goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

update local and mpv plugins, add file cmdlet, update docs

Browse Source
This commit is contained in:
Nose
2026-05-14 17:15:13 -07:00
parent 9f0eb29289
commit 036977832b
10 changed files with 653 additions and 297 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmdlet/file/add.py
+187 -268
View File
@@ -2,7 +2,6 @@ from __future__ import annotations
from typing import Any, Dict, Optional, Sequence, Tuple, List
from pathlib import Path
from copy import deepcopy
import sys
import shutil
import tempfile
@@ -11,7 +10,7 @@ from urllib.parse import urlparse
from SYS import models
from SYS import pipeline as ctx
from SYS.logger import log, debug, debug_panel, is_debug_enabled
from SYS.logger import log, debug, debug_panel
from SYS.payload_builders import build_table_result_payload
from SYS.pipeline_progress import PipelineProgress
from SYS.result_publication import overlay_existing_result_table, publish_result_table
@@ -92,41 +91,11 @@ class _CommandDependencies:
self._plugins[cache_key] = plugin
return plugin
DEBUG_PIPE_NOTE_PREVIEW_LENGTH = 256
# Protocol schemes that identify a remote resource / not a local file path.
# Used by multiple methods in this file to guard against URL strings being
# treated as local file paths.
_REMOTE_URL_PREFIXES: tuple[str, ...] = (
"http://", "https://", "ftp://", "ftps://", "magnet:", "torrent:", "tidal:", "hydrus:",
)
def _truncate_debug_note_text(value: Any) -> str:
raw = str(value or "")
if len(raw) <= DEBUG_PIPE_NOTE_PREVIEW_LENGTH:
return raw
return raw[:DEBUG_PIPE_NOTE_PREVIEW_LENGTH].rstrip() + "..."
def _sanitize_pipe_object_for_debug(pipe_obj: models.PipeObject) -> models.PipeObject:
safe_po = deepcopy(pipe_obj)
try:
extra = safe_po.extra
if isinstance(extra, dict):
sanitized = dict(extra)
notes = sanitized.get("notes")
if isinstance(notes, dict):
truncated_notes: Dict[str, str] = {}
for note_name, note_value in notes.items():
truncated_notes[str(note_name)] = _truncate_debug_note_text(note_value)
sanitized["notes"] = truncated_notes
safe_po.extra = sanitized
except Exception:
pass
return safe_po
def _maybe_apply_florencevision_tags(
media_path: Path,
tags: List[str],
@@ -224,9 +193,9 @@ class Add_File(Cmdlet):
super().__init__(
name="add-file",
summary=
"Ingest a local media file to a configured instance, upload plugin, or local directory.",
"Ingest a local media file to a configured store or plugin destination.",
usage=
"add-file (-path <filepath> | <piped>) (-instance <name|path> | -plugin <upload-plugin>) [-delete]",
"add-file (-path <filepath> | <piped>) (-instance <store-name> | -plugin <plugin> [-instance <name|path>]) [-delete]",
arg=[
SharedArgs.PATH,
SharedArgs.INSTANCE,
@@ -242,11 +211,10 @@ class Add_File(Cmdlet):
],
detail=[
"Note: add-file ingests local files. To fetch remote sources, use download-file and pipe into add-file.",
"- Instance/location options (use -instance):",
"- Store options (use -instance without -plugin):",
" hydrus: Upload to Hydrus database with metadata tagging",
" local: Copy file to local directory",
" <path>: Copy file to specified directory",
"- Upload plugin options (use -plugin):",
"- Plugin options (use -plugin):",
" local: Copy file to a configured local destination or direct path via -instance",
" 0x0: Upload to 0x0.st for temporary hosting",
" file.io: Upload to file.io for temporary hosting",
" internetarchive: Upload to archive.org (optional tag: ia:<identifier> to upload into an existing item)",
@@ -254,6 +222,7 @@ class Add_File(Cmdlet):
],
examples=[
'download-file "https://themathesontrust.org/papers/christianity/alcock-alphabet1.pdf" | add-file -instance tutorial',
'@1 | add-file -plugin local -instance C:\\Users\\Me\\Downloads',
'add-file -plugin ftp -instance archive -path C:\\Media\\report.pdf',
],
exec=self.run,
@@ -275,25 +244,19 @@ class Add_File(Cmdlet):
source_url_arg = parsed.get("url")
plugin_name = parsed.get("plugin")
delete_after = parsed.get("delete", False)
local_export_destination: Optional[str] = None
if plugin_name and not plugin_instance and location:
plugin_instance = location
# Convenience: when piping a file into add-file, allow `-path <existing dir>`
# to act as the destination export directory.
# Example: screen-shot "https://..." | add-file -path "C:\Users\Admin\Desktop"
# Backward-compatible shorthand: when piping a file into add-file, allow
# `-path <existing dir>` to normalize into the local export plugin path.
if path_arg and not location and not plugin_name:
try:
candidate_dir = Path(str(path_arg))
if candidate_dir.exists() and candidate_dir.is_dir():
debug_panel(
"add-file destination",
[
("mode", "local export"),
("path", candidate_dir),
],
border_style="cyan",
)
location = str(candidate_dir)
plugin_name = "local"
plugin_instance = str(candidate_dir)
local_export_destination = str(candidate_dir)
path_arg = None
except Exception:
pass
@@ -397,13 +360,44 @@ class Add_File(Cmdlet):
is_storage_backend_location = False
if location and not plugin_name and not is_storage_backend_location:
if not Add_File._looks_like_local_export_target(str(location)):
resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target(
location,
config,
deps=deps,
require_explicit=True,
)
if resolved_local_path:
plugin_name = "local"
plugin_instance = resolved_local_instance or str(location)
location = None
local_export_destination = resolved_local_path
else:
log(
f"Storage backend '{location}' not found. Use -path for local export or configure that store backend.",
f"Storage backend '{location}' not found. Use -plugin local -instance <name|path> for local export or configure that store backend.",
file=sys.stderr,
)
return 1
normalized_plugin_name = Add_File._normalize_provider_key(plugin_name)
if normalized_plugin_name == "local":
resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target(
plugin_instance or location,
config,
deps=deps,
require_explicit=bool(plugin_instance or location),
)
if not resolved_local_path:
requested_local = str(plugin_instance or location or "").strip() or "<default>"
log(
f"Local destination '{requested_local}' is not configured. Use -plugin local -instance <name|path>.",
file=sys.stderr,
)
return 1
plugin_name = "local"
plugin_instance = resolved_local_instance or str(plugin_instance or location or "").strip() or None
location = None
local_export_destination = resolved_local_path
plugin_storage_backend = None
if plugin_name:
plugin_storage_backend = Add_File._resolve_plugin_storage_backend(
@@ -469,46 +463,8 @@ class Add_File(Cmdlet):
except Exception:
use_steps = False
try:
debug_panel(
"add-file",
[
("result_type", type(result).__name__),
("items", total_items),
("location", location),
("plugin", plugin_name),
("instance", plugin_instance),
("delete", delete_after),
],
border_style="cyan",
)
except Exception:
pass
# add-file is ingestion-only: it does not download URLs here.
# Show a concise PipeObject preview when debug logging is enabled to aid pipeline troubleshooting.
if is_debug_enabled():
preview_items = (
items_to_process if isinstance(items_to_process, list)
else [items_to_process]
)
max_preview = 5
for idx, item in enumerate(preview_items[:max_preview]):
po = item if isinstance(item, models.PipeObject) else None
if po is None:
try:
po = coerce_to_pipe_object(item, path_arg)
except Exception:
po = None
if po is None:
continue
try:
safe_po = _sanitize_pipe_object_for_debug(po)
safe_po.debug_table()
except Exception:
pass
should_present_directory_selector = bool(dir_scan_mode and not has_downstream_stage)
if dir_scan_mode and has_downstream_stage:
debug(
@@ -666,12 +622,19 @@ class Add_File(Cmdlet):
if use_steps and steps_started:
progress.step("resolving source")
export_destination = (
Path(local_export_destination)
if local_export_destination
else Path(location)
if location and not is_storage_backend_location
else None
)
media_path, file_hash, temp_dir_to_cleanup = self._resolve_source(
item,
path_arg,
pipe_obj,
config,
export_destination=(Path(location) if location and not is_storage_backend_location else None),
export_destination=export_destination,
store_instance=storage_registry,
deps=deps,
)
@@ -679,19 +642,6 @@ class Add_File(Cmdlet):
media_path, file_hash, temp_dir_to_cleanup = Add_File._download_piped_source(
pipe_obj, config, storage_registry, deps=deps
)
if media_path:
try:
debug_panel(
f"add-file source {idx}/{max(1, total_items)}",
[
("path", media_path),
("hash", file_hash or "N/A"),
("plugin", plugin_name or "local"),
],
border_style="green",
)
except Exception:
pass
if not media_path:
failures += 1
continue
@@ -768,13 +718,8 @@ class Add_File(Cmdlet):
store_instance=storage_registry,
)
else:
code = self._handle_local_export(
media_path,
location,
pipe_obj,
config,
delete_after_item
)
log(f"Invalid storage backend: {location}", file=sys.stderr)
code = 1
except Exception as exc:
debug(f"[add-file] ERROR: Failed to resolve location: {exc}")
log(f"Invalid location: {location}", file=sys.stderr)
@@ -1371,27 +1316,6 @@ class Add_File(Cmdlet):
pass
return None
@staticmethod
def _looks_like_local_export_target(location: str) -> bool:
target = str(location or "").strip()
if not target:
return False
target_path = Path(target).expanduser()
try:
if target_path.exists():
return True
except Exception:
pass
if target.startswith((".", "~")):
return True
if "\\" in target or "/" in target:
return True
if len(target) >= 2 and target[1] == ":":
return True
return False
@staticmethod
def _resolve_source(
result: Any,
@@ -1608,6 +1532,45 @@ class Add_File(Cmdlet):
return resolved_text
@staticmethod
def _resolve_local_export_plugin_target(
requested: Optional[Any],
config: Dict[str, Any],
*,
deps: Optional[_CommandDependencies] = None,
require_explicit: bool = False,
) -> tuple[Optional[str], Optional[str]]:
if deps is None:
deps = _CommandDependencies(config)
file_provider = deps.get_plugin_with_capability("local", "upload")
if file_provider is None:
return None, None
resolver = getattr(file_provider, "resolve_destination", None)
if not callable(resolver):
return None, None
requested_text = str(requested or "").strip() or None
try:
resolved_name, settings = resolver(
requested_text,
require_explicit=require_explicit,
)
except TypeError:
try:
resolved_name, settings = resolver(requested_text)
except Exception:
return None, None
except Exception:
return None, None
path_value = str((settings or {}).get("path") or "").strip()
if not path_value:
return None, None
resolved_text = str(resolved_name or requested_text or "").strip() or None
return resolved_text, path_value
@staticmethod
def _maybe_download_plugin_result(
result: Any,
@@ -2294,136 +2257,72 @@ class Add_File(Cmdlet):
return None
@staticmethod
def _handle_local_export(
media_path: Path,
location: str,
def _emit_plugin_upload_payload(
upload_payload: Dict[str, Any],
plugin_name: str,
instance_name: Optional[str],
pipe_obj: models.PipeObject,
config: Dict[str,
Any],
media_path: Path,
delete_after: bool,
) -> int:
"""Handle exporting to a specific local path (Copy)."""
try:
destination_root = Path(location)
except Exception as exc:
log(f"❌ Invalid destination path '{location}': {exc}", file=sys.stderr)
return 1
payload = dict(upload_payload or {})
extra_updates: Dict[str, Any] = {}
raw_extra = payload.get("extra")
if isinstance(raw_extra, dict):
extra_updates.update(raw_extra)
direct_export_download = False
try:
if isinstance(pipe_obj.extra, dict):
direct_export_download = bool(pipe_obj.extra.pop("_direct_export_download", False))
except Exception:
direct_export_download = False
if plugin_name:
extra_updates.setdefault("plugin", plugin_name)
if instance_name:
extra_updates.setdefault("instance", instance_name)
try:
debug_panel(
"add-file export",
[
("destination", destination_root),
("source", media_path),
],
border_style="green",
)
except Exception:
pass
raw_urls = payload.get("url")
if isinstance(raw_urls, str):
url_values = [raw_urls.strip()] if raw_urls.strip() else []
extra_updates["url"] = url_values
elif isinstance(raw_urls, (list, tuple, set)):
url_values = [str(item).strip() for item in raw_urls if str(item).strip()]
extra_updates["url"] = url_values
result = None
tags, url, title, f_hash = Add_File._prepare_metadata(result, media_path, pipe_obj, config)
# Determine Filename (Title-based)
title_value = title
if not title_value:
# Try to find title in tags
title_tag = next(
(t for t in tags if str(t).strip().lower().startswith("title:")),
None
)
if title_tag:
title_value = title_tag.split(":", 1)[1].strip()
if not title_value:
title_value = media_path.stem.replace("_", " ").strip()
safe_title = "".join(
c for c in title_value if c.isalnum() or c in " ._-()[]{}'`"
).strip()
base_name = safe_title or media_path.stem
# Fix to prevent double extensions (e.g., file.exe.exe)
# If the base name already ends with the extension of the media file,
# don't append it again.
file_ext = media_path.suffix
if file_ext and base_name.lower().endswith(file_ext.lower()):
new_name = base_name
else:
new_name = base_name + file_ext
destination_root.mkdir(parents=True, exist_ok=True)
target_path = destination_root / new_name
if direct_export_download:
target_path = media_path
else:
if target_path.exists():
target_path = unique_path(target_path)
# COPY Operation (Safe Export)
try:
shutil.copy2(str(media_path), target_path)
except Exception as exc:
log(f"❌ Failed to export file: {exc}", file=sys.stderr)
return 1
# Copy Sidecars
Add_File._copy_sidecars(media_path, target_path)
# Ensure hash for exported copy
if not f_hash:
try:
f_hash = sha256_file(target_path)
except Exception:
f_hash = None
# Write Metadata Sidecars (since it's an export)
relationships = Add_File._get_relationships(result, pipe_obj)
try:
write_sidecar(target_path, tags, url, f_hash)
from SYS.metadata import write_metadata # lazy: avoids 1000+ module chain at startup
write_metadata(
target_path,
hash_value=f_hash,
url=url,
relationships=relationships or []
)
except Exception:
pass
# Update PipeObject and emit
extra_updates = {
"url": url,
"export_path": str(destination_root),
}
relationships = payload.get("relationships")
if relationships:
extra_updates["relationships"] = relationships
try:
pipe_obj.relationships = relationships
except Exception:
pass
chosen_title = title or title_value or pipe_obj.title or target_path.name
tags = payload.get("tag")
if isinstance(tags, list):
tag_values = [str(tag) for tag in tags]
else:
tag_values = list(pipe_obj.tag or [])
title_value = str(payload.get("title") or pipe_obj.title or media_path.name).strip() or media_path.name
path_value = str(payload.get("path") or pipe_obj.path or media_path).strip()
hash_value = str(
payload.get("hash")
or payload.get("file_hash")
or getattr(pipe_obj, "hash", None)
or "unknown"
).strip() or "unknown"
store_value = str(payload.get("store") or "").strip()
provider_value = payload.get("provider")
if provider_value is None and plugin_name:
provider_value = plugin_name
Add_File._update_pipe_object_destination(
pipe_obj,
hash_value=f_hash or "unknown",
store="local",
path=str(target_path),
tag=tags,
title=chosen_title,
hash_value=hash_value,
store=store_value,
provider=str(provider_value) if provider_value else None,
path=path_value,
tag=tag_values,
title=title_value,
extra_updates=extra_updates,
)
Add_File._emit_pipe_object(pipe_obj)
# Cleanup
# Only delete if explicitly requested!
Add_File._cleanup_after_success(media_path, delete_source=delete_after)
return 0
@staticmethod
@@ -2459,10 +2358,37 @@ class Add_File(Cmdlet):
show_available_plugins_panel(sorted(available_uploads))
return 1
hoster_url = file_provider.upload(
upload_kwargs: Dict[str, Any] = {
"pipe_obj": pipe_obj,
"instance": instance_name,
}
normalized_plugin_name = Add_File._normalize_provider_key(plugin_name)
f_hash = Add_File._resolve_file_hash(None, media_path, pipe_obj, None)
if normalized_plugin_name == "local":
result = None
tags, urls, title, f_hash = Add_File._prepare_metadata(result, media_path, pipe_obj, config)
relationships = Add_File._get_relationships(result, pipe_obj)
direct_export_download = False
try:
if isinstance(pipe_obj.extra, dict):
direct_export_download = bool(pipe_obj.extra.pop("_direct_export_download", False))
except Exception:
direct_export_download = False
upload_kwargs.update(
{
"title": title,
"tags": tags,
"urls": urls,
"hash_value": f_hash,
"relationships": relationships,
"direct_export_download": direct_export_download,
}
)
upload_result = file_provider.upload(
str(media_path),
pipe_obj=pipe_obj,
instance=instance_name,
**upload_kwargs,
)
duplicate_upload = False
@@ -2478,29 +2404,22 @@ class Add_File(Cmdlet):
duplicate_rule = ""
duplicate_target = ""
try:
debug_panel(
"add-file plugin upload",
[
("plugin", plugin_name),
("instance", instance_name or "<default>"),
("source", media_path),
("duplicate", duplicate_upload),
("rule", duplicate_rule or "none"),
("target", duplicate_target or ""),
("url", hoster_url),
],
border_style="yellow" if duplicate_upload else "green",
)
except Exception:
pass
f_hash = Add_File._resolve_file_hash(None, media_path, pipe_obj, None)
except Exception as exc:
log(f"Upload failed: {exc}", file=sys.stderr)
return 1
if isinstance(upload_result, dict):
return Add_File._emit_plugin_upload_payload(
upload_result,
plugin_name,
instance_name,
pipe_obj,
media_path,
delete_after,
)
hoster_url = str(upload_result or "").strip()
# Update PipeObject and emit
extra_updates: Dict[str,
Any] = {
cmdlet/file_cmdlet.py
+20 -6
View File
@@ -21,7 +21,6 @@ class File(Cmdlet):
"get": {"-get", "--get"},
"merge": {"-merge", "--merge"},
"download": {"-download", "--download", "-dl", "--dl"},
"search": {"-search", "--search"},
"convert": {"-convert", "--convert"},
"trim": {"-trim", "--trim"},
"archive": {"-archive", "--archive"},
@@ -45,9 +44,10 @@ class File(Cmdlet):
super().__init__(
name="file",
summary="Manage file operations with one command",
usage='file (-add|-delete|-get|-merge|-download|-search|-convert|-trim|-archive|-screenshot) [args]',
usage='file -query <query> [args] | file (-add|-delete|-get|-merge|-download|-convert|-trim|-archive|-screenshot) [args]',
arg=[
SharedArgs.QUERY,
SharedArgs.PLUGIN,
SharedArgs.INSTANCE,
SharedArgs.PATH,
CmdletArg("-add", type="flag", required=False, description="Run add-file"),
@@ -55,21 +55,32 @@ class File(Cmdlet):
CmdletArg("-get", type="flag", required=False, description="Run get-file"),
CmdletArg("-merge", type="flag", required=False, description="Run merge-file"),
CmdletArg("-download", type="flag", required=False, description="Run download-file", alias="dl"),
CmdletArg("-search", type="flag", required=False, description="Run search-file"),
CmdletArg("-convert", type="flag", required=False, description="Run convert-file"),
CmdletArg("-trim", type="flag", required=False, description="Run trim-file"),
CmdletArg("-archive", type="flag", required=False, description="Run archive-file"),
CmdletArg("-screenshot", type="flag", required=False, description="Run screen-shot", alias="shot"),
],
detail=[
"- Exactly one action flag is required.",
"- Use -query to run search-file through the unified file command.",
"- Otherwise, exactly one non-search action flag is required.",
"- Remaining args are passed through to the selected file cmdlet.",
"- Examples: file -add ..., file -delete ..., file -merge ...",
"- Examples: file -query ..., file -add ..., file -delete ...",
],
exec=self.run,
)
self.register()
@staticmethod
def _has_query_arg(args: Sequence[str]) -> bool:
query_flags = {"-query", "--query"}
for token in args or []:
text = str(token or "").strip().lower()
if text in query_flags:
return True
if any(text.startswith(f"{flag}=") for flag in query_flags):
return True
return False
@classmethod
def _extract_action(cls, args: Sequence[str]) -> tuple[str | None, List[str], List[str]]:
matched_actions: List[str] = []
@@ -93,6 +104,9 @@ class File(Cmdlet):
if action not in unique_actions:
unique_actions.append(action)
if not unique_actions and cls._has_query_arg(passthrough):
return "search", passthrough, unique_actions
if len(unique_actions) != 1:
return None, passthrough, unique_actions
return unique_actions[0], passthrough, unique_actions
@@ -125,7 +139,7 @@ class File(Cmdlet):
if action is None:
if not seen:
log(
"file: missing action flag; choose exactly one of -add, -delete, -get, -merge, -download, -search, -convert, -trim, -archive, -screenshot",
"file: missing action; use -query for search or choose exactly one of -add, -delete, -get, -merge, -download, -convert, -trim, -archive, -screenshot",
file=sys.stderr,
)
else: