goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

syntax revamp

Browse Source
This commit is contained in:
Nose
2026-05-24 12:32:57 -07:00
parent 6c0a1b4415
commit 5041d9fbb9
20 changed files with 1512 additions and 1060 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmdlet/file/add.py
+245 -100
View File
@@ -195,9 +195,14 @@ class Add_File(Cmdlet):
summary=
"Ingest a local media file to a configured store or plugin destination.",
usage=
"add-file (-path <filepath> | <piped>) (-instance <store-name> | -plugin <plugin> [-instance <name|path>]) [-delete]",
"add-file (<source> | <piped>) (-instance <store-name> | -plugin <plugin> [-instance <name|path>]) [-delete]",
arg=[
SharedArgs.PATH,
CmdletArg(
name="source",
type="string",
required=False,
description="Local file or directory path to ingest or scan.",
),
SharedArgs.INSTANCE,
SharedArgs.URL,
SharedArgs.PLUGIN,
@@ -218,19 +223,38 @@ class Add_File(Cmdlet):
" 0x0: Upload to 0x0.st for temporary hosting",
" file.io: Upload to file.io for temporary hosting",
" internetarchive: Upload to archive.org (optional tag: ia:<identifier> to upload into an existing item)",
"- Use -instance with -plugin to target a named provider config: add-file -plugin ftp -instance archive -path C:\\Media\\file.pdf",
"- Use a positional source path with -instance and -plugin to target a named provider config: add-file C:\\Media\\file.pdf -plugin ftp -instance archive",
],
examples=[
'download-file "https://themathesontrust.org/papers/christianity/alcock-alphabet1.pdf" | add-file -instance tutorial',
'@1 | add-file -plugin local -instance C:\\Users\\Me\\Downloads',
'add-file -plugin ftp -instance archive -path C:\\Media\\report.pdf',
'add-file C:\\Media\\report.pdf -plugin ftp -instance archive',
],
exec=self.run,
)
self.register()
@staticmethod
def _uses_legacy_path_flag(args: Sequence[str]) -> bool:
for token in args or []:
lowered = str(token or "").strip().lower()
if lowered in {"-path", "--path", "-p"}:
return True
return False
@staticmethod
def _legacy_path_flag_message() -> str:
return (
"add-file no longer supports -path. Pass the source file or directory as a positional argument, "
"and use -plugin local -instance <name|path> for local export."
)
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
"""Main execution entry point."""
if Add_File._uses_legacy_path_flag(args):
log(Add_File._legacy_path_flag_message(), file=sys.stderr)
return 1
parsed = parse_cmdlet_args(args, self)
progress = PipelineProgress(ctx)
@@ -238,7 +262,7 @@ class Add_File(Cmdlet):
deps = _CommandDependencies(config)
storage_registry = deps.get_backend_registry()
path_arg = parsed.get("path")
source_arg = parsed.get("source")
location = parsed.get("instance")
plugin_instance = parsed.get("instance")
source_url_arg = parsed.get("url")
@@ -248,19 +272,6 @@ class Add_File(Cmdlet):
if plugin_name and not plugin_instance and location:
plugin_instance = location
# Backward-compatible shorthand: when piping a file into add-file, allow
# `-path <existing dir>` to normalize into the local export plugin path.
if path_arg and not location and not plugin_name:
try:
candidate_dir = Path(str(path_arg))
if candidate_dir.exists() and candidate_dir.is_dir():
plugin_name = "local"
plugin_instance = str(candidate_dir)
local_export_destination = str(candidate_dir)
path_arg = None
except Exception:
pass
stage_ctx = ctx.get_stage_context()
is_last_stage = (stage_ctx
is None) or bool(getattr(stage_ctx,
@@ -269,24 +280,24 @@ class Add_File(Cmdlet):
has_downstream_stage = bool(stage_ctx is not None and not is_last_stage)
# Directory-mode selector:
# - Terminal use: `add-file -instance X -path <DIR>` shows a selectable table.
# - Pipelined use: `add-file -instance X -path <DIR> | ...` processes the full batch
# - Terminal use: `add-file <DIR> -instance X` shows a selectable table.
# - Pipelined use: `add-file <DIR> -instance X | ...` processes the full batch
# immediately so downstream stages receive the uploaded items.
# - Selection replay: `@N` re-runs add-file with `-path file1,file2,...`.
# - Selection replay: `@N` re-runs add-file with `file1,file2,...` as the source token.
dir_scan_mode = False
dir_scan_results: Optional[List[Dict[str, Any]]] = None
explicit_path_list_results: Optional[List[Dict[str, Any]]] = None
explicit_source_list_results: Optional[List[Dict[str, Any]]] = None
if path_arg and location and not plugin_name:
# Support comma-separated path lists: -path "file1,file2,file3"
if source_arg and location and not plugin_name:
# Support comma-separated source lists: "file1,file2,file3"
# This is the mechanism used by @N expansion for directory tables.
try:
path_text = str(path_arg)
source_text = str(source_arg)
except Exception:
path_text = ""
source_text = ""
if "," in path_text:
parts = [p.strip().strip('"') for p in path_text.split(",")]
if "," in source_text:
parts = [p.strip().strip('"') for p in source_text.split(",")]
parts = [p for p in parts if p]
batch: List[Dict[str, Any]] = []
@@ -319,13 +330,13 @@ class Add_File(Cmdlet):
)
if batch:
explicit_path_list_results = batch
# Clear path_arg so add-file doesn't treat it as a single path.
path_arg = None
explicit_source_list_results = batch
# Clear source_arg so add-file doesn't treat it as a single path.
source_arg = None
else:
# Directory scan (selector table, no ingest yet)
try:
candidate_dir = Path(str(path_arg))
candidate_dir = Path(str(source_arg))
if candidate_dir.exists() and candidate_dir.is_dir():
dir_scan_mode = True
debug(
@@ -338,12 +349,12 @@ class Add_File(Cmdlet):
debug(
f"[add-file] Found {len(dir_scan_results)} supported files in directory"
)
# Clear path_arg so it doesn't trigger single-item mode.
path_arg = None
# Clear source_arg so it doesn't trigger single-item mode.
source_arg = None
except Exception as exc:
debug(f"[add-file] Directory scan failed: {exc}")
if result is None and not path_arg and not explicit_path_list_results and not dir_scan_results:
if result is None and not source_arg and not explicit_source_list_results and not dir_scan_results:
try:
if ctx.get_stage_context() is not None:
return 0
@@ -414,15 +425,15 @@ class Add_File(Cmdlet):
# Decide which items to process.
# - If directory scan was performed, use those results
# - If user provided -path (and it was not reinterpreted as destination), treat this invocation as single-item.
# - If user provided a positional source path, treat this invocation as single-item.
# - Otherwise, if piped input is a list, ingest each item.
if explicit_path_list_results:
items_to_process = explicit_path_list_results
debug(f"[add-file] Using {len(items_to_process)} files from -path list")
if explicit_source_list_results:
items_to_process = explicit_source_list_results
debug(f"[add-file] Using {len(items_to_process)} files from source list")
elif dir_scan_results:
items_to_process = dir_scan_results
debug(f"[add-file] Using {len(items_to_process)} files from directory scan")
elif path_arg:
elif source_arg:
items_to_process: List[Any] = [result]
elif isinstance(result, list) and result:
items_to_process = list(result)
@@ -472,26 +483,21 @@ class Add_File(Cmdlet):
)
# If this invocation was terminal directory selector mode, show a selectable table and stop.
# The user then runs @N (optionally piped), which replays add-file with selected paths.
# The user then runs @N (optionally piped), which replays add-file with selected source paths.
if should_present_directory_selector:
try:
from SYS.result_table import Table
from pathlib import Path as _Path
# Build base args to replay: keep everything except the directory -path.
base_args: List[str] = []
skip_next = False
for tok in list(args or []):
if skip_next:
skip_next = False
continue
t = str(tok)
if t in {"-path",
"--path",
"-p"}:
skip_next = True
continue
base_args.append(t)
if plugin_name:
base_args.extend(["-plugin", str(plugin_name)])
if location:
base_args.extend(["-instance", str(location)])
if source_url_arg:
base_args.extend(["-url", str(source_url_arg)])
if bool(delete_after):
base_args.append("-delete")
table = Table(title="Files in Directory", preserve_order=True)
table.set_table("add-file.directory")
@@ -517,7 +523,7 @@ class Add_File(Cmdlet):
("Size", size),
("Ext", ext),
],
selection_args=["-path", str(p) if p is not None else ""],
selection_args=[str(p) if p is not None else ""],
path=str(p) if p is not None else "",
hash=hp,
)
@@ -631,7 +637,7 @@ class Add_File(Cmdlet):
)
media_path, file_hash, temp_dir_to_cleanup = self._resolve_source(
item,
path_arg,
source_arg,
pipe_obj,
config,
export_destination=export_destination,
@@ -649,8 +655,8 @@ class Add_File(Cmdlet):
# Update pipe_obj with resolved path
pipe_obj.path = str(media_path)
# When using -path (filesystem export), allow all file types.
# When using -instance (backend), restrict to SUPPORTED_MEDIA_EXTENSIONS.
# Local/plugin exports can accept any file type.
# Storage backends stay restricted to SUPPORTED_MEDIA_EXTENSIONS.
allow_all_files = not bool(effective_storage_backend_name)
if not self._validate_source(media_path, allow_all_extensions=allow_all_files):
failures += 1
@@ -780,30 +786,7 @@ class Add_File(Cmdlet):
# Stop the live pipeline progress UI before rendering the details panels.
# This prevents the progress display from lingering on screen.
try:
live_progress = ctx.get_live_progress()
except Exception:
live_progress = None
if live_progress is not None:
try:
stage_ctx = ctx.get_stage_context()
pipe_idx = getattr(stage_ctx, "pipe_index", None)
if isinstance(pipe_idx, int):
live_progress.finish_pipe(
int(pipe_idx),
force_complete=True
)
except Exception:
pass
try:
live_progress.stop()
except Exception:
pass
try:
if hasattr(ctx, "set_live_progress"):
ctx.set_live_progress(None)
except Exception:
pass
Add_File._stop_live_progress_for_terminal_render()
subject = collected_payloads[0] if len(collected_payloads) == 1 else collected_payloads
# Use helper to display items and make them @-selectable
@@ -1108,6 +1091,8 @@ class Add_File(Cmdlet):
backend: Any,
file_hash: str,
pipe_obj: models.PipeObject,
*,
output_dir: Optional[Path] = None,
) -> Tuple[Optional[Path], Optional[Path]]:
"""Best-effort fetch of a backend file when get_file returns a URL.
@@ -1133,30 +1118,68 @@ class Add_File(Cmdlet):
metadata = getattr(pipe_obj, "metadata", {})
if isinstance(metadata, dict):
suffix = metadata.get("ext")
tmp_dir = Path(tempfile.mkdtemp(prefix="add-file-src-"))
# Introspect downloader to pass supported args (suffix, progress_callback)
download_root = output_dir
if download_root is None:
tmp_dir = Path(tempfile.mkdtemp(prefix="add-file-src-"))
download_root = tmp_dir
if download_root is None:
return None, None
# Introspect downloader to pass supported args.
import inspect
sig = inspect.signature(downloader)
kwargs = {"temp_root": tmp_dir}
kwargs = {"temp_root": download_root}
if "suffix" in sig.parameters:
kwargs["suffix"] = suffix
# Hook into global PipelineProgress if available
pp = PipelineProgress.get()
if pp and "progress_callback" in sig.parameters:
pipeline_progress = PipelineProgress(ctx)
transfer_label = "peer transfer"
try:
transfer_label = str(getattr(pipe_obj, "title", "") or "").strip() or transfer_label
except Exception:
transfer_label = "peer transfer"
if "pipeline_progress" in sig.parameters:
kwargs["pipeline_progress"] = pipeline_progress
if "transfer_label" in sig.parameters:
kwargs["transfer_label"] = transfer_label
if "progress_callback" in sig.parameters:
def _cb(done, total):
# Show fetch progress instead of just 'resolving'
pp.update(downloaded=done, total=total, label="peer transfer")
try:
total_val = int(total) if total is not None else None
except Exception:
total_val = None
try:
if int(done or 0) <= 0:
pipeline_progress.begin_transfer(
label=transfer_label,
total=total_val,
)
except Exception:
pass
try:
pipeline_progress.update_transfer(
label=transfer_label,
completed=int(done or 0),
total=total_val,
)
except Exception:
pass
kwargs["progress_callback"] = _cb
downloaded = downloader(str(file_hash), **kwargs)
if isinstance(downloaded, Path) and downloaded.exists():
if output_dir is not None:
pipe_obj.is_temp = False
if isinstance(pipe_obj.extra, dict):
pipe_obj.extra["_direct_export_download"] = True
else:
pipe_obj.extra = {"_direct_export_download": True}
return downloaded, None
pipe_obj.is_temp = True
return downloaded, tmp_dir
except Exception:
@@ -1208,6 +1231,11 @@ class Add_File(Cmdlet):
source_url=url_text,
)
pipeline_progress = PipelineProgress(ctx)
try:
destination_label = str(download_root) if download_root is not None else "temporary workspace"
pipeline_progress.set_status(f"downloading {suggested_name} to {destination_label}")
except Exception:
pass
downloaded = _download_direct_file(
url_text,
@@ -1230,6 +1258,11 @@ class Add_File(Cmdlet):
return downloaded_path, tmp_dir
except Exception:
pass
finally:
try:
PipelineProgress(ctx).clear_status()
except Exception:
pass
if tmp_dir is not None:
try:
@@ -1319,7 +1352,7 @@ class Add_File(Cmdlet):
@staticmethod
def _resolve_source(
result: Any,
path_arg: Optional[str],
source_arg: Optional[str],
pipe_obj: models.PipeObject,
config: Dict[str,
Any],
@@ -1329,7 +1362,7 @@ class Add_File(Cmdlet):
) -> Tuple[Optional[Path],
Optional[str],
Optional[Path]]:
"""Resolve the source file path from args or pipeline result.
"""Resolve the source file path from the positional source arg or pipeline result.
Returns (media_path, file_hash, temp_dir_to_cleanup).
"""
@@ -1374,7 +1407,10 @@ class Add_File(Cmdlet):
return mp_path, str(r_hash), None
dl_path, tmp_dir = Add_File._maybe_download_backend_file(
backend, str(r_hash), pipe_obj
backend,
str(r_hash),
pipe_obj,
output_dir=export_destination,
)
if dl_path and dl_path.exists():
pipe_obj.path = str(dl_path)
@@ -1395,8 +1431,8 @@ class Add_File(Cmdlet):
# PRIORITY 2: Generic Coercion (Path arg > PipeObject > Result)
candidate: Optional[Path] = None
if path_arg:
candidate = Path(path_arg)
if source_arg:
candidate = Path(source_arg)
elif pipe_obj.path:
candidate = Path(pipe_obj.path)
@@ -1471,6 +1507,83 @@ class Add_File(Cmdlet):
normalized = normalized.split(".", 1)[0]
return normalized
@staticmethod
def validate_preflight_args(
args: Sequence[str],
config: Optional[Dict[str, Any]] = None,
) -> Optional[str]:
cfg = config if isinstance(config, dict) else {}
if Add_File._uses_legacy_path_flag(args):
return f"Pipeline error: {Add_File._legacy_path_flag_message()}"
try:
parsed = parse_cmdlet_args(args, CMDLET)
except Exception as exc:
return f"Pipeline error: invalid add-file arguments: {exc}"
deps = _CommandDependencies(cfg)
storage_registry = deps.get_backend_registry()
location = parsed.get("instance")
plugin_instance = parsed.get("instance")
plugin_name = parsed.get("plugin")
is_storage_backend_location = False
if location:
try:
backend_registry_for_lookup = storage_registry or deps.get_backend_registry()
is_storage_backend_location = Add_File._resolve_backend_by_name(
backend_registry_for_lookup,
str(location),
) is not None
except Exception:
is_storage_backend_location = False
if location and not plugin_name and not is_storage_backend_location:
resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target(
location,
cfg,
deps=deps,
require_explicit=True,
)
if resolved_local_path:
return None
return (
f"Pipeline error: storage backend '{location}' not found. "
"Use -plugin local -instance <name|path> for local export or configure that store backend."
)
normalized_plugin_name = Add_File._normalize_provider_key(plugin_name)
if normalized_plugin_name:
upload_plugin = deps.get_plugin_with_capability(normalized_plugin_name, "upload")
if upload_plugin is None:
plugin_exists = deps.get_plugin(normalized_plugin_name) is not None
if plugin_exists:
if normalized_plugin_name == "loc":
return (
"Pipeline error: plugin 'loc' does not support add-file/upload. "
"Use -plugin local -instance <name|path> for local export."
)
return f"Pipeline error: plugin '{normalized_plugin_name}' does not support add-file/upload."
return f"Pipeline error: unknown upload plugin '{plugin_name}'."
if normalized_plugin_name == "local":
requested_local = str(plugin_instance or location or "").strip() or "<default>"
resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target(
plugin_instance or location,
cfg,
deps=deps,
require_explicit=bool(plugin_instance or location),
)
if not resolved_local_path:
return (
f"Pipeline error: local destination '{requested_local}' is not configured. "
"Use -plugin local -instance <name|path>."
)
return None
@staticmethod
def _resolve_plugin_storage_backend(
plugin_name: Optional[Any],
@@ -1730,8 +1843,8 @@ class Add_File(Cmdlet):
Args:
media_path: Path to the file to validate
allow_all_extensions: If True, skip file type filtering (used for -path exports).
If False, only allow SUPPORTED_MEDIA_EXTENSIONS (used for -instance).
allow_all_extensions: If True, skip file type filtering for non-backend exports.
If False, only allow SUPPORTED_MEDIA_EXTENSIONS for backend ingest.
"""
if media_path is None:
return False
@@ -1740,7 +1853,7 @@ class Add_File(Cmdlet):
log(f"File not found: {media_path}")
return False
# Validate file type: only when adding to -instance backend, not for -path exports
# Validate file type only when ingesting into a storage backend.
if not allow_all_extensions:
file_extension = media_path.suffix.lower()
if file_extension not in SUPPORTED_MEDIA_EXTENSIONS:
@@ -1947,12 +2060,42 @@ class Add_File(Cmdlet):
return
try:
Add_File._stop_live_progress_for_terminal_render()
from .._shared import display_and_persist_items
display_and_persist_items([payload], title="Result", subject=payload)
except Exception:
pass
@staticmethod
def _stop_live_progress_for_terminal_render() -> None:
try:
live_progress = ctx.get_live_progress()
except Exception:
live_progress = None
if live_progress is None:
return
try:
stage_ctx = ctx.get_stage_context()
pipe_idx = getattr(stage_ctx, "pipe_index", None)
if isinstance(pipe_idx, int):
live_progress.finish_pipe(int(pipe_idx), force_complete=True)
except Exception:
pass
try:
live_progress.stop()
except Exception:
pass
try:
if hasattr(ctx, "set_live_progress"):
ctx.set_live_progress(None)
except Exception:
pass
@staticmethod
def _emit_storage_result(
payload: Dict[str,
@@ -2362,6 +2505,7 @@ class Add_File(Cmdlet):
"pipe_obj": pipe_obj,
"instance": instance_name,
}
pipeline_progress = PipelineProgress(ctx)
normalized_plugin_name = Add_File._normalize_provider_key(plugin_name)
f_hash = Add_File._resolve_file_hash(None, media_path, pipe_obj, None)
if normalized_plugin_name == "local":
@@ -2383,6 +2527,7 @@ class Add_File(Cmdlet):
"hash_value": f_hash,
"relationships": relationships,
"direct_export_download": direct_export_download,
"pipeline_progress": pipeline_progress,
}
)
@@ -2705,7 +2850,7 @@ class Add_File(Cmdlet):
)
# Emit a search-file-like payload for consistent tables and natural piping.
# Keep hash/store for downstream commands (get-tag, get-file, etc.).
# Keep hash/store for downstream commands (get-tag, download-file, etc.).
resolved_hash = chosen_hash
if prefer_defer_tags and tags: