syntax revamp

This commit is contained in:
2026-05-24 12:32:57 -07:00
parent 6c0a1b4415
commit 5041d9fbb9
20 changed files with 1512 additions and 1060 deletions
+245 -100
View File
@@ -195,9 +195,14 @@ class Add_File(Cmdlet):
summary=
"Ingest a local media file to a configured store or plugin destination.",
usage=
"add-file (-path <filepath> | <piped>) (-instance <store-name> | -plugin <plugin> [-instance <name|path>]) [-delete]",
"add-file (<source> | <piped>) (-instance <store-name> | -plugin <plugin> [-instance <name|path>]) [-delete]",
arg=[
SharedArgs.PATH,
CmdletArg(
name="source",
type="string",
required=False,
description="Local file or directory path to ingest or scan.",
),
SharedArgs.INSTANCE,
SharedArgs.URL,
SharedArgs.PLUGIN,
@@ -218,19 +223,38 @@ class Add_File(Cmdlet):
" 0x0: Upload to 0x0.st for temporary hosting",
" file.io: Upload to file.io for temporary hosting",
" internetarchive: Upload to archive.org (optional tag: ia:<identifier> to upload into an existing item)",
"- Use -instance with -plugin to target a named provider config: add-file -plugin ftp -instance archive -path C:\\Media\\file.pdf",
"- Use a positional source path with -instance and -plugin to target a named provider config: add-file C:\\Media\\file.pdf -plugin ftp -instance archive",
],
examples=[
'download-file "https://themathesontrust.org/papers/christianity/alcock-alphabet1.pdf" | add-file -instance tutorial',
'@1 | add-file -plugin local -instance C:\\Users\\Me\\Downloads',
'add-file -plugin ftp -instance archive -path C:\\Media\\report.pdf',
'add-file C:\\Media\\report.pdf -plugin ftp -instance archive',
],
exec=self.run,
)
self.register()
@staticmethod
def _uses_legacy_path_flag(args: Sequence[str]) -> bool:
for token in args or []:
lowered = str(token or "").strip().lower()
if lowered in {"-path", "--path", "-p"}:
return True
return False
@staticmethod
def _legacy_path_flag_message() -> str:
return (
"add-file no longer supports -path. Pass the source file or directory as a positional argument, "
"and use -plugin local -instance <name|path> for local export."
)
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
"""Main execution entry point."""
if Add_File._uses_legacy_path_flag(args):
log(Add_File._legacy_path_flag_message(), file=sys.stderr)
return 1
parsed = parse_cmdlet_args(args, self)
progress = PipelineProgress(ctx)
@@ -238,7 +262,7 @@ class Add_File(Cmdlet):
deps = _CommandDependencies(config)
storage_registry = deps.get_backend_registry()
path_arg = parsed.get("path")
source_arg = parsed.get("source")
location = parsed.get("instance")
plugin_instance = parsed.get("instance")
source_url_arg = parsed.get("url")
@@ -248,19 +272,6 @@ class Add_File(Cmdlet):
if plugin_name and not plugin_instance and location:
plugin_instance = location
# Backward-compatible shorthand: when piping a file into add-file, allow
# `-path <existing dir>` to normalize into the local export plugin path.
if path_arg and not location and not plugin_name:
try:
candidate_dir = Path(str(path_arg))
if candidate_dir.exists() and candidate_dir.is_dir():
plugin_name = "local"
plugin_instance = str(candidate_dir)
local_export_destination = str(candidate_dir)
path_arg = None
except Exception:
pass
stage_ctx = ctx.get_stage_context()
is_last_stage = (stage_ctx
is None) or bool(getattr(stage_ctx,
@@ -269,24 +280,24 @@ class Add_File(Cmdlet):
has_downstream_stage = bool(stage_ctx is not None and not is_last_stage)
# Directory-mode selector:
# - Terminal use: `add-file -instance X -path <DIR>` shows a selectable table.
# - Pipelined use: `add-file -instance X -path <DIR> | ...` processes the full batch
# - Terminal use: `add-file <DIR> -instance X` shows a selectable table.
# - Pipelined use: `add-file <DIR> -instance X | ...` processes the full batch
# immediately so downstream stages receive the uploaded items.
# - Selection replay: `@N` re-runs add-file with `-path file1,file2,...`.
# - Selection replay: `@N` re-runs add-file with `file1,file2,...` as the source token.
dir_scan_mode = False
dir_scan_results: Optional[List[Dict[str, Any]]] = None
explicit_path_list_results: Optional[List[Dict[str, Any]]] = None
explicit_source_list_results: Optional[List[Dict[str, Any]]] = None
if path_arg and location and not plugin_name:
# Support comma-separated path lists: -path "file1,file2,file3"
if source_arg and location and not plugin_name:
# Support comma-separated source lists: "file1,file2,file3"
# This is the mechanism used by @N expansion for directory tables.
try:
path_text = str(path_arg)
source_text = str(source_arg)
except Exception:
path_text = ""
source_text = ""
if "," in path_text:
parts = [p.strip().strip('"') for p in path_text.split(",")]
if "," in source_text:
parts = [p.strip().strip('"') for p in source_text.split(",")]
parts = [p for p in parts if p]
batch: List[Dict[str, Any]] = []
@@ -319,13 +330,13 @@ class Add_File(Cmdlet):
)
if batch:
explicit_path_list_results = batch
# Clear path_arg so add-file doesn't treat it as a single path.
path_arg = None
explicit_source_list_results = batch
# Clear source_arg so add-file doesn't treat it as a single path.
source_arg = None
else:
# Directory scan (selector table, no ingest yet)
try:
candidate_dir = Path(str(path_arg))
candidate_dir = Path(str(source_arg))
if candidate_dir.exists() and candidate_dir.is_dir():
dir_scan_mode = True
debug(
@@ -338,12 +349,12 @@ class Add_File(Cmdlet):
debug(
f"[add-file] Found {len(dir_scan_results)} supported files in directory"
)
# Clear path_arg so it doesn't trigger single-item mode.
path_arg = None
# Clear source_arg so it doesn't trigger single-item mode.
source_arg = None
except Exception as exc:
debug(f"[add-file] Directory scan failed: {exc}")
if result is None and not path_arg and not explicit_path_list_results and not dir_scan_results:
if result is None and not source_arg and not explicit_source_list_results and not dir_scan_results:
try:
if ctx.get_stage_context() is not None:
return 0
@@ -414,15 +425,15 @@ class Add_File(Cmdlet):
# Decide which items to process.
# - If directory scan was performed, use those results
# - If user provided -path (and it was not reinterpreted as destination), treat this invocation as single-item.
# - If user provided a positional source path, treat this invocation as single-item.
# - Otherwise, if piped input is a list, ingest each item.
if explicit_path_list_results:
items_to_process = explicit_path_list_results
debug(f"[add-file] Using {len(items_to_process)} files from -path list")
if explicit_source_list_results:
items_to_process = explicit_source_list_results
debug(f"[add-file] Using {len(items_to_process)} files from source list")
elif dir_scan_results:
items_to_process = dir_scan_results
debug(f"[add-file] Using {len(items_to_process)} files from directory scan")
elif path_arg:
elif source_arg:
items_to_process: List[Any] = [result]
elif isinstance(result, list) and result:
items_to_process = list(result)
@@ -472,26 +483,21 @@ class Add_File(Cmdlet):
)
# If this invocation was terminal directory selector mode, show a selectable table and stop.
# The user then runs @N (optionally piped), which replays add-file with selected paths.
# The user then runs @N (optionally piped), which replays add-file with selected source paths.
if should_present_directory_selector:
try:
from SYS.result_table import Table
from pathlib import Path as _Path
# Build base args to replay: keep everything except the directory -path.
base_args: List[str] = []
skip_next = False
for tok in list(args or []):
if skip_next:
skip_next = False
continue
t = str(tok)
if t in {"-path",
"--path",
"-p"}:
skip_next = True
continue
base_args.append(t)
if plugin_name:
base_args.extend(["-plugin", str(plugin_name)])
if location:
base_args.extend(["-instance", str(location)])
if source_url_arg:
base_args.extend(["-url", str(source_url_arg)])
if bool(delete_after):
base_args.append("-delete")
table = Table(title="Files in Directory", preserve_order=True)
table.set_table("add-file.directory")
@@ -517,7 +523,7 @@ class Add_File(Cmdlet):
("Size", size),
("Ext", ext),
],
selection_args=["-path", str(p) if p is not None else ""],
selection_args=[str(p) if p is not None else ""],
path=str(p) if p is not None else "",
hash=hp,
)
@@ -631,7 +637,7 @@ class Add_File(Cmdlet):
)
media_path, file_hash, temp_dir_to_cleanup = self._resolve_source(
item,
path_arg,
source_arg,
pipe_obj,
config,
export_destination=export_destination,
@@ -649,8 +655,8 @@ class Add_File(Cmdlet):
# Update pipe_obj with resolved path
pipe_obj.path = str(media_path)
# When using -path (filesystem export), allow all file types.
# When using -instance (backend), restrict to SUPPORTED_MEDIA_EXTENSIONS.
# Local/plugin exports can accept any file type.
# Storage backends stay restricted to SUPPORTED_MEDIA_EXTENSIONS.
allow_all_files = not bool(effective_storage_backend_name)
if not self._validate_source(media_path, allow_all_extensions=allow_all_files):
failures += 1
@@ -780,30 +786,7 @@ class Add_File(Cmdlet):
# Stop the live pipeline progress UI before rendering the details panels.
# This prevents the progress display from lingering on screen.
try:
live_progress = ctx.get_live_progress()
except Exception:
live_progress = None
if live_progress is not None:
try:
stage_ctx = ctx.get_stage_context()
pipe_idx = getattr(stage_ctx, "pipe_index", None)
if isinstance(pipe_idx, int):
live_progress.finish_pipe(
int(pipe_idx),
force_complete=True
)
except Exception:
pass
try:
live_progress.stop()
except Exception:
pass
try:
if hasattr(ctx, "set_live_progress"):
ctx.set_live_progress(None)
except Exception:
pass
Add_File._stop_live_progress_for_terminal_render()
subject = collected_payloads[0] if len(collected_payloads) == 1 else collected_payloads
# Use helper to display items and make them @-selectable
@@ -1108,6 +1091,8 @@ class Add_File(Cmdlet):
backend: Any,
file_hash: str,
pipe_obj: models.PipeObject,
*,
output_dir: Optional[Path] = None,
) -> Tuple[Optional[Path], Optional[Path]]:
"""Best-effort fetch of a backend file when get_file returns a URL.
@@ -1133,30 +1118,68 @@ class Add_File(Cmdlet):
metadata = getattr(pipe_obj, "metadata", {})
if isinstance(metadata, dict):
suffix = metadata.get("ext")
tmp_dir = Path(tempfile.mkdtemp(prefix="add-file-src-"))
# Introspect downloader to pass supported args (suffix, progress_callback)
download_root = output_dir
if download_root is None:
tmp_dir = Path(tempfile.mkdtemp(prefix="add-file-src-"))
download_root = tmp_dir
if download_root is None:
return None, None
# Introspect downloader to pass supported args.
import inspect
sig = inspect.signature(downloader)
kwargs = {"temp_root": tmp_dir}
kwargs = {"temp_root": download_root}
if "suffix" in sig.parameters:
kwargs["suffix"] = suffix
# Hook into global PipelineProgress if available
pp = PipelineProgress.get()
if pp and "progress_callback" in sig.parameters:
pipeline_progress = PipelineProgress(ctx)
transfer_label = "peer transfer"
try:
transfer_label = str(getattr(pipe_obj, "title", "") or "").strip() or transfer_label
except Exception:
transfer_label = "peer transfer"
if "pipeline_progress" in sig.parameters:
kwargs["pipeline_progress"] = pipeline_progress
if "transfer_label" in sig.parameters:
kwargs["transfer_label"] = transfer_label
if "progress_callback" in sig.parameters:
def _cb(done, total):
# Show fetch progress instead of just 'resolving'
pp.update(downloaded=done, total=total, label="peer transfer")
try:
total_val = int(total) if total is not None else None
except Exception:
total_val = None
try:
if int(done or 0) <= 0:
pipeline_progress.begin_transfer(
label=transfer_label,
total=total_val,
)
except Exception:
pass
try:
pipeline_progress.update_transfer(
label=transfer_label,
completed=int(done or 0),
total=total_val,
)
except Exception:
pass
kwargs["progress_callback"] = _cb
downloaded = downloader(str(file_hash), **kwargs)
if isinstance(downloaded, Path) and downloaded.exists():
if output_dir is not None:
pipe_obj.is_temp = False
if isinstance(pipe_obj.extra, dict):
pipe_obj.extra["_direct_export_download"] = True
else:
pipe_obj.extra = {"_direct_export_download": True}
return downloaded, None
pipe_obj.is_temp = True
return downloaded, tmp_dir
except Exception:
@@ -1208,6 +1231,11 @@ class Add_File(Cmdlet):
source_url=url_text,
)
pipeline_progress = PipelineProgress(ctx)
try:
destination_label = str(download_root) if download_root is not None else "temporary workspace"
pipeline_progress.set_status(f"downloading {suggested_name} to {destination_label}")
except Exception:
pass
downloaded = _download_direct_file(
url_text,
@@ -1230,6 +1258,11 @@ class Add_File(Cmdlet):
return downloaded_path, tmp_dir
except Exception:
pass
finally:
try:
PipelineProgress(ctx).clear_status()
except Exception:
pass
if tmp_dir is not None:
try:
@@ -1319,7 +1352,7 @@ class Add_File(Cmdlet):
@staticmethod
def _resolve_source(
result: Any,
path_arg: Optional[str],
source_arg: Optional[str],
pipe_obj: models.PipeObject,
config: Dict[str,
Any],
@@ -1329,7 +1362,7 @@ class Add_File(Cmdlet):
) -> Tuple[Optional[Path],
Optional[str],
Optional[Path]]:
"""Resolve the source file path from args or pipeline result.
"""Resolve the source file path from the positional source arg or pipeline result.
Returns (media_path, file_hash, temp_dir_to_cleanup).
"""
@@ -1374,7 +1407,10 @@ class Add_File(Cmdlet):
return mp_path, str(r_hash), None
dl_path, tmp_dir = Add_File._maybe_download_backend_file(
backend, str(r_hash), pipe_obj
backend,
str(r_hash),
pipe_obj,
output_dir=export_destination,
)
if dl_path and dl_path.exists():
pipe_obj.path = str(dl_path)
@@ -1395,8 +1431,8 @@ class Add_File(Cmdlet):
# PRIORITY 2: Generic Coercion (Path arg > PipeObject > Result)
candidate: Optional[Path] = None
if path_arg:
candidate = Path(path_arg)
if source_arg:
candidate = Path(source_arg)
elif pipe_obj.path:
candidate = Path(pipe_obj.path)
@@ -1471,6 +1507,83 @@ class Add_File(Cmdlet):
normalized = normalized.split(".", 1)[0]
return normalized
@staticmethod
def validate_preflight_args(
args: Sequence[str],
config: Optional[Dict[str, Any]] = None,
) -> Optional[str]:
cfg = config if isinstance(config, dict) else {}
if Add_File._uses_legacy_path_flag(args):
return f"Pipeline error: {Add_File._legacy_path_flag_message()}"
try:
parsed = parse_cmdlet_args(args, CMDLET)
except Exception as exc:
return f"Pipeline error: invalid add-file arguments: {exc}"
deps = _CommandDependencies(cfg)
storage_registry = deps.get_backend_registry()
location = parsed.get("instance")
plugin_instance = parsed.get("instance")
plugin_name = parsed.get("plugin")
is_storage_backend_location = False
if location:
try:
backend_registry_for_lookup = storage_registry or deps.get_backend_registry()
is_storage_backend_location = Add_File._resolve_backend_by_name(
backend_registry_for_lookup,
str(location),
) is not None
except Exception:
is_storage_backend_location = False
if location and not plugin_name and not is_storage_backend_location:
resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target(
location,
cfg,
deps=deps,
require_explicit=True,
)
if resolved_local_path:
return None
return (
f"Pipeline error: storage backend '{location}' not found. "
"Use -plugin local -instance <name|path> for local export or configure that store backend."
)
normalized_plugin_name = Add_File._normalize_provider_key(plugin_name)
if normalized_plugin_name:
upload_plugin = deps.get_plugin_with_capability(normalized_plugin_name, "upload")
if upload_plugin is None:
plugin_exists = deps.get_plugin(normalized_plugin_name) is not None
if plugin_exists:
if normalized_plugin_name == "loc":
return (
"Pipeline error: plugin 'loc' does not support add-file/upload. "
"Use -plugin local -instance <name|path> for local export."
)
return f"Pipeline error: plugin '{normalized_plugin_name}' does not support add-file/upload."
return f"Pipeline error: unknown upload plugin '{plugin_name}'."
if normalized_plugin_name == "local":
requested_local = str(plugin_instance or location or "").strip() or "<default>"
resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target(
plugin_instance or location,
cfg,
deps=deps,
require_explicit=bool(plugin_instance or location),
)
if not resolved_local_path:
return (
f"Pipeline error: local destination '{requested_local}' is not configured. "
"Use -plugin local -instance <name|path>."
)
return None
@staticmethod
def _resolve_plugin_storage_backend(
plugin_name: Optional[Any],
@@ -1730,8 +1843,8 @@ class Add_File(Cmdlet):
Args:
media_path: Path to the file to validate
allow_all_extensions: If True, skip file type filtering (used for -path exports).
If False, only allow SUPPORTED_MEDIA_EXTENSIONS (used for -instance).
allow_all_extensions: If True, skip file type filtering for non-backend exports.
If False, only allow SUPPORTED_MEDIA_EXTENSIONS for backend ingest.
"""
if media_path is None:
return False
@@ -1740,7 +1853,7 @@ class Add_File(Cmdlet):
log(f"File not found: {media_path}")
return False
# Validate file type: only when adding to -instance backend, not for -path exports
# Validate file type only when ingesting into a storage backend.
if not allow_all_extensions:
file_extension = media_path.suffix.lower()
if file_extension not in SUPPORTED_MEDIA_EXTENSIONS:
@@ -1947,12 +2060,42 @@ class Add_File(Cmdlet):
return
try:
Add_File._stop_live_progress_for_terminal_render()
from .._shared import display_and_persist_items
display_and_persist_items([payload], title="Result", subject=payload)
except Exception:
pass
@staticmethod
def _stop_live_progress_for_terminal_render() -> None:
try:
live_progress = ctx.get_live_progress()
except Exception:
live_progress = None
if live_progress is None:
return
try:
stage_ctx = ctx.get_stage_context()
pipe_idx = getattr(stage_ctx, "pipe_index", None)
if isinstance(pipe_idx, int):
live_progress.finish_pipe(int(pipe_idx), force_complete=True)
except Exception:
pass
try:
live_progress.stop()
except Exception:
pass
try:
if hasattr(ctx, "set_live_progress"):
ctx.set_live_progress(None)
except Exception:
pass
@staticmethod
def _emit_storage_result(
payload: Dict[str,
@@ -2362,6 +2505,7 @@ class Add_File(Cmdlet):
"pipe_obj": pipe_obj,
"instance": instance_name,
}
pipeline_progress = PipelineProgress(ctx)
normalized_plugin_name = Add_File._normalize_provider_key(plugin_name)
f_hash = Add_File._resolve_file_hash(None, media_path, pipe_obj, None)
if normalized_plugin_name == "local":
@@ -2383,6 +2527,7 @@ class Add_File(Cmdlet):
"hash_value": f_hash,
"relationships": relationships,
"direct_export_download": direct_export_download,
"pipeline_progress": pipeline_progress,
}
)
@@ -2705,7 +2850,7 @@ class Add_File(Cmdlet):
)
# Emit a search-file-like payload for consistent tables and natural piping.
# Keep hash/store for downstream commands (get-tag, get-file, etc.).
# Keep hash/store for downstream commands (get-tag, download-file, etc.).
resolved_hash = chosen_hash
if prefer_defer_tags and tags:
+317 -3
View File
@@ -15,6 +15,8 @@ from pathlib import Path
from typing import Any, Dict, List, Optional, Sequence
from urllib.parse import urlparse
from contextlib import AbstractContextManager, nullcontext
import shutil
import webbrowser
from API.HTTP import _download_direct_file
@@ -25,6 +27,7 @@ from SYS.pipeline_progress import PipelineProgress
from SYS.result_table import Table, build_display_row
from SYS.rich_display import stderr_console as get_stderr_console
from SYS import pipeline as pipeline_context
from SYS.item_accessors import get_result_title
from rich.prompt import Prompt
# SYS.metadata import deferred: normalize_urls loaded lazily at call site to avoid
# pulling in Cryptodome (~900ms) at module import time.
@@ -64,7 +67,7 @@ class Download_File(Cmdlet):
name="download-file",
summary="Download files or streaming media",
usage=
"download-file <url> [-plugin NAME] [-instance NAME] [-path DIR] [options] OR @N | download-file [-plugin NAME] [-instance NAME] [-path DIR] [options]",
"download-file <url|path> [-plugin NAME] [-instance NAME] [-path DIR] [options] OR @N | download-file [-plugin NAME] [-instance NAME] [-path DIR] [options] OR download-file -query \"hash:<sha256>\" -instance <store> [-browser]",
alias=["dl-file",
"download-http"],
arg=[
@@ -73,6 +76,16 @@ class Download_File(Cmdlet):
SharedArgs.INSTANCE,
SharedArgs.PATH,
SharedArgs.QUERY,
CmdletArg(
name="name",
type="string",
description="Output filename override for store exports.",
),
CmdletArg(
name="browser",
type="flag",
description="Open a backend-provided browser URL instead of exporting to disk when available.",
),
QueryArg(
"clip",
key="clip",
@@ -95,6 +108,7 @@ class Download_File(Cmdlet):
],
detail=[
"Download files directly via HTTP or streaming media via yt-dlp.",
"Also exports store-backed files via hash+store selection or -query \"hash:<sha256>\" -instance <store>.",
"Use -plugin with -instance to target a named provider config when a plugin exposes multiple instances.",
"For Internet Archive item pages (archive.org/details/...), shows a selectable file/format list; pick with @N to download.",
],
@@ -924,6 +938,283 @@ class Download_File(Cmdlet):
pipeline_context.emit(payload)
@staticmethod
def _path_looks_local(value: Any) -> bool:
text = str(value or "").strip()
if not text:
return False
if text.startswith(("http://", "https://", "ftp://", "ftps://", "magnet:", "torrent:")):
return False
if len(text) >= 2 and text[1] == ":":
return True
if text.startswith(("\\", "/", ".", "~")):
return True
return Path(text).exists()
@staticmethod
def _resolve_display_title(result: Any, metadata: Optional[Dict[str, Any]]) -> str:
candidates = [
get_result_title(result, "title", "name", "filename"),
get_result_title(metadata or {}, "title", "name", "filename"),
]
for candidate in candidates:
if candidate is None:
continue
text = str(candidate).strip()
if text:
return text
return ""
@staticmethod
def _sanitize_export_filename(name: str) -> str:
allowed_chars: List[str] = []
for ch in str(name or ""):
if ch.isalnum() or ch in {"-", "_", " ", "."}:
allowed_chars.append(ch)
else:
allowed_chars.append(" ")
sanitized = " ".join("".join(allowed_chars).split())
return sanitized or "export"
@staticmethod
def _unique_export_path(path: Path) -> Path:
if not path.exists():
return path
stem = path.stem
suffix = path.suffix
parent = path.parent
counter = 1
while True:
candidate = parent / f"{stem} ({counter}){suffix}"
if not candidate.exists():
return candidate
counter += 1
@staticmethod
def _iter_storage_export_refs(
parsed: Dict[str, Any],
piped_items: Sequence[Any],
) -> tuple[List[Dict[str, Any]], List[Any], Optional[int]]:
refs: List[Dict[str, Any]] = []
residual_items: List[Any] = []
query_text = str(parsed.get("query") or "").strip()
query_hash: Optional[str] = None
if query_text:
query_hash = sh.parse_single_hash_query(query_text)
if query_text.lower().startswith("hash") and not query_hash:
log('Error: -query must be of the form hash:<sha256>', file=sys.stderr)
return [], list(piped_items or []), 1
explicit_store = str(parsed.get("instance") or "").strip()
if query_hash:
if not explicit_store:
log('Error: No store name provided', file=sys.stderr)
return [], list(piped_items or []), 1
refs.append(
{
"hash": query_hash,
"store": explicit_store,
"result": None,
}
)
for item in piped_items or []:
normalized_hash = sh.normalize_hash(
str(get_field(item, "hash") or get_field(item, "file_hash") or get_field(item, "hash_hex") or "")
)
store_name = str(parsed.get("instance") or get_field(item, "store") or "").strip()
if normalized_hash and store_name:
refs.append(
{
"hash": normalized_hash,
"store": store_name,
"result": item,
}
)
else:
residual_items.append(item)
return refs, residual_items, None
def _export_store_file(
self,
*,
file_hash: str,
store_name: str,
result: Any,
parsed: Dict[str, Any],
config: Dict[str, Any],
final_output_dir: Path,
) -> int:
output_path = parsed.get("path")
explicit_output_requested = bool(output_path)
output_name = parsed.get("name")
browser_flag = bool(parsed.get("browser"))
backend, _store_registry, _exc = sh.get_preferred_store_backend(
config,
store_name,
suppress_debug=True,
)
if backend is None:
log(f"Error: Storage backend '{store_name}' not found", file=sys.stderr)
return 1
metadata = backend.get_metadata(file_hash)
if not metadata:
log(f"Error: File metadata not found for hash {file_hash}", file=sys.stderr)
return 1
try:
debug_panel(
"download-file store export",
[
("hash", file_hash),
("instance", store_name),
("output_path", output_path or "<default>"),
("output_name", output_name or "<auto>"),
("browser", browser_flag),
],
border_style="blue",
)
except Exception:
pass
want_url = browser_flag
source_path = backend.get_file(file_hash, url=want_url)
download_url = None
if isinstance(source_path, str):
if source_path.startswith(("http://", "https://")):
download_url = source_path
else:
source_path = Path(source_path)
if download_url and (browser_flag or not explicit_output_requested):
try:
webbrowser.open(download_url)
except Exception as exc:
log(f"Error opening browser: {exc}", file=sys.stderr)
return 1
pipeline_context.emit(
build_file_result_payload(
title=self._resolve_display_title(result, metadata) or "Opened",
hash_value=file_hash,
store=store_name,
url=download_url,
)
)
return 0
if download_url is None:
if not source_path or not Path(source_path).exists():
log(f"Error: Backend could not retrieve file for hash {file_hash}", file=sys.stderr)
return 1
filename = str(output_name or "").strip()
if not filename:
title = (metadata.get("title") if isinstance(metadata, dict) else None) or self._resolve_display_title(result, metadata) or "export"
filename = self._sanitize_export_filename(str(title))
ext = metadata.get("ext") if isinstance(metadata, dict) else None
if ext and not filename.endswith(str(ext)):
ext_text = str(ext)
if not ext_text.startswith("."):
ext_text = "." + ext_text
filename += ext_text
if download_url:
result_obj = _download_direct_file(
download_url,
final_output_dir,
quiet=True,
suggested_filename=filename,
pipeline_progress=config.get("_pipeline_progress") if isinstance(config, dict) else None,
)
dest_path = self._path_from_download_result(result_obj)
else:
dest_path = self._unique_export_path(final_output_dir / filename)
shutil.copy2(Path(source_path), dest_path)
pipeline_context.emit(
build_file_result_payload(
title=filename,
hash_value=file_hash,
store=store_name,
path=str(dest_path),
)
)
return 0
def _process_storage_items(
self,
*,
piped_items: Sequence[Any],
parsed: Dict[str, Any],
config: Dict[str, Any],
final_output_dir: Path,
) -> tuple[int, List[Any], Optional[int]]:
refs, residual_items, early_exit = self._iter_storage_export_refs(parsed, piped_items)
if early_exit is not None:
return 0, list(residual_items), early_exit
if not refs:
return 0, list(residual_items), None
successes = 0
for ref in refs:
exit_code = self._export_store_file(
file_hash=str(ref.get("hash") or ""),
store_name=str(ref.get("store") or ""),
result=ref.get("result"),
parsed=parsed,
config=config,
final_output_dir=final_output_dir,
)
if exit_code != 0:
return successes, list(residual_items), exit_code
successes += 1
return successes, list(residual_items), None
def _process_explicit_local_sources(
self,
*,
local_sources: Sequence[str],
final_output_dir: Path,
parsed: Dict[str, Any],
progress: PipelineProgress,
config: Dict[str, Any],
) -> int:
explicit_output_requested = bool(parsed.get("path"))
downloaded_count = 0
for raw_source in local_sources or []:
source_path = Path(str(raw_source or "")).expanduser()
if not source_path.exists() or not source_path.is_file():
log(f"File not found: {source_path}", file=sys.stderr)
continue
if explicit_output_requested:
destination = final_output_dir / source_path.name
destination = self._unique_export_path(destination)
shutil.copy2(source_path, destination)
emit_path = destination
else:
emit_path = source_path
self._emit_local_file(
downloaded_path=emit_path,
source=str(source_path),
title_hint=emit_path.stem,
tags_hint=None,
media_kind_hint="file",
full_metadata=None,
progress=progress,
config=config,
)
downloaded_count += 1
return downloaded_count
def _maybe_render_download_details(self, *, config: Dict[str, Any]) -> None:
try:
stage_ctx = pipeline_context.get_stage_context()
@@ -2377,6 +2668,7 @@ class Download_File(Cmdlet):
parsed = parse_cmdlet_args(args, self)
registry = self._load_provider_registry()
selection_url_prefixes = self._selection_url_prefixes(registry)
explicit_input = parsed.get("url")
# Resolve URLs from -url or positional arguments
url_candidates = parsed.get("url") or [
@@ -2389,6 +2681,9 @@ class Download_File(Cmdlet):
]
from SYS.metadata import normalize_urls as normalize_url_list # lazy: avoids Cryptodome at startup
raw_url = normalize_url_list(url_candidates)
local_source_inputs: List[str] = []
if not raw_url and isinstance(explicit_input, str) and self._path_looks_local(explicit_input):
local_source_inputs = [str(explicit_input)]
quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False
@@ -2608,7 +2903,7 @@ class Download_File(Cmdlet):
raw_url = []
piped_items = self._collect_piped_items_if_no_urls(result, raw_url)
if not raw_url and not piped_items:
if not raw_url and not piped_items and not local_source_inputs:
log("No url or piped items to download", file=sys.stderr)
return 1
@@ -2673,8 +2968,27 @@ class Download_File(Cmdlet):
downloaded_count = 0
if local_source_inputs:
downloaded_count += self._process_explicit_local_sources(
local_sources=local_source_inputs,
final_output_dir=final_output_dir,
parsed=parsed,
progress=progress,
config=config,
)
storage_downloaded, piped_items, storage_exit = self._process_storage_items(
piped_items=piped_items,
parsed=parsed,
config=config,
final_output_dir=final_output_dir,
)
downloaded_count += int(storage_downloaded)
if storage_exit is not None:
return int(storage_exit)
if skipped_dupe_count and not raw_url and not piped_items:
return 0
return 0 if downloaded_count > 0 else 0
urls_downloaded, early_exit = self._process_explicit_urls(
raw_urls=raw_url,
-497
View File
@@ -1,497 +0,0 @@
from __future__ import annotations
from typing import Any, Dict, Sequence
from pathlib import Path
import os
import sys
import shutil
import subprocess
import tempfile
import threading
import time
import http.server
from urllib.parse import quote
import webbrowser
from urllib.parse import urljoin
from urllib.request import pathname2url
from SYS import pipeline as ctx
from .. import _shared as sh
from SYS.item_accessors import get_result_title
from SYS.logger import log, debug, debug_panel
from SYS.config import resolve_output_dir
from API.HTTP import _download_direct_file
from SYS.payload_builders import build_file_result_payload
class Get_File(sh.Cmdlet):
"""Export files to local path via hash+store."""
def __init__(self) -> None:
"""Initialize get-file cmdlet."""
super().__init__(
name="get-file",
summary="Export file to local path",
usage="@1 | get-file -path ./output",
arg=[
sh.SharedArgs.QUERY,
sh.SharedArgs.INSTANCE,
sh.SharedArgs.PATH,
sh.CmdletArg(
"name",
description="Output filename (default: from metadata title)"
),
sh.CmdletArg(
"browser",
flag=True,
description="Open file in browser instead of saving to disk"
),
],
detail=[
"- Exports file from storage backend to local path",
'- Uses selected item\'s hash, or -query "hash:<sha256>"',
"- Preserves file extension and metadata",
],
exec=self.run,
)
self.register()
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
"""Export file via hash+store backend."""
parsed = sh.parse_cmdlet_args(args, self)
try:
debug_panel(
"get-file",
[
("result_type", type(result).__name__),
("parsed_args", parsed),
],
border_style="cyan",
)
except Exception:
pass
query_hash, query_valid = sh.require_single_hash_query(
parsed.get("query"),
"Error: -query must be of the form hash:<sha256>",
)
if not query_valid:
return 1
# Extract hash and store from result or args
file_hash = query_hash or sh.get_field(result, "hash")
store_name = parsed.get("instance") or sh.get_field(result, "store")
output_path = parsed.get("path")
output_name = parsed.get("name")
browser_flag = bool(parsed.get("browser"))
if not file_hash:
log(
'Error: No file hash provided (pipe an item or use -query "hash:<sha256>")'
)
return 1
if not store_name:
log("Error: No store name provided")
return 1
# Normalize hash
file_hash = sh.normalize_hash(file_hash)
if not file_hash:
log("Error: Invalid hash format")
return 1
try:
debug_panel(
"get-file selection",
[
("hash", file_hash),
("instance", store_name),
("output_path", output_path or "<default>"),
("output_name", output_name or "<auto>"),
],
border_style="blue",
)
except Exception:
pass
backend, _store_registry, _exc = sh.get_preferred_store_backend(
config,
store_name,
suppress_debug=True,
)
if backend is None:
log(f"Error: Storage backend '{store_name}' not found", file=sys.stderr)
return 1
# Get file metadata to determine name and extension
metadata = backend.get_metadata(file_hash)
if not metadata:
log(f"Error: File metadata not found for hash {file_hash}")
return 1
try:
debug_panel(
"get-file backend",
[
("backend", type(backend).__name__),
("title", metadata.get("title") or ""),
("ext", metadata.get("ext") or ""),
],
border_style="green",
)
except Exception:
pass
def resolve_display_title() -> str:
candidates = [
get_result_title(result, "title", "name", "filename"),
get_result_title(metadata, "title", "name", "filename"),
]
for candidate in candidates:
if candidate is None:
continue
text = str(candidate).strip()
if text:
return text
return ""
# Get file from backend (may return Path or URL string depending on backend).
# If -browser is given, request a URL (for Hydrus viewer). If -path is given,
# always retrieve a local file. Otherwise default to local export.
want_url = browser_flag
source_path = backend.get_file(file_hash, url=want_url)
download_url = None
if isinstance(source_path, str):
if source_path.startswith("http://") or source_path.startswith("https://"):
download_url = source_path
else:
source_path = Path(source_path)
try:
debug_panel(
"get-file fetch",
[
("url_hint", want_url),
("mode", "browser-url" if download_url else "local-path"),
("source", download_url or source_path or "<missing>"),
],
border_style="magenta",
)
except Exception:
pass
if download_url and (browser_flag or output_path is None):
# Open in browser: explicit -browser flag, or Hydrus returned a URL with no output path
try:
webbrowser.open(download_url)
except Exception as exc:
log(f"Error opening browser: {exc}", file=sys.stderr)
else:
try:
debug_panel(
"get-file open",
[
("action", "browser-open"),
("url", download_url),
],
file=sys.stderr,
border_style="green",
)
except Exception:
pass
ctx.emit(
build_file_result_payload(
title=resolve_display_title() or "Opened",
hash_value=file_hash,
store=store_name,
url=download_url,
)
)
return 0
if download_url is None:
if not source_path or not source_path.exists():
log(f"Error: Backend could not retrieve file for hash {file_hash}")
return 1
# Otherwise: export/copy to output_dir.
if output_path:
output_dir = Path(output_path).expanduser()
else:
output_dir = resolve_output_dir(config)
output_dir.mkdir(parents=True, exist_ok=True)
# Determine output filename (only when exporting)
if output_name:
filename = output_name
else:
title = (
(metadata.get("title") if isinstance(metadata,
dict) else None)
or resolve_display_title() or "export"
)
filename = self._sanitize_filename(title)
# Add extension if metadata has it
ext = metadata.get("ext")
if ext and not filename.endswith(ext):
if not ext.startswith("."):
ext = "." + ext
filename += ext
dest_path: Path
if download_url:
downloaded = _download_direct_file(
download_url,
output_dir,
quiet=True,
suggested_filename=filename,
)
dest_path = downloaded.path
else:
dest_path = self._unique_path(output_dir / filename)
# Copy file to destination
shutil.copy2(source_path, dest_path)
try:
debug_panel(
"get-file export",
[
("mode", "download" if download_url else "copy"),
("destination", dest_path),
("filename", filename),
],
file=sys.stderr,
border_style="green",
)
except Exception:
pass
log(f"Exported: {dest_path}", file=sys.stderr)
# Emit result for pipeline
ctx.emit(
build_file_result_payload(
title=filename,
hash_value=file_hash,
store=store_name,
path=str(dest_path),
)
)
return 0
def _open_file_default(self, path: Path) -> None:
"""Open a local file in the OS default application."""
try:
suffix = str(path.suffix or "").lower()
if sys.platform.startswith("win"):
# On Windows, file associations for common media types can point at
# editors (Paint/VS Code). Prefer opening a localhost URL.
if self._open_local_file_in_browser_via_http(path):
return
if suffix in {
".png",
".jpg",
".jpeg",
".gif",
".webp",
".bmp",
".tif",
".tiff",
".svg",
}:
# Use default web browser for images.
if self._open_image_in_default_browser(path):
return
if sys.platform.startswith("win"):
os.startfile(str(path)) # type: ignore[attr-defined]
return
if sys.platform == "darwin":
subprocess.Popen(
["open",
str(path)],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
return
subprocess.Popen(
["xdg-open",
str(path)],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
except Exception as exc:
log(f"Error opening file: {exc}", file=sys.stderr)
def _open_local_file_in_browser_via_http(self, file_path: Path) -> bool:
"""Serve a single local file via localhost HTTP and open in browser.
This avoids Windows file-association issues (e.g., PNG -> Paint, HTML -> VS Code).
The server is bound to 127.0.0.1 on an ephemeral port and is shut down after
a timeout.
"""
try:
resolved = file_path.resolve()
directory = resolved.parent
filename = resolved.name
except Exception:
return False
class OneFileHandler(http.server.SimpleHTTPRequestHandler):
def __init__(self, *handler_args, **handler_kwargs):
super().__init__(
*handler_args,
directory=str(directory),
**handler_kwargs
)
def log_message(self, format: str, *args) -> None: # noqa: A003
# Keep normal output clean.
return
def do_GET(self) -> None: # noqa: N802
if self.path in {"/",
""}:
self.path = "/" + filename
return super().do_GET()
if self.path == "/" + filename or self.path == "/" + quote(filename):
return super().do_GET()
self.send_error(404)
def do_HEAD(self) -> None: # noqa: N802
if self.path in {"/",
""}:
self.path = "/" + filename
return super().do_HEAD()
if self.path == "/" + filename or self.path == "/" + quote(filename):
return super().do_HEAD()
self.send_error(404)
try:
httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), OneFileHandler)
except Exception:
return False
port = httpd.server_address[1]
url = f"http://127.0.0.1:{port}/{quote(filename)}"
# Run server in the background.
server_thread = threading.Thread(
target=httpd.serve_forever,
kwargs={
"poll_interval": 0.2
},
daemon=True
)
server_thread.start()
# Auto-shutdown after a timeout to avoid lingering servers.
def shutdown_later() -> None:
time.sleep(10 * 60)
try:
httpd.shutdown()
except Exception:
pass
try:
httpd.server_close()
except Exception:
pass
threading.Thread(target=shutdown_later, daemon=True).start()
try:
debug(f"[get-file] Opening via localhost: {url}")
return bool(webbrowser.open(url))
except Exception:
return False
def _open_image_in_default_browser(self, image_path: Path) -> bool:
"""Open an image file in the user's default web browser.
We intentionally avoid opening the image path directly on Windows because
file associations may point to editors/viewers (e.g., Paint). Instead we
generate a tiny HTML wrapper and open that (HTML is typically associated
with the default browser).
"""
try:
resolved = image_path.resolve()
image_url = urljoin("file:", pathname2url(str(resolved)))
except Exception:
return False
# Create a stable wrapper filename to reduce temp-file spam.
wrapper_path = Path(
tempfile.gettempdir()
) / f"medeia-open-image-{resolved.stem}.html"
try:
wrapper_path.write_text(
"\n".join(
[
"<!doctype html>",
'<meta charset="utf-8">',
f"<title>{resolved.name}</title>",
"<style>html,body{margin:0;padding:0;background:#000}img{display:block;max-width:100vw;max-height:100vh;margin:auto}</style>",
f'<img src="{image_url}" alt="{resolved.name}">',
]
),
encoding="utf-8",
)
except Exception:
return False
# Prefer localhost server when possible (reliable on Windows).
if self._open_local_file_in_browser_via_http(image_path):
return True
wrapper_url = wrapper_path.as_uri()
try:
return bool(webbrowser.open(wrapper_url))
except Exception:
return False
def _sanitize_filename(self, name: str) -> str:
"""Sanitize filename by removing invalid characters."""
allowed_chars = []
for ch in str(name):
if ch.isalnum() or ch in {"-",
"_",
" ",
"."}:
allowed_chars.append(ch)
else:
allowed_chars.append(" ")
# Collapse multiple spaces
sanitized = " ".join("".join(allowed_chars).split())
return sanitized or "export"
def _unique_path(self, path: Path) -> Path:
"""Generate unique path by adding (1), (2), etc. if file exists."""
if not path.exists():
return path
stem = path.stem
suffix = path.suffix
parent = path.parent
counter = 1
while True:
new_path = parent / f"{stem} ({counter}){suffix}"
if not new_path.exists():
return new_path
counter += 1
# Instantiate and register cmdlet
Add_File_Instance = Get_File()
+54 -28
View File
@@ -193,7 +193,7 @@ class search_file(Cmdlet):
"URL search: url:* (any URL) or url:<value> (URL substring)",
"Extension search: ext:<value> (e.g., ext:png)",
"Hydrus-style extension: system:filetype = png",
"Results include hash for downstream commands (get-file, add-tag, etc.)",
"Results include hash for downstream commands (download-file, add-tag, etc.)",
"Examples:",
"search-file -query foo # Search all storage backends",
"search-file -instance home -query '*' # Search 'home' Hydrus instance",
@@ -1755,6 +1755,13 @@ class search_file(Cmdlet):
f.lower()
for f in (flag_registry.get("open") or {"-open", "--open"})
}
valued_flags = (
query_flags
| instance_flags
| limit_flags
| plugin_flags
| open_flags
)
# Parse arguments
query = ""
@@ -1771,37 +1778,56 @@ class search_file(Cmdlet):
while i < len(args_list):
arg = args_list[i]
low = arg.lower()
if low in query_flags and i + 1 < len(args_list):
chunk = args_list[i + 1]
query = f"{query} {chunk}".strip() if query else chunk
i += 2
next_arg = args_list[i + 1] if i + 1 < len(args_list) else None
next_low = str(next_arg or "").lower()
next_is_flag = bool(next_arg) and next_low in valued_flags
if low in query_flags:
if next_arg is not None and not next_is_flag:
chunk = next_arg
query = f"{query} {chunk}".strip() if query else chunk
i += 2
continue
i += 1
continue
if low in plugin_flags and i + 1 < len(args_list):
plugin_name = args_list[i + 1]
i += 2
if low in plugin_flags:
if next_arg is not None and not next_is_flag:
plugin_name = next_arg
i += 2
continue
i += 1
continue
if low in instance_flags and i + 1 < len(args_list):
instance_name = args_list[i + 1]
i += 2
if low in instance_flags:
if next_arg is not None and not next_is_flag:
instance_name = next_arg
i += 2
continue
i += 1
continue
if low in open_flags and i + 1 < len(args_list):
try:
open_id = int(args_list[i + 1])
except ValueError:
log(
f"Warning: Invalid open value '{args_list[i + 1]}', ignoring",
file=sys.stderr,
)
open_id = None
i += 2
if low in open_flags:
if next_arg is not None and not next_is_flag:
try:
open_id = int(next_arg)
except ValueError:
log(
f"Warning: Invalid open value '{next_arg}', ignoring",
file=sys.stderr,
)
open_id = None
i += 2
continue
i += 1
continue
if low in limit_flags and i + 1 < len(args_list):
limit_set = True
try:
limit = int(args_list[i + 1])
except ValueError:
limit = 100
i += 2
if low in limit_flags:
if next_arg is not None and not next_is_flag:
limit_set = True
try:
limit = int(next_arg)
except ValueError:
limit = 100
i += 2
continue
i += 1
elif not arg.startswith("-"):
positional_args.append(arg)
query = f"{query} {arg}".strip() if query else arg
+8 -9
View File
@@ -13,12 +13,12 @@ SharedArgs = sh.SharedArgs
class File(Cmdlet):
"""Unified file command: file -add|-delete|-get|-merge|..."""
"""Unified file command: file -search|-add|-delete|-download|-merge|..."""
_ACTION_FLAGS = {
"search": {"-search", "--search"},
"add": {"-add", "--add"},
"delete": {"-delete", "--delete", "-del", "--del"},
"get": {"-get", "--get"},
"merge": {"-merge", "--merge"},
"download": {"-download", "--download", "-dl", "--dl"},
"convert": {"-convert", "--convert"},
@@ -30,7 +30,6 @@ class File(Cmdlet):
_ACTION_MODULE = {
"add": "cmdlet.file.add",
"delete": "cmdlet.file.delete",
"get": "cmdlet.file.get",
"merge": "cmdlet.file.merge",
"download": "cmdlet.file.download",
"search": "cmdlet.file.search",
@@ -44,15 +43,14 @@ class File(Cmdlet):
super().__init__(
name="file",
summary="Manage file operations with one command",
usage='file -query <query> [args] | file (-add|-delete|-get|-merge|-download|-convert|-trim|-archive|-screenshot) [args]',
usage='file -query <query> [args] | file (-search|-add|-delete|-merge|-download|-convert|-trim|-archive|-screenshot) [args]',
arg=[
SharedArgs.QUERY,
SharedArgs.PLUGIN,
SharedArgs.INSTANCE,
SharedArgs.PATH,
CmdletArg("-search", type="flag", required=False, description="Run search-file"),
CmdletArg("-add", type="flag", required=False, description="Run add-file"),
CmdletArg("-delete", type="flag", required=False, description="Run delete-file", alias="del"),
CmdletArg("-get", type="flag", required=False, description="Run get-file"),
CmdletArg("-merge", type="flag", required=False, description="Run merge-file"),
CmdletArg("-download", type="flag", required=False, description="Run download-file", alias="dl"),
CmdletArg("-convert", type="flag", required=False, description="Run convert-file"),
@@ -61,10 +59,11 @@ class File(Cmdlet):
CmdletArg("-screenshot", type="flag", required=False, description="Run screen-shot", alias="shot"),
],
detail=[
"- Use -query to run search-file through the unified file command.",
"- Use -search for explicit search mode, then add -plugin/-instance and -query as needed.",
"- Plain -query still routes to search-file for direct search entry.",
"- Otherwise, exactly one non-search action flag is required.",
"- Remaining args are passed through to the selected file cmdlet.",
"- Examples: file -query ..., file -add ..., file -delete ...",
"- Examples: file -search -plugin hydrusnetwork -query ..., file -add ..., file -delete ...",
],
exec=self.run,
)
@@ -139,7 +138,7 @@ class File(Cmdlet):
if action is None:
if not seen:
log(
"file: missing action; use -query for search or choose exactly one of -add, -delete, -get, -merge, -download, -convert, -trim, -archive, -screenshot",
"file: missing action; use -search/-query for search or choose exactly one of -search, -add, -delete, -merge, -download, -convert, -trim, -archive, -screenshot",
file=sys.stderr,
)
else: