syntax revamp

This commit is contained in:
2026-05-24 12:32:57 -07:00
parent 6c0a1b4415
commit 5041d9fbb9
20 changed files with 1512 additions and 1060 deletions
+485 -312
View File
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -169,7 +169,7 @@ class Provider(ABC):
# Declare which top-level cmdlet names this plugin handles.
# Cmdlet dispatch and capability discovery use this to route operations.
# Example: frozenset({"add-file", "get-file", "get-tag", "search-file"})
# Example: frozenset({"add-file", "download-file", "get-tag", "search-file"})
SUPPORTED_CMDLETS: frozenset = frozenset()
def __init__(self, config: Optional[Dict[str, Any]] = None):
+1 -1
View File
@@ -471,7 +471,7 @@ class PluginRegistry:
if not info.is_multi_instance:
continue
if not info.supported_cmdlets.intersection(
{"add-file", "get-file", "tag"}
{"add-file", "download-file", "tag"}
):
continue
try:
+101 -12
View File
@@ -70,6 +70,17 @@ def _tokenize_stage(stage_text: str) -> list[str]:
return text.split()
def _parse_pipeline_tokens(raw: str) -> list[tuple[str, list[str]]]:
parsed: list[tuple[str, list[str]]] = []
for stage in _split_pipeline_stages(raw):
tokens = _tokenize_stage(stage)
if not tokens:
continue
cmd = str(tokens[0]).replace("_", "-").strip().lower()
parsed.append((cmd, tokens))
return parsed
def _has_flag(tokens: list[str], *flags: str) -> bool:
want = {str(f).strip().lower() for f in flags if str(f).strip()}
if not want:
@@ -115,18 +126,10 @@ def _validate_add_note_requires_add_file_order(raw: str) -> Optional[SyntaxError
Rationale: add-note requires a known (store, hash) target; piping before add-file
means the item likely has no hash yet.
"""
stages = _split_pipeline_stages(raw)
if len(stages) <= 1:
parsed = _parse_pipeline_tokens(raw)
if len(parsed) <= 1:
return None
parsed: list[tuple[str, list[str]]] = []
for stage in stages:
tokens = _tokenize_stage(stage)
if not tokens:
continue
cmd = str(tokens[0]).replace("_", "-").strip().lower()
parsed.append((cmd, tokens))
add_file_positions = [i for i, (cmd, _toks) in enumerate(parsed) if cmd == "add-file"]
if not add_file_positions:
return None
@@ -165,7 +168,85 @@ def _validate_add_note_requires_add_file_order(raw: str) -> Optional[SyntaxError
return None
def validate_pipeline_text(text: str) -> Optional[SyntaxErrorDetail]:
def _validate_file_cmdlet_stage_actions(raw: str) -> Optional[SyntaxErrorDetail]:
parsed = _parse_pipeline_tokens(raw)
if not parsed:
return None
try:
from cmdlet.file_cmdlet import File as FileCmdlet
except Exception:
return None
for cmd, tokens in parsed:
if cmd != "file":
continue
action, _passthrough, seen = FileCmdlet._extract_action(tokens[1:])
if action is not None:
continue
if seen:
rendered = ", ".join(f"-{name}" for name in seen)
return SyntaxErrorDetail(
f"Pipeline error: 'file' has conflicting actions ({rendered}); choose exactly one."
)
if _has_flag(tokens[1:], "-plugin", "--plugin", "-instance", "--instance", "-path", "--path"):
return SyntaxErrorDetail(
"Pipeline error: 'file' requires an explicit action here. "
"Use 'file -add -plugin local -instance <name|path>' for local export, or 'file -search ...' for search."
)
return SyntaxErrorDetail(
"Pipeline error: 'file' requires -search/-query for search or exactly one action flag "
"(-search, -add, -delete, -merge, -download, -convert, -trim, -archive, -screenshot)."
)
return None
def _validate_add_file_stage_preflight(
raw: str,
config: Optional[Dict[str, Any]],
) -> Optional[SyntaxErrorDetail]:
if not isinstance(config, dict):
return None
parsed = _parse_pipeline_tokens(raw)
if not parsed:
return None
try:
from cmdlet.file.add import Add_File
from cmdlet.file_cmdlet import File as FileCmdlet
except Exception:
return None
for cmd, tokens in parsed:
stage_args: Optional[list[str]] = None
if cmd == "add-file":
stage_args = tokens[1:]
elif cmd == "file":
action, passthrough, _seen = FileCmdlet._extract_action(tokens[1:])
if action == "add":
stage_args = passthrough
if stage_args is None:
continue
message = Add_File.validate_preflight_args(stage_args, config)
if message:
return SyntaxErrorDetail(message)
return None
def validate_pipeline_text(
text: str,
config: Optional[Dict[str, Any]] = None,
) -> Optional[SyntaxErrorDetail]:
"""Validate raw CLI input before tokenization/execution.
This is intentionally lightweight and focuses on user-facing syntax issues:
@@ -252,11 +333,19 @@ def validate_pipeline_text(text: str) -> Optional[SyntaxErrorDetail]:
if not in_single and not in_double and not ch.isspace():
seen_nonspace_since_pipe = True
# Semantic rules (still lightweight; no cmdlet imports)
# Pipeline-only semantic rules.
semantic_error = _validate_add_note_requires_add_file_order(raw)
if semantic_error is not None:
return semantic_error
semantic_error = _validate_file_cmdlet_stage_actions(raw)
if semantic_error is not None:
return semantic_error
semantic_error = _validate_add_file_stage_preflight(raw, config)
if semantic_error is not None:
return semantic_error
return None
+8 -2
View File
@@ -612,6 +612,8 @@ def write_tags(
url: Iterable[str],
hash_value: Optional[str] = None,
db=None,
*,
emit_debug: bool = True,
) -> None:
"""Write tags to database or sidecar file (tags only).
@@ -665,7 +667,8 @@ def write_tags(
if lines:
sidecar.write_text("\n".join(lines) + "\n", encoding="utf-8")
debug(f"Tags: {sidecar}")
if emit_debug:
debug(f"Tags: {sidecar}")
else:
try:
sidecar.unlink()
@@ -681,6 +684,8 @@ def write_metadata(
url: Optional[Iterable[str]] = None,
relationships: Optional[Iterable[str]] = None,
db=None,
*,
emit_debug: bool = True,
) -> None:
"""Write metadata to database or sidecar file.
@@ -753,7 +758,8 @@ def write_metadata(
# Write metadata file
if lines:
sidecar.write_text("\n".join(lines) + "\n", encoding="utf-8")
debug(f"Wrote metadata to {sidecar}")
if emit_debug:
debug(f"Wrote metadata to {sidecar}")
else:
# Remove if no content
try:
+5 -9
View File
@@ -2089,7 +2089,7 @@ class PipelineExecutor:
# Command expansion via @N:
# - Default behavior: expand ONLY for single-row selections.
# - Special case: allow multi-row expansion for add-file directory tables by
# combining selected rows into a single `-path file1,file2,...` argument.
# combining selected rows into one comma-separated positional source token.
if source_cmd and not skip_pipe_expansion and not prefer_row_action:
src = str(source_cmd).replace("_", "-").strip().lower()
@@ -2107,19 +2107,15 @@ class PipelineExecutor:
[str(x) for x in row_args if x is not None]
)
# Combine `['-path', <file>]` from each row into one `-path` arg.
# Combine `[<file>]` from each row into one positional source token.
paths: List[str] = []
can_merge = bool(row_args_list) and (
len(row_args_list) == len(selection_indices)
)
if can_merge:
for ra in row_args_list:
if len(ra) == 2 and str(ra[0]).strip().lower() in {
"-path",
"--path",
"-p",
}:
p = str(ra[1]).strip()
if len(ra) == 1:
p = str(ra[0]).strip()
if p:
paths.append(p)
else:
@@ -2127,7 +2123,7 @@ class PipelineExecutor:
break
if can_merge and paths:
selected_row_args.extend(["-path", ",".join(paths)])
selected_row_args.append(",".join(paths))
elif len(selection_indices) == 1 and row_args_list:
selected_row_args.extend(row_args_list[0])
else:
+70
View File
@@ -1,16 +1,86 @@
from __future__ import annotations
import json
import os
import time
import uuid
from pathlib import Path
from typing import Any, Dict, List, Optional
_REPL_STATE_FILENAME = "medeia-repl-state.json"
def repl_queue_dir(root: Path) -> Path:
return Path(root) / "Log" / "repl_queue"
def repl_state_path(root: Path) -> Path:
return Path(root) / "Log" / _REPL_STATE_FILENAME
def _write_json_atomic(path: Path, payload: Dict[str, Any]) -> Path:
path.parent.mkdir(parents=True, exist_ok=True)
temp_path = path.with_suffix(path.suffix + ".tmp")
temp_path.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
temp_path.replace(path)
return path
def touch_repl_state(
root: Path,
*,
session_id: Optional[str] = None,
pid: Optional[int] = None,
status: str = "running",
) -> Path:
payload: Dict[str, Any] = {
"status": str(status or "running").strip() or "running",
"updated_at": time.time(),
"pid": int(pid) if pid is not None else int(os.getpid()),
}
if isinstance(session_id, str) and session_id.strip():
payload["session_id"] = session_id.strip()
return _write_json_atomic(repl_state_path(root), payload)
def read_repl_state(root: Path) -> Optional[Dict[str, Any]]:
path = repl_state_path(root)
try:
if not path.exists() or not path.is_file():
return None
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception:
return None
return payload if isinstance(payload, dict) else None
def clear_repl_state(root: Path) -> None:
path = repl_state_path(root)
try:
path.unlink()
except Exception:
return
def repl_state_is_alive(root: Path, *, max_age_seconds: float = 3.0) -> bool:
payload = read_repl_state(root)
if not isinstance(payload, dict):
return False
if str(payload.get("status") or "").strip().lower() != "running":
return False
try:
updated_at = float(payload.get("updated_at") or 0.0)
except Exception:
return False
if updated_at <= 0:
return False
try:
return (time.time() - updated_at) <= max(0.0, float(max_age_seconds))
except Exception:
return False
def _legacy_repl_queue_glob(root: Path) -> list[Path]:
log_dir = Path(root) / "Log"
if not log_dir.exists():
+2 -2
View File
@@ -75,8 +75,8 @@ def build_default_selection(
except Exception:
resolved_path = path_text
args = ["-path", resolved_path]
return args, ["get-file", "-path", resolved_path]
args = [resolved_path]
return args, ["download-file", resolved_path]
return hash_args, hash_action
+245 -100
View File
@@ -195,9 +195,14 @@ class Add_File(Cmdlet):
summary=
"Ingest a local media file to a configured store or plugin destination.",
usage=
"add-file (-path <filepath> | <piped>) (-instance <store-name> | -plugin <plugin> [-instance <name|path>]) [-delete]",
"add-file (<source> | <piped>) (-instance <store-name> | -plugin <plugin> [-instance <name|path>]) [-delete]",
arg=[
SharedArgs.PATH,
CmdletArg(
name="source",
type="string",
required=False,
description="Local file or directory path to ingest or scan.",
),
SharedArgs.INSTANCE,
SharedArgs.URL,
SharedArgs.PLUGIN,
@@ -218,19 +223,38 @@ class Add_File(Cmdlet):
" 0x0: Upload to 0x0.st for temporary hosting",
" file.io: Upload to file.io for temporary hosting",
" internetarchive: Upload to archive.org (optional tag: ia:<identifier> to upload into an existing item)",
"- Use -instance with -plugin to target a named provider config: add-file -plugin ftp -instance archive -path C:\\Media\\file.pdf",
"- Use a positional source path with -instance and -plugin to target a named provider config: add-file C:\\Media\\file.pdf -plugin ftp -instance archive",
],
examples=[
'download-file "https://themathesontrust.org/papers/christianity/alcock-alphabet1.pdf" | add-file -instance tutorial',
'@1 | add-file -plugin local -instance C:\\Users\\Me\\Downloads',
'add-file -plugin ftp -instance archive -path C:\\Media\\report.pdf',
'add-file C:\\Media\\report.pdf -plugin ftp -instance archive',
],
exec=self.run,
)
self.register()
@staticmethod
def _uses_legacy_path_flag(args: Sequence[str]) -> bool:
for token in args or []:
lowered = str(token or "").strip().lower()
if lowered in {"-path", "--path", "-p"}:
return True
return False
@staticmethod
def _legacy_path_flag_message() -> str:
return (
"add-file no longer supports -path. Pass the source file or directory as a positional argument, "
"and use -plugin local -instance <name|path> for local export."
)
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
"""Main execution entry point."""
if Add_File._uses_legacy_path_flag(args):
log(Add_File._legacy_path_flag_message(), file=sys.stderr)
return 1
parsed = parse_cmdlet_args(args, self)
progress = PipelineProgress(ctx)
@@ -238,7 +262,7 @@ class Add_File(Cmdlet):
deps = _CommandDependencies(config)
storage_registry = deps.get_backend_registry()
path_arg = parsed.get("path")
source_arg = parsed.get("source")
location = parsed.get("instance")
plugin_instance = parsed.get("instance")
source_url_arg = parsed.get("url")
@@ -248,19 +272,6 @@ class Add_File(Cmdlet):
if plugin_name and not plugin_instance and location:
plugin_instance = location
# Backward-compatible shorthand: when piping a file into add-file, allow
# `-path <existing dir>` to normalize into the local export plugin path.
if path_arg and not location and not plugin_name:
try:
candidate_dir = Path(str(path_arg))
if candidate_dir.exists() and candidate_dir.is_dir():
plugin_name = "local"
plugin_instance = str(candidate_dir)
local_export_destination = str(candidate_dir)
path_arg = None
except Exception:
pass
stage_ctx = ctx.get_stage_context()
is_last_stage = (stage_ctx
is None) or bool(getattr(stage_ctx,
@@ -269,24 +280,24 @@ class Add_File(Cmdlet):
has_downstream_stage = bool(stage_ctx is not None and not is_last_stage)
# Directory-mode selector:
# - Terminal use: `add-file -instance X -path <DIR>` shows a selectable table.
# - Pipelined use: `add-file -instance X -path <DIR> | ...` processes the full batch
# - Terminal use: `add-file <DIR> -instance X` shows a selectable table.
# - Pipelined use: `add-file <DIR> -instance X | ...` processes the full batch
# immediately so downstream stages receive the uploaded items.
# - Selection replay: `@N` re-runs add-file with `-path file1,file2,...`.
# - Selection replay: `@N` re-runs add-file with `file1,file2,...` as the source token.
dir_scan_mode = False
dir_scan_results: Optional[List[Dict[str, Any]]] = None
explicit_path_list_results: Optional[List[Dict[str, Any]]] = None
explicit_source_list_results: Optional[List[Dict[str, Any]]] = None
if path_arg and location and not plugin_name:
# Support comma-separated path lists: -path "file1,file2,file3"
if source_arg and location and not plugin_name:
# Support comma-separated source lists: "file1,file2,file3"
# This is the mechanism used by @N expansion for directory tables.
try:
path_text = str(path_arg)
source_text = str(source_arg)
except Exception:
path_text = ""
source_text = ""
if "," in path_text:
parts = [p.strip().strip('"') for p in path_text.split(",")]
if "," in source_text:
parts = [p.strip().strip('"') for p in source_text.split(",")]
parts = [p for p in parts if p]
batch: List[Dict[str, Any]] = []
@@ -319,13 +330,13 @@ class Add_File(Cmdlet):
)
if batch:
explicit_path_list_results = batch
# Clear path_arg so add-file doesn't treat it as a single path.
path_arg = None
explicit_source_list_results = batch
# Clear source_arg so add-file doesn't treat it as a single path.
source_arg = None
else:
# Directory scan (selector table, no ingest yet)
try:
candidate_dir = Path(str(path_arg))
candidate_dir = Path(str(source_arg))
if candidate_dir.exists() and candidate_dir.is_dir():
dir_scan_mode = True
debug(
@@ -338,12 +349,12 @@ class Add_File(Cmdlet):
debug(
f"[add-file] Found {len(dir_scan_results)} supported files in directory"
)
# Clear path_arg so it doesn't trigger single-item mode.
path_arg = None
# Clear source_arg so it doesn't trigger single-item mode.
source_arg = None
except Exception as exc:
debug(f"[add-file] Directory scan failed: {exc}")
if result is None and not path_arg and not explicit_path_list_results and not dir_scan_results:
if result is None and not source_arg and not explicit_source_list_results and not dir_scan_results:
try:
if ctx.get_stage_context() is not None:
return 0
@@ -414,15 +425,15 @@ class Add_File(Cmdlet):
# Decide which items to process.
# - If directory scan was performed, use those results
# - If user provided -path (and it was not reinterpreted as destination), treat this invocation as single-item.
# - If user provided a positional source path, treat this invocation as single-item.
# - Otherwise, if piped input is a list, ingest each item.
if explicit_path_list_results:
items_to_process = explicit_path_list_results
debug(f"[add-file] Using {len(items_to_process)} files from -path list")
if explicit_source_list_results:
items_to_process = explicit_source_list_results
debug(f"[add-file] Using {len(items_to_process)} files from source list")
elif dir_scan_results:
items_to_process = dir_scan_results
debug(f"[add-file] Using {len(items_to_process)} files from directory scan")
elif path_arg:
elif source_arg:
items_to_process: List[Any] = [result]
elif isinstance(result, list) and result:
items_to_process = list(result)
@@ -472,26 +483,21 @@ class Add_File(Cmdlet):
)
# If this invocation was terminal directory selector mode, show a selectable table and stop.
# The user then runs @N (optionally piped), which replays add-file with selected paths.
# The user then runs @N (optionally piped), which replays add-file with selected source paths.
if should_present_directory_selector:
try:
from SYS.result_table import Table
from pathlib import Path as _Path
# Build base args to replay: keep everything except the directory -path.
base_args: List[str] = []
skip_next = False
for tok in list(args or []):
if skip_next:
skip_next = False
continue
t = str(tok)
if t in {"-path",
"--path",
"-p"}:
skip_next = True
continue
base_args.append(t)
if plugin_name:
base_args.extend(["-plugin", str(plugin_name)])
if location:
base_args.extend(["-instance", str(location)])
if source_url_arg:
base_args.extend(["-url", str(source_url_arg)])
if bool(delete_after):
base_args.append("-delete")
table = Table(title="Files in Directory", preserve_order=True)
table.set_table("add-file.directory")
@@ -517,7 +523,7 @@ class Add_File(Cmdlet):
("Size", size),
("Ext", ext),
],
selection_args=["-path", str(p) if p is not None else ""],
selection_args=[str(p) if p is not None else ""],
path=str(p) if p is not None else "",
hash=hp,
)
@@ -631,7 +637,7 @@ class Add_File(Cmdlet):
)
media_path, file_hash, temp_dir_to_cleanup = self._resolve_source(
item,
path_arg,
source_arg,
pipe_obj,
config,
export_destination=export_destination,
@@ -649,8 +655,8 @@ class Add_File(Cmdlet):
# Update pipe_obj with resolved path
pipe_obj.path = str(media_path)
# When using -path (filesystem export), allow all file types.
# When using -instance (backend), restrict to SUPPORTED_MEDIA_EXTENSIONS.
# Local/plugin exports can accept any file type.
# Storage backends stay restricted to SUPPORTED_MEDIA_EXTENSIONS.
allow_all_files = not bool(effective_storage_backend_name)
if not self._validate_source(media_path, allow_all_extensions=allow_all_files):
failures += 1
@@ -780,30 +786,7 @@ class Add_File(Cmdlet):
# Stop the live pipeline progress UI before rendering the details panels.
# This prevents the progress display from lingering on screen.
try:
live_progress = ctx.get_live_progress()
except Exception:
live_progress = None
if live_progress is not None:
try:
stage_ctx = ctx.get_stage_context()
pipe_idx = getattr(stage_ctx, "pipe_index", None)
if isinstance(pipe_idx, int):
live_progress.finish_pipe(
int(pipe_idx),
force_complete=True
)
except Exception:
pass
try:
live_progress.stop()
except Exception:
pass
try:
if hasattr(ctx, "set_live_progress"):
ctx.set_live_progress(None)
except Exception:
pass
Add_File._stop_live_progress_for_terminal_render()
subject = collected_payloads[0] if len(collected_payloads) == 1 else collected_payloads
# Use helper to display items and make them @-selectable
@@ -1108,6 +1091,8 @@ class Add_File(Cmdlet):
backend: Any,
file_hash: str,
pipe_obj: models.PipeObject,
*,
output_dir: Optional[Path] = None,
) -> Tuple[Optional[Path], Optional[Path]]:
"""Best-effort fetch of a backend file when get_file returns a URL.
@@ -1133,30 +1118,68 @@ class Add_File(Cmdlet):
metadata = getattr(pipe_obj, "metadata", {})
if isinstance(metadata, dict):
suffix = metadata.get("ext")
tmp_dir = Path(tempfile.mkdtemp(prefix="add-file-src-"))
# Introspect downloader to pass supported args (suffix, progress_callback)
download_root = output_dir
if download_root is None:
tmp_dir = Path(tempfile.mkdtemp(prefix="add-file-src-"))
download_root = tmp_dir
if download_root is None:
return None, None
# Introspect downloader to pass supported args.
import inspect
sig = inspect.signature(downloader)
kwargs = {"temp_root": tmp_dir}
kwargs = {"temp_root": download_root}
if "suffix" in sig.parameters:
kwargs["suffix"] = suffix
# Hook into global PipelineProgress if available
pp = PipelineProgress.get()
if pp and "progress_callback" in sig.parameters:
pipeline_progress = PipelineProgress(ctx)
transfer_label = "peer transfer"
try:
transfer_label = str(getattr(pipe_obj, "title", "") or "").strip() or transfer_label
except Exception:
transfer_label = "peer transfer"
if "pipeline_progress" in sig.parameters:
kwargs["pipeline_progress"] = pipeline_progress
if "transfer_label" in sig.parameters:
kwargs["transfer_label"] = transfer_label
if "progress_callback" in sig.parameters:
def _cb(done, total):
# Show fetch progress instead of just 'resolving'
pp.update(downloaded=done, total=total, label="peer transfer")
try:
total_val = int(total) if total is not None else None
except Exception:
total_val = None
try:
if int(done or 0) <= 0:
pipeline_progress.begin_transfer(
label=transfer_label,
total=total_val,
)
except Exception:
pass
try:
pipeline_progress.update_transfer(
label=transfer_label,
completed=int(done or 0),
total=total_val,
)
except Exception:
pass
kwargs["progress_callback"] = _cb
downloaded = downloader(str(file_hash), **kwargs)
if isinstance(downloaded, Path) and downloaded.exists():
if output_dir is not None:
pipe_obj.is_temp = False
if isinstance(pipe_obj.extra, dict):
pipe_obj.extra["_direct_export_download"] = True
else:
pipe_obj.extra = {"_direct_export_download": True}
return downloaded, None
pipe_obj.is_temp = True
return downloaded, tmp_dir
except Exception:
@@ -1208,6 +1231,11 @@ class Add_File(Cmdlet):
source_url=url_text,
)
pipeline_progress = PipelineProgress(ctx)
try:
destination_label = str(download_root) if download_root is not None else "temporary workspace"
pipeline_progress.set_status(f"downloading {suggested_name} to {destination_label}")
except Exception:
pass
downloaded = _download_direct_file(
url_text,
@@ -1230,6 +1258,11 @@ class Add_File(Cmdlet):
return downloaded_path, tmp_dir
except Exception:
pass
finally:
try:
PipelineProgress(ctx).clear_status()
except Exception:
pass
if tmp_dir is not None:
try:
@@ -1319,7 +1352,7 @@ class Add_File(Cmdlet):
@staticmethod
def _resolve_source(
result: Any,
path_arg: Optional[str],
source_arg: Optional[str],
pipe_obj: models.PipeObject,
config: Dict[str,
Any],
@@ -1329,7 +1362,7 @@ class Add_File(Cmdlet):
) -> Tuple[Optional[Path],
Optional[str],
Optional[Path]]:
"""Resolve the source file path from args or pipeline result.
"""Resolve the source file path from the positional source arg or pipeline result.
Returns (media_path, file_hash, temp_dir_to_cleanup).
"""
@@ -1374,7 +1407,10 @@ class Add_File(Cmdlet):
return mp_path, str(r_hash), None
dl_path, tmp_dir = Add_File._maybe_download_backend_file(
backend, str(r_hash), pipe_obj
backend,
str(r_hash),
pipe_obj,
output_dir=export_destination,
)
if dl_path and dl_path.exists():
pipe_obj.path = str(dl_path)
@@ -1395,8 +1431,8 @@ class Add_File(Cmdlet):
# PRIORITY 2: Generic Coercion (Path arg > PipeObject > Result)
candidate: Optional[Path] = None
if path_arg:
candidate = Path(path_arg)
if source_arg:
candidate = Path(source_arg)
elif pipe_obj.path:
candidate = Path(pipe_obj.path)
@@ -1471,6 +1507,83 @@ class Add_File(Cmdlet):
normalized = normalized.split(".", 1)[0]
return normalized
@staticmethod
def validate_preflight_args(
args: Sequence[str],
config: Optional[Dict[str, Any]] = None,
) -> Optional[str]:
cfg = config if isinstance(config, dict) else {}
if Add_File._uses_legacy_path_flag(args):
return f"Pipeline error: {Add_File._legacy_path_flag_message()}"
try:
parsed = parse_cmdlet_args(args, CMDLET)
except Exception as exc:
return f"Pipeline error: invalid add-file arguments: {exc}"
deps = _CommandDependencies(cfg)
storage_registry = deps.get_backend_registry()
location = parsed.get("instance")
plugin_instance = parsed.get("instance")
plugin_name = parsed.get("plugin")
is_storage_backend_location = False
if location:
try:
backend_registry_for_lookup = storage_registry or deps.get_backend_registry()
is_storage_backend_location = Add_File._resolve_backend_by_name(
backend_registry_for_lookup,
str(location),
) is not None
except Exception:
is_storage_backend_location = False
if location and not plugin_name and not is_storage_backend_location:
resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target(
location,
cfg,
deps=deps,
require_explicit=True,
)
if resolved_local_path:
return None
return (
f"Pipeline error: storage backend '{location}' not found. "
"Use -plugin local -instance <name|path> for local export or configure that store backend."
)
normalized_plugin_name = Add_File._normalize_provider_key(plugin_name)
if normalized_plugin_name:
upload_plugin = deps.get_plugin_with_capability(normalized_plugin_name, "upload")
if upload_plugin is None:
plugin_exists = deps.get_plugin(normalized_plugin_name) is not None
if plugin_exists:
if normalized_plugin_name == "loc":
return (
"Pipeline error: plugin 'loc' does not support add-file/upload. "
"Use -plugin local -instance <name|path> for local export."
)
return f"Pipeline error: plugin '{normalized_plugin_name}' does not support add-file/upload."
return f"Pipeline error: unknown upload plugin '{plugin_name}'."
if normalized_plugin_name == "local":
requested_local = str(plugin_instance or location or "").strip() or "<default>"
resolved_local_instance, resolved_local_path = Add_File._resolve_local_export_plugin_target(
plugin_instance or location,
cfg,
deps=deps,
require_explicit=bool(plugin_instance or location),
)
if not resolved_local_path:
return (
f"Pipeline error: local destination '{requested_local}' is not configured. "
"Use -plugin local -instance <name|path>."
)
return None
@staticmethod
def _resolve_plugin_storage_backend(
plugin_name: Optional[Any],
@@ -1730,8 +1843,8 @@ class Add_File(Cmdlet):
Args:
media_path: Path to the file to validate
allow_all_extensions: If True, skip file type filtering (used for -path exports).
If False, only allow SUPPORTED_MEDIA_EXTENSIONS (used for -instance).
allow_all_extensions: If True, skip file type filtering for non-backend exports.
If False, only allow SUPPORTED_MEDIA_EXTENSIONS for backend ingest.
"""
if media_path is None:
return False
@@ -1740,7 +1853,7 @@ class Add_File(Cmdlet):
log(f"File not found: {media_path}")
return False
# Validate file type: only when adding to -instance backend, not for -path exports
# Validate file type only when ingesting into a storage backend.
if not allow_all_extensions:
file_extension = media_path.suffix.lower()
if file_extension not in SUPPORTED_MEDIA_EXTENSIONS:
@@ -1947,12 +2060,42 @@ class Add_File(Cmdlet):
return
try:
Add_File._stop_live_progress_for_terminal_render()
from .._shared import display_and_persist_items
display_and_persist_items([payload], title="Result", subject=payload)
except Exception:
pass
@staticmethod
def _stop_live_progress_for_terminal_render() -> None:
try:
live_progress = ctx.get_live_progress()
except Exception:
live_progress = None
if live_progress is None:
return
try:
stage_ctx = ctx.get_stage_context()
pipe_idx = getattr(stage_ctx, "pipe_index", None)
if isinstance(pipe_idx, int):
live_progress.finish_pipe(int(pipe_idx), force_complete=True)
except Exception:
pass
try:
live_progress.stop()
except Exception:
pass
try:
if hasattr(ctx, "set_live_progress"):
ctx.set_live_progress(None)
except Exception:
pass
@staticmethod
def _emit_storage_result(
payload: Dict[str,
@@ -2362,6 +2505,7 @@ class Add_File(Cmdlet):
"pipe_obj": pipe_obj,
"instance": instance_name,
}
pipeline_progress = PipelineProgress(ctx)
normalized_plugin_name = Add_File._normalize_provider_key(plugin_name)
f_hash = Add_File._resolve_file_hash(None, media_path, pipe_obj, None)
if normalized_plugin_name == "local":
@@ -2383,6 +2527,7 @@ class Add_File(Cmdlet):
"hash_value": f_hash,
"relationships": relationships,
"direct_export_download": direct_export_download,
"pipeline_progress": pipeline_progress,
}
)
@@ -2705,7 +2850,7 @@ class Add_File(Cmdlet):
)
# Emit a search-file-like payload for consistent tables and natural piping.
# Keep hash/store for downstream commands (get-tag, get-file, etc.).
# Keep hash/store for downstream commands (get-tag, download-file, etc.).
resolved_hash = chosen_hash
if prefer_defer_tags and tags:
+317 -3
View File
@@ -15,6 +15,8 @@ from pathlib import Path
from typing import Any, Dict, List, Optional, Sequence
from urllib.parse import urlparse
from contextlib import AbstractContextManager, nullcontext
import shutil
import webbrowser
from API.HTTP import _download_direct_file
@@ -25,6 +27,7 @@ from SYS.pipeline_progress import PipelineProgress
from SYS.result_table import Table, build_display_row
from SYS.rich_display import stderr_console as get_stderr_console
from SYS import pipeline as pipeline_context
from SYS.item_accessors import get_result_title
from rich.prompt import Prompt
# SYS.metadata import deferred: normalize_urls loaded lazily at call site to avoid
# pulling in Cryptodome (~900ms) at module import time.
@@ -64,7 +67,7 @@ class Download_File(Cmdlet):
name="download-file",
summary="Download files or streaming media",
usage=
"download-file <url> [-plugin NAME] [-instance NAME] [-path DIR] [options] OR @N | download-file [-plugin NAME] [-instance NAME] [-path DIR] [options]",
"download-file <url|path> [-plugin NAME] [-instance NAME] [-path DIR] [options] OR @N | download-file [-plugin NAME] [-instance NAME] [-path DIR] [options] OR download-file -query \"hash:<sha256>\" -instance <store> [-browser]",
alias=["dl-file",
"download-http"],
arg=[
@@ -73,6 +76,16 @@ class Download_File(Cmdlet):
SharedArgs.INSTANCE,
SharedArgs.PATH,
SharedArgs.QUERY,
CmdletArg(
name="name",
type="string",
description="Output filename override for store exports.",
),
CmdletArg(
name="browser",
type="flag",
description="Open a backend-provided browser URL instead of exporting to disk when available.",
),
QueryArg(
"clip",
key="clip",
@@ -95,6 +108,7 @@ class Download_File(Cmdlet):
],
detail=[
"Download files directly via HTTP or streaming media via yt-dlp.",
"Also exports store-backed files via hash+store selection or -query \"hash:<sha256>\" -instance <store>.",
"Use -plugin with -instance to target a named provider config when a plugin exposes multiple instances.",
"For Internet Archive item pages (archive.org/details/...), shows a selectable file/format list; pick with @N to download.",
],
@@ -924,6 +938,283 @@ class Download_File(Cmdlet):
pipeline_context.emit(payload)
@staticmethod
def _path_looks_local(value: Any) -> bool:
text = str(value or "").strip()
if not text:
return False
if text.startswith(("http://", "https://", "ftp://", "ftps://", "magnet:", "torrent:")):
return False
if len(text) >= 2 and text[1] == ":":
return True
if text.startswith(("\\", "/", ".", "~")):
return True
return Path(text).exists()
@staticmethod
def _resolve_display_title(result: Any, metadata: Optional[Dict[str, Any]]) -> str:
candidates = [
get_result_title(result, "title", "name", "filename"),
get_result_title(metadata or {}, "title", "name", "filename"),
]
for candidate in candidates:
if candidate is None:
continue
text = str(candidate).strip()
if text:
return text
return ""
@staticmethod
def _sanitize_export_filename(name: str) -> str:
allowed_chars: List[str] = []
for ch in str(name or ""):
if ch.isalnum() or ch in {"-", "_", " ", "."}:
allowed_chars.append(ch)
else:
allowed_chars.append(" ")
sanitized = " ".join("".join(allowed_chars).split())
return sanitized or "export"
@staticmethod
def _unique_export_path(path: Path) -> Path:
if not path.exists():
return path
stem = path.stem
suffix = path.suffix
parent = path.parent
counter = 1
while True:
candidate = parent / f"{stem} ({counter}){suffix}"
if not candidate.exists():
return candidate
counter += 1
@staticmethod
def _iter_storage_export_refs(
parsed: Dict[str, Any],
piped_items: Sequence[Any],
) -> tuple[List[Dict[str, Any]], List[Any], Optional[int]]:
refs: List[Dict[str, Any]] = []
residual_items: List[Any] = []
query_text = str(parsed.get("query") or "").strip()
query_hash: Optional[str] = None
if query_text:
query_hash = sh.parse_single_hash_query(query_text)
if query_text.lower().startswith("hash") and not query_hash:
log('Error: -query must be of the form hash:<sha256>', file=sys.stderr)
return [], list(piped_items or []), 1
explicit_store = str(parsed.get("instance") or "").strip()
if query_hash:
if not explicit_store:
log('Error: No store name provided', file=sys.stderr)
return [], list(piped_items or []), 1
refs.append(
{
"hash": query_hash,
"store": explicit_store,
"result": None,
}
)
for item in piped_items or []:
normalized_hash = sh.normalize_hash(
str(get_field(item, "hash") or get_field(item, "file_hash") or get_field(item, "hash_hex") or "")
)
store_name = str(parsed.get("instance") or get_field(item, "store") or "").strip()
if normalized_hash and store_name:
refs.append(
{
"hash": normalized_hash,
"store": store_name,
"result": item,
}
)
else:
residual_items.append(item)
return refs, residual_items, None
def _export_store_file(
self,
*,
file_hash: str,
store_name: str,
result: Any,
parsed: Dict[str, Any],
config: Dict[str, Any],
final_output_dir: Path,
) -> int:
output_path = parsed.get("path")
explicit_output_requested = bool(output_path)
output_name = parsed.get("name")
browser_flag = bool(parsed.get("browser"))
backend, _store_registry, _exc = sh.get_preferred_store_backend(
config,
store_name,
suppress_debug=True,
)
if backend is None:
log(f"Error: Storage backend '{store_name}' not found", file=sys.stderr)
return 1
metadata = backend.get_metadata(file_hash)
if not metadata:
log(f"Error: File metadata not found for hash {file_hash}", file=sys.stderr)
return 1
try:
debug_panel(
"download-file store export",
[
("hash", file_hash),
("instance", store_name),
("output_path", output_path or "<default>"),
("output_name", output_name or "<auto>"),
("browser", browser_flag),
],
border_style="blue",
)
except Exception:
pass
want_url = browser_flag
source_path = backend.get_file(file_hash, url=want_url)
download_url = None
if isinstance(source_path, str):
if source_path.startswith(("http://", "https://")):
download_url = source_path
else:
source_path = Path(source_path)
if download_url and (browser_flag or not explicit_output_requested):
try:
webbrowser.open(download_url)
except Exception as exc:
log(f"Error opening browser: {exc}", file=sys.stderr)
return 1
pipeline_context.emit(
build_file_result_payload(
title=self._resolve_display_title(result, metadata) or "Opened",
hash_value=file_hash,
store=store_name,
url=download_url,
)
)
return 0
if download_url is None:
if not source_path or not Path(source_path).exists():
log(f"Error: Backend could not retrieve file for hash {file_hash}", file=sys.stderr)
return 1
filename = str(output_name or "").strip()
if not filename:
title = (metadata.get("title") if isinstance(metadata, dict) else None) or self._resolve_display_title(result, metadata) or "export"
filename = self._sanitize_export_filename(str(title))
ext = metadata.get("ext") if isinstance(metadata, dict) else None
if ext and not filename.endswith(str(ext)):
ext_text = str(ext)
if not ext_text.startswith("."):
ext_text = "." + ext_text
filename += ext_text
if download_url:
result_obj = _download_direct_file(
download_url,
final_output_dir,
quiet=True,
suggested_filename=filename,
pipeline_progress=config.get("_pipeline_progress") if isinstance(config, dict) else None,
)
dest_path = self._path_from_download_result(result_obj)
else:
dest_path = self._unique_export_path(final_output_dir / filename)
shutil.copy2(Path(source_path), dest_path)
pipeline_context.emit(
build_file_result_payload(
title=filename,
hash_value=file_hash,
store=store_name,
path=str(dest_path),
)
)
return 0
def _process_storage_items(
self,
*,
piped_items: Sequence[Any],
parsed: Dict[str, Any],
config: Dict[str, Any],
final_output_dir: Path,
) -> tuple[int, List[Any], Optional[int]]:
refs, residual_items, early_exit = self._iter_storage_export_refs(parsed, piped_items)
if early_exit is not None:
return 0, list(residual_items), early_exit
if not refs:
return 0, list(residual_items), None
successes = 0
for ref in refs:
exit_code = self._export_store_file(
file_hash=str(ref.get("hash") or ""),
store_name=str(ref.get("store") or ""),
result=ref.get("result"),
parsed=parsed,
config=config,
final_output_dir=final_output_dir,
)
if exit_code != 0:
return successes, list(residual_items), exit_code
successes += 1
return successes, list(residual_items), None
def _process_explicit_local_sources(
self,
*,
local_sources: Sequence[str],
final_output_dir: Path,
parsed: Dict[str, Any],
progress: PipelineProgress,
config: Dict[str, Any],
) -> int:
explicit_output_requested = bool(parsed.get("path"))
downloaded_count = 0
for raw_source in local_sources or []:
source_path = Path(str(raw_source or "")).expanduser()
if not source_path.exists() or not source_path.is_file():
log(f"File not found: {source_path}", file=sys.stderr)
continue
if explicit_output_requested:
destination = final_output_dir / source_path.name
destination = self._unique_export_path(destination)
shutil.copy2(source_path, destination)
emit_path = destination
else:
emit_path = source_path
self._emit_local_file(
downloaded_path=emit_path,
source=str(source_path),
title_hint=emit_path.stem,
tags_hint=None,
media_kind_hint="file",
full_metadata=None,
progress=progress,
config=config,
)
downloaded_count += 1
return downloaded_count
def _maybe_render_download_details(self, *, config: Dict[str, Any]) -> None:
try:
stage_ctx = pipeline_context.get_stage_context()
@@ -2377,6 +2668,7 @@ class Download_File(Cmdlet):
parsed = parse_cmdlet_args(args, self)
registry = self._load_provider_registry()
selection_url_prefixes = self._selection_url_prefixes(registry)
explicit_input = parsed.get("url")
# Resolve URLs from -url or positional arguments
url_candidates = parsed.get("url") or [
@@ -2389,6 +2681,9 @@ class Download_File(Cmdlet):
]
from SYS.metadata import normalize_urls as normalize_url_list # lazy: avoids Cryptodome at startup
raw_url = normalize_url_list(url_candidates)
local_source_inputs: List[str] = []
if not raw_url and isinstance(explicit_input, str) and self._path_looks_local(explicit_input):
local_source_inputs = [str(explicit_input)]
quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False
@@ -2608,7 +2903,7 @@ class Download_File(Cmdlet):
raw_url = []
piped_items = self._collect_piped_items_if_no_urls(result, raw_url)
if not raw_url and not piped_items:
if not raw_url and not piped_items and not local_source_inputs:
log("No url or piped items to download", file=sys.stderr)
return 1
@@ -2673,8 +2968,27 @@ class Download_File(Cmdlet):
downloaded_count = 0
if local_source_inputs:
downloaded_count += self._process_explicit_local_sources(
local_sources=local_source_inputs,
final_output_dir=final_output_dir,
parsed=parsed,
progress=progress,
config=config,
)
storage_downloaded, piped_items, storage_exit = self._process_storage_items(
piped_items=piped_items,
parsed=parsed,
config=config,
final_output_dir=final_output_dir,
)
downloaded_count += int(storage_downloaded)
if storage_exit is not None:
return int(storage_exit)
if skipped_dupe_count and not raw_url and not piped_items:
return 0
return 0 if downloaded_count > 0 else 0
urls_downloaded, early_exit = self._process_explicit_urls(
raw_urls=raw_url,
-497
View File
@@ -1,497 +0,0 @@
from __future__ import annotations
from typing import Any, Dict, Sequence
from pathlib import Path
import os
import sys
import shutil
import subprocess
import tempfile
import threading
import time
import http.server
from urllib.parse import quote
import webbrowser
from urllib.parse import urljoin
from urllib.request import pathname2url
from SYS import pipeline as ctx
from .. import _shared as sh
from SYS.item_accessors import get_result_title
from SYS.logger import log, debug, debug_panel
from SYS.config import resolve_output_dir
from API.HTTP import _download_direct_file
from SYS.payload_builders import build_file_result_payload
class Get_File(sh.Cmdlet):
"""Export files to local path via hash+store."""
def __init__(self) -> None:
"""Initialize get-file cmdlet."""
super().__init__(
name="get-file",
summary="Export file to local path",
usage="@1 | get-file -path ./output",
arg=[
sh.SharedArgs.QUERY,
sh.SharedArgs.INSTANCE,
sh.SharedArgs.PATH,
sh.CmdletArg(
"name",
description="Output filename (default: from metadata title)"
),
sh.CmdletArg(
"browser",
flag=True,
description="Open file in browser instead of saving to disk"
),
],
detail=[
"- Exports file from storage backend to local path",
'- Uses selected item\'s hash, or -query "hash:<sha256>"',
"- Preserves file extension and metadata",
],
exec=self.run,
)
self.register()
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
"""Export file via hash+store backend."""
parsed = sh.parse_cmdlet_args(args, self)
try:
debug_panel(
"get-file",
[
("result_type", type(result).__name__),
("parsed_args", parsed),
],
border_style="cyan",
)
except Exception:
pass
query_hash, query_valid = sh.require_single_hash_query(
parsed.get("query"),
"Error: -query must be of the form hash:<sha256>",
)
if not query_valid:
return 1
# Extract hash and store from result or args
file_hash = query_hash or sh.get_field(result, "hash")
store_name = parsed.get("instance") or sh.get_field(result, "store")
output_path = parsed.get("path")
output_name = parsed.get("name")
browser_flag = bool(parsed.get("browser"))
if not file_hash:
log(
'Error: No file hash provided (pipe an item or use -query "hash:<sha256>")'
)
return 1
if not store_name:
log("Error: No store name provided")
return 1
# Normalize hash
file_hash = sh.normalize_hash(file_hash)
if not file_hash:
log("Error: Invalid hash format")
return 1
try:
debug_panel(
"get-file selection",
[
("hash", file_hash),
("instance", store_name),
("output_path", output_path or "<default>"),
("output_name", output_name or "<auto>"),
],
border_style="blue",
)
except Exception:
pass
backend, _store_registry, _exc = sh.get_preferred_store_backend(
config,
store_name,
suppress_debug=True,
)
if backend is None:
log(f"Error: Storage backend '{store_name}' not found", file=sys.stderr)
return 1
# Get file metadata to determine name and extension
metadata = backend.get_metadata(file_hash)
if not metadata:
log(f"Error: File metadata not found for hash {file_hash}")
return 1
try:
debug_panel(
"get-file backend",
[
("backend", type(backend).__name__),
("title", metadata.get("title") or ""),
("ext", metadata.get("ext") or ""),
],
border_style="green",
)
except Exception:
pass
def resolve_display_title() -> str:
candidates = [
get_result_title(result, "title", "name", "filename"),
get_result_title(metadata, "title", "name", "filename"),
]
for candidate in candidates:
if candidate is None:
continue
text = str(candidate).strip()
if text:
return text
return ""
# Get file from backend (may return Path or URL string depending on backend).
# If -browser is given, request a URL (for Hydrus viewer). If -path is given,
# always retrieve a local file. Otherwise default to local export.
want_url = browser_flag
source_path = backend.get_file(file_hash, url=want_url)
download_url = None
if isinstance(source_path, str):
if source_path.startswith("http://") or source_path.startswith("https://"):
download_url = source_path
else:
source_path = Path(source_path)
try:
debug_panel(
"get-file fetch",
[
("url_hint", want_url),
("mode", "browser-url" if download_url else "local-path"),
("source", download_url or source_path or "<missing>"),
],
border_style="magenta",
)
except Exception:
pass
if download_url and (browser_flag or output_path is None):
# Open in browser: explicit -browser flag, or Hydrus returned a URL with no output path
try:
webbrowser.open(download_url)
except Exception as exc:
log(f"Error opening browser: {exc}", file=sys.stderr)
else:
try:
debug_panel(
"get-file open",
[
("action", "browser-open"),
("url", download_url),
],
file=sys.stderr,
border_style="green",
)
except Exception:
pass
ctx.emit(
build_file_result_payload(
title=resolve_display_title() or "Opened",
hash_value=file_hash,
store=store_name,
url=download_url,
)
)
return 0
if download_url is None:
if not source_path or not source_path.exists():
log(f"Error: Backend could not retrieve file for hash {file_hash}")
return 1
# Otherwise: export/copy to output_dir.
if output_path:
output_dir = Path(output_path).expanduser()
else:
output_dir = resolve_output_dir(config)
output_dir.mkdir(parents=True, exist_ok=True)
# Determine output filename (only when exporting)
if output_name:
filename = output_name
else:
title = (
(metadata.get("title") if isinstance(metadata,
dict) else None)
or resolve_display_title() or "export"
)
filename = self._sanitize_filename(title)
# Add extension if metadata has it
ext = metadata.get("ext")
if ext and not filename.endswith(ext):
if not ext.startswith("."):
ext = "." + ext
filename += ext
dest_path: Path
if download_url:
downloaded = _download_direct_file(
download_url,
output_dir,
quiet=True,
suggested_filename=filename,
)
dest_path = downloaded.path
else:
dest_path = self._unique_path(output_dir / filename)
# Copy file to destination
shutil.copy2(source_path, dest_path)
try:
debug_panel(
"get-file export",
[
("mode", "download" if download_url else "copy"),
("destination", dest_path),
("filename", filename),
],
file=sys.stderr,
border_style="green",
)
except Exception:
pass
log(f"Exported: {dest_path}", file=sys.stderr)
# Emit result for pipeline
ctx.emit(
build_file_result_payload(
title=filename,
hash_value=file_hash,
store=store_name,
path=str(dest_path),
)
)
return 0
def _open_file_default(self, path: Path) -> None:
"""Open a local file in the OS default application."""
try:
suffix = str(path.suffix or "").lower()
if sys.platform.startswith("win"):
# On Windows, file associations for common media types can point at
# editors (Paint/VS Code). Prefer opening a localhost URL.
if self._open_local_file_in_browser_via_http(path):
return
if suffix in {
".png",
".jpg",
".jpeg",
".gif",
".webp",
".bmp",
".tif",
".tiff",
".svg",
}:
# Use default web browser for images.
if self._open_image_in_default_browser(path):
return
if sys.platform.startswith("win"):
os.startfile(str(path)) # type: ignore[attr-defined]
return
if sys.platform == "darwin":
subprocess.Popen(
["open",
str(path)],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
return
subprocess.Popen(
["xdg-open",
str(path)],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
except Exception as exc:
log(f"Error opening file: {exc}", file=sys.stderr)
def _open_local_file_in_browser_via_http(self, file_path: Path) -> bool:
"""Serve a single local file via localhost HTTP and open in browser.
This avoids Windows file-association issues (e.g., PNG -> Paint, HTML -> VS Code).
The server is bound to 127.0.0.1 on an ephemeral port and is shut down after
a timeout.
"""
try:
resolved = file_path.resolve()
directory = resolved.parent
filename = resolved.name
except Exception:
return False
class OneFileHandler(http.server.SimpleHTTPRequestHandler):
def __init__(self, *handler_args, **handler_kwargs):
super().__init__(
*handler_args,
directory=str(directory),
**handler_kwargs
)
def log_message(self, format: str, *args) -> None: # noqa: A003
# Keep normal output clean.
return
def do_GET(self) -> None: # noqa: N802
if self.path in {"/",
""}:
self.path = "/" + filename
return super().do_GET()
if self.path == "/" + filename or self.path == "/" + quote(filename):
return super().do_GET()
self.send_error(404)
def do_HEAD(self) -> None: # noqa: N802
if self.path in {"/",
""}:
self.path = "/" + filename
return super().do_HEAD()
if self.path == "/" + filename or self.path == "/" + quote(filename):
return super().do_HEAD()
self.send_error(404)
try:
httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), OneFileHandler)
except Exception:
return False
port = httpd.server_address[1]
url = f"http://127.0.0.1:{port}/{quote(filename)}"
# Run server in the background.
server_thread = threading.Thread(
target=httpd.serve_forever,
kwargs={
"poll_interval": 0.2
},
daemon=True
)
server_thread.start()
# Auto-shutdown after a timeout to avoid lingering servers.
def shutdown_later() -> None:
time.sleep(10 * 60)
try:
httpd.shutdown()
except Exception:
pass
try:
httpd.server_close()
except Exception:
pass
threading.Thread(target=shutdown_later, daemon=True).start()
try:
debug(f"[get-file] Opening via localhost: {url}")
return bool(webbrowser.open(url))
except Exception:
return False
def _open_image_in_default_browser(self, image_path: Path) -> bool:
"""Open an image file in the user's default web browser.
We intentionally avoid opening the image path directly on Windows because
file associations may point to editors/viewers (e.g., Paint). Instead we
generate a tiny HTML wrapper and open that (HTML is typically associated
with the default browser).
"""
try:
resolved = image_path.resolve()
image_url = urljoin("file:", pathname2url(str(resolved)))
except Exception:
return False
# Create a stable wrapper filename to reduce temp-file spam.
wrapper_path = Path(
tempfile.gettempdir()
) / f"medeia-open-image-{resolved.stem}.html"
try:
wrapper_path.write_text(
"\n".join(
[
"<!doctype html>",
'<meta charset="utf-8">',
f"<title>{resolved.name}</title>",
"<style>html,body{margin:0;padding:0;background:#000}img{display:block;max-width:100vw;max-height:100vh;margin:auto}</style>",
f'<img src="{image_url}" alt="{resolved.name}">',
]
),
encoding="utf-8",
)
except Exception:
return False
# Prefer localhost server when possible (reliable on Windows).
if self._open_local_file_in_browser_via_http(image_path):
return True
wrapper_url = wrapper_path.as_uri()
try:
return bool(webbrowser.open(wrapper_url))
except Exception:
return False
def _sanitize_filename(self, name: str) -> str:
"""Sanitize filename by removing invalid characters."""
allowed_chars = []
for ch in str(name):
if ch.isalnum() or ch in {"-",
"_",
" ",
"."}:
allowed_chars.append(ch)
else:
allowed_chars.append(" ")
# Collapse multiple spaces
sanitized = " ".join("".join(allowed_chars).split())
return sanitized or "export"
def _unique_path(self, path: Path) -> Path:
"""Generate unique path by adding (1), (2), etc. if file exists."""
if not path.exists():
return path
stem = path.stem
suffix = path.suffix
parent = path.parent
counter = 1
while True:
new_path = parent / f"{stem} ({counter}){suffix}"
if not new_path.exists():
return new_path
counter += 1
# Instantiate and register cmdlet
Add_File_Instance = Get_File()
+54 -28
View File
@@ -193,7 +193,7 @@ class search_file(Cmdlet):
"URL search: url:* (any URL) or url:<value> (URL substring)",
"Extension search: ext:<value> (e.g., ext:png)",
"Hydrus-style extension: system:filetype = png",
"Results include hash for downstream commands (get-file, add-tag, etc.)",
"Results include hash for downstream commands (download-file, add-tag, etc.)",
"Examples:",
"search-file -query foo # Search all storage backends",
"search-file -instance home -query '*' # Search 'home' Hydrus instance",
@@ -1755,6 +1755,13 @@ class search_file(Cmdlet):
f.lower()
for f in (flag_registry.get("open") or {"-open", "--open"})
}
valued_flags = (
query_flags
| instance_flags
| limit_flags
| plugin_flags
| open_flags
)
# Parse arguments
query = ""
@@ -1771,37 +1778,56 @@ class search_file(Cmdlet):
while i < len(args_list):
arg = args_list[i]
low = arg.lower()
if low in query_flags and i + 1 < len(args_list):
chunk = args_list[i + 1]
query = f"{query} {chunk}".strip() if query else chunk
i += 2
next_arg = args_list[i + 1] if i + 1 < len(args_list) else None
next_low = str(next_arg or "").lower()
next_is_flag = bool(next_arg) and next_low in valued_flags
if low in query_flags:
if next_arg is not None and not next_is_flag:
chunk = next_arg
query = f"{query} {chunk}".strip() if query else chunk
i += 2
continue
i += 1
continue
if low in plugin_flags and i + 1 < len(args_list):
plugin_name = args_list[i + 1]
i += 2
if low in plugin_flags:
if next_arg is not None and not next_is_flag:
plugin_name = next_arg
i += 2
continue
i += 1
continue
if low in instance_flags and i + 1 < len(args_list):
instance_name = args_list[i + 1]
i += 2
if low in instance_flags:
if next_arg is not None and not next_is_flag:
instance_name = next_arg
i += 2
continue
i += 1
continue
if low in open_flags and i + 1 < len(args_list):
try:
open_id = int(args_list[i + 1])
except ValueError:
log(
f"Warning: Invalid open value '{args_list[i + 1]}', ignoring",
file=sys.stderr,
)
open_id = None
i += 2
if low in open_flags:
if next_arg is not None and not next_is_flag:
try:
open_id = int(next_arg)
except ValueError:
log(
f"Warning: Invalid open value '{next_arg}', ignoring",
file=sys.stderr,
)
open_id = None
i += 2
continue
i += 1
continue
if low in limit_flags and i + 1 < len(args_list):
limit_set = True
try:
limit = int(args_list[i + 1])
except ValueError:
limit = 100
i += 2
if low in limit_flags:
if next_arg is not None and not next_is_flag:
limit_set = True
try:
limit = int(next_arg)
except ValueError:
limit = 100
i += 2
continue
i += 1
elif not arg.startswith("-"):
positional_args.append(arg)
query = f"{query} {arg}".strip() if query else arg
+8 -9
View File
@@ -13,12 +13,12 @@ SharedArgs = sh.SharedArgs
class File(Cmdlet):
"""Unified file command: file -add|-delete|-get|-merge|..."""
"""Unified file command: file -search|-add|-delete|-download|-merge|..."""
_ACTION_FLAGS = {
"search": {"-search", "--search"},
"add": {"-add", "--add"},
"delete": {"-delete", "--delete", "-del", "--del"},
"get": {"-get", "--get"},
"merge": {"-merge", "--merge"},
"download": {"-download", "--download", "-dl", "--dl"},
"convert": {"-convert", "--convert"},
@@ -30,7 +30,6 @@ class File(Cmdlet):
_ACTION_MODULE = {
"add": "cmdlet.file.add",
"delete": "cmdlet.file.delete",
"get": "cmdlet.file.get",
"merge": "cmdlet.file.merge",
"download": "cmdlet.file.download",
"search": "cmdlet.file.search",
@@ -44,15 +43,14 @@ class File(Cmdlet):
super().__init__(
name="file",
summary="Manage file operations with one command",
usage='file -query <query> [args] | file (-add|-delete|-get|-merge|-download|-convert|-trim|-archive|-screenshot) [args]',
usage='file -query <query> [args] | file (-search|-add|-delete|-merge|-download|-convert|-trim|-archive|-screenshot) [args]',
arg=[
SharedArgs.QUERY,
SharedArgs.PLUGIN,
SharedArgs.INSTANCE,
SharedArgs.PATH,
CmdletArg("-search", type="flag", required=False, description="Run search-file"),
CmdletArg("-add", type="flag", required=False, description="Run add-file"),
CmdletArg("-delete", type="flag", required=False, description="Run delete-file", alias="del"),
CmdletArg("-get", type="flag", required=False, description="Run get-file"),
CmdletArg("-merge", type="flag", required=False, description="Run merge-file"),
CmdletArg("-download", type="flag", required=False, description="Run download-file", alias="dl"),
CmdletArg("-convert", type="flag", required=False, description="Run convert-file"),
@@ -61,10 +59,11 @@ class File(Cmdlet):
CmdletArg("-screenshot", type="flag", required=False, description="Run screen-shot", alias="shot"),
],
detail=[
"- Use -query to run search-file through the unified file command.",
"- Use -search for explicit search mode, then add -plugin/-instance and -query as needed.",
"- Plain -query still routes to search-file for direct search entry.",
"- Otherwise, exactly one non-search action flag is required.",
"- Remaining args are passed through to the selected file cmdlet.",
"- Examples: file -query ..., file -add ..., file -delete ...",
"- Examples: file -search -plugin hydrusnetwork -query ..., file -add ..., file -delete ...",
],
exec=self.run,
)
@@ -139,7 +138,7 @@ class File(Cmdlet):
if action is None:
if not seen:
log(
"file: missing action; use -query for search or choose exactly one of -add, -delete, -get, -merge, -download, -convert, -trim, -archive, -screenshot",
"file: missing action; use -search/-query for search or choose exactly one of -search, -add, -delete, -merge, -download, -convert, -trim, -archive, -screenshot",
file=sys.stderr,
)
else:
+3 -3
View File
@@ -21,7 +21,7 @@ storage-style integration:
- `selector()` turns folder rows into a follow-up table when the user runs `@N`.
- `download()` and `download_url()` fetch FTP files into `download-file` output paths.
- `resolve_pipe_result_download()` lets `@N | add-file -instance ...` materialize a remote FTP file first.
- `upload()` lets `add-file -plugin ftp -instance <name> -path ...` push a local file to the configured FTP server.
- `upload()` lets `add-file <local-file> -plugin ftp -instance <name>` push a local file to the configured FTP server.
## Example config
@@ -112,7 +112,7 @@ Why this works:
Uploading uses the same plugin, through `add-file -plugin ftp -instance <name>`:
```powershell
add-file -plugin ftp -instance archive -path C:\Media\report.pdf
add-file C:\Media\report.pdf -plugin ftp -instance archive
```
That sends the file to the selected instance's FTP `base_path` and returns the
@@ -148,5 +148,5 @@ search-file -plugin ftp -instance work "path:/incoming depth:2 *.pdf"
@1
@1 | download-file -path C:\Downloads
@1 | add-file -instance tutorial
add-file -plugin ftp -instance archive -path C:\Media\report.pdf
add-file C:\Media\report.pdf -plugin ftp -instance archive
```
+3 -3
View File
@@ -15,7 +15,7 @@ The SCP plugin mirrors the FTP walkthrough, but on top of SSH:
- plain `@N` on a folder drills into that directory
- plain `@N` on a file runs `download-file -plugin scp -instance <name> -url ...`
- `@N | add-file -instance ...` downloads first, then ingests the local temp file
- `add-file -plugin scp -instance <name> -path ...` uploads a local file to the configured remote path
- `add-file <local-file> -plugin scp -instance <name>` uploads a local file to the configured remote path
## Example config
@@ -102,7 +102,7 @@ Why this works:
## Upload flow
```powershell
add-file -plugin scp -instance archive -path C:\Media\report.pdf
add-file C:\Media\report.pdf -plugin scp -instance archive
```
## Implementation notes
@@ -120,5 +120,5 @@ search-file -plugin scp -instance work "path:/srv/files depth:2 *.zip"
@1
@1 | download-file -path C:\Downloads
@1 | add-file -instance tutorial
add-file -plugin scp -instance archive -path C:\Media\report.pdf
add-file C:\Media\report.pdf -plugin scp -instance archive
```
+1 -1
View File
@@ -73,7 +73,7 @@ class FTP(Provider):
PLUGIN_NAME = "ftp"
URL = ("ftp://", "ftps://")
MULTI_INSTANCE = True
SUPPORTED_CMDLETS = frozenset({"add-file", "delete-file", "get-file", "search-file"})
SUPPORTED_CMDLETS = frozenset({"add-file", "delete-file", "download-file", "search-file"})
@property
def label(self) -> str:
+158 -73
View File
@@ -28,6 +28,54 @@ def _copy_sidecars(source_path: Path, target_path: Path) -> None:
continue
def _copy_with_progress(
source_path: Path,
target_path: Path,
*,
pipeline_progress: Any = None,
label: str = "local export",
chunk_size: int = 1024 * 1024,
) -> None:
total_bytes: Optional[int] = None
try:
total_bytes = int(source_path.stat().st_size)
except Exception:
total_bytes = None
transfer_started = False
completed = 0
transfer_label = str(label or target_path.name or source_path.name)
try:
if pipeline_progress is not None and hasattr(pipeline_progress, "begin_transfer"):
pipeline_progress.begin_transfer(
label=transfer_label,
total=total_bytes if isinstance(total_bytes, int) and total_bytes > 0 else None,
)
transfer_started = True
with source_path.open("rb") as src, target_path.open("wb") as dst:
while True:
chunk = src.read(max(4096, int(chunk_size or 0) or 1024 * 1024))
if not chunk:
break
dst.write(chunk)
completed += len(chunk)
if pipeline_progress is not None and hasattr(pipeline_progress, "update_transfer"):
pipeline_progress.update_transfer(
label=transfer_label,
completed=completed,
total=total_bytes if isinstance(total_bytes, int) and total_bytes > 0 else None,
)
shutil.copystat(str(source_path), str(target_path))
finally:
if pipeline_progress is not None and transfer_started and hasattr(pipeline_progress, "finish_transfer"):
try:
pipeline_progress.finish_transfer(label=transfer_label)
except Exception:
pass
class Local(Provider):
PLUGIN_NAME = "local"
PLUGIN_ALIASES = ("filesystem", "fs")
@@ -122,84 +170,121 @@ class Local(Provider):
if not source_path.exists() or not source_path.is_file():
raise FileNotFoundError(f"File not found: {source_path}")
requested_instance = str(kwargs.get("instance") or kwargs.get("store") or "").strip() or None
resolved_name, settings = self.resolve_destination(
requested_instance,
require_explicit=bool(requested_instance),
)
destination_text = str(settings.get("path") or "").strip()
if not destination_text:
requested_label = requested_instance or "<default>"
raise ValueError(
f"Local destination '{requested_label}' is not configured. Use -plugin local -instance <name|path>."
)
pipeline_progress = kwargs.get("pipeline_progress")
destination_root = Path(destination_text).expanduser()
create_dirs = bool(settings.get("create_dirs", True))
if create_dirs:
destination_root.mkdir(parents=True, exist_ok=True)
elif not destination_root.exists():
raise FileNotFoundError(f"Destination directory does not exist: {destination_root}")
elif not destination_root.is_dir():
raise NotADirectoryError(f"Destination is not a directory: {destination_root}")
title = str(kwargs.get("title") or "").strip()
if not title:
title = source_path.stem.replace("_", " ").strip()
base_name = sanitize_filename(title or source_path.stem)
file_ext = source_path.suffix
if file_ext and base_name.lower().endswith(file_ext.lower()):
target_name = base_name
else:
target_name = base_name + file_ext
direct_export_download = bool(kwargs.get("direct_export_download", False))
target_path = source_path if direct_export_download else destination_root / target_name
if not direct_export_download:
if target_path.exists():
target_path = unique_path(target_path)
shutil.copy2(str(source_path), target_path)
_copy_sidecars(source_path, target_path)
tags = list(kwargs.get("tags") or [])
urls = list(kwargs.get("urls") or [])
hash_value = str(kwargs.get("hash_value") or "").strip() or None
if not hash_value:
def _set_status(text: str) -> None:
if pipeline_progress is None or not hasattr(pipeline_progress, "set_status"):
return
try:
hash_value = sha256_file(target_path)
pipeline_progress.set_status(f"local: {text}")
except Exception:
hash_value = None
pass
def _clear_status() -> None:
if pipeline_progress is None or not hasattr(pipeline_progress, "clear_status"):
return
try:
pipeline_progress.clear_status()
except Exception:
pass
relationships = kwargs.get("relationships")
try:
write_tags(target_path, tags, urls, hash_value=hash_value)
write_metadata(
target_path,
hash_value=hash_value,
url=urls,
relationships=relationships or [],
requested_instance = str(kwargs.get("instance") or kwargs.get("store") or "").strip() or None
resolved_name, settings = self.resolve_destination(
requested_instance,
require_explicit=bool(requested_instance),
)
except Exception:
pass
destination_text = str(settings.get("path") or "").strip()
if not destination_text:
requested_label = requested_instance or "<default>"
raise ValueError(
f"Local destination '{requested_label}' is not configured. Use -plugin local -instance <name|path>."
)
extra_updates: Dict[str, Any] = {
"url": urls,
"export_path": str(destination_root),
}
if resolved_name:
extra_updates["instance"] = resolved_name
if relationships:
extra_updates["relationships"] = relationships
destination_root = Path(destination_text).expanduser()
create_dirs = bool(settings.get("create_dirs", True))
if create_dirs:
destination_root.mkdir(parents=True, exist_ok=True)
elif not destination_root.exists():
raise FileNotFoundError(f"Destination directory does not exist: {destination_root}")
elif not destination_root.is_dir():
raise NotADirectoryError(f"Destination is not a directory: {destination_root}")
return {
"hash": hash_value or "unknown",
"store": "local",
"provider": self.name,
"path": str(target_path),
"tag": tags,
"title": title or target_path.name,
"relationships": relationships,
"extra": extra_updates,
}
title = str(kwargs.get("title") or "").strip()
if not title:
title = source_path.stem.replace("_", " ").strip()
base_name = sanitize_filename(title or source_path.stem)
file_ext = source_path.suffix
if file_ext and base_name.lower().endswith(file_ext.lower()):
target_name = base_name
else:
target_name = base_name + file_ext
direct_export_download = bool(kwargs.get("direct_export_download", False))
target_path = source_path if direct_export_download else destination_root / target_name
if not direct_export_download:
if target_path.exists():
target_path = unique_path(target_path)
_set_status(f"copying {target_path.name}")
_copy_with_progress(
source_path,
target_path,
pipeline_progress=pipeline_progress,
label=str(target_path.name or source_path.name or "local export"),
)
_copy_sidecars(source_path, target_path)
else:
_set_status(f"finalizing {target_path.name}")
tags = list(kwargs.get("tags") or [])
urls = list(kwargs.get("urls") or [])
hash_value = str(kwargs.get("hash_value") or "").strip() or None
if not hash_value:
try:
hash_value = sha256_file(target_path)
except Exception:
hash_value = None
relationships = kwargs.get("relationships")
try:
_set_status(f"writing metadata for {target_path.name}")
write_tags(
target_path,
tags,
urls,
hash_value=hash_value,
emit_debug=False,
)
write_metadata(
target_path,
hash_value=hash_value,
url=urls,
relationships=relationships or [],
emit_debug=False,
)
except Exception:
pass
extra_updates: Dict[str, Any] = {
"url": urls,
"export_path": str(destination_root),
}
if resolved_name:
extra_updates["instance"] = resolved_name
if relationships:
extra_updates["relationships"] = relationships
return {
"hash": hash_value or "unknown",
"store": "local",
"provider": self.name,
"path": str(target_path),
"tag": tags,
"title": title or target_path.name,
"relationships": relationships,
"extra": extra_updates,
}
finally:
_clear_status()
+36 -1
View File
@@ -2658,6 +2658,41 @@ local function _queue_pipeline_in_repl(pipeline_cmd, queued_message, failure_pre
return false
end
do
local repo_root = _detect_repo_root()
local detail = 'REPL not running'
if repo_root ~= '' then
local log_dir = utils.join_path(repo_root, 'Log')
if _path_exists(log_dir) then
local state_path = utils.join_path(log_dir, 'medeia-repl-state.json')
local fh = io.open(state_path, 'r')
if fh then
local raw = fh:read('*a')
fh:close()
raw = trim(tostring(raw or ''))
if raw ~= '' then
local ok, payload = pcall(utils.parse_json, raw)
if ok and type(payload) == 'table' then
local status = trim(tostring(payload.status or 'running')):lower()
local updated_at = tonumber(payload.updated_at or 0)
local now = (os and os.time) and os.time() or nil
if status == '' or status == 'running' then
if updated_at and updated_at > 0 and now and (now - updated_at) <= 3 then
detail = ''
end
end
end
end
end
end
end
if detail ~= '' then
_lua_log(queue_label .. ': repl unavailable err=' .. detail)
mp.osd_message((failure_prefix or 'REPL queue failed') .. ': ' .. detail, 5)
return false
end
end
local queue_metadata = { kind = 'mpv-download' }
if type(metadata) == 'table' then
for key, value in pairs(metadata) do
@@ -5566,7 +5601,7 @@ local function _start_download_flow_for_current()
end
ensure_mpv_ipc_server()
local pipeline_cmd = 'file -get -store ' .. quote_pipeline_arg(store_hash.store) .. ' -query ' .. quote_pipeline_arg('hash:' .. store_hash.hash) .. ' -path ' .. quote_pipeline_arg(folder)
local pipeline_cmd = 'file -download -instance ' .. quote_pipeline_arg(store_hash.store) .. ' -query ' .. quote_pipeline_arg('hash:' .. store_hash.hash) .. ' -path ' .. quote_pipeline_arg(folder)
_queue_pipeline_in_repl(
pipeline_cmd,
'Queued in REPL: store copy',
+13 -2
View File
@@ -68,7 +68,7 @@ if _ROOT not in sys.path:
from plugins.mpv.mpv_ipc import MPVIPCClient, _windows_kill_pids, _windows_hidden_subprocess_kwargs, _windows_list_mpv_pids # noqa: E402
from SYS.config import load_config, reload_config # noqa: E402
from SYS.logger import set_debug, debug, set_thread_stream # noqa: E402
from SYS.repl_queue import enqueue_repl_command # noqa: E402
from SYS.repl_queue import enqueue_repl_command, repl_state_is_alive # noqa: E402
from SYS.utils import format_bytes # noqa: E402
from PluginCore.registry import get_plugin, get_plugin_class # noqa: E402
from tool.ytdlp import get_display_format_id, get_selection_format_id # noqa: E402
@@ -628,8 +628,19 @@ def _run_op(op: str, data: Any) -> Dict[str, Any]:
"table": None,
}
repo_root = _repo_root()
if not repl_state_is_alive(repo_root):
return {
"success": False,
"stdout": "",
"stderr": "",
"error": "REPL not running",
"table": None,
"queued": False,
}
queue_path = enqueue_repl_command(
_repo_root(),
repo_root,
command_text,
source=source,
metadata=metadata,
+1 -1
View File
@@ -122,7 +122,7 @@ Ingest a selected remote result into a configured backend:
Upload a local file through a plugin:
```powershell
add-file -plugin ftp -instance archive -path C:\Media\report.pdf
add-file C:\Media\report.pdf -plugin ftp -instance archive
```
The exact meaning of `@1` depends on the current table and plugin. For example, one row may open a nested directory table while another row may download or replay a file-specific action.