h
This commit is contained in:
@@ -1153,6 +1153,16 @@ class PipelineLiveProgress:
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
# Auto-stop Live rendering once all pipes are complete so the progress
|
||||||
|
# UI clears itself even if callers forget to stop it explicitly.
|
||||||
|
try:
|
||||||
|
if self._live is not None and self._pipe_labels:
|
||||||
|
total_pipes = len(self._pipe_labels)
|
||||||
|
if total_pipes > 0 and completed >= total_pipes:
|
||||||
|
self.stop()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
def begin_pipe_steps(self, pipe_index: int, *, total_steps: int) -> None:
|
def begin_pipe_steps(self, pipe_index: int, *, total_steps: int) -> None:
|
||||||
"""Initialize step tracking for a pipe.
|
"""Initialize step tracking for a pipe.
|
||||||
|
|
||||||
|
|||||||
@@ -2821,6 +2821,7 @@ class PipelineExecutor:
|
|||||||
|
|
||||||
pipe_idx = pipe_index_by_stage.get(stage_index)
|
pipe_idx = pipe_index_by_stage.get(stage_index)
|
||||||
|
|
||||||
|
overlay_table: Any | None = None
|
||||||
session = WorkerStages.begin_stage(
|
session = WorkerStages.begin_stage(
|
||||||
worker_manager,
|
worker_manager,
|
||||||
cmd_name=cmd_name,
|
cmd_name=cmd_name,
|
||||||
@@ -2856,24 +2857,17 @@ class PipelineExecutor:
|
|||||||
# Pipeline overlay tables (e.g., get-url detail views) need to be
|
# Pipeline overlay tables (e.g., get-url detail views) need to be
|
||||||
# rendered when running inside a pipeline because the CLI path
|
# rendered when running inside a pipeline because the CLI path
|
||||||
# normally handles rendering. The overlay is only useful when
|
# normally handles rendering. The overlay is only useful when
|
||||||
# we're at the terminal stage of the pipeline.
|
# we're at the terminal stage of the pipeline. Save the table so
|
||||||
|
# it can be printed after the pipe finishes.
|
||||||
|
overlay_table = None
|
||||||
if stage_index + 1 >= len(stages):
|
if stage_index + 1 >= len(stages):
|
||||||
display_table = None
|
|
||||||
try:
|
try:
|
||||||
display_table = (
|
overlay_table = (
|
||||||
ctx.get_display_table()
|
ctx.get_display_table()
|
||||||
if hasattr(ctx, "get_display_table") else None
|
if hasattr(ctx, "get_display_table") else None
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
display_table = None
|
overlay_table = None
|
||||||
if display_table is not None:
|
|
||||||
try:
|
|
||||||
from SYS.rich_display import stdout_console
|
|
||||||
|
|
||||||
stdout_console().print()
|
|
||||||
stdout_console().print(display_table)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
# Update piped_result for next stage from emitted items
|
# Update piped_result for next stage from emitted items
|
||||||
stage_emits = list(stage_ctx.emits)
|
stage_emits = list(stage_ctx.emits)
|
||||||
@@ -2884,6 +2878,14 @@ class PipelineExecutor:
|
|||||||
finally:
|
finally:
|
||||||
if progress_ui is not None and pipe_idx is not None:
|
if progress_ui is not None and pipe_idx is not None:
|
||||||
progress_ui.finish_pipe(pipe_idx)
|
progress_ui.finish_pipe(pipe_idx)
|
||||||
|
if overlay_table is not None:
|
||||||
|
try:
|
||||||
|
from SYS.rich_display import stdout_console
|
||||||
|
|
||||||
|
stdout_console().print()
|
||||||
|
stdout_console().print(overlay_table)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
if session:
|
if session:
|
||||||
try:
|
try:
|
||||||
session.close()
|
session.close()
|
||||||
|
|||||||
@@ -554,6 +554,14 @@ class Add_File(Cmdlet):
|
|||||||
media_path, file_hash, temp_dir_to_cleanup = self._resolve_source(
|
media_path, file_hash, temp_dir_to_cleanup = self._resolve_source(
|
||||||
item, path_arg, pipe_obj, config, store_instance=storage_registry
|
item, path_arg, pipe_obj, config, store_instance=storage_registry
|
||||||
)
|
)
|
||||||
|
if not media_path and provider_name:
|
||||||
|
media_path, file_hash, temp_dir_to_cleanup = Add_File._download_provider_source(
|
||||||
|
pipe_obj, config, storage_registry
|
||||||
|
)
|
||||||
|
if media_path:
|
||||||
|
debug(
|
||||||
|
f"[add-file] Provider source downloaded: {media_path}"
|
||||||
|
)
|
||||||
debug(
|
debug(
|
||||||
f"[add-file] RESOLVED source: path={media_path}, hash={file_hash if file_hash else 'N/A'}..."
|
f"[add-file] RESOLVED source: path={media_path}, hash={file_hash if file_hash else 'N/A'}..."
|
||||||
)
|
)
|
||||||
@@ -1071,6 +1079,28 @@ class Add_File(Cmdlet):
|
|||||||
pass
|
pass
|
||||||
return None, None
|
return None, None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _resolve_backend_by_name(store: Any, backend_name: str) -> Optional[Any]:
|
||||||
|
if not store or not backend_name:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return store[backend_name]
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
target = str(backend_name or "").strip().lower()
|
||||||
|
if not target:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
for candidate in store.list_backends():
|
||||||
|
if isinstance(candidate, str) and candidate.strip().lower() == target:
|
||||||
|
try:
|
||||||
|
return store[candidate]
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _resolve_source(
|
def _resolve_source(
|
||||||
result: Any,
|
result: Any,
|
||||||
@@ -1111,15 +1141,12 @@ class Add_File(Cmdlet):
|
|||||||
if not store:
|
if not store:
|
||||||
store = Store(config)
|
store = Store(config)
|
||||||
|
|
||||||
if r_store in store.list_backends():
|
backend = Add_File._resolve_backend_by_name(store, r_store)
|
||||||
backend = store[r_store]
|
if backend is not None:
|
||||||
# Try direct access (Path)
|
|
||||||
mp = backend.get_file(r_hash)
|
mp = backend.get_file(r_hash)
|
||||||
if isinstance(mp, Path) and mp.exists():
|
if isinstance(mp, Path) and mp.exists():
|
||||||
pipe_obj.path = str(mp)
|
pipe_obj.path = str(mp)
|
||||||
return mp, str(r_hash), None
|
return mp, str(r_hash), None
|
||||||
|
|
||||||
# Try download to temp
|
|
||||||
if isinstance(mp, str) and mp.strip():
|
if isinstance(mp, str) and mp.strip():
|
||||||
dl_path, tmp_dir = Add_File._maybe_download_backend_file(
|
dl_path, tmp_dir = Add_File._maybe_download_backend_file(
|
||||||
backend, str(r_hash), pipe_obj
|
backend, str(r_hash), pipe_obj
|
||||||
@@ -1162,6 +1189,41 @@ class Add_File(Cmdlet):
|
|||||||
log("File path could not be resolved")
|
log("File path could not be resolved")
|
||||||
return None, None, None
|
return None, None, None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _download_provider_source(
|
||||||
|
pipe_obj: models.PipeObject,
|
||||||
|
config: Dict[str, Any],
|
||||||
|
store_instance: Optional[Any],
|
||||||
|
) -> Tuple[Optional[Path], Optional[str], Optional[Path]]:
|
||||||
|
r_hash = str(getattr(pipe_obj, "hash", None) or getattr(pipe_obj, "file_hash", None) or "").strip()
|
||||||
|
r_store = str(getattr(pipe_obj, "store", None) or "").strip()
|
||||||
|
if not (r_hash and r_store):
|
||||||
|
return None, None, None
|
||||||
|
|
||||||
|
try:
|
||||||
|
store = store_instance or Store(config)
|
||||||
|
except Exception:
|
||||||
|
store = None
|
||||||
|
backend = Add_File._resolve_backend_by_name(store, r_store) if store is not None else None
|
||||||
|
if backend is None:
|
||||||
|
return None, None, None
|
||||||
|
|
||||||
|
try:
|
||||||
|
source = backend.get_file(r_hash.lower())
|
||||||
|
if isinstance(source, Path) and source.exists():
|
||||||
|
pipe_obj.path = str(source)
|
||||||
|
return source, str(r_hash), None
|
||||||
|
if isinstance(source, str) and source.strip():
|
||||||
|
dl_path, tmp_dir = Add_File._maybe_download_backend_file(
|
||||||
|
backend, str(r_hash), pipe_obj
|
||||||
|
)
|
||||||
|
if dl_path and dl_path.exists():
|
||||||
|
return dl_path, str(r_hash), tmp_dir
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return None, None, None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _scan_directory_for_files(directory: Path, compute_hash: bool = True) -> List[Dict[str, Any]]:
|
def _scan_directory_for_files(directory: Path, compute_hash: bool = True) -> List[Dict[str, Any]]:
|
||||||
"""Scan a directory for supported media files and return list of file info dicts.
|
"""Scan a directory for supported media files and return list of file info dicts.
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from . import _shared as sh
|
|||||||
from SYS.logger import log, debug
|
from SYS.logger import log, debug
|
||||||
from Store import Store
|
from Store import Store
|
||||||
from SYS.config import resolve_output_dir
|
from SYS.config import resolve_output_dir
|
||||||
|
from API.HTTP import _download_direct_file
|
||||||
|
|
||||||
|
|
||||||
class Get_File(sh.Cmdlet):
|
class Get_File(sh.Cmdlet):
|
||||||
@@ -148,36 +149,36 @@ class Get_File(sh.Cmdlet):
|
|||||||
|
|
||||||
debug(f"[get-file] backend.get_file returned: {source_path}")
|
debug(f"[get-file] backend.get_file returned: {source_path}")
|
||||||
|
|
||||||
# Check if backend returned a URL (HydrusNetwork case)
|
download_url = None
|
||||||
if isinstance(source_path,
|
if isinstance(source_path, str):
|
||||||
str) and (source_path.startswith("http://")
|
if source_path.startswith("http://") or source_path.startswith("https://"):
|
||||||
or source_path.startswith("https://")):
|
download_url = source_path
|
||||||
# Hydrus backend returns a URL; open it only for this explicit user action.
|
else:
|
||||||
|
source_path = Path(source_path)
|
||||||
|
|
||||||
|
if download_url and output_path is None:
|
||||||
|
# Hydrus backend returns a URL; open it only when no output path
|
||||||
try:
|
try:
|
||||||
webbrowser.open(source_path)
|
webbrowser.open(download_url)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log(f"Error opening browser: {exc}", file=sys.stderr)
|
log(f"Error opening browser: {exc}", file=sys.stderr)
|
||||||
else:
|
else:
|
||||||
debug(f"Opened in browser: {source_path}", file=sys.stderr)
|
debug(f"Opened in browser: {download_url}", file=sys.stderr)
|
||||||
|
|
||||||
# Emit result for pipeline
|
|
||||||
ctx.emit(
|
ctx.emit(
|
||||||
{
|
{
|
||||||
"hash": file_hash,
|
"hash": file_hash,
|
||||||
"store": store_name,
|
"store": store_name,
|
||||||
"url": source_path,
|
"url": download_url,
|
||||||
"title": resolve_display_title() or "Opened",
|
"title": resolve_display_title() or "Opened",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
# Otherwise treat as file path (local/folder backends)
|
if download_url is None:
|
||||||
if isinstance(source_path, str):
|
if not source_path or not source_path.exists():
|
||||||
source_path = Path(source_path)
|
log(f"Error: Backend could not retrieve file for hash {file_hash}")
|
||||||
|
return 1
|
||||||
if not source_path or not source_path.exists():
|
|
||||||
log(f"Error: Backend could not retrieve file for hash {file_hash}")
|
|
||||||
return 1
|
|
||||||
|
|
||||||
# Otherwise: export/copy to output_dir.
|
# Otherwise: export/copy to output_dir.
|
||||||
if output_path:
|
if output_path:
|
||||||
@@ -206,11 +207,21 @@ class Get_File(sh.Cmdlet):
|
|||||||
ext = "." + ext
|
ext = "." + ext
|
||||||
filename += ext
|
filename += ext
|
||||||
|
|
||||||
dest_path = self._unique_path(output_dir / filename)
|
dest_path: Path
|
||||||
|
if download_url:
|
||||||
# Copy file to destination
|
downloaded = _download_direct_file(
|
||||||
debug(f"[get-file] Copying {source_path} -> {dest_path}", file=sys.stderr)
|
download_url,
|
||||||
shutil.copy2(source_path, dest_path)
|
output_dir,
|
||||||
|
quiet=True,
|
||||||
|
suggested_filename=filename,
|
||||||
|
)
|
||||||
|
dest_path = downloaded.path
|
||||||
|
debug(f"[get-file] Downloaded remote file to {dest_path}", file=sys.stderr)
|
||||||
|
else:
|
||||||
|
dest_path = self._unique_path(output_dir / filename)
|
||||||
|
# Copy file to destination
|
||||||
|
debug(f"[get-file] Copying {source_path} -> {dest_path}", file=sys.stderr)
|
||||||
|
shutil.copy2(source_path, dest_path)
|
||||||
|
|
||||||
log(f"Exported: {dest_path}", file=sys.stderr)
|
log(f"Exported: {dest_path}", file=sys.stderr)
|
||||||
|
|
||||||
|
|||||||
@@ -59,7 +59,11 @@ class _WorkerLogger:
|
|||||||
|
|
||||||
def update_worker_status(self, worker_id: str, status: str) -> None:
|
def update_worker_status(self, worker_id: str, status: str) -> None:
|
||||||
try:
|
try:
|
||||||
update_worker(worker_id, status=status)
|
normalized = (status or "").lower()
|
||||||
|
kwargs: dict[str, str] = {"status": status}
|
||||||
|
if normalized in {"completed", "error", "cancelled"}:
|
||||||
|
kwargs["result"] = normalized
|
||||||
|
update_worker(worker_id, **kwargs)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user