goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

updated panel display

Browse Source
This commit is contained in:
Nose
2026-04-16 17:18:50 -07:00
parent 97e310be70
commit 343a7b37a0
14 changed files with 711 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmdlet/_shared.py
+70 -20
View File
@@ -14,7 +14,7 @@ from collections.abc import Iterable as IterableABC
from functools import lru_cache
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
from SYS.logger import log, debug
from SYS.logger import log, debug, debug_panel
from pathlib import Path
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple
from dataclasses import dataclass, field
@@ -3158,6 +3158,8 @@ def check_url_exists_in_storage(
storage: Any,
hydrus_available: bool,
final_output_dir: Optional[Path] = None,
*,
auto_continue_duplicates: bool = True,
) -> bool:
"""Pre-flight check to see if URLs already exist in storage.
@@ -3187,7 +3189,6 @@ def check_url_exists_in_storage(
in_pipeline = bool(stage_ctx is not None or ("|" in str(current_cmd_text or "")))
start_time = time.monotonic()
time_budget = 45.0
debug(f"[preflight] check_url_exists_in_storage: checking {len(urls)} url(s)")
if in_pipeline:
try:
already_checked = bool(
@@ -3243,7 +3244,7 @@ def check_url_exists_in_storage(
return False
return False
if in_pipeline:
if in_pipeline and auto_continue_duplicates:
try:
cached_cmd = pipeline_context.load_value("preflight.url_duplicates.command", default="")
cached_decision = pipeline_context.load_value("preflight.url_duplicates.continue", default=None)
@@ -3706,6 +3707,20 @@ def check_url_exists_in_storage(
debug("Bulk URL preflight skipped: no searchable backends")
return True
try:
debug_panel(
"URL preflight",
[
("url_count", len(unique_urls)),
("pipeline", in_pipeline),
("bulk_mode", bulk_mode),
("backends", ", ".join(str(name) for name in backend_names)),
],
border_style="yellow",
)
except Exception:
pass
seen_pairs: set[tuple[str, str]] = set()
matched_urls: set[str] = set()
match_rows: List[Dict[str, Any]] = []
@@ -3726,12 +3741,7 @@ def check_url_exists_in_storage(
except Exception:
continue
debug(f"[preflight] Scanning backend: {backend_name}")
if HydrusNetwork is not None and isinstance(backend, HydrusNetwork):
client = getattr(backend, "_client", None)
if client is None:
continue
if not hydrus_available:
debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup")
@@ -3748,7 +3758,33 @@ def check_url_exists_in_storage(
found_hash: Optional[str] = None
found = False
lookup_exact = getattr(backend, "find_hashes_by_url", None)
if callable(lookup_exact):
for needle in [original_url, *(needles or [])][:7]:
needle_text = str(needle or "").strip()
if not _httpish(needle_text):
continue
try:
exact_hashes = lookup_exact(needle_text) or []
except Exception:
continue
if not isinstance(exact_hashes, list) or not exact_hashes:
continue
try:
found_hash = str(exact_hashes[0] or "").strip().lower()
except Exception:
found_hash = None
found = True
break
client = getattr(backend, "_client", None)
if found:
pass
elif client is None:
continue
for needle in (needles or [])[:6]:
if found:
break
if not _httpish(needle):
continue
try:
@@ -3868,7 +3904,6 @@ def check_url_exists_in_storage(
match_rows.append(display_row)
if not match_rows:
debug("Bulk URL preflight: no matches")
if in_pipeline:
preflight_cache = _load_preflight_cache()
url_dup_cache = preflight_cache.get("url_duplicates")
@@ -3935,24 +3970,39 @@ def check_url_exists_in_storage(
auto_confirm_reason = "non-interactive stdin"
answered_yes = True
auto_declined = False
with cm:
get_stderr_console().print(table)
setattr(table, "_rendered_by_cmdlet", True)
if auto_confirm_reason is None:
answered_yes = bool(Confirm.ask("Continue anyway?", default=False, console=get_stderr_console()))
else:
debug(
f"Bulk URL preflight auto-confirmed duplicates ({auto_confirm_reason}); continuing without user input."
)
try:
log(
f"Auto-confirmed duplicate URL warning ({auto_confirm_reason}). Continuing...",
file=sys.stderr,
answered_yes = bool(auto_continue_duplicates)
auto_declined = not answered_yes
if answered_yes:
debug(
f"Bulk URL preflight auto-confirmed duplicates ({auto_confirm_reason}); continuing without user input."
)
except Exception:
pass
try:
log(
f"Auto-confirmed duplicate URL warning ({auto_confirm_reason}). Continuing...",
file=sys.stderr,
)
except Exception:
pass
else:
debug(
f"Bulk URL preflight auto-skipped duplicates ({auto_confirm_reason}); skipping without user input."
)
try:
log(
f"Duplicate URL detected ({auto_confirm_reason}). Skipping download.",
file=sys.stderr,
)
except Exception:
pass
if in_pipeline:
if in_pipeline and auto_continue_duplicates:
try:
existing = pipeline_context.load_value("preflight", default=None)
except Exception:
@@ -3977,7 +4027,7 @@ def check_url_exists_in_storage(
pass
if not answered_yes:
if in_pipeline:
if in_pipeline and not auto_declined:
try:
pipeline_context.request_pipeline_stop(reason="duplicate-url declined", exit_code=0)
except Exception:
cmdlet/add_file.py
+68 -32
View File
@@ -11,7 +11,7 @@ from urllib.parse import urlparse
from SYS import models
from SYS import pipeline as ctx
from SYS.logger import log, debug, is_debug_enabled
from SYS.logger import log, debug, debug_panel, is_debug_enabled
from SYS.payload_builders import build_table_result_payload
from SYS.pipeline_progress import PipelineProgress
from SYS.result_publication import overlay_existing_result_table, publish_result_table
@@ -247,11 +247,13 @@ class Add_File(Cmdlet):
is None) or bool(getattr(stage_ctx,
"is_last_stage",
False))
has_downstream_stage = bool(stage_ctx is not None and not is_last_stage)
# Directory-mode selector:
# - First pass: `add-file -store X -path <DIR>` should ONLY show a selectable table.
# - Second pass (triggered by @ selection expansion): re-run add-file with `-path file1,file2,...`
# and actually ingest/copy.
# - Terminal use: `add-file -store X -path <DIR>` shows a selectable table.
# - Pipelined use: `add-file -store X -path <DIR> | ...` processes the full batch
# immediately so downstream stages receive the uploaded items.
# - Selection replay: `@N` re-runs add-file with `-path file1,file2,...`.
dir_scan_mode = False
dir_scan_results: Optional[List[Dict[str, Any]]] = None
explicit_path_list_results: Optional[List[Dict[str, Any]]] = None
@@ -350,6 +352,19 @@ class Add_File(Cmdlet):
total_items = len(items_to_process) if isinstance(items_to_process, list) else 0
processed_items = 0
try:
ui, _ = progress.ui_and_pipe_index()
if ui is not None and total_items:
preview_items = (
list(items_to_process)
if isinstance(items_to_process, list) else [items_to_process]
)
progress.begin_pipe(
total_items=total_items,
items_preview=preview_items,
)
except Exception:
pass
try:
if total_items:
progress.set_percent(0)
@@ -369,12 +384,20 @@ class Add_File(Cmdlet):
except Exception:
use_steps = False
debug(f"[add-file] INPUT result type={type(result).__name__}")
if isinstance(result, list):
debug(f"[add-file] INPUT result is list with {len(result)} items")
debug(
f"[add-file] PARSED args: location={location}, provider={provider_name}, delete={delete_after}"
)
try:
debug_panel(
"add-file",
[
("result_type", type(result).__name__),
("items", total_items),
("location", location),
("provider", provider_name),
("delete", delete_after),
],
border_style="cyan",
)
except Exception:
pass
# add-file is ingestion-only: it does not download URLs here.
@@ -393,22 +416,22 @@ class Add_File(Cmdlet):
except Exception:
po = None
if po is None:
debug(f"[add-file] PIPE item[{idx}] preview (non-PipeObject)")
continue
debug(f"[add-file] PIPE item[{idx}] PipeObject preview")
try:
safe_po = _sanitize_pipe_object_for_debug(po)
safe_po.debug_table()
except Exception:
pass
if len(preview_items) > max_preview:
debug(
f"[add-file] Skipping {len(preview_items) - max_preview} additional piped item(s) in debug preview"
)
# If this invocation was directory selector mode, show a selectable table and stop.
should_present_directory_selector = bool(dir_scan_mode and not has_downstream_stage)
if dir_scan_mode and has_downstream_stage:
debug(
"[add-file] Continuing with directory batch ingest because downstream stages exist"
)
# If this invocation was terminal directory selector mode, show a selectable table and stop.
# The user then runs @N (optionally piped), which replays add-file with selected paths.
if dir_scan_mode:
if should_present_directory_selector:
try:
from SYS.result_table import Table
from pathlib import Path as _Path
@@ -563,13 +586,19 @@ class Add_File(Cmdlet):
media_path, file_hash, temp_dir_to_cleanup = Add_File._download_provider_source(
pipe_obj, config, storage_registry
)
if media_path:
debug(
f"[add-file] Provider source downloaded: {media_path}"
if media_path:
try:
debug_panel(
f"add-file source {idx}/{max(1, total_items)}",
[
("path", media_path),
("hash", file_hash or "N/A"),
("provider", provider_name or "local"),
],
border_style="green",
)
debug(
f"[add-file] RESOLVED source: path={media_path}, hash={file_hash if file_hash else 'N/A'}..."
)
except Exception:
pass
if not media_path:
failures += 1
continue
@@ -1616,6 +1645,7 @@ class Add_File(Cmdlet):
) -> None:
pipe_obj.hash = hash_value
pipe_obj.store = store
pipe_obj.is_temp = False
pipe_obj.path = path
pipe_obj.tag = tag
if title:
@@ -2211,11 +2241,20 @@ class Add_File(Cmdlet):
upload_tags = tags
if prefer_defer_tags and upload_tags:
upload_tags = []
debug(f"[add-file] Deferring tag application for {backend_name} (backend preference)")
debug(
f"[add-file] Storing into backend '{backend_name}' path='{media_path}' title='{title}' hash='{f_hash[:12] if f_hash else 'N/A'}'"
)
try:
debug_panel(
"add-file store",
[
("backend", backend_name),
("path", media_path),
("title", title),
("hash_hint", f_hash[:12] if f_hash else "N/A"),
("defer_tags", bool(prefer_defer_tags and tags)),
],
border_style="yellow",
)
except Exception:
pass
# Call backend's add_file with full metadata
# Backend returns hash as identifier. If we already know the hash from _resolve_source
@@ -2227,9 +2266,6 @@ class Add_File(Cmdlet):
url=[] if (defer_url_association and url) else url,
file_hash=f_hash,
)
debug(
f"[add-file] backend.add_file returned identifier {file_identifier} (len={len(str(file_identifier)) if file_identifier is not None else 'None'})"
)
##log(f"✓ File added to '{backend_name}': {file_identifier}", file=sys.stderr)
stored_path: Optional[str] = None
cmdlet/download_file.py
+80 -40
View File
@@ -18,7 +18,7 @@ from contextlib import AbstractContextManager, nullcontext
from API.HTTP import _download_direct_file
from SYS.models import DownloadError, DownloadOptions, DownloadMediaResult
from SYS.logger import log, debug, is_debug_enabled
from SYS.logger import log, debug, debug_panel, is_debug_enabled
from SYS.payload_builders import build_file_result_payload, build_table_result_payload
from SYS.pipeline_progress import PipelineProgress
from SYS.result_table import Table
@@ -113,7 +113,17 @@ class Download_File(Cmdlet):
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
"""Main execution method."""
debug(f"[download-file] run invoked with args: {list(args)}")
try:
debug_panel(
"download-file",
[
("args", list(args)),
("has_piped_input", bool(result)),
],
border_style="cyan",
)
except Exception:
debug(f"[download-file] run invoked with args: {list(args)}")
return self._run_impl(result, args, config)
@staticmethod
@@ -1008,7 +1018,6 @@ class Download_File(Cmdlet):
from Store import Store
from API.HydrusNetwork import is_hydrus_available
debug("[download-file] Initializing storage interface...")
storage = Store(config=config or {}, suppress_debug=True)
hydrus_available = bool(is_hydrus_available(config or {}))
@@ -1126,7 +1135,6 @@ class Download_File(Cmdlet):
@staticmethod
def _canonicalize_url_for_storage(*, requested_url: str, ytdlp_tool: YtDlpTool, playlist_items: Optional[str]) -> str:
if playlist_items:
debug(f"[download-file] Skipping canonicalization for playlist item(s): {playlist_items}")
return str(requested_url)
try:
cf = None
@@ -1136,16 +1144,13 @@ class Download_File(Cmdlet):
cf = str(cookie_path)
except Exception:
cf = None
debug(f"[download-file] Canonicalizing URL: {requested_url}")
pr = probe_url(requested_url, no_playlist=False, timeout_seconds=15, cookiefile=cf)
if isinstance(pr, dict):
for key in ("webpage_url", "original_url", "url", "requested_url"):
value = pr.get(key)
if isinstance(value, str) and value.strip():
canon = value.strip()
if canon != requested_url:
debug(f"[download-file] Resolved canonical URL: {requested_url} -> {canon}")
return canon
except Exception as e:
debug(f"[download-file] Canonicalization error for {requested_url}: {e}")
@@ -1180,7 +1185,8 @@ class Download_File(Cmdlet):
urls=unique_to_check,
storage=storage,
hydrus_available=hydrus_available,
final_output_dir=final_output_dir
final_output_dir=final_output_dir,
auto_continue_duplicates=False,
)
def _preflight_url_duplicates_bulk(
@@ -1204,7 +1210,8 @@ class Download_File(Cmdlet):
urls=unique_urls,
storage=storage,
hydrus_available=hydrus_available,
final_output_dir=final_output_dir
final_output_dir=final_output_dir,
auto_continue_duplicates=False,
)
@@ -1512,6 +1519,7 @@ class Download_File(Cmdlet):
download_timeout_seconds: int,
) -> int:
downloaded_count = 0
duplicate_skipped_count = 0
downloaded_pipe_objects: List[Dict[str, Any]] = []
pipe_seq = 0
clip_sections_spec = self._build_clip_sections_spec(clip_ranges)
@@ -1527,9 +1535,6 @@ class Download_File(Cmdlet):
for url_index, url in enumerate(supported_url, 1):
try:
debug(f"[download-file] Processing URL in loop: {url}")
debug(f"[download-file] ytdl_format parameter passed in: {ytdl_format}")
display_total = batch_total if batch_total > 0 else total_urls
display_index = batch_index if batch_total > 0 else url_index
display_label = batch_label or str(url)
@@ -1543,7 +1548,6 @@ class Download_File(Cmdlet):
)
if not skip_per_url_preflight:
debug(f"[download-file] Running duplicate preflight for: {canonical_url}")
if not self._preflight_url_duplicate(
storage=storage,
hydrus_available=hydrus_available,
@@ -1551,9 +1555,25 @@ class Download_File(Cmdlet):
candidate_url=canonical_url,
extra_urls=[url],
):
duplicate_skipped_count += 1
log(f"Skipping download (duplicate found): {url}", file=sys.stderr)
continue
try:
debug_panel(
f"Download item {display_index}/{display_total or total_urls}",
[
("url", url),
("canonical_url", canonical_url),
("mode", mode),
("format", ytdl_format or "auto"),
("duplicate_preflight", not skip_per_url_preflight),
],
border_style="green",
)
except Exception:
pass
if aggregate_status_mode:
try:
if display_total > 0:
@@ -1682,11 +1702,7 @@ class Download_File(Cmdlet):
actual_format = f"{actual_format}+bestaudio"
except Exception as e:
pass
debug(
"[download-file] Resolved format for download: "
f"mode={mode}, format={actual_format or 'default'}, playlist_items={actual_playlist_items}"
)
attempted_single_format_fallback = False
attempted_audio_fallback_specific = False
@@ -1709,9 +1725,7 @@ class Download_File(Cmdlet):
if not aggregate_status_mode:
PipelineProgress(pipeline_context).step("downloading")
debug(f"Starting download for {url} (format: {actual_format or 'default'}) with {download_timeout_seconds}s activity timeout...")
result_obj = _download_with_timeout(opts, timeout_seconds=download_timeout_seconds, config=config)
debug(f"Download completed for {url}, building pipe object...")
break
except DownloadError as e:
cause = getattr(e, "__cause__", None)
@@ -2028,8 +2042,6 @@ class Download_File(Cmdlet):
except Exception:
pass
debug(f"Emitting {len(pipe_objects)} result(s) to pipeline...")
if not aggregate_status_mode:
PipelineProgress(pipeline_context).step("finalized")
@@ -2049,7 +2061,17 @@ class Download_File(Cmdlet):
pass
downloaded_count += len(pipe_objects)
debug("✓ Downloaded and emitted")
try:
debug_panel(
"download-file result",
[
("emitted", len(pipe_objects)),
("url", url),
],
border_style="green",
)
except Exception:
pass
except DownloadError as e:
log(f"Download failed for {url}: {e}", file=sys.stderr)
@@ -2057,7 +2079,8 @@ class Download_File(Cmdlet):
log(f"Error processing {url}: {e}", file=sys.stderr)
if downloaded_count > 0:
debug(f"✓ Successfully processed {downloaded_count} URL(s)")
return 0
if duplicate_skipped_count > 0:
return 0
log("No downloads completed", file=sys.stderr)
@@ -2072,7 +2095,6 @@ class Download_File(Cmdlet):
parsed: Dict[str, Any],
) -> int:
try:
debug("Starting streaming download handler")
suppress_nested, _batch_total, _batch_index, _batch_label = self._batch_progress_state(config)
ytdlp_tool = YtDlpTool(config)
@@ -2091,21 +2113,18 @@ class Download_File(Cmdlet):
if not final_output_dir:
return 1
debug(f"Output directory: {final_output_dir}")
progress = PipelineProgress(pipeline_context)
using_shared_ui = pipeline_context.get_stage_context() is not None
try:
# If we are already in a pipeline stage, the parent UI is already handling progress.
# Calling ensure_local_ui can cause re-initialization hangs on some platforms.
if pipeline_context.get_stage_context() is None:
debug("[download-file] Initializing local UI...")
if not using_shared_ui:
progress.ensure_local_ui(
label="download-file",
total_items=len(supported_url),
items_preview=supported_url,
)
else:
debug("[download-file] Skipping local UI: running inside pipeline stage")
try:
if not suppress_nested:
progress.begin_pipe(
@@ -2116,8 +2135,6 @@ class Download_File(Cmdlet):
debug(f"[download-file] PipelineProgress begin_pipe error: {err}")
except Exception as e:
debug(f"[download-file] PipelineProgress update error: {e}")
debug("[download-file] Parsing clip and query specs...")
clip_spec = parsed.get("clip")
query_spec = parsed.get("query")
@@ -2223,7 +2240,6 @@ class Download_File(Cmdlet):
ytdl_format = query_format
if not ytdl_format:
debug(f"[download-file] Checking for playlist at {candidate_url}...")
if self._maybe_show_playlist_table(url=candidate_url, ytdlp_tool=ytdlp_tool):
playlist_selection_handled = True
# ... (existing logging code) ...
@@ -2270,7 +2286,6 @@ class Download_File(Cmdlet):
forced_single_format_id = None
forced_single_format_for_batch = False
debug("[download-file] Checking if format table should be shown...")
early_ret = self._maybe_show_format_table_for_single_url(
mode=mode,
clip_spec=clip_spec,
@@ -2343,7 +2358,23 @@ class Download_File(Cmdlet):
except Exception:
timeout_seconds = 300
debug(f"[download-file] Proceeding to final download call for {len(supported_url)} URL(s)...")
try:
debug_panel(
"Streaming download",
[
("urls", len(supported_url)),
("mode", mode),
("format", ytdl_format or "auto"),
("output_dir", final_output_dir),
("ui", "shared pipeline" if using_shared_ui else "local"),
("playlist_items", playlist_items),
("skip_preflight", skip_per_url_preflight),
("timeout_seconds", timeout_seconds),
],
border_style="blue",
)
except Exception:
pass
return self._download_supported_urls(
supported_url=supported_url,
ytdlp_tool=ytdlp_tool,
@@ -2855,8 +2886,6 @@ class Download_File(Cmdlet):
prev_progress = None
had_progress_key = False
try:
debug("Starting download-file")
# Allow providers to tap into the active PipelineProgress (optional).
try:
if isinstance(config, dict):
@@ -3117,7 +3146,6 @@ class Download_File(Cmdlet):
streaming_exit_code: Optional[int] = None
streaming_downloaded = 0
if supported_streaming:
debug(f"[download-file] Using ytdlp provider for {len(supported_streaming)} URL(s)")
streaming_exit_code = self._run_streaming_urls(
streaming_urls=supported_streaming,
args=args,
@@ -3161,7 +3189,19 @@ class Download_File(Cmdlet):
if not final_output_dir:
return 1
debug(f"Output directory: {final_output_dir}")
try:
debug_panel(
"download-file plan",
[
("output_dir", final_output_dir),
("streaming_urls", len(supported_streaming)),
("remaining_urls", len(raw_url)),
("piped_items", len(piped_items) if isinstance(piped_items, list) else int(bool(piped_items))),
],
border_style="cyan",
)
except Exception:
debug(f"Output directory: {final_output_dir}")
# If the caller isn't running the shared pipeline Live progress UI (e.g. direct
# cmdlet execution), start a minimal local pipeline progress panel so downloads
cmdlet/search_file.py
-8
View File
@@ -2158,11 +2158,7 @@ class search_file(Cmdlet):
)
db.update_worker_status(worker_id, "error")
return 1
debug(f"[search-file] Searching '{backend_to_search}'")
results = target_backend.search(query, limit=limit)
debug(
f"[search-file] '{backend_to_search}' -> {len(results or [])} result(s)"
)
else:
all_results = []
store_registry = None
@@ -2184,14 +2180,10 @@ class search_file(Cmdlet):
if type(backend).search is BaseStore.search:
continue
debug(f"[search-file] Searching '{backend_name}'")
backend_results = backend.search(
query,
limit=limit - len(all_results)
)
debug(
f"[search-file] '{backend_name}' -> {len(backend_results or [])} result(s)"
)
if backend_results:
all_results.extend(backend_results)
if len(all_results) >= limit: