updated panel display
This commit is contained in:
+70
-20
@@ -14,7 +14,7 @@ from collections.abc import Iterable as IterableABC
|
||||
from functools import lru_cache
|
||||
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
|
||||
|
||||
from SYS.logger import log, debug
|
||||
from SYS.logger import log, debug, debug_panel
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple
|
||||
from dataclasses import dataclass, field
|
||||
@@ -3158,6 +3158,8 @@ def check_url_exists_in_storage(
|
||||
storage: Any,
|
||||
hydrus_available: bool,
|
||||
final_output_dir: Optional[Path] = None,
|
||||
*,
|
||||
auto_continue_duplicates: bool = True,
|
||||
) -> bool:
|
||||
"""Pre-flight check to see if URLs already exist in storage.
|
||||
|
||||
@@ -3187,7 +3189,6 @@ def check_url_exists_in_storage(
|
||||
in_pipeline = bool(stage_ctx is not None or ("|" in str(current_cmd_text or "")))
|
||||
start_time = time.monotonic()
|
||||
time_budget = 45.0
|
||||
debug(f"[preflight] check_url_exists_in_storage: checking {len(urls)} url(s)")
|
||||
if in_pipeline:
|
||||
try:
|
||||
already_checked = bool(
|
||||
@@ -3243,7 +3244,7 @@ def check_url_exists_in_storage(
|
||||
return False
|
||||
return False
|
||||
|
||||
if in_pipeline:
|
||||
if in_pipeline and auto_continue_duplicates:
|
||||
try:
|
||||
cached_cmd = pipeline_context.load_value("preflight.url_duplicates.command", default="")
|
||||
cached_decision = pipeline_context.load_value("preflight.url_duplicates.continue", default=None)
|
||||
@@ -3706,6 +3707,20 @@ def check_url_exists_in_storage(
|
||||
debug("Bulk URL preflight skipped: no searchable backends")
|
||||
return True
|
||||
|
||||
try:
|
||||
debug_panel(
|
||||
"URL preflight",
|
||||
[
|
||||
("url_count", len(unique_urls)),
|
||||
("pipeline", in_pipeline),
|
||||
("bulk_mode", bulk_mode),
|
||||
("backends", ", ".join(str(name) for name in backend_names)),
|
||||
],
|
||||
border_style="yellow",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
seen_pairs: set[tuple[str, str]] = set()
|
||||
matched_urls: set[str] = set()
|
||||
match_rows: List[Dict[str, Any]] = []
|
||||
@@ -3726,12 +3741,7 @@ def check_url_exists_in_storage(
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
debug(f"[preflight] Scanning backend: {backend_name}")
|
||||
|
||||
if HydrusNetwork is not None and isinstance(backend, HydrusNetwork):
|
||||
client = getattr(backend, "_client", None)
|
||||
if client is None:
|
||||
continue
|
||||
if not hydrus_available:
|
||||
debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup")
|
||||
|
||||
@@ -3748,7 +3758,33 @@ def check_url_exists_in_storage(
|
||||
|
||||
found_hash: Optional[str] = None
|
||||
found = False
|
||||
lookup_exact = getattr(backend, "find_hashes_by_url", None)
|
||||
if callable(lookup_exact):
|
||||
for needle in [original_url, *(needles or [])][:7]:
|
||||
needle_text = str(needle or "").strip()
|
||||
if not _httpish(needle_text):
|
||||
continue
|
||||
try:
|
||||
exact_hashes = lookup_exact(needle_text) or []
|
||||
except Exception:
|
||||
continue
|
||||
if not isinstance(exact_hashes, list) or not exact_hashes:
|
||||
continue
|
||||
try:
|
||||
found_hash = str(exact_hashes[0] or "").strip().lower()
|
||||
except Exception:
|
||||
found_hash = None
|
||||
found = True
|
||||
break
|
||||
|
||||
client = getattr(backend, "_client", None)
|
||||
if found:
|
||||
pass
|
||||
elif client is None:
|
||||
continue
|
||||
for needle in (needles or [])[:6]:
|
||||
if found:
|
||||
break
|
||||
if not _httpish(needle):
|
||||
continue
|
||||
try:
|
||||
@@ -3868,7 +3904,6 @@ def check_url_exists_in_storage(
|
||||
match_rows.append(display_row)
|
||||
|
||||
if not match_rows:
|
||||
debug("Bulk URL preflight: no matches")
|
||||
if in_pipeline:
|
||||
preflight_cache = _load_preflight_cache()
|
||||
url_dup_cache = preflight_cache.get("url_duplicates")
|
||||
@@ -3935,24 +3970,39 @@ def check_url_exists_in_storage(
|
||||
auto_confirm_reason = "non-interactive stdin"
|
||||
|
||||
answered_yes = True
|
||||
auto_declined = False
|
||||
with cm:
|
||||
get_stderr_console().print(table)
|
||||
setattr(table, "_rendered_by_cmdlet", True)
|
||||
if auto_confirm_reason is None:
|
||||
answered_yes = bool(Confirm.ask("Continue anyway?", default=False, console=get_stderr_console()))
|
||||
else:
|
||||
debug(
|
||||
f"Bulk URL preflight auto-confirmed duplicates ({auto_confirm_reason}); continuing without user input."
|
||||
)
|
||||
try:
|
||||
log(
|
||||
f"Auto-confirmed duplicate URL warning ({auto_confirm_reason}). Continuing...",
|
||||
file=sys.stderr,
|
||||
answered_yes = bool(auto_continue_duplicates)
|
||||
auto_declined = not answered_yes
|
||||
if answered_yes:
|
||||
debug(
|
||||
f"Bulk URL preflight auto-confirmed duplicates ({auto_confirm_reason}); continuing without user input."
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
log(
|
||||
f"Auto-confirmed duplicate URL warning ({auto_confirm_reason}). Continuing...",
|
||||
file=sys.stderr,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
debug(
|
||||
f"Bulk URL preflight auto-skipped duplicates ({auto_confirm_reason}); skipping without user input."
|
||||
)
|
||||
try:
|
||||
log(
|
||||
f"Duplicate URL detected ({auto_confirm_reason}). Skipping download.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if in_pipeline:
|
||||
if in_pipeline and auto_continue_duplicates:
|
||||
try:
|
||||
existing = pipeline_context.load_value("preflight", default=None)
|
||||
except Exception:
|
||||
@@ -3977,7 +4027,7 @@ def check_url_exists_in_storage(
|
||||
pass
|
||||
|
||||
if not answered_yes:
|
||||
if in_pipeline:
|
||||
if in_pipeline and not auto_declined:
|
||||
try:
|
||||
pipeline_context.request_pipeline_stop(reason="duplicate-url declined", exit_code=0)
|
||||
except Exception:
|
||||
|
||||
Reference in New Issue
Block a user