From d9e736172a98b333bd23958e938c1323d5347490 Mon Sep 17 00:00:00 2001 From: Nose Date: Fri, 17 Apr 2026 16:17:16 -0700 Subject: [PATCH] update --- Store/HydrusNetwork.py | 76 +++++++++++++++--- cmdlet/_shared.py | 5 +- cmdlet/add_file.py | 166 +++++++++++++++++++++++++++++++++++----- cmdlet/download_file.py | 104 +++++++++++++++++++++++++ 4 files changed, 322 insertions(+), 29 deletions(-) diff --git a/Store/HydrusNetwork.py b/Store/HydrusNetwork.py index 1d659e0..dd8e29f 100644 --- a/Store/HydrusNetwork.py +++ b/Store/HydrusNetwork.py @@ -13,7 +13,7 @@ from urllib.parse import quote, parse_qsl, urlencode, urlsplit, urlunsplit import httpx from API.httpx_shared import get_shared_httpx_client -from SYS.logger import debug, log +from SYS.logger import debug, debug_panel, log from SYS.utils_constant import mime_maps _KNOWN_EXTS = { @@ -1533,7 +1533,17 @@ class HydrusNetwork(Store): Only explicit user actions (e.g. the get-file cmdlet) should open files. """ file_hash = str(file_hash or "").strip().lower() - debug(f"{self._log_prefix()} get_file(hash={file_hash[:12]}..., url={kwargs.get('url')})") + try: + debug_panel( + "Hydrus get_file", + [ + ("hash", file_hash), + ("prefer_url", bool(kwargs.get("url"))), + ], + border_style="blue", + ) + except Exception: + pass # If 'url=True' is passed, we preference the browser URL even if a local path is available. # This is typically used by the 'get-file' cmdlet for interactive viewing. @@ -1543,7 +1553,17 @@ class HydrusNetwork(Store): browser_url = ( f"{base_url}/get_files/file?hash={file_hash}&Hydrus-Client-API-Access-Key={access_key}" ) - debug(f"{self._log_prefix()} get_file: returning browser URL per request: {browser_url}") + try: + debug_panel( + "Hydrus get_file", + [ + ("mode", "browser-url"), + ("url", browser_url), + ], + border_style="blue", + ) + except Exception: + pass return browser_url # Try to get the local disk path if possible (works if Hydrus is on same machine) @@ -1555,18 +1575,46 @@ class HydrusNetwork(Store): if server_path: local_path = Path(server_path) if local_path.exists(): - debug(f"{self._log_prefix()} get_file: found local path: {local_path}") + try: + debug_panel( + "Hydrus get_file", + [ + ("mode", "local-path"), + ("path", local_path), + ], + border_style="green", + ) + except Exception: + pass return local_path except Exception as e: - debug(f"{self._log_prefix()} get_file: could not resolve path from API: {e}") + try: + debug_panel( + "Hydrus get_file", + [ + ("mode", "path-lookup-error"), + ("error", e), + ], + border_style="yellow", + ) + except Exception: + pass # If we found a path on the server but it's not locally accessible, # keep it for logging but continue to the browser URL fallback so the UI # can still open the file via the Hydrus web UI. if server_path: - debug( - f"{self._log_prefix()} get_file: server path not locally accessible, falling back to HTTP: {server_path}" - ) + try: + debug_panel( + "Hydrus get_file fallback", + [ + ("mode", "remote-http"), + ("server_path", server_path), + ], + border_style="yellow", + ) + except Exception: + pass # Fallback to browser URL with access key base_url = str(self.URL).rstrip("/") @@ -1574,7 +1622,17 @@ class HydrusNetwork(Store): browser_url = ( f"{base_url}/get_files/file?hash={file_hash}&Hydrus-Client-API-Access-Key={access_key}" ) - debug(f"{self._log_prefix()} get_file: falling back to url={browser_url}") + try: + debug_panel( + "Hydrus get_file fallback", + [ + ("mode", "remote-http"), + ("url", browser_url), + ], + border_style="yellow", + ) + except Exception: + pass return browser_url def download_to_temp( diff --git a/cmdlet/_shared.py b/cmdlet/_shared.py index 57dab32..901910c 100644 --- a/cmdlet/_shared.py +++ b/cmdlet/_shared.py @@ -3160,6 +3160,7 @@ def check_url_exists_in_storage( final_output_dir: Optional[Path] = None, *, auto_continue_duplicates: bool = True, + force_prompt_in_pipeline: bool = False, ) -> bool: """Pre-flight check to see if URLs already exist in storage. @@ -3252,7 +3253,7 @@ def check_url_exists_in_storage( cached_cmd = "" cached_decision = None - if cached_decision is not None and str(cached_cmd or "") == str(current_cmd_text or ""): + if (not force_prompt_in_pipeline) and cached_decision is not None and str(cached_cmd or "") == str(current_cmd_text or ""): _mark_preflight_checked() if bool(cached_decision): return True @@ -3959,7 +3960,7 @@ def check_url_exists_in_storage( is_last_stage = bool(getattr(stage_ctx, "is_last_stage", False)) except Exception: is_last_stage = False - if total_stages > 1 and not is_last_stage: + if total_stages > 1 and not is_last_stage and not force_prompt_in_pipeline: auto_confirm_reason = "pipeline stage (pre-last)" if auto_confirm_reason is None: try: diff --git a/cmdlet/add_file.py b/cmdlet/add_file.py index 774978f..3120543 100644 --- a/cmdlet/add_file.py +++ b/cmdlet/add_file.py @@ -234,8 +234,13 @@ class Add_File(Cmdlet): try: candidate_dir = Path(str(path_arg)) if candidate_dir.exists() and candidate_dir.is_dir(): - debug( - f"[add-file] Treating -path directory as destination: {candidate_dir}" + debug_panel( + "add-file destination", + [ + ("mode", "local export"), + ("path", candidate_dir), + ], + border_style="cyan", ) location = str(candidate_dir) path_arg = None @@ -350,6 +355,13 @@ class Add_File(Cmdlet): else: items_to_process = [result] + if result is None and not path_arg and not explicit_path_list_results and not dir_scan_results: + try: + if ctx.get_stage_context() is not None: + return 0 + except Exception: + pass + total_items = len(items_to_process) if isinstance(items_to_process, list) else 0 processed_items = 0 try: @@ -580,7 +592,12 @@ class Add_File(Cmdlet): progress.step("resolving source") media_path, file_hash, temp_dir_to_cleanup = self._resolve_source( - item, path_arg, pipe_obj, config, store_instance=storage_registry + item, + path_arg, + pipe_obj, + config, + export_destination=(Path(location) if location and not is_storage_backend_location else None), + store_instance=storage_registry, ) if not media_path and provider_name: media_path, file_hash, temp_dir_to_cleanup = Add_File._download_provider_source( @@ -1103,6 +1120,70 @@ class Add_File(Cmdlet): pass return None, None + @staticmethod + def _download_remote_backend_url( + remote_url: str, + pipe_obj: models.PipeObject, + *, + file_hash: Optional[str] = None, + output_dir: Optional[Path] = None, + ) -> Tuple[Optional[Path], Optional[Path]]: + """Best-effort fetch of a remote backend URL. + + Returns (downloaded_path, temp_dir_to_cleanup). + When ``output_dir`` is provided, the file is downloaded directly there and no + temp cleanup path is returned. + """ + + url_text = str(remote_url or "").strip() + if not url_text: + return None, None + if not url_text.lower().startswith(_REMOTE_URL_PREFIXES): + return None, None + + tmp_dir: Optional[Path] = None + try: + download_root = output_dir + if download_root is None: + tmp_dir = Path(tempfile.mkdtemp(prefix="add-file-src-")) + download_root = tmp_dir + + suggested_name = Add_File._build_provider_filename( + pipe_obj, + fallback_hash=file_hash, + source_url=url_text, + ) + pipeline_progress = PipelineProgress(ctx) + + downloaded = _download_direct_file( + url_text, + download_root, + quiet=False, + suggested_filename=suggested_name, + pipeline_progress=pipeline_progress, + ) + downloaded_path = getattr(downloaded, "path", None) + if isinstance(downloaded_path, Path) and downloaded_path.exists(): + if output_dir is not None: + pipe_obj.is_temp = False + if isinstance(pipe_obj.extra, dict): + pipe_obj.extra["_direct_export_download"] = True + else: + pipe_obj.extra = {"_direct_export_download": True} + return downloaded_path, None + + pipe_obj.is_temp = True + return downloaded_path, tmp_dir + except Exception: + pass + + if tmp_dir is not None: + try: + shutil.rmtree(tmp_dir, ignore_errors=True) + except Exception: + pass + return None, None + @staticmethod def _build_provider_filename( pipe_obj: models.PipeObject, @@ -1188,6 +1269,7 @@ class Add_File(Cmdlet): pipe_obj: models.PipeObject, config: Dict[str, Any], + export_destination: Optional[Path] = None, store_instance: Optional[Any] = None, ) -> Tuple[Optional[Path], Optional[str], @@ -1228,12 +1310,30 @@ class Add_File(Cmdlet): pipe_obj.path = str(mp) return mp, str(r_hash), None if isinstance(mp, str) and mp.strip(): + try: + mp_path = Path(str(mp)) + except Exception: + mp_path = None + if mp_path is not None and mp_path.exists() and mp_path.is_file(): + pipe_obj.path = str(mp_path) + return mp_path, str(r_hash), None + dl_path, tmp_dir = Add_File._maybe_download_backend_file( backend, str(r_hash), pipe_obj ) if dl_path and dl_path.exists(): pipe_obj.path = str(dl_path) return dl_path, str(r_hash), tmp_dir + + dl_path, tmp_dir = Add_File._download_remote_backend_url( + str(mp), + pipe_obj, + file_hash=str(r_hash), + output_dir=export_destination, + ) + if dl_path and dl_path.exists(): + pipe_obj.path = str(dl_path) + return dl_path, str(r_hash), tmp_dir except Exception as exc: debug(f"[add-file] _resolve_source backend fetch failed for {r_store}/{r_hash}: {exc}") @@ -1657,12 +1757,22 @@ class Add_File(Cmdlet): @staticmethod def _emit_pipe_object(pipe_obj: models.PipeObject) -> None: - from SYS.result_table import format_result - - log(format_result(pipe_obj, title="Result"), file=sys.stderr) - ctx.emit(pipe_obj.to_dict()) + payload = pipe_obj.to_dict() + ctx.emit(payload) ctx.set_current_stage_table(None) + stage_ctx = ctx.get_stage_context() + is_last = (stage_ctx is None) or bool(getattr(stage_ctx, "is_last_stage", False)) + if not is_last: + return + + try: + from ._shared import display_and_persist_items + + display_and_persist_items([payload], title="Result", subject=payload) + except Exception: + pass + @staticmethod def _emit_storage_result( payload: Dict[str, @@ -1925,7 +2035,24 @@ class Add_File(Cmdlet): log(f"❌ Invalid destination path '{location}': {exc}", file=sys.stderr) return 1 - log(f"Exporting to local path: {destination_root}", file=sys.stderr) + direct_export_download = False + try: + if isinstance(pipe_obj.extra, dict): + direct_export_download = bool(pipe_obj.extra.pop("_direct_export_download", False)) + except Exception: + direct_export_download = False + + try: + debug_panel( + "add-file export", + [ + ("destination", destination_root), + ("source", media_path), + ], + border_style="green", + ) + except Exception: + pass result = None tags, url, title, f_hash = Add_File._prepare_metadata(result, media_path, pipe_obj, config) @@ -1961,18 +2088,21 @@ class Add_File(Cmdlet): destination_root.mkdir(parents=True, exist_ok=True) target_path = destination_root / new_name - if target_path.exists(): - target_path = unique_path(target_path) + if direct_export_download: + target_path = media_path + else: + if target_path.exists(): + target_path = unique_path(target_path) - # COPY Operation (Safe Export) - try: - shutil.copy2(str(media_path), target_path) - except Exception as exc: - log(f"❌ Failed to export file: {exc}", file=sys.stderr) - return 1 + # COPY Operation (Safe Export) + try: + shutil.copy2(str(media_path), target_path) + except Exception as exc: + log(f"❌ Failed to export file: {exc}", file=sys.stderr) + return 1 - # Copy Sidecars - Add_File._copy_sidecars(media_path, target_path) + # Copy Sidecars + Add_File._copy_sidecars(media_path, target_path) # Ensure hash for exported copy if not f_hash: diff --git a/cmdlet/download_file.py b/cmdlet/download_file.py index 2ba9828..afea22e 100644 --- a/cmdlet/download_file.py +++ b/cmdlet/download_file.py @@ -1187,6 +1187,7 @@ class Download_File(Cmdlet): hydrus_available=hydrus_available, final_output_dir=final_output_dir, auto_continue_duplicates=False, + force_prompt_in_pipeline=bool(kwargs.get("force_prompt_in_pipeline")), ) def _preflight_url_duplicates_bulk( @@ -1554,6 +1555,7 @@ class Download_File(Cmdlet): final_output_dir=final_output_dir, candidate_url=canonical_url, extra_urls=[url], + force_prompt_in_pipeline=bool(clip_ranges), ): duplicate_skipped_count += 1 log(f"Skipping download (duplicate found): {url}", file=sys.stderr) @@ -2823,6 +2825,101 @@ class Download_File(Cmdlet): return f"{hours:02d}:{minutes:02d}:{secs:02d}" return f"{minutes:02d}:{secs:02d}" + @staticmethod + def _rebase_subtitle_timestamp_text(text: str, offset_seconds: int) -> str: + if not text: + return text + + try: + offset_value = float(offset_seconds) + except Exception: + return text + + if offset_value <= 0: + return text + + timestamp_re = re.compile(r"(?(?:\d{2}:)?\d{2}:\d{2}(?:[\.,]\d{1,3})?)(?!\d)") + + def _shift(match: re.Match[str]) -> str: + original = str(match.group("ts") or "") + if not original: + return original + + frac_sep = "." + frac_digits = 0 + base = original + frac_seconds = 0.0 + if "." in original: + base, frac = original.split(".", 1) + frac_sep = "." + frac_digits = len(frac) + try: + frac_seconds = float(f"0.{frac}") if frac else 0.0 + except Exception: + frac_seconds = 0.0 + elif "," in original: + base, frac = original.split(",", 1) + frac_sep = "," + frac_digits = len(frac) + try: + frac_seconds = float(f"0.{frac}") if frac else 0.0 + except Exception: + frac_seconds = 0.0 + + parts = base.split(":") + if len(parts) == 3: + hours_s, minutes_s, seconds_s = parts + include_hours = True + elif len(parts) == 2: + hours_s = "0" + minutes_s, seconds_s = parts + include_hours = False + else: + return original + + try: + total = ( + (int(hours_s) * 3600) + + (int(minutes_s) * 60) + + int(seconds_s) + + frac_seconds + + offset_value + ) + except Exception: + return original + + total = max(0.0, total) + whole_seconds = int(total) + fraction = total - whole_seconds + hours, remainder = divmod(whole_seconds, 3600) + minutes, seconds = divmod(remainder, 60) + + if frac_digits > 0: + scale = 10 ** frac_digits + frac_value = int(round(fraction * scale)) + if frac_value >= scale: + frac_value = 0 + seconds += 1 + if seconds >= 60: + seconds = 0 + minutes += 1 + if minutes >= 60: + minutes = 0 + hours += 1 + frac_text = f"{frac_value:0{frac_digits}d}" + if include_hours or hours > 0: + return f"{hours:02d}:{minutes:02d}:{seconds:02d}{frac_sep}{frac_text}" + return f"{minutes:02d}:{seconds:02d}{frac_sep}{frac_text}" + + if include_hours or hours > 0: + return f"{hours:02d}:{minutes:02d}:{seconds:02d}" + return f"{minutes:02d}:{seconds:02d}" + + try: + return timestamp_re.sub(_shift, str(text)) + except Exception: + return text + @classmethod def _format_clip_range(cls, start_s: int, end_s: int) -> str: force_hours = bool(start_s >= 3600 or end_s >= 3600) @@ -2854,6 +2951,13 @@ class Download_File(Cmdlet): po["tag"] = tags + notes = po.get("notes") + if isinstance(notes, dict): + sub_text = notes.get("sub") + if isinstance(sub_text, str) and sub_text.strip(): + notes["sub"] = cls._rebase_subtitle_timestamp_text(sub_text, start_s) + po["notes"] = notes + if len(pipe_objects) < 2: return