hh

SYS/models.py

@@ -7,6 +7,7 @@ import os
 import shutil
 import sys
 import time
+from threading import RLock
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Callable, Dict, List, Optional, Protocol, TextIO
@@ -755,6 +756,7 @@ class PipelineLiveProgress:
     def __init__(self, pipe_labels: List[str], *, enabled: bool = True) -> None:
         self._enabled = bool(enabled)
         self._pipe_labels = [str(x) for x in (pipe_labels or [])]
+        self._lock = RLock()
 
         self._console: Optional[Console] = None
         self._live: Optional[Live] = None
@@ -826,26 +828,27 @@ class PipelineLiveProgress:
         the spinner without needing manual Live.update() calls.
         """
 
-        pipe_progress = self._pipe_progress
-        status = self._status
-        transfers = self._transfers
-        overall = self._overall
-        if pipe_progress is None or transfers is None or overall is None:
-            # Not started (or stopped).
-            yield Panel("", title="Pipeline", expand=False)
-            return
+        with self._lock:
+            pipe_progress = self._pipe_progress
+            status = self._status
+            transfers = self._transfers
+            overall = self._overall
+            if pipe_progress is None or transfers is None or overall is None:
+                # Not started (or stopped).
+                yield Panel("", title="Pipeline", expand=False)
+                return
 
-        body_parts: List[Any] = [pipe_progress]
-        if status is not None and self._status_tasks:
-            body_parts.append(status)
-        body_parts.append(transfers)
+            body_parts: List[Any] = [pipe_progress]
+            if status is not None and self._status_tasks:
+                body_parts.append(status)
+            body_parts.append(transfers)
 
-        yield Group(
-            Panel(Group(*body_parts),
-                  title=self._title_text(),
-                  expand=False),
-            overall
-        )
+            yield Group(
+                Panel(Group(*body_parts),
+                      title=self._title_text(),
+                      expand=False),
+                overall
+            )
 
     def _render_group(self) -> Group:
         # Backward-compatible helper (some callers may still expect a Group).
@@ -1029,52 +1032,58 @@ class PipelineLiveProgress:
             return
         if not self._ensure_pipe(int(pipe_index)):
             return
-        prog = self._status
-        if prog is None:
-            return
+        
+        with self._lock:
+            prog = self._status
+            if prog is None:
+                return
 
-        try:
-            pidx = int(pipe_index)
-            msg = str(text or "").strip()
-        except Exception:
-            return
-
-        # For long single-item work, hide the per-item spinner line and use this
-        # dedicated status line instead.
-        if self._pipe_percent_mode.get(pidx, False):
             try:
-                self._hide_pipe_subtasks(pidx)
+                pidx = int(pipe_index)
+                msg = str(text or "").strip()
+            except Exception:
+                return
+
+            # For long single-item work, hide the per-item spinner line and use this
+            # dedicated status line instead.
+            if self._pipe_percent_mode.get(pidx, False):
+                try:
+                    self._hide_pipe_subtasks(pidx)
+                except Exception:
+                    pass
+
+            task_id = self._status_tasks.get(pidx)
+            if task_id is None:
+                try:
+                    task_id = prog.add_task(msg)
+                except Exception:
+                    return
+                self._status_tasks[pidx] = task_id
+
+            try:
+                prog.update(task_id, description=msg, refresh=True)
             except Exception:
                 pass
 
-        task_id = self._status_tasks.get(pidx)
-        if task_id is None:
+    def clear_pipe_status_text(self, pipe_index: int) -> None:
+        if not self._enabled:
+            return
+        
+        with self._lock:
+            prog = self._status
+            if prog is None:
+                return
             try:
-                task_id = prog.add_task(msg)
+                pidx = int(pipe_index)
             except Exception:
                 return
-            self._status_tasks[pidx] = task_id
-
-        try:
-            prog.update(task_id, description=msg, refresh=True)
-        except Exception:
-            pass
-
-    def clear_pipe_status_text(self, pipe_index: int) -> None:
-        prog = self._status
-        if prog is None:
-            return
-        try:
-            pidx = int(pipe_index)
-        except Exception:
-            return
-        task_id = self._status_tasks.pop(pidx, None)
-        if task_id is None:
-            return
-        try:
-            prog.remove_task(task_id)
-        except Exception:
-            pass
+            task_id = self._status_tasks.pop(pidx, None)
+            if task_id is None:
+                return
+            try:
+                prog.remove_task(task_id)
+            except Exception:
+                pass
 
     def set_pipe_percent(self, pipe_index: int, percent: int) -> None:
         """Update the pipe bar as a percent (only when single-item mode is enabled)."""
@@ -1095,6 +1104,31 @@ class PipelineLiveProgress:
             pct = max(0, min(100, int(percent)))
             pipe_task = self._pipe_tasks[pidx]
             pipe_progress.update(pipe_task, completed=pct, total=100, refresh=True)
+            self._update_overall()
+        except Exception:
+            pass
+
+    def _update_overall(self) -> None:
+        """Update the overall pipeline progress task."""
+        if self._overall is None or self._overall_task is None:
+            return
+
+        completed = 0
+        try:
+            # Count a pipe as completed if its 'done' count matches or exceeds the advertised total.
+            completed = sum(
+                1 for i in range(len(self._pipe_labels))
+                if self._pipe_done[i] >= max(1, self._pipe_totals[i])
+            )
+        except Exception:
+            completed = 0
+
+        try:
+            self._overall.update(
+                self._overall_task,
+                completed=min(completed, max(1, len(self._pipe_labels))),
+                description=f"Pipeline: {completed}/{len(self._pipe_labels)} pipes completed",
+            )
         except Exception:
             pass
 
@@ -1108,24 +1142,25 @@ class PipelineLiveProgress:
         if not self._ensure_pipe(int(pipe_index)):
             return
 
-        try:
-            pidx = int(pipe_index)
-            tot = max(1, int(total_steps))
-        except Exception:
-            return
+        with self._lock:
+            try:
+                pidx = int(pipe_index)
+                tot = max(1, int(total_steps))
+            except Exception:
+                return
 
-        self._pipe_step_total[pidx] = tot
-        self._pipe_step_done[pidx] = 0
+            self._pipe_step_total[pidx] = tot
+            self._pipe_step_done[pidx] = 0
 
-        # Reset status line and percent.
-        try:
-            self.clear_pipe_status_text(pidx)
-        except Exception:
-            pass
-        try:
-            self.set_pipe_percent(pidx, 0)
-        except Exception:
-            pass
+            # Reset status line and percent.
+            try:
+                self.clear_pipe_status_text(pidx)
+            except Exception:
+                pass
+            try:
+                self.set_pipe_percent(pidx, 0)
+            except Exception:
+                pass
 
     def advance_pipe_step(self, pipe_index: int, text: str) -> None:
         """Advance the pipe's step counter by one.
@@ -1287,6 +1322,8 @@ class PipelineLiveProgress:
         except Exception:
             pass
 
+        self._update_overall()
+
         labels: List[str] = []
         if isinstance(items_preview, list) and items_preview:
             labels = [_pipeline_progress_item_label(x) for x in items_preview]
@@ -1372,6 +1409,8 @@ class PipelineLiveProgress:
         else:
             pipe_progress.update(pipe_task, completed=done)
 
+        self._update_overall()
+
         # Clear any status line now that it emitted.
         try:
             self.clear_pipe_status_text(pipe_index)
@@ -1452,23 +1491,7 @@ class PipelineLiveProgress:
         except Exception:
             pass
 
-        if self._overall_task is not None:
-            completed = 0
-            try:
-                completed = sum(
-                    1 for i in range(len(self._pipe_labels))
-                    if self._pipe_done[i] >= max(1, self._pipe_totals[i])
-                )
-            except Exception:
-                completed = 0
-            overall.update(
-                self._overall_task,
-                completed=min(completed,
-                              max(1,
-                                  len(self._pipe_labels))),
-                description=
-                f"Pipeline: {completed}/{len(self._pipe_labels)} pipes completed",
-            )
+        self._update_overall()
 
 
 class PipelineStageContext:

Store/HydrusNetwork.py

@@ -325,8 +325,11 @@ class HydrusNetwork(Store):
         ]
 
         try:
-            # Compute file hash
-            file_hash = sha256_file(file_path)
+            # Compute file hash (or use hint from kwargs to avoid redundant IO)
+            file_hash = kwargs.get("hash") or kwargs.get("file_hash")
+            if not file_hash:
+                file_hash = sha256_file(file_path)
+            
             debug(f"{self._log_prefix()} file hash: {file_hash}")
 
             # Use persistent client with session key

cmdlet/add_file.py

@@ -369,11 +369,12 @@ class Add_File(Cmdlet):
         # Many add-file flows don't emit intermediate items, so without steps the pipe can look "stuck".
         use_steps = False
         steps_started = False
-        step2_done = False
-        step3_done = False
         try:
             ui, _ = progress.ui_and_pipe_index()
             use_steps = (ui is not None) and (len(items_to_process) == 1)
+            if use_steps:
+                progress.begin_steps(5)
+                steps_started = True
         except Exception:
             use_steps = False
 
@@ -545,10 +546,8 @@ class Add_File(Cmdlet):
             temp_dir_to_cleanup: Optional[Path] = None
             delete_after_item = delete_after
             try:
-                if use_steps and (not steps_started):
-                    progress.begin_steps(5)
+                if use_steps and steps_started:
                     progress.step("resolving source")
-                    steps_started = True
 
                 media_path, file_hash, temp_dir_to_cleanup = self._resolve_source(
                     item, path_arg, pipe_obj, config, store_instance=storage_registry
@@ -560,32 +559,20 @@ class Add_File(Cmdlet):
                     failures += 1
                     continue
 
+                # Update pipe_obj with resolved path
+                pipe_obj.path = str(media_path)
+
+                # When using -path (filesystem export), allow all file types.
+                # When using -store (backend), restrict to SUPPORTED_MEDIA_EXTENSIONS.
+                allow_all_files = not (location and is_storage_backend_location)
+                if not self._validate_source(media_path, allow_all_extensions=allow_all_files):
+                    failures += 1
+                    continue
+
                 if use_steps and steps_started:
-                    progress.step("hashing file")
-
-                # Update pipe_obj with resolved path
-                pipe_obj.path = str(media_path)
-
-                # When using -path (filesystem export), allow all file types.
-                # When using -store (backend), restrict to SUPPORTED_MEDIA_EXTENSIONS.
-                allow_all_files = not (location and is_storage_backend_location)
-                if not self._validate_source(media_path, allow_all_extensions=allow_all_files):
-                    failures += 1
-                    continue
-
-                if use_steps and steps_started and (not step2_done):
+                    if not file_hash:
+                        progress.step("hashing file")
                     progress.step("ingesting file")
-                    step2_done = True
-
-                # Update pipe_obj with resolved path
-                pipe_obj.path = str(media_path)
-
-                # When using -path (filesystem export), allow all file types.
-                # When using -store (backend), restrict to SUPPORTED_MEDIA_EXTENSIONS.
-                allow_all_files = not (location and is_storage_backend_location)
-                if not self._validate_source(media_path, allow_all_extensions=allow_all_files):
-                    failures += 1
-                    continue
 
                 if provider_name:
                     if str(provider_name).strip().lower() == "matrix":
@@ -690,9 +677,8 @@ class Add_File(Cmdlet):
                 seen: set[str] = set()
                 hashes = [h for h in hashes if not (h in seen or seen.add(h))]
 
-                if use_steps and steps_started and (not step3_done):
+                if use_steps and steps_started:
                     progress.step("refreshing display")
-                    step3_done = True
 
                 refreshed_items = Add_File._try_emit_search_file_by_hashes(
                     store=str(location),
@@ -700,6 +686,7 @@ class Add_File(Cmdlet):
                     config=config,
                     store_instance=storage_registry,
                 )
+                debug(f"[add-file] Internal refresh returned refreshed_items count={len(refreshed_items) if refreshed_items else 0}")
                 if not refreshed_items:
                     # Fallback: at least show the add-file payloads as a display overlay
                     from SYS.result_table import ResultTable
@@ -756,7 +743,7 @@ class Add_File(Cmdlet):
             from cmdlet.search_file import CMDLET as search_file_cmdlet
 
             query = "hash:" + ",".join(hashes)
-            args = ["-store", str(store), query]
+            args = ["-store", str(store), "-internal-refresh", query]
             debug(f'[add-file] Refresh: search-file -store {store} "{query}"')
 
             # Run search-file under a temporary stage context so its ctx.emit() calls

cmdlet/search_file.py

@@ -258,9 +258,6 @@ class search_file(Cmdlet):
         try:
             results_list: List[Dict[str, Any]] = []
 
-            from SYS import result_table
-
-            importlib.reload(result_table)
             from SYS.result_table import ResultTable
 
             provider_text = str(provider_name or "").strip()
@@ -453,8 +450,8 @@ class search_file(Cmdlet):
         args_list = [str(arg) for arg in (args or [])]
 
         refresh_mode = any(
-            str(a).strip().lower() in {"--refresh",
-                                       "-refresh"} for a in args_list
+            str(a).strip().lower() in {"--refresh", "-refresh", "-internal-refresh"}
+            for a in args_list
         )
 
         def _format_command_title(command: str, raw_args: List[str]) -> str:
@@ -470,7 +467,7 @@ class search_file(Cmdlet):
 
             cleaned = [
                 str(a) for a in (raw_args or [])
-                if str(a).strip().lower() not in {"--refresh", "-refresh"}
+                if str(a).strip().lower() not in {"--refresh", "-refresh", "-internal-refresh"}
             ]
             if not cleaned:
                 return command
@@ -626,6 +623,10 @@ class search_file(Cmdlet):
                         continue
 
         if not library_root:
+            # Internal refreshes should not trigger config panels or stop progress.
+            if "-internal-refresh" in args_list:
+                return 1
+
             from SYS import pipeline as ctx_mod
             progress = None
             if hasattr(ctx_mod, "get_pipeline_state"):
@@ -641,19 +642,16 @@ class search_file(Cmdlet):
         # Use context manager to ensure database is always closed
         with API_folder_store(library_root) as db:
             try:
-                db.insert_worker(
-                    worker_id,
-                    "search-file",
-                    title=f"Search: {query}",
-                    description=f"Query: {query}",
-                    pipe=ctx.get_current_command_text(),
-                )
+                if "-internal-refresh" not in args_list:
+                    db.insert_worker(
+                        worker_id,
+                        "search-file",
+                        title=f"Search: {query}",
+                        description=f"Query: {query}",
+                        pipe=ctx.get_current_command_text(),
+                    )
 
                 results_list = []
-                from SYS import result_table
-                import importlib
-
-                importlib.reload(result_table)
                 from SYS.result_table import ResultTable
 
                 table = ResultTable(command_title)
@@ -802,6 +800,16 @@ class search_file(Cmdlet):
                     if found_any:
                         table.title = command_title
 
+                        # Add-file refresh quality-of-life: if exactly 1 item is being refreshed,
+                        # show the detailed item panel instead of a single-row table.
+                        if refresh_mode and len(results_list) == 1:
+                            try:
+                                from SYS.rich_display import render_item_details_panel
+                                render_item_details_panel(results_list[0])
+                                table._rendered_by_cmdlet = True
+                            except Exception:
+                                pass
+
                         if refresh_mode:
                             ctx.set_last_result_table_preserve_history(
                                 table,
@@ -918,6 +926,15 @@ class search_file(Cmdlet):
 
                     table.title = command_title
 
+                    # If exactly 1 item is being refreshed, show the detailed item panel.
+                    if refresh_mode and len(results_list) == 1:
+                        try:
+                            from SYS.rich_display import render_item_details_panel
+                            render_item_details_panel(results_list[0])
+                            table._rendered_by_cmdlet = True
+                        except Exception:
+                            pass
+
                     if refresh_mode:
                         ctx.set_last_result_table_preserve_history(table, results_list)
                     else: