From dabc8f9d511c686d0b3188dc55855f34f9da884e Mon Sep 17 00:00:00 2001 From: Nose Date: Thu, 15 Jan 2026 03:20:52 -0800 Subject: [PATCH] d --- API/folder.py | 378 +++++++++++++++++++++++++++++---------------- Store/Folder.py | 18 ++- cmdlet/add_file.py | 65 ++++++-- 3 files changed, 313 insertions(+), 148 deletions(-) diff --git a/API/folder.py b/API/folder.py index 35da959..d6f1bd1 100644 --- a/API/folder.py +++ b/API/folder.py @@ -16,6 +16,7 @@ import logging import subprocess import shutil import time +from contextlib import contextmanager from datetime import datetime from pathlib import Path, PurePosixPath from threading import RLock @@ -218,6 +219,26 @@ class API_folder_store: self._db_lock = self._shared_db_lock self._init_db() + @contextmanager + def _with_db_lock(self, *, timeout: float = 8.0): + """Acquire the shared DB lock with a bounded wait to avoid indefinite stalls.""" + locked = False + try: + locked = self._db_lock.acquire(timeout=timeout) + if not locked: + mm_debug(f"[folder-db] lock acquisition timed out after {timeout:.1f}s; proceeding unlocked") + except Exception as exc: + locked = False + mm_debug(f"[folder-db] lock acquisition failed ({exc}); proceeding unlocked") + try: + yield + finally: + if locked: + try: + self._db_lock.release() + except RuntimeError: + pass + def _normalize_input_path(self, file_path: Path) -> Path: p = expand_path(file_path).resolve() # If the path is relative to the current working directory, we check if it's meant to be in the library_root. @@ -261,7 +282,7 @@ class API_folder_store: def _init_db(self) -> None: """Initialize database connection and create tables if needed.""" - with self._db_lock: + with self._with_db_lock(): try: # Ensure the library root exists; sqlite cannot create parent dirs. try: @@ -723,7 +744,7 @@ class API_folder_store: @_db_retry() def _update_metadata_modified_time(self, file_hash: str) -> None: """Update the time_modified timestamp for a file's metadata.""" - with self._db_lock: + with self._with_db_lock(): try: cursor = self.connection.cursor() cursor.execute( @@ -770,7 +791,7 @@ class API_folder_store: attempt = 0 while True: try: - with self._db_lock: + with self._with_db_lock(): cursor = self.connection.cursor() mm_debug("[folder-db] SELECT files by file_path") @@ -877,46 +898,61 @@ class API_folder_store: def get_metadata(self, file_hash: str) -> Optional[Dict[str, Any]]: """Get metadata for a file by hash.""" - try: - with self._db_lock: - cursor = self.connection.cursor() + max_attempts = 5 + attempt = 0 + while True: + try: + with self._with_db_lock(): + cursor = self.connection.cursor() - cursor.execute( - """ - SELECT m.* FROM metadata m - WHERE m.hash = ? - """, - (file_hash, - ), + cursor.execute( + """ + SELECT m.* FROM metadata m + WHERE m.hash = ? + """, + (file_hash, + ), + ) + + row = cursor.fetchone() + if not row: + return None + + metadata = dict(row) + + # Parse JSON fields + for field in ["url", "relationships"]: + if metadata.get(field): + try: + metadata[field] = json.loads(metadata[field]) + except (json.JSONDecodeError, TypeError): + metadata[field] = [] if field == "url" else {} + + # Ensure relationships is always a dict + if metadata.get("relationships") is None: + metadata["relationships"] = {} + if not isinstance(metadata.get("relationships"), dict): + metadata["relationships"] = {} + + return metadata + except sqlite3.OperationalError as e: + msg = str(e or "").lower() + if "database is locked" in msg and attempt < max_attempts: + attempt += 1 + sleep_time = min(0.1 * (2 ** (attempt - 1)), 1.0) + time.sleep(sleep_time) + continue + logger.error( + f"Error getting metadata for hash {file_hash}: {e}", + exc_info=True ) - - row = cursor.fetchone() - if not row: - return None - - metadata = dict(row) - - # Parse JSON fields - for field in ["url", "relationships"]: - if metadata.get(field): - try: - metadata[field] = json.loads(metadata[field]) - except (json.JSONDecodeError, TypeError): - metadata[field] = [] if field == "url" else {} - - # Ensure relationships is always a dict - if metadata.get("relationships") is None: - metadata["relationships"] = {} - if not isinstance(metadata.get("relationships"), dict): - metadata["relationships"] = {} - - return metadata - except Exception as e: - logger.error( - f"Error getting metadata for hash {file_hash}: {e}", - exc_info=True - ) - return None + return None + except Exception as e: + logger.error( + f"Error getting metadata for hash {file_hash}: {e}", + exc_info=True + ) + return None def set_relationship_by_hash( self, @@ -1113,7 +1149,7 @@ class API_folder_store: file_type = get_type_from_ext(str(ext)) - with self._db_lock: + with self._with_db_lock(): cursor = self.connection.cursor() cursor.execute( """ @@ -1163,7 +1199,7 @@ class API_folder_store: tags: List[str] ) -> None: """Save metadata and tags for a file in a single transaction.""" - with self._db_lock: + with self._with_db_lock(): try: abs_path = self._normalize_input_path(file_path) db_path = self._to_db_file_path(abs_path) @@ -1247,24 +1283,36 @@ class API_folder_store: def get_tags(self, file_hash: str) -> List[str]: """Get all tags for a file by hash.""" - try: - with self._db_lock: - cursor = self.connection.cursor() + max_attempts = 5 + attempt = 0 + while True: + try: + with self._with_db_lock(): + cursor = self.connection.cursor() - cursor.execute( - """ - SELECT t.tag FROM tag t - WHERE t.hash = ? - ORDER BY t.tag - """, - (file_hash, - ), - ) + cursor.execute( + """ + SELECT t.tag FROM tag t + WHERE t.hash = ? + ORDER BY t.tag + """, + (file_hash, + ), + ) - return [row[0] for row in cursor.fetchall()] - except Exception as e: - logger.error(f"Error getting tags for hash {file_hash}: {e}", exc_info=True) - return [] + return [row[0] for row in cursor.fetchall()] + except sqlite3.OperationalError as e: + msg = str(e or "").lower() + if "database is locked" in msg and attempt < max_attempts: + attempt += 1 + sleep_time = min(0.1 * (2 ** (attempt - 1)), 1.0) + time.sleep(sleep_time) + continue + logger.error(f"Error getting tags for hash {file_hash}: {e}", exc_info=True) + return [] + except Exception as e: + logger.error(f"Error getting tags for hash {file_hash}: {e}", exc_info=True) + return [] @_db_retry() def save_tags(self, file_path: Path, tags: List[str]) -> None: @@ -1357,7 +1405,7 @@ class API_folder_store: @_db_retry() def add_tags(self, file_path: Path, tags: List[str]) -> None: """Add tags to a file.""" - with self._db_lock: + with self._with_db_lock(): try: file_hash = self.get_or_create_file_entry(file_path) cursor = self.connection.cursor() @@ -1425,7 +1473,7 @@ class API_folder_store: @_db_retry() def remove_tags(self, file_path: Path, tags: List[str]) -> None: """Remove specific tags from a file.""" - with self._db_lock: + with self._with_db_lock(): try: file_hash = self.get_or_create_file_entry(file_path) cursor = self.connection.cursor() @@ -1452,7 +1500,7 @@ class API_folder_store: @_db_retry() def add_tags_to_hash(self, file_hash: str, tags: List[str]) -> None: """Add tags to a file by hash.""" - with self._db_lock: + with self._with_db_lock(): try: cursor = self.connection.cursor() @@ -1495,7 +1543,7 @@ class API_folder_store: @_db_retry() def remove_tags_from_hash(self, file_hash: str, tags: List[str]) -> None: """Remove specific tags from a file by hash.""" - with self._db_lock: + with self._with_db_lock(): try: cursor = self.connection.cursor() @@ -1529,7 +1577,7 @@ class API_folder_store: Any] ) -> None: """Update metadata for a file by hash.""" - with self._db_lock: + with self._with_db_lock(): try: cursor = self.connection.cursor() @@ -1582,7 +1630,7 @@ class API_folder_store: related_file_path: Path to the related file rel_type: Type of relationship ('king', 'alt', 'related') """ - with self._db_lock: + with self._with_db_lock(): try: str_path = str(file_path.resolve()) str_related_path = str(related_file_path.resolve()) @@ -1734,75 +1782,141 @@ class API_folder_store: ) return [] - def get_note(self, file_hash: str) -> Optional[str]: - """Get the default note for a file by hash.""" - try: - notes = self.get_notes(file_hash) - if not notes: - return None - return notes.get("default") - except Exception as e: - logger.error(f"Error getting note for hash {file_hash}: {e}", exc_info=True) + def get_note(self, file_hash: str, name: str = "default") -> Optional[str]: + """Get a named note (default note by default) for a file hash.""" + normalized_hash = str(file_hash or "").strip().lower() + if len(normalized_hash) != 64: return None - def get_notes(self, file_hash: str) -> Dict[str, str]: - """Get all notes for a file by hash.""" - try: - cursor = self.connection.cursor() - cursor.execute( - "SELECT name, note FROM note WHERE hash = ? ORDER BY name ASC", - (file_hash, - ), - ) - out: Dict[str, - str] = {} - for name, note in cursor.fetchall() or []: - if not name: - continue - out[str(name)] = str(note or "") - return out - except Exception as e: - logger.error( - f"Error getting notes for hash {file_hash}: {e}", - exc_info=True - ) - return {} + note_name = str(name or "default").strip() or "default" + max_attempts = 5 + import time - def save_note(self, file_path: Path, note: str) -> None: - """Save the default note for a file.""" - self.set_note(file_path, "default", note) + for attempt in range(max_attempts): + try: + with self._with_db_lock(): + cursor = self.connection.cursor() + cursor.execute( + "SELECT note FROM note WHERE hash = ? AND name = ?", + (normalized_hash, + note_name), + ) + row = cursor.fetchone() + if row: + return row[0] + + if note_name != "default": + return None + + cursor.execute( + "SELECT note FROM note WHERE hash = ? ORDER BY updated_at DESC LIMIT 1", + (normalized_hash, + ), + ) + row = cursor.fetchone() + return row[0] if row else None + except sqlite3.OperationalError as e: + msg = str(e or "").lower() + if "database is locked" in msg and attempt < (max_attempts - 1): + sleep_time = min(0.1 * (2 ** attempt), 1.0) + time.sleep(sleep_time) + continue + logger.error( + f"Error getting note for hash {file_hash}: {e}", + exc_info=True + ) + return None + except Exception as e: + logger.error( + f"Error getting note for hash {file_hash}: {e}", + exc_info=True + ) + return None + return None + + def set_note_by_hash(self, file_hash: str, name: str, note: str) -> None: + """Set a named note using a known file hash (no re-hash).""" + note_name = str(name or "").strip() + normalized_hash = str(file_hash or "").strip().lower() + if not note_name: + raise ValueError("Note name is required") + if len(normalized_hash) != 64: + raise ValueError("File hash must be a 64-character hex string") + + max_attempts = 5 + import time + + for attempt in range(max_attempts): + try: + with self._with_db_lock(): + cursor = self.connection.cursor() + + cursor.execute( + "SELECT 1 FROM file WHERE hash = ?", + (normalized_hash, + ), + ) + exists = cursor.fetchone() is not None + if not exists: + raise ValueError( + f"Hash {normalized_hash} not found in file table" + ) + + cursor.execute( + """ + INSERT INTO note (hash, name, note) + VALUES (?, ?, ?) + ON CONFLICT(hash, name) DO UPDATE SET + note = excluded.note, + updated_at = CURRENT_TIMESTAMP + """, + (normalized_hash, + note_name, + note), + ) + self.connection.commit() + logger.debug( + f"Saved note '{note_name}' for hash {normalized_hash}" + ) + return + except sqlite3.OperationalError as e: + msg = str(e or "").lower() + if "database is locked" in msg and attempt < (max_attempts - 1): + sleep_time = min(0.1 * (2 ** attempt), 1.0) + time.sleep(sleep_time) + continue + logger.error( + f"Error saving note for hash {normalized_hash}: {e}", + exc_info=True + ) + raise + except Exception as e: + logger.error( + f"Error saving note for hash {normalized_hash}: {e}", + exc_info=True + ) + raise def set_note(self, file_path: Path, name: str, note: str) -> None: - """Set a named note for a file.""" - with self._db_lock: - try: - note_name = str(name or "").strip() - if not note_name: - raise ValueError("Note name is required") + """Set a named note for a file path, computing hash if needed.""" + note_name = str(name or "").strip() + if not note_name: + raise ValueError("Note name is required") - file_hash = self.get_or_create_file_entry(file_path) - cursor = self.connection.cursor() - cursor.execute( - """ - INSERT INTO note (hash, name, note) - VALUES (?, ?, ?) - ON CONFLICT(hash, name) DO UPDATE SET - note = excluded.note, - updated_at = CURRENT_TIMESTAMP - """, - (file_hash, - note_name, - note), - ) - self.connection.commit() - logger.debug(f"Saved note '{note_name}' for {file_path}") - except Exception as e: - logger.error(f"Error saving note for {file_path}: {e}", exc_info=True) - raise + try: + file_hash = self.get_or_create_file_entry(file_path) + self.set_note_by_hash(file_hash, note_name, note) + except Exception as e: + logger.error(f"Error saving note for {file_path}: {e}", exc_info=True) + raise + + def save_note(self, file_path: Path, note: str, name: str = "default") -> None: + """Backward-compatible helper to store a note for a file path.""" + self.set_note(file_path, name, note) def delete_note(self, file_hash: str, name: str) -> None: """Delete a named note for a file by hash.""" - with self._db_lock: + with self._with_db_lock(): try: note_name = str(name or "").strip() if not note_name: @@ -1854,7 +1968,7 @@ class API_folder_store: def search_hash(self, file_hash: str) -> Optional[Path]: """Search for a file by hash.""" try: - with self._db_lock: + with self._with_db_lock(): cursor = self.connection.cursor() cursor.execute( @@ -1939,7 +2053,7 @@ class API_folder_store: backlinks in other files so no file retains dangling references to the deleted hash. """ - with self._db_lock: + with self._with_db_lock(): try: abs_path = self._normalize_input_path(file_path) str_path = self._to_db_file_path(abs_path) @@ -2048,7 +2162,7 @@ class API_folder_store: pipe: Optional[str] = None, ) -> int: """Insert a new worker entry into the database.""" - with self._db_lock: + with self._with_db_lock(): try: cursor = self.connection.cursor() cursor.execute( @@ -2085,7 +2199,7 @@ class API_folder_store: def update_worker(self, worker_id: str, **kwargs) -> bool: """Update worker entry with given fields.""" - with self._db_lock: + with self._with_db_lock(): try: allowed_fields = { "status", @@ -2129,7 +2243,7 @@ class API_folder_store: def update_worker_status(self, worker_id: str, status: str) -> int: """Update worker status and return its database ID.""" - with self._db_lock: + with self._with_db_lock(): try: cursor = self.connection.cursor() @@ -2208,7 +2322,7 @@ class API_folder_store: def delete_worker(self, worker_id: str) -> bool: """Delete a worker entry.""" - with self._db_lock: + with self._with_db_lock(): try: cursor = self.connection.cursor() cursor.execute("DELETE FROM worker WHERE worker_id = ?", @@ -2316,7 +2430,7 @@ class API_folder_store: """Append text to a worker's stdout log and timeline.""" if not text: return True - with self._db_lock: + with self._with_db_lock(): try: # Check if connection is valid if not self.connection: diff --git a/Store/Folder.py b/Store/Folder.py index 9f3f927..e93c659 100644 --- a/Store/Folder.py +++ b/Store/Folder.py @@ -2002,18 +2002,26 @@ class Folder(Store): if not self._location: return False file_hash = str(file_identifier or "").strip().lower() + note_name = str(name or "").strip() if not _normalize_hash(file_hash): return False - - file_path = self.get_file(file_hash, **kwargs) - if not file_path or not isinstance(file_path, - Path) or not file_path.exists(): + if not note_name: return False with API_folder_store(Path(self._location)) as db: + setter_hash = getattr(db, "set_note_by_hash", None) + if callable(setter_hash): + setter_hash(file_hash, note_name, str(text)) + return True + + file_path = self.get_file(file_hash, **kwargs) + if not file_path or not isinstance(file_path, + Path) or not file_path.exists(): + return False + setter = getattr(db, "set_note", None) if callable(setter): - setter(file_path, str(name), str(text)) + setter(file_path, note_name, str(text)) return True db.save_note(file_path, str(text)) return True diff --git a/cmdlet/add_file.py b/cmdlet/add_file.py index f60558c..457d4f0 100644 --- a/cmdlet/add_file.py +++ b/cmdlet/add_file.py @@ -691,14 +691,35 @@ class Add_File(Cmdlet): # Fallback: at least show the add-file payloads as a display overlay from SYS.result_table import ResultTable - table = ResultTable("Result") - for payload in collected_payloads: - table.add_result(payload) - ctx.set_last_result_table_overlay( - table, - collected_payloads, - subject=collected_payloads - ) + # If this was a single-item ingest, render the detailed item display + # directly from the payload to avoid DB refresh contention. + detail_rendered = False + if len(collected_payloads) == 1: + try: + from SYS.rich_display import render_item_details_panel + + render_item_details_panel(collected_payloads[0]) + table = ResultTable("Result") + table.add_result(collected_payloads[0]) + setattr(table, "_rendered_by_cmdlet", True) + ctx.set_last_result_table_overlay( + table, + collected_payloads, + subject=collected_payloads[0] + ) + detail_rendered = True + except Exception: + detail_rendered = False + + if not detail_rendered: + table = ResultTable("Result") + for payload in collected_payloads: + table.add_result(payload) + ctx.set_last_result_table_overlay( + table, + collected_payloads, + subject=collected_payloads + ) except Exception: pass @@ -1875,6 +1896,12 @@ class Add_File(Cmdlet): except Exception: hydrus_like_backend = False + is_folder_backend = False + try: + is_folder_backend = type(backend).__name__ == "Folder" + except Exception: + is_folder_backend = False + # Prepare metadata from pipe_obj and sidecars tags, url, title, f_hash = Add_File._prepare_metadata( result, media_path, pipe_obj, config @@ -1996,7 +2023,20 @@ class Add_File(Cmdlet): # For Hydrus, get_file() returns a browser URL (often with an access key) and should # only be invoked by explicit user commands (e.g. get-file). try: - if type(backend).__name__ == "Folder": + if is_folder_backend: + # Avoid extra DB round-trips for Folder; we can derive the stored path. + hash_for_path: Optional[str] = None + if isinstance(file_identifier, str) and len(file_identifier) == 64: + hash_for_path = file_identifier + elif f_hash and isinstance(f_hash, str) and len(f_hash) == 64: + hash_for_path = f_hash + if hash_for_path: + suffix = media_path.suffix if media_path else "" + filename = f"{hash_for_path}{suffix}" if suffix else hash_for_path + location_path = getattr(backend, "_location", None) + if location_path: + stored_path = str(Path(location_path) / filename) + else: maybe_path = backend.get_file(file_identifier) if isinstance(maybe_path, Path): stored_path = str(maybe_path) @@ -2050,7 +2090,9 @@ class Add_File(Cmdlet): pass else: try: - backend.add_url(resolved_hash, list(url)) + # Folder.add_file already persists URLs, avoid extra DB traffic here. + if not is_folder_backend: + backend.add_url(resolved_hash, list(url)) except Exception: pass @@ -2107,7 +2149,8 @@ class Add_File(Cmdlet): meta: Dict[str, Any] = {} try: - meta = backend.get_metadata(resolved_hash) or {} + if not is_folder_backend: + meta = backend.get_metadata(resolved_hash) or {} except Exception: meta = {}