From 1dbaabac73f572e3a877fd42214c6cd4007b2b52 Mon Sep 17 00:00:00 2001 From: Nose Date: Sat, 31 Jan 2026 19:57:09 -0800 Subject: [PATCH] h --- SYS/background_notifier.py | 14 ++++---- SYS/cli_parsing.py | 9 ++--- SYS/cmdlet_catalog.py | 36 ++++++++++++-------- SYS/config.py | 59 +++++++++++++++++---------------- SYS/database.py | 68 ++++++++++++++++++++++++-------------- SYS/html_table.py | 19 ++++++----- SYS/pipeline.py | 8 +++-- 7 files changed, 125 insertions(+), 88 deletions(-) diff --git a/SYS/background_notifier.py b/SYS/background_notifier.py index 39ab9d3..2785ecf 100644 --- a/SYS/background_notifier.py +++ b/SYS/background_notifier.py @@ -12,6 +12,8 @@ from __future__ import annotations from typing import Any, Callable, Dict, Optional, Set from SYS.logger import log, debug +import logging +logger = logging.getLogger(__name__) class BackgroundNotifier: @@ -103,7 +105,7 @@ class BackgroundNotifier: try: self.output(line) except Exception: - pass + logger.exception("BackgroundNotifier failed to output overlay line for worker %s", worker_id) self._last_state.pop(worker_id, None) self.session_worker_ids.discard(worker_id) @@ -120,7 +122,7 @@ class BackgroundNotifier: try: self.output(line) except Exception: - pass + logger.exception("BackgroundNotifier failed to output terminal-only line for worker %s", worker_id) # Stop tracking this worker after terminal notification self.session_worker_ids.discard(worker_id) continue @@ -148,7 +150,7 @@ class BackgroundNotifier: try: self.output(line) except Exception: - pass + logger.exception("BackgroundNotifier failed to output line for worker %s", worker_id) if self.overlay_mode: try: @@ -156,7 +158,7 @@ class BackgroundNotifier: if overlay_active_workers == 0: self.output("") except Exception: - pass + logger.exception("BackgroundNotifier failed to clear overlay for session") def ensure_background_notifier( @@ -201,6 +203,6 @@ def ensure_background_notifier( ) try: manager._background_notifier = notifier # type: ignore[attr-defined] - except Exception: - pass + except Exception as exc: + logger.exception("Failed to attach background notifier to manager: %s", exc) return notifier diff --git a/SYS/cli_parsing.py b/SYS/cli_parsing.py index 8508833..dac8d6f 100644 --- a/SYS/cli_parsing.py +++ b/SYS/cli_parsing.py @@ -8,6 +8,7 @@ from __future__ import annotations import re from typing import Any, Callable, Dict, List, Optional, Set, Tuple +from SYS.logger import debug # Prompt-toolkit lexer types are optional at import time; fall back to lightweight # stubs if prompt_toolkit is not available so imports remain safe for testing. @@ -130,8 +131,8 @@ class SelectionFilterSyntax: try: if SelectionSyntax.parse(str(token)) is not None: return None - except Exception: - pass + except Exception as exc: + debug("SelectionSyntax.parse failed during filter detection: %s", exc, exc_info=True) raw = str(token)[1:].strip() if not raw: @@ -211,8 +212,8 @@ class SelectionFilterSyntax: for k in ("title", "path", "detail", "provider", "store", "table"): try: _set(k, getattr(item, k, None)) - except Exception: - pass + except Exception as exc: + debug("SelectionFilterSyntax: failed to _set attribute %s on item: %s", k, exc, exc_info=True) return out diff --git a/SYS/cmdlet_catalog.py b/SYS/cmdlet_catalog.py index eef9a62..cdb76ad 100644 --- a/SYS/cmdlet_catalog.py +++ b/SYS/cmdlet_catalog.py @@ -3,6 +3,8 @@ from __future__ import annotations from importlib import import_module from types import ModuleType from typing import Any, Dict, List, Optional +import logging +logger = logging.getLogger(__name__) try: from .config import get_local_storage_path @@ -31,7 +33,8 @@ def _get_cmdlet_package() -> Optional[ModuleType]: return _cmdlet_pkg try: _cmdlet_pkg = import_module("cmdlet") - except Exception: + except Exception as exc: + logger.exception("Failed to import cmdlet package: %s", exc) _cmdlet_pkg = None return _cmdlet_pkg @@ -52,8 +55,8 @@ def ensure_registry_loaded(force: bool = False) -> None: if callable(ensure_fn): try: ensure_fn(force=force) - except Exception: - pass + except Exception as exc: + logger.exception("ensure_registry_loaded: ensure_cmdlet_modules_loaded failed: %s", exc) def _normalize_mod_name(mod_name: str) -> str: @@ -76,7 +79,8 @@ def import_cmd_module(mod_name: str): return import_module(qualified) except ModuleNotFoundError: continue - except Exception: + except Exception as exc: + logger.exception("Unexpected error importing module %s: %s", qualified, exc) continue return None @@ -127,7 +131,8 @@ def get_cmdlet_metadata( if owner_mod: owner = import_module(owner_mod) data = getattr(owner, "CMDLET", None) - except Exception: + except Exception as exc: + logger.exception("Registry fallback failed while resolving cmdlet %s: %s", cmd_name, exc) data = None if not data: @@ -303,14 +308,16 @@ def get_cmdlet_arg_choices( from SYS.config import load_config config = load_config() - except Exception: + except Exception as exc: + logger.exception("Failed to load config for matrix default choices: %s", exc) config = config or {} matrix_conf = {} try: providers = config.get("provider") or {} matrix_conf = providers.get("matrix") or {} - except Exception: + except Exception as exc: + logger.exception("Failed to read matrix provider config: %s", exc) matrix_conf = {} raw = None @@ -328,7 +335,8 @@ def get_cmdlet_arg_choices( import re ids = [p.strip() for p in re.split(r"[,\s]+", text) if p and p.strip()] - except Exception: + except Exception as exc: + logger.exception("Failed to parse matrix room ids from config: %r", raw) ids = [] if ids: @@ -350,12 +358,12 @@ def get_cmdlet_arg_choices( choices.append(name or rid) if choices: return choices - except Exception: - pass - except Exception: - pass - except Exception: - pass + except Exception as exc: + logger.exception("Matrix provider failed while listing rooms: %s", exc) + except Exception as exc: + logger.exception("Failed to import Matrix provider or initialize: %s", exc) + except Exception as exc: + logger.exception("Failed to resolve matrix rooms: %s", exc) # Fallback: return raw ids as choices return ids diff --git a/SYS/config.py b/SYS/config.py index 243a02d..bb7b89d 100644 --- a/SYS/config.py +++ b/SYS/config.py @@ -15,6 +15,8 @@ from copy import deepcopy from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from SYS.logger import log +import logging +logger = logging.getLogger(__name__) from SYS.utils import expand_path from SYS.database import db, get_config_all, save_config_value, rows_to_config @@ -186,16 +188,16 @@ def resolve_output_dir(config: Dict[str, Any]) -> Path: # Verify we can access it (not a system directory with permission issues) if path.exists() or path.parent.exists(): return path - except Exception: - pass + except Exception as exc: + logger.debug("resolve_output_dir: failed to expand temp value %r: %s", temp_value, exc, exc_info=True) # Then try outfile setting outfile_value = config.get("outfile") if outfile_value: try: return expand_path(outfile_value) - except Exception: - pass + except Exception as exc: + logger.debug("resolve_output_dir: failed to expand outfile value %r: %s", outfile_value, exc, exc_info=True) # Fallback to system temp directory return Path(tempfile.gettempdir()) @@ -313,8 +315,8 @@ def resolve_cookies_path( values: list[Any] = [] try: values.append(config.get("cookies")) - except Exception: - pass + except Exception as exc: + logger.debug("resolve_cookies_path: failed to read top-level cookies: %s", exc, exc_info=True) try: tool = config.get("tool") @@ -323,16 +325,16 @@ def resolve_cookies_path( if isinstance(ytdlp, dict): values.append(ytdlp.get("cookies")) values.append(ytdlp.get("cookiefile")) - except Exception: - pass + except Exception as exc: + logger.debug("resolve_cookies_path: failed to read tool.ytdlp cookies: %s", exc, exc_info=True) try: ytdlp_block = config.get("ytdlp") if isinstance(ytdlp_block, dict): values.append(ytdlp_block.get("cookies")) values.append(ytdlp_block.get("cookiefile")) - except Exception: - pass + except Exception as exc: + logger.debug("resolve_cookies_path: failed to read ytdlp cookies block: %s", exc, exc_info=True) base_dir = script_dir or SCRIPT_DIR for value in values: @@ -488,7 +490,7 @@ def load_config() -> Dict[str, Any]: # Forensics disabled: audit/mismatch/backup detection removed to simplify code. except Exception: - pass + logger.exception("Failed to build config load summary from %s", db.db_path) return db_config _LAST_SAVED_CONFIG = {} @@ -518,8 +520,8 @@ def _acquire_save_lock(timeout: float = _SAVE_LOCK_TIMEOUT): "ts": time.time(), "cmdline": " ".join(sys.argv), })) - except Exception: - pass + except Exception as exc: + logger.exception("Failed to write save lock owner metadata %s: %s", lock_dir, exc) return lock_dir except FileExistsError: # Check for stale lock @@ -533,8 +535,8 @@ def _acquire_save_lock(timeout: float = _SAVE_LOCK_TIMEOUT): import shutil shutil.rmtree(lock_dir) continue - except Exception: - pass + except Exception as exc: + logger.exception("Failed to remove stale save lock dir %s", lock_dir) else: # No owner file; if directory is old enough consider it stale try: @@ -542,10 +544,10 @@ def _acquire_save_lock(timeout: float = _SAVE_LOCK_TIMEOUT): import shutil shutil.rmtree(lock_dir) continue - except Exception: - pass - except Exception: - pass + except Exception as exc: + logger.exception("Failed to stat/remove stale save lock dir %s", lock_dir) + except Exception as exc: + logger.exception("Failed to inspect save lock directory %s: %s", lock_dir, exc) if time.time() - start > timeout: raise ConfigSaveConflict("Save lock busy; could not acquire in time") time.sleep(0.1) @@ -558,10 +560,10 @@ def _release_save_lock(lock_dir: Path) -> None: if owner.exists(): owner.unlink() except Exception: - pass + logger.exception("Failed to remove save lock owner file %s", owner) lock_dir.rmdir() except Exception: - pass + logger.exception("Failed to release save lock directory %s", lock_dir) def save_config(config: Dict[str, Any]) -> int: @@ -681,8 +683,8 @@ def save_config(config: Dict[str, Any]) -> int: try: if lock_dir is not None and lock_dir.exists(): _release_save_lock(lock_dir) - except Exception: - pass + except Exception as exc: + logger.exception("Failed to release save lock during save flow: %s", exc) break except sqlite3.OperationalError as exc: @@ -694,8 +696,8 @@ def save_config(config: Dict[str, Any]) -> int: try: if lock_dir is not None and lock_dir.exists(): _release_save_lock(lock_dir) - except Exception: - pass + except Exception as exc: + logger.exception("Failed to release save lock after DB write failure: %s", exc) raise delay = _CONFIG_SAVE_RETRY_DELAY * attempts log(f"Database locked; retry {attempts}/{_CONFIG_SAVE_MAX_RETRIES} in {delay:.2f}s") @@ -705,8 +707,8 @@ def save_config(config: Dict[str, Any]) -> int: try: if lock_dir is not None and lock_dir.exists(): _release_save_lock(lock_dir) - except Exception: - pass + except Exception as exc: + logger.exception("Failed to release save lock after CRITICAL configuration save failure: %s", exc) raise clear_config_cache() @@ -762,7 +764,8 @@ def save_config_and_verify(config: Dict[str, Any], retries: int = 3, delay: floa break elif isinstance(srv, str) and srv.strip(): expected_key = srv.strip() - except Exception: + except Exception as exc: + logger.debug("Failed to determine expected key for save verification: %s", exc, exc_info=True) expected_key = None last_exc: Exception | None = None diff --git a/SYS/database.py b/SYS/database.py index 0904275..fd8c3da 100644 --- a/SYS/database.py +++ b/SYS/database.py @@ -12,6 +12,8 @@ from contextlib import contextmanager import time import datetime from SYS.logger import debug, log +import logging +logger = logging.getLogger(__name__) # DB execute retry settings (for transient 'database is locked' errors) _DB_EXEC_RETRY_MAX = 5 @@ -29,8 +31,8 @@ def _resolve_root_dir() -> Path: candidate = Path(env_root).expanduser().resolve() if candidate.exists(): return candidate - except Exception: - pass + except Exception as exc: + logger.debug("_resolve_root_dir: failed to resolve env_root %r: %s", env_root, exc, exc_info=True) cwd = Path.cwd().resolve() for base in [cwd, *cwd.parents]: @@ -173,23 +175,24 @@ class Database: try: if not self.conn.in_transaction: self.conn.rollback() - except Exception: - pass + except Exception as exc: + logger.exception("Rollback failed while retrying locked execute: %s", exc) time.sleep(delay) continue # Not recoverable or out of retries if not self.conn.in_transaction: try: self.conn.rollback() - except Exception: - pass + except Exception as exc: + logger.exception("Rollback failed in non-recoverable execute path: %s", exc) raise - except Exception: + except Exception as exc: if not self.conn.in_transaction: try: self.conn.rollback() - except Exception: - pass + except Exception as rb_exc: + logger.exception("Rollback failed during unexpected execute exception: %s", rb_exc) + logger.exception("Unexpected exception during DB execute: %s", exc) raise def executemany(self, query: str, param_list: List[tuple]): @@ -211,22 +214,23 @@ class Database: try: if not self.conn.in_transaction: self.conn.rollback() - except Exception: - pass + except Exception as exc: + logger.exception("Rollback failed while retrying locked executemany: %s", exc) time.sleep(delay) continue if not self.conn.in_transaction: try: self.conn.rollback() - except Exception: - pass + except Exception as exc: + logger.exception("Rollback failed in non-recoverable executemany path: %s", exc) raise - except Exception: + except Exception as exc: if not self.conn.in_transaction: try: self.conn.rollback() - except Exception: - pass + except Exception as rb_exc: + logger.exception("Rollback failed during unexpected executemany exception: %s", rb_exc) + logger.exception("Unexpected exception during DB executemany: %s", exc) raise @contextmanager @@ -255,7 +259,7 @@ class Database: try: self._conn_lock.release() except Exception: - pass + logger.exception("Failed to release DB connection lock") def fetchall(self, query: str, params: tuple = ()): with self._conn_lock: @@ -321,17 +325,28 @@ def _log_worker_loop() -> None: fallback_file = fallback_dir / "log_fallback.txt" with fallback_file.open("a", encoding="utf-8") as fh: fh.write(f"{datetime.datetime.utcnow().isoformat()}Z [{level}] {module}: {message}\n") - except Exception: + except Exception as exc: # Last resort: print to stderr try: log(f"ERROR: Could not persist log message: {level} {module} {message}") except Exception: pass + try: + import sys as _sys, traceback as _tb + _sys.stderr.write(f"CRITICAL: Could not persist log message to fallback file: {exc}\n") + _tb.print_exc(file=_sys.stderr) + except Exception: + pass finally: try: _LOG_QUEUE.task_done() - except Exception: - pass + except Exception as exc: + try: + import sys as _sys, traceback as _tb + _sys.stderr.write(f"CRITICAL: Failed to mark log task done: {exc}\n") + _tb.print_exc(file=_sys.stderr) + except Exception: + pass def _ensure_log_thread() -> None: @@ -433,7 +448,8 @@ def rows_to_config(rows) -> Dict[str, Any]: parsed_val = val except Exception: parsed_val = val - except Exception: + except Exception as exc: + logger.debug("rows_to_config: failed to parse value for key %s; using raw value", key, exc_info=True) parsed_val = val if cat == 'global': @@ -469,7 +485,8 @@ def insert_worker(worker_id: str, worker_type: str, title: str = "", description (worker_id, worker_type, title, description) ) return True - except Exception: + except Exception as exc: + logger.exception("Failed to insert worker %s: %s", worker_id, exc) return False def update_worker(worker_id: str, **kwargs) -> bool: @@ -495,7 +512,8 @@ def update_worker(worker_id: str, **kwargs) -> bool: try: db.execute(query, tuple(vals)) return True - except Exception: + except Exception as exc: + logger.exception("Failed to update worker %s: %s", worker_id, exc) return False def append_worker_stdout(worker_id: str, content: str, channel: str = 'stdout'): @@ -504,8 +522,8 @@ def append_worker_stdout(worker_id: str, content: str, channel: str = 'stdout'): "INSERT INTO worker_stdout (worker_id, channel, content) VALUES (?, ?, ?)", (worker_id, channel, content) ) - except Exception: - pass + except Exception as exc: + logger.exception("Failed to append worker stdout for %s", worker_id) def get_worker_stdout(worker_id: str, channel: Optional[str] = None) -> str: query = "SELECT content FROM worker_stdout WHERE worker_id = ?" diff --git a/SYS/html_table.py b/SYS/html_table.py index 1d0cc40..1ca9473 100644 --- a/SYS/html_table.py +++ b/SYS/html_table.py @@ -19,6 +19,7 @@ from typing import Any, Dict, List, Optional, Tuple from lxml import html as lxml_html from urllib.parse import urljoin import re +from SYS.logger import debug # Default xpaths for candidate result containers _DEFAULT_XPATHS = [ @@ -50,8 +51,8 @@ def _text_or_img_title(el) -> str: imgs = el.xpath('.//img/@title') if imgs: return str(imgs[0]).strip() - except Exception: - pass + except Exception as exc: + debug("Failed to retrieve img title from element: %s", exc, exc_info=True) return (el.text_content() or "").strip() @@ -66,7 +67,8 @@ def find_candidate_nodes(doc_or_html: Any, xpaths: Optional[List[str]] = None) - found = doc.xpath(xp) if found: return list(found), xp - except Exception: + except Exception as exc: + debug("Failed to execute xpath %s: %s", xp, exc, exc_info=True) continue return [], None @@ -214,7 +216,8 @@ def extract_records(doc_or_html: Any, base_url: Optional[str] = None, xpaths: Op df = max(dfs, key=lambda d: getattr(d, "shape", (len(getattr(d, 'index', [])), 0))[0]) try: rows = df.to_dict("records") - except Exception: + except Exception as exc: + debug("pandas HTML table parse: df.to_dict('records') failed: %s", exc, exc_info=True) # Some DataFrame-like objects may have slightly different APIs rows = [dict(r) for r in df] @@ -240,13 +243,13 @@ def extract_records(doc_or_html: Any, base_url: Optional[str] = None, xpaths: Op href = anchors.get(rec["title"]) if href: rec["path"] = urljoin(base_url, href) if base_url else href - except Exception: - pass + except Exception as exc: + debug("pandas: failed to recover anchor hrefs for table rows: %s", exc, exc_info=True) return records, "pandas" - except Exception: + except Exception as exc: # Pandas not present or parsing failed; fall back to node parsing - pass + debug("pandas: not available or parsing failed: %s", exc, exc_info=True) # Fallback to node-based parsing nodes, chosen = find_candidate_nodes(doc_or_html, xpaths=xpaths) diff --git a/SYS/pipeline.py b/SYS/pipeline.py index deac443..49732fb 100644 --- a/SYS/pipeline.py +++ b/SYS/pipeline.py @@ -52,7 +52,8 @@ def suspend_live_progress(): try: ui.pause() paused = True - except Exception: + except Exception as exc: + logger.exception("Failed to pause live progress UI: %s", exc) paused = False yield finally: @@ -1427,7 +1428,8 @@ class PipelineExecutor: logger.exception("is_known_provider_name predicate failed for key %s; falling back", key) try: provider = get_provider(key, config) - except Exception: + except Exception as exc: + logger.exception("Failed to load provider '%s' during selector resolution: %s", key, exc) continue selector = getattr(provider, "selector", None) if selector is None: @@ -1439,7 +1441,7 @@ class PipelineExecutor: stage_is_last=True) ) except Exception as exc: - print(f"{key} selector failed: {exc}\n") + logger.exception("%s selector failed during selection: %s", key, exc) return True if handled: return True