goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

j

Browse Source
This commit is contained in:
Nose
2026-01-02 02:28:59 -08:00
parent deb05c0d44
commit 6e9a0c28ff
13 changed files with 1402 additions and 2334 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
SYS/cmdlet_api.py Normal file
View File

@@ -0,0 +1,220 @@
from __future__ import annotations
import contextlib
import io
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Optional, Sequence
from SYS import pipeline as ctx
from SYS.models import PipelineStageContext
from SYS.rich_display import capture_rich_output
CmdletCallable = Callable[[Any, Sequence[str], Dict[str, Any]], int]
@dataclass(slots=True)
class CmdletRunResult:
"""Programmatic result for a single cmdlet invocation."""
name: str
args: Sequence[str]
exit_code: int = 0
emitted: List[Any] = field(default_factory=list)
# Best-effort: cmdlets can publish tables/items via pipeline state even when
# they don't emit pipeline items.
result_table: Optional[Any] = None
result_items: List[Any] = field(default_factory=list)
result_subject: Optional[Any] = None
stdout: str = ""
stderr: str = ""
error: Optional[str] = None
def _normalize_cmd_name(name: str) -> str:
return str(name or "").replace("_", "-").strip().lower()
def resolve_cmdlet(cmd_name: str) -> Optional[CmdletCallable]:
"""Resolve a cmdlet callable by name from the registry (aliases supported)."""
try:
from SYS.cmdlet_catalog import ensure_registry_loaded
ensure_registry_loaded()
except Exception:
pass
try:
import cmdlet as cmdlet_pkg
return cmdlet_pkg.get(cmd_name)
except Exception:
return None
def run_cmdlet(
cmd: str | CmdletCallable,
args: Sequence[str] | None,
config: Dict[str, Any],
*,
piped: Any = None,
isolate: bool = True,
capture_output: bool = True,
stage_index: int = 0,
total_stages: int = 1,
pipe_index: Optional[int] = None,
worker_id: Optional[str] = None,
) -> CmdletRunResult:
"""Run a single cmdlet programmatically and return structured results.
This is intended for TUI/webapp consumers that want cmdlet behavior without
going through the interactive CLI loop.
Notes:
- When `isolate=True` (default) this runs inside `ctx.new_pipeline_state()` so
global CLI pipeline state is not mutated.
- Output capturing covers both normal `print()` and Rich output via
`capture_rich_output()`.
"""
normalized_args: Sequence[str] = list(args or [])
if isinstance(cmd, str):
name = _normalize_cmd_name(cmd)
cmd_fn = resolve_cmdlet(name)
else:
name = getattr(cmd, "__name__", "cmdlet")
cmd_fn = cmd
result = CmdletRunResult(name=name, args=normalized_args)
if not callable(cmd_fn):
result.exit_code = 1
result.error = f"Unknown command: {name}"
result.stderr = result.error
return result
stage_ctx = PipelineStageContext(
stage_index=int(stage_index),
total_stages=int(total_stages),
pipe_index=pipe_index,
worker_id=worker_id,
)
stdout_buffer = io.StringIO()
stderr_buffer = io.StringIO()
stage_text = " ".join([name, *list(normalized_args)]).strip()
state_cm = ctx.new_pipeline_state() if isolate else contextlib.nullcontext()
with state_cm:
# Keep behavior predictable: start from a clean slate.
try:
ctx.reset()
except Exception:
pass
try:
ctx.set_stage_context(stage_ctx)
except Exception:
pass
try:
ctx.set_current_cmdlet_name(name)
except Exception:
pass
try:
ctx.set_current_stage_text(stage_text)
except Exception:
pass
try:
ctx.set_current_command_text(stage_text)
except Exception:
pass
try:
run_cm = (
capture_rich_output(stdout=stdout_buffer, stderr=stderr_buffer)
if capture_output
else contextlib.nullcontext()
)
with run_cm:
with (
contextlib.redirect_stdout(stdout_buffer)
if capture_output
else contextlib.nullcontext()
):
with (
contextlib.redirect_stderr(stderr_buffer)
if capture_output
else contextlib.nullcontext()
):
result.exit_code = int(cmd_fn(piped, list(normalized_args), config))
except Exception as exc:
result.exit_code = 1
result.error = f"{type(exc).__name__}: {exc}"
finally:
result.stdout = stdout_buffer.getvalue()
result.stderr = stderr_buffer.getvalue()
# Prefer cmdlet emits (pipeline semantics).
try:
result.emitted = list(stage_ctx.emits or [])
except Exception:
result.emitted = []
# Mirror CLI behavior: if cmdlet emitted items and there is no overlay table,
# make emitted items the last result items for downstream consumers.
try:
has_overlay = bool(ctx.get_display_table())
except Exception:
has_overlay = False
if result.emitted and not has_overlay:
try:
ctx.set_last_result_items_only(list(result.emitted))
except Exception:
pass
# Best-effort snapshot of visible results.
try:
result.result_table = (
ctx.get_display_table() or ctx.get_current_stage_table() or ctx.get_last_result_table()
)
except Exception:
result.result_table = None
try:
result.result_items = list(ctx.get_last_result_items() or [])
except Exception:
result.result_items = []
try:
result.result_subject = ctx.get_last_result_subject()
except Exception:
result.result_subject = None
# Cleanup stage-local markers.
try:
ctx.clear_current_stage_text()
except Exception:
pass
try:
ctx.clear_current_cmdlet_name()
except Exception:
pass
try:
ctx.clear_current_command_text()
except Exception:
pass
try:
ctx.set_stage_context(None)
except Exception:
pass
return result

71
SYS/metadata.py
View File

@@ -512,36 +512,46 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
if not base_path.exists():
continue
# Ensure file entry exists
file_id: Optional[int] = None
# Ensure file entry exists (folder store schema is keyed by hash).
file_hash_value: Optional[str] = None
if sha256_file and base_path.exists():
try:
file_hash_value = sha256_file(base_path)
except Exception:
file_hash_value = None
if not file_hash_value:
continue
try:
db_file_path = (
db._to_db_file_path(base_path) # type: ignore[attr-defined]
if hasattr(db, "_to_db_file_path")
else str(base_path)
)
except Exception:
db_file_path = str(base_path)
try:
file_modified = float(base_path.stat().st_mtime)
except Exception:
file_modified = None
try:
cursor = db.connection.cursor() if db.connection else None
if cursor:
cursor.execute(
"SELECT id FROM files WHERE file_path = ?",
(str(base_path),
)
"SELECT hash FROM file WHERE file_path = ?",
(str(db_file_path),),
)
result = cursor.fetchone()
file_id = result[0] if result else None
except Exception:
file_id = None
if not file_id:
try:
cursor = db.connection.cursor() if db.connection else None
if cursor:
if not result:
cursor.execute(
'INSERT INTO files (file_path, indexed_at, updated_at) VALUES (?, datetime("now"), datetime("now"))',
(str(base_path),
),
"INSERT INTO file (hash, file_path, file_modified) VALUES (?, ?, ?)",
(file_hash_value, str(db_file_path), file_modified),
)
db.connection.commit()
file_id = cursor.lastrowid
except Exception:
continue
if not file_id:
except Exception:
continue
if sidecar_path.suffix == ".tag":
@@ -557,15 +567,9 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
try:
cursor = db.connection.cursor() if db.connection else None
if cursor:
file_hash_value: Optional[str] = None
if hasattr(db, "get_file_hash"):
try:
file_hash_value = db.get_file_hash(file_id)
except Exception:
file_hash_value = None
for tag in tags:
cursor.execute(
"INSERT OR IGNORE INTO tags (hash, tag) VALUES (?, ?)",
"INSERT OR IGNORE INTO tag (hash, tag) VALUES (?, ?)",
(file_hash_value,
tag),
)
@@ -608,13 +612,15 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
except Exception:
pass
if not hash_value:
hash_value = file_hash_value
try:
cursor = db.connection.cursor() if db.connection else None
if cursor:
cursor.execute(
'INSERT OR REPLACE INTO metadata (file_id, hash, url, relationships, time_imported, time_modified) VALUES (?, ?, ?, ?, datetime("now"), datetime("now"))',
'INSERT OR REPLACE INTO metadata (hash, url, relationships, time_imported, time_modified) VALUES (?, ?, ?, datetime("now"), datetime("now"))',
(
file_id,
hash_value,
json.dumps(url),
json.dumps(relationships),
@@ -634,9 +640,8 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
cursor = db.connection.cursor() if db.connection else None
if cursor:
cursor.execute(
'INSERT INTO notes (file_id, note, created_at, updated_at) VALUES (?, ?, datetime("now"), datetime("now")) ON CONFLICT(file_id) DO UPDATE SET note = excluded.note, updated_at = datetime("now")',
(file_id,
content),
'INSERT INTO note (hash, name, note, created_at, updated_at) VALUES (?, ?, ?, datetime("now"), datetime("now")) ON CONFLICT(hash, name) DO UPDATE SET note = excluded.note, updated_at = datetime("now")',
(file_hash_value, "default", content),
)
db.connection.commit()
except Exception:

85
SYS/worker_manager.py
View File

@@ -263,6 +263,9 @@ class WorkerManager:
self.refresh_thread: Optional[Thread] = None
self._stop_refresh = False
self._lock = Lock()
# Reuse the DB's own lock so there is exactly one lock guarding the
# sqlite connection (and it is safe for re-entrant/nested DB usage).
self._db_lock = self.db._db_lock
self.worker_handlers: Dict[str,
WorkerLoggingHandler] = {} # Track active handlers
self._worker_last_step: Dict[str,
@@ -272,7 +275,8 @@ class WorkerManager:
"""Close the database connection."""
if self.db:
try:
self.db.close()
with self._db_lock:
self.db.close()
except Exception:
pass
@@ -317,12 +321,13 @@ class WorkerManager:
Count of workers updated.
"""
try:
return self.db.expire_running_workers(
older_than_seconds=older_than_seconds,
status=status,
reason=reason,
worker_id_prefix=worker_id_prefix,
)
with self._db_lock:
return self.db.expire_running_workers(
older_than_seconds=older_than_seconds,
status=status,
reason=reason,
worker_id_prefix=worker_id_prefix,
)
except Exception as exc:
logger.error(f"Failed to expire stale workers: {exc}", exc_info=True)
return 0
@@ -419,14 +424,15 @@ class WorkerManager:
True if worker was inserted successfully
"""
try:
result = self.db.insert_worker(
worker_id,
worker_type,
title,
description,
total_steps,
pipe=pipe
)
with self._db_lock:
result = self.db.insert_worker(
worker_id,
worker_type,
title,
description,
total_steps,
pipe=pipe
)
if result > 0:
logger.debug(
f"[WorkerManager] Tracking worker: {worker_id} ({worker_type})"
@@ -473,7 +479,8 @@ class WorkerManager:
kwargs["last_updated"] = datetime.now().isoformat()
if "current_step" in kwargs and kwargs["current_step"]:
self._worker_last_step[worker_id] = str(kwargs["current_step"])
return self.db.update_worker(worker_id, **kwargs)
with self._db_lock:
return self.db.update_worker(worker_id, **kwargs)
return True
except Exception as e:
logger.error(
@@ -510,7 +517,8 @@ class WorkerManager:
if result_data:
kwargs["result_data"] = result_data
success = self.db.update_worker(worker_id, **kwargs)
with self._db_lock:
success = self.db.update_worker(worker_id, **kwargs)
logger.info(f"[WorkerManager] Worker finished: {worker_id} ({result})")
self._worker_last_step.pop(worker_id, None)
return success
@@ -528,7 +536,8 @@ class WorkerManager:
List of active worker dictionaries
"""
try:
return self.db.get_active_workers()
with self._db_lock:
return self.db.get_active_workers()
except Exception as e:
logger.error(
f"[WorkerManager] Error getting active workers: {e}",
@@ -546,7 +555,8 @@ class WorkerManager:
List of finished worker dictionaries
"""
try:
all_workers = self.db.get_all_workers(limit=limit)
with self._db_lock:
all_workers = self.db.get_all_workers(limit=limit)
# Filter to only finished workers
finished = [
w for w in all_workers
@@ -570,7 +580,8 @@ class WorkerManager:
Worker data or None if not found
"""
try:
return self.db.get_worker(worker_id)
with self._db_lock:
return self.db.get_worker(worker_id)
except Exception as e:
logger.error(
f"[WorkerManager] Error getting worker {worker_id}: {e}",
@@ -583,7 +594,8 @@ class WorkerManager:
limit: int = 500) -> List[Dict[str,
Any]]:
"""Fetch recorded worker timeline events."""
return self.db.get_worker_events(worker_id, limit)
with self._db_lock:
return self.db.get_worker_events(worker_id, limit)
def log_step(self, worker_id: str, step_text: str) -> bool:
"""Log a step to a worker's step history.
@@ -596,7 +608,8 @@ class WorkerManager:
True if successful
"""
try:
success = self.db.append_worker_steps(worker_id, step_text)
with self._db_lock:
success = self.db.append_worker_steps(worker_id, step_text)
if success:
self._worker_last_step[worker_id] = step_text
return success
@@ -621,7 +634,8 @@ class WorkerManager:
Steps text or empty string if not found
"""
try:
return self.db.get_worker_steps(worker_id)
with self._db_lock:
return self.db.get_worker_steps(worker_id)
except Exception as e:
logger.error(
f"[WorkerManager] Error getting steps for worker {worker_id}: {e}",
@@ -705,7 +719,8 @@ class WorkerManager:
Number of workers deleted
"""
try:
count = self.db.cleanup_old_workers(days)
with self._db_lock:
count = self.db.cleanup_old_workers(days)
if count > 0:
logger.info(f"[WorkerManager] Cleaned up {count} old workers")
return count
@@ -729,12 +744,13 @@ class WorkerManager:
"""
try:
step_label = self._get_last_step(worker_id)
return self.db.append_worker_stdout(
worker_id,
text,
step=step_label,
channel=channel
)
with self._db_lock:
return self.db.append_worker_stdout(
worker_id,
text,
step=step_label,
channel=channel
)
except Exception as e:
logger.error(f"[WorkerManager] Error appending stdout: {e}", exc_info=True)
return False
@@ -749,7 +765,8 @@ class WorkerManager:
Worker's stdout or empty string
"""
try:
return self.db.get_worker_stdout(worker_id)
with self._db_lock:
return self.db.get_worker_stdout(worker_id)
except Exception as e:
logger.error(f"[WorkerManager] Error getting stdout: {e}", exc_info=True)
return ""
@@ -773,7 +790,8 @@ class WorkerManager:
True if clear was successful
"""
try:
return self.db.clear_worker_stdout(worker_id)
with self._db_lock:
return self.db.clear_worker_stdout(worker_id)
except Exception as e:
logger.error(f"[WorkerManager] Error clearing stdout: {e}", exc_info=True)
return False
@@ -781,5 +799,6 @@ class WorkerManager:
def close(self) -> None:
"""Close the worker manager and database connection."""
self.stop_auto_refresh()
self.db.close()
with self._db_lock:
self.db.close()
logger.info("[WorkerManager] Closed")