goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Add YAPF style + ignore, and format tracked Python files

Browse Source
This commit is contained in:
Nose
2025-12-29 18:42:02 -08:00
parent c019c00aed
commit 507946a3e4
108 changed files with 11664 additions and 6494 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
SYS/worker_manager.py
View File

@@ -170,7 +170,8 @@ class WorkerLoggingHandler(logging.StreamHandler):
# Set a format that includes timestamp and level
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
"%(asctime)s - %(name)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S"
)
self.setFormatter(formatter)
@@ -196,7 +197,10 @@ class WorkerLoggingHandler(logging.StreamHandler):
# Add timestamp and level if not already in message
import time
timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(record.created))
timestamp = time.strftime(
"%Y-%m-%d %H:%M:%S",
time.localtime(record.created)
)
msg = f"{timestamp} - {record.name} - {record.levelname} - {msg}"
with self._lock:
@@ -214,9 +218,17 @@ class WorkerLoggingHandler(logging.StreamHandler):
log_text = "\n".join(self.buffer)
try:
if self.manager:
self.manager.append_worker_stdout(self.worker_id, log_text, channel="log")
self.manager.append_worker_stdout(
self.worker_id,
log_text,
channel="log"
)
else:
self.db.append_worker_stdout(self.worker_id, log_text, channel="log")
self.db.append_worker_stdout(
self.worker_id,
log_text,
channel="log"
)
except Exception as e:
# If we can't write to DB, at least log it
log(f"Error flushing worker logs: {e}")
@@ -251,8 +263,10 @@ class WorkerManager:
self.refresh_thread: Optional[Thread] = None
self._stop_refresh = False
self._lock = Lock()
self.worker_handlers: Dict[str, WorkerLoggingHandler] = {} # Track active handlers
self._worker_last_step: Dict[str, str] = {}
self.worker_handlers: Dict[str,
WorkerLoggingHandler] = {} # Track active handlers
self._worker_last_step: Dict[str,
str] = {}
def close(self) -> None:
"""Close the database connection."""
@@ -270,7 +284,12 @@ class WorkerManager:
"""Context manager exit - close database."""
self.close()
def add_refresh_callback(self, callback: Callable[[List[Dict[str, Any]]], None]) -> None:
def add_refresh_callback(
self,
callback: Callable[[List[Dict[str,
Any]]],
None]
) -> None:
"""Register a callback to be called on worker updates.
Args:
@@ -318,7 +337,8 @@ class WorkerManager:
if callback in self.refresh_callbacks:
self.refresh_callbacks.remove(callback)
def enable_logging_for_worker(self, worker_id: str) -> Optional[WorkerLoggingHandler]:
def enable_logging_for_worker(self,
worker_id: str) -> Optional[WorkerLoggingHandler]:
"""Enable logging capture for a worker.
Creates a logging handler that captures all logs for this worker.
@@ -343,7 +363,8 @@ class WorkerManager:
return handler
except Exception as e:
logger.error(
f"[WorkerManager] Error enabling logging for worker {worker_id}: {e}", exc_info=True
f"[WorkerManager] Error enabling logging for worker {worker_id}: {e}",
exc_info=True
)
return None
@@ -366,7 +387,9 @@ class WorkerManager:
root_logger = logging.getLogger()
root_logger.removeHandler(handler)
logger.debug(f"[WorkerManager] Disabled logging for worker: {worker_id}")
logger.debug(
f"[WorkerManager] Disabled logging for worker: {worker_id}"
)
except Exception as e:
logger.error(
f"[WorkerManager] Error disabling logging for worker {worker_id}: {e}",
@@ -397,10 +420,17 @@ class WorkerManager:
"""
try:
result = self.db.insert_worker(
worker_id, worker_type, title, description, total_steps, pipe=pipe
worker_id,
worker_type,
title,
description,
total_steps,
pipe=pipe
)
if result > 0:
logger.debug(f"[WorkerManager] Tracking worker: {worker_id} ({worker_type})")
logger.debug(
f"[WorkerManager] Tracking worker: {worker_id} ({worker_type})"
)
self._start_refresh_if_needed()
return True
return False
@@ -446,11 +476,18 @@ class WorkerManager:
return self.db.update_worker(worker_id, **kwargs)
return True
except Exception as e:
logger.error(f"[WorkerManager] Error updating worker {worker_id}: {e}", exc_info=True)
logger.error(
f"[WorkerManager] Error updating worker {worker_id}: {e}",
exc_info=True
)
return False
def finish_worker(
self, worker_id: str, result: str = "completed", error_msg: str = "", result_data: str = ""
self,
worker_id: str,
result: str = "completed",
error_msg: str = "",
result_data: str = ""
) -> bool:
"""Mark a worker as finished.
@@ -464,7 +501,10 @@ class WorkerManager:
True if update was successful
"""
try:
kwargs = {"status": result, "completed_at": datetime.now().isoformat()}
kwargs = {
"status": result,
"completed_at": datetime.now().isoformat()
}
if error_msg:
kwargs["error_message"] = error_msg
if result_data:
@@ -475,7 +515,10 @@ class WorkerManager:
self._worker_last_step.pop(worker_id, None)
return success
except Exception as e:
logger.error(f"[WorkerManager] Error finishing worker {worker_id}: {e}", exc_info=True)
logger.error(
f"[WorkerManager] Error finishing worker {worker_id}: {e}",
exc_info=True
)
return False
def get_active_workers(self) -> List[Dict[str, Any]]:
@@ -487,7 +530,10 @@ class WorkerManager:
try:
return self.db.get_active_workers()
except Exception as e:
logger.error(f"[WorkerManager] Error getting active workers: {e}", exc_info=True)
logger.error(
f"[WorkerManager] Error getting active workers: {e}",
exc_info=True
)
return []
def get_finished_workers(self, limit: int = 100) -> List[Dict[str, Any]]:
@@ -503,11 +549,15 @@ class WorkerManager:
all_workers = self.db.get_all_workers(limit=limit)
# Filter to only finished workers
finished = [
w for w in all_workers if w.get("status") in ["completed", "error", "cancelled"]
w for w in all_workers
if w.get("status") in ["completed", "error", "cancelled"]
]
return finished
except Exception as e:
logger.error(f"[WorkerManager] Error getting finished workers: {e}", exc_info=True)
logger.error(
f"[WorkerManager] Error getting finished workers: {e}",
exc_info=True
)
return []
def get_worker(self, worker_id: str) -> Optional[Dict[str, Any]]:
@@ -522,10 +572,16 @@ class WorkerManager:
try:
return self.db.get_worker(worker_id)
except Exception as e:
logger.error(f"[WorkerManager] Error getting worker {worker_id}: {e}", exc_info=True)
logger.error(
f"[WorkerManager] Error getting worker {worker_id}: {e}",
exc_info=True
)
return None
def get_worker_events(self, worker_id: str, limit: int = 500) -> List[Dict[str, Any]]:
def get_worker_events(self,
worker_id: str,
limit: int = 500) -> List[Dict[str,
Any]]:
"""Fetch recorded worker timeline events."""
return self.db.get_worker_events(worker_id, limit)
@@ -546,7 +602,8 @@ class WorkerManager:
return success
except Exception as e:
logger.error(
f"[WorkerManager] Error logging step for worker {worker_id}: {e}", exc_info=True
f"[WorkerManager] Error logging step for worker {worker_id}: {e}",
exc_info=True
)
return False
@@ -567,7 +624,8 @@ class WorkerManager:
return self.db.get_worker_steps(worker_id)
except Exception as e:
logger.error(
f"[WorkerManager] Error getting steps for worker {worker_id}: {e}", exc_info=True
f"[WorkerManager] Error getting steps for worker {worker_id}: {e}",
exc_info=True
)
return ""
@@ -613,7 +671,9 @@ class WorkerManager:
if not active:
# No more active workers, stop refreshing
logger.debug("[WorkerManager] No active workers, stopping auto-refresh")
logger.debug(
"[WorkerManager] No active workers, stopping auto-refresh"
)
break
# Call all registered callbacks with the active workers
@@ -623,11 +683,15 @@ class WorkerManager:
callback(active)
except Exception as e:
logger.error(
f"[WorkerManager] Error in refresh callback: {e}", exc_info=True
f"[WorkerManager] Error in refresh callback: {e}",
exc_info=True
)
except Exception as e:
logger.error(f"[WorkerManager] Error in auto-refresh loop: {e}", exc_info=True)
logger.error(
f"[WorkerManager] Error in auto-refresh loop: {e}",
exc_info=True
)
finally:
logger.debug("[WorkerManager] Auto-refresh loop ended")
@@ -646,7 +710,10 @@ class WorkerManager:
logger.info(f"[WorkerManager] Cleaned up {count} old workers")
return count
except Exception as e:
logger.error(f"[WorkerManager] Error cleaning up old workers: {e}", exc_info=True)
logger.error(
f"[WorkerManager] Error cleaning up old workers: {e}",
exc_info=True
)
return 0
def append_stdout(self, worker_id: str, text: str, channel: str = "stdout") -> bool:
@@ -662,7 +729,12 @@ class WorkerManager:
"""
try:
step_label = self._get_last_step(worker_id)
return self.db.append_worker_stdout(worker_id, text, step=step_label, channel=channel)
return self.db.append_worker_stdout(
worker_id,
text,
step=step_label,
channel=channel
)
except Exception as e:
logger.error(f"[WorkerManager] Error appending stdout: {e}", exc_info=True)
return False
@@ -682,7 +754,12 @@ class WorkerManager:
logger.error(f"[WorkerManager] Error getting stdout: {e}", exc_info=True)
return ""
def append_worker_stdout(self, worker_id: str, text: str, channel: str = "stdout") -> bool:
def append_worker_stdout(
self,
worker_id: str,
text: str,
channel: str = "stdout"
) -> bool:
"""Compatibility wrapper for append_stdout."""
return self.append_stdout(worker_id, text, channel=channel)