goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Add YAPF style + ignore, and format tracked Python files

Browse Source
This commit is contained in:
Nose
2025-12-29 18:42:02 -08:00
parent c019c00aed
commit 507946a3e4
108 changed files with 11664 additions and 6494 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
SYS/background_notifier.py
View File

@@ -20,18 +20,21 @@ class BackgroundNotifier:
def __init__(
self,
manager: Any,
output: Callable[[str], None] = log,
output: Callable[[str],
None] = log,
session_worker_ids: Optional[Set[str]] = None,
only_terminal_updates: bool = False,
overlay_mode: bool = False,
) -> None:
self.manager = manager
self.output = output
self.session_worker_ids = session_worker_ids if session_worker_ids is not None else set()
self.session_worker_ids = session_worker_ids if session_worker_ids is not None else set(
)
self.only_terminal_updates = only_terminal_updates
self.overlay_mode = overlay_mode
self._filter_enabled = session_worker_ids is not None
self._last_state: Dict[str, str] = {}
self._last_state: Dict[str,
str] = {}
try:
self.manager.add_refresh_callback(self._on_refresh)
@@ -56,7 +59,8 @@ class BackgroundNotifier:
elif progress_val:
progress = f" {progress_val}"
step = str(worker.get("current_step") or worker.get("description") or "").strip()
step = str(worker.get("current_step") or worker.get("description")
or "").strip()
parts = [f"[worker:{worker_id}] {status}{progress}"]
if step:
parts.append(step)
@@ -83,7 +87,8 @@ class BackgroundNotifier:
# Overlay mode: only emit on completion; suppress start/progress spam
if self.overlay_mode:
if status in ("completed", "finished", "error"):
progress_val = worker.get("progress") or worker.get("progress_percent") or ""
progress_val = worker.get("progress"
) or worker.get("progress_percent") or ""
step = str(
worker.get("current_step") or worker.get("description") or ""
).strip()
@@ -128,8 +133,10 @@ class BackgroundNotifier:
self.session_worker_ids.discard(worker_id)
continue
progress_val = worker.get("progress") or worker.get("progress_percent") or ""
step = str(worker.get("current_step") or worker.get("description") or "").strip()
progress_val = worker.get("progress"
) or worker.get("progress_percent") or ""
step = str(worker.get("current_step") or worker.get("description")
or "").strip()
signature = f"{status}|{progress_val}|{step}"
if self._last_state.get(worker_id) == signature:
@@ -154,7 +161,8 @@ class BackgroundNotifier:
def ensure_background_notifier(
manager: Any,
output: Callable[[str], None] = log,
output: Callable[[str],
None] = log,
session_worker_ids: Optional[Set[str]] = None,
only_terminal_updates: bool = False,
overlay_mode: bool = False,

228
SYS/download.py
View File

@@ -99,8 +99,11 @@ def is_url_supported_by_ytdlp(url: str) -> bool:
def list_formats(
url: str, no_playlist: bool = False, playlist_items: Optional[str] = None
) -> Optional[List[Dict[str, Any]]]:
url: str,
no_playlist: bool = False,
playlist_items: Optional[str] = None
) -> Optional[List[Dict[str,
Any]]]:
"""Get list of available formats for a URL using yt-dlp."""
_ensure_yt_dlp_ready()
@@ -130,15 +133,21 @@ def list_formats(
for fmt in formats:
result_formats.append(
{
"format_id": fmt.get("format_id", ""),
"format": fmt.get("format", ""),
"ext": fmt.get("ext", ""),
"resolution": fmt.get("resolution", ""),
"format_id": fmt.get("format_id",
""),
"format": fmt.get("format",
""),
"ext": fmt.get("ext",
""),
"resolution": fmt.get("resolution",
""),
"width": fmt.get("width"),
"height": fmt.get("height"),
"fps": fmt.get("fps"),
"vcodec": fmt.get("vcodec", "none"),
"acodec": fmt.get("acodec", "none"),
"vcodec": fmt.get("vcodec",
"none"),
"acodec": fmt.get("acodec",
"none"),
"filesize": fmt.get("filesize"),
"tbr": fmt.get("tbr"),
}
@@ -153,8 +162,14 @@ def list_formats(
def _download_with_sections_via_cli(
url: str, ytdl_options: Dict[str, Any], sections: List[str], quiet: bool = False
) -> tuple[Optional[str], Dict[str, Any]]:
url: str,
ytdl_options: Dict[str,
Any],
sections: List[str],
quiet: bool = False
) -> tuple[Optional[str],
Dict[str,
Any]]:
"""Download each section separately so merge-file can combine them.
yt-dlp with multiple --download-sections args merges them into one file.
@@ -174,7 +189,8 @@ def _download_with_sections_via_cli(
# Generate a unique hash-based ID for this download session
# This ensures different videos/downloads don't have filename collisions
session_id = hashlib.md5(
(url + str(time.time()) + "".join(random.choices(string.ascii_letters, k=10))).encode()
(url + str(time.time()) + "".join(random.choices(string.ascii_letters,
k=10))).encode()
).hexdigest()[:12]
first_section_info = None
@@ -207,7 +223,11 @@ def _download_with_sections_via_cli(
metadata_cmd.append(url)
try:
meta_result = subprocess.run(metadata_cmd, capture_output=True, text=True)
meta_result = subprocess.run(
metadata_cmd,
capture_output=True,
text=True
)
if meta_result.returncode == 0 and meta_result.stdout:
try:
info_dict = json.loads(meta_result.stdout.strip())
@@ -253,7 +273,9 @@ def _download_with_sections_via_cli(
cmd.append(url)
if not quiet:
debug(f"Running yt-dlp for section {section_idx}/{len(sections_list)}: {section}")
debug(
f"Running yt-dlp for section {section_idx}/{len(sections_list)}: {section}"
)
debug(f"Command: {' '.join(cmd)}")
# Run the subprocess - don't capture output so progress is shown
@@ -280,24 +302,26 @@ def _build_ytdlp_options(opts: DownloadOptions) -> Dict[str, Any]:
# When downloading sections, each section will have .section_N_of_M added by _download_with_sections_via_cli
outtmpl = str((opts.output_dir / "%(title)s.%(ext)s").resolve())
base_options: Dict[str, Any] = {
"outtmpl": outtmpl,
"quiet": True,
"no_warnings": True,
"noprogress": True,
"socket_timeout": 30,
"retries": 10,
"fragment_retries": 10,
"http_chunk_size": 10_485_760,
"restrictfilenames": True,
"progress_hooks": [] if opts.quiet else [_progress_callback],
}
base_options: Dict[str,
Any] = {
"outtmpl": outtmpl,
"quiet": True,
"no_warnings": True,
"noprogress": True,
"socket_timeout": 30,
"retries": 10,
"fragment_retries": 10,
"http_chunk_size": 10_485_760,
"restrictfilenames": True,
"progress_hooks": [] if opts.quiet else [_progress_callback],
}
if opts.cookies_path and opts.cookies_path.is_file():
base_options["cookiefile"] = str(opts.cookies_path)
else:
# Fallback to browser cookies
base_options["cookiesfrombrowser"] = ("chrome",)
base_options["cookiesfrombrowser"] = ("chrome",
)
# Add no-playlist option if specified (for single video from playlist url)
if opts.no_playlist:
@@ -306,7 +330,9 @@ def _build_ytdlp_options(opts: DownloadOptions) -> Dict[str, Any]:
# Configure based on mode
if opts.mode == "audio":
base_options["format"] = opts.ytdl_format or "251/140/bestaudio"
base_options["postprocessors"] = [{"key": "FFmpegExtractAudio"}]
base_options["postprocessors"] = [{
"key": "FFmpegExtractAudio"
}]
else: # video
base_options["format"] = opts.ytdl_format or "bestvideo+bestaudio/best"
base_options["format_sort"] = [
@@ -396,7 +422,11 @@ def _candidate_paths(entry: Dict[str, Any], output_dir: Path) -> Iterator[Path]:
yield output_dir / entry["filename"]
def _resolve_entry_and_path(info: Dict[str, Any], output_dir: Path) -> tuple[Dict[str, Any], Path]:
def _resolve_entry_and_path(info: Dict[str,
Any],
output_dir: Path) -> tuple[Dict[str,
Any],
Path]:
"""Find downloaded file in yt-dlp metadata."""
for entry in _iter_download_entries(info):
for candidate in _candidate_paths(entry, output_dir):
@@ -454,7 +484,10 @@ def _get_libgen_download_url(libgen_url: str) -> Optional[str]:
# LibGen redirects to actual mirrors, follow redirects to get final URL
session = requests.Session()
session.headers.update(
{"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"}
{
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
}
)
debug(f"Following LibGen redirect chain for: {libgen_url}")
@@ -479,30 +512,36 @@ def _get_libgen_download_url(libgen_url: str) -> Optional[str]:
continue
href_lower = href.lower()
if "get.php" in href_lower or href_lower.endswith(
(".pdf", ".epub", ".djvu", ".mobi")
):
if "get.php" in href_lower or href_lower.endswith((".pdf",
".epub",
".djvu",
".mobi")):
download_url = (
href if href.startswith("http") else urljoin(final_url, href)
href if href.startswith("http") else
urljoin(final_url,
href)
)
debug(f"Found download link: {download_url}")
return download_url
else:
# Regex fallback
for m in re.finditer(
r"href=[\"\']([^\"\']+)[\"\']",
response.text or "",
flags=re.IGNORECASE,
r"href=[\"\']([^\"\']+)[\"\']",
response.text or "",
flags=re.IGNORECASE,
):
href = str(m.group(1) or "").strip()
if not href or href.lower().startswith("javascript:"):
continue
href_lower = href.lower()
if "get.php" in href_lower or href_lower.endswith(
(".pdf", ".epub", ".djvu", ".mobi")
):
if "get.php" in href_lower or href_lower.endswith((".pdf",
".epub",
".djvu",
".mobi")):
download_url = (
href if href.startswith("http") else urljoin(final_url, href)
href if href.startswith("http") else
urljoin(final_url,
href)
)
debug(f"Found download link: {download_url}")
return download_url
@@ -616,13 +655,17 @@ def _download_direct_file(
response = client._request("HEAD", url, follow_redirects=True)
content_disposition = response.headers.get("content-disposition", "")
try:
content_type = str(response.headers.get("content-type", "") or "").strip().lower()
content_type = str(response.headers.get("content-type",
"") or "").strip().lower()
except Exception:
content_type = ""
if content_disposition:
# Extract filename from Content-Disposition header
# Format: attachment; filename="filename.pdf" or filename=filename.pdf
match = re.search(r'filename\*?=(?:"([^"]*)"|([^;\s]*))', content_disposition)
match = re.search(
r'filename\*?=(?:"([^"]*)"|([^;\s]*))',
content_disposition
)
if match:
extracted_name = match.group(1) or match.group(2)
if extracted_name:
@@ -638,7 +681,11 @@ def _download_direct_file(
# servers block/lie on HEAD, and a URL path like `edition.php` would otherwise
# be saved as a bogus file.
try:
page_like_exts = {".php", ".asp", ".aspx", ".jsp", ".cgi"}
page_like_exts = {".php",
".asp",
".aspx",
".jsp",
".cgi"}
ext = ""
try:
ext = Path(str(filename or "")).suffix.lower()
@@ -653,13 +700,14 @@ def _download_direct_file(
with client._request_stream("GET", url, follow_redirects=True) as resp:
resp.raise_for_status()
ct = (
str(resp.headers.get("content-type", "") or "")
.split(";", 1)[0]
.strip()
.lower()
str(resp.headers.get("content-type",
"") or "").split(";",
1)[0].strip().lower()
)
if ct.startswith("text/html"):
raise DownloadError("URL appears to be an HTML page, not a direct file")
raise DownloadError(
"URL appears to be an HTML page, not a direct file"
)
except DownloadError:
raise
except Exception:
@@ -722,7 +770,8 @@ def _download_direct_file(
# Prefer pipeline transfer bars when a Live UI is active.
use_pipeline_transfer = False
try:
if pipeline_progress is not None and hasattr(pipeline_progress, "update_transfer"):
if pipeline_progress is not None and hasattr(pipeline_progress,
"update_transfer"):
ui = None
if hasattr(pipeline_progress, "ui_and_pipe_index"):
ui, _ = pipeline_progress.ui_and_pipe_index() # type: ignore[attr-defined]
@@ -753,15 +802,16 @@ def _download_direct_file(
try:
total_val: Optional[int] = (
int(content_length)
if isinstance(content_length, int) and content_length > 0
else None
if isinstance(content_length,
int) and content_length > 0 else None
)
except Exception:
total_val = None
try:
if hasattr(pipeline_progress, "begin_transfer"):
pipeline_progress.begin_transfer(
label=str(filename or "download"), total=total_val
label=str(filename or "download"),
total=total_val
)
transfer_started[0] = True
except Exception:
@@ -773,16 +823,18 @@ def _download_direct_file(
# Update pipeline transfer bar when present.
try:
if pipeline_progress is not None and hasattr(pipeline_progress, "update_transfer"):
if pipeline_progress is not None and hasattr(pipeline_progress,
"update_transfer"):
_maybe_begin_transfer(content_length)
total_val: Optional[int] = (
int(content_length)
if isinstance(content_length, int) and content_length > 0
else None
if isinstance(content_length,
int) and content_length > 0 else None
)
pipeline_progress.update_transfer(
label=str(filename or "download"),
completed=int(bytes_downloaded) if bytes_downloaded is not None else None,
completed=int(bytes_downloaded)
if bytes_downloaded is not None else None,
total=total_val,
)
except Exception:
@@ -796,12 +848,17 @@ def _download_direct_file(
return
elapsed = now - start_time
percent = (bytes_downloaded / content_length) * 100 if content_length > 0 else 0
percent = (
bytes_downloaded / content_length
) * 100 if content_length > 0 else 0
speed = bytes_downloaded / elapsed if elapsed > 0 else 0
eta_str: Optional[str] = None
if content_length > 0 and speed > 0:
try:
eta_seconds = max(0.0, float(content_length - bytes_downloaded) / float(speed))
eta_seconds = max(
0.0,
float(content_length - bytes_downloaded) / float(speed)
)
minutes, seconds = divmod(int(eta_seconds), 60)
hours, minutes = divmod(minutes, 60)
eta_str = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
@@ -832,11 +889,9 @@ def _download_direct_file(
pass
try:
if (
pipeline_progress is not None
and transfer_started[0]
and hasattr(pipeline_progress, "finish_transfer")
):
if (pipeline_progress is not None and transfer_started[0]
and hasattr(pipeline_progress,
"finish_transfer")):
pipeline_progress.finish_transfer(label=str(filename or "download"))
except Exception:
pass
@@ -844,8 +899,9 @@ def _download_direct_file(
try:
if progress_bar is not None:
avg_speed_str = (
progress_bar.format_bytes(downloaded_bytes[0] / elapsed if elapsed > 0 else 0)
+ "/s"
progress_bar.
format_bytes(downloaded_bytes[0] / elapsed if elapsed > 0 else 0) +
"/s"
)
else:
avg_speed_str = f"{(downloaded_bytes[0] / elapsed if elapsed > 0 else 0):.1f} B/s"
@@ -864,7 +920,8 @@ def _download_direct_file(
ext = ""
info = {
"id": str(filename).rsplit(".", 1)[0] if "." in str(filename) else str(filename),
"id": str(filename).rsplit(".",
1)[0] if "." in str(filename) else str(filename),
"ext": ext,
"webpage_url": url,
}
@@ -897,7 +954,11 @@ def _download_direct_file(
if debug_logger is not None:
debug_logger.write_record(
"direct-file-downloaded",
{"url": url, "path": str(file_path), "hash": hash_value},
{
"url": url,
"path": str(file_path),
"hash": hash_value
},
)
return DownloadMediaResult(
@@ -915,11 +976,9 @@ def _download_direct_file(
except Exception:
pass
try:
if (
pipeline_progress is not None
and transfer_started[0]
and hasattr(pipeline_progress, "finish_transfer")
):
if (pipeline_progress is not None and transfer_started[0]
and hasattr(pipeline_progress,
"finish_transfer")):
pipeline_progress.finish_transfer(label=str(filename or "download"))
except Exception:
pass
@@ -927,7 +986,11 @@ def _download_direct_file(
if debug_logger is not None:
debug_logger.write_record(
"exception",
{"phase": "direct-file", "url": url, "error": str(exc)},
{
"phase": "direct-file",
"url": url,
"error": str(exc)
},
)
raise DownloadError(f"Failed to download {url}: {exc}") from exc
except Exception as exc:
@@ -937,11 +1000,9 @@ def _download_direct_file(
except Exception:
pass
try:
if (
pipeline_progress is not None
and transfer_started[0]
and hasattr(pipeline_progress, "finish_transfer")
):
if (pipeline_progress is not None and transfer_started[0]
and hasattr(pipeline_progress,
"finish_transfer")):
pipeline_progress.finish_transfer(label=str(filename or "download"))
except Exception:
pass
@@ -959,9 +1020,10 @@ def _download_direct_file(
raise DownloadError(f"Error downloading file: {exc}") from exc
def probe_url(
url: str, no_playlist: bool = False, timeout_seconds: int = 15
) -> Optional[Dict[str, Any]]:
def probe_url(url: str,
no_playlist: bool = False,
timeout_seconds: int = 15) -> Optional[Dict[str,
Any]]:
"""Probe URL to extract metadata WITHOUT downloading.
Args:
@@ -1032,7 +1094,9 @@ def probe_url(
if thread.is_alive():
# Probe timed out - return None to fall back to direct download
debug(f"Probe timeout for {url} (>={timeout_seconds}s), proceeding with download")
debug(
f"Probe timeout for {url} (>={timeout_seconds}s), proceeding with download"
)
return None
if result_container[1] is not None:

11
SYS/env_check.py
View File

@@ -102,8 +102,11 @@ def check_urllib3_compat() -> Tuple[bool, str]:
# Looks good
debug(
"urllib3 appears usable: version=%s, exceptions=%s",
getattr(urllib3, "__version__", "<unknown>"),
hasattr(urllib3, "exceptions"),
getattr(urllib3,
"__version__",
"<unknown>"),
hasattr(urllib3,
"exceptions"),
)
return True, "OK"
@@ -129,7 +132,9 @@ def ensure_urllib3_ok(exit_on_error: bool = True) -> bool:
log(border)
if exit_on_error:
log("Please follow the steps above to fix your environment, then re-run this command.")
log(
"Please follow the steps above to fix your environment, then re-run this command."
)
try:
sys.exit(2)
except SystemExit:

8
SYS/file_server.py
View File

@@ -54,7 +54,8 @@ class FileServerHandler(SimpleHTTPRequestHandler):
self.send_header("Content-type", content_type)
self.send_header("Content-Length", str(len(file_content)))
self.send_header(
"Content-Disposition", f'attachment; filename="{full_path.name}"'
"Content-Disposition",
f'attachment; filename="{full_path.name}"'
)
self.end_headers()
self.wfile.write(file_content)
@@ -118,7 +119,10 @@ def start_file_server(port: int = 8001) -> Optional[str]:
_file_server = HTTPServer(server_address, FileServerHandler)
# Start in daemon thread
_server_thread = threading.Thread(target=_file_server.serve_forever, daemon=True)
_server_thread = threading.Thread(
target=_file_server.serve_forever,
daemon=True
)
_server_thread.start()
logger.info(f"File server started on port {port}")

39
SYS/pipeline_progress.py
View File

@@ -27,16 +27,24 @@ class PipelineProgress:
def ui_and_pipe_index(self) -> Tuple[Optional[Any], int]:
ui = None
try:
ui = self._ctx.get_live_progress() if hasattr(self._ctx, "get_live_progress") else None
ui = self._ctx.get_live_progress(
) if hasattr(self._ctx,
"get_live_progress") else None
except Exception:
ui = None
pipe_idx: int = 0
try:
stage_ctx = (
self._ctx.get_stage_context() if hasattr(self._ctx, "get_stage_context") else None
self._ctx.get_stage_context()
if hasattr(self._ctx,
"get_stage_context") else None
)
maybe_idx = getattr(stage_ctx, "pipe_index", None) if stage_ctx is not None else None
maybe_idx = getattr(
stage_ctx,
"pipe_index",
None
) if stage_ctx is not None else None
if isinstance(maybe_idx, int):
pipe_idx = int(maybe_idx)
except Exception:
@@ -111,7 +119,11 @@ class PipelineProgress:
return
def update_transfer(
self, *, label: str, completed: Optional[int], total: Optional[int] = None
self,
*,
label: str,
completed: Optional[int],
total: Optional[int] = None
) -> None:
ui, _ = self.ui_and_pipe_index()
if ui is None:
@@ -149,13 +161,19 @@ class PipelineProgress:
return
def ensure_local_ui(
self, *, label: str, total_items: int, items_preview: Optional[Sequence[Any]] = None
self,
*,
label: str,
total_items: int,
items_preview: Optional[Sequence[Any]] = None
) -> bool:
"""Start a local PipelineLiveProgress panel if no shared UI exists."""
try:
existing = (
self._ctx.get_live_progress() if hasattr(self._ctx, "get_live_progress") else None
self._ctx.get_live_progress()
if hasattr(self._ctx,
"get_live_progress") else None
)
except Exception:
existing = None
@@ -179,7 +197,10 @@ class PipelineProgress:
try:
ui.begin_pipe(
0, total_items=max(1, int(total_items)), items_preview=list(items_preview or [])
0,
total_items=max(1,
int(total_items)),
items_preview=list(items_preview or [])
)
except Exception:
pass
@@ -221,7 +242,9 @@ class PipelineProgress:
items_preview: Optional[Sequence[Any]] = None,
) -> Iterator["PipelineProgress"]:
created = self.ensure_local_ui(
label=label, total_items=total_items, items_preview=items_preview
label=label,
total_items=total_items,
items_preview=items_preview
)
try:
yield self

7
SYS/progress.py
View File

@@ -10,12 +10,15 @@ import sys
from models import ProgressBar
_BAR = ProgressBar()
def print_progress(
filename: str, current: int, total: int, speed: float = 0, end: str = "\r"
filename: str,
current: int,
total: int,
speed: float = 0,
end: str = "\r"
) -> None:
_BAR.update(
downloaded=int(current),

35
SYS/tasks.py
View File

@@ -31,7 +31,10 @@ def connect_ipc(path: str, timeout: float = 5.0) -> IO[bytes] | None:
except OSError as exc: # Pipe busy
# Windows named pipes can intermittently raise EINVAL while the pipe exists
# but is not ready/accepting connections yet.
if exc.errno not in (errno.ENOENT, errno.EPIPE, errno.EBUSY, errno.EINVAL):
if exc.errno not in (errno.ENOENT,
errno.EPIPE,
errno.EBUSY,
errno.EINVAL):
raise
if time.time() > deadline:
return None
@@ -66,7 +69,12 @@ def ipc_sender(ipc: IO[bytes] | None):
def _send(event: str, payload: dict) -> None:
message = json.dumps(
{"command": ["script-message", event, json.dumps(payload)]}, ensure_ascii=False
{
"command": ["script-message",
event,
json.dumps(payload)]
},
ensure_ascii=False
)
encoded = message.encode("utf-8") + b"\n"
with lock:
@@ -86,7 +94,9 @@ def iter_stream(stream: Iterable[str]) -> Iterable[str]:
def _run_task(args, parser) -> int:
if not args.command:
parser.error('run-task requires a command to execute (use "--" before the command).')
parser.error(
'run-task requires a command to execute (use "--" before the command).'
)
env = os.environ.copy()
for entry in args.env:
key, sep, value = entry.partition("=")
@@ -110,7 +120,12 @@ def _run_task(args, parser) -> int:
return 1
if command and isinstance(command[0], str) and sys.executable:
first = command[0].lower()
if first in {"python", "python3", "py", "python.exe", "python3.exe", "py.exe"}:
if first in {"python",
"python3",
"py",
"python.exe",
"python3.exe",
"py.exe"}:
command[0] = sys.executable
if os.environ.get("DOWNLOW_DEBUG"):
log(f"Launching command: {command}", file=sys.stderr)
@@ -181,13 +196,21 @@ def _run_task(args, parser) -> int:
threads = []
if process.stdout:
t_out = threading.Thread(
target=pump, args=(process.stdout, "stdout", stdout_lines), daemon=True
target=pump,
args=(process.stdout,
"stdout",
stdout_lines),
daemon=True
)
t_out.start()
threads.append(t_out)
if process.stderr:
t_err = threading.Thread(
target=pump, args=(process.stderr, "stderr", stderr_lines), daemon=True
target=pump,
args=(process.stderr,
"stderr",
stderr_lines),
daemon=True
)
t_err.start()
threads.append(t_err)

81
SYS/utils.py
View File

@@ -123,7 +123,9 @@ def create_metadata_sidecar(file_path: Path, metadata: dict) -> None:
with open(metadata_path, "w", encoding="utf-8") as f:
json.dump(metadata, f, ensure_ascii=False, indent=2)
except OSError as exc:
raise RuntimeError(f"Failed to write metadata sidecar {metadata_path}: {exc}") from exc
raise RuntimeError(
f"Failed to write metadata sidecar {metadata_path}: {exc}"
) from exc
def create_tags_sidecar(file_path: Path, tags: set) -> None:
@@ -194,7 +196,8 @@ def ffprobe(file_path: str) -> dict:
return {}
metadata = {}
fmt = probe.get("format", {})
fmt = probe.get("format",
{})
metadata["duration"] = float(fmt.get("duration", 0)) if "duration" in fmt else None
metadata["size"] = int(fmt.get("size", 0)) if "size" in fmt else None
metadata["format_name"] = fmt.get("format_name", None)
@@ -204,19 +207,38 @@ def ffprobe(file_path: str) -> dict:
codec_type = stream.get("codec_type")
if codec_type == "audio":
metadata["audio_codec"] = stream.get("codec_name")
metadata["bitrate"] = int(stream.get("bit_rate", 0)) if "bit_rate" in stream else None
metadata["bitrate"] = int(
stream.get("bit_rate",
0)
) if "bit_rate" in stream else None
metadata["samplerate"] = (
int(stream.get("sample_rate", 0)) if "sample_rate" in stream else None
int(stream.get("sample_rate",
0)) if "sample_rate" in stream else None
)
metadata["channels"] = int(stream.get("channels", 0)) if "channels" in stream else None
metadata["channels"] = int(
stream.get("channels",
0)
) if "channels" in stream else None
elif codec_type == "video":
metadata["video_codec"] = stream.get("codec_name")
metadata["width"] = int(stream.get("width", 0)) if "width" in stream else None
metadata["height"] = int(stream.get("height", 0)) if "height" in stream else None
metadata["width"] = int(
stream.get("width",
0)
) if "width" in stream else None
metadata["height"] = int(
stream.get("height",
0)
) if "height" in stream else None
elif codec_type == "image":
metadata["image_codec"] = stream.get("codec_name")
metadata["width"] = int(stream.get("width", 0)) if "width" in stream else None
metadata["height"] = int(stream.get("height", 0)) if "height" in stream else None
metadata["width"] = int(
stream.get("width",
0)
) if "width" in stream else None
metadata["height"] = int(
stream.get("height",
0)
) if "height" in stream else None
return metadata
@@ -239,11 +261,16 @@ def decode_cbor(data: bytes) -> Any:
def jsonify(value: Any) -> Any:
"""Convert *value* into a JSON-friendly structure."""
if isinstance(value, dict):
return {str(key): jsonify(val) for key, val in value.items()}
return {
str(key): jsonify(val)
for key, val in value.items()
}
if isinstance(value, list):
return [jsonify(item) for item in value]
if isinstance(value, bytes):
return {"__bytes__": base64.b64encode(value).decode("ascii")}
return {
"__bytes__": base64.b64encode(value).decode("ascii")
}
return value
@@ -362,12 +389,12 @@ def format_metadata_value(key: str, value) -> str:
elif key in ("duration", "length"):
return format_duration(value)
elif key in (
"time_modified",
"time_imported",
"created_at",
"updated_at",
"indexed_at",
"timestamp",
"time_modified",
"time_imported",
"created_at",
"updated_at",
"indexed_at",
"timestamp",
):
return format_timestamp(value)
else:
@@ -413,9 +440,13 @@ def extract_link_from_result(result: Any) -> Any | None:
return result.get("url") or result.get("link") or result.get("href")
return (
getattr(result, "url", None)
or getattr(result, "link", None)
or getattr(result, "href", None)
getattr(result,
"url",
None) or getattr(result,
"link",
None) or getattr(result,
"href",
None)
)
@@ -466,7 +497,11 @@ def get_api_key(config: dict[str, Any], service: str, key_path: str) -> str | No
return None
def add_direct_link_to_result(result: Any, direct_link: str, original_link: str) -> None:
def add_direct_link_to_result(
result: Any,
direct_link: str,
original_link: str
) -> None:
"""Add direct link information to result object.
Args:
@@ -515,7 +550,9 @@ def _normalise_rule(rule: dict[str, Any]) -> dict[str, Any] | None:
force_screenshot = bool(rule.get("force_screenshot"))
extra_tags_raw = rule.get("extra_tags")
if isinstance(extra_tags_raw, str):
extra_tags = [part.strip() for part in extra_tags_raw.split(",") if part.strip()]
extra_tags = [
part.strip() for part in extra_tags_raw.split(",") if part.strip()
]
elif isinstance(extra_tags_raw, (list, tuple, set)):
extra_tags = [str(item).strip() for item in extra_tags_raw if str(item).strip()]
else:

326
SYS/utils_constant.py
View File

@@ -1,98 +1,296 @@
mime_maps = {
"image": {
"jpg": {"ext": ".jpg", "mimes": ["image/jpeg", "image/jpg"]},
"png": {"ext": ".png", "mimes": ["image/png"]},
"gif": {"ext": ".gif", "mimes": ["image/gif"]},
"webp": {"ext": ".webp", "mimes": ["image/webp"]},
"avif": {"ext": ".avif", "mimes": ["image/avif"]},
"jxl": {"ext": ".jxl", "mimes": ["image/jxl"]},
"bmp": {"ext": ".bmp", "mimes": ["image/bmp"]},
"heic": {"ext": ".heic", "mimes": ["image/heic"]},
"heif": {"ext": ".heif", "mimes": ["image/heif"]},
"ico": {"ext": ".ico", "mimes": ["image/x-icon", "image/vnd.microsoft.icon"]},
"qoi": {"ext": ".qoi", "mimes": ["image/qoi"]},
"tiff": {"ext": ".tiff", "mimes": ["image/tiff", "image/x-tiff"]},
"svg": {"ext": ".svg", "mimes": ["image/svg+xml"]},
"jpg": {
"ext": ".jpg",
"mimes": ["image/jpeg",
"image/jpg"]
},
"png": {
"ext": ".png",
"mimes": ["image/png"]
},
"gif": {
"ext": ".gif",
"mimes": ["image/gif"]
},
"webp": {
"ext": ".webp",
"mimes": ["image/webp"]
},
"avif": {
"ext": ".avif",
"mimes": ["image/avif"]
},
"jxl": {
"ext": ".jxl",
"mimes": ["image/jxl"]
},
"bmp": {
"ext": ".bmp",
"mimes": ["image/bmp"]
},
"heic": {
"ext": ".heic",
"mimes": ["image/heic"]
},
"heif": {
"ext": ".heif",
"mimes": ["image/heif"]
},
"ico": {
"ext": ".ico",
"mimes": ["image/x-icon",
"image/vnd.microsoft.icon"]
},
"qoi": {
"ext": ".qoi",
"mimes": ["image/qoi"]
},
"tiff": {
"ext": ".tiff",
"mimes": ["image/tiff",
"image/x-tiff"]
},
"svg": {
"ext": ".svg",
"mimes": ["image/svg+xml"]
},
},
"image_sequence": {
"apng": {"ext": ".apng", "mimes": ["image/apng"], "sequence": True},
"avifs": {"ext": ".avifs", "mimes": ["image/avif-sequence"], "sequence": True},
"heics": {"ext": ".heics", "mimes": ["image/heic-sequence"], "sequence": True},
"heifs": {"ext": ".heifs", "mimes": ["image/heif-sequence"], "sequence": True},
"apng": {
"ext": ".apng",
"mimes": ["image/apng"],
"sequence": True
},
"avifs": {
"ext": ".avifs",
"mimes": ["image/avif-sequence"],
"sequence": True
},
"heics": {
"ext": ".heics",
"mimes": ["image/heic-sequence"],
"sequence": True
},
"heifs": {
"ext": ".heifs",
"mimes": ["image/heif-sequence"],
"sequence": True
},
},
"video": {
"mp4": {"ext": ".mp4", "mimes": ["video/mp4", "audio/mp4"]},
"webm": {"ext": ".webm", "mimes": ["video/webm", "audio/webm"]},
"mov": {"ext": ".mov", "mimes": ["video/quicktime"]},
"ogv": {"ext": ".ogv", "mimes": ["video/ogg"]},
"mpeg": {"ext": ".mpeg", "mimes": ["video/mpeg"]},
"avi": {"ext": ".avi", "mimes": ["video/x-msvideo", "video/avi"]},
"flv": {"ext": ".flv", "mimes": ["video/x-flv"]},
"mp4": {
"ext": ".mp4",
"mimes": ["video/mp4",
"audio/mp4"]
},
"webm": {
"ext": ".webm",
"mimes": ["video/webm",
"audio/webm"]
},
"mov": {
"ext": ".mov",
"mimes": ["video/quicktime"]
},
"ogv": {
"ext": ".ogv",
"mimes": ["video/ogg"]
},
"mpeg": {
"ext": ".mpeg",
"mimes": ["video/mpeg"]
},
"avi": {
"ext": ".avi",
"mimes": ["video/x-msvideo",
"video/avi"]
},
"flv": {
"ext": ".flv",
"mimes": ["video/x-flv"]
},
"mkv": {
"ext": ".mkv",
"mimes": ["video/x-matroska", "application/x-matroska"],
"mimes": ["video/x-matroska",
"application/x-matroska"],
"audio_only_ext": ".mka",
},
"wmv": {"ext": ".wmv", "mimes": ["video/x-ms-wmv"]},
"rv": {"ext": ".rv", "mimes": ["video/vnd.rn-realvideo"]},
"wmv": {
"ext": ".wmv",
"mimes": ["video/x-ms-wmv"]
},
"rv": {
"ext": ".rv",
"mimes": ["video/vnd.rn-realvideo"]
},
},
"audio": {
"mp3": {"ext": ".mp3", "mimes": ["audio/mpeg", "audio/mp3"]},
"m4a": {"ext": ".m4a", "mimes": ["audio/mp4", "audio/x-m4a"]},
"ogg": {"ext": ".ogg", "mimes": ["audio/ogg"]},
"opus": {"ext": ".opus", "mimes": ["audio/opus"]},
"flac": {"ext": ".flac", "mimes": ["audio/flac"]},
"wav": {"ext": ".wav", "mimes": ["audio/wav", "audio/x-wav", "audio/vnd.wave"]},
"wma": {"ext": ".wma", "mimes": ["audio/x-ms-wma"]},
"tta": {"ext": ".tta", "mimes": ["audio/x-tta"]},
"wv": {"ext": ".wv", "mimes": ["audio/x-wavpack", "audio/wavpack"]},
"mka": {"ext": ".mka", "mimes": ["audio/x-matroska", "video/x-matroska"]},
"mp3": {
"ext": ".mp3",
"mimes": ["audio/mpeg",
"audio/mp3"]
},
"m4a": {
"ext": ".m4a",
"mimes": ["audio/mp4",
"audio/x-m4a"]
},
"ogg": {
"ext": ".ogg",
"mimes": ["audio/ogg"]
},
"opus": {
"ext": ".opus",
"mimes": ["audio/opus"]
},
"flac": {
"ext": ".flac",
"mimes": ["audio/flac"]
},
"wav": {
"ext": ".wav",
"mimes": ["audio/wav",
"audio/x-wav",
"audio/vnd.wave"]
},
"wma": {
"ext": ".wma",
"mimes": ["audio/x-ms-wma"]
},
"tta": {
"ext": ".tta",
"mimes": ["audio/x-tta"]
},
"wv": {
"ext": ".wv",
"mimes": ["audio/x-wavpack",
"audio/wavpack"]
},
"mka": {
"ext": ".mka",
"mimes": ["audio/x-matroska",
"video/x-matroska"]
},
},
"document": {
"pdf": {"ext": ".pdf", "mimes": ["application/pdf"]},
"epub": {"ext": ".epub", "mimes": ["application/epub+zip"]},
"djvu": {"ext": ".djvu", "mimes": ["application/vnd.djvu"]},
"rtf": {"ext": ".rtf", "mimes": ["application/rtf"]},
"pdf": {
"ext": ".pdf",
"mimes": ["application/pdf"]
},
"epub": {
"ext": ".epub",
"mimes": ["application/epub+zip"]
},
"djvu": {
"ext": ".djvu",
"mimes": ["application/vnd.djvu"]
},
"rtf": {
"ext": ".rtf",
"mimes": ["application/rtf"]
},
"docx": {
"ext": ".docx",
"mimes": ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"],
"ext":
".docx",
"mimes":
["application/vnd.openxmlformats-officedocument.wordprocessingml.document"],
},
"xlsx": {
"ext": ".xlsx",
"mimes": ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"],
"mimes":
["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"],
},
"pptx": {
"ext": ".pptx",
"mimes": ["application/vnd.openxmlformats-officedocument.presentationml.presentation"],
"ext":
".pptx",
"mimes": [
"application/vnd.openxmlformats-officedocument.presentationml.presentation"
],
},
"doc": {
"ext": ".doc",
"mimes": ["application/msword"]
},
"xls": {
"ext": ".xls",
"mimes": ["application/vnd.ms-excel"]
},
"ppt": {
"ext": ".ppt",
"mimes": ["application/vnd.ms-powerpoint"]
},
"doc": {"ext": ".doc", "mimes": ["application/msword"]},
"xls": {"ext": ".xls", "mimes": ["application/vnd.ms-excel"]},
"ppt": {"ext": ".ppt", "mimes": ["application/vnd.ms-powerpoint"]},
},
"archive": {
"zip": {"ext": ".zip", "mimes": ["application/zip"]},
"7z": {"ext": ".7z", "mimes": ["application/x-7z-compressed"]},
"rar": {"ext": ".rar", "mimes": ["application/x-rar-compressed", "application/vnd.rar"]},
"gz": {"ext": ".gz", "mimes": ["application/gzip", "application/x-gzip"]},
"tar": {"ext": ".tar", "mimes": ["application/x-tar"]},
"zip": {
"ext": ".zip",
"mimes": ["application/zip"]
},
"7z": {
"ext": ".7z",
"mimes": ["application/x-7z-compressed"]
},
"rar": {
"ext": ".rar",
"mimes": ["application/x-rar-compressed",
"application/vnd.rar"]
},
"gz": {
"ext": ".gz",
"mimes": ["application/gzip",
"application/x-gzip"]
},
"tar": {
"ext": ".tar",
"mimes": ["application/x-tar"]
},
"cbz": {
"ext": ".cbz",
"mimes": ["application/zip"],
"note": "zip archive of images; prefer extension-based detection for comics",
"note":
"zip archive of images; prefer extension-based detection for comics",
},
},
"project": {
"clip": {"ext": ".clip", "mimes": ["application/clip"]},
"kra": {"ext": ".kra", "mimes": ["application/x-krita"]},
"procreate": {"ext": ".procreate", "mimes": ["application/x-procreate"]},
"psd": {"ext": ".psd", "mimes": ["image/vnd.adobe.photoshop"]},
"swf": {"ext": ".swf", "mimes": ["application/x-shockwave-flash"]},
"clip": {
"ext": ".clip",
"mimes": ["application/clip"]
},
"kra": {
"ext": ".kra",
"mimes": ["application/x-krita"]
},
"procreate": {
"ext": ".procreate",
"mimes": ["application/x-procreate"]
},
"psd": {
"ext": ".psd",
"mimes": ["image/vnd.adobe.photoshop"]
},
"swf": {
"ext": ".swf",
"mimes": ["application/x-shockwave-flash"]
},
},
"other": {
"octet-stream": {"ext": "", "mimes": ["application/octet-stream"]},
"json": {"ext": ".json", "mimes": ["application/json"]},
"xml": {"ext": ".xml", "mimes": ["application/xml", "text/xml"]},
"csv": {"ext": ".csv", "mimes": ["text/csv"]},
"octet-stream": {
"ext": "",
"mimes": ["application/octet-stream"]
},
"json": {
"ext": ".json",
"mimes": ["application/json"]
},
"xml": {
"ext": ".xml",
"mimes": ["application/xml",
"text/xml"]
},
"csv": {
"ext": ".csv",
"mimes": ["text/csv"]
},
},
}

135
SYS/worker_manager.py
View File

@@ -170,7 +170,8 @@ class WorkerLoggingHandler(logging.StreamHandler):
# Set a format that includes timestamp and level
formatter = logging.Formatter(
"%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
"%(asctime)s - %(name)s - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S"
)
self.setFormatter(formatter)
@@ -196,7 +197,10 @@ class WorkerLoggingHandler(logging.StreamHandler):
# Add timestamp and level if not already in message
import time
timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(record.created))
timestamp = time.strftime(
"%Y-%m-%d %H:%M:%S",
time.localtime(record.created)
)
msg = f"{timestamp} - {record.name} - {record.levelname} - {msg}"
with self._lock:
@@ -214,9 +218,17 @@ class WorkerLoggingHandler(logging.StreamHandler):
log_text = "\n".join(self.buffer)
try:
if self.manager:
self.manager.append_worker_stdout(self.worker_id, log_text, channel="log")
self.manager.append_worker_stdout(
self.worker_id,
log_text,
channel="log"
)
else:
self.db.append_worker_stdout(self.worker_id, log_text, channel="log")
self.db.append_worker_stdout(
self.worker_id,
log_text,
channel="log"
)
except Exception as e:
# If we can't write to DB, at least log it
log(f"Error flushing worker logs: {e}")
@@ -251,8 +263,10 @@ class WorkerManager:
self.refresh_thread: Optional[Thread] = None
self._stop_refresh = False
self._lock = Lock()
self.worker_handlers: Dict[str, WorkerLoggingHandler] = {} # Track active handlers
self._worker_last_step: Dict[str, str] = {}
self.worker_handlers: Dict[str,
WorkerLoggingHandler] = {} # Track active handlers
self._worker_last_step: Dict[str,
str] = {}
def close(self) -> None:
"""Close the database connection."""
@@ -270,7 +284,12 @@ class WorkerManager:
"""Context manager exit - close database."""
self.close()
def add_refresh_callback(self, callback: Callable[[List[Dict[str, Any]]], None]) -> None:
def add_refresh_callback(
self,
callback: Callable[[List[Dict[str,
Any]]],
None]
) -> None:
"""Register a callback to be called on worker updates.
Args:
@@ -318,7 +337,8 @@ class WorkerManager:
if callback in self.refresh_callbacks:
self.refresh_callbacks.remove(callback)
def enable_logging_for_worker(self, worker_id: str) -> Optional[WorkerLoggingHandler]:
def enable_logging_for_worker(self,
worker_id: str) -> Optional[WorkerLoggingHandler]:
"""Enable logging capture for a worker.
Creates a logging handler that captures all logs for this worker.
@@ -343,7 +363,8 @@ class WorkerManager:
return handler
except Exception as e:
logger.error(
f"[WorkerManager] Error enabling logging for worker {worker_id}: {e}", exc_info=True
f"[WorkerManager] Error enabling logging for worker {worker_id}: {e}",
exc_info=True
)
return None
@@ -366,7 +387,9 @@ class WorkerManager:
root_logger = logging.getLogger()
root_logger.removeHandler(handler)
logger.debug(f"[WorkerManager] Disabled logging for worker: {worker_id}")
logger.debug(
f"[WorkerManager] Disabled logging for worker: {worker_id}"
)
except Exception as e:
logger.error(
f"[WorkerManager] Error disabling logging for worker {worker_id}: {e}",
@@ -397,10 +420,17 @@ class WorkerManager:
"""
try:
result = self.db.insert_worker(
worker_id, worker_type, title, description, total_steps, pipe=pipe
worker_id,
worker_type,
title,
description,
total_steps,
pipe=pipe
)
if result > 0:
logger.debug(f"[WorkerManager] Tracking worker: {worker_id} ({worker_type})")
logger.debug(
f"[WorkerManager] Tracking worker: {worker_id} ({worker_type})"
)
self._start_refresh_if_needed()
return True
return False
@@ -446,11 +476,18 @@ class WorkerManager:
return self.db.update_worker(worker_id, **kwargs)
return True
except Exception as e:
logger.error(f"[WorkerManager] Error updating worker {worker_id}: {e}", exc_info=True)
logger.error(
f"[WorkerManager] Error updating worker {worker_id}: {e}",
exc_info=True
)
return False
def finish_worker(
self, worker_id: str, result: str = "completed", error_msg: str = "", result_data: str = ""
self,
worker_id: str,
result: str = "completed",
error_msg: str = "",
result_data: str = ""
) -> bool:
"""Mark a worker as finished.
@@ -464,7 +501,10 @@ class WorkerManager:
True if update was successful
"""
try:
kwargs = {"status": result, "completed_at": datetime.now().isoformat()}
kwargs = {
"status": result,
"completed_at": datetime.now().isoformat()
}
if error_msg:
kwargs["error_message"] = error_msg
if result_data:
@@ -475,7 +515,10 @@ class WorkerManager:
self._worker_last_step.pop(worker_id, None)
return success
except Exception as e:
logger.error(f"[WorkerManager] Error finishing worker {worker_id}: {e}", exc_info=True)
logger.error(
f"[WorkerManager] Error finishing worker {worker_id}: {e}",
exc_info=True
)
return False
def get_active_workers(self) -> List[Dict[str, Any]]:
@@ -487,7 +530,10 @@ class WorkerManager:
try:
return self.db.get_active_workers()
except Exception as e:
logger.error(f"[WorkerManager] Error getting active workers: {e}", exc_info=True)
logger.error(
f"[WorkerManager] Error getting active workers: {e}",
exc_info=True
)
return []
def get_finished_workers(self, limit: int = 100) -> List[Dict[str, Any]]:
@@ -503,11 +549,15 @@ class WorkerManager:
all_workers = self.db.get_all_workers(limit=limit)
# Filter to only finished workers
finished = [
w for w in all_workers if w.get("status") in ["completed", "error", "cancelled"]
w for w in all_workers
if w.get("status") in ["completed", "error", "cancelled"]
]
return finished
except Exception as e:
logger.error(f"[WorkerManager] Error getting finished workers: {e}", exc_info=True)
logger.error(
f"[WorkerManager] Error getting finished workers: {e}",
exc_info=True
)
return []
def get_worker(self, worker_id: str) -> Optional[Dict[str, Any]]:
@@ -522,10 +572,16 @@ class WorkerManager:
try:
return self.db.get_worker(worker_id)
except Exception as e:
logger.error(f"[WorkerManager] Error getting worker {worker_id}: {e}", exc_info=True)
logger.error(
f"[WorkerManager] Error getting worker {worker_id}: {e}",
exc_info=True
)
return None
def get_worker_events(self, worker_id: str, limit: int = 500) -> List[Dict[str, Any]]:
def get_worker_events(self,
worker_id: str,
limit: int = 500) -> List[Dict[str,
Any]]:
"""Fetch recorded worker timeline events."""
return self.db.get_worker_events(worker_id, limit)
@@ -546,7 +602,8 @@ class WorkerManager:
return success
except Exception as e:
logger.error(
f"[WorkerManager] Error logging step for worker {worker_id}: {e}", exc_info=True
f"[WorkerManager] Error logging step for worker {worker_id}: {e}",
exc_info=True
)
return False
@@ -567,7 +624,8 @@ class WorkerManager:
return self.db.get_worker_steps(worker_id)
except Exception as e:
logger.error(
f"[WorkerManager] Error getting steps for worker {worker_id}: {e}", exc_info=True
f"[WorkerManager] Error getting steps for worker {worker_id}: {e}",
exc_info=True
)
return ""
@@ -613,7 +671,9 @@ class WorkerManager:
if not active:
# No more active workers, stop refreshing
logger.debug("[WorkerManager] No active workers, stopping auto-refresh")
logger.debug(
"[WorkerManager] No active workers, stopping auto-refresh"
)
break
# Call all registered callbacks with the active workers
@@ -623,11 +683,15 @@ class WorkerManager:
callback(active)
except Exception as e:
logger.error(
f"[WorkerManager] Error in refresh callback: {e}", exc_info=True
f"[WorkerManager] Error in refresh callback: {e}",
exc_info=True
)
except Exception as e:
logger.error(f"[WorkerManager] Error in auto-refresh loop: {e}", exc_info=True)
logger.error(
f"[WorkerManager] Error in auto-refresh loop: {e}",
exc_info=True
)
finally:
logger.debug("[WorkerManager] Auto-refresh loop ended")
@@ -646,7 +710,10 @@ class WorkerManager:
logger.info(f"[WorkerManager] Cleaned up {count} old workers")
return count
except Exception as e:
logger.error(f"[WorkerManager] Error cleaning up old workers: {e}", exc_info=True)
logger.error(
f"[WorkerManager] Error cleaning up old workers: {e}",
exc_info=True
)
return 0
def append_stdout(self, worker_id: str, text: str, channel: str = "stdout") -> bool:
@@ -662,7 +729,12 @@ class WorkerManager:
"""
try:
step_label = self._get_last_step(worker_id)
return self.db.append_worker_stdout(worker_id, text, step=step_label, channel=channel)
return self.db.append_worker_stdout(
worker_id,
text,
step=step_label,
channel=channel
)
except Exception as e:
logger.error(f"[WorkerManager] Error appending stdout: {e}", exc_info=True)
return False
@@ -682,7 +754,12 @@ class WorkerManager:
logger.error(f"[WorkerManager] Error getting stdout: {e}", exc_info=True)
return ""
def append_worker_stdout(self, worker_id: str, text: str, channel: str = "stdout") -> bool:
def append_worker_stdout(
self,
worker_id: str,
text: str,
channel: str = "stdout"
) -> bool:
"""Compatibility wrapper for append_stdout."""
return self.append_stdout(worker_id, text, channel=channel)