update refactoring and add new features

This commit is contained in:
2026-07-24 20:55:58 -07:00
parent 7f12bc7f40
commit 03fbbbcf28
69 changed files with 16332 additions and 17600 deletions
+8 -4
View File
@@ -26,12 +26,16 @@ SCRIPT_DIR = Path(__file__).resolve().parent
_SAVE_LOCK_DIRNAME = ".medios_save_lock"
_SAVE_LOCK_TIMEOUT = 30.0 # seconds to wait for save lock
_SAVE_LOCK_STALE_SECONDS = 3600 # consider lock stale after 1 hour
_SAVE_LOCK_POLL_INTERVAL = 0.1 # seconds between lock acquisition attempts
_WAL_CHECKPOINT_TIMEOUT = 5.0 # seconds for WAL checkpoint connection
_CONFIG_CACHE: Dict[str, Any] = {}
_LAST_SAVED_CONFIG: Dict[str, Any] = {}
_CONFIG_SUMMARY_PENDING = False
_CONFIG_SAVE_MAX_RETRIES = 5
_CONFIG_SAVE_RETRY_DELAY = 0.15
_CONFIG_SAVE_VERIFY_RETRIES = 3
_CONFIG_MISSING = object()
_PATH_ALIAS_TOKEN_RE = re.compile(r"^\$(?:\((?P<braced>[^)]+)\)|(?P<plain>[A-Za-z0-9_.-]+))$")
@@ -890,7 +894,7 @@ def _acquire_save_lock(timeout: float = _SAVE_LOCK_TIMEOUT):
logger.exception("Failed to inspect save lock directory %s: %s", lock_dir, exc)
if time.time() - start > timeout:
raise ConfigSaveConflict("Save lock busy; could not acquire in time")
time.sleep(0.1)
time.sleep(_SAVE_LOCK_POLL_INTERVAL)
def _release_save_lock(lock_dir: Path) -> None:
@@ -1030,10 +1034,10 @@ def save_config(config: Dict[str, Any]) -> int:
# we don't contend with our main connection lock or active transactions.
try:
try:
with sqlite3.connect(str(db.db_path), timeout=5.0) as _con:
with sqlite3.connect(str(db.db_path), timeout=_WAL_CHECKPOINT_TIMEOUT) as _con:
_con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
except Exception:
with sqlite3.connect(str(db.db_path), timeout=5.0) as _con:
with sqlite3.connect(str(db.db_path), timeout=_WAL_CHECKPOINT_TIMEOUT) as _con:
_con.execute("PRAGMA wal_checkpoint")
except Exception as exc:
log(f"Warning: WAL checkpoint failed: {exc}")
@@ -1084,7 +1088,7 @@ def save(config: Dict[str, Any]) -> int:
return save_config(config)
def save_config_and_verify(config: Dict[str, Any], retries: int = 3, delay: float = 0.15) -> int:
def save_config_and_verify(config: Dict[str, Any], retries: int = _CONFIG_SAVE_VERIFY_RETRIES, delay: float = _CONFIG_SAVE_RETRY_DELAY) -> int:
"""Save configuration and verify crucial keys persisted to disk.
This helper performs a best-effort verification loop that reloads the
+58 -17
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import atexit
import sqlite3
import json
import ast
@@ -18,6 +19,8 @@ logger = logging.getLogger(__name__)
# DB execute retry settings (for transient 'database is locked' errors)
_DB_EXEC_RETRY_MAX = 5
_DB_EXEC_RETRY_BASE_DELAY = 0.05
_DB_CONNECT_TIMEOUT = 30.0
_DB_BUSY_TIMEOUT_MS = 30000
# The database is located in the project root (prefer explicit repo hints).
def _resolve_root_dir() -> Path:
@@ -72,7 +75,7 @@ class Database:
self.conn = sqlite3.connect(
str(self.db_path),
check_same_thread=False,
timeout=30.0 # Increase timeout to 30s to avoid locking issues
timeout=_DB_CONNECT_TIMEOUT
)
self.conn.row_factory = sqlite3.Row
# Reentrant lock to allow nested DB calls within the same thread (e.g., transaction ->
@@ -82,7 +85,7 @@ class Database:
# Use WAL mode for better concurrency (allows multiple readers + 1 writer)
# Set a busy timeout so SQLite waits for short locks rather than immediately failing
try:
self.conn.execute("PRAGMA busy_timeout = 30000")
self.conn.execute(f"PRAGMA busy_timeout = {_DB_BUSY_TIMEOUT_MS}")
self.conn.execute("PRAGMA journal_mode=WAL")
self.conn.execute("PRAGMA synchronous=NORMAL")
except sqlite3.Error:
@@ -282,23 +285,27 @@ class Database:
# Singleton instance
db = Database()
_LOG_QUEUE: Queue[tuple[str, str, str]] = Queue()
_LOG_QUEUE: Queue = Queue()
_LOG_THREAD_STARTED = False
_LOG_THREAD_LOCK = threading.Lock()
_LOG_WRITE_COUNT = 0
_LOG_PRUNE_INTERVAL = 500 # prune every N successful writes
_LOG_MAX_AGE_DAYS = 7 # keep logs for this many days
_LOG_WRITE_RETRY_MAX = 3
_LOG_WRITE_RETRY_BASE_DELAY = 0.05
_LOG_SENTINEL = object()
_LOG_THREAD_REF: Optional[threading.Thread] = None
def _ensure_log_db_schema() -> None:
try:
conn = sqlite3.connect(
str(LOG_DB_PATH),
timeout=30.0,
timeout=_DB_CONNECT_TIMEOUT,
check_same_thread=False,
)
try:
conn.execute("PRAGMA busy_timeout = 30000")
conn.execute(f"PRAGMA busy_timeout = {_DB_BUSY_TIMEOUT_MS}")
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA synchronous=NORMAL")
conn.execute(
@@ -328,15 +335,20 @@ def _log_worker_loop() -> None:
"""
global _LOG_WRITE_COUNT
while True:
level, module, message = _LOG_QUEUE.get()
item = _LOG_QUEUE.get()
if item is _LOG_SENTINEL:
break
level, module, message = item
try:
attempts = 0
written = False
while attempts < 3 and not written:
while attempts < _LOG_WRITE_RETRY_MAX and not written:
conn = None
cur = None
try:
conn = sqlite3.connect(str(LOG_DB_PATH), timeout=30.0)
conn = sqlite3.connect(str(LOG_DB_PATH), timeout=_DB_CONNECT_TIMEOUT)
try:
conn.execute("PRAGMA busy_timeout = 30000")
conn.execute(f"PRAGMA busy_timeout = {_DB_BUSY_TIMEOUT_MS}")
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA synchronous=NORMAL")
except sqlite3.Error:
@@ -352,13 +364,11 @@ def _log_worker_loop() -> None:
conn.commit()
except Exception:
pass
cur.close()
conn.close()
written = True
except sqlite3.OperationalError as exc:
attempts += 1
if 'locked' in str(exc).lower():
time.sleep(0.05 * attempts)
time.sleep(_LOG_WRITE_RETRY_BASE_DELAY * attempts)
continue
# Non-lock operational errors: abort attempts
log(f"Warning: Failed to write log entry (operational): {exc}")
@@ -366,6 +376,17 @@ def _log_worker_loop() -> None:
except Exception as exc:
log(f"Warning: Failed to write log entry: {exc}")
break
finally:
if cur is not None:
try:
cur.close()
except Exception:
pass
if conn is not None:
try:
conn.close()
except Exception:
pass
if not written:
# Fallback to a file-based log so we never lose the message silently
try:
@@ -399,7 +420,7 @@ def _log_worker_loop() -> None:
def _ensure_log_thread() -> None:
global _LOG_THREAD_STARTED
global _LOG_THREAD_STARTED, _LOG_THREAD_REF
if _LOG_THREAD_STARTED:
return
with _LOG_THREAD_LOCK:
@@ -411,8 +432,24 @@ def _ensure_log_thread() -> None:
daemon=True
)
thread.start()
_LOG_THREAD_REF = thread
_LOG_THREAD_STARTED = True
def _shutdown_log_thread() -> None:
global _LOG_THREAD_REF
_LOG_QUEUE.put(_LOG_SENTINEL)
thread = _LOG_THREAD_REF
if thread is not None:
try:
thread.join(timeout=3.0)
except Exception:
pass
_LOG_THREAD_REF = None
atexit.register(_shutdown_log_thread)
def get_db() -> Database:
return db
@@ -533,7 +570,11 @@ def get_config_all() -> Dict[str, Any]:
# Worker Management Methods for medios.db
def _worker_db_connect(timeout: float = 0.75) -> sqlite3.Connection:
_WORKER_DB_DEFAULT_TIMEOUT = 0.75
_WORKER_DB_DEFAULT_RETRIES = 1
_WORKER_DB_RETRY_BASE_DELAY = 0.05
def _worker_db_connect(timeout: float = _WORKER_DB_DEFAULT_TIMEOUT) -> sqlite3.Connection:
conn = sqlite3.connect(
str(DB_PATH),
timeout=timeout,
@@ -555,8 +596,8 @@ def _worker_db_execute(
params: tuple = (),
*,
fetch: Optional[str] = None,
timeout: float = 0.75,
retries: int = 1,
timeout: float = _WORKER_DB_DEFAULT_TIMEOUT,
retries: int = _WORKER_DB_DEFAULT_RETRIES,
) -> Any:
attempts = 0
while True:
@@ -580,7 +621,7 @@ def _worker_db_execute(
msg = str(exc).lower()
if "locked" in msg and attempts < retries:
attempts += 1
time.sleep(0.05 * attempts)
time.sleep(_WORKER_DB_RETRY_BASE_DELAY * attempts)
continue
raise
finally:
+12 -5
View File
@@ -1,9 +1,11 @@
"""Simple HTTP file server for serving files in web mode."""
import atexit
import os
import threading
import socket
import logging
from http.server import HTTPServer, SimpleHTTPRequestHandler
from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler
from pathlib import Path
from typing import Optional
import mimetypes
@@ -11,9 +13,12 @@ import urllib.parse
logger = logging.getLogger(__name__)
_DNS_FALLBACK_HOST = os.environ.get("MM_DNS_SERVER", "8.8.8.8")
_DNS_FALLBACK_PORT = 80
# Global server instance
_file_server: Optional[HTTPServer] = None
_server_thread: Optional[threading.Thread] = None
_file_server: Optional[ThreadingHTTPServer] = None
_file_server_thread: Optional[threading.Thread] = None
_server_port: int = 8001
@@ -84,7 +89,7 @@ def get_local_ip() -> Optional[str]:
try:
# Connect to a remote server to determine local IP
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(("8.8.8.8", 80))
s.connect((_DNS_FALLBACK_HOST, _DNS_FALLBACK_PORT))
ip = s.getsockname()[0]
s.close()
return ip
@@ -116,7 +121,7 @@ def start_file_server(port: int = 8001) -> Optional[str]:
# Create server
server_address = ("", port)
_file_server = HTTPServer(server_address, FileServerHandler)
_file_server = ThreadingHTTPServer(server_address, FileServerHandler)
# Start in daemon thread
_server_thread = threading.Thread(
@@ -125,6 +130,8 @@ def start_file_server(port: int = 8001) -> Optional[str]:
)
_server_thread.start()
atexit.register(stop_file_server)
logger.info(f"File server started on port {port}")
# Get local IP
+5 -90
View File
@@ -523,8 +523,8 @@ def _read_sidecar_metadata(
lower = line.lower()
if lower.startswith("hash:"):
hash_value = line.split(":", 1)[1].strip() if ":" in line else ""
elif lower.startswith("url:") or lower.startswith("url:"):
# Parse url (handle legacy 'url:' format)
elif lower.startswith("url:"):
# Parse url
url_part = line.split(":", 1)[1].strip() if ":" in line else ""
if url_part:
for url_segment in url_part.split(","):
@@ -638,44 +638,7 @@ def write_tags(
return
except Exception as e:
debug(f"Failed to add tags to database: {e}", file=sys.stderr)
# Fall through to sidecar creation as fallback
# Create sidecar path
try:
sidecar = media_path.parent / (media_path.name + ".tag")
except Exception:
sidecar = media_path.with_name(media_path.name + ".tag")
# Handle edge case: empty/invalid base name
try:
if not sidecar.stem or sidecar.name in {".tag",
"-.tag",
"_.tag"}:
fallback_base = (
media_path.stem
or _sanitize_title_for_filename(extract_title(tag_list) or "")
or "untitled"
)
sidecar = media_path.parent / f"{fallback_base}.tag"
except Exception:
logger.exception("Failed to determine fallback .tag sidecar base for %s", media_path)
# Write via consolidated function
try:
lines: List[str] = []
lines.extend(str(tag).strip().lower() for tag in tag_list if str(tag).strip())
if lines:
sidecar.write_text("\n".join(lines) + "\n", encoding="utf-8")
if emit_debug:
debug(f"Tags: {sidecar}")
else:
try:
sidecar.unlink()
except FileNotFoundError:
pass
except OSError as exc:
debug(f"Failed to write tag sidecar {sidecar}: {exc}", file=sys.stderr)
return
def write_metadata(
@@ -729,45 +692,7 @@ def write_metadata(
return
except Exception as e:
debug(f"Failed to add metadata to database: {e}", file=sys.stderr)
# Fall through to sidecar creation as fallback
# Create sidecar path
try:
sidecar = media_path.parent / (media_path.name + ".metadata")
except Exception:
sidecar = media_path.with_name(media_path.name + ".metadata")
try:
lines = []
# Add hash if available
if hash_value:
lines.append(f"hash:{hash_value}")
# Add known url
for url in url_list:
if str(url).strip():
clean = str(url).strip()
lines.append(f"url:{clean}")
# Add relationships
for rel in rel_list:
if str(rel).strip():
lines.append(f"relationship:{str(rel).strip()}")
# Write metadata file
if lines:
sidecar.write_text("\n".join(lines) + "\n", encoding="utf-8")
if emit_debug:
debug(f"Wrote metadata to {sidecar}")
else:
# Remove if no content
try:
sidecar.unlink()
except FileNotFoundError:
pass
except OSError as exc:
debug(f"Failed to write metadata sidecar {sidecar}: {exc}", file=sys.stderr)
return
def extract_title(tags: Iterable[str]) -> Optional[str]:
@@ -2864,17 +2789,7 @@ def build_ffmpeg_command(
"192k",
])
cmd.extend(["-f", "opus"])
elif fmt == "audio":
# Legacy format name for mp3
cmd.extend([
"-vn",
"-c:a",
"libmp3lame",
"-b:a",
"192k",
])
cmd.extend(["-f", "mp3"])
elif fmt != "copy":
if fmt not in ("mp4", "webm", "mp3", "flac", "wav", "aac", "m4a", "ogg", "opus", "copy"):
raise ValueError(f"Unsupported format: {fmt}")
cmd.append(str(output_path))
+12 -3314
View File
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+41 -158
View File
@@ -45,22 +45,6 @@ def _rich():
return _rich_mod
# Optional Textual imports - lazily loaded to avoid pulling in ~300ms of textual
# at import time when the TUI is not being used.
import importlib.util as _importlib_util
TEXTUAL_AVAILABLE: bool = _importlib_util.find_spec("textual") is not None
# Tree is populated lazily on first call to build_metadata_tree().
_textual_Tree: Any = None
def _get_textual_Tree() -> Any:
global _textual_Tree
if _textual_Tree is None:
from textual.widgets import Tree as _Tree
_textual_Tree = _Tree
return _textual_Tree
# Import ResultModel from the API for typing; avoid runtime redefinition issues
from typing import TYPE_CHECKING
@@ -69,6 +53,8 @@ if TYPE_CHECKING:
else:
ResultModel = None # type: ignore[assignment]
# Reuse the existing format_bytes helper under a clearer alias
from SYS.utils import format_bytes as format_mb
@@ -587,30 +573,6 @@ class InputOption:
}
@dataclass
class TUIResultCard:
"""Represents a result as a UI card with title, metadata, and actions.
Used in hub-ui and TUI contexts to render individual search results
as grouped components with visual structure.
"""
title: str
subtitle: Optional[str] = None
metadata: Optional[Dict[str, str]] = None
media_kind: Optional[str] = None
tag: Optional[List[str]] = None
file_hash: Optional[str] = None
file_size: Optional[str] = None
duration: Optional[str] = None
def __post_init__(self):
"""Initialize default values."""
if self.metadata is None:
self.metadata = {}
if self.tag is None:
self.tag = []
@dataclass
class Column:
@@ -2113,122 +2075,6 @@ class Table:
return self.rows[idx]
return None
# TUI-specific formatting methods
def to_datatable_rows(self, source: str = "unknown") -> List[List[str]]:
"""Convert results to rows suitable for Textual DataTable widget.
Args:
source: Source type for formatting context (openlibrary, soulseek, etc.)
Returns:
List of row value lists
"""
rows = []
for result in self.rows:
row_values = self._format_datatable_row(result, source)
rows.append(row_values)
return rows
def _format_datatable_row(self,
row: Row,
source: str = "unknown") -> List[str]:
"""Format a ResultRow for DataTable display.
Args:
row: ResultRow to format
source: Source type
Returns:
List of column values as strings
"""
# Extract values from row columns
values = [col.value for col in row.columns]
# Truncate to reasonable lengths for table display
return [v[:60] if len(v) > 60 else v for v in values]
def to_result_cards(self) -> List[TUIResultCard]:
"""Convert all rows to TUIResultCard objects for card-based UI display.
Returns:
List of TUIResultCard objects
"""
cards = []
for row in self.rows:
card = self._row_to_card(row)
cards.append(card)
return cards
def _row_to_card(self, row: Row) -> TUIResultCard:
"""Convert a ResultRow to a TUIResultCard.
Args:
row: ResultRow to convert
Returns:
TUIResultCard with extracted metadata
"""
# Build metadata dict from row columns
metadata = {}
title = ""
for col in row.columns:
if col.name.lower() == "title":
title = col.value
metadata[col.name] = col.value
# Extract tag values if present
tag = []
if "Tag" in metadata:
tag_val = metadata["Tag"]
if tag_val:
tag = [t.strip() for t in tag_val.split(",")][:5]
# Try to find useful metadata fields
subtitle = metadata.get("Artist", metadata.get("Author", ""))
media_kind = metadata.get("Type", metadata.get("Media Kind", ""))
file_size = metadata.get("Size", "")
duration = metadata.get("Duration", "")
file_hash = metadata.get("Hash", "")
return TUIResultCard(
title=title or "Unknown",
subtitle=subtitle,
metadata=metadata,
media_kind=media_kind,
tag=tag,
file_hash=file_hash or None,
file_size=file_size or None,
duration=duration or None,
)
def build_metadata_tree(self, tree_widget: "Tree") -> None:
"""Populate a Textual Tree widget with result metadata hierarchy.
Args:
tree_widget: Textual Tree widget to populate
Raises:
ImportError: If Textual not available
"""
if not TEXTUAL_AVAILABLE:
raise ImportError("Textual not available for tree building")
tree_widget.reset(self.title or "Results")
root = tree_widget.root
# Add each row as a top-level node
for i, row in enumerate(self.rows, 1):
row_node = root.add(f"[bold]Result {i}[/bold]")
# Add columns as children
for col in row.columns:
value_str = col.value
if len(value_str) > 100:
value_str = value_str[:97] + "..."
row_node.add_leaf(f"[cyan]{col.name}[/cyan]: {value_str}")
def _format_size(size: Any, integer_only: bool = False) -> str:
"""Format file size as human-readable string.
@@ -2577,9 +2423,46 @@ class ItemDetailView(Table):
return f"{label[:max_len - 3]}..."
return label
def _looks_like_local_path(value: Any) -> bool:
text = str(value or "").strip()
if not text:
return False
try:
p = Path(text)
if p.exists():
return True
return text.startswith("/") or text.startswith("\\\\") or (len(text) >= 2 and text[1] == ":" and text[2:3] in ("\\", "/"))
except Exception:
return False
def _short_path_label(path_value: str, *, max_len: int = 68) -> str:
text = str(path_value or "").strip()
if not text:
return ""
if len(text) <= max_len:
return text
parts = text.replace("\\", "/").rstrip("/").split("/")
if len(parts) >= 3:
head = parts[0]
tail = parts[-1]
mid = "..."
label = f"{head}/{mid}/{tail}"
if len(label) > max_len:
return f"{text[:max_len - 3]}..."
return label
return f"{text[:max_len - 3]}..."
def _render_detail_value(key: str, value: Any) -> Any:
# Keep terminal links compact while preserving full click target.
if str(key or "").strip().lower() in {"path", "url"} and _looks_like_http_url(value):
key_lower = str(key or "").strip().lower()
if key_lower == "path" and _looks_like_local_path(value):
full_path = str(value).strip()
try:
file_uri = Path(full_path).as_uri()
except Exception:
return str(value)
label = _short_path_label(full_path)
return _rich().Text(label, style=f"underline cyan link {file_uri}")
if key_lower in {"path", "url"} and _looks_like_http_url(value):
full_url = str(value).strip()
label = _short_link_label(full_url)
return _rich().Text(label, style=f"underline cyan link {full_url}")
+1 -1
View File
@@ -109,7 +109,7 @@ def show_plugin_config_panel(
group = Group(
Text("The following plugins are not configured and cannot be used:\n"),
table,
Text.from_markup("\nTo configure them, run the command with [bold cyan].config[/bold cyan] or use the [bold green]TUI[/bold green] config menu.")
Text.from_markup("\nTo configure them, use the [bold cyan].config[/bold cyan] command in the REPL.")
)
panel = Panel(
+8 -3
View File
@@ -260,6 +260,11 @@ class WorkerLoggingHandler(logging.StreamHandler):
super().close()
_DEFAULT_STDOUT_FLUSH_BYTES = 4096
_DEFAULT_STDOUT_FLUSH_INTERVAL = 0.75
_REFRESH_THREAD_JOIN_TIMEOUT = 5
class WorkerManager:
"""Manages persistent worker tasks using the central medios.db."""
@@ -284,8 +289,8 @@ class WorkerManager:
self._stdout_buffer_sizes: Dict[Tuple[str, str], int] = {}
self._stdout_buffer_steps: Dict[Tuple[str, str], Optional[str]] = {}
self._stdout_last_flush: Dict[Tuple[str, str], float] = {}
self._stdout_flush_bytes = 4096
self._stdout_flush_interval = 0.75
self._stdout_flush_bytes = _DEFAULT_STDOUT_FLUSH_BYTES
self._stdout_flush_interval = _DEFAULT_STDOUT_FLUSH_INTERVAL
def __enter__(self):
@@ -707,7 +712,7 @@ class WorkerManager:
self._stop_refresh = True
self._refresh_stop_event.set()
if self.refresh_thread:
self.refresh_thread.join(timeout=5)
self.refresh_thread.join(timeout=_REFRESH_THREAD_JOIN_TIMEOUT)
self.refresh_thread = None
def _start_refresh_if_needed(self) -> None: