goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

update refactoring and add new features

Browse Source
This commit is contained in:
goyimnose
2026-07-24 20:55:58 -07:00
parent 7f12bc7f40
commit 03fbbbcf28
69 changed files with 16332 additions and 17600 deletions
Download Patch File Download Diff File Expand all files Collapse all files
SYS/database.py
+58 -17
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import atexit
import sqlite3
import json
import ast
@@ -18,6 +19,8 @@ logger = logging.getLogger(__name__)
# DB execute retry settings (for transient 'database is locked' errors)
_DB_EXEC_RETRY_MAX = 5
_DB_EXEC_RETRY_BASE_DELAY = 0.05
_DB_CONNECT_TIMEOUT = 30.0
_DB_BUSY_TIMEOUT_MS = 30000
# The database is located in the project root (prefer explicit repo hints).
def _resolve_root_dir() -> Path:
@@ -72,7 +75,7 @@ class Database:
self.conn = sqlite3.connect(
str(self.db_path),
check_same_thread=False,
timeout=30.0 # Increase timeout to 30s to avoid locking issues
timeout=_DB_CONNECT_TIMEOUT
)
self.conn.row_factory = sqlite3.Row
# Reentrant lock to allow nested DB calls within the same thread (e.g., transaction ->
@@ -82,7 +85,7 @@ class Database:
# Use WAL mode for better concurrency (allows multiple readers + 1 writer)
# Set a busy timeout so SQLite waits for short locks rather than immediately failing
try:
self.conn.execute("PRAGMA busy_timeout = 30000")
self.conn.execute(f"PRAGMA busy_timeout = {_DB_BUSY_TIMEOUT_MS}")
self.conn.execute("PRAGMA journal_mode=WAL")
self.conn.execute("PRAGMA synchronous=NORMAL")
except sqlite3.Error:
@@ -282,23 +285,27 @@ class Database:
# Singleton instance
db = Database()
_LOG_QUEUE: Queue[tuple[str, str, str]] = Queue()
_LOG_QUEUE: Queue = Queue()
_LOG_THREAD_STARTED = False
_LOG_THREAD_LOCK = threading.Lock()
_LOG_WRITE_COUNT = 0
_LOG_PRUNE_INTERVAL = 500 # prune every N successful writes
_LOG_MAX_AGE_DAYS = 7 # keep logs for this many days
_LOG_WRITE_RETRY_MAX = 3
_LOG_WRITE_RETRY_BASE_DELAY = 0.05
_LOG_SENTINEL = object()
_LOG_THREAD_REF: Optional[threading.Thread] = None
def _ensure_log_db_schema() -> None:
try:
conn = sqlite3.connect(
str(LOG_DB_PATH),
timeout=30.0,
timeout=_DB_CONNECT_TIMEOUT,
check_same_thread=False,
)
try:
conn.execute("PRAGMA busy_timeout = 30000")
conn.execute(f"PRAGMA busy_timeout = {_DB_BUSY_TIMEOUT_MS}")
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA synchronous=NORMAL")
conn.execute(
@@ -328,15 +335,20 @@ def _log_worker_loop() -> None:
"""
global _LOG_WRITE_COUNT
while True:
level, module, message = _LOG_QUEUE.get()
item = _LOG_QUEUE.get()
if item is _LOG_SENTINEL:
break
level, module, message = item
try:
attempts = 0
written = False
while attempts < 3 and not written:
while attempts < _LOG_WRITE_RETRY_MAX and not written:
conn = None
cur = None
try:
conn = sqlite3.connect(str(LOG_DB_PATH), timeout=30.0)
conn = sqlite3.connect(str(LOG_DB_PATH), timeout=_DB_CONNECT_TIMEOUT)
try:
conn.execute("PRAGMA busy_timeout = 30000")
conn.execute(f"PRAGMA busy_timeout = {_DB_BUSY_TIMEOUT_MS}")
conn.execute("PRAGMA journal_mode=WAL")
conn.execute("PRAGMA synchronous=NORMAL")
except sqlite3.Error:
@@ -352,13 +364,11 @@ def _log_worker_loop() -> None:
conn.commit()
except Exception:
pass
cur.close()
conn.close()
written = True
except sqlite3.OperationalError as exc:
attempts += 1
if 'locked' in str(exc).lower():
time.sleep(0.05 * attempts)
time.sleep(_LOG_WRITE_RETRY_BASE_DELAY * attempts)
continue
# Non-lock operational errors: abort attempts
log(f"Warning: Failed to write log entry (operational): {exc}")
@@ -366,6 +376,17 @@ def _log_worker_loop() -> None:
except Exception as exc:
log(f"Warning: Failed to write log entry: {exc}")
break
finally:
if cur is not None:
try:
cur.close()
except Exception:
pass
if conn is not None:
try:
conn.close()
except Exception:
pass
if not written:
# Fallback to a file-based log so we never lose the message silently
try:
@@ -399,7 +420,7 @@ def _log_worker_loop() -> None:
def _ensure_log_thread() -> None:
global _LOG_THREAD_STARTED
global _LOG_THREAD_STARTED, _LOG_THREAD_REF
if _LOG_THREAD_STARTED:
return
with _LOG_THREAD_LOCK:
@@ -411,8 +432,24 @@ def _ensure_log_thread() -> None:
daemon=True
)
thread.start()
_LOG_THREAD_REF = thread
_LOG_THREAD_STARTED = True
def _shutdown_log_thread() -> None:
global _LOG_THREAD_REF
_LOG_QUEUE.put(_LOG_SENTINEL)
thread = _LOG_THREAD_REF
if thread is not None:
try:
thread.join(timeout=3.0)
except Exception:
pass
_LOG_THREAD_REF = None
atexit.register(_shutdown_log_thread)
def get_db() -> Database:
return db
@@ -533,7 +570,11 @@ def get_config_all() -> Dict[str, Any]:
# Worker Management Methods for medios.db
def _worker_db_connect(timeout: float = 0.75) -> sqlite3.Connection:
_WORKER_DB_DEFAULT_TIMEOUT = 0.75
_WORKER_DB_DEFAULT_RETRIES = 1
_WORKER_DB_RETRY_BASE_DELAY = 0.05
def _worker_db_connect(timeout: float = _WORKER_DB_DEFAULT_TIMEOUT) -> sqlite3.Connection:
conn = sqlite3.connect(
str(DB_PATH),
timeout=timeout,
@@ -555,8 +596,8 @@ def _worker_db_execute(
params: tuple = (),
*,
fetch: Optional[str] = None,
timeout: float = 0.75,
retries: int = 1,
timeout: float = _WORKER_DB_DEFAULT_TIMEOUT,
retries: int = _WORKER_DB_DEFAULT_RETRIES,
) -> Any:
attempts = 0
while True:
@@ -580,7 +621,7 @@ def _worker_db_execute(
msg = str(exc).lower()
if "locked" in msg and attempts < retries:
attempts += 1
time.sleep(0.05 * attempts)
time.sleep(_WORKER_DB_RETRY_BASE_DELAY * attempts)
continue
raise
finally: