goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

fdsfdsd

Browse Source
This commit is contained in:
Nose
2026-01-30 10:47:47 -08:00
parent a44b80fd1d
commit ab94c57244
5 changed files with 872 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
API/data/alldebrid.json
View File

@@ -507,7 +507,7 @@
"mediafire\\.com/(\\?|download/|file/|download\\.php\\?)([0-9a-z]{15})"
],
"regexp": "mediafire\\.com/(\\?|download/|file/|download\\.php\\?)([0-9a-z]{15})",
"status": false
"status": true
},
"mexashare": {
"name": "mexashare",
@@ -679,7 +679,7 @@
"(uploadboy\\.com/[0-9a-zA-Z]{12})"
],
"regexp": "(uploadboy\\.com/[0-9a-zA-Z]{12})",
"status": false
"status": true
},
"uploader": {
"name": "uploader",

393
SYS/config.py
View File

@@ -5,21 +5,37 @@ from __future__ import annotations
import json
import sqlite3
import time
import os
import traceback
import datetime
import sys
import getpass
import hashlib
from copy import deepcopy
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from SYS.logger import log
from SYS.utils import expand_path
from SYS.database import db, get_config_all, save_config_value
from SYS.database import db, get_config_all, save_config_value, rows_to_config
SCRIPT_DIR = Path(__file__).resolve().parent
# Save lock settings (cross-process)
_SAVE_LOCK_DIRNAME = ".medios_save_lock"
_SAVE_LOCK_TIMEOUT = 30.0 # seconds to wait for save lock
_SAVE_LOCK_STALE_SECONDS = 3600 # consider lock stale after 1 hour
_CONFIG_CACHE: Dict[str, Any] = {}
_LAST_SAVED_CONFIG: Dict[str, Any] = {}
_CONFIG_SAVE_MAX_RETRIES = 5
_CONFIG_SAVE_RETRY_DELAY = 0.15
class ConfigSaveConflict(Exception):
"""Raised when a save would overwrite external changes present on disk."""
pass
def global_config() -> List[Dict[str, Any]]:
"""Return configuration schema for global settings."""
return [
@@ -455,6 +471,70 @@ def load_config() -> Dict[str, Any]:
_sync_alldebrid_api_key(db_config)
_CONFIG_CACHE = db_config
_LAST_SAVED_CONFIG = deepcopy(db_config)
try:
# Log a compact summary to help detect startup overwrites/mismatches
provs = list(db_config.get("provider", {}).keys()) if isinstance(db_config.get("provider"), dict) else []
stores = list(db_config.get("store", {}).keys()) if isinstance(db_config.get("store"), dict) else []
mtime = None
try:
mtime = datetime.datetime.utcfromtimestamp(db.db_path.stat().st_mtime).isoformat() + "Z"
except Exception:
mtime = None
summary = (
f"Loaded config from {db.db_path.name}: providers={len(provs)} ({', '.join(provs[:10])}{'...' if len(provs)>10 else ''}), "
f"stores={len(stores)} ({', '.join(stores[:10])}{'...' if len(stores)>10 else ''}), mtime={mtime}"
)
log(summary)
# Try to detect if the most recent audit indicates we previously saved items
# that are no longer present in the loaded config (possible overwrite/restore)
try:
audit_path = Path(db.db_path).with_name("config_audit.log")
if audit_path.exists():
last_line = None
with audit_path.open("r", encoding="utf-8") as fh:
for line in fh:
if line and line.strip():
last_line = line
if last_line:
try:
last_entry = json.loads(last_line)
last_provs = set(last_entry.get("providers") or [])
current_provs = set(provs)
missing = sorted(list(last_provs - current_provs))
if missing:
log(
f"WARNING: Config mismatch on load - last saved providers {sorted(list(last_provs))} "
f"are missing from loaded config: {missing} (last saved {last_entry.get('dt')})"
)
try:
# Write a forensic mismatch record to help diagnose potential overwrites
mismatch_path = Path(db.db_path).with_name("config_mismatch.log")
record = {
"detected": datetime.datetime.utcnow().isoformat() + "Z",
"db": str(db.db_path),
"db_mtime": mtime,
"last_saved_dt": last_entry.get("dt"),
"last_saved_providers": sorted(list(last_provs)),
"missing": missing,
}
try:
backup_dir = Path(db.db_path).with_name("config_backups")
if backup_dir.exists():
files = sorted(backup_dir.glob("medios-backup-*.db"), key=lambda p: p.stat().st_mtime, reverse=True)
record["latest_backup"] = str(files[0]) if files else None
except Exception:
pass
with mismatch_path.open("a", encoding="utf-8") as fh:
fh.write(json.dumps(record) + "\n")
except Exception:
pass
except Exception:
pass
except Exception:
pass
except Exception:
pass
return db_config
_LAST_SAVED_CONFIG = {}
@@ -466,64 +546,320 @@ def reload_config() -> Dict[str, Any]:
return load_config()
def _acquire_save_lock(timeout: float = _SAVE_LOCK_TIMEOUT):
"""Acquire a cross-process save lock implemented as a directory.
Returns the Path to the created lock directory. Raises ConfigSaveConflict
if the lock cannot be acquired within the timeout.
"""
lock_dir = Path(db.db_path).with_name(_SAVE_LOCK_DIRNAME)
start = time.time()
while True:
try:
lock_dir.mkdir(exist_ok=False)
# Write owner metadata for diagnostics
try:
(lock_dir / "owner.json").write_text(json.dumps({
"pid": os.getpid(),
"ts": time.time(),
"cmdline": " ".join(sys.argv),
}))
except Exception:
pass
return lock_dir
except FileExistsError:
# Check for stale lock
try:
owner = lock_dir / "owner.json"
if owner.exists():
data = json.loads(owner.read_text())
ts = data.get("ts") or 0
if time.time() - ts > _SAVE_LOCK_STALE_SECONDS:
try:
import shutil
shutil.rmtree(lock_dir)
continue
except Exception:
pass
else:
# No owner file; if directory is old enough consider it stale
try:
if time.time() - lock_dir.stat().st_mtime > _SAVE_LOCK_STALE_SECONDS:
import shutil
shutil.rmtree(lock_dir)
continue
except Exception:
pass
except Exception:
pass
if time.time() - start > timeout:
raise ConfigSaveConflict("Save lock busy; could not acquire in time")
time.sleep(0.1)
def _release_save_lock(lock_dir: Path) -> None:
try:
owner = lock_dir / "owner.json"
try:
if owner.exists():
owner.unlink()
except Exception:
pass
lock_dir.rmdir()
except Exception:
pass
def save_config(config: Dict[str, Any]) -> int:
global _CONFIG_CACHE, _LAST_SAVED_CONFIG
_sync_alldebrid_api_key(config)
# Acquire cross-process save lock to avoid concurrent saves from different
# processes which can lead to race conditions and DB-level overwrite.
lock_dir = None
try:
lock_dir = _acquire_save_lock()
except ConfigSaveConflict:
# Surface a clear exception to callers so they can retry or handle it.
raise
previous_config = deepcopy(_LAST_SAVED_CONFIG)
changed_count = _count_changed_entries(previous_config, config)
def _write_entries() -> int:
global _CONFIG_CACHE, _LAST_SAVED_CONFIG
count = 0
with db.transaction():
db.execute("DELETE FROM config")
# Use the transaction-provided connection directly to avoid re-acquiring
# the connection lock via db.* helpers which can lead to deadlock.
with db.transaction() as conn:
# Detect concurrent changes by reading the current DB state inside the
# same transaction before mutating it. Use the transaction connection
# directly to avoid acquiring the connection lock again (deadlock).
try:
cur = conn.cursor()
cur.execute("SELECT category, subtype, item_name, key, value FROM config")
rows = cur.fetchall()
current_disk = rows_to_config(rows)
cur.close()
except Exception:
current_disk = {}
if current_disk != _LAST_SAVED_CONFIG:
# If we have no local changes, refresh caches and skip the write.
if changed_count == 0:
log("Skip save: disk configuration changed since last load and no local changes; not writing to DB.")
# Refresh local caches to match the disk
_CONFIG_CACHE = current_disk
_LAST_SAVED_CONFIG = deepcopy(current_disk)
return 0
# Otherwise, abort to avoid overwriting external changes
raise ConfigSaveConflict(
"Configuration on disk changed since you started editing; save aborted to prevent overwrite. Reload and reapply your changes."
)
# Proceed with writing when no conflicting external changes detected
conn.execute("DELETE FROM config")
for key, value in config.items():
if key in ('store', 'provider', 'tool'):
if isinstance(value, dict):
for subtype, instances in value.items():
if not isinstance(instances, dict):
continue
if key == 'store':
for name, settings in instances.items():
if isinstance(settings, dict):
for k, v in settings.items():
save_config_value(key, subtype, name, k, v)
count += 1
else:
normalized_subtype = subtype
if key == 'provider':
normalized_subtype = _normalize_provider_name(subtype)
if not normalized_subtype:
continue
for k, v in instances.items():
save_config_value(key, normalized_subtype, "default", k, v)
count += 1
if key in ('store', 'provider', 'tool') and isinstance(value, dict):
for subtype, instances in value.items():
if not isinstance(instances, dict):
continue
if key == 'store':
for name, settings in instances.items():
if isinstance(settings, dict):
for k, v in settings.items():
val_str = json.dumps(v) if not isinstance(v, str) else v
conn.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
(key, subtype, name, k, val_str),
)
count += 1
else:
normalized_subtype = subtype
if key == 'provider':
normalized_subtype = _normalize_provider_name(subtype)
if not normalized_subtype:
continue
for k, v in instances.items():
val_str = json.dumps(v) if not isinstance(v, str) else v
conn.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
(key, normalized_subtype, "default", k, val_str),
)
count += 1
else:
if not key.startswith("_") and value is not None:
save_config_value("global", "none", "none", key, value)
val_str = json.dumps(value) if not isinstance(value, str) else value
conn.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
("global", "none", "none", key, val_str),
)
count += 1
return count
saved_entries = 0
attempts = 0
while True:
try:
saved_entries = _write_entries()
# Central log entry
log(
f"Synced {saved_entries} entries to {db.db_path} "
f"({changed_count} changed entries)"
)
# Try to checkpoint WAL to ensure main DB file reflects latest state.
# Use a separate short-lived connection to perform the checkpoint so
# we don't contend with our main connection lock or active transactions.
try:
try:
with sqlite3.connect(str(db.db_path), timeout=5.0) as _con:
_con.execute("PRAGMA wal_checkpoint(TRUNCATE)")
except Exception:
with sqlite3.connect(str(db.db_path), timeout=5.0) as _con:
_con.execute("PRAGMA wal_checkpoint")
except Exception as exc:
log(f"Warning: WAL checkpoint failed: {exc}")
# Audit to disk so we can correlate saves across restarts and processes.
# Audit to disk so we can correlate saves across restarts and processes.
try:
audit_path = Path(db.db_path).with_name("config_audit.log")
# Gather non-secret summary info (provider/store names)
provider_names = []
store_names = []
try:
pblock = config.get("provider")
if isinstance(pblock, dict):
provider_names = [str(k) for k in pblock.keys()]
except Exception:
provider_names = []
try:
sblock = config.get("store")
if isinstance(sblock, dict):
store_names = [str(k) for k in sblock.keys()]
except Exception:
store_names = []
stack = traceback.format_stack()
caller = stack[-1].strip() if stack else ""
# Try to include the database file modification time for correlation
db_mtime = None
try:
db_mtime = datetime.datetime.utcfromtimestamp(db.db_path.stat().st_mtime).isoformat() + "Z"
except Exception:
db_mtime = None
# Create a consistent timestamped backup of the DB so we can recover later
backup_path = None
try:
backup_dir = Path(db.db_path).with_name("config_backups")
backup_dir.mkdir(parents=False, exist_ok=True)
ts = datetime.datetime.utcnow().strftime("%Y%m%dT%H%M%SZ")
candidate = backup_dir / f"medios-backup-{ts}.db"
try:
# Use sqlite backup API for a consistent copy
src_con = sqlite3.connect(str(db.db_path))
dest_con = sqlite3.connect(str(candidate))
src_con.backup(dest_con)
dest_con.close()
src_con.close()
backup_path = str(candidate)
except Exception as e:
log(f"Warning: Failed to create DB backup: {e}")
# Prune older backups (keep last 20)
try:
files = sorted(backup_dir.glob("medios-backup-*.db"), key=lambda p: p.stat().st_mtime, reverse=True)
for old in files[20:]:
try:
old.unlink()
except Exception:
pass
except Exception:
pass
except Exception:
backup_path = None
# Collect process/exec info and a short hash of the config for forensic tracing
try:
exe = sys.executable
argv = list(sys.argv)
cwd = os.getcwd()
user = getpass.getuser()
try:
cfg_hash = hashlib.md5(json.dumps(config, sort_keys=True).encode('utf-8')).hexdigest()
except Exception:
cfg_hash = None
except Exception:
exe = None
argv = None
cwd = None
user = None
cfg_hash = None
entry = {
"ts": time.time(),
"dt": datetime.datetime.utcnow().isoformat() + "Z",
"pid": os.getpid(),
"exe": exe,
"argv": argv,
"cwd": cwd,
"user": user,
"stack": "".join(stack[-20:]),
"caller": caller,
"config_hash": cfg_hash,
"saved_entries": saved_entries,
"changed_count": changed_count,
"db": str(db.db_path),
"db_mtime": db_mtime,
"backup": backup_path,
"providers": provider_names,
"stores": store_names,
}
try:
with audit_path.open("a", encoding="utf-8") as fh:
fh.write(json.dumps(entry) + "\n")
except Exception:
# Best-effort; don't fail the save if audit write fails
log("Warning: Failed to write config audit file")
except Exception:
pass
finally:
# Release the save lock we acquired earlier
try:
if lock_dir is not None and lock_dir.exists():
_release_save_lock(lock_dir)
except Exception:
pass
break
except sqlite3.OperationalError as exc:
attempts += 1
locked_error = "locked" in str(exc).lower()
if not locked_error or attempts >= _CONFIG_SAVE_MAX_RETRIES:
log(f"CRITICAL: Database write failed: {exc}")
# Ensure we release potential save lock before bubbling error
try:
if lock_dir is not None and lock_dir.exists():
_release_save_lock(lock_dir)
except Exception:
pass
raise
delay = _CONFIG_SAVE_RETRY_DELAY * attempts
log(f"Database locked; retry {attempts}/{_CONFIG_SAVE_MAX_RETRIES} in {delay:.2f}s")
time.sleep(delay)
except Exception as exc:
log(f"CRITICAL: Configuration save failed: {exc}")
try:
if lock_dir is not None and lock_dir.exists():
_release_save_lock(lock_dir)
except Exception:
pass
raise
clear_config_cache()
@@ -540,3 +876,12 @@ def load() -> Dict[str, Any]:
def save(config: Dict[str, Any]) -> int:
"""Persist *config* back to disk."""
return save_config(config)
def count_changed_entries(config: Dict[str, Any]) -> int:
"""Return the number of changed configuration entries compared to the last saved snapshot.
This is useful for user-facing messages that want to indicate how many entries
were actually modified, not the total number of rows persisted to the database.
"""
return _count_changed_entries(_LAST_SAVED_CONFIG, config)

257
SYS/database.py
View File

@@ -8,8 +8,14 @@ from queue import Queue
from pathlib import Path
from typing import Any, Dict, List, Optional
from contextlib import contextmanager
import time
import datetime
from SYS.logger import log
# DB execute retry settings (for transient 'database is locked' errors)
_DB_EXEC_RETRY_MAX = 5
_DB_EXEC_RETRY_BASE_DELAY = 0.05
# The database is located in the project root (prefer explicit repo hints).
def _resolve_root_dir() -> Path:
env_root = (
@@ -65,9 +71,14 @@ class Database:
timeout=30.0 # Increase timeout to 30s to avoid locking issues
)
self.conn.row_factory = sqlite3.Row
# Reentrant lock to allow nested DB calls within the same thread (e.g., transaction ->
# get_config_all / save_config_value) without deadlocking.
self._conn_lock = threading.RLock()
# Use WAL mode for better concurrency (allows multiple readers + 1 writer)
# Set a busy timeout so SQLite waits for short locks rather than immediately failing
try:
self.conn.execute("PRAGMA busy_timeout = 30000")
self.conn.execute("PRAGMA journal_mode=WAL")
self.conn.execute("PRAGMA synchronous=NORMAL")
except sqlite3.Error:
@@ -139,61 +150,129 @@ class Database:
def get_connection(self):
return self.conn
def execute(self, query: str, params: tuple = ()):
cursor = self.conn.cursor()
try:
cursor.execute(query, params)
if not self.conn.in_transaction:
self.conn.commit()
return cursor
except Exception:
if not self.conn.in_transaction:
self.conn.rollback()
raise
def execute(self, query: str, params: tuple = ()):
attempts = 0
while True:
# Serialize access to the underlying sqlite connection to avoid
# concurrent use from multiple threads which can trigger locks.
with self._conn_lock:
cursor = self.conn.cursor()
try:
cursor.execute(query, params)
if not self.conn.in_transaction:
self.conn.commit()
return cursor
except sqlite3.OperationalError as exc:
msg = str(exc).lower()
# Retry a few times on transient lock errors
if 'locked' in msg and attempts < _DB_EXEC_RETRY_MAX:
attempts += 1
delay = _DB_EXEC_RETRY_BASE_DELAY * attempts
log(f"Database locked on execute; retry {attempts}/{_DB_EXEC_RETRY_MAX} in {delay:.2f}s")
try:
if not self.conn.in_transaction:
self.conn.rollback()
except Exception:
pass
time.sleep(delay)
continue
# Not recoverable or out of retries
if not self.conn.in_transaction:
try:
self.conn.rollback()
except Exception:
pass
raise
except Exception:
if not self.conn.in_transaction:
try:
self.conn.rollback()
except Exception:
pass
raise
def executemany(self, query: str, param_list: List[tuple]):
cursor = self.conn.cursor()
try:
cursor.executemany(query, param_list)
if not self.conn.in_transaction:
self.conn.commit()
return cursor
except Exception:
if not self.conn.in_transaction:
self.conn.rollback()
raise
attempts = 0
while True:
with self._conn_lock:
cursor = self.conn.cursor()
try:
cursor.executemany(query, param_list)
if not self.conn.in_transaction:
self.conn.commit()
return cursor
except sqlite3.OperationalError as exc:
msg = str(exc).lower()
if 'locked' in msg and attempts < _DB_EXEC_RETRY_MAX:
attempts += 1
delay = _DB_EXEC_RETRY_BASE_DELAY * attempts
log(f"Database locked on executemany; retry {attempts}/{_DB_EXEC_RETRY_MAX} in {delay:.2f}s")
try:
if not self.conn.in_transaction:
self.conn.rollback()
except Exception:
pass
time.sleep(delay)
continue
if not self.conn.in_transaction:
try:
self.conn.rollback()
except Exception:
pass
raise
except Exception:
if not self.conn.in_transaction:
try:
self.conn.rollback()
except Exception:
pass
raise
@contextmanager
def transaction(self):
"""Context manager for a database transaction."""
"""Context manager for a database transaction.
Transactions acquire the connection lock for the duration of the transaction
to prevent other threads from performing concurrent operations on the
same sqlite connection which can lead to locking issues.
"""
if self.conn.in_transaction:
# Already in a transaction, just yield
yield self.conn
else:
# Hold the connection lock for the lifetime of the transaction
self._conn_lock.acquire()
try:
self.conn.execute("BEGIN")
yield self.conn
self.conn.commit()
except Exception:
self.conn.rollback()
raise
try:
yield self.conn
self.conn.commit()
except Exception:
self.conn.rollback()
raise
finally:
try:
self._conn_lock.release()
except Exception:
pass
def fetchall(self, query: str, params: tuple = ()):
cursor = self.conn.cursor()
try:
cursor.execute(query, params)
return cursor.fetchall()
finally:
cursor.close()
def fetchone(self, query: str, params: tuple = ()):
cursor = self.conn.cursor()
try:
cursor.execute(query, params)
return cursor.fetchone()
finally:
cursor.close()
with self._conn_lock:
cursor = self.conn.cursor()
try:
cursor.execute(query, params)
return cursor.fetchall()
finally:
cursor.close()
def fetchone(self, query: str, params: tuple = ()):
with self._conn_lock:
cursor = self.conn.cursor()
try:
cursor.execute(query, params)
return cursor.fetchone()
finally:
cursor.close()
# Singleton instance
db = Database()
@@ -203,15 +282,50 @@ _LOG_THREAD_LOCK = threading.Lock()
def _log_worker_loop() -> None:
"""Background log writer using a temporary per-write connection with
small retry/backoff and a file fallback when writes fail repeatedly.
"""
while True:
level, module, message = _LOG_QUEUE.get()
try:
db.execute(
"INSERT INTO logs (level, module, message) VALUES (?, ?, ?)",
(level, module, message)
)
except Exception:
pass
attempts = 0
written = False
while attempts < 3 and not written:
try:
# Create a short-lived connection for the logging write so the
# logging thread does not contend with the main connection lock.
conn = sqlite3.connect(str(db.db_path), timeout=5.0)
cur = conn.cursor()
cur.execute("INSERT INTO logs (level, module, message) VALUES (?, ?, ?)", (level, module, message))
conn.commit()
cur.close()
conn.close()
written = True
except sqlite3.OperationalError as exc:
attempts += 1
if 'locked' in str(exc).lower():
time.sleep(0.05 * attempts)
continue
# Non-lock operational errors: abort attempts
log(f"Warning: Failed to write log entry (operational): {exc}")
break
except Exception as exc:
log(f"Warning: Failed to write log entry: {exc}")
break
if not written:
# Fallback to a file-based log so we never lose the message silently
try:
fallback_dir = Path(db.db_path).with_name("logs")
fallback_dir.mkdir(parents=True, exist_ok=True)
fallback_file = fallback_dir / "log_fallback.txt"
with fallback_file.open("a", encoding="utf-8") as fh:
fh.write(f"{datetime.datetime.utcnow().isoformat()}Z [{level}] {module}: {message}\n")
except Exception:
# Last resort: print to stderr
try:
log(f"ERROR: Could not persist log message: {level} {module} {message}")
except Exception:
pass
finally:
try:
_LOG_QUEUE.task_done()
@@ -261,11 +375,14 @@ def save_config_value(category: str, subtype: str, item_name: str, key: str, val
(category, subtype, item_name, key, val_str)
)
def get_config_all() -> Dict[str, Any]:
"""Retrieve all configuration from the database in the legacy dict format."""
rows = db.fetchall("SELECT category, subtype, item_name, key, value FROM config")
def rows_to_config(rows) -> Dict[str, Any]:
"""Convert DB rows (category, subtype, item_name, key, value) into a config dict.
This central helper is used by `get_config_all` and callers that need to
parse rows fetched with a transaction connection to avoid nested lock
acquisitions.
"""
config: Dict[str, Any] = {}
for row in rows:
cat = row['category']
sub = row['subtype']
@@ -276,13 +393,33 @@ def get_config_all() -> Dict[str, Any]:
# Drop legacy folder store entries (folder store is removed).
if cat == 'store' and str(sub).strip().lower() == 'folder':
continue
# Try to parse JSON value, fallback to string
# Conservative JSON parsing: only attempt to decode when the value
# looks like JSON (object/array/quoted string/true/false/null/number).
parsed_val = val
try:
parsed_val = json.loads(val)
if isinstance(val, str):
s = val.strip()
if s == "":
parsed_val = ""
else:
first = s[0]
lowered = s.lower()
if first in ('{', '[', '"') or lowered in ('true', 'false', 'null') or __import__('re').fullmatch(r'-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?', s):
try:
parsed_val = json.loads(val)
except Exception:
parsed_val = val
else:
parsed_val = val
else:
try:
parsed_val = json.loads(val)
except Exception:
parsed_val = val
except Exception:
parsed_val = val
if cat == 'global':
config[key] = parsed_val
else:
@@ -298,9 +435,15 @@ def get_config_all() -> Dict[str, Any]:
name_dict[key] = parsed_val
else:
config.setdefault(cat, {})[key] = parsed_val
return config
def get_config_all() -> Dict[str, Any]:
"""Retrieve all configuration from the database in the legacy dict format."""
rows = db.fetchall("SELECT category, subtype, item_name, key, value FROM config")
return rows_to_config(rows)
# Worker Management Methods for medios.db
def insert_worker(worker_id: str, worker_type: str, title: str = "", description: str = "") -> bool:

12
TUI.py
View File

@@ -42,6 +42,7 @@ from TUI.pipeline_runner import PipelineRunResult # type: ignore # noqa: E402
from SYS.result_table import Table, extract_hash_value, extract_store_value # type: ignore # noqa: E402
from SYS.config import load_config # type: ignore # noqa: E402
from SYS.database import db
from Store.registry import Store as StoreRegistry # type: ignore # noqa: E402
from SYS.cmdlet_catalog import ensure_registry_loaded, list_cmdlet_names # type: ignore # noqa: E402
from SYS.cli_syntax import validate_pipeline_text # type: ignore # noqa: E402
@@ -525,6 +526,17 @@ class PipelineHubApp(App):
# Run startup check automatically
self._run_pipeline_background(".status")
# Provide a visible startup summary of configured providers/stores for debugging
try:
cfg = load_config() or {}
provs = list(cfg.get("provider", {}).keys()) if isinstance(cfg.get("provider"), dict) else []
stores = list(cfg.get("store", {}).keys()) if isinstance(cfg.get("store"), dict) else []
prov_display = ", ".join(provs[:10]) + ("..." if len(provs) > 10 else "")
store_display = ", ".join(stores[:10]) + ("..." if len(stores) > 10 else "")
self._append_log_line(f"Startup config: providers={len(provs)} ({prov_display or '(none)'}), stores={len(stores)} ({store_display or '(none)')}, db={db.db_path.name}")
except Exception:
pass
# ------------------------------------------------------------------
# Actions
# ------------------------------------------------------------------

305
TUI/modalscreen/config_modal.py
View File

@@ -7,8 +7,10 @@ from textual.app import ComposeResult
from textual.containers import Container, Horizontal, Vertical, ScrollableContainer
from textual.screen import ModalScreen
from textual.widgets import Static, Button, Input, Label, ListView, ListItem, Rule, Select
from pathlib import Path
from SYS.config import load_config, save_config, reload_config, global_config
from SYS.config import load_config, save_config, reload_config, global_config, count_changed_entries, ConfigSaveConflict
from SYS.database import db
from SYS.logger import log
from Store.registry import _discover_store_classes, _required_keys_for
from ProviderCore.registry import list_providers
@@ -124,6 +126,8 @@ class ConfigModal(ModalScreen):
self._matrix_status: Optional[Static] = None
self._matrix_test_running = False
self._editor_snapshot: Optional[Dict[str, Any]] = None
# Path to the database file used by this process (for diagnostics)
self._db_path = str(db.db_path)
def _capture_editor_snapshot(self) -> None:
self._editor_snapshot = deepcopy(self.config_data)
@@ -141,8 +145,10 @@ class ConfigModal(ModalScreen):
def compose(self) -> ComposeResult:
with Container(id="config-container"):
yield Static("CONFIGURATION EDITOR", classes="section-title")
yield Static(f"DB: {self._db_path}", classes="config-label", id="config-db-path")
yield Static("Last saved: unknown", classes="config-label", id="config-last-save")
with Horizontal():
with Vertical(id="config-sidebar"):
with Vertical(id="config-sidebar"):
yield Label("Categories", classes="config-label")
with ListView(id="category-list"):
yield ListItem(Label("Global Settings"), id="cat-globals")
@@ -156,11 +162,28 @@ class ConfigModal(ModalScreen):
yield Button("Add Store", variant="primary", id="add-store-btn")
yield Button("Add Provider", variant="primary", id="add-provider-btn")
yield Button("Back", id="back-btn")
yield Button("Restore Backup", id="restore-backup-btn")
yield Button("Copy DB Path", id="copy-db-btn")
yield Button("Close", variant="error", id="cancel-btn")
def on_mount(self) -> None:
self.query_one("#add-store-btn", Button).display = False
self.query_one("#add-provider-btn", Button).display = False
# Update DB path and last-saved on mount
try:
self.query_one("#config-db-path", Static).update(self._db_path)
except Exception:
pass
try:
mtime = None
try:
mtime = db.db_path.stat().st_mtime
mtime = __import__('datetime').datetime.utcfromtimestamp(mtime).isoformat() + "Z"
except Exception:
mtime = None
self.query_one("#config-last-save", Static).update(f"Last saved: {mtime or '(unknown)'}")
except Exception:
pass
self.refresh_view()
def refresh_view(self) -> None:
@@ -568,19 +591,37 @@ class ConfigModal(ModalScreen):
if not self.validate_current_editor():
return
if self.editing_item_name and not self._editor_has_changes():
self.notify("No changes to save", severity="warning")
self.notify("No changes to save", severity="warning", timeout=3)
return
try:
saved = self.save_all()
msg = f"Configuration saved ({saved} entries)"
if saved == 0:
msg = "Configuration saved (no rows changed)"
self.notify(msg)
msg = f"Configuration saved (no rows changed) to {db.db_path.name}"
else:
msg = f"Configuration saved ({saved} change(s)) to {db.db_path.name}"
# Make the success notification visible a bit longer so it's not missed
self.notify(msg, timeout=5)
# Return to the main list view within the current category
self.editing_item_name = None
self.editing_item_type = None
self.refresh_view()
self._editor_snapshot = None
except ConfigSaveConflict as exc:
# A concurrent on-disk change was detected; do not overwrite it.
self.notify(
"Save aborted: configuration changed on disk. The editor will refresh.",
severity="error",
timeout=10,
)
# Refresh our in-memory view from disk and drop the editor snapshot
try:
self.config_data = reload_config()
except Exception:
pass
self._editor_snapshot = None
self.editing_item_name = None
self.editing_item_type = None
self.refresh_view()
except Exception as exc:
self.notify(f"Save failed: {exc}", severity="error", timeout=10)
elif bid in self._button_id_map:
@@ -602,13 +643,12 @@ class ConfigModal(ModalScreen):
if "provider" in self.config_data and name in self.config_data["provider"]:
del self.config_data["provider"][name]
removed = True
if str(name).strip().lower() == "alldebrid":
self._remove_alldebrid_store_entry()
if removed:
try:
saved = self.save_all()
msg = f"Configuration saved ({saved} entries)"
if saved == 0:
msg = "Configuration saved (no rows changed)"
self.notify(msg)
self.notify("Saving configuration...", timeout=3)
except Exception as exc:
self.notify(f"Save failed: {exc}", severity="error", timeout=10)
self.refresh_view()
@@ -640,6 +680,27 @@ class ConfigModal(ModalScreen):
self._request_matrix_test()
elif bid == "matrix-rooms-btn":
self._open_matrix_room_picker()
elif bid == "restore-backup-btn":
try:
backup = self._get_last_backup_path()
if not backup:
self.notify("No backups available", severity="warning")
else:
# Ask for confirmation via a simple notification and perform restore
self.notify(f"Restoring {backup.name}...", timeout=2)
self._restore_backup_background(str(backup))
except Exception as exc:
self.notify(f"Restore failed: {exc}", severity="error")
elif bid == "copy-db-btn":
try:
if hasattr(self.app, "copy_to_clipboard"):
self.app.copy_to_clipboard(str(db.db_path))
self.notify("DB path copied to clipboard")
else:
# Fall back to a visible notification
self.notify(str(db.db_path))
except Exception:
self.notify("Failed to copy DB path", severity="warning")
elif bid.startswith("paste-"):
# Programmatic paste button
target_id = bid.replace("paste-", "")
@@ -674,6 +735,65 @@ class ConfigModal(ModalScreen):
else:
self.notify("Clipboard not supported in this terminal", severity="warning")
def _get_last_backup_path(self):
try:
backup_dir = Path(db.db_path).with_name("config_backups")
if not backup_dir.exists():
return None
files = sorted(backup_dir.glob("medios-backup-*.db"), key=lambda p: p.stat().st_mtime, reverse=True)
return files[0] if files else None
except Exception:
return None
@work(thread=True)
def _restore_backup_background(self, backup_path: str) -> None:
try:
import sqlite3, json
cfg = {}
with sqlite3.connect(backup_path) as conn:
cur = conn.cursor()
cur.execute("SELECT category, subtype, item_name, key, value FROM config")
rows = cur.fetchall()
for cat, sub, name, key, val in rows:
try:
parsed = json.loads(val)
except Exception:
parsed = val
if cat == 'global':
cfg[key] = parsed
elif cat in ('provider', 'tool'):
cd = cfg.setdefault(cat, {})
sd = cd.setdefault(sub, {})
sd[key] = parsed
elif cat == 'store':
cd = cfg.setdefault('store', {})
sd = cd.setdefault(sub, {})
nd = sd.setdefault(name, {})
nd[key] = parsed
else:
cfg.setdefault(cat, {})[key] = parsed
# Persist restored config using save_config
from SYS.config import save_config, reload_config
saved = save_config(cfg)
# Reload and update UI from main thread
self.app.call_from_thread(self._on_restore_complete, True, backup_path, saved)
except Exception as exc:
self.app.call_from_thread(self._on_restore_complete, False, backup_path, str(exc))
def _on_restore_complete(self, success: bool, backup_path: str, saved_or_error):
if success:
# Refresh our in-memory view and UI
try:
from SYS.config import reload_config
self.config_data = reload_config()
self.refresh_view()
except Exception:
pass
self.notify(f"Restore complete: re-saved {saved_or_error} entries from {Path(backup_path).name}")
else:
self.notify(f"Restore failed: {saved_or_error}", severity="error")
def on_store_type_selected(self, stype: str) -> None:
if not stype:
return
@@ -847,6 +967,28 @@ class ConfigModal(ModalScreen):
self._update_config_value(widget_id, value)
def _remove_alldebrid_store_entry(self) -> bool:
"""Remove the mirrored AllDebrid store entry that would recreate the provider."""
store_block = self.config_data.get("store")
if not isinstance(store_block, dict):
return False
debrid = store_block.get("debrid")
if not isinstance(debrid, dict):
return False
removed = False
for key in list(debrid.keys()):
if str(key or "").strip().lower() == "all-debrid":
debrid.pop(key, None)
removed = True
if not debrid:
store_block.pop("debrid", None)
if not store_block:
self.config_data.pop("store", None)
return removed
def _get_matrix_provider_block(self) -> Dict[str, Any]:
providers = self.config_data.get("provider")
if not isinstance(providers, dict):
@@ -870,6 +1012,7 @@ class ConfigModal(ModalScreen):
self._synchronize_inputs_to_config()
if self._matrix_status:
self._matrix_status.update("Saving configuration before testing…")
changed = count_changed_entries(self.config_data)
try:
entries = save_config(self.config_data)
except Exception as exc:
@@ -879,7 +1022,7 @@ class ConfigModal(ModalScreen):
return
self.config_data = reload_config()
if self._matrix_status:
self._matrix_status.update(f"Saved configuration ({entries} entries). Testing Matrix connection…")
self._matrix_status.update(f"Saved configuration ({changed} change(s)) to {db.db_path.name}. Testing Matrix connection…")
self._matrix_test_running = True
self._matrix_test_background()
@@ -941,6 +1084,7 @@ class ConfigModal(ModalScreen):
return
matrix_block = self.config_data.setdefault("provider", {}).setdefault("matrix", {})
matrix_block["rooms"] = ", ".join(cleaned)
changed = count_changed_entries(self.config_data)
try:
entries = save_config(self.config_data)
except Exception as exc:
@@ -949,7 +1093,7 @@ class ConfigModal(ModalScreen):
return
self.config_data = reload_config()
if self._matrix_status:
status = f"Saved {len(cleaned)} default room(s) ({entries} rows persisted)."
status = f"Saved {len(cleaned)} default room(s) ({changed} change(s)) to {db.db_path.name}."
self._matrix_status.update(status)
self.refresh_view()
@@ -967,10 +1111,139 @@ class ConfigModal(ModalScreen):
def save_all(self) -> int:
self._synchronize_inputs_to_config()
entries = save_config(self.config_data)
self.config_data = reload_config()
log(f"ConfigModal saved {entries} configuration entries")
return entries
# Compute change count prior to persisting so callers can report the number of
# actual changes rather than the total number of rows written to the DB.
changed = count_changed_entries(self.config_data)
# Snapshot config for background save
snapshot = deepcopy(self.config_data)
# Schedule the save to run on a background worker so the UI doesn't block.
try:
# Prefer Textual's worker when running inside the app
self._save_background(snapshot, changed)
except Exception:
# Fallback: start a plain thread that runs the underlying task body
import threading
func = getattr(self._save_background, "__wrapped__", None)
if func:
thread = threading.Thread(target=lambda: func(self, snapshot, changed), daemon=True)
else:
thread = threading.Thread(target=lambda: self._save_background(snapshot, changed), daemon=True)
thread.start()
# Ensure DB path indicator is current and show saving status
self._db_path = str(db.db_path)
try:
self.query_one("#config-db-path", Static).update(self._db_path)
except Exception:
pass
try:
self.query_one("#config-last-save", Static).update("Last saved: (saving...)")
except Exception:
pass
log(f"ConfigModal scheduled save (changed={changed})")
return changed
@work(thread=True)
def _save_background(self, cfg: Dict[str, Any], changed: int) -> None:
try:
saved_entries = save_config(cfg)
try:
appobj = self.app
except Exception:
appobj = None
if appobj and hasattr(appobj, 'call_from_thread'):
appobj.call_from_thread(self._on_save_complete, True, None, changed, saved_entries)
else:
# If no app (e.g., running under tests), call completion directly
self._on_save_complete(True, None, changed, saved_entries)
except ConfigSaveConflict as exc:
try:
appobj = self.app
except Exception:
appobj = None
if appobj and hasattr(appobj, 'call_from_thread'):
appobj.call_from_thread(self._on_save_complete, False, str(exc), changed, 0)
else:
self._on_save_complete(False, str(exc), changed, 0)
except Exception as exc:
try:
appobj = self.app
except Exception:
appobj = None
if appobj and hasattr(appobj, 'call_from_thread'):
appobj.call_from_thread(self._on_save_complete, False, str(exc), changed, 0)
else:
self._on_save_complete(False, str(exc), changed, 0)
def _on_save_complete(self, success: bool, error: Optional[str], changed: int, saved_entries: int) -> None:
# Safely determine whether a Textual app context is available. Accessing
# `self.app` can raise when not running inside the TUI; handle that.
try:
appobj = self.app
except Exception:
appobj = None
if success:
try:
self.config_data = reload_config()
except Exception:
pass
# Update last-saved label with file timestamp for visibility
db_mtime = None
try:
db_mtime = db.db_path.stat().st_mtime
db_mtime = __import__('datetime').datetime.utcfromtimestamp(db_mtime).isoformat() + "Z"
except Exception:
db_mtime = None
if appobj:
try:
if changed == 0:
label_text = f"Last saved: (no changes)"
else:
label_text = f"Last saved: {changed} change(s) at {db_mtime or '(unknown)'}"
try:
self.query_one("#config-last-save", Static).update(label_text)
except Exception:
pass
except Exception:
pass
try:
self.refresh_view()
except Exception:
pass
try:
self.notify(f"Configuration saved ({changed} change(s)) to {db.db_path.name}", timeout=5)
except Exception:
pass
else:
# No TUI available; log instead of updating UI
log(f"Configuration saved ({changed} change(s)) to {db.db_path.name}")
log(f"ConfigModal saved {saved_entries} configuration entries (changed={changed})")
else:
# Save failed; notify via UI if available, otherwise log
if appobj:
try:
self.notify(f"Save failed: {error}", severity="error", timeout=10)
except Exception:
pass
try:
self.config_data = reload_config()
except Exception:
pass
try:
self.refresh_view()
except Exception:
pass
else:
log(f"Save failed: {error}")
def validate_current_editor(self) -> bool:
"""Ensure all required fields for the current item are filled."""