goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

f

Browse Source
This commit is contained in:
Nose
2026-01-22 01:53:13 -08:00
parent b3e7f3e277
commit 33406a6ecf
17 changed files with 857 additions and 877 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
CLI.py
View File

@@ -15,17 +15,33 @@ import os
from pathlib import Path from pathlib import Path
if not os.environ.get("MM_DEBUG"): if not os.environ.get("MM_DEBUG"):
try: try:
conf_path = Path(__file__).resolve().parent / "config.conf" # Check database first
if conf_path.exists(): db_path = Path(__file__).resolve().parent / "medios.db"
for ln in conf_path.read_text(encoding="utf-8").splitlines(): if db_path.exists():
ln_strip = ln.strip() import sqlite3
if ln_strip.startswith("debug"): with sqlite3.connect(str(db_path)) as conn:
parts = ln_strip.split("=", 1) cur = conn.cursor()
if len(parts) >= 2: # Check for global debug key
val = parts[1].strip().strip('"').strip("'").strip().lower() cur.execute("SELECT value FROM config WHERE key = 'debug' AND category = 'global'")
if val in ("1", "true", "yes", "on"): row = cur.fetchone()
os.environ["MM_DEBUG"] = "1" if row:
break val = str(row[0]).strip().lower()
if val in ("1", "true", "yes", "on"):
os.environ["MM_DEBUG"] = "1"
# Fallback to legacy config.conf if not set by DB
if not os.environ.get("MM_DEBUG"):
conf_path = Path(__file__).resolve().parent / "config.conf"
if conf_path.exists():
for ln in conf_path.read_text(encoding="utf-8").splitlines():
ln_strip = ln.strip()
if ln_strip.startswith("debug"):
parts = ln_strip.split("=", 1)
if len(parts) >= 2:
val = parts[1].strip().strip('"').strip("'").strip().lower()
if val in ("1", "true", "yes", "on"):
os.environ["MM_DEBUG"] = "1"
break
except Exception: except Exception:
pass pass
@@ -297,7 +313,7 @@ class CmdletIntrospection:
if normalized_arg in ("storage", "store"): if normalized_arg in ("storage", "store"):
# Use cached/lightweight names for completions to avoid instantiating backends # Use cached/lightweight names for completions to avoid instantiating backends
# (instantiating backends may perform initialization such as opening folder DBs). # (instantiating backends may perform heavy initialization).
backends = cls.store_choices(config, force=False) backends = cls.store_choices(config, force=False)
if backends: if backends:
return backends return backends
@@ -1484,7 +1500,7 @@ class CLI:
@app.command("remote-server") @app.command("remote-server")
def remote_server( def remote_server(
storage_path: str = typer.Argument( storage_path: str = typer.Argument(
None, help="Path to the store folder or store name from config" None, help="Path to the storage root"
), ),
port: int = typer.Option(None, "--port", help="Port to run the server on"), port: int = typer.Option(None, "--port", help="Port to run the server on"),
api_key: str | None = typer.Option(None, "--api-key", help="API key for authentication"), api_key: str | None = typer.Option(None, "--api-key", help="API key for authentication"),
@@ -1492,81 +1508,16 @@ class CLI:
debug_server: bool = False, debug_server: bool = False,
background: bool = False, background: bool = False,
) -> None: ) -> None:
"""Start the remote storage Flask server. """Start the remote storage server.
If no path is provided, it looks for [networking=zerotier] 'serve' and 'port' in config. NOTE: The legacy local storage server has been removed. Use HydrusNetwork
'serve' can be a path or the name of a [store=folder] entry. integrations instead.
Examples:
mm remote-server C:\\path\\to\\store --port 999 --api-key mykey
mm remote-server my_folder_name
mm remote-server --background
""" """
try: print(
from scripts import remote_storage_server as rss "Error: remote-server is no longer available because legacy local storage has been removed.",
except Exception as exc: file=sys.stderr,
print( )
"Error: remote_storage_server script not available:", return
exc,
file=sys.stderr,
)
return
# Ensure Flask present
if not getattr(rss, "HAS_FLASK", False):
print(
"ERROR: Flask and flask-cors required; install with: pip install flask flask-cors",
file=sys.stderr,
)
return
from SYS.config import load_config
conf = load_config()
# Resolve from Networking config if omitted
zt_conf = conf.get("networking", {}).get("zerotier", {})
if not storage_path:
storage_path = zt_conf.get("serve")
if port is None:
port = int(zt_conf.get("port") or 999)
if api_key is None:
api_key = zt_conf.get("api_key")
if not storage_path:
print(
"Error: No storage path provided and no [networking=zerotier] 'serve' configured.",
file=sys.stderr,
)
return
from pathlib import Path
# Check if storage_path is a named folder store
folders = conf.get("store", {}).get("folder", {})
found_path = None
for name, block in folders.items():
if name.lower() == storage_path.lower():
found_path = block.get("path") or block.get("PATH")
break
if found_path:
storage = Path(found_path).resolve()
else:
storage = Path(storage_path).resolve()
if not storage.exists():
print(f"Error: Storage path does not exist: {storage}", file=sys.stderr)
return
rss.STORAGE_PATH = storage
rss.API_KEY = api_key
try:
app_obj = rss.create_app()
except Exception as exc:
print("Failed to create remote_storage_server app:", exc, file=sys.stderr)
return
print( print(
f"Starting remote storage server at http://{host}:{port}, storage: {storage}" f"Starting remote storage server at http://{host}:{port}, storage: {storage}"
@@ -2102,64 +2053,6 @@ Come to love it when others take what you share, as there is no greater joy
detail=str(exc) detail=str(exc)
) )
if _has_store_subtype(config, "folder"):
store_cfg = config.get("store")
folder_cfg = store_cfg.get("folder",
{}) if isinstance(store_cfg,
dict) else {}
if isinstance(folder_cfg, dict) and folder_cfg:
for instance_name, instance_cfg in folder_cfg.items():
if not isinstance(instance_cfg, dict):
continue
name_key = str(instance_cfg.get("NAME") or instance_name)
path_val = str(
instance_cfg.get("PATH") or instance_cfg.get("path")
or ""
).strip()
ok = bool(
store_registry
and store_registry.is_available(name_key)
)
if ok and store_registry:
backend = store_registry[name_key]
scan_ok = bool(getattr(backend, "scan_ok", True))
scan_detail = str(
getattr(backend,
"scan_detail",
"") or ""
)
stats = getattr(backend, "scan_stats", None)
files = None
if isinstance(stats, dict):
total_db = stats.get("files_total_db")
if isinstance(total_db, (int, float)):
files = int(total_db)
status = "SCANNED" if scan_ok else "ERROR"
detail = (path_val + (" - " if path_val else "")
) + (scan_detail or "Up to date")
_add_startup_check(
status,
name_key,
store="folder",
files=files,
detail=detail
)
else:
err = None
if store_registry:
err = store_registry.get_backend_error(
instance_name
) or store_registry.get_backend_error(name_key)
detail = (path_val + (" - " if path_val else "")
) + (err or "Unavailable")
_add_startup_check(
"ERROR",
name_key,
store="folder",
detail=detail
)
if _has_store_subtype(config, "debrid"): if _has_store_subtype(config, "debrid"):
try: try:
from SYS.config import get_debrid_api_key from SYS.config import get_debrid_api_key

145
SYS/config.py
View File

@@ -8,6 +8,7 @@ from pathlib import Path
from typing import Any, Dict, Optional, List from typing import Any, Dict, Optional, List
from SYS.logger import log from SYS.logger import log
from SYS.utils import expand_path from SYS.utils import expand_path
from SYS.database import db, get_config_all, save_config_value
DEFAULT_CONFIG_FILENAME = "config.conf" DEFAULT_CONFIG_FILENAME = "config.conf"
SCRIPT_DIR = Path(__file__).resolve().parent SCRIPT_DIR = Path(__file__).resolve().parent
@@ -208,10 +209,9 @@ def parse_conf_text(text: str, *, base: Optional[Dict[str, Any]] = None) -> Dict
Supported patterns: Supported patterns:
- Top-level key/value: temp="./temp" - Top-level key/value: temp="./temp"
- Sections: [store=folder] + name/path lines
- Sections: [store=hydrusnetwork] + name/access key/url lines - Sections: [store=hydrusnetwork] + name/access key/url lines
- Sections: [provider=OpenLibrary] + email/password lines - Sections: [provider=OpenLibrary] + email/password lines
- Dotted keys: store.folder.default.path="C:\\Media" (optional) - Dotted keys: store.hydrusnetwork.<name>.url="http://..." (optional)
""" """
config: Dict[str, Any] = dict(base or {}) config: Dict[str, Any] = dict(base or {})
@@ -519,7 +519,6 @@ def get_local_storage_path(config: Dict[str, Any]) -> Optional[Path]:
"""Get local storage path from config. """Get local storage path from config.
Supports multiple formats: Supports multiple formats:
- New: config["store"]["folder"]["any_name"]["path"]
- Old: config["storage"]["local"]["path"] - Old: config["storage"]["local"]["path"]
- Old: config["Local"]["path"] - Old: config["Local"]["path"]
@@ -529,17 +528,6 @@ def get_local_storage_path(config: Dict[str, Any]) -> Optional[Path]:
Returns: Returns:
Path object if found, None otherwise Path object if found, None otherwise
""" """
# Try new format: iterate all folder stores and use the first valid path found.
store = config.get("store", {})
if isinstance(store, dict):
folder_config = store.get("folder", {})
if isinstance(folder_config, dict):
for name, inst_cfg in folder_config.items():
if isinstance(inst_cfg, dict):
p = inst_cfg.get("path") or inst_cfg.get("PATH")
if p:
return expand_path(p)
# Fall back to storage.local.path format # Fall back to storage.local.path format
storage = config.get("storage", {}) storage = config.get("storage", {})
if isinstance(storage, dict): if isinstance(storage, dict):
@@ -686,32 +674,76 @@ def resolve_debug_log(config: Dict[str, Any]) -> Optional[Path]:
return path return path
def migrate_conf_to_db(config: Dict[str, Any]) -> None:
"""Migrate the configuration dictionary to the database."""
log("Migrating configuration from .conf to database...")
for key, value in config.items():
if key in ("store", "provider", "tool", "networking"):
cat = key
sub_dict = value
if isinstance(sub_dict, dict):
for subtype, subtype_items in sub_dict.items():
if isinstance(subtype_items, dict):
# For provider/tool/networking, subtype is the name (e.g. alldebrid)
# but for store, it's the type (e.g. hydrusnetwork)
if cat == "store" and str(subtype).strip().lower() == "folder":
continue
if cat != "store":
for k, v in subtype_items.items():
save_config_value(cat, subtype, "", k, v)
else:
for name, items in subtype_items.items():
if isinstance(items, dict):
for k, v in items.items():
save_config_value(cat, subtype, name, k, v)
else:
# Global setting
save_config_value("global", "", "", key, value)
log("Configuration migration complete!")
def load_config( def load_config(
config_dir: Optional[Path] = None, filename: str = DEFAULT_CONFIG_FILENAME config_dir: Optional[Path] = None, filename: str = DEFAULT_CONFIG_FILENAME
) -> Dict[str, Any]: ) -> Dict[str, Any]:
base_dir = config_dir or SCRIPT_DIR base_dir = config_dir or SCRIPT_DIR
config_path = base_dir / filename config_path = base_dir / filename
cache_key = _make_cache_key(config_dir, filename, config_path) cache_key = _make_cache_key(config_dir, filename, config_path)
if cache_key in _CONFIG_CACHE: if cache_key in _CONFIG_CACHE:
return _CONFIG_CACHE[cache_key] return _CONFIG_CACHE[cache_key]
if config_path.suffix.lower() != ".conf": # 1. Try loading from database first
log(f"Unsupported config format: {config_path.name} (only .conf is supported)") db_config = get_config_all()
_CONFIG_CACHE[cache_key] = {} if db_config:
return {} _CONFIG_CACHE[cache_key] = db_config
return db_config
try: # 2. If DB is empty, try loading from legacy config.conf
data = _load_conf_config(base_dir, config_path) if config_path.exists():
except FileNotFoundError: if config_path.suffix.lower() != ".conf":
_CONFIG_CACHE[cache_key] = {} log(f"Unsupported config format: {config_path.name} (only .conf is supported)")
return {} return {}
except OSError as exc:
log(f"Failed to read {config_path}: {exc}") try:
_CONFIG_CACHE[cache_key] = {} config = _load_conf_config(base_dir, config_path)
return {} # Migrate to database
migrate_conf_to_db(config)
# Optional: Rename old config file to mark as migrated
try:
migrated_path = config_path.with_name(config_path.name + ".migrated")
config_path.rename(migrated_path)
log(f"Legacy config file renamed to {migrated_path.name}")
except Exception as e:
log(f"Could not rename legacy config file: {e}")
_CONFIG_CACHE[cache_key] = data _CONFIG_CACHE[cache_key] = config
return data return config
except Exception as e:
log(f"Failed to load legacy config at {config_path}: {e}")
return {}
return {}
def reload_config( def reload_config(
@@ -723,55 +755,12 @@ def reload_config(
def _validate_config_safety(config: Dict[str, Any]) -> None: def _validate_config_safety(config: Dict[str, Any]) -> None:
"""Check for dangerous configurations, like folder stores in non-empty dirs.""" """Validate configuration safety.
store = config.get("store")
if not isinstance(store, dict):
return
folder_stores = store.get("folder") Folder store validation has been removed because the folder store backend
if not isinstance(folder_stores, dict): is no longer supported.
return """
return
for name, cfg in folder_stores.items():
if not isinstance(cfg, dict):
continue
path_str = cfg.get("path") or cfg.get("PATH")
if not path_str:
continue
try:
p = expand_path(path_str).resolve()
# If the path doesn't exist yet, it's fine (will be created empty)
if not p.exists():
continue
if not p.is_dir():
continue
# DB name from API/folder.py
db_file = p / "medios-macina.db"
if db_file.exists():
# Existing portable library, allowed to re-attach
continue
# Check if directory has any files (other than the DB we just checked)
items = list(p.iterdir())
if items:
item_names = [i.name for i in items[:3]]
if len(items) > 3:
item_names.append("...")
raise RuntimeError(
f"Configuration Error: Local library '{name}' target directory is not empty.\n"
f"Path: {p}\n"
f"Found {len(items)} items: {', '.join(item_names)}\n"
f"To prevent accidental mass-hashing, new libraries must be set to unique, empty folders."
)
except RuntimeError:
raise
except Exception:
# We don't want to crash on invalid paths during validation if they aren't 'unsafe'
pass
def save_config( def save_config(
@@ -787,7 +776,7 @@ def save_config(
f"Unsupported config format: {config_path.name} (only .conf is supported)" f"Unsupported config format: {config_path.name} (only .conf is supported)"
) )
# Safety Check: Validate folder stores are in empty dirs or existing libraries # Safety Check: placeholder (folder store validation removed)
_validate_config_safety(config) _validate_config_safety(config)
try: try:

253
SYS/database.py Normal file
View File

@@ -0,0 +1,253 @@
from __future__ import annotations
import sqlite3
import json
from pathlib import Path
from typing import Any, Dict, List, Optional
# The database is located in the project root
ROOT_DIR = Path(__file__).resolve().parent.parent
DB_PATH = ROOT_DIR / "medios.db"
class Database:
_instance: Optional[Database] = None
def __new__(cls):
if cls._instance is None:
cls._instance = super(Database, cls).__new__(cls)
cls._instance._init_db()
return cls._instance
def _init_db(self):
self.conn = sqlite3.connect(str(DB_PATH), check_same_thread=False)
self.conn.row_factory = sqlite3.Row
self._create_tables()
def _create_tables(self):
cursor = self.conn.cursor()
# Config table: stores all settings previously in config.conf
# category: global, store, provider, tool, networking
# subtype: e.g., hydrusnetwork, folder, alldebrid
# item_name: e.g., hn-local, default
# key: the setting key
# value: the setting value (serialized to string)
cursor.execute("""
CREATE TABLE IF NOT EXISTS config (
category TEXT,
subtype TEXT,
item_name TEXT,
key TEXT,
value TEXT,
PRIMARY KEY (category, subtype, item_name, key)
)
""")
# Logs table
cursor.execute("""
CREATE TABLE IF NOT EXISTS logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
level TEXT,
module TEXT,
message TEXT
)
""")
# Workers table (for background tasks)
cursor.execute("""
CREATE TABLE IF NOT EXISTS workers (
id TEXT PRIMARY KEY,
type TEXT,
title TEXT,
description TEXT,
status TEXT DEFAULT 'running',
progress REAL DEFAULT 0.0,
details TEXT,
result TEXT DEFAULT 'pending',
error_message TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
""")
# Worker stdout/logs
cursor.execute("""
CREATE TABLE IF NOT EXISTS worker_stdout (
worker_id TEXT,
channel TEXT DEFAULT 'stdout',
content TEXT,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY(worker_id) REFERENCES workers(id)
)
""")
self.conn.commit()
def get_connection(self):
return self.conn
def execute(self, query: str, params: tuple = ()):
cursor = self.conn.cursor()
cursor.execute(query, params)
self.conn.commit()
return cursor
def fetchall(self, query: str, params: tuple = ()):
cursor = self.conn.cursor()
cursor.execute(query, params)
return cursor.fetchall()
def fetchone(self, query: str, params: tuple = ()):
cursor = self.conn.cursor()
cursor.execute(query, params)
return cursor.fetchone()
# Singleton instance
db = Database()
def get_db() -> Database:
return db
def log_to_db(level: str, module: str, message: str):
"""Log a message to the database."""
try:
db.execute(
"INSERT INTO logs (level, module, message) VALUES (?, ?, ?)",
(level, module, message)
)
except Exception:
# Avoid recursive logging errors if DB is locked
pass
# Initialize DB logger in the unified logger
try:
from SYS.logger import set_db_logger
set_db_logger(log_to_db)
except ImportError:
pass
def save_config_value(category: str, subtype: str, item_name: str, key: str, value: Any):
"""Save a configuration value to the database."""
val_str = json.dumps(value) if not isinstance(value, str) else value
db.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
(category, subtype, item_name, key, val_str)
)
def get_config_all() -> Dict[str, Any]:
"""Retrieve all configuration from the database in the legacy dict format."""
try:
db.execute("DELETE FROM config WHERE category='store' AND LOWER(subtype)='folder'")
except Exception:
pass
rows = db.fetchall("SELECT category, subtype, item_name, key, value FROM config")
config: Dict[str, Any] = {}
for row in rows:
cat = row['category']
sub = row['subtype']
name = row['item_name']
key = row['key']
val = row['value']
# Drop legacy folder store entries (folder store is removed).
if cat == 'store' and str(sub).strip().lower() == 'folder':
continue
# Try to parse JSON value, fallback to string
try:
parsed_val = json.loads(val)
except Exception:
parsed_val = val
if cat == 'global':
config[key] = parsed_val
else:
# Modular structure: config[cat][sub][name][key]
if cat in ('provider', 'tool', 'networking'):
cat_dict = config.setdefault(cat, {})
sub_dict = cat_dict.setdefault(sub, {})
sub_dict[key] = parsed_val
elif cat == 'store':
cat_dict = config.setdefault(cat, {})
sub_dict = cat_dict.setdefault(sub, {})
name_dict = sub_dict.setdefault(name, {})
name_dict[key] = parsed_val
else:
config.setdefault(cat, {})[key] = parsed_val
return config
# Worker Management Methods for medios.db
def insert_worker(worker_id: str, worker_type: str, title: str = "", description: str = "") -> bool:
try:
db.execute(
"INSERT INTO workers (id, type, title, description, status) VALUES (?, ?, ?, ?, 'running')",
(worker_id, worker_type, title, description)
)
return True
except Exception:
return False
def update_worker(worker_id: str, **kwargs) -> bool:
if not kwargs:
return True
# Filter valid columns
valid_cols = {'type', 'title', 'description', 'status', 'progress', 'details', 'result', 'error_message'}
cols = []
vals = []
for k, v in kwargs.items():
if k in valid_cols:
cols.append(f"{k} = ?")
vals.append(v)
if not cols:
return True
cols.append("updated_at = CURRENT_TIMESTAMP")
query = f"UPDATE workers SET {', '.join(cols)} WHERE id = ?"
vals.append(worker_id)
try:
db.execute(query, tuple(vals))
return True
except Exception:
return False
def append_worker_stdout(worker_id: str, content: str, channel: str = 'stdout'):
try:
db.execute(
"INSERT INTO worker_stdout (worker_id, channel, content) VALUES (?, ?, ?)",
(worker_id, channel, content)
)
except Exception:
pass
def get_worker_stdout(worker_id: str, channel: Optional[str] = None) -> str:
query = "SELECT content FROM worker_stdout WHERE worker_id = ?"
params = [worker_id]
if channel:
query += " AND channel = ?"
params.append(channel)
query += " ORDER BY timestamp ASC"
rows = db.fetchall(query, tuple(params))
return "\n".join(row['content'] for row in rows)
def get_active_workers() -> List[Dict[str, Any]]:
rows = db.fetchall("SELECT * FROM workers WHERE status = 'running' ORDER BY created_at DESC")
return [dict(row) for row in rows]
def get_worker(worker_id: str) -> Optional[Dict[str, Any]]:
row = db.fetchone("SELECT * FROM workers WHERE id = ?", (worker_id,))
return dict(row) if row else None
def expire_running_workers(older_than_seconds: int = 300, status: str = 'error', reason: str = 'timeout') -> int:
# SQLITE doesn't have a simple way to do DATETIME - INTERVAL, so we'll use strftime/unixepoch if available
# or just do regular update for all running ones for now as a simple fallback
query = f"UPDATE workers SET status = ?, error_message = ? WHERE status = 'running'"
db.execute(query, (status, reason))
return 0 # We don't easily get the rowcount from db.execute right now

17
SYS/logger.py
View File

@@ -4,9 +4,17 @@ import sys
import inspect import inspect
import threading import threading
from pathlib import Path from pathlib import Path
from typing import Optional
from SYS.rich_display import console_for from SYS.rich_display import console_for
# Global DB logger set later to avoid circular imports
_DB_LOGGER = None
def set_db_logger(func):
global _DB_LOGGER
_DB_LOGGER = func
_DEBUG_ENABLED = False _DEBUG_ENABLED = False
_thread_local = threading.local() _thread_local = threading.local()
@@ -207,6 +215,15 @@ def log(*args, **kwargs) -> None:
console_for(file).print(prefix, *args, sep=sep, end=end) console_for(file).print(prefix, *args, sep=sep, end=end)
else: else:
console_for(file).print(*args, sep=sep, end=end) console_for(file).print(*args, sep=sep, end=end)
# Log to database if available
if _DB_LOGGER:
try:
msg = sep.join(map(str, args))
level = "DEBUG" if add_prefix else "INFO"
_DB_LOGGER(level, f"{file_name}.{func_name}", msg)
except Exception:
pass
finally: finally:
del frame del frame
del caller_frame del caller_frame

146
SYS/worker_manager.py
View File

@@ -11,8 +11,17 @@ from datetime import datetime
from threading import Thread, Lock from threading import Thread, Lock
import time import time
from API.folder import API_folder_store
from SYS.logger import log from SYS.logger import log
from SYS.database import (
db,
insert_worker,
update_worker,
append_worker_stdout,
get_worker_stdout as db_get_worker_stdout,
get_active_workers as db_get_active_workers,
get_worker as db_get_worker,
expire_running_workers as db_expire_running_workers
)
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -157,7 +166,6 @@ class WorkerLoggingHandler(logging.StreamHandler):
def __init__( def __init__(
self, self,
worker_id: str, worker_id: str,
db: API_folder_store,
manager: Optional["WorkerManager"] = None, manager: Optional["WorkerManager"] = None,
buffer_size: int = 50, buffer_size: int = 50,
): ):
@@ -165,12 +173,10 @@ class WorkerLoggingHandler(logging.StreamHandler):
Args: Args:
worker_id: ID of the worker to capture logs for worker_id: ID of the worker to capture logs for
db: Reference to LocalLibraryDB for storing logs
buffer_size: Number of logs to buffer before flushing to DB buffer_size: Number of logs to buffer before flushing to DB
""" """
super().__init__() super().__init__()
self.worker_id = worker_id self.worker_id = worker_id
self.db = db
self.manager = manager self.manager = manager
self.buffer_size = buffer_size self.buffer_size = buffer_size
self.buffer: list[str] = [] self.buffer: list[str] = []
@@ -232,7 +238,7 @@ class WorkerLoggingHandler(logging.StreamHandler):
channel="log" channel="log"
) )
else: else:
self.db.append_worker_stdout( append_worker_stdout(
self.worker_id, self.worker_id,
log_text, log_text,
channel="log" channel="log"
@@ -255,29 +261,22 @@ class WorkerLoggingHandler(logging.StreamHandler):
class WorkerManager: class WorkerManager:
"""Manages persistent worker tasks with auto-refresh capability.""" """Manages persistent worker tasks using the central medios.db."""
def __init__(self, library_root: Path, auto_refresh_interval: float = 2.0): def __init__(self, auto_refresh_interval: float = 2.0):
"""Initialize the worker manager. """Initialize the worker manager.
Args: Args:
library_root: Root directory for the local library database
auto_refresh_interval: Seconds between auto-refresh checks (0 = disabled) auto_refresh_interval: Seconds between auto-refresh checks (0 = disabled)
""" """
self.library_root = Path(library_root)
self.db = API_folder_store(library_root)
self.auto_refresh_interval = auto_refresh_interval self.auto_refresh_interval = auto_refresh_interval
self.refresh_callbacks: List[Callable] = [] self.refresh_callbacks: List[Callable] = []
self.refresh_thread: Optional[Thread] = None self.refresh_thread: Optional[Thread] = None
self._stop_refresh = False self._stop_refresh = False
self._lock = Lock() self._lock = Lock()
# Reuse the DB's own lock so there is exactly one lock guarding the self.worker_handlers: Dict[str, WorkerLoggingHandler] = {}
# sqlite connection (and it is safe for re-entrant/nested DB usage). self._worker_last_step: Dict[str, str] = {}
self._db_lock = self.db._db_lock
self.worker_handlers: Dict[str,
WorkerLoggingHandler] = {} # Track active handlers
self._worker_last_step: Dict[str,
str] = {}
# Buffered stdout/log batching to reduce DB lock contention. # Buffered stdout/log batching to reduce DB lock contention.
self._stdout_buffers: Dict[Tuple[str, str], List[str]] = {} self._stdout_buffers: Dict[Tuple[str, str], List[str]] = {}
self._stdout_buffer_sizes: Dict[Tuple[str, str], int] = {} self._stdout_buffer_sizes: Dict[Tuple[str, str], int] = {}
@@ -328,13 +327,11 @@ class WorkerManager:
Count of workers updated. Count of workers updated.
""" """
try: try:
with self._db_lock: return db_expire_running_workers(
return self.db.expire_running_workers( older_than_seconds=older_than_seconds,
older_than_seconds=older_than_seconds, status=status,
status=status, reason=reason or "Stale worker expired"
reason=reason, )
worker_id_prefix=worker_id_prefix,
)
except Exception as exc: except Exception as exc:
logger.error(f"Failed to expire stale workers: {exc}", exc_info=True) logger.error(f"Failed to expire stale workers: {exc}", exc_info=True)
return 0 return 0
@@ -362,7 +359,7 @@ class WorkerManager:
The logging handler that was created, or None if there was an error The logging handler that was created, or None if there was an error
""" """
try: try:
handler = WorkerLoggingHandler(worker_id, self.db, manager=self) handler = WorkerLoggingHandler(worker_id, manager=self)
with self._lock: with self._lock:
self.worker_handlers[worker_id] = handler self.worker_handlers[worker_id] = handler
@@ -437,16 +434,13 @@ class WorkerManager:
True if worker was inserted successfully True if worker was inserted successfully
""" """
try: try:
with self._db_lock: success = insert_worker(
result = self.db.insert_worker( worker_id,
worker_id, worker_type,
worker_type, title,
title, description
description, )
total_steps, if success:
pipe=pipe
)
if result > 0:
logger.debug( logger.debug(
f"[WorkerManager] Tracking worker: {worker_id} ({worker_type})" f"[WorkerManager] Tracking worker: {worker_id} ({worker_type})"
) )
@@ -482,18 +476,16 @@ class WorkerManager:
if progress > 0: if progress > 0:
kwargs["progress"] = progress kwargs["progress"] = progress
if current_step: if current_step:
kwargs["current_step"] = current_step kwargs["details"] = current_step
if details: if details:
kwargs["description"] = details kwargs["description"] = details
if error: if error:
kwargs["error_message"] = error kwargs["error_message"] = error
if kwargs: if kwargs:
kwargs["last_updated"] = datetime.now().isoformat() if "details" in kwargs and kwargs["details"]:
if "current_step" in kwargs and kwargs["current_step"]: self._worker_last_step[worker_id] = str(kwargs["details"])
self._worker_last_step[worker_id] = str(kwargs["current_step"]) return update_worker(worker_id, **kwargs)
with self._db_lock:
return self.db.update_worker(worker_id, **kwargs)
return True return True
except Exception as e: except Exception as e:
logger.error( logger.error(
@@ -515,7 +507,7 @@ class WorkerManager:
worker_id: Unique identifier for the worker worker_id: Unique identifier for the worker
result: Result status ('completed', 'error', 'cancelled') result: Result status ('completed', 'error', 'cancelled')
error_msg: Error message if any error_msg: Error message if any
result_data: Result data as JSON string result_data: Result data as JSON string (saved in details)
Returns: Returns:
True if update was successful True if update was successful
@@ -526,16 +518,15 @@ class WorkerManager:
except Exception: except Exception:
pass pass
kwargs = { kwargs = {
"status": result, "status": "finished",
"completed_at": datetime.now().isoformat() "result": result
} }
if error_msg: if error_msg:
kwargs["error_message"] = error_msg kwargs["error_message"] = error_msg
if result_data: if result_data:
kwargs["result_data"] = result_data kwargs["details"] = result_data
with self._db_lock: success = update_worker(worker_id, **kwargs)
success = self.db.update_worker(worker_id, **kwargs)
logger.info(f"[WorkerManager] Worker finished: {worker_id} ({result})") logger.info(f"[WorkerManager] Worker finished: {worker_id} ({result})")
self._worker_last_step.pop(worker_id, None) self._worker_last_step.pop(worker_id, None)
return success return success
@@ -553,8 +544,7 @@ class WorkerManager:
List of active worker dictionaries List of active worker dictionaries
""" """
try: try:
with self._db_lock: return db_get_active_workers()
return self.db.get_active_workers()
except Exception as e: except Exception as e:
logger.error( logger.error(
f"[WorkerManager] Error getting active workers: {e}", f"[WorkerManager] Error getting active workers: {e}",
@@ -572,14 +562,9 @@ class WorkerManager:
List of finished worker dictionaries List of finished worker dictionaries
""" """
try: try:
with self._db_lock: # We don't have a get_all_workers in database.py yet, but we'll use a local query
all_workers = self.db.get_all_workers(limit=limit) rows = db.fetchall(f"SELECT * FROM workers WHERE status = 'finished' ORDER BY updated_at DESC LIMIT {limit}")
# Filter to only finished workers return [dict(row) for row in rows]
finished = [
w for w in all_workers
if w.get("status") in ["completed", "error", "cancelled"]
]
return finished
except Exception as e: except Exception as e:
logger.error( logger.error(
f"[WorkerManager] Error getting finished workers: {e}", f"[WorkerManager] Error getting finished workers: {e}",
@@ -597,7 +582,13 @@ class WorkerManager:
Worker data or None if not found Worker data or None if not found
""" """
try: try:
with self._db_lock: return db_get_worker(worker_id)
except Exception as e:
logger.error(
f"[WorkerManager] Error getting worker {worker_id}: {e}",
exc_info=True
)
return None
return self.db.get_worker(worker_id) return self.db.get_worker(worker_id)
except Exception as e: except Exception as e:
logger.error( logger.error(
@@ -815,14 +806,11 @@ class WorkerManager:
ok = True ok = True
for wid, ch, step, payload in pending_flush: for wid, ch, step, payload in pending_flush:
try: try:
with self._db_lock: append_worker_stdout(
result = self.db.append_worker_stdout( wid,
wid, payload,
payload, channel=ch
step=step, )
channel=ch
)
ok = ok and result
except Exception as e: except Exception as e:
logger.error( logger.error(
f"[WorkerManager] Error flushing stdout for {wid}: {e}", f"[WorkerManager] Error flushing stdout for {wid}: {e}",
@@ -851,7 +839,6 @@ class WorkerManager:
if not chunks: if not chunks:
return True return True
text = "".join(chunks) text = "".join(chunks)
step = self._stdout_buffer_steps.get(key)
self._stdout_buffers[key] = [] self._stdout_buffers[key] = []
self._stdout_buffer_sizes[key] = 0 self._stdout_buffer_sizes[key] = 0
self._stdout_last_flush[key] = time.monotonic() self._stdout_last_flush[key] = time.monotonic()
@@ -860,13 +847,12 @@ class WorkerManager:
if not text: if not text:
return True return True
try: try:
with self._db_lock: append_worker_stdout(
return self.db.append_worker_stdout( worker_id,
worker_id, text,
text, channel=channel,
step=step, )
channel=channel, return True
)
except Exception as e: except Exception as e:
logger.error( logger.error(
f"[WorkerManager] Error flushing stdout for {worker_id}: {e}", f"[WorkerManager] Error flushing stdout for {worker_id}: {e}",
@@ -884,8 +870,7 @@ class WorkerManager:
Worker's stdout or empty string Worker's stdout or empty string
""" """
try: try:
with self._db_lock: return db_get_worker_stdout(worker_id)
return self.db.get_worker_stdout(worker_id)
except Exception as e: except Exception as e:
logger.error(f"[WorkerManager] Error getting stdout: {e}", exc_info=True) logger.error(f"[WorkerManager] Error getting stdout: {e}", exc_info=True)
return "" return ""
@@ -909,21 +894,20 @@ class WorkerManager:
True if clear was successful True if clear was successful
""" """
try: try:
with self._db_lock: # Not implemented in database.py yet, but we'll add it or skip it
return self.db.clear_worker_stdout(worker_id) db.execute("DELETE FROM worker_stdout WHERE worker_id = ?", (worker_id,))
return True
except Exception as e: except Exception as e:
logger.error(f"[WorkerManager] Error clearing stdout: {e}", exc_info=True) logger.error(f"[WorkerManager] Error clearing stdout: {e}", exc_info=True)
return False return False
def close(self) -> None: def close(self) -> None:
"""Close the worker manager and database connection.""" """Close the worker manager."""
self.stop_auto_refresh() self.stop_auto_refresh()
try: try:
self._flush_all_stdout_buffers() self._flush_all_stdout_buffers()
except Exception: except Exception:
pass pass
with self._db_lock:
self.db.close()
logger.info("[WorkerManager] Closed") logger.info("[WorkerManager] Closed")
def _flush_all_stdout_buffers(self) -> None: def _flush_all_stdout_buffers(self) -> None:

1
Store/Folder.py
View File

@@ -90,6 +90,7 @@ class Folder(Store):
NAME: Optional[str] = None, NAME: Optional[str] = None,
PATH: Optional[str] = None, PATH: Optional[str] = None,
) -> None: ) -> None:
log("WARNING: Folder store is DEPRECATED and will be removed in a future version. Please migrate to HydrusNetwork.")
if name is None and NAME is not None: if name is None and NAME is not None:
name = str(NAME) name = str(NAME)
if location is None and PATH is not None: if location is None and PATH is not None:

4
Store/registry.py
View File

@@ -66,6 +66,8 @@ def _discover_store_classes() -> Dict[str, Type[BaseStore]]:
module_name = module_info.name module_name = module_info.name
if module_name in {"__init__", "_base", "registry"}: if module_name in {"__init__", "_base", "registry"}:
continue continue
if module_name.lower() == "folder":
continue
try: try:
module = importlib.import_module(f"Store.{module_name}") module = importlib.import_module(f"Store.{module_name}")
@@ -215,6 +217,8 @@ class Store:
continue continue
store_type = _normalize_store_type(str(raw_store_type)) store_type = _normalize_store_type(str(raw_store_type))
if store_type == "folder":
continue
store_cls = classes_by_type.get(store_type) store_cls = classes_by_type.get(store_type)
if store_cls is None: if store_cls is None:
if not self._suppress_debug: if not self._suppress_debug:

54
cmdlet/_shared.py
View File

@@ -2751,58 +2751,8 @@ def register_url_with_local_library(
Returns: Returns:
True if url were registered, False otherwise True if url were registered, False otherwise
""" """
# Folder store removed; local library URL registration is disabled.
try: return False
from SYS.config import get_local_storage_path
from API.folder import API_folder_store
file_path = get_field(pipe_obj, "path")
url_field = get_field(pipe_obj, "url", [])
urls: List[str] = []
if isinstance(url_field, str):
urls = [u.strip() for u in url_field.split(",") if u.strip()]
elif isinstance(url_field, (list, tuple)):
urls = [u for u in url_field if isinstance(u, str) and u.strip()]
if not file_path or not urls:
return False
path_obj = Path(file_path)
if not path_obj.exists():
return False
storage_path = get_local_storage_path(config)
if not storage_path:
return False
# Optimization: Don't open DB if file isn't in library root
try:
path_obj.resolve().relative_to(Path(storage_path).resolve())
except ValueError:
return False
with API_folder_store(storage_path) as db:
file_hash = db.get_file_hash(path_obj)
if not file_hash:
return False
metadata = db.get_metadata(file_hash) or {}
existing_url = metadata.get("url") or []
# Add any new url
changed = False
for u in urls:
if u not in existing_url:
existing_url.append(u)
changed = True
if changed:
metadata["url"] = existing_url
db.save_metadata(path_obj, metadata)
return True
return True # url already existed
except Exception:
return False
def resolve_tidal_manifest_path(item: Any) -> Optional[str]: def resolve_tidal_manifest_path(item: Any) -> Optional[str]:

91
cmdlet/delete_file.py
View File

@@ -7,7 +7,6 @@ import sys
from pathlib import Path from pathlib import Path
from SYS.logger import debug, log from SYS.logger import debug, log
from Store.Folder import Folder
from Store import Store from Store import Store
from . import _shared as sh from . import _shared as sh
from API import HydrusNetwork as hydrus_wrapper from API import HydrusNetwork as hydrus_wrapper
@@ -280,61 +279,23 @@ class Delete_File(sh.Cmdlet):
except Exception: except Exception:
size_bytes = None size_bytes = None
# If lib_root is provided and this is from a folder store, use the Folder class # Delete the local file directly
if lib_root: try:
try: if path.exists() and path.is_file():
folder = Folder(Path(lib_root), name=store or "local") path.unlink()
if folder.delete_file(str(path)): local_deleted = True
local_deleted = True deleted_rows.append(
deleted_rows.append( {
{ "title":
"title": str(title_val).strip() if title_val else path.name,
str(title_val).strip() if title_val else path.name, "store": store_label,
"store": store_label, "hash": hash_hex or sh.normalize_hash(path.stem) or "",
"hash": hash_hex or sh.normalize_hash(path.stem) or "", "size_bytes": size_bytes,
"size_bytes": size_bytes, "ext": _get_ext_from_item() or path.suffix.lstrip("."),
"ext": _get_ext_from_item() or path.suffix.lstrip("."), }
} )
) except Exception as exc:
except Exception as exc: log(f"Local delete failed: {exc}", file=sys.stderr)
debug(f"Folder.delete_file failed: {exc}", file=sys.stderr)
# Fallback to manual deletion
try:
if path.exists() and path.is_file():
path.unlink()
local_deleted = True
deleted_rows.append(
{
"title":
str(title_val).strip() if title_val else path.name,
"store": store_label,
"hash": hash_hex or sh.normalize_hash(path.stem)
or "",
"size_bytes": size_bytes,
"ext": _get_ext_from_item()
or path.suffix.lstrip("."),
}
)
except Exception as exc:
log(f"Local delete failed: {exc}", file=sys.stderr)
else:
# No lib_root, just delete the file
try:
if path.exists() and path.is_file():
path.unlink()
local_deleted = True
deleted_rows.append(
{
"title":
str(title_val).strip() if title_val else path.name,
"store": store_label,
"hash": hash_hex or sh.normalize_hash(path.stem) or "",
"size_bytes": size_bytes,
"ext": _get_ext_from_item() or path.suffix.lstrip("."),
}
)
except Exception as exc:
log(f"Local delete failed: {exc}", file=sys.stderr)
# Remove common sidecars regardless of file removal success # Remove common sidecars regardless of file removal success
for sidecar in ( for sidecar in (
@@ -533,24 +494,6 @@ class Delete_File(sh.Cmdlet):
log("Invalid -query value (expected hash:<sha256>)", file=sys.stderr) log("Invalid -query value (expected hash:<sha256>)", file=sys.stderr)
return 1 return 1
# If no lib_root provided, try to get the first folder store from config
if not lib_root:
try:
storage_config = config.get("storage",
{})
folder_config = storage_config.get("folder",
{})
if folder_config:
# Get first folder store path
for store_name, store_config in folder_config.items():
if isinstance(store_config, dict):
path = store_config.get("path")
if path:
lib_root = path
break
except Exception:
pass
reason = " ".join(token for token in reason_tokens reason = " ".join(token for token in reason_tokens
if str(token).strip()).strip() if str(token).strip()).strip()

21
cmdlet/get_file.py
View File

@@ -175,27 +175,6 @@ class Get_File(sh.Cmdlet):
log(f"Error: Backend could not retrieve file for hash {file_hash}") log(f"Error: Backend could not retrieve file for hash {file_hash}")
return 1 return 1
# Folder store UX: without -path, just open the file in the default app.
# Only export/copy when -path is explicitly provided.
backend_name = type(backend).__name__
is_folder_backend = backend_name.lower() == "folder"
if is_folder_backend and not output_path:
display_title = resolve_display_title() or source_path.stem or "Opened"
ext_for_emit = metadata.get("ext") or source_path.suffix.lstrip(".")
self._open_file_default(source_path)
log(f"Opened: {source_path}", file=sys.stderr)
ctx.emit(
{
"hash": file_hash,
"store": store_name,
"path": str(source_path),
"title": str(display_title),
"ext": str(ext_for_emit or ""),
}
)
debug("[get-file] Completed successfully")
return 0
# Otherwise: export/copy to output_dir. # Otherwise: export/copy to output_dir.
if output_path: if output_path:
output_dir = Path(output_path).expanduser() output_dir = Path(output_path).expanduser()

156
cmdlet/get_relationship.py
View File

@@ -2,7 +2,6 @@ from __future__ import annotations
from typing import Any, Dict, Sequence, Optional from typing import Any, Dict, Sequence, Optional
import sys import sys
from pathlib import Path
from SYS.logger import log from SYS.logger import log
@@ -19,12 +18,11 @@ get_hash_for_operation = sh.get_hash_for_operation
fetch_hydrus_metadata = sh.fetch_hydrus_metadata fetch_hydrus_metadata = sh.fetch_hydrus_metadata
should_show_help = sh.should_show_help should_show_help = sh.should_show_help
get_field = sh.get_field get_field = sh.get_field
from API.folder import API_folder_store
from Store import Store from Store import Store
CMDLET = Cmdlet( CMDLET = Cmdlet(
name="get-relationship", name="get-relationship",
summary="Print relationships for the selected file (Hydrus or Local).", summary="Print relationships for the selected file (Hydrus).",
usage='get-relationship [-query "hash:<sha256>"]', usage='get-relationship [-query "hash:<sha256>"]',
alias=[], alias=[],
arg=[ arg=[
@@ -32,155 +30,12 @@ CMDLET = Cmdlet(
SharedArgs.STORE, SharedArgs.STORE,
], ],
detail=[ detail=[
"- Lists relationship data as returned by Hydrus or Local DB.", "- Lists relationship data as returned by Hydrus.",
], ],
) )
def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
# Help
if should_show_help(_args):
log(f"Cmdlet: {CMDLET.name}\nSummary: {CMDLET.summary}\nUsage: {CMDLET.usage}")
return 0
# Parse -query and -store override
override_query: str | None = None
override_store: str | None = None
args_list = list(_args)
i = 0
while i < len(args_list):
a = args_list[i]
low = str(a).lower()
if low in {"-query",
"--query",
"query"} and i + 1 < len(args_list):
override_query = str(args_list[i + 1]).strip()
i += 2
continue
if low in {"-store",
"--store",
"store"} and i + 1 < len(args_list):
override_store = str(args_list[i + 1]).strip()
i += 2
continue
i += 1
override_hash: str | None = (
sh.parse_single_hash_query(override_query) if override_query else None
)
if override_query and not override_hash:
log('get-relationship requires -query "hash:<sha256>"', file=sys.stderr)
return 1
# Handle @N selection which creates a list
# This cmdlet is single-subject; require disambiguation when multiple items are provided.
if isinstance(result, list):
if len(result) == 0:
result = None
elif len(result) > 1 and not override_hash:
log(
'get-relationship expects a single item; select one row (e.g. @1) or pass -query "hash:<sha256>"',
file=sys.stderr,
)
return 1
else:
result = result[0]
# Initialize results collection
found_relationships = [] # List of dicts: {hash, type, title, path, store}
source_title = "Unknown"
def _add_relationship(entry: Dict[str, Any]) -> None:
"""Add relationship if not already present by hash or path."""
for existing in found_relationships:
if (entry.get("hash")
and str(existing.get("hash",
"")).lower() == str(entry["hash"]).lower()):
return
if (entry.get("path")
and str(existing.get("path",
"")).lower() == str(entry["path"]).lower()):
return
found_relationships.append(entry)
# Store/hash-first subject resolution
store_name: Optional[str] = override_store
if not store_name:
store_name = get_field(result, "store")
hash_hex = (
normalize_hash(override_hash)
if override_hash else normalize_hash(get_hash_for_operation(None,
result))
)
if not source_title or source_title == "Unknown":
source_title = (
get_field(result,
"title") or get_field(result,
"name")
or (hash_hex[:16] + "..." if hash_hex else "Unknown")
)
local_db_checked = False
if store_name and hash_hex:
try:
store = Store(config)
backend = store[str(store_name)]
# Folder store relationships
# IMPORTANT: only treat the Folder backend as a local DB store.
# Other backends may expose a location() method but are not SQLite folder stores.
if (type(backend).__name__ == "Folder" and hasattr(backend,
"location")
and callable(getattr(backend,
"location"))):
storage_path = Path(str(backend.location()))
with API_folder_store(storage_path) as db:
local_db_checked = True
# Update source title from tags if possible
try:
tags = db.get_tags(hash_hex)
for t in tags:
if isinstance(t, str) and t.lower().startswith("title:"):
source_title = t[6:].strip()
break
except Exception:
pass
metadata = db.get_metadata(hash_hex)
rels = (metadata or {}).get("relationships")
king_hashes: list[str] = []
# Forward relationships
if isinstance(rels, dict):
for rel_type, hashes in rels.items():
if not isinstance(hashes, list):
continue
for related_hash in hashes:
related_hash = normalize_hash(str(related_hash))
if not related_hash or related_hash == hash_hex:
continue
entry_type = (
"king" if str(rel_type).lower() == "alt" else
str(rel_type)
)
if entry_type == "king":
king_hashes.append(related_hash)
related_title = related_hash[:16] + "..."
try:
rel_tags = db.get_tags(related_hash)
for t in rel_tags:
if isinstance(
t,
str) and t.lower().startswith("title:"):
related_title = t[6:].strip()
break
except Exception:
pass pass
_add_relationship( _add_relationship(
@@ -270,10 +125,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
except Exception as e: except Exception as e:
log(f"Error checking store relationships: {e}", file=sys.stderr) log(f"Error checking store relationships: {e}", file=sys.stderr)
# If we found local relationships, we can stop or merge with Hydrus? # Fetch Hydrus relationships if we have a hash.
# For now, if we found local ones, let's show them.
# But if the file is also in Hydrus, we might want those too.
# Let's try Hydrus if we have a hash.
hash_hex = ( hash_hex = (
normalize_hash(override_hash) normalize_hash(override_hash)
@@ -281,7 +133,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
result)) result))
) )
if hash_hex and not local_db_checked: if hash_hex:
try: try:
client = None client = None
store_label = "hydrus" store_label = "hydrus"

178
cmdlet/search_file.py
View File

@@ -1,4 +1,4 @@
"""search-file cmdlet: Search for files in storage backends (Folder, Hydrus).""" """search-file cmdlet: Search for files in storage backends (Hydrus)."""
from __future__ import annotations from __future__ import annotations
@@ -11,12 +11,12 @@ import sys
from SYS.logger import log, debug from SYS.logger import log, debug
from ProviderCore.registry import get_search_provider, list_search_providers from ProviderCore.registry import get_search_provider, list_search_providers
from SYS.config import get_local_storage_path
from SYS.rich_display import ( from SYS.rich_display import (
show_provider_config_panel, show_provider_config_panel,
show_store_config_panel, show_store_config_panel,
show_available_providers_panel, show_available_providers_panel,
) )
from SYS.database import insert_worker, update_worker, append_worker_stdout
from ._shared import ( from ._shared import (
Cmdlet, Cmdlet,
@@ -32,17 +32,52 @@ from SYS import pipeline as ctx
STORAGE_ORIGINS = {"local", STORAGE_ORIGINS = {"local",
"hydrus", "hydrus",
"folder",
"zerotier"} "zerotier"}
class _WorkerLogger:
def __init__(self, worker_id: str) -> None:
self.worker_id = worker_id
def __enter__(self) -> "_WorkerLogger":
return self
def __exit__(self, exc_type, exc, tb) -> None: # type: ignore[override]
return None
def insert_worker(
self,
worker_id: str,
worker_type: str,
title: str = "",
description: str = "",
**kwargs: Any,
) -> None:
try:
insert_worker(worker_id, worker_type, title=title, description=description)
except Exception:
pass
def update_worker_status(self, worker_id: str, status: str) -> None:
try:
update_worker(worker_id, status=status)
except Exception:
pass
def append_worker_stdout(self, worker_id: str, content: str) -> None:
try:
append_worker_stdout(worker_id, content)
except Exception:
pass
class search_file(Cmdlet): class search_file(Cmdlet):
"""Class-based search-file cmdlet for searching storage backends.""" """Class-based search-file cmdlet for searching storage backends."""
def __init__(self) -> None: def __init__(self) -> None:
super().__init__( super().__init__(
name="search-file", name="search-file",
summary="Search storage backends (Folder, Hydrus) or external providers (via -provider).", summary="Search storage backends (Hydrus) or external providers (via -provider).",
usage="search-file [-query <query>] [-store BACKEND] [-limit N] [-provider NAME]", usage="search-file [-query <query>] [-store BACKEND] [-limit N] [-provider NAME]",
arg=[ arg=[
CmdletArg( CmdletArg(
@@ -65,7 +100,7 @@ class search_file(Cmdlet):
), ),
], ],
detail=[ detail=[
"Search across storage backends: Folder stores and Hydrus instances", "Search across storage backends: Hydrus instances",
"Use -store to search a specific backend by name", "Use -store to search a specific backend by name",
"URL search: url:* (any URL) or url:<value> (URL substring)", "URL search: url:* (any URL) or url:<value> (URL substring)",
"Extension search: ext:<value> (e.g., ext:png)", "Extension search: ext:<value> (e.g., ext:png)",
@@ -74,12 +109,12 @@ class search_file(Cmdlet):
"Examples:", "Examples:",
"search-file -query foo # Search all storage backends", "search-file -query foo # Search all storage backends",
"search-file -store home -query '*' # Search 'home' Hydrus instance", "search-file -store home -query '*' # Search 'home' Hydrus instance",
"search-file -store test -query 'video' # Search 'test' folder store", "search-file -store home -query 'video' # Search 'home' Hydrus instance",
"search-file -query 'hash:deadbeef...' # Search by SHA256 hash", "search-file -query 'hash:deadbeef...' # Search by SHA256 hash",
"search-file -query 'url:*' # Files that have any URL", "search-file -query 'url:*' # Files that have any URL",
"search-file -query 'url:youtube.com' # Files whose URL contains substring", "search-file -query 'url:youtube.com' # Files whose URL contains substring",
"search-file -query 'ext:png' # Files whose metadata ext is png", "search-file -query 'ext:png' # Files whose metadata ext is png",
"search-file -query 'system:filetype = png' # Hydrus: native; Folder: maps to metadata.ext", "search-file -query 'system:filetype = png' # Hydrus: native",
"", "",
"Provider search (-provider):", "Provider search (-provider):",
"search-file -provider youtube 'tutorial' # Search YouTube provider", "search-file -provider youtube 'tutorial' # Search YouTube provider",
@@ -210,49 +245,15 @@ class search_file(Cmdlet):
return 1 return 1
worker_id = str(uuid.uuid4()) worker_id = str(uuid.uuid4())
library_root = get_local_storage_path(config or {}) if get_local_storage_path else None try:
insert_worker(
if not library_root: worker_id,
try: "search-file",
from Store.registry import get_backend_instance title=f"Search: {query}",
# Try the first configured folder backend without instantiating all backends description=f"Provider: {provider_name}, Query: {query}",
store_cfg = (config or {}).get("store") or {} )
folder_cfg = None except Exception:
for raw_store_type, instances in store_cfg.items(): pass
if _normalize_store_type(str(raw_store_type)) == "folder":
folder_cfg = instances
break
if isinstance(folder_cfg, dict):
for instance_name, instance_config in folder_cfg.items():
try:
backend = get_backend_instance(config, instance_name, suppress_debug=True)
if backend and type(backend).__name__ == "Folder":
library_root = expand_path(getattr(backend, "_location", None))
if library_root:
break
except Exception:
pass
except Exception:
pass
db = None
# Disable Folder DB usage for "external" searches when not using a folder store
# db = None
if library_root and False: # Disabled to prevent 'database is locked' errors during external searches
try:
from API.folder import API_folder_store
db = API_folder_store(library_root)
db.__enter__()
db.insert_worker(
worker_id,
"search-file",
title=f"Search: {query}",
description=f"Provider: {provider_name}, Query: {query}",
pipe=ctx.get_current_command_text(),
)
except Exception:
db = None
try: try:
results_list: List[Dict[str, Any]] = [] results_list: List[Dict[str, Any]] = []
@@ -381,9 +382,11 @@ class search_file(Cmdlet):
if not results: if not results:
log(f"No results found for query: {query}", file=sys.stderr) log(f"No results found for query: {query}", file=sys.stderr)
if db is not None: try:
db.append_worker_stdout(worker_id, json.dumps([], indent=2)) append_worker_stdout(worker_id, json.dumps([], indent=2))
db.update_worker_status(worker_id, "completed") update_worker(worker_id, status="completed")
except Exception:
pass
return 0 return 0
for search_result in results: for search_result in results:
@@ -415,9 +418,11 @@ class search_file(Cmdlet):
ctx.set_current_stage_table(table) ctx.set_current_stage_table(table)
if db is not None: try:
db.append_worker_stdout(worker_id, json.dumps(results_list, indent=2)) append_worker_stdout(worker_id, json.dumps(results_list, indent=2))
db.update_worker_status(worker_id, "completed") update_worker(worker_id, status="completed")
except Exception:
pass
return 0 return 0
@@ -426,18 +431,11 @@ class search_file(Cmdlet):
import traceback import traceback
debug(traceback.format_exc()) debug(traceback.format_exc())
if db is not None: try:
try: update_worker(worker_id, status="error")
db.update_worker_status(worker_id, "error") except Exception:
except Exception: pass
pass
return 1 return 1
finally:
if db is not None:
try:
db.__exit__(None, None, None)
except Exception:
pass
# --- Execution ------------------------------------------------------ # --- Execution ------------------------------------------------------
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
@@ -591,37 +589,12 @@ class search_file(Cmdlet):
log("Provide a search query", file=sys.stderr) log("Provide a search query", file=sys.stderr)
return 1 return 1
from API.folder import API_folder_store
worker_id = str(uuid.uuid4()) worker_id = str(uuid.uuid4())
from Store import Store from Store import Store
storage_registry = Store(config=config or {}) storage_registry = Store(config=config or {})
library_root = get_local_storage_path(config or {})
if not library_root:
# Fallback for search-file: if no global folder path is found,
# try to use the specific backend mentioned in -store or the first available folder backend.
if storage_backend:
try:
backend = storage_registry[storage_backend]
if backend and type(backend).__name__ == "Folder":
library_root = expand_path(getattr(backend, "_location", None))
except Exception:
pass
else:
# Try all backends until we find a Folder one
for name in storage_registry.list_backends():
try:
backend = storage_registry[name]
if type(backend).__name__ == "Folder":
library_root = expand_path(getattr(backend, "_location", None))
if library_root:
break
except Exception:
continue
if not library_root: if not storage_registry.list_backends():
# Internal refreshes should not trigger config panels or stop progress. # Internal refreshes should not trigger config panels or stop progress.
if "-internal-refresh" in args_list: if "-internal-refresh" in args_list:
return 1 return 1
@@ -635,11 +608,11 @@ class search_file(Cmdlet):
progress.stop() progress.stop()
except Exception: except Exception:
pass pass
show_store_config_panel(["Folder Store"]) show_store_config_panel(["Hydrus Network"])
return 1 return 1
# Use context manager to ensure database is always closed # Use a lightweight worker logger to track search results in the central DB
with API_folder_store(library_root) as db: with _WorkerLogger(worker_id) as db:
try: try:
if "-internal-refresh" not in args_list: if "-internal-refresh" not in args_list:
db.insert_worker( db.insert_worker(
@@ -713,18 +686,7 @@ class search_file(Cmdlet):
# Resolve a path/URL string if possible # Resolve a path/URL string if possible
path_str: Optional[str] = None path_str: Optional[str] = None
# IMPORTANT: avoid calling get_file() for remote backends. # Avoid calling get_file() for remote backends during search/refresh.
# For Hydrus, get_file() returns a browser URL (and may include access keys),
# which should not be pulled during search/refresh.
try:
if type(resolved_backend).__name__ == "Folder":
maybe_path = resolved_backend.get_file(h)
if isinstance(maybe_path, Path):
path_str = str(maybe_path)
elif isinstance(maybe_path, str) and maybe_path:
path_str = maybe_path
except Exception:
path_str = None
meta_obj: Dict[str, meta_obj: Dict[str,
Any] = {} Any] = {}

259
cmdnat/pipe.py
View File

@@ -4,6 +4,7 @@ import sys
import json import json
import socket import socket
import re import re
from datetime import datetime
from urllib.parse import urlparse, parse_qs from urllib.parse import urlparse, parse_qs
from pathlib import Path from pathlib import Path
from cmdlet._shared import Cmdlet, CmdletArg, parse_cmdlet_args, resolve_tidal_manifest_path from cmdlet._shared import Cmdlet, CmdletArg, parse_cmdlet_args, resolve_tidal_manifest_path
@@ -13,8 +14,7 @@ from MPV.mpv_ipc import MPV
from SYS import pipeline as ctx from SYS import pipeline as ctx
from SYS.models import PipeObject from SYS.models import PipeObject
from API.folder import LocalLibrarySearchOptimizer from SYS.config import get_hydrus_access_key, get_hydrus_url
from SYS.config import get_local_storage_path, get_hydrus_access_key, get_hydrus_url
_ALLDEBRID_UNLOCK_CACHE: Dict[str, _ALLDEBRID_UNLOCK_CACHE: Dict[str,
str] = {} str] = {}
@@ -27,6 +27,94 @@ def _repo_root() -> Path:
return Path(os.getcwd()) return Path(os.getcwd())
def _playlist_store_path() -> Path:
return _repo_root() / "mpv_playlists.json"
def _load_playlist_store(path: Path) -> Dict[str, Any]:
if not path.exists():
return {"next_id": 1, "playlists": []}
try:
data = json.loads(path.read_text(encoding="utf-8"))
if not isinstance(data, dict):
return {"next_id": 1, "playlists": []}
data.setdefault("next_id", 1)
data.setdefault("playlists", [])
if not isinstance(data["playlists"], list):
data["playlists"] = []
return data
except Exception:
return {"next_id": 1, "playlists": []}
def _save_playlist_store(path: Path, data: Dict[str, Any]) -> bool:
try:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, indent=2), encoding="utf-8")
return True
except Exception:
return False
def _save_playlist(name: str, items: List[Any]) -> bool:
path = _playlist_store_path()
data = _load_playlist_store(path)
playlists = data.get("playlists", [])
now = datetime.utcnow().isoformat(timespec="seconds") + "Z"
for pl in playlists:
if str(pl.get("name")).strip().lower() == str(name).strip().lower():
pl["items"] = list(items)
pl["updated_at"] = now
return _save_playlist_store(path, data)
new_id = int(data.get("next_id") or 1)
data["next_id"] = new_id + 1
playlists.append({
"id": new_id,
"name": name,
"items": list(items),
"updated_at": now,
})
data["playlists"] = playlists
return _save_playlist_store(path, data)
def _get_playlist_by_id(playlist_id: int) -> Optional[tuple[str, List[Any]]]:
data = _load_playlist_store(_playlist_store_path())
for pl in data.get("playlists", []):
try:
if int(pl.get("id")) == int(playlist_id):
return str(pl.get("name") or ""), list(pl.get("items") or [])
except Exception:
continue
return None
def _delete_playlist(playlist_id: int) -> bool:
path = _playlist_store_path()
data = _load_playlist_store(path)
playlists = data.get("playlists", [])
kept = []
removed = False
for pl in playlists:
try:
if int(pl.get("id")) == int(playlist_id):
removed = True
continue
except Exception:
pass
kept.append(pl)
data["playlists"] = kept
return _save_playlist_store(path, data) if removed else False
def _get_playlists() -> List[Dict[str, Any]]:
data = _load_playlist_store(_playlist_store_path())
playlists = data.get("playlists", [])
return [dict(pl) for pl in playlists if isinstance(pl, dict)]
def _repo_log_dir() -> Path: def _repo_log_dir() -> Path:
d = _repo_root() / "Log" d = _repo_root() / "Log"
try: try:
@@ -828,23 +916,8 @@ def _get_playable_path(
backend_class = type(backend).__name__ backend_class = type(backend).__name__
backend_target_resolved = True backend_target_resolved = True
# Folder stores: resolve to an on-disk file path.
if (hasattr(backend, "get_file") and callable(getattr(backend, "get_file"))
and backend_class == "Folder"):
try:
resolved = backend.get_file(file_hash)
if isinstance(resolved, Path):
path = str(resolved)
elif resolved is not None:
path = str(resolved)
except Exception as e:
debug(
f"Error resolving file path from store '{store}': {e}",
file=sys.stderr
)
# HydrusNetwork: build a playable API file URL without browser side-effects. # HydrusNetwork: build a playable API file URL without browser side-effects.
elif backend_class == "HydrusNetwork": if backend_class == "HydrusNetwork":
try: try:
client = getattr(backend, "_client", None) client = getattr(backend, "_client", None)
base_url = getattr(client, "url", None) base_url = getattr(client, "url", None)
@@ -1367,58 +1440,38 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
# If we save 'memory://...', it will work when loaded back. # If we save 'memory://...', it will work when loaded back.
clean_items.append(item) clean_items.append(item)
# Use config from context or load it if _save_playlist(playlist_name, clean_items):
config_data = config if config else {} debug(f"Playlist saved as '{playlist_name}'")
return 0
storage_path = get_local_storage_path(config_data) or _default_state_dir() debug(f"Failed to save playlist '{playlist_name}'")
try: return 1
Path(storage_path).mkdir(parents=True, exist_ok=True)
except Exception:
pass
with LocalLibrarySearchOptimizer(storage_path) as db:
if db.save_playlist(playlist_name, clean_items):
debug(f"Playlist saved as '{playlist_name}'")
return 0
else:
debug(f"Failed to save playlist '{playlist_name}'")
return 1
# Handle Load Playlist # Handle Load Playlist
current_playlist_name = None current_playlist_name = None
if load_mode: if load_mode:
# Use config from context or load it if index_arg:
config_data = config if config else {} try:
pl_id = int(index_arg)
storage_path = get_local_storage_path(config_data) # Handle Delete Playlist (if -clear is also passed)
if not storage_path: if clear_mode:
debug("Local storage path not configured.") if _delete_playlist(pl_id):
return 1 debug(f"Playlist ID {pl_id} deleted.")
# Clear index_arg so we fall through to list mode and show updated list
with LocalLibrarySearchOptimizer(storage_path) as db: index_arg = None
if index_arg: # Don't return, let it list the remaining playlists
try:
pl_id = int(index_arg)
# Handle Delete Playlist (if -clear is also passed)
if clear_mode:
if db.delete_playlist(pl_id):
debug(f"Playlist ID {pl_id} deleted.")
# Clear index_arg so we fall through to list mode and show updated list
index_arg = None
# Don't return, let it list the remaining playlists
else:
debug(f"Failed to delete playlist ID {pl_id}.")
return 1
else: else:
# Handle Load Playlist debug(f"Failed to delete playlist ID {pl_id}.")
result = db.get_playlist_by_id(pl_id) return 1
if result is None: else:
debug(f"Playlist ID {pl_id} not found.") # Handle Load Playlist
return 1 result = _get_playlist_by_id(pl_id)
if result is None:
debug(f"Playlist ID {pl_id} not found.")
return 1
name, items = result name, items = result
current_playlist_name = name current_playlist_name = name
# Queue items (replacing current playlist) # Queue items (replacing current playlist)
if items: if items:
@@ -1446,42 +1499,42 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
debug(f"Invalid playlist ID: {index_arg}") debug(f"Invalid playlist ID: {index_arg}")
return 1 return 1
# If we deleted or didn't have an index, list playlists # If we deleted or didn't have an index, list playlists
if not index_arg: if not index_arg:
playlists = db.get_playlists() playlists = _get_playlists()
if not playlists: if not playlists:
debug("No saved playlists found.") debug("No saved playlists found.")
return 0
table = Table("Saved Playlists")
for i, pl in enumerate(playlists):
item_count = len(pl.get("items", []))
row = table.add_row()
# row.add_column("ID", str(pl['id'])) # Hidden as per user request
row.add_column("Name", pl["name"])
row.add_column("Items", str(item_count))
row.add_column("Updated", pl["updated_at"])
# Set the playlist items as the result object for this row
# When user selects @N, they get the list of items
# We also set the source command to .pipe -load <ID> so it loads it
table.set_row_selection_args(i, ["-load", str(pl["id"])])
table.set_source_command(".mpv")
# Register results
ctx.set_last_result_table_overlay(
table,
[p["items"] for p in playlists]
)
ctx.set_current_stage_table(table)
# Do not print directly here.
# Both CmdletExecutor and PipelineExecutor render the current-stage/overlay table,
# so printing here would duplicate output.
return 0 return 0
table = Table("Saved Playlists")
for i, pl in enumerate(playlists):
item_count = len(pl.get("items", []))
row = table.add_row()
# row.add_column("ID", str(pl['id'])) # Hidden as per user request
row.add_column("Name", pl["name"])
row.add_column("Items", str(item_count))
row.add_column("Updated", pl.get("updated_at") or "")
# Set the playlist items as the result object for this row
# When user selects @N, they get the list of items
# We also set the source command to .pipe -load <ID> so it loads it
table.set_row_selection_args(i, ["-load", str(pl["id"])])
table.set_source_command(".mpv")
# Register results
ctx.set_last_result_table_overlay(
table,
[p["items"] for p in playlists]
)
ctx.set_current_stage_table(table)
# Do not print directly here.
# Both CmdletExecutor and PipelineExecutor render the current-stage/overlay table,
# so printing here would duplicate output.
return 0
# Everything below was originally outside a try block; keep it inside so `start_opts` is in scope. # Everything below was originally outside a try block; keep it inside so `start_opts` is in scope.
# Handle Play/Pause commands (but skip if we have index_arg to play a specific item) # Handle Play/Pause commands (but skip if we have index_arg to play a specific item)
@@ -1850,20 +1903,6 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
if len(stem) == 64 and all(c in "0123456789abcdef" if len(stem) == 64 and all(c in "0123456789abcdef"
for c in stem.lower()): for c in stem.lower()):
file_hash = stem.lower() file_hash = stem.lower()
# Find which folder store has this file
if file_storage:
for backend_name in file_storage.list_backends():
backend = file_storage[backend_name]
if type(backend).__name__ == "Folder":
# Check if this backend has the file
try:
result_path = backend.get_file(file_hash)
if isinstance(result_path,
Path) and result_path.exists():
store_name = backend_name
break
except Exception:
pass
# Fallback to inferred store if we couldn't find it # Fallback to inferred store if we couldn't find it
if not store_name: if not store_name:

22
cmdnat/status.py
View File

@@ -242,28 +242,6 @@ def _run(result: Any, args: List[str], config: Dict[str, Any]) -> int:
_add_startup_check(startup_table, "DISABLED", "Matrix", provider="matrix", detail=str(exc)) _add_startup_check(startup_table, "DISABLED", "Matrix", provider="matrix", detail=str(exc))
debug(f"Matrix instantiation failed: {exc}") debug(f"Matrix instantiation failed: {exc}")
# Folders
if _has_store_subtype(config, "folder"):
fcfg = config.get("store", {}).get("folder", {})
for iname, icfg in fcfg.items():
if not isinstance(icfg, dict): continue
nkey = str(icfg.get("NAME") or iname)
pval = str(icfg.get("PATH") or icfg.get("path") or "").strip()
debug(f"Folder store check: name={nkey}, path={pval}")
ok = bool(store_registry and store_registry.is_available(nkey))
if ok and store_registry:
backend = store_registry[nkey]
scan_ok = getattr(backend, "scan_ok", True)
sdet = getattr(backend, "scan_detail", "Up to date")
stats = getattr(backend, "scan_stats", {})
files = int(stats.get("files_total_db", 0)) if stats else None
debug(f"Folder backend '{nkey}': scan_ok={scan_ok}, scan_detail={sdet}, stats={stats}")
_add_startup_check(startup_table, "SCANNED" if scan_ok else "ERROR", nkey, store="folder", files=files, detail=f"{pval} - {sdet}")
else:
err = store_registry.get_backend_error(iname) if store_registry else None
debug(f"Folder backend '{nkey}' error: {err}")
_add_startup_check(startup_table, "ERROR", nkey, store="folder", detail=f"{pval} - {err or 'Unavailable'}")
# Cookies # Cookies
try: try:
from tool.ytdlp import YtDlpTool from tool.ytdlp import YtDlpTool

115
cmdnat/worker.py
View File

@@ -12,7 +12,7 @@ from cmdlet import register
from cmdlet._shared import Cmdlet, CmdletArg from cmdlet._shared import Cmdlet, CmdletArg
from SYS import pipeline as ctx from SYS import pipeline as ctx
from SYS.logger import log from SYS.logger import log
from SYS.config import get_local_storage_path from SYS.database import db as _db, get_worker_stdout
DEFAULT_LIMIT = 100 DEFAULT_LIMIT = 100
WORKER_STATUS_FILTERS = {"running", WORKER_STATUS_FILTERS = {"running",
@@ -74,6 +74,69 @@ CMDLET = Cmdlet(
) )
def _normalize_worker_row(row: Dict[str, Any]) -> Dict[str, Any]:
worker_id = row.get("id")
created = row.get("created_at") or ""
updated = row.get("updated_at") or ""
payload = dict(row)
payload["worker_id"] = worker_id
payload["started_at"] = created
payload["last_updated"] = updated
payload["completed_at"] = updated
payload["pipe"] = row.get("details") or row.get("title") or ""
return payload
class _WorkerDB:
def clear_finished_workers(self) -> int:
try:
cur = _db.execute("DELETE FROM workers WHERE status != 'running'")
return int(getattr(cur, "rowcount", 0) or 0)
except Exception:
return 0
def get_worker(self, worker_id: str) -> Dict[str, Any] | None:
row = _db.fetchone("SELECT * FROM workers WHERE id = ?", (worker_id,))
if not row:
return None
worker = _normalize_worker_row(dict(row))
try:
worker["stdout"] = get_worker_stdout(worker_id)
except Exception:
worker["stdout"] = ""
return worker
def get_worker_events(self, worker_id: str) -> List[Dict[str, Any]]:
try:
rows = _db.fetchall(
"SELECT content, channel, timestamp FROM worker_stdout WHERE worker_id = ? ORDER BY timestamp ASC",
(worker_id,),
)
except Exception:
rows = []
events: List[Dict[str, Any]] = []
for row in rows:
try:
events.append({
"message": row.get("content"),
"channel": row.get("channel") or "stdout",
"created_at": row.get("timestamp"),
})
except Exception:
continue
return events
def get_all_workers(self, limit: int = 100) -> List[Dict[str, Any]]:
try:
rows = _db.fetchall(
"SELECT * FROM workers ORDER BY created_at DESC LIMIT ?",
(int(limit or 100),),
)
except Exception:
rows = []
return [_normalize_worker_row(dict(row)) for row in rows]
def _has_help_flag(args_list: Sequence[str]) -> bool: def _has_help_flag(args_list: Sequence[str]) -> bool:
return any(str(arg).lower() in HELP_FLAGS for arg in args_list) return any(str(arg).lower() in HELP_FLAGS for arg in args_list)
@@ -101,39 +164,33 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
options = _parse_worker_args(args_list) options = _parse_worker_args(args_list)
library_root = get_local_storage_path(config or {})
if not library_root:
log("No library root configured. Use the .config command to set up storage.", file=sys.stderr)
return 1
try: try:
from API.folder import API_folder_store db = _WorkerDB()
with API_folder_store(library_root) as db: if options.clear:
if options.clear: count = db.clear_finished_workers()
count = db.clear_finished_workers() log(f"Cleared {count} finished workers.")
log(f"Cleared {count} finished workers.") return 0
if options.worker_id:
worker = db.get_worker(options.worker_id)
if worker:
events: List[Dict[str, Any]] = []
try:
wid = worker.get("worker_id")
if wid:
events = db.get_worker_events(wid)
except Exception:
pass
_emit_worker_detail(worker, events)
return 0 return 0
log(f"Worker not found: {options.worker_id}", file=sys.stderr)
return 1
if options.worker_id: if selection_requested:
worker = db.get_worker(options.worker_id) return _render_worker_selection(db, result)
if worker:
events: List[Dict[str, Any]] = []
try:
wid = worker.get("worker_id")
if wid and hasattr(db, "get_worker_events"):
events = db.get_worker_events(wid)
except Exception:
pass
_emit_worker_detail(worker, events)
return 0
log(f"Worker not found: {options.worker_id}", file=sys.stderr)
return 1
if selection_requested: return _render_worker_list(db, options.status, options.limit)
return _render_worker_selection(db, result)
return _render_worker_list(db, options.status, options.limit)
except Exception as exc: except Exception as exc:
log(f"Workers query failed: {exc}", file=sys.stderr) log(f"Workers query failed: {exc}", file=sys.stderr)
import traceback import traceback

40
scripts/bootstrap.py
View File

@@ -738,7 +738,40 @@ def main() -> int:
return 0 return 0
def _update_config_value(root: Path, key: str, value: str) -> bool: def _update_config_value(root: Path, key: str, value: str) -> bool:
db_path = root / "medios.db"
config_path = root / "config.conf" config_path = root / "config.conf"
# Try database first
if db_path.exists():
try:
import sqlite3
import json
with sqlite3.connect(str(db_path)) as conn:
# We want to set store.hydrusnetwork.hydrus.<key>
cur = conn.cursor()
# Check if hydrusnetwork store exists
cur.execute("SELECT 1 FROM config WHERE category='store' AND subtype='hydrusnetwork'")
if cur.fetchone():
cur.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
('store', 'hydrusnetwork', 'hydrus', key, value)
)
else:
# Create the section
cur.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
('store', 'hydrusnetwork', 'hydrus', 'name', 'hydrus')
)
cur.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
('store', 'hydrusnetwork', 'hydrus', key, value)
)
conn.commit()
return True
except Exception as e:
print(f"Error updating database config: {e}")
# Fallback to config.conf
if not config_path.exists(): if not config_path.exists():
fallback = root / "config.conf.remove" fallback = root / "config.conf.remove"
if fallback.exists(): if fallback.exists():
@@ -759,7 +792,7 @@ def main() -> int:
config_path.write_text(new_content, encoding="utf-8") config_path.write_text(new_content, encoding="utf-8")
return True return True
except Exception as e: except Exception as e:
print(f"Error updating config: {e}") print(f"Error updating legacy config: {e}")
return False return False
def _interactive_menu() -> str | int: def _interactive_menu() -> str | int:
@@ -1512,7 +1545,10 @@ if (Test-Path (Join-Path $repo 'CLI.py')) {
"if not defined MM_NO_UPDATE (\n" "if not defined MM_NO_UPDATE (\n"
" if exist \"!REPO!\\.git\" (\n" " if exist \"!REPO!\\.git\" (\n"
" set \"AUTO_UPDATE=true\"\n" " set \"AUTO_UPDATE=true\"\n"
" if exist \"!REPO!\\config.conf\" (\n" " if exist \"!REPO!\\medios.db\" (\n"
" \"sqlite3\" \"!REPO!\\medios.db\" \"SELECT value FROM config WHERE key='auto_update' AND category='global'\" | findstr /i /r \"false no off 0\" >nul 2>&1\n"
" if !errorlevel! == 0 set \"AUTO_UPDATE=false\"\n"
" ) else if exist \"!REPO!\\config.conf\" (\n"
" findstr /i /r \"auto_update.*=.*false auto_update.*=.*no auto_update.*=.*off auto_update.*=.*0\" \"!REPO!\\config.conf\" >nul 2>&1\n" " findstr /i /r \"auto_update.*=.*false auto_update.*=.*no auto_update.*=.*off auto_update.*=.*0\" \"!REPO!\\config.conf\" >nul 2>&1\n"
" if !errorlevel! == 0 set \"AUTO_UPDATE=false\"\n" " if !errorlevel! == 0 set \"AUTO_UPDATE=false\"\n"
" )\n" " )\n"

51
scripts/hydrusnetwork.py
View File

@@ -144,23 +144,63 @@ def run_git_pull(git: str, dest: Path) -> None:
def update_medios_config(hydrus_path: Path) -> bool: def update_medios_config(hydrus_path: Path) -> bool:
"""Attempt to update config.conf in the Medios-Macina root with the hydrus path. """Attempt to update Medios-Macina root configuration with the hydrus path.
This helps link the newly installed Hydrus instance with the main project. This helps link the newly installed Hydrus instance with the main project.
We check for medios.db first, then fall back to config.conf.
""" """
# Scripts is in <root>/scripts, so parent is root.
script_dir = Path(__file__).resolve().parent script_dir = Path(__file__).resolve().parent
root = script_dir.parent root = script_dir.parent
db_path = root / "medios.db"
config_path = root / "config.conf" config_path = root / "config.conf"
hydrus_abs_path = str(hydrus_path.resolve())
# Try database first
if db_path.exists():
try:
import sqlite3
import json
with sqlite3.connect(str(db_path)) as conn:
conn.row_factory = sqlite3.Row
cur = conn.cursor()
# We want to set store.hydrusnetwork.hydrus.gitclone
# First check if hydrusnetwork store exists
cur.execute("SELECT 1 FROM config WHERE category='store' AND subtype='hydrusnetwork'")
if cur.fetchone():
# Update or insert gitclone for the hydrus subtype
# Note: we assume the name is 'hydrus' or 'hn-local' or something.
# Usually it's 'hydrus' if newly created.
cur.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
('store', 'hydrusnetwork', 'hydrus', 'gitclone', hydrus_abs_path)
)
else:
# Create the section
cur.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
('store', 'hydrusnetwork', 'hydrus', 'name', 'hydrus')
)
cur.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
('store', 'hydrusnetwork', 'hydrus', 'gitclone', hydrus_abs_path)
)
conn.commit()
logging.info("✅ Linked Hydrus installation in medios.db (gitclone=\"%s\")", hydrus_abs_path)
return True
except Exception as e:
logging.error("Failed to update medios.db: %s", e)
# Fallback to config.conf
if not config_path.exists(): if not config_path.exists():
logging.debug("MM config.conf not found at %s; skipping auto-link.", config_path) logging.debug("MM config.conf not found at %s; skipping legacy auto-link.", config_path)
return False return False
try: try:
content = config_path.read_text(encoding="utf-8") content = config_path.read_text(encoding="utf-8")
key = "gitclone" key = "gitclone"
value = str(hydrus_path.resolve()) value = hydrus_abs_path
# Pattern to replace existing gitclone in the hydrusnetwork section # Pattern to replace existing gitclone in the hydrusnetwork section
pattern = rf'^(\s*{re.escape(key)}\s*=\s*)(.*)$' pattern = rf'^(\s*{re.escape(key)}\s*=\s*)(.*)$'
@@ -178,6 +218,9 @@ def update_medios_config(hydrus_path: Path) -> bool:
logging.info("✅ Linked Hydrus installation in Medios-Macina config (gitclone=\"%s\")", value) logging.info("✅ Linked Hydrus installation in Medios-Macina config (gitclone=\"%s\")", value)
return True return True
except Exception as e: except Exception as e:
logging.error("Failed to update config.conf: %s", e)
return False
return False
logging.debug("Failed to update MM config: %s", e) logging.debug("Failed to update MM config: %s", e)
return False return False