update and cleanup repo

This commit is contained in:
2026-05-26 15:32:01 -07:00
parent 5041d9fbb9
commit 0db899d0c3
72 changed files with 788 additions and 1884 deletions
+3 -3
View File
@@ -410,10 +410,10 @@ def get_cmdlet_arg_choices(
matrix_conf = {}
try:
providers = config.get("provider") or {}
matrix_conf = providers.get("matrix") or {}
plugins = config.get("plugin") or {}
matrix_conf = plugins.get("matrix") or {}
except Exception as exc:
logger.exception("Failed to read matrix provider config: %s", exc)
logger.exception("Failed to read matrix plugin config: %s", exc)
matrix_conf = {}
raw = None
+51
View File
@@ -77,6 +77,50 @@ def QueryArg(
)
def collect_registered_cmdlet_names(
cmdlet_obj: Any,
*,
fallback_name: Optional[str] = None,
) -> List[str]:
"""Return normalized registration keys for a cmdlet object.
Prefers the cmdlet object's own `_collect_names()` implementation when
available, then normalizes names to the registry key form used by callers.
"""
raw_names: List[Any] = []
collector = getattr(cmdlet_obj, "_collect_names", None)
if callable(collector):
try:
raw_names.extend(list(collector() or []))
except Exception:
pass
if fallback_name:
raw_names.append(fallback_name)
if not raw_names:
raw_name = getattr(cmdlet_obj, "name", None)
if raw_name:
raw_names.append(raw_name)
for alias_attr in ("alias", "aliases"):
alias_values = getattr(cmdlet_obj, alias_attr, None)
if not alias_values:
continue
raw_names.extend(list(alias_values))
seen: Set[str] = set()
normalized_names: List[str] = []
for raw_name in raw_names:
key = str(raw_name or "").replace("_", "-").lower().strip()
if not key or key in seen:
continue
seen.add(key)
normalized_names.append(key)
return normalized_names
class SharedArgs:
"""Registry of shared CmdletArg definitions used across multiple cmdlet."""
@@ -100,6 +144,13 @@ class SharedArgs:
description="selects plugin",
)
INSTANCE = CmdletArg(
name="instance",
type="string",
description="Selects a plugin instance",
query_key="instance",
)
@staticmethod
def get_store_choices(config: Optional[Dict[str, Any]] = None, force: bool = False) -> List[str]:
if not force and hasattr(SharedArgs, "_cached_available_stores"):
+122 -228
View File
@@ -95,8 +95,6 @@ def clear_config_cache() -> None:
def _log_config_load_summary(config: Dict[str, Any]) -> None:
try:
plugin_block = config.get("plugin")
if not isinstance(plugin_block, dict):
plugin_block = config.get("provider")
if isinstance(plugin_block, dict):
# Count distinct plugin names; note multi-instance plugins appear once per name
plugin_names = list(plugin_block.keys())
@@ -265,7 +263,9 @@ def set_nested_config_value(
def get_hydrus_instance(
config: Dict[str, Any], instance_name: str = "home"
) -> Optional[Dict[str, Any]]:
"""Get a specific Hydrus instance config by name from plugin/provider config."""
"""Get a specific Hydrus instance config by name from plugin config."""
_canonicalize_plugin_config(config)
def _lookup_in(source: Dict[str, Any]) -> Optional[Dict[str, Any]]:
if not isinstance(source, dict) or not source:
return None
@@ -286,16 +286,13 @@ def get_hydrus_instance(
candidate = source.get(first_key) if first_key else None
return candidate if isinstance(candidate, dict) else None
# New format: config["plugin"]["hydrusnetwork"] or config["provider"]["hydrusnetwork"]
# (both point to the same dict after normalization)
for section in ("plugin", "provider"):
section_cfg = config.get(section)
if isinstance(section_cfg, dict):
hydrus_cfg = section_cfg.get("hydrusnetwork")
if isinstance(hydrus_cfg, dict):
result = _lookup_in(hydrus_cfg)
if result is not None:
return result
plugin_cfg = config.get("plugin")
if isinstance(plugin_cfg, dict):
hydrus_cfg = plugin_cfg.get("hydrusnetwork")
if isinstance(hydrus_cfg, dict):
result = _lookup_in(hydrus_cfg)
if result is not None:
return result
return None
@@ -339,17 +336,17 @@ def get_hydrus_url(config: Dict[str, Any], instance_name: str = "home") -> Optio
return str(url).strip() if url else None
def get_provider_block(config: Dict[str, Any], name: str) -> Dict[str, Any]:
_normalize_plugin_config_aliases(config)
provider_cfg = config.get("provider")
if not isinstance(provider_cfg, dict):
def get_plugin_block(config: Dict[str, Any], name: str) -> Dict[str, Any]:
_canonicalize_plugin_config(config)
plugin_cfg = config.get("plugin")
if not isinstance(plugin_cfg, dict):
return {}
normalized = _normalize_provider_name(name)
if normalized:
block = provider_cfg.get(normalized)
block = plugin_cfg.get(normalized)
if isinstance(block, dict):
return block
for key, block in provider_cfg.items():
for key, block in plugin_cfg.items():
if not isinstance(block, dict):
continue
if _normalize_provider_name(key) == normalized:
@@ -358,13 +355,13 @@ def get_provider_block(config: Dict[str, Any], name: str) -> Dict[str, Any]:
def get_soulseek_username(config: Dict[str, Any]) -> Optional[str]:
block = get_provider_block(config, "soulseek")
block = get_plugin_block(config, "soulseek")
val = block.get("username") or block.get("USERNAME")
return str(val).strip() if val else None
def get_soulseek_password(config: Dict[str, Any]) -> Optional[str]:
block = get_provider_block(config, "soulseek")
block = get_plugin_block(config, "soulseek")
val = block.get("password") or block.get("PASSWORD")
return str(val).strip() if val else None
@@ -415,33 +412,33 @@ def resolve_output_dir(config: Dict[str, Any]) -> Path:
def get_local_storage_path(config: Dict[str, Any]) -> Optional[Path]:
"""Get local storage path from config.
"""Return the configured default local plugin destination path.
Supports multiple formats:
- Old: config["storage"]["local"]["path"]
- Old: config["Local"]["path"]
Args:
config: Configuration dict
Returns:
Path object if found, None otherwise
This helper is intentionally narrow: it reports a real local library/export
root only when the canonical `plugin.local` config defines one. Callers that
want a staging/output directory should use `resolve_output_dir(...)` instead.
"""
# Fall back to storage.local.path format
storage = config.get("storage", {})
if isinstance(storage, dict):
local_config = storage.get("local", {})
if isinstance(local_config, dict):
path_str = local_config.get("path")
if path_str:
return expand_path(path_str)
local_block = get_plugin_block(config, "local")
if not isinstance(local_block, dict) or not local_block:
return None
# Fall back to old Local format
local_config = config.get("Local", {})
if isinstance(local_config, dict):
path_str = local_config.get("path")
if path_str:
return expand_path(path_str)
if _is_multi_instance_plugin_config(local_block):
if "default" in local_block and isinstance(local_block.get("default"), dict):
local_config = local_block.get("default")
else:
local_config = next(
(value for value in local_block.values() if isinstance(value, dict)),
None,
)
else:
local_config = local_block
if not isinstance(local_config, dict):
return None
path_str = local_config.get("path") or local_config.get("PATH")
if path_str:
return expand_path(path_str)
return None
@@ -449,7 +446,7 @@ def get_local_storage_path(config: Dict[str, Any]) -> Optional[Path]:
def get_debrid_api_key(config: Dict[str, Any], service: str = "All-debrid") -> Optional[str]:
"""Get Debrid API key from config.
Checks the plugin/provider block first (canonical format).
Checks the plugin block first (canonical format).
Args:
config: Configuration dict
@@ -458,37 +455,23 @@ def get_debrid_api_key(config: Dict[str, Any], service: str = "All-debrid") -> O
Returns:
API key string if found, None otherwise
"""
# 1) Canonical plugin/provider block: config["plugin"]["alldebrid"]["api_key"]
provider_block = config.get("provider") or config.get("plugin")
if isinstance(provider_block, dict):
alldebrid_entry = provider_block.get("alldebrid")
_canonicalize_plugin_config(config)
# 1) Canonical plugin block: config["plugin"]["alldebrid"]["api_key"]
plugin_block = config.get("plugin")
if isinstance(plugin_block, dict):
alldebrid_entry = plugin_block.get("alldebrid")
if isinstance(alldebrid_entry, dict):
for k in ("api_key", "API_KEY", "apikey", "APIKEY"):
val = alldebrid_entry.get(k)
if isinstance(val, str) and val.strip():
return val.strip()
# 2) Migrated legacy debrid plugin entry: config["plugin"]["debrid"]["all-debrid"]["api_key"]
if isinstance(provider_block, dict):
service_key = str(service).strip().lower()
debrid_plugin = provider_block.get("debrid")
if isinstance(debrid_plugin, dict):
entry = debrid_plugin.get(service_key)
if isinstance(entry, dict):
api_key = entry.get("api_key")
return str(api_key).strip() if api_key else None
if isinstance(entry, str):
return entry.strip() or None
return None
def get_provider_credentials(config: Dict[str, Any], provider: str) -> Optional[Dict[str, str]]:
"""Get provider credentials (email/password) from config.
Supports both formats:
- New: config["provider"][provider] = {"email": "...", "password": "..."}
- Old: config[provider.capitalize()] = {"email": "...", "password": "..."}
def get_plugin_credentials(config: Dict[str, Any], provider: str) -> Optional[Dict[str, str]]:
"""Get plugin credentials (email/password) from config.
Args:
config: Configuration dict
@@ -497,22 +480,11 @@ def get_provider_credentials(config: Dict[str, Any], provider: str) -> Optional[
Returns:
Dict with credentials if found, None otherwise
"""
# Try new format first
provider_config = config.get("provider", {})
if isinstance(provider_config, dict):
creds = provider_config.get(provider.lower(), {})
if isinstance(creds, dict) and creds:
return creds
_canonicalize_plugin_config(config)
# Fall back to old format (capitalized key)
old_key_map = {
"openlibrary": "OpenLibrary",
"archive": "Archive",
"soulseek": "Soulseek",
}
old_key = old_key_map.get(provider.lower())
if old_key:
creds = config.get(old_key, {})
plugin_config = config.get("plugin", {})
if isinstance(plugin_config, dict):
creds = plugin_config.get(provider.lower(), {})
if isinstance(creds, dict) and creds:
return creds
@@ -522,19 +494,19 @@ def get_provider_credentials(config: Dict[str, Any], provider: str) -> Optional[
def resolve_cookies_path(
config: Dict[str, Any], script_dir: Optional[Path] = None
) -> Optional[Path]:
# Only support modular config style:
# [tool=ytdlp]
# Only support plugin config style:
# [plugin=ytdlp]
# cookies="C:\\path\\cookies.txt"
values: list[Any] = []
try:
tool = config.get("tool")
if isinstance(tool, dict):
ytdlp = tool.get("ytdlp")
plugin = config.get("plugin")
if isinstance(plugin, dict):
ytdlp = plugin.get("ytdlp")
if isinstance(ytdlp, dict):
values.append(ytdlp.get("cookies"))
values.append(ytdlp.get("cookiefile"))
except Exception as exc:
logger.debug("resolve_cookies_path: failed to read tool.ytdlp cookies: %s", exc, exc_info=True)
logger.debug("resolve_cookies_path: failed to read plugin.ytdlp cookies: %s", exc, exc_info=True)
base_dir = _resolve_app_root(script_dir)
for value in values:
@@ -627,54 +599,26 @@ def resolve_plugin_asset_path(
return None
def _normalize_plugin_config_aliases(config: Dict[str, Any]) -> None:
def _canonicalize_plugin_config(config: Dict[str, Any]) -> None:
if not isinstance(config, dict):
return
config.pop("provider", None)
config.pop("store", None)
plugin_block = config.get("plugin")
provider_block = config.get("provider")
normalized_provider: Dict[str, Any] = {}
if isinstance(provider_block, dict):
for key, value in provider_block.items():
normalized_key = _normalize_provider_name(key)
if normalized_key and normalized_key not in normalized_provider:
normalized_provider[normalized_key] = value
normalized_plugin: Dict[str, Any] = {}
if isinstance(plugin_block, dict):
for key, value in plugin_block.items():
normalized_key = _normalize_provider_name(key)
if normalized_key and normalized_key not in normalized_provider:
normalized_provider[normalized_key] = value
if normalized_key:
normalized_plugin[normalized_key] = value
# Fold legacy config["store"] entries into the plugin namespace.
# store format: {type: {instance_name: {key: val}}} — multi-instance.
# After folding, remove config["store"] so it is no longer consulted.
store_block = config.pop("store", None)
if isinstance(store_block, dict):
for store_type, instances in store_block.items():
if not isinstance(instances, dict):
continue
normalized_key = _normalize_provider_name(store_type)
if not normalized_key:
continue
existing = normalized_provider.get(normalized_key)
if not isinstance(existing, dict):
existing = {}
normalized_provider[normalized_key] = existing
for instance_name, settings in instances.items():
if isinstance(settings, dict) and instance_name not in existing:
existing[instance_name] = dict(settings)
if normalized_provider:
config["provider"] = normalized_provider
config["plugin"] = normalized_provider
if normalized_plugin or isinstance(plugin_block, dict):
config["plugin"] = normalized_plugin
else:
if isinstance(provider_block, dict):
config["plugin"] = provider_block
elif isinstance(plugin_block, dict):
config["provider"] = plugin_block
config.pop("plugin", None)
def _extract_api_key(value: Any) -> Optional[str]:
if isinstance(value, dict):
@@ -698,40 +642,24 @@ def _sync_alldebrid_api_key(config: Dict[str, Any]) -> None:
if not isinstance(config, dict):
return
_normalize_plugin_config_aliases(config)
providers = config.get("provider")
if not isinstance(providers, dict):
providers = {}
config["provider"] = providers
provider_entry = providers.get("alldebrid")
provider_section: Dict[str, Any] | None = None
provider_key = None
if isinstance(provider_entry, dict):
provider_section = provider_entry
provider_key = _extract_api_key(provider_section)
elif isinstance(provider_entry, str):
provider_key = provider_entry.strip()
if provider_key:
provider_section = {"api_key": provider_key}
providers["alldebrid"] = provider_section
# If no key found in provider block, check for a migrated debrid plugin entry.
# (rows_to_config migrates store.debrid.all-debrid → plugin.debrid.all-debrid)
if not provider_key:
plugin_block = config.get("plugin") or providers
debrid_plugin = plugin_block.get("debrid") if isinstance(plugin_block, dict) else None
if isinstance(debrid_plugin, dict):
service_entry = debrid_plugin.get("all-debrid")
legacy_key = _extract_api_key(service_entry) if service_entry else None
if legacy_key:
if provider_section is None:
provider_section = {}
providers["alldebrid"] = provider_section
provider_section.setdefault("api_key", legacy_key)
_canonicalize_plugin_config(config)
plugins = config.get("plugin")
if not isinstance(plugins, dict):
plugins = {}
config["plugin"] = plugins
plugin_entry = plugins.get("alldebrid")
plugin_section: Dict[str, Any] | None = None
plugin_key = None
if isinstance(plugin_entry, dict):
plugin_section = plugin_entry
plugin_key = _extract_api_key(plugin_section)
elif isinstance(plugin_entry, str):
plugin_key = plugin_entry.strip()
if plugin_key:
plugin_section = {"api_key": plugin_key}
plugins["alldebrid"] = plugin_section
def _is_multi_instance_plugin_config(value: Any) -> bool:
"""Return True if `value` looks like a multi-instance plugin config (dict-of-dicts).
@@ -754,12 +682,9 @@ def _is_multi_instance_plugin_config(value: Any) -> bool:
def _flatten_config_entries(config: Dict[str, Any]) -> Dict[Tuple[str, str, str, str], Any]:
entries: Dict[Tuple[str, str, str, str], Any] = {}
_normalize_plugin_config_aliases(config)
_canonicalize_plugin_config(config)
for key, value in config.items():
if key == 'plugin':
# plugin == provider after normalization; skip duplicate
continue
if key == 'provider' and isinstance(value, dict):
if key == 'plugin' and isinstance(value, dict):
for subtype, plugin_cfg in value.items():
if not isinstance(plugin_cfg, dict):
continue
@@ -773,21 +698,13 @@ def _flatten_config_entries(config: Dict[str, Any]) -> Dict[Tuple[str, str, str,
else:
# Single-instance: {key: val}
for k, v in plugin_cfg.items():
entries[('provider', subtype, 'default', k)] = v
elif key in ('store', 'tool') and isinstance(value, dict):
entries[('plugin', subtype, 'default', k)] = v
elif key == 'tool' and isinstance(value, dict):
for subtype, instances in value.items():
if not isinstance(instances, dict):
continue
if key == 'store':
# Legacy store: migrate to plugin category
for name, settings in instances.items():
if not isinstance(settings, dict):
continue
for k, v in settings.items():
entries[('plugin', subtype, name, k)] = v
else: # tool
for k, v in instances.items():
entries[(key, subtype, 'default', k)] = v
for k, v in instances.items():
entries[('tool', subtype, 'default', k)] = v
elif not key.startswith('_') and value is not None:
entries[('global', 'none', 'none', key)] = value
return entries
@@ -817,14 +734,6 @@ def _config_from_flattened_entries(
config[key] = value
continue
if category == "store":
# Legacy: migrate to plugin namespace at reconstitution time
plugin_block = config.setdefault("plugin", {})
subtype_block = plugin_block.setdefault(subtype, {})
item_block = subtype_block.setdefault(item_name, {})
item_block[key] = value
continue
if category == "plugin":
plugin_block = config.setdefault("plugin", {})
subtype_block = plugin_block.setdefault(subtype, {})
@@ -835,7 +744,7 @@ def _config_from_flattened_entries(
item_block[key] = value
continue
if category in {"provider", "tool"}:
if category == "tool":
category_block = config.setdefault(category, {})
subtype_block = category_block.setdefault(subtype, {})
subtype_block[key] = value
@@ -849,7 +758,7 @@ def _config_from_flattened_entries(
if isinstance(item_block, dict):
item_block[key] = value
_normalize_plugin_config_aliases(config)
_canonicalize_plugin_config(config)
_sync_alldebrid_api_key(config)
return config
@@ -880,9 +789,9 @@ def _merge_non_conflicting_config_changes(
def _extract_expected_alldebrid_key(config: Dict[str, Any]) -> Optional[str]:
expected_key = None
try:
providers = config.get("provider", {}) if isinstance(config, dict) else {}
if isinstance(providers, dict):
entry = providers.get("alldebrid")
plugins = config.get("plugin", {}) if isinstance(config, dict) else {}
if isinstance(plugins, dict):
entry = plugins.get("alldebrid")
if entry is not None:
if isinstance(entry, dict):
for k in ("api_key", "API_KEY", "apikey", "APIKEY"):
@@ -908,18 +817,10 @@ def load_config(*, emit_summary: bool = False) -> Dict[str, Any]:
_CONFIG_SUMMARY_PENDING = False
return _CONFIG_CACHE
# One-time DB migration: move category='store' rows to category='plugin'.
# This is idempotent — a no-op if no store rows exist.
try:
from SYS.database import migrate_store_category_to_plugin
migrate_store_category_to_plugin()
except Exception:
logger.debug("Store→plugin DB migration skipped or failed", exc_info=True)
# Load strictly from database
db_config = get_config_all()
if db_config:
_normalize_plugin_config_aliases(db_config)
_canonicalize_plugin_config(db_config)
_sync_alldebrid_api_key(db_config)
_CONFIG_CACHE = db_config
_LAST_SAVED_CONFIG = deepcopy(db_config)
@@ -1007,7 +908,7 @@ def _release_save_lock(lock_dir: Path) -> None:
def save_config(config: Dict[str, Any]) -> int:
global _CONFIG_CACHE, _LAST_SAVED_CONFIG
_normalize_plugin_config_aliases(config)
_canonicalize_plugin_config(config)
_sync_alldebrid_api_key(config)
# Acquire cross-process save lock to avoid concurrent saves from different
@@ -1065,31 +966,39 @@ def save_config(config: Dict[str, Any]) -> int:
# Proceed with writing when no conflicting external changes detected
conn.execute("DELETE FROM config")
for key, value in config_to_write.items():
if key in ('store', 'provider', 'tool') and isinstance(value, dict):
if key in ('plugin', 'tool') and isinstance(value, dict):
for subtype, instances in value.items():
if not isinstance(instances, dict):
continue
if key == 'store':
for name, settings in instances.items():
if isinstance(settings, dict):
if key == 'plugin':
normalized_subtype = _normalize_provider_name(subtype)
if not normalized_subtype:
continue
if _is_multi_instance_plugin_config(instances):
for name, settings in instances.items():
if not isinstance(settings, dict):
continue
for k, v in settings.items():
val_str = json.dumps(v) if not isinstance(v, str) else v
conn.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
(key, subtype, name, k, val_str),
("plugin", normalized_subtype, name, k, val_str),
)
count += 1
else:
for k, v in instances.items():
val_str = json.dumps(v) if not isinstance(v, str) else v
conn.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
("plugin", normalized_subtype, "default", k, val_str),
)
count += 1
else:
normalized_subtype = subtype
if key == 'provider':
normalized_subtype = _normalize_provider_name(subtype)
if not normalized_subtype:
continue
for k, v in instances.items():
val_str = json.dumps(v) if not isinstance(v, str) else v
conn.execute(
"INSERT OR REPLACE INTO config (category, subtype, item_name, key, value) VALUES (?, ?, ?, ?, ?)",
(key, normalized_subtype, "default", k, val_str),
("tool", subtype, "default", k, val_str),
)
count += 1
else:
@@ -1197,30 +1106,15 @@ def save_config_and_verify(config: Dict[str, Any], retries: int = 3, delay: floa
# Nothing special to verify; return success.
return saved
# Reload directly from disk and compare the canonical debrid/provider keys
# Reload directly from disk and compare the canonical plugin key.
clear_config_cache()
reloaded = load_config()
# Provider-level key
prov_block = reloaded.get("provider", {}) if isinstance(reloaded, dict) else {}
prov_key = None
if isinstance(prov_block, dict):
aentry = prov_block.get("alldebrid")
if isinstance(aentry, dict):
for k in ("api_key", "API_KEY", "apikey", "APIKEY"):
v = aentry.get(k)
if isinstance(v, str) and v.strip():
prov_key = v.strip()
break
elif isinstance(aentry, str) and aentry.strip():
prov_key = aentry.strip()
# Store-level key
try:
store_key = get_debrid_api_key(reloaded, service="All-debrid")
reloaded_key = _extract_expected_alldebrid_key(reloaded)
except Exception:
store_key = None
reloaded_key = None
if prov_key == expected_key or store_key == expected_key:
if reloaded_key == expected_key:
try:
# Log a short, masked fingerprint to aid debugging without exposing the key itself
import hashlib
+13 -51
View File
@@ -504,67 +504,29 @@ def rows_to_config(rows) -> Dict[str, Any]:
if cat == 'global':
config[key] = parsed_val
else:
# Modular structure: config[cat][sub][name][key]
if cat in ('provider', 'tool'):
# Modular structure: config[category][subtype][item_name?][key]
if cat == 'plugin':
cat_dict = config.setdefault('plugin', {})
sub_dict = cat_dict.setdefault(sub, {})
if str(name or '').strip().lower() == 'default':
sub_dict[key] = parsed_val
else:
name_dict = sub_dict.setdefault(name, {})
name_dict[key] = parsed_val
elif cat in ('provider', 'store'):
continue
elif cat == 'tool':
cat_dict = config.setdefault(cat, {})
sub_dict = cat_dict.setdefault(sub, {})
sub_dict[key] = parsed_val
elif cat == 'store':
# Migrate legacy store rows into the unified plugin namespace.
# store config used a 4-level path: (store, type, instance_name, key).
# Plugin config uses: config["plugin"][type][instance_name][key].
cat_dict = config.setdefault('plugin', {})
sub_dict = cat_dict.setdefault(sub, {})
name_dict = sub_dict.setdefault(name, {})
name_dict[key] = parsed_val
else:
config.setdefault(cat, {})[key] = parsed_val
return config
def migrate_store_category_to_plugin() -> int:
"""One-time migration: re-key category='store' DB rows to category='plugin'.
The 'store' category used ``(store, type, instance_name, key)`` tuples;
the unified plugin system uses the same 4-level path under category='plugin'.
Existing 'plugin' rows for the same (subtype, item_name, key) are overwritten.
Returns the number of rows that were migrated (0 if already migrated).
"""
try:
count_row = db.fetchone(
"SELECT COUNT(*) AS n FROM config WHERE category='store' AND subtype != 'folder'"
)
count = int(count_row['n']) if count_row else 0
if count == 0:
# Also clean up any lingering folder-store rows
db.execute("DELETE FROM config WHERE category='store'")
with db._conn_lock:
db.conn.commit()
return 0
# Copy store rows to plugin, replacing any pre-existing plugin rows for
# the same (subtype, item_name, key), then delete the old store rows.
db.execute(
"""
INSERT OR REPLACE INTO config (category, subtype, item_name, key, value)
SELECT 'plugin', subtype, item_name, key, value
FROM config
WHERE category = 'store' AND subtype != 'folder'
"""
)
db.execute("DELETE FROM config WHERE category = 'store'")
with db._conn_lock:
db.conn.commit()
logger.info("Migrated %d config rows from category='store' to category='plugin'", count)
return count
except Exception:
logger.exception("Failed to migrate store config rows to plugin category")
return 0
def get_config_all() -> Dict[str, Any]:
"""Retrieve all configuration from the database in the legacy dict format."""
"""Retrieve all configuration from the database in the canonical plugin-centric dict format."""
rows = db.fetchall("SELECT category, subtype, item_name, key, value FROM config")
return rows_to_config(rows)
+2 -2
View File
@@ -277,12 +277,12 @@ def extract_records(doc_or_html: Any, base_url: Optional[str] = None, xpaths: Op
return normed, chosen
# Small convenience: convert records to SearchResult. Providers can call this or
# Small convenience: convert records to SearchResult. Plugins can call this or
# use their own mapping when they need full SearchResult objects.
from PluginCore.base import SearchResult # local import to avoid circular issues
def records_to_search_results(records: List[Dict[str, str]], table: str = "provider") -> List[SearchResult]:
def records_to_search_results(records: List[Dict[str, str]], table: str = "plugin") -> List[SearchResult]:
out: List[SearchResult] = []
for rec in records:
title = rec.get("title") or rec.get("name") or ""
+3 -3
View File
@@ -49,7 +49,7 @@ class PipeObject:
hash: str
store: str
provider: Optional[str] = None
plugin: Optional[str] = None
tag: List[str] = field(default_factory=list)
title: Optional[str] = None
url: Optional[str] = None
@@ -144,8 +144,8 @@ class PipeObject:
"store": self.store,
}
if self.provider:
data["provider"] = self.provider
if self.plugin:
data["plugin"] = self.plugin
if self.tag:
data["tag"] = self.tag
+3 -3
View File
@@ -61,12 +61,12 @@ def florencevision_missing_modules() -> List[str]:
def _provider_missing_modules(config: Dict[str, Any]) -> Dict[str, List[str]]:
missing: Dict[str, List[str]] = {}
provider_cfg = (config or {}).get("provider")
if not isinstance(provider_cfg, dict):
plugin_cfg = (config or {}).get("plugin")
if not isinstance(plugin_cfg, dict):
return missing
for provider_name, requirements in _PROVIDER_DEPENDENCIES.items():
block = provider_cfg.get(provider_name)
block = plugin_cfg.get(provider_name)
if not isinstance(block, dict) or not block:
continue
missing_for_provider = [
+4 -4
View File
@@ -191,11 +191,11 @@ def coerce_to_pipe_object(
pipe_obj = models.PipeObject(
hash=hash_val,
store=store_val,
provider=str(
value.get("provider")
plugin=str(
value.get("plugin")
or value.get("prov")
or value.get("source")
or extra.get("provider")
or extra.get("plugin")
or extra.get("source")
or ""
).strip()
@@ -253,7 +253,7 @@ def coerce_to_pipe_object(
pipe_obj = models.PipeObject(
hash=hash_val,
store=store_val,
provider=None,
plugin=None,
path=str(path_val) if path_val and path_val != "unknown" else None,
title=title_val,
url=url_val,
+3 -19
View File
@@ -1545,19 +1545,16 @@ class PipelineExecutor:
table_meta = meta if isinstance(meta, dict) else None
if isinstance(meta, dict):
_add(meta.get("plugin"))
_add(meta.get("provider"))
except Exception:
logger.exception("Failed to inspect current_table/table metadata in _maybe_run_class_selector")
for item in selected_items or []:
if isinstance(item, dict):
_add(item.get("plugin"))
_add(item.get("provider"))
_add(item.get("store"))
_add(item.get("table"))
else:
_add(getattr(item, "plugin", None))
_add(getattr(item, "provider", None))
_add(getattr(item, "store", None))
_add(getattr(item, "table", None))
@@ -1664,17 +1661,14 @@ class PipelineExecutor:
meta = None
if isinstance(meta, dict):
_add(meta.get("plugin"))
_add(meta.get("provider"))
for item in selected_items or []:
if isinstance(item, dict):
_add(item.get("plugin"))
_add(item.get("provider"))
_add(item.get("table"))
_add(item.get("source"))
else:
_add(getattr(item, "plugin", None))
_add(getattr(item, "provider", None))
_add(getattr(item, "table", None))
_add(getattr(item, "source", None))
@@ -3129,20 +3123,10 @@ class PipelineExecutor:
mod = import_cmd_module(cmd_name, reload_loaded=True)
data = getattr(mod, "CMDLET", None) if mod else None
if data and hasattr(data, "exec") and callable(getattr(data, "exec")):
from SYS.cmdlet_spec import collect_registered_cmdlet_names
run_fn = getattr(data, "exec")
registered_names = set()
raw_name = getattr(data, "name", None)
if raw_name:
registered_names.add(str(raw_name).replace("_", "-").lower())
registered_names.add(str(cmd_name).replace("_", "-").lower())
for alias_attr in ("alias", "aliases"):
alias_values = getattr(data, alias_attr, None)
if alias_values:
for alias in alias_values:
alias_text = str(alias or "").replace("_", "-").lower().strip()
if alias_text:
registered_names.add(alias_text)
for registered_name in registered_names:
for registered_name in collect_registered_cmdlet_names(data, fallback_name=cmd_name):
REGISTRY[registered_name] = run_fn
cmd_fn = run_fn
except Exception:
+51 -51
View File
@@ -14,6 +14,36 @@ logger = logging.getLogger(__name__)
ConfigField = Dict[str, Any]
def _import_plugin_support_module(plugin_name: str) -> Optional[Any]:
normalized = str(plugin_name or "").strip()
if not normalized:
return None
try:
return importlib.import_module(f"plugins.{normalized}")
except Exception:
return None
def _iter_plugin_module_names() -> List[str]:
names: List[str] = []
try:
import plugins as plugin_package
except Exception:
logger.exception("Failed to import plugins package for config discovery")
return names
package_path = getattr(plugin_package, "__path__", None)
if not package_path:
return names
for module_info in pkgutil.iter_modules(package_path):
name = str(module_info.name or "").strip()
if not name or name.startswith("_"):
continue
names.append(name)
return names
def _normalize_schema(fields: Optional[Iterable[Any]]) -> List[ConfigField]:
normalized: List[ConfigField] = []
seen: set[str] = set()
@@ -55,48 +85,38 @@ def _call_schema(owner: Any, label: str) -> List[ConfigField]:
def get_store_schema(store_type: str) -> List[ConfigField]:
"""Return config schema for a store type.
After the storeplugin migration, store types are plugins. We look up the
plugin schema by name; if not found we return an empty list.
Store types are now plugins. We look up the plugin schema by name; if not
found we return an empty list.
"""
normalized = str(store_type or "").strip()
# Strip a legacy "store-" prefix so callers using the old type name still work
if normalized.startswith("store-"):
normalized = normalized[len("store-"):]
return get_plugin_schema(normalized)
return get_plugin_schema(str(store_type or "").strip())
def get_plugin_schema(plugin_name: str) -> List[ConfigField]:
plugin_class = get_plugin_class(str(plugin_name or "").strip())
if plugin_class is None:
normalized_name = str(plugin_name or "").strip()
if not normalized_name:
return []
return _call_schema(plugin_class, f"plugin '{plugin_name}'")
plugin_class = get_plugin_class(normalized_name)
if plugin_class is not None:
schema = _call_schema(plugin_class, f"plugin '{normalized_name}'")
if schema:
return schema
def get_tool_schema(tool_name: str) -> List[ConfigField]:
tool_name = str(tool_name or "").strip()
if not tool_name:
module = _import_plugin_support_module(normalized_name)
if module is None:
return []
try:
module = importlib.import_module(f"tool.{tool_name}")
except Exception:
logger.exception("Failed to import tool module 'tool.%s'", tool_name)
return []
return _call_schema(module, f"tool '{tool_name}'")
return _call_schema(module, f"plugin support '{normalized_name}'")
def get_item_schema(item_type: str, item_name: str) -> List[ConfigField]:
normalized_type = str(item_type or "").strip()
normalized_name = str(item_name or "").strip()
if normalized_type.startswith("store-"):
return get_store_schema(normalized_type.replace("store-", "", 1))
if normalized_type.startswith("plugin-"):
# Multi-instance plugin: plugin-{ptype}; item_name is the instance name
ptype = normalized_type[len("plugin-"):]
return get_plugin_schema(ptype)
if normalized_type in {"provider", "plugin"}:
if normalized_type == "plugin":
return get_plugin_schema(normalized_name)
if normalized_type == "tool":
return get_tool_schema(normalized_name)
return []
@@ -143,13 +163,6 @@ def build_default_plugin_config(plugin_name: str) -> Dict[str, Any]:
return config
def build_default_tool_config(tool_name: str) -> Dict[str, Any]:
config: Dict[str, Any] = {}
for field in get_tool_schema(tool_name):
config[field["key"]] = field.get("default", "")
return config
def get_required_config_keys(item_type: str, item_name: str) -> List[str]:
normalized_type = str(item_type or "").strip()
normalized_name = str(item_name or "").strip()
@@ -170,9 +183,9 @@ def get_required_config_keys(item_type: str, item_name: str) -> List[str]:
if field.get("required"):
_add_key(field.get("key"))
if normalized_type.startswith("plugin-") or normalized_type.startswith("store-"):
# Multi-instance plugin (plugin-{ptype}) or legacy store-{type}: look up by plugin name
ptype = normalized_type.replace("plugin-", "", 1).replace("store-", "", 1)
if normalized_type.startswith("plugin-"):
# Multi-instance plugin (plugin-{ptype}): look up by plugin name.
ptype = normalized_type.replace("plugin-", "", 1)
plugin_class = get_plugin_class(ptype)
if plugin_class is not None:
try:
@@ -180,7 +193,7 @@ def get_required_config_keys(item_type: str, item_name: str) -> List[str]:
_add_key(required_key)
except Exception:
logger.exception("Failed to load required config keys for plugin '%s'", ptype)
elif normalized_type in {"provider", "plugin"}:
elif normalized_type == "plugin":
plugin_class = get_plugin_class(normalized_name)
if plugin_class is not None:
try:
@@ -211,20 +224,7 @@ def get_configurable_plugin_types() -> List[str]:
plugin_cls = info.plugin_class
if get_plugin_schema(info.canonical_name) or getattr(plugin_cls, 'MULTI_INSTANCE', False):
options.append(info.canonical_name)
return sorted(set(options))
def get_configurable_tool_types() -> List[str]:
options: List[str] = []
try:
import tool as tool_package
for module_info in pkgutil.iter_modules(tool_package.__path__):
tool_name = str(module_info.name or "").strip()
if not tool_name:
continue
if get_tool_schema(tool_name):
options.append(tool_name)
except Exception:
logger.exception("Failed to discover configurable tool modules")
for module_name in _iter_plugin_module_names():
if get_plugin_schema(module_name):
options.append(module_name)
return sorted(set(options))
+1 -1
View File
@@ -51,7 +51,7 @@ class Plugin:
raise RuntimeError(f"plugin '{self.name}' adapter failed") from exc
cols = self.get_columns(rows)
return ResultTable(provider=self.name, rows=rows, columns=cols, meta=self.metadata or {})
return ResultTable(plugin=self.name, rows=rows, columns=cols, meta=self.metadata or {})
def serialize_row(self, row: ResultModel) -> Dict[str, Any]:
r = ensure_result_model(row)
+5 -5
View File
@@ -35,21 +35,21 @@ class ResultModel:
@dataclass(frozen=True)
class ResultTable:
"""Concrete, provider-owned table of rows/columns.
"""Concrete, plugin-owned table of rows/columns.
This is intentionally minimal: it only stores rows, column specs, and
optional metadata used by renderers. It does not auto-normalize legacy
objects or infer columns.
"""
provider: str
plugin: str
rows: List[ResultModel]
columns: List[ColumnSpec]
meta: Dict[str, Any] = field(default_factory=dict)
def __post_init__(self) -> None:
if not str(self.provider or "").strip():
raise ValueError("provider required for ResultTable")
if not str(self.plugin or "").strip():
raise ValueError("plugin required for ResultTable")
object.__setattr__(self, "rows", [ensure_result_model(r) for r in self.rows])
if not self.columns:
raise ValueError("columns are required for ResultTable")
@@ -70,7 +70,7 @@ class ResultTable:
"ext": r.ext,
"size_bytes": r.size_bytes,
"metadata": r.metadata or {},
"source": r.source or self.provider,
"source": r.source or self.plugin,
"_selection_args": list(selection or []),
}
+2 -2
View File
@@ -516,12 +516,12 @@ def extract_link(result: Any, args: Iterable[str]) -> Any | None:
def get_api_key(config: dict[str, Any], service: str, key_path: str) -> str | None:
"""Get API key from config with fallback support.
"""Get API key from a dot-notation config path.
Args:
config: Configuration dictionary
service: Service name for logging
key_path: Dot-notation path to key (e.g., "Debrid.All-debrid")
key_path: Dot-notation path to key (e.g., "plugin.alldebrid.api_key")
Returns:
API key if found and not empty, None otherwise
+2 -11
View File
@@ -81,7 +81,7 @@ class Worker:
"""
try:
if self.manager:
self.manager.append_worker_stdout(self.id, text)
self.manager.append_stdout(self.id, text)
else:
self._stdout_buffer.append(text)
except Exception as e:
@@ -232,7 +232,7 @@ class WorkerLoggingHandler(logging.StreamHandler):
log_text = "\n".join(self.buffer)
try:
if self.manager:
self.manager.append_worker_stdout(
self.manager.append_stdout(
self.worker_id,
log_text,
channel="log"
@@ -872,15 +872,6 @@ class WorkerManager:
logger.error(f"[WorkerManager] Error getting stdout: {e}", exc_info=True)
return ""
def append_worker_stdout(
self,
worker_id: str,
text: str,
channel: str = "stdout"
) -> bool:
"""Compatibility wrapper for append_stdout."""
return self.append_stdout(worker_id, text, channel=channel)
def clear_stdout(self, worker_id: str) -> bool:
"""Clear stdout logs for a worker.