From 8685fbb723839a95c2107bb4b598ac6f7bb3ec5a Mon Sep 17 00:00:00 2001 From: Nose Date: Mon, 27 Apr 2026 21:17:53 -0700 Subject: [PATCH] updated plugin refactor and added FTP and SCP plugins , also hydrusnetwork plugin migration --- CLI.py | 1 + ProviderCore/base.py | 18 + ProviderCore/registry.py | 31 +- SYS/config.py | 183 +++- Store/HydrusNetwork.py | 29 + TUI/modalscreen/config_modal.py | 123 ++- cmdlet/_shared.py | 124 +-- cmdlet/add_relationship.py | 17 +- cmdlet/archive_file.py | 73 +- cmdlet/delete_file.py | 121 +-- cmdlet/download_file.py | 12 +- cmdlet/get_relationship.py | 53 +- cmdnat/config.py | 148 ++- docs/ftp_plugin_tutorial.md | 164 +++ docs/scp_plugin_tutorial.md | 136 +++ plugins/README.md | 10 +- plugins/alldebrid/__init__.py | 67 +- .../data => plugins/alldebrid}/alldebrid.json | 6 +- plugins/ftp/__init__.py | 778 +++++++++++++++ plugins/openlibrary/__init__.py | 896 ++++++++++++++++- plugins/scp/__init__.py | 938 ++++++++++++++++++ plugins/soulseek/__init__.py | 98 +- plugins/ytdlp/__init__.py | 27 +- scripts/pyproject.toml | 2 + 24 files changed, 3650 insertions(+), 405 deletions(-) create mode 100644 docs/ftp_plugin_tutorial.md create mode 100644 docs/scp_plugin_tutorial.md rename {API/data => plugins/alldebrid}/alldebrid.json (99%) create mode 100644 plugins/ftp/__init__.py create mode 100644 plugins/scp/__init__.py diff --git a/CLI.py b/CLI.py index 158a196..114f782 100644 --- a/CLI.py +++ b/CLI.py @@ -657,6 +657,7 @@ class CmdletCompleter(Completer): return value def _used_arg_logicals( + self, cmd_name: str, stage_tokens: List[str], config: Dict[str, diff --git a/ProviderCore/base.py b/ProviderCore/base.py index 6cfb629..dd099b6 100644 --- a/ProviderCore/base.py +++ b/ProviderCore/base.py @@ -422,6 +422,24 @@ class Provider(ABC): _ = quiet_mode return None + def config_helper_text(self) -> str: + """Optional helper text shown in the config editor.""" + + return "" + + def config_actions(self) -> List[Dict[str, Any]]: + """Optional actions exposed in the config editor for this provider.""" + + return [] + + def run_config_action(self, action_id: str, **_kwargs: Any) -> Dict[str, Any]: + """Execute a provider-owned config action from the config editor.""" + + return { + "ok": False, + "message": f"Provider '{self.name}' does not support config action '{action_id}'.", + } + def upload(self, file_path: str, **kwargs: Any) -> str: """Upload a file and return a URL or identifier.""" raise NotImplementedError(f"Provider '{self.name}' does not support upload") diff --git a/ProviderCore/registry.py b/ProviderCore/registry.py index ac91af2..1a510b4 100644 --- a/ProviderCore/registry.py +++ b/ProviderCore/registry.py @@ -136,7 +136,28 @@ class ProviderRegistry: self._discovered = False self._external_dirs_scanned = False + def _ensure_builtin_package_dirs(self) -> None: + if self._builtin_package_dirs or not self.package_name: + return + try: + package = importlib.import_module(self.package_name) + except Exception: + return + + package_path = getattr(package, "__path__", None) + if not package_path: + return + + builtin_dirs: List[Path] = [] + for entry in package_path: + try: + builtin_dirs.append(Path(str(entry)).resolve()) + except Exception: + builtin_dirs.append(Path(str(entry))) + self._builtin_package_dirs = tuple(builtin_dirs) + def _is_builtin_package_dir(self, candidate: Path) -> bool: + self._ensure_builtin_package_dirs() try: resolved = candidate.resolve() except Exception: @@ -253,6 +274,7 @@ class ProviderRegistry: if self._external_dirs_scanned: return self._external_dirs_scanned = True + self._ensure_builtin_package_dirs() for plugin_dir in _iter_external_plugin_dirs(): if self._is_builtin_package_dir(plugin_dir): @@ -302,15 +324,8 @@ class ProviderRegistry: return self._register_module(package) + self._ensure_builtin_package_dirs() package_path = getattr(package, "__path__", None) - if package_path: - builtin_dirs: List[Path] = [] - for entry in package_path: - try: - builtin_dirs.append(Path(str(entry)).resolve()) - except Exception: - builtin_dirs.append(Path(str(entry))) - self._builtin_package_dirs = tuple(builtin_dirs) if not package_path: self._discover_external_plugins() return diff --git a/SYS/config.py b/SYS/config.py index 820047f..79ff0da 100644 --- a/SYS/config.py +++ b/SYS/config.py @@ -8,6 +8,7 @@ import time import os import re import datetime +import shutil import sys import tempfile from copy import deepcopy @@ -532,7 +533,7 @@ def resolve_cookies_path( if candidate.is_file(): return candidate - plugin_cookie = resolve_plugin_asset_path("ytdlp", "cookies.txt", script_dir=base_dir) + plugin_cookie = _resolve_ytdlp_plugin_cookie_path(base_dir) if plugin_cookie is not None: return plugin_cookie @@ -542,6 +543,30 @@ def resolve_cookies_path( return None +def _resolve_ytdlp_plugin_cookie_path(base_dir: Path) -> Optional[Path]: + plugin_cookie = resolve_plugin_asset_path("ytdlp", "cookies.txt", script_dir=base_dir) + if plugin_cookie is not None: + return plugin_cookie + + plugin_dir = _resolve_app_root(base_dir) / "plugins" / "ytdlp" + if not plugin_dir.is_dir(): + return None + + plugin_cookie = plugin_dir / "cookies.txt" + legacy_cookie = _resolve_app_root(base_dir) / "cookies.txt" + try: + if legacy_cookie.is_file() and not plugin_cookie.exists(): + plugin_cookie.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(legacy_cookie, plugin_cookie) + return plugin_cookie + except Exception: + return None + + if plugin_cookie.is_file(): + return plugin_cookie + return None + + def resolve_debug_log(config: Dict[str, Any]) -> Optional[Path]: value = config.get("download_debug_log") if not value: @@ -721,6 +746,105 @@ def _count_changed_entries(old_config: Dict[str, Any], new_config: Dict[str, Any return len(changed) + len(removed) +def _changed_entry_keys(old_config: Dict[str, Any], new_config: Dict[str, Any]) -> set[Tuple[str, str, str, str]]: + old_entries = _flatten_config_entries(old_config or {}) + new_entries = _flatten_config_entries(new_config or {}) + keys = set(old_entries) | set(new_entries) + return {key for key in keys if old_entries.get(key, _CONFIG_MISSING) != new_entries.get(key, _CONFIG_MISSING)} + + +def _config_from_flattened_entries( + entries: Dict[Tuple[str, str, str, str], Any], +) -> Dict[str, Any]: + config: Dict[str, Any] = {} + for (category, subtype, item_name, key), value in entries.items(): + if category == "global": + config[key] = value + continue + + if category == "store": + store_block = config.setdefault("store", {}) + subtype_block = store_block.setdefault(subtype, {}) + item_block = subtype_block.setdefault(item_name, {}) + item_block[key] = value + continue + + if category in {"provider", "tool"}: + category_block = config.setdefault(category, {}) + subtype_block = category_block.setdefault(subtype, {}) + subtype_block[key] = value + continue + + category_block = config.setdefault(category, {}) + if isinstance(category_block, dict): + subtype_block = category_block.setdefault(subtype, {}) + if isinstance(subtype_block, dict): + item_block = subtype_block.setdefault(item_name, {}) + if isinstance(item_block, dict): + item_block[key] = value + + _normalize_plugin_config_aliases(config) + _sync_alldebrid_api_key(config) + return config + + +def _merge_non_conflicting_config_changes( + base_config: Dict[str, Any], + disk_config: Dict[str, Any], + local_config: Dict[str, Any], +) -> Optional[Dict[str, Any]]: + local_changed = _changed_entry_keys(base_config, local_config) + if not local_changed: + return deepcopy(disk_config) + + disk_changed = _changed_entry_keys(base_config, disk_config) + if local_changed & disk_changed: + return None + + merged_entries = dict(_flatten_config_entries(disk_config or {})) + local_entries = _flatten_config_entries(local_config or {}) + for key in local_changed: + if key in local_entries: + merged_entries[key] = local_entries[key] + else: + merged_entries.pop(key, None) + return _config_from_flattened_entries(merged_entries) + + +def _extract_expected_alldebrid_key(config: Dict[str, Any]) -> Optional[str]: + expected_key = None + try: + providers = config.get("provider", {}) if isinstance(config, dict) else {} + if isinstance(providers, dict): + entry = providers.get("alldebrid") + if entry is not None: + if isinstance(entry, dict): + for k in ("api_key", "API_KEY", "apikey", "APIKEY"): + v = entry.get(k) + if isinstance(v, str) and v.strip(): + expected_key = v.strip() + break + elif isinstance(entry, str) and entry.strip(): + expected_key = entry.strip() + if not expected_key: + store_block = config.get("store", {}) if isinstance(config, dict) else {} + debrid = store_block.get("debrid") if isinstance(store_block, dict) else None + if isinstance(debrid, dict): + srv = debrid.get("all-debrid") + if isinstance(srv, dict): + for k in ("api_key", "API_KEY", "apikey", "APIKEY"): + v = srv.get(k) + if isinstance(v, str) and v.strip(): + expected_key = v.strip() + break + elif isinstance(srv, str) and srv.strip(): + expected_key = srv.strip() + except Exception as exc: + logger.debug("Failed to determine expected AllDebrid key: %s", exc, exc_info=True) + expected_key = None + return expected_key + + def load_config(*, emit_summary: bool = True) -> Dict[str, Any]: global _CONFIG_CACHE, _LAST_SAVED_CONFIG, _CONFIG_SUMMARY_PENDING if _CONFIG_CACHE: @@ -838,6 +962,7 @@ def save_config(config: Dict[str, Any]) -> int: def _write_entries() -> int: global _CONFIG_CACHE, _LAST_SAVED_CONFIG count = 0 + config_to_write = config # Use the transaction-provided connection directly to avoid re-acquiring # the connection lock via db.* helpers which can lead to deadlock. with db.transaction() as conn: @@ -861,14 +986,22 @@ def save_config(config: Dict[str, Any]) -> int: _CONFIG_CACHE = current_disk _LAST_SAVED_CONFIG = deepcopy(current_disk) return 0 - # Otherwise, abort to avoid overwriting external changes - raise ConfigSaveConflict( - "Configuration on disk changed since you started editing; save aborted to prevent overwrite. Reload and reapply your changes." + merged_config = _merge_non_conflicting_config_changes( + previous_config, + current_disk, + config, ) + if merged_config is None: + # Otherwise, abort to avoid overwriting external changes + raise ConfigSaveConflict( + "Configuration on disk changed since you started editing; save aborted to prevent overwrite. Reload and reapply your changes." + ) + config_to_write = merged_config + log("Config save rebased local changes onto newer disk configuration.") # Proceed with writing when no conflicting external changes detected conn.execute("DELETE FROM config") - for key, value in config.items(): + for key, value in config_to_write.items(): if key in ('store', 'provider', 'tool') and isinstance(value, dict): for subtype, instances in value.items(): if not isinstance(instances, dict): @@ -904,6 +1037,8 @@ def save_config(config: Dict[str, Any]) -> int: ("global", "none", "none", key, val_str), ) count += 1 + _CONFIG_CACHE = config_to_write + _LAST_SAVED_CONFIG = deepcopy(config_to_write) return count @@ -964,9 +1099,6 @@ def save_config(config: Dict[str, Any]) -> int: logger.exception("Failed to release save lock after CRITICAL configuration save failure: %s", exc) raise - clear_config_cache() - _CONFIG_CACHE = config - _LAST_SAVED_CONFIG = deepcopy(config) return saved_entries @@ -988,37 +1120,10 @@ def save_config_and_verify(config: Dict[str, Any], retries: int = 3, delay: floa AllDebrid) were written successfully. If verification fails after the configured number of retries, a RuntimeError is raised. """ - # Detect an API key that should be verified (provider or store-backed) - expected_key = None - try: - providers = config.get("provider", {}) if isinstance(config, dict) else {} - if isinstance(providers, dict): - entry = providers.get("alldebrid") - if entry is not None: - # _extract_api_key is a small internal helper; reuse the implementation here - if isinstance(entry, dict): - for k in ("api_key", "API_KEY", "apikey", "APIKEY"): - v = entry.get(k) - if isinstance(v, str) and v.strip(): - expected_key = v.strip() - break - elif isinstance(entry, str) and entry.strip(): - expected_key = entry.strip() - if not expected_key: - store_block = config.get("store", {}) if isinstance(config, dict) else {} - debrid = store_block.get("debrid") if isinstance(store_block, dict) else None - if isinstance(debrid, dict): - srv = debrid.get("all-debrid") - if isinstance(srv, dict): - for k in ("api_key", "API_KEY", "apikey", "APIKEY"): - v = srv.get(k) - if isinstance(v, str) and v.strip(): - expected_key = v.strip() - break - elif isinstance(srv, str) and srv.strip(): - expected_key = srv.strip() - except Exception as exc: - logger.debug("Failed to determine expected key for save verification: %s", exc, exc_info=True) + # Only perform the extra verification loop when the AllDebrid key actually changed. + expected_key = _extract_expected_alldebrid_key(config) + baseline_key = _extract_expected_alldebrid_key(_LAST_SAVED_CONFIG) + if expected_key == baseline_key: expected_key = None last_exc: Exception | None = None diff --git a/Store/HydrusNetwork.py b/Store/HydrusNetwork.py index 52f7526..06aabec 100644 --- a/Store/HydrusNetwork.py +++ b/Store/HydrusNetwork.py @@ -89,6 +89,8 @@ class HydrusNetwork(Store): Maintains its own HydrusClient. """ + STORE_TYPE = "hydrusnetwork" + @classmethod def config_schema(cls) -> List[Dict[str, Any]]: return [ @@ -1776,6 +1778,33 @@ class HydrusNetwork(Store): debug(f"{self._log_prefix()} delete_file failed: {exc}") return False + def build_file_url(self, file_hash: str, *, include_access_key: bool = True) -> str: + normalized = str(file_hash or "").strip().lower() + base_url = str(self.URL).rstrip("/") + url = f"{base_url}/get_files/file?hash={quote(normalized)}" + if include_access_key and str(self.API or "").strip(): + url = f"{url}&Hydrus-Client-API-Access-Key={quote(str(self.API))}" + return url + + def fetch_file_metadata(self, file_hash: str, **kwargs: Any) -> Optional[Dict[str, Any]]: + try: + client = self._client + if client is None: + return None + return client.fetch_file_metadata(hashes=[str(file_hash or "").strip().lower()], **kwargs) + except Exception: + return None + + def get_relationships(self, file_hash: str) -> Optional[Dict[str, Any]]: + try: + client = self._client + if client is None: + return None + payload = client.get_file_relationships(str(file_hash or "").strip().lower()) + return payload if isinstance(payload, dict) else None + except Exception: + return None + def get_metadata(self, file_hash: str, **kwargs: Any) -> Optional[Dict[str, Any]]: """Get metadata for a file from Hydrus by hash. diff --git a/TUI/modalscreen/config_modal.py b/TUI/modalscreen/config_modal.py index 5530851..8512a58 100644 --- a/TUI/modalscreen/config_modal.py +++ b/TUI/modalscreen/config_modal.py @@ -32,7 +32,7 @@ from SYS.plugin_config import ( get_item_schema_map, get_required_config_keys, ) -from ProviderCore.registry import get_plugin +from ProviderCore.registry import get_plugin, get_plugin_class from TUI.modalscreen.matrix_room_picker import MatrixRoomPicker from TUI.modalscreen.selection_modal import SelectionModal import logging @@ -164,9 +164,12 @@ class ConfigModal(ModalScreen): self.editing_item_type = None # 'store' or 'provider' self.editing_item_name = None self._button_id_map = {} + self._provider_button_map: Dict[str, tuple[str, str]] = {} self._input_id_map = {} self._matrix_status: Optional[Static] = None self._matrix_test_running = False + self._provider_status: Optional[Static] = None + self._provider_action_running = False self._editor_snapshot: Optional[Dict[str, Any]] = None # Inline matrix rooms controls self._matrix_inline_list: Optional[ListView] = None @@ -256,6 +259,7 @@ class ConfigModal(ModalScreen): return self._button_id_map.clear() + self._provider_button_map.clear() self._input_id_map.clear() # Clear existing @@ -594,6 +598,33 @@ class ConfigModal(ModalScreen): row.mount(Button("Paste", id=f"paste-{inp_id}", classes="paste-btn")) idx += 1 + if item_type == "provider" and isinstance(item_name, str): + provider = self._instantiate_provider_for_editor(item_name, self.config_data) + if provider is not None: + provider_actions = provider.config_actions() or [] + if provider_actions: + container.mount(Rule()) + container.mount(Label(f"{provider.label} helpers", classes="config-label")) + helper_text = str(provider.config_helper_text() or "Use these helpers to validate provider settings.").strip() + status = Static(helper_text, id="provider-status") + container.mount(status) + self._provider_status = status + row = Horizontal(classes="field-row") + container.mount(row) + for action in provider_actions: + action_id = str(action.get("id") or "").strip() + if not action_id: + continue + button_id = f"provider-action-{item_name}-{action_id}".replace(" ", "-") + self._provider_button_map[button_id] = (item_name, action_id) + row.mount( + Button( + str(action.get("label") or action_id.replace("_", " ").title()), + id=button_id, + variant=str(action.get("variant") or "default"), + ) + ) + if ( item_type == "provider" and isinstance(item_name, str) @@ -755,6 +786,10 @@ class ConfigModal(ModalScreen): self.editing_item_type = None self.refresh_view() except Exception as exc: + try: + log(f"Configuration save failed: {exc}") + except Exception: + logger.exception("Failed to write save failure to logs") self.notify(f"Save failed: {exc}", severity="error", timeout=10) elif bid == "save-durable-btn": # Perform a synchronous, verified save and notify status to the user. @@ -788,6 +823,10 @@ class ConfigModal(ModalScreen): self.refresh_view() self._editor_snapshot = None except Exception as exc: + try: + log(f"Durable configuration save failed: {exc}") + except Exception: + logger.exception("Failed to write durable save failure to logs") self.notify(f"Durable save failed: {exc}", severity="error", timeout=10) try: log(f"Durable save failed: {exc}") @@ -823,8 +862,15 @@ class ConfigModal(ModalScreen): saved = self.save_all() self.notify("Saving configuration...", timeout=3) except Exception as exc: + try: + log(f"Configuration save failed while deleting config entry: {exc}") + except Exception: + logger.exception("Failed to write config delete save failure to logs") self.notify(f"Save failed: {exc}", severity="error", timeout=10) self.refresh_view() + elif bid in self._provider_button_map: + provider_name, action_id = self._provider_button_map[bid] + self._request_provider_action(provider_name, action_id) elif bid == "add-store-btn": options = get_configurable_store_types() self.app.push_screen(SelectionModal("Select Store Type", options), callback=self.on_store_type_selected) @@ -882,6 +928,10 @@ class ConfigModal(ModalScreen): try: entries = save_config(self.config_data) except Exception as exc: + try: + log(f"Saving Matrix default rooms failed: {exc}") + except Exception: + logger.exception("Failed to write Matrix room save failure to logs") if self._matrix_status: self._matrix_status.update(f"Saving default rooms failed: {exc}") return @@ -921,6 +971,69 @@ class ConfigModal(ModalScreen): else: self.notify("Clipboard not supported in this terminal", severity="warning") + def _instantiate_provider_for_editor(self, provider_name: str, config_data: Optional[Dict[str, Any]] = None) -> Optional[Any]: + try: + provider_class = get_plugin_class(provider_name) + except Exception: + provider_class = None + if provider_class is None: + return None + try: + return provider_class(config_data or self.config_data) + except Exception: + logger.exception("Failed to instantiate provider '%s' for config helper", provider_name) + return None + + def _request_provider_action(self, provider_name: str, action_id: str) -> None: + if self._provider_action_running: + return + self._synchronize_inputs_to_config() + self._provider_action_running = True + if self._provider_status is not None: + self._provider_status.update(f"Running {action_id.replace('_', ' ')}…") + self._provider_action_background(provider_name, action_id, deepcopy(self.config_data)) + + @work(thread=True) + def _provider_action_background(self, provider_name: str, action_id: str, config_snapshot: Dict[str, Any]) -> None: + try: + provider = self._instantiate_provider_for_editor(provider_name, config_snapshot) + if provider is None: + raise RuntimeError(f"Provider '{provider_name}' is unavailable") + result = provider.run_config_action(action_id) + if not isinstance(result, dict): + result = {"ok": False, "message": f"Provider '{provider_name}' returned an invalid config action result."} + except Exception as exc: + result = {"ok": False, "message": str(exc) or f"Provider action '{action_id}' failed."} + + try: + self.app.call_from_thread(self._provider_action_complete, provider_name, action_id, result) + except Exception: + self._provider_action_complete(provider_name, action_id, result) + + def _provider_action_complete(self, provider_name: str, action_id: str, result: Dict[str, Any]) -> None: + self._provider_action_running = False + ok = bool(result.get("ok")) + message = str(result.get("message") or f"Provider action '{action_id}' finished.") + updates = result.get("config_updates") + + if ok and isinstance(updates, dict): + provider_block = self.config_data.setdefault("provider", {}).setdefault(provider_name, {}) + if isinstance(provider_block, dict): + provider_block.update(updates) + message = f"{message}" + try: + self.refresh_view() + except Exception: + logger.exception("Failed to refresh config view after provider action") + + if self._provider_status is not None: + self._provider_status.update(message) + + try: + self.notify(message, severity="error" if not ok else "information", timeout=8) + except Exception: + logger.exception("Failed to notify provider action result for %s/%s", provider_name, action_id) + # Backup/restore helpers removed: forensics/audit mode disabled and restore UI removed. def on_store_type_selected(self, stype: str) -> None: @@ -1130,6 +1243,10 @@ class ConfigModal(ModalScreen): try: entries = save_config(self.config_data) except Exception as exc: + try: + log(f"Saving configuration before Matrix test failed: {exc}") + except Exception: + logger.exception("Failed to write Matrix test pre-save failure to logs") if self._matrix_status: self._matrix_status.update(f"Saving configuration failed: {exc}") self._matrix_test_running = False @@ -1290,6 +1407,10 @@ class ConfigModal(ModalScreen): try: entries = save_config(self.config_data) except Exception as exc: + try: + log(f"Saving configuration before Matrix room load failed: {exc}") + except Exception: + logger.exception("Failed to write Matrix load pre-save failure to logs") if self._matrix_status: self._matrix_status.update(f"Saving configuration failed: {exc}") self._matrix_test_running = False diff --git a/cmdlet/_shared.py b/cmdlet/_shared.py index 6c979ae..d69e518 100644 --- a/cmdlet/_shared.py +++ b/cmdlet/_shared.py @@ -1410,7 +1410,7 @@ def fetch_hydrus_metadata( Eliminates repeated boilerplate: client initialization, error handling, metadata extraction. Args: - config: Configuration object (passed to hydrus_wrapper.get_client) + config: Configuration object used to resolve the Hydrus provider/store hash_hex: File hash to fetch metadata for store_name: Optional Hydrus store name. When provided, do not fall back to a global/default Hydrus client. hydrus_client: Optional explicit Hydrus client. When provided, takes precedence. @@ -1422,38 +1422,53 @@ def fetch_hydrus_metadata( - metadata_dict: Dict from Hydrus (first item in metadata list) or None if unavailable - error_code: 0 on success, 1 on any error (suitable for returning from cmdlet execute()) """ - from API import HydrusNetwork - - hydrus_wrapper = HydrusNetwork - client = hydrus_client + hydrus_provider = None + try: + from ProviderCore.registry import get_plugin + + hydrus_provider = get_plugin("hydrusnetwork", config) + except Exception: + hydrus_provider = None + if client is None: - if store_name: - # Store specified: do not fall back to a global/default Hydrus client. + if hydrus_provider is not None: try: - from Store import Store - - store = Store(config) - backend = store[str(store_name)] - candidate = getattr(backend, "_client", None) - if candidate is not None and hasattr(candidate, "fetch_file_metadata"): - client = candidate + client = hydrus_provider.get_client( + store_name=store_name if store_name else None, + allow_default=not bool(store_name), + ) except Exception as exc: - log(f"Hydrus client unavailable for store '{store_name}': {exc}") + if store_name: + log(f"Hydrus client unavailable for store '{store_name}': {exc}") + else: + log(f"Hydrus client unavailable: {exc}") client = None - if client is None: - log(f"Hydrus client unavailable for store '{store_name}'") - return None, 1 - else: - try: - client = hydrus_wrapper.get_client(config) - except Exception as exc: - log(f"Hydrus client unavailable: {exc}") - return None, 1 + if client is None and store_name: + log(f"Hydrus client unavailable for store '{store_name}'") + return None, 1 + if client is None and hydrus_provider is None: + log("Hydrus provider unavailable") + return None, 1 - if client is None: - log("Hydrus client unavailable") - return None, 1 + if hydrus_provider is not None: + try: + metadata = hydrus_provider.fetch_metadata( + hash_hex, + store_name=store_name if store_name else None, + **kwargs, + ) + except Exception as exc: + log(f"Hydrus metadata fetch failed: {exc}") + return None, 1 + if isinstance(metadata, dict): + return metadata, 0 + if client is None: + if store_name: + log(f"Hydrus client unavailable for store '{store_name}'") + else: + log("Hydrus metadata unavailable") + return None, 1 try: payload = client.fetch_file_metadata(hashes=[hash_hex], **kwargs) @@ -3725,10 +3740,13 @@ def check_url_exists_in_storage( match_rows: List[Dict[str, Any]] = [] max_rows = 200 + hydrus_provider = None try: - from Store.HydrusNetwork import HydrusNetwork + from ProviderCore.registry import get_plugin + + hydrus_provider = get_plugin("hydrusnetwork", config) except Exception: - HydrusNetwork = None # type: ignore + hydrus_provider = None for backend_name in backend_names: if _timed_out("backend scan"): @@ -3739,8 +3757,14 @@ def check_url_exists_in_storage( backend = storage[backend_name] except Exception: continue - - if HydrusNetwork is not None and isinstance(backend, HydrusNetwork): + + is_hydrus_backend = False + try: + is_hydrus_backend = bool(hydrus_provider and hydrus_provider.is_backend(backend, str(backend_name))) + except Exception: + is_hydrus_backend = False + + if is_hydrus_backend: if not hydrus_available: debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup") @@ -3776,44 +3800,6 @@ def check_url_exists_in_storage( found = True break - client = getattr(backend, "_client", None) - if found: - pass - elif client is None: - continue - for needle in (needles or [])[:6]: - if found: - break - if not _httpish(needle): - continue - try: - from API.HydrusNetwork import HydrusRequestSpec - - spec = HydrusRequestSpec( - method="GET", - endpoint="/add_urls/get_url_files", - query={"url": needle}, - ) - if hasattr(client, "_perform_request"): - response = client._perform_request(spec) - raw_hashes = None - if isinstance(response, dict): - raw_hashes = response.get("hashes") or response.get("file_hashes") - raw_ids = response.get("file_ids") - hash_list = raw_hashes if isinstance(raw_hashes, list) else [] - has_ids = isinstance(raw_ids, list) and len(raw_ids) > 0 - has_hashes = len(hash_list) > 0 - if has_hashes: - try: - found_hash = str(hash_list[0]).strip() - except Exception: - found_hash = None - if has_ids or has_hashes: - found = True - break - except Exception: - continue - if not found: continue diff --git a/cmdlet/add_relationship.py b/cmdlet/add_relationship.py index d9ec4f2..a550280 100644 --- a/cmdlet/add_relationship.py +++ b/cmdlet/add_relationship.py @@ -9,9 +9,9 @@ import sys from SYS.logger import log from SYS.item_accessors import get_sha256_hex, get_store_name +from ProviderCore.registry import get_plugin from SYS import pipeline as ctx -from API import HydrusNetwork as hydrus_wrapper from . import _shared as sh Cmdlet = sh.Cmdlet @@ -617,16 +617,20 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: # - If no store is specified, use the default Hydrus client. # NOTE: When a store is specified, we do not fall back to a global/default Hydrus client. hydrus_client = None + hydrus_provider = get_plugin("hydrusnetwork", config) if store_name and (not is_folder_store) and backend is not None: try: - candidate = getattr(backend, "_client", None) - if candidate is not None and hasattr(candidate, "set_relationship"): - hydrus_client = candidate + if hydrus_provider is not None: + hydrus_client = hydrus_provider.get_client( + store_name=str(store_name), + allow_default=False, + ) except Exception: hydrus_client = None elif not store_name: try: - hydrus_client = hydrus_wrapper.get_client(config) + if hydrus_provider is not None: + hydrus_client = hydrus_provider.get_client() except Exception: hydrus_client = None @@ -1049,8 +1053,9 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: return 1 # Build Hydrus client + hydrus_provider = get_plugin("hydrusnetwork", config) try: - hydrus_client = hydrus_wrapper.get_client(config) + hydrus_client = hydrus_provider.get_client() if hydrus_provider is not None else None except Exception as exc: log(f"Hydrus client unavailable: {exc}", file=sys.stderr) return 1 diff --git a/cmdlet/archive_file.py b/cmdlet/archive_file.py index 5c9d78f..eb3aa20 100644 --- a/cmdlet/archive_file.py +++ b/cmdlet/archive_file.py @@ -13,6 +13,7 @@ from typing import Any, Dict, List, Sequence, Set from urllib.parse import parse_qs, urlparse from SYS.logger import log +from ProviderCore.registry import get_plugin from SYS.item_accessors import get_http_url, get_sha256_hex, get_store_name from SYS.utils import extract_hydrus_hash_from_url @@ -71,10 +72,8 @@ def _maybe_download_hydrus_item( This is intentionally side-effect free except for writing the local temp file. """ - try: - from SYS.config import get_hydrus_access_key, get_hydrus_url - from API.HydrusNetwork import HydrusNetwork as HydrusClient, download_hydrus_file - except Exception: + hydrus_provider = get_plugin("hydrusnetwork", config) + if hydrus_provider is None: return None store_name = _extract_store_name(item) @@ -102,68 +101,10 @@ def _maybe_download_hydrus_item( is_hydrus_url = False if not (is_hydrus_url or store_hint): return None - - # Prefer store name as instance key; fall back to "home". - access_key = None - hydrus_url = None - for inst in [s for s in [store_lower, "home"] if s]: - try: - access_key = (get_hydrus_access_key(config, inst) or "").strip() or None - hydrus_url = (get_hydrus_url(config, inst) or "").strip() or None - if access_key and hydrus_url: - break - except Exception: - access_key = None - hydrus_url = None - - if not access_key or not hydrus_url: - return None - - client = HydrusClient(url=hydrus_url, access_key=access_key, timeout=60.0) - file_url = url if (url and is_hydrus_url) else client.file_url(file_hash) - - # Best-effort extension from Hydrus metadata. - suffix = ".hydrus" - try: - meta_response = client.fetch_file_metadata( - hashes=[file_hash], - include_mime=True - ) - entries = meta_response.get("metadata" - ) if isinstance(meta_response, - dict) else None - if isinstance(entries, list) and entries: - entry = entries[0] - if isinstance(entry, dict): - ext = entry.get("ext") - if isinstance(ext, str) and ext.strip(): - cleaned = ext.strip() - if not cleaned.startswith("."): - cleaned = "." + cleaned.lstrip(".") - if len(cleaned) <= 12: - suffix = cleaned - except Exception: - pass - - try: - output_dir.mkdir(parents=True, exist_ok=True) - except Exception: - pass - - dest = output_dir / f"{file_hash}{suffix}" - if dest.exists(): - dest = output_dir / f"{file_hash}_{uuid.uuid4().hex[:10]}{suffix}" - - headers = { - "Hydrus-Client-API-Access-Key": access_key - } - download_hydrus_file(file_url, headers, dest, timeout=60.0) - try: - if dest.exists() and dest.is_file(): - return dest - except Exception: - return None - return None + preferred_store = store_name or None + if url and is_hydrus_url: + return hydrus_provider.download_url(url, output_dir) + return hydrus_provider.download_hash_to_temp(file_hash, store_name=preferred_store, temp_root=output_dir) def _resolve_existing_or_fetch_path(item: Any, diff --git a/cmdlet/delete_file.py b/cmdlet/delete_file.py index 990fa3c..1a8ba4c 100644 --- a/cmdlet/delete_file.py +++ b/cmdlet/delete_file.py @@ -7,9 +7,9 @@ import sys from pathlib import Path from SYS.logger import debug, log +from ProviderCore.registry import get_plugin from Store import Store from . import _shared as sh -from API import HydrusNetwork as hydrus_wrapper from SYS import pipeline as ctx from SYS.result_table_helpers import add_row_columns from SYS.result_table import Table, _format_size @@ -129,6 +129,7 @@ class Delete_File(sh.Cmdlet): store = sh.get_field(item, "store") store_lower = str(store).lower() if store else "" + hydrus_provider = get_plugin("hydrusnetwork", config) backend = None try: @@ -144,18 +145,17 @@ class Delete_File(sh.Cmdlet): # so checking only the store name is unreliable. is_hydrus_store = False try: - if backend is not None: - from Store.HydrusNetwork import HydrusNetwork as HydrusStore - - is_hydrus_store = isinstance(backend, HydrusStore) + if hydrus_provider is not None and backend is not None: + is_hydrus_store = bool(hydrus_provider.is_backend(backend, str(store or ""))) except Exception: is_hydrus_store = False # Backwards-compatible fallback heuristic (older items might only carry a name). - if ((not is_hydrus_store) and bool(store_lower) - and ("hydrus" in store_lower or store_lower in {"home", - "work"})): - is_hydrus_store = True + if (not is_hydrus_store) and hydrus_provider is not None and bool(store_lower): + try: + is_hydrus_store = bool(hydrus_provider.is_store_name(store_lower)) + except Exception: + is_hydrus_store = False store_label = str(store) if store else "default" hydrus_prefix = f"[hydrusnetwork:{store_label}]" @@ -318,18 +318,20 @@ class Delete_File(sh.Cmdlet): should_try_hydrus = False if should_try_hydrus and hash_hex: - # Prefer deleting via the resolved store backend when it is a HydrusNetwork store. - # This ensures store-specific post-delete hooks run (e.g., clearing Hydrus deletion records). - did_backend_delete = False + did_hydrus_delete = False try: - if backend is not None: - deleter = getattr(backend, "delete_file", None) - if callable(deleter): - did_backend_delete = bool(deleter(hash_hex, reason=reason)) + if hydrus_provider is not None: + did_hydrus_delete = bool( + hydrus_provider.delete_hash( + hash_hex, + store_name=str(store) if store else None, + reason=reason or None, + ) + ) except Exception: - did_backend_delete = False + did_hydrus_delete = False - if did_backend_delete: + if did_hydrus_delete: hydrus_deleted = True title_str = str(title_val).strip() if title_val else "" if title_str: @@ -340,85 +342,12 @@ class Delete_File(sh.Cmdlet): else: debug(f"{hydrus_prefix} Deleted hash:{hash_hex}", file=sys.stderr) else: - # Fallback to direct client calls. - client = None - if store: - # Store specified: do not fall back to a global/default Hydrus client. - try: - registry = Store(config) - backend = registry[str(store)] - candidate = getattr(backend, "_client", None) - if candidate is not None and hasattr(candidate, "_post"): - client = candidate - except Exception as exc: - if not local_deleted: - log( - f"Hydrus client unavailable for store '{store}': {exc}", - file=sys.stderr, - ) - return False - if client is None: - if not local_deleted: - log( - f"Hydrus client unavailable for store '{store}'", - file=sys.stderr - ) - return False - else: - # No store context; use default Hydrus client. - try: - client = hydrus_wrapper.get_client(config) - except Exception as exc: - if not local_deleted: - log(f"Hydrus client unavailable: {exc}", file=sys.stderr) - return False - if client is None: - if not local_deleted: - log("Hydrus client unavailable", file=sys.stderr) - return False - - payload: Dict[str, - Any] = { - "hashes": [hash_hex] - } - if reason: - payload["reason"] = reason - try: - client._post( - "/add_files/delete_files", - data=payload - ) # type: ignore[attr-defined] - # Best-effort clear deletion record if supported by this client. - try: - clearer = getattr(client, "clear_file_deletion_record", None) - if callable(clearer): - clearer([hash_hex]) - else: - client._post( - "/add_files/clear_file_deletion_record", - data={ - "hashes": [hash_hex] - } - ) # type: ignore[attr-defined] - except Exception: - pass - - hydrus_deleted = True - title_str = str(title_val).strip() if title_val else "" - if title_str: - debug( - f"{hydrus_prefix} Deleted title:{title_str} hash:{hash_hex}", - file=sys.stderr, - ) + if not local_deleted: + if store: + log(f"Hydrus store unavailable for '{store}'", file=sys.stderr) else: - debug( - f"{hydrus_prefix} Deleted hash:{hash_hex}", - file=sys.stderr - ) - except Exception: - # If it's not in Hydrus (e.g. 404 or similar), that's fine - if not local_deleted: - return [] + log("Hydrus delete failed", file=sys.stderr) + return [] if hydrus_deleted and hash_hex: size_hint = None diff --git a/cmdlet/download_file.py b/cmdlet/download_file.py index 2a4fab8..dafe35e 100644 --- a/cmdlet/download_file.py +++ b/cmdlet/download_file.py @@ -980,10 +980,14 @@ class Download_File(Cmdlet): ) -> Optional[str]: if storage is None or not canonical_url: return None + hydrus_provider = None try: - from Store.HydrusNetwork import HydrusNetwork + registry_helpers = cls._load_provider_registry() + get_plugin = registry_helpers.get("get_plugin") + if callable(get_plugin): + hydrus_provider = get_plugin("hydrusnetwork", {}) except Exception: - HydrusNetwork = None # type: ignore + hydrus_provider = None try: backend_names = list(storage.list_searchable_backends() or []) @@ -1001,13 +1005,13 @@ class Download_File(Cmdlet): except Exception: pass try: - if HydrusNetwork is not None and isinstance(backend, HydrusNetwork) and not hydrus_available: + if hydrus_provider is not None and hydrus_provider.is_backend(backend, str(backend_name)) and not hydrus_available: continue except Exception: pass try: - if HydrusNetwork is not None and isinstance(backend, HydrusNetwork): + if hydrus_provider is not None and hydrus_provider.is_backend(backend, str(backend_name)): hashes = backend.find_hashes_by_url(canonical_url) or [] for existing_hash in hashes: normalized = sh.normalize_hash(existing_hash) diff --git a/cmdlet/get_relationship.py b/cmdlet/get_relationship.py index a87bb48..fc45942 100644 --- a/cmdlet/get_relationship.py +++ b/cmdlet/get_relationship.py @@ -5,12 +5,12 @@ import sys from SYS.detail_view_helpers import create_detail_view, prepare_detail_metadata from SYS.logger import log +from ProviderCore.registry import get_plugin from SYS.result_table_helpers import add_row_columns from SYS.selection_builder import build_hash_store_selection from SYS.result_publication import publish_result_table from SYS import pipeline as ctx -from API import HydrusNetwork as hydrus_wrapper from . import _shared as sh Cmdlet = sh.Cmdlet @@ -22,7 +22,6 @@ get_hash_for_operation = sh.get_hash_for_operation fetch_hydrus_metadata = sh.fetch_hydrus_metadata should_show_help = sh.should_show_help get_field = sh.get_field -from Store import Store CMDLET = Cmdlet( name="get-relationship", @@ -109,6 +108,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: return 1 # Fetch Hydrus relationships if we have a hash. + hydrus_provider = get_plugin("hydrusnetwork", config) hash_hex = ( normalize_hash(override_hash) @@ -118,29 +118,18 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: if hash_hex: try: - client = None store_label = "hydrus" - backend_obj = None if store_name: - # Store specified: do not fall back to a global/default Hydrus client. store_label = str(store_name) - try: - store = Store(config) - backend_obj = store[str(store_name)] - candidate = getattr(backend_obj, "_client", None) - if candidate is not None and hasattr(candidate, - "get_file_relationships"): - client = candidate - except Exception: - client = None - if client is None: + if hydrus_provider is None: log( f"Hydrus client unavailable for store '{store_name}'", file=sys.stderr ) return 1 + relationships = hydrus_provider.get_relationships(hash_hex, store_name=store_name) else: - client = hydrus_wrapper.get_client(config) + relationships = hydrus_provider.get_relationships(hash_hex) if hydrus_provider is not None else None def _resolve_related_title(rel_hash: str) -> str: """Best-effort resolve a Hydrus hash to a human title. @@ -154,22 +143,15 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: if not h: return str(rel_hash) - # Prefer backend tag extraction when available. - if backend_obj is not None and hasattr(backend_obj, "get_tag"): + # Prefer provider-backed title resolution when available. + if hydrus_provider is not None: try: - tag_result = backend_obj.get_tag(h) - tags = ( - tag_result[0] - if isinstance(tag_result, - tuple) and tag_result else tag_result + resolved_title = hydrus_provider.get_title( + h, + store_name=store_label if store_name else None, ) - if isinstance(tags, list): - for t in tags: - if isinstance(t, - str) and t.lower().startswith("title:"): - val = t.split(":", 1)[1].strip() - if val: - return val + if isinstance(resolved_title, str) and resolved_title.strip(): + return resolved_title.strip() except Exception: pass @@ -179,7 +161,6 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: config, h, store_name=store_label if store_name else None, - hydrus_client=client, include_service_keys_to_tags=True, include_file_url=False, include_duration=False, @@ -224,14 +205,12 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int: return h[:16] + "..." - if client: - rel = client.get_file_relationships(hash_hex) - if rel: - file_rels = rel.get("file_relationships", + if relationships: + file_rels = relationships.get("file_relationships", {}) - this_file_rels = file_rels.get(hash_hex) + this_file_rels = file_rels.get(hash_hex) - if this_file_rels: + if this_file_rels: # Map Hydrus relationship IDs to names. # For /manage_file_relationships/get_file_relationships, the Hydrus docs define: # 0=potential duplicates, 1=false positives, 3=alternates, 8=duplicates diff --git a/cmdnat/config.py b/cmdnat/config.py index ee65a87..c43b41b 100644 --- a/cmdnat/config.py +++ b/cmdnat/config.py @@ -1,3 +1,6 @@ +import datetime +import sqlite3 +from pathlib import Path from typing import List, Dict, Any, Optional, Sequence from SYS.cmdlet_spec import Cmdlet, CmdletArg @@ -7,17 +10,20 @@ from SYS.config import ( save_config_and_verify, set_nested_config_value, ) +from SYS.database import LOG_DB_PATH, db +from SYS.logger import log from SYS import pipeline as ctx from SYS.result_table import Table from cmdnat._parsing import ( extract_piped_value as _extract_piped_value, extract_value_arg as _extract_value_arg, + has_flag as _has_flag, ) CMDLET = Cmdlet( name=".config", summary="Manage configuration settings", - usage=".config [key] [value]", + usage=".config [key] [value] | .config -log [count]", arg=[ CmdletArg( name="key", @@ -33,6 +39,140 @@ CMDLET = Cmdlet( ) +def _extract_log_limit(args: Sequence[str], default: int = 30) -> int: + try: + tokens = [str(arg).strip() for arg in (args or []) if str(arg).strip()] + except Exception: + return default + + for idx, token in enumerate(tokens): + lowered = token.lower() + if lowered in {"-log", "--log"}: + if idx + 1 < len(tokens): + candidate = tokens[idx + 1] + if candidate and not candidate.startswith("-"): + try: + return max(1, min(200, int(candidate))) + except Exception: + return default + return default + if lowered.startswith("-log=") or lowered.startswith("--log="): + _, value = lowered.split("=", 1) + try: + return max(1, min(200, int(value))) + except Exception: + return default + return default + + +def _fallback_log_path() -> Path: + return Path(db.db_path).with_name("logs") / "log_fallback.txt" + + +def _load_recent_config_logs(limit: int = 30) -> List[Dict[str, str]]: + rows: List[Dict[str, str]] = [] + sql = """ + SELECT timestamp, level, module, message + FROM logs + WHERE lower(module) LIKE ? + OR lower(message) LIKE ? + OR lower(message) LIKE ? + OR lower(message) LIKE ? + ORDER BY id DESC + LIMIT ? + """ + params = ( + "%config%", + "%config%", + "%save failed%", + "%saving configuration failed%", + int(limit), + ) + + try: + with sqlite3.connect(str(LOG_DB_PATH), timeout=5.0) as conn: + conn.row_factory = sqlite3.Row + cur = conn.cursor() + cur.execute(sql, params) + fetched = cur.fetchall() + cur.close() + for row in fetched: + rows.append( + { + "timestamp": str(row["timestamp"] or ""), + "level": str(row["level"] or ""), + "module": str(row["module"] or ""), + "message": str(row["message"] or ""), + } + ) + except Exception: + rows = [] + + if rows: + return rows + + fallback = _fallback_log_path() + try: + if not fallback.exists(): + return [] + lines = fallback.read_text(encoding="utf-8", errors="replace").splitlines() + matches = [ + line for line in lines + if any(term in line.lower() for term in ("config", "save failed", "saving configuration failed")) + ] + for line in reversed(matches[-limit:]): + rows.append( + { + "timestamp": "", + "level": "FALLBACK", + "module": "fallback", + "message": line, + } + ) + except Exception: + return [] + return rows + + +def _format_log_timestamp_local(raw_value: str) -> str: + text = str(raw_value or "").strip() + if not text: + return "" + for pattern in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S.%f"): + try: + parsed = datetime.datetime.strptime(text, pattern).replace(tzinfo=datetime.timezone.utc) + return parsed.astimezone().strftime("%Y-%m-%d %H:%M:%S") + except Exception: + continue + return text + + +def _show_config_logs(args: Sequence[str]) -> int: + limit = _extract_log_limit(args) + rows = _load_recent_config_logs(limit=limit) + if not rows: + print( + f"No recent config/save logs found in {LOG_DB_PATH.name} or {_fallback_log_path().name}." + ) + return 0 + + table = Table("Configuration Logs") + table.set_table("config.logs") + table.set_source_command(".config", ["-log", str(limit)]) + + for row_data in rows: + row = table.add_row() + row.add_column("Time (local)", _format_log_timestamp_local(row_data.get("timestamp", ""))) + row.add_column("Level", row_data.get("level", "")) + row.add_column("Module", row_data.get("module", "")) + row.add_column("Message", row_data.get("message", "")) + + ctx.set_last_result_table_overlay(table, rows) + ctx.set_current_stage_table(table) + print(f"Showing {len(rows)} recent configuration log entries.") + return 0 + + def flatten_config(config: Dict[str, Any], parent_key: str = "", sep: str = ".") -> List[Dict[str, Any]]: items: List[Dict[str, Any]] = [] for k, v in config.items(): @@ -108,6 +248,9 @@ def _strip_value_quotes(value: str) -> str: def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int: import sys + if _has_flag(args, "-log") or _has_flag(args, "--log"): + return _show_config_logs(args) + # Load configuration from the database current_config = load_config() @@ -135,6 +278,7 @@ def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int: try: save_config_and_verify(current_config) except Exception as exc: + log(f"Configuration save verification failed for '{selection_key}': {exc}") print(f"Error saving configuration (verification failed): {exc}") return 1 else: @@ -142,6 +286,7 @@ def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int: print(f"Updated '{selection_key}' to '{new_value}'") return 0 except Exception as exc: + log(f"Error updating config '{selection_key}': {exc}") print(f"Error updating config: {exc}") return 1 @@ -201,6 +346,7 @@ def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int: print(f"Updated '{key}' to '{value}'") return 0 except Exception as exc: + log(f"Error updating config '{key}': {exc}") print(f"Error updating config: {exc}") return 1 diff --git a/docs/ftp_plugin_tutorial.md b/docs/ftp_plugin_tutorial.md new file mode 100644 index 0000000..3a393cd --- /dev/null +++ b/docs/ftp_plugin_tutorial.md @@ -0,0 +1,164 @@ +# FTP Plugin Walkthrough + +This walkthrough adds a real bundled `ftp` plugin so users can: + +- run `search-file -plugin ftp ...` +- browse remote folders as result tables +- select file rows to `download-file` +- pipe selected file rows into `add-file` +- upload local files with `add-file -plugin ftp` + +The implementation lives in [plugins/ftp/__init__.py](plugins/ftp/__init__.py). + +## What The Plugin Does + +The FTP plugin demonstrates the main provider hooks that matter for a storage-style integration: + +- `config_schema()` exposes host, credentials, base path, TLS, and search depth. +- `extract_query_arguments()` supports inline query fields like `path:` and `depth:`. +- `search()` walks an FTP directory tree and returns `SearchResult` rows. +- `selector()` turns folder rows into a follow-up table when the user runs `@N`. +- `download()` and `download_url()` fetch FTP files into `download-file` output paths. +- `resolve_pipe_result_download()` lets `@N | add-file -store ...` materialize a remote FTP file first. +- `upload()` lets `add-file -plugin ftp -path ...` push a local file to the configured FTP server. + +## Example Config + +Add an FTP provider block to your config: + +```toml +[provider.ftp] +host = "ftp.example.com" +port = 21 +username = "demo" +password = "secret" +base_path = "/incoming" +tls = false +passive = true +timeout = 20 +search_depth = 1 +``` + +Notes: + +- `host` is the only required field for the plugin to validate. +- `username` defaults to `anonymous` and `password` defaults to `anonymous@`. +- `base_path` is both the default search root and the upload target directory. +- `search_depth` controls how many folder levels `search-file -plugin ftp` scans by default. + +## Search Flow + +Basic listing from the configured base path: + +```powershell +search-file -plugin ftp "*" +``` + +Search by filename fragment: + +```powershell +search-file -plugin ftp "invoice" +``` + +Search a different subtree and recurse deeper: + +```powershell +search-file -plugin ftp "path:/pub depth:2 invoice" +``` + +Filter to folders only: + +```powershell +search-file -plugin ftp "path:/pub type:folder *" +``` + +The plugin returns rows with explicit columns for name, type, directory, size, and modification time. + +## Selection Flow + +Folder rows are navigation rows. If the selected row is a directory, plain `@N` opens a new FTP table for that directory: + +```powershell +search-file -plugin ftp "*" +@2 +``` + +File rows carry an explicit row action: + +```powershell +download-file -plugin ftp -url ftp://ftp.example.com/incoming/report.pdf +``` + +That means plain `@N` on a file row downloads it immediately: + +```powershell +search-file -plugin ftp "report" +@1 +``` + +## Download And Add-File Flow + +If you want the downloaded file in a specific local directory: + +```powershell +search-file -plugin ftp "report" +@1 | download-file -path C:\Downloads +``` + +If you want to ingest the selected FTP file into a configured store backend: + +```powershell +search-file -plugin ftp "report" +@1 | add-file -store tutorial +``` + +Why this works: + +- the file row advertises a `download-file` row action +- the pipeline auto-inserts that download before `add-file` +- the FTP plugin also implements `resolve_pipe_result_download()` so provider-owned FTP rows can be materialized for ingestion + +## Upload Flow + +Uploading uses the same provider name, but through `add-file -plugin ftp`: + +```powershell +add-file -plugin ftp -path C:\Media\report.pdf +``` + +That sends the file to the configured FTP `base_path` and returns the FTP URL as the uploaded result. + +## Why The Row Metadata Matters + +The critical part of this plugin is the file-row metadata: + +- file rows emit `_selection_args` as `['-url', '']` +- file rows emit `_selection_action` as `['download-file', '-plugin', 'ftp', '-url', '']` +- folder rows do not emit a download action, so `selector()` can own drill-in behavior instead + +That split is what keeps these two user experiences compatible: + +- `@N` on a folder opens a new table +- `@N` on a file downloads the file +- `@N | add-file -store ...` first downloads, then ingests + +## Implementation Notes + +The plugin prefers `MLSD` for directory listings and falls back to `NLST` plus directory probes when the server does not support machine-readable listings. + +The code is intentionally small and uses only Python stdlib pieces: + +- `ftplib` for FTP and FTPS +- `fnmatch` for wildcard-style search tokens +- `tempfile` for `add-file` handoff downloads + +## Recommended Commands To Demo The Walkthrough + +```powershell +search-file -plugin ftp "*" +search-file -plugin ftp "path:/incoming depth:2 *.pdf" +@1 +@1 | download-file -path C:\Downloads +@1 | add-file -store tutorial +add-file -plugin ftp -path C:\Media\report.pdf +``` \ No newline at end of file diff --git a/docs/scp_plugin_tutorial.md b/docs/scp_plugin_tutorial.md new file mode 100644 index 0000000..87758a3 --- /dev/null +++ b/docs/scp_plugin_tutorial.md @@ -0,0 +1,136 @@ +# SCP Plugin Walkthrough + +This walkthrough adds a bundled `scp` plugin backed by existing SSH libraries: + +- `paramiko` for SSH and SFTP directory listing +- `scp` for file transfers + +The implementation lives in [plugins/scp/__init__.py](plugins/scp/__init__.py). + +## What The Plugin Does + +The SCP plugin mirrors the FTP walkthrough, but on top of SSH: + +- `search-file -plugin scp ...` lists remote files and folders over SFTP. +- plain `@N` on a folder drills into that directory. +- plain `@N` on a file runs `download-file -plugin scp -url ...`. +- `@N | add-file -store ...` downloads first, then ingests the local temp file. +- `add-file -plugin scp -path ...` uploads a local file to the configured remote path. + +## Example Config + +```toml +[provider.scp] +host = "ssh.example.com" +port = 22 +username = "deploy" +password = "secret" +key_path = "C:/Users/Admin/.ssh/id_ed25519" +base_path = "/srv/files" +timeout = 20 +search_depth = 1 +allow_agent = true +look_for_keys = true +``` + +Notes: + +- `host` and `username` are required for the plugin to validate. +- You can use password auth, key auth, or both. +- `base_path` is both the default search root and the default upload directory. + +## Search Flow + +List the configured base path: + +```powershell +search-file -plugin scp "*" +``` + +Search by filename: + +```powershell +search-file -plugin scp "invoice" +``` + +Search another subtree with deeper recursion: + +```powershell +search-file -plugin scp "path:/srv/files/releases depth:2 *.zip" +``` + +Show only folders: + +```powershell +search-file -plugin scp "path:/srv/files type:folder *" +``` + +## Selection Flow + +Folder rows are navigation rows: + +```powershell +search-file -plugin scp "*" +@2 +``` + +File rows carry an explicit row action, so terminal selection downloads directly: + +```powershell +search-file -plugin scp "report" +@1 +``` + +That expands to the equivalent of: + +```powershell +download-file -plugin scp -url scp://ssh.example.com/srv/files/report.pdf +``` + +## Download And Add-File Flow + +Download into a local folder: + +```powershell +search-file -plugin scp "report" +@1 | download-file -path C:\Downloads +``` + +Ingest a selected remote file into a configured store backend: + +```powershell +search-file -plugin scp "report" +@1 | add-file -store tutorial +``` + +Why this works: + +- file rows advertise `_selection_action` for `download-file` +- `add-file` selection replay inserts that provider download stage before ingest +- the plugin also implements `resolve_pipe_result_download()` for provider-owned SCP rows + +## Upload Flow + +Upload a local file to the configured remote `base_path`: + +```powershell +add-file -plugin scp -path C:\Media\report.pdf +``` + +## Implementation Notes + +The plugin uses SFTP for directory listing because SCP itself is a transfer protocol, not a browse/search protocol. That split keeps the provider simple: + +- browse and metadata via Paramiko SFTP +- file transfer via the `scp` package + +## Recommended Demo Commands + +```powershell +search-file -plugin scp "*" +search-file -plugin scp "path:/srv/files depth:2 *.zip" +@1 +@1 | download-file -path C:\Downloads +@1 | add-file -store tutorial +add-file -plugin scp -path C:\Media\report.pdf +``` \ No newline at end of file diff --git a/plugins/README.md b/plugins/README.md index 83513e7..be88913 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -48,4 +48,12 @@ class MyPlugin(Provider): path=f"https://example.com/{text}", ) ] -``` \ No newline at end of file +``` + +Bundled walkthrough: + +- The repo now includes a real FTP example plugin in [plugins/ftp/__init__.py](plugins/ftp/__init__.py). +- The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp`, folder drill-in via `@N`, file download routing, `@N | add-file -store ...`, and `add-file -plugin ftp` uploads. +- The repo also includes an SCP example plugin in [plugins/scp/__init__.py](plugins/scp/__init__.py). +- The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp`, SSH-backed directory drill-in, file download routing, `@N | add-file -store ...`, and `add-file -plugin scp` uploads. +- The repo now also includes a built-in HydrusNetwork provider in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). It delegates to configured `store.hydrusnetwork.*` backends so Hydrus features can be reached through the normal plugin registry without cmdlets importing Hydrus modules directly. \ No newline at end of file diff --git a/plugins/alldebrid/__init__.py b/plugins/alldebrid/__init__.py index e147995..37aad1c 100644 --- a/plugins/alldebrid/__init__.py +++ b/plugins/alldebrid/__init__.py @@ -23,24 +23,56 @@ from SYS.models import DownloadError, PipeObject _HOSTS_CACHE_TTL_SECONDS = 24 * 60 * 60 -def _repo_root() -> Path: +def _plugin_dir() -> Path: try: - return Path(__file__).resolve().parents[1] + return Path(__file__).resolve().parent except Exception: return Path(".") +def _legacy_hosts_cache_paths() -> Tuple[Path, ...]: + try: + repo_root = Path(__file__).resolve().parents[2] + plugins_root = Path(__file__).resolve().parents[1] + except Exception: + return tuple() + return ( + plugins_root / "API" / "data" / "alldebrid.json", + repo_root / "API" / "data" / "alldebrid.json", + ) + + def _hosts_cache_path() -> Path: - # Keep this local to the repo so it works in portable installs. - # The registry's URL routing can read this file without instantiating providers. + # Keep this local to the plugin so plugin-specific cache/state stays bundled + # with the plugin itself in portable installs. # # This file is expected to be the JSON payload shape from AllDebrid: # {"status":"success","data":{"hosts":[...],"streams":[...],"redirectors":[...]}} - return _repo_root() / "API" / "data" / "alldebrid.json" + return _plugin_dir() / "alldebrid.json" + + +def _resolve_hosts_cache_path() -> Path: + path = _hosts_cache_path() + try: + if path.exists() and path.is_file(): + return path + except Exception: + return path + + for legacy in _legacy_hosts_cache_paths(): + try: + if not legacy.exists() or not legacy.is_file(): + continue + path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(legacy, path) + return path + except Exception: + continue + return path def _load_cached_domains(category: str) -> List[str]: - """Load cached domain list from API/data/alldebrid.json. + """Load cached domain list from the plugin-local alldebrid.json cache. category: "hosts" | "streams" | "redirectors" """ @@ -49,7 +81,7 @@ def _load_cached_domains(category: str) -> List[str]: if wanted not in {"hosts", "streams", "redirectors"}: return [] - path = _hosts_cache_path() + path = _resolve_hosts_cache_path() try: if not path.exists() or not path.is_file(): return [] @@ -68,12 +100,27 @@ def _load_cached_domains(category: str) -> List[str]: return [] raw_list = data.get(wanted) - if not isinstance(raw_list, list): + if not isinstance(raw_list, (list, dict)): return [] out: List[str] = [] seen: set[str] = set() - for d in raw_list: + + domain_candidates: List[Any] = [] + if isinstance(raw_list, list): + domain_candidates.extend(raw_list) + else: + for entry in raw_list.values(): + if isinstance(entry, dict): + nested_domains = entry.get("domains") + if isinstance(nested_domains, list): + domain_candidates.extend(nested_domains) + elif isinstance(nested_domains, str): + domain_candidates.append(nested_domains) + elif isinstance(entry, str): + domain_candidates.append(entry) + + for d in domain_candidates: try: dom = str(d or "").strip().lower() except Exception: @@ -115,7 +162,7 @@ def _save_cached_hosts_payload(payload: Dict[str, Any]) -> None: def _cache_is_fresh() -> bool: - path = _hosts_cache_path() + path = _resolve_hosts_cache_path() try: if not path.exists() or not path.is_file(): return False diff --git a/API/data/alldebrid.json b/plugins/alldebrid/alldebrid.json similarity index 99% rename from API/data/alldebrid.json rename to plugins/alldebrid/alldebrid.json index ca709f8..7926790 100644 --- a/API/data/alldebrid.json +++ b/plugins/alldebrid/alldebrid.json @@ -37,7 +37,7 @@ "(rapidgator\\.net/file/[0-9]{7,8})" ], "regexp": "((rapidgator\\.net|rg\\.to|rapidgator\\.asia)/file/([0-9a-zA-Z]{32}))|((rapidgator\\.net/file/[0-9]{7,8}))", - "status": true + "status": false }, "turbobit": { "name": "turbobit", @@ -71,7 +71,7 @@ "(wayupload\\.com/[a-z0-9]{12}\\.html)" ], "regexp": "(turbobit5?a?\\.(net|cc|com)/([a-z0-9]{12}))|(turbobif\\.(net|cc|com)/([a-z0-9]{12}))|(turb[o]?\\.(to|cc|pw)\\/([a-z0-9]{12}))|(turbobit\\.(net|cc)/download/free/([a-z0-9]{12}))|((trbbt|tourbobit|torbobit|tbit|turbobita|trbt)\\.(net|cc|com|to)/([a-z0-9]{12}))|((turbobit\\.cloud/turbo/[a-z0-9]+))|((wayupload\\.com/[a-z0-9]{12}\\.html))", - "status": false + "status": true }, "hitfile": { "name": "hitfile", @@ -375,7 +375,7 @@ "(filespace\\.com/[a-zA-Z0-9]{12})" ], "regexp": "(filespace\\.com/fd/([a-zA-Z0-9]{12}))|((filespace\\.com/[a-zA-Z0-9]{12}))", - "status": false + "status": true }, "filezip": { "name": "filezip", diff --git a/plugins/ftp/__init__.py b/plugins/ftp/__init__.py new file mode 100644 index 0000000..fe61fe8 --- /dev/null +++ b/plugins/ftp/__init__.py @@ -0,0 +1,778 @@ +from __future__ import annotations + +import fnmatch +import ftplib +import posixpath +import tempfile +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import quote, unquote, urlparse + +from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments + + +def _pick_provider_config(config: Any) -> Dict[str, Any]: + if not isinstance(config, dict): + return {} + provider = config.get("provider") + if not isinstance(provider, dict): + return {} + entry = provider.get("ftp") + if isinstance(entry, dict): + return entry + return {} + + +def _coerce_bool(value: Any, default: bool = False) -> bool: + if isinstance(value, bool): + return value + if value is None: + return default + text = str(value).strip().lower() + if not text: + return default + if text in {"1", "true", "yes", "on"}: + return True + if text in {"0", "false", "no", "off"}: + return False + return default + + +def _coerce_int(value: Any, default: int) -> int: + try: + return int(value) + except Exception: + return default + + +def _format_timestamp(raw_value: Any) -> str: + text = str(raw_value or "").strip() + if not text: + return "" + for pattern in ("%Y%m%d%H%M%S", "%Y%m%d%H%M%S.%f"): + try: + parsed = datetime.strptime(text, pattern) + return parsed.strftime("%Y-%m-%d %H:%M") + except Exception: + continue + return text + + +def _safe_filename(name: Any) -> str: + raw = str(name or "").strip() + if not raw: + raw = "download" + cleaned = "".join(ch if ch.isalnum() or ch in {"-", "_", ".", " "} else "_" for ch in raw) + cleaned = cleaned.strip(" ._") + return cleaned or "download" + + +def _unique_path(path: Path) -> Path: + if not path.exists(): + return path + stem = path.stem or "download" + suffix = path.suffix + counter = 1 + while True: + candidate = path.with_name(f"{stem}_{counter}{suffix}") + if not candidate.exists(): + return candidate + counter += 1 + + +class FTP(Provider): + PLUGIN_NAME = "ftp" + URL = ("ftp://", "ftps://") + + @property + def label(self) -> str: + return "FTP" + + @property + def preserve_order(self) -> bool: + return True + + @classmethod + def config_schema(cls) -> List[Dict[str, Any]]: + return [ + { + "key": "host", + "label": "Host", + "default": "", + "required": True, + "placeholder": "ftp.example.com", + }, + { + "key": "port", + "label": "Port", + "type": "integer", + "default": 21, + }, + { + "key": "username", + "label": "Username", + "default": "anonymous", + }, + { + "key": "password", + "label": "Password", + "type": "secret", + "secret": True, + "default": "", + }, + { + "key": "base_path", + "label": "Base Path", + "default": "/", + "placeholder": "/incoming", + }, + { + "key": "tls", + "label": "Use FTPS", + "type": "boolean", + "default": False, + }, + { + "key": "passive", + "label": "Passive Mode", + "type": "boolean", + "default": True, + }, + { + "key": "timeout", + "label": "Timeout Seconds", + "type": "integer", + "default": 20, + }, + { + "key": "search_depth", + "label": "Default Search Depth", + "type": "integer", + "default": 1, + }, + ] + + def __init__(self, config: Optional[Dict[str, Any]] = None): + super().__init__(config) + conf = _pick_provider_config(self.config) + self._host = str(conf.get("host") or "").strip() + self._tls = _coerce_bool(conf.get("tls"), False) + self._port = _coerce_int(conf.get("port"), 21) + self._username = str(conf.get("username") or conf.get("user") or "anonymous").strip() or "anonymous" + password_value = conf.get("password") + self._password = str(password_value).strip() if password_value not in (None, "") else "anonymous@" + self._passive = _coerce_bool(conf.get("passive"), True) + self._timeout = max(1, _coerce_int(conf.get("timeout"), 20)) + self._search_depth = max(0, _coerce_int(conf.get("search_depth"), 1)) + self._base_path = self._normalize_remote_path(conf.get("base_path") or "/", default="/") + + def validate(self) -> bool: + return bool(self._host) + + def config_helper_text(self) -> str: + return "Test the configured FTP/FTPS settings before searching or uploading." + + def config_actions(self) -> List[Dict[str, Any]]: + return [ + { + "id": "test_connection", + "label": "Test connection", + "variant": "primary", + } + ] + + def run_config_action(self, action_id: str, **_kwargs: Any) -> Dict[str, Any]: + if str(action_id or "").strip().lower() != "test_connection": + return super().run_config_action(action_id, **_kwargs) + + if not self._host: + return {"ok": False, "message": "Set 'host' before testing the FTP connection."} + + ftp = None + try: + ftp = self._connect() + active_path = self._base_path or "/" + try: + ftp.cwd(active_path) + resolved_path = ftp.pwd() + except Exception: + resolved_path = active_path + return { + "ok": True, + "message": f"Connected to FTP {self._host}:{self._port} and reached {resolved_path}.", + } + except Exception as exc: + return {"ok": False, "message": f"FTP connection failed: {exc}"} + finally: + self._close(ftp) + + def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]: + text, inline = parse_inline_query_arguments(query) + filters: Dict[str, Any] = {} + + if inline.get("path"): + filters["path"] = inline.get("path") + if inline.get("depth"): + filters["depth"] = max(0, _coerce_int(inline.get("depth"), self._search_depth)) + if inline.get("type"): + filters["type"] = str(inline.get("type") or "").strip().lower() + + return text, filters + + def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str: + active_path = self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path) + text = str(query or "").strip() + if not text or text == "*": + return f"FTP: {active_path}" + return f"FTP: {text} @ {active_path}" + + def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + return { + "plugin": self.name, + "host": self._host, + "path": self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path), + "query": str(query or "").strip(), + } + + def search( + self, + query: str, + limit: int = 50, + filters: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> List[SearchResult]: + _ = kwargs + active_filters = dict(filters or {}) + start_path = self._normalize_remote_path(active_filters.get("path") or self._base_path, default=self._base_path) + search_depth = max(0, _coerce_int(active_filters.get("depth"), self._search_depth)) + type_filter = str(active_filters.get("type") or "any").strip().lower() + needle = str(query or "").strip() + max_results = max(0, int(limit or 0)) + if max_results <= 0: + return [] + + ftp = self._connect() + try: + return self._search_directory( + ftp, + start_path, + needle=needle, + limit=max_results, + search_depth=search_depth, + type_filter=type_filter, + ) + finally: + self._close(ftp) + + def selector( + self, + selected_items: List[Any], + *, + ctx: Any, + stage_is_last: bool = True, + **_kwargs: Any, + ) -> bool: + if not stage_is_last: + return False + + target_path = "" + target_title = "" + for item in selected_items or []: + metadata = self._item_metadata(item) + if not metadata.get("is_dir"): + continue + target_path = self._normalize_remote_path(metadata.get("ftp_path") or metadata.get("selection_path"), default=self._base_path) + target_title = str(metadata.get("title") or metadata.get("name") or "").strip() + if target_path: + break + + if not target_path: + return False + + ftp = self._connect() + try: + rows = self._search_directory( + ftp, + target_path, + needle="*", + limit=500, + search_depth=0, + type_filter="any", + ) + finally: + self._close(ftp) + + try: + from SYS.result_table import Table + from SYS.rich_display import stdout_console + except Exception: + return True + + title = target_title or target_path + table = Table(f"FTP: {title}")._perseverance(True) + table.set_table("ftp") + try: + table.set_table_metadata({ + "provider": "ftp", + "host": self._host, + "path": target_path, + "view": "directory", + }) + except Exception: + pass + table.set_source_command("search-file", ["-plugin", "ftp", f"path:{target_path}", "*"]) + + payloads: List[Dict[str, Any]] = [] + for row in rows: + table.add_result(row) + payloads.append(row.to_dict()) + + try: + ctx.set_last_result_table(table, payloads, subject={"plugin": "ftp", "path": target_path}) + ctx.set_current_stage_table(table) + except Exception: + pass + + try: + stdout_console().print() + stdout_console().print(table) + except Exception: + pass + + return True + + def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]: + metadata = getattr(result, "full_metadata", None) + if isinstance(metadata, dict) and metadata.get("is_dir"): + return None + target = str(getattr(result, "path", "") or "").strip() + if not target: + return None + return self.download_url(target, output_dir, title=getattr(result, "title", None)) + + def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]: + settings = self._connection_settings_for_url(url) + remote_path = settings["path"] + if not remote_path or remote_path == "/": + return None + + filename_hint = str(kwargs.get("title") or "").strip() + parsed_name = posixpath.basename(remote_path.rstrip("/")) + filename = _safe_filename(filename_hint or unquote(parsed_name) or "download") + + destination_dir = Path(output_dir) + destination_dir.mkdir(parents=True, exist_ok=True) + destination = _unique_path(destination_dir / filename) + + ftp = self._connect( + host=settings["host"], + port=settings["port"], + username=settings["username"], + password=settings["password"], + tls=settings["tls"], + ) + try: + with destination.open("wb") as handle: + ftp.retrbinary(f"RETR {remote_path}", handle.write) + return destination + except Exception: + try: + destination.unlink(missing_ok=True) + except Exception: + pass + return None + finally: + self._close(ftp) + + def resolve_pipe_result_download( + self, + result: Any, + pipe_obj: Any, + ) -> Tuple[Optional[Path], Optional[str], Optional[Path]]: + metadata = self._item_metadata(result, pipe_obj=pipe_obj) + if metadata.get("is_dir"): + return None, None, None + + download_url = str( + metadata.get("selection_url") + or metadata.get("ftp_url") + or metadata.get("path") + or "" + ).strip() + if not download_url.startswith(("ftp://", "ftps://")): + return None, None, None + + temp_dir = Path(tempfile.mkdtemp(prefix="ftp-add-file-")) + downloaded = self.download_url(download_url, temp_dir, title=metadata.get("title")) + if downloaded is None: + try: + temp_dir.rmdir() + except Exception: + pass + return None, None, None + + try: + if pipe_obj is not None: + pipe_obj.is_temp = True + except Exception: + pass + return downloaded, None, temp_dir + + def upload(self, file_path: str, **kwargs: Any) -> str: + local_path = Path(str(file_path or "")).expanduser() + if not local_path.exists() or not local_path.is_file(): + raise FileNotFoundError(f"File not found: {local_path}") + + remote_dir = self._normalize_remote_path(kwargs.get("remote_path") or kwargs.get("path") or self._base_path, default=self._base_path) + remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name + remote_path = self._join_remote_path(remote_dir, remote_name) + + ftp = self._connect() + try: + self._ensure_directory(ftp, remote_dir) + with local_path.open("rb") as handle: + ftp.storbinary(f"STOR {remote_path}", handle) + finally: + self._close(ftp) + + return self._build_url(remote_path) + + def _connect( + self, + *, + host: Optional[str] = None, + port: Optional[int] = None, + username: Optional[str] = None, + password: Optional[str] = None, + tls: Optional[bool] = None, + ) -> ftplib.FTP: + use_tls = self._tls if tls is None else bool(tls) + ftp: ftplib.FTP = ftplib.FTP_TLS() if use_tls else ftplib.FTP() + ftp.connect(host or self._host, int(port or self._port), timeout=self._timeout) + ftp.login(username or self._username, password or self._password) + try: + ftp.set_pasv(self._passive) + except Exception: + pass + if use_tls and isinstance(ftp, ftplib.FTP_TLS): + ftp.prot_p() + return ftp + + def _close(self, ftp: Optional[ftplib.FTP]) -> None: + if ftp is None: + return + try: + ftp.quit() + except Exception: + try: + ftp.close() + except Exception: + pass + + def _normalize_remote_path(self, value: Any, *, default: str) -> str: + text = str(value or "").strip().replace("\\", "/") + if not text: + text = default + elif text.startswith(("ftp://", "ftps://")): + try: + text = unquote(urlparse(text).path or "/") + except Exception: + text = default + elif not text.startswith("/"): + text = posixpath.join(default, text) + + normalized = posixpath.normpath(text) + normalized = "/" + normalized.lstrip("/") + return normalized or "/" + + def _join_remote_path(self, parent: Any, child: Any) -> str: + left = self._normalize_remote_path(parent, default=self._base_path) + right = str(child or "").strip().replace("\\", "/") + if not right: + return left + return self._normalize_remote_path(posixpath.join(left, right), default="/") + + def _build_url( + self, + remote_path: Any, + *, + host: Optional[str] = None, + port: Optional[int] = None, + tls: Optional[bool] = None, + ) -> str: + path_text = self._normalize_remote_path(remote_path, default="/") + scheme = "ftps" if (self._tls if tls is None else bool(tls)) else "ftp" + host_text = str(host or self._host).strip() + port_value = int(port or self._port) + port_suffix = f":{port_value}" if port_value and port_value != 21 else "" + return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}" + + def _connection_settings_for_url(self, url: str) -> Dict[str, Any]: + parsed = urlparse(str(url or "").strip()) + scheme = (parsed.scheme or "ftp").strip().lower() + host = parsed.hostname or self._host + port = parsed.port or self._port + username = parsed.username or self._username + password = parsed.password or self._password + path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default="/") + return { + "tls": scheme == "ftps", + "host": host, + "port": port, + "username": username, + "password": password, + "path": path_text, + } + + def _search_directory( + self, + ftp: ftplib.FTP, + start_path: str, + *, + needle: str, + limit: int, + search_depth: int, + type_filter: str, + ) -> List[SearchResult]: + results: List[SearchResult] = [] + visited: set[str] = set() + + def walk(current_path: str, depth_left: int) -> None: + normalized = self._normalize_remote_path(current_path, default=self._base_path) + if normalized in visited or len(results) >= limit: + return + visited.add(normalized) + + for entry in self._list_directory(ftp, normalized): + if len(results) >= limit: + return + if self._matches_entry(entry, needle=needle, type_filter=type_filter): + results.append(self._build_result(entry)) + if entry.get("is_dir") and depth_left > 0: + walk(str(entry.get("ftp_path") or normalized), depth_left - 1) + + walk(start_path, max(0, search_depth)) + return results + + def _matches_entry(self, entry: Dict[str, Any], *, needle: str, type_filter: str) -> bool: + is_dir = bool(entry.get("is_dir")) + if type_filter in {"dir", "dirs", "folder", "folders"} and not is_dir: + return False + if type_filter in {"file", "files"} and is_dir: + return False + + text = str(needle or "").strip().lower() + if not text or text in {"*", "all", "list"}: + return True + + haystacks = [ + str(entry.get("name") or "").lower(), + str(entry.get("ftp_path") or "").lower(), + ] + for token in [part for part in text.split() if part]: + if any(ch in token for ch in "*?[]"): + if not any(fnmatch.fnmatch(haystack, token) for haystack in haystacks): + return False + elif not any(token in haystack for haystack in haystacks): + return False + return True + + def _build_result(self, entry: Dict[str, Any]) -> SearchResult: + ftp_path = str(entry.get("ftp_path") or "/") + ftp_url = self._build_url(ftp_path) + is_dir = bool(entry.get("is_dir")) + size_value = entry.get("size") + modified = str(entry.get("modified") or "") + parent = posixpath.dirname(ftp_path.rstrip("/")) or "/" + metadata = { + "provider": "ftp", + "host": self._host, + "ftp_path": ftp_path, + "ftp_url": ftp_url, + "selection_url": ftp_url, + "is_dir": is_dir, + "name": str(entry.get("name") or "").strip(), + } + if size_value is not None: + metadata["size"] = size_value + if modified: + metadata["modified"] = modified + + return SearchResult( + table="ftp", + title=str(entry.get("name") or ftp_path), + path=ftp_url, + detail=parent, + annotations=["folder" if is_dir else "file"], + media_kind="folder" if is_dir else "file", + size_bytes=int(size_value) if isinstance(size_value, int) else None, + tag={"ftp", "folder" if is_dir else "file"}, + columns=[ + ("Name", str(entry.get("name") or "")), + ("Type", "dir" if is_dir else "file"), + ("Directory", parent), + ("Size", "" if size_value is None else str(size_value)), + ("Modified", modified), + ], + selection_args=None if is_dir else ["-url", ftp_url], + selection_action=None if is_dir else ["download-file", "-plugin", "ftp", "-url", ftp_url], + full_metadata=metadata, + ) + + def _list_directory(self, ftp: ftplib.FTP, remote_path: str) -> List[Dict[str, Any]]: + normalized = self._normalize_remote_path(remote_path, default=self._base_path) + try: + entries: List[Dict[str, Any]] = [] + for name, facts in ftp.mlsd(normalized): + name_text = str(name or "").strip() + if not name_text or name_text in {".", ".."}: + continue + entry_type = str((facts or {}).get("type") or "").strip().lower() + if entry_type in {"cdir", "pdir"}: + continue + size_value = None + raw_size = (facts or {}).get("size") + if raw_size not in (None, ""): + try: + size_value = int(raw_size) + except Exception: + size_value = None + entries.append( + { + "name": name_text, + "ftp_path": self._join_remote_path(normalized, name_text), + "is_dir": entry_type == "dir", + "size": size_value, + "modified": _format_timestamp((facts or {}).get("modify")), + } + ) + return entries + except Exception: + return self._list_directory_fallback(ftp, normalized) + + def _list_directory_fallback(self, ftp: ftplib.FTP, remote_path: str) -> List[Dict[str, Any]]: + try: + listed = ftp.nlst(remote_path) + except Exception: + return [] + + entries: List[Dict[str, Any]] = [] + seen: set[str] = set() + for raw_entry in listed: + entry_text = str(raw_entry or "").strip() + if not entry_text: + continue + entry_path = entry_text if entry_text.startswith("/") else self._join_remote_path(remote_path, entry_text) + name_text = posixpath.basename(entry_path.rstrip("/")) or entry_path.rstrip("/") + if not name_text or name_text in {".", ".."} or name_text in seen: + continue + seen.add(name_text) + is_dir = self._is_directory(ftp, entry_path) + size_value = None + if not is_dir: + try: + size_raw = ftp.size(entry_path) + if size_raw is not None: + size_value = int(size_raw) + except Exception: + size_value = None + entries.append( + { + "name": name_text, + "ftp_path": entry_path, + "is_dir": is_dir, + "size": size_value, + "modified": self._read_modified(ftp, entry_path), + } + ) + return entries + + def _is_directory(self, ftp: ftplib.FTP, remote_path: str) -> bool: + current = None + try: + current = ftp.pwd() + except Exception: + current = None + try: + ftp.cwd(remote_path) + return True + except Exception: + return False + finally: + if current is not None: + try: + ftp.cwd(current) + except Exception: + pass + + def _read_modified(self, ftp: ftplib.FTP, remote_path: str) -> str: + try: + response = ftp.sendcmd(f"MDTM {remote_path}") + except Exception: + return "" + parts = str(response or "").split() + if len(parts) >= 2: + return _format_timestamp(parts[1]) + return "" + + def _ensure_directory(self, ftp: ftplib.FTP, remote_path: str) -> None: + normalized = self._normalize_remote_path(remote_path, default=self._base_path) + if normalized == "/": + return + partial = "" + for segment in [part for part in normalized.split("/") if part]: + partial = f"{partial}/{segment}" + if self._is_directory(ftp, partial): + continue + try: + ftp.mkd(partial) + except Exception: + if not self._is_directory(ftp, partial): + raise + + def _item_metadata(self, item: Any, *, pipe_obj: Any = None) -> Dict[str, Any]: + metadata: Dict[str, Any] = {} + for source in (item, pipe_obj): + if isinstance(source, dict): + for key in ("title", "path", "url"): + if source.get(key) is not None and key not in metadata: + metadata[key] = source.get(key) + nested = source.get("full_metadata") or source.get("metadata") + if isinstance(nested, dict): + metadata.update(nested) + elif source is not None: + for attr in ("title", "path", "url"): + try: + value = getattr(source, attr, None) + except Exception: + value = None + if value is not None and attr not in metadata: + metadata[attr] = value + try: + nested = getattr(source, "full_metadata", None) or getattr(source, "metadata", None) + except Exception: + nested = None + if isinstance(nested, dict): + metadata.update(nested) + + ftp_path = metadata.get("ftp_path") or metadata.get("selection_path") + if not ftp_path: + path_value = metadata.get("path") or metadata.get("url") or metadata.get("ftp_url") + path_text = str(path_value or "").strip() + if path_text.startswith(("ftp://", "ftps://")): + ftp_path = self._normalize_remote_path(path_text, default=self._base_path) + if ftp_path: + metadata["ftp_path"] = self._normalize_remote_path(ftp_path, default=self._base_path) + metadata.setdefault("selection_path", metadata["ftp_path"]) + + if metadata.get("ftp_path") and not metadata.get("ftp_url"): + metadata["ftp_url"] = self._build_url(metadata["ftp_path"]) + if metadata.get("ftp_url") and not metadata.get("selection_url"): + metadata["selection_url"] = metadata["ftp_url"] + + is_dir = metadata.get("is_dir") + if is_dir is None and metadata.get("media_kind"): + is_dir = str(metadata.get("media_kind") or "").strip().lower() == "folder" + metadata["is_dir"] = bool(is_dir) + return metadata \ No newline at end of file diff --git a/plugins/openlibrary/__init__.py b/plugins/openlibrary/__init__.py index 0dda51e..84c93f0 100644 --- a/plugins/openlibrary/__init__.py +++ b/plugins/openlibrary/__init__.py @@ -29,11 +29,63 @@ from plugins.metadata_provider import ( from SYS.utils import unique_path _DEFAULT_ARCHIVE_SCALE = 4 +_DEFAULT_PREFERRED_LANGUAGE = "eng" _QUALITY_TO_ARCHIVE_SCALE = { "high": 2, "medium": 5, "low": 8, } +_LANGUAGE_NAME_TO_CODE = { + "english": "eng", + "eng": "eng", + "en": "eng", + "spanish": "spa", + "spa": "spa", + "es": "spa", + "french": "fre", + "fre": "fre", + "fra": "fre", + "fr": "fre", + "german": "ger", + "ger": "ger", + "deu": "ger", + "de": "ger", + "italian": "ita", + "ita": "ita", + "it": "ita", + "portuguese": "por", + "por": "por", + "pt": "por", + "polish": "pol", + "pol": "pol", + "pl": "pol", + "russian": "rus", + "rus": "rus", + "ru": "rus", + "chinese": "chi", + "chi": "chi", + "zho": "chi", + "zh": "chi", + "japanese": "jpn", + "jpn": "jpn", + "ja": "jpn", +} +_LANGUAGE_CODE_TO_NAME = { + "arm": "Armenian", + "chi": "Chinese", + "eng": "English", + "fre": "French", + "spa": "Spanish", + "ger": "German", + "ice": "Icelandic", + "ita": "Italian", + "jpn": "Japanese", + "kor": "Korean", + "por": "Portuguese", + "pol": "Polish", + "rus": "Russian", + "swe": "Swedish", +} def _create_archive_session() -> requests.Session: @@ -120,24 +172,124 @@ def _first_str(value: Any) -> Optional[str]: def _resolve_edition_id(doc: Dict[str, Any]) -> str: - # OpenLibrary Search API typically provides edition_key: ["OL...M", ...] + candidate_ids = _resolve_candidate_edition_ids(doc) + return candidate_ids[0] if candidate_ids else "" + + +def _resolve_candidate_edition_ids(doc: Dict[str, Any]) -> List[str]: + out: List[str] = [] + + def _add(value: Any) -> None: + text = str(value or "").strip() + if text and text not in out: + out.append(text) + + _add(doc.get("lending_edition_s")) + edition_key = doc.get("edition_key") - if isinstance(edition_key, list) and edition_key: - return str(edition_key[0]).strip() - if isinstance(edition_key, str) and edition_key.strip(): - return edition_key.strip() + if isinstance(edition_key, list): + for value in edition_key: + _add(value) + elif isinstance(edition_key, str): + _add(edition_key) - # Often present even when edition_key is missing. - cover_edition_key = doc.get("cover_edition_key") - if isinstance(cover_edition_key, str) and cover_edition_key.strip(): - return cover_edition_key.strip() + _add(doc.get("cover_edition_key")) + _add(doc.get("openlibrary_id")) - # Fallback: sometimes key can be /books/OL...M key = doc.get("key") if isinstance(key, str) and key.startswith("/books/"): - return key.split("/books/", 1)[1].strip("/") + _add(key.split("/books/", 1)[1].strip("/")) - return "" + return out + + +def _normalize_language_code(value: Any) -> str: + text = str(value or "").strip().lower() + if not text: + return "" + if text.startswith("/languages/"): + text = text.rsplit("/", 1)[-1].strip().lower() + return _LANGUAGE_NAME_TO_CODE.get(text, text) + + +def _extract_language_codes(value: Any) -> List[str]: + out: List[str] = [] + + def _add(raw: Any) -> None: + code = _normalize_language_code(raw) + if code and code not in out: + out.append(code) + + if isinstance(value, list): + for item in value: + if isinstance(item, dict): + _add(item.get("key") or item.get("code") or item.get("name")) + else: + _add(item) + elif isinstance(value, dict): + _add(value.get("key") or value.get("code") or value.get("name")) + else: + _add(value) + + return out + + +def _language_label(codes: List[str]) -> str: + labels = [ + _LANGUAGE_CODE_TO_NAME.get(code, str(code or "").upper()) + for code in codes + if str(code or "").strip() + ] + if not labels: + return "Unknown" + if len(labels) == 1: + return labels[0] + return ", ".join(labels[:3]) + + +def _order_language_codes(codes: List[str], preferred_language: str) -> List[str]: + cleaned: List[str] = [] + for code in codes: + text = str(code or "").strip().lower() + if text and text not in cleaned: + cleaned.append(text) + + preferred = str(preferred_language or "").strip().lower() or _DEFAULT_PREFERRED_LANGUAGE + indexed_codes = list(enumerate(cleaned)) + indexed_codes.sort(key=lambda item: (0 if item[1] == preferred else 1, item[0])) + return [code for _, code in indexed_codes] + + +def _extract_archive_candidates(payload: Any) -> List[str]: + if not isinstance(payload, dict): + return [] + + out: List[str] = [] + + def _add(raw: Any) -> None: + text = str(raw or "").strip() + if text and text not in out: + out.append(text) + + _add(payload.get("ocaid")) + for key in ("ia", "internet_archive", "archive_id", "ocaids"): + value = payload.get(key) + if isinstance(value, list): + for item in value: + _add(item) + else: + _add(value) + + identifiers = payload.get("identifiers") + if isinstance(identifiers, dict): + ia_value = identifiers.get("internet_archive") + if isinstance(ia_value, list): + for item in ia_value: + _add(item) + else: + _add(ia_value) + + return out def _check_lendable(session: requests.Session, edition_id: str) -> Tuple[bool, str]: @@ -210,6 +362,49 @@ def _resolve_archive_id( return "" +def _fetch_work_editions( + session: requests.Session, + work_key: str, + *, + limit: int = 200, +) -> List[Dict[str, Any]]: + work_path = str(work_key or "").strip() + if not work_path.startswith("/works/"): + return [] + + try: + resp = session.get( + f"https://openlibrary.org{work_path}/editions.json", + params={"limit": int(limit)}, + timeout=10, + ) + resp.raise_for_status() + data = resp.json() or {} + except Exception: + return [] + + entries = data.get("entries") if isinstance(data, dict) else None + if not isinstance(entries, list): + return [] + + out: List[Dict[str, Any]] = [] + seen: set[str] = set() + for entry in entries: + if not isinstance(entry, dict): + continue + edition_id = _resolve_edition_id(entry) + if not edition_id or edition_id in seen: + continue + seen.add(edition_id) + out.append({ + "edition_id": edition_id, + "raw": dict(entry), + "language_codes": _extract_language_codes(entry.get("languages") or entry.get("language")), + "archive_candidates": _extract_archive_candidates(entry), + }) + return out + + def _fetch_openlibrary_edition_metadata( session: requests.Session, edition_id: str, @@ -248,6 +443,10 @@ def _fetch_openlibrary_edition_metadata( "openlibrary_id": str(edition_id).strip(), "openlibrary": str(edition_id).strip(), } + language_codes = _extract_language_codes(data.get("languages") or data.get("language")) + if language_codes: + out["language_codes"] = language_codes + out["language_label"] = _language_label(language_codes) if isbn_10: out["isbn_10"] = isbn_10 if isbn_13: @@ -410,7 +609,7 @@ def title_hint_from_url_slug(u: str) -> str: class OpenLibrary(Provider): TABLE_AUTO_STAGES = { - "openlibrary": ["download-file"], + "openlibrary.edition": ["download-file"], } @classmethod @@ -434,6 +633,23 @@ class OpenLibrary(Provider): "label": "Image Quality", "default": "medium", "choices": ["high", "medium", "low"] + }, + { + "key": "preferred_language", + "label": "Preferred Edition Language", + "default": "English", + "choices": [ + "English", + "Spanish", + "French", + "German", + "Italian", + "Portuguese", + "Polish", + "Russian", + "Chinese", + "Japanese", + ] } ] @@ -453,16 +669,499 @@ class OpenLibrary(Provider): class BookNotAvailableError(Exception): """Raised when a book is not available for borrowing (waitlisted/in use).""" + @staticmethod + def _preferred_language_from_config(config: Dict[str, Any]) -> str: + if not isinstance(config, dict): + return _DEFAULT_PREFERRED_LANGUAGE + + entry = config.get("provider", {}).get("openlibrary", {}) + if not isinstance(entry, dict): + return _DEFAULT_PREFERRED_LANGUAGE + + value = entry.get("preferred_language") or entry.get("language") + code = _normalize_language_code(value) + return code or _DEFAULT_PREFERRED_LANGUAGE + + @staticmethod + def _edition_language_sort_key(language_codes: List[str], preferred_language: str, ordinal: int) -> Tuple[int, int, int]: + codes = [str(code or "").strip().lower() for code in language_codes if str(code or "").strip()] + preferred = str(preferred_language or "").strip().lower() or _DEFAULT_PREFERRED_LANGUAGE + preferred_rank = 0 if preferred in codes else 1 + unknown_rank = 1 if not codes else 0 + return preferred_rank, unknown_rank, ordinal + + def _build_edition_candidates(self, payload: Dict[str, Any]) -> List[Dict[str, Any]]: + meta = payload.get("full_metadata") or payload.get("metadata") or {} + if not isinstance(meta, dict): + meta = {} + + raw_doc = meta.get("raw") if isinstance(meta.get("raw"), dict) else {} + candidate_map: Dict[str, Dict[str, Any]] = {} + order: List[str] = [] + + def _is_edition_raw(raw_entry: Optional[Dict[str, Any]]) -> bool: + if not isinstance(raw_entry, dict): + return False + key = str(raw_entry.get("key") or "").strip() + return key.startswith("/books/") + + def _upsert(edition_id: str, raw_entry: Optional[Dict[str, Any]] = None) -> None: + text = str(edition_id or "").strip() + if not text: + return + existing = candidate_map.get(text) + if existing is None: + existing = { + "edition_id": text, + "raw": {}, + "language_codes": [], + "archive_candidates": [], + "ordinal": len(order), + } + candidate_map[text] = existing + order.append(text) + + if _is_edition_raw(raw_entry): + existing_raw = existing.get("raw") + if not isinstance(existing_raw, dict) or not existing_raw: + existing["raw"] = dict(raw_entry) + language_codes = existing.get("language_codes") or [] + if not language_codes: + existing["language_codes"] = _extract_language_codes(raw_entry.get("languages") or raw_entry.get("language")) + archive_candidates = existing.get("archive_candidates") or [] + if not archive_candidates: + existing["archive_candidates"] = _extract_archive_candidates(raw_entry) + + if isinstance(raw_doc, dict): + for edition_id in _resolve_candidate_edition_ids(raw_doc): + _upsert(edition_id) + for edition_id in _resolve_candidate_edition_ids(meta): + _upsert(edition_id) + + work_key = str(meta.get("openlibrary_key") or "").strip() + if work_key: + for entry in _fetch_work_editions(self._session, work_key): + if not isinstance(entry, dict): + continue + _upsert( + str(entry.get("edition_id") or "").strip(), + entry.get("raw") if isinstance(entry.get("raw"), dict) else None, + ) + existing = candidate_map.get(str(entry.get("edition_id") or "").strip()) + if isinstance(existing, dict): + if not existing.get("language_codes"): + existing["language_codes"] = list(entry.get("language_codes") or []) + if not existing.get("archive_candidates"): + existing["archive_candidates"] = list(entry.get("archive_candidates") or []) + + preferred_language = self._preferred_language_from_config(self.config) + candidates = [candidate_map[edition_id] for edition_id in order if edition_id in candidate_map] + candidates.sort( + key=lambda item: self._edition_language_sort_key( + list(item.get("language_codes") or []), + preferred_language, + int(item.get("ordinal") or 0), + ) + ) + return candidates + + def get_table_type(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str: + filters = filters or {} + view = str(filters.get("view") or "").strip().lower() + if view in {"edition", "editions", "borrowable-editions", "borrowable_editions"}: + return "openlibrary.edition" + return "openlibrary.work" + + @staticmethod + def _selection_payload(item: Any) -> Dict[str, Any]: + if isinstance(item, dict): + return dict(item) + try: + if hasattr(item, "to_dict"): + payload = item.to_dict() # type: ignore[attr-defined] + if isinstance(payload, dict): + return payload + except Exception: + pass + try: + return { + "table": getattr(item, "table", None), + "title": getattr(item, "title", None), + "path": getattr(item, "path", None), + "detail": getattr(item, "detail", None), + "annotations": getattr(item, "annotations", None), + "media_kind": getattr(item, "media_kind", None), + "full_metadata": getattr(item, "full_metadata", None), + } + except Exception: + return {} + + def _build_borrowable_edition_results(self, payload: Dict[str, Any]) -> List[SearchResult]: + meta = payload.get("full_metadata") or payload.get("metadata") or {} + if not isinstance(meta, dict): + meta = {} + + raw_doc = meta.get("raw") if isinstance(meta.get("raw"), dict) else {} + + candidates = self._build_edition_candidates(payload) + if not candidates: + return [] + + parent_title = str(payload.get("title") or meta.get("title") or raw_doc.get("title") or "Unknown").strip() or "Unknown" + + authors_value = meta.get("authors") or raw_doc.get("author_name") or [] + if isinstance(authors_value, str): + authors_value = [authors_value] + if not isinstance(authors_value, list): + authors_value = [] + authors_list = [str(author).strip() for author in authors_value if str(author or "").strip()] + + parent_year = str(meta.get("year") or raw_doc.get("first_publish_year") or "").strip() + + ia_candidates: List[str] = [] + for source in (meta.get("ia"), raw_doc.get("ia")): + if isinstance(source, str): + source = [source] + if isinstance(source, list): + for value in source: + text = str(value or "").strip() + if text and text not in ia_candidates: + ia_candidates.append(text) + + preferred_language = self._preferred_language_from_config(self.config) + + return self._build_borrowable_edition_results_from_candidates( + candidates, + raw_doc=raw_doc, + meta=meta, + parent_title=parent_title, + authors_list=authors_list, + parent_year=parent_year, + ia_candidates=ia_candidates, + preferred_language=preferred_language, + ) + + def _build_borrowable_edition_results_from_candidates( + self, + candidates: List[Dict[str, Any]], + *, + raw_doc: Dict[str, Any], + meta: Dict[str, Any], + parent_title: str, + authors_list: List[str], + parent_year: str, + ia_candidates: List[str], + preferred_language: str, + ) -> List[SearchResult]: + if not candidates: + return [] + + def _build_one(candidate: Dict[str, Any]) -> Optional[SearchResult]: + return self._build_borrowable_edition_result( + candidate, + raw_doc=raw_doc, + meta=meta, + parent_title=parent_title, + authors_list=authors_list, + parent_year=parent_year, + ia_candidates=ia_candidates, + preferred_language=preferred_language, + ) + + results: List[SearchResult] = [] + max_workers = min(12, max(1, len(candidates))) + with futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + future_to_id = { + executor.submit(_build_one, candidate): str(candidate.get("edition_id") or "").strip() + for candidate in candidates + } + resolved: Dict[str, SearchResult] = {} + for future in futures.as_completed(list(future_to_id.keys())): + edition_id = future_to_id[future] + try: + built = future.result() + except Exception: + built = None + if built is not None: + resolved[edition_id] = built + + for candidate in candidates: + edition_id = str(candidate.get("edition_id") or "").strip() + built = resolved.get(edition_id) + if built is not None: + results.append(built) + return results + + def _build_borrowable_edition_result( + self, + candidate: Dict[str, Any], + *, + raw_doc: Dict[str, Any], + meta: Dict[str, Any], + parent_title: str, + authors_list: List[str], + parent_year: str, + ia_candidates: List[str], + preferred_language: str, + ) -> Optional[SearchResult]: + edition_id = str(candidate.get("edition_id") or "").strip() + if not edition_id: + return None + session_local = _create_archive_session() + lendable, reason = _check_lendable(session_local, edition_id) + archive_candidates = list(candidate.get("archive_candidates") or []) + for fallback_candidate in ia_candidates: + if fallback_candidate not in archive_candidates: + archive_candidates.append(fallback_candidate) + + archive_id = _first_str(archive_candidates) or "" + if lendable and not archive_id: + archive_id = _resolve_archive_id(session_local, edition_id, ia_candidates) + + if not lendable: + if not archive_id: + archive_id = _resolve_archive_id(session_local, edition_id, ia_candidates) + if not archive_id: + return None + lendable2, reason2 = self._archive_is_lendable(archive_id) + if not lendable2: + return None + reason = reason2 or reason + + edition_meta = _fetch_openlibrary_edition_metadata(session_local, edition_id) + if not archive_id: + archive_id = str(edition_meta.get("archive_id") or "").strip() + if not archive_id: + return None + + isbn_10 = str(edition_meta.get("isbn_10") or meta.get("isbn_10") or "").strip() + isbn_13 = str(edition_meta.get("isbn_13") or meta.get("isbn_13") or "").strip() + language_codes = list(edition_meta.get("language_codes") or candidate.get("language_codes") or []) + language_codes = _order_language_codes(language_codes, preferred_language) + language_label = _language_label(language_codes) + book_path = f"https://openlibrary.org/books/{edition_id}" + selection_url = ( + f"https://archive.org/details/{archive_id}" + if archive_id else book_path + ) + + annotations: List[str] = ["borrow", f"edition:{edition_id}"] + if archive_id: + annotations.append("archive") + if language_codes: + annotations.append(f"lang:{language_codes[0]}") + if isbn_13: + annotations.append(f"isbn_13:{isbn_13}") + elif isbn_10: + annotations.append(f"isbn_10:{isbn_10}") + + edition_metadata = { + "openlibrary_id": edition_id, + "openlibrary_key": f"/books/{edition_id}", + "authors": authors_list, + "year": parent_year, + "isbn_10": isbn_10, + "isbn_13": isbn_13, + "language_codes": language_codes, + "language": language_label, + "ia": [archive_id] if archive_id else [], + "availability": "borrow", + "availability_reason": reason, + "archive_id": archive_id, + "direct_url": "", + "selection_view": "edition", + "selection_url": selection_url, + "raw": raw_doc, + "_selection_args": ["-url", selection_url], + "_selection_action": ["download-file", "-url", selection_url], + } + + return SearchResult( + table="openlibrary.edition", + title=parent_title, + path=book_path, + detail=( + (f"By: {', '.join(authors_list)}" if authors_list else "") + + (f" ({parent_year})" if parent_year else "") + ).strip(), + annotations=annotations, + media_kind="book", + columns=[ + ("Title", parent_title), + ("Author", ", ".join(authors_list)), + ("Language", language_label), + ("Year", parent_year), + ("Avail", "borrow"), + ("OLID", edition_id), + ], + full_metadata=edition_metadata, + ) + + def _build_preferred_borrowable_edition(self, payload: Dict[str, Any]) -> Optional[SearchResult]: + meta = payload.get("full_metadata") or payload.get("metadata") or {} + if not isinstance(meta, dict): + meta = {} + + raw_doc = meta.get("raw") if isinstance(meta.get("raw"), dict) else {} + candidates = self._build_edition_candidates(payload) + if not candidates: + return None + + parent_title = str(payload.get("title") or meta.get("title") or raw_doc.get("title") or "Unknown").strip() or "Unknown" + authors_value = meta.get("authors") or raw_doc.get("author_name") or [] + if isinstance(authors_value, str): + authors_value = [authors_value] + if not isinstance(authors_value, list): + authors_value = [] + authors_list = [str(author).strip() for author in authors_value if str(author or "").strip()] + parent_year = str(meta.get("year") or raw_doc.get("first_publish_year") or "").strip() + + ia_candidates: List[str] = [] + for source in (meta.get("ia"), raw_doc.get("ia")): + if isinstance(source, str): + source = [source] + if isinstance(source, list): + for value in source: + text = str(value or "").strip() + if text and text not in ia_candidates: + ia_candidates.append(text) + + preferred_language = self._preferred_language_from_config(self.config) + for candidate in candidates: + built = self._build_borrowable_edition_result( + candidate, + raw_doc=raw_doc, + meta=meta, + parent_title=parent_title, + authors_list=authors_list, + parent_year=parent_year, + ia_candidates=ia_candidates, + preferred_language=preferred_language, + ) + if built is not None: + return built + return None + + def expand_selection( + self, + selected_items: List[Any], + *, + ctx: Any, + stage_is_last: bool = True, + table_type: str = "", + **_kwargs: Any, + ) -> Optional[List[Any]]: + _ = ctx + if stage_is_last: + return None + + normalized_table = str(table_type or "").strip().lower() + if normalized_table != "openlibrary.work": + return None + + for item in selected_items or []: + payload = self._selection_payload(item) + meta = payload.get("full_metadata") or payload.get("metadata") or {} + if not isinstance(meta, dict): + continue + if str(meta.get("selection_view") or "").strip().lower() != "work": + continue + preferred_edition = self._build_preferred_borrowable_edition(payload) + if preferred_edition is not None: + return [preferred_edition] + return None + + def selector( + self, + selected_items: List[Any], + *, + ctx: Any, + stage_is_last: bool = True, + **_kwargs: Any, + ) -> bool: + _ = stage_is_last + + chosen_payload: Optional[Dict[str, Any]] = None + for item in selected_items or []: + payload = self._selection_payload(item) + meta = payload.get("full_metadata") or payload.get("metadata") or {} + if not isinstance(meta, dict): + meta = {} + selection_view = str(meta.get("selection_view") or "").strip().lower() + table_type = str(payload.get("table") or "").strip().lower() + if selection_view == "edition" or table_type == "openlibrary.edition": + continue + if selection_view == "work" or table_type == "openlibrary.work": + chosen_payload = payload + break + + if chosen_payload is None: + return False + + try: + editions = self._build_borrowable_edition_results(chosen_payload) + except Exception as exc: + print(f"openlibrary selector failed: {exc}\n") + return True + + if not editions: + print("No borrowable OpenLibrary editions were found for that work.\n") + return True + + try: + from SYS.result_table import Table + from SYS.rich_display import stdout_console + except Exception: + return True + + title = str(chosen_payload.get("title") or "OpenLibrary").strip() or "OpenLibrary" + table = Table(f"OpenLibrary Editions: {title}")._perseverance(True) + table.set_table("openlibrary.edition") + try: + table.set_table_metadata({"provider": "openlibrary", "view": "borrowable_editions"}) + except Exception: + pass + table.set_source_command("search-file", ["-plugin", "openlibrary"]) + + results_payload: List[Dict[str, Any]] = [] + for edition in editions: + table.add_result(edition) + try: + results_payload.append(edition.to_dict()) + except Exception: + results_payload.append({ + "table": getattr(edition, "table", "openlibrary.edition"), + "title": getattr(edition, "title", ""), + "path": getattr(edition, "path", ""), + "full_metadata": getattr(edition, "full_metadata", None), + }) + + try: + ctx.set_last_result_table(table, results_payload) + ctx.set_current_stage_table(table) + except Exception: + pass + + stdout_console().print() + stdout_console().print(table) + return True + def search_result_from_url(self, url: str) -> Optional[SearchResult]: """Build a minimal SearchResult from a bare OpenLibrary/Archive URL.""" edition_id = edition_id_from_url(url) + archive_id = _archive_id_from_url(url) title_hint = title_hint_from_url_slug(url) + metadata: Dict[str, Any] = {} + if edition_id: + metadata["openlibrary_id"] = edition_id + if archive_id: + metadata["archive_id"] = archive_id return SearchResult( table="openlibrary", title=title_hint, path=str(url), media_kind="book", - full_metadata={"openlibrary_id": edition_id} if edition_id else {}, + full_metadata=metadata, ) def download_url( @@ -475,7 +1174,7 @@ class OpenLibrary(Provider): Returns a dict with the downloaded path and SearchResult when successful. """ - sr = self.search_result_from_url(url) + self, if sr is None: return None @@ -488,6 +1187,87 @@ class OpenLibrary(Provider): "search_result": sr, } + def resolve_pipe_result_download( + self, + result: Any, + pipe_obj: Any, + ) -> Tuple[Optional[Path], Optional[str], Optional[Path]]: + download_url = "" + for source in ( + getattr(pipe_obj, "url", None) if pipe_obj is not None else None, + getattr(pipe_obj, "source_url", None) if pipe_obj is not None else None, + getattr(pipe_obj, "metadata", {}).get("selection_url") if pipe_obj is not None and isinstance(getattr(pipe_obj, "metadata", None), dict) else None, + getattr(pipe_obj, "metadata", {}).get("selection_action", [None, None])[-1] if pipe_obj is not None and isinstance(getattr(pipe_obj, "metadata", None), dict) and isinstance(getattr(pipe_obj, "metadata", {}).get("selection_action"), list) else None, + ): + text = str(source or "").strip() + if text.startswith(("http://", "https://")): + download_url = text + break + + if not download_url and isinstance(result, dict): + for source in ( + result.get("url"), + result.get("path"), + result.get("full_metadata", {}).get("selection_url") if isinstance(result.get("full_metadata"), dict) else None, + ): + text = str(source or "").strip() + if text.startswith(("http://", "https://")): + download_url = text + break + + if not download_url: + return None, None, None + + progress_callback = None + if isinstance(self.config, dict): + pipeline_progress = self.config.get("_pipeline_progress") + if pipeline_progress is not None: + label = "" + for source in ( + getattr(pipe_obj, "title", None) if pipe_obj is not None else None, + result.get("title") if isinstance(result, dict) else None, + getattr(pipe_obj, "metadata", {}).get("openlibrary_id") if pipe_obj is not None and isinstance(getattr(pipe_obj, "metadata", None), dict) else None, + ): + text = str(source or "").strip() + if text: + label = text + break + progress_callback = _build_pipeline_progress_callback( + pipeline_progress, + label or "openlibrary", + ) + + tmp_dir = Path(tempfile.mkdtemp(prefix="openlibrary-add-file-")) + try: + downloaded = self.download_url( + download_url, + tmp_dir, + progress_callback=progress_callback, + ) + except Exception: + try: + shutil.rmtree(tmp_dir, ignore_errors=True) + except Exception: + pass + return None, None, None + + if not isinstance(downloaded, dict): + try: + shutil.rmtree(tmp_dir, ignore_errors=True) + except Exception: + pass + return None, None, None + + downloaded_path = downloaded.get("path") + if isinstance(downloaded_path, Path) and downloaded_path.exists(): + return downloaded_path, None, tmp_dir + + try: + shutil.rmtree(tmp_dir, ignore_errors=True) + except Exception: + pass + return None, None, None + @staticmethod def _credential_archive(config: Dict[str, Any]) -> Tuple[Optional[str], Optional[str]]: """Get Archive.org email/password from config.""" @@ -1007,12 +1787,13 @@ class OpenLibrary(Provider): def _compute_availability(doc_dict: Dict[str, Any]) -> Tuple[str, + str, str, str, str]: - edition_id_local = _resolve_edition_id(doc_dict) - if not edition_id_local: - return "no-olid", "", "", "" + candidate_edition_ids = _resolve_candidate_edition_ids(doc_dict) + if not candidate_edition_ids: + return "no-olid", "", "", "", "" ia_val_local = doc_dict.get("ia") or [] if isinstance(ia_val_local, str): @@ -1023,38 +1804,48 @@ class OpenLibrary(Provider): session_local = _create_archive_session() - try: - archive_id_local = _resolve_archive_id( - session_local, - edition_id_local, - ia_ids_local - ) - except Exception: - archive_id_local = "" + last_reason = "" + last_archive_id = "" + last_edition_id = candidate_edition_ids[0] + for edition_id_local in candidate_edition_ids[:25]: + last_edition_id = edition_id_local + try: + archive_id_local = _resolve_archive_id( + session_local, + edition_id_local, + ia_ids_local + ) + except Exception: + archive_id_local = "" - if not archive_id_local: - return "no-archive", "", "", "" + if not archive_id_local: + continue - # Prefer the fastest signal first: OpenLibrary lendable status. - lendable_local, reason_local = _check_lendable(session_local, edition_id_local) - if lendable_local: - return "borrow", reason_local, archive_id_local, "" + last_archive_id = archive_id_local + lendable_local, reason_local = _check_lendable(session_local, edition_id_local) + if lendable_local: + return "borrow", reason_local, archive_id_local, "", edition_id_local - # OpenLibrary API can be a false-negative; fall back to Archive metadata. - try: - lendable2, reason2 = self._archive_is_lendable(archive_id_local) - if lendable2: - return "borrow", reason2 or reason_local, archive_id_local, "" - except Exception: - pass + try: + lendable2, reason2 = self._archive_is_lendable(archive_id_local) + if lendable2: + return "borrow", reason2 or reason_local, archive_id_local, "", edition_id_local + except Exception: + pass - return "unavailable", reason_local, archive_id_local, "" + last_reason = reason_local + + if last_archive_id: + return "unavailable", last_reason, last_archive_id, "", last_edition_id + return "no-archive", "", "", "", last_edition_id availability_rows: List[Tuple[str, + str, str, str, str]] = [ ("unknown", + "", "", "", "") for _ in range(len(docs)) @@ -1073,7 +1864,7 @@ class OpenLibrary(Provider): try: availability_rows[i] = fut.result() except Exception: - availability_rows[i] = ("unknown", "", "", "") + availability_rows[i] = ("unknown", "", "", "", "") done += 1 for idx, doc in enumerate(docs): @@ -1125,19 +1916,23 @@ class OpenLibrary(Provider): ] # Determine availability using the concurrently computed enrichment. - availability, availability_reason, archive_id, direct_url = ("unknown", "", "", "") + availability, availability_reason, archive_id, direct_url, preferred_edition_id = ("unknown", "", "", "", "") if 0 <= idx < len(availability_rows): - availability, availability_reason, archive_id, direct_url = availability_rows[idx] + availability, availability_reason, archive_id, direct_url, preferred_edition_id = availability_rows[idx] # UX requirement: OpenLibrary provider should ONLY show borrowable books. # Ignore printdisabled-only and non-borrow items. if availability != "borrow": continue + candidate_edition_ids = _resolve_candidate_edition_ids(doc) + if preferred_edition_id and preferred_edition_id not in candidate_edition_ids: + candidate_edition_ids.insert(0, preferred_edition_id) + # Patch the display column. - for idx, (name, _val) in enumerate(columns): + for column_idx, (name, _val) in enumerate(columns): if name == "Avail": - columns[idx] = ("Avail", availability) + columns[column_idx] = ("Avail", availability) break annotations: List[str] = [] @@ -1151,8 +1946,9 @@ class OpenLibrary(Provider): "borrow"}: annotations.append(availability) + selected_edition_id = preferred_edition_id or edition_id book_path = ( - f"https://openlibrary.org/books/{edition_id}" if edition_id else + f"https://openlibrary.org/books/{selected_edition_id}" if selected_edition_id else ( f"https://openlibrary.org{work_key}" if isinstance(work_key, str) and work_key.startswith("/") else @@ -1160,27 +1956,27 @@ class OpenLibrary(Provider): ) ) metadata = { - "openlibrary_id": edition_id, + "openlibrary_id": selected_edition_id, "openlibrary_key": work_key, "authors": authors_list, "year": year, "isbn_10": isbn_10, "isbn_13": isbn_13, "ia": ia_ids, + "candidate_edition_ids": candidate_edition_ids, "availability": availability, "availability_reason": availability_reason, "archive_id": archive_id, "direct_url": direct_url, + "selection_view": "work", "raw": doc, } if book_path: metadata["selection_url"] = book_path - metadata["_selection_args"] = ["-url", book_path] - metadata["_selection_action"] = ["download-file", "-url", book_path] results.append( SearchResult( - table="openlibrary", + table="openlibrary.work", title=book_title, path=book_path, detail=( diff --git a/plugins/scp/__init__.py b/plugins/scp/__init__.py new file mode 100644 index 0000000..0d4c8ef --- /dev/null +++ b/plugins/scp/__init__.py @@ -0,0 +1,938 @@ +from __future__ import annotations + +import fnmatch +import posixpath +import shlex +import stat +import tempfile +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import quote, unquote, urlparse + +import paramiko +from scp import SCPClient + +from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments + + +def _pick_provider_config(config: Any) -> Dict[str, Any]: + if not isinstance(config, dict): + return {} + provider = config.get("provider") + if not isinstance(provider, dict): + return {} + entry = provider.get("scp") + if isinstance(entry, dict): + return entry + return {} + + +def _coerce_bool(value: Any, default: bool = False) -> bool: + if isinstance(value, bool): + return value + if value is None: + return default + text = str(value).strip().lower() + if not text: + return default + if text in {"1", "true", "yes", "on"}: + return True + if text in {"0", "false", "no", "off"}: + return False + return default + + +def _coerce_int(value: Any, default: int) -> int: + try: + return int(value) + except Exception: + return default + + +def _format_epoch(raw_value: Any) -> str: + try: + stamp = int(raw_value) + except Exception: + return "" + try: + return datetime.fromtimestamp(stamp).strftime("%Y-%m-%d %H:%M") + except Exception: + return str(raw_value or "") + + +def _safe_filename(name: Any) -> str: + raw = str(name or "").strip() + if not raw: + raw = "download" + cleaned = "".join(ch if ch.isalnum() or ch in {"-", "_", ".", " "} else "_" for ch in raw) + cleaned = cleaned.strip(" ._") + return cleaned or "download" + + +def _unique_path(path: Path) -> Path: + if not path.exists(): + return path + stem = path.stem or "download" + suffix = path.suffix + counter = 1 + while True: + candidate = path.with_name(f"{stem}_{counter}{suffix}") + if not candidate.exists(): + return candidate + counter += 1 + + +class SCP(Provider): + PLUGIN_NAME = "scp" + URL = ("scp://", "sftp://") + + @property + def label(self) -> str: + return "SCP" + + @property + def preserve_order(self) -> bool: + return True + + @classmethod + def config_schema(cls) -> List[Dict[str, Any]]: + return [ + { + "key": "host", + "label": "Host", + "default": "", + "required": True, + "placeholder": "ssh.example.com", + }, + { + "key": "port", + "label": "Port", + "type": "integer", + "default": 22, + }, + { + "key": "username", + "label": "Username", + "default": "", + "required": True, + "placeholder": "deploy", + }, + { + "key": "password", + "label": "Password", + "type": "secret", + "secret": True, + "default": "", + }, + { + "key": "key_path", + "label": "SSH Key Path", + "type": "path", + "default": "", + "placeholder": "C:/Users/Admin/.ssh/id_ed25519", + }, + { + "key": "base_path", + "label": "Base Path", + "default": "/", + "placeholder": "/srv/files", + }, + { + "key": "timeout", + "label": "Timeout Seconds", + "type": "integer", + "default": 20, + }, + { + "key": "search_depth", + "label": "Default Search Depth", + "type": "integer", + "default": 1, + }, + { + "key": "allow_agent", + "label": "Use SSH Agent", + "type": "boolean", + "default": True, + }, + { + "key": "look_for_keys", + "label": "Look For Default Keys", + "type": "boolean", + "default": True, + }, + ] + + def __init__(self, config: Optional[Dict[str, Any]] = None): + super().__init__(config) + conf = _pick_provider_config(self.config) + self._host = str(conf.get("host") or "").strip() + self._port = _coerce_int(conf.get("port"), 22) + self._username = str(conf.get("username") or conf.get("user") or "").strip() + self._password = str(conf.get("password") or "").strip() + self._key_path = str(conf.get("key_path") or conf.get("identity_file") or "").strip() + self._timeout = max(1, _coerce_int(conf.get("timeout"), 20)) + self._search_depth = max(0, _coerce_int(conf.get("search_depth"), 1)) + self._allow_agent = _coerce_bool(conf.get("allow_agent"), True) + self._look_for_keys = _coerce_bool(conf.get("look_for_keys"), True) + self._base_path = self._normalize_remote_path(conf.get("base_path") or "/", default="/") + + def validate(self) -> bool: + return bool(self._host and self._username) + + def config_helper_text(self) -> str: + return "Test the SSH/SCP connection before searching. You can also generate an RSA key pair from here." + + def config_actions(self) -> List[Dict[str, Any]]: + return [ + { + "id": "test_connection", + "label": "Test connection", + "variant": "primary", + }, + { + "id": "generate_ssh_key", + "label": "Generate SSH key", + "variant": "default", + }, + ] + + def run_config_action(self, action_id: str, **_kwargs: Any) -> Dict[str, Any]: + normalized = str(action_id or "").strip().lower() + if normalized == "test_connection": + return self._run_test_connection() + if normalized == "generate_ssh_key": + return self._generate_ssh_keypair() + return super().run_config_action(action_id, **_kwargs) + + def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]: + text, inline = parse_inline_query_arguments(query) + filters: Dict[str, Any] = {} + + if inline.get("path"): + filters["path"] = inline.get("path") + if inline.get("depth"): + filters["depth"] = max(0, _coerce_int(inline.get("depth"), self._search_depth)) + if inline.get("type"): + filters["type"] = str(inline.get("type") or "").strip().lower() + + return text, filters + + def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str: + active_path = self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path) + text = str(query or "").strip() + if not text or text == "*": + return f"SCP: {active_path}" + return f"SCP: {text} @ {active_path}" + + def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + return { + "plugin": self.name, + "host": self._host, + "path": self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path), + "query": str(query or "").strip(), + } + + def search( + self, + query: str, + limit: int = 50, + filters: Optional[Dict[str, Any]] = None, + **kwargs: Any, + ) -> List[SearchResult]: + _ = kwargs + active_filters = dict(filters or {}) + start_path = self._normalize_remote_path(active_filters.get("path") or self._base_path, default=self._base_path) + search_depth = max(0, _coerce_int(active_filters.get("depth"), self._search_depth)) + type_filter = str(active_filters.get("type") or "any").strip().lower() + needle = str(query or "").strip() + max_results = max(0, int(limit or 0)) + if max_results <= 0: + return [] + + ssh = self._connect_ssh() + sftp = None + try: + try: + sftp = self._open_sftp(ssh) + except Exception as exc: + if not self._is_sftp_negotiation_error(exc): + raise + return self._search_directory_via_ssh( + ssh, + start_path, + needle=needle, + limit=max_results, + search_depth=search_depth, + type_filter=type_filter, + ) + + return self._search_directory( + sftp, + start_path, + needle=needle, + limit=max_results, + search_depth=search_depth, + type_filter=type_filter, + ) + finally: + self._close_client(sftp) + self._close_client(ssh) + + def selector( + self, + selected_items: List[Any], + *, + ctx: Any, + stage_is_last: bool = True, + **_kwargs: Any, + ) -> bool: + if not stage_is_last: + return False + + target_path = "" + target_title = "" + for item in selected_items or []: + metadata = self._item_metadata(item) + if not metadata.get("is_dir"): + continue + target_path = self._normalize_remote_path(metadata.get("scp_path") or metadata.get("selection_path"), default=self._base_path) + target_title = str(metadata.get("title") or metadata.get("name") or "").strip() + if target_path: + break + + if not target_path: + return False + + ssh = self._connect_ssh() + sftp = None + try: + try: + sftp = self._open_sftp(ssh) + except Exception as exc: + if not self._is_sftp_negotiation_error(exc): + raise + rows = self._search_directory_via_ssh( + ssh, + target_path, + needle="*", + limit=500, + search_depth=0, + type_filter="any", + ) + else: + rows = self._search_directory( + sftp, + target_path, + needle="*", + limit=500, + search_depth=0, + type_filter="any", + ) + finally: + self._close_client(sftp) + self._close_client(ssh) + + try: + from SYS.result_table import Table + from SYS.rich_display import stdout_console + except Exception: + return True + + title = target_title or target_path + table = Table(f"SCP: {title}")._perseverance(True) + table.set_table("scp") + try: + table.set_table_metadata({ + "provider": "scp", + "host": self._host, + "path": target_path, + "view": "directory", + }) + except Exception: + pass + table.set_source_command("search-file", ["-plugin", "scp", f"path:{target_path}", "*"]) + + payloads: List[Dict[str, Any]] = [] + for row in rows: + table.add_result(row) + payloads.append(row.to_dict()) + + try: + ctx.set_last_result_table(table, payloads, subject={"plugin": "scp", "path": target_path}) + ctx.set_current_stage_table(table) + except Exception: + pass + + try: + stdout_console().print() + stdout_console().print(table) + except Exception: + pass + + return True + + def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]: + metadata = getattr(result, "full_metadata", None) + if isinstance(metadata, dict) and metadata.get("is_dir"): + return None + target = str(getattr(result, "path", "") or "").strip() + if not target: + return None + return self.download_url(target, output_dir, title=getattr(result, "title", None)) + + def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]: + settings = self._connection_settings_for_url(url) + remote_path = settings["path"] + if not remote_path or remote_path == "/": + return None + + filename_hint = str(kwargs.get("title") or "").strip() + parsed_name = posixpath.basename(remote_path.rstrip("/")) + filename = _safe_filename(filename_hint or unquote(parsed_name) or "download") + + destination_dir = Path(output_dir) + destination_dir.mkdir(parents=True, exist_ok=True) + destination = _unique_path(destination_dir / filename) + + ssh = self._connect_ssh(settings) + scp_client = None + try: + scp_client = self._open_scp(ssh) + scp_client.get(remote_path, local_path=str(destination)) + return destination + except Exception: + try: + destination.unlink(missing_ok=True) + except Exception: + pass + return None + finally: + self._close_client(scp_client) + self._close_client(ssh) + + def resolve_pipe_result_download( + self, + result: Any, + pipe_obj: Any, + ) -> Tuple[Optional[Path], Optional[str], Optional[Path]]: + metadata = self._item_metadata(result, pipe_obj=pipe_obj) + if metadata.get("is_dir"): + return None, None, None + + download_url = str( + metadata.get("selection_url") + or metadata.get("scp_url") + or metadata.get("path") + or "" + ).strip() + if not download_url.startswith(("scp://", "sftp://")): + return None, None, None + + temp_dir = Path(tempfile.mkdtemp(prefix="scp-add-file-")) + downloaded = self.download_url(download_url, temp_dir, title=metadata.get("title")) + if downloaded is None: + try: + temp_dir.rmdir() + except Exception: + pass + return None, None, None + + try: + if pipe_obj is not None: + pipe_obj.is_temp = True + except Exception: + pass + return downloaded, None, temp_dir + + def upload(self, file_path: str, **kwargs: Any) -> str: + local_path = Path(str(file_path or "")).expanduser() + if not local_path.exists() or not local_path.is_file(): + raise FileNotFoundError(f"File not found: {local_path}") + + remote_dir = self._normalize_remote_path(kwargs.get("remote_path") or kwargs.get("path") or self._base_path, default=self._base_path) + remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name + remote_path = self._join_remote_path(remote_dir, remote_name) + + ssh = self._connect_ssh() + sftp = None + scp_client = None + try: + try: + sftp = self._open_sftp(ssh) + except Exception as exc: + if not self._is_sftp_negotiation_error(exc): + raise + self._ensure_directory_via_ssh(ssh, remote_dir) + else: + self._ensure_directory(sftp, remote_dir) + scp_client = self._open_scp(ssh) + scp_client.put(str(local_path), remote_path=remote_path) + finally: + self._close_client(scp_client) + self._close_client(sftp) + self._close_client(ssh) + + return self._build_url(remote_path) + + def _run_test_connection(self) -> Dict[str, Any]: + if not self._host: + return {"ok": False, "message": "Set 'host' before testing the SCP connection."} + if not self._username: + return {"ok": False, "message": "Set 'username' before testing the SCP connection."} + + ssh = None + sftp = None + try: + ssh = self._connect_ssh() + base_path = self._base_path or "/" + transport_detail = "SFTP available" + try: + sftp = self._open_sftp(ssh) + except Exception as exc: + if not self._is_sftp_negotiation_error(exc): + raise + is_dir = self._path_exists_via_ssh(ssh, base_path) + transport_detail = "SFTP unavailable; using SSH command fallback" + else: + try: + attrs = sftp.stat(base_path) + is_dir = stat.S_ISDIR(getattr(attrs, "st_mode", 0)) + except Exception: + is_dir = False + detail = f" and confirmed {base_path}" if is_dir else "" + auth_mode = f"key {self._key_path}" if self._key_path else "password/agent auth" + return { + "ok": True, + "message": f"Connected to SCP {self._host}:{self._port} as {self._username} via {auth_mode}. {transport_detail}{detail}.", + } + except Exception as exc: + return {"ok": False, "message": f"SCP connection failed: {exc}"} + finally: + self._close_client(sftp) + self._close_client(ssh) + + def _generate_ssh_keypair(self) -> Dict[str, Any]: + target = Path(self._key_path).expanduser() if self._key_path else (Path.home() / ".ssh" / "medeia_scp_rsa") + try: + target.parent.mkdir(parents=True, exist_ok=True) + except Exception as exc: + return {"ok": False, "message": f"Could not create key directory: {exc}"} + + public_path = target.with_name(target.name + ".pub") + if target.exists() or public_path.exists(): + return { + "ok": False, + "message": f"SSH key already exists at {target}. Remove it or choose a different key_path first.", + } + + try: + key = paramiko.RSAKey.generate(bits=4096) + key.write_private_key_file(str(target)) + comment = f"{self._username or 'medeia'}@{self._host or 'scp'}" + public_path.write_text(f"{key.get_name()} {key.get_base64()} {comment}\n", encoding="utf-8") + try: + target.chmod(0o600) + except Exception: + pass + return { + "ok": True, + "message": f"Generated SSH key pair at {target}. Save the config to persist key_path.", + "config_updates": {"key_path": str(target)}, + } + except Exception as exc: + try: + target.unlink(missing_ok=True) + except Exception: + pass + try: + public_path.unlink(missing_ok=True) + except Exception: + pass + return {"ok": False, "message": f"SSH key generation failed: {exc}"} + + def _connect_ssh(self, overrides: Optional[Dict[str, Any]] = None) -> paramiko.SSHClient: + settings = dict(overrides or {}) + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect( + hostname=str(settings.get("host") or self._host), + port=int(settings.get("port") or self._port), + username=str(settings.get("username") or self._username), + password=str(settings.get("password") or self._password) or None, + key_filename=str(settings.get("key_path") or self._key_path) or None, + timeout=self._timeout, + allow_agent=self._allow_agent if "allow_agent" not in settings else bool(settings.get("allow_agent")), + look_for_keys=self._look_for_keys if "look_for_keys" not in settings else bool(settings.get("look_for_keys")), + ) + return client + + def _open_sftp(self, ssh: Any) -> Any: + return ssh.open_sftp() + + def _open_scp(self, ssh: Any) -> Any: + return SCPClient(ssh.get_transport()) + + def _is_sftp_negotiation_error(self, exc: Exception) -> bool: + text = str(exc or "").strip().lower() + if isinstance(exc, EOFError): + return True + return any( + marker in text + for marker in ( + "eof during negotiation", + "open failed", + "channel closed", + "administratively prohibited", + "subsystem request failed", + ) + ) + + def _run_ssh_command(self, ssh: Any, command: str) -> Tuple[int, str, str]: + stdin, stdout, stderr = ssh.exec_command(command, timeout=self._timeout) + try: + stdin.close() + except Exception: + pass + output = stdout.read().decode("utf-8", errors="replace") + error = stderr.read().decode("utf-8", errors="replace") + status = 0 + try: + status = int(stdout.channel.recv_exit_status()) + except Exception: + status = 0 + return status, output, error + + def _path_exists_via_ssh(self, ssh: Any, remote_path: str) -> bool: + normalized = self._normalize_remote_path(remote_path, default=self._base_path) + quoted_path = shlex.quote(normalized) + status, _, _ = self._run_ssh_command(ssh, f"test -d {quoted_path}") + return status == 0 + + def _ensure_directory_via_ssh(self, ssh: Any, remote_path: str) -> None: + normalized = self._normalize_remote_path(remote_path, default=self._base_path) + if normalized == "/": + return + quoted_path = shlex.quote(normalized) + status, _, error = self._run_ssh_command(ssh, f"mkdir -p {quoted_path}") + if status != 0: + raise RuntimeError(error.strip() or f"mkdir -p failed for {normalized}") + + def _close_client(self, client: Any) -> None: + if client is None: + return + try: + client.close() + except Exception: + pass + + def _normalize_remote_path(self, value: Any, *, default: str) -> str: + text = str(value or "").strip().replace("\\", "/") + if not text: + text = default + elif text.startswith(("scp://", "sftp://")): + try: + text = unquote(urlparse(text).path or "/") + except Exception: + text = default + elif not text.startswith("/"): + text = posixpath.join(default, text) + + normalized = posixpath.normpath(text) + normalized = "/" + normalized.lstrip("/") + return normalized or "/" + + def _join_remote_path(self, parent: Any, child: Any) -> str: + left = self._normalize_remote_path(parent, default=self._base_path) + right = str(child or "").strip().replace("\\", "/") + if not right: + return left + return self._normalize_remote_path(posixpath.join(left, right), default="/") + + def _build_url( + self, + remote_path: Any, + *, + host: Optional[str] = None, + port: Optional[int] = None, + scheme: str = "scp", + ) -> str: + path_text = self._normalize_remote_path(remote_path, default="/") + host_text = str(host or self._host).strip() + port_value = int(port or self._port) + port_suffix = f":{port_value}" if port_value and port_value != 22 else "" + return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}" + + def _connection_settings_for_url(self, url: str) -> Dict[str, Any]: + parsed = urlparse(str(url or "").strip()) + scheme = (parsed.scheme or "scp").strip().lower() + host = parsed.hostname or self._host + port = parsed.port or self._port + username = parsed.username or self._username + password = parsed.password or self._password + path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default="/") + return { + "scheme": scheme, + "host": host, + "port": port, + "username": username, + "password": password, + "key_path": self._key_path, + "allow_agent": self._allow_agent, + "look_for_keys": self._look_for_keys, + "path": path_text, + } + + def _search_directory( + self, + sftp: Any, + start_path: str, + *, + needle: str, + limit: int, + search_depth: int, + type_filter: str, + ) -> List[SearchResult]: + results: List[SearchResult] = [] + visited: set[str] = set() + + def walk(current_path: str, depth_left: int) -> None: + normalized = self._normalize_remote_path(current_path, default=self._base_path) + if normalized in visited or len(results) >= limit: + return + visited.add(normalized) + + for entry in self._list_directory(sftp, normalized): + if len(results) >= limit: + return + if self._matches_entry(entry, needle=needle, type_filter=type_filter): + results.append(self._build_result(entry)) + if entry.get("is_dir") and depth_left > 0: + walk(str(entry.get("scp_path") or normalized), depth_left - 1) + + walk(start_path, max(0, search_depth)) + return results + + def _search_directory_via_ssh( + self, + ssh: Any, + start_path: str, + *, + needle: str, + limit: int, + search_depth: int, + type_filter: str, + ) -> List[SearchResult]: + entries = self._list_directory_via_ssh(ssh, start_path, depth=search_depth) + results: List[SearchResult] = [] + for entry in entries: + if len(results) >= limit: + break + if self._matches_entry(entry, needle=needle, type_filter=type_filter): + results.append(self._build_result(entry)) + return results + + def _matches_entry(self, entry: Dict[str, Any], *, needle: str, type_filter: str) -> bool: + is_dir = bool(entry.get("is_dir")) + if type_filter in {"dir", "dirs", "folder", "folders"} and not is_dir: + return False + if type_filter in {"file", "files"} and is_dir: + return False + + text = str(needle or "").strip().lower() + if not text or text in {"*", "all", "list"}: + return True + + haystacks = [ + str(entry.get("name") or "").lower(), + str(entry.get("scp_path") or "").lower(), + ] + for token in [part for part in text.split() if part]: + if any(ch in token for ch in "*?[]"): + if not any(fnmatch.fnmatch(haystack, token) for haystack in haystacks): + return False + elif not any(token in haystack for haystack in haystacks): + return False + return True + + def _build_result(self, entry: Dict[str, Any]) -> SearchResult: + scp_path = str(entry.get("scp_path") or "/") + scp_url = self._build_url(scp_path) + is_dir = bool(entry.get("is_dir")) + size_value = entry.get("size") + modified = str(entry.get("modified") or "") + parent = posixpath.dirname(scp_path.rstrip("/")) or "/" + metadata = { + "provider": "scp", + "host": self._host, + "scp_path": scp_path, + "scp_url": scp_url, + "selection_url": scp_url, + "is_dir": is_dir, + "name": str(entry.get("name") or "").strip(), + } + if size_value is not None: + metadata["size"] = size_value + if modified: + metadata["modified"] = modified + + return SearchResult( + table="scp", + title=str(entry.get("name") or scp_path), + path=scp_url, + detail=parent, + annotations=["folder" if is_dir else "file"], + media_kind="folder" if is_dir else "file", + size_bytes=int(size_value) if isinstance(size_value, int) else None, + tag={"scp", "folder" if is_dir else "file"}, + columns=[ + ("Name", str(entry.get("name") or "")), + ("Type", "dir" if is_dir else "file"), + ("Directory", parent), + ("Size", "" if size_value is None else str(size_value)), + ("Modified", modified), + ], + selection_args=None if is_dir else ["-url", scp_url], + selection_action=None if is_dir else ["download-file", "-plugin", "scp", "-url", scp_url], + full_metadata=metadata, + ) + + def _list_directory(self, sftp: Any, remote_path: str) -> List[Dict[str, Any]]: + try: + attrs = sftp.listdir_attr(remote_path) + except Exception: + return [] + + entries: List[Dict[str, Any]] = [] + for attr in attrs: + name_text = str(getattr(attr, "filename", "") or "").strip() + if not name_text or name_text in {".", ".."}: + continue + mode = getattr(attr, "st_mode", 0) + is_dir = stat.S_ISDIR(mode) + size_value = getattr(attr, "st_size", None) + try: + size_int = int(size_value) if size_value is not None else None + except Exception: + size_int = None + entries.append( + { + "name": name_text, + "scp_path": self._join_remote_path(remote_path, name_text), + "is_dir": is_dir, + "size": size_int, + "modified": _format_epoch(getattr(attr, "st_mtime", None)), + } + ) + return entries + + def _list_directory_via_ssh(self, ssh: Any, remote_path: str, *, depth: int) -> List[Dict[str, Any]]: + normalized = self._normalize_remote_path(remote_path, default=self._base_path) + max_depth = max(1, int(depth) + 1) + quoted_path = shlex.quote(normalized) + command = ( + f"find {quoted_path} -mindepth 1 -maxdepth {max_depth} " + f"\\( -type d -o -type f \\) -exec sh -c 'for path do " + f"if [ -d \"$path\" ]; then kind=d; else kind=f; fi; " + f"name=$(basename \"$path\"); " + f"printf \"%s\\0%s\\0%s\\0\" \"$kind\" \"$path\" \"$name\"; " + f"done' sh {{}} +" + ) + status, output, error = self._run_ssh_command(ssh, command) + if status != 0: + error_text = error.strip().lower() + if "no such file" in error_text or "cannot access" in error_text: + return [] + raise RuntimeError(error.strip() or f"SSH listing failed for {normalized}") + + chunks = [part for part in output.split("\0") if part] + entries: List[Dict[str, Any]] = [] + for index in range(0, len(chunks), 3): + if index + 2 >= len(chunks): + break + kind = chunks[index] + scp_path = self._normalize_remote_path(chunks[index + 1], default=normalized) + name_text = str(chunks[index + 2] or "").strip() + if not name_text or name_text in {".", ".."}: + continue + entries.append( + { + "name": name_text, + "scp_path": scp_path, + "is_dir": kind == "d", + "size": None, + "modified": "", + } + ) + return entries + + def _ensure_directory(self, sftp: Any, remote_path: str) -> None: + normalized = self._normalize_remote_path(remote_path, default=self._base_path) + if normalized == "/": + return + partial = "" + for segment in [part for part in normalized.split("/") if part]: + partial = f"{partial}/{segment}" + try: + attrs = sftp.stat(partial) + if stat.S_ISDIR(getattr(attrs, "st_mode", 0)): + continue + except Exception: + pass + try: + sftp.mkdir(partial) + except Exception: + try: + attrs = sftp.stat(partial) + if stat.S_ISDIR(getattr(attrs, "st_mode", 0)): + continue + except Exception: + pass + raise + + def _item_metadata(self, item: Any, *, pipe_obj: Any = None) -> Dict[str, Any]: + metadata: Dict[str, Any] = {} + for source in (item, pipe_obj): + if isinstance(source, dict): + for key in ("title", "path", "url"): + if source.get(key) is not None and key not in metadata: + metadata[key] = source.get(key) + nested = source.get("full_metadata") or source.get("metadata") + if isinstance(nested, dict): + metadata.update(nested) + elif source is not None: + for attr in ("title", "path", "url"): + try: + value = getattr(source, attr, None) + except Exception: + value = None + if value is not None and attr not in metadata: + metadata[attr] = value + try: + nested = getattr(source, "full_metadata", None) or getattr(source, "metadata", None) + except Exception: + nested = None + if isinstance(nested, dict): + metadata.update(nested) + + scp_path = metadata.get("scp_path") or metadata.get("selection_path") + if not scp_path: + path_value = metadata.get("path") or metadata.get("url") or metadata.get("scp_url") + path_text = str(path_value or "").strip() + if path_text.startswith(("scp://", "sftp://")): + scp_path = self._normalize_remote_path(path_text, default=self._base_path) + if scp_path: + metadata["scp_path"] = self._normalize_remote_path(scp_path, default=self._base_path) + metadata.setdefault("selection_path", metadata["scp_path"]) + + if metadata.get("scp_path") and not metadata.get("scp_url"): + metadata["scp_url"] = self._build_url(metadata["scp_path"]) + if metadata.get("scp_url") and not metadata.get("selection_url"): + metadata["selection_url"] = metadata["scp_url"] + + is_dir = metadata.get("is_dir") + if is_dir is None and metadata.get("media_kind"): + is_dir = str(metadata.get("media_kind") or "").strip().lower() == "folder" + metadata["is_dir"] = bool(is_dir) + return metadata \ No newline at end of file diff --git a/plugins/soulseek/__init__.py b/plugins/soulseek/__init__.py index b694632..5336643 100644 --- a/plugins/soulseek/__init__.py +++ b/plugins/soulseek/__init__.py @@ -12,10 +12,18 @@ from pathlib import Path from typing import Any, Dict, List, Optional from ProviderCore.base import Provider, SearchResult -from SYS.logger import log, debug +from SYS.logger import log, debug, debug_panel from SYS.models import ProgressBar _SOULSEEK_NOISE_SUBSTRINGS = ( + "unhandled exception on loop", + "Task exception was never retrieved", + "future: Any: if msg == "Task exception was never retrieved": cls = getattr(exc, "__class__", None) name = getattr(cls, "__name__", "") - mod = getattr(cls, "__module__", "") + exc_text = str(exc or "").lower() - # Suppress ConnectionFailedError from aioslsk - if name == "ConnectionFailedError" and str(mod).startswith("aioslsk"): + # Suppress expected peer direct-connect failures from aioslsk. + if name == "ConnectionFailedError" or "failed to connect" in exc_text: return except Exception: # If our filter logic fails, fall through to default handling. @@ -117,6 +125,9 @@ class _LineFilterStream(io.TextIOBase): self._in_tb = False self._tb_lines: list[str] = [] self._tb_suppress = False + self._in_task_block = False + self._task_lines: list[str] = [] + self._task_suppress = False def writable(self) -> bool: # pragma: no cover return True @@ -137,6 +148,19 @@ class _LineFilterStream(io.TextIOBase): self._tb_suppress = False self._in_tb = False + def _flush_task_block(self) -> None: + if not self._task_lines: + return + if not self._task_suppress: + for l in self._task_lines: + try: + self._underlying.write(l + "\n") + except Exception: + pass + self._task_lines = [] + self._task_suppress = False + self._in_task_block = False + def write(self, s: str) -> int: self._buf += str(s) while "\n" in self._buf: @@ -145,6 +169,29 @@ class _LineFilterStream(io.TextIOBase): return len(s) def _handle_line(self, line: str) -> None: + if not self._in_task_block and self._should_suppress_line(line) and ( + line.startswith("Task exception was never retrieved") + or line.startswith("future: None: - end = time.time() + timeout + ) -> Dict[str, Any]: + start = time.time() + end = start + timeout last_count = 0 + update_count = 0 while time.time() < end: current_count = len(getattr(search_request, "results", [])) if current_count > last_count: - debug(f"[soulseek] Got {current_count} result(s)...") last_count = current_count + update_count += 1 await asyncio.sleep(0.5) + return { + "peer_hits": last_count, + "count_updates": update_count, + "elapsed_seconds": round(max(0.0, time.time() - start), 1), + } + def search( self, query: str, @@ -503,7 +562,7 @@ class Soulseek(Provider): base_tmp.mkdir(parents=True, exist_ok=True) try: - flat_results = asyncio.run( + flat_results, search_summary = asyncio.run( self.perform_search(query, timeout=9.0, limit=limit) @@ -636,6 +695,23 @@ class Soulseek(Provider): ) ) + try: + debug_panel( + "soulseek search", + [ + ("query", query), + ("peer_hits", search_summary.get("peer_hits", 0)), + ("file_hits", len(flat_results)), + ("audio_hits", len(music_results)), + ("results", len(results)), + ("poll_updates", search_summary.get("count_updates", 0)), + ("elapsed_s", search_summary.get("elapsed_seconds", 0.0)), + ], + border_style="magenta", + ) + except Exception: + pass + return results except Exception as exc: diff --git a/plugins/ytdlp/__init__.py b/plugins/ytdlp/__init__.py index 26bb54a..e1cdd9c 100644 --- a/plugins/ytdlp/__init__.py +++ b/plugins/ytdlp/__init__.py @@ -1286,9 +1286,30 @@ class ytdlp(TableProviderMixin, Provider): try: - from SYS.result_table_adapters import register_plugin + from SYS.result_table_adapters import get_plugin, register_plugin from SYS.result_table_api import ResultModel, ColumnSpec, metadata_column, title_column + def _register_table_plugin_once( + name: str, + adapter: Any, + *, + columns: Any, + selection_fn: Any, + metadata: Optional[Dict[str, Any]] = None, + ) -> None: + try: + get_plugin(name) + return + except KeyError: + pass + register_plugin( + name, + adapter, + columns=columns, + selection_fn=selection_fn, + metadata=metadata, + ) + def _convert_format_result_to_model(sr: Any) -> ResultModel: d = sr.to_dict() if hasattr(sr, "to_dict") else (sr if isinstance(sr, dict) else {}) title = d.get("title") or f"Format {d.get('format_id', 'unknown')}" @@ -1361,7 +1382,7 @@ try: return result_args return [] - register_plugin( + _register_table_plugin_once( "ytdlp.formatlist", _adapter, columns=_columns_factory, @@ -1421,7 +1442,7 @@ try: return ["-url", row.path] return ["-title", row.title or ""] - register_plugin( + _register_table_plugin_once( "ytdlp.search", _search_adapter, columns=_search_columns_factory, diff --git a/scripts/pyproject.toml b/scripts/pyproject.toml index d2030ff..d67eede 100644 --- a/scripts/pyproject.toml +++ b/scripts/pyproject.toml @@ -70,6 +70,8 @@ dependencies = [ # Browser automation "playwright>=1.40.0", + "paramiko>=3.5.0", + "scp>=0.15.0", # Development and utilities "python-dateutil>=2.8.0",