updated plugin refactor and added FTP and SCP plugins , also hydrusnetwork plugin migration

This commit is contained in:
2026-04-27 21:17:53 -07:00
parent bfd5c20dc3
commit 8685fbb723
24 changed files with 3650 additions and 405 deletions
+1
View File
@@ -657,6 +657,7 @@ class CmdletCompleter(Completer):
return value
def _used_arg_logicals(
self,
cmd_name: str,
stage_tokens: List[str],
config: Dict[str,
+18
View File
@@ -422,6 +422,24 @@ class Provider(ABC):
_ = quiet_mode
return None
def config_helper_text(self) -> str:
"""Optional helper text shown in the config editor."""
return ""
def config_actions(self) -> List[Dict[str, Any]]:
"""Optional actions exposed in the config editor for this provider."""
return []
def run_config_action(self, action_id: str, **_kwargs: Any) -> Dict[str, Any]:
"""Execute a provider-owned config action from the config editor."""
return {
"ok": False,
"message": f"Provider '{self.name}' does not support config action '{action_id}'.",
}
def upload(self, file_path: str, **kwargs: Any) -> str:
"""Upload a file and return a URL or identifier."""
raise NotImplementedError(f"Provider '{self.name}' does not support upload")
+23 -8
View File
@@ -136,7 +136,28 @@ class ProviderRegistry:
self._discovered = False
self._external_dirs_scanned = False
def _ensure_builtin_package_dirs(self) -> None:
if self._builtin_package_dirs or not self.package_name:
return
try:
package = importlib.import_module(self.package_name)
except Exception:
return
package_path = getattr(package, "__path__", None)
if not package_path:
return
builtin_dirs: List[Path] = []
for entry in package_path:
try:
builtin_dirs.append(Path(str(entry)).resolve())
except Exception:
builtin_dirs.append(Path(str(entry)))
self._builtin_package_dirs = tuple(builtin_dirs)
def _is_builtin_package_dir(self, candidate: Path) -> bool:
self._ensure_builtin_package_dirs()
try:
resolved = candidate.resolve()
except Exception:
@@ -253,6 +274,7 @@ class ProviderRegistry:
if self._external_dirs_scanned:
return
self._external_dirs_scanned = True
self._ensure_builtin_package_dirs()
for plugin_dir in _iter_external_plugin_dirs():
if self._is_builtin_package_dir(plugin_dir):
@@ -302,15 +324,8 @@ class ProviderRegistry:
return
self._register_module(package)
self._ensure_builtin_package_dirs()
package_path = getattr(package, "__path__", None)
if package_path:
builtin_dirs: List[Path] = []
for entry in package_path:
try:
builtin_dirs.append(Path(str(entry)).resolve())
except Exception:
builtin_dirs.append(Path(str(entry)))
self._builtin_package_dirs = tuple(builtin_dirs)
if not package_path:
self._discover_external_plugins()
return
+144 -39
View File
@@ -8,6 +8,7 @@ import time
import os
import re
import datetime
import shutil
import sys
import tempfile
from copy import deepcopy
@@ -532,7 +533,7 @@ def resolve_cookies_path(
if candidate.is_file():
return candidate
plugin_cookie = resolve_plugin_asset_path("ytdlp", "cookies.txt", script_dir=base_dir)
plugin_cookie = _resolve_ytdlp_plugin_cookie_path(base_dir)
if plugin_cookie is not None:
return plugin_cookie
@@ -542,6 +543,30 @@ def resolve_cookies_path(
return None
def _resolve_ytdlp_plugin_cookie_path(base_dir: Path) -> Optional[Path]:
plugin_cookie = resolve_plugin_asset_path("ytdlp", "cookies.txt", script_dir=base_dir)
if plugin_cookie is not None:
return plugin_cookie
plugin_dir = _resolve_app_root(base_dir) / "plugins" / "ytdlp"
if not plugin_dir.is_dir():
return None
plugin_cookie = plugin_dir / "cookies.txt"
legacy_cookie = _resolve_app_root(base_dir) / "cookies.txt"
try:
if legacy_cookie.is_file() and not plugin_cookie.exists():
plugin_cookie.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(legacy_cookie, plugin_cookie)
return plugin_cookie
except Exception:
return None
if plugin_cookie.is_file():
return plugin_cookie
return None
def resolve_debug_log(config: Dict[str, Any]) -> Optional[Path]:
value = config.get("download_debug_log")
if not value:
@@ -721,6 +746,105 @@ def _count_changed_entries(old_config: Dict[str, Any], new_config: Dict[str, Any
return len(changed) + len(removed)
def _changed_entry_keys(old_config: Dict[str, Any], new_config: Dict[str, Any]) -> set[Tuple[str, str, str, str]]:
old_entries = _flatten_config_entries(old_config or {})
new_entries = _flatten_config_entries(new_config or {})
keys = set(old_entries) | set(new_entries)
return {key for key in keys if old_entries.get(key, _CONFIG_MISSING) != new_entries.get(key, _CONFIG_MISSING)}
def _config_from_flattened_entries(
entries: Dict[Tuple[str, str, str, str], Any],
) -> Dict[str, Any]:
config: Dict[str, Any] = {}
for (category, subtype, item_name, key), value in entries.items():
if category == "global":
config[key] = value
continue
if category == "store":
store_block = config.setdefault("store", {})
subtype_block = store_block.setdefault(subtype, {})
item_block = subtype_block.setdefault(item_name, {})
item_block[key] = value
continue
if category in {"provider", "tool"}:
category_block = config.setdefault(category, {})
subtype_block = category_block.setdefault(subtype, {})
subtype_block[key] = value
continue
category_block = config.setdefault(category, {})
if isinstance(category_block, dict):
subtype_block = category_block.setdefault(subtype, {})
if isinstance(subtype_block, dict):
item_block = subtype_block.setdefault(item_name, {})
if isinstance(item_block, dict):
item_block[key] = value
_normalize_plugin_config_aliases(config)
_sync_alldebrid_api_key(config)
return config
def _merge_non_conflicting_config_changes(
base_config: Dict[str, Any],
disk_config: Dict[str, Any],
local_config: Dict[str, Any],
) -> Optional[Dict[str, Any]]:
local_changed = _changed_entry_keys(base_config, local_config)
if not local_changed:
return deepcopy(disk_config)
disk_changed = _changed_entry_keys(base_config, disk_config)
if local_changed & disk_changed:
return None
merged_entries = dict(_flatten_config_entries(disk_config or {}))
local_entries = _flatten_config_entries(local_config or {})
for key in local_changed:
if key in local_entries:
merged_entries[key] = local_entries[key]
else:
merged_entries.pop(key, None)
return _config_from_flattened_entries(merged_entries)
def _extract_expected_alldebrid_key(config: Dict[str, Any]) -> Optional[str]:
expected_key = None
try:
providers = config.get("provider", {}) if isinstance(config, dict) else {}
if isinstance(providers, dict):
entry = providers.get("alldebrid")
if entry is not None:
if isinstance(entry, dict):
for k in ("api_key", "API_KEY", "apikey", "APIKEY"):
v = entry.get(k)
if isinstance(v, str) and v.strip():
expected_key = v.strip()
break
elif isinstance(entry, str) and entry.strip():
expected_key = entry.strip()
if not expected_key:
store_block = config.get("store", {}) if isinstance(config, dict) else {}
debrid = store_block.get("debrid") if isinstance(store_block, dict) else None
if isinstance(debrid, dict):
srv = debrid.get("all-debrid")
if isinstance(srv, dict):
for k in ("api_key", "API_KEY", "apikey", "APIKEY"):
v = srv.get(k)
if isinstance(v, str) and v.strip():
expected_key = v.strip()
break
elif isinstance(srv, str) and srv.strip():
expected_key = srv.strip()
except Exception as exc:
logger.debug("Failed to determine expected AllDebrid key: %s", exc, exc_info=True)
expected_key = None
return expected_key
def load_config(*, emit_summary: bool = True) -> Dict[str, Any]:
global _CONFIG_CACHE, _LAST_SAVED_CONFIG, _CONFIG_SUMMARY_PENDING
if _CONFIG_CACHE:
@@ -838,6 +962,7 @@ def save_config(config: Dict[str, Any]) -> int:
def _write_entries() -> int:
global _CONFIG_CACHE, _LAST_SAVED_CONFIG
count = 0
config_to_write = config
# Use the transaction-provided connection directly to avoid re-acquiring
# the connection lock via db.* helpers which can lead to deadlock.
with db.transaction() as conn:
@@ -861,14 +986,22 @@ def save_config(config: Dict[str, Any]) -> int:
_CONFIG_CACHE = current_disk
_LAST_SAVED_CONFIG = deepcopy(current_disk)
return 0
# Otherwise, abort to avoid overwriting external changes
raise ConfigSaveConflict(
"Configuration on disk changed since you started editing; save aborted to prevent overwrite. Reload and reapply your changes."
merged_config = _merge_non_conflicting_config_changes(
previous_config,
current_disk,
config,
)
if merged_config is None:
# Otherwise, abort to avoid overwriting external changes
raise ConfigSaveConflict(
"Configuration on disk changed since you started editing; save aborted to prevent overwrite. Reload and reapply your changes."
)
config_to_write = merged_config
log("Config save rebased local changes onto newer disk configuration.")
# Proceed with writing when no conflicting external changes detected
conn.execute("DELETE FROM config")
for key, value in config.items():
for key, value in config_to_write.items():
if key in ('store', 'provider', 'tool') and isinstance(value, dict):
for subtype, instances in value.items():
if not isinstance(instances, dict):
@@ -904,6 +1037,8 @@ def save_config(config: Dict[str, Any]) -> int:
("global", "none", "none", key, val_str),
)
count += 1
_CONFIG_CACHE = config_to_write
_LAST_SAVED_CONFIG = deepcopy(config_to_write)
return count
@@ -964,9 +1099,6 @@ def save_config(config: Dict[str, Any]) -> int:
logger.exception("Failed to release save lock after CRITICAL configuration save failure: %s", exc)
raise
clear_config_cache()
_CONFIG_CACHE = config
_LAST_SAVED_CONFIG = deepcopy(config)
return saved_entries
@@ -988,37 +1120,10 @@ def save_config_and_verify(config: Dict[str, Any], retries: int = 3, delay: floa
AllDebrid) were written successfully. If verification fails after the
configured number of retries, a RuntimeError is raised.
"""
# Detect an API key that should be verified (provider or store-backed)
expected_key = None
try:
providers = config.get("provider", {}) if isinstance(config, dict) else {}
if isinstance(providers, dict):
entry = providers.get("alldebrid")
if entry is not None:
# _extract_api_key is a small internal helper; reuse the implementation here
if isinstance(entry, dict):
for k in ("api_key", "API_KEY", "apikey", "APIKEY"):
v = entry.get(k)
if isinstance(v, str) and v.strip():
expected_key = v.strip()
break
elif isinstance(entry, str) and entry.strip():
expected_key = entry.strip()
if not expected_key:
store_block = config.get("store", {}) if isinstance(config, dict) else {}
debrid = store_block.get("debrid") if isinstance(store_block, dict) else None
if isinstance(debrid, dict):
srv = debrid.get("all-debrid")
if isinstance(srv, dict):
for k in ("api_key", "API_KEY", "apikey", "APIKEY"):
v = srv.get(k)
if isinstance(v, str) and v.strip():
expected_key = v.strip()
break
elif isinstance(srv, str) and srv.strip():
expected_key = srv.strip()
except Exception as exc:
logger.debug("Failed to determine expected key for save verification: %s", exc, exc_info=True)
# Only perform the extra verification loop when the AllDebrid key actually changed.
expected_key = _extract_expected_alldebrid_key(config)
baseline_key = _extract_expected_alldebrid_key(_LAST_SAVED_CONFIG)
if expected_key == baseline_key:
expected_key = None
last_exc: Exception | None = None
+29
View File
@@ -89,6 +89,8 @@ class HydrusNetwork(Store):
Maintains its own HydrusClient.
"""
STORE_TYPE = "hydrusnetwork"
@classmethod
def config_schema(cls) -> List[Dict[str, Any]]:
return [
@@ -1776,6 +1778,33 @@ class HydrusNetwork(Store):
debug(f"{self._log_prefix()} delete_file failed: {exc}")
return False
def build_file_url(self, file_hash: str, *, include_access_key: bool = True) -> str:
normalized = str(file_hash or "").strip().lower()
base_url = str(self.URL).rstrip("/")
url = f"{base_url}/get_files/file?hash={quote(normalized)}"
if include_access_key and str(self.API or "").strip():
url = f"{url}&Hydrus-Client-API-Access-Key={quote(str(self.API))}"
return url
def fetch_file_metadata(self, file_hash: str, **kwargs: Any) -> Optional[Dict[str, Any]]:
try:
client = self._client
if client is None:
return None
return client.fetch_file_metadata(hashes=[str(file_hash or "").strip().lower()], **kwargs)
except Exception:
return None
def get_relationships(self, file_hash: str) -> Optional[Dict[str, Any]]:
try:
client = self._client
if client is None:
return None
payload = client.get_file_relationships(str(file_hash or "").strip().lower())
return payload if isinstance(payload, dict) else None
except Exception:
return None
def get_metadata(self, file_hash: str, **kwargs: Any) -> Optional[Dict[str, Any]]:
"""Get metadata for a file from Hydrus by hash.
+122 -1
View File
@@ -32,7 +32,7 @@ from SYS.plugin_config import (
get_item_schema_map,
get_required_config_keys,
)
from ProviderCore.registry import get_plugin
from ProviderCore.registry import get_plugin, get_plugin_class
from TUI.modalscreen.matrix_room_picker import MatrixRoomPicker
from TUI.modalscreen.selection_modal import SelectionModal
import logging
@@ -164,9 +164,12 @@ class ConfigModal(ModalScreen):
self.editing_item_type = None # 'store' or 'provider'
self.editing_item_name = None
self._button_id_map = {}
self._provider_button_map: Dict[str, tuple[str, str]] = {}
self._input_id_map = {}
self._matrix_status: Optional[Static] = None
self._matrix_test_running = False
self._provider_status: Optional[Static] = None
self._provider_action_running = False
self._editor_snapshot: Optional[Dict[str, Any]] = None
# Inline matrix rooms controls
self._matrix_inline_list: Optional[ListView] = None
@@ -256,6 +259,7 @@ class ConfigModal(ModalScreen):
return
self._button_id_map.clear()
self._provider_button_map.clear()
self._input_id_map.clear()
# Clear existing
@@ -594,6 +598,33 @@ class ConfigModal(ModalScreen):
row.mount(Button("Paste", id=f"paste-{inp_id}", classes="paste-btn"))
idx += 1
if item_type == "provider" and isinstance(item_name, str):
provider = self._instantiate_provider_for_editor(item_name, self.config_data)
if provider is not None:
provider_actions = provider.config_actions() or []
if provider_actions:
container.mount(Rule())
container.mount(Label(f"{provider.label} helpers", classes="config-label"))
helper_text = str(provider.config_helper_text() or "Use these helpers to validate provider settings.").strip()
status = Static(helper_text, id="provider-status")
container.mount(status)
self._provider_status = status
row = Horizontal(classes="field-row")
container.mount(row)
for action in provider_actions:
action_id = str(action.get("id") or "").strip()
if not action_id:
continue
button_id = f"provider-action-{item_name}-{action_id}".replace(" ", "-")
self._provider_button_map[button_id] = (item_name, action_id)
row.mount(
Button(
str(action.get("label") or action_id.replace("_", " ").title()),
id=button_id,
variant=str(action.get("variant") or "default"),
)
)
if (
item_type == "provider"
and isinstance(item_name, str)
@@ -755,6 +786,10 @@ class ConfigModal(ModalScreen):
self.editing_item_type = None
self.refresh_view()
except Exception as exc:
try:
log(f"Configuration save failed: {exc}")
except Exception:
logger.exception("Failed to write save failure to logs")
self.notify(f"Save failed: {exc}", severity="error", timeout=10)
elif bid == "save-durable-btn":
# Perform a synchronous, verified save and notify status to the user.
@@ -788,6 +823,10 @@ class ConfigModal(ModalScreen):
self.refresh_view()
self._editor_snapshot = None
except Exception as exc:
try:
log(f"Durable configuration save failed: {exc}")
except Exception:
logger.exception("Failed to write durable save failure to logs")
self.notify(f"Durable save failed: {exc}", severity="error", timeout=10)
try:
log(f"Durable save failed: {exc}")
@@ -823,8 +862,15 @@ class ConfigModal(ModalScreen):
saved = self.save_all()
self.notify("Saving configuration...", timeout=3)
except Exception as exc:
try:
log(f"Configuration save failed while deleting config entry: {exc}")
except Exception:
logger.exception("Failed to write config delete save failure to logs")
self.notify(f"Save failed: {exc}", severity="error", timeout=10)
self.refresh_view()
elif bid in self._provider_button_map:
provider_name, action_id = self._provider_button_map[bid]
self._request_provider_action(provider_name, action_id)
elif bid == "add-store-btn":
options = get_configurable_store_types()
self.app.push_screen(SelectionModal("Select Store Type", options), callback=self.on_store_type_selected)
@@ -882,6 +928,10 @@ class ConfigModal(ModalScreen):
try:
entries = save_config(self.config_data)
except Exception as exc:
try:
log(f"Saving Matrix default rooms failed: {exc}")
except Exception:
logger.exception("Failed to write Matrix room save failure to logs")
if self._matrix_status:
self._matrix_status.update(f"Saving default rooms failed: {exc}")
return
@@ -921,6 +971,69 @@ class ConfigModal(ModalScreen):
else:
self.notify("Clipboard not supported in this terminal", severity="warning")
def _instantiate_provider_for_editor(self, provider_name: str, config_data: Optional[Dict[str, Any]] = None) -> Optional[Any]:
try:
provider_class = get_plugin_class(provider_name)
except Exception:
provider_class = None
if provider_class is None:
return None
try:
return provider_class(config_data or self.config_data)
except Exception:
logger.exception("Failed to instantiate provider '%s' for config helper", provider_name)
return None
def _request_provider_action(self, provider_name: str, action_id: str) -> None:
if self._provider_action_running:
return
self._synchronize_inputs_to_config()
self._provider_action_running = True
if self._provider_status is not None:
self._provider_status.update(f"Running {action_id.replace('_', ' ')}")
self._provider_action_background(provider_name, action_id, deepcopy(self.config_data))
@work(thread=True)
def _provider_action_background(self, provider_name: str, action_id: str, config_snapshot: Dict[str, Any]) -> None:
try:
provider = self._instantiate_provider_for_editor(provider_name, config_snapshot)
if provider is None:
raise RuntimeError(f"Provider '{provider_name}' is unavailable")
result = provider.run_config_action(action_id)
if not isinstance(result, dict):
result = {"ok": False, "message": f"Provider '{provider_name}' returned an invalid config action result."}
except Exception as exc:
result = {"ok": False, "message": str(exc) or f"Provider action '{action_id}' failed."}
try:
self.app.call_from_thread(self._provider_action_complete, provider_name, action_id, result)
except Exception:
self._provider_action_complete(provider_name, action_id, result)
def _provider_action_complete(self, provider_name: str, action_id: str, result: Dict[str, Any]) -> None:
self._provider_action_running = False
ok = bool(result.get("ok"))
message = str(result.get("message") or f"Provider action '{action_id}' finished.")
updates = result.get("config_updates")
if ok and isinstance(updates, dict):
provider_block = self.config_data.setdefault("provider", {}).setdefault(provider_name, {})
if isinstance(provider_block, dict):
provider_block.update(updates)
message = f"{message}"
try:
self.refresh_view()
except Exception:
logger.exception("Failed to refresh config view after provider action")
if self._provider_status is not None:
self._provider_status.update(message)
try:
self.notify(message, severity="error" if not ok else "information", timeout=8)
except Exception:
logger.exception("Failed to notify provider action result for %s/%s", provider_name, action_id)
# Backup/restore helpers removed: forensics/audit mode disabled and restore UI removed.
def on_store_type_selected(self, stype: str) -> None:
@@ -1130,6 +1243,10 @@ class ConfigModal(ModalScreen):
try:
entries = save_config(self.config_data)
except Exception as exc:
try:
log(f"Saving configuration before Matrix test failed: {exc}")
except Exception:
logger.exception("Failed to write Matrix test pre-save failure to logs")
if self._matrix_status:
self._matrix_status.update(f"Saving configuration failed: {exc}")
self._matrix_test_running = False
@@ -1290,6 +1407,10 @@ class ConfigModal(ModalScreen):
try:
entries = save_config(self.config_data)
except Exception as exc:
try:
log(f"Saving configuration before Matrix room load failed: {exc}")
except Exception:
logger.exception("Failed to write Matrix load pre-save failure to logs")
if self._matrix_status:
self._matrix_status.update(f"Saving configuration failed: {exc}")
self._matrix_test_running = False
+55 -69
View File
@@ -1410,7 +1410,7 @@ def fetch_hydrus_metadata(
Eliminates repeated boilerplate: client initialization, error handling, metadata extraction.
Args:
config: Configuration object (passed to hydrus_wrapper.get_client)
config: Configuration object used to resolve the Hydrus provider/store
hash_hex: File hash to fetch metadata for
store_name: Optional Hydrus store name. When provided, do not fall back to a global/default Hydrus client.
hydrus_client: Optional explicit Hydrus client. When provided, takes precedence.
@@ -1422,38 +1422,53 @@ def fetch_hydrus_metadata(
- metadata_dict: Dict from Hydrus (first item in metadata list) or None if unavailable
- error_code: 0 on success, 1 on any error (suitable for returning from cmdlet execute())
"""
from API import HydrusNetwork
hydrus_wrapper = HydrusNetwork
client = hydrus_client
hydrus_provider = None
try:
from ProviderCore.registry import get_plugin
hydrus_provider = get_plugin("hydrusnetwork", config)
except Exception:
hydrus_provider = None
if client is None:
if store_name:
# Store specified: do not fall back to a global/default Hydrus client.
if hydrus_provider is not None:
try:
from Store import Store
store = Store(config)
backend = store[str(store_name)]
candidate = getattr(backend, "_client", None)
if candidate is not None and hasattr(candidate, "fetch_file_metadata"):
client = candidate
client = hydrus_provider.get_client(
store_name=store_name if store_name else None,
allow_default=not bool(store_name),
)
except Exception as exc:
log(f"Hydrus client unavailable for store '{store_name}': {exc}")
if store_name:
log(f"Hydrus client unavailable for store '{store_name}': {exc}")
else:
log(f"Hydrus client unavailable: {exc}")
client = None
if client is None:
log(f"Hydrus client unavailable for store '{store_name}'")
return None, 1
else:
try:
client = hydrus_wrapper.get_client(config)
except Exception as exc:
log(f"Hydrus client unavailable: {exc}")
return None, 1
if client is None and store_name:
log(f"Hydrus client unavailable for store '{store_name}'")
return None, 1
if client is None and hydrus_provider is None:
log("Hydrus provider unavailable")
return None, 1
if client is None:
log("Hydrus client unavailable")
return None, 1
if hydrus_provider is not None:
try:
metadata = hydrus_provider.fetch_metadata(
hash_hex,
store_name=store_name if store_name else None,
**kwargs,
)
except Exception as exc:
log(f"Hydrus metadata fetch failed: {exc}")
return None, 1
if isinstance(metadata, dict):
return metadata, 0
if client is None:
if store_name:
log(f"Hydrus client unavailable for store '{store_name}'")
else:
log("Hydrus metadata unavailable")
return None, 1
try:
payload = client.fetch_file_metadata(hashes=[hash_hex], **kwargs)
@@ -3725,10 +3740,13 @@ def check_url_exists_in_storage(
match_rows: List[Dict[str, Any]] = []
max_rows = 200
hydrus_provider = None
try:
from Store.HydrusNetwork import HydrusNetwork
from ProviderCore.registry import get_plugin
hydrus_provider = get_plugin("hydrusnetwork", config)
except Exception:
HydrusNetwork = None # type: ignore
hydrus_provider = None
for backend_name in backend_names:
if _timed_out("backend scan"):
@@ -3739,8 +3757,14 @@ def check_url_exists_in_storage(
backend = storage[backend_name]
except Exception:
continue
if HydrusNetwork is not None and isinstance(backend, HydrusNetwork):
is_hydrus_backend = False
try:
is_hydrus_backend = bool(hydrus_provider and hydrus_provider.is_backend(backend, str(backend_name)))
except Exception:
is_hydrus_backend = False
if is_hydrus_backend:
if not hydrus_available:
debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup")
@@ -3776,44 +3800,6 @@ def check_url_exists_in_storage(
found = True
break
client = getattr(backend, "_client", None)
if found:
pass
elif client is None:
continue
for needle in (needles or [])[:6]:
if found:
break
if not _httpish(needle):
continue
try:
from API.HydrusNetwork import HydrusRequestSpec
spec = HydrusRequestSpec(
method="GET",
endpoint="/add_urls/get_url_files",
query={"url": needle},
)
if hasattr(client, "_perform_request"):
response = client._perform_request(spec)
raw_hashes = None
if isinstance(response, dict):
raw_hashes = response.get("hashes") or response.get("file_hashes")
raw_ids = response.get("file_ids")
hash_list = raw_hashes if isinstance(raw_hashes, list) else []
has_ids = isinstance(raw_ids, list) and len(raw_ids) > 0
has_hashes = len(hash_list) > 0
if has_hashes:
try:
found_hash = str(hash_list[0]).strip()
except Exception:
found_hash = None
if has_ids or has_hashes:
found = True
break
except Exception:
continue
if not found:
continue
+11 -6
View File
@@ -9,9 +9,9 @@ import sys
from SYS.logger import log
from SYS.item_accessors import get_sha256_hex, get_store_name
from ProviderCore.registry import get_plugin
from SYS import pipeline as ctx
from API import HydrusNetwork as hydrus_wrapper
from . import _shared as sh
Cmdlet = sh.Cmdlet
@@ -617,16 +617,20 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
# - If no store is specified, use the default Hydrus client.
# NOTE: When a store is specified, we do not fall back to a global/default Hydrus client.
hydrus_client = None
hydrus_provider = get_plugin("hydrusnetwork", config)
if store_name and (not is_folder_store) and backend is not None:
try:
candidate = getattr(backend, "_client", None)
if candidate is not None and hasattr(candidate, "set_relationship"):
hydrus_client = candidate
if hydrus_provider is not None:
hydrus_client = hydrus_provider.get_client(
store_name=str(store_name),
allow_default=False,
)
except Exception:
hydrus_client = None
elif not store_name:
try:
hydrus_client = hydrus_wrapper.get_client(config)
if hydrus_provider is not None:
hydrus_client = hydrus_provider.get_client()
except Exception:
hydrus_client = None
@@ -1049,8 +1053,9 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
return 1
# Build Hydrus client
hydrus_provider = get_plugin("hydrusnetwork", config)
try:
hydrus_client = hydrus_wrapper.get_client(config)
hydrus_client = hydrus_provider.get_client() if hydrus_provider is not None else None
except Exception as exc:
log(f"Hydrus client unavailable: {exc}", file=sys.stderr)
return 1
+7 -66
View File
@@ -13,6 +13,7 @@ from typing import Any, Dict, List, Sequence, Set
from urllib.parse import parse_qs, urlparse
from SYS.logger import log
from ProviderCore.registry import get_plugin
from SYS.item_accessors import get_http_url, get_sha256_hex, get_store_name
from SYS.utils import extract_hydrus_hash_from_url
@@ -71,10 +72,8 @@ def _maybe_download_hydrus_item(
This is intentionally side-effect free except for writing the local temp file.
"""
try:
from SYS.config import get_hydrus_access_key, get_hydrus_url
from API.HydrusNetwork import HydrusNetwork as HydrusClient, download_hydrus_file
except Exception:
hydrus_provider = get_plugin("hydrusnetwork", config)
if hydrus_provider is None:
return None
store_name = _extract_store_name(item)
@@ -102,68 +101,10 @@ def _maybe_download_hydrus_item(
is_hydrus_url = False
if not (is_hydrus_url or store_hint):
return None
# Prefer store name as instance key; fall back to "home".
access_key = None
hydrus_url = None
for inst in [s for s in [store_lower, "home"] if s]:
try:
access_key = (get_hydrus_access_key(config, inst) or "").strip() or None
hydrus_url = (get_hydrus_url(config, inst) or "").strip() or None
if access_key and hydrus_url:
break
except Exception:
access_key = None
hydrus_url = None
if not access_key or not hydrus_url:
return None
client = HydrusClient(url=hydrus_url, access_key=access_key, timeout=60.0)
file_url = url if (url and is_hydrus_url) else client.file_url(file_hash)
# Best-effort extension from Hydrus metadata.
suffix = ".hydrus"
try:
meta_response = client.fetch_file_metadata(
hashes=[file_hash],
include_mime=True
)
entries = meta_response.get("metadata"
) if isinstance(meta_response,
dict) else None
if isinstance(entries, list) and entries:
entry = entries[0]
if isinstance(entry, dict):
ext = entry.get("ext")
if isinstance(ext, str) and ext.strip():
cleaned = ext.strip()
if not cleaned.startswith("."):
cleaned = "." + cleaned.lstrip(".")
if len(cleaned) <= 12:
suffix = cleaned
except Exception:
pass
try:
output_dir.mkdir(parents=True, exist_ok=True)
except Exception:
pass
dest = output_dir / f"{file_hash}{suffix}"
if dest.exists():
dest = output_dir / f"{file_hash}_{uuid.uuid4().hex[:10]}{suffix}"
headers = {
"Hydrus-Client-API-Access-Key": access_key
}
download_hydrus_file(file_url, headers, dest, timeout=60.0)
try:
if dest.exists() and dest.is_file():
return dest
except Exception:
return None
return None
preferred_store = store_name or None
if url and is_hydrus_url:
return hydrus_provider.download_url(url, output_dir)
return hydrus_provider.download_hash_to_temp(file_hash, store_name=preferred_store, temp_root=output_dir)
def _resolve_existing_or_fetch_path(item: Any,
+25 -96
View File
@@ -7,9 +7,9 @@ import sys
from pathlib import Path
from SYS.logger import debug, log
from ProviderCore.registry import get_plugin
from Store import Store
from . import _shared as sh
from API import HydrusNetwork as hydrus_wrapper
from SYS import pipeline as ctx
from SYS.result_table_helpers import add_row_columns
from SYS.result_table import Table, _format_size
@@ -129,6 +129,7 @@ class Delete_File(sh.Cmdlet):
store = sh.get_field(item, "store")
store_lower = str(store).lower() if store else ""
hydrus_provider = get_plugin("hydrusnetwork", config)
backend = None
try:
@@ -144,18 +145,17 @@ class Delete_File(sh.Cmdlet):
# so checking only the store name is unreliable.
is_hydrus_store = False
try:
if backend is not None:
from Store.HydrusNetwork import HydrusNetwork as HydrusStore
is_hydrus_store = isinstance(backend, HydrusStore)
if hydrus_provider is not None and backend is not None:
is_hydrus_store = bool(hydrus_provider.is_backend(backend, str(store or "")))
except Exception:
is_hydrus_store = False
# Backwards-compatible fallback heuristic (older items might only carry a name).
if ((not is_hydrus_store) and bool(store_lower)
and ("hydrus" in store_lower or store_lower in {"home",
"work"})):
is_hydrus_store = True
if (not is_hydrus_store) and hydrus_provider is not None and bool(store_lower):
try:
is_hydrus_store = bool(hydrus_provider.is_store_name(store_lower))
except Exception:
is_hydrus_store = False
store_label = str(store) if store else "default"
hydrus_prefix = f"[hydrusnetwork:{store_label}]"
@@ -318,18 +318,20 @@ class Delete_File(sh.Cmdlet):
should_try_hydrus = False
if should_try_hydrus and hash_hex:
# Prefer deleting via the resolved store backend when it is a HydrusNetwork store.
# This ensures store-specific post-delete hooks run (e.g., clearing Hydrus deletion records).
did_backend_delete = False
did_hydrus_delete = False
try:
if backend is not None:
deleter = getattr(backend, "delete_file", None)
if callable(deleter):
did_backend_delete = bool(deleter(hash_hex, reason=reason))
if hydrus_provider is not None:
did_hydrus_delete = bool(
hydrus_provider.delete_hash(
hash_hex,
store_name=str(store) if store else None,
reason=reason or None,
)
)
except Exception:
did_backend_delete = False
did_hydrus_delete = False
if did_backend_delete:
if did_hydrus_delete:
hydrus_deleted = True
title_str = str(title_val).strip() if title_val else ""
if title_str:
@@ -340,85 +342,12 @@ class Delete_File(sh.Cmdlet):
else:
debug(f"{hydrus_prefix} Deleted hash:{hash_hex}", file=sys.stderr)
else:
# Fallback to direct client calls.
client = None
if store:
# Store specified: do not fall back to a global/default Hydrus client.
try:
registry = Store(config)
backend = registry[str(store)]
candidate = getattr(backend, "_client", None)
if candidate is not None and hasattr(candidate, "_post"):
client = candidate
except Exception as exc:
if not local_deleted:
log(
f"Hydrus client unavailable for store '{store}': {exc}",
file=sys.stderr,
)
return False
if client is None:
if not local_deleted:
log(
f"Hydrus client unavailable for store '{store}'",
file=sys.stderr
)
return False
else:
# No store context; use default Hydrus client.
try:
client = hydrus_wrapper.get_client(config)
except Exception as exc:
if not local_deleted:
log(f"Hydrus client unavailable: {exc}", file=sys.stderr)
return False
if client is None:
if not local_deleted:
log("Hydrus client unavailable", file=sys.stderr)
return False
payload: Dict[str,
Any] = {
"hashes": [hash_hex]
}
if reason:
payload["reason"] = reason
try:
client._post(
"/add_files/delete_files",
data=payload
) # type: ignore[attr-defined]
# Best-effort clear deletion record if supported by this client.
try:
clearer = getattr(client, "clear_file_deletion_record", None)
if callable(clearer):
clearer([hash_hex])
else:
client._post(
"/add_files/clear_file_deletion_record",
data={
"hashes": [hash_hex]
}
) # type: ignore[attr-defined]
except Exception:
pass
hydrus_deleted = True
title_str = str(title_val).strip() if title_val else ""
if title_str:
debug(
f"{hydrus_prefix} Deleted title:{title_str} hash:{hash_hex}",
file=sys.stderr,
)
if not local_deleted:
if store:
log(f"Hydrus store unavailable for '{store}'", file=sys.stderr)
else:
debug(
f"{hydrus_prefix} Deleted hash:{hash_hex}",
file=sys.stderr
)
except Exception:
# If it's not in Hydrus (e.g. 404 or similar), that's fine
if not local_deleted:
return []
log("Hydrus delete failed", file=sys.stderr)
return []
if hydrus_deleted and hash_hex:
size_hint = None
+8 -4
View File
@@ -980,10 +980,14 @@ class Download_File(Cmdlet):
) -> Optional[str]:
if storage is None or not canonical_url:
return None
hydrus_provider = None
try:
from Store.HydrusNetwork import HydrusNetwork
registry_helpers = cls._load_provider_registry()
get_plugin = registry_helpers.get("get_plugin")
if callable(get_plugin):
hydrus_provider = get_plugin("hydrusnetwork", {})
except Exception:
HydrusNetwork = None # type: ignore
hydrus_provider = None
try:
backend_names = list(storage.list_searchable_backends() or [])
@@ -1001,13 +1005,13 @@ class Download_File(Cmdlet):
except Exception:
pass
try:
if HydrusNetwork is not None and isinstance(backend, HydrusNetwork) and not hydrus_available:
if hydrus_provider is not None and hydrus_provider.is_backend(backend, str(backend_name)) and not hydrus_available:
continue
except Exception:
pass
try:
if HydrusNetwork is not None and isinstance(backend, HydrusNetwork):
if hydrus_provider is not None and hydrus_provider.is_backend(backend, str(backend_name)):
hashes = backend.find_hashes_by_url(canonical_url) or []
for existing_hash in hashes:
normalized = sh.normalize_hash(existing_hash)
+16 -37
View File
@@ -5,12 +5,12 @@ import sys
from SYS.detail_view_helpers import create_detail_view, prepare_detail_metadata
from SYS.logger import log
from ProviderCore.registry import get_plugin
from SYS.result_table_helpers import add_row_columns
from SYS.selection_builder import build_hash_store_selection
from SYS.result_publication import publish_result_table
from SYS import pipeline as ctx
from API import HydrusNetwork as hydrus_wrapper
from . import _shared as sh
Cmdlet = sh.Cmdlet
@@ -22,7 +22,6 @@ get_hash_for_operation = sh.get_hash_for_operation
fetch_hydrus_metadata = sh.fetch_hydrus_metadata
should_show_help = sh.should_show_help
get_field = sh.get_field
from Store import Store
CMDLET = Cmdlet(
name="get-relationship",
@@ -109,6 +108,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
return 1
# Fetch Hydrus relationships if we have a hash.
hydrus_provider = get_plugin("hydrusnetwork", config)
hash_hex = (
normalize_hash(override_hash)
@@ -118,29 +118,18 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
if hash_hex:
try:
client = None
store_label = "hydrus"
backend_obj = None
if store_name:
# Store specified: do not fall back to a global/default Hydrus client.
store_label = str(store_name)
try:
store = Store(config)
backend_obj = store[str(store_name)]
candidate = getattr(backend_obj, "_client", None)
if candidate is not None and hasattr(candidate,
"get_file_relationships"):
client = candidate
except Exception:
client = None
if client is None:
if hydrus_provider is None:
log(
f"Hydrus client unavailable for store '{store_name}'",
file=sys.stderr
)
return 1
relationships = hydrus_provider.get_relationships(hash_hex, store_name=store_name)
else:
client = hydrus_wrapper.get_client(config)
relationships = hydrus_provider.get_relationships(hash_hex) if hydrus_provider is not None else None
def _resolve_related_title(rel_hash: str) -> str:
"""Best-effort resolve a Hydrus hash to a human title.
@@ -154,22 +143,15 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
if not h:
return str(rel_hash)
# Prefer backend tag extraction when available.
if backend_obj is not None and hasattr(backend_obj, "get_tag"):
# Prefer provider-backed title resolution when available.
if hydrus_provider is not None:
try:
tag_result = backend_obj.get_tag(h)
tags = (
tag_result[0]
if isinstance(tag_result,
tuple) and tag_result else tag_result
resolved_title = hydrus_provider.get_title(
h,
store_name=store_label if store_name else None,
)
if isinstance(tags, list):
for t in tags:
if isinstance(t,
str) and t.lower().startswith("title:"):
val = t.split(":", 1)[1].strip()
if val:
return val
if isinstance(resolved_title, str) and resolved_title.strip():
return resolved_title.strip()
except Exception:
pass
@@ -179,7 +161,6 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
config,
h,
store_name=store_label if store_name else None,
hydrus_client=client,
include_service_keys_to_tags=True,
include_file_url=False,
include_duration=False,
@@ -224,14 +205,12 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
return h[:16] + "..."
if client:
rel = client.get_file_relationships(hash_hex)
if rel:
file_rels = rel.get("file_relationships",
if relationships:
file_rels = relationships.get("file_relationships",
{})
this_file_rels = file_rels.get(hash_hex)
this_file_rels = file_rels.get(hash_hex)
if this_file_rels:
if this_file_rels:
# Map Hydrus relationship IDs to names.
# For /manage_file_relationships/get_file_relationships, the Hydrus docs define:
# 0=potential duplicates, 1=false positives, 3=alternates, 8=duplicates
+147 -1
View File
@@ -1,3 +1,6 @@
import datetime
import sqlite3
from pathlib import Path
from typing import List, Dict, Any, Optional, Sequence
from SYS.cmdlet_spec import Cmdlet, CmdletArg
@@ -7,17 +10,20 @@ from SYS.config import (
save_config_and_verify,
set_nested_config_value,
)
from SYS.database import LOG_DB_PATH, db
from SYS.logger import log
from SYS import pipeline as ctx
from SYS.result_table import Table
from cmdnat._parsing import (
extract_piped_value as _extract_piped_value,
extract_value_arg as _extract_value_arg,
has_flag as _has_flag,
)
CMDLET = Cmdlet(
name=".config",
summary="Manage configuration settings",
usage=".config [key] [value]",
usage=".config [key] [value] | .config -log [count]",
arg=[
CmdletArg(
name="key",
@@ -33,6 +39,140 @@ CMDLET = Cmdlet(
)
def _extract_log_limit(args: Sequence[str], default: int = 30) -> int:
try:
tokens = [str(arg).strip() for arg in (args or []) if str(arg).strip()]
except Exception:
return default
for idx, token in enumerate(tokens):
lowered = token.lower()
if lowered in {"-log", "--log"}:
if idx + 1 < len(tokens):
candidate = tokens[idx + 1]
if candidate and not candidate.startswith("-"):
try:
return max(1, min(200, int(candidate)))
except Exception:
return default
return default
if lowered.startswith("-log=") or lowered.startswith("--log="):
_, value = lowered.split("=", 1)
try:
return max(1, min(200, int(value)))
except Exception:
return default
return default
def _fallback_log_path() -> Path:
return Path(db.db_path).with_name("logs") / "log_fallback.txt"
def _load_recent_config_logs(limit: int = 30) -> List[Dict[str, str]]:
rows: List[Dict[str, str]] = []
sql = """
SELECT timestamp, level, module, message
FROM logs
WHERE lower(module) LIKE ?
OR lower(message) LIKE ?
OR lower(message) LIKE ?
OR lower(message) LIKE ?
ORDER BY id DESC
LIMIT ?
"""
params = (
"%config%",
"%config%",
"%save failed%",
"%saving configuration failed%",
int(limit),
)
try:
with sqlite3.connect(str(LOG_DB_PATH), timeout=5.0) as conn:
conn.row_factory = sqlite3.Row
cur = conn.cursor()
cur.execute(sql, params)
fetched = cur.fetchall()
cur.close()
for row in fetched:
rows.append(
{
"timestamp": str(row["timestamp"] or ""),
"level": str(row["level"] or ""),
"module": str(row["module"] or ""),
"message": str(row["message"] or ""),
}
)
except Exception:
rows = []
if rows:
return rows
fallback = _fallback_log_path()
try:
if not fallback.exists():
return []
lines = fallback.read_text(encoding="utf-8", errors="replace").splitlines()
matches = [
line for line in lines
if any(term in line.lower() for term in ("config", "save failed", "saving configuration failed"))
]
for line in reversed(matches[-limit:]):
rows.append(
{
"timestamp": "",
"level": "FALLBACK",
"module": "fallback",
"message": line,
}
)
except Exception:
return []
return rows
def _format_log_timestamp_local(raw_value: str) -> str:
text = str(raw_value or "").strip()
if not text:
return ""
for pattern in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S.%f"):
try:
parsed = datetime.datetime.strptime(text, pattern).replace(tzinfo=datetime.timezone.utc)
return parsed.astimezone().strftime("%Y-%m-%d %H:%M:%S")
except Exception:
continue
return text
def _show_config_logs(args: Sequence[str]) -> int:
limit = _extract_log_limit(args)
rows = _load_recent_config_logs(limit=limit)
if not rows:
print(
f"No recent config/save logs found in {LOG_DB_PATH.name} or {_fallback_log_path().name}."
)
return 0
table = Table("Configuration Logs")
table.set_table("config.logs")
table.set_source_command(".config", ["-log", str(limit)])
for row_data in rows:
row = table.add_row()
row.add_column("Time (local)", _format_log_timestamp_local(row_data.get("timestamp", "")))
row.add_column("Level", row_data.get("level", ""))
row.add_column("Module", row_data.get("module", ""))
row.add_column("Message", row_data.get("message", ""))
ctx.set_last_result_table_overlay(table, rows)
ctx.set_current_stage_table(table)
print(f"Showing {len(rows)} recent configuration log entries.")
return 0
def flatten_config(config: Dict[str, Any], parent_key: str = "", sep: str = ".") -> List[Dict[str, Any]]:
items: List[Dict[str, Any]] = []
for k, v in config.items():
@@ -108,6 +248,9 @@ def _strip_value_quotes(value: str) -> str:
def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int:
import sys
if _has_flag(args, "-log") or _has_flag(args, "--log"):
return _show_config_logs(args)
# Load configuration from the database
current_config = load_config()
@@ -135,6 +278,7 @@ def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int:
try:
save_config_and_verify(current_config)
except Exception as exc:
log(f"Configuration save verification failed for '{selection_key}': {exc}")
print(f"Error saving configuration (verification failed): {exc}")
return 1
else:
@@ -142,6 +286,7 @@ def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int:
print(f"Updated '{selection_key}' to '{new_value}'")
return 0
except Exception as exc:
log(f"Error updating config '{selection_key}': {exc}")
print(f"Error updating config: {exc}")
return 1
@@ -201,6 +346,7 @@ def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int:
print(f"Updated '{key}' to '{value}'")
return 0
except Exception as exc:
log(f"Error updating config '{key}': {exc}")
print(f"Error updating config: {exc}")
return 1
+164
View File
@@ -0,0 +1,164 @@
# FTP Plugin Walkthrough
This walkthrough adds a real bundled `ftp` plugin so users can:
- run `search-file -plugin ftp ...`
- browse remote folders as result tables
- select file rows to `download-file`
- pipe selected file rows into `add-file`
- upload local files with `add-file -plugin ftp`
The implementation lives in [plugins/ftp/__init__.py](plugins/ftp/__init__.py).
## What The Plugin Does
The FTP plugin demonstrates the main provider hooks that matter for a storage-style integration:
- `config_schema()` exposes host, credentials, base path, TLS, and search depth.
- `extract_query_arguments()` supports inline query fields like `path:` and `depth:`.
- `search()` walks an FTP directory tree and returns `SearchResult` rows.
- `selector()` turns folder rows into a follow-up table when the user runs `@N`.
- `download()` and `download_url()` fetch FTP files into `download-file` output paths.
- `resolve_pipe_result_download()` lets `@N | add-file -store ...` materialize a remote FTP file first.
- `upload()` lets `add-file -plugin ftp -path ...` push a local file to the configured FTP server.
## Example Config
Add an FTP provider block to your config:
```toml
[provider.ftp]
host = "ftp.example.com"
port = 21
username = "demo"
password = "secret"
base_path = "/incoming"
tls = false
passive = true
timeout = 20
search_depth = 1
```
Notes:
- `host` is the only required field for the plugin to validate.
- `username` defaults to `anonymous` and `password` defaults to `anonymous@`.
- `base_path` is both the default search root and the upload target directory.
- `search_depth` controls how many folder levels `search-file -plugin ftp` scans by default.
## Search Flow
Basic listing from the configured base path:
```powershell
search-file -plugin ftp "*"
```
Search by filename fragment:
```powershell
search-file -plugin ftp "invoice"
```
Search a different subtree and recurse deeper:
```powershell
search-file -plugin ftp "path:/pub depth:2 invoice"
```
Filter to folders only:
```powershell
search-file -plugin ftp "path:/pub type:folder *"
```
The plugin returns rows with explicit columns for name, type, directory, size, and modification time.
## Selection Flow
Folder rows are navigation rows. If the selected row is a directory, plain `@N` opens a new FTP table for that directory:
```powershell
search-file -plugin ftp "*"
@2
```
File rows carry an explicit row action:
```powershell
download-file -plugin ftp -url ftp://ftp.example.com/incoming/report.pdf
```
That means plain `@N` on a file row downloads it immediately:
```powershell
search-file -plugin ftp "report"
@1
```
## Download And Add-File Flow
If you want the downloaded file in a specific local directory:
```powershell
search-file -plugin ftp "report"
@1 | download-file -path C:\Downloads
```
If you want to ingest the selected FTP file into a configured store backend:
```powershell
search-file -plugin ftp "report"
@1 | add-file -store tutorial
```
Why this works:
- the file row advertises a `download-file` row action
- the pipeline auto-inserts that download before `add-file`
- the FTP plugin also implements `resolve_pipe_result_download()` so provider-owned FTP rows can be materialized for ingestion
## Upload Flow
Uploading uses the same provider name, but through `add-file -plugin ftp`:
```powershell
add-file -plugin ftp -path C:\Media\report.pdf
```
That sends the file to the configured FTP `base_path` and returns the FTP URL as the uploaded result.
## Why The Row Metadata Matters
The critical part of this plugin is the file-row metadata:
- file rows emit `_selection_args` as `['-url', '<ftp-url>']`
- file rows emit `_selection_action` as `['download-file', '-plugin', 'ftp', '-url', '<ftp-url>']`
- folder rows do not emit a download action, so `selector()` can own drill-in behavior instead
That split is what keeps these two user experiences compatible:
- `@N` on a folder opens a new table
- `@N` on a file downloads the file
- `@N | add-file -store ...` first downloads, then ingests
## Implementation Notes
The plugin prefers `MLSD` for directory listings and falls back to `NLST` plus directory probes when the server does not support machine-readable listings.
The code is intentionally small and uses only Python stdlib pieces:
- `ftplib` for FTP and FTPS
- `fnmatch` for wildcard-style search tokens
- `tempfile` for `add-file` handoff downloads
## Recommended Commands To Demo The Walkthrough
```powershell
search-file -plugin ftp "*"
search-file -plugin ftp "path:/incoming depth:2 *.pdf"
@1
@1 | download-file -path C:\Downloads
@1 | add-file -store tutorial
add-file -plugin ftp -path C:\Media\report.pdf
```
+136
View File
@@ -0,0 +1,136 @@
# SCP Plugin Walkthrough
This walkthrough adds a bundled `scp` plugin backed by existing SSH libraries:
- `paramiko` for SSH and SFTP directory listing
- `scp` for file transfers
The implementation lives in [plugins/scp/__init__.py](plugins/scp/__init__.py).
## What The Plugin Does
The SCP plugin mirrors the FTP walkthrough, but on top of SSH:
- `search-file -plugin scp ...` lists remote files and folders over SFTP.
- plain `@N` on a folder drills into that directory.
- plain `@N` on a file runs `download-file -plugin scp -url ...`.
- `@N | add-file -store ...` downloads first, then ingests the local temp file.
- `add-file -plugin scp -path ...` uploads a local file to the configured remote path.
## Example Config
```toml
[provider.scp]
host = "ssh.example.com"
port = 22
username = "deploy"
password = "secret"
key_path = "C:/Users/Admin/.ssh/id_ed25519"
base_path = "/srv/files"
timeout = 20
search_depth = 1
allow_agent = true
look_for_keys = true
```
Notes:
- `host` and `username` are required for the plugin to validate.
- You can use password auth, key auth, or both.
- `base_path` is both the default search root and the default upload directory.
## Search Flow
List the configured base path:
```powershell
search-file -plugin scp "*"
```
Search by filename:
```powershell
search-file -plugin scp "invoice"
```
Search another subtree with deeper recursion:
```powershell
search-file -plugin scp "path:/srv/files/releases depth:2 *.zip"
```
Show only folders:
```powershell
search-file -plugin scp "path:/srv/files type:folder *"
```
## Selection Flow
Folder rows are navigation rows:
```powershell
search-file -plugin scp "*"
@2
```
File rows carry an explicit row action, so terminal selection downloads directly:
```powershell
search-file -plugin scp "report"
@1
```
That expands to the equivalent of:
```powershell
download-file -plugin scp -url scp://ssh.example.com/srv/files/report.pdf
```
## Download And Add-File Flow
Download into a local folder:
```powershell
search-file -plugin scp "report"
@1 | download-file -path C:\Downloads
```
Ingest a selected remote file into a configured store backend:
```powershell
search-file -plugin scp "report"
@1 | add-file -store tutorial
```
Why this works:
- file rows advertise `_selection_action` for `download-file`
- `add-file` selection replay inserts that provider download stage before ingest
- the plugin also implements `resolve_pipe_result_download()` for provider-owned SCP rows
## Upload Flow
Upload a local file to the configured remote `base_path`:
```powershell
add-file -plugin scp -path C:\Media\report.pdf
```
## Implementation Notes
The plugin uses SFTP for directory listing because SCP itself is a transfer protocol, not a browse/search protocol. That split keeps the provider simple:
- browse and metadata via Paramiko SFTP
- file transfer via the `scp` package
## Recommended Demo Commands
```powershell
search-file -plugin scp "*"
search-file -plugin scp "path:/srv/files depth:2 *.zip"
@1
@1 | download-file -path C:\Downloads
@1 | add-file -store tutorial
add-file -plugin scp -path C:\Media\report.pdf
```
+9 -1
View File
@@ -48,4 +48,12 @@ class MyPlugin(Provider):
path=f"https://example.com/{text}",
)
]
```
```
Bundled walkthrough:
- The repo now includes a real FTP example plugin in [plugins/ftp/__init__.py](plugins/ftp/__init__.py).
- The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp`, folder drill-in via `@N`, file download routing, `@N | add-file -store ...`, and `add-file -plugin ftp` uploads.
- The repo also includes an SCP example plugin in [plugins/scp/__init__.py](plugins/scp/__init__.py).
- The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp`, SSH-backed directory drill-in, file download routing, `@N | add-file -store ...`, and `add-file -plugin scp` uploads.
- The repo now also includes a built-in HydrusNetwork provider in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). It delegates to configured `store.hydrusnetwork.*` backends so Hydrus features can be reached through the normal plugin registry without cmdlets importing Hydrus modules directly.
+57 -10
View File
@@ -23,24 +23,56 @@ from SYS.models import DownloadError, PipeObject
_HOSTS_CACHE_TTL_SECONDS = 24 * 60 * 60
def _repo_root() -> Path:
def _plugin_dir() -> Path:
try:
return Path(__file__).resolve().parents[1]
return Path(__file__).resolve().parent
except Exception:
return Path(".")
def _legacy_hosts_cache_paths() -> Tuple[Path, ...]:
try:
repo_root = Path(__file__).resolve().parents[2]
plugins_root = Path(__file__).resolve().parents[1]
except Exception:
return tuple()
return (
plugins_root / "API" / "data" / "alldebrid.json",
repo_root / "API" / "data" / "alldebrid.json",
)
def _hosts_cache_path() -> Path:
# Keep this local to the repo so it works in portable installs.
# The registry's URL routing can read this file without instantiating providers.
# Keep this local to the plugin so plugin-specific cache/state stays bundled
# with the plugin itself in portable installs.
#
# This file is expected to be the JSON payload shape from AllDebrid:
# {"status":"success","data":{"hosts":[...],"streams":[...],"redirectors":[...]}}
return _repo_root() / "API" / "data" / "alldebrid.json"
return _plugin_dir() / "alldebrid.json"
def _resolve_hosts_cache_path() -> Path:
path = _hosts_cache_path()
try:
if path.exists() and path.is_file():
return path
except Exception:
return path
for legacy in _legacy_hosts_cache_paths():
try:
if not legacy.exists() or not legacy.is_file():
continue
path.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(legacy, path)
return path
except Exception:
continue
return path
def _load_cached_domains(category: str) -> List[str]:
"""Load cached domain list from API/data/alldebrid.json.
"""Load cached domain list from the plugin-local alldebrid.json cache.
category: "hosts" | "streams" | "redirectors"
"""
@@ -49,7 +81,7 @@ def _load_cached_domains(category: str) -> List[str]:
if wanted not in {"hosts", "streams", "redirectors"}:
return []
path = _hosts_cache_path()
path = _resolve_hosts_cache_path()
try:
if not path.exists() or not path.is_file():
return []
@@ -68,12 +100,27 @@ def _load_cached_domains(category: str) -> List[str]:
return []
raw_list = data.get(wanted)
if not isinstance(raw_list, list):
if not isinstance(raw_list, (list, dict)):
return []
out: List[str] = []
seen: set[str] = set()
for d in raw_list:
domain_candidates: List[Any] = []
if isinstance(raw_list, list):
domain_candidates.extend(raw_list)
else:
for entry in raw_list.values():
if isinstance(entry, dict):
nested_domains = entry.get("domains")
if isinstance(nested_domains, list):
domain_candidates.extend(nested_domains)
elif isinstance(nested_domains, str):
domain_candidates.append(nested_domains)
elif isinstance(entry, str):
domain_candidates.append(entry)
for d in domain_candidates:
try:
dom = str(d or "").strip().lower()
except Exception:
@@ -115,7 +162,7 @@ def _save_cached_hosts_payload(payload: Dict[str, Any]) -> None:
def _cache_is_fresh() -> bool:
path = _hosts_cache_path()
path = _resolve_hosts_cache_path()
try:
if not path.exists() or not path.is_file():
return False
@@ -37,7 +37,7 @@
"(rapidgator\\.net/file/[0-9]{7,8})"
],
"regexp": "((rapidgator\\.net|rg\\.to|rapidgator\\.asia)/file/([0-9a-zA-Z]{32}))|((rapidgator\\.net/file/[0-9]{7,8}))",
"status": true
"status": false
},
"turbobit": {
"name": "turbobit",
@@ -71,7 +71,7 @@
"(wayupload\\.com/[a-z0-9]{12}\\.html)"
],
"regexp": "(turbobit5?a?\\.(net|cc|com)/([a-z0-9]{12}))|(turbobif\\.(net|cc|com)/([a-z0-9]{12}))|(turb[o]?\\.(to|cc|pw)\\/([a-z0-9]{12}))|(turbobit\\.(net|cc)/download/free/([a-z0-9]{12}))|((trbbt|tourbobit|torbobit|tbit|turbobita|trbt)\\.(net|cc|com|to)/([a-z0-9]{12}))|((turbobit\\.cloud/turbo/[a-z0-9]+))|((wayupload\\.com/[a-z0-9]{12}\\.html))",
"status": false
"status": true
},
"hitfile": {
"name": "hitfile",
@@ -375,7 +375,7 @@
"(filespace\\.com/[a-zA-Z0-9]{12})"
],
"regexp": "(filespace\\.com/fd/([a-zA-Z0-9]{12}))|((filespace\\.com/[a-zA-Z0-9]{12}))",
"status": false
"status": true
},
"filezip": {
"name": "filezip",
+778
View File
@@ -0,0 +1,778 @@
from __future__ import annotations
import fnmatch
import ftplib
import posixpath
import tempfile
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from urllib.parse import quote, unquote, urlparse
from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments
def _pick_provider_config(config: Any) -> Dict[str, Any]:
if not isinstance(config, dict):
return {}
provider = config.get("provider")
if not isinstance(provider, dict):
return {}
entry = provider.get("ftp")
if isinstance(entry, dict):
return entry
return {}
def _coerce_bool(value: Any, default: bool = False) -> bool:
if isinstance(value, bool):
return value
if value is None:
return default
text = str(value).strip().lower()
if not text:
return default
if text in {"1", "true", "yes", "on"}:
return True
if text in {"0", "false", "no", "off"}:
return False
return default
def _coerce_int(value: Any, default: int) -> int:
try:
return int(value)
except Exception:
return default
def _format_timestamp(raw_value: Any) -> str:
text = str(raw_value or "").strip()
if not text:
return ""
for pattern in ("%Y%m%d%H%M%S", "%Y%m%d%H%M%S.%f"):
try:
parsed = datetime.strptime(text, pattern)
return parsed.strftime("%Y-%m-%d %H:%M")
except Exception:
continue
return text
def _safe_filename(name: Any) -> str:
raw = str(name or "").strip()
if not raw:
raw = "download"
cleaned = "".join(ch if ch.isalnum() or ch in {"-", "_", ".", " "} else "_" for ch in raw)
cleaned = cleaned.strip(" ._")
return cleaned or "download"
def _unique_path(path: Path) -> Path:
if not path.exists():
return path
stem = path.stem or "download"
suffix = path.suffix
counter = 1
while True:
candidate = path.with_name(f"{stem}_{counter}{suffix}")
if not candidate.exists():
return candidate
counter += 1
class FTP(Provider):
PLUGIN_NAME = "ftp"
URL = ("ftp://", "ftps://")
@property
def label(self) -> str:
return "FTP"
@property
def preserve_order(self) -> bool:
return True
@classmethod
def config_schema(cls) -> List[Dict[str, Any]]:
return [
{
"key": "host",
"label": "Host",
"default": "",
"required": True,
"placeholder": "ftp.example.com",
},
{
"key": "port",
"label": "Port",
"type": "integer",
"default": 21,
},
{
"key": "username",
"label": "Username",
"default": "anonymous",
},
{
"key": "password",
"label": "Password",
"type": "secret",
"secret": True,
"default": "",
},
{
"key": "base_path",
"label": "Base Path",
"default": "/",
"placeholder": "/incoming",
},
{
"key": "tls",
"label": "Use FTPS",
"type": "boolean",
"default": False,
},
{
"key": "passive",
"label": "Passive Mode",
"type": "boolean",
"default": True,
},
{
"key": "timeout",
"label": "Timeout Seconds",
"type": "integer",
"default": 20,
},
{
"key": "search_depth",
"label": "Default Search Depth",
"type": "integer",
"default": 1,
},
]
def __init__(self, config: Optional[Dict[str, Any]] = None):
super().__init__(config)
conf = _pick_provider_config(self.config)
self._host = str(conf.get("host") or "").strip()
self._tls = _coerce_bool(conf.get("tls"), False)
self._port = _coerce_int(conf.get("port"), 21)
self._username = str(conf.get("username") or conf.get("user") or "anonymous").strip() or "anonymous"
password_value = conf.get("password")
self._password = str(password_value).strip() if password_value not in (None, "") else "anonymous@"
self._passive = _coerce_bool(conf.get("passive"), True)
self._timeout = max(1, _coerce_int(conf.get("timeout"), 20))
self._search_depth = max(0, _coerce_int(conf.get("search_depth"), 1))
self._base_path = self._normalize_remote_path(conf.get("base_path") or "/", default="/")
def validate(self) -> bool:
return bool(self._host)
def config_helper_text(self) -> str:
return "Test the configured FTP/FTPS settings before searching or uploading."
def config_actions(self) -> List[Dict[str, Any]]:
return [
{
"id": "test_connection",
"label": "Test connection",
"variant": "primary",
}
]
def run_config_action(self, action_id: str, **_kwargs: Any) -> Dict[str, Any]:
if str(action_id or "").strip().lower() != "test_connection":
return super().run_config_action(action_id, **_kwargs)
if not self._host:
return {"ok": False, "message": "Set 'host' before testing the FTP connection."}
ftp = None
try:
ftp = self._connect()
active_path = self._base_path or "/"
try:
ftp.cwd(active_path)
resolved_path = ftp.pwd()
except Exception:
resolved_path = active_path
return {
"ok": True,
"message": f"Connected to FTP {self._host}:{self._port} and reached {resolved_path}.",
}
except Exception as exc:
return {"ok": False, "message": f"FTP connection failed: {exc}"}
finally:
self._close(ftp)
def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]:
text, inline = parse_inline_query_arguments(query)
filters: Dict[str, Any] = {}
if inline.get("path"):
filters["path"] = inline.get("path")
if inline.get("depth"):
filters["depth"] = max(0, _coerce_int(inline.get("depth"), self._search_depth))
if inline.get("type"):
filters["type"] = str(inline.get("type") or "").strip().lower()
return text, filters
def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str:
active_path = self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path)
text = str(query or "").strip()
if not text or text == "*":
return f"FTP: {active_path}"
return f"FTP: {text} @ {active_path}"
def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
return {
"plugin": self.name,
"host": self._host,
"path": self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path),
"query": str(query or "").strip(),
}
def search(
self,
query: str,
limit: int = 50,
filters: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> List[SearchResult]:
_ = kwargs
active_filters = dict(filters or {})
start_path = self._normalize_remote_path(active_filters.get("path") or self._base_path, default=self._base_path)
search_depth = max(0, _coerce_int(active_filters.get("depth"), self._search_depth))
type_filter = str(active_filters.get("type") or "any").strip().lower()
needle = str(query or "").strip()
max_results = max(0, int(limit or 0))
if max_results <= 0:
return []
ftp = self._connect()
try:
return self._search_directory(
ftp,
start_path,
needle=needle,
limit=max_results,
search_depth=search_depth,
type_filter=type_filter,
)
finally:
self._close(ftp)
def selector(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
**_kwargs: Any,
) -> bool:
if not stage_is_last:
return False
target_path = ""
target_title = ""
for item in selected_items or []:
metadata = self._item_metadata(item)
if not metadata.get("is_dir"):
continue
target_path = self._normalize_remote_path(metadata.get("ftp_path") or metadata.get("selection_path"), default=self._base_path)
target_title = str(metadata.get("title") or metadata.get("name") or "").strip()
if target_path:
break
if not target_path:
return False
ftp = self._connect()
try:
rows = self._search_directory(
ftp,
target_path,
needle="*",
limit=500,
search_depth=0,
type_filter="any",
)
finally:
self._close(ftp)
try:
from SYS.result_table import Table
from SYS.rich_display import stdout_console
except Exception:
return True
title = target_title or target_path
table = Table(f"FTP: {title}")._perseverance(True)
table.set_table("ftp")
try:
table.set_table_metadata({
"provider": "ftp",
"host": self._host,
"path": target_path,
"view": "directory",
})
except Exception:
pass
table.set_source_command("search-file", ["-plugin", "ftp", f"path:{target_path}", "*"])
payloads: List[Dict[str, Any]] = []
for row in rows:
table.add_result(row)
payloads.append(row.to_dict())
try:
ctx.set_last_result_table(table, payloads, subject={"plugin": "ftp", "path": target_path})
ctx.set_current_stage_table(table)
except Exception:
pass
try:
stdout_console().print()
stdout_console().print(table)
except Exception:
pass
return True
def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]:
metadata = getattr(result, "full_metadata", None)
if isinstance(metadata, dict) and metadata.get("is_dir"):
return None
target = str(getattr(result, "path", "") or "").strip()
if not target:
return None
return self.download_url(target, output_dir, title=getattr(result, "title", None))
def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]:
settings = self._connection_settings_for_url(url)
remote_path = settings["path"]
if not remote_path or remote_path == "/":
return None
filename_hint = str(kwargs.get("title") or "").strip()
parsed_name = posixpath.basename(remote_path.rstrip("/"))
filename = _safe_filename(filename_hint or unquote(parsed_name) or "download")
destination_dir = Path(output_dir)
destination_dir.mkdir(parents=True, exist_ok=True)
destination = _unique_path(destination_dir / filename)
ftp = self._connect(
host=settings["host"],
port=settings["port"],
username=settings["username"],
password=settings["password"],
tls=settings["tls"],
)
try:
with destination.open("wb") as handle:
ftp.retrbinary(f"RETR {remote_path}", handle.write)
return destination
except Exception:
try:
destination.unlink(missing_ok=True)
except Exception:
pass
return None
finally:
self._close(ftp)
def resolve_pipe_result_download(
self,
result: Any,
pipe_obj: Any,
) -> Tuple[Optional[Path], Optional[str], Optional[Path]]:
metadata = self._item_metadata(result, pipe_obj=pipe_obj)
if metadata.get("is_dir"):
return None, None, None
download_url = str(
metadata.get("selection_url")
or metadata.get("ftp_url")
or metadata.get("path")
or ""
).strip()
if not download_url.startswith(("ftp://", "ftps://")):
return None, None, None
temp_dir = Path(tempfile.mkdtemp(prefix="ftp-add-file-"))
downloaded = self.download_url(download_url, temp_dir, title=metadata.get("title"))
if downloaded is None:
try:
temp_dir.rmdir()
except Exception:
pass
return None, None, None
try:
if pipe_obj is not None:
pipe_obj.is_temp = True
except Exception:
pass
return downloaded, None, temp_dir
def upload(self, file_path: str, **kwargs: Any) -> str:
local_path = Path(str(file_path or "")).expanduser()
if not local_path.exists() or not local_path.is_file():
raise FileNotFoundError(f"File not found: {local_path}")
remote_dir = self._normalize_remote_path(kwargs.get("remote_path") or kwargs.get("path") or self._base_path, default=self._base_path)
remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name
remote_path = self._join_remote_path(remote_dir, remote_name)
ftp = self._connect()
try:
self._ensure_directory(ftp, remote_dir)
with local_path.open("rb") as handle:
ftp.storbinary(f"STOR {remote_path}", handle)
finally:
self._close(ftp)
return self._build_url(remote_path)
def _connect(
self,
*,
host: Optional[str] = None,
port: Optional[int] = None,
username: Optional[str] = None,
password: Optional[str] = None,
tls: Optional[bool] = None,
) -> ftplib.FTP:
use_tls = self._tls if tls is None else bool(tls)
ftp: ftplib.FTP = ftplib.FTP_TLS() if use_tls else ftplib.FTP()
ftp.connect(host or self._host, int(port or self._port), timeout=self._timeout)
ftp.login(username or self._username, password or self._password)
try:
ftp.set_pasv(self._passive)
except Exception:
pass
if use_tls and isinstance(ftp, ftplib.FTP_TLS):
ftp.prot_p()
return ftp
def _close(self, ftp: Optional[ftplib.FTP]) -> None:
if ftp is None:
return
try:
ftp.quit()
except Exception:
try:
ftp.close()
except Exception:
pass
def _normalize_remote_path(self, value: Any, *, default: str) -> str:
text = str(value or "").strip().replace("\\", "/")
if not text:
text = default
elif text.startswith(("ftp://", "ftps://")):
try:
text = unquote(urlparse(text).path or "/")
except Exception:
text = default
elif not text.startswith("/"):
text = posixpath.join(default, text)
normalized = posixpath.normpath(text)
normalized = "/" + normalized.lstrip("/")
return normalized or "/"
def _join_remote_path(self, parent: Any, child: Any) -> str:
left = self._normalize_remote_path(parent, default=self._base_path)
right = str(child or "").strip().replace("\\", "/")
if not right:
return left
return self._normalize_remote_path(posixpath.join(left, right), default="/")
def _build_url(
self,
remote_path: Any,
*,
host: Optional[str] = None,
port: Optional[int] = None,
tls: Optional[bool] = None,
) -> str:
path_text = self._normalize_remote_path(remote_path, default="/")
scheme = "ftps" if (self._tls if tls is None else bool(tls)) else "ftp"
host_text = str(host or self._host).strip()
port_value = int(port or self._port)
port_suffix = f":{port_value}" if port_value and port_value != 21 else ""
return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}"
def _connection_settings_for_url(self, url: str) -> Dict[str, Any]:
parsed = urlparse(str(url or "").strip())
scheme = (parsed.scheme or "ftp").strip().lower()
host = parsed.hostname or self._host
port = parsed.port or self._port
username = parsed.username or self._username
password = parsed.password or self._password
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default="/")
return {
"tls": scheme == "ftps",
"host": host,
"port": port,
"username": username,
"password": password,
"path": path_text,
}
def _search_directory(
self,
ftp: ftplib.FTP,
start_path: str,
*,
needle: str,
limit: int,
search_depth: int,
type_filter: str,
) -> List[SearchResult]:
results: List[SearchResult] = []
visited: set[str] = set()
def walk(current_path: str, depth_left: int) -> None:
normalized = self._normalize_remote_path(current_path, default=self._base_path)
if normalized in visited or len(results) >= limit:
return
visited.add(normalized)
for entry in self._list_directory(ftp, normalized):
if len(results) >= limit:
return
if self._matches_entry(entry, needle=needle, type_filter=type_filter):
results.append(self._build_result(entry))
if entry.get("is_dir") and depth_left > 0:
walk(str(entry.get("ftp_path") or normalized), depth_left - 1)
walk(start_path, max(0, search_depth))
return results
def _matches_entry(self, entry: Dict[str, Any], *, needle: str, type_filter: str) -> bool:
is_dir = bool(entry.get("is_dir"))
if type_filter in {"dir", "dirs", "folder", "folders"} and not is_dir:
return False
if type_filter in {"file", "files"} and is_dir:
return False
text = str(needle or "").strip().lower()
if not text or text in {"*", "all", "list"}:
return True
haystacks = [
str(entry.get("name") or "").lower(),
str(entry.get("ftp_path") or "").lower(),
]
for token in [part for part in text.split() if part]:
if any(ch in token for ch in "*?[]"):
if not any(fnmatch.fnmatch(haystack, token) for haystack in haystacks):
return False
elif not any(token in haystack for haystack in haystacks):
return False
return True
def _build_result(self, entry: Dict[str, Any]) -> SearchResult:
ftp_path = str(entry.get("ftp_path") or "/")
ftp_url = self._build_url(ftp_path)
is_dir = bool(entry.get("is_dir"))
size_value = entry.get("size")
modified = str(entry.get("modified") or "")
parent = posixpath.dirname(ftp_path.rstrip("/")) or "/"
metadata = {
"provider": "ftp",
"host": self._host,
"ftp_path": ftp_path,
"ftp_url": ftp_url,
"selection_url": ftp_url,
"is_dir": is_dir,
"name": str(entry.get("name") or "").strip(),
}
if size_value is not None:
metadata["size"] = size_value
if modified:
metadata["modified"] = modified
return SearchResult(
table="ftp",
title=str(entry.get("name") or ftp_path),
path=ftp_url,
detail=parent,
annotations=["folder" if is_dir else "file"],
media_kind="folder" if is_dir else "file",
size_bytes=int(size_value) if isinstance(size_value, int) else None,
tag={"ftp", "folder" if is_dir else "file"},
columns=[
("Name", str(entry.get("name") or "")),
("Type", "dir" if is_dir else "file"),
("Directory", parent),
("Size", "" if size_value is None else str(size_value)),
("Modified", modified),
],
selection_args=None if is_dir else ["-url", ftp_url],
selection_action=None if is_dir else ["download-file", "-plugin", "ftp", "-url", ftp_url],
full_metadata=metadata,
)
def _list_directory(self, ftp: ftplib.FTP, remote_path: str) -> List[Dict[str, Any]]:
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
try:
entries: List[Dict[str, Any]] = []
for name, facts in ftp.mlsd(normalized):
name_text = str(name or "").strip()
if not name_text or name_text in {".", ".."}:
continue
entry_type = str((facts or {}).get("type") or "").strip().lower()
if entry_type in {"cdir", "pdir"}:
continue
size_value = None
raw_size = (facts or {}).get("size")
if raw_size not in (None, ""):
try:
size_value = int(raw_size)
except Exception:
size_value = None
entries.append(
{
"name": name_text,
"ftp_path": self._join_remote_path(normalized, name_text),
"is_dir": entry_type == "dir",
"size": size_value,
"modified": _format_timestamp((facts or {}).get("modify")),
}
)
return entries
except Exception:
return self._list_directory_fallback(ftp, normalized)
def _list_directory_fallback(self, ftp: ftplib.FTP, remote_path: str) -> List[Dict[str, Any]]:
try:
listed = ftp.nlst(remote_path)
except Exception:
return []
entries: List[Dict[str, Any]] = []
seen: set[str] = set()
for raw_entry in listed:
entry_text = str(raw_entry or "").strip()
if not entry_text:
continue
entry_path = entry_text if entry_text.startswith("/") else self._join_remote_path(remote_path, entry_text)
name_text = posixpath.basename(entry_path.rstrip("/")) or entry_path.rstrip("/")
if not name_text or name_text in {".", ".."} or name_text in seen:
continue
seen.add(name_text)
is_dir = self._is_directory(ftp, entry_path)
size_value = None
if not is_dir:
try:
size_raw = ftp.size(entry_path)
if size_raw is not None:
size_value = int(size_raw)
except Exception:
size_value = None
entries.append(
{
"name": name_text,
"ftp_path": entry_path,
"is_dir": is_dir,
"size": size_value,
"modified": self._read_modified(ftp, entry_path),
}
)
return entries
def _is_directory(self, ftp: ftplib.FTP, remote_path: str) -> bool:
current = None
try:
current = ftp.pwd()
except Exception:
current = None
try:
ftp.cwd(remote_path)
return True
except Exception:
return False
finally:
if current is not None:
try:
ftp.cwd(current)
except Exception:
pass
def _read_modified(self, ftp: ftplib.FTP, remote_path: str) -> str:
try:
response = ftp.sendcmd(f"MDTM {remote_path}")
except Exception:
return ""
parts = str(response or "").split()
if len(parts) >= 2:
return _format_timestamp(parts[1])
return ""
def _ensure_directory(self, ftp: ftplib.FTP, remote_path: str) -> None:
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
if normalized == "/":
return
partial = ""
for segment in [part for part in normalized.split("/") if part]:
partial = f"{partial}/{segment}"
if self._is_directory(ftp, partial):
continue
try:
ftp.mkd(partial)
except Exception:
if not self._is_directory(ftp, partial):
raise
def _item_metadata(self, item: Any, *, pipe_obj: Any = None) -> Dict[str, Any]:
metadata: Dict[str, Any] = {}
for source in (item, pipe_obj):
if isinstance(source, dict):
for key in ("title", "path", "url"):
if source.get(key) is not None and key not in metadata:
metadata[key] = source.get(key)
nested = source.get("full_metadata") or source.get("metadata")
if isinstance(nested, dict):
metadata.update(nested)
elif source is not None:
for attr in ("title", "path", "url"):
try:
value = getattr(source, attr, None)
except Exception:
value = None
if value is not None and attr not in metadata:
metadata[attr] = value
try:
nested = getattr(source, "full_metadata", None) or getattr(source, "metadata", None)
except Exception:
nested = None
if isinstance(nested, dict):
metadata.update(nested)
ftp_path = metadata.get("ftp_path") or metadata.get("selection_path")
if not ftp_path:
path_value = metadata.get("path") or metadata.get("url") or metadata.get("ftp_url")
path_text = str(path_value or "").strip()
if path_text.startswith(("ftp://", "ftps://")):
ftp_path = self._normalize_remote_path(path_text, default=self._base_path)
if ftp_path:
metadata["ftp_path"] = self._normalize_remote_path(ftp_path, default=self._base_path)
metadata.setdefault("selection_path", metadata["ftp_path"])
if metadata.get("ftp_path") and not metadata.get("ftp_url"):
metadata["ftp_url"] = self._build_url(metadata["ftp_path"])
if metadata.get("ftp_url") and not metadata.get("selection_url"):
metadata["selection_url"] = metadata["ftp_url"]
is_dir = metadata.get("is_dir")
if is_dir is None and metadata.get("media_kind"):
is_dir = str(metadata.get("media_kind") or "").strip().lower() == "folder"
metadata["is_dir"] = bool(is_dir)
return metadata
File diff suppressed because it is too large Load Diff
+938
View File
@@ -0,0 +1,938 @@
from __future__ import annotations
import fnmatch
import posixpath
import shlex
import stat
import tempfile
from datetime import datetime
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
from urllib.parse import quote, unquote, urlparse
import paramiko
from scp import SCPClient
from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments
def _pick_provider_config(config: Any) -> Dict[str, Any]:
if not isinstance(config, dict):
return {}
provider = config.get("provider")
if not isinstance(provider, dict):
return {}
entry = provider.get("scp")
if isinstance(entry, dict):
return entry
return {}
def _coerce_bool(value: Any, default: bool = False) -> bool:
if isinstance(value, bool):
return value
if value is None:
return default
text = str(value).strip().lower()
if not text:
return default
if text in {"1", "true", "yes", "on"}:
return True
if text in {"0", "false", "no", "off"}:
return False
return default
def _coerce_int(value: Any, default: int) -> int:
try:
return int(value)
except Exception:
return default
def _format_epoch(raw_value: Any) -> str:
try:
stamp = int(raw_value)
except Exception:
return ""
try:
return datetime.fromtimestamp(stamp).strftime("%Y-%m-%d %H:%M")
except Exception:
return str(raw_value or "")
def _safe_filename(name: Any) -> str:
raw = str(name or "").strip()
if not raw:
raw = "download"
cleaned = "".join(ch if ch.isalnum() or ch in {"-", "_", ".", " "} else "_" for ch in raw)
cleaned = cleaned.strip(" ._")
return cleaned or "download"
def _unique_path(path: Path) -> Path:
if not path.exists():
return path
stem = path.stem or "download"
suffix = path.suffix
counter = 1
while True:
candidate = path.with_name(f"{stem}_{counter}{suffix}")
if not candidate.exists():
return candidate
counter += 1
class SCP(Provider):
PLUGIN_NAME = "scp"
URL = ("scp://", "sftp://")
@property
def label(self) -> str:
return "SCP"
@property
def preserve_order(self) -> bool:
return True
@classmethod
def config_schema(cls) -> List[Dict[str, Any]]:
return [
{
"key": "host",
"label": "Host",
"default": "",
"required": True,
"placeholder": "ssh.example.com",
},
{
"key": "port",
"label": "Port",
"type": "integer",
"default": 22,
},
{
"key": "username",
"label": "Username",
"default": "",
"required": True,
"placeholder": "deploy",
},
{
"key": "password",
"label": "Password",
"type": "secret",
"secret": True,
"default": "",
},
{
"key": "key_path",
"label": "SSH Key Path",
"type": "path",
"default": "",
"placeholder": "C:/Users/Admin/.ssh/id_ed25519",
},
{
"key": "base_path",
"label": "Base Path",
"default": "/",
"placeholder": "/srv/files",
},
{
"key": "timeout",
"label": "Timeout Seconds",
"type": "integer",
"default": 20,
},
{
"key": "search_depth",
"label": "Default Search Depth",
"type": "integer",
"default": 1,
},
{
"key": "allow_agent",
"label": "Use SSH Agent",
"type": "boolean",
"default": True,
},
{
"key": "look_for_keys",
"label": "Look For Default Keys",
"type": "boolean",
"default": True,
},
]
def __init__(self, config: Optional[Dict[str, Any]] = None):
super().__init__(config)
conf = _pick_provider_config(self.config)
self._host = str(conf.get("host") or "").strip()
self._port = _coerce_int(conf.get("port"), 22)
self._username = str(conf.get("username") or conf.get("user") or "").strip()
self._password = str(conf.get("password") or "").strip()
self._key_path = str(conf.get("key_path") or conf.get("identity_file") or "").strip()
self._timeout = max(1, _coerce_int(conf.get("timeout"), 20))
self._search_depth = max(0, _coerce_int(conf.get("search_depth"), 1))
self._allow_agent = _coerce_bool(conf.get("allow_agent"), True)
self._look_for_keys = _coerce_bool(conf.get("look_for_keys"), True)
self._base_path = self._normalize_remote_path(conf.get("base_path") or "/", default="/")
def validate(self) -> bool:
return bool(self._host and self._username)
def config_helper_text(self) -> str:
return "Test the SSH/SCP connection before searching. You can also generate an RSA key pair from here."
def config_actions(self) -> List[Dict[str, Any]]:
return [
{
"id": "test_connection",
"label": "Test connection",
"variant": "primary",
},
{
"id": "generate_ssh_key",
"label": "Generate SSH key",
"variant": "default",
},
]
def run_config_action(self, action_id: str, **_kwargs: Any) -> Dict[str, Any]:
normalized = str(action_id or "").strip().lower()
if normalized == "test_connection":
return self._run_test_connection()
if normalized == "generate_ssh_key":
return self._generate_ssh_keypair()
return super().run_config_action(action_id, **_kwargs)
def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]:
text, inline = parse_inline_query_arguments(query)
filters: Dict[str, Any] = {}
if inline.get("path"):
filters["path"] = inline.get("path")
if inline.get("depth"):
filters["depth"] = max(0, _coerce_int(inline.get("depth"), self._search_depth))
if inline.get("type"):
filters["type"] = str(inline.get("type") or "").strip().lower()
return text, filters
def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str:
active_path = self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path)
text = str(query or "").strip()
if not text or text == "*":
return f"SCP: {active_path}"
return f"SCP: {text} @ {active_path}"
def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
return {
"plugin": self.name,
"host": self._host,
"path": self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path),
"query": str(query or "").strip(),
}
def search(
self,
query: str,
limit: int = 50,
filters: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> List[SearchResult]:
_ = kwargs
active_filters = dict(filters or {})
start_path = self._normalize_remote_path(active_filters.get("path") or self._base_path, default=self._base_path)
search_depth = max(0, _coerce_int(active_filters.get("depth"), self._search_depth))
type_filter = str(active_filters.get("type") or "any").strip().lower()
needle = str(query or "").strip()
max_results = max(0, int(limit or 0))
if max_results <= 0:
return []
ssh = self._connect_ssh()
sftp = None
try:
try:
sftp = self._open_sftp(ssh)
except Exception as exc:
if not self._is_sftp_negotiation_error(exc):
raise
return self._search_directory_via_ssh(
ssh,
start_path,
needle=needle,
limit=max_results,
search_depth=search_depth,
type_filter=type_filter,
)
return self._search_directory(
sftp,
start_path,
needle=needle,
limit=max_results,
search_depth=search_depth,
type_filter=type_filter,
)
finally:
self._close_client(sftp)
self._close_client(ssh)
def selector(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
**_kwargs: Any,
) -> bool:
if not stage_is_last:
return False
target_path = ""
target_title = ""
for item in selected_items or []:
metadata = self._item_metadata(item)
if not metadata.get("is_dir"):
continue
target_path = self._normalize_remote_path(metadata.get("scp_path") or metadata.get("selection_path"), default=self._base_path)
target_title = str(metadata.get("title") or metadata.get("name") or "").strip()
if target_path:
break
if not target_path:
return False
ssh = self._connect_ssh()
sftp = None
try:
try:
sftp = self._open_sftp(ssh)
except Exception as exc:
if not self._is_sftp_negotiation_error(exc):
raise
rows = self._search_directory_via_ssh(
ssh,
target_path,
needle="*",
limit=500,
search_depth=0,
type_filter="any",
)
else:
rows = self._search_directory(
sftp,
target_path,
needle="*",
limit=500,
search_depth=0,
type_filter="any",
)
finally:
self._close_client(sftp)
self._close_client(ssh)
try:
from SYS.result_table import Table
from SYS.rich_display import stdout_console
except Exception:
return True
title = target_title or target_path
table = Table(f"SCP: {title}")._perseverance(True)
table.set_table("scp")
try:
table.set_table_metadata({
"provider": "scp",
"host": self._host,
"path": target_path,
"view": "directory",
})
except Exception:
pass
table.set_source_command("search-file", ["-plugin", "scp", f"path:{target_path}", "*"])
payloads: List[Dict[str, Any]] = []
for row in rows:
table.add_result(row)
payloads.append(row.to_dict())
try:
ctx.set_last_result_table(table, payloads, subject={"plugin": "scp", "path": target_path})
ctx.set_current_stage_table(table)
except Exception:
pass
try:
stdout_console().print()
stdout_console().print(table)
except Exception:
pass
return True
def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]:
metadata = getattr(result, "full_metadata", None)
if isinstance(metadata, dict) and metadata.get("is_dir"):
return None
target = str(getattr(result, "path", "") or "").strip()
if not target:
return None
return self.download_url(target, output_dir, title=getattr(result, "title", None))
def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]:
settings = self._connection_settings_for_url(url)
remote_path = settings["path"]
if not remote_path or remote_path == "/":
return None
filename_hint = str(kwargs.get("title") or "").strip()
parsed_name = posixpath.basename(remote_path.rstrip("/"))
filename = _safe_filename(filename_hint or unquote(parsed_name) or "download")
destination_dir = Path(output_dir)
destination_dir.mkdir(parents=True, exist_ok=True)
destination = _unique_path(destination_dir / filename)
ssh = self._connect_ssh(settings)
scp_client = None
try:
scp_client = self._open_scp(ssh)
scp_client.get(remote_path, local_path=str(destination))
return destination
except Exception:
try:
destination.unlink(missing_ok=True)
except Exception:
pass
return None
finally:
self._close_client(scp_client)
self._close_client(ssh)
def resolve_pipe_result_download(
self,
result: Any,
pipe_obj: Any,
) -> Tuple[Optional[Path], Optional[str], Optional[Path]]:
metadata = self._item_metadata(result, pipe_obj=pipe_obj)
if metadata.get("is_dir"):
return None, None, None
download_url = str(
metadata.get("selection_url")
or metadata.get("scp_url")
or metadata.get("path")
or ""
).strip()
if not download_url.startswith(("scp://", "sftp://")):
return None, None, None
temp_dir = Path(tempfile.mkdtemp(prefix="scp-add-file-"))
downloaded = self.download_url(download_url, temp_dir, title=metadata.get("title"))
if downloaded is None:
try:
temp_dir.rmdir()
except Exception:
pass
return None, None, None
try:
if pipe_obj is not None:
pipe_obj.is_temp = True
except Exception:
pass
return downloaded, None, temp_dir
def upload(self, file_path: str, **kwargs: Any) -> str:
local_path = Path(str(file_path or "")).expanduser()
if not local_path.exists() or not local_path.is_file():
raise FileNotFoundError(f"File not found: {local_path}")
remote_dir = self._normalize_remote_path(kwargs.get("remote_path") or kwargs.get("path") or self._base_path, default=self._base_path)
remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name
remote_path = self._join_remote_path(remote_dir, remote_name)
ssh = self._connect_ssh()
sftp = None
scp_client = None
try:
try:
sftp = self._open_sftp(ssh)
except Exception as exc:
if not self._is_sftp_negotiation_error(exc):
raise
self._ensure_directory_via_ssh(ssh, remote_dir)
else:
self._ensure_directory(sftp, remote_dir)
scp_client = self._open_scp(ssh)
scp_client.put(str(local_path), remote_path=remote_path)
finally:
self._close_client(scp_client)
self._close_client(sftp)
self._close_client(ssh)
return self._build_url(remote_path)
def _run_test_connection(self) -> Dict[str, Any]:
if not self._host:
return {"ok": False, "message": "Set 'host' before testing the SCP connection."}
if not self._username:
return {"ok": False, "message": "Set 'username' before testing the SCP connection."}
ssh = None
sftp = None
try:
ssh = self._connect_ssh()
base_path = self._base_path or "/"
transport_detail = "SFTP available"
try:
sftp = self._open_sftp(ssh)
except Exception as exc:
if not self._is_sftp_negotiation_error(exc):
raise
is_dir = self._path_exists_via_ssh(ssh, base_path)
transport_detail = "SFTP unavailable; using SSH command fallback"
else:
try:
attrs = sftp.stat(base_path)
is_dir = stat.S_ISDIR(getattr(attrs, "st_mode", 0))
except Exception:
is_dir = False
detail = f" and confirmed {base_path}" if is_dir else ""
auth_mode = f"key {self._key_path}" if self._key_path else "password/agent auth"
return {
"ok": True,
"message": f"Connected to SCP {self._host}:{self._port} as {self._username} via {auth_mode}. {transport_detail}{detail}.",
}
except Exception as exc:
return {"ok": False, "message": f"SCP connection failed: {exc}"}
finally:
self._close_client(sftp)
self._close_client(ssh)
def _generate_ssh_keypair(self) -> Dict[str, Any]:
target = Path(self._key_path).expanduser() if self._key_path else (Path.home() / ".ssh" / "medeia_scp_rsa")
try:
target.parent.mkdir(parents=True, exist_ok=True)
except Exception as exc:
return {"ok": False, "message": f"Could not create key directory: {exc}"}
public_path = target.with_name(target.name + ".pub")
if target.exists() or public_path.exists():
return {
"ok": False,
"message": f"SSH key already exists at {target}. Remove it or choose a different key_path first.",
}
try:
key = paramiko.RSAKey.generate(bits=4096)
key.write_private_key_file(str(target))
comment = f"{self._username or 'medeia'}@{self._host or 'scp'}"
public_path.write_text(f"{key.get_name()} {key.get_base64()} {comment}\n", encoding="utf-8")
try:
target.chmod(0o600)
except Exception:
pass
return {
"ok": True,
"message": f"Generated SSH key pair at {target}. Save the config to persist key_path.",
"config_updates": {"key_path": str(target)},
}
except Exception as exc:
try:
target.unlink(missing_ok=True)
except Exception:
pass
try:
public_path.unlink(missing_ok=True)
except Exception:
pass
return {"ok": False, "message": f"SSH key generation failed: {exc}"}
def _connect_ssh(self, overrides: Optional[Dict[str, Any]] = None) -> paramiko.SSHClient:
settings = dict(overrides or {})
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(
hostname=str(settings.get("host") or self._host),
port=int(settings.get("port") or self._port),
username=str(settings.get("username") or self._username),
password=str(settings.get("password") or self._password) or None,
key_filename=str(settings.get("key_path") or self._key_path) or None,
timeout=self._timeout,
allow_agent=self._allow_agent if "allow_agent" not in settings else bool(settings.get("allow_agent")),
look_for_keys=self._look_for_keys if "look_for_keys" not in settings else bool(settings.get("look_for_keys")),
)
return client
def _open_sftp(self, ssh: Any) -> Any:
return ssh.open_sftp()
def _open_scp(self, ssh: Any) -> Any:
return SCPClient(ssh.get_transport())
def _is_sftp_negotiation_error(self, exc: Exception) -> bool:
text = str(exc or "").strip().lower()
if isinstance(exc, EOFError):
return True
return any(
marker in text
for marker in (
"eof during negotiation",
"open failed",
"channel closed",
"administratively prohibited",
"subsystem request failed",
)
)
def _run_ssh_command(self, ssh: Any, command: str) -> Tuple[int, str, str]:
stdin, stdout, stderr = ssh.exec_command(command, timeout=self._timeout)
try:
stdin.close()
except Exception:
pass
output = stdout.read().decode("utf-8", errors="replace")
error = stderr.read().decode("utf-8", errors="replace")
status = 0
try:
status = int(stdout.channel.recv_exit_status())
except Exception:
status = 0
return status, output, error
def _path_exists_via_ssh(self, ssh: Any, remote_path: str) -> bool:
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
quoted_path = shlex.quote(normalized)
status, _, _ = self._run_ssh_command(ssh, f"test -d {quoted_path}")
return status == 0
def _ensure_directory_via_ssh(self, ssh: Any, remote_path: str) -> None:
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
if normalized == "/":
return
quoted_path = shlex.quote(normalized)
status, _, error = self._run_ssh_command(ssh, f"mkdir -p {quoted_path}")
if status != 0:
raise RuntimeError(error.strip() or f"mkdir -p failed for {normalized}")
def _close_client(self, client: Any) -> None:
if client is None:
return
try:
client.close()
except Exception:
pass
def _normalize_remote_path(self, value: Any, *, default: str) -> str:
text = str(value or "").strip().replace("\\", "/")
if not text:
text = default
elif text.startswith(("scp://", "sftp://")):
try:
text = unquote(urlparse(text).path or "/")
except Exception:
text = default
elif not text.startswith("/"):
text = posixpath.join(default, text)
normalized = posixpath.normpath(text)
normalized = "/" + normalized.lstrip("/")
return normalized or "/"
def _join_remote_path(self, parent: Any, child: Any) -> str:
left = self._normalize_remote_path(parent, default=self._base_path)
right = str(child or "").strip().replace("\\", "/")
if not right:
return left
return self._normalize_remote_path(posixpath.join(left, right), default="/")
def _build_url(
self,
remote_path: Any,
*,
host: Optional[str] = None,
port: Optional[int] = None,
scheme: str = "scp",
) -> str:
path_text = self._normalize_remote_path(remote_path, default="/")
host_text = str(host or self._host).strip()
port_value = int(port or self._port)
port_suffix = f":{port_value}" if port_value and port_value != 22 else ""
return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}"
def _connection_settings_for_url(self, url: str) -> Dict[str, Any]:
parsed = urlparse(str(url or "").strip())
scheme = (parsed.scheme or "scp").strip().lower()
host = parsed.hostname or self._host
port = parsed.port or self._port
username = parsed.username or self._username
password = parsed.password or self._password
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default="/")
return {
"scheme": scheme,
"host": host,
"port": port,
"username": username,
"password": password,
"key_path": self._key_path,
"allow_agent": self._allow_agent,
"look_for_keys": self._look_for_keys,
"path": path_text,
}
def _search_directory(
self,
sftp: Any,
start_path: str,
*,
needle: str,
limit: int,
search_depth: int,
type_filter: str,
) -> List[SearchResult]:
results: List[SearchResult] = []
visited: set[str] = set()
def walk(current_path: str, depth_left: int) -> None:
normalized = self._normalize_remote_path(current_path, default=self._base_path)
if normalized in visited or len(results) >= limit:
return
visited.add(normalized)
for entry in self._list_directory(sftp, normalized):
if len(results) >= limit:
return
if self._matches_entry(entry, needle=needle, type_filter=type_filter):
results.append(self._build_result(entry))
if entry.get("is_dir") and depth_left > 0:
walk(str(entry.get("scp_path") or normalized), depth_left - 1)
walk(start_path, max(0, search_depth))
return results
def _search_directory_via_ssh(
self,
ssh: Any,
start_path: str,
*,
needle: str,
limit: int,
search_depth: int,
type_filter: str,
) -> List[SearchResult]:
entries = self._list_directory_via_ssh(ssh, start_path, depth=search_depth)
results: List[SearchResult] = []
for entry in entries:
if len(results) >= limit:
break
if self._matches_entry(entry, needle=needle, type_filter=type_filter):
results.append(self._build_result(entry))
return results
def _matches_entry(self, entry: Dict[str, Any], *, needle: str, type_filter: str) -> bool:
is_dir = bool(entry.get("is_dir"))
if type_filter in {"dir", "dirs", "folder", "folders"} and not is_dir:
return False
if type_filter in {"file", "files"} and is_dir:
return False
text = str(needle or "").strip().lower()
if not text or text in {"*", "all", "list"}:
return True
haystacks = [
str(entry.get("name") or "").lower(),
str(entry.get("scp_path") or "").lower(),
]
for token in [part for part in text.split() if part]:
if any(ch in token for ch in "*?[]"):
if not any(fnmatch.fnmatch(haystack, token) for haystack in haystacks):
return False
elif not any(token in haystack for haystack in haystacks):
return False
return True
def _build_result(self, entry: Dict[str, Any]) -> SearchResult:
scp_path = str(entry.get("scp_path") or "/")
scp_url = self._build_url(scp_path)
is_dir = bool(entry.get("is_dir"))
size_value = entry.get("size")
modified = str(entry.get("modified") or "")
parent = posixpath.dirname(scp_path.rstrip("/")) or "/"
metadata = {
"provider": "scp",
"host": self._host,
"scp_path": scp_path,
"scp_url": scp_url,
"selection_url": scp_url,
"is_dir": is_dir,
"name": str(entry.get("name") or "").strip(),
}
if size_value is not None:
metadata["size"] = size_value
if modified:
metadata["modified"] = modified
return SearchResult(
table="scp",
title=str(entry.get("name") or scp_path),
path=scp_url,
detail=parent,
annotations=["folder" if is_dir else "file"],
media_kind="folder" if is_dir else "file",
size_bytes=int(size_value) if isinstance(size_value, int) else None,
tag={"scp", "folder" if is_dir else "file"},
columns=[
("Name", str(entry.get("name") or "")),
("Type", "dir" if is_dir else "file"),
("Directory", parent),
("Size", "" if size_value is None else str(size_value)),
("Modified", modified),
],
selection_args=None if is_dir else ["-url", scp_url],
selection_action=None if is_dir else ["download-file", "-plugin", "scp", "-url", scp_url],
full_metadata=metadata,
)
def _list_directory(self, sftp: Any, remote_path: str) -> List[Dict[str, Any]]:
try:
attrs = sftp.listdir_attr(remote_path)
except Exception:
return []
entries: List[Dict[str, Any]] = []
for attr in attrs:
name_text = str(getattr(attr, "filename", "") or "").strip()
if not name_text or name_text in {".", ".."}:
continue
mode = getattr(attr, "st_mode", 0)
is_dir = stat.S_ISDIR(mode)
size_value = getattr(attr, "st_size", None)
try:
size_int = int(size_value) if size_value is not None else None
except Exception:
size_int = None
entries.append(
{
"name": name_text,
"scp_path": self._join_remote_path(remote_path, name_text),
"is_dir": is_dir,
"size": size_int,
"modified": _format_epoch(getattr(attr, "st_mtime", None)),
}
)
return entries
def _list_directory_via_ssh(self, ssh: Any, remote_path: str, *, depth: int) -> List[Dict[str, Any]]:
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
max_depth = max(1, int(depth) + 1)
quoted_path = shlex.quote(normalized)
command = (
f"find {quoted_path} -mindepth 1 -maxdepth {max_depth} "
f"\\( -type d -o -type f \\) -exec sh -c 'for path do "
f"if [ -d \"$path\" ]; then kind=d; else kind=f; fi; "
f"name=$(basename \"$path\"); "
f"printf \"%s\\0%s\\0%s\\0\" \"$kind\" \"$path\" \"$name\"; "
f"done' sh {{}} +"
)
status, output, error = self._run_ssh_command(ssh, command)
if status != 0:
error_text = error.strip().lower()
if "no such file" in error_text or "cannot access" in error_text:
return []
raise RuntimeError(error.strip() or f"SSH listing failed for {normalized}")
chunks = [part for part in output.split("\0") if part]
entries: List[Dict[str, Any]] = []
for index in range(0, len(chunks), 3):
if index + 2 >= len(chunks):
break
kind = chunks[index]
scp_path = self._normalize_remote_path(chunks[index + 1], default=normalized)
name_text = str(chunks[index + 2] or "").strip()
if not name_text or name_text in {".", ".."}:
continue
entries.append(
{
"name": name_text,
"scp_path": scp_path,
"is_dir": kind == "d",
"size": None,
"modified": "",
}
)
return entries
def _ensure_directory(self, sftp: Any, remote_path: str) -> None:
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
if normalized == "/":
return
partial = ""
for segment in [part for part in normalized.split("/") if part]:
partial = f"{partial}/{segment}"
try:
attrs = sftp.stat(partial)
if stat.S_ISDIR(getattr(attrs, "st_mode", 0)):
continue
except Exception:
pass
try:
sftp.mkdir(partial)
except Exception:
try:
attrs = sftp.stat(partial)
if stat.S_ISDIR(getattr(attrs, "st_mode", 0)):
continue
except Exception:
pass
raise
def _item_metadata(self, item: Any, *, pipe_obj: Any = None) -> Dict[str, Any]:
metadata: Dict[str, Any] = {}
for source in (item, pipe_obj):
if isinstance(source, dict):
for key in ("title", "path", "url"):
if source.get(key) is not None and key not in metadata:
metadata[key] = source.get(key)
nested = source.get("full_metadata") or source.get("metadata")
if isinstance(nested, dict):
metadata.update(nested)
elif source is not None:
for attr in ("title", "path", "url"):
try:
value = getattr(source, attr, None)
except Exception:
value = None
if value is not None and attr not in metadata:
metadata[attr] = value
try:
nested = getattr(source, "full_metadata", None) or getattr(source, "metadata", None)
except Exception:
nested = None
if isinstance(nested, dict):
metadata.update(nested)
scp_path = metadata.get("scp_path") or metadata.get("selection_path")
if not scp_path:
path_value = metadata.get("path") or metadata.get("url") or metadata.get("scp_url")
path_text = str(path_value or "").strip()
if path_text.startswith(("scp://", "sftp://")):
scp_path = self._normalize_remote_path(path_text, default=self._base_path)
if scp_path:
metadata["scp_path"] = self._normalize_remote_path(scp_path, default=self._base_path)
metadata.setdefault("selection_path", metadata["scp_path"])
if metadata.get("scp_path") and not metadata.get("scp_url"):
metadata["scp_url"] = self._build_url(metadata["scp_path"])
if metadata.get("scp_url") and not metadata.get("selection_url"):
metadata["selection_url"] = metadata["scp_url"]
is_dir = metadata.get("is_dir")
if is_dir is None and metadata.get("media_kind"):
is_dir = str(metadata.get("media_kind") or "").strip().lower() == "folder"
metadata["is_dir"] = bool(is_dir)
return metadata
+87 -11
View File
@@ -12,10 +12,18 @@ from pathlib import Path
from typing import Any, Dict, List, Optional
from ProviderCore.base import Provider, SearchResult
from SYS.logger import log, debug
from SYS.logger import log, debug, debug_panel
from SYS.models import ProgressBar
_SOULSEEK_NOISE_SUBSTRINGS = (
"unhandled exception on loop",
"Task exception was never retrieved",
"future: <Task finished",
"ConnectionFailedError",
"PeerConnectionError",
"indirect connection failed",
"indirect connection timed out",
"failed to connect",
"search reply ticket does not match any search request",
"failed to receive transfer ticket on file connection",
"aioslsk.exceptions.ConnectionReadError",
@@ -59,10 +67,10 @@ async def _suppress_aioslsk_asyncio_task_noise() -> Any:
if msg == "Task exception was never retrieved":
cls = getattr(exc, "__class__", None)
name = getattr(cls, "__name__", "")
mod = getattr(cls, "__module__", "")
exc_text = str(exc or "").lower()
# Suppress ConnectionFailedError from aioslsk
if name == "ConnectionFailedError" and str(mod).startswith("aioslsk"):
# Suppress expected peer direct-connect failures from aioslsk.
if name == "ConnectionFailedError" or "failed to connect" in exc_text:
return
except Exception:
# If our filter logic fails, fall through to default handling.
@@ -117,6 +125,9 @@ class _LineFilterStream(io.TextIOBase):
self._in_tb = False
self._tb_lines: list[str] = []
self._tb_suppress = False
self._in_task_block = False
self._task_lines: list[str] = []
self._task_suppress = False
def writable(self) -> bool: # pragma: no cover
return True
@@ -137,6 +148,19 @@ class _LineFilterStream(io.TextIOBase):
self._tb_suppress = False
self._in_tb = False
def _flush_task_block(self) -> None:
if not self._task_lines:
return
if not self._task_suppress:
for l in self._task_lines:
try:
self._underlying.write(l + "\n")
except Exception:
pass
self._task_lines = []
self._task_suppress = False
self._in_task_block = False
def write(self, s: str) -> int:
self._buf += str(s)
while "\n" in self._buf:
@@ -145,6 +169,29 @@ class _LineFilterStream(io.TextIOBase):
return len(s)
def _handle_line(self, line: str) -> None:
if not self._in_task_block and self._should_suppress_line(line) and (
line.startswith("Task exception was never retrieved")
or line.startswith("future: <Task finished")
or line.startswith("unhandled exception on loop")
):
self._in_task_block = True
self._task_lines = [line]
self._task_suppress = True
return
if self._in_task_block:
if line.startswith("Traceback (most recent call last):"):
self._in_tb = True
self._tb_lines = [line]
self._tb_suppress = True
return
self._task_lines.append(line)
if self._should_suppress_line(line):
self._task_suppress = True
if line.strip() == "":
self._flush_task_block()
return
# Start capturing tracebacks so we can suppress the whole block if it matches.
if not self._in_tb and line.startswith("Traceback (most recent call last):"):
self._in_tb = True
@@ -159,6 +206,8 @@ class _LineFilterStream(io.TextIOBase):
# End traceback block on blank line.
if line.strip() == "":
self._flush_tb()
if self._in_task_block:
self._flush_task_block()
return
# Non-traceback line
@@ -174,6 +223,8 @@ class _LineFilterStream(io.TextIOBase):
if self._in_tb:
# If the traceback ends without a trailing blank line, decide here.
self._flush_tb()
if self._in_task_block:
self._flush_task_block()
if self._buf:
line = self._buf
self._buf = ""
@@ -422,14 +473,14 @@ class Soulseek(Provider):
try:
search_request = await client.searches.search(query)
await self._collect_results(search_request, timeout=timeout)
return self._flatten_results(search_request)[:limit]
summary = await self._collect_results(search_request, timeout=timeout)
return self._flatten_results(search_request)[:limit], summary
except Exception as exc:
log(
f"[soulseek] Search error: {type(exc).__name__}: {exc}",
file=sys.stderr
)
return []
return [], {}
finally:
# Best-effort: try to cancel/close the search request before stopping
# the client to reduce stray reply spam.
@@ -477,16 +528,24 @@ class Soulseek(Provider):
self,
search_request: Any,
timeout: float = 75.0
) -> None:
end = time.time() + timeout
) -> Dict[str, Any]:
start = time.time()
end = start + timeout
last_count = 0
update_count = 0
while time.time() < end:
current_count = len(getattr(search_request, "results", []))
if current_count > last_count:
debug(f"[soulseek] Got {current_count} result(s)...")
last_count = current_count
update_count += 1
await asyncio.sleep(0.5)
return {
"peer_hits": last_count,
"count_updates": update_count,
"elapsed_seconds": round(max(0.0, time.time() - start), 1),
}
def search(
self,
query: str,
@@ -503,7 +562,7 @@ class Soulseek(Provider):
base_tmp.mkdir(parents=True, exist_ok=True)
try:
flat_results = asyncio.run(
flat_results, search_summary = asyncio.run(
self.perform_search(query,
timeout=9.0,
limit=limit)
@@ -636,6 +695,23 @@ class Soulseek(Provider):
)
)
try:
debug_panel(
"soulseek search",
[
("query", query),
("peer_hits", search_summary.get("peer_hits", 0)),
("file_hits", len(flat_results)),
("audio_hits", len(music_results)),
("results", len(results)),
("poll_updates", search_summary.get("count_updates", 0)),
("elapsed_s", search_summary.get("elapsed_seconds", 0.0)),
],
border_style="magenta",
)
except Exception:
pass
return results
except Exception as exc:
+24 -3
View File
@@ -1286,9 +1286,30 @@ class ytdlp(TableProviderMixin, Provider):
try:
from SYS.result_table_adapters import register_plugin
from SYS.result_table_adapters import get_plugin, register_plugin
from SYS.result_table_api import ResultModel, ColumnSpec, metadata_column, title_column
def _register_table_plugin_once(
name: str,
adapter: Any,
*,
columns: Any,
selection_fn: Any,
metadata: Optional[Dict[str, Any]] = None,
) -> None:
try:
get_plugin(name)
return
except KeyError:
pass
register_plugin(
name,
adapter,
columns=columns,
selection_fn=selection_fn,
metadata=metadata,
)
def _convert_format_result_to_model(sr: Any) -> ResultModel:
d = sr.to_dict() if hasattr(sr, "to_dict") else (sr if isinstance(sr, dict) else {})
title = d.get("title") or f"Format {d.get('format_id', 'unknown')}"
@@ -1361,7 +1382,7 @@ try:
return result_args
return []
register_plugin(
_register_table_plugin_once(
"ytdlp.formatlist",
_adapter,
columns=_columns_factory,
@@ -1421,7 +1442,7 @@ try:
return ["-url", row.path]
return ["-title", row.title or ""]
register_plugin(
_register_table_plugin_once(
"ytdlp.search",
_search_adapter,
columns=_search_columns_factory,
+2
View File
@@ -70,6 +70,8 @@ dependencies = [
# Browser automation
"playwright>=1.40.0",
"paramiko>=3.5.0",
"scp>=0.15.0",
# Development and utilities
"python-dateutil>=2.8.0",