updated plugin refactor and added FTP and SCP plugins , also hydrusnetwork plugin migration
This commit is contained in:
@@ -657,6 +657,7 @@ class CmdletCompleter(Completer):
|
||||
return value
|
||||
|
||||
def _used_arg_logicals(
|
||||
self,
|
||||
cmd_name: str,
|
||||
stage_tokens: List[str],
|
||||
config: Dict[str,
|
||||
|
||||
@@ -422,6 +422,24 @@ class Provider(ABC):
|
||||
_ = quiet_mode
|
||||
return None
|
||||
|
||||
def config_helper_text(self) -> str:
|
||||
"""Optional helper text shown in the config editor."""
|
||||
|
||||
return ""
|
||||
|
||||
def config_actions(self) -> List[Dict[str, Any]]:
|
||||
"""Optional actions exposed in the config editor for this provider."""
|
||||
|
||||
return []
|
||||
|
||||
def run_config_action(self, action_id: str, **_kwargs: Any) -> Dict[str, Any]:
|
||||
"""Execute a provider-owned config action from the config editor."""
|
||||
|
||||
return {
|
||||
"ok": False,
|
||||
"message": f"Provider '{self.name}' does not support config action '{action_id}'.",
|
||||
}
|
||||
|
||||
def upload(self, file_path: str, **kwargs: Any) -> str:
|
||||
"""Upload a file and return a URL or identifier."""
|
||||
raise NotImplementedError(f"Provider '{self.name}' does not support upload")
|
||||
|
||||
@@ -136,7 +136,28 @@ class ProviderRegistry:
|
||||
self._discovered = False
|
||||
self._external_dirs_scanned = False
|
||||
|
||||
def _ensure_builtin_package_dirs(self) -> None:
|
||||
if self._builtin_package_dirs or not self.package_name:
|
||||
return
|
||||
try:
|
||||
package = importlib.import_module(self.package_name)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
package_path = getattr(package, "__path__", None)
|
||||
if not package_path:
|
||||
return
|
||||
|
||||
builtin_dirs: List[Path] = []
|
||||
for entry in package_path:
|
||||
try:
|
||||
builtin_dirs.append(Path(str(entry)).resolve())
|
||||
except Exception:
|
||||
builtin_dirs.append(Path(str(entry)))
|
||||
self._builtin_package_dirs = tuple(builtin_dirs)
|
||||
|
||||
def _is_builtin_package_dir(self, candidate: Path) -> bool:
|
||||
self._ensure_builtin_package_dirs()
|
||||
try:
|
||||
resolved = candidate.resolve()
|
||||
except Exception:
|
||||
@@ -253,6 +274,7 @@ class ProviderRegistry:
|
||||
if self._external_dirs_scanned:
|
||||
return
|
||||
self._external_dirs_scanned = True
|
||||
self._ensure_builtin_package_dirs()
|
||||
|
||||
for plugin_dir in _iter_external_plugin_dirs():
|
||||
if self._is_builtin_package_dir(plugin_dir):
|
||||
@@ -302,15 +324,8 @@ class ProviderRegistry:
|
||||
return
|
||||
|
||||
self._register_module(package)
|
||||
self._ensure_builtin_package_dirs()
|
||||
package_path = getattr(package, "__path__", None)
|
||||
if package_path:
|
||||
builtin_dirs: List[Path] = []
|
||||
for entry in package_path:
|
||||
try:
|
||||
builtin_dirs.append(Path(str(entry)).resolve())
|
||||
except Exception:
|
||||
builtin_dirs.append(Path(str(entry)))
|
||||
self._builtin_package_dirs = tuple(builtin_dirs)
|
||||
if not package_path:
|
||||
self._discover_external_plugins()
|
||||
return
|
||||
|
||||
+144
-39
@@ -8,6 +8,7 @@ import time
|
||||
import os
|
||||
import re
|
||||
import datetime
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from copy import deepcopy
|
||||
@@ -532,7 +533,7 @@ def resolve_cookies_path(
|
||||
if candidate.is_file():
|
||||
return candidate
|
||||
|
||||
plugin_cookie = resolve_plugin_asset_path("ytdlp", "cookies.txt", script_dir=base_dir)
|
||||
plugin_cookie = _resolve_ytdlp_plugin_cookie_path(base_dir)
|
||||
if plugin_cookie is not None:
|
||||
return plugin_cookie
|
||||
|
||||
@@ -542,6 +543,30 @@ def resolve_cookies_path(
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_ytdlp_plugin_cookie_path(base_dir: Path) -> Optional[Path]:
|
||||
plugin_cookie = resolve_plugin_asset_path("ytdlp", "cookies.txt", script_dir=base_dir)
|
||||
if plugin_cookie is not None:
|
||||
return plugin_cookie
|
||||
|
||||
plugin_dir = _resolve_app_root(base_dir) / "plugins" / "ytdlp"
|
||||
if not plugin_dir.is_dir():
|
||||
return None
|
||||
|
||||
plugin_cookie = plugin_dir / "cookies.txt"
|
||||
legacy_cookie = _resolve_app_root(base_dir) / "cookies.txt"
|
||||
try:
|
||||
if legacy_cookie.is_file() and not plugin_cookie.exists():
|
||||
plugin_cookie.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(legacy_cookie, plugin_cookie)
|
||||
return plugin_cookie
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
if plugin_cookie.is_file():
|
||||
return plugin_cookie
|
||||
return None
|
||||
|
||||
|
||||
def resolve_debug_log(config: Dict[str, Any]) -> Optional[Path]:
|
||||
value = config.get("download_debug_log")
|
||||
if not value:
|
||||
@@ -721,6 +746,105 @@ def _count_changed_entries(old_config: Dict[str, Any], new_config: Dict[str, Any
|
||||
return len(changed) + len(removed)
|
||||
|
||||
|
||||
def _changed_entry_keys(old_config: Dict[str, Any], new_config: Dict[str, Any]) -> set[Tuple[str, str, str, str]]:
|
||||
old_entries = _flatten_config_entries(old_config or {})
|
||||
new_entries = _flatten_config_entries(new_config or {})
|
||||
keys = set(old_entries) | set(new_entries)
|
||||
return {key for key in keys if old_entries.get(key, _CONFIG_MISSING) != new_entries.get(key, _CONFIG_MISSING)}
|
||||
|
||||
|
||||
def _config_from_flattened_entries(
|
||||
entries: Dict[Tuple[str, str, str, str], Any],
|
||||
) -> Dict[str, Any]:
|
||||
config: Dict[str, Any] = {}
|
||||
for (category, subtype, item_name, key), value in entries.items():
|
||||
if category == "global":
|
||||
config[key] = value
|
||||
continue
|
||||
|
||||
if category == "store":
|
||||
store_block = config.setdefault("store", {})
|
||||
subtype_block = store_block.setdefault(subtype, {})
|
||||
item_block = subtype_block.setdefault(item_name, {})
|
||||
item_block[key] = value
|
||||
continue
|
||||
|
||||
if category in {"provider", "tool"}:
|
||||
category_block = config.setdefault(category, {})
|
||||
subtype_block = category_block.setdefault(subtype, {})
|
||||
subtype_block[key] = value
|
||||
continue
|
||||
|
||||
category_block = config.setdefault(category, {})
|
||||
if isinstance(category_block, dict):
|
||||
subtype_block = category_block.setdefault(subtype, {})
|
||||
if isinstance(subtype_block, dict):
|
||||
item_block = subtype_block.setdefault(item_name, {})
|
||||
if isinstance(item_block, dict):
|
||||
item_block[key] = value
|
||||
|
||||
_normalize_plugin_config_aliases(config)
|
||||
_sync_alldebrid_api_key(config)
|
||||
return config
|
||||
|
||||
|
||||
def _merge_non_conflicting_config_changes(
|
||||
base_config: Dict[str, Any],
|
||||
disk_config: Dict[str, Any],
|
||||
local_config: Dict[str, Any],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
local_changed = _changed_entry_keys(base_config, local_config)
|
||||
if not local_changed:
|
||||
return deepcopy(disk_config)
|
||||
|
||||
disk_changed = _changed_entry_keys(base_config, disk_config)
|
||||
if local_changed & disk_changed:
|
||||
return None
|
||||
|
||||
merged_entries = dict(_flatten_config_entries(disk_config or {}))
|
||||
local_entries = _flatten_config_entries(local_config or {})
|
||||
for key in local_changed:
|
||||
if key in local_entries:
|
||||
merged_entries[key] = local_entries[key]
|
||||
else:
|
||||
merged_entries.pop(key, None)
|
||||
return _config_from_flattened_entries(merged_entries)
|
||||
|
||||
|
||||
def _extract_expected_alldebrid_key(config: Dict[str, Any]) -> Optional[str]:
|
||||
expected_key = None
|
||||
try:
|
||||
providers = config.get("provider", {}) if isinstance(config, dict) else {}
|
||||
if isinstance(providers, dict):
|
||||
entry = providers.get("alldebrid")
|
||||
if entry is not None:
|
||||
if isinstance(entry, dict):
|
||||
for k in ("api_key", "API_KEY", "apikey", "APIKEY"):
|
||||
v = entry.get(k)
|
||||
if isinstance(v, str) and v.strip():
|
||||
expected_key = v.strip()
|
||||
break
|
||||
elif isinstance(entry, str) and entry.strip():
|
||||
expected_key = entry.strip()
|
||||
if not expected_key:
|
||||
store_block = config.get("store", {}) if isinstance(config, dict) else {}
|
||||
debrid = store_block.get("debrid") if isinstance(store_block, dict) else None
|
||||
if isinstance(debrid, dict):
|
||||
srv = debrid.get("all-debrid")
|
||||
if isinstance(srv, dict):
|
||||
for k in ("api_key", "API_KEY", "apikey", "APIKEY"):
|
||||
v = srv.get(k)
|
||||
if isinstance(v, str) and v.strip():
|
||||
expected_key = v.strip()
|
||||
break
|
||||
elif isinstance(srv, str) and srv.strip():
|
||||
expected_key = srv.strip()
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to determine expected AllDebrid key: %s", exc, exc_info=True)
|
||||
expected_key = None
|
||||
return expected_key
|
||||
|
||||
|
||||
def load_config(*, emit_summary: bool = True) -> Dict[str, Any]:
|
||||
global _CONFIG_CACHE, _LAST_SAVED_CONFIG, _CONFIG_SUMMARY_PENDING
|
||||
if _CONFIG_CACHE:
|
||||
@@ -838,6 +962,7 @@ def save_config(config: Dict[str, Any]) -> int:
|
||||
def _write_entries() -> int:
|
||||
global _CONFIG_CACHE, _LAST_SAVED_CONFIG
|
||||
count = 0
|
||||
config_to_write = config
|
||||
# Use the transaction-provided connection directly to avoid re-acquiring
|
||||
# the connection lock via db.* helpers which can lead to deadlock.
|
||||
with db.transaction() as conn:
|
||||
@@ -861,14 +986,22 @@ def save_config(config: Dict[str, Any]) -> int:
|
||||
_CONFIG_CACHE = current_disk
|
||||
_LAST_SAVED_CONFIG = deepcopy(current_disk)
|
||||
return 0
|
||||
# Otherwise, abort to avoid overwriting external changes
|
||||
raise ConfigSaveConflict(
|
||||
"Configuration on disk changed since you started editing; save aborted to prevent overwrite. Reload and reapply your changes."
|
||||
merged_config = _merge_non_conflicting_config_changes(
|
||||
previous_config,
|
||||
current_disk,
|
||||
config,
|
||||
)
|
||||
if merged_config is None:
|
||||
# Otherwise, abort to avoid overwriting external changes
|
||||
raise ConfigSaveConflict(
|
||||
"Configuration on disk changed since you started editing; save aborted to prevent overwrite. Reload and reapply your changes."
|
||||
)
|
||||
config_to_write = merged_config
|
||||
log("Config save rebased local changes onto newer disk configuration.")
|
||||
|
||||
# Proceed with writing when no conflicting external changes detected
|
||||
conn.execute("DELETE FROM config")
|
||||
for key, value in config.items():
|
||||
for key, value in config_to_write.items():
|
||||
if key in ('store', 'provider', 'tool') and isinstance(value, dict):
|
||||
for subtype, instances in value.items():
|
||||
if not isinstance(instances, dict):
|
||||
@@ -904,6 +1037,8 @@ def save_config(config: Dict[str, Any]) -> int:
|
||||
("global", "none", "none", key, val_str),
|
||||
)
|
||||
count += 1
|
||||
_CONFIG_CACHE = config_to_write
|
||||
_LAST_SAVED_CONFIG = deepcopy(config_to_write)
|
||||
return count
|
||||
|
||||
|
||||
@@ -964,9 +1099,6 @@ def save_config(config: Dict[str, Any]) -> int:
|
||||
logger.exception("Failed to release save lock after CRITICAL configuration save failure: %s", exc)
|
||||
raise
|
||||
|
||||
clear_config_cache()
|
||||
_CONFIG_CACHE = config
|
||||
_LAST_SAVED_CONFIG = deepcopy(config)
|
||||
return saved_entries
|
||||
|
||||
|
||||
@@ -988,37 +1120,10 @@ def save_config_and_verify(config: Dict[str, Any], retries: int = 3, delay: floa
|
||||
AllDebrid) were written successfully. If verification fails after the
|
||||
configured number of retries, a RuntimeError is raised.
|
||||
"""
|
||||
# Detect an API key that should be verified (provider or store-backed)
|
||||
expected_key = None
|
||||
try:
|
||||
providers = config.get("provider", {}) if isinstance(config, dict) else {}
|
||||
if isinstance(providers, dict):
|
||||
entry = providers.get("alldebrid")
|
||||
if entry is not None:
|
||||
# _extract_api_key is a small internal helper; reuse the implementation here
|
||||
if isinstance(entry, dict):
|
||||
for k in ("api_key", "API_KEY", "apikey", "APIKEY"):
|
||||
v = entry.get(k)
|
||||
if isinstance(v, str) and v.strip():
|
||||
expected_key = v.strip()
|
||||
break
|
||||
elif isinstance(entry, str) and entry.strip():
|
||||
expected_key = entry.strip()
|
||||
if not expected_key:
|
||||
store_block = config.get("store", {}) if isinstance(config, dict) else {}
|
||||
debrid = store_block.get("debrid") if isinstance(store_block, dict) else None
|
||||
if isinstance(debrid, dict):
|
||||
srv = debrid.get("all-debrid")
|
||||
if isinstance(srv, dict):
|
||||
for k in ("api_key", "API_KEY", "apikey", "APIKEY"):
|
||||
v = srv.get(k)
|
||||
if isinstance(v, str) and v.strip():
|
||||
expected_key = v.strip()
|
||||
break
|
||||
elif isinstance(srv, str) and srv.strip():
|
||||
expected_key = srv.strip()
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to determine expected key for save verification: %s", exc, exc_info=True)
|
||||
# Only perform the extra verification loop when the AllDebrid key actually changed.
|
||||
expected_key = _extract_expected_alldebrid_key(config)
|
||||
baseline_key = _extract_expected_alldebrid_key(_LAST_SAVED_CONFIG)
|
||||
if expected_key == baseline_key:
|
||||
expected_key = None
|
||||
|
||||
last_exc: Exception | None = None
|
||||
|
||||
@@ -89,6 +89,8 @@ class HydrusNetwork(Store):
|
||||
Maintains its own HydrusClient.
|
||||
"""
|
||||
|
||||
STORE_TYPE = "hydrusnetwork"
|
||||
|
||||
@classmethod
|
||||
def config_schema(cls) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
@@ -1776,6 +1778,33 @@ class HydrusNetwork(Store):
|
||||
debug(f"{self._log_prefix()} delete_file failed: {exc}")
|
||||
return False
|
||||
|
||||
def build_file_url(self, file_hash: str, *, include_access_key: bool = True) -> str:
|
||||
normalized = str(file_hash or "").strip().lower()
|
||||
base_url = str(self.URL).rstrip("/")
|
||||
url = f"{base_url}/get_files/file?hash={quote(normalized)}"
|
||||
if include_access_key and str(self.API or "").strip():
|
||||
url = f"{url}&Hydrus-Client-API-Access-Key={quote(str(self.API))}"
|
||||
return url
|
||||
|
||||
def fetch_file_metadata(self, file_hash: str, **kwargs: Any) -> Optional[Dict[str, Any]]:
|
||||
try:
|
||||
client = self._client
|
||||
if client is None:
|
||||
return None
|
||||
return client.fetch_file_metadata(hashes=[str(file_hash or "").strip().lower()], **kwargs)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def get_relationships(self, file_hash: str) -> Optional[Dict[str, Any]]:
|
||||
try:
|
||||
client = self._client
|
||||
if client is None:
|
||||
return None
|
||||
payload = client.get_file_relationships(str(file_hash or "").strip().lower())
|
||||
return payload if isinstance(payload, dict) else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def get_metadata(self, file_hash: str, **kwargs: Any) -> Optional[Dict[str, Any]]:
|
||||
"""Get metadata for a file from Hydrus by hash.
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ from SYS.plugin_config import (
|
||||
get_item_schema_map,
|
||||
get_required_config_keys,
|
||||
)
|
||||
from ProviderCore.registry import get_plugin
|
||||
from ProviderCore.registry import get_plugin, get_plugin_class
|
||||
from TUI.modalscreen.matrix_room_picker import MatrixRoomPicker
|
||||
from TUI.modalscreen.selection_modal import SelectionModal
|
||||
import logging
|
||||
@@ -164,9 +164,12 @@ class ConfigModal(ModalScreen):
|
||||
self.editing_item_type = None # 'store' or 'provider'
|
||||
self.editing_item_name = None
|
||||
self._button_id_map = {}
|
||||
self._provider_button_map: Dict[str, tuple[str, str]] = {}
|
||||
self._input_id_map = {}
|
||||
self._matrix_status: Optional[Static] = None
|
||||
self._matrix_test_running = False
|
||||
self._provider_status: Optional[Static] = None
|
||||
self._provider_action_running = False
|
||||
self._editor_snapshot: Optional[Dict[str, Any]] = None
|
||||
# Inline matrix rooms controls
|
||||
self._matrix_inline_list: Optional[ListView] = None
|
||||
@@ -256,6 +259,7 @@ class ConfigModal(ModalScreen):
|
||||
return
|
||||
|
||||
self._button_id_map.clear()
|
||||
self._provider_button_map.clear()
|
||||
self._input_id_map.clear()
|
||||
|
||||
# Clear existing
|
||||
@@ -594,6 +598,33 @@ class ConfigModal(ModalScreen):
|
||||
row.mount(Button("Paste", id=f"paste-{inp_id}", classes="paste-btn"))
|
||||
idx += 1
|
||||
|
||||
if item_type == "provider" and isinstance(item_name, str):
|
||||
provider = self._instantiate_provider_for_editor(item_name, self.config_data)
|
||||
if provider is not None:
|
||||
provider_actions = provider.config_actions() or []
|
||||
if provider_actions:
|
||||
container.mount(Rule())
|
||||
container.mount(Label(f"{provider.label} helpers", classes="config-label"))
|
||||
helper_text = str(provider.config_helper_text() or "Use these helpers to validate provider settings.").strip()
|
||||
status = Static(helper_text, id="provider-status")
|
||||
container.mount(status)
|
||||
self._provider_status = status
|
||||
row = Horizontal(classes="field-row")
|
||||
container.mount(row)
|
||||
for action in provider_actions:
|
||||
action_id = str(action.get("id") or "").strip()
|
||||
if not action_id:
|
||||
continue
|
||||
button_id = f"provider-action-{item_name}-{action_id}".replace(" ", "-")
|
||||
self._provider_button_map[button_id] = (item_name, action_id)
|
||||
row.mount(
|
||||
Button(
|
||||
str(action.get("label") or action_id.replace("_", " ").title()),
|
||||
id=button_id,
|
||||
variant=str(action.get("variant") or "default"),
|
||||
)
|
||||
)
|
||||
|
||||
if (
|
||||
item_type == "provider"
|
||||
and isinstance(item_name, str)
|
||||
@@ -755,6 +786,10 @@ class ConfigModal(ModalScreen):
|
||||
self.editing_item_type = None
|
||||
self.refresh_view()
|
||||
except Exception as exc:
|
||||
try:
|
||||
log(f"Configuration save failed: {exc}")
|
||||
except Exception:
|
||||
logger.exception("Failed to write save failure to logs")
|
||||
self.notify(f"Save failed: {exc}", severity="error", timeout=10)
|
||||
elif bid == "save-durable-btn":
|
||||
# Perform a synchronous, verified save and notify status to the user.
|
||||
@@ -788,6 +823,10 @@ class ConfigModal(ModalScreen):
|
||||
self.refresh_view()
|
||||
self._editor_snapshot = None
|
||||
except Exception as exc:
|
||||
try:
|
||||
log(f"Durable configuration save failed: {exc}")
|
||||
except Exception:
|
||||
logger.exception("Failed to write durable save failure to logs")
|
||||
self.notify(f"Durable save failed: {exc}", severity="error", timeout=10)
|
||||
try:
|
||||
log(f"Durable save failed: {exc}")
|
||||
@@ -823,8 +862,15 @@ class ConfigModal(ModalScreen):
|
||||
saved = self.save_all()
|
||||
self.notify("Saving configuration...", timeout=3)
|
||||
except Exception as exc:
|
||||
try:
|
||||
log(f"Configuration save failed while deleting config entry: {exc}")
|
||||
except Exception:
|
||||
logger.exception("Failed to write config delete save failure to logs")
|
||||
self.notify(f"Save failed: {exc}", severity="error", timeout=10)
|
||||
self.refresh_view()
|
||||
elif bid in self._provider_button_map:
|
||||
provider_name, action_id = self._provider_button_map[bid]
|
||||
self._request_provider_action(provider_name, action_id)
|
||||
elif bid == "add-store-btn":
|
||||
options = get_configurable_store_types()
|
||||
self.app.push_screen(SelectionModal("Select Store Type", options), callback=self.on_store_type_selected)
|
||||
@@ -882,6 +928,10 @@ class ConfigModal(ModalScreen):
|
||||
try:
|
||||
entries = save_config(self.config_data)
|
||||
except Exception as exc:
|
||||
try:
|
||||
log(f"Saving Matrix default rooms failed: {exc}")
|
||||
except Exception:
|
||||
logger.exception("Failed to write Matrix room save failure to logs")
|
||||
if self._matrix_status:
|
||||
self._matrix_status.update(f"Saving default rooms failed: {exc}")
|
||||
return
|
||||
@@ -921,6 +971,69 @@ class ConfigModal(ModalScreen):
|
||||
else:
|
||||
self.notify("Clipboard not supported in this terminal", severity="warning")
|
||||
|
||||
def _instantiate_provider_for_editor(self, provider_name: str, config_data: Optional[Dict[str, Any]] = None) -> Optional[Any]:
|
||||
try:
|
||||
provider_class = get_plugin_class(provider_name)
|
||||
except Exception:
|
||||
provider_class = None
|
||||
if provider_class is None:
|
||||
return None
|
||||
try:
|
||||
return provider_class(config_data or self.config_data)
|
||||
except Exception:
|
||||
logger.exception("Failed to instantiate provider '%s' for config helper", provider_name)
|
||||
return None
|
||||
|
||||
def _request_provider_action(self, provider_name: str, action_id: str) -> None:
|
||||
if self._provider_action_running:
|
||||
return
|
||||
self._synchronize_inputs_to_config()
|
||||
self._provider_action_running = True
|
||||
if self._provider_status is not None:
|
||||
self._provider_status.update(f"Running {action_id.replace('_', ' ')}…")
|
||||
self._provider_action_background(provider_name, action_id, deepcopy(self.config_data))
|
||||
|
||||
@work(thread=True)
|
||||
def _provider_action_background(self, provider_name: str, action_id: str, config_snapshot: Dict[str, Any]) -> None:
|
||||
try:
|
||||
provider = self._instantiate_provider_for_editor(provider_name, config_snapshot)
|
||||
if provider is None:
|
||||
raise RuntimeError(f"Provider '{provider_name}' is unavailable")
|
||||
result = provider.run_config_action(action_id)
|
||||
if not isinstance(result, dict):
|
||||
result = {"ok": False, "message": f"Provider '{provider_name}' returned an invalid config action result."}
|
||||
except Exception as exc:
|
||||
result = {"ok": False, "message": str(exc) or f"Provider action '{action_id}' failed."}
|
||||
|
||||
try:
|
||||
self.app.call_from_thread(self._provider_action_complete, provider_name, action_id, result)
|
||||
except Exception:
|
||||
self._provider_action_complete(provider_name, action_id, result)
|
||||
|
||||
def _provider_action_complete(self, provider_name: str, action_id: str, result: Dict[str, Any]) -> None:
|
||||
self._provider_action_running = False
|
||||
ok = bool(result.get("ok"))
|
||||
message = str(result.get("message") or f"Provider action '{action_id}' finished.")
|
||||
updates = result.get("config_updates")
|
||||
|
||||
if ok and isinstance(updates, dict):
|
||||
provider_block = self.config_data.setdefault("provider", {}).setdefault(provider_name, {})
|
||||
if isinstance(provider_block, dict):
|
||||
provider_block.update(updates)
|
||||
message = f"{message}"
|
||||
try:
|
||||
self.refresh_view()
|
||||
except Exception:
|
||||
logger.exception("Failed to refresh config view after provider action")
|
||||
|
||||
if self._provider_status is not None:
|
||||
self._provider_status.update(message)
|
||||
|
||||
try:
|
||||
self.notify(message, severity="error" if not ok else "information", timeout=8)
|
||||
except Exception:
|
||||
logger.exception("Failed to notify provider action result for %s/%s", provider_name, action_id)
|
||||
|
||||
# Backup/restore helpers removed: forensics/audit mode disabled and restore UI removed.
|
||||
|
||||
def on_store_type_selected(self, stype: str) -> None:
|
||||
@@ -1130,6 +1243,10 @@ class ConfigModal(ModalScreen):
|
||||
try:
|
||||
entries = save_config(self.config_data)
|
||||
except Exception as exc:
|
||||
try:
|
||||
log(f"Saving configuration before Matrix test failed: {exc}")
|
||||
except Exception:
|
||||
logger.exception("Failed to write Matrix test pre-save failure to logs")
|
||||
if self._matrix_status:
|
||||
self._matrix_status.update(f"Saving configuration failed: {exc}")
|
||||
self._matrix_test_running = False
|
||||
@@ -1290,6 +1407,10 @@ class ConfigModal(ModalScreen):
|
||||
try:
|
||||
entries = save_config(self.config_data)
|
||||
except Exception as exc:
|
||||
try:
|
||||
log(f"Saving configuration before Matrix room load failed: {exc}")
|
||||
except Exception:
|
||||
logger.exception("Failed to write Matrix load pre-save failure to logs")
|
||||
if self._matrix_status:
|
||||
self._matrix_status.update(f"Saving configuration failed: {exc}")
|
||||
self._matrix_test_running = False
|
||||
|
||||
+55
-69
@@ -1410,7 +1410,7 @@ def fetch_hydrus_metadata(
|
||||
Eliminates repeated boilerplate: client initialization, error handling, metadata extraction.
|
||||
|
||||
Args:
|
||||
config: Configuration object (passed to hydrus_wrapper.get_client)
|
||||
config: Configuration object used to resolve the Hydrus provider/store
|
||||
hash_hex: File hash to fetch metadata for
|
||||
store_name: Optional Hydrus store name. When provided, do not fall back to a global/default Hydrus client.
|
||||
hydrus_client: Optional explicit Hydrus client. When provided, takes precedence.
|
||||
@@ -1422,38 +1422,53 @@ def fetch_hydrus_metadata(
|
||||
- metadata_dict: Dict from Hydrus (first item in metadata list) or None if unavailable
|
||||
- error_code: 0 on success, 1 on any error (suitable for returning from cmdlet execute())
|
||||
"""
|
||||
from API import HydrusNetwork
|
||||
|
||||
hydrus_wrapper = HydrusNetwork
|
||||
|
||||
client = hydrus_client
|
||||
hydrus_provider = None
|
||||
try:
|
||||
from ProviderCore.registry import get_plugin
|
||||
|
||||
hydrus_provider = get_plugin("hydrusnetwork", config)
|
||||
except Exception:
|
||||
hydrus_provider = None
|
||||
|
||||
if client is None:
|
||||
if store_name:
|
||||
# Store specified: do not fall back to a global/default Hydrus client.
|
||||
if hydrus_provider is not None:
|
||||
try:
|
||||
from Store import Store
|
||||
|
||||
store = Store(config)
|
||||
backend = store[str(store_name)]
|
||||
candidate = getattr(backend, "_client", None)
|
||||
if candidate is not None and hasattr(candidate, "fetch_file_metadata"):
|
||||
client = candidate
|
||||
client = hydrus_provider.get_client(
|
||||
store_name=store_name if store_name else None,
|
||||
allow_default=not bool(store_name),
|
||||
)
|
||||
except Exception as exc:
|
||||
log(f"Hydrus client unavailable for store '{store_name}': {exc}")
|
||||
if store_name:
|
||||
log(f"Hydrus client unavailable for store '{store_name}': {exc}")
|
||||
else:
|
||||
log(f"Hydrus client unavailable: {exc}")
|
||||
client = None
|
||||
if client is None:
|
||||
log(f"Hydrus client unavailable for store '{store_name}'")
|
||||
return None, 1
|
||||
else:
|
||||
try:
|
||||
client = hydrus_wrapper.get_client(config)
|
||||
except Exception as exc:
|
||||
log(f"Hydrus client unavailable: {exc}")
|
||||
return None, 1
|
||||
if client is None and store_name:
|
||||
log(f"Hydrus client unavailable for store '{store_name}'")
|
||||
return None, 1
|
||||
if client is None and hydrus_provider is None:
|
||||
log("Hydrus provider unavailable")
|
||||
return None, 1
|
||||
|
||||
if client is None:
|
||||
log("Hydrus client unavailable")
|
||||
return None, 1
|
||||
if hydrus_provider is not None:
|
||||
try:
|
||||
metadata = hydrus_provider.fetch_metadata(
|
||||
hash_hex,
|
||||
store_name=store_name if store_name else None,
|
||||
**kwargs,
|
||||
)
|
||||
except Exception as exc:
|
||||
log(f"Hydrus metadata fetch failed: {exc}")
|
||||
return None, 1
|
||||
if isinstance(metadata, dict):
|
||||
return metadata, 0
|
||||
if client is None:
|
||||
if store_name:
|
||||
log(f"Hydrus client unavailable for store '{store_name}'")
|
||||
else:
|
||||
log("Hydrus metadata unavailable")
|
||||
return None, 1
|
||||
|
||||
try:
|
||||
payload = client.fetch_file_metadata(hashes=[hash_hex], **kwargs)
|
||||
@@ -3725,10 +3740,13 @@ def check_url_exists_in_storage(
|
||||
match_rows: List[Dict[str, Any]] = []
|
||||
max_rows = 200
|
||||
|
||||
hydrus_provider = None
|
||||
try:
|
||||
from Store.HydrusNetwork import HydrusNetwork
|
||||
from ProviderCore.registry import get_plugin
|
||||
|
||||
hydrus_provider = get_plugin("hydrusnetwork", config)
|
||||
except Exception:
|
||||
HydrusNetwork = None # type: ignore
|
||||
hydrus_provider = None
|
||||
|
||||
for backend_name in backend_names:
|
||||
if _timed_out("backend scan"):
|
||||
@@ -3739,8 +3757,14 @@ def check_url_exists_in_storage(
|
||||
backend = storage[backend_name]
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if HydrusNetwork is not None and isinstance(backend, HydrusNetwork):
|
||||
|
||||
is_hydrus_backend = False
|
||||
try:
|
||||
is_hydrus_backend = bool(hydrus_provider and hydrus_provider.is_backend(backend, str(backend_name)))
|
||||
except Exception:
|
||||
is_hydrus_backend = False
|
||||
|
||||
if is_hydrus_backend:
|
||||
if not hydrus_available:
|
||||
debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup")
|
||||
|
||||
@@ -3776,44 +3800,6 @@ def check_url_exists_in_storage(
|
||||
found = True
|
||||
break
|
||||
|
||||
client = getattr(backend, "_client", None)
|
||||
if found:
|
||||
pass
|
||||
elif client is None:
|
||||
continue
|
||||
for needle in (needles or [])[:6]:
|
||||
if found:
|
||||
break
|
||||
if not _httpish(needle):
|
||||
continue
|
||||
try:
|
||||
from API.HydrusNetwork import HydrusRequestSpec
|
||||
|
||||
spec = HydrusRequestSpec(
|
||||
method="GET",
|
||||
endpoint="/add_urls/get_url_files",
|
||||
query={"url": needle},
|
||||
)
|
||||
if hasattr(client, "_perform_request"):
|
||||
response = client._perform_request(spec)
|
||||
raw_hashes = None
|
||||
if isinstance(response, dict):
|
||||
raw_hashes = response.get("hashes") or response.get("file_hashes")
|
||||
raw_ids = response.get("file_ids")
|
||||
hash_list = raw_hashes if isinstance(raw_hashes, list) else []
|
||||
has_ids = isinstance(raw_ids, list) and len(raw_ids) > 0
|
||||
has_hashes = len(hash_list) > 0
|
||||
if has_hashes:
|
||||
try:
|
||||
found_hash = str(hash_list[0]).strip()
|
||||
except Exception:
|
||||
found_hash = None
|
||||
if has_ids or has_hashes:
|
||||
found = True
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not found:
|
||||
continue
|
||||
|
||||
|
||||
@@ -9,9 +9,9 @@ import sys
|
||||
|
||||
from SYS.logger import log
|
||||
from SYS.item_accessors import get_sha256_hex, get_store_name
|
||||
from ProviderCore.registry import get_plugin
|
||||
|
||||
from SYS import pipeline as ctx
|
||||
from API import HydrusNetwork as hydrus_wrapper
|
||||
from . import _shared as sh
|
||||
|
||||
Cmdlet = sh.Cmdlet
|
||||
@@ -617,16 +617,20 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
# - If no store is specified, use the default Hydrus client.
|
||||
# NOTE: When a store is specified, we do not fall back to a global/default Hydrus client.
|
||||
hydrus_client = None
|
||||
hydrus_provider = get_plugin("hydrusnetwork", config)
|
||||
if store_name and (not is_folder_store) and backend is not None:
|
||||
try:
|
||||
candidate = getattr(backend, "_client", None)
|
||||
if candidate is not None and hasattr(candidate, "set_relationship"):
|
||||
hydrus_client = candidate
|
||||
if hydrus_provider is not None:
|
||||
hydrus_client = hydrus_provider.get_client(
|
||||
store_name=str(store_name),
|
||||
allow_default=False,
|
||||
)
|
||||
except Exception:
|
||||
hydrus_client = None
|
||||
elif not store_name:
|
||||
try:
|
||||
hydrus_client = hydrus_wrapper.get_client(config)
|
||||
if hydrus_provider is not None:
|
||||
hydrus_client = hydrus_provider.get_client()
|
||||
except Exception:
|
||||
hydrus_client = None
|
||||
|
||||
@@ -1049,8 +1053,9 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
return 1
|
||||
|
||||
# Build Hydrus client
|
||||
hydrus_provider = get_plugin("hydrusnetwork", config)
|
||||
try:
|
||||
hydrus_client = hydrus_wrapper.get_client(config)
|
||||
hydrus_client = hydrus_provider.get_client() if hydrus_provider is not None else None
|
||||
except Exception as exc:
|
||||
log(f"Hydrus client unavailable: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
+7
-66
@@ -13,6 +13,7 @@ from typing import Any, Dict, List, Sequence, Set
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from SYS.logger import log
|
||||
from ProviderCore.registry import get_plugin
|
||||
from SYS.item_accessors import get_http_url, get_sha256_hex, get_store_name
|
||||
from SYS.utils import extract_hydrus_hash_from_url
|
||||
|
||||
@@ -71,10 +72,8 @@ def _maybe_download_hydrus_item(
|
||||
|
||||
This is intentionally side-effect free except for writing the local temp file.
|
||||
"""
|
||||
try:
|
||||
from SYS.config import get_hydrus_access_key, get_hydrus_url
|
||||
from API.HydrusNetwork import HydrusNetwork as HydrusClient, download_hydrus_file
|
||||
except Exception:
|
||||
hydrus_provider = get_plugin("hydrusnetwork", config)
|
||||
if hydrus_provider is None:
|
||||
return None
|
||||
|
||||
store_name = _extract_store_name(item)
|
||||
@@ -102,68 +101,10 @@ def _maybe_download_hydrus_item(
|
||||
is_hydrus_url = False
|
||||
if not (is_hydrus_url or store_hint):
|
||||
return None
|
||||
|
||||
# Prefer store name as instance key; fall back to "home".
|
||||
access_key = None
|
||||
hydrus_url = None
|
||||
for inst in [s for s in [store_lower, "home"] if s]:
|
||||
try:
|
||||
access_key = (get_hydrus_access_key(config, inst) or "").strip() or None
|
||||
hydrus_url = (get_hydrus_url(config, inst) or "").strip() or None
|
||||
if access_key and hydrus_url:
|
||||
break
|
||||
except Exception:
|
||||
access_key = None
|
||||
hydrus_url = None
|
||||
|
||||
if not access_key or not hydrus_url:
|
||||
return None
|
||||
|
||||
client = HydrusClient(url=hydrus_url, access_key=access_key, timeout=60.0)
|
||||
file_url = url if (url and is_hydrus_url) else client.file_url(file_hash)
|
||||
|
||||
# Best-effort extension from Hydrus metadata.
|
||||
suffix = ".hydrus"
|
||||
try:
|
||||
meta_response = client.fetch_file_metadata(
|
||||
hashes=[file_hash],
|
||||
include_mime=True
|
||||
)
|
||||
entries = meta_response.get("metadata"
|
||||
) if isinstance(meta_response,
|
||||
dict) else None
|
||||
if isinstance(entries, list) and entries:
|
||||
entry = entries[0]
|
||||
if isinstance(entry, dict):
|
||||
ext = entry.get("ext")
|
||||
if isinstance(ext, str) and ext.strip():
|
||||
cleaned = ext.strip()
|
||||
if not cleaned.startswith("."):
|
||||
cleaned = "." + cleaned.lstrip(".")
|
||||
if len(cleaned) <= 12:
|
||||
suffix = cleaned
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
dest = output_dir / f"{file_hash}{suffix}"
|
||||
if dest.exists():
|
||||
dest = output_dir / f"{file_hash}_{uuid.uuid4().hex[:10]}{suffix}"
|
||||
|
||||
headers = {
|
||||
"Hydrus-Client-API-Access-Key": access_key
|
||||
}
|
||||
download_hydrus_file(file_url, headers, dest, timeout=60.0)
|
||||
try:
|
||||
if dest.exists() and dest.is_file():
|
||||
return dest
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
preferred_store = store_name or None
|
||||
if url and is_hydrus_url:
|
||||
return hydrus_provider.download_url(url, output_dir)
|
||||
return hydrus_provider.download_hash_to_temp(file_hash, store_name=preferred_store, temp_root=output_dir)
|
||||
|
||||
|
||||
def _resolve_existing_or_fetch_path(item: Any,
|
||||
|
||||
+25
-96
@@ -7,9 +7,9 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
from SYS.logger import debug, log
|
||||
from ProviderCore.registry import get_plugin
|
||||
from Store import Store
|
||||
from . import _shared as sh
|
||||
from API import HydrusNetwork as hydrus_wrapper
|
||||
from SYS import pipeline as ctx
|
||||
from SYS.result_table_helpers import add_row_columns
|
||||
from SYS.result_table import Table, _format_size
|
||||
@@ -129,6 +129,7 @@ class Delete_File(sh.Cmdlet):
|
||||
store = sh.get_field(item, "store")
|
||||
|
||||
store_lower = str(store).lower() if store else ""
|
||||
hydrus_provider = get_plugin("hydrusnetwork", config)
|
||||
|
||||
backend = None
|
||||
try:
|
||||
@@ -144,18 +145,17 @@ class Delete_File(sh.Cmdlet):
|
||||
# so checking only the store name is unreliable.
|
||||
is_hydrus_store = False
|
||||
try:
|
||||
if backend is not None:
|
||||
from Store.HydrusNetwork import HydrusNetwork as HydrusStore
|
||||
|
||||
is_hydrus_store = isinstance(backend, HydrusStore)
|
||||
if hydrus_provider is not None and backend is not None:
|
||||
is_hydrus_store = bool(hydrus_provider.is_backend(backend, str(store or "")))
|
||||
except Exception:
|
||||
is_hydrus_store = False
|
||||
|
||||
# Backwards-compatible fallback heuristic (older items might only carry a name).
|
||||
if ((not is_hydrus_store) and bool(store_lower)
|
||||
and ("hydrus" in store_lower or store_lower in {"home",
|
||||
"work"})):
|
||||
is_hydrus_store = True
|
||||
if (not is_hydrus_store) and hydrus_provider is not None and bool(store_lower):
|
||||
try:
|
||||
is_hydrus_store = bool(hydrus_provider.is_store_name(store_lower))
|
||||
except Exception:
|
||||
is_hydrus_store = False
|
||||
store_label = str(store) if store else "default"
|
||||
hydrus_prefix = f"[hydrusnetwork:{store_label}]"
|
||||
|
||||
@@ -318,18 +318,20 @@ class Delete_File(sh.Cmdlet):
|
||||
should_try_hydrus = False
|
||||
|
||||
if should_try_hydrus and hash_hex:
|
||||
# Prefer deleting via the resolved store backend when it is a HydrusNetwork store.
|
||||
# This ensures store-specific post-delete hooks run (e.g., clearing Hydrus deletion records).
|
||||
did_backend_delete = False
|
||||
did_hydrus_delete = False
|
||||
try:
|
||||
if backend is not None:
|
||||
deleter = getattr(backend, "delete_file", None)
|
||||
if callable(deleter):
|
||||
did_backend_delete = bool(deleter(hash_hex, reason=reason))
|
||||
if hydrus_provider is not None:
|
||||
did_hydrus_delete = bool(
|
||||
hydrus_provider.delete_hash(
|
||||
hash_hex,
|
||||
store_name=str(store) if store else None,
|
||||
reason=reason or None,
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
did_backend_delete = False
|
||||
did_hydrus_delete = False
|
||||
|
||||
if did_backend_delete:
|
||||
if did_hydrus_delete:
|
||||
hydrus_deleted = True
|
||||
title_str = str(title_val).strip() if title_val else ""
|
||||
if title_str:
|
||||
@@ -340,85 +342,12 @@ class Delete_File(sh.Cmdlet):
|
||||
else:
|
||||
debug(f"{hydrus_prefix} Deleted hash:{hash_hex}", file=sys.stderr)
|
||||
else:
|
||||
# Fallback to direct client calls.
|
||||
client = None
|
||||
if store:
|
||||
# Store specified: do not fall back to a global/default Hydrus client.
|
||||
try:
|
||||
registry = Store(config)
|
||||
backend = registry[str(store)]
|
||||
candidate = getattr(backend, "_client", None)
|
||||
if candidate is not None and hasattr(candidate, "_post"):
|
||||
client = candidate
|
||||
except Exception as exc:
|
||||
if not local_deleted:
|
||||
log(
|
||||
f"Hydrus client unavailable for store '{store}': {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return False
|
||||
if client is None:
|
||||
if not local_deleted:
|
||||
log(
|
||||
f"Hydrus client unavailable for store '{store}'",
|
||||
file=sys.stderr
|
||||
)
|
||||
return False
|
||||
else:
|
||||
# No store context; use default Hydrus client.
|
||||
try:
|
||||
client = hydrus_wrapper.get_client(config)
|
||||
except Exception as exc:
|
||||
if not local_deleted:
|
||||
log(f"Hydrus client unavailable: {exc}", file=sys.stderr)
|
||||
return False
|
||||
if client is None:
|
||||
if not local_deleted:
|
||||
log("Hydrus client unavailable", file=sys.stderr)
|
||||
return False
|
||||
|
||||
payload: Dict[str,
|
||||
Any] = {
|
||||
"hashes": [hash_hex]
|
||||
}
|
||||
if reason:
|
||||
payload["reason"] = reason
|
||||
try:
|
||||
client._post(
|
||||
"/add_files/delete_files",
|
||||
data=payload
|
||||
) # type: ignore[attr-defined]
|
||||
# Best-effort clear deletion record if supported by this client.
|
||||
try:
|
||||
clearer = getattr(client, "clear_file_deletion_record", None)
|
||||
if callable(clearer):
|
||||
clearer([hash_hex])
|
||||
else:
|
||||
client._post(
|
||||
"/add_files/clear_file_deletion_record",
|
||||
data={
|
||||
"hashes": [hash_hex]
|
||||
}
|
||||
) # type: ignore[attr-defined]
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
hydrus_deleted = True
|
||||
title_str = str(title_val).strip() if title_val else ""
|
||||
if title_str:
|
||||
debug(
|
||||
f"{hydrus_prefix} Deleted title:{title_str} hash:{hash_hex}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
if not local_deleted:
|
||||
if store:
|
||||
log(f"Hydrus store unavailable for '{store}'", file=sys.stderr)
|
||||
else:
|
||||
debug(
|
||||
f"{hydrus_prefix} Deleted hash:{hash_hex}",
|
||||
file=sys.stderr
|
||||
)
|
||||
except Exception:
|
||||
# If it's not in Hydrus (e.g. 404 or similar), that's fine
|
||||
if not local_deleted:
|
||||
return []
|
||||
log("Hydrus delete failed", file=sys.stderr)
|
||||
return []
|
||||
|
||||
if hydrus_deleted and hash_hex:
|
||||
size_hint = None
|
||||
|
||||
@@ -980,10 +980,14 @@ class Download_File(Cmdlet):
|
||||
) -> Optional[str]:
|
||||
if storage is None or not canonical_url:
|
||||
return None
|
||||
hydrus_provider = None
|
||||
try:
|
||||
from Store.HydrusNetwork import HydrusNetwork
|
||||
registry_helpers = cls._load_provider_registry()
|
||||
get_plugin = registry_helpers.get("get_plugin")
|
||||
if callable(get_plugin):
|
||||
hydrus_provider = get_plugin("hydrusnetwork", {})
|
||||
except Exception:
|
||||
HydrusNetwork = None # type: ignore
|
||||
hydrus_provider = None
|
||||
|
||||
try:
|
||||
backend_names = list(storage.list_searchable_backends() or [])
|
||||
@@ -1001,13 +1005,13 @@ class Download_File(Cmdlet):
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if HydrusNetwork is not None and isinstance(backend, HydrusNetwork) and not hydrus_available:
|
||||
if hydrus_provider is not None and hydrus_provider.is_backend(backend, str(backend_name)) and not hydrus_available:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
if HydrusNetwork is not None and isinstance(backend, HydrusNetwork):
|
||||
if hydrus_provider is not None and hydrus_provider.is_backend(backend, str(backend_name)):
|
||||
hashes = backend.find_hashes_by_url(canonical_url) or []
|
||||
for existing_hash in hashes:
|
||||
normalized = sh.normalize_hash(existing_hash)
|
||||
|
||||
+16
-37
@@ -5,12 +5,12 @@ import sys
|
||||
|
||||
from SYS.detail_view_helpers import create_detail_view, prepare_detail_metadata
|
||||
from SYS.logger import log
|
||||
from ProviderCore.registry import get_plugin
|
||||
from SYS.result_table_helpers import add_row_columns
|
||||
from SYS.selection_builder import build_hash_store_selection
|
||||
from SYS.result_publication import publish_result_table
|
||||
|
||||
from SYS import pipeline as ctx
|
||||
from API import HydrusNetwork as hydrus_wrapper
|
||||
from . import _shared as sh
|
||||
|
||||
Cmdlet = sh.Cmdlet
|
||||
@@ -22,7 +22,6 @@ get_hash_for_operation = sh.get_hash_for_operation
|
||||
fetch_hydrus_metadata = sh.fetch_hydrus_metadata
|
||||
should_show_help = sh.should_show_help
|
||||
get_field = sh.get_field
|
||||
from Store import Store
|
||||
|
||||
CMDLET = Cmdlet(
|
||||
name="get-relationship",
|
||||
@@ -109,6 +108,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
return 1
|
||||
|
||||
# Fetch Hydrus relationships if we have a hash.
|
||||
hydrus_provider = get_plugin("hydrusnetwork", config)
|
||||
|
||||
hash_hex = (
|
||||
normalize_hash(override_hash)
|
||||
@@ -118,29 +118,18 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
|
||||
if hash_hex:
|
||||
try:
|
||||
client = None
|
||||
store_label = "hydrus"
|
||||
backend_obj = None
|
||||
if store_name:
|
||||
# Store specified: do not fall back to a global/default Hydrus client.
|
||||
store_label = str(store_name)
|
||||
try:
|
||||
store = Store(config)
|
||||
backend_obj = store[str(store_name)]
|
||||
candidate = getattr(backend_obj, "_client", None)
|
||||
if candidate is not None and hasattr(candidate,
|
||||
"get_file_relationships"):
|
||||
client = candidate
|
||||
except Exception:
|
||||
client = None
|
||||
if client is None:
|
||||
if hydrus_provider is None:
|
||||
log(
|
||||
f"Hydrus client unavailable for store '{store_name}'",
|
||||
file=sys.stderr
|
||||
)
|
||||
return 1
|
||||
relationships = hydrus_provider.get_relationships(hash_hex, store_name=store_name)
|
||||
else:
|
||||
client = hydrus_wrapper.get_client(config)
|
||||
relationships = hydrus_provider.get_relationships(hash_hex) if hydrus_provider is not None else None
|
||||
|
||||
def _resolve_related_title(rel_hash: str) -> str:
|
||||
"""Best-effort resolve a Hydrus hash to a human title.
|
||||
@@ -154,22 +143,15 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
if not h:
|
||||
return str(rel_hash)
|
||||
|
||||
# Prefer backend tag extraction when available.
|
||||
if backend_obj is not None and hasattr(backend_obj, "get_tag"):
|
||||
# Prefer provider-backed title resolution when available.
|
||||
if hydrus_provider is not None:
|
||||
try:
|
||||
tag_result = backend_obj.get_tag(h)
|
||||
tags = (
|
||||
tag_result[0]
|
||||
if isinstance(tag_result,
|
||||
tuple) and tag_result else tag_result
|
||||
resolved_title = hydrus_provider.get_title(
|
||||
h,
|
||||
store_name=store_label if store_name else None,
|
||||
)
|
||||
if isinstance(tags, list):
|
||||
for t in tags:
|
||||
if isinstance(t,
|
||||
str) and t.lower().startswith("title:"):
|
||||
val = t.split(":", 1)[1].strip()
|
||||
if val:
|
||||
return val
|
||||
if isinstance(resolved_title, str) and resolved_title.strip():
|
||||
return resolved_title.strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -179,7 +161,6 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
config,
|
||||
h,
|
||||
store_name=store_label if store_name else None,
|
||||
hydrus_client=client,
|
||||
include_service_keys_to_tags=True,
|
||||
include_file_url=False,
|
||||
include_duration=False,
|
||||
@@ -224,14 +205,12 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
|
||||
return h[:16] + "..."
|
||||
|
||||
if client:
|
||||
rel = client.get_file_relationships(hash_hex)
|
||||
if rel:
|
||||
file_rels = rel.get("file_relationships",
|
||||
if relationships:
|
||||
file_rels = relationships.get("file_relationships",
|
||||
{})
|
||||
this_file_rels = file_rels.get(hash_hex)
|
||||
this_file_rels = file_rels.get(hash_hex)
|
||||
|
||||
if this_file_rels:
|
||||
if this_file_rels:
|
||||
# Map Hydrus relationship IDs to names.
|
||||
# For /manage_file_relationships/get_file_relationships, the Hydrus docs define:
|
||||
# 0=potential duplicates, 1=false positives, 3=alternates, 8=duplicates
|
||||
|
||||
+147
-1
@@ -1,3 +1,6 @@
|
||||
import datetime
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any, Optional, Sequence
|
||||
|
||||
from SYS.cmdlet_spec import Cmdlet, CmdletArg
|
||||
@@ -7,17 +10,20 @@ from SYS.config import (
|
||||
save_config_and_verify,
|
||||
set_nested_config_value,
|
||||
)
|
||||
from SYS.database import LOG_DB_PATH, db
|
||||
from SYS.logger import log
|
||||
from SYS import pipeline as ctx
|
||||
from SYS.result_table import Table
|
||||
from cmdnat._parsing import (
|
||||
extract_piped_value as _extract_piped_value,
|
||||
extract_value_arg as _extract_value_arg,
|
||||
has_flag as _has_flag,
|
||||
)
|
||||
|
||||
CMDLET = Cmdlet(
|
||||
name=".config",
|
||||
summary="Manage configuration settings",
|
||||
usage=".config [key] [value]",
|
||||
usage=".config [key] [value] | .config -log [count]",
|
||||
arg=[
|
||||
CmdletArg(
|
||||
name="key",
|
||||
@@ -33,6 +39,140 @@ CMDLET = Cmdlet(
|
||||
)
|
||||
|
||||
|
||||
def _extract_log_limit(args: Sequence[str], default: int = 30) -> int:
|
||||
try:
|
||||
tokens = [str(arg).strip() for arg in (args or []) if str(arg).strip()]
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
for idx, token in enumerate(tokens):
|
||||
lowered = token.lower()
|
||||
if lowered in {"-log", "--log"}:
|
||||
if idx + 1 < len(tokens):
|
||||
candidate = tokens[idx + 1]
|
||||
if candidate and not candidate.startswith("-"):
|
||||
try:
|
||||
return max(1, min(200, int(candidate)))
|
||||
except Exception:
|
||||
return default
|
||||
return default
|
||||
if lowered.startswith("-log=") or lowered.startswith("--log="):
|
||||
_, value = lowered.split("=", 1)
|
||||
try:
|
||||
return max(1, min(200, int(value)))
|
||||
except Exception:
|
||||
return default
|
||||
return default
|
||||
|
||||
|
||||
def _fallback_log_path() -> Path:
|
||||
return Path(db.db_path).with_name("logs") / "log_fallback.txt"
|
||||
|
||||
|
||||
def _load_recent_config_logs(limit: int = 30) -> List[Dict[str, str]]:
|
||||
rows: List[Dict[str, str]] = []
|
||||
sql = """
|
||||
SELECT timestamp, level, module, message
|
||||
FROM logs
|
||||
WHERE lower(module) LIKE ?
|
||||
OR lower(message) LIKE ?
|
||||
OR lower(message) LIKE ?
|
||||
OR lower(message) LIKE ?
|
||||
ORDER BY id DESC
|
||||
LIMIT ?
|
||||
"""
|
||||
params = (
|
||||
"%config%",
|
||||
"%config%",
|
||||
"%save failed%",
|
||||
"%saving configuration failed%",
|
||||
int(limit),
|
||||
)
|
||||
|
||||
try:
|
||||
with sqlite3.connect(str(LOG_DB_PATH), timeout=5.0) as conn:
|
||||
conn.row_factory = sqlite3.Row
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, params)
|
||||
fetched = cur.fetchall()
|
||||
cur.close()
|
||||
for row in fetched:
|
||||
rows.append(
|
||||
{
|
||||
"timestamp": str(row["timestamp"] or ""),
|
||||
"level": str(row["level"] or ""),
|
||||
"module": str(row["module"] or ""),
|
||||
"message": str(row["message"] or ""),
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
rows = []
|
||||
|
||||
if rows:
|
||||
return rows
|
||||
|
||||
fallback = _fallback_log_path()
|
||||
try:
|
||||
if not fallback.exists():
|
||||
return []
|
||||
lines = fallback.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
matches = [
|
||||
line for line in lines
|
||||
if any(term in line.lower() for term in ("config", "save failed", "saving configuration failed"))
|
||||
]
|
||||
for line in reversed(matches[-limit:]):
|
||||
rows.append(
|
||||
{
|
||||
"timestamp": "",
|
||||
"level": "FALLBACK",
|
||||
"module": "fallback",
|
||||
"message": line,
|
||||
}
|
||||
)
|
||||
except Exception:
|
||||
return []
|
||||
return rows
|
||||
|
||||
|
||||
def _format_log_timestamp_local(raw_value: str) -> str:
|
||||
text = str(raw_value or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
for pattern in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S.%f"):
|
||||
try:
|
||||
parsed = datetime.datetime.strptime(text, pattern).replace(tzinfo=datetime.timezone.utc)
|
||||
return parsed.astimezone().strftime("%Y-%m-%d %H:%M:%S")
|
||||
except Exception:
|
||||
continue
|
||||
return text
|
||||
|
||||
|
||||
def _show_config_logs(args: Sequence[str]) -> int:
|
||||
limit = _extract_log_limit(args)
|
||||
rows = _load_recent_config_logs(limit=limit)
|
||||
if not rows:
|
||||
print(
|
||||
f"No recent config/save logs found in {LOG_DB_PATH.name} or {_fallback_log_path().name}."
|
||||
)
|
||||
return 0
|
||||
|
||||
table = Table("Configuration Logs")
|
||||
table.set_table("config.logs")
|
||||
table.set_source_command(".config", ["-log", str(limit)])
|
||||
|
||||
for row_data in rows:
|
||||
row = table.add_row()
|
||||
row.add_column("Time (local)", _format_log_timestamp_local(row_data.get("timestamp", "")))
|
||||
row.add_column("Level", row_data.get("level", ""))
|
||||
row.add_column("Module", row_data.get("module", ""))
|
||||
row.add_column("Message", row_data.get("message", ""))
|
||||
|
||||
ctx.set_last_result_table_overlay(table, rows)
|
||||
ctx.set_current_stage_table(table)
|
||||
print(f"Showing {len(rows)} recent configuration log entries.")
|
||||
return 0
|
||||
|
||||
|
||||
def flatten_config(config: Dict[str, Any], parent_key: str = "", sep: str = ".") -> List[Dict[str, Any]]:
|
||||
items: List[Dict[str, Any]] = []
|
||||
for k, v in config.items():
|
||||
@@ -108,6 +248,9 @@ def _strip_value_quotes(value: str) -> str:
|
||||
def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int:
|
||||
import sys
|
||||
|
||||
if _has_flag(args, "-log") or _has_flag(args, "--log"):
|
||||
return _show_config_logs(args)
|
||||
|
||||
# Load configuration from the database
|
||||
current_config = load_config()
|
||||
|
||||
@@ -135,6 +278,7 @@ def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int:
|
||||
try:
|
||||
save_config_and_verify(current_config)
|
||||
except Exception as exc:
|
||||
log(f"Configuration save verification failed for '{selection_key}': {exc}")
|
||||
print(f"Error saving configuration (verification failed): {exc}")
|
||||
return 1
|
||||
else:
|
||||
@@ -142,6 +286,7 @@ def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int:
|
||||
print(f"Updated '{selection_key}' to '{new_value}'")
|
||||
return 0
|
||||
except Exception as exc:
|
||||
log(f"Error updating config '{selection_key}': {exc}")
|
||||
print(f"Error updating config: {exc}")
|
||||
return 1
|
||||
|
||||
@@ -201,6 +346,7 @@ def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int:
|
||||
print(f"Updated '{key}' to '{value}'")
|
||||
return 0
|
||||
except Exception as exc:
|
||||
log(f"Error updating config '{key}': {exc}")
|
||||
print(f"Error updating config: {exc}")
|
||||
return 1
|
||||
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
# FTP Plugin Walkthrough
|
||||
|
||||
This walkthrough adds a real bundled `ftp` plugin so users can:
|
||||
|
||||
- run `search-file -plugin ftp ...`
|
||||
- browse remote folders as result tables
|
||||
- select file rows to `download-file`
|
||||
- pipe selected file rows into `add-file`
|
||||
- upload local files with `add-file -plugin ftp`
|
||||
|
||||
The implementation lives in [plugins/ftp/__init__.py](plugins/ftp/__init__.py).
|
||||
|
||||
## What The Plugin Does
|
||||
|
||||
The FTP plugin demonstrates the main provider hooks that matter for a storage-style integration:
|
||||
|
||||
- `config_schema()` exposes host, credentials, base path, TLS, and search depth.
|
||||
- `extract_query_arguments()` supports inline query fields like `path:` and `depth:`.
|
||||
- `search()` walks an FTP directory tree and returns `SearchResult` rows.
|
||||
- `selector()` turns folder rows into a follow-up table when the user runs `@N`.
|
||||
- `download()` and `download_url()` fetch FTP files into `download-file` output paths.
|
||||
- `resolve_pipe_result_download()` lets `@N | add-file -store ...` materialize a remote FTP file first.
|
||||
- `upload()` lets `add-file -plugin ftp -path ...` push a local file to the configured FTP server.
|
||||
|
||||
## Example Config
|
||||
|
||||
Add an FTP provider block to your config:
|
||||
|
||||
```toml
|
||||
[provider.ftp]
|
||||
host = "ftp.example.com"
|
||||
port = 21
|
||||
username = "demo"
|
||||
password = "secret"
|
||||
base_path = "/incoming"
|
||||
tls = false
|
||||
passive = true
|
||||
timeout = 20
|
||||
search_depth = 1
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- `host` is the only required field for the plugin to validate.
|
||||
- `username` defaults to `anonymous` and `password` defaults to `anonymous@`.
|
||||
- `base_path` is both the default search root and the upload target directory.
|
||||
- `search_depth` controls how many folder levels `search-file -plugin ftp` scans by default.
|
||||
|
||||
## Search Flow
|
||||
|
||||
Basic listing from the configured base path:
|
||||
|
||||
```powershell
|
||||
search-file -plugin ftp "*"
|
||||
```
|
||||
|
||||
Search by filename fragment:
|
||||
|
||||
```powershell
|
||||
search-file -plugin ftp "invoice"
|
||||
```
|
||||
|
||||
Search a different subtree and recurse deeper:
|
||||
|
||||
```powershell
|
||||
search-file -plugin ftp "path:/pub depth:2 invoice"
|
||||
```
|
||||
|
||||
Filter to folders only:
|
||||
|
||||
```powershell
|
||||
search-file -plugin ftp "path:/pub type:folder *"
|
||||
```
|
||||
|
||||
The plugin returns rows with explicit columns for name, type, directory, size, and modification time.
|
||||
|
||||
## Selection Flow
|
||||
|
||||
Folder rows are navigation rows. If the selected row is a directory, plain `@N` opens a new FTP table for that directory:
|
||||
|
||||
```powershell
|
||||
search-file -plugin ftp "*"
|
||||
@2
|
||||
```
|
||||
|
||||
File rows carry an explicit row action:
|
||||
|
||||
```powershell
|
||||
download-file -plugin ftp -url ftp://ftp.example.com/incoming/report.pdf
|
||||
```
|
||||
|
||||
That means plain `@N` on a file row downloads it immediately:
|
||||
|
||||
```powershell
|
||||
search-file -plugin ftp "report"
|
||||
@1
|
||||
```
|
||||
|
||||
## Download And Add-File Flow
|
||||
|
||||
If you want the downloaded file in a specific local directory:
|
||||
|
||||
```powershell
|
||||
search-file -plugin ftp "report"
|
||||
@1 | download-file -path C:\Downloads
|
||||
```
|
||||
|
||||
If you want to ingest the selected FTP file into a configured store backend:
|
||||
|
||||
```powershell
|
||||
search-file -plugin ftp "report"
|
||||
@1 | add-file -store tutorial
|
||||
```
|
||||
|
||||
Why this works:
|
||||
|
||||
- the file row advertises a `download-file` row action
|
||||
- the pipeline auto-inserts that download before `add-file`
|
||||
- the FTP plugin also implements `resolve_pipe_result_download()` so provider-owned FTP rows can be materialized for ingestion
|
||||
|
||||
## Upload Flow
|
||||
|
||||
Uploading uses the same provider name, but through `add-file -plugin ftp`:
|
||||
|
||||
```powershell
|
||||
add-file -plugin ftp -path C:\Media\report.pdf
|
||||
```
|
||||
|
||||
That sends the file to the configured FTP `base_path` and returns the FTP URL as the uploaded result.
|
||||
|
||||
## Why The Row Metadata Matters
|
||||
|
||||
The critical part of this plugin is the file-row metadata:
|
||||
|
||||
- file rows emit `_selection_args` as `['-url', '<ftp-url>']`
|
||||
- file rows emit `_selection_action` as `['download-file', '-plugin', 'ftp', '-url', '<ftp-url>']`
|
||||
- folder rows do not emit a download action, so `selector()` can own drill-in behavior instead
|
||||
|
||||
That split is what keeps these two user experiences compatible:
|
||||
|
||||
- `@N` on a folder opens a new table
|
||||
- `@N` on a file downloads the file
|
||||
- `@N | add-file -store ...` first downloads, then ingests
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The plugin prefers `MLSD` for directory listings and falls back to `NLST` plus directory probes when the server does not support machine-readable listings.
|
||||
|
||||
The code is intentionally small and uses only Python stdlib pieces:
|
||||
|
||||
- `ftplib` for FTP and FTPS
|
||||
- `fnmatch` for wildcard-style search tokens
|
||||
- `tempfile` for `add-file` handoff downloads
|
||||
|
||||
## Recommended Commands To Demo The Walkthrough
|
||||
|
||||
```powershell
|
||||
search-file -plugin ftp "*"
|
||||
search-file -plugin ftp "path:/incoming depth:2 *.pdf"
|
||||
@1
|
||||
@1 | download-file -path C:\Downloads
|
||||
@1 | add-file -store tutorial
|
||||
add-file -plugin ftp -path C:\Media\report.pdf
|
||||
```
|
||||
@@ -0,0 +1,136 @@
|
||||
# SCP Plugin Walkthrough
|
||||
|
||||
This walkthrough adds a bundled `scp` plugin backed by existing SSH libraries:
|
||||
|
||||
- `paramiko` for SSH and SFTP directory listing
|
||||
- `scp` for file transfers
|
||||
|
||||
The implementation lives in [plugins/scp/__init__.py](plugins/scp/__init__.py).
|
||||
|
||||
## What The Plugin Does
|
||||
|
||||
The SCP plugin mirrors the FTP walkthrough, but on top of SSH:
|
||||
|
||||
- `search-file -plugin scp ...` lists remote files and folders over SFTP.
|
||||
- plain `@N` on a folder drills into that directory.
|
||||
- plain `@N` on a file runs `download-file -plugin scp -url ...`.
|
||||
- `@N | add-file -store ...` downloads first, then ingests the local temp file.
|
||||
- `add-file -plugin scp -path ...` uploads a local file to the configured remote path.
|
||||
|
||||
## Example Config
|
||||
|
||||
```toml
|
||||
[provider.scp]
|
||||
host = "ssh.example.com"
|
||||
port = 22
|
||||
username = "deploy"
|
||||
password = "secret"
|
||||
key_path = "C:/Users/Admin/.ssh/id_ed25519"
|
||||
base_path = "/srv/files"
|
||||
timeout = 20
|
||||
search_depth = 1
|
||||
allow_agent = true
|
||||
look_for_keys = true
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- `host` and `username` are required for the plugin to validate.
|
||||
- You can use password auth, key auth, or both.
|
||||
- `base_path` is both the default search root and the default upload directory.
|
||||
|
||||
## Search Flow
|
||||
|
||||
List the configured base path:
|
||||
|
||||
```powershell
|
||||
search-file -plugin scp "*"
|
||||
```
|
||||
|
||||
Search by filename:
|
||||
|
||||
```powershell
|
||||
search-file -plugin scp "invoice"
|
||||
```
|
||||
|
||||
Search another subtree with deeper recursion:
|
||||
|
||||
```powershell
|
||||
search-file -plugin scp "path:/srv/files/releases depth:2 *.zip"
|
||||
```
|
||||
|
||||
Show only folders:
|
||||
|
||||
```powershell
|
||||
search-file -plugin scp "path:/srv/files type:folder *"
|
||||
```
|
||||
|
||||
## Selection Flow
|
||||
|
||||
Folder rows are navigation rows:
|
||||
|
||||
```powershell
|
||||
search-file -plugin scp "*"
|
||||
@2
|
||||
```
|
||||
|
||||
File rows carry an explicit row action, so terminal selection downloads directly:
|
||||
|
||||
```powershell
|
||||
search-file -plugin scp "report"
|
||||
@1
|
||||
```
|
||||
|
||||
That expands to the equivalent of:
|
||||
|
||||
```powershell
|
||||
download-file -plugin scp -url scp://ssh.example.com/srv/files/report.pdf
|
||||
```
|
||||
|
||||
## Download And Add-File Flow
|
||||
|
||||
Download into a local folder:
|
||||
|
||||
```powershell
|
||||
search-file -plugin scp "report"
|
||||
@1 | download-file -path C:\Downloads
|
||||
```
|
||||
|
||||
Ingest a selected remote file into a configured store backend:
|
||||
|
||||
```powershell
|
||||
search-file -plugin scp "report"
|
||||
@1 | add-file -store tutorial
|
||||
```
|
||||
|
||||
Why this works:
|
||||
|
||||
- file rows advertise `_selection_action` for `download-file`
|
||||
- `add-file` selection replay inserts that provider download stage before ingest
|
||||
- the plugin also implements `resolve_pipe_result_download()` for provider-owned SCP rows
|
||||
|
||||
## Upload Flow
|
||||
|
||||
Upload a local file to the configured remote `base_path`:
|
||||
|
||||
```powershell
|
||||
add-file -plugin scp -path C:\Media\report.pdf
|
||||
```
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
The plugin uses SFTP for directory listing because SCP itself is a transfer protocol, not a browse/search protocol. That split keeps the provider simple:
|
||||
|
||||
- browse and metadata via Paramiko SFTP
|
||||
- file transfer via the `scp` package
|
||||
|
||||
## Recommended Demo Commands
|
||||
|
||||
```powershell
|
||||
search-file -plugin scp "*"
|
||||
search-file -plugin scp "path:/srv/files depth:2 *.zip"
|
||||
@1
|
||||
@1 | download-file -path C:\Downloads
|
||||
@1 | add-file -store tutorial
|
||||
add-file -plugin scp -path C:\Media\report.pdf
|
||||
```
|
||||
+9
-1
@@ -48,4 +48,12 @@ class MyPlugin(Provider):
|
||||
path=f"https://example.com/{text}",
|
||||
)
|
||||
]
|
||||
```
|
||||
```
|
||||
|
||||
Bundled walkthrough:
|
||||
|
||||
- The repo now includes a real FTP example plugin in [plugins/ftp/__init__.py](plugins/ftp/__init__.py).
|
||||
- The walkthrough is in [docs/ftp_plugin_tutorial.md](docs/ftp_plugin_tutorial.md) and shows `search-file -plugin ftp`, folder drill-in via `@N`, file download routing, `@N | add-file -store ...`, and `add-file -plugin ftp` uploads.
|
||||
- The repo also includes an SCP example plugin in [plugins/scp/__init__.py](plugins/scp/__init__.py).
|
||||
- The walkthrough is in [docs/scp_plugin_tutorial.md](docs/scp_plugin_tutorial.md) and shows `search-file -plugin scp`, SSH-backed directory drill-in, file download routing, `@N | add-file -store ...`, and `add-file -plugin scp` uploads.
|
||||
- The repo now also includes a built-in HydrusNetwork provider in [plugins/hydrusnetwork/__init__.py](plugins/hydrusnetwork/__init__.py). It delegates to configured `store.hydrusnetwork.*` backends so Hydrus features can be reached through the normal plugin registry without cmdlets importing Hydrus modules directly.
|
||||
@@ -23,24 +23,56 @@ from SYS.models import DownloadError, PipeObject
|
||||
_HOSTS_CACHE_TTL_SECONDS = 24 * 60 * 60
|
||||
|
||||
|
||||
def _repo_root() -> Path:
|
||||
def _plugin_dir() -> Path:
|
||||
try:
|
||||
return Path(__file__).resolve().parents[1]
|
||||
return Path(__file__).resolve().parent
|
||||
except Exception:
|
||||
return Path(".")
|
||||
|
||||
|
||||
def _legacy_hosts_cache_paths() -> Tuple[Path, ...]:
|
||||
try:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
plugins_root = Path(__file__).resolve().parents[1]
|
||||
except Exception:
|
||||
return tuple()
|
||||
return (
|
||||
plugins_root / "API" / "data" / "alldebrid.json",
|
||||
repo_root / "API" / "data" / "alldebrid.json",
|
||||
)
|
||||
|
||||
|
||||
def _hosts_cache_path() -> Path:
|
||||
# Keep this local to the repo so it works in portable installs.
|
||||
# The registry's URL routing can read this file without instantiating providers.
|
||||
# Keep this local to the plugin so plugin-specific cache/state stays bundled
|
||||
# with the plugin itself in portable installs.
|
||||
#
|
||||
# This file is expected to be the JSON payload shape from AllDebrid:
|
||||
# {"status":"success","data":{"hosts":[...],"streams":[...],"redirectors":[...]}}
|
||||
return _repo_root() / "API" / "data" / "alldebrid.json"
|
||||
return _plugin_dir() / "alldebrid.json"
|
||||
|
||||
|
||||
def _resolve_hosts_cache_path() -> Path:
|
||||
path = _hosts_cache_path()
|
||||
try:
|
||||
if path.exists() and path.is_file():
|
||||
return path
|
||||
except Exception:
|
||||
return path
|
||||
|
||||
for legacy in _legacy_hosts_cache_paths():
|
||||
try:
|
||||
if not legacy.exists() or not legacy.is_file():
|
||||
continue
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(legacy, path)
|
||||
return path
|
||||
except Exception:
|
||||
continue
|
||||
return path
|
||||
|
||||
|
||||
def _load_cached_domains(category: str) -> List[str]:
|
||||
"""Load cached domain list from API/data/alldebrid.json.
|
||||
"""Load cached domain list from the plugin-local alldebrid.json cache.
|
||||
|
||||
category: "hosts" | "streams" | "redirectors"
|
||||
"""
|
||||
@@ -49,7 +81,7 @@ def _load_cached_domains(category: str) -> List[str]:
|
||||
if wanted not in {"hosts", "streams", "redirectors"}:
|
||||
return []
|
||||
|
||||
path = _hosts_cache_path()
|
||||
path = _resolve_hosts_cache_path()
|
||||
try:
|
||||
if not path.exists() or not path.is_file():
|
||||
return []
|
||||
@@ -68,12 +100,27 @@ def _load_cached_domains(category: str) -> List[str]:
|
||||
return []
|
||||
|
||||
raw_list = data.get(wanted)
|
||||
if not isinstance(raw_list, list):
|
||||
if not isinstance(raw_list, (list, dict)):
|
||||
return []
|
||||
|
||||
out: List[str] = []
|
||||
seen: set[str] = set()
|
||||
for d in raw_list:
|
||||
|
||||
domain_candidates: List[Any] = []
|
||||
if isinstance(raw_list, list):
|
||||
domain_candidates.extend(raw_list)
|
||||
else:
|
||||
for entry in raw_list.values():
|
||||
if isinstance(entry, dict):
|
||||
nested_domains = entry.get("domains")
|
||||
if isinstance(nested_domains, list):
|
||||
domain_candidates.extend(nested_domains)
|
||||
elif isinstance(nested_domains, str):
|
||||
domain_candidates.append(nested_domains)
|
||||
elif isinstance(entry, str):
|
||||
domain_candidates.append(entry)
|
||||
|
||||
for d in domain_candidates:
|
||||
try:
|
||||
dom = str(d or "").strip().lower()
|
||||
except Exception:
|
||||
@@ -115,7 +162,7 @@ def _save_cached_hosts_payload(payload: Dict[str, Any]) -> None:
|
||||
|
||||
|
||||
def _cache_is_fresh() -> bool:
|
||||
path = _hosts_cache_path()
|
||||
path = _resolve_hosts_cache_path()
|
||||
try:
|
||||
if not path.exists() or not path.is_file():
|
||||
return False
|
||||
|
||||
@@ -37,7 +37,7 @@
|
||||
"(rapidgator\\.net/file/[0-9]{7,8})"
|
||||
],
|
||||
"regexp": "((rapidgator\\.net|rg\\.to|rapidgator\\.asia)/file/([0-9a-zA-Z]{32}))|((rapidgator\\.net/file/[0-9]{7,8}))",
|
||||
"status": true
|
||||
"status": false
|
||||
},
|
||||
"turbobit": {
|
||||
"name": "turbobit",
|
||||
@@ -71,7 +71,7 @@
|
||||
"(wayupload\\.com/[a-z0-9]{12}\\.html)"
|
||||
],
|
||||
"regexp": "(turbobit5?a?\\.(net|cc|com)/([a-z0-9]{12}))|(turbobif\\.(net|cc|com)/([a-z0-9]{12}))|(turb[o]?\\.(to|cc|pw)\\/([a-z0-9]{12}))|(turbobit\\.(net|cc)/download/free/([a-z0-9]{12}))|((trbbt|tourbobit|torbobit|tbit|turbobita|trbt)\\.(net|cc|com|to)/([a-z0-9]{12}))|((turbobit\\.cloud/turbo/[a-z0-9]+))|((wayupload\\.com/[a-z0-9]{12}\\.html))",
|
||||
"status": false
|
||||
"status": true
|
||||
},
|
||||
"hitfile": {
|
||||
"name": "hitfile",
|
||||
@@ -375,7 +375,7 @@
|
||||
"(filespace\\.com/[a-zA-Z0-9]{12})"
|
||||
],
|
||||
"regexp": "(filespace\\.com/fd/([a-zA-Z0-9]{12}))|((filespace\\.com/[a-zA-Z0-9]{12}))",
|
||||
"status": false
|
||||
"status": true
|
||||
},
|
||||
"filezip": {
|
||||
"name": "filezip",
|
||||
@@ -0,0 +1,778 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import ftplib
|
||||
import posixpath
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from urllib.parse import quote, unquote, urlparse
|
||||
|
||||
from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments
|
||||
|
||||
|
||||
def _pick_provider_config(config: Any) -> Dict[str, Any]:
|
||||
if not isinstance(config, dict):
|
||||
return {}
|
||||
provider = config.get("provider")
|
||||
if not isinstance(provider, dict):
|
||||
return {}
|
||||
entry = provider.get("ftp")
|
||||
if isinstance(entry, dict):
|
||||
return entry
|
||||
return {}
|
||||
|
||||
|
||||
def _coerce_bool(value: Any, default: bool = False) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if value is None:
|
||||
return default
|
||||
text = str(value).strip().lower()
|
||||
if not text:
|
||||
return default
|
||||
if text in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if text in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
return default
|
||||
|
||||
|
||||
def _coerce_int(value: Any, default: int) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def _format_timestamp(raw_value: Any) -> str:
|
||||
text = str(raw_value or "").strip()
|
||||
if not text:
|
||||
return ""
|
||||
for pattern in ("%Y%m%d%H%M%S", "%Y%m%d%H%M%S.%f"):
|
||||
try:
|
||||
parsed = datetime.strptime(text, pattern)
|
||||
return parsed.strftime("%Y-%m-%d %H:%M")
|
||||
except Exception:
|
||||
continue
|
||||
return text
|
||||
|
||||
|
||||
def _safe_filename(name: Any) -> str:
|
||||
raw = str(name or "").strip()
|
||||
if not raw:
|
||||
raw = "download"
|
||||
cleaned = "".join(ch if ch.isalnum() or ch in {"-", "_", ".", " "} else "_" for ch in raw)
|
||||
cleaned = cleaned.strip(" ._")
|
||||
return cleaned or "download"
|
||||
|
||||
|
||||
def _unique_path(path: Path) -> Path:
|
||||
if not path.exists():
|
||||
return path
|
||||
stem = path.stem or "download"
|
||||
suffix = path.suffix
|
||||
counter = 1
|
||||
while True:
|
||||
candidate = path.with_name(f"{stem}_{counter}{suffix}")
|
||||
if not candidate.exists():
|
||||
return candidate
|
||||
counter += 1
|
||||
|
||||
|
||||
class FTP(Provider):
|
||||
PLUGIN_NAME = "ftp"
|
||||
URL = ("ftp://", "ftps://")
|
||||
|
||||
@property
|
||||
def label(self) -> str:
|
||||
return "FTP"
|
||||
|
||||
@property
|
||||
def preserve_order(self) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def config_schema(cls) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"key": "host",
|
||||
"label": "Host",
|
||||
"default": "",
|
||||
"required": True,
|
||||
"placeholder": "ftp.example.com",
|
||||
},
|
||||
{
|
||||
"key": "port",
|
||||
"label": "Port",
|
||||
"type": "integer",
|
||||
"default": 21,
|
||||
},
|
||||
{
|
||||
"key": "username",
|
||||
"label": "Username",
|
||||
"default": "anonymous",
|
||||
},
|
||||
{
|
||||
"key": "password",
|
||||
"label": "Password",
|
||||
"type": "secret",
|
||||
"secret": True,
|
||||
"default": "",
|
||||
},
|
||||
{
|
||||
"key": "base_path",
|
||||
"label": "Base Path",
|
||||
"default": "/",
|
||||
"placeholder": "/incoming",
|
||||
},
|
||||
{
|
||||
"key": "tls",
|
||||
"label": "Use FTPS",
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
},
|
||||
{
|
||||
"key": "passive",
|
||||
"label": "Passive Mode",
|
||||
"type": "boolean",
|
||||
"default": True,
|
||||
},
|
||||
{
|
||||
"key": "timeout",
|
||||
"label": "Timeout Seconds",
|
||||
"type": "integer",
|
||||
"default": 20,
|
||||
},
|
||||
{
|
||||
"key": "search_depth",
|
||||
"label": "Default Search Depth",
|
||||
"type": "integer",
|
||||
"default": 1,
|
||||
},
|
||||
]
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||
super().__init__(config)
|
||||
conf = _pick_provider_config(self.config)
|
||||
self._host = str(conf.get("host") or "").strip()
|
||||
self._tls = _coerce_bool(conf.get("tls"), False)
|
||||
self._port = _coerce_int(conf.get("port"), 21)
|
||||
self._username = str(conf.get("username") or conf.get("user") or "anonymous").strip() or "anonymous"
|
||||
password_value = conf.get("password")
|
||||
self._password = str(password_value).strip() if password_value not in (None, "") else "anonymous@"
|
||||
self._passive = _coerce_bool(conf.get("passive"), True)
|
||||
self._timeout = max(1, _coerce_int(conf.get("timeout"), 20))
|
||||
self._search_depth = max(0, _coerce_int(conf.get("search_depth"), 1))
|
||||
self._base_path = self._normalize_remote_path(conf.get("base_path") or "/", default="/")
|
||||
|
||||
def validate(self) -> bool:
|
||||
return bool(self._host)
|
||||
|
||||
def config_helper_text(self) -> str:
|
||||
return "Test the configured FTP/FTPS settings before searching or uploading."
|
||||
|
||||
def config_actions(self) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": "test_connection",
|
||||
"label": "Test connection",
|
||||
"variant": "primary",
|
||||
}
|
||||
]
|
||||
|
||||
def run_config_action(self, action_id: str, **_kwargs: Any) -> Dict[str, Any]:
|
||||
if str(action_id or "").strip().lower() != "test_connection":
|
||||
return super().run_config_action(action_id, **_kwargs)
|
||||
|
||||
if not self._host:
|
||||
return {"ok": False, "message": "Set 'host' before testing the FTP connection."}
|
||||
|
||||
ftp = None
|
||||
try:
|
||||
ftp = self._connect()
|
||||
active_path = self._base_path or "/"
|
||||
try:
|
||||
ftp.cwd(active_path)
|
||||
resolved_path = ftp.pwd()
|
||||
except Exception:
|
||||
resolved_path = active_path
|
||||
return {
|
||||
"ok": True,
|
||||
"message": f"Connected to FTP {self._host}:{self._port} and reached {resolved_path}.",
|
||||
}
|
||||
except Exception as exc:
|
||||
return {"ok": False, "message": f"FTP connection failed: {exc}"}
|
||||
finally:
|
||||
self._close(ftp)
|
||||
|
||||
def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]:
|
||||
text, inline = parse_inline_query_arguments(query)
|
||||
filters: Dict[str, Any] = {}
|
||||
|
||||
if inline.get("path"):
|
||||
filters["path"] = inline.get("path")
|
||||
if inline.get("depth"):
|
||||
filters["depth"] = max(0, _coerce_int(inline.get("depth"), self._search_depth))
|
||||
if inline.get("type"):
|
||||
filters["type"] = str(inline.get("type") or "").strip().lower()
|
||||
|
||||
return text, filters
|
||||
|
||||
def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str:
|
||||
active_path = self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path)
|
||||
text = str(query or "").strip()
|
||||
if not text or text == "*":
|
||||
return f"FTP: {active_path}"
|
||||
return f"FTP: {text} @ {active_path}"
|
||||
|
||||
def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
return {
|
||||
"plugin": self.name,
|
||||
"host": self._host,
|
||||
"path": self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path),
|
||||
"query": str(query or "").strip(),
|
||||
}
|
||||
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
limit: int = 50,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
**kwargs: Any,
|
||||
) -> List[SearchResult]:
|
||||
_ = kwargs
|
||||
active_filters = dict(filters or {})
|
||||
start_path = self._normalize_remote_path(active_filters.get("path") or self._base_path, default=self._base_path)
|
||||
search_depth = max(0, _coerce_int(active_filters.get("depth"), self._search_depth))
|
||||
type_filter = str(active_filters.get("type") or "any").strip().lower()
|
||||
needle = str(query or "").strip()
|
||||
max_results = max(0, int(limit or 0))
|
||||
if max_results <= 0:
|
||||
return []
|
||||
|
||||
ftp = self._connect()
|
||||
try:
|
||||
return self._search_directory(
|
||||
ftp,
|
||||
start_path,
|
||||
needle=needle,
|
||||
limit=max_results,
|
||||
search_depth=search_depth,
|
||||
type_filter=type_filter,
|
||||
)
|
||||
finally:
|
||||
self._close(ftp)
|
||||
|
||||
def selector(
|
||||
self,
|
||||
selected_items: List[Any],
|
||||
*,
|
||||
ctx: Any,
|
||||
stage_is_last: bool = True,
|
||||
**_kwargs: Any,
|
||||
) -> bool:
|
||||
if not stage_is_last:
|
||||
return False
|
||||
|
||||
target_path = ""
|
||||
target_title = ""
|
||||
for item in selected_items or []:
|
||||
metadata = self._item_metadata(item)
|
||||
if not metadata.get("is_dir"):
|
||||
continue
|
||||
target_path = self._normalize_remote_path(metadata.get("ftp_path") or metadata.get("selection_path"), default=self._base_path)
|
||||
target_title = str(metadata.get("title") or metadata.get("name") or "").strip()
|
||||
if target_path:
|
||||
break
|
||||
|
||||
if not target_path:
|
||||
return False
|
||||
|
||||
ftp = self._connect()
|
||||
try:
|
||||
rows = self._search_directory(
|
||||
ftp,
|
||||
target_path,
|
||||
needle="*",
|
||||
limit=500,
|
||||
search_depth=0,
|
||||
type_filter="any",
|
||||
)
|
||||
finally:
|
||||
self._close(ftp)
|
||||
|
||||
try:
|
||||
from SYS.result_table import Table
|
||||
from SYS.rich_display import stdout_console
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
title = target_title or target_path
|
||||
table = Table(f"FTP: {title}")._perseverance(True)
|
||||
table.set_table("ftp")
|
||||
try:
|
||||
table.set_table_metadata({
|
||||
"provider": "ftp",
|
||||
"host": self._host,
|
||||
"path": target_path,
|
||||
"view": "directory",
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
table.set_source_command("search-file", ["-plugin", "ftp", f"path:{target_path}", "*"])
|
||||
|
||||
payloads: List[Dict[str, Any]] = []
|
||||
for row in rows:
|
||||
table.add_result(row)
|
||||
payloads.append(row.to_dict())
|
||||
|
||||
try:
|
||||
ctx.set_last_result_table(table, payloads, subject={"plugin": "ftp", "path": target_path})
|
||||
ctx.set_current_stage_table(table)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
stdout_console().print()
|
||||
stdout_console().print(table)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return True
|
||||
|
||||
def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]:
|
||||
metadata = getattr(result, "full_metadata", None)
|
||||
if isinstance(metadata, dict) and metadata.get("is_dir"):
|
||||
return None
|
||||
target = str(getattr(result, "path", "") or "").strip()
|
||||
if not target:
|
||||
return None
|
||||
return self.download_url(target, output_dir, title=getattr(result, "title", None))
|
||||
|
||||
def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]:
|
||||
settings = self._connection_settings_for_url(url)
|
||||
remote_path = settings["path"]
|
||||
if not remote_path or remote_path == "/":
|
||||
return None
|
||||
|
||||
filename_hint = str(kwargs.get("title") or "").strip()
|
||||
parsed_name = posixpath.basename(remote_path.rstrip("/"))
|
||||
filename = _safe_filename(filename_hint or unquote(parsed_name) or "download")
|
||||
|
||||
destination_dir = Path(output_dir)
|
||||
destination_dir.mkdir(parents=True, exist_ok=True)
|
||||
destination = _unique_path(destination_dir / filename)
|
||||
|
||||
ftp = self._connect(
|
||||
host=settings["host"],
|
||||
port=settings["port"],
|
||||
username=settings["username"],
|
||||
password=settings["password"],
|
||||
tls=settings["tls"],
|
||||
)
|
||||
try:
|
||||
with destination.open("wb") as handle:
|
||||
ftp.retrbinary(f"RETR {remote_path}", handle.write)
|
||||
return destination
|
||||
except Exception:
|
||||
try:
|
||||
destination.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
finally:
|
||||
self._close(ftp)
|
||||
|
||||
def resolve_pipe_result_download(
|
||||
self,
|
||||
result: Any,
|
||||
pipe_obj: Any,
|
||||
) -> Tuple[Optional[Path], Optional[str], Optional[Path]]:
|
||||
metadata = self._item_metadata(result, pipe_obj=pipe_obj)
|
||||
if metadata.get("is_dir"):
|
||||
return None, None, None
|
||||
|
||||
download_url = str(
|
||||
metadata.get("selection_url")
|
||||
or metadata.get("ftp_url")
|
||||
or metadata.get("path")
|
||||
or ""
|
||||
).strip()
|
||||
if not download_url.startswith(("ftp://", "ftps://")):
|
||||
return None, None, None
|
||||
|
||||
temp_dir = Path(tempfile.mkdtemp(prefix="ftp-add-file-"))
|
||||
downloaded = self.download_url(download_url, temp_dir, title=metadata.get("title"))
|
||||
if downloaded is None:
|
||||
try:
|
||||
temp_dir.rmdir()
|
||||
except Exception:
|
||||
pass
|
||||
return None, None, None
|
||||
|
||||
try:
|
||||
if pipe_obj is not None:
|
||||
pipe_obj.is_temp = True
|
||||
except Exception:
|
||||
pass
|
||||
return downloaded, None, temp_dir
|
||||
|
||||
def upload(self, file_path: str, **kwargs: Any) -> str:
|
||||
local_path = Path(str(file_path or "")).expanduser()
|
||||
if not local_path.exists() or not local_path.is_file():
|
||||
raise FileNotFoundError(f"File not found: {local_path}")
|
||||
|
||||
remote_dir = self._normalize_remote_path(kwargs.get("remote_path") or kwargs.get("path") or self._base_path, default=self._base_path)
|
||||
remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name
|
||||
remote_path = self._join_remote_path(remote_dir, remote_name)
|
||||
|
||||
ftp = self._connect()
|
||||
try:
|
||||
self._ensure_directory(ftp, remote_dir)
|
||||
with local_path.open("rb") as handle:
|
||||
ftp.storbinary(f"STOR {remote_path}", handle)
|
||||
finally:
|
||||
self._close(ftp)
|
||||
|
||||
return self._build_url(remote_path)
|
||||
|
||||
def _connect(
|
||||
self,
|
||||
*,
|
||||
host: Optional[str] = None,
|
||||
port: Optional[int] = None,
|
||||
username: Optional[str] = None,
|
||||
password: Optional[str] = None,
|
||||
tls: Optional[bool] = None,
|
||||
) -> ftplib.FTP:
|
||||
use_tls = self._tls if tls is None else bool(tls)
|
||||
ftp: ftplib.FTP = ftplib.FTP_TLS() if use_tls else ftplib.FTP()
|
||||
ftp.connect(host or self._host, int(port or self._port), timeout=self._timeout)
|
||||
ftp.login(username or self._username, password or self._password)
|
||||
try:
|
||||
ftp.set_pasv(self._passive)
|
||||
except Exception:
|
||||
pass
|
||||
if use_tls and isinstance(ftp, ftplib.FTP_TLS):
|
||||
ftp.prot_p()
|
||||
return ftp
|
||||
|
||||
def _close(self, ftp: Optional[ftplib.FTP]) -> None:
|
||||
if ftp is None:
|
||||
return
|
||||
try:
|
||||
ftp.quit()
|
||||
except Exception:
|
||||
try:
|
||||
ftp.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _normalize_remote_path(self, value: Any, *, default: str) -> str:
|
||||
text = str(value or "").strip().replace("\\", "/")
|
||||
if not text:
|
||||
text = default
|
||||
elif text.startswith(("ftp://", "ftps://")):
|
||||
try:
|
||||
text = unquote(urlparse(text).path or "/")
|
||||
except Exception:
|
||||
text = default
|
||||
elif not text.startswith("/"):
|
||||
text = posixpath.join(default, text)
|
||||
|
||||
normalized = posixpath.normpath(text)
|
||||
normalized = "/" + normalized.lstrip("/")
|
||||
return normalized or "/"
|
||||
|
||||
def _join_remote_path(self, parent: Any, child: Any) -> str:
|
||||
left = self._normalize_remote_path(parent, default=self._base_path)
|
||||
right = str(child or "").strip().replace("\\", "/")
|
||||
if not right:
|
||||
return left
|
||||
return self._normalize_remote_path(posixpath.join(left, right), default="/")
|
||||
|
||||
def _build_url(
|
||||
self,
|
||||
remote_path: Any,
|
||||
*,
|
||||
host: Optional[str] = None,
|
||||
port: Optional[int] = None,
|
||||
tls: Optional[bool] = None,
|
||||
) -> str:
|
||||
path_text = self._normalize_remote_path(remote_path, default="/")
|
||||
scheme = "ftps" if (self._tls if tls is None else bool(tls)) else "ftp"
|
||||
host_text = str(host or self._host).strip()
|
||||
port_value = int(port or self._port)
|
||||
port_suffix = f":{port_value}" if port_value and port_value != 21 else ""
|
||||
return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}"
|
||||
|
||||
def _connection_settings_for_url(self, url: str) -> Dict[str, Any]:
|
||||
parsed = urlparse(str(url or "").strip())
|
||||
scheme = (parsed.scheme or "ftp").strip().lower()
|
||||
host = parsed.hostname or self._host
|
||||
port = parsed.port or self._port
|
||||
username = parsed.username or self._username
|
||||
password = parsed.password or self._password
|
||||
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default="/")
|
||||
return {
|
||||
"tls": scheme == "ftps",
|
||||
"host": host,
|
||||
"port": port,
|
||||
"username": username,
|
||||
"password": password,
|
||||
"path": path_text,
|
||||
}
|
||||
|
||||
def _search_directory(
|
||||
self,
|
||||
ftp: ftplib.FTP,
|
||||
start_path: str,
|
||||
*,
|
||||
needle: str,
|
||||
limit: int,
|
||||
search_depth: int,
|
||||
type_filter: str,
|
||||
) -> List[SearchResult]:
|
||||
results: List[SearchResult] = []
|
||||
visited: set[str] = set()
|
||||
|
||||
def walk(current_path: str, depth_left: int) -> None:
|
||||
normalized = self._normalize_remote_path(current_path, default=self._base_path)
|
||||
if normalized in visited or len(results) >= limit:
|
||||
return
|
||||
visited.add(normalized)
|
||||
|
||||
for entry in self._list_directory(ftp, normalized):
|
||||
if len(results) >= limit:
|
||||
return
|
||||
if self._matches_entry(entry, needle=needle, type_filter=type_filter):
|
||||
results.append(self._build_result(entry))
|
||||
if entry.get("is_dir") and depth_left > 0:
|
||||
walk(str(entry.get("ftp_path") or normalized), depth_left - 1)
|
||||
|
||||
walk(start_path, max(0, search_depth))
|
||||
return results
|
||||
|
||||
def _matches_entry(self, entry: Dict[str, Any], *, needle: str, type_filter: str) -> bool:
|
||||
is_dir = bool(entry.get("is_dir"))
|
||||
if type_filter in {"dir", "dirs", "folder", "folders"} and not is_dir:
|
||||
return False
|
||||
if type_filter in {"file", "files"} and is_dir:
|
||||
return False
|
||||
|
||||
text = str(needle or "").strip().lower()
|
||||
if not text or text in {"*", "all", "list"}:
|
||||
return True
|
||||
|
||||
haystacks = [
|
||||
str(entry.get("name") or "").lower(),
|
||||
str(entry.get("ftp_path") or "").lower(),
|
||||
]
|
||||
for token in [part for part in text.split() if part]:
|
||||
if any(ch in token for ch in "*?[]"):
|
||||
if not any(fnmatch.fnmatch(haystack, token) for haystack in haystacks):
|
||||
return False
|
||||
elif not any(token in haystack for haystack in haystacks):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _build_result(self, entry: Dict[str, Any]) -> SearchResult:
|
||||
ftp_path = str(entry.get("ftp_path") or "/")
|
||||
ftp_url = self._build_url(ftp_path)
|
||||
is_dir = bool(entry.get("is_dir"))
|
||||
size_value = entry.get("size")
|
||||
modified = str(entry.get("modified") or "")
|
||||
parent = posixpath.dirname(ftp_path.rstrip("/")) or "/"
|
||||
metadata = {
|
||||
"provider": "ftp",
|
||||
"host": self._host,
|
||||
"ftp_path": ftp_path,
|
||||
"ftp_url": ftp_url,
|
||||
"selection_url": ftp_url,
|
||||
"is_dir": is_dir,
|
||||
"name": str(entry.get("name") or "").strip(),
|
||||
}
|
||||
if size_value is not None:
|
||||
metadata["size"] = size_value
|
||||
if modified:
|
||||
metadata["modified"] = modified
|
||||
|
||||
return SearchResult(
|
||||
table="ftp",
|
||||
title=str(entry.get("name") or ftp_path),
|
||||
path=ftp_url,
|
||||
detail=parent,
|
||||
annotations=["folder" if is_dir else "file"],
|
||||
media_kind="folder" if is_dir else "file",
|
||||
size_bytes=int(size_value) if isinstance(size_value, int) else None,
|
||||
tag={"ftp", "folder" if is_dir else "file"},
|
||||
columns=[
|
||||
("Name", str(entry.get("name") or "")),
|
||||
("Type", "dir" if is_dir else "file"),
|
||||
("Directory", parent),
|
||||
("Size", "" if size_value is None else str(size_value)),
|
||||
("Modified", modified),
|
||||
],
|
||||
selection_args=None if is_dir else ["-url", ftp_url],
|
||||
selection_action=None if is_dir else ["download-file", "-plugin", "ftp", "-url", ftp_url],
|
||||
full_metadata=metadata,
|
||||
)
|
||||
|
||||
def _list_directory(self, ftp: ftplib.FTP, remote_path: str) -> List[Dict[str, Any]]:
|
||||
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
|
||||
try:
|
||||
entries: List[Dict[str, Any]] = []
|
||||
for name, facts in ftp.mlsd(normalized):
|
||||
name_text = str(name or "").strip()
|
||||
if not name_text or name_text in {".", ".."}:
|
||||
continue
|
||||
entry_type = str((facts or {}).get("type") or "").strip().lower()
|
||||
if entry_type in {"cdir", "pdir"}:
|
||||
continue
|
||||
size_value = None
|
||||
raw_size = (facts or {}).get("size")
|
||||
if raw_size not in (None, ""):
|
||||
try:
|
||||
size_value = int(raw_size)
|
||||
except Exception:
|
||||
size_value = None
|
||||
entries.append(
|
||||
{
|
||||
"name": name_text,
|
||||
"ftp_path": self._join_remote_path(normalized, name_text),
|
||||
"is_dir": entry_type == "dir",
|
||||
"size": size_value,
|
||||
"modified": _format_timestamp((facts or {}).get("modify")),
|
||||
}
|
||||
)
|
||||
return entries
|
||||
except Exception:
|
||||
return self._list_directory_fallback(ftp, normalized)
|
||||
|
||||
def _list_directory_fallback(self, ftp: ftplib.FTP, remote_path: str) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
listed = ftp.nlst(remote_path)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
entries: List[Dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for raw_entry in listed:
|
||||
entry_text = str(raw_entry or "").strip()
|
||||
if not entry_text:
|
||||
continue
|
||||
entry_path = entry_text if entry_text.startswith("/") else self._join_remote_path(remote_path, entry_text)
|
||||
name_text = posixpath.basename(entry_path.rstrip("/")) or entry_path.rstrip("/")
|
||||
if not name_text or name_text in {".", ".."} or name_text in seen:
|
||||
continue
|
||||
seen.add(name_text)
|
||||
is_dir = self._is_directory(ftp, entry_path)
|
||||
size_value = None
|
||||
if not is_dir:
|
||||
try:
|
||||
size_raw = ftp.size(entry_path)
|
||||
if size_raw is not None:
|
||||
size_value = int(size_raw)
|
||||
except Exception:
|
||||
size_value = None
|
||||
entries.append(
|
||||
{
|
||||
"name": name_text,
|
||||
"ftp_path": entry_path,
|
||||
"is_dir": is_dir,
|
||||
"size": size_value,
|
||||
"modified": self._read_modified(ftp, entry_path),
|
||||
}
|
||||
)
|
||||
return entries
|
||||
|
||||
def _is_directory(self, ftp: ftplib.FTP, remote_path: str) -> bool:
|
||||
current = None
|
||||
try:
|
||||
current = ftp.pwd()
|
||||
except Exception:
|
||||
current = None
|
||||
try:
|
||||
ftp.cwd(remote_path)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
finally:
|
||||
if current is not None:
|
||||
try:
|
||||
ftp.cwd(current)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _read_modified(self, ftp: ftplib.FTP, remote_path: str) -> str:
|
||||
try:
|
||||
response = ftp.sendcmd(f"MDTM {remote_path}")
|
||||
except Exception:
|
||||
return ""
|
||||
parts = str(response or "").split()
|
||||
if len(parts) >= 2:
|
||||
return _format_timestamp(parts[1])
|
||||
return ""
|
||||
|
||||
def _ensure_directory(self, ftp: ftplib.FTP, remote_path: str) -> None:
|
||||
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
|
||||
if normalized == "/":
|
||||
return
|
||||
partial = ""
|
||||
for segment in [part for part in normalized.split("/") if part]:
|
||||
partial = f"{partial}/{segment}"
|
||||
if self._is_directory(ftp, partial):
|
||||
continue
|
||||
try:
|
||||
ftp.mkd(partial)
|
||||
except Exception:
|
||||
if not self._is_directory(ftp, partial):
|
||||
raise
|
||||
|
||||
def _item_metadata(self, item: Any, *, pipe_obj: Any = None) -> Dict[str, Any]:
|
||||
metadata: Dict[str, Any] = {}
|
||||
for source in (item, pipe_obj):
|
||||
if isinstance(source, dict):
|
||||
for key in ("title", "path", "url"):
|
||||
if source.get(key) is not None and key not in metadata:
|
||||
metadata[key] = source.get(key)
|
||||
nested = source.get("full_metadata") or source.get("metadata")
|
||||
if isinstance(nested, dict):
|
||||
metadata.update(nested)
|
||||
elif source is not None:
|
||||
for attr in ("title", "path", "url"):
|
||||
try:
|
||||
value = getattr(source, attr, None)
|
||||
except Exception:
|
||||
value = None
|
||||
if value is not None and attr not in metadata:
|
||||
metadata[attr] = value
|
||||
try:
|
||||
nested = getattr(source, "full_metadata", None) or getattr(source, "metadata", None)
|
||||
except Exception:
|
||||
nested = None
|
||||
if isinstance(nested, dict):
|
||||
metadata.update(nested)
|
||||
|
||||
ftp_path = metadata.get("ftp_path") or metadata.get("selection_path")
|
||||
if not ftp_path:
|
||||
path_value = metadata.get("path") or metadata.get("url") or metadata.get("ftp_url")
|
||||
path_text = str(path_value or "").strip()
|
||||
if path_text.startswith(("ftp://", "ftps://")):
|
||||
ftp_path = self._normalize_remote_path(path_text, default=self._base_path)
|
||||
if ftp_path:
|
||||
metadata["ftp_path"] = self._normalize_remote_path(ftp_path, default=self._base_path)
|
||||
metadata.setdefault("selection_path", metadata["ftp_path"])
|
||||
|
||||
if metadata.get("ftp_path") and not metadata.get("ftp_url"):
|
||||
metadata["ftp_url"] = self._build_url(metadata["ftp_path"])
|
||||
if metadata.get("ftp_url") and not metadata.get("selection_url"):
|
||||
metadata["selection_url"] = metadata["ftp_url"]
|
||||
|
||||
is_dir = metadata.get("is_dir")
|
||||
if is_dir is None and metadata.get("media_kind"):
|
||||
is_dir = str(metadata.get("media_kind") or "").strip().lower() == "folder"
|
||||
metadata["is_dir"] = bool(is_dir)
|
||||
return metadata
|
||||
+846
-50
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,938 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import posixpath
|
||||
import shlex
|
||||
import stat
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from urllib.parse import quote, unquote, urlparse
|
||||
|
||||
import paramiko
|
||||
from scp import SCPClient
|
||||
|
||||
from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments
|
||||
|
||||
|
||||
def _pick_provider_config(config: Any) -> Dict[str, Any]:
|
||||
if not isinstance(config, dict):
|
||||
return {}
|
||||
provider = config.get("provider")
|
||||
if not isinstance(provider, dict):
|
||||
return {}
|
||||
entry = provider.get("scp")
|
||||
if isinstance(entry, dict):
|
||||
return entry
|
||||
return {}
|
||||
|
||||
|
||||
def _coerce_bool(value: Any, default: bool = False) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if value is None:
|
||||
return default
|
||||
text = str(value).strip().lower()
|
||||
if not text:
|
||||
return default
|
||||
if text in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if text in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
return default
|
||||
|
||||
|
||||
def _coerce_int(value: Any, default: int) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def _format_epoch(raw_value: Any) -> str:
|
||||
try:
|
||||
stamp = int(raw_value)
|
||||
except Exception:
|
||||
return ""
|
||||
try:
|
||||
return datetime.fromtimestamp(stamp).strftime("%Y-%m-%d %H:%M")
|
||||
except Exception:
|
||||
return str(raw_value or "")
|
||||
|
||||
|
||||
def _safe_filename(name: Any) -> str:
|
||||
raw = str(name or "").strip()
|
||||
if not raw:
|
||||
raw = "download"
|
||||
cleaned = "".join(ch if ch.isalnum() or ch in {"-", "_", ".", " "} else "_" for ch in raw)
|
||||
cleaned = cleaned.strip(" ._")
|
||||
return cleaned or "download"
|
||||
|
||||
|
||||
def _unique_path(path: Path) -> Path:
|
||||
if not path.exists():
|
||||
return path
|
||||
stem = path.stem or "download"
|
||||
suffix = path.suffix
|
||||
counter = 1
|
||||
while True:
|
||||
candidate = path.with_name(f"{stem}_{counter}{suffix}")
|
||||
if not candidate.exists():
|
||||
return candidate
|
||||
counter += 1
|
||||
|
||||
|
||||
class SCP(Provider):
|
||||
PLUGIN_NAME = "scp"
|
||||
URL = ("scp://", "sftp://")
|
||||
|
||||
@property
|
||||
def label(self) -> str:
|
||||
return "SCP"
|
||||
|
||||
@property
|
||||
def preserve_order(self) -> bool:
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def config_schema(cls) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"key": "host",
|
||||
"label": "Host",
|
||||
"default": "",
|
||||
"required": True,
|
||||
"placeholder": "ssh.example.com",
|
||||
},
|
||||
{
|
||||
"key": "port",
|
||||
"label": "Port",
|
||||
"type": "integer",
|
||||
"default": 22,
|
||||
},
|
||||
{
|
||||
"key": "username",
|
||||
"label": "Username",
|
||||
"default": "",
|
||||
"required": True,
|
||||
"placeholder": "deploy",
|
||||
},
|
||||
{
|
||||
"key": "password",
|
||||
"label": "Password",
|
||||
"type": "secret",
|
||||
"secret": True,
|
||||
"default": "",
|
||||
},
|
||||
{
|
||||
"key": "key_path",
|
||||
"label": "SSH Key Path",
|
||||
"type": "path",
|
||||
"default": "",
|
||||
"placeholder": "C:/Users/Admin/.ssh/id_ed25519",
|
||||
},
|
||||
{
|
||||
"key": "base_path",
|
||||
"label": "Base Path",
|
||||
"default": "/",
|
||||
"placeholder": "/srv/files",
|
||||
},
|
||||
{
|
||||
"key": "timeout",
|
||||
"label": "Timeout Seconds",
|
||||
"type": "integer",
|
||||
"default": 20,
|
||||
},
|
||||
{
|
||||
"key": "search_depth",
|
||||
"label": "Default Search Depth",
|
||||
"type": "integer",
|
||||
"default": 1,
|
||||
},
|
||||
{
|
||||
"key": "allow_agent",
|
||||
"label": "Use SSH Agent",
|
||||
"type": "boolean",
|
||||
"default": True,
|
||||
},
|
||||
{
|
||||
"key": "look_for_keys",
|
||||
"label": "Look For Default Keys",
|
||||
"type": "boolean",
|
||||
"default": True,
|
||||
},
|
||||
]
|
||||
|
||||
def __init__(self, config: Optional[Dict[str, Any]] = None):
|
||||
super().__init__(config)
|
||||
conf = _pick_provider_config(self.config)
|
||||
self._host = str(conf.get("host") or "").strip()
|
||||
self._port = _coerce_int(conf.get("port"), 22)
|
||||
self._username = str(conf.get("username") or conf.get("user") or "").strip()
|
||||
self._password = str(conf.get("password") or "").strip()
|
||||
self._key_path = str(conf.get("key_path") or conf.get("identity_file") or "").strip()
|
||||
self._timeout = max(1, _coerce_int(conf.get("timeout"), 20))
|
||||
self._search_depth = max(0, _coerce_int(conf.get("search_depth"), 1))
|
||||
self._allow_agent = _coerce_bool(conf.get("allow_agent"), True)
|
||||
self._look_for_keys = _coerce_bool(conf.get("look_for_keys"), True)
|
||||
self._base_path = self._normalize_remote_path(conf.get("base_path") or "/", default="/")
|
||||
|
||||
def validate(self) -> bool:
|
||||
return bool(self._host and self._username)
|
||||
|
||||
def config_helper_text(self) -> str:
|
||||
return "Test the SSH/SCP connection before searching. You can also generate an RSA key pair from here."
|
||||
|
||||
def config_actions(self) -> List[Dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": "test_connection",
|
||||
"label": "Test connection",
|
||||
"variant": "primary",
|
||||
},
|
||||
{
|
||||
"id": "generate_ssh_key",
|
||||
"label": "Generate SSH key",
|
||||
"variant": "default",
|
||||
},
|
||||
]
|
||||
|
||||
def run_config_action(self, action_id: str, **_kwargs: Any) -> Dict[str, Any]:
|
||||
normalized = str(action_id or "").strip().lower()
|
||||
if normalized == "test_connection":
|
||||
return self._run_test_connection()
|
||||
if normalized == "generate_ssh_key":
|
||||
return self._generate_ssh_keypair()
|
||||
return super().run_config_action(action_id, **_kwargs)
|
||||
|
||||
def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]:
|
||||
text, inline = parse_inline_query_arguments(query)
|
||||
filters: Dict[str, Any] = {}
|
||||
|
||||
if inline.get("path"):
|
||||
filters["path"] = inline.get("path")
|
||||
if inline.get("depth"):
|
||||
filters["depth"] = max(0, _coerce_int(inline.get("depth"), self._search_depth))
|
||||
if inline.get("type"):
|
||||
filters["type"] = str(inline.get("type") or "").strip().lower()
|
||||
|
||||
return text, filters
|
||||
|
||||
def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str:
|
||||
active_path = self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path)
|
||||
text = str(query or "").strip()
|
||||
if not text or text == "*":
|
||||
return f"SCP: {active_path}"
|
||||
return f"SCP: {text} @ {active_path}"
|
||||
|
||||
def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
||||
return {
|
||||
"plugin": self.name,
|
||||
"host": self._host,
|
||||
"path": self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path),
|
||||
"query": str(query or "").strip(),
|
||||
}
|
||||
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
limit: int = 50,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
**kwargs: Any,
|
||||
) -> List[SearchResult]:
|
||||
_ = kwargs
|
||||
active_filters = dict(filters or {})
|
||||
start_path = self._normalize_remote_path(active_filters.get("path") or self._base_path, default=self._base_path)
|
||||
search_depth = max(0, _coerce_int(active_filters.get("depth"), self._search_depth))
|
||||
type_filter = str(active_filters.get("type") or "any").strip().lower()
|
||||
needle = str(query or "").strip()
|
||||
max_results = max(0, int(limit or 0))
|
||||
if max_results <= 0:
|
||||
return []
|
||||
|
||||
ssh = self._connect_ssh()
|
||||
sftp = None
|
||||
try:
|
||||
try:
|
||||
sftp = self._open_sftp(ssh)
|
||||
except Exception as exc:
|
||||
if not self._is_sftp_negotiation_error(exc):
|
||||
raise
|
||||
return self._search_directory_via_ssh(
|
||||
ssh,
|
||||
start_path,
|
||||
needle=needle,
|
||||
limit=max_results,
|
||||
search_depth=search_depth,
|
||||
type_filter=type_filter,
|
||||
)
|
||||
|
||||
return self._search_directory(
|
||||
sftp,
|
||||
start_path,
|
||||
needle=needle,
|
||||
limit=max_results,
|
||||
search_depth=search_depth,
|
||||
type_filter=type_filter,
|
||||
)
|
||||
finally:
|
||||
self._close_client(sftp)
|
||||
self._close_client(ssh)
|
||||
|
||||
def selector(
|
||||
self,
|
||||
selected_items: List[Any],
|
||||
*,
|
||||
ctx: Any,
|
||||
stage_is_last: bool = True,
|
||||
**_kwargs: Any,
|
||||
) -> bool:
|
||||
if not stage_is_last:
|
||||
return False
|
||||
|
||||
target_path = ""
|
||||
target_title = ""
|
||||
for item in selected_items or []:
|
||||
metadata = self._item_metadata(item)
|
||||
if not metadata.get("is_dir"):
|
||||
continue
|
||||
target_path = self._normalize_remote_path(metadata.get("scp_path") or metadata.get("selection_path"), default=self._base_path)
|
||||
target_title = str(metadata.get("title") or metadata.get("name") or "").strip()
|
||||
if target_path:
|
||||
break
|
||||
|
||||
if not target_path:
|
||||
return False
|
||||
|
||||
ssh = self._connect_ssh()
|
||||
sftp = None
|
||||
try:
|
||||
try:
|
||||
sftp = self._open_sftp(ssh)
|
||||
except Exception as exc:
|
||||
if not self._is_sftp_negotiation_error(exc):
|
||||
raise
|
||||
rows = self._search_directory_via_ssh(
|
||||
ssh,
|
||||
target_path,
|
||||
needle="*",
|
||||
limit=500,
|
||||
search_depth=0,
|
||||
type_filter="any",
|
||||
)
|
||||
else:
|
||||
rows = self._search_directory(
|
||||
sftp,
|
||||
target_path,
|
||||
needle="*",
|
||||
limit=500,
|
||||
search_depth=0,
|
||||
type_filter="any",
|
||||
)
|
||||
finally:
|
||||
self._close_client(sftp)
|
||||
self._close_client(ssh)
|
||||
|
||||
try:
|
||||
from SYS.result_table import Table
|
||||
from SYS.rich_display import stdout_console
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
title = target_title or target_path
|
||||
table = Table(f"SCP: {title}")._perseverance(True)
|
||||
table.set_table("scp")
|
||||
try:
|
||||
table.set_table_metadata({
|
||||
"provider": "scp",
|
||||
"host": self._host,
|
||||
"path": target_path,
|
||||
"view": "directory",
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
table.set_source_command("search-file", ["-plugin", "scp", f"path:{target_path}", "*"])
|
||||
|
||||
payloads: List[Dict[str, Any]] = []
|
||||
for row in rows:
|
||||
table.add_result(row)
|
||||
payloads.append(row.to_dict())
|
||||
|
||||
try:
|
||||
ctx.set_last_result_table(table, payloads, subject={"plugin": "scp", "path": target_path})
|
||||
ctx.set_current_stage_table(table)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
stdout_console().print()
|
||||
stdout_console().print(table)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return True
|
||||
|
||||
def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]:
|
||||
metadata = getattr(result, "full_metadata", None)
|
||||
if isinstance(metadata, dict) and metadata.get("is_dir"):
|
||||
return None
|
||||
target = str(getattr(result, "path", "") or "").strip()
|
||||
if not target:
|
||||
return None
|
||||
return self.download_url(target, output_dir, title=getattr(result, "title", None))
|
||||
|
||||
def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]:
|
||||
settings = self._connection_settings_for_url(url)
|
||||
remote_path = settings["path"]
|
||||
if not remote_path or remote_path == "/":
|
||||
return None
|
||||
|
||||
filename_hint = str(kwargs.get("title") or "").strip()
|
||||
parsed_name = posixpath.basename(remote_path.rstrip("/"))
|
||||
filename = _safe_filename(filename_hint or unquote(parsed_name) or "download")
|
||||
|
||||
destination_dir = Path(output_dir)
|
||||
destination_dir.mkdir(parents=True, exist_ok=True)
|
||||
destination = _unique_path(destination_dir / filename)
|
||||
|
||||
ssh = self._connect_ssh(settings)
|
||||
scp_client = None
|
||||
try:
|
||||
scp_client = self._open_scp(ssh)
|
||||
scp_client.get(remote_path, local_path=str(destination))
|
||||
return destination
|
||||
except Exception:
|
||||
try:
|
||||
destination.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
finally:
|
||||
self._close_client(scp_client)
|
||||
self._close_client(ssh)
|
||||
|
||||
def resolve_pipe_result_download(
|
||||
self,
|
||||
result: Any,
|
||||
pipe_obj: Any,
|
||||
) -> Tuple[Optional[Path], Optional[str], Optional[Path]]:
|
||||
metadata = self._item_metadata(result, pipe_obj=pipe_obj)
|
||||
if metadata.get("is_dir"):
|
||||
return None, None, None
|
||||
|
||||
download_url = str(
|
||||
metadata.get("selection_url")
|
||||
or metadata.get("scp_url")
|
||||
or metadata.get("path")
|
||||
or ""
|
||||
).strip()
|
||||
if not download_url.startswith(("scp://", "sftp://")):
|
||||
return None, None, None
|
||||
|
||||
temp_dir = Path(tempfile.mkdtemp(prefix="scp-add-file-"))
|
||||
downloaded = self.download_url(download_url, temp_dir, title=metadata.get("title"))
|
||||
if downloaded is None:
|
||||
try:
|
||||
temp_dir.rmdir()
|
||||
except Exception:
|
||||
pass
|
||||
return None, None, None
|
||||
|
||||
try:
|
||||
if pipe_obj is not None:
|
||||
pipe_obj.is_temp = True
|
||||
except Exception:
|
||||
pass
|
||||
return downloaded, None, temp_dir
|
||||
|
||||
def upload(self, file_path: str, **kwargs: Any) -> str:
|
||||
local_path = Path(str(file_path or "")).expanduser()
|
||||
if not local_path.exists() or not local_path.is_file():
|
||||
raise FileNotFoundError(f"File not found: {local_path}")
|
||||
|
||||
remote_dir = self._normalize_remote_path(kwargs.get("remote_path") or kwargs.get("path") or self._base_path, default=self._base_path)
|
||||
remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name
|
||||
remote_path = self._join_remote_path(remote_dir, remote_name)
|
||||
|
||||
ssh = self._connect_ssh()
|
||||
sftp = None
|
||||
scp_client = None
|
||||
try:
|
||||
try:
|
||||
sftp = self._open_sftp(ssh)
|
||||
except Exception as exc:
|
||||
if not self._is_sftp_negotiation_error(exc):
|
||||
raise
|
||||
self._ensure_directory_via_ssh(ssh, remote_dir)
|
||||
else:
|
||||
self._ensure_directory(sftp, remote_dir)
|
||||
scp_client = self._open_scp(ssh)
|
||||
scp_client.put(str(local_path), remote_path=remote_path)
|
||||
finally:
|
||||
self._close_client(scp_client)
|
||||
self._close_client(sftp)
|
||||
self._close_client(ssh)
|
||||
|
||||
return self._build_url(remote_path)
|
||||
|
||||
def _run_test_connection(self) -> Dict[str, Any]:
|
||||
if not self._host:
|
||||
return {"ok": False, "message": "Set 'host' before testing the SCP connection."}
|
||||
if not self._username:
|
||||
return {"ok": False, "message": "Set 'username' before testing the SCP connection."}
|
||||
|
||||
ssh = None
|
||||
sftp = None
|
||||
try:
|
||||
ssh = self._connect_ssh()
|
||||
base_path = self._base_path or "/"
|
||||
transport_detail = "SFTP available"
|
||||
try:
|
||||
sftp = self._open_sftp(ssh)
|
||||
except Exception as exc:
|
||||
if not self._is_sftp_negotiation_error(exc):
|
||||
raise
|
||||
is_dir = self._path_exists_via_ssh(ssh, base_path)
|
||||
transport_detail = "SFTP unavailable; using SSH command fallback"
|
||||
else:
|
||||
try:
|
||||
attrs = sftp.stat(base_path)
|
||||
is_dir = stat.S_ISDIR(getattr(attrs, "st_mode", 0))
|
||||
except Exception:
|
||||
is_dir = False
|
||||
detail = f" and confirmed {base_path}" if is_dir else ""
|
||||
auth_mode = f"key {self._key_path}" if self._key_path else "password/agent auth"
|
||||
return {
|
||||
"ok": True,
|
||||
"message": f"Connected to SCP {self._host}:{self._port} as {self._username} via {auth_mode}. {transport_detail}{detail}.",
|
||||
}
|
||||
except Exception as exc:
|
||||
return {"ok": False, "message": f"SCP connection failed: {exc}"}
|
||||
finally:
|
||||
self._close_client(sftp)
|
||||
self._close_client(ssh)
|
||||
|
||||
def _generate_ssh_keypair(self) -> Dict[str, Any]:
|
||||
target = Path(self._key_path).expanduser() if self._key_path else (Path.home() / ".ssh" / "medeia_scp_rsa")
|
||||
try:
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
except Exception as exc:
|
||||
return {"ok": False, "message": f"Could not create key directory: {exc}"}
|
||||
|
||||
public_path = target.with_name(target.name + ".pub")
|
||||
if target.exists() or public_path.exists():
|
||||
return {
|
||||
"ok": False,
|
||||
"message": f"SSH key already exists at {target}. Remove it or choose a different key_path first.",
|
||||
}
|
||||
|
||||
try:
|
||||
key = paramiko.RSAKey.generate(bits=4096)
|
||||
key.write_private_key_file(str(target))
|
||||
comment = f"{self._username or 'medeia'}@{self._host or 'scp'}"
|
||||
public_path.write_text(f"{key.get_name()} {key.get_base64()} {comment}\n", encoding="utf-8")
|
||||
try:
|
||||
target.chmod(0o600)
|
||||
except Exception:
|
||||
pass
|
||||
return {
|
||||
"ok": True,
|
||||
"message": f"Generated SSH key pair at {target}. Save the config to persist key_path.",
|
||||
"config_updates": {"key_path": str(target)},
|
||||
}
|
||||
except Exception as exc:
|
||||
try:
|
||||
target.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
public_path.unlink(missing_ok=True)
|
||||
except Exception:
|
||||
pass
|
||||
return {"ok": False, "message": f"SSH key generation failed: {exc}"}
|
||||
|
||||
def _connect_ssh(self, overrides: Optional[Dict[str, Any]] = None) -> paramiko.SSHClient:
|
||||
settings = dict(overrides or {})
|
||||
client = paramiko.SSHClient()
|
||||
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||
client.connect(
|
||||
hostname=str(settings.get("host") or self._host),
|
||||
port=int(settings.get("port") or self._port),
|
||||
username=str(settings.get("username") or self._username),
|
||||
password=str(settings.get("password") or self._password) or None,
|
||||
key_filename=str(settings.get("key_path") or self._key_path) or None,
|
||||
timeout=self._timeout,
|
||||
allow_agent=self._allow_agent if "allow_agent" not in settings else bool(settings.get("allow_agent")),
|
||||
look_for_keys=self._look_for_keys if "look_for_keys" not in settings else bool(settings.get("look_for_keys")),
|
||||
)
|
||||
return client
|
||||
|
||||
def _open_sftp(self, ssh: Any) -> Any:
|
||||
return ssh.open_sftp()
|
||||
|
||||
def _open_scp(self, ssh: Any) -> Any:
|
||||
return SCPClient(ssh.get_transport())
|
||||
|
||||
def _is_sftp_negotiation_error(self, exc: Exception) -> bool:
|
||||
text = str(exc or "").strip().lower()
|
||||
if isinstance(exc, EOFError):
|
||||
return True
|
||||
return any(
|
||||
marker in text
|
||||
for marker in (
|
||||
"eof during negotiation",
|
||||
"open failed",
|
||||
"channel closed",
|
||||
"administratively prohibited",
|
||||
"subsystem request failed",
|
||||
)
|
||||
)
|
||||
|
||||
def _run_ssh_command(self, ssh: Any, command: str) -> Tuple[int, str, str]:
|
||||
stdin, stdout, stderr = ssh.exec_command(command, timeout=self._timeout)
|
||||
try:
|
||||
stdin.close()
|
||||
except Exception:
|
||||
pass
|
||||
output = stdout.read().decode("utf-8", errors="replace")
|
||||
error = stderr.read().decode("utf-8", errors="replace")
|
||||
status = 0
|
||||
try:
|
||||
status = int(stdout.channel.recv_exit_status())
|
||||
except Exception:
|
||||
status = 0
|
||||
return status, output, error
|
||||
|
||||
def _path_exists_via_ssh(self, ssh: Any, remote_path: str) -> bool:
|
||||
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
|
||||
quoted_path = shlex.quote(normalized)
|
||||
status, _, _ = self._run_ssh_command(ssh, f"test -d {quoted_path}")
|
||||
return status == 0
|
||||
|
||||
def _ensure_directory_via_ssh(self, ssh: Any, remote_path: str) -> None:
|
||||
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
|
||||
if normalized == "/":
|
||||
return
|
||||
quoted_path = shlex.quote(normalized)
|
||||
status, _, error = self._run_ssh_command(ssh, f"mkdir -p {quoted_path}")
|
||||
if status != 0:
|
||||
raise RuntimeError(error.strip() or f"mkdir -p failed for {normalized}")
|
||||
|
||||
def _close_client(self, client: Any) -> None:
|
||||
if client is None:
|
||||
return
|
||||
try:
|
||||
client.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _normalize_remote_path(self, value: Any, *, default: str) -> str:
|
||||
text = str(value or "").strip().replace("\\", "/")
|
||||
if not text:
|
||||
text = default
|
||||
elif text.startswith(("scp://", "sftp://")):
|
||||
try:
|
||||
text = unquote(urlparse(text).path or "/")
|
||||
except Exception:
|
||||
text = default
|
||||
elif not text.startswith("/"):
|
||||
text = posixpath.join(default, text)
|
||||
|
||||
normalized = posixpath.normpath(text)
|
||||
normalized = "/" + normalized.lstrip("/")
|
||||
return normalized or "/"
|
||||
|
||||
def _join_remote_path(self, parent: Any, child: Any) -> str:
|
||||
left = self._normalize_remote_path(parent, default=self._base_path)
|
||||
right = str(child or "").strip().replace("\\", "/")
|
||||
if not right:
|
||||
return left
|
||||
return self._normalize_remote_path(posixpath.join(left, right), default="/")
|
||||
|
||||
def _build_url(
|
||||
self,
|
||||
remote_path: Any,
|
||||
*,
|
||||
host: Optional[str] = None,
|
||||
port: Optional[int] = None,
|
||||
scheme: str = "scp",
|
||||
) -> str:
|
||||
path_text = self._normalize_remote_path(remote_path, default="/")
|
||||
host_text = str(host or self._host).strip()
|
||||
port_value = int(port or self._port)
|
||||
port_suffix = f":{port_value}" if port_value and port_value != 22 else ""
|
||||
return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}"
|
||||
|
||||
def _connection_settings_for_url(self, url: str) -> Dict[str, Any]:
|
||||
parsed = urlparse(str(url or "").strip())
|
||||
scheme = (parsed.scheme or "scp").strip().lower()
|
||||
host = parsed.hostname or self._host
|
||||
port = parsed.port or self._port
|
||||
username = parsed.username or self._username
|
||||
password = parsed.password or self._password
|
||||
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default="/")
|
||||
return {
|
||||
"scheme": scheme,
|
||||
"host": host,
|
||||
"port": port,
|
||||
"username": username,
|
||||
"password": password,
|
||||
"key_path": self._key_path,
|
||||
"allow_agent": self._allow_agent,
|
||||
"look_for_keys": self._look_for_keys,
|
||||
"path": path_text,
|
||||
}
|
||||
|
||||
def _search_directory(
|
||||
self,
|
||||
sftp: Any,
|
||||
start_path: str,
|
||||
*,
|
||||
needle: str,
|
||||
limit: int,
|
||||
search_depth: int,
|
||||
type_filter: str,
|
||||
) -> List[SearchResult]:
|
||||
results: List[SearchResult] = []
|
||||
visited: set[str] = set()
|
||||
|
||||
def walk(current_path: str, depth_left: int) -> None:
|
||||
normalized = self._normalize_remote_path(current_path, default=self._base_path)
|
||||
if normalized in visited or len(results) >= limit:
|
||||
return
|
||||
visited.add(normalized)
|
||||
|
||||
for entry in self._list_directory(sftp, normalized):
|
||||
if len(results) >= limit:
|
||||
return
|
||||
if self._matches_entry(entry, needle=needle, type_filter=type_filter):
|
||||
results.append(self._build_result(entry))
|
||||
if entry.get("is_dir") and depth_left > 0:
|
||||
walk(str(entry.get("scp_path") or normalized), depth_left - 1)
|
||||
|
||||
walk(start_path, max(0, search_depth))
|
||||
return results
|
||||
|
||||
def _search_directory_via_ssh(
|
||||
self,
|
||||
ssh: Any,
|
||||
start_path: str,
|
||||
*,
|
||||
needle: str,
|
||||
limit: int,
|
||||
search_depth: int,
|
||||
type_filter: str,
|
||||
) -> List[SearchResult]:
|
||||
entries = self._list_directory_via_ssh(ssh, start_path, depth=search_depth)
|
||||
results: List[SearchResult] = []
|
||||
for entry in entries:
|
||||
if len(results) >= limit:
|
||||
break
|
||||
if self._matches_entry(entry, needle=needle, type_filter=type_filter):
|
||||
results.append(self._build_result(entry))
|
||||
return results
|
||||
|
||||
def _matches_entry(self, entry: Dict[str, Any], *, needle: str, type_filter: str) -> bool:
|
||||
is_dir = bool(entry.get("is_dir"))
|
||||
if type_filter in {"dir", "dirs", "folder", "folders"} and not is_dir:
|
||||
return False
|
||||
if type_filter in {"file", "files"} and is_dir:
|
||||
return False
|
||||
|
||||
text = str(needle or "").strip().lower()
|
||||
if not text or text in {"*", "all", "list"}:
|
||||
return True
|
||||
|
||||
haystacks = [
|
||||
str(entry.get("name") or "").lower(),
|
||||
str(entry.get("scp_path") or "").lower(),
|
||||
]
|
||||
for token in [part for part in text.split() if part]:
|
||||
if any(ch in token for ch in "*?[]"):
|
||||
if not any(fnmatch.fnmatch(haystack, token) for haystack in haystacks):
|
||||
return False
|
||||
elif not any(token in haystack for haystack in haystacks):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _build_result(self, entry: Dict[str, Any]) -> SearchResult:
|
||||
scp_path = str(entry.get("scp_path") or "/")
|
||||
scp_url = self._build_url(scp_path)
|
||||
is_dir = bool(entry.get("is_dir"))
|
||||
size_value = entry.get("size")
|
||||
modified = str(entry.get("modified") or "")
|
||||
parent = posixpath.dirname(scp_path.rstrip("/")) or "/"
|
||||
metadata = {
|
||||
"provider": "scp",
|
||||
"host": self._host,
|
||||
"scp_path": scp_path,
|
||||
"scp_url": scp_url,
|
||||
"selection_url": scp_url,
|
||||
"is_dir": is_dir,
|
||||
"name": str(entry.get("name") or "").strip(),
|
||||
}
|
||||
if size_value is not None:
|
||||
metadata["size"] = size_value
|
||||
if modified:
|
||||
metadata["modified"] = modified
|
||||
|
||||
return SearchResult(
|
||||
table="scp",
|
||||
title=str(entry.get("name") or scp_path),
|
||||
path=scp_url,
|
||||
detail=parent,
|
||||
annotations=["folder" if is_dir else "file"],
|
||||
media_kind="folder" if is_dir else "file",
|
||||
size_bytes=int(size_value) if isinstance(size_value, int) else None,
|
||||
tag={"scp", "folder" if is_dir else "file"},
|
||||
columns=[
|
||||
("Name", str(entry.get("name") or "")),
|
||||
("Type", "dir" if is_dir else "file"),
|
||||
("Directory", parent),
|
||||
("Size", "" if size_value is None else str(size_value)),
|
||||
("Modified", modified),
|
||||
],
|
||||
selection_args=None if is_dir else ["-url", scp_url],
|
||||
selection_action=None if is_dir else ["download-file", "-plugin", "scp", "-url", scp_url],
|
||||
full_metadata=metadata,
|
||||
)
|
||||
|
||||
def _list_directory(self, sftp: Any, remote_path: str) -> List[Dict[str, Any]]:
|
||||
try:
|
||||
attrs = sftp.listdir_attr(remote_path)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
entries: List[Dict[str, Any]] = []
|
||||
for attr in attrs:
|
||||
name_text = str(getattr(attr, "filename", "") or "").strip()
|
||||
if not name_text or name_text in {".", ".."}:
|
||||
continue
|
||||
mode = getattr(attr, "st_mode", 0)
|
||||
is_dir = stat.S_ISDIR(mode)
|
||||
size_value = getattr(attr, "st_size", None)
|
||||
try:
|
||||
size_int = int(size_value) if size_value is not None else None
|
||||
except Exception:
|
||||
size_int = None
|
||||
entries.append(
|
||||
{
|
||||
"name": name_text,
|
||||
"scp_path": self._join_remote_path(remote_path, name_text),
|
||||
"is_dir": is_dir,
|
||||
"size": size_int,
|
||||
"modified": _format_epoch(getattr(attr, "st_mtime", None)),
|
||||
}
|
||||
)
|
||||
return entries
|
||||
|
||||
def _list_directory_via_ssh(self, ssh: Any, remote_path: str, *, depth: int) -> List[Dict[str, Any]]:
|
||||
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
|
||||
max_depth = max(1, int(depth) + 1)
|
||||
quoted_path = shlex.quote(normalized)
|
||||
command = (
|
||||
f"find {quoted_path} -mindepth 1 -maxdepth {max_depth} "
|
||||
f"\\( -type d -o -type f \\) -exec sh -c 'for path do "
|
||||
f"if [ -d \"$path\" ]; then kind=d; else kind=f; fi; "
|
||||
f"name=$(basename \"$path\"); "
|
||||
f"printf \"%s\\0%s\\0%s\\0\" \"$kind\" \"$path\" \"$name\"; "
|
||||
f"done' sh {{}} +"
|
||||
)
|
||||
status, output, error = self._run_ssh_command(ssh, command)
|
||||
if status != 0:
|
||||
error_text = error.strip().lower()
|
||||
if "no such file" in error_text or "cannot access" in error_text:
|
||||
return []
|
||||
raise RuntimeError(error.strip() or f"SSH listing failed for {normalized}")
|
||||
|
||||
chunks = [part for part in output.split("\0") if part]
|
||||
entries: List[Dict[str, Any]] = []
|
||||
for index in range(0, len(chunks), 3):
|
||||
if index + 2 >= len(chunks):
|
||||
break
|
||||
kind = chunks[index]
|
||||
scp_path = self._normalize_remote_path(chunks[index + 1], default=normalized)
|
||||
name_text = str(chunks[index + 2] or "").strip()
|
||||
if not name_text or name_text in {".", ".."}:
|
||||
continue
|
||||
entries.append(
|
||||
{
|
||||
"name": name_text,
|
||||
"scp_path": scp_path,
|
||||
"is_dir": kind == "d",
|
||||
"size": None,
|
||||
"modified": "",
|
||||
}
|
||||
)
|
||||
return entries
|
||||
|
||||
def _ensure_directory(self, sftp: Any, remote_path: str) -> None:
|
||||
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
|
||||
if normalized == "/":
|
||||
return
|
||||
partial = ""
|
||||
for segment in [part for part in normalized.split("/") if part]:
|
||||
partial = f"{partial}/{segment}"
|
||||
try:
|
||||
attrs = sftp.stat(partial)
|
||||
if stat.S_ISDIR(getattr(attrs, "st_mode", 0)):
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
sftp.mkdir(partial)
|
||||
except Exception:
|
||||
try:
|
||||
attrs = sftp.stat(partial)
|
||||
if stat.S_ISDIR(getattr(attrs, "st_mode", 0)):
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
def _item_metadata(self, item: Any, *, pipe_obj: Any = None) -> Dict[str, Any]:
|
||||
metadata: Dict[str, Any] = {}
|
||||
for source in (item, pipe_obj):
|
||||
if isinstance(source, dict):
|
||||
for key in ("title", "path", "url"):
|
||||
if source.get(key) is not None and key not in metadata:
|
||||
metadata[key] = source.get(key)
|
||||
nested = source.get("full_metadata") or source.get("metadata")
|
||||
if isinstance(nested, dict):
|
||||
metadata.update(nested)
|
||||
elif source is not None:
|
||||
for attr in ("title", "path", "url"):
|
||||
try:
|
||||
value = getattr(source, attr, None)
|
||||
except Exception:
|
||||
value = None
|
||||
if value is not None and attr not in metadata:
|
||||
metadata[attr] = value
|
||||
try:
|
||||
nested = getattr(source, "full_metadata", None) or getattr(source, "metadata", None)
|
||||
except Exception:
|
||||
nested = None
|
||||
if isinstance(nested, dict):
|
||||
metadata.update(nested)
|
||||
|
||||
scp_path = metadata.get("scp_path") or metadata.get("selection_path")
|
||||
if not scp_path:
|
||||
path_value = metadata.get("path") or metadata.get("url") or metadata.get("scp_url")
|
||||
path_text = str(path_value or "").strip()
|
||||
if path_text.startswith(("scp://", "sftp://")):
|
||||
scp_path = self._normalize_remote_path(path_text, default=self._base_path)
|
||||
if scp_path:
|
||||
metadata["scp_path"] = self._normalize_remote_path(scp_path, default=self._base_path)
|
||||
metadata.setdefault("selection_path", metadata["scp_path"])
|
||||
|
||||
if metadata.get("scp_path") and not metadata.get("scp_url"):
|
||||
metadata["scp_url"] = self._build_url(metadata["scp_path"])
|
||||
if metadata.get("scp_url") and not metadata.get("selection_url"):
|
||||
metadata["selection_url"] = metadata["scp_url"]
|
||||
|
||||
is_dir = metadata.get("is_dir")
|
||||
if is_dir is None and metadata.get("media_kind"):
|
||||
is_dir = str(metadata.get("media_kind") or "").strip().lower() == "folder"
|
||||
metadata["is_dir"] = bool(is_dir)
|
||||
return metadata
|
||||
@@ -12,10 +12,18 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from ProviderCore.base import Provider, SearchResult
|
||||
from SYS.logger import log, debug
|
||||
from SYS.logger import log, debug, debug_panel
|
||||
from SYS.models import ProgressBar
|
||||
|
||||
_SOULSEEK_NOISE_SUBSTRINGS = (
|
||||
"unhandled exception on loop",
|
||||
"Task exception was never retrieved",
|
||||
"future: <Task finished",
|
||||
"ConnectionFailedError",
|
||||
"PeerConnectionError",
|
||||
"indirect connection failed",
|
||||
"indirect connection timed out",
|
||||
"failed to connect",
|
||||
"search reply ticket does not match any search request",
|
||||
"failed to receive transfer ticket on file connection",
|
||||
"aioslsk.exceptions.ConnectionReadError",
|
||||
@@ -59,10 +67,10 @@ async def _suppress_aioslsk_asyncio_task_noise() -> Any:
|
||||
if msg == "Task exception was never retrieved":
|
||||
cls = getattr(exc, "__class__", None)
|
||||
name = getattr(cls, "__name__", "")
|
||||
mod = getattr(cls, "__module__", "")
|
||||
exc_text = str(exc or "").lower()
|
||||
|
||||
# Suppress ConnectionFailedError from aioslsk
|
||||
if name == "ConnectionFailedError" and str(mod).startswith("aioslsk"):
|
||||
# Suppress expected peer direct-connect failures from aioslsk.
|
||||
if name == "ConnectionFailedError" or "failed to connect" in exc_text:
|
||||
return
|
||||
except Exception:
|
||||
# If our filter logic fails, fall through to default handling.
|
||||
@@ -117,6 +125,9 @@ class _LineFilterStream(io.TextIOBase):
|
||||
self._in_tb = False
|
||||
self._tb_lines: list[str] = []
|
||||
self._tb_suppress = False
|
||||
self._in_task_block = False
|
||||
self._task_lines: list[str] = []
|
||||
self._task_suppress = False
|
||||
|
||||
def writable(self) -> bool: # pragma: no cover
|
||||
return True
|
||||
@@ -137,6 +148,19 @@ class _LineFilterStream(io.TextIOBase):
|
||||
self._tb_suppress = False
|
||||
self._in_tb = False
|
||||
|
||||
def _flush_task_block(self) -> None:
|
||||
if not self._task_lines:
|
||||
return
|
||||
if not self._task_suppress:
|
||||
for l in self._task_lines:
|
||||
try:
|
||||
self._underlying.write(l + "\n")
|
||||
except Exception:
|
||||
pass
|
||||
self._task_lines = []
|
||||
self._task_suppress = False
|
||||
self._in_task_block = False
|
||||
|
||||
def write(self, s: str) -> int:
|
||||
self._buf += str(s)
|
||||
while "\n" in self._buf:
|
||||
@@ -145,6 +169,29 @@ class _LineFilterStream(io.TextIOBase):
|
||||
return len(s)
|
||||
|
||||
def _handle_line(self, line: str) -> None:
|
||||
if not self._in_task_block and self._should_suppress_line(line) and (
|
||||
line.startswith("Task exception was never retrieved")
|
||||
or line.startswith("future: <Task finished")
|
||||
or line.startswith("unhandled exception on loop")
|
||||
):
|
||||
self._in_task_block = True
|
||||
self._task_lines = [line]
|
||||
self._task_suppress = True
|
||||
return
|
||||
|
||||
if self._in_task_block:
|
||||
if line.startswith("Traceback (most recent call last):"):
|
||||
self._in_tb = True
|
||||
self._tb_lines = [line]
|
||||
self._tb_suppress = True
|
||||
return
|
||||
self._task_lines.append(line)
|
||||
if self._should_suppress_line(line):
|
||||
self._task_suppress = True
|
||||
if line.strip() == "":
|
||||
self._flush_task_block()
|
||||
return
|
||||
|
||||
# Start capturing tracebacks so we can suppress the whole block if it matches.
|
||||
if not self._in_tb and line.startswith("Traceback (most recent call last):"):
|
||||
self._in_tb = True
|
||||
@@ -159,6 +206,8 @@ class _LineFilterStream(io.TextIOBase):
|
||||
# End traceback block on blank line.
|
||||
if line.strip() == "":
|
||||
self._flush_tb()
|
||||
if self._in_task_block:
|
||||
self._flush_task_block()
|
||||
return
|
||||
|
||||
# Non-traceback line
|
||||
@@ -174,6 +223,8 @@ class _LineFilterStream(io.TextIOBase):
|
||||
if self._in_tb:
|
||||
# If the traceback ends without a trailing blank line, decide here.
|
||||
self._flush_tb()
|
||||
if self._in_task_block:
|
||||
self._flush_task_block()
|
||||
if self._buf:
|
||||
line = self._buf
|
||||
self._buf = ""
|
||||
@@ -422,14 +473,14 @@ class Soulseek(Provider):
|
||||
|
||||
try:
|
||||
search_request = await client.searches.search(query)
|
||||
await self._collect_results(search_request, timeout=timeout)
|
||||
return self._flatten_results(search_request)[:limit]
|
||||
summary = await self._collect_results(search_request, timeout=timeout)
|
||||
return self._flatten_results(search_request)[:limit], summary
|
||||
except Exception as exc:
|
||||
log(
|
||||
f"[soulseek] Search error: {type(exc).__name__}: {exc}",
|
||||
file=sys.stderr
|
||||
)
|
||||
return []
|
||||
return [], {}
|
||||
finally:
|
||||
# Best-effort: try to cancel/close the search request before stopping
|
||||
# the client to reduce stray reply spam.
|
||||
@@ -477,16 +528,24 @@ class Soulseek(Provider):
|
||||
self,
|
||||
search_request: Any,
|
||||
timeout: float = 75.0
|
||||
) -> None:
|
||||
end = time.time() + timeout
|
||||
) -> Dict[str, Any]:
|
||||
start = time.time()
|
||||
end = start + timeout
|
||||
last_count = 0
|
||||
update_count = 0
|
||||
while time.time() < end:
|
||||
current_count = len(getattr(search_request, "results", []))
|
||||
if current_count > last_count:
|
||||
debug(f"[soulseek] Got {current_count} result(s)...")
|
||||
last_count = current_count
|
||||
update_count += 1
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
return {
|
||||
"peer_hits": last_count,
|
||||
"count_updates": update_count,
|
||||
"elapsed_seconds": round(max(0.0, time.time() - start), 1),
|
||||
}
|
||||
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
@@ -503,7 +562,7 @@ class Soulseek(Provider):
|
||||
base_tmp.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
flat_results = asyncio.run(
|
||||
flat_results, search_summary = asyncio.run(
|
||||
self.perform_search(query,
|
||||
timeout=9.0,
|
||||
limit=limit)
|
||||
@@ -636,6 +695,23 @@ class Soulseek(Provider):
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
debug_panel(
|
||||
"soulseek search",
|
||||
[
|
||||
("query", query),
|
||||
("peer_hits", search_summary.get("peer_hits", 0)),
|
||||
("file_hits", len(flat_results)),
|
||||
("audio_hits", len(music_results)),
|
||||
("results", len(results)),
|
||||
("poll_updates", search_summary.get("count_updates", 0)),
|
||||
("elapsed_s", search_summary.get("elapsed_seconds", 0.0)),
|
||||
],
|
||||
border_style="magenta",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return results
|
||||
|
||||
except Exception as exc:
|
||||
|
||||
@@ -1286,9 +1286,30 @@ class ytdlp(TableProviderMixin, Provider):
|
||||
|
||||
|
||||
try:
|
||||
from SYS.result_table_adapters import register_plugin
|
||||
from SYS.result_table_adapters import get_plugin, register_plugin
|
||||
from SYS.result_table_api import ResultModel, ColumnSpec, metadata_column, title_column
|
||||
|
||||
def _register_table_plugin_once(
|
||||
name: str,
|
||||
adapter: Any,
|
||||
*,
|
||||
columns: Any,
|
||||
selection_fn: Any,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
) -> None:
|
||||
try:
|
||||
get_plugin(name)
|
||||
return
|
||||
except KeyError:
|
||||
pass
|
||||
register_plugin(
|
||||
name,
|
||||
adapter,
|
||||
columns=columns,
|
||||
selection_fn=selection_fn,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
def _convert_format_result_to_model(sr: Any) -> ResultModel:
|
||||
d = sr.to_dict() if hasattr(sr, "to_dict") else (sr if isinstance(sr, dict) else {})
|
||||
title = d.get("title") or f"Format {d.get('format_id', 'unknown')}"
|
||||
@@ -1361,7 +1382,7 @@ try:
|
||||
return result_args
|
||||
return []
|
||||
|
||||
register_plugin(
|
||||
_register_table_plugin_once(
|
||||
"ytdlp.formatlist",
|
||||
_adapter,
|
||||
columns=_columns_factory,
|
||||
@@ -1421,7 +1442,7 @@ try:
|
||||
return ["-url", row.path]
|
||||
return ["-title", row.title or ""]
|
||||
|
||||
register_plugin(
|
||||
_register_table_plugin_once(
|
||||
"ytdlp.search",
|
||||
_search_adapter,
|
||||
columns=_search_columns_factory,
|
||||
|
||||
@@ -70,6 +70,8 @@ dependencies = [
|
||||
|
||||
# Browser automation
|
||||
"playwright>=1.40.0",
|
||||
"paramiko>=3.5.0",
|
||||
"scp>=0.15.0",
|
||||
|
||||
# Development and utilities
|
||||
"python-dateutil>=2.8.0",
|
||||
|
||||
Reference in New Issue
Block a user