goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

updating and refining plugin system refactor

Browse Source
This commit is contained in:
Nose
2026-04-28 22:20:54 -07:00
parent 8685fbb723
commit 323c24f4f4
33 changed files with 4287 additions and 3312 deletions
Download Patch File Download Diff File Expand all files Collapse all files
plugins/ftp/__init__.py
+252 -83
View File
@@ -12,18 +12,6 @@ from urllib.parse import quote, unquote, urlparse
from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments
def _pick_provider_config(config: Any) -> Dict[str, Any]:
if not isinstance(config, dict):
return {}
provider = config.get("provider")
if not isinstance(provider, dict):
return {}
entry = provider.get("ftp")
if isinstance(entry, dict):
return entry
return {}
def _coerce_bool(value: Any, default: bool = False) -> bool:
if isinstance(value, bool):
return value
@@ -155,20 +143,54 @@ class FTP(Provider):
def __init__(self, config: Optional[Dict[str, Any]] = None):
super().__init__(config)
conf = _pick_provider_config(self.config)
self._host = str(conf.get("host") or "").strip()
self._tls = _coerce_bool(conf.get("tls"), False)
self._port = _coerce_int(conf.get("port"), 21)
self._username = str(conf.get("username") or conf.get("user") or "anonymous").strip() or "anonymous"
password_value = conf.get("password")
self._password = str(password_value).strip() if password_value not in (None, "") else "anonymous@"
self._passive = _coerce_bool(conf.get("passive"), True)
self._timeout = max(1, _coerce_int(conf.get("timeout"), 20))
self._search_depth = max(0, _coerce_int(conf.get("search_depth"), 1))
self._base_path = self._normalize_remote_path(conf.get("base_path") or "/", default="/")
_instance_name, conf = self.resolve_plugin_instance()
defaults = self._settings_from_config(conf)
self._host = str(defaults.get("host") or "").strip()
self._tls = bool(defaults.get("tls"))
self._port = int(defaults.get("port") or 21)
self._username = str(defaults.get("username") or "anonymous").strip() or "anonymous"
self._password = str(defaults.get("password") or "anonymous@").strip() or "anonymous@"
self._passive = bool(defaults.get("passive"))
self._timeout = max(1, int(defaults.get("timeout") or 20))
self._search_depth = max(0, int(defaults.get("search_depth") or 1))
self._base_path = self._normalize_remote_path(defaults.get("base_path") or "/", default="/")
def _settings_from_config(self, conf: Optional[Dict[str, Any]], *, instance_name: Optional[str] = None) -> Dict[str, Any]:
entry = dict(conf or {})
password_value = entry.get("password")
return {
"instance": str(instance_name or entry.get("_instance_name") or "").strip() or None,
"host": str(entry.get("host") or "").strip(),
"tls": _coerce_bool(entry.get("tls"), False),
"port": _coerce_int(entry.get("port"), 21),
"username": str(entry.get("username") or entry.get("user") or "anonymous").strip() or "anonymous",
"password": str(password_value).strip() if password_value not in (None, "") else "anonymous@",
"passive": _coerce_bool(entry.get("passive"), True),
"timeout": max(1, _coerce_int(entry.get("timeout"), 20)),
"search_depth": max(0, _coerce_int(entry.get("search_depth"), 1)),
"base_path": self._normalize_remote_path(entry.get("base_path") or "/", default="/"),
}
def _resolve_settings(
self,
*,
filters: Optional[Dict[str, Any]] = None,
instance_name: Optional[str] = None,
require_explicit: bool = False,
) -> Dict[str, Any]:
requested = self.requested_instance_name(filters, instance=instance_name)
resolved_name, conf = self.resolve_plugin_instance(
requested,
require_explicit=require_explicit or bool(requested),
)
settings = self._settings_from_config(conf, instance_name=resolved_name)
if settings.get("instance") is None and requested:
settings["instance"] = requested
return settings
def validate(self) -> bool:
return bool(self._host)
settings = self._resolve_settings()
return bool(settings.get("host"))
def config_helper_text(self) -> str:
return "Test the configured FTP/FTPS settings before searching or uploading."
@@ -186,13 +208,14 @@ class FTP(Provider):
if str(action_id or "").strip().lower() != "test_connection":
return super().run_config_action(action_id, **_kwargs)
if not self._host:
settings = self._resolve_settings()
if not settings.get("host"):
return {"ok": False, "message": "Set 'host' before testing the FTP connection."}
ftp = None
try:
ftp = self._connect()
active_path = self._base_path or "/"
ftp = self._connect(settings=settings)
active_path = str(settings.get("base_path") or "/")
try:
ftp.cwd(active_path)
resolved_path = ftp.pwd()
@@ -200,7 +223,7 @@ class FTP(Provider):
resolved_path = active_path
return {
"ok": True,
"message": f"Connected to FTP {self._host}:{self._port} and reached {resolved_path}.",
"message": f"Connected to FTP {settings.get('host')}:{settings.get('port')} and reached {resolved_path}.",
}
except Exception as exc:
return {"ok": False, "message": f"FTP connection failed: {exc}"}
@@ -211,6 +234,10 @@ class FTP(Provider):
text, inline = parse_inline_query_arguments(query)
filters: Dict[str, Any] = {}
instance_name = str(inline.get("instance") or inline.get("store") or "").strip()
if instance_name:
filters["instance"] = instance_name
if inline.get("path"):
filters["path"] = inline.get("path")
if inline.get("depth"):
@@ -221,17 +248,21 @@ class FTP(Provider):
return text, filters
def get_table_title(self, query: str, filters: Optional[Dict[str, Any]] = None) -> str:
active_path = self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path)
settings = self._resolve_settings(filters=filters)
active_path = self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/"))
instance_name = str(settings.get("instance") or "").strip()
text = str(query or "").strip()
if not text or text == "*":
return f"FTP: {active_path}"
return f"FTP: {text} @ {active_path}"
return f"FTP{f'[{instance_name}]' if instance_name else ''}: {active_path}"
return f"FTP{f'[{instance_name}]' if instance_name else ''}: {text} @ {active_path}"
def get_table_metadata(self, query: str, filters: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
settings = self._resolve_settings(filters=filters)
return {
"plugin": self.name,
"host": self._host,
"path": self._normalize_remote_path((filters or {}).get("path") or self._base_path, default=self._base_path),
"instance": settings.get("instance"),
"host": settings.get("host"),
"path": self._normalize_remote_path((filters or {}).get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/")),
"query": str(query or "").strip(),
}
@@ -244,15 +275,21 @@ class FTP(Provider):
) -> List[SearchResult]:
_ = kwargs
active_filters = dict(filters or {})
start_path = self._normalize_remote_path(active_filters.get("path") or self._base_path, default=self._base_path)
search_depth = max(0, _coerce_int(active_filters.get("depth"), self._search_depth))
settings = self._resolve_settings(filters=active_filters, require_explicit=True)
if not settings.get("host"):
requested = self.requested_instance_name(active_filters)
if requested:
raise RuntimeError(f"FTP instance '{requested}' is unavailable")
return []
start_path = self._normalize_remote_path(active_filters.get("path") or settings.get("base_path") or "/", default=str(settings.get("base_path") or "/"))
search_depth = max(0, _coerce_int(active_filters.get("depth"), int(settings.get("search_depth") or self._search_depth)))
type_filter = str(active_filters.get("type") or "any").strip().lower()
needle = str(query or "").strip()
max_results = max(0, int(limit or 0))
if max_results <= 0:
return []
ftp = self._connect()
ftp = self._connect(settings=settings)
try:
return self._search_directory(
ftp,
@@ -261,6 +298,7 @@ class FTP(Provider):
limit=max_results,
search_depth=search_depth,
type_filter=type_filter,
settings=settings,
)
finally:
self._close(ftp)
@@ -278,19 +316,23 @@ class FTP(Provider):
target_path = ""
target_title = ""
instance_name = ""
for item in selected_items or []:
metadata = self._item_metadata(item)
if not metadata.get("is_dir"):
continue
target_path = self._normalize_remote_path(metadata.get("ftp_path") or metadata.get("selection_path"), default=self._base_path)
settings = self._resolve_settings(instance_name=str(metadata.get("instance") or "").strip() or None, require_explicit=bool(metadata.get("instance")))
target_path = self._normalize_remote_path(metadata.get("ftp_path") or metadata.get("selection_path"), default=str(settings.get("base_path") or "/"))
target_title = str(metadata.get("title") or metadata.get("name") or "").strip()
instance_name = str(settings.get("instance") or metadata.get("instance") or "").strip()
if target_path:
break
if not target_path:
return False
ftp = self._connect()
settings = self._resolve_settings(instance_name=instance_name or None, require_explicit=bool(instance_name))
ftp = self._connect(settings=settings)
try:
rows = self._search_directory(
ftp,
@@ -299,6 +341,7 @@ class FTP(Provider):
limit=500,
search_depth=0,
type_filter="any",
settings=settings,
)
finally:
self._close(ftp)
@@ -310,18 +353,23 @@ class FTP(Provider):
return True
title = target_title or target_path
table = Table(f"FTP: {title}")._perseverance(True)
table = Table(f"FTP{f'[{instance_name}]' if instance_name else ''}: {title}")._perseverance(True)
table.set_table("ftp")
try:
table.set_table_metadata({
"provider": "ftp",
"host": self._host,
"instance": instance_name or None,
"host": settings.get("host"),
"path": target_path,
"view": "directory",
})
except Exception:
pass
table.set_source_command("search-file", ["-plugin", "ftp", f"path:{target_path}", "*"])
source_args = ["-plugin", "ftp"]
if instance_name:
source_args.extend(["-instance", instance_name])
source_args.extend([f"path:{target_path}", "*"])
table.set_source_command("search-file", source_args)
payloads: List[Dict[str, Any]] = []
for row in rows:
@@ -329,7 +377,7 @@ class FTP(Provider):
payloads.append(row.to_dict())
try:
ctx.set_last_result_table(table, payloads, subject={"plugin": "ftp", "path": target_path})
ctx.set_last_result_table(table, payloads, subject={"plugin": "ftp", "instance": instance_name or None, "path": target_path})
ctx.set_current_stage_table(table)
except Exception:
pass
@@ -342,6 +390,77 @@ class FTP(Provider):
return True
def show_selection_details(
self,
selected_items: List[Any],
*,
ctx: Any,
stage_is_last: bool = True,
source_command: str = "",
table_type: str = "",
table_metadata: Optional[Dict[str, Any]] = None,
**_kwargs: Any,
) -> bool:
_ = table_type
item, _payload, _meta = self.resolve_selection_detail_subject(
selected_items,
stage_is_last=stage_is_last,
source_command=source_command,
require_media_kind="file",
)
if item is None:
return False
metadata = self._item_metadata(item)
if bool(metadata.get("is_dir")):
return False
title = str(metadata.get("title") or metadata.get("name") or metadata.get("path") or "").strip() or "FTP Item"
instance_name = str(metadata.get("instance") or (table_metadata or {}).get("instance") or "").strip()
ftp_url = str(metadata.get("ftp_url") or metadata.get("selection_url") or metadata.get("path") or "").strip()
remote_path = str(metadata.get("ftp_path") or "").strip()
host = str(metadata.get("host") or "").strip()
modified = str(metadata.get("modified") or "").strip()
try:
from SYS.detail_view_helpers import prepare_detail_metadata, render_selection_detail_view
except Exception:
return super().show_selection_details(
selected_items,
ctx=ctx,
stage_is_last=stage_is_last,
source_command=source_command,
table_type=table_type,
table_metadata=table_metadata,
)
detail_metadata = prepare_detail_metadata(
item,
title=title,
store=instance_name or self.name,
path=ftp_url or remote_path or None,
tags=metadata.get("tag") or metadata.get("tags"),
extra_fields={
"Plugin": self.name,
"Host": host or None,
"Instance": instance_name or None,
"Remote Path": remote_path or None,
"Directory": str(metadata.get("detail") or "").strip() or None,
"Modified": modified or None,
"Ftp Url": ftp_url or None,
},
)
return render_selection_detail_view(
ctx=ctx,
item=item,
title=f"FTP Item: {title}",
metadata=detail_metadata,
table_name=self.name,
detail_order=["Title", "Store", "Host", "Instance", "Remote Path", "Directory", "Modified", "Path", "Ext", "FTP URL", "Plugin"],
value_case="preserve",
)
def download(self, result: SearchResult, output_dir: Path) -> Optional[Path]:
metadata = getattr(result, "full_metadata", None)
if isinstance(metadata, dict) and metadata.get("is_dir"):
@@ -349,10 +468,15 @@ class FTP(Provider):
target = str(getattr(result, "path", "") or "").strip()
if not target:
return None
return self.download_url(target, output_dir, title=getattr(result, "title", None))
instance_name = str(metadata.get("instance") or "").strip() if isinstance(metadata, dict) else ""
return self.download_url(target, output_dir, title=getattr(result, "title", None), instance=instance_name or None)
def download_url(self, url: str, output_dir: Path, **kwargs: Any) -> Optional[Path]:
settings = self._connection_settings_for_url(url)
parsed = kwargs.get("parsed") if isinstance(kwargs.get("parsed"), dict) else {}
settings = self._connection_settings_for_url(
url,
instance_name=str(kwargs.get("instance") or parsed.get("instance") or "").strip() or None,
)
remote_path = settings["path"]
if not remote_path or remote_path == "/":
return None
@@ -365,13 +489,7 @@ class FTP(Provider):
destination_dir.mkdir(parents=True, exist_ok=True)
destination = _unique_path(destination_dir / filename)
ftp = self._connect(
host=settings["host"],
port=settings["port"],
username=settings["username"],
password=settings["password"],
tls=settings["tls"],
)
ftp = self._connect(settings=settings)
try:
with destination.open("wb") as handle:
ftp.retrbinary(f"RETR {remote_path}", handle.write)
@@ -404,7 +522,12 @@ class FTP(Provider):
return None, None, None
temp_dir = Path(tempfile.mkdtemp(prefix="ftp-add-file-"))
downloaded = self.download_url(download_url, temp_dir, title=metadata.get("title"))
downloaded = self.download_url(
download_url,
temp_dir,
title=metadata.get("title"),
instance=metadata.get("instance"),
)
if downloaded is None:
try:
temp_dir.rmdir()
@@ -424,35 +547,57 @@ class FTP(Provider):
if not local_path.exists() or not local_path.is_file():
raise FileNotFoundError(f"File not found: {local_path}")
remote_dir = self._normalize_remote_path(kwargs.get("remote_path") or kwargs.get("path") or self._base_path, default=self._base_path)
settings = self._resolve_settings(
instance_name=str(kwargs.get("instance") or kwargs.get("store") or "").strip() or None,
require_explicit=bool(kwargs.get("instance") or kwargs.get("store")),
)
if not settings.get("host"):
requested = str(kwargs.get("instance") or kwargs.get("store") or "").strip()
if requested:
raise RuntimeError(f"FTP instance '{requested}' is unavailable")
raise RuntimeError("No configured FTP instance is available")
remote_dir = self._normalize_remote_path(
kwargs.get("remote_path") or kwargs.get("path") or settings.get("base_path") or "/",
default=str(settings.get("base_path") or "/"),
)
remote_name = posixpath.basename(str(kwargs.get("remote_name") or local_path.name).replace("\\", "/")) or local_path.name
remote_path = self._join_remote_path(remote_dir, remote_name)
ftp = self._connect()
ftp = self._connect(settings=settings)
try:
self._ensure_directory(ftp, remote_dir)
self._ensure_directory(ftp, remote_dir, base_path=str(settings.get("base_path") or "/"))
with local_path.open("rb") as handle:
ftp.storbinary(f"STOR {remote_path}", handle)
finally:
self._close(ftp)
return self._build_url(remote_path)
return self._build_url(remote_path, settings=settings)
def _connect(
self,
*,
settings: Optional[Dict[str, Any]] = None,
host: Optional[str] = None,
port: Optional[int] = None,
username: Optional[str] = None,
password: Optional[str] = None,
tls: Optional[bool] = None,
) -> ftplib.FTP:
use_tls = self._tls if tls is None else bool(tls)
resolved = dict(settings or {})
use_tls = bool(resolved.get("tls")) if tls is None else bool(tls)
ftp: ftplib.FTP = ftplib.FTP_TLS() if use_tls else ftplib.FTP()
ftp.connect(host or self._host, int(port or self._port), timeout=self._timeout)
ftp.login(username or self._username, password or self._password)
ftp.connect(
host or str(resolved.get("host") or self._host),
int(port or resolved.get("port") or self._port),
timeout=max(1, int(resolved.get("timeout") or self._timeout)),
)
ftp.login(
username or str(resolved.get("username") or self._username),
password or str(resolved.get("password") or self._password),
)
try:
ftp.set_pasv(self._passive)
ftp.set_pasv(bool(resolved.get("passive")) if "passive" in resolved else self._passive)
except Exception:
pass
if use_tls and isinstance(ftp, ftplib.FTP_TLS):
@@ -497,32 +642,39 @@ class FTP(Provider):
self,
remote_path: Any,
*,
settings: Optional[Dict[str, Any]] = None,
host: Optional[str] = None,
port: Optional[int] = None,
tls: Optional[bool] = None,
) -> str:
resolved = dict(settings or {})
path_text = self._normalize_remote_path(remote_path, default="/")
scheme = "ftps" if (self._tls if tls is None else bool(tls)) else "ftp"
host_text = str(host or self._host).strip()
port_value = int(port or self._port)
scheme = "ftps" if ((bool(resolved.get("tls")) if tls is None else bool(tls))) else "ftp"
host_text = str(host or resolved.get("host") or self._host).strip()
port_value = int(port or resolved.get("port") or self._port)
port_suffix = f":{port_value}" if port_value and port_value != 21 else ""
return f"{scheme}://{host_text}{port_suffix}{quote(path_text, safe='/-._~!$&\'()*+,;=:@')}"
def _connection_settings_for_url(self, url: str) -> Dict[str, Any]:
def _connection_settings_for_url(self, url: str, *, instance_name: Optional[str] = None) -> Dict[str, Any]:
settings = self._resolve_settings(instance_name=instance_name, require_explicit=bool(instance_name))
parsed = urlparse(str(url or "").strip())
scheme = (parsed.scheme or "ftp").strip().lower()
host = parsed.hostname or self._host
port = parsed.port or self._port
username = parsed.username or self._username
password = parsed.password or self._password
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default="/")
host = parsed.hostname or settings.get("host") or self._host
port = parsed.port or settings.get("port") or self._port
username = parsed.username or settings.get("username") or self._username
password = parsed.password or settings.get("password") or self._password
path_text = self._normalize_remote_path(unquote(parsed.path or "/"), default=str(settings.get("base_path") or "/"))
return {
"instance": settings.get("instance"),
"tls": scheme == "ftps",
"host": host,
"port": port,
"username": username,
"password": password,
"path": path_text,
"passive": settings.get("passive", self._passive),
"timeout": settings.get("timeout", self._timeout),
"base_path": settings.get("base_path", self._base_path),
}
def _search_directory(
@@ -534,21 +686,22 @@ class FTP(Provider):
limit: int,
search_depth: int,
type_filter: str,
settings: Dict[str, Any],
) -> List[SearchResult]:
results: List[SearchResult] = []
visited: set[str] = set()
def walk(current_path: str, depth_left: int) -> None:
normalized = self._normalize_remote_path(current_path, default=self._base_path)
normalized = self._normalize_remote_path(current_path, default=str(settings.get("base_path") or self._base_path))
if normalized in visited or len(results) >= limit:
return
visited.add(normalized)
for entry in self._list_directory(ftp, normalized):
for entry in self._list_directory(ftp, normalized, base_path=str(settings.get("base_path") or self._base_path)):
if len(results) >= limit:
return
if self._matches_entry(entry, needle=needle, type_filter=type_filter):
results.append(self._build_result(entry))
results.append(self._build_result(entry, settings=settings))
if entry.get("is_dir") and depth_left > 0:
walk(str(entry.get("ftp_path") or normalized), depth_left - 1)
@@ -578,16 +731,18 @@ class FTP(Provider):
return False
return True
def _build_result(self, entry: Dict[str, Any]) -> SearchResult:
def _build_result(self, entry: Dict[str, Any], *, settings: Dict[str, Any]) -> SearchResult:
ftp_path = str(entry.get("ftp_path") or "/")
ftp_url = self._build_url(ftp_path)
ftp_url = self._build_url(ftp_path, settings=settings)
is_dir = bool(entry.get("is_dir"))
size_value = entry.get("size")
modified = str(entry.get("modified") or "")
parent = posixpath.dirname(ftp_path.rstrip("/")) or "/"
instance_name = str(settings.get("instance") or "").strip()
metadata = {
"provider": "ftp",
"host": self._host,
"instance": instance_name or None,
"host": settings.get("host"),
"ftp_path": ftp_path,
"ftp_url": ftp_url,
"selection_url": ftp_url,
@@ -599,6 +754,13 @@ class FTP(Provider):
if modified:
metadata["modified"] = modified
selection_args = ["-url", ftp_url]
selection_action = ["download-file", "-plugin", "ftp"]
if instance_name:
selection_args = ["-instance", instance_name, *selection_args]
selection_action.extend(["-instance", instance_name])
selection_action.extend(["-url", ftp_url])
return SearchResult(
table="ftp",
title=str(entry.get("name") or ftp_path),
@@ -615,13 +777,13 @@ class FTP(Provider):
("Size", "" if size_value is None else str(size_value)),
("Modified", modified),
],
selection_args=None if is_dir else ["-url", ftp_url],
selection_action=None if is_dir else ["download-file", "-plugin", "ftp", "-url", ftp_url],
selection_args=None if is_dir else selection_args,
selection_action=None if is_dir else selection_action,
full_metadata=metadata,
)
def _list_directory(self, ftp: ftplib.FTP, remote_path: str) -> List[Dict[str, Any]]:
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
def _list_directory(self, ftp: ftplib.FTP, remote_path: str, *, base_path: str) -> List[Dict[str, Any]]:
normalized = self._normalize_remote_path(remote_path, default=base_path)
try:
entries: List[Dict[str, Any]] = []
for name, facts in ftp.mlsd(normalized):
@@ -716,8 +878,8 @@ class FTP(Provider):
return _format_timestamp(parts[1])
return ""
def _ensure_directory(self, ftp: ftplib.FTP, remote_path: str) -> None:
normalized = self._normalize_remote_path(remote_path, default=self._base_path)
def _ensure_directory(self, ftp: ftplib.FTP, remote_path: str, *, base_path: str) -> None:
normalized = self._normalize_remote_path(remote_path, default=base_path)
if normalized == "/":
return
partial = ""
@@ -763,11 +925,18 @@ class FTP(Provider):
if path_text.startswith(("ftp://", "ftps://")):
ftp_path = self._normalize_remote_path(path_text, default=self._base_path)
if ftp_path:
metadata["ftp_path"] = self._normalize_remote_path(ftp_path, default=self._base_path)
base_path = str(metadata.get("base_path") or self._base_path)
metadata["ftp_path"] = self._normalize_remote_path(ftp_path, default=base_path)
metadata.setdefault("selection_path", metadata["ftp_path"])
if metadata.get("ftp_path") and not metadata.get("ftp_url"):
metadata["ftp_url"] = self._build_url(metadata["ftp_path"])
metadata["ftp_url"] = self._build_url(
metadata["ftp_path"],
settings={
"host": metadata.get("host") or self._host,
"instance": metadata.get("instance"),
},
)
if metadata.get("ftp_url") and not metadata.get("selection_url"):
metadata["selection_url"] = metadata["ftp_url"]