updated plugin refactor and added FTP and SCP plugins , also hydrusnetwork plugin migration

cmdlet/_shared.py

@@ -1410,7 +1410,7 @@ def fetch_hydrus_metadata(
     Eliminates repeated boilerplate: client initialization, error handling, metadata extraction.
 
     Args:
-            config: Configuration object (passed to hydrus_wrapper.get_client)
+            config: Configuration object used to resolve the Hydrus provider/store
             hash_hex: File hash to fetch metadata for
             store_name: Optional Hydrus store name. When provided, do not fall back to a global/default Hydrus client.
             hydrus_client: Optional explicit Hydrus client. When provided, takes precedence.
@@ -1422,38 +1422,53 @@ def fetch_hydrus_metadata(
             - metadata_dict: Dict from Hydrus (first item in metadata list) or None if unavailable
             - error_code: 0 on success, 1 on any error (suitable for returning from cmdlet execute())
     """
-    from API import HydrusNetwork
-
-    hydrus_wrapper = HydrusNetwork
-
     client = hydrus_client
+    hydrus_provider = None
+    try:
+        from ProviderCore.registry import get_plugin
+
+        hydrus_provider = get_plugin("hydrusnetwork", config)
+    except Exception:
+        hydrus_provider = None
+
     if client is None:
-        if store_name:
-            # Store specified: do not fall back to a global/default Hydrus client.
+        if hydrus_provider is not None:
             try:
-                from Store import Store
-
-                store = Store(config)
-                backend = store[str(store_name)]
-                candidate = getattr(backend, "_client", None)
-                if candidate is not None and hasattr(candidate, "fetch_file_metadata"):
-                    client = candidate
+                client = hydrus_provider.get_client(
+                    store_name=store_name if store_name else None,
+                    allow_default=not bool(store_name),
+                )
             except Exception as exc:
-                log(f"Hydrus client unavailable for store '{store_name}': {exc}")
+                if store_name:
+                    log(f"Hydrus client unavailable for store '{store_name}': {exc}")
+                else:
+                    log(f"Hydrus client unavailable: {exc}")
                 client = None
-            if client is None:
-                log(f"Hydrus client unavailable for store '{store_name}'")
-                return None, 1
-        else:
-            try:
-                client = hydrus_wrapper.get_client(config)
-            except Exception as exc:
-                log(f"Hydrus client unavailable: {exc}")
-                return None, 1
+        if client is None and store_name:
+            log(f"Hydrus client unavailable for store '{store_name}'")
+            return None, 1
+        if client is None and hydrus_provider is None:
+            log("Hydrus provider unavailable")
+            return None, 1
 
-            if client is None:
-                log("Hydrus client unavailable")
-                return None, 1
+    if hydrus_provider is not None:
+        try:
+            metadata = hydrus_provider.fetch_metadata(
+                hash_hex,
+                store_name=store_name if store_name else None,
+                **kwargs,
+            )
+        except Exception as exc:
+            log(f"Hydrus metadata fetch failed: {exc}")
+            return None, 1
+        if isinstance(metadata, dict):
+            return metadata, 0
+        if client is None:
+            if store_name:
+                log(f"Hydrus client unavailable for store '{store_name}'")
+            else:
+                log("Hydrus metadata unavailable")
+            return None, 1
 
     try:
         payload = client.fetch_file_metadata(hashes=[hash_hex], **kwargs)
@@ -3725,10 +3740,13 @@ def check_url_exists_in_storage(
     match_rows: List[Dict[str, Any]] = []
     max_rows = 200
 
+    hydrus_provider = None
     try:
-        from Store.HydrusNetwork import HydrusNetwork
+        from ProviderCore.registry import get_plugin
+
+        hydrus_provider = get_plugin("hydrusnetwork", config)
     except Exception:
-        HydrusNetwork = None  # type: ignore
+        hydrus_provider = None
 
     for backend_name in backend_names:
         if _timed_out("backend scan"):
@@ -3739,8 +3757,14 @@ def check_url_exists_in_storage(
             backend = storage[backend_name]
         except Exception:
             continue
-        
-        if HydrusNetwork is not None and isinstance(backend, HydrusNetwork):
+
+        is_hydrus_backend = False
+        try:
+            is_hydrus_backend = bool(hydrus_provider and hydrus_provider.is_backend(backend, str(backend_name)))
+        except Exception:
+            is_hydrus_backend = False
+
+        if is_hydrus_backend:
             if not hydrus_available:
                 debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup")
 
@@ -3776,44 +3800,6 @@ def check_url_exists_in_storage(
                         found = True
                         break
 
-                client = getattr(backend, "_client", None)
-                if found:
-                    pass
-                elif client is None:
-                    continue
-                for needle in (needles or [])[:6]:
-                    if found:
-                        break
-                    if not _httpish(needle):
-                        continue
-                    try:
-                        from API.HydrusNetwork import HydrusRequestSpec
-
-                        spec = HydrusRequestSpec(
-                            method="GET",
-                            endpoint="/add_urls/get_url_files",
-                            query={"url": needle},
-                        )
-                        if hasattr(client, "_perform_request"):
-                            response = client._perform_request(spec)
-                            raw_hashes = None
-                            if isinstance(response, dict):
-                                raw_hashes = response.get("hashes") or response.get("file_hashes")
-                                raw_ids = response.get("file_ids")
-                                hash_list = raw_hashes if isinstance(raw_hashes, list) else []
-                                has_ids = isinstance(raw_ids, list) and len(raw_ids) > 0
-                                has_hashes = len(hash_list) > 0
-                                if has_hashes:
-                                    try:
-                                        found_hash = str(hash_list[0]).strip()
-                                    except Exception:
-                                        found_hash = None
-                                if has_ids or has_hashes:
-                                    found = True
-                                    break
-                    except Exception:
-                        continue
-
                 if not found:
                     continue
 

cmdlet/add_relationship.py

@@ -9,9 +9,9 @@ import sys
 
 from SYS.logger import log
 from SYS.item_accessors import get_sha256_hex, get_store_name
+from ProviderCore.registry import get_plugin
 
 from SYS import pipeline as ctx
-from API import HydrusNetwork as hydrus_wrapper
 from . import _shared as sh
 
 Cmdlet = sh.Cmdlet
@@ -617,16 +617,20 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
     # - If no store is specified, use the default Hydrus client.
     # NOTE: When a store is specified, we do not fall back to a global/default Hydrus client.
     hydrus_client = None
+    hydrus_provider = get_plugin("hydrusnetwork", config)
     if store_name and (not is_folder_store) and backend is not None:
         try:
-            candidate = getattr(backend, "_client", None)
-            if candidate is not None and hasattr(candidate, "set_relationship"):
-                hydrus_client = candidate
+            if hydrus_provider is not None:
+                hydrus_client = hydrus_provider.get_client(
+                    store_name=str(store_name),
+                    allow_default=False,
+                )
         except Exception:
             hydrus_client = None
     elif not store_name:
         try:
-            hydrus_client = hydrus_wrapper.get_client(config)
+            if hydrus_provider is not None:
+                hydrus_client = hydrus_provider.get_client()
         except Exception:
             hydrus_client = None
 
@@ -1049,8 +1053,9 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
         return 1
 
     # Build Hydrus client
+    hydrus_provider = get_plugin("hydrusnetwork", config)
     try:
-        hydrus_client = hydrus_wrapper.get_client(config)
+        hydrus_client = hydrus_provider.get_client() if hydrus_provider is not None else None
     except Exception as exc:
         log(f"Hydrus client unavailable: {exc}", file=sys.stderr)
         return 1

cmdlet/archive_file.py

@@ -13,6 +13,7 @@ from typing import Any, Dict, List, Sequence, Set
 from urllib.parse import parse_qs, urlparse
 
 from SYS.logger import log
+from ProviderCore.registry import get_plugin
 from SYS.item_accessors import get_http_url, get_sha256_hex, get_store_name
 from SYS.utils import extract_hydrus_hash_from_url
 
@@ -71,10 +72,8 @@ def _maybe_download_hydrus_item(
 
     This is intentionally side-effect free except for writing the local temp file.
     """
-    try:
-        from SYS.config import get_hydrus_access_key, get_hydrus_url
-        from API.HydrusNetwork import HydrusNetwork as HydrusClient, download_hydrus_file
-    except Exception:
+    hydrus_provider = get_plugin("hydrusnetwork", config)
+    if hydrus_provider is None:
         return None
 
     store_name = _extract_store_name(item)
@@ -102,68 +101,10 @@ def _maybe_download_hydrus_item(
             is_hydrus_url = False
     if not (is_hydrus_url or store_hint):
         return None
-
-    # Prefer store name as instance key; fall back to "home".
-    access_key = None
-    hydrus_url = None
-    for inst in [s for s in [store_lower, "home"] if s]:
-        try:
-            access_key = (get_hydrus_access_key(config, inst) or "").strip() or None
-            hydrus_url = (get_hydrus_url(config, inst) or "").strip() or None
-            if access_key and hydrus_url:
-                break
-        except Exception:
-            access_key = None
-            hydrus_url = None
-
-    if not access_key or not hydrus_url:
-        return None
-
-    client = HydrusClient(url=hydrus_url, access_key=access_key, timeout=60.0)
-    file_url = url if (url and is_hydrus_url) else client.file_url(file_hash)
-
-    # Best-effort extension from Hydrus metadata.
-    suffix = ".hydrus"
-    try:
-        meta_response = client.fetch_file_metadata(
-            hashes=[file_hash],
-            include_mime=True
-        )
-        entries = meta_response.get("metadata"
-                                    ) if isinstance(meta_response,
-                                                    dict) else None
-        if isinstance(entries, list) and entries:
-            entry = entries[0]
-            if isinstance(entry, dict):
-                ext = entry.get("ext")
-                if isinstance(ext, str) and ext.strip():
-                    cleaned = ext.strip()
-                    if not cleaned.startswith("."):
-                        cleaned = "." + cleaned.lstrip(".")
-                    if len(cleaned) <= 12:
-                        suffix = cleaned
-    except Exception:
-        pass
-
-    try:
-        output_dir.mkdir(parents=True, exist_ok=True)
-    except Exception:
-        pass
-
-    dest = output_dir / f"{file_hash}{suffix}"
-    if dest.exists():
-        dest = output_dir / f"{file_hash}_{uuid.uuid4().hex[:10]}{suffix}"
-
-    headers = {
-        "Hydrus-Client-API-Access-Key": access_key
-    }
-    download_hydrus_file(file_url, headers, dest, timeout=60.0)
-    try:
-        if dest.exists() and dest.is_file():
-            return dest
-    except Exception:
-        return None
-    return None
+    preferred_store = store_name or None
+    if url and is_hydrus_url:
+        return hydrus_provider.download_url(url, output_dir)
+    return hydrus_provider.download_hash_to_temp(file_hash, store_name=preferred_store, temp_root=output_dir)
 
 
 def _resolve_existing_or_fetch_path(item: Any,

cmdlet/delete_file.py

@@ -7,9 +7,9 @@ import sys
 from pathlib import Path
 
 from SYS.logger import debug, log
+from ProviderCore.registry import get_plugin
 from Store import Store
 from . import _shared as sh
-from API import HydrusNetwork as hydrus_wrapper
 from SYS import pipeline as ctx
 from SYS.result_table_helpers import add_row_columns
 from SYS.result_table import Table, _format_size
@@ -129,6 +129,7 @@ class Delete_File(sh.Cmdlet):
             store = sh.get_field(item, "store")
 
         store_lower = str(store).lower() if store else ""
+        hydrus_provider = get_plugin("hydrusnetwork", config)
 
         backend = None
         try:
@@ -144,18 +145,17 @@ class Delete_File(sh.Cmdlet):
         # so checking only the store name is unreliable.
         is_hydrus_store = False
         try:
-            if backend is not None:
-                from Store.HydrusNetwork import HydrusNetwork as HydrusStore
-
-                is_hydrus_store = isinstance(backend, HydrusStore)
+            if hydrus_provider is not None and backend is not None:
+                is_hydrus_store = bool(hydrus_provider.is_backend(backend, str(store or "")))
         except Exception:
             is_hydrus_store = False
 
         # Backwards-compatible fallback heuristic (older items might only carry a name).
-        if ((not is_hydrus_store) and bool(store_lower)
-                and ("hydrus" in store_lower or store_lower in {"home",
-                                                                "work"})):
-            is_hydrus_store = True
+        if (not is_hydrus_store) and hydrus_provider is not None and bool(store_lower):
+            try:
+                is_hydrus_store = bool(hydrus_provider.is_store_name(store_lower))
+            except Exception:
+                is_hydrus_store = False
         store_label = str(store) if store else "default"
         hydrus_prefix = f"[hydrusnetwork:{store_label}]"
 
@@ -318,18 +318,20 @@ class Delete_File(sh.Cmdlet):
             should_try_hydrus = False
 
         if should_try_hydrus and hash_hex:
-            # Prefer deleting via the resolved store backend when it is a HydrusNetwork store.
-            # This ensures store-specific post-delete hooks run (e.g., clearing Hydrus deletion records).
-            did_backend_delete = False
+            did_hydrus_delete = False
             try:
-                if backend is not None:
-                    deleter = getattr(backend, "delete_file", None)
-                    if callable(deleter):
-                        did_backend_delete = bool(deleter(hash_hex, reason=reason))
+                if hydrus_provider is not None:
+                    did_hydrus_delete = bool(
+                        hydrus_provider.delete_hash(
+                            hash_hex,
+                            store_name=str(store) if store else None,
+                            reason=reason or None,
+                        )
+                    )
             except Exception:
-                did_backend_delete = False
+                did_hydrus_delete = False
 
-            if did_backend_delete:
+            if did_hydrus_delete:
                 hydrus_deleted = True
                 title_str = str(title_val).strip() if title_val else ""
                 if title_str:
@@ -340,85 +342,12 @@ class Delete_File(sh.Cmdlet):
                 else:
                     debug(f"{hydrus_prefix} Deleted hash:{hash_hex}", file=sys.stderr)
             else:
-                # Fallback to direct client calls.
-                client = None
-                if store:
-                    # Store specified: do not fall back to a global/default Hydrus client.
-                    try:
-                        registry = Store(config)
-                        backend = registry[str(store)]
-                        candidate = getattr(backend, "_client", None)
-                        if candidate is not None and hasattr(candidate, "_post"):
-                            client = candidate
-                    except Exception as exc:
-                        if not local_deleted:
-                            log(
-                                f"Hydrus client unavailable for store '{store}': {exc}",
-                                file=sys.stderr,
-                            )
-                            return False
-                    if client is None:
-                        if not local_deleted:
-                            log(
-                                f"Hydrus client unavailable for store '{store}'",
-                                file=sys.stderr
-                            )
-                            return False
-                else:
-                    # No store context; use default Hydrus client.
-                    try:
-                        client = hydrus_wrapper.get_client(config)
-                    except Exception as exc:
-                        if not local_deleted:
-                            log(f"Hydrus client unavailable: {exc}", file=sys.stderr)
-                            return False
-                    if client is None:
-                        if not local_deleted:
-                            log("Hydrus client unavailable", file=sys.stderr)
-                            return False
-
-                payload: Dict[str,
-                              Any] = {
-                                  "hashes": [hash_hex]
-                              }
-                if reason:
-                    payload["reason"] = reason
-                try:
-                    client._post(
-                        "/add_files/delete_files",
-                        data=payload
-                    )  # type: ignore[attr-defined]
-                    # Best-effort clear deletion record if supported by this client.
-                    try:
-                        clearer = getattr(client, "clear_file_deletion_record", None)
-                        if callable(clearer):
-                            clearer([hash_hex])
-                        else:
-                            client._post(
-                                "/add_files/clear_file_deletion_record",
-                                data={
-                                    "hashes": [hash_hex]
-                                }
-                            )  # type: ignore[attr-defined]
-                    except Exception:
-                        pass
-
-                    hydrus_deleted = True
-                    title_str = str(title_val).strip() if title_val else ""
-                    if title_str:
-                        debug(
-                            f"{hydrus_prefix} Deleted title:{title_str} hash:{hash_hex}",
-                            file=sys.stderr,
-                        )
+                if not local_deleted:
+                    if store:
+                        log(f"Hydrus store unavailable for '{store}'", file=sys.stderr)
                     else:
-                        debug(
-                            f"{hydrus_prefix} Deleted hash:{hash_hex}",
-                            file=sys.stderr
-                        )
-                except Exception:
-                    # If it's not in Hydrus (e.g. 404 or similar), that's fine
-                    if not local_deleted:
-                        return []
+                        log("Hydrus delete failed", file=sys.stderr)
+                    return []
 
         if hydrus_deleted and hash_hex:
             size_hint = None

cmdlet/download_file.py

@@ -980,10 +980,14 @@ class Download_File(Cmdlet):
     ) -> Optional[str]:
         if storage is None or not canonical_url:
             return None
+        hydrus_provider = None
         try:
-            from Store.HydrusNetwork import HydrusNetwork
+            registry_helpers = cls._load_provider_registry()
+            get_plugin = registry_helpers.get("get_plugin")
+            if callable(get_plugin):
+                hydrus_provider = get_plugin("hydrusnetwork", {})
         except Exception:
-            HydrusNetwork = None  # type: ignore
+            hydrus_provider = None
 
         try:
             backend_names = list(storage.list_searchable_backends() or [])
@@ -1001,13 +1005,13 @@ class Download_File(Cmdlet):
             except Exception:
                 pass
             try:
-                if HydrusNetwork is not None and isinstance(backend, HydrusNetwork) and not hydrus_available:
+                if hydrus_provider is not None and hydrus_provider.is_backend(backend, str(backend_name)) and not hydrus_available:
                     continue
             except Exception:
                 pass
 
             try:
-                if HydrusNetwork is not None and isinstance(backend, HydrusNetwork):
+                if hydrus_provider is not None and hydrus_provider.is_backend(backend, str(backend_name)):
                     hashes = backend.find_hashes_by_url(canonical_url) or []
                     for existing_hash in hashes:
                         normalized = sh.normalize_hash(existing_hash)

cmdlet/get_relationship.py

@@ -5,12 +5,12 @@ import sys
 
 from SYS.detail_view_helpers import create_detail_view, prepare_detail_metadata
 from SYS.logger import log
+from ProviderCore.registry import get_plugin
 from SYS.result_table_helpers import add_row_columns
 from SYS.selection_builder import build_hash_store_selection
 from SYS.result_publication import publish_result_table
 
 from SYS import pipeline as ctx
-from API import HydrusNetwork as hydrus_wrapper
 from . import _shared as sh
 
 Cmdlet = sh.Cmdlet
@@ -22,7 +22,6 @@ get_hash_for_operation = sh.get_hash_for_operation
 fetch_hydrus_metadata = sh.fetch_hydrus_metadata
 should_show_help = sh.should_show_help
 get_field = sh.get_field
-from Store import Store
 
 CMDLET = Cmdlet(
     name="get-relationship",
@@ -109,6 +108,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
         return 1
 
     # Fetch Hydrus relationships if we have a hash.
+    hydrus_provider = get_plugin("hydrusnetwork", config)
 
     hash_hex = (
         normalize_hash(override_hash)
@@ -118,29 +118,18 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
 
     if hash_hex:
         try:
-            client = None
             store_label = "hydrus"
-            backend_obj = None
             if store_name:
-                # Store specified: do not fall back to a global/default Hydrus client.
                 store_label = str(store_name)
-                try:
-                    store = Store(config)
-                    backend_obj = store[str(store_name)]
-                    candidate = getattr(backend_obj, "_client", None)
-                    if candidate is not None and hasattr(candidate,
-                                                         "get_file_relationships"):
-                        client = candidate
-                except Exception:
-                    client = None
-                if client is None:
+                if hydrus_provider is None:
                     log(
                         f"Hydrus client unavailable for store '{store_name}'",
                         file=sys.stderr
                     )
                     return 1
+                relationships = hydrus_provider.get_relationships(hash_hex, store_name=store_name)
             else:
-                client = hydrus_wrapper.get_client(config)
+                relationships = hydrus_provider.get_relationships(hash_hex) if hydrus_provider is not None else None
 
             def _resolve_related_title(rel_hash: str) -> str:
                 """Best-effort resolve a Hydrus hash to a human title.
@@ -154,22 +143,15 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
                 if not h:
                     return str(rel_hash)
 
-                # Prefer backend tag extraction when available.
-                if backend_obj is not None and hasattr(backend_obj, "get_tag"):
+                # Prefer provider-backed title resolution when available.
+                if hydrus_provider is not None:
                     try:
-                        tag_result = backend_obj.get_tag(h)
-                        tags = (
-                            tag_result[0]
-                            if isinstance(tag_result,
-                                          tuple) and tag_result else tag_result
+                        resolved_title = hydrus_provider.get_title(
+                            h,
+                            store_name=store_label if store_name else None,
                         )
-                        if isinstance(tags, list):
-                            for t in tags:
-                                if isinstance(t,
-                                              str) and t.lower().startswith("title:"):
-                                    val = t.split(":", 1)[1].strip()
-                                    if val:
-                                        return val
+                        if isinstance(resolved_title, str) and resolved_title.strip():
+                            return resolved_title.strip()
                     except Exception:
                         pass
 
@@ -179,7 +161,6 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
                         config,
                         h,
                         store_name=store_label if store_name else None,
-                        hydrus_client=client,
                         include_service_keys_to_tags=True,
                         include_file_url=False,
                         include_duration=False,
@@ -224,14 +205,12 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
 
                 return h[:16] + "..."
 
-            if client:
-                rel = client.get_file_relationships(hash_hex)
-                if rel:
-                    file_rels = rel.get("file_relationships",
+            if relationships:
+                file_rels = relationships.get("file_relationships",
                                         {})
-                    this_file_rels = file_rels.get(hash_hex)
+                this_file_rels = file_rels.get(hash_hex)
 
-                    if this_file_rels:
+                if this_file_rels:
                         # Map Hydrus relationship IDs to names.
                         # For /manage_file_relationships/get_file_relationships, the Hydrus docs define:
                         #   0=potential duplicates, 1=false positives, 3=alternates, 8=duplicates