diff --git a/.gitignore b/.gitignore index 9a67c49..7a805ad 100644 --- a/.gitignore +++ b/.gitignore @@ -242,4 +242,5 @@ tmp_* authtoken.secret mypy. -.idea \ No newline at end of file +.idea +medios.db \ No newline at end of file diff --git a/API/zerotier.py b/API/zerotier.py deleted file mode 100644 index c33998c..0000000 --- a/API/zerotier.py +++ /dev/null @@ -1,646 +0,0 @@ -"""ZeroTier helpers and discovery utilities. - -This module provides a small, dependency-light API for interacting with a -local zerotier-one node (preferred via Python module when available, else via -`zerotier-cli`), discovering peers on a given ZeroTier network, and probing -for services running on those peers (e.g., our remote storage server or a -Hydrus instance). - -Notes: -- This is intentionally conservative and all operations are best-effort and - fail gracefully when the local system does not have ZeroTier installed. -- The implementation prefers a Python ZeroTier binding when available, else - falls back to calling the `zerotier-cli` binary (if present) and parsing - JSON output where possible. - -Example usage: - from API import zerotier - if zerotier.is_available(): - nets = zerotier.list_networks() - zerotier.join_network("8056c2e21c000001") - services = zerotier.discover_services_on_network("8056c2e21c000001", ports=[999], paths=["/health","/api_version"]) # noqa: E501 -""" - -from __future__ import annotations - -import json -import os -import shutil -import subprocess -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple - -from SYS.logger import debug - -# Optional Python ZeroTier bindings - prefer them when available -_HAVE_PY_ZEROTIER = False -try: - # Try common package names; not all installations will have this available - # This import is optional and callers should still work via the CLI fallback. - import zerotier as _zt # type: ignore - _HAVE_PY_ZEROTIER = True -except Exception: - _HAVE_PY_ZEROTIER = False - - -@dataclass -class ZeroTierNetwork: - id: str - name: str - status: str - assigned_addresses: List[str] - - -@dataclass -class ZeroTierServiceProbe: - address: str - port: int - path: str - url: str - ok: bool - status_code: Optional[int] - payload: Optional[Any] - service_hint: Optional[str] = None - - -def _get_cli_path() -> Optional[str]: - """Find the zerotier-cli binary or script across common locations.""" - # 1. Check PATH - p = shutil.which("zerotier-cli") - if p: - return p - - # 2. Check common installation paths - candidates = [] - if sys.platform == "win32": - # Check various Program Files locations and both .bat and .exe - roots = [ - os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)"), - os.environ.get("ProgramFiles", r"C:\Program Files"), - os.environ.get("ProgramData", r"C:\ProgramData"), - ] - for root in roots: - base = os.path.join(root, "ZeroTier", "One", "zerotier-cli") - candidates.append(base + ".bat") - candidates.append(base + ".exe") - else: - # Linux / macOS - candidates = [ - "/usr/sbin/zerotier-cli", - "/usr/local/bin/zerotier-cli", - "/sbin/zerotier-cli", - "/var/lib/zerotier-one/zerotier-cli", - ] - - for c in candidates: - try: - if os.path.isfile(c): - return str(c) - except Exception: - pass - - return None - - -def _get_home_path() -> Optional[str]: - """Return the ZeroTier home directory (containing authtoken.secret).""" - if sys.platform == "win32": - path = os.path.join(os.environ.get("ProgramData", r"C:\ProgramData"), "ZeroTier", "One") - if os.path.isdir(path): - return path - else: - # Linux - if os.path.isdir("/var/lib/zerotier-one"): - return "/var/lib/zerotier-one" - # macOS - if os.path.isdir("/Library/Application Support/ZeroTier/One"): - return "/Library/Application Support/ZeroTier/One" - return None - - -def _get_authtoken() -> Optional[str]: - """Try to read the local ZeroTier authtoken.secret from the ZeroTier home dir.""" - home = _get_home_path() - if home: - token_file = os.path.join(home, "authtoken.secret") - if os.path.isfile(token_file): - try: - with open(token_file, "r") as f: - return f.read().strip() - except Exception: - pass - return None - - -def _read_token_file(path: str) -> Optional[str]: - """Read a token from an arbitrary file path (safely). - - Returns the stripped token string or None on error. - """ - try: - with open(path, "r") as f: - t = f.read().strip() - return t if t else None - except Exception as exc: - debug(f"read_token_file failed: {exc}") - return None - - -def _find_file_upwards(filename: str, start: Optional[str] = None) -> Optional[str]: - """Search for `filename` by walking up parent directories starting at `start` (or CWD). - - Returns the first matching path or None. - """ - start_dir = Path(start or os.getcwd()).resolve() - for p in [start_dir] + list(start_dir.parents): - candidate = p / filename - if candidate.is_file(): - return str(candidate) - return None - - -def _find_repo_root(start: Optional[str] = None) -> Optional[str]: - """Find a probable repository root by looking for .git/pyproject.toml/setup.py upwards from start. - - Returns the directory path or None. - """ - start_dir = Path(start or Path(__file__).resolve().parent).resolve() - for p in [start_dir] + list(start_dir.parents): - if (p / ".git").exists() or (p / "pyproject.toml").exists() or (p / "setup.py").exists(): - return str(p) - return None - - -def _get_token_path() -> Optional[str]: - """Return the source of an auth token: 'env' or a filesystem path to authtoken.secret. - - This checks in order: env token string, env token file, CWD (and parents), repo root, - user home, and finally the system ZeroTier home. - """ - # 1: token provided directly in env - if os.environ.get("ZEROTIER_AUTH_TOKEN") or os.environ.get("ZEROTIER_AUTHTOKEN"): - return "env" - - # 2: token file path provided - p = os.environ.get("ZEROTIER_AUTH_TOKEN_FILE") or os.environ.get("ZEROTIER_AUTHTOKEN_FILE") - if p and os.path.isfile(p): - return p - - # 3: token file in current working dir or any parent - up = _find_file_upwards("authtoken.secret", start=os.getcwd()) - if up: - return up - - # 4: token file at repository root (helpful if TUI runs with a different CWD) - repo = _find_repo_root() - if repo: - rp = os.path.join(repo, "authtoken.secret") - if os.path.isfile(rp): - return rp - - # 5: token file in user's home - home_candidate = os.path.join(str(Path.home()), "authtoken.secret") - if os.path.isfile(home_candidate): - return home_candidate - - # 6: fallback to the ZeroTier home location - zhome = _get_home_path() - if zhome: - tz = os.path.join(zhome, "authtoken.secret") - if os.path.isfile(tz): - return tz - - return None - - -def _get_token_override() -> Optional[str]: - """Read the token value using the path determined by `_get_token_path()` or env. - - Returns the token string, or None if no token is available. - """ - path_or_env = _get_token_path() - if path_or_env == "env": - t = os.environ.get("ZEROTIER_AUTH_TOKEN") or os.environ.get("ZEROTIER_AUTHTOKEN") - return t.strip() if t else None - if path_or_env: - return _read_token_file(path_or_env) - return None - - -def _cli_available() -> bool: - return _get_cli_path() is not None - - -def is_available() -> bool: - """Return True if we can interact with ZeroTier locally (module or CLI).""" - return _HAVE_PY_ZEROTIER or _cli_available() - - -def _run_cli_capture(*args: str, timeout: float = 5.0) -> Tuple[int, str, str]: - """Run zerotier-cli and return (returncode, stdout, stderr). - - This centralizes how we call the CLI so we can always capture stderr and - returncodes and make debugging failures much easier. - """ - bin_path = _get_cli_path() - if not bin_path: - raise RuntimeError("zerotier-cli not found") - - full_args = list(args) - token = _get_token_override() - if token and not any(a.startswith("-T") for a in full_args): - # Do not log the token itself; we log only its presence/length for debugging - debug(f"Using external authtoken (len={len(token)}) for CLI auth") - full_args.insert(0, f"-T{token}") - - home = _get_home_path() - if home and not any(a.startswith("-D") for a in full_args): - full_args.insert(0, f"-D{home}") - - cmd = [bin_path, *full_args] - debug(f"Running zerotier-cli: {cmd}") - use_shell = sys.platform == "win32" and str(bin_path).lower().endswith(".bat") - proc = subprocess.run(cmd, timeout=timeout, capture_output=True, text=True, shell=use_shell) - return proc.returncode, proc.stdout, proc.stderr - - -def _run_cli_json(*args: str, timeout: float = 5.0) -> Any: - """Run zerotier-cli with arguments and parse JSON output if possible. - - Returns parsed JSON on success, or raises an exception with stderr when non-zero exit. - """ - rc, out, err = _run_cli_capture(*args, timeout=timeout) - if rc != 0: - # Surface stderr or stdout in the exception so callers (and logs) can show - # the actionable message instead of a blind CalledProcessError. - raise RuntimeError(f"zerotier-cli failed (rc={rc}): {err.strip() or out.strip()}") - try: - return json.loads(out) - except Exception: - # Some CLI invocations might print non-json; return as raw string - return out - - -def list_networks() -> List[ZeroTierNetwork]: - """Return a list of configured ZeroTier networks on this node. - - Best-effort: prefers Python binding, then `zerotier-cli listnetworks -j`. - """ - nets: List[ZeroTierNetwork] = [] - - if _HAVE_PY_ZEROTIER: - try: - # Attempt to use common API shape (best-effort) - raw = _zt.list_networks() # type: ignore[attr-defined] - # If the Python binding returned results, use them. If it returned - # an empty list/None, fall back to the CLI so we don't return a - # false-empty result to the UI. - if raw: - for n in raw: - # raw entries are expected to be dict-like - nets.append(ZeroTierNetwork( - id=str(n.get("id") or n.get("networkId") or ""), - name=str(n.get("name") or ""), - status=str(n.get("status") or ""), - assigned_addresses=list(n.get("assignedAddresses") or []), - )) - return nets - else: - debug("py-zerotier returned no networks; falling back to CLI") - except Exception as exc: # pragma: no cover - optional dependency - debug(f"py-zerotier listing failed: {exc}") - - # CLI fallback - try: - data = _run_cli_json("listnetworks", "-j") - if isinstance(data, list): - for entry in data: - nets.append(ZeroTierNetwork( - id=str(entry.get("id") or ""), - name=str(entry.get("name") or ""), - status=str(entry.get("status") or ""), - assigned_addresses=list(entry.get("assignedAddresses") or []), - )) - except Exception as exc: - debug(f"list_networks failed: {exc}") - - return nets - - -def join_network(network_id: str) -> bool: - """Join the given ZeroTier network (best-effort). - - Returns True on success, False otherwise. - """ - network_id = str(network_id or "").strip() - if not network_id: - raise ValueError("network_id is required") - - if _HAVE_PY_ZEROTIER: - try: - _zt.join_network(network_id) # type: ignore[attr-defined] - return True - except Exception as exc: # pragma: no cover - optional dependency - debug(f"py-zerotier join failed: {exc}") - - if _cli_available(): - try: - rc, out, err = _run_cli_capture("join", network_id, timeout=10) - if rc == 0: - return True - # Surface the CLI's stderr/stdout to callers as an exception so the TUI - # can show a helpful error (instead of a generic 'failed to join'). - raise RuntimeError(f"zerotier-cli join failed (rc={rc}): {err.strip() or out.strip()}") - except Exception: - # Re-raise so callers (UI/tests) can react to the exact error - raise - return False - - -def leave_network(network_id: str) -> bool: - network_id = str(network_id or "").strip() - if not network_id: - raise ValueError("network_id is required") - - if _HAVE_PY_ZEROTIER: - try: - _zt.leave_network(network_id) # type: ignore[attr-defined] - return True - except Exception as exc: # pragma: no cover - optional dependency - debug(f"py-zerotier leave failed: {exc}") - - if _cli_available(): - try: - rc, out, err = _run_cli_capture("leave", network_id, timeout=10) - if rc == 0: - return True - raise RuntimeError(f"zerotier-cli leave failed (rc={rc}): {err.strip() or out.strip()}") - except Exception: - raise - return False - - -def _strip_addr(addr: str) -> str: - # Remove trailing CID parts like '/24' and zone IDs like '%eth0' - if not addr: - return addr - a = addr.split("/")[0] - if "%" in a: - a = a.split("%", 1)[0] - return a - - -def get_assigned_addresses(network_id: str) -> List[str]: - """Return assigned ZeroTier addresses for the local node on the given network.""" - network_id = str(network_id or "").strip() - if not network_id: - return [] - - for n in list_networks(): - if n.id == network_id: - return [str(_strip_addr(a)) for a in n.assigned_addresses if a] - return [] - - -def get_assigned_subnets(network_id: str) -> List[str]: - """Return CIDR subnets (e.g. '10.147.17.0/24') for the given network.""" - network_id = str(network_id or "").strip() - if not network_id: - return [] - - subnets = [] - for n in list_networks(): - if n.id == network_id: - for addr in n.assigned_addresses: - if addr and "/" in addr: - # Calculate subnet base - try: - import ipaddress - net = ipaddress.ip_network(addr, strict=False) - subnets.append(str(net)) - except Exception: - pass - return subnets - - -def fetch_central_members(network_id: str, api_token: str) -> List[Dict[str, Any]]: - """Fetch member details from ZeroTier Central API. - - Requires a valid ZeroTier Central API token. - Returns a list of member objects containing 'config' with 'ipAssignments', etc. - """ - url = f"https://my.zerotier.com/api/v1/network/{network_id}/member" - headers = {"Authorization": f"token {api_token}"} - try: - import httpx - - resp = httpx.get(url, headers=headers, timeout=10) - resp.raise_for_status() - return resp.json() - except Exception: - try: - import requests - - resp = requests.get(url, headers=headers, timeout=10) - resp.raise_for_status() - return resp.json() - except Exception as exc: - debug(f"ZeroTier Central API fetch failed: {exc}") - return [] - - -def list_peers() -> List[Dict[str, Any]]: - """Return peers known to the local ZeroTier node (best-effort parsing). - - If CLI supports JSON output for peers it will be parsed, otherwise we return - an empty list. - """ - if _HAVE_PY_ZEROTIER: - try: - peers = _zt.list_peers() # type: ignore[attr-defined] - return list(peers or []) - except Exception as exc: # pragma: no cover - optional dependency - debug(f"py-zerotier list_peers failed: {exc}") - - if _cli_available(): - try: - data = _run_cli_json("peers", "-j") - if isinstance(data, list): - return data - except Exception as exc: - debug(f"zerotier-cli peers failed: {exc}") - return [] - - -def _probe_url(url: str, *, timeout: float = 2.0, accept_json: bool = True) -> Tuple[bool, Optional[int], Optional[Any]]: - """Try fetching the URL and return (ok, status_code, payload). - - Uses httpx if available, otherwise falls back to requests. - """ - try: - try: - import httpx - resp = httpx.get(url, timeout=timeout) - code = int(resp.status_code if hasattr(resp, "status_code") else resp.status) - content_type = str(resp.headers.get("content-type") or "").lower() - if code == 200 and accept_json and "json" in content_type: - try: - return True, code, resp.json() - except Exception: - return True, code, resp.text - return (code == 200), code, resp.text - except Exception: - import requests # type: ignore - resp = requests.get(url, timeout=timeout) - code = int(resp.status_code) - content_type = str(resp.headers.get("content-type") or "").lower() - if code == 200 and accept_json and "json" in content_type: - try: - return True, code, resp.json() - except Exception: - return True, code, resp.text - return (code == 200), code, resp.text - except Exception as exc: - debug(f"Probe failed: {url} -> {exc}") - return False, None, None - - -def discover_services_on_network( - network_id: str, - *, - ports: Optional[List[int]] = None, - paths: Optional[List[str]] = None, - timeout: float = 2.0, - accept_json: bool = True, - api_token: Optional[str] = None, -) -> List[ZeroTierServiceProbe]: - """Probe peers on the given network for HTTP services. - - If api_token is provided, it fetches all member IPs from ZeroTier Central. - Otherwise, it only probes the local node's assigned addresses (for now). - """ - net = str(network_id or "").strip() - if not net: - raise ValueError("network_id required") - - ports = list(ports or [999]) - paths = list(paths or ["/health", "/api_version"]) - - addresses = get_assigned_addresses(net) - - if api_token: - members = fetch_central_members(net, api_token) - for m in members: - # Look for online members with IP assignments - if m.get("online") and m.get("config", {}).get("ipAssignments"): - for ip in m["config"]["ipAssignments"]: - addr = str(ip).split("/")[0] - if addr not in addresses: - addresses.append(addr) - else: - # Fallback: if no Central token, and we are on a likely /24 subnet, - # we can try to guess/probe peers on that same subnet. - subnets = get_assigned_subnets(net) - for subnet_str in subnets: - try: - import ipaddress - subnet = ipaddress.ip_network(subnet_str, strict=False) - # Only scan if subnet is reasonably small (e.g. <= /24 = 256 hosts) - if subnet.num_addresses <= 256: - for ip in subnet.hosts(): - addr = str(ip) - if addr not in addresses: - addresses.append(addr) - except Exception: - pass - - probes: List[ZeroTierServiceProbe] = [] - - # Parallelize probes to make subnet scanning feasible - import concurrent.futures - - def do_probe(host): - host_probes = [] - for port in ports: - # Try HTTP first as it's the common case for local storage - for scheme in ("http", "https"): - # Fast probe of just the first path - path = paths[0] - url = f"{scheme}://{host}:{port}{path}" - ok, code, payload = _probe_url(url, timeout=timeout, accept_json=accept_json) - if ok or code == 401: - hint = None - try: - # remote_storage_server returns {"status": "ok", ...} - if code == 401: - hint = "remote_storage" # Most likely - elif isinstance(payload, dict) and payload.get("status"): - hint = "remote_storage" - # hydrus returns {"api_version": ...} - elif isinstance(payload, dict) and payload.get("api_version"): - hint = "hydrus" - except Exception: - pass - - host_probes.append(ZeroTierServiceProbe( - address=host, - port=int(port), - path=path, - url=url, - ok=(code == 200), - status_code=code, - payload=payload, - service_hint=hint, - )) - # Stop probing other schemes/paths for this host/port - break - return host_probes - - # Use ThreadPoolExecutor for concurrent I/O probes - max_workers = min(50, len(addresses) or 1) - with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: - future_to_addr = {executor.submit(do_probe, addr): addr for addr in addresses} - for future in concurrent.futures.as_completed(future_to_addr): - try: - probes.extend(future.result()) - except Exception: - pass - - return probes - - -def find_peer_service( - network_id: str, - *, - service_hint: Optional[str] = None, - port: Optional[int] = None, - path_candidates: Optional[List[str]] = None, - timeout: float = 2.0, - api_token: Optional[str] = None, -) -> Optional[ZeroTierServiceProbe]: - """Return the first probe that matches service_hint or is successful. - - Useful for selecting a peer to configure a store against. - """ - paths = path_candidates or ["/health", "/api_version", "/session_key"] - ports = [port] if port is not None else [999, 5000, 45869, 80, 443] - - probes = discover_services_on_network( - network_id, ports=ports, paths=paths, timeout=timeout, api_token=api_token - ) - if not probes: - return None - if service_hint: - for p in probes: - if p.service_hint and service_hint.lower() in str(p.service_hint).lower(): - return p - # Hydrus detection: check payload for 'api_version' - try: - if service_hint.lower() == "hydrus" and isinstance(p.payload, dict) and p.payload.get("api_version"): - return p - except Exception: - pass - # Fallback: return the first OK probe - return probes[0] if probes else None diff --git a/CLI.py b/CLI.py index 9f3ec03..c8cf5f8 100644 --- a/CLI.py +++ b/CLI.py @@ -93,7 +93,6 @@ _install_rich_traceback(show_locals=False) from SYS.logger import debug, set_debug from SYS.worker_manager import WorkerManager -from SYS.background_services import ensure_zerotier_server_running, stop_zerotier_server from SYS.cmdlet_catalog import ( get_cmdlet_arg_choices, @@ -1587,8 +1586,6 @@ class CLI: return app def run(self) -> None: - ensure_zerotier_server_running() - # Ensure Rich tracebacks are active even when invoking subcommands. try: config = self._config_loader.load() @@ -2402,8 +2399,6 @@ Come to love it when others take what you share, as there is no greater joy if pipeline_ctx_ref: pipeline_ctx_ref.clear_current_command_text() - stop_zerotier_server() - _PTK_Lexer = object # type: ignore diff --git a/SYS/background_services.py b/SYS/background_services.py deleted file mode 100644 index e631f4b..0000000 --- a/SYS/background_services.py +++ /dev/null @@ -1,143 +0,0 @@ -from __future__ import annotations - -import os -import sys -import subprocess -from pathlib import Path -from typing import Optional - -from SYS.config import load_config -from SYS.logger import debug, log - -_zt_server_proc: Optional[subprocess.Popen] = None -_zt_server_last_config: Optional[str] = None - -# We no longer use atexit here because explicit lifecycle management -# is preferred in TUI/REPL, and background servers use a monitor thread -# to shut down when the parent dies. -# atexit.register(lambda: stop_zerotier_server()) - -def ensure_zerotier_server_running() -> None: - """Check config and ensure the ZeroTier storage server is running if needed.""" - global _zt_server_proc, _zt_server_last_config - - try: - # Load config from the project root (where config.conf typically lives) - repo_root = Path(__file__).resolve().parent.parent - cfg = load_config(repo_root) - except Exception: - return - - zt_conf = cfg.get("networking", {}).get("zerotier", {}) - serve_target = zt_conf.get("serve") - port = zt_conf.get("port") or 999 - api_key = zt_conf.get("api_key") - - # Config hash to detect changes - config_id = f"{serve_target}|{port}|{api_key}" - - # Check if proc is still alive - if _zt_server_proc: - if _zt_server_proc.poll() is not None: - # Process died - debug("ZeroTier background server died. Restarting...") - _zt_server_proc = None - elif config_id == _zt_server_last_config: - # Already running with correct config - return - - # If config changed and we have a proc, stop it - if _zt_server_proc and config_id != _zt_server_last_config: - debug("ZeroTier server config changed. Stopping old process...") - try: - _zt_server_proc.terminate() - _zt_server_proc.wait(timeout=2) - except Exception: - try: - _zt_server_proc.kill() - except Exception: - pass - _zt_server_proc = None - - _zt_server_last_config = config_id - - if not serve_target: - return - - # Resolve path - storage_path = None - folders = cfg.get("store", {}).get("folder", {}) - for name, block in folders.items(): - if name.lower() == serve_target.lower(): - storage_path = block.get("path") or block.get("PATH") - break - - if not storage_path: - # Fallback to direct path - storage_path = serve_target - - if not storage_path or not Path(storage_path).exists(): - debug(f"ZeroTier host target '{serve_target}' not found at {storage_path}. Cannot start server.") - return - - repo_root = Path(__file__).resolve().parent.parent - server_script = repo_root / "scripts" / "remote_storage_server.py" - - if not server_script.exists(): - debug(f"ZeroTier server script not found at {server_script}") - return - - # Use the same python executable that is currently running - # On Windows, explicitly prefer the .venv python if it exists - python_exe = sys.executable - if sys.platform == "win32": - venv_py = repo_root / ".venv" / "Scripts" / "python.exe" - if venv_py.exists(): - python_exe = str(venv_py) - - cmd = [python_exe, str(server_script), - "--storage-path", str(storage_path), - "--port", str(port), - "--monitor"] - cmd += ["--parent-pid", str(os.getpid())] - if api_key: - cmd += ["--api-key", str(api_key)] - - try: - debug(f"Starting ZeroTier storage server: {cmd}") - # Capture errors to a log file instead of DEVNULL - log_file = repo_root / "zt_server_error.log" - with open(log_file, "a") as f: - f.write(f"\n--- Starting server at {__import__('datetime').datetime.now()} ---\n") - f.write(f"Command: {' '.join(cmd)}\n") - f.write(f"CWD: {repo_root}\n") - f.write(f"Python: {python_exe}\n") - - err_f = open(log_file, "a") - # On Windows, CREATE_NO_WINDOW = 0x08000000 ensures no console pops up - import subprocess - _zt_server_proc = subprocess.Popen( - cmd, - stdout=subprocess.DEVNULL, - stderr=err_f, - cwd=str(repo_root), - creationflags=0x08000000 if sys.platform == "win32" else 0 - ) - log(f"ZeroTier background server started on port {port} (sharing {serve_target})") - except Exception as e: - debug(f"Failed to start ZeroTier server: {e}") - _zt_server_proc = None - -def stop_zerotier_server() -> None: - """Stop the background server if it is running.""" - global _zt_server_proc - if _zt_server_proc: - try: - _zt_server_proc.terminate() - _zt_server_proc.wait(timeout=2) - except Exception: - try: - _zt_server_proc.kill() - except Exception: - pass - _zt_server_proc = None diff --git a/SYS/config.py b/SYS/config.py index 1ae0846..b0a45a4 100644 --- a/SYS/config.py +++ b/SYS/config.py @@ -188,19 +188,6 @@ def _apply_conf_block( tool[tool_name] = dict(block) return - if kind_l == "networking": - net_name = str(subtype).strip().lower() - if not net_name: - return - net = config.setdefault("networking", {}) - if not isinstance(net, dict): - config["networking"] = {} - net = config["networking"] - existing = net.get(net_name) - if isinstance(existing, dict): - _merge_dict_inplace(existing, block) - else: - net[net_name] = dict(block) return @@ -366,24 +353,6 @@ def _serialize_conf(config: Dict[str, Any]) -> str: seen_keys.add(k_upper) lines.append(f"{k}={_format_conf_value(block.get(k))}") - # Networking blocks - networking = config.get("networking") - if isinstance(networking, dict): - for name in sorted(networking.keys()): - block = networking.get(name) - if not isinstance(block, dict): - continue - lines.append("") - lines.append(f"[networking={name}]") - - seen_keys = set() - for k in sorted(block.keys()): - k_upper = k.upper() - if k_upper in seen_keys: - continue - seen_keys.add(k_upper) - lines.append(f"{k}={_format_conf_value(block.get(k))}") - return "\n".join(lines).rstrip() + "\n" @@ -674,34 +643,6 @@ def resolve_debug_log(config: Dict[str, Any]) -> Optional[Path]: return path -def migrate_conf_to_db(config: Dict[str, Any]) -> None: - """Migrate the configuration dictionary to the database.""" - log("Migrating configuration from .conf to database...") - for key, value in config.items(): - if key in ("store", "provider", "tool", "networking"): - cat = key - sub_dict = value - if isinstance(sub_dict, dict): - for subtype, subtype_items in sub_dict.items(): - if isinstance(subtype_items, dict): - # For provider/tool/networking, subtype is the name (e.g. alldebrid) - # but for store, it's the type (e.g. hydrusnetwork) - if cat == "store" and str(subtype).strip().lower() == "folder": - continue - if cat != "store": - for k, v in subtype_items.items(): - save_config_value(cat, subtype, "", k, v) - else: - for name, items in subtype_items.items(): - if isinstance(items, dict): - for k, v in items.items(): - save_config_value(cat, subtype, name, k, v) - else: - # Global setting - save_config_value("global", "", "", key, value) - log("Configuration migration complete!") - - def load_config( config_dir: Optional[Path] = None, filename: str = DEFAULT_CONFIG_FILENAME ) -> Dict[str, Any]: @@ -712,37 +653,12 @@ def load_config( if cache_key in _CONFIG_CACHE: return _CONFIG_CACHE[cache_key] - # 1. Try loading from database first + # Load from database db_config = get_config_all() if db_config: _CONFIG_CACHE[cache_key] = db_config return db_config - # 2. If DB is empty, try loading from legacy config.conf - if config_path.exists(): - if config_path.suffix.lower() != ".conf": - log(f"Unsupported config format: {config_path.name} (only .conf is supported)") - return {} - - try: - config = _load_conf_config(base_dir, config_path) - # Migrate to database - migrate_conf_to_db(config) - - # Optional: Rename old config file to mark as migrated - try: - migrated_path = config_path.with_name(config_path.name + ".migrated") - config_path.rename(migrated_path) - log(f"Legacy config file renamed to {migrated_path.name}") - except Exception as e: - log(f"Could not rename legacy config file: {e}") - - _CONFIG_CACHE[cache_key] = config - return config - except Exception as e: - log(f"Failed to load legacy config at {config_path}: {e}") - return {} - return {} @@ -771,18 +687,43 @@ def save_config( base_dir = config_dir or SCRIPT_DIR config_path = base_dir / filename - if config_path.suffix.lower() != ".conf": - raise RuntimeError( - f"Unsupported config format: {config_path.name} (only .conf is supported)" - ) - - # Safety Check: placeholder (folder store validation removed) - _validate_config_safety(config) - + # 1. Save to Database try: - config_path.write_text(_serialize_conf(config), encoding="utf-8") - except OSError as exc: - raise RuntimeError(f"Failed to write config to {config_path}: {exc}") from exc + from SYS.database import db, save_config_value + + # We want to clear and re-save or just update? + # For simplicity, we'll iterate and update. + for key, value in config.items(): + if key in ('store', 'provider', 'tool'): + if isinstance(value, dict): + for subtype, instances in value.items(): + if isinstance(instances, dict): + # provider/tool are usually config[cat][subtype][key] + # but store is config['store'][subtype][name][key] + if key == 'store': + for name, settings in instances.items(): + if isinstance(settings, dict): + for k, v in settings.items(): + save_config_value(key, subtype, name, k, v) + else: + for k, v in instances.items(): + save_config_value(key, subtype, "default", k, v) + else: + # global settings + if not key.startswith("_"): + save_config_value("global", "none", "none", key, value) + except Exception as e: + log(f"Failed to save config to database: {e}") + + # 2. Legacy fallback: write to .conf for now (optional, but keep for backward compat for a bit) + if config_path.suffix.lower() == ".conf": + # Safety Check: placeholder (folder store validation removed) + _validate_config_safety(config) + + try: + config_path.write_text(_serialize_conf(config), encoding="utf-8") + except OSError as exc: + log(f"Failed to write legacy config to {config_path}: {exc}") cache_key = _make_cache_key(config_dir, filename, config_path) _CONFIG_CACHE[cache_key] = config diff --git a/SYS/database.py b/SYS/database.py index f1df271..47d4c41 100644 --- a/SYS/database.py +++ b/SYS/database.py @@ -138,7 +138,8 @@ def save_config_value(category: str, subtype: str, item_name: str, key: str, val def get_config_all() -> Dict[str, Any]: """Retrieve all configuration from the database in the legacy dict format.""" try: - db.execute("DELETE FROM config WHERE category='store' AND LOWER(subtype)='folder'") + db.execute("DELETE FROM config WHERE category='store' AND LOWER(subtype) in ('folder', 'zerotier')") + db.execute("DELETE FROM config WHERE category='networking'") except Exception: pass rows = db.fetchall("SELECT category, subtype, item_name, key, value FROM config") @@ -165,7 +166,7 @@ def get_config_all() -> Dict[str, Any]: config[key] = parsed_val else: # Modular structure: config[cat][sub][name][key] - if cat in ('provider', 'tool', 'networking'): + if cat in ('provider', 'tool'): cat_dict = config.setdefault(cat, {}) sub_dict = cat_dict.setdefault(sub, {}) sub_dict[key] = parsed_val diff --git a/SYS/optional_deps.py b/SYS/optional_deps.py index cef870c..725d045 100644 --- a/SYS/optional_deps.py +++ b/SYS/optional_deps.py @@ -48,14 +48,6 @@ _PROVIDER_DEPENDENCIES: Dict[str, List[Tuple[str, str]]] = { "soulseek": [("aioslsk", "aioslsk>=1.6.0")], } -# Dependencies required when ZeroTier features are configured (auto-install when enabled) -_ZEROTIER_DEPENDENCIES: List[Tuple[str, str]] = [ - ("flask", "flask>=2.3.0"), - ("flask_cors", "flask-cors>=3.0.1"), - ("werkzeug", "werkzeug>=2.3.0"), -] - - def florencevision_missing_modules() -> List[str]: return [ requirement @@ -151,29 +143,5 @@ def maybe_auto_install_configured_tools(config: Dict[str, Any]) -> None: label = f"{provider_name.title()} provider" _install_requirements(label, requirements) - # ZeroTier: if a zerotier section is present OR a zerotier store is configured, - # optionally auto-install Flask-based remote server dependencies so the - # `remote_storage_server.py` and CLI helper will run out-of-the-box. - try: - zerotier_cfg = (config or {}).get("zerotier") - store_cfg = (config or {}).get("store") if isinstance(config, dict) else {} - store_has_zerotier = isinstance(store_cfg, dict) and bool(store_cfg.get("zerotier")) - - if (isinstance(zerotier_cfg, dict) and zerotier_cfg) or store_has_zerotier: - auto_install = True - if isinstance(zerotier_cfg, dict) and "auto_install" in zerotier_cfg: - auto_install = _as_bool(zerotier_cfg.get("auto_install"), True) - if auto_install: - missing = [ - requirement - for import_name, requirement in _ZEROTIER_DEPENDENCIES - if not _try_import(import_name) - ] - if missing: - _install_requirements("ZeroTier", missing) - except Exception: - # Don't let optional-dep logic raise at startup - pass - __all__ = ["maybe_auto_install_configured_tools", "florencevision_missing_modules"] diff --git a/Store/Folder.py b/Store/Folder.py deleted file mode 100644 index 5cb0139..0000000 --- a/Store/Folder.py +++ /dev/null @@ -1,2312 +0,0 @@ -from __future__ import annotations - -import json -import re -import shutil -import sys -from fnmatch import fnmatch -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple - -from SYS.logger import debug, log -from SYS.utils import sha256_file, expand_path -from SYS.config import get_local_storage_path - -from Store._base import Store - - -def _normalize_hash(value: Any) -> Optional[str]: - candidate = str(value or "").strip().lower() - if len(candidate) != 64: - return None - if any(ch not in "0123456789abcdef" for ch in candidate): - return None - return candidate - - -def _resolve_file_hash(db_hash: Optional[str], file_path: Path) -> Optional[str]: - normalized = _normalize_hash(db_hash) if db_hash else None - if normalized: - return normalized - return _normalize_hash(file_path.stem) - - -def _normalize_url_for_search(url: str) -> str: - value = str(url or "").strip() - value = re.sub(r"^[a-z][a-z0-9+.-]*://", "", value, flags=re.IGNORECASE) - value = re.sub(r"^www\.", "", value, flags=re.IGNORECASE) - return value.lower() - - -def _match_url_pattern(url: str, pattern: str) -> bool: - normalized_url = _normalize_url_for_search(url) - normalized_pattern = _normalize_url_for_search(pattern) - if not normalized_pattern: - return False - has_wildcards = any(ch in normalized_pattern for ch in ("*", "?")) - if has_wildcards: - return fnmatch(normalized_url, normalized_pattern) - normalized_url_no_slash = normalized_url.rstrip("/") - normalized_pattern_no_slash = normalized_pattern.rstrip("/") - if normalized_pattern_no_slash and normalized_pattern_no_slash == normalized_url_no_slash: - return True - return normalized_pattern in normalized_url - - -class Folder(Store): - """""" - - # Track which locations have already been migrated to avoid repeated migrations - _migrated_locations: set[str] = set() - # Cache scan results to avoid repeated full scans across repeated instantiations - _scan_cache: Dict[str, - Tuple[bool, - str, - Dict[str, - int]]] = {} - - @classmethod - def config_schema(cls) -> List[Dict[str, Any]]: - return [ - { - "key": "NAME", - "label": "Store Name", - "default": "", - "required": True - }, - { - "key": "PATH", - "label": "Folder Path", - "default": "", - "required": True - } - ] - - def __init__( - self, - location: Optional[str] = None, - name: Optional[str] = None, - *, - NAME: Optional[str] = None, - PATH: Optional[str] = None, - ) -> None: - log("WARNING: Folder store is DEPRECATED and will be removed in a future version. Please migrate to HydrusNetwork.") - if name is None and NAME is not None: - name = str(NAME) - if location is None and PATH is not None: - location = str(PATH) - - self._location = str(location) if location is not None else "" - self._name = name - - # Scan status (set during init) - self.scan_ok: bool = True - self.scan_detail: str = "" - self.scan_stats: Dict[str, - int] = {} - - if self._location: - try: - from API.folder import API_folder_store - from API.folder import LocalLibraryInitializer - - location_path = expand_path(self._location) - - # Use context manager to ensure connection is properly closed - with API_folder_store(location_path) as db: - if db.connection: - db.connection.commit() - - # Call migration and discovery at startup - Folder.migrate_location(self._location) - - # Local library scan/index (one-time per location per process) - location_key = str(location_path) - cached = Folder._scan_cache.get(location_key) - if cached is None: - try: - debug(f"[folder] Initializing library scan for {location_path}...") - initializer = LocalLibraryInitializer(location_path) - stats = initializer.scan_and_index() or {} - debug(f"[folder] Scan complete. Stats: {stats}") - files_new = int(stats.get("files_new", 0) or 0) - sidecars = int(stats.get("sidecars_imported", 0) or 0) - total_db = int(stats.get("files_total_db", 0) or 0) - if files_new > 0 or sidecars > 0: - detail = f"New: {files_new}, Sidecars: {sidecars}" + ( - f" (Total: {total_db})" if total_db else "" - ) - else: - detail = "Up to date" + ( - f" (Total: {total_db})" if total_db else "" - ) - Folder._scan_cache[location_key] = (True, detail, dict(stats)) - except Exception as exc: - Folder._scan_cache[location_key] = ( - False, - f"Scan failed: {exc}", - {} - ) - - ok, detail, stats = Folder._scan_cache.get(location_key, (True, "", {})) - self.scan_ok = bool(ok) - self.scan_detail = str(detail or "") - self.scan_stats = dict(stats or {}) - except Exception as exc: - debug(f"Failed to initialize database for '{name}': {exc}") - - @classmethod - def migrate_location(cls, location: Optional[str]) -> None: - """Migrate a location to hash-based storage (one-time operation, call explicitly at startup).""" - if not location: - return - - location_path = expand_path(location) - location_str = str(location_path) - - # Only migrate once per location - if location_str in cls._migrated_locations: - return - - cls._migrated_locations.add(location_str) - - cls._migrate_to_hash_storage(location_path) - - @classmethod - def _migrate_to_hash_storage(cls, location_path: Path) -> None: - """Migrate existing files from filename-based to hash-based storage. - - Checks for sidecars (.metadata, .tag) and imports them before renaming. - Also ensures all files have a title: tag. - """ - from API.folder import API_folder_store, read_sidecar, find_sidecar - - try: - with API_folder_store(location_path) as db: - cursor = db.connection.cursor() - - # First pass: migrate filename-based files and add title tags - # Scan all files in the storage directory - for file_path in sorted(location_path.iterdir()): - if not file_path.is_file(): - continue - - # Skip database files and sidecars - if file_path.suffix in (".db", ".metadata", ".tag", "-shm", "-wal"): - continue - # Also skip if the file ends with -shm or -wal (SQLite journal files) - if file_path.name.endswith(("-shm", "-wal")): - continue - - # Check if filename is already a hash (without extension) - if len(file_path.stem) == 64 and all( - c in "0123456789abcdef" for c in file_path.stem.lower()): - continue # Already migrated, will process in second pass - - try: - # Compute file hash - file_hash = sha256_file(file_path) - # Preserve extension in the hash-based filename - file_ext = file_path.suffix # e.g., '.mp4' - hash_filename = file_hash + file_ext if file_ext else file_hash - hash_path = location_path / hash_filename - - # Check for sidecars and import them - sidecar_path = find_sidecar(file_path) - tags_to_add = [] - url_to_add = [] - has_title_tag = False - - if sidecar_path and sidecar_path.exists(): - try: - _, tags, url = read_sidecar(sidecar_path) - if tags: - tags_to_add = list(tags) - # Check if title tag exists - has_title_tag = any( - t.lower().startswith("title:") - for t in tags_to_add - ) - if url: - url_to_add = list(url) - debug( - f"Found sidecar for {file_path.name}: {len(tags_to_add)} tags, {len(url_to_add)} url", - file=sys.stderr, - ) - # Delete the sidecar after importing - sidecar_path.unlink() - except Exception as exc: - debug( - f"Failed to read sidecar for {file_path.name}: {exc}", - file=sys.stderr, - ) - - # Ensure there's a title tag (use original filename if not present) - if not has_title_tag: - tags_to_add.append(f"title:{file_path.name}") - - # Rename file to hash if needed - if hash_path != file_path and not hash_path.exists(): - debug( - f"Migrating: {file_path.name} -> {hash_filename}", - file=sys.stderr - ) - file_path.rename(hash_path) - - # Ensure DB points to the renamed path (update by hash). - try: - cursor.execute( - "UPDATE file SET file_path = ?, updated_at = CURRENT_TIMESTAMP WHERE hash = ?", - (db._to_db_file_path(hash_path), - file_hash), - ) - except Exception: - pass - - # Create or update database entry - db.get_or_create_file_entry(hash_path) - - # Save extension metadata - ext_clean = file_ext.lstrip(".") if file_ext else "" - db.save_metadata( - hash_path, - { - "hash": file_hash, - "ext": ext_clean, - "size": hash_path.stat().st_size, - }, - ) - - # Add all tags (including title tag) - if tags_to_add: - db.save_tags(hash_path, tags_to_add) - debug( - f"Added {len(tags_to_add)} tags to {file_hash}", - file=sys.stderr - ) - - # Note: url would need a separate table if you want to store them - # For now, we're just noting them in debug - if url_to_add: - debug( - f"Imported {len(url_to_add)} url for {file_hash}: {url_to_add}", - file=sys.stderr, - ) - - except Exception as exc: - debug( - f"Failed to migrate file {file_path.name}: {exc}", - file=sys.stderr - ) - - # Second pass: ensure all files in database have a title: tag - db.connection.commit() - cursor.execute( - """ - SELECT f.hash, f.file_path - FROM file f - WHERE NOT EXISTS ( - SELECT 1 FROM tag t WHERE t.hash = f.hash AND LOWER(t.tag) LIKE 'title:%' - ) - """ - ) - files_without_title = cursor.fetchall() - - for file_hash, file_path_str in files_without_title: - try: - file_path = location_path / str(file_path_str) - if file_path.exists(): - # Use the filename as the title - title_tag = f"title:{file_path.name}" - db.save_tags(file_path, [title_tag]) - debug( - f"Added title tag to {file_path.name}", - file=sys.stderr - ) - except Exception as exc: - debug( - f"Failed to add title tag to file {file_path_str}: {exc}", - file=sys.stderr, - ) - - db.connection.commit() - - # Third pass: discover files on disk that aren't in the database yet - # These are hash-named files that were added after initial indexing - cursor.execute("SELECT LOWER(hash) FROM file") - db_hashes = {row[0] - for row in cursor.fetchall()} - - discovered = 0 - for file_path in sorted(location_path.rglob("*")): - if file_path.is_file(): - # Check if file name (without extension) is a 64-char hex hash - name_without_ext = file_path.stem - if len(name_without_ext) == 64 and all( - c in "0123456789abcdef" - for c in name_without_ext.lower()): - file_hash = name_without_ext.lower() - - # Skip if already in DB - if file_hash in db_hashes: - continue - - try: - # Add file to DB (creates entry and auto-adds title: tag) - db.get_or_create_file_entry(file_path) - - # Save extension metadata - file_ext = file_path.suffix - ext_clean = file_ext.lstrip(".") if file_ext else "" - db.save_metadata( - file_path, - { - "hash": file_hash, - "ext": ext_clean, - "size": file_path.stat().st_size, - }, - ) - - discovered += 1 - except Exception as e: - debug( - f"Failed to discover file {file_path.name}: {e}", - file=sys.stderr, - ) - - if discovered > 0: - debug( - f"Discovered and indexed {discovered} undiscovered files in {location_path.name}", - file=sys.stderr, - ) - db.connection.commit() - except Exception as exc: - debug(f"Migration to hash storage failed: {exc}", file=sys.stderr) - - def location(self) -> str: - return self._location - - def name(self) -> str: - return self._name - - def add_file(self, file_path: Path, **kwargs: Any) -> str: - """Add file to local folder storage with full metadata support. - - Args: - file_path: Path to the file to add - move: If True, move file instead of copy (default: False) - tag: Optional list of tag values to add - url: Optional list of url to associate with the file - title: Optional title (will be added as 'title:value' tag) - file_hash: Optional pre-calculated SHA256 hash (skips re-hashing) - - Returns: - File hash (SHA256 hex string) as identifier - """ - move_file = bool(kwargs.get("move")) - tag_list = kwargs.get("tag", []) - url = kwargs.get("url", []) - title = kwargs.get("title") - file_hash = kwargs.get("file_hash") - - # Extract title from tags if not explicitly provided - if not title: - for candidate in tag_list: - if isinstance(candidate, - str) and candidate.lower().startswith("title:"): - title = candidate.split(":", 1)[1].strip() - break - - # Fallback to filename if no title - if not title: - title = file_path.name - - # Ensure title is in tags - title_tag = f"title:{title}" - if not any(str(candidate).lower().startswith("title:") - for candidate in tag_list): - tag_list = [title_tag] + list(tag_list) - - try: - if not file_hash or len(str(file_hash)) != 64: - debug(f"[folder] Re-hashing file: {file_path}", file=sys.stderr) - file_hash = sha256_file(file_path) - - debug(f"File hash: {file_hash}", file=sys.stderr) - - # Preserve extension in the stored filename - file_ext = file_path.suffix # e.g., '.mp4' - save_filename = file_hash + file_ext if file_ext else file_hash - save_file = Path(self._location) / save_filename - - # Check if file already exists - from API.folder import API_folder_store - - with API_folder_store(Path(self._location)) as db: - existing_path = db.search_hash(file_hash) - if existing_path and existing_path.exists(): - log( - f"āœ“ File already in local storage: {existing_path}", - file=sys.stderr, - ) - # Still add tags and url if provided - if tag_list: - self.add_tag(file_hash, tag_list) - if url: - self.add_url(file_hash, url) - return file_hash - - # Move or copy file (with progress bar on actual byte transfer). - # Note: a same-volume move may be a fast rename and won't show progress. - def _copy_with_progress(src: Path, dst: Path, *, label: str) -> None: - from SYS.models import ProgressFileReader - - total_bytes = None - try: - total_bytes = int(src.stat().st_size) - except Exception: - total_bytes = None - - with src.open("rb") as r, dst.open("wb") as w: - reader = ProgressFileReader(r, total_bytes=total_bytes, label=label) - while True: - chunk = reader.read(1024 * 1024) - if not chunk: - break - w.write(chunk) - - # Preserve file metadata similar to shutil.copy2 - try: - shutil.copystat(str(src), str(dst)) - except Exception: - pass - - if move_file: - # Prefer native move; fall back to copy+delete with progress on failure. - try: - shutil.move(str(file_path), str(save_file)) - debug(f"Local move: {save_file}", file=sys.stderr) - # After a move, the original path no longer exists; use destination for subsequent ops. - file_path = save_file - except Exception: - _copy_with_progress( - file_path, - save_file, - label=f"folder:{self._name} move" - ) - try: - file_path.unlink(missing_ok=True) # type: ignore[arg-type] - except Exception: - try: - if file_path.exists(): - file_path.unlink() - except Exception: - pass - debug(f"Local move (copy+delete): {save_file}", file=sys.stderr) - file_path = save_file - else: - _copy_with_progress( - file_path, - save_file, - label=f"folder:{self._name} copy" - ) - debug(f"Local copy: {save_file}", file=sys.stderr) - - # Best-effort: capture duration for media - duration_value: float | None = None - try: - from SYS.utils import ffprobe - - probe = ffprobe(str(save_file)) - duration = probe.get("duration") - if isinstance(duration, (int, float)) and duration > 0: - duration_value = float(duration) - except Exception: - duration_value = None - - # Save to database (metadata + tag/url updates share one connection) - with API_folder_store(Path(self._location)) as db: - conn = getattr(db, "connection", None) - if conn is None: - raise RuntimeError("Folder store DB connection unavailable") - cursor = conn.cursor() - - debug( - f"[Folder.add_file] saving metadata for hash {file_hash}", - file=sys.stderr, - ) - ext_clean = file_ext.lstrip(".") if file_ext else "" - db.save_metadata( - save_file, - { - "hash": file_hash, - "ext": ext_clean, - "size": save_file.stat().st_size, - "duration": duration_value, - }, - ) - debug( - f"[Folder.add_file] metadata stored for hash {file_hash}", - file=sys.stderr, - ) - - if tag_list: - try: - debug( - f"[Folder.add_file] merging {len(tag_list)} tags for {file_hash}", - file=sys.stderr, - ) - from SYS.metadata import compute_namespaced_tag_overwrite - - existing_tags = [ - t for t in (db.get_tags(file_hash) or []) - if isinstance(t, str) and t.strip() - ] - _to_remove, _to_add, merged = compute_namespaced_tag_overwrite( - existing_tags, tag_list or [] - ) - if _to_remove or _to_add: - cursor.execute("DELETE FROM tag WHERE hash = ?", - (file_hash,)) - for t in merged: - tag_val = str(t).strip().lower() - if tag_val: - cursor.execute( - "INSERT OR IGNORE INTO tag (hash, tag) VALUES (?, ?)", - (file_hash, tag_val), - ) - conn.commit() - debug( - f"[Folder.add_file] tags rewritten for {file_hash}", - file=sys.stderr, - ) - try: - db._update_metadata_modified_time(file_hash) - except Exception: - pass - except Exception as exc: - debug(f"Local DB tag merge failed: {exc}", file=sys.stderr) - - if url: - try: - from SYS.metadata import normalize_urls - - existing_meta = db.get_metadata(file_hash) or {} - existing_urls = normalize_urls(existing_meta.get("url")) - incoming_urls = normalize_urls(url) - debug( - f"[Folder.add_file] merging {len(incoming_urls)} URLs for {file_hash}: {incoming_urls}", - file=sys.stderr, - ) - changed = False - for entry in list(incoming_urls or []): - if not entry: - continue - if entry not in existing_urls: - existing_urls.append(entry) - changed = True - if changed: - db.update_metadata_by_hash( - file_hash, - {"url": existing_urls}, - ) - debug( - f"[Folder.add_file] URLs merged for {file_hash}: {existing_urls}", - file=sys.stderr, - ) - except Exception as exc: - debug(f"Local DB URL merge failed: {exc}", file=sys.stderr) - - ##log(f"āœ“ Added to local storage: {save_file.name}", file=sys.stderr) - return file_hash - - except Exception as exc: - log(f"āŒ Local storage failed: {exc}", file=sys.stderr) - raise - - def search(self, query: str, **kwargs: Any) -> list[Dict[str, Any]]: - """Search local database for files by title tag or filename.""" - from fnmatch import fnmatch - from API.folder import DatabaseAPI - import unicodedata - - limit = kwargs.get("limit") - try: - limit = int(limit) if limit is not None else None - except (TypeError, ValueError): - limit = None - if isinstance(limit, int) and limit <= 0: - limit = None - - query = query.lower() - query_lower = query # Ensure query_lower is defined for all code paths - - def _normalize_namespace_text(text: str, *, allow_wildcards: bool) -> str: - """Normalize tag namespace values for consistent matching. - - Removes control/format chars (e.g. zero-width spaces) that frequently appear in scraped tags, - collapses whitespace, and lowercases. - """ - s = str(text or "") - # Normalize newlines/tabs/etc to spaces early. - s = s.replace("\r", " ").replace("\n", " ").replace("\t", " ") - # Drop control / format chars (Cc/Cf) while preserving wildcard tokens when requested. - cleaned_chars: list[str] = [] - for ch in s: - if allow_wildcards and ch in {"*", - "?"}: - cleaned_chars.append(ch) - continue - cat = unicodedata.category(ch) - if cat in {"Cc", - "Cf"}: - continue - cleaned_chars.append(ch) - s = "".join(cleaned_chars) - # Collapse any remaining unicode whitespace runs. - s = " ".join(s.split()) - return s.strip().lower() - - def _normalize_ext_filter(value: str) -> str: - v = str(value or "").strip().lower().lstrip(".") - v = "".join(ch for ch in v if ch.isalnum()) - return v - - def _extract_system_filetype_ext(text: str) -> Optional[str]: - # Match: system:filetype = png (allow optional '=' and flexible spaces) - m = re.search(r"\bsystem:filetype\s*(?:=\s*)?([^\s,]+)", text) - if not m: - m = re.search(r"\bsystem:filetype\s*=\s*([^\s,]+)", text) - if not m: - return None - return _normalize_ext_filter(m.group(1)) or None - - # Support `ext:` and Hydrus-style `system:filetype = ` anywhere - # in the query (space or comma separated). - ext_filter: Optional[str] = None - try: - sys_ext = _extract_system_filetype_ext(query_lower) - if sys_ext: - ext_filter = sys_ext - query_lower = re.sub( - r"\s*\bsystem:filetype\s*(?:=\s*)?[^\s,]+", - " ", - query_lower - ) - query_lower = re.sub(r"\s{2,}", " ", query_lower).strip().strip(",") - query = query_lower - - m = re.search(r"\bext:([^\s,]+)", query_lower) - if not m: - m = re.search(r"\bextension:([^\s,]+)", query_lower) - if m: - ext_filter = _normalize_ext_filter(m.group(1)) or None - query_lower = re.sub( - r"\s*\b(?:ext|extension):[^\s,]+", - " ", - query_lower - ) - query_lower = re.sub(r"\s{2,}", " ", query_lower).strip().strip(",") - query = query_lower - except Exception: - ext_filter = None - - match_all = query == "*" or (not query and bool(ext_filter)) - results = [] - search_dir = expand_path(self._location) - backend_label = str( - getattr(self, "_name", "") or getattr(self, "NAME", "") or "folder" - ) - debug( - f"[folder:{backend_label}] search start: query={query} limit={limit} root={search_dir}" - ) - - def _url_like_pattern(value: str) -> str: - # Interpret user patterns as substring matches (with optional glob wildcards). - v = (value or "").strip().lower() - if not v or v == "*": - return "%" - v = v.replace("%", "\\%").replace("_", "\\_") - v = v.replace("*", "%").replace("?", "_") - if "%" not in v and "_" not in v: - return f"%{v}%" - if not v.startswith("%"): - v = "%" + v - if not v.endswith("%"): - v = v + "%" - return v - - def _like_pattern(term: str) -> str: - # Convert glob-like tokens to SQL LIKE wildcards. - return str(term or "").replace("*", "%").replace("?", "_") - - tokens = [t.strip() for t in query.split(",") if t.strip()] - - if not match_all and len(tokens) == 1 and _normalize_hash(query): - debug("Hash queries require 'hash:' prefix for local search") - return results - - if not match_all and _normalize_hash(query): - debug("Hash queries require 'hash:' prefix for local search") - return results - - def _create_entry( - file_path: Path, - tags: list[str], - size_bytes: int | None, - db_hash: Optional[str] - ) -> dict[str, - Any]: - path_str = str(file_path) - # Get title from tags if available, otherwise use hash as fallback - title = next( - (t.split(":", - 1)[1] for t in tags if t.lower().startswith("title:")), - None - ) - if not title: - # Fallback to hash if no title tag exists - hash_value = _resolve_file_hash(db_hash, file_path) - title = hash_value if hash_value else file_path.stem - - # Extract extension from file path - ext = file_path.suffix.lstrip(".") - if not ext: - # Fallback: try to extract from title (original filename might be in title) - title_path = Path(title) - ext = title_path.suffix.lstrip(".") - - # Build clean entry with only necessary fields - hash_value = _resolve_file_hash(db_hash, file_path) - entry = { - "title": title, - "ext": ext, - "path": path_str, - "target": path_str, - "store": self._name, - "size": size_bytes, - "hash": hash_value, - "tag": tags, - } - return entry - - try: - if not search_dir.exists(): - debug(f"Search directory does not exist: {search_dir}") - return results - - try: - with DatabaseAPI(search_dir) as api: - ext_hashes: set[str] | None = None - if ext_filter: - # Fetch a bounded set of hashes to intersect with other filters. - ext_fetch_limit = (limit or 45) * 50 - ext_hashes = api.get_file_hashes_by_ext( - ext_filter, - limit=ext_fetch_limit - ) - - # ext-only search: query is empty (or coerced to match_all above). - if ext_filter and (not query_lower or query_lower == "*"): - rows = api.get_files_by_ext(ext_filter, limit) - for file_hash, file_path_str, size_bytes, ext in rows: - if not file_path_str: - continue - file_path = search_dir / str(file_path_str) - if not file_path.exists(): - continue - if size_bytes is None: - try: - size_bytes = file_path.stat().st_size - except OSError: - size_bytes = None - tags = api.get_tags_for_file(file_hash) - entry = _create_entry( - file_path, - tags, - size_bytes, - file_hash - ) - try: - db_ext = str(ext or "").strip().lstrip(".") - if db_ext: - entry["ext"] = db_ext - except Exception: - pass - results.append(entry) - if limit is not None and len(results) >= limit: - return results - backend_label = str( - getattr(self, - "_name", - "") or getattr(self, - "NAME", - "") or "folder" - ) - debug(f"[folder:{backend_label}] {len(results)} result(s)") - return results - - if tokens and len(tokens) > 1: - url_fetch_limit = (limit or 45) * 50 - - def _ids_for_token(token: str) -> set[int]: - token = token.strip() - if not token: - return set() - - if ":" in token and not token.startswith(":"): - namespace, pattern = token.split(":", 1) - namespace = namespace.strip().lower() - pattern = pattern.strip().lower() - - if namespace == "hash": - normalized_hash = _normalize_hash(pattern) - if not normalized_hash: - return set() - h = api.get_file_hash_by_hash(normalized_hash) - return {h} if h else set() - - if namespace == "url": - if not pattern or pattern == "*": - return api.get_file_hashes_with_any_url( - limit=url_fetch_limit - ) - return api.get_file_hashes_by_url_like( - _url_like_pattern(pattern), - limit=url_fetch_limit - ) - - if namespace == "system": - # Hydrus-compatible query: system:filetype = png - m_ft = re.match( - r"^filetype\s*(?:=\s*)?(.+)$", - pattern - ) - if m_ft: - normalized_ext = _normalize_ext_filter( - m_ft.group(1) - ) - if not normalized_ext: - return set() - return api.get_file_hashes_by_ext( - normalized_ext, - limit=url_fetch_limit - ) - return set() - - if namespace in {"ext", - "extension"}: - normalized_ext = _normalize_ext_filter(pattern) - if not normalized_ext: - return set() - return api.get_file_hashes_by_ext( - normalized_ext, - limit=url_fetch_limit - ) - - if namespace == "store": - if pattern not in {"local", - "file", - "filesystem"}: - return set() - return api.get_all_file_hashes() - - query_pattern = f"{namespace}:%" - tag_rows = api.get_file_hashes_by_tag_pattern( - query_pattern - ) - matched: set[str] = set() - for file_hash, tag_val in tag_rows: - if not tag_val: - continue - tag_lower = str(tag_val).lower() - if not tag_lower.startswith(f"{namespace}:"): - continue - value = _normalize_namespace_text( - tag_lower[len(namespace) + 1:], - allow_wildcards=False - ) - pat = _normalize_namespace_text( - pattern, - allow_wildcards=True - ) - if fnmatch(value, pat): - matched.add(file_hash) - return matched - - term = token.lower() - like_pattern = f"%{_like_pattern(term)}%" - # Unqualified token: match file path, title: tags, and non-namespaced tags. - # Do NOT match other namespaces by default (e.g., artist:men at work). - hashes = set( - api.get_file_hashes_by_path_pattern(like_pattern) - or set() - ) - - try: - title_rows = api.get_files_by_namespace_pattern( - f"title:{like_pattern}", - url_fetch_limit - ) - hashes.update( - { - row[0] - for row in (title_rows or []) if row and row[0] - } - ) - except Exception: - pass - - try: - simple_rows = api.get_files_by_simple_tag_pattern( - like_pattern, - url_fetch_limit - ) - hashes.update( - { - row[0] - for row in (simple_rows or []) if row and row[0] - } - ) - except Exception: - pass - - return hashes - - try: - matching_hashes: set[str] | None = None - for token in tokens: - hashes = _ids_for_token(token) - matching_hashes = ( - hashes if matching_hashes is None else - matching_hashes & hashes - ) - if not matching_hashes: - return results - - if ext_hashes is not None: - matching_hashes = ( - matching_hashes or set() - ) & ext_hashes - if not matching_hashes: - return results - - if not matching_hashes: - return results - - rows = api.get_file_metadata(matching_hashes, limit) - for file_hash, file_path_str, size_bytes, ext in rows: - if not file_path_str: - continue - file_path = search_dir / str(file_path_str) - if not file_path.exists(): - continue - if size_bytes is None: - try: - size_bytes = file_path.stat().st_size - except OSError: - size_bytes = None - tags = api.get_tags_for_file(file_hash) - entry = _create_entry( - file_path, - tags, - size_bytes, - file_hash - ) - try: - db_ext = str(ext or "").strip().lstrip(".") - if db_ext: - entry["ext"] = db_ext - except Exception: - pass - results.append(entry) - if limit is not None and len(results) >= limit: - return results - return results - except Exception as exc: - log(f"āš ļø AND search failed: {exc}", file=sys.stderr) - debug(f"AND search exception details: {exc}") - return [] - - if ":" in query and not query.startswith(":"): - namespace, pattern = query.split(":", 1) - namespace = namespace.strip().lower() - pattern = pattern.strip().lower() - debug(f"[folder:{backend_label}] namespace search: {namespace}:{pattern}") - - if namespace == "hash": - normalized_hash = _normalize_hash(pattern) - if not normalized_hash: - return results - h = api.get_file_hash_by_hash(normalized_hash) - hashes = {h} if h else set() - rows = api.get_file_metadata(hashes, limit) - for file_hash, file_path_str, size_bytes, ext in rows: - if not file_path_str: - continue - file_path = search_dir / str(file_path_str) - if not file_path.exists(): - continue - if size_bytes is None: - try: - size_bytes = file_path.stat().st_size - except OSError: - size_bytes = None - tags = api.get_tags_for_file(file_hash) - entry = _create_entry( - file_path, - tags, - size_bytes, - file_hash - ) - try: - db_ext = str(ext or "").strip().lstrip(".") - if db_ext: - entry["ext"] = db_ext - except Exception: - pass - results.append(entry) - if limit is not None and len(results) >= limit: - return results - return results - - if namespace == "url": - pattern_hint = kwargs.get("pattern_hint") - - def _pattern_candidates(raw: Any) -> List[str]: - if raw is None: - return [] - if isinstance(raw, (list, tuple, set)): - out: List[str] = [] - for item in raw: - text = str(item or "").strip() - if text and text not in out: - out.append(text) - return out - if isinstance(raw, str): - text = raw.strip() - return [text] if text else [] - return [] - - pattern_candidates = _pattern_candidates(pattern_hint) - if len(pattern_candidates) > 200: - pattern_candidates = pattern_candidates[:200] - - def _parse_url_value(raw: Any) -> list[str]: - if raw is None: - return [] - if isinstance(raw, list): - return [str(u).strip() for u in raw if str(u).strip()] - if isinstance(raw, str): - text = raw.strip() - if not text: - return [] - try: - parsed = json.loads(text) - if isinstance(parsed, list): - return [ - str(u).strip() - for u in parsed - if str(u).strip() - ] - except Exception: - pass - return [text] - return [] - - def _matches_pattern(url_list: list[str]) -> bool: - if not pattern_candidates: - return True - for candidate_url in url_list: - for pat in pattern_candidates: - if _match_url_pattern(candidate_url, pat): - return True - return False - - if not pattern or pattern == "*": - if pattern_candidates: - debug( - f"[folder:{backend_label}] url search: any-url (limit={limit}) pattern_hint={len(pattern_candidates)}" - ) - rows = api.get_files_by_url_like_any( - [_url_like_pattern(p) for p in pattern_candidates], - limit, - ) - else: - debug(f"[folder:{backend_label}] url search: any-url (limit={limit})") - rows = api.get_files_with_any_url(limit) - else: - debug( - f"[folder:{backend_label}] url search: like={pattern} (limit={limit})" - ) - rows = api.get_files_by_url_like( - _url_like_pattern(pattern), - limit - ) - for file_hash, file_path_str, size_bytes, ext, url_raw in rows: - if not file_path_str: - continue - file_path = search_dir / str(file_path_str) - if not file_path.exists(): - continue - if size_bytes is None: - try: - size_bytes = file_path.stat().st_size - except OSError: - size_bytes = None - urls = _parse_url_value(url_raw) - if not urls or not _matches_pattern(urls): - continue - tags = api.get_tags_for_file(file_hash) - entry = _create_entry( - file_path, - tags, - size_bytes, - file_hash - ) - entry["urls"] = urls - results.append(entry) - if limit is not None and len(results) >= limit: - return results - return results - - if namespace == "system": - # Hydrus-compatible query: system:filetype = png - m_ft = re.match(r"^filetype\s*(?:=\s*)?(.+)$", pattern) - if m_ft: - normalized_ext = _normalize_ext_filter(m_ft.group(1)) - if not normalized_ext: - return results - rows = api.get_files_by_ext(normalized_ext, limit) - for file_hash, file_path_str, size_bytes, ext in rows: - if not file_path_str: - continue - file_path = search_dir / str(file_path_str) - if not file_path.exists(): - continue - if size_bytes is None: - try: - size_bytes = file_path.stat().st_size - except OSError: - size_bytes = None - tags = api.get_tags_for_file(file_hash) - entry = _create_entry( - file_path, - tags, - size_bytes, - file_hash - ) - try: - db_ext = str(ext or "").strip().lstrip(".") - if db_ext: - entry["ext"] = db_ext - except Exception: - pass - results.append(entry) - if limit is not None and len(results) >= limit: - return results - return results - - if namespace in {"ext", - "extension"}: - normalized_ext = _normalize_ext_filter(pattern) - if not normalized_ext: - return results - rows = api.get_files_by_ext(normalized_ext, limit) - for file_hash, file_path_str, size_bytes, ext in rows: - if not file_path_str: - continue - file_path = search_dir / str(file_path_str) - if not file_path.exists(): - continue - if size_bytes is None: - try: - size_bytes = file_path.stat().st_size - except OSError: - size_bytes = None - tags = api.get_tags_for_file(file_hash) - entry = _create_entry( - file_path, - tags, - size_bytes, - file_hash - ) - try: - db_ext = str(ext or "").strip().lstrip(".") - if db_ext: - entry["ext"] = db_ext - except Exception: - pass - results.append(entry) - if limit is not None and len(results) >= limit: - return results - return results - - query_pattern = f"{namespace}:%" - rows = api.get_files_by_namespace_pattern(query_pattern, limit) - debug(f"Found {len(rows)} potential matches in DB") - - for file_hash, file_path_str, size_bytes, ext in rows: - if not file_path_str: - continue - - tags = api.get_tags_by_namespace_and_file( - file_hash, - query_pattern - ) - - for tag in tags: - tag_lower = tag.lower() - if tag_lower.startswith(f"{namespace}:"): - value = _normalize_namespace_text( - tag_lower[len(namespace) + 1:], - allow_wildcards=False - ) - pat = _normalize_namespace_text( - pattern, - allow_wildcards=True - ) - if fnmatch(value, pat): - if ext_hashes is not None and file_hash not in ext_hashes: - break - file_path = search_dir / str(file_path_str) - if file_path.exists(): - if size_bytes is None: - size_bytes = file_path.stat().st_size - all_tags = api.get_tags_for_file(file_hash) - entry = _create_entry( - file_path, - all_tags, - size_bytes, - file_hash - ) - try: - db_ext = str(ext - or "").strip().lstrip(".") - if db_ext: - entry["ext"] = db_ext - except Exception: - pass - results.append(entry) - else: - debug(f"File missing on disk: {file_path}") - break - - if limit is not None and len(results) >= limit: - return results - elif not match_all: - # Default (unqualified) search: AND semantics across terms. - # Each term must match at least one of: - # - file path (filename) - # - title: namespace tag - # - non-namespaced tag - # Other namespaces (artist:, series:, etc.) are excluded unless explicitly queried. - terms = [ - t.strip() for t in query_lower.replace(",", " ").split() - if t.strip() - ] - if not terms: - terms = [query_lower] - - fetch_limit = (limit or 45) * 50 - - matching_hashes: Optional[set[str]] = None - for term in terms: - if not term: - continue - like_term = _like_pattern(term) - like_pattern = f"%{like_term}%" - - term_hashes: set[str] = set() - try: - term_hashes.update( - api.get_file_hashes_by_path_pattern(like_pattern) - ) - except Exception: - pass - - try: - title_rows = api.get_files_by_namespace_pattern( - f"title:{like_pattern}", - fetch_limit - ) - term_hashes.update( - { - row[0] - for row in (title_rows or []) if row and row[0] - } - ) - except Exception: - pass - - try: - simple_rows = api.get_files_by_simple_tag_pattern( - like_pattern, - fetch_limit - ) - term_hashes.update( - { - row[0] - for row in (simple_rows or []) if row and row[0] - } - ) - except Exception: - pass - - if ext_hashes is not None: - term_hashes &= ext_hashes - - matching_hashes = ( - term_hashes if matching_hashes is None else - (matching_hashes & term_hashes) - ) - if not matching_hashes: - return results - - if not matching_hashes: - return results - - rows = api.get_file_metadata(set(matching_hashes), limit) - for file_hash, file_path_str, size_bytes, ext in rows: - if not file_path_str: - continue - file_path = search_dir / str(file_path_str) - if not file_path.exists(): - continue - if size_bytes is None: - try: - size_bytes = file_path.stat().st_size - except OSError: - size_bytes = None - tags = api.get_tags_for_file(file_hash) - entry_obj = _create_entry( - file_path, - tags, - size_bytes, - file_hash - ) - try: - db_ext = str(ext or "").strip().lstrip(".") - if db_ext: - entry_obj["ext"] = db_ext - except Exception: - pass - results.append(entry_obj) - if limit is not None and len(results) >= limit: - break - - else: - rows = api.get_all_files(limit) - for file_hash, file_path_str, size_bytes, ext in rows: - if file_path_str: - if ext_hashes is not None and file_hash not in ext_hashes: - continue - file_path = search_dir / str(file_path_str) - if file_path.exists(): - if size_bytes is None: - size_bytes = file_path.stat().st_size - - tags = api.get_tags_for_file(file_hash) - entry = _create_entry( - file_path, - tags, - size_bytes, - file_hash - ) - try: - db_ext = str(ext or "").strip().lstrip(".") - if db_ext: - entry["ext"] = db_ext - except Exception: - pass - results.append(entry) - - backend_label = str( - getattr(self, - "_name", - "") or getattr(self, - "NAME", - "") or "folder" - ) - debug(f"[folder:{backend_label}] {len(results)} result(s)") - return results - - except Exception as e: - log(f"āš ļø Database search failed: {e}", file=sys.stderr) - debug(f"DB search exception details: {e}") - return [] - - except Exception as exc: - log(f"āŒ Local search failed: {exc}", file=sys.stderr) - raise - - def _resolve_library_root(self, - file_path: Path, - config: Dict[str, - Any]) -> Optional[Path]: - """Return the library root containing medios-macina.db. - - Prefer the store's configured location, then config override, then walk parents - of the file path to find a directory with medios-macina.db.""" - candidates: list[Path] = [] - if self._location: - candidates.append(expand_path(self._location)) - cfg_root = get_local_storage_path(config) if config else None - if cfg_root: - candidates.append(expand_path(cfg_root)) - - for root in candidates: - db_path = root / "medios-macina.db" - if db_path.exists(): - return root - - try: - for parent in [file_path] + list(file_path.parents): - db_path = parent / "medios-macina.db" - if db_path.exists(): - return parent - except Exception: - pass - return None - - def get_file(self, file_hash: str, **kwargs: Any) -> Optional[Path]: - """Retrieve file by hash, returning path to the file. - - Args: - file_hash: SHA256 hash of the file (64-char hex string) - - Returns: - Path to the file or None if not found - """ - try: - # Normalize the hash - normalized_hash = _normalize_hash(file_hash) - if not normalized_hash: - return None - - search_dir = expand_path(self._location) - from API.folder import API_folder_store - - with API_folder_store(search_dir) as db: - # Search for file by hash - file_path = db.search_hash(normalized_hash) - - if file_path and file_path.exists(): - return file_path - - return None - - except Exception as exc: - debug(f"Failed to get file for hash {file_hash}: {exc}") - return None - - def get_metadata(self, file_hash: str, **kwargs: Any) -> Optional[Dict[str, Any]]: - """Get metadata for a file from the database by hash. - - Args: - file_hash: SHA256 hash of the file (64-char hex string) - **kwargs: Additional options - - Returns: - Dict with metadata fields (ext, size, hash, duration, etc.) or None if not found - """ - try: - # Normalize the hash - normalized_hash = _normalize_hash(file_hash) - if not normalized_hash: - return None - - search_dir = expand_path(self._location) - from API.folder import DatabaseAPI - - with DatabaseAPI(search_dir) as api: - # Get file hash - file_hash_result = api.get_file_hash_by_hash(normalized_hash) - if not file_hash_result: - return None - - # Query metadata directly from database - cursor = api.get_cursor() - cursor.execute( - """ - SELECT * FROM metadata WHERE hash = ? - """, - (file_hash_result, - ), - ) - - row = cursor.fetchone() - if not row: - return None - - metadata = dict(row) - - # Canonicalize metadata keys (no legacy aliases) - if "file_path" in metadata and "path" not in metadata: - metadata["path"] = metadata.get("file_path") - metadata.pop("file_path", None) - - # Parse JSON fields - for field in ["url", "relationships"]: - if metadata.get(field): - try: - metadata[field] = json.loads(metadata[field]) - except (json.JSONDecodeError, TypeError): - metadata[field] = [] if field == "url" else [] - - return metadata - except Exception as exc: - debug(f"Failed to get metadata for hash {file_hash}: {exc}") - return None - - def set_relationship(self, alt_hash: str, king_hash: str, kind: str = "alt") -> bool: - """Persist a relationship in the folder store DB. - - This is a thin wrapper around the folder DB API so cmdlets can avoid - backend-specific branching. - """ - try: - if not self._location: - return False - - alt_norm = _normalize_hash(alt_hash) - king_norm = _normalize_hash(king_hash) - if not alt_norm or not king_norm or alt_norm == king_norm: - return False - - from API.folder import API_folder_store - - with API_folder_store(expand_path(self._location)) as db: - db.set_relationship_by_hash( - alt_norm, - king_norm, - str(kind or "alt"), - bidirectional=False, - ) - return True - except Exception: - return False - - def get_tag(self, file_identifier: str, **kwargs: Any) -> Tuple[List[str], str]: - """Get tags for a local file by hash. - - Returns: - Tuple of (tags_list, store_name) where store_name is the actual store name - """ - from API.folder import API_folder_store - - try: - file_hash = file_identifier - if self._location: - try: - with API_folder_store(Path(self._location)) as db: - db_tags = db.get_tags(file_hash) - if db_tags: - # Return actual store name instead of generic "local_db" - store_name = self._name if self._name else "local" - return [ - str(t).strip().lower() - for t in db_tags - if isinstance(t, str) and t.strip() - ], store_name - except Exception as exc: - debug(f"Local DB lookup failed: {exc}") - return [], "unknown" - except Exception as exc: - debug(f"get_tags failed for local file: {exc}") - return [], "unknown" - - def add_tag(self, file_identifier: str, tags: List[str], **kwargs: Any) -> bool: - """Add tags to a local file by hash (via API_folder_store). - - Handles namespace collapsing: when adding namespace:value, removes existing namespace:* tags. - Returns True if tags were successfully added. - """ - from API.folder import API_folder_store - - try: - if not self._location: - return False - - try: - with API_folder_store(Path(self._location)) as db: - existing_tags = [ - t for t in (db.get_tags(file_identifier) or []) - if isinstance(t, str) and t.strip() - ] - - from SYS.metadata import compute_namespaced_tag_overwrite - - _to_remove, _to_add, merged = compute_namespaced_tag_overwrite( - existing_tags, tags or [] - ) - if not _to_remove and not _to_add: - return True - - # Folder DB tag table is case-sensitive and add_tags_to_hash() is additive. - # To enforce lowercase-only tags and namespace overwrites, rewrite the full tag set. - cursor = db.connection.cursor() - cursor.execute("DELETE FROM tag WHERE hash = ?", - (file_identifier, - )) - for t in merged: - t = str(t).strip().lower() - if t: - cursor.execute( - "INSERT OR IGNORE INTO tag (hash, tag) VALUES (?, ?)", - (file_identifier, - t), - ) - db.connection.commit() - try: - db._update_metadata_modified_time(file_identifier) - except Exception: - pass - return True - except Exception as exc: - debug(f"Local DB add_tags failed: {exc}") - return False - except Exception as exc: - debug(f"add_tag failed for local file: {exc}") - return False - - def delete_tag(self, file_identifier: str, tags: List[str], **kwargs: Any) -> bool: - """Remove tags from a local file by hash.""" - from API.folder import API_folder_store - - try: - file_hash = file_identifier - if self._location: - try: - with API_folder_store(Path(self._location)) as db: - tag_list = [ - str(t).strip().lower() for t in (tags or []) - if isinstance(t, str) and str(t).strip() - ] - if not tag_list: - return True - db.remove_tags_from_hash(file_hash, tag_list) - return True - except Exception as exc: - debug(f"Local DB remove_tags failed: {exc}") - return False - except Exception as exc: - debug(f"delete_tag failed for local file: {exc}") - return False - - def get_url(self, file_identifier: str, **kwargs: Any) -> List[str]: - """Get known url for a local file by hash.""" - from API.folder import API_folder_store - - try: - file_hash = file_identifier - if self._location: - try: - from SYS.metadata import normalize_urls - - with API_folder_store(Path(self._location)) as db: - meta = db.get_metadata(file_hash) or {} - urls = normalize_urls(meta.get("url")) - return urls - except Exception as exc: - debug(f"Local DB get_metadata failed: {exc}") - return [] - except Exception as exc: - debug(f"get_url failed for local file: {exc}") - return [] - - def add_url(self, file_identifier: str, url: List[str], **kwargs: Any) -> bool: - """Add known url to a local file by hash.""" - from API.folder import API_folder_store - - try: - file_hash = file_identifier - if self._location: - try: - from SYS.metadata import normalize_urls - - with API_folder_store(Path(self._location)) as db: - meta = db.get_metadata(file_hash) or {} - existing_urls = normalize_urls(meta.get("url")) - incoming_urls = normalize_urls(url) - changed = False - for u in list(incoming_urls or []): - if not u: - continue - if u not in existing_urls: - existing_urls.append(u) - changed = True - if changed: - db.update_metadata_by_hash( - file_hash, - { - "url": existing_urls - } - ) - return True - except Exception as exc: - debug(f"Local DB add_url failed: {exc}") - return False - except Exception as exc: - debug(f"add_url failed for local file: {exc}") - return False - - def add_url_bulk(self, items: List[tuple[str, List[str]]], **kwargs: Any) -> bool: - """Add known urls to many local files in one DB session. - - This is a performance optimization used by cmdlets that receive many PipeObjects. - """ - from API.folder import API_folder_store - - try: - if not self._location: - return False - - # Normalize + coalesce duplicates per hash. - try: - from SYS.metadata import normalize_urls - except Exception: - normalize_urls = None # type: ignore - - merged_by_hash: Dict[str, - List[str]] = {} - for file_identifier, url_list in items or []: - file_hash = str(file_identifier or "").strip().lower() - if not file_hash: - continue - - incoming: List[str] - if normalize_urls is not None: - try: - incoming = normalize_urls(url_list) - except Exception: - incoming = [ - str(u).strip() for u in (url_list or []) if str(u).strip() - ] - else: - incoming = [ - str(u).strip() for u in (url_list or []) if str(u).strip() - ] - - if not incoming: - continue - - existing = merged_by_hash.get(file_hash) or [] - for u in incoming: - if u and u not in existing: - existing.append(u) - merged_by_hash[file_hash] = existing - - if not merged_by_hash: - return True - - import json - - with API_folder_store(Path(self._location)) as db: - conn = getattr(db, "connection", None) - if conn is None: - return False - cursor = conn.cursor() - - # Ensure metadata rows exist (may be needed for older entries). - for file_hash in merged_by_hash.keys(): - try: - cursor.execute( - "INSERT OR IGNORE INTO metadata (hash) VALUES (?)", - (file_hash, - ) - ) - except Exception: - continue - - # Load existing urls for all hashes in chunks. - existing_urls_by_hash: Dict[str, - List[str]] = { - h: [] - for h in merged_by_hash.keys() - } - hashes = list(merged_by_hash.keys()) - chunk_size = 400 - for i in range(0, len(hashes), chunk_size): - chunk = hashes[i:i + chunk_size] - if not chunk: - continue - placeholders = ",".join(["?"] * len(chunk)) - try: - cursor.execute( - f"SELECT hash, url FROM metadata WHERE hash IN ({placeholders})", - chunk - ) - rows = cursor.fetchall() or [] - except Exception: - rows = [] - - for row in rows: - try: - row_hash = str(row[0]).strip().lower() - except Exception: - continue - raw_urls = None - try: - raw_urls = row[1] - except Exception: - raw_urls = None - - parsed_urls: List[str] = [] - if raw_urls: - try: - parsed = json.loads(raw_urls) - if normalize_urls is not None: - parsed_urls = normalize_urls(parsed) - else: - if isinstance(parsed, list): - parsed_urls = [ - str(u).strip() for u in parsed - if str(u).strip() - ] - except Exception: - parsed_urls = [] - - existing_urls_by_hash[row_hash] = parsed_urls - - # Compute updates and write in one commit. - updates: List[tuple[str, str]] = [] - for file_hash, incoming_urls in merged_by_hash.items(): - existing_urls = existing_urls_by_hash.get(file_hash) or [] - final = list(existing_urls) - for u in incoming_urls: - if u and u not in final: - final.append(u) - if final != existing_urls: - try: - updates.append((json.dumps(final), file_hash)) - except Exception: - continue - - if updates: - cursor.executemany( - "UPDATE metadata SET url = ?, time_modified = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP WHERE hash = ?", - updates, - ) - - conn.commit() - return True - except Exception as exc: - debug(f"add_url_bulk failed for local file: {exc}") - return False - - def delete_url(self, file_identifier: str, url: List[str], **kwargs: Any) -> bool: - """Delete known url from a local file by hash.""" - from API.folder import API_folder_store - - try: - file_hash = file_identifier - if self._location: - try: - from SYS.metadata import normalize_urls - - with API_folder_store(Path(self._location)) as db: - meta = db.get_metadata(file_hash) or {} - existing_urls = normalize_urls(meta.get("url")) - remove_set = {u - for u in normalize_urls(url) if u} - if not remove_set: - return False - new_urls = [u for u in existing_urls if u not in remove_set] - if new_urls != existing_urls: - db.update_metadata_by_hash(file_hash, - { - "url": new_urls - }) - return True - except Exception as exc: - debug(f"Local DB delete_url failed: {exc}") - return False - except Exception as exc: - debug(f"delete_url failed for local file: {exc}") - return False - - def delete_url_bulk( - self, - items: List[tuple[str, - List[str]]], - **kwargs: Any - ) -> bool: - """Delete known urls from many local files in one DB session.""" - from API.folder import API_folder_store - - try: - if not self._location: - return False - - try: - from SYS.metadata import normalize_urls - except Exception: - normalize_urls = None # type: ignore - - remove_by_hash: Dict[str, - set[str]] = {} - for file_identifier, url_list in items or []: - file_hash = str(file_identifier or "").strip().lower() - if not file_hash: - continue - - incoming: List[str] - if normalize_urls is not None: - try: - incoming = normalize_urls(url_list) - except Exception: - incoming = [ - str(u).strip() for u in (url_list or []) if str(u).strip() - ] - else: - incoming = [ - str(u).strip() for u in (url_list or []) if str(u).strip() - ] - - remove = {u - for u in incoming if u} - if not remove: - continue - remove_by_hash.setdefault(file_hash, set()).update(remove) - - if not remove_by_hash: - return True - - import json - - with API_folder_store(Path(self._location)) as db: - conn = getattr(db, "connection", None) - if conn is None: - return False - cursor = conn.cursor() - - # Ensure metadata rows exist. - for file_hash in remove_by_hash.keys(): - try: - cursor.execute( - "INSERT OR IGNORE INTO metadata (hash) VALUES (?)", - (file_hash, - ) - ) - except Exception: - continue - - # Load existing urls for hashes in chunks. - existing_urls_by_hash: Dict[str, - List[str]] = { - h: [] - for h in remove_by_hash.keys() - } - hashes = list(remove_by_hash.keys()) - chunk_size = 400 - for i in range(0, len(hashes), chunk_size): - chunk = hashes[i:i + chunk_size] - if not chunk: - continue - placeholders = ",".join(["?"] * len(chunk)) - try: - cursor.execute( - f"SELECT hash, url FROM metadata WHERE hash IN ({placeholders})", - chunk - ) - rows = cursor.fetchall() or [] - except Exception: - rows = [] - - for row in rows: - try: - row_hash = str(row[0]).strip().lower() - except Exception: - continue - raw_urls = None - try: - raw_urls = row[1] - except Exception: - raw_urls = None - - parsed_urls: List[str] = [] - if raw_urls: - try: - parsed = json.loads(raw_urls) - if normalize_urls is not None: - parsed_urls = normalize_urls(parsed) - else: - if isinstance(parsed, list): - parsed_urls = [ - str(u).strip() for u in parsed - if str(u).strip() - ] - except Exception: - parsed_urls = [] - - existing_urls_by_hash[row_hash] = parsed_urls - - # Apply removals + write updates. - updates: List[tuple[str, str]] = [] - for file_hash, remove_set in remove_by_hash.items(): - existing_urls = existing_urls_by_hash.get(file_hash) or [] - new_urls = [u for u in existing_urls if u not in remove_set] - if new_urls != existing_urls: - try: - updates.append((json.dumps(new_urls), file_hash)) - except Exception: - continue - - if updates: - cursor.executemany( - "UPDATE metadata SET url = ?, time_modified = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP WHERE hash = ?", - updates, - ) - - conn.commit() - return True - except Exception as exc: - debug(f"delete_url_bulk failed for local file: {exc}") - return False - - def get_note(self, file_identifier: str, **kwargs: Any) -> Dict[str, str]: - """Get notes for a local file by hash.""" - from API.folder import API_folder_store - - try: - if not self._location: - return {} - file_hash = str(file_identifier or "").strip().lower() - if not _normalize_hash(file_hash): - return {} - with API_folder_store(Path(self._location)) as db: - getter = getattr(db, "get_notes", None) - if callable(getter): - notes = getter(file_hash) - return notes if isinstance(notes, - dict) else {} - # Fallback: default-only - note = db.get_note(file_hash) - return { - "default": str(note or "") - } if note else {} - except Exception as exc: - debug(f"get_note failed for local file: {exc}") - return {} - - def set_note( - self, - file_identifier: str, - name: str, - text: str, - **kwargs: Any - ) -> bool: - """Set a named note for a local file by hash.""" - from API.folder import API_folder_store - - try: - if not self._location: - return False - file_hash = str(file_identifier or "").strip().lower() - note_name = str(name or "").strip() - if not _normalize_hash(file_hash): - return False - if not note_name: - return False - - with API_folder_store(Path(self._location)) as db: - setter_hash = getattr(db, "set_note_by_hash", None) - if callable(setter_hash): - setter_hash(file_hash, note_name, str(text)) - return True - - file_path = self.get_file(file_hash, **kwargs) - if not file_path or not isinstance(file_path, - Path) or not file_path.exists(): - return False - - setter = getattr(db, "set_note", None) - if callable(setter): - setter(file_path, note_name, str(text)) - return True - try: - db.save_note(file_path, str(text), name=note_name) - except TypeError: - db.save_note(file_path, str(text)) - return True - except Exception as exc: - debug(f"set_note failed for local file: {exc}") - return False - - def set_note_bulk(self, items: List[tuple[str, str, str]], **kwargs: Any) -> bool: - """Set notes for many local files in one DB session. - - Preserves existing semantics by only setting notes for hashes that still - map to a file path that exists on disk. - """ - from API.folder import API_folder_store - - try: - if not self._location: - return False - - # Normalize input. - normalized: List[tuple[str, str, str]] = [] - for file_identifier, name, text in items or []: - file_hash = str(file_identifier or "").strip().lower() - note_name = str(name or "").strip() - note_text = str(text or "") - if not file_hash or not _normalize_hash(file_hash) or not note_name: - continue - normalized.append((file_hash, note_name, note_text)) - - if not normalized: - return True - - with API_folder_store(Path(self._location)) as db: - conn = getattr(db, "connection", None) - if conn is None: - return False - cursor = conn.cursor() - - # Look up file paths for hashes in chunks (to verify existence). - wanted_hashes = sorted({h - for (h, _n, _t) in normalized}) - hash_to_path: Dict[str, - str] = {} - chunk_size = 400 - for i in range(0, len(wanted_hashes), chunk_size): - chunk = wanted_hashes[i:i + chunk_size] - if not chunk: - continue - placeholders = ",".join(["?"] * len(chunk)) - try: - cursor.execute( - f"SELECT hash, file_path FROM file WHERE hash IN ({placeholders})", - chunk, - ) - rows = cursor.fetchall() or [] - except Exception: - rows = [] - for row in rows: - try: - h = str(row[0]).strip().lower() - p = str(row[1]).strip() - except Exception: - continue - if h and p: - hash_to_path[h] = p - - # Ensure notes rows exist and only write for existing files. - inserts: List[tuple[str, str, str]] = [] - for h, note_name, note_text in normalized: - p = hash_to_path.get(h) - if not p: - continue - try: - if not (Path(self._location) / p).exists(): - continue - except Exception: - continue - inserts.append((h, note_name, note_text)) - - if not inserts: - return False - - # Prefer upsert when supported, else fall back to INSERT OR REPLACE. - try: - cursor.executemany( - "INSERT INTO note (hash, name, note) VALUES (?, ?, ?) " - "ON CONFLICT(hash, name) DO UPDATE SET note = excluded.note, updated_at = CURRENT_TIMESTAMP", - inserts, - ) - except Exception: - cursor.executemany( - "INSERT OR REPLACE INTO note (hash, name, note) VALUES (?, ?, ?)", - inserts, - ) - - conn.commit() - return True - except Exception as exc: - debug(f"set_note_bulk failed for local file: {exc}") - return False - - def delete_note(self, file_identifier: str, name: str, **kwargs: Any) -> bool: - """Delete a named note for a local file by hash.""" - from API.folder import API_folder_store - - try: - if not self._location: - return False - file_hash = str(file_identifier or "").strip().lower() - if not _normalize_hash(file_hash): - return False - with API_folder_store(Path(self._location)) as db: - deleter = getattr(db, "delete_note", None) - if callable(deleter): - deleter(file_hash, str(name)) - return True - # Default-only fallback - if str(name).strip().lower() == "default": - deleter2 = getattr(db, "save_note", None) - if callable(deleter2): - file_path = self.get_file(file_hash, **kwargs) - if file_path and isinstance(file_path, - Path) and file_path.exists(): - deleter2(file_path, "") - return True - return False - except Exception as exc: - debug(f"delete_note failed for local file: {exc}") - return False - - def delete_file(self, file_identifier: str, **kwargs: Any) -> bool: - """Delete a file from the folder store. - - Args: - file_identifier: The file path (as string) or hash of the file to delete - **kwargs: Optional parameters - - Returns: - True if deletion succeeded, False otherwise - """ - from API.folder import API_folder_store - - try: - if not self._location: - return False - - raw = str(file_identifier or "").strip() - if not raw: - return False - - store_root = expand_path(self._location) - - # Support deletion by hash (common for store items where `path` is the hash). - file_hash = _normalize_hash(raw) - resolved_path: Optional[Path] = None - with API_folder_store(store_root) as db: - if file_hash: - resolved_path = db.search_hash(file_hash) - else: - p = expand_path(raw) - resolved_path = p if p.is_absolute() else (store_root / p) - - if resolved_path is None: - debug(f"delete_file: could not resolve identifier: {raw}") - return False - - # Delete from database (also cleans up relationship backlinks). - db.delete_file(resolved_path) - - # Delete the actual file from disk (best-effort). - try: - if resolved_path.exists(): - resolved_path.unlink() - debug(f"Deleted file: {resolved_path}") - else: - debug(f"File not found on disk: {resolved_path}") - except Exception: - pass - - return True - except Exception as exc: - debug(f"delete_file failed: {exc}") - return False diff --git a/Store/ZeroTier.py b/Store/ZeroTier.py deleted file mode 100644 index f90f890..0000000 --- a/Store/ZeroTier.py +++ /dev/null @@ -1,659 +0,0 @@ -"""ZeroTier-backed Store implementation. - -This store locates a service running on peers in a ZeroTier network and -proxies store operations to that remote service. The remote service can be -our `remote_storage_server` (default) or a Hydrus API server (`service=hydrus`). - -Configuration keys: -- NAME: store instance name (required) -- NETWORK_ID: ZeroTier network ID to use for discovery (required) -- SERVICE: 'remote' or 'hydrus' (default: 'remote') -- PORT: service port (default: 999 for remote, 45869 for hydrus) -- API_KEY: optional API key to include in requests -- HOST: optional preferred peer address (skip discovery if provided) - -Notes: -- This implementation focuses on read operations (search, get_file, get_metadata, - tag/url ops). Uploads can be implemented later when the remote server - supports a robust, authenticated upload endpoint. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple - -from SYS.logger import debug, log -from Store._base import Store - - -class ZeroTier(Store): - - @classmethod - def config_schema(cls) -> List[Dict[str, Any]]: - return [ - {"key": "NAME", "label": "Store Name", "default": "", "required": True}, - {"key": "NETWORK_ID", "label": "ZeroTier Network ID", "default": "", "required": True}, - {"key": "HOST", "label": "Peer address (IP)", "default": "", "required": True}, - {"key": "PORT", "label": "Service Port", "default": "999", "required": False}, - {"key": "SERVICE", "label": "Service Type (remote|hydrus)", "default": "remote", "required": False}, - {"key": "API_KEY", "label": "API Key (optional)", "default": "", "required": False, "secret": True}, - {"key": "TIMEOUT", "label": "Request timeout (s)", "default": "5", "required": False}, - ] - - def __new__(cls, *args: Any, **kwargs: Any) -> "ZeroTier": - inst = super().__new__(cls) - name = kwargs.get("NAME") - if name is not None: - setattr(inst, "NAME", str(name)) - return inst - - def __init__( - self, - instance_name: Optional[str] = None, - network_id: Optional[str] = None, - service: Optional[str] = None, - port: Optional[int] = None, - api_key: Optional[str] = None, - host: Optional[str] = None, - timeout: Optional[int] = None, - *, - NAME: Optional[str] = None, - NETWORK_ID: Optional[str] = None, - SERVICE: Optional[str] = None, - PORT: Optional[int] = None, - API_KEY: Optional[str] = None, - HOST: Optional[str] = None, - TIMEOUT: Optional[int] = None, - ) -> None: - if instance_name is None and NAME is not None: - instance_name = str(NAME) - if network_id is None and NETWORK_ID is not None: - network_id = str(NETWORK_ID) - if service is None and SERVICE is not None: - service = str(SERVICE) - if port is None and PORT is not None: - try: - port = int(PORT) - except Exception: - port = None - if api_key is None and API_KEY is not None: - api_key = str(API_KEY) - if host is None and HOST is not None: - host = str(HOST) - if timeout is None and TIMEOUT is not None: - try: - timeout = int(TIMEOUT) - except Exception: - timeout = None - - self._name = str(instance_name or "") - self._network_id = str(network_id or "").strip() - self._service = (str(service or "remote") or "remote").lower() - self._port = int(port if port is not None else (45869 if self._service == "hydrus" else 999)) - self._api_key = str(api_key or "").strip() or None - self._preferred_host = str(host or "").strip() or None - self._timeout = int(timeout or 5) - - # Cached discovery result - self._cached_peer: Optional[Tuple[str, int]] = None - self._cached_client: Optional[Any] = None - - def name(self) -> str: - return str(getattr(self, "_name", "zerotier")) - - # -------------------- internal helpers -------------------- - def _discover_peer(self, *, refresh: bool = False) -> Optional[Tuple[str, int]]: - """Discover a peer host:port for this service on the configured network. - - Returns (host, port) or None. - """ - if self._preferred_host and not refresh: - return (self._preferred_host, self._port) - - if self._cached_peer and not refresh: - return self._cached_peer - - try: - from API import zerotier as zt - except Exception as exc: - debug(f"ZeroTier discovery helper not available: {exc}") - return None - - # Try to find a central API key for better discovery - from SYS.config import load_config - - conf = load_config() - net_conf = conf.get("networking", {}).get("zerotier", {}) - central_token = net_conf.get("api_key") - - # Look for a matching service on the network - probe = zt.find_peer_service( - self._network_id, - service_hint=("hydrus" if self._service == "hydrus" else None), - port=self._port, - api_token=central_token, - ) - if probe: - # Extract host:port - host = probe.address - port = probe.port or self._port - self._cached_peer = (host, int(port)) - debug(f"ZeroTier store '{self.name()}' discovered peer {host}:{port}") - return self._cached_peer - - debug(f"ZeroTier store '{self.name()}' found no peers on network {self._network_id}") - return None - - def _ensure_client(self, *, refresh: bool = False) -> Optional[Any]: - """Return a remote client object or base URL depending on service type. - - For 'hydrus' service we return an API.HydrusNetwork instance; for 'remote' - service we return a base URL string to send HTTP requests to. - """ - if self._cached_client and not refresh: - return self._cached_client - - peer = self._discover_peer(refresh=refresh) - if not peer: - return None - host, port = peer - - if self._service == "hydrus": - try: - from API.HydrusNetwork import HydrusNetwork as HydrusClient - base_url = f"http://{host}:{port}" - client = HydrusClient(url=base_url, access_key=(self._api_key or ""), timeout=self._timeout) - self._cached_client = client - return client - except Exception as exc: - debug(f"Failed to instantiate Hydrus client for ZeroTier peer {host}:{port}: {exc}") - return None - - # Default: remote_storage 'http' style API - self._cached_client = f"http://{host}:{port}" - return self._cached_client - - def _request_remote(self, method: str, path: str, *, params: Optional[Dict[str, Any]] = None, json_body: Optional[Any] = None, timeout: Optional[int] = None) -> Optional[Any]: - base = self._ensure_client() - if base is None or not isinstance(base, str): - debug("No remote base URL available for ZeroTier store") - return None - - url = base.rstrip("/") + path - headers = {} - if self._api_key: - headers["X-API-Key"] = self._api_key - - try: - import httpx - resp = httpx.request(method, url, params=params, json=json_body, headers=headers, timeout=timeout or self._timeout) - if resp.status_code == 401: - log(f"[Store={self._name}] Remote service at {url} requires an API Key. Please configure 'API_KEY' for this store.", severity="warning") - resp.raise_for_status() - try: - return resp.json() - except Exception: - return resp.text - except Exception as exc: - debug(f"ZeroTier HTTP request failed: {method} {url} -> {exc}") - return None - - # -------------------- Store API -------------------- - def search(self, query: str, **kwargs: Any) -> List[Dict[str, Any]]: - """Search for files on the remote service.""" - client = self._ensure_client() - if client is None: - debug("ZeroTier search: no client available") - return [] - - if self._service == "hydrus": - # Hydrus API expects tags list; best-effort: treat query as a single tag or raw search term - try: - tags = [query] - payload = client.search_files(tags, return_hashes=True, return_file_ids=False, return_file_count=False) - # Hydrus JSON shape varies; normalize to simple list - files = [] - try: - if isinstance(payload, dict): - rows = payload.get("files") or payload.get("metadata") or [] - for r in rows: - files.append(r if isinstance(r, dict) else {}) - except Exception: - pass - return files - except Exception as exc: - debug(f"Hydrus search failed: {exc}") - return [] - - # remote_storage path - params = {"q": query, "limit": int(kwargs.get("limit", 100))} - res = self._request_remote("GET", "/files/search", params=params) - if isinstance(res, dict): - files = list(res.get("files") or []) - # Inject store name and normalize keys for the CLI - for f in files: - if isinstance(f, dict): - f["store"] = self._name - # remote_storage_server returns 'file_path' and 'size' - # CLI prefers 'path' and 'size_bytes' - if "file_path" in f and "path" not in f: - f["path"] = f["file_path"] - - # Try to extract title from tags - tags = f.get("tag") or [] - title_tag = next((t for t in tags if str(t).lower().startswith("title:")), None) - if title_tag and ":" in title_tag: - f["title"] = title_tag.split(":", 1)[1].strip() - elif "title" not in f: - try: - f["title"] = Path(f["file_path"]).stem - except Exception: - f["title"] = f["file_path"] - - if "size" in f and "size_bytes" not in f: - f["size_bytes"] = f["size"] - return files - return [] - - def get_file(self, file_hash: str, **kwargs: Any) -> Optional[Path | str]: - """Return either a URL (hydrus or remote capable) or local path (not implemented). - - For Hydrus: return the direct file URL (Hydrus client URL with access token appended if needed). - For remote_storage: currently return the metadata path (if available) or None. - """ - client = self._ensure_client() - if client is None: - return None - - if self._service == "hydrus": - try: - # Hydrus wrapper provides file_url() convenience - return client.file_url(file_hash) - except Exception as exc: - debug(f"Hydrus get_file failed: {exc}") - return None - - # remote storage: return download URL - base = self._ensure_client() - if not base or not isinstance(base, str): - return None - - url = f"{base.rstrip('/')}/files/raw/{file_hash}" - if self._api_key: - sep = "&" if "?" in url else "?" - url += f"{sep}api_key={self._api_key}" - return url - - def download_to_temp( - self, - file_hash: str, - temp_root: Optional[Path] = None, - suffix: Optional[str] = None, - progress_callback: Optional[Callable[[int, int], None]] = None, - ) -> Optional[Path]: - """Download a file from the remote peer to a local temporary file.""" - import os - import httpx - import tempfile - - if self._service == "hydrus": - return None - - url = self.get_file(file_hash) - if not url or not isinstance(url, str) or not url.startswith("http"): - return None - - # Ensure suffix starts with a dot if provided - if suffix and not suffix.startswith("."): - suffix = f".{suffix}" - if not suffix: - suffix = ".tmp" - - try: - # Use provided temp_root or system temp - if temp_root: - temp_root.mkdir(parents=True, exist_ok=True) - fd, tmp_path = tempfile.mkstemp(dir=str(temp_root), suffix=suffix) - else: - fd, tmp_path = tempfile.mkstemp(suffix=suffix) - - os_fd = os.fdopen(fd, "wb") - - headers = {} - if self._api_key: - headers["X-API-Key"] = self._api_key - - downloaded = 0 - total = 0 - with httpx.stream("GET", url, headers=headers, timeout=self._timeout) as r: - r.raise_for_status() - total = int(r.headers.get("Content-Length", 0)) - # Use a larger chunk size for ZeroTier/P2P efficiency - for chunk in r.iter_bytes(chunk_size=128 * 1024): - if chunk: - os_fd.write(chunk) - downloaded += len(chunk) - if progress_callback: - try: - progress_callback(downloaded, total) - except Exception: - pass - - os_fd.close() - return Path(tmp_path) - - except Exception as exc: - debug(f"ZeroTier download_to_temp failed for {file_hash}: {exc}") - return None - - def add_file(self, file_path: Path, **kwargs: Any) -> Optional[str]: - """Upload a local file to the remote ZeroTier peer (supports 'remote' and 'hydrus' services). - - Returns the file hash on success, or None on failure. - """ - - p = Path(file_path) - if not p.exists(): - debug(f"ZeroTier add_file: local file not found: {p}") - return None - - # Hydrus: delegate to Hydrus client add_file() - if self._service == "hydrus": - try: - client = self._ensure_client() - if client is None: - debug("ZeroTier add_file: Hydrus client unavailable") - return None - return client.add_file(p, **kwargs) - except Exception as exc: - debug(f"ZeroTier hydrus add_file failed: {exc}") - return None - - # Remote server: POST /files/upload multipart/form-data - base = self._ensure_client() - if base is None or not isinstance(base, str): - debug("ZeroTier add_file: no remote base URL available") - return None - - url = base.rstrip("/") + "/files/upload" - headers = {} - if self._api_key: - headers["X-API-Key"] = self._api_key - - try: - import httpx - with open(p, "rb") as fh: - # Build form fields for tags/urls (support list or comma-separated) - data = [] - if "tag" in kwargs: - tags = kwargs.get("tag") or [] - if isinstance(tags, str): - tags = [t.strip() for t in tags.split(",") if t.strip()] - for t in tags: - data.append(("tag", t)) - if "url" in kwargs: - urls = kwargs.get("url") or [] - if isinstance(urls, str): - urls = [u.strip() for u in urls.split(",") if u.strip()] - for u in urls: - data.append(("url", u)) - - files = {"file": (p.name, fh, "application/octet-stream")} - # Prefer `requests` for local testing / WSGI servers which may not accept - # chunked uploads reliably with httpx/httpcore. Fall back to httpx otherwise. - try: - try: - import requests - # Convert data list-of-tuples to dict for requests (acceptable for repeated fields) - data_dict = {} - for k, v in data: - if k in data_dict: - existing = data_dict[k] - if not isinstance(existing, list): - data_dict[k] = [existing] - data_dict[k].append(v) - else: - data_dict[k] = v - r = requests.post(url, headers=headers, files=files, data=data_dict or None, timeout=self._timeout) - if r.status_code in (200, 201): - try: - payload = r.json() - file_hash = payload.get("hash") or payload.get("file_hash") - return file_hash - except Exception: - return None - try: - debug(f"[zerotier-debug] upload failed (requests) status={r.status_code} body={r.text}") - except Exception: - pass - debug(f"ZeroTier add_file failed (requests): status {r.status_code} body={getattr(r, 'text', '')}") - return None - except Exception: - import httpx - resp = httpx.post(url, headers=headers, files=files, data=data, timeout=self._timeout) - # Note: some environments may not create request.files correctly; capture body for debugging - try: - if resp.status_code in (200, 201): - try: - payload = resp.json() - file_hash = payload.get("hash") or payload.get("file_hash") - return file_hash - except Exception: - return None - # Debug output to help tests capture server response - try: - debug(f"[zerotier-debug] upload failed status={resp.status_code} body={resp.text}") - except Exception: - pass - debug(f"ZeroTier add_file failed: status {resp.status_code} body={getattr(resp, 'text', '')}") - return None - except Exception as exc: - debug(f"ZeroTier add_file exception: {exc}") - return None - except Exception as exc: - debug(f"ZeroTier add_file exception: {exc}") - return None - except Exception as exc: - debug(f"ZeroTier add_file exception: {exc}") - return None - - def get_metadata(self, file_hash: str, **kwargs: Any) -> Optional[Dict[str, Any]]: - client = self._ensure_client() - if client is None: - return None - - if self._service == "hydrus": - try: - payload = client.fetch_file_metadata(hashes=[file_hash], include_file_url=True, include_size=True, include_mime=True) - return payload - except Exception as exc: - debug(f"Hydrus fetch_file_metadata failed: {exc}") - return None - - res = self._request_remote("GET", f"/files/{file_hash}") - if isinstance(res, dict): - # Extract title from tags for the details panel/metadata view - tags = res.get("tag") or [] - title_tag = next((t for t in tags if str(t).lower().startswith("title:")), None) - if title_tag and ":" in title_tag: - res["title"] = title_tag.split(":", 1)[1].strip() - return res - return None - - def get_tag(self, file_identifier: str, **kwargs: Any) -> Tuple[List[str], str]: - # Return (tags, service). For hydrus use fetch_file_metadata service keys. - client = self._ensure_client() - if client is None: - return ([], "") - - if self._service == "hydrus": - try: - payload = client.fetch_file_metadata(hashes=[file_identifier], include_service_keys_to_tags=True) - tags = [] - if isinstance(payload, dict): - metas = payload.get("metadata") or [] - if metas and isinstance(metas, list) and metas: - md = metas[0] - if isinstance(md, dict): - tags = md.get("service_keys_to_tags") or [] - return (tags, "hydrus") - except Exception as exc: - debug(f"Hydrus get_tag failed: {exc}") - return ([], "hydrus") - - res = self._request_remote("GET", f"/tags/{file_identifier}") - if isinstance(res, dict): - return (list(res.get("tag") or []), "remote") - return ([], "remote") - - def add_tag(self, file_identifier: str, tags: List[str], **kwargs: Any) -> bool: - client = self._ensure_client() - if client is None: - return False - - if self._service == "hydrus": - try: - service_name = kwargs.get("service_name") or "my tags" - client.add_tag(file_identifier, tags, service_name) - return True - except Exception as exc: - debug(f"Hydrus add_tag failed: {exc}") - return False - - payload = {"tag": tags} - res = self._request_remote("POST", f"/tags/{file_identifier}", json_body=payload) - return bool(res) - - def delete_tag(self, file_identifier: str, tags: List[str], **kwargs: Any) -> bool: - client = self._ensure_client() - if client is None: - return False - - if self._service == "hydrus": - try: - service_name = kwargs.get("service_name") or "my tags" - client.delete_tag(file_identifier, tags, service_name) - return True - except Exception as exc: - debug(f"Hydrus delete_tag failed: {exc}") - return False - - # remote_storage DELETE /tags/?tag=tag1,tag2 - query = {"tag": ",".join(tags)} - res = self._request_remote("DELETE", f"/tags/{file_identifier}", params=query) - return bool(res) - - def get_url(self, file_identifier: str, **kwargs: Any) -> List[str]: - # For Hydrus, use fetch_file_metadata to include file URL; for remote, GET tags endpoint includes urls - client = self._ensure_client() - if client is None: - return [] - - if self._service == "hydrus": - try: - payload = client.fetch_file_metadata(hashes=[file_identifier], include_file_url=True) - try: - metas = payload.get("metadata") or [] - if metas and isinstance(metas, list) and metas: - md = metas[0] - if isinstance(md, dict): - urls = md.get("file_urls") or [] - return list(urls) - except Exception: - pass - return [] - except Exception as exc: - debug(f"Hydrus get_url failed: {exc}") - return [] - - meta = self._request_remote("GET", f"/files/{file_identifier}") - if isinstance(meta, dict): - urls = meta.get("url") or [] - return list(urls) - return [] - - def add_url(self, file_identifier: str, url: List[str], **kwargs: Any) -> bool: - client = self._ensure_client() - if client is None: - return False - - if self._service == "hydrus": - try: - client.associate_url(hashes=[file_identifier], url=url[0]) - return True - except Exception as exc: - debug(f"Hydrus add_url failed: {exc}") - return False - - payload = {"url": url} - res = self._request_remote("POST", f"/files/{file_identifier}/url", json_body=payload) - return bool(res) - - def delete_url(self, file_identifier: str, url: List[str], **kwargs: Any) -> bool: - client = self._ensure_client() - if client is None: - return False - - if self._service == "hydrus": - try: - client.delete_urls(hashes=[file_identifier], urls=url) - return True - except Exception as exc: - debug(f"Hydrus delete_url failed: {exc}") - return False - - payload = {"url": url} - res = self._request_remote("DELETE", f"/files/{file_identifier}/url", json_body=payload) - return bool(res) - - def get_note(self, file_identifier: str, **kwargs: Any) -> Dict[str, str]: - """Get named notes for a file. Returns a mapping of name->text.""" - client = self._ensure_client() - if client is None: - return {} - - if self._service == "hydrus": - try: - # Hydrus API may expose notes via fetch_file_metadata; best-effort - payload = client.fetch_file_metadata(hashes=[file_identifier], include_notes=True) - if isinstance(payload, dict): - metas = payload.get("metadata") or [] - if metas and isinstance(metas, list): - md = metas[0] - notes = md.get("notes") or {} - return dict(notes) - except Exception: - return {} - - # Remote storage has no notes API yet - return {} - - def set_note(self, file_identifier: str, name: str, text: str, **kwargs: Any) -> bool: - client = self._ensure_client() - if client is None: - return False - - if self._service == "hydrus": - try: - client.set_note(file_identifier, name, text) - return True - except Exception: - return False - - # Remote storage: not supported - return False - - def delete_note(self, file_identifier: str, name: str, **kwargs: Any) -> bool: - client = self._ensure_client() - if client is None: - return False - - if self._service == "hydrus": - try: - client.delete_note(file_identifier, name) - return True - except Exception: - return False - - return False diff --git a/Store/registry.py b/Store/registry.py index c808cd5..c6f800b 100644 --- a/Store/registry.py +++ b/Store/registry.py @@ -64,9 +64,7 @@ def _discover_store_classes() -> Dict[str, Type[BaseStore]]: discovered: Dict[str, Type[BaseStore]] = {} for module_info in pkgutil.iter_modules(store_pkg.__path__): module_name = module_info.name - if module_name in {"__init__", "_base", "registry"}: - continue - if module_name.lower() == "folder": + if module_name.startswith(("_", "registry")): continue try: diff --git a/TUI.py b/TUI.py index 8a29637..87da8ad 100644 --- a/TUI.py +++ b/TUI.py @@ -47,7 +47,6 @@ from SYS.cmdlet_catalog import ensure_registry_loaded, list_cmdlet_names # type from SYS.cli_syntax import validate_pipeline_text # type: ignore # noqa: E402 from TUI.pipeline_runner import PipelineRunner # type: ignore # noqa: E402 -from SYS.background_services import ensure_zerotier_server_running, stop_zerotier_server def _dedup_preserve_order(items: List[str]) -> List[str]: @@ -503,13 +502,7 @@ class PipelineHubApp(App): if self.worker_table: self.worker_table.add_columns("ID", "Type", "Status", "Details") - self.set_interval(5.0, ensure_zerotier_server_running) - def on_unmount(self) -> None: - stop_zerotier_server() - - async def _manage_zerotier_server(self) -> None: - # Method removed - logic moved to SYS.background_services pass # Initialize the store choices cache at startup (filters disabled stores) diff --git a/TUI/modalscreen/config_modal.py b/TUI/modalscreen/config_modal.py index c96f7e7..b40b3f7 100644 --- a/TUI/modalscreen/config_modal.py +++ b/TUI/modalscreen/config_modal.py @@ -128,7 +128,6 @@ class ConfigModal(ModalScreen): yield Label("Categories", classes="config-label") with ListView(id="category-list"): yield ListItem(Label("Global Settings"), id="cat-globals") - yield ListItem(Label("Connectors"), id="cat-networking") yield ListItem(Label("Stores"), id="cat-stores") yield ListItem(Label("Providers"), id="cat-providers") @@ -138,44 +137,62 @@ class ConfigModal(ModalScreen): yield Button("Save", variant="success", id="save-btn") yield Button("Add Store", variant="primary", id="add-store-btn") yield Button("Add Provider", variant="primary", id="add-provider-btn") - yield Button("Add Net", variant="primary", id="add-net-btn") yield Button("Back", id="back-btn") yield Button("Close", variant="error", id="cancel-btn") def on_mount(self) -> None: self.query_one("#add-store-btn", Button).display = False self.query_one("#add-provider-btn", Button).display = False - self.query_one("#add-net-btn", Button).display = False self.refresh_view() def refresh_view(self) -> None: - container = self.query_one("#fields-container", ScrollableContainer) + """ + Refresh the content area. We debounce this call and use a render_id + to avoid race conditions with Textual's async widget mounting. + """ + self._render_id = getattr(self, "_render_id", 0) + 1 + + if hasattr(self, "_refresh_timer"): + self._refresh_timer.stop() + self._refresh_timer = self.set_timer(0.02, self._actual_refresh) + + def _actual_refresh(self) -> None: + try: + container = self.query_one("#fields-container", ScrollableContainer) + except Exception: + return + self._button_id_map.clear() self._input_id_map.clear() - # Clear existing synchronously - for child in list(container.children): - child.remove() - + # Clear existing + container.query("*").remove() + # Update visibility of buttons try: self.query_one("#add-store-btn", Button).display = (self.current_category == "stores" and self.editing_item_name is None) self.query_one("#add-provider-btn", Button).display = (self.current_category == "providers" and self.editing_item_name is None) - self.query_one("#add-net-btn", Button).display = (self.current_category == "networking" and self.editing_item_name is None) self.query_one("#back-btn", Button).display = (self.editing_item_name is not None) self.query_one("#save-btn", Button).display = (self.editing_item_name is not None or self.current_category == "globals") except Exception: pass - # We mount using call_after_refresh to ensure the removals are processed by Textual - # before we try to mount new widgets with potentially duplicate IDs. + render_id = self._render_id + def do_mount(): + # If a new refresh was started, ignore this old mount request + if getattr(self, "_render_id", 0) != render_id: + return + + # Final check that container is empty. remove() is async. + if container.children: + for child in list(container.children): + child.remove() + if self.editing_item_name: self.render_item_editor(container) elif self.current_category == "globals": self.render_globals(container) - elif self.current_category == "networking": - self.render_networking(container) elif self.current_category == "stores": self.render_stores(container) elif self.current_category == "providers": @@ -241,73 +258,6 @@ class ConfigModal(ModalScreen): row.mount(Button("Paste", id=f"paste-{inp_id}", classes="paste-btn")) idx += 1 - def render_networking(self, container: ScrollableContainer) -> None: - container.mount(Label("ZeroTier Networks (local)", classes="config-label")) - - from API import zerotier as zt - # Show whether we have an explicit authtoken available and its source - try: - token_src = zt._get_token_path() - except Exception: - token_src = None - - if token_src == "env": - container.mount(Static("Auth: authtoken provided via env var (ZEROTIER_AUTH_TOKEN) ā€” no admin required", classes="config-note")) - elif token_src: - container.mount(Static(f"Auth: authtoken file found: {token_src} ā€” no admin required", classes="config-note")) - else: - container.mount(Static("Auth: authtoken not found in workspace; TUI may need admin to join networks", classes="config-warning")) - - try: - local_nets = zt.list_networks() - if not local_nets: - container.mount(Static("No active ZeroTier networks found on this machine.")) - else: - for n in local_nets: - row = Horizontal( - Static(f"{n.name} [{n.id}] - {n.status}", classes="item-label"), - Button("Leave", variant="error", id=f"zt-leave-{n.id}"), - classes="item-row" - ) - container.mount(row) - except Exception as exc: - container.mount(Static(f"Error listing ZeroTier networks: {exc}")) - - container.mount(Rule()) - container.mount(Label("Connectors", classes="config-label")) - net = self.config_data.get("networking", {}) - if not net: - container.mount(Static("No connectors configured.")) - else: - idx = 0 - for ntype, conf in net.items(): - edit_id = f"edit-net-{idx}" - del_id = f"del-net-{idx}" - self._button_id_map[edit_id] = ("edit", "networking", ntype) - self._button_id_map[del_id] = ("del", "networking", ntype) - idx += 1 - - label = ntype - if ntype == "zerotier": - serve = conf.get("serve", "Unknown") - net_id = conf.get("network_id", "Unknown") - net_name = net_id - try: - for ln in local_nets: - if ln.id == net_id: - net_name = ln.name - break - except Exception: pass - label = f"{serve} ---> {net_name}" - - row = Horizontal( - Static(label, classes="item-label"), - Button("Edit", id=edit_id), - Button("Delete", variant="error", id=del_id), - classes="item-row" - ) - container.mount(row) - def render_stores(self, container: ScrollableContainer) -> None: container.mount(Label("Configured Stores", classes="config-label")) stores = self.config_data.get("store", {}) @@ -402,30 +352,6 @@ class ConfigModal(ModalScreen): except Exception: pass - # Fetch Networking schema - if item_type == "networking": - if item_name == "zerotier": - from API import zerotier as zt - local_net_choices = [] - try: - for n in zt.list_networks(): - local_net_choices.append((f"{n.name} ({n.id})", n.id)) - except Exception: pass - - local_store_choices = [] - for s_type, s_data in self.config_data.get("store", {}).items(): - for s_name in s_data.keys(): - local_store_choices.append(s_name) - - schema = [ - {"key": "network_id", "label": "Network to Share on", "choices": local_net_choices}, - {"key": "serve", "label": "Local Store to Share", "choices": local_store_choices}, - {"key": "port", "label": "Port", "default": "999"}, - {"key": "api_key", "label": "Access Key (API Key)", "default": "", "secret": True}, - ] - for f in schema: - provider_schema_map[f["key"].upper()] = f - # Use columns for better layout of inputs with paste buttons container.mount(Label("Edit Settings")) # render_item_editor will handle the inputs for us if we set these @@ -583,8 +509,6 @@ class ConfigModal(ModalScreen): if not event.item: return if event.item.id == "cat-globals": self.current_category = "globals" - elif event.item.id == "cat-networking": - self.current_category = "networking" elif event.item.id == "cat-stores": self.current_category = "stores" elif event.item.id == "cat-providers": @@ -608,24 +532,7 @@ class ConfigModal(ModalScreen): if not self.validate_current_editor(): return try: - # If we are editing networking.zerotier, check if network_id changed and join it - if self.editing_item_type == "networking" and self.editing_item_name == "zerotier": - old_id = str(self.config_data.get("networking", {}).get("zerotier", {}).get("network_id") or "").strip() - self.save_all() - new_id = str(self.config_data.get("networking", {}).get("zerotier", {}).get("network_id") or "").strip() - - if new_id and new_id != old_id: - from API import zerotier as zt - try: - if zt.join_network(new_id): - self.notify(f"Joined ZeroTier network {new_id}") - else: - self.notify(f"Config saved, but failed to join network {new_id}", severity="warning") - except Exception as exc: - self.notify(f"Join error: {exc}", severity="error") - else: - self.save_all() - + self.save_all() self.notify("Configuration saved!") # Return to the main list view within the current category self.editing_item_name = None @@ -633,15 +540,6 @@ class ConfigModal(ModalScreen): self.refresh_view() except Exception as exc: self.notify(f"Save failed: {exc}", severity="error", timeout=10) - elif bid.startswith("zt-leave-"): - nid = bid.replace("zt-leave-", "") - from API import zerotier as zt - try: - zt.leave_network(nid) - self.notify(f"Left ZeroTier network {nid}") - self.refresh_view() - except Exception as exc: - self.notify(f"Failed to leave: {exc}", severity="error") elif bid in self._button_id_map: action, itype, name = self._button_id_map[bid] if action == "edit": @@ -657,9 +555,6 @@ class ConfigModal(ModalScreen): elif itype == "provider": if "provider" in self.config_data and name in self.config_data["provider"]: del self.config_data["provider"][name] - elif itype == "networking": - if "networking" in self.config_data and name in self.config_data["networking"]: - del self.config_data["networking"][name] self.refresh_view() elif bid == "add-store-btn": all_classes = _discover_store_classes() @@ -685,9 +580,6 @@ class ConfigModal(ModalScreen): except Exception: pass self.app.push_screen(SelectionModal("Select Provider Type", options), callback=self.on_provider_type_selected) - elif bid == "add-net-btn": - options = ["zerotier"] - self.app.push_screen(SelectionModal("Select Networking Service", options), callback=self.on_net_type_selected) elif bid.startswith("paste-"): # Programmatic paste button target_id = bid.replace("paste-", "") @@ -725,124 +617,6 @@ class ConfigModal(ModalScreen): def on_store_type_selected(self, stype: str) -> None: if not stype: return - if stype == "zerotier": - # Push a discovery wizard - from TUI.modalscreen.selection_modal import SelectionModal - from API import zerotier as zt - - # 1. Choose Network - joined = zt.list_networks() - if not joined: - self.notify("Error: Join a ZeroTier network first in 'Connectors'", severity="error") - return - - net_options = [f"{n.name or 'Network'} ({n.id})" for n in joined] - - def on_net_selected(net_choice: str): - if not net_choice: return - net_id = net_choice.split("(")[-1].rstrip(")") - - # 2. Host or Connect? - def on_mode_selected(mode: str): - if not mode: return - - if mode == "Host (Share a local store)": - # 3a. Select Local Store to Share - local_stores = [] - for s_type, s_data in self.config_data.get("store", {}).items(): - if s_type == "zerotier": continue - for s_name in s_data.keys(): - local_stores.append(f"{s_name} ({s_type})") - - if not local_stores: - self.notify("No local stores available to share.", severity="error") - return - - def on_share_selected(share_choice: str): - if not share_choice: return - share_name = share_choice.split(" (")[0] - - # Update networking config - if "networking" not in self.config_data: self.config_data["networking"] = {} - zt_net = self.config_data["networking"].setdefault("zerotier", {}) - zt_net["serve"] = share_name - zt_net["network_id"] = net_id - if not zt_net.get("port"): - zt_net["port"] = "999" - - try: - self.save_all() - from SYS.background_services import ensure_zerotier_server_running - ensure_zerotier_server_running() - self.notify(f"ZeroTier auto-saved: Sharing '{share_name}' on network {net_id}") - except Exception as e: - self.notify(f"Auto-save failed: {e}", severity="error") - - self.refresh_view() - - self.app.push_screen(SelectionModal("Select Local Store to Share", local_stores), callback=on_share_selected) - - else: - # 3b. Connect to Remote Peer - Background Discovery - @work - async def run_discovery(node): - self.notify(f"Discovery: Scanning {net_id} for peers...", timeout=5) - central_token = self.config_data.get("networking", {}).get("zerotier", {}).get("api_key") - try: - import asyncio - from functools import partial - loop = asyncio.get_event_loop() - probes = await loop.run_in_executor(None, partial( - zt.discover_services_on_network, net_id, ports=[999, 45869], api_token=central_token - )) - except Exception as e: - self.notify(f"Discovery error: {e}", severity="error") - return - - if not probes: - self.notify("No peers found. Check firewall or server status.", severity="warning") - return - - peer_options = [] - for p in probes: - label = "Remote" - if isinstance(p.payload, dict): - label = p.payload.get("name") or p.payload.get("peer_id") or label - status = " [Locked]" if p.status_code == 401 else "" - peer_options.append(f"{p.address} ({label}){status}") - - def on_peer_selected(choice: str): - if not choice: return - addr = choice.split(" ")[0] - match = next((p for p in probes if p.address == addr), None) - if match: - save_connected_store(match) - - self.app.push_screen(SelectionModal("Select Peer to Connect", peer_options), callback=on_peer_selected) - - def save_connected_store(p: zt.ZeroTierServiceProbe): - new_name = f"zt_{p.address.replace('.', '_')}" - if "store" not in self.config_data: self.config_data["store"] = {} - store_cfg = self.config_data["store"].setdefault("zerotier", {}) - - store_cfg[new_name] = { - "NAME": new_name, - "NETWORK_ID": net_id, - "HOST": p.address, - "PORT": str(p.port), - "SERVICE": p.service_hint or "remote" - } - self.save_all() - self.notify(f"Connected to {p.address}") - self.refresh_view() - - run_discovery(self) - - self.app.push_screen(SelectionModal("ZeroTier Mode", ["Host (Share a local store)", "Connect (Use a remote store)"]), callback=on_mode_selected) - - self.app.push_screen(SelectionModal("Select ZeroTier Network", net_options), callback=on_net_selected) - return - new_name = f"new_{stype}" if "store" not in self.config_data: self.config_data["store"] = {} @@ -907,18 +681,6 @@ class ConfigModal(ModalScreen): self.editing_item_name = ptype self.refresh_view() - def on_net_type_selected(self, ntype: str) -> None: - if not ntype: return - self.editing_item_type = "networking" - self.editing_item_name = ntype - - # Ensure it exists in config_data - net = self.config_data.setdefault("networking", {}) - if ntype not in net: - net[ntype] = {} - - self.refresh_view() - def _update_config_value(self, widget_id: str, value: Any) -> None: if widget_id not in self._input_id_map: return diff --git a/cmdlet/search_file.py b/cmdlet/search_file.py index e86870e..7795c10 100644 --- a/cmdlet/search_file.py +++ b/cmdlet/search_file.py @@ -31,8 +31,7 @@ from ._shared import ( from SYS import pipeline as ctx STORAGE_ORIGINS = {"local", - "hydrus", - "zerotier"} + "hydrus"} class _WorkerLogger: diff --git a/cmdnat/status.py b/cmdnat/status.py index 40923b0..ec3b7aa 100644 --- a/cmdnat/status.py +++ b/cmdnat/status.py @@ -251,35 +251,6 @@ def _run(result: Any, args: List[str], config: Dict[str, Any]) -> int: except Exception as exc: debug(f"Cookies check failed: {exc}") - # ZeroTier Hosting - zt_conf = config.get("networking", {}).get("zerotier", {}) - if zt_conf.get("serve"): - from SYS.background_services import ensure_zerotier_server_running - try: - debug("ZeroTier hosting enabled; ensuring server is running") - ensure_zerotier_server_running() - except Exception as exc: - debug(f"ensure_zerotier_server_running failed: {exc}") - - serve_target = zt_conf.get("serve") - port = zt_conf.get("port") or 999 - status = "OFFLINE" - detail = f"Sharing: {serve_target} on port {port}" - try: - from API.HTTP import HTTPClient - debug(f"Probing ZeroTier health on 127.0.0.1:{port}") - # Probing 127.0.0.1 is more reliable on Windows than localhost - with HTTPClient(timeout=1.0, retries=0) as client: - resp = client.get(f"http://127.0.0.1:{port}/health") - if resp.status_code == 200: - status = "ONLINE" - payload = resp.json() - detail += f" (Live: {payload.get('name', 'unknown')})" - debug(f"ZeroTier host responded: status={resp.status_code}, payload_keys={list(payload.keys()) if isinstance(payload, dict) else 'unknown'}") - except Exception as exc: - debug(f"ZeroTier probe failed: {exc}") - _add_startup_check(startup_table, status, "ZeroTier Host", detail=detail) - except Exception as exc: debug(f"Status check failed: {exc}") diff --git a/cmdnat/zerotier.py b/cmdnat/zerotier.py deleted file mode 100644 index a26b57c..0000000 --- a/cmdnat/zerotier.py +++ /dev/null @@ -1,152 +0,0 @@ -import sys -import requests -from pathlib import Path -from typing import Any, Dict, Sequence - -# Add project root to sys.path -root = Path(__file__).resolve().parent.parent -if str(root) not in sys.path: - sys.path.insert(0, str(root)) - -from cmdlet._shared import Cmdlet -from SYS.config import load_config -from SYS.result_table import Table -from API import zerotier as zt - -def exec_zerotier(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: - # Use provided config or fall back to CWD load - cfg = config if config else load_config(Path.cwd()) - - table = Table("ZeroTier Status") - - # 1. Local Hub Status - row = table.add_row() - row.add_column("TYPE", "HOST") - row.add_column("NAME", "localhost") - - # Try to get node ID via CLI info - node_id = "???" - try: - if hasattr(zt, "_run_cli_json"): - info = zt._run_cli_json("info", "-j") - node_id = info.get("address", "???") - except: - pass - row.add_column("ID", node_id) - - # Check if local server is responsive - try: - # endpoint is /health for remote_storage_server - # We try 127.0.0.1 first with a more generous timeout - # Using a list of potential local hits to be robust against Windows networking quirks - import socket - - def get_local_ip(): - try: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.connect(("8.8.8.8", 80)) - ip = s.getsockname()[0] - s.close() - return ip - except: - return None - - hosts = ["127.0.0.1", "localhost"] - local_ip = get_local_ip() - if local_ip: - hosts.append(local_ip) - - success = False - last_err = "" - - # Try multiple times if server just started - import time - for attempt in range(3): - for host in hosts: - try: - resp = requests.get(f"http://{host}:999/health", timeout=3, proxies={"http": None, "https": None}) - if resp.status_code == 200: - row.add_column("STATUS", "ONLINE") - row.add_column("ADDRESS", f"{host}:999") - row.add_column("DETAIL", f"Serving {cfg.get('active_store', 'default')}") - success = True - break - elif resp.status_code == 401: - row.add_column("STATUS", "Serving (Locked)") - row.add_column("ADDRESS", f"{host}:999") - row.add_column("DETAIL", "401 Unauthorized - API Key required") - success = True - break - except Exception as e: - last_err = str(e) - continue - if success: - break - time.sleep(1) # Wait between attempts - - if not success: - row.add_column("STATUS", "OFFLINE") - row.add_column("ADDRESS", "127.0.0.1:999") - row.add_column("DETAIL", f"Server not responding on port 999. Last attempt ({hosts[-1]}): {last_err}") - - except Exception as e: - row.add_column("STATUS", "OFFLINE") - row.add_column("ADDRESS", "127.0.0.1:999") - row.add_column("DETAIL", f"Status check failed: {e}") - - # 2. Add Networks - if zt.is_available(): - try: - networks = zt.list_networks() - for net in networks: - row = table.add_row() - row.add_column("TYPE", "NETWORK") - row.add_column("NAME", getattr(net, "name", "Unnamed")) - row.add_column("ID", getattr(net, "id", "")) - - status = getattr(net, "status", "OK") - assigned = getattr(net, "assigned_addresses", []) - ip_str = assigned[0] if assigned else "" - - row.add_column("STATUS", status) - row.add_column("ADDRESS", ip_str) - except Exception as e: - row = table.add_row() - row.add_column("TYPE", "ERROR") - row.add_column("DETAIL", f"Failed to list networks: {e}") - else: - row = table.add_row() - row.add_column("TYPE", "SYSTEM") - row.add_column("NAME", "ZeroTier") - row.add_column("STATUS", "NOT FOUND") - row.add_column("DETAIL", "zerotier-cli not in path") - - # Output - try: - from cmdnat.out_table import TableOutput - TableOutput().render(table) - except Exception: - # Fallback for raw CLI - print(f"\n--- {table.title} ---") - for r in table.rows: - # Use the get_column method from ResultRow - t = r.get_column("TYPE") or "" - n = r.get_column("NAME") or "" - s = r.get_column("STATUS") or "" - a = r.get_column("ADDRESS") or "" - id = r.get_column("ID") or "" - d = r.get_column("DETAIL") or "" - print(f"[{t:7}] {n:15} | {s:15} | {a:20} | {id} | {d}") - print("-" * 100) - - return 0 - -CMDLET = Cmdlet( - name=".zerotier", - summary="Check ZeroTier node and hosting status", - usage=".zerotier", - exec=exec_zerotier, -) - -if __name__ == "__main__": - exec_zerotier(None, sys.argv[1:], {}) \ No newline at end of file diff --git a/docs/zerotier.md b/docs/zerotier.md deleted file mode 100644 index 8a2d22c..0000000 --- a/docs/zerotier.md +++ /dev/null @@ -1,91 +0,0 @@ -# ZeroTier integration (store sharing) - -This document describes how Medios-Macina integrates with ZeroTier to share -storage backends between machines on a private virtual network. - -Goals -- Allow you to expose stores (folder-based, remote storage server, Hydrus client) - to other members of your ZeroTier network. -- Keep the CLI experience identical: remote stores appear as normal `-store` backends. -- Use secure authentication (API keys / per-store tokens) and limit exposure to private network. - -Prerequisites -- Each machine must run `zerotier-one` and be a member of your ZeroTier network. -- The Medios-Macina instance on each machine should run the `remote_storage_server.py` - or a Hydrus client instance you want to expose. -- The remote storage server requires Flask and Flask-CORS to run (install with: `pip install flask flask-cors`). - -Auto-install behavior -- When a `zerotier` section is present in `config.conf` **or** a `store=zerotier` instance is configured, the CLI will attempt to auto-install the required packages (`flask`, `flask-cors`, and `werkzeug`) on startup unless you disable it with `auto_install = false` in the `zerotier` config block. This mirrors the behavior for other optional features (e.g., Soulseek). -- On your controller/management machine, authorize members via ZeroTier Central. - -Configuration (conceptual) - -You can configure networks and Zerotier-backed stores in your `config.conf`. Here -are example snippets and recommendations. - -## Top-level ZeroTier networks (recommended) - -Use a `zerotier` section to list networks your instance is willing to use/auto-join: - -```ini -[zerotier] -# Example config (implementation treats this as a dict via the loader) -# networks: -# home: -# network_id: 8056c2e21c000001 -# api_key: my-zt-central-token ; optional, only needed for automating member authorization -# auto_join: true -# prefer_hosts: ["192.168.86.42"] ; optional peer IP inside the ZT network -``` - -## Store config (ZeroTier store instances) - -Add a `store=zerotier` block so the Store registry can create a ZeroTier store instance: - -```ini -[store=zerotier] -my-remote = { "NAME": "my-remote", "NETWORK_ID": "8056c2e21c000001", "SERVICE": "remote", "PORT": 999, "API_KEY": "myremotekey" } -hydrus-remote = { "NAME": "hydrus-remote", "NETWORK_ID": "8056c2e21c000001", "SERVICE": "hydrus", "PORT": 45869, "API_KEY": "hydrus-access-key" } -``` - -- `SERVICE` can be `remote` (our `remote_storage_server`), or `hydrus` (Hydrus API). -- `HOST` is optional; if present, discovery is skipped and the host:port is used. -- `API_KEY` will be sent as `X-API-Key` (and Hydrus access keys, when relevant). - -Operation & discovery -- The local ZeroTier store wrapper will attempt to discover peers on the configured - ZeroTier network by inspecting assigned addresses on this node and probing common - service endpoints (e.g., `/health`, `/api_version`). -- For `hydrus` service types we look for Hydrus-style `/api_version` responses. -- For `remote` service types we look for our `remote_storage_server` `/health` endpoint. - -Security notes -- Your ZeroTier network provides a private IP layer, but the exposed services - should still require authentication (API keys) and enforce scope (read/write). -- If you plan to expose stores to other users, consider per-store API keys with - roles (read-only, write, admin) and monitor/audit access. - -Next steps / prototyping -- The first prototype in this repo adds `API/zerotier.py` (discovery + join helpers) - and `Store/ZeroTier.py` (a store wrapper that proxies to `hydrus` or `remote` endpoints). -- Upload support (server-side `POST /files/upload`) is now implemented allowing authenticated multipart uploads; the ZeroTier store wrapper supports `add_file()` and the `add-file` cmdlet can be used with a configured ZeroTier store for end-to-end uploads. - -Example: upload via the helper script (discovers a remote on the network and uploads the file): - -```powershell -python .\scripts\zerotier_setup.py --upload 8056c2e21c000001 --file "C:\path\to\file.mp4" --api-key myremotekey --tag tag1 --tag tag2 -``` - -Or using curl directly against a discovered ZeroTier peer's IP: - -```powershell -curl -X POST -H "X-API-Key: myremotekey" -F "file=@/path/to/file.mp4" -F "tag=tag1" http://:999/files/upload -``` - -If you'd like I can: -- Add an example `scripts/zt-join.py` helper that uses the API wrapper to join a network; -- Add a presigned-upload + multipart upload flow to `scripts/remote_storage_server.py` so - ZeroTier stores can support `add-file` uploads directly. - -Tell me which of the above you want next (upload support, auto-join helper, or presigned flow) and I'll proceed. \ No newline at end of file diff --git a/scripts/remote_storage_server.py b/scripts/remote_storage_server.py deleted file mode 100644 index 7fe83f6..0000000 --- a/scripts/remote_storage_server.py +++ /dev/null @@ -1,773 +0,0 @@ -"""Remote Storage Server - REST API for file management on mobile devices. - -This server runs on a mobile device (Android with Termux, iOS with iSH, etc.) -and exposes the local library database as a REST API. Your PC connects to this -server and uses it as a remote storage backend through the RemoteStorageBackend. - -## INSTALLATION - -### On Android (Termux): -1. Install Termux from Play Store: https://play.google.com/store/apps/details?id=com.termux -2. In Termux: - $ apt update && apt install python - $ pip install flask flask-cors -3. Copy this file to your device -4. Run it (with optional API key): - $ python remote_storage_server.py --storage-path /path/to/storage --port 999 - $ python remote_storage_server.py --storage-path /path/to/storage --api-key mysecretkey -5. Server prints connection info automatically (IP, port, API key) - -### On PC: -1. Install requests: pip install requests -2. Add to config.conf: - [store=remote] - name="phone" - url="http://192.168.1.100:999" - api_key="mysecretkey" - timeout=30 - Note: API key is optional. Works on WiFi or cellular data. - -## USAGE - -After setup, all cmdlet work with the phone: -$ search-file zohar -store phone -$ @1-3 | add-relationship -king @4 -store phone -$ @1 | get-relationship -store phone - -The server exposes REST endpoints that RemoteStorageBackend uses internally. -""" - -from __future__ import annotations - -import os -import sys -import argparse -import logging -import threading -import time -from pathlib import Path -from typing import Optional, Dict, Any -from datetime import datetime -from functools import wraps - -# Add parent directory to path for imports -sys.path.insert(0, str(Path(__file__).parent.parent)) - - -# ============================================================================ -# CONFIGURATION -# ============================================================================ - -logging.basicConfig( - level=logging.INFO, - format="[%(asctime)s] %(levelname)s: %(message)s" -) -logger = logging.getLogger(__name__) - -STORAGE_PATH: Optional[Path] = None -API_KEY: Optional[str] = None # API key for authentication (None = no auth required) - -# Cache for database connection to prevent "database is locked" on high frequency requests -_DB_CACHE: Dict[str, Any] = {} - -def get_db(path: Path): - from API.folder import LocalLibrarySearchOptimizer - p_str = str(path) - if p_str not in _DB_CACHE: - _DB_CACHE[p_str] = LocalLibrarySearchOptimizer(path) - _DB_CACHE[p_str].__enter__() - return _DB_CACHE[p_str] - -# Try importing Flask - will be used in main() only -try: - from flask import Flask, request, jsonify - from flask_cors import CORS - - HAS_FLASK = True -except ImportError: - HAS_FLASK = False - -# ============================================================================ -# UTILITY FUNCTIONS -# ============================================================================ - - -def monitor_parent(parent_pid: int): - """Monitor the parent process and shut down if it dies.""" - if parent_pid <= 1: - return - - logger.info(f"Monitoring parent process {parent_pid}") - - # On Windows, we might need a different approach if os.kill(pid, 0) is unreliable - is_windows = sys.platform == "win32" - - while True: - try: - if is_windows: - # OpenProcess with PROCESS_QUERY_LIMITED_INFORMATION (0x1000) - # This is safer than os.kill on Windows for existence checks - import ctypes - PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 - handle = ctypes.windll.kernel32.OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, False, parent_pid) - if handle: - exit_code = ctypes.c_ulong() - ctypes.windll.kernel32.GetExitCodeProcess(handle, ctypes.byref(exit_code)) - ctypes.windll.kernel32.CloseHandle(handle) - # STILL_ACTIVE is 259 - if exit_code.value != 259: - logger.info(f"Parent process {parent_pid} finished with code {exit_code.value}. Shutting down...") - os._exit(0) - else: - # On Windows, sometimes we lose access to the handle if the parent is transitioning - # or if it was started from a shell that already closed. - # We'll ignore handle failures for now unless we want to be very strict. - pass - else: - os.kill(parent_pid, 0) - except Exception as e: - # Parent is dead or inaccessible - logger.info(f"Parent process {parent_pid} no longer accessible: {e}. Shutting down server...") - os._exit(0) - time.sleep(5) # Increase check interval to be less aggressive - - -def get_local_ip() -> Optional[str]: - """Get the local IP address that would be used for external connections.""" - import socket - - try: - # Create a socket to determine which interface would be used - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.connect(("8.8.8.8", 80)) # Google DNS - ip = s.getsockname()[0] - s.close() - return ip - except Exception: - return None - - -# ============================================================================ -# FLASK APP FACTORY -# ============================================================================ - - -def create_app(): - """Create and configure Flask app with all routes.""" - if not HAS_FLASK: - raise ImportError( - "Flask not installed. Install with: pip install flask flask-cors" - ) - - from flask import Flask, request, jsonify, send_file - from flask_cors import CORS - - app = Flask(__name__) - CORS(app) - - # ======================================================================== - # HELPER DECORATORS - # ======================================================================== - - def require_auth(): - """Decorator to check API key authentication if configured.""" - - def decorator(f): - - @wraps(f) - def decorated_function(*args, **kwargs): - if API_KEY: - # Get API key from header or query parameter - provided_key = request.headers.get("X-API-Key" - ) or request.args.get("api_key") - if not provided_key or provided_key != API_KEY: - return jsonify({"error": "Unauthorized. Invalid or missing API key."}), 401 - return f(*args, **kwargs) - - return decorated_function - - return decorator - - def require_storage(): - """Decorator to ensure storage path is configured.""" - - def decorator(f): - - @wraps(f) - def decorated_function(*args, **kwargs): - if not STORAGE_PATH: - return jsonify({"error": "Storage path not configured"}), 500 - return f(*args, **kwargs) - - return decorated_function - - return decorator - - # ======================================================================== - # HEALTH CHECK - # ======================================================================== - - @app.route("/health", methods=["GET"]) - def health(): - """Check server health and storage availability.""" - # Check auth manually to allow discovery even if locked - authed = True - if API_KEY: - provided_key = request.headers.get("X-API-Key") or request.args.get("api_key") - if not provided_key or provided_key != API_KEY: - authed = False - - status = { - "status": "ok", - "service": "remote_storage", - "name": os.environ.get("MM_SERVER_NAME", "Remote Storage"), - "storage_configured": STORAGE_PATH is not None, - "timestamp": datetime.now().isoformat(), - "locked": not authed and API_KEY is not None - } - - # If not authed but API_KEY is required, return minimal info for discovery - if not authed and API_KEY: - return jsonify(status), 200 - - if STORAGE_PATH: - status["storage_path"] = str(STORAGE_PATH) - status["storage_exists"] = STORAGE_PATH.exists() - try: - search_db = get_db(STORAGE_PATH) - status["database_accessible"] = True - except Exception as e: - status["database_accessible"] = False - status["database_error"] = str(e) - - return jsonify(status), 200 - - # ======================================================================== - # FILE OPERATIONS - # ======================================================================== - - @app.route("/files/search", methods=["GET"]) - @require_auth() - @require_storage() - def search_files(): - """Search for files by name or tag.""" - query = request.args.get("q", "") - limit = request.args.get("limit", 100, type=int) - - # Allow empty query or '*' for "list everything" - db_query = query if query and query != "*" else "" - - try: - search_db = get_db(STORAGE_PATH) - results = search_db.search_by_name(db_query, limit) - tag_results = search_db.search_by_tag(db_query, limit) - all_results_dict = { - r["hash"]: r - for r in (results + tag_results) - } - - # Fetch tags for each result to support title extraction on client - if search_db.db: - for res in all_results_dict.values(): - file_hash = res.get("hash") - if file_hash: - tags = search_db.db.get_tags(file_hash) - res["tag"] = tags - - return ( - jsonify( - { - "query": query, - "count": len(all_results_dict), - "files": list(all_results_dict.values()), - } - ), - 200, - ) - except Exception as e: - logger.error(f"Search error: {e}", exc_info=True) - return jsonify({"error": f"Search failed: {str(e)}"}), 500 - - @app.route("/files/", methods=["GET"]) - @require_auth() - @require_storage() - def get_file_metadata(file_hash: str): - """Get metadata for a specific file by hash.""" - try: - search_db = get_db(STORAGE_PATH) - db = search_db.db - if not db: - return jsonify({"error": "Database unavailable"}), 500 - - file_path = db.search_hash(file_hash) - - if not file_path or not file_path.exists(): - return jsonify({"error": "File not found"}), 404 - - metadata = db.get_metadata(file_hash) - tags = db.get_tags(file_hash) # Use hash string - - return ( - jsonify( - { - "hash": file_hash, - "path": str(file_path), - "size": file_path.stat().st_size, - "metadata": metadata, - "tag": tags, - } - ), - 200, - ) - except Exception as e: - logger.error(f"Get metadata error: {e}", exc_info=True) - return jsonify({"error": f"Failed to get metadata: {str(e)}"}), 500 - - @app.route("/files/raw/", methods=["GET"]) - @require_auth() - @require_storage() - def download_file(file_hash: str): - """Download a raw file by hash.""" - try: - search_db = get_db(STORAGE_PATH) - db = search_db.db - if not db: - return jsonify({"error": "Database unavailable"}), 500 - - file_path = db.search_hash(file_hash) - - if not file_path or not file_path.exists(): - return jsonify({"error": "File not found"}), 404 - - return send_file(file_path) - except Exception as e: - logger.error(f"Download error: {e}", exc_info=True) - return jsonify({"error": f"Download failed: {str(e)}"}), 500 - - @app.route("/files/index", methods=["POST"]) - @require_auth() - @require_storage() - def index_file(): - """Index a new file in the storage.""" - from SYS.utils import sha256_file - - data = request.get_json() or {} - file_path_str = data.get("path") - tags = data.get("tag", []) - url = data.get("url", []) - - if not file_path_str: - return jsonify({"error": "File path required"}), 400 - - try: - file_path = Path(file_path_str) - - if not file_path.exists(): - return jsonify({"error": "File does not exist"}), 404 - - search_db = get_db(STORAGE_PATH) - db = search_db.db - if not db: - return jsonify({"error": "Database unavailable"}), 500 - - db.get_or_create_file_entry(file_path) - - if tags: - db.add_tags(file_path, tags) - - if url: - db.add_url(file_path, url) - - file_hash = sha256_file(file_path) - - return ( - jsonify( - { - "hash": file_hash, - "path": str(file_path), - "tags_added": len(tags), - "url_added": len(url), - } - ), - 201, - ) - except Exception as e: - logger.error(f"Index error: {e}", exc_info=True) - return jsonify({"error": f"Indexing failed: {str(e)}"}), 500 - - @app.route("/files/upload", methods=["POST"]) - @require_auth() - @require_storage() - def upload_file(): - """Upload a file into storage (multipart/form-data). - - Accepts form fields: - - file: uploaded file (required) - - tag: repeated tag parameters or comma-separated string - - url: repeated url parameters or comma-separated string - """ - from API.folder import API_folder_store - from SYS.utils import sha256_file, sanitize_filename, ensure_directory, unique_path - - if 'file' not in request.files: - return jsonify({"error": "file required"}), 400 - file_storage = request.files.get('file') - if file_storage is None: - return jsonify({"error": "file required"}), 400 - - filename = sanitize_filename(file_storage.filename or "upload") - incoming_dir = STORAGE_PATH / "incoming" - target_path = incoming_dir / filename - target_path = unique_path(target_path) - - try: - # Initialize the DB first (run safety checks) before creating any files. - with API_folder_store(STORAGE_PATH) as db: - # Ensure the incoming directory exists only after DB safety checks pass. - ensure_directory(incoming_dir) - - # Save uploaded file to storage - file_storage.save(str(target_path)) - - # Extract optional metadata - tags = [] - if 'tag' in request.form: - # Support repeated form fields or comma-separated list - tags = request.form.getlist('tag') or [] - if not tags and request.form.get('tag'): - tags = [t.strip() for t in str(request.form.get('tag') or "").split(",") if t.strip()] - - urls = [] - if 'url' in request.form: - urls = request.form.getlist('url') or [] - if not urls and request.form.get('url'): - urls = [u.strip() for u in str(request.form.get('url') or "").split(",") if u.strip()] - - db.get_or_create_file_entry(target_path) - - if tags: - db.add_tags(target_path, tags) - - if urls: - db.add_url(target_path, urls) - - file_hash = sha256_file(target_path) - - return ( - jsonify({ - "hash": file_hash, - "path": str(target_path), - "tags_added": len(tags), - "url_added": len(urls), - }), - 201, - ) - except Exception as e: - logger.error(f"Upload error: {e}", exc_info=True) - return jsonify({"error": f"Upload failed: {str(e)}"}), 500 - - # ======================================================================== - # TAG OPERATIONS - # ======================================================================== - - @app.route("/tags/", methods=["GET"]) - @require_auth() - @require_storage() - def get_tags(file_hash: str): - """Get tags for a file.""" - from API.folder import API_folder_store - - try: - with API_folder_store(STORAGE_PATH) as db: - file_path = db.search_hash(file_hash) - if not file_path: - return jsonify({"error": "File not found"}), 404 - - tags = db.get_tags(file_path) - return jsonify({"hash": file_hash, "tag": tags}), 200 - except Exception as e: - logger.error(f"Get tags error: {e}", exc_info=True) - return jsonify({"error": f"Failed: {str(e)}"}), 500 - - @app.route("/tags/", methods=["POST"]) - @require_auth() - @require_storage() - def add_tags(file_hash: str): - """Add tags to a file.""" - from API.folder import API_folder_store - - data = request.get_json() or {} - tags = data.get("tag", []) - mode = data.get("mode", "add") - - if not tags: - return jsonify({"error": "Tag required"}), 400 - - try: - with API_folder_store(STORAGE_PATH) as db: - file_path = db.search_hash(file_hash) - if not file_path: - return jsonify({"error": "File not found"}), 404 - - if mode == "replace": - db.remove_tags(file_path, db.get_tags(file_path)) - - db.add_tags(file_path, tags) - return jsonify({"hash": file_hash, "tag_added": len(tags), "mode": mode}), 200 - except Exception as e: - logger.error(f"Add tags error: {e}", exc_info=True) - return jsonify({"error": f"Failed: {str(e)}"}), 500 - - @app.route("/tags/", methods=["DELETE"]) - @require_auth() - @require_storage() - def remove_tags(file_hash: str): - """Remove tags from a file.""" - from API.folder import API_folder_store - - tags_str = request.args.get("tag", "") - - try: - with API_folder_store(STORAGE_PATH) as db: - file_path = db.search_hash(file_hash) - if not file_path: - return jsonify({"error": "File not found"}), 404 - - if tags_str: - tags_to_remove = [t.strip() for t in tags_str.split(",")] - else: - tags_to_remove = db.get_tags(file_path) - - db.remove_tags(file_path, tags_to_remove) - return jsonify({"hash": file_hash, "tags_removed": len(tags_to_remove)}), 200 - except Exception as e: - logger.error(f"Remove tags error: {e}", exc_info=True) - return jsonify({"error": f"Failed: {str(e)}"}), 500 - - # ======================================================================== - # RELATIONSHIP OPERATIONS - # ======================================================================== - - @app.route("/relationships/", methods=["GET"]) - @require_auth() - @require_storage() - def get_relationships(file_hash: str): - """Get relationships for a file.""" - from API.folder import API_folder_store - - try: - with API_folder_store(STORAGE_PATH) as db: - file_path = db.search_hash(file_hash) - if not file_path: - return jsonify({"error": "File not found"}), 404 - - metadata = db.get_metadata(file_path) - relationships = metadata.get("relationships", - {}) if metadata else {} - return jsonify({"hash": file_hash, "relationships": relationships}), 200 - except Exception as e: - logger.error(f"Get relationships error: {e}", exc_info=True) - return jsonify({"error": f"Failed: {str(e)}"}), 500 - - @app.route("/relationships", methods=["POST"]) - @require_auth() - @require_storage() - def set_relationship(): - """Set a relationship between two files.""" - from API.folder import API_folder_store - - data = request.get_json() or {} - from_hash = data.get("from_hash") - to_hash = data.get("to_hash") - rel_type = data.get("type", "alt") - - if not from_hash or not to_hash: - return jsonify({"error": "from_hash and to_hash required"}), 400 - - try: - with API_folder_store(STORAGE_PATH) as db: - from_path = db.search_hash(from_hash) - to_path = db.search_hash(to_hash) - - if not from_path or not to_path: - return jsonify({"error": "File not found"}), 404 - - db.set_relationship(from_path, to_path, rel_type) - return jsonify({"from_hash": from_hash, "to_hash": to_hash, "type": rel_type}), 200 - except Exception as e: - logger.error(f"Set relationship error: {e}", exc_info=True) - return jsonify({"error": f"Failed: {str(e)}"}), 500 - - # ======================================================================== - # URL OPERATIONS - # ======================================================================== - - @app.route("/url/", methods=["GET"]) - @require_auth() - @require_storage() - def get_url(file_hash: str): - """Get known url for a file.""" - from API.folder import API_folder_store - - try: - with API_folder_store(STORAGE_PATH) as db: - file_path = db.search_hash(file_hash) - if not file_path: - return jsonify({"error": "File not found"}), 404 - - metadata = db.get_metadata(file_path) - url = metadata.get("url", []) if metadata else [] - return jsonify({"hash": file_hash, "url": url}), 200 - except Exception as e: - logger.error(f"Get url error: {e}", exc_info=True) - return jsonify({"error": f"Failed: {str(e)}"}), 500 - - @app.route("/url/", methods=["POST"]) - @require_auth() - @require_storage() - def add_url(file_hash: str): - """Add url to a file.""" - from API.folder import API_folder_store - - data = request.get_json() or {} - url = data.get("url", []) - - if not url: - return jsonify({"error": "url required"}), 400 - - try: - with API_folder_store(STORAGE_PATH) as db: - file_path = db.search_hash(file_hash) - if not file_path: - return jsonify({"error": "File not found"}), 404 - - db.add_url(file_path, url) - return jsonify({"hash": file_hash, "url_added": len(url)}), 200 - except Exception as e: - logger.error(f"Add url error: {e}", exc_info=True) - return jsonify({"error": f"Failed: {str(e)}"}), 500 - - return app - - -# ============================================================================ -# MAIN -# ============================================================================ - - -def main(): - if not HAS_FLASK: - print("ERROR: Flask and flask-cors required") - print("Install with: pip install flask flask-cors") - sys.exit(1) - - parser = argparse.ArgumentParser( - description="Remote Storage Server for Medios-Macina", - epilog= - "Example: python remote_storage_server.py --storage-path /storage/media --port 999 --api-key mysecretkey", - ) - parser.add_argument( - "--storage-path", - type=str, - required=True, - help="Path to storage directory" - ) - parser.add_argument( - "--host", - type=str, - default="0.0.0.0", - help="Server host (default: 0.0.0.0)" - ) - parser.add_argument( - "--port", - type=int, - default=999, - help="Server port (default: 999)" - ) - parser.add_argument( - "--api-key", - type=str, - default=None, - help="API key for authentication (optional)" - ) - parser.add_argument("--debug", action="store_true", help="Enable debug mode") - parser.add_argument( - "--monitor", - action="store_true", - help="Shut down if parent process dies" - ) - parser.add_argument( - "--parent-pid", - type=int, - default=None, - help="Explicit PID to monitor (defaults to the immediate parent process)", - ) - - args = parser.parse_args() - - # Start monitor thread if requested - if args.monitor: - monitor_pid = args.parent_pid or os.getppid() - if monitor_pid > 1: - monitor_thread = threading.Thread( - target=monitor_parent, - args=(monitor_pid, ), - daemon=True - ) - monitor_thread.start() - - global STORAGE_PATH, API_KEY - STORAGE_PATH = Path(args.storage_path).resolve() - API_KEY = args.api_key - - if not STORAGE_PATH.exists(): - print(f"ERROR: Storage path does not exist: {STORAGE_PATH}") - sys.exit(1) - - # Get local IP address - local_ip = get_local_ip() - if not local_ip: - local_ip = "127.0.0.1" - - print(f"\n{'='*70}") - print("Remote Storage Server - Medios-Macina") - print(f"{'='*70}") - print(f"Storage Path: {STORAGE_PATH}") - print(f"Local IP: {local_ip}") - print(f"Server URL: http://{local_ip}:{args.port}") - print(f"Health URL: http://{local_ip}:{args.port}/health") - print( - f"API Key: {'Enabled - ' + ('***' + args.api_key[-4:]) if args.api_key else 'Disabled (no auth)'}" - ) - print(f"Debug Mode: {args.debug}") - print("\nšŸ“‹ Config for config.conf:") - print("[store=remote]") - print('name="phone"') - print(f'url="http://{local_ip}:{args.port}"') - if args.api_key: - print(f'api_key="{args.api_key}"') - print("timeout=30") - - print("\nOR use ZeroTier Networking (Server Side):") - print("[networking=zerotier]") - print(f'serve="{STORAGE_PATH.name}"') - print(f'port="{args.port}"') - if args.api_key: - print(f'api_key="{args.api_key}"') - print(f"\n{'='*70}\n") - - try: - from API.folder import API_folder_store - - with API_folder_store(STORAGE_PATH) as db: - logger.info("Database initialized successfully") - except Exception as e: - logger.error(f"Failed to initialize database: {e}") - sys.exit(1) - - app = create_app() - app.run(host=args.host, port=args.port, debug=args.debug, use_reloader=False) - - -if __name__ == "__main__": - main() diff --git a/scripts/zerotier_setup.py b/scripts/zerotier_setup.py deleted file mode 100644 index c88b42d..0000000 --- a/scripts/zerotier_setup.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python3 -"""Simple ZeroTier helper for joining networks and discovering peers. - -Usage: - python scripts/zerotier_setup.py --join - python scripts/zerotier_setup.py --list - python scripts/zerotier_setup.py --discover - -This is a convenience tool to exercise the API/zerotier.py functionality while -prototyping and bringing up remote peers for store testing. -""" -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path - -from SYS.logger import log - -try: - from API import zerotier -except Exception: - zerotier = None - - -def main(argv=None): - parser = argparse.ArgumentParser(description="ZeroTier helper for Medios-Macina") - parser.add_argument("--list", action="store_true", help="List local ZeroTier networks") - parser.add_argument("--join", type=str, help="Join a ZeroTier network by ID") - parser.add_argument("--leave", type=str, help="Leave a ZeroTier network by ID") - parser.add_argument("--discover", type=str, help="Discover services on a ZeroTier network ID") - parser.add_argument("--upload", type=str, help="Upload a file to a discovered 'remote' service on this ZeroTier network ID") - parser.add_argument("--file", type=str, help="Local file to upload (used with --upload)") - parser.add_argument("--tag", action="append", help="Tag to attach (repeatable)", default=[]) - parser.add_argument("--url", action="append", help="URL to associate (repeatable)", default=[]) - parser.add_argument("--api-key", type=str, help="API key to use for uploads (optional)") - parser.add_argument("--json", action="store_true", help="Output JSON when appropriate") - args = parser.parse_args(argv) - - if zerotier is None: - log("ZeroTier API module not available; ensure API/zerotier.py is importable and zerotier or zerotier-cli is installed") - return 1 - - if args.list: - nets = zerotier.list_networks() - if args.json: - print(json.dumps([n.__dict__ for n in nets], indent=2)) - else: - for n in nets: - print(f"{n.id}\t{name:=}{n.name}\t{n.status}\t{n.assigned_addresses}") - return 0 - - if args.join: - try: - ok = zerotier.join_network(args.join) - print("Joined" if ok else "Failed to join") - return 0 if ok else 2 - except Exception as exc: - log(f"Join failed: {exc}") - print(f"Join failed: {exc}") - return 2 - - if args.leave: - try: - ok = zerotier.leave_network(args.leave) - print("Left" if ok else "Failed to leave") - return 0 if ok else 2 - except Exception as exc: - log(f"Leave failed: {exc}") - print(f"Leave failed: {exc}") - return 2 - - if args.discover: - probes = zerotier.discover_services_on_network(args.discover) - if args.json: - print(json.dumps([p.__dict__ for p in probes], indent=2, default=str)) - else: - for p in probes: - print(f"{p.address}:{p.port}{p.path} -> status={p.status_code} hint={p.service_hint}") - return 0 - - if args.upload: - # Upload a file to the first discovered remote service on the network - if not args.file: - print("ERROR: --file is required for --upload") - return 2 - - probe = zerotier.find_peer_service(args.upload, service_hint="remote") - if not probe: - print("No remote service found on network") - return 2 - - base = f"http://{probe.address}:{probe.port}" - try: - import httpx - url = base.rstrip("/") + "/files/upload" - headers = {} - if args.api_key: - headers["X-API-Key"] = args.api_key - with open(args.file, "rb") as fh: - files = {"file": (Path(args.file).name, fh)} - data = [] - for t in (args.tag or []): - data.append(("tag", t)) - for u in (args.url or []): - data.append(("url", u)) - resp = httpx.post(url, files=files, data=data, headers=headers, timeout=30) - print(resp.status_code, resp.text) - return 0 if resp.status_code in (200, 201) else 2 - except Exception: - import requests - url = base.rstrip("/") + "/files/upload" - headers = {} - if args.api_key: - headers["X-API-Key"] = args.api_key - with open(args.file, "rb") as fh: - files = {"file": (Path(args.file).name, fh)} - data = [] - for t in (args.tag or []): - data.append(("tag", t)) - for u in (args.url or []): - data.append(("url", u)) - resp = requests.post(url, files=files, data=data, headers=headers, timeout=30) - print(resp.status_code, resp.text) - return 0 if resp.status_code in (200, 201) else 2 - - parser.print_help() - return 0 - - -if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file