From 226367a6eae9b7325636fa8b919355f6f7f8ed73 Mon Sep 17 00:00:00 2001 From: Nose Date: Tue, 13 Jan 2026 20:04:24 -0800 Subject: [PATCH] df --- API/data/alldebrid.json | 6 +- API/zerotier.py | 352 +++++++++++++++++++++ MPV/portable_config/mpv.conf | 2 +- Store/ZeroTier.py | 515 +++++++++++++++++++++++++++++++ docs/zerotier.md | 88 ++++++ scripts/remote_storage_server.py | 68 ++++ scripts/zerotier_setup.py | 124 ++++++++ 7 files changed, 1151 insertions(+), 4 deletions(-) create mode 100644 API/zerotier.py create mode 100644 Store/ZeroTier.py create mode 100644 docs/zerotier.md create mode 100644 scripts/zerotier_setup.py diff --git a/API/data/alldebrid.json b/API/data/alldebrid.json index 28ba01d..c2f1d35 100644 --- a/API/data/alldebrid.json +++ b/API/data/alldebrid.json @@ -353,7 +353,7 @@ "filedot\\.(xyz|to|top)/([0-9a-zA-Z]{12})" ], "regexp": "filedot\\.(xyz|to|top)/([0-9a-zA-Z]{12})", - "status": true + "status": false }, "filefactory": { "name": "filefactory", @@ -389,7 +389,7 @@ "(filespace\\.com/[a-zA-Z0-9]{12})" ], "regexp": "(filespace\\.com/fd/([a-zA-Z0-9]{12}))|((filespace\\.com/[a-zA-Z0-9]{12}))", - "status": false + "status": true }, "filezip": { "name": "filezip", @@ -786,7 +786,7 @@ "(upl\\.wf/d/[0-9a-zA-Z]+)" ], "regexp": "((world\\-files\\.com/[0-9a-zA-Z]{12}))|((upl\\.wf/d/[0-9a-zA-Z]+))", - "status": true, + "status": false, "hardRedirect": [ "world\\-files\\.com/([0-9a-zA-Z]{12})" ] diff --git a/API/zerotier.py b/API/zerotier.py new file mode 100644 index 0000000..283099d --- /dev/null +++ b/API/zerotier.py @@ -0,0 +1,352 @@ +"""ZeroTier helpers and discovery utilities. + +This module provides a small, dependency-light API for interacting with a +local zerotier-one node (preferred via Python module when available, else via +`zerotier-cli`), discovering peers on a given ZeroTier network, and probing +for services running on those peers (e.g., our remote storage server or a +Hydrus instance). + +Notes: +- This is intentionally conservative and all operations are best-effort and + fail gracefully when the local system does not have ZeroTier installed. +- The implementation prefers a Python ZeroTier binding when available, else + falls back to calling the `zerotier-cli` binary (if present) and parsing + JSON output where possible. + +Example usage: + from API import zerotier + if zerotier.is_available(): + nets = zerotier.list_networks() + zerotier.join_network("8056c2e21c000001") + services = zerotier.discover_services_on_network("8056c2e21c000001", ports=[5000], paths=["/health","/api_version"]) # noqa: E501 +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from SYS.logger import debug, log + +# Optional Python ZeroTier bindings - prefer them when available +_HAVE_PY_ZEROTIER = False +try: + # Try common package names; not all installations will have this available + # This import is optional and callers should still work via the CLI fallback. + import zerotier as _zt # type: ignore + _HAVE_PY_ZEROTIER = True +except Exception: + _HAVE_PY_ZEROTIER = False + + +@dataclass +class ZeroTierNetwork: + id: str + name: str + status: str + assigned_addresses: List[str] + + +@dataclass +class ZeroTierServiceProbe: + address: str + port: int + path: str + url: str + ok: bool + status_code: Optional[int] + payload: Optional[Any] + service_hint: Optional[str] = None + + +def _cli_available() -> bool: + return bool(shutil.which("zerotier-cli")) + + +def is_available() -> bool: + """Return True if we can interact with ZeroTier locally (module or CLI).""" + return _HAVE_PY_ZEROTIER or _cli_available() + + +def _run_cli_json(*args: str, timeout: float = 5.0) -> Any: + """Run zerotier-cli with arguments and parse JSON output if possible. + + Returns parsed JSON on success, or raises an exception. + """ + bin_path = shutil.which("zerotier-cli") + if not bin_path: + raise RuntimeError("zerotier-cli not found") + + cmd = [bin_path, *args] + debug(f"Running zerotier-cli: {cmd}") + out = subprocess.check_output(cmd, timeout=timeout) + try: + return json.loads(out.decode("utf-8")) + except Exception: + # Some CLI invocations might print non-json; return as raw string + return out.decode("utf-8", "replace") + + +def list_networks() -> List[ZeroTierNetwork]: + """Return a list of configured ZeroTier networks on this node. + + Best-effort: prefers Python binding, then `zerotier-cli listnetworks -j`. + """ + nets: List[ZeroTierNetwork] = [] + + if _HAVE_PY_ZEROTIER: + try: + # Attempt to use common API shape (best-effort) + raw = _zt.list_networks() # type: ignore[attr-defined] + for n in raw or []: + nets.append(ZeroTierNetwork( + id=str(n.get("id") or n.get("networkId") or ""), + name=str(n.get("name") or ""), + status=str(n.get("status") or ""), + assigned_addresses=list(n.get("assignedAddresses") or []), + )) + return nets + except Exception as exc: # pragma: no cover - optional dependency + debug(f"py-zerotier listing failed: {exc}") + + # CLI fallback + try: + data = _run_cli_json("listnetworks", "-j") + if isinstance(data, list): + for entry in data: + nets.append(ZeroTierNetwork( + id=str(entry.get("id") or ""), + name=str(entry.get("name") or ""), + status=str(entry.get("status") or ""), + assigned_addresses=list(entry.get("assignedAddresses") or []), + )) + except Exception as exc: + debug(f"list_networks failed: {exc}") + + return nets + + +def join_network(network_id: str) -> bool: + """Join the given ZeroTier network (best-effort). + + Returns True on success, False otherwise. + """ + network_id = str(network_id or "").strip() + if not network_id: + raise ValueError("network_id is required") + + if _HAVE_PY_ZEROTIER: + try: + _zt.join_network(network_id) # type: ignore[attr-defined] + return True + except Exception as exc: # pragma: no cover - optional dependency + debug(f"py-zerotier join failed: {exc}") + + if _cli_available(): + try: + subprocess.check_call([shutil.which("zerotier-cli"), "join", network_id], timeout=10) + return True + except Exception as exc: + debug(f"zerotier-cli join failed: {exc}") + return False + + +def leave_network(network_id: str) -> bool: + network_id = str(network_id or "").strip() + if not network_id: + raise ValueError("network_id is required") + + if _HAVE_PY_ZEROTIER: + try: + _zt.leave_network(network_id) # type: ignore[attr-defined] + return True + except Exception as exc: # pragma: no cover - optional dependency + debug(f"py-zerotier leave failed: {exc}") + + if _cli_available(): + try: + subprocess.check_call([shutil.which("zerotier-cli"), "leave", network_id], timeout=10) + return True + except Exception as exc: + debug(f"zerotier-cli leave failed: {exc}") + return False + + +def _strip_addr(addr: str) -> str: + # Remove trailing CID parts like '/24' and zone IDs like '%eth0' + if not addr: + return addr + a = addr.split("/")[0] + if "%" in a: + a = a.split("%", 1)[0] + return a + + +def get_assigned_addresses(network_id: str) -> List[str]: + """Return assigned ZeroTier addresses for the local node on the given network.""" + network_id = str(network_id or "").strip() + if not network_id: + return [] + + for n in list_networks(): + if n.id == network_id: + return [str(_strip_addr(a)) for a in n.assigned_addresses if a] + return [] + + +def list_peers() -> List[Dict[str, Any]]: + """Return peers known to the local ZeroTier node (best-effort parsing). + + If CLI supports JSON output for peers it will be parsed, otherwise we return + an empty list. + """ + if _HAVE_PY_ZEROTIER: + try: + peers = _zt.list_peers() # type: ignore[attr-defined] + return list(peers or []) + except Exception as exc: # pragma: no cover - optional dependency + debug(f"py-zerotier list_peers failed: {exc}") + + if _cli_available(): + try: + data = _run_cli_json("peers", "-j") + if isinstance(data, list): + return data + except Exception as exc: + debug(f"zerotier-cli peers failed: {exc}") + return [] + + +def _probe_url(url: str, *, timeout: float = 2.0, accept_json: bool = True) -> Tuple[bool, Optional[int], Optional[Any]]: + """Try fetching the URL and return (ok, status_code, payload). + + Uses httpx if available, otherwise falls back to requests. + """ + try: + try: + import httpx + resp = httpx.get(url, timeout=timeout) + code = int(resp.status_code if hasattr(resp, "status_code") else resp.status) + content_type = str(resp.headers.get("content-type") or "").lower() + if code == 200 and accept_json and "json" in content_type: + try: + return True, code, resp.json() + except Exception: + return True, code, resp.text + return (code == 200), code, resp.text + except Exception: + import requests # type: ignore + resp = requests.get(url, timeout=timeout) + code = int(resp.status_code) + content_type = str(resp.headers.get("content-type") or "").lower() + if code == 200 and accept_json and "json" in content_type: + try: + return True, code, resp.json() + except Exception: + return True, code, resp.text + return (code == 200), code, resp.text + except Exception as exc: + debug(f"Probe failed: {url} -> {exc}") + return False, None, None + + +def discover_services_on_network( + network_id: str, + *, + ports: Optional[List[int]] = None, + paths: Optional[List[str]] = None, + timeout: float = 2.0, + accept_json: bool = True, +) -> List[ZeroTierServiceProbe]: + """Probe assigned addresses on the given network for HTTP services. + + Returns a list of ZeroTierServiceProbe entries for successful probes. + + By default probes `ports=[5000]` (our remote_storage_server default) and + `paths=["/health","/api_version"]` which should detect either our + remote_storage_server or Hydrus instances. + """ + net = str(network_id or "").strip() + if not net: + raise ValueError("network_id required") + + ports = list(ports or [5000]) + paths = list(paths or ["/health", "/api_version", "/api_version/", "/session_key"]) + + addresses = get_assigned_addresses(net) + probes: List[ZeroTierServiceProbe] = [] + + for addr in addresses: + host = str(addr or "").strip() + if not host: + continue + # Try both http and https schemes + for port in ports: + for path in paths: + for scheme in ("http", "https"): + url = f"{scheme}://{host}:{port}{path}" + ok, code, payload = _probe_url(url, timeout=timeout, accept_json=accept_json) + if ok: + hint = None + # Heuristics: hydrus exposes /api_version with a JSON payload + try: + if isinstance(payload, dict) and payload.get("api_version"): + hint = "hydrus" + except Exception: + pass + try: + if isinstance(payload, dict) and payload.get("status"): + hint = hint or "remote_storage" + except Exception: + pass + + probes.append(ZeroTierServiceProbe( + address=host, + port=int(port), + path=path, + url=url, + ok=True, + status_code=code, + payload=payload, + service_hint=hint, + )) + # stop probing other schemes for this host/port/path + break + return probes + + +def find_peer_service( + network_id: str, + *, + service_hint: Optional[str] = None, + port: Optional[int] = None, + path_candidates: Optional[List[str]] = None, + timeout: float = 2.0, +) -> Optional[ZeroTierServiceProbe]: + """Return the first probe that matches service_hint or is successful. + + Useful for selecting a peer to configure a store against. + """ + paths = path_candidates or ["/health", "/api_version", "/session_key"] + ports = [port] if port is not None else [5000, 45869, 80, 443] + + probes = discover_services_on_network(network_id, ports=ports, paths=paths, timeout=timeout) + if not probes: + return None + if service_hint: + for p in probes: + if p.service_hint and service_hint.lower() in str(p.service_hint).lower(): + return p + # Hydrus detection: check payload for 'api_version' + try: + if service_hint.lower() == "hydrus" and isinstance(p.payload, dict) and p.payload.get("api_version"): + return p + except Exception: + pass + # Fallback: return the first OK probe + return probes[0] if probes else None diff --git a/MPV/portable_config/mpv.conf b/MPV/portable_config/mpv.conf index c1750d7..39e96b3 100644 --- a/MPV/portable_config/mpv.conf +++ b/MPV/portable_config/mpv.conf @@ -10,7 +10,7 @@ osd-fonts-dir=~~/scripts/uosc/fonts sub-fonts-dir=~~/scripts/uosc/ ontop=yes -autofit=100% +autofit=45% # Avoid showing embedded cover art for audio-only files if uosc isn't working, # but we keep it enabled for now to ensure a window exists. audio-display=yes diff --git a/Store/ZeroTier.py b/Store/ZeroTier.py new file mode 100644 index 0000000..d9bcdcc --- /dev/null +++ b/Store/ZeroTier.py @@ -0,0 +1,515 @@ +"""ZeroTier-backed Store implementation. + +This store locates a service running on peers in a ZeroTier network and +proxies store operations to that remote service. The remote service can be +our `remote_storage_server` (default) or a Hydrus API server (`service=hydrus`). + +Configuration keys: +- NAME: store instance name (required) +- NETWORK_ID: ZeroTier network ID to use for discovery (required) +- SERVICE: 'remote' or 'hydrus' (default: 'remote') +- PORT: service port (default: 5000 for remote, 45869 for hydrus) +- API_KEY: optional API key to include in requests +- HOST: optional preferred peer address (skip discovery if provided) + +Notes: +- This implementation focuses on read operations (search, get_file, get_metadata, + tag/url ops). Uploads can be implemented later when the remote server + supports a robust, authenticated upload endpoint. +""" + +from __future__ import annotations + +import json +import sys +import time +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from SYS.logger import debug, log +from Store._base import Store + + +class ZeroTier(Store): + + @classmethod + def config(cls) -> List[Dict[str, Any]]: + return [ + {"key": "NAME", "label": "Store Name", "default": "", "required": True}, + {"key": "NETWORK_ID", "label": "ZeroTier Network ID", "default": "", "required": True}, + {"key": "SERVICE", "label": "Service Type (remote|hydrus)", "default": "remote", "required": True}, + {"key": "PORT", "label": "Service Port", "default": "5000", "required": False}, + {"key": "API_KEY", "label": "API Key (optional)", "default": "", "required": False, "secret": True}, + {"key": "HOST", "label": "Preferred peer host (optional)", "default": "", "required": False}, + {"key": "TIMEOUT", "label": "Request timeout (s)", "default": "5", "required": False}, + ] + + def __new__(cls, *args: Any, **kwargs: Any) -> "ZeroTier": + inst = super().__new__(cls) + name = kwargs.get("NAME") + if name is not None: + setattr(inst, "NAME", str(name)) + return inst + + def __init__( + self, + instance_name: Optional[str] = None, + network_id: Optional[str] = None, + service: Optional[str] = None, + port: Optional[int] = None, + api_key: Optional[str] = None, + host: Optional[str] = None, + timeout: Optional[int] = None, + *, + NAME: Optional[str] = None, + NETWORK_ID: Optional[str] = None, + SERVICE: Optional[str] = None, + PORT: Optional[int] = None, + API_KEY: Optional[str] = None, + HOST: Optional[str] = None, + TIMEOUT: Optional[int] = None, + ) -> None: + if instance_name is None and NAME is not None: + instance_name = str(NAME) + if network_id is None and NETWORK_ID is not None: + network_id = str(NETWORK_ID) + if service is None and SERVICE is not None: + service = str(SERVICE) + if port is None and PORT is not None: + try: + port = int(PORT) + except Exception: + port = None + if api_key is None and API_KEY is not None: + api_key = str(API_KEY) + if host is None and HOST is not None: + host = str(HOST) + if timeout is None and TIMEOUT is not None: + try: + timeout = int(TIMEOUT) + except Exception: + timeout = None + + self._name = str(instance_name or "") + self._network_id = str(network_id or "").strip() + self._service = (str(service or "remote") or "remote").lower() + self._port = int(port if port is not None else (45869 if self._service == "hydrus" else 5000)) + self._api_key = str(api_key or "").strip() or None + self._preferred_host = str(host or "").strip() or None + self._timeout = int(timeout or 5) + + # Cached discovery result + self._cached_peer: Optional[Tuple[str, int]] = None + self._cached_client: Optional[Any] = None + + def name(self) -> str: + return str(getattr(self, "_name", "zerotier")) + + # -------------------- internal helpers -------------------- + def _discover_peer(self, *, refresh: bool = False) -> Optional[Tuple[str, int]]: + """Discover a peer host:port for this service on the configured network. + + Returns (host, port) or None. + """ + if self._preferred_host and not refresh: + return (self._preferred_host, self._port) + + if self._cached_peer and not refresh: + return self._cached_peer + + try: + from API import zerotier as zt + except Exception as exc: + debug(f"ZeroTier discovery helper not available: {exc}") + return None + + # Look for a matching service on the network + probe = zt.find_peer_service(self._network_id, service_hint=("hydrus" if self._service == "hydrus" else None), port=self._port) + if probe: + # Extract host:port + host = probe.address + port = probe.port or self._port + self._cached_peer = (host, int(port)) + debug(f"ZeroTier store '{self.name()}' discovered peer {host}:{port}") + return self._cached_peer + + debug(f"ZeroTier store '{self.name()}' found no peers on network {self._network_id}") + return None + + def _ensure_client(self, *, refresh: bool = False) -> Optional[Any]: + """Return a remote client object or base URL depending on service type. + + For 'hydrus' service we return an API.HydrusNetwork instance; for 'remote' + service we return a base URL string to send HTTP requests to. + """ + if self._cached_client and not refresh: + return self._cached_client + + peer = self._discover_peer(refresh=refresh) + if not peer: + return None + host, port = peer + + if self._service == "hydrus": + try: + from API.HydrusNetwork import HydrusNetwork as HydrusClient + base_url = f"http://{host}:{port}" + client = HydrusClient(url=base_url, access_key=(self._api_key or ""), timeout=self._timeout) + self._cached_client = client + return client + except Exception as exc: + debug(f"Failed to instantiate Hydrus client for ZeroTier peer {host}:{port}: {exc}") + return None + + # Default: remote_storage 'http' style API + self._cached_client = f"http://{host}:{port}" + return self._cached_client + + def _request_remote(self, method: str, path: str, *, params: Optional[Dict[str, Any]] = None, json_body: Optional[Any] = None, timeout: Optional[int] = None) -> Optional[Any]: + base = self._ensure_client() + if base is None or not isinstance(base, str): + debug("No remote base URL available for ZeroTier store") + return None + + url = base.rstrip("/") + path + headers = {} + if self._api_key: + headers["X-API-Key"] = self._api_key + + try: + import httpx + resp = httpx.request(method, url, params=params, json=json_body, headers=headers, timeout=timeout or self._timeout) + resp.raise_for_status() + try: + return resp.json() + except Exception: + return resp.text + except Exception as exc: + debug(f"ZeroTier HTTP request failed: {method} {url} -> {exc}") + return None + + # -------------------- Store API -------------------- + def search(self, query: str, **kwargs: Any) -> List[Dict[str, Any]]: + """Search for files on the remote service.""" + client = self._ensure_client() + if client is None: + debug("ZeroTier search: no client available") + return [] + + if self._service == "hydrus": + # Hydrus API expects tags list; best-effort: treat query as a single tag or raw search term + try: + tags = [query] + payload = client.search_files(tags, return_hashes=True, return_file_ids=False, return_file_count=False) + # Hydrus JSON shape varies; normalize to simple list + files = [] + try: + if isinstance(payload, dict): + rows = payload.get("files") or payload.get("metadata") or [] + for r in rows: + files.append(r if isinstance(r, dict) else {}) + except Exception: + pass + return files + except Exception as exc: + debug(f"Hydrus search failed: {exc}") + return [] + + # remote_storage path + params = {"q": query, "limit": int(kwargs.get("limit", 100))} + res = self._request_remote("GET", "/files/search", params=params) + if isinstance(res, dict): + return list(res.get("files") or []) + return [] + + def get_file(self, file_hash: str, **kwargs: Any) -> Optional[Path | str]: + """Return either a URL (hydrus or remote capable) or local path (not implemented). + + For Hydrus: return the direct file URL (Hydrus client URL with access token appended if needed). + For remote_storage: currently return the metadata path (if available) or None. + """ + client = self._ensure_client() + if client is None: + return None + + if self._service == "hydrus": + try: + # Hydrus wrapper provides file_url() convenience + return client.file_url(file_hash) + except Exception as exc: + debug(f"Hydrus get_file failed: {exc}") + return None + + # remote storage: try metadata endpoint + res = self._request_remote("GET", f"/files/{file_hash}") + if isinstance(res, dict): + # remote server returns a 'path' to the file (server-local path) + p = res.get("path") or res.get("file") or None + if isinstance(p, str) and p.startswith("http"): + return p + return p + return None + + def add_file(self, file_path: Path, **kwargs: Any) -> Optional[str]: + """Upload a local file to the remote ZeroTier peer (supports 'remote' and 'hydrus' services). + + Returns the file hash on success, or None on failure. + """ + from SYS.utils import sha256_file + + p = Path(file_path) + if not p.exists(): + debug(f"ZeroTier add_file: local file not found: {p}") + return None + + # Hydrus: delegate to Hydrus client add_file() + if self._service == "hydrus": + try: + client = self._ensure_client() + if client is None: + debug("ZeroTier add_file: Hydrus client unavailable") + return None + return client.add_file(p, **kwargs) + except Exception as exc: + debug(f"ZeroTier hydrus add_file failed: {exc}") + return None + + # Remote server: POST /files/upload multipart/form-data + base = self._ensure_client() + if base is None or not isinstance(base, str): + debug("ZeroTier add_file: no remote base URL available") + return None + + url = base.rstrip("/") + "/files/upload" + headers = {} + if self._api_key: + headers["X-API-Key"] = self._api_key + + try: + import httpx + with open(p, "rb") as fh: + # Build form fields for tags/urls (support list or comma-separated) + data = [] + if "tag" in kwargs: + tags = kwargs.get("tag") or [] + if isinstance(tags, str): + tags = [t.strip() for t in tags.split(",") if t.strip()] + for t in tags: + data.append(("tag", t)) + if "url" in kwargs: + urls = kwargs.get("url") or [] + if isinstance(urls, str): + urls = [u.strip() for u in urls.split(",") if u.strip()] + for u in urls: + data.append(("url", u)) + + files = {"file": (p.name, fh, "application/octet-stream")} + resp = httpx.post(url, headers=headers, files=files, data=data, timeout=self._timeout) + resp.raise_for_status() + if resp.status_code in (200, 201): + try: + payload = resp.json() + file_hash = payload.get("hash") or payload.get("file_hash") + return file_hash + except Exception: + return None + debug(f"ZeroTier add_file failed: status {resp.status_code}") + return None + except Exception as exc: + debug(f"ZeroTier add_file exception: {exc}") + return None + + def get_metadata(self, file_hash: str, **kwargs: Any) -> Optional[Dict[str, Any]]: + client = self._ensure_client() + if client is None: + return None + + if self._service == "hydrus": + try: + payload = client.fetch_file_metadata(hashes=[file_hash], include_file_url=True, include_size=True, include_mime=True) + return payload + except Exception as exc: + debug(f"Hydrus fetch_file_metadata failed: {exc}") + return None + + res = self._request_remote("GET", f"/files/{file_hash}") + if isinstance(res, dict): + return res + return None + + def get_tag(self, file_identifier: str, **kwargs: Any) -> Tuple[List[str], str]: + # Return (tags, service). For hydrus use fetch_file_metadata service keys. + client = self._ensure_client() + if client is None: + return ([], "") + + if self._service == "hydrus": + try: + payload = client.fetch_file_metadata(hashes=[file_identifier], include_service_keys_to_tags=True) + tags = [] + if isinstance(payload, dict): + metas = payload.get("metadata") or [] + if metas and isinstance(metas, list) and metas: + md = metas[0] + if isinstance(md, dict): + tags = md.get("service_keys_to_tags") or [] + return (tags, "hydrus") + except Exception as exc: + debug(f"Hydrus get_tag failed: {exc}") + return ([], "hydrus") + + res = self._request_remote("GET", f"/tags/{file_identifier}") + if isinstance(res, dict): + return (list(res.get("tag") or []), "remote") + return ([], "remote") + + def add_tag(self, file_identifier: str, tags: List[str], **kwargs: Any) -> bool: + client = self._ensure_client() + if client is None: + return False + + if self._service == "hydrus": + try: + service_name = kwargs.get("service_name") or "my tags" + client.add_tag(file_identifier, tags, service_name) + return True + except Exception as exc: + debug(f"Hydrus add_tag failed: {exc}") + return False + + payload = {"tag": tags} + res = self._request_remote("POST", f"/tags/{file_identifier}", json_body=payload) + return bool(res) + + def delete_tag(self, file_identifier: str, tags: List[str], **kwargs: Any) -> bool: + client = self._ensure_client() + if client is None: + return False + + if self._service == "hydrus": + try: + service_name = kwargs.get("service_name") or "my tags" + client.delete_tag(file_identifier, tags, service_name) + return True + except Exception as exc: + debug(f"Hydrus delete_tag failed: {exc}") + return False + + # remote_storage DELETE /tags/?tag=tag1,tag2 + query = {"tag": ",".join(tags)} + res = self._request_remote("DELETE", f"/tags/{file_identifier}", params=query) + return bool(res) + + def get_url(self, file_identifier: str, **kwargs: Any) -> List[str]: + # For Hydrus, use fetch_file_metadata to include file URL; for remote, GET tags endpoint includes urls + client = self._ensure_client() + if client is None: + return [] + + if self._service == "hydrus": + try: + payload = client.fetch_file_metadata(hashes=[file_identifier], include_file_url=True) + try: + metas = payload.get("metadata") or [] + if metas and isinstance(metas, list) and metas: + md = metas[0] + if isinstance(md, dict): + urls = md.get("file_urls") or [] + return list(urls) + except Exception: + pass + return [] + except Exception as exc: + debug(f"Hydrus get_url failed: {exc}") + return [] + + meta = self._request_remote("GET", f"/files/{file_identifier}") + if isinstance(meta, dict): + urls = meta.get("url") or [] + return list(urls) + return [] + + def add_url(self, file_identifier: str, url: List[str], **kwargs: Any) -> bool: + client = self._ensure_client() + if client is None: + return False + + if self._service == "hydrus": + try: + client.associate_url(hashes=[file_identifier], url=url[0]) + return True + except Exception as exc: + debug(f"Hydrus add_url failed: {exc}") + return False + + payload = {"url": url} + res = self._request_remote("POST", f"/files/{file_identifier}/url", json_body=payload) + return bool(res) + + def delete_url(self, file_identifier: str, url: List[str], **kwargs: Any) -> bool: + client = self._ensure_client() + if client is None: + return False + + if self._service == "hydrus": + try: + client.delete_urls(hashes=[file_identifier], urls=url) + return True + except Exception as exc: + debug(f"Hydrus delete_url failed: {exc}") + return False + + payload = {"url": url} + res = self._request_remote("DELETE", f"/files/{file_identifier}/url", json_body=payload) + return bool(res) + + def get_note(self, file_identifier: str, **kwargs: Any) -> Dict[str, str]: + """Get named notes for a file. Returns a mapping of name->text.""" + client = self._ensure_client() + if client is None: + return {} + + if self._service == "hydrus": + try: + # Hydrus API may expose notes via fetch_file_metadata; best-effort + payload = client.fetch_file_metadata(hashes=[file_identifier], include_notes=True) + if isinstance(payload, dict): + metas = payload.get("metadata") or [] + if metas and isinstance(metas, list): + md = metas[0] + notes = md.get("notes") or {} + return dict(notes) + except Exception: + return {} + + # Remote storage has no notes API yet + return {} + + def set_note(self, file_identifier: str, name: str, text: str, **kwargs: Any) -> bool: + client = self._ensure_client() + if client is None: + return False + + if self._service == "hydrus": + try: + client.set_note(file_identifier, name, text) + return True + except Exception: + return False + + # Remote storage: not supported + return False + + def delete_note(self, file_identifier: str, name: str, **kwargs: Any) -> bool: + client = self._ensure_client() + if client is None: + return False + + if self._service == "hydrus": + try: + client.delete_note(file_identifier, name) + return True + except Exception: + return False + + return False diff --git a/docs/zerotier.md b/docs/zerotier.md new file mode 100644 index 0000000..bc65ccc --- /dev/null +++ b/docs/zerotier.md @@ -0,0 +1,88 @@ +# ZeroTier integration (store sharing) + +This document describes how Medios-Macina integrates with ZeroTier to share +storage backends between machines on a private virtual network. + +Goals +- Allow you to expose stores (folder-based, remote storage server, Hydrus client) + to other members of your ZeroTier network. +- Keep the CLI experience identical: remote stores appear as normal `-store` backends. +- Use secure authentication (API keys / per-store tokens) and limit exposure to private network. + +Prerequisites +- Each machine must run `zerotier-one` and be a member of your ZeroTier network. +- The Medios-Macina instance on each machine should run the `remote_storage_server.py` + or a Hydrus client instance you want to expose. +- The remote storage server requires Flask and Flask-CORS to run (install with: `pip install flask flask-cors`). +- On your controller/management machine, authorize members via ZeroTier Central. + +Configuration (conceptual) + +You can configure networks and Zerotier-backed stores in your `config.conf`. Here +are example snippets and recommendations. + +## Top-level ZeroTier networks (recommended) + +Use a `zerotier` section to list networks your instance is willing to use/auto-join: + +```ini +[zerotier] +# Example config (implementation treats this as a dict via the loader) +# networks: +# home: +# network_id: 8056c2e21c000001 +# api_key: my-zt-central-token ; optional, only needed for automating member authorization +# auto_join: true +# prefer_hosts: ["192.168.86.42"] ; optional peer IP inside the ZT network +``` + +## Store config (ZeroTier store instances) + +Add a `store=zerotier` block so the Store registry can create a ZeroTier store instance: + +```ini +[store=zerotier] +my-remote = { "NAME": "my-remote", "NETWORK_ID": "8056c2e21c000001", "SERVICE": "remote", "PORT": 5000, "API_KEY": "myremotekey" } +hydrus-remote = { "NAME": "hydrus-remote", "NETWORK_ID": "8056c2e21c000001", "SERVICE": "hydrus", "PORT": 45869, "API_KEY": "hydrus-access-key" } +``` + +- `SERVICE` can be `remote` (our `remote_storage_server`), or `hydrus` (Hydrus API). +- `HOST` is optional; if present, discovery is skipped and the host:port is used. +- `API_KEY` will be sent as `X-API-Key` (and Hydrus access keys, when relevant). + +Operation & discovery +- The local ZeroTier store wrapper will attempt to discover peers on the configured + ZeroTier network by inspecting assigned addresses on this node and probing common + service endpoints (e.g., `/health`, `/api_version`). +- For `hydrus` service types we look for Hydrus-style `/api_version` responses. +- For `remote` service types we look for our `remote_storage_server` `/health` endpoint. + +Security notes +- Your ZeroTier network provides a private IP layer, but the exposed services + should still require authentication (API keys) and enforce scope (read/write). +- If you plan to expose stores to other users, consider per-store API keys with + roles (read-only, write, admin) and monitor/audit access. + +Next steps / prototyping +- The first prototype in this repo adds `API/zerotier.py` (discovery + join helpers) + and `Store/ZeroTier.py` (a store wrapper that proxies to `hydrus` or `remote` endpoints). +- Upload support (server-side `POST /files/upload`) is now implemented allowing authenticated multipart uploads; the ZeroTier store wrapper supports `add_file()` and the `add-file` cmdlet can be used with a configured ZeroTier store for end-to-end uploads. + +Example: upload via the helper script (discovers a remote on the network and uploads the file): + +```powershell +python .\scripts\zerotier_setup.py --upload 8056c2e21c000001 --file "C:\path\to\file.mp4" --api-key myremotekey --tag tag1 --tag tag2 +``` + +Or using curl directly against a discovered ZeroTier peer's IP: + +```powershell +curl -X POST -H "X-API-Key: myremotekey" -F "file=@/path/to/file.mp4" -F "tag=tag1" http://:5000/files/upload +``` + +If you'd like I can: +- Add an example `scripts/zt-join.py` helper that uses the API wrapper to join a network; +- Add a presigned-upload + multipart upload flow to `scripts/remote_storage_server.py` so + ZeroTier stores can support `add-file` uploads directly. + +Tell me which of the above you want next (upload support, auto-join helper, or presigned flow) and I'll proceed. \ No newline at end of file diff --git a/scripts/remote_storage_server.py b/scripts/remote_storage_server.py index 303db71..ddac81f 100644 --- a/scripts/remote_storage_server.py +++ b/scripts/remote_storage_server.py @@ -301,6 +301,74 @@ def create_app(): logger.error(f"Index error: {e}", exc_info=True) return jsonify({"error": f"Indexing failed: {str(e)}"}), 500 + @app.route("/files/upload", methods=["POST"]) + @require_auth() + @require_storage() + def upload_file(): + """Upload a file into storage (multipart/form-data). + + Accepts form fields: + - file: uploaded file (required) + - tag: repeated tag parameters or comma-separated string + - url: repeated url parameters or comma-separated string + """ + from API.folder import API_folder_store + from SYS.utils import sha256_file, sanitize_filename, ensure_directory, unique_path + + if 'file' not in request.files: + return jsonify({"error": "file required"}), 400 + file_storage = request.files.get('file') + if file_storage is None: + return jsonify({"error": "file required"}), 400 + + filename = sanitize_filename(file_storage.filename or "upload") + incoming_dir = STORAGE_PATH / "incoming" + ensure_directory(incoming_dir) + target_path = incoming_dir / filename + target_path = unique_path(target_path) + + try: + # Save uploaded file to storage + file_storage.save(str(target_path)) + + # Extract optional metadata + tags = [] + if 'tag' in request.form: + # Support repeated form fields or comma-separated list + tags = request.form.getlist('tag') or [] + if not tags and request.form.get('tag'): + tags = [t.strip() for t in str(request.form.get('tag') or "").split(",") if t.strip()] + + urls = [] + if 'url' in request.form: + urls = request.form.getlist('url') or [] + if not urls and request.form.get('url'): + urls = [u.strip() for u in str(request.form.get('url') or "").split(",") if u.strip()] + + with API_folder_store(STORAGE_PATH) as db: + db.get_or_create_file_entry(target_path) + + if tags: + db.add_tags(target_path, tags) + + if urls: + db.add_url(target_path, urls) + + file_hash = sha256_file(target_path) + + return ( + jsonify({ + "hash": file_hash, + "path": str(target_path), + "tags_added": len(tags), + "url_added": len(urls), + }), + 201, + ) + except Exception as e: + logger.error(f"Upload error: {e}", exc_info=True) + return jsonify({"error": f"Upload failed: {str(e)}"}), 500 + # ======================================================================== # TAG OPERATIONS # ======================================================================== diff --git a/scripts/zerotier_setup.py b/scripts/zerotier_setup.py new file mode 100644 index 0000000..ebe4924 --- /dev/null +++ b/scripts/zerotier_setup.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Simple ZeroTier helper for joining networks and discovering peers. + +Usage: + python scripts/zerotier_setup.py --join + python scripts/zerotier_setup.py --list + python scripts/zerotier_setup.py --discover + +This is a convenience tool to exercise the API/zerotier.py functionality while +prototyping and bringing up remote peers for store testing. +""" +from __future__ import annotations + +import argparse +import json +import sys +from typing import Any +from pathlib import Path + +from SYS.logger import log, debug + +try: + from API import zerotier +except Exception: + zerotier = None + + +def main(argv=None): + parser = argparse.ArgumentParser(description="ZeroTier helper for Medios-Macina") + parser.add_argument("--list", action="store_true", help="List local ZeroTier networks") + parser.add_argument("--join", type=str, help="Join a ZeroTier network by ID") + parser.add_argument("--leave", type=str, help="Leave a ZeroTier network by ID") + parser.add_argument("--discover", type=str, help="Discover services on a ZeroTier network ID") + parser.add_argument("--upload", type=str, help="Upload a file to a discovered 'remote' service on this ZeroTier network ID") + parser.add_argument("--file", type=str, help="Local file to upload (used with --upload)") + parser.add_argument("--tag", action="append", help="Tag to attach (repeatable)", default=[]) + parser.add_argument("--url", action="append", help="URL to associate (repeatable)", default=[]) + parser.add_argument("--api-key", type=str, help="API key to use for uploads (optional)") + parser.add_argument("--json", action="store_true", help="Output JSON when appropriate") + args = parser.parse_args(argv) + + if zerotier is None: + log("ZeroTier API module not available; ensure API/zerotier.py is importable and zerotier or zerotier-cli is installed") + return 1 + + if args.list: + nets = zerotier.list_networks() + if args.json: + print(json.dumps([n.__dict__ for n in nets], indent=2)) + else: + for n in nets: + print(f"{n.id}\t{name:=}{n.name}\t{n.status}\t{n.assigned_addresses}") + return 0 + + if args.join: + ok = zerotier.join_network(args.join) + print("Joined" if ok else "Failed to join") + return 0 if ok else 2 + + if args.leave: + ok = zerotier.leave_network(args.leave) + print("Left" if ok else "Failed to leave") + return 0 if ok else 2 + + if args.discover: + probes = zerotier.discover_services_on_network(args.discover) + if args.json: + print(json.dumps([p.__dict__ for p in probes], indent=2, default=str)) + else: + for p in probes: + print(f"{p.address}:{p.port}{p.path} -> status={p.status_code} hint={p.service_hint}") + return 0 + + if args.upload: + # Upload a file to the first discovered remote service on the network + if not args.file: + print("ERROR: --file is required for --upload") + return 2 + + probe = zerotier.find_peer_service(args.upload, service_hint="remote") + if not probe: + print("No remote service found on network") + return 2 + + base = f"http://{probe.address}:{probe.port}" + try: + import httpx + url = base.rstrip("/") + "/files/upload" + headers = {} + if args.api_key: + headers["X-API-Key"] = args.api_key + with open(args.file, "rb") as fh: + files = {"file": (Path(args.file).name, fh)} + data = [] + for t in (args.tag or []): + data.append(("tag", t)) + for u in (args.url or []): + data.append(("url", u)) + resp = httpx.post(url, files=files, data=data, headers=headers, timeout=30) + print(resp.status_code, resp.text) + return 0 if resp.status_code in (200, 201) else 2 + except Exception: + import requests + url = base.rstrip("/") + "/files/upload" + headers = {} + if args.api_key: + headers["X-API-Key"] = args.api_key + with open(args.file, "rb") as fh: + files = {"file": (Path(args.file).name, fh)} + data = [] + for t in (args.tag or []): + data.append(("tag", t)) + for u in (args.url or []): + data.append(("url", u)) + resp = requests.post(url, files=files, data=data, headers=headers, timeout=30) + print(resp.status_code, resp.text) + return 0 if resp.status_code in (200, 201) else 2 + + parser.print_help() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file