This commit is contained in:
2026-03-25 22:39:30 -07:00
parent c31402c8f1
commit 562acd809c
46 changed files with 2367 additions and 1868 deletions

View File

@@ -38,6 +38,9 @@ from API.httpx_shared import get_shared_httpx_client
# Default configuration # Default configuration
DEFAULT_TIMEOUT = 30.0 DEFAULT_TIMEOUT = 30.0
_CONTENT_DISPOSITION_FILENAME_RE = re.compile(
r'filename\*?=(?:"([^"]*)"|([^;\s]*))'
)
DEFAULT_RETRIES = 3 DEFAULT_RETRIES = 3
DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
@@ -661,7 +664,7 @@ def download_direct_file(
content_type = "" content_type = ""
if content_disposition: if content_disposition:
match = re.search(r'filename\*?=(?:"([^"]*)"|([^;\s]*))', content_disposition) match = _CONTENT_DISPOSITION_FILENAME_RE.search(content_disposition)
if match: if match:
extracted_name = match.group(1) or match.group(2) extracted_name = match.group(1) or match.group(2)
if extracted_name: if extracted_name:

107
CLI.py
View File

@@ -31,7 +31,6 @@ if not os.environ.get("MM_DEBUG"):
except Exception: except Exception:
pass pass
import httpx
import json import json
import shlex import shlex
import sys import sys
@@ -58,6 +57,16 @@ from SYS.rich_display import (
stderr_console, stderr_console,
stdout_console, stdout_console,
) )
from cmdnat._status_shared import (
add_startup_check as _shared_add_startup_check,
default_provider_ping_targets as _default_provider_ping_targets,
has_provider as _has_provider,
has_store_subtype as _has_store_subtype,
has_tool as _has_tool,
ping_first as _ping_first,
ping_url as _ping_url,
provider_display_name as _provider_display_name,
)
def _install_rich_traceback(*, show_locals: bool = False) -> None: def _install_rich_traceback(*, show_locals: bool = False) -> None:
@@ -1858,10 +1867,6 @@ Come to love it when others take what you share, as there is no greater joy
startup_table._interactive(True)._perseverance(True) startup_table._interactive(True)._perseverance(True)
startup_table.set_value_case("upper") startup_table.set_value_case("upper")
def _upper(value: Any) -> str:
text = "" if value is None else str(value)
return text.upper()
def _add_startup_check( def _add_startup_check(
status: str, status: str,
name: str, name: str,
@@ -1871,50 +1876,15 @@ Come to love it when others take what you share, as there is no greater joy
files: int | str | None = None, files: int | str | None = None,
detail: str = "", detail: str = "",
) -> None: ) -> None:
row = startup_table.add_row() _shared_add_startup_check(
row.add_column("STATUS", _upper(status)) startup_table,
row.add_column("NAME", _upper(name)) status,
row.add_column("PROVIDER", _upper(provider or "")) name,
row.add_column("STORE", _upper(store or "")) provider=provider,
row.add_column("FILES", "" if files is None else str(files)) store=store,
row.add_column("DETAIL", _upper(detail or "")) files=files,
detail=detail,
def _has_store_subtype(cfg: dict, subtype: str) -> bool: )
store_cfg = cfg.get("store")
if not isinstance(store_cfg, dict):
return False
bucket = store_cfg.get(subtype)
if not isinstance(bucket, dict):
return False
return any(isinstance(v, dict) and bool(v) for v in bucket.values())
def _has_provider(cfg: dict, name: str) -> bool:
provider_cfg = cfg.get("provider")
if not isinstance(provider_cfg, dict):
return False
block = provider_cfg.get(str(name).strip().lower())
return isinstance(block, dict) and bool(block)
def _has_tool(cfg: dict, name: str) -> bool:
tool_cfg = cfg.get("tool")
if not isinstance(tool_cfg, dict):
return False
block = tool_cfg.get(str(name).strip().lower())
return isinstance(block, dict) and bool(block)
def _ping_url(url: str, timeout: float = 3.0) -> tuple[bool, str]:
try:
from API.HTTP import HTTPClient
with HTTPClient(timeout=timeout, retries=1) as client:
resp = client.get(url, allow_redirects=True)
code = int(getattr(resp, "status_code", 0) or 0)
ok = 200 <= code < 500
return ok, f"{url} (HTTP {code})"
except httpx.TimeoutException:
return False, f"{url} (timeout)"
except Exception as exc:
return False, f"{url} ({type(exc).__name__})"
config = self._config_loader.load() config = self._config_loader.load()
debug_enabled = bool(config.get("debug", False)) debug_enabled = bool(config.get("debug", False))
@@ -2015,47 +1985,8 @@ Come to love it when others take what you share, as there is no greater joy
file_availability = list_file_providers(config) or {} file_availability = list_file_providers(config) or {}
meta_availability = list_metadata_providers(config) or {} meta_availability = list_metadata_providers(config) or {}
def _provider_display_name(key: str) -> str:
k = (key or "").strip()
low = k.lower()
if low == "openlibrary":
return "OpenLibrary"
if low == "alldebrid":
return "AllDebrid"
if low == "youtube":
return "YouTube"
return k[:1].upper() + k[1:] if k else "Provider"
already_checked = {"matrix"} already_checked = {"matrix"}
def _default_provider_ping_targets(provider_key: str) -> list[str]:
prov = (provider_key or "").strip().lower()
if prov == "openlibrary":
return ["https://openlibrary.org"]
if prov == "youtube":
return ["https://www.youtube.com"]
if prov == "bandcamp":
return ["https://bandcamp.com"]
if prov == "libgen":
from Provider.libgen import MIRRORS
mirrors = [
str(x).rstrip("/") for x in (MIRRORS or [])
if str(x).strip()
]
return [m + "/json.php" for m in mirrors]
return []
def _ping_first(urls: list[str]) -> tuple[bool, str]:
for u in urls:
ok, detail = _ping_url(u)
if ok:
return True, detail
if urls:
ok, detail = _ping_url(urls[0])
return ok, detail
return False, "No ping target"
for provider_name in provider_cfg.keys(): for provider_name in provider_cfg.keys():
prov = str(provider_name or "").strip().lower() prov = str(provider_name or "").strip().lower()
if not prov or prov in already_checked: if not prov or prov in already_checked:

View File

@@ -15,6 +15,7 @@ from API.HTTP import HTTPClient, _download_direct_file
from API.alldebrid import AllDebridClient, parse_magnet_or_hash, is_torrent_file from API.alldebrid import AllDebridClient, parse_magnet_or_hash, is_torrent_file
from ProviderCore.base import Provider, SearchResult from ProviderCore.base import Provider, SearchResult
from SYS.provider_helpers import TableProviderMixin from SYS.provider_helpers import TableProviderMixin
from SYS.item_accessors import get_field as _extract_value
from SYS.utils import sanitize_filename from SYS.utils import sanitize_filename
from SYS.logger import log, debug from SYS.logger import log, debug
from SYS.models import DownloadError, PipeObject from SYS.models import DownloadError, PipeObject
@@ -339,25 +340,6 @@ def _looks_like_torrent_source(candidate: str) -> bool:
return False return False
def _extract_value(source: Any, field: str) -> Any:
if source is None:
return None
if isinstance(source, dict):
if field in source:
return source.get(field)
else:
try:
value = getattr(source, field)
except Exception:
value = None
if value is not None:
return value
extra = getattr(source, "extra", None)
if isinstance(extra, dict) and field in extra:
return extra.get(field)
return None
def _dispatch_alldebrid_magnet_search( def _dispatch_alldebrid_magnet_search(
magnet_id: int, magnet_id: int,
config: Dict[str, Any], config: Dict[str, Any],

View File

@@ -11,7 +11,7 @@ import sys
import tempfile import tempfile
from copy import deepcopy from copy import deepcopy
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
from SYS.logger import log from SYS.logger import log
import logging import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -29,6 +29,7 @@ _CONFIG_CACHE: Dict[str, Any] = {}
_LAST_SAVED_CONFIG: Dict[str, Any] = {} _LAST_SAVED_CONFIG: Dict[str, Any] = {}
_CONFIG_SAVE_MAX_RETRIES = 5 _CONFIG_SAVE_MAX_RETRIES = 5
_CONFIG_SAVE_RETRY_DELAY = 0.15 _CONFIG_SAVE_RETRY_DELAY = 0.15
_CONFIG_MISSING = object()
class ConfigSaveConflict(Exception): class ConfigSaveConflict(Exception):
@@ -61,6 +62,94 @@ def clear_config_cache() -> None:
_LAST_SAVED_CONFIG = {} _LAST_SAVED_CONFIG = {}
def get_nested_config_value(config: Dict[str, Any], *path: str) -> Any:
cur: Any = config
for key in path:
if not isinstance(cur, dict):
return None
cur = cur.get(key)
return cur
def coerce_config_value(
value: Any,
existing_value: Any = _CONFIG_MISSING,
*,
on_error: Optional[Callable[[str], None]] = None,
) -> Any:
if not isinstance(value, str):
return value
text = value.strip()
lowered = text.lower()
if existing_value is _CONFIG_MISSING:
if lowered in {"true", "false"}:
return lowered == "true"
if text.isdigit():
return int(text)
return value
if isinstance(existing_value, bool):
if lowered in {"true", "yes", "1", "on"}:
return True
if lowered in {"false", "no", "0", "off"}:
return False
if on_error is not None:
on_error(f"Warning: Could not convert '{value}' to boolean. Using string.")
return value
if isinstance(existing_value, int) and not isinstance(existing_value, bool):
try:
return int(text)
except ValueError:
if on_error is not None:
on_error(f"Warning: Could not convert '{value}' to int. Using string.")
return value
if isinstance(existing_value, float):
try:
return float(text)
except ValueError:
if on_error is not None:
on_error(f"Warning: Could not convert '{value}' to float. Using string.")
return value
return value
def set_nested_config_value(
config: Dict[str, Any],
key_path: str | Sequence[str],
value: Any,
*,
on_error: Optional[Callable[[str], None]] = None,
) -> bool:
if not isinstance(config, dict):
return False
if isinstance(key_path, str):
keys = [part for part in key_path.split(".") if part]
else:
keys = [str(part) for part in (key_path or []) if str(part)]
if not keys:
return False
current = config
for key in keys[:-1]:
next_value = current.get(key)
if not isinstance(next_value, dict):
next_value = {}
current[key] = next_value
current = next_value
last_key = keys[-1]
existing_value = current[last_key] if last_key in current else _CONFIG_MISSING
current[last_key] = coerce_config_value(value, existing_value, on_error=on_error)
return True
def get_hydrus_instance( def get_hydrus_instance(
config: Dict[str, Any], instance_name: str = "home" config: Dict[str, Any], instance_name: str = "home"
) -> Optional[Dict[str, Any]]: ) -> Optional[Dict[str, Any]]:

104
SYS/detail_view_helpers.py Normal file
View File

@@ -0,0 +1,104 @@
from __future__ import annotations
from typing import Any, Iterable, Optional, Sequence
def _labelize_key(key: str) -> str:
return str(key or "").replace("_", " ").title()
def _normalize_tags_value(tags: Any) -> Optional[str]:
if tags is None:
return None
if isinstance(tags, str):
text = tags.strip()
return text or None
if isinstance(tags, Sequence):
seen: list[str] = []
for tag in tags:
text = str(tag or "").strip()
if text and text not in seen:
seen.append(text)
return ", ".join(seen) if seen else None
text = str(tags).strip()
return text or None
def prepare_detail_metadata(
subject: Any,
*,
include_subject_fields: bool = False,
title: Optional[str] = None,
hash_value: Optional[str] = None,
store: Optional[str] = None,
path: Optional[str] = None,
tags: Any = None,
prefer_existing_tags: bool = True,
extra_fields: Optional[dict[str, Any]] = None,
) -> dict[str, Any]:
from SYS.result_table import extract_item_metadata
metadata = extract_item_metadata(subject) or {}
if include_subject_fields and isinstance(subject, dict):
for key, value in subject.items():
if str(key).startswith("_") or key in {"selection_action", "selection_args"}:
continue
label = _labelize_key(str(key))
if label not in metadata and value is not None:
metadata[label] = value
if title:
metadata["Title"] = title
if hash_value:
metadata["Hash"] = hash_value
if store:
metadata["Store"] = store
if path:
metadata["Path"] = path
tags_text = _normalize_tags_value(tags)
if tags_text and (not prefer_existing_tags or not metadata.get("Tags")):
metadata["Tags"] = tags_text
for key, value in (extra_fields or {}).items():
if value is not None:
metadata[str(key)] = value
return metadata
def create_detail_view(
title: str,
metadata: dict[str, Any],
*,
table_name: Optional[str] = None,
source_command: Optional[tuple[str, Sequence[str]]] = None,
init_command: Optional[tuple[str, Sequence[str]]] = None,
max_columns: Optional[int] = None,
exclude_tags: bool = False,
value_case: Optional[str] = "preserve",
perseverance: bool = True,
) -> Any:
from SYS.result_table import ItemDetailView
kwargs: dict[str, Any] = {"item_metadata": metadata}
if max_columns is not None:
kwargs["max_columns"] = max_columns
if exclude_tags:
kwargs["exclude_tags"] = True
table = ItemDetailView(title, **kwargs)
if table_name:
table = table.set_table(table_name)
if value_case:
table = table.set_value_case(value_case)
if perseverance:
table = table._perseverance(True)
if source_command:
name, args = source_command
table.set_source_command(name, list(args))
if init_command:
name, args = init_command
table = table.init_command(name, list(args))
return table

132
SYS/item_accessors.py Normal file
View File

@@ -0,0 +1,132 @@
from __future__ import annotations
import re
from typing import Any, Iterable, Optional
_SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
def get_field(obj: Any, field: str, default: Optional[Any] = None) -> Any:
if isinstance(obj, list):
if not obj:
return default
obj = obj[0]
if isinstance(obj, dict):
return obj.get(field, default)
value = getattr(obj, field, None)
if value is not None:
return value
extra_value = getattr(obj, "extra", None)
if isinstance(extra_value, dict):
return extra_value.get(field, default)
return default
def first_field(obj: Any, fields: Iterable[str], default: Optional[Any] = None) -> Any:
for field in fields:
value = get_field(obj, str(field), None)
if value is not None:
return value
return default
def get_text_field(obj: Any, *fields: str, default: str = "") -> str:
value = first_field(obj, fields, default=None)
if value is None:
return default
text = str(value).strip()
return text if text else default
def get_column_text(obj: Any, *labels: str) -> Optional[str]:
columns = get_field(obj, "columns")
if not isinstance(columns, list):
return None
wanted = {str(label or "").strip().lower() for label in labels if str(label or "").strip()}
if not wanted:
return None
for pair in columns:
try:
if not isinstance(pair, (list, tuple)) or len(pair) != 2:
continue
key, value = pair
if str(key or "").strip().lower() not in wanted:
continue
text = str(value or "").strip()
if text:
return text
except Exception:
continue
return None
def get_int_field(obj: Any, *fields: str) -> Optional[int]:
value = first_field(obj, fields, default=None)
if value is None:
return None
if isinstance(value, (int, float)):
return int(value)
try:
return int(value)
except Exception:
return None
def get_extension_field(obj: Any, *fields: str) -> str:
text = get_text_field(obj, *(fields or ("ext", "extension")), default="")
return text.lstrip(".") if text else ""
def get_result_title(obj: Any, *fields: str) -> Optional[str]:
text = get_text_field(obj, *(fields or ("title", "name", "filename")), default="")
if text:
return text
return get_column_text(obj, "title", "name")
def extract_item_tags(obj: Any) -> list[str]:
return get_string_list(obj, "tag")
def get_string_list(obj: Any, field: str) -> list[str]:
value = get_field(obj, field)
if isinstance(value, list):
return [str(item).strip() for item in value if item is not None and str(item).strip()]
if isinstance(value, str):
text = value.strip()
return [text] if text else []
return []
def set_field(obj: Any, field: str, value: Any) -> bool:
if isinstance(obj, dict):
obj[field] = value
return True
try:
setattr(obj, field, value)
return True
except Exception:
return False
def get_sha256_hex(obj: Any, *fields: str) -> Optional[str]:
value = get_text_field(obj, *(fields or ("hash",)))
if value and _SHA256_RE.fullmatch(value):
return value.lower()
return None
def get_store_name(obj: Any, *fields: str) -> Optional[str]:
value = get_text_field(obj, *(fields or ("store",)))
return value or None
def get_http_url(obj: Any, *fields: str) -> Optional[str]:
value = get_text_field(obj, *(fields or ("url", "target")))
if value.lower().startswith(("http://", "https://")):
return value
return None

158
SYS/payload_builders.py Normal file
View File

@@ -0,0 +1,158 @@
from __future__ import annotations
from pathlib import Path
from typing import Any, Dict, Optional, Sequence
from urllib.parse import unquote, urlparse
def normalize_file_extension(ext_value: Any) -> str:
ext = str(ext_value or "").strip().lstrip(".")
for sep in (" ", "|", "(", "[", "{", ",", ";"):
if sep in ext:
ext = ext.split(sep, 1)[0]
break
if "." in ext:
ext = ext.split(".")[-1]
ext = "".join(ch for ch in ext if ch.isalnum())
return ext[:5]
def extract_title_tag_value(tags: Sequence[str]) -> Optional[str]:
for tag in tags:
text = str(tag or "").strip()
if text.lower().startswith("title:"):
value = text.split(":", 1)[1].strip()
if value:
return value
return None
def _derive_title(
title: Optional[str],
fallback_title: Optional[str],
path: Optional[str],
url: Any,
hash_value: Optional[str],
) -> str:
for candidate in (title, fallback_title):
text = str(candidate or "").strip()
if text:
return text
path_text = str(path or "").strip()
if path_text:
try:
return Path(path_text).stem or path_text
except Exception:
return path_text
if isinstance(url, str):
try:
parsed = urlparse(url)
name = Path(unquote(parsed.path)).stem
if name:
return name
except Exception:
pass
text = url.strip()
if text:
return text
if isinstance(url, list):
for candidate in url:
text = str(candidate or "").strip()
if text:
return text
return str(hash_value or "").strip()
def build_file_result_payload(
*,
title: Optional[str] = None,
fallback_title: Optional[str] = None,
path: Optional[str] = None,
url: Any = None,
hash_value: Optional[str] = None,
store: Optional[str] = None,
tag: Optional[Sequence[str]] = None,
ext: Any = None,
size_bytes: Optional[int] = None,
columns: Optional[Sequence[tuple[str, Any]]] = None,
source: Optional[str] = None,
table: Optional[str] = None,
detail: Optional[str] = None,
**extra: Any,
) -> Dict[str, Any]:
resolved_title = _derive_title(title, fallback_title, path, url, hash_value)
resolved_path = str(path).strip() if path is not None and str(path).strip() else None
resolved_store = str(store).strip() if store is not None and str(store).strip() else None
resolved_ext = normalize_file_extension(ext)
if not resolved_ext:
for candidate in (resolved_path, resolved_title):
text = str(candidate or "").strip()
if not text:
continue
try:
resolved_ext = normalize_file_extension(Path(text).suffix)
except Exception:
resolved_ext = ""
if resolved_ext:
break
payload: Dict[str, Any] = {"title": resolved_title}
if resolved_path is not None:
payload["path"] = resolved_path
if hash_value:
payload["hash"] = str(hash_value)
if url not in (None, "", []):
payload["url"] = url
if resolved_store is not None:
payload["store"] = resolved_store
if tag is not None:
payload["tag"] = list(tag)
if resolved_ext:
payload["ext"] = resolved_ext
if size_bytes is not None:
payload["size_bytes"] = size_bytes
if columns is not None:
payload["columns"] = list(columns)
if source:
payload["source"] = source
if table:
payload["table"] = table
if detail is not None:
payload["detail"] = str(detail)
payload.update(extra)
return payload
def build_table_result_payload(
*,
columns: Sequence[tuple[str, Any]],
title: Optional[str] = None,
table: Optional[str] = None,
detail: Optional[str] = None,
selection_args: Optional[Sequence[Any]] = None,
selection_action: Optional[Sequence[Any]] = None,
**extra: Any,
) -> Dict[str, Any]:
payload: Dict[str, Any] = {
"columns": [(str(label), value) for label, value in columns],
}
if title is not None:
payload["title"] = str(title)
if table:
payload["table"] = table
if detail is not None:
payload["detail"] = str(detail)
if selection_args:
payload["_selection_args"] = [str(arg) for arg in selection_args if arg is not None]
if selection_action:
payload["_selection_action"] = [str(arg) for arg in selection_action if arg is not None]
payload.update(extra)
return payload

60
SYS/result_publication.py Normal file
View File

@@ -0,0 +1,60 @@
from __future__ import annotations
from typing import Any, Iterable, Optional
def resolve_publication_subject(
items: Iterable[Any] | None,
subject: Any = None,
) -> Any:
if subject is not None:
return subject
resolved_items = list(items or [])
if not resolved_items:
return None
if len(resolved_items) == 1:
return resolved_items[0]
return resolved_items
def publish_result_table(
pipeline_context: Any,
result_table: Any,
items: Iterable[Any] | None = None,
*,
subject: Any = None,
overlay: bool = False,
) -> None:
resolved_items = list(items or [])
resolved_subject = resolve_publication_subject(resolved_items, subject)
if overlay:
pipeline_context.set_last_result_table_overlay(
result_table,
resolved_items,
subject=resolved_subject,
)
return
pipeline_context.set_last_result_table(
result_table,
resolved_items,
subject=resolved_subject,
)
def overlay_existing_result_table(
pipeline_context: Any,
*,
subject: Any = None,
) -> bool:
table = pipeline_context.get_last_result_table()
items = list(pipeline_context.get_last_result_items() or [])
if table is None or not items:
return False
publish_result_table(
pipeline_context,
table,
items,
subject=subject,
overlay=True,
)
return True

View File

@@ -0,0 +1,10 @@
from __future__ import annotations
from typing import Any, Iterable
def add_row_columns(table: Any, columns: Iterable[tuple[str, Any]]) -> Any:
row = table.add_row()
for label, value in columns:
row.add_column(str(label), "" if value is None else str(value))
return row

157
SYS/selection_builder.py Normal file
View File

@@ -0,0 +1,157 @@
from __future__ import annotations
import re
from typing import Any, Iterable, List, Optional, Sequence, Tuple
_SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
def looks_like_url(value: Any, *, extra_prefixes: Iterable[str] = ()) -> bool:
try:
text = str(value or "").strip().lower()
except Exception:
return False
if not text:
return False
prefixes = ("http://", "https://", "magnet:", "torrent:") + tuple(
str(prefix).strip().lower() for prefix in extra_prefixes if str(prefix).strip()
)
return text.startswith(prefixes)
def normalize_selection_args(selection_args: Any) -> Optional[List[str]]:
if isinstance(selection_args, (list, tuple)):
return [str(arg) for arg in selection_args if arg is not None]
if selection_args is not None:
return [str(selection_args)]
return None
def normalize_hash_for_selection(value: Any) -> str:
text = str(value or "").strip()
if _SHA256_RE.fullmatch(text):
return text.lower()
return text
def build_hash_store_selection(
hash_value: Any,
store_value: Any,
*,
action_name: str = "get-metadata",
) -> Tuple[Optional[List[str]], Optional[List[str]]]:
hash_text = normalize_hash_for_selection(hash_value)
store_text = str(store_value or "").strip()
if not hash_text or not store_text:
return None, None
args = ["-query", f"hash:{hash_text}", "-store", store_text]
return args, [action_name] + list(args)
def build_default_selection(
*,
path_value: Any,
hash_value: Any = None,
store_value: Any = None,
) -> Tuple[Optional[List[str]], Optional[List[str]]]:
path_text = str(path_value or "").strip()
hash_args, hash_action = build_hash_store_selection(hash_value, store_value)
if path_text:
if looks_like_url(path_text):
if hash_args and "/view_file" in path_text:
return hash_args, hash_action
args = ["-url", path_text]
return args, ["download-file", "-url", path_text]
if hash_args:
return hash_args, hash_action
try:
from SYS.utils import expand_path
resolved_path = str(expand_path(path_text))
except Exception:
resolved_path = path_text
args = ["-path", resolved_path]
return args, ["get-file", "-path", resolved_path]
return hash_args, hash_action
def extract_selection_fields(
item: Any,
*,
extra_url_prefixes: Iterable[str] = (),
) -> Tuple[Optional[List[str]], Optional[List[str]], Optional[str]]:
selection_args: Any = None
selection_action: Any = None
item_url: Any = None
if isinstance(item, dict):
selection_args = item.get("_selection_args") or item.get("selection_args")
selection_action = item.get("_selection_action") or item.get("selection_action")
item_url = item.get("url") or item.get("path") or item.get("target")
nested_values = [item.get("metadata"), item.get("full_metadata"), item.get("extra")]
else:
item_url = getattr(item, "url", None) or getattr(item, "path", None) or getattr(item, "target", None)
nested_values = [
getattr(item, "metadata", None),
getattr(item, "full_metadata", None),
getattr(item, "extra", None),
]
for nested in nested_values:
if not isinstance(nested, dict):
continue
selection_args = selection_args or nested.get("_selection_args") or nested.get("selection_args")
selection_action = selection_action or nested.get("_selection_action") or nested.get("selection_action")
item_url = item_url or nested.get("url") or nested.get("source_url") or nested.get("target")
normalized_args = normalize_selection_args(selection_args)
normalized_action = normalize_selection_args(selection_action)
if item_url and not looks_like_url(item_url, extra_prefixes=extra_url_prefixes):
item_url = None
return normalized_args, normalized_action, str(item_url) if item_url else None
def selection_args_have_url(
args_list: Sequence[str],
*,
extra_url_prefixes: Iterable[str] = (),
) -> bool:
for idx, arg in enumerate(args_list):
low = str(arg or "").strip().lower()
if low in {"-url", "--url"} and idx + 1 < len(args_list):
return True
if looks_like_url(arg, extra_prefixes=extra_url_prefixes):
return True
return False
def extract_urls_from_selection_args(
args_list: Sequence[str],
*,
extra_url_prefixes: Iterable[str] = (),
) -> List[str]:
urls: List[str] = []
idx = 0
while idx < len(args_list):
token = str(args_list[idx] or "")
low = token.strip().lower()
if low in {"-url", "--url"} and idx + 1 < len(args_list):
candidate = str(args_list[idx + 1] or "").strip()
if looks_like_url(candidate, extra_prefixes=extra_url_prefixes) and candidate not in urls:
urls.append(candidate)
idx += 2
continue
if looks_like_url(token, extra_prefixes=extra_url_prefixes):
candidate = token.strip()
if candidate not in urls:
urls.append(candidate)
idx += 1
return urls

View File

@@ -10,7 +10,16 @@ from textual.screen import ModalScreen
from textual.widgets import Static, Button, Input, Label, ListView, ListItem, Rule, Select, Checkbox from textual.widgets import Static, Button, Input, Label, ListView, ListItem, Rule, Select, Checkbox
from pathlib import Path from pathlib import Path
from SYS.config import load_config, save_config, save_config_and_verify, reload_config, global_config, count_changed_entries, ConfigSaveConflict from SYS.config import (
load_config,
save_config,
save_config_and_verify,
reload_config,
global_config,
count_changed_entries,
ConfigSaveConflict,
coerce_config_value,
)
from SYS.database import db from SYS.database import db
from SYS.logger import log, debug from SYS.logger import log, debug
from Store.registry import _discover_store_classes, _required_keys_for from Store.registry import _discover_store_classes, _required_keys_for
@@ -1142,15 +1151,7 @@ class ConfigModal(ModalScreen):
return return
# Try to preserve boolean/integer types # Try to preserve boolean/integer types
processed_value = raw_value processed_value = coerce_config_value(raw_value, existing_value)
if isinstance(raw_value, str):
low = raw_value.lower()
if low == "true":
processed_value = True
elif low == "false":
processed_value = False
elif raw_value.isdigit():
processed_value = int(raw_value)
if widget_id.startswith("global-"): if widget_id.startswith("global-"):
self.config_data[key] = processed_value self.config_data[key] = processed_value

View File

@@ -11,14 +11,18 @@ import sys
import tempfile import tempfile
import time import time
from collections.abc import Iterable as IterableABC from collections.abc import Iterable as IterableABC
from functools import lru_cache
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
from SYS.logger import log, debug from SYS.logger import log, debug
from pathlib import Path from pathlib import Path
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Set from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple
from dataclasses import dataclass, field from dataclasses import dataclass, field
from SYS import models from SYS import models
from SYS import pipeline as pipeline_context from SYS import pipeline as pipeline_context
from SYS.item_accessors import get_field as _item_accessor_get_field
from SYS.payload_builders import build_file_result_payload, build_table_result_payload
from SYS.result_publication import publish_result_table
from SYS.result_table import Table from SYS.result_table import Table
from SYS.rich_display import stderr_console as get_stderr_console from SYS.rich_display import stderr_console as get_stderr_console
from rich.prompt import Confirm from rich.prompt import Confirm
@@ -944,6 +948,18 @@ def build_pipeline_preview(raw_urls: Sequence[str], piped_items: Sequence[Any])
return preview return preview
@lru_cache(maxsize=4096)
def _normalize_hash_cached(hash_hex: str) -> Optional[str]:
text = hash_hex.strip().lower()
if not text:
return None
if len(text) != 64:
return None
if not all(ch in "0123456789abcdef" for ch in text):
return None
return text
def normalize_hash(hash_hex: Optional[str]) -> Optional[str]: def normalize_hash(hash_hex: Optional[str]) -> Optional[str]:
"""Normalize a hash string to lowercase, or return None if invalid. """Normalize a hash string to lowercase, or return None if invalid.
@@ -955,14 +971,7 @@ def normalize_hash(hash_hex: Optional[str]) -> Optional[str]:
""" """
if not isinstance(hash_hex, str): if not isinstance(hash_hex, str):
return None return None
text = hash_hex.strip().lower() return _normalize_hash_cached(hash_hex)
if not text:
return None
if len(text) != 64:
return None
if not all(ch in "0123456789abcdef" for ch in text):
return None
return text
def resolve_hash_for_cmdlet( def resolve_hash_for_cmdlet(
@@ -1007,6 +1016,270 @@ def resolve_hash_for_cmdlet(
return None return None
def resolve_item_store_hash(
item: Any,
*,
override_store: Optional[str] = None,
override_hash: Optional[str] = None,
hash_field: str = "hash",
store_field: str = "store",
path_fields: Sequence[str] = ("path", "target"),
) -> Tuple[str, Optional[str]]:
"""Resolve store name and normalized hash from a result item."""
store_name = str(override_store or get_field(item, store_field) or "").strip()
raw_hash = get_field(item, hash_field)
raw_path = None
for field_name in path_fields:
candidate = get_field(item, field_name)
if candidate:
raw_path = candidate
break
resolved_hash = resolve_hash_for_cmdlet(
str(raw_hash) if raw_hash else None,
str(raw_path) if raw_path else None,
str(override_hash) if override_hash else None,
)
return store_name, resolved_hash
def get_store_backend(
config: Optional[Dict[str, Any]],
store_name: Optional[str],
*,
store_registry: Any = None,
suppress_debug: bool = False,
) -> Tuple[Optional[Any], Any, Optional[Exception]]:
"""Resolve a store backend, optionally reusing an existing registry.
Returns a tuple of ``(backend, store_registry, exc)`` so callers can keep
their command-specific error messages while avoiding repeated registry setup
and ``store[name]`` boilerplate.
"""
registry = store_registry
if registry is None:
try:
from Store import Store
registry = Store(config or {}, suppress_debug=suppress_debug)
except Exception as exc:
return None, None, exc
backend_name = str(store_name or "").strip()
if not backend_name:
return None, registry, KeyError("Missing store name")
try:
return registry[backend_name], registry, None
except Exception as exc:
return None, registry, exc
def get_preferred_store_backend(
config: Optional[Dict[str, Any]],
store_name: Optional[str],
*,
store_registry: Any = None,
suppress_debug: bool = True,
) -> Tuple[Optional[Any], Any, Optional[Exception]]:
"""Prefer a targeted backend instance before falling back to registry lookup."""
direct_exc: Optional[Exception] = None
try:
from Store.registry import get_backend_instance
backend = get_backend_instance(
config or {},
str(store_name or ""),
suppress_debug=suppress_debug,
)
if backend is not None:
return backend, store_registry, None
except Exception as exc:
direct_exc = exc
backend, registry, lookup_exc = get_store_backend(
config,
store_name,
store_registry=store_registry,
suppress_debug=suppress_debug,
)
if backend is not None:
return backend, registry, None
return None, registry, direct_exc or lookup_exc
def coalesce_hash_value_pairs(
pairs: Sequence[Tuple[str, Sequence[str]]],
) -> List[Tuple[str, List[str]]]:
"""Merge duplicate hash/value pairs while preserving first-seen value order."""
merged: Dict[str, List[str]] = {}
for hash_value, values in pairs:
normalized_hash = str(hash_value or "").strip()
if not normalized_hash:
continue
bucket = merged.setdefault(normalized_hash, [])
seen = set(bucket)
for value in values or []:
text = str(value or "").strip()
if not text or text in seen:
continue
seen.add(text)
bucket.append(text)
return [(hash_value, items) for hash_value, items in merged.items() if items]
def run_store_hash_value_batches(
config: Optional[Dict[str, Any]],
batch: Dict[str, List[Tuple[str, Sequence[str]]]],
*,
bulk_method_name: str,
single_method_name: str,
store_registry: Any = None,
suppress_debug: bool = False,
pass_config_to_bulk: bool = True,
pass_config_to_single: bool = True,
) -> Tuple[Any, List[Tuple[str, int, int]]]:
"""Dispatch grouped hash/value batches across stores.
Returns ``(store_registry, stats)`` where ``stats`` contains
``(store_name, item_count, value_count)`` for each dispatched store.
Missing stores are skipped so callers can preserve existing warning behavior.
"""
registry = store_registry
stats: List[Tuple[str, int, int]] = []
for store_name, pairs in batch.items():
backend, registry, _exc = get_store_backend(
config,
store_name,
store_registry=registry,
suppress_debug=suppress_debug,
)
if backend is None:
continue
bulk_pairs = coalesce_hash_value_pairs(pairs)
if not bulk_pairs:
continue
bulk_fn = getattr(backend, bulk_method_name, None)
if callable(bulk_fn):
if pass_config_to_bulk:
bulk_fn(bulk_pairs, config=config)
else:
bulk_fn(bulk_pairs)
else:
single_fn = getattr(backend, single_method_name)
for hash_value, values in bulk_pairs:
if pass_config_to_single:
single_fn(hash_value, values, config=config)
else:
single_fn(hash_value, values)
stats.append(
(
store_name,
len(bulk_pairs),
sum(len(values or []) for _hash_value, values in bulk_pairs),
)
)
return registry, stats
def run_store_note_batches(
config: Optional[Dict[str, Any]],
batch: Dict[str, List[Tuple[str, str, str]]],
*,
store_registry: Any = None,
suppress_debug: bool = False,
on_store_error: Optional[Callable[[str, Exception], None]] = None,
on_unsupported_store: Optional[Callable[[str], None]] = None,
on_item_error: Optional[Callable[[str, str, str, Exception], None]] = None,
) -> Tuple[Any, int]:
"""Dispatch grouped note writes across stores while preserving item-level errors."""
registry = store_registry
success_count = 0
for store_name, items in batch.items():
backend, registry, exc = get_store_backend(
config,
store_name,
store_registry=registry,
suppress_debug=suppress_debug,
)
if backend is None:
if on_store_error is not None and exc is not None:
on_store_error(store_name, exc)
continue
if not hasattr(backend, "set_note"):
if on_unsupported_store is not None:
on_unsupported_store(store_name)
continue
for hash_value, note_name, note_text in items:
try:
if backend.set_note(hash_value, note_name, note_text, config=config):
success_count += 1
except Exception as item_exc:
if on_item_error is not None:
on_item_error(store_name, hash_value, note_name, item_exc)
return registry, success_count
def collect_store_hash_value_batch(
items: Sequence[Any],
*,
store_registry: Any,
value_resolver: Callable[[Any], Optional[Sequence[str]]],
override_hash: Optional[str] = None,
override_store: Optional[str] = None,
on_warning: Optional[Callable[[str], None]] = None,
) -> Tuple[Dict[str, List[Tuple[str, List[str]]]], List[Any]]:
"""Collect validated store/hash/value batches while preserving passthrough items."""
batch: Dict[str, List[Tuple[str, List[str]]]] = {}
pass_through: List[Any] = []
for item in items:
pass_through.append(item)
raw_hash = override_hash or get_field(item, "hash")
raw_store = override_store or get_field(item, "store")
if not raw_hash or not raw_store:
if on_warning is not None:
on_warning("Item missing hash/store; skipping")
continue
normalized = normalize_hash(raw_hash)
if not normalized:
if on_warning is not None:
on_warning("Item has invalid hash; skipping")
continue
store_text = str(raw_store).strip()
if not store_text:
if on_warning is not None:
on_warning("Item has empty store; skipping")
continue
try:
is_available = bool(store_registry.is_available(store_text))
except Exception:
is_available = False
if not is_available:
if on_warning is not None:
on_warning(f"Store '{store_text}' not configured; skipping")
continue
values = [str(value).strip() for value in (value_resolver(item) or []) if str(value).strip()]
if not values:
continue
batch.setdefault(store_text, []).append((normalized, values))
return batch, pass_through
def parse_hash_query(query: Optional[str]) -> List[str]: def parse_hash_query(query: Optional[str]) -> List[str]:
"""Parse a unified query string for `hash:` into normalized SHA256 hashes. """Parse a unified query string for `hash:` into normalized SHA256 hashes.
@@ -1054,6 +1327,36 @@ def parse_single_hash_query(query: Optional[str]) -> Optional[str]:
return hashes[0] return hashes[0]
def require_hash_query(
query: Optional[str],
error_message: str,
*,
log_file: Any = None,
) -> Tuple[List[str], bool]:
"""Parse a multi-hash query and log a caller-provided error on invalid input."""
hashes = parse_hash_query(query)
if query and not hashes:
kwargs = {"file": log_file} if log_file is not None else {}
log(error_message, **kwargs)
return [], False
return hashes, True
def require_single_hash_query(
query: Optional[str],
error_message: str,
*,
log_file: Any = None,
) -> Tuple[Optional[str], bool]:
"""Parse a single-hash query and log a caller-provided error on invalid input."""
query_hash = parse_single_hash_query(query)
if query and not query_hash:
kwargs = {"file": log_file} if log_file is not None else {}
log(error_message, **kwargs)
return None, False
return query_hash, True
def get_hash_for_operation( def get_hash_for_operation(
override_hash: Optional[str], override_hash: Optional[str],
result: Any, result: Any,
@@ -1180,26 +1483,7 @@ def get_field(obj: Any, field: str, default: Optional[Any] = None) -> Any:
get_field(result, "hash") # From dict or object get_field(result, "hash") # From dict or object
get_field(result, "table", "unknown") # With default get_field(result, "table", "unknown") # With default
""" """
# Handle lists by accessing the first element return _item_accessor_get_field(obj, field, default)
if isinstance(obj, list):
if not obj:
return default
obj = obj[0]
if isinstance(obj, dict):
return obj.get(field, default)
else:
# Try direct attribute access first
value = getattr(obj, field, None)
if value is not None:
return value
# For PipeObjects, also check the extra field
extra_val = getattr(obj, "extra", None)
if isinstance(extra_val, dict):
return extra_val.get(field, default)
return default
def should_show_help(args: Sequence[str]) -> bool: def should_show_help(args: Sequence[str]) -> bool:
@@ -1636,33 +1920,22 @@ def create_pipe_object_result(
Returns: Returns:
Dict with all PipeObject fields for emission Dict with all PipeObject fields for emission
""" """
result: Dict[str, Any] = { result = build_file_result_payload(
"source": source, title=title,
"id": identifier, path=file_path,
"path": file_path, hash_value=hash_value,
"action": f"cmdlet:{cmdlet_name}", # Format: cmdlet:cmdlet_name store=source,
} tag=tag,
source=source,
id=identifier,
action=f"cmdlet:{cmdlet_name}",
**extra,
)
if title:
result["title"] = title
if hash_value:
result["hash"] = hash_value
if is_temp: if is_temp:
result["is_temp"] = True result["is_temp"] = True
if parent_hash: if parent_hash:
result["parent_hash"] = parent_hash result["parent_hash"] = parent_hash
if tag:
result["tag"] = tag
# Canonical store field: use source for compatibility
try:
if source:
result["store"] = source
except Exception:
pass
# Add any extra fields
result.update(extra)
return result return result
@@ -2153,6 +2426,32 @@ def normalize_result_input(result: Any) -> List[Dict[str, Any]]:
return [] return []
def normalize_result_items(
result: Any,
*,
include_falsey_single: bool = False,
) -> List[Any]:
"""Normalize piped input to a raw item list without converting item types."""
if isinstance(result, list):
return list(result)
if result is None:
return []
if include_falsey_single or result:
return [result]
return []
def value_has_content(value: Any) -> bool:
"""Return True when a value should be treated as present for payload building."""
if value is None:
return False
if isinstance(value, str):
return bool(value.strip())
if isinstance(value, (list, tuple, set)):
return len(value) > 0
return True
def filter_results_by_temp(results: List[Any], include_temp: bool = False) -> List[Any]: def filter_results_by_temp(results: List[Any], include_temp: bool = False) -> List[Any]:
"""Filter results by temporary status. """Filter results by temporary status.
@@ -2380,6 +2679,46 @@ def extract_url_from_result(result: Any) -> list[str]:
return normalize_urls(url) return normalize_urls(url)
def merge_urls(existing: Any, incoming: Sequence[Any]) -> list[str]:
"""Merge URL values into a normalized, de-duplicated list."""
from SYS.metadata import normalize_urls
merged: list[str] = []
for value in normalize_urls(existing):
if value not in merged:
merged.append(value)
for value in normalize_urls(list(incoming or [])):
if value not in merged:
merged.append(value)
return merged
def remove_urls(existing: Any, remove: Sequence[Any]) -> list[str]:
"""Remove URL values from an existing URL field and return survivors."""
from SYS.metadata import normalize_urls
current = normalize_urls(existing)
remove_set = {value for value in normalize_urls(list(remove or [])) if value}
if not remove_set:
return current
return [value for value in current if value not in remove_set]
def set_item_urls(item: Any, urls: Sequence[Any]) -> None:
"""Persist normalized URL values back onto a dict/object result item."""
normalized = merge_urls([], list(urls or []))
payload: Any = normalized[0] if len(normalized) == 1 else list(normalized)
try:
if isinstance(item, dict):
item["url"] = payload
return
if hasattr(item, "url"):
setattr(item, "url", payload)
except Exception:
return
def extract_relationships(result: Any) -> Optional[Dict[str, Any]]: def extract_relationships(result: Any) -> Optional[Dict[str, Any]]:
if isinstance(result, models.PipeObject): if isinstance(result, models.PipeObject):
relationships = result.get_relationships() relationships = result.get_relationships()
@@ -3270,14 +3609,9 @@ def check_url_exists_in_storage(
ext = extracted.get("ext") if isinstance(extracted, dict) else "" ext = extracted.get("ext") if isinstance(extracted, dict) else ""
size_val = extracted.get("size") if isinstance(extracted, dict) else None size_val = extracted.get("size") if isinstance(extracted, dict) else None
return { return build_table_result_payload(
"title": str(title), title=str(title),
"store": str(get_field(hit, "store") or backend_name), columns=[
"hash": str(file_hash or ""),
"ext": str(ext or ""),
"size": size_val,
"url": original_url,
"columns": [
("Title", str(title)), ("Title", str(title)),
("Store", str(get_field(hit, "store") or backend_name)), ("Store", str(get_field(hit, "store") or backend_name)),
("Hash", str(file_hash or "")), ("Hash", str(file_hash or "")),
@@ -3285,7 +3619,12 @@ def check_url_exists_in_storage(
("Size", size_val), ("Size", size_val),
("URL", original_url), ("URL", original_url),
], ],
} store=str(get_field(hit, "store") or backend_name),
hash=str(file_hash or ""),
ext=str(ext or ""),
size=size_val,
url=original_url,
)
def _search_backend_url_hits( def _search_backend_url_hits(
backend: Any, backend: Any,
@@ -3443,18 +3782,18 @@ def check_url_exists_in_storage(
seen_pairs.add((original_url, str(backend_name))) seen_pairs.add((original_url, str(backend_name)))
matched_urls.add(original_url) matched_urls.add(original_url)
display_row = { display_row = build_table_result_payload(
"title": "(exists)", title="(exists)",
"store": str(backend_name), columns=[
"hash": found_hash or "",
"url": original_url,
"columns": [
("Title", "(exists)"), ("Title", "(exists)"),
("Store", str(backend_name)), ("Store", str(backend_name)),
("Hash", found_hash or ""), ("Hash", found_hash or ""),
("URL", original_url), ("URL", original_url),
], ],
} store=str(backend_name),
hash=found_hash or "",
url=original_url,
)
match_rows.append(display_row) match_rows.append(display_row)
continue continue
@@ -3700,11 +4039,7 @@ def display_and_persist_items(
setattr(table, "_rendered_by_cmdlet", True) setattr(table, "_rendered_by_cmdlet", True)
# Use provided subject or default to first item
if subject is None:
subject = items[0] if len(items) == 1 else list(items)
# Persist table for @N selection across command boundaries # Persist table for @N selection across command boundaries
pipeline_context.set_last_result_table(table, list(items), subject=subject) publish_result_table(pipeline_context, table, items, subject=subject)
except Exception: except Exception:
pass pass

View File

@@ -12,7 +12,9 @@ from urllib.parse import urlparse
from SYS import models from SYS import models
from SYS import pipeline as ctx from SYS import pipeline as ctx
from SYS.logger import log, debug, is_debug_enabled from SYS.logger import log, debug, is_debug_enabled
from SYS.payload_builders import build_table_result_payload
from SYS.pipeline_progress import PipelineProgress from SYS.pipeline_progress import PipelineProgress
from SYS.result_publication import overlay_existing_result_table, publish_result_table
from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS from SYS.utils_constant import ALL_SUPPORTED_EXTENSIONS
from Store import Store from Store import Store
from API.HTTP import _download_direct_file from API.HTTP import _download_direct_file
@@ -444,27 +446,18 @@ class Add_File(Cmdlet):
ext = str(file_info.get("ext") or "").lstrip(".") ext = str(file_info.get("ext") or "").lstrip(".")
size = file_info.get("size", 0) size = file_info.get("size", 0)
row_item = { row_item = build_table_result_payload(
"path": title=clean_title,
str(p) if p is not None else "", columns=[
"hash": ("Title", clean_title),
hp, ("Hash", hp),
"title": ("Size", size),
clean_title, ("Ext", ext),
"columns": [
("Title",
clean_title),
("Hash",
hp),
("Size",
size),
("Ext",
ext),
], ],
# Used by @N replay (CLI will combine selected rows into -path file1,file2,...) selection_args=["-path", str(p) if p is not None else ""],
"_selection_args": ["-path", path=str(p) if p is not None else "",
str(p) if p is not None else ""], hash=hp,
} )
rows.append(row_item) rows.append(row_item)
table.add_result(row_item) table.add_result(row_item)
@@ -537,8 +530,7 @@ class Add_File(Cmdlet):
else: else:
pipe_obj.extra = {} pipe_obj.extra = {}
merged_urls.extend(cli_urls) merged_urls = sh.merge_urls(merged_urls, cli_urls)
merged_urls = normalize_urls(merged_urls)
if merged_urls: if merged_urls:
pipe_obj.extra["url"] = merged_urls pipe_obj.extra["url"] = merged_urls
except Exception: except Exception:
@@ -827,13 +819,15 @@ class Add_File(Cmdlet):
except Exception as exc: except Exception as exc:
debug(f"[add-file] Item details render failed: {exc}") debug(f"[add-file] Item details render failed: {exc}")
ctx.set_last_result_table_overlay( publish_result_table(
ctx,
table, table,
items, items,
subject={ subject={
"store": store, "store": store,
"hash": hashes "hash": hashes
} },
overlay=True,
) )
except Exception: except Exception:
pass pass
@@ -1673,7 +1667,7 @@ class Add_File(Cmdlet):
table = Table("Result") table = Table("Result")
table.add_result(payload) table.add_result(payload)
# Overlay so @1 refers to this add-file result without overwriting search history # Overlay so @1 refers to this add-file result without overwriting search history
ctx.set_last_result_table_overlay(table, [payload], subject=payload) publish_result_table(ctx, table, [payload], subject=payload, overlay=True)
except Exception: except Exception:
# If table rendering fails, still keep @ selection items # If table rendering fails, still keep @ selection items
try: try:
@@ -1734,15 +1728,13 @@ class Add_File(Cmdlet):
try: try:
table = ctx.get_last_result_table() table = ctx.get_last_result_table()
items = ctx.get_last_result_items() items = ctx.get_last_result_items()
if table is not None and items: overlay_existing_result_table(
ctx.set_last_result_table_overlay( ctx,
table, subject={
items, "store": store,
subject={ "hash": hash_value
"store": store, },
"hash": hash_value )
}
)
except Exception: except Exception:
pass pass
@@ -2484,58 +2476,36 @@ class Add_File(Cmdlet):
if not pairs: if not pairs:
continue continue
try: try:
backend = store[backend_name] backend, store, _exc = sh.get_store_backend(
config,
backend_name,
store_registry=store,
)
if backend is None:
continue
items = sh.coalesce_hash_value_pairs(pairs)
if not items:
continue
bulk = getattr(backend, "add_url_bulk", None)
if callable(bulk):
try:
bulk(items)
continue
except Exception:
pass
single = getattr(backend, "add_url", None)
if callable(single):
for h, u in items:
try:
single(h, u)
except Exception:
continue
except Exception: except Exception:
continue continue
# Merge URLs per hash and de-duplicate.
merged: Dict[str,
List[str]] = {}
for file_hash, urls in pairs:
h = str(file_hash or "").strip().lower()
if len(h) != 64:
continue
url_list: List[str] = []
try:
for u in urls or []:
s = str(u or "").strip()
if s:
url_list.append(s)
except Exception:
url_list = []
if not url_list:
continue
bucket = merged.setdefault(h, [])
seen = set(bucket)
for u in url_list:
if u in seen:
continue
seen.add(u)
bucket.append(u)
items: List[tuple[str,
List[str]]] = [(h,
u) for h, u in merged.items() if u]
if not items:
continue
bulk = getattr(backend, "add_url_bulk", None)
if callable(bulk):
try:
bulk(items)
continue
except Exception:
pass
single = getattr(backend, "add_url", None)
if callable(single):
for h, u in items:
try:
single(h, u)
except Exception:
continue
@staticmethod @staticmethod
def _apply_pending_tag_associations( def _apply_pending_tag_associations(
pending: Dict[str, pending: Dict[str,
@@ -2552,30 +2522,15 @@ class Add_File(Cmdlet):
except Exception: except Exception:
return return
for backend_name, pairs in (pending or {}).items(): sh.run_store_hash_value_batches(
if not pairs: config,
continue pending or {},
try: bulk_method_name="add_tags_bulk",
backend = store[backend_name] single_method_name="add_tag",
except Exception: store_registry=store,
continue pass_config_to_bulk=False,
pass_config_to_single=False,
# Try bulk variant first )
bulk = getattr(backend, "add_tags_bulk", None)
if callable(bulk):
try:
bulk([(h, t) for h, t in pairs])
continue
except Exception:
pass
single = getattr(backend, "add_tag", None)
if callable(single):
for h, t in pairs:
try:
single(h, t)
except Exception:
continue
@staticmethod @staticmethod
def _load_sidecar_bundle( def _load_sidecar_bundle(

View File

@@ -18,8 +18,6 @@ normalize_hash = sh.normalize_hash
parse_cmdlet_args = sh.parse_cmdlet_args parse_cmdlet_args = sh.parse_cmdlet_args
normalize_result_input = sh.normalize_result_input normalize_result_input = sh.normalize_result_input
should_show_help = sh.should_show_help should_show_help = sh.should_show_help
from Store import Store
from SYS.utils import sha256_file
class Add_Note(Cmdlet): class Add_Note(Cmdlet):
@@ -171,14 +169,6 @@ class Add_Note(Cmdlet):
return tokens return tokens
def _resolve_hash(
self,
raw_hash: Optional[str],
raw_path: Optional[str],
override_hash: Optional[str],
) -> Optional[str]:
return sh.resolve_hash_for_cmdlet(raw_hash, raw_path, override_hash)
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
if should_show_help(args): if should_show_help(args):
log(f"Cmdlet: {self.name}\nSummary: {self.summary}\nUsage: {self.usage}") log(f"Cmdlet: {self.name}\nSummary: {self.summary}\nUsage: {self.usage}")
@@ -217,8 +207,12 @@ class Add_Note(Cmdlet):
# Direct targeting mode: apply note once to the explicit target and # Direct targeting mode: apply note once to the explicit target and
# pass through any piped items unchanged. # pass through any piped items unchanged.
try: try:
store_registry = Store(config) backend, _store_registry, exc = sh.get_store_backend(
backend = store_registry[str(store_override)] config,
str(store_override),
)
if backend is None:
raise exc or KeyError(store_override)
ok = bool( ok = bool(
backend.set_note( backend.set_note(
str(hash_override), str(hash_override),
@@ -262,7 +256,7 @@ class Add_Note(Cmdlet):
) )
return 1 return 1
store_registry = Store(config) store_registry = None
planned_ops = 0 planned_ops = 0
# Batch write plan: store -> [(hash, name, text), ...] # Batch write plan: store -> [(hash, name, text), ...]
@@ -307,9 +301,12 @@ class Add_Note(Cmdlet):
ctx.emit(res) ctx.emit(res)
continue continue
store_name = str(store_override or res.get("store") or "").strip() store_name, resolved_hash = sh.resolve_item_store_hash(
raw_hash = res.get("hash") res,
raw_path = res.get("path") override_store=str(store_override) if store_override else None,
override_hash=str(hash_override) if hash_override else None,
path_fields=("path",),
)
if not store_name: if not store_name:
log( log(
@@ -318,11 +315,6 @@ class Add_Note(Cmdlet):
) )
continue continue
resolved_hash = self._resolve_hash(
raw_hash=str(raw_hash) if raw_hash else None,
raw_path=str(raw_path) if raw_path else None,
override_hash=str(hash_override) if hash_override else None,
)
if not resolved_hash: if not resolved_hash:
log( log(
"[add_note] Warning: Item missing usable hash; skipping", "[add_note] Warning: Item missing usable hash; skipping",
@@ -343,23 +335,23 @@ class Add_Note(Cmdlet):
# Execute batch operations # Execute batch operations
success_count = 0 def _on_store_error(store_name: str, exc: Exception) -> None:
for store_name, ops in note_ops.items(): log(f"[add_note] Store access failed '{store_name}': {exc}", file=sys.stderr)
try:
backend = store_registry[store_name] def _on_unsupported_store(store_name: str) -> None:
if not hasattr(backend, "set_note"): log(f"[add_note] Store '{store_name}' does not support notes", file=sys.stderr)
log(f"[add_note] Store '{store_name}' does not support notes", file=sys.stderr)
continue def _on_item_error(store_name: str, hash_value: str, note_name_value: str, exc: Exception) -> None:
log(f"[add_note] Write failed {store_name}:{hash_value} ({note_name_value}): {exc}", file=sys.stderr)
for (h, name, text) in ops:
try: store_registry, success_count = sh.run_store_note_batches(
if backend.set_note(h, name, text, config=config): config,
success_count += 1 note_ops,
except Exception as e: store_registry=store_registry,
log(f"[add_note] Write failed {store_name}:{h} ({name}): {e}", file=sys.stderr) on_store_error=_on_store_error,
on_unsupported_store=_on_unsupported_store,
except Exception as e: on_item_error=_on_item_error,
log(f"[add_note] Store access failed '{store_name}': {e}", file=sys.stderr) )
if planned_ops > 0: if planned_ops > 0:
msg = f"✓ add-note: Updated {success_count}/{planned_ops} notes across {len(note_ops)} stores" msg = f"✓ add-note: Updated {success_count}/{planned_ops} notes across {len(note_ops)} stores"

View File

@@ -8,6 +8,7 @@ from pathlib import Path
import sys import sys
from SYS.logger import log from SYS.logger import log
from SYS.item_accessors import get_sha256_hex, get_store_name
from SYS import pipeline as ctx from SYS import pipeline as ctx
from API import HydrusNetwork as hydrus_wrapper from API import HydrusNetwork as hydrus_wrapper
@@ -20,7 +21,6 @@ parse_cmdlet_args = sh.parse_cmdlet_args
normalize_result_input = sh.normalize_result_input normalize_result_input = sh.normalize_result_input
should_show_help = sh.should_show_help should_show_help = sh.should_show_help
get_field = sh.get_field get_field = sh.get_field
from Store import Store
CMDLET = Cmdlet( CMDLET = Cmdlet(
name="add-relationship", name="add-relationship",
@@ -68,14 +68,7 @@ CMDLET = Cmdlet(
) )
def _normalize_hash_hex(value: Optional[str]) -> Optional[str]: _normalize_hash_hex = sh.normalize_hash
"""Normalize a hash hex string to lowercase 64-char format."""
if not value or not isinstance(value, str):
return None
normalized = value.strip().lower()
if len(normalized) == 64 and all(c in "0123456789abcdef" for c in normalized):
return normalized
return None
def _extract_relationships_from_tag(tag_value: str) -> Dict[str, list[str]]: def _extract_relationships_from_tag(tag_value: str) -> Dict[str, list[str]]:
@@ -279,23 +272,10 @@ def _resolve_items_from_at(token: str) -> Optional[list[Any]]:
def _extract_hash_and_store(item: Any) -> tuple[Optional[str], Optional[str]]: def _extract_hash_and_store(item: Any) -> tuple[Optional[str], Optional[str]]:
"""Extract (hash_hex, store) from a result item (dict/object).""" """Extract (hash_hex, store) from a result item (dict/object)."""
try: try:
h = get_field(item, return (
"hash_hex") or get_field(item, get_sha256_hex(item, "hash_hex", "hash", "file_hash"),
"hash") or get_field(item, get_store_name(item, "store"),
"file_hash") )
s = get_field(item, "store")
hash_norm = _normalize_hash_hex(str(h) if h is not None else None)
store_norm: Optional[str]
if s is None:
store_norm = None
else:
store_norm = str(s).strip()
if not store_norm:
store_norm = None
return hash_norm, store_norm
except Exception: except Exception:
return None, None return None, None
@@ -461,9 +441,12 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
parsed = parse_cmdlet_args(_args, CMDLET) parsed = parse_cmdlet_args(_args, CMDLET)
arg_path: Optional[Path] = None arg_path: Optional[Path] = None
override_store = parsed.get("store") override_store = parsed.get("store")
override_hashes = sh.parse_hash_query(parsed.get("query")) override_hashes, query_valid = sh.require_hash_query(
if parsed.get("query") and not override_hashes: parsed.get("query"),
log("Invalid -query value (expected hash:<sha256>)", file=sys.stderr) "Invalid -query value (expected hash:<sha256>)",
log_file=sys.stderr,
)
if not query_valid:
return 1 return 1
king_arg = parsed.get("king") king_arg = parsed.get("king")
alt_arg = parsed.get("alt") alt_arg = parsed.get("alt")
@@ -618,14 +601,13 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
is_folder_store = False is_folder_store = False
store_root: Optional[Path] = None store_root: Optional[Path] = None
if store_name: if store_name:
try: backend, _store_registry, _exc = sh.get_store_backend(config, str(store_name))
store = Store(config) if backend is not None:
backend = store[str(store_name)]
loc = getattr(backend, "location", None) loc = getattr(backend, "location", None)
if callable(loc): if callable(loc):
is_folder_store = True is_folder_store = True
store_root = Path(str(loc())) store_root = Path(str(loc()))
except Exception: else:
backend = None backend = None
is_folder_store = False is_folder_store = False
store_root = None store_root = None

View File

@@ -6,6 +6,9 @@ import sys
import re import re
from SYS.logger import log, debug from SYS.logger import log, debug
from SYS.item_accessors import extract_item_tags, get_string_list, set_field
from SYS.payload_builders import extract_title_tag_value
from SYS.result_publication import publish_result_table
from SYS import models from SYS import models
from SYS import pipeline as ctx from SYS import pipeline as ctx
@@ -24,7 +27,6 @@ collapse_namespace_tag = sh.collapse_namespace_tag
should_show_help = sh.should_show_help should_show_help = sh.should_show_help
get_field = sh.get_field get_field = sh.get_field
from Store import Store from Store import Store
from SYS.utils import sha256_file
_FIELD_NAME_RE = re.compile(r"^[A-Za-z0-9_]+$") _FIELD_NAME_RE = re.compile(r"^[A-Za-z0-9_]+$")
@@ -239,33 +241,15 @@ def _try_compile_extract_template(
def _extract_title_tag(tags: List[str]) -> Optional[str]: def _extract_title_tag(tags: List[str]) -> Optional[str]:
"""Return the value of the first title: tag if present.""" """Return the value of the first title: tag if present."""
for t in tags: return extract_title_tag_value(tags)
if t.lower().startswith("title:"):
value = t.split(":", 1)[1].strip()
return value or None
return None
def _extract_item_tags(res: Any) -> List[str]: def _extract_item_tags(res: Any) -> List[str]:
if isinstance(res, models.PipeObject): return extract_item_tags(res)
raw = getattr(res, "tag", None)
elif isinstance(res, dict):
raw = res.get("tag")
else:
raw = None
if isinstance(raw, list):
return [str(t) for t in raw if t is not None]
if isinstance(raw, str) and raw.strip():
return [raw]
return []
def _set_item_tags(res: Any, tags: List[str]) -> None: def _set_item_tags(res: Any, tags: List[str]) -> None:
if isinstance(res, models.PipeObject): set_field(res, "tag", tags)
res.tag = tags
elif isinstance(res, dict):
res["tag"] = tags
def _apply_title_to_result(res: Any, title_value: Optional[str]) -> None: def _apply_title_to_result(res: Any, title_value: Optional[str]) -> None:
@@ -401,7 +385,7 @@ def _refresh_result_table_title(
# Keep the underlying history intact; update only the overlay so @.. can # Keep the underlying history intact; update only the overlay so @.. can
# clear the overlay then continue back to prior tables (e.g., the search list). # clear the overlay then continue back to prior tables (e.g., the search list).
ctx.set_last_result_table_overlay(new_table, updated_items) publish_result_table(ctx, new_table, updated_items, overlay=True)
except Exception: except Exception:
pass pass
@@ -439,30 +423,21 @@ def _refresh_tag_view(
refresh_args: List[str] = ["-query", f"hash:{target_hash}"] refresh_args: List[str] = ["-query", f"hash:{target_hash}"]
# Build a lean subject so get-tag fetches fresh tags instead of reusing cached payloads. # Build a lean subject so get-tag fetches fresh tags instead of reusing cached payloads.
def _value_has_content(value: Any) -> bool:
if value is None:
return False
if isinstance(value, str):
return bool(value.strip())
if isinstance(value, (list, tuple, set)):
return len(value) > 0
return True
def _build_refresh_subject() -> Dict[str, Any]: def _build_refresh_subject() -> Dict[str, Any]:
payload: Dict[str, Any] = {} payload: Dict[str, Any] = {}
payload["hash"] = target_hash payload["hash"] = target_hash
if _value_has_content(store_name): if sh.value_has_content(store_name):
payload["store"] = store_name payload["store"] = store_name
path_value = target_path or get_field(subject, "path") path_value = target_path or get_field(subject, "path")
if not _value_has_content(path_value): if not sh.value_has_content(path_value):
path_value = get_field(subject, "target") path_value = get_field(subject, "target")
if _value_has_content(path_value): if sh.value_has_content(path_value):
payload["path"] = path_value payload["path"] = path_value
for key in ("title", "name", "url", "relations", "service_name"): for key in ("title", "name", "url", "relations", "service_name"):
val = get_field(subject, key) val = get_field(subject, key)
if _value_has_content(val): if sh.value_has_content(val):
payload[key] = val payload[key] = val
extra_value = get_field(subject, "extra") extra_value = get_field(subject, "extra")
@@ -473,7 +448,7 @@ def _refresh_tag_view(
} }
if cleaned: if cleaned:
payload["extra"] = cleaned payload["extra"] = cleaned
elif _value_has_content(extra_value): elif sh.value_has_content(extra_value):
payload["extra"] = extra_value payload["extra"] = extra_value
return payload return payload
@@ -570,15 +545,15 @@ class Add_Tag(Cmdlet):
extract_debug = bool(parsed.get("extract-debug", False)) extract_debug = bool(parsed.get("extract-debug", False))
extract_debug_rx, extract_debug_err = _try_compile_extract_template(extract_template) extract_debug_rx, extract_debug_err = _try_compile_extract_template(extract_template)
query_hash = sh.parse_single_hash_query(parsed.get("query")) query_hash, query_valid = sh.require_single_hash_query(
if parsed.get("query") and not query_hash: parsed.get("query"),
log( "[add_tag] Error: -query must be of the form hash:<sha256>",
"[add_tag] Error: -query must be of the form hash:<sha256>", log_file=sys.stderr,
file=sys.stderr )
) if not query_valid:
return 1 return 1
hash_override = normalize_hash(query_hash) if query_hash else None hash_override = query_hash
# If add-tag is in the middle of a pipeline (has downstream stages), default to # If add-tag is in the middle of a pipeline (has downstream stages), default to
# including temp files. This enables common flows like: # including temp files. This enables common flows like:
@@ -879,21 +854,11 @@ class Add_Tag(Cmdlet):
) )
return 1 return 1
resolved_hash = ( resolved_hash = sh.resolve_hash_for_cmdlet(
normalize_hash(hash_override) str(raw_hash) if raw_hash else None,
if hash_override else normalize_hash(raw_hash) str(raw_path) if raw_path else None,
str(hash_override) if hash_override else None,
) )
if not resolved_hash and raw_path:
try:
p = Path(str(raw_path))
stem = p.stem
if len(stem) == 64 and all(c in "0123456789abcdef"
for c in stem.lower()):
resolved_hash = stem.lower()
elif p.exists() and p.is_file():
resolved_hash = sha256_file(p)
except Exception:
resolved_hash = None
if not resolved_hash: if not resolved_hash:
log( log(
@@ -903,9 +868,13 @@ class Add_Tag(Cmdlet):
ctx.emit(res) ctx.emit(res)
continue continue
try: backend, store_registry, exc = sh.get_store_backend(
backend = store_registry[str(store_name)] config,
except Exception as exc: str(store_name),
store_registry=store_registry,
suppress_debug=True,
)
if backend is None:
log( log(
f"[add_tag] Error: Unknown store '{store_name}': {exc}", f"[add_tag] Error: Unknown store '{store_name}': {exc}",
file=sys.stderr file=sys.stderr

View File

@@ -49,9 +49,11 @@ class Add_Url(sh.Cmdlet):
except Exception: except Exception:
pass pass
query_hash = sh.parse_single_hash_query(parsed.get("query")) query_hash, query_valid = sh.require_single_hash_query(
if parsed.get("query") and not query_hash: parsed.get("query"),
log("Error: -query must be of the form hash:<sha256>") "Error: -query must be of the form hash:<sha256>",
)
if not query_valid:
return 1 return 1
# Bulk input is common in pipelines; treat a list of PipeObjects as a batch. # Bulk input is common in pipelines; treat a list of PipeObjects as a batch.
@@ -117,125 +119,53 @@ class Add_Url(sh.Cmdlet):
try: try:
storage = Store(config) storage = Store(config)
def _merge_urls(existing: Any, incoming: List[str]) -> List[str]:
out: List[str] = []
try:
if isinstance(existing, str):
out.extend(
[p.strip() for p in existing.split(",") if p.strip()]
)
elif isinstance(existing, (list, tuple)):
out.extend([str(u).strip() for u in existing if str(u).strip()])
except Exception:
out = []
for u in incoming:
if u and u not in out:
out.append(u)
return out
def _set_item_url(item: Any, merged: List[str]) -> None:
try:
if isinstance(item, dict):
if len(merged) == 1:
item["url"] = merged[0]
else:
item["url"] = list(merged)
return
# PipeObject-like
if hasattr(item, "url"):
if len(merged) == 1:
setattr(item, "url", merged[0])
else:
setattr(item, "url", list(merged))
except Exception:
return
# Build batches per store. # Build batches per store.
store_override = parsed.get("store") store_override = parsed.get("store")
batch: Dict[str,
List[Tuple[str,
List[str]]]] = {}
pass_through: List[Any] = []
if results: if results:
for item in results: def _warn(message: str) -> None:
pass_through.append(item) ctx.print_if_visible(f"[add-url] Warning: {message}", file=sys.stderr)
raw_hash = query_hash or sh.get_field(item, "hash") batch, pass_through = sh.collect_store_hash_value_batch(
raw_store = store_override or sh.get_field(item, "store") results,
if not raw_hash or not raw_store: store_registry=storage,
ctx.print_if_visible( value_resolver=lambda _item: list(urls),
"[add-url] Warning: Item missing hash/store; skipping", override_hash=query_hash,
file=sys.stderr override_store=store_override,
) on_warning=_warn,
continue )
normalized = sh.normalize_hash(raw_hash)
if not normalized:
ctx.print_if_visible(
"[add-url] Warning: Item has invalid hash; skipping",
file=sys.stderr
)
continue
store_text = str(raw_store).strip()
if not store_text:
ctx.print_if_visible(
"[add-url] Warning: Item has empty store; skipping",
file=sys.stderr
)
continue
# Validate backend exists (skip PATH/unknown).
if not storage.is_available(store_text):
ctx.print_if_visible(
f"[add-url] Warning: Store '{store_text}' not configured; skipping",
file=sys.stderr,
)
continue
batch.setdefault(store_text, []).append((normalized, list(urls)))
# Execute per-store batches. # Execute per-store batches.
for store_text, pairs in batch.items(): storage, batch_stats = sh.run_store_hash_value_batches(
try: config,
backend = storage[store_text] batch,
except Exception: bulk_method_name="add_url_bulk",
continue single_method_name="add_url",
store_registry=storage,
# Coalesce duplicates per hash before passing to backend. )
merged: Dict[str, for store_text, item_count, _value_count in batch_stats:
List[str]] = {}
for h, ulist in pairs:
merged.setdefault(h, [])
for u in ulist or []:
if u and u not in merged[h]:
merged[h].append(u)
bulk_pairs = [(h, merged[h]) for h in merged.keys()]
bulk_fn = getattr(backend, "add_url_bulk", None)
if callable(bulk_fn):
bulk_fn(bulk_pairs, config=config)
else:
for h, ulist in bulk_pairs:
backend.add_url(h, ulist, config=config)
ctx.print_if_visible( ctx.print_if_visible(
f"✓ add-url: {len(urls)} url(s) for {len(bulk_pairs)} item(s) in '{store_text}'", f"✓ add-url: {len(urls)} url(s) for {item_count} item(s) in '{store_text}'",
file=sys.stderr, file=sys.stderr,
) )
# Pass items through unchanged (but update url field for convenience). # Pass items through unchanged (but update url field for convenience).
for item in pass_through: for item in pass_through:
existing = sh.get_field(item, "url") existing = sh.get_field(item, "url")
merged = _merge_urls(existing, list(urls)) merged = sh.merge_urls(existing, list(urls))
_set_item_url(item, merged) sh.set_item_urls(item, merged)
ctx.emit(item) ctx.emit(item)
return 0 return 0
# Single-item mode # Single-item mode
backend = storage[str(store_name)] backend, storage, exc = sh.get_store_backend(
config,
str(store_name),
store_registry=storage,
)
if backend is None:
log(f"Error: Storage backend '{store_name}' not configured")
return 1
backend.add_url(str(file_hash), urls, config=config) backend.add_url(str(file_hash), urls, config=config)
ctx.print_if_visible( ctx.print_if_visible(
f"✓ add-url: {len(urls)} url(s) added", f"✓ add-url: {len(urls)} url(s) added",
@@ -243,14 +173,11 @@ class Add_Url(sh.Cmdlet):
) )
if result is not None: if result is not None:
existing = sh.get_field(result, "url") existing = sh.get_field(result, "url")
merged = _merge_urls(existing, list(urls)) merged = sh.merge_urls(existing, list(urls))
_set_item_url(result, merged) sh.set_item_urls(result, merged)
ctx.emit(result) ctx.emit(result)
return 0 return 0
except KeyError:
log(f"Error: Storage backend '{store_name}' not configured")
return 1
except Exception as exc: except Exception as exc:
log(f"Error adding URL: {exc}", file=sys.stderr) log(f"Error adding URL: {exc}", file=sys.stderr)
return 1 return 1

View File

@@ -13,6 +13,7 @@ from typing import Any, Dict, List, Sequence, Set
from urllib.parse import parse_qs, urlparse from urllib.parse import parse_qs, urlparse
from SYS.logger import log from SYS.logger import log
from SYS.item_accessors import get_http_url, get_sha256_hex, get_store_name
from SYS.utils import extract_hydrus_hash_from_url from SYS.utils import extract_hydrus_hash_from_url
from SYS import pipeline as ctx from SYS import pipeline as ctx
@@ -27,41 +28,16 @@ create_pipe_object_result = sh.create_pipe_object_result
parse_cmdlet_args = sh.parse_cmdlet_args parse_cmdlet_args = sh.parse_cmdlet_args
should_show_help = sh.should_show_help should_show_help = sh.should_show_help
_SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
def _extract_sha256_hex(item: Any) -> str: def _extract_sha256_hex(item: Any) -> str:
try: return get_sha256_hex(item, "hash") or ""
if isinstance(item, dict):
h = item.get("hash")
else:
h = getattr(item, "hash", None)
if isinstance(h, str) and _SHA256_RE.fullmatch(h.strip()):
return h.strip().lower()
except Exception:
pass
return ""
def _extract_store_name(item: Any) -> str: def _extract_store_name(item: Any) -> str:
try: return get_store_name(item, "store") or ""
if isinstance(item, dict):
s = item.get("store")
else:
s = getattr(item, "store", None)
return str(s or "").strip()
except Exception:
return ""
def _extract_url(item: Any) -> str: def _extract_url(item: Any) -> str:
try: return get_http_url(item, "url", "target") or ""
u = sh.get_field(item, "url") or sh.get_field(item, "target")
if isinstance(u, str) and u.strip().lower().startswith(("http://", "https://")):
return u.strip()
except Exception:
pass
return ""
def _extract_hash_from_hydrus_file_url(url: str) -> str: def _extract_hash_from_hydrus_file_url(url: str) -> str:
@@ -217,10 +193,9 @@ def _resolve_existing_or_fetch_path(item: Any,
store_name = _extract_store_name(item) store_name = _extract_store_name(item)
if file_hash and store_name: if file_hash and store_name:
try: try:
from Store import Store backend, _store_registry, _exc = sh.get_store_backend(config, store_name)
if backend is None:
store = Store(config) return None, None
backend = store[store_name]
src = backend.get_file(file_hash) src = backend.get_file(file_hash)
if isinstance(src, Path): if isinstance(src, Path):
if src.exists(): if src.exists():
@@ -320,11 +295,10 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
# This cmdlet always creates the archive in the configured output directory and emits it. # This cmdlet always creates the archive in the configured output directory and emits it.
# Collect piped items; archive-file is a batch command (single output). # Collect piped items; archive-file is a batch command (single output).
items: List[Any] = [] items: List[Any] = sh.normalize_result_items(
if isinstance(result, list): result,
items = list(result) include_falsey_single=True,
elif result is not None: )
items = [result]
if not items: if not items:
log("No piped items provided to archive-file", file=sys.stderr) log("No piped items provided to archive-file", file=sys.stderr)

View File

@@ -7,6 +7,7 @@ import shutil
import subprocess import subprocess
from SYS.logger import log, debug from SYS.logger import log, debug
from SYS.payload_builders import build_file_result_payload
from SYS.utils import sha256_file from SYS.utils import sha256_file
from . import _shared as sh from . import _shared as sh
from SYS import pipeline as ctx from SYS import pipeline as ctx
@@ -279,13 +280,15 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
title = extract_title_from_result(item) or output_path.stem title = extract_title_from_result(item) or output_path.stem
ctx.emit({ ctx.emit(
"path": str(output_path), build_file_result_payload(
"title": title, title=title,
"hash": out_hash, path=str(output_path),
"media_kind": target_kind, hash_value=out_hash,
"source_path": str(input_path), media_kind=target_kind,
}) source_path=str(input_path),
)
)
if delete_src: if delete_src:
try: try:

View File

@@ -11,6 +11,7 @@ from Store import Store
from . import _shared as sh from . import _shared as sh
from API import HydrusNetwork as hydrus_wrapper from API import HydrusNetwork as hydrus_wrapper
from SYS import pipeline as ctx from SYS import pipeline as ctx
from SYS.result_table_helpers import add_row_columns
from SYS.result_table import Table, _format_size from SYS.result_table import Table, _format_size
from SYS.rich_display import stdout_console from SYS.rich_display import stdout_console
@@ -487,21 +488,18 @@ class Delete_File(sh.Cmdlet):
reason_tokens.append(token) reason_tokens.append(token)
i += 1 i += 1
override_hash = sh.parse_single_hash_query( override_hash, query_valid = sh.require_single_hash_query(
override_query override_query,
) if override_query else None "Invalid -query value (expected hash:<sha256>)",
if override_query and not override_hash: log_file=sys.stderr,
log("Invalid -query value (expected hash:<sha256>)", file=sys.stderr) )
if not query_valid:
return 1 return 1
reason = " ".join(token for token in reason_tokens reason = " ".join(token for token in reason_tokens
if str(token).strip()).strip() if str(token).strip()).strip()
items = [] items = sh.normalize_result_items(result)
if isinstance(result, list):
items = result
elif result:
items = [result]
if not items: if not items:
log("No items to delete", file=sys.stderr) log("No items to delete", file=sys.stderr)
@@ -526,16 +524,16 @@ class Delete_File(sh.Cmdlet):
table = Table("Deleted") table = Table("Deleted")
table._interactive(True)._perseverance(True) table._interactive(True)._perseverance(True)
for row in deleted_rows: for row in deleted_rows:
result_row = table.add_row() add_row_columns(
result_row.add_column("Title", row.get("title", "")) table,
result_row.add_column("Store", row.get("store", "")) [
result_row.add_column("Hash", row.get("hash", "")) ("Title", row.get("title", "")),
result_row.add_column( ("Store", row.get("store", "")),
"Size", ("Hash", row.get("hash", "")),
_format_size(row.get("size_bytes"), ("Size", _format_size(row.get("size_bytes"), integer_only=False)),
integer_only=False) ("Ext", row.get("ext", "")),
],
) )
result_row.add_column("Ext", row.get("ext", ""))
# Display-only: print directly and do not affect selection/history. # Display-only: print directly and do not affect selection/history.
try: try:

View File

@@ -1,7 +1,6 @@
from __future__ import annotations from __future__ import annotations
from pathlib import Path from typing import Any, Dict, Sequence
from typing import Any, Dict, Optional, Sequence
import sys import sys
from SYS.logger import log from SYS.logger import log
@@ -17,8 +16,6 @@ parse_cmdlet_args = sh.parse_cmdlet_args
normalize_result_input = sh.normalize_result_input normalize_result_input = sh.normalize_result_input
get_field = sh.get_field get_field = sh.get_field
should_show_help = sh.should_show_help should_show_help = sh.should_show_help
from Store import Store
from SYS.utils import sha256_file
class Delete_Note(Cmdlet): class Delete_Note(Cmdlet):
@@ -50,14 +47,6 @@ class Delete_Note(Cmdlet):
pass pass
self.register() self.register()
def _resolve_hash(
self,
raw_hash: Optional[str],
raw_path: Optional[str],
override_hash: Optional[str],
) -> Optional[str]:
return sh.resolve_hash_for_cmdlet(raw_hash, raw_path, override_hash)
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
if should_show_help(args): if should_show_help(args):
log(f"Cmdlet: {self.name}\nSummary: {self.summary}\nUsage: {self.usage}") log(f"Cmdlet: {self.name}\nSummary: {self.summary}\nUsage: {self.usage}")
@@ -66,12 +55,12 @@ class Delete_Note(Cmdlet):
parsed = parse_cmdlet_args(args, self) parsed = parse_cmdlet_args(args, self)
store_override = parsed.get("store") store_override = parsed.get("store")
query_hash = sh.parse_single_hash_query(parsed.get("query")) query_hash, query_valid = sh.require_single_hash_query(
if parsed.get("query") and not query_hash: parsed.get("query"),
log( "[delete_note] Error: -query must be of the form hash:<sha256>",
"[delete_note] Error: -query must be of the form hash:<sha256>", log_file=sys.stderr,
file=sys.stderr )
) if not query_valid:
return 1 return 1
note_name_override = str(parsed.get("name") or "").strip() note_name_override = str(parsed.get("name") or "").strip()
# Allow piping note rows from get-note: the selected item carries note_name. # Allow piping note rows from get-note: the selected item carries note_name.
@@ -97,7 +86,7 @@ class Delete_Note(Cmdlet):
) )
return 1 return 1
store_registry = Store(config) store_registry = None
deleted = 0 deleted = 0
for res in results: for res in results:
@@ -117,9 +106,12 @@ class Delete_Note(Cmdlet):
) )
return 1 return 1
store_name = str(store_override or res.get("store") or "").strip() store_name, resolved_hash = sh.resolve_item_store_hash(
raw_hash = res.get("hash") res,
raw_path = res.get("path") override_store=str(store_override) if store_override else None,
override_hash=str(query_hash) if query_hash else None,
path_fields=("path",),
)
if not store_name: if not store_name:
log( log(
@@ -128,18 +120,16 @@ class Delete_Note(Cmdlet):
) )
return 1 return 1
resolved_hash = self._resolve_hash(
raw_hash=str(raw_hash) if raw_hash else None,
raw_path=str(raw_path) if raw_path else None,
override_hash=str(query_hash) if query_hash else None,
)
if not resolved_hash: if not resolved_hash:
ctx.emit(res) ctx.emit(res)
continue continue
try: backend, store_registry, exc = sh.get_store_backend(
backend = store_registry[store_name] config,
except Exception as exc: store_name,
store_registry=store_registry,
)
if backend is None:
log( log(
f"[delete_note] Error: Unknown store '{store_name}': {exc}", f"[delete_note] Error: Unknown store '{store_name}': {exc}",
file=sys.stderr file=sys.stderr

View File

@@ -1,7 +1,6 @@
from __future__ import annotations from __future__ import annotations
from typing import Any, Dict, Sequence from typing import Any, Dict, Sequence
from pathlib import Path
import sys import sys
from SYS import pipeline as ctx from SYS import pipeline as ctx
@@ -15,7 +14,6 @@ parse_tag_arguments = sh.parse_tag_arguments
should_show_help = sh.should_show_help should_show_help = sh.should_show_help
get_field = sh.get_field get_field = sh.get_field
from SYS.logger import debug, log from SYS.logger import debug, log
from Store import Store
def _refresh_tag_view_if_current( def _refresh_tag_view_if_current(
@@ -80,31 +78,22 @@ def _refresh_tag_view_if_current(
refresh_args.extend(["-query", f"hash:{file_hash}"]) refresh_args.extend(["-query", f"hash:{file_hash}"])
# Build a lean subject so get-tag fetches fresh tags instead of reusing cached payloads. # Build a lean subject so get-tag fetches fresh tags instead of reusing cached payloads.
def _value_has_content(value: Any) -> bool:
if value is None:
return False
if isinstance(value, str):
return bool(value.strip())
if isinstance(value, (list, tuple, set)):
return len(value) > 0
return True
def _build_refresh_subject() -> Dict[str, Any]: def _build_refresh_subject() -> Dict[str, Any]:
payload: Dict[str, Any] = {} payload: Dict[str, Any] = {}
payload["hash"] = file_hash payload["hash"] = file_hash
store_value = store_name or get_field(subject, "store") store_value = store_name or get_field(subject, "store")
if _value_has_content(store_value): if sh.value_has_content(store_value):
payload["store"] = store_value payload["store"] = store_value
path_value = path or get_field(subject, "path") path_value = path or get_field(subject, "path")
if not _value_has_content(path_value): if not sh.value_has_content(path_value):
path_value = get_field(subject, "target") path_value = get_field(subject, "target")
if _value_has_content(path_value): if sh.value_has_content(path_value):
payload["path"] = path_value payload["path"] = path_value
for key in ("title", "name", "url", "relations", "service_name"): for key in ("title", "name", "url", "relations", "service_name"):
val = get_field(subject, key) val = get_field(subject, key)
if _value_has_content(val): if sh.value_has_content(val):
payload[key] = val payload[key] = val
extra_value = get_field(subject, "extra") extra_value = get_field(subject, "extra")
@@ -115,7 +104,7 @@ def _refresh_tag_view_if_current(
} }
if cleaned: if cleaned:
payload["extra"] = cleaned payload["extra"] = cleaned
elif _value_has_content(extra_value): elif sh.value_has_content(extra_value):
payload["extra"] = extra_value payload["extra"] = extra_value
return payload return payload
@@ -201,11 +190,12 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
rest.append(a) rest.append(a)
i += 1 i += 1
override_hash = sh.parse_single_hash_query( override_hash, query_valid = sh.require_single_hash_query(
override_query override_query,
) if override_query else None "Invalid -query value (expected hash:<sha256>)",
if override_query and not override_hash: log_file=sys.stderr,
log("Invalid -query value (expected hash:<sha256>)", file=sys.stderr) )
if not query_valid:
return 1 return 1
# Selection syntax (@...) is handled by the pipeline runner, not by this cmdlet. # Selection syntax (@...) is handled by the pipeline runner, not by this cmdlet.
@@ -242,11 +232,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
return 1 return 1
# Normalize result to a list for processing # Normalize result to a list for processing
items_to_process = [] items_to_process = sh.normalize_result_items(result)
if isinstance(result, list):
items_to_process = result
elif result:
items_to_process = [result]
# Process each item # Process each item
success_count = 0 success_count = 0
@@ -358,14 +344,7 @@ def _process_deletion(
) )
return False return False
resolved_hash = normalize_hash(file_hash) if file_hash else None resolved_hash = sh.resolve_hash_for_cmdlet(file_hash, path, None)
if not resolved_hash and path:
try:
from SYS.utils import sha256_file
resolved_hash = sha256_file(Path(path))
except Exception:
resolved_hash = None
if not resolved_hash: if not resolved_hash:
log( log(
@@ -376,7 +355,13 @@ def _process_deletion(
def _fetch_existing_tags() -> list[str]: def _fetch_existing_tags() -> list[str]:
try: try:
backend = Store(config, suppress_debug=True)[store_name] backend, _store_registry, _exc = sh.get_store_backend(
config,
store_name,
suppress_debug=True,
)
if backend is None:
return []
existing, _src = backend.get_tag(resolved_hash, config=config) existing, _src = backend.get_tag(resolved_hash, config=config)
return list(existing or []) return list(existing or [])
except Exception: except Exception:
@@ -403,7 +388,13 @@ def _process_deletion(
return False return False
try: try:
backend = Store(config, suppress_debug=True)[store_name] backend, _store_registry, exc = sh.get_store_backend(
config,
store_name,
suppress_debug=True,
)
if backend is None:
raise exc or KeyError(store_name)
ok = backend.delete_tag(resolved_hash, list(tags), config=config) ok = backend.delete_tag(resolved_hash, list(tags), config=config)
if ok: if ok:
preview = resolved_hash[:12] + ("" if len(resolved_hash) > 12 else "") preview = resolved_hash[:12] + ("" if len(resolved_hash) > 12 else "")

View File

@@ -4,6 +4,7 @@ from typing import Any, Dict, List, Sequence, Tuple
import sys import sys
from SYS import pipeline as ctx from SYS import pipeline as ctx
from . import _shared as sh
from ._shared import ( from ._shared import (
Cmdlet, Cmdlet,
CmdletArg, CmdletArg,
@@ -45,9 +46,11 @@ class Delete_Url(Cmdlet):
"""Delete URL from file via hash+store backend.""" """Delete URL from file via hash+store backend."""
parsed = parse_cmdlet_args(args, self) parsed = parse_cmdlet_args(args, self)
query_hash = sh.parse_single_hash_query(parsed.get("query")) query_hash, query_valid = sh.require_single_hash_query(
if parsed.get("query") and not query_hash: parsed.get("query"),
log("Error: -query must be of the form hash:<sha256>") "Error: -query must be of the form hash:<sha256>",
)
if not query_valid:
return 1 return 1
# Bulk input is common in pipelines; treat a list of PipeObjects as a batch. # Bulk input is common in pipelines; treat a list of PipeObjects as a batch.
@@ -105,77 +108,13 @@ class Delete_Url(Cmdlet):
try: try:
storage = Store(config) storage = Store(config)
def _remove_urls(existing: Any, remove: List[str]) -> Any:
# Preserve prior shape: keep str when 1 url, list when multiple.
current: List[str] = []
try:
if isinstance(existing, str):
current = [p.strip() for p in existing.split(",") if p.strip()]
elif isinstance(existing, (list, tuple)):
current = [str(u).strip() for u in existing if str(u).strip()]
except Exception:
current = []
remove_set = {u
for u in (remove or []) if u}
new_urls = [u for u in current if u not in remove_set]
if len(new_urls) == 1:
return new_urls[0]
return new_urls
def _set_item_url(item: Any, merged: Any) -> None:
try:
if isinstance(item, dict):
item["url"] = merged
return
if hasattr(item, "url"):
setattr(item, "url", merged)
except Exception:
return
store_override = parsed.get("store") store_override = parsed.get("store")
batch: Dict[str,
List[Tuple[str,
List[str]]]] = {}
pass_through: List[Any] = []
if results: if results:
for item in results: def _warn(message: str) -> None:
pass_through.append(item) ctx.print_if_visible(f"[delete-url] Warning: {message}", file=sys.stderr)
raw_hash = query_hash or get_field(item, "hash") def _resolve_item_urls(item: Any) -> List[str]:
raw_store = store_override or get_field(item, "store")
if not raw_hash or not raw_store:
ctx.print_if_visible(
"[delete-url] Warning: Item missing hash/store; skipping",
file=sys.stderr,
)
continue
normalized = normalize_hash(raw_hash)
if not normalized:
ctx.print_if_visible(
"[delete-url] Warning: Item has invalid hash; skipping",
file=sys.stderr
)
continue
store_text = str(raw_store).strip()
if not store_text:
ctx.print_if_visible(
"[delete-url] Warning: Item has empty store; skipping",
file=sys.stderr
)
continue
if not storage.is_available(store_text):
ctx.print_if_visible(
f"[delete-url] Warning: Store '{store_text}' not configured; skipping",
file=sys.stderr,
)
continue
# Determine which URLs to delete.
# - If user passed an explicit <url>, apply it to all items.
# - Otherwise, when piping url rows from get-url, delete the url(s) from each item.
item_urls = list(urls_from_cli) item_urls = list(urls_from_cli)
if not item_urls: if not item_urls:
item_urls = [ item_urls = [
@@ -184,41 +123,28 @@ class Delete_Url(Cmdlet):
) if str(u).strip() ) if str(u).strip()
] ]
if not item_urls: if not item_urls:
ctx.print_if_visible( _warn("Item has no url field; skipping")
"[delete-url] Warning: Item has no url field; skipping", return item_urls
file=sys.stderr
)
continue
batch.setdefault(store_text, []).append((normalized, item_urls)) batch, pass_through = sh.collect_store_hash_value_batch(
results,
store_registry=storage,
value_resolver=_resolve_item_urls,
override_hash=query_hash,
override_store=store_override,
on_warning=_warn,
)
for store_text, pairs in batch.items(): storage, batch_stats = sh.run_store_hash_value_batches(
try: config,
backend = storage[store_text] batch,
except Exception: bulk_method_name="delete_url_bulk",
continue single_method_name="delete_url",
store_registry=storage,
merged: Dict[str, )
List[str]] = {} for store_text, item_count, deleted_count in batch_stats:
for h, ulist in pairs:
merged.setdefault(h, [])
for u in ulist or []:
if u and u not in merged[h]:
merged[h].append(u)
bulk_pairs = [(h, merged[h]) for h in merged.keys()]
bulk_fn = getattr(backend, "delete_url_bulk", None)
if callable(bulk_fn):
bulk_fn(bulk_pairs, config=config)
else:
for h, ulist in bulk_pairs:
backend.delete_url(h, ulist, config=config)
deleted_count = 0
for _h, ulist in bulk_pairs:
deleted_count += len(ulist or [])
ctx.print_if_visible( ctx.print_if_visible(
f"✓ delete-url: {deleted_count} url(s) for {len(bulk_pairs)} item(s) in '{store_text}'", f"✓ delete-url: {deleted_count} url(s) for {item_count} item(s) in '{store_text}'",
file=sys.stderr, file=sys.stderr,
) )
@@ -234,7 +160,7 @@ class Delete_Url(Cmdlet):
get_field(item, "url") or get_field(item, "source_url") get_field(item, "url") or get_field(item, "source_url")
) if str(u).strip() ) if str(u).strip()
] ]
_set_item_url(item, _remove_urls(existing, list(remove_set))) sh.set_item_urls(item, sh.remove_urls(existing, list(remove_set)))
ctx.emit(item) ctx.emit(item)
return 0 return 0
@@ -249,7 +175,14 @@ class Delete_Url(Cmdlet):
log("Error: No URL provided") log("Error: No URL provided")
return 1 return 1
backend = storage[str(store_name)] backend, storage, exc = sh.get_store_backend(
config,
str(store_name),
store_registry=storage,
)
if backend is None:
log(f"Error: Storage backend '{store_name}' not configured")
return 1
backend.delete_url(str(file_hash), list(urls_from_cli), config=config) backend.delete_url(str(file_hash), list(urls_from_cli), config=config)
ctx.print_if_visible( ctx.print_if_visible(
f"✓ delete-url: {len(urls_from_cli)} url(s) removed", f"✓ delete-url: {len(urls_from_cli)} url(s) removed",
@@ -257,13 +190,10 @@ class Delete_Url(Cmdlet):
) )
if result is not None: if result is not None:
existing = get_field(result, "url") existing = get_field(result, "url")
_set_item_url(result, _remove_urls(existing, list(urls_from_cli))) sh.set_item_urls(result, sh.remove_urls(existing, list(urls_from_cli)))
ctx.emit(result) ctx.emit(result)
return 0 return 0
except KeyError:
log(f"Error: Storage backend '{store_name}' not configured")
return 1
except Exception as exc: except Exception as exc:
log(f"Error deleting URL: {exc}", file=sys.stderr) log(f"Error deleting URL: {exc}", file=sys.stderr)
return 1 return 1

View File

@@ -19,11 +19,17 @@ from contextlib import AbstractContextManager, nullcontext
from API.HTTP import _download_direct_file from API.HTTP import _download_direct_file
from SYS.models import DownloadError, DownloadOptions, DownloadMediaResult from SYS.models import DownloadError, DownloadOptions, DownloadMediaResult
from SYS.logger import log, debug, is_debug_enabled from SYS.logger import log, debug, is_debug_enabled
from SYS.payload_builders import build_file_result_payload, build_table_result_payload
from SYS.pipeline_progress import PipelineProgress from SYS.pipeline_progress import PipelineProgress
from SYS.result_table import Table from SYS.result_table import Table
from SYS.rich_display import stderr_console as get_stderr_console from SYS.rich_display import stderr_console as get_stderr_console
from SYS import pipeline as pipeline_context from SYS import pipeline as pipeline_context
from SYS.metadata import normalize_urls as normalize_url_list from SYS.metadata import normalize_urls as normalize_url_list
from SYS.selection_builder import (
extract_selection_fields,
extract_urls_from_selection_args,
selection_args_have_url,
)
from SYS.utils import sha256_file from SYS.utils import sha256_file
from tool.ytdlp import ( from tool.ytdlp import (
@@ -57,6 +63,7 @@ build_pipeline_preview = sh.build_pipeline_preview
# URI scheme prefixes owned by AllDebrid (magic-link and emoji shorthand). # URI scheme prefixes owned by AllDebrid (magic-link and emoji shorthand).
# Defined once here so every method in this file references the same constant. # Defined once here so every method in this file references the same constant.
_ALLDEBRID_PREFIXES: tuple[str, ...] = ("alldebrid:", "alldebrid🧲") _ALLDEBRID_PREFIXES: tuple[str, ...] = ("alldebrid:", "alldebrid🧲")
_FORMAT_INDEX_RE = re.compile(r"^\s*#?\d+\s*$")
class Download_File(Cmdlet): class Download_File(Cmdlet):
@@ -1008,9 +1015,7 @@ class Download_File(Cmdlet):
formats_cache: Dict[str, Optional[List[Dict[str, Any]]]], formats_cache: Dict[str, Optional[List[Dict[str, Any]]]],
ytdlp_tool: YtDlpTool, ytdlp_tool: YtDlpTool,
) -> Optional[str]: ) -> Optional[str]:
import re if not query_format or not _FORMAT_INDEX_RE.match(str(query_format)):
if not query_format or not re.match(r"^\s*#?\d+\s*$", str(query_format)):
return None return None
try: try:
@@ -1221,22 +1226,24 @@ class Download_File(Cmdlet):
except Exception: except Exception:
pass pass
row: Dict[str, Any] = { row = build_table_result_payload(
"table": "download-file", table="download-file",
"title": str(title or f"Item {idx}"), title=str(title or f"Item {idx}"),
"detail": str(uploader or ""), detail=str(uploader or ""),
"media_kind": "playlist-item", columns=[
"playlist_index": idx,
"_selection_args": (["-url", str(entry_url)] if entry_url else ["-url", str(url), "-item", str(idx)]),
"url": entry_url,
"target": entry_url,
"columns": [
("#", str(idx)), ("#", str(idx)),
("Title", str(title or "")), ("Title", str(title or "")),
("Duration", str(duration or "")), ("Duration", str(duration or "")),
("Uploader", str(uploader or "")), ("Uploader", str(uploader or "")),
], ],
} selection_args=(
["-url", str(entry_url)] if entry_url else ["-url", str(url), "-item", str(idx)]
),
media_kind="playlist-item",
playlist_index=idx,
url=entry_url,
target=entry_url,
)
results_list.append(row) results_list.append(row)
table.add_result(row) table.add_result(row)
@@ -1782,14 +1789,11 @@ class Download_File(Cmdlet):
desc_parts.append(size_str) desc_parts.append(size_str)
format_desc = " | ".join(desc_parts) format_desc = " | ".join(desc_parts)
format_dict: Dict[str, Any] = { format_dict = build_table_result_payload(
"table": "download-file", table="download-file",
"title": f"Format {format_id}", title=f"Format {format_id}",
"url": url, detail=format_desc,
"target": url, columns=[
"detail": format_desc,
"media_kind": "format",
"columns": [
("ID", format_id), ("ID", format_id),
("Resolution", resolution or "N/A"), ("Resolution", resolution or "N/A"),
("Ext", ext), ("Ext", ext),
@@ -1797,13 +1801,16 @@ class Download_File(Cmdlet):
("Video", vcodec), ("Video", vcodec),
("Audio", acodec), ("Audio", acodec),
], ],
"full_metadata": { selection_args=["-query", f"format:{selection_format_id}"],
url=url,
target=url,
media_kind="format",
full_metadata={
"format_id": format_id, "format_id": format_id,
"url": url, "url": url,
"item_selector": selection_format_id, "item_selector": selection_format_id,
}, },
"_selection_args": ["-query", f"format:{selection_format_id}"], )
}
results_list.append(format_dict) results_list.append(format_dict)
table.add_result(format_dict) table.add_result(format_dict)
@@ -2379,18 +2386,18 @@ class Download_File(Cmdlet):
if not final_url and url: if not final_url and url:
final_url = str(url) final_url = str(url)
return { return build_file_result_payload(
"path": str(media_path), title=title,
"hash": hash_value, path=str(media_path),
"title": title, hash_value=hash_value,
"url": final_url, url=final_url,
"tag": tag, tag=tag,
"action": "cmdlet:download-file", store=getattr(opts, "storage_name", None) or getattr(opts, "storage_location", None) or "PATH",
"is_temp": True, action="cmdlet:download-file",
"ytdl_format": getattr(opts, "ytdl_format", None), is_temp=True,
"store": getattr(opts, "storage_name", None) or getattr(opts, "storage_location", None) or "PATH", ytdl_format=getattr(opts, "ytdl_format", None),
"media_kind": "video" if opts.mode == "video" else "audio", media_kind="video" if opts.mode == "video" else "audio",
} )
@staticmethod @staticmethod
def download_streaming_url_as_pipe_objects( def download_streaming_url_as_pipe_objects(
@@ -2609,22 +2616,13 @@ class Download_File(Cmdlet):
return out return out
@staticmethod
def _normalize_hash_hex(value: Optional[str]) -> Optional[str]:
if not value or not isinstance(value, str):
return None
candidate = value.strip().lower()
if len(candidate) == 64 and all(c in "0123456789abcdef" for c in candidate):
return candidate
return None
@classmethod @classmethod
def _extract_hash_from_search_hit(cls, hit: Any) -> Optional[str]: def _extract_hash_from_search_hit(cls, hit: Any) -> Optional[str]:
if not isinstance(hit, dict): if not isinstance(hit, dict):
return None return None
for key in ("hash", "hash_hex", "file_hash", "hydrus_hash"): for key in ("hash", "hash_hex", "file_hash", "hydrus_hash"):
v = hit.get(key) v = hit.get(key)
normalized = cls._normalize_hash_hex(str(v) if v is not None else None) normalized = sh.normalize_hash(str(v) if v is not None else None)
if normalized: if normalized:
return normalized return normalized
return None return None
@@ -2717,10 +2715,10 @@ class Download_File(Cmdlet):
hashes: List[str] = [] hashes: List[str] = []
for po in pipe_objects: for po in pipe_objects:
h_val = cls._normalize_hash_hex(str(po.get("hash") or "")) h_val = sh.normalize_hash(str(po.get("hash") or ""))
hashes.append(h_val or "") hashes.append(h_val or "")
king_hash = cls._normalize_hash_hex(source_king_hash) if source_king_hash else None king_hash = sh.normalize_hash(source_king_hash) if source_king_hash else None
if not king_hash: if not king_hash:
king_hash = hashes[0] if hashes and hashes[0] else None king_hash = hashes[0] if hashes and hashes[0] else None
if not king_hash: if not king_hash:
@@ -2774,10 +2772,10 @@ class Download_File(Cmdlet):
# Fallback to piped items if no explicit URLs provided # Fallback to piped items if no explicit URLs provided
piped_items = [] piped_items = []
if not raw_url: if not raw_url:
if isinstance(result, list): piped_items = sh.normalize_result_items(
piped_items = list(result) result,
elif result is not None: include_falsey_single=True,
piped_items = [result] )
# Handle TABLE_AUTO_STAGES routing: if a piped item has _selection_args, # Handle TABLE_AUTO_STAGES routing: if a piped item has _selection_args,
# re-invoke download-file with those args instead of processing the PipeObject itself. # re-invoke download-file with those args instead of processing the PipeObject itself.
@@ -2785,68 +2783,18 @@ class Download_File(Cmdlet):
selection_runs: List[List[str]] = [] selection_runs: List[List[str]] = []
residual_items: List[Any] = [] residual_items: List[Any] = []
def _looks_like_url(value: Any) -> bool:
try:
s_val = str(value or "").strip().lower()
except Exception:
return False
return s_val.startswith(
("http://", "https://", "magnet:", "torrent:") + _ALLDEBRID_PREFIXES
)
def _extract_selection_args(item: Any) -> tuple[Optional[List[str]], Optional[str]]:
selection_args: Optional[List[str]] = None
item_url: Optional[str] = None
if isinstance(item, dict):
selection_args = item.get("_selection_args") or item.get("selection_args")
item_url = item.get("url") or item.get("path") or item.get("target")
md = item.get("metadata") or item.get("full_metadata")
if isinstance(md, dict):
selection_args = selection_args or md.get("_selection_args") or md.get("selection_args")
item_url = item_url or md.get("url") or md.get("source_url")
extra = item.get("extra")
if isinstance(extra, dict):
selection_args = selection_args or extra.get("_selection_args") or extra.get("selection_args")
item_url = item_url or extra.get("url") or extra.get("source_url")
else:
item_url = getattr(item, "url", None) or getattr(item, "path", None) or getattr(item, "target", None)
md = getattr(item, "metadata", None)
if isinstance(md, dict):
selection_args = md.get("_selection_args") or md.get("selection_args")
item_url = item_url or md.get("url") or md.get("source_url")
extra = getattr(item, "extra", None)
if isinstance(extra, dict):
selection_args = selection_args or extra.get("_selection_args") or extra.get("selection_args")
item_url = item_url or extra.get("url") or extra.get("source_url")
if isinstance(selection_args, (list, tuple)):
normalized_args = [str(arg) for arg in selection_args if arg is not None]
elif selection_args is not None:
normalized_args = [str(selection_args)]
else:
normalized_args = None
if item_url and not _looks_like_url(item_url):
item_url = None
return normalized_args, item_url
def _selection_args_have_url(args_list: Sequence[str]) -> bool:
for idx, arg in enumerate(args_list):
low = str(arg or "").strip().lower()
if low in {"-url", "--url"}:
return True
if _looks_like_url(arg):
return True
return False
for item in piped_items: for item in piped_items:
handled = False handled = False
try: try:
normalized_args, item_url = _extract_selection_args(item) normalized_args, _normalized_action, item_url = extract_selection_fields(
item,
extra_url_prefixes=_ALLDEBRID_PREFIXES,
)
if normalized_args: if normalized_args:
if _selection_args_have_url(normalized_args): if selection_args_have_url(
normalized_args,
extra_url_prefixes=_ALLDEBRID_PREFIXES,
):
selection_runs.append(list(normalized_args)) selection_runs.append(list(normalized_args))
handled = True handled = True
elif item_url: elif item_url:
@@ -2860,25 +2808,11 @@ class Download_File(Cmdlet):
if selection_runs: if selection_runs:
selection_urls: List[str] = [] selection_urls: List[str] = []
def _extract_urls_from_args(args_list: Sequence[str]) -> List[str]:
urls: List[str] = []
idx = 0
while idx < len(args_list):
token = str(args_list[idx] or "")
low = token.strip().lower()
if low in {"-url", "--url"} and idx + 1 < len(args_list):
candidate = str(args_list[idx + 1] or "").strip()
if _looks_like_url(candidate):
urls.append(candidate)
idx += 2
continue
if _looks_like_url(token):
urls.append(token.strip())
idx += 1
return urls
for run_args in selection_runs: for run_args in selection_runs:
for u in _extract_urls_from_args(run_args): for u in extract_urls_from_selection_args(
run_args,
extra_url_prefixes=_ALLDEBRID_PREFIXES,
):
if u not in selection_urls: if u not in selection_urls:
selection_urls.append(u) selection_urls.append(u)

View File

@@ -17,10 +17,11 @@ from urllib.request import pathname2url
from SYS import pipeline as ctx from SYS import pipeline as ctx
from . import _shared as sh from . import _shared as sh
from SYS.item_accessors import get_result_title
from SYS.logger import log, debug from SYS.logger import log, debug
from Store import Store
from SYS.config import resolve_output_dir from SYS.config import resolve_output_dir
from API.HTTP import _download_direct_file from API.HTTP import _download_direct_file
from SYS.payload_builders import build_file_result_payload
class Get_File(sh.Cmdlet): class Get_File(sh.Cmdlet):
@@ -56,9 +57,11 @@ class Get_File(sh.Cmdlet):
parsed = sh.parse_cmdlet_args(args, self) parsed = sh.parse_cmdlet_args(args, self)
debug(f"[get-file] parsed args: {parsed}") debug(f"[get-file] parsed args: {parsed}")
query_hash = sh.parse_single_hash_query(parsed.get("query")) query_hash, query_valid = sh.require_single_hash_query(
if parsed.get("query") and not query_hash: parsed.get("query"),
log("Error: -query must be of the form hash:<sha256>") "Error: -query must be of the form hash:<sha256>",
)
if not query_valid:
return 1 return 1
# Extract hash and store from result or args # Extract hash and store from result or args
@@ -87,21 +90,14 @@ class Get_File(sh.Cmdlet):
debug(f"[get-file] Getting storage backend: {store_name}") debug(f"[get-file] Getting storage backend: {store_name}")
# Prefer instantiating only the named backend to avoid initializing all configured backends backend, _store_registry, _exc = sh.get_preferred_store_backend(
try: config,
from Store.registry import get_backend_instance store_name,
backend = get_backend_instance(config, store_name, suppress_debug=True) suppress_debug=True,
except Exception: )
backend = None
if backend is None: if backend is None:
# Fallback to full registry when targeted instantiation fails log(f"Error: Storage backend '{store_name}' not found", file=sys.stderr)
try: return 1
store = Store(config)
backend = store[store_name]
except Exception:
log(f"Error: Storage backend '{store_name}' not found", file=sys.stderr)
return 1
debug(f"[get-file] Backend retrieved: {type(backend).__name__}") debug(f"[get-file] Backend retrieved: {type(backend).__name__}")
@@ -117,18 +113,8 @@ class Get_File(sh.Cmdlet):
def resolve_display_title() -> str: def resolve_display_title() -> str:
candidates = [ candidates = [
sh.get_field(result, get_result_title(result, "title", "name", "filename"),
"title"), get_result_title(metadata, "title", "name", "filename"),
sh.get_field(result,
"name"),
sh.get_field(result,
"filename"),
(metadata.get("title") if isinstance(metadata,
dict) else None),
(metadata.get("name") if isinstance(metadata,
dict) else None),
(metadata.get("filename") if isinstance(metadata,
dict) else None),
] ]
for candidate in candidates: for candidate in candidates:
if candidate is None: if candidate is None:
@@ -166,12 +152,12 @@ class Get_File(sh.Cmdlet):
debug(f"Opened in browser: {download_url}", file=sys.stderr) debug(f"Opened in browser: {download_url}", file=sys.stderr)
ctx.emit( ctx.emit(
{ build_file_result_payload(
"hash": file_hash, title=resolve_display_title() or "Opened",
"store": store_name, hash_value=file_hash,
"url": download_url, store=store_name,
"title": resolve_display_title() or "Opened", url=download_url,
} )
) )
return 0 return 0
@@ -227,12 +213,12 @@ class Get_File(sh.Cmdlet):
# Emit result for pipeline # Emit result for pipeline
ctx.emit( ctx.emit(
{ build_file_result_payload(
"hash": file_hash, title=filename,
"store": store_name, hash_value=file_hash,
"path": str(dest_path), store=store_name,
"title": filename, path=str(dest_path),
} )
) )
debug("[get-file] Completed successfully") debug("[get-file] Completed successfully")

View File

@@ -4,7 +4,9 @@ from typing import Any, Dict, Sequence, Optional
import json import json
import sys import sys
from SYS.item_accessors import get_extension_field, get_int_field
from SYS.logger import log from SYS.logger import log
from SYS.payload_builders import build_file_result_payload
from . import _shared as sh from . import _shared as sh
@@ -15,6 +17,7 @@ parse_cmdlet_args = sh.parse_cmdlet_args
get_field = sh.get_field get_field = sh.get_field
from SYS import pipeline as ctx from SYS import pipeline as ctx
from SYS.result_table import Table from SYS.result_table import Table
from SYS.result_table_helpers import add_row_columns
class Get_Metadata(Cmdlet): class Get_Metadata(Cmdlet):
@@ -176,22 +179,28 @@ class Get_Metadata(Cmdlet):
store or ""), store or ""),
] ]
return { payload = build_file_result_payload(
"title": title or path, title=title,
"path": path, fallback_title=path,
"store": store, path=path,
"mime": mime, url=url,
"ext": ext or "", hash_value=hash_value,
"size_bytes": size_int, store=store,
"duration_seconds": dur_int, tag=tag or [],
"pages": pages_int, ext=ext,
"imported_ts": imported_ts, size_bytes=size_int,
"imported": imported_label, columns=columns,
"hash": hash_value, )
"url": url, payload.update(
"tag": tag or [], {
"columns": columns, "mime": mime,
} "duration_seconds": dur_int,
"pages": pages_int,
"imported_ts": imported_ts,
"imported": imported_label,
}
)
return payload
@staticmethod @staticmethod
def _add_table_body_row(table: Table, row: Dict[str, Any]) -> None: def _add_table_body_row(table: Table, row: Dict[str, Any]) -> None:
@@ -213,16 +222,18 @@ class Get_Metadata(Cmdlet):
label, value = col label, value = col
lookup[str(label)] = value lookup[str(label)] = value
row_obj = table.add_row() columns_to_add = [
row_obj.add_column("Hash", lookup.get("Hash", "")) ("Hash", lookup.get("Hash", "")),
row_obj.add_column("MIME", lookup.get("MIME", "")) ("MIME", lookup.get("MIME", "")),
row_obj.add_column("Size(MB)", lookup.get("Size(MB)", "")) ("Size(MB)", lookup.get("Size(MB)", "")),
]
if "Duration(s)" in lookup: if "Duration(s)" in lookup:
row_obj.add_column("Duration(s)", lookup.get("Duration(s)", "")) columns_to_add.append(("Duration(s)", lookup.get("Duration(s)", "")))
elif "Pages" in lookup: elif "Pages" in lookup:
row_obj.add_column("Pages", lookup.get("Pages", "")) columns_to_add.append(("Pages", lookup.get("Pages", "")))
else: else:
row_obj.add_column("Duration(s)", "") columns_to_add.append(("Duration(s)", ""))
add_row_columns(table, columns_to_add)
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
"""Execute get-metadata cmdlet - retrieve and display file metadata. """Execute get-metadata cmdlet - retrieve and display file metadata.
@@ -247,9 +258,12 @@ class Get_Metadata(Cmdlet):
# Parse arguments # Parse arguments
parsed = parse_cmdlet_args(args, self) parsed = parse_cmdlet_args(args, self)
query_hash = sh.parse_single_hash_query(parsed.get("query")) query_hash, query_valid = sh.require_single_hash_query(
if parsed.get("query") and not query_hash: parsed.get("query"),
log('No hash available - use -query "hash:<sha256>"', file=sys.stderr) 'No hash available - use -query "hash:<sha256>"',
log_file=sys.stderr,
)
if not query_valid:
return 1 return 1
# Get hash and store from parsed args or result # Get hash and store from parsed args or result
@@ -266,21 +280,14 @@ class Get_Metadata(Cmdlet):
# Use storage backend to get metadata # Use storage backend to get metadata
try: try:
# Instantiate only the required backend when possible to avoid initializing all configured backends backend, _store_registry, _exc = sh.get_preferred_store_backend(
try: config,
from Store.registry import get_backend_instance storage_source,
backend = get_backend_instance(config, storage_source, suppress_debug=True) suppress_debug=True,
except Exception: )
backend = None
if backend is None: if backend is None:
try: log(f"Storage backend '{storage_source}' not found", file=sys.stderr)
from Store import Store return 1
storage = Store(config)
backend = storage[storage_source]
except Exception:
log(f"Storage backend '{storage_source}' not found", file=sys.stderr)
return 1
# Get metadata from backend # Get metadata from backend
metadata = backend.get_metadata(file_hash) metadata = backend.get_metadata(file_hash)
@@ -330,8 +337,8 @@ class Get_Metadata(Cmdlet):
# Extract metadata fields # Extract metadata fields
mime_type = metadata.get("mime") or metadata.get("ext", "") mime_type = metadata.get("mime") or metadata.get("ext", "")
file_ext = metadata.get("ext", "") # Extract file extension separately file_ext = get_extension_field(metadata, "ext", "extension")
file_size = metadata.get("size") file_size = get_int_field(metadata, "size", "size_bytes")
duration_seconds = metadata.get("duration") duration_seconds = metadata.get("duration")
if duration_seconds is None: if duration_seconds is None:
duration_seconds = metadata.get("duration_seconds") duration_seconds = metadata.get("duration_seconds")

View File

@@ -1,10 +1,13 @@
from __future__ import annotations from __future__ import annotations
from pathlib import Path from typing import Any, Dict, List, Sequence
from typing import Any, Dict, List, Optional, Sequence
import sys import sys
from SYS.logger import log from SYS.logger import log
from SYS.detail_view_helpers import create_detail_view, prepare_detail_metadata
from SYS.payload_builders import build_table_result_payload
from SYS.result_publication import publish_result_table
from SYS.result_table_helpers import add_row_columns
from SYS import pipeline as ctx from SYS import pipeline as ctx
from . import _shared as sh from . import _shared as sh
@@ -16,8 +19,6 @@ normalize_hash = sh.normalize_hash
parse_cmdlet_args = sh.parse_cmdlet_args parse_cmdlet_args = sh.parse_cmdlet_args
normalize_result_input = sh.normalize_result_input normalize_result_input = sh.normalize_result_input
should_show_help = sh.should_show_help should_show_help = sh.should_show_help
from Store import Store
from SYS.utils import sha256_file
class Get_Note(Cmdlet): class Get_Note(Cmdlet):
@@ -45,14 +46,6 @@ class Get_Note(Cmdlet):
pass pass
self.register() self.register()
def _resolve_hash(
self,
raw_hash: Optional[str],
raw_path: Optional[str],
override_hash: Optional[str],
) -> Optional[str]:
return sh.resolve_hash_for_cmdlet(raw_hash, raw_path, override_hash)
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
if should_show_help(args): if should_show_help(args):
log(f"Cmdlet: {self.name}\nSummary: {self.summary}\nUsage: {self.usage}") log(f"Cmdlet: {self.name}\nSummary: {self.summary}\nUsage: {self.usage}")
@@ -60,12 +53,12 @@ class Get_Note(Cmdlet):
parsed = parse_cmdlet_args(args, self) parsed = parse_cmdlet_args(args, self)
store_override = parsed.get("store") store_override = parsed.get("store")
query_hash = sh.parse_single_hash_query(parsed.get("query")) query_hash, query_valid = sh.require_single_hash_query(
if parsed.get("query") and not query_hash: parsed.get("query"),
log( "[get_note] Error: -query must be of the form hash:<sha256>",
"[get_note] Error: -query must be of the form hash:<sha256>", log_file=sys.stderr,
file=sys.stderr )
) if not query_valid:
return 1 return 1
results = normalize_result_input(result) results = normalize_result_input(result)
@@ -82,31 +75,32 @@ class Get_Note(Cmdlet):
) )
return 1 return 1
store_registry = Store(config) store_registry = None
any_notes = False any_notes = False
display_items: List[Dict[str, Any]] = [] display_items: List[Dict[str, Any]] = []
# We assume single subject for get-note detail view # We assume single subject for get-note detail view
main_res = results[0] main_res = results[0]
from SYS.result_table import ItemDetailView, extract_item_metadata metadata = prepare_detail_metadata(main_res)
metadata = extract_item_metadata(main_res)
note_table = ( note_table = create_detail_view(
ItemDetailView("Notes", item_metadata=metadata) "Notes",
.set_table("note") metadata,
.set_value_case("preserve") table_name="note",
._perseverance(True) source_command=("get-note", []),
) )
note_table.set_source_command("get-note", [])
for res in results: for res in results:
if not isinstance(res, dict): if not isinstance(res, dict):
continue continue
store_name = str(store_override or res.get("store") or "").strip() store_name, resolved_hash = sh.resolve_item_store_hash(
raw_hash = res.get("hash") res,
raw_path = res.get("path") override_store=str(store_override) if store_override else None,
override_hash=str(query_hash) if query_hash else None,
path_fields=("path",),
)
if not store_name: if not store_name:
log( log(
@@ -115,11 +109,6 @@ class Get_Note(Cmdlet):
) )
return 1 return 1
resolved_hash = self._resolve_hash(
raw_hash=str(raw_hash) if raw_hash else None,
raw_path=str(raw_path) if raw_path else None,
override_hash=str(query_hash) if query_hash else None,
)
if not resolved_hash: if not resolved_hash:
continue continue
@@ -129,9 +118,12 @@ class Get_Note(Cmdlet):
if store_name and not metadata.get("Store"): if store_name and not metadata.get("Store"):
metadata["Store"] = store_name metadata["Store"] = store_name
try: backend, store_registry, exc = sh.get_store_backend(
backend = store_registry[store_name] config,
except Exception as exc: store_name,
store_registry=store_registry,
)
if backend is None:
log( log(
f"[get_note] Error: Unknown store '{store_name}': {exc}", f"[get_note] Error: Unknown store '{store_name}': {exc}",
file=sys.stderr file=sys.stderr
@@ -158,28 +150,27 @@ class Get_Note(Cmdlet):
# Keep payload small for IPC/pipes. # Keep payload small for IPC/pipes.
raw_text = raw_text[:999] raw_text = raw_text[:999]
preview = " ".join(raw_text.replace("\r", "").split("\n")) preview = " ".join(raw_text.replace("\r", "").split("\n"))
payload: Dict[str, Any] = { payload = build_table_result_payload(
"store": store_name, columns=[
"hash": resolved_hash, ("Name", str(k)),
"note_name": str(k), ("Text", preview.strip()),
"note_text": raw_text,
"columns": [
("Name",
str(k)),
("Text",
preview.strip()),
], ],
} store=store_name,
hash=resolved_hash,
note_name=str(k),
note_text=raw_text,
)
display_items.append(payload) display_items.append(payload)
if note_table is not None: if note_table is not None:
row = note_table.add_row() add_row_columns(
row.add_column("Name", str(k)) note_table,
row.add_column("Text", preview.strip()) [("Name", str(k)), ("Text", preview.strip())],
)
ctx.emit(payload) ctx.emit(payload)
# Always set the table overlay even if empty to show item details # Always set the table overlay even if empty to show item details
ctx.set_last_result_table_overlay(note_table, display_items, subject=result) publish_result_table(ctx, note_table, display_items, subject=result, overlay=True)
if not any_notes: if not any_notes:
log("No notes found.") log("No notes found.")

View File

@@ -3,7 +3,11 @@ from __future__ import annotations
from typing import Any, Dict, Sequence, Optional from typing import Any, Dict, Sequence, Optional
import sys import sys
from SYS.detail_view_helpers import create_detail_view, prepare_detail_metadata
from SYS.logger import log from SYS.logger import log
from SYS.result_table_helpers import add_row_columns
from SYS.selection_builder import build_hash_store_selection
from SYS.result_publication import publish_result_table
from SYS import pipeline as ctx from SYS import pipeline as ctx
from API import HydrusNetwork as hydrus_wrapper from API import HydrusNetwork as hydrus_wrapper
@@ -59,11 +63,12 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
continue continue
i += 1 i += 1
override_hash: str | None = ( override_hash, query_valid = sh.require_single_hash_query(
sh.parse_single_hash_query(override_query) if override_query else None override_query,
'get-relationship requires -query "hash:<sha256>"',
log_file=sys.stderr,
) )
if override_query and not override_hash: if not query_valid:
log('get-relationship requires -query "hash:<sha256>"', file=sys.stderr)
return 1 return 1
# Handle @N selection which creates a list # Handle @N selection which creates a list
@@ -326,21 +331,19 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
log(f"Hydrus relationships fetch failed: {exc}", file=sys.stderr) log(f"Hydrus relationships fetch failed: {exc}", file=sys.stderr)
# Display results # Display results
from SYS.result_table import ItemDetailView, extract_item_metadata metadata = prepare_detail_metadata(
result,
# Prepare metadata for the detail view title=(source_title if source_title and source_title != "Unknown" else None),
metadata = extract_item_metadata(result) hash_value=hash_hex,
)
if hash_hex:
metadata["Hash"] = hash_hex
# Overlays
if source_title and source_title != "Unknown":
metadata["Title"] = source_title
table = ItemDetailView("Relationships", item_metadata=metadata table = create_detail_view(
).init_command("get-relationship", "Relationships",
[]) metadata,
init_command=("get-relationship", []),
value_case=None,
perseverance=False,
)
# Sort by type then title # Sort by type then title
# Custom sort order: King first, then Derivative, then others # Custom sort order: King first, then Derivative, then others
@@ -364,11 +367,14 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
pipeline_results = [] pipeline_results = []
for i, item in enumerate(found_relationships): for i, item in enumerate(found_relationships):
row = table.add_row() add_row_columns(
row.add_column("Type", item["type"].title()) table,
row.add_column("Title", item["title"]) [
# row.add_column("Hash", item['hash'][:16] + "...") # User requested removal ("Type", item["type"].title()),
row.add_column("Store", item["store"]) ("Title", item["title"]),
("Store", item["store"]),
],
)
# Create result object for pipeline # Create result object for pipeline
res_obj = { res_obj = {
@@ -384,16 +390,15 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
pipeline_results.append(res_obj) pipeline_results.append(res_obj)
# Set selection args # Set selection args
table.set_row_selection_args( selection_args, _selection_action = build_hash_store_selection(
i, item["hash"],
["-store", item["store"],
str(item["store"]),
"-query",
f"hash:{item['hash']}"]
) )
if selection_args:
table.set_row_selection_args(i, selection_args)
# Ensure empty state is still navigable/visible # Ensure empty state is still navigable/visible
ctx.set_last_result_table_overlay(table, pipeline_results) publish_result_table(ctx, table, pipeline_results, overlay=True)
from SYS.rich_display import stdout_console from SYS.rich_display import stdout_console
stdout_console().print(table) stdout_console().print(table)

View File

@@ -26,6 +26,10 @@ from typing import Any, Dict, List, Optional, Sequence, Tuple
from SYS import pipeline as ctx from SYS import pipeline as ctx
from SYS.pipeline_progress import PipelineProgress from SYS.pipeline_progress import PipelineProgress
from SYS.detail_view_helpers import create_detail_view, prepare_detail_metadata
from SYS.payload_builders import extract_title_tag_value
from SYS.result_publication import publish_result_table
from SYS.result_table_helpers import add_row_columns
from . import _shared as sh from . import _shared as sh
from SYS.field_access import get_field from SYS.field_access import get_field
@@ -259,36 +263,24 @@ def _emit_tags_as_table(
subject: Full context object (should preserve original metadata) subject: Full context object (should preserve original metadata)
quiet: If True, don't display (emit-only mode) quiet: If True, don't display (emit-only mode)
""" """
from SYS.result_table import ItemDetailView, extract_item_metadata metadata = prepare_detail_metadata(
subject,
# Prepare metadata for the detail view, extracting all fields from subject first include_subject_fields=True,
metadata = extract_item_metadata(subject) or {} title=item_title,
hash_value=file_hash,
# Preserve all additional fields from subject dict if it's a dict-like object store=(service_name if service_name else store),
if isinstance(subject, dict): path=path,
for key, value in subject.items(): )
# Skip internal/control fields
if not key.startswith("_") and key not in {"selection_action", "selection_args"}:
# Convert keys to readable labels (snake_case -> Title Case)
label = str(key).replace("_", " ").title()
# Only add if not already present from extract_item_metadata
if label not in metadata and value is not None:
metadata[label] = value
# Apply explicit parameter overrides (these take priority)
if item_title:
metadata["Title"] = item_title
if file_hash:
metadata["Hash"] = file_hash
if store:
metadata["Store"] = service_name if service_name else store
if path:
metadata["Path"] = path
# Create ItemDetailView with exclude_tags=True so the panel shows file info # Create ItemDetailView with exclude_tags=True so the panel shows file info
# but doesn't duplicate the tag list that we show as a table below. # but doesn't duplicate the tag list that we show as a table below.
table = ItemDetailView("Tags", item_metadata=metadata, max_columns=1, exclude_tags=True) table = create_detail_view(
table.set_source_command("get-tag", []) "Tags",
metadata,
max_columns=1,
exclude_tags=True,
source_command=("get-tag", []),
)
# Create TagItem for each tag and add to table # Create TagItem for each tag and add to table
tag_items = [] tag_items = []
@@ -383,12 +375,7 @@ def _extract_title_from(tags_list: List[str]) -> Optional[str]:
return extract_title(tags_list) return extract_title(tags_list)
except Exception: except Exception:
pass pass
for t in tags_list: return extract_title_tag_value(tags_list)
if isinstance(t, str) and t.lower().startswith("title:"):
val = t.split(":", 1)[1].strip()
if val:
return val
return None
def _rename_file_if_title_tag(media: Optional[Path], tags_added: List[str]) -> bool: def _rename_file_if_title_tag(media: Optional[Path], tags_added: List[str]) -> bool:
@@ -1002,9 +989,12 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
# Extract values # Extract values
query_raw = parsed_args.get("query") query_raw = parsed_args.get("query")
hash_override = sh.parse_single_hash_query(query_raw) hash_override, query_valid = sh.require_single_hash_query(
if query_raw and not hash_override: query_raw,
log("Invalid -query value (expected hash:<sha256>)", file=sys.stderr) "Invalid -query value (expected hash:<sha256>)",
log_file=sys.stderr,
)
if not query_valid:
return 1 return 1
store_key = parsed_args.get("store") store_key = parsed_args.get("store")
emit_requested = parsed_args.get("emit", False) emit_requested = parsed_args.get("emit", False)
@@ -1023,25 +1013,16 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
except Exception: except Exception:
display_subject = None display_subject = None
def _value_has_content(value: Any) -> bool:
if value is None:
return False
if isinstance(value, str):
return bool(value.strip())
if isinstance(value, (list, tuple, set)):
return len(value) > 0
return True
def _resolve_subject_value(*keys: str) -> Any: def _resolve_subject_value(*keys: str) -> Any:
for key in keys: for key in keys:
val = get_field(result, key, None) val = get_field(result, key, None)
if _value_has_content(val): if sh.value_has_content(val):
return val return val
if display_subject is None: if display_subject is None:
return None return None
for key in keys: for key in keys:
val = get_field(display_subject, key, None) val = get_field(display_subject, key, None)
if _value_has_content(val): if sh.value_has_content(val):
return val return val
return None return None
@@ -1422,11 +1403,15 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
) )
for idx, item in enumerate(items): for idx, item in enumerate(items):
tags = _filter_scraped_tags(provider.to_tags(item)) tags = _filter_scraped_tags(provider.to_tags(item))
row = table.add_row() add_row_columns(
row.add_column("Title", item.get("title", "")) table,
row.add_column("Artist", item.get("artist", "")) [
row.add_column("Album", item.get("album", "")) ("Title", item.get("title", "")),
row.add_column("Year", item.get("year", "")) ("Artist", item.get("artist", "")),
("Album", item.get("album", "")),
("Year", item.get("year", "")),
],
)
payload = { payload = {
"tag": tags, "tag": tags,
"provider": provider.name, "provider": provider.name,
@@ -1447,7 +1432,7 @@ def _run_impl(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
# Store an overlay so that a subsequent `@N` selects from THIS metadata table, # Store an overlay so that a subsequent `@N` selects from THIS metadata table,
# not from the previous searchable table. # not from the previous searchable table.
ctx.set_last_result_table_overlay(table, selection_payload) publish_result_table(ctx, table, selection_payload, overlay=True)
ctx.set_current_stage_table(table) ctx.set_current_stage_table(table)
return 0 return 0

View File

@@ -16,7 +16,11 @@ from ._shared import (
normalize_hash, normalize_hash,
) )
from . import _shared as sh from . import _shared as sh
from SYS.detail_view_helpers import create_detail_view, prepare_detail_metadata
from SYS.item_accessors import get_extension_field, get_int_field, get_result_title
from SYS.logger import log from SYS.logger import log
from SYS.payload_builders import build_file_result_payload
from SYS.result_publication import publish_result_table
from SYS.result_table import Table from SYS.result_table import Table
from Store import Store from Store import Store
from SYS import pipeline as ctx from SYS import pipeline as ctx
@@ -221,52 +225,15 @@ class Get_Url(Cmdlet):
@staticmethod @staticmethod
def _extract_title_from_result(result: Any) -> Optional[str]: def _extract_title_from_result(result: Any) -> Optional[str]:
# Prefer explicit title field. return get_result_title(result, "title", "name", "filename")
# Fall back to ResultTable-style columns list.
cols = None
if isinstance(result, dict):
cols = result.get("columns")
else:
cols = getattr(result, "columns", None)
if isinstance(cols, list):
for pair in cols:
try:
if isinstance(pair, (list, tuple)) and len(pair) == 2:
k, v = pair
if str(k or "").strip().lower() in {"title", "name"}:
if isinstance(v, str) and v.strip():
return v.strip()
except Exception:
continue
return None
@staticmethod @staticmethod
def _extract_size_from_hit(hit: Any) -> int | None: def _extract_size_from_hit(hit: Any) -> int | None:
for key in ("size", "file_size", "filesize", "size_bytes"): return get_int_field(hit, "size", "file_size", "filesize", "size_bytes")
try:
val = get_field(hit, key)
except Exception:
val = None
if val is None:
continue
if isinstance(val, (int, float)):
return int(val)
try:
return int(val)
except Exception:
continue
return None
@staticmethod @staticmethod
def _extract_ext_from_hit(hit: Any) -> str: def _extract_ext_from_hit(hit: Any) -> str:
for key in ("ext", "extension"): return get_extension_field(hit, "ext", "extension")
try:
ext_val = get_field(hit, key)
except Exception:
ext_val = None
if isinstance(ext_val, str) and ext_val.strip():
return ext_val.strip().lstrip(".")
return ""
def _search_urls_across_stores(self, def _search_urls_across_stores(self,
pattern: str, pattern: str,
@@ -488,27 +455,25 @@ class Get_Url(Cmdlet):
table.set_source_command("get-url", ["-url", search_pattern]) table.set_source_command("get-url", ["-url", search_pattern])
for item in items: for item in items:
payload: Dict[str, Any] = { payload = build_file_result_payload(
# Keep fields for downstream cmdlets. title=item.title,
"hash": item.hash, hash_value=item.hash,
"store": item.store, store=item.store,
"url": item.url, url=item.url,
"title": item.title, ext=item.ext,
"size": item.size, columns=[
"ext": item.ext,
# Force the visible table columns + ordering.
"columns": [
("Title", item.title), ("Title", item.title),
("Url", item.url), ("Url", item.url),
("Size", item.size), ("Size", item.size),
("Ext", item.ext), ("Ext", item.ext),
("Store", item.store), ("Store", item.store),
], ],
} size=item.size,
)
display_items.append(payload) display_items.append(payload)
table.add_result(payload) table.add_result(payload)
ctx.set_last_result_table(table if display_items else None, display_items, subject=result) publish_result_table(ctx, table if display_items else None, display_items, subject=result)
# Emit after table state is finalized to prevent side effects in TUI rendering # Emit after table state is finalized to prevent side effects in TUI rendering
for d in display_items: for d in display_items:
@@ -520,9 +485,11 @@ class Get_Url(Cmdlet):
return 0 return 0
# Original mode: Get URLs for a specific file by hash+store # Original mode: Get URLs for a specific file by hash+store
query_hash = sh.parse_single_hash_query(parsed.get("query")) query_hash, query_valid = sh.require_single_hash_query(
if parsed.get("query") and not query_hash: parsed.get("query"),
log("Error: -query must be of the form hash:<sha256>") "Error: -query must be of the form hash:<sha256>",
)
if not query_valid:
return 1 return 1
# Extract hash and store from result or args # Extract hash and store from result or args
@@ -550,10 +517,9 @@ class Get_Url(Cmdlet):
from SYS.metadata import normalize_urls from SYS.metadata import normalize_urls
urls = normalize_urls(urls) urls = normalize_urls(urls)
from SYS.result_table import ItemDetailView, extract_item_metadata
# Prepare metadata for the detail view # Prepare metadata for the detail view
metadata = extract_item_metadata(result) metadata = prepare_detail_metadata(result)
tag_values = None
# Enrich the metadata with tags if missing # Enrich the metadata with tags if missing
if not metadata.get("Tags"): if not metadata.get("Tags"):
@@ -577,24 +543,24 @@ class Get_Url(Cmdlet):
pass pass
if row_tags: if row_tags:
row_tags = sorted(list(set(row_tags))) tag_values = sorted(list(set(row_tags)))
metadata["Tags"] = ", ".join(row_tags)
except Exception: except Exception:
pass pass
if file_hash: metadata = prepare_detail_metadata(
metadata["Hash"] = file_hash result,
if store_name: hash_value=file_hash,
metadata["Store"] = store_name store=store_name,
tags=tag_values,
table = ( )
ItemDetailView(
"Urls", table = create_detail_view(
item_metadata=metadata, "Urls",
max_columns=1 metadata,
)._perseverance(True).set_table("url").set_value_case("preserve") max_columns=1,
table_name="url",
source_command=("get-url", []),
) )
table.set_source_command("get-url", [])
items: List[UrlItem] = [] items: List[UrlItem] = []
for u in list(urls or []): for u in list(urls or []):
@@ -609,7 +575,7 @@ class Get_Url(Cmdlet):
# Use overlay mode to avoid "merging" with the previous status/table state. # Use overlay mode to avoid "merging" with the previous status/table state.
# This is idiomatic for detail views and prevents the search table from being # This is idiomatic for detail views and prevents the search table from being
# contaminated by partial re-renders. # contaminated by partial re-renders.
ctx.set_last_result_table_overlay(table, items, subject=result) publish_result_table(ctx, table, items, subject=result, overlay=True)
# Emit items at the end for pipeline continuity # Emit items at the end for pipeline continuity
for item in items: for item in items:

View File

@@ -28,6 +28,9 @@ should_show_help = sh.should_show_help
from SYS import pipeline as ctx from SYS import pipeline as ctx
_CHAPTER_TITLE_SPLIT_RE = _re.compile(r"^(?P<prefix>.+?)\s+-\s+(?P<chapter>.+)$")
_FFMPEG_TIME_RE = _re.compile(r"time=(\d{2}):(\d{2}):(\d{2})\.(\d{2})")
try: try:
from pypdf import PdfWriter, PdfReader from pypdf import PdfWriter, PdfReader
@@ -611,13 +614,12 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool:
# "Book Name - Chapter" # "Book Name - Chapter"
# If *all* titles share the same "Book Name" prefix, strip it. # If *all* titles share the same "Book Name" prefix, strip it.
if len(chapters) >= 2: if len(chapters) >= 2:
split_re = _re.compile(r"^(?P<prefix>.+?)\s+-\s+(?P<chapter>.+)$")
prefixes: List[str] = [] prefixes: List[str] = []
stripped_titles: List[str] = [] stripped_titles: List[str] = []
all_match = True all_match = True
for ch in chapters: for ch in chapters:
raw_title = str(ch.get("title") or "").strip() raw_title = str(ch.get("title") or "").strip()
m = split_re.match(raw_title) m = _CHAPTER_TITLE_SPLIT_RE.match(raw_title)
if not m: if not m:
all_match = False all_match = False
break break
@@ -721,7 +723,6 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool:
) )
# Monitor progress # Monitor progress
duration_re = re.compile(r"time=(\d{2}):(\d{2}):(\d{2})\.(\d{2})")
total_duration_sec = current_time_ms / 1000.0 total_duration_sec = current_time_ms / 1000.0
while True: while True:
@@ -733,7 +734,7 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool:
if line: if line:
# Parse time=HH:MM:SS.mm # Parse time=HH:MM:SS.mm
match = duration_re.search(line) match = _FFMPEG_TIME_RE.search(line)
if match and total_duration_sec > 0: if match and total_duration_sec > 0:
h, m, s, cs = map(int, match.groups()) h, m, s, cs = map(int, match.groups())
current_sec = h * 3600 + m * 60 + s + cs / 100.0 current_sec = h * 3600 + m * 60 + s + cs / 100.0

View File

@@ -18,6 +18,7 @@ from typing import Any, Dict, List, Optional, Sequence, Tuple
from urllib.parse import urlsplit, quote, urljoin, unquote from urllib.parse import urlsplit, quote, urljoin, unquote
from SYS.logger import log, debug, is_debug_enabled from SYS.logger import log, debug, is_debug_enabled
from SYS.item_accessors import extract_item_tags, get_result_title
from API.HTTP import HTTPClient from API.HTTP import HTTPClient
from SYS.pipeline_progress import PipelineProgress from SYS.pipeline_progress import PipelineProgress
from SYS.utils import ensure_directory, unique_path, unique_preserve_order from SYS.utils import ensure_directory, unique_path, unique_preserve_order
@@ -1005,26 +1006,10 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
# ======================================================================== # ========================================================================
def _extract_item_tags(item: Any) -> List[str]: def _extract_item_tags(item: Any) -> List[str]:
if item is None: return extract_item_tags(item)
return []
raw = get_field(item, "tag")
if isinstance(raw, list):
return [str(t) for t in raw if t is not None and str(t).strip()]
if isinstance(raw, str) and raw.strip():
return [raw.strip()]
return []
def _extract_item_title(item: Any) -> str: def _extract_item_title(item: Any) -> str:
if item is None: return get_result_title(item, "title", "name", "filename") or ""
return ""
for key in ("title", "name", "filename"):
val = get_field(item, key)
if val is None:
continue
text = str(val).strip()
if text:
return text
return ""
def _clean_title(text: str) -> str: def _clean_title(text: str) -> str:
value = (text or "").strip() value = (text or "").strip()

View File

@@ -14,6 +14,7 @@ import time
from urllib.parse import urlparse, parse_qs, unquote, urljoin from urllib.parse import urlparse, parse_qs, unquote, urljoin
from SYS.logger import log, debug from SYS.logger import log, debug
from SYS.payload_builders import build_file_result_payload, normalize_file_extension
from ProviderCore.registry import get_search_provider, list_search_providers from ProviderCore.registry import get_search_provider, list_search_providers
from SYS.rich_display import ( from SYS.rich_display import (
show_provider_config_panel, show_provider_config_panel,
@@ -21,12 +22,16 @@ from SYS.rich_display import (
show_available_providers_panel, show_available_providers_panel,
) )
from SYS.database import insert_worker, update_worker, append_worker_stdout from SYS.database import insert_worker, update_worker, append_worker_stdout
from SYS.item_accessors import get_extension_field, get_int_field, get_result_title
from SYS.selection_builder import build_default_selection
from SYS.result_publication import publish_result_table
from ._shared import ( from ._shared import (
Cmdlet, Cmdlet,
CmdletArg, CmdletArg,
SharedArgs, SharedArgs,
get_field, get_field,
get_preferred_store_backend,
should_show_help, should_show_help,
normalize_hash, normalize_hash,
first_title_tag, first_title_tag,
@@ -34,6 +39,35 @@ from ._shared import (
) )
from SYS import pipeline as ctx from SYS import pipeline as ctx
_WHITESPACE_RE = re.compile(r"\s+")
_SITE_TOKEN_RE = re.compile(r"(?:^|\s)site:([^\s,]+)", flags=re.IGNORECASE)
_FILETYPE_TOKEN_RE = re.compile(
r"(?:^|\s)(?:ext|filetype|type):\.?([a-z0-9]{1,12})\b",
flags=re.IGNORECASE,
)
_SITE_REMOVE_RE = re.compile(r"(?:^|\s)site:[^\s,]+", flags=re.IGNORECASE)
_FILETYPE_REMOVE_RE = re.compile(
r"(?:^|\s)(?:ext|filetype|type):\.?[a-z0-9]{1,12}\b",
flags=re.IGNORECASE,
)
_SCHEME_PREFIX_RE = re.compile(r"^[a-z]+:")
_YAHOO_RU_RE = re.compile(r"/RU=([^/]+)/RK=", flags=re.IGNORECASE)
_HTML_TAG_RE = re.compile(r"<[^>]+>")
_DDG_RESULT_ANCHOR_RE = re.compile(
r'<a[^>]+class="[^"]*result__a[^"]*"[^>]+href="([^"]+)"[^>]*>(.*?)</a>',
flags=re.IGNORECASE | re.DOTALL,
)
_GENERIC_ANCHOR_RE = re.compile(
r'<a[^>]+href=["\']([^"\']+)["\'][^>]*>(.*?)</a>',
flags=re.IGNORECASE | re.DOTALL,
)
_BING_RESULT_ANCHOR_RE = re.compile(
r'<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>',
flags=re.IGNORECASE | re.DOTALL,
)
_STORE_FILTER_RE = re.compile(r"\bstore:([^\s,]+)", flags=re.IGNORECASE)
_STORE_FILTER_REMOVE_RE = re.compile(r"\s*[,]?\s*store:[^\s,]+", flags=re.IGNORECASE)
class _WorkerLogger: class _WorkerLogger:
def __init__(self, worker_id: str) -> None: def __init__(self, worker_id: str) -> None:
@@ -230,7 +264,7 @@ class search_file(Cmdlet):
@staticmethod @staticmethod
def _normalize_space(text: Any) -> str: def _normalize_space(text: Any) -> str:
return re.sub(r"\s+", " ", str(text or "")).strip() return _WHITESPACE_RE.sub(" ", str(text or "")).strip()
@classmethod @classmethod
def _build_web_search_plan( def _build_web_search_plan(
@@ -266,7 +300,7 @@ class search_file(Cmdlet):
site_token_to_strip = "" site_token_to_strip = ""
seed_url = "" seed_url = ""
site_match = re.search(r"(?:^|\s)site:([^\s,]+)", text, flags=re.IGNORECASE) site_match = _SITE_TOKEN_RE.search(text)
if site_match: if site_match:
site_host = cls._extract_site_host(site_match.group(1)) site_host = cls._extract_site_host(site_match.group(1))
seed_url = str(site_match.group(1) or "").strip() seed_url = str(site_match.group(1) or "").strip()
@@ -286,7 +320,7 @@ class search_file(Cmdlet):
lower_candidate = candidate.lower() lower_candidate = candidate.lower()
if lower_candidate.startswith(("ext:", "filetype:", "type:", "site:")): if lower_candidate.startswith(("ext:", "filetype:", "type:", "site:")):
continue continue
if re.match(r"^[a-z]+:", lower_candidate) and not lower_candidate.startswith( if _SCHEME_PREFIX_RE.match(lower_candidate) and not lower_candidate.startswith(
("http://", "https://") ("http://", "https://")
): ):
continue continue
@@ -299,11 +333,7 @@ class search_file(Cmdlet):
if not site_host: if not site_host:
return None return None
filetype_match = re.search( filetype_match = _FILETYPE_TOKEN_RE.search(text)
r"(?:^|\s)(?:ext|filetype|type):\.?([a-z0-9]{1,12})\b",
text,
flags=re.IGNORECASE,
)
filetype = cls._normalize_extension(filetype_match.group(1)) if filetype_match else "" filetype = cls._normalize_extension(filetype_match.group(1)) if filetype_match else ""
# Feature gate: trigger this web-search mode when filetype is present # Feature gate: trigger this web-search mode when filetype is present
@@ -313,13 +343,8 @@ class search_file(Cmdlet):
return None return None
residual = text residual = text
residual = re.sub(r"(?:^|\s)site:[^\s,]+", " ", residual, flags=re.IGNORECASE) residual = _SITE_REMOVE_RE.sub(" ", residual)
residual = re.sub( residual = _FILETYPE_REMOVE_RE.sub(" ", residual)
r"(?:^|\s)(?:ext|filetype|type):\.?[a-z0-9]{1,12}\b",
" ",
residual,
flags=re.IGNORECASE,
)
if site_from_positional and positional_args: if site_from_positional and positional_args:
first = str(positional_args[0] or "").strip() first = str(positional_args[0] or "").strip()
@@ -631,7 +656,7 @@ class search_file(Cmdlet):
# Yahoo result links often look like: # Yahoo result links often look like:
# https://r.search.yahoo.com/.../RU=<url-encoded-target>/RK=... # https://r.search.yahoo.com/.../RU=<url-encoded-target>/RK=...
ru_match = re.search(r"/RU=([^/]+)/RK=", raw_href, flags=re.IGNORECASE) ru_match = _YAHOO_RU_RE.search(raw_href)
if ru_match: if ru_match:
try: try:
return str(unquote(ru_match.group(1))).strip() return str(unquote(ru_match.group(1))).strip()
@@ -664,6 +689,75 @@ class search_file(Cmdlet):
return False return False
return host == target or host.endswith(f".{target}") return host == target or host.endswith(f".{target}")
@staticmethod
def _itertext_join(node: Any) -> str:
try:
return " ".join([str(text).strip() for text in node.itertext() if str(text).strip()])
except Exception:
return ""
@staticmethod
def _html_fragment_to_text(fragment: Any) -> str:
text = _HTML_TAG_RE.sub(" ", str(fragment or ""))
return html.unescape(text)
@classmethod
def _append_web_result(
cls,
items: List[Dict[str, str]],
seen_urls: set[str],
*,
site_host: str,
url_text: str,
title_text: str,
snippet_text: str,
) -> None:
url_clean = str(url_text or "").strip()
if not url_clean or not url_clean.startswith(("http://", "https://")):
return
if not cls._url_matches_site(url_clean, site_host):
return
if url_clean in seen_urls:
return
seen_urls.add(url_clean)
items.append(
{
"url": url_clean,
"title": cls._normalize_space(title_text) or url_clean,
"snippet": cls._normalize_space(snippet_text),
}
)
@classmethod
def _parse_web_results_with_fallback(
cls,
*,
html_text: str,
limit: int,
lxml_parser: Any,
regex_parser: Any,
fallback_when_empty: bool = False,
) -> List[Dict[str, str]]:
"""Run an lxml-based parser with an optional regex fallback."""
items: List[Dict[str, str]] = []
seen_urls: set[str] = set()
should_run_regex = False
try:
from lxml import html as lxml_html
doc = lxml_html.fromstring(html_text or "")
lxml_parser(doc, items, seen_urls)
should_run_regex = fallback_when_empty and not items
except Exception:
should_run_regex = True
if should_run_regex:
regex_parser(html_text or "", items, seen_urls)
return items[:limit]
@classmethod @classmethod
def _parse_duckduckgo_results( def _parse_duckduckgo_results(
cls, cls,
@@ -673,36 +767,7 @@ class search_file(Cmdlet):
limit: int, limit: int,
) -> List[Dict[str, str]]: ) -> List[Dict[str, str]]:
"""Parse DuckDuckGo HTML results into normalized rows.""" """Parse DuckDuckGo HTML results into normalized rows."""
items: List[Dict[str, str]] = [] def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
seen_urls: set[str] = set()
def _add_item(url_text: str, title_text: str, snippet_text: str) -> None:
url_clean = str(url_text or "").strip()
if not url_clean:
return
if not url_clean.startswith(("http://", "https://")):
return
if not cls._url_matches_site(url_clean, site_host):
return
if url_clean in seen_urls:
return
seen_urls.add(url_clean)
title_clean = cls._normalize_space(title_text)
snippet_clean = cls._normalize_space(snippet_text)
items.append(
{
"url": url_clean,
"title": title_clean or url_clean,
"snippet": snippet_clean,
}
)
# Preferred parser path (lxml is already a project dependency).
try:
from lxml import html as lxml_html
doc = lxml_html.fromstring(html_text or "")
result_nodes = doc.xpath("//div[contains(@class, 'result')]") result_nodes = doc.xpath("//div[contains(@class, 'result')]")
for node in result_nodes: for node in result_nodes:
@@ -712,40 +777,47 @@ class search_file(Cmdlet):
link = links[0] link = links[0]
href = cls._extract_duckduckgo_target_url(link.get("href")) href = cls._extract_duckduckgo_target_url(link.get("href"))
title = " ".join([str(t).strip() for t in link.itertext() if str(t).strip()]) title = cls._itertext_join(link)
snippet_nodes = node.xpath(".//*[contains(@class, 'result__snippet')]") snippet_nodes = node.xpath(".//*[contains(@class, 'result__snippet')]")
snippet = "" snippet = ""
if snippet_nodes: if snippet_nodes:
snippet = " ".join( snippet = cls._itertext_join(snippet_nodes[0])
[str(t).strip() for t in snippet_nodes[0].itertext() if str(t).strip()]
)
_add_item(href, title, snippet) cls._append_web_result(
items,
seen_urls,
site_host=site_host,
url_text=href,
title_text=title,
snippet_text=snippet,
)
if len(items) >= limit: if len(items) >= limit:
break break
except Exception:
# Fallback to regex parser below.
pass
if items: def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
return items[:limit] for match in _DDG_RESULT_ANCHOR_RE.finditer(raw_html):
href = cls._extract_duckduckgo_target_url(match.group(1))
title_html = match.group(2)
title = cls._html_fragment_to_text(title_html)
cls._append_web_result(
items,
seen_urls,
site_host=site_host,
url_text=href,
title_text=title,
snippet_text="",
)
if len(items) >= limit:
break
# Regex fallback for environments where HTML parsing fails. return cls._parse_web_results_with_fallback(
anchor_pattern = re.compile( html_text=html_text,
r'<a[^>]+class="[^"]*result__a[^"]*"[^>]+href="([^"]+)"[^>]*>(.*?)</a>', limit=limit,
flags=re.IGNORECASE | re.DOTALL, lxml_parser=_parse_lxml,
regex_parser=_parse_regex,
fallback_when_empty=True,
) )
for match in anchor_pattern.finditer(html_text or ""):
href = cls._extract_duckduckgo_target_url(match.group(1))
title_html = match.group(2)
title = re.sub(r"<[^>]+>", " ", str(title_html or ""))
title = html.unescape(title)
_add_item(href, title, "")
if len(items) >= limit:
break
return items[:limit]
@classmethod @classmethod
def _parse_yahoo_results( def _parse_yahoo_results(
@@ -756,51 +828,43 @@ class search_file(Cmdlet):
limit: int, limit: int,
) -> List[Dict[str, str]]: ) -> List[Dict[str, str]]:
"""Parse Yahoo HTML search results into normalized rows.""" """Parse Yahoo HTML search results into normalized rows."""
items: List[Dict[str, str]] = [] def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
seen_urls: set[str] = set()
def _add_item(url_text: str, title_text: str, snippet_text: str) -> None:
url_clean = str(url_text or "").strip()
if not url_clean or not url_clean.startswith(("http://", "https://")):
return
if not cls._url_matches_site(url_clean, site_host):
return
if url_clean in seen_urls:
return
seen_urls.add(url_clean)
items.append(
{
"url": url_clean,
"title": cls._normalize_space(title_text) or url_clean,
"snippet": cls._normalize_space(snippet_text),
}
)
try:
from lxml import html as lxml_html
doc = lxml_html.fromstring(html_text or "")
for node in doc.xpath("//a[@href]"): for node in doc.xpath("//a[@href]"):
href = cls._extract_yahoo_target_url(node.get("href")) href = cls._extract_yahoo_target_url(node.get("href"))
title = " ".join([str(t).strip() for t in node.itertext() if str(t).strip()]) title = cls._itertext_join(node)
_add_item(href, title, "") cls._append_web_result(
if len(items) >= limit: items,
break seen_urls,
except Exception: site_host=site_host,
anchor_pattern = re.compile( url_text=href,
r'<a[^>]+href=["\']([^"\']+)["\'][^>]*>(.*?)</a>', title_text=title,
flags=re.IGNORECASE | re.DOTALL, snippet_text="",
) )
for match in anchor_pattern.finditer(html_text or ""):
href = cls._extract_yahoo_target_url(match.group(1))
title_html = match.group(2)
title = re.sub(r"<[^>]+>", " ", str(title_html or ""))
title = html.unescape(title)
_add_item(href, title, "")
if len(items) >= limit: if len(items) >= limit:
break break
return items[:limit] def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
for match in _GENERIC_ANCHOR_RE.finditer(raw_html):
href = cls._extract_yahoo_target_url(match.group(1))
title_html = match.group(2)
title = cls._html_fragment_to_text(title_html)
cls._append_web_result(
items,
seen_urls,
site_host=site_host,
url_text=href,
title_text=title,
snippet_text="",
)
if len(items) >= limit:
break
return cls._parse_web_results_with_fallback(
html_text=html_text,
limit=limit,
lxml_parser=_parse_lxml,
regex_parser=_parse_regex,
)
@classmethod @classmethod
def _query_yahoo( def _query_yahoo(
@@ -881,30 +945,7 @@ class search_file(Cmdlet):
limit: int, limit: int,
) -> List[Dict[str, str]]: ) -> List[Dict[str, str]]:
"""Parse Bing HTML search results into normalized rows.""" """Parse Bing HTML search results into normalized rows."""
items: List[Dict[str, str]] = [] def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
seen_urls: set[str] = set()
def _add_item(url_text: str, title_text: str, snippet_text: str) -> None:
url_clean = str(url_text or "").strip()
if not url_clean or not url_clean.startswith(("http://", "https://")):
return
if not cls._url_matches_site(url_clean, site_host):
return
if url_clean in seen_urls:
return
seen_urls.add(url_clean)
items.append(
{
"url": url_clean,
"title": cls._normalize_space(title_text) or url_clean,
"snippet": cls._normalize_space(snippet_text),
}
)
try:
from lxml import html as lxml_html
doc = lxml_html.fromstring(html_text or "")
result_nodes = doc.xpath("//li[contains(@class, 'b_algo')]") result_nodes = doc.xpath("//li[contains(@class, 'b_algo')]")
for node in result_nodes: for node in result_nodes:
@@ -913,7 +954,7 @@ class search_file(Cmdlet):
continue continue
link = links[0] link = links[0]
href = str(link.get("href") or "").strip() href = str(link.get("href") or "").strip()
title = " ".join([str(t).strip() for t in link.itertext() if str(t).strip()]) title = cls._itertext_join(link)
snippet = "" snippet = ""
for sel in ( for sel in (
@@ -923,28 +964,41 @@ class search_file(Cmdlet):
): ):
snip_nodes = node.xpath(sel) snip_nodes = node.xpath(sel)
if snip_nodes: if snip_nodes:
snippet = " ".join( snippet = cls._itertext_join(snip_nodes[0])
[str(t).strip() for t in snip_nodes[0].itertext() if str(t).strip()]
)
break break
_add_item(href, title, snippet) cls._append_web_result(
if len(items) >= limit: items,
break seen_urls,
except Exception: site_host=site_host,
anchor_pattern = re.compile( url_text=href,
r"<h2[^>]*>\s*<a[^>]+href=\"([^\"]+)\"[^>]*>(.*?)</a>", title_text=title,
flags=re.IGNORECASE | re.DOTALL, snippet_text=snippet,
) )
for match in anchor_pattern.finditer(html_text or ""):
href = match.group(1)
title = re.sub(r"<[^>]+>", " ", str(match.group(2) or ""))
title = html.unescape(title)
_add_item(href, title, "")
if len(items) >= limit: if len(items) >= limit:
break break
return items[:limit] def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
for match in _BING_RESULT_ANCHOR_RE.finditer(raw_html):
href = match.group(1)
title = cls._html_fragment_to_text(match.group(2))
cls._append_web_result(
items,
seen_urls,
site_host=site_host,
url_text=href,
title_text=title,
snippet_text="",
)
if len(items) >= limit:
break
return cls._parse_web_results_with_fallback(
html_text=html_text,
limit=limit,
lxml_parser=_parse_lxml,
regex_parser=_parse_regex,
)
@classmethod @classmethod
def _query_web_search( def _query_web_search(
@@ -1218,33 +1272,30 @@ class search_file(Cmdlet):
if file_name: if file_name:
title = file_name title = file_name
payload: Dict[str, Any] = { payload = build_file_result_payload(
"title": title, title=title,
"path": target_url, path=target_url,
"url": target_url, url=target_url,
"source": "web", source="web",
"store": "web", store="web",
"table": "web.search", table="web.search",
"ext": detected_ext, ext=detected_ext,
"detail": snippet, detail=snippet,
"tag": [f"site:{site_host}"] + ([f"type:{detected_ext}"] if detected_ext else []), tag=[f"site:{site_host}"] + ([f"type:{detected_ext}"] if detected_ext else []),
"columns": [ columns=[
("Title", title), ("Title", title),
("Type", detected_ext), ("Type", detected_ext),
("URL", target_url), ("URL", target_url),
], ],
"_selection_args": ["-url", target_url], _selection_args=["-url", target_url],
"_selection_action": ["download-file", "-url", target_url], _selection_action=["download-file", "-url", target_url],
} )
table.add_result(payload) table.add_result(payload)
results_list.append(payload) results_list.append(payload)
ctx.emit(payload) ctx.emit(payload)
if refresh_mode: publish_result_table(ctx, table, results_list, overlay=refresh_mode)
ctx.set_last_result_table_preserve_history(table, results_list)
else:
ctx.set_last_result_table(table, results_list)
ctx.set_current_stage_table(table) ctx.set_current_stage_table(table)
@@ -1267,15 +1318,7 @@ class search_file(Cmdlet):
@staticmethod @staticmethod
def _normalize_extension(ext_value: Any) -> str: def _normalize_extension(ext_value: Any) -> str:
"""Sanitize extension strings to alphanumerics and cap at 5 chars.""" """Sanitize extension strings to alphanumerics and cap at 5 chars."""
ext = str(ext_value or "").strip().lstrip(".") return normalize_file_extension(ext_value)
for sep in (" ", "|", "(", "[", "{", ",", ";"):
if sep in ext:
ext = ext.split(sep, 1)[0]
break
if "." in ext:
ext = ext.split(".")[-1]
ext = "".join(ch for ch in ext if ch.isalnum())
return ext[:5]
@staticmethod @staticmethod
def _normalize_lookup_target(value: Optional[str]) -> str: def _normalize_lookup_target(value: Optional[str]) -> str:
@@ -1580,10 +1623,7 @@ class search_file(Cmdlet):
results_list.append(item_dict) results_list.append(item_dict)
ctx.emit(item_dict) ctx.emit(item_dict)
if refresh_mode: publish_result_table(ctx, table, results_list, overlay=refresh_mode)
ctx.set_last_result_table_preserve_history(table, results_list)
else:
ctx.set_last_result_table(table, results_list)
ctx.set_current_stage_table(table) ctx.set_current_stage_table(table)
@@ -1764,11 +1804,11 @@ class search_file(Cmdlet):
store_filter: Optional[str] = None store_filter: Optional[str] = None
if query: if query:
match = re.search(r"\bstore:([^\s,]+)", query, flags=re.IGNORECASE) match = _STORE_FILTER_RE.search(query)
if match: if match:
store_filter = match.group(1).strip() or None store_filter = match.group(1).strip() or None
query = re.sub(r"\s*[,]?\s*store:[^\s,]+", " ", query, flags=re.IGNORECASE) query = _STORE_FILTER_REMOVE_RE.sub(" ", query)
query = re.sub(r"\s{2,}", " ", query) query = _WHITESPACE_RE.sub(" ", query)
query = query.strip().strip(",") query = query.strip().strip(",")
if store_filter and not storage_backend: if store_filter and not storage_backend:
@@ -1912,19 +1952,15 @@ class search_file(Cmdlet):
for h in hash_query: for h in hash_query:
resolved_backend_name: Optional[str] = None resolved_backend_name: Optional[str] = None
resolved_backend = None resolved_backend = None
store_registry = None
for backend_name in backends_to_try: for backend_name in backends_to_try:
backend = None backend, store_registry, _exc = get_preferred_store_backend(
try: config,
backend = get_backend_instance(config, backend_name, suppress_debug=True) backend_name,
if backend is None: store_registry=store_registry,
# Last-resort: instantiate full registry for this backend only suppress_debug=True,
from Store import Store as _Store )
_store = _Store(config=config, suppress_debug=True)
if _store.is_available(backend_name):
backend = _store[backend_name]
except Exception:
backend = None
if backend is None: if backend is None:
continue continue
try: try:
@@ -2017,16 +2053,14 @@ class search_file(Cmdlet):
except Exception: except Exception:
title_from_tag = None title_from_tag = None
title = title_from_tag or meta_obj.get("title") or meta_obj.get( title = title_from_tag or get_result_title(meta_obj, "title", "name")
"name"
)
if not title and path_str: if not title and path_str:
try: try:
title = Path(path_str).stem title = Path(path_str).stem
except Exception: except Exception:
title = path_str title = path_str
ext_val = meta_obj.get("ext") or meta_obj.get("extension") ext_val = get_extension_field(meta_obj, "ext", "extension")
if not ext_val and path_str: if not ext_val and path_str:
try: try:
ext_val = Path(path_str).suffix ext_val = Path(path_str).suffix
@@ -2038,27 +2072,19 @@ class search_file(Cmdlet):
except Exception: except Exception:
ext_val = None ext_val = None
size_bytes = meta_obj.get("size") size_bytes_int = get_int_field(meta_obj, "size", "size_bytes")
if size_bytes is None:
size_bytes = meta_obj.get("size_bytes")
try:
size_bytes_int: Optional[int] = (
int(size_bytes) if size_bytes is not None else None
)
except Exception:
size_bytes_int = None
payload: Dict[str, payload = build_file_result_payload(
Any] = { title=title,
"title": str(title or h), fallback_title=h,
"hash": h, hash_value=h,
"store": resolved_backend_name, store=resolved_backend_name,
"path": path_str, path=path_str,
"ext": self._normalize_extension(ext_val), ext=ext_val,
"size_bytes": size_bytes_int, size_bytes=size_bytes_int,
"tag": tags_list, tag=tags_list,
"url": meta_obj.get("url") or [], url=meta_obj.get("url") or [],
} )
self._set_storage_display_columns(payload) self._set_storage_display_columns(payload)
@@ -2106,16 +2132,20 @@ class search_file(Cmdlet):
if backend_to_search: if backend_to_search:
searched_backends.append(backend_to_search) searched_backends.append(backend_to_search)
target_backend, _store_registry, exc = get_preferred_store_backend(
config,
backend_to_search,
suppress_debug=True,
)
if target_backend is None:
if exc is not None:
log(f"Backend '{backend_to_search}' not found: {exc}", file=sys.stderr)
db.update_worker_status(worker_id, "error")
return 1
debug(f"[search-file] Requested backend '{backend_to_search}' not found")
return 1
try: try:
target_backend = get_backend_instance(config, backend_to_search, suppress_debug=True) pass
if target_backend is None:
from Store import Store as _Store
_store = _Store(config=config, suppress_debug=True)
if _store.is_available(backend_to_search):
target_backend = _store[backend_to_search]
else:
debug(f"[search-file] Requested backend '{backend_to_search}' not found")
return 1
except Exception as exc: except Exception as exc:
log(f"Backend '{backend_to_search}' not found: {exc}", file=sys.stderr) log(f"Backend '{backend_to_search}' not found: {exc}", file=sys.stderr)
db.update_worker_status(worker_id, "error") db.update_worker_status(worker_id, "error")
@@ -2135,18 +2165,19 @@ class search_file(Cmdlet):
) )
else: else:
all_results = [] all_results = []
store_registry = None
for backend_name in list_configured_backend_names(config or {}): for backend_name in list_configured_backend_names(config or {}):
try: try:
backend = get_backend_instance(config, backend_name, suppress_debug=True) backend, store_registry, _exc = get_preferred_store_backend(
config,
backend_name,
store_registry=store_registry,
suppress_debug=True,
)
if backend is None: if backend is None:
from Store import Store as _Store # Configured backend name exists but has no registered implementation or failed to load.
_store = _Store(config=config, suppress_debug=True) # (e.g. 'all-debrid' being treated as a store but having no store provider).
if _store.is_available(backend_name): continue
backend = _store[backend_name]
else:
# Configured backend name exists but has no registered implementation or failed to load.
# (e.g. 'all-debrid' being treated as a store but having no store provider).
continue
searched_backends.append(backend_name) searched_backends.append(backend_name)
@@ -2216,63 +2247,11 @@ class search_file(Cmdlet):
# Populate default selection args for interactive @N selection/hash/url handling # Populate default selection args for interactive @N selection/hash/url handling
try: try:
sel_args: Optional[List[str]] = None sel_args, sel_action = build_default_selection(
sel_action: Optional[List[str]] = None path_value=normalized.get("path") or normalized.get("target") or normalized.get("url"),
hash_value=normalized.get("hash") or normalized.get("file_hash") or normalized.get("hash_hex"),
# Prefer explicit path when available store_value=normalized.get("store"),
p_val = normalized.get("path") or normalized.get("target") or normalized.get("url") )
if p_val:
p_str = str(p_val or "").strip()
if p_str:
if p_str.startswith(("http://", "https://", "magnet:", "torrent:")):
h = normalized.get("hash") or normalized.get("file_hash") or normalized.get("hash_hex")
s_val = normalized.get("store")
if h and s_val and "/view_file" in p_str:
try:
h_norm = normalize_hash(h)
except Exception:
h_norm = str(h)
sel_args = ["-query", f"hash:{h_norm}", "-store", str(s_val)]
sel_action = ["get-metadata", "-query", f"hash:{h_norm}", "-store", str(s_val)]
else:
sel_args = ["-url", p_str]
sel_action = ["download-file", "-url", p_str]
else:
try:
from SYS.utils import expand_path
full_path = expand_path(p_str)
# Prefer showing metadata details when we have a hash+store context
h = normalized.get("hash") or normalized.get("file_hash") or normalized.get("hash_hex")
s_val = normalized.get("store")
if h and s_val:
try:
h_norm = normalize_hash(h)
except Exception:
h_norm = str(h)
sel_args = ["-query", f"hash:{h_norm}", "-store", str(s_val)]
sel_action = ["get-metadata", "-query", f"hash:{h_norm}", "-store", str(s_val)]
else:
sel_args = ["-path", str(full_path)]
# Default action for local paths: get-file to fetch or operate on the path
sel_action = ["get-file", "-path", str(full_path)]
except Exception:
sel_args = ["-path", p_str]
sel_action = ["get-file", "-path", p_str]
# Fallback: use hash+store when available
if sel_args is None:
h = normalized.get("hash") or normalized.get("file_hash") or normalized.get("hash_hex")
s_val = normalized.get("store")
if h and s_val:
try:
h_norm = normalize_hash(h)
except Exception:
h_norm = str(h)
sel_args = ["-query", f"hash:{h_norm}", "-store", str(s_val)]
# Show metadata details by default for store/hash selections
sel_action = ["get-metadata", "-query", f"hash:{h_norm}", "-store", str(s_val)]
if sel_args: if sel_args:
normalized["_selection_args"] = [str(x) for x in sel_args] normalized["_selection_args"] = [str(x) for x in sel_args]
if sel_action: if sel_action:
@@ -2305,11 +2284,17 @@ class search_file(Cmdlet):
subject_hash = query.split("hash:")[1].split(",")[0].strip() subject_hash = query.split("hash:")[1].split(",")[0].strip()
subject_context = {"store": backend_to_search, "hash": subject_hash} subject_context = {"store": backend_to_search, "hash": subject_hash}
ctx.set_last_result_table_overlay(table, results_list, subject=subject_context) publish_result_table(
ctx,
table,
results_list,
subject=subject_context,
overlay=True,
)
except Exception: except Exception:
ctx.set_last_result_table_preserve_history(table, results_list) publish_result_table(ctx, table, results_list, overlay=True)
else: else:
ctx.set_last_result_table(table, results_list) publish_result_table(ctx, table, results_list)
db.append_worker_stdout( db.append_worker_stdout(
worker_id, worker_id,
_summarize_worker_results(results_list) _summarize_worker_results(results_list)

View File

@@ -12,9 +12,9 @@ import time
from urllib.parse import urlparse from urllib.parse import urlparse
from SYS.logger import log, debug from SYS.logger import log, debug
from SYS.item_accessors import get_store_name
from SYS.utils import sha256_file from SYS.utils import sha256_file
from . import _shared as sh from . import _shared as sh
from Store import Store
Cmdlet = sh.Cmdlet Cmdlet = sh.Cmdlet
CmdletArg = sh.CmdletArg CmdletArg = sh.CmdletArg
@@ -153,12 +153,7 @@ def _sanitize_filename(name: str, *, max_len: int = 140) -> str:
def _extract_store_name(item: Any) -> Optional[str]: def _extract_store_name(item: Any) -> Optional[str]:
try: return get_store_name(item, "store")
store_val = get_field(item, "store")
s = str(store_val or "").strip()
return s if s else None
except Exception:
return None
def _persist_alt_relationship( def _persist_alt_relationship(
@@ -437,9 +432,11 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
if store_name: if store_name:
try: try:
store = Store(config) backend, _store_registry, _exc = sh.get_store_backend(
if store.is_available(store_name): config,
backend = store[str(store_name)] store_name,
)
if backend is not None:
stored_hash = backend.add_file( stored_hash = backend.add_file(
Path(str(output_path)), Path(str(output_path)),
title=new_title, title=new_title,

79
cmdnat/_parsing.py Normal file
View File

@@ -0,0 +1,79 @@
from __future__ import annotations
from typing import Any, Iterable, List, Optional, Sequence
VALUE_ARG_FLAGS = frozenset({"-value", "--value", "-set-value", "--set-value"})
def extract_piped_value(result: Any) -> Optional[str]:
if isinstance(result, str):
return result.strip() if result.strip() else None
if isinstance(result, (int, float)):
return str(result)
if isinstance(result, dict):
value = result.get("value")
if value is not None:
return str(value).strip()
return None
def extract_arg_value(
args: Sequence[str],
*,
flags: Iterable[str],
allow_positional: bool = False,
) -> Optional[str]:
if not args:
return None
tokens = [str(tok) for tok in args if tok is not None]
normalized_flags = {
str(flag).strip().lower() for flag in flags if str(flag).strip()
}
if not normalized_flags:
return None
for idx, tok in enumerate(tokens):
text = tok.strip()
if not text:
continue
low = text.lower()
if low in normalized_flags and idx + 1 < len(tokens):
candidate = str(tokens[idx + 1]).strip()
if candidate:
return candidate
if "=" in low:
head, value = low.split("=", 1)
if head in normalized_flags and value:
return value.strip()
if not allow_positional:
return None
for tok in tokens:
text = str(tok).strip()
if text and not text.startswith("-"):
return text
return None
def extract_value_arg(args: Sequence[str]) -> Optional[str]:
return extract_arg_value(args, flags=VALUE_ARG_FLAGS, allow_positional=True)
def has_flag(args: Sequence[str], flag: str) -> bool:
try:
want = str(flag or "").strip().lower()
if not want:
return False
return any(str(arg).strip().lower() == want for arg in (args or []))
except Exception:
return False
def normalize_to_list(value: Any) -> List[Any]:
if value is None:
return []
if isinstance(value, list):
return value
return [value]

112
cmdnat/_status_shared.py Normal file
View File

@@ -0,0 +1,112 @@
from __future__ import annotations
from typing import Any
import httpx
from SYS.result_table import Table
def upper_text(value: Any) -> str:
text = "" if value is None else str(value)
return text.upper()
def add_startup_check(
table: Table,
status: str,
name: str,
*,
provider: str = "",
store: str = "",
files: int | str | None = None,
detail: str = "",
) -> None:
row = table.add_row()
row.add_column("STATUS", upper_text(status))
row.add_column("NAME", upper_text(name))
row.add_column("PROVIDER", upper_text(provider or ""))
row.add_column("STORE", upper_text(store or ""))
row.add_column("FILES", "" if files is None else str(files))
row.add_column("DETAIL", upper_text(detail or ""))
def has_store_subtype(cfg: dict, subtype: str) -> bool:
store_cfg = cfg.get("store")
if not isinstance(store_cfg, dict):
return False
bucket = store_cfg.get(subtype)
if not isinstance(bucket, dict):
return False
return any(isinstance(value, dict) and bool(value) for value in bucket.values())
def has_provider(cfg: dict, name: str) -> bool:
provider_cfg = cfg.get("provider")
if not isinstance(provider_cfg, dict):
return False
block = provider_cfg.get(str(name).strip().lower())
return isinstance(block, dict) and bool(block)
def has_tool(cfg: dict, name: str) -> bool:
tool_cfg = cfg.get("tool")
if not isinstance(tool_cfg, dict):
return False
block = tool_cfg.get(str(name).strip().lower())
return isinstance(block, dict) and bool(block)
def ping_url(url: str, timeout: float = 3.0) -> tuple[bool, str]:
try:
from API.HTTP import HTTPClient
with HTTPClient(timeout=timeout, retries=1) as client:
response = client.get(url, allow_redirects=True)
code = int(getattr(response, "status_code", 0) or 0)
ok = 200 <= code < 500
return ok, f"{url} (HTTP {code})"
except httpx.TimeoutException:
return False, f"{url} (timeout)"
except Exception as exc:
return False, f"{url} ({type(exc).__name__})"
def provider_display_name(key: str) -> str:
label = (key or "").strip()
lower = label.lower()
if lower == "openlibrary":
return "OpenLibrary"
if lower == "alldebrid":
return "AllDebrid"
if lower == "youtube":
return "YouTube"
return label[:1].upper() + label[1:] if label else "Provider"
def default_provider_ping_targets(provider_key: str) -> list[str]:
provider = (provider_key or "").strip().lower()
if provider == "openlibrary":
return ["https://openlibrary.org"]
if provider == "youtube":
return ["https://www.youtube.com"]
if provider == "bandcamp":
return ["https://bandcamp.com"]
if provider == "libgen":
try:
from Provider.libgen import MIRRORS
return [str(url).rstrip("/") + "/json.php" for url in (MIRRORS or []) if str(url).strip()]
except ImportError:
return []
return []
def ping_first(urls: list[str]) -> tuple[bool, str]:
for url in urls:
ok, detail = ping_url(url)
if ok:
return True, detail
if urls:
return ping_url(urls[0])
return False, "No ping target"

View File

@@ -1,7 +1,7 @@
import json import json
import os import os
import sys import sys
from typing import List, Dict, Any, Sequence from typing import List, Dict, Any, Sequence, Optional
from SYS.cmdlet_spec import Cmdlet, CmdletArg from SYS.cmdlet_spec import Cmdlet, CmdletArg
from SYS.logger import log from SYS.logger import log
from SYS.result_table import Table from SYS.result_table import Table
@@ -12,22 +12,45 @@ ADJECTIVE_FILE = os.path.join(
"cmdnat", "cmdnat",
"adjective.json" "adjective.json"
) )
_ADJECTIVE_CACHE: Optional[Dict[str, List[str]]] = None
_ADJECTIVE_CACHE_MTIME_NS: Optional[int] = None
def _load_adjectives() -> Dict[str, List[str]]: def _load_adjectives() -> Dict[str, List[str]]:
global _ADJECTIVE_CACHE, _ADJECTIVE_CACHE_MTIME_NS
try: try:
if os.path.exists(ADJECTIVE_FILE): if not os.path.exists(ADJECTIVE_FILE):
with open(ADJECTIVE_FILE, "r", encoding="utf-8") as f: _ADJECTIVE_CACHE = {}
return json.load(f) _ADJECTIVE_CACHE_MTIME_NS = None
return {}
current_mtime_ns = os.stat(ADJECTIVE_FILE).st_mtime_ns
if (_ADJECTIVE_CACHE is not None and
_ADJECTIVE_CACHE_MTIME_NS == current_mtime_ns):
return _ADJECTIVE_CACHE
with open(ADJECTIVE_FILE, "r", encoding="utf-8") as f:
loaded = json.load(f)
if not isinstance(loaded, dict):
loaded = {}
_ADJECTIVE_CACHE = loaded
_ADJECTIVE_CACHE_MTIME_NS = current_mtime_ns
return _ADJECTIVE_CACHE
except Exception as e: except Exception as e:
log(f"Error loading adjectives: {e}", file=sys.stderr) log(f"Error loading adjectives: {e}", file=sys.stderr)
_ADJECTIVE_CACHE = {}
_ADJECTIVE_CACHE_MTIME_NS = None
return {} return {}
def _save_adjectives(data: Dict[str, List[str]]) -> bool: def _save_adjectives(data: Dict[str, List[str]]) -> bool:
global _ADJECTIVE_CACHE, _ADJECTIVE_CACHE_MTIME_NS
try: try:
with open(ADJECTIVE_FILE, "w", encoding="utf-8") as f: with open(ADJECTIVE_FILE, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2) json.dump(data, f, indent=2)
_ADJECTIVE_CACHE = data
_ADJECTIVE_CACHE_MTIME_NS = os.stat(ADJECTIVE_FILE).st_mtime_ns
return True return True
except Exception as e: except Exception as e:
log(f"Error saving adjectives: {e}", file=sys.stderr) log(f"Error saving adjectives: {e}", file=sys.stderr)

View File

@@ -1,9 +1,18 @@
from typing import List, Dict, Any, Optional, Sequence from typing import List, Dict, Any, Optional, Sequence
from SYS.cmdlet_spec import Cmdlet, CmdletArg from SYS.cmdlet_spec import Cmdlet, CmdletArg
from SYS.config import load_config, save_config, save_config_and_verify from SYS.config import (
load_config,
save_config,
save_config_and_verify,
set_nested_config_value,
)
from SYS import pipeline as ctx from SYS import pipeline as ctx
from SYS.result_table import Table from SYS.result_table import Table
from cmdnat._parsing import (
extract_piped_value as _extract_piped_value,
extract_value_arg as _extract_value_arg,
)
CMDLET = Cmdlet( CMDLET = Cmdlet(
name=".config", name=".config",
@@ -43,91 +52,7 @@ def flatten_config(config: Dict[str, Any], parent_key: str = "", sep: str = ".")
def set_nested_config(config: Dict[str, Any], key: str, value: str) -> bool: def set_nested_config(config: Dict[str, Any], key: str, value: str) -> bool:
keys = key.split(".") return set_nested_config_value(config, key, value, on_error=print)
d = config
# Navigate to the parent dict
for k in keys[:-1]:
if k not in d or not isinstance(d[k], dict):
d[k] = {}
d = d[k]
last_key = keys[-1]
# Try to preserve type if key exists
if last_key in d:
current_val = d[last_key]
if isinstance(current_val, bool):
if value.lower() in ("true", "yes", "1", "on"):
d[last_key] = True
elif value.lower() in ("false", "no", "0", "off"):
d[last_key] = False
else:
# Fallback to boolean conversion of string (usually True for non-empty)
# But for config, explicit is better.
print(f"Warning: Could not convert '{value}' to boolean. Using string.")
d[last_key] = value
elif isinstance(current_val, int):
try:
d[last_key] = int(value)
except ValueError:
print(f"Warning: Could not convert '{value}' to int. Using string.")
d[last_key] = value
elif isinstance(current_val, float):
try:
d[last_key] = float(value)
except ValueError:
print(f"Warning: Could not convert '{value}' to float. Using string.")
d[last_key] = value
else:
d[last_key] = value
else:
# New key, try to infer type
if value.lower() in ("true", "false"):
d[last_key] = value.lower() == "true"
elif value.isdigit():
d[last_key] = int(value)
else:
d[last_key] = value
return True
def _extract_piped_value(result: Any) -> Optional[str]:
if isinstance(result, str):
return result.strip() if result.strip() else None
if isinstance(result, (int, float)):
return str(result)
if isinstance(result, dict):
val = result.get("value")
if val is not None:
return str(val).strip()
return None
def _extract_value_arg(args: Sequence[str]) -> Optional[str]:
if not args:
return None
tokens = [str(tok) for tok in args if tok is not None]
flags = {"-value", "--value", "-set-value", "--set-value"}
for idx, tok in enumerate(tokens):
text = tok.strip()
if not text:
continue
low = text.lower()
if low in flags and idx + 1 < len(tokens):
candidate = str(tokens[idx + 1]).strip()
if candidate:
return candidate
if "=" in low:
head, val = low.split("=", 1)
if head in flags and val:
return val.strip()
for tok in tokens:
text = str(tok).strip()
if text and not text.startswith("-"):
return text
return None
def _get_selected_config_key() -> Optional[str]: def _get_selected_config_key() -> Optional[str]:

View File

@@ -12,8 +12,16 @@ from SYS.cmdlet_spec import Cmdlet, CmdletArg
from SYS.config import load_config, save_config from SYS.config import load_config, save_config
from SYS.logger import log, debug from SYS.logger import log, debug
from SYS.result_table import Table from SYS.result_table import Table
from SYS.item_accessors import get_sha256_hex
from SYS.utils import extract_hydrus_hash_from_url from SYS.utils import extract_hydrus_hash_from_url
from SYS import pipeline as ctx from SYS import pipeline as ctx
from cmdnat._parsing import (
extract_arg_value,
extract_piped_value as _extract_piped_value,
extract_value_arg as _extract_value_arg,
has_flag as _has_flag,
normalize_to_list as _normalize_to_list,
)
_MATRIX_PENDING_ITEMS_KEY = "matrix_pending_items" _MATRIX_PENDING_ITEMS_KEY = "matrix_pending_items"
_MATRIX_PENDING_TEXT_KEY = "matrix_pending_text" _MATRIX_PENDING_TEXT_KEY = "matrix_pending_text"
@@ -21,62 +29,9 @@ _MATRIX_MENU_STATE_KEY = "matrix_menu_state"
_MATRIX_SELECTED_SETTING_KEY_KEY = "matrix_selected_setting_key" _MATRIX_SELECTED_SETTING_KEY_KEY = "matrix_selected_setting_key"
def _extract_piped_value(result: Any) -> Optional[str]:
"""Extract the piped value from result (string, number, or dict with 'value' key)."""
if isinstance(result, str):
return result.strip() if result.strip() else None
if isinstance(result, (int, float)):
return str(result)
if isinstance(result, dict):
# Fallback to value field if it's a dict
val = result.get("value")
if val is not None:
return str(val).strip()
return None
def _extract_value_arg(args: Sequence[str]) -> Optional[str]:
"""Extract a fallback value from command-line args (value flag or positional)."""
if not args:
return None
tokens = [str(tok) for tok in args if tok is not None]
value_flags = {"-value", "--value", "-set-value", "--set-value"}
for idx, tok in enumerate(tokens):
low = tok.strip()
if not low:
continue
low_lower = low.lower()
if low_lower in value_flags and idx + 1 < len(tokens):
candidate = str(tokens[idx + 1]).strip()
if candidate:
return candidate
if "=" in low_lower:
head, val = low_lower.split("=", 1)
if head in value_flags and val:
return val.strip()
# Fallback to first non-flag token
for tok in tokens:
text = str(tok).strip()
if text and not text.startswith("-"):
return text
return None
def _extract_set_value_arg(args: Sequence[str]) -> Optional[str]: def _extract_set_value_arg(args: Sequence[str]) -> Optional[str]:
"""Extract the value from -set-value flag.""" """Extract the value from -set-value flag."""
if not args: return extract_arg_value(args, flags={"-set-value"})
return None
try:
tokens = list(args)
except Exception:
return None
for i, tok in enumerate(tokens):
try:
if str(tok).lower() == "-set-value" and i + 1 < len(tokens):
return str(tokens[i + 1]).strip()
except Exception:
continue
return None
def _update_matrix_config(config: Dict[str, Any], key: str, value: Any) -> bool: def _update_matrix_config(config: Dict[str, Any], key: str, value: Any) -> bool:
@@ -122,16 +77,6 @@ def _update_matrix_config(config: Dict[str, Any], key: str, value: Any) -> bool:
return False return False
def _has_flag(args: Sequence[str], flag: str) -> bool:
try:
want = str(flag or "").strip().lower()
if not want:
return False
return any(str(a).strip().lower() == want for a in (args or []))
except Exception:
return False
def _parse_config_room_filter_ids(config: Dict[str, Any]) -> List[str]: def _parse_config_room_filter_ids(config: Dict[str, Any]) -> List[str]:
try: try:
if not isinstance(config, dict): if not isinstance(config, dict):
@@ -426,14 +371,6 @@ def _extract_text_arg(args: Sequence[str]) -> str:
return "" return ""
def _normalize_to_list(value: Any) -> List[Any]:
if value is None:
return []
if isinstance(value, list):
return value
return [value]
def _extract_room_id(room_obj: Any) -> Optional[str]: def _extract_room_id(room_obj: Any) -> Optional[str]:
try: try:
# PipeObject stores unknown fields in .extra # PipeObject stores unknown fields in .extra
@@ -525,22 +462,8 @@ def _extract_url(item: Any) -> Optional[str]:
return None return None
_SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
def _extract_sha256_hex(item: Any) -> Optional[str]: def _extract_sha256_hex(item: Any) -> Optional[str]:
try: return get_sha256_hex(item, "hash")
if hasattr(item, "hash"):
h = getattr(item, "hash")
if isinstance(h, str) and _SHA256_RE.fullmatch(h.strip()):
return h.strip().lower()
if isinstance(item, dict):
h = item.get("hash")
if isinstance(h, str) and _SHA256_RE.fullmatch(h.strip()):
return h.strip().lower()
except Exception:
pass
return None
def _extract_hash_from_hydrus_file_url(url: str) -> Optional[str]: def _extract_hash_from_hydrus_file_url(url: str) -> Optional[str]:

View File

@@ -39,6 +39,7 @@ _WINDOWS_RESERVED_NAMES = {
*(f"com{i}" for i in range(1, 10)), *(f"com{i}" for i in range(1, 10)),
*(f"lpt{i}" for i in range(1, 10)), *(f"lpt{i}" for i in range(1, 10)),
} }
_ILLEGAL_FILENAME_CHARS_RE = re.compile(r'[<>:"/\\|?*]')
def _sanitize_filename_base(text: str) -> str: def _sanitize_filename_base(text: str) -> str:
@@ -48,7 +49,7 @@ def _sanitize_filename_base(text: str) -> str:
return "table" return "table"
# Replace characters illegal on Windows (and generally unsafe cross-platform). # Replace characters illegal on Windows (and generally unsafe cross-platform).
s = re.sub(r'[<>:"/\\|?*]', " ", s) s = _ILLEGAL_FILENAME_CHARS_RE.sub(" ", s)
# Drop control characters. # Drop control characters.
s = "".join(ch for ch in s if ch.isprintable()) s = "".join(ch for ch in s if ch.isprintable())

View File

@@ -23,6 +23,15 @@ _ALLDEBRID_UNLOCK_CACHE: Dict[str,
str] = {} str] = {}
_NOTES_PREFETCH_INFLIGHT: set[str] = set() _NOTES_PREFETCH_INFLIGHT: set[str] = set()
_NOTES_PREFETCH_LOCK = threading.Lock() _NOTES_PREFETCH_LOCK = threading.Lock()
_PLAYLIST_STORE_CACHE: Optional[Dict[str, Any]] = None
_PLAYLIST_STORE_MTIME_NS: Optional[int] = None
_SHA256_RE = re.compile(r"[0-9a-f]{64}")
_SHA256_FULL_RE = re.compile(r"^[0-9a-f]{64}$")
_EXTINF_TITLE_RE = re.compile(r"#EXTINF:-1,(.*?)(?:\n|\r|$)")
_WINDOWS_PATH_RE = re.compile(r"^[a-z]:[\\/]", flags=re.IGNORECASE)
_HASH_QUERY_RE = re.compile(r"hash=([0-9a-f]{64})")
_IPV4_RE = re.compile(r"^\d+\.\d+\.\d+\.\d+$")
_MPD_PATH_RE = re.compile(r"\.mpd($|\?)")
def _repo_root() -> Path: def _repo_root() -> Path:
@@ -36,26 +45,56 @@ def _playlist_store_path() -> Path:
return _repo_root() / "mpv_playlists.json" return _repo_root() / "mpv_playlists.json"
def _load_playlist_store(path: Path) -> Dict[str, Any]: def _new_playlist_store() -> Dict[str, Any]:
if not path.exists(): return {"next_id": 1, "playlists": []}
return {"next_id": 1, "playlists": []}
def _normalize_playlist_store(data: Any) -> Dict[str, Any]:
if not isinstance(data, dict):
return _new_playlist_store()
normalized = dict(data)
try: try:
data = json.loads(path.read_text(encoding="utf-8")) next_id = int(normalized.get("next_id") or 1)
if not isinstance(data, dict): except Exception:
return {"next_id": 1, "playlists": []} next_id = 1
data.setdefault("next_id", 1) normalized["next_id"] = max(next_id, 1)
data.setdefault("playlists", [])
if not isinstance(data["playlists"], list): playlists = normalized.get("playlists")
data["playlists"] = [] normalized["playlists"] = playlists if isinstance(playlists, list) else []
return normalized
def _load_playlist_store(path: Path) -> Dict[str, Any]:
global _PLAYLIST_STORE_CACHE, _PLAYLIST_STORE_MTIME_NS
if not path.exists():
_PLAYLIST_STORE_CACHE = _new_playlist_store()
_PLAYLIST_STORE_MTIME_NS = None
return _PLAYLIST_STORE_CACHE
try:
current_mtime_ns = path.stat().st_mtime_ns
if (_PLAYLIST_STORE_CACHE is not None and
_PLAYLIST_STORE_MTIME_NS == current_mtime_ns):
return _PLAYLIST_STORE_CACHE
data = _normalize_playlist_store(json.loads(path.read_text(encoding="utf-8")))
_PLAYLIST_STORE_CACHE = data
_PLAYLIST_STORE_MTIME_NS = current_mtime_ns
return data return data
except Exception: except Exception:
return {"next_id": 1, "playlists": []} _PLAYLIST_STORE_CACHE = _new_playlist_store()
_PLAYLIST_STORE_MTIME_NS = None
return _PLAYLIST_STORE_CACHE
def _save_playlist_store(path: Path, data: Dict[str, Any]) -> bool: def _save_playlist_store(path: Path, data: Dict[str, Any]) -> bool:
global _PLAYLIST_STORE_CACHE, _PLAYLIST_STORE_MTIME_NS
try: try:
normalized = _normalize_playlist_store(data)
path.parent.mkdir(parents=True, exist_ok=True) path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, indent=2), encoding="utf-8") path.write_text(json.dumps(normalized, indent=2), encoding="utf-8")
_PLAYLIST_STORE_CACHE = normalized
_PLAYLIST_STORE_MTIME_NS = path.stat().st_mtime_ns
return True return True
except Exception: except Exception:
return False return False
@@ -559,7 +598,7 @@ def _extract_store_and_hash(item: Any) -> tuple[Optional[str], Optional[str]]:
else: else:
text = getattr(item, "path", None) or getattr(item, "url", None) text = getattr(item, "path", None) or getattr(item, "url", None)
if text: if text:
m = re.search(r"[0-9a-f]{64}", str(text).lower()) m = _SHA256_RE.search(str(text).lower())
if m: if m:
file_hash = m.group(0) file_hash = m.group(0)
except Exception: except Exception:
@@ -707,7 +746,7 @@ def _extract_title_from_item(item: Dict[str, Any]) -> str:
try: try:
# Extract title from #EXTINF:-1,Title # Extract title from #EXTINF:-1,Title
# Use regex to find title between #EXTINF:-1, and newline # Use regex to find title between #EXTINF:-1, and newline
match = re.search(r"#EXTINF:-1,(.*?)(?:\n|\r|$)", filename) match = _EXTINF_TITLE_RE.search(filename)
if match: if match:
extracted_title = match.group(1).strip() extracted_title = match.group(1).strip()
if not title or title == "memory://": if not title or title == "memory://":
@@ -817,7 +856,7 @@ def _normalize_playlist_path(text: Optional[str]) -> Optional[str]:
return None return None
# If it's already a bare hydrus hash, use it directly # If it's already a bare hydrus hash, use it directly
lower_real = real.lower() lower_real = real.lower()
if re.fullmatch(r"[0-9a-f]{64}", lower_real): if _SHA256_FULL_RE.fullmatch(lower_real):
return lower_real return lower_real
# If it's a hydrus file URL, normalize to the hash for dedupe # If it's a hydrus file URL, normalize to the hash for dedupe
@@ -829,7 +868,7 @@ def _normalize_playlist_path(text: Optional[str]) -> Optional[str]:
if parsed.path.endswith("/get_files/file"): if parsed.path.endswith("/get_files/file"):
qs = parse_qs(parsed.query) qs = parse_qs(parsed.query)
h = qs.get("hash", [None])[0] h = qs.get("hash", [None])[0]
if h and re.fullmatch(r"[0-9a-f]{64}", h.lower()): if h and _SHA256_FULL_RE.fullmatch(h.lower()):
return h.lower() return h.lower()
except Exception: except Exception:
pass pass
@@ -862,7 +901,7 @@ def _infer_store_from_playlist_item(
target = memory_target target = memory_target
# Hydrus hashes: bare 64-hex entries # Hydrus hashes: bare 64-hex entries
if re.fullmatch(r"[0-9a-f]{64}", target.lower()): if _SHA256_FULL_RE.fullmatch(target.lower()):
# If we have file_storage, query each Hydrus instance to find which one has this hash # If we have file_storage, query each Hydrus instance to find which one has this hash
if file_storage: if file_storage:
hash_str = target.lower() hash_str = target.lower()
@@ -877,7 +916,7 @@ def _infer_store_from_playlist_item(
if lower.startswith("hydrus://"): if lower.startswith("hydrus://"):
# Extract hash from hydrus:// URL if possible # Extract hash from hydrus:// URL if possible
if file_storage: if file_storage:
hash_match = re.search(r"[0-9a-f]{64}", target.lower()) hash_match = _SHA256_RE.search(target.lower())
if hash_match: if hash_match:
hash_str = hash_match.group(0) hash_str = hash_match.group(0)
hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage) hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage)
@@ -886,9 +925,7 @@ def _infer_store_from_playlist_item(
return "hydrus" return "hydrus"
# Windows / UNC paths # Windows / UNC paths
if re.match(r"^[a-z]:[\\/]", if _WINDOWS_PATH_RE.match(target) or target.startswith("\\\\"):
target,
flags=re.IGNORECASE) or target.startswith("\\\\"):
return "local" return "local"
# file:// url # file:// url
@@ -918,7 +955,7 @@ def _infer_store_from_playlist_item(
# Hydrus API URL - try to extract hash and find instance # Hydrus API URL - try to extract hash and find instance
if file_storage: if file_storage:
# Try to extract hash from URL parameters # Try to extract hash from URL parameters
hash_match = re.search(r"hash=([0-9a-f]{64})", target.lower()) hash_match = _HASH_QUERY_RE.search(target.lower())
if hash_match: if hash_match:
hash_str = hash_match.group(1) hash_str = hash_match.group(1)
hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage) hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage)
@@ -929,10 +966,10 @@ def _infer_store_from_playlist_item(
if hydrus_instance: if hydrus_instance:
return hydrus_instance return hydrus_instance
return "hydrus" return "hydrus"
if re.match(r"^\d+\.\d+\.\d+\.\d+$", host_stripped) and "get_files" in path: if _IPV4_RE.match(host_stripped) and "get_files" in path:
# IP-based Hydrus URL # IP-based Hydrus URL
if file_storage: if file_storage:
hash_match = re.search(r"hash=([0-9a-f]{64})", target.lower()) hash_match = _HASH_QUERY_RE.search(target.lower())
if hash_match: if hash_match:
hash_str = hash_match.group(1) hash_str = hash_match.group(1)
hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage) hydrus_instance = _find_hydrus_instance_for_hash(hash_str, file_storage)
@@ -1002,7 +1039,7 @@ def _is_hydrus_path(path: str, hydrus_url: Optional[str]) -> bool:
pass pass
if "get_files" in path_part or "file?hash=" in path_part: if "get_files" in path_part or "file?hash=" in path_part:
return True return True
if re.match(r"^\d+\.\d+\.\d+\.\d+$", host) and "get_files" in path_part: if _IPV4_RE.match(host) and "get_files" in path_part:
return True return True
return False return False
@@ -1493,7 +1530,7 @@ def _queue_items(
# Set it via IPC before loadfile so the currently running MPV can play the manifest. # Set it via IPC before loadfile so the currently running MPV can play the manifest.
try: try:
target_str = str(target or "") target_str = str(target or "")
if re.search(r"\.mpd($|\?)", target_str.lower()): if _MPD_PATH_RE.search(target_str.lower()):
_send_ipc_command( _send_ipc_command(
{ {
"command": [ "command": [
@@ -1556,8 +1593,9 @@ def _queue_items(
if target: if target:
# If we just have a hydrus hash, build a direct file URL for MPV # If we just have a hydrus hash, build a direct file URL for MPV
if re.fullmatch(r"[0-9a-f]{64}", if _SHA256_FULL_RE.fullmatch(
str(target).strip().lower()) and effective_hydrus_url: str(target).strip().lower()
) and effective_hydrus_url:
target = ( target = (
f"{effective_hydrus_url.rstrip('/')}/get_files/file?hash={str(target).strip()}" f"{effective_hydrus_url.rstrip('/')}/get_files/file?hash={str(target).strip()}"
) )
@@ -2337,7 +2375,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
# Check if it's a Hydrus URL # Check if it's a Hydrus URL
if "get_files/file" in real_path or "hash=" in real_path: if "get_files/file" in real_path or "hash=" in real_path:
# Extract hash from Hydrus URL # Extract hash from Hydrus URL
hash_match = re.search(r"hash=([0-9a-f]{64})", real_path.lower()) hash_match = _HASH_QUERY_RE.search(real_path.lower())
if hash_match: if hash_match:
file_hash = hash_match.group(1) file_hash = hash_match.group(1)
# Try to find which Hydrus instance has this file # Try to find which Hydrus instance has this file
@@ -2576,7 +2614,7 @@ def _start_mpv(
candidate = it.get("path") or it.get("url") candidate = it.get("path") or it.get("url")
else: else:
candidate = getattr(it, "path", None) or getattr(it, "url", None) candidate = getattr(it, "path", None) or getattr(it, "url", None)
if candidate and re.search(r"\.mpd($|\?)", str(candidate).lower()): if candidate and _MPD_PATH_RE.search(str(candidate).lower()):
needs_mpd_whitelist = True needs_mpd_whitelist = True
break break
if needs_mpd_whitelist: if needs_mpd_whitelist:

View File

@@ -7,6 +7,16 @@ from SYS.cmdlet_spec import Cmdlet
from SYS import pipeline as ctx from SYS import pipeline as ctx
from SYS.result_table import Table from SYS.result_table import Table
from SYS.logger import set_debug, debug from SYS.logger import set_debug, debug
from cmdnat._status_shared import (
add_startup_check as _add_startup_check,
default_provider_ping_targets as _default_provider_ping_targets,
has_provider as _has_provider,
has_store_subtype as _has_store_subtype,
has_tool as _has_tool,
ping_first as _ping_first,
ping_url as _ping_url,
provider_display_name as _provider_display_name,
)
CMDLET = Cmdlet( CMDLET = Cmdlet(
name=".status", name=".status",
@@ -15,91 +25,6 @@ CMDLET = Cmdlet(
arg=[], arg=[],
) )
def _upper(value: Any) -> str:
text = "" if value is None else str(value)
return text.upper()
def _add_startup_check(
table: Table,
status: str,
name: str,
*,
provider: str = "",
store: str = "",
files: int | str | None = None,
detail: str = "",
) -> None:
row = table.add_row()
row.add_column("STATUS", _upper(status))
row.add_column("NAME", _upper(name))
row.add_column("PROVIDER", _upper(provider or ""))
row.add_column("STORE", _upper(store or ""))
row.add_column("FILES", "" if files is None else str(files))
row.add_column("DETAIL", _upper(detail or ""))
def _has_store_subtype(cfg: dict, subtype: str) -> bool:
store_cfg = cfg.get("store")
if not isinstance(store_cfg, dict):
return False
bucket = store_cfg.get(subtype)
if not isinstance(bucket, dict):
return False
return any(isinstance(v, dict) and bool(v) for v in bucket.values())
def _has_provider(cfg: dict, name: str) -> bool:
provider_cfg = cfg.get("provider")
if not isinstance(provider_cfg, dict):
return False
block = provider_cfg.get(str(name).strip().lower())
return isinstance(block, dict) and bool(block)
def _has_tool(cfg: dict, name: str) -> bool:
tool_cfg = cfg.get("tool")
if not isinstance(tool_cfg, dict):
return False
block = tool_cfg.get(str(name).strip().lower())
return isinstance(block, dict) and bool(block)
def _ping_url(url: str, timeout: float = 3.0) -> tuple[bool, str]:
try:
from API.HTTP import HTTPClient
with HTTPClient(timeout=timeout, retries=1) as client:
resp = client.get(url, allow_redirects=True)
code = int(getattr(resp, "status_code", 0) or 0)
ok = 200 <= code < 500
return ok, f"{url} (HTTP {code})"
except Exception as exc:
return False, f"{url} ({type(exc).__name__})"
def _provider_display_name(key: str) -> str:
k = (key or "").strip()
low = k.lower()
if low == "openlibrary": return "OpenLibrary"
if low == "alldebrid": return "AllDebrid"
if low == "youtube": return "YouTube"
return k[:1].upper() + k[1:] if k else "Provider"
def _default_provider_ping_targets(provider_key: str) -> list[str]:
prov = (provider_key or "").strip().lower()
if prov == "openlibrary": return ["https://openlibrary.org"]
if prov == "youtube": return ["https://www.youtube.com"]
if prov == "bandcamp": return ["https://bandcamp.com"]
if prov == "libgen":
try:
from Provider.libgen import MIRRORS
return [str(x).rstrip("/") + "/json.php" for x in (MIRRORS or []) if str(x).strip()]
except ImportError: return []
return []
def _ping_first(urls: list[str]) -> tuple[bool, str]:
for u in urls:
ok, detail = _ping_url(u)
if ok: return True, detail
if urls:
ok, detail = _ping_url(urls[0])
return ok, detail
return False, "No ping target"
def _run(result: Any, args: List[str], config: Dict[str, Any]) -> int: def _run(result: Any, args: List[str], config: Dict[str, Any]) -> int:
startup_table = Table( startup_table = Table(
"*********<IGNITIO>*********<NOUSEMPEH>*********<RUGRAPOG>*********<OMEGHAU>*********" "*********<IGNITIO>*********<NOUSEMPEH>*********<RUGRAPOG>*********<OMEGHAU>*********"

View File

@@ -8,28 +8,10 @@ from SYS.cmdlet_spec import Cmdlet, CmdletArg
from SYS.logger import log from SYS.logger import log
from SYS.result_table import Table from SYS.result_table import Table
from SYS import pipeline as ctx from SYS import pipeline as ctx
from cmdnat._parsing import has_flag as _has_flag, normalize_to_list as _normalize_to_list
_TELEGRAM_PENDING_ITEMS_KEY = "telegram_pending_items" _TELEGRAM_PENDING_ITEMS_KEY = "telegram_pending_items"
def _has_flag(args: Sequence[str], flag: str) -> bool:
try:
want = str(flag or "").strip().lower()
if not want:
return False
return any(str(a).strip().lower() == want for a in (args or []))
except Exception:
return False
def _normalize_to_list(value: Any) -> List[Any]:
if value is None:
return []
if isinstance(value, list):
return value
return [value]
def _extract_chat_id(chat_obj: Any) -> Optional[int]: def _extract_chat_id(chat_obj: Any) -> Optional[int]:
try: try:
if isinstance(chat_obj, dict): if isinstance(chat_obj, dict):

View File

@@ -4,6 +4,7 @@ from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List, Optional, Sequence, Tuple from typing import Any, Dict, List, Optional, Sequence, Tuple
from SYS.config import get_nested_config_value as _get_nested
from SYS.logger import debug from SYS.logger import debug
@@ -28,15 +29,6 @@ def _debug_repr(value: Any, max_chars: int = 12000) -> str:
return _truncate_debug_text(s, max_chars=max_chars) return _truncate_debug_text(s, max_chars=max_chars)
def _get_nested(config: Dict[str, Any], *path: str) -> Any:
cur: Any = config
for key in path:
if not isinstance(cur, dict):
return None
cur = cur.get(key)
return cur
def _as_bool(value: Any, default: bool = False) -> bool: def _as_bool(value: Any, default: bool = False) -> bool:
if value is None: if value is None:
return default return default

View File

@@ -10,6 +10,7 @@ from dataclasses import dataclass
from pathlib import Path from pathlib import Path
from typing import Any, Dict, Iterator, Optional, Union from typing import Any, Dict, Iterator, Optional, Union
from SYS.config import get_nested_config_value as _get_nested
from SYS.logger import debug from SYS.logger import debug
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
@@ -24,15 +25,6 @@ __all__ = [
] ]
def _get_nested(config: Dict[str, Any], *path: str) -> Any:
cur: Any = config
for key in path:
if not isinstance(cur, dict):
return None
cur = cur.get(key)
return cur
def _resolve_out_dir(arg_outdir: Optional[Union[str, Path]]) -> Path: def _resolve_out_dir(arg_outdir: Optional[Union[str, Path]]) -> Path:
"""Resolve an output directory using config when possible.""" """Resolve an output directory using config when possible."""
if arg_outdir: if arg_outdir:

View File

@@ -18,6 +18,7 @@ from typing import Any, Dict, Iterator, List, Optional, Sequence, cast
from urllib.parse import urlparse from urllib.parse import urlparse
from SYS import pipeline as pipeline_context from SYS import pipeline as pipeline_context
from SYS.config import get_nested_config_value as _get_nested
from SYS.logger import debug, log from SYS.logger import debug, log
from SYS.models import ( from SYS.models import (
DebugLogger, DebugLogger,
@@ -137,15 +138,6 @@ def _build_supported_domains() -> set[str]:
return _SUPPORTED_DOMAINS return _SUPPORTED_DOMAINS
def _get_nested(config: Dict[str, Any], *path: str) -> Any:
cur: Any = config
for key in path:
if not isinstance(cur, dict):
return None
cur = cur.get(key)
return cur
def _parse_csv_list(value: Any) -> Optional[List[str]]: def _parse_csv_list(value: Any) -> Optional[List[str]]:
if value is None: if value is None:
return None return None