This commit is contained in:
2026-01-02 02:28:59 -08:00
parent deb05c0d44
commit 6e9a0c28ff
13 changed files with 1402 additions and 2334 deletions

File diff suppressed because it is too large Load Diff

23
CLI.py
View File

@@ -3824,9 +3824,28 @@ class PipelineExecutor:
# Fall back to default selection rendering on any failure.
pass
items = piped_result if isinstance(piped_result, list) else [piped_result]
# Special-case: selecting notes should show the text content directly.
note_like_items = [
i for i in items
if isinstance(i, dict) and ("note_text" in i or "note" in i)
]
if note_like_items:
for idx, item in enumerate(note_like_items, 1):
note_name = str(
item.get("note_name")
or item.get("name")
or f"note {idx}"
).strip()
note_text = str(item.get("note_text") or item.get("note") or "")
note_text = note_text[:999]
stdout_console().print()
stdout_console().print(f"{note_name}:\n{note_text}")
ctx.set_last_result_items_only(items)
return
table = ResultTable("Selection Result")
items = piped_result if isinstance(piped_result,
list) else [piped_result]
for item in items:
table.add_result(item)
ctx.set_last_result_items_only(items)

220
SYS/cmdlet_api.py Normal file
View File

@@ -0,0 +1,220 @@
from __future__ import annotations
import contextlib
import io
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Optional, Sequence
from SYS import pipeline as ctx
from SYS.models import PipelineStageContext
from SYS.rich_display import capture_rich_output
CmdletCallable = Callable[[Any, Sequence[str], Dict[str, Any]], int]
@dataclass(slots=True)
class CmdletRunResult:
"""Programmatic result for a single cmdlet invocation."""
name: str
args: Sequence[str]
exit_code: int = 0
emitted: List[Any] = field(default_factory=list)
# Best-effort: cmdlets can publish tables/items via pipeline state even when
# they don't emit pipeline items.
result_table: Optional[Any] = None
result_items: List[Any] = field(default_factory=list)
result_subject: Optional[Any] = None
stdout: str = ""
stderr: str = ""
error: Optional[str] = None
def _normalize_cmd_name(name: str) -> str:
return str(name or "").replace("_", "-").strip().lower()
def resolve_cmdlet(cmd_name: str) -> Optional[CmdletCallable]:
"""Resolve a cmdlet callable by name from the registry (aliases supported)."""
try:
from SYS.cmdlet_catalog import ensure_registry_loaded
ensure_registry_loaded()
except Exception:
pass
try:
import cmdlet as cmdlet_pkg
return cmdlet_pkg.get(cmd_name)
except Exception:
return None
def run_cmdlet(
cmd: str | CmdletCallable,
args: Sequence[str] | None,
config: Dict[str, Any],
*,
piped: Any = None,
isolate: bool = True,
capture_output: bool = True,
stage_index: int = 0,
total_stages: int = 1,
pipe_index: Optional[int] = None,
worker_id: Optional[str] = None,
) -> CmdletRunResult:
"""Run a single cmdlet programmatically and return structured results.
This is intended for TUI/webapp consumers that want cmdlet behavior without
going through the interactive CLI loop.
Notes:
- When `isolate=True` (default) this runs inside `ctx.new_pipeline_state()` so
global CLI pipeline state is not mutated.
- Output capturing covers both normal `print()` and Rich output via
`capture_rich_output()`.
"""
normalized_args: Sequence[str] = list(args or [])
if isinstance(cmd, str):
name = _normalize_cmd_name(cmd)
cmd_fn = resolve_cmdlet(name)
else:
name = getattr(cmd, "__name__", "cmdlet")
cmd_fn = cmd
result = CmdletRunResult(name=name, args=normalized_args)
if not callable(cmd_fn):
result.exit_code = 1
result.error = f"Unknown command: {name}"
result.stderr = result.error
return result
stage_ctx = PipelineStageContext(
stage_index=int(stage_index),
total_stages=int(total_stages),
pipe_index=pipe_index,
worker_id=worker_id,
)
stdout_buffer = io.StringIO()
stderr_buffer = io.StringIO()
stage_text = " ".join([name, *list(normalized_args)]).strip()
state_cm = ctx.new_pipeline_state() if isolate else contextlib.nullcontext()
with state_cm:
# Keep behavior predictable: start from a clean slate.
try:
ctx.reset()
except Exception:
pass
try:
ctx.set_stage_context(stage_ctx)
except Exception:
pass
try:
ctx.set_current_cmdlet_name(name)
except Exception:
pass
try:
ctx.set_current_stage_text(stage_text)
except Exception:
pass
try:
ctx.set_current_command_text(stage_text)
except Exception:
pass
try:
run_cm = (
capture_rich_output(stdout=stdout_buffer, stderr=stderr_buffer)
if capture_output
else contextlib.nullcontext()
)
with run_cm:
with (
contextlib.redirect_stdout(stdout_buffer)
if capture_output
else contextlib.nullcontext()
):
with (
contextlib.redirect_stderr(stderr_buffer)
if capture_output
else contextlib.nullcontext()
):
result.exit_code = int(cmd_fn(piped, list(normalized_args), config))
except Exception as exc:
result.exit_code = 1
result.error = f"{type(exc).__name__}: {exc}"
finally:
result.stdout = stdout_buffer.getvalue()
result.stderr = stderr_buffer.getvalue()
# Prefer cmdlet emits (pipeline semantics).
try:
result.emitted = list(stage_ctx.emits or [])
except Exception:
result.emitted = []
# Mirror CLI behavior: if cmdlet emitted items and there is no overlay table,
# make emitted items the last result items for downstream consumers.
try:
has_overlay = bool(ctx.get_display_table())
except Exception:
has_overlay = False
if result.emitted and not has_overlay:
try:
ctx.set_last_result_items_only(list(result.emitted))
except Exception:
pass
# Best-effort snapshot of visible results.
try:
result.result_table = (
ctx.get_display_table() or ctx.get_current_stage_table() or ctx.get_last_result_table()
)
except Exception:
result.result_table = None
try:
result.result_items = list(ctx.get_last_result_items() or [])
except Exception:
result.result_items = []
try:
result.result_subject = ctx.get_last_result_subject()
except Exception:
result.result_subject = None
# Cleanup stage-local markers.
try:
ctx.clear_current_stage_text()
except Exception:
pass
try:
ctx.clear_current_cmdlet_name()
except Exception:
pass
try:
ctx.clear_current_command_text()
except Exception:
pass
try:
ctx.set_stage_context(None)
except Exception:
pass
return result

View File

@@ -512,36 +512,46 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
if not base_path.exists():
continue
# Ensure file entry exists
file_id: Optional[int] = None
# Ensure file entry exists (folder store schema is keyed by hash).
file_hash_value: Optional[str] = None
if sha256_file and base_path.exists():
try:
cursor = db.connection.cursor() if db.connection else None
if cursor:
cursor.execute(
"SELECT id FROM files WHERE file_path = ?",
(str(base_path),
)
)
result = cursor.fetchone()
file_id = result[0] if result else None
file_hash_value = sha256_file(base_path)
except Exception:
file_id = None
file_hash_value = None
if not file_id:
try:
cursor = db.connection.cursor() if db.connection else None
if cursor:
cursor.execute(
'INSERT INTO files (file_path, indexed_at, updated_at) VALUES (?, datetime("now"), datetime("now"))',
(str(base_path),
),
)
db.connection.commit()
file_id = cursor.lastrowid
except Exception:
if not file_hash_value:
continue
if not file_id:
try:
db_file_path = (
db._to_db_file_path(base_path) # type: ignore[attr-defined]
if hasattr(db, "_to_db_file_path")
else str(base_path)
)
except Exception:
db_file_path = str(base_path)
try:
file_modified = float(base_path.stat().st_mtime)
except Exception:
file_modified = None
try:
cursor = db.connection.cursor() if db.connection else None
if cursor:
cursor.execute(
"SELECT hash FROM file WHERE file_path = ?",
(str(db_file_path),),
)
result = cursor.fetchone()
if not result:
cursor.execute(
"INSERT INTO file (hash, file_path, file_modified) VALUES (?, ?, ?)",
(file_hash_value, str(db_file_path), file_modified),
)
db.connection.commit()
except Exception:
continue
if sidecar_path.suffix == ".tag":
@@ -557,15 +567,9 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
try:
cursor = db.connection.cursor() if db.connection else None
if cursor:
file_hash_value: Optional[str] = None
if hasattr(db, "get_file_hash"):
try:
file_hash_value = db.get_file_hash(file_id)
except Exception:
file_hash_value = None
for tag in tags:
cursor.execute(
"INSERT OR IGNORE INTO tags (hash, tag) VALUES (?, ?)",
"INSERT OR IGNORE INTO tag (hash, tag) VALUES (?, ?)",
(file_hash_value,
tag),
)
@@ -608,13 +612,15 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
except Exception:
pass
if not hash_value:
hash_value = file_hash_value
try:
cursor = db.connection.cursor() if db.connection else None
if cursor:
cursor.execute(
'INSERT OR REPLACE INTO metadata (file_id, hash, url, relationships, time_imported, time_modified) VALUES (?, ?, ?, ?, datetime("now"), datetime("now"))',
'INSERT OR REPLACE INTO metadata (hash, url, relationships, time_imported, time_modified) VALUES (?, ?, ?, datetime("now"), datetime("now"))',
(
file_id,
hash_value,
json.dumps(url),
json.dumps(relationships),
@@ -634,9 +640,8 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
cursor = db.connection.cursor() if db.connection else None
if cursor:
cursor.execute(
'INSERT INTO notes (file_id, note, created_at, updated_at) VALUES (?, ?, datetime("now"), datetime("now")) ON CONFLICT(file_id) DO UPDATE SET note = excluded.note, updated_at = datetime("now")',
(file_id,
content),
'INSERT INTO note (hash, name, note, created_at, updated_at) VALUES (?, ?, ?, datetime("now"), datetime("now")) ON CONFLICT(hash, name) DO UPDATE SET note = excluded.note, updated_at = datetime("now")',
(file_hash_value, "default", content),
)
db.connection.commit()
except Exception:

View File

@@ -263,6 +263,9 @@ class WorkerManager:
self.refresh_thread: Optional[Thread] = None
self._stop_refresh = False
self._lock = Lock()
# Reuse the DB's own lock so there is exactly one lock guarding the
# sqlite connection (and it is safe for re-entrant/nested DB usage).
self._db_lock = self.db._db_lock
self.worker_handlers: Dict[str,
WorkerLoggingHandler] = {} # Track active handlers
self._worker_last_step: Dict[str,
@@ -272,6 +275,7 @@ class WorkerManager:
"""Close the database connection."""
if self.db:
try:
with self._db_lock:
self.db.close()
except Exception:
pass
@@ -317,6 +321,7 @@ class WorkerManager:
Count of workers updated.
"""
try:
with self._db_lock:
return self.db.expire_running_workers(
older_than_seconds=older_than_seconds,
status=status,
@@ -419,6 +424,7 @@ class WorkerManager:
True if worker was inserted successfully
"""
try:
with self._db_lock:
result = self.db.insert_worker(
worker_id,
worker_type,
@@ -473,6 +479,7 @@ class WorkerManager:
kwargs["last_updated"] = datetime.now().isoformat()
if "current_step" in kwargs and kwargs["current_step"]:
self._worker_last_step[worker_id] = str(kwargs["current_step"])
with self._db_lock:
return self.db.update_worker(worker_id, **kwargs)
return True
except Exception as e:
@@ -510,6 +517,7 @@ class WorkerManager:
if result_data:
kwargs["result_data"] = result_data
with self._db_lock:
success = self.db.update_worker(worker_id, **kwargs)
logger.info(f"[WorkerManager] Worker finished: {worker_id} ({result})")
self._worker_last_step.pop(worker_id, None)
@@ -528,6 +536,7 @@ class WorkerManager:
List of active worker dictionaries
"""
try:
with self._db_lock:
return self.db.get_active_workers()
except Exception as e:
logger.error(
@@ -546,6 +555,7 @@ class WorkerManager:
List of finished worker dictionaries
"""
try:
with self._db_lock:
all_workers = self.db.get_all_workers(limit=limit)
# Filter to only finished workers
finished = [
@@ -570,6 +580,7 @@ class WorkerManager:
Worker data or None if not found
"""
try:
with self._db_lock:
return self.db.get_worker(worker_id)
except Exception as e:
logger.error(
@@ -583,6 +594,7 @@ class WorkerManager:
limit: int = 500) -> List[Dict[str,
Any]]:
"""Fetch recorded worker timeline events."""
with self._db_lock:
return self.db.get_worker_events(worker_id, limit)
def log_step(self, worker_id: str, step_text: str) -> bool:
@@ -596,6 +608,7 @@ class WorkerManager:
True if successful
"""
try:
with self._db_lock:
success = self.db.append_worker_steps(worker_id, step_text)
if success:
self._worker_last_step[worker_id] = step_text
@@ -621,6 +634,7 @@ class WorkerManager:
Steps text or empty string if not found
"""
try:
with self._db_lock:
return self.db.get_worker_steps(worker_id)
except Exception as e:
logger.error(
@@ -705,6 +719,7 @@ class WorkerManager:
Number of workers deleted
"""
try:
with self._db_lock:
count = self.db.cleanup_old_workers(days)
if count > 0:
logger.info(f"[WorkerManager] Cleaned up {count} old workers")
@@ -729,6 +744,7 @@ class WorkerManager:
"""
try:
step_label = self._get_last_step(worker_id)
with self._db_lock:
return self.db.append_worker_stdout(
worker_id,
text,
@@ -749,6 +765,7 @@ class WorkerManager:
Worker's stdout or empty string
"""
try:
with self._db_lock:
return self.db.get_worker_stdout(worker_id)
except Exception as e:
logger.error(f"[WorkerManager] Error getting stdout: {e}", exc_info=True)
@@ -773,6 +790,7 @@ class WorkerManager:
True if clear was successful
"""
try:
with self._db_lock:
return self.db.clear_worker_stdout(worker_id)
except Exception as e:
logger.error(f"[WorkerManager] Error clearing stdout: {e}", exc_info=True)
@@ -781,5 +799,6 @@ class WorkerManager:
def close(self) -> None:
"""Close the worker manager and database connection."""
self.stop_auto_refresh()
with self._db_lock:
self.db.close()
logger.info("[WorkerManager] Closed")

View File

@@ -60,7 +60,7 @@ class Folder(Store):
if location is None and PATH is not None:
location = str(PATH)
self._location = location
self._location = str(location) if location is not None else ""
self._name = name
# Scan status (set during init)
@@ -221,7 +221,7 @@ class Folder(Store):
# Ensure DB points to the renamed path (update by hash).
try:
cursor.execute(
"UPDATE files SET file_path = ?, updated_at = CURRENT_TIMESTAMP WHERE hash = ?",
"UPDATE file SET file_path = ?, updated_at = CURRENT_TIMESTAMP WHERE hash = ?",
(db._to_db_file_path(hash_path),
file_hash),
)
@@ -269,9 +269,9 @@ class Folder(Store):
cursor.execute(
"""
SELECT f.hash, f.file_path
FROM files f
FROM file f
WHERE NOT EXISTS (
SELECT 1 FROM tags t WHERE t.hash = f.hash AND LOWER(t.tag) LIKE 'title:%'
SELECT 1 FROM tag t WHERE t.hash = f.hash AND LOWER(t.tag) LIKE 'title:%'
)
"""
)
@@ -298,7 +298,7 @@ class Folder(Store):
# Third pass: discover files on disk that aren't in the database yet
# These are hash-named files that were added after initial indexing
cursor.execute("SELECT LOWER(hash) FROM files")
cursor.execute("SELECT LOWER(hash) FROM file")
db_hashes = {row[0]
for row in cursor.fetchall()}
@@ -484,10 +484,17 @@ class Folder(Store):
except Exception:
duration_value = None
# Save to database
# Save to database (metadata + tag/url updates share one connection)
with API_folder_store(Path(self._location)) as db:
db.get_or_create_file_entry(save_file)
# Save metadata including extension
conn = getattr(db, "connection", None)
if conn is None:
raise RuntimeError("Folder store DB connection unavailable")
cursor = conn.cursor()
debug(
f"[Folder.add_file] saving metadata for hash {file_hash}",
file=sys.stderr,
)
ext_clean = file_ext.lstrip(".") if file_ext else ""
db.save_metadata(
save_file,
@@ -498,14 +505,77 @@ class Folder(Store):
"duration": duration_value,
},
)
debug(
f"[Folder.add_file] metadata stored for hash {file_hash}",
file=sys.stderr,
)
# Add tags if provided
if tag_list:
self.add_tag(file_hash, tag_list)
try:
debug(
f"[Folder.add_file] merging {len(tag_list)} tags for {file_hash}",
file=sys.stderr,
)
from SYS.metadata import compute_namespaced_tag_overwrite
existing_tags = [
t for t in (db.get_tags(file_hash) or [])
if isinstance(t, str) and t.strip()
]
_to_remove, _to_add, merged = compute_namespaced_tag_overwrite(
existing_tags, tag_list or []
)
if _to_remove or _to_add:
cursor.execute("DELETE FROM tag WHERE hash = ?",
(file_hash,))
for t in merged:
tag_val = str(t).strip().lower()
if tag_val:
cursor.execute(
"INSERT OR IGNORE INTO tag (hash, tag) VALUES (?, ?)",
(file_hash, tag_val),
)
conn.commit()
debug(
f"[Folder.add_file] tags rewritten for {file_hash}",
file=sys.stderr,
)
try:
db._update_metadata_modified_time(file_hash)
except Exception:
pass
except Exception as exc:
debug(f"Local DB tag merge failed: {exc}", file=sys.stderr)
# Add url if provided
if url:
self.add_url(file_hash, url)
try:
debug(
f"[Folder.add_file] merging {len(url)} URLs for {file_hash}",
file=sys.stderr,
)
from SYS.metadata import normalize_urls
existing_meta = db.get_metadata(file_hash) or {}
existing_urls = normalize_urls(existing_meta.get("url"))
incoming_urls = normalize_urls(url)
changed = False
for entry in list(incoming_urls or []):
if not entry:
continue
if entry not in existing_urls:
existing_urls.append(entry)
changed = True
if changed:
db.update_metadata_by_hash(
file_hash,
{"url": existing_urls},
)
debug(
f"[Folder.add_file] URLs merged for {file_hash}",
file=sys.stderr,
)
except Exception as exc:
debug(f"Local DB URL merge failed: {exc}", file=sys.stderr)
##log(f"✓ Added to local storage: {save_file.name}", file=sys.stderr)
return file_hash
@@ -1373,6 +1443,34 @@ class Folder(Store):
debug(f"Failed to get metadata for hash {file_hash}: {exc}")
return None
def set_relationship(self, alt_hash: str, king_hash: str, kind: str = "alt") -> bool:
"""Persist a relationship in the folder store DB.
This is a thin wrapper around the folder DB API so cmdlets can avoid
backend-specific branching.
"""
try:
if not self._location:
return False
alt_norm = _normalize_hash(alt_hash)
king_norm = _normalize_hash(king_hash)
if not alt_norm or not king_norm or alt_norm == king_norm:
return False
from API.folder import API_folder_store
with API_folder_store(Path(self._location).expanduser()) as db:
db.set_relationship_by_hash(
alt_norm,
king_norm,
str(kind or "alt"),
bidirectional=False,
)
return True
except Exception:
return False
def get_tag(self, file_identifier: str, **kwargs: Any) -> Tuple[List[str], str]:
"""Get tags for a local file by hash.
@@ -1432,14 +1530,14 @@ class Folder(Store):
# Folder DB tag table is case-sensitive and add_tags_to_hash() is additive.
# To enforce lowercase-only tags and namespace overwrites, rewrite the full tag set.
cursor = db.connection.cursor()
cursor.execute("DELETE FROM tags WHERE hash = ?",
cursor.execute("DELETE FROM tag WHERE hash = ?",
(hash,
))
for t in merged:
t = str(t).strip().lower()
if t:
cursor.execute(
"INSERT OR IGNORE INTO tags (hash, tag) VALUES (?, ?)",
"INSERT OR IGNORE INTO tag (hash, tag) VALUES (?, ?)",
(hash,
t),
)
@@ -1953,7 +2051,7 @@ class Folder(Store):
placeholders = ",".join(["?"] * len(chunk))
try:
cursor.execute(
f"SELECT hash, file_path FROM files WHERE hash IN ({placeholders})",
f"SELECT hash, file_path FROM file WHERE hash IN ({placeholders})",
chunk,
)
rows = cursor.fetchall() or []
@@ -1987,13 +2085,13 @@ class Folder(Store):
# Prefer upsert when supported, else fall back to INSERT OR REPLACE.
try:
cursor.executemany(
"INSERT INTO notes (hash, name, note) VALUES (?, ?, ?) "
"INSERT INTO note (hash, name, note) VALUES (?, ?, ?) "
"ON CONFLICT(hash, name) DO UPDATE SET note = excluded.note, updated_at = CURRENT_TIMESTAMP",
inserts,
)
except Exception:
cursor.executemany(
"INSERT OR REPLACE INTO notes (hash, name, note) VALUES (?, ?, ?)",
"INSERT OR REPLACE INTO note (hash, name, note) VALUES (?, ?, ?)",
inserts,
)

View File

@@ -218,6 +218,23 @@ class HydrusNetwork(Store):
def get_name(self) -> str:
return self.NAME
def set_relationship(self, alt_hash: str, king_hash: str, kind: str = "alt") -> bool:
"""Persist a relationship via the Hydrus client API for this backend instance."""
try:
alt_norm = str(alt_hash or "").strip().lower()
king_norm = str(king_hash or "").strip().lower()
if len(alt_norm) != 64 or len(king_norm) != 64 or alt_norm == king_norm:
return False
client = getattr(self, "_client", None)
if client is None or not hasattr(client, "set_relationship"):
return False
client.set_relationship(alt_norm, king_norm, str(kind or "alt"))
return True
except Exception:
return False
def add_file(self, file_path: Path, **kwargs: Any) -> str:
"""Upload file to Hydrus with full metadata support.
@@ -284,9 +301,8 @@ class HydrusNetwork(Store):
file_exists = True
break
if file_exists:
log(
f" Duplicate detected - file already in Hydrus with hash: {file_hash}",
file=sys.stderr,
debug(
f"{self._log_prefix()} Duplicate detected - file already in Hydrus with hash: {file_hash}"
)
except Exception:
pass
@@ -301,9 +317,8 @@ class HydrusNetwork(Store):
# Upload file if not already present
if not file_exists:
log(
f"{self._log_prefix()} Uploading: {file_path.name}",
file=sys.stderr
debug(
f"{self._log_prefix()} Uploading: {file_path.name}"
)
response = client.add_file(file_path)
@@ -320,7 +335,7 @@ class HydrusNetwork(Store):
raise Exception(f"Hydrus response missing file hash: {response}")
file_hash = hydrus_hash
log(f"{self._log_prefix()} hash: {file_hash}", file=sys.stderr)
debug(f"{self._log_prefix()} hash: {file_hash}")
# Add tags if provided (both for new and existing files)
if tag_list:
@@ -335,9 +350,8 @@ class HydrusNetwork(Store):
f"{self._log_prefix()} Adding {len(tag_list)} tag(s): {tag_list}"
)
client.add_tag(file_hash, tag_list, service_name)
log(
f"{self._log_prefix()} Tags added via '{service_name}'",
file=sys.stderr
debug(
f"{self._log_prefix()} Tags added via '{service_name}'"
)
except Exception as exc:
log(
@@ -347,9 +361,8 @@ class HydrusNetwork(Store):
# Associate url if provided (both for new and existing files)
if url:
log(
f"{self._log_prefix()} Associating {len(url)} URL(s) with file",
file=sys.stderr
debug(
f"{self._log_prefix()} Associating {len(url)} URL(s) with file"
)
for url in url:
if url:

View File

@@ -186,7 +186,7 @@ class SharedArgs:
name="path",
type="string",
choices=[], # Dynamically populated via get_store_choices()
description="Selects store",
description="selects store",
)
URL = CmdletArg(
@@ -194,6 +194,11 @@ class SharedArgs:
type="string",
description="http parser",
)
PROVIDER = CmdletArg(
name="provider",
type="string",
description="selects provider",
)
@staticmethod
def get_store_choices(config: Optional[Dict[str, Any]] = None) -> List[str]:

File diff suppressed because it is too large Load Diff

View File

@@ -23,6 +23,14 @@ from SYS.utils import sha256_file
class Add_Note(Cmdlet):
DEFAULT_QUERY_HINTS = (
"title:",
"text:",
"hash:",
"caption:",
"sub:",
"subtitle:",
)
def __init__(self) -> None:
super().__init__(
@@ -124,6 +132,45 @@ class Add_Note(Cmdlet):
note_text = text_match.group(1).strip() if text_match else ""
return (note_name or None, note_text or None)
@classmethod
def _looks_like_note_query_token(cls, token: Any) -> bool:
text = str(token or "").strip().lower()
if not text:
return False
return any(hint in text for hint in cls.DEFAULT_QUERY_HINTS)
@classmethod
def _default_query_args(cls, args: Sequence[str]) -> List[str]:
tokens: List[str] = list(args or [])
lower_tokens = {str(tok).lower() for tok in tokens if tok is not None}
if "-query" in lower_tokens or "--query" in lower_tokens:
return tokens
for idx, tok in enumerate(tokens):
token_text = str(tok or "")
if not token_text or token_text.startswith("-"):
continue
if not cls._looks_like_note_query_token(token_text):
continue
combined_parts = [token_text]
end = idx + 1
while end < len(tokens):
next_text = str(tokens[end] or "")
if not next_text or next_text.startswith("-"):
break
if not cls._looks_like_note_query_token(next_text):
break
combined_parts.append(next_text)
end += 1
combined_query = " ".join(combined_parts)
tokens[idx:end] = [combined_query]
tokens.insert(idx, "-query")
return tokens
return tokens
def _resolve_hash(
self,
raw_hash: Optional[str],
@@ -153,11 +200,14 @@ class Add_Note(Cmdlet):
log(f"Cmdlet: {self.name}\nSummary: {self.summary}\nUsage: {self.usage}")
return 0
parsed = parse_cmdlet_args(args, self)
parsed_args = self._default_query_args(args)
parsed = parse_cmdlet_args(parsed_args, self)
store_override = parsed.get("store")
hash_override = normalize_hash(parsed.get("hash"))
note_name, note_text = self._parse_note_query(str(parsed.get("query") or ""))
note_name = str(note_name or "").strip()
note_text = str(note_text or "").strip()
if not note_name or not note_text:
log(
"[add_note] Error: -query must include title:<title> and text:<text>",
@@ -173,7 +223,6 @@ class Add_Note(Cmdlet):
return 1
explicit_target = bool(hash_override and store_override)
results = normalize_result_input(result)
if results and explicit_target:
# Direct targeting mode: apply note once to the explicit target and
@@ -194,13 +243,21 @@ class Add_Note(Cmdlet):
f"✓ add-note: 1 item in '{store_override}'",
file=sys.stderr
)
except Exception as exc:
log(f"[add_note] Error: Failed to set note: {exc}", file=sys.stderr)
return 1
log(
"[add_note] Updated 1/1 item(s)",
file=sys.stderr
)
for res in results:
ctx.emit(res)
return 0
log(
"[add_note] Warning: Note write reported failure",
file=sys.stderr
)
return 1
except Exception as exc:
log(f"[add_note] Error: Failed to set note: {exc}", file=sys.stderr)
return 1
if not results:
if explicit_target:
@@ -217,7 +274,7 @@ class Add_Note(Cmdlet):
return 1
store_registry = Store(config)
updated = 0
planned_ops = 0
# Batch write plan: store -> [(hash, name, text), ...]
note_ops: Dict[str,
@@ -271,12 +328,12 @@ class Add_Note(Cmdlet):
[]).append((resolved_hash,
note_name,
item_note_text))
updated += 1
planned_ops += 1
ctx.emit(res)
# Execute bulk writes per store.
wrote_any = False
successful_writes = 0
for store_name, ops in note_ops.items():
if not ops:
continue
@@ -285,16 +342,23 @@ class Add_Note(Cmdlet):
except Exception:
continue
store_success = 0
bulk_fn = getattr(backend, "set_note_bulk", None)
if callable(bulk_fn):
try:
ok = bool(bulk_fn(list(ops), config=config))
wrote_any = wrote_any or ok or True
if ok:
store_success += len(ops)
ctx.print_if_visible(
f"✓ add-note: {len(ops)} item(s) in '{store_name}'",
file=sys.stderr
)
successful_writes += store_success
continue
log(
f"[add_note] Warning: bulk set_note returned False for '{store_name}'",
file=sys.stderr,
)
except Exception as exc:
log(
f"[add_note] Warning: bulk set_note failed for '{store_name}': {exc}; falling back",
@@ -305,12 +369,23 @@ class Add_Note(Cmdlet):
for file_hash, name, text in ops:
try:
ok = bool(backend.set_note(file_hash, name, text, config=config))
wrote_any = wrote_any or ok
if ok:
store_success += 1
except Exception:
continue
log(f"[add_note] Updated {updated} item(s)", file=sys.stderr)
return 0 if (updated > 0 and wrote_any) else (0 if updated > 0 else 1)
if store_success:
successful_writes += store_success
ctx.print_if_visible(
f"✓ add-note: {store_success} item(s) in '{store_name}'",
file=sys.stderr
)
log(
f"[add_note] Updated {successful_writes}/{planned_ops} item(s)",
file=sys.stderr
)
return 0 if successful_writes > 0 else 1
CMDLET = Add_Note()

View File

@@ -33,6 +33,7 @@ from rich.prompt import Confirm
from tool.ytdlp import (
YtDlpTool,
_best_subtitle_sidecar,
_SUBTITLE_EXTS,
_download_with_timeout,
_format_chapters_note,
_read_text_file,
@@ -2413,7 +2414,7 @@ class Download_File(Cmdlet):
except Exception:
continue
try:
if p_path.suffix.lower() in _best_subtitle_sidecar.__defaults__[0]:
if p_path.suffix.lower() in _SUBTITLE_EXTS:
continue
except Exception:
pass
@@ -2936,6 +2937,223 @@ class Download_File(Cmdlet):
"media_kind": "video" if opts.mode == "video" else "audio",
}
@staticmethod
def download_streaming_url_as_pipe_objects(
url: str,
config: Dict[str, Any],
*,
mode_hint: Optional[str] = None,
ytdl_format_hint: Optional[str] = None,
) -> List[Dict[str, Any]]:
"""Download a yt-dlp-supported URL and return PipeObject-style dict(s).
This is a lightweight helper intended for cmdlets that need to expand streaming URLs
into local files without re-implementing yt-dlp glue.
"""
url_str = str(url or "").strip()
if not url_str:
return []
if not is_url_supported_by_ytdlp(url_str):
return []
try:
from SYS.config import resolve_output_dir
out_dir = resolve_output_dir(config)
if out_dir is None:
return []
except Exception:
return []
cookies_path = None
try:
cookie_candidate = YtDlpTool(config).resolve_cookiefile()
if cookie_candidate is not None and cookie_candidate.is_file():
cookies_path = cookie_candidate
except Exception:
cookies_path = None
quiet_download = False
try:
quiet_download = bool((config or {}).get("_quiet_background_output"))
except Exception:
quiet_download = False
mode = str(mode_hint or "").strip().lower() if mode_hint else ""
if mode not in {"audio", "video"}:
mode = "video"
try:
cf = (
str(cookies_path)
if cookies_path is not None and cookies_path.is_file() else None
)
fmts_probe = list_formats(
url_str,
no_playlist=False,
playlist_items=None,
cookiefile=cf,
)
if isinstance(fmts_probe, list) and fmts_probe:
has_video = False
for f in fmts_probe:
if not isinstance(f, dict):
continue
vcodec = str(f.get("vcodec", "none") or "none").strip().lower()
if vcodec and vcodec != "none":
has_video = True
break
mode = "video" if has_video else "audio"
except Exception:
mode = "video"
fmt_hint = str(ytdl_format_hint).strip() if ytdl_format_hint else ""
chosen_format: Optional[str]
if fmt_hint:
chosen_format = fmt_hint
else:
chosen_format = None
if mode == "audio":
chosen_format = "bestaudio/best"
opts = DownloadOptions(
url=url_str,
mode=mode,
output_dir=Path(out_dir),
cookies_path=cookies_path,
ytdl_format=chosen_format,
quiet=quiet_download,
embed_chapters=True,
write_sub=True,
)
try:
result_obj = _download_with_timeout(opts, timeout_seconds=300)
except Exception as exc:
log(f"[download-file] Download failed for {url_str}: {exc}", file=sys.stderr)
return []
results: List[Any]
if isinstance(result_obj, list):
results = list(result_obj)
else:
paths = getattr(result_obj, "paths", None)
if isinstance(paths, list) and paths:
results = []
for p in paths:
try:
p_path = Path(p)
except Exception:
continue
if not p_path.exists() or p_path.is_dir():
continue
try:
hv = sha256_file(p_path)
except Exception:
hv = None
try:
results.append(
DownloadMediaResult(
path=p_path,
info=getattr(result_obj, "info", {}) or {},
tag=list(getattr(result_obj, "tag", []) or []),
source_url=getattr(result_obj, "source_url", None) or url_str,
hash_value=hv,
)
)
except Exception:
continue
else:
results = [result_obj]
out: List[Dict[str, Any]] = []
for downloaded in results:
try:
info = (
downloaded.info
if isinstance(getattr(downloaded, "info", None), dict) else {}
)
except Exception:
info = {}
try:
media_path = Path(str(getattr(downloaded, "path", "") or ""))
except Exception:
continue
if not media_path.exists() or media_path.is_dir():
continue
try:
hash_value = getattr(downloaded, "hash_value", None) or sha256_file(media_path)
except Exception:
hash_value = None
title = None
try:
title = info.get("title")
except Exception:
title = None
title = title or media_path.stem
tags = list(getattr(downloaded, "tag", []) or [])
if title and f"title:{title}" not in tags:
tags.insert(0, f"title:{title}")
final_url = None
try:
page_url = info.get("webpage_url") or info.get("original_url") or info.get("url")
if page_url:
final_url = str(page_url)
except Exception:
final_url = None
if not final_url:
final_url = url_str
po: Dict[str, Any] = {
"path": str(media_path),
"hash": hash_value,
"title": title,
"url": final_url,
"tag": tags,
"action": "cmdlet:download-file",
"is_temp": True,
"ytdl_format": getattr(opts, "ytdl_format", None),
"store": getattr(opts, "storage_name", None) or getattr(opts, "storage_location", None) or "PATH",
"media_kind": "video" if opts.mode == "video" else "audio",
}
try:
chapters_text = _format_chapters_note(info)
except Exception:
chapters_text = None
if chapters_text:
notes = po.get("notes")
if not isinstance(notes, dict):
notes = {}
notes.setdefault("chapters", chapters_text)
po["notes"] = notes
try:
sub_path = _best_subtitle_sidecar(media_path)
except Exception:
sub_path = None
if sub_path is not None:
sub_text = _read_text_file(sub_path)
if sub_text:
notes = po.get("notes")
if not isinstance(notes, dict):
notes = {}
notes["sub"] = sub_text
po["notes"] = notes
try:
sub_path.unlink()
except Exception:
pass
out.append(po)
return out
@staticmethod
def _normalise_hash_hex(value: Optional[str]) -> Optional[str]:
if not value or not isinstance(value, str):

View File

@@ -191,7 +191,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
mode_hint = None
forced_format = None
from cmdlet.add_file import Add_File
from cmdlet.download_file import Download_File
expanded: List[Dict[str, Any]] = []
downloaded_any = False
@@ -204,7 +204,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
expanded.append(it)
continue
downloaded = Add_File._download_streaming_url_as_pipe_objects(
downloaded = Download_File.download_streaming_url_as_pipe_objects(
u,
config,
mode_hint=mode_hint,

View File

@@ -1,10 +1,11 @@
from __future__ import annotations
from typing import Any, Dict, Sequence, List, Optional
from typing import Any, Dict, Sequence, List, Optional, Tuple
import shlex
import sys
from cmdlet._shared import Cmdlet, CmdletArg, parse_cmdlet_args
from cmdlet import REGISTRY as CMDLET_REGISTRY, ensure_cmdlet_modules_loaded
from SYS.logger import log
from SYS.result_table import ResultTable
from SYS import pipeline as ctx
@@ -27,6 +28,118 @@ def _examples_for_cmd(name: str) -> List[str]:
return lookup.get(key, [])
def _normalize_cmdlet_key(name: Optional[str]) -> str:
return str(name or "").replace("_", "-").lower().strip()
def _cmdlet_aliases(cmdlet_obj: Cmdlet) -> List[str]:
aliases: List[str] = []
for attr in ("alias", "aliases"):
raw_aliases = getattr(cmdlet_obj, attr, None)
if isinstance(raw_aliases, (list, tuple, set)):
candidates = raw_aliases
else:
candidates = (raw_aliases,)
for alias in candidates or ():
text = str(alias or "").strip()
if text:
aliases.append(text)
seen: set[str] = set()
deduped: List[str] = []
for alias in aliases:
key = alias.lower()
if key in seen:
continue
seen.add(key)
deduped.append(alias)
return deduped
def _cmdlet_arg_to_dict(arg: CmdletArg) -> Dict[str, Any]:
return {
"name": str(getattr(arg, "name", "") or ""),
"type": str(getattr(arg, "type", "") or ""),
"required": bool(getattr(arg, "required", False)),
"description": str(getattr(arg, "description", "") or ""),
"choices": [str(c) for c in list(getattr(arg, "choices", []) or [])],
"alias": str(getattr(arg, "alias", "") or ""),
"variadic": bool(getattr(arg, "variadic", False)),
"usage": str(getattr(arg, "usage", "") or ""),
"query_key": getattr(arg, "query_key", None),
"query_aliases": [str(c) for c in list(getattr(arg, "query_aliases", []) or [])],
"query_only": bool(getattr(arg, "query_only", False)),
"requires_db": bool(getattr(arg, "requires_db", False)),
}
def _build_alias_map_from_metadata(metadata: Dict[str, Dict[str, Any]]) -> Dict[str, str]:
mapping: Dict[str, str] = {}
for name, meta in metadata.items():
canonical = _normalize_cmdlet_key(name)
if canonical:
mapping[canonical] = name
for alias in meta.get("aliases", []) or []:
alias_key = _normalize_cmdlet_key(alias)
if alias_key:
mapping[alias_key] = name
return mapping
def _gather_metadata_from_cmdlet_classes() -> Tuple[Dict[str, Dict[str, Any]], Dict[str, str]]:
metadata: Dict[str, Dict[str, Any]] = {}
alias_map: Dict[str, str] = {}
try:
ensure_cmdlet_modules_loaded()
except Exception:
pass
for module in list(sys.modules.values()):
mod_name = getattr(module, "__name__", "") or ""
if not (mod_name.startswith("cmdlet.") or mod_name == "cmdlet" or mod_name.startswith("cmdnat.")):
continue
cmdlet_obj = getattr(module, "CMDLET", None)
if not isinstance(cmdlet_obj, Cmdlet):
continue
canonical_key = _normalize_cmdlet_key(getattr(cmdlet_obj, "name", None) or "")
if not canonical_key:
continue
entry = {
"name": str(getattr(cmdlet_obj, "name", "") or canonical_key),
"summary": str(getattr(cmdlet_obj, "summary", "") or ""),
"usage": str(getattr(cmdlet_obj, "usage", "") or ""),
"aliases": _cmdlet_aliases(cmdlet_obj),
"details": list(getattr(cmdlet_obj, "detail", []) or []),
"args": [_cmdlet_arg_to_dict(a) for a in getattr(cmdlet_obj, "arg", []) or []],
"raw": getattr(cmdlet_obj, "raw", None),
}
metadata[canonical_key] = entry
alias_map[canonical_key] = canonical_key
for alias in entry["aliases"]:
alias_key = _normalize_cmdlet_key(alias)
if alias_key:
alias_map[alias_key] = canonical_key
for registry_name in CMDLET_REGISTRY.keys():
normalized = _normalize_cmdlet_key(registry_name)
if not normalized or normalized in alias_map:
continue
alias_map[normalized] = normalized
metadata.setdefault(
normalized,
{
"name": normalized,
"aliases": [],
"usage": "",
"summary": "",
"details": [],
"args": [],
"raw": None,
},
)
return metadata, alias_map
def _find_cmd_metadata(name: str,
metadata: Dict[str,
Dict[str,
@@ -148,17 +261,90 @@ def _render_detail(meta: Dict[str, Any], args: Sequence[str]) -> None:
def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
catalog: Any | None = None
cmdlet_names: List[str] = []
metadata: Dict[str, Dict[str, Any]] = {}
alias_map: Dict[str, str] = {}
try:
import cmdlet_catalog as _catalog
CMDLET.arg[0].choices = _normalize_choice_list(
_catalog.list_cmdlet_names(config=config)
)
metadata = _catalog.list_cmdlet_metadata(config=config)
catalog = _catalog
except Exception:
catalog = None
if catalog is not None:
try:
cmdlet_names = catalog.list_cmdlet_names(config=config)
except Exception:
cmdlet_names = []
try:
metadata = catalog.list_cmdlet_metadata(config=config)
except Exception:
CMDLET.arg[0].choices = []
metadata = {}
if metadata:
alias_map = _build_alias_map_from_metadata(metadata)
else:
metadata, alias_map = _gather_metadata_from_cmdlet_classes()
if not metadata:
fallback_names = sorted(set(cmdlet_names or list(CMDLET_REGISTRY.keys())))
if fallback_names:
base_meta: Dict[str, Dict[str, Any]] = {}
for cmdname in fallback_names:
canonical = str(cmdname or "").replace("_", "-").lower()
entry: Dict[str, Any]
candidate: Dict[str, Any] | None = None
if catalog is not None:
try:
candidate = catalog.get_cmdlet_metadata(cmdname, config=config)
except Exception:
candidate = None
if candidate:
canonical = candidate.get("name", canonical)
entry = candidate
else:
entry = {
"name": canonical,
"aliases": [],
"usage": "",
"summary": "",
"details": [],
"args": [],
"raw": None,
}
base = base_meta.setdefault(
canonical,
{
"name": canonical,
"aliases": [],
"usage": "",
"summary": "",
"details": [],
"args": [],
"raw": entry.get("raw"),
},
)
if entry.get("aliases"):
base_aliases = set(base.get("aliases", []))
base_aliases.update([a for a in entry.get("aliases", []) if a])
base["aliases"] = sorted(base_aliases)
if not base.get("usage") and entry.get("usage"):
base["usage"] = entry["usage"]
if not base.get("summary") and entry.get("summary"):
base["summary"] = entry["summary"]
if not base.get("details") and entry.get("details"):
base["details"] = entry["details"]
if not base.get("args") and entry.get("args"):
base["args"] = entry["args"]
if not base.get("raw") and entry.get("raw"):
base["raw"] = entry["raw"]
metadata = base_meta
alias_map = _build_alias_map_from_metadata(metadata)
choice_candidates = list(alias_map.keys()) if alias_map else list(metadata.keys())
CMDLET.arg[0].choices = _normalize_choice_list(choice_candidates)
parsed = parse_cmdlet_args(args, CMDLET)
filter_text = parsed.get("filter")