j
This commit is contained in:
597
API/folder.py
597
API/folder.py
File diff suppressed because it is too large
Load Diff
23
CLI.py
23
CLI.py
@@ -3824,9 +3824,28 @@ class PipelineExecutor:
|
|||||||
# Fall back to default selection rendering on any failure.
|
# Fall back to default selection rendering on any failure.
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
items = piped_result if isinstance(piped_result, list) else [piped_result]
|
||||||
|
|
||||||
|
# Special-case: selecting notes should show the text content directly.
|
||||||
|
note_like_items = [
|
||||||
|
i for i in items
|
||||||
|
if isinstance(i, dict) and ("note_text" in i or "note" in i)
|
||||||
|
]
|
||||||
|
if note_like_items:
|
||||||
|
for idx, item in enumerate(note_like_items, 1):
|
||||||
|
note_name = str(
|
||||||
|
item.get("note_name")
|
||||||
|
or item.get("name")
|
||||||
|
or f"note {idx}"
|
||||||
|
).strip()
|
||||||
|
note_text = str(item.get("note_text") or item.get("note") or "")
|
||||||
|
note_text = note_text[:999]
|
||||||
|
stdout_console().print()
|
||||||
|
stdout_console().print(f"{note_name}:\n{note_text}")
|
||||||
|
ctx.set_last_result_items_only(items)
|
||||||
|
return
|
||||||
|
|
||||||
table = ResultTable("Selection Result")
|
table = ResultTable("Selection Result")
|
||||||
items = piped_result if isinstance(piped_result,
|
|
||||||
list) else [piped_result]
|
|
||||||
for item in items:
|
for item in items:
|
||||||
table.add_result(item)
|
table.add_result(item)
|
||||||
ctx.set_last_result_items_only(items)
|
ctx.set_last_result_items_only(items)
|
||||||
|
|||||||
220
SYS/cmdlet_api.py
Normal file
220
SYS/cmdlet_api.py
Normal file
@@ -0,0 +1,220 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import contextlib
|
||||||
|
import io
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from typing import Any, Callable, Dict, List, Optional, Sequence
|
||||||
|
|
||||||
|
from SYS import pipeline as ctx
|
||||||
|
from SYS.models import PipelineStageContext
|
||||||
|
from SYS.rich_display import capture_rich_output
|
||||||
|
|
||||||
|
|
||||||
|
CmdletCallable = Callable[[Any, Sequence[str], Dict[str, Any]], int]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class CmdletRunResult:
|
||||||
|
"""Programmatic result for a single cmdlet invocation."""
|
||||||
|
|
||||||
|
name: str
|
||||||
|
args: Sequence[str]
|
||||||
|
exit_code: int = 0
|
||||||
|
emitted: List[Any] = field(default_factory=list)
|
||||||
|
|
||||||
|
# Best-effort: cmdlets can publish tables/items via pipeline state even when
|
||||||
|
# they don't emit pipeline items.
|
||||||
|
result_table: Optional[Any] = None
|
||||||
|
result_items: List[Any] = field(default_factory=list)
|
||||||
|
result_subject: Optional[Any] = None
|
||||||
|
|
||||||
|
stdout: str = ""
|
||||||
|
stderr: str = ""
|
||||||
|
error: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_cmd_name(name: str) -> str:
|
||||||
|
return str(name or "").replace("_", "-").strip().lower()
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_cmdlet(cmd_name: str) -> Optional[CmdletCallable]:
|
||||||
|
"""Resolve a cmdlet callable by name from the registry (aliases supported)."""
|
||||||
|
try:
|
||||||
|
from SYS.cmdlet_catalog import ensure_registry_loaded
|
||||||
|
|
||||||
|
ensure_registry_loaded()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
import cmdlet as cmdlet_pkg
|
||||||
|
|
||||||
|
return cmdlet_pkg.get(cmd_name)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def run_cmdlet(
|
||||||
|
cmd: str | CmdletCallable,
|
||||||
|
args: Sequence[str] | None,
|
||||||
|
config: Dict[str, Any],
|
||||||
|
*,
|
||||||
|
piped: Any = None,
|
||||||
|
isolate: bool = True,
|
||||||
|
capture_output: bool = True,
|
||||||
|
stage_index: int = 0,
|
||||||
|
total_stages: int = 1,
|
||||||
|
pipe_index: Optional[int] = None,
|
||||||
|
worker_id: Optional[str] = None,
|
||||||
|
) -> CmdletRunResult:
|
||||||
|
"""Run a single cmdlet programmatically and return structured results.
|
||||||
|
|
||||||
|
This is intended for TUI/webapp consumers that want cmdlet behavior without
|
||||||
|
going through the interactive CLI loop.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
- When `isolate=True` (default) this runs inside `ctx.new_pipeline_state()` so
|
||||||
|
global CLI pipeline state is not mutated.
|
||||||
|
- Output capturing covers both normal `print()` and Rich output via
|
||||||
|
`capture_rich_output()`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
normalized_args: Sequence[str] = list(args or [])
|
||||||
|
|
||||||
|
if isinstance(cmd, str):
|
||||||
|
name = _normalize_cmd_name(cmd)
|
||||||
|
cmd_fn = resolve_cmdlet(name)
|
||||||
|
else:
|
||||||
|
name = getattr(cmd, "__name__", "cmdlet")
|
||||||
|
cmd_fn = cmd
|
||||||
|
|
||||||
|
result = CmdletRunResult(name=name, args=normalized_args)
|
||||||
|
|
||||||
|
if not callable(cmd_fn):
|
||||||
|
result.exit_code = 1
|
||||||
|
result.error = f"Unknown command: {name}"
|
||||||
|
result.stderr = result.error
|
||||||
|
return result
|
||||||
|
|
||||||
|
stage_ctx = PipelineStageContext(
|
||||||
|
stage_index=int(stage_index),
|
||||||
|
total_stages=int(total_stages),
|
||||||
|
pipe_index=pipe_index,
|
||||||
|
worker_id=worker_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
stdout_buffer = io.StringIO()
|
||||||
|
stderr_buffer = io.StringIO()
|
||||||
|
|
||||||
|
stage_text = " ".join([name, *list(normalized_args)]).strip()
|
||||||
|
|
||||||
|
state_cm = ctx.new_pipeline_state() if isolate else contextlib.nullcontext()
|
||||||
|
|
||||||
|
with state_cm:
|
||||||
|
# Keep behavior predictable: start from a clean slate.
|
||||||
|
try:
|
||||||
|
ctx.reset()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
ctx.set_stage_context(stage_ctx)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
ctx.set_current_cmdlet_name(name)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
ctx.set_current_stage_text(stage_text)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
ctx.set_current_command_text(stage_text)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
run_cm = (
|
||||||
|
capture_rich_output(stdout=stdout_buffer, stderr=stderr_buffer)
|
||||||
|
if capture_output
|
||||||
|
else contextlib.nullcontext()
|
||||||
|
)
|
||||||
|
with run_cm:
|
||||||
|
with (
|
||||||
|
contextlib.redirect_stdout(stdout_buffer)
|
||||||
|
if capture_output
|
||||||
|
else contextlib.nullcontext()
|
||||||
|
):
|
||||||
|
with (
|
||||||
|
contextlib.redirect_stderr(stderr_buffer)
|
||||||
|
if capture_output
|
||||||
|
else contextlib.nullcontext()
|
||||||
|
):
|
||||||
|
result.exit_code = int(cmd_fn(piped, list(normalized_args), config))
|
||||||
|
except Exception as exc:
|
||||||
|
result.exit_code = 1
|
||||||
|
result.error = f"{type(exc).__name__}: {exc}"
|
||||||
|
finally:
|
||||||
|
result.stdout = stdout_buffer.getvalue()
|
||||||
|
result.stderr = stderr_buffer.getvalue()
|
||||||
|
|
||||||
|
# Prefer cmdlet emits (pipeline semantics).
|
||||||
|
try:
|
||||||
|
result.emitted = list(stage_ctx.emits or [])
|
||||||
|
except Exception:
|
||||||
|
result.emitted = []
|
||||||
|
|
||||||
|
# Mirror CLI behavior: if cmdlet emitted items and there is no overlay table,
|
||||||
|
# make emitted items the last result items for downstream consumers.
|
||||||
|
try:
|
||||||
|
has_overlay = bool(ctx.get_display_table())
|
||||||
|
except Exception:
|
||||||
|
has_overlay = False
|
||||||
|
|
||||||
|
if result.emitted and not has_overlay:
|
||||||
|
try:
|
||||||
|
ctx.set_last_result_items_only(list(result.emitted))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Best-effort snapshot of visible results.
|
||||||
|
try:
|
||||||
|
result.result_table = (
|
||||||
|
ctx.get_display_table() or ctx.get_current_stage_table() or ctx.get_last_result_table()
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
result.result_table = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
result.result_items = list(ctx.get_last_result_items() or [])
|
||||||
|
except Exception:
|
||||||
|
result.result_items = []
|
||||||
|
|
||||||
|
try:
|
||||||
|
result.result_subject = ctx.get_last_result_subject()
|
||||||
|
except Exception:
|
||||||
|
result.result_subject = None
|
||||||
|
|
||||||
|
# Cleanup stage-local markers.
|
||||||
|
try:
|
||||||
|
ctx.clear_current_stage_text()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
ctx.clear_current_cmdlet_name()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
ctx.clear_current_command_text()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
ctx.set_stage_context(None)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
return result
|
||||||
@@ -512,36 +512,46 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
|
|||||||
if not base_path.exists():
|
if not base_path.exists():
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Ensure file entry exists
|
# Ensure file entry exists (folder store schema is keyed by hash).
|
||||||
file_id: Optional[int] = None
|
file_hash_value: Optional[str] = None
|
||||||
|
if sha256_file and base_path.exists():
|
||||||
|
try:
|
||||||
|
file_hash_value = sha256_file(base_path)
|
||||||
|
except Exception:
|
||||||
|
file_hash_value = None
|
||||||
|
|
||||||
|
if not file_hash_value:
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
db_file_path = (
|
||||||
|
db._to_db_file_path(base_path) # type: ignore[attr-defined]
|
||||||
|
if hasattr(db, "_to_db_file_path")
|
||||||
|
else str(base_path)
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
db_file_path = str(base_path)
|
||||||
|
|
||||||
|
try:
|
||||||
|
file_modified = float(base_path.stat().st_mtime)
|
||||||
|
except Exception:
|
||||||
|
file_modified = None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
cursor = db.connection.cursor() if db.connection else None
|
cursor = db.connection.cursor() if db.connection else None
|
||||||
if cursor:
|
if cursor:
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
"SELECT id FROM files WHERE file_path = ?",
|
"SELECT hash FROM file WHERE file_path = ?",
|
||||||
(str(base_path),
|
(str(db_file_path),),
|
||||||
)
|
|
||||||
)
|
)
|
||||||
result = cursor.fetchone()
|
result = cursor.fetchone()
|
||||||
file_id = result[0] if result else None
|
if not result:
|
||||||
except Exception:
|
|
||||||
file_id = None
|
|
||||||
|
|
||||||
if not file_id:
|
|
||||||
try:
|
|
||||||
cursor = db.connection.cursor() if db.connection else None
|
|
||||||
if cursor:
|
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
'INSERT INTO files (file_path, indexed_at, updated_at) VALUES (?, datetime("now"), datetime("now"))',
|
"INSERT INTO file (hash, file_path, file_modified) VALUES (?, ?, ?)",
|
||||||
(str(base_path),
|
(file_hash_value, str(db_file_path), file_modified),
|
||||||
),
|
|
||||||
)
|
)
|
||||||
db.connection.commit()
|
db.connection.commit()
|
||||||
file_id = cursor.lastrowid
|
except Exception:
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if not file_id:
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if sidecar_path.suffix == ".tag":
|
if sidecar_path.suffix == ".tag":
|
||||||
@@ -557,15 +567,9 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
|
|||||||
try:
|
try:
|
||||||
cursor = db.connection.cursor() if db.connection else None
|
cursor = db.connection.cursor() if db.connection else None
|
||||||
if cursor:
|
if cursor:
|
||||||
file_hash_value: Optional[str] = None
|
|
||||||
if hasattr(db, "get_file_hash"):
|
|
||||||
try:
|
|
||||||
file_hash_value = db.get_file_hash(file_id)
|
|
||||||
except Exception:
|
|
||||||
file_hash_value = None
|
|
||||||
for tag in tags:
|
for tag in tags:
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
"INSERT OR IGNORE INTO tags (hash, tag) VALUES (?, ?)",
|
"INSERT OR IGNORE INTO tag (hash, tag) VALUES (?, ?)",
|
||||||
(file_hash_value,
|
(file_hash_value,
|
||||||
tag),
|
tag),
|
||||||
)
|
)
|
||||||
@@ -608,13 +612,15 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
if not hash_value:
|
||||||
|
hash_value = file_hash_value
|
||||||
|
|
||||||
try:
|
try:
|
||||||
cursor = db.connection.cursor() if db.connection else None
|
cursor = db.connection.cursor() if db.connection else None
|
||||||
if cursor:
|
if cursor:
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
'INSERT OR REPLACE INTO metadata (file_id, hash, url, relationships, time_imported, time_modified) VALUES (?, ?, ?, ?, datetime("now"), datetime("now"))',
|
'INSERT OR REPLACE INTO metadata (hash, url, relationships, time_imported, time_modified) VALUES (?, ?, ?, datetime("now"), datetime("now"))',
|
||||||
(
|
(
|
||||||
file_id,
|
|
||||||
hash_value,
|
hash_value,
|
||||||
json.dumps(url),
|
json.dumps(url),
|
||||||
json.dumps(relationships),
|
json.dumps(relationships),
|
||||||
@@ -634,9 +640,8 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
|
|||||||
cursor = db.connection.cursor() if db.connection else None
|
cursor = db.connection.cursor() if db.connection else None
|
||||||
if cursor:
|
if cursor:
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
'INSERT INTO notes (file_id, note, created_at, updated_at) VALUES (?, ?, datetime("now"), datetime("now")) ON CONFLICT(file_id) DO UPDATE SET note = excluded.note, updated_at = datetime("now")',
|
'INSERT INTO note (hash, name, note, created_at, updated_at) VALUES (?, ?, ?, datetime("now"), datetime("now")) ON CONFLICT(hash, name) DO UPDATE SET note = excluded.note, updated_at = datetime("now")',
|
||||||
(file_id,
|
(file_hash_value, "default", content),
|
||||||
content),
|
|
||||||
)
|
)
|
||||||
db.connection.commit()
|
db.connection.commit()
|
||||||
except Exception:
|
except Exception:
|
||||||
|
|||||||
@@ -263,6 +263,9 @@ class WorkerManager:
|
|||||||
self.refresh_thread: Optional[Thread] = None
|
self.refresh_thread: Optional[Thread] = None
|
||||||
self._stop_refresh = False
|
self._stop_refresh = False
|
||||||
self._lock = Lock()
|
self._lock = Lock()
|
||||||
|
# Reuse the DB's own lock so there is exactly one lock guarding the
|
||||||
|
# sqlite connection (and it is safe for re-entrant/nested DB usage).
|
||||||
|
self._db_lock = self.db._db_lock
|
||||||
self.worker_handlers: Dict[str,
|
self.worker_handlers: Dict[str,
|
||||||
WorkerLoggingHandler] = {} # Track active handlers
|
WorkerLoggingHandler] = {} # Track active handlers
|
||||||
self._worker_last_step: Dict[str,
|
self._worker_last_step: Dict[str,
|
||||||
@@ -272,7 +275,8 @@ class WorkerManager:
|
|||||||
"""Close the database connection."""
|
"""Close the database connection."""
|
||||||
if self.db:
|
if self.db:
|
||||||
try:
|
try:
|
||||||
self.db.close()
|
with self._db_lock:
|
||||||
|
self.db.close()
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@@ -317,12 +321,13 @@ class WorkerManager:
|
|||||||
Count of workers updated.
|
Count of workers updated.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
return self.db.expire_running_workers(
|
with self._db_lock:
|
||||||
older_than_seconds=older_than_seconds,
|
return self.db.expire_running_workers(
|
||||||
status=status,
|
older_than_seconds=older_than_seconds,
|
||||||
reason=reason,
|
status=status,
|
||||||
worker_id_prefix=worker_id_prefix,
|
reason=reason,
|
||||||
)
|
worker_id_prefix=worker_id_prefix,
|
||||||
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
logger.error(f"Failed to expire stale workers: {exc}", exc_info=True)
|
logger.error(f"Failed to expire stale workers: {exc}", exc_info=True)
|
||||||
return 0
|
return 0
|
||||||
@@ -419,14 +424,15 @@ class WorkerManager:
|
|||||||
True if worker was inserted successfully
|
True if worker was inserted successfully
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
result = self.db.insert_worker(
|
with self._db_lock:
|
||||||
worker_id,
|
result = self.db.insert_worker(
|
||||||
worker_type,
|
worker_id,
|
||||||
title,
|
worker_type,
|
||||||
description,
|
title,
|
||||||
total_steps,
|
description,
|
||||||
pipe=pipe
|
total_steps,
|
||||||
)
|
pipe=pipe
|
||||||
|
)
|
||||||
if result > 0:
|
if result > 0:
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"[WorkerManager] Tracking worker: {worker_id} ({worker_type})"
|
f"[WorkerManager] Tracking worker: {worker_id} ({worker_type})"
|
||||||
@@ -473,7 +479,8 @@ class WorkerManager:
|
|||||||
kwargs["last_updated"] = datetime.now().isoformat()
|
kwargs["last_updated"] = datetime.now().isoformat()
|
||||||
if "current_step" in kwargs and kwargs["current_step"]:
|
if "current_step" in kwargs and kwargs["current_step"]:
|
||||||
self._worker_last_step[worker_id] = str(kwargs["current_step"])
|
self._worker_last_step[worker_id] = str(kwargs["current_step"])
|
||||||
return self.db.update_worker(worker_id, **kwargs)
|
with self._db_lock:
|
||||||
|
return self.db.update_worker(worker_id, **kwargs)
|
||||||
return True
|
return True
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(
|
logger.error(
|
||||||
@@ -510,7 +517,8 @@ class WorkerManager:
|
|||||||
if result_data:
|
if result_data:
|
||||||
kwargs["result_data"] = result_data
|
kwargs["result_data"] = result_data
|
||||||
|
|
||||||
success = self.db.update_worker(worker_id, **kwargs)
|
with self._db_lock:
|
||||||
|
success = self.db.update_worker(worker_id, **kwargs)
|
||||||
logger.info(f"[WorkerManager] Worker finished: {worker_id} ({result})")
|
logger.info(f"[WorkerManager] Worker finished: {worker_id} ({result})")
|
||||||
self._worker_last_step.pop(worker_id, None)
|
self._worker_last_step.pop(worker_id, None)
|
||||||
return success
|
return success
|
||||||
@@ -528,7 +536,8 @@ class WorkerManager:
|
|||||||
List of active worker dictionaries
|
List of active worker dictionaries
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
return self.db.get_active_workers()
|
with self._db_lock:
|
||||||
|
return self.db.get_active_workers()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(
|
logger.error(
|
||||||
f"[WorkerManager] Error getting active workers: {e}",
|
f"[WorkerManager] Error getting active workers: {e}",
|
||||||
@@ -546,7 +555,8 @@ class WorkerManager:
|
|||||||
List of finished worker dictionaries
|
List of finished worker dictionaries
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
all_workers = self.db.get_all_workers(limit=limit)
|
with self._db_lock:
|
||||||
|
all_workers = self.db.get_all_workers(limit=limit)
|
||||||
# Filter to only finished workers
|
# Filter to only finished workers
|
||||||
finished = [
|
finished = [
|
||||||
w for w in all_workers
|
w for w in all_workers
|
||||||
@@ -570,7 +580,8 @@ class WorkerManager:
|
|||||||
Worker data or None if not found
|
Worker data or None if not found
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
return self.db.get_worker(worker_id)
|
with self._db_lock:
|
||||||
|
return self.db.get_worker(worker_id)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(
|
logger.error(
|
||||||
f"[WorkerManager] Error getting worker {worker_id}: {e}",
|
f"[WorkerManager] Error getting worker {worker_id}: {e}",
|
||||||
@@ -583,7 +594,8 @@ class WorkerManager:
|
|||||||
limit: int = 500) -> List[Dict[str,
|
limit: int = 500) -> List[Dict[str,
|
||||||
Any]]:
|
Any]]:
|
||||||
"""Fetch recorded worker timeline events."""
|
"""Fetch recorded worker timeline events."""
|
||||||
return self.db.get_worker_events(worker_id, limit)
|
with self._db_lock:
|
||||||
|
return self.db.get_worker_events(worker_id, limit)
|
||||||
|
|
||||||
def log_step(self, worker_id: str, step_text: str) -> bool:
|
def log_step(self, worker_id: str, step_text: str) -> bool:
|
||||||
"""Log a step to a worker's step history.
|
"""Log a step to a worker's step history.
|
||||||
@@ -596,7 +608,8 @@ class WorkerManager:
|
|||||||
True if successful
|
True if successful
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
success = self.db.append_worker_steps(worker_id, step_text)
|
with self._db_lock:
|
||||||
|
success = self.db.append_worker_steps(worker_id, step_text)
|
||||||
if success:
|
if success:
|
||||||
self._worker_last_step[worker_id] = step_text
|
self._worker_last_step[worker_id] = step_text
|
||||||
return success
|
return success
|
||||||
@@ -621,7 +634,8 @@ class WorkerManager:
|
|||||||
Steps text or empty string if not found
|
Steps text or empty string if not found
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
return self.db.get_worker_steps(worker_id)
|
with self._db_lock:
|
||||||
|
return self.db.get_worker_steps(worker_id)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(
|
logger.error(
|
||||||
f"[WorkerManager] Error getting steps for worker {worker_id}: {e}",
|
f"[WorkerManager] Error getting steps for worker {worker_id}: {e}",
|
||||||
@@ -705,7 +719,8 @@ class WorkerManager:
|
|||||||
Number of workers deleted
|
Number of workers deleted
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
count = self.db.cleanup_old_workers(days)
|
with self._db_lock:
|
||||||
|
count = self.db.cleanup_old_workers(days)
|
||||||
if count > 0:
|
if count > 0:
|
||||||
logger.info(f"[WorkerManager] Cleaned up {count} old workers")
|
logger.info(f"[WorkerManager] Cleaned up {count} old workers")
|
||||||
return count
|
return count
|
||||||
@@ -729,12 +744,13 @@ class WorkerManager:
|
|||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
step_label = self._get_last_step(worker_id)
|
step_label = self._get_last_step(worker_id)
|
||||||
return self.db.append_worker_stdout(
|
with self._db_lock:
|
||||||
worker_id,
|
return self.db.append_worker_stdout(
|
||||||
text,
|
worker_id,
|
||||||
step=step_label,
|
text,
|
||||||
channel=channel
|
step=step_label,
|
||||||
)
|
channel=channel
|
||||||
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"[WorkerManager] Error appending stdout: {e}", exc_info=True)
|
logger.error(f"[WorkerManager] Error appending stdout: {e}", exc_info=True)
|
||||||
return False
|
return False
|
||||||
@@ -749,7 +765,8 @@ class WorkerManager:
|
|||||||
Worker's stdout or empty string
|
Worker's stdout or empty string
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
return self.db.get_worker_stdout(worker_id)
|
with self._db_lock:
|
||||||
|
return self.db.get_worker_stdout(worker_id)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"[WorkerManager] Error getting stdout: {e}", exc_info=True)
|
logger.error(f"[WorkerManager] Error getting stdout: {e}", exc_info=True)
|
||||||
return ""
|
return ""
|
||||||
@@ -773,7 +790,8 @@ class WorkerManager:
|
|||||||
True if clear was successful
|
True if clear was successful
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
return self.db.clear_worker_stdout(worker_id)
|
with self._db_lock:
|
||||||
|
return self.db.clear_worker_stdout(worker_id)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"[WorkerManager] Error clearing stdout: {e}", exc_info=True)
|
logger.error(f"[WorkerManager] Error clearing stdout: {e}", exc_info=True)
|
||||||
return False
|
return False
|
||||||
@@ -781,5 +799,6 @@ class WorkerManager:
|
|||||||
def close(self) -> None:
|
def close(self) -> None:
|
||||||
"""Close the worker manager and database connection."""
|
"""Close the worker manager and database connection."""
|
||||||
self.stop_auto_refresh()
|
self.stop_auto_refresh()
|
||||||
self.db.close()
|
with self._db_lock:
|
||||||
|
self.db.close()
|
||||||
logger.info("[WorkerManager] Closed")
|
logger.info("[WorkerManager] Closed")
|
||||||
|
|||||||
136
Store/Folder.py
136
Store/Folder.py
@@ -60,7 +60,7 @@ class Folder(Store):
|
|||||||
if location is None and PATH is not None:
|
if location is None and PATH is not None:
|
||||||
location = str(PATH)
|
location = str(PATH)
|
||||||
|
|
||||||
self._location = location
|
self._location = str(location) if location is not None else ""
|
||||||
self._name = name
|
self._name = name
|
||||||
|
|
||||||
# Scan status (set during init)
|
# Scan status (set during init)
|
||||||
@@ -221,7 +221,7 @@ class Folder(Store):
|
|||||||
# Ensure DB points to the renamed path (update by hash).
|
# Ensure DB points to the renamed path (update by hash).
|
||||||
try:
|
try:
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
"UPDATE files SET file_path = ?, updated_at = CURRENT_TIMESTAMP WHERE hash = ?",
|
"UPDATE file SET file_path = ?, updated_at = CURRENT_TIMESTAMP WHERE hash = ?",
|
||||||
(db._to_db_file_path(hash_path),
|
(db._to_db_file_path(hash_path),
|
||||||
file_hash),
|
file_hash),
|
||||||
)
|
)
|
||||||
@@ -269,9 +269,9 @@ class Folder(Store):
|
|||||||
cursor.execute(
|
cursor.execute(
|
||||||
"""
|
"""
|
||||||
SELECT f.hash, f.file_path
|
SELECT f.hash, f.file_path
|
||||||
FROM files f
|
FROM file f
|
||||||
WHERE NOT EXISTS (
|
WHERE NOT EXISTS (
|
||||||
SELECT 1 FROM tags t WHERE t.hash = f.hash AND LOWER(t.tag) LIKE 'title:%'
|
SELECT 1 FROM tag t WHERE t.hash = f.hash AND LOWER(t.tag) LIKE 'title:%'
|
||||||
)
|
)
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
@@ -298,7 +298,7 @@ class Folder(Store):
|
|||||||
|
|
||||||
# Third pass: discover files on disk that aren't in the database yet
|
# Third pass: discover files on disk that aren't in the database yet
|
||||||
# These are hash-named files that were added after initial indexing
|
# These are hash-named files that were added after initial indexing
|
||||||
cursor.execute("SELECT LOWER(hash) FROM files")
|
cursor.execute("SELECT LOWER(hash) FROM file")
|
||||||
db_hashes = {row[0]
|
db_hashes = {row[0]
|
||||||
for row in cursor.fetchall()}
|
for row in cursor.fetchall()}
|
||||||
|
|
||||||
@@ -484,10 +484,17 @@ class Folder(Store):
|
|||||||
except Exception:
|
except Exception:
|
||||||
duration_value = None
|
duration_value = None
|
||||||
|
|
||||||
# Save to database
|
# Save to database (metadata + tag/url updates share one connection)
|
||||||
with API_folder_store(Path(self._location)) as db:
|
with API_folder_store(Path(self._location)) as db:
|
||||||
db.get_or_create_file_entry(save_file)
|
conn = getattr(db, "connection", None)
|
||||||
# Save metadata including extension
|
if conn is None:
|
||||||
|
raise RuntimeError("Folder store DB connection unavailable")
|
||||||
|
cursor = conn.cursor()
|
||||||
|
|
||||||
|
debug(
|
||||||
|
f"[Folder.add_file] saving metadata for hash {file_hash}",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
ext_clean = file_ext.lstrip(".") if file_ext else ""
|
ext_clean = file_ext.lstrip(".") if file_ext else ""
|
||||||
db.save_metadata(
|
db.save_metadata(
|
||||||
save_file,
|
save_file,
|
||||||
@@ -498,14 +505,77 @@ class Folder(Store):
|
|||||||
"duration": duration_value,
|
"duration": duration_value,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
debug(
|
||||||
|
f"[Folder.add_file] metadata stored for hash {file_hash}",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
|
||||||
# Add tags if provided
|
if tag_list:
|
||||||
if tag_list:
|
try:
|
||||||
self.add_tag(file_hash, tag_list)
|
debug(
|
||||||
|
f"[Folder.add_file] merging {len(tag_list)} tags for {file_hash}",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
from SYS.metadata import compute_namespaced_tag_overwrite
|
||||||
|
|
||||||
# Add url if provided
|
existing_tags = [
|
||||||
if url:
|
t for t in (db.get_tags(file_hash) or [])
|
||||||
self.add_url(file_hash, url)
|
if isinstance(t, str) and t.strip()
|
||||||
|
]
|
||||||
|
_to_remove, _to_add, merged = compute_namespaced_tag_overwrite(
|
||||||
|
existing_tags, tag_list or []
|
||||||
|
)
|
||||||
|
if _to_remove or _to_add:
|
||||||
|
cursor.execute("DELETE FROM tag WHERE hash = ?",
|
||||||
|
(file_hash,))
|
||||||
|
for t in merged:
|
||||||
|
tag_val = str(t).strip().lower()
|
||||||
|
if tag_val:
|
||||||
|
cursor.execute(
|
||||||
|
"INSERT OR IGNORE INTO tag (hash, tag) VALUES (?, ?)",
|
||||||
|
(file_hash, tag_val),
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
debug(
|
||||||
|
f"[Folder.add_file] tags rewritten for {file_hash}",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
db._update_metadata_modified_time(file_hash)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
except Exception as exc:
|
||||||
|
debug(f"Local DB tag merge failed: {exc}", file=sys.stderr)
|
||||||
|
|
||||||
|
if url:
|
||||||
|
try:
|
||||||
|
debug(
|
||||||
|
f"[Folder.add_file] merging {len(url)} URLs for {file_hash}",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
from SYS.metadata import normalize_urls
|
||||||
|
|
||||||
|
existing_meta = db.get_metadata(file_hash) or {}
|
||||||
|
existing_urls = normalize_urls(existing_meta.get("url"))
|
||||||
|
incoming_urls = normalize_urls(url)
|
||||||
|
changed = False
|
||||||
|
for entry in list(incoming_urls or []):
|
||||||
|
if not entry:
|
||||||
|
continue
|
||||||
|
if entry not in existing_urls:
|
||||||
|
existing_urls.append(entry)
|
||||||
|
changed = True
|
||||||
|
if changed:
|
||||||
|
db.update_metadata_by_hash(
|
||||||
|
file_hash,
|
||||||
|
{"url": existing_urls},
|
||||||
|
)
|
||||||
|
debug(
|
||||||
|
f"[Folder.add_file] URLs merged for {file_hash}",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
debug(f"Local DB URL merge failed: {exc}", file=sys.stderr)
|
||||||
|
|
||||||
##log(f"✓ Added to local storage: {save_file.name}", file=sys.stderr)
|
##log(f"✓ Added to local storage: {save_file.name}", file=sys.stderr)
|
||||||
return file_hash
|
return file_hash
|
||||||
@@ -1373,6 +1443,34 @@ class Folder(Store):
|
|||||||
debug(f"Failed to get metadata for hash {file_hash}: {exc}")
|
debug(f"Failed to get metadata for hash {file_hash}: {exc}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def set_relationship(self, alt_hash: str, king_hash: str, kind: str = "alt") -> bool:
|
||||||
|
"""Persist a relationship in the folder store DB.
|
||||||
|
|
||||||
|
This is a thin wrapper around the folder DB API so cmdlets can avoid
|
||||||
|
backend-specific branching.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
if not self._location:
|
||||||
|
return False
|
||||||
|
|
||||||
|
alt_norm = _normalize_hash(alt_hash)
|
||||||
|
king_norm = _normalize_hash(king_hash)
|
||||||
|
if not alt_norm or not king_norm or alt_norm == king_norm:
|
||||||
|
return False
|
||||||
|
|
||||||
|
from API.folder import API_folder_store
|
||||||
|
|
||||||
|
with API_folder_store(Path(self._location).expanduser()) as db:
|
||||||
|
db.set_relationship_by_hash(
|
||||||
|
alt_norm,
|
||||||
|
king_norm,
|
||||||
|
str(kind or "alt"),
|
||||||
|
bidirectional=False,
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
def get_tag(self, file_identifier: str, **kwargs: Any) -> Tuple[List[str], str]:
|
def get_tag(self, file_identifier: str, **kwargs: Any) -> Tuple[List[str], str]:
|
||||||
"""Get tags for a local file by hash.
|
"""Get tags for a local file by hash.
|
||||||
|
|
||||||
@@ -1432,14 +1530,14 @@ class Folder(Store):
|
|||||||
# Folder DB tag table is case-sensitive and add_tags_to_hash() is additive.
|
# Folder DB tag table is case-sensitive and add_tags_to_hash() is additive.
|
||||||
# To enforce lowercase-only tags and namespace overwrites, rewrite the full tag set.
|
# To enforce lowercase-only tags and namespace overwrites, rewrite the full tag set.
|
||||||
cursor = db.connection.cursor()
|
cursor = db.connection.cursor()
|
||||||
cursor.execute("DELETE FROM tags WHERE hash = ?",
|
cursor.execute("DELETE FROM tag WHERE hash = ?",
|
||||||
(hash,
|
(hash,
|
||||||
))
|
))
|
||||||
for t in merged:
|
for t in merged:
|
||||||
t = str(t).strip().lower()
|
t = str(t).strip().lower()
|
||||||
if t:
|
if t:
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
"INSERT OR IGNORE INTO tags (hash, tag) VALUES (?, ?)",
|
"INSERT OR IGNORE INTO tag (hash, tag) VALUES (?, ?)",
|
||||||
(hash,
|
(hash,
|
||||||
t),
|
t),
|
||||||
)
|
)
|
||||||
@@ -1953,7 +2051,7 @@ class Folder(Store):
|
|||||||
placeholders = ",".join(["?"] * len(chunk))
|
placeholders = ",".join(["?"] * len(chunk))
|
||||||
try:
|
try:
|
||||||
cursor.execute(
|
cursor.execute(
|
||||||
f"SELECT hash, file_path FROM files WHERE hash IN ({placeholders})",
|
f"SELECT hash, file_path FROM file WHERE hash IN ({placeholders})",
|
||||||
chunk,
|
chunk,
|
||||||
)
|
)
|
||||||
rows = cursor.fetchall() or []
|
rows = cursor.fetchall() or []
|
||||||
@@ -1987,13 +2085,13 @@ class Folder(Store):
|
|||||||
# Prefer upsert when supported, else fall back to INSERT OR REPLACE.
|
# Prefer upsert when supported, else fall back to INSERT OR REPLACE.
|
||||||
try:
|
try:
|
||||||
cursor.executemany(
|
cursor.executemany(
|
||||||
"INSERT INTO notes (hash, name, note) VALUES (?, ?, ?) "
|
"INSERT INTO note (hash, name, note) VALUES (?, ?, ?) "
|
||||||
"ON CONFLICT(hash, name) DO UPDATE SET note = excluded.note, updated_at = CURRENT_TIMESTAMP",
|
"ON CONFLICT(hash, name) DO UPDATE SET note = excluded.note, updated_at = CURRENT_TIMESTAMP",
|
||||||
inserts,
|
inserts,
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
cursor.executemany(
|
cursor.executemany(
|
||||||
"INSERT OR REPLACE INTO notes (hash, name, note) VALUES (?, ?, ?)",
|
"INSERT OR REPLACE INTO note (hash, name, note) VALUES (?, ?, ?)",
|
||||||
inserts,
|
inserts,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -218,6 +218,23 @@ class HydrusNetwork(Store):
|
|||||||
def get_name(self) -> str:
|
def get_name(self) -> str:
|
||||||
return self.NAME
|
return self.NAME
|
||||||
|
|
||||||
|
def set_relationship(self, alt_hash: str, king_hash: str, kind: str = "alt") -> bool:
|
||||||
|
"""Persist a relationship via the Hydrus client API for this backend instance."""
|
||||||
|
try:
|
||||||
|
alt_norm = str(alt_hash or "").strip().lower()
|
||||||
|
king_norm = str(king_hash or "").strip().lower()
|
||||||
|
if len(alt_norm) != 64 or len(king_norm) != 64 or alt_norm == king_norm:
|
||||||
|
return False
|
||||||
|
|
||||||
|
client = getattr(self, "_client", None)
|
||||||
|
if client is None or not hasattr(client, "set_relationship"):
|
||||||
|
return False
|
||||||
|
|
||||||
|
client.set_relationship(alt_norm, king_norm, str(kind or "alt"))
|
||||||
|
return True
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
def add_file(self, file_path: Path, **kwargs: Any) -> str:
|
def add_file(self, file_path: Path, **kwargs: Any) -> str:
|
||||||
"""Upload file to Hydrus with full metadata support.
|
"""Upload file to Hydrus with full metadata support.
|
||||||
|
|
||||||
@@ -284,9 +301,8 @@ class HydrusNetwork(Store):
|
|||||||
file_exists = True
|
file_exists = True
|
||||||
break
|
break
|
||||||
if file_exists:
|
if file_exists:
|
||||||
log(
|
debug(
|
||||||
f"ℹ️ Duplicate detected - file already in Hydrus with hash: {file_hash}",
|
f"{self._log_prefix()} Duplicate detected - file already in Hydrus with hash: {file_hash}"
|
||||||
file=sys.stderr,
|
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
@@ -301,9 +317,8 @@ class HydrusNetwork(Store):
|
|||||||
|
|
||||||
# Upload file if not already present
|
# Upload file if not already present
|
||||||
if not file_exists:
|
if not file_exists:
|
||||||
log(
|
debug(
|
||||||
f"{self._log_prefix()} Uploading: {file_path.name}",
|
f"{self._log_prefix()} Uploading: {file_path.name}"
|
||||||
file=sys.stderr
|
|
||||||
)
|
)
|
||||||
response = client.add_file(file_path)
|
response = client.add_file(file_path)
|
||||||
|
|
||||||
@@ -320,7 +335,7 @@ class HydrusNetwork(Store):
|
|||||||
raise Exception(f"Hydrus response missing file hash: {response}")
|
raise Exception(f"Hydrus response missing file hash: {response}")
|
||||||
|
|
||||||
file_hash = hydrus_hash
|
file_hash = hydrus_hash
|
||||||
log(f"{self._log_prefix()} hash: {file_hash}", file=sys.stderr)
|
debug(f"{self._log_prefix()} hash: {file_hash}")
|
||||||
|
|
||||||
# Add tags if provided (both for new and existing files)
|
# Add tags if provided (both for new and existing files)
|
||||||
if tag_list:
|
if tag_list:
|
||||||
@@ -335,9 +350,8 @@ class HydrusNetwork(Store):
|
|||||||
f"{self._log_prefix()} Adding {len(tag_list)} tag(s): {tag_list}"
|
f"{self._log_prefix()} Adding {len(tag_list)} tag(s): {tag_list}"
|
||||||
)
|
)
|
||||||
client.add_tag(file_hash, tag_list, service_name)
|
client.add_tag(file_hash, tag_list, service_name)
|
||||||
log(
|
debug(
|
||||||
f"{self._log_prefix()} Tags added via '{service_name}'",
|
f"{self._log_prefix()} Tags added via '{service_name}'"
|
||||||
file=sys.stderr
|
|
||||||
)
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log(
|
log(
|
||||||
@@ -347,9 +361,8 @@ class HydrusNetwork(Store):
|
|||||||
|
|
||||||
# Associate url if provided (both for new and existing files)
|
# Associate url if provided (both for new and existing files)
|
||||||
if url:
|
if url:
|
||||||
log(
|
debug(
|
||||||
f"{self._log_prefix()} Associating {len(url)} URL(s) with file",
|
f"{self._log_prefix()} Associating {len(url)} URL(s) with file"
|
||||||
file=sys.stderr
|
|
||||||
)
|
)
|
||||||
for url in url:
|
for url in url:
|
||||||
if url:
|
if url:
|
||||||
|
|||||||
@@ -186,7 +186,7 @@ class SharedArgs:
|
|||||||
name="path",
|
name="path",
|
||||||
type="string",
|
type="string",
|
||||||
choices=[], # Dynamically populated via get_store_choices()
|
choices=[], # Dynamically populated via get_store_choices()
|
||||||
description="Selects store",
|
description="selects store",
|
||||||
)
|
)
|
||||||
|
|
||||||
URL = CmdletArg(
|
URL = CmdletArg(
|
||||||
@@ -194,6 +194,11 @@ class SharedArgs:
|
|||||||
type="string",
|
type="string",
|
||||||
description="http parser",
|
description="http parser",
|
||||||
)
|
)
|
||||||
|
PROVIDER = CmdletArg(
|
||||||
|
name="provider",
|
||||||
|
type="string",
|
||||||
|
description="selects provider",
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def get_store_choices(config: Optional[Dict[str, Any]] = None) -> List[str]:
|
def get_store_choices(config: Optional[Dict[str, Any]] = None) -> List[str]:
|
||||||
|
|||||||
2025
cmdlet/add_file.py
2025
cmdlet/add_file.py
File diff suppressed because it is too large
Load Diff
@@ -23,6 +23,14 @@ from SYS.utils import sha256_file
|
|||||||
|
|
||||||
|
|
||||||
class Add_Note(Cmdlet):
|
class Add_Note(Cmdlet):
|
||||||
|
DEFAULT_QUERY_HINTS = (
|
||||||
|
"title:",
|
||||||
|
"text:",
|
||||||
|
"hash:",
|
||||||
|
"caption:",
|
||||||
|
"sub:",
|
||||||
|
"subtitle:",
|
||||||
|
)
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
super().__init__(
|
super().__init__(
|
||||||
@@ -124,6 +132,45 @@ class Add_Note(Cmdlet):
|
|||||||
note_text = text_match.group(1).strip() if text_match else ""
|
note_text = text_match.group(1).strip() if text_match else ""
|
||||||
return (note_name or None, note_text or None)
|
return (note_name or None, note_text or None)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _looks_like_note_query_token(cls, token: Any) -> bool:
|
||||||
|
text = str(token or "").strip().lower()
|
||||||
|
if not text:
|
||||||
|
return False
|
||||||
|
return any(hint in text for hint in cls.DEFAULT_QUERY_HINTS)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _default_query_args(cls, args: Sequence[str]) -> List[str]:
|
||||||
|
tokens: List[str] = list(args or [])
|
||||||
|
lower_tokens = {str(tok).lower() for tok in tokens if tok is not None}
|
||||||
|
if "-query" in lower_tokens or "--query" in lower_tokens:
|
||||||
|
return tokens
|
||||||
|
|
||||||
|
for idx, tok in enumerate(tokens):
|
||||||
|
token_text = str(tok or "")
|
||||||
|
if not token_text or token_text.startswith("-"):
|
||||||
|
continue
|
||||||
|
if not cls._looks_like_note_query_token(token_text):
|
||||||
|
continue
|
||||||
|
|
||||||
|
combined_parts = [token_text]
|
||||||
|
end = idx + 1
|
||||||
|
while end < len(tokens):
|
||||||
|
next_text = str(tokens[end] or "")
|
||||||
|
if not next_text or next_text.startswith("-"):
|
||||||
|
break
|
||||||
|
if not cls._looks_like_note_query_token(next_text):
|
||||||
|
break
|
||||||
|
combined_parts.append(next_text)
|
||||||
|
end += 1
|
||||||
|
|
||||||
|
combined_query = " ".join(combined_parts)
|
||||||
|
tokens[idx:end] = [combined_query]
|
||||||
|
tokens.insert(idx, "-query")
|
||||||
|
return tokens
|
||||||
|
|
||||||
|
return tokens
|
||||||
|
|
||||||
def _resolve_hash(
|
def _resolve_hash(
|
||||||
self,
|
self,
|
||||||
raw_hash: Optional[str],
|
raw_hash: Optional[str],
|
||||||
@@ -153,11 +200,14 @@ class Add_Note(Cmdlet):
|
|||||||
log(f"Cmdlet: {self.name}\nSummary: {self.summary}\nUsage: {self.usage}")
|
log(f"Cmdlet: {self.name}\nSummary: {self.summary}\nUsage: {self.usage}")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
parsed = parse_cmdlet_args(args, self)
|
parsed_args = self._default_query_args(args)
|
||||||
|
parsed = parse_cmdlet_args(parsed_args, self)
|
||||||
|
|
||||||
store_override = parsed.get("store")
|
store_override = parsed.get("store")
|
||||||
hash_override = normalize_hash(parsed.get("hash"))
|
hash_override = normalize_hash(parsed.get("hash"))
|
||||||
note_name, note_text = self._parse_note_query(str(parsed.get("query") or ""))
|
note_name, note_text = self._parse_note_query(str(parsed.get("query") or ""))
|
||||||
|
note_name = str(note_name or "").strip()
|
||||||
|
note_text = str(note_text or "").strip()
|
||||||
if not note_name or not note_text:
|
if not note_name or not note_text:
|
||||||
log(
|
log(
|
||||||
"[add_note] Error: -query must include title:<title> and text:<text>",
|
"[add_note] Error: -query must include title:<title> and text:<text>",
|
||||||
@@ -173,7 +223,6 @@ class Add_Note(Cmdlet):
|
|||||||
return 1
|
return 1
|
||||||
|
|
||||||
explicit_target = bool(hash_override and store_override)
|
explicit_target = bool(hash_override and store_override)
|
||||||
|
|
||||||
results = normalize_result_input(result)
|
results = normalize_result_input(result)
|
||||||
if results and explicit_target:
|
if results and explicit_target:
|
||||||
# Direct targeting mode: apply note once to the explicit target and
|
# Direct targeting mode: apply note once to the explicit target and
|
||||||
@@ -194,14 +243,22 @@ class Add_Note(Cmdlet):
|
|||||||
f"✓ add-note: 1 item in '{store_override}'",
|
f"✓ add-note: 1 item in '{store_override}'",
|
||||||
file=sys.stderr
|
file=sys.stderr
|
||||||
)
|
)
|
||||||
|
log(
|
||||||
|
"[add_note] Updated 1/1 item(s)",
|
||||||
|
file=sys.stderr
|
||||||
|
)
|
||||||
|
for res in results:
|
||||||
|
ctx.emit(res)
|
||||||
|
return 0
|
||||||
|
log(
|
||||||
|
"[add_note] Warning: Note write reported failure",
|
||||||
|
file=sys.stderr
|
||||||
|
)
|
||||||
|
return 1
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log(f"[add_note] Error: Failed to set note: {exc}", file=sys.stderr)
|
log(f"[add_note] Error: Failed to set note: {exc}", file=sys.stderr)
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
for res in results:
|
|
||||||
ctx.emit(res)
|
|
||||||
return 0
|
|
||||||
|
|
||||||
if not results:
|
if not results:
|
||||||
if explicit_target:
|
if explicit_target:
|
||||||
# Allow standalone use (no piped input) and enable piping the target forward.
|
# Allow standalone use (no piped input) and enable piping the target forward.
|
||||||
@@ -217,7 +274,7 @@ class Add_Note(Cmdlet):
|
|||||||
return 1
|
return 1
|
||||||
|
|
||||||
store_registry = Store(config)
|
store_registry = Store(config)
|
||||||
updated = 0
|
planned_ops = 0
|
||||||
|
|
||||||
# Batch write plan: store -> [(hash, name, text), ...]
|
# Batch write plan: store -> [(hash, name, text), ...]
|
||||||
note_ops: Dict[str,
|
note_ops: Dict[str,
|
||||||
@@ -271,12 +328,12 @@ class Add_Note(Cmdlet):
|
|||||||
[]).append((resolved_hash,
|
[]).append((resolved_hash,
|
||||||
note_name,
|
note_name,
|
||||||
item_note_text))
|
item_note_text))
|
||||||
updated += 1
|
planned_ops += 1
|
||||||
|
|
||||||
ctx.emit(res)
|
ctx.emit(res)
|
||||||
|
|
||||||
# Execute bulk writes per store.
|
# Execute bulk writes per store.
|
||||||
wrote_any = False
|
successful_writes = 0
|
||||||
for store_name, ops in note_ops.items():
|
for store_name, ops in note_ops.items():
|
||||||
if not ops:
|
if not ops:
|
||||||
continue
|
continue
|
||||||
@@ -285,16 +342,23 @@ class Add_Note(Cmdlet):
|
|||||||
except Exception:
|
except Exception:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
store_success = 0
|
||||||
bulk_fn = getattr(backend, "set_note_bulk", None)
|
bulk_fn = getattr(backend, "set_note_bulk", None)
|
||||||
if callable(bulk_fn):
|
if callable(bulk_fn):
|
||||||
try:
|
try:
|
||||||
ok = bool(bulk_fn(list(ops), config=config))
|
ok = bool(bulk_fn(list(ops), config=config))
|
||||||
wrote_any = wrote_any or ok or True
|
if ok:
|
||||||
ctx.print_if_visible(
|
store_success += len(ops)
|
||||||
f"✓ add-note: {len(ops)} item(s) in '{store_name}'",
|
ctx.print_if_visible(
|
||||||
file=sys.stderr
|
f"✓ add-note: {len(ops)} item(s) in '{store_name}'",
|
||||||
|
file=sys.stderr
|
||||||
|
)
|
||||||
|
successful_writes += store_success
|
||||||
|
continue
|
||||||
|
log(
|
||||||
|
f"[add_note] Warning: bulk set_note returned False for '{store_name}'",
|
||||||
|
file=sys.stderr,
|
||||||
)
|
)
|
||||||
continue
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log(
|
log(
|
||||||
f"[add_note] Warning: bulk set_note failed for '{store_name}': {exc}; falling back",
|
f"[add_note] Warning: bulk set_note failed for '{store_name}': {exc}; falling back",
|
||||||
@@ -305,12 +369,23 @@ class Add_Note(Cmdlet):
|
|||||||
for file_hash, name, text in ops:
|
for file_hash, name, text in ops:
|
||||||
try:
|
try:
|
||||||
ok = bool(backend.set_note(file_hash, name, text, config=config))
|
ok = bool(backend.set_note(file_hash, name, text, config=config))
|
||||||
wrote_any = wrote_any or ok
|
if ok:
|
||||||
|
store_success += 1
|
||||||
except Exception:
|
except Exception:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
log(f"[add_note] Updated {updated} item(s)", file=sys.stderr)
|
if store_success:
|
||||||
return 0 if (updated > 0 and wrote_any) else (0 if updated > 0 else 1)
|
successful_writes += store_success
|
||||||
|
ctx.print_if_visible(
|
||||||
|
f"✓ add-note: {store_success} item(s) in '{store_name}'",
|
||||||
|
file=sys.stderr
|
||||||
|
)
|
||||||
|
|
||||||
|
log(
|
||||||
|
f"[add_note] Updated {successful_writes}/{planned_ops} item(s)",
|
||||||
|
file=sys.stderr
|
||||||
|
)
|
||||||
|
return 0 if successful_writes > 0 else 1
|
||||||
|
|
||||||
|
|
||||||
CMDLET = Add_Note()
|
CMDLET = Add_Note()
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ from rich.prompt import Confirm
|
|||||||
from tool.ytdlp import (
|
from tool.ytdlp import (
|
||||||
YtDlpTool,
|
YtDlpTool,
|
||||||
_best_subtitle_sidecar,
|
_best_subtitle_sidecar,
|
||||||
|
_SUBTITLE_EXTS,
|
||||||
_download_with_timeout,
|
_download_with_timeout,
|
||||||
_format_chapters_note,
|
_format_chapters_note,
|
||||||
_read_text_file,
|
_read_text_file,
|
||||||
@@ -2413,7 +2414,7 @@ class Download_File(Cmdlet):
|
|||||||
except Exception:
|
except Exception:
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
if p_path.suffix.lower() in _best_subtitle_sidecar.__defaults__[0]:
|
if p_path.suffix.lower() in _SUBTITLE_EXTS:
|
||||||
continue
|
continue
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
@@ -2936,6 +2937,223 @@ class Download_File(Cmdlet):
|
|||||||
"media_kind": "video" if opts.mode == "video" else "audio",
|
"media_kind": "video" if opts.mode == "video" else "audio",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def download_streaming_url_as_pipe_objects(
|
||||||
|
url: str,
|
||||||
|
config: Dict[str, Any],
|
||||||
|
*,
|
||||||
|
mode_hint: Optional[str] = None,
|
||||||
|
ytdl_format_hint: Optional[str] = None,
|
||||||
|
) -> List[Dict[str, Any]]:
|
||||||
|
"""Download a yt-dlp-supported URL and return PipeObject-style dict(s).
|
||||||
|
|
||||||
|
This is a lightweight helper intended for cmdlets that need to expand streaming URLs
|
||||||
|
into local files without re-implementing yt-dlp glue.
|
||||||
|
"""
|
||||||
|
url_str = str(url or "").strip()
|
||||||
|
if not url_str:
|
||||||
|
return []
|
||||||
|
|
||||||
|
if not is_url_supported_by_ytdlp(url_str):
|
||||||
|
return []
|
||||||
|
|
||||||
|
try:
|
||||||
|
from SYS.config import resolve_output_dir
|
||||||
|
|
||||||
|
out_dir = resolve_output_dir(config)
|
||||||
|
if out_dir is None:
|
||||||
|
return []
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
|
||||||
|
cookies_path = None
|
||||||
|
try:
|
||||||
|
cookie_candidate = YtDlpTool(config).resolve_cookiefile()
|
||||||
|
if cookie_candidate is not None and cookie_candidate.is_file():
|
||||||
|
cookies_path = cookie_candidate
|
||||||
|
except Exception:
|
||||||
|
cookies_path = None
|
||||||
|
|
||||||
|
quiet_download = False
|
||||||
|
try:
|
||||||
|
quiet_download = bool((config or {}).get("_quiet_background_output"))
|
||||||
|
except Exception:
|
||||||
|
quiet_download = False
|
||||||
|
|
||||||
|
mode = str(mode_hint or "").strip().lower() if mode_hint else ""
|
||||||
|
if mode not in {"audio", "video"}:
|
||||||
|
mode = "video"
|
||||||
|
try:
|
||||||
|
cf = (
|
||||||
|
str(cookies_path)
|
||||||
|
if cookies_path is not None and cookies_path.is_file() else None
|
||||||
|
)
|
||||||
|
fmts_probe = list_formats(
|
||||||
|
url_str,
|
||||||
|
no_playlist=False,
|
||||||
|
playlist_items=None,
|
||||||
|
cookiefile=cf,
|
||||||
|
)
|
||||||
|
if isinstance(fmts_probe, list) and fmts_probe:
|
||||||
|
has_video = False
|
||||||
|
for f in fmts_probe:
|
||||||
|
if not isinstance(f, dict):
|
||||||
|
continue
|
||||||
|
vcodec = str(f.get("vcodec", "none") or "none").strip().lower()
|
||||||
|
if vcodec and vcodec != "none":
|
||||||
|
has_video = True
|
||||||
|
break
|
||||||
|
mode = "video" if has_video else "audio"
|
||||||
|
except Exception:
|
||||||
|
mode = "video"
|
||||||
|
|
||||||
|
fmt_hint = str(ytdl_format_hint).strip() if ytdl_format_hint else ""
|
||||||
|
chosen_format: Optional[str]
|
||||||
|
if fmt_hint:
|
||||||
|
chosen_format = fmt_hint
|
||||||
|
else:
|
||||||
|
chosen_format = None
|
||||||
|
if mode == "audio":
|
||||||
|
chosen_format = "bestaudio/best"
|
||||||
|
|
||||||
|
opts = DownloadOptions(
|
||||||
|
url=url_str,
|
||||||
|
mode=mode,
|
||||||
|
output_dir=Path(out_dir),
|
||||||
|
cookies_path=cookies_path,
|
||||||
|
ytdl_format=chosen_format,
|
||||||
|
quiet=quiet_download,
|
||||||
|
embed_chapters=True,
|
||||||
|
write_sub=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
result_obj = _download_with_timeout(opts, timeout_seconds=300)
|
||||||
|
except Exception as exc:
|
||||||
|
log(f"[download-file] Download failed for {url_str}: {exc}", file=sys.stderr)
|
||||||
|
return []
|
||||||
|
|
||||||
|
results: List[Any]
|
||||||
|
if isinstance(result_obj, list):
|
||||||
|
results = list(result_obj)
|
||||||
|
else:
|
||||||
|
paths = getattr(result_obj, "paths", None)
|
||||||
|
if isinstance(paths, list) and paths:
|
||||||
|
results = []
|
||||||
|
for p in paths:
|
||||||
|
try:
|
||||||
|
p_path = Path(p)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
if not p_path.exists() or p_path.is_dir():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
hv = sha256_file(p_path)
|
||||||
|
except Exception:
|
||||||
|
hv = None
|
||||||
|
try:
|
||||||
|
results.append(
|
||||||
|
DownloadMediaResult(
|
||||||
|
path=p_path,
|
||||||
|
info=getattr(result_obj, "info", {}) or {},
|
||||||
|
tag=list(getattr(result_obj, "tag", []) or []),
|
||||||
|
source_url=getattr(result_obj, "source_url", None) or url_str,
|
||||||
|
hash_value=hv,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
results = [result_obj]
|
||||||
|
|
||||||
|
out: List[Dict[str, Any]] = []
|
||||||
|
for downloaded in results:
|
||||||
|
try:
|
||||||
|
info = (
|
||||||
|
downloaded.info
|
||||||
|
if isinstance(getattr(downloaded, "info", None), dict) else {}
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
info = {}
|
||||||
|
|
||||||
|
try:
|
||||||
|
media_path = Path(str(getattr(downloaded, "path", "") or ""))
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
if not media_path.exists() or media_path.is_dir():
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
hash_value = getattr(downloaded, "hash_value", None) or sha256_file(media_path)
|
||||||
|
except Exception:
|
||||||
|
hash_value = None
|
||||||
|
|
||||||
|
title = None
|
||||||
|
try:
|
||||||
|
title = info.get("title")
|
||||||
|
except Exception:
|
||||||
|
title = None
|
||||||
|
title = title or media_path.stem
|
||||||
|
|
||||||
|
tags = list(getattr(downloaded, "tag", []) or [])
|
||||||
|
if title and f"title:{title}" not in tags:
|
||||||
|
tags.insert(0, f"title:{title}")
|
||||||
|
|
||||||
|
final_url = None
|
||||||
|
try:
|
||||||
|
page_url = info.get("webpage_url") or info.get("original_url") or info.get("url")
|
||||||
|
if page_url:
|
||||||
|
final_url = str(page_url)
|
||||||
|
except Exception:
|
||||||
|
final_url = None
|
||||||
|
if not final_url:
|
||||||
|
final_url = url_str
|
||||||
|
|
||||||
|
po: Dict[str, Any] = {
|
||||||
|
"path": str(media_path),
|
||||||
|
"hash": hash_value,
|
||||||
|
"title": title,
|
||||||
|
"url": final_url,
|
||||||
|
"tag": tags,
|
||||||
|
"action": "cmdlet:download-file",
|
||||||
|
"is_temp": True,
|
||||||
|
"ytdl_format": getattr(opts, "ytdl_format", None),
|
||||||
|
"store": getattr(opts, "storage_name", None) or getattr(opts, "storage_location", None) or "PATH",
|
||||||
|
"media_kind": "video" if opts.mode == "video" else "audio",
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
chapters_text = _format_chapters_note(info)
|
||||||
|
except Exception:
|
||||||
|
chapters_text = None
|
||||||
|
if chapters_text:
|
||||||
|
notes = po.get("notes")
|
||||||
|
if not isinstance(notes, dict):
|
||||||
|
notes = {}
|
||||||
|
notes.setdefault("chapters", chapters_text)
|
||||||
|
po["notes"] = notes
|
||||||
|
|
||||||
|
try:
|
||||||
|
sub_path = _best_subtitle_sidecar(media_path)
|
||||||
|
except Exception:
|
||||||
|
sub_path = None
|
||||||
|
if sub_path is not None:
|
||||||
|
sub_text = _read_text_file(sub_path)
|
||||||
|
if sub_text:
|
||||||
|
notes = po.get("notes")
|
||||||
|
if not isinstance(notes, dict):
|
||||||
|
notes = {}
|
||||||
|
notes["sub"] = sub_text
|
||||||
|
po["notes"] = notes
|
||||||
|
try:
|
||||||
|
sub_path.unlink()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
out.append(po)
|
||||||
|
|
||||||
|
return out
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _normalise_hash_hex(value: Optional[str]) -> Optional[str]:
|
def _normalise_hash_hex(value: Optional[str]) -> Optional[str]:
|
||||||
if not value or not isinstance(value, str):
|
if not value or not isinstance(value, str):
|
||||||
|
|||||||
@@ -191,7 +191,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
|||||||
mode_hint = None
|
mode_hint = None
|
||||||
forced_format = None
|
forced_format = None
|
||||||
|
|
||||||
from cmdlet.add_file import Add_File
|
from cmdlet.download_file import Download_File
|
||||||
|
|
||||||
expanded: List[Dict[str, Any]] = []
|
expanded: List[Dict[str, Any]] = []
|
||||||
downloaded_any = False
|
downloaded_any = False
|
||||||
@@ -204,7 +204,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
|||||||
expanded.append(it)
|
expanded.append(it)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
downloaded = Add_File._download_streaming_url_as_pipe_objects(
|
downloaded = Download_File.download_streaming_url_as_pipe_objects(
|
||||||
u,
|
u,
|
||||||
config,
|
config,
|
||||||
mode_hint=mode_hint,
|
mode_hint=mode_hint,
|
||||||
|
|||||||
200
cmdnat/help.py
200
cmdnat/help.py
@@ -1,10 +1,11 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from typing import Any, Dict, Sequence, List, Optional
|
from typing import Any, Dict, Sequence, List, Optional, Tuple
|
||||||
import shlex
|
import shlex
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from cmdlet._shared import Cmdlet, CmdletArg, parse_cmdlet_args
|
from cmdlet._shared import Cmdlet, CmdletArg, parse_cmdlet_args
|
||||||
|
from cmdlet import REGISTRY as CMDLET_REGISTRY, ensure_cmdlet_modules_loaded
|
||||||
from SYS.logger import log
|
from SYS.logger import log
|
||||||
from SYS.result_table import ResultTable
|
from SYS.result_table import ResultTable
|
||||||
from SYS import pipeline as ctx
|
from SYS import pipeline as ctx
|
||||||
@@ -27,6 +28,118 @@ def _examples_for_cmd(name: str) -> List[str]:
|
|||||||
return lookup.get(key, [])
|
return lookup.get(key, [])
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_cmdlet_key(name: Optional[str]) -> str:
|
||||||
|
return str(name or "").replace("_", "-").lower().strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _cmdlet_aliases(cmdlet_obj: Cmdlet) -> List[str]:
|
||||||
|
aliases: List[str] = []
|
||||||
|
for attr in ("alias", "aliases"):
|
||||||
|
raw_aliases = getattr(cmdlet_obj, attr, None)
|
||||||
|
if isinstance(raw_aliases, (list, tuple, set)):
|
||||||
|
candidates = raw_aliases
|
||||||
|
else:
|
||||||
|
candidates = (raw_aliases,)
|
||||||
|
for alias in candidates or ():
|
||||||
|
text = str(alias or "").strip()
|
||||||
|
if text:
|
||||||
|
aliases.append(text)
|
||||||
|
seen: set[str] = set()
|
||||||
|
deduped: List[str] = []
|
||||||
|
for alias in aliases:
|
||||||
|
key = alias.lower()
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
deduped.append(alias)
|
||||||
|
return deduped
|
||||||
|
|
||||||
|
|
||||||
|
def _cmdlet_arg_to_dict(arg: CmdletArg) -> Dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"name": str(getattr(arg, "name", "") or ""),
|
||||||
|
"type": str(getattr(arg, "type", "") or ""),
|
||||||
|
"required": bool(getattr(arg, "required", False)),
|
||||||
|
"description": str(getattr(arg, "description", "") or ""),
|
||||||
|
"choices": [str(c) for c in list(getattr(arg, "choices", []) or [])],
|
||||||
|
"alias": str(getattr(arg, "alias", "") or ""),
|
||||||
|
"variadic": bool(getattr(arg, "variadic", False)),
|
||||||
|
"usage": str(getattr(arg, "usage", "") or ""),
|
||||||
|
"query_key": getattr(arg, "query_key", None),
|
||||||
|
"query_aliases": [str(c) for c in list(getattr(arg, "query_aliases", []) or [])],
|
||||||
|
"query_only": bool(getattr(arg, "query_only", False)),
|
||||||
|
"requires_db": bool(getattr(arg, "requires_db", False)),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _build_alias_map_from_metadata(metadata: Dict[str, Dict[str, Any]]) -> Dict[str, str]:
|
||||||
|
mapping: Dict[str, str] = {}
|
||||||
|
for name, meta in metadata.items():
|
||||||
|
canonical = _normalize_cmdlet_key(name)
|
||||||
|
if canonical:
|
||||||
|
mapping[canonical] = name
|
||||||
|
for alias in meta.get("aliases", []) or []:
|
||||||
|
alias_key = _normalize_cmdlet_key(alias)
|
||||||
|
if alias_key:
|
||||||
|
mapping[alias_key] = name
|
||||||
|
return mapping
|
||||||
|
|
||||||
|
|
||||||
|
def _gather_metadata_from_cmdlet_classes() -> Tuple[Dict[str, Dict[str, Any]], Dict[str, str]]:
|
||||||
|
metadata: Dict[str, Dict[str, Any]] = {}
|
||||||
|
alias_map: Dict[str, str] = {}
|
||||||
|
try:
|
||||||
|
ensure_cmdlet_modules_loaded()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
for module in list(sys.modules.values()):
|
||||||
|
mod_name = getattr(module, "__name__", "") or ""
|
||||||
|
if not (mod_name.startswith("cmdlet.") or mod_name == "cmdlet" or mod_name.startswith("cmdnat.")):
|
||||||
|
continue
|
||||||
|
cmdlet_obj = getattr(module, "CMDLET", None)
|
||||||
|
if not isinstance(cmdlet_obj, Cmdlet):
|
||||||
|
continue
|
||||||
|
canonical_key = _normalize_cmdlet_key(getattr(cmdlet_obj, "name", None) or "")
|
||||||
|
if not canonical_key:
|
||||||
|
continue
|
||||||
|
entry = {
|
||||||
|
"name": str(getattr(cmdlet_obj, "name", "") or canonical_key),
|
||||||
|
"summary": str(getattr(cmdlet_obj, "summary", "") or ""),
|
||||||
|
"usage": str(getattr(cmdlet_obj, "usage", "") or ""),
|
||||||
|
"aliases": _cmdlet_aliases(cmdlet_obj),
|
||||||
|
"details": list(getattr(cmdlet_obj, "detail", []) or []),
|
||||||
|
"args": [_cmdlet_arg_to_dict(a) for a in getattr(cmdlet_obj, "arg", []) or []],
|
||||||
|
"raw": getattr(cmdlet_obj, "raw", None),
|
||||||
|
}
|
||||||
|
metadata[canonical_key] = entry
|
||||||
|
alias_map[canonical_key] = canonical_key
|
||||||
|
for alias in entry["aliases"]:
|
||||||
|
alias_key = _normalize_cmdlet_key(alias)
|
||||||
|
if alias_key:
|
||||||
|
alias_map[alias_key] = canonical_key
|
||||||
|
|
||||||
|
for registry_name in CMDLET_REGISTRY.keys():
|
||||||
|
normalized = _normalize_cmdlet_key(registry_name)
|
||||||
|
if not normalized or normalized in alias_map:
|
||||||
|
continue
|
||||||
|
alias_map[normalized] = normalized
|
||||||
|
metadata.setdefault(
|
||||||
|
normalized,
|
||||||
|
{
|
||||||
|
"name": normalized,
|
||||||
|
"aliases": [],
|
||||||
|
"usage": "",
|
||||||
|
"summary": "",
|
||||||
|
"details": [],
|
||||||
|
"args": [],
|
||||||
|
"raw": None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
return metadata, alias_map
|
||||||
|
|
||||||
|
|
||||||
def _find_cmd_metadata(name: str,
|
def _find_cmd_metadata(name: str,
|
||||||
metadata: Dict[str,
|
metadata: Dict[str,
|
||||||
Dict[str,
|
Dict[str,
|
||||||
@@ -148,17 +261,90 @@ def _render_detail(meta: Dict[str, Any], args: Sequence[str]) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||||
|
catalog: Any | None = None
|
||||||
|
cmdlet_names: List[str] = []
|
||||||
|
metadata: Dict[str, Dict[str, Any]] = {}
|
||||||
|
alias_map: Dict[str, str] = {}
|
||||||
|
|
||||||
try:
|
try:
|
||||||
import cmdlet_catalog as _catalog
|
import cmdlet_catalog as _catalog
|
||||||
|
|
||||||
CMDLET.arg[0].choices = _normalize_choice_list(
|
catalog = _catalog
|
||||||
_catalog.list_cmdlet_names(config=config)
|
|
||||||
)
|
|
||||||
metadata = _catalog.list_cmdlet_metadata(config=config)
|
|
||||||
except Exception:
|
except Exception:
|
||||||
CMDLET.arg[0].choices = []
|
catalog = None
|
||||||
metadata = {}
|
|
||||||
|
|
||||||
|
if catalog is not None:
|
||||||
|
try:
|
||||||
|
cmdlet_names = catalog.list_cmdlet_names(config=config)
|
||||||
|
except Exception:
|
||||||
|
cmdlet_names = []
|
||||||
|
try:
|
||||||
|
metadata = catalog.list_cmdlet_metadata(config=config)
|
||||||
|
except Exception:
|
||||||
|
metadata = {}
|
||||||
|
|
||||||
|
if metadata:
|
||||||
|
alias_map = _build_alias_map_from_metadata(metadata)
|
||||||
|
else:
|
||||||
|
metadata, alias_map = _gather_metadata_from_cmdlet_classes()
|
||||||
|
|
||||||
|
if not metadata:
|
||||||
|
fallback_names = sorted(set(cmdlet_names or list(CMDLET_REGISTRY.keys())))
|
||||||
|
if fallback_names:
|
||||||
|
base_meta: Dict[str, Dict[str, Any]] = {}
|
||||||
|
for cmdname in fallback_names:
|
||||||
|
canonical = str(cmdname or "").replace("_", "-").lower()
|
||||||
|
entry: Dict[str, Any]
|
||||||
|
candidate: Dict[str, Any] | None = None
|
||||||
|
if catalog is not None:
|
||||||
|
try:
|
||||||
|
candidate = catalog.get_cmdlet_metadata(cmdname, config=config)
|
||||||
|
except Exception:
|
||||||
|
candidate = None
|
||||||
|
if candidate:
|
||||||
|
canonical = candidate.get("name", canonical)
|
||||||
|
entry = candidate
|
||||||
|
else:
|
||||||
|
entry = {
|
||||||
|
"name": canonical,
|
||||||
|
"aliases": [],
|
||||||
|
"usage": "",
|
||||||
|
"summary": "",
|
||||||
|
"details": [],
|
||||||
|
"args": [],
|
||||||
|
"raw": None,
|
||||||
|
}
|
||||||
|
base = base_meta.setdefault(
|
||||||
|
canonical,
|
||||||
|
{
|
||||||
|
"name": canonical,
|
||||||
|
"aliases": [],
|
||||||
|
"usage": "",
|
||||||
|
"summary": "",
|
||||||
|
"details": [],
|
||||||
|
"args": [],
|
||||||
|
"raw": entry.get("raw"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if entry.get("aliases"):
|
||||||
|
base_aliases = set(base.get("aliases", []))
|
||||||
|
base_aliases.update([a for a in entry.get("aliases", []) if a])
|
||||||
|
base["aliases"] = sorted(base_aliases)
|
||||||
|
if not base.get("usage") and entry.get("usage"):
|
||||||
|
base["usage"] = entry["usage"]
|
||||||
|
if not base.get("summary") and entry.get("summary"):
|
||||||
|
base["summary"] = entry["summary"]
|
||||||
|
if not base.get("details") and entry.get("details"):
|
||||||
|
base["details"] = entry["details"]
|
||||||
|
if not base.get("args") and entry.get("args"):
|
||||||
|
base["args"] = entry["args"]
|
||||||
|
if not base.get("raw") and entry.get("raw"):
|
||||||
|
base["raw"] = entry["raw"]
|
||||||
|
metadata = base_meta
|
||||||
|
alias_map = _build_alias_map_from_metadata(metadata)
|
||||||
|
|
||||||
|
choice_candidates = list(alias_map.keys()) if alias_map else list(metadata.keys())
|
||||||
|
CMDLET.arg[0].choices = _normalize_choice_list(choice_candidates)
|
||||||
parsed = parse_cmdlet_args(args, CMDLET)
|
parsed = parse_cmdlet_args(args, CMDLET)
|
||||||
|
|
||||||
filter_text = parsed.get("filter")
|
filter_text = parsed.get("filter")
|
||||||
|
|||||||
Reference in New Issue
Block a user