j
This commit is contained in:
597
API/folder.py
597
API/folder.py
File diff suppressed because it is too large
Load Diff
23
CLI.py
23
CLI.py
@@ -3824,9 +3824,28 @@ class PipelineExecutor:
|
||||
# Fall back to default selection rendering on any failure.
|
||||
pass
|
||||
|
||||
items = piped_result if isinstance(piped_result, list) else [piped_result]
|
||||
|
||||
# Special-case: selecting notes should show the text content directly.
|
||||
note_like_items = [
|
||||
i for i in items
|
||||
if isinstance(i, dict) and ("note_text" in i or "note" in i)
|
||||
]
|
||||
if note_like_items:
|
||||
for idx, item in enumerate(note_like_items, 1):
|
||||
note_name = str(
|
||||
item.get("note_name")
|
||||
or item.get("name")
|
||||
or f"note {idx}"
|
||||
).strip()
|
||||
note_text = str(item.get("note_text") or item.get("note") or "")
|
||||
note_text = note_text[:999]
|
||||
stdout_console().print()
|
||||
stdout_console().print(f"{note_name}:\n{note_text}")
|
||||
ctx.set_last_result_items_only(items)
|
||||
return
|
||||
|
||||
table = ResultTable("Selection Result")
|
||||
items = piped_result if isinstance(piped_result,
|
||||
list) else [piped_result]
|
||||
for item in items:
|
||||
table.add_result(item)
|
||||
ctx.set_last_result_items_only(items)
|
||||
|
||||
220
SYS/cmdlet_api.py
Normal file
220
SYS/cmdlet_api.py
Normal file
@@ -0,0 +1,220 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import io
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable, Dict, List, Optional, Sequence
|
||||
|
||||
from SYS import pipeline as ctx
|
||||
from SYS.models import PipelineStageContext
|
||||
from SYS.rich_display import capture_rich_output
|
||||
|
||||
|
||||
CmdletCallable = Callable[[Any, Sequence[str], Dict[str, Any]], int]
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CmdletRunResult:
|
||||
"""Programmatic result for a single cmdlet invocation."""
|
||||
|
||||
name: str
|
||||
args: Sequence[str]
|
||||
exit_code: int = 0
|
||||
emitted: List[Any] = field(default_factory=list)
|
||||
|
||||
# Best-effort: cmdlets can publish tables/items via pipeline state even when
|
||||
# they don't emit pipeline items.
|
||||
result_table: Optional[Any] = None
|
||||
result_items: List[Any] = field(default_factory=list)
|
||||
result_subject: Optional[Any] = None
|
||||
|
||||
stdout: str = ""
|
||||
stderr: str = ""
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
def _normalize_cmd_name(name: str) -> str:
|
||||
return str(name or "").replace("_", "-").strip().lower()
|
||||
|
||||
|
||||
def resolve_cmdlet(cmd_name: str) -> Optional[CmdletCallable]:
|
||||
"""Resolve a cmdlet callable by name from the registry (aliases supported)."""
|
||||
try:
|
||||
from SYS.cmdlet_catalog import ensure_registry_loaded
|
||||
|
||||
ensure_registry_loaded()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
import cmdlet as cmdlet_pkg
|
||||
|
||||
return cmdlet_pkg.get(cmd_name)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def run_cmdlet(
|
||||
cmd: str | CmdletCallable,
|
||||
args: Sequence[str] | None,
|
||||
config: Dict[str, Any],
|
||||
*,
|
||||
piped: Any = None,
|
||||
isolate: bool = True,
|
||||
capture_output: bool = True,
|
||||
stage_index: int = 0,
|
||||
total_stages: int = 1,
|
||||
pipe_index: Optional[int] = None,
|
||||
worker_id: Optional[str] = None,
|
||||
) -> CmdletRunResult:
|
||||
"""Run a single cmdlet programmatically and return structured results.
|
||||
|
||||
This is intended for TUI/webapp consumers that want cmdlet behavior without
|
||||
going through the interactive CLI loop.
|
||||
|
||||
Notes:
|
||||
- When `isolate=True` (default) this runs inside `ctx.new_pipeline_state()` so
|
||||
global CLI pipeline state is not mutated.
|
||||
- Output capturing covers both normal `print()` and Rich output via
|
||||
`capture_rich_output()`.
|
||||
"""
|
||||
|
||||
normalized_args: Sequence[str] = list(args or [])
|
||||
|
||||
if isinstance(cmd, str):
|
||||
name = _normalize_cmd_name(cmd)
|
||||
cmd_fn = resolve_cmdlet(name)
|
||||
else:
|
||||
name = getattr(cmd, "__name__", "cmdlet")
|
||||
cmd_fn = cmd
|
||||
|
||||
result = CmdletRunResult(name=name, args=normalized_args)
|
||||
|
||||
if not callable(cmd_fn):
|
||||
result.exit_code = 1
|
||||
result.error = f"Unknown command: {name}"
|
||||
result.stderr = result.error
|
||||
return result
|
||||
|
||||
stage_ctx = PipelineStageContext(
|
||||
stage_index=int(stage_index),
|
||||
total_stages=int(total_stages),
|
||||
pipe_index=pipe_index,
|
||||
worker_id=worker_id,
|
||||
)
|
||||
|
||||
stdout_buffer = io.StringIO()
|
||||
stderr_buffer = io.StringIO()
|
||||
|
||||
stage_text = " ".join([name, *list(normalized_args)]).strip()
|
||||
|
||||
state_cm = ctx.new_pipeline_state() if isolate else contextlib.nullcontext()
|
||||
|
||||
with state_cm:
|
||||
# Keep behavior predictable: start from a clean slate.
|
||||
try:
|
||||
ctx.reset()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
ctx.set_stage_context(stage_ctx)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
ctx.set_current_cmdlet_name(name)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
ctx.set_current_stage_text(stage_text)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
ctx.set_current_command_text(stage_text)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
run_cm = (
|
||||
capture_rich_output(stdout=stdout_buffer, stderr=stderr_buffer)
|
||||
if capture_output
|
||||
else contextlib.nullcontext()
|
||||
)
|
||||
with run_cm:
|
||||
with (
|
||||
contextlib.redirect_stdout(stdout_buffer)
|
||||
if capture_output
|
||||
else contextlib.nullcontext()
|
||||
):
|
||||
with (
|
||||
contextlib.redirect_stderr(stderr_buffer)
|
||||
if capture_output
|
||||
else contextlib.nullcontext()
|
||||
):
|
||||
result.exit_code = int(cmd_fn(piped, list(normalized_args), config))
|
||||
except Exception as exc:
|
||||
result.exit_code = 1
|
||||
result.error = f"{type(exc).__name__}: {exc}"
|
||||
finally:
|
||||
result.stdout = stdout_buffer.getvalue()
|
||||
result.stderr = stderr_buffer.getvalue()
|
||||
|
||||
# Prefer cmdlet emits (pipeline semantics).
|
||||
try:
|
||||
result.emitted = list(stage_ctx.emits or [])
|
||||
except Exception:
|
||||
result.emitted = []
|
||||
|
||||
# Mirror CLI behavior: if cmdlet emitted items and there is no overlay table,
|
||||
# make emitted items the last result items for downstream consumers.
|
||||
try:
|
||||
has_overlay = bool(ctx.get_display_table())
|
||||
except Exception:
|
||||
has_overlay = False
|
||||
|
||||
if result.emitted and not has_overlay:
|
||||
try:
|
||||
ctx.set_last_result_items_only(list(result.emitted))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Best-effort snapshot of visible results.
|
||||
try:
|
||||
result.result_table = (
|
||||
ctx.get_display_table() or ctx.get_current_stage_table() or ctx.get_last_result_table()
|
||||
)
|
||||
except Exception:
|
||||
result.result_table = None
|
||||
|
||||
try:
|
||||
result.result_items = list(ctx.get_last_result_items() or [])
|
||||
except Exception:
|
||||
result.result_items = []
|
||||
|
||||
try:
|
||||
result.result_subject = ctx.get_last_result_subject()
|
||||
except Exception:
|
||||
result.result_subject = None
|
||||
|
||||
# Cleanup stage-local markers.
|
||||
try:
|
||||
ctx.clear_current_stage_text()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
ctx.clear_current_cmdlet_name()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
ctx.clear_current_command_text()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
ctx.set_stage_context(None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return result
|
||||
@@ -512,36 +512,46 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
|
||||
if not base_path.exists():
|
||||
continue
|
||||
|
||||
# Ensure file entry exists
|
||||
file_id: Optional[int] = None
|
||||
# Ensure file entry exists (folder store schema is keyed by hash).
|
||||
file_hash_value: Optional[str] = None
|
||||
if sha256_file and base_path.exists():
|
||||
try:
|
||||
file_hash_value = sha256_file(base_path)
|
||||
except Exception:
|
||||
file_hash_value = None
|
||||
|
||||
if not file_hash_value:
|
||||
continue
|
||||
|
||||
try:
|
||||
db_file_path = (
|
||||
db._to_db_file_path(base_path) # type: ignore[attr-defined]
|
||||
if hasattr(db, "_to_db_file_path")
|
||||
else str(base_path)
|
||||
)
|
||||
except Exception:
|
||||
db_file_path = str(base_path)
|
||||
|
||||
try:
|
||||
file_modified = float(base_path.stat().st_mtime)
|
||||
except Exception:
|
||||
file_modified = None
|
||||
|
||||
try:
|
||||
cursor = db.connection.cursor() if db.connection else None
|
||||
if cursor:
|
||||
cursor.execute(
|
||||
"SELECT id FROM files WHERE file_path = ?",
|
||||
(str(base_path),
|
||||
)
|
||||
"SELECT hash FROM file WHERE file_path = ?",
|
||||
(str(db_file_path),),
|
||||
)
|
||||
result = cursor.fetchone()
|
||||
file_id = result[0] if result else None
|
||||
except Exception:
|
||||
file_id = None
|
||||
|
||||
if not file_id:
|
||||
try:
|
||||
cursor = db.connection.cursor() if db.connection else None
|
||||
if cursor:
|
||||
if not result:
|
||||
cursor.execute(
|
||||
'INSERT INTO files (file_path, indexed_at, updated_at) VALUES (?, datetime("now"), datetime("now"))',
|
||||
(str(base_path),
|
||||
),
|
||||
"INSERT INTO file (hash, file_path, file_modified) VALUES (?, ?, ?)",
|
||||
(file_hash_value, str(db_file_path), file_modified),
|
||||
)
|
||||
db.connection.commit()
|
||||
file_id = cursor.lastrowid
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not file_id:
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if sidecar_path.suffix == ".tag":
|
||||
@@ -557,15 +567,9 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
|
||||
try:
|
||||
cursor = db.connection.cursor() if db.connection else None
|
||||
if cursor:
|
||||
file_hash_value: Optional[str] = None
|
||||
if hasattr(db, "get_file_hash"):
|
||||
try:
|
||||
file_hash_value = db.get_file_hash(file_id)
|
||||
except Exception:
|
||||
file_hash_value = None
|
||||
for tag in tags:
|
||||
cursor.execute(
|
||||
"INSERT OR IGNORE INTO tags (hash, tag) VALUES (?, ?)",
|
||||
"INSERT OR IGNORE INTO tag (hash, tag) VALUES (?, ?)",
|
||||
(file_hash_value,
|
||||
tag),
|
||||
)
|
||||
@@ -608,13 +612,15 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not hash_value:
|
||||
hash_value = file_hash_value
|
||||
|
||||
try:
|
||||
cursor = db.connection.cursor() if db.connection else None
|
||||
if cursor:
|
||||
cursor.execute(
|
||||
'INSERT OR REPLACE INTO metadata (file_id, hash, url, relationships, time_imported, time_modified) VALUES (?, ?, ?, ?, datetime("now"), datetime("now"))',
|
||||
'INSERT OR REPLACE INTO metadata (hash, url, relationships, time_imported, time_modified) VALUES (?, ?, ?, datetime("now"), datetime("now"))',
|
||||
(
|
||||
file_id,
|
||||
hash_value,
|
||||
json.dumps(url),
|
||||
json.dumps(relationships),
|
||||
@@ -634,9 +640,8 @@ def import_pending_sidecars(db_root: Path, db: Any) -> None:
|
||||
cursor = db.connection.cursor() if db.connection else None
|
||||
if cursor:
|
||||
cursor.execute(
|
||||
'INSERT INTO notes (file_id, note, created_at, updated_at) VALUES (?, ?, datetime("now"), datetime("now")) ON CONFLICT(file_id) DO UPDATE SET note = excluded.note, updated_at = datetime("now")',
|
||||
(file_id,
|
||||
content),
|
||||
'INSERT INTO note (hash, name, note, created_at, updated_at) VALUES (?, ?, ?, datetime("now"), datetime("now")) ON CONFLICT(hash, name) DO UPDATE SET note = excluded.note, updated_at = datetime("now")',
|
||||
(file_hash_value, "default", content),
|
||||
)
|
||||
db.connection.commit()
|
||||
except Exception:
|
||||
|
||||
@@ -263,6 +263,9 @@ class WorkerManager:
|
||||
self.refresh_thread: Optional[Thread] = None
|
||||
self._stop_refresh = False
|
||||
self._lock = Lock()
|
||||
# Reuse the DB's own lock so there is exactly one lock guarding the
|
||||
# sqlite connection (and it is safe for re-entrant/nested DB usage).
|
||||
self._db_lock = self.db._db_lock
|
||||
self.worker_handlers: Dict[str,
|
||||
WorkerLoggingHandler] = {} # Track active handlers
|
||||
self._worker_last_step: Dict[str,
|
||||
@@ -272,7 +275,8 @@ class WorkerManager:
|
||||
"""Close the database connection."""
|
||||
if self.db:
|
||||
try:
|
||||
self.db.close()
|
||||
with self._db_lock:
|
||||
self.db.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@@ -317,12 +321,13 @@ class WorkerManager:
|
||||
Count of workers updated.
|
||||
"""
|
||||
try:
|
||||
return self.db.expire_running_workers(
|
||||
older_than_seconds=older_than_seconds,
|
||||
status=status,
|
||||
reason=reason,
|
||||
worker_id_prefix=worker_id_prefix,
|
||||
)
|
||||
with self._db_lock:
|
||||
return self.db.expire_running_workers(
|
||||
older_than_seconds=older_than_seconds,
|
||||
status=status,
|
||||
reason=reason,
|
||||
worker_id_prefix=worker_id_prefix,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.error(f"Failed to expire stale workers: {exc}", exc_info=True)
|
||||
return 0
|
||||
@@ -419,14 +424,15 @@ class WorkerManager:
|
||||
True if worker was inserted successfully
|
||||
"""
|
||||
try:
|
||||
result = self.db.insert_worker(
|
||||
worker_id,
|
||||
worker_type,
|
||||
title,
|
||||
description,
|
||||
total_steps,
|
||||
pipe=pipe
|
||||
)
|
||||
with self._db_lock:
|
||||
result = self.db.insert_worker(
|
||||
worker_id,
|
||||
worker_type,
|
||||
title,
|
||||
description,
|
||||
total_steps,
|
||||
pipe=pipe
|
||||
)
|
||||
if result > 0:
|
||||
logger.debug(
|
||||
f"[WorkerManager] Tracking worker: {worker_id} ({worker_type})"
|
||||
@@ -473,7 +479,8 @@ class WorkerManager:
|
||||
kwargs["last_updated"] = datetime.now().isoformat()
|
||||
if "current_step" in kwargs and kwargs["current_step"]:
|
||||
self._worker_last_step[worker_id] = str(kwargs["current_step"])
|
||||
return self.db.update_worker(worker_id, **kwargs)
|
||||
with self._db_lock:
|
||||
return self.db.update_worker(worker_id, **kwargs)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
@@ -510,7 +517,8 @@ class WorkerManager:
|
||||
if result_data:
|
||||
kwargs["result_data"] = result_data
|
||||
|
||||
success = self.db.update_worker(worker_id, **kwargs)
|
||||
with self._db_lock:
|
||||
success = self.db.update_worker(worker_id, **kwargs)
|
||||
logger.info(f"[WorkerManager] Worker finished: {worker_id} ({result})")
|
||||
self._worker_last_step.pop(worker_id, None)
|
||||
return success
|
||||
@@ -528,7 +536,8 @@ class WorkerManager:
|
||||
List of active worker dictionaries
|
||||
"""
|
||||
try:
|
||||
return self.db.get_active_workers()
|
||||
with self._db_lock:
|
||||
return self.db.get_active_workers()
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"[WorkerManager] Error getting active workers: {e}",
|
||||
@@ -546,7 +555,8 @@ class WorkerManager:
|
||||
List of finished worker dictionaries
|
||||
"""
|
||||
try:
|
||||
all_workers = self.db.get_all_workers(limit=limit)
|
||||
with self._db_lock:
|
||||
all_workers = self.db.get_all_workers(limit=limit)
|
||||
# Filter to only finished workers
|
||||
finished = [
|
||||
w for w in all_workers
|
||||
@@ -570,7 +580,8 @@ class WorkerManager:
|
||||
Worker data or None if not found
|
||||
"""
|
||||
try:
|
||||
return self.db.get_worker(worker_id)
|
||||
with self._db_lock:
|
||||
return self.db.get_worker(worker_id)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"[WorkerManager] Error getting worker {worker_id}: {e}",
|
||||
@@ -583,7 +594,8 @@ class WorkerManager:
|
||||
limit: int = 500) -> List[Dict[str,
|
||||
Any]]:
|
||||
"""Fetch recorded worker timeline events."""
|
||||
return self.db.get_worker_events(worker_id, limit)
|
||||
with self._db_lock:
|
||||
return self.db.get_worker_events(worker_id, limit)
|
||||
|
||||
def log_step(self, worker_id: str, step_text: str) -> bool:
|
||||
"""Log a step to a worker's step history.
|
||||
@@ -596,7 +608,8 @@ class WorkerManager:
|
||||
True if successful
|
||||
"""
|
||||
try:
|
||||
success = self.db.append_worker_steps(worker_id, step_text)
|
||||
with self._db_lock:
|
||||
success = self.db.append_worker_steps(worker_id, step_text)
|
||||
if success:
|
||||
self._worker_last_step[worker_id] = step_text
|
||||
return success
|
||||
@@ -621,7 +634,8 @@ class WorkerManager:
|
||||
Steps text or empty string if not found
|
||||
"""
|
||||
try:
|
||||
return self.db.get_worker_steps(worker_id)
|
||||
with self._db_lock:
|
||||
return self.db.get_worker_steps(worker_id)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"[WorkerManager] Error getting steps for worker {worker_id}: {e}",
|
||||
@@ -705,7 +719,8 @@ class WorkerManager:
|
||||
Number of workers deleted
|
||||
"""
|
||||
try:
|
||||
count = self.db.cleanup_old_workers(days)
|
||||
with self._db_lock:
|
||||
count = self.db.cleanup_old_workers(days)
|
||||
if count > 0:
|
||||
logger.info(f"[WorkerManager] Cleaned up {count} old workers")
|
||||
return count
|
||||
@@ -729,12 +744,13 @@ class WorkerManager:
|
||||
"""
|
||||
try:
|
||||
step_label = self._get_last_step(worker_id)
|
||||
return self.db.append_worker_stdout(
|
||||
worker_id,
|
||||
text,
|
||||
step=step_label,
|
||||
channel=channel
|
||||
)
|
||||
with self._db_lock:
|
||||
return self.db.append_worker_stdout(
|
||||
worker_id,
|
||||
text,
|
||||
step=step_label,
|
||||
channel=channel
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"[WorkerManager] Error appending stdout: {e}", exc_info=True)
|
||||
return False
|
||||
@@ -749,7 +765,8 @@ class WorkerManager:
|
||||
Worker's stdout or empty string
|
||||
"""
|
||||
try:
|
||||
return self.db.get_worker_stdout(worker_id)
|
||||
with self._db_lock:
|
||||
return self.db.get_worker_stdout(worker_id)
|
||||
except Exception as e:
|
||||
logger.error(f"[WorkerManager] Error getting stdout: {e}", exc_info=True)
|
||||
return ""
|
||||
@@ -773,7 +790,8 @@ class WorkerManager:
|
||||
True if clear was successful
|
||||
"""
|
||||
try:
|
||||
return self.db.clear_worker_stdout(worker_id)
|
||||
with self._db_lock:
|
||||
return self.db.clear_worker_stdout(worker_id)
|
||||
except Exception as e:
|
||||
logger.error(f"[WorkerManager] Error clearing stdout: {e}", exc_info=True)
|
||||
return False
|
||||
@@ -781,5 +799,6 @@ class WorkerManager:
|
||||
def close(self) -> None:
|
||||
"""Close the worker manager and database connection."""
|
||||
self.stop_auto_refresh()
|
||||
self.db.close()
|
||||
with self._db_lock:
|
||||
self.db.close()
|
||||
logger.info("[WorkerManager] Closed")
|
||||
|
||||
136
Store/Folder.py
136
Store/Folder.py
@@ -60,7 +60,7 @@ class Folder(Store):
|
||||
if location is None and PATH is not None:
|
||||
location = str(PATH)
|
||||
|
||||
self._location = location
|
||||
self._location = str(location) if location is not None else ""
|
||||
self._name = name
|
||||
|
||||
# Scan status (set during init)
|
||||
@@ -221,7 +221,7 @@ class Folder(Store):
|
||||
# Ensure DB points to the renamed path (update by hash).
|
||||
try:
|
||||
cursor.execute(
|
||||
"UPDATE files SET file_path = ?, updated_at = CURRENT_TIMESTAMP WHERE hash = ?",
|
||||
"UPDATE file SET file_path = ?, updated_at = CURRENT_TIMESTAMP WHERE hash = ?",
|
||||
(db._to_db_file_path(hash_path),
|
||||
file_hash),
|
||||
)
|
||||
@@ -269,9 +269,9 @@ class Folder(Store):
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT f.hash, f.file_path
|
||||
FROM files f
|
||||
FROM file f
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM tags t WHERE t.hash = f.hash AND LOWER(t.tag) LIKE 'title:%'
|
||||
SELECT 1 FROM tag t WHERE t.hash = f.hash AND LOWER(t.tag) LIKE 'title:%'
|
||||
)
|
||||
"""
|
||||
)
|
||||
@@ -298,7 +298,7 @@ class Folder(Store):
|
||||
|
||||
# Third pass: discover files on disk that aren't in the database yet
|
||||
# These are hash-named files that were added after initial indexing
|
||||
cursor.execute("SELECT LOWER(hash) FROM files")
|
||||
cursor.execute("SELECT LOWER(hash) FROM file")
|
||||
db_hashes = {row[0]
|
||||
for row in cursor.fetchall()}
|
||||
|
||||
@@ -484,10 +484,17 @@ class Folder(Store):
|
||||
except Exception:
|
||||
duration_value = None
|
||||
|
||||
# Save to database
|
||||
# Save to database (metadata + tag/url updates share one connection)
|
||||
with API_folder_store(Path(self._location)) as db:
|
||||
db.get_or_create_file_entry(save_file)
|
||||
# Save metadata including extension
|
||||
conn = getattr(db, "connection", None)
|
||||
if conn is None:
|
||||
raise RuntimeError("Folder store DB connection unavailable")
|
||||
cursor = conn.cursor()
|
||||
|
||||
debug(
|
||||
f"[Folder.add_file] saving metadata for hash {file_hash}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
ext_clean = file_ext.lstrip(".") if file_ext else ""
|
||||
db.save_metadata(
|
||||
save_file,
|
||||
@@ -498,14 +505,77 @@ class Folder(Store):
|
||||
"duration": duration_value,
|
||||
},
|
||||
)
|
||||
debug(
|
||||
f"[Folder.add_file] metadata stored for hash {file_hash}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
# Add tags if provided
|
||||
if tag_list:
|
||||
self.add_tag(file_hash, tag_list)
|
||||
if tag_list:
|
||||
try:
|
||||
debug(
|
||||
f"[Folder.add_file] merging {len(tag_list)} tags for {file_hash}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
from SYS.metadata import compute_namespaced_tag_overwrite
|
||||
|
||||
# Add url if provided
|
||||
if url:
|
||||
self.add_url(file_hash, url)
|
||||
existing_tags = [
|
||||
t for t in (db.get_tags(file_hash) or [])
|
||||
if isinstance(t, str) and t.strip()
|
||||
]
|
||||
_to_remove, _to_add, merged = compute_namespaced_tag_overwrite(
|
||||
existing_tags, tag_list or []
|
||||
)
|
||||
if _to_remove or _to_add:
|
||||
cursor.execute("DELETE FROM tag WHERE hash = ?",
|
||||
(file_hash,))
|
||||
for t in merged:
|
||||
tag_val = str(t).strip().lower()
|
||||
if tag_val:
|
||||
cursor.execute(
|
||||
"INSERT OR IGNORE INTO tag (hash, tag) VALUES (?, ?)",
|
||||
(file_hash, tag_val),
|
||||
)
|
||||
conn.commit()
|
||||
debug(
|
||||
f"[Folder.add_file] tags rewritten for {file_hash}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
try:
|
||||
db._update_metadata_modified_time(file_hash)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
debug(f"Local DB tag merge failed: {exc}", file=sys.stderr)
|
||||
|
||||
if url:
|
||||
try:
|
||||
debug(
|
||||
f"[Folder.add_file] merging {len(url)} URLs for {file_hash}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
from SYS.metadata import normalize_urls
|
||||
|
||||
existing_meta = db.get_metadata(file_hash) or {}
|
||||
existing_urls = normalize_urls(existing_meta.get("url"))
|
||||
incoming_urls = normalize_urls(url)
|
||||
changed = False
|
||||
for entry in list(incoming_urls or []):
|
||||
if not entry:
|
||||
continue
|
||||
if entry not in existing_urls:
|
||||
existing_urls.append(entry)
|
||||
changed = True
|
||||
if changed:
|
||||
db.update_metadata_by_hash(
|
||||
file_hash,
|
||||
{"url": existing_urls},
|
||||
)
|
||||
debug(
|
||||
f"[Folder.add_file] URLs merged for {file_hash}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
except Exception as exc:
|
||||
debug(f"Local DB URL merge failed: {exc}", file=sys.stderr)
|
||||
|
||||
##log(f"✓ Added to local storage: {save_file.name}", file=sys.stderr)
|
||||
return file_hash
|
||||
@@ -1373,6 +1443,34 @@ class Folder(Store):
|
||||
debug(f"Failed to get metadata for hash {file_hash}: {exc}")
|
||||
return None
|
||||
|
||||
def set_relationship(self, alt_hash: str, king_hash: str, kind: str = "alt") -> bool:
|
||||
"""Persist a relationship in the folder store DB.
|
||||
|
||||
This is a thin wrapper around the folder DB API so cmdlets can avoid
|
||||
backend-specific branching.
|
||||
"""
|
||||
try:
|
||||
if not self._location:
|
||||
return False
|
||||
|
||||
alt_norm = _normalize_hash(alt_hash)
|
||||
king_norm = _normalize_hash(king_hash)
|
||||
if not alt_norm or not king_norm or alt_norm == king_norm:
|
||||
return False
|
||||
|
||||
from API.folder import API_folder_store
|
||||
|
||||
with API_folder_store(Path(self._location).expanduser()) as db:
|
||||
db.set_relationship_by_hash(
|
||||
alt_norm,
|
||||
king_norm,
|
||||
str(kind or "alt"),
|
||||
bidirectional=False,
|
||||
)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def get_tag(self, file_identifier: str, **kwargs: Any) -> Tuple[List[str], str]:
|
||||
"""Get tags for a local file by hash.
|
||||
|
||||
@@ -1432,14 +1530,14 @@ class Folder(Store):
|
||||
# Folder DB tag table is case-sensitive and add_tags_to_hash() is additive.
|
||||
# To enforce lowercase-only tags and namespace overwrites, rewrite the full tag set.
|
||||
cursor = db.connection.cursor()
|
||||
cursor.execute("DELETE FROM tags WHERE hash = ?",
|
||||
cursor.execute("DELETE FROM tag WHERE hash = ?",
|
||||
(hash,
|
||||
))
|
||||
for t in merged:
|
||||
t = str(t).strip().lower()
|
||||
if t:
|
||||
cursor.execute(
|
||||
"INSERT OR IGNORE INTO tags (hash, tag) VALUES (?, ?)",
|
||||
"INSERT OR IGNORE INTO tag (hash, tag) VALUES (?, ?)",
|
||||
(hash,
|
||||
t),
|
||||
)
|
||||
@@ -1953,7 +2051,7 @@ class Folder(Store):
|
||||
placeholders = ",".join(["?"] * len(chunk))
|
||||
try:
|
||||
cursor.execute(
|
||||
f"SELECT hash, file_path FROM files WHERE hash IN ({placeholders})",
|
||||
f"SELECT hash, file_path FROM file WHERE hash IN ({placeholders})",
|
||||
chunk,
|
||||
)
|
||||
rows = cursor.fetchall() or []
|
||||
@@ -1987,13 +2085,13 @@ class Folder(Store):
|
||||
# Prefer upsert when supported, else fall back to INSERT OR REPLACE.
|
||||
try:
|
||||
cursor.executemany(
|
||||
"INSERT INTO notes (hash, name, note) VALUES (?, ?, ?) "
|
||||
"INSERT INTO note (hash, name, note) VALUES (?, ?, ?) "
|
||||
"ON CONFLICT(hash, name) DO UPDATE SET note = excluded.note, updated_at = CURRENT_TIMESTAMP",
|
||||
inserts,
|
||||
)
|
||||
except Exception:
|
||||
cursor.executemany(
|
||||
"INSERT OR REPLACE INTO notes (hash, name, note) VALUES (?, ?, ?)",
|
||||
"INSERT OR REPLACE INTO note (hash, name, note) VALUES (?, ?, ?)",
|
||||
inserts,
|
||||
)
|
||||
|
||||
|
||||
@@ -218,6 +218,23 @@ class HydrusNetwork(Store):
|
||||
def get_name(self) -> str:
|
||||
return self.NAME
|
||||
|
||||
def set_relationship(self, alt_hash: str, king_hash: str, kind: str = "alt") -> bool:
|
||||
"""Persist a relationship via the Hydrus client API for this backend instance."""
|
||||
try:
|
||||
alt_norm = str(alt_hash or "").strip().lower()
|
||||
king_norm = str(king_hash or "").strip().lower()
|
||||
if len(alt_norm) != 64 or len(king_norm) != 64 or alt_norm == king_norm:
|
||||
return False
|
||||
|
||||
client = getattr(self, "_client", None)
|
||||
if client is None or not hasattr(client, "set_relationship"):
|
||||
return False
|
||||
|
||||
client.set_relationship(alt_norm, king_norm, str(kind or "alt"))
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def add_file(self, file_path: Path, **kwargs: Any) -> str:
|
||||
"""Upload file to Hydrus with full metadata support.
|
||||
|
||||
@@ -284,9 +301,8 @@ class HydrusNetwork(Store):
|
||||
file_exists = True
|
||||
break
|
||||
if file_exists:
|
||||
log(
|
||||
f"ℹ️ Duplicate detected - file already in Hydrus with hash: {file_hash}",
|
||||
file=sys.stderr,
|
||||
debug(
|
||||
f"{self._log_prefix()} Duplicate detected - file already in Hydrus with hash: {file_hash}"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -301,9 +317,8 @@ class HydrusNetwork(Store):
|
||||
|
||||
# Upload file if not already present
|
||||
if not file_exists:
|
||||
log(
|
||||
f"{self._log_prefix()} Uploading: {file_path.name}",
|
||||
file=sys.stderr
|
||||
debug(
|
||||
f"{self._log_prefix()} Uploading: {file_path.name}"
|
||||
)
|
||||
response = client.add_file(file_path)
|
||||
|
||||
@@ -320,7 +335,7 @@ class HydrusNetwork(Store):
|
||||
raise Exception(f"Hydrus response missing file hash: {response}")
|
||||
|
||||
file_hash = hydrus_hash
|
||||
log(f"{self._log_prefix()} hash: {file_hash}", file=sys.stderr)
|
||||
debug(f"{self._log_prefix()} hash: {file_hash}")
|
||||
|
||||
# Add tags if provided (both for new and existing files)
|
||||
if tag_list:
|
||||
@@ -335,9 +350,8 @@ class HydrusNetwork(Store):
|
||||
f"{self._log_prefix()} Adding {len(tag_list)} tag(s): {tag_list}"
|
||||
)
|
||||
client.add_tag(file_hash, tag_list, service_name)
|
||||
log(
|
||||
f"{self._log_prefix()} Tags added via '{service_name}'",
|
||||
file=sys.stderr
|
||||
debug(
|
||||
f"{self._log_prefix()} Tags added via '{service_name}'"
|
||||
)
|
||||
except Exception as exc:
|
||||
log(
|
||||
@@ -347,9 +361,8 @@ class HydrusNetwork(Store):
|
||||
|
||||
# Associate url if provided (both for new and existing files)
|
||||
if url:
|
||||
log(
|
||||
f"{self._log_prefix()} Associating {len(url)} URL(s) with file",
|
||||
file=sys.stderr
|
||||
debug(
|
||||
f"{self._log_prefix()} Associating {len(url)} URL(s) with file"
|
||||
)
|
||||
for url in url:
|
||||
if url:
|
||||
|
||||
@@ -186,7 +186,7 @@ class SharedArgs:
|
||||
name="path",
|
||||
type="string",
|
||||
choices=[], # Dynamically populated via get_store_choices()
|
||||
description="Selects store",
|
||||
description="selects store",
|
||||
)
|
||||
|
||||
URL = CmdletArg(
|
||||
@@ -194,6 +194,11 @@ class SharedArgs:
|
||||
type="string",
|
||||
description="http parser",
|
||||
)
|
||||
PROVIDER = CmdletArg(
|
||||
name="provider",
|
||||
type="string",
|
||||
description="selects provider",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_store_choices(config: Optional[Dict[str, Any]] = None) -> List[str]:
|
||||
|
||||
2025
cmdlet/add_file.py
2025
cmdlet/add_file.py
File diff suppressed because it is too large
Load Diff
@@ -23,6 +23,14 @@ from SYS.utils import sha256_file
|
||||
|
||||
|
||||
class Add_Note(Cmdlet):
|
||||
DEFAULT_QUERY_HINTS = (
|
||||
"title:",
|
||||
"text:",
|
||||
"hash:",
|
||||
"caption:",
|
||||
"sub:",
|
||||
"subtitle:",
|
||||
)
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
@@ -124,6 +132,45 @@ class Add_Note(Cmdlet):
|
||||
note_text = text_match.group(1).strip() if text_match else ""
|
||||
return (note_name or None, note_text or None)
|
||||
|
||||
@classmethod
|
||||
def _looks_like_note_query_token(cls, token: Any) -> bool:
|
||||
text = str(token or "").strip().lower()
|
||||
if not text:
|
||||
return False
|
||||
return any(hint in text for hint in cls.DEFAULT_QUERY_HINTS)
|
||||
|
||||
@classmethod
|
||||
def _default_query_args(cls, args: Sequence[str]) -> List[str]:
|
||||
tokens: List[str] = list(args or [])
|
||||
lower_tokens = {str(tok).lower() for tok in tokens if tok is not None}
|
||||
if "-query" in lower_tokens or "--query" in lower_tokens:
|
||||
return tokens
|
||||
|
||||
for idx, tok in enumerate(tokens):
|
||||
token_text = str(tok or "")
|
||||
if not token_text or token_text.startswith("-"):
|
||||
continue
|
||||
if not cls._looks_like_note_query_token(token_text):
|
||||
continue
|
||||
|
||||
combined_parts = [token_text]
|
||||
end = idx + 1
|
||||
while end < len(tokens):
|
||||
next_text = str(tokens[end] or "")
|
||||
if not next_text or next_text.startswith("-"):
|
||||
break
|
||||
if not cls._looks_like_note_query_token(next_text):
|
||||
break
|
||||
combined_parts.append(next_text)
|
||||
end += 1
|
||||
|
||||
combined_query = " ".join(combined_parts)
|
||||
tokens[idx:end] = [combined_query]
|
||||
tokens.insert(idx, "-query")
|
||||
return tokens
|
||||
|
||||
return tokens
|
||||
|
||||
def _resolve_hash(
|
||||
self,
|
||||
raw_hash: Optional[str],
|
||||
@@ -153,11 +200,14 @@ class Add_Note(Cmdlet):
|
||||
log(f"Cmdlet: {self.name}\nSummary: {self.summary}\nUsage: {self.usage}")
|
||||
return 0
|
||||
|
||||
parsed = parse_cmdlet_args(args, self)
|
||||
parsed_args = self._default_query_args(args)
|
||||
parsed = parse_cmdlet_args(parsed_args, self)
|
||||
|
||||
store_override = parsed.get("store")
|
||||
hash_override = normalize_hash(parsed.get("hash"))
|
||||
note_name, note_text = self._parse_note_query(str(parsed.get("query") or ""))
|
||||
note_name = str(note_name or "").strip()
|
||||
note_text = str(note_text or "").strip()
|
||||
if not note_name or not note_text:
|
||||
log(
|
||||
"[add_note] Error: -query must include title:<title> and text:<text>",
|
||||
@@ -173,7 +223,6 @@ class Add_Note(Cmdlet):
|
||||
return 1
|
||||
|
||||
explicit_target = bool(hash_override and store_override)
|
||||
|
||||
results = normalize_result_input(result)
|
||||
if results and explicit_target:
|
||||
# Direct targeting mode: apply note once to the explicit target and
|
||||
@@ -194,14 +243,22 @@ class Add_Note(Cmdlet):
|
||||
f"✓ add-note: 1 item in '{store_override}'",
|
||||
file=sys.stderr
|
||||
)
|
||||
log(
|
||||
"[add_note] Updated 1/1 item(s)",
|
||||
file=sys.stderr
|
||||
)
|
||||
for res in results:
|
||||
ctx.emit(res)
|
||||
return 0
|
||||
log(
|
||||
"[add_note] Warning: Note write reported failure",
|
||||
file=sys.stderr
|
||||
)
|
||||
return 1
|
||||
except Exception as exc:
|
||||
log(f"[add_note] Error: Failed to set note: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
for res in results:
|
||||
ctx.emit(res)
|
||||
return 0
|
||||
|
||||
if not results:
|
||||
if explicit_target:
|
||||
# Allow standalone use (no piped input) and enable piping the target forward.
|
||||
@@ -217,7 +274,7 @@ class Add_Note(Cmdlet):
|
||||
return 1
|
||||
|
||||
store_registry = Store(config)
|
||||
updated = 0
|
||||
planned_ops = 0
|
||||
|
||||
# Batch write plan: store -> [(hash, name, text), ...]
|
||||
note_ops: Dict[str,
|
||||
@@ -271,12 +328,12 @@ class Add_Note(Cmdlet):
|
||||
[]).append((resolved_hash,
|
||||
note_name,
|
||||
item_note_text))
|
||||
updated += 1
|
||||
planned_ops += 1
|
||||
|
||||
ctx.emit(res)
|
||||
|
||||
# Execute bulk writes per store.
|
||||
wrote_any = False
|
||||
successful_writes = 0
|
||||
for store_name, ops in note_ops.items():
|
||||
if not ops:
|
||||
continue
|
||||
@@ -285,16 +342,23 @@ class Add_Note(Cmdlet):
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
store_success = 0
|
||||
bulk_fn = getattr(backend, "set_note_bulk", None)
|
||||
if callable(bulk_fn):
|
||||
try:
|
||||
ok = bool(bulk_fn(list(ops), config=config))
|
||||
wrote_any = wrote_any or ok or True
|
||||
ctx.print_if_visible(
|
||||
f"✓ add-note: {len(ops)} item(s) in '{store_name}'",
|
||||
file=sys.stderr
|
||||
if ok:
|
||||
store_success += len(ops)
|
||||
ctx.print_if_visible(
|
||||
f"✓ add-note: {len(ops)} item(s) in '{store_name}'",
|
||||
file=sys.stderr
|
||||
)
|
||||
successful_writes += store_success
|
||||
continue
|
||||
log(
|
||||
f"[add_note] Warning: bulk set_note returned False for '{store_name}'",
|
||||
file=sys.stderr,
|
||||
)
|
||||
continue
|
||||
except Exception as exc:
|
||||
log(
|
||||
f"[add_note] Warning: bulk set_note failed for '{store_name}': {exc}; falling back",
|
||||
@@ -305,12 +369,23 @@ class Add_Note(Cmdlet):
|
||||
for file_hash, name, text in ops:
|
||||
try:
|
||||
ok = bool(backend.set_note(file_hash, name, text, config=config))
|
||||
wrote_any = wrote_any or ok
|
||||
if ok:
|
||||
store_success += 1
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
log(f"[add_note] Updated {updated} item(s)", file=sys.stderr)
|
||||
return 0 if (updated > 0 and wrote_any) else (0 if updated > 0 else 1)
|
||||
if store_success:
|
||||
successful_writes += store_success
|
||||
ctx.print_if_visible(
|
||||
f"✓ add-note: {store_success} item(s) in '{store_name}'",
|
||||
file=sys.stderr
|
||||
)
|
||||
|
||||
log(
|
||||
f"[add_note] Updated {successful_writes}/{planned_ops} item(s)",
|
||||
file=sys.stderr
|
||||
)
|
||||
return 0 if successful_writes > 0 else 1
|
||||
|
||||
|
||||
CMDLET = Add_Note()
|
||||
|
||||
@@ -33,6 +33,7 @@ from rich.prompt import Confirm
|
||||
from tool.ytdlp import (
|
||||
YtDlpTool,
|
||||
_best_subtitle_sidecar,
|
||||
_SUBTITLE_EXTS,
|
||||
_download_with_timeout,
|
||||
_format_chapters_note,
|
||||
_read_text_file,
|
||||
@@ -2413,7 +2414,7 @@ class Download_File(Cmdlet):
|
||||
except Exception:
|
||||
continue
|
||||
try:
|
||||
if p_path.suffix.lower() in _best_subtitle_sidecar.__defaults__[0]:
|
||||
if p_path.suffix.lower() in _SUBTITLE_EXTS:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
@@ -2936,6 +2937,223 @@ class Download_File(Cmdlet):
|
||||
"media_kind": "video" if opts.mode == "video" else "audio",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def download_streaming_url_as_pipe_objects(
|
||||
url: str,
|
||||
config: Dict[str, Any],
|
||||
*,
|
||||
mode_hint: Optional[str] = None,
|
||||
ytdl_format_hint: Optional[str] = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Download a yt-dlp-supported URL and return PipeObject-style dict(s).
|
||||
|
||||
This is a lightweight helper intended for cmdlets that need to expand streaming URLs
|
||||
into local files without re-implementing yt-dlp glue.
|
||||
"""
|
||||
url_str = str(url or "").strip()
|
||||
if not url_str:
|
||||
return []
|
||||
|
||||
if not is_url_supported_by_ytdlp(url_str):
|
||||
return []
|
||||
|
||||
try:
|
||||
from SYS.config import resolve_output_dir
|
||||
|
||||
out_dir = resolve_output_dir(config)
|
||||
if out_dir is None:
|
||||
return []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
cookies_path = None
|
||||
try:
|
||||
cookie_candidate = YtDlpTool(config).resolve_cookiefile()
|
||||
if cookie_candidate is not None and cookie_candidate.is_file():
|
||||
cookies_path = cookie_candidate
|
||||
except Exception:
|
||||
cookies_path = None
|
||||
|
||||
quiet_download = False
|
||||
try:
|
||||
quiet_download = bool((config or {}).get("_quiet_background_output"))
|
||||
except Exception:
|
||||
quiet_download = False
|
||||
|
||||
mode = str(mode_hint or "").strip().lower() if mode_hint else ""
|
||||
if mode not in {"audio", "video"}:
|
||||
mode = "video"
|
||||
try:
|
||||
cf = (
|
||||
str(cookies_path)
|
||||
if cookies_path is not None and cookies_path.is_file() else None
|
||||
)
|
||||
fmts_probe = list_formats(
|
||||
url_str,
|
||||
no_playlist=False,
|
||||
playlist_items=None,
|
||||
cookiefile=cf,
|
||||
)
|
||||
if isinstance(fmts_probe, list) and fmts_probe:
|
||||
has_video = False
|
||||
for f in fmts_probe:
|
||||
if not isinstance(f, dict):
|
||||
continue
|
||||
vcodec = str(f.get("vcodec", "none") or "none").strip().lower()
|
||||
if vcodec and vcodec != "none":
|
||||
has_video = True
|
||||
break
|
||||
mode = "video" if has_video else "audio"
|
||||
except Exception:
|
||||
mode = "video"
|
||||
|
||||
fmt_hint = str(ytdl_format_hint).strip() if ytdl_format_hint else ""
|
||||
chosen_format: Optional[str]
|
||||
if fmt_hint:
|
||||
chosen_format = fmt_hint
|
||||
else:
|
||||
chosen_format = None
|
||||
if mode == "audio":
|
||||
chosen_format = "bestaudio/best"
|
||||
|
||||
opts = DownloadOptions(
|
||||
url=url_str,
|
||||
mode=mode,
|
||||
output_dir=Path(out_dir),
|
||||
cookies_path=cookies_path,
|
||||
ytdl_format=chosen_format,
|
||||
quiet=quiet_download,
|
||||
embed_chapters=True,
|
||||
write_sub=True,
|
||||
)
|
||||
|
||||
try:
|
||||
result_obj = _download_with_timeout(opts, timeout_seconds=300)
|
||||
except Exception as exc:
|
||||
log(f"[download-file] Download failed for {url_str}: {exc}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
results: List[Any]
|
||||
if isinstance(result_obj, list):
|
||||
results = list(result_obj)
|
||||
else:
|
||||
paths = getattr(result_obj, "paths", None)
|
||||
if isinstance(paths, list) and paths:
|
||||
results = []
|
||||
for p in paths:
|
||||
try:
|
||||
p_path = Path(p)
|
||||
except Exception:
|
||||
continue
|
||||
if not p_path.exists() or p_path.is_dir():
|
||||
continue
|
||||
try:
|
||||
hv = sha256_file(p_path)
|
||||
except Exception:
|
||||
hv = None
|
||||
try:
|
||||
results.append(
|
||||
DownloadMediaResult(
|
||||
path=p_path,
|
||||
info=getattr(result_obj, "info", {}) or {},
|
||||
tag=list(getattr(result_obj, "tag", []) or []),
|
||||
source_url=getattr(result_obj, "source_url", None) or url_str,
|
||||
hash_value=hv,
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
else:
|
||||
results = [result_obj]
|
||||
|
||||
out: List[Dict[str, Any]] = []
|
||||
for downloaded in results:
|
||||
try:
|
||||
info = (
|
||||
downloaded.info
|
||||
if isinstance(getattr(downloaded, "info", None), dict) else {}
|
||||
)
|
||||
except Exception:
|
||||
info = {}
|
||||
|
||||
try:
|
||||
media_path = Path(str(getattr(downloaded, "path", "") or ""))
|
||||
except Exception:
|
||||
continue
|
||||
if not media_path.exists() or media_path.is_dir():
|
||||
continue
|
||||
|
||||
try:
|
||||
hash_value = getattr(downloaded, "hash_value", None) or sha256_file(media_path)
|
||||
except Exception:
|
||||
hash_value = None
|
||||
|
||||
title = None
|
||||
try:
|
||||
title = info.get("title")
|
||||
except Exception:
|
||||
title = None
|
||||
title = title or media_path.stem
|
||||
|
||||
tags = list(getattr(downloaded, "tag", []) or [])
|
||||
if title and f"title:{title}" not in tags:
|
||||
tags.insert(0, f"title:{title}")
|
||||
|
||||
final_url = None
|
||||
try:
|
||||
page_url = info.get("webpage_url") or info.get("original_url") or info.get("url")
|
||||
if page_url:
|
||||
final_url = str(page_url)
|
||||
except Exception:
|
||||
final_url = None
|
||||
if not final_url:
|
||||
final_url = url_str
|
||||
|
||||
po: Dict[str, Any] = {
|
||||
"path": str(media_path),
|
||||
"hash": hash_value,
|
||||
"title": title,
|
||||
"url": final_url,
|
||||
"tag": tags,
|
||||
"action": "cmdlet:download-file",
|
||||
"is_temp": True,
|
||||
"ytdl_format": getattr(opts, "ytdl_format", None),
|
||||
"store": getattr(opts, "storage_name", None) or getattr(opts, "storage_location", None) or "PATH",
|
||||
"media_kind": "video" if opts.mode == "video" else "audio",
|
||||
}
|
||||
|
||||
try:
|
||||
chapters_text = _format_chapters_note(info)
|
||||
except Exception:
|
||||
chapters_text = None
|
||||
if chapters_text:
|
||||
notes = po.get("notes")
|
||||
if not isinstance(notes, dict):
|
||||
notes = {}
|
||||
notes.setdefault("chapters", chapters_text)
|
||||
po["notes"] = notes
|
||||
|
||||
try:
|
||||
sub_path = _best_subtitle_sidecar(media_path)
|
||||
except Exception:
|
||||
sub_path = None
|
||||
if sub_path is not None:
|
||||
sub_text = _read_text_file(sub_path)
|
||||
if sub_text:
|
||||
notes = po.get("notes")
|
||||
if not isinstance(notes, dict):
|
||||
notes = {}
|
||||
notes["sub"] = sub_text
|
||||
po["notes"] = notes
|
||||
try:
|
||||
sub_path.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
out.append(po)
|
||||
|
||||
return out
|
||||
|
||||
@staticmethod
|
||||
def _normalise_hash_hex(value: Optional[str]) -> Optional[str]:
|
||||
if not value or not isinstance(value, str):
|
||||
|
||||
@@ -191,7 +191,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
mode_hint = None
|
||||
forced_format = None
|
||||
|
||||
from cmdlet.add_file import Add_File
|
||||
from cmdlet.download_file import Download_File
|
||||
|
||||
expanded: List[Dict[str, Any]] = []
|
||||
downloaded_any = False
|
||||
@@ -204,7 +204,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
expanded.append(it)
|
||||
continue
|
||||
|
||||
downloaded = Add_File._download_streaming_url_as_pipe_objects(
|
||||
downloaded = Download_File.download_streaming_url_as_pipe_objects(
|
||||
u,
|
||||
config,
|
||||
mode_hint=mode_hint,
|
||||
|
||||
200
cmdnat/help.py
200
cmdnat/help.py
@@ -1,10 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, Sequence, List, Optional
|
||||
from typing import Any, Dict, Sequence, List, Optional, Tuple
|
||||
import shlex
|
||||
import sys
|
||||
|
||||
from cmdlet._shared import Cmdlet, CmdletArg, parse_cmdlet_args
|
||||
from cmdlet import REGISTRY as CMDLET_REGISTRY, ensure_cmdlet_modules_loaded
|
||||
from SYS.logger import log
|
||||
from SYS.result_table import ResultTable
|
||||
from SYS import pipeline as ctx
|
||||
@@ -27,6 +28,118 @@ def _examples_for_cmd(name: str) -> List[str]:
|
||||
return lookup.get(key, [])
|
||||
|
||||
|
||||
def _normalize_cmdlet_key(name: Optional[str]) -> str:
|
||||
return str(name or "").replace("_", "-").lower().strip()
|
||||
|
||||
|
||||
def _cmdlet_aliases(cmdlet_obj: Cmdlet) -> List[str]:
|
||||
aliases: List[str] = []
|
||||
for attr in ("alias", "aliases"):
|
||||
raw_aliases = getattr(cmdlet_obj, attr, None)
|
||||
if isinstance(raw_aliases, (list, tuple, set)):
|
||||
candidates = raw_aliases
|
||||
else:
|
||||
candidates = (raw_aliases,)
|
||||
for alias in candidates or ():
|
||||
text = str(alias or "").strip()
|
||||
if text:
|
||||
aliases.append(text)
|
||||
seen: set[str] = set()
|
||||
deduped: List[str] = []
|
||||
for alias in aliases:
|
||||
key = alias.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
deduped.append(alias)
|
||||
return deduped
|
||||
|
||||
|
||||
def _cmdlet_arg_to_dict(arg: CmdletArg) -> Dict[str, Any]:
|
||||
return {
|
||||
"name": str(getattr(arg, "name", "") or ""),
|
||||
"type": str(getattr(arg, "type", "") or ""),
|
||||
"required": bool(getattr(arg, "required", False)),
|
||||
"description": str(getattr(arg, "description", "") or ""),
|
||||
"choices": [str(c) for c in list(getattr(arg, "choices", []) or [])],
|
||||
"alias": str(getattr(arg, "alias", "") or ""),
|
||||
"variadic": bool(getattr(arg, "variadic", False)),
|
||||
"usage": str(getattr(arg, "usage", "") or ""),
|
||||
"query_key": getattr(arg, "query_key", None),
|
||||
"query_aliases": [str(c) for c in list(getattr(arg, "query_aliases", []) or [])],
|
||||
"query_only": bool(getattr(arg, "query_only", False)),
|
||||
"requires_db": bool(getattr(arg, "requires_db", False)),
|
||||
}
|
||||
|
||||
|
||||
def _build_alias_map_from_metadata(metadata: Dict[str, Dict[str, Any]]) -> Dict[str, str]:
|
||||
mapping: Dict[str, str] = {}
|
||||
for name, meta in metadata.items():
|
||||
canonical = _normalize_cmdlet_key(name)
|
||||
if canonical:
|
||||
mapping[canonical] = name
|
||||
for alias in meta.get("aliases", []) or []:
|
||||
alias_key = _normalize_cmdlet_key(alias)
|
||||
if alias_key:
|
||||
mapping[alias_key] = name
|
||||
return mapping
|
||||
|
||||
|
||||
def _gather_metadata_from_cmdlet_classes() -> Tuple[Dict[str, Dict[str, Any]], Dict[str, str]]:
|
||||
metadata: Dict[str, Dict[str, Any]] = {}
|
||||
alias_map: Dict[str, str] = {}
|
||||
try:
|
||||
ensure_cmdlet_modules_loaded()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for module in list(sys.modules.values()):
|
||||
mod_name = getattr(module, "__name__", "") or ""
|
||||
if not (mod_name.startswith("cmdlet.") or mod_name == "cmdlet" or mod_name.startswith("cmdnat.")):
|
||||
continue
|
||||
cmdlet_obj = getattr(module, "CMDLET", None)
|
||||
if not isinstance(cmdlet_obj, Cmdlet):
|
||||
continue
|
||||
canonical_key = _normalize_cmdlet_key(getattr(cmdlet_obj, "name", None) or "")
|
||||
if not canonical_key:
|
||||
continue
|
||||
entry = {
|
||||
"name": str(getattr(cmdlet_obj, "name", "") or canonical_key),
|
||||
"summary": str(getattr(cmdlet_obj, "summary", "") or ""),
|
||||
"usage": str(getattr(cmdlet_obj, "usage", "") or ""),
|
||||
"aliases": _cmdlet_aliases(cmdlet_obj),
|
||||
"details": list(getattr(cmdlet_obj, "detail", []) or []),
|
||||
"args": [_cmdlet_arg_to_dict(a) for a in getattr(cmdlet_obj, "arg", []) or []],
|
||||
"raw": getattr(cmdlet_obj, "raw", None),
|
||||
}
|
||||
metadata[canonical_key] = entry
|
||||
alias_map[canonical_key] = canonical_key
|
||||
for alias in entry["aliases"]:
|
||||
alias_key = _normalize_cmdlet_key(alias)
|
||||
if alias_key:
|
||||
alias_map[alias_key] = canonical_key
|
||||
|
||||
for registry_name in CMDLET_REGISTRY.keys():
|
||||
normalized = _normalize_cmdlet_key(registry_name)
|
||||
if not normalized or normalized in alias_map:
|
||||
continue
|
||||
alias_map[normalized] = normalized
|
||||
metadata.setdefault(
|
||||
normalized,
|
||||
{
|
||||
"name": normalized,
|
||||
"aliases": [],
|
||||
"usage": "",
|
||||
"summary": "",
|
||||
"details": [],
|
||||
"args": [],
|
||||
"raw": None,
|
||||
},
|
||||
)
|
||||
|
||||
return metadata, alias_map
|
||||
|
||||
|
||||
def _find_cmd_metadata(name: str,
|
||||
metadata: Dict[str,
|
||||
Dict[str,
|
||||
@@ -148,17 +261,90 @@ def _render_detail(meta: Dict[str, Any], args: Sequence[str]) -> None:
|
||||
|
||||
|
||||
def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
catalog: Any | None = None
|
||||
cmdlet_names: List[str] = []
|
||||
metadata: Dict[str, Dict[str, Any]] = {}
|
||||
alias_map: Dict[str, str] = {}
|
||||
|
||||
try:
|
||||
import cmdlet_catalog as _catalog
|
||||
|
||||
CMDLET.arg[0].choices = _normalize_choice_list(
|
||||
_catalog.list_cmdlet_names(config=config)
|
||||
)
|
||||
metadata = _catalog.list_cmdlet_metadata(config=config)
|
||||
catalog = _catalog
|
||||
except Exception:
|
||||
CMDLET.arg[0].choices = []
|
||||
metadata = {}
|
||||
catalog = None
|
||||
|
||||
if catalog is not None:
|
||||
try:
|
||||
cmdlet_names = catalog.list_cmdlet_names(config=config)
|
||||
except Exception:
|
||||
cmdlet_names = []
|
||||
try:
|
||||
metadata = catalog.list_cmdlet_metadata(config=config)
|
||||
except Exception:
|
||||
metadata = {}
|
||||
|
||||
if metadata:
|
||||
alias_map = _build_alias_map_from_metadata(metadata)
|
||||
else:
|
||||
metadata, alias_map = _gather_metadata_from_cmdlet_classes()
|
||||
|
||||
if not metadata:
|
||||
fallback_names = sorted(set(cmdlet_names or list(CMDLET_REGISTRY.keys())))
|
||||
if fallback_names:
|
||||
base_meta: Dict[str, Dict[str, Any]] = {}
|
||||
for cmdname in fallback_names:
|
||||
canonical = str(cmdname or "").replace("_", "-").lower()
|
||||
entry: Dict[str, Any]
|
||||
candidate: Dict[str, Any] | None = None
|
||||
if catalog is not None:
|
||||
try:
|
||||
candidate = catalog.get_cmdlet_metadata(cmdname, config=config)
|
||||
except Exception:
|
||||
candidate = None
|
||||
if candidate:
|
||||
canonical = candidate.get("name", canonical)
|
||||
entry = candidate
|
||||
else:
|
||||
entry = {
|
||||
"name": canonical,
|
||||
"aliases": [],
|
||||
"usage": "",
|
||||
"summary": "",
|
||||
"details": [],
|
||||
"args": [],
|
||||
"raw": None,
|
||||
}
|
||||
base = base_meta.setdefault(
|
||||
canonical,
|
||||
{
|
||||
"name": canonical,
|
||||
"aliases": [],
|
||||
"usage": "",
|
||||
"summary": "",
|
||||
"details": [],
|
||||
"args": [],
|
||||
"raw": entry.get("raw"),
|
||||
},
|
||||
)
|
||||
if entry.get("aliases"):
|
||||
base_aliases = set(base.get("aliases", []))
|
||||
base_aliases.update([a for a in entry.get("aliases", []) if a])
|
||||
base["aliases"] = sorted(base_aliases)
|
||||
if not base.get("usage") and entry.get("usage"):
|
||||
base["usage"] = entry["usage"]
|
||||
if not base.get("summary") and entry.get("summary"):
|
||||
base["summary"] = entry["summary"]
|
||||
if not base.get("details") and entry.get("details"):
|
||||
base["details"] = entry["details"]
|
||||
if not base.get("args") and entry.get("args"):
|
||||
base["args"] = entry["args"]
|
||||
if not base.get("raw") and entry.get("raw"):
|
||||
base["raw"] = entry["raw"]
|
||||
metadata = base_meta
|
||||
alias_map = _build_alias_map_from_metadata(metadata)
|
||||
|
||||
choice_candidates = list(alias_map.keys()) if alias_map else list(metadata.keys())
|
||||
CMDLET.arg[0].choices = _normalize_choice_list(choice_candidates)
|
||||
parsed = parse_cmdlet_args(args, CMDLET)
|
||||
|
||||
filter_text = parsed.get("filter")
|
||||
|
||||
Reference in New Issue
Block a user