goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

khh
Some checks failed
smoke-mm / Install & smoke test mm --help (push) Has been cancelled

Browse Source
This commit is contained in:
nose
2025-12-24 02:13:21 -08:00
parent 8bf04c6b71
commit 24dd18de7e
20 changed files with 1792 additions and 636 deletions
Download Patch File Download Diff File Expand all files Collapse all files

455
TUI/pipeline_runner.py
View File

@@ -1,18 +1,16 @@
"""Pipeline execution utilities for the Textual UI.
This module mirrors the CLI pipeline behaviour while exposing a class-based
interface that the TUI can call. It keeps all pipeline/cmdlet integration in
one place so the interface layer stays focused on presentation.
The TUI is a frontend to the CLI, so it must use the same pipeline executor
implementation as the CLI (`CLI.PipelineExecutor`).
"""
from __future__ import annotations
import contextlib
import io
import shlex
import uuid
from dataclasses import dataclass, field
import sys
from pathlib import Path
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Optional, Sequence
BASE_DIR = Path(__file__).resolve().parent
@@ -23,11 +21,10 @@ for path in (ROOT_DIR, BASE_DIR):
sys.path.insert(0, str_path)
import pipeline as ctx
from cmdlet import REGISTRY
from config import get_local_storage_path, load_config
from SYS.worker_manager import WorkerManager
from CLI import MedeiaCLI
from CLI import ConfigLoader, PipelineExecutor as CLIPipelineExecutor, WorkerManagerRegistry
from SYS.logger import set_debug
from rich_display import capture_rich_output
from result_table import ResultTable
@dataclass(slots=True)
@@ -73,24 +70,16 @@ class PipelineRunResult:
}
class PipelineExecutor:
"""Thin wrapper over the cmdlet registry + pipeline context."""
class PipelineRunner:
"""TUI wrapper that delegates to the canonical CLI pipeline executor."""
def __init__(
self,
*,
config: Optional[Dict[str, Any]] = None,
worker_manager: Optional[WorkerManager] = None,
) -> None:
self._config = config or load_config()
self._worker_manager = worker_manager
if self._worker_manager is None:
self._worker_manager = self._ensure_worker_manager()
if self._worker_manager:
self._config["_worker_manager"] = self._worker_manager
def __init__(self) -> None:
self._config_loader = ConfigLoader(root=ROOT_DIR)
self._executor = CLIPipelineExecutor(config_loader=self._config_loader)
self._worker_manager = None
@property
def worker_manager(self) -> Optional[WorkerManager]:
def worker_manager(self):
return self._worker_manager
def run_pipeline(
@@ -98,290 +87,214 @@ class PipelineExecutor:
pipeline_text: str,
*,
seeds: Optional[Any] = None,
isolate: bool = False,
on_log: Optional[Callable[[str], None]] = None,
) -> PipelineRunResult:
"""Execute a pipeline string and return structured results.
snapshot: Optional[Dict[str, Any]] = None
if isolate:
snapshot = self._snapshot_ctx_state()
Args:
pipeline_text: Raw pipeline text entered by the user.
on_log: Optional callback that receives human-readable log lines.
"""
normalized = pipeline_text.strip()
normalized = str(pipeline_text or "").strip()
result = PipelineRunResult(pipeline=normalized, success=False)
if not normalized:
result.error = "Pipeline is empty"
return result
tokens = self._tokenize(normalized)
stages = self._split_stages(tokens)
if not stages:
result.error = "Pipeline contains no stages"
try:
from cli_syntax import validate_pipeline_text
syntax_error = validate_pipeline_text(normalized)
if syntax_error:
result.error = syntax_error.message
result.stderr = syntax_error.message
return result
except Exception:
pass
try:
tokens = shlex.split(normalized)
except Exception as exc:
result.error = f"Syntax error: {exc}"
result.stderr = result.error
return result
if not tokens:
result.error = "Pipeline contains no tokens"
return result
config = self._config_loader.load()
try:
set_debug(bool(config.get("debug", False)))
except Exception:
pass
try:
self._worker_manager = WorkerManagerRegistry.ensure(config)
except Exception:
self._worker_manager = None
ctx.reset()
ctx.set_current_command_text(normalized)
if seeds is not None:
try:
# Mirror CLI behavior: treat seeds as output of a virtual previous stage.
if not isinstance(seeds, list):
seeds = [seeds]
setter = getattr(ctx, "set_last_result_items_only", None)
if callable(setter):
setter(seeds)
else:
ctx.set_last_items(list(seeds))
ctx.set_last_result_items_only(list(seeds))
except Exception:
pass
stdout_buffer = io.StringIO()
stderr_buffer = io.StringIO()
piped_result: Any = None
worker_session = self._start_worker_session(normalized)
try:
with contextlib.redirect_stdout(stdout_buffer), contextlib.redirect_stderr(
stderr_buffer
):
for index, stage_tokens in enumerate(stages):
stage = self._execute_stage(
index=index,
total=len(stages),
stage_tokens=stage_tokens,
piped_input=piped_result,
on_log=on_log,
)
result.stages.append(stage)
if stage.status != "completed":
result.error = stage.error or f"Stage {stage.name} failed"
return result
if index == len(stages) - 1:
result.emitted = stage.emitted
result.result_table = stage.result_table
else:
piped_result = stage.emitted
result.success = True
return result
with capture_rich_output(stdout=stdout_buffer, stderr=stderr_buffer):
with contextlib.redirect_stdout(stdout_buffer), contextlib.redirect_stderr(stderr_buffer):
if on_log:
on_log("Executing pipeline via CLI executor...")
self._executor.execute_tokens(list(tokens))
except Exception as exc:
result.error = f"{type(exc).__name__}: {exc}"
finally:
try:
ctx.clear_current_command_text()
except Exception:
pass
result.stdout = stdout_buffer.getvalue()
result.stderr = stderr_buffer.getvalue()
ctx.clear_current_command_text()
if worker_session is not None:
status = "completed" if result.success else "error"
worker_session.finish(status=status, message=result.error or "")
# ------------------------------------------------------------------
# Stage execution helpers
# ------------------------------------------------------------------
def _execute_stage(
self,
*,
index: int,
total: int,
stage_tokens: Sequence[str],
piped_input: Any,
on_log: Optional[Callable[[str], None]],
) -> PipelineStageResult:
if not stage_tokens:
return PipelineStageResult(name="(empty)", args=[], status="skipped")
cmd_name = stage_tokens[0].replace("_", "-").lower()
stage_args = stage_tokens[1:]
stage = PipelineStageResult(name=cmd_name, args=stage_args)
if cmd_name.startswith("@"):
return self._apply_selection_stage(
token=cmd_name,
stage=stage,
piped_input=piped_input,
on_log=on_log,
)
cmd_fn = REGISTRY.get(cmd_name)
if not cmd_fn:
stage.status = "failed"
stage.error = f"Unknown command: {cmd_name}"
return stage
pipeline_ctx = ctx.PipelineStageContext(stage_index=index, total_stages=total, pipe_index=index)
ctx.set_stage_context(pipeline_ctx)
# Pull the canonical state out of pipeline context.
table = None
try:
return_code = cmd_fn(piped_input, list(stage_args), self._config)
except Exception as exc: # pragma: no cover - surfaced in UI
stage.status = "failed"
stage.error = f"{type(exc).__name__}: {exc}"
if on_log:
on_log(stage.error)
return stage
finally:
ctx.set_stage_context(None)
emitted = list(getattr(pipeline_ctx, "emits", []) or [])
stage.emitted = emitted
# Capture the ResultTable if the cmdlet set one
# Check display table first (overlay), then last result table
stage.result_table = ctx.get_display_table() or ctx.get_last_result_table()
if return_code != 0:
stage.status = "failed"
stage.error = f"Exit code {return_code}"
else:
stage.status = "completed"
stage.error = None
worker_id = self._current_worker_id()
if self._worker_manager and worker_id:
label = f"[Stage {index + 1}/{total}] {cmd_name} {stage.status}"
self._worker_manager.log_step(worker_id, label)
# Don't clear the table if we just captured it, but ensure items are set for next stage
# If we have a table, we should probably keep it in ctx for history if needed
# But for pipeline execution, we mainly care about passing items to next stage
# ctx.set_last_result_table(None, emitted) <-- This was clearing it
# Ensure items are available for next stage
ctx.set_last_items(emitted)
return stage
def _apply_selection_stage(
self,
*,
token: str,
stage: PipelineStageResult,
piped_input: Any,
on_log: Optional[Callable[[str], None]],
) -> PipelineStageResult:
# Bare '@' means use the subject associated with the current result table (e.g., the file shown in a tag/URL view)
if token == "@":
subject = ctx.get_last_result_subject()
if subject is None:
stage.status = "failed"
stage.error = "Selection requested (@) but there is no current result context"
return stage
stage.emitted = subject if isinstance(subject, list) else [subject]
ctx.set_last_items(stage.emitted)
stage.status = "completed"
if on_log:
on_log("Selected current table subject via @")
return stage
selection = self._parse_selection(token)
items = piped_input or []
if not isinstance(items, list):
items = list(items if isinstance(items, Sequence) else [items])
if not items:
stage.status = "failed"
stage.error = "Selection requested but there is no upstream data"
return stage
if selection is None:
stage.emitted = list(items)
else:
zero_based = sorted(i - 1 for i in selection if i > 0)
stage.emitted = [items[i] for i in zero_based if 0 <= i < len(items)]
if not stage.emitted:
stage.status = "failed"
stage.error = "Selection matched no rows"
return stage
ctx.set_last_items(stage.emitted)
ctx.set_last_result_table(None, stage.emitted)
stage.status = "completed"
if on_log:
on_log(f"Selected {len(stage.emitted)} item(s) via {token}")
return stage
# ------------------------------------------------------------------
# Worker/session helpers
# ------------------------------------------------------------------
def _start_worker_session(self, pipeline_text: str) -> Optional[_WorkerSession]:
manager = self._ensure_worker_manager()
if manager is None:
return None
worker_id = f"tui_pipeline_{uuid.uuid4().hex[:8]}"
tracked = manager.track_worker(
worker_id,
worker_type="pipeline",
title="Pipeline run",
description=pipeline_text,
pipe=pipeline_text,
)
if not tracked:
return None
manager.log_step(worker_id, "Pipeline started")
self._config["_current_worker_id"] = worker_id
return _WorkerSession(manager=manager, worker_id=worker_id, config=self._config)
def _ensure_worker_manager(self) -> Optional[WorkerManager]:
if self._worker_manager:
return self._worker_manager
library_root = get_local_storage_path(self._config)
if not library_root:
return None
try:
self._worker_manager = WorkerManager(Path(library_root), auto_refresh_interval=0)
self._config["_worker_manager"] = self._worker_manager
table = ctx.get_display_table() or ctx.get_current_stage_table() or ctx.get_last_result_table()
except Exception:
self._worker_manager = None
return self._worker_manager
table = None
def _current_worker_id(self) -> Optional[str]:
worker_id = self._config.get("_current_worker_id")
return str(worker_id) if worker_id else None
# ------------------------------------------------------------------
# Parsing helpers
# ------------------------------------------------------------------
@staticmethod
def _tokenize(pipeline_text: str) -> List[str]:
items: List[Any] = []
try:
return shlex.split(pipeline_text)
except ValueError:
return pipeline_text.split()
items = list(ctx.get_last_result_items() or [])
except Exception:
items = []
if table is None and items:
try:
synth = ResultTable("Results")
for item in items:
synth.add_result(item)
table = synth
except Exception:
table = None
result.emitted = items
result.result_table = table
combined = (result.stdout + "\n" + result.stderr).strip().lower()
failure_markers = (
"unknown command:",
"pipeline order error:",
"invalid selection:",
"invalid pipeline syntax",
"failed to execute pipeline",
"[error]",
)
if result.error:
result.success = False
elif any(m in combined for m in failure_markers):
result.success = False
if not result.error:
result.error = "Pipeline failed"
else:
result.success = True
if isolate and snapshot is not None:
try:
self._restore_ctx_state(snapshot)
except Exception:
# Best-effort; isolation should never break normal operation.
pass
return result
@staticmethod
def _split_stages(tokens: Sequence[str]) -> List[List[str]]:
stages: List[List[str]] = []
current: List[str] = []
for token in tokens:
if token == "|":
if current:
stages.append(current)
current = []
else:
current.append(token)
if current:
stages.append(current)
return stages
def _snapshot_ctx_state() -> Dict[str, Any]:
"""Best-effort snapshot of pipeline context so TUI popups don't clobber UI state."""
@staticmethod
def _parse_selection(token: str) -> Optional[Sequence[int]]:
parsed = MedeiaCLI.parse_selection_syntax(token)
return sorted(parsed) if parsed else None
def _copy(val: Any) -> Any:
if isinstance(val, list):
return val.copy()
if isinstance(val, dict):
return val.copy()
return val
snap: Dict[str, Any] = {}
keys = [
"_LIVE_PROGRESS",
"_CURRENT_CONTEXT",
"_LAST_SEARCH_QUERY",
"_PIPELINE_REFRESHED",
"_PIPELINE_LAST_ITEMS",
"_LAST_RESULT_TABLE",
"_LAST_RESULT_ITEMS",
"_LAST_RESULT_SUBJECT",
"_RESULT_TABLE_HISTORY",
"_RESULT_TABLE_FORWARD",
"_CURRENT_STAGE_TABLE",
"_DISPLAY_ITEMS",
"_DISPLAY_TABLE",
"_DISPLAY_SUBJECT",
"_PIPELINE_LAST_SELECTION",
"_PIPELINE_COMMAND_TEXT",
"_CURRENT_CMDLET_NAME",
"_CURRENT_STAGE_TEXT",
"_PIPELINE_VALUES",
"_PENDING_PIPELINE_TAIL",
"_PENDING_PIPELINE_SOURCE",
"_UI_LIBRARY_REFRESH_CALLBACK",
]
class _WorkerSession:
"""Minimal worker session wrapper for the TUI executor."""
for k in keys:
snap[k] = _copy(getattr(ctx, k, None))
def __init__(self, *, manager: WorkerManager, worker_id: str, config: Optional[Dict[str, Any]] = None) -> None:
self._manager = manager
self.worker_id = worker_id
self._config = config
def finish(self, *, status: str, message: str) -> None:
# Deepen copies where nested lists are common.
try:
self._manager.finish_worker(self.worker_id, result=status, error_msg=message)
self._manager.log_step(self.worker_id, f"Pipeline {status}")
hist = list(getattr(ctx, "_RESULT_TABLE_HISTORY", []) or [])
snap["_RESULT_TABLE_HISTORY"] = [
(t, (items.copy() if isinstance(items, list) else list(items) if items else []), subj)
for (t, items, subj) in hist
if isinstance((t, items, subj), tuple)
]
except Exception:
pass
if self._config and self._config.get("_current_worker_id") == self.worker_id:
self._config.pop("_current_worker_id", None)
try:
fwd = list(getattr(ctx, "_RESULT_TABLE_FORWARD", []) or [])
snap["_RESULT_TABLE_FORWARD"] = [
(t, (items.copy() if isinstance(items, list) else list(items) if items else []), subj)
for (t, items, subj) in fwd
if isinstance((t, items, subj), tuple)
]
except Exception:
pass
try:
tail = list(getattr(ctx, "_PENDING_PIPELINE_TAIL", []) or [])
snap["_PENDING_PIPELINE_TAIL"] = [list(stage) for stage in tail if isinstance(stage, list)]
except Exception:
pass
try:
values = getattr(ctx, "_PIPELINE_VALUES", None)
if isinstance(values, dict):
snap["_PIPELINE_VALUES"] = values.copy()
except Exception:
pass
return snap
@staticmethod
def _restore_ctx_state(snapshot: Dict[str, Any]) -> None:
for k, v in (snapshot or {}).items():
try:
setattr(ctx, k, v)
except Exception:
pass

754
TUI/tui.py
View File

@@ -1,26 +1,20 @@
"""Modern Textual UI for driving Medeia-Macina pipelines."""
from __future__ import annotations
import json
import re
import sys
from pathlib import Path
from typing import Any, List, Optional, Sequence
from typing import Any, List, Optional, Sequence, Tuple
from textual import work
from textual import on, work
from textual.app import App, ComposeResult
from textual.binding import Binding
from textual.containers import Container, Horizontal, Vertical, VerticalScroll
from textual.widgets import (
Button,
DataTable,
Footer,
Header,
Input,
ListItem,
ListView,
Static,
TextArea,
Tree,
)
from textual.events import Key
from textual.containers import Container, Horizontal, Vertical
from textual.screen import ModalScreen
from textual.widgets import Button, DataTable, Footer, Header, Input, Label, OptionList, Select, Static, TextArea
from textual.widgets.option_list import Option
BASE_DIR = Path(__file__).resolve().parent
ROOT_DIR = BASE_DIR.parent
@@ -29,25 +23,198 @@ for path in (BASE_DIR, ROOT_DIR):
if str_path not in sys.path:
sys.path.insert(0, str_path)
from menu_actions import ( # type: ignore # noqa: E402
PIPELINE_PRESETS,
PipelinePreset,
)
from pipeline_runner import PipelineExecutor, PipelineRunResult # type: ignore # noqa: E402
from pipeline_runner import PipelineRunResult # type: ignore # noqa: E402
from result_table import ResultTable # type: ignore # noqa: E402
from config import load_config # type: ignore # noqa: E402
from Store.registry import Store as StoreRegistry # type: ignore # noqa: E402
from cmdlet_catalog import ensure_registry_loaded, list_cmdlet_names # type: ignore # noqa: E402
from cli_syntax import validate_pipeline_text # type: ignore # noqa: E402
class PresetListItem(ListItem):
"""List entry that stores its pipeline preset."""
from pipeline_runner import PipelineRunner # type: ignore # noqa: E402
def __init__(self, preset: PipelinePreset) -> None:
super().__init__(
Static(
f"[b]{preset.label}[/b]\n[pale_green4]{preset.description}[/pale_green4]",
classes="preset-entry",
)
)
self.preset = preset
def _dedup_preserve_order(items: List[str]) -> List[str]:
out: List[str] = []
seen: set[str] = set()
for raw in items:
s = str(raw or "").strip()
if not s:
continue
key = s.lower()
if key in seen:
continue
seen.add(key)
out.append(s)
return out
def _extract_tag_names(emitted: Sequence[Any]) -> List[str]:
tags: List[str] = []
for obj in emitted or []:
try:
if hasattr(obj, "tag_name"):
val = getattr(obj, "tag_name")
if val:
tags.append(str(val))
continue
except Exception:
pass
if isinstance(obj, dict):
for k in ("tag_name", "tag", "name", "value"):
v = obj.get(k)
if isinstance(v, str) and v.strip():
tags.append(v.strip())
break
continue
return _dedup_preserve_order(tags)
class TextPopup(ModalScreen[None]):
def __init__(self, *, title: str, text: str) -> None:
super().__init__()
self._title = str(title)
self._text = str(text or "")
def compose(self) -> ComposeResult:
yield Static(self._title, id="popup-title")
yield TextArea(self._text, id="popup-text", read_only=True)
yield Button("Close", id="popup-close")
def on_button_pressed(self, event: Button.Pressed) -> None:
if event.button.id == "popup-close":
self.dismiss(None)
class TagEditorPopup(ModalScreen[None]):
def __init__(self, *, seeds: Any, store_name: str, file_hash: Optional[str]) -> None:
super().__init__()
self._seeds = seeds
self._store = str(store_name or "").strip()
self._hash = str(file_hash or "").strip() if file_hash else ""
self._original_tags: List[str] = []
self._status: Optional[Static] = None
self._editor: Optional[TextArea] = None
def compose(self) -> ComposeResult:
yield Static("Tags", id="popup-title")
yield TextArea("", id="tags-editor")
with Horizontal(id="tags-buttons"):
yield Button("Save", id="tags-save")
yield Button("Close", id="tags-close")
yield Static("", id="tags-status")
def on_mount(self) -> None:
self._status = self.query_one("#tags-status", Static)
self._editor = self.query_one("#tags-editor", TextArea)
self._set_status("Loading tags…")
self._load_tags_background()
def _set_status(self, msg: str) -> None:
if self._status:
self._status.update(str(msg or ""))
@work(thread=True)
def _load_tags_background(self) -> None:
app = self.app # PipelineHubApp
try:
runner: PipelineRunner = getattr(app, "executor")
cmd = f"@1 | get-tag -emit"
res = runner.run_pipeline(cmd, seeds=self._seeds, isolate=True)
tags = _extract_tag_names(res.emitted)
except Exception as exc:
tags = []
try:
app.call_from_thread(self._set_status, f"Error: {type(exc).__name__}: {exc}")
except Exception:
self._set_status(f"Error: {type(exc).__name__}: {exc}")
self._original_tags = tags
try:
app.call_from_thread(self._apply_loaded_tags, tags)
except Exception:
self._apply_loaded_tags(tags)
def _apply_loaded_tags(self, tags: List[str]) -> None:
if self._editor:
self._editor.text = "\n".join(tags)
self._set_status(f"Loaded {len(tags)} tag(s)")
def _parse_editor_tags(self) -> List[str]:
raw = ""
try:
raw = str(self._editor.text or "") if self._editor else ""
except Exception:
raw = ""
lines = [t.strip() for t in raw.replace("\r\n", "\n").split("\n")]
return _dedup_preserve_order([t for t in lines if t])
def on_button_pressed(self, event: Button.Pressed) -> None:
if event.button.id == "tags-close":
self.dismiss(None)
return
if event.button.id == "tags-save":
self._save_tags()
def _save_tags(self) -> None:
desired = self._parse_editor_tags()
current = _dedup_preserve_order(list(self._original_tags or []))
desired_set = {t.lower() for t in desired}
current_set = {t.lower() for t in current}
to_add = [t for t in desired if t.lower() not in current_set]
to_del = [t for t in current if t.lower() not in desired_set]
if not to_add and not to_del:
self._set_status("No changes")
return
self._set_status("Saving…")
self._save_tags_background(to_add, to_del, desired)
@work(thread=True)
def _save_tags_background(self, to_add: List[str], to_del: List[str], desired: List[str]) -> None:
app = self.app # PipelineHubApp
try:
runner: PipelineRunner = getattr(app, "executor")
store_tok = json.dumps(self._store)
query_chunk = f" -query {json.dumps(f'hash:{self._hash}')}" if self._hash else ""
failures: List[str] = []
if to_del:
del_args = " ".join(json.dumps(t) for t in to_del)
del_cmd = f"@1 | delete-tag -store {store_tok}{query_chunk} {del_args}"
del_res = runner.run_pipeline(del_cmd, seeds=self._seeds, isolate=True)
if not getattr(del_res, "success", False):
failures.append(str(getattr(del_res, "error", "") or getattr(del_res, "stderr", "") or "delete-tag failed").strip())
if to_add:
add_args = " ".join(json.dumps(t) for t in to_add)
add_cmd = f"@1 | add-tag -store {store_tok}{query_chunk} {add_args}"
add_res = runner.run_pipeline(add_cmd, seeds=self._seeds, isolate=True)
if not getattr(add_res, "success", False):
failures.append(str(getattr(add_res, "error", "") or getattr(add_res, "stderr", "") or "add-tag failed").strip())
if failures:
msg = failures[0]
try:
app.call_from_thread(self._set_status, f"Error: {msg}")
except Exception:
self._set_status(f"Error: {msg}")
return
self._original_tags = list(desired)
try:
app.call_from_thread(self._set_status, f"Saved (+{len(to_add)}, -{len(to_del)})")
except Exception:
self._set_status(f"Saved (+{len(to_add)}, -{len(to_del)})")
except Exception as exc:
try:
app.call_from_thread(self._set_status, f"Error: {type(exc).__name__}: {exc}")
except Exception:
self._set_status(f"Error: {type(exc).__name__}: {exc}")
class PipelineHubApp(App):
@@ -58,22 +225,27 @@ class PipelineHubApp(App):
Binding("ctrl+enter", "run_pipeline", "Run Pipeline"),
Binding("f5", "refresh_workers", "Refresh Workers"),
Binding("ctrl+l", "focus_command", "Focus Input", show=False),
Binding("ctrl+g", "focus_logs", "Focus Logs", show=False),
]
def __init__(self) -> None:
super().__init__()
self.executor = PipelineExecutor()
self.executor = PipelineRunner()
self.result_items: List[Any] = []
self.log_lines: List[str] = []
self.command_input: Optional[Input] = None
self.store_select: Optional[Select] = None
self.path_input: Optional[Input] = None
self.log_output: Optional[TextArea] = None
self.results_table: Optional[DataTable] = None
self.metadata_tree: Optional[Tree] = None
self.worker_table: Optional[DataTable] = None
self.preset_list: Optional[ListView] = None
self.status_panel: Optional[Static] = None
self.current_result_table: Optional[ResultTable] = None
self.suggestion_list: Optional[OptionList] = None
self._cmdlet_names: List[str] = []
self._pipeline_running = False
self._pipeline_worker: Any = None
self._selected_row_index: int = 0
# ------------------------------------------------------------------
# Layout
@@ -81,43 +253,58 @@ class PipelineHubApp(App):
def compose(self) -> ComposeResult: # noqa: D401 - Textual compose hook
yield Header(show_clock=True)
with Container(id="app-shell"):
with Horizontal(id="command-pane"):
self.command_input = Input(
placeholder='download-data "<url>" | merge-file | add-tags -store local | add-file -storage local',
id="pipeline-input",
)
yield self.command_input
yield Button("Run", id="run-button", variant="primary")
self.status_panel = Static("Idle", id="status-panel")
yield self.status_panel
with Horizontal(id="content-row"):
with VerticalScroll(id="left-pane"):
yield Static("Pipeline Presets", classes="section-title")
self.preset_list = ListView(
*(PresetListItem(preset) for preset in PIPELINE_PRESETS),
id="preset-list",
)
yield self.preset_list
yield Static("Logs", classes="section-title")
self.log_output = TextArea(id="log-output", read_only=True)
yield self.log_output
yield Static("Workers", classes="section-title")
self.worker_table = DataTable(id="workers-table")
yield self.worker_table
with Vertical(id="right-pane"):
yield Static("Results", classes="section-title")
self.results_table = DataTable(id="results-table")
yield self.results_table
yield Static("Metadata", classes="section-title")
self.metadata_tree = Tree("Run a pipeline", id="metadata-tree")
yield self.metadata_tree
with Vertical(id="command-pane"):
with Horizontal(id="command-row"):
yield Input(placeholder="Enter pipeline command...", id="pipeline-input")
yield Button("Run", id="run-button")
yield Button("Tags", id="tags-button")
yield Button("Metadata", id="metadata-button")
yield Button("Relationships", id="relationships-button")
yield Static("Ready", id="status-panel")
yield OptionList(id="cmd-suggestions")
with Vertical(id="results-pane"):
yield Label("Results", classes="section-title")
yield DataTable(id="results-table")
with Vertical(id="bottom-pane"):
yield Label("Store + Output", classes="section-title")
with Horizontal(id="store-row"):
yield Select([], id="store-select")
yield Input(placeholder="Output path (optional)", id="output-path")
with Horizontal(id="logs-workers-row"):
with Vertical(id="logs-pane"):
yield Label("Logs", classes="section-title")
yield TextArea(id="log-output", read_only=True)
with Vertical(id="workers-pane"):
yield Label("Workers", classes="section-title")
yield DataTable(id="workers-table")
yield Footer()
def on_mount(self) -> None:
self.command_input = self.query_one("#pipeline-input", Input)
self.status_panel = self.query_one("#status-panel", Static)
self.results_table = self.query_one("#results-table", DataTable)
self.worker_table = self.query_one("#workers-table", DataTable)
self.log_output = self.query_one("#log-output", TextArea)
self.store_select = self.query_one("#store-select", Select)
self.path_input = self.query_one("#output-path", Input)
self.suggestion_list = self.query_one("#cmd-suggestions", OptionList)
if self.suggestion_list:
self.suggestion_list.display = False
if self.results_table:
self.results_table.cursor_type = "row"
self.results_table.zebra_stripes = True
self.results_table.add_columns("Row", "Title", "Source", "File")
if self.worker_table:
self.worker_table.add_columns("ID", "Type", "Status", "Details")
self._populate_store_options()
self._load_cmdlet_names()
if self.executor.worker_manager:
self.set_interval(2.0, self.refresh_workers)
self.refresh_workers()
@@ -131,10 +318,24 @@ class PipelineHubApp(App):
if self.command_input:
self.command_input.focus()
def action_focus_logs(self) -> None:
if self.log_output:
self.log_output.focus()
def action_run_pipeline(self) -> None:
if self._pipeline_running:
self.notify("Pipeline already running", severity="warning", timeout=3)
return
# Self-heal if the background worker already stopped (e.g. error in thread).
worker = self._pipeline_worker
try:
is_running = bool(getattr(worker, "is_running", False))
except Exception:
is_running = True
if (worker is None) or (not is_running):
self._pipeline_running = False
self._pipeline_worker = None
else:
self.notify("Pipeline already running", severity="warning", timeout=3)
return
if not self.command_input:
return
pipeline_text = self.command_input.value.strip()
@@ -142,12 +343,33 @@ class PipelineHubApp(App):
self.notify("Enter a pipeline to run", severity="warning", timeout=3)
return
pipeline_text = self._apply_store_path_and_tags(pipeline_text)
self._pipeline_running = True
self._set_status("Running…", level="info")
self._clear_log()
self._append_log_line(f"$ {pipeline_text}")
self._clear_results()
self._run_pipeline_background(pipeline_text)
self._pipeline_worker = self._run_pipeline_background(pipeline_text)
@on(Input.Changed, "#pipeline-input")
def on_pipeline_input_changed(self, event: Input.Changed) -> None:
text = str(event.value or "")
self._update_suggestions(text)
self._update_syntax_status(text)
@on(OptionList.OptionSelected, "#cmd-suggestions")
def on_suggestion_selected(self, event: OptionList.OptionSelected) -> None:
if not self.command_input or not self.suggestion_list:
return
try:
suggestion = str(event.option.prompt)
except Exception:
return
new_text = self._apply_suggestion_to_text(str(self.command_input.value or ""), suggestion)
self.command_input.value = new_text
self.suggestion_list.display = False
self.command_input.focus()
def action_refresh_workers(self) -> None:
self.refresh_workers()
@@ -158,34 +380,178 @@ class PipelineHubApp(App):
def on_button_pressed(self, event: Button.Pressed) -> None:
if event.button.id == "run-button":
self.action_run_pipeline()
elif event.button.id == "tags-button":
self._open_tags_popup()
elif event.button.id == "metadata-button":
self._open_metadata_popup()
elif event.button.id == "relationships-button":
self._open_relationships_popup()
def on_input_submitted(self, event: Input.Submitted) -> None:
if event.input.id == "pipeline-input":
self.action_run_pipeline()
def on_list_view_selected(self, event: ListView.Selected) -> None:
if isinstance(event.item, PresetListItem) and self.command_input:
self.command_input.value = event.item.preset.pipeline
self.notify(f"Loaded preset: {event.item.preset.label}", timeout=2)
event.stop()
def on_key(self, event: Key) -> None:
# Make Tab accept autocomplete when typing commands.
if event.key != "tab":
return
if not self.command_input or not self.command_input.has_focus:
return
suggestion = self._get_first_suggestion()
if not suggestion:
return
self.command_input.value = self._apply_suggestion_to_text(str(self.command_input.value or ""), suggestion)
if self.suggestion_list:
self.suggestion_list.display = False
event.prevent_default()
event.stop()
def _get_first_suggestion(self) -> str:
if not self.suggestion_list or not bool(getattr(self.suggestion_list, "display", False)):
return ""
# Textual OptionList API differs across versions; handle best-effort.
try:
options = list(getattr(self.suggestion_list, "options", []) or [])
if options:
first = options[0]
return str(getattr(first, "prompt", "") or "")
except Exception:
pass
return ""
def _populate_store_options(self) -> None:
"""Populate the store dropdown from the configured Store registry."""
if not self.store_select:
return
try:
cfg = load_config() or {}
except Exception:
cfg = {}
stores: List[str] = []
try:
stores = StoreRegistry(config=cfg, suppress_debug=True).list_backends()
except Exception:
stores = []
# Always offer a reasonable default even if config is missing.
if "local" not in [s.lower() for s in stores]:
stores = ["local", *stores]
options = [(name, name) for name in stores]
try:
self.store_select.set_options(options)
if options:
current = getattr(self.store_select, "value", None)
# Textual Select uses a sentinel for "no selection".
if (current is None) or (current == "") or (current is Select.BLANK):
self.store_select.value = options[0][1]
except Exception:
pass
def _get_selected_store(self) -> Optional[str]:
if not self.store_select:
return None
try:
value = getattr(self.store_select, "value", None)
except Exception:
return None
if value is None or value is Select.BLANK:
return None
try:
text = str(value).strip()
except Exception:
return None
if not text or text == "Select.BLANK":
return None
return text
def _apply_store_path_and_tags(self, pipeline_text: str) -> str:
"""Apply store/path/tags UI fields to the pipeline text.
Rules (simple + non-destructive):
- If output path is set and the first stage is download-media and has no -path/--path, append -path.
- If a store is selected and pipeline has no add-file stage, append add-file -store <store>.
"""
base = str(pipeline_text or "").strip()
if not base:
return base
selected_store = self._get_selected_store()
output_path = ""
if self.path_input:
try:
output_path = str(self.path_input.value or "").strip()
except Exception:
output_path = ""
stages = [s.strip() for s in base.split("|") if s.strip()]
if not stages:
return base
# Identify first stage command name for conservative auto-augmentation.
first_stage_cmd = ""
try:
first_stage_cmd = str(stages[0].split()[0]).replace("_", "-").strip().lower() if stages[0].split() else ""
except Exception:
first_stage_cmd = ""
# Apply -path to download-media first stage (only if missing)
if output_path:
first = stages[0]
low = first.lower()
if low.startswith("download-media") and " -path" not in low and " --path" not in low:
stages[0] = f"{first} -path {json.dumps(output_path)}"
joined = " | ".join(stages)
low_joined = joined.lower()
# Only auto-append add-file for download pipelines.
should_auto_add_file = bool(
selected_store
and ("add-file" not in low_joined)
and (first_stage_cmd in {"download-media", "download-file", "download-torrent"})
)
if should_auto_add_file:
store_token = json.dumps(selected_store)
joined = f"{joined} | add-file -store {store_token}"
return joined
def on_data_table_row_highlighted(self, event: DataTable.RowHighlighted) -> None:
if not self.results_table or event.control is not self.results_table:
return
index = event.cursor_row
if 0 <= index < len(self.result_items):
self._display_metadata(index)
index = int(event.cursor_row or 0)
if index < 0:
index = 0
self._selected_row_index = index
# ------------------------------------------------------------------
# Pipeline execution helpers
# ------------------------------------------------------------------
@work(exclusive=True, thread=True)
def _run_pipeline_background(self, pipeline_text: str) -> None:
run_result = self.executor.run_pipeline(pipeline_text, on_log=self._log_from_worker)
try:
run_result = self.executor.run_pipeline(pipeline_text, on_log=self._log_from_worker)
except Exception as exc:
# Ensure the UI never gets stuck in "running" state.
run_result = PipelineRunResult(
pipeline=str(pipeline_text or ""),
success=False,
error=f"{type(exc).__name__}: {exc}",
stderr=f"{type(exc).__name__}: {exc}",
)
self.call_from_thread(self._on_pipeline_finished, run_result)
def _on_pipeline_finished(self, run_result: PipelineRunResult) -> None:
self._pipeline_running = False
self._pipeline_worker = None
status_level = "success" if run_result.success else "error"
status_text = "Completed" if run_result.success else "Failed"
self._set_status(status_text, level=status_level)
@@ -219,6 +585,8 @@ class PipelineHubApp(App):
self.current_result_table = run_result.result_table
self._populate_results_table()
self.refresh_workers()
if self.result_items:
self._selected_row_index = 0
def _log_from_worker(self, message: str) -> None:
self.call_from_thread(self._append_log_line, message)
@@ -251,35 +619,213 @@ class PipelineHubApp(App):
for idx, item in enumerate(self.result_items, start=1):
self.results_table.add_row(str(idx), str(item), "—", "—", key=str(idx - 1))
def _display_metadata(self, index: int) -> None:
if not self.metadata_tree:
def _load_cmdlet_names(self) -> None:
try:
ensure_registry_loaded()
names = list_cmdlet_names() or []
self._cmdlet_names = sorted({str(n).replace("_", "-") for n in names if str(n).strip()})
except Exception:
self._cmdlet_names = []
def _update_syntax_status(self, text: str) -> None:
if self._pipeline_running:
return
root = self.metadata_tree.root
root.label = "Metadata"
root.remove_children()
if self.current_result_table and 0 <= index < len(self.current_result_table.rows):
row = self.current_result_table.rows[index]
for col in row.columns:
root.add(f"[b]{col.name}[/b]: {col.value}")
elif 0 <= index < len(self.result_items):
item = self.result_items[index]
if isinstance(item, dict):
self._populate_tree_node(root, item)
else:
root.add(str(item))
def _populate_tree_node(self, node, data: Any) -> None:
if isinstance(data, dict):
for key, value in data.items():
child = node.add(f"[b]{key}[/b]")
self._populate_tree_node(child, value)
elif isinstance(data, Sequence) and not isinstance(data, (str, bytes)):
for idx, value in enumerate(data):
child = node.add(f"[{idx}]")
self._populate_tree_node(child, value)
raw = str(text or "").strip()
if not raw:
self._set_status("Ready", level="info")
return
try:
err = validate_pipeline_text(raw)
except Exception:
err = None
if err:
self._set_status(err.message, level="error")
else:
node.add(str(data))
self._set_status("Ready", level="info")
def _update_suggestions(self, text: str) -> None:
if not self.suggestion_list:
return
raw = str(text or "")
prefix = self._current_cmd_prefix(raw)
if not prefix:
self.suggestion_list.display = False
return
pref_low = prefix.lower()
matches = [n for n in self._cmdlet_names if n.lower().startswith(pref_low)]
matches = matches[:10]
if not matches:
self.suggestion_list.display = False
return
try:
self.suggestion_list.clear_options() # type: ignore[attr-defined]
except Exception:
try:
# Fallback for older/newer Textual APIs.
self.suggestion_list.options = [] # type: ignore[attr-defined]
except Exception:
pass
try:
self.suggestion_list.add_options([Option(m) for m in matches]) # type: ignore[attr-defined]
except Exception:
try:
self.suggestion_list.options = [Option(m) for m in matches] # type: ignore[attr-defined]
except Exception:
pass
self.suggestion_list.display = True
@staticmethod
def _current_cmd_prefix(text: str) -> str:
"""Best-effort prefix for cmdlet name completion.
Completes the token immediately after start-of-line or a '|'.
"""
raw = str(text or "")
# Find the segment after the last pipe.
segment = raw.split("|")[-1]
# Remove leading whitespace.
segment = segment.lstrip()
if not segment:
return ""
# Only complete the first token of the segment.
m = re.match(r"([A-Za-z0-9_\-]*)", segment)
return m.group(1) if m else ""
@staticmethod
def _apply_suggestion_to_text(text: str, suggestion: str) -> str:
raw = str(text or "")
parts = raw.split("|")
if not parts:
return suggestion
last = parts[-1]
# Preserve leading spaces after the pipe.
leading = "".join(ch for ch in last if ch.isspace())
trimmed = last.lstrip()
# Replace first token in last segment.
replaced = re.sub(r"^[A-Za-z0-9_\-]*", suggestion, trimmed)
parts[-1] = leading + replaced
return "|".join(parts)
def _resolve_selected_item(self) -> Tuple[Optional[Any], Optional[str], Optional[str]]:
"""Return (item, store_name, hash) for the currently selected row."""
index = int(getattr(self, "_selected_row_index", 0) or 0)
if index < 0:
index = 0
item: Any = None
# Prefer mapping displayed table row -> source item.
if self.current_result_table and 0 <= index < len(getattr(self.current_result_table, "rows", []) or []):
row = self.current_result_table.rows[index]
src_idx = getattr(row, "source_index", None)
if isinstance(src_idx, int) and 0 <= src_idx < len(self.result_items):
item = self.result_items[src_idx]
if item is None and 0 <= index < len(self.result_items):
item = self.result_items[index]
store_name = None
file_hash = None
if isinstance(item, dict):
store_name = item.get("store")
file_hash = item.get("hash")
else:
store_name = getattr(item, "store", None)
file_hash = getattr(item, "hash", None)
store_text = str(store_name).strip() if store_name is not None else ""
hash_text = str(file_hash).strip() if file_hash is not None else ""
if not store_text:
# Fallback to UI store selection when item doesn't carry it.
store_text = self._get_selected_store() or ""
return item, (store_text or None), (hash_text or None)
def _open_tags_popup(self) -> None:
if self._pipeline_running:
self.notify("Pipeline already running", severity="warning", timeout=3)
return
item, store_name, file_hash = self._resolve_selected_item()
if item is None:
self.notify("No selected item", severity="warning", timeout=3)
return
if not store_name:
self.notify("Selected item missing store", severity="warning", timeout=4)
return
seeds: Any = item
if isinstance(item, dict):
seeds = dict(item)
try:
if store_name and not str(seeds.get("store") or "").strip():
seeds["store"] = store_name
except Exception:
pass
try:
if file_hash and not str(seeds.get("hash") or "").strip():
seeds["hash"] = file_hash
except Exception:
pass
self.push_screen(TagEditorPopup(seeds=seeds, store_name=store_name, file_hash=file_hash))
def _open_metadata_popup(self) -> None:
item, _store_name, _file_hash = self._resolve_selected_item()
if item is None:
self.notify("No selected item", severity="warning", timeout=3)
return
text = ""
idx = int(getattr(self, "_selected_row_index", 0) or 0)
if self.current_result_table and 0 <= idx < len(getattr(self.current_result_table, "rows", []) or []):
row = self.current_result_table.rows[idx]
lines = [f"{col.name}: {col.value}" for col in getattr(row, "columns", []) or []]
text = "\n".join(lines)
elif isinstance(item, dict):
try:
text = json.dumps(item, indent=2, ensure_ascii=False)
except Exception:
text = str(item)
else:
text = str(item)
self.push_screen(TextPopup(title="Metadata", text=text))
def _open_relationships_popup(self) -> None:
item, _store_name, _file_hash = self._resolve_selected_item()
if item is None:
self.notify("No selected item", severity="warning", timeout=3)
return
relationships = None
if isinstance(item, dict):
relationships = item.get("relationships") or item.get("relationship")
else:
relationships = getattr(item, "relationships", None)
if not relationships:
relationships = getattr(item, "get_relationships", lambda: None)()
if not relationships:
self.push_screen(TextPopup(title="Relationships", text="No relationships"))
return
lines: List[str] = []
if isinstance(relationships, dict):
for rel_type, value in relationships.items():
if isinstance(value, list):
if not value:
lines.append(f"{rel_type}: (empty)")
for v in value:
lines.append(f"{rel_type}: {v}")
else:
lines.append(f"{rel_type}: {value}")
else:
lines.append(str(relationships))
self.push_screen(TextPopup(title="Relationships", text="\n".join(lines)))
def _clear_log(self) -> None:
self.log_lines = []
@@ -301,9 +847,7 @@ class PipelineHubApp(App):
self.result_items = []
if self.results_table:
self.results_table.clear()
if self.metadata_tree:
self.metadata_tree.root.label = "Awaiting results"
self.metadata_tree.root.remove_children()
self._selected_row_index = 0
def _set_status(self, message: str, *, level: str = "info") -> None:
if not self.status_panel:

113
TUI/tui.tcss
View File

@@ -14,6 +14,11 @@
border: round $primary;
}
#command-row {
width: 100%;
height: auto;
}
#pipeline-input {
width: 1fr;
min-height: 3;
@@ -38,22 +43,61 @@
border: solid $panel-darken-1;
}
#content-row {
#cmd-suggestions {
width: 100%;
height: 1fr;
height: auto;
max-height: 8;
margin-top: 1;
background: $surface;
border: round $panel-darken-2;
}
#left-pane,
#right-pane {
#results-pane {
width: 100%;
height: 2fr;
padding: 1;
background: $panel;
border: round $panel-darken-2;
margin-top: 1;
}
#store-select {
width: 24;
margin-right: 2;
height: 3;
}
#output-path {
width: 1fr;
height: 100%;
height: 3;
}
#bottom-pane {
width: 100%;
height: 1fr;
padding: 1;
background: $panel;
border: round $panel-darken-2;
}
#left-pane {
max-width: 60;
#store-row {
width: 100%;
height: auto;
}
#logs-workers-row {
width: 100%;
height: 1fr;
margin-top: 1;
}
#logs-pane,
#workers-pane {
width: 1fr;
height: 100%;
padding: 0 1;
}
.section-title {
@@ -62,33 +106,19 @@
margin-top: 1;
}
.preset-entry {
padding: 1;
border: tall $panel-darken-1;
margin-bottom: 1;
}
#preset-list {
height: 25;
border: solid $secondary;
}
#log-output {
height: 16;
height: 1fr;
}
#workers-table {
height: auto;
height: 1fr;
}
#results-table {
height: 1fr;
}
#metadata-tree {
height: 1fr;
border: round $panel-darken-1;
}
.status-info {
background: $boost;
@@ -109,4 +139,39 @@
width: auto;
min-width: 10;
margin: 0 1;
}
#tags-button,
#metadata-button,
#relationships-button {
width: auto;
min-width: 12;
margin: 0 1;
}
#popup-title {
width: 100%;
height: 3;
text-style: bold;
content-align: center middle;
border: round $panel-darken-2;
background: $boost;
}
#popup-text,
#tags-editor {
height: 1fr;
border: round $panel-darken-2;
}
#tags-buttons {
width: 100%;
height: auto;
margin-top: 1;
}
#tags-status {
width: 1fr;
height: 3;
content-align: left middle;
}