goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

f

Browse Source
This commit is contained in:
Nose
2026-01-19 21:25:44 -08:00
parent 37e2ff6651
commit fcab85455d
13 changed files with 820 additions and 393 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
CLI.py
View File

@@ -7,6 +7,28 @@ This module intentionally uses a class-based architecture:
- all REPL/pipeline/cmdlet execution state lives on objects
"""
# When running the CLI directly (not via the 'mm' launcher), honor the
# repository config `debug` flag by enabling `MM_DEBUG` so import-time
# diagnostics and bootstrap debug output are visible without setting the
# environment variable manually.
import os
from pathlib import Path
if not os.environ.get("MM_DEBUG"):
try:
conf_path = Path(__file__).resolve().parent / "config.conf"
if conf_path.exists():
for ln in conf_path.read_text(encoding="utf-8").splitlines():
ln_strip = ln.strip()
if ln_strip.startswith("debug"):
parts = ln_strip.split("=", 1)
if len(parts) >= 2:
val = parts[1].strip().strip('"').strip("'").strip().lower()
if val in ("1", "true", "yes", "on"):
os.environ["MM_DEBUG"] = "1"
break
except Exception:
pass
import json
import re
import shlex
@@ -1325,7 +1347,7 @@ class CmdletExecutor:
console = Console()
class MedeiaCLI:
class CLI:
"""Main CLI application object."""
ROOT = Path(__file__).resolve().parent
@@ -2469,4 +2491,4 @@ Come to love it when others take what you share, as there is no greater joy
if __name__ == "__main__":
MedeiaCLI().run()
CLI().run()

800
SYS/pipeline.py
View File

@@ -622,7 +622,9 @@ def set_last_result_table(
src_idx = getattr(row, "source_index", None)
if isinstance(src_idx, int) and 0 <= src_idx < len(state.last_result_items):
sorted_items.append(state.last_result_items[src_idx])
if len(sorted_items) == len(result_table.rows):
# Only reassign when the table actually contains rows and the reordering
# produced a complete mapping. Avoid clearing items when the table has no rows.
if result_table.rows and len(sorted_items) == len(result_table.rows):
state.last_result_items = sorted_items
except Exception:
pass
@@ -2136,6 +2138,54 @@ class PipelineExecutor:
stages[-1] = [cmd] + [str(x) for x in row_args] + tail
except Exception:
pass
# If no auto stage inserted and there are selection-action tokens available
# for the single selected row, apply it as the pipeline stage so a bare
# `@N` runs the intended action (e.g., get-file for hash-backed rows).
if not stages and selection_indices and len(selection_indices) == 1:
try:
idx = selection_indices[0]
debug(f"@N initial selection idx={idx} last_items={len(ctx.get_last_result_items() or [])}")
row_action = None
try:
row_action = ctx.get_current_stage_table_row_selection_action(idx)
except Exception:
row_action = None
if not row_action:
try:
items = ctx.get_last_result_items() or []
if 0 <= idx < len(items):
maybe = items[idx]
# Provide explicit debug output about the payload selected
try:
if isinstance(maybe, dict):
debug(f"@N payload: hash={maybe.get('hash')} store={maybe.get('store')} _selection_args={maybe.get('_selection_args')} _selection_action={maybe.get('_selection_action')}")
else:
debug(f"@N payload object type: {type(maybe).__name__}")
except Exception:
pass
if isinstance(maybe, dict):
candidate = maybe.get("_selection_action")
if isinstance(candidate, (list, tuple)):
row_action = [str(x) for x in candidate if x is not None]
debug(f"@N restored row_action from payload: {row_action}")
except Exception:
row_action = None
if row_action:
debug(f"@N applying row action -> {row_action}")
stages.append(row_action)
if pipeline_session and worker_manager:
try:
worker_manager.log_step(
pipeline_session.worker_id,
f"@N applied row action -> {' '.join(row_action)}",
)
except Exception:
pass
except Exception:
pass
else:
first_cmd = stages[0][0] if stages and stages[0] else None
if isinstance(table_type, str) and table_type.startswith("metadata.") and first_cmd not in (
@@ -2372,387 +2422,447 @@ class PipelineExecutor:
pipe_index_by_stage: Dict[int,
int] = {}
try:
ok, initial_piped = self._maybe_apply_initial_selection(
ctx,
config,
stages,
selection_indices=first_stage_selection_indices,
first_stage_had_extra_args=first_stage_had_extra_args,
worker_manager=worker_manager,
pipeline_session=pipeline_session,
)
if not ok:
return
if initial_piped is not None:
piped_result = initial_piped
except Exception as exc:
ok, initial_piped = self._maybe_apply_initial_selection(
ctx,
config,
stages,
selection_indices=first_stage_selection_indices,
first_stage_had_extra_args=first_stage_had_extra_args,
worker_manager=worker_manager,
pipeline_session=pipeline_session,
)
if not ok:
return
if initial_piped is not None:
piped_result = initial_piped
# REPL guard: prevent add-relationship before add-file for download-file pipelines.
if not self._validate_download_file_relationship_order(stages):
pipeline_status = "failed"
pipeline_error = f"{type(exc).__name__}: {exc}"
print(f"[error] {type(exc).__name__}: {exc}\n")
pipeline_error = "Invalid pipeline order"
return
# REPL guard: prevent add-relationship before add-file for download-file pipelines.
if not self._validate_download_file_relationship_order(stages):
pipeline_status = "failed"
pipeline_error = "Invalid pipeline order"
return
# ------------------------------------------------------------------
# Multi-level pipeline progress (pipes = stages, tasks = items)
# ------------------------------------------------------------------
progress_ui, pipe_index_by_stage = self._maybe_start_live_progress(config, stages)
# ------------------------------------------------------------------
# Multi-level pipeline progress (pipes = stages, tasks = items)
# ------------------------------------------------------------------
progress_ui, pipe_index_by_stage = self._maybe_start_live_progress(config, stages)
for stage_index, stage_tokens in enumerate(stages):
if not stage_tokens:
continue
for stage_index, stage_tokens in enumerate(stages):
if not stage_tokens:
continue
raw_stage_name = str(stage_tokens[0])
cmd_name = raw_stage_name.replace("_", "-").lower()
stage_args = stage_tokens[1:]
raw_stage_name = str(stage_tokens[0])
cmd_name = raw_stage_name.replace("_", "-").lower()
stage_args = stage_tokens[1:]
if cmd_name == "@":
# Prefer piping the last emitted/visible items (e.g. add-file results)
# over the result-table subject. The subject can refer to older context
# (e.g. a playlist row) and may not contain store+hash.
if cmd_name == "@":
# Prefer piping the last emitted/visible items (e.g. add-file results)
# over the result-table subject. The subject can refer to older context
# (e.g. a playlist row) and may not contain store+hash.
last_items = None
try:
last_items = ctx.get_last_result_items()
except Exception:
last_items = None
if last_items:
from cmdlet._shared import coerce_to_pipe_object
try:
last_items = ctx.get_last_result_items()
pipe_items = [
coerce_to_pipe_object(x) for x in list(last_items)
]
except Exception:
last_items = None
if last_items:
from cmdlet._shared import coerce_to_pipe_object
try:
pipe_items = [
coerce_to_pipe_object(x) for x in list(last_items)
]
except Exception:
pipe_items = list(last_items)
piped_result = pipe_items if len(pipe_items
) > 1 else pipe_items[0]
try:
ctx.set_last_items(pipe_items)
except Exception:
pass
if pipeline_session and worker_manager:
try:
worker_manager.log_step(
pipeline_session.worker_id,
"@ used last result items"
)
except Exception:
pass
continue
subject = ctx.get_last_result_subject()
if subject is None:
print("No current result context available for '@'\n")
pipeline_status = "failed"
pipeline_error = "No result items/subject for @"
return
piped_result = subject
pipe_items = list(last_items)
piped_result = pipe_items if len(pipe_items
) > 1 else pipe_items[0]
try:
subject_items = subject if isinstance(subject,
list) else [subject]
ctx.set_last_items(subject_items)
ctx.set_last_items(pipe_items)
except Exception:
pass
if pipeline_session and worker_manager:
try:
worker_manager.log_step(
pipeline_session.worker_id,
"@ used current table subject"
"@ used last result items"
)
except Exception:
pass
continue
if cmd_name.startswith("@"): # selection stage
selection_token = raw_stage_name
selection = SelectionSyntax.parse(selection_token)
filter_spec = SelectionFilterSyntax.parse(selection_token)
is_select_all = selection_token.strip() == "@*"
if selection is None and filter_spec is None and not is_select_all:
print(f"Invalid selection: {selection_token}\n")
pipeline_status = "failed"
pipeline_error = f"Invalid selection {selection_token}"
return
selected_indices = []
# Prefer selecting from the last selectable *table* (search/playlist)
# rather than from display-only emitted items, unless we're explicitly
# selecting from an overlay table.
display_table = None
subject = ctx.get_last_result_subject()
if subject is None:
print("No current result context available for '@'\n")
pipeline_status = "failed"
pipeline_error = "No result items/subject for @"
return
piped_result = subject
try:
subject_items = subject if isinstance(subject,
list) else [subject]
ctx.set_last_items(subject_items)
except Exception:
pass
if pipeline_session and worker_manager:
try:
display_table = (
ctx.get_display_table()
if hasattr(ctx,
"get_display_table") else None
worker_manager.log_step(
pipeline_session.worker_id,
"@ used current table subject"
)
except Exception:
display_table = None
stage_table = ctx.get_current_stage_table()
# Selection should operate on the table the user sees.
# If a display overlay table exists, force it as the current-stage table
# so provider selectors (e.g. tidal.album -> tracks) behave consistently.
try:
if display_table is not None and hasattr(ctx, "set_current_stage_table"):
ctx.set_current_stage_table(display_table)
stage_table = display_table
except Exception:
pass
continue
if not stage_table and display_table is not None:
if cmd_name.startswith("@"): # selection stage
selection_token = raw_stage_name
selection = SelectionSyntax.parse(selection_token)
filter_spec = SelectionFilterSyntax.parse(selection_token)
is_select_all = selection_token.strip() == "@*"
if selection is None and filter_spec is None and not is_select_all:
print(f"Invalid selection: {selection_token}\n")
pipeline_status = "failed"
pipeline_error = f"Invalid selection {selection_token}"
return
selected_indices = []
# Prefer selecting from the last selectable *table* (search/playlist)
# rather than from display-only emitted items, unless we're explicitly
# selecting from an overlay table.
display_table = None
try:
display_table = (
ctx.get_display_table()
if hasattr(ctx,
"get_display_table") else None
)
except Exception:
display_table = None
stage_table = ctx.get_current_stage_table()
# Selection should operate on the table the user sees.
# If a display overlay table exists, force it as the current-stage table
# so provider selectors (e.g. tidal.album -> tracks) behave consistently.
try:
if display_table is not None and hasattr(ctx, "set_current_stage_table"):
ctx.set_current_stage_table(display_table)
stage_table = display_table
if not stage_table:
stage_table = ctx.get_last_result_table()
except Exception:
pass
if not stage_table and display_table is not None:
stage_table = display_table
if not stage_table:
stage_table = ctx.get_last_result_table()
try:
if hasattr(ctx, "debug_table_state"):
ctx.debug_table_state(f"selection {selection_token}")
except Exception:
pass
if display_table is not None and stage_table is display_table:
items_list = ctx.get_last_result_items() or []
else:
if hasattr(ctx, "get_last_selectable_result_items"):
items_list = ctx.get_last_selectable_result_items(
) or []
else:
items_list = ctx.get_last_result_items() or []
if is_select_all:
selected_indices = list(range(len(items_list)))
elif filter_spec is not None:
selected_indices = [
i for i, item in enumerate(items_list)
if SelectionFilterSyntax.matches(item, filter_spec)
]
else:
selected_indices = sorted(
[i - 1 for i in selection]
) # type: ignore[arg-type]
resolved_items = items_list if items_list else []
filtered = [
resolved_items[i] for i in selected_indices
if 0 <= i < len(resolved_items)
]
# Debug: show selection resolution and sample payload info
try:
debug(f"Selection {selection_token} -> resolved_indices={selected_indices} filtered_count={len(filtered)}")
if filtered:
sample = filtered[0]
if isinstance(sample, dict):
debug(f"Selection sample: hash={sample.get('hash')} store={sample.get('store')} _selection_args={sample.get('_selection_args')} _selection_action={sample.get('_selection_action')}")
else:
try:
debug(f"Selection sample object: provider={getattr(sample, 'provider', None)} store={getattr(sample, 'store', None)}")
except Exception:
pass
except Exception:
pass
if not filtered:
print("No items matched selection\n")
pipeline_status = "failed"
pipeline_error = "Empty selection"
return
# Filter UX: if the stage token is a filter and it's terminal,
# render a filtered table overlay rather than selecting/auto-downloading.
stage_is_last = (stage_index + 1 >= len(stages))
if filter_spec is not None and stage_is_last:
try:
if hasattr(ctx, "debug_table_state"):
ctx.debug_table_state(f"selection {selection_token}")
base_table = stage_table
if base_table is None:
base_table = ctx.get_last_result_table()
if base_table is not None and hasattr(base_table, "copy_with_title"):
new_table = base_table.copy_with_title(getattr(base_table, "title", "") or "Results")
else:
new_table = Table(getattr(base_table, "title", "") if base_table is not None else "Results")
try:
if base_table is not None and getattr(base_table, "table", None):
new_table.set_table(str(getattr(base_table, "table")))
except Exception:
pass
try:
# Attach a one-line header so users see the active filter.
safe = str(selection_token)[1:].strip()
new_table.set_header_line(f'filter: "{safe}"')
except Exception:
pass
for item in filtered:
new_table.add_result(item)
try:
ctx.set_last_result_table_overlay(new_table, items=list(filtered), subject=ctx.get_last_result_subject())
except Exception:
pass
try:
stdout_console().print()
stdout_console().print(new_table)
except Exception:
pass
except Exception:
pass
if display_table is not None and stage_table is display_table:
items_list = ctx.get_last_result_items() or []
else:
if hasattr(ctx, "get_last_selectable_result_items"):
items_list = ctx.get_last_selectable_result_items(
) or []
else:
items_list = ctx.get_last_result_items() or []
if is_select_all:
selected_indices = list(range(len(items_list)))
elif filter_spec is not None:
selected_indices = [
i for i, item in enumerate(items_list)
if SelectionFilterSyntax.matches(item, filter_spec)
]
else:
selected_indices = sorted(
[i - 1 for i in selection]
) # type: ignore[arg-type]
resolved_items = items_list if items_list else []
filtered = [
resolved_items[i] for i in selected_indices
if 0 <= i < len(resolved_items)
]
if not filtered:
print("No items matched selection\n")
pipeline_status = "failed"
pipeline_error = "Empty selection"
return
# Filter UX: if the stage token is a filter and it's terminal,
# render a filtered table overlay rather than selecting/auto-downloading.
stage_is_last = (stage_index + 1 >= len(stages))
if filter_spec is not None and stage_is_last:
try:
base_table = stage_table
if base_table is None:
base_table = ctx.get_last_result_table()
if base_table is not None and hasattr(base_table, "copy_with_title"):
new_table = base_table.copy_with_title(getattr(base_table, "title", "") or "Results")
else:
new_table = Table(getattr(base_table, "title", "") if base_table is not None else "Results")
try:
if base_table is not None and getattr(base_table, "table", None):
new_table.set_table(str(getattr(base_table, "table")))
except Exception:
pass
try:
# Attach a one-line header so users see the active filter.
safe = str(selection_token)[1:].strip()
new_table.set_header_line(f'filter: "{safe}"')
except Exception:
pass
for item in filtered:
new_table.add_result(item)
try:
ctx.set_last_result_table_overlay(new_table, items=list(filtered), subject=ctx.get_last_result_subject())
except Exception:
pass
try:
stdout_console().print()
stdout_console().print(new_table)
except Exception:
pass
except Exception:
pass
continue
# UX: selecting a single URL row from get-url tables should open it.
# Only do this when the selection stage is terminal to avoid surprising
# side-effects in pipelines like `@1 | download-file`.
current_table = ctx.get_current_stage_table(
) or ctx.get_last_result_table()
if (not is_select_all) and (len(filtered) == 1):
try:
PipelineExecutor._maybe_open_url_selection(
current_table,
filtered,
stage_is_last=(stage_index + 1 >= len(stages)),
)
except Exception:
pass
if PipelineExecutor._maybe_run_class_selector(
ctx,
config,
filtered,
stage_is_last=(stage_index + 1 >= len(stages))):
return
# Special case: selecting multiple tags from get-tag and piping into delete-tag
# should batch into a single operation (one backend call).
next_cmd = None
try:
if stage_index + 1 < len(stages) and stages[stage_index + 1]:
next_cmd = str(stages[stage_index + 1][0]
).replace("_",
"-").lower()
except Exception:
next_cmd = None
def _is_tag_row(obj: Any) -> bool:
try:
if (hasattr(obj,
"__class__")
and obj.__class__.__name__ == "TagItem"
and hasattr(obj,
"tag_name")):
return True
except Exception:
pass
try:
if isinstance(obj, dict) and obj.get("tag_name"):
return True
except Exception:
pass
return False
if (next_cmd in {"delete-tag",
"delete_tag"} and len(filtered) > 1
and all(_is_tag_row(x) for x in filtered)):
from cmdlet._shared import get_field
tags: List[str] = []
first_hash = None
first_store = None
first_path = None
for item in filtered:
tag_name = get_field(item, "tag_name")
if tag_name:
tags.append(str(tag_name))
if first_hash is None:
first_hash = get_field(item, "hash")
if first_store is None:
first_store = get_field(item, "store")
if first_path is None:
first_path = get_field(item,
"path") or get_field(
item,
"target"
)
if tags:
grouped = {
"table": "tag.selection",
"media_kind": "tag",
"hash": first_hash,
"store": first_store,
"path": first_path,
"tag": tags,
}
piped_result = grouped
continue
from cmdlet._shared import coerce_to_pipe_object
filtered_pipe_objs = [
coerce_to_pipe_object(item) for item in filtered
]
piped_result = (
filtered_pipe_objs
if len(filtered_pipe_objs) > 1 else filtered_pipe_objs[0]
)
current_table = ctx.get_current_stage_table(
) or ctx.get_last_result_table()
table_type = (
current_table.table
if current_table and hasattr(current_table,
"table") else None
)
def _norm_stage_cmd(name: Any) -> str:
return str(name or "").replace("_", "-").strip().lower()
next_cmd = None
if stage_index + 1 < len(stages) and stages[stage_index + 1]:
next_cmd = _norm_stage_cmd(stages[stage_index + 1][0])
auto_stage = None
if isinstance(table_type, str) and table_type:
try:
from ProviderCore.registry import selection_auto_stage_for_table
# Preserve historical behavior: only forward selection-stage args
# to the auto stage when we are appending a new last stage.
at_end = bool(stage_index + 1 >= len(stages))
auto_stage = selection_auto_stage_for_table(
table_type,
stage_args if at_end else None,
)
except Exception:
auto_stage = None
# Auto-insert downloader stages for provider tables.
# IMPORTANT: do not auto-download for filter selections; they may match many rows.
if filter_spec is None:
if stage_index + 1 >= len(stages):
if auto_stage:
try:
print(f"Auto-running selection via {auto_stage[0]}")
except Exception:
pass
stages.append(list(auto_stage))
else:
if auto_stage:
auto_cmd = _norm_stage_cmd(auto_stage[0])
if next_cmd not in (auto_cmd, ".pipe", ".mpv"):
debug(f"Auto-inserting {auto_cmd} after selection")
stages.insert(stage_index + 1, list(auto_stage))
continue
cmd_fn = REGISTRY.get(cmd_name)
if not cmd_fn:
# UX: selecting a single URL row from get-url tables should open it.
# Only do this when the selection stage is terminal to avoid surprising
# side-effects in pipelines like `@1 | download-file`.
current_table = ctx.get_current_stage_table(
) or ctx.get_last_result_table()
if (not is_select_all) and (len(filtered) == 1):
try:
mod = import_cmd_module(cmd_name)
data = getattr(mod, "CMDLET", None) if mod else None
if data and hasattr(data, "exec") and callable(getattr(data, "exec")):
run_fn = getattr(data, "exec")
REGISTRY[cmd_name] = run_fn
cmd_fn = run_fn
PipelineExecutor._maybe_open_url_selection(
current_table,
filtered,
stage_is_last=(stage_index + 1 >= len(stages)),
)
except Exception:
cmd_fn = None
pass
if not cmd_fn:
print(f"Unknown command: {cmd_name}\n")
pipeline_status = "failed"
pipeline_error = f"Unknown command: {cmd_name}"
if PipelineExecutor._maybe_run_class_selector(
ctx,
config,
filtered,
stage_is_last=(stage_index + 1 >= len(stages))):
return
# Special case: selecting multiple tags from get-tag and piping into delete-tag
# should batch into a single operation (one backend call).
next_cmd = None
try:
if stage_index + 1 < len(stages) and stages[stage_index + 1]:
next_cmd = str(stages[stage_index + 1][0]
).replace("_",
"-").lower()
except Exception:
next_cmd = None
def _is_tag_row(obj: Any) -> bool:
try:
if (hasattr(obj,
"__class__")
and obj.__class__.__name__ == "TagItem"
and hasattr(obj,
"tag_name")):
return True
except Exception:
pass
try:
if isinstance(obj, dict) and obj.get("tag_name"):
return True
except Exception:
pass
return False
if (next_cmd in {"delete-tag",
"delete_tag"} and len(filtered) > 1
and all(_is_tag_row(x) for x in filtered)):
from cmdlet._shared import get_field
tags: List[str] = []
first_hash = None
first_store = None
first_path = None
for item in filtered:
tag_name = get_field(item, "tag_name")
if tag_name:
tags.append(str(tag_name))
if first_hash is None:
first_hash = get_field(item, "hash")
if first_store is None:
first_store = get_field(item, "store")
if first_path is None:
first_path = get_field(item,
"path") or get_field(
item,
"target"
)
if tags:
grouped = {
"table": "tag.selection",
"media_kind": "tag",
"hash": first_hash,
"store": first_store,
"path": first_path,
"tag": tags,
}
piped_result = grouped
continue
from cmdlet._shared import coerce_to_pipe_object
filtered_pipe_objs = [
coerce_to_pipe_object(item) for item in filtered
]
piped_result = (
filtered_pipe_objs
if len(filtered_pipe_objs) > 1 else filtered_pipe_objs[0]
)
current_table = ctx.get_current_stage_table(
) or ctx.get_last_result_table()
table_type = (
current_table.table
if current_table and hasattr(current_table,
"table") else None
)
def _norm_stage_cmd(name: Any) -> str:
return str(name or "").replace("_", "-").strip().lower()
next_cmd = None
if stage_index + 1 < len(stages) and stages[stage_index + 1]:
next_cmd = _norm_stage_cmd(stages[stage_index + 1][0])
auto_stage = None
if isinstance(table_type, str) and table_type:
try:
from ProviderCore.registry import selection_auto_stage_for_table
# Preserve historical behavior: only forward selection-stage args
# to the auto stage when we are appending a new last stage.
at_end = bool(stage_index + 1 >= len(stages))
auto_stage = selection_auto_stage_for_table(
table_type,
stage_args if at_end else None,
)
except Exception:
auto_stage = None
# Auto-insert downloader stages for provider tables.
# IMPORTANT: do not auto-download for filter selections; they may match many rows.
if filter_spec is None:
if stage_index + 1 >= len(stages):
if auto_stage:
try:
print(f"Auto-running selection via {auto_stage[0]}")
except Exception:
pass
stages.append(list(auto_stage))
else:
if auto_stage:
auto_cmd = _norm_stage_cmd(auto_stage[0])
if next_cmd not in (auto_cmd, ".pipe", ".mpv"):
debug(f"Auto-inserting {auto_cmd} after selection")
stages.insert(stage_index + 1, list(auto_stage))
continue
cmd_fn = REGISTRY.get(cmd_name)
if not cmd_fn:
try:
mod = import_cmd_module(cmd_name)
data = getattr(mod, "CMDLET", None) if mod else None
if data and hasattr(data, "exec") and callable(getattr(data, "exec")):
run_fn = getattr(data, "exec")
REGISTRY[cmd_name] = run_fn
cmd_fn = run_fn
except Exception:
cmd_fn = None
if not cmd_fn:
print(f"Unknown command: {cmd_name}\n")
pipeline_status = "failed"
pipeline_error = f"Unknown command: {cmd_name}"
return
try:
from SYS.models import PipelineStageContext
pipe_idx = pipe_index_by_stage.get(stage_index)
session = WorkerStages.begin_stage(
worker_manager,
cmd_name=cmd_name,
stage_tokens=stage_tokens,
config=config,
command_text=pipeline_text if pipeline_text else " ".join(stage_tokens),
)
try:
stage_ctx = PipelineStageContext(
stage_index=stage_index,
total_stages=len(stages),
pipe_index=pipe_idx,
worker_id=session.worker_id if session else None,
)
# Set context for the current run
ctx.set_stage_context(stage_ctx)
ctx.set_current_cmdlet_name(cmd_name)
ctx.set_current_stage_text(" ".join(stage_tokens))
ctx.clear_emits()
# RUN THE CMDLET
cmd_fn(piped_result, stage_args, config)
# Update piped_result for next stage from emitted items
stage_emits = list(stage_ctx.emits)
if stage_emits:
piped_result = stage_emits if len(stage_emits) > 1 else stage_emits[0]
else:
piped_result = None
if progress_ui is not None and pipe_idx is not None:
progress_ui.complete_stage(pipe_idx)
finally:
if session:
try:
session.close()
except Exception:
pass
except Exception as exc:
pipeline_status = "failed"
pipeline_error = f"{cmd_name}: {exc}"
debug(f"Error in stage {stage_index} ({cmd_name}): {exc}")
return
except Exception as exc:
pipeline_status = "failed"
pipeline_error = f"{type(exc).__name__}: {exc}"

2
Store/HydrusNetwork.py
View File

@@ -1934,7 +1934,7 @@ class HydrusNetwork(Store):
try:
if service_key:
# Mutate tags for many hashes in a single request
client.mutate_tags_by_key(hash=hashes, service_key=service_key, add_tags=list(tag_tuple))
client.mutate_tags_by_key(hashes=hashes, service_key=service_key, add_tags=list(tag_tuple))
any_success = True
continue
except Exception as exc:

154
Store/registry.py
View File

@@ -28,9 +28,7 @@ _DISCOVERED_CLASSES_CACHE: Optional[Dict[str, Type[BaseStore]]] = None
# Backends that failed to initialize earlier in the current process.
# Keyed by (store_type, instance_key) where instance_key is the name used under config.store.<type>.<instance_key>.
_FAILED_BACKEND_CACHE: Dict[tuple[str,
str],
str] = {}
_FAILED_BACKEND_CACHE: Dict[tuple[str, str], str] = {}
def _normalize_store_type(value: str) -> str:
@@ -63,13 +61,10 @@ def _discover_store_classes() -> Dict[str, Type[BaseStore]]:
import Store as store_pkg
discovered: Dict[str,
Type[BaseStore]] = {}
discovered: Dict[str, Type[BaseStore]] = {}
for module_info in pkgutil.iter_modules(store_pkg.__path__):
module_name = module_info.name
if module_name in {"__init__",
"_base",
"registry"}:
if module_name in {"__init__", "_base", "registry"}:
continue
try:
@@ -122,10 +117,7 @@ def _required_keys_for(store_cls: Type[BaseStore]) -> list[str]:
)
def _build_kwargs(store_cls: Type[BaseStore],
instance_name: str,
instance_config: Any) -> Dict[str,
Any]:
def _build_kwargs(store_cls: Type[BaseStore], instance_name: str, instance_config: Any) -> Dict[str, Any]:
if isinstance(instance_config, dict):
cfg_dict = dict(instance_config)
else:
@@ -134,13 +126,10 @@ def _build_kwargs(store_cls: Type[BaseStore],
required = _required_keys_for(store_cls)
# If NAME is required but not present, allow the instance key to provide it.
if (any(_normalize_config_key(k) == "NAME" for k in required)
and _get_case_insensitive(cfg_dict,
"NAME") is None):
if (any(_normalize_config_key(k) == "NAME" for k in required) and _get_case_insensitive(cfg_dict, "NAME") is None):
cfg_dict["NAME"] = str(instance_name)
kwargs: Dict[str,
Any] = {}
kwargs: Dict[str, Any] = {}
missing: list[str] = []
for key in required:
value = _get_case_insensitive(cfg_dict, key)
@@ -257,8 +246,7 @@ class Store:
# Convenience normalization for filesystem-like paths.
for key in list(kwargs.keys()):
if _normalize_config_key(key) in {"PATH",
"LOCATION"}:
if _normalize_config_key(key) in {"PATH", "LOCATION"}:
kwargs[key] = str(expand_path(kwargs[key]))
backend = store_cls(**kwargs)
@@ -283,8 +271,50 @@ class Store:
f"[Store] Failed to register {store_cls.__name__} instance '{instance_name}': {exc}"
)
def _resolve_backend_name(self,
backend_name: str) -> tuple[Optional[str], Optional[str]]:
def _resolve_backend_name(self, backend_name: str) -> tuple[Optional[str], Optional[str]]:
requested = str(backend_name or "")
if requested in self._backends:
return requested, None
requested_norm = _normalize_store_type(requested)
ci_matches = [
name for name in self._backends
if _normalize_store_type(name) == requested_norm
]
if len(ci_matches) == 1:
return ci_matches[0], None
if len(ci_matches) > 1:
return None, f"Ambiguous store alias '{backend_name}' matches {ci_matches}"
type_matches = [
name for name, store_type in self._backend_types.items()
if store_type == requested_norm
]
if len(type_matches) == 1:
return type_matches[0], None
if len(type_matches) > 1:
return None, (
f"Ambiguous store alias '{backend_name}' matches type '{requested_norm}': {type_matches}"
)
prefix_matches = [
name for name, store_type in self._backend_types.items()
if store_type.startswith(requested_norm)
]
if len(prefix_matches) == 1:
return prefix_matches[0], None
if len(prefix_matches) > 1:
return None, (
f"Ambiguous store alias '{backend_name}' matches type prefix '{requested_norm}': {prefix_matches}"
)
return None, None
# get_backend_instance implementation moved to the bottom of this file to avoid
# instantiating all backends during startup (see function `get_backend_instance`).
def _resolve_backend_name(self, backend_name: str) -> tuple[Optional[str], Optional[str]]:
requested = str(backend_name or "")
if requested in self._backends:
return requested, None
@@ -461,3 +491,85 @@ def list_configured_backend_names(config: Optional[Dict[str, Any]]) -> list[str]
return sorted(set(names))
except Exception:
return []
def get_backend_instance(config: Optional[Dict[str, Any]], backend_name: str, *, suppress_debug: bool = False) -> Optional[BaseStore]:
"""Instantiate and return a single store backend by configured name.
This avoids creating all configured backends (and opening their DBs)
when only a single backend is needed (common in `get-file`/`get-metadata`).
The function first tries a lightweight match against raw config NAME/value to
avoid calling `_build_kwargs` (which can raise if keys are missing).
Returns None when no matching backend is found or instantiation fails.
"""
if not backend_name:
return None
store_cfg = (config or {}).get("store") or {}
if not isinstance(store_cfg, dict):
return None
classes_by_type = _discover_store_classes()
desired = str(backend_name or "").strip().lower()
for raw_store_type, instances in store_cfg.items():
if not isinstance(instances, dict):
continue
store_type = _normalize_store_type(str(raw_store_type))
store_cls = classes_by_type.get(store_type)
if store_cls is None:
continue
# Fast path: match using raw 'NAME' or 'name' in config without building full kwargs
for instance_name, instance_cfg in instances.items():
candidate_alias = None
if isinstance(instance_cfg, dict):
candidate_alias = (
instance_cfg.get("NAME") or instance_cfg.get("name")
)
candidate_alias = str(candidate_alias or instance_name).strip()
if candidate_alias.lower() == desired:
try:
kwargs = _build_kwargs(store_cls, str(instance_name), instance_cfg)
except Exception as exc:
if not suppress_debug:
debug(f"[Store] Can't build kwargs for '{instance_name}' ({store_type}): {exc}")
return None
try:
for key in list(kwargs.keys()):
if _normalize_config_key(key) in {"PATH", "LOCATION"}:
kwargs[key] = str(expand_path(kwargs[key]))
except Exception:
pass
try:
backend = store_cls(**kwargs)
return backend
except Exception as exc:
if not suppress_debug:
debug(f"[Store] Failed to instantiate backend '{candidate_alias}': {exc}")
return None
# Fallback: build kwargs for each instance and compare resolved NAME
for instance_name, instance_cfg in instances.items():
try:
kwargs = _build_kwargs(store_cls, str(instance_name), instance_cfg)
except Exception:
continue
alias = str(kwargs.get("NAME") or instance_name).strip()
if alias.lower() != desired:
continue
try:
for key in list(kwargs.keys()):
if _normalize_config_key(key) in {"PATH", "LOCATION"}:
kwargs[key] = str(expand_path(kwargs[key]))
except Exception:
pass
try:
backend = store_cls(**kwargs)
return backend
except Exception as exc:
if not suppress_debug:
debug(f"[Store] Failed to instantiate backend '{alias}': {exc}")
return None
if not suppress_debug:
debug(f"[Store] Backend '{backend_name}' not found in config")
return None

19
cmdlet/get_file.py
View File

@@ -86,9 +86,22 @@ class Get_File(sh.Cmdlet):
debug(f"[get-file] Getting storage backend: {store_name}")
# Get storage backend
store = Store(config)
backend = store[store_name]
# Prefer instantiating only the named backend to avoid initializing all configured backends
try:
from Store.registry import get_backend_instance
backend = get_backend_instance(config, store_name, suppress_debug=True)
except Exception:
backend = None
if backend is None:
# Fallback to full registry when targeted instantiation fails
try:
store = Store(config)
backend = store[store_name]
except Exception:
log(f"Error: Storage backend '{store_name}' not found", file=sys.stderr)
return 1
debug(f"[get-file] Backend retrieved: {type(backend).__name__}")
# Get file metadata to determine name and extension

24
cmdlet/get_metadata.py
View File

@@ -192,10 +192,21 @@ class Get_Metadata(Cmdlet):
# Use storage backend to get metadata
try:
from Store import Store
# Instantiate only the required backend when possible to avoid initializing all configured backends
try:
from Store.registry import get_backend_instance
backend = get_backend_instance(config, storage_source, suppress_debug=True)
except Exception:
backend = None
storage = Store(config)
backend = storage[storage_source]
if backend is None:
try:
from Store import Store
storage = Store(config)
backend = storage[storage_source]
except Exception:
log(f"Storage backend '{storage_source}' not found", file=sys.stderr)
return 1
# Get metadata from backend
metadata = backend.get_metadata(file_hash)
@@ -290,6 +301,13 @@ class Get_Metadata(Cmdlet):
list(args))
self._add_table_body_row(table, row)
ctx.set_last_result_table_overlay(table, [row], row)
try:
from SYS.rich_display import render_item_details_panel
render_item_details_panel(row)
table._rendered_by_cmdlet = True
except Exception:
pass
ctx.emit(row)
return 0

124
cmdlet/search_file.py
View File

@@ -211,18 +211,27 @@ class search_file(Cmdlet):
worker_id = str(uuid.uuid4())
library_root = get_local_storage_path(config or {}) if get_local_storage_path else None
if not library_root:
try:
from Store import Store
storage_registry = Store(config=config or {})
# Try the first folder backend
for name in storage_registry.list_backends():
backend = storage_registry[name]
if type(backend).__name__ == "Folder":
library_root = expand_path(getattr(backend, "_location", None))
if library_root:
break
from Store.registry import get_backend_instance
# Try the first configured folder backend without instantiating all backends
store_cfg = (config or {}).get("store") or {}
folder_cfg = None
for raw_store_type, instances in store_cfg.items():
if _normalize_store_type(str(raw_store_type)) == "folder":
folder_cfg = instances
break
if isinstance(folder_cfg, dict):
for instance_name, instance_config in folder_cfg.items():
try:
backend = get_backend_instance(config, instance_name, suppress_debug=True)
if backend and type(backend).__name__ == "Folder":
library_root = expand_path(getattr(backend, "_location", None))
if library_root:
break
except Exception:
pass
except Exception:
pass
@@ -655,10 +664,9 @@ class search_file(Cmdlet):
except Exception:
pass
from Store import Store
from Store.registry import list_configured_backend_names, get_backend_instance
from Store._base import Store as BaseStore
storage = storage_registry
backend_to_search = storage_backend or None
if hash_query:
# Explicit hash list search: build rows from backend metadata.
@@ -666,7 +674,7 @@ class search_file(Cmdlet):
if backend_to_search:
backends_to_try = [backend_to_search]
else:
backends_to_try = list(storage.list_backends())
backends_to_try = list_configured_backend_names(config or {})
found_any = False
for h in hash_query:
@@ -674,9 +682,17 @@ class search_file(Cmdlet):
resolved_backend = None
for backend_name in backends_to_try:
backend = None
try:
backend = storage[backend_name]
backend = get_backend_instance(config, backend_name, suppress_debug=True)
if backend is None:
# Last-resort: instantiate full registry for this backend only
from Store import Store as _Store
_store = _Store(config=config)
backend = _store[backend_name]
except Exception:
backend = None
if backend is None:
continue
try:
# If get_metadata works, consider it a hit; get_file can be optional (e.g. remote URL).
@@ -828,7 +844,17 @@ class search_file(Cmdlet):
if backend_to_search:
searched_backends.append(backend_to_search)
target_backend = storage[backend_to_search]
try:
target_backend = get_backend_instance(config, backend_to_search, suppress_debug=True)
if target_backend is None:
from Store import Store as _Store
_store = _Store(config=config)
target_backend = _store[backend_to_search]
except Exception as exc:
log(f"Backend '{backend_to_search}' not found: {exc}", file=sys.stderr)
db.update_worker_status(worker_id, "error")
return 1
if type(target_backend).search is BaseStore.search:
log(
f"Backend '{backend_to_search}' does not support searching",
@@ -843,11 +869,19 @@ class search_file(Cmdlet):
)
else:
all_results = []
for backend_name in storage.list_searchable_backends():
for backend_name in list_configured_backend_names(config or {}):
try:
backend = storage[backend_name]
backend = get_backend_instance(config, backend_name, suppress_debug=True)
if backend is None:
from Store import Store as _Store
_store = _Store(config=config)
backend = _store[backend_name]
searched_backends.append(backend_name)
if type(backend).search is BaseStore.search:
continue
debug(f"[search-file] Searching '{backend_name}'")
backend_results = backend.search(
query,
@@ -909,6 +943,62 @@ class search_file(Cmdlet):
if store_val and not normalized.get("store"):
normalized["store"] = store_val
# Populate default selection args for interactive @N selection/hash/url handling
try:
sel_args: Optional[List[str]] = None
sel_action: Optional[List[str]] = None
# Prefer explicit path when available
p_val = normalized.get("path") or normalized.get("target") or normalized.get("url")
if p_val:
p_str = str(p_val or "").strip()
if p_str:
if p_str.startswith(("http://", "https://", "magnet:", "torrent:")):
sel_args = ["-url", p_str]
sel_action = ["download-file", "-url", p_str]
else:
try:
from SYS.utils import expand_path
full_path = expand_path(p_str)
# Prefer showing metadata details when we have a hash+store context
h = normalized.get("hash") or normalized.get("file_hash") or normalized.get("hash_hex")
s_val = normalized.get("store")
if h and s_val:
try:
h_norm = normalize_hash(h)
except Exception:
h_norm = str(h)
sel_args = ["-query", f"hash:{h_norm}", "-store", str(s_val)]
sel_action = ["get-metadata", "-query", f"hash:{h_norm}", "-store", str(s_val)]
else:
sel_args = ["-path", str(full_path)]
# Default action for local paths: get-file to fetch or operate on the path
sel_action = ["get-file", "-path", str(full_path)]
except Exception:
sel_args = ["-path", p_str]
sel_action = ["get-file", "-path", p_str]
# Fallback: use hash+store when available
if sel_args is None:
h = normalized.get("hash") or normalized.get("file_hash") or normalized.get("hash_hex")
s_val = normalized.get("store")
if h and s_val:
try:
h_norm = normalize_hash(h)
except Exception:
h_norm = str(h)
sel_args = ["-query", f"hash:{h_norm}", "-store", str(s_val)]
# Show metadata details by default for store/hash selections
sel_action = ["get-metadata", "-query", f"hash:{h_norm}", "-store", str(s_val)]
if sel_args:
normalized["_selection_args"] = [str(x) for x in sel_args]
if sel_action:
normalized["_selection_action"] = [str(x) for x in sel_action]
except Exception:
pass
table.add_result(normalized)
results_list.append(normalized)

23
scripts/check_indentation.py Normal file
View File

@@ -0,0 +1,23 @@
from pathlib import Path
p = Path('Store/registry.py')
lines = p.read_text(encoding='utf-8').splitlines()
stack = [0]
for i, line in enumerate(lines, start=1):
if not line.strip():
continue
leading = len(line) - len(line.lstrip(' '))
if leading > stack[-1]:
# increased indent -> push
stack.append(leading)
else:
# dedent: must match an existing level
if leading != stack[-1]:
if leading in stack:
# pop until match
while stack and stack[-1] != leading:
stack.pop()
else:
print(f"Line {i}: inconsistent indent {leading}, stack levels {stack}")
print(repr(line))
break
print('Done')

12
scripts/check_store.py Normal file
View File

@@ -0,0 +1,12 @@
import traceback
try:
from Store import Store
s = Store(config={}, suppress_debug=True)
print('INSTANCE TYPE:', type(s))
print('HAS is_available:', hasattr(s, 'is_available'))
if hasattr(s, 'is_available'):
print('is_available callable:', callable(getattr(s, 'is_available')))
print('DIR:', sorted([n for n in dir(s) if not n.startswith('__')]))
except Exception:
traceback.print_exc()

2
scripts/cli_entry.py
View File

@@ -184,7 +184,7 @@ def _run_cli(clean_args: List[str]) -> int:
if MedeiaCLI is None:
try:
repo_root = _ensure_repo_root_on_sys_path()
from CLI import MedeiaCLI as _M # type: ignore
from CLI import CLI as _M # type: ignore
MedeiaCLI = _M
except Exception as exc:
# Provide diagnostic information

6
scripts/find_big_indent.py Normal file
View File

@@ -0,0 +1,6 @@
from pathlib import Path
p = Path('Store/registry.py')
for i, line in enumerate(p.read_text(encoding='utf-8').splitlines(), start=1):
leading = len(line) - len(line.lstrip(' '))
if leading > 20:
print(i, leading, repr(line))

11
scripts/indent_check.py Normal file
View File

@@ -0,0 +1,11 @@
import pathlib
p = pathlib.Path('Store/registry.py')
with p.open('r', encoding='utf-8') as f:
lines = f.readlines()
for i in range(312, 328):
if i-1 < len(lines):
line = lines[i-1]
leading = line[:len(line)-len(line.lstrip('\t '))]
print(f"{i}: {repr(line.rstrip())} | leading={repr(leading)} len={len(leading)} chars={[ord(c) for c in leading]}")
else:
print(f"{i}: <EOF>")

10
scripts/indent_stats.py Normal file
View File

@@ -0,0 +1,10 @@
from pathlib import Path
p = Path('Store/registry.py')
counts = {}
for i, line in enumerate(p.read_text(encoding='utf-8').splitlines(), start=1):
if not line.strip():
continue
leading = len(line) - len(line.lstrip(' '))
counts[leading] = counts.get(leading, 0) + 1
for k in sorted(counts.keys()):
print(k, counts[k])