From fcab85455d205a13e5c98e81fb24cc73c87f98c2 Mon Sep 17 00:00:00 2001 From: Nose Date: Mon, 19 Jan 2026 21:25:44 -0800 Subject: [PATCH] f --- CLI.py | 26 +- SYS/pipeline.py | 800 ++++++++++++++++++++--------------- Store/HydrusNetwork.py | 2 +- Store/registry.py | 154 ++++++- cmdlet/get_file.py | 19 +- cmdlet/get_metadata.py | 24 +- cmdlet/search_file.py | 124 +++++- scripts/check_indentation.py | 23 + scripts/check_store.py | 12 + scripts/cli_entry.py | 2 +- scripts/find_big_indent.py | 6 + scripts/indent_check.py | 11 + scripts/indent_stats.py | 10 + 13 files changed, 820 insertions(+), 393 deletions(-) create mode 100644 scripts/check_indentation.py create mode 100644 scripts/check_store.py create mode 100644 scripts/find_big_indent.py create mode 100644 scripts/indent_check.py create mode 100644 scripts/indent_stats.py diff --git a/CLI.py b/CLI.py index 798f876..eb6b55f 100644 --- a/CLI.py +++ b/CLI.py @@ -7,6 +7,28 @@ This module intentionally uses a class-based architecture: - all REPL/pipeline/cmdlet execution state lives on objects """ +# When running the CLI directly (not via the 'mm' launcher), honor the +# repository config `debug` flag by enabling `MM_DEBUG` so import-time +# diagnostics and bootstrap debug output are visible without setting the +# environment variable manually. +import os +from pathlib import Path +if not os.environ.get("MM_DEBUG"): + try: + conf_path = Path(__file__).resolve().parent / "config.conf" + if conf_path.exists(): + for ln in conf_path.read_text(encoding="utf-8").splitlines(): + ln_strip = ln.strip() + if ln_strip.startswith("debug"): + parts = ln_strip.split("=", 1) + if len(parts) >= 2: + val = parts[1].strip().strip('"').strip("'").strip().lower() + if val in ("1", "true", "yes", "on"): + os.environ["MM_DEBUG"] = "1" + break + except Exception: + pass + import json import re import shlex @@ -1325,7 +1347,7 @@ class CmdletExecutor: console = Console() -class MedeiaCLI: +class CLI: """Main CLI application object.""" ROOT = Path(__file__).resolve().parent @@ -2469,4 +2491,4 @@ Come to love it when others take what you share, as there is no greater joy if __name__ == "__main__": - MedeiaCLI().run() + CLI().run() diff --git a/SYS/pipeline.py b/SYS/pipeline.py index 4b5e620..fa1eeaa 100644 --- a/SYS/pipeline.py +++ b/SYS/pipeline.py @@ -622,7 +622,9 @@ def set_last_result_table( src_idx = getattr(row, "source_index", None) if isinstance(src_idx, int) and 0 <= src_idx < len(state.last_result_items): sorted_items.append(state.last_result_items[src_idx]) - if len(sorted_items) == len(result_table.rows): + # Only reassign when the table actually contains rows and the reordering + # produced a complete mapping. Avoid clearing items when the table has no rows. + if result_table.rows and len(sorted_items) == len(result_table.rows): state.last_result_items = sorted_items except Exception: pass @@ -2136,6 +2138,54 @@ class PipelineExecutor: stages[-1] = [cmd] + [str(x) for x in row_args] + tail except Exception: pass + + # If no auto stage inserted and there are selection-action tokens available + # for the single selected row, apply it as the pipeline stage so a bare + # `@N` runs the intended action (e.g., get-file for hash-backed rows). + if not stages and selection_indices and len(selection_indices) == 1: + try: + idx = selection_indices[0] + debug(f"@N initial selection idx={idx} last_items={len(ctx.get_last_result_items() or [])}") + row_action = None + try: + row_action = ctx.get_current_stage_table_row_selection_action(idx) + except Exception: + row_action = None + + if not row_action: + try: + items = ctx.get_last_result_items() or [] + if 0 <= idx < len(items): + maybe = items[idx] + # Provide explicit debug output about the payload selected + try: + if isinstance(maybe, dict): + debug(f"@N payload: hash={maybe.get('hash')} store={maybe.get('store')} _selection_args={maybe.get('_selection_args')} _selection_action={maybe.get('_selection_action')}") + else: + debug(f"@N payload object type: {type(maybe).__name__}") + except Exception: + pass + if isinstance(maybe, dict): + candidate = maybe.get("_selection_action") + if isinstance(candidate, (list, tuple)): + row_action = [str(x) for x in candidate if x is not None] + debug(f"@N restored row_action from payload: {row_action}") + except Exception: + row_action = None + + if row_action: + debug(f"@N applying row action -> {row_action}") + stages.append(row_action) + if pipeline_session and worker_manager: + try: + worker_manager.log_step( + pipeline_session.worker_id, + f"@N applied row action -> {' '.join(row_action)}", + ) + except Exception: + pass + except Exception: + pass else: first_cmd = stages[0][0] if stages and stages[0] else None if isinstance(table_type, str) and table_type.startswith("metadata.") and first_cmd not in ( @@ -2372,387 +2422,447 @@ class PipelineExecutor: pipe_index_by_stage: Dict[int, int] = {} - try: - ok, initial_piped = self._maybe_apply_initial_selection( - ctx, - config, - stages, - selection_indices=first_stage_selection_indices, - first_stage_had_extra_args=first_stage_had_extra_args, - worker_manager=worker_manager, - pipeline_session=pipeline_session, - ) - if not ok: - return - if initial_piped is not None: - piped_result = initial_piped - except Exception as exc: + ok, initial_piped = self._maybe_apply_initial_selection( + ctx, + config, + stages, + selection_indices=first_stage_selection_indices, + first_stage_had_extra_args=first_stage_had_extra_args, + worker_manager=worker_manager, + pipeline_session=pipeline_session, + ) + if not ok: + return + if initial_piped is not None: + piped_result = initial_piped + + # REPL guard: prevent add-relationship before add-file for download-file pipelines. + if not self._validate_download_file_relationship_order(stages): pipeline_status = "failed" - pipeline_error = f"{type(exc).__name__}: {exc}" - print(f"[error] {type(exc).__name__}: {exc}\n") + pipeline_error = "Invalid pipeline order" return - # REPL guard: prevent add-relationship before add-file for download-file pipelines. - if not self._validate_download_file_relationship_order(stages): - pipeline_status = "failed" - pipeline_error = "Invalid pipeline order" - return + # ------------------------------------------------------------------ + # Multi-level pipeline progress (pipes = stages, tasks = items) + # ------------------------------------------------------------------ + progress_ui, pipe_index_by_stage = self._maybe_start_live_progress(config, stages) - # ------------------------------------------------------------------ - # Multi-level pipeline progress (pipes = stages, tasks = items) - # ------------------------------------------------------------------ - progress_ui, pipe_index_by_stage = self._maybe_start_live_progress(config, stages) + for stage_index, stage_tokens in enumerate(stages): + if not stage_tokens: + continue - for stage_index, stage_tokens in enumerate(stages): - if not stage_tokens: - continue + raw_stage_name = str(stage_tokens[0]) + cmd_name = raw_stage_name.replace("_", "-").lower() + stage_args = stage_tokens[1:] - raw_stage_name = str(stage_tokens[0]) - cmd_name = raw_stage_name.replace("_", "-").lower() - stage_args = stage_tokens[1:] - - if cmd_name == "@": - # Prefer piping the last emitted/visible items (e.g. add-file results) - # over the result-table subject. The subject can refer to older context - # (e.g. a playlist row) and may not contain store+hash. + if cmd_name == "@": + # Prefer piping the last emitted/visible items (e.g. add-file results) + # over the result-table subject. The subject can refer to older context + # (e.g. a playlist row) and may not contain store+hash. + last_items = None + try: + last_items = ctx.get_last_result_items() + except Exception: last_items = None + + if last_items: + from cmdlet._shared import coerce_to_pipe_object + try: - last_items = ctx.get_last_result_items() + pipe_items = [ + coerce_to_pipe_object(x) for x in list(last_items) + ] except Exception: - last_items = None - - if last_items: - from cmdlet._shared import coerce_to_pipe_object - - try: - pipe_items = [ - coerce_to_pipe_object(x) for x in list(last_items) - ] - except Exception: - pipe_items = list(last_items) - piped_result = pipe_items if len(pipe_items - ) > 1 else pipe_items[0] - try: - ctx.set_last_items(pipe_items) - except Exception: - pass - if pipeline_session and worker_manager: - try: - worker_manager.log_step( - pipeline_session.worker_id, - "@ used last result items" - ) - except Exception: - pass - continue - - subject = ctx.get_last_result_subject() - if subject is None: - print("No current result context available for '@'\n") - pipeline_status = "failed" - pipeline_error = "No result items/subject for @" - return - piped_result = subject + pipe_items = list(last_items) + piped_result = pipe_items if len(pipe_items + ) > 1 else pipe_items[0] try: - subject_items = subject if isinstance(subject, - list) else [subject] - ctx.set_last_items(subject_items) + ctx.set_last_items(pipe_items) except Exception: pass if pipeline_session and worker_manager: try: worker_manager.log_step( pipeline_session.worker_id, - "@ used current table subject" + "@ used last result items" ) except Exception: pass continue - if cmd_name.startswith("@"): # selection stage - selection_token = raw_stage_name - selection = SelectionSyntax.parse(selection_token) - filter_spec = SelectionFilterSyntax.parse(selection_token) - is_select_all = selection_token.strip() == "@*" - if selection is None and filter_spec is None and not is_select_all: - print(f"Invalid selection: {selection_token}\n") - pipeline_status = "failed" - pipeline_error = f"Invalid selection {selection_token}" - return - - selected_indices = [] - # Prefer selecting from the last selectable *table* (search/playlist) - # rather than from display-only emitted items, unless we're explicitly - # selecting from an overlay table. - display_table = None + subject = ctx.get_last_result_subject() + if subject is None: + print("No current result context available for '@'\n") + pipeline_status = "failed" + pipeline_error = "No result items/subject for @" + return + piped_result = subject + try: + subject_items = subject if isinstance(subject, + list) else [subject] + ctx.set_last_items(subject_items) + except Exception: + pass + if pipeline_session and worker_manager: try: - display_table = ( - ctx.get_display_table() - if hasattr(ctx, - "get_display_table") else None + worker_manager.log_step( + pipeline_session.worker_id, + "@ used current table subject" ) - except Exception: - display_table = None - - stage_table = ctx.get_current_stage_table() - # Selection should operate on the table the user sees. - # If a display overlay table exists, force it as the current-stage table - # so provider selectors (e.g. tidal.album -> tracks) behave consistently. - try: - if display_table is not None and hasattr(ctx, "set_current_stage_table"): - ctx.set_current_stage_table(display_table) - stage_table = display_table except Exception: pass + continue - if not stage_table and display_table is not None: + if cmd_name.startswith("@"): # selection stage + selection_token = raw_stage_name + selection = SelectionSyntax.parse(selection_token) + filter_spec = SelectionFilterSyntax.parse(selection_token) + is_select_all = selection_token.strip() == "@*" + if selection is None and filter_spec is None and not is_select_all: + print(f"Invalid selection: {selection_token}\n") + pipeline_status = "failed" + pipeline_error = f"Invalid selection {selection_token}" + return + + selected_indices = [] + # Prefer selecting from the last selectable *table* (search/playlist) + # rather than from display-only emitted items, unless we're explicitly + # selecting from an overlay table. + display_table = None + try: + display_table = ( + ctx.get_display_table() + if hasattr(ctx, + "get_display_table") else None + ) + except Exception: + display_table = None + + stage_table = ctx.get_current_stage_table() + # Selection should operate on the table the user sees. + # If a display overlay table exists, force it as the current-stage table + # so provider selectors (e.g. tidal.album -> tracks) behave consistently. + try: + if display_table is not None and hasattr(ctx, "set_current_stage_table"): + ctx.set_current_stage_table(display_table) stage_table = display_table - if not stage_table: - stage_table = ctx.get_last_result_table() + except Exception: + pass + if not stage_table and display_table is not None: + stage_table = display_table + if not stage_table: + stage_table = ctx.get_last_result_table() + + try: + if hasattr(ctx, "debug_table_state"): + ctx.debug_table_state(f"selection {selection_token}") + except Exception: + pass + + if display_table is not None and stage_table is display_table: + items_list = ctx.get_last_result_items() or [] + else: + if hasattr(ctx, "get_last_selectable_result_items"): + items_list = ctx.get_last_selectable_result_items( + ) or [] + else: + items_list = ctx.get_last_result_items() or [] + + if is_select_all: + selected_indices = list(range(len(items_list))) + elif filter_spec is not None: + selected_indices = [ + i for i, item in enumerate(items_list) + if SelectionFilterSyntax.matches(item, filter_spec) + ] + else: + selected_indices = sorted( + [i - 1 for i in selection] + ) # type: ignore[arg-type] + + resolved_items = items_list if items_list else [] + filtered = [ + resolved_items[i] for i in selected_indices + if 0 <= i < len(resolved_items) + ] + # Debug: show selection resolution and sample payload info + try: + debug(f"Selection {selection_token} -> resolved_indices={selected_indices} filtered_count={len(filtered)}") + if filtered: + sample = filtered[0] + if isinstance(sample, dict): + debug(f"Selection sample: hash={sample.get('hash')} store={sample.get('store')} _selection_args={sample.get('_selection_args')} _selection_action={sample.get('_selection_action')}") + else: + try: + debug(f"Selection sample object: provider={getattr(sample, 'provider', None)} store={getattr(sample, 'store', None)}") + except Exception: + pass + except Exception: + pass + + if not filtered: + print("No items matched selection\n") + pipeline_status = "failed" + pipeline_error = "Empty selection" + return + + # Filter UX: if the stage token is a filter and it's terminal, + # render a filtered table overlay rather than selecting/auto-downloading. + stage_is_last = (stage_index + 1 >= len(stages)) + if filter_spec is not None and stage_is_last: try: - if hasattr(ctx, "debug_table_state"): - ctx.debug_table_state(f"selection {selection_token}") + base_table = stage_table + if base_table is None: + base_table = ctx.get_last_result_table() + + if base_table is not None and hasattr(base_table, "copy_with_title"): + new_table = base_table.copy_with_title(getattr(base_table, "title", "") or "Results") + else: + new_table = Table(getattr(base_table, "title", "") if base_table is not None else "Results") + + try: + if base_table is not None and getattr(base_table, "table", None): + new_table.set_table(str(getattr(base_table, "table"))) + except Exception: + pass + + try: + # Attach a one-line header so users see the active filter. + safe = str(selection_token)[1:].strip() + new_table.set_header_line(f'filter: "{safe}"') + except Exception: + pass + + for item in filtered: + new_table.add_result(item) + + try: + ctx.set_last_result_table_overlay(new_table, items=list(filtered), subject=ctx.get_last_result_subject()) + except Exception: + pass + + try: + stdout_console().print() + stdout_console().print(new_table) + except Exception: + pass except Exception: pass - - if display_table is not None and stage_table is display_table: - items_list = ctx.get_last_result_items() or [] - else: - if hasattr(ctx, "get_last_selectable_result_items"): - items_list = ctx.get_last_selectable_result_items( - ) or [] - else: - items_list = ctx.get_last_result_items() or [] - - if is_select_all: - selected_indices = list(range(len(items_list))) - elif filter_spec is not None: - selected_indices = [ - i for i, item in enumerate(items_list) - if SelectionFilterSyntax.matches(item, filter_spec) - ] - else: - selected_indices = sorted( - [i - 1 for i in selection] - ) # type: ignore[arg-type] - - resolved_items = items_list if items_list else [] - filtered = [ - resolved_items[i] for i in selected_indices - if 0 <= i < len(resolved_items) - ] - if not filtered: - print("No items matched selection\n") - pipeline_status = "failed" - pipeline_error = "Empty selection" - return - - # Filter UX: if the stage token is a filter and it's terminal, - # render a filtered table overlay rather than selecting/auto-downloading. - stage_is_last = (stage_index + 1 >= len(stages)) - if filter_spec is not None and stage_is_last: - try: - base_table = stage_table - if base_table is None: - base_table = ctx.get_last_result_table() - - if base_table is not None and hasattr(base_table, "copy_with_title"): - new_table = base_table.copy_with_title(getattr(base_table, "title", "") or "Results") - else: - new_table = Table(getattr(base_table, "title", "") if base_table is not None else "Results") - - try: - if base_table is not None and getattr(base_table, "table", None): - new_table.set_table(str(getattr(base_table, "table"))) - except Exception: - pass - - try: - # Attach a one-line header so users see the active filter. - safe = str(selection_token)[1:].strip() - new_table.set_header_line(f'filter: "{safe}"') - except Exception: - pass - - for item in filtered: - new_table.add_result(item) - - try: - ctx.set_last_result_table_overlay(new_table, items=list(filtered), subject=ctx.get_last_result_subject()) - except Exception: - pass - - try: - stdout_console().print() - stdout_console().print(new_table) - except Exception: - pass - except Exception: - pass - continue - - # UX: selecting a single URL row from get-url tables should open it. - # Only do this when the selection stage is terminal to avoid surprising - # side-effects in pipelines like `@1 | download-file`. - current_table = ctx.get_current_stage_table( - ) or ctx.get_last_result_table() - if (not is_select_all) and (len(filtered) == 1): - try: - PipelineExecutor._maybe_open_url_selection( - current_table, - filtered, - stage_is_last=(stage_index + 1 >= len(stages)), - ) - except Exception: - pass - - if PipelineExecutor._maybe_run_class_selector( - ctx, - config, - filtered, - stage_is_last=(stage_index + 1 >= len(stages))): - return - - # Special case: selecting multiple tags from get-tag and piping into delete-tag - # should batch into a single operation (one backend call). - next_cmd = None - try: - if stage_index + 1 < len(stages) and stages[stage_index + 1]: - next_cmd = str(stages[stage_index + 1][0] - ).replace("_", - "-").lower() - except Exception: - next_cmd = None - - def _is_tag_row(obj: Any) -> bool: - try: - if (hasattr(obj, - "__class__") - and obj.__class__.__name__ == "TagItem" - and hasattr(obj, - "tag_name")): - return True - except Exception: - pass - try: - if isinstance(obj, dict) and obj.get("tag_name"): - return True - except Exception: - pass - return False - - if (next_cmd in {"delete-tag", - "delete_tag"} and len(filtered) > 1 - and all(_is_tag_row(x) for x in filtered)): - from cmdlet._shared import get_field - - tags: List[str] = [] - first_hash = None - first_store = None - first_path = None - for item in filtered: - tag_name = get_field(item, "tag_name") - if tag_name: - tags.append(str(tag_name)) - if first_hash is None: - first_hash = get_field(item, "hash") - if first_store is None: - first_store = get_field(item, "store") - if first_path is None: - first_path = get_field(item, - "path") or get_field( - item, - "target" - ) - - if tags: - grouped = { - "table": "tag.selection", - "media_kind": "tag", - "hash": first_hash, - "store": first_store, - "path": first_path, - "tag": tags, - } - piped_result = grouped - continue - - from cmdlet._shared import coerce_to_pipe_object - - filtered_pipe_objs = [ - coerce_to_pipe_object(item) for item in filtered - ] - piped_result = ( - filtered_pipe_objs - if len(filtered_pipe_objs) > 1 else filtered_pipe_objs[0] - ) - - current_table = ctx.get_current_stage_table( - ) or ctx.get_last_result_table() - table_type = ( - current_table.table - if current_table and hasattr(current_table, - "table") else None - ) - - def _norm_stage_cmd(name: Any) -> str: - return str(name or "").replace("_", "-").strip().lower() - - next_cmd = None - if stage_index + 1 < len(stages) and stages[stage_index + 1]: - next_cmd = _norm_stage_cmd(stages[stage_index + 1][0]) - - auto_stage = None - if isinstance(table_type, str) and table_type: - try: - from ProviderCore.registry import selection_auto_stage_for_table - - # Preserve historical behavior: only forward selection-stage args - # to the auto stage when we are appending a new last stage. - at_end = bool(stage_index + 1 >= len(stages)) - auto_stage = selection_auto_stage_for_table( - table_type, - stage_args if at_end else None, - ) - except Exception: - auto_stage = None - - # Auto-insert downloader stages for provider tables. - # IMPORTANT: do not auto-download for filter selections; they may match many rows. - if filter_spec is None: - if stage_index + 1 >= len(stages): - if auto_stage: - try: - print(f"Auto-running selection via {auto_stage[0]}") - except Exception: - pass - stages.append(list(auto_stage)) - else: - if auto_stage: - auto_cmd = _norm_stage_cmd(auto_stage[0]) - if next_cmd not in (auto_cmd, ".pipe", ".mpv"): - debug(f"Auto-inserting {auto_cmd} after selection") - stages.insert(stage_index + 1, list(auto_stage)) continue - cmd_fn = REGISTRY.get(cmd_name) - if not cmd_fn: + # UX: selecting a single URL row from get-url tables should open it. + # Only do this when the selection stage is terminal to avoid surprising + # side-effects in pipelines like `@1 | download-file`. + current_table = ctx.get_current_stage_table( + ) or ctx.get_last_result_table() + if (not is_select_all) and (len(filtered) == 1): try: - mod = import_cmd_module(cmd_name) - data = getattr(mod, "CMDLET", None) if mod else None - if data and hasattr(data, "exec") and callable(getattr(data, "exec")): - run_fn = getattr(data, "exec") - REGISTRY[cmd_name] = run_fn - cmd_fn = run_fn + PipelineExecutor._maybe_open_url_selection( + current_table, + filtered, + stage_is_last=(stage_index + 1 >= len(stages)), + ) except Exception: - cmd_fn = None + pass - if not cmd_fn: - print(f"Unknown command: {cmd_name}\n") - pipeline_status = "failed" - pipeline_error = f"Unknown command: {cmd_name}" + if PipelineExecutor._maybe_run_class_selector( + ctx, + config, + filtered, + stage_is_last=(stage_index + 1 >= len(stages))): return + + # Special case: selecting multiple tags from get-tag and piping into delete-tag + # should batch into a single operation (one backend call). + next_cmd = None + try: + if stage_index + 1 < len(stages) and stages[stage_index + 1]: + next_cmd = str(stages[stage_index + 1][0] + ).replace("_", + "-").lower() + except Exception: + next_cmd = None + + def _is_tag_row(obj: Any) -> bool: + try: + if (hasattr(obj, + "__class__") + and obj.__class__.__name__ == "TagItem" + and hasattr(obj, + "tag_name")): + return True + except Exception: + pass + try: + if isinstance(obj, dict) and obj.get("tag_name"): + return True + except Exception: + pass + return False + + if (next_cmd in {"delete-tag", + "delete_tag"} and len(filtered) > 1 + and all(_is_tag_row(x) for x in filtered)): + from cmdlet._shared import get_field + + tags: List[str] = [] + first_hash = None + first_store = None + first_path = None + for item in filtered: + tag_name = get_field(item, "tag_name") + if tag_name: + tags.append(str(tag_name)) + if first_hash is None: + first_hash = get_field(item, "hash") + if first_store is None: + first_store = get_field(item, "store") + if first_path is None: + first_path = get_field(item, + "path") or get_field( + item, + "target" + ) + + if tags: + grouped = { + "table": "tag.selection", + "media_kind": "tag", + "hash": first_hash, + "store": first_store, + "path": first_path, + "tag": tags, + } + piped_result = grouped + continue + + from cmdlet._shared import coerce_to_pipe_object + + filtered_pipe_objs = [ + coerce_to_pipe_object(item) for item in filtered + ] + piped_result = ( + filtered_pipe_objs + if len(filtered_pipe_objs) > 1 else filtered_pipe_objs[0] + ) + + current_table = ctx.get_current_stage_table( + ) or ctx.get_last_result_table() + table_type = ( + current_table.table + if current_table and hasattr(current_table, + "table") else None + ) + + def _norm_stage_cmd(name: Any) -> str: + return str(name or "").replace("_", "-").strip().lower() + + next_cmd = None + if stage_index + 1 < len(stages) and stages[stage_index + 1]: + next_cmd = _norm_stage_cmd(stages[stage_index + 1][0]) + + auto_stage = None + if isinstance(table_type, str) and table_type: + try: + from ProviderCore.registry import selection_auto_stage_for_table + + # Preserve historical behavior: only forward selection-stage args + # to the auto stage when we are appending a new last stage. + at_end = bool(stage_index + 1 >= len(stages)) + auto_stage = selection_auto_stage_for_table( + table_type, + stage_args if at_end else None, + ) + except Exception: + auto_stage = None + + # Auto-insert downloader stages for provider tables. + # IMPORTANT: do not auto-download for filter selections; they may match many rows. + if filter_spec is None: + if stage_index + 1 >= len(stages): + if auto_stage: + try: + print(f"Auto-running selection via {auto_stage[0]}") + except Exception: + pass + stages.append(list(auto_stage)) + else: + if auto_stage: + auto_cmd = _norm_stage_cmd(auto_stage[0]) + if next_cmd not in (auto_cmd, ".pipe", ".mpv"): + debug(f"Auto-inserting {auto_cmd} after selection") + stages.insert(stage_index + 1, list(auto_stage)) + continue + + cmd_fn = REGISTRY.get(cmd_name) + if not cmd_fn: + try: + mod = import_cmd_module(cmd_name) + data = getattr(mod, "CMDLET", None) if mod else None + if data and hasattr(data, "exec") and callable(getattr(data, "exec")): + run_fn = getattr(data, "exec") + REGISTRY[cmd_name] = run_fn + cmd_fn = run_fn + except Exception: + cmd_fn = None + + if not cmd_fn: + print(f"Unknown command: {cmd_name}\n") + pipeline_status = "failed" + pipeline_error = f"Unknown command: {cmd_name}" + return + + try: + from SYS.models import PipelineStageContext + + pipe_idx = pipe_index_by_stage.get(stage_index) + + session = WorkerStages.begin_stage( + worker_manager, + cmd_name=cmd_name, + stage_tokens=stage_tokens, + config=config, + command_text=pipeline_text if pipeline_text else " ".join(stage_tokens), + ) + try: + stage_ctx = PipelineStageContext( + stage_index=stage_index, + total_stages=len(stages), + pipe_index=pipe_idx, + worker_id=session.worker_id if session else None, + ) + + # Set context for the current run + ctx.set_stage_context(stage_ctx) + ctx.set_current_cmdlet_name(cmd_name) + ctx.set_current_stage_text(" ".join(stage_tokens)) + ctx.clear_emits() + + # RUN THE CMDLET + cmd_fn(piped_result, stage_args, config) + + # Update piped_result for next stage from emitted items + stage_emits = list(stage_ctx.emits) + if stage_emits: + piped_result = stage_emits if len(stage_emits) > 1 else stage_emits[0] + else: + piped_result = None + + if progress_ui is not None and pipe_idx is not None: + progress_ui.complete_stage(pipe_idx) + finally: + if session: + try: + session.close() + except Exception: + pass + + except Exception as exc: + pipeline_status = "failed" + pipeline_error = f"{cmd_name}: {exc}" + debug(f"Error in stage {stage_index} ({cmd_name}): {exc}") + return except Exception as exc: pipeline_status = "failed" pipeline_error = f"{type(exc).__name__}: {exc}" diff --git a/Store/HydrusNetwork.py b/Store/HydrusNetwork.py index 756f5fe..c2e84f9 100644 --- a/Store/HydrusNetwork.py +++ b/Store/HydrusNetwork.py @@ -1934,7 +1934,7 @@ class HydrusNetwork(Store): try: if service_key: # Mutate tags for many hashes in a single request - client.mutate_tags_by_key(hash=hashes, service_key=service_key, add_tags=list(tag_tuple)) + client.mutate_tags_by_key(hashes=hashes, service_key=service_key, add_tags=list(tag_tuple)) any_success = True continue except Exception as exc: diff --git a/Store/registry.py b/Store/registry.py index fd08f30..540a6f0 100644 --- a/Store/registry.py +++ b/Store/registry.py @@ -28,9 +28,7 @@ _DISCOVERED_CLASSES_CACHE: Optional[Dict[str, Type[BaseStore]]] = None # Backends that failed to initialize earlier in the current process. # Keyed by (store_type, instance_key) where instance_key is the name used under config.store... -_FAILED_BACKEND_CACHE: Dict[tuple[str, - str], - str] = {} +_FAILED_BACKEND_CACHE: Dict[tuple[str, str], str] = {} def _normalize_store_type(value: str) -> str: @@ -63,13 +61,10 @@ def _discover_store_classes() -> Dict[str, Type[BaseStore]]: import Store as store_pkg - discovered: Dict[str, - Type[BaseStore]] = {} + discovered: Dict[str, Type[BaseStore]] = {} for module_info in pkgutil.iter_modules(store_pkg.__path__): module_name = module_info.name - if module_name in {"__init__", - "_base", - "registry"}: + if module_name in {"__init__", "_base", "registry"}: continue try: @@ -122,10 +117,7 @@ def _required_keys_for(store_cls: Type[BaseStore]) -> list[str]: ) -def _build_kwargs(store_cls: Type[BaseStore], - instance_name: str, - instance_config: Any) -> Dict[str, - Any]: +def _build_kwargs(store_cls: Type[BaseStore], instance_name: str, instance_config: Any) -> Dict[str, Any]: if isinstance(instance_config, dict): cfg_dict = dict(instance_config) else: @@ -134,13 +126,10 @@ def _build_kwargs(store_cls: Type[BaseStore], required = _required_keys_for(store_cls) # If NAME is required but not present, allow the instance key to provide it. - if (any(_normalize_config_key(k) == "NAME" for k in required) - and _get_case_insensitive(cfg_dict, - "NAME") is None): + if (any(_normalize_config_key(k) == "NAME" for k in required) and _get_case_insensitive(cfg_dict, "NAME") is None): cfg_dict["NAME"] = str(instance_name) - kwargs: Dict[str, - Any] = {} + kwargs: Dict[str, Any] = {} missing: list[str] = [] for key in required: value = _get_case_insensitive(cfg_dict, key) @@ -257,8 +246,7 @@ class Store: # Convenience normalization for filesystem-like paths. for key in list(kwargs.keys()): - if _normalize_config_key(key) in {"PATH", - "LOCATION"}: + if _normalize_config_key(key) in {"PATH", "LOCATION"}: kwargs[key] = str(expand_path(kwargs[key])) backend = store_cls(**kwargs) @@ -283,8 +271,50 @@ class Store: f"[Store] Failed to register {store_cls.__name__} instance '{instance_name}': {exc}" ) - def _resolve_backend_name(self, - backend_name: str) -> tuple[Optional[str], Optional[str]]: + def _resolve_backend_name(self, backend_name: str) -> tuple[Optional[str], Optional[str]]: + requested = str(backend_name or "") + if requested in self._backends: + return requested, None + + requested_norm = _normalize_store_type(requested) + + ci_matches = [ + name for name in self._backends + if _normalize_store_type(name) == requested_norm + ] + if len(ci_matches) == 1: + return ci_matches[0], None + if len(ci_matches) > 1: + return None, f"Ambiguous store alias '{backend_name}' matches {ci_matches}" + + type_matches = [ + name for name, store_type in self._backend_types.items() + if store_type == requested_norm + ] + if len(type_matches) == 1: + return type_matches[0], None + if len(type_matches) > 1: + return None, ( + f"Ambiguous store alias '{backend_name}' matches type '{requested_norm}': {type_matches}" + ) + + prefix_matches = [ + name for name, store_type in self._backend_types.items() + if store_type.startswith(requested_norm) + ] + if len(prefix_matches) == 1: + return prefix_matches[0], None + if len(prefix_matches) > 1: + return None, ( + f"Ambiguous store alias '{backend_name}' matches type prefix '{requested_norm}': {prefix_matches}" + ) + + return None, None + + # get_backend_instance implementation moved to the bottom of this file to avoid + # instantiating all backends during startup (see function `get_backend_instance`). + + def _resolve_backend_name(self, backend_name: str) -> tuple[Optional[str], Optional[str]]: requested = str(backend_name or "") if requested in self._backends: return requested, None @@ -461,3 +491,85 @@ def list_configured_backend_names(config: Optional[Dict[str, Any]]) -> list[str] return sorted(set(names)) except Exception: return [] + + +def get_backend_instance(config: Optional[Dict[str, Any]], backend_name: str, *, suppress_debug: bool = False) -> Optional[BaseStore]: + """Instantiate and return a single store backend by configured name. + + This avoids creating all configured backends (and opening their DBs) + when only a single backend is needed (common in `get-file`/`get-metadata`). + The function first tries a lightweight match against raw config NAME/value to + avoid calling `_build_kwargs` (which can raise if keys are missing). + Returns None when no matching backend is found or instantiation fails. + """ + if not backend_name: + return None + store_cfg = (config or {}).get("store") or {} + if not isinstance(store_cfg, dict): + return None + classes_by_type = _discover_store_classes() + desired = str(backend_name or "").strip().lower() + + for raw_store_type, instances in store_cfg.items(): + if not isinstance(instances, dict): + continue + store_type = _normalize_store_type(str(raw_store_type)) + store_cls = classes_by_type.get(store_type) + if store_cls is None: + continue + + # Fast path: match using raw 'NAME' or 'name' in config without building full kwargs + for instance_name, instance_cfg in instances.items(): + candidate_alias = None + if isinstance(instance_cfg, dict): + candidate_alias = ( + instance_cfg.get("NAME") or instance_cfg.get("name") + ) + candidate_alias = str(candidate_alias or instance_name).strip() + if candidate_alias.lower() == desired: + try: + kwargs = _build_kwargs(store_cls, str(instance_name), instance_cfg) + except Exception as exc: + if not suppress_debug: + debug(f"[Store] Can't build kwargs for '{instance_name}' ({store_type}): {exc}") + return None + try: + for key in list(kwargs.keys()): + if _normalize_config_key(key) in {"PATH", "LOCATION"}: + kwargs[key] = str(expand_path(kwargs[key])) + except Exception: + pass + try: + backend = store_cls(**kwargs) + return backend + except Exception as exc: + if not suppress_debug: + debug(f"[Store] Failed to instantiate backend '{candidate_alias}': {exc}") + return None + + # Fallback: build kwargs for each instance and compare resolved NAME + for instance_name, instance_cfg in instances.items(): + try: + kwargs = _build_kwargs(store_cls, str(instance_name), instance_cfg) + except Exception: + continue + alias = str(kwargs.get("NAME") or instance_name).strip() + if alias.lower() != desired: + continue + try: + for key in list(kwargs.keys()): + if _normalize_config_key(key) in {"PATH", "LOCATION"}: + kwargs[key] = str(expand_path(kwargs[key])) + except Exception: + pass + try: + backend = store_cls(**kwargs) + return backend + except Exception as exc: + if not suppress_debug: + debug(f"[Store] Failed to instantiate backend '{alias}': {exc}") + return None + + if not suppress_debug: + debug(f"[Store] Backend '{backend_name}' not found in config") + return None diff --git a/cmdlet/get_file.py b/cmdlet/get_file.py index b9ba2eb..22533a8 100644 --- a/cmdlet/get_file.py +++ b/cmdlet/get_file.py @@ -86,9 +86,22 @@ class Get_File(sh.Cmdlet): debug(f"[get-file] Getting storage backend: {store_name}") - # Get storage backend - store = Store(config) - backend = store[store_name] + # Prefer instantiating only the named backend to avoid initializing all configured backends + try: + from Store.registry import get_backend_instance + backend = get_backend_instance(config, store_name, suppress_debug=True) + except Exception: + backend = None + + if backend is None: + # Fallback to full registry when targeted instantiation fails + try: + store = Store(config) + backend = store[store_name] + except Exception: + log(f"Error: Storage backend '{store_name}' not found", file=sys.stderr) + return 1 + debug(f"[get-file] Backend retrieved: {type(backend).__name__}") # Get file metadata to determine name and extension diff --git a/cmdlet/get_metadata.py b/cmdlet/get_metadata.py index 816e2d5..2394a0d 100644 --- a/cmdlet/get_metadata.py +++ b/cmdlet/get_metadata.py @@ -192,10 +192,21 @@ class Get_Metadata(Cmdlet): # Use storage backend to get metadata try: - from Store import Store + # Instantiate only the required backend when possible to avoid initializing all configured backends + try: + from Store.registry import get_backend_instance + backend = get_backend_instance(config, storage_source, suppress_debug=True) + except Exception: + backend = None - storage = Store(config) - backend = storage[storage_source] + if backend is None: + try: + from Store import Store + storage = Store(config) + backend = storage[storage_source] + except Exception: + log(f"Storage backend '{storage_source}' not found", file=sys.stderr) + return 1 # Get metadata from backend metadata = backend.get_metadata(file_hash) @@ -290,6 +301,13 @@ class Get_Metadata(Cmdlet): list(args)) self._add_table_body_row(table, row) ctx.set_last_result_table_overlay(table, [row], row) + try: + from SYS.rich_display import render_item_details_panel + + render_item_details_panel(row) + table._rendered_by_cmdlet = True + except Exception: + pass ctx.emit(row) return 0 diff --git a/cmdlet/search_file.py b/cmdlet/search_file.py index 7077027..d8d2495 100644 --- a/cmdlet/search_file.py +++ b/cmdlet/search_file.py @@ -211,18 +211,27 @@ class search_file(Cmdlet): worker_id = str(uuid.uuid4()) library_root = get_local_storage_path(config or {}) if get_local_storage_path else None - + if not library_root: try: - from Store import Store - storage_registry = Store(config=config or {}) - # Try the first folder backend - for name in storage_registry.list_backends(): - backend = storage_registry[name] - if type(backend).__name__ == "Folder": - library_root = expand_path(getattr(backend, "_location", None)) - if library_root: - break + from Store.registry import get_backend_instance + # Try the first configured folder backend without instantiating all backends + store_cfg = (config or {}).get("store") or {} + folder_cfg = None + for raw_store_type, instances in store_cfg.items(): + if _normalize_store_type(str(raw_store_type)) == "folder": + folder_cfg = instances + break + if isinstance(folder_cfg, dict): + for instance_name, instance_config in folder_cfg.items(): + try: + backend = get_backend_instance(config, instance_name, suppress_debug=True) + if backend and type(backend).__name__ == "Folder": + library_root = expand_path(getattr(backend, "_location", None)) + if library_root: + break + except Exception: + pass except Exception: pass @@ -655,10 +664,9 @@ class search_file(Cmdlet): except Exception: pass - from Store import Store + from Store.registry import list_configured_backend_names, get_backend_instance from Store._base import Store as BaseStore - storage = storage_registry backend_to_search = storage_backend or None if hash_query: # Explicit hash list search: build rows from backend metadata. @@ -666,7 +674,7 @@ class search_file(Cmdlet): if backend_to_search: backends_to_try = [backend_to_search] else: - backends_to_try = list(storage.list_backends()) + backends_to_try = list_configured_backend_names(config or {}) found_any = False for h in hash_query: @@ -674,9 +682,17 @@ class search_file(Cmdlet): resolved_backend = None for backend_name in backends_to_try: + backend = None try: - backend = storage[backend_name] + backend = get_backend_instance(config, backend_name, suppress_debug=True) + if backend is None: + # Last-resort: instantiate full registry for this backend only + from Store import Store as _Store + _store = _Store(config=config) + backend = _store[backend_name] except Exception: + backend = None + if backend is None: continue try: # If get_metadata works, consider it a hit; get_file can be optional (e.g. remote URL). @@ -828,7 +844,17 @@ class search_file(Cmdlet): if backend_to_search: searched_backends.append(backend_to_search) - target_backend = storage[backend_to_search] + try: + target_backend = get_backend_instance(config, backend_to_search, suppress_debug=True) + if target_backend is None: + from Store import Store as _Store + _store = _Store(config=config) + target_backend = _store[backend_to_search] + except Exception as exc: + log(f"Backend '{backend_to_search}' not found: {exc}", file=sys.stderr) + db.update_worker_status(worker_id, "error") + return 1 + if type(target_backend).search is BaseStore.search: log( f"Backend '{backend_to_search}' does not support searching", @@ -843,11 +869,19 @@ class search_file(Cmdlet): ) else: all_results = [] - for backend_name in storage.list_searchable_backends(): + for backend_name in list_configured_backend_names(config or {}): try: - backend = storage[backend_name] + backend = get_backend_instance(config, backend_name, suppress_debug=True) + if backend is None: + from Store import Store as _Store + _store = _Store(config=config) + backend = _store[backend_name] + searched_backends.append(backend_name) + if type(backend).search is BaseStore.search: + continue + debug(f"[search-file] Searching '{backend_name}'") backend_results = backend.search( query, @@ -909,6 +943,62 @@ class search_file(Cmdlet): if store_val and not normalized.get("store"): normalized["store"] = store_val + # Populate default selection args for interactive @N selection/hash/url handling + try: + sel_args: Optional[List[str]] = None + sel_action: Optional[List[str]] = None + + # Prefer explicit path when available + p_val = normalized.get("path") or normalized.get("target") or normalized.get("url") + if p_val: + p_str = str(p_val or "").strip() + if p_str: + if p_str.startswith(("http://", "https://", "magnet:", "torrent:")): + sel_args = ["-url", p_str] + sel_action = ["download-file", "-url", p_str] + else: + try: + from SYS.utils import expand_path + + full_path = expand_path(p_str) + # Prefer showing metadata details when we have a hash+store context + h = normalized.get("hash") or normalized.get("file_hash") or normalized.get("hash_hex") + s_val = normalized.get("store") + if h and s_val: + try: + h_norm = normalize_hash(h) + except Exception: + h_norm = str(h) + sel_args = ["-query", f"hash:{h_norm}", "-store", str(s_val)] + sel_action = ["get-metadata", "-query", f"hash:{h_norm}", "-store", str(s_val)] + else: + sel_args = ["-path", str(full_path)] + # Default action for local paths: get-file to fetch or operate on the path + sel_action = ["get-file", "-path", str(full_path)] + except Exception: + sel_args = ["-path", p_str] + sel_action = ["get-file", "-path", p_str] + + # Fallback: use hash+store when available + if sel_args is None: + h = normalized.get("hash") or normalized.get("file_hash") or normalized.get("hash_hex") + s_val = normalized.get("store") + if h and s_val: + try: + h_norm = normalize_hash(h) + except Exception: + h_norm = str(h) + sel_args = ["-query", f"hash:{h_norm}", "-store", str(s_val)] + # Show metadata details by default for store/hash selections + sel_action = ["get-metadata", "-query", f"hash:{h_norm}", "-store", str(s_val)] + + if sel_args: + normalized["_selection_args"] = [str(x) for x in sel_args] + if sel_action: + normalized["_selection_action"] = [str(x) for x in sel_action] + except Exception: + pass + table.add_result(normalized) results_list.append(normalized) diff --git a/scripts/check_indentation.py b/scripts/check_indentation.py new file mode 100644 index 0000000..91fe493 --- /dev/null +++ b/scripts/check_indentation.py @@ -0,0 +1,23 @@ +from pathlib import Path +p = Path('Store/registry.py') +lines = p.read_text(encoding='utf-8').splitlines() +stack = [0] +for i, line in enumerate(lines, start=1): + if not line.strip(): + continue + leading = len(line) - len(line.lstrip(' ')) + if leading > stack[-1]: + # increased indent -> push + stack.append(leading) + else: + # dedent: must match an existing level + if leading != stack[-1]: + if leading in stack: + # pop until match + while stack and stack[-1] != leading: + stack.pop() + else: + print(f"Line {i}: inconsistent indent {leading}, stack levels {stack}") + print(repr(line)) + break +print('Done') \ No newline at end of file diff --git a/scripts/check_store.py b/scripts/check_store.py new file mode 100644 index 0000000..113a13e --- /dev/null +++ b/scripts/check_store.py @@ -0,0 +1,12 @@ +import traceback + +try: + from Store import Store + s = Store(config={}, suppress_debug=True) + print('INSTANCE TYPE:', type(s)) + print('HAS is_available:', hasattr(s, 'is_available')) + if hasattr(s, 'is_available'): + print('is_available callable:', callable(getattr(s, 'is_available'))) + print('DIR:', sorted([n for n in dir(s) if not n.startswith('__')])) +except Exception: + traceback.print_exc() \ No newline at end of file diff --git a/scripts/cli_entry.py b/scripts/cli_entry.py index ebb7ab2..956c63a 100644 --- a/scripts/cli_entry.py +++ b/scripts/cli_entry.py @@ -184,7 +184,7 @@ def _run_cli(clean_args: List[str]) -> int: if MedeiaCLI is None: try: repo_root = _ensure_repo_root_on_sys_path() - from CLI import MedeiaCLI as _M # type: ignore + from CLI import CLI as _M # type: ignore MedeiaCLI = _M except Exception as exc: # Provide diagnostic information diff --git a/scripts/find_big_indent.py b/scripts/find_big_indent.py new file mode 100644 index 0000000..2c57e66 --- /dev/null +++ b/scripts/find_big_indent.py @@ -0,0 +1,6 @@ +from pathlib import Path +p = Path('Store/registry.py') +for i, line in enumerate(p.read_text(encoding='utf-8').splitlines(), start=1): + leading = len(line) - len(line.lstrip(' ')) + if leading > 20: + print(i, leading, repr(line)) \ No newline at end of file diff --git a/scripts/indent_check.py b/scripts/indent_check.py new file mode 100644 index 0000000..03b1c39 --- /dev/null +++ b/scripts/indent_check.py @@ -0,0 +1,11 @@ +import pathlib +p = pathlib.Path('Store/registry.py') +with p.open('r', encoding='utf-8') as f: + lines = f.readlines() +for i in range(312, 328): + if i-1 < len(lines): + line = lines[i-1] + leading = line[:len(line)-len(line.lstrip('\t '))] + print(f"{i}: {repr(line.rstrip())} | leading={repr(leading)} len={len(leading)} chars={[ord(c) for c in leading]}") + else: + print(f"{i}: ") \ No newline at end of file diff --git a/scripts/indent_stats.py b/scripts/indent_stats.py new file mode 100644 index 0000000..e6c4d58 --- /dev/null +++ b/scripts/indent_stats.py @@ -0,0 +1,10 @@ +from pathlib import Path +p = Path('Store/registry.py') +counts = {} +for i, line in enumerate(p.read_text(encoding='utf-8').splitlines(), start=1): + if not line.strip(): + continue + leading = len(line) - len(line.lstrip(' ')) + counts[leading] = counts.get(leading, 0) + 1 +for k in sorted(counts.keys()): + print(k, counts[k])