f
This commit is contained in:
26
CLI.py
26
CLI.py
@@ -7,6 +7,28 @@ This module intentionally uses a class-based architecture:
|
|||||||
- all REPL/pipeline/cmdlet execution state lives on objects
|
- all REPL/pipeline/cmdlet execution state lives on objects
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
# When running the CLI directly (not via the 'mm' launcher), honor the
|
||||||
|
# repository config `debug` flag by enabling `MM_DEBUG` so import-time
|
||||||
|
# diagnostics and bootstrap debug output are visible without setting the
|
||||||
|
# environment variable manually.
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
if not os.environ.get("MM_DEBUG"):
|
||||||
|
try:
|
||||||
|
conf_path = Path(__file__).resolve().parent / "config.conf"
|
||||||
|
if conf_path.exists():
|
||||||
|
for ln in conf_path.read_text(encoding="utf-8").splitlines():
|
||||||
|
ln_strip = ln.strip()
|
||||||
|
if ln_strip.startswith("debug"):
|
||||||
|
parts = ln_strip.split("=", 1)
|
||||||
|
if len(parts) >= 2:
|
||||||
|
val = parts[1].strip().strip('"').strip("'").strip().lower()
|
||||||
|
if val in ("1", "true", "yes", "on"):
|
||||||
|
os.environ["MM_DEBUG"] = "1"
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
import shlex
|
import shlex
|
||||||
@@ -1325,7 +1347,7 @@ class CmdletExecutor:
|
|||||||
console = Console()
|
console = Console()
|
||||||
|
|
||||||
|
|
||||||
class MedeiaCLI:
|
class CLI:
|
||||||
"""Main CLI application object."""
|
"""Main CLI application object."""
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parent
|
ROOT = Path(__file__).resolve().parent
|
||||||
@@ -2469,4 +2491,4 @@ Come to love it when others take what you share, as there is no greater joy
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
MedeiaCLI().run()
|
CLI().run()
|
||||||
|
|||||||
124
SYS/pipeline.py
124
SYS/pipeline.py
@@ -622,7 +622,9 @@ def set_last_result_table(
|
|||||||
src_idx = getattr(row, "source_index", None)
|
src_idx = getattr(row, "source_index", None)
|
||||||
if isinstance(src_idx, int) and 0 <= src_idx < len(state.last_result_items):
|
if isinstance(src_idx, int) and 0 <= src_idx < len(state.last_result_items):
|
||||||
sorted_items.append(state.last_result_items[src_idx])
|
sorted_items.append(state.last_result_items[src_idx])
|
||||||
if len(sorted_items) == len(result_table.rows):
|
# Only reassign when the table actually contains rows and the reordering
|
||||||
|
# produced a complete mapping. Avoid clearing items when the table has no rows.
|
||||||
|
if result_table.rows and len(sorted_items) == len(result_table.rows):
|
||||||
state.last_result_items = sorted_items
|
state.last_result_items = sorted_items
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
@@ -2136,6 +2138,54 @@ class PipelineExecutor:
|
|||||||
stages[-1] = [cmd] + [str(x) for x in row_args] + tail
|
stages[-1] = [cmd] + [str(x) for x in row_args] + tail
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
# If no auto stage inserted and there are selection-action tokens available
|
||||||
|
# for the single selected row, apply it as the pipeline stage so a bare
|
||||||
|
# `@N` runs the intended action (e.g., get-file for hash-backed rows).
|
||||||
|
if not stages and selection_indices and len(selection_indices) == 1:
|
||||||
|
try:
|
||||||
|
idx = selection_indices[0]
|
||||||
|
debug(f"@N initial selection idx={idx} last_items={len(ctx.get_last_result_items() or [])}")
|
||||||
|
row_action = None
|
||||||
|
try:
|
||||||
|
row_action = ctx.get_current_stage_table_row_selection_action(idx)
|
||||||
|
except Exception:
|
||||||
|
row_action = None
|
||||||
|
|
||||||
|
if not row_action:
|
||||||
|
try:
|
||||||
|
items = ctx.get_last_result_items() or []
|
||||||
|
if 0 <= idx < len(items):
|
||||||
|
maybe = items[idx]
|
||||||
|
# Provide explicit debug output about the payload selected
|
||||||
|
try:
|
||||||
|
if isinstance(maybe, dict):
|
||||||
|
debug(f"@N payload: hash={maybe.get('hash')} store={maybe.get('store')} _selection_args={maybe.get('_selection_args')} _selection_action={maybe.get('_selection_action')}")
|
||||||
|
else:
|
||||||
|
debug(f"@N payload object type: {type(maybe).__name__}")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
if isinstance(maybe, dict):
|
||||||
|
candidate = maybe.get("_selection_action")
|
||||||
|
if isinstance(candidate, (list, tuple)):
|
||||||
|
row_action = [str(x) for x in candidate if x is not None]
|
||||||
|
debug(f"@N restored row_action from payload: {row_action}")
|
||||||
|
except Exception:
|
||||||
|
row_action = None
|
||||||
|
|
||||||
|
if row_action:
|
||||||
|
debug(f"@N applying row action -> {row_action}")
|
||||||
|
stages.append(row_action)
|
||||||
|
if pipeline_session and worker_manager:
|
||||||
|
try:
|
||||||
|
worker_manager.log_step(
|
||||||
|
pipeline_session.worker_id,
|
||||||
|
f"@N applied row action -> {' '.join(row_action)}",
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
else:
|
else:
|
||||||
first_cmd = stages[0][0] if stages and stages[0] else None
|
first_cmd = stages[0][0] if stages and stages[0] else None
|
||||||
if isinstance(table_type, str) and table_type.startswith("metadata.") and first_cmd not in (
|
if isinstance(table_type, str) and table_type.startswith("metadata.") and first_cmd not in (
|
||||||
@@ -2372,7 +2422,6 @@ class PipelineExecutor:
|
|||||||
pipe_index_by_stage: Dict[int,
|
pipe_index_by_stage: Dict[int,
|
||||||
int] = {}
|
int] = {}
|
||||||
|
|
||||||
try:
|
|
||||||
ok, initial_piped = self._maybe_apply_initial_selection(
|
ok, initial_piped = self._maybe_apply_initial_selection(
|
||||||
ctx,
|
ctx,
|
||||||
config,
|
config,
|
||||||
@@ -2386,11 +2435,6 @@ class PipelineExecutor:
|
|||||||
return
|
return
|
||||||
if initial_piped is not None:
|
if initial_piped is not None:
|
||||||
piped_result = initial_piped
|
piped_result = initial_piped
|
||||||
except Exception as exc:
|
|
||||||
pipeline_status = "failed"
|
|
||||||
pipeline_error = f"{type(exc).__name__}: {exc}"
|
|
||||||
print(f"[error] {type(exc).__name__}: {exc}\n")
|
|
||||||
return
|
|
||||||
|
|
||||||
# REPL guard: prevent add-relationship before add-file for download-file pipelines.
|
# REPL guard: prevent add-relationship before add-file for download-file pipelines.
|
||||||
if not self._validate_download_file_relationship_order(stages):
|
if not self._validate_download_file_relationship_order(stages):
|
||||||
@@ -2542,6 +2586,21 @@ class PipelineExecutor:
|
|||||||
resolved_items[i] for i in selected_indices
|
resolved_items[i] for i in selected_indices
|
||||||
if 0 <= i < len(resolved_items)
|
if 0 <= i < len(resolved_items)
|
||||||
]
|
]
|
||||||
|
# Debug: show selection resolution and sample payload info
|
||||||
|
try:
|
||||||
|
debug(f"Selection {selection_token} -> resolved_indices={selected_indices} filtered_count={len(filtered)}")
|
||||||
|
if filtered:
|
||||||
|
sample = filtered[0]
|
||||||
|
if isinstance(sample, dict):
|
||||||
|
debug(f"Selection sample: hash={sample.get('hash')} store={sample.get('store')} _selection_args={sample.get('_selection_args')} _selection_action={sample.get('_selection_action')}")
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
debug(f"Selection sample object: provider={getattr(sample, 'provider', None)} store={getattr(sample, 'store', None)}")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
if not filtered:
|
if not filtered:
|
||||||
print("No items matched selection\n")
|
print("No items matched selection\n")
|
||||||
pipeline_status = "failed"
|
pipeline_status = "failed"
|
||||||
@@ -2753,6 +2812,57 @@ class PipelineExecutor:
|
|||||||
pipeline_status = "failed"
|
pipeline_status = "failed"
|
||||||
pipeline_error = f"Unknown command: {cmd_name}"
|
pipeline_error = f"Unknown command: {cmd_name}"
|
||||||
return
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
from SYS.models import PipelineStageContext
|
||||||
|
|
||||||
|
pipe_idx = pipe_index_by_stage.get(stage_index)
|
||||||
|
|
||||||
|
session = WorkerStages.begin_stage(
|
||||||
|
worker_manager,
|
||||||
|
cmd_name=cmd_name,
|
||||||
|
stage_tokens=stage_tokens,
|
||||||
|
config=config,
|
||||||
|
command_text=pipeline_text if pipeline_text else " ".join(stage_tokens),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
stage_ctx = PipelineStageContext(
|
||||||
|
stage_index=stage_index,
|
||||||
|
total_stages=len(stages),
|
||||||
|
pipe_index=pipe_idx,
|
||||||
|
worker_id=session.worker_id if session else None,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Set context for the current run
|
||||||
|
ctx.set_stage_context(stage_ctx)
|
||||||
|
ctx.set_current_cmdlet_name(cmd_name)
|
||||||
|
ctx.set_current_stage_text(" ".join(stage_tokens))
|
||||||
|
ctx.clear_emits()
|
||||||
|
|
||||||
|
# RUN THE CMDLET
|
||||||
|
cmd_fn(piped_result, stage_args, config)
|
||||||
|
|
||||||
|
# Update piped_result for next stage from emitted items
|
||||||
|
stage_emits = list(stage_ctx.emits)
|
||||||
|
if stage_emits:
|
||||||
|
piped_result = stage_emits if len(stage_emits) > 1 else stage_emits[0]
|
||||||
|
else:
|
||||||
|
piped_result = None
|
||||||
|
|
||||||
|
if progress_ui is not None and pipe_idx is not None:
|
||||||
|
progress_ui.complete_stage(pipe_idx)
|
||||||
|
finally:
|
||||||
|
if session:
|
||||||
|
try:
|
||||||
|
session.close()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
except Exception as exc:
|
||||||
|
pipeline_status = "failed"
|
||||||
|
pipeline_error = f"{cmd_name}: {exc}"
|
||||||
|
debug(f"Error in stage {stage_index} ({cmd_name}): {exc}")
|
||||||
|
return
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
pipeline_status = "failed"
|
pipeline_status = "failed"
|
||||||
pipeline_error = f"{type(exc).__name__}: {exc}"
|
pipeline_error = f"{type(exc).__name__}: {exc}"
|
||||||
|
|||||||
@@ -1934,7 +1934,7 @@ class HydrusNetwork(Store):
|
|||||||
try:
|
try:
|
||||||
if service_key:
|
if service_key:
|
||||||
# Mutate tags for many hashes in a single request
|
# Mutate tags for many hashes in a single request
|
||||||
client.mutate_tags_by_key(hash=hashes, service_key=service_key, add_tags=list(tag_tuple))
|
client.mutate_tags_by_key(hashes=hashes, service_key=service_key, add_tags=list(tag_tuple))
|
||||||
any_success = True
|
any_success = True
|
||||||
continue
|
continue
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
|
|||||||
@@ -28,9 +28,7 @@ _DISCOVERED_CLASSES_CACHE: Optional[Dict[str, Type[BaseStore]]] = None
|
|||||||
|
|
||||||
# Backends that failed to initialize earlier in the current process.
|
# Backends that failed to initialize earlier in the current process.
|
||||||
# Keyed by (store_type, instance_key) where instance_key is the name used under config.store.<type>.<instance_key>.
|
# Keyed by (store_type, instance_key) where instance_key is the name used under config.store.<type>.<instance_key>.
|
||||||
_FAILED_BACKEND_CACHE: Dict[tuple[str,
|
_FAILED_BACKEND_CACHE: Dict[tuple[str, str], str] = {}
|
||||||
str],
|
|
||||||
str] = {}
|
|
||||||
|
|
||||||
|
|
||||||
def _normalize_store_type(value: str) -> str:
|
def _normalize_store_type(value: str) -> str:
|
||||||
@@ -63,13 +61,10 @@ def _discover_store_classes() -> Dict[str, Type[BaseStore]]:
|
|||||||
|
|
||||||
import Store as store_pkg
|
import Store as store_pkg
|
||||||
|
|
||||||
discovered: Dict[str,
|
discovered: Dict[str, Type[BaseStore]] = {}
|
||||||
Type[BaseStore]] = {}
|
|
||||||
for module_info in pkgutil.iter_modules(store_pkg.__path__):
|
for module_info in pkgutil.iter_modules(store_pkg.__path__):
|
||||||
module_name = module_info.name
|
module_name = module_info.name
|
||||||
if module_name in {"__init__",
|
if module_name in {"__init__", "_base", "registry"}:
|
||||||
"_base",
|
|
||||||
"registry"}:
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -122,10 +117,7 @@ def _required_keys_for(store_cls: Type[BaseStore]) -> list[str]:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _build_kwargs(store_cls: Type[BaseStore],
|
def _build_kwargs(store_cls: Type[BaseStore], instance_name: str, instance_config: Any) -> Dict[str, Any]:
|
||||||
instance_name: str,
|
|
||||||
instance_config: Any) -> Dict[str,
|
|
||||||
Any]:
|
|
||||||
if isinstance(instance_config, dict):
|
if isinstance(instance_config, dict):
|
||||||
cfg_dict = dict(instance_config)
|
cfg_dict = dict(instance_config)
|
||||||
else:
|
else:
|
||||||
@@ -134,13 +126,10 @@ def _build_kwargs(store_cls: Type[BaseStore],
|
|||||||
required = _required_keys_for(store_cls)
|
required = _required_keys_for(store_cls)
|
||||||
|
|
||||||
# If NAME is required but not present, allow the instance key to provide it.
|
# If NAME is required but not present, allow the instance key to provide it.
|
||||||
if (any(_normalize_config_key(k) == "NAME" for k in required)
|
if (any(_normalize_config_key(k) == "NAME" for k in required) and _get_case_insensitive(cfg_dict, "NAME") is None):
|
||||||
and _get_case_insensitive(cfg_dict,
|
|
||||||
"NAME") is None):
|
|
||||||
cfg_dict["NAME"] = str(instance_name)
|
cfg_dict["NAME"] = str(instance_name)
|
||||||
|
|
||||||
kwargs: Dict[str,
|
kwargs: Dict[str, Any] = {}
|
||||||
Any] = {}
|
|
||||||
missing: list[str] = []
|
missing: list[str] = []
|
||||||
for key in required:
|
for key in required:
|
||||||
value = _get_case_insensitive(cfg_dict, key)
|
value = _get_case_insensitive(cfg_dict, key)
|
||||||
@@ -257,8 +246,7 @@ class Store:
|
|||||||
|
|
||||||
# Convenience normalization for filesystem-like paths.
|
# Convenience normalization for filesystem-like paths.
|
||||||
for key in list(kwargs.keys()):
|
for key in list(kwargs.keys()):
|
||||||
if _normalize_config_key(key) in {"PATH",
|
if _normalize_config_key(key) in {"PATH", "LOCATION"}:
|
||||||
"LOCATION"}:
|
|
||||||
kwargs[key] = str(expand_path(kwargs[key]))
|
kwargs[key] = str(expand_path(kwargs[key]))
|
||||||
|
|
||||||
backend = store_cls(**kwargs)
|
backend = store_cls(**kwargs)
|
||||||
@@ -283,8 +271,50 @@ class Store:
|
|||||||
f"[Store] Failed to register {store_cls.__name__} instance '{instance_name}': {exc}"
|
f"[Store] Failed to register {store_cls.__name__} instance '{instance_name}': {exc}"
|
||||||
)
|
)
|
||||||
|
|
||||||
def _resolve_backend_name(self,
|
def _resolve_backend_name(self, backend_name: str) -> tuple[Optional[str], Optional[str]]:
|
||||||
backend_name: str) -> tuple[Optional[str], Optional[str]]:
|
requested = str(backend_name or "")
|
||||||
|
if requested in self._backends:
|
||||||
|
return requested, None
|
||||||
|
|
||||||
|
requested_norm = _normalize_store_type(requested)
|
||||||
|
|
||||||
|
ci_matches = [
|
||||||
|
name for name in self._backends
|
||||||
|
if _normalize_store_type(name) == requested_norm
|
||||||
|
]
|
||||||
|
if len(ci_matches) == 1:
|
||||||
|
return ci_matches[0], None
|
||||||
|
if len(ci_matches) > 1:
|
||||||
|
return None, f"Ambiguous store alias '{backend_name}' matches {ci_matches}"
|
||||||
|
|
||||||
|
type_matches = [
|
||||||
|
name for name, store_type in self._backend_types.items()
|
||||||
|
if store_type == requested_norm
|
||||||
|
]
|
||||||
|
if len(type_matches) == 1:
|
||||||
|
return type_matches[0], None
|
||||||
|
if len(type_matches) > 1:
|
||||||
|
return None, (
|
||||||
|
f"Ambiguous store alias '{backend_name}' matches type '{requested_norm}': {type_matches}"
|
||||||
|
)
|
||||||
|
|
||||||
|
prefix_matches = [
|
||||||
|
name for name, store_type in self._backend_types.items()
|
||||||
|
if store_type.startswith(requested_norm)
|
||||||
|
]
|
||||||
|
if len(prefix_matches) == 1:
|
||||||
|
return prefix_matches[0], None
|
||||||
|
if len(prefix_matches) > 1:
|
||||||
|
return None, (
|
||||||
|
f"Ambiguous store alias '{backend_name}' matches type prefix '{requested_norm}': {prefix_matches}"
|
||||||
|
)
|
||||||
|
|
||||||
|
return None, None
|
||||||
|
|
||||||
|
# get_backend_instance implementation moved to the bottom of this file to avoid
|
||||||
|
# instantiating all backends during startup (see function `get_backend_instance`).
|
||||||
|
|
||||||
|
def _resolve_backend_name(self, backend_name: str) -> tuple[Optional[str], Optional[str]]:
|
||||||
requested = str(backend_name or "")
|
requested = str(backend_name or "")
|
||||||
if requested in self._backends:
|
if requested in self._backends:
|
||||||
return requested, None
|
return requested, None
|
||||||
@@ -461,3 +491,85 @@ def list_configured_backend_names(config: Optional[Dict[str, Any]]) -> list[str]
|
|||||||
return sorted(set(names))
|
return sorted(set(names))
|
||||||
except Exception:
|
except Exception:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def get_backend_instance(config: Optional[Dict[str, Any]], backend_name: str, *, suppress_debug: bool = False) -> Optional[BaseStore]:
|
||||||
|
"""Instantiate and return a single store backend by configured name.
|
||||||
|
|
||||||
|
This avoids creating all configured backends (and opening their DBs)
|
||||||
|
when only a single backend is needed (common in `get-file`/`get-metadata`).
|
||||||
|
The function first tries a lightweight match against raw config NAME/value to
|
||||||
|
avoid calling `_build_kwargs` (which can raise if keys are missing).
|
||||||
|
Returns None when no matching backend is found or instantiation fails.
|
||||||
|
"""
|
||||||
|
if not backend_name:
|
||||||
|
return None
|
||||||
|
store_cfg = (config or {}).get("store") or {}
|
||||||
|
if not isinstance(store_cfg, dict):
|
||||||
|
return None
|
||||||
|
classes_by_type = _discover_store_classes()
|
||||||
|
desired = str(backend_name or "").strip().lower()
|
||||||
|
|
||||||
|
for raw_store_type, instances in store_cfg.items():
|
||||||
|
if not isinstance(instances, dict):
|
||||||
|
continue
|
||||||
|
store_type = _normalize_store_type(str(raw_store_type))
|
||||||
|
store_cls = classes_by_type.get(store_type)
|
||||||
|
if store_cls is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Fast path: match using raw 'NAME' or 'name' in config without building full kwargs
|
||||||
|
for instance_name, instance_cfg in instances.items():
|
||||||
|
candidate_alias = None
|
||||||
|
if isinstance(instance_cfg, dict):
|
||||||
|
candidate_alias = (
|
||||||
|
instance_cfg.get("NAME") or instance_cfg.get("name")
|
||||||
|
)
|
||||||
|
candidate_alias = str(candidate_alias or instance_name).strip()
|
||||||
|
if candidate_alias.lower() == desired:
|
||||||
|
try:
|
||||||
|
kwargs = _build_kwargs(store_cls, str(instance_name), instance_cfg)
|
||||||
|
except Exception as exc:
|
||||||
|
if not suppress_debug:
|
||||||
|
debug(f"[Store] Can't build kwargs for '{instance_name}' ({store_type}): {exc}")
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
for key in list(kwargs.keys()):
|
||||||
|
if _normalize_config_key(key) in {"PATH", "LOCATION"}:
|
||||||
|
kwargs[key] = str(expand_path(kwargs[key]))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
backend = store_cls(**kwargs)
|
||||||
|
return backend
|
||||||
|
except Exception as exc:
|
||||||
|
if not suppress_debug:
|
||||||
|
debug(f"[Store] Failed to instantiate backend '{candidate_alias}': {exc}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Fallback: build kwargs for each instance and compare resolved NAME
|
||||||
|
for instance_name, instance_cfg in instances.items():
|
||||||
|
try:
|
||||||
|
kwargs = _build_kwargs(store_cls, str(instance_name), instance_cfg)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
alias = str(kwargs.get("NAME") or instance_name).strip()
|
||||||
|
if alias.lower() != desired:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
for key in list(kwargs.keys()):
|
||||||
|
if _normalize_config_key(key) in {"PATH", "LOCATION"}:
|
||||||
|
kwargs[key] = str(expand_path(kwargs[key]))
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
backend = store_cls(**kwargs)
|
||||||
|
return backend
|
||||||
|
except Exception as exc:
|
||||||
|
if not suppress_debug:
|
||||||
|
debug(f"[Store] Failed to instantiate backend '{alias}': {exc}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
if not suppress_debug:
|
||||||
|
debug(f"[Store] Backend '{backend_name}' not found in config")
|
||||||
|
return None
|
||||||
|
|||||||
@@ -86,9 +86,22 @@ class Get_File(sh.Cmdlet):
|
|||||||
|
|
||||||
debug(f"[get-file] Getting storage backend: {store_name}")
|
debug(f"[get-file] Getting storage backend: {store_name}")
|
||||||
|
|
||||||
# Get storage backend
|
# Prefer instantiating only the named backend to avoid initializing all configured backends
|
||||||
|
try:
|
||||||
|
from Store.registry import get_backend_instance
|
||||||
|
backend = get_backend_instance(config, store_name, suppress_debug=True)
|
||||||
|
except Exception:
|
||||||
|
backend = None
|
||||||
|
|
||||||
|
if backend is None:
|
||||||
|
# Fallback to full registry when targeted instantiation fails
|
||||||
|
try:
|
||||||
store = Store(config)
|
store = Store(config)
|
||||||
backend = store[store_name]
|
backend = store[store_name]
|
||||||
|
except Exception:
|
||||||
|
log(f"Error: Storage backend '{store_name}' not found", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
debug(f"[get-file] Backend retrieved: {type(backend).__name__}")
|
debug(f"[get-file] Backend retrieved: {type(backend).__name__}")
|
||||||
|
|
||||||
# Get file metadata to determine name and extension
|
# Get file metadata to determine name and extension
|
||||||
|
|||||||
@@ -192,10 +192,21 @@ class Get_Metadata(Cmdlet):
|
|||||||
|
|
||||||
# Use storage backend to get metadata
|
# Use storage backend to get metadata
|
||||||
try:
|
try:
|
||||||
from Store import Store
|
# Instantiate only the required backend when possible to avoid initializing all configured backends
|
||||||
|
try:
|
||||||
|
from Store.registry import get_backend_instance
|
||||||
|
backend = get_backend_instance(config, storage_source, suppress_debug=True)
|
||||||
|
except Exception:
|
||||||
|
backend = None
|
||||||
|
|
||||||
|
if backend is None:
|
||||||
|
try:
|
||||||
|
from Store import Store
|
||||||
storage = Store(config)
|
storage = Store(config)
|
||||||
backend = storage[storage_source]
|
backend = storage[storage_source]
|
||||||
|
except Exception:
|
||||||
|
log(f"Storage backend '{storage_source}' not found", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
# Get metadata from backend
|
# Get metadata from backend
|
||||||
metadata = backend.get_metadata(file_hash)
|
metadata = backend.get_metadata(file_hash)
|
||||||
@@ -290,6 +301,13 @@ class Get_Metadata(Cmdlet):
|
|||||||
list(args))
|
list(args))
|
||||||
self._add_table_body_row(table, row)
|
self._add_table_body_row(table, row)
|
||||||
ctx.set_last_result_table_overlay(table, [row], row)
|
ctx.set_last_result_table_overlay(table, [row], row)
|
||||||
|
try:
|
||||||
|
from SYS.rich_display import render_item_details_panel
|
||||||
|
|
||||||
|
render_item_details_panel(row)
|
||||||
|
table._rendered_by_cmdlet = True
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
ctx.emit(row)
|
ctx.emit(row)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|||||||
@@ -214,17 +214,26 @@ class search_file(Cmdlet):
|
|||||||
|
|
||||||
if not library_root:
|
if not library_root:
|
||||||
try:
|
try:
|
||||||
from Store import Store
|
from Store.registry import get_backend_instance
|
||||||
storage_registry = Store(config=config or {})
|
# Try the first configured folder backend without instantiating all backends
|
||||||
# Try the first folder backend
|
store_cfg = (config or {}).get("store") or {}
|
||||||
for name in storage_registry.list_backends():
|
folder_cfg = None
|
||||||
backend = storage_registry[name]
|
for raw_store_type, instances in store_cfg.items():
|
||||||
if type(backend).__name__ == "Folder":
|
if _normalize_store_type(str(raw_store_type)) == "folder":
|
||||||
|
folder_cfg = instances
|
||||||
|
break
|
||||||
|
if isinstance(folder_cfg, dict):
|
||||||
|
for instance_name, instance_config in folder_cfg.items():
|
||||||
|
try:
|
||||||
|
backend = get_backend_instance(config, instance_name, suppress_debug=True)
|
||||||
|
if backend and type(backend).__name__ == "Folder":
|
||||||
library_root = expand_path(getattr(backend, "_location", None))
|
library_root = expand_path(getattr(backend, "_location", None))
|
||||||
if library_root:
|
if library_root:
|
||||||
break
|
break
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
db = None
|
db = None
|
||||||
# Disable Folder DB usage for "external" searches when not using a folder store
|
# Disable Folder DB usage for "external" searches when not using a folder store
|
||||||
@@ -655,10 +664,9 @@ class search_file(Cmdlet):
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
from Store import Store
|
from Store.registry import list_configured_backend_names, get_backend_instance
|
||||||
from Store._base import Store as BaseStore
|
from Store._base import Store as BaseStore
|
||||||
|
|
||||||
storage = storage_registry
|
|
||||||
backend_to_search = storage_backend or None
|
backend_to_search = storage_backend or None
|
||||||
if hash_query:
|
if hash_query:
|
||||||
# Explicit hash list search: build rows from backend metadata.
|
# Explicit hash list search: build rows from backend metadata.
|
||||||
@@ -666,7 +674,7 @@ class search_file(Cmdlet):
|
|||||||
if backend_to_search:
|
if backend_to_search:
|
||||||
backends_to_try = [backend_to_search]
|
backends_to_try = [backend_to_search]
|
||||||
else:
|
else:
|
||||||
backends_to_try = list(storage.list_backends())
|
backends_to_try = list_configured_backend_names(config or {})
|
||||||
|
|
||||||
found_any = False
|
found_any = False
|
||||||
for h in hash_query:
|
for h in hash_query:
|
||||||
@@ -674,9 +682,17 @@ class search_file(Cmdlet):
|
|||||||
resolved_backend = None
|
resolved_backend = None
|
||||||
|
|
||||||
for backend_name in backends_to_try:
|
for backend_name in backends_to_try:
|
||||||
|
backend = None
|
||||||
try:
|
try:
|
||||||
backend = storage[backend_name]
|
backend = get_backend_instance(config, backend_name, suppress_debug=True)
|
||||||
|
if backend is None:
|
||||||
|
# Last-resort: instantiate full registry for this backend only
|
||||||
|
from Store import Store as _Store
|
||||||
|
_store = _Store(config=config)
|
||||||
|
backend = _store[backend_name]
|
||||||
except Exception:
|
except Exception:
|
||||||
|
backend = None
|
||||||
|
if backend is None:
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
# If get_metadata works, consider it a hit; get_file can be optional (e.g. remote URL).
|
# If get_metadata works, consider it a hit; get_file can be optional (e.g. remote URL).
|
||||||
@@ -828,7 +844,17 @@ class search_file(Cmdlet):
|
|||||||
|
|
||||||
if backend_to_search:
|
if backend_to_search:
|
||||||
searched_backends.append(backend_to_search)
|
searched_backends.append(backend_to_search)
|
||||||
target_backend = storage[backend_to_search]
|
try:
|
||||||
|
target_backend = get_backend_instance(config, backend_to_search, suppress_debug=True)
|
||||||
|
if target_backend is None:
|
||||||
|
from Store import Store as _Store
|
||||||
|
_store = _Store(config=config)
|
||||||
|
target_backend = _store[backend_to_search]
|
||||||
|
except Exception as exc:
|
||||||
|
log(f"Backend '{backend_to_search}' not found: {exc}", file=sys.stderr)
|
||||||
|
db.update_worker_status(worker_id, "error")
|
||||||
|
return 1
|
||||||
|
|
||||||
if type(target_backend).search is BaseStore.search:
|
if type(target_backend).search is BaseStore.search:
|
||||||
log(
|
log(
|
||||||
f"Backend '{backend_to_search}' does not support searching",
|
f"Backend '{backend_to_search}' does not support searching",
|
||||||
@@ -843,11 +869,19 @@ class search_file(Cmdlet):
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
all_results = []
|
all_results = []
|
||||||
for backend_name in storage.list_searchable_backends():
|
for backend_name in list_configured_backend_names(config or {}):
|
||||||
try:
|
try:
|
||||||
backend = storage[backend_name]
|
backend = get_backend_instance(config, backend_name, suppress_debug=True)
|
||||||
|
if backend is None:
|
||||||
|
from Store import Store as _Store
|
||||||
|
_store = _Store(config=config)
|
||||||
|
backend = _store[backend_name]
|
||||||
|
|
||||||
searched_backends.append(backend_name)
|
searched_backends.append(backend_name)
|
||||||
|
|
||||||
|
if type(backend).search is BaseStore.search:
|
||||||
|
continue
|
||||||
|
|
||||||
debug(f"[search-file] Searching '{backend_name}'")
|
debug(f"[search-file] Searching '{backend_name}'")
|
||||||
backend_results = backend.search(
|
backend_results = backend.search(
|
||||||
query,
|
query,
|
||||||
@@ -909,6 +943,62 @@ class search_file(Cmdlet):
|
|||||||
if store_val and not normalized.get("store"):
|
if store_val and not normalized.get("store"):
|
||||||
normalized["store"] = store_val
|
normalized["store"] = store_val
|
||||||
|
|
||||||
|
# Populate default selection args for interactive @N selection/hash/url handling
|
||||||
|
try:
|
||||||
|
sel_args: Optional[List[str]] = None
|
||||||
|
sel_action: Optional[List[str]] = None
|
||||||
|
|
||||||
|
# Prefer explicit path when available
|
||||||
|
p_val = normalized.get("path") or normalized.get("target") or normalized.get("url")
|
||||||
|
if p_val:
|
||||||
|
p_str = str(p_val or "").strip()
|
||||||
|
if p_str:
|
||||||
|
if p_str.startswith(("http://", "https://", "magnet:", "torrent:")):
|
||||||
|
sel_args = ["-url", p_str]
|
||||||
|
sel_action = ["download-file", "-url", p_str]
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
from SYS.utils import expand_path
|
||||||
|
|
||||||
|
full_path = expand_path(p_str)
|
||||||
|
# Prefer showing metadata details when we have a hash+store context
|
||||||
|
h = normalized.get("hash") or normalized.get("file_hash") or normalized.get("hash_hex")
|
||||||
|
s_val = normalized.get("store")
|
||||||
|
if h and s_val:
|
||||||
|
try:
|
||||||
|
h_norm = normalize_hash(h)
|
||||||
|
except Exception:
|
||||||
|
h_norm = str(h)
|
||||||
|
sel_args = ["-query", f"hash:{h_norm}", "-store", str(s_val)]
|
||||||
|
sel_action = ["get-metadata", "-query", f"hash:{h_norm}", "-store", str(s_val)]
|
||||||
|
else:
|
||||||
|
sel_args = ["-path", str(full_path)]
|
||||||
|
# Default action for local paths: get-file to fetch or operate on the path
|
||||||
|
sel_action = ["get-file", "-path", str(full_path)]
|
||||||
|
except Exception:
|
||||||
|
sel_args = ["-path", p_str]
|
||||||
|
sel_action = ["get-file", "-path", p_str]
|
||||||
|
|
||||||
|
# Fallback: use hash+store when available
|
||||||
|
if sel_args is None:
|
||||||
|
h = normalized.get("hash") or normalized.get("file_hash") or normalized.get("hash_hex")
|
||||||
|
s_val = normalized.get("store")
|
||||||
|
if h and s_val:
|
||||||
|
try:
|
||||||
|
h_norm = normalize_hash(h)
|
||||||
|
except Exception:
|
||||||
|
h_norm = str(h)
|
||||||
|
sel_args = ["-query", f"hash:{h_norm}", "-store", str(s_val)]
|
||||||
|
# Show metadata details by default for store/hash selections
|
||||||
|
sel_action = ["get-metadata", "-query", f"hash:{h_norm}", "-store", str(s_val)]
|
||||||
|
|
||||||
|
if sel_args:
|
||||||
|
normalized["_selection_args"] = [str(x) for x in sel_args]
|
||||||
|
if sel_action:
|
||||||
|
normalized["_selection_action"] = [str(x) for x in sel_action]
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
table.add_result(normalized)
|
table.add_result(normalized)
|
||||||
|
|
||||||
results_list.append(normalized)
|
results_list.append(normalized)
|
||||||
|
|||||||
23
scripts/check_indentation.py
Normal file
23
scripts/check_indentation.py
Normal file
@@ -0,0 +1,23 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
p = Path('Store/registry.py')
|
||||||
|
lines = p.read_text(encoding='utf-8').splitlines()
|
||||||
|
stack = [0]
|
||||||
|
for i, line in enumerate(lines, start=1):
|
||||||
|
if not line.strip():
|
||||||
|
continue
|
||||||
|
leading = len(line) - len(line.lstrip(' '))
|
||||||
|
if leading > stack[-1]:
|
||||||
|
# increased indent -> push
|
||||||
|
stack.append(leading)
|
||||||
|
else:
|
||||||
|
# dedent: must match an existing level
|
||||||
|
if leading != stack[-1]:
|
||||||
|
if leading in stack:
|
||||||
|
# pop until match
|
||||||
|
while stack and stack[-1] != leading:
|
||||||
|
stack.pop()
|
||||||
|
else:
|
||||||
|
print(f"Line {i}: inconsistent indent {leading}, stack levels {stack}")
|
||||||
|
print(repr(line))
|
||||||
|
break
|
||||||
|
print('Done')
|
||||||
12
scripts/check_store.py
Normal file
12
scripts/check_store.py
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
import traceback
|
||||||
|
|
||||||
|
try:
|
||||||
|
from Store import Store
|
||||||
|
s = Store(config={}, suppress_debug=True)
|
||||||
|
print('INSTANCE TYPE:', type(s))
|
||||||
|
print('HAS is_available:', hasattr(s, 'is_available'))
|
||||||
|
if hasattr(s, 'is_available'):
|
||||||
|
print('is_available callable:', callable(getattr(s, 'is_available')))
|
||||||
|
print('DIR:', sorted([n for n in dir(s) if not n.startswith('__')]))
|
||||||
|
except Exception:
|
||||||
|
traceback.print_exc()
|
||||||
@@ -184,7 +184,7 @@ def _run_cli(clean_args: List[str]) -> int:
|
|||||||
if MedeiaCLI is None:
|
if MedeiaCLI is None:
|
||||||
try:
|
try:
|
||||||
repo_root = _ensure_repo_root_on_sys_path()
|
repo_root = _ensure_repo_root_on_sys_path()
|
||||||
from CLI import MedeiaCLI as _M # type: ignore
|
from CLI import CLI as _M # type: ignore
|
||||||
MedeiaCLI = _M
|
MedeiaCLI = _M
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
# Provide diagnostic information
|
# Provide diagnostic information
|
||||||
|
|||||||
6
scripts/find_big_indent.py
Normal file
6
scripts/find_big_indent.py
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
p = Path('Store/registry.py')
|
||||||
|
for i, line in enumerate(p.read_text(encoding='utf-8').splitlines(), start=1):
|
||||||
|
leading = len(line) - len(line.lstrip(' '))
|
||||||
|
if leading > 20:
|
||||||
|
print(i, leading, repr(line))
|
||||||
11
scripts/indent_check.py
Normal file
11
scripts/indent_check.py
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
import pathlib
|
||||||
|
p = pathlib.Path('Store/registry.py')
|
||||||
|
with p.open('r', encoding='utf-8') as f:
|
||||||
|
lines = f.readlines()
|
||||||
|
for i in range(312, 328):
|
||||||
|
if i-1 < len(lines):
|
||||||
|
line = lines[i-1]
|
||||||
|
leading = line[:len(line)-len(line.lstrip('\t '))]
|
||||||
|
print(f"{i}: {repr(line.rstrip())} | leading={repr(leading)} len={len(leading)} chars={[ord(c) for c in leading]}")
|
||||||
|
else:
|
||||||
|
print(f"{i}: <EOF>")
|
||||||
10
scripts/indent_stats.py
Normal file
10
scripts/indent_stats.py
Normal file
@@ -0,0 +1,10 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
p = Path('Store/registry.py')
|
||||||
|
counts = {}
|
||||||
|
for i, line in enumerate(p.read_text(encoding='utf-8').splitlines(), start=1):
|
||||||
|
if not line.strip():
|
||||||
|
continue
|
||||||
|
leading = len(line) - len(line.lstrip(' '))
|
||||||
|
counts[leading] = counts.get(leading, 0) + 1
|
||||||
|
for k in sorted(counts.keys()):
|
||||||
|
print(k, counts[k])
|
||||||
Reference in New Issue
Block a user