diff --git a/SYS/pipeline.py b/SYS/pipeline.py index 922d26d..cf8bb30 100644 --- a/SYS/pipeline.py +++ b/SYS/pipeline.py @@ -3174,7 +3174,9 @@ class PipelineExecutor: pipe_idx = pipe_index_by_stage.get(stage_index) - overlay_table: Any | None = None + output_table: Any | None = None + pre_stage_table: Any | None = None + pre_last_result_table: Any | None = None session = _worker().WorkerStages.begin_stage( worker_manager, cmd_name=cmd_name, @@ -3204,6 +3206,22 @@ class PipelineExecutor: # should call begin_pipe themselves with the actual count. progress_ui.begin_pipe(pipe_idx, total_items=1) + if stage_index + 1 >= len(stages): + try: + pre_stage_table = ( + ctx.get_current_stage_table() + if hasattr(ctx, "get_current_stage_table") else None + ) + except Exception: + pre_stage_table = None + try: + pre_last_result_table = ( + ctx.get_last_result_table() + if hasattr(ctx, "get_last_result_table") else None + ) + except Exception: + pre_last_result_table = None + # RUN THE CMDLET ret_code = cmd_fn(piped_result, stage_args, config) if ret_code is not None: @@ -3216,20 +3234,41 @@ class PipelineExecutor: pipeline_error = f"Stage '{cmd_name}' failed with exit code {normalized_ret}" return - # Pipeline overlay tables (e.g., get-url detail views) need to be - # rendered when running inside a pipeline because the CLI path - # normally handles rendering. The overlay is only useful when - # we're at the terminal stage of the pipeline. Save the table so - # it can be printed after the pipe finishes. - overlay_table = None + # Terminal pipeline stages need to render overlay tables and also + # newly produced standard result tables from row actions like + # `.config -browse ...`, because there is no outer CLI render pass. + output_table = None if stage_index + 1 >= len(stages): try: - overlay_table = ( + output_table = ( ctx.get_display_table() if hasattr(ctx, "get_display_table") else None ) except Exception: - overlay_table = None + output_table = None + + if output_table is None: + current_stage_table = None + last_result_table = None + try: + current_stage_table = ( + ctx.get_current_stage_table() + if hasattr(ctx, "get_current_stage_table") else None + ) + except Exception: + current_stage_table = None + try: + last_result_table = ( + ctx.get_last_result_table() + if hasattr(ctx, "get_last_result_table") else None + ) + except Exception: + last_result_table = None + + if current_stage_table is not None and current_stage_table is not pre_stage_table: + output_table = current_stage_table + elif last_result_table is not None and last_result_table is not pre_last_result_table: + output_table = last_result_table # Update piped_result for next stage from emitted items stage_emits = list(stage_ctx.emits) @@ -3240,14 +3279,14 @@ class PipelineExecutor: finally: if progress_ui is not None and pipe_idx is not None: progress_ui.finish_pipe(pipe_idx) - if overlay_table is not None: + if output_table is not None: try: from SYS.rich_display import stdout_console stdout_console().print() - stdout_console().print(overlay_table) + stdout_console().print(output_table) except Exception: - logger.exception("Failed to render overlay_table to stdout_console") + logger.exception("Failed to render output_table to stdout_console") if session: try: session.close() diff --git a/cmdlet/file/download.py b/cmdlet/file/download.py index 635bff0..4a736ac 100644 --- a/cmdlet/file/download.py +++ b/cmdlet/file/download.py @@ -8,6 +8,7 @@ Supports: from __future__ import annotations +from collections.abc import Mapping, Sequence as SequenceABC import sys import re from pathlib import Path @@ -21,13 +22,14 @@ from SYS.models import DownloadError, DownloadOptions, DownloadMediaResult from SYS.logger import log, debug_panel, is_debug_enabled from SYS.payload_builders import build_file_result_payload, build_table_result_payload from SYS.pipeline_progress import PipelineProgress -from SYS.result_table import Table +from SYS.result_table import Table, build_display_row from SYS.rich_display import stderr_console as get_stderr_console from SYS import pipeline as pipeline_context from rich.prompt import Prompt # SYS.metadata import deferred: normalize_urls loaded lazily at call site to avoid # pulling in Cryptodome (~900ms) at module import time. from SYS.selection_builder import ( + build_hash_store_selection, extract_selection_fields, extract_urls_from_selection_args, selection_args_have_url, @@ -267,6 +269,8 @@ class Download_File(Cmdlet): ) -> tuple[int, Optional[int]]: downloaded_count = 0 + skipped_duplicate_only = 0 + attempted_download = False suppress_nested, batch_total, batch_index, batch_label = self._batch_progress_state(config) total_urls = len(raw_urls or []) @@ -408,17 +412,49 @@ class Download_File(Cmdlet): # Try generic download_url if not already handled if not handled and hasattr(provider, "download_url"): + parsed_for_provider = parsed + provider_preflight_items = self._resolve_provider_preflight_items( + provider, + url=str(url), + parsed=parsed, + args=args, + ) + if provider_preflight_items: + provider_preflight_urls = [ + str(item.get("url") or "").strip() + for item in provider_preflight_items + if str(item.get("url") or "").strip() + ] + provider_preflight_urls, preflight_exit, provider_skipped = self._preflight_explicit_url_duplicates( + raw_urls=provider_preflight_urls, + config=config, + ) + if preflight_exit is not None: + return downloaded_count, int(preflight_exit) + if provider_skipped: + if not provider_preflight_urls: + skipped_duplicate_only += 1 + continue + selector = self._build_provider_playlist_item_selector( + provider_preflight_items, + remaining_urls=provider_preflight_urls, + ) + if selector: + parsed_for_provider = dict(parsed) + parsed_for_provider["item"] = selector try: + attempted_download = True res = provider.download_url( str(url), final_output_dir, - parsed=parsed, + parsed=parsed_for_provider, args=list(args), progress=progress, quiet_mode=quiet_mode, context_items=list(context_items or []), ) except TypeError: + attempted_download = True res = provider.download_url(str(url), final_output_dir) plugin_downloaded, plugin_exit, plugin_handled = self._consume_plugin_download_result( @@ -471,6 +507,7 @@ class Download_File(Cmdlet): continue # Direct Download Fallback + attempted_download = True result_obj = _download_direct_file( str(url), final_output_dir, @@ -496,6 +533,8 @@ class Download_File(Cmdlet): except Exception as e: log(f"Error processing {url}: {e}", file=sys.stderr) + if downloaded_count == 0 and skipped_duplicate_only > 0 and not attempted_download: + return downloaded_count, 0 return downloaded_count, None def _normalize_provider_key(self, value: Optional[Any]) -> Optional[str]: @@ -981,6 +1020,630 @@ class Download_File(Cmdlet): return normalized return None + @staticmethod + def _iter_duplicate_tag_values(item: Any) -> List[str]: + def _append_tag(out: List[str], value: Any) -> None: + text = "" + if isinstance(value, bytes): + try: + text = value.decode("utf-8", errors="ignore") + except Exception: + text = str(value) + elif isinstance(value, str): + text = value + if not text: + return + cleaned = text.strip() + if cleaned: + out.append(cleaned) + + def _collect_current(container: Any, out: List[str]) -> None: + if isinstance(container, SequenceABC) and not isinstance(container, (str, bytes, bytearray, Mapping)): + for tag in container: + _append_tag(out, tag) + return + if not isinstance(container, Mapping): + return + current = container.get("0") + if current is None: + current = container.get(0) + if isinstance(current, SequenceABC) and not isinstance(current, (str, bytes, bytearray, Mapping)): + for tag in current: + _append_tag(out, tag) + + def _collect_service_data(service_data: Any, out: List[str]) -> None: + if not isinstance(service_data, Mapping): + return + for key in ( + "display_tags", + "display_friendly_tags", + "display", + "storage_tags", + "statuses_to_tags", + "tags", + ): + _collect_current(service_data.get(key), out) + + collected: List[str] = [] + for raw_tags in ( + get_field(item, "tags_flat"), + get_field(item, "tags"), + get_field(item, "tag"), + ): + if isinstance(raw_tags, str): + _append_tag(collected, raw_tags) + continue + if isinstance(raw_tags, (list, tuple, set)): + for raw_tag in raw_tags: + _append_tag(collected, raw_tag) + continue + if not isinstance(raw_tags, Mapping): + continue + + statuses_map = raw_tags.get("service_keys_to_statuses_to_tags") + if isinstance(statuses_map, Mapping): + for status_payload in statuses_map.values(): + _collect_current(status_payload, collected) + + names_map = raw_tags.get("service_keys_to_names") + if isinstance(names_map, Mapping): + _ = names_map + + _collect_service_data(raw_tags, collected) + for maybe_service in raw_tags.values(): + _collect_service_data(maybe_service, collected) + + deduped: List[str] = [] + seen: set[str] = set() + for raw_tag in collected: + text = str(raw_tag or "").strip() + key = text.lower() + if not text or key in seen: + continue + seen.add(key) + deduped.append(text) + return deduped + + @staticmethod + def _extract_duplicate_namespace_tags(item: Any) -> List[str]: + tag_values = Download_File._iter_duplicate_tag_values(item) + + namespace_tags: List[str] = [] + seen: set[str] = set() + for raw_tag in tag_values: + text = str(raw_tag or "").strip() + if not text: + continue + lower = text.lower() + if ":" not in text or lower.startswith("title:"): + continue + if lower in seen: + continue + seen.add(lower) + namespace_tags.append(text) + return namespace_tags + + @staticmethod + def _extract_duplicate_title_tag(item: Any) -> Optional[str]: + for raw_tag in Download_File._iter_duplicate_tag_values(item): + tag_text = str(raw_tag or "").strip() + if not tag_text or not tag_text.lower().startswith("title:"): + continue + value = tag_text.split(":", 1)[1].strip() + if value: + return value + return None + + @classmethod + def _extract_duplicate_title(cls, item: Any) -> str: + for key in ("title", "name", "filename", "target"): + value = get_field(item, key) + text = str(value or "").strip() + if text: + return text + + tag_title = cls._extract_duplicate_title_tag(item) + if tag_title: + return tag_title + + path_value = str(get_field(item, "path") or "").strip() + if path_value and not path_value.lower().startswith(("http://", "https://", "file://")): + return path_value + + return "(exists)" + + @classmethod + def _has_duplicate_title(cls, item: Any) -> bool: + return cls._extract_duplicate_title(item) != "(exists)" + + @staticmethod + def _normalize_duplicate_preflight_policy(value: Any) -> Optional[str]: + text = str(value or "").strip().lower() + if not text: + return "skip" + mapping = { + "i": "ignore", + "ignore": "ignore", + "s": "skip", + "skip": "skip", + "c": "cancel", + "cancel": "cancel", + } + return mapping.get(text) + + @classmethod + def _build_duplicate_display_row( + cls, + item: Any, + *, + backend_name: str, + original_url: str, + ) -> Dict[str, Any]: + try: + extracted = build_display_row(item, keys=["title", "store", "hash", "ext", "size"]) + except Exception: + extracted = {} + + title = extracted.get("title") or cls._extract_duplicate_title(item) + store_name = extracted.get("store") or get_field(item, "store") or backend_name + file_hash = extracted.get("hash") or get_field(item, "hash") or get_field(item, "file_hash") or get_field(item, "hash_hex") or "" + ext_text = str(extracted.get("ext") or get_field(item, "ext") or "").strip() + size_raw = extracted.get("size") + if size_raw is None: + size_raw = get_field(item, "size_bytes") + if size_raw is None: + size_raw = get_field(item, "size") + + if not ext_text: + for candidate in (get_field(item, "path"), get_field(item, "title"), get_field(item, "name")): + candidate_text = str(candidate or "").strip() + if not candidate_text: + continue + suffix = Path(candidate_text).suffix.lstrip(".") + if suffix: + ext_text = suffix + break + + title_text = str(title) + tag_text = ", ".join(cls._extract_duplicate_namespace_tags(item)) + store_text = str(store_name or backend_name) + file_hash_text = str(file_hash or "") + selection_args = None + selection_action = None + selection_url = None + if file_hash_text and store_text and file_hash_text.strip().lower() != "unknown": + selection_args, selection_action = build_hash_store_selection( + file_hash_text, + store_text, + ) + if selection_args and len(selection_args) >= 2: + normalized_hash = str(selection_args[1]).split("hash:", 1)[-1].strip() + if normalized_hash: + file_hash_text = normalized_hash + selection_url = f"hydrus://{store_text}/{normalized_hash}" + + columns: List[tuple[str, Any]] = [("Title", title_text)] + if tag_text: + columns.append(("Tag", tag_text)) + columns.extend( + [ + ("Store", store_text), + ("Size", size_raw), + ("Ext", ext_text), + ("URL", original_url), + ] + ) + + metadata = dict(item) if isinstance(item, dict) else {} + if file_hash_text: + metadata.setdefault("hash", file_hash_text) + if store_text: + metadata.setdefault("store", store_text) + if ext_text: + metadata.setdefault("ext", ext_text) + if size_raw is not None: + metadata.setdefault("size", size_raw) + metadata.setdefault("size_bytes", size_raw) + metadata.setdefault("url", original_url) + if selection_url: + metadata.setdefault("selection_url", selection_url) + + payload = build_table_result_payload( + title=title_text, + columns=columns, + selection_args=selection_args, + selection_action=selection_action, + store=store_text, + hash=file_hash_text, + ext=ext_text, + size=size_raw, + size_bytes=size_raw, + url=original_url, + tags_flat=metadata.get("tags_flat"), + full_metadata=metadata, + ) + if selection_url: + payload["path"] = selection_url + payload["selection_url"] = selection_url + return payload + + @classmethod + def _fetch_duplicate_metadata_for_hash( + cls, + backend: Any, + *, + backend_name: str, + file_hash: str, + ) -> Dict[str, Any]: + metadata: Optional[Dict[str, Any]] = None + + fetcher = getattr(backend, "fetch_file_metadata", None) + if callable(fetcher): + try: + payload = fetcher(file_hash) + except TypeError: + try: + payload = fetcher(file_hash=file_hash) + except Exception: + payload = None + except Exception: + payload = None + + if isinstance(payload, dict): + meta_list = payload.get("metadata") + if isinstance(meta_list, list) and meta_list and isinstance(meta_list[0], dict): + metadata = dict(meta_list[0]) + else: + metadata = dict(payload) + + metadata = cls._enrich_duplicate_metadata( + metadata, + backend, + backend_name=backend_name, + file_hash=file_hash, + ) + + metadata.setdefault("hash", file_hash) + metadata.setdefault("store", backend_name) + return metadata + + @classmethod + def _enrich_duplicate_metadata( + cls, + metadata: Optional[Dict[str, Any]], + backend: Any, + *, + backend_name: str, + file_hash: str, + ) -> Dict[str, Any]: + result = dict(metadata) if isinstance(metadata, dict) else {} + + if result: + extractor = getattr(backend, "_extract_title_and_tags", None) + if callable(extractor): + file_id_value = get_field(result, "file_id") or file_hash + try: + extracted_title, extracted_tags = extractor(result, file_id_value) + except Exception: + extracted_title, extracted_tags = None, None + + if not get_field(result, "tags_flat") and isinstance(extracted_tags, SequenceABC) and not isinstance(extracted_tags, (str, bytes, bytearray, Mapping)): + deduped_tags: List[str] = [] + seen_tags: set[str] = set() + for raw_tag in extracted_tags: + tag_text = str(raw_tag or "").strip() + lowered = tag_text.lower() + if not tag_text or lowered in seen_tags: + continue + seen_tags.add(lowered) + deduped_tags.append(tag_text) + if deduped_tags: + result["tags_flat"] = deduped_tags + + title_text = str(extracted_title or "").strip() + generic_title = f"Hydrus File {file_id_value}".strip() + if title_text and title_text != generic_title: + result.setdefault("title", title_text) + + if not result: + getter = getattr(backend, "get_metadata", None) + if callable(getter): + try: + payload = getter(file_hash) + except Exception: + payload = None + if isinstance(payload, dict): + result = dict(payload) + + getter = getattr(backend, "get_metadata", None) + if callable(getter) and not cls._has_duplicate_title(result): + try: + getter_payload = getter(file_hash) + except Exception: + getter_payload = None + if isinstance(getter_payload, dict): + for key, value in getter_payload.items(): + current = result.get(key) + if current not in (None, "", [], {}, ()): + continue + if value in (None, "", [], {}, ()): + continue + result[key] = value + + return result + + @classmethod + def _fetch_duplicate_metadata_for_hashes( + cls, + backend: Any, + *, + backend_name: str, + file_hashes: Sequence[str], + ) -> Dict[str, Dict[str, Any]]: + normalized_hashes: List[str] = [] + seen_hashes: set[str] = set() + for raw_hash in file_hashes or []: + normalized_hash = sh.normalize_hash(str(raw_hash) if raw_hash is not None else None) + if not normalized_hash or normalized_hash in seen_hashes: + continue + seen_hashes.add(normalized_hash) + normalized_hashes.append(normalized_hash) + + if not normalized_hashes: + return {} + + metadata_by_hash: Dict[str, Dict[str, Any]] = {} + fetcher = getattr(backend, "fetch_files_metadata", None) + if callable(fetcher): + try: + payload = fetcher( + normalized_hashes, + include_service_keys_to_tags=True, + include_file_url=True, + include_duration=True, + include_size=True, + include_mime=True, + ) + except TypeError: + try: + payload = fetcher( + file_hashes=normalized_hashes, + include_service_keys_to_tags=True, + include_file_url=True, + include_duration=True, + include_size=True, + include_mime=True, + ) + except Exception: + payload = None + except Exception: + payload = None + + if isinstance(payload, dict): + meta_list = payload.get("metadata") + if isinstance(meta_list, list): + for entry in meta_list: + if not isinstance(entry, dict): + continue + entry_hash = sh.normalize_hash(str(entry.get("hash") or entry.get("hash_hex") or entry.get("file_hash") or "")) + if not entry_hash: + continue + metadata_by_hash[entry_hash] = cls._enrich_duplicate_metadata( + dict(entry), + backend, + backend_name=backend_name, + file_hash=entry_hash, + ) + + for normalized_hash in normalized_hashes: + metadata = metadata_by_hash.get(normalized_hash) + if metadata is None: + metadata = cls._fetch_duplicate_metadata_for_hash( + backend, + backend_name=backend_name, + file_hash=normalized_hash, + ) + metadata.setdefault("hash", normalized_hash) + metadata.setdefault("store", backend_name) + metadata_by_hash[normalized_hash] = metadata + + return metadata_by_hash + + @classmethod + def _collect_existing_url_match_refs_for_url( + cls, + storage: Any, + canonical_url: str, + *, + hydrus_available: bool, + config: Optional[Dict[str, Any]] = None, + ) -> List[Dict[str, Any]]: + if not canonical_url: + return [] + + config_dict = config if isinstance(config, dict) else {} + refs: List[Dict[str, Any]] = [] + seen_pairs: set[tuple[str, str]] = set() + seen_backends: set[str] = set() + + def _append_ref(backend_name: str, backend: Any, *, item: Any = None, file_hash_hint: Optional[str] = None, is_exact: bool = False) -> None: + normalized_hash = sh.normalize_hash(str(file_hash_hint) if file_hash_hint is not None else None) + if not normalized_hash: + normalized_hash = cls._extract_hash_from_search_hit(item) + pair_key = (str(backend_name or "").strip().lower(), str(normalized_hash or "")) + if pair_key in seen_pairs: + return + seen_pairs.add(pair_key) + refs.append( + { + "backend_name": str(backend_name or "").strip(), + "backend": backend, + "hash": normalized_hash, + "item": dict(item) if isinstance(item, dict) else item, + "is_exact": bool(is_exact), + } + ) + + def _iter_backends() -> List[tuple[str, Any]]: + backends: List[tuple[str, Any]] = [] + if storage is not None: + try: + backend_names = list(storage.list_searchable_backends() or []) + except Exception: + backend_names = [] + + for backend_name in backend_names: + try: + backend = storage[backend_name] + except Exception: + continue + name_text = str(backend_name).strip() + if not name_text or name_text.lower() == "temp": + continue + key = name_text.lower() + if key in seen_backends: + continue + seen_backends.add(key) + backends.append((name_text, backend)) + + try: + registry_helpers = cls._load_provider_registry() + get_plugin = registry_helpers.get("get_plugin") + hydrus_provider = get_plugin("hydrusnetwork", config_dict) if callable(get_plugin) else None + if hydrus_provider is not None: + for backend_name, backend in hydrus_provider.iter_backends(): + name_text = str(backend_name or "").strip() + if not name_text: + continue + key = name_text.lower() + if key in seen_backends: + continue + seen_backends.add(key) + backends.append((name_text, backend)) + except Exception: + pass + + return backends + + for backend_name, backend in _iter_backends(): + try: + if not hydrus_available and str(getattr(backend, "STORE_TYPE", "")).strip().lower() == "hydrusnetwork": + continue + except Exception: + pass + + found_exact = False + lookup_exact = getattr(backend, "find_hashes_by_url", None) + if callable(lookup_exact): + try: + hashes = lookup_exact(canonical_url) or [] + except Exception: + hashes = [] + if isinstance(hashes, (list, tuple, set)): + for existing_hash in hashes: + normalized_hash = sh.normalize_hash(str(existing_hash) if existing_hash is not None else None) + if not normalized_hash: + continue + found_exact = True + _append_ref( + backend_name, + backend, + file_hash_hint=normalized_hash, + is_exact=True, + ) + if found_exact: + continue + + searcher = getattr(backend, "search", None) + if callable(searcher): + try: + hits = searcher(f"url:{canonical_url}", limit=5, minimal=True) or [] + except Exception: + hits = [] + for hit in hits: + _append_ref(backend_name, backend, item=hit) + + return refs + + @classmethod + def _find_existing_url_matches_for_url( + cls, + storage: Any, + canonical_url: str, + *, + hydrus_available: bool, + config: Optional[Dict[str, Any]] = None, + ) -> List[Dict[str, Any]]: + refs = cls._collect_existing_url_match_refs_for_url( + storage, + canonical_url, + hydrus_available=hydrus_available, + config=config, + ) + if not refs: + return [] + + matches: List[Dict[str, Any]] = [] + exact_hashes_by_backend: Dict[str, Dict[str, Any]] = {} + prefetched_metadata: Dict[tuple[str, str], Dict[str, Any]] = {} + + for ref in refs: + if not ref.get("is_exact"): + continue + backend_name = str(ref.get("backend_name") or "").strip() + backend_key = backend_name.lower() + normalized_hash = sh.normalize_hash(str(ref.get("hash") or "")) + if not backend_key or not normalized_hash: + continue + bucket = exact_hashes_by_backend.setdefault( + backend_key, + { + "backend_name": backend_name, + "backend": ref.get("backend"), + "hashes": [], + }, + ) + if normalized_hash not in bucket["hashes"]: + bucket["hashes"].append(normalized_hash) + + for backend_key, bucket in exact_hashes_by_backend.items(): + metadata_map = cls._fetch_duplicate_metadata_for_hashes( + bucket.get("backend"), + backend_name=str(bucket.get("backend_name") or backend_key), + file_hashes=list(bucket.get("hashes") or []), + ) + for normalized_hash, metadata in metadata_map.items(): + prefetched_metadata[(backend_key, normalized_hash)] = metadata + + for ref in refs: + backend_name = str(ref.get("backend_name") or "").strip() + backend_key = backend_name.lower() + normalized_hash = sh.normalize_hash(str(ref.get("hash") or "")) + if ref.get("is_exact") and normalized_hash: + candidate = prefetched_metadata.get((backend_key, normalized_hash)) + if candidate is None: + candidate = cls._fetch_duplicate_metadata_for_hash( + ref.get("backend"), + backend_name=backend_name, + file_hash=normalized_hash, + ) + else: + item = ref.get("item") + candidate = dict(item) if isinstance(item, dict) else {"hash": normalized_hash or "", "store": backend_name} + + if normalized_hash: + candidate.setdefault("hash", normalized_hash) + candidate.setdefault("store", backend_name) + matches.append( + cls._build_duplicate_display_row( + candidate, + backend_name=backend_name, + original_url=canonical_url, + ) + ) + + return matches + @classmethod def _find_existing_hash_for_url( cls, storage: Any, canonical_url: str, *, hydrus_available: bool @@ -1232,95 +1895,21 @@ class Download_File(Cmdlet): hydrus_available: bool, config: Optional[Dict[str, Any]] = None, ) -> List[str]: - if not canonical_url: - return [] - - config_dict = config if isinstance(config, dict) else {} - found_hashes: List[str] = [] + refs = cls._collect_existing_url_match_refs_for_url( + storage, + canonical_url, + hydrus_available=hydrus_available, + config=config, + ) + hashes: List[str] = [] seen_hashes: set[str] = set() - seen_backends: set[str] = set() - - def _add_hash(value: Any) -> None: - normalized = sh.normalize_hash(str(value) if value is not None else None) + for ref in refs: + normalized = sh.normalize_hash(str(ref.get("hash") or "")) if not normalized or normalized in seen_hashes: - return + continue seen_hashes.add(normalized) - found_hashes.append(normalized) - - def _iter_backends() -> List[tuple[str, Any]]: - backends: List[tuple[str, Any]] = [] - if storage is not None: - try: - backend_names = list(storage.list_searchable_backends() or []) - except Exception: - backend_names = [] - - for backend_name in backend_names: - try: - backend = storage[backend_name] - except Exception: - continue - name_text = str(backend_name).strip() - if not name_text: - continue - if name_text.lower() == "temp": - continue - key = name_text.lower() - if key in seen_backends: - continue - seen_backends.add(key) - backends.append((name_text, backend)) - - # Hydrus can be plugin-configured without appearing in Store.list_searchable_backends(). - try: - registry_helpers = cls._load_provider_registry() - get_plugin = registry_helpers.get("get_plugin") - hydrus_provider = get_plugin("hydrusnetwork", config_dict) if callable(get_plugin) else None - if hydrus_provider is not None: - for backend_name, backend in hydrus_provider.iter_backends(): - name_text = str(backend_name or "").strip() - if not name_text: - continue - key = name_text.lower() - if key in seen_backends: - continue - seen_backends.add(key) - backends.append((name_text, backend)) - except Exception: - pass - - return backends - - for backend_name, backend in _iter_backends(): - try: - if not hydrus_available and str(getattr(backend, "STORE_TYPE", "")).strip().lower() == "hydrusnetwork": - continue - except Exception: - pass - - lookup_exact = getattr(backend, "find_hashes_by_url", None) - if callable(lookup_exact): - try: - hashes = lookup_exact(canonical_url) or [] - except Exception: - hashes = [] - if isinstance(hashes, (list, tuple, set)): - for existing_hash in hashes: - _add_hash(existing_hash) - if found_hashes: - continue - - searcher = getattr(backend, "search", None) - if callable(searcher): - try: - hits = searcher(f"url:{canonical_url}", limit=5, minimal=True) or [] - except Exception: - hits = [] - for hit in hits: - extracted = cls._extract_hash_from_search_hit(hit) - _add_hash(extracted) - - return found_hashes + hashes.append(normalized) + return hashes def _preflight_explicit_url_duplicates( self, @@ -1337,20 +1926,82 @@ class Download_File(Cmdlet): return urls, None, 0 storage, hydrus_available = self._init_storage(config) - duplicates: Dict[str, List[str]] = {} + duplicate_refs: Dict[str, List[Dict[str, Any]]] = {} + exact_hashes_by_backend: Dict[str, Dict[str, Any]] = {} for url in urls: - found = self._find_existing_hashes_for_url( + refs = self._collect_existing_url_match_refs_for_url( storage, url, hydrus_available=hydrus_available, config=config, ) - if found: - duplicates[url] = found + if not refs: + continue + duplicate_refs[url] = refs + for ref in refs: + if not ref.get("is_exact"): + continue + backend_name = str(ref.get("backend_name") or "").strip() + backend_key = backend_name.lower() + normalized_hash = sh.normalize_hash(str(ref.get("hash") or "")) + if not backend_key or not normalized_hash: + continue + bucket = exact_hashes_by_backend.setdefault( + backend_key, + { + "backend_name": backend_name, + "backend": ref.get("backend"), + "hashes": [], + }, + ) + if normalized_hash not in bucket["hashes"]: + bucket["hashes"].append(normalized_hash) - if not duplicates: + if not duplicate_refs: return urls, None, 0 + prefetched_metadata: Dict[tuple[str, str], Dict[str, Any]] = {} + for backend_key, bucket in exact_hashes_by_backend.items(): + metadata_map = self._fetch_duplicate_metadata_for_hashes( + bucket.get("backend"), + backend_name=str(bucket.get("backend_name") or backend_key), + file_hashes=list(bucket.get("hashes") or []), + ) + for normalized_hash, metadata in metadata_map.items(): + prefetched_metadata[(backend_key, normalized_hash)] = metadata + + duplicates: Dict[str, List[Dict[str, Any]]] = {} + for url, refs in duplicate_refs.items(): + rows: List[Dict[str, Any]] = [] + for ref in refs: + backend_name = str(ref.get("backend_name") or "").strip() + backend_key = backend_name.lower() + normalized_hash = sh.normalize_hash(str(ref.get("hash") or "")) + if ref.get("is_exact") and normalized_hash: + candidate = prefetched_metadata.get((backend_key, normalized_hash)) + if candidate is None: + candidate = self._fetch_duplicate_metadata_for_hash( + ref.get("backend"), + backend_name=backend_name, + file_hash=normalized_hash, + ) + else: + item = ref.get("item") + candidate = dict(item) if isinstance(item, dict) else {"hash": normalized_hash or "", "store": backend_name} + + if normalized_hash: + candidate.setdefault("hash", normalized_hash) + candidate.setdefault("store", backend_name) + rows.append( + self._build_duplicate_display_row( + candidate, + backend_name=backend_name, + original_url=url, + ) + ) + if rows: + duplicates[url] = rows + duplicate_count = len(duplicates) total_count = len(urls) try: @@ -1365,43 +2016,68 @@ class Download_File(Cmdlet): except Exception: pass - table = Table(f"Duplicate URLs detected ({duplicate_count}/{total_count})", max_columns=8) + table = Table(f"Duplicate URLs detected ({duplicate_count}/{total_count})", max_columns=12) table._interactive(False) - for url, hashes in duplicates.items(): - table.add_result( - build_table_result_payload( - title="(exists)", - columns=[ - ("URL", url), - ("Hash", str((hashes[0] if hashes else "") or "")), - ], - url=url, - hash=str((hashes[0] if hashes else "") or ""), - ) - ) + duplicate_rows: List[Dict[str, Any]] = [] + for _url, rows in duplicates.items(): + for row in rows: + payload = dict(row) if isinstance(row, dict) else {} + duplicate_rows.append(payload) + table.add_result(payload) + + try: + pipeline_context.set_last_result_table_overlay(table, duplicate_rows) + except Exception: + pass try: stdin_interactive = bool(sys.stdin and sys.stdin.isatty()) except Exception: stdin_interactive = False - policy = "skip" - if stdin_interactive: - console = get_stderr_console() - console.print(table) - policy = Prompt.ask( - "Duplicate URLs found. Action?", - choices=["ignore", "skip", "cancel"], - default="skip", - console=console, - ) - else: - # Safe default in non-interactive runs: avoid redownloading known duplicates. - policy = "skip" + suspend = getattr(pipeline_context, "suspend_live_progress", None) + cm: AbstractContextManager[Any] = nullcontext() + if callable(suspend): try: - get_stderr_console().print(table) + maybe_cm = suspend() + if maybe_cm is not None: + cm = maybe_cm # type: ignore[assignment] + except Exception: + cm = nullcontext() + + policy = "skip" + with cm: + console = get_stderr_console() + try: + console.print(table) except Exception: pass + setattr(table, "_rendered_by_cmdlet", True) + + if stdin_interactive: + while True: + try: + raw_policy = Prompt.ask( + "Duplicate URLs found. Action? [I]gnore/[S]kip/[C]ancel", + default="skip", + console=console, + ) + except (EOFError, KeyboardInterrupt): + policy = "cancel" + break + + normalized_policy = self._normalize_duplicate_preflight_policy(raw_policy) + if normalized_policy is not None: + policy = normalized_policy + break + + try: + console.print("Please select one of: I, S, C, ignore, skip, cancel") + except Exception: + pass + else: + # Safe default in non-interactive runs: avoid redownloading known duplicates. + policy = "skip" if policy == "cancel": try: @@ -1422,6 +2098,82 @@ class Download_File(Cmdlet): pass return filtered, None, skipped + @staticmethod + def _resolve_provider_preflight_items( + provider: Any, + *, + url: str, + parsed: Dict[str, Any], + args: Sequence[str], + ) -> List[Dict[str, Any]]: + resolver = getattr(provider, "resolve_preflight_items", None) + if not callable(resolver): + return [] + + try: + items = resolver(url, parsed=parsed, args=list(args)) + except TypeError: + try: + items = resolver(url) + except Exception: + items = None + except Exception: + items = None + + if not isinstance(items, list): + return [] + + normalized: List[Dict[str, Any]] = [] + for idx, item in enumerate(items, 1): + if not isinstance(item, dict): + continue + item_url = str(item.get("url") or "").strip() + if not item_url: + continue + playlist_index = item.get("playlist_index") + try: + playlist_index_value = int(playlist_index) + except Exception: + playlist_index_value = idx + normalized.append( + { + "url": item_url, + "playlist_index": playlist_index_value, + } + ) + + return normalized + + @staticmethod + def _build_provider_playlist_item_selector( + items: Sequence[Dict[str, Any]], + *, + remaining_urls: Sequence[str], + ) -> Optional[str]: + allowed_urls = { + str(url or "").strip() for url in (remaining_urls or []) if str(url or "").strip() + } + if not allowed_urls: + return None + + selectors: List[str] = [] + for idx, item in enumerate(items, 1): + item_url = str(item.get("url") or "").strip() + if not item_url or item_url not in allowed_urls: + continue + playlist_index = item.get("playlist_index") + try: + playlist_index_value = int(playlist_index) + except Exception: + playlist_index_value = idx + if playlist_index_value <= 0: + continue + selectors.append(str(playlist_index_value)) + + if not selectors: + return None + return ",".join(selectors) + @staticmethod def _format_timecode(seconds: int, *, force_hours: bool) -> str: total = max(0, int(seconds)) diff --git a/cmdlet/metadata/tag_add.py b/cmdlet/metadata/tag_add.py index 1af8b2e..7408226 100644 --- a/cmdlet/metadata/tag_add.py +++ b/cmdlet/metadata/tag_add.py @@ -553,13 +553,26 @@ class Add_Tag(Cmdlet): extract_debug = bool(parsed.get("extract-debug", False)) extract_debug_rx, extract_debug_err = _try_compile_extract_template(extract_template) + raw_tag = parsed.get("tag", []) + if isinstance(raw_tag, str): + raw_tag = [raw_tag] + + # Normalize input early so a non-hash -query can be treated as the tag payload + # when the target item is already coming from the pipeline. + results = normalize_result_input(result) + + query_value = parsed.get("query") query_hash, query_valid = sh.require_single_hash_query( - parsed.get("query"), + query_value, "[add_tag] Error: -query must be of the form hash:", log_file=sys.stderr, ) if not query_valid: - return 1 + if not raw_tag and results and query_value: + raw_tag = [str(query_value)] + query_hash = None + else: + return 1 hash_override = query_hash @@ -581,9 +594,6 @@ class Add_Tag(Cmdlet): if has_downstream and not include_temp and not store_override: include_temp = True - # Normalize input to list - results = normalize_result_input(result) - # Filter by temp status (unless --all is set) if not include_temp: results = filter_results_by_temp(results, include_temp=False) @@ -599,11 +609,6 @@ class Add_Tag(Cmdlet): ) return 1 - # Get tag from arguments (or fallback to pipeline payload) - raw_tag = parsed.get("tag", []) - if isinstance(raw_tag, str): - raw_tag = [raw_tag] - # Fallback: if no tag provided explicitly, try to pull from first result payload. # IMPORTANT: when -extract is used, users typically want *only* extracted tags, # not "re-add whatever tags are already in the payload". diff --git a/cmdlet/metadata/tag_delete.py b/cmdlet/metadata/tag_delete.py index a1f93c1..ac789ea 100644 --- a/cmdlet/metadata/tag_delete.py +++ b/cmdlet/metadata/tag_delete.py @@ -462,13 +462,23 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: rest.append(a) i += 1 + # Normalize the incoming target list early so a non-hash -query can be treated + # as the delete-tag payload when the file target already comes from the pipeline. + items_to_process = sh.normalize_result_items(result) + tags_arg = _parse_delete_tag_arguments(rest) + override_hash, query_valid = sh.require_single_hash_query( override_query, "Invalid -query value (expected hash:)", log_file=sys.stderr, ) if not query_valid: - return 1 + if (not tags_arg and override_query and items_to_process and not has_piped_tag + and not has_piped_tag_list): + tags_arg = _parse_delete_tag_arguments([override_query]) + override_hash = None + else: + return 1 # Selection syntax (@...) is handled by the pipeline runner, not by this cmdlet. # If @ reaches here as a literal argument, it's almost certainly user error. @@ -485,7 +495,6 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: except Exception: grouped_table = "" grouped_tags = get_field(result, "tag") if result is not None else None - tags_arg = _parse_delete_tag_arguments(rest) if (grouped_table == "tag.selection" and isinstance(grouped_tags, list) and grouped_tags and not tags_arg): @@ -503,9 +512,6 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: log("Requires at least one tag argument") return 1 - # Normalize result to a list for processing - items_to_process = sh.normalize_result_items(result) - # Process each item success_count = 0 diff --git a/cmdnat/config.py b/cmdnat/config.py index cedab9a..4c801cc 100644 --- a/cmdnat/config.py +++ b/cmdnat/config.py @@ -15,11 +15,44 @@ from SYS.logger import log from SYS import pipeline as ctx from SYS.result_table import Table from cmdnat._parsing import ( + VALUE_ARG_FLAGS, extract_piped_value as _extract_piped_value, + extract_arg_value as _extract_arg_value, extract_value_arg as _extract_value_arg, has_flag as _has_flag, ) + +_PREFERENCES_BROWSE_PATH = "__preferences__" +_PLUGINS_BROWSE_PATH = "__plugins__" +_PLUGIN_CATEGORY_KEYS = ("plugin", "provider", "tool") +_KNOWN_SECTION_LABELS = { + "plugin": "Plugins", + "provider": "Plugins", + "tool": "Plugins", +} +_KNOWN_SECTION_DESCRIPTIONS = { + _PREFERENCES_BROWSE_PATH: "Global preferences and simple values", + _PLUGINS_BROWSE_PATH: "All configured plugins and plugin instances", + "provider": "Plugin configuration", + "plugin": "Plugin configuration", + "tool": "Plugin configuration", +} +_SENSITIVE_CONFIG_KEYS = { + "access_key", + "access_token", + "api", + "api_key", + "apikey", + "authorization", + "bearer_token", + "cookie", + "cookies", + "password", + "secret", + "token", +} + CMDLET = Cmdlet( name=".config", summary="Manage configuration settings", @@ -173,29 +206,316 @@ def _show_config_logs(args: Sequence[str]) -> int: return 0 -def flatten_config(config: Dict[str, Any], parent_key: str = "", sep: str = ".") -> List[Dict[str, Any]]: - items: List[Dict[str, Any]] = [] - for k, v in config.items(): - if k.startswith("_"): - continue - new_key = f"{parent_key}{sep}{k}" if parent_key else k - if isinstance(v, dict): - items.extend(flatten_config(v, new_key, sep=sep)) - else: - items.append({ - "key": new_key, - "value": v, - "value_display": str(v), - "type": type(v).__name__, - }) - return items - - def set_nested_config(config: Dict[str, Any], key: str, value: str) -> bool: return set_nested_config_value(config, key, value, on_error=print) -def _get_selected_config_key() -> Optional[str]: +def _visible_config_entries(config_data: Any) -> List[tuple[str, Any]]: + if not isinstance(config_data, dict): + return [] + return [ + (str(key), value) + for key, value in config_data.items() + if isinstance(key, str) and not key.startswith("_") + ] + + +def _format_config_label(value: Any) -> str: + text = str(value or "").strip() + if not text: + return "Configuration" + if text == _PREFERENCES_BROWSE_PATH: + return "Preferences" + if text == _PLUGINS_BROWSE_PATH: + return "Plugins" + return text.replace("_", " ").replace("-", " ").strip().title() + + +def _format_config_path_label(browse_path: Optional[str]) -> str: + text = str(browse_path or "").strip() + if not text: + return "Root" + if text == _PREFERENCES_BROWSE_PATH: + return "Preferences" + if text == _PLUGINS_BROWSE_PATH: + return "Plugins" + parts = [part for part in text.split(".") if part] + formatted: List[str] = [] + for idx, part in enumerate(parts): + if idx == 0 and part in _PLUGIN_CATEGORY_KEYS: + formatted.append("Plugins") + else: + formatted.append(_format_config_label(part)) + return " / ".join(formatted) + + +def _format_config_value(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if value is None: + return "null" + if isinstance(value, (list, tuple, set)): + return ", ".join(str(item) for item in value) + return str(value) + + +def _is_sensitive_config_key(key_path: str) -> bool: + leaf = str(key_path or "").split(".")[-1].strip().lower() + return leaf in _SENSITIVE_CONFIG_KEYS + + +def _format_config_entry_count(value: Any) -> str: + count = len(_visible_config_entries(value)) if isinstance(value, dict) else 0 + if count == 1: + return "1 entry" + return f"{count} entries" + + +def _iter_plugin_branches(config_data: Dict[str, Any]) -> List[tuple[str, str, Any]]: + branches: List[tuple[str, str, Any]] = [] + if not isinstance(config_data, dict): + return branches + + for category in _PLUGIN_CATEGORY_KEYS: + category_block = config_data.get(category) + if not isinstance(category_block, dict): + continue + for name, value in _visible_config_entries(category_block): + branches.append((category, name, value)) + return branches + + +def _collect_plugin_root_items(config_data: Dict[str, Any]) -> List[Dict[str, Any]]: + plugin_items: Dict[str, Dict[str, Any]] = {} + for category, name, value in _iter_plugin_branches(config_data): + key = str(name or "").strip().lower() + if not key: + continue + existing = plugin_items.get(key) + if existing is None: + plugin_items[key] = { + "kind": "section", + "title": _format_config_label(name), + "browse_path": f"{category}.{name}", + "summary": _format_config_entry_count(value), + "type": "section", + "description": "Plugin configuration", + } + continue + + if str(category) == "plugin" and not str(existing.get("browse_path") or "").startswith("plugin."): + existing["browse_path"] = f"{category}.{name}" + try: + current_count = int(str(existing.get("summary") or "0").split()[0]) + except Exception: + current_count = 0 + extra_count = len(_visible_config_entries(value)) if isinstance(value, dict) else 0 + merged_count = current_count + extra_count + existing["summary"] = "1 entry" if merged_count == 1 else f"{merged_count} entries" + + return sorted(plugin_items.values(), key=lambda item: str(item.get("title") or "").lower()) + + +def _resolve_config_branch( + config_data: Dict[str, Any], + browse_path: Optional[str], +) -> Optional[Dict[str, Any]]: + text = str(browse_path or "").strip() + if not text: + return config_data if isinstance(config_data, dict) else None + + if text == _PREFERENCES_BROWSE_PATH: + return { + key: value + for key, value in _visible_config_entries(config_data) + if not isinstance(value, dict) + } + + if text == _PLUGINS_BROWSE_PATH: + return { + str(item.get("title") or ""): item + for item in _collect_plugin_root_items(config_data) + } + + current: Any = config_data + for part in text.split("."): + if not isinstance(current, dict): + return None + current = current.get(part) + return current if isinstance(current, dict) else None + + +def _build_section_item( + *, + title: str, + browse_path: str, + value: Any, + description: Optional[str] = None, +) -> Dict[str, Any]: + return { + "kind": "section", + "title": title, + "browse_path": browse_path, + "summary": _format_config_entry_count(value), + "type": "section", + "description": str(description or "").strip() or _KNOWN_SECTION_DESCRIPTIONS.get(browse_path, ""), + } + + +def _build_value_item( + *, + key_path: str, + name: str, + value: Any, +) -> Dict[str, Any]: + display_value = "***" if _is_sensitive_config_key(key_path) else _format_config_value(value) + path_parts = [part for part in str(key_path or "").split(".") if part] + display_path = " / ".join( + [_format_config_path_label(".".join(path_parts[:-1]))] if len(path_parts) > 1 else [] + + [_format_config_label(path_parts[-1])] if path_parts else [_format_config_label(name)] + ) + return { + "kind": "value", + "key": key_path, + "name": name, + "title": _format_config_label(name), + "value": value, + "value_display": display_value, + "display_path": display_path, + "type": type(value).__name__, + } + + +def _build_root_config_items(config_data: Dict[str, Any]) -> List[Dict[str, Any]]: + items: List[Dict[str, Any]] = [] + visible_entries = _visible_config_entries(config_data) + + preferences = { + key: value + for key, value in visible_entries + if not isinstance(value, dict) + } + if preferences: + items.append( + _build_section_item( + title="Preferences", + browse_path=_PREFERENCES_BROWSE_PATH, + value=preferences, + ) + ) + + plugin_items = _collect_plugin_root_items(config_data) + if plugin_items: + items.append( + _build_section_item( + title="Plugins", + browse_path=_PLUGINS_BROWSE_PATH, + value={item["title"]: item for item in plugin_items}, + ) + ) + + other_sections: List[Dict[str, Any]] = [] + for key, value in visible_entries: + if key in set(_PLUGIN_CATEGORY_KEYS) or not isinstance(value, dict): + continue + other_sections.append( + _build_section_item( + title=_format_config_label(key), + browse_path=key, + value=value, + ) + ) + + other_sections.sort(key=lambda item: str(item.get("title") or "").lower()) + items.extend(other_sections) + return items + + +def _build_nested_config_items( + config_data: Dict[str, Any], + browse_path: str, +) -> List[Dict[str, Any]]: + if browse_path == _PLUGINS_BROWSE_PATH: + return _collect_plugin_root_items(config_data) + + branch = _resolve_config_branch(config_data, browse_path) + if branch is None: + return [] + + section_items: List[Dict[str, Any]] = [] + value_items: List[Dict[str, Any]] = [] + is_preferences_view = browse_path == _PREFERENCES_BROWSE_PATH + + for key, value in _visible_config_entries(branch): + full_key = key if is_preferences_view else f"{browse_path}.{key}" + if isinstance(value, dict): + section_items.append( + _build_section_item( + title=_format_config_label(key), + browse_path=full_key, + value=value, + ) + ) + else: + value_items.append( + _build_value_item( + key_path=full_key, + name=key, + value=value, + ) + ) + + section_items.sort(key=lambda item: str(item.get("title") or "").lower()) + value_items.sort(key=lambda item: str(item.get("name") or "").lower()) + return section_items + value_items + + +def _build_config_items( + config_data: Dict[str, Any], + browse_path: Optional[str] = None, +) -> List[Dict[str, Any]]: + text = str(browse_path or "").strip() + if not text: + return _build_root_config_items(config_data) + return _build_nested_config_items(config_data, text) + + +def _build_config_table_title(browse_path: Optional[str]) -> str: + text = str(browse_path or "").strip() + if not text: + return "Configuration" + return f"Configuration: {_format_config_path_label(text)}" + + +def _build_config_header_lines(browse_path: Optional[str]) -> List[str]: + text = str(browse_path or "").strip() + if not text: + return [ + "Use @N on a section to drill in. Use @.. to go back.", + ] + return [ + f"Path: {_format_config_path_label(text)}", + "Use @N on a section to drill in. Use @N | .config to update a setting. Use @.. to go back.", + ] + + +def _extract_browse_arg(args: Sequence[str]) -> Optional[str]: + return _extract_arg_value(args, flags={"-browse", "--browse"}, allow_positional=False) + + +def _extract_selected_update_value(args: Sequence[str]) -> Optional[str]: + explicit = _extract_arg_value(args, flags=VALUE_ARG_FLAGS, allow_positional=False) + if explicit is not None: + return explicit + + tokens = [str(arg).strip() for arg in (args or []) if str(arg).strip()] + positional = [token for token in tokens if not token.startswith("-")] + if len(positional) == 1: + return positional[0] + return None + + +def _get_selected_config_item() -> Optional[Dict[str, Any]]: try: indices = ctx.get_last_selection() or [] except Exception: @@ -211,32 +531,83 @@ def _get_selected_config_key() -> Optional[str]: return None item = items[idx] if isinstance(item, dict): - return item.get("key") - return getattr(item, "key", None) + return item + + normalized: Dict[str, Any] = {} + for key in ("kind", "key", "title", "browse_path", "name", "value", "value_display", "type"): + try: + value = getattr(item, key, None) + except Exception: + value = None + if value is not None: + normalized[key] = value + return normalized or None -def _show_config_table(config_data: Dict[str, Any]) -> int: - items = flatten_config(config_data) +def _show_config_table( + config_data: Dict[str, Any], + *, + browse_path: Optional[str] = None, +) -> int: + items = _build_config_items(config_data, browse_path=browse_path) if not items: - print("No configuration entries available.") + path_text = _format_config_path_label(browse_path) + print(f"No configuration entries available for {path_text}.") return 0 - items.sort(key=lambda x: x.get("key")) - table = Table("Configuration") + table = Table(_build_config_table_title(browse_path), preserve_order=True) table.set_table("config") - table.set_source_command(".config", []) + if browse_path: + table.set_source_command(".config", ["-browse", str(browse_path)]) + else: + table.set_source_command(".config", []) + table.set_header_lines(_build_config_header_lines(browse_path)) - for item in items: + for idx, item in enumerate(items): row = table.add_row() - row.add_column("Key", item.get("key", "")) - row.add_column("Value", item.get("value_display", "")) + row.add_column("Name", item.get("title", "")) + row.add_column("Value", item.get("summary") or item.get("value_display", "")) row.add_column("Type", item.get("type", "")) + if item.get("kind") == "section" and item.get("browse_path"): + table.set_row_selection_action( + idx, + [".config", "-browse", str(item.get("browse_path"))], + ) - ctx.set_last_result_table_overlay(table, items) + ctx.set_last_result_table(table, items) ctx.set_current_stage_table(table) return 0 +def _save_updated_config(config_data: Dict[str, Any], key_path: str) -> None: + try: + key_l = str(key_path or "").lower() + except Exception: + key_l = "" + if "alldebrid" in key_l or "all-debrid" in key_l: + save_config_and_verify(config_data) + return + save_config(config_data) + + +def _resolve_direct_browse_path( + config_data: Dict[str, Any], + token: str, +) -> Optional[str]: + text = str(token or "").strip() + if not text: + return None + lowered = text.lower() + if lowered in {"preferences", "prefs"}: + return _PREFERENCES_BROWSE_PATH + if lowered in {"plugins", "plugin", "providers", "provider", "tools", "tool"}: + return _PLUGINS_BROWSE_PATH + branch = _resolve_config_branch(config_data, text) + if isinstance(branch, dict): + return text + return None + + def _strip_value_quotes(value: str) -> str: if not value: return value @@ -254,41 +625,33 @@ def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int: # Load configuration from the database current_config = load_config() - selection_key = _get_selected_config_key() - value_from_args = _extract_value_arg(args) if selection_key else None - value_from_pipe = _extract_piped_value(piped_result) + browse_path = _extract_browse_arg(args) + if browse_path: + return _show_config_table(current_config, browse_path=browse_path) - if selection_key: - new_value = value_from_pipe or value_from_args - if not new_value: - print( - "Provide a new value via pipe or argument: @N | .config " - ) - return 1 - new_value = _strip_value_quotes(new_value) - try: - set_nested_config(current_config, selection_key, new_value) - # For AllDebrid API changes, use the verified save path to ensure - # the new API key persisted to disk; otherwise fall back to normal save. + selection_item = _get_selected_config_item() + value_from_pipe = _extract_piped_value(piped_result) + selection_kind = str((selection_item or {}).get("kind") or "").strip().lower() + selection_key = str((selection_item or {}).get("key") or "").strip() or None + selection_browse_path = str((selection_item or {}).get("browse_path") or "").strip() or None + selection_display_path = str((selection_item or {}).get("display_path") or selection_key or "").strip() or selection_key + + if selection_kind == "section" and selection_browse_path and not args and value_from_pipe is None: + return _show_config_table(current_config, browse_path=selection_browse_path) + + if selection_kind == "value" and selection_key: + new_value = value_from_pipe or _extract_selected_update_value(args) + if new_value is not None: + new_value = _strip_value_quotes(new_value) try: - key_l = str(selection_key or "").lower() - except Exception: - key_l = "" - if "alldebrid" in key_l or "all-debrid" in key_l: - try: - save_config_and_verify(current_config) - except Exception as exc: - log(f"Configuration save verification failed for '{selection_key}': {exc}") - print(f"Error saving configuration (verification failed): {exc}") - return 1 - else: - save_config(current_config) - print(f"Updated '{selection_key}' to '{new_value}'") - return 0 - except Exception as exc: - log(f"Error updating config '{selection_key}': {exc}") - print(f"Error updating config: {exc}") - return 1 + set_nested_config(current_config, selection_key, new_value) + _save_updated_config(current_config, selection_key) + print(f"Updated '{selection_display_path}' to '{new_value}'") + return 0 + except Exception as exc: + log(f"Error updating config '{selection_key}': {exc}") + print(f"Error updating config: {exc}") + return 1 if not args: if sys.stdin.isatty() and not piped_result: @@ -300,13 +663,16 @@ def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int: key = args[0] if len(args) < 2: + browse_target = _resolve_direct_browse_path(current_config, key) + if browse_target: + return _show_config_table(current_config, browse_path=browse_target) print(f"Error: Value required for key '{key}'") return 1 value = _strip_value_quotes(" ".join(args[1:])) try: set_nested_config(current_config, key, value) - save_config(current_config) + _save_updated_config(current_config, key) print(f"Updated '{key}' to '{value}'") return 0 except Exception as exc: diff --git a/plugins/alldebrid/alldebrid.json b/plugins/alldebrid/alldebrid.json index 17957dd..a09cadc 100644 --- a/plugins/alldebrid/alldebrid.json +++ b/plugins/alldebrid/alldebrid.json @@ -92,7 +92,7 @@ "(hitfile\\.net/[a-z0-9A-Z]{4,9})" ], "regexp": "(hitf\\.(to|cc)/([a-z0-9A-Z]{4,9}))|(htfl\\.(net|to|cc)/([a-z0-9A-Z]{4,9}))|(hitfile\\.(net)/download/free/([a-z0-9A-Z]{4,9}))|((hitfile\\.net/[a-z0-9A-Z]{4,9}))", - "status": true + "status": false }, "mega": { "name": "mega", @@ -463,7 +463,7 @@ "isra\\.cloud/\\?op=report_file&id=([0-9a-zA-Z]{12})" ], "regexp": "((isra\\.cloud/[0-9a-zA-Z]{12}))|(isra\\.cloud/\\?op=report_file&id=([0-9a-zA-Z]{12}))", - "status": true, + "status": false, "hardRedirect": [ "isra\\.cloud/([0-9a-zA-Z]{12})" ] @@ -478,11 +478,11 @@ "katfile.vip" ], "regexps": [ - "katfile\\.(cloud|online|vip|ws)/([0-9a-zA-Z]{12})", + "katfile\\.(cloud|online|vip|ws|space)/([0-9a-zA-Z]{12})", "(katfile\\.com/[0-9a-zA-Z]{12})" ], - "regexp": "(katfile\\.(cloud|online|vip|ws)/([0-9a-zA-Z]{12}))|((katfile\\.com/[0-9a-zA-Z]{12}))", - "status": true + "regexp": "(katfile\\.(cloud|online|vip|ws|space)/([0-9a-zA-Z]{12}))|((katfile\\.com/[0-9a-zA-Z]{12}))", + "status": false }, "mediafire": { "name": "mediafire", diff --git a/plugins/mpv/LUA/main.lua b/plugins/mpv/LUA/main.lua index a3bad49..98280d2 100644 --- a/plugins/mpv/LUA/main.lua +++ b/plugins/mpv/LUA/main.lua @@ -6535,6 +6535,7 @@ mp.register_script_message('medios-load-url-event', function(json) if not M._reset_uosc_input_state('load-url-submit') then _lua_log('[LOAD-URL] UOSC not loaded, cannot close menu') end + M._schedule_uosc_cursor_resync('load-url-submit') end -- Close the URL prompt immediately once the user submits. Playback may still @@ -6602,6 +6603,7 @@ mp.register_script_message('medios-load-url-event', function(json) _lua_log('[LOAD-URL] URL is yt-dlp compatible, prefetching formats in background') mp.add_timeout(0.5, function() _prefetch_formats_for_url(url) + M._schedule_uosc_cursor_resync('file-loaded-web') end) end end) diff --git a/plugins/mpv/commands.py b/plugins/mpv/commands.py index 4987c54..f653453 100644 --- a/plugins/mpv/commands.py +++ b/plugins/mpv/commands.py @@ -895,7 +895,7 @@ def _get_playlist(silent: bool = False) -> Optional[List[Dict[str, Any]]]: def _extract_title_from_item(item: Dict[str, Any]) -> str: """Extract a clean title from an MPV playlist item, handling memory:// M3U hacks.""" title = item.get("title") - filename = item.get("filename") or "" + filename = item.get("filename") or item.get("playlist-path") or "" # Special handling for memory:// M3U playlists (used to pass titles via IPC) if "memory://" in filename and "#EXTINF:" in filename: @@ -923,6 +923,163 @@ def _extract_title_from_item(item: Dict[str, Any]) -> str: return title or filename or "Unknown" +def _looks_like_raw_playlist_title( + title: Optional[str], + target: Optional[str], +) -> bool: + text = str(title or "").strip() + if not text or text == "Unknown": + return True + + target_text = str(target or "").strip() + if target_text and text == target_text: + return True + + lower = text.lower() + if lower.startswith(("http://", "https://", "hydrus://", "file://", "memory://")): + return True + if _WINDOWS_PATH_RE.match(text) or text.startswith("\\\\"): + return True + return False + + +def _resolve_hydrus_playlist_title( + target: str, + *, + store_name: Optional[str], + file_hash: Optional[str], + config: Optional[Dict[str, Any]], +) -> Optional[str]: + raw_target = str(target or "").strip() + if not raw_target: + return None + + resolved_store = str(store_name or "").strip() or None + resolved_hash = str(file_hash or "").strip().lower() or None + looks_hydrus = bool(resolved_store) or bool( + _SHA256_FULL_RE.fullmatch(raw_target.lower()) + ) or raw_target.lower().startswith("hydrus://") or _is_hydrus_path(raw_target, None) + if not looks_hydrus: + return None + + try: + hydrus_plugin = get_plugin("hydrusnetwork", config or {}) + except Exception: + hydrus_plugin = None + if hydrus_plugin is None: + return None + + try: + parsed_store, parsed_hash = hydrus_plugin.parse_hydrus_url(raw_target) + except Exception: + parsed_store, parsed_hash = None, "" + + if not resolved_store and parsed_store: + resolved_store = str(parsed_store).strip() or None + if not resolved_hash and parsed_hash: + resolved_hash = str(parsed_hash).strip().lower() or None + + if not resolved_store: + try: + inferred_store = hydrus_plugin.infer_playlist_store( + None, + target=raw_target, + file_storage=None, + ) + except Exception: + inferred_store = None + if inferred_store: + resolved_store = str(inferred_store).strip() or None + + if not resolved_hash: + try: + hashes = hydrus_plugin.find_hashes_by_url( + raw_target, + store_name=resolved_store, + ) + except TypeError: + try: + hashes = hydrus_plugin.find_hashes_by_url(raw_target) + except Exception: + hashes = [] + except Exception: + hashes = [] + if isinstance(hashes, list) and hashes: + resolved_hash = str(hashes[0] or "").strip().lower() or None + + if not resolved_hash: + return None + + try: + resolved_title = hydrus_plugin.get_title( + resolved_hash, + store_name=resolved_store, + ) + except Exception: + resolved_title = "" + + title_text = str(resolved_title or "").strip() + if not title_text: + return None + + if title_text.lower() in {resolved_hash, resolved_hash[:16] + "..."}: + return None + return title_text + + +def _resolve_playlist_display_title( + item: Dict[str, Any], + *, + config: Optional[Dict[str, Any]] = None, + file_storage: Optional[Any] = None, + store_name: Optional[str] = None, + file_hash: Optional[str] = None, + title_cache: Optional[Dict[tuple[str, str, str], Optional[str]]] = None, +) -> str: + title = _extract_title_from_item(item) + filename = item.get("filename") or item.get("playlist-path") or "" + real_path = _extract_target_from_memory_uri(filename) or filename + if not _looks_like_raw_playlist_title(title, real_path): + return title + + resolved_store = str(store_name or "").strip() or None + resolved_hash = str(file_hash or "").strip().lower() or None + if not resolved_store or not resolved_hash: + extracted_store, extracted_hash = _extract_store_and_hash( + { + "store": resolved_store, + "hash": resolved_hash, + "path": real_path, + "filename": filename, + "title": title, + }, + config=config, + ) + if not resolved_store and extracted_store: + resolved_store = str(extracted_store).strip() or None + if not resolved_hash and extracted_hash: + resolved_hash = str(extracted_hash).strip().lower() or None + + cache_key = ( + str(real_path or "").strip().lower(), + str(resolved_store or "").strip().lower(), + str(resolved_hash or "").strip().lower(), + ) + if title_cache is not None and cache_key in title_cache: + cached_title = title_cache[cache_key] + return cached_title or title + + resolved_title = _resolve_hydrus_playlist_title( + real_path, + store_name=resolved_store, + file_hash=resolved_hash, + config=config, + ) + if title_cache is not None: + title_cache[cache_key] = resolved_title + return resolved_title or title + + def _extract_target_from_memory_uri(text: str) -> Optional[str]: """Extract the real target URL/path from a memory:// M3U payload.""" if not isinstance(text, str) or not text.startswith("memory://"): @@ -1927,7 +2084,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: return 1 # Build result object with file info - title = _extract_title_from_item(current_item) + title = _resolve_playlist_display_title(current_item, config=config) filename = current_item.get("filename", "") # Emit the current item to pipeline @@ -2340,7 +2497,11 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: return 1 item = items[idx] - title = _extract_title_from_item(item) + title = _resolve_playlist_display_title( + item, + config=config, + file_storage=file_storage, + ) filename = item.get("filename", "") if isinstance(item, dict) else "" hydrus_header = _build_hydrus_header(config or {}) hydrus_url = None @@ -2446,9 +2607,9 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: # Convert MPV items to PipeObjects with proper hash and store pipe_objects = [] + title_cache: Dict[tuple[str, str, str], Optional[str]] = {} for i, item in enumerate(items): is_current = item.get("current", False) - title = _extract_title_from_item(item) filename = item.get("filename", "") # Extract the real path/URL from memory:// wrapper if present @@ -2458,7 +2619,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: store_name, file_hash = _extract_store_and_hash( { "path": real_path, - "title": title, + "filename": filename, }, config=config, ) @@ -2480,6 +2641,15 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: config=config, ) + title = _resolve_playlist_display_title( + item, + config=config, + file_storage=file_storage, + store_name=store_name, + file_hash=file_hash, + title_cache=title_cache, + ) + # Build PipeObject with proper metadata pipe_obj = PipeObject( hash=file_hash or "unknown", diff --git a/plugins/ytdlp/__init__.py b/plugins/ytdlp/__init__.py index 2f30941..2b2b8f3 100644 --- a/plugins/ytdlp/__init__.py +++ b/plugins/ytdlp/__init__.py @@ -544,6 +544,81 @@ class ytdlp(TableProviderMixin, Provider): } AUTO_STAGE_USE_SELECTION_ARGS = True + @staticmethod + def _playlist_entry_to_url(entry: Any, *, extractor_name: str) -> Optional[str]: + if not isinstance(entry, dict): + return None + for key in ("webpage_url", "original_url", "url"): + value = entry.get(key) + if isinstance(value, str) and value.strip(): + cleaned = value.strip() + try: + if urlparse(cleaned).scheme in {"http", "https"}: + return cleaned + except Exception: + return cleaned + entry_id = entry.get("id") + if isinstance(entry_id, str) and entry_id.strip() and "youtube" in extractor_name: + return f"https://www.youtube.com/watch?v={entry_id.strip()}" + return None + + def resolve_preflight_items(self, url: str, **kwargs: Any) -> Optional[List[Dict[str, Any]]]: + url_str = str(url or "").strip() + if not url_str or not is_url_supported_by_ytdlp(url_str): + return None + + parsed = kwargs.get("parsed") if isinstance(kwargs.get("parsed"), dict) else {} + query_spec = parsed.get("query") + query_keyed = _parse_query_keyed_spec(str(query_spec) if query_spec is not None else None) + + playlist_items = str(parsed.get("item")) if parsed.get("item") else None + item_values: List[str] = [] + if isinstance(query_keyed, dict): + item_values.extend(query_keyed.get("item", []) or []) + if item_values and not playlist_items: + playlist_items = ",".join([value for value in item_values if value]) + + ytdlp_tool = YtDlpTool(self.config) + try: + probe = probe_url( + url_str, + no_playlist=False, + playlist_items=playlist_items, + timeout_seconds=15, + cookiefile=_cookiefile_str(ytdlp_tool), + ) + except Exception: + probe = None + + if not isinstance(probe, dict): + return None + + entries = probe.get("entries") + if not isinstance(entries, list) or not entries: + return None + + extractor_name = str(probe.get("extractor") or probe.get("extractor_key") or "").strip().lower() + items: List[Dict[str, Any]] = [] + for idx, entry in enumerate(entries, 1): + entry_url = self._playlist_entry_to_url(entry, extractor_name=extractor_name) + if not entry_url: + continue + playlist_index = None + if isinstance(entry, dict): + playlist_index = entry.get("playlist_index") + try: + playlist_index_value = int(playlist_index) + except Exception: + playlist_index_value = idx + items.append( + { + "url": entry_url, + "playlist_index": playlist_index_value, + } + ) + + return items or None + def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]: normalized_query, inline_args = parse_inline_query_arguments(query) search_parts: List[str] = [] @@ -745,23 +820,6 @@ class ytdlp(TableProviderMixin, Provider): elif "youtube" in extractor_name: table_type = "youtube" - def _entry_to_url(entry: Any) -> Optional[str]: - if not isinstance(entry, dict): - return None - for key in ("webpage_url", "original_url", "url"): - value = entry.get(key) - if isinstance(value, str) and value.strip(): - cleaned = value.strip() - try: - if urlparse(cleaned).scheme in {"http", "https"}: - return cleaned - except Exception: - return cleaned - entry_id = entry.get("id") - if isinstance(entry_id, str) and entry_id.strip() and "youtube" in extractor_name: - return f"https://www.youtube.com/watch?v={entry_id.strip()}" - return None - table = Table(preserve_order=True) safe_url = str(url or "").strip() table.title = f'download-file -url "{safe_url}"' if safe_url else "download-file" @@ -781,7 +839,7 @@ class ytdlp(TableProviderMixin, Provider): title = entry.get("title") if isinstance(entry, dict) else None uploader = entry.get("uploader") if isinstance(entry, dict) else None duration = entry.get("duration") if isinstance(entry, dict) else None - entry_url = _entry_to_url(entry) + entry_url = self._playlist_entry_to_url(entry, extractor_name=extractor_name) row = build_table_result_payload( table="download-file", title=str(title or f"Item {idx}"), diff --git a/tool/ytdlp.py b/tool/ytdlp.py index 9a28b74..958a1c6 100644 --- a/tool/ytdlp.py +++ b/tool/ytdlp.py @@ -520,6 +520,7 @@ def probe_url( timeout_seconds: int = 15, *, cookiefile: Optional[str] = None, + playlist_items: Optional[str] = None, ) -> Optional[Dict[str, Any]]: """Probe URL metadata without downloading. @@ -529,8 +530,12 @@ def probe_url( if not is_url_supported_by_ytdlp(url): return None + playlist_items_key = str(playlist_items or "").strip() or None + # Simple in-memory cache to avoid duplicate probes for the same URL/options in a short window. - cache_key = hashlib.md5(f"{url}|{no_playlist}|{cookiefile}".encode()).hexdigest() + cache_key = hashlib.md5( + f"{url}|{no_playlist}|{cookiefile}|{playlist_items_key or ''}".encode() + ).hexdigest() now = time.monotonic() if cache_key in _PROBE_CACHE: ts, result = _PROBE_CACHE[cache_key] @@ -562,6 +567,8 @@ def probe_url( if no_playlist: ydl_opts["noplaylist"] = True + elif playlist_items_key: + ydl_opts["playlist_items"] = playlist_items_key with yt_dlp.YoutubeDL(ydl_opts) as ydl: # type: ignore[arg-type] info = ydl.extract_info(url, download=False)