From 343a7b37a0e49f954246c2c8777f69edf1da3bc1 Mon Sep 17 00:00:00 2001 From: Nose Date: Thu, 16 Apr 2026 17:18:50 -0700 Subject: [PATCH] updated panel display --- API/HTTP.py | 20 +---- API/data/alldebrid.json | 8 +- Provider/alldebrid.py | 12 ++- SYS/config.py | 6 ++ SYS/logger.py | 142 ++++++++++++++++++++++++++------ SYS/pipeline.py | 93 +++++++++++++++++---- SYS/result_table.py | 135 +++++++++++++++++++++++++----- SYS/result_table_renderers.py | 18 ++-- Store/HydrusNetwork.py | 150 +++++++++++++++++++++++++--------- cmdlet/_shared.py | 90 +++++++++++++++----- cmdlet/add_file.py | 100 +++++++++++++++-------- cmdlet/download_file.py | 120 ++++++++++++++++++--------- cmdlet/search_file.py | 8 -- tool/ytdlp.py | 73 +++++++++-------- 14 files changed, 711 insertions(+), 264 deletions(-) diff --git a/API/HTTP.py b/API/HTTP.py index af2de2f..14b79c1 100644 --- a/API/HTTP.py +++ b/API/HTTP.py @@ -21,7 +21,7 @@ from pathlib import Path from urllib.parse import unquote, urlparse, parse_qs import logging -from SYS.logger import debug, is_debug_enabled, log +from SYS.logger import debug, debug_panel, is_debug_enabled, log from SYS.models import DebugLogger, DownloadError, DownloadMediaResult, ProgressBar from SYS.utils import ensure_directory, sha256_file @@ -80,23 +80,7 @@ class HTTPClient: def _debug_panel(self, title: str, rows: List[tuple[str, Any]]) -> None: if not is_debug_enabled(): return - try: - from rich.table import Table as RichTable - from rich.panel import Panel - - grid = RichTable.grid(padding=(0, 1)) - grid.add_column("Key", style="cyan", no_wrap=True) - grid.add_column("Value") - for key, val in rows: - try: - grid.add_row(str(key), str(val)) - except Exception: - grid.add_row(str(key), "") - - debug(Panel(grid, title=title, expand=False)) - except Exception: - # Fallback to simple debug output - debug(title, rows) + debug_panel(title, rows, border_style="bright_blue") def __enter__(self): """Context manager entry.""" diff --git a/API/data/alldebrid.json b/API/data/alldebrid.json index 66708bf..e4862a8 100644 --- a/API/data/alldebrid.json +++ b/API/data/alldebrid.json @@ -425,7 +425,7 @@ "(hexupload\\.net|hexload\\.com)/([a-zA-Z0-9]{12})" ], "regexp": "(hexupload\\.net|hexload\\.com)/([a-zA-Z0-9]{12})", - "status": true + "status": false }, "hot4share": { "name": "hot4share", @@ -482,7 +482,7 @@ "(katfile\\.com/[0-9a-zA-Z]{12})" ], "regexp": "(katfile\\.(cloud|online|vip)/([0-9a-zA-Z]{12}))|((katfile\\.com/[0-9a-zA-Z]{12}))", - "status": true + "status": false }, "mediafire": { "name": "mediafire", @@ -595,7 +595,7 @@ "(simfileshare\\.net/download/[0-9]+/)" ], "regexp": "(simfileshare\\.net/download/[0-9]+/)", - "status": true + "status": false }, "streamtape": { "name": "streamtape", @@ -690,7 +690,7 @@ "uploadrar\\.(net|com)/([0-9a-z]{12})" ], "regexp": "((get|cloud)\\.rahim-soft\\.com/([0-9a-z]{12}))|((fingau\\.com/([0-9a-z]{12})))|((tech|miui|cloud|flash)\\.getpczone\\.com/([0-9a-z]{12}))|(miui.rahim-soft\\.com/([0-9a-z]{12}))|(uploadrar\\.(net|com)/([0-9a-z]{12}))", - "status": true, + "status": false, "hardRedirect": [ "uploadrar.com/([0-9a-zA-Z]{12})" ] diff --git a/Provider/alldebrid.py b/Provider/alldebrid.py index 5d00e6d..83cae0c 100644 --- a/Provider/alldebrid.py +++ b/Provider/alldebrid.py @@ -17,7 +17,7 @@ from ProviderCore.base import Provider, SearchResult from SYS.provider_helpers import TableProviderMixin from SYS.item_accessors import get_field as _extract_value from SYS.utils import sanitize_filename -from SYS.logger import log, debug +from SYS.logger import log, debug, debug_panel from SYS.models import DownloadError, PipeObject _HOSTS_CACHE_TTL_SECONDS = 24 * 60 * 60 @@ -757,9 +757,13 @@ class AllDebrid(TableProviderMixin, Provider): dom = str(d or "").strip().lower() if dom and dom not in patterns: patterns.append(dom) - log( - f"[alldebrid] url_patterns loaded {len(cached)} cached host domains; total patterns={len(patterns)}", - file=sys.stderr, + debug_panel( + "AllDebrid host cache", + [ + ("cached_domains", len(cached)), + ("total_patterns", len(patterns)), + ], + border_style="magenta", ) except Exception: pass diff --git a/SYS/config.py b/SYS/config.py index 6a5c007..2152f2b 100644 --- a/SYS/config.py +++ b/SYS/config.py @@ -51,6 +51,12 @@ def global_config() -> List[Dict[str, Any]]: "label": "Auto-Update", "default": "true", "choices": ["true", "false"] + }, + { + "key": "table_appearance", + "label": "Table Appearance", + "default": "rainbow", + "choices": ["plain", "bw-striped", "rainbow"] } ] diff --git a/SYS/logger.py b/SYS/logger.py index 56aad4e..623d96c 100644 --- a/SYS/logger.py +++ b/SYS/logger.py @@ -5,7 +5,7 @@ import inspect import logging import threading from pathlib import Path -from typing import Optional +from typing import Any, Optional, Sequence from SYS.rich_display import console_for @@ -43,6 +43,106 @@ def is_debug_enabled() -> bool: return _DEBUG_ENABLED +def _debug_output_suppressed() -> bool: + try: + stderr_name = getattr(sys.stderr, "name", "") + return "nul" in str(stderr_name).lower() or "/dev/null" in str(stderr_name) + except Exception: + return False + + +def _debug_output_file(file=None): + stream = get_thread_stream() + if stream is not None: + return stream + if file is not None: + return file + return sys.stderr + + +def _is_rich_renderable(value: Any) -> bool: + if value is None: + return False + if isinstance(value, (str, bytes, bytearray)): + return False + return bool( + hasattr(value, "__rich_console__") + or hasattr(value, "__rich__") + or value.__class__.__module__.startswith("rich.") + ) + + +def _caller_location(depth: int = 1) -> tuple[str, str]: + frame = inspect.currentframe() + current = frame + try: + for _ in range(max(0, int(depth))): + if current is None: + break + current = current.f_back + + if current is None: + return "", "" + + return Path(current.f_code.co_filename).stem, current.f_code.co_name + finally: + del frame + + +def _debug_db_log(*, caller_name: str, message: str, level: str = "DEBUG") -> None: + if not _DB_LOGGER: + return + try: + _DB_LOGGER(level, caller_name, message) + except Exception: + pass + + +def debug_panel( + title: str, + rows: Sequence[tuple[str, Any]], + *, + file=None, + border_style: str = "cyan", +) -> None: + """Render a compact key/value debug panel when debug logging is enabled.""" + if not _DEBUG_ENABLED or _debug_output_suppressed(): + return + + target_file = _debug_output_file(file) + try: + from rich.panel import Panel + from rich.table import Table as RichTable + + grid = RichTable.grid(padding=(0, 1)) + grid.add_column("Key", style="cyan", no_wrap=True) + grid.add_column("Value") + for key, val in rows: + try: + grid.add_row(str(key), str(val)) + except Exception: + grid.add_row(str(key), "") + + debug( + Panel( + grid, + title=str(title or "Debug"), + expand=False, + border_style=border_style, + ), + file=target_file, + ) + + file_name, func_name = _caller_location(depth=2) + caller_name = f"{file_name}.{func_name}" if file_name and func_name else "" + _debug_db_log( + caller_name=caller_name, + message=f"[{title}] " + "; ".join(f"{k}={v}" for k, v in rows), + ) + except Exception: + debug(title, list(rows), file=target_file) + + def debug(*args, **kwargs) -> None: """Print debug message if debug logging is enabled. @@ -53,26 +153,24 @@ def debug(*args, **kwargs) -> None: # Check if stderr has been redirected to /dev/null (quiet mode) # If so, skip output to avoid queuing in background worker's capture - try: - stderr_name = getattr(sys.stderr, "name", "") - if "nul" in str(stderr_name).lower() or "/dev/null" in str(stderr_name): - return - except Exception: - pass + if _debug_output_suppressed(): + return - # Check for thread-local stream first - stream = get_thread_stream() - if stream: - kwargs["file"] = stream - # Set default to stderr for debug messages - elif "file" not in kwargs: - kwargs["file"] = sys.stderr + target_file = _debug_output_file(kwargs.pop("file", None)) + + if len(args) == 1 and _is_rich_renderable(args[0]): + renderable = args[0] + console_for(target_file).print(renderable) + file_name, func_name = _caller_location(depth=1) + caller_name = f"{file_name}.{func_name}" if file_name and func_name else "" + _debug_db_log(caller_name=caller_name, message=f"") + return # Prepend DEBUG label args = ("DEBUG:", *args) # Use the same logic as log() - log(*args, **kwargs) + log(*args, file=target_file, **kwargs) def debug_inspect( @@ -97,19 +195,11 @@ def debug_inspect( return # Mirror debug() quiet-mode guard. - try: - stderr_name = getattr(sys.stderr, "name", "") - if "nul" in str(stderr_name).lower() or "/dev/null" in str(stderr_name): - return - except Exception: - pass + if _debug_output_suppressed(): + return # Resolve destination stream. - stream = get_thread_stream() - if stream is not None: - file = stream - elif file is None: - file = sys.stderr + file = _debug_output_file(file) # Compute caller prefix (same as log()). prefix = None diff --git a/SYS/pipeline.py b/SYS/pipeline.py index 67fb031..545edbd 100644 --- a/SYS/pipeline.py +++ b/SYS/pipeline.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field from contextvars import ContextVar from typing import Any, Dict, List, Optional, Sequence, Callable from SYS.models import PipelineStageContext -from SYS.logger import log, debug, is_debug_enabled +from SYS.logger import log, debug, debug_panel, is_debug_enabled import logging logger = logging.getLogger(__name__) from SYS.worker import WorkerManagerRegistry, WorkerStages @@ -28,6 +28,47 @@ HELP_EXAMPLE_SOURCE_COMMANDS = { } +def _emit_selection_debug_panel( + *, + selection_token: Any, + selection_indices: Sequence[int], + item_count: int, + filtered_count: int, + stage_table_present: bool, + display_table_present: bool, + stage_is_last: bool, + row_action: Optional[Sequence[Any]] = None, + downstream_stages: Optional[Sequence[Sequence[Any]]] = None, + mode: Optional[str] = None, +) -> None: + if not is_debug_enabled(): + return + + try: + rows: List[tuple[str, Any]] = [ + ("selection", str(selection_token or "")), + ("indices", [int(idx) + 1 for idx in (selection_indices or [])]), + ("items", int(item_count)), + ("filtered", int(filtered_count)), + ("stage_table", bool(stage_table_present)), + ("display_table", bool(display_table_present)), + ("stage_is_last", bool(stage_is_last)), + ("downstream_stages", len(list(downstream_stages or []))), + ] + if mode: + rows.insert(1, ("mode", str(mode))) + if row_action: + rows.append(("row_action", " ".join(str(part) for part in row_action if part is not None))) + + debug_panel( + f"Selection replay {selection_token}", + rows, + border_style="magenta", + ) + except Exception: + pass + + def set_live_progress(progress_ui: Any) -> None: """Register the current Live progress UI so cmdlets can suspend it during prompts.""" state = _get_pipeline_state() @@ -1883,6 +1924,7 @@ class PipelineExecutor: selected_row_args: List[str] = [] skip_pipe_expansion = source_cmd in {".pipe", ".mpv"} and len(stages) > 0 prefer_row_action = False + preferred_row_action = None if len(selection_indices) == 1 and not stages: try: row_action = _get_row_action(selection_indices[0]) @@ -1890,10 +1932,7 @@ class PipelineExecutor: row_action = None if row_action: prefer_row_action = True - debug( - "@N: skipping source command expansion because row has explicit selection_action " - f"{row_action}" - ) + preferred_row_action = list(row_action) # Command expansion via @N: # - Default behavior: expand ONLY for single-row selections. # - Special case: allow multi-row expansion for add-file directory tables by @@ -1978,7 +2017,7 @@ class PipelineExecutor: except Exception: logger.exception("Failed to record pipeline log step for @N expansion (pipeline_session=%r)", getattr(pipeline_session, 'worker_id', None)) elif selected_row_args and stages: - debug("@N: skipping source command expansion because downstream stages exist") + pass stage_table = None try: @@ -2003,8 +2042,6 @@ class PipelineExecutor: except Exception: stage_table = None - debug(f"@N: stage_table={stage_table is not None}, display_table={display_table is not None}") - # ==================================================================== # PHASE 4: Retrieve and filter items from current result set # ==================================================================== @@ -2015,14 +2052,31 @@ class PipelineExecutor: except Exception as exc: debug(f"@N: Exception getting items_list: {exc}") items_list = [] - - debug(f"@N: selection_indices={selection_indices}, items_list length={len(items_list)}") resolved_items = items_list if items_list else [] if items_list: filtered = [ resolved_items[i] for i in selection_indices if 0 <= i < len(resolved_items) ] + if selection_indices: + if len(selection_indices) == 1: + selection_label = f"@{selection_indices[0] + 1}" + else: + selection_label = "@{" + ",".join(str(idx + 1) for idx in selection_indices) + "}" + else: + selection_label = "@selection" + _emit_selection_debug_panel( + selection_token=selection_label, + selection_indices=selection_indices, + item_count=len(items_list), + filtered_count=len(filtered), + stage_table_present=(stage_table is not None), + display_table_present=(display_table is not None), + stage_is_last=(not stages), + row_action=preferred_row_action, + downstream_stages=stages, + mode=("row_action" if preferred_row_action else "selection"), + ) if not filtered: print("No items matched selection in pipeline\n") return False, None @@ -2088,15 +2142,12 @@ class PipelineExecutor: filtered = track_items table_type_hint = "tidal.track" - debug(f"@N: calling _maybe_run_class_selector with filtered={len(filtered)} items, stage_is_last={not stages}") if PipelineExecutor._maybe_run_class_selector( ctx, config, filtered, stage_is_last=(not stages)): - debug(f"@N: _maybe_run_class_selector returned True, returning False") return False, None - debug(f"@N: _maybe_run_class_selector returned False, continuing") from SYS.pipe_object import coerce_to_pipe_object @@ -2105,7 +2156,6 @@ class PipelineExecutor: filtered_pipe_objs if len(filtered_pipe_objs) > 1 else filtered_pipe_objs[0] ) - debug(f"@N: coerced piped_result, stages={stages}") if pipeline_session and worker_manager: try: @@ -2204,7 +2254,6 @@ class PipelineExecutor: print("Auto-applying metadata selection via get-tag") stages.append(["get-tag"]) elif auto_stage: - debug(f"@N: Found auto_stage={auto_stage}, appending") try: print(f"Auto-running selection via {auto_stage[0]}") except Exception: @@ -2238,7 +2287,6 @@ class PipelineExecutor: if not stages and selection_indices and len(selection_indices) == 1: row_action = _get_row_action(selection_indices[0], items_list) if row_action: - debug(f"@N: applying row_action {row_action}") stages.append(row_action) if pipeline_session and worker_manager: try: @@ -2464,7 +2512,18 @@ class PipelineExecutor: ctx = sys.modules[__name__] try: - debug(f"execute_tokens: tokens={tokens}") + try: + from SYS.logger import debug_panel + + debug_panel( + "Pipeline execution", + [ + ("command", " ".join(str(tok) for tok in tokens)), + ("token_count", len(tokens)), + ], + ) + except Exception: + debug(f"execute_tokens: tokens={tokens}") self._try_clear_pipeline_stop(ctx) # REPL guard: stage-local tables should not persist across independent diff --git a/SYS/result_table.py b/SYS/result_table.py index 6d42579..977ad9f 100644 --- a/SYS/result_table.py +++ b/SYS/result_table.py @@ -16,6 +16,7 @@ from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, Callable, Set from pathlib import Path import json +import re from rich.box import SIMPLE from rich.console import Group @@ -113,14 +114,97 @@ _RESULT_TABLE_ROW_STYLE_LOOP: List[tuple[str, str]] = [ ("#ffffff", "#000000"), ] +_RESULT_TABLE_BW_ROW_STYLE_LOOP: List[tuple[str, str]] = [ + ("#000000", "#ffffff"), + ("#ffffff", "#000000"), +] + RESULT_TABLE_HEADER_STYLE = "bold #000000 on #ffffff" RESULT_TABLE_BORDER_STYLE = "#000000 on #ffffff" +RESULT_TABLE_PLAIN_HEADER_STYLE = "bold" + +_RESULT_TABLE_APPEARANCE_ALIASES: Dict[str, str] = { + "": "rainbow", + "rainbow": "rainbow", + "default": "rainbow", + "plain": "plain", + "none": "plain", + "bw": "bw-striped", + "b-w": "bw-striped", + "b-w-striped": "bw-striped", + "bw-striped": "bw-striped", + "b-w-stripes": "bw-striped", + "bw-stripes": "bw-striped", + "black-white": "bw-striped", + "black-white-striped": "bw-striped", + "black-white-stripes": "bw-striped", + "black-and-white": "bw-striped", + "black-and-white-striped": "bw-striped", + "black-and-white-stripes": "bw-striped", +} -def get_result_table_row_style(row_index: int) -> str: - text_color, bg_color = _RESULT_TABLE_ROW_STYLE_LOOP[ - row_index % len(_RESULT_TABLE_ROW_STYLE_LOOP) - ] +def normalize_result_table_appearance_mode(value: Any) -> str: + text = str(value or "").strip().lower() + if not text: + return "rainbow" + + collapsed = re.sub(r"[^a-z0-9]+", "-", text).strip("-") + return _RESULT_TABLE_APPEARANCE_ALIASES.get(collapsed, "rainbow") + + +def get_result_table_appearance_mode(config: Optional[Dict[str, Any]] = None) -> str: + cfg = config + if cfg is None: + try: + from SYS.config import load_config + + cfg = load_config() or {} + except Exception: + cfg = {} + + raw_value = None + if isinstance(cfg, dict): + raw_value = cfg.get("table_appearance") + + return normalize_result_table_appearance_mode(raw_value) + + +def get_result_table_header_style(config: Optional[Dict[str, Any]] = None) -> str: + mode = get_result_table_appearance_mode(config) + if mode == "plain": + return RESULT_TABLE_PLAIN_HEADER_STYLE + return RESULT_TABLE_HEADER_STYLE + + +def get_result_table_border_style(config: Optional[Dict[str, Any]] = None) -> str: + mode = get_result_table_appearance_mode(config) + if mode == "plain": + return "" + return RESULT_TABLE_BORDER_STYLE + + +def get_result_table_panel_style(config: Optional[Dict[str, Any]] = None) -> str: + mode = get_result_table_appearance_mode(config) + if mode == "plain": + return "" + return "on #ffffff" + + +def get_result_table_row_style( + row_index: int, + appearance_mode: Optional[str] = None, + config: Optional[Dict[str, Any]] = None, +) -> str: + mode = appearance_mode or get_result_table_appearance_mode(config) + if mode == "plain": + return "" + + style_loop = ( + _RESULT_TABLE_BW_ROW_STYLE_LOOP + if mode == "bw-striped" else _RESULT_TABLE_ROW_STYLE_LOOP + ) + text_color, bg_color = style_loop[row_index % len(style_loop)] return f"{text_color} on {bg_color}" @@ -1435,16 +1519,21 @@ class Table: def to_rich(self): """Return a Rich renderable representing this table.""" + appearance_mode = get_result_table_appearance_mode() + header_style = get_result_table_header_style({"table_appearance": appearance_mode}) + border_style = get_result_table_border_style({"table_appearance": appearance_mode}) + panel_style = get_result_table_panel_style({"table_appearance": appearance_mode}) + if not self.rows: empty = Text("No results") return ( Panel( empty, - title=Text(str(self.title), style=RESULT_TABLE_HEADER_STYLE), - border_style=RESULT_TABLE_BORDER_STYLE, + title=Text(str(self.title), style=header_style), + border_style=border_style, padding=(0, 0), expand=False, - style="on #ffffff", + style=panel_style, ) if self.title else empty @@ -1460,8 +1549,8 @@ class Table: table = RichTable( show_header=True, - header_style=RESULT_TABLE_HEADER_STYLE, - border_style=RESULT_TABLE_BORDER_STYLE, + header_style=header_style, + border_style=border_style, box=None, expand=False, show_lines=False, @@ -1497,7 +1586,13 @@ class Table: for name in col_names: val = row.get_column(name) or "" cells.append(self._apply_value_case(_sanitize_cell_text(val))) - table.add_row(*cells, style=get_result_table_row_style(row_idx - 1)) + table.add_row( + *cells, + style=get_result_table_row_style( + row_idx - 1, + appearance_mode=appearance_mode, + ), + ) if self.title or self.header_lines: header_bits = [Text(line) for line in (self.header_lines or [])] @@ -1505,11 +1600,11 @@ class Table: return ( Panel( renderable, - title=Text(str(self.title), style=RESULT_TABLE_HEADER_STYLE), - border_style=RESULT_TABLE_BORDER_STYLE, + title=Text(str(self.title), style=header_style), + border_style=border_style, padding=(0, 0), expand=False, - style="on #ffffff", + style=panel_style, ) if self.title else renderable @@ -2385,11 +2480,13 @@ class ItemDetailView(Table): elements = [] if has_details: + header_style = get_result_table_header_style() + border_style = get_result_table_border_style() detail_title = str(self.detail_title or "Item Details").strip() or "Item Details" elements.append(Panel( details_table, - title=Text(detail_title, style=RESULT_TABLE_HEADER_STYLE), - border_style=RESULT_TABLE_BORDER_STYLE, + title=Text(detail_title, style=header_style), + border_style=border_style, padding=(1, 2) )) @@ -2397,11 +2494,11 @@ class ItemDetailView(Table): # If it's a Panel already (from super().to_rich() with title), use it directly # but force the border style to the result-table standard for consistency if isinstance(results_renderable, Panel): - results_renderable.border_style = RESULT_TABLE_BORDER_STYLE + results_renderable.border_style = get_result_table_border_style() if results_renderable.title: results_renderable.title = Text( str(results_renderable.title), - style=RESULT_TABLE_HEADER_STYLE, + style=get_result_table_header_style(), ) # Add a bit of padding inside if it contains a table elements.append(results_renderable) @@ -2416,8 +2513,8 @@ class ItemDetailView(Table): elements.append( Panel( results_group, - title=Text(str(display_title), style=RESULT_TABLE_HEADER_STYLE), - border_style=RESULT_TABLE_BORDER_STYLE, + title=Text(str(display_title), style=get_result_table_header_style()), + border_style=get_result_table_border_style(), ) ) diff --git a/SYS/result_table_renderers.py b/SYS/result_table_renderers.py index 2c45f9d..e547402 100644 --- a/SYS/result_table_renderers.py +++ b/SYS/result_table_renderers.py @@ -13,9 +13,10 @@ import logging logger = logging.getLogger(__name__) from SYS.result_table import ( - RESULT_TABLE_BORDER_STYLE, - RESULT_TABLE_HEADER_STYLE, apply_result_table_layout, + get_result_table_appearance_mode, + get_result_table_border_style, + get_result_table_header_style, get_result_table_row_style, ) from SYS.result_table_api import ColumnSpec, ResultModel, ResultTable, Renderer @@ -38,8 +39,8 @@ class RichRenderer(Renderer): table = RichTable( show_header=True, - header_style=RESULT_TABLE_HEADER_STYLE, - border_style=RESULT_TABLE_BORDER_STYLE, + header_style=get_result_table_header_style(), + border_style=get_result_table_border_style(), box=None, padding=(0, 1), pad_edge=False, @@ -47,6 +48,7 @@ class RichRenderer(Renderer): expand=False, ) apply_result_table_layout(table) + appearance_mode = get_result_table_appearance_mode() cols = list(columns) for col in cols: if str(col.header or "").strip().lower() == "tag": @@ -71,7 +73,13 @@ class RichRenderer(Renderer): logger.exception("Column extractor failed for '%s': %s", col.header, exc) cell = "" cells.append(cell) - table.add_row(*cells, style=get_result_table_row_style(row_idx)) + table.add_row( + *cells, + style=get_result_table_row_style( + row_idx, + appearance_mode=appearance_mode, + ), + ) return table diff --git a/Store/HydrusNetwork.py b/Store/HydrusNetwork.py index c3e789c..1d659e0 100644 --- a/Store/HydrusNetwork.py +++ b/Store/HydrusNetwork.py @@ -8,7 +8,7 @@ from collections import deque from pathlib import Path from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple -from urllib.parse import quote +from urllib.parse import quote, parse_qsl, urlencode, urlsplit, urlunsplit import httpx from API.httpx_shared import get_shared_httpx_client @@ -417,8 +417,6 @@ class HydrusNetwork(Store): if not file_hash: file_hash = sha256_file(file_path) - debug(f"{self._log_prefix()} file hash: {file_hash}") - # Use persistent client with session key client = self._client if client is None: @@ -564,7 +562,6 @@ class HydrusNetwork(Store): raise Exception(f"Hydrus response missing file hash: {response}") file_hash = hydrus_hash - debug(f"{self._log_prefix()} hash: {file_hash}") # Add tags if provided (both for new and existing files) if tag_list: @@ -575,13 +572,7 @@ class HydrusNetwork(Store): service_name = "my tags" try: - debug( - f"{self._log_prefix()} Adding {len(tag_list)} tag(s): {tag_list}" - ) client.add_tag(file_hash, tag_list, service_name) - debug( - f"{self._log_prefix()} Tags added via '{service_name}'" - ) except Exception as exc: log( f"{self._log_prefix()} ⚠️ Failed to add tags: {exc}", @@ -590,14 +581,10 @@ class HydrusNetwork(Store): # Associate url if provided (both for new and existing files) if url: - debug( - f"{self._log_prefix()} Associating {len(url)} URL(s) with file" - ) for url in url: if url: try: client.associate_url(file_hash, str(url)) - debug(f"{self._log_prefix()} Associated URL: {url}") except Exception as exc: log( f"{self._log_prefix()} ⚠️ Failed to associate URL {url}: {exc}", @@ -634,7 +621,6 @@ class HydrusNetwork(Store): raise Exception("Hydrus client unavailable") prefix = self._log_prefix() - debug(f"{prefix} Searching for: {query}") def _extract_urls(meta_obj: Any) -> list[str]: if not isinstance(meta_obj, dict): @@ -720,6 +706,70 @@ class HydrusNetwork(Store): ) -> list[dict[str, Any]]: """Best-effort URL search by scanning Hydrus metadata with include_file_url=True.""" + try: + from API.HydrusNetwork import _generate_hydrus_url_variants + except Exception: + _generate_hydrus_url_variants = None # type: ignore[assignment] + + def _normalize_url_match_token(value: str | None) -> str: + token = str(value or "").strip() + if not token: + return "" + + token = token.split("#", 1)[0] + try: + parsed_url = urlsplit(token) + except Exception: + return token.lower() + + if parsed_url.scheme and parsed_url.scheme.lower() not in {"http", "https"}: + return token.lower() + + netloc = str(parsed_url.netloc or "").strip().lower() + if netloc.startswith("www."): + netloc = netloc[4:] + + try: + query_pairs = parse_qsl(parsed_url.query, keep_blank_values=True) + except Exception: + query_pairs = [] + + filtered_pairs = [] + for key, val in query_pairs: + key_norm = str(key or "").lower() + if key_norm in {"t", "start", "time_continue", "timestamp", "time", "begin"}: + continue + if key_norm.startswith("utm_"): + continue + filtered_pairs.append((key, val)) + + normalized_query = urlencode(filtered_pairs, doseq=True) if filtered_pairs else "" + normalized = urlunsplit(("", netloc, parsed_url.path or "", normalized_query, "")) + return str(normalized or token).lstrip("/").lower() + + def _append_url_needles(output: list[str], candidate: str | None) -> None: + raw_candidate = str(candidate or "").strip() + if not raw_candidate: + return + + expanded_candidates = [raw_candidate] + if callable(_generate_hydrus_url_variants): + try: + expanded_candidates.extend(_generate_hydrus_url_variants(raw_candidate) or []) + except Exception: + pass + + for expanded in expanded_candidates: + expanded_text = str(expanded or "").strip() + if not expanded_text: + continue + lowered = expanded_text.lower() + if lowered and lowered not in output: + output.append(lowered) + normalized = _normalize_url_match_token(expanded_text) + if normalized and normalized not in output: + output.append(normalized) + candidate_file_ids: list[int] = [] candidate_hashes: list[str] = [] seen_file_ids: set[int] = set() @@ -775,13 +825,9 @@ class HydrusNetwork(Store): needle_list: list[str] = [] if isinstance(needles, (list, tuple, set)): for item in needles: - text = str(item or "").strip().lower() - if text and text not in needle_list: - needle_list.append(text) + _append_url_needles(needle_list, str(item or "")) if not needle_list: - needle = (url_value or "").strip().lower() - if needle: - needle_list = [needle] + _append_url_needles(needle_list, url_value) chunk_size = 200 out: list[dict[str, Any]] = [] if scan_limit is None: @@ -855,7 +901,14 @@ class HydrusNetwork(Store): continue if not needle_list: continue - if any(any(n in u.lower() for n in needle_list) for u in urls): + normalized_urls = [_normalize_url_match_token(u) for u in urls] + if any( + any( + n in str(u or "").lower() or (normalized_urls[idx] and n in normalized_urls[idx]) + for idx, u in enumerate(urls) + ) + for n in needle_list + ): out.append(meta) continue @@ -1001,7 +1054,8 @@ class HydrusNetwork(Store): return ids_out, hashes_out - query_lower = query.lower().strip() + raw_query = str(query or "").strip() + query_lower = raw_query.lower().strip() # Support `ext:` anywhere in the query. We filter results by the # Hydrus metadata extension field. @@ -1055,10 +1109,10 @@ class HydrusNetwork(Store): hashes: list[str] = [] file_ids: list[int] = [] - if ":" in query_lower and not query_lower.startswith(":"): - namespace, pattern = query_lower.split(":", 1) - namespace = namespace.strip().lower() - pattern = pattern.strip() + if ":" in raw_query and not raw_query.startswith(":"): + namespace_raw, pattern_raw = raw_query.split(":", 1) + namespace = namespace_raw.strip().lower() + pattern = pattern_raw.strip() if namespace == "url": try: fetch_limit_raw = int(limit) if limit else 100 @@ -1066,7 +1120,7 @@ class HydrusNetwork(Store): fetch_limit_raw = 100 if url_only: metadata_list = _search_url_query_metadata( - query_lower, + f"url:{pattern}", fetch_limit_raw, minimal=minimal, ) @@ -1092,12 +1146,13 @@ class HydrusNetwork(Store): token = str(value or "").strip().lower() if not token: return "" - return token.replace("*", "").replace("?", "") + return token.replace("*", "") # Fast-path: exact URL via /add_urls/get_url_files when a full URL is provided. exact_url_attempted = False try: - if pattern.startswith("http://") or pattern.startswith("https://"): + has_wildcards = ("*" in pattern) + if (pattern.startswith("http://") or pattern.startswith("https://")) and not has_wildcards: exact_url_attempted = True metadata_list = self.lookup_url_metadata(pattern, minimal=minimal) except Exception: @@ -1123,7 +1178,7 @@ class HydrusNetwork(Store): minimal=minimal, ) elif namespace == "system": - normalized_system_predicate = pattern.strip() + normalized_system_predicate = pattern.strip().lower() if normalized_system_predicate == "has url": try: fetch_limit = int(limit) if limit else 100 @@ -1173,7 +1228,6 @@ class HydrusNetwork(Store): if freeform_union_search: if not title_predicates and not freeform_predicates: - debug(f"{prefix} 0 result(s)") return [] payloads: list[Any] = [] @@ -1223,7 +1277,6 @@ class HydrusNetwork(Store): hashes = [] else: if not tags: - debug(f"{prefix} 0 result(s)") return [] search_result = client.search_files( @@ -1239,7 +1292,6 @@ class HydrusNetwork(Store): if ext_only and ext_filter: results = [] if not file_ids and not hashes: - debug(f"{prefix} 0 result(s)") return [] # Prefer file_ids if available. @@ -1301,14 +1353,11 @@ class HydrusNetwork(Store): "ext": _resolve_ext_from_meta(meta, mime_type), } ) - - debug(f"{prefix} {len(results)} result(s)") return results[:limit] # If we only got hashes, fall back to the normal flow below. if not file_ids and not hashes: - debug(f"{prefix} 0 result(s)") return [] file_ids, hashes = _cap_metadata_candidates( @@ -1457,8 +1506,6 @@ class HydrusNetwork(Store): "ext": ext, } ) - - debug(f"{prefix} {len(results)} result(s)") if ext_filter: wanted = ext_filter filtered: list[dict[str, Any]] = [] @@ -2110,6 +2157,31 @@ class HydrusNetwork(Store): seen_ids.add(file_id) file_ids.append(file_id) + statuses = response.get("url_file_statuses") + if isinstance(statuses, list): + for entry in statuses: + if not isinstance(entry, dict): + continue + + status_hash = entry.get("hash") or entry.get("file_hash") + try: + normalized_hash = str(status_hash or "").strip().lower() + except Exception: + normalized_hash = "" + if normalized_hash and normalized_hash not in seen_hashes: + seen_hashes.add(normalized_hash) + hashes.append(normalized_hash) + + status_id = entry.get("file_id") or entry.get("fileid") + try: + file_id = int(status_id) if status_id is not None else None + except (TypeError, ValueError): + file_id = None + if file_id is None or file_id in seen_ids: + continue + seen_ids.add(file_id) + file_ids.append(file_id) + for key in ("normalized_url", "redirect_url", "url"): value = response.get(key) if isinstance(value, str): diff --git a/cmdlet/_shared.py b/cmdlet/_shared.py index d6227c8..57dab32 100644 --- a/cmdlet/_shared.py +++ b/cmdlet/_shared.py @@ -14,7 +14,7 @@ from collections.abc import Iterable as IterableABC from functools import lru_cache from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse -from SYS.logger import log, debug +from SYS.logger import log, debug, debug_panel from pathlib import Path from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple from dataclasses import dataclass, field @@ -3158,6 +3158,8 @@ def check_url_exists_in_storage( storage: Any, hydrus_available: bool, final_output_dir: Optional[Path] = None, + *, + auto_continue_duplicates: bool = True, ) -> bool: """Pre-flight check to see if URLs already exist in storage. @@ -3187,7 +3189,6 @@ def check_url_exists_in_storage( in_pipeline = bool(stage_ctx is not None or ("|" in str(current_cmd_text or ""))) start_time = time.monotonic() time_budget = 45.0 - debug(f"[preflight] check_url_exists_in_storage: checking {len(urls)} url(s)") if in_pipeline: try: already_checked = bool( @@ -3243,7 +3244,7 @@ def check_url_exists_in_storage( return False return False - if in_pipeline: + if in_pipeline and auto_continue_duplicates: try: cached_cmd = pipeline_context.load_value("preflight.url_duplicates.command", default="") cached_decision = pipeline_context.load_value("preflight.url_duplicates.continue", default=None) @@ -3706,6 +3707,20 @@ def check_url_exists_in_storage( debug("Bulk URL preflight skipped: no searchable backends") return True + try: + debug_panel( + "URL preflight", + [ + ("url_count", len(unique_urls)), + ("pipeline", in_pipeline), + ("bulk_mode", bulk_mode), + ("backends", ", ".join(str(name) for name in backend_names)), + ], + border_style="yellow", + ) + except Exception: + pass + seen_pairs: set[tuple[str, str]] = set() matched_urls: set[str] = set() match_rows: List[Dict[str, Any]] = [] @@ -3726,12 +3741,7 @@ def check_url_exists_in_storage( except Exception: continue - debug(f"[preflight] Scanning backend: {backend_name}") - if HydrusNetwork is not None and isinstance(backend, HydrusNetwork): - client = getattr(backend, "_client", None) - if client is None: - continue if not hydrus_available: debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup") @@ -3748,7 +3758,33 @@ def check_url_exists_in_storage( found_hash: Optional[str] = None found = False + lookup_exact = getattr(backend, "find_hashes_by_url", None) + if callable(lookup_exact): + for needle in [original_url, *(needles or [])][:7]: + needle_text = str(needle or "").strip() + if not _httpish(needle_text): + continue + try: + exact_hashes = lookup_exact(needle_text) or [] + except Exception: + continue + if not isinstance(exact_hashes, list) or not exact_hashes: + continue + try: + found_hash = str(exact_hashes[0] or "").strip().lower() + except Exception: + found_hash = None + found = True + break + + client = getattr(backend, "_client", None) + if found: + pass + elif client is None: + continue for needle in (needles or [])[:6]: + if found: + break if not _httpish(needle): continue try: @@ -3868,7 +3904,6 @@ def check_url_exists_in_storage( match_rows.append(display_row) if not match_rows: - debug("Bulk URL preflight: no matches") if in_pipeline: preflight_cache = _load_preflight_cache() url_dup_cache = preflight_cache.get("url_duplicates") @@ -3935,24 +3970,39 @@ def check_url_exists_in_storage( auto_confirm_reason = "non-interactive stdin" answered_yes = True + auto_declined = False with cm: get_stderr_console().print(table) setattr(table, "_rendered_by_cmdlet", True) if auto_confirm_reason is None: answered_yes = bool(Confirm.ask("Continue anyway?", default=False, console=get_stderr_console())) else: - debug( - f"Bulk URL preflight auto-confirmed duplicates ({auto_confirm_reason}); continuing without user input." - ) - try: - log( - f"Auto-confirmed duplicate URL warning ({auto_confirm_reason}). Continuing...", - file=sys.stderr, + answered_yes = bool(auto_continue_duplicates) + auto_declined = not answered_yes + if answered_yes: + debug( + f"Bulk URL preflight auto-confirmed duplicates ({auto_confirm_reason}); continuing without user input." ) - except Exception: - pass + try: + log( + f"Auto-confirmed duplicate URL warning ({auto_confirm_reason}). Continuing...", + file=sys.stderr, + ) + except Exception: + pass + else: + debug( + f"Bulk URL preflight auto-skipped duplicates ({auto_confirm_reason}); skipping without user input." + ) + try: + log( + f"Duplicate URL detected ({auto_confirm_reason}). Skipping download.", + file=sys.stderr, + ) + except Exception: + pass - if in_pipeline: + if in_pipeline and auto_continue_duplicates: try: existing = pipeline_context.load_value("preflight", default=None) except Exception: @@ -3977,7 +4027,7 @@ def check_url_exists_in_storage( pass if not answered_yes: - if in_pipeline: + if in_pipeline and not auto_declined: try: pipeline_context.request_pipeline_stop(reason="duplicate-url declined", exit_code=0) except Exception: diff --git a/cmdlet/add_file.py b/cmdlet/add_file.py index a66cda4..774978f 100644 --- a/cmdlet/add_file.py +++ b/cmdlet/add_file.py @@ -11,7 +11,7 @@ from urllib.parse import urlparse from SYS import models from SYS import pipeline as ctx -from SYS.logger import log, debug, is_debug_enabled +from SYS.logger import log, debug, debug_panel, is_debug_enabled from SYS.payload_builders import build_table_result_payload from SYS.pipeline_progress import PipelineProgress from SYS.result_publication import overlay_existing_result_table, publish_result_table @@ -247,11 +247,13 @@ class Add_File(Cmdlet): is None) or bool(getattr(stage_ctx, "is_last_stage", False)) + has_downstream_stage = bool(stage_ctx is not None and not is_last_stage) # Directory-mode selector: - # - First pass: `add-file -store X -path ` should ONLY show a selectable table. - # - Second pass (triggered by @ selection expansion): re-run add-file with `-path file1,file2,...` - # and actually ingest/copy. + # - Terminal use: `add-file -store X -path ` shows a selectable table. + # - Pipelined use: `add-file -store X -path | ...` processes the full batch + # immediately so downstream stages receive the uploaded items. + # - Selection replay: `@N` re-runs add-file with `-path file1,file2,...`. dir_scan_mode = False dir_scan_results: Optional[List[Dict[str, Any]]] = None explicit_path_list_results: Optional[List[Dict[str, Any]]] = None @@ -350,6 +352,19 @@ class Add_File(Cmdlet): total_items = len(items_to_process) if isinstance(items_to_process, list) else 0 processed_items = 0 + try: + ui, _ = progress.ui_and_pipe_index() + if ui is not None and total_items: + preview_items = ( + list(items_to_process) + if isinstance(items_to_process, list) else [items_to_process] + ) + progress.begin_pipe( + total_items=total_items, + items_preview=preview_items, + ) + except Exception: + pass try: if total_items: progress.set_percent(0) @@ -369,12 +384,20 @@ class Add_File(Cmdlet): except Exception: use_steps = False - debug(f"[add-file] INPUT result type={type(result).__name__}") - if isinstance(result, list): - debug(f"[add-file] INPUT result is list with {len(result)} items") - debug( - f"[add-file] PARSED args: location={location}, provider={provider_name}, delete={delete_after}" - ) + try: + debug_panel( + "add-file", + [ + ("result_type", type(result).__name__), + ("items", total_items), + ("location", location), + ("provider", provider_name), + ("delete", delete_after), + ], + border_style="cyan", + ) + except Exception: + pass # add-file is ingestion-only: it does not download URLs here. @@ -393,22 +416,22 @@ class Add_File(Cmdlet): except Exception: po = None if po is None: - debug(f"[add-file] PIPE item[{idx}] preview (non-PipeObject)") continue - debug(f"[add-file] PIPE item[{idx}] PipeObject preview") try: safe_po = _sanitize_pipe_object_for_debug(po) safe_po.debug_table() except Exception: pass - if len(preview_items) > max_preview: - debug( - f"[add-file] Skipping {len(preview_items) - max_preview} additional piped item(s) in debug preview" - ) - # If this invocation was directory selector mode, show a selectable table and stop. + should_present_directory_selector = bool(dir_scan_mode and not has_downstream_stage) + if dir_scan_mode and has_downstream_stage: + debug( + "[add-file] Continuing with directory batch ingest because downstream stages exist" + ) + + # If this invocation was terminal directory selector mode, show a selectable table and stop. # The user then runs @N (optionally piped), which replays add-file with selected paths. - if dir_scan_mode: + if should_present_directory_selector: try: from SYS.result_table import Table from pathlib import Path as _Path @@ -563,13 +586,19 @@ class Add_File(Cmdlet): media_path, file_hash, temp_dir_to_cleanup = Add_File._download_provider_source( pipe_obj, config, storage_registry ) - if media_path: - debug( - f"[add-file] Provider source downloaded: {media_path}" + if media_path: + try: + debug_panel( + f"add-file source {idx}/{max(1, total_items)}", + [ + ("path", media_path), + ("hash", file_hash or "N/A"), + ("provider", provider_name or "local"), + ], + border_style="green", ) - debug( - f"[add-file] RESOLVED source: path={media_path}, hash={file_hash if file_hash else 'N/A'}..." - ) + except Exception: + pass if not media_path: failures += 1 continue @@ -1616,6 +1645,7 @@ class Add_File(Cmdlet): ) -> None: pipe_obj.hash = hash_value pipe_obj.store = store + pipe_obj.is_temp = False pipe_obj.path = path pipe_obj.tag = tag if title: @@ -2211,11 +2241,20 @@ class Add_File(Cmdlet): upload_tags = tags if prefer_defer_tags and upload_tags: upload_tags = [] - debug(f"[add-file] Deferring tag application for {backend_name} (backend preference)") - - debug( - f"[add-file] Storing into backend '{backend_name}' path='{media_path}' title='{title}' hash='{f_hash[:12] if f_hash else 'N/A'}'" - ) + try: + debug_panel( + "add-file store", + [ + ("backend", backend_name), + ("path", media_path), + ("title", title), + ("hash_hint", f_hash[:12] if f_hash else "N/A"), + ("defer_tags", bool(prefer_defer_tags and tags)), + ], + border_style="yellow", + ) + except Exception: + pass # Call backend's add_file with full metadata # Backend returns hash as identifier. If we already know the hash from _resolve_source @@ -2227,9 +2266,6 @@ class Add_File(Cmdlet): url=[] if (defer_url_association and url) else url, file_hash=f_hash, ) - debug( - f"[add-file] backend.add_file returned identifier {file_identifier} (len={len(str(file_identifier)) if file_identifier is not None else 'None'})" - ) ##log(f"✓ File added to '{backend_name}': {file_identifier}", file=sys.stderr) stored_path: Optional[str] = None diff --git a/cmdlet/download_file.py b/cmdlet/download_file.py index 2ba266c..2ba9828 100644 --- a/cmdlet/download_file.py +++ b/cmdlet/download_file.py @@ -18,7 +18,7 @@ from contextlib import AbstractContextManager, nullcontext from API.HTTP import _download_direct_file from SYS.models import DownloadError, DownloadOptions, DownloadMediaResult -from SYS.logger import log, debug, is_debug_enabled +from SYS.logger import log, debug, debug_panel, is_debug_enabled from SYS.payload_builders import build_file_result_payload, build_table_result_payload from SYS.pipeline_progress import PipelineProgress from SYS.result_table import Table @@ -113,7 +113,17 @@ class Download_File(Cmdlet): def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: """Main execution method.""" - debug(f"[download-file] run invoked with args: {list(args)}") + try: + debug_panel( + "download-file", + [ + ("args", list(args)), + ("has_piped_input", bool(result)), + ], + border_style="cyan", + ) + except Exception: + debug(f"[download-file] run invoked with args: {list(args)}") return self._run_impl(result, args, config) @staticmethod @@ -1008,7 +1018,6 @@ class Download_File(Cmdlet): from Store import Store from API.HydrusNetwork import is_hydrus_available - debug("[download-file] Initializing storage interface...") storage = Store(config=config or {}, suppress_debug=True) hydrus_available = bool(is_hydrus_available(config or {})) @@ -1126,7 +1135,6 @@ class Download_File(Cmdlet): @staticmethod def _canonicalize_url_for_storage(*, requested_url: str, ytdlp_tool: YtDlpTool, playlist_items: Optional[str]) -> str: if playlist_items: - debug(f"[download-file] Skipping canonicalization for playlist item(s): {playlist_items}") return str(requested_url) try: cf = None @@ -1136,16 +1144,13 @@ class Download_File(Cmdlet): cf = str(cookie_path) except Exception: cf = None - - debug(f"[download-file] Canonicalizing URL: {requested_url}") + pr = probe_url(requested_url, no_playlist=False, timeout_seconds=15, cookiefile=cf) if isinstance(pr, dict): for key in ("webpage_url", "original_url", "url", "requested_url"): value = pr.get(key) if isinstance(value, str) and value.strip(): canon = value.strip() - if canon != requested_url: - debug(f"[download-file] Resolved canonical URL: {requested_url} -> {canon}") return canon except Exception as e: debug(f"[download-file] Canonicalization error for {requested_url}: {e}") @@ -1180,7 +1185,8 @@ class Download_File(Cmdlet): urls=unique_to_check, storage=storage, hydrus_available=hydrus_available, - final_output_dir=final_output_dir + final_output_dir=final_output_dir, + auto_continue_duplicates=False, ) def _preflight_url_duplicates_bulk( @@ -1204,7 +1210,8 @@ class Download_File(Cmdlet): urls=unique_urls, storage=storage, hydrus_available=hydrus_available, - final_output_dir=final_output_dir + final_output_dir=final_output_dir, + auto_continue_duplicates=False, ) @@ -1512,6 +1519,7 @@ class Download_File(Cmdlet): download_timeout_seconds: int, ) -> int: downloaded_count = 0 + duplicate_skipped_count = 0 downloaded_pipe_objects: List[Dict[str, Any]] = [] pipe_seq = 0 clip_sections_spec = self._build_clip_sections_spec(clip_ranges) @@ -1527,9 +1535,6 @@ class Download_File(Cmdlet): for url_index, url in enumerate(supported_url, 1): try: - debug(f"[download-file] Processing URL in loop: {url}") - debug(f"[download-file] ytdl_format parameter passed in: {ytdl_format}") - display_total = batch_total if batch_total > 0 else total_urls display_index = batch_index if batch_total > 0 else url_index display_label = batch_label or str(url) @@ -1543,7 +1548,6 @@ class Download_File(Cmdlet): ) if not skip_per_url_preflight: - debug(f"[download-file] Running duplicate preflight for: {canonical_url}") if not self._preflight_url_duplicate( storage=storage, hydrus_available=hydrus_available, @@ -1551,9 +1555,25 @@ class Download_File(Cmdlet): candidate_url=canonical_url, extra_urls=[url], ): + duplicate_skipped_count += 1 log(f"Skipping download (duplicate found): {url}", file=sys.stderr) continue + try: + debug_panel( + f"Download item {display_index}/{display_total or total_urls}", + [ + ("url", url), + ("canonical_url", canonical_url), + ("mode", mode), + ("format", ytdl_format or "auto"), + ("duplicate_preflight", not skip_per_url_preflight), + ], + border_style="green", + ) + except Exception: + pass + if aggregate_status_mode: try: if display_total > 0: @@ -1682,11 +1702,7 @@ class Download_File(Cmdlet): actual_format = f"{actual_format}+bestaudio" except Exception as e: pass - - debug( - "[download-file] Resolved format for download: " - f"mode={mode}, format={actual_format or 'default'}, playlist_items={actual_playlist_items}" - ) + attempted_single_format_fallback = False attempted_audio_fallback_specific = False @@ -1709,9 +1725,7 @@ class Download_File(Cmdlet): if not aggregate_status_mode: PipelineProgress(pipeline_context).step("downloading") - debug(f"Starting download for {url} (format: {actual_format or 'default'}) with {download_timeout_seconds}s activity timeout...") result_obj = _download_with_timeout(opts, timeout_seconds=download_timeout_seconds, config=config) - debug(f"Download completed for {url}, building pipe object...") break except DownloadError as e: cause = getattr(e, "__cause__", None) @@ -2028,8 +2042,6 @@ class Download_File(Cmdlet): except Exception: pass - debug(f"Emitting {len(pipe_objects)} result(s) to pipeline...") - if not aggregate_status_mode: PipelineProgress(pipeline_context).step("finalized") @@ -2049,7 +2061,17 @@ class Download_File(Cmdlet): pass downloaded_count += len(pipe_objects) - debug("✓ Downloaded and emitted") + try: + debug_panel( + "download-file result", + [ + ("emitted", len(pipe_objects)), + ("url", url), + ], + border_style="green", + ) + except Exception: + pass except DownloadError as e: log(f"Download failed for {url}: {e}", file=sys.stderr) @@ -2057,7 +2079,8 @@ class Download_File(Cmdlet): log(f"Error processing {url}: {e}", file=sys.stderr) if downloaded_count > 0: - debug(f"✓ Successfully processed {downloaded_count} URL(s)") + return 0 + if duplicate_skipped_count > 0: return 0 log("No downloads completed", file=sys.stderr) @@ -2072,7 +2095,6 @@ class Download_File(Cmdlet): parsed: Dict[str, Any], ) -> int: try: - debug("Starting streaming download handler") suppress_nested, _batch_total, _batch_index, _batch_label = self._batch_progress_state(config) ytdlp_tool = YtDlpTool(config) @@ -2091,21 +2113,18 @@ class Download_File(Cmdlet): if not final_output_dir: return 1 - debug(f"Output directory: {final_output_dir}") - progress = PipelineProgress(pipeline_context) + using_shared_ui = pipeline_context.get_stage_context() is not None try: # If we are already in a pipeline stage, the parent UI is already handling progress. # Calling ensure_local_ui can cause re-initialization hangs on some platforms. - if pipeline_context.get_stage_context() is None: - debug("[download-file] Initializing local UI...") + if not using_shared_ui: progress.ensure_local_ui( label="download-file", total_items=len(supported_url), items_preview=supported_url, ) else: - debug("[download-file] Skipping local UI: running inside pipeline stage") try: if not suppress_nested: progress.begin_pipe( @@ -2116,8 +2135,6 @@ class Download_File(Cmdlet): debug(f"[download-file] PipelineProgress begin_pipe error: {err}") except Exception as e: debug(f"[download-file] PipelineProgress update error: {e}") - - debug("[download-file] Parsing clip and query specs...") clip_spec = parsed.get("clip") query_spec = parsed.get("query") @@ -2223,7 +2240,6 @@ class Download_File(Cmdlet): ytdl_format = query_format if not ytdl_format: - debug(f"[download-file] Checking for playlist at {candidate_url}...") if self._maybe_show_playlist_table(url=candidate_url, ytdlp_tool=ytdlp_tool): playlist_selection_handled = True # ... (existing logging code) ... @@ -2270,7 +2286,6 @@ class Download_File(Cmdlet): forced_single_format_id = None forced_single_format_for_batch = False - debug("[download-file] Checking if format table should be shown...") early_ret = self._maybe_show_format_table_for_single_url( mode=mode, clip_spec=clip_spec, @@ -2343,7 +2358,23 @@ class Download_File(Cmdlet): except Exception: timeout_seconds = 300 - debug(f"[download-file] Proceeding to final download call for {len(supported_url)} URL(s)...") + try: + debug_panel( + "Streaming download", + [ + ("urls", len(supported_url)), + ("mode", mode), + ("format", ytdl_format or "auto"), + ("output_dir", final_output_dir), + ("ui", "shared pipeline" if using_shared_ui else "local"), + ("playlist_items", playlist_items), + ("skip_preflight", skip_per_url_preflight), + ("timeout_seconds", timeout_seconds), + ], + border_style="blue", + ) + except Exception: + pass return self._download_supported_urls( supported_url=supported_url, ytdlp_tool=ytdlp_tool, @@ -2855,8 +2886,6 @@ class Download_File(Cmdlet): prev_progress = None had_progress_key = False try: - debug("Starting download-file") - # Allow providers to tap into the active PipelineProgress (optional). try: if isinstance(config, dict): @@ -3117,7 +3146,6 @@ class Download_File(Cmdlet): streaming_exit_code: Optional[int] = None streaming_downloaded = 0 if supported_streaming: - debug(f"[download-file] Using ytdlp provider for {len(supported_streaming)} URL(s)") streaming_exit_code = self._run_streaming_urls( streaming_urls=supported_streaming, args=args, @@ -3161,7 +3189,19 @@ class Download_File(Cmdlet): if not final_output_dir: return 1 - debug(f"Output directory: {final_output_dir}") + try: + debug_panel( + "download-file plan", + [ + ("output_dir", final_output_dir), + ("streaming_urls", len(supported_streaming)), + ("remaining_urls", len(raw_url)), + ("piped_items", len(piped_items) if isinstance(piped_items, list) else int(bool(piped_items))), + ], + border_style="cyan", + ) + except Exception: + debug(f"Output directory: {final_output_dir}") # If the caller isn't running the shared pipeline Live progress UI (e.g. direct # cmdlet execution), start a minimal local pipeline progress panel so downloads diff --git a/cmdlet/search_file.py b/cmdlet/search_file.py index c298a63..1e0578e 100644 --- a/cmdlet/search_file.py +++ b/cmdlet/search_file.py @@ -2158,11 +2158,7 @@ class search_file(Cmdlet): ) db.update_worker_status(worker_id, "error") return 1 - debug(f"[search-file] Searching '{backend_to_search}'") results = target_backend.search(query, limit=limit) - debug( - f"[search-file] '{backend_to_search}' -> {len(results or [])} result(s)" - ) else: all_results = [] store_registry = None @@ -2184,14 +2180,10 @@ class search_file(Cmdlet): if type(backend).search is BaseStore.search: continue - debug(f"[search-file] Searching '{backend_name}'") backend_results = backend.search( query, limit=limit - len(all_results) ) - debug( - f"[search-file] '{backend_name}' -> {len(backend_results or [])} result(s)" - ) if backend_results: all_results.extend(backend_results) if len(all_results) >= limit: diff --git a/tool/ytdlp.py b/tool/ytdlp.py index f2d4575..18e1256 100644 --- a/tool/ytdlp.py +++ b/tool/ytdlp.py @@ -19,7 +19,7 @@ from urllib.parse import urlparse from SYS import pipeline as pipeline_context from SYS.config import get_nested_config_value as _get_nested -from SYS.logger import debug, log +from SYS.logger import debug, debug_panel, log from SYS.models import ( DebugLogger, DownloadError, @@ -505,7 +505,6 @@ def probe_url( def _do_probe() -> None: try: - debug(f"[probe] Starting probe for {url}") ensure_yt_dlp_ready() assert yt_dlp is not None @@ -529,9 +528,7 @@ def probe_url( ydl_opts["noplaylist"] = True with yt_dlp.YoutubeDL(ydl_opts) as ydl: # type: ignore[arg-type] - debug(f"[probe] ytdlp extract_info (download=False) start: {url}") info = ydl.extract_info(url, download=False) - debug(f"[probe] ytdlp extract_info (download=False) done: {url}") if not isinstance(info, dict): result_container[0] = None @@ -550,6 +547,16 @@ def probe_url( "webpage_url": webpage_url, "url": webpage_url or url, } + debug_panel( + "yt-dlp probe", + [ + ("url", url), + ("extractor", info.get("extractor", "")), + ("title", info.get("title", "")), + ("playlist", bool(info.get("entries"))), + ], + border_style="green", + ) except Exception as exc: debug(f"Probe error for {url}: {exc}") result_container[1] = exc @@ -1112,12 +1119,6 @@ class YtDlpTool: if clip_safe_fmt != fmt: debug(f"[ytdlp] clip format normalized: {fmt} -> {clip_safe_fmt}") fmt = clip_safe_fmt - debug( - "[ytdlp] build options: " - f"mode={opts.mode}, ytdl_format={ytdl_format}, default_format={default_fmt}, final_format={fmt}, " - f"cookiefile={base_options.get('cookiefile')}, cookiesfrombrowser={base_options.get('cookiesfrombrowser')}, " - f"player_client={((base_options.get('extractor_args') or {}).get('youtube') or {}).get('player_client')}" - ) base_options["format"] = fmt if opts.mode == "audio": @@ -1199,9 +1200,6 @@ class YtDlpTool: if opts.playlist_items: base_options["playlist_items"] = opts.playlist_items - if not opts.quiet: - debug(f"yt-dlp: mode={opts.mode}, format={base_options.get('format')}, cookiefile={base_options.get('cookiefile')}") - return base_options def build_yt_dlp_cli_args( @@ -2100,7 +2098,6 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] = format) are applied when constructing the YtDlpTool instance. """ - debug(f"[download_media] start: {opts.url}") try: netloc = urlparse(opts.url).netloc.lower() except Exception: @@ -2123,9 +2120,6 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] = raise DownloadError(msg) if opts.playlist_items: - debug( - f"Skipping probe for playlist (item selection: {opts.playlist_items}), proceeding with download" - ) probe_result: Optional[Dict[str, Any]] = {"url": opts.url} else: probe_cookiefile = None @@ -2138,8 +2132,6 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] = probe_result = probe_url(opts.url, no_playlist=opts.no_playlist, timeout_seconds=15, cookiefile=probe_cookiefile) if probe_result is None: - if not opts.quiet: - debug("yt-dlp probe returned no metadata; continuing with direct download attempt") if debug_logger is not None: debug_logger.write_record("ytdlp-probe-miss-continue", {"url": opts.url}) @@ -2155,7 +2147,18 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] = if _progress_callback not in hooks: hooks.append(_progress_callback) if not opts.quiet: - debug(f"Starting yt-dlp download: {opts.url}") + debug_panel( + "yt-dlp job", + [ + ("url", opts.url), + ("mode", opts.mode), + ("format", ytdl_options.get("format") or "default"), + ("cookiefile", ytdl_options.get("cookiefile")), + ("download_sections", ytdl_options.get("download_sections") or "None"), + ("force_keyframes", ytdl_options.get("force_keyframes_at_cuts", False)), + ], + border_style="blue", + ) if debug_logger is not None: debug_logger.write_record("ytdlp-start", {"url": opts.url}) @@ -2164,11 +2167,6 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] = session_id = None first_section_info: Dict[str, Any] = {} try: - if not opts.quiet: - if ytdl_options.get("download_sections"): - debug(f"[yt-dlp] download_sections: {ytdl_options['download_sections']}") - debug(f"[yt-dlp] force_keyframes_at_cuts: {ytdl_options.get('force_keyframes_at_cuts', False)}") - if ytdl_options.get("download_sections"): live_ui, _ = PipelineProgress(pipeline_context).ui_and_pipe_index() quiet_sections = bool(opts.quiet) or (live_ui is not None) @@ -2354,18 +2352,21 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] = media_paths = renamed_media_files if renamed_media_files else None if not opts.quiet: count = len(media_paths) if isinstance(media_paths, list) else 1 - debug(f"✓ Downloaded {count} section media file(s) (session: {session_id})") + debug_panel( + "yt-dlp result", + [ + ("files", count), + ("session", session_id), + ("file", media_path.name), + ], + border_style="green", + ) else: media_path = files[0] media_paths = None - if not opts.quiet: - debug(f"✓ Downloaded section file (pattern not found): {media_path.name}") else: media_path = files[0] media_paths = None - - if not opts.quiet: - debug(f"✓ Downloaded: {media_path.name}") if debug_logger is not None: debug_logger.write_record("ytdlp-file-found", {"path": str(media_path)}) except Exception as exc: @@ -2462,7 +2463,15 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] = source_url = entry.get("webpage_url") or entry.get("original_url") or entry.get("url") if not opts.quiet: - debug(f"✓ Downloaded: {media_path.name} ({len(tags_res)} tags)") + debug_panel( + "yt-dlp result", + [ + ("file", media_path.name), + ("tag_count", len(tags_res)), + ("source_url", source_url or opts.url), + ], + border_style="green", + ) if debug_logger is not None: debug_logger.write_record( "downloaded",