goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

updated panel display

Browse Source
This commit is contained in:
Nose
2026-04-16 17:18:50 -07:00
parent 97e310be70
commit 343a7b37a0
14 changed files with 711 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files
API/HTTP.py
+2 -18
View File
@@ -21,7 +21,7 @@ from pathlib import Path
from urllib.parse import unquote, urlparse, parse_qs
import logging
from SYS.logger import debug, is_debug_enabled, log
from SYS.logger import debug, debug_panel, is_debug_enabled, log
from SYS.models import DebugLogger, DownloadError, DownloadMediaResult, ProgressBar
from SYS.utils import ensure_directory, sha256_file
@@ -80,23 +80,7 @@ class HTTPClient:
def _debug_panel(self, title: str, rows: List[tuple[str, Any]]) -> None:
if not is_debug_enabled():
return
try:
from rich.table import Table as RichTable
from rich.panel import Panel
grid = RichTable.grid(padding=(0, 1))
grid.add_column("Key", style="cyan", no_wrap=True)
grid.add_column("Value")
for key, val in rows:
try:
grid.add_row(str(key), str(val))
except Exception:
grid.add_row(str(key), "<unprintable>")
debug(Panel(grid, title=title, expand=False))
except Exception:
# Fallback to simple debug output
debug(title, rows)
debug_panel(title, rows, border_style="bright_blue")
def __enter__(self):
"""Context manager entry."""
API/data/alldebrid.json
+4 -4
View File
@@ -425,7 +425,7 @@
"(hexupload\\.net|hexload\\.com)/([a-zA-Z0-9]{12})"
],
"regexp": "(hexupload\\.net|hexload\\.com)/([a-zA-Z0-9]{12})",
"status": true
"status": false
},
"hot4share": {
"name": "hot4share",
@@ -482,7 +482,7 @@
"(katfile\\.com/[0-9a-zA-Z]{12})"
],
"regexp": "(katfile\\.(cloud|online|vip)/([0-9a-zA-Z]{12}))|((katfile\\.com/[0-9a-zA-Z]{12}))",
"status": true
"status": false
},
"mediafire": {
"name": "mediafire",
@@ -595,7 +595,7 @@
"(simfileshare\\.net/download/[0-9]+/)"
],
"regexp": "(simfileshare\\.net/download/[0-9]+/)",
"status": true
"status": false
},
"streamtape": {
"name": "streamtape",
@@ -690,7 +690,7 @@
"uploadrar\\.(net|com)/([0-9a-z]{12})"
],
"regexp": "((get|cloud)\\.rahim-soft\\.com/([0-9a-z]{12}))|((fingau\\.com/([0-9a-z]{12})))|((tech|miui|cloud|flash)\\.getpczone\\.com/([0-9a-z]{12}))|(miui.rahim-soft\\.com/([0-9a-z]{12}))|(uploadrar\\.(net|com)/([0-9a-z]{12}))",
"status": true,
"status": false,
"hardRedirect": [
"uploadrar.com/([0-9a-zA-Z]{12})"
]
Provider/alldebrid.py
+8 -4
View File
@@ -17,7 +17,7 @@ from ProviderCore.base import Provider, SearchResult
from SYS.provider_helpers import TableProviderMixin
from SYS.item_accessors import get_field as _extract_value
from SYS.utils import sanitize_filename
from SYS.logger import log, debug
from SYS.logger import log, debug, debug_panel
from SYS.models import DownloadError, PipeObject
_HOSTS_CACHE_TTL_SECONDS = 24 * 60 * 60
@@ -757,9 +757,13 @@ class AllDebrid(TableProviderMixin, Provider):
dom = str(d or "").strip().lower()
if dom and dom not in patterns:
patterns.append(dom)
log(
f"[alldebrid] url_patterns loaded {len(cached)} cached host domains; total patterns={len(patterns)}",
file=sys.stderr,
debug_panel(
"AllDebrid host cache",
[
("cached_domains", len(cached)),
("total_patterns", len(patterns)),
],
border_style="magenta",
)
except Exception:
pass
SYS/config.py
+6
View File
@@ -51,6 +51,12 @@ def global_config() -> List[Dict[str, Any]]:
"label": "Auto-Update",
"default": "true",
"choices": ["true", "false"]
},
{
"key": "table_appearance",
"label": "Table Appearance",
"default": "rainbow",
"choices": ["plain", "bw-striped", "rainbow"]
}
]
SYS/logger.py
+116 -26
View File
@@ -5,7 +5,7 @@ import inspect
import logging
import threading
from pathlib import Path
from typing import Optional
from typing import Any, Optional, Sequence
from SYS.rich_display import console_for
@@ -43,6 +43,106 @@ def is_debug_enabled() -> bool:
return _DEBUG_ENABLED
def _debug_output_suppressed() -> bool:
try:
stderr_name = getattr(sys.stderr, "name", "")
return "nul" in str(stderr_name).lower() or "/dev/null" in str(stderr_name)
except Exception:
return False
def _debug_output_file(file=None):
stream = get_thread_stream()
if stream is not None:
return stream
if file is not None:
return file
return sys.stderr
def _is_rich_renderable(value: Any) -> bool:
if value is None:
return False
if isinstance(value, (str, bytes, bytearray)):
return False
return bool(
hasattr(value, "__rich_console__")
or hasattr(value, "__rich__")
or value.__class__.__module__.startswith("rich.")
)
def _caller_location(depth: int = 1) -> tuple[str, str]:
frame = inspect.currentframe()
current = frame
try:
for _ in range(max(0, int(depth))):
if current is None:
break
current = current.f_back
if current is None:
return "", ""
return Path(current.f_code.co_filename).stem, current.f_code.co_name
finally:
del frame
def _debug_db_log(*, caller_name: str, message: str, level: str = "DEBUG") -> None:
if not _DB_LOGGER:
return
try:
_DB_LOGGER(level, caller_name, message)
except Exception:
pass
def debug_panel(
title: str,
rows: Sequence[tuple[str, Any]],
*,
file=None,
border_style: str = "cyan",
) -> None:
"""Render a compact key/value debug panel when debug logging is enabled."""
if not _DEBUG_ENABLED or _debug_output_suppressed():
return
target_file = _debug_output_file(file)
try:
from rich.panel import Panel
from rich.table import Table as RichTable
grid = RichTable.grid(padding=(0, 1))
grid.add_column("Key", style="cyan", no_wrap=True)
grid.add_column("Value")
for key, val in rows:
try:
grid.add_row(str(key), str(val))
except Exception:
grid.add_row(str(key), "<unprintable>")
debug(
Panel(
grid,
title=str(title or "Debug"),
expand=False,
border_style=border_style,
),
file=target_file,
)
file_name, func_name = _caller_location(depth=2)
caller_name = f"{file_name}.{func_name}" if file_name and func_name else ""
_debug_db_log(
caller_name=caller_name,
message=f"[{title}] " + "; ".join(f"{k}={v}" for k, v in rows),
)
except Exception:
debug(title, list(rows), file=target_file)
def debug(*args, **kwargs) -> None:
"""Print debug message if debug logging is enabled.
@@ -53,26 +153,24 @@ def debug(*args, **kwargs) -> None:
# Check if stderr has been redirected to /dev/null (quiet mode)
# If so, skip output to avoid queuing in background worker's capture
try:
stderr_name = getattr(sys.stderr, "name", "")
if "nul" in str(stderr_name).lower() or "/dev/null" in str(stderr_name):
return
except Exception:
pass
if _debug_output_suppressed():
return
# Check for thread-local stream first
stream = get_thread_stream()
if stream:
kwargs["file"] = stream
# Set default to stderr for debug messages
elif "file" not in kwargs:
kwargs["file"] = sys.stderr
target_file = _debug_output_file(kwargs.pop("file", None))
if len(args) == 1 and _is_rich_renderable(args[0]):
renderable = args[0]
console_for(target_file).print(renderable)
file_name, func_name = _caller_location(depth=1)
caller_name = f"{file_name}.{func_name}" if file_name and func_name else ""
_debug_db_log(caller_name=caller_name, message=f"<rich:{type(renderable).__name__}>")
return
# Prepend DEBUG label
args = ("DEBUG:", *args)
# Use the same logic as log()
log(*args, **kwargs)
log(*args, file=target_file, **kwargs)
def debug_inspect(
@@ -97,19 +195,11 @@ def debug_inspect(
return
# Mirror debug() quiet-mode guard.
try:
stderr_name = getattr(sys.stderr, "name", "")
if "nul" in str(stderr_name).lower() or "/dev/null" in str(stderr_name):
return
except Exception:
pass
if _debug_output_suppressed():
return
# Resolve destination stream.
stream = get_thread_stream()
if stream is not None:
file = stream
elif file is None:
file = sys.stderr
file = _debug_output_file(file)
# Compute caller prefix (same as log()).
prefix = None
SYS/pipeline.py
+76 -17
View File
@@ -10,7 +10,7 @@ from dataclasses import dataclass, field
from contextvars import ContextVar
from typing import Any, Dict, List, Optional, Sequence, Callable
from SYS.models import PipelineStageContext
from SYS.logger import log, debug, is_debug_enabled
from SYS.logger import log, debug, debug_panel, is_debug_enabled
import logging
logger = logging.getLogger(__name__)
from SYS.worker import WorkerManagerRegistry, WorkerStages
@@ -28,6 +28,47 @@ HELP_EXAMPLE_SOURCE_COMMANDS = {
}
def _emit_selection_debug_panel(
*,
selection_token: Any,
selection_indices: Sequence[int],
item_count: int,
filtered_count: int,
stage_table_present: bool,
display_table_present: bool,
stage_is_last: bool,
row_action: Optional[Sequence[Any]] = None,
downstream_stages: Optional[Sequence[Sequence[Any]]] = None,
mode: Optional[str] = None,
) -> None:
if not is_debug_enabled():
return
try:
rows: List[tuple[str, Any]] = [
("selection", str(selection_token or "")),
("indices", [int(idx) + 1 for idx in (selection_indices or [])]),
("items", int(item_count)),
("filtered", int(filtered_count)),
("stage_table", bool(stage_table_present)),
("display_table", bool(display_table_present)),
("stage_is_last", bool(stage_is_last)),
("downstream_stages", len(list(downstream_stages or []))),
]
if mode:
rows.insert(1, ("mode", str(mode)))
if row_action:
rows.append(("row_action", " ".join(str(part) for part in row_action if part is not None)))
debug_panel(
f"Selection replay {selection_token}",
rows,
border_style="magenta",
)
except Exception:
pass
def set_live_progress(progress_ui: Any) -> None:
"""Register the current Live progress UI so cmdlets can suspend it during prompts."""
state = _get_pipeline_state()
@@ -1883,6 +1924,7 @@ class PipelineExecutor:
selected_row_args: List[str] = []
skip_pipe_expansion = source_cmd in {".pipe", ".mpv"} and len(stages) > 0
prefer_row_action = False
preferred_row_action = None
if len(selection_indices) == 1 and not stages:
try:
row_action = _get_row_action(selection_indices[0])
@@ -1890,10 +1932,7 @@ class PipelineExecutor:
row_action = None
if row_action:
prefer_row_action = True
debug(
"@N: skipping source command expansion because row has explicit selection_action "
f"{row_action}"
)
preferred_row_action = list(row_action)
# Command expansion via @N:
# - Default behavior: expand ONLY for single-row selections.
# - Special case: allow multi-row expansion for add-file directory tables by
@@ -1978,7 +2017,7 @@ class PipelineExecutor:
except Exception:
logger.exception("Failed to record pipeline log step for @N expansion (pipeline_session=%r)", getattr(pipeline_session, 'worker_id', None))
elif selected_row_args and stages:
debug("@N: skipping source command expansion because downstream stages exist")
pass
stage_table = None
try:
@@ -2003,8 +2042,6 @@ class PipelineExecutor:
except Exception:
stage_table = None
debug(f"@N: stage_table={stage_table is not None}, display_table={display_table is not None}")
# ====================================================================
# PHASE 4: Retrieve and filter items from current result set
# ====================================================================
@@ -2015,14 +2052,31 @@ class PipelineExecutor:
except Exception as exc:
debug(f"@N: Exception getting items_list: {exc}")
items_list = []
debug(f"@N: selection_indices={selection_indices}, items_list length={len(items_list)}")
resolved_items = items_list if items_list else []
if items_list:
filtered = [
resolved_items[i] for i in selection_indices
if 0 <= i < len(resolved_items)
]
if selection_indices:
if len(selection_indices) == 1:
selection_label = f"@{selection_indices[0] + 1}"
else:
selection_label = "@{" + ",".join(str(idx + 1) for idx in selection_indices) + "}"
else:
selection_label = "@selection"
_emit_selection_debug_panel(
selection_token=selection_label,
selection_indices=selection_indices,
item_count=len(items_list),
filtered_count=len(filtered),
stage_table_present=(stage_table is not None),
display_table_present=(display_table is not None),
stage_is_last=(not stages),
row_action=preferred_row_action,
downstream_stages=stages,
mode=("row_action" if preferred_row_action else "selection"),
)
if not filtered:
print("No items matched selection in pipeline\n")
return False, None
@@ -2088,15 +2142,12 @@ class PipelineExecutor:
filtered = track_items
table_type_hint = "tidal.track"
debug(f"@N: calling _maybe_run_class_selector with filtered={len(filtered)} items, stage_is_last={not stages}")
if PipelineExecutor._maybe_run_class_selector(
ctx,
config,
filtered,
stage_is_last=(not stages)):
debug(f"@N: _maybe_run_class_selector returned True, returning False")
return False, None
debug(f"@N: _maybe_run_class_selector returned False, continuing")
from SYS.pipe_object import coerce_to_pipe_object
@@ -2105,7 +2156,6 @@ class PipelineExecutor:
filtered_pipe_objs
if len(filtered_pipe_objs) > 1 else filtered_pipe_objs[0]
)
debug(f"@N: coerced piped_result, stages={stages}")
if pipeline_session and worker_manager:
try:
@@ -2204,7 +2254,6 @@ class PipelineExecutor:
print("Auto-applying metadata selection via get-tag")
stages.append(["get-tag"])
elif auto_stage:
debug(f"@N: Found auto_stage={auto_stage}, appending")
try:
print(f"Auto-running selection via {auto_stage[0]}")
except Exception:
@@ -2238,7 +2287,6 @@ class PipelineExecutor:
if not stages and selection_indices and len(selection_indices) == 1:
row_action = _get_row_action(selection_indices[0], items_list)
if row_action:
debug(f"@N: applying row_action {row_action}")
stages.append(row_action)
if pipeline_session and worker_manager:
try:
@@ -2464,7 +2512,18 @@ class PipelineExecutor:
ctx = sys.modules[__name__]
try:
debug(f"execute_tokens: tokens={tokens}")
try:
from SYS.logger import debug_panel
debug_panel(
"Pipeline execution",
[
("command", " ".join(str(tok) for tok in tokens)),
("token_count", len(tokens)),
],
)
except Exception:
debug(f"execute_tokens: tokens={tokens}")
self._try_clear_pipeline_stop(ctx)
# REPL guard: stage-local tables should not persist across independent
SYS/result_table.py
+116 -19
View File
@@ -16,6 +16,7 @@ from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Callable, Set
from pathlib import Path
import json
import re
from rich.box import SIMPLE
from rich.console import Group
@@ -113,14 +114,97 @@ _RESULT_TABLE_ROW_STYLE_LOOP: List[tuple[str, str]] = [
("#ffffff", "#000000"),
]
_RESULT_TABLE_BW_ROW_STYLE_LOOP: List[tuple[str, str]] = [
("#000000", "#ffffff"),
("#ffffff", "#000000"),
]
RESULT_TABLE_HEADER_STYLE = "bold #000000 on #ffffff"
RESULT_TABLE_BORDER_STYLE = "#000000 on #ffffff"
RESULT_TABLE_PLAIN_HEADER_STYLE = "bold"
_RESULT_TABLE_APPEARANCE_ALIASES: Dict[str, str] = {
"": "rainbow",
"rainbow": "rainbow",
"default": "rainbow",
"plain": "plain",
"none": "plain",
"bw": "bw-striped",
"b-w": "bw-striped",
"b-w-striped": "bw-striped",
"bw-striped": "bw-striped",
"b-w-stripes": "bw-striped",
"bw-stripes": "bw-striped",
"black-white": "bw-striped",
"black-white-striped": "bw-striped",
"black-white-stripes": "bw-striped",
"black-and-white": "bw-striped",
"black-and-white-striped": "bw-striped",
"black-and-white-stripes": "bw-striped",
}
def get_result_table_row_style(row_index: int) -> str:
text_color, bg_color = _RESULT_TABLE_ROW_STYLE_LOOP[
row_index % len(_RESULT_TABLE_ROW_STYLE_LOOP)
]
def normalize_result_table_appearance_mode(value: Any) -> str:
text = str(value or "").strip().lower()
if not text:
return "rainbow"
collapsed = re.sub(r"[^a-z0-9]+", "-", text).strip("-")
return _RESULT_TABLE_APPEARANCE_ALIASES.get(collapsed, "rainbow")
def get_result_table_appearance_mode(config: Optional[Dict[str, Any]] = None) -> str:
cfg = config
if cfg is None:
try:
from SYS.config import load_config
cfg = load_config() or {}
except Exception:
cfg = {}
raw_value = None
if isinstance(cfg, dict):
raw_value = cfg.get("table_appearance")
return normalize_result_table_appearance_mode(raw_value)
def get_result_table_header_style(config: Optional[Dict[str, Any]] = None) -> str:
mode = get_result_table_appearance_mode(config)
if mode == "plain":
return RESULT_TABLE_PLAIN_HEADER_STYLE
return RESULT_TABLE_HEADER_STYLE
def get_result_table_border_style(config: Optional[Dict[str, Any]] = None) -> str:
mode = get_result_table_appearance_mode(config)
if mode == "plain":
return ""
return RESULT_TABLE_BORDER_STYLE
def get_result_table_panel_style(config: Optional[Dict[str, Any]] = None) -> str:
mode = get_result_table_appearance_mode(config)
if mode == "plain":
return ""
return "on #ffffff"
def get_result_table_row_style(
row_index: int,
appearance_mode: Optional[str] = None,
config: Optional[Dict[str, Any]] = None,
) -> str:
mode = appearance_mode or get_result_table_appearance_mode(config)
if mode == "plain":
return ""
style_loop = (
_RESULT_TABLE_BW_ROW_STYLE_LOOP
if mode == "bw-striped" else _RESULT_TABLE_ROW_STYLE_LOOP
)
text_color, bg_color = style_loop[row_index % len(style_loop)]
return f"{text_color} on {bg_color}"
@@ -1435,16 +1519,21 @@ class Table:
def to_rich(self):
"""Return a Rich renderable representing this table."""
appearance_mode = get_result_table_appearance_mode()
header_style = get_result_table_header_style({"table_appearance": appearance_mode})
border_style = get_result_table_border_style({"table_appearance": appearance_mode})
panel_style = get_result_table_panel_style({"table_appearance": appearance_mode})
if not self.rows:
empty = Text("No results")
return (
Panel(
empty,
title=Text(str(self.title), style=RESULT_TABLE_HEADER_STYLE),
border_style=RESULT_TABLE_BORDER_STYLE,
title=Text(str(self.title), style=header_style),
border_style=border_style,
padding=(0, 0),
expand=False,
style="on #ffffff",
style=panel_style,
)
if self.title
else empty
@@ -1460,8 +1549,8 @@ class Table:
table = RichTable(
show_header=True,
header_style=RESULT_TABLE_HEADER_STYLE,
border_style=RESULT_TABLE_BORDER_STYLE,
header_style=header_style,
border_style=border_style,
box=None,
expand=False,
show_lines=False,
@@ -1497,7 +1586,13 @@ class Table:
for name in col_names:
val = row.get_column(name) or ""
cells.append(self._apply_value_case(_sanitize_cell_text(val)))
table.add_row(*cells, style=get_result_table_row_style(row_idx - 1))
table.add_row(
*cells,
style=get_result_table_row_style(
row_idx - 1,
appearance_mode=appearance_mode,
),
)
if self.title or self.header_lines:
header_bits = [Text(line) for line in (self.header_lines or [])]
@@ -1505,11 +1600,11 @@ class Table:
return (
Panel(
renderable,
title=Text(str(self.title), style=RESULT_TABLE_HEADER_STYLE),
border_style=RESULT_TABLE_BORDER_STYLE,
title=Text(str(self.title), style=header_style),
border_style=border_style,
padding=(0, 0),
expand=False,
style="on #ffffff",
style=panel_style,
)
if self.title
else renderable
@@ -2385,11 +2480,13 @@ class ItemDetailView(Table):
elements = []
if has_details:
header_style = get_result_table_header_style()
border_style = get_result_table_border_style()
detail_title = str(self.detail_title or "Item Details").strip() or "Item Details"
elements.append(Panel(
details_table,
title=Text(detail_title, style=RESULT_TABLE_HEADER_STYLE),
border_style=RESULT_TABLE_BORDER_STYLE,
title=Text(detail_title, style=header_style),
border_style=border_style,
padding=(1, 2)
))
@@ -2397,11 +2494,11 @@ class ItemDetailView(Table):
# If it's a Panel already (from super().to_rich() with title), use it directly
# but force the border style to the result-table standard for consistency
if isinstance(results_renderable, Panel):
results_renderable.border_style = RESULT_TABLE_BORDER_STYLE
results_renderable.border_style = get_result_table_border_style()
if results_renderable.title:
results_renderable.title = Text(
str(results_renderable.title),
style=RESULT_TABLE_HEADER_STYLE,
style=get_result_table_header_style(),
)
# Add a bit of padding inside if it contains a table
elements.append(results_renderable)
@@ -2416,8 +2513,8 @@ class ItemDetailView(Table):
elements.append(
Panel(
results_group,
title=Text(str(display_title), style=RESULT_TABLE_HEADER_STYLE),
border_style=RESULT_TABLE_BORDER_STYLE,
title=Text(str(display_title), style=get_result_table_header_style()),
border_style=get_result_table_border_style(),
)
)
SYS/result_table_renderers.py
+13 -5
View File
@@ -13,9 +13,10 @@ import logging
logger = logging.getLogger(__name__)
from SYS.result_table import (
RESULT_TABLE_BORDER_STYLE,
RESULT_TABLE_HEADER_STYLE,
apply_result_table_layout,
get_result_table_appearance_mode,
get_result_table_border_style,
get_result_table_header_style,
get_result_table_row_style,
)
from SYS.result_table_api import ColumnSpec, ResultModel, ResultTable, Renderer
@@ -38,8 +39,8 @@ class RichRenderer(Renderer):
table = RichTable(
show_header=True,
header_style=RESULT_TABLE_HEADER_STYLE,
border_style=RESULT_TABLE_BORDER_STYLE,
header_style=get_result_table_header_style(),
border_style=get_result_table_border_style(),
box=None,
padding=(0, 1),
pad_edge=False,
@@ -47,6 +48,7 @@ class RichRenderer(Renderer):
expand=False,
)
apply_result_table_layout(table)
appearance_mode = get_result_table_appearance_mode()
cols = list(columns)
for col in cols:
if str(col.header or "").strip().lower() == "tag":
@@ -71,7 +73,13 @@ class RichRenderer(Renderer):
logger.exception("Column extractor failed for '%s': %s", col.header, exc)
cell = ""
cells.append(cell)
table.add_row(*cells, style=get_result_table_row_style(row_idx))
table.add_row(
*cells,
style=get_result_table_row_style(
row_idx,
appearance_mode=appearance_mode,
),
)
return table
Store/HydrusNetwork.py
+111 -39
View File
@@ -8,7 +8,7 @@ from collections import deque
from pathlib import Path
from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple
from urllib.parse import quote
from urllib.parse import quote, parse_qsl, urlencode, urlsplit, urlunsplit
import httpx
from API.httpx_shared import get_shared_httpx_client
@@ -417,8 +417,6 @@ class HydrusNetwork(Store):
if not file_hash:
file_hash = sha256_file(file_path)
debug(f"{self._log_prefix()} file hash: {file_hash}")
# Use persistent client with session key
client = self._client
if client is None:
@@ -564,7 +562,6 @@ class HydrusNetwork(Store):
raise Exception(f"Hydrus response missing file hash: {response}")
file_hash = hydrus_hash
debug(f"{self._log_prefix()} hash: {file_hash}")
# Add tags if provided (both for new and existing files)
if tag_list:
@@ -575,13 +572,7 @@ class HydrusNetwork(Store):
service_name = "my tags"
try:
debug(
f"{self._log_prefix()} Adding {len(tag_list)} tag(s): {tag_list}"
)
client.add_tag(file_hash, tag_list, service_name)
debug(
f"{self._log_prefix()} Tags added via '{service_name}'"
)
except Exception as exc:
log(
f"{self._log_prefix()} ⚠️ Failed to add tags: {exc}",
@@ -590,14 +581,10 @@ class HydrusNetwork(Store):
# Associate url if provided (both for new and existing files)
if url:
debug(
f"{self._log_prefix()} Associating {len(url)} URL(s) with file"
)
for url in url:
if url:
try:
client.associate_url(file_hash, str(url))
debug(f"{self._log_prefix()} Associated URL: {url}")
except Exception as exc:
log(
f"{self._log_prefix()} ⚠️ Failed to associate URL {url}: {exc}",
@@ -634,7 +621,6 @@ class HydrusNetwork(Store):
raise Exception("Hydrus client unavailable")
prefix = self._log_prefix()
debug(f"{prefix} Searching for: {query}")
def _extract_urls(meta_obj: Any) -> list[str]:
if not isinstance(meta_obj, dict):
@@ -720,6 +706,70 @@ class HydrusNetwork(Store):
) -> list[dict[str, Any]]:
"""Best-effort URL search by scanning Hydrus metadata with include_file_url=True."""
try:
from API.HydrusNetwork import _generate_hydrus_url_variants
except Exception:
_generate_hydrus_url_variants = None # type: ignore[assignment]
def _normalize_url_match_token(value: str | None) -> str:
token = str(value or "").strip()
if not token:
return ""
token = token.split("#", 1)[0]
try:
parsed_url = urlsplit(token)
except Exception:
return token.lower()
if parsed_url.scheme and parsed_url.scheme.lower() not in {"http", "https"}:
return token.lower()
netloc = str(parsed_url.netloc or "").strip().lower()
if netloc.startswith("www."):
netloc = netloc[4:]
try:
query_pairs = parse_qsl(parsed_url.query, keep_blank_values=True)
except Exception:
query_pairs = []
filtered_pairs = []
for key, val in query_pairs:
key_norm = str(key or "").lower()
if key_norm in {"t", "start", "time_continue", "timestamp", "time", "begin"}:
continue
if key_norm.startswith("utm_"):
continue
filtered_pairs.append((key, val))
normalized_query = urlencode(filtered_pairs, doseq=True) if filtered_pairs else ""
normalized = urlunsplit(("", netloc, parsed_url.path or "", normalized_query, ""))
return str(normalized or token).lstrip("/").lower()
def _append_url_needles(output: list[str], candidate: str | None) -> None:
raw_candidate = str(candidate or "").strip()
if not raw_candidate:
return
expanded_candidates = [raw_candidate]
if callable(_generate_hydrus_url_variants):
try:
expanded_candidates.extend(_generate_hydrus_url_variants(raw_candidate) or [])
except Exception:
pass
for expanded in expanded_candidates:
expanded_text = str(expanded or "").strip()
if not expanded_text:
continue
lowered = expanded_text.lower()
if lowered and lowered not in output:
output.append(lowered)
normalized = _normalize_url_match_token(expanded_text)
if normalized and normalized not in output:
output.append(normalized)
candidate_file_ids: list[int] = []
candidate_hashes: list[str] = []
seen_file_ids: set[int] = set()
@@ -775,13 +825,9 @@ class HydrusNetwork(Store):
needle_list: list[str] = []
if isinstance(needles, (list, tuple, set)):
for item in needles:
text = str(item or "").strip().lower()
if text and text not in needle_list:
needle_list.append(text)
_append_url_needles(needle_list, str(item or ""))
if not needle_list:
needle = (url_value or "").strip().lower()
if needle:
needle_list = [needle]
_append_url_needles(needle_list, url_value)
chunk_size = 200
out: list[dict[str, Any]] = []
if scan_limit is None:
@@ -855,7 +901,14 @@ class HydrusNetwork(Store):
continue
if not needle_list:
continue
if any(any(n in u.lower() for n in needle_list) for u in urls):
normalized_urls = [_normalize_url_match_token(u) for u in urls]
if any(
any(
n in str(u or "").lower() or (normalized_urls[idx] and n in normalized_urls[idx])
for idx, u in enumerate(urls)
)
for n in needle_list
):
out.append(meta)
continue
@@ -1001,7 +1054,8 @@ class HydrusNetwork(Store):
return ids_out, hashes_out
query_lower = query.lower().strip()
raw_query = str(query or "").strip()
query_lower = raw_query.lower().strip()
# Support `ext:<value>` anywhere in the query. We filter results by the
# Hydrus metadata extension field.
@@ -1055,10 +1109,10 @@ class HydrusNetwork(Store):
hashes: list[str] = []
file_ids: list[int] = []
if ":" in query_lower and not query_lower.startswith(":"):
namespace, pattern = query_lower.split(":", 1)
namespace = namespace.strip().lower()
pattern = pattern.strip()
if ":" in raw_query and not raw_query.startswith(":"):
namespace_raw, pattern_raw = raw_query.split(":", 1)
namespace = namespace_raw.strip().lower()
pattern = pattern_raw.strip()
if namespace == "url":
try:
fetch_limit_raw = int(limit) if limit else 100
@@ -1066,7 +1120,7 @@ class HydrusNetwork(Store):
fetch_limit_raw = 100
if url_only:
metadata_list = _search_url_query_metadata(
query_lower,
f"url:{pattern}",
fetch_limit_raw,
minimal=minimal,
)
@@ -1092,12 +1146,13 @@ class HydrusNetwork(Store):
token = str(value or "").strip().lower()
if not token:
return ""
return token.replace("*", "").replace("?", "")
return token.replace("*", "")
# Fast-path: exact URL via /add_urls/get_url_files when a full URL is provided.
exact_url_attempted = False
try:
if pattern.startswith("http://") or pattern.startswith("https://"):
has_wildcards = ("*" in pattern)
if (pattern.startswith("http://") or pattern.startswith("https://")) and not has_wildcards:
exact_url_attempted = True
metadata_list = self.lookup_url_metadata(pattern, minimal=minimal)
except Exception:
@@ -1123,7 +1178,7 @@ class HydrusNetwork(Store):
minimal=minimal,
)
elif namespace == "system":
normalized_system_predicate = pattern.strip()
normalized_system_predicate = pattern.strip().lower()
if normalized_system_predicate == "has url":
try:
fetch_limit = int(limit) if limit else 100
@@ -1173,7 +1228,6 @@ class HydrusNetwork(Store):
if freeform_union_search:
if not title_predicates and not freeform_predicates:
debug(f"{prefix} 0 result(s)")
return []
payloads: list[Any] = []
@@ -1223,7 +1277,6 @@ class HydrusNetwork(Store):
hashes = []
else:
if not tags:
debug(f"{prefix} 0 result(s)")
return []
search_result = client.search_files(
@@ -1239,7 +1292,6 @@ class HydrusNetwork(Store):
if ext_only and ext_filter:
results = []
if not file_ids and not hashes:
debug(f"{prefix} 0 result(s)")
return []
# Prefer file_ids if available.
@@ -1301,14 +1353,11 @@ class HydrusNetwork(Store):
"ext": _resolve_ext_from_meta(meta, mime_type),
}
)
debug(f"{prefix} {len(results)} result(s)")
return results[:limit]
# If we only got hashes, fall back to the normal flow below.
if not file_ids and not hashes:
debug(f"{prefix} 0 result(s)")
return []
file_ids, hashes = _cap_metadata_candidates(
@@ -1457,8 +1506,6 @@ class HydrusNetwork(Store):
"ext": ext,
}
)
debug(f"{prefix} {len(results)} result(s)")
if ext_filter:
wanted = ext_filter
filtered: list[dict[str, Any]] = []
@@ -2110,6 +2157,31 @@ class HydrusNetwork(Store):
seen_ids.add(file_id)
file_ids.append(file_id)
statuses = response.get("url_file_statuses")
if isinstance(statuses, list):
for entry in statuses:
if not isinstance(entry, dict):
continue
status_hash = entry.get("hash") or entry.get("file_hash")
try:
normalized_hash = str(status_hash or "").strip().lower()
except Exception:
normalized_hash = ""
if normalized_hash and normalized_hash not in seen_hashes:
seen_hashes.add(normalized_hash)
hashes.append(normalized_hash)
status_id = entry.get("file_id") or entry.get("fileid")
try:
file_id = int(status_id) if status_id is not None else None
except (TypeError, ValueError):
file_id = None
if file_id is None or file_id in seen_ids:
continue
seen_ids.add(file_id)
file_ids.append(file_id)
for key in ("normalized_url", "redirect_url", "url"):
value = response.get(key)
if isinstance(value, str):
cmdlet/_shared.py
+70 -20
View File
@@ -14,7 +14,7 @@ from collections.abc import Iterable as IterableABC
from functools import lru_cache
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
from SYS.logger import log, debug
from SYS.logger import log, debug, debug_panel
from pathlib import Path
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple
from dataclasses import dataclass, field
@@ -3158,6 +3158,8 @@ def check_url_exists_in_storage(
storage: Any,
hydrus_available: bool,
final_output_dir: Optional[Path] = None,
*,
auto_continue_duplicates: bool = True,
) -> bool:
"""Pre-flight check to see if URLs already exist in storage.
@@ -3187,7 +3189,6 @@ def check_url_exists_in_storage(
in_pipeline = bool(stage_ctx is not None or ("|" in str(current_cmd_text or "")))
start_time = time.monotonic()
time_budget = 45.0
debug(f"[preflight] check_url_exists_in_storage: checking {len(urls)} url(s)")
if in_pipeline:
try:
already_checked = bool(
@@ -3243,7 +3244,7 @@ def check_url_exists_in_storage(
return False
return False
if in_pipeline:
if in_pipeline and auto_continue_duplicates:
try:
cached_cmd = pipeline_context.load_value("preflight.url_duplicates.command", default="")
cached_decision = pipeline_context.load_value("preflight.url_duplicates.continue", default=None)
@@ -3706,6 +3707,20 @@ def check_url_exists_in_storage(
debug("Bulk URL preflight skipped: no searchable backends")
return True
try:
debug_panel(
"URL preflight",
[
("url_count", len(unique_urls)),
("pipeline", in_pipeline),
("bulk_mode", bulk_mode),
("backends", ", ".join(str(name) for name in backend_names)),
],
border_style="yellow",
)
except Exception:
pass
seen_pairs: set[tuple[str, str]] = set()
matched_urls: set[str] = set()
match_rows: List[Dict[str, Any]] = []
@@ -3726,12 +3741,7 @@ def check_url_exists_in_storage(
except Exception:
continue
debug(f"[preflight] Scanning backend: {backend_name}")
if HydrusNetwork is not None and isinstance(backend, HydrusNetwork):
client = getattr(backend, "_client", None)
if client is None:
continue
if not hydrus_available:
debug("Bulk URL preflight: global Hydrus availability check failed; attempting per-backend best-effort lookup")
@@ -3748,7 +3758,33 @@ def check_url_exists_in_storage(
found_hash: Optional[str] = None
found = False
lookup_exact = getattr(backend, "find_hashes_by_url", None)
if callable(lookup_exact):
for needle in [original_url, *(needles or [])][:7]:
needle_text = str(needle or "").strip()
if not _httpish(needle_text):
continue
try:
exact_hashes = lookup_exact(needle_text) or []
except Exception:
continue
if not isinstance(exact_hashes, list) or not exact_hashes:
continue
try:
found_hash = str(exact_hashes[0] or "").strip().lower()
except Exception:
found_hash = None
found = True
break
client = getattr(backend, "_client", None)
if found:
pass
elif client is None:
continue
for needle in (needles or [])[:6]:
if found:
break
if not _httpish(needle):
continue
try:
@@ -3868,7 +3904,6 @@ def check_url_exists_in_storage(
match_rows.append(display_row)
if not match_rows:
debug("Bulk URL preflight: no matches")
if in_pipeline:
preflight_cache = _load_preflight_cache()
url_dup_cache = preflight_cache.get("url_duplicates")
@@ -3935,24 +3970,39 @@ def check_url_exists_in_storage(
auto_confirm_reason = "non-interactive stdin"
answered_yes = True
auto_declined = False
with cm:
get_stderr_console().print(table)
setattr(table, "_rendered_by_cmdlet", True)
if auto_confirm_reason is None:
answered_yes = bool(Confirm.ask("Continue anyway?", default=False, console=get_stderr_console()))
else:
debug(
f"Bulk URL preflight auto-confirmed duplicates ({auto_confirm_reason}); continuing without user input."
)
try:
log(
f"Auto-confirmed duplicate URL warning ({auto_confirm_reason}). Continuing...",
file=sys.stderr,
answered_yes = bool(auto_continue_duplicates)
auto_declined = not answered_yes
if answered_yes:
debug(
f"Bulk URL preflight auto-confirmed duplicates ({auto_confirm_reason}); continuing without user input."
)
except Exception:
pass
try:
log(
f"Auto-confirmed duplicate URL warning ({auto_confirm_reason}). Continuing...",
file=sys.stderr,
)
except Exception:
pass
else:
debug(
f"Bulk URL preflight auto-skipped duplicates ({auto_confirm_reason}); skipping without user input."
)
try:
log(
f"Duplicate URL detected ({auto_confirm_reason}). Skipping download.",
file=sys.stderr,
)
except Exception:
pass
if in_pipeline:
if in_pipeline and auto_continue_duplicates:
try:
existing = pipeline_context.load_value("preflight", default=None)
except Exception:
@@ -3977,7 +4027,7 @@ def check_url_exists_in_storage(
pass
if not answered_yes:
if in_pipeline:
if in_pipeline and not auto_declined:
try:
pipeline_context.request_pipeline_stop(reason="duplicate-url declined", exit_code=0)
except Exception:
cmdlet/add_file.py
+68 -32
View File
@@ -11,7 +11,7 @@ from urllib.parse import urlparse
from SYS import models
from SYS import pipeline as ctx
from SYS.logger import log, debug, is_debug_enabled
from SYS.logger import log, debug, debug_panel, is_debug_enabled
from SYS.payload_builders import build_table_result_payload
from SYS.pipeline_progress import PipelineProgress
from SYS.result_publication import overlay_existing_result_table, publish_result_table
@@ -247,11 +247,13 @@ class Add_File(Cmdlet):
is None) or bool(getattr(stage_ctx,
"is_last_stage",
False))
has_downstream_stage = bool(stage_ctx is not None and not is_last_stage)
# Directory-mode selector:
# - First pass: `add-file -store X -path <DIR>` should ONLY show a selectable table.
# - Second pass (triggered by @ selection expansion): re-run add-file with `-path file1,file2,...`
# and actually ingest/copy.
# - Terminal use: `add-file -store X -path <DIR>` shows a selectable table.
# - Pipelined use: `add-file -store X -path <DIR> | ...` processes the full batch
# immediately so downstream stages receive the uploaded items.
# - Selection replay: `@N` re-runs add-file with `-path file1,file2,...`.
dir_scan_mode = False
dir_scan_results: Optional[List[Dict[str, Any]]] = None
explicit_path_list_results: Optional[List[Dict[str, Any]]] = None
@@ -350,6 +352,19 @@ class Add_File(Cmdlet):
total_items = len(items_to_process) if isinstance(items_to_process, list) else 0
processed_items = 0
try:
ui, _ = progress.ui_and_pipe_index()
if ui is not None and total_items:
preview_items = (
list(items_to_process)
if isinstance(items_to_process, list) else [items_to_process]
)
progress.begin_pipe(
total_items=total_items,
items_preview=preview_items,
)
except Exception:
pass
try:
if total_items:
progress.set_percent(0)
@@ -369,12 +384,20 @@ class Add_File(Cmdlet):
except Exception:
use_steps = False
debug(f"[add-file] INPUT result type={type(result).__name__}")
if isinstance(result, list):
debug(f"[add-file] INPUT result is list with {len(result)} items")
debug(
f"[add-file] PARSED args: location={location}, provider={provider_name}, delete={delete_after}"
)
try:
debug_panel(
"add-file",
[
("result_type", type(result).__name__),
("items", total_items),
("location", location),
("provider", provider_name),
("delete", delete_after),
],
border_style="cyan",
)
except Exception:
pass
# add-file is ingestion-only: it does not download URLs here.
@@ -393,22 +416,22 @@ class Add_File(Cmdlet):
except Exception:
po = None
if po is None:
debug(f"[add-file] PIPE item[{idx}] preview (non-PipeObject)")
continue
debug(f"[add-file] PIPE item[{idx}] PipeObject preview")
try:
safe_po = _sanitize_pipe_object_for_debug(po)
safe_po.debug_table()
except Exception:
pass
if len(preview_items) > max_preview:
debug(
f"[add-file] Skipping {len(preview_items) - max_preview} additional piped item(s) in debug preview"
)
# If this invocation was directory selector mode, show a selectable table and stop.
should_present_directory_selector = bool(dir_scan_mode and not has_downstream_stage)
if dir_scan_mode and has_downstream_stage:
debug(
"[add-file] Continuing with directory batch ingest because downstream stages exist"
)
# If this invocation was terminal directory selector mode, show a selectable table and stop.
# The user then runs @N (optionally piped), which replays add-file with selected paths.
if dir_scan_mode:
if should_present_directory_selector:
try:
from SYS.result_table import Table
from pathlib import Path as _Path
@@ -563,13 +586,19 @@ class Add_File(Cmdlet):
media_path, file_hash, temp_dir_to_cleanup = Add_File._download_provider_source(
pipe_obj, config, storage_registry
)
if media_path:
debug(
f"[add-file] Provider source downloaded: {media_path}"
if media_path:
try:
debug_panel(
f"add-file source {idx}/{max(1, total_items)}",
[
("path", media_path),
("hash", file_hash or "N/A"),
("provider", provider_name or "local"),
],
border_style="green",
)
debug(
f"[add-file] RESOLVED source: path={media_path}, hash={file_hash if file_hash else 'N/A'}..."
)
except Exception:
pass
if not media_path:
failures += 1
continue
@@ -1616,6 +1645,7 @@ class Add_File(Cmdlet):
) -> None:
pipe_obj.hash = hash_value
pipe_obj.store = store
pipe_obj.is_temp = False
pipe_obj.path = path
pipe_obj.tag = tag
if title:
@@ -2211,11 +2241,20 @@ class Add_File(Cmdlet):
upload_tags = tags
if prefer_defer_tags and upload_tags:
upload_tags = []
debug(f"[add-file] Deferring tag application for {backend_name} (backend preference)")
debug(
f"[add-file] Storing into backend '{backend_name}' path='{media_path}' title='{title}' hash='{f_hash[:12] if f_hash else 'N/A'}'"
)
try:
debug_panel(
"add-file store",
[
("backend", backend_name),
("path", media_path),
("title", title),
("hash_hint", f_hash[:12] if f_hash else "N/A"),
("defer_tags", bool(prefer_defer_tags and tags)),
],
border_style="yellow",
)
except Exception:
pass
# Call backend's add_file with full metadata
# Backend returns hash as identifier. If we already know the hash from _resolve_source
@@ -2227,9 +2266,6 @@ class Add_File(Cmdlet):
url=[] if (defer_url_association and url) else url,
file_hash=f_hash,
)
debug(
f"[add-file] backend.add_file returned identifier {file_identifier} (len={len(str(file_identifier)) if file_identifier is not None else 'None'})"
)
##log(f"✓ File added to '{backend_name}': {file_identifier}", file=sys.stderr)
stored_path: Optional[str] = None
cmdlet/download_file.py
+80 -40
View File
@@ -18,7 +18,7 @@ from contextlib import AbstractContextManager, nullcontext
from API.HTTP import _download_direct_file
from SYS.models import DownloadError, DownloadOptions, DownloadMediaResult
from SYS.logger import log, debug, is_debug_enabled
from SYS.logger import log, debug, debug_panel, is_debug_enabled
from SYS.payload_builders import build_file_result_payload, build_table_result_payload
from SYS.pipeline_progress import PipelineProgress
from SYS.result_table import Table
@@ -113,7 +113,17 @@ class Download_File(Cmdlet):
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
"""Main execution method."""
debug(f"[download-file] run invoked with args: {list(args)}")
try:
debug_panel(
"download-file",
[
("args", list(args)),
("has_piped_input", bool(result)),
],
border_style="cyan",
)
except Exception:
debug(f"[download-file] run invoked with args: {list(args)}")
return self._run_impl(result, args, config)
@staticmethod
@@ -1008,7 +1018,6 @@ class Download_File(Cmdlet):
from Store import Store
from API.HydrusNetwork import is_hydrus_available
debug("[download-file] Initializing storage interface...")
storage = Store(config=config or {}, suppress_debug=True)
hydrus_available = bool(is_hydrus_available(config or {}))
@@ -1126,7 +1135,6 @@ class Download_File(Cmdlet):
@staticmethod
def _canonicalize_url_for_storage(*, requested_url: str, ytdlp_tool: YtDlpTool, playlist_items: Optional[str]) -> str:
if playlist_items:
debug(f"[download-file] Skipping canonicalization for playlist item(s): {playlist_items}")
return str(requested_url)
try:
cf = None
@@ -1136,16 +1144,13 @@ class Download_File(Cmdlet):
cf = str(cookie_path)
except Exception:
cf = None
debug(f"[download-file] Canonicalizing URL: {requested_url}")
pr = probe_url(requested_url, no_playlist=False, timeout_seconds=15, cookiefile=cf)
if isinstance(pr, dict):
for key in ("webpage_url", "original_url", "url", "requested_url"):
value = pr.get(key)
if isinstance(value, str) and value.strip():
canon = value.strip()
if canon != requested_url:
debug(f"[download-file] Resolved canonical URL: {requested_url} -> {canon}")
return canon
except Exception as e:
debug(f"[download-file] Canonicalization error for {requested_url}: {e}")
@@ -1180,7 +1185,8 @@ class Download_File(Cmdlet):
urls=unique_to_check,
storage=storage,
hydrus_available=hydrus_available,
final_output_dir=final_output_dir
final_output_dir=final_output_dir,
auto_continue_duplicates=False,
)
def _preflight_url_duplicates_bulk(
@@ -1204,7 +1210,8 @@ class Download_File(Cmdlet):
urls=unique_urls,
storage=storage,
hydrus_available=hydrus_available,
final_output_dir=final_output_dir
final_output_dir=final_output_dir,
auto_continue_duplicates=False,
)
@@ -1512,6 +1519,7 @@ class Download_File(Cmdlet):
download_timeout_seconds: int,
) -> int:
downloaded_count = 0
duplicate_skipped_count = 0
downloaded_pipe_objects: List[Dict[str, Any]] = []
pipe_seq = 0
clip_sections_spec = self._build_clip_sections_spec(clip_ranges)
@@ -1527,9 +1535,6 @@ class Download_File(Cmdlet):
for url_index, url in enumerate(supported_url, 1):
try:
debug(f"[download-file] Processing URL in loop: {url}")
debug(f"[download-file] ytdl_format parameter passed in: {ytdl_format}")
display_total = batch_total if batch_total > 0 else total_urls
display_index = batch_index if batch_total > 0 else url_index
display_label = batch_label or str(url)
@@ -1543,7 +1548,6 @@ class Download_File(Cmdlet):
)
if not skip_per_url_preflight:
debug(f"[download-file] Running duplicate preflight for: {canonical_url}")
if not self._preflight_url_duplicate(
storage=storage,
hydrus_available=hydrus_available,
@@ -1551,9 +1555,25 @@ class Download_File(Cmdlet):
candidate_url=canonical_url,
extra_urls=[url],
):
duplicate_skipped_count += 1
log(f"Skipping download (duplicate found): {url}", file=sys.stderr)
continue
try:
debug_panel(
f"Download item {display_index}/{display_total or total_urls}",
[
("url", url),
("canonical_url", canonical_url),
("mode", mode),
("format", ytdl_format or "auto"),
("duplicate_preflight", not skip_per_url_preflight),
],
border_style="green",
)
except Exception:
pass
if aggregate_status_mode:
try:
if display_total > 0:
@@ -1682,11 +1702,7 @@ class Download_File(Cmdlet):
actual_format = f"{actual_format}+bestaudio"
except Exception as e:
pass
debug(
"[download-file] Resolved format for download: "
f"mode={mode}, format={actual_format or 'default'}, playlist_items={actual_playlist_items}"
)
attempted_single_format_fallback = False
attempted_audio_fallback_specific = False
@@ -1709,9 +1725,7 @@ class Download_File(Cmdlet):
if not aggregate_status_mode:
PipelineProgress(pipeline_context).step("downloading")
debug(f"Starting download for {url} (format: {actual_format or 'default'}) with {download_timeout_seconds}s activity timeout...")
result_obj = _download_with_timeout(opts, timeout_seconds=download_timeout_seconds, config=config)
debug(f"Download completed for {url}, building pipe object...")
break
except DownloadError as e:
cause = getattr(e, "__cause__", None)
@@ -2028,8 +2042,6 @@ class Download_File(Cmdlet):
except Exception:
pass
debug(f"Emitting {len(pipe_objects)} result(s) to pipeline...")
if not aggregate_status_mode:
PipelineProgress(pipeline_context).step("finalized")
@@ -2049,7 +2061,17 @@ class Download_File(Cmdlet):
pass
downloaded_count += len(pipe_objects)
debug("✓ Downloaded and emitted")
try:
debug_panel(
"download-file result",
[
("emitted", len(pipe_objects)),
("url", url),
],
border_style="green",
)
except Exception:
pass
except DownloadError as e:
log(f"Download failed for {url}: {e}", file=sys.stderr)
@@ -2057,7 +2079,8 @@ class Download_File(Cmdlet):
log(f"Error processing {url}: {e}", file=sys.stderr)
if downloaded_count > 0:
debug(f"✓ Successfully processed {downloaded_count} URL(s)")
return 0
if duplicate_skipped_count > 0:
return 0
log("No downloads completed", file=sys.stderr)
@@ -2072,7 +2095,6 @@ class Download_File(Cmdlet):
parsed: Dict[str, Any],
) -> int:
try:
debug("Starting streaming download handler")
suppress_nested, _batch_total, _batch_index, _batch_label = self._batch_progress_state(config)
ytdlp_tool = YtDlpTool(config)
@@ -2091,21 +2113,18 @@ class Download_File(Cmdlet):
if not final_output_dir:
return 1
debug(f"Output directory: {final_output_dir}")
progress = PipelineProgress(pipeline_context)
using_shared_ui = pipeline_context.get_stage_context() is not None
try:
# If we are already in a pipeline stage, the parent UI is already handling progress.
# Calling ensure_local_ui can cause re-initialization hangs on some platforms.
if pipeline_context.get_stage_context() is None:
debug("[download-file] Initializing local UI...")
if not using_shared_ui:
progress.ensure_local_ui(
label="download-file",
total_items=len(supported_url),
items_preview=supported_url,
)
else:
debug("[download-file] Skipping local UI: running inside pipeline stage")
try:
if not suppress_nested:
progress.begin_pipe(
@@ -2116,8 +2135,6 @@ class Download_File(Cmdlet):
debug(f"[download-file] PipelineProgress begin_pipe error: {err}")
except Exception as e:
debug(f"[download-file] PipelineProgress update error: {e}")
debug("[download-file] Parsing clip and query specs...")
clip_spec = parsed.get("clip")
query_spec = parsed.get("query")
@@ -2223,7 +2240,6 @@ class Download_File(Cmdlet):
ytdl_format = query_format
if not ytdl_format:
debug(f"[download-file] Checking for playlist at {candidate_url}...")
if self._maybe_show_playlist_table(url=candidate_url, ytdlp_tool=ytdlp_tool):
playlist_selection_handled = True
# ... (existing logging code) ...
@@ -2270,7 +2286,6 @@ class Download_File(Cmdlet):
forced_single_format_id = None
forced_single_format_for_batch = False
debug("[download-file] Checking if format table should be shown...")
early_ret = self._maybe_show_format_table_for_single_url(
mode=mode,
clip_spec=clip_spec,
@@ -2343,7 +2358,23 @@ class Download_File(Cmdlet):
except Exception:
timeout_seconds = 300
debug(f"[download-file] Proceeding to final download call for {len(supported_url)} URL(s)...")
try:
debug_panel(
"Streaming download",
[
("urls", len(supported_url)),
("mode", mode),
("format", ytdl_format or "auto"),
("output_dir", final_output_dir),
("ui", "shared pipeline" if using_shared_ui else "local"),
("playlist_items", playlist_items),
("skip_preflight", skip_per_url_preflight),
("timeout_seconds", timeout_seconds),
],
border_style="blue",
)
except Exception:
pass
return self._download_supported_urls(
supported_url=supported_url,
ytdlp_tool=ytdlp_tool,
@@ -2855,8 +2886,6 @@ class Download_File(Cmdlet):
prev_progress = None
had_progress_key = False
try:
debug("Starting download-file")
# Allow providers to tap into the active PipelineProgress (optional).
try:
if isinstance(config, dict):
@@ -3117,7 +3146,6 @@ class Download_File(Cmdlet):
streaming_exit_code: Optional[int] = None
streaming_downloaded = 0
if supported_streaming:
debug(f"[download-file] Using ytdlp provider for {len(supported_streaming)} URL(s)")
streaming_exit_code = self._run_streaming_urls(
streaming_urls=supported_streaming,
args=args,
@@ -3161,7 +3189,19 @@ class Download_File(Cmdlet):
if not final_output_dir:
return 1
debug(f"Output directory: {final_output_dir}")
try:
debug_panel(
"download-file plan",
[
("output_dir", final_output_dir),
("streaming_urls", len(supported_streaming)),
("remaining_urls", len(raw_url)),
("piped_items", len(piped_items) if isinstance(piped_items, list) else int(bool(piped_items))),
],
border_style="cyan",
)
except Exception:
debug(f"Output directory: {final_output_dir}")
# If the caller isn't running the shared pipeline Live progress UI (e.g. direct
# cmdlet execution), start a minimal local pipeline progress panel so downloads
cmdlet/search_file.py
-8
View File
@@ -2158,11 +2158,7 @@ class search_file(Cmdlet):
)
db.update_worker_status(worker_id, "error")
return 1
debug(f"[search-file] Searching '{backend_to_search}'")
results = target_backend.search(query, limit=limit)
debug(
f"[search-file] '{backend_to_search}' -> {len(results or [])} result(s)"
)
else:
all_results = []
store_registry = None
@@ -2184,14 +2180,10 @@ class search_file(Cmdlet):
if type(backend).search is BaseStore.search:
continue
debug(f"[search-file] Searching '{backend_name}'")
backend_results = backend.search(
query,
limit=limit - len(all_results)
)
debug(
f"[search-file] '{backend_name}' -> {len(backend_results or [])} result(s)"
)
if backend_results:
all_results.extend(backend_results)
if len(all_results) >= limit:
tool/ytdlp.py
+41 -32
View File
@@ -19,7 +19,7 @@ from urllib.parse import urlparse
from SYS import pipeline as pipeline_context
from SYS.config import get_nested_config_value as _get_nested
from SYS.logger import debug, log
from SYS.logger import debug, debug_panel, log
from SYS.models import (
DebugLogger,
DownloadError,
@@ -505,7 +505,6 @@ def probe_url(
def _do_probe() -> None:
try:
debug(f"[probe] Starting probe for {url}")
ensure_yt_dlp_ready()
assert yt_dlp is not None
@@ -529,9 +528,7 @@ def probe_url(
ydl_opts["noplaylist"] = True
with yt_dlp.YoutubeDL(ydl_opts) as ydl: # type: ignore[arg-type]
debug(f"[probe] ytdlp extract_info (download=False) start: {url}")
info = ydl.extract_info(url, download=False)
debug(f"[probe] ytdlp extract_info (download=False) done: {url}")
if not isinstance(info, dict):
result_container[0] = None
@@ -550,6 +547,16 @@ def probe_url(
"webpage_url": webpage_url,
"url": webpage_url or url,
}
debug_panel(
"yt-dlp probe",
[
("url", url),
("extractor", info.get("extractor", "")),
("title", info.get("title", "")),
("playlist", bool(info.get("entries"))),
],
border_style="green",
)
except Exception as exc:
debug(f"Probe error for {url}: {exc}")
result_container[1] = exc
@@ -1112,12 +1119,6 @@ class YtDlpTool:
if clip_safe_fmt != fmt:
debug(f"[ytdlp] clip format normalized: {fmt} -> {clip_safe_fmt}")
fmt = clip_safe_fmt
debug(
"[ytdlp] build options: "
f"mode={opts.mode}, ytdl_format={ytdl_format}, default_format={default_fmt}, final_format={fmt}, "
f"cookiefile={base_options.get('cookiefile')}, cookiesfrombrowser={base_options.get('cookiesfrombrowser')}, "
f"player_client={((base_options.get('extractor_args') or {}).get('youtube') or {}).get('player_client')}"
)
base_options["format"] = fmt
if opts.mode == "audio":
@@ -1199,9 +1200,6 @@ class YtDlpTool:
if opts.playlist_items:
base_options["playlist_items"] = opts.playlist_items
if not opts.quiet:
debug(f"yt-dlp: mode={opts.mode}, format={base_options.get('format')}, cookiefile={base_options.get('cookiefile')}")
return base_options
def build_yt_dlp_cli_args(
@@ -2100,7 +2098,6 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] =
format) are applied when constructing the YtDlpTool instance.
"""
debug(f"[download_media] start: {opts.url}")
try:
netloc = urlparse(opts.url).netloc.lower()
except Exception:
@@ -2123,9 +2120,6 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] =
raise DownloadError(msg)
if opts.playlist_items:
debug(
f"Skipping probe for playlist (item selection: {opts.playlist_items}), proceeding with download"
)
probe_result: Optional[Dict[str, Any]] = {"url": opts.url}
else:
probe_cookiefile = None
@@ -2138,8 +2132,6 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] =
probe_result = probe_url(opts.url, no_playlist=opts.no_playlist, timeout_seconds=15, cookiefile=probe_cookiefile)
if probe_result is None:
if not opts.quiet:
debug("yt-dlp probe returned no metadata; continuing with direct download attempt")
if debug_logger is not None:
debug_logger.write_record("ytdlp-probe-miss-continue", {"url": opts.url})
@@ -2155,7 +2147,18 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] =
if _progress_callback not in hooks:
hooks.append(_progress_callback)
if not opts.quiet:
debug(f"Starting yt-dlp download: {opts.url}")
debug_panel(
"yt-dlp job",
[
("url", opts.url),
("mode", opts.mode),
("format", ytdl_options.get("format") or "default"),
("cookiefile", ytdl_options.get("cookiefile")),
("download_sections", ytdl_options.get("download_sections") or "None"),
("force_keyframes", ytdl_options.get("force_keyframes_at_cuts", False)),
],
border_style="blue",
)
if debug_logger is not None:
debug_logger.write_record("ytdlp-start", {"url": opts.url})
@@ -2164,11 +2167,6 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] =
session_id = None
first_section_info: Dict[str, Any] = {}
try:
if not opts.quiet:
if ytdl_options.get("download_sections"):
debug(f"[yt-dlp] download_sections: {ytdl_options['download_sections']}")
debug(f"[yt-dlp] force_keyframes_at_cuts: {ytdl_options.get('force_keyframes_at_cuts', False)}")
if ytdl_options.get("download_sections"):
live_ui, _ = PipelineProgress(pipeline_context).ui_and_pipe_index()
quiet_sections = bool(opts.quiet) or (live_ui is not None)
@@ -2354,18 +2352,21 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] =
media_paths = renamed_media_files if renamed_media_files else None
if not opts.quiet:
count = len(media_paths) if isinstance(media_paths, list) else 1
debug(f"✓ Downloaded {count} section media file(s) (session: {session_id})")
debug_panel(
"yt-dlp result",
[
("files", count),
("session", session_id),
("file", media_path.name),
],
border_style="green",
)
else:
media_path = files[0]
media_paths = None
if not opts.quiet:
debug(f"✓ Downloaded section file (pattern not found): {media_path.name}")
else:
media_path = files[0]
media_paths = None
if not opts.quiet:
debug(f"✓ Downloaded: {media_path.name}")
if debug_logger is not None:
debug_logger.write_record("ytdlp-file-found", {"path": str(media_path)})
except Exception as exc:
@@ -2462,7 +2463,15 @@ def download_media(opts: DownloadOptions, *, config: Optional[Dict[str, Any]] =
source_url = entry.get("webpage_url") or entry.get("original_url") or entry.get("url")
if not opts.quiet:
debug(f"✓ Downloaded: {media_path.name} ({len(tags_res)} tags)")
debug_panel(
"yt-dlp result",
[
("file", media_path.name),
("tag_count", len(tags_res)),
("source_url", source_url or opts.url),
],
border_style="green",
)
if debug_logger is not None:
debug_logger.write_record(
"downloaded",