update refactoring and add new features
This commit is contained in:
@@ -0,0 +1,897 @@
|
||||
"""Generic file/stream downloader — core orchestration.
|
||||
|
||||
Supports:
|
||||
- Direct HTTP file URLs (PDFs, images, documents; non-yt-dlp)
|
||||
- Piped plugin items (uses plugin.download when available)
|
||||
- Streaming sites via yt-dlp (YouTube, Bandcamp, etc.)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence as SequenceABC
|
||||
import sys
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
from urllib.parse import urlparse
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
import shutil
|
||||
import webbrowser
|
||||
|
||||
from API.HTTP import download_direct_file
|
||||
from SYS.models import DownloadError, DownloadOptions, DownloadMediaResult
|
||||
from SYS.logger import log, debug_panel, is_debug_enabled
|
||||
from SYS.payload_builders import build_file_result_payload, build_table_result_payload
|
||||
from SYS.pipeline_progress import PipelineProgress
|
||||
from SYS.result_table import Table, build_display_row
|
||||
from SYS.rich_display import stderr_console as get_stderr_console
|
||||
from SYS import pipeline as pipeline_context
|
||||
from SYS.item_accessors import get_result_title
|
||||
from rich.prompt import Prompt
|
||||
from SYS.selection_builder import (
|
||||
build_hash_store_selection,
|
||||
extract_selection_fields,
|
||||
extract_urls_from_selection_args,
|
||||
selection_args_have_url,
|
||||
)
|
||||
from SYS.utils import sha256_file
|
||||
|
||||
try:
|
||||
from plugins.ytdlp import YtDlpTool # type: ignore
|
||||
except Exception: # pragma: no cover - optional dependency for tests/runtime wrappers
|
||||
YtDlpTool = None # type: ignore
|
||||
|
||||
from .. import _shared as sh
|
||||
|
||||
Cmdlet = sh.Cmdlet
|
||||
CmdletArg = sh.CmdletArg
|
||||
SharedArgs = sh.SharedArgs
|
||||
QueryArg = sh.QueryArg
|
||||
parse_cmdlet_args = sh.parse_cmdlet_args
|
||||
register_url_with_local_library = sh.register_url_with_local_library
|
||||
coerce_to_pipe_object = sh.coerce_to_pipe_object
|
||||
get_field = sh.get_field
|
||||
resolve_target_dir = sh.resolve_target_dir
|
||||
coerce_to_path = sh.coerce_to_path
|
||||
build_pipeline_preview = sh.build_pipeline_preview
|
||||
|
||||
|
||||
class Download_File(Cmdlet):
|
||||
"""Class-based download-file cmdlet - direct HTTP downloads."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""Initialize download-file cmdlet."""
|
||||
super().__init__(
|
||||
name="download-file",
|
||||
summary="Download files or streaming media",
|
||||
usage=
|
||||
"download-file <url|path> [-plugin NAME] [-instance NAME] [-path DIR] [options] OR @N | download-file [-plugin NAME] [-instance NAME] [-path DIR] [options] OR download-file -query \"hash:<sha256>\" -instance <store> [-browser]",
|
||||
alias=["dl-file",
|
||||
"download-http"],
|
||||
arg=[
|
||||
SharedArgs.URL,
|
||||
SharedArgs.PLUGIN,
|
||||
SharedArgs.INSTANCE,
|
||||
SharedArgs.PATH,
|
||||
SharedArgs.QUERY,
|
||||
CmdletArg(
|
||||
name="name",
|
||||
type="string",
|
||||
description="Output filename override for store exports.",
|
||||
),
|
||||
CmdletArg(
|
||||
name="browser",
|
||||
type="flag",
|
||||
description="Open a backend-provided browser URL instead of exporting to disk when available.",
|
||||
),
|
||||
QueryArg(
|
||||
"clip",
|
||||
key="clip",
|
||||
aliases=["range",
|
||||
"section",
|
||||
"sections"],
|
||||
type="string",
|
||||
required=False,
|
||||
description=(
|
||||
"Clip time ranges via -query keyed fields (e.g. clip:1m-2m or clip:00:01-00:10). "
|
||||
"Comma-separated values supported."
|
||||
),
|
||||
query_only=True,
|
||||
),
|
||||
CmdletArg(
|
||||
name="item",
|
||||
type="string",
|
||||
description="Item selection for playlists/formats",
|
||||
),
|
||||
],
|
||||
detail=[
|
||||
"Download files directly via HTTP or streaming media via yt-dlp.",
|
||||
"Also exports store-backed files via hash+store selection or -query \"hash:<sha256>\" -instance <store>.",
|
||||
"Use -plugin with -instance to target a named provider config when a plugin exposes multiple instances.",
|
||||
"For Internet Archive item pages (archive.org/details/...), shows a selectable file/format list; pick with @N to download.",
|
||||
],
|
||||
exec=self.run,
|
||||
)
|
||||
self.register()
|
||||
|
||||
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
"""Main execution method."""
|
||||
try:
|
||||
debug_panel(
|
||||
"download-file",
|
||||
[
|
||||
("args", list(args)),
|
||||
("has_piped_input", bool(result)),
|
||||
],
|
||||
border_style="cyan",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return self._run_impl(result, args, config)
|
||||
|
||||
@staticmethod
|
||||
def _path_from_download_result(result_obj: Any) -> Path:
|
||||
"""Normalize downloader return values to a concrete filesystem path."""
|
||||
resolved = coerce_to_path(result_obj)
|
||||
if resolved is None:
|
||||
raise DownloadError("Could not determine downloaded file path")
|
||||
return resolved
|
||||
|
||||
@staticmethod
|
||||
def _selection_run_label(
|
||||
run_args: Sequence[str],
|
||||
*,
|
||||
extra_url_prefixes: Sequence[str] = (),
|
||||
) -> str:
|
||||
try:
|
||||
urls = extract_urls_from_selection_args(
|
||||
run_args,
|
||||
extra_url_prefixes=extra_url_prefixes,
|
||||
)
|
||||
if urls:
|
||||
return str(urls[0])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for arg in run_args:
|
||||
text = str(arg or "").strip()
|
||||
if text and not text.startswith("-"):
|
||||
return text
|
||||
return "item"
|
||||
|
||||
@staticmethod
|
||||
def _batch_progress_state(config: Optional[Dict[str, Any]]) -> tuple[bool, int, int, str]:
|
||||
if not isinstance(config, dict):
|
||||
return False, 0, 0, ""
|
||||
|
||||
suppress_nested = bool(config.get("_download_file_suppress_nested_pipe_progress"))
|
||||
if not suppress_nested:
|
||||
return False, 0, 0, ""
|
||||
|
||||
try:
|
||||
total = max(0, int(config.get("_download_file_batch_total") or 0))
|
||||
except Exception:
|
||||
total = 0
|
||||
try:
|
||||
index = max(0, int(config.get("_download_file_batch_index") or 0))
|
||||
except Exception:
|
||||
index = 0
|
||||
try:
|
||||
label = str(config.get("_download_file_batch_label") or "").strip()
|
||||
except Exception:
|
||||
label = ""
|
||||
|
||||
return True, total, index, label
|
||||
|
||||
@staticmethod
|
||||
def _selection_url_prefixes(registry: Dict[str, Any]) -> List[str]:
|
||||
loader = registry.get("list_selection_url_prefixes")
|
||||
if not callable(loader):
|
||||
return []
|
||||
try:
|
||||
values = loader() or []
|
||||
except Exception:
|
||||
return []
|
||||
return [str(value).strip().lower() for value in values if str(value or "").strip()]
|
||||
|
||||
def _normalize_provider_key(self, value: Optional[Any]) -> Optional[str]:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
normalized = str(value).strip()
|
||||
except Exception:
|
||||
return None
|
||||
if not normalized:
|
||||
return None
|
||||
if "." in normalized:
|
||||
normalized = normalized.split(".", 1)[0]
|
||||
return normalized.lower()
|
||||
|
||||
def _provider_key_from_item(self, item: Any) -> Optional[str]:
|
||||
table_hint = get_field(item, "table")
|
||||
key = self._normalize_provider_key(table_hint)
|
||||
if key:
|
||||
return key
|
||||
provider_hint = get_field(item, "plugin")
|
||||
key = self._normalize_provider_key(provider_hint)
|
||||
if key:
|
||||
return key
|
||||
return self._normalize_provider_key(get_field(item, "source"))
|
||||
|
||||
@staticmethod
|
||||
def _path_looks_local(value: Any) -> bool:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
if text.startswith(("http://", "https://", "ftp://", "ftps://", "magnet:", "torrent:")):
|
||||
return False
|
||||
if len(text) >= 2 and text[1] == ":":
|
||||
return True
|
||||
if text.startswith(("\\", "/", ".", "~")):
|
||||
return True
|
||||
return Path(text).exists()
|
||||
|
||||
@staticmethod
|
||||
def _resolve_display_title(result: Any, metadata: Optional[Dict[str, Any]]) -> str:
|
||||
candidates = [
|
||||
get_result_title(result, "title", "name", "filename"),
|
||||
get_result_title(metadata or {}, "title", "name", "filename"),
|
||||
]
|
||||
for candidate in candidates:
|
||||
if candidate is None:
|
||||
continue
|
||||
text = str(candidate).strip()
|
||||
if text:
|
||||
return text
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_export_filename(name: str) -> str:
|
||||
allowed_chars: List[str] = []
|
||||
for ch in str(name or ""):
|
||||
if ch.isalnum() or ch in {"-", "_", " ", "."}:
|
||||
allowed_chars.append(ch)
|
||||
else:
|
||||
allowed_chars.append(" ")
|
||||
sanitized = " ".join("".join(allowed_chars).split())
|
||||
return sanitized or "export"
|
||||
|
||||
@staticmethod
|
||||
def _unique_export_path(path: Path) -> Path:
|
||||
if not path.exists():
|
||||
return path
|
||||
stem = path.stem
|
||||
suffix = path.suffix
|
||||
parent = path.parent
|
||||
counter = 1
|
||||
while True:
|
||||
candidate = parent / f"{stem} ({counter}){suffix}"
|
||||
if not candidate.exists():
|
||||
return candidate
|
||||
counter += 1
|
||||
|
||||
@staticmethod
|
||||
def _load_provider_registry() -> Dict[str, Any]:
|
||||
"""Lightweight accessor for plugin helpers without hard dependencies."""
|
||||
try:
|
||||
from PluginCore import registry as provider_registry # type: ignore
|
||||
from PluginCore.base import SearchResult # type: ignore
|
||||
|
||||
return {
|
||||
"get_plugin": getattr(provider_registry, "get_plugin", None),
|
||||
"match_plugin_name_for_url": getattr(provider_registry, "match_plugin_name_for_url", None),
|
||||
"list_selection_url_prefixes": getattr(provider_registry, "list_selection_url_prefixes", None),
|
||||
"SearchResult": SearchResult,
|
||||
}
|
||||
except Exception:
|
||||
return {
|
||||
"get_plugin": None,
|
||||
"match_plugin_name_for_url": None,
|
||||
"list_selection_url_prefixes": None,
|
||||
"SearchResult": None,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def _normalize_duplicate_preflight_policy(cls, value: Any) -> Optional[str]:
|
||||
text = str(value or "").strip().lower()
|
||||
if not text:
|
||||
return "skip"
|
||||
mapping = {
|
||||
"i": "ignore",
|
||||
"ignore": "ignore",
|
||||
"s": "skip",
|
||||
"skip": "skip",
|
||||
"c": "cancel",
|
||||
"cancel": "cancel",
|
||||
}
|
||||
return mapping.get(text)
|
||||
|
||||
def _maybe_render_download_details(self, *, config: Dict[str, Any]) -> None:
|
||||
try:
|
||||
stage_ctx = pipeline_context.get_stage_context()
|
||||
except Exception:
|
||||
stage_ctx = None
|
||||
|
||||
is_last_stage = (stage_ctx is None) or bool(getattr(stage_ctx, "is_last_stage", False))
|
||||
if not is_last_stage:
|
||||
return
|
||||
|
||||
try:
|
||||
quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False
|
||||
except Exception:
|
||||
quiet_mode = False
|
||||
if quiet_mode:
|
||||
return
|
||||
|
||||
emitted_items: List[Any] = []
|
||||
try:
|
||||
emitted_items = list(getattr(stage_ctx, "emits", None) or []) if stage_ctx is not None else []
|
||||
except Exception:
|
||||
emitted_items = []
|
||||
|
||||
if not emitted_items:
|
||||
return
|
||||
|
||||
try:
|
||||
live_progress = pipeline_context.get_live_progress()
|
||||
except Exception:
|
||||
live_progress = None
|
||||
|
||||
if live_progress is not None:
|
||||
try:
|
||||
pipe_idx = getattr(stage_ctx, "pipe_index", None) if stage_ctx is not None else None
|
||||
if isinstance(pipe_idx, int):
|
||||
live_progress.finish_pipe(int(pipe_idx), force_complete=True)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
live_progress.stop()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if hasattr(pipeline_context, "set_live_progress"):
|
||||
pipeline_context.set_live_progress(None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
subject = emitted_items[0] if len(emitted_items) == 1 else list(emitted_items)
|
||||
from .._shared import display_and_persist_items
|
||||
display_and_persist_items(list(emitted_items), title="Result", subject=subject)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
if stage_ctx is not None:
|
||||
stage_ctx.emits = []
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _maybe_show_provider_picker(
|
||||
self,
|
||||
*,
|
||||
raw_urls: Sequence[str],
|
||||
piped_items: Sequence[Any],
|
||||
parsed: Dict[str, Any],
|
||||
config: Dict[str, Any],
|
||||
registry: Dict[str, Any],
|
||||
) -> Optional[int]:
|
||||
"""Generic hook for providers to show a selection table (e.g. Internet Archive format picker)."""
|
||||
total_inputs = len(raw_urls or []) + len(piped_items or [])
|
||||
if total_inputs != 1:
|
||||
return None
|
||||
|
||||
target_url = None
|
||||
if raw_urls:
|
||||
target_url = str(raw_urls[0])
|
||||
elif piped_items:
|
||||
target_url = str(get_field(piped_items[0], "path") or get_field(piped_items[0], "url") or "")
|
||||
|
||||
if not target_url:
|
||||
return None
|
||||
|
||||
match_provider_name_for_url = registry.get("match_plugin_name_for_url")
|
||||
get_provider = registry.get("get_plugin")
|
||||
|
||||
provider_name = None
|
||||
if match_provider_name_for_url:
|
||||
try:
|
||||
provider_name = match_provider_name_for_url(target_url)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if provider_name and get_provider:
|
||||
provider = get_provider(provider_name, config)
|
||||
if provider and hasattr(provider, "maybe_show_picker"):
|
||||
try:
|
||||
quiet_mode = bool(config.get("_quiet_background_output"))
|
||||
res = provider.maybe_show_picker(
|
||||
url=target_url,
|
||||
item=piped_items[0] if piped_items else None,
|
||||
parsed=parsed,
|
||||
config=config,
|
||||
quiet_mode=quiet_mode,
|
||||
)
|
||||
if res is not None:
|
||||
return int(res)
|
||||
except Exception as e:
|
||||
debug_panel(
|
||||
"download-file picker error",
|
||||
[
|
||||
("plugin", provider_name),
|
||||
("url", target_url),
|
||||
("error", e),
|
||||
],
|
||||
border_style="yellow",
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
def _run_impl(
|
||||
self,
|
||||
result: Any,
|
||||
args: Sequence[str],
|
||||
config: Dict[str, Any]
|
||||
) -> int:
|
||||
"""Main download implementation for direct HTTP files."""
|
||||
progress = PipelineProgress(pipeline_context)
|
||||
prev_progress = None
|
||||
had_progress_key = False
|
||||
try:
|
||||
try:
|
||||
if isinstance(config, dict):
|
||||
had_progress_key = "_pipeline_progress" in config
|
||||
prev_progress = config.get("_pipeline_progress")
|
||||
config["_pipeline_progress"] = progress
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
parsed = parse_cmdlet_args(args, self)
|
||||
registry = self._load_provider_registry()
|
||||
selection_url_prefixes = self._selection_url_prefixes(registry)
|
||||
explicit_input = parsed.get("url")
|
||||
|
||||
url_candidates = parsed.get("url") or [
|
||||
a for a in parsed.get("args", [])
|
||||
if isinstance(a, str) and (
|
||||
a.startswith("http") or "://" in a or ":" in a
|
||||
or "\U0001f9f2" in a
|
||||
and not a.startswith("-")
|
||||
)
|
||||
]
|
||||
from SYS.metadata import normalize_urls as normalize_url_list # lazy: avoids Cryptodome at startup
|
||||
raw_url = normalize_url_list(url_candidates)
|
||||
local_source_inputs: List[str] = []
|
||||
if not raw_url and isinstance(explicit_input, str) and self._path_looks_local(explicit_input):
|
||||
local_source_inputs = [str(explicit_input)]
|
||||
|
||||
quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False
|
||||
|
||||
piped_items = []
|
||||
if not raw_url:
|
||||
piped_items = sh.normalize_result_items(
|
||||
result,
|
||||
include_falsey_single=True,
|
||||
)
|
||||
|
||||
if piped_items and not raw_url:
|
||||
selection_runs: List[List[str]] = []
|
||||
residual_items: List[Any] = []
|
||||
|
||||
for item in piped_items:
|
||||
handled = False
|
||||
try:
|
||||
normalized_args, _normalized_action, item_url = extract_selection_fields(
|
||||
item,
|
||||
extra_url_prefixes=selection_url_prefixes,
|
||||
)
|
||||
if normalized_args:
|
||||
if selection_args_have_url(
|
||||
normalized_args,
|
||||
extra_url_prefixes=selection_url_prefixes,
|
||||
):
|
||||
selection_runs.append(list(normalized_args))
|
||||
handled = True
|
||||
elif item_url:
|
||||
selection_runs.append([str(item_url)] + list(normalized_args))
|
||||
handled = True
|
||||
except Exception as e:
|
||||
debug_panel(
|
||||
"download-file selection args failed",
|
||||
[("error", e)],
|
||||
border_style="yellow",
|
||||
)
|
||||
handled = False
|
||||
if not handled:
|
||||
residual_items.append(item)
|
||||
if selection_runs:
|
||||
selection_urls: List[str] = []
|
||||
|
||||
for run_args in selection_runs:
|
||||
for u in extract_urls_from_selection_args(
|
||||
run_args,
|
||||
extra_url_prefixes=selection_url_prefixes,
|
||||
):
|
||||
if u not in selection_urls:
|
||||
selection_urls.append(u)
|
||||
|
||||
original_skip_preflight = None
|
||||
original_timeout = None
|
||||
original_skip_direct = None
|
||||
original_batch_total = None
|
||||
original_batch_index = None
|
||||
original_batch_label = None
|
||||
original_suppress_nested = None
|
||||
try:
|
||||
if isinstance(config, dict):
|
||||
original_skip_preflight = config.get("_skip_url_preflight")
|
||||
original_timeout = config.get("_pipeobject_timeout_seconds")
|
||||
original_skip_direct = config.get("_skip_direct_on_streaming_failure")
|
||||
original_batch_total = config.get("_download_file_batch_total")
|
||||
original_batch_index = config.get("_download_file_batch_index")
|
||||
original_batch_label = config.get("_download_file_batch_label")
|
||||
original_suppress_nested = config.get("_download_file_suppress_nested_pipe_progress")
|
||||
except Exception:
|
||||
original_skip_preflight = None
|
||||
original_timeout = None
|
||||
original_batch_total = None
|
||||
original_batch_index = None
|
||||
original_batch_label = None
|
||||
original_suppress_nested = None
|
||||
|
||||
try:
|
||||
if selection_urls:
|
||||
if isinstance(config, dict):
|
||||
config["_skip_url_preflight"] = True
|
||||
config["_skip_direct_on_streaming_failure"] = True
|
||||
|
||||
if isinstance(config, dict) and config.get("_pipeobject_timeout_seconds") is None:
|
||||
config["_pipeobject_timeout_seconds"] = 600
|
||||
|
||||
successes = 0
|
||||
failures = 0
|
||||
last_code = 0
|
||||
total_selection = len(selection_runs)
|
||||
preview_items = list(selection_urls[:5]) or [
|
||||
self._selection_run_label(
|
||||
run_args,
|
||||
extra_url_prefixes=selection_url_prefixes,
|
||||
)
|
||||
for run_args in selection_runs[:5]
|
||||
]
|
||||
try:
|
||||
progress.ensure_local_ui(
|
||||
label="download-file",
|
||||
total_items=total_selection,
|
||||
items_preview=preview_items,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
progress.begin_pipe(
|
||||
total_items=total_selection,
|
||||
items_preview=preview_items,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
for idx, run_args in enumerate(selection_runs, 1):
|
||||
run_label = self._selection_run_label(
|
||||
run_args,
|
||||
extra_url_prefixes=selection_url_prefixes,
|
||||
)
|
||||
try:
|
||||
progress.set_status(
|
||||
f"downloading {idx}/{total_selection}: {run_label}"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if isinstance(config, dict):
|
||||
config["_download_file_batch_total"] = total_selection
|
||||
config["_download_file_batch_index"] = idx
|
||||
config["_download_file_batch_label"] = run_label
|
||||
config["_download_file_suppress_nested_pipe_progress"] = True
|
||||
except Exception:
|
||||
pass
|
||||
exit_code = self._run_impl(None, run_args, config)
|
||||
if exit_code == 0:
|
||||
successes += 1
|
||||
else:
|
||||
failures += 1
|
||||
last_code = exit_code
|
||||
|
||||
piped_items = residual_items
|
||||
if not piped_items:
|
||||
if successes > 0:
|
||||
return 0
|
||||
return last_code or 1
|
||||
finally:
|
||||
try:
|
||||
if isinstance(config, dict):
|
||||
if original_skip_preflight is None:
|
||||
config.pop("_skip_url_preflight", None)
|
||||
else:
|
||||
config["_skip_url_preflight"] = original_skip_preflight
|
||||
if original_timeout is None:
|
||||
config.pop("_pipeobject_timeout_seconds", None)
|
||||
else:
|
||||
config["_pipeobject_timeout_seconds"] = original_timeout
|
||||
if original_skip_direct is None:
|
||||
config.pop("_skip_direct_on_streaming_failure", None)
|
||||
else:
|
||||
config["_skip_direct_on_streaming_failure"] = original_skip_direct
|
||||
if original_batch_total is None:
|
||||
config.pop("_download_file_batch_total", None)
|
||||
else:
|
||||
config["_download_file_batch_total"] = original_batch_total
|
||||
if original_batch_index is None:
|
||||
config.pop("_download_file_batch_index", None)
|
||||
else:
|
||||
config["_download_file_batch_index"] = original_batch_index
|
||||
if original_batch_label is None:
|
||||
config.pop("_download_file_batch_label", None)
|
||||
else:
|
||||
config["_download_file_batch_label"] = original_batch_label
|
||||
if original_suppress_nested is None:
|
||||
config.pop("_download_file_suppress_nested_pipe_progress", None)
|
||||
else:
|
||||
config["_download_file_suppress_nested_pipe_progress"] = original_suppress_nested
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
had_piped_input = False
|
||||
try:
|
||||
if isinstance(result, list):
|
||||
had_piped_input = bool(result)
|
||||
else:
|
||||
had_piped_input = bool(result)
|
||||
except Exception:
|
||||
had_piped_input = False
|
||||
|
||||
if (had_piped_input and raw_url and len(raw_url) == 1
|
||||
and (not parsed.get("path"))):
|
||||
candidate = str(raw_url[0] or "").strip()
|
||||
low = candidate.lower()
|
||||
looks_like_url = low.startswith(
|
||||
("http://", "https://", "ftp://", "magnet:", "torrent:")
|
||||
+ tuple(selection_url_prefixes)
|
||||
)
|
||||
looks_like_provider = (
|
||||
":" in candidate and not candidate.startswith(
|
||||
("http:", "https:", "ftp:", "ftps:", "file:")
|
||||
+ tuple(selection_url_prefixes)
|
||||
)
|
||||
)
|
||||
looks_like_windows_path = (
|
||||
(len(candidate) >= 2 and candidate[1] == ":")
|
||||
or candidate.startswith("\\\\") or candidate.startswith("\\")
|
||||
or candidate.endswith(("\\", "/"))
|
||||
)
|
||||
if (not looks_like_url) and (
|
||||
not looks_like_provider) and looks_like_windows_path:
|
||||
parsed["path"] = candidate
|
||||
raw_url = []
|
||||
piped_items = sh.normalize_result_items(result, include_falsey_single=True)
|
||||
|
||||
if not raw_url and not piped_items and not local_source_inputs:
|
||||
log("No url or piped items to download", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
picker_result = self._maybe_show_provider_picker(
|
||||
raw_urls=raw_url,
|
||||
piped_items=piped_items,
|
||||
parsed=parsed,
|
||||
config=config,
|
||||
registry=registry,
|
||||
)
|
||||
if picker_result is not None:
|
||||
return int(picker_result)
|
||||
|
||||
picker_result = self._maybe_show_provider_picker(
|
||||
raw_urls=raw_url,
|
||||
piped_items=piped_items,
|
||||
parsed=parsed,
|
||||
config=config,
|
||||
registry=registry,
|
||||
)
|
||||
if picker_result is not None:
|
||||
return int(picker_result)
|
||||
|
||||
final_output_dir = resolve_target_dir(parsed, config)
|
||||
if not final_output_dir:
|
||||
return 1
|
||||
|
||||
try:
|
||||
debug_panel(
|
||||
"download-file plan",
|
||||
[
|
||||
("output_dir", final_output_dir),
|
||||
("remaining_urls", len(raw_url)),
|
||||
("piped_items", len(piped_items) if isinstance(piped_items, list) else int(bool(piped_items))),
|
||||
],
|
||||
border_style="cyan",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
total_items = max(1, len(raw_url or []) + len(piped_items or []))
|
||||
preview = build_pipeline_preview(raw_url, piped_items)
|
||||
|
||||
progress.ensure_local_ui(
|
||||
label="download-file",
|
||||
total_items=total_items,
|
||||
items_preview=preview
|
||||
)
|
||||
|
||||
raw_url, preflight_exit, skipped_dupe_count = self._preflight_explicit_url_duplicates(
|
||||
raw_urls=raw_url,
|
||||
config=config,
|
||||
)
|
||||
if preflight_exit is not None:
|
||||
return int(preflight_exit)
|
||||
|
||||
downloaded_count = 0
|
||||
|
||||
if local_source_inputs:
|
||||
downloaded_count += self._process_explicit_local_sources(
|
||||
local_sources=local_source_inputs,
|
||||
final_output_dir=final_output_dir,
|
||||
parsed=parsed,
|
||||
progress=progress,
|
||||
config=config,
|
||||
)
|
||||
|
||||
storage_downloaded, piped_items, storage_exit = self._process_storage_items(
|
||||
piped_items=piped_items,
|
||||
parsed=parsed,
|
||||
config=config,
|
||||
final_output_dir=final_output_dir,
|
||||
)
|
||||
downloaded_count += int(storage_downloaded)
|
||||
if storage_exit is not None:
|
||||
return int(storage_exit)
|
||||
|
||||
if skipped_dupe_count and not raw_url and not piped_items:
|
||||
return 0 if downloaded_count > 0 else 0
|
||||
|
||||
urls_downloaded, early_exit = self._process_explicit_urls(
|
||||
raw_urls=raw_url,
|
||||
final_output_dir=final_output_dir,
|
||||
config=config,
|
||||
quiet_mode=quiet_mode,
|
||||
registry=registry,
|
||||
progress=progress,
|
||||
parsed=parsed,
|
||||
args=args,
|
||||
context_items=(result if isinstance(result, list) else ([result] if result else [])),
|
||||
)
|
||||
downloaded_count += int(urls_downloaded)
|
||||
if early_exit is not None:
|
||||
return int(early_exit)
|
||||
|
||||
provider_downloaded, magnet_submissions = self._process_provider_items(
|
||||
piped_items=piped_items,
|
||||
final_output_dir=final_output_dir,
|
||||
config=config,
|
||||
quiet_mode=quiet_mode,
|
||||
registry=registry,
|
||||
progress=progress,
|
||||
)
|
||||
downloaded_count += provider_downloaded
|
||||
|
||||
if downloaded_count > 0 or magnet_submissions > 0:
|
||||
self._maybe_render_download_details(config=config)
|
||||
return 0
|
||||
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
log(f"Error in download-file: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
finally:
|
||||
try:
|
||||
if isinstance(config, dict):
|
||||
if had_progress_key:
|
||||
config["_pipeline_progress"] = prev_progress
|
||||
else:
|
||||
config.pop("_pipeline_progress", None)
|
||||
except Exception:
|
||||
pass
|
||||
progress.close_local_ui(force_complete=True)
|
||||
|
||||
|
||||
# Late-binding: import sub-module functions and attach them as methods.
|
||||
from .download_fetch import ( # noqa: E402
|
||||
_emit_plugin_items,
|
||||
_consume_plugin_download_result,
|
||||
_process_explicit_urls,
|
||||
_expand_provider_items,
|
||||
_process_provider_items,
|
||||
_download_provider_items,
|
||||
_emit_local_file,
|
||||
_resolve_provider_preflight_items,
|
||||
_build_provider_playlist_item_selector,
|
||||
_format_timecode,
|
||||
_rebase_subtitle_timestamp_text,
|
||||
_format_clip_range,
|
||||
_apply_clip_decorations,
|
||||
_parse_clip_spec_to_ranges,
|
||||
)
|
||||
|
||||
from .download_storage import ( # noqa: E402
|
||||
_iter_storage_export_refs,
|
||||
_export_store_file,
|
||||
_process_storage_items,
|
||||
_process_explicit_local_sources,
|
||||
_extract_hash_from_search_hit,
|
||||
_iter_duplicate_tag_values,
|
||||
_extract_duplicate_namespace_tags,
|
||||
_extract_duplicate_title_tag,
|
||||
_extract_duplicate_title,
|
||||
_has_duplicate_title,
|
||||
_build_duplicate_display_row,
|
||||
_fetch_duplicate_metadata_for_hash,
|
||||
_enrich_duplicate_metadata,
|
||||
_fetch_duplicate_metadata_for_hashes,
|
||||
_collect_existing_url_match_refs_for_url,
|
||||
_find_existing_url_matches_for_url,
|
||||
_find_existing_hash_for_url,
|
||||
_find_existing_hashes_for_url,
|
||||
_init_storage,
|
||||
_supports_storage_duplicate_lookup,
|
||||
_filter_supported_urls,
|
||||
_canonicalize_url_for_storage,
|
||||
_preflight_url_duplicate,
|
||||
_preflight_explicit_url_duplicates,
|
||||
_download_supported_urls,
|
||||
_maybe_show_playlist_table,
|
||||
_maybe_show_format_table_for_single_url,
|
||||
_run_streaming_urls,
|
||||
)
|
||||
|
||||
Download_File._emit_plugin_items = _emit_plugin_items
|
||||
Download_File._consume_plugin_download_result = _consume_plugin_download_result
|
||||
Download_File._process_explicit_urls = _process_explicit_urls
|
||||
Download_File._expand_provider_items = _expand_provider_items
|
||||
Download_File._process_provider_items = _process_provider_items
|
||||
Download_File._download_provider_items = _download_provider_items
|
||||
Download_File._emit_local_file = _emit_local_file
|
||||
Download_File._resolve_provider_preflight_items = staticmethod(_resolve_provider_preflight_items)
|
||||
Download_File._build_provider_playlist_item_selector = staticmethod(_build_provider_playlist_item_selector)
|
||||
Download_File._format_timecode = staticmethod(_format_timecode)
|
||||
Download_File._rebase_subtitle_timestamp_text = staticmethod(_rebase_subtitle_timestamp_text)
|
||||
Download_File._format_clip_range = classmethod(_format_clip_range)
|
||||
Download_File._apply_clip_decorations = classmethod(_apply_clip_decorations)
|
||||
Download_File._parse_clip_spec_to_ranges = staticmethod(_parse_clip_spec_to_ranges)
|
||||
|
||||
Download_File._iter_storage_export_refs = staticmethod(_iter_storage_export_refs)
|
||||
Download_File._export_store_file = _export_store_file
|
||||
Download_File._process_storage_items = _process_storage_items
|
||||
Download_File._process_explicit_local_sources = _process_explicit_local_sources
|
||||
Download_File._extract_hash_from_search_hit = classmethod(_extract_hash_from_search_hit)
|
||||
Download_File._iter_duplicate_tag_values = staticmethod(_iter_duplicate_tag_values)
|
||||
Download_File._extract_duplicate_namespace_tags = staticmethod(_extract_duplicate_namespace_tags)
|
||||
Download_File._extract_duplicate_title_tag = staticmethod(_extract_duplicate_title_tag)
|
||||
Download_File._extract_duplicate_title = classmethod(_extract_duplicate_title)
|
||||
Download_File._has_duplicate_title = classmethod(_has_duplicate_title)
|
||||
Download_File._build_duplicate_display_row = classmethod(_build_duplicate_display_row)
|
||||
Download_File._fetch_duplicate_metadata_for_hash = classmethod(_fetch_duplicate_metadata_for_hash)
|
||||
Download_File._enrich_duplicate_metadata = classmethod(_enrich_duplicate_metadata)
|
||||
Download_File._fetch_duplicate_metadata_for_hashes = classmethod(_fetch_duplicate_metadata_for_hashes)
|
||||
Download_File._collect_existing_url_match_refs_for_url = classmethod(_collect_existing_url_match_refs_for_url)
|
||||
Download_File._find_existing_url_matches_for_url = classmethod(_find_existing_url_matches_for_url)
|
||||
Download_File._find_existing_hash_for_url = classmethod(_find_existing_hash_for_url)
|
||||
Download_File._find_existing_hashes_for_url = classmethod(_find_existing_hashes_for_url)
|
||||
Download_File._init_storage = staticmethod(_init_storage)
|
||||
Download_File._supports_storage_duplicate_lookup = staticmethod(_supports_storage_duplicate_lookup)
|
||||
Download_File._filter_supported_urls = staticmethod(_filter_supported_urls)
|
||||
Download_File._canonicalize_url_for_storage = staticmethod(_canonicalize_url_for_storage)
|
||||
Download_File._preflight_url_duplicate = staticmethod(_preflight_url_duplicate)
|
||||
Download_File._preflight_explicit_url_duplicates = _preflight_explicit_url_duplicates
|
||||
Download_File._download_supported_urls = _download_supported_urls
|
||||
Download_File._maybe_show_playlist_table = _maybe_show_playlist_table
|
||||
Download_File._maybe_show_format_table_for_single_url = _maybe_show_format_table_for_single_url
|
||||
Download_File._run_streaming_urls = _run_streaming_urls
|
||||
|
||||
CMDLET = Download_File()
|
||||
Reference in New Issue
Block a user