1034 lines
37 KiB
Python
1034 lines
37 KiB
Python
"""URL/plugin fetching, streaming, and emission for download-file."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any, Dict, List, Optional, Sequence
|
|
from collections.abc import Sequence as SequenceABC
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
from SYS.logger import log, debug_panel
|
|
from SYS.pipeline_progress import PipelineProgress
|
|
from SYS import pipeline as pipeline_context
|
|
from SYS.utils import sha256_file
|
|
from API.HTTP import download_direct_file
|
|
|
|
from .. import _shared as sh
|
|
|
|
from .download_core import Download_File
|
|
|
|
coerce_to_path = sh.coerce_to_path
|
|
coerce_to_pipe_object = sh.coerce_to_pipe_object
|
|
register_url_with_local_library = sh.register_url_with_local_library
|
|
get_field = sh.get_field
|
|
|
|
|
|
def _emit_plugin_items(
|
|
self,
|
|
*,
|
|
items: Sequence[Any],
|
|
config: Dict[str, Any],
|
|
) -> int:
|
|
emitted = 0
|
|
for item in items:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
pipeline_context.emit(item)
|
|
if item.get("url"):
|
|
try:
|
|
pipe_obj = coerce_to_pipe_object(item)
|
|
register_url_with_local_library(pipe_obj, config)
|
|
except Exception:
|
|
pass
|
|
emitted += 1
|
|
return emitted
|
|
|
|
|
|
def _consume_plugin_download_result(
|
|
self,
|
|
*,
|
|
result: Any,
|
|
config: Dict[str, Any],
|
|
) -> tuple[int, Optional[int], bool]:
|
|
if result is None:
|
|
return 0, None, False
|
|
|
|
if isinstance(result, list):
|
|
if result and all(isinstance(item, dict) for item in result):
|
|
return _emit_plugin_items(self, items=result, config=config), 0, True
|
|
return 0, None, False
|
|
|
|
if not isinstance(result, dict):
|
|
return 0, None, False
|
|
|
|
action = str(
|
|
result.get("action")
|
|
or result.get("plugin_action")
|
|
or ""
|
|
).strip().lower()
|
|
|
|
if action in {"emit_items", "emit_pipe_objects"}:
|
|
items = result.get("items") or []
|
|
exit_code = result.get("exit_code")
|
|
emitted = _emit_plugin_items(
|
|
self,
|
|
items=items if isinstance(items, list) else [],
|
|
config=config,
|
|
)
|
|
try:
|
|
normalized_exit = int(exit_code) if exit_code is not None else 0
|
|
except Exception:
|
|
normalized_exit = 0
|
|
return emitted, normalized_exit, True
|
|
|
|
if action == "handled":
|
|
exit_code = result.get("exit_code")
|
|
try:
|
|
normalized_exit = int(exit_code) if exit_code is not None else 0
|
|
except Exception:
|
|
normalized_exit = 0
|
|
try:
|
|
downloaded = int(result.get("downloaded") or 0)
|
|
except Exception:
|
|
downloaded = 0
|
|
return downloaded, normalized_exit, True
|
|
|
|
return 0, None, False
|
|
|
|
|
|
def _emit_local_file(
|
|
self,
|
|
*,
|
|
downloaded_path: Path,
|
|
source: Optional[str],
|
|
title_hint: Optional[str],
|
|
tags_hint: Optional[List[str]],
|
|
media_kind_hint: Optional[str],
|
|
full_metadata: Optional[Dict[str, Any]],
|
|
progress: PipelineProgress,
|
|
config: Dict[str, Any],
|
|
provider_hint: Optional[str] = None,
|
|
) -> None:
|
|
title_val = (title_hint or downloaded_path.stem or "Unknown").strip() or downloaded_path.stem
|
|
hash_value = sha256_file(downloaded_path)
|
|
notes: Optional[Dict[str, str]] = None
|
|
try:
|
|
if isinstance(full_metadata, dict):
|
|
_provider_notes = full_metadata.get("_notes")
|
|
if isinstance(_provider_notes, dict) and _provider_notes:
|
|
notes = {str(k): str(v) for k, v in _provider_notes.items() if k and v}
|
|
except Exception:
|
|
notes = None
|
|
tag: List[str] = []
|
|
if tags_hint:
|
|
tag.extend([str(t) for t in tags_hint if t])
|
|
if not any(str(t).lower().startswith("title:") for t in tag):
|
|
tag.insert(0, f"title:{title_val}")
|
|
|
|
payload: Dict[str, Any] = {
|
|
"path": str(downloaded_path),
|
|
"hash": hash_value,
|
|
"title": title_val,
|
|
"action": "cmdlet:download-file",
|
|
"download_mode": "file",
|
|
"store": "local",
|
|
"media_kind": media_kind_hint or "file",
|
|
"tag": tag,
|
|
}
|
|
if provider_hint:
|
|
payload["plugin"] = str(provider_hint)
|
|
if full_metadata:
|
|
payload["metadata"] = full_metadata
|
|
if notes:
|
|
payload["notes"] = notes
|
|
if source and str(source).startswith("http"):
|
|
payload["url"] = source
|
|
elif source:
|
|
payload["source_url"] = source
|
|
|
|
pipeline_context.emit(payload)
|
|
|
|
|
|
def _expand_provider_items(
|
|
self,
|
|
*,
|
|
piped_items: Sequence[Any],
|
|
registry: Dict[str, Any],
|
|
config: Dict[str, Any],
|
|
) -> List[Any]:
|
|
get_provider = registry.get("get_plugin")
|
|
expanded_items: List[Any] = []
|
|
|
|
for item in piped_items:
|
|
try:
|
|
provider_key = self._provider_key_from_item(item)
|
|
provider = get_provider(provider_key, config) if provider_key and get_provider else None
|
|
|
|
if provider and hasattr(provider, "expand_item") and callable(provider.expand_item):
|
|
try:
|
|
sub_items = provider.expand_item(item)
|
|
if sub_items:
|
|
expanded_items.extend(sub_items)
|
|
continue
|
|
except Exception as e:
|
|
debug_panel(
|
|
"download-file expand_item failed",
|
|
[
|
|
("plugin", provider_key),
|
|
("error", e),
|
|
],
|
|
border_style="yellow",
|
|
)
|
|
|
|
expanded_items.append(item)
|
|
except Exception:
|
|
expanded_items.append(item)
|
|
|
|
return expanded_items
|
|
|
|
|
|
def _download_provider_items(
|
|
self,
|
|
*,
|
|
provider: Any,
|
|
provider_name: str,
|
|
search_result: Any,
|
|
output_dir: Path,
|
|
progress: PipelineProgress,
|
|
quiet_mode: bool,
|
|
config: Dict[str, Any],
|
|
) -> int:
|
|
if provider is None or not hasattr(provider, "download_items"):
|
|
return 0
|
|
|
|
def _on_emit(path: Path, file_url: str, relpath: str, metadata: Dict[str, Any]) -> None:
|
|
title_hint = None
|
|
try:
|
|
title_hint = metadata.get("name") or relpath
|
|
except Exception:
|
|
title_hint = relpath
|
|
title_hint = title_hint or (Path(path).name if path else "download")
|
|
|
|
_emit_local_file(
|
|
self,
|
|
downloaded_path=path,
|
|
source=file_url,
|
|
title_hint=title_hint,
|
|
tags_hint=None,
|
|
media_kind_hint="file",
|
|
full_metadata=metadata if isinstance(metadata, dict) else None,
|
|
progress=progress,
|
|
config=config,
|
|
provider_hint=provider_name,
|
|
)
|
|
|
|
try:
|
|
downloaded_count = provider.download_items(
|
|
search_result,
|
|
output_dir,
|
|
emit=_on_emit,
|
|
progress=progress,
|
|
quiet_mode=quiet_mode,
|
|
path_from_result=coerce_to_path,
|
|
config=config,
|
|
)
|
|
except TypeError:
|
|
downloaded_count = provider.download_items(
|
|
search_result,
|
|
output_dir,
|
|
emit=_on_emit,
|
|
progress=progress,
|
|
quiet_mode=quiet_mode,
|
|
path_from_result=coerce_to_path,
|
|
)
|
|
except Exception as exc:
|
|
log(f"Provider {provider_name} download_items error: {exc}", file=sys.stderr)
|
|
return 0
|
|
|
|
try:
|
|
return int(downloaded_count or 0)
|
|
except Exception:
|
|
return 0
|
|
|
|
|
|
def _process_explicit_urls(
|
|
self,
|
|
*,
|
|
raw_urls: Sequence[str],
|
|
final_output_dir: Path,
|
|
config: Dict[str, Any],
|
|
quiet_mode: bool,
|
|
registry: Dict[str, Any],
|
|
progress: PipelineProgress,
|
|
parsed: Dict[str, Any],
|
|
args: Sequence[str],
|
|
context_items: Sequence[Any] = (),
|
|
) -> tuple[int, Optional[int]]:
|
|
downloaded_count = 0
|
|
skipped_duplicate_only = 0
|
|
attempted_download = False
|
|
suppress_nested, batch_total, batch_index, batch_label = self._batch_progress_state(config)
|
|
total_urls = len(raw_urls or [])
|
|
|
|
try:
|
|
if total_urls > 1 and not suppress_nested:
|
|
progress.begin_pipe(total_items=total_urls, items_preview=list(raw_urls[:5]))
|
|
except Exception:
|
|
pass
|
|
|
|
SearchResult = registry.get("SearchResult")
|
|
get_plugin = registry.get("get_plugin")
|
|
match_plugin_name_for_url = registry.get("match_plugin_name_for_url")
|
|
|
|
for idx, url in enumerate(raw_urls, 1):
|
|
try:
|
|
try:
|
|
display_total = batch_total if batch_total > 0 else total_urls
|
|
display_index = batch_index if batch_total > 0 else idx
|
|
display_label = batch_label or str(url)
|
|
if display_total > 0:
|
|
progress.set_status(
|
|
f"downloading {display_index}/{display_total}: {display_label}"
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
provider_name = None
|
|
if match_plugin_name_for_url:
|
|
try:
|
|
provider_name = match_plugin_name_for_url(str(url))
|
|
except Exception:
|
|
pass
|
|
|
|
provider = None
|
|
if provider_name and get_plugin:
|
|
provider = get_plugin(provider_name, config)
|
|
|
|
if provider:
|
|
try:
|
|
handled = False
|
|
if hasattr(provider, "handle_url"):
|
|
try:
|
|
handled, path = provider.handle_url(str(url), output_dir=final_output_dir)
|
|
if handled:
|
|
extra_meta = None
|
|
title_hint = None
|
|
tags_hint: Optional[List[str]] = None
|
|
media_kind_hint = None
|
|
path_value: Optional[Any] = path
|
|
|
|
if isinstance(path, dict):
|
|
plugin_action = str(
|
|
path.get("action")
|
|
or path.get("plugin_action")
|
|
or ""
|
|
).strip().lower()
|
|
if plugin_action == "download_items" or bool(path.get("download_items")):
|
|
request_metadata = path.get("metadata") or path.get("full_metadata") or {}
|
|
if not isinstance(request_metadata, dict):
|
|
request_metadata = {}
|
|
magnet_id = path.get("magnet_id") or request_metadata.get("magnet_id")
|
|
if magnet_id is not None:
|
|
request_metadata.setdefault("magnet_id", magnet_id)
|
|
|
|
if SearchResult is None:
|
|
continue
|
|
|
|
sr = SearchResult(
|
|
table=str(provider_name),
|
|
title=str(path.get("title") or path.get("name") or f"{provider_name} item"),
|
|
path=str(path.get("path") or path.get("url") or url),
|
|
full_metadata=request_metadata,
|
|
)
|
|
downloaded_extra = _download_provider_items(
|
|
self,
|
|
provider=provider,
|
|
provider_name=str(provider_name),
|
|
search_result=sr,
|
|
output_dir=final_output_dir,
|
|
progress=progress,
|
|
quiet_mode=quiet_mode,
|
|
config=config,
|
|
)
|
|
if downloaded_extra:
|
|
downloaded_count += int(downloaded_extra)
|
|
continue
|
|
|
|
plugin_downloaded, plugin_exit, plugin_handled = _consume_plugin_download_result(
|
|
self,
|
|
result=path,
|
|
config=config,
|
|
)
|
|
if plugin_handled:
|
|
downloaded_count += plugin_downloaded
|
|
if plugin_exit is not None and plugin_downloaded == 0:
|
|
return downloaded_count, int(plugin_exit)
|
|
if plugin_downloaded:
|
|
continue
|
|
|
|
path_value = path.get("path") or path.get("file_path")
|
|
extra_meta = path.get("metadata") or path.get("full_metadata")
|
|
title_hint = path.get("title") or path.get("name")
|
|
media_kind_hint = path.get("media_kind")
|
|
tags_val = path.get("tags") or path.get("tag")
|
|
if isinstance(tags_val, (list, tuple, set)):
|
|
tags_hint = [str(t) for t in tags_val if t]
|
|
elif isinstance(tags_val, str) and tags_val.strip():
|
|
tags_hint = [str(tags_val).strip()]
|
|
|
|
if path_value:
|
|
p_val = Path(str(path_value))
|
|
if not title_hint and isinstance(extra_meta, dict):
|
|
title_hint = extra_meta.get("title") or extra_meta.get("name")
|
|
|
|
_emit_local_file(
|
|
self,
|
|
downloaded_path=p_val,
|
|
source=str(url),
|
|
title_hint=str(title_hint) if title_hint else p_val.stem,
|
|
tags_hint=tags_hint,
|
|
media_kind_hint=str(media_kind_hint) if media_kind_hint else "file",
|
|
full_metadata=extra_meta,
|
|
progress=progress,
|
|
config=config,
|
|
provider_hint=provider_name
|
|
)
|
|
downloaded_count += 1
|
|
continue
|
|
except Exception as e:
|
|
debug_panel(
|
|
"download-file provider error",
|
|
[
|
|
("plugin", provider_name),
|
|
("url", url),
|
|
("operation", "handle_url"),
|
|
("error", e),
|
|
],
|
|
border_style="yellow",
|
|
)
|
|
|
|
if not handled and hasattr(provider, "download_url"):
|
|
parsed_for_provider = parsed
|
|
provider_preflight_items = _resolve_provider_preflight_items(
|
|
provider,
|
|
url=str(url),
|
|
parsed=parsed,
|
|
args=args,
|
|
)
|
|
if provider_preflight_items:
|
|
provider_preflight_urls = [
|
|
str(item.get("url") or "").strip()
|
|
for item in provider_preflight_items
|
|
if str(item.get("url") or "").strip()
|
|
]
|
|
provider_preflight_urls, preflight_exit, provider_skipped = self._preflight_explicit_url_duplicates(
|
|
raw_urls=provider_preflight_urls,
|
|
config=config,
|
|
)
|
|
if preflight_exit is not None:
|
|
return downloaded_count, int(preflight_exit)
|
|
if provider_skipped:
|
|
if not provider_preflight_urls:
|
|
skipped_duplicate_only += 1
|
|
continue
|
|
selector = _build_provider_playlist_item_selector(
|
|
provider_preflight_items,
|
|
remaining_urls=provider_preflight_urls,
|
|
)
|
|
if selector:
|
|
parsed_for_provider = dict(parsed)
|
|
parsed_for_provider["item"] = selector
|
|
try:
|
|
attempted_download = True
|
|
res = provider.download_url(
|
|
str(url),
|
|
final_output_dir,
|
|
parsed=parsed_for_provider,
|
|
args=list(args),
|
|
progress=progress,
|
|
quiet_mode=quiet_mode,
|
|
context_items=list(context_items or []),
|
|
)
|
|
except TypeError:
|
|
attempted_download = True
|
|
res = provider.download_url(str(url), final_output_dir)
|
|
|
|
plugin_downloaded, plugin_exit, plugin_handled = _consume_plugin_download_result(
|
|
self,
|
|
result=res,
|
|
config=config,
|
|
)
|
|
if plugin_handled:
|
|
downloaded_count += plugin_downloaded
|
|
if plugin_exit is not None and plugin_downloaded == 0:
|
|
return downloaded_count, int(plugin_exit)
|
|
if plugin_downloaded:
|
|
continue
|
|
|
|
if res:
|
|
p_val = None
|
|
extra_meta = None
|
|
if isinstance(res, (str, Path)):
|
|
p_val = Path(res)
|
|
elif isinstance(res, tuple) and len(res) > 0:
|
|
p_val = Path(res[0])
|
|
if len(res) > 1 and isinstance(res[1], dict):
|
|
extra_meta = res[1]
|
|
elif isinstance(res, dict):
|
|
path_candidate = res.get("path") or res.get("file_path")
|
|
if path_candidate:
|
|
p_val = Path(path_candidate)
|
|
extra_meta = res
|
|
|
|
if p_val:
|
|
_emit_local_file(
|
|
self,
|
|
downloaded_path=p_val,
|
|
source=str(url),
|
|
title_hint=p_val.stem,
|
|
tags_hint=None,
|
|
media_kind_hint=extra_meta.get("media_kind") if extra_meta else "file",
|
|
full_metadata=extra_meta,
|
|
provider_hint=provider_name,
|
|
progress=progress,
|
|
config=config,
|
|
)
|
|
downloaded_count += 1
|
|
continue
|
|
|
|
except Exception as e:
|
|
log(f"Provider {provider_name} error handling {url}: {e}", file=sys.stderr)
|
|
pass
|
|
|
|
if not handled:
|
|
continue
|
|
|
|
# Direct Download Fallback
|
|
attempted_download = True
|
|
result_obj = download_direct_file(
|
|
str(url),
|
|
final_output_dir,
|
|
quiet=quiet_mode,
|
|
pipeline_progress=progress,
|
|
)
|
|
downloaded_path = self._path_from_download_result(result_obj)
|
|
|
|
_emit_local_file(
|
|
self,
|
|
downloaded_path=downloaded_path,
|
|
source=str(url),
|
|
title_hint=downloaded_path.stem,
|
|
tags_hint=[f"title:{downloaded_path.stem}"],
|
|
media_kind_hint="file",
|
|
full_metadata=None,
|
|
progress=progress,
|
|
config=config,
|
|
)
|
|
downloaded_count += 1
|
|
|
|
except Exception as e:
|
|
log(f"Download failed for {url}: {e}", file=sys.stderr)
|
|
|
|
if downloaded_count == 0 and skipped_duplicate_only > 0 and not attempted_download:
|
|
return downloaded_count, 0
|
|
return downloaded_count, None
|
|
|
|
|
|
def _process_provider_items(self,
|
|
*,
|
|
piped_items: Sequence[Any],
|
|
final_output_dir: Path,
|
|
config: Dict[str, Any],
|
|
quiet_mode: bool,
|
|
registry: Dict[str, Any],
|
|
progress: PipelineProgress,
|
|
) -> tuple[int, int]:
|
|
downloaded_count = 0
|
|
queued_magnet_submissions = 0
|
|
get_provider = registry.get("get_plugin")
|
|
SearchResult = registry.get("SearchResult")
|
|
|
|
expanded_items = _expand_provider_items(
|
|
self,
|
|
piped_items=piped_items,
|
|
registry=registry,
|
|
config=config
|
|
)
|
|
|
|
total_items = len(expanded_items)
|
|
processed_items = 0
|
|
|
|
try:
|
|
if total_items:
|
|
progress.set_percent(0)
|
|
except Exception:
|
|
pass
|
|
|
|
for idx, item in enumerate(expanded_items, 1):
|
|
try:
|
|
label = "item"
|
|
table = get_field(item, "table")
|
|
title = get_field(item, "title")
|
|
target = get_field(item, "path") or get_field(item, "url")
|
|
|
|
media_kind = get_field(item, "media_kind")
|
|
tags_val = get_field(item, "tag")
|
|
tags_list: Optional[List[str]]
|
|
if isinstance(tags_val, (list, set)):
|
|
tags_list = sorted([str(t) for t in tags_val if t])
|
|
else:
|
|
tags_list = None
|
|
|
|
full_metadata = get_field(item, "full_metadata")
|
|
if ((not full_metadata) and isinstance(item, dict)
|
|
and isinstance(item.get("extra"), dict)):
|
|
extra_md = item["extra"].get("full_metadata")
|
|
if isinstance(extra_md, dict):
|
|
full_metadata = extra_md
|
|
|
|
try:
|
|
label = title or target
|
|
label = str(label or "item").strip()
|
|
if total_items:
|
|
pct = int(round((processed_items / max(1, total_items)) * 100))
|
|
progress.set_percent(pct)
|
|
progress.set_status(
|
|
f"downloading {processed_items + 1}/{total_items}: {label}"
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
transfer_label = label
|
|
|
|
downloaded_path: Optional[Path] = None
|
|
attempted_provider_download = False
|
|
provider_sr = None
|
|
provider_obj = None
|
|
provider_key = self._provider_key_from_item(item)
|
|
if provider_key and get_provider and SearchResult:
|
|
provider_obj = get_provider(provider_key, config)
|
|
|
|
if provider_obj is not None and getattr(provider_obj, "prefers_transfer_progress", False):
|
|
try:
|
|
progress.begin_transfer(label=transfer_label, total=None)
|
|
except Exception:
|
|
pass
|
|
|
|
if provider_obj is not None:
|
|
attempted_provider_download = True
|
|
sr = SearchResult(
|
|
table=str(table),
|
|
title=str(title or "Unknown"),
|
|
path=str(target or ""),
|
|
tag=set(tags_list) if tags_list else set(),
|
|
media_kind=str(media_kind or "file"),
|
|
full_metadata=full_metadata
|
|
if isinstance(full_metadata, dict) else {},
|
|
)
|
|
|
|
output_dir = final_output_dir
|
|
|
|
downloaded_path = provider_obj.download(sr, output_dir)
|
|
provider_sr = sr
|
|
|
|
if downloaded_path is None:
|
|
try:
|
|
downloaded_extra = _download_provider_items(
|
|
self,
|
|
provider=provider_obj,
|
|
provider_name=str(provider_key),
|
|
search_result=sr,
|
|
output_dir=output_dir,
|
|
progress=progress,
|
|
quiet_mode=quiet_mode,
|
|
config=config,
|
|
)
|
|
except Exception:
|
|
downloaded_extra = 0
|
|
|
|
if downloaded_extra:
|
|
downloaded_count += int(downloaded_extra)
|
|
continue
|
|
|
|
if (downloaded_path is None and not attempted_provider_download
|
|
and isinstance(target, str) and target.startswith("http")):
|
|
|
|
suggested_name = str(title).strip() if title is not None else None
|
|
result_obj = download_direct_file(
|
|
target,
|
|
final_output_dir,
|
|
quiet=quiet_mode,
|
|
suggested_filename=suggested_name,
|
|
pipeline_progress=progress,
|
|
)
|
|
downloaded_path = coerce_to_path(result_obj)
|
|
|
|
if downloaded_path is None:
|
|
log(
|
|
f"Cannot download item (no provider handler / unsupported target): {title or target}",
|
|
file=sys.stderr,
|
|
)
|
|
continue
|
|
|
|
if provider_sr is not None:
|
|
try:
|
|
sr_md = getattr(provider_sr, "full_metadata", None)
|
|
if isinstance(sr_md, dict) and sr_md:
|
|
full_metadata = sr_md
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
if isinstance(full_metadata, dict):
|
|
t = str(full_metadata.get("title") or "").strip()
|
|
if t:
|
|
title = t
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
sr_tags = getattr(provider_sr, "tag", None)
|
|
if isinstance(sr_tags, (set, list)) and sr_tags:
|
|
tags_list = sorted([str(t) for t in sr_tags if t])
|
|
except Exception:
|
|
pass
|
|
|
|
_emit_local_file(
|
|
self,
|
|
downloaded_path=downloaded_path,
|
|
source=str(target) if target else None,
|
|
title_hint=str(title) if title else downloaded_path.stem,
|
|
tags_hint=tags_list,
|
|
media_kind_hint=str(media_kind) if media_kind else None,
|
|
full_metadata=full_metadata if isinstance(full_metadata, dict) else None,
|
|
progress=progress,
|
|
config=config,
|
|
provider_hint=provider_key
|
|
)
|
|
downloaded_count += 1
|
|
|
|
except Exception as e:
|
|
log(f"Download failed: {e}", file=sys.stderr)
|
|
finally:
|
|
if provider_obj is not None and getattr(provider_obj, "prefers_transfer_progress", False):
|
|
try:
|
|
progress.finish_transfer(label=transfer_label)
|
|
except Exception:
|
|
pass
|
|
processed_items += 1
|
|
try:
|
|
pct = int(round((processed_items / max(1, total_items)) * 100))
|
|
progress.set_percent(pct)
|
|
if processed_items >= total_items:
|
|
progress.clear_status()
|
|
except Exception:
|
|
pass
|
|
|
|
return downloaded_count, queued_magnet_submissions
|
|
|
|
|
|
@staticmethod
|
|
def _resolve_provider_preflight_items(
|
|
provider: Any,
|
|
*,
|
|
url: str,
|
|
parsed: Dict[str, Any],
|
|
args: Sequence[str],
|
|
) -> List[Dict[str, Any]]:
|
|
resolver = getattr(provider, "resolve_preflight_items", None)
|
|
if not callable(resolver):
|
|
return []
|
|
|
|
try:
|
|
items = resolver(url, parsed=parsed, args=list(args))
|
|
except TypeError:
|
|
try:
|
|
items = resolver(url)
|
|
except Exception:
|
|
items = None
|
|
except Exception:
|
|
items = None
|
|
|
|
if not isinstance(items, list):
|
|
return []
|
|
|
|
normalized: List[Dict[str, Any]] = []
|
|
for idx, item in enumerate(items, 1):
|
|
if not isinstance(item, dict):
|
|
continue
|
|
item_url = str(item.get("url") or "").strip()
|
|
if not item_url:
|
|
continue
|
|
playlist_index = item.get("playlist_index")
|
|
try:
|
|
playlist_index_value = int(playlist_index)
|
|
except Exception:
|
|
playlist_index_value = idx
|
|
normalized.append(
|
|
{
|
|
"url": item_url,
|
|
"playlist_index": playlist_index_value,
|
|
}
|
|
)
|
|
|
|
return normalized
|
|
|
|
|
|
@staticmethod
|
|
def _build_provider_playlist_item_selector(
|
|
items: Sequence[Dict[str, Any]],
|
|
*,
|
|
remaining_urls: Sequence[str],
|
|
) -> Optional[str]:
|
|
allowed_urls = {
|
|
str(url or "").strip() for url in (remaining_urls or []) if str(url or "").strip()
|
|
}
|
|
if not allowed_urls:
|
|
return None
|
|
|
|
selectors: List[str] = []
|
|
for idx, item in enumerate(items, 1):
|
|
item_url = str(item.get("url") or "").strip()
|
|
if not item_url or item_url not in allowed_urls:
|
|
continue
|
|
playlist_index = item.get("playlist_index")
|
|
try:
|
|
playlist_index_value = int(playlist_index)
|
|
except Exception:
|
|
playlist_index_value = idx
|
|
if playlist_index_value <= 0:
|
|
continue
|
|
selectors.append(str(playlist_index_value))
|
|
|
|
if not selectors:
|
|
return None
|
|
return ",".join(selectors)
|
|
|
|
|
|
@staticmethod
|
|
def _format_timecode(seconds: int, *, force_hours: bool) -> str:
|
|
total = max(0, int(seconds))
|
|
minutes, secs = divmod(total, 60)
|
|
hours, minutes = divmod(minutes, 60)
|
|
if force_hours:
|
|
return f"{hours:02d}:{minutes:02d}:{secs:02d}"
|
|
return f"{minutes:02d}:{secs:02d}"
|
|
|
|
|
|
@staticmethod
|
|
def _rebase_subtitle_timestamp_text(text: str, offset_seconds: int) -> str:
|
|
import re as _re
|
|
|
|
if not text:
|
|
return text
|
|
|
|
try:
|
|
offset_value = float(offset_seconds)
|
|
except Exception:
|
|
return text
|
|
|
|
if offset_value <= 0:
|
|
return text
|
|
|
|
timestamp_re = _re.compile(r"(?<!\d)(?P<ts>(?:\d{2}:)?\d{2}:\d{2}(?:[\\.,]\d{1,3})?)(?!\d)")
|
|
|
|
def _shift(match) -> str:
|
|
original = str(match.group("ts") or "")
|
|
if not original:
|
|
return original
|
|
|
|
frac_sep = "."
|
|
frac_digits = 0
|
|
base = original
|
|
frac_seconds = 0.0
|
|
if "." in original:
|
|
base, frac = original.split(".", 1)
|
|
frac_sep = "."
|
|
frac_digits = len(frac)
|
|
try:
|
|
frac_seconds = float(f"0.{frac}") if frac else 0.0
|
|
except Exception:
|
|
frac_seconds = 0.0
|
|
elif "," in original:
|
|
base, frac = original.split(",", 1)
|
|
frac_sep = ","
|
|
frac_digits = len(frac)
|
|
try:
|
|
frac_seconds = float(f"0.{frac}") if frac else 0.0
|
|
except Exception:
|
|
frac_seconds = 0.0
|
|
|
|
parts = base.split(":")
|
|
if len(parts) == 3:
|
|
hours_s, minutes_s, seconds_s = parts
|
|
include_hours = True
|
|
elif len(parts) == 2:
|
|
hours_s = "0"
|
|
minutes_s, seconds_s = parts
|
|
include_hours = False
|
|
else:
|
|
return original
|
|
|
|
try:
|
|
total = (
|
|
(int(hours_s) * 3600)
|
|
+ (int(minutes_s) * 60)
|
|
+ int(seconds_s)
|
|
+ frac_seconds
|
|
+ offset_value
|
|
)
|
|
except Exception:
|
|
return original
|
|
|
|
total = max(0.0, total)
|
|
whole_seconds = int(total)
|
|
fraction = total - whole_seconds
|
|
hours, remainder = divmod(whole_seconds, 3600)
|
|
minutes, seconds = divmod(remainder, 60)
|
|
|
|
if frac_digits > 0:
|
|
scale = 10 ** frac_digits
|
|
frac_value = int(round(fraction * scale))
|
|
if frac_value >= scale:
|
|
frac_value = 0
|
|
seconds += 1
|
|
if seconds >= 60:
|
|
seconds = 0
|
|
minutes += 1
|
|
if minutes >= 60:
|
|
minutes = 0
|
|
hours += 1
|
|
frac_text = f"{frac_value:0{frac_digits}d}"
|
|
if include_hours or hours > 0:
|
|
return f"{hours:02d}:{minutes:02d}:{seconds:02d}{frac_sep}{frac_text}"
|
|
return f"{minutes:02d}:{seconds:02d}{frac_sep}{frac_text}"
|
|
|
|
if include_hours or hours > 0:
|
|
return f"{hours:02d}:{minutes:02d}:{seconds:02d}"
|
|
return f"{minutes:02d}:{seconds:02d}"
|
|
|
|
try:
|
|
return timestamp_re.sub(_shift, str(text))
|
|
except Exception:
|
|
return text
|
|
|
|
|
|
@classmethod
|
|
def _format_clip_range(cls, start_s: int, end_s: int) -> str:
|
|
force_hours = bool(start_s >= 3600 or end_s >= 3600)
|
|
return f"{_format_timecode(start_s, force_hours=force_hours)}-{_format_timecode(end_s, force_hours=force_hours)}"
|
|
|
|
|
|
@classmethod
|
|
def _apply_clip_decorations(
|
|
cls, pipe_objects: List[Dict[str, Any]], clip_ranges: List[tuple[int, int]], *, source_king_hash: Optional[str]
|
|
) -> None:
|
|
if not pipe_objects or len(pipe_objects) != len(clip_ranges):
|
|
return
|
|
|
|
for po, (start_s, end_s) in zip(pipe_objects, clip_ranges):
|
|
clip_range = _format_clip_range(start_s, end_s)
|
|
clip_tag = f"clip:{clip_range}"
|
|
|
|
po["title"] = clip_tag
|
|
|
|
tags = po.get("tag")
|
|
if not isinstance(tags, list):
|
|
tags = []
|
|
|
|
tags = [t for t in tags if not str(t).strip().lower().startswith("title:")]
|
|
tags = [t for t in tags if not str(t).strip().lower().startswith("relationship:")]
|
|
tags.insert(0, f"title:{clip_tag}")
|
|
|
|
if clip_tag not in tags:
|
|
tags.append(clip_tag)
|
|
|
|
po["tag"] = tags
|
|
|
|
notes = po.get("notes")
|
|
if isinstance(notes, dict):
|
|
sub_text = notes.get("sub")
|
|
if isinstance(sub_text, str) and sub_text.strip():
|
|
notes["sub"] = _rebase_subtitle_timestamp_text(sub_text, start_s)
|
|
po["notes"] = notes
|
|
|
|
if len(pipe_objects) < 2:
|
|
return
|
|
|
|
hashes: List[str] = []
|
|
for po in pipe_objects:
|
|
h_val = sh.normalize_hash(str(po.get("hash") or ""))
|
|
hashes.append(h_val or "")
|
|
|
|
king_hash = sh.normalize_hash(source_king_hash) if source_king_hash else None
|
|
if not king_hash:
|
|
king_hash = hashes[0] if hashes and hashes[0] else None
|
|
if not king_hash:
|
|
return
|
|
|
|
alt_hashes: List[str] = [h for h in hashes if h and h != king_hash]
|
|
if not alt_hashes:
|
|
return
|
|
|
|
for po in pipe_objects:
|
|
po["relationships"] = {"king": [king_hash], "alt": list(alt_hashes)}
|
|
|
|
|
|
@staticmethod
|
|
def _parse_clip_spec_to_ranges(clip_spec: Optional[str]) -> Optional[List[tuple[int, int]]]:
|
|
"""Parse clip spec strings like '2m-2m20s,5m-6m'."""
|
|
text = str(clip_spec or "").strip()
|
|
if not text:
|
|
return None
|
|
|
|
def _parse_time(value: str) -> Optional[int]:
|
|
s = str(value or "").strip().lower()
|
|
if not s:
|
|
return None
|
|
try:
|
|
if ":" in s:
|
|
parts = [int(p) for p in s.split(":")]
|
|
if len(parts) == 2:
|
|
return (parts[0] * 60) + parts[1]
|
|
if len(parts) == 3:
|
|
return (parts[0] * 3600) + (parts[1] * 60) + parts[2]
|
|
return None
|
|
total = 0
|
|
number = ""
|
|
units_seen = False
|
|
for ch in s:
|
|
if ch.isdigit():
|
|
number += ch
|
|
continue
|
|
if ch in {"h", "m", "s"} and number:
|
|
units_seen = True
|
|
val = int(number)
|
|
if ch == "h":
|
|
total += val * 3600
|
|
elif ch == "m":
|
|
total += val * 60
|
|
else:
|
|
total += val
|
|
number = ""
|
|
continue
|
|
return None
|
|
if number:
|
|
total += int(number)
|
|
if total == 0 and units_seen:
|
|
return 0
|
|
return total if total >= 0 else None
|
|
except Exception:
|
|
return None
|
|
|
|
ranges: List[tuple[int, int]] = []
|
|
for chunk in [c.strip() for c in text.split(",") if c.strip()]:
|
|
if "-" not in chunk:
|
|
return None
|
|
left, right = chunk.split("-", 1)
|
|
start = _parse_time(left)
|
|
end = _parse_time(right)
|
|
if start is None or end is None or end < start:
|
|
return None
|
|
ranges.append((start, end))
|
|
return ranges or None
|