update download plugin
This commit is contained in:
-11
@@ -652,8 +652,6 @@ def download_direct_file(
|
|||||||
extracted_name = match.group(1) or match.group(2)
|
extracted_name = match.group(1) or match.group(2)
|
||||||
if extracted_name:
|
if extracted_name:
|
||||||
filename = unquote(extracted_name)
|
filename = unquote(extracted_name)
|
||||||
if not quiet:
|
|
||||||
debug(f"Filename from Content-Disposition: {filename}")
|
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
if not quiet:
|
if not quiet:
|
||||||
log(f"Could not get filename from headers: {exc}", file=sys.stderr)
|
log(f"Could not get filename from headers: {exc}", file=sys.stderr)
|
||||||
@@ -748,11 +746,7 @@ def download_direct_file(
|
|||||||
|
|
||||||
transfer_started = [False]
|
transfer_started = [False]
|
||||||
|
|
||||||
if not quiet:
|
|
||||||
debug(f"Direct download: {filename}")
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
start_time = time.time()
|
|
||||||
downloaded_bytes = [0]
|
downloaded_bytes = [0]
|
||||||
transfer_started[0] = False
|
transfer_started[0] = False
|
||||||
|
|
||||||
@@ -807,8 +801,6 @@ def download_direct_file(
|
|||||||
with HTTPClient(timeout=30.0) as client:
|
with HTTPClient(timeout=30.0) as client:
|
||||||
client.download(url, str(file_path), progress_callback=progress_callback)
|
client.download(url, str(file_path), progress_callback=progress_callback)
|
||||||
|
|
||||||
elapsed = time.time() - start_time
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if progress_bar is not None:
|
if progress_bar is not None:
|
||||||
progress_bar.finish()
|
progress_bar.finish()
|
||||||
@@ -823,9 +815,6 @@ def download_direct_file(
|
|||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Failed to finish pipeline transfer")
|
logger.exception("Failed to finish pipeline transfer")
|
||||||
|
|
||||||
if not quiet:
|
|
||||||
debug(f"✓ Downloaded in {elapsed:.1f}s")
|
|
||||||
|
|
||||||
ext_out = ""
|
ext_out = ""
|
||||||
try:
|
try:
|
||||||
ext_out = Path(str(filename)).suffix.lstrip(".")
|
ext_out = Path(str(filename)).suffix.lstrip(".")
|
||||||
|
|||||||
@@ -13,32 +13,6 @@ def coerce_to_pipe_object(
|
|||||||
|
|
||||||
Uses hash+store canonical pattern.
|
Uses hash+store canonical pattern.
|
||||||
"""
|
"""
|
||||||
# Debug: Print ResultItem details if coming from search_file.py
|
|
||||||
try:
|
|
||||||
from SYS.logger import is_debug_enabled, debug
|
|
||||||
|
|
||||||
if (
|
|
||||||
is_debug_enabled()
|
|
||||||
and hasattr(value, "__class__")
|
|
||||||
and value.__class__.__name__ == "ResultItem"
|
|
||||||
):
|
|
||||||
debug("[ResultItem -> PipeObject conversion]")
|
|
||||||
debug(f" title={getattr(value, 'title', None)}")
|
|
||||||
debug(f" target={getattr(value, 'target', None)}")
|
|
||||||
debug(f" hash={getattr(value, 'hash', None)}")
|
|
||||||
debug(f" media_kind={getattr(value, 'media_kind', None)}")
|
|
||||||
debug(f" tag={getattr(value, 'tag', None)}")
|
|
||||||
debug(f" tag_summary={getattr(value, 'tag_summary', None)}")
|
|
||||||
debug(f" size_bytes={getattr(value, 'size_bytes', None)}")
|
|
||||||
debug(f" duration_seconds={getattr(value, 'duration_seconds', None)}")
|
|
||||||
debug(f" relationships={getattr(value, 'relationships', None)}")
|
|
||||||
debug(f" url={getattr(value, 'url', None)}")
|
|
||||||
debug(
|
|
||||||
f" full_metadata keys={list(getattr(value, 'full_metadata', {}).keys()) if hasattr(value, 'full_metadata') and value.full_metadata else []}"
|
|
||||||
)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
if isinstance(value, models.PipeObject):
|
if isinstance(value, models.PipeObject):
|
||||||
return value
|
return value
|
||||||
|
|
||||||
|
|||||||
@@ -2445,30 +2445,6 @@ def coerce_to_pipe_object(
|
|||||||
|
|
||||||
Uses hash+store canonical pattern.
|
Uses hash+store canonical pattern.
|
||||||
"""
|
"""
|
||||||
# Debug: Print ResultItem details if coming from search_file.py
|
|
||||||
try:
|
|
||||||
from SYS.logger import is_debug_enabled, debug
|
|
||||||
|
|
||||||
if (is_debug_enabled() and hasattr(value,
|
|
||||||
"__class__")
|
|
||||||
and value.__class__.__name__ == "ResultItem"):
|
|
||||||
debug("[ResultItem -> PipeObject conversion]")
|
|
||||||
debug(f" title={getattr(value, 'title', None)}")
|
|
||||||
debug(f" target={getattr(value, 'target', None)}")
|
|
||||||
debug(f" hash={getattr(value, 'hash', None)}")
|
|
||||||
debug(f" media_kind={getattr(value, 'media_kind', None)}")
|
|
||||||
debug(f" tag={getattr(value, 'tag', None)}")
|
|
||||||
debug(f" tag_summary={getattr(value, 'tag_summary', None)}")
|
|
||||||
debug(f" size_bytes={getattr(value, 'size_bytes', None)}")
|
|
||||||
debug(f" duration_seconds={getattr(value, 'duration_seconds', None)}")
|
|
||||||
debug(f" relationships={getattr(value, 'relationships', None)}")
|
|
||||||
debug(f" url={getattr(value, 'url', None)}")
|
|
||||||
debug(
|
|
||||||
f" full_metadata keys={list(getattr(value, 'full_metadata', {}).keys()) if hasattr(value, 'full_metadata') and value.full_metadata else []}"
|
|
||||||
)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
if isinstance(value, models.PipeObject):
|
if isinstance(value, models.PipeObject):
|
||||||
return value
|
return value
|
||||||
|
|
||||||
|
|||||||
+13
-2
@@ -199,7 +199,7 @@ class Add_File(Cmdlet):
|
|||||||
summary=
|
summary=
|
||||||
"Ingest a local media file to a configured store or plugin destination.",
|
"Ingest a local media file to a configured store or plugin destination.",
|
||||||
usage=
|
usage=
|
||||||
"add-file (<source> | <piped>) (-instance <store-name> | -plugin <plugin> [-instance <name|path>]) [-delete]",
|
"add-file (<source> | <piped>) (-instance <store-name> | -plugin <plugin> [-instance <name|path>]) [-folder <name>] [-delete]",
|
||||||
arg=[
|
arg=[
|
||||||
CmdletArg(
|
CmdletArg(
|
||||||
name="source",
|
name="source",
|
||||||
@@ -217,6 +217,11 @@ class Add_File(Cmdlet):
|
|||||||
description="Delete file after successful upload",
|
description="Delete file after successful upload",
|
||||||
alias="del",
|
alias="del",
|
||||||
),
|
),
|
||||||
|
CmdletArg(
|
||||||
|
name="folder",
|
||||||
|
description="Folder name override for local plugin exports.",
|
||||||
|
alias="folder-name",
|
||||||
|
),
|
||||||
],
|
],
|
||||||
detail=[
|
detail=[
|
||||||
"Note: add-file ingests local files. To fetch remote sources, use download-file and pipe into add-file.",
|
"Note: add-file ingests local files. To fetch remote sources, use download-file and pipe into add-file.",
|
||||||
@@ -224,6 +229,7 @@ class Add_File(Cmdlet):
|
|||||||
" hydrus: Upload to Hydrus database with metadata tagging",
|
" hydrus: Upload to Hydrus database with metadata tagging",
|
||||||
"- Plugin options (use -plugin):",
|
"- Plugin options (use -plugin):",
|
||||||
" local: Copy file to a configured local destination or direct path via -instance",
|
" local: Copy file to a configured local destination or direct path via -instance",
|
||||||
|
" local folder exports: use -folder <name> (or -folder-name); piped AllDebrid downloads default to their magnet folder",
|
||||||
" 0x0: Upload to 0x0.st for temporary hosting",
|
" 0x0: Upload to 0x0.st for temporary hosting",
|
||||||
" file.io: Upload to file.io for temporary hosting",
|
" file.io: Upload to file.io for temporary hosting",
|
||||||
" internetarchive: Upload to archive.org (optional tag: ia:<identifier> to upload into an existing item)",
|
" internetarchive: Upload to archive.org (optional tag: ia:<identifier> to upload into an existing item)",
|
||||||
@@ -272,6 +278,7 @@ class Add_File(Cmdlet):
|
|||||||
source_url_arg = parsed.get("url")
|
source_url_arg = parsed.get("url")
|
||||||
plugin_name = parsed.get("plugin")
|
plugin_name = parsed.get("plugin")
|
||||||
delete_after = parsed.get("delete", False)
|
delete_after = parsed.get("delete", False)
|
||||||
|
folder_name = parsed.get("folder")
|
||||||
local_export_destination: Optional[str] = None
|
local_export_destination: Optional[str] = None
|
||||||
if plugin_name and not plugin_instance and location:
|
if plugin_name and not plugin_instance and location:
|
||||||
plugin_instance = location
|
plugin_instance = location
|
||||||
@@ -700,7 +707,8 @@ class Add_File(Cmdlet):
|
|||||||
plugin_instance,
|
plugin_instance,
|
||||||
pipe_obj,
|
pipe_obj,
|
||||||
config,
|
config,
|
||||||
delete_after_item
|
delete_after_item,
|
||||||
|
folder_name=folder_name,
|
||||||
)
|
)
|
||||||
if code == 0:
|
if code == 0:
|
||||||
successes += 1
|
successes += 1
|
||||||
@@ -2532,6 +2540,7 @@ class Add_File(Cmdlet):
|
|||||||
config: Dict[str,
|
config: Dict[str,
|
||||||
Any],
|
Any],
|
||||||
delete_after: bool,
|
delete_after: bool,
|
||||||
|
folder_name: Optional[str] = None,
|
||||||
) -> int:
|
) -> int:
|
||||||
"""Handle uploading via an add-file plugin (e.g. 0x0)."""
|
"""Handle uploading via an add-file plugin (e.g. 0x0)."""
|
||||||
from PluginCore.registry import (
|
from PluginCore.registry import (
|
||||||
@@ -2585,6 +2594,8 @@ class Add_File(Cmdlet):
|
|||||||
"pipeline_progress": pipeline_progress,
|
"pipeline_progress": pipeline_progress,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
if folder_name is not None and str(folder_name).strip():
|
||||||
|
upload_kwargs["folder_name"] = str(folder_name).strip()
|
||||||
|
|
||||||
upload_result = file_provider.upload(
|
upload_result = file_provider.upload(
|
||||||
str(media_path),
|
str(media_path),
|
||||||
|
|||||||
@@ -429,9 +429,11 @@ def prepare_magnet(
|
|||||||
def _flatten_files_with_relpath(items: Any) -> Iterable[Dict[str, Any]]:
|
def _flatten_files_with_relpath(items: Any) -> Iterable[Dict[str, Any]]:
|
||||||
for node in AllDebrid._flatten_files(items):
|
for node in AllDebrid._flatten_files(items):
|
||||||
enriched = dict(node)
|
enriched = dict(node)
|
||||||
|
enriched.setdefault("name", node.get("n") or node.get("name"))
|
||||||
|
enriched.setdefault("link", node.get("l") or node.get("link"))
|
||||||
rel = node.get("_relpath") or node.get("relpath")
|
rel = node.get("_relpath") or node.get("relpath")
|
||||||
if not rel:
|
if not rel:
|
||||||
name = node.get("n") or node.get("name")
|
name = enriched.get("name")
|
||||||
rel = str(name or "").strip()
|
rel = str(name or "").strip()
|
||||||
enriched["relpath"] = rel
|
enriched["relpath"] = rel
|
||||||
yield enriched
|
yield enriched
|
||||||
@@ -447,7 +449,7 @@ def download_magnet(
|
|||||||
path_from_result: Callable[[Any], Path],
|
path_from_result: Callable[[Any], Path],
|
||||||
on_emit: Callable[[Path, str, str, Dict[str, Any]], None],
|
on_emit: Callable[[Path, str, str, Dict[str, Any]], None],
|
||||||
) -> tuple[int, Optional[int]]:
|
) -> tuple[int, Optional[int]]:
|
||||||
client, magnet_id, _magnet_info = prepare_magnet(magnet_spec, config)
|
client, magnet_id, magnet_info = prepare_magnet(magnet_spec, config)
|
||||||
if client is None or magnet_id is None:
|
if client is None or magnet_id is None:
|
||||||
return 0, None
|
return 0, None
|
||||||
|
|
||||||
@@ -486,6 +488,11 @@ def download_magnet(
|
|||||||
log(f"AllDebrid magnet {magnet_id} produced no files", file=sys.stderr)
|
log(f"AllDebrid magnet {magnet_id} produced no files", file=sys.stderr)
|
||||||
return 0, magnet_id
|
return 0, magnet_id
|
||||||
|
|
||||||
|
magnet_folder_name = _resolve_alldebrid_folder_name(
|
||||||
|
magnet_info,
|
||||||
|
fallback_metadata=magnet_files,
|
||||||
|
magnet_id=magnet_id,
|
||||||
|
)
|
||||||
downloaded = 0
|
downloaded = 0
|
||||||
for node in _flatten_files_with_relpath(file_nodes):
|
for node in _flatten_files_with_relpath(file_nodes):
|
||||||
file_url = str(node.get("link") or "").strip()
|
file_url = str(node.get("link") or "").strip()
|
||||||
@@ -494,15 +501,12 @@ def download_magnet(
|
|||||||
if not file_url or not relpath:
|
if not file_url or not relpath:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
target_path = final_output_dir
|
|
||||||
rel_path_obj = Path(relpath)
|
rel_path_obj = Path(relpath)
|
||||||
output_dir = target_path
|
output_dir = adjust_output_dir_for_alldebrid(
|
||||||
if rel_path_obj.parent:
|
final_output_dir,
|
||||||
output_dir = target_path / rel_path_obj.parent
|
{"folder": magnet_folder_name, "relpath": relpath},
|
||||||
try:
|
magnet_files,
|
||||||
output_dir.mkdir(parents=True, exist_ok=True)
|
)
|
||||||
except Exception:
|
|
||||||
output_dir = target_path
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result_obj = download_direct_file(
|
result_obj = download_direct_file(
|
||||||
@@ -519,6 +523,7 @@ def download_magnet(
|
|||||||
downloaded_path = path_from_result(result_obj)
|
downloaded_path = path_from_result(result_obj)
|
||||||
metadata = {
|
metadata = {
|
||||||
"magnet_id": magnet_id,
|
"magnet_id": magnet_id,
|
||||||
|
"folder": magnet_folder_name,
|
||||||
"relpath": relpath,
|
"relpath": relpath,
|
||||||
"name": file_name,
|
"name": file_name,
|
||||||
}
|
}
|
||||||
@@ -627,6 +632,29 @@ def adjust_output_dir_for_alldebrid(
|
|||||||
return output_dir
|
return output_dir
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_alldebrid_folder_name(
|
||||||
|
metadata: Any,
|
||||||
|
*,
|
||||||
|
fallback_metadata: Any = None,
|
||||||
|
magnet_id: Optional[int] = None,
|
||||||
|
) -> str:
|
||||||
|
sources = [
|
||||||
|
source
|
||||||
|
for source in (metadata, fallback_metadata)
|
||||||
|
if isinstance(source, dict)
|
||||||
|
]
|
||||||
|
for key in ("folder", "magnet_name", "filename", "name"):
|
||||||
|
for source in sources:
|
||||||
|
value = str(source.get(key) or "").strip()
|
||||||
|
if value:
|
||||||
|
return value
|
||||||
|
for source in sources:
|
||||||
|
value = str(source.get("hash") or "").strip()
|
||||||
|
if value:
|
||||||
|
return value
|
||||||
|
return f"magnet-{magnet_id}" if magnet_id is not None else "download"
|
||||||
|
|
||||||
|
|
||||||
class AllDebrid(TablePluginMixin, Provider):
|
class AllDebrid(TablePluginMixin, Provider):
|
||||||
"""AllDebrid account provider with magnet folder/file browsing and downloads.
|
"""AllDebrid account provider with magnet folder/file browsing and downloads.
|
||||||
|
|
||||||
@@ -1278,13 +1306,27 @@ class AllDebrid(TablePluginMixin, Provider):
|
|||||||
log(f"AllDebrid magnet {magnet_id} has no downloadable files", file=sys.stderr)
|
log(f"AllDebrid magnet {magnet_id} has no downloadable files", file=sys.stderr)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
magnet_metadata: Dict[str, Any] = {}
|
||||||
|
try:
|
||||||
|
for magnet in client.magnet_list() or []:
|
||||||
|
if not isinstance(magnet, dict):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
listed_id = int(magnet.get("id") or 0)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
continue
|
||||||
|
if listed_id == magnet_id:
|
||||||
|
magnet_metadata = magnet
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
magnet_metadata = {}
|
||||||
|
|
||||||
magnet_path_metadata: Dict[str, Any] = {}
|
magnet_path_metadata: Dict[str, Any] = {}
|
||||||
magnet_folder_name = str(
|
magnet_folder_name = _resolve_alldebrid_folder_name(
|
||||||
magnet_files.get("filename")
|
magnet_metadata,
|
||||||
or magnet_files.get("name")
|
fallback_metadata=magnet_files,
|
||||||
or magnet_files.get("hash")
|
magnet_id=magnet_id,
|
||||||
or f"magnet-{magnet_id}"
|
)
|
||||||
).strip()
|
|
||||||
if magnet_folder_name:
|
if magnet_folder_name:
|
||||||
magnet_path_metadata["folder"] = magnet_folder_name
|
magnet_path_metadata["folder"] = magnet_folder_name
|
||||||
|
|
||||||
@@ -1340,6 +1382,7 @@ class AllDebrid(TablePluginMixin, Provider):
|
|||||||
downloaded_path = path_from_result(result_obj)
|
downloaded_path = path_from_result(result_obj)
|
||||||
metadata = {
|
metadata = {
|
||||||
"magnet_id": magnet_id,
|
"magnet_id": magnet_id,
|
||||||
|
"folder": magnet_folder_name,
|
||||||
"relpath": relpath,
|
"relpath": relpath,
|
||||||
"name": file_name,
|
"name": file_name,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -165,6 +165,43 @@ class Local(Provider):
|
|||||||
def validate(self) -> bool:
|
def validate(self) -> bool:
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _folder_name_from_pipe(pipe_obj: Any) -> str:
|
||||||
|
metadata = getattr(pipe_obj, "metadata", None)
|
||||||
|
extra = getattr(pipe_obj, "extra", None)
|
||||||
|
if not isinstance(metadata, dict):
|
||||||
|
metadata = {}
|
||||||
|
if not isinstance(extra, dict):
|
||||||
|
extra = {}
|
||||||
|
|
||||||
|
plugin_name = str(
|
||||||
|
getattr(pipe_obj, "plugin", None)
|
||||||
|
or metadata.get("plugin")
|
||||||
|
or extra.get("plugin")
|
||||||
|
or ""
|
||||||
|
).strip().lower()
|
||||||
|
has_alldebrid_metadata = any(
|
||||||
|
isinstance(source, dict)
|
||||||
|
and (source.get("magnet_id") is not None or source.get("plugin") == "alldebrid")
|
||||||
|
for source in (metadata, extra)
|
||||||
|
)
|
||||||
|
if plugin_name != "alldebrid" and not has_alldebrid_metadata:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
for source in (metadata, extra):
|
||||||
|
for key in ("folder_name", "folder", "magnet_name"):
|
||||||
|
value = str(source.get(key) or "").strip()
|
||||||
|
if value:
|
||||||
|
return value
|
||||||
|
|
||||||
|
magnet = source.get("magnet")
|
||||||
|
if isinstance(magnet, dict):
|
||||||
|
for key in ("filename", "name", "hash"):
|
||||||
|
value = str(magnet.get(key) or "").strip()
|
||||||
|
if value:
|
||||||
|
return value
|
||||||
|
return ""
|
||||||
|
|
||||||
def upload(self, file_path: str, **kwargs: Any) -> Dict[str, Any]:
|
def upload(self, file_path: str, **kwargs: Any) -> Dict[str, Any]:
|
||||||
source_path = Path(str(file_path or "")).expanduser()
|
source_path = Path(str(file_path or "")).expanduser()
|
||||||
if not source_path.exists() or not source_path.is_file():
|
if not source_path.exists() or not source_path.is_file():
|
||||||
@@ -210,6 +247,19 @@ class Local(Provider):
|
|||||||
elif not destination_root.is_dir():
|
elif not destination_root.is_dir():
|
||||||
raise NotADirectoryError(f"Destination is not a directory: {destination_root}")
|
raise NotADirectoryError(f"Destination is not a directory: {destination_root}")
|
||||||
|
|
||||||
|
direct_export_download = bool(kwargs.get("direct_export_download", False))
|
||||||
|
folder_name = str(kwargs.get("folder_name") or "").strip()
|
||||||
|
if not folder_name and not direct_export_download:
|
||||||
|
folder_name = self._folder_name_from_pipe(kwargs.get("pipe_obj"))
|
||||||
|
|
||||||
|
export_root = destination_root
|
||||||
|
if folder_name and not direct_export_download:
|
||||||
|
export_root = destination_root / sanitize_filename(folder_name)
|
||||||
|
if create_dirs:
|
||||||
|
export_root.mkdir(parents=True, exist_ok=True)
|
||||||
|
elif not export_root.is_dir():
|
||||||
|
raise FileNotFoundError(f"Destination directory does not exist: {export_root}")
|
||||||
|
|
||||||
title = str(kwargs.get("title") or "").strip()
|
title = str(kwargs.get("title") or "").strip()
|
||||||
if not title:
|
if not title:
|
||||||
title = source_path.stem.replace("_", " ").strip()
|
title = source_path.stem.replace("_", " ").strip()
|
||||||
@@ -221,8 +271,7 @@ class Local(Provider):
|
|||||||
else:
|
else:
|
||||||
target_name = base_name + file_ext
|
target_name = base_name + file_ext
|
||||||
|
|
||||||
direct_export_download = bool(kwargs.get("direct_export_download", False))
|
target_path = source_path if direct_export_download else export_root / target_name
|
||||||
target_path = source_path if direct_export_download else destination_root / target_name
|
|
||||||
|
|
||||||
if not direct_export_download:
|
if not direct_export_download:
|
||||||
if target_path.exists():
|
if target_path.exists():
|
||||||
@@ -271,6 +320,8 @@ class Local(Provider):
|
|||||||
"url": urls,
|
"url": urls,
|
||||||
"export_path": str(destination_root),
|
"export_path": str(destination_root),
|
||||||
}
|
}
|
||||||
|
if export_root != destination_root:
|
||||||
|
extra_updates["export_folder"] = str(export_root)
|
||||||
if resolved_name:
|
if resolved_name:
|
||||||
extra_updates["instance"] = resolved_name
|
extra_updates["instance"] = resolved_name
|
||||||
if relationships:
|
if relationships:
|
||||||
|
|||||||
Reference in New Issue
Block a user