syntax revamp
This commit is contained in:
+158
-73
@@ -28,6 +28,54 @@ def _copy_sidecars(source_path: Path, target_path: Path) -> None:
|
||||
continue
|
||||
|
||||
|
||||
def _copy_with_progress(
|
||||
source_path: Path,
|
||||
target_path: Path,
|
||||
*,
|
||||
pipeline_progress: Any = None,
|
||||
label: str = "local export",
|
||||
chunk_size: int = 1024 * 1024,
|
||||
) -> None:
|
||||
total_bytes: Optional[int] = None
|
||||
try:
|
||||
total_bytes = int(source_path.stat().st_size)
|
||||
except Exception:
|
||||
total_bytes = None
|
||||
|
||||
transfer_started = False
|
||||
completed = 0
|
||||
transfer_label = str(label or target_path.name or source_path.name)
|
||||
try:
|
||||
if pipeline_progress is not None and hasattr(pipeline_progress, "begin_transfer"):
|
||||
pipeline_progress.begin_transfer(
|
||||
label=transfer_label,
|
||||
total=total_bytes if isinstance(total_bytes, int) and total_bytes > 0 else None,
|
||||
)
|
||||
transfer_started = True
|
||||
|
||||
with source_path.open("rb") as src, target_path.open("wb") as dst:
|
||||
while True:
|
||||
chunk = src.read(max(4096, int(chunk_size or 0) or 1024 * 1024))
|
||||
if not chunk:
|
||||
break
|
||||
dst.write(chunk)
|
||||
completed += len(chunk)
|
||||
if pipeline_progress is not None and hasattr(pipeline_progress, "update_transfer"):
|
||||
pipeline_progress.update_transfer(
|
||||
label=transfer_label,
|
||||
completed=completed,
|
||||
total=total_bytes if isinstance(total_bytes, int) and total_bytes > 0 else None,
|
||||
)
|
||||
|
||||
shutil.copystat(str(source_path), str(target_path))
|
||||
finally:
|
||||
if pipeline_progress is not None and transfer_started and hasattr(pipeline_progress, "finish_transfer"):
|
||||
try:
|
||||
pipeline_progress.finish_transfer(label=transfer_label)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class Local(Provider):
|
||||
PLUGIN_NAME = "local"
|
||||
PLUGIN_ALIASES = ("filesystem", "fs")
|
||||
@@ -122,84 +170,121 @@ class Local(Provider):
|
||||
if not source_path.exists() or not source_path.is_file():
|
||||
raise FileNotFoundError(f"File not found: {source_path}")
|
||||
|
||||
requested_instance = str(kwargs.get("instance") or kwargs.get("store") or "").strip() or None
|
||||
resolved_name, settings = self.resolve_destination(
|
||||
requested_instance,
|
||||
require_explicit=bool(requested_instance),
|
||||
)
|
||||
destination_text = str(settings.get("path") or "").strip()
|
||||
if not destination_text:
|
||||
requested_label = requested_instance or "<default>"
|
||||
raise ValueError(
|
||||
f"Local destination '{requested_label}' is not configured. Use -plugin local -instance <name|path>."
|
||||
)
|
||||
pipeline_progress = kwargs.get("pipeline_progress")
|
||||
|
||||
destination_root = Path(destination_text).expanduser()
|
||||
create_dirs = bool(settings.get("create_dirs", True))
|
||||
if create_dirs:
|
||||
destination_root.mkdir(parents=True, exist_ok=True)
|
||||
elif not destination_root.exists():
|
||||
raise FileNotFoundError(f"Destination directory does not exist: {destination_root}")
|
||||
elif not destination_root.is_dir():
|
||||
raise NotADirectoryError(f"Destination is not a directory: {destination_root}")
|
||||
|
||||
title = str(kwargs.get("title") or "").strip()
|
||||
if not title:
|
||||
title = source_path.stem.replace("_", " ").strip()
|
||||
base_name = sanitize_filename(title or source_path.stem)
|
||||
|
||||
file_ext = source_path.suffix
|
||||
if file_ext and base_name.lower().endswith(file_ext.lower()):
|
||||
target_name = base_name
|
||||
else:
|
||||
target_name = base_name + file_ext
|
||||
|
||||
direct_export_download = bool(kwargs.get("direct_export_download", False))
|
||||
target_path = source_path if direct_export_download else destination_root / target_name
|
||||
|
||||
if not direct_export_download:
|
||||
if target_path.exists():
|
||||
target_path = unique_path(target_path)
|
||||
shutil.copy2(str(source_path), target_path)
|
||||
_copy_sidecars(source_path, target_path)
|
||||
|
||||
tags = list(kwargs.get("tags") or [])
|
||||
urls = list(kwargs.get("urls") or [])
|
||||
hash_value = str(kwargs.get("hash_value") or "").strip() or None
|
||||
if not hash_value:
|
||||
def _set_status(text: str) -> None:
|
||||
if pipeline_progress is None or not hasattr(pipeline_progress, "set_status"):
|
||||
return
|
||||
try:
|
||||
hash_value = sha256_file(target_path)
|
||||
pipeline_progress.set_status(f"local: {text}")
|
||||
except Exception:
|
||||
hash_value = None
|
||||
pass
|
||||
|
||||
def _clear_status() -> None:
|
||||
if pipeline_progress is None or not hasattr(pipeline_progress, "clear_status"):
|
||||
return
|
||||
try:
|
||||
pipeline_progress.clear_status()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
relationships = kwargs.get("relationships")
|
||||
try:
|
||||
write_tags(target_path, tags, urls, hash_value=hash_value)
|
||||
write_metadata(
|
||||
target_path,
|
||||
hash_value=hash_value,
|
||||
url=urls,
|
||||
relationships=relationships or [],
|
||||
requested_instance = str(kwargs.get("instance") or kwargs.get("store") or "").strip() or None
|
||||
resolved_name, settings = self.resolve_destination(
|
||||
requested_instance,
|
||||
require_explicit=bool(requested_instance),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
destination_text = str(settings.get("path") or "").strip()
|
||||
if not destination_text:
|
||||
requested_label = requested_instance or "<default>"
|
||||
raise ValueError(
|
||||
f"Local destination '{requested_label}' is not configured. Use -plugin local -instance <name|path>."
|
||||
)
|
||||
|
||||
extra_updates: Dict[str, Any] = {
|
||||
"url": urls,
|
||||
"export_path": str(destination_root),
|
||||
}
|
||||
if resolved_name:
|
||||
extra_updates["instance"] = resolved_name
|
||||
if relationships:
|
||||
extra_updates["relationships"] = relationships
|
||||
destination_root = Path(destination_text).expanduser()
|
||||
create_dirs = bool(settings.get("create_dirs", True))
|
||||
if create_dirs:
|
||||
destination_root.mkdir(parents=True, exist_ok=True)
|
||||
elif not destination_root.exists():
|
||||
raise FileNotFoundError(f"Destination directory does not exist: {destination_root}")
|
||||
elif not destination_root.is_dir():
|
||||
raise NotADirectoryError(f"Destination is not a directory: {destination_root}")
|
||||
|
||||
return {
|
||||
"hash": hash_value or "unknown",
|
||||
"store": "local",
|
||||
"provider": self.name,
|
||||
"path": str(target_path),
|
||||
"tag": tags,
|
||||
"title": title or target_path.name,
|
||||
"relationships": relationships,
|
||||
"extra": extra_updates,
|
||||
}
|
||||
title = str(kwargs.get("title") or "").strip()
|
||||
if not title:
|
||||
title = source_path.stem.replace("_", " ").strip()
|
||||
base_name = sanitize_filename(title or source_path.stem)
|
||||
|
||||
file_ext = source_path.suffix
|
||||
if file_ext and base_name.lower().endswith(file_ext.lower()):
|
||||
target_name = base_name
|
||||
else:
|
||||
target_name = base_name + file_ext
|
||||
|
||||
direct_export_download = bool(kwargs.get("direct_export_download", False))
|
||||
target_path = source_path if direct_export_download else destination_root / target_name
|
||||
|
||||
if not direct_export_download:
|
||||
if target_path.exists():
|
||||
target_path = unique_path(target_path)
|
||||
_set_status(f"copying {target_path.name}")
|
||||
_copy_with_progress(
|
||||
source_path,
|
||||
target_path,
|
||||
pipeline_progress=pipeline_progress,
|
||||
label=str(target_path.name or source_path.name or "local export"),
|
||||
)
|
||||
_copy_sidecars(source_path, target_path)
|
||||
else:
|
||||
_set_status(f"finalizing {target_path.name}")
|
||||
|
||||
tags = list(kwargs.get("tags") or [])
|
||||
urls = list(kwargs.get("urls") or [])
|
||||
hash_value = str(kwargs.get("hash_value") or "").strip() or None
|
||||
if not hash_value:
|
||||
try:
|
||||
hash_value = sha256_file(target_path)
|
||||
except Exception:
|
||||
hash_value = None
|
||||
|
||||
relationships = kwargs.get("relationships")
|
||||
try:
|
||||
_set_status(f"writing metadata for {target_path.name}")
|
||||
write_tags(
|
||||
target_path,
|
||||
tags,
|
||||
urls,
|
||||
hash_value=hash_value,
|
||||
emit_debug=False,
|
||||
)
|
||||
write_metadata(
|
||||
target_path,
|
||||
hash_value=hash_value,
|
||||
url=urls,
|
||||
relationships=relationships or [],
|
||||
emit_debug=False,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
extra_updates: Dict[str, Any] = {
|
||||
"url": urls,
|
||||
"export_path": str(destination_root),
|
||||
}
|
||||
if resolved_name:
|
||||
extra_updates["instance"] = resolved_name
|
||||
if relationships:
|
||||
extra_updates["relationships"] = relationships
|
||||
|
||||
return {
|
||||
"hash": hash_value or "unknown",
|
||||
"store": "local",
|
||||
"provider": self.name,
|
||||
"path": str(target_path),
|
||||
"tag": tags,
|
||||
"title": title or target_path.name,
|
||||
"relationships": relationships,
|
||||
"extra": extra_updates,
|
||||
}
|
||||
finally:
|
||||
_clear_status()
|
||||
Reference in New Issue
Block a user