fdf
This commit is contained in:
@@ -49,9 +49,11 @@ class Add_Url(sh.Cmdlet):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
query_hash = sh.parse_single_hash_query(parsed.get("query"))
|
||||
if parsed.get("query") and not query_hash:
|
||||
log("Error: -query must be of the form hash:<sha256>")
|
||||
query_hash, query_valid = sh.require_single_hash_query(
|
||||
parsed.get("query"),
|
||||
"Error: -query must be of the form hash:<sha256>",
|
||||
)
|
||||
if not query_valid:
|
||||
return 1
|
||||
|
||||
# Bulk input is common in pipelines; treat a list of PipeObjects as a batch.
|
||||
@@ -117,125 +119,53 @@ class Add_Url(sh.Cmdlet):
|
||||
try:
|
||||
storage = Store(config)
|
||||
|
||||
def _merge_urls(existing: Any, incoming: List[str]) -> List[str]:
|
||||
out: List[str] = []
|
||||
try:
|
||||
if isinstance(existing, str):
|
||||
out.extend(
|
||||
[p.strip() for p in existing.split(",") if p.strip()]
|
||||
)
|
||||
elif isinstance(existing, (list, tuple)):
|
||||
out.extend([str(u).strip() for u in existing if str(u).strip()])
|
||||
except Exception:
|
||||
out = []
|
||||
for u in incoming:
|
||||
if u and u not in out:
|
||||
out.append(u)
|
||||
return out
|
||||
|
||||
def _set_item_url(item: Any, merged: List[str]) -> None:
|
||||
try:
|
||||
if isinstance(item, dict):
|
||||
if len(merged) == 1:
|
||||
item["url"] = merged[0]
|
||||
else:
|
||||
item["url"] = list(merged)
|
||||
return
|
||||
# PipeObject-like
|
||||
if hasattr(item, "url"):
|
||||
if len(merged) == 1:
|
||||
setattr(item, "url", merged[0])
|
||||
else:
|
||||
setattr(item, "url", list(merged))
|
||||
except Exception:
|
||||
return
|
||||
|
||||
# Build batches per store.
|
||||
store_override = parsed.get("store")
|
||||
batch: Dict[str,
|
||||
List[Tuple[str,
|
||||
List[str]]]] = {}
|
||||
pass_through: List[Any] = []
|
||||
|
||||
if results:
|
||||
for item in results:
|
||||
pass_through.append(item)
|
||||
def _warn(message: str) -> None:
|
||||
ctx.print_if_visible(f"[add-url] Warning: {message}", file=sys.stderr)
|
||||
|
||||
raw_hash = query_hash or sh.get_field(item, "hash")
|
||||
raw_store = store_override or sh.get_field(item, "store")
|
||||
if not raw_hash or not raw_store:
|
||||
ctx.print_if_visible(
|
||||
"[add-url] Warning: Item missing hash/store; skipping",
|
||||
file=sys.stderr
|
||||
)
|
||||
continue
|
||||
|
||||
normalized = sh.normalize_hash(raw_hash)
|
||||
if not normalized:
|
||||
ctx.print_if_visible(
|
||||
"[add-url] Warning: Item has invalid hash; skipping",
|
||||
file=sys.stderr
|
||||
)
|
||||
continue
|
||||
|
||||
store_text = str(raw_store).strip()
|
||||
if not store_text:
|
||||
ctx.print_if_visible(
|
||||
"[add-url] Warning: Item has empty store; skipping",
|
||||
file=sys.stderr
|
||||
)
|
||||
continue
|
||||
|
||||
# Validate backend exists (skip PATH/unknown).
|
||||
if not storage.is_available(store_text):
|
||||
ctx.print_if_visible(
|
||||
f"[add-url] Warning: Store '{store_text}' not configured; skipping",
|
||||
file=sys.stderr,
|
||||
)
|
||||
continue
|
||||
|
||||
batch.setdefault(store_text, []).append((normalized, list(urls)))
|
||||
batch, pass_through = sh.collect_store_hash_value_batch(
|
||||
results,
|
||||
store_registry=storage,
|
||||
value_resolver=lambda _item: list(urls),
|
||||
override_hash=query_hash,
|
||||
override_store=store_override,
|
||||
on_warning=_warn,
|
||||
)
|
||||
|
||||
# Execute per-store batches.
|
||||
for store_text, pairs in batch.items():
|
||||
try:
|
||||
backend = storage[store_text]
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Coalesce duplicates per hash before passing to backend.
|
||||
merged: Dict[str,
|
||||
List[str]] = {}
|
||||
for h, ulist in pairs:
|
||||
merged.setdefault(h, [])
|
||||
for u in ulist or []:
|
||||
if u and u not in merged[h]:
|
||||
merged[h].append(u)
|
||||
|
||||
bulk_pairs = [(h, merged[h]) for h in merged.keys()]
|
||||
|
||||
bulk_fn = getattr(backend, "add_url_bulk", None)
|
||||
if callable(bulk_fn):
|
||||
bulk_fn(bulk_pairs, config=config)
|
||||
else:
|
||||
for h, ulist in bulk_pairs:
|
||||
backend.add_url(h, ulist, config=config)
|
||||
|
||||
storage, batch_stats = sh.run_store_hash_value_batches(
|
||||
config,
|
||||
batch,
|
||||
bulk_method_name="add_url_bulk",
|
||||
single_method_name="add_url",
|
||||
store_registry=storage,
|
||||
)
|
||||
for store_text, item_count, _value_count in batch_stats:
|
||||
ctx.print_if_visible(
|
||||
f"✓ add-url: {len(urls)} url(s) for {len(bulk_pairs)} item(s) in '{store_text}'",
|
||||
f"✓ add-url: {len(urls)} url(s) for {item_count} item(s) in '{store_text}'",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
# Pass items through unchanged (but update url field for convenience).
|
||||
for item in pass_through:
|
||||
existing = sh.get_field(item, "url")
|
||||
merged = _merge_urls(existing, list(urls))
|
||||
_set_item_url(item, merged)
|
||||
merged = sh.merge_urls(existing, list(urls))
|
||||
sh.set_item_urls(item, merged)
|
||||
ctx.emit(item)
|
||||
return 0
|
||||
|
||||
# Single-item mode
|
||||
backend = storage[str(store_name)]
|
||||
backend, storage, exc = sh.get_store_backend(
|
||||
config,
|
||||
str(store_name),
|
||||
store_registry=storage,
|
||||
)
|
||||
if backend is None:
|
||||
log(f"Error: Storage backend '{store_name}' not configured")
|
||||
return 1
|
||||
backend.add_url(str(file_hash), urls, config=config)
|
||||
ctx.print_if_visible(
|
||||
f"✓ add-url: {len(urls)} url(s) added",
|
||||
@@ -243,14 +173,11 @@ class Add_Url(sh.Cmdlet):
|
||||
)
|
||||
if result is not None:
|
||||
existing = sh.get_field(result, "url")
|
||||
merged = _merge_urls(existing, list(urls))
|
||||
_set_item_url(result, merged)
|
||||
merged = sh.merge_urls(existing, list(urls))
|
||||
sh.set_item_urls(result, merged)
|
||||
ctx.emit(result)
|
||||
return 0
|
||||
|
||||
except KeyError:
|
||||
log(f"Error: Storage backend '{store_name}' not configured")
|
||||
return 1
|
||||
except Exception as exc:
|
||||
log(f"Error adding URL: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
Reference in New Issue
Block a user