goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

dfdf

Browse Source
This commit is contained in:
nose
2025-12-20 23:57:44 -08:00
parent b75faa49a2
commit 8ca5783970
39 changed files with 4294 additions and 1722 deletions
Download Patch File Download Diff File Expand all files Collapse all files

629
cmdlet/add_file.py
View File

@@ -109,6 +109,7 @@ class Add_File(Cmdlet):
collected_payloads: List[Dict[str, Any]] = []
pending_relationship_pairs: Dict[str, set[tuple[str, str]]] = {}
pending_url_associations: Dict[str, List[tuple[str, List[str]]]] = {}
successes = 0
failures = 0
@@ -118,6 +119,110 @@ class Add_File(Cmdlet):
want_final_search_store = bool(is_last_stage) and bool(is_storage_backend_location) and bool(location)
auto_search_store_after_add = False
# When ingesting multiple items into a backend store, defer URL association and
# apply it once at the end (bulk) to avoid per-item URL API calls.
defer_url_association = bool(is_storage_backend_location) and bool(location) and len(items_to_process) > 1
# If we are going to persist results (-store / -provider) and the piped input contains
# URL download targets (e.g. playlist rows), preflight URL duplicates once up-front.
# IMPORTANT: Do not treat a *source URL* on an already-local file (e.g. screen-shot)
# as a download target; that would trigger yt-dlp preflights for non-yt-dlp URLs.
skip_url_downloads: set[str] = set()
download_mode_hint: Optional[str] = None
forced_ytdl_format: Optional[str] = None
if (provider_name or location) and isinstance(items_to_process, list) and items_to_process:
url_candidates: List[str] = []
for it in items_to_process:
try:
po_probe = coerce_to_pipe_object(it, path_arg)
except Exception:
continue
# If the piped item already points at a local file, we are *ingesting* it,
# not downloading it. Skip URL-preflight and yt-dlp probing for those.
try:
po_path = getattr(po_probe, "path", None)
po_path_s = str(po_path or "").strip()
if po_path_s and not po_path_s.lower().startswith(("http://", "https://", "magnet:", "torrent:")):
continue
except Exception:
pass
try:
for u in (self._get_url(it, po_probe) or []):
s = str(u or "").strip()
if not s:
continue
if s.lower().startswith(("http://", "https://", "magnet:", "torrent:")):
url_candidates.append(s)
except Exception:
continue
# Only meaningful when targeting a registered backend store.
if url_candidates and is_storage_backend_location and location:
# De-dupe in-order to keep logs stable.
seen: set[str] = set()
unique_urls: List[str] = []
for u in url_candidates:
if u in seen:
continue
seen.add(u)
unique_urls.append(u)
try:
skip_url_downloads = self._preflight_url_duplicates_bulk(unique_urls, config)
except Exception:
skip_url_downloads = set()
# Batch-level format preflight:
# - If the sample URL only has one available format, force it for the batch.
# - If the sample URL appears audio-only (no video codecs), prefer audio mode.
try:
from cmdlet.download_media import is_url_supported_by_ytdlp, list_formats
from tool.ytdlp import YtDlpTool
sample_url = unique_urls[0] if unique_urls else None
if sample_url and is_url_supported_by_ytdlp(str(sample_url)):
cf = None
try:
cookie_path = YtDlpTool(config).resolve_cookiefile()
if cookie_path is not None and cookie_path.is_file():
cf = str(cookie_path)
except Exception:
cf = None
fmts = list_formats(
str(sample_url),
no_playlist=False,
playlist_items=None,
cookiefile=cf,
)
if isinstance(fmts, list) and fmts:
has_video = False
try:
for f in fmts:
if not isinstance(f, dict):
continue
vcodec = str(f.get("vcodec", "none") or "none").strip().lower()
if vcodec and vcodec != "none":
has_video = True
break
except Exception:
has_video = False
download_mode_hint = "video" if has_video else "audio"
if len(fmts) == 1 and isinstance(fmts[0], dict):
fid = str(fmts[0].get("format_id") or "").strip()
if fid:
forced_ytdl_format = fid
except Exception:
download_mode_hint = download_mode_hint
forced_ytdl_format = forced_ytdl_format
processed_url_items: set[str] = set()
for item in items_to_process:
pipe_obj = coerce_to_pipe_object(item, path_arg)
@@ -244,7 +349,148 @@ class Add_File(Cmdlet):
if isinstance(media_path_or_url, str) and media_path_or_url.lower().startswith(
("http://", "https://", "magnet:", "torrent:")
):
code = self._delegate_to_download_data(item, media_path_or_url, location, provider_name, args, config)
# If the user provided a destination (-store / -provider), download here and then
# continue normal add-file logic so the downloaded file is actually ingested.
url_str = str(media_path_or_url)
if (provider_name or location):
# Avoid re-processing the same URL multiple times in a batch.
if url_str in processed_url_items:
successes += 1
continue
processed_url_items.add(url_str)
# If bulk preflight found this URL already stored, skip downloading.
if url_str in skip_url_downloads:
log(f"Skipping download (already stored): {url_str}", file=sys.stderr)
successes += 1
continue
downloaded_pipe_dicts = self._download_streaming_url_as_pipe_objects(
url_str,
config,
mode_hint=download_mode_hint,
ytdl_format_hint=forced_ytdl_format,
)
if not downloaded_pipe_dicts:
failures += 1
continue
# Merge original tags/notes/relationships into each downloaded item and ingest.
for dl_item in downloaded_pipe_dicts:
try:
if isinstance(dl_item, dict):
# Merge tags
base_tags = list(getattr(pipe_obj, "tag", None) or [])
if base_tags:
dl_tags = list(dl_item.get("tag") or [])
dl_item["tag"] = merge_sequences(dl_tags, base_tags, case_sensitive=False)
# Carry notes/relationships forward when present on the original.
base_notes = getattr(pipe_obj, "notes", None)
if base_notes and ("notes" not in dl_item):
dl_item["notes"] = base_notes
base_rels = getattr(pipe_obj, "relationships", None)
if base_rels and ("relationships" not in dl_item):
dl_item["relationships"] = base_rels
except Exception:
pass
dl_pipe_obj = coerce_to_pipe_object(dl_item, None)
try:
dl_media_path = Path(str(getattr(dl_pipe_obj, "path", "") or ""))
except Exception:
dl_media_path = None
if dl_media_path is None or not self._validate_source(dl_media_path):
failures += 1
continue
if provider_name:
if str(provider_name).strip().lower() == "matrix":
room_id = None
if provider_room:
room_id = str(provider_room).strip()
if not room_id:
try:
matrix_conf = config.get("provider", {}).get("matrix", {}) if isinstance(config, dict) else {}
room_id = str(matrix_conf.get("room_id") or "").strip() or None
except Exception:
room_id = None
if not room_id:
pending = [
{
"path": str(dl_media_path),
"pipe_obj": dl_pipe_obj,
"delete_after": bool(delete_after_item),
}
]
return self._matrix_prompt_room_selection(pending, config, list(args))
code = self._handle_matrix_upload(
dl_media_path,
dl_pipe_obj,
config,
delete_after_item,
room_id=room_id,
)
else:
code = self._handle_provider_upload(
dl_media_path,
provider_name,
dl_pipe_obj,
config,
delete_after_item,
)
if code == 0:
successes += 1
else:
failures += 1
continue
if location:
try:
store = Store(config)
backends = store.list_backends()
if location in backends:
code = self._handle_storage_backend(
dl_item,
dl_media_path,
location,
dl_pipe_obj,
config,
delete_after_item,
collect_payloads=collected_payloads,
collect_relationship_pairs=pending_relationship_pairs,
defer_url_association=defer_url_association,
pending_url_associations=pending_url_associations,
suppress_last_stage_overlay=want_final_search_store,
auto_search_store=auto_search_store_after_add,
)
else:
code = self._handle_local_export(
dl_media_path,
location,
dl_pipe_obj,
config,
delete_after_item,
)
except Exception as exc:
debug(f"[add-file] ERROR: Failed to resolve location: {exc}")
log(f"Invalid location: {location}", file=sys.stderr)
failures += 1
continue
if code == 0:
successes += 1
else:
failures += 1
continue
# Finished processing all downloaded items for this URL.
continue
# No destination specified: keep legacy behavior (download-media only).
code = self._delegate_to_download_data(item, url_str, location, provider_name, args, config)
if code == 0:
successes += 1
else:
@@ -303,6 +549,8 @@ class Add_File(Cmdlet):
delete_after_item,
collect_payloads=collected_payloads,
collect_relationship_pairs=pending_relationship_pairs,
defer_url_association=defer_url_association,
pending_url_associations=pending_url_associations,
suppress_last_stage_overlay=want_final_search_store,
auto_search_store=auto_search_store_after_add,
)
@@ -329,6 +577,13 @@ class Add_File(Cmdlet):
except Exception:
pass
# Apply deferred url associations (bulk) before showing the final store table.
if pending_url_associations:
try:
Add_File._apply_pending_url_associations(pending_url_associations, config)
except Exception:
pass
# Always end add-file -store (when last stage) by showing the canonical store table.
# This keeps output consistent and ensures @N selection works for multi-item ingests.
if want_final_search_store and collected_payloads:
@@ -383,7 +638,7 @@ class Add_File(Cmdlet):
query = "hash:" + ",".join(hashes)
args = ["-store", str(store), query]
log(f"[add-file] Refresh: search-store -store {store} \"{query}\"", file=sys.stderr)
debug(f"[add-file] Refresh: search-store -store {store} \"{query}\"")
# Run search-store under a temporary stage context so its ctx.emit() calls
# don't interfere with the outer add-file pipeline stage.
@@ -1440,6 +1695,292 @@ class Add_File(Cmdlet):
return 0
@staticmethod
def _preflight_url_duplicates_bulk(urls: Sequence[str], config: Dict[str, Any]) -> set[str]:
"""Return a set of URLs that appear to already exist in any searchable backend.
This is a best-effort check used to avoid re-downloading already-stored media when
a batch of URL items is piped into add-file.
"""
skip: set[str] = set()
try:
storage = Store(config)
backend_names = list(storage.list_searchable_backends() or [])
except Exception:
return skip
for raw in urls:
u = str(raw or "").strip()
if not u:
continue
for backend_name in backend_names:
try:
if str(backend_name).strip().lower() == "temp":
continue
except Exception:
pass
try:
backend = storage[backend_name]
except Exception:
continue
try:
hits = backend.search(f"url:{u}", limit=1) or []
except Exception:
hits = []
if hits:
skip.add(u)
break
return skip
@staticmethod
def _download_streaming_url_as_pipe_objects(
url: str,
config: Dict[str, Any],
*,
mode_hint: Optional[str] = None,
ytdl_format_hint: Optional[str] = None,
) -> List[Dict[str, Any]]:
"""Download a yt-dlp-supported URL and return PipeObject-style dict(s).
This does not rely on pipeline stage context and is used so add-file can ingest
URL selections directly (download -> add to store/provider) in one invocation.
"""
url_str = str(url or "").strip()
if not url_str:
return []
try:
from cmdlet.download_media import (
CMDLET as dl_cmdlet,
_download_with_timeout,
is_url_supported_by_ytdlp,
list_formats,
_format_chapters_note,
_best_subtitle_sidecar,
_read_text_file,
)
from models import DownloadOptions
from tool.ytdlp import YtDlpTool
except Exception:
return []
if not is_url_supported_by_ytdlp(url_str):
return []
try:
from config import resolve_output_dir
out_dir = resolve_output_dir(config)
if out_dir is None:
return []
except Exception:
return []
cookies_path = None
try:
cookie_candidate = YtDlpTool(config).resolve_cookiefile()
if cookie_candidate is not None and cookie_candidate.is_file():
cookies_path = cookie_candidate
except Exception:
cookies_path = None
quiet_download = False
try:
quiet_download = bool((config or {}).get("_quiet_background_output"))
except Exception:
quiet_download = False
# Decide download mode.
# Default to video unless we have a hint or the URL appears to be audio-only.
mode = str(mode_hint or "").strip().lower() if mode_hint else ""
if mode not in {"audio", "video"}:
mode = "video"
# Best-effort: infer from formats for this URL (one-time, no playlist probing).
try:
cf = str(cookies_path) if cookies_path is not None and cookies_path.is_file() else None
fmts_probe = list_formats(url_str, no_playlist=False, playlist_items=None, cookiefile=cf)
if isinstance(fmts_probe, list) and fmts_probe:
has_video = False
for f in fmts_probe:
if not isinstance(f, dict):
continue
vcodec = str(f.get("vcodec", "none") or "none").strip().lower()
if vcodec and vcodec != "none":
has_video = True
break
mode = "video" if has_video else "audio"
except Exception:
mode = "video"
# Pick a safe initial format selector.
# Important: yt-dlp defaults like "251/140" are YouTube-specific and break Bandcamp.
fmt_hint = str(ytdl_format_hint).strip() if ytdl_format_hint else ""
if fmt_hint:
chosen_format: Optional[str] = fmt_hint
else:
chosen_format = None
if mode == "audio":
# Generic audio selector that works across extractors.
chosen_format = "bestaudio/best"
opts = DownloadOptions(
url=url_str,
mode=mode,
output_dir=Path(out_dir),
cookies_path=cookies_path,
ytdl_format=chosen_format,
quiet=quiet_download,
embed_chapters=True,
write_sub=True,
)
# Download with a small amount of resilience for format errors.
try:
result_obj = _download_with_timeout(opts, timeout_seconds=300)
except Exception as exc:
msg = str(exc)
# If a format is invalid/unsupported, try:
# - if only one format exists, retry with that id
# - else for audio-only sources, retry with bestaudio/best
try:
format_error = "Requested format is not available" in msg
except Exception:
format_error = False
if format_error:
try:
cf = str(cookies_path) if cookies_path is not None and cookies_path.is_file() else None
fmts = list_formats(url_str, no_playlist=False, playlist_items=None, cookiefile=cf)
if isinstance(fmts, list) and len(fmts) == 1 and isinstance(fmts[0], dict):
fid = str(fmts[0].get("format_id") or "").strip()
if fid:
opts = DownloadOptions(
url=url_str,
mode=mode,
output_dir=Path(out_dir),
cookies_path=cookies_path,
ytdl_format=fid,
quiet=quiet_download,
embed_chapters=True,
write_sub=True,
)
result_obj = _download_with_timeout(opts, timeout_seconds=300)
# proceed
else:
raise
elif mode == "audio" and (not chosen_format or chosen_format != "bestaudio/best"):
opts = DownloadOptions(
url=url_str,
mode=mode,
output_dir=Path(out_dir),
cookies_path=cookies_path,
ytdl_format="bestaudio/best",
quiet=quiet_download,
embed_chapters=True,
write_sub=True,
)
result_obj = _download_with_timeout(opts, timeout_seconds=300)
else:
raise
except Exception as exc2:
log(f"[add-file] Download failed for {url_str}: {exc2}", file=sys.stderr)
return []
else:
log(f"[add-file] Download failed for {url_str}: {exc}", file=sys.stderr)
return []
results: List[Any]
if isinstance(result_obj, list):
results = list(result_obj)
else:
paths = getattr(result_obj, "paths", None)
if isinstance(paths, list) and paths:
# Section downloads: create one result per file.
from models import DownloadMediaResult
results = []
for p in paths:
try:
p_path = Path(p)
except Exception:
continue
if not p_path.exists() or p_path.is_dir():
continue
try:
hv = sha256_file(p_path)
except Exception:
hv = None
try:
results.append(
DownloadMediaResult(
path=p_path,
info=getattr(result_obj, "info", {}) or {},
tag=list(getattr(result_obj, "tag", []) or []),
source_url=getattr(result_obj, "source_url", None) or url_str,
hash_value=hv,
)
)
except Exception:
continue
else:
results = [result_obj]
out: List[Dict[str, Any]] = []
for downloaded in results:
try:
po = dl_cmdlet._build_pipe_object(downloaded, url_str, opts)
# Attach chapter timestamps note (best-effort).
try:
info = downloaded.info if isinstance(getattr(downloaded, "info", None), dict) else {}
except Exception:
info = {}
try:
chapters_text = _format_chapters_note(info)
except Exception:
chapters_text = None
if chapters_text:
notes = po.get("notes")
if not isinstance(notes, dict):
notes = {}
notes.setdefault("chapters", chapters_text)
po["notes"] = notes
# Capture subtitle sidecar into notes and remove it so add-file won't ingest it later.
try:
media_path = Path(str(po.get("path") or ""))
except Exception:
media_path = None
if media_path is not None and media_path.exists() and media_path.is_file():
try:
sub_path = _best_subtitle_sidecar(media_path)
except Exception:
sub_path = None
if sub_path is not None:
sub_text = _read_text_file(sub_path)
if sub_text:
notes = po.get("notes")
if not isinstance(notes, dict):
notes = {}
notes["sub"] = sub_text
po["notes"] = notes
try:
sub_path.unlink()
except Exception:
pass
# Mark as temp artifact from download-media so add-file can auto-delete after ingest.
po["action"] = "cmdlet:download-media"
po["is_temp"] = True
out.append(po)
except Exception:
continue
return out
@staticmethod
def _download_soulseek_file(
result: Any,
@@ -1640,7 +2181,9 @@ class Add_File(Cmdlet):
ctx.set_current_stage_table(table)
print()
print(table.format_plain())
from rich_display import stdout_console
stdout_console().print(table)
print("\nSelect room(s) with @N (e.g. @1 or @1-3) to upload the selected item(s)")
return 0
@@ -1710,6 +2253,8 @@ class Add_File(Cmdlet):
*,
collect_payloads: Optional[List[Dict[str, Any]]] = None,
collect_relationship_pairs: Optional[Dict[str, set[tuple[str, str]]]] = None,
defer_url_association: bool = False,
pending_url_associations: Optional[Dict[str, List[tuple[str, List[str]]]]] = None,
suppress_last_stage_overlay: bool = False,
auto_search_store: bool = True,
) -> int:
@@ -1822,7 +2367,7 @@ class Add_File(Cmdlet):
media_path,
title=title,
tag=tags,
url=url
url=[] if (defer_url_association and url) else url
)
##log(f"✓ File added to '{backend_name}': {file_identifier}", file=sys.stderr)
@@ -1859,10 +2404,16 @@ class Add_File(Cmdlet):
# If we have url(s), ensure they get associated with the destination file.
# This mirrors `add-url` behavior but avoids emitting extra pipeline noise.
if url:
try:
backend.add_url(resolved_hash, list(url))
except Exception:
pass
if defer_url_association and pending_url_associations is not None:
try:
pending_url_associations.setdefault(str(backend_name), []).append((str(resolved_hash), list(url)))
except Exception:
pass
else:
try:
backend.add_url(resolved_hash, list(url))
except Exception:
pass
# If a subtitle note was provided upstream (e.g., download-media writes notes.sub),
# persist it automatically like add-note would.
@@ -1965,6 +2516,68 @@ class Add_File(Cmdlet):
# --- Helpers ---
@staticmethod
def _apply_pending_url_associations(pending: Dict[str, List[tuple[str, List[str]]]], config: Dict[str, Any]) -> None:
"""Apply deferred URL associations in bulk, grouped per backend."""
try:
store = Store(config)
except Exception:
return
for backend_name, pairs in (pending or {}).items():
if not pairs:
continue
try:
backend = store[backend_name]
except Exception:
continue
# Merge URLs per hash and de-duplicate.
merged: Dict[str, List[str]] = {}
for file_hash, urls in pairs:
h = str(file_hash or "").strip().lower()
if len(h) != 64:
continue
url_list: List[str] = []
try:
for u in (urls or []):
s = str(u or "").strip()
if s:
url_list.append(s)
except Exception:
url_list = []
if not url_list:
continue
bucket = merged.setdefault(h, [])
seen = set(bucket)
for u in url_list:
if u in seen:
continue
seen.add(u)
bucket.append(u)
items: List[tuple[str, List[str]]] = [(h, u) for h, u in merged.items() if u]
if not items:
continue
bulk = getattr(backend, "add_url_bulk", None)
if callable(bulk):
try:
bulk(items)
continue
except Exception:
pass
single = getattr(backend, "add_url", None)
if callable(single):
for h, u in items:
try:
single(h, u)
except Exception:
continue
@staticmethod
def _load_sidecar_bundle(
media_path: Path,

48
cmdlet/add_note.py
View File

@@ -1,7 +1,7 @@
from __future__ import annotations
from pathlib import Path
from typing import Any, Dict, Optional, Sequence
from typing import Any, Dict, List, Optional, Sequence, Tuple
import sys
from SYS.logger import log
@@ -103,6 +103,9 @@ class Add_Note(Cmdlet):
store_registry = Store(config)
updated = 0
# Batch write plan: store -> [(hash, name, text), ...]
note_ops: Dict[str, List[Tuple[str, str, str]]] = {}
# Optional global fallback for note text from pipeline values.
# Allows patterns like: ... | add-note sub
pipeline_default_text = None
@@ -177,20 +180,43 @@ class Add_Note(Cmdlet):
log(f"[add_note] Error: Unknown store '{store_name}': {exc}", file=sys.stderr)
return 1
ok = False
try:
ok = bool(backend.set_note(resolved_hash, note_name, item_note_text, config=config))
except Exception as exc:
log(f"[add_note] Error: Failed to set note: {exc}", file=sys.stderr)
ok = False
if ok:
updated += 1
# Queue for bulk write per store. We still emit items immediately;
# the pipeline only advances after this cmdlet returns.
note_ops.setdefault(store_name, []).append((resolved_hash, note_name, item_note_text))
updated += 1
ctx.emit(res)
# Execute bulk writes per store.
wrote_any = False
for store_name, ops in note_ops.items():
if not ops:
continue
try:
backend = store_registry[store_name]
except Exception:
continue
bulk_fn = getattr(backend, "set_note_bulk", None)
if callable(bulk_fn):
try:
ok = bool(bulk_fn(list(ops), config=config))
wrote_any = wrote_any or ok or True
ctx.print_if_visible(f"✓ add-note: {len(ops)} item(s) in '{store_name}'", file=sys.stderr)
continue
except Exception as exc:
log(f"[add_note] Warning: bulk set_note failed for '{store_name}': {exc}; falling back", file=sys.stderr)
# Fallback: per-item writes
for file_hash, name, text in ops:
try:
ok = bool(backend.set_note(file_hash, name, text, config=config))
wrote_any = wrote_any or ok
except Exception:
continue
log(f"[add_note] Updated {updated} item(s)", file=sys.stderr)
return 0 if updated > 0 else 1
return 0 if (updated > 0 and wrote_any) else (0 if updated > 0 else 1)
CMDLET = Add_Note()

54
cmdlet/add_tag.py
View File

@@ -520,45 +520,13 @@ class Add_Tag(Cmdlet):
if new_tag.lower() not in existing_lower:
item_tag_to_add.append(new_tag)
# Namespace replacement: delete old namespace:* when adding namespace:value
removed_namespace_tag: list[str] = []
for new_tag in item_tag_to_add:
if not isinstance(new_tag, str) or ":" not in new_tag:
continue
ns = new_tag.split(":", 1)[0].strip()
if not ns:
continue
ns_prefix = ns.lower() + ":"
for t in existing_tag_list:
if t.lower().startswith(ns_prefix) and t.lower() != new_tag.lower():
removed_namespace_tag.append(t)
removed_namespace_tag = sorted({t for t in removed_namespace_tag})
actual_tag_to_add = [t for t in item_tag_to_add if isinstance(t, str) and t.lower() not in existing_lower]
changed = False
if removed_namespace_tag:
try:
ok_del = backend.delete_tag(resolved_hash, removed_namespace_tag, config=config)
if ok_del:
changed = True
except Exception as exc:
log(f"[add_tag] Warning: Failed deleting namespace tag: {exc}", file=sys.stderr)
if actual_tag_to_add:
try:
ok_add = backend.add_tag(resolved_hash, actual_tag_to_add, config=config)
if ok_add:
changed = True
else:
log("[add_tag] Warning: Store rejected tag update", file=sys.stderr)
except Exception as exc:
log(f"[add_tag] Warning: Failed adding tag: {exc}", file=sys.stderr)
if changed:
total_added += len(actual_tag_to_add)
total_modified += 1
try:
ok_add = backend.add_tag(resolved_hash, item_tag_to_add, config=config)
if not ok_add:
log("[add_tag] Warning: Store rejected tag update", file=sys.stderr)
except Exception as exc:
log(f"[add_tag] Warning: Failed adding tag: {exc}", file=sys.stderr)
try:
refreshed_tag, _src2 = backend.get_tag(resolved_hash, config=config)
@@ -566,6 +534,14 @@ class Add_Tag(Cmdlet):
except Exception:
refreshed_list = existing_tag_list
# Decide whether anything actually changed (case-sensitive so title casing updates count).
if set(refreshed_list) != set(existing_tag_list):
changed = True
before_lower = {t.lower() for t in existing_tag_list}
after_lower = {t.lower() for t in refreshed_list}
total_added += len(after_lower - before_lower)
total_modified += 1
# Update the result's tag using canonical field
if isinstance(res, models.PipeObject):
res.tag = refreshed_list
@@ -575,7 +551,7 @@ class Add_Tag(Cmdlet):
final_title = _extract_title_tag(refreshed_list)
_apply_title_to_result(res, final_title)
if final_title and (not original_title or final_title.lower() != original_title.lower()):
if final_title and (not original_title or final_title != original_title):
_refresh_result_table_title(final_title, resolved_hash, str(store_name), raw_path)
if changed:

155
cmdlet/add_url.py
View File

@@ -1,6 +1,6 @@
from __future__ import annotations
from typing import Any, Dict, Sequence
from typing import Any, Dict, List, Optional, Sequence, Tuple
import sys
import pipeline as ctx
@@ -39,28 +39,37 @@ class Add_Url(sh.Cmdlet):
log("Error: -query must be of the form hash:<sha256>")
return 1
# Bulk input is common in pipelines; treat a list of PipeObjects as a batch.
results: List[Any] = result if isinstance(result, list) else ([result] if result is not None else [])
if query_hash and len(results) > 1:
log("Error: -query hash:<sha256> cannot be used with multiple piped items")
return 1
# Extract hash and store from result or args
file_hash = query_hash or sh.get_field(result, "hash")
store_name = parsed.get("store") or sh.get_field(result, "store")
file_hash = query_hash or (sh.get_field(result, "hash") if result is not None else None)
store_name = parsed.get("store") or (sh.get_field(result, "store") if result is not None else None)
url_arg = parsed.get("url")
if not file_hash:
log("Error: No file hash provided (pipe an item or use -query \"hash:<sha256>\")")
return 1
if not store_name:
log("Error: No store name provided")
return 1
# If we have multiple piped items, we will resolve hash/store per item below.
if not results:
if not file_hash:
log("Error: No file hash provided (pipe an item or use -query \"hash:<sha256>\")")
return 1
if not store_name:
log("Error: No store name provided")
return 1
if not url_arg:
log("Error: No URL provided")
return 1
# Normalize hash
file_hash = sh.normalize_hash(file_hash)
if not file_hash:
log("Error: Invalid hash format")
return 1
# Normalize hash (single-item mode)
if not results and file_hash:
file_hash = sh.normalize_hash(file_hash)
if not file_hash:
log("Error: Invalid hash format")
return 1
# Parse url (comma-separated)
urls = [u.strip() for u in str(url_arg).split(',') if u.strip()]
@@ -71,12 +80,118 @@ class Add_Url(sh.Cmdlet):
# Get backend and add url
try:
storage = Store(config)
backend = storage[store_name]
backend.add_url(file_hash, urls)
for u in urls:
ctx.emit(f"Added URL: {u}")
def _merge_urls(existing: Any, incoming: List[str]) -> List[str]:
out: List[str] = []
try:
if isinstance(existing, str):
out.extend([p.strip() for p in existing.split(",") if p.strip()])
elif isinstance(existing, (list, tuple)):
out.extend([str(u).strip() for u in existing if str(u).strip()])
except Exception:
out = []
for u in incoming:
if u and u not in out:
out.append(u)
return out
def _set_item_url(item: Any, merged: List[str]) -> None:
try:
if isinstance(item, dict):
if len(merged) == 1:
item["url"] = merged[0]
else:
item["url"] = list(merged)
return
# PipeObject-like
if hasattr(item, "url"):
if len(merged) == 1:
setattr(item, "url", merged[0])
else:
setattr(item, "url", list(merged))
except Exception:
return
# Build batches per store.
store_override = parsed.get("store")
batch: Dict[str, List[Tuple[str, List[str]]]] = {}
pass_through: List[Any] = []
if results:
for item in results:
pass_through.append(item)
raw_hash = query_hash or sh.get_field(item, "hash")
raw_store = store_override or sh.get_field(item, "store")
if not raw_hash or not raw_store:
ctx.print_if_visible("[add-url] Warning: Item missing hash/store; skipping", file=sys.stderr)
continue
normalized = sh.normalize_hash(raw_hash)
if not normalized:
ctx.print_if_visible("[add-url] Warning: Item has invalid hash; skipping", file=sys.stderr)
continue
store_text = str(raw_store).strip()
if not store_text:
ctx.print_if_visible("[add-url] Warning: Item has empty store; skipping", file=sys.stderr)
continue
# Validate backend exists (skip PATH/unknown).
if not storage.is_available(store_text):
ctx.print_if_visible(
f"[add-url] Warning: Store '{store_text}' not configured; skipping", file=sys.stderr
)
continue
batch.setdefault(store_text, []).append((normalized, list(urls)))
# Execute per-store batches.
for store_text, pairs in batch.items():
try:
backend = storage[store_text]
except Exception:
continue
# Coalesce duplicates per hash before passing to backend.
merged: Dict[str, List[str]] = {}
for h, ulist in pairs:
merged.setdefault(h, [])
for u in (ulist or []):
if u and u not in merged[h]:
merged[h].append(u)
bulk_pairs = [(h, merged[h]) for h in merged.keys()]
bulk_fn = getattr(backend, "add_url_bulk", None)
if callable(bulk_fn):
bulk_fn(bulk_pairs, config=config)
else:
for h, ulist in bulk_pairs:
backend.add_url(h, ulist, config=config)
ctx.print_if_visible(
f"✓ add-url: {len(urls)} url(s) for {len(bulk_pairs)} item(s) in '{store_text}'",
file=sys.stderr,
)
# Pass items through unchanged (but update url field for convenience).
for item in pass_through:
existing = sh.get_field(item, "url")
merged = _merge_urls(existing, list(urls))
_set_item_url(item, merged)
ctx.emit(item)
return 0
# Single-item mode
backend = storage[str(store_name)]
backend.add_url(str(file_hash), urls, config=config)
ctx.print_if_visible(f"✓ add-url: {len(urls)} url(s) added", file=sys.stderr)
if result is not None:
existing = sh.get_field(result, "url")
merged = _merge_urls(existing, list(urls))
_set_item_url(result, merged)
ctx.emit(result)
return 0
except KeyError:

161
cmdlet/delete_file.py
View File

@@ -1,16 +1,19 @@
"""Delete-file cmdlet: Delete files from local storage and/or Hydrus."""
from __future__ import annotations
from typing import Any, Dict, Sequence
from typing import Any, Dict, List, Sequence
import sys
from pathlib import Path
from SYS.logger import debug, log
from SYS.utils import format_bytes
from Store.Folder import Folder
from Store import Store
from . import _shared as sh
from API import HydrusNetwork as hydrus_wrapper
import pipeline as ctx
from result_table import ResultTable, _format_size
from rich_display import stdout_console
class Delete_File(sh.Cmdlet):
@@ -38,9 +41,20 @@ class Delete_File(sh.Cmdlet):
)
self.register()
def _process_single_item(self, item: Any, override_hash: str | None, conserve: str | None,
lib_root: str | None, reason: str, config: Dict[str, Any]) -> bool:
"""Process deletion for a single item."""
def _process_single_item(
self,
item: Any,
override_hash: str | None,
conserve: str | None,
lib_root: str | None,
reason: str,
config: Dict[str, Any],
) -> List[Dict[str, Any]]:
"""Process deletion for a single item.
Returns display rows (for the final Rich table). Returning an empty list
indicates no delete occurred.
"""
# Handle item as either dict or object
if isinstance(item, dict):
hash_hex_raw = item.get("hash_hex") or item.get("hash")
@@ -50,6 +64,44 @@ class Delete_File(sh.Cmdlet):
hash_hex_raw = sh.get_field(item, "hash_hex") or sh.get_field(item, "hash")
target = sh.get_field(item, "target") or sh.get_field(item, "file_path") or sh.get_field(item, "path")
title_val = sh.get_field(item, "title") or sh.get_field(item, "name")
def _get_ext_from_item() -> str:
try:
if isinstance(item, dict):
ext_val = item.get("ext")
if ext_val:
return str(ext_val)
extra = item.get("extra")
if isinstance(extra, dict) and extra.get("ext"):
return str(extra.get("ext"))
else:
ext_val = sh.get_field(item, "ext")
if ext_val:
return str(ext_val)
extra = sh.get_field(item, "extra")
if isinstance(extra, dict) and extra.get("ext"):
return str(extra.get("ext"))
except Exception:
pass
# Fallback: infer from target path or title if it looks like a filename
try:
if isinstance(target, str) and target:
suffix = Path(target).suffix
if suffix:
return suffix.lstrip(".")
except Exception:
pass
try:
if title_val:
suffix = Path(str(title_val)).suffix
if suffix:
return suffix.lstrip(".")
except Exception:
pass
return ""
store = None
if isinstance(item, dict):
@@ -70,9 +122,16 @@ class Delete_File(sh.Cmdlet):
local_deleted = False
local_target = isinstance(target, str) and target.strip() and not str(target).lower().startswith(("http://", "https://"))
deleted_rows: List[Dict[str, Any]] = []
if conserve != "local" and local_target:
path = Path(str(target))
size_bytes: int | None = None
try:
if path.exists() and path.is_file():
size_bytes = int(path.stat().st_size)
except Exception:
size_bytes = None
# If lib_root is provided and this is from a folder store, use the Folder class
if lib_root:
@@ -80,8 +139,15 @@ class Delete_File(sh.Cmdlet):
folder = Folder(Path(lib_root), name=store or "local")
if folder.delete_file(str(path)):
local_deleted = True
ctx.emit(f"Removed file: {path.name}")
log(f"Deleted: {path.name}", file=sys.stderr)
deleted_rows.append(
{
"title": str(title_val).strip() if title_val else path.name,
"store": store_label,
"hash": hash_hex or sh.normalize_hash(path.stem) or "",
"size_bytes": size_bytes,
"ext": _get_ext_from_item() or path.suffix.lstrip("."),
}
)
except Exception as exc:
debug(f"Folder.delete_file failed: {exc}", file=sys.stderr)
# Fallback to manual deletion
@@ -89,8 +155,15 @@ class Delete_File(sh.Cmdlet):
if path.exists() and path.is_file():
path.unlink()
local_deleted = True
ctx.emit(f"Removed local file: {path}")
log(f"Deleted: {path.name}", file=sys.stderr)
deleted_rows.append(
{
"title": str(title_val).strip() if title_val else path.name,
"store": store_label,
"hash": hash_hex or sh.normalize_hash(path.stem) or "",
"size_bytes": size_bytes,
"ext": _get_ext_from_item() or path.suffix.lstrip("."),
}
)
except Exception as exc:
log(f"Local delete failed: {exc}", file=sys.stderr)
else:
@@ -99,8 +172,15 @@ class Delete_File(sh.Cmdlet):
if path.exists() and path.is_file():
path.unlink()
local_deleted = True
ctx.emit(f"Removed local file: {path}")
log(f"Deleted: {path.name}", file=sys.stderr)
deleted_rows.append(
{
"title": str(title_val).strip() if title_val else path.name,
"store": store_label,
"hash": hash_hex or sh.normalize_hash(path.stem) or "",
"size_bytes": size_bytes,
"ext": _get_ext_from_item() or path.suffix.lstrip("."),
}
)
except Exception as exc:
log(f"Local delete failed: {exc}", file=sys.stderr)
@@ -168,26 +248,32 @@ class Delete_File(sh.Cmdlet):
except Exception:
# If it's not in Hydrus (e.g. 404 or similar), that's fine
if not local_deleted:
return False
return []
if hydrus_deleted and hash_hex:
title_str = str(title_val).strip() if title_val else ""
if reason:
if title_str:
ctx.emit(f"{hydrus_prefix} Deleted title:{title_str} hash:{hash_hex} (reason: {reason}).")
size_hint = None
try:
if isinstance(item, dict):
size_hint = item.get("size_bytes") or item.get("size")
else:
ctx.emit(f"{hydrus_prefix} Deleted hash:{hash_hex} (reason: {reason}).")
else:
if title_str:
ctx.emit(f"{hydrus_prefix} Deleted title:{title_str} hash:{hash_hex}.")
else:
ctx.emit(f"{hydrus_prefix} Deleted hash:{hash_hex}.")
size_hint = sh.get_field(item, "size_bytes") or sh.get_field(item, "size")
except Exception:
size_hint = None
deleted_rows.append(
{
"title": str(title_val).strip() if title_val else "",
"store": store_label,
"hash": hash_hex,
"size_bytes": size_hint,
"ext": _get_ext_from_item(),
}
)
if hydrus_deleted or local_deleted:
return True
return deleted_rows
log("Selected result has neither Hydrus hash nor local file target")
return False
return []
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
"""Execute delete-file command."""
@@ -257,15 +343,34 @@ class Delete_File(sh.Cmdlet):
return 1
success_count = 0
deleted_rows: List[Dict[str, Any]] = []
for item in items:
if self._process_single_item(item, override_hash, conserve, lib_root, reason, config):
rows = self._process_single_item(item, override_hash, conserve, lib_root, reason, config)
if rows:
success_count += 1
deleted_rows.extend(rows)
if success_count > 0:
# Clear cached tables/items so deleted entries are not redisplayed
if deleted_rows:
table = ResultTable("Deleted")
table.set_no_choice(True).set_preserve_order(True)
for row in deleted_rows:
result_row = table.add_row()
result_row.add_column("Title", row.get("title", ""))
result_row.add_column("Store", row.get("store", ""))
result_row.add_column("Hash", row.get("hash", ""))
result_row.add_column("Size", _format_size(row.get("size_bytes"), integer_only=False))
result_row.add_column("Ext", row.get("ext", ""))
# Display-only: print directly and do not affect selection/history.
try:
stdout_console().print()
stdout_console().print(table)
setattr(table, "_rendered_by_cmdlet", True)
except Exception:
pass
# Ensure no stale overlay/selection carries forward.
try:
ctx.set_last_result_table_overlay(None, None, None)
ctx.set_last_result_table(None, [])
ctx.set_last_result_items_only([])
ctx.set_current_stage_table(None)
except Exception:

141
cmdlet/delete_url.py
View File

@@ -1,6 +1,6 @@
from __future__ import annotations
from typing import Any, Dict, Sequence
from typing import Any, Dict, List, Optional, Sequence, Tuple
import sys
import pipeline as ctx
@@ -48,28 +48,37 @@ class Delete_Url(Cmdlet):
log("Error: -query must be of the form hash:<sha256>")
return 1
# Bulk input is common in pipelines; treat a list of PipeObjects as a batch.
results: List[Any] = result if isinstance(result, list) else ([result] if result is not None else [])
if query_hash and len(results) > 1:
log("Error: -query hash:<sha256> cannot be used with multiple piped items")
return 1
# Extract hash and store from result or args
file_hash = query_hash or get_field(result, "hash")
store_name = parsed.get("store") or get_field(result, "store")
file_hash = query_hash or (get_field(result, "hash") if result is not None else None)
store_name = parsed.get("store") or (get_field(result, "store") if result is not None else None)
url_arg = parsed.get("url")
if not file_hash:
log("Error: No file hash provided (pipe an item or use -query \"hash:<sha256>\")")
return 1
if not store_name:
log("Error: No store name provided")
return 1
# If we have multiple piped items, we will resolve hash/store per item below.
if not results:
if not file_hash:
log("Error: No file hash provided (pipe an item or use -query \"hash:<sha256>\")")
return 1
if not store_name:
log("Error: No store name provided")
return 1
if not url_arg:
log("Error: No URL provided")
return 1
# Normalize hash
file_hash = normalize_hash(file_hash)
if not file_hash:
log("Error: Invalid hash format")
return 1
# Normalize hash (single-item mode)
if not results and file_hash:
file_hash = normalize_hash(file_hash)
if not file_hash:
log("Error: Invalid hash format")
return 1
# Parse url (comma-separated)
urls = [u.strip() for u in str(url_arg).split(',') if u.strip()]
@@ -80,12 +89,104 @@ class Delete_Url(Cmdlet):
# Get backend and delete url
try:
storage = Store(config)
backend = storage[store_name]
backend.delete_url(file_hash, urls)
for u in urls:
ctx.emit(f"Deleted URL: {u}")
def _remove_urls(existing: Any, remove: List[str]) -> Any:
# Preserve prior shape: keep str when 1 url, list when multiple.
current: List[str] = []
try:
if isinstance(existing, str):
current = [p.strip() for p in existing.split(",") if p.strip()]
elif isinstance(existing, (list, tuple)):
current = [str(u).strip() for u in existing if str(u).strip()]
except Exception:
current = []
remove_set = {u for u in (remove or []) if u}
new_urls = [u for u in current if u not in remove_set]
if len(new_urls) == 1:
return new_urls[0]
return new_urls
def _set_item_url(item: Any, merged: Any) -> None:
try:
if isinstance(item, dict):
item["url"] = merged
return
if hasattr(item, "url"):
setattr(item, "url", merged)
except Exception:
return
store_override = parsed.get("store")
batch: Dict[str, List[Tuple[str, List[str]]]] = {}
pass_through: List[Any] = []
if results:
for item in results:
pass_through.append(item)
raw_hash = query_hash or get_field(item, "hash")
raw_store = store_override or get_field(item, "store")
if not raw_hash or not raw_store:
ctx.print_if_visible("[delete-url] Warning: Item missing hash/store; skipping", file=sys.stderr)
continue
normalized = normalize_hash(raw_hash)
if not normalized:
ctx.print_if_visible("[delete-url] Warning: Item has invalid hash; skipping", file=sys.stderr)
continue
store_text = str(raw_store).strip()
if not store_text:
ctx.print_if_visible("[delete-url] Warning: Item has empty store; skipping", file=sys.stderr)
continue
if not storage.is_available(store_text):
ctx.print_if_visible(
f"[delete-url] Warning: Store '{store_text}' not configured; skipping", file=sys.stderr
)
continue
batch.setdefault(store_text, []).append((normalized, list(urls)))
for store_text, pairs in batch.items():
try:
backend = storage[store_text]
except Exception:
continue
merged: Dict[str, List[str]] = {}
for h, ulist in pairs:
merged.setdefault(h, [])
for u in (ulist or []):
if u and u not in merged[h]:
merged[h].append(u)
bulk_pairs = [(h, merged[h]) for h in merged.keys()]
bulk_fn = getattr(backend, "delete_url_bulk", None)
if callable(bulk_fn):
bulk_fn(bulk_pairs, config=config)
else:
for h, ulist in bulk_pairs:
backend.delete_url(h, ulist, config=config)
ctx.print_if_visible(
f"✓ delete-url: {len(urls)} url(s) for {len(bulk_pairs)} item(s) in '{store_text}'",
file=sys.stderr,
)
for item in pass_through:
existing = get_field(item, "url")
_set_item_url(item, _remove_urls(existing, list(urls)))
ctx.emit(item)
return 0
# Single-item mode
backend = storage[str(store_name)]
backend.delete_url(str(file_hash), urls, config=config)
ctx.print_if_visible(f"✓ delete-url: {len(urls)} url(s) removed", file=sys.stderr)
if result is not None:
existing = get_field(result, "url")
_set_item_url(result, _remove_urls(existing, list(urls)))
ctx.emit(result)
return 0
except KeyError:

1213
cmdlet/download_media.py
View File

File diff suppressed because it is too large Load Diff

143
cmdlet/get_file.py
View File

@@ -6,7 +6,14 @@ import os
import sys
import shutil
import subprocess
import tempfile
import threading
import time
import http.server
from urllib.parse import quote
import webbrowser
from urllib.parse import urljoin
from urllib.request import pathname2url
import pipeline as ctx
from . import _shared as sh
@@ -56,7 +63,7 @@ class Get_File(sh.Cmdlet):
output_path = parsed.get("path")
output_name = parsed.get("name")
debug(f"[get-file] file_hash={file_hash[:12] if file_hash else None}... store_name={store_name}")
debug(f"[get-file] file_hash={file_hash} store_name={store_name}")
if not file_hash:
log("Error: No file hash provided (pipe an item or use -query \"hash:<sha256>\")")
@@ -83,7 +90,7 @@ class Get_File(sh.Cmdlet):
debug(f"[get-file] Getting metadata for hash...")
metadata = backend.get_metadata(file_hash)
if not metadata:
log(f"Error: File metadata not found for hash {file_hash[:12]}...")
log(f"Error: File metadata not found for hash {file_hash}")
return 1
debug(f"[get-file] Metadata retrieved: title={metadata.get('title')}, ext={metadata.get('ext')}")
@@ -104,7 +111,7 @@ class Get_File(sh.Cmdlet):
return text
return ""
debug(f"[get-file] Calling backend.get_file({file_hash[:12]}...)")
debug(f"[get-file] Calling backend.get_file({file_hash})")
# Get file from backend (may return Path or URL string depending on backend)
source_path = backend.get_file(file_hash)
@@ -135,7 +142,7 @@ class Get_File(sh.Cmdlet):
source_path = Path(source_path)
if not source_path or not source_path.exists():
log(f"Error: Backend could not retrieve file for hash {file_hash[:12]}...")
log(f"Error: Backend could not retrieve file for hash {file_hash}")
return 1
# Folder store UX: without -path, just open the file in the default app.
@@ -202,6 +209,18 @@ class Get_File(sh.Cmdlet):
def _open_file_default(self, path: Path) -> None:
"""Open a local file in the OS default application."""
try:
suffix = str(path.suffix or "").lower()
if sys.platform.startswith("win"):
# On Windows, file associations for common media types can point at
# editors (Paint/VS Code). Prefer opening a localhost URL.
if self._open_local_file_in_browser_via_http(path):
return
if suffix in {".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tif", ".tiff", ".svg"}:
# Use default web browser for images.
if self._open_image_in_default_browser(path):
return
if sys.platform.startswith("win"):
os.startfile(str(path)) # type: ignore[attr-defined]
return
@@ -211,6 +230,122 @@ class Get_File(sh.Cmdlet):
subprocess.Popen(["xdg-open", str(path)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
except Exception as exc:
log(f"Error opening file: {exc}", file=sys.stderr)
def _open_local_file_in_browser_via_http(self, file_path: Path) -> bool:
"""Serve a single local file via localhost HTTP and open in browser.
This avoids Windows file-association issues (e.g., PNG -> Paint, HTML -> VS Code).
The server is bound to 127.0.0.1 on an ephemeral port and is shut down after
a timeout.
"""
try:
resolved = file_path.resolve()
directory = resolved.parent
filename = resolved.name
except Exception:
return False
class OneFileHandler(http.server.SimpleHTTPRequestHandler):
def __init__(self, *handler_args, **handler_kwargs):
super().__init__(*handler_args, directory=str(directory), **handler_kwargs)
def log_message(self, format: str, *args) -> None: # noqa: A003
# Keep normal output clean.
return
def do_GET(self) -> None: # noqa: N802
if self.path in {"/", ""}:
self.path = "/" + filename
return super().do_GET()
if self.path == "/" + filename or self.path == "/" + quote(filename):
return super().do_GET()
self.send_error(404)
def do_HEAD(self) -> None: # noqa: N802
if self.path in {"/", ""}:
self.path = "/" + filename
return super().do_HEAD()
if self.path == "/" + filename or self.path == "/" + quote(filename):
return super().do_HEAD()
self.send_error(404)
try:
httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), OneFileHandler)
except Exception:
return False
port = httpd.server_address[1]
url = f"http://127.0.0.1:{port}/{quote(filename)}"
# Run server in the background.
server_thread = threading.Thread(target=httpd.serve_forever, kwargs={"poll_interval": 0.2}, daemon=True)
server_thread.start()
# Auto-shutdown after a timeout to avoid lingering servers.
def shutdown_later() -> None:
time.sleep(10 * 60)
try:
httpd.shutdown()
except Exception:
pass
try:
httpd.server_close()
except Exception:
pass
threading.Thread(target=shutdown_later, daemon=True).start()
try:
debug(f"[get-file] Opening via localhost: {url}")
return bool(webbrowser.open(url))
except Exception:
return False
def _open_image_in_default_browser(self, image_path: Path) -> bool:
"""Open an image file in the user's default web browser.
We intentionally avoid opening the image path directly on Windows because
file associations may point to editors/viewers (e.g., Paint). Instead we
generate a tiny HTML wrapper and open that (HTML is typically associated
with the default browser).
"""
try:
resolved = image_path.resolve()
image_url = urljoin("file:", pathname2url(str(resolved)))
except Exception:
return False
# Create a stable wrapper filename to reduce temp-file spam.
wrapper_path = Path(tempfile.gettempdir()) / f"medeia-open-image-{resolved.stem}.html"
try:
wrapper_path.write_text(
"\n".join(
[
"<!doctype html>",
"<meta charset=\"utf-8\">",
f"<title>{resolved.name}</title>",
"<style>html,body{margin:0;padding:0;background:#000}img{display:block;max-width:100vw;max-height:100vh;margin:auto}</style>",
f"<img src=\"{image_url}\" alt=\"{resolved.name}\">",
]
),
encoding="utf-8",
)
except Exception:
return False
# Prefer localhost server when possible (reliable on Windows).
if self._open_local_file_in_browser_via_http(image_path):
return True
wrapper_url = wrapper_path.as_uri()
try:
return bool(webbrowser.open(wrapper_url))
except Exception:
return False
def _sanitize_filename(self, name: str) -> str:
"""Sanitize filename by removing invalid characters."""

4
cmdlet/get_relationship.py
View File

@@ -450,7 +450,9 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
table.set_row_selection_args(i, ["-store", str(item['store']), "-query", f"hash:{item['hash']}"])
ctx.set_last_result_table(table, pipeline_results)
print(table)
from rich_display import stdout_console
stdout_console().print(table)
return 0

110
cmdlet/merge_file.py
View File

@@ -112,6 +112,107 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
item = files_to_merge[0]
ctx.emit(item)
return 0
def _resolve_existing_path(item: Dict[str, Any]) -> Optional[Path]:
raw_path = get_pipe_object_path(item)
target_path: Optional[Path] = None
if isinstance(raw_path, Path):
target_path = raw_path
elif isinstance(raw_path, str) and raw_path.strip():
candidate = Path(raw_path).expanduser()
if candidate.exists():
target_path = candidate
if target_path and target_path.exists():
return target_path
return None
def _extract_url(item: Dict[str, Any]) -> Optional[str]:
u = get_field(item, "url") or get_field(item, "target")
if isinstance(u, str):
s = u.strip()
if s.lower().startswith(("http://", "https://")):
return s
return None
# If the user piped URL-only playlist selections (no local paths yet), download first.
# This keeps the pipeline order intuitive:
# @* | merge-file | add-file -store ...
urls_to_download: List[str] = []
for it in files_to_merge:
if _resolve_existing_path(it) is not None:
continue
u = _extract_url(it)
if u:
urls_to_download.append(u)
if urls_to_download and len(urls_to_download) >= 2:
try:
# Compute a batch hint (audio vs video + single-format id) once.
mode_hint: Optional[str] = None
forced_format: Optional[str] = None
try:
from cmdlet.download_media import list_formats
from tool.ytdlp import YtDlpTool
sample_url = urls_to_download[0]
cookiefile = None
try:
cookie_path = YtDlpTool(config).resolve_cookiefile()
if cookie_path is not None and cookie_path.is_file():
cookiefile = str(cookie_path)
except Exception:
cookiefile = None
fmts = list_formats(sample_url, no_playlist=False, playlist_items=None, cookiefile=cookiefile)
if isinstance(fmts, list) and fmts:
has_video = False
for f in fmts:
if not isinstance(f, dict):
continue
vcodec = str(f.get("vcodec", "none") or "none").strip().lower()
if vcodec and vcodec != "none":
has_video = True
break
mode_hint = "video" if has_video else "audio"
if len(fmts) == 1 and isinstance(fmts[0], dict):
fid = str(fmts[0].get("format_id") or "").strip()
if fid:
forced_format = fid
except Exception:
mode_hint = None
forced_format = None
from cmdlet.add_file import Add_File
expanded: List[Dict[str, Any]] = []
downloaded_any = False
for it in files_to_merge:
if _resolve_existing_path(it) is not None:
expanded.append(it)
continue
u = _extract_url(it)
if not u:
expanded.append(it)
continue
downloaded = Add_File._download_streaming_url_as_pipe_objects(
u,
config,
mode_hint=mode_hint,
ytdl_format_hint=forced_format,
)
if downloaded:
expanded.extend(downloaded)
downloaded_any = True
else:
expanded.append(it)
if downloaded_any:
files_to_merge = expanded
except Exception:
# If downloads fail, we fall back to the existing path-based merge behavior.
pass
# Extract file paths and metadata from result objects
source_files: List[Path] = []
@@ -120,14 +221,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
source_tags: List[str] = [] # tags read from .tag sidecars
source_item_tag_lists: List[List[str]] = [] # tags carried in-memory on piped items
for item in files_to_merge:
raw_path = get_pipe_object_path(item)
target_path = None
if isinstance(raw_path, Path):
target_path = raw_path
elif isinstance(raw_path, str) and raw_path.strip():
candidate = Path(raw_path).expanduser()
if candidate.exists():
target_path = candidate
target_path = _resolve_existing_path(item)
if target_path and target_path.exists():
source_files.append(target_path)

72
cmdlet/screen_shot.py
View File

@@ -266,27 +266,27 @@ def _archive_url(url: str, timeout: float) -> Tuple[List[str], List[str]]:
(_submit_archive_ph, "archive.ph"),
):
try:
log(f"Archiving to {label}...", flush=True)
debug(f"Archiving to {label}...")
archived = submitter(url, timeout)
except httpx.HTTPStatusError as exc:
if exc.response.status_code == 429:
warnings.append(f"archive {label} rate limited (HTTP 429)")
log(f"{label}: Rate limited (HTTP 429)", flush=True)
debug(f"{label}: Rate limited (HTTP 429)")
else:
warnings.append(f"archive {label} failed: HTTP {exc.response.status_code}")
log(f"{label}: HTTP {exc.response.status_code}", flush=True)
debug(f"{label}: HTTP {exc.response.status_code}")
except httpx.RequestError as exc:
warnings.append(f"archive {label} failed: {exc}")
log(f"{label}: Connection error: {exc}", flush=True)
debug(f"{label}: Connection error: {exc}")
except Exception as exc:
warnings.append(f"archive {label} failed: {exc}")
log(f"{label}: {exc}", flush=True)
debug(f"{label}: {exc}")
else:
if archived:
archives.append(archived)
log(f"{label}: Success - {archived}", flush=True)
debug(f"{label}: Success - {archived}")
else:
log(f"{label}: No archive link returned", flush=True)
debug(f"{label}: No archive link returned")
return archives, warnings
@@ -335,7 +335,7 @@ def _capture(options: ScreenshotOptions, destination: Path, warnings: List[str])
tool.debug_dump()
log("Launching browser...", flush=True)
debug("Launching browser...")
format_name = _normalise_format(options.output_format)
headless = options.headless or format_name == "pdf"
debug(f"[_capture] Format: {format_name}, Headless: {headless}")
@@ -345,29 +345,29 @@ def _capture(options: ScreenshotOptions, destination: Path, warnings: List[str])
try:
with tool.open_page(headless=headless) as page:
log(f"Navigating to {options.url}...", flush=True)
debug(f"Navigating to {options.url}...")
try:
tool.goto(page, options.url)
log("Page loaded successfully", flush=True)
debug("Page loaded successfully")
except PlaywrightTimeoutError:
warnings.append("navigation timeout; capturing current page state")
log("Navigation timeout; proceeding with current state", flush=True)
debug("Navigation timeout; proceeding with current state")
# Skip article lookup by default (wait_for_article defaults to False)
if options.wait_for_article:
try:
log("Waiting for article element...", flush=True)
debug("Waiting for article element...")
page.wait_for_selector("article", timeout=10_000)
log("Article element found", flush=True)
debug("Article element found")
except PlaywrightTimeoutError:
warnings.append("<article> selector not found; capturing fallback")
log("Article element not found; using fallback", flush=True)
debug("Article element not found; using fallback")
if options.wait_after_load > 0:
log(f"Waiting {options.wait_after_load}s for page stabilization...", flush=True)
debug(f"Waiting {options.wait_after_load}s for page stabilization...")
time.sleep(min(10.0, max(0.0, options.wait_after_load)))
if options.replace_video_posters:
log("Replacing video elements with posters...", flush=True)
debug("Replacing video elements with posters...")
page.evaluate(
"""
document.querySelectorAll('video').forEach(v => {
@@ -384,7 +384,7 @@ def _capture(options: ScreenshotOptions, destination: Path, warnings: List[str])
# Attempt platform-specific target capture if requested (and not PDF)
element_captured = False
if options.prefer_platform_target and format_name != "pdf":
log("Attempting platform-specific content capture...", flush=True)
debug("Attempting platform-specific content capture...")
try:
_platform_preprocess(options.url, page, warnings)
except Exception as e:
@@ -397,36 +397,36 @@ def _capture(options: ScreenshotOptions, destination: Path, warnings: List[str])
debug(f"[_capture] Trying selectors: {selectors}")
for sel in selectors:
try:
log(f"Trying selector: {sel}", flush=True)
debug(f"Trying selector: {sel}")
el = page.wait_for_selector(sel, timeout=max(0, int(options.selector_timeout_ms)))
except PlaywrightTimeoutError:
log(f"Selector not found: {sel}", flush=True)
debug(f"Selector not found: {sel}")
continue
try:
if el is not None:
log(f"Found element with selector: {sel}", flush=True)
debug(f"Found element with selector: {sel}")
try:
el.scroll_into_view_if_needed(timeout=1000)
except Exception:
pass
log(f"Capturing element to {destination}...", flush=True)
debug(f"Capturing element to {destination}...")
el.screenshot(path=str(destination), type=("jpeg" if format_name == "jpeg" else None))
element_captured = True
log("Element captured successfully", flush=True)
debug("Element captured successfully")
break
except Exception as exc:
warnings.append(f"element capture failed for '{sel}': {exc}")
log(f"Failed to capture element: {exc}", flush=True)
debug(f"Failed to capture element: {exc}")
# Fallback to default capture paths
if element_captured:
pass
elif format_name == "pdf":
log("Generating PDF...", flush=True)
debug("Generating PDF...")
page.emulate_media(media="print")
page.pdf(path=str(destination), print_background=True)
log(f"PDF saved to {destination}", flush=True)
debug(f"PDF saved to {destination}")
else:
log(f"Capturing full page to {destination}...", flush=True)
debug(f"Capturing full page to {destination}...")
screenshot_kwargs: Dict[str, Any] = {"path": str(destination)}
if format_name == "jpeg":
screenshot_kwargs["type"] = "jpeg"
@@ -441,7 +441,7 @@ def _capture(options: ScreenshotOptions, destination: Path, warnings: List[str])
article.screenshot(**article_kwargs)
else:
page.screenshot(**screenshot_kwargs)
log(f"Screenshot saved to {destination}", flush=True)
debug(f"Screenshot saved to {destination}")
except Exception as exc:
debug(f"[_capture] Exception launching browser/page: {exc}")
msg = str(exc).lower()
@@ -587,7 +587,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
if storage_value:
try:
screenshot_dir = SharedArgs.resolve_storage(storage_value)
log(f"[screen_shot] Using --storage {storage_value}: {screenshot_dir}", flush=True)
debug(f"[screen_shot] Using --storage {storage_value}: {screenshot_dir}")
except ValueError as e:
log(str(e), file=sys.stderr)
return 1
@@ -596,7 +596,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
if screenshot_dir is None and resolve_output_dir is not None:
try:
screenshot_dir = resolve_output_dir(config)
log(f"[screen_shot] Using config resolver: {screenshot_dir}", flush=True)
debug(f"[screen_shot] Using config resolver: {screenshot_dir}")
except Exception:
pass
@@ -604,14 +604,14 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
if screenshot_dir is None and config and config.get("outfile"):
try:
screenshot_dir = Path(config["outfile"]).expanduser()
log(f"[screen_shot] Using config outfile: {screenshot_dir}", flush=True)
debug(f"[screen_shot] Using config outfile: {screenshot_dir}")
except Exception:
pass
# Default: User's Videos directory
if screenshot_dir is None:
screenshot_dir = Path.home() / "Videos"
log(f"[screen_shot] Using default directory: {screenshot_dir}", flush=True)
debug(f"[screen_shot] Using default directory: {screenshot_dir}")
ensure_directory(screenshot_dir)
@@ -693,11 +693,11 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
screenshot_result = _capture_screenshot(options)
# Log results and warnings
log(f"Screenshot captured to {screenshot_result.path}", flush=True)
debug(f"Screenshot captured to {screenshot_result.path}")
if screenshot_result.archive_url:
log(f"Archives: {', '.join(screenshot_result.archive_url)}", flush=True)
debug(f"Archives: {', '.join(screenshot_result.archive_url)}")
for warning in screenshot_result.warnings:
log(f"Warning: {warning}", flush=True)
debug(f"Warning: {warning}")
# Compute hash of screenshot file
screenshot_hash = None
@@ -762,8 +762,8 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
log(f"No screenshots were successfully captured", file=sys.stderr)
return 1
# Log completion message
log(f"✓ Successfully captured {len(all_emitted)} screenshot(s)", flush=True)
# Log completion message (keep this as normal output)
log(f"✓ Successfully captured {len(all_emitted)} screenshot(s)")
return exit_code
CMDLET = Cmdlet(

89
cmdlet/search_store.py
View File

@@ -45,6 +45,8 @@ class Search_Store(Cmdlet):
"Search across storage backends: Folder stores and Hydrus instances",
"Use -store to search a specific backend by name",
"URL search: url:* (any URL) or url:<value> (URL substring)",
"Extension search: ext:<value> (e.g., ext:png)",
"Hydrus-style extension: system:filetype = png",
"Results include hash for downstream commands (get-file, add-tag, etc.)",
"Examples:",
"search-store -query foo # Search all storage backends",
@@ -53,6 +55,8 @@ class Search_Store(Cmdlet):
"search-store -query 'hash:deadbeef...' # Search by SHA256 hash",
"search-store -query 'url:*' # Files that have any URL",
"search-store -query 'url:youtube.com' # Files whose URL contains substring",
"search-store -query 'ext:png' # Files whose metadata ext is png",
"search-store -query 'system:filetype = png' # Hydrus: native; Folder: maps to metadata.ext",
],
exec=self.run,
)
@@ -107,6 +111,35 @@ class Search_Store(Cmdlet):
args_list = [str(arg) for arg in (args or [])]
refresh_mode = any(str(a).strip().lower() in {"--refresh", "-refresh"} for a in args_list)
def _format_command_title(command: str, raw_args: List[str]) -> str:
def _quote(value: str) -> str:
text = str(value)
if not text:
return '""'
needs_quotes = any(ch.isspace() for ch in text) or '"' in text
if not needs_quotes:
return text
return '"' + text.replace('"', '\\"') + '"'
cleaned = [
str(a)
for a in (raw_args or [])
if str(a).strip().lower() not in {"--refresh", "-refresh"}
]
if not cleaned:
return command
return " ".join([command, *[_quote(a) for a in cleaned]])
raw_title = None
try:
raw_title = ctx.get_current_stage_text("") if hasattr(ctx, "get_current_stage_text") else None
except Exception:
raw_title = None
command_title = (str(raw_title).strip() if raw_title else "") or _format_command_title("search-store", list(args_list))
# Build dynamic flag variants from cmdlet arg definitions.
# This avoids hardcoding flag spellings in parsing loops.
flag_registry = self.build_flag_registry()
@@ -188,11 +221,7 @@ class Search_Store(Cmdlet):
importlib.reload(result_table)
from result_table import ResultTable
table_title = f"Search: {query}"
if storage_backend:
table_title += f" [{storage_backend}]"
table = ResultTable(table_title)
table = ResultTable(command_title)
try:
table.set_source_command("search-store", list(args_list))
except Exception:
@@ -326,26 +355,23 @@ class Search_Store(Cmdlet):
ctx.emit(payload)
if found_any:
# Title should reflect the command, query, and only stores present in the table.
store_counts: "OrderedDict[str, int]" = OrderedDict()
for row_item in results_list:
store_val = str(row_item.get("store") or "").strip()
if not store_val:
continue
if store_val not in store_counts:
store_counts[store_val] = 0
store_counts[store_val] += 1
table.title = command_title
counts_part = " ".join(f"{name}:{count}" for name, count in store_counts.items() if count > 0)
base_title = f"search-store: {query}".strip()
table.title = f"{base_title} | {counts_part}" if counts_part else base_title
ctx.set_last_result_table(table, results_list)
if refresh_mode:
ctx.set_last_result_table_preserve_history(table, results_list)
else:
ctx.set_last_result_table(table, results_list)
db.append_worker_stdout(worker_id, json.dumps(results_list, indent=2))
db.update_worker_status(worker_id, 'completed')
return 0
log("No results found", file=sys.stderr)
if refresh_mode:
try:
table.title = command_title
ctx.set_last_result_table_preserve_history(table, [])
except Exception:
pass
db.append_worker_stdout(worker_id, json.dumps([], indent=2))
db.update_worker_status(worker_id, 'completed')
return 0
@@ -413,24 +439,21 @@ class Search_Store(Cmdlet):
results_list.append(normalized)
ctx.emit(normalized)
# Title should reflect the command, query, and only stores present in the table.
store_counts: "OrderedDict[str, int]" = OrderedDict()
for row_item in results_list:
store_val = str(row_item.get("store") or "").strip()
if not store_val:
continue
if store_val not in store_counts:
store_counts[store_val] = 0
store_counts[store_val] += 1
table.title = command_title
counts_part = " ".join(f"{name}:{count}" for name, count in store_counts.items() if count > 0)
base_title = f"search-store: {query}".strip()
table.title = f"{base_title} | {counts_part}" if counts_part else base_title
ctx.set_last_result_table(table, results_list)
if refresh_mode:
ctx.set_last_result_table_preserve_history(table, results_list)
else:
ctx.set_last_result_table(table, results_list)
db.append_worker_stdout(worker_id, json.dumps(results_list, indent=2))
else:
log("No results found", file=sys.stderr)
if refresh_mode:
try:
table.title = command_title
ctx.set_last_result_table_preserve_history(table, [])
except Exception:
pass
db.append_worker_stdout(worker_id, json.dumps([], indent=2))
db.update_worker_status(worker_id, 'completed')