goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

update refactoring and add new features

Browse Source
This commit is contained in:
goyimnose
2026-07-24 20:55:58 -07:00
parent 7f12bc7f40
commit 03fbbbcf28
69 changed files with 16332 additions and 17600 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmdlet/file/add.py
+56 -3263
View File
File diff suppressed because it is too large Load Diff
cmdlet/file/add_core.py
+1000
View File
File diff suppressed because it is too large Load Diff
cmdlet/file/add_storage.py
+1056
View File
File diff suppressed because it is too large Load Diff
cmdlet/file/add_tagging.py
+562
View File
@@ -0,0 +1,562 @@
"""Tag resolution, FlorenceVision integration, and metadata extraction for add-file."""
from __future__ import annotations
from typing import Any, Dict, List, Optional, Sequence, Tuple
from pathlib import Path
import sys
import re
from SYS import models
from SYS.logger import log, debug
from .. import _shared as sh
# Import from add_core (safe: add_core defines these before importing this module)
from .add_core import (
Add_File,
_SCREENSHOT_TIME_SUFFIX_RE,
_REMOTE_URL_PREFIXES,
)
extract_tag_from_result = sh.extract_tag_from_result
extract_title_from_result = sh.extract_title_from_result
extract_url_from_result = sh.extract_url_from_result
merge_sequences = sh.merge_sequences
extract_relationships = sh.extract_relationships
extract_duration = sh.extract_duration
collapse_namespace_tags = sh.collapse_namespace_tags
resolve_media_kind_by_extension = sh.resolve_media_kind_by_extension
get_field = sh.get_field
def _maybe_apply_florencevision_tags(
media_path: Path,
tags: List[str],
config: Dict[str, Any],
pipe_obj: Optional[models.PipeObject] = None,
) -> List[str]:
"""Optionally auto-tag images using the FlorenceVision plugin helper.
Controlled via config:
[plugin=florencevision]
enabled=true
strict=false
If strict=false (default), failures log a warning and return the original tags.
If strict=true, failures raise to abort the ingest.
"""
strict = False
try:
plugin_block = (config or {}).get("plugin")
fv_block = plugin_block.get("florencevision") if isinstance(plugin_block, dict) else None
enabled = False
if isinstance(fv_block, dict):
enabled = bool(fv_block.get("enabled"))
strict = bool(fv_block.get("strict"))
if not enabled:
return tags
from plugins.florencevision import FlorenceVisionTool
cfg_for_tool: Dict[str, Any] = config
try:
action = str(getattr(pipe_obj, "action", "") or "") if pipe_obj is not None else ""
cmdlet_name = ""
if action.lower().startswith("cmdlet:"):
cmdlet_name = action.split(":", 1)[1].strip().lower()
if cmdlet_name in {"screen-shot", "screen_shot", "screenshot"}:
plugin_block2 = dict((config or {}).get("plugin") or {})
fv_block2 = dict(plugin_block2.get("florencevision") or {})
fv_block2["task"] = "ocr"
plugin_block2["florencevision"] = fv_block2
cfg_for_tool = dict(config or {})
cfg_for_tool["plugin"] = plugin_block2
except Exception:
cfg_for_tool = config
fv = FlorenceVisionTool(cfg_for_tool)
if not fv.enabled() or not fv.applicable_path(media_path):
return tags
auto_tags = fv.tags_for_file(media_path)
try:
caption_text = getattr(fv, "last_caption", None)
if caption_text and pipe_obj is not None:
if not isinstance(pipe_obj.extra, dict):
pipe_obj.extra = {}
notes = pipe_obj.extra.get("notes")
if not isinstance(notes, dict):
notes = {}
notes.setdefault("caption", caption_text)
pipe_obj.extra["notes"] = notes
except Exception:
pass
if not auto_tags:
return tags
merged = merge_sequences(tags or [], auto_tags, case_sensitive=False)
debug(f"[add-file] FlorenceVision added {len(auto_tags)} tag(s)")
return merged
except Exception as exc:
strict2 = False
try:
tool_block = (config or {}).get("tool")
fv_block = tool_block.get("florencevision") if isinstance(tool_block, dict) else None
strict2 = bool(fv_block.get("strict")) if isinstance(fv_block, dict) else False
except Exception:
strict2 = False
if strict or strict2:
raise
log(f"[add-file] Warning: FlorenceVision tagging failed: {exc}", file=sys.stderr)
return tags
def _normalize_hash_candidate(value: Any) -> str:
text = str(value or "").strip().lower()
if len(text) != 64:
return ""
if any(ch not in "0123456789abcdef" for ch in text):
return ""
return text
def _parse_relationship_tag_king_alts(
tag_value: str
) -> tuple[Optional[str], List[str]]:
"""Parse a relationship tag into (king_hash, alt_hashes).
Supported formats:
- New: relationship: <KING_HASH>,<ALT_HASH>,<ALT_HASH>
- Old: relationship: hash(king)<KING_HASH>,hash(alt)<ALT_HASH>...
relationship: hash(king)KING,hash(alt)ALT
For the local DB we treat the first hash listed as the king.
"""
if not isinstance(tag_value, str):
return None, []
raw = tag_value.strip()
if not raw:
return None, []
rhs = raw
if ":" in raw:
prefix, rest = raw.split(":", 1)
if prefix.strip().lower() == "relationship":
rhs = rest.strip()
typed = re.findall(r"hash\((\w+)\)<?([a-fA-F0-9]{64})>?", rhs)
if typed:
king: Optional[str] = None
alts: List[str] = []
for rel_type, h in typed:
h_norm = str(h).strip().lower()
if rel_type.strip().lower() == "king":
king = h_norm
elif rel_type.strip().lower() in {"alt", "related"}:
alts.append(h_norm)
if not king:
all_hashes = [str(h).strip().lower() for _, h in typed]
king = all_hashes[0] if all_hashes else None
alts = [h for h in all_hashes[1:] if h]
seen: set[str] = set()
alts = [
h for h in alts
if h and len(h) == 64 and not (h in seen or seen.add(h))
]
if king and len(king) == 64:
return king, [h for h in alts if h != king]
return None, []
hashes = re.findall(r"\b[a-fA-F0-9]{64}\b", rhs)
hashes = [h.strip().lower() for h in hashes if isinstance(h, str)]
if not hashes:
return None, []
king = hashes[0]
alts = hashes[1:]
seen2: set[str] = set()
alts = [
h for h in alts if h and len(h) == 64 and not (h in seen2 or seen2.add(h))
]
return king, [h for h in alts if h != king]
def _parse_relationships_king_alts(
relationships: Dict[str, Any],
) -> tuple[Optional[str], List[str]]:
"""Parse a PipeObject.relationships dict into (king_hash, alt_hashes).
Supported shapes:
- {"king": [KING], "alt": [ALT1, ALT2]}
- {"king": KING, "alt": ALT} (strings)
- Also treats "related" hashes as alts for persistence purposes.
"""
if not isinstance(relationships, dict) or not relationships:
return None, []
def _first_hash(val: Any) -> Optional[str]:
if isinstance(val, str):
h = val.strip().lower()
return h if len(h) == 64 else None
if isinstance(val, list):
for item in val:
if isinstance(item, str):
h = item.strip().lower()
if len(h) == 64:
return h
return None
def _many_hashes(val: Any) -> List[str]:
out: List[str] = []
if isinstance(val, str):
h = val.strip().lower()
if len(h) == 64:
out.append(h)
elif isinstance(val, list):
for item in val:
if isinstance(item, str):
h = item.strip().lower()
if len(h) == 64:
out.append(h)
return out
king = _first_hash(relationships.get("king"))
if not king:
return None, []
alts = _many_hashes(relationships.get("alt"))
alts.extend(_many_hashes(relationships.get("related")))
seen: set[str] = set()
alts = [h for h in alts if h and h != king and not (h in seen or seen.add(h))]
return king, alts
def _get_url(result: Any, pipe_obj: models.PipeObject) -> List[str]:
"""Extract valid URLs from pipe object or result dict."""
from SYS.metadata import normalize_urls
candidates: List[str] = []
if pipe_obj.url:
candidates.append(pipe_obj.url)
if pipe_obj.source_url:
candidates.append(pipe_obj.source_url)
if isinstance(pipe_obj.extra, dict):
u = pipe_obj.extra.get("url")
if isinstance(u, list):
candidates.extend(str(x) for x in u if x)
elif isinstance(u, str):
candidates.append(u)
raw_from_result = extract_url_from_result(result)
if raw_from_result:
candidates.extend(raw_from_result)
normalized = normalize_urls(candidates)
return [u for u in normalized if Add_File._is_probable_url(u)]
def _get_relationships(result: Any, pipe_obj: models.PipeObject) -> Optional[Dict[str, Any]]:
try:
rels = pipe_obj.get_relationships()
if rels:
return rels
except Exception:
pass
if isinstance(result, dict) and result.get("relationships"):
return result.get("relationships")
try:
return extract_relationships(result)
except Exception:
return None
def _get_duration(result: Any, pipe_obj: models.PipeObject) -> Optional[float]:
def _parse_duration(value: Any) -> Optional[float]:
if value is None:
return None
if isinstance(value, (int, float)):
return float(value) if value > 0 else None
if isinstance(value, str):
s = value.strip()
if not s:
return None
try:
candidate = float(s)
return candidate if candidate > 0 else None
except ValueError:
pass
if ":" in s:
parts = [p.strip() for p in s.split(":") if p.strip()]
if len(parts) in {2, 3} and all(p.isdigit() for p in parts):
nums = [int(p) for p in parts]
if len(nums) == 2:
minutes, seconds = nums
return float(minutes * 60 + seconds)
hours, minutes, seconds = nums
return float(hours * 3600 + minutes * 60 + seconds)
return None
parsed = _parse_duration(getattr(pipe_obj, "duration", None))
if parsed is not None:
return parsed
try:
return _parse_duration(extract_duration(result))
except Exception:
return None
def _get_note_text(result: Any, pipe_obj: models.PipeObject, note_name: str) -> Optional[str]:
"""Extract a named note text from a piped item.
Supports:
- pipe_obj.extra["notes"][note_name]
- result["notes"][note_name] for dict results
- pipe_obj.extra[note_name] / result[note_name] as fallback
"""
def _normalize(val: Any) -> Optional[str]:
if val is None:
return None
if isinstance(val, bytes):
try:
val = val.decode("utf-8", errors="ignore")
except Exception:
val = str(val)
if isinstance(val, str):
text = val.strip()
return text if text else None
try:
text = str(val).strip()
return text if text else None
except Exception:
return None
note_key = str(note_name or "").strip()
if not note_key:
return None
try:
if isinstance(pipe_obj.extra, dict):
notes_val = pipe_obj.extra.get("notes")
if isinstance(notes_val, dict) and note_key in notes_val:
return _normalize(notes_val.get(note_key))
if note_key in pipe_obj.extra:
return _normalize(pipe_obj.extra.get(note_key))
except Exception:
pass
if isinstance(result, dict):
try:
notes_val = result.get("notes")
if isinstance(notes_val, dict) and note_key in notes_val:
return _normalize(notes_val.get(note_key))
if note_key in result:
return _normalize(result.get(note_key))
except Exception:
pass
return None
def _load_sidecar_bundle(
media_path: Path,
instance: Optional[str],
config: Dict[str, Any],
) -> Tuple[Optional[Path], Optional[str], List[str], List[str]]:
"""Load sidecar metadata (placeholder — overridden by active plugins)."""
return None, None, [], []
def _resolve_file_hash(
result: Any,
media_path: Path,
pipe_obj: models.PipeObject,
fallback_hash: Optional[str],
) -> Optional[str]:
from SYS.utils import sha256_file
if pipe_obj.hash and pipe_obj.hash != "unknown":
return pipe_obj.hash
if fallback_hash:
return fallback_hash
if isinstance(result, dict):
candidate = result.get("hash")
if candidate:
return str(candidate)
try:
return sha256_file(media_path)
except Exception:
return None
def _resolve_media_kind(path: Path) -> str:
return resolve_media_kind_by_extension(path)
def _prepare_metadata(
result: Any,
media_path: Path,
pipe_obj: models.PipeObject,
config: Dict[str, Any],
) -> Tuple[List[str], List[str], Optional[str], Optional[str]]:
"""
Prepare tags, url, and title for the file.
Returns (tags, url, preferred_title, file_hash)
"""
tags_from_result = list(pipe_obj.tag or [])
if not tags_from_result:
try:
tags_from_result = list(extract_tag_from_result(result) or [])
except Exception:
tags_from_result = []
url_from_result = Add_File._get_url(result, pipe_obj)
def _has_namespace_tag(tags: Sequence[str], namespace: str) -> bool:
namespace_text = str(namespace or "").strip().lower()
if not namespace_text:
return False
prefix = f"{namespace_text}:"
for tag in tags or []:
text = str(tag or "").strip().lower()
if text.startswith(prefix):
return True
return False
def _extract_screenshot_time_title() -> tuple[Optional[str], Optional[str]]:
current_title = str(preferred_title or "").strip()
filename_title = str(media_path.stem or "").strip()
if current_title and current_title != filename_title:
return None, None
if not url_from_result:
return None, None
suffix = str(media_path.suffix or "").strip().lower()
if suffix not in {".png", ".jpg", ".jpeg", ".webp", ".gif", ".bmp", ".avif", ".mhtml"}:
return None, None
match = _SCREENSHOT_TIME_SUFFIX_RE.match(str(media_path.stem or "").strip())
if not match:
return None, None
title_text = str(match.group("title") or "").strip().replace("_", " ").strip()
label_text = str(match.group("label") or "").strip().lower()
if not title_text or not label_text:
return None, None
return title_text, label_text
preferred_title = pipe_obj.title
if not preferred_title:
for t in tags_from_result:
if str(t).strip().lower().startswith("title:"):
candidate = t.split(":", 1)[1].strip().replace("_", " ").strip()
if candidate:
preferred_title = candidate
break
if not preferred_title:
preferred_title = extract_title_from_result(result)
if preferred_title:
preferred_title = preferred_title.replace("_", " ").strip()
derived_screenshot_title, derived_time_tag = _extract_screenshot_time_title()
if derived_screenshot_title and (
not preferred_title or str(preferred_title or "").strip() == str(media_path.stem or "").strip()
):
preferred_title = derived_screenshot_title
store = getattr(pipe_obj, "store", None)
_, sidecar_hash, sidecar_tags, sidecar_url = Add_File._load_sidecar_bundle(
media_path, store, config
)
def normalize_title_tag(tag: str) -> str:
if str(tag).strip().lower().startswith("title:"):
parts = tag.split(":", 1)
if len(parts) == 2:
value = parts[1].replace("_", " ").strip()
return f"title:{value}"
return tag
tags_from_result_no_title = [
t for t in tags_from_result
if not str(t).strip().lower().startswith("title:")
]
sidecar_tags = collapse_namespace_tags(
[normalize_title_tag(t) for t in sidecar_tags],
"title",
prefer="last"
)
sidecar_tags_filtered = [
t for t in sidecar_tags if not str(t).strip().lower().startswith("title:")
]
merged_tags = merge_sequences(
tags_from_result_no_title,
sidecar_tags_filtered,
case_sensitive=True
)
if derived_time_tag and not _has_namespace_tag(merged_tags, "time") and not _has_namespace_tag(merged_tags, "timestamp"):
merged_tags.append(f"time:{derived_time_tag}")
if preferred_title:
merged_tags.append(f"title:{preferred_title}")
merged_url = merge_sequences(url_from_result, sidecar_url, case_sensitive=False)
merged_url = [u for u in merged_url if Add_File._is_probable_url(u)]
file_hash = Add_File._resolve_file_hash(
result,
media_path,
pipe_obj,
sidecar_hash
)
relationship_tags = [
t for t in merged_tags
if isinstance(t, str) and t.strip().lower().startswith("relationship:")
]
if relationship_tags:
try:
if (not isinstance(getattr(pipe_obj, "relationships", None), dict) or not pipe_obj.relationships):
king: Optional[str] = None
alts: List[str] = []
for rel_tag in relationship_tags:
k, a = _parse_relationship_tag_king_alts(rel_tag)
if k and not king:
king = k
if a:
alts.extend(a)
if king:
seen_alt: set[str] = set()
alts = [
h for h in alts if h and h != king and len(h) == 64
and not (h in seen_alt or seen_alt.add(h))
]
payload: Dict[str, Any] = {"king": [king]}
if alts:
payload["alt"] = alts
pipe_obj.relationships = payload
except Exception:
pass
merged_tags = [
t for t in merged_tags if
not (isinstance(t, str) and t.strip().lower().startswith("relationship:"))
]
pipe_obj.tag = merged_tags
if preferred_title and not pipe_obj.title:
pipe_obj.title = preferred_title
if file_hash and not pipe_obj.hash:
pipe_obj.hash = file_hash
if isinstance(pipe_obj.extra, dict):
pipe_obj.extra["url"] = merged_url
return merged_tags, merged_url, preferred_title, file_hash
cmdlet/file/add_validation.py
+611
View File
@@ -0,0 +1,611 @@
"""File validation, path resolution, and source fetching for add-file."""
from __future__ import annotations
from typing import Any, Dict, Optional, List
from pathlib import Path
import sys
import tempfile
import shutil
from urllib.parse import urlparse
from SYS import models
from SYS import pipeline as ctx
from SYS.logger import log, debug
from SYS.pipeline_progress import PipelineProgress
from SYS.utils import sha256_file, sanitize_filename
from API.HTTP import download_direct_file
from .. import _shared as sh
# Import from add_core (safe: add_core defines these before importing this module)
from .add_core import (
Add_File,
_CommandDependencies,
SUPPORTED_MEDIA_EXTENSIONS,
_REMOTE_URL_PREFIXES,
)
coerce_to_path = sh.coerce_to_path
get_field = sh.get_field
def _resolve_backend_by_name(instance: Any, backend_name: str) -> Optional[Any]:
if not instance or not backend_name:
return None
try:
return instance[backend_name]
except Exception:
pass
target = str(backend_name or "").strip().lower()
if not target:
return None
try:
for candidate in instance.list_backends():
if isinstance(candidate, str) and candidate.strip().lower() == target:
try:
return instance[candidate]
except Exception:
continue
except Exception:
pass
return None
def _build_provider_filename(
pipe_obj: models.PipeObject,
fallback_hash: Optional[str] = None,
source_url: Optional[str] = None,
) -> str:
title_candidates: List[str] = []
title_value = getattr(pipe_obj, "title", "")
if title_value:
title_candidates.append(str(title_value))
extra = getattr(pipe_obj, "extra", {})
if isinstance(extra, dict):
candid = extra.get("name") or extra.get("title")
if candid:
title_candidates.append(str(candid))
metadata = getattr(pipe_obj, "metadata", {})
if isinstance(metadata, dict):
meta_name = metadata.get("title") or metadata.get("name")
if meta_name:
title_candidates.append(str(meta_name))
text = ""
for candidate in title_candidates:
if candidate:
text = candidate.strip()
if text:
break
if not text and fallback_hash:
text = fallback_hash[:8]
safe_name = sanitize_filename(text or "download")
ext = ""
if isinstance(metadata, dict):
ext = metadata.get("ext") or metadata.get("extension") or ""
if not ext and isinstance(extra, dict):
ext = extra.get("ext") or ""
if not ext and source_url:
try:
parsed = urlparse(source_url)
ext = Path(parsed.path).suffix.lstrip(".")
except Exception:
ext = ""
if ext:
ext_text = str(ext)
if not ext_text.startswith("."):
ext_text = "." + ext_text.lstrip(".")
if not safe_name.lower().endswith(ext_text.lower()):
safe_name = f"{safe_name}{ext_text}"
return safe_name or "download"
def _maybe_download_backend_file(
backend: Any,
file_hash: str,
pipe_obj: models.PipeObject,
*,
output_dir: Optional[Path] = None,
) -> tuple[Optional[Path], Optional[Path]]:
"""Best-effort fetch of a backend file when get_file returns a URL.
Returns (downloaded_path, temp_dir_to_cleanup).
"""
downloader = getattr(backend, "download_to_temp", None)
if not callable(downloader):
return None, None
tmp_dir: Optional[Path] = None
try:
suffix = None
if pipe_obj.path:
try:
suffix = Path(pipe_obj.path).suffix
except Exception:
pass
if not suffix:
metadata = getattr(pipe_obj, "metadata", {})
if isinstance(metadata, dict):
suffix = metadata.get("ext")
download_root = output_dir
if download_root is None:
tmp_dir = Path(tempfile.mkdtemp(prefix="add-file-src-"))
download_root = tmp_dir
if download_root is None:
return None, None
import inspect
sig = inspect.signature(downloader)
kwargs = {"temp_root": download_root}
if "suffix" in sig.parameters:
kwargs["suffix"] = suffix
pipeline_progress = PipelineProgress(ctx)
transfer_label = "peer transfer"
try:
transfer_label = str(getattr(pipe_obj, "title", "") or "").strip() or transfer_label
except Exception:
transfer_label = "peer transfer"
if "pipeline_progress" in sig.parameters:
kwargs["pipeline_progress"] = pipeline_progress
if "transfer_label" in sig.parameters:
kwargs["transfer_label"] = transfer_label
if "progress_callback" in sig.parameters:
def _cb(done, total):
try:
total_val = int(total) if total is not None else None
except Exception:
total_val = None
try:
if int(done or 0) <= 0:
pipeline_progress.begin_transfer(
label=transfer_label,
total=total_val,
)
except Exception:
pass
try:
pipeline_progress.update_transfer(
label=transfer_label,
completed=int(done or 0),
total=total_val,
)
except Exception:
pass
kwargs["progress_callback"] = _cb
downloaded = downloader(str(file_hash), **kwargs)
if isinstance(downloaded, Path) and downloaded.exists():
if output_dir is not None:
pipe_obj.is_temp = False
if isinstance(pipe_obj.extra, dict):
pipe_obj.extra["_direct_export_download"] = True
else:
pipe_obj.extra = {"_direct_export_download": True}
return downloaded, None
pipe_obj.is_temp = True
return downloaded, tmp_dir
except Exception:
pass
if tmp_dir is not None:
try:
shutil.rmtree(tmp_dir, ignore_errors=True)
except Exception:
pass
return None, None
def _download_remote_backend_url(
remote_url: str,
pipe_obj: models.PipeObject,
*,
file_hash: Optional[str] = None,
output_dir: Optional[Path] = None,
) -> tuple[Optional[Path], Optional[Path]]:
"""Best-effort fetch of a remote backend URL.
Returns (downloaded_path, temp_dir_to_cleanup).
When ``output_dir`` is provided, the file is downloaded directly there and no
temp cleanup path is returned.
"""
url_text = str(remote_url or "").strip()
if not url_text:
return None, None
if not url_text.lower().startswith(_REMOTE_URL_PREFIXES):
return None, None
if not url_text.lower().startswith(("http://", "https://")):
return None, None
tmp_dir: Optional[Path] = None
try:
download_root = output_dir
if download_root is None:
tmp_dir = Path(tempfile.mkdtemp(prefix="add-file-src-"))
download_root = tmp_dir
suggested_name = _build_provider_filename(
pipe_obj,
fallback_hash=file_hash,
source_url=url_text,
)
pipeline_progress = PipelineProgress(ctx)
try:
destination_label = str(download_root) if download_root is not None else "temporary workspace"
pipeline_progress.set_status(f"downloading {suggested_name} to {destination_label}")
except Exception:
pass
downloaded = download_direct_file(
url_text,
download_root,
quiet=False,
suggested_filename=suggested_name,
pipeline_progress=pipeline_progress,
)
downloaded_path = getattr(downloaded, "path", None)
if isinstance(downloaded_path, Path) and downloaded_path.exists():
if output_dir is not None:
pipe_obj.is_temp = False
if isinstance(pipe_obj.extra, dict):
pipe_obj.extra["_direct_export_download"] = True
else:
pipe_obj.extra = {"_direct_export_download": True}
return downloaded_path, None
pipe_obj.is_temp = True
return downloaded_path, tmp_dir
except Exception:
pass
finally:
try:
PipelineProgress(ctx).clear_status()
except Exception:
pass
if tmp_dir is not None:
try:
shutil.rmtree(tmp_dir, ignore_errors=True)
except Exception:
pass
return None, None
def _maybe_download_plugin_result(
result: Any,
pipe_obj: models.PipeObject,
config: Dict[str, Any],
deps: Optional[_CommandDependencies] = None,
) -> tuple[Optional[Path], Optional[str], Optional[Path]]:
plugin_key = None
for source in (
pipe_obj.plugin,
get_field(result, "plugin"),
get_field(result, "table"),
):
candidate = Add_File._normalize_provider_key(source)
if candidate:
plugin_key = candidate
break
if not plugin_key:
return None, None, None
if deps is None:
deps = _CommandDependencies(config)
plugin = deps.get_plugin(plugin_key)
if plugin is None:
return None, None, None
try:
return plugin.resolve_pipe_result_download(result, pipe_obj)
except Exception as exc:
debug(f"[add-file] Plugin '{plugin_key}' download helper failed: {exc}")
return None, None, None
def _download_piped_source(
pipe_obj: models.PipeObject,
config: Dict[str, Any],
store_instance: Optional[Any],
deps: Optional[_CommandDependencies] = None,
) -> tuple[Optional[Path], Optional[str], Optional[Path]]:
r_hash = str(getattr(pipe_obj, "hash", None) or getattr(pipe_obj, "file_hash", None) or "").strip()
r_store = str(getattr(pipe_obj, "store", None) or "").strip()
if not (r_hash and r_store):
return None, None, None
if deps is None:
deps = _CommandDependencies(config)
backend_registry = store_instance or deps.get_backend_registry()
backend = _resolve_backend_by_name(backend_registry, r_store) if backend_registry is not None else None
if backend is None:
return None, None, None
try:
source = backend.get_file(r_hash.lower())
if isinstance(source, Path) and source.exists():
pipe_obj.path = str(source)
return source, str(r_hash), None
if isinstance(source, str) and source.strip():
dl_path, tmp_dir = _maybe_download_backend_file(
backend, str(r_hash), pipe_obj
)
if dl_path and dl_path.exists():
return dl_path, str(r_hash), tmp_dir
source_url = str(source).strip()
if source_url.lower().startswith(("http://", "https://")):
download_dir = Path(tempfile.mkdtemp(prefix="add-file-src-"))
try:
filename = _build_provider_filename(
pipe_obj,
str(r_hash),
source_url,
)
downloaded = download_direct_file(
source_url,
download_dir,
quiet=True,
suggested_filename=filename,
)
downloaded_path = downloaded.path
if downloaded_path and downloaded_path.exists():
pipe_obj.is_temp = True
pipe_obj.path = str(downloaded_path)
return downloaded_path, str(r_hash), download_dir
except Exception as exc:
debug(f"[add-file] Provider download failed: {exc}")
try:
shutil.rmtree(download_dir, ignore_errors=True)
except Exception:
pass
except Exception:
pass
return None, None, None
def _scan_directory_for_files(directory: Path, compute_hash: bool = True) -> List[Dict[str, Any]]:
"""Scan a directory for supported media files and return list of file info dicts.
Each dict contains:
- path: Path object
- name: filename
- hash: sha256 hash (or None if compute_hash=False)
- size: file size in bytes
- ext: file extension
"""
if not directory.exists() or not directory.is_dir():
return []
files_info: List[Dict[str, Any]] = []
try:
for item in directory.iterdir():
if not item.is_file():
continue
ext = item.suffix.lower()
if ext not in SUPPORTED_MEDIA_EXTENSIONS:
continue
file_hash = None
if compute_hash:
try:
file_hash = sha256_file(item)
except Exception as exc:
debug(f"Failed to hash {item}: {exc}")
continue
try:
size = item.stat().st_size
except Exception:
size = 0
files_info.append(
{
"path": item,
"name": item.name,
"hash": file_hash,
"size": size,
"ext": ext,
}
)
except Exception as exc:
debug(f"Error scanning directory {directory}: {exc}")
return files_info
def _validate_source(media_path: Optional[Path], allow_all_extensions: bool = False) -> bool:
"""Validate that the source file exists and is supported.
Args:
media_path: Path to the file to validate
allow_all_extensions: If True, skip file type filtering for non-backend exports.
If False, only allow SUPPORTED_MEDIA_EXTENSIONS for backend ingest.
"""
if media_path is None:
return False
if not media_path.exists() or not media_path.is_file():
log(f"File not found: {media_path}")
return False
if not allow_all_extensions:
file_extension = media_path.suffix.lower()
if file_extension not in SUPPORTED_MEDIA_EXTENSIONS:
log(f"❌ Unsupported file type: {file_extension}", file=sys.stderr)
return False
return True
def _is_probable_url(s: Any) -> bool:
"""Check if a string looks like a URL/magnet/identifier (vs a tag/title)."""
if not isinstance(s, str):
return False
val = s.strip().lower()
if not val:
return False
if val.startswith(_REMOTE_URL_PREFIXES):
return True
if "://" in val:
return True
if val.startswith("hash:"):
return False
return False
def _resolve_source(
result: Any,
source_arg: Optional[str],
pipe_obj: models.PipeObject,
config: Dict[str, Any],
export_destination: Optional[Path] = None,
store_instance: Optional[Any] = None,
deps: Optional[_CommandDependencies] = None,
) -> tuple[Optional[Path], Optional[str], Optional[Path]]:
"""Resolve the source file path from the positional source arg or pipeline result.
Returns (media_path, file_hash, temp_dir_to_cleanup).
"""
# PRIORITY 1a: Prefer an explicit local path when it exists.
if isinstance(result, dict):
r_path = result.get("path")
r_hash = result.get("hash")
if r_path:
try:
p = coerce_to_path(r_path)
if p.exists() and p.is_file():
pipe_obj.path = str(p)
return p, str(r_hash) if r_hash else None, None
except Exception:
pass
# PRIORITY 1b: Try hash+store from result (fetch from backend)
r_hash = get_field(result, "hash") or get_field(result, "file_hash")
r_store = get_field(result, "store")
if r_hash and r_store:
try:
if deps is None:
deps = _CommandDependencies(config)
backend_registry = store_instance or deps.get_backend_registry()
backend = _resolve_backend_by_name(backend_registry, r_store)
if backend is not None:
mp = backend.get_file(r_hash)
if isinstance(mp, Path) and mp.exists():
pipe_obj.path = str(mp)
return mp, str(r_hash), None
if isinstance(mp, str) and mp.strip():
try:
mp_path = Path(str(mp))
except Exception:
mp_path = None
if mp_path is not None and mp_path.exists() and mp_path.is_file():
pipe_obj.path = str(mp_path)
return mp_path, str(r_hash), None
dl_path, tmp_dir = _maybe_download_backend_file(
backend,
str(r_hash),
pipe_obj,
output_dir=export_destination,
)
if dl_path and dl_path.exists():
pipe_obj.path = str(dl_path)
return dl_path, str(r_hash), tmp_dir
dl_path, tmp_dir = _download_remote_backend_url(
str(mp),
pipe_obj,
file_hash=str(r_hash),
output_dir=export_destination,
)
if dl_path and dl_path.exists():
pipe_obj.path = str(dl_path)
return dl_path, str(r_hash), tmp_dir
except Exception as exc:
debug(f"[add-file] _resolve_source backend fetch failed for {r_store}/{r_hash}: {exc}")
# PRIORITY 2: Generic Coercion (Path arg > PipeObject > Result)
candidate: Optional[Path] = None
if source_arg:
candidate = Path(source_arg)
elif pipe_obj.path:
candidate = Path(pipe_obj.path)
if not candidate:
obj = result[0] if isinstance(result, list) and result else result
if obj:
try:
candidate = coerce_to_path(obj)
except ValueError:
pass
if candidate:
s = str(candidate).lower()
if s.startswith(_REMOTE_URL_PREFIXES):
downloaded_path, hash_hint, tmp_dir = _maybe_download_plugin_result(
result,
pipe_obj,
config,
deps=deps,
)
if downloaded_path:
pipe_obj.path = str(downloaded_path)
return downloaded_path, hash_hint, tmp_dir
dl_path, tmp_dir = _download_remote_backend_url(
str(candidate),
pipe_obj,
file_hash=get_field(result, "hash") or get_field(result, "file_hash"),
output_dir=export_destination,
)
if dl_path:
pipe_obj.path = str(dl_path)
hash_hint = get_field(result, "hash") or get_field(result, "file_hash")
return dl_path, hash_hint, tmp_dir
log("add-file could not auto-fetch remote source. Use download-file first.", file=sys.stderr)
return None, None, None
pipe_obj.path = str(candidate)
hash_hint = get_field(result, "hash") or get_field(result, "file_hash") or getattr(pipe_obj, "hash", None)
return candidate, hash_hint, None
downloaded_path, hash_hint, tmp_dir = _maybe_download_plugin_result(
result,
pipe_obj,
config,
deps=deps,
)
if downloaded_path:
pipe_obj.path = str(downloaded_path)
return downloaded_path, hash_hint, tmp_dir
debug(f"No resolution path matched. result type={type(result).__name__}")
log("File path could not be resolved")
return None, None, None
cmdlet/file/download.py
+50 -3095
View File
File diff suppressed because it is too large Load Diff
cmdlet/file/download_core.py
+897
View File
@@ -0,0 +1,897 @@
"""Generic file/stream downloader — core orchestration.
Supports:
- Direct HTTP file URLs (PDFs, images, documents; non-yt-dlp)
- Piped plugin items (uses plugin.download when available)
- Streaming sites via yt-dlp (YouTube, Bandcamp, etc.)
"""
from __future__ import annotations
from collections.abc import Mapping, Sequence as SequenceABC
import sys
import re
from pathlib import Path
from typing import Any, Dict, List, Optional, Sequence
from urllib.parse import urlparse
from contextlib import AbstractContextManager, nullcontext
import shutil
import webbrowser
from API.HTTP import download_direct_file
from SYS.models import DownloadError, DownloadOptions, DownloadMediaResult
from SYS.logger import log, debug_panel, is_debug_enabled
from SYS.payload_builders import build_file_result_payload, build_table_result_payload
from SYS.pipeline_progress import PipelineProgress
from SYS.result_table import Table, build_display_row
from SYS.rich_display import stderr_console as get_stderr_console
from SYS import pipeline as pipeline_context
from SYS.item_accessors import get_result_title
from rich.prompt import Prompt
from SYS.selection_builder import (
build_hash_store_selection,
extract_selection_fields,
extract_urls_from_selection_args,
selection_args_have_url,
)
from SYS.utils import sha256_file
try:
from plugins.ytdlp import YtDlpTool # type: ignore
except Exception: # pragma: no cover - optional dependency for tests/runtime wrappers
YtDlpTool = None # type: ignore
from .. import _shared as sh
Cmdlet = sh.Cmdlet
CmdletArg = sh.CmdletArg
SharedArgs = sh.SharedArgs
QueryArg = sh.QueryArg
parse_cmdlet_args = sh.parse_cmdlet_args
register_url_with_local_library = sh.register_url_with_local_library
coerce_to_pipe_object = sh.coerce_to_pipe_object
get_field = sh.get_field
resolve_target_dir = sh.resolve_target_dir
coerce_to_path = sh.coerce_to_path
build_pipeline_preview = sh.build_pipeline_preview
class Download_File(Cmdlet):
"""Class-based download-file cmdlet - direct HTTP downloads."""
def __init__(self) -> None:
"""Initialize download-file cmdlet."""
super().__init__(
name="download-file",
summary="Download files or streaming media",
usage=
"download-file <url|path> [-plugin NAME] [-instance NAME] [-path DIR] [options] OR @N | download-file [-plugin NAME] [-instance NAME] [-path DIR] [options] OR download-file -query \"hash:<sha256>\" -instance <store> [-browser]",
alias=["dl-file",
"download-http"],
arg=[
SharedArgs.URL,
SharedArgs.PLUGIN,
SharedArgs.INSTANCE,
SharedArgs.PATH,
SharedArgs.QUERY,
CmdletArg(
name="name",
type="string",
description="Output filename override for store exports.",
),
CmdletArg(
name="browser",
type="flag",
description="Open a backend-provided browser URL instead of exporting to disk when available.",
),
QueryArg(
"clip",
key="clip",
aliases=["range",
"section",
"sections"],
type="string",
required=False,
description=(
"Clip time ranges via -query keyed fields (e.g. clip:1m-2m or clip:00:01-00:10). "
"Comma-separated values supported."
),
query_only=True,
),
CmdletArg(
name="item",
type="string",
description="Item selection for playlists/formats",
),
],
detail=[
"Download files directly via HTTP or streaming media via yt-dlp.",
"Also exports store-backed files via hash+store selection or -query \"hash:<sha256>\" -instance <store>.",
"Use -plugin with -instance to target a named provider config when a plugin exposes multiple instances.",
"For Internet Archive item pages (archive.org/details/...), shows a selectable file/format list; pick with @N to download.",
],
exec=self.run,
)
self.register()
def run(self, result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
"""Main execution method."""
try:
debug_panel(
"download-file",
[
("args", list(args)),
("has_piped_input", bool(result)),
],
border_style="cyan",
)
except Exception:
pass
return self._run_impl(result, args, config)
@staticmethod
def _path_from_download_result(result_obj: Any) -> Path:
"""Normalize downloader return values to a concrete filesystem path."""
resolved = coerce_to_path(result_obj)
if resolved is None:
raise DownloadError("Could not determine downloaded file path")
return resolved
@staticmethod
def _selection_run_label(
run_args: Sequence[str],
*,
extra_url_prefixes: Sequence[str] = (),
) -> str:
try:
urls = extract_urls_from_selection_args(
run_args,
extra_url_prefixes=extra_url_prefixes,
)
if urls:
return str(urls[0])
except Exception:
pass
for arg in run_args:
text = str(arg or "").strip()
if text and not text.startswith("-"):
return text
return "item"
@staticmethod
def _batch_progress_state(config: Optional[Dict[str, Any]]) -> tuple[bool, int, int, str]:
if not isinstance(config, dict):
return False, 0, 0, ""
suppress_nested = bool(config.get("_download_file_suppress_nested_pipe_progress"))
if not suppress_nested:
return False, 0, 0, ""
try:
total = max(0, int(config.get("_download_file_batch_total") or 0))
except Exception:
total = 0
try:
index = max(0, int(config.get("_download_file_batch_index") or 0))
except Exception:
index = 0
try:
label = str(config.get("_download_file_batch_label") or "").strip()
except Exception:
label = ""
return True, total, index, label
@staticmethod
def _selection_url_prefixes(registry: Dict[str, Any]) -> List[str]:
loader = registry.get("list_selection_url_prefixes")
if not callable(loader):
return []
try:
values = loader() or []
except Exception:
return []
return [str(value).strip().lower() for value in values if str(value or "").strip()]
def _normalize_provider_key(self, value: Optional[Any]) -> Optional[str]:
if value is None:
return None
try:
normalized = str(value).strip()
except Exception:
return None
if not normalized:
return None
if "." in normalized:
normalized = normalized.split(".", 1)[0]
return normalized.lower()
def _provider_key_from_item(self, item: Any) -> Optional[str]:
table_hint = get_field(item, "table")
key = self._normalize_provider_key(table_hint)
if key:
return key
provider_hint = get_field(item, "plugin")
key = self._normalize_provider_key(provider_hint)
if key:
return key
return self._normalize_provider_key(get_field(item, "source"))
@staticmethod
def _path_looks_local(value: Any) -> bool:
text = str(value or "").strip()
if not text:
return False
if text.startswith(("http://", "https://", "ftp://", "ftps://", "magnet:", "torrent:")):
return False
if len(text) >= 2 and text[1] == ":":
return True
if text.startswith(("\\", "/", ".", "~")):
return True
return Path(text).exists()
@staticmethod
def _resolve_display_title(result: Any, metadata: Optional[Dict[str, Any]]) -> str:
candidates = [
get_result_title(result, "title", "name", "filename"),
get_result_title(metadata or {}, "title", "name", "filename"),
]
for candidate in candidates:
if candidate is None:
continue
text = str(candidate).strip()
if text:
return text
return ""
@staticmethod
def _sanitize_export_filename(name: str) -> str:
allowed_chars: List[str] = []
for ch in str(name or ""):
if ch.isalnum() or ch in {"-", "_", " ", "."}:
allowed_chars.append(ch)
else:
allowed_chars.append(" ")
sanitized = " ".join("".join(allowed_chars).split())
return sanitized or "export"
@staticmethod
def _unique_export_path(path: Path) -> Path:
if not path.exists():
return path
stem = path.stem
suffix = path.suffix
parent = path.parent
counter = 1
while True:
candidate = parent / f"{stem} ({counter}){suffix}"
if not candidate.exists():
return candidate
counter += 1
@staticmethod
def _load_provider_registry() -> Dict[str, Any]:
"""Lightweight accessor for plugin helpers without hard dependencies."""
try:
from PluginCore import registry as provider_registry # type: ignore
from PluginCore.base import SearchResult # type: ignore
return {
"get_plugin": getattr(provider_registry, "get_plugin", None),
"match_plugin_name_for_url": getattr(provider_registry, "match_plugin_name_for_url", None),
"list_selection_url_prefixes": getattr(provider_registry, "list_selection_url_prefixes", None),
"SearchResult": SearchResult,
}
except Exception:
return {
"get_plugin": None,
"match_plugin_name_for_url": None,
"list_selection_url_prefixes": None,
"SearchResult": None,
}
@classmethod
def _normalize_duplicate_preflight_policy(cls, value: Any) -> Optional[str]:
text = str(value or "").strip().lower()
if not text:
return "skip"
mapping = {
"i": "ignore",
"ignore": "ignore",
"s": "skip",
"skip": "skip",
"c": "cancel",
"cancel": "cancel",
}
return mapping.get(text)
def _maybe_render_download_details(self, *, config: Dict[str, Any]) -> None:
try:
stage_ctx = pipeline_context.get_stage_context()
except Exception:
stage_ctx = None
is_last_stage = (stage_ctx is None) or bool(getattr(stage_ctx, "is_last_stage", False))
if not is_last_stage:
return
try:
quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False
except Exception:
quiet_mode = False
if quiet_mode:
return
emitted_items: List[Any] = []
try:
emitted_items = list(getattr(stage_ctx, "emits", None) or []) if stage_ctx is not None else []
except Exception:
emitted_items = []
if not emitted_items:
return
try:
live_progress = pipeline_context.get_live_progress()
except Exception:
live_progress = None
if live_progress is not None:
try:
pipe_idx = getattr(stage_ctx, "pipe_index", None) if stage_ctx is not None else None
if isinstance(pipe_idx, int):
live_progress.finish_pipe(int(pipe_idx), force_complete=True)
except Exception:
pass
try:
live_progress.stop()
except Exception:
pass
try:
if hasattr(pipeline_context, "set_live_progress"):
pipeline_context.set_live_progress(None)
except Exception:
pass
try:
subject = emitted_items[0] if len(emitted_items) == 1 else list(emitted_items)
from .._shared import display_and_persist_items
display_and_persist_items(list(emitted_items), title="Result", subject=subject)
except Exception:
pass
try:
if stage_ctx is not None:
stage_ctx.emits = []
except Exception:
pass
def _maybe_show_provider_picker(
self,
*,
raw_urls: Sequence[str],
piped_items: Sequence[Any],
parsed: Dict[str, Any],
config: Dict[str, Any],
registry: Dict[str, Any],
) -> Optional[int]:
"""Generic hook for providers to show a selection table (e.g. Internet Archive format picker)."""
total_inputs = len(raw_urls or []) + len(piped_items or [])
if total_inputs != 1:
return None
target_url = None
if raw_urls:
target_url = str(raw_urls[0])
elif piped_items:
target_url = str(get_field(piped_items[0], "path") or get_field(piped_items[0], "url") or "")
if not target_url:
return None
match_provider_name_for_url = registry.get("match_plugin_name_for_url")
get_provider = registry.get("get_plugin")
provider_name = None
if match_provider_name_for_url:
try:
provider_name = match_provider_name_for_url(target_url)
except Exception:
pass
if provider_name and get_provider:
provider = get_provider(provider_name, config)
if provider and hasattr(provider, "maybe_show_picker"):
try:
quiet_mode = bool(config.get("_quiet_background_output"))
res = provider.maybe_show_picker(
url=target_url,
item=piped_items[0] if piped_items else None,
parsed=parsed,
config=config,
quiet_mode=quiet_mode,
)
if res is not None:
return int(res)
except Exception as e:
debug_panel(
"download-file picker error",
[
("plugin", provider_name),
("url", target_url),
("error", e),
],
border_style="yellow",
)
return None
def _run_impl(
self,
result: Any,
args: Sequence[str],
config: Dict[str, Any]
) -> int:
"""Main download implementation for direct HTTP files."""
progress = PipelineProgress(pipeline_context)
prev_progress = None
had_progress_key = False
try:
try:
if isinstance(config, dict):
had_progress_key = "_pipeline_progress" in config
prev_progress = config.get("_pipeline_progress")
config["_pipeline_progress"] = progress
except Exception:
pass
parsed = parse_cmdlet_args(args, self)
registry = self._load_provider_registry()
selection_url_prefixes = self._selection_url_prefixes(registry)
explicit_input = parsed.get("url")
url_candidates = parsed.get("url") or [
a for a in parsed.get("args", [])
if isinstance(a, str) and (
a.startswith("http") or "://" in a or ":" in a
or "\U0001f9f2" in a
and not a.startswith("-")
)
]
from SYS.metadata import normalize_urls as normalize_url_list # lazy: avoids Cryptodome at startup
raw_url = normalize_url_list(url_candidates)
local_source_inputs: List[str] = []
if not raw_url and isinstance(explicit_input, str) and self._path_looks_local(explicit_input):
local_source_inputs = [str(explicit_input)]
quiet_mode = bool(config.get("_quiet_background_output")) if isinstance(config, dict) else False
piped_items = []
if not raw_url:
piped_items = sh.normalize_result_items(
result,
include_falsey_single=True,
)
if piped_items and not raw_url:
selection_runs: List[List[str]] = []
residual_items: List[Any] = []
for item in piped_items:
handled = False
try:
normalized_args, _normalized_action, item_url = extract_selection_fields(
item,
extra_url_prefixes=selection_url_prefixes,
)
if normalized_args:
if selection_args_have_url(
normalized_args,
extra_url_prefixes=selection_url_prefixes,
):
selection_runs.append(list(normalized_args))
handled = True
elif item_url:
selection_runs.append([str(item_url)] + list(normalized_args))
handled = True
except Exception as e:
debug_panel(
"download-file selection args failed",
[("error", e)],
border_style="yellow",
)
handled = False
if not handled:
residual_items.append(item)
if selection_runs:
selection_urls: List[str] = []
for run_args in selection_runs:
for u in extract_urls_from_selection_args(
run_args,
extra_url_prefixes=selection_url_prefixes,
):
if u not in selection_urls:
selection_urls.append(u)
original_skip_preflight = None
original_timeout = None
original_skip_direct = None
original_batch_total = None
original_batch_index = None
original_batch_label = None
original_suppress_nested = None
try:
if isinstance(config, dict):
original_skip_preflight = config.get("_skip_url_preflight")
original_timeout = config.get("_pipeobject_timeout_seconds")
original_skip_direct = config.get("_skip_direct_on_streaming_failure")
original_batch_total = config.get("_download_file_batch_total")
original_batch_index = config.get("_download_file_batch_index")
original_batch_label = config.get("_download_file_batch_label")
original_suppress_nested = config.get("_download_file_suppress_nested_pipe_progress")
except Exception:
original_skip_preflight = None
original_timeout = None
original_batch_total = None
original_batch_index = None
original_batch_label = None
original_suppress_nested = None
try:
if selection_urls:
if isinstance(config, dict):
config["_skip_url_preflight"] = True
config["_skip_direct_on_streaming_failure"] = True
if isinstance(config, dict) and config.get("_pipeobject_timeout_seconds") is None:
config["_pipeobject_timeout_seconds"] = 600
successes = 0
failures = 0
last_code = 0
total_selection = len(selection_runs)
preview_items = list(selection_urls[:5]) or [
self._selection_run_label(
run_args,
extra_url_prefixes=selection_url_prefixes,
)
for run_args in selection_runs[:5]
]
try:
progress.ensure_local_ui(
label="download-file",
total_items=total_selection,
items_preview=preview_items,
)
except Exception:
pass
try:
progress.begin_pipe(
total_items=total_selection,
items_preview=preview_items,
)
except Exception:
pass
for idx, run_args in enumerate(selection_runs, 1):
run_label = self._selection_run_label(
run_args,
extra_url_prefixes=selection_url_prefixes,
)
try:
progress.set_status(
f"downloading {idx}/{total_selection}: {run_label}"
)
except Exception:
pass
try:
if isinstance(config, dict):
config["_download_file_batch_total"] = total_selection
config["_download_file_batch_index"] = idx
config["_download_file_batch_label"] = run_label
config["_download_file_suppress_nested_pipe_progress"] = True
except Exception:
pass
exit_code = self._run_impl(None, run_args, config)
if exit_code == 0:
successes += 1
else:
failures += 1
last_code = exit_code
piped_items = residual_items
if not piped_items:
if successes > 0:
return 0
return last_code or 1
finally:
try:
if isinstance(config, dict):
if original_skip_preflight is None:
config.pop("_skip_url_preflight", None)
else:
config["_skip_url_preflight"] = original_skip_preflight
if original_timeout is None:
config.pop("_pipeobject_timeout_seconds", None)
else:
config["_pipeobject_timeout_seconds"] = original_timeout
if original_skip_direct is None:
config.pop("_skip_direct_on_streaming_failure", None)
else:
config["_skip_direct_on_streaming_failure"] = original_skip_direct
if original_batch_total is None:
config.pop("_download_file_batch_total", None)
else:
config["_download_file_batch_total"] = original_batch_total
if original_batch_index is None:
config.pop("_download_file_batch_index", None)
else:
config["_download_file_batch_index"] = original_batch_index
if original_batch_label is None:
config.pop("_download_file_batch_label", None)
else:
config["_download_file_batch_label"] = original_batch_label
if original_suppress_nested is None:
config.pop("_download_file_suppress_nested_pipe_progress", None)
else:
config["_download_file_suppress_nested_pipe_progress"] = original_suppress_nested
except Exception:
pass
had_piped_input = False
try:
if isinstance(result, list):
had_piped_input = bool(result)
else:
had_piped_input = bool(result)
except Exception:
had_piped_input = False
if (had_piped_input and raw_url and len(raw_url) == 1
and (not parsed.get("path"))):
candidate = str(raw_url[0] or "").strip()
low = candidate.lower()
looks_like_url = low.startswith(
("http://", "https://", "ftp://", "magnet:", "torrent:")
+ tuple(selection_url_prefixes)
)
looks_like_provider = (
":" in candidate and not candidate.startswith(
("http:", "https:", "ftp:", "ftps:", "file:")
+ tuple(selection_url_prefixes)
)
)
looks_like_windows_path = (
(len(candidate) >= 2 and candidate[1] == ":")
or candidate.startswith("\\\\") or candidate.startswith("\\")
or candidate.endswith(("\\", "/"))
)
if (not looks_like_url) and (
not looks_like_provider) and looks_like_windows_path:
parsed["path"] = candidate
raw_url = []
piped_items = sh.normalize_result_items(result, include_falsey_single=True)
if not raw_url and not piped_items and not local_source_inputs:
log("No url or piped items to download", file=sys.stderr)
return 1
picker_result = self._maybe_show_provider_picker(
raw_urls=raw_url,
piped_items=piped_items,
parsed=parsed,
config=config,
registry=registry,
)
if picker_result is not None:
return int(picker_result)
picker_result = self._maybe_show_provider_picker(
raw_urls=raw_url,
piped_items=piped_items,
parsed=parsed,
config=config,
registry=registry,
)
if picker_result is not None:
return int(picker_result)
final_output_dir = resolve_target_dir(parsed, config)
if not final_output_dir:
return 1
try:
debug_panel(
"download-file plan",
[
("output_dir", final_output_dir),
("remaining_urls", len(raw_url)),
("piped_items", len(piped_items) if isinstance(piped_items, list) else int(bool(piped_items))),
],
border_style="cyan",
)
except Exception:
pass
total_items = max(1, len(raw_url or []) + len(piped_items or []))
preview = build_pipeline_preview(raw_url, piped_items)
progress.ensure_local_ui(
label="download-file",
total_items=total_items,
items_preview=preview
)
raw_url, preflight_exit, skipped_dupe_count = self._preflight_explicit_url_duplicates(
raw_urls=raw_url,
config=config,
)
if preflight_exit is not None:
return int(preflight_exit)
downloaded_count = 0
if local_source_inputs:
downloaded_count += self._process_explicit_local_sources(
local_sources=local_source_inputs,
final_output_dir=final_output_dir,
parsed=parsed,
progress=progress,
config=config,
)
storage_downloaded, piped_items, storage_exit = self._process_storage_items(
piped_items=piped_items,
parsed=parsed,
config=config,
final_output_dir=final_output_dir,
)
downloaded_count += int(storage_downloaded)
if storage_exit is not None:
return int(storage_exit)
if skipped_dupe_count and not raw_url and not piped_items:
return 0 if downloaded_count > 0 else 0
urls_downloaded, early_exit = self._process_explicit_urls(
raw_urls=raw_url,
final_output_dir=final_output_dir,
config=config,
quiet_mode=quiet_mode,
registry=registry,
progress=progress,
parsed=parsed,
args=args,
context_items=(result if isinstance(result, list) else ([result] if result else [])),
)
downloaded_count += int(urls_downloaded)
if early_exit is not None:
return int(early_exit)
provider_downloaded, magnet_submissions = self._process_provider_items(
piped_items=piped_items,
final_output_dir=final_output_dir,
config=config,
quiet_mode=quiet_mode,
registry=registry,
progress=progress,
)
downloaded_count += provider_downloaded
if downloaded_count > 0 or magnet_submissions > 0:
self._maybe_render_download_details(config=config)
return 0
return 1
except Exception as e:
log(f"Error in download-file: {e}", file=sys.stderr)
return 1
finally:
try:
if isinstance(config, dict):
if had_progress_key:
config["_pipeline_progress"] = prev_progress
else:
config.pop("_pipeline_progress", None)
except Exception:
pass
progress.close_local_ui(force_complete=True)
# Late-binding: import sub-module functions and attach them as methods.
from .download_fetch import ( # noqa: E402
_emit_plugin_items,
_consume_plugin_download_result,
_process_explicit_urls,
_expand_provider_items,
_process_provider_items,
_download_provider_items,
_emit_local_file,
_resolve_provider_preflight_items,
_build_provider_playlist_item_selector,
_format_timecode,
_rebase_subtitle_timestamp_text,
_format_clip_range,
_apply_clip_decorations,
_parse_clip_spec_to_ranges,
)
from .download_storage import ( # noqa: E402
_iter_storage_export_refs,
_export_store_file,
_process_storage_items,
_process_explicit_local_sources,
_extract_hash_from_search_hit,
_iter_duplicate_tag_values,
_extract_duplicate_namespace_tags,
_extract_duplicate_title_tag,
_extract_duplicate_title,
_has_duplicate_title,
_build_duplicate_display_row,
_fetch_duplicate_metadata_for_hash,
_enrich_duplicate_metadata,
_fetch_duplicate_metadata_for_hashes,
_collect_existing_url_match_refs_for_url,
_find_existing_url_matches_for_url,
_find_existing_hash_for_url,
_find_existing_hashes_for_url,
_init_storage,
_supports_storage_duplicate_lookup,
_filter_supported_urls,
_canonicalize_url_for_storage,
_preflight_url_duplicate,
_preflight_explicit_url_duplicates,
_download_supported_urls,
_maybe_show_playlist_table,
_maybe_show_format_table_for_single_url,
_run_streaming_urls,
)
Download_File._emit_plugin_items = _emit_plugin_items
Download_File._consume_plugin_download_result = _consume_plugin_download_result
Download_File._process_explicit_urls = _process_explicit_urls
Download_File._expand_provider_items = _expand_provider_items
Download_File._process_provider_items = _process_provider_items
Download_File._download_provider_items = _download_provider_items
Download_File._emit_local_file = _emit_local_file
Download_File._resolve_provider_preflight_items = staticmethod(_resolve_provider_preflight_items)
Download_File._build_provider_playlist_item_selector = staticmethod(_build_provider_playlist_item_selector)
Download_File._format_timecode = staticmethod(_format_timecode)
Download_File._rebase_subtitle_timestamp_text = staticmethod(_rebase_subtitle_timestamp_text)
Download_File._format_clip_range = classmethod(_format_clip_range)
Download_File._apply_clip_decorations = classmethod(_apply_clip_decorations)
Download_File._parse_clip_spec_to_ranges = staticmethod(_parse_clip_spec_to_ranges)
Download_File._iter_storage_export_refs = staticmethod(_iter_storage_export_refs)
Download_File._export_store_file = _export_store_file
Download_File._process_storage_items = _process_storage_items
Download_File._process_explicit_local_sources = _process_explicit_local_sources
Download_File._extract_hash_from_search_hit = classmethod(_extract_hash_from_search_hit)
Download_File._iter_duplicate_tag_values = staticmethod(_iter_duplicate_tag_values)
Download_File._extract_duplicate_namespace_tags = staticmethod(_extract_duplicate_namespace_tags)
Download_File._extract_duplicate_title_tag = staticmethod(_extract_duplicate_title_tag)
Download_File._extract_duplicate_title = classmethod(_extract_duplicate_title)
Download_File._has_duplicate_title = classmethod(_has_duplicate_title)
Download_File._build_duplicate_display_row = classmethod(_build_duplicate_display_row)
Download_File._fetch_duplicate_metadata_for_hash = classmethod(_fetch_duplicate_metadata_for_hash)
Download_File._enrich_duplicate_metadata = classmethod(_enrich_duplicate_metadata)
Download_File._fetch_duplicate_metadata_for_hashes = classmethod(_fetch_duplicate_metadata_for_hashes)
Download_File._collect_existing_url_match_refs_for_url = classmethod(_collect_existing_url_match_refs_for_url)
Download_File._find_existing_url_matches_for_url = classmethod(_find_existing_url_matches_for_url)
Download_File._find_existing_hash_for_url = classmethod(_find_existing_hash_for_url)
Download_File._find_existing_hashes_for_url = classmethod(_find_existing_hashes_for_url)
Download_File._init_storage = staticmethod(_init_storage)
Download_File._supports_storage_duplicate_lookup = staticmethod(_supports_storage_duplicate_lookup)
Download_File._filter_supported_urls = staticmethod(_filter_supported_urls)
Download_File._canonicalize_url_for_storage = staticmethod(_canonicalize_url_for_storage)
Download_File._preflight_url_duplicate = staticmethod(_preflight_url_duplicate)
Download_File._preflight_explicit_url_duplicates = _preflight_explicit_url_duplicates
Download_File._download_supported_urls = _download_supported_urls
Download_File._maybe_show_playlist_table = _maybe_show_playlist_table
Download_File._maybe_show_format_table_for_single_url = _maybe_show_format_table_for_single_url
Download_File._run_streaming_urls = _run_streaming_urls
CMDLET = Download_File()
cmdlet/file/download_fetch.py
+1033
View File
File diff suppressed because it is too large Load Diff
cmdlet/file/download_storage.py
+1321
View File
File diff suppressed because it is too large Load Diff
cmdlet/file/screenshot.py
+2 -14
View File
@@ -43,29 +43,17 @@ from SYS import pipeline as pipeline_context
# Playwright & Screenshot Dependencies
# ============================================================================
from plugins.playwright import PlaywrightTimeoutError, PlaywrightTool
from plugins.playwright import PlaywrightTimeoutError, PlaywrightTool, USER_AGENT
try:
from SYS.config import resolve_output_dir
except ImportError:
try:
_parent_dir = str(Path(__file__).parent.parent)
if _parent_dir not in sys.path:
sys.path.insert(0, _parent_dir)
from SYS.config import resolve_output_dir
except ImportError:
resolve_output_dir = None
resolve_output_dir = None
# ============================================================================
# Screenshot Constants & Configuration
# ============================================================================
USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/120.0.0.0 Safari/537.36"
)
DEFAULT_VIEWPORT: dict[str,
int] = {
"width": 1920,
cmdlet/file/search.py
+456 -1201
View File
File diff suppressed because it is too large Load Diff
cmdlet/file/search_engines.py
+994
View File
@@ -0,0 +1,994 @@
"""Search engine result parsing — Bing, DuckDuckGo, Yahoo, and site crawling.
Extracted from search.py to keep the cmdlet class focused on orchestration.
"""
from __future__ import annotations
from typing import Any, Dict, List, Optional, Sequence
from collections import deque
from pathlib import Path
import re
import time
import html as _html
from urllib.parse import urlparse, parse_qs, unquote, urljoin
from SYS.logger import debug
from SYS.payload_builders import normalize_file_extension
_WHITESPACE_RE = re.compile(r"\s+")
_SITE_TOKEN_RE = re.compile(r"(?:^|\s)site:([^\s,]+)", flags=re.IGNORECASE)
_FILETYPE_TOKEN_RE = re.compile(
r"(?:^|\s)(?:ext|filetype|type):\.?([a-z0-9]{1,12})\b",
flags=re.IGNORECASE,
)
_SITE_REMOVE_RE = re.compile(r"(?:^|\s)site:[^\s,]+", flags=re.IGNORECASE)
_FILETYPE_REMOVE_RE = re.compile(
r"(?:^|\s)(?:ext|filetype|type):\.?[a-z0-9]{1,12}\b",
flags=re.IGNORECASE,
)
_SCHEME_PREFIX_RE = re.compile(r"^[a-z]+:")
_YAHOO_RU_RE = re.compile(r"/RU=([^/]+)/RK=", flags=re.IGNORECASE)
_HTML_TAG_RE = re.compile(r"<[^>]+>")
_DDG_RESULT_ANCHOR_RE = re.compile(
r'<a[^>]+class="[^"]*result__a[^"]*"[^>]+href="([^"]+)"[^>]*>(.*?)</a>',
flags=re.IGNORECASE | re.DOTALL,
)
_GENERIC_ANCHOR_RE = re.compile(
r'<a[^>]+href=["\']([^"\']+)["\'][^>]*>(.*?)</a>',
flags=re.IGNORECASE | re.DOTALL,
)
_BING_RESULT_ANCHOR_RE = re.compile(
r'<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>',
flags=re.IGNORECASE | re.DOTALL,
)
def _normalize_extension(ext_value: Any) -> str:
"""Sanitize extension strings to alphanumerics and cap at 5 chars."""
return normalize_file_extension(ext_value)
def _normalize_host(value: Any) -> str:
"""Normalize host names for matching/filtering."""
host = str(value or "").strip().lower()
if host.startswith("www."):
host = host[4:]
if ":" in host:
host = host.split(":", 1)[0]
return host
def _normalize_space(text: Any) -> str:
return _WHITESPACE_RE.sub(" ", str(text or "")).strip()
def _url_matches_site(cls, url: str, site_host: str) -> bool:
"""Return True when URL host is the requested site/subdomain."""
try:
parsed = urlparse(str(url or ""))
host = _normalize_host(getattr(parsed, "hostname", "") or "")
except Exception:
return False
target = _normalize_host(site_host)
if not host or not target:
return False
return host == target or host.endswith(f".{target}")
def _itertext_join(node: Any) -> str:
try:
return " ".join([str(text).strip() for text in node.itertext() if str(text).strip()])
except Exception:
return ""
def _html_fragment_to_text(fragment: Any) -> str:
text = _HTML_TAG_RE.sub(" ", str(fragment or ""))
return _html.unescape(text)
def _append_web_result(
cls,
items: List[Dict[str, str]],
seen_urls: set[str],
*,
site_host: str,
url_text: str,
title_text: str,
snippet_text: str,
) -> None:
url_clean = str(url_text or "").strip()
if not url_clean or not url_clean.startswith(("http://", "https://")):
return
if not _url_matches_site(url_clean, site_host):
return
if url_clean in seen_urls:
return
seen_urls.add(url_clean)
items.append(
{
"url": url_clean,
"title": _normalize_space(title_text) or url_clean,
"snippet": _normalize_space(snippet_text),
}
)
def _parse_web_results_with_fallback(
cls,
*,
html_text: str,
limit: int,
lxml_parser: Any,
regex_parser: Any,
fallback_when_empty: bool = False,
) -> List[Dict[str, str]]:
"""Run an lxml-based parser with an optional regex fallback."""
items: List[Dict[str, str]] = []
seen_urls: set[str] = set()
should_run_regex = False
try:
from lxml import html as lxml_html
doc = lxml_html.fromstring(html_text or "")
lxml_parser(doc, items, seen_urls)
should_run_regex = fallback_when_empty and not items
except Exception:
should_run_regex = True
if should_run_regex:
regex_parser(html_text or "", items, seen_urls)
return items[:limit]
def _extract_duckduckgo_target_url(href: Any) -> str:
"""Extract direct target URL from DuckDuckGo result links."""
raw_href = str(href or "").strip()
if not raw_href:
return ""
if raw_href.startswith("//"):
raw_href = f"https:{raw_href}"
if raw_href.startswith("/"):
raw_href = f"https://duckduckgo.com{raw_href}"
parsed = None
try:
parsed = urlparse(raw_href)
except Exception:
parsed = None
try:
host = str(getattr(parsed, "hostname", "") or "").strip().lower()
except Exception:
host = ""
if host.endswith("duckduckgo.com"):
try:
query = parse_qs(str(getattr(parsed, "query", "") or ""))
candidate = (query.get("uddg") or [""])[0]
if candidate:
return str(unquote(candidate)).strip()
except Exception:
pass
return raw_href
def _extract_yahoo_target_url(href: Any) -> str:
"""Extract direct target URL from Yahoo redirect links."""
raw_href = str(href or "").strip()
if not raw_href:
return ""
ru_match = _YAHOO_RU_RE.search(raw_href)
if ru_match:
try:
return str(unquote(ru_match.group(1))).strip()
except Exception:
pass
try:
parsed = urlparse(raw_href)
query = parse_qs(str(getattr(parsed, "query", "") or ""))
candidate = (query.get("RU") or query.get("ru") or [""])[0]
if candidate:
return str(unquote(candidate)).strip()
except Exception:
pass
return raw_href
def parse_duckduckgo_results(
cls,
*,
html_text: str,
site_host: str,
limit: int,
) -> List[Dict[str, str]]:
"""Parse DuckDuckGo HTML results into normalized rows."""
def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
result_nodes = doc.xpath("//div[contains(@class, 'result')]")
for node in result_nodes:
links = node.xpath(".//a[contains(@class, 'result__a')]")
if not links:
continue
link = links[0]
href = _extract_duckduckgo_target_url(link.get("href"))
title = _itertext_join(link)
snippet_nodes = node.xpath(".//*[contains(@class, 'result__snippet')]")
snippet = ""
if snippet_nodes:
snippet = _itertext_join(snippet_nodes[0])
_append_web_result(
cls,
items,
seen_urls,
site_host=site_host,
url_text=href,
title_text=title,
snippet_text=snippet,
)
if len(items) >= limit:
break
def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
for match in _DDG_RESULT_ANCHOR_RE.finditer(raw_html):
href = _extract_duckduckgo_target_url(match.group(1))
title_html = match.group(2)
title = _html_fragment_to_text(title_html)
_append_web_result(
cls,
items,
seen_urls,
site_host=site_host,
url_text=href,
title_text=title,
snippet_text="",
)
if len(items) >= limit:
break
return _parse_web_results_with_fallback(
cls,
html_text=html_text,
limit=limit,
lxml_parser=_parse_lxml,
regex_parser=_parse_regex,
fallback_when_empty=True,
)
def parse_yahoo_results(
cls,
*,
html_text: str,
site_host: str,
limit: int,
) -> List[Dict[str, str]]:
"""Parse Yahoo HTML search results into normalized rows."""
def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
for node in doc.xpath("//a[@href]"):
href = _extract_yahoo_target_url(node.get("href"))
title = _itertext_join(node)
_append_web_result(
cls,
items,
seen_urls,
site_host=site_host,
url_text=href,
title_text=title,
snippet_text="",
)
if len(items) >= limit:
break
def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
for match in _GENERIC_ANCHOR_RE.finditer(raw_html):
href = _extract_yahoo_target_url(match.group(1))
title_html = match.group(2)
title = _html_fragment_to_text(title_html)
_append_web_result(
cls,
items,
seen_urls,
site_host=site_host,
url_text=href,
title_text=title,
snippet_text="",
)
if len(items) >= limit:
break
return _parse_web_results_with_fallback(
cls,
html_text=html_text,
limit=limit,
lxml_parser=_parse_lxml,
regex_parser=_parse_regex,
)
def query_yahoo(
cls,
*,
search_query: str,
site_host: str,
limit: int,
session: Any,
deadline: Optional[float] = None,
) -> List[Dict[str, str]]:
"""Fetch results from Yahoo search (robust fallback in bot-protected envs)."""
all_rows: List[Dict[str, str]] = []
seen_urls: set[str] = set()
max_pages = max(1, min((max(1, int(limit or 1)) + 9) // 10, 3))
for page_idx in range(max_pages):
if deadline is not None and time.monotonic() >= deadline:
break
params = {
"p": search_query,
"n": "10",
"b": str((page_idx * 10) + 1),
}
try:
read_timeout = 10.0
if deadline is not None:
remaining = max(0.0, float(deadline - time.monotonic()))
if remaining <= 0.0:
break
read_timeout = max(3.0, min(10.0, remaining))
response = session.get(
"https://search.yahoo.com/search",
params=params,
timeout=(3, read_timeout),
headers={
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/124.0.0.0 Safari/537.36"
),
"Accept-Language": "en-US,en;q=0.9",
},
)
response.raise_for_status()
except Exception:
break
page_rows = parse_yahoo_results(
cls,
html_text=response.text,
site_host=site_host,
limit=max(1, limit - len(all_rows)),
)
new_rows = 0
for row in page_rows:
url_value = str(row.get("url") or "").strip()
if not url_value or url_value in seen_urls:
continue
seen_urls.add(url_value)
all_rows.append(row)
new_rows += 1
if len(all_rows) >= limit:
break
if len(all_rows) >= limit or new_rows == 0:
break
return all_rows[:limit]
def parse_bing_results(
cls,
*,
html_text: str,
site_host: str,
limit: int,
) -> List[Dict[str, str]]:
"""Parse Bing HTML search results into normalized rows."""
def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
result_nodes = doc.xpath("//li[contains(@class, 'b_algo')]")
for node in result_nodes:
links = node.xpath(".//h2/a")
if not links:
continue
link = links[0]
href = str(link.get("href") or "").strip()
title = _itertext_join(link)
snippet = ""
for sel in (
".//*[contains(@class,'b_caption')]//p",
".//*[contains(@class,'b_snippet')]",
".//p",
):
snip_nodes = node.xpath(sel)
if snip_nodes:
snippet = _itertext_join(snip_nodes[0])
break
_append_web_result(
cls,
items,
seen_urls,
site_host=site_host,
url_text=href,
title_text=title,
snippet_text=snippet,
)
if len(items) >= limit:
break
def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
for match in _BING_RESULT_ANCHOR_RE.finditer(raw_html):
href = match.group(1)
title = _html_fragment_to_text(match.group(2))
_append_web_result(
cls,
items,
seen_urls,
site_host=site_host,
url_text=href,
title_text=title,
snippet_text="",
)
if len(items) >= limit:
break
return _parse_web_results_with_fallback(
cls,
html_text=html_text,
limit=limit,
lxml_parser=_parse_lxml,
regex_parser=_parse_regex,
)
def query_bing(
cls,
*,
search_query: str,
site_host: str,
limit: int,
session: Any,
deadline: Optional[float] = None,
) -> List[Dict[str, str]]:
"""Fetch results from Bing (supports filetype: and site: natively)."""
all_rows: List[Dict[str, str]] = []
seen_urls: set[str] = set()
page_start = 1
pages_checked = 0
max_pages = max(1, min((max(1, int(limit or 1)) + 49) // 50, 3))
while len(all_rows) < limit and pages_checked < max_pages:
if deadline is not None and time.monotonic() >= deadline:
break
params = {"q": search_query, "first": str(page_start), "count": "50"}
try:
read_timeout = 10.0
if deadline is not None:
remaining = max(0.0, float(deadline - time.monotonic()))
if remaining <= 0.0:
break
read_timeout = max(3.0, min(10.0, remaining))
response = session.get(
"https://www.bing.com/search",
params=params,
timeout=(3, read_timeout),
headers={
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/124.0.0.0 Safari/537.36"
),
"Accept-Language": "en-US,en;q=0.9",
},
)
response.raise_for_status()
except Exception:
break
page_rows = parse_bing_results(
cls,
html_text=response.text,
site_host=site_host,
limit=max(1, limit - len(all_rows)),
)
new_rows = 0
for row in page_rows:
url_value = str(row.get("url") or "").strip()
if not url_value or url_value in seen_urls:
continue
seen_urls.add(url_value)
all_rows.append(row)
new_rows += 1
if len(all_rows) >= limit:
break
if new_rows == 0 or len(all_rows) >= limit:
break
page_start += 50
pages_checked += 1
return all_rows
def query_web_search(
cls,
*,
search_query: str,
site_host: str,
limit: int,
) -> List[Dict[str, str]]:
"""Execute web search and return parsed result rows.
Uses Yahoo first (works in environments where Bing/DDG HTML endpoints
are challenge-gated), then Bing, then DuckDuckGo.
"""
from API.requests_client import get_requests_session
session = get_requests_session()
normalized_limit = max(1, min(int(limit or 1), 100))
engine_deadline = time.monotonic() + 12.0
all_rows = query_yahoo(
cls,
search_query=search_query,
site_host=site_host,
limit=normalized_limit,
session=session,
deadline=engine_deadline,
)
if all_rows:
return all_rows[:normalized_limit]
all_rows = query_bing(
cls,
search_query=search_query,
site_host=site_host,
limit=normalized_limit,
session=session,
deadline=engine_deadline,
)
if all_rows:
return all_rows[:normalized_limit]
all_rows_ddg: List[Dict[str, str]] = []
seen_urls: set[str] = set()
endpoints = [
"https://html.duckduckgo.com/html/",
"https://duckduckgo.com/html/",
]
for endpoint in endpoints:
if time.monotonic() >= engine_deadline:
break
max_offsets = min(3, max(1, (normalized_limit + 29) // 30))
for page_idx in range(max_offsets):
if time.monotonic() >= engine_deadline:
break
offset = page_idx * 30
params = {"q": search_query, "s": str(offset)}
remaining = max(0.0, float(engine_deadline - time.monotonic()))
if remaining <= 0.0:
break
read_timeout = max(3.0, min(10.0, remaining))
response = session.get(
endpoint,
params=params,
timeout=(3, read_timeout),
headers={"Referer": "https://duckduckgo.com/"},
)
response.raise_for_status()
page_rows = parse_duckduckgo_results(
cls,
html_text=response.text,
site_host=site_host,
limit=max(1, normalized_limit - len(all_rows_ddg)),
)
new_rows = 0
for row in page_rows:
url_value = str(row.get("url") or "").strip()
if not url_value or url_value in seen_urls:
continue
seen_urls.add(url_value)
all_rows_ddg.append(row)
new_rows += 1
if len(all_rows_ddg) >= normalized_limit:
break
if len(all_rows_ddg) >= normalized_limit or new_rows == 0:
break
if all_rows_ddg:
break
return all_rows_ddg[:normalized_limit]
def _is_probable_html_path(path_value: str) -> bool:
"""Return True when URL path likely points to an HTML page."""
path = str(path_value or "").strip()
if not path:
return True
suffix = Path(path).suffix.lower()
if not suffix:
return True
return suffix in {".html", ".htm", ".php", ".asp", ".aspx", ".jsp", ".shtml", ".xhtml"}
def _extract_html_links(cls, *, html_text: str, base_url: str) -> List[str]:
"""Extract absolute links from an HTML document."""
links: List[str] = []
seen: set[str] = set()
def _add_link(raw_href: Any) -> None:
href = str(raw_href or "").strip()
if not href or href.startswith(("#", "javascript:", "mailto:")):
return
try:
absolute = urljoin(base_url, href)
parsed = urlparse(absolute)
except Exception:
return
if str(getattr(parsed, "scheme", "") or "").lower() not in {"http", "https"}:
return
clean = parsed._replace(fragment="").geturl()
if clean in seen:
return
seen.add(clean)
links.append(clean)
try:
from lxml import html as lxml_html
doc = lxml_html.fromstring(html_text or "")
for node in doc.xpath("//a[@href]"):
_add_link(node.get("href"))
except Exception:
href_pattern = re.compile(r'<a[^>]+href=["\']([^"\']+)["\']', flags=re.IGNORECASE)
for match in href_pattern.finditer(html_text or ""):
_add_link(match.group(1))
return links
def crawl_site_for_extension(
cls,
*,
seed_url: str,
site_host: str,
extension: str,
limit: int,
max_duration_seconds: float = 15.0,
) -> List[Dict[str, str]]:
"""Fallback crawler that discovers in-site file links by extension."""
from API.requests_client import get_requests_session
normalized_ext = _normalize_extension(extension)
if not normalized_ext:
return []
start_url = _normalize_seed_url(cls, seed_url, site_host)
if not start_url:
return []
session = get_requests_session()
headers = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/124.0.0.0 Safari/537.36"
),
"Accept-Language": "en-US,en;q=0.9",
}
queue: deque[str] = deque([start_url])
queued: set[str] = {start_url}
visited_pages: set[str] = set()
seen_files: set[str] = set()
rows: List[Dict[str, str]] = []
normalized_limit = max(1, min(int(limit or 1), 100))
max_pages = max(8, min(normalized_limit * 4, 64))
crawl_deadline = time.monotonic() + max(5.0, float(max_duration_seconds or 0.0))
while (
queue
and len(visited_pages) < max_pages
and len(rows) < normalized_limit
and time.monotonic() < crawl_deadline
):
page_url = queue.popleft()
queued.discard(page_url)
if page_url in visited_pages:
continue
visited_pages.add(page_url)
if time.monotonic() >= crawl_deadline:
break
try:
response = session.get(page_url, timeout=(4, 8), headers=headers)
response.raise_for_status()
except Exception:
continue
final_url = str(getattr(response, "url", "") or page_url)
try:
parsed_final = urlparse(final_url)
except Exception:
continue
final_host = _normalize_host(getattr(parsed_final, "hostname", "") or "")
if not _url_matches_site(final_url, site_host):
continue
final_path = str(getattr(parsed_final, "path", "") or "")
direct_ext = _normalize_extension(Path(final_path).suffix)
if direct_ext == normalized_ext:
file_url = parsed_final._replace(fragment="").geturl()
if file_url not in seen_files:
seen_files.add(file_url)
title = Path(unquote(final_path)).name or file_url
rows.append(
{
"url": file_url,
"title": title,
"snippet": "Discovered via in-site crawl",
}
)
continue
content_type = str((response.headers or {}).get("content-type", "") or "").lower()
if "html" not in content_type and "xhtml" not in content_type:
continue
html_text = str(getattr(response, "text", "") or "")
if not html_text:
continue
if len(html_text) > 2_500_000:
continue
discovered_links = _extract_html_links(cls, html_text=html_text, base_url=final_url)
for idx, target in enumerate(discovered_links):
if len(rows) >= normalized_limit:
break
if idx >= 300:
break
if time.monotonic() >= crawl_deadline:
break
try:
parsed_target = urlparse(target)
except Exception:
continue
target_host = _normalize_host(getattr(parsed_target, "hostname", "") or "")
if not target_host or not (target_host == final_host or target_host.endswith(f".{site_host}")):
if not _url_matches_site(target, site_host):
continue
target_clean = parsed_target._replace(fragment="").geturl()
target_path = str(getattr(parsed_target, "path", "") or "")
target_ext = _normalize_extension(Path(target_path).suffix)
if target_ext == normalized_ext:
if target_clean in seen_files:
continue
seen_files.add(target_clean)
title = Path(unquote(target_path)).name or target_clean
rows.append(
{
"url": target_clean,
"title": title,
"snippet": f"Discovered via crawl from {final_path or '/'}",
}
)
continue
if _is_probable_html_path(target_path):
if target_clean not in visited_pages and target_clean not in queued:
queue.append(target_clean)
queued.add(target_clean)
if time.monotonic() >= crawl_deadline:
debug(
"Web crawl fallback reached time budget",
{
"site": site_host,
"visited_pages": len(visited_pages),
"queued_pages": len(queue),
"results": len(rows),
"time_budget_seconds": max_duration_seconds,
},
)
return rows[:normalized_limit]
def _extract_site_host(cls, candidate: Any) -> Optional[str]:
"""Extract a host/domain from URL-like input."""
raw = str(candidate or "").strip().strip('"').strip("'")
if not raw:
return None
if raw.lower().startswith("site:"):
raw = raw.split(":", 1)[1].strip()
parsed = None
try:
parsed = urlparse(raw)
except Exception:
parsed = None
if parsed is None or not getattr(parsed, "hostname", None):
try:
parsed = urlparse(f"https://{raw}")
except Exception:
parsed = None
host = ""
try:
host = str(getattr(parsed, "hostname", "") or "").strip().lower()
except Exception:
host = ""
host = _normalize_host(host)
if not host or "." not in host:
return None
return host
def _normalize_seed_url(cls, seed_value: Any, site_host: str) -> str:
"""Build a safe crawl starting URL from user input and resolved host."""
raw = str(seed_value or "").strip().strip("'\"")
if not raw:
raw = str(site_host or "").strip()
if raw and not raw.startswith(("http://", "https://")):
raw = f"https://{raw}"
try:
parsed = urlparse(raw)
except Exception:
parsed = urlparse("")
target = _normalize_host(site_host)
host = _normalize_host(getattr(parsed, "hostname", "") or "")
if target and host and not (host == target or host.endswith(f".{target}")):
return f"https://{target}/"
scheme = str(getattr(parsed, "scheme", "") or "https").lower()
if scheme not in {"http", "https"}:
scheme = "https"
netloc = str(getattr(parsed, "netloc", "") or "").strip()
if not netloc:
netloc = target
path = str(getattr(parsed, "path", "") or "").strip()
if not path:
path = "/"
return f"{scheme}://{netloc}{path}"
def build_web_search_plan(
cls,
*,
query: str,
positional_args: List[str],
storage_backend: Optional[str],
store_filter: Optional[str],
hash_query: List[str],
) -> Optional[Dict[str, Any]]:
"""Build web-search plan for URL + ext/filetype query syntax.
Example input:
search-file "example.com/foo" -query "ext:pdf"
Produces:
site:example.com filetype:pdf
"""
if storage_backend or store_filter or hash_query:
return None
text = _normalize_space(query)
if not text:
return None
local_markers = ("url:", "hash:", "tag:", "instance:", "system:")
if any(marker in text.lower() for marker in local_markers):
return None
site_host: Optional[str] = None
site_from_positional = False
site_token_to_strip = ""
seed_url = ""
site_match = _SITE_TOKEN_RE.search(text)
if site_match:
site_host = _extract_site_host(cls, site_match.group(1))
seed_url = str(site_match.group(1) or "").strip()
if not site_host and positional_args:
site_host = _extract_site_host(cls, positional_args[0])
site_from_positional = bool(site_host)
if site_from_positional:
site_token_to_strip = str(positional_args[0] or "").strip()
seed_url = site_token_to_strip
if not site_host:
for token in text.split():
candidate = str(token or "").strip().strip(",")
if not candidate:
continue
lower_candidate = candidate.lower()
if lower_candidate.startswith(("ext:", "filetype:", "type:", "site:")):
continue
if _SCHEME_PREFIX_RE.match(lower_candidate) and not lower_candidate.startswith(
("http://", "https://")
):
continue
guessed = _extract_site_host(cls, candidate)
if guessed:
site_host = guessed
site_token_to_strip = candidate
break
if not site_host:
return None
filetype_match = _FILETYPE_TOKEN_RE.search(text)
filetype = _normalize_extension(filetype_match.group(1)) if filetype_match else ""
has_explicit_site = bool(site_match)
if not filetype and not has_explicit_site:
return None
residual = text
residual = _SITE_REMOVE_RE.sub(" ", residual)
residual = _FILETYPE_REMOVE_RE.sub(" ", residual)
if site_from_positional and positional_args:
first = str(positional_args[0] or "").strip()
if first:
residual = re.sub(rf"(?:^|\s){re.escape(first)}(?:\s|$)", " ", residual, count=1)
elif site_token_to_strip:
residual = re.sub(
rf"(?:^|\s){re.escape(site_token_to_strip)}(?:\s|$)",
" ",
residual,
count=1,
)
residual = _normalize_space(residual)
search_terms: List[str] = [f"site:{site_host}"]
if filetype:
search_terms.append(f"filetype:{filetype}")
if residual:
search_terms.append(residual)
search_query = " ".join(search_terms).strip()
if not search_query:
return None
normalized_seed_url = _normalize_seed_url(cls, seed_url, site_host)
return {
"site_host": site_host,
"filetype": filetype,
"search_query": search_query,
"residual": residual,
"seed_url": normalized_seed_url,
}