update refactoring and add new features
This commit is contained in:
@@ -0,0 +1,994 @@
|
||||
"""Search engine result parsing — Bing, DuckDuckGo, Yahoo, and site crawling.
|
||||
|
||||
Extracted from search.py to keep the cmdlet class focused on orchestration.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
import re
|
||||
import time
|
||||
import html as _html
|
||||
from urllib.parse import urlparse, parse_qs, unquote, urljoin
|
||||
|
||||
from SYS.logger import debug
|
||||
from SYS.payload_builders import normalize_file_extension
|
||||
|
||||
_WHITESPACE_RE = re.compile(r"\s+")
|
||||
_SITE_TOKEN_RE = re.compile(r"(?:^|\s)site:([^\s,]+)", flags=re.IGNORECASE)
|
||||
_FILETYPE_TOKEN_RE = re.compile(
|
||||
r"(?:^|\s)(?:ext|filetype|type):\.?([a-z0-9]{1,12})\b",
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
_SITE_REMOVE_RE = re.compile(r"(?:^|\s)site:[^\s,]+", flags=re.IGNORECASE)
|
||||
_FILETYPE_REMOVE_RE = re.compile(
|
||||
r"(?:^|\s)(?:ext|filetype|type):\.?[a-z0-9]{1,12}\b",
|
||||
flags=re.IGNORECASE,
|
||||
)
|
||||
_SCHEME_PREFIX_RE = re.compile(r"^[a-z]+:")
|
||||
_YAHOO_RU_RE = re.compile(r"/RU=([^/]+)/RK=", flags=re.IGNORECASE)
|
||||
_HTML_TAG_RE = re.compile(r"<[^>]+>")
|
||||
_DDG_RESULT_ANCHOR_RE = re.compile(
|
||||
r'<a[^>]+class="[^"]*result__a[^"]*"[^>]+href="([^"]+)"[^>]*>(.*?)</a>',
|
||||
flags=re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_GENERIC_ANCHOR_RE = re.compile(
|
||||
r'<a[^>]+href=["\']([^"\']+)["\'][^>]*>(.*?)</a>',
|
||||
flags=re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
_BING_RESULT_ANCHOR_RE = re.compile(
|
||||
r'<h2[^>]*>\s*<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>',
|
||||
flags=re.IGNORECASE | re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
def _normalize_extension(ext_value: Any) -> str:
|
||||
"""Sanitize extension strings to alphanumerics and cap at 5 chars."""
|
||||
return normalize_file_extension(ext_value)
|
||||
|
||||
|
||||
def _normalize_host(value: Any) -> str:
|
||||
"""Normalize host names for matching/filtering."""
|
||||
host = str(value or "").strip().lower()
|
||||
if host.startswith("www."):
|
||||
host = host[4:]
|
||||
if ":" in host:
|
||||
host = host.split(":", 1)[0]
|
||||
return host
|
||||
|
||||
|
||||
def _normalize_space(text: Any) -> str:
|
||||
return _WHITESPACE_RE.sub(" ", str(text or "")).strip()
|
||||
|
||||
|
||||
def _url_matches_site(cls, url: str, site_host: str) -> bool:
|
||||
"""Return True when URL host is the requested site/subdomain."""
|
||||
try:
|
||||
parsed = urlparse(str(url or ""))
|
||||
host = _normalize_host(getattr(parsed, "hostname", "") or "")
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
target = _normalize_host(site_host)
|
||||
if not host or not target:
|
||||
return False
|
||||
return host == target or host.endswith(f".{target}")
|
||||
|
||||
|
||||
def _itertext_join(node: Any) -> str:
|
||||
try:
|
||||
return " ".join([str(text).strip() for text in node.itertext() if str(text).strip()])
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _html_fragment_to_text(fragment: Any) -> str:
|
||||
text = _HTML_TAG_RE.sub(" ", str(fragment or ""))
|
||||
return _html.unescape(text)
|
||||
|
||||
|
||||
def _append_web_result(
|
||||
cls,
|
||||
items: List[Dict[str, str]],
|
||||
seen_urls: set[str],
|
||||
*,
|
||||
site_host: str,
|
||||
url_text: str,
|
||||
title_text: str,
|
||||
snippet_text: str,
|
||||
) -> None:
|
||||
url_clean = str(url_text or "").strip()
|
||||
if not url_clean or not url_clean.startswith(("http://", "https://")):
|
||||
return
|
||||
if not _url_matches_site(url_clean, site_host):
|
||||
return
|
||||
if url_clean in seen_urls:
|
||||
return
|
||||
|
||||
seen_urls.add(url_clean)
|
||||
items.append(
|
||||
{
|
||||
"url": url_clean,
|
||||
"title": _normalize_space(title_text) or url_clean,
|
||||
"snippet": _normalize_space(snippet_text),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _parse_web_results_with_fallback(
|
||||
cls,
|
||||
*,
|
||||
html_text: str,
|
||||
limit: int,
|
||||
lxml_parser: Any,
|
||||
regex_parser: Any,
|
||||
fallback_when_empty: bool = False,
|
||||
) -> List[Dict[str, str]]:
|
||||
"""Run an lxml-based parser with an optional regex fallback."""
|
||||
items: List[Dict[str, str]] = []
|
||||
seen_urls: set[str] = set()
|
||||
should_run_regex = False
|
||||
|
||||
try:
|
||||
from lxml import html as lxml_html
|
||||
|
||||
doc = lxml_html.fromstring(html_text or "")
|
||||
lxml_parser(doc, items, seen_urls)
|
||||
should_run_regex = fallback_when_empty and not items
|
||||
except Exception:
|
||||
should_run_regex = True
|
||||
|
||||
if should_run_regex:
|
||||
regex_parser(html_text or "", items, seen_urls)
|
||||
|
||||
return items[:limit]
|
||||
|
||||
|
||||
def _extract_duckduckgo_target_url(href: Any) -> str:
|
||||
"""Extract direct target URL from DuckDuckGo result links."""
|
||||
raw_href = str(href or "").strip()
|
||||
if not raw_href:
|
||||
return ""
|
||||
|
||||
if raw_href.startswith("//"):
|
||||
raw_href = f"https:{raw_href}"
|
||||
|
||||
if raw_href.startswith("/"):
|
||||
raw_href = f"https://duckduckgo.com{raw_href}"
|
||||
|
||||
parsed = None
|
||||
try:
|
||||
parsed = urlparse(raw_href)
|
||||
except Exception:
|
||||
parsed = None
|
||||
|
||||
try:
|
||||
host = str(getattr(parsed, "hostname", "") or "").strip().lower()
|
||||
except Exception:
|
||||
host = ""
|
||||
|
||||
if host.endswith("duckduckgo.com"):
|
||||
try:
|
||||
query = parse_qs(str(getattr(parsed, "query", "") or ""))
|
||||
candidate = (query.get("uddg") or [""])[0]
|
||||
if candidate:
|
||||
return str(unquote(candidate)).strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return raw_href
|
||||
|
||||
|
||||
def _extract_yahoo_target_url(href: Any) -> str:
|
||||
"""Extract direct target URL from Yahoo redirect links."""
|
||||
raw_href = str(href or "").strip()
|
||||
if not raw_href:
|
||||
return ""
|
||||
|
||||
ru_match = _YAHOO_RU_RE.search(raw_href)
|
||||
if ru_match:
|
||||
try:
|
||||
return str(unquote(ru_match.group(1))).strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
parsed = urlparse(raw_href)
|
||||
query = parse_qs(str(getattr(parsed, "query", "") or ""))
|
||||
candidate = (query.get("RU") or query.get("ru") or [""])[0]
|
||||
if candidate:
|
||||
return str(unquote(candidate)).strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return raw_href
|
||||
|
||||
|
||||
def parse_duckduckgo_results(
|
||||
cls,
|
||||
*,
|
||||
html_text: str,
|
||||
site_host: str,
|
||||
limit: int,
|
||||
) -> List[Dict[str, str]]:
|
||||
"""Parse DuckDuckGo HTML results into normalized rows."""
|
||||
def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
|
||||
result_nodes = doc.xpath("//div[contains(@class, 'result')]")
|
||||
|
||||
for node in result_nodes:
|
||||
links = node.xpath(".//a[contains(@class, 'result__a')]")
|
||||
if not links:
|
||||
continue
|
||||
|
||||
link = links[0]
|
||||
href = _extract_duckduckgo_target_url(link.get("href"))
|
||||
title = _itertext_join(link)
|
||||
|
||||
snippet_nodes = node.xpath(".//*[contains(@class, 'result__snippet')]")
|
||||
snippet = ""
|
||||
if snippet_nodes:
|
||||
snippet = _itertext_join(snippet_nodes[0])
|
||||
|
||||
_append_web_result(
|
||||
cls,
|
||||
items,
|
||||
seen_urls,
|
||||
site_host=site_host,
|
||||
url_text=href,
|
||||
title_text=title,
|
||||
snippet_text=snippet,
|
||||
)
|
||||
if len(items) >= limit:
|
||||
break
|
||||
|
||||
def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
|
||||
for match in _DDG_RESULT_ANCHOR_RE.finditer(raw_html):
|
||||
href = _extract_duckduckgo_target_url(match.group(1))
|
||||
title_html = match.group(2)
|
||||
title = _html_fragment_to_text(title_html)
|
||||
_append_web_result(
|
||||
cls,
|
||||
items,
|
||||
seen_urls,
|
||||
site_host=site_host,
|
||||
url_text=href,
|
||||
title_text=title,
|
||||
snippet_text="",
|
||||
)
|
||||
if len(items) >= limit:
|
||||
break
|
||||
|
||||
return _parse_web_results_with_fallback(
|
||||
cls,
|
||||
html_text=html_text,
|
||||
limit=limit,
|
||||
lxml_parser=_parse_lxml,
|
||||
regex_parser=_parse_regex,
|
||||
fallback_when_empty=True,
|
||||
)
|
||||
|
||||
|
||||
def parse_yahoo_results(
|
||||
cls,
|
||||
*,
|
||||
html_text: str,
|
||||
site_host: str,
|
||||
limit: int,
|
||||
) -> List[Dict[str, str]]:
|
||||
"""Parse Yahoo HTML search results into normalized rows."""
|
||||
def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
|
||||
for node in doc.xpath("//a[@href]"):
|
||||
href = _extract_yahoo_target_url(node.get("href"))
|
||||
title = _itertext_join(node)
|
||||
_append_web_result(
|
||||
cls,
|
||||
items,
|
||||
seen_urls,
|
||||
site_host=site_host,
|
||||
url_text=href,
|
||||
title_text=title,
|
||||
snippet_text="",
|
||||
)
|
||||
if len(items) >= limit:
|
||||
break
|
||||
|
||||
def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
|
||||
for match in _GENERIC_ANCHOR_RE.finditer(raw_html):
|
||||
href = _extract_yahoo_target_url(match.group(1))
|
||||
title_html = match.group(2)
|
||||
title = _html_fragment_to_text(title_html)
|
||||
_append_web_result(
|
||||
cls,
|
||||
items,
|
||||
seen_urls,
|
||||
site_host=site_host,
|
||||
url_text=href,
|
||||
title_text=title,
|
||||
snippet_text="",
|
||||
)
|
||||
if len(items) >= limit:
|
||||
break
|
||||
|
||||
return _parse_web_results_with_fallback(
|
||||
cls,
|
||||
html_text=html_text,
|
||||
limit=limit,
|
||||
lxml_parser=_parse_lxml,
|
||||
regex_parser=_parse_regex,
|
||||
)
|
||||
|
||||
|
||||
def query_yahoo(
|
||||
cls,
|
||||
*,
|
||||
search_query: str,
|
||||
site_host: str,
|
||||
limit: int,
|
||||
session: Any,
|
||||
deadline: Optional[float] = None,
|
||||
) -> List[Dict[str, str]]:
|
||||
"""Fetch results from Yahoo search (robust fallback in bot-protected envs)."""
|
||||
all_rows: List[Dict[str, str]] = []
|
||||
seen_urls: set[str] = set()
|
||||
|
||||
max_pages = max(1, min((max(1, int(limit or 1)) + 9) // 10, 3))
|
||||
for page_idx in range(max_pages):
|
||||
if deadline is not None and time.monotonic() >= deadline:
|
||||
break
|
||||
|
||||
params = {
|
||||
"p": search_query,
|
||||
"n": "10",
|
||||
"b": str((page_idx * 10) + 1),
|
||||
}
|
||||
try:
|
||||
read_timeout = 10.0
|
||||
if deadline is not None:
|
||||
remaining = max(0.0, float(deadline - time.monotonic()))
|
||||
if remaining <= 0.0:
|
||||
break
|
||||
read_timeout = max(3.0, min(10.0, remaining))
|
||||
|
||||
response = session.get(
|
||||
"https://search.yahoo.com/search",
|
||||
params=params,
|
||||
timeout=(3, read_timeout),
|
||||
headers={
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
except Exception:
|
||||
break
|
||||
|
||||
page_rows = parse_yahoo_results(
|
||||
cls,
|
||||
html_text=response.text,
|
||||
site_host=site_host,
|
||||
limit=max(1, limit - len(all_rows)),
|
||||
)
|
||||
new_rows = 0
|
||||
for row in page_rows:
|
||||
url_value = str(row.get("url") or "").strip()
|
||||
if not url_value or url_value in seen_urls:
|
||||
continue
|
||||
seen_urls.add(url_value)
|
||||
all_rows.append(row)
|
||||
new_rows += 1
|
||||
if len(all_rows) >= limit:
|
||||
break
|
||||
|
||||
if len(all_rows) >= limit or new_rows == 0:
|
||||
break
|
||||
|
||||
return all_rows[:limit]
|
||||
|
||||
|
||||
def parse_bing_results(
|
||||
cls,
|
||||
*,
|
||||
html_text: str,
|
||||
site_host: str,
|
||||
limit: int,
|
||||
) -> List[Dict[str, str]]:
|
||||
"""Parse Bing HTML search results into normalized rows."""
|
||||
def _parse_lxml(doc: Any, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
|
||||
result_nodes = doc.xpath("//li[contains(@class, 'b_algo')]")
|
||||
|
||||
for node in result_nodes:
|
||||
links = node.xpath(".//h2/a")
|
||||
if not links:
|
||||
continue
|
||||
link = links[0]
|
||||
href = str(link.get("href") or "").strip()
|
||||
title = _itertext_join(link)
|
||||
|
||||
snippet = ""
|
||||
for sel in (
|
||||
".//*[contains(@class,'b_caption')]//p",
|
||||
".//*[contains(@class,'b_snippet')]",
|
||||
".//p",
|
||||
):
|
||||
snip_nodes = node.xpath(sel)
|
||||
if snip_nodes:
|
||||
snippet = _itertext_join(snip_nodes[0])
|
||||
break
|
||||
|
||||
_append_web_result(
|
||||
cls,
|
||||
items,
|
||||
seen_urls,
|
||||
site_host=site_host,
|
||||
url_text=href,
|
||||
title_text=title,
|
||||
snippet_text=snippet,
|
||||
)
|
||||
if len(items) >= limit:
|
||||
break
|
||||
|
||||
def _parse_regex(raw_html: str, items: List[Dict[str, str]], seen_urls: set[str]) -> None:
|
||||
for match in _BING_RESULT_ANCHOR_RE.finditer(raw_html):
|
||||
href = match.group(1)
|
||||
title = _html_fragment_to_text(match.group(2))
|
||||
_append_web_result(
|
||||
cls,
|
||||
items,
|
||||
seen_urls,
|
||||
site_host=site_host,
|
||||
url_text=href,
|
||||
title_text=title,
|
||||
snippet_text="",
|
||||
)
|
||||
if len(items) >= limit:
|
||||
break
|
||||
|
||||
return _parse_web_results_with_fallback(
|
||||
cls,
|
||||
html_text=html_text,
|
||||
limit=limit,
|
||||
lxml_parser=_parse_lxml,
|
||||
regex_parser=_parse_regex,
|
||||
)
|
||||
|
||||
|
||||
def query_bing(
|
||||
cls,
|
||||
*,
|
||||
search_query: str,
|
||||
site_host: str,
|
||||
limit: int,
|
||||
session: Any,
|
||||
deadline: Optional[float] = None,
|
||||
) -> List[Dict[str, str]]:
|
||||
"""Fetch results from Bing (supports filetype: and site: natively)."""
|
||||
all_rows: List[Dict[str, str]] = []
|
||||
seen_urls: set[str] = set()
|
||||
|
||||
page_start = 1
|
||||
pages_checked = 0
|
||||
max_pages = max(1, min((max(1, int(limit or 1)) + 49) // 50, 3))
|
||||
while len(all_rows) < limit and pages_checked < max_pages:
|
||||
if deadline is not None and time.monotonic() >= deadline:
|
||||
break
|
||||
|
||||
params = {"q": search_query, "first": str(page_start), "count": "50"}
|
||||
try:
|
||||
read_timeout = 10.0
|
||||
if deadline is not None:
|
||||
remaining = max(0.0, float(deadline - time.monotonic()))
|
||||
if remaining <= 0.0:
|
||||
break
|
||||
read_timeout = max(3.0, min(10.0, remaining))
|
||||
|
||||
response = session.get(
|
||||
"https://www.bing.com/search",
|
||||
params=params,
|
||||
timeout=(3, read_timeout),
|
||||
headers={
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
except Exception:
|
||||
break
|
||||
|
||||
page_rows = parse_bing_results(
|
||||
cls,
|
||||
html_text=response.text,
|
||||
site_host=site_host,
|
||||
limit=max(1, limit - len(all_rows)),
|
||||
)
|
||||
new_rows = 0
|
||||
for row in page_rows:
|
||||
url_value = str(row.get("url") or "").strip()
|
||||
if not url_value or url_value in seen_urls:
|
||||
continue
|
||||
seen_urls.add(url_value)
|
||||
all_rows.append(row)
|
||||
new_rows += 1
|
||||
if len(all_rows) >= limit:
|
||||
break
|
||||
|
||||
if new_rows == 0 or len(all_rows) >= limit:
|
||||
break
|
||||
page_start += 50
|
||||
pages_checked += 1
|
||||
|
||||
return all_rows
|
||||
|
||||
|
||||
def query_web_search(
|
||||
cls,
|
||||
*,
|
||||
search_query: str,
|
||||
site_host: str,
|
||||
limit: int,
|
||||
) -> List[Dict[str, str]]:
|
||||
"""Execute web search and return parsed result rows.
|
||||
|
||||
Uses Yahoo first (works in environments where Bing/DDG HTML endpoints
|
||||
are challenge-gated), then Bing, then DuckDuckGo.
|
||||
"""
|
||||
from API.requests_client import get_requests_session
|
||||
|
||||
session = get_requests_session()
|
||||
normalized_limit = max(1, min(int(limit or 1), 100))
|
||||
engine_deadline = time.monotonic() + 12.0
|
||||
|
||||
all_rows = query_yahoo(
|
||||
cls,
|
||||
search_query=search_query,
|
||||
site_host=site_host,
|
||||
limit=normalized_limit,
|
||||
session=session,
|
||||
deadline=engine_deadline,
|
||||
)
|
||||
if all_rows:
|
||||
return all_rows[:normalized_limit]
|
||||
|
||||
all_rows = query_bing(
|
||||
cls,
|
||||
search_query=search_query,
|
||||
site_host=site_host,
|
||||
limit=normalized_limit,
|
||||
session=session,
|
||||
deadline=engine_deadline,
|
||||
)
|
||||
if all_rows:
|
||||
return all_rows[:normalized_limit]
|
||||
|
||||
all_rows_ddg: List[Dict[str, str]] = []
|
||||
seen_urls: set[str] = set()
|
||||
endpoints = [
|
||||
"https://html.duckduckgo.com/html/",
|
||||
"https://duckduckgo.com/html/",
|
||||
]
|
||||
for endpoint in endpoints:
|
||||
if time.monotonic() >= engine_deadline:
|
||||
break
|
||||
max_offsets = min(3, max(1, (normalized_limit + 29) // 30))
|
||||
for page_idx in range(max_offsets):
|
||||
if time.monotonic() >= engine_deadline:
|
||||
break
|
||||
offset = page_idx * 30
|
||||
params = {"q": search_query, "s": str(offset)}
|
||||
remaining = max(0.0, float(engine_deadline - time.monotonic()))
|
||||
if remaining <= 0.0:
|
||||
break
|
||||
read_timeout = max(3.0, min(10.0, remaining))
|
||||
response = session.get(
|
||||
endpoint,
|
||||
params=params,
|
||||
timeout=(3, read_timeout),
|
||||
headers={"Referer": "https://duckduckgo.com/"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
page_rows = parse_duckduckgo_results(
|
||||
cls,
|
||||
html_text=response.text,
|
||||
site_host=site_host,
|
||||
limit=max(1, normalized_limit - len(all_rows_ddg)),
|
||||
)
|
||||
new_rows = 0
|
||||
for row in page_rows:
|
||||
url_value = str(row.get("url") or "").strip()
|
||||
if not url_value or url_value in seen_urls:
|
||||
continue
|
||||
seen_urls.add(url_value)
|
||||
all_rows_ddg.append(row)
|
||||
new_rows += 1
|
||||
if len(all_rows_ddg) >= normalized_limit:
|
||||
break
|
||||
if len(all_rows_ddg) >= normalized_limit or new_rows == 0:
|
||||
break
|
||||
if all_rows_ddg:
|
||||
break
|
||||
|
||||
return all_rows_ddg[:normalized_limit]
|
||||
|
||||
|
||||
def _is_probable_html_path(path_value: str) -> bool:
|
||||
"""Return True when URL path likely points to an HTML page."""
|
||||
path = str(path_value or "").strip()
|
||||
if not path:
|
||||
return True
|
||||
suffix = Path(path).suffix.lower()
|
||||
if not suffix:
|
||||
return True
|
||||
return suffix in {".html", ".htm", ".php", ".asp", ".aspx", ".jsp", ".shtml", ".xhtml"}
|
||||
|
||||
|
||||
def _extract_html_links(cls, *, html_text: str, base_url: str) -> List[str]:
|
||||
"""Extract absolute links from an HTML document."""
|
||||
links: List[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def _add_link(raw_href: Any) -> None:
|
||||
href = str(raw_href or "").strip()
|
||||
if not href or href.startswith(("#", "javascript:", "mailto:")):
|
||||
return
|
||||
try:
|
||||
absolute = urljoin(base_url, href)
|
||||
parsed = urlparse(absolute)
|
||||
except Exception:
|
||||
return
|
||||
if str(getattr(parsed, "scheme", "") or "").lower() not in {"http", "https"}:
|
||||
return
|
||||
clean = parsed._replace(fragment="").geturl()
|
||||
if clean in seen:
|
||||
return
|
||||
seen.add(clean)
|
||||
links.append(clean)
|
||||
|
||||
try:
|
||||
from lxml import html as lxml_html
|
||||
|
||||
doc = lxml_html.fromstring(html_text or "")
|
||||
for node in doc.xpath("//a[@href]"):
|
||||
_add_link(node.get("href"))
|
||||
except Exception:
|
||||
href_pattern = re.compile(r'<a[^>]+href=["\']([^"\']+)["\']', flags=re.IGNORECASE)
|
||||
for match in href_pattern.finditer(html_text or ""):
|
||||
_add_link(match.group(1))
|
||||
|
||||
return links
|
||||
|
||||
|
||||
def crawl_site_for_extension(
|
||||
cls,
|
||||
*,
|
||||
seed_url: str,
|
||||
site_host: str,
|
||||
extension: str,
|
||||
limit: int,
|
||||
max_duration_seconds: float = 15.0,
|
||||
) -> List[Dict[str, str]]:
|
||||
"""Fallback crawler that discovers in-site file links by extension."""
|
||||
from API.requests_client import get_requests_session
|
||||
|
||||
normalized_ext = _normalize_extension(extension)
|
||||
if not normalized_ext:
|
||||
return []
|
||||
|
||||
start_url = _normalize_seed_url(cls, seed_url, site_host)
|
||||
if not start_url:
|
||||
return []
|
||||
|
||||
session = get_requests_session()
|
||||
headers = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
}
|
||||
|
||||
queue: deque[str] = deque([start_url])
|
||||
queued: set[str] = {start_url}
|
||||
visited_pages: set[str] = set()
|
||||
seen_files: set[str] = set()
|
||||
rows: List[Dict[str, str]] = []
|
||||
normalized_limit = max(1, min(int(limit or 1), 100))
|
||||
max_pages = max(8, min(normalized_limit * 4, 64))
|
||||
crawl_deadline = time.monotonic() + max(5.0, float(max_duration_seconds or 0.0))
|
||||
|
||||
while (
|
||||
queue
|
||||
and len(visited_pages) < max_pages
|
||||
and len(rows) < normalized_limit
|
||||
and time.monotonic() < crawl_deadline
|
||||
):
|
||||
page_url = queue.popleft()
|
||||
queued.discard(page_url)
|
||||
if page_url in visited_pages:
|
||||
continue
|
||||
visited_pages.add(page_url)
|
||||
|
||||
if time.monotonic() >= crawl_deadline:
|
||||
break
|
||||
|
||||
try:
|
||||
response = session.get(page_url, timeout=(4, 8), headers=headers)
|
||||
response.raise_for_status()
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
final_url = str(getattr(response, "url", "") or page_url)
|
||||
try:
|
||||
parsed_final = urlparse(final_url)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
final_host = _normalize_host(getattr(parsed_final, "hostname", "") or "")
|
||||
if not _url_matches_site(final_url, site_host):
|
||||
continue
|
||||
|
||||
final_path = str(getattr(parsed_final, "path", "") or "")
|
||||
direct_ext = _normalize_extension(Path(final_path).suffix)
|
||||
if direct_ext == normalized_ext:
|
||||
file_url = parsed_final._replace(fragment="").geturl()
|
||||
if file_url not in seen_files:
|
||||
seen_files.add(file_url)
|
||||
title = Path(unquote(final_path)).name or file_url
|
||||
rows.append(
|
||||
{
|
||||
"url": file_url,
|
||||
"title": title,
|
||||
"snippet": "Discovered via in-site crawl",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
content_type = str((response.headers or {}).get("content-type", "") or "").lower()
|
||||
if "html" not in content_type and "xhtml" not in content_type:
|
||||
continue
|
||||
|
||||
html_text = str(getattr(response, "text", "") or "")
|
||||
if not html_text:
|
||||
continue
|
||||
if len(html_text) > 2_500_000:
|
||||
continue
|
||||
|
||||
discovered_links = _extract_html_links(cls, html_text=html_text, base_url=final_url)
|
||||
for idx, target in enumerate(discovered_links):
|
||||
if len(rows) >= normalized_limit:
|
||||
break
|
||||
if idx >= 300:
|
||||
break
|
||||
if time.monotonic() >= crawl_deadline:
|
||||
break
|
||||
try:
|
||||
parsed_target = urlparse(target)
|
||||
except Exception:
|
||||
continue
|
||||
target_host = _normalize_host(getattr(parsed_target, "hostname", "") or "")
|
||||
if not target_host or not (target_host == final_host or target_host.endswith(f".{site_host}")):
|
||||
if not _url_matches_site(target, site_host):
|
||||
continue
|
||||
|
||||
target_clean = parsed_target._replace(fragment="").geturl()
|
||||
target_path = str(getattr(parsed_target, "path", "") or "")
|
||||
target_ext = _normalize_extension(Path(target_path).suffix)
|
||||
|
||||
if target_ext == normalized_ext:
|
||||
if target_clean in seen_files:
|
||||
continue
|
||||
seen_files.add(target_clean)
|
||||
title = Path(unquote(target_path)).name or target_clean
|
||||
rows.append(
|
||||
{
|
||||
"url": target_clean,
|
||||
"title": title,
|
||||
"snippet": f"Discovered via crawl from {final_path or '/'}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
if _is_probable_html_path(target_path):
|
||||
if target_clean not in visited_pages and target_clean not in queued:
|
||||
queue.append(target_clean)
|
||||
queued.add(target_clean)
|
||||
|
||||
if time.monotonic() >= crawl_deadline:
|
||||
debug(
|
||||
"Web crawl fallback reached time budget",
|
||||
{
|
||||
"site": site_host,
|
||||
"visited_pages": len(visited_pages),
|
||||
"queued_pages": len(queue),
|
||||
"results": len(rows),
|
||||
"time_budget_seconds": max_duration_seconds,
|
||||
},
|
||||
)
|
||||
|
||||
return rows[:normalized_limit]
|
||||
|
||||
|
||||
def _extract_site_host(cls, candidate: Any) -> Optional[str]:
|
||||
"""Extract a host/domain from URL-like input."""
|
||||
raw = str(candidate or "").strip().strip('"').strip("'")
|
||||
if not raw:
|
||||
return None
|
||||
|
||||
if raw.lower().startswith("site:"):
|
||||
raw = raw.split(":", 1)[1].strip()
|
||||
|
||||
parsed = None
|
||||
try:
|
||||
parsed = urlparse(raw)
|
||||
except Exception:
|
||||
parsed = None
|
||||
|
||||
if parsed is None or not getattr(parsed, "hostname", None):
|
||||
try:
|
||||
parsed = urlparse(f"https://{raw}")
|
||||
except Exception:
|
||||
parsed = None
|
||||
|
||||
host = ""
|
||||
try:
|
||||
host = str(getattr(parsed, "hostname", "") or "").strip().lower()
|
||||
except Exception:
|
||||
host = ""
|
||||
|
||||
host = _normalize_host(host)
|
||||
if not host or "." not in host:
|
||||
return None
|
||||
return host
|
||||
|
||||
|
||||
def _normalize_seed_url(cls, seed_value: Any, site_host: str) -> str:
|
||||
"""Build a safe crawl starting URL from user input and resolved host."""
|
||||
raw = str(seed_value or "").strip().strip("'\"")
|
||||
if not raw:
|
||||
raw = str(site_host or "").strip()
|
||||
|
||||
if raw and not raw.startswith(("http://", "https://")):
|
||||
raw = f"https://{raw}"
|
||||
|
||||
try:
|
||||
parsed = urlparse(raw)
|
||||
except Exception:
|
||||
parsed = urlparse("")
|
||||
|
||||
target = _normalize_host(site_host)
|
||||
host = _normalize_host(getattr(parsed, "hostname", "") or "")
|
||||
if target and host and not (host == target or host.endswith(f".{target}")):
|
||||
return f"https://{target}/"
|
||||
|
||||
scheme = str(getattr(parsed, "scheme", "") or "https").lower()
|
||||
if scheme not in {"http", "https"}:
|
||||
scheme = "https"
|
||||
|
||||
netloc = str(getattr(parsed, "netloc", "") or "").strip()
|
||||
if not netloc:
|
||||
netloc = target
|
||||
path = str(getattr(parsed, "path", "") or "").strip()
|
||||
if not path:
|
||||
path = "/"
|
||||
|
||||
return f"{scheme}://{netloc}{path}"
|
||||
|
||||
|
||||
def build_web_search_plan(
|
||||
cls,
|
||||
*,
|
||||
query: str,
|
||||
positional_args: List[str],
|
||||
storage_backend: Optional[str],
|
||||
store_filter: Optional[str],
|
||||
hash_query: List[str],
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Build web-search plan for URL + ext/filetype query syntax.
|
||||
|
||||
Example input:
|
||||
search-file "example.com/foo" -query "ext:pdf"
|
||||
Produces:
|
||||
site:example.com filetype:pdf
|
||||
"""
|
||||
if storage_backend or store_filter or hash_query:
|
||||
return None
|
||||
|
||||
text = _normalize_space(query)
|
||||
if not text:
|
||||
return None
|
||||
|
||||
local_markers = ("url:", "hash:", "tag:", "instance:", "system:")
|
||||
if any(marker in text.lower() for marker in local_markers):
|
||||
return None
|
||||
|
||||
site_host: Optional[str] = None
|
||||
site_from_positional = False
|
||||
site_token_to_strip = ""
|
||||
seed_url = ""
|
||||
|
||||
site_match = _SITE_TOKEN_RE.search(text)
|
||||
if site_match:
|
||||
site_host = _extract_site_host(cls, site_match.group(1))
|
||||
seed_url = str(site_match.group(1) or "").strip()
|
||||
|
||||
if not site_host and positional_args:
|
||||
site_host = _extract_site_host(cls, positional_args[0])
|
||||
site_from_positional = bool(site_host)
|
||||
if site_from_positional:
|
||||
site_token_to_strip = str(positional_args[0] or "").strip()
|
||||
seed_url = site_token_to_strip
|
||||
|
||||
if not site_host:
|
||||
for token in text.split():
|
||||
candidate = str(token or "").strip().strip(",")
|
||||
if not candidate:
|
||||
continue
|
||||
lower_candidate = candidate.lower()
|
||||
if lower_candidate.startswith(("ext:", "filetype:", "type:", "site:")):
|
||||
continue
|
||||
if _SCHEME_PREFIX_RE.match(lower_candidate) and not lower_candidate.startswith(
|
||||
("http://", "https://")
|
||||
):
|
||||
continue
|
||||
guessed = _extract_site_host(cls, candidate)
|
||||
if guessed:
|
||||
site_host = guessed
|
||||
site_token_to_strip = candidate
|
||||
break
|
||||
|
||||
if not site_host:
|
||||
return None
|
||||
|
||||
filetype_match = _FILETYPE_TOKEN_RE.search(text)
|
||||
filetype = _normalize_extension(filetype_match.group(1)) if filetype_match else ""
|
||||
|
||||
has_explicit_site = bool(site_match)
|
||||
if not filetype and not has_explicit_site:
|
||||
return None
|
||||
|
||||
residual = text
|
||||
residual = _SITE_REMOVE_RE.sub(" ", residual)
|
||||
residual = _FILETYPE_REMOVE_RE.sub(" ", residual)
|
||||
|
||||
if site_from_positional and positional_args:
|
||||
first = str(positional_args[0] or "").strip()
|
||||
if first:
|
||||
residual = re.sub(rf"(?:^|\s){re.escape(first)}(?:\s|$)", " ", residual, count=1)
|
||||
elif site_token_to_strip:
|
||||
residual = re.sub(
|
||||
rf"(?:^|\s){re.escape(site_token_to_strip)}(?:\s|$)",
|
||||
" ",
|
||||
residual,
|
||||
count=1,
|
||||
)
|
||||
|
||||
residual = _normalize_space(residual)
|
||||
|
||||
search_terms: List[str] = [f"site:{site_host}"]
|
||||
if filetype:
|
||||
search_terms.append(f"filetype:{filetype}")
|
||||
if residual:
|
||||
search_terms.append(residual)
|
||||
|
||||
search_query = " ".join(search_terms).strip()
|
||||
if not search_query:
|
||||
return None
|
||||
|
||||
normalized_seed_url = _normalize_seed_url(cls, seed_url, site_host)
|
||||
|
||||
return {
|
||||
"site_host": site_host,
|
||||
"filetype": filetype,
|
||||
"search_query": search_query,
|
||||
"residual": residual,
|
||||
"seed_url": normalized_seed_url,
|
||||
}
|
||||
Reference in New Issue
Block a user