refactor(download): remove ProviderCore/download.py, move sanitize_filename to SYS.utils, replace callers to use API.HTTP.HTTPClient
This commit is contained in:
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
@@ -46,9 +48,51 @@ class SearchResult:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
selection_args = getattr(self, "selection_args", None)
|
||||
except Exception:
|
||||
selection_args = None
|
||||
if selection_args is None:
|
||||
try:
|
||||
fm = getattr(self, "full_metadata", None)
|
||||
if isinstance(fm, dict):
|
||||
selection_args = fm.get("_selection_args") or fm.get("selection_args")
|
||||
except Exception:
|
||||
selection_args = None
|
||||
if selection_args:
|
||||
out["_selection_args"] = selection_args
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def parse_inline_query_arguments(raw_query: str) -> Tuple[str, Dict[str, str]]:
|
||||
"""Extract inline key:value arguments from a provider search query."""
|
||||
|
||||
query_text = str(raw_query or "").strip()
|
||||
if not query_text:
|
||||
return "", {}
|
||||
|
||||
tokens = re.split(r"[,\s]+", query_text)
|
||||
leftover: List[str] = []
|
||||
parsed_args: Dict[str, str] = {}
|
||||
|
||||
for token in tokens:
|
||||
if not token:
|
||||
continue
|
||||
sep_index = token.find(":")
|
||||
if sep_index < 0:
|
||||
sep_index = token.find("=")
|
||||
if sep_index > 0:
|
||||
key = token[:sep_index].strip().lower()
|
||||
value = token[sep_index + 1 :].strip()
|
||||
if key and value:
|
||||
parsed_args[key] = value
|
||||
continue
|
||||
leftover.append(token)
|
||||
|
||||
return " ".join(leftover).strip(), parsed_args
|
||||
|
||||
|
||||
class Provider(ABC):
|
||||
"""Unified provider base class.
|
||||
|
||||
@@ -97,6 +141,12 @@ class Provider(ABC):
|
||||
return []
|
||||
return out
|
||||
|
||||
def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]:
|
||||
"""Allow providers to normalize query text and parse inline arguments."""
|
||||
|
||||
normalized = str(query or "").strip()
|
||||
return normalized, {}
|
||||
|
||||
# Standard lifecycle/auth hook.
|
||||
def login(self, **_kwargs: Any) -> bool:
|
||||
return True
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional
|
||||
import sys
|
||||
|
||||
import requests
|
||||
|
||||
from SYS.models import ProgressBar
|
||||
|
||||
|
||||
def sanitize_filename(name: str, *, max_len: int = 150) -> str:
|
||||
text = str(name or "").strip()
|
||||
if not text:
|
||||
return "download"
|
||||
|
||||
forbidden = set('<>:"/\\|?*')
|
||||
cleaned = "".join("_" if c in forbidden else c for c in text)
|
||||
cleaned = " ".join(cleaned.split()).strip().strip(".")
|
||||
if not cleaned:
|
||||
cleaned = "download"
|
||||
return cleaned[:max_len]
|
||||
|
||||
|
||||
def download_file(
|
||||
url: str,
|
||||
output_path: Path,
|
||||
*,
|
||||
session: Optional[requests.Session] = None,
|
||||
timeout_s: float = 30.0,
|
||||
progress_callback: Optional[Callable[[int,
|
||||
Optional[int],
|
||||
str],
|
||||
None]] = None,
|
||||
) -> bool:
|
||||
output_path = Path(output_path)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
s = session or requests.Session()
|
||||
|
||||
bar = ProgressBar() if progress_callback is None else None
|
||||
downloaded = 0
|
||||
total = None
|
||||
|
||||
try:
|
||||
with s.get(url, stream=True, timeout=timeout_s) as resp:
|
||||
resp.raise_for_status()
|
||||
try:
|
||||
total_val = int(resp.headers.get("content-length") or 0)
|
||||
total = total_val if total_val > 0 else None
|
||||
except Exception:
|
||||
total = None
|
||||
|
||||
label = str(output_path.name or "download")
|
||||
|
||||
# Render once immediately so fast downloads still show something.
|
||||
try:
|
||||
if progress_callback is not None:
|
||||
progress_callback(0, total, label)
|
||||
elif bar is not None:
|
||||
bar.update(downloaded=0, total=total, label=label, file=sys.stderr)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
with open(output_path, "wb") as f:
|
||||
for chunk in resp.iter_content(chunk_size=1024 * 256):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
try:
|
||||
if progress_callback is not None:
|
||||
progress_callback(downloaded, total, label)
|
||||
elif bar is not None:
|
||||
bar.update(
|
||||
downloaded=downloaded,
|
||||
total=total,
|
||||
label=label,
|
||||
file=sys.stderr
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
if bar is not None:
|
||||
bar.finish()
|
||||
except Exception:
|
||||
pass
|
||||
return output_path.exists() and output_path.stat().st_size > 0
|
||||
except Exception:
|
||||
try:
|
||||
if bar is not None:
|
||||
bar.finish()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if output_path.exists():
|
||||
output_path.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
127
ProviderCore/inline_utils.py
Normal file
127
ProviderCore/inline_utils.py
Normal file
@@ -0,0 +1,127 @@
|
||||
"""Inline query helpers for providers (choice normalization and filter resolution)."""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
def _normalize_choice(entry: Any) -> Optional[Dict[str, Any]]:
|
||||
if entry is None:
|
||||
return None
|
||||
if isinstance(entry, dict):
|
||||
value = entry.get("value")
|
||||
text = entry.get("text") or entry.get("label") or value
|
||||
aliases = entry.get("alias") or entry.get("aliases") or []
|
||||
value_str = str(value) if value is not None else (str(text) if text is not None else None)
|
||||
text_str = str(text) if text is not None else value_str
|
||||
if not value_str or not text_str:
|
||||
return None
|
||||
alias_list = [str(a) for a in aliases if a is not None]
|
||||
return {"value": value_str, "text": text_str, "aliases": alias_list}
|
||||
return {"value": str(entry), "text": str(entry), "aliases": []}
|
||||
|
||||
|
||||
def collect_choice(provider: Any) -> Dict[str, List[Dict[str, Any]]]:
|
||||
"""Collect normalized inline/query argument choice entries from a provider.
|
||||
|
||||
Supports QUERY_ARG_CHOICES, INLINE_QUERY_FIELD_CHOICES, and the
|
||||
helper methods valued by Providers (`query_field_choices` /
|
||||
`inline_query_field_choices`). Each choice is normalized to {value,text,aliases}.
|
||||
"""
|
||||
|
||||
mapping: Dict[str, List[Dict[str, Any]]] = {}
|
||||
|
||||
def _ingest(source: Any, target_key: str) -> None:
|
||||
normalized: List[Dict[str, Any]] = []
|
||||
seq = source
|
||||
try:
|
||||
if callable(seq):
|
||||
seq = seq()
|
||||
except Exception:
|
||||
seq = source
|
||||
if isinstance(seq, dict):
|
||||
seq = seq.get("choices") or seq.get("values") or seq
|
||||
if isinstance(seq, (list, tuple, set)):
|
||||
for entry in seq:
|
||||
n = _normalize_choice(entry)
|
||||
if n:
|
||||
normalized.append(n)
|
||||
if normalized:
|
||||
mapping[target_key] = normalized
|
||||
|
||||
base = getattr(provider, "QUERY_ARG_CHOICES", None)
|
||||
if isinstance(base, dict):
|
||||
for k, v in base.items():
|
||||
key_norm = str(k).strip().lower()
|
||||
if not key_norm:
|
||||
continue
|
||||
_ingest(v, key_norm)
|
||||
|
||||
try:
|
||||
fn = getattr(provider, "inline_query_field_choices", None)
|
||||
if callable(fn):
|
||||
extra = fn()
|
||||
if isinstance(extra, dict):
|
||||
for k, v in extra.items():
|
||||
key_norm = str(k).strip().lower()
|
||||
if not key_norm:
|
||||
continue
|
||||
_ingest(v, key_norm)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return mapping
|
||||
|
||||
|
||||
def resolve_filter(
|
||||
provider: Any,
|
||||
inline_args: Dict[str, Any],
|
||||
*,
|
||||
field_transforms: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, str]:
|
||||
"""Map inline query args to provider filter values using declared choices.
|
||||
|
||||
- Uses provider choice mapping (value/text/aliases) to resolve user text.
|
||||
- Applies optional per-field transforms (e.g., str.upper).
|
||||
- Returns normalized filters suitable for provider.search.
|
||||
"""
|
||||
|
||||
filters: Dict[str, str] = {}
|
||||
if not inline_args:
|
||||
return filters
|
||||
|
||||
mapping = collect_choice(provider)
|
||||
transforms = field_transforms or {}
|
||||
|
||||
for raw_key, raw_val in inline_args.items():
|
||||
if raw_val is None:
|
||||
continue
|
||||
key = str(raw_key or "").strip().lower()
|
||||
val_str = str(raw_val).strip()
|
||||
if not key or not val_str:
|
||||
continue
|
||||
|
||||
entries = mapping.get(key, [])
|
||||
resolved: Optional[str] = None
|
||||
val_lower = val_str.lower()
|
||||
for entry in entries:
|
||||
text = str(entry.get("text") or "").strip()
|
||||
value = str(entry.get("value") or "").strip()
|
||||
aliases = [str(a).strip() for a in entry.get("aliases", []) if a is not None]
|
||||
alias_lowers = {a.lower() for a in aliases}
|
||||
if val_lower in {text.lower(), value.lower()} or val_lower in alias_lowers:
|
||||
resolved = value or text or val_str
|
||||
break
|
||||
|
||||
if resolved is None:
|
||||
resolved = val_str
|
||||
|
||||
transform = transforms.get(key)
|
||||
if callable(transform):
|
||||
try:
|
||||
resolved = transform(resolved)
|
||||
except Exception:
|
||||
pass
|
||||
if resolved:
|
||||
filters[key] = str(resolved)
|
||||
|
||||
return filters
|
||||
@@ -89,7 +89,6 @@ class ProviderRegistry:
|
||||
replace: bool = False,
|
||||
) -> ProviderInfo:
|
||||
"""Register a provider class with canonical and alias names."""
|
||||
|
||||
candidates = self._candidate_names(provider_class, override_name)
|
||||
if not candidates:
|
||||
raise ValueError("provider name candidates are required")
|
||||
@@ -397,6 +396,125 @@ def match_provider_name_for_url(url: str) -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
def provider_inline_query_choices(
|
||||
provider_name: str,
|
||||
field_name: str,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
) -> List[str]:
|
||||
"""Return provider-declared inline query choices for a field (e.g., system:GBA).
|
||||
|
||||
Providers can expose a mapping via ``QUERY_ARG_CHOICES`` (preferred) or
|
||||
``INLINE_QUERY_FIELD_CHOICES`` / ``inline_query_field_choices()``. The helper
|
||||
keeps completion logic simple and reusable.
|
||||
This helper keeps completion logic simple and reusable.
|
||||
"""
|
||||
|
||||
pname = str(provider_name or "").strip().lower()
|
||||
field = str(field_name or "").strip().lower()
|
||||
if not pname or not field:
|
||||
return []
|
||||
|
||||
provider = get_search_provider(pname, config)
|
||||
if provider is None:
|
||||
provider = get_provider(pname, config)
|
||||
if provider is None:
|
||||
return []
|
||||
|
||||
def _normalize_choice_entry(entry: Any) -> Optional[Dict[str, Any]]:
|
||||
if entry is None:
|
||||
return None
|
||||
if isinstance(entry, dict):
|
||||
value = entry.get("value")
|
||||
text = entry.get("text") or entry.get("label") or value
|
||||
aliases = entry.get("alias") or entry.get("aliases") or []
|
||||
value_str = str(value) if value is not None else (str(text) if text is not None else None)
|
||||
text_str = str(text) if text is not None else value_str
|
||||
if not value_str or not text_str:
|
||||
return None
|
||||
alias_list = [str(a) for a in aliases if a is not None]
|
||||
return {"value": value_str, "text": text_str, "aliases": alias_list}
|
||||
# string/other primitives
|
||||
return {"value": str(entry), "text": str(entry), "aliases": []}
|
||||
|
||||
def _collect_mapping(p) -> Dict[str, List[Dict[str, Any]]]:
|
||||
mapping: Dict[str, List[Dict[str, Any]]] = {}
|
||||
base = getattr(p, "QUERY_ARG_CHOICES", None)
|
||||
if not isinstance(base, dict):
|
||||
base = getattr(p, "INLINE_QUERY_FIELD_CHOICES", None)
|
||||
if isinstance(base, dict):
|
||||
for k, v in base.items():
|
||||
normalized: List[Dict[str, Any]] = []
|
||||
seq = v
|
||||
try:
|
||||
if callable(seq):
|
||||
seq = seq()
|
||||
except Exception:
|
||||
seq = v
|
||||
if isinstance(seq, dict):
|
||||
seq = seq.get("choices") or seq.get("values") or seq
|
||||
if isinstance(seq, (list, tuple, set)):
|
||||
for entry in seq:
|
||||
n = _normalize_choice_entry(entry)
|
||||
if n:
|
||||
normalized.append(n)
|
||||
if normalized:
|
||||
mapping[str(k).strip().lower()] = normalized
|
||||
try:
|
||||
fn = getattr(p, "inline_query_field_choices", None)
|
||||
if callable(fn):
|
||||
extra = fn()
|
||||
if isinstance(extra, dict):
|
||||
for k, v in extra.items():
|
||||
normalized: List[Dict[str, Any]] = []
|
||||
seq = v
|
||||
try:
|
||||
if callable(seq):
|
||||
seq = seq()
|
||||
except Exception:
|
||||
seq = v
|
||||
if isinstance(seq, dict):
|
||||
seq = seq.get("choices") or seq.get("values") or seq
|
||||
if isinstance(seq, (list, tuple, set)):
|
||||
for entry in seq:
|
||||
n = _normalize_choice_entry(entry)
|
||||
if n:
|
||||
normalized.append(n)
|
||||
if normalized:
|
||||
mapping[str(k).strip().lower()] = normalized
|
||||
except Exception:
|
||||
pass
|
||||
return mapping
|
||||
|
||||
try:
|
||||
mapping = _collect_mapping(provider)
|
||||
if not mapping:
|
||||
return []
|
||||
|
||||
entries = mapping.get(field, [])
|
||||
if not entries:
|
||||
return []
|
||||
|
||||
seen: set[str] = set()
|
||||
out: List[str] = []
|
||||
for entry in entries:
|
||||
text = entry.get("text") or entry.get("value")
|
||||
if not text:
|
||||
continue
|
||||
text_str = str(text)
|
||||
if text_str in seen:
|
||||
continue
|
||||
seen.add(text_str)
|
||||
out.append(text_str)
|
||||
for alias in entry.get("aliases", []):
|
||||
alias_str = str(alias)
|
||||
if alias_str and alias_str not in seen:
|
||||
seen.add(alias_str)
|
||||
out.append(alias_str)
|
||||
return out
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def get_provider_for_url(url: str,
|
||||
config: Optional[Dict[str, Any]] = None) -> Optional[Provider]:
|
||||
name = match_provider_name_for_url(url)
|
||||
@@ -405,6 +523,60 @@ def get_provider_for_url(url: str,
|
||||
return get_provider(name, config)
|
||||
|
||||
|
||||
def resolve_inline_filters(
|
||||
provider: Provider,
|
||||
inline_args: Dict[str, Any],
|
||||
*,
|
||||
field_transforms: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, str]:
|
||||
"""Map inline query args to provider filter values using declared choices.
|
||||
|
||||
- Uses provider's inline choice mapping (value/text/aliases) to resolve user text.
|
||||
- Applies optional per-field transforms (e.g., str.upper).
|
||||
- Returns normalized filters suitable for provider.search.
|
||||
"""
|
||||
|
||||
filters: Dict[str, str] = {}
|
||||
if not inline_args:
|
||||
return filters
|
||||
|
||||
mapping = _collect_mapping(provider)
|
||||
transforms = field_transforms or {}
|
||||
|
||||
for raw_key, raw_val in inline_args.items():
|
||||
if raw_val is None:
|
||||
continue
|
||||
key = str(raw_key or "").strip().lower()
|
||||
val_str = str(raw_val).strip()
|
||||
if not key or not val_str:
|
||||
continue
|
||||
|
||||
entries = mapping.get(key, [])
|
||||
resolved: Optional[str] = None
|
||||
val_lower = val_str.lower()
|
||||
for entry in entries:
|
||||
text = str(entry.get("text") or "").strip()
|
||||
value = str(entry.get("value") or "").strip()
|
||||
aliases = [str(a).strip() for a in entry.get("aliases", []) if a is not None]
|
||||
if val_lower in {text.lower(), value.lower()} or val_lower in {a.lower() for a in aliases}:
|
||||
resolved = value or text or val_str
|
||||
break
|
||||
|
||||
if resolved is None:
|
||||
resolved = val_str
|
||||
|
||||
transform = transforms.get(key)
|
||||
if callable(transform):
|
||||
try:
|
||||
resolved = transform(resolved)
|
||||
except Exception:
|
||||
pass
|
||||
if resolved:
|
||||
filters[key] = str(resolved)
|
||||
|
||||
return filters
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ProviderInfo",
|
||||
"Provider",
|
||||
@@ -423,4 +595,5 @@ __all__ = [
|
||||
"get_provider_class",
|
||||
"selection_auto_stage_for_table",
|
||||
"download_soulseek_file",
|
||||
"provider_inline_query_choices",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user