This commit is contained in:
2026-02-11 18:16:07 -08:00
parent cc715e1fef
commit 1d0de1118b
27 changed files with 1167 additions and 1075 deletions

View File

@@ -17,7 +17,8 @@ from API.Tidal import (
stringify,
)
from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments
from cmdlet._shared import get_field
from SYS.field_access import get_field
from Provider.tidal_manifest import resolve_tidal_manifest_path
from SYS import pipeline as pipeline_context
from SYS.logger import debug, log
@@ -1178,11 +1179,6 @@ class HIFI(Provider):
except Exception:
pass
try:
from cmdlet._shared import resolve_tidal_manifest_path
except Exception:
return None
resolved = resolve_tidal_manifest_path({"full_metadata": md, "path": raw_path, "title": getattr(result, "title", "")})
if not resolved:
return None
@@ -1223,9 +1219,11 @@ class HIFI(Provider):
# As a fallback, try downloading the URL directly if it looks like a file.
try:
import httpx
from API.httpx_shared import get_shared_httpx_client
resp = httpx.get(resolved_text, timeout=float(getattr(self, "api_timeout", 10.0)))
timeout_val = float(getattr(self, "api_timeout", 10.0))
client = get_shared_httpx_client(timeout=timeout_val)
resp = client.get(resolved_text, timeout=timeout_val)
resp.raise_for_status()
content = resp.content
direct_path = output_dir / f"{stem}.bin"

View File

@@ -17,7 +17,8 @@ from API.Tidal import (
stringify,
)
from ProviderCore.base import Provider, SearchResult
from cmdlet._shared import get_field
from SYS.field_access import get_field
from Provider.tidal_manifest import resolve_tidal_manifest_path
from SYS import pipeline as pipeline_context
from SYS.logger import debug, log
@@ -144,6 +145,62 @@ class Tidal(Provider):
meta["view"] = self._get_view(query, filters)
return meta
def postprocess_search_results(
self,
*,
query: str,
results: List[SearchResult],
filters: Optional[Dict[str, Any]] = None,
limit: int = 50,
table_type: str = "",
table_meta: Optional[Dict[str, Any]] = None,
) -> Tuple[List[SearchResult], Optional[str], Optional[Dict[str, Any]]]:
_ = query
_ = filters
_ = table_type
# Provider-specific UX: if an artist search yields exactly one artist row,
# auto-expand directly to albums (preserves historical cmdlet behavior).
try:
view = (table_meta or {}).get("view") if isinstance(table_meta, dict) else None
if str(view or "").strip().lower() != "artist":
return results, None, None
except Exception:
return results, None, None
if not isinstance(results, list) or len(results) != 1:
return results, None, None
artist_res = results[0]
artist_name = str(getattr(artist_res, "title", "") or "").strip()
artist_md = getattr(artist_res, "full_metadata", None)
artist_id: Optional[int] = None
if isinstance(artist_md, dict):
raw_id = artist_md.get("artistId") or artist_md.get("id")
try:
artist_id = int(raw_id) if raw_id is not None else None
except Exception:
artist_id = None
# Use a floor of 200 to keep the expanded album list useful.
want = max(int(limit or 0), 200)
try:
album_results = self._albums_for_artist(
artist_id=artist_id,
artist_name=artist_name,
limit=want,
)
except Exception:
album_results = []
if not album_results:
return results, None, None
meta_out: Dict[str, Any] = dict(table_meta or {}) if isinstance(table_meta, dict) else {}
meta_out["view"] = "album"
return album_results, "tidal.album", meta_out
def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
super().__init__(config)
self.api_urls = self._resolve_api_urls()
@@ -1304,11 +1361,6 @@ class Tidal(Provider):
except Exception:
pass
try:
from cmdlet._shared import resolve_tidal_manifest_path
except Exception:
return None
resolved = resolve_tidal_manifest_path({"full_metadata": md, "path": raw_path, "title": getattr(result, "title", "")})
if not resolved:
return None
@@ -1349,9 +1401,11 @@ class Tidal(Provider):
# As a fallback, try downloading the URL directly if it looks like a file.
try:
import httpx
from API.httpx_shared import get_shared_httpx_client
resp = httpx.get(resolved_text, timeout=float(getattr(self, "api_timeout", 10.0)))
timeout_val = float(getattr(self, "api_timeout", 10.0))
client = get_shared_httpx_client(timeout=timeout_val)
resp = client.get(resolved_text, timeout=timeout_val)
resp.raise_for_status()
content = resp.content
direct_path = output_dir / f"{stem}.bin"

View File

@@ -514,7 +514,7 @@ class InternetArchive(Provider):
quiet_mode: bool,
) -> Optional[int]:
"""Generic hook for download-file to show a selection table for IA items."""
from cmdlet._shared import get_field as sh_get_field
from SYS.field_access import get_field as sh_get_field
return maybe_show_formats_table(
raw_urls=[url] if url else [],
piped_items=[item] if item else [],

View File

@@ -4,6 +4,8 @@ import html as html_std
import logging
import re
import requests
from API.requests_client import get_requests_session
import sys
import time
from pathlib import Path
@@ -294,7 +296,7 @@ def _enrich_book_tags_from_isbn(isbn: str,
# 1) OpenLibrary API lookup by ISBN (short timeout, silent failure).
try:
url = f"https://openlibrary.org/api/books?bibkeys=ISBN:{isbn_clean}&jscmd=data&format=json"
resp = requests.get(url, timeout=4)
resp = get_requests_session().get(url, timeout=4)
resp.raise_for_status()
data = resp.json()
if isinstance(data, dict) and data:
@@ -407,14 +409,11 @@ def _fetch_libgen_details_html(
try:
if timeout is None:
timeout = (DEFAULT_CONNECT_TIMEOUT, DEFAULT_READ_TIMEOUT)
session = requests.Session()
session.headers.update(
{
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
}
)
with session.get(str(url), stream=True, timeout=timeout) as resp:
session = get_requests_session()
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36",
}
with session.get(str(url), stream=True, timeout=timeout, headers=headers) as resp:
resp.raise_for_status()
ct = str(resp.headers.get("Content-Type", "")).lower()
if "text/html" not in ct:
@@ -1111,13 +1110,15 @@ class LibgenSearch:
"""Robust LibGen searcher."""
def __init__(self, session: Optional[requests.Session] = None):
self.session = session or requests.Session()
self.session.headers.update(
{
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
}
)
self.session = session or get_requests_session()
# Ensure a modern browser UA is present without clobbering existing one.
if not any(k.lower() == "user-agent" for k in (self.session.headers or {})):
self.session.headers.update(
{
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
}
)
def _search_libgen_json(
self,
@@ -1901,7 +1902,7 @@ def download_from_mirror(
) -> Tuple[bool,
Optional[Path]]:
"""Download file from a LibGen mirror URL with optional progress tracking."""
session = session or requests.Session()
session = session or get_requests_session()
# Ensure a modern browser User-Agent is used for downloads to avoid mirror blocks.
if not any(
k.lower() == "user-agent"

View File

@@ -8,7 +8,7 @@ from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional, Tuple
from urllib.parse import quote
import requests
from API.requests_client import get_requests_session
from ProviderCore.base import Provider, SearchResult
from SYS.provider_helpers import TableProviderMixin
@@ -189,7 +189,7 @@ def _matrix_health_check(*,
if not base:
return False, "Matrix homeserver missing"
resp = requests.get(f"{base}/_matrix/client/versions", timeout=5)
resp = get_requests_session().get(f"{base}/_matrix/client/versions", timeout=5)
if resp.status_code != 200:
return False, f"Homeserver returned {resp.status_code}"
@@ -197,7 +197,7 @@ def _matrix_health_check(*,
headers = {
"Authorization": f"Bearer {access_token}"
}
resp = requests.get(
resp = get_requests_session().get(
f"{base}/_matrix/client/v3/account/whoami",
headers=headers,
timeout=5
@@ -234,6 +234,8 @@ class Matrix(TableProviderMixin, Provider):
4. Selection triggers upload of pending files to selected rooms
"""
EXPOSE_AS_FILE_PROVIDER = False
@classmethod
def config_schema(cls) -> List[Dict[str, Any]]:
return [
@@ -388,7 +390,7 @@ class Matrix(TableProviderMixin, Provider):
headers = {
"Authorization": f"Bearer {token}"
}
resp = requests.get(
resp = get_requests_session().get(
f"{base}/_matrix/client/v3/joined_rooms",
headers=headers,
timeout=10
@@ -442,7 +444,7 @@ class Matrix(TableProviderMixin, Provider):
# Best-effort room name lookup (safe to fail).
try:
encoded = quote(room_id, safe="")
name_resp = requests.get(
name_resp = get_requests_session().get(
f"{base}/_matrix/client/v3/rooms/{encoded}/state/m.room.name",
headers=headers,
timeout=5,
@@ -491,7 +493,7 @@ class Matrix(TableProviderMixin, Provider):
total_bytes=int(path.stat().st_size),
label="upload"
)
resp = requests.post(
resp = get_requests_session().post(
upload_url,
headers=headers,
data=wrapped,
@@ -539,7 +541,7 @@ class Matrix(TableProviderMixin, Provider):
send_headers = {
"Authorization": f"Bearer {token}"
}
send_resp = requests.put(send_url, headers=send_headers, json=payload)
send_resp = get_requests_session().put(send_url, headers=send_headers, json=payload)
if send_resp.status_code != 200:
raise Exception(f"Matrix send message failed: {send_resp.text}")
@@ -588,7 +590,7 @@ class Matrix(TableProviderMixin, Provider):
"msgtype": "m.text",
"body": message
}
send_resp = requests.put(send_url, headers=send_headers, json=payload)
send_resp = get_requests_session().put(send_url, headers=send_headers, json=payload)
if send_resp.status_code != 200:
raise Exception(f"Matrix send text failed: {send_resp.text}")

View File

@@ -4,12 +4,12 @@ from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional, Type, cast
import html as html_std
import re
import requests
import sys
import json
import subprocess
from API.HTTP import HTTPClient
from API.requests_client import get_requests_session
from ProviderCore.base import SearchResult
try:
from Provider.Tidal import Tidal
@@ -86,7 +86,7 @@ class ITunesProvider(MetadataProvider):
"limit": limit
}
try:
resp = requests.get(
resp = get_requests_session().get(
"https://itunes.apple.com/search",
params=params,
timeout=10
@@ -137,7 +137,7 @@ class OpenLibraryMetadataProvider(MetadataProvider):
else:
q = query_clean
resp = requests.get(
resp = get_requests_session().get(
"https://openlibrary.org/search.json",
params={
"q": q,
@@ -243,7 +243,7 @@ class GoogleBooksMetadataProvider(MetadataProvider):
q = query_clean
try:
resp = requests.get(
resp = get_requests_session().get(
"https://www.googleapis.com/books/v1/volumes",
params={
"q": q,
@@ -369,7 +369,7 @@ class ISBNsearchMetadataProvider(MetadataProvider):
url = f"https://isbnsearch.org/isbn/{isbn}"
try:
resp = requests.get(url, timeout=10)
resp = get_requests_session().get(url, timeout=10)
resp.raise_for_status()
html = str(resp.text or "")
if not html:
@@ -1059,7 +1059,10 @@ def fetch_archive_item_metadata(archive_id: str,
ident = str(archive_id or "").strip()
if not ident:
return {}
resp = requests.get(f"https://archive.org/metadata/{ident}", timeout=int(timeout))
resp = get_requests_session().get(
f"https://archive.org/metadata/{ident}",
timeout=int(timeout),
)
resp.raise_for_status()
data = resp.json() if resp is not None else {}
if not isinstance(data, dict):

View File

@@ -16,7 +16,8 @@ from urllib.parse import urlparse
import requests
from API.HTTP import HTTPClient, get_requests_verify_value
from API.HTTP import HTTPClient
from API.requests_client import get_requests_session
from ProviderCore.base import Provider, SearchResult
from SYS.utils import sanitize_filename
from SYS.cli_syntax import get_field, get_free_text, parse_query
@@ -27,8 +28,6 @@ from Provider.metadata_provider import (
)
from SYS.utils import unique_path
# Resolve lazily to avoid import-time module checks (prevents debugger first-chance noise)
_ARCHIVE_VERIFY_VALUE = None # will be resolved on first session creation
_DEFAULT_ARCHIVE_SCALE = 4
_QUALITY_TO_ARCHIVE_SCALE = {
"high": 2,
@@ -38,12 +37,7 @@ _QUALITY_TO_ARCHIVE_SCALE = {
def _create_archive_session() -> requests.Session:
session = requests.Session()
global _ARCHIVE_VERIFY_VALUE
if _ARCHIVE_VERIFY_VALUE is None:
_ARCHIVE_VERIFY_VALUE = get_requests_verify_value()
session.verify = _ARCHIVE_VERIFY_VALUE
return session
return get_requests_session()
try:
from Crypto.Cipher import AES # type: ignore
@@ -590,10 +584,9 @@ class OpenLibrary(Provider):
if not ident:
return False, "no-archive-id"
try:
resp = requests.get(
resp = get_requests_session().get(
f"https://archive.org/metadata/{ident}",
timeout=8,
verify=_ARCHIVE_VERIFY_VALUE,
)
resp.raise_for_status()
data = resp.json() if resp is not None else {}
@@ -804,10 +797,9 @@ class OpenLibrary(Provider):
"""Check for a directly downloadable original PDF in Archive.org metadata."""
try:
metadata_url = f"https://archive.org/metadata/{book_id}"
response = requests.get(
response = get_requests_session().get(
metadata_url,
timeout=6,
verify=_ARCHIVE_VERIFY_VALUE,
)
response.raise_for_status()
metadata = response.json()
@@ -822,11 +814,10 @@ class OpenLibrary(Provider):
pdf_url = (
f"https://archive.org/download/{book_id}/{filename.replace(' ', '%20')}"
)
check_response = requests.head(
check_response = get_requests_session().head(
pdf_url,
timeout=4,
allow_redirects=True,
verify=_ARCHIVE_VERIFY_VALUE,
)
if check_response.status_code == 200:
return True, pdf_url

284
Provider/tidal_manifest.py Normal file
View File

@@ -0,0 +1,284 @@
"""Tidal/HIFI manifest helpers.
This module intentionally lives with the provider code (not cmdlets).
It contains best-effort helpers for turning proxy-provided Tidal "manifest"
values into a playable input reference:
- A local MPD file path (persisted to temp)
- Or a direct URL (when the manifest is JSON with `urls`)
Callers may pass either a SearchResult-like object (with `.full_metadata`) or
pipeline dicts.
"""
from __future__ import annotations
import base64
import hashlib
import json
import re
import sys
import tempfile
from pathlib import Path
from typing import Any, Dict, Optional
from API.httpx_shared import get_shared_httpx_client
from SYS.logger import log
def resolve_tidal_manifest_path(item: Any) -> Optional[str]:
"""Persist the Tidal manifest (MPD) and return a local path or URL.
Resolution order:
1) `_tidal_manifest_path` (existing local file)
2) `_tidal_manifest_url` (existing remote URL)
3) decode `manifest` and:
- if JSON with `urls`: return the first URL
- if MPD XML: persist under `%TEMP%/medeia/tidal/` and return path
If `manifest` is missing but a track id exists, the function will attempt a
best-effort fetch from the public proxy endpoints to populate `manifest`.
"""
metadata: Any = None
if isinstance(item, dict):
metadata = item.get("full_metadata") or item.get("metadata")
else:
metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None)
if not isinstance(metadata, dict):
return None
existing_path = metadata.get("_tidal_manifest_path")
if existing_path:
try:
resolved = Path(str(existing_path))
if resolved.is_file():
return str(resolved)
except Exception:
pass
existing_url = metadata.get("_tidal_manifest_url")
if existing_url and isinstance(existing_url, str):
candidate = existing_url.strip()
if candidate:
return candidate
raw_manifest = metadata.get("manifest")
if not raw_manifest:
_maybe_fetch_track_manifest(item, metadata)
raw_manifest = metadata.get("manifest")
if not raw_manifest:
return None
manifest_str = "".join(str(raw_manifest or "").split())
if not manifest_str:
return None
manifest_bytes: bytes
try:
manifest_bytes = base64.b64decode(manifest_str, validate=True)
except Exception:
try:
manifest_bytes = base64.b64decode(manifest_str, validate=False)
except Exception:
try:
manifest_bytes = manifest_str.encode("utf-8")
except Exception:
return None
if not manifest_bytes:
return None
head = (manifest_bytes[:1024] or b"").lstrip()
if head.startswith((b"{", b"[")):
return _resolve_json_manifest_urls(metadata, manifest_bytes)
looks_like_mpd = head.startswith((b"<?xml", b"<MPD")) or (b"<MPD" in head)
if not looks_like_mpd:
manifest_mime = str(metadata.get("manifestMimeType") or "").strip().lower()
try:
metadata["_tidal_manifest_error"] = (
f"Decoded manifest is not an MPD XML (mime: {manifest_mime or 'unknown'})"
)
except Exception:
pass
try:
log(
f"[tidal] Decoded manifest is not an MPD XML for track {metadata.get('trackId') or metadata.get('id')} (mime {manifest_mime or 'unknown'})",
file=sys.stderr,
)
except Exception:
pass
return None
return _persist_mpd_bytes(item, metadata, manifest_bytes)
def _maybe_fetch_track_manifest(item: Any, metadata: Dict[str, Any]) -> None:
"""If we only have a track id, fetch details from the proxy to populate `manifest`."""
try:
already = bool(metadata.get("_tidal_track_details_fetched"))
except Exception:
already = False
track_id = metadata.get("trackId") or metadata.get("id")
if track_id is None:
try:
if isinstance(item, dict):
candidate_path = item.get("path") or item.get("url")
else:
candidate_path = getattr(item, "path", None) or getattr(item, "url", None)
except Exception:
candidate_path = None
if candidate_path:
m = re.search(
r"(tidal|hifi):(?://)?track[\\/](\d+)",
str(candidate_path),
flags=re.IGNORECASE,
)
if m:
track_id = m.group(2)
if already or track_id is None:
return
try:
track_int = int(track_id)
except Exception:
track_int = None
if not track_int or track_int <= 0:
return
try:
client = get_shared_httpx_client()
resp = client.get(
"https://tidal-api.binimum.org/track/",
params={"id": str(track_int)},
timeout=10.0,
)
resp.raise_for_status()
payload = resp.json()
data = payload.get("data") if isinstance(payload, dict) else None
if isinstance(data, dict) and data:
try:
metadata.update(data)
except Exception:
pass
try:
metadata["_tidal_track_details_fetched"] = True
except Exception:
pass
if not metadata.get("url"):
try:
resp_info = client.get(
"https://tidal-api.binimum.org/info/",
params={"id": str(track_int)},
timeout=10.0,
)
resp_info.raise_for_status()
info_payload = resp_info.json()
info_data = info_payload.get("data") if isinstance(info_payload, dict) else None
if isinstance(info_data, dict) and info_data:
try:
for k, v in info_data.items():
if k not in metadata:
metadata[k] = v
except Exception:
pass
try:
if info_data.get("url"):
metadata["url"] = info_data.get("url")
except Exception:
pass
except Exception:
pass
except Exception:
return
def _resolve_json_manifest_urls(metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]:
try:
text = manifest_bytes.decode("utf-8", errors="ignore")
payload = json.loads(text)
urls = payload.get("urls") or []
selected_url = None
for candidate in urls:
if isinstance(candidate, str):
candidate = candidate.strip()
if candidate:
selected_url = candidate
break
if selected_url:
try:
metadata["_tidal_manifest_url"] = selected_url
except Exception:
pass
return selected_url
try:
metadata["_tidal_manifest_error"] = "JSON manifest contained no urls"
except Exception:
pass
log(
f"[tidal] JSON manifest for track {metadata.get('trackId') or metadata.get('id')} had no playable urls",
file=sys.stderr,
)
except Exception as exc:
try:
metadata["_tidal_manifest_error"] = f"Failed to parse JSON manifest: {exc}"
except Exception:
pass
log(
f"[tidal] Failed to parse JSON manifest for track {metadata.get('trackId') or metadata.get('id')}: {exc}",
file=sys.stderr,
)
return None
def _persist_mpd_bytes(item: Any, metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]:
manifest_hash = str(metadata.get("manifestHash") or "").strip()
track_id = metadata.get("trackId") or metadata.get("id")
identifier = manifest_hash or hashlib.sha256(manifest_bytes).hexdigest()
identifier_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", identifier)[:64]
if not identifier_safe:
identifier_safe = hashlib.sha256(manifest_bytes).hexdigest()[:12]
track_safe = "tidal"
if track_id is not None:
track_safe = re.sub(r"[^A-Za-z0-9_-]+", "_", str(track_id))[:32] or "tidal"
manifest_dir = Path(tempfile.gettempdir()) / "medeia" / "tidal"
try:
manifest_dir.mkdir(parents=True, exist_ok=True)
except Exception:
pass
filename = f"tidal-{track_safe}-{identifier_safe[:24]}.mpd"
target_path = manifest_dir / filename
try:
with open(target_path, "wb") as fh:
fh.write(manifest_bytes)
metadata["_tidal_manifest_path"] = str(target_path)
# Best-effort: propagate back into the caller object/dict.
if isinstance(item, dict):
if item.get("full_metadata") is metadata:
item["full_metadata"] = metadata
elif item.get("metadata") is metadata:
item["metadata"] = metadata
else:
extra = getattr(item, "extra", None)
if isinstance(extra, dict):
extra["_tidal_manifest_path"] = str(target_path)
return str(target_path)
except Exception:
return None

View File

@@ -6,6 +6,8 @@ from dataclasses import dataclass
from typing import Any, Dict, List, Optional
import requests
from API.requests_client import get_requests_session
from ProviderCore.base import Provider, SearchResult
from SYS.logger import debug, log
try: # Preferred HTML parser
@@ -66,7 +68,7 @@ class Scraper:
def _get_page(self, page: int) -> List[TorrentInfo]:
url, payload = self._request_data(page)
try:
resp = requests.get(
resp = get_requests_session().get(
url,
params=payload,
headers=self.headers,
@@ -86,7 +88,7 @@ class Scraper:
def _parse_detail(self, url: str) -> Optional[str]: # optional override
try:
resp = requests.get(url, headers=self.headers, timeout=self.timeout)
resp = get_requests_session().get(url, headers=self.headers, timeout=self.timeout)
resp.raise_for_status()
return self._parse_detail_response(resp)
except Exception: