update
This commit is contained in:
@@ -73,6 +73,8 @@ class HydrusNetwork:
|
|||||||
url: str
|
url: str
|
||||||
access_key: str = ""
|
access_key: str = ""
|
||||||
timeout: float = 9.0
|
timeout: float = 9.0
|
||||||
|
upload_io_timeout: float = 120.0
|
||||||
|
upload_chunk_size: int = 64 * 1024
|
||||||
instance_name: str = "" # Optional store name (e.g., 'home') for namespaced logs
|
instance_name: str = "" # Optional store name (e.g., 'home') for namespaced logs
|
||||||
|
|
||||||
scheme: str = field(init=False)
|
scheme: str = field(init=False)
|
||||||
@@ -167,6 +169,37 @@ class HydrusNetwork:
|
|||||||
f"{self._log_prefix()} Uploading file {file_path.name} ({file_size} bytes)"
|
f"{self._log_prefix()} Uploading file {file_path.name} ({file_size} bytes)"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Upload timeout policy:
|
||||||
|
# - Keep normal requests fast via `self.timeout`.
|
||||||
|
# - For streaming uploads, use an activity timeout for read/write so
|
||||||
|
# long transfers do not fail due to a short generic timeout.
|
||||||
|
# - If upload_io_timeout <= 0, disable read/write timeout entirely.
|
||||||
|
try:
|
||||||
|
upload_io_timeout = float(self.upload_io_timeout)
|
||||||
|
except Exception:
|
||||||
|
upload_io_timeout = 120.0
|
||||||
|
if upload_io_timeout <= 0:
|
||||||
|
upload_timeout = httpx.Timeout(
|
||||||
|
connect=float(self.timeout),
|
||||||
|
read=None,
|
||||||
|
write=None,
|
||||||
|
pool=float(self.timeout),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
upload_timeout = httpx.Timeout(
|
||||||
|
connect=float(self.timeout),
|
||||||
|
read=upload_io_timeout,
|
||||||
|
write=upload_io_timeout,
|
||||||
|
pool=float(self.timeout),
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
chunk_size = int(self.upload_chunk_size)
|
||||||
|
except Exception:
|
||||||
|
chunk_size = 64 * 1024
|
||||||
|
if chunk_size <= 0:
|
||||||
|
chunk_size = 64 * 1024
|
||||||
|
|
||||||
# Stream upload body with a stderr progress bar (pipeline-safe).
|
# Stream upload body with a stderr progress bar (pipeline-safe).
|
||||||
from SYS.models import ProgressBar
|
from SYS.models import ProgressBar
|
||||||
|
|
||||||
@@ -198,7 +231,7 @@ class HydrusNetwork:
|
|||||||
try:
|
try:
|
||||||
with file_path.open("rb") as handle:
|
with file_path.open("rb") as handle:
|
||||||
while True:
|
while True:
|
||||||
chunk = handle.read(256 * 1024)
|
chunk = handle.read(chunk_size)
|
||||||
if not chunk:
|
if not chunk:
|
||||||
break
|
break
|
||||||
sent[0] += len(chunk)
|
sent[0] += len(chunk)
|
||||||
@@ -216,6 +249,7 @@ class HydrusNetwork:
|
|||||||
url,
|
url,
|
||||||
content=file_gen(),
|
content=file_gen(),
|
||||||
headers=headers,
|
headers=headers,
|
||||||
|
timeout=upload_timeout,
|
||||||
raise_for_status=False,
|
raise_for_status=False,
|
||||||
log_http_errors=False,
|
log_http_errors=False,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -22,7 +22,7 @@
|
|||||||
"((1fichier\\.com|megadl\\.fr|alterupload\\.com|cjoint\\.net|desfichiers\\.com|dfichiers\\.com|mesfichiers\\.org|piecejointe\\.net|pjointe\\.com|tenvoi\\.com|dl4free\\.com)/\\?[a-zA-Z0-9]{5,30}(&pw=[^&]+)?)"
|
"((1fichier\\.com|megadl\\.fr|alterupload\\.com|cjoint\\.net|desfichiers\\.com|dfichiers\\.com|mesfichiers\\.org|piecejointe\\.net|pjointe\\.com|tenvoi\\.com|dl4free\\.com)/\\?[a-zA-Z0-9]{5,30}(&pw=[^&]+)?)"
|
||||||
],
|
],
|
||||||
"regexp": "((1fichier\\.com|megadl\\.fr|alterupload\\.com|cjoint\\.net|desfichiers\\.com|dfichiers\\.com|mesfichiers\\.org|piecejointe\\.net|pjointe\\.com|tenvoi\\.com|dl4free\\.com)/\\?[a-zA-Z0-9]{5,30}(&pw=[^&]+)?)",
|
"regexp": "((1fichier\\.com|megadl\\.fr|alterupload\\.com|cjoint\\.net|desfichiers\\.com|dfichiers\\.com|mesfichiers\\.org|piecejointe\\.net|pjointe\\.com|tenvoi\\.com|dl4free\\.com)/\\?[a-zA-Z0-9]{5,30}(&pw=[^&]+)?)",
|
||||||
"status": true
|
"status": false
|
||||||
},
|
},
|
||||||
"rapidgator": {
|
"rapidgator": {
|
||||||
"name": "rapidgator",
|
"name": "rapidgator",
|
||||||
@@ -398,7 +398,7 @@
|
|||||||
"(gigapeta\\.com/dl/[0-9a-zA-Z]{13,15})"
|
"(gigapeta\\.com/dl/[0-9a-zA-Z]{13,15})"
|
||||||
],
|
],
|
||||||
"regexp": "(gigapeta\\.com/dl/[0-9a-zA-Z]{13,15})",
|
"regexp": "(gigapeta\\.com/dl/[0-9a-zA-Z]{13,15})",
|
||||||
"status": true
|
"status": false
|
||||||
},
|
},
|
||||||
"google": {
|
"google": {
|
||||||
"name": "google",
|
"name": "google",
|
||||||
@@ -425,7 +425,7 @@
|
|||||||
"(hexupload\\.net|hexload\\.com)/([a-zA-Z0-9]{12})"
|
"(hexupload\\.net|hexload\\.com)/([a-zA-Z0-9]{12})"
|
||||||
],
|
],
|
||||||
"regexp": "(hexupload\\.net|hexload\\.com)/([a-zA-Z0-9]{12})",
|
"regexp": "(hexupload\\.net|hexload\\.com)/([a-zA-Z0-9]{12})",
|
||||||
"status": true
|
"status": false
|
||||||
},
|
},
|
||||||
"hot4share": {
|
"hot4share": {
|
||||||
"name": "hot4share",
|
"name": "hot4share",
|
||||||
@@ -494,7 +494,7 @@
|
|||||||
"mediafire\\.com/(\\?|download/|file/|download\\.php\\?)([0-9a-z]{15})"
|
"mediafire\\.com/(\\?|download/|file/|download\\.php\\?)([0-9a-z]{15})"
|
||||||
],
|
],
|
||||||
"regexp": "mediafire\\.com/(\\?|download/|file/|download\\.php\\?)([0-9a-z]{15})",
|
"regexp": "mediafire\\.com/(\\?|download/|file/|download\\.php\\?)([0-9a-z]{15})",
|
||||||
"status": true
|
"status": false
|
||||||
},
|
},
|
||||||
"mixdrop": {
|
"mixdrop": {
|
||||||
"name": "mixdrop",
|
"name": "mixdrop",
|
||||||
@@ -652,7 +652,7 @@
|
|||||||
"(uploadboy\\.com/[0-9a-zA-Z]{12})"
|
"(uploadboy\\.com/[0-9a-zA-Z]{12})"
|
||||||
],
|
],
|
||||||
"regexp": "(uploadboy\\.com/[0-9a-zA-Z]{12})",
|
"regexp": "(uploadboy\\.com/[0-9a-zA-Z]{12})",
|
||||||
"status": true
|
"status": false
|
||||||
},
|
},
|
||||||
"uploader": {
|
"uploader": {
|
||||||
"name": "uploader",
|
"name": "uploader",
|
||||||
@@ -690,7 +690,7 @@
|
|||||||
"uploadrar\\.(net|com)/([0-9a-z]{12})"
|
"uploadrar\\.(net|com)/([0-9a-z]{12})"
|
||||||
],
|
],
|
||||||
"regexp": "((get|cloud)\\.rahim-soft\\.com/([0-9a-z]{12}))|((fingau\\.com/([0-9a-z]{12})))|((tech|miui|cloud|flash)\\.getpczone\\.com/([0-9a-z]{12}))|(miui.rahim-soft\\.com/([0-9a-z]{12}))|(uploadrar\\.(net|com)/([0-9a-z]{12}))",
|
"regexp": "((get|cloud)\\.rahim-soft\\.com/([0-9a-z]{12}))|((fingau\\.com/([0-9a-z]{12})))|((tech|miui|cloud|flash)\\.getpczone\\.com/([0-9a-z]{12}))|(miui.rahim-soft\\.com/([0-9a-z]{12}))|(uploadrar\\.(net|com)/([0-9a-z]{12}))",
|
||||||
"status": true,
|
"status": false,
|
||||||
"hardRedirect": [
|
"hardRedirect": [
|
||||||
"uploadrar.com/([0-9a-zA-Z]{12})"
|
"uploadrar.com/([0-9a-zA-Z]{12})"
|
||||||
]
|
]
|
||||||
@@ -775,7 +775,7 @@
|
|||||||
"(worldbytez\\.(net|com)/[a-zA-Z0-9]{12})"
|
"(worldbytez\\.(net|com)/[a-zA-Z0-9]{12})"
|
||||||
],
|
],
|
||||||
"regexp": "(worldbytez\\.(net|com)/[a-zA-Z0-9]{12})",
|
"regexp": "(worldbytez\\.(net|com)/[a-zA-Z0-9]{12})",
|
||||||
"status": true
|
"status": false
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"streams": {
|
"streams": {
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import importlib.util
|
||||||
|
import inspect
|
||||||
import re
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import sys
|
import sys
|
||||||
@@ -11,6 +13,7 @@ from typing import Any, Dict, List, Optional, Sequence, Tuple
|
|||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
from ProviderCore.base import Provider, SearchResult
|
from ProviderCore.base import Provider, SearchResult
|
||||||
|
from SYS.logger import debug
|
||||||
|
|
||||||
_TELEGRAM_DEFAULT_TIMESTAMP_STEM_RE = re.compile(
|
_TELEGRAM_DEFAULT_TIMESTAMP_STEM_RE = re.compile(
|
||||||
r"^(?P<prefix>photo|video|document|audio|voice|animation)_(?P<date>\d{4}-\d{2}-\d{2})_(?P<time>\d{2}-\d{2}-\d{2})(?: \(\d+\))?$",
|
r"^(?P<prefix>photo|video|document|audio|voice|animation)_(?P<date>\d{4}-\d{2}-\d{2})_(?P<time>\d{2}-\d{2}-\d{2})(?: \(\d+\))?$",
|
||||||
@@ -170,6 +173,21 @@ class Telegram(Provider):
|
|||||||
"label": "Bot Token (optional)",
|
"label": "Bot Token (optional)",
|
||||||
"default": "",
|
"default": "",
|
||||||
"secret": True
|
"secret": True
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "part_size_kb",
|
||||||
|
"label": "Transfer chunk size KB (4-512)",
|
||||||
|
"default": "512",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "connection_mode",
|
||||||
|
"label": "Connection mode (abridged|full)",
|
||||||
|
"default": "abridged",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"key": "receive_updates",
|
||||||
|
"label": "Receive updates during transfers",
|
||||||
|
"default": False,
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -185,13 +203,108 @@ class Telegram(Provider):
|
|||||||
self._api_hash = telegram_conf.get("api_hash")
|
self._api_hash = telegram_conf.get("api_hash")
|
||||||
self._bot_token = telegram_conf.get("bot_token")
|
self._bot_token = telegram_conf.get("bot_token")
|
||||||
self._last_login_error: Optional[str] = None
|
self._last_login_error: Optional[str] = None
|
||||||
# Telethon downloads are chunked; larger parts mean fewer round-trips.
|
# Telethon transfers are chunked; larger parts mean fewer round-trips.
|
||||||
# Telethon typically expects 4..1024 KB and divisible by 4.
|
# Telethon typically expects 4..512 KB and divisible by 4.
|
||||||
self._part_size_kb = telegram_conf.get("part_size_kb")
|
self._part_size_kb = telegram_conf.get("part_size_kb")
|
||||||
if self._part_size_kb is None:
|
if self._part_size_kb is None:
|
||||||
self._part_size_kb = telegram_conf.get("chunk_kb")
|
self._part_size_kb = telegram_conf.get("chunk_kb")
|
||||||
if self._part_size_kb is None:
|
if self._part_size_kb is None:
|
||||||
self._part_size_kb = telegram_conf.get("download_part_kb")
|
self._part_size_kb = telegram_conf.get("download_part_kb")
|
||||||
|
self._connection_mode = str(
|
||||||
|
telegram_conf.get("connection_mode") or telegram_conf.get("connection")
|
||||||
|
or "abridged"
|
||||||
|
).strip().lower()
|
||||||
|
self._receive_updates = self._coerce_bool(
|
||||||
|
telegram_conf.get("receive_updates"), default=False
|
||||||
|
)
|
||||||
|
self._cryptg_available = self._detect_cryptg_available()
|
||||||
|
self._emitted_cryptg_hint = False
|
||||||
|
self._download_media_accepts_part_size = self._detect_download_media_accepts_part_size()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _coerce_bool(value: Any, *, default: bool = False) -> bool:
|
||||||
|
if value is None:
|
||||||
|
return default
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return value
|
||||||
|
try:
|
||||||
|
s = str(value).strip().lower()
|
||||||
|
except Exception:
|
||||||
|
return default
|
||||||
|
if s in {"1", "true", "yes", "on"}:
|
||||||
|
return True
|
||||||
|
if s in {"0", "false", "no", "off"}:
|
||||||
|
return False
|
||||||
|
return default
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _detect_cryptg_available() -> bool:
|
||||||
|
try:
|
||||||
|
return importlib.util.find_spec("cryptg") is not None
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _detect_download_media_accepts_part_size() -> bool:
|
||||||
|
try:
|
||||||
|
from telethon import TelegramClient
|
||||||
|
|
||||||
|
sig = inspect.signature(TelegramClient.download_media)
|
||||||
|
params = sig.parameters
|
||||||
|
if "part_size_kb" in params:
|
||||||
|
return True
|
||||||
|
return any(
|
||||||
|
p.kind == inspect.Parameter.VAR_KEYWORD
|
||||||
|
for p in params.values()
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _emit_cryptg_speed_hint_once(self) -> None:
|
||||||
|
if self._cryptg_available or self._emitted_cryptg_hint:
|
||||||
|
return
|
||||||
|
self._emitted_cryptg_hint = True
|
||||||
|
try:
|
||||||
|
sys.stderr.write(
|
||||||
|
"[telegram] Tip: install 'cryptg' for faster Telegram media transfer performance.\n"
|
||||||
|
)
|
||||||
|
sys.stderr.flush()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _new_client(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
session_base: Path,
|
||||||
|
app_id: int,
|
||||||
|
api_hash: str,
|
||||||
|
receive_updates: Optional[bool] = None,
|
||||||
|
):
|
||||||
|
from telethon import TelegramClient
|
||||||
|
|
||||||
|
kwargs: Dict[str, Any] = {
|
||||||
|
"receive_updates": bool(
|
||||||
|
self._receive_updates
|
||||||
|
if receive_updates is None else receive_updates
|
||||||
|
)
|
||||||
|
}
|
||||||
|
mode = str(self._connection_mode or "").strip().lower()
|
||||||
|
try:
|
||||||
|
if mode in {"abridged", "tcpabridged", "fast"}:
|
||||||
|
from telethon.network.connection.tcpabridged import ConnectionTcpAbridged
|
||||||
|
|
||||||
|
kwargs["connection"] = ConnectionTcpAbridged
|
||||||
|
elif mode in {"full", "tcpfull"}:
|
||||||
|
from telethon.network.connection.tcpfull import ConnectionTcpFull
|
||||||
|
|
||||||
|
kwargs["connection"] = ConnectionTcpFull
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
return TelegramClient(str(session_base), app_id, api_hash, **kwargs)
|
||||||
|
except TypeError:
|
||||||
|
return TelegramClient(str(session_base), app_id, api_hash)
|
||||||
|
|
||||||
def _has_running_event_loop(self) -> bool:
|
def _has_running_event_loop(self) -> bool:
|
||||||
try:
|
try:
|
||||||
@@ -342,7 +455,11 @@ class Telegram(Provider):
|
|||||||
session_base = self._session_base_path()
|
session_base = self._session_base_path()
|
||||||
|
|
||||||
async def _check_async() -> bool:
|
async def _check_async() -> bool:
|
||||||
client = TelegramClient(str(session_base), app_id, api_hash)
|
client = self._new_client(
|
||||||
|
session_base=session_base,
|
||||||
|
app_id=app_id,
|
||||||
|
api_hash=api_hash,
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
await client.connect()
|
await client.connect()
|
||||||
return bool(await client.is_user_authorized())
|
return bool(await client.is_user_authorized())
|
||||||
@@ -418,7 +535,12 @@ class Telegram(Provider):
|
|||||||
session_base = self._session_base_path()
|
session_base = self._session_base_path()
|
||||||
|
|
||||||
async def _auth_async() -> None:
|
async def _auth_async() -> None:
|
||||||
client = TelegramClient(str(session_base), app_id, api_hash)
|
client = self._new_client(
|
||||||
|
session_base=session_base,
|
||||||
|
app_id=app_id,
|
||||||
|
api_hash=api_hash,
|
||||||
|
receive_updates=True,
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
if use_bot:
|
if use_bot:
|
||||||
await client.start(bot_token=bot_token)
|
await client.start(bot_token=bot_token)
|
||||||
@@ -515,7 +637,12 @@ class Telegram(Provider):
|
|||||||
session_base = self._session_base_path()
|
session_base = self._session_base_path()
|
||||||
|
|
||||||
async def _auth_async() -> None:
|
async def _auth_async() -> None:
|
||||||
client = TelegramClient(str(session_base), app_id, api_hash)
|
client = self._new_client(
|
||||||
|
session_base=session_base,
|
||||||
|
app_id=app_id,
|
||||||
|
api_hash=api_hash,
|
||||||
|
receive_updates=True,
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
await client.start(bot_token=bot_token)
|
await client.start(bot_token=bot_token)
|
||||||
finally:
|
finally:
|
||||||
@@ -545,15 +672,15 @@ class Telegram(Provider):
|
|||||||
if val not in (None, ""):
|
if val not in (None, ""):
|
||||||
ps = int(str(val).strip())
|
ps = int(str(val).strip())
|
||||||
else:
|
else:
|
||||||
ps = 1024
|
ps = 512
|
||||||
except Exception:
|
except Exception:
|
||||||
ps = 1024
|
ps = 512
|
||||||
|
|
||||||
# Clamp to Telethon-safe range.
|
# Clamp to Telethon-safe range.
|
||||||
if ps < 4:
|
if ps < 4:
|
||||||
ps = 4
|
ps = 4
|
||||||
if ps > 1024:
|
if ps > 512:
|
||||||
ps = 1024
|
ps = 512
|
||||||
# Must be divisible by 4.
|
# Must be divisible by 4.
|
||||||
ps = int(ps / 4) * 4
|
ps = int(ps / 4) * 4
|
||||||
if ps <= 0:
|
if ps <= 0:
|
||||||
@@ -640,7 +767,11 @@ class Telegram(Provider):
|
|||||||
session_base = self._session_base_path()
|
session_base = self._session_base_path()
|
||||||
|
|
||||||
async def _list_async() -> list[Dict[str, Any]]:
|
async def _list_async() -> list[Dict[str, Any]]:
|
||||||
client = TelegramClient(str(session_base), app_id, api_hash)
|
client = self._new_client(
|
||||||
|
session_base=session_base,
|
||||||
|
app_id=app_id,
|
||||||
|
api_hash=api_hash,
|
||||||
|
)
|
||||||
rows: list[Dict[str, Any]] = []
|
rows: list[Dict[str, Any]] = []
|
||||||
try:
|
try:
|
||||||
await client.connect()
|
await client.connect()
|
||||||
@@ -840,13 +971,18 @@ class Telegram(Provider):
|
|||||||
raise Exception("No chat selected")
|
raise Exception("No chat selected")
|
||||||
|
|
||||||
async def _send_async() -> None:
|
async def _send_async() -> None:
|
||||||
client = TelegramClient(str(session_base), app_id, api_hash)
|
client = self._new_client(
|
||||||
|
session_base=session_base,
|
||||||
|
app_id=app_id,
|
||||||
|
api_hash=api_hash,
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
await client.connect()
|
await client.connect()
|
||||||
if not bool(await client.is_user_authorized()):
|
if not bool(await client.is_user_authorized()):
|
||||||
raise Exception(
|
raise Exception(
|
||||||
"Telegram session is not authorized. Run: .telegram -login"
|
"Telegram session is not authorized. Run: .telegram -login"
|
||||||
)
|
)
|
||||||
|
self._emit_cryptg_speed_hint_once()
|
||||||
|
|
||||||
# Resolve entities: prefer IDs. Only fall back to usernames when IDs are absent.
|
# Resolve entities: prefer IDs. Only fall back to usernames when IDs are absent.
|
||||||
entities: list[Any] = []
|
entities: list[Any] = []
|
||||||
@@ -999,7 +1135,11 @@ class Telegram(Provider):
|
|||||||
chat, message_id = _parse_telegram_message_url(url)
|
chat, message_id = _parse_telegram_message_url(url)
|
||||||
|
|
||||||
async def _download_async() -> Tuple[Path, Dict[str, Any]]:
|
async def _download_async() -> Tuple[Path, Dict[str, Any]]:
|
||||||
client = TelegramClient(str(session_base), app_id, api_hash)
|
client = self._new_client(
|
||||||
|
session_base=session_base,
|
||||||
|
app_id=app_id,
|
||||||
|
api_hash=api_hash,
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
await client.connect()
|
await client.connect()
|
||||||
if not bool(await client.is_user_authorized()):
|
if not bool(await client.is_user_authorized()):
|
||||||
@@ -1104,18 +1244,20 @@ class Telegram(Provider):
|
|||||||
)
|
)
|
||||||
|
|
||||||
part_kb = self._resolve_part_size_kb(file_size)
|
part_kb = self._resolve_part_size_kb(file_size)
|
||||||
|
self._emit_cryptg_speed_hint_once()
|
||||||
|
download_kwargs: Dict[str, Any] = {
|
||||||
|
"file": str(output_dir),
|
||||||
|
"progress_callback": _progress,
|
||||||
|
}
|
||||||
|
if self._download_media_accepts_part_size:
|
||||||
|
download_kwargs["part_size_kb"] = part_kb
|
||||||
try:
|
try:
|
||||||
downloaded = await client.download_media(
|
downloaded = await client.download_media(message, **download_kwargs)
|
||||||
message,
|
|
||||||
file=str(output_dir),
|
|
||||||
progress_callback=_progress,
|
|
||||||
part_size_kb=part_kb,
|
|
||||||
)
|
|
||||||
except TypeError:
|
except TypeError:
|
||||||
downloaded = await client.download_media(
|
downloaded = await client.download_media(
|
||||||
message,
|
message,
|
||||||
file=str(output_dir),
|
file=str(output_dir),
|
||||||
progress_callback=_progress
|
progress_callback=_progress,
|
||||||
)
|
)
|
||||||
progress_bar.finish()
|
progress_bar.finish()
|
||||||
if not downloaded:
|
if not downloaded:
|
||||||
|
|||||||
@@ -25,6 +25,18 @@ from API.httpx_shared import get_shared_httpx_client
|
|||||||
from SYS.logger import log
|
from SYS.logger import log
|
||||||
|
|
||||||
|
|
||||||
|
_DEFAULT_TIDAL_TRACK_API_BASES = (
|
||||||
|
"https://triton.squid.wtf",
|
||||||
|
"https://wolf.qqdl.site",
|
||||||
|
"https://maus.qqdl.site",
|
||||||
|
"https://vogel.qqdl.site",
|
||||||
|
"https://katze.qqdl.site",
|
||||||
|
"https://hund.qqdl.site",
|
||||||
|
"https://tidal.kinoplus.online",
|
||||||
|
"https://tidal-api.binimum.org",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def resolve_tidal_manifest_path(item: Any) -> Optional[str]:
|
def resolve_tidal_manifest_path(item: Any) -> Optional[str]:
|
||||||
"""Persist the Tidal manifest (MPD) and return a local path or URL.
|
"""Persist the Tidal manifest (MPD) and return a local path or URL.
|
||||||
|
|
||||||
@@ -114,6 +126,41 @@ def resolve_tidal_manifest_path(item: Any) -> Optional[str]:
|
|||||||
return _persist_mpd_bytes(item, metadata, manifest_bytes)
|
return _persist_mpd_bytes(item, metadata, manifest_bytes)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_api_base(candidate: Any) -> Optional[str]:
|
||||||
|
text = str(candidate or "").strip()
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
if not re.match(r"^https?://", text, flags=re.IGNORECASE):
|
||||||
|
return None
|
||||||
|
return text.rstrip("/")
|
||||||
|
|
||||||
|
|
||||||
|
def _iter_track_api_bases(metadata: Dict[str, Any]) -> list[str]:
|
||||||
|
bases: list[str] = []
|
||||||
|
seen: set[str] = set()
|
||||||
|
|
||||||
|
dynamic_candidates = [
|
||||||
|
metadata.get("_tidal_api_base"),
|
||||||
|
metadata.get("_api_base"),
|
||||||
|
metadata.get("api_base"),
|
||||||
|
metadata.get("base_url"),
|
||||||
|
]
|
||||||
|
|
||||||
|
for candidate in dynamic_candidates:
|
||||||
|
normalized = _normalize_api_base(candidate)
|
||||||
|
if normalized and normalized not in seen:
|
||||||
|
seen.add(normalized)
|
||||||
|
bases.append(normalized)
|
||||||
|
|
||||||
|
for candidate in _DEFAULT_TIDAL_TRACK_API_BASES:
|
||||||
|
normalized = _normalize_api_base(candidate)
|
||||||
|
if normalized and normalized not in seen:
|
||||||
|
seen.add(normalized)
|
||||||
|
bases.append(normalized)
|
||||||
|
|
||||||
|
return bases
|
||||||
|
|
||||||
|
|
||||||
def _maybe_fetch_track_manifest(item: Any, metadata: Dict[str, Any]) -> None:
|
def _maybe_fetch_track_manifest(item: Any, metadata: Dict[str, Any]) -> None:
|
||||||
"""If we only have a track id, fetch details from the proxy to populate `manifest`."""
|
"""If we only have a track id, fetch details from the proxy to populate `manifest`."""
|
||||||
|
|
||||||
@@ -155,29 +202,40 @@ def _maybe_fetch_track_manifest(item: Any, metadata: Dict[str, Any]) -> None:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
client = get_shared_httpx_client()
|
client = get_shared_httpx_client()
|
||||||
|
except Exception:
|
||||||
|
return
|
||||||
|
|
||||||
|
attempted = False
|
||||||
|
for base in _iter_track_api_bases(metadata):
|
||||||
|
attempted = True
|
||||||
|
|
||||||
|
track_data: Optional[Dict[str, Any]] = None
|
||||||
|
for params in ({"id": str(track_int)}, {"id": str(track_int), "quality": "LOSSLESS"}):
|
||||||
|
try:
|
||||||
resp = client.get(
|
resp = client.get(
|
||||||
"https://tidal-api.binimum.org/track/",
|
f"{base}/track/",
|
||||||
params={"id": str(track_int)},
|
params=params,
|
||||||
timeout=10.0,
|
timeout=10.0,
|
||||||
)
|
)
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
payload = resp.json()
|
payload = resp.json()
|
||||||
data = payload.get("data") if isinstance(payload, dict) else None
|
data = payload.get("data") if isinstance(payload, dict) else None
|
||||||
if isinstance(data, dict) and data:
|
if isinstance(data, dict) and data:
|
||||||
try:
|
track_data = data
|
||||||
metadata.update(data)
|
break
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
continue
|
||||||
|
|
||||||
|
if isinstance(track_data, dict) and track_data:
|
||||||
try:
|
try:
|
||||||
metadata["_tidal_track_details_fetched"] = True
|
metadata.update(track_data)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if not metadata.get("url"):
|
if not metadata.get("manifest") or not metadata.get("url"):
|
||||||
try:
|
try:
|
||||||
resp_info = client.get(
|
resp_info = client.get(
|
||||||
"https://tidal-api.binimum.org/info/",
|
f"{base}/info/",
|
||||||
params={"id": str(track_int)},
|
params={"id": str(track_int)},
|
||||||
timeout=10.0,
|
timeout=10.0,
|
||||||
)
|
)
|
||||||
@@ -186,21 +244,22 @@ def _maybe_fetch_track_manifest(item: Any, metadata: Dict[str, Any]) -> None:
|
|||||||
info_data = info_payload.get("data") if isinstance(info_payload, dict) else None
|
info_data = info_payload.get("data") if isinstance(info_payload, dict) else None
|
||||||
if isinstance(info_data, dict) and info_data:
|
if isinstance(info_data, dict) and info_data:
|
||||||
try:
|
try:
|
||||||
for k, v in info_data.items():
|
for key, value in info_data.items():
|
||||||
if k not in metadata:
|
if key not in metadata or not metadata.get(key):
|
||||||
metadata[k] = v
|
metadata[key] = value
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
try:
|
|
||||||
if info_data.get("url"):
|
|
||||||
metadata["url"] = info_data.get("url")
|
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
if metadata.get("manifest"):
|
||||||
|
break
|
||||||
|
|
||||||
|
if attempted:
|
||||||
|
try:
|
||||||
|
metadata["_tidal_track_details_fetched"] = True
|
||||||
except Exception:
|
except Exception:
|
||||||
return
|
pass
|
||||||
|
|
||||||
|
|
||||||
def _resolve_json_manifest_urls(metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]:
|
def _resolve_json_manifest_urls(metadata: Dict[str, Any], manifest_bytes: bytes) -> Optional[str]:
|
||||||
|
|||||||
@@ -1195,13 +1195,6 @@ class HydrusNetwork(Store):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
try:
|
try:
|
||||||
title_ids, title_hashes = _extract_search_ids(
|
|
||||||
payloads[0] if payloads else None
|
|
||||||
)
|
|
||||||
# Optimization: for single-term queries, skip the freeform query
|
|
||||||
# to avoid duplicate requests.
|
|
||||||
single_term = bool(search_terms and len(search_terms) == 1)
|
|
||||||
if not single_term:
|
|
||||||
payloads.append(
|
payloads.append(
|
||||||
client.search_files(
|
client.search_files(
|
||||||
tags=freeform_predicates,
|
tags=freeform_predicates,
|
||||||
|
|||||||
@@ -844,7 +844,6 @@ def _tail_text_file(path: str,
|
|||||||
return []
|
return []
|
||||||
except Exception:
|
except Exception:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with open(p, "rb") as f:
|
with open(p, "rb") as f:
|
||||||
try:
|
try:
|
||||||
@@ -864,6 +863,64 @@ def _tail_text_file(path: str,
|
|||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_tidal_stream_fallback_url(item: Any) -> Optional[str]:
|
||||||
|
"""Best-effort HTTP streaming fallback for unresolved tidal:// placeholders."""
|
||||||
|
|
||||||
|
def _http_candidate(value: Any) -> Optional[str]:
|
||||||
|
if isinstance(value, list):
|
||||||
|
for entry in value:
|
||||||
|
candidate = _http_candidate(entry)
|
||||||
|
if candidate:
|
||||||
|
return candidate
|
||||||
|
return None
|
||||||
|
text = str(value or "").strip()
|
||||||
|
if not text:
|
||||||
|
return None
|
||||||
|
if text.lower().startswith(("http://", "https://")):
|
||||||
|
return text
|
||||||
|
return None
|
||||||
|
|
||||||
|
metadata: Optional[Dict[str, Any]] = None
|
||||||
|
|
||||||
|
if isinstance(item, dict):
|
||||||
|
metadata = item.get("full_metadata") or item.get("metadata")
|
||||||
|
for key in ("url", "source_url", "target"):
|
||||||
|
candidate = _http_candidate(item.get(key))
|
||||||
|
if candidate:
|
||||||
|
return candidate
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
metadata = getattr(item, "full_metadata", None) or getattr(item, "metadata", None)
|
||||||
|
except Exception:
|
||||||
|
metadata = None
|
||||||
|
for key in ("url", "source_url", "target"):
|
||||||
|
try:
|
||||||
|
candidate = _http_candidate(getattr(item, key, None))
|
||||||
|
except Exception:
|
||||||
|
candidate = None
|
||||||
|
if candidate:
|
||||||
|
return candidate
|
||||||
|
|
||||||
|
if not isinstance(metadata, dict):
|
||||||
|
return None
|
||||||
|
|
||||||
|
for key in (
|
||||||
|
"_tidal_manifest_url",
|
||||||
|
"streamUrl",
|
||||||
|
"audioUrl",
|
||||||
|
"assetUrl",
|
||||||
|
"playbackUrl",
|
||||||
|
"manifestUrl",
|
||||||
|
"manifestURL",
|
||||||
|
"url",
|
||||||
|
):
|
||||||
|
candidate = _http_candidate(metadata.get(key))
|
||||||
|
if candidate:
|
||||||
|
return candidate
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _get_playable_path(
|
def _get_playable_path(
|
||||||
item: Any,
|
item: Any,
|
||||||
file_storage: Optional[Any],
|
file_storage: Optional[Any],
|
||||||
@@ -950,9 +1007,26 @@ def _get_playable_path(
|
|||||||
if manifest_path:
|
if manifest_path:
|
||||||
path = manifest_path
|
path = manifest_path
|
||||||
else:
|
else:
|
||||||
# If this is a tidal:// placeholder and we couldn't resolve a manifest, do not fall back.
|
# If this is a tidal:// placeholder and we couldn't resolve a manifest, do not fall back —
|
||||||
|
# UNLESS the item has already been stored in a backend (store+hash present), in which case
|
||||||
|
# we clear the tidal:// path so the store-resolution logic below can build a playable URL.
|
||||||
try:
|
try:
|
||||||
if isinstance(path, str) and path.strip().lower().startswith("tidal:"):
|
if isinstance(path, str) and path.strip().lower().startswith("tidal:"):
|
||||||
|
if store and file_hash and str(file_hash).strip().lower() not in ("", "unknown"):
|
||||||
|
# Item is stored in a backend — clear the tidal:// placeholder and let
|
||||||
|
# the hash+store resolution further below build the real playable URL.
|
||||||
|
path = None
|
||||||
|
else:
|
||||||
|
fallback_stream_url = _extract_tidal_stream_fallback_url(item)
|
||||||
|
if fallback_stream_url:
|
||||||
|
path = fallback_stream_url
|
||||||
|
try:
|
||||||
|
debug(
|
||||||
|
f"_get_playable_path: using fallback Tidal stream URL {fallback_stream_url}"
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
try:
|
try:
|
||||||
meta = None
|
meta = None
|
||||||
if isinstance(item, dict):
|
if isinstance(item, dict):
|
||||||
|
|||||||
Reference in New Issue
Block a user