This commit is contained in:
2026-02-18 13:59:45 -08:00
parent eab6ded855
commit 615a4fd1a4
4 changed files with 229 additions and 89 deletions

View File

@@ -92,7 +92,7 @@
"(hitfile\\.net/[a-z0-9A-Z]{4,9})" "(hitfile\\.net/[a-z0-9A-Z]{4,9})"
], ],
"regexp": "(hitf\\.(to|cc)/([a-z0-9A-Z]{4,9}))|(htfl\\.(net|to|cc)/([a-z0-9A-Z]{4,9}))|(hitfile\\.(net)/download/free/([a-z0-9A-Z]{4,9}))|((hitfile\\.net/[a-z0-9A-Z]{4,9}))", "regexp": "(hitf\\.(to|cc)/([a-z0-9A-Z]{4,9}))|(htfl\\.(net|to|cc)/([a-z0-9A-Z]{4,9}))|(hitfile\\.(net)/download/free/([a-z0-9A-Z]{4,9}))|((hitfile\\.net/[a-z0-9A-Z]{4,9}))",
"status": false "status": true
}, },
"mega": { "mega": {
"name": "mega", "name": "mega",
@@ -375,7 +375,7 @@
"(filespace\\.com/[a-zA-Z0-9]{12})" "(filespace\\.com/[a-zA-Z0-9]{12})"
], ],
"regexp": "(filespace\\.com/fd/([a-zA-Z0-9]{12}))|((filespace\\.com/[a-zA-Z0-9]{12}))", "regexp": "(filespace\\.com/fd/([a-zA-Z0-9]{12}))|((filespace\\.com/[a-zA-Z0-9]{12}))",
"status": false "status": true
}, },
"filezip": { "filezip": {
"name": "filezip", "name": "filezip",
@@ -689,7 +689,7 @@
"uploadrar\\.(net|com)/([0-9a-z]{12})" "uploadrar\\.(net|com)/([0-9a-z]{12})"
], ],
"regexp": "((get|cloud)\\.rahim-soft\\.com/([0-9a-z]{12}))|((fingau\\.com/([0-9a-z]{12})))|((tech|miui|cloud|flash)\\.getpczone\\.com/([0-9a-z]{12}))|(miui.rahim-soft\\.com/([0-9a-z]{12}))|(uploadrar\\.(net|com)/([0-9a-z]{12}))", "regexp": "((get|cloud)\\.rahim-soft\\.com/([0-9a-z]{12}))|((fingau\\.com/([0-9a-z]{12})))|((tech|miui|cloud|flash)\\.getpczone\\.com/([0-9a-z]{12}))|(miui.rahim-soft\\.com/([0-9a-z]{12}))|(uploadrar\\.(net|com)/([0-9a-z]{12}))",
"status": true, "status": false,
"hardRedirect": [ "hardRedirect": [
"uploadrar.com/([0-9a-zA-Z]{12})" "uploadrar.com/([0-9a-zA-Z]{12})"
] ]

View File

@@ -9,6 +9,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple
from urllib.parse import quote from urllib.parse import quote
from API.requests_client import get_requests_session from API.requests_client import get_requests_session
from SYS.utils import ffprobe as probe_media_metadata
from ProviderCore.base import Provider, SearchResult from ProviderCore.base import Provider, SearchResult
from SYS.provider_helpers import TableProviderMixin from SYS.provider_helpers import TableProviderMixin
@@ -166,6 +167,71 @@ def _classify_matrix_upload(path: Path,
return (mime_type or "application/octet-stream"), msgtype return (mime_type or "application/octet-stream"), msgtype
def _build_matrix_media_info(path: Path, *, mime_type: str, msgtype: str) -> Dict[str, Any]:
"""Build Matrix `info` payload with dimensions/duration when available.
Matrix clients use `info.w`/`info.h` to size image/video placeholders. Supplying
these fields keeps the preview aspect ratio aligned with the actual media.
"""
info: Dict[str, Any] = {
"mimetype": str(mime_type or "application/octet-stream"),
"size": int(path.stat().st_size),
}
metadata: Dict[str, Any] = {}
try:
metadata = probe_media_metadata(str(path)) or {}
except Exception:
metadata = {}
width: Optional[int] = None
height: Optional[int] = None
try:
raw_w = metadata.get("width") if isinstance(metadata, dict) else None
if isinstance(raw_w, (int, float)) and raw_w > 0:
width = int(raw_w)
except Exception:
width = None
try:
raw_h = metadata.get("height") if isinstance(metadata, dict) else None
if isinstance(raw_h, (int, float)) and raw_h > 0:
height = int(raw_h)
except Exception:
height = None
# Fallback for images when ffprobe metadata is unavailable.
if msgtype == "m.image" and (width is None or height is None):
try:
from PIL import Image # type: ignore
with Image.open(path) as img:
iw, ih = img.size
if isinstance(iw, int) and iw > 0:
width = iw
if isinstance(ih, int) and ih > 0:
height = ih
except Exception:
pass
if msgtype in {"m.image", "m.video"}:
if width is not None:
info["w"] = width
if height is not None:
info["h"] = height
if msgtype in {"m.video", "m.audio"}:
try:
raw_duration = metadata.get("duration") if isinstance(metadata, dict) else None
if isinstance(raw_duration, (int, float)) and raw_duration > 0:
info["duration"] = int(round(float(raw_duration) * 1000.0))
except Exception:
pass
return info
def _normalize_homeserver(value: str) -> str: def _normalize_homeserver(value: str) -> str:
text = str(value or "").strip() text = str(value or "").strip()
if not text: if not text:
@@ -523,10 +589,7 @@ class Matrix(TableProviderMixin, Provider):
except Exception: except Exception:
download_url_for_store = "" download_url_for_store = ""
info = { info = _build_matrix_media_info(path, mime_type=mime_type, msgtype=msgtype)
"mimetype": mime_type,
"size": path.stat().st_size
}
payload = { payload = {
"msgtype": msgtype, "msgtype": msgtype,
"body": filename, "body": filename,

View File

@@ -1480,41 +1480,136 @@ def _download_with_sections_via_cli(
pipeline = PipelineProgress(pipeline_context) pipeline = PipelineProgress(pipeline_context)
class _SectionProgressSimulator: section_hms_re = re.compile(r"\*?(\d{2}:\d{2}:\d{2})-(\d{2}:\d{2}:\d{2})")
def __init__(self, start_pct: int, max_pct: int, interval: float = 0.5) -> None: ytdlp_pct_re = re.compile(r"(?P<pct>\d{1,3}(?:\.\d+)?)%")
self._start_pct = max(0, min(int(start_pct), 99)) ffmpeg_time_re = re.compile(r"time=(?P<h>\d{2}):(?P<m>\d{2}):(?P<s>\d{2}(?:\.\d+)?)")
self._max_pct = max(self._start_pct, min(int(max_pct), 98))
self._interval = max(0.1, float(interval))
self._stop_event = threading.Event()
self._thread: Optional[threading.Thread] = None
def _run(self) -> None: def _hms_to_seconds(hms: str) -> Optional[float]:
current = self._start_pct
while not self._stop_event.wait(self._interval):
if current < self._max_pct:
current += 1
try: try:
_set_pipe_percent(current) hh, mm, ss = str(hms).split(":", 2)
return float(int(hh) * 3600 + int(mm) * 60 + float(ss))
except Exception:
return None
def _section_duration_seconds(section_text: str) -> Optional[float]:
match = section_hms_re.search(str(section_text or "").strip())
if not match:
return None
start_sec = _hms_to_seconds(match.group(1))
end_sec = _hms_to_seconds(match.group(2))
if start_sec is None or end_sec is None:
return None
duration = float(end_sec) - float(start_sec)
if duration <= 0:
return None
return duration
def _safe_set_percent(percent: int) -> None:
try:
_set_pipe_percent(max(0, min(int(percent), 99)))
except Exception: except Exception:
from SYS.logger import logger from SYS.logger import logger
logger.exception("Failed to set pipeline percent to %d", current) logger.exception("Failed to set pipeline percent to %s", percent)
def start(self) -> None: def _run_section_command(
if self._thread is not None or self._start_pct >= self._max_pct: cmd: List[str],
*,
section_idx: int,
total_sections: int,
section_label: str,
section_duration: Optional[float],
) -> None:
section_start_pct = int(((section_idx - 1) / max(1, total_sections)) * 99)
section_end_pct = int((section_idx / max(1, total_sections)) * 99)
if section_end_pct <= section_start_pct:
section_end_pct = min(99, section_start_pct + 1)
section_span = max(1, section_end_pct - section_start_pct)
observed_progress = 0.0
def _apply_progress(progress01: float, phase: str) -> None:
nonlocal observed_progress
p = max(0.0, min(float(progress01), 1.0))
if p <= observed_progress:
return return
self._thread = threading.Thread(target=self._run, daemon=True) observed_progress = p
self._thread.start() mapped = section_start_pct + int(round(section_span * p))
_safe_set_percent(mapped)
def stop(self) -> None:
self._stop_event.set()
if self._thread is not None:
self._thread.join(timeout=0.5)
self._thread = None
try: try:
_set_pipe_percent(self._max_pct) if phase == "clip":
pipeline.set_status(f"Clipping section {section_idx}/{total_sections} ({int(round(p * 100))}%)")
else:
pipeline.set_status(f"Downloading section {section_idx}/{total_sections} ({int(round(p * 100))}%)")
except Exception: except Exception:
from SYS.logger import logger pass
logger.exception("Failed to set pipeline percent to max %d", self._max_pct)
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
encoding="utf-8",
errors="replace",
bufsize=1,
)
assert proc.stdout is not None
tail_lines: List[str] = []
try:
_record_progress_activity()
for raw_line in proc.stdout:
line = str(raw_line or "").strip()
if not line:
continue
_record_progress_activity()
if len(tail_lines) >= 120:
tail_lines.pop(0)
tail_lines.append(line)
m_pct = ytdlp_pct_re.search(line)
if m_pct is not None:
try:
pct_value = float(m_pct.group("pct"))
except Exception:
pct_value = 0.0
# Reserve final 15% for ffmpeg clipping phase if present.
_apply_progress(min(0.85, max(0.0, pct_value / 100.0) * 0.85), "download")
continue
m_time = ffmpeg_time_re.search(line)
if m_time is not None and section_duration and section_duration > 0:
try:
elapsed = (
int(m_time.group("h")) * 3600
+ int(m_time.group("m")) * 60
+ float(m_time.group("s"))
)
except Exception:
elapsed = 0.0
clip_progress = min(1.0, max(0.0, float(elapsed) / float(section_duration)))
_apply_progress(0.85 + (clip_progress * 0.15), "clip")
continue
# Non-empty output line from yt-dlp/ffmpeg still means the process is active.
_apply_progress(min(0.98, observed_progress + 0.002), "download")
return_code = proc.wait()
if return_code != 0:
tail = "\n".join(tail_lines[-12:]).strip()
details = f"\n{tail}" if tail else ""
raise DownloadError(
f"yt-dlp failed for section {section_label} (exit {return_code}){details}"
)
_apply_progress(1.0, "clip")
_record_progress_activity()
finally:
try:
if proc.poll() is None:
proc.kill()
except Exception:
pass
session_id = hashlib.md5((url + str(time.time()) + "".join(random.choices(string.ascii_letters, k=10))).encode()).hexdigest()[:12] session_id = hashlib.md5((url + str(time.time()) + "".join(random.choices(string.ascii_letters, k=10))).encode()).hexdigest()[:12]
first_section_info = None first_section_info = None
@@ -1522,16 +1617,8 @@ def _download_with_sections_via_cli(
total_sections = len(sections_list) total_sections = len(sections_list)
try: try:
for section_idx, section in enumerate(sections_list, 1): for section_idx, section in enumerate(sections_list, 1):
display_pct = 50
if total_sections > 0:
display_pct = 50 + int(((section_idx - 1) / max(1, total_sections)) * 49)
try:
_set_pipe_percent(display_pct)
except Exception:
from SYS.logger import logger
logger.exception("Failed to set pipeline percent to display_pct %d for section %d", display_pct, section_idx)
pipeline.set_status(f"Downloading & clipping clip section {section_idx}/{total_sections}") pipeline.set_status(f"Downloading & clipping clip section {section_idx}/{total_sections}")
section_duration = _section_duration_seconds(section)
base_outtmpl = ytdl_options.get("outtmpl", "%(title)s.%(ext)s") base_outtmpl = ytdl_options.get("outtmpl", "%(title)s.%(ext)s")
output_dir_path = Path(base_outtmpl).parent output_dir_path = Path(base_outtmpl).parent
@@ -1565,10 +1652,10 @@ def _download_with_sections_via_cli(
cmd = ["yt-dlp"] cmd = ["yt-dlp"]
if quiet: if quiet:
cmd.append("--quiet")
cmd.append("--no-warnings") cmd.append("--no-warnings")
cmd.append("--no-progress") # Emit line-oriented progress and capture it for real clip activity tracking.
cmd.extend(["--postprocessor-args", "ffmpeg:-hide_banner -loglevel error"]) cmd.append("--newline")
cmd.extend(["--postprocessor-args", "ffmpeg:-hide_banner -nostdin -stats"])
if ytdl_options.get("ffmpeg_location"): if ytdl_options.get("ffmpeg_location"):
try: try:
cmd.extend(["--ffmpeg-location", str(ytdl_options["ffmpeg_location"])]) cmd.extend(["--ffmpeg-location", str(ytdl_options["ffmpeg_location"])])
@@ -1614,23 +1701,13 @@ def _download_with_sections_via_cli(
if not quiet: if not quiet:
debug(f"Running yt-dlp for section: {section}") debug(f"Running yt-dlp for section: {section}")
progress_end_pct = min(display_pct + 45, 98) _run_section_command(
simulator = _SectionProgressSimulator(display_pct, progress_end_pct) cmd,
simulator.start() section_idx=section_idx,
try: total_sections=total_sections,
if quiet: section_label=section,
subprocess.run(cmd, check=True, capture_output=True, text=True) section_duration=section_duration,
else: )
subprocess.run(cmd, check=True)
except subprocess.CalledProcessError as exc:
stderr_text = exc.stderr or ""
tail = "\n".join(stderr_text.splitlines()[-12:]).strip()
details = f"\n{tail}" if tail else ""
raise DownloadError(f"yt-dlp failed for section {section} (exit {exc.returncode}){details}") from exc
except Exception as exc:
raise DownloadError(f"yt-dlp failed for section {section}: {exc}") from exc
finally:
simulator.stop()
finally: finally:
pipeline.clear_status() pipeline.clear_status()