diff --git a/API/data/alldebrid.json b/API/data/alldebrid.json index f883242..58ea42b 100644 --- a/API/data/alldebrid.json +++ b/API/data/alldebrid.json @@ -92,7 +92,7 @@ "(hitfile\\.net/[a-z0-9A-Z]{4,9})" ], "regexp": "(hitf\\.(to|cc)/([a-z0-9A-Z]{4,9}))|(htfl\\.(net|to|cc)/([a-z0-9A-Z]{4,9}))|(hitfile\\.(net)/download/free/([a-z0-9A-Z]{4,9}))|((hitfile\\.net/[a-z0-9A-Z]{4,9}))", - "status": false + "status": true }, "mega": { "name": "mega", @@ -375,7 +375,7 @@ "(filespace\\.com/[a-zA-Z0-9]{12})" ], "regexp": "(filespace\\.com/fd/([a-zA-Z0-9]{12}))|((filespace\\.com/[a-zA-Z0-9]{12}))", - "status": false + "status": true }, "filezip": { "name": "filezip", @@ -689,7 +689,7 @@ "uploadrar\\.(net|com)/([0-9a-z]{12})" ], "regexp": "((get|cloud)\\.rahim-soft\\.com/([0-9a-z]{12}))|((fingau\\.com/([0-9a-z]{12})))|((tech|miui|cloud|flash)\\.getpczone\\.com/([0-9a-z]{12}))|(miui.rahim-soft\\.com/([0-9a-z]{12}))|(uploadrar\\.(net|com)/([0-9a-z]{12}))", - "status": true, + "status": false, "hardRedirect": [ "uploadrar.com/([0-9a-zA-Z]{12})" ] diff --git a/Provider/matrix.py b/Provider/matrix.py index 2f9de50..dd877d8 100644 --- a/Provider/matrix.py +++ b/Provider/matrix.py @@ -9,6 +9,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple from urllib.parse import quote from API.requests_client import get_requests_session +from SYS.utils import ffprobe as probe_media_metadata from ProviderCore.base import Provider, SearchResult from SYS.provider_helpers import TableProviderMixin @@ -166,6 +167,71 @@ def _classify_matrix_upload(path: Path, return (mime_type or "application/octet-stream"), msgtype +def _build_matrix_media_info(path: Path, *, mime_type: str, msgtype: str) -> Dict[str, Any]: + """Build Matrix `info` payload with dimensions/duration when available. + + Matrix clients use `info.w`/`info.h` to size image/video placeholders. Supplying + these fields keeps the preview aspect ratio aligned with the actual media. + """ + info: Dict[str, Any] = { + "mimetype": str(mime_type or "application/octet-stream"), + "size": int(path.stat().st_size), + } + + metadata: Dict[str, Any] = {} + try: + metadata = probe_media_metadata(str(path)) or {} + except Exception: + metadata = {} + + width: Optional[int] = None + height: Optional[int] = None + + try: + raw_w = metadata.get("width") if isinstance(metadata, dict) else None + if isinstance(raw_w, (int, float)) and raw_w > 0: + width = int(raw_w) + except Exception: + width = None + + try: + raw_h = metadata.get("height") if isinstance(metadata, dict) else None + if isinstance(raw_h, (int, float)) and raw_h > 0: + height = int(raw_h) + except Exception: + height = None + + # Fallback for images when ffprobe metadata is unavailable. + if msgtype == "m.image" and (width is None or height is None): + try: + from PIL import Image # type: ignore + + with Image.open(path) as img: + iw, ih = img.size + if isinstance(iw, int) and iw > 0: + width = iw + if isinstance(ih, int) and ih > 0: + height = ih + except Exception: + pass + + if msgtype in {"m.image", "m.video"}: + if width is not None: + info["w"] = width + if height is not None: + info["h"] = height + + if msgtype in {"m.video", "m.audio"}: + try: + raw_duration = metadata.get("duration") if isinstance(metadata, dict) else None + if isinstance(raw_duration, (int, float)) and raw_duration > 0: + info["duration"] = int(round(float(raw_duration) * 1000.0)) + except Exception: + pass + + return info + + def _normalize_homeserver(value: str) -> str: text = str(value or "").strip() if not text: @@ -523,10 +589,7 @@ class Matrix(TableProviderMixin, Provider): except Exception: download_url_for_store = "" - info = { - "mimetype": mime_type, - "size": path.stat().st_size - } + info = _build_matrix_media_info(path, mime_type=mime_type, msgtype=msgtype) payload = { "msgtype": msgtype, "body": filename, diff --git a/SYS/pipeline.py b/SYS/pipeline.py index 513f299..b3505de 100644 --- a/SYS/pipeline.py +++ b/SYS/pipeline.py @@ -1903,31 +1903,31 @@ class PipelineExecutor: else: cmd_list = [] - # IMPORTANT: Put selected row args *before* source_args. - # Rationale: The cmdlet argument parser treats the *first* unknown - # token as a positional value (e.g., URL). If `source_args` - # contain unknown flags (like -provider which download-file does - # not declare), they could be misinterpreted as the positional - # URL argument and cause attempts to download strings like - # "-provider" (which is invalid). By placing selection args - # first we ensure the intended URL/selection token is parsed - # as the positional URL and avoid this class of parsing errors. - expanded_stage: List[str] = cmd_list + selected_row_args + source_args + # IMPORTANT: Put selected row args *before* source_args. + # Rationale: The cmdlet argument parser treats the *first* unknown + # token as a positional value (e.g., URL). If `source_args` + # contain unknown flags (like -provider which download-file does + # not declare), they could be misinterpreted as the positional + # URL argument and cause attempts to download strings like + # "-provider" (which is invalid). By placing selection args + # first we ensure the intended URL/selection token is parsed + # as the positional URL and avoid this class of parsing errors. + expanded_stage: List[str] = cmd_list + selected_row_args + source_args - if first_stage_had_extra_args and stages: - expanded_stage += stages[0] - stages[0] = expanded_stage - else: - stages.insert(0, expanded_stage) + if first_stage_had_extra_args and stages: + expanded_stage += stages[0] + stages[0] = expanded_stage + else: + stages.insert(0, expanded_stage) - if pipeline_session and worker_manager: - try: - worker_manager.log_step( - pipeline_session.worker_id, - f"@N expansion: {source_cmd} + selected_args={selected_row_args} + source_args={source_args}", - ) - except Exception: - logger.exception("Failed to record pipeline log step for @N expansion (pipeline_session=%r)", getattr(pipeline_session, 'worker_id', None)) + if pipeline_session and worker_manager: + try: + worker_manager.log_step( + pipeline_session.worker_id, + f"@N expansion: {source_cmd} + selected_args={selected_row_args} + source_args={source_args}", + ) + except Exception: + logger.exception("Failed to record pipeline log step for @N expansion (pipeline_session=%r)", getattr(pipeline_session, 'worker_id', None)) stage_table = None try: diff --git a/tool/ytdlp.py b/tool/ytdlp.py index 047fd93..142183f 100644 --- a/tool/ytdlp.py +++ b/tool/ytdlp.py @@ -1480,41 +1480,136 @@ def _download_with_sections_via_cli( pipeline = PipelineProgress(pipeline_context) - class _SectionProgressSimulator: - def __init__(self, start_pct: int, max_pct: int, interval: float = 0.5) -> None: - self._start_pct = max(0, min(int(start_pct), 99)) - self._max_pct = max(self._start_pct, min(int(max_pct), 98)) - self._interval = max(0.1, float(interval)) - self._stop_event = threading.Event() - self._thread: Optional[threading.Thread] = None + section_hms_re = re.compile(r"\*?(\d{2}:\d{2}:\d{2})-(\d{2}:\d{2}:\d{2})") + ytdlp_pct_re = re.compile(r"(?P\d{1,3}(?:\.\d+)?)%") + ffmpeg_time_re = re.compile(r"time=(?P\d{2}):(?P\d{2}):(?P\d{2}(?:\.\d+)?)") - def _run(self) -> None: - current = self._start_pct - while not self._stop_event.wait(self._interval): - if current < self._max_pct: - current += 1 - try: - _set_pipe_percent(current) - except Exception: - from SYS.logger import logger - logger.exception("Failed to set pipeline percent to %d", current) + def _hms_to_seconds(hms: str) -> Optional[float]: + try: + hh, mm, ss = str(hms).split(":", 2) + return float(int(hh) * 3600 + int(mm) * 60 + float(ss)) + except Exception: + return None - def start(self) -> None: - if self._thread is not None or self._start_pct >= self._max_pct: + def _section_duration_seconds(section_text: str) -> Optional[float]: + match = section_hms_re.search(str(section_text or "").strip()) + if not match: + return None + start_sec = _hms_to_seconds(match.group(1)) + end_sec = _hms_to_seconds(match.group(2)) + if start_sec is None or end_sec is None: + return None + duration = float(end_sec) - float(start_sec) + if duration <= 0: + return None + return duration + + def _safe_set_percent(percent: int) -> None: + try: + _set_pipe_percent(max(0, min(int(percent), 99))) + except Exception: + from SYS.logger import logger + logger.exception("Failed to set pipeline percent to %s", percent) + + def _run_section_command( + cmd: List[str], + *, + section_idx: int, + total_sections: int, + section_label: str, + section_duration: Optional[float], + ) -> None: + section_start_pct = int(((section_idx - 1) / max(1, total_sections)) * 99) + section_end_pct = int((section_idx / max(1, total_sections)) * 99) + if section_end_pct <= section_start_pct: + section_end_pct = min(99, section_start_pct + 1) + section_span = max(1, section_end_pct - section_start_pct) + + observed_progress = 0.0 + + def _apply_progress(progress01: float, phase: str) -> None: + nonlocal observed_progress + p = max(0.0, min(float(progress01), 1.0)) + if p <= observed_progress: return - self._thread = threading.Thread(target=self._run, daemon=True) - self._thread.start() - - def stop(self) -> None: - self._stop_event.set() - if self._thread is not None: - self._thread.join(timeout=0.5) - self._thread = None + observed_progress = p + mapped = section_start_pct + int(round(section_span * p)) + _safe_set_percent(mapped) try: - _set_pipe_percent(self._max_pct) + if phase == "clip": + pipeline.set_status(f"Clipping section {section_idx}/{total_sections} ({int(round(p * 100))}%)") + else: + pipeline.set_status(f"Downloading section {section_idx}/{total_sections} ({int(round(p * 100))}%)") except Exception: - from SYS.logger import logger - logger.exception("Failed to set pipeline percent to max %d", self._max_pct) + pass + + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", + bufsize=1, + ) + + assert proc.stdout is not None + tail_lines: List[str] = [] + try: + _record_progress_activity() + for raw_line in proc.stdout: + line = str(raw_line or "").strip() + if not line: + continue + + _record_progress_activity() + + if len(tail_lines) >= 120: + tail_lines.pop(0) + tail_lines.append(line) + + m_pct = ytdlp_pct_re.search(line) + if m_pct is not None: + try: + pct_value = float(m_pct.group("pct")) + except Exception: + pct_value = 0.0 + # Reserve final 15% for ffmpeg clipping phase if present. + _apply_progress(min(0.85, max(0.0, pct_value / 100.0) * 0.85), "download") + continue + + m_time = ffmpeg_time_re.search(line) + if m_time is not None and section_duration and section_duration > 0: + try: + elapsed = ( + int(m_time.group("h")) * 3600 + + int(m_time.group("m")) * 60 + + float(m_time.group("s")) + ) + except Exception: + elapsed = 0.0 + clip_progress = min(1.0, max(0.0, float(elapsed) / float(section_duration))) + _apply_progress(0.85 + (clip_progress * 0.15), "clip") + continue + + # Non-empty output line from yt-dlp/ffmpeg still means the process is active. + _apply_progress(min(0.98, observed_progress + 0.002), "download") + + return_code = proc.wait() + if return_code != 0: + tail = "\n".join(tail_lines[-12:]).strip() + details = f"\n{tail}" if tail else "" + raise DownloadError( + f"yt-dlp failed for section {section_label} (exit {return_code}){details}" + ) + _apply_progress(1.0, "clip") + _record_progress_activity() + finally: + try: + if proc.poll() is None: + proc.kill() + except Exception: + pass session_id = hashlib.md5((url + str(time.time()) + "".join(random.choices(string.ascii_letters, k=10))).encode()).hexdigest()[:12] first_section_info = None @@ -1522,16 +1617,8 @@ def _download_with_sections_via_cli( total_sections = len(sections_list) try: for section_idx, section in enumerate(sections_list, 1): - display_pct = 50 - if total_sections > 0: - display_pct = 50 + int(((section_idx - 1) / max(1, total_sections)) * 49) - try: - _set_pipe_percent(display_pct) - except Exception: - from SYS.logger import logger - logger.exception("Failed to set pipeline percent to display_pct %d for section %d", display_pct, section_idx) - pipeline.set_status(f"Downloading & clipping clip section {section_idx}/{total_sections}") + section_duration = _section_duration_seconds(section) base_outtmpl = ytdl_options.get("outtmpl", "%(title)s.%(ext)s") output_dir_path = Path(base_outtmpl).parent @@ -1565,10 +1652,10 @@ def _download_with_sections_via_cli( cmd = ["yt-dlp"] if quiet: - cmd.append("--quiet") cmd.append("--no-warnings") - cmd.append("--no-progress") - cmd.extend(["--postprocessor-args", "ffmpeg:-hide_banner -loglevel error"]) + # Emit line-oriented progress and capture it for real clip activity tracking. + cmd.append("--newline") + cmd.extend(["--postprocessor-args", "ffmpeg:-hide_banner -nostdin -stats"]) if ytdl_options.get("ffmpeg_location"): try: cmd.extend(["--ffmpeg-location", str(ytdl_options["ffmpeg_location"])]) @@ -1614,23 +1701,13 @@ def _download_with_sections_via_cli( if not quiet: debug(f"Running yt-dlp for section: {section}") - progress_end_pct = min(display_pct + 45, 98) - simulator = _SectionProgressSimulator(display_pct, progress_end_pct) - simulator.start() - try: - if quiet: - subprocess.run(cmd, check=True, capture_output=True, text=True) - else: - subprocess.run(cmd, check=True) - except subprocess.CalledProcessError as exc: - stderr_text = exc.stderr or "" - tail = "\n".join(stderr_text.splitlines()[-12:]).strip() - details = f"\n{tail}" if tail else "" - raise DownloadError(f"yt-dlp failed for section {section} (exit {exc.returncode}){details}") from exc - except Exception as exc: - raise DownloadError(f"yt-dlp failed for section {section}: {exc}") from exc - finally: - simulator.stop() + _run_section_command( + cmd, + section_idx=section_idx, + total_sections=total_sections, + section_label=section, + section_duration=section_duration, + ) finally: pipeline.clear_status()