hh

tool/ytdlp.py

@@ -1480,41 +1480,136 @@ def _download_with_sections_via_cli(
 
     pipeline = PipelineProgress(pipeline_context)
 
-    class _SectionProgressSimulator:
-        def __init__(self, start_pct: int, max_pct: int, interval: float = 0.5) -> None:
-            self._start_pct = max(0, min(int(start_pct), 99))
-            self._max_pct = max(self._start_pct, min(int(max_pct), 98))
-            self._interval = max(0.1, float(interval))
-            self._stop_event = threading.Event()
-            self._thread: Optional[threading.Thread] = None
+    section_hms_re = re.compile(r"\*?(\d{2}:\d{2}:\d{2})-(\d{2}:\d{2}:\d{2})")
+    ytdlp_pct_re = re.compile(r"(?P<pct>\d{1,3}(?:\.\d+)?)%")
+    ffmpeg_time_re = re.compile(r"time=(?P<h>\d{2}):(?P<m>\d{2}):(?P<s>\d{2}(?:\.\d+)?)")
 
-        def _run(self) -> None:
-            current = self._start_pct
-            while not self._stop_event.wait(self._interval):
-                if current < self._max_pct:
-                    current += 1
-                    try:
-                        _set_pipe_percent(current)
-                    except Exception:
-                        from SYS.logger import logger
-                        logger.exception("Failed to set pipeline percent to %d", current)
+    def _hms_to_seconds(hms: str) -> Optional[float]:
+        try:
+            hh, mm, ss = str(hms).split(":", 2)
+            return float(int(hh) * 3600 + int(mm) * 60 + float(ss))
+        except Exception:
+            return None
 
-        def start(self) -> None:
-            if self._thread is not None or self._start_pct >= self._max_pct:
+    def _section_duration_seconds(section_text: str) -> Optional[float]:
+        match = section_hms_re.search(str(section_text or "").strip())
+        if not match:
+            return None
+        start_sec = _hms_to_seconds(match.group(1))
+        end_sec = _hms_to_seconds(match.group(2))
+        if start_sec is None or end_sec is None:
+            return None
+        duration = float(end_sec) - float(start_sec)
+        if duration <= 0:
+            return None
+        return duration
+
+    def _safe_set_percent(percent: int) -> None:
+        try:
+            _set_pipe_percent(max(0, min(int(percent), 99)))
+        except Exception:
+            from SYS.logger import logger
+            logger.exception("Failed to set pipeline percent to %s", percent)
+
+    def _run_section_command(
+        cmd: List[str],
+        *,
+        section_idx: int,
+        total_sections: int,
+        section_label: str,
+        section_duration: Optional[float],
+    ) -> None:
+        section_start_pct = int(((section_idx - 1) / max(1, total_sections)) * 99)
+        section_end_pct = int((section_idx / max(1, total_sections)) * 99)
+        if section_end_pct <= section_start_pct:
+            section_end_pct = min(99, section_start_pct + 1)
+        section_span = max(1, section_end_pct - section_start_pct)
+
+        observed_progress = 0.0
+
+        def _apply_progress(progress01: float, phase: str) -> None:
+            nonlocal observed_progress
+            p = max(0.0, min(float(progress01), 1.0))
+            if p <= observed_progress:
                 return
-            self._thread = threading.Thread(target=self._run, daemon=True)
-            self._thread.start()
-
-        def stop(self) -> None:
-            self._stop_event.set()
-            if self._thread is not None:
-                self._thread.join(timeout=0.5)
-                self._thread = None
+            observed_progress = p
+            mapped = section_start_pct + int(round(section_span * p))
+            _safe_set_percent(mapped)
             try:
-                _set_pipe_percent(self._max_pct)
+                if phase == "clip":
+                    pipeline.set_status(f"Clipping section {section_idx}/{total_sections} ({int(round(p * 100))}%)")
+                else:
+                    pipeline.set_status(f"Downloading section {section_idx}/{total_sections} ({int(round(p * 100))}%)")
             except Exception:
-                from SYS.logger import logger
-                logger.exception("Failed to set pipeline percent to max %d", self._max_pct)
+                pass
+
+        proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            bufsize=1,
+        )
+
+        assert proc.stdout is not None
+        tail_lines: List[str] = []
+        try:
+            _record_progress_activity()
+            for raw_line in proc.stdout:
+                line = str(raw_line or "").strip()
+                if not line:
+                    continue
+
+                _record_progress_activity()
+
+                if len(tail_lines) >= 120:
+                    tail_lines.pop(0)
+                tail_lines.append(line)
+
+                m_pct = ytdlp_pct_re.search(line)
+                if m_pct is not None:
+                    try:
+                        pct_value = float(m_pct.group("pct"))
+                    except Exception:
+                        pct_value = 0.0
+                    # Reserve final 15% for ffmpeg clipping phase if present.
+                    _apply_progress(min(0.85, max(0.0, pct_value / 100.0) * 0.85), "download")
+                    continue
+
+                m_time = ffmpeg_time_re.search(line)
+                if m_time is not None and section_duration and section_duration > 0:
+                    try:
+                        elapsed = (
+                            int(m_time.group("h")) * 3600
+                            + int(m_time.group("m")) * 60
+                            + float(m_time.group("s"))
+                        )
+                    except Exception:
+                        elapsed = 0.0
+                    clip_progress = min(1.0, max(0.0, float(elapsed) / float(section_duration)))
+                    _apply_progress(0.85 + (clip_progress * 0.15), "clip")
+                    continue
+
+                # Non-empty output line from yt-dlp/ffmpeg still means the process is active.
+                _apply_progress(min(0.98, observed_progress + 0.002), "download")
+
+            return_code = proc.wait()
+            if return_code != 0:
+                tail = "\n".join(tail_lines[-12:]).strip()
+                details = f"\n{tail}" if tail else ""
+                raise DownloadError(
+                    f"yt-dlp failed for section {section_label} (exit {return_code}){details}"
+                )
+            _apply_progress(1.0, "clip")
+            _record_progress_activity()
+        finally:
+            try:
+                if proc.poll() is None:
+                    proc.kill()
+            except Exception:
+                pass
 
     session_id = hashlib.md5((url + str(time.time()) + "".join(random.choices(string.ascii_letters, k=10))).encode()).hexdigest()[:12]
     first_section_info = None
@@ -1522,16 +1617,8 @@ def _download_with_sections_via_cli(
     total_sections = len(sections_list)
     try:
         for section_idx, section in enumerate(sections_list, 1):
-            display_pct = 50
-            if total_sections > 0:
-                display_pct = 50 + int(((section_idx - 1) / max(1, total_sections)) * 49)
-            try:
-                _set_pipe_percent(display_pct)
-            except Exception:
-                from SYS.logger import logger
-                logger.exception("Failed to set pipeline percent to display_pct %d for section %d", display_pct, section_idx)
-
             pipeline.set_status(f"Downloading & clipping clip section {section_idx}/{total_sections}")
+            section_duration = _section_duration_seconds(section)
 
             base_outtmpl = ytdl_options.get("outtmpl", "%(title)s.%(ext)s")
             output_dir_path = Path(base_outtmpl).parent
@@ -1565,10 +1652,10 @@ def _download_with_sections_via_cli(
 
             cmd = ["yt-dlp"]
             if quiet:
-                cmd.append("--quiet")
                 cmd.append("--no-warnings")
-                cmd.append("--no-progress")
-                cmd.extend(["--postprocessor-args", "ffmpeg:-hide_banner -loglevel error"])
+            # Emit line-oriented progress and capture it for real clip activity tracking.
+            cmd.append("--newline")
+            cmd.extend(["--postprocessor-args", "ffmpeg:-hide_banner -nostdin -stats"])
             if ytdl_options.get("ffmpeg_location"):
                 try:
                     cmd.extend(["--ffmpeg-location", str(ytdl_options["ffmpeg_location"])])
@@ -1614,23 +1701,13 @@ def _download_with_sections_via_cli(
             if not quiet:
                 debug(f"Running yt-dlp for section: {section}")
 
-            progress_end_pct = min(display_pct + 45, 98)
-            simulator = _SectionProgressSimulator(display_pct, progress_end_pct)
-            simulator.start()
-            try:
-                if quiet:
-                    subprocess.run(cmd, check=True, capture_output=True, text=True)
-                else:
-                    subprocess.run(cmd, check=True)
-            except subprocess.CalledProcessError as exc:
-                stderr_text = exc.stderr or ""
-                tail = "\n".join(stderr_text.splitlines()[-12:]).strip()
-                details = f"\n{tail}" if tail else ""
-                raise DownloadError(f"yt-dlp failed for section {section} (exit {exc.returncode}){details}") from exc
-            except Exception as exc:
-                raise DownloadError(f"yt-dlp failed for section {section}: {exc}") from exc
-            finally:
-                simulator.stop()
+            _run_section_command(
+                cmd,
+                section_idx=section_idx,
+                total_sections=total_sections,
+                section_label=section,
+                section_duration=section_duration,
+            )
     finally:
         pipeline.clear_status()