From d0b821b5dd180a77a0a49f03ec67bd8e67ee4a0c Mon Sep 17 00:00:00 2001 From: nose Date: Sun, 21 Dec 2025 16:59:37 -0800 Subject: [PATCH] sdfsdf --- CLI.py | 31 ++++- TUI/pipeline_runner.py | 2 +- cmdlet/add_file.py | 4 +- cmdlet/download_media.py | 149 ++++++++++++++++---- cmdlet/screen_shot.py | 167 +++++++++++++---------- models.py | 287 +++++++++++++++++++++++++++++++++++---- tool/ytdlp.py | 4 + 7 files changed, 508 insertions(+), 136 deletions(-) diff --git a/CLI.py b/CLI.py index a14f250..ffe7973 100644 --- a/CLI.py +++ b/CLI.py @@ -1099,7 +1099,7 @@ class CmdletExecutor: ) stage_worker_id = stage_session.worker_id if stage_session else None - pipeline_ctx = ctx.PipelineStageContext(stage_index=0, total_stages=1, worker_id=stage_worker_id) + pipeline_ctx = ctx.PipelineStageContext(stage_index=0, total_stages=1, pipe_index=0, worker_id=stage_worker_id) ctx.set_stage_context(pipeline_ctx) stage_status = "completed" stage_error = "" @@ -1584,7 +1584,7 @@ class PipelineExecutor: "download_media", ".pipe", ): - print("Auto-inserting download-file after Soulseek selection") + debug("Auto-inserting download-file after Soulseek selection") stages.insert(0, ["download-file"]) if table_type == "youtube" and first_cmd not in ( "download-media", @@ -1592,7 +1592,7 @@ class PipelineExecutor: "download-file", ".pipe", ): - print("Auto-inserting download-media after YouTube selection") + debug("Auto-inserting download-media after YouTube selection") stages.insert(0, ["download-media"]) if table_type == "bandcamp" and first_cmd not in ( "download-media", @@ -1636,6 +1636,11 @@ class PipelineExecutor: name = str(tokens[0]).replace("_", "-").lower() if name == "@" or name.startswith("@"): continue + # `.pipe` (MPV) is an interactive launcher; disable pipeline Live progress + # for it because it doesn't meaningfully "complete" (mpv may keep running) + # and Live output interferes with MPV playlist UI. + if name == ".pipe": + continue pipe_stage_indices.append(idx) pipe_labels.append(name) @@ -1846,6 +1851,7 @@ class PipelineExecutor: pipeline_ctx = ctx.PipelineStageContext( stage_index=stage_index, total_stages=len(stages), + pipe_index=pipe_idx, worker_id=stage_worker_id, on_emit=on_emit, ) @@ -1889,6 +1895,21 @@ class PipelineExecutor: except Exception: pass + # `.pipe` is typically the terminal interactive stage (MPV UI). + # Stop Live progress before running it so output doesn't get stuck behind Live. + if cmd_name == ".pipe" and progress_ui is not None and (stage_index + 1 >= len(stages)): + try: + progress_ui.stop() + except Exception: + pass + try: + import pipeline as _pipeline_ctx + if hasattr(_pipeline_ctx, "set_live_progress"): + _pipeline_ctx.set_live_progress(None) + except Exception: + pass + progress_ui = None + ret_code = cmd_fn(piped_result, list(stage_args), config) stage_is_last = stage_index + 1 >= len(stages) @@ -2127,7 +2148,7 @@ class PipelineExecutor: Welcome = """ # MEDIOS-MACINA -Romans 1:22 Professing themselves to be wise, they became fools, +[red]Romans 1:22[/red] Professing themselves to be wise, they became fools, dfd @@ -2278,7 +2299,7 @@ class MedeiaCLI: def run_repl(self) -> None: # (Startup banner is optional; keep the REPL quiet by default.) - prompt_text = "<🜂🜄🜁🜃>" + prompt_text = "<🜂🜄|🜁🜃>" startup_table = ResultTable( "*********************************************" diff --git a/TUI/pipeline_runner.py b/TUI/pipeline_runner.py index e58aca5..7444d8c 100644 --- a/TUI/pipeline_runner.py +++ b/TUI/pipeline_runner.py @@ -206,7 +206,7 @@ class PipelineExecutor: stage.error = f"Unknown command: {cmd_name}" return stage - pipeline_ctx = ctx.PipelineStageContext(stage_index=index, total_stages=total) + pipeline_ctx = ctx.PipelineStageContext(stage_index=index, total_stages=total, pipe_index=index) ctx.set_stage_context(pipeline_ctx) try: diff --git a/cmdlet/add_file.py b/cmdlet/add_file.py index 8a20233..b2e469c 100644 --- a/cmdlet/add_file.py +++ b/cmdlet/add_file.py @@ -643,7 +643,7 @@ class Add_File(Cmdlet): # Run search-store under a temporary stage context so its ctx.emit() calls # don't interfere with the outer add-file pipeline stage. prev_ctx = ctx.get_stage_context() - temp_ctx = ctx.PipelineStageContext(stage_index=0, total_stages=1, worker_id=getattr(prev_ctx, "worker_id", None)) + temp_ctx = ctx.PipelineStageContext(stage_index=0, total_stages=1, pipe_index=0, worker_id=getattr(prev_ctx, "worker_id", None)) ctx.set_stage_context(temp_ctx) try: code = search_store_cmdlet.run(None, args, config) @@ -1472,7 +1472,7 @@ class Add_File(Cmdlet): # Run search-store under a temporary stage context so its ctx.emit() calls # don't interfere with the outer add-file pipeline stage. prev_ctx = ctx.get_stage_context() - temp_ctx = ctx.PipelineStageContext(stage_index=0, total_stages=1, worker_id=getattr(prev_ctx, "worker_id", None)) + temp_ctx = ctx.PipelineStageContext(stage_index=0, total_stages=1, pipe_index=0, worker_id=getattr(prev_ctx, "worker_id", None)) ctx.set_stage_context(temp_ctx) try: code = search_store_cmdlet.run(None, args, config) diff --git a/cmdlet/download_media.py b/cmdlet/download_media.py index 73a697c..b1de2dc 100644 --- a/cmdlet/download_media.py +++ b/cmdlet/download_media.py @@ -48,6 +48,64 @@ coerce_to_pipe_object = sh.coerce_to_pipe_object get_field = sh.get_field +def _live_ui_and_pipe_index() -> tuple[Optional[Any], int]: + ui = None + try: + ui = pipeline_context.get_live_progress() if hasattr(pipeline_context, "get_live_progress") else None + except Exception: + ui = None + + pipe_idx: int = 0 + try: + stage_ctx = pipeline_context.get_stage_context() if hasattr(pipeline_context, "get_stage_context") else None + maybe_idx = getattr(stage_ctx, "pipe_index", None) if stage_ctx is not None else None + if isinstance(maybe_idx, int): + pipe_idx = int(maybe_idx) + except Exception: + pipe_idx = 0 + + return ui, pipe_idx + + +def _begin_live_steps(total_steps: int) -> None: + """Declare the total number of steps for the current pipe.""" + ui, pipe_idx = _live_ui_and_pipe_index() + if ui is None: + return + try: + begin = getattr(ui, "begin_pipe_steps", None) + if callable(begin): + begin(int(pipe_idx), total_steps=int(total_steps)) + except Exception: + return + + +def _step(text: str) -> None: + """Emit a *new* step (increments i/N and advances percent automatically).""" + ui, pipe_idx = _live_ui_and_pipe_index() + if ui is None: + return + try: + adv = getattr(ui, "advance_pipe_step", None) + if callable(adv): + adv(int(pipe_idx), str(text)) + except Exception: + return + + +def _set_pipe_percent(percent: int) -> None: + """Best-effort percent update without changing step text.""" + ui, pipe_idx = _live_ui_and_pipe_index() + if ui is None: + return + try: + set_pct = getattr(ui, "set_pipe_percent", None) + if callable(set_pct): + set_pct(int(pipe_idx), int(percent)) + except Exception: + return + + # Minimal inlined helpers from helper/download.py (is_url_supported_by_ytdlp, list_formats) try: import yt_dlp # type: ignore @@ -353,7 +411,17 @@ def _download_with_sections_via_cli(url: str, ytdl_options: Dict[str, Any], sect session_id = hashlib.md5((url + str(time.time()) + ''.join(random.choices(string.ascii_letters, k=10))).encode()).hexdigest()[:12] first_section_info = None + total_sections = len(sections_list) for section_idx, section in enumerate(sections_list, 1): + # While step 1/2 is "downloading", keep the pipe bar moving for multi-section clips. + # Map sections onto 50..99 so step 2/2 can still jump to 100. + try: + if total_sections > 0: + pct = 50 + int(((section_idx - 1) / max(1, total_sections)) * 49) + _set_pipe_percent(pct) + except Exception: + pass + base_outtmpl = ytdl_options.get("outtmpl", "%(title)s.%(ext)s") output_dir_path = Path(base_outtmpl).parent filename_tmpl = f"{session_id}_{section_idx}" @@ -385,6 +453,17 @@ def _download_with_sections_via_cli(url: str, ytdl_options: Dict[str, Any], sect debug(f"Error extracting metadata: {e}") cmd = ["yt-dlp"] + if quiet: + cmd.append("--quiet") + cmd.append("--no-warnings") + cmd.append("--no-progress") + # Keep ffmpeg/merger output from taking over the terminal. + cmd.extend(["--postprocessor-args", "ffmpeg:-hide_banner -loglevel error"]) + if ytdl_options.get("ffmpeg_location"): + try: + cmd.extend(["--ffmpeg-location", str(ytdl_options["ffmpeg_location"])]) + except Exception: + pass if ytdl_options.get("format"): cmd.extend(["-f", ytdl_options["format"]]) if ytdl_options.get("merge_output_format"): @@ -413,7 +492,7 @@ def _download_with_sections_via_cli(url: str, ytdl_options: Dict[str, Any], sect cmd.append("--write-auto-sub") cmd.extend(["--sub-format", "vtt"]) if ytdl_options.get("force_keyframes_at_cuts"): - cmd.extend(["--force-keyframes-at-cuts"]) if ytdl_options.get("force_keyframes_at_cuts") else None + cmd.append("--force-keyframes-at-cuts") cmd.extend(["-o", section_outtmpl]) if ytdl_options.get("cookiefile"): cookies_path = ytdl_options["cookiefile"].replace("\\", "/") @@ -428,10 +507,23 @@ def _download_with_sections_via_cli(url: str, ytdl_options: Dict[str, Any], sect if not quiet: debug(f"Running yt-dlp for section: {section}") try: - subprocess.run(cmd, check=True) + if quiet: + subprocess.run(cmd, check=True, capture_output=True, text=True) + else: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as exc: + stderr_text = (exc.stderr or "") + tail = "\n".join(stderr_text.splitlines()[-12:]).strip() + details = f"\n{tail}" if tail else "" + raise DownloadError(f"yt-dlp failed for section {section} (exit {exc.returncode}){details}") from exc except Exception as exc: - if not quiet: - debug(f"yt-dlp error for section {section}: {exc}") + raise DownloadError(f"yt-dlp failed for section {section}: {exc}") from exc + + # Mark near-complete before returning so the runner can finalize cleanly. + try: + _set_pipe_percent(99) + except Exception: + pass return session_id, first_section_info or {} @@ -720,30 +812,16 @@ def download_media( session_id = None first_section_info = {} if ytdl_options.get("download_sections"): - # The CLI path emits yt-dlp's own progress output; pause the pipeline Live UI - # so those progress bars remain visible instead of being clobbered. - try: - from contextlib import nullcontext - except Exception: - nullcontext = None # type: ignore - - suspend = getattr(pipeline_context, "suspend_live_progress", None) - cm = suspend() if callable(suspend) else (nullcontext() if nullcontext else None) - if cm is None: - session_id, first_section_info = _download_with_sections_via_cli( - opts.url, - ytdl_options, - ytdl_options.get("download_sections", []), - quiet=opts.quiet, - ) - else: - with cm: - session_id, first_section_info = _download_with_sections_via_cli( - opts.url, - ytdl_options, - ytdl_options.get("download_sections", []), - quiet=opts.quiet, - ) + # For clip (download_sections), keep pipeline Live UI active and suppress + # yt-dlp/ffmpeg CLI spam when running in quiet/pipeline mode. + live_ui, _ = _live_ui_and_pipe_index() + quiet_sections = bool(opts.quiet) or (live_ui is not None) + session_id, first_section_info = _download_with_sections_via_cli( + opts.url, + ytdl_options, + ytdl_options.get("download_sections", []), + quiet=quiet_sections, + ) info = None else: with yt_dlp.YoutubeDL(ytdl_options) as ydl: # type: ignore[arg-type] @@ -2168,7 +2246,6 @@ class Download_Media(Cmdlet): pipeline_context.set_last_result_table(table, results_list) log(f"", file=sys.stderr) - log(f"Use: @N to select and download format", file=sys.stderr) return 0 # Download each URL @@ -2196,6 +2273,11 @@ class Download_Media(Cmdlet): log(f"Skipping download: {url}", file=sys.stderr) continue + # Step progress is per-URL download. + # Keep steps meaningful: long-running download + finalize. + # (Fast internal bookkeeping should not be steps.) + _begin_live_steps(2) + # If playlist_items is specified but looks like a format ID (e.g. from table selection), # treat it as a format selector instead of playlist items. # This handles the case where @N selection passes -item @@ -2274,6 +2356,7 @@ class Download_Media(Cmdlet): write_sub=write_sub, ) + _step("downloading") # Use timeout wrapper to prevent hanging debug(f"Starting download with 5-minute timeout...") result_obj = _download_with_timeout(opts, timeout_seconds=300) @@ -2416,6 +2499,10 @@ class Download_Media(Cmdlet): except Exception: pass + # Complete the step sequence: we return here and the user must + # re-run with @N selection. + _step("awaiting selection") + log("Requested format is not available; select a working format with @N", file=sys.stderr) return 0 @@ -2522,6 +2609,10 @@ class Download_Media(Cmdlet): debug(f"Emitting {len(pipe_objects)} result(s) to pipeline...") + # Mark complete *before* the first emit, because the pipeline clears the + # status line on emit(). + _step("finalized") + stage_ctx = pipeline_context.get_stage_context() emit_enabled = bool(stage_ctx is not None and not getattr(stage_ctx, "is_last_stage", False)) for pipe_obj_dict in pipe_objects: diff --git a/cmdlet/screen_shot.py b/cmdlet/screen_shot.py index f5ddcfc..3eb2a87 100644 --- a/cmdlet/screen_shot.py +++ b/cmdlet/screen_shot.py @@ -32,20 +32,52 @@ parse_cmdlet_args = sh.parse_cmdlet_args import pipeline as pipeline_context -def _set_live_step(text: str) -> None: - """Best-effort update to the pipeline Live progress title (if enabled).""" +def _live_ui_and_pipe_index() -> tuple[Optional[Any], int]: + ui = None try: ui = pipeline_context.get_live_progress() if hasattr(pipeline_context, "get_live_progress") else None except Exception: ui = None + + pipe_idx: int = 0 + try: + stage_ctx = pipeline_context.get_stage_context() if hasattr(pipeline_context, "get_stage_context") else None + maybe_idx = getattr(stage_ctx, "pipe_index", None) if stage_ctx is not None else None + if isinstance(maybe_idx, int): + pipe_idx = int(maybe_idx) + except Exception: + pipe_idx = 0 + + return ui, pipe_idx + + +def _begin_live_steps(total_steps: int) -> None: + """Declare the total number of steps for this cmdlet run (per-pipe).""" + ui, pipe_idx = _live_ui_and_pipe_index() if ui is None: return try: - setter = getattr(ui, "set_active_subtask_text", None) - if callable(setter): - setter(str(text or "").strip()) + begin = getattr(ui, "begin_pipe_steps", None) + if callable(begin): + begin(int(pipe_idx), total_steps=int(total_steps)) except Exception: - pass + return + + +def _step(text: str) -> None: + """Emit a *new* step. + + Each call increments the step counter and advances percent automatically. + """ + ui, pipe_idx = _live_ui_and_pipe_index() + if ui is None: + return + try: + adv = getattr(ui, "advance_pipe_step", None) + if callable(adv): + adv(int(pipe_idx), str(text)) + except Exception: + return # ============================================================================ # CMDLET Metadata Declaration @@ -186,28 +218,6 @@ def _format_suffix(fmt: str) -> str: return ".jpg" return f".{fmt}" - -def _convert_to_webp(source_path: Path, dest_path: Path) -> None: - """Convert an image file to WebP using Pillow.""" - from PIL import Image - - with Image.open(source_path) as img: - # Keep a sensible default: good quality + small size. - img.save(dest_path, format="WEBP", quality=100, method=6) - - -def _selectors_for_url(url: str) -> List[str]: - """Return a list of likely content selectors for known platforms.""" - u = url.lower() - sels: List[str] = [] - - for domain, selectors in SITE_SELECTORS.items(): - if domain in u: - sels.extend(selectors) - - return sels or ["article"] - - def _matched_site_selectors(url: str) -> List[str]: """Return SITE_SELECTORS for a matched domain; empty if no match. @@ -223,46 +233,47 @@ def _matched_site_selectors(url: str) -> List[str]: def _platform_preprocess(url: str, page: Any, warnings: List[str], timeout_ms: int = 10_000) -> None: """Best-effort page tweaks for popular platforms before capture.""" - u = url.lower() + try: + u = str(url or "").lower() - def _try_click_texts(texts: List[str], passes: int = 2, per_timeout: int = 700) -> int: - clicks = 0 - for _ in range(max(1, passes)): - for t in texts: - try: - page.locator(f"text=/{t}/i").first.click(timeout=per_timeout) - clicks += 1 - except PlaywrightTimeoutError: - pass - except Exception: - pass - time.sleep(0.1) - return clicks + def _try_click_buttons(names: List[str], passes: int = 2, per_timeout: int = 700) -> int: + clicks = 0 + for _ in range(max(1, int(passes))): + for name in names: + try: + locator = page.get_by_role("button", name=name) + locator.first.click(timeout=int(per_timeout)) + clicks += 1 + except Exception: + pass + return clicks - # Dismiss common cookie/consent prompts - _try_click_texts(["accept", "i agree", "agree", "got it", "allow all", "consent"]) + # Dismiss common cookie / consent prompts. + _try_click_buttons([ + "Accept all", + "Accept", + "I agree", + "Agree", + "Allow all", + "OK", + ]) - # Platform-specific expansions - if "reddit.com" in u: - _try_click_texts(["see more", "read more", "show more", "more"]) - if ("twitter.com" in u) or ("x.com" in u): - _try_click_texts(["show more", "more"]) - if "instagram.com" in u: - _try_click_texts(["more", "see more"]) - if "tiktok.com" in u: - _try_click_texts(["more", "see more"]) - if ("facebook.com" in u) or ("fb.watch" in u): - _try_click_texts(["see more", "show more", "more"]) - if "rumble.com" in u: - _try_click_texts(["accept", "agree", "close"]) + # Some sites need small nudges (best-effort). + if "reddit.com" in u: + _try_click_buttons(["Accept all", "Accept"]) + if ("twitter.com" in u) or ("x.com" in u): + _try_click_buttons(["Accept all", "Accept"]) + if "instagram.com" in u: + _try_click_buttons(["Allow all", "Accept all", "Accept"]) + except Exception as exc: + debug(f"[_platform_preprocess] skipped: {exc}") + return def _submit_wayback(url: str, timeout: float) -> Optional[str]: - """Submit URL to Internet Archive Wayback Machine.""" encoded = quote(url, safe="/:?=&") - with HTTPClient() as client: + with HTTPClient(headers={"User-Agent": USER_AGENT}) as client: response = client.get(f"https://web.archive.org/save/{encoded}") - response.raise_for_status() content_location = response.headers.get("Content-Location") if content_location: return urljoin("https://web.archive.org", content_location) @@ -359,10 +370,7 @@ def _capture(options: ScreenshotOptions, destination: Path, warnings: List[str]) """Capture screenshot using Playwright.""" debug(f"[_capture] Starting capture for {options.url} -> {destination}") try: - # Two-phase Live progress: - # 1) load + stabilize (ends right after the wait_after_load sleep) - # 2) capture + save (and any post-processing) - _set_live_step("screen-shot: loading") + _step("loading launching browser") tool = options.playwright_tool or PlaywrightTool({}) # Ensure Chromium engine is used for the screen-shot cmdlet (force for consistency) @@ -397,13 +405,16 @@ def _capture(options: ScreenshotOptions, destination: Path, warnings: List[str]) try: with tool.open_page(headless=headless) as page: + _step("loading navigating") debug(f"Navigating to {options.url}...") try: tool.goto(page, options.url) debug("Page loaded successfully") + _step("loading page loaded") except PlaywrightTimeoutError: warnings.append("navigation timeout; capturing current page state") debug("Navigation timeout; proceeding with current state") + _step("loading navigation timeout") # Skip article lookup by default (wait_for_article defaults to False) if options.wait_for_article: @@ -419,8 +430,9 @@ def _capture(options: ScreenshotOptions, destination: Path, warnings: List[str]) debug(f"Waiting {options.wait_after_load}s for page stabilization...") time.sleep(min(10.0, max(0.0, options.wait_after_load))) - # Phase 2 begins here (per request). - _set_live_step("screen-shot: capturing") + _step("loading stabilized") + + _step("capturing preparing") if options.replace_video_posters: debug("Replacing video elements with posters...") page.evaluate( @@ -441,6 +453,7 @@ def _capture(options: ScreenshotOptions, destination: Path, warnings: List[str]) if options.prefer_platform_target and format_name != "pdf": debug(f"[_capture] Target capture enabled") debug("Attempting platform-specific content capture...") + _step("capturing locating target") try: _platform_preprocess(options.url, page, warnings) except Exception as e: @@ -465,6 +478,7 @@ def _capture(options: ScreenshotOptions, destination: Path, warnings: List[str]) el.scroll_into_view_if_needed(timeout=1000) except Exception: pass + _step("capturing output") debug(f"Capturing element to {destination}...") el.screenshot(path=str(destination), type=("jpeg" if format_name == "jpeg" else None)) element_captured = True @@ -475,12 +489,14 @@ def _capture(options: ScreenshotOptions, destination: Path, warnings: List[str]) debug(f"Failed to capture element: {exc}") # Fallback to default capture paths if element_captured: - pass + _step("capturing saved") elif format_name == "pdf": debug("Generating PDF...") page.emulate_media(media="print") + _step("capturing output") page.pdf(path=str(destination), print_background=True) debug(f"PDF saved to {destination}") + _step("capturing saved") else: debug(f"Capturing full page to {destination}...") screenshot_kwargs: Dict[str, Any] = {"path": str(destination)} @@ -488,16 +504,20 @@ def _capture(options: ScreenshotOptions, destination: Path, warnings: List[str]) screenshot_kwargs["type"] = "jpeg" screenshot_kwargs["quality"] = 90 if options.full_page: + _step("capturing output") page.screenshot(full_page=True, **screenshot_kwargs) else: article = page.query_selector("article") if article is not None: article_kwargs = dict(screenshot_kwargs) article_kwargs.pop("full_page", None) + _step("capturing output") article.screenshot(**article_kwargs) else: + _step("capturing output") page.screenshot(**screenshot_kwargs) debug(f"Screenshot saved to {destination}") + _step("capturing saved") except Exception as exc: debug(f"[_capture] Exception launching browser/page: {exc}") msg = str(exc).lower() @@ -519,6 +539,13 @@ def _capture_screenshot(options: ScreenshotOptions) -> ScreenshotResult: destination = _prepare_output_path(options) warnings: List[str] = [] + will_target = bool(options.prefer_platform_target) and requested_format != "pdf" + will_convert = requested_format == "webp" + will_archive = bool(options.archive and options.url) + total_steps = 9 + (1 if will_target else 0) + (1 if will_convert else 0) + (1 if will_archive else 0) + _begin_live_steps(total_steps) + _step("loading starting") + # Playwright screenshots do not natively support WebP output. # Capture as PNG, then convert via Pillow. capture_path = destination @@ -529,6 +556,7 @@ def _capture_screenshot(options: ScreenshotOptions) -> ScreenshotResult: _capture(options, capture_path, warnings) if requested_format == "webp": + _step("capturing converting to webp") debug(f"[_capture_screenshot] Converting png -> webp: {destination}") try: _convert_to_webp(capture_path, destination) @@ -544,7 +572,7 @@ def _capture_screenshot(options: ScreenshotOptions) -> ScreenshotResult: url: List[str] = [options.url] if options.url else [] archive_url: List[str] = [] if options.archive and options.url: - _set_live_step("screen-shot: archiving") + _step("capturing archiving") debug(f"[_capture_screenshot] Archiving enabled for {options.url}") archives, archive_warnings = _archive_url(options.url, options.archive_timeout) archive_url.extend(archives) @@ -552,6 +580,8 @@ def _capture_screenshot(options: ScreenshotOptions) -> ScreenshotResult: if archives: url = unique_preserve_order([*url, *archives]) + _step("capturing finalized") + applied_tag = unique_preserve_order(list(tag for tag in options.tag if tag.strip())) return ScreenshotResult( @@ -768,7 +798,6 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: continue try: - _set_live_step("screen-shot: starting") # Create screenshot with provided options # Force the Playwright engine to Chromium for the screen-shot cmdlet # (this ensures consistent rendering and supports PDF output requirements). diff --git a/models.py b/models.py index 162e6ab..c106460 100644 --- a/models.py +++ b/models.py @@ -681,6 +681,7 @@ class PipelineLiveProgress: self._overall: Optional[Progress] = None self._pipe_progress: Optional[Progress] = None self._subtasks: Optional[Progress] = None + self._status: Optional[Progress] = None self._transfers: Optional[Progress] = None self._overall_task: Optional[TaskID] = None @@ -688,6 +689,17 @@ class PipelineLiveProgress: self._transfer_tasks: Dict[str, TaskID] = {} + # Per-pipe status line shown below the pipe bars. + self._status_tasks: Dict[int, TaskID] = {} + + # When a pipe is operating on a single item, allow percent-based progress + # updates on the pipe bar (0..100) so it doesn't sit at 0% until emit(). + self._pipe_percent_mode: Dict[int, bool] = {} + + # Per-pipe step counters used for status lines and percent mapping. + self._pipe_step_total: Dict[int, int] = {} + self._pipe_step_done: Dict[int, int] = {} + # Per-pipe state self._pipe_totals: List[int] = [0 for _ in self._pipe_labels] self._pipe_done: List[int] = [0 for _ in self._pipe_labels] @@ -700,26 +712,11 @@ class PipelineLiveProgress: def _title_text(self) -> str: """Compute the Pipeline panel title. - We keep per-pipe elapsed time on the pipe rows. The panel title is used - to show the currently active item (cmd + url/path) with a lightweight - spinner so the UI reads as "working on X". + The title remains stable ("Pipeline"). Per-item step detail is rendered + using a dedicated progress bar within the panel. """ - active = str(self._active_subtask_text or "").strip() - if not active: - return "Pipeline" - - # Lightweight spinner frames (similar intent to Rich's simpleDots). - try: - import time - - frames = [".", "..", "..."] - idx = int(time.monotonic() * 4) % len(frames) - prefix = frames[idx] - except Exception: - prefix = "..." - - return f"{prefix} {active}" + return "Pipeline" def set_active_subtask_text(self, text: Optional[str]) -> None: """Update the Pipeline panel title to reflect the current in-item step. @@ -744,6 +741,7 @@ class PipelineLiveProgress: """ pipe_progress = self._pipe_progress + status = self._status transfers = self._transfers overall = self._overall if pipe_progress is None or transfers is None or overall is None: @@ -751,23 +749,27 @@ class PipelineLiveProgress: yield Panel("", title="Pipeline", expand=False) return - yield Group( - Panel(Group(pipe_progress, transfers), title=self._title_text(), expand=False), - overall, - ) + body_parts: List[Any] = [pipe_progress] + if status is not None and self._status_tasks: + body_parts.append(status) + body_parts.append(transfers) + + yield Group(Panel(Group(*body_parts), title=self._title_text(), expand=False), overall) def _render_group(self) -> Group: # Backward-compatible helper (some callers may still expect a Group). pipe_progress = self._pipe_progress + status = self._status transfers = self._transfers overall = self._overall assert pipe_progress is not None assert transfers is not None assert overall is not None - return Group( - Panel(Group(pipe_progress, transfers), title=self._title_text(), expand=False), - overall, - ) + body_parts: List[Any] = [pipe_progress] + if status is not None and self._status_tasks: + body_parts.append(status) + body_parts.append(transfers) + return Group(Panel(Group(*body_parts), title=self._title_text(), expand=False), overall) def start(self) -> None: if not self._enabled: @@ -803,6 +805,14 @@ class PipelineLiveProgress: transient=False, ) + # Status line below the pipe bars. Kept simple (no extra bar) so it + # doesn't visually offset the main pipe bar columns. + self._status = Progress( + TextColumn(" [bold]└─ {task.description}[/bold]"), + console=self._console, + transient=False, + ) + # Byte-based transfer bars (download/upload) integrated into the Live view. self._transfers = Progress( TextColumn(" {task.description}"), @@ -878,12 +888,178 @@ class PipelineLiveProgress: self._overall = None self._pipe_progress = None self._subtasks = None + self._status = None self._transfers = None self._overall_task = None self._pipe_tasks = [] self._transfer_tasks = {} + self._status_tasks = {} + self._pipe_percent_mode = {} + self._pipe_step_total = {} + self._pipe_step_done = {} self._active_subtask_text = None + def _hide_pipe_subtasks(self, pipe_index: int) -> None: + """Hide any visible per-item spinner rows for a pipe.""" + subtasks = self._subtasks + if subtasks is None: + return + try: + for sub_id in self._subtask_ids[int(pipe_index)]: + try: + subtasks.stop_task(sub_id) + subtasks.update(sub_id, visible=False) + except Exception: + pass + except Exception: + pass + + def set_pipe_status_text(self, pipe_index: int, text: str) -> None: + """Set a status line under the pipe bars for the given pipe.""" + if not self._enabled: + return + if not self._ensure_pipe(int(pipe_index)): + return + prog = self._status + if prog is None: + return + + try: + pidx = int(pipe_index) + msg = str(text or "").strip() + except Exception: + return + + # For long single-item work, hide the per-item spinner line and use this + # dedicated status line instead. + if self._pipe_percent_mode.get(pidx, False): + try: + self._hide_pipe_subtasks(pidx) + except Exception: + pass + + task_id = self._status_tasks.get(pidx) + if task_id is None: + try: + task_id = prog.add_task(msg) + except Exception: + return + self._status_tasks[pidx] = task_id + + try: + prog.update(task_id, description=msg, refresh=True) + except Exception: + pass + + def clear_pipe_status_text(self, pipe_index: int) -> None: + prog = self._status + if prog is None: + return + try: + pidx = int(pipe_index) + except Exception: + return + task_id = self._status_tasks.pop(pidx, None) + if task_id is None: + return + try: + prog.remove_task(task_id) + except Exception: + pass + + def set_pipe_percent(self, pipe_index: int, percent: int) -> None: + """Update the pipe bar as a percent (only when single-item mode is enabled).""" + if not self._enabled: + return + if not self._ensure_pipe(int(pipe_index)): + return + pipe_progress = self._pipe_progress + if pipe_progress is None: + return + try: + pidx = int(pipe_index) + except Exception: + return + if not self._pipe_percent_mode.get(pidx, False): + return + try: + pct = max(0, min(100, int(percent))) + pipe_task = self._pipe_tasks[pidx] + pipe_progress.update(pipe_task, completed=pct, total=100, refresh=True) + except Exception: + pass + + def begin_pipe_steps(self, pipe_index: int, *, total_steps: int) -> None: + """Initialize step tracking for a pipe. + + The cmdlet must call this once up-front so we can map steps to percent. + """ + if not self._enabled: + return + if not self._ensure_pipe(int(pipe_index)): + return + + try: + pidx = int(pipe_index) + tot = max(1, int(total_steps)) + except Exception: + return + + self._pipe_step_total[pidx] = tot + self._pipe_step_done[pidx] = 0 + + # Reset status line and percent. + try: + self.clear_pipe_status_text(pidx) + except Exception: + pass + try: + self.set_pipe_percent(pidx, 0) + except Exception: + pass + + def advance_pipe_step(self, pipe_index: int, text: str) -> None: + """Advance the pipe's step counter by one. + + Each call is treated as a new step (no in-place text rewrites). + Updates: + - status line: "i/N step: {text}" + - pipe percent (single-item pipes only): round(i/N*100) + """ + if not self._enabled: + return + if not self._ensure_pipe(int(pipe_index)): + return + + try: + pidx = int(pipe_index) + except Exception: + return + + total = int(self._pipe_step_total.get(pidx, 0) or 0) + if total <= 0: + # If steps weren't declared, treat as a single-step operation. + total = 1 + self._pipe_step_total[pidx] = total + + done = int(self._pipe_step_done.get(pidx, 0) or 0) + 1 + done = min(done, total) + self._pipe_step_done[pidx] = done + + msg = str(text or "").strip() + line = f"{done}/{total} step: {msg}" if msg else f"{done}/{total} step" + try: + self.set_pipe_status_text(pidx, line) + except Exception: + pass + + # Percent mapping only applies when the pipe is in percent mode (single-item). + try: + pct = 100 if done >= total else int(round((done / max(1, total)) * 100.0)) + self.set_pipe_percent(pidx, pct) + except Exception: + pass + def begin_transfer(self, *, label: str, total: Optional[int] = None) -> None: if not self._enabled: return @@ -962,8 +1138,23 @@ class PipelineLiveProgress: self._subtask_active_index[pipe_index] = 0 self._subtask_ids[pipe_index] = [] + # Reset per-item step progress for this pipe. + try: + self.clear_pipe_status_text(pipe_index) + except Exception: + pass + try: + self._pipe_step_total.pop(pipe_index, None) + self._pipe_step_done.pop(pipe_index, None) + except Exception: + pass + + # If this pipe will process exactly one item, allow percent-based updates. + percent_mode = bool(int(total_items) == 1) + self._pipe_percent_mode[pipe_index] = percent_mode + pipe_task = self._pipe_tasks[pipe_index] - pipe_progress.update(pipe_task, completed=0, total=total_items) + pipe_progress.update(pipe_task, completed=0, total=(100 if percent_mode else total_items)) # Start the per-pipe timer now that the pipe is actually running. try: pipe_progress.start_task(pipe_task) @@ -974,6 +1165,12 @@ class PipelineLiveProgress: if isinstance(items_preview, list) and items_preview: labels = [_pipeline_progress_item_label(x) for x in items_preview] + # For single-item pipes, keep the UI clean: don't show a spinner row. + if percent_mode: + self._subtask_ids[pipe_index] = [] + self._active_subtask_text = None + return + for i in range(total_items): suffix = labels[i] if i < len(labels) else f"item {i + 1}/{total_items}" # Use start=False so elapsed time starts when we explicitly start_task(). @@ -1038,7 +1235,21 @@ class PipelineLiveProgress: self._pipe_done[pipe_index] = done pipe_task = self._pipe_tasks[pipe_index] - pipe_progress.update(pipe_task, completed=done) + if self._pipe_percent_mode.get(pipe_index, False): + pipe_progress.update(pipe_task, completed=100, total=100) + else: + pipe_progress.update(pipe_task, completed=done) + + # Clear any status line now that it emitted. + try: + self.clear_pipe_status_text(pipe_index) + except Exception: + pass + try: + self._pipe_step_total.pop(pipe_index, None) + self._pipe_step_done.pop(pipe_index, None) + except Exception: + pass # Start next subtask spinner. next_index = active + 1 @@ -1072,7 +1283,10 @@ class PipelineLiveProgress: # Ensure the pipe bar finishes even if cmdlet didn’t emit per item. if force_complete and done < total: pipe_task = self._pipe_tasks[pipe_index] - pipe_progress.update(pipe_task, completed=total) + if self._pipe_percent_mode.get(pipe_index, False): + pipe_progress.update(pipe_task, completed=100, total=100) + else: + pipe_progress.update(pipe_task, completed=total) self._pipe_done[pipe_index] = total # Hide any remaining subtask spinners. @@ -1086,6 +1300,17 @@ class PipelineLiveProgress: # If we just finished the active pipe, clear the title context. self._active_subtask_text = None + # Ensure status line is cleared when a pipe finishes. + try: + self.clear_pipe_status_text(pipe_index) + except Exception: + pass + try: + self._pipe_step_total.pop(pipe_index, None) + self._pipe_step_done.pop(pipe_index, None) + except Exception: + pass + # Stop the per-pipe timer once the pipe is finished. try: pipe_task = self._pipe_tasks[pipe_index] @@ -1112,12 +1337,14 @@ class PipelineStageContext: self, stage_index: int, total_stages: int, + pipe_index: Optional[int] = None, worker_id: Optional[str] = None, on_emit: Optional[Callable[[Any], None]] = None, ): self.stage_index = stage_index self.total_stages = total_stages self.is_last_stage = (stage_index == total_stages - 1) + self.pipe_index = int(pipe_index) if pipe_index is not None else None self.worker_id = worker_id self._on_emit = on_emit self.emits: List[Any] = [] @@ -1141,7 +1368,7 @@ class PipelineStageContext: def __repr__(self) -> str: return ( f"PipelineStageContext(stage={self.stage_index}/{self.total_stages}, " - f"is_last={self.is_last_stage}, worker_id={self.worker_id})" + f"pipe_index={self.pipe_index}, is_last={self.is_last_stage}, worker_id={self.worker_id})" ) diff --git a/tool/ytdlp.py b/tool/ytdlp.py index b68c722..8451362 100644 --- a/tool/ytdlp.py +++ b/tool/ytdlp.py @@ -257,6 +257,10 @@ class YtDlpTool: if sections: base_options["download_sections"] = sections + # Clipped outputs should begin with a keyframe; otherwise players (notably mpv) + # can show audio before video or a black screen until the next keyframe. + # yt-dlp implements this by forcing keyframes at cut points. + base_options["force_keyframes_at_cuts"] = True debug(f"Download sections configured: {', '.join(sections)}") if opts.playlist_items: