From 5323b7c76fe46e740cc86401b1c8391d1597c8c8 Mon Sep 17 00:00:00 2001 From: Nose Date: Tue, 10 Feb 2026 23:00:30 -0800 Subject: [PATCH] F --- API/data/alldebrid.json | 30 +++------ SYS/models.py | 3 +- cmdlet/download_file.py | 38 ++++++++++- cmdlet/screen_shot.py | 51 +++++++++++++-- tool/playwright.py | 141 ++++++++++++++++++++-------------------- tool/ytdlp.py | 33 ++++++++-- 6 files changed, 192 insertions(+), 104 deletions(-) diff --git a/API/data/alldebrid.json b/API/data/alldebrid.json index 15a2dab..89f522a 100644 --- a/API/data/alldebrid.json +++ b/API/data/alldebrid.json @@ -22,7 +22,7 @@ "((1fichier\\.com|megadl\\.fr|alterupload\\.com|cjoint\\.net|desfichiers\\.com|dfichiers\\.com|mesfichiers\\.org|piecejointe\\.net|pjointe\\.com|tenvoi\\.com|dl4free\\.com)/\\?[a-zA-Z0-9]{5,30}(&pw=[^&]+)?)" ], "regexp": "((1fichier\\.com|megadl\\.fr|alterupload\\.com|cjoint\\.net|desfichiers\\.com|dfichiers\\.com|mesfichiers\\.org|piecejointe\\.net|pjointe\\.com|tenvoi\\.com|dl4free\\.com)/\\?[a-zA-Z0-9]{5,30}(&pw=[^&]+)?)", - "status": false + "status": true }, "rapidgator": { "name": "rapidgator", @@ -37,7 +37,7 @@ "(rapidgator\\.net/file/[0-9]{7,8})" ], "regexp": "((rapidgator\\.net|rg\\.to|rapidgator\\.asia)/file/([0-9a-zA-Z]{32}))|((rapidgator\\.net/file/[0-9]{7,8}))", - "status": true + "status": false }, "turbobit": { "name": "turbobit", @@ -439,7 +439,7 @@ "(hexupload\\.net|hexload\\.com)/([a-zA-Z0-9]{12})" ], "regexp": "(hexupload\\.net|hexload\\.com)/([a-zA-Z0-9]{12})", - "status": false + "status": true }, "hot4share": { "name": "hot4share", @@ -495,7 +495,7 @@ "(katfile\\.com/[0-9a-zA-Z]{12})" ], "regexp": "(katfile\\.(cloud|online)/([0-9a-zA-Z]{12}))|((katfile\\.com/[0-9a-zA-Z]{12}))", - "status": false + "status": true }, "mediafire": { "name": "mediafire", @@ -507,20 +507,6 @@ "mediafire\\.com/(\\?|download/|file/|download\\.php\\?)([0-9a-z]{15})" ], "regexp": "mediafire\\.com/(\\?|download/|file/|download\\.php\\?)([0-9a-z]{15})", - "status": true - }, - "mexashare": { - "name": "mexashare", - "type": "premium", - "domains": [ - "mexashare.com", - "mx-sh.net", - "mexa.sh" - ], - "regexps": [ - "((mexashare\\.com|mx-sh\\.net|mexa\\.sh)/[0-9a-zA-Z]{12})" - ], - "regexp": "((mexashare\\.com|mx-sh\\.net|mexa\\.sh)/[0-9a-zA-Z]{12})", "status": false }, "mixdrop": { @@ -622,7 +608,7 @@ "(simfileshare\\.net/download/[0-9]+/)" ], "regexp": "(simfileshare\\.net/download/[0-9]+/)", - "status": false + "status": true }, "streamtape": { "name": "streamtape", @@ -802,7 +788,7 @@ "(worldbytez\\.(net|com)/[a-zA-Z0-9]{12})" ], "regexp": "(worldbytez\\.(net|com)/[a-zA-Z0-9]{12})", - "status": false + "status": true } }, "streams": { @@ -17956,9 +17942,9 @@ "generic.tld" ], "regexps": [ - "((example.com|1fichier.com|4shared.com|vev.io|clipwatching.com|clicknupload.click|playvidto.com|uploadrar.com|simfileshare.net|usersdrive.com|fastbit.cc|dropgalaxy.in|uploadboy.com|file.al|filespace.com|uploader.link|9xupload.asia|hexupload.net|filefactory.com|filerio.in|drive.google.com|gigapeta.com|isra.cloud|katfile.com|mediafire.com|mega.co.nz|alldebrid.com|prefiles.com|rapidgator.net|alfafile.net|scribd.com|turbobit.net|hitfile.net|sendit.cloud|ddl.to|exload.com|uploadhaven.com|vidoza.net|mixdrop.co|dropapk.to|indishare.me|world-files.com|uploadbox.io|worldbytez.com|mp4upload.com|mexashare.com|upload42.com|uploading.vn|filedot.to|zofile.com|spicyfile.com|modsbase.com|sharemods.com|dl-file.com|dosya.co|loadstar.club|dailyuploads.net|file-upload.com|uploadbank.com|filezip.cc|hot4share.com|streamtape.com)/folders?/[^'\"<>;]+)" + "((example.com|1fichier.com|4shared.com|vev.io|clipwatching.com|clicknupload.click|playvidto.com|uploadrar.com|simfileshare.net|usersdrive.com|fastbit.cc|dropgalaxy.in|uploadboy.com|file.al|filespace.com|uploader.link|9xupload.asia|hexupload.net|filefactory.com|filerio.in|drive.google.com|gigapeta.com|isra.cloud|katfile.com|mediafire.com|mega.co.nz|alldebrid.com|prefiles.com|rapidgator.net|alfafile.net|scribd.com|turbobit.net|hitfile.net|sendit.cloud|ddl.to|exload.com|uploadhaven.com|vidoza.net|mixdrop.co|dropapk.to|indishare.me|world-files.com|uploadbox.io|worldbytez.com|mp4upload.com|upload42.com|uploading.vn|filedot.to|zofile.com|spicyfile.com|modsbase.com|sharemods.com|dl-file.com|dosya.co|loadstar.club|dailyuploads.net|file-upload.com|uploadbank.com|filezip.cc|hot4share.com|streamtape.com)/folders?/[^'\"<>;]+)" ], - "regexp": "((example.com|1fichier.com|4shared.com|vev.io|clipwatching.com|clicknupload.click|playvidto.com|uploadrar.com|simfileshare.net|usersdrive.com|fastbit.cc|dropgalaxy.in|uploadboy.com|file.al|filespace.com|uploader.link|9xupload.asia|hexupload.net|filefactory.com|filerio.in|drive.google.com|gigapeta.com|isra.cloud|katfile.com|mediafire.com|mega.co.nz|alldebrid.com|prefiles.com|rapidgator.net|alfafile.net|scribd.com|turbobit.net|hitfile.net|sendit.cloud|ddl.to|exload.com|uploadhaven.com|vidoza.net|mixdrop.co|dropapk.to|indishare.me|world-files.com|uploadbox.io|worldbytez.com|mp4upload.com|mexashare.com|upload42.com|uploading.vn|filedot.to|zofile.com|spicyfile.com|modsbase.com|sharemods.com|dl-file.com|dosya.co|loadstar.club|dailyuploads.net|file-upload.com|uploadbank.com|filezip.cc|hot4share.com|streamtape.com)/folders?/[^'\"<>;]+)" + "regexp": "((example.com|1fichier.com|4shared.com|vev.io|clipwatching.com|clicknupload.click|playvidto.com|uploadrar.com|simfileshare.net|usersdrive.com|fastbit.cc|dropgalaxy.in|uploadboy.com|file.al|filespace.com|uploader.link|9xupload.asia|hexupload.net|filefactory.com|filerio.in|drive.google.com|gigapeta.com|isra.cloud|katfile.com|mediafire.com|mega.co.nz|alldebrid.com|prefiles.com|rapidgator.net|alfafile.net|scribd.com|turbobit.net|hitfile.net|sendit.cloud|ddl.to|exload.com|uploadhaven.com|vidoza.net|mixdrop.co|dropapk.to|indishare.me|world-files.com|uploadbox.io|worldbytez.com|mp4upload.com|upload42.com|uploading.vn|filedot.to|zofile.com|spicyfile.com|modsbase.com|sharemods.com|dl-file.com|dosya.co|loadstar.club|dailyuploads.net|file-upload.com|uploadbank.com|filezip.cc|hot4share.com|streamtape.com)/folders?/[^'\"<>;]+)" }, "google": { "name": "google", diff --git a/SYS/models.py b/SYS/models.py index 1a4c064..c39b465 100644 --- a/SYS/models.py +++ b/SYS/models.py @@ -145,7 +145,8 @@ class PipeObject: title_text = cmdlet_name # Color the title (requested: yellow instead of Rich's default blue-ish title). - debug_inspect(self, title=f"[yellow]{title_text}[/yellow]") + # We disable value=False to avoid duplicating the object repr which is redundant with the attribute listing + debug_inspect(self, title=f"[yellow]{title_text}[/yellow]", value=False) def to_dict(self) -> Dict[str, Any]: """Serialize to dictionary, excluding None and empty values.""" diff --git a/cmdlet/download_file.py b/cmdlet/download_file.py index a037833..3916a47 100644 --- a/cmdlet/download_file.py +++ b/cmdlet/download_file.py @@ -18,12 +18,13 @@ from contextlib import AbstractContextManager, nullcontext from API.HTTP import _download_direct_file from SYS.models import DownloadError, DownloadOptions, DownloadMediaResult -from SYS.logger import log, debug +from SYS.logger import log, debug, is_debug_enabled from SYS.pipeline_progress import PipelineProgress from SYS.result_table import Table from SYS.rich_display import stderr_console as get_stderr_console from SYS import pipeline as pipeline_context from SYS.metadata import normalize_urls as normalize_url_list +from SYS.utils import sha256_file from tool.ytdlp import ( YtDlpTool, @@ -1495,6 +1496,41 @@ class Download_File(Cmdlet): forced_single_applied = True # Proactive fallback for single audio formats which might be unstable + if ( + actual_format + and isinstance(actual_format, str) + and actual_format == "audio" + ): + actual_format = "bestaudio/best" + + # DEBUG: Render config panel for tracking pipeline state + if is_debug_enabled(): + try: + from rich.table import Table as RichTable + from rich import box as RichBox + from tool.ytdlp import YtDlpDefaults + + t = RichTable(title="Download Config", show_header=True, header_style="bold magenta", box=RichBox.ROUNDED) + t.add_column("Property", style="cyan") + t.add_column("Value", style="green") + t.add_row("URL", url) + t.add_row("Mode", mode) + t.add_row("Format", str(actual_format)) + t.add_row("Playlist Items", str(actual_playlist_items)) + + # Browser/Cookie info from ytdlp tool + defaults = getattr(ytdlp_tool, "defaults", None) + if isinstance(defaults, YtDlpDefaults): + t.add_row("Cookie File", str(defaults.cookiefile or "None")) + t.add_row("Browser Cookies", str(defaults.cookies_from_browser or "None")) + t.add_row("User Agent", str(defaults.user_agent or "default")) + + debug(t) + except Exception: + pass + + # If no format explicitly chosen, we might want to check available formats + # and maybe show a table if multiple are available? if ( actual_format and isinstance(actual_format, str) diff --git a/cmdlet/screen_shot.py b/cmdlet/screen_shot.py index d908701..01eca2f 100644 --- a/cmdlet/screen_shot.py +++ b/cmdlet/screen_shot.py @@ -17,7 +17,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional, Sequence, Tuple from urllib.parse import urlsplit, quote, urljoin, unquote -from SYS.logger import log, debug +from SYS.logger import log, debug, is_debug_enabled from API.HTTP import HTTPClient from SYS.pipeline_progress import PipelineProgress from SYS.utils import ensure_directory, unique_path, unique_preserve_order @@ -27,6 +27,7 @@ Cmdlet = sh.Cmdlet CmdletArg = sh.CmdletArg SharedArgs = sh.SharedArgs create_pipe_object_result = sh.create_pipe_object_result +coerce_to_pipe_object = sh.coerce_to_pipe_object normalize_result_input = sh.normalize_result_input should_show_help = sh.should_show_help get_field = sh.get_field @@ -592,13 +593,33 @@ def _capture( } }) - tool.debug_dump() + if is_debug_enabled(): + try: + from rich.table import Table + from rich import box + t = Table(title="Screenshot Config", show_header=True, header_style="bold magenta", box=box.ROUNDED) + t.add_column("Property", style="cyan") + t.add_column("Value", style="green") + t.add_row("URL", options.url) + t.add_row("Format", _normalize_format(options.output_format)) + + # Browser details + defaults = getattr(tool, "defaults", None) + if defaults: + t.add_row("Browser", getattr(defaults, "browser", "unknown")) + t.add_row("Headless", str(getattr(defaults, "headless", "unknown"))) + t.add_row("Viewport", f"{getattr(defaults, 'viewport_width', '?')}x{getattr(defaults, 'viewport_height', '?')}") + t.add_row("Timeout", f"{getattr(defaults, 'navigation_timeout_ms', '?')}ms") + + t.add_row("Full Page", str(options.full_page)) + t.add_row("Destination", str(destination)) + debug(t) + except Exception: + pass - debug("Launching browser...") format_name = _normalize_format(options.output_format) headless = options.headless or format_name == "pdf" - debug(f"[_capture] Format: {format_name}, Headless: {headless}") - + if format_name == "pdf" and not options.headless: warnings.append( "pdf output requires headless Chromium; overriding headless mode" @@ -1129,6 +1150,8 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: is_temp=True, parent_hash=hashlib.sha256(url.encode()).hexdigest(), tag=merged_tags, + url=url, # Explicitly map url to top-level PipeObject field + source_url=url, # Map source_url as well extra={ "source_url": url, "archive_url": screenshot_result.archive_url, @@ -1141,6 +1164,24 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int: pipeline_context.emit(pipe_obj) all_emitted.append(pipe_obj) + # Debug: show PipeObject preview if enabled + if is_debug_enabled(): + try: + debug("[screen-shot] Output PipeObject preview") + po = coerce_to_pipe_object(pipe_obj) + from SYS.logger import _sanitize_pipe_object_for_debug as _sanitize # Or use helper if avail + # Add simple sanitize helper if not available + def _safe_table(obj): + try: + # Try calling debug_table on the object + if hasattr(obj, "debug_table"): + obj.debug_table() + except Exception: + pass + _safe_table(po) + except Exception: + pass + # If we created a local progress UI, advance it per completed item. progress.on_emit(pipe_obj) diff --git a/tool/playwright.py b/tool/playwright.py index 4561426..a787671 100644 --- a/tool/playwright.py +++ b/tool/playwright.py @@ -253,76 +253,6 @@ class PlaywrightTool: ) -def config_schema() -> List[Dict[str, Any]]: - """Return a schema describing editable Playwright tool defaults for the config UI. - - Notes: - - `user_agent` is a dropdown with a `custom` option; put the real UA in - `user_agent_custom` when choosing `custom`. - - Viewport dimensions are offered as convenient choices. - - `ffmpeg_path` intentionally defaults to empty; Playwright will consult - a global `FFMPEG_PATH` environment variable (or fallback to bundled/system). - """ - _defaults = PlaywrightDefaults() - - browser_choices = ["chromium", "firefox", "webkit"] - viewport_width_choices = [1920, 1366, 1280, 1024, 800] - viewport_height_choices = [1080, 900, 768, 720, 600] - - return [ - { - "key": "browser", - "label": "Playwright browser", - "default": _defaults.browser, - "choices": browser_choices, - }, - { - "key": "headless", - "label": "Headless", - "default": str(_defaults.headless), - "choices": ["true", "false"], - }, - { - "key": "user_agent", - "label": "User Agent", - "default": "default", - "choices": ["default", "native", "custom"], - }, - { - "key": "user_agent_custom", - "label": "Custom User Agent (used when User Agent = custom)", - "default": "", - }, - { - "key": "viewport_width", - "label": "Viewport width", - "default": _defaults.viewport_width, - "choices": viewport_width_choices, - }, - { - "key": "viewport_height", - "label": "Viewport height", - "default": _defaults.viewport_height, - "choices": viewport_height_choices, - }, - { - "key": "navigation_timeout_ms", - "label": "Navigation timeout (ms)", - "default": _defaults.navigation_timeout_ms, - }, - { - "key": "ignore_https_errors", - "label": "Ignore HTTPS errors", - "default": str(_defaults.ignore_https_errors), - "choices": ["true", "false"], - }, - { - "key": "ffmpeg_path", - "label": "FFmpeg path (leave empty to use global/bundled)", - "default": "", - }, - ] - def require(self) -> None: """Ensure Playwright is present; raise a helpful RuntimeError if not.""" try: @@ -635,3 +565,74 @@ def config_schema() -> List[Dict[str, Any]]: except Exception: pass return None + + +def config_schema() -> List[Dict[str, Any]]: + """Return a schema describing editable Playwright tool defaults for the config UI. + + Notes: + - `user_agent` is a dropdown with a `custom` option; put the real UA in + `user_agent_custom` when choosing `custom`. + - Viewport dimensions are offered as convenient choices. + - `ffmpeg_path` intentionally defaults to empty; Playwright will consult + a global `FFMPEG_PATH` environment variable (or fallback to bundled/system). + """ + _defaults = PlaywrightDefaults() + + browser_choices = ["chromium", "firefox", "webkit"] + viewport_width_choices = [1920, 1366, 1280, 1024, 800] + viewport_height_choices = [1080, 900, 768, 720, 600] + + return [ + { + "key": "browser", + "label": "Playwright browser", + "default": _defaults.browser, + "choices": browser_choices, + }, + { + "key": "headless", + "label": "Headless", + "default": str(_defaults.headless), + "choices": ["true", "false"], + }, + { + "key": "user_agent", + "label": "User Agent", + "default": "default", + "choices": ["default", "native", "custom"], + }, + { + "key": "user_agent_custom", + "label": "Custom User Agent (used when User Agent = custom)", + "default": "", + }, + { + "key": "viewport_width", + "label": "Viewport width", + "default": _defaults.viewport_width, + "choices": viewport_width_choices, + }, + { + "key": "viewport_height", + "label": "Viewport height", + "default": _defaults.viewport_height, + "choices": viewport_height_choices, + }, + { + "key": "navigation_timeout_ms", + "label": "Navigation timeout (ms)", + "default": _defaults.navigation_timeout_ms, + }, + { + "key": "ignore_https_errors", + "label": "Ignore HTTPS errors", + "default": str(_defaults.ignore_https_errors), + "choices": ["true", "false"], + }, + { + "key": "ffmpeg_path", + "label": "FFmpeg path (leave empty to use global/bundled)", + "default": "", + }, + ] diff --git a/tool/ytdlp.py b/tool/ytdlp.py index f0b8676..dd0d015 100644 --- a/tool/ytdlp.py +++ b/tool/ytdlp.py @@ -770,8 +770,8 @@ class YtDlpTool: """Resolve numeric heights (720, 1080p) to yt-dlp height selectors. Examples: - "720" -> "bv*[height<=720]+ba" - "1080p" -> "bv*[height<=1080]+ba" + "720" -> "bestvideo[height<=720]+bestaudio/best[height<=720]" + "1080p" -> "bestvideo[height<=1080]+bestaudio/best[height<=1080]" """ if not format_str or not isinstance(format_str, str): return None @@ -783,10 +783,33 @@ class YtDlpTool: # Strip trailing 'p' if present (e.g. 720p -> 720) if s.endswith('p'): s = s[:-1] + + # Heuristic: 240/360/480/720/1080/1440/2160 are common height inputs + # But small IDs like 18, 22, 137 are format IDs. + # YouTube Format IDs are usually 2-3 digits. + # Heights are also 3-4 digits. + # "240" is ambiguous (Format 240 vs Height 240). + # We assume common video heights are intended as heights. if s.isdigit(): - height = int(s) - if height >= 144: - return f"bv*[height<={height}]+ba" + val = int(s) + + # Common video heights that overlap with format IDs: + # - None currently overlap with common legacy itag IDs (17,18,22,34-38,43-46) + # or dash video IDs (133-137, 160, 242-248, 264, 271, 278, 298-315...). + # - 240, 360, 480, 720, 1080, 1440, 2160 + # + # Format 240 is likely not a thing (242, 243, ... exist). + # Format 480 ... none in common lists. + # Format 720 ... none. + # So if it looks like a standard resolution, treat as height constraint. + + if val in {144, 240, 360, 480, 540, 720, 1080, 1440, 2160, 2880, 4320}: + return f"bestvideo[height<={val}]+bestaudio/best[height<={val}]" + + # If user types something like 500, we can also treat as height constraint if > 100 + if val >= 100 and val not in {133, 134, 135, 136, 137, 160, 242, 243, 244, 247, 248, 278, 394, 395, 396, 397, 398, 399}: + return f"bestvideo[height<={val}]+bestaudio/best[height<={val}]" + return None def _load_defaults(self) -> YtDlpDefaults: