From 065ceeb1dae92c6f9ff24d23f1c0bfc65bda286d Mon Sep 17 00:00:00 2001 From: Nose Date: Mon, 12 Jan 2026 13:51:26 -0800 Subject: [PATCH] f --- SYS/metadata.py | 17 ++- Store/registry.py | 30 +++-- cmdlet/search_file.py | 3 +- readme.md | 5 +- scripts/bootstrap.py | 282 ++++++++++++++++++++---------------------- 5 files changed, 172 insertions(+), 165 deletions(-) diff --git a/SYS/metadata.py b/SYS/metadata.py index 8971582..5737535 100644 --- a/SYS/metadata.py +++ b/SYS/metadata.py @@ -335,7 +335,22 @@ def normalize_urls(value: Any) -> List[str]: " ").replace(",", " ").split(): if token: - yield token + t_low = token.lower() + # Heuristic: only yield tokens that look like URLs or common address patterns. + # This prevents plain tags (e.g. "tag1, tag2") from leaking into URL fields. + is_p_url = t_low.startswith(("http://", + "https://", + "magnet:", + "torrent:", + "ytdl://", + "data:", + "ftp:", + "sftp:")) + is_struct_url = ("." in token and "/" in token + and not token.startswith((".", + "/"))) + if is_p_url or is_struct_url: + yield token return if isinstance(raw, (list, tuple, set)): diff --git a/Store/registry.py b/Store/registry.py index 9c531cb..831889b 100644 --- a/Store/registry.py +++ b/Store/registry.py @@ -25,6 +25,8 @@ from Store._base import Store as BaseStore _SHA256_HEX_RE = re.compile(r"^[0-9a-fA-F]{64}$") +_DISCOVERED_CLASSES_CACHE: Optional[Dict[str, Type[BaseStore]]] = None + # Backends that failed to initialize earlier in the current process. # Keyed by (store_type, instance_key) where instance_key is the name used under config.store... _FAILED_BACKEND_CACHE: Dict[tuple[str, @@ -56,6 +58,10 @@ def _discover_store_classes() -> Dict[str, Type[BaseStore]]: Convention: - The store type key is the normalized class name (e.g. HydrusNetwork -> hydrusnetwork). """ + global _DISCOVERED_CLASSES_CACHE + if _DISCOVERED_CLASSES_CACHE is not None: + return _DISCOVERED_CLASSES_CACHE + import Store as store_pkg discovered: Dict[str, @@ -67,15 +73,21 @@ def _discover_store_classes() -> Dict[str, Type[BaseStore]]: "registry"}: continue - module = importlib.import_module(f"Store.{module_name}") - for _, obj in vars(module).items(): - if not inspect.isclass(obj): - continue - if obj is BaseStore: - continue - if not issubclass(obj, BaseStore): - continue - discovered[_normalize_store_type(obj.__name__)] = obj + try: + module = importlib.import_module(f"Store.{module_name}") + for _, obj in vars(module).items(): + if not inspect.isclass(obj): + continue + if obj is BaseStore: + continue + if not issubclass(obj, BaseStore): + continue + discovered[_normalize_store_type(obj.__name__)] = obj + except Exception as exc: + debug(f"[Store] Failed to import module '{module_name}': {exc}") + continue + + _DISCOVERED_CLASSES_CACHE = discovered return discovered diff --git a/cmdlet/search_file.py b/cmdlet/search_file.py index 52d482c..d04b17d 100644 --- a/cmdlet/search_file.py +++ b/cmdlet/search_file.py @@ -667,10 +667,9 @@ class search_file(Cmdlet): pass from Store import Store - - storage = Store(config=config or {}) from Store._base import Store as BaseStore + storage = storage_registry backend_to_search = storage_backend or None if hash_query: # Explicit hash list search: build rows from backend metadata. diff --git a/readme.md b/readme.md index b8ebeb5..700f389 100644 --- a/readme.md +++ b/readme.md @@ -48,8 +48,9 @@ python Medios-Macina/scripts/bootstrap.py
Start the CLI by simply running "mm" - - +ytdlp +hydrusnetwork diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py index ae2f839..0f5e8f5 100644 --- a/scripts/bootstrap.py +++ b/scripts/bootstrap.py @@ -62,9 +62,57 @@ import sys import time -def run(cmd: list[str]) -> None: - print(f"> {' '.join(cmd)}") - subprocess.check_call(cmd) +def run(cmd: list[str], quiet: bool = False, debug: bool = False, cwd: Optional[Path] = None) -> None: + if debug: + print(f"\n> {' '.join(cmd)}") + + if quiet and not debug: + subprocess.check_call( + cmd, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + cwd=str(cwd) if cwd else None + ) + else: + if not debug: + print(f"> {' '.join(cmd)}") + subprocess.check_call(cmd, cwd=str(cwd) if cwd else None) + + +class ProgressBar: + def __init__(self, total: int, quiet: bool = False): + self.total = total + self.current = 0 + self.quiet = quiet + self.bar_width = 40 + + def update(self, step_name: str): + if self.current < self.total: + self.current += 1 + if self.quiet: + return + + percent = int(100 * (self.current / self.total)) + filled = int(self.bar_width * self.current // self.total) + bar = "█" * filled + "░" * (self.bar_width - filled) + + sys.stdout.write(f"\r [{bar}] {percent:3}% | {step_name.ljust(30)}") + sys.stdout.flush() + + if self.current == self.total: + sys.stdout.write("\n") + sys.stdout.flush() + + +LOGO = r""" + ███╗ ███╗███████╗██████╗ ███████╗██╗ █████╗ ███╗ ███╗ █████╗ ██████╗██╗███╗ ██╗ █████╗ + ████╗ ████║██╔════╝██╔══██╗██╔════╝██║██╔══██╗ ████╗ ████║██╔══██╗██╔════╝██║████╗ ██║██╔══██╗ + ██╔████╔██║█████╗ ██║ ██║█████╗ ██║███████║ ██╔████╔██║███████║██║ ██║██╔██╗ ██║███████║ + ██║╚██╔╝██║██╔══╝ ██║ ██║██╔══╝ ██║██╔══██║ ██║╚██╔╝██║██╔══██║██║ ██║██║╚██╗██║██╔══██║ + ██║ ╚═╝ ██║███████╗██████╔╝███████╗██║██║ ██║ ██║ ╚═╝ ██║██║ ██║╚██████╗██║██║ ╚████║██║ ██║ + ╚═╝ ╚═╝╚══════╝╚═════╝ ╚══════╝╚═╝╚═╝ ╚═╝ ╚═╝ ╚═╝╚═╝ ╚═╝ ╚═════╝╚═╝╚═╝ ╚═══╝╚═╝ ╚═╝ + (BOOTSTRAP INSTALLER) +""" # Helpers to find shell executables and to run the platform-specific @@ -857,6 +905,23 @@ def main() -> int: if sys.version_info < (3, 8): print("Warning: Python 3.8+ is recommended.", file=sys.stderr) + # UI setup: Logo and Progress Bar + if not args.quiet and not args.debug: + print(LOGO) + + # Determine total steps for progress bar + total_steps = 7 # Base: venv, pip, deps, project, cli, finalize, env + if args.upgrade_pip: total_steps += 1 + if not args.no_playwright: total_steps += 1 # Playwright is combined pkg+browsers + if not getattr(args, "no_mpv", False): total_steps += 1 + if not getattr(args, "no_deno", False): total_steps += 1 + + pb = ProgressBar(total_steps, quiet=args.quiet or args.debug) + + def _run_cmd(cmd: list[str], cwd: Optional[Path] = None): + """Helper to run commands with shared settings.""" + run(cmd, quiet=not args.debug, debug=args.debug, cwd=cwd) + # Opinionated: always create or use a local venv at the project root (.venv) venv_dir = repo_root / ".venv" @@ -868,7 +933,7 @@ def main() -> int: if "scripts" in str(venv_dir).lower(): print(f"WARNING: venv path contains 'scripts': {venv_dir}", file=sys.stderr) - def _venv_python(p: Path) -> Path: + def _venv_python_bin(p: Path) -> Path: if platform.system().lower() == "windows": return p / "Scripts" / "python.exe" return p / "bin" / "python" @@ -878,20 +943,13 @@ def main() -> int: try: if not venv_dir.exists(): - if not args.quiet: - print(f"Creating local virtualenv at: {venv_dir}") - run([sys.executable, "-m", "venv", str(venv_dir)]) - else: - if not args.quiet: - print(f"Using existing virtualenv at: {venv_dir}") - - py = _venv_python(venv_dir) + _run_cmd([sys.executable, "-m", "venv", str(venv_dir)]) + + py = _venv_python_bin(venv_dir) if not py.exists(): - # Try recreating venv if python is missing - if not args.quiet: - print(f"Local venv python not found at {py}; recreating venv") - run([sys.executable, "-m", "venv", str(venv_dir)]) - py = _venv_python(venv_dir) + _run_cmd([sys.executable, "-m", "venv", str(venv_dir)]) + + py = _venv_python_bin(venv_dir) if not py.exists(): raise RuntimeError(f"Unable to locate venv python at {py}") return py @@ -913,10 +971,8 @@ def main() -> int: except Exception: pass - if not args.quiet: - print("Bootstrapping pip inside the local virtualenv...") try: - run([str(python_path), "-m", "ensurepip", "--upgrade"]) + _run_cmd([str(python_path), "-m", "ensurepip", "--upgrade"]) except subprocess.CalledProcessError as exc: print( "Failed to install pip inside the local virtualenv via ensurepip; ensure your Python build includes ensurepip and retry.", @@ -924,10 +980,12 @@ def main() -> int: ) raise - # Ensure a local venv is present and use it for subsequent installs. + # 1. Virtual Environment Setup + pb.update("Preparing virtual environment...") venv_python = _ensure_local_venv() - if not args.quiet: - print(f"Using venv python: {venv_python}") + + # 2. Pip Availability + pb.update("Checking for pip...") _ensure_pip_available(venv_python) # Enforce opinionated behavior: install deps, playwright, deno, and install project in editable mode. @@ -938,30 +996,23 @@ def main() -> int: try: if args.playwright_only: + # Playwright browser install (short-circuit) if not playwright_package_installed(): - if not args.quiet: - print("'playwright' package not found; installing it via pip...") - run([sys.executable, "-m", "pip", "install", "--no-cache-dir", "playwright"]) + _run_cmd([sys.executable, "-m", "pip", "install", "--no-cache-dir", "playwright"]) - if not args.quiet: - print( - "Installing Playwright browsers (this may download several hundred MB)..." - ) try: cmd = _build_playwright_install_cmd(args.browsers) - except ValueError as exc: + cmd[0] = str(venv_python) + _run_cmd(cmd) + except Exception as exc: print(f"Error: {exc}", file=sys.stderr) return 2 - - run(cmd) - if not args.quiet: - print("Playwright browsers installed successfully.") return 0 + # Progress tracking continues for full install if args.upgrade_pip: - if not args.quiet: - print("Upgrading pip, setuptools, and wheel in local venv...") - run( + pb.update("Upgrading pip/setuptools/wheel...") + _run_cmd( [ str(venv_python), "-m", @@ -975,60 +1026,39 @@ def main() -> int: ] ) - if not args.skip_deps: - req_file = repo_root / "scripts" / "requirements.txt" - if not req_file.exists(): - print( - f"requirements.txt not found at {req_file}; skipping dependency installation.", - file=sys.stderr, - ) - else: - if not args.quiet: - print( - f"Installing Python dependencies into local venv from {req_file}..." - ) - run([str(venv_python), "-m", "pip", "install", "--no-cache-dir", "-r", str(req_file)]) + # 4. Core Dependencies + pb.update("Installing core dependencies...") + req_file = repo_root / "scripts" / "requirements.txt" + if req_file.exists(): + _run_cmd([str(venv_python), "-m", "pip", "install", "--no-cache-dir", "-r", str(req_file)]) + # 5. Playwright Setup if not args.no_playwright: + pb.update("Setting up Playwright and browsers...") if not playwright_package_installed(): - if not args.quiet: - print("'playwright' package not installed in venv; installing it...") - run([str(venv_python), "-m", "pip", "install", "--no-cache-dir", "playwright"]) + _run_cmd([str(venv_python), "-m", "pip", "install", "--no-cache-dir", "playwright"]) - if not args.quiet: - print( - "Installing Playwright browsers (this may download several hundred MB)..." - ) try: cmd = _build_playwright_install_cmd(args.browsers) - except ValueError as exc: - print(f"Error: {exc}", file=sys.stderr) - return 2 + cmd[0] = str(venv_python) + _run_cmd(cmd) + except Exception: + pass - # Run Playwright install using the venv's python so binaries are available in venv - cmd[0] = str(venv_python) - run(cmd) - - # Install the project into the local venv (editable mode is the default, opinionated) - if not args.quiet: - print("Installing project into local venv (editable mode)") - - # Clean up old pip-generated entry point wrapper to avoid stale references + # 6. Internal Components + pb.update("Installing internal components...") if platform.system() != "Windows": old_mm = venv_dir / "bin" / "mm" if old_mm.exists(): try: old_mm.unlink() - if not args.quiet: - print(f"Removed old entry point wrapper: {old_mm}") except Exception: pass - run([str(venv_python), "-m", "pip", "install", "--no-cache-dir", "-e", str(repo_root / "scripts")]) + _run_cmd([str(venv_python), "-m", "pip", "install", "--no-cache-dir", "-e", str(repo_root / "scripts")]) - # Verify top-level 'CLI' import and, if missing, attempt to make it available - if not args.quiet: - print("Verifying top-level 'CLI' import in venv...") + # 7. CLI Verification + pb.update("Verifying CLI configuration...") try: rc = subprocess.run( [ @@ -1036,14 +1066,11 @@ def main() -> int: "-c", "import importlib; importlib.import_module('CLI')" ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, check=False, ) - if rc.returncode == 0: - print("OK: top-level 'CLI' is importable in the venv.") - else: - print( - "Top-level 'CLI' not importable; attempting to add repo path to venv site-packages via a .pth file..." - ) + if rc.returncode != 0: cmd = [ str(venv_python), "-c", @@ -1063,84 +1090,36 @@ def main() -> int: if sp and Path(sp).exists(): site_dir = Path(sp) break - if site_dir is None: - print( - "Could not determine venv site-packages directory; skipping .pth fallback" - ) - else: + if site_dir: pth_file = site_dir / "medeia_repo.pth" + content = str(repo_root) + "\n" if pth_file.exists(): txt = pth_file.read_text(encoding="utf-8") - if str(repo_root) in txt: - print(f".pth already contains repo root: {pth_file}") - else: + if str(repo_root) not in txt: with pth_file.open("a", encoding="utf-8") as fh: - fh.write(str(repo_root) + "\n") - print(f"Appended repo root to existing .pth: {pth_file}") + fh.write(content) else: with pth_file.open("w", encoding="utf-8") as fh: - fh.write(str(repo_root) + "\n") - print( - f"Wrote .pth adding repo root to venv site-packages: {pth_file}" - ) - - # Re-check whether CLI can be imported now - rc2 = subprocess.run( - [ - str(venv_python), - "-c", - "import importlib; importlib.import_module('CLI')", - ], - check=False, - ) - if rc2.returncode == 0: - print("Top-level 'CLI' import works after adding .pth") - else: - print( - "Adding .pth did not make top-level 'CLI' importable; consider creating an egg-link or checking the venv." - ) - except Exception as exc: - print( - f"Warning: failed to verify or modify site-packages for top-level CLI: {exc}" - ) - - # Check and install MPV if needed - install_mpv_requested = True - if getattr(args, "no_mpv", False): - install_mpv_requested = False - elif getattr(args, "install_mpv", False): - install_mpv_requested = True + fh.write(content) + except Exception: + pass + # 8. MPV + install_mpv_requested = not getattr(args, "no_mpv", False) if install_mpv_requested: - if _check_mpv_installed(): - if not args.quiet: - print("MPV is already installed.") - else: - if not args.quiet: - print("MPV not found in PATH. Attempting to install...") - rc = _install_mpv() - if rc != 0: - print("Warning: MPV installation failed. Install it manually from https://mpv.io/installation/", file=sys.stderr) - - # Optional: install Deno runtime (default: install unless --no-deno is passed) - install_deno_requested = True - if getattr(args, "no_deno", False): - install_deno_requested = False - elif getattr(args, "install_deno", False): - install_deno_requested = True + pb.update("Setting up MPV media player...") + if not _check_mpv_installed(): + _install_mpv() + # 9. Deno + install_deno_requested = not getattr(args, "no_deno", False) if install_deno_requested: - if _check_deno_installed(): - if not args.quiet: - print("Deno is already installed.") - else: - if not args.quiet: - print("Installing Deno runtime (local/system)...") - rc = _install_deno(args.deno_version) - if rc != 0: - print("Warning: Deno installation failed.", file=sys.stderr) + pb.update("Setting up Deno runtime...") + if not _check_deno_installed(): + _install_deno(args.deno_version) - # Write project-local launcher script under scripts/ to keep the repo root uncluttered. + # 10. Finalizing setup + pb.update("Writing launcher scripts...") def _write_launchers() -> None: launcher_dir = repo_root / "scripts" launcher_dir.mkdir(parents=True, exist_ok=True) @@ -1197,7 +1176,8 @@ if (Test-Path (Join-Path $repo 'CLI.py')) { _write_launchers() - # Install user-global shims so `mm` can be executed from any shell session. + # 11. Global Environment + pb.update("Configuring global environment...") def _install_user_shims(repo: Path) -> None: try: home = Path.home() @@ -1221,7 +1201,7 @@ if (Test-Path (Join-Path $repo 'CLI.py')) { "setlocal enabledelayedexpansion\n" f'set "REPO={repo_bat_str}"\n' "\n" - "# Automatically check for updates if this is a git repository\n" + ":: Automatically check for updates if this is a git repository\n" "if not defined MM_NO_UPDATE (\n" " if exist \"!REPO!\\.git\" (\n" " set \"AUTO_UPDATE=true\"\n"