goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

f

Browse Source
This commit is contained in:
Nose
2026-01-12 13:51:26 -08:00
parent b7b58f0e42
commit 065ceeb1da
5 changed files with 172 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
SYS/metadata.py
View File

@@ -335,7 +335,22 @@ def normalize_urls(value: Any) -> List[str]:
" ").replace(",", " ").replace(",",
" ").split(): " ").split():
if token: if token:
yield token t_low = token.lower()
# Heuristic: only yield tokens that look like URLs or common address patterns.
# This prevents plain tags (e.g. "tag1, tag2") from leaking into URL fields.
is_p_url = t_low.startswith(("http://",
"https://",
"magnet:",
"torrent:",
"ytdl://",
"data:",
"ftp:",
"sftp:"))
is_struct_url = ("." in token and "/" in token
and not token.startswith((".",
"/")))
if is_p_url or is_struct_url:
yield token
return return
if isinstance(raw, (list, tuple, set)): if isinstance(raw, (list, tuple, set)):

30
Store/registry.py
View File

@@ -25,6 +25,8 @@ from Store._base import Store as BaseStore
_SHA256_HEX_RE = re.compile(r"^[0-9a-fA-F]{64}$") _SHA256_HEX_RE = re.compile(r"^[0-9a-fA-F]{64}$")
_DISCOVERED_CLASSES_CACHE: Optional[Dict[str, Type[BaseStore]]] = None
# Backends that failed to initialize earlier in the current process. # Backends that failed to initialize earlier in the current process.
# Keyed by (store_type, instance_key) where instance_key is the name used under config.store.<type>.<instance_key>. # Keyed by (store_type, instance_key) where instance_key is the name used under config.store.<type>.<instance_key>.
_FAILED_BACKEND_CACHE: Dict[tuple[str, _FAILED_BACKEND_CACHE: Dict[tuple[str,
@@ -56,6 +58,10 @@ def _discover_store_classes() -> Dict[str, Type[BaseStore]]:
Convention: Convention:
- The store type key is the normalized class name (e.g. HydrusNetwork -> hydrusnetwork). - The store type key is the normalized class name (e.g. HydrusNetwork -> hydrusnetwork).
""" """
global _DISCOVERED_CLASSES_CACHE
if _DISCOVERED_CLASSES_CACHE is not None:
return _DISCOVERED_CLASSES_CACHE
import Store as store_pkg import Store as store_pkg
discovered: Dict[str, discovered: Dict[str,
@@ -67,15 +73,21 @@ def _discover_store_classes() -> Dict[str, Type[BaseStore]]:
"registry"}: "registry"}:
continue continue
module = importlib.import_module(f"Store.{module_name}") try:
for _, obj in vars(module).items(): module = importlib.import_module(f"Store.{module_name}")
if not inspect.isclass(obj): for _, obj in vars(module).items():
continue if not inspect.isclass(obj):
if obj is BaseStore: continue
continue if obj is BaseStore:
if not issubclass(obj, BaseStore): continue
continue if not issubclass(obj, BaseStore):
discovered[_normalize_store_type(obj.__name__)] = obj continue
discovered[_normalize_store_type(obj.__name__)] = obj
except Exception as exc:
debug(f"[Store] Failed to import module '{module_name}': {exc}")
continue
_DISCOVERED_CLASSES_CACHE = discovered
return discovered return discovered

3
cmdlet/search_file.py
View File

@@ -667,10 +667,9 @@ class search_file(Cmdlet):
pass pass
from Store import Store from Store import Store
storage = Store(config=config or {})
from Store._base import Store as BaseStore from Store._base import Store as BaseStore
storage = storage_registry
backend_to_search = storage_backend or None backend_to_search = storage_backend or None
if hash_query: if hash_query:
# Explicit hash list search: build rows from backend metadata. # Explicit hash list search: build rows from backend metadata.

5
readme.md
View File

@@ -48,8 +48,9 @@ python Medios-Macina/scripts/bootstrap.py
</details> </details>
<br> <br>
<b>Start the CLI by simply running "mm"</b> <b>Start the CLI by simply running "mm"</b>
</div> </div>
<img src="https://avatars.githubusercontent.com/u/79589310?s=48&v=4">ytdlp</img>
<img src="https://avatars.githubusercontent.com/u/3878678?s=48&v=4
">hydrusnetwork</img>

282
scripts/bootstrap.py
View File

@@ -62,9 +62,57 @@ import sys
import time import time
def run(cmd: list[str]) -> None: def run(cmd: list[str], quiet: bool = False, debug: bool = False, cwd: Optional[Path] = None) -> None:
print(f"> {' '.join(cmd)}") if debug:
subprocess.check_call(cmd) print(f"\n> {' '.join(cmd)}")
if quiet and not debug:
subprocess.check_call(
cmd,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
cwd=str(cwd) if cwd else None
)
else:
if not debug:
print(f"> {' '.join(cmd)}")
subprocess.check_call(cmd, cwd=str(cwd) if cwd else None)
class ProgressBar:
def __init__(self, total: int, quiet: bool = False):
self.total = total
self.current = 0
self.quiet = quiet
self.bar_width = 40
def update(self, step_name: str):
if self.current < self.total:
self.current += 1
if self.quiet:
return
percent = int(100 * (self.current / self.total))
filled = int(self.bar_width * self.current // self.total)
bar = "█" * filled + "░" * (self.bar_width - filled)
sys.stdout.write(f"\r [{bar}] {percent:3}% | {step_name.ljust(30)}")
sys.stdout.flush()
if self.current == self.total:
sys.stdout.write("\n")
sys.stdout.flush()
LOGO = r"""
███╗ ███╗███████╗██████╗ ███████╗██╗ █████╗ ███╗ ███╗ █████╗ ██████╗██╗███╗ ██╗ █████╗
████╗ ████║██╔════╝██╔══██╗██╔════╝██║██╔══██╗ ████╗ ████║██╔══██╗██╔════╝██║████╗ ██║██╔══██╗
██╔████╔██║█████╗ ██║ ██║█████╗ ██║███████║ ██╔████╔██║███████║██║ ██║██╔██╗ ██║███████║
██║╚██╔╝██║██╔══╝ ██║ ██║██╔══╝ ██║██╔══██║ ██║╚██╔╝██║██╔══██║██║ ██║██║╚██╗██║██╔══██║
██║ ╚═╝ ██║███████╗██████╔╝███████╗██║██║ ██║ ██║ ╚═╝ ██║██║ ██║╚██████╗██║██║ ╚████║██║ ██║
╚═╝ ╚═╝╚══════╝╚═════╝ ╚══════╝╚═╝╚═╝ ╚═╝ ╚═╝ ╚═╝╚═╝ ╚═╝ ╚═════╝╚═╝╚═╝ ╚═══╝╚═╝ ╚═╝
(BOOTSTRAP INSTALLER)
"""
# Helpers to find shell executables and to run the platform-specific # Helpers to find shell executables and to run the platform-specific
@@ -857,6 +905,23 @@ def main() -> int:
if sys.version_info < (3, 8): if sys.version_info < (3, 8):
print("Warning: Python 3.8+ is recommended.", file=sys.stderr) print("Warning: Python 3.8+ is recommended.", file=sys.stderr)
# UI setup: Logo and Progress Bar
if not args.quiet and not args.debug:
print(LOGO)
# Determine total steps for progress bar
total_steps = 7 # Base: venv, pip, deps, project, cli, finalize, env
if args.upgrade_pip: total_steps += 1
if not args.no_playwright: total_steps += 1 # Playwright is combined pkg+browsers
if not getattr(args, "no_mpv", False): total_steps += 1
if not getattr(args, "no_deno", False): total_steps += 1
pb = ProgressBar(total_steps, quiet=args.quiet or args.debug)
def _run_cmd(cmd: list[str], cwd: Optional[Path] = None):
"""Helper to run commands with shared settings."""
run(cmd, quiet=not args.debug, debug=args.debug, cwd=cwd)
# Opinionated: always create or use a local venv at the project root (.venv) # Opinionated: always create or use a local venv at the project root (.venv)
venv_dir = repo_root / ".venv" venv_dir = repo_root / ".venv"
@@ -868,7 +933,7 @@ def main() -> int:
if "scripts" in str(venv_dir).lower(): if "scripts" in str(venv_dir).lower():
print(f"WARNING: venv path contains 'scripts': {venv_dir}", file=sys.stderr) print(f"WARNING: venv path contains 'scripts': {venv_dir}", file=sys.stderr)
def _venv_python(p: Path) -> Path: def _venv_python_bin(p: Path) -> Path:
if platform.system().lower() == "windows": if platform.system().lower() == "windows":
return p / "Scripts" / "python.exe" return p / "Scripts" / "python.exe"
return p / "bin" / "python" return p / "bin" / "python"
@@ -878,20 +943,13 @@ def main() -> int:
try: try:
if not venv_dir.exists(): if not venv_dir.exists():
if not args.quiet: _run_cmd([sys.executable, "-m", "venv", str(venv_dir)])
print(f"Creating local virtualenv at: {venv_dir}")
run([sys.executable, "-m", "venv", str(venv_dir)]) py = _venv_python_bin(venv_dir)
else:
if not args.quiet:
print(f"Using existing virtualenv at: {venv_dir}")
py = _venv_python(venv_dir)
if not py.exists(): if not py.exists():
# Try recreating venv if python is missing _run_cmd([sys.executable, "-m", "venv", str(venv_dir)])
if not args.quiet:
print(f"Local venv python not found at {py}; recreating venv") py = _venv_python_bin(venv_dir)
run([sys.executable, "-m", "venv", str(venv_dir)])
py = _venv_python(venv_dir)
if not py.exists(): if not py.exists():
raise RuntimeError(f"Unable to locate venv python at {py}") raise RuntimeError(f"Unable to locate venv python at {py}")
return py return py
@@ -913,10 +971,8 @@ def main() -> int:
except Exception: except Exception:
pass pass
if not args.quiet:
print("Bootstrapping pip inside the local virtualenv...")
try: try:
run([str(python_path), "-m", "ensurepip", "--upgrade"]) _run_cmd([str(python_path), "-m", "ensurepip", "--upgrade"])
except subprocess.CalledProcessError as exc: except subprocess.CalledProcessError as exc:
print( print(
"Failed to install pip inside the local virtualenv via ensurepip; ensure your Python build includes ensurepip and retry.", "Failed to install pip inside the local virtualenv via ensurepip; ensure your Python build includes ensurepip and retry.",
@@ -924,10 +980,12 @@ def main() -> int:
) )
raise raise
# Ensure a local venv is present and use it for subsequent installs. # 1. Virtual Environment Setup
pb.update("Preparing virtual environment...")
venv_python = _ensure_local_venv() venv_python = _ensure_local_venv()
if not args.quiet:
print(f"Using venv python: {venv_python}") # 2. Pip Availability
pb.update("Checking for pip...")
_ensure_pip_available(venv_python) _ensure_pip_available(venv_python)
# Enforce opinionated behavior: install deps, playwright, deno, and install project in editable mode. # Enforce opinionated behavior: install deps, playwright, deno, and install project in editable mode.
@@ -938,30 +996,23 @@ def main() -> int:
try: try:
if args.playwright_only: if args.playwright_only:
# Playwright browser install (short-circuit)
if not playwright_package_installed(): if not playwright_package_installed():
if not args.quiet: _run_cmd([sys.executable, "-m", "pip", "install", "--no-cache-dir", "playwright"])
print("'playwright' package not found; installing it via pip...")
run([sys.executable, "-m", "pip", "install", "--no-cache-dir", "playwright"])
if not args.quiet:
print(
"Installing Playwright browsers (this may download several hundred MB)..."
)
try: try:
cmd = _build_playwright_install_cmd(args.browsers) cmd = _build_playwright_install_cmd(args.browsers)
except ValueError as exc: cmd[0] = str(venv_python)
_run_cmd(cmd)
except Exception as exc:
print(f"Error: {exc}", file=sys.stderr) print(f"Error: {exc}", file=sys.stderr)
return 2 return 2
run(cmd)
if not args.quiet:
print("Playwright browsers installed successfully.")
return 0 return 0
# Progress tracking continues for full install
if args.upgrade_pip: if args.upgrade_pip:
if not args.quiet: pb.update("Upgrading pip/setuptools/wheel...")
print("Upgrading pip, setuptools, and wheel in local venv...") _run_cmd(
run(
[ [
str(venv_python), str(venv_python),
"-m", "-m",
@@ -975,60 +1026,39 @@ def main() -> int:
] ]
) )
if not args.skip_deps: # 4. Core Dependencies
req_file = repo_root / "scripts" / "requirements.txt" pb.update("Installing core dependencies...")
if not req_file.exists(): req_file = repo_root / "scripts" / "requirements.txt"
print( if req_file.exists():
f"requirements.txt not found at {req_file}; skipping dependency installation.", _run_cmd([str(venv_python), "-m", "pip", "install", "--no-cache-dir", "-r", str(req_file)])
file=sys.stderr,
)
else:
if not args.quiet:
print(
f"Installing Python dependencies into local venv from {req_file}..."
)
run([str(venv_python), "-m", "pip", "install", "--no-cache-dir", "-r", str(req_file)])
# 5. Playwright Setup
if not args.no_playwright: if not args.no_playwright:
pb.update("Setting up Playwright and browsers...")
if not playwright_package_installed(): if not playwright_package_installed():
if not args.quiet: _run_cmd([str(venv_python), "-m", "pip", "install", "--no-cache-dir", "playwright"])
print("'playwright' package not installed in venv; installing it...")
run([str(venv_python), "-m", "pip", "install", "--no-cache-dir", "playwright"])
if not args.quiet:
print(
"Installing Playwright browsers (this may download several hundred MB)..."
)
try: try:
cmd = _build_playwright_install_cmd(args.browsers) cmd = _build_playwright_install_cmd(args.browsers)
except ValueError as exc: cmd[0] = str(venv_python)
print(f"Error: {exc}", file=sys.stderr) _run_cmd(cmd)
return 2 except Exception:
pass
# Run Playwright install using the venv's python so binaries are available in venv # 6. Internal Components
cmd[0] = str(venv_python) pb.update("Installing internal components...")
run(cmd)
# Install the project into the local venv (editable mode is the default, opinionated)
if not args.quiet:
print("Installing project into local venv (editable mode)")
# Clean up old pip-generated entry point wrapper to avoid stale references
if platform.system() != "Windows": if platform.system() != "Windows":
old_mm = venv_dir / "bin" / "mm" old_mm = venv_dir / "bin" / "mm"
if old_mm.exists(): if old_mm.exists():
try: try:
old_mm.unlink() old_mm.unlink()
if not args.quiet:
print(f"Removed old entry point wrapper: {old_mm}")
except Exception: except Exception:
pass pass
run([str(venv_python), "-m", "pip", "install", "--no-cache-dir", "-e", str(repo_root / "scripts")]) _run_cmd([str(venv_python), "-m", "pip", "install", "--no-cache-dir", "-e", str(repo_root / "scripts")])
# Verify top-level 'CLI' import and, if missing, attempt to make it available # 7. CLI Verification
if not args.quiet: pb.update("Verifying CLI configuration...")
print("Verifying top-level 'CLI' import in venv...")
try: try:
rc = subprocess.run( rc = subprocess.run(
[ [
@@ -1036,14 +1066,11 @@ def main() -> int:
"-c", "-c",
"import importlib; importlib.import_module('CLI')" "import importlib; importlib.import_module('CLI')"
], ],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False, check=False,
) )
if rc.returncode == 0: if rc.returncode != 0:
print("OK: top-level 'CLI' is importable in the venv.")
else:
print(
"Top-level 'CLI' not importable; attempting to add repo path to venv site-packages via a .pth file..."
)
cmd = [ cmd = [
str(venv_python), str(venv_python),
"-c", "-c",
@@ -1063,84 +1090,36 @@ def main() -> int:
if sp and Path(sp).exists(): if sp and Path(sp).exists():
site_dir = Path(sp) site_dir = Path(sp)
break break
if site_dir is None: if site_dir:
print(
"Could not determine venv site-packages directory; skipping .pth fallback"
)
else:
pth_file = site_dir / "medeia_repo.pth" pth_file = site_dir / "medeia_repo.pth"
content = str(repo_root) + "\n"
if pth_file.exists(): if pth_file.exists():
txt = pth_file.read_text(encoding="utf-8") txt = pth_file.read_text(encoding="utf-8")
if str(repo_root) in txt: if str(repo_root) not in txt:
print(f".pth already contains repo root: {pth_file}")
else:
with pth_file.open("a", encoding="utf-8") as fh: with pth_file.open("a", encoding="utf-8") as fh:
fh.write(str(repo_root) + "\n") fh.write(content)
print(f"Appended repo root to existing .pth: {pth_file}")
else: else:
with pth_file.open("w", encoding="utf-8") as fh: with pth_file.open("w", encoding="utf-8") as fh:
fh.write(str(repo_root) + "\n") fh.write(content)
print( except Exception:
f"Wrote .pth adding repo root to venv site-packages: {pth_file}" pass
)
# Re-check whether CLI can be imported now
rc2 = subprocess.run(
[
str(venv_python),
"-c",
"import importlib; importlib.import_module('CLI')",
],
check=False,
)
if rc2.returncode == 0:
print("Top-level 'CLI' import works after adding .pth")
else:
print(
"Adding .pth did not make top-level 'CLI' importable; consider creating an egg-link or checking the venv."
)
except Exception as exc:
print(
f"Warning: failed to verify or modify site-packages for top-level CLI: {exc}"
)
# Check and install MPV if needed
install_mpv_requested = True
if getattr(args, "no_mpv", False):
install_mpv_requested = False
elif getattr(args, "install_mpv", False):
install_mpv_requested = True
# 8. MPV
install_mpv_requested = not getattr(args, "no_mpv", False)
if install_mpv_requested: if install_mpv_requested:
if _check_mpv_installed(): pb.update("Setting up MPV media player...")
if not args.quiet: if not _check_mpv_installed():
print("MPV is already installed.") _install_mpv()
else:
if not args.quiet:
print("MPV not found in PATH. Attempting to install...")
rc = _install_mpv()
if rc != 0:
print("Warning: MPV installation failed. Install it manually from https://mpv.io/installation/", file=sys.stderr)
# Optional: install Deno runtime (default: install unless --no-deno is passed)
install_deno_requested = True
if getattr(args, "no_deno", False):
install_deno_requested = False
elif getattr(args, "install_deno", False):
install_deno_requested = True
# 9. Deno
install_deno_requested = not getattr(args, "no_deno", False)
if install_deno_requested: if install_deno_requested:
if _check_deno_installed(): pb.update("Setting up Deno runtime...")
if not args.quiet: if not _check_deno_installed():
print("Deno is already installed.") _install_deno(args.deno_version)
else:
if not args.quiet:
print("Installing Deno runtime (local/system)...")
rc = _install_deno(args.deno_version)
if rc != 0:
print("Warning: Deno installation failed.", file=sys.stderr)
# Write project-local launcher script under scripts/ to keep the repo root uncluttered. # 10. Finalizing setup
pb.update("Writing launcher scripts...")
def _write_launchers() -> None: def _write_launchers() -> None:
launcher_dir = repo_root / "scripts" launcher_dir = repo_root / "scripts"
launcher_dir.mkdir(parents=True, exist_ok=True) launcher_dir.mkdir(parents=True, exist_ok=True)
@@ -1197,7 +1176,8 @@ if (Test-Path (Join-Path $repo 'CLI.py')) {
_write_launchers() _write_launchers()
# Install user-global shims so `mm` can be executed from any shell session. # 11. Global Environment
pb.update("Configuring global environment...")
def _install_user_shims(repo: Path) -> None: def _install_user_shims(repo: Path) -> None:
try: try:
home = Path.home() home = Path.home()
@@ -1221,7 +1201,7 @@ if (Test-Path (Join-Path $repo 'CLI.py')) {
"setlocal enabledelayedexpansion\n" "setlocal enabledelayedexpansion\n"
f'set "REPO={repo_bat_str}"\n' f'set "REPO={repo_bat_str}"\n'
"\n" "\n"
"# Automatically check for updates if this is a git repository\n" ":: Automatically check for updates if this is a git repository\n"
"if not defined MM_NO_UPDATE (\n" "if not defined MM_NO_UPDATE (\n"
" if exist \"!REPO!\\.git\" (\n" " if exist \"!REPO!\\.git\" (\n"
" set \"AUTO_UPDATE=true\"\n" " set \"AUTO_UPDATE=true\"\n"