dfd
This commit is contained in:
@@ -20,7 +20,7 @@ import time
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterator, List, Optional
|
||||
from urllib.parse import urljoin
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
@@ -62,14 +62,11 @@ def _progress_callback(status: Dict[str, Any]) -> None:
|
||||
percent = status.get("_percent_str", "?")
|
||||
speed = status.get("_speed_str", "?")
|
||||
eta = status.get("_eta_str", "?")
|
||||
# Print progress to stdout with carriage return to update in place
|
||||
sys.stdout.write(f"\r[download] {percent} at {speed} ETA {eta} ")
|
||||
sys.stdout.flush()
|
||||
elif event == "finished":
|
||||
# Clear the progress line
|
||||
sys.stdout.write("\r" + " " * 70 + "\r")
|
||||
sys.stdout.flush()
|
||||
# Log finished message (visible)
|
||||
debug(f"✓ Download finished: {status.get('filename')}")
|
||||
elif event in ("postprocessing", "processing"):
|
||||
debug(f"Post-processing: {status.get('postprocessor')}")
|
||||
@@ -99,17 +96,7 @@ def is_url_supported_by_ytdlp(url: str) -> bool:
|
||||
|
||||
|
||||
def list_formats(url: str, no_playlist: bool = False, playlist_items: Optional[str] = None) -> Optional[List[Dict[str, Any]]]:
|
||||
"""Get list of available formats for a URL using yt-dlp.
|
||||
|
||||
Args:
|
||||
url: URL to get formats for
|
||||
no_playlist: If True, ignore playlists and list formats for single video
|
||||
playlist_items: If specified, only list formats for these playlist items (e.g., "1,3,5-8")
|
||||
|
||||
Returns:
|
||||
List of format dictionaries with keys: format_id, format, resolution, fps, vcodec, acodec, filesize, etc.
|
||||
Returns None if yt-dlp is not available or format listing fails.
|
||||
"""
|
||||
"""Get list of available formats for a URL using yt-dlp."""
|
||||
_ensure_yt_dlp_ready()
|
||||
|
||||
try:
|
||||
@@ -118,28 +105,25 @@ def list_formats(url: str, no_playlist: bool = False, playlist_items: Optional[s
|
||||
"no_warnings": True,
|
||||
"socket_timeout": 30,
|
||||
}
|
||||
|
||||
# Add no_playlist option if specified
|
||||
|
||||
if no_playlist:
|
||||
ydl_opts["noplaylist"] = True
|
||||
|
||||
# Add playlist_items filter if specified
|
||||
|
||||
if playlist_items:
|
||||
ydl_opts["playlist_items"] = playlist_items
|
||||
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
debug(f"Fetching format list for: {url}")
|
||||
info = ydl.extract_info(url, download=False)
|
||||
|
||||
|
||||
formats = info.get("formats", [])
|
||||
if not formats:
|
||||
log("No formats available", file=sys.stderr)
|
||||
return None
|
||||
|
||||
# Parse and extract relevant format info
|
||||
|
||||
result_formats = []
|
||||
for fmt in formats:
|
||||
format_info = {
|
||||
result_formats.append({
|
||||
"format_id": fmt.get("format_id", ""),
|
||||
"format": fmt.get("format", ""),
|
||||
"ext": fmt.get("ext", ""),
|
||||
@@ -150,13 +134,12 @@ def list_formats(url: str, no_playlist: bool = False, playlist_items: Optional[s
|
||||
"vcodec": fmt.get("vcodec", "none"),
|
||||
"acodec": fmt.get("acodec", "none"),
|
||||
"filesize": fmt.get("filesize"),
|
||||
"tbr": fmt.get("tbr"), # Total bitrate
|
||||
}
|
||||
result_formats.append(format_info)
|
||||
|
||||
"tbr": fmt.get("tbr"),
|
||||
})
|
||||
|
||||
debug(f"Found {len(result_formats)} available formats")
|
||||
return result_formats
|
||||
|
||||
|
||||
except Exception as e:
|
||||
log(f"✗ Error fetching formats: {e}", file=sys.stderr)
|
||||
return None
|
||||
@@ -779,8 +762,28 @@ def download_media(
|
||||
debug_logger.write_record("libgen-resolve-failed", {"url": opts.url})
|
||||
return _download_direct_file(opts.url, opts.output_dir, debug_logger)
|
||||
|
||||
# Try yt-dlp first if URL is supported
|
||||
if not is_url_supported_by_ytdlp(opts.url):
|
||||
# Handle GoFile shares with a dedicated resolver before yt-dlp/direct fallbacks
|
||||
try:
|
||||
netloc = urlparse(opts.url).netloc.lower()
|
||||
except Exception:
|
||||
netloc = ""
|
||||
if "gofile.io" in netloc:
|
||||
msg = "GoFile links are currently unsupported"
|
||||
debug(msg)
|
||||
if debug_logger is not None:
|
||||
debug_logger.write_record("gofile-unsupported", {"url": opts.url})
|
||||
raise DownloadError(msg)
|
||||
|
||||
# Determine if yt-dlp should be used
|
||||
ytdlp_supported = is_url_supported_by_ytdlp(opts.url)
|
||||
if ytdlp_supported:
|
||||
probe_result = probe_url(opts.url, no_playlist=opts.no_playlist)
|
||||
if probe_result is None:
|
||||
log(f"URL supported by yt-dlp but no media detected, falling back to direct download: {opts.url}")
|
||||
if debug_logger is not None:
|
||||
debug_logger.write_record("ytdlp-skip-no-media", {"url": opts.url})
|
||||
return _download_direct_file(opts.url, opts.output_dir, debug_logger)
|
||||
else:
|
||||
log(f"URL not supported by yt-dlp, trying direct download: {opts.url}")
|
||||
if debug_logger is not None:
|
||||
debug_logger.write_record("direct-file-attempt", {"url": opts.url})
|
||||
|
||||
@@ -28,6 +28,41 @@ import re
|
||||
|
||||
from helper.logger import log, debug
|
||||
from helper.utils_constant import mime_maps
|
||||
from helper.utils import sha256_file
|
||||
|
||||
|
||||
HEX_DIGITS = set("0123456789abcdef")
|
||||
|
||||
|
||||
def _normalize_hex_hash(value: Optional[str]) -> Optional[str]:
|
||||
"""Return a normalized 64-character lowercase hash or None."""
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
cleaned = ''.join(ch for ch in str(value).strip().lower() if ch in HEX_DIGITS)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
if len(cleaned) == 64:
|
||||
return cleaned
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_file_hash(candidate: Optional[str], path: Path) -> Optional[str]:
|
||||
"""Return the given hash if valid, otherwise compute sha256 from disk."""
|
||||
normalized = _normalize_hex_hash(candidate)
|
||||
if normalized is not None:
|
||||
return normalized
|
||||
|
||||
if not path.exists():
|
||||
return None
|
||||
|
||||
try:
|
||||
return sha256_file(path)
|
||||
except Exception as exc:
|
||||
debug(f"Failed to compute hash for {path}: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
class StorageBackend(ABC):
|
||||
@@ -198,6 +233,39 @@ class LocalStorageBackend(StorageBackend):
|
||||
search_dir = Path(location).expanduser()
|
||||
debug(f"Searching local storage at: {search_dir}")
|
||||
|
||||
# Support comma-separated AND queries (token1,token2,...). Each token must match.
|
||||
tokens = [t.strip() for t in query.split(',') if t.strip()]
|
||||
|
||||
# Require explicit namespace for hash lookups to avoid accidental filename matches
|
||||
if not match_all and len(tokens) == 1 and _normalize_hex_hash(query_lower):
|
||||
debug("Hash queries require 'hash:' prefix for local search")
|
||||
return results
|
||||
|
||||
# Require explicit namespace for hash lookups to avoid accidental filename matches
|
||||
if not match_all and _normalize_hex_hash(query_lower):
|
||||
debug("Hash queries require 'hash:' prefix for local search")
|
||||
return results
|
||||
|
||||
def _create_entry(file_path: Path, tags: list[str], size_bytes: int | None, db_hash: Optional[str]) -> dict[str, Any]:
|
||||
path_str = str(file_path)
|
||||
entry = {
|
||||
"name": file_path.stem,
|
||||
"title": next((t.split(':', 1)[1] for t in tags if t.lower().startswith('title:')), file_path.stem),
|
||||
"ext": file_path.suffix.lstrip('.'),
|
||||
"path": path_str,
|
||||
"target": path_str,
|
||||
"origin": "local",
|
||||
"size": size_bytes,
|
||||
"size_bytes": size_bytes,
|
||||
"tags": tags,
|
||||
}
|
||||
hash_value = _resolve_file_hash(db_hash, file_path)
|
||||
if hash_value:
|
||||
entry["hash"] = hash_value
|
||||
entry["hash_hex"] = hash_value
|
||||
entry["file_hash"] = hash_value
|
||||
return entry
|
||||
|
||||
try:
|
||||
if not search_dir.exists():
|
||||
debug(f"Search directory does not exist: {search_dir}")
|
||||
@@ -209,17 +277,196 @@ class LocalStorageBackend(StorageBackend):
|
||||
cursor = db.connection.cursor()
|
||||
|
||||
# Check if query is a tag namespace search (format: "namespace:pattern")
|
||||
if tokens and len(tokens) > 1:
|
||||
# AND mode across comma-separated tokens
|
||||
def _like_pattern(term: str) -> str:
|
||||
return term.replace('*', '%').replace('?', '_')
|
||||
|
||||
def _ids_for_token(token: str, cursor) -> set[int]:
|
||||
token = token.strip()
|
||||
if not token:
|
||||
return set()
|
||||
|
||||
# Namespaced token
|
||||
if ':' in token and not token.startswith(':'):
|
||||
namespace, pattern = token.split(':', 1)
|
||||
namespace = namespace.strip().lower()
|
||||
pattern = pattern.strip().lower()
|
||||
|
||||
if namespace == 'hash':
|
||||
normalized_hash = _normalize_hex_hash(pattern)
|
||||
if not normalized_hash:
|
||||
return set()
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT id FROM files
|
||||
WHERE LOWER(file_hash) = ?
|
||||
""",
|
||||
(normalized_hash,)
|
||||
)
|
||||
return {row[0] for row in cursor.fetchall()}
|
||||
|
||||
if namespace == 'store':
|
||||
# Local backend only serves local store
|
||||
if pattern not in {'local', 'file', 'filesystem'}:
|
||||
return set()
|
||||
cursor.execute("SELECT id FROM files")
|
||||
return {row[0] for row in cursor.fetchall()}
|
||||
|
||||
# Generic namespace match on tags
|
||||
query_pattern = f"{namespace}:%"
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT DISTINCT f.id, t.tag
|
||||
FROM files f
|
||||
JOIN tags t ON f.id = t.file_id
|
||||
WHERE LOWER(t.tag) LIKE ?
|
||||
""",
|
||||
(query_pattern,)
|
||||
)
|
||||
matched: set[int] = set()
|
||||
for file_id, tag_val in cursor.fetchall():
|
||||
if not tag_val:
|
||||
continue
|
||||
tag_lower = str(tag_val).lower()
|
||||
if not tag_lower.startswith(f"{namespace}:"):
|
||||
continue
|
||||
value = tag_lower[len(namespace)+1:]
|
||||
if fnmatch(value, pattern):
|
||||
matched.add(int(file_id))
|
||||
return matched
|
||||
|
||||
# Bare token: match filename OR any tag (including title)
|
||||
term = token.lower()
|
||||
like_pattern = f"%{_like_pattern(term)}%"
|
||||
|
||||
ids: set[int] = set()
|
||||
# Filename match
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT DISTINCT id FROM files
|
||||
WHERE LOWER(file_path) LIKE ?
|
||||
""",
|
||||
(like_pattern,)
|
||||
)
|
||||
ids.update(int(row[0]) for row in cursor.fetchall())
|
||||
|
||||
# Tag match (any namespace, including title)
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT DISTINCT f.id
|
||||
FROM files f
|
||||
JOIN tags t ON f.id = t.file_id
|
||||
WHERE LOWER(t.tag) LIKE ?
|
||||
""",
|
||||
(like_pattern,)
|
||||
)
|
||||
ids.update(int(row[0]) for row in cursor.fetchall())
|
||||
return ids
|
||||
|
||||
try:
|
||||
with LocalLibraryDB(search_dir) as db:
|
||||
cursor = db.connection.cursor()
|
||||
matching_ids: set[int] | None = None
|
||||
for token in tokens:
|
||||
ids = _ids_for_token(token, cursor)
|
||||
matching_ids = ids if matching_ids is None else matching_ids & ids
|
||||
if not matching_ids:
|
||||
return results
|
||||
|
||||
if not matching_ids:
|
||||
return results
|
||||
|
||||
# Fetch rows for matching IDs
|
||||
placeholders = ",".join(["?"] * len(matching_ids))
|
||||
fetch_sql = f"""
|
||||
SELECT id, file_path, file_size, file_hash
|
||||
FROM files
|
||||
WHERE id IN ({placeholders})
|
||||
ORDER BY file_path
|
||||
LIMIT ?
|
||||
"""
|
||||
cursor.execute(fetch_sql, (*matching_ids, limit or len(matching_ids)))
|
||||
rows = cursor.fetchall()
|
||||
for file_id, file_path_str, size_bytes, file_hash in rows:
|
||||
if not file_path_str:
|
||||
continue
|
||||
file_path = Path(file_path_str)
|
||||
if not file_path.exists():
|
||||
continue
|
||||
if size_bytes is None:
|
||||
try:
|
||||
size_bytes = file_path.stat().st_size
|
||||
except OSError:
|
||||
size_bytes = None
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT tag FROM tags WHERE file_id = ?
|
||||
""",
|
||||
(file_id,),
|
||||
)
|
||||
tags = [row[0] for row in cursor.fetchall()]
|
||||
entry = _create_entry(file_path, tags, size_bytes, file_hash)
|
||||
results.append(entry)
|
||||
if limit is not None and len(results) >= limit:
|
||||
return results
|
||||
return results
|
||||
except Exception as exc:
|
||||
log(f"⚠️ AND search failed: {exc}", file=sys.stderr)
|
||||
debug(f"AND search exception details: {exc}")
|
||||
return []
|
||||
|
||||
if ":" in query and not query.startswith(":"):
|
||||
namespace, pattern = query.split(":", 1)
|
||||
namespace = namespace.strip().lower()
|
||||
pattern = pattern.strip().lower()
|
||||
debug(f"Performing namespace search: {namespace}:{pattern}")
|
||||
|
||||
# Special-case hash: lookups against file_hash column
|
||||
if namespace == "hash":
|
||||
normalized_hash = _normalize_hex_hash(pattern)
|
||||
if not normalized_hash:
|
||||
return results
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT id, file_path, file_size, file_hash
|
||||
FROM files
|
||||
WHERE LOWER(file_hash) = ?
|
||||
ORDER BY file_path
|
||||
LIMIT ?
|
||||
""",
|
||||
(normalized_hash, limit or 1000),
|
||||
)
|
||||
|
||||
for file_id, file_path_str, size_bytes, file_hash in cursor.fetchall():
|
||||
if not file_path_str:
|
||||
continue
|
||||
file_path = Path(file_path_str)
|
||||
if not file_path.exists():
|
||||
continue
|
||||
if size_bytes is None:
|
||||
try:
|
||||
size_bytes = file_path.stat().st_size
|
||||
except OSError:
|
||||
size_bytes = None
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT tag FROM tags WHERE file_id = ?
|
||||
""",
|
||||
(file_id,),
|
||||
)
|
||||
all_tags = [row[0] for row in cursor.fetchall()]
|
||||
entry = _create_entry(file_path, all_tags, size_bytes, file_hash)
|
||||
results.append(entry)
|
||||
if limit is not None and len(results) >= limit:
|
||||
return results
|
||||
return results
|
||||
|
||||
# Search for tags matching the namespace and pattern
|
||||
query_pattern = f"{namespace}:%"
|
||||
|
||||
cursor.execute("""
|
||||
SELECT DISTINCT f.id, f.file_path, f.file_size
|
||||
SELECT DISTINCT f.id, f.file_path, f.file_size, f.file_hash
|
||||
FROM files f
|
||||
JOIN tags t ON f.id = t.file_id
|
||||
WHERE LOWER(t.tag) LIKE ?
|
||||
@@ -231,7 +478,7 @@ class LocalStorageBackend(StorageBackend):
|
||||
debug(f"Found {len(rows)} potential matches in DB")
|
||||
|
||||
# Filter results by pattern match
|
||||
for file_id, file_path_str, size_bytes in rows:
|
||||
for file_id, file_path_str, size_bytes, file_hash in rows:
|
||||
if not file_path_str:
|
||||
continue
|
||||
|
||||
@@ -254,30 +501,14 @@ class LocalStorageBackend(StorageBackend):
|
||||
if fnmatch(value, pattern):
|
||||
file_path = Path(file_path_str)
|
||||
if file_path.exists():
|
||||
path_str = str(file_path)
|
||||
if size_bytes is None:
|
||||
size_bytes = file_path.stat().st_size
|
||||
|
||||
# Fetch all tags for this file
|
||||
cursor.execute("""
|
||||
SELECT tag FROM tags WHERE file_id = ?
|
||||
""", (file_id,))
|
||||
all_tags = [row[0] for row in cursor.fetchall()]
|
||||
|
||||
# Use title tag if present
|
||||
title_tag = next((t.split(':', 1)[1] for t in all_tags if t.lower().startswith('title:')), None)
|
||||
|
||||
results.append({
|
||||
"name": file_path.stem,
|
||||
"title": title_tag or file_path.stem,
|
||||
"ext": file_path.suffix.lstrip('.'),
|
||||
"path": path_str,
|
||||
"target": path_str,
|
||||
"origin": "local",
|
||||
"size": size_bytes,
|
||||
"size_bytes": size_bytes,
|
||||
"tags": all_tags,
|
||||
})
|
||||
entry = _create_entry(file_path, all_tags, size_bytes, file_hash)
|
||||
results.append(entry)
|
||||
else:
|
||||
debug(f"File missing on disk: {file_path}")
|
||||
break # Don't add same file multiple times
|
||||
@@ -309,7 +540,7 @@ class LocalStorageBackend(StorageBackend):
|
||||
where_clause = " AND ".join(conditions)
|
||||
|
||||
cursor.execute(f"""
|
||||
SELECT DISTINCT f.id, f.file_path, f.file_size
|
||||
SELECT DISTINCT f.id, f.file_path, f.file_size, f.file_hash
|
||||
FROM files f
|
||||
WHERE {where_clause}
|
||||
ORDER BY f.file_path
|
||||
@@ -344,7 +575,7 @@ class LocalStorageBackend(StorageBackend):
|
||||
word_regex = None
|
||||
|
||||
seen_files = set()
|
||||
for file_id, file_path_str, size_bytes in rows:
|
||||
for file_id, file_path_str, size_bytes, file_hash in rows:
|
||||
if not file_path_str or file_path_str in seen_files:
|
||||
continue
|
||||
|
||||
@@ -361,26 +592,12 @@ class LocalStorageBackend(StorageBackend):
|
||||
if size_bytes is None:
|
||||
size_bytes = file_path.stat().st_size
|
||||
|
||||
# Fetch tags for this file
|
||||
cursor.execute("""
|
||||
SELECT tag FROM tags WHERE file_id = ?
|
||||
""", (file_id,))
|
||||
tags = [row[0] for row in cursor.fetchall()]
|
||||
|
||||
# Use title tag if present
|
||||
title_tag = next((t.split(':', 1)[1] for t in tags if t.lower().startswith('title:')), None)
|
||||
|
||||
results.append({
|
||||
"name": file_path.stem,
|
||||
"title": title_tag or file_path.stem,
|
||||
"ext": file_path.suffix.lstrip('.'),
|
||||
"path": path_str,
|
||||
"target": path_str,
|
||||
"origin": "local",
|
||||
"size": size_bytes,
|
||||
"size_bytes": size_bytes,
|
||||
"tags": tags,
|
||||
})
|
||||
entry = _create_entry(file_path, tags, size_bytes, file_hash)
|
||||
results.append(entry)
|
||||
if limit is not None and len(results) >= limit:
|
||||
return results
|
||||
|
||||
@@ -390,7 +607,7 @@ class LocalStorageBackend(StorageBackend):
|
||||
for term in terms:
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT DISTINCT f.id, f.file_path, f.file_size
|
||||
SELECT DISTINCT f.id, f.file_path, f.file_size, f.file_hash
|
||||
FROM files f
|
||||
JOIN tags t ON f.id = t.file_id
|
||||
WHERE LOWER(t.tag) LIKE ?
|
||||
@@ -399,7 +616,7 @@ class LocalStorageBackend(StorageBackend):
|
||||
""",
|
||||
(f"title:%{term}%", fetch_limit),
|
||||
)
|
||||
for file_id, file_path_str, size_bytes in cursor.fetchall():
|
||||
for file_id, file_path_str, size_bytes, file_hash in cursor.fetchall():
|
||||
if not file_path_str:
|
||||
continue
|
||||
entry = title_hits.get(file_id)
|
||||
@@ -411,6 +628,7 @@ class LocalStorageBackend(StorageBackend):
|
||||
title_hits[file_id] = {
|
||||
"path": file_path_str,
|
||||
"size": size_bytes,
|
||||
"hash": file_hash,
|
||||
"count": 1,
|
||||
}
|
||||
|
||||
@@ -441,19 +659,8 @@ class LocalStorageBackend(StorageBackend):
|
||||
(file_id,),
|
||||
)
|
||||
tags = [row[0] for row in cursor.fetchall()]
|
||||
title_tag = next((t.split(':', 1)[1] for t in tags if t.lower().startswith('title:')), None)
|
||||
|
||||
results.append({
|
||||
"name": file_path.stem,
|
||||
"title": title_tag or file_path.stem,
|
||||
"ext": file_path.suffix.lstrip('.'),
|
||||
"path": str(file_path),
|
||||
"target": str(file_path),
|
||||
"origin": "local",
|
||||
"size": size_bytes,
|
||||
"size_bytes": size_bytes,
|
||||
"tags": tags,
|
||||
})
|
||||
entry = _create_entry(file_path, tags, size_bytes, info.get("hash"))
|
||||
results.append(entry)
|
||||
if limit is not None and len(results) >= limit:
|
||||
return results
|
||||
|
||||
@@ -465,7 +672,7 @@ class LocalStorageBackend(StorageBackend):
|
||||
query_pattern = f"%{query_lower}%"
|
||||
|
||||
cursor.execute("""
|
||||
SELECT DISTINCT f.id, f.file_path, f.file_size
|
||||
SELECT DISTINCT f.id, f.file_path, f.file_size, f.file_hash
|
||||
FROM files f
|
||||
JOIN tags t ON f.id = t.file_id
|
||||
WHERE LOWER(t.tag) LIKE ? AND LOWER(t.tag) NOT LIKE '%:%'
|
||||
@@ -474,7 +681,7 @@ class LocalStorageBackend(StorageBackend):
|
||||
""", (query_pattern, limit or 1000))
|
||||
|
||||
tag_rows = cursor.fetchall()
|
||||
for file_id, file_path_str, size_bytes in tag_rows:
|
||||
for file_id, file_path_str, size_bytes, file_hash in tag_rows:
|
||||
if not file_path_str or file_path_str in seen_files:
|
||||
continue
|
||||
seen_files.add(file_path_str)
|
||||
@@ -490,21 +697,8 @@ class LocalStorageBackend(StorageBackend):
|
||||
SELECT tag FROM tags WHERE file_id = ?
|
||||
""", (file_id,))
|
||||
tags = [row[0] for row in cursor.fetchall()]
|
||||
|
||||
# Use title tag if present
|
||||
title_tag = next((t.split(':', 1)[1] for t in tags if t.lower().startswith('title:')), None)
|
||||
|
||||
results.append({
|
||||
"name": file_path.stem,
|
||||
"title": title_tag or file_path.stem,
|
||||
"ext": file_path.suffix.lstrip('.'),
|
||||
"path": path_str,
|
||||
"target": path_str,
|
||||
"origin": "local",
|
||||
"size": size_bytes,
|
||||
"size_bytes": size_bytes,
|
||||
"tags": tags,
|
||||
})
|
||||
entry = _create_entry(file_path, tags, size_bytes, file_hash)
|
||||
results.append(entry)
|
||||
|
||||
if limit is not None and len(results) >= limit:
|
||||
return results
|
||||
@@ -512,14 +706,14 @@ class LocalStorageBackend(StorageBackend):
|
||||
else:
|
||||
# Match all - get all files from database
|
||||
cursor.execute("""
|
||||
SELECT id, file_path, file_size
|
||||
SELECT id, file_path, file_size, file_hash
|
||||
FROM files
|
||||
ORDER BY file_path
|
||||
LIMIT ?
|
||||
""", (limit or 1000,))
|
||||
|
||||
rows = cursor.fetchall()
|
||||
for file_id, file_path_str, size_bytes in rows:
|
||||
for file_id, file_path_str, size_bytes, file_hash in rows:
|
||||
if file_path_str:
|
||||
file_path = Path(file_path_str)
|
||||
if file_path.exists():
|
||||
@@ -532,21 +726,8 @@ class LocalStorageBackend(StorageBackend):
|
||||
SELECT tag FROM tags WHERE file_id = ?
|
||||
""", (file_id,))
|
||||
tags = [row[0] for row in cursor.fetchall()]
|
||||
|
||||
# Use title tag if present
|
||||
title_tag = next((t.split(':', 1)[1] for t in tags if t.lower().startswith('title:')), None)
|
||||
|
||||
results.append({
|
||||
"name": file_path.stem,
|
||||
"title": title_tag or file_path.stem,
|
||||
"ext": file_path.suffix.lstrip('.'),
|
||||
"path": path_str,
|
||||
"target": path_str,
|
||||
"origin": "local",
|
||||
"size": size_bytes,
|
||||
"size_bytes": size_bytes,
|
||||
"tags": tags,
|
||||
})
|
||||
entry = _create_entry(file_path, tags, size_bytes, file_hash)
|
||||
results.append(entry)
|
||||
|
||||
if results:
|
||||
debug(f"Returning {len(results)} results from DB")
|
||||
|
||||
@@ -22,6 +22,7 @@ from typing import Optional, Dict, Any, List, Tuple, Set
|
||||
from .utils import sha256_file
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
WORKER_LOG_MAX_ENTRIES = 99
|
||||
|
||||
# Try to import optional dependencies
|
||||
try:
|
||||
@@ -352,6 +353,29 @@ class LocalLibraryDB:
|
||||
INSERT INTO worker_log (worker_id, event_type, step, channel, message)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
""", (worker_id, event_type, step, channel, message))
|
||||
self._prune_worker_log_entries(cursor, worker_id)
|
||||
|
||||
def _prune_worker_log_entries(self, cursor, worker_id: str) -> None:
|
||||
"""Keep at most WORKER_LOG_MAX_ENTRIES rows per worker by trimming oldest ones."""
|
||||
if WORKER_LOG_MAX_ENTRIES <= 0:
|
||||
return
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT id FROM worker_log
|
||||
WHERE worker_id = ?
|
||||
ORDER BY id DESC
|
||||
LIMIT 1 OFFSET ?
|
||||
""",
|
||||
(worker_id, WORKER_LOG_MAX_ENTRIES - 1),
|
||||
)
|
||||
row = cursor.fetchone()
|
||||
if not row:
|
||||
return
|
||||
cutoff_id = row[0]
|
||||
cursor.execute(
|
||||
"DELETE FROM worker_log WHERE worker_id = ? AND id < ?",
|
||||
(worker_id, cutoff_id),
|
||||
)
|
||||
|
||||
def get_worker_events(self, worker_id: str, limit: int = 500) -> List[Dict[str, Any]]:
|
||||
"""Return chronological worker log events for timelines."""
|
||||
|
||||
@@ -7,6 +7,11 @@ import sys
|
||||
|
||||
from helper.logger import log, debug
|
||||
|
||||
try: # Optional dependency
|
||||
import musicbrainzngs # type: ignore
|
||||
except ImportError: # pragma: no cover - optional
|
||||
musicbrainzngs = None
|
||||
|
||||
|
||||
class MetadataProvider(ABC):
|
||||
"""Base class for metadata providers (music, movies, books, etc.)."""
|
||||
@@ -266,6 +271,86 @@ class GoogleBooksMetadataProvider(MetadataProvider):
|
||||
return tags
|
||||
|
||||
|
||||
class MusicBrainzMetadataProvider(MetadataProvider):
|
||||
"""Metadata provider for MusicBrainz recordings."""
|
||||
|
||||
@property
|
||||
def name(self) -> str: # type: ignore[override]
|
||||
return "musicbrainz"
|
||||
|
||||
def search(self, query: str, limit: int = 10) -> List[Dict[str, Any]]:
|
||||
if not musicbrainzngs:
|
||||
log("musicbrainzngs is not installed; skipping MusicBrainz scrape", file=sys.stderr)
|
||||
return []
|
||||
|
||||
q = (query or "").strip()
|
||||
if not q:
|
||||
return []
|
||||
|
||||
try:
|
||||
# Ensure user agent is set (required by MusicBrainz)
|
||||
musicbrainzngs.set_useragent("Medeia-Macina", "0.1")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
resp = musicbrainzngs.search_recordings(query=q, limit=limit)
|
||||
recordings = resp.get("recording-list") or resp.get("recordings") or []
|
||||
except Exception as exc:
|
||||
log(f"MusicBrainz search failed: {exc}", file=sys.stderr)
|
||||
return []
|
||||
|
||||
items: List[Dict[str, Any]] = []
|
||||
for rec in recordings[:limit]:
|
||||
if not isinstance(rec, dict):
|
||||
continue
|
||||
title = rec.get("title") or ""
|
||||
|
||||
artist = ""
|
||||
artist_credit = rec.get("artist-credit") or rec.get("artist_credit")
|
||||
if isinstance(artist_credit, list) and artist_credit:
|
||||
first = artist_credit[0]
|
||||
if isinstance(first, dict):
|
||||
artist = first.get("name") or first.get("artist", {}).get("name", "")
|
||||
elif isinstance(first, str):
|
||||
artist = first
|
||||
|
||||
album = ""
|
||||
release_list = rec.get("release-list") or rec.get("releases") or rec.get("release")
|
||||
if isinstance(release_list, list) and release_list:
|
||||
first_rel = release_list[0]
|
||||
if isinstance(first_rel, dict):
|
||||
album = first_rel.get("title", "") or ""
|
||||
release_date = first_rel.get("date") or ""
|
||||
else:
|
||||
album = str(first_rel)
|
||||
release_date = ""
|
||||
else:
|
||||
release_date = rec.get("first-release-date") or ""
|
||||
|
||||
year = str(release_date)[:4] if release_date else ""
|
||||
mbid = rec.get("id") or ""
|
||||
|
||||
items.append({
|
||||
"title": title,
|
||||
"artist": artist,
|
||||
"album": album,
|
||||
"year": year,
|
||||
"provider": self.name,
|
||||
"mbid": mbid,
|
||||
"raw": rec,
|
||||
})
|
||||
|
||||
return items
|
||||
|
||||
def to_tags(self, item: Dict[str, Any]) -> List[str]:
|
||||
tags = super().to_tags(item)
|
||||
mbid = item.get("mbid")
|
||||
if mbid:
|
||||
tags.append(f"musicbrainz:{mbid}")
|
||||
return tags
|
||||
|
||||
|
||||
# Registry ---------------------------------------------------------------
|
||||
|
||||
_METADATA_PROVIDERS: Dict[str, Type[MetadataProvider]] = {
|
||||
@@ -273,6 +358,7 @@ _METADATA_PROVIDERS: Dict[str, Type[MetadataProvider]] = {
|
||||
"openlibrary": OpenLibraryMetadataProvider,
|
||||
"googlebooks": GoogleBooksMetadataProvider,
|
||||
"google": GoogleBooksMetadataProvider,
|
||||
"musicbrainz": MusicBrainzMetadataProvider,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ import os
|
||||
import platform
|
||||
import socket
|
||||
import time as _time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, List
|
||||
|
||||
from helper.logger import debug
|
||||
@@ -19,6 +20,7 @@ from helper.logger import debug
|
||||
|
||||
# Fixed pipe name for persistent MPV connection across all Python sessions
|
||||
FIXED_IPC_PIPE_NAME = "mpv-medeia-macina"
|
||||
MPV_LUA_SCRIPT_PATH = str(Path(__file__).resolve().parent.parent / "LUA" / "main.lua")
|
||||
|
||||
|
||||
class MPVIPCError(Exception):
|
||||
@@ -45,6 +47,48 @@ def get_ipc_pipe_path() -> str:
|
||||
return f"/tmp/{FIXED_IPC_PIPE_NAME}.sock"
|
||||
|
||||
|
||||
def _unwrap_memory_target(text: Optional[str]) -> Optional[str]:
|
||||
"""Return the real target from a memory:// M3U payload if present."""
|
||||
if not isinstance(text, str) or not text.startswith("memory://"):
|
||||
return text
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#') or line.startswith('memory://'):
|
||||
continue
|
||||
return line
|
||||
return text
|
||||
|
||||
|
||||
def _normalize_target(text: Optional[str]) -> Optional[str]:
|
||||
"""Normalize playlist targets for deduping across raw/memory:// wrappers."""
|
||||
if not text:
|
||||
return None
|
||||
real = _unwrap_memory_target(text)
|
||||
if not real:
|
||||
return None
|
||||
real = real.strip()
|
||||
if not real:
|
||||
return None
|
||||
|
||||
lower = real.lower()
|
||||
# Hydrus bare hash
|
||||
if len(lower) == 64 and all(ch in "0123456789abcdef" for ch in lower):
|
||||
return lower
|
||||
|
||||
# Hydrus file URL with hash query
|
||||
try:
|
||||
parsed = __import__("urllib.parse").parse.urlparse(real)
|
||||
qs = __import__("urllib.parse").parse.parse_qs(parsed.query)
|
||||
hash_qs = qs.get("hash", [None])[0]
|
||||
if hash_qs and len(hash_qs) == 64 and all(ch in "0123456789abcdef" for ch in hash_qs.lower()):
|
||||
return hash_qs.lower()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Normalize paths/urls for comparison
|
||||
return lower.replace('\\', '\\')
|
||||
|
||||
|
||||
class MPVIPCClient:
|
||||
"""Client for communicating with mpv via IPC socket/pipe.
|
||||
|
||||
@@ -171,11 +215,18 @@ class MPVIPCClient:
|
||||
# Check if this is the response to our request
|
||||
if resp.get("request_id") == request.get("request_id"):
|
||||
return resp
|
||||
|
||||
# If it's an error without request_id (shouldn't happen for commands)
|
||||
if "error" in resp and "request_id" not in resp:
|
||||
# Might be an event or async error
|
||||
pass
|
||||
|
||||
# Handle async log messages/events for visibility
|
||||
event_type = resp.get("event")
|
||||
if event_type == "log-message":
|
||||
level = resp.get("level", "info")
|
||||
prefix = resp.get("prefix", "")
|
||||
text = resp.get("text", "").strip()
|
||||
debug(f"[MPV {level}] {prefix} {text}".strip())
|
||||
elif event_type:
|
||||
debug(f"[MPV event] {event_type}: {resp}")
|
||||
elif "error" in resp and "request_id" not in resp:
|
||||
debug(f"[MPV error] {resp}")
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
@@ -230,7 +281,13 @@ def send_to_mpv(file_url: str, title: str, headers: Optional[Dict[str, str]] = N
|
||||
return False
|
||||
|
||||
try:
|
||||
# Command 1: Set headers if provided
|
||||
# Command 0: Subscribe to log messages so MPV console errors surface in REPL
|
||||
_subscribe_log_messages(client)
|
||||
|
||||
# Command 1: Ensure our Lua helper is loaded for in-window controls
|
||||
_ensure_lua_script_loaded(client)
|
||||
|
||||
# Command 2: Set headers if provided
|
||||
if headers:
|
||||
header_str = ",".join([f"{k}: {v}" for k, v in headers.items()])
|
||||
cmd_headers = {
|
||||
@@ -238,22 +295,46 @@ def send_to_mpv(file_url: str, title: str, headers: Optional[Dict[str, str]] = N
|
||||
"request_id": 0
|
||||
}
|
||||
client.send_command(cmd_headers)
|
||||
|
||||
# Deduplicate: if target already exists in playlist, just play it
|
||||
normalized_new = _normalize_target(file_url)
|
||||
existing_index = None
|
||||
existing_title = None
|
||||
if normalized_new:
|
||||
playlist_resp = client.send_command({"command": ["get_property", "playlist"], "request_id": 98})
|
||||
if playlist_resp and playlist_resp.get("error") == "success":
|
||||
for idx, item in enumerate(playlist_resp.get("data", []) or []):
|
||||
for key in ("playlist-path", "filename"):
|
||||
norm_existing = _normalize_target(item.get(key)) if isinstance(item, dict) else None
|
||||
if norm_existing and norm_existing == normalized_new:
|
||||
existing_index = idx
|
||||
existing_title = item.get("title") if isinstance(item, dict) else None
|
||||
break
|
||||
if existing_index is not None:
|
||||
break
|
||||
|
||||
if existing_index is not None and append:
|
||||
play_cmd = {"command": ["playlist-play-index", existing_index], "request_id": 99}
|
||||
play_resp = client.send_command(play_cmd)
|
||||
if play_resp and play_resp.get("error") == "success":
|
||||
client.send_command({"command": ["set_property", "pause", False], "request_id": 100})
|
||||
safe_title = (title or existing_title or "").replace("\n", " ").replace("\r", " ").strip()
|
||||
if safe_title:
|
||||
client.send_command({"command": ["set_property", "force-media-title", safe_title], "request_id": 101})
|
||||
debug(f"Already in playlist, playing existing entry: {safe_title or file_url}")
|
||||
return True
|
||||
|
||||
# Command 2: Load file
|
||||
# Use memory:// M3U to preserve title in playlist if provided
|
||||
# This is required for YouTube URLs and proper playlist display
|
||||
if title:
|
||||
# Sanitize title for M3U (remove newlines)
|
||||
safe_title = title.replace("\n", " ").replace("\r", "")
|
||||
# M3U format: #EXTM3U\n#EXTINF:-1,Title\nURL
|
||||
m3u_content = f"#EXTM3U\n#EXTINF:-1,{safe_title}\n{file_url}"
|
||||
target = f"memory://{m3u_content}"
|
||||
else:
|
||||
target = file_url
|
||||
|
||||
# Command 2: Load file and inject title via memory:// wrapper so playlist shows friendly names immediately
|
||||
target = file_url
|
||||
load_mode = "append-play" if append else "replace"
|
||||
safe_title = (title or "").replace("\n", " ").replace("\r", " ").strip()
|
||||
target_to_send = target
|
||||
if safe_title and not str(target).startswith("memory://"):
|
||||
m3u_content = f"#EXTM3U\n#EXTINF:-1,{safe_title}\n{target}"
|
||||
target_to_send = f"memory://{m3u_content}"
|
||||
|
||||
cmd_load = {
|
||||
"command": ["loadfile", target, load_mode],
|
||||
"command": ["loadfile", target_to_send, load_mode],
|
||||
"request_id": 1
|
||||
}
|
||||
|
||||
@@ -263,14 +344,14 @@ def send_to_mpv(file_url: str, title: str, headers: Optional[Dict[str, str]] = N
|
||||
return False
|
||||
|
||||
# Command 3: Set title (metadata for display) - still useful for window title
|
||||
if title:
|
||||
if safe_title:
|
||||
cmd_title = {
|
||||
"command": ["set_property", "force-media-title", title],
|
||||
"command": ["set_property", "force-media-title", safe_title],
|
||||
"request_id": 2
|
||||
}
|
||||
client.send_command(cmd_title)
|
||||
|
||||
debug(f"Sent to existing MPV: {title}")
|
||||
debug(f"Sent to existing MPV: {safe_title or title}")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
@@ -295,3 +376,29 @@ def get_mpv_client(socket_path: Optional[str] = None) -> Optional[MPVIPCClient]:
|
||||
return client
|
||||
return None
|
||||
|
||||
|
||||
def _subscribe_log_messages(client: MPVIPCClient) -> None:
|
||||
"""Ask MPV to emit log messages over IPC so we can surface console errors."""
|
||||
try:
|
||||
client.send_command({"command": ["request_log_messages", "warn"], "request_id": 11})
|
||||
except Exception as exc:
|
||||
debug(f"Failed to subscribe to MPV logs: {exc}")
|
||||
|
||||
|
||||
def _ensure_lua_script_loaded(client: MPVIPCClient) -> None:
|
||||
"""Load the bundled MPV Lua script to enable in-window controls.
|
||||
|
||||
Safe to call repeatedly; mpv will simply reload the script if already present.
|
||||
"""
|
||||
try:
|
||||
script_path = MPV_LUA_SCRIPT_PATH
|
||||
if not script_path or not os.path.exists(script_path):
|
||||
return
|
||||
resp = client.send_command({"command": ["load-script", script_path], "request_id": 12})
|
||||
if resp and resp.get("error") == "success":
|
||||
debug(f"Loaded MPV Lua script: {script_path}")
|
||||
else:
|
||||
debug(f"MPV Lua load response: {resp}")
|
||||
except Exception as exc:
|
||||
debug(f"Failed to load MPV Lua script: {exc}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user