huge refactor of the entire codebase, with the goal of improving maintainability, readability, and extensibility. This commit includes changes to almost every file in the project, including:
This commit is contained in:
+55
-3
@@ -64,8 +64,7 @@ def _format_total_seconds(seconds: Any) -> str:
|
||||
|
||||
|
||||
class HIFI(Provider):
|
||||
|
||||
PROVIDER_NAME = "hifi"
|
||||
PLUGIN_NAME = "hifi"
|
||||
|
||||
TABLE_AUTO_STAGES = {
|
||||
"hifi.track": ["download-file"],
|
||||
@@ -2091,4 +2090,57 @@ class HIFI(Provider):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return True
|
||||
return True
|
||||
|
||||
def expand_selection(
|
||||
self,
|
||||
selected_items: List[Any],
|
||||
*,
|
||||
ctx: Any,
|
||||
stage_is_last: bool = True,
|
||||
table_type: str = "",
|
||||
**_kwargs: Any,
|
||||
) -> Optional[List[Any]]:
|
||||
_ = ctx
|
||||
if stage_is_last:
|
||||
return None
|
||||
|
||||
normalized_table = str(table_type or "").strip().lower()
|
||||
if normalized_table != "hifi.album":
|
||||
return None
|
||||
|
||||
try:
|
||||
contexts = self._extract_album_selection_context(selected_items)
|
||||
except Exception:
|
||||
return None
|
||||
if not contexts:
|
||||
return None
|
||||
|
||||
track_items: List[Any] = []
|
||||
seen_track_ids: set[int] = set()
|
||||
for album_id, album_title, artist_name in contexts:
|
||||
try:
|
||||
track_results = self._tracks_for_album(
|
||||
album_id=album_id,
|
||||
album_title=album_title,
|
||||
artist_name=artist_name,
|
||||
limit=500,
|
||||
)
|
||||
except Exception:
|
||||
track_results = []
|
||||
for track in track_results or []:
|
||||
try:
|
||||
metadata = getattr(track, "full_metadata", None)
|
||||
track_id = None
|
||||
if isinstance(metadata, dict):
|
||||
raw_id = metadata.get("trackId") or metadata.get("id")
|
||||
track_id = int(raw_id) if raw_id is not None else None
|
||||
if track_id is not None:
|
||||
if track_id in seen_track_ids:
|
||||
continue
|
||||
seen_track_ids.add(track_id)
|
||||
except Exception:
|
||||
pass
|
||||
track_items.append(track)
|
||||
|
||||
return track_items or None
|
||||
+59
-3
@@ -64,7 +64,7 @@ def _format_total_seconds(seconds: Any) -> str:
|
||||
|
||||
|
||||
class Tidal(Provider):
|
||||
PROVIDER_NAME = "tidal"
|
||||
PLUGIN_NAME = "tidal"
|
||||
|
||||
TABLE_AUTO_STAGES = {
|
||||
"tidal.track": ["download-file"],
|
||||
@@ -82,7 +82,7 @@ class Tidal(Provider):
|
||||
"tidal.com",
|
||||
"listen.tidal.com",
|
||||
)
|
||||
URL = URL_DOMAINS
|
||||
URL = URL_DOMAINS + ("tidal:",)
|
||||
"""Provider that targets the Tidal search endpoint.
|
||||
|
||||
The CLI can supply a list of fail-over URLs via ``provider.tidal.api_urls`` or
|
||||
@@ -210,6 +210,9 @@ class Tidal(Provider):
|
||||
self.api_timeout = 10.0
|
||||
self.api_clients = [TidalApiClient(base_url=url, timeout=self.api_timeout) for url in self.api_urls]
|
||||
|
||||
def resolve_playback_path(self, item: Any, **_kwargs: Any) -> Optional[str]:
|
||||
return resolve_tidal_manifest_path(item)
|
||||
|
||||
def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]:
|
||||
"""Parse inline `key:value` query arguments.
|
||||
|
||||
@@ -2398,4 +2401,57 @@ class Tidal(Provider):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return True
|
||||
return True
|
||||
|
||||
def expand_selection(
|
||||
self,
|
||||
selected_items: List[Any],
|
||||
*,
|
||||
ctx: Any,
|
||||
stage_is_last: bool = True,
|
||||
table_type: str = "",
|
||||
**_kwargs: Any,
|
||||
) -> Optional[List[Any]]:
|
||||
_ = ctx
|
||||
if stage_is_last:
|
||||
return None
|
||||
|
||||
normalized_table = str(table_type or "").strip().lower()
|
||||
if normalized_table != "tidal.album":
|
||||
return None
|
||||
|
||||
try:
|
||||
contexts = self._extract_album_selection_context(selected_items)
|
||||
except Exception:
|
||||
return None
|
||||
if not contexts:
|
||||
return None
|
||||
|
||||
track_items: List[Any] = []
|
||||
seen_track_ids: set[int] = set()
|
||||
for album_id, album_title, artist_name in contexts:
|
||||
try:
|
||||
track_results = self._tracks_for_album(
|
||||
album_id=album_id,
|
||||
album_title=album_title,
|
||||
artist_name=artist_name,
|
||||
limit=500,
|
||||
)
|
||||
except Exception:
|
||||
track_results = []
|
||||
for track in track_results or []:
|
||||
try:
|
||||
metadata = getattr(track, "full_metadata", None)
|
||||
track_id = None
|
||||
if isinstance(metadata, dict):
|
||||
raw_id = metadata.get("trackId") or metadata.get("id")
|
||||
track_id = int(raw_id) if raw_id is not None else None
|
||||
if track_id is not None:
|
||||
if track_id in seen_track_ids:
|
||||
continue
|
||||
seen_track_ids.add(track_id)
|
||||
except Exception:
|
||||
pass
|
||||
track_items.append(track)
|
||||
|
||||
return track_items or None
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Provider plugin modules.
|
||||
"""Built-in plugin modules.
|
||||
|
||||
Concrete provider implementations live in this package.
|
||||
The public entrypoint/registry is ProviderCore.registry.
|
||||
Concrete built-in plugins live in this package.
|
||||
The public registry lives in ProviderCore.registry.
|
||||
"""
|
||||
|
||||
# Register providers with the strict ResultTable adapter system
|
||||
|
||||
+72
-19
@@ -351,7 +351,7 @@ def _dispatch_alldebrid_magnet_search(
|
||||
if callable(exec_fn):
|
||||
exec_fn(
|
||||
None,
|
||||
["-provider", "alldebrid", f"ID={magnet_id}"],
|
||||
["-plugin", "alldebrid", f"ID={magnet_id}"],
|
||||
config,
|
||||
)
|
||||
except Exception:
|
||||
@@ -493,7 +493,7 @@ def download_magnet(
|
||||
|
||||
def expand_folder_item(
|
||||
item: Any,
|
||||
get_search_provider: Optional[Callable[[str, Dict[str, Any]], Any]],
|
||||
get_search_plugin: Optional[Callable[[str, Dict[str, Any]], Any]],
|
||||
config: Dict[str, Any],
|
||||
) -> Tuple[List[Any], Optional[str]]:
|
||||
table = getattr(item, "table", None) if not isinstance(item, dict) else item.get("table")
|
||||
@@ -517,15 +517,15 @@ def expand_folder_item(
|
||||
except Exception:
|
||||
magnet_id = None
|
||||
|
||||
if magnet_id is None or get_search_provider is None:
|
||||
if magnet_id is None or get_search_plugin is None:
|
||||
return [], None
|
||||
|
||||
provider = get_search_provider("alldebrid", config) if get_search_provider else None
|
||||
if provider is None:
|
||||
plugin = get_search_plugin("alldebrid", config) if get_search_plugin else None
|
||||
if plugin is None:
|
||||
return [], None
|
||||
|
||||
try:
|
||||
files = provider.search("*", limit=10_000, filters={"view": "files", "magnet_id": int(magnet_id)})
|
||||
files = plugin.search("*", limit=10_000, filters={"view": "files", "magnet_id": int(magnet_id)})
|
||||
except Exception:
|
||||
files = []
|
||||
|
||||
@@ -609,7 +609,7 @@ class AllDebrid(TableProviderMixin, Provider):
|
||||
- Drill-down: Selecting a folder row (@N) fetches and displays all files
|
||||
|
||||
SELECTION FLOW:
|
||||
1. User runs: search-file -provider alldebrid "ubuntu"
|
||||
1. User runs: search-file -plugin alldebrid "ubuntu"
|
||||
2. Results show magnet folders and (optionally) files
|
||||
3. User selects a row: @1
|
||||
4. Selection metadata routes to download-file with -url alldebrid:magnet:<id>
|
||||
@@ -619,7 +619,7 @@ class AllDebrid(TableProviderMixin, Provider):
|
||||
# Magnet URIs should be routed through this provider.
|
||||
TABLE_AUTO_STAGES = {"alldebrid": ["download-file"]}
|
||||
AUTO_STAGE_USE_SELECTION_ARGS = True
|
||||
URL = ("magnet:", "alldebrid:magnet:", "alldebrid:", "alldebrid🧲")
|
||||
URL = ("magnet:", "alldebrid:magnet:", "alldebrid:", "alldebrid🧲", "alldebrid.com")
|
||||
URL_DOMAINS = ()
|
||||
|
||||
def extract_query_arguments(self, query: str) -> Tuple[str, Dict[str, Any]]:
|
||||
@@ -949,12 +949,10 @@ class AllDebrid(TableProviderMixin, Provider):
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def download_for_pipe_result(
|
||||
cls,
|
||||
def resolve_pipe_result_download(
|
||||
self,
|
||||
result: Any,
|
||||
pipe_obj: Optional[PipeObject],
|
||||
config: Dict[str, Any],
|
||||
) -> Tuple[Optional[Path], Optional[str], Optional[Path]]:
|
||||
"""Download a remote provider result on behalf of add-file."""
|
||||
|
||||
@@ -1026,8 +1024,7 @@ class AllDebrid(TableProviderMixin, Provider):
|
||||
|
||||
download_dir = Path(tempfile.mkdtemp(prefix="add-file-alldebrid-"))
|
||||
try:
|
||||
provider = cls(config)
|
||||
downloaded_path = provider.download(search_result, download_dir)
|
||||
downloaded_path = self.download(search_result, download_dir)
|
||||
if not downloaded_path:
|
||||
shutil.rmtree(download_dir, ignore_errors=True)
|
||||
return None, None, None
|
||||
@@ -1049,6 +1046,62 @@ class AllDebrid(TableProviderMixin, Provider):
|
||||
log(f"[alldebrid] add-file download failed: {exc}", file=sys.stderr)
|
||||
shutil.rmtree(download_dir, ignore_errors=True)
|
||||
return None, None, None
|
||||
|
||||
def status_summary(self) -> Dict[str, Any]:
|
||||
try:
|
||||
api_key = _get_debrid_api_key(self.config)
|
||||
if not api_key:
|
||||
return {
|
||||
"status": "DISABLED",
|
||||
"name": self.label,
|
||||
"plugin": self.name,
|
||||
"detail": "Not configured",
|
||||
}
|
||||
client = AllDebridClient(api_key)
|
||||
base_url = str(getattr(client, "base_url", "") or "").strip()
|
||||
return {
|
||||
"status": "ENABLED",
|
||||
"name": self.label,
|
||||
"plugin": self.name,
|
||||
"detail": base_url or "Connected",
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"status": "DISABLED",
|
||||
"name": self.label,
|
||||
"plugin": self.name,
|
||||
"detail": str(exc),
|
||||
}
|
||||
|
||||
def resolve_url(self, url: str, **_kwargs: Any) -> str:
|
||||
target = str(url or "").strip()
|
||||
if not target.startswith(("http://", "https://")):
|
||||
return target
|
||||
|
||||
try:
|
||||
parsed = urlparse(target)
|
||||
host = str(parsed.netloc or "").lower()
|
||||
path = str(parsed.path or "")
|
||||
except Exception:
|
||||
return target
|
||||
|
||||
if host != "alldebrid.com" or not path.startswith("/f/"):
|
||||
return target
|
||||
|
||||
api_key = _get_debrid_api_key(self.config)
|
||||
if not api_key:
|
||||
return target
|
||||
|
||||
try:
|
||||
client = AllDebridClient(str(api_key))
|
||||
unlocked = client.unlock_link(target)
|
||||
if isinstance(unlocked, str) and unlocked.strip():
|
||||
return unlocked.strip()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return target
|
||||
|
||||
def download_items(
|
||||
self,
|
||||
result: SearchResult,
|
||||
@@ -1413,7 +1466,7 @@ class AllDebrid(TableProviderMixin, Provider):
|
||||
"provider_view": "files",
|
||||
# Selection metadata for table system
|
||||
"_selection_args": ["-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"],
|
||||
"_selection_action": ["download-file", "-provider", "alldebrid", "-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"],
|
||||
"_selection_action": ["download-file", "-plugin", "alldebrid", "-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"],
|
||||
}
|
||||
|
||||
results.append(
|
||||
@@ -1528,7 +1581,7 @@ class AllDebrid(TableProviderMixin, Provider):
|
||||
"magnet_name": magnet_name,
|
||||
# Selection metadata: allow @N expansion to drive downloads directly
|
||||
"_selection_args": ["-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"],
|
||||
"_selection_action": ["download-file", "-provider", "alldebrid", "-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"],
|
||||
"_selection_action": ["download-file", "-plugin", "alldebrid", "-url", f"{_ALD_MAGNET_PREFIX}{magnet_id}"],
|
||||
},
|
||||
)
|
||||
)
|
||||
@@ -1629,7 +1682,7 @@ class AllDebrid(TableProviderMixin, Provider):
|
||||
table.set_table_metadata({"provider": "alldebrid", "view": "files", "magnet_id": magnet_id})
|
||||
except Exception:
|
||||
pass
|
||||
table.set_source_command("download-file", ["-provider", "alldebrid"])
|
||||
table.set_source_command("download-file", ["-plugin", "alldebrid"])
|
||||
|
||||
results_payload: List[Dict[str, Any]] = []
|
||||
for r in files or []:
|
||||
@@ -1662,7 +1715,7 @@ class AllDebrid(TableProviderMixin, Provider):
|
||||
|
||||
|
||||
try:
|
||||
from SYS.result_table_adapters import register_provider
|
||||
from SYS.result_table_adapters import register_plugin
|
||||
from SYS.result_table_api import ColumnSpec, ResultModel, metadata_column, title_column
|
||||
|
||||
def _as_payload(item: Any) -> Dict[str, Any]:
|
||||
@@ -1853,7 +1906,7 @@ try:
|
||||
return ["-title", row.title or ""]
|
||||
|
||||
|
||||
register_provider(
|
||||
register_plugin(
|
||||
"alldebrid",
|
||||
_adapter,
|
||||
columns=_columns_factory,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Example provider that uses the new `ResultTable` API.
|
||||
"""Example plugin that uses the new `ResultTable` API.
|
||||
|
||||
This module demonstrates a minimal provider adapter that yields `ResultModel`
|
||||
instances, a set of `ColumnSpec` definitions, and a tiny CLI-friendly renderer
|
||||
@@ -8,7 +8,7 @@ Run this to see sample output:
|
||||
python -m Provider.example_provider
|
||||
|
||||
Example usage (piped selector):
|
||||
provider-table -provider example -sample | select -select 1 | add-file -store default
|
||||
plugin-table -plugin example -sample | select -select 1 | add-file -store default
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -105,9 +105,9 @@ def selection_fn(row: ResultModel) -> List[str]:
|
||||
return ["-title", row.title]
|
||||
|
||||
|
||||
# Register the provider with the registry so callers can discover it by name
|
||||
from SYS.result_table_adapters import register_provider
|
||||
register_provider(
|
||||
# Register the plugin with the registry so callers can discover it by name
|
||||
from SYS.result_table_adapters import register_plugin
|
||||
register_plugin(
|
||||
"example",
|
||||
adapter,
|
||||
columns=columns_factory,
|
||||
@@ -223,17 +223,17 @@ def demo() -> None:
|
||||
|
||||
|
||||
def demo_with_selection(idx: int = 0) -> None:
|
||||
"""Demonstrate how a cmdlet would use provider registration and selection args.
|
||||
"""Demonstrate how a cmdlet would use plugin registration and selection args.
|
||||
|
||||
- Fetch the registered provider by name
|
||||
- Fetch the registered plugin by name
|
||||
- Build rows via adapter
|
||||
- Render the table
|
||||
- Show the selection args for the chosen row; these are the args a cmdlet
|
||||
would append when the user picks that row.
|
||||
"""
|
||||
from SYS.result_table_adapters import get_provider
|
||||
from SYS.result_table_adapters import get_plugin
|
||||
|
||||
provider = get_provider("example")
|
||||
provider = get_plugin("example")
|
||||
rows = list(provider.adapter(SAMPLE_ITEMS))
|
||||
cols = provider.get_columns(rows)
|
||||
|
||||
|
||||
+1
-1
@@ -50,7 +50,7 @@ def _extract_key(payload: Any) -> Optional[str]:
|
||||
|
||||
class FileIO(Provider):
|
||||
"""File provider for file.io."""
|
||||
PROVIDER_NAME = "file.io"
|
||||
PLUGIN_NAME = "file.io"
|
||||
|
||||
@classmethod
|
||||
def config_schema(cls) -> List[Dict[str, Any]]:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Example provider template for use as a starter kit.
|
||||
"""Example plugin template for use as a starter kit.
|
||||
|
||||
This minimal provider demonstrates the typical hooks a provider may implement:
|
||||
This minimal plugin demonstrates the typical hooks a plugin may implement:
|
||||
- `validate()` to assert it's usable
|
||||
- `search()` to return `SearchResult` items
|
||||
- `download()` to persist a sample file (useful for local tests)
|
||||
@@ -17,13 +17,14 @@ from ProviderCore.base import Provider, SearchResult
|
||||
|
||||
|
||||
class HelloProvider(Provider):
|
||||
"""Very small example provider suitable as a template.
|
||||
"""Very small example plugin suitable as a template.
|
||||
|
||||
- Table name: `hello`
|
||||
- Usage: `search-file -provider hello "query"`
|
||||
- Usage: `search-file -plugin hello "query"`
|
||||
- Selecting a row and piping into `download-file` will call `download()`.
|
||||
"""
|
||||
|
||||
PLUGIN_NAME = "hello"
|
||||
URL = ("hello:",)
|
||||
URL_DOMAINS = ()
|
||||
|
||||
|
||||
@@ -594,9 +594,9 @@ class InternetArchive(Provider):
|
||||
"""Internet Archive provider using the `internetarchive` Python module.
|
||||
|
||||
Supports:
|
||||
- search-file -provider internetarchive <query>
|
||||
- search-file -plugin internetarchive <query>
|
||||
- download-file / provider.download() from search results
|
||||
- add-file -provider internetarchive (uploads)
|
||||
- add-file -plugin internetarchive (uploads)
|
||||
"""
|
||||
URL = ("archive.org",)
|
||||
|
||||
|
||||
+18
-3
@@ -294,7 +294,7 @@ class Matrix(TableProviderMixin, Provider):
|
||||
- MIME detection: Automatic content type classification for Matrix msgtype
|
||||
|
||||
SELECTION FLOW:
|
||||
1. User runs: search-file -provider matrix "room" (or .matrix -list-rooms)
|
||||
1. User runs: search-file -plugin matrix "room" (or .matrix -list-rooms)
|
||||
2. Results show available joined rooms
|
||||
3. User selects rooms: @1 @2 (or @1,2)
|
||||
4. Selection triggers upload of pending files to selected rooms
|
||||
@@ -368,6 +368,21 @@ class Matrix(TableProviderMixin, Provider):
|
||||
and matrix_conf.get("access_token")
|
||||
)
|
||||
|
||||
def status_summary(self) -> Dict[str, Any]:
|
||||
matrix_conf = self.config.get("provider", {}).get("matrix", {}) if isinstance(self.config, dict) else {}
|
||||
homeserver = str(matrix_conf.get("homeserver") or "").strip()
|
||||
room_id = str(matrix_conf.get("room_id") or "").strip()
|
||||
detail = homeserver
|
||||
if room_id:
|
||||
detail = (detail + (" " if detail else "")) + f"room:{room_id}"
|
||||
enabled = bool(self.validate())
|
||||
return {
|
||||
"status": "ENABLED" if enabled else "DISABLED",
|
||||
"name": self.label,
|
||||
"plugin": self.name,
|
||||
"detail": detail or ("Connected" if enabled else "Not configured"),
|
||||
}
|
||||
|
||||
def search(
|
||||
self,
|
||||
query: str,
|
||||
@@ -767,7 +782,7 @@ class Matrix(TableProviderMixin, Provider):
|
||||
|
||||
# Minimal provider registration for the new table system
|
||||
try:
|
||||
from SYS.result_table_adapters import register_provider
|
||||
from SYS.result_table_adapters import register_plugin
|
||||
from SYS.result_table_api import ResultModel, ColumnSpec, metadata_column, title_column
|
||||
|
||||
def _convert_search_result_to_model(sr: Any) -> ResultModel:
|
||||
@@ -850,7 +865,7 @@ try:
|
||||
|
||||
return ["-title", row.title or ""]
|
||||
|
||||
register_provider(
|
||||
register_plugin(
|
||||
"matrix",
|
||||
_adapter,
|
||||
columns=_columns_factory,
|
||||
|
||||
@@ -40,6 +40,42 @@ except ImportError: # pragma: no cover - optional
|
||||
yt_dlp = None
|
||||
|
||||
|
||||
def _dedup_text_values(values: List[str]) -> List[str]:
|
||||
out: List[str] = []
|
||||
seen: set[str] = set()
|
||||
for value in values or []:
|
||||
if value is None:
|
||||
continue
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
continue
|
||||
key = text.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(text)
|
||||
return out
|
||||
|
||||
|
||||
def _filter_default_scraped_tags(tags: List[str]) -> List[str]:
|
||||
blocked = {"title", "artist", "source"}
|
||||
out: List[str] = []
|
||||
seen: set[str] = set()
|
||||
for tag in tags or []:
|
||||
text = str(tag or "").strip()
|
||||
if not text:
|
||||
continue
|
||||
namespace = text.split(":", 1)[0].strip().lower() if ":" in text else ""
|
||||
if namespace in blocked:
|
||||
continue
|
||||
key = text.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(text)
|
||||
return out
|
||||
|
||||
|
||||
class MetadataProvider(ABC):
|
||||
"""Base class for metadata providers (music, movies, books, etc.)."""
|
||||
|
||||
@@ -122,6 +158,64 @@ class MetadataProvider(ABC):
|
||||
|
||||
return False
|
||||
|
||||
def default_subject_scrape_priority(self) -> int:
|
||||
"""Priority used when `get-tag -scrape` is invoked without an explicit provider."""
|
||||
|
||||
return 0
|
||||
|
||||
def url_scrape_priority(self, url: str) -> int:
|
||||
"""Priority for handling a raw URL passed to `get-tag -scrape <url>`."""
|
||||
|
||||
_ = url
|
||||
return 0
|
||||
|
||||
def resolve_subject_query(
|
||||
self,
|
||||
result: Any,
|
||||
get_field: Any,
|
||||
*,
|
||||
backend: Any = None,
|
||||
file_hash: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
"""Resolve a provider-specific query from the current subject/result."""
|
||||
|
||||
_ = backend
|
||||
_ = file_hash
|
||||
return self.extract_url_query(result, get_field)
|
||||
|
||||
def prefers_store_tag_overwrite(self) -> bool:
|
||||
"""Whether direct subject scrapes should replace the store tag set."""
|
||||
|
||||
return False
|
||||
|
||||
def filter_tags_for_selection(self, tags: List[str]) -> List[str]:
|
||||
"""Filter scraped tags before presenting a selectable metadata row."""
|
||||
|
||||
return _filter_default_scraped_tags(tags)
|
||||
|
||||
def filter_tags_for_store_apply(self, tags: List[str]) -> List[str]:
|
||||
"""Filter scraped tags before applying them to an existing store-backed item."""
|
||||
|
||||
return self.filter_tags_for_selection(tags)
|
||||
|
||||
def scrape_url_payload(self, url: str) -> Optional[Dict[str, Any]]:
|
||||
"""Return a URL scrape payload for `get-tag -scrape <url>` when supported."""
|
||||
|
||||
items = self.search(url, limit=1)
|
||||
if not items:
|
||||
return None
|
||||
item = items[0] if isinstance(items[0], dict) else {}
|
||||
try:
|
||||
tags = [str(t) for t in self.to_tags(item) if t is not None]
|
||||
except Exception:
|
||||
tags = []
|
||||
return {
|
||||
"title": item.get("title"),
|
||||
"tag": _dedup_text_values(tags),
|
||||
"formats": [],
|
||||
"playlist_items": [],
|
||||
}
|
||||
|
||||
|
||||
class ITunesProvider(MetadataProvider):
|
||||
"""Metadata provider using the iTunes Search API."""
|
||||
@@ -1015,6 +1109,226 @@ class YtdlpMetadataProvider(MetadataProvider):
|
||||
def emits_direct_tags(self) -> bool:
|
||||
return True
|
||||
|
||||
def default_subject_scrape_priority(self) -> int:
|
||||
return 100
|
||||
|
||||
def url_scrape_priority(self, url: str) -> int:
|
||||
text = str(url or "").strip()
|
||||
if not text.startswith(("http://", "https://")):
|
||||
return 0
|
||||
return 100
|
||||
|
||||
def prefers_store_tag_overwrite(self) -> bool:
|
||||
return True
|
||||
|
||||
def filter_tags_for_store_apply(self, tags: List[str]) -> List[str]:
|
||||
return _dedup_text_values(tags)
|
||||
|
||||
def _resolve_candidate_urls_for_subject(
|
||||
self,
|
||||
result: Any,
|
||||
get_field: Any,
|
||||
*,
|
||||
backend: Any = None,
|
||||
file_hash: Optional[str] = None,
|
||||
) -> List[str]:
|
||||
try:
|
||||
from SYS.metadata import normalize_urls
|
||||
except Exception:
|
||||
normalize_urls = None # type: ignore[assignment]
|
||||
|
||||
urls: List[str] = []
|
||||
|
||||
if backend is not None and file_hash:
|
||||
try:
|
||||
backend_urls = backend.get_url(file_hash, config=self.config)
|
||||
if backend_urls:
|
||||
if normalize_urls:
|
||||
urls.extend(normalize_urls(backend_urls))
|
||||
else:
|
||||
urls.extend(
|
||||
[str(u).strip() for u in backend_urls if isinstance(u, str) and str(u).strip()]
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
meta = backend.get_metadata(file_hash, config=self.config)
|
||||
if isinstance(meta, dict) and meta.get("url"):
|
||||
raw = meta.get("url")
|
||||
if normalize_urls:
|
||||
urls.extend(normalize_urls(raw))
|
||||
elif isinstance(raw, list):
|
||||
urls.extend([str(u).strip() for u in raw if isinstance(u, str) and str(u).strip()])
|
||||
elif isinstance(raw, str) and raw.strip():
|
||||
urls.append(raw.strip())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for key in ("url", "webpage_url", "source_url", "target"):
|
||||
val = get_field(result, key, None)
|
||||
if not val:
|
||||
continue
|
||||
if normalize_urls:
|
||||
urls.extend(normalize_urls(val))
|
||||
continue
|
||||
if isinstance(val, str) and val.strip():
|
||||
urls.append(val.strip())
|
||||
elif isinstance(val, list):
|
||||
urls.extend([str(u).strip() for u in val if isinstance(u, str) and str(u).strip()])
|
||||
|
||||
meta_field = get_field(result, "metadata", None)
|
||||
if isinstance(meta_field, dict) and meta_field.get("url"):
|
||||
raw = meta_field.get("url")
|
||||
if normalize_urls:
|
||||
urls.extend(normalize_urls(raw))
|
||||
elif isinstance(raw, list):
|
||||
urls.extend([str(u).strip() for u in raw if isinstance(u, str) and str(u).strip()])
|
||||
elif isinstance(raw, str) and raw.strip():
|
||||
urls.append(raw.strip())
|
||||
|
||||
return _dedup_text_values(urls)
|
||||
|
||||
def _pick_supported_subject_url(self, urls: List[str]) -> Optional[str]:
|
||||
if not urls:
|
||||
return None
|
||||
|
||||
def _is_hydrus_file_url(u: str) -> bool:
|
||||
text = str(u or "").strip().lower()
|
||||
return bool(text and "/get_files/file" in text and "hash=" in text)
|
||||
|
||||
candidates = []
|
||||
for url in urls:
|
||||
text = str(url or "").strip()
|
||||
if not text.startswith(("http://", "https://")):
|
||||
continue
|
||||
if _is_hydrus_file_url(text):
|
||||
continue
|
||||
candidates.append(text)
|
||||
if not candidates:
|
||||
return None
|
||||
|
||||
try:
|
||||
from tool.ytdlp import is_url_supported_by_ytdlp
|
||||
|
||||
for text in candidates:
|
||||
try:
|
||||
if is_url_supported_by_ytdlp(text):
|
||||
return text
|
||||
except Exception:
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
def resolve_subject_query(
|
||||
self,
|
||||
result: Any,
|
||||
get_field: Any,
|
||||
*,
|
||||
backend: Any = None,
|
||||
file_hash: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
candidate_urls = self._resolve_candidate_urls_for_subject(
|
||||
result,
|
||||
get_field,
|
||||
backend=backend,
|
||||
file_hash=file_hash,
|
||||
)
|
||||
return self._pick_supported_subject_url(candidate_urls)
|
||||
|
||||
@staticmethod
|
||||
def _extract_url_formats(formats: Any) -> List[tuple[str, str]]:
|
||||
if not isinstance(formats, list):
|
||||
return []
|
||||
|
||||
video_formats: Dict[str, Dict[str, Any]] = {}
|
||||
audio_formats: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
for fmt in formats:
|
||||
if not isinstance(fmt, dict):
|
||||
continue
|
||||
vcodec = fmt.get("vcodec", "none")
|
||||
acodec = fmt.get("acodec", "none")
|
||||
height = fmt.get("height")
|
||||
ext = fmt.get("ext", "unknown")
|
||||
format_id = fmt.get("format_id", "")
|
||||
tbr = fmt.get("tbr", 0)
|
||||
abr = fmt.get("abr", 0)
|
||||
|
||||
if vcodec and vcodec != "none" and height:
|
||||
if int(height) < 480:
|
||||
continue
|
||||
res_key = f"{int(height)}p"
|
||||
if res_key not in video_formats or tbr > video_formats[res_key].get("tbr", 0):
|
||||
video_formats[res_key] = {
|
||||
"label": f"{int(height)}p ({ext})",
|
||||
"format_id": str(format_id),
|
||||
"tbr": tbr,
|
||||
}
|
||||
elif acodec and acodec != "none" and (not vcodec or vcodec == "none"):
|
||||
audio_key = f"audio_{abr}"
|
||||
if audio_key not in audio_formats or abr > audio_formats[audio_key].get("abr", 0):
|
||||
audio_formats[audio_key] = {
|
||||
"label": f"audio ({ext})",
|
||||
"format_id": str(format_id),
|
||||
"abr": abr,
|
||||
}
|
||||
|
||||
result: List[tuple[str, str]] = []
|
||||
for res in sorted(video_formats.keys(), key=lambda value: int(value.replace("p", "")), reverse=True):
|
||||
fmt = video_formats[res]
|
||||
result.append((str(fmt.get("label") or res), str(fmt.get("format_id") or "")))
|
||||
if audio_formats:
|
||||
best_audio_key = max(audio_formats.keys(), key=lambda key: float(audio_formats[key].get("abr", 0) or 0))
|
||||
fmt = audio_formats[best_audio_key]
|
||||
result.append((str(fmt.get("label") or "audio"), str(fmt.get("format_id") or "")))
|
||||
return [entry for entry in result if entry[1]]
|
||||
|
||||
@staticmethod
|
||||
def _build_playlist_items(raw: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
entries = raw.get("entries")
|
||||
if not isinstance(entries, list):
|
||||
return []
|
||||
|
||||
playlist_items: List[Dict[str, Any]] = []
|
||||
for idx, entry in enumerate(entries, 1):
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
playlist_items.append(
|
||||
{
|
||||
"index": idx,
|
||||
"id": entry.get("id", f"track_{idx}"),
|
||||
"title": entry.get("title", entry.get("id", f"Track {idx}")),
|
||||
"duration": entry.get("duration", 0),
|
||||
"url": entry.get("url") or entry.get("webpage_url", ""),
|
||||
}
|
||||
)
|
||||
return playlist_items
|
||||
|
||||
def scrape_url_payload(self, url: str) -> Optional[Dict[str, Any]]:
|
||||
info = self._extract_info(url)
|
||||
if not isinstance(info, dict):
|
||||
return None
|
||||
|
||||
item = {
|
||||
"title": info.get("title") or "",
|
||||
"artist": str(info.get("artist") or info.get("uploader") or info.get("channel") or ""),
|
||||
"album": str(info.get("album") or info.get("playlist_title") or ""),
|
||||
"year": str((str(info.get("release_date") or "") or str(info.get("upload_date") or ""))[:4]),
|
||||
"provider": self.name,
|
||||
"url": str(url or "").strip(),
|
||||
"raw": info,
|
||||
}
|
||||
tags = _dedup_text_values([str(tag) for tag in self.to_tags(item) if tag is not None])
|
||||
return {
|
||||
"title": item.get("title") or None,
|
||||
"tag": tags,
|
||||
"formats": self._extract_url_formats(info.get("formats", [])),
|
||||
"playlist_items": self._build_playlist_items(info),
|
||||
}
|
||||
|
||||
|
||||
def _coerce_archive_field_list(value: Any) -> List[str]:
|
||||
"""Coerce an Archive.org metadata field to a list of strings."""
|
||||
@@ -1420,7 +1734,7 @@ try:
|
||||
from typing import Iterable
|
||||
|
||||
from SYS.result_table_api import ColumnSpec, ResultModel, metadata_column, title_column
|
||||
from SYS.result_table_adapters import register_provider
|
||||
from SYS.result_table_adapters import register_plugin
|
||||
|
||||
def _ensure_search_result(item: Any) -> SearchResult:
|
||||
if isinstance(item, SearchResult):
|
||||
@@ -1526,7 +1840,7 @@ try:
|
||||
return ["-url", url]
|
||||
return ["-title", row.title or ""]
|
||||
|
||||
register_provider(
|
||||
register_plugin(
|
||||
"openlibrary",
|
||||
_adapter,
|
||||
columns=_columns_factory,
|
||||
@@ -1671,3 +1985,42 @@ def get_metadata_provider(name: str,
|
||||
except Exception as exc:
|
||||
log(f"Provider init failed for '{name}': {exc}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
|
||||
def get_default_subject_scrape_provider(
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
) -> Optional[MetadataProvider]:
|
||||
best_provider: Optional[MetadataProvider] = None
|
||||
best_priority = 0
|
||||
for cls in _METADATA_PROVIDERS.values():
|
||||
try:
|
||||
provider = cls(config)
|
||||
priority = int(provider.default_subject_scrape_priority())
|
||||
except Exception:
|
||||
continue
|
||||
if priority > best_priority:
|
||||
best_priority = priority
|
||||
best_provider = provider
|
||||
return best_provider
|
||||
|
||||
|
||||
def get_metadata_provider_for_url(
|
||||
url: str,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
) -> Optional[MetadataProvider]:
|
||||
text = str(url or "").strip()
|
||||
if not text:
|
||||
return None
|
||||
|
||||
best_provider: Optional[MetadataProvider] = None
|
||||
best_priority = 0
|
||||
for cls in _METADATA_PROVIDERS.values():
|
||||
try:
|
||||
provider = cls(config)
|
||||
priority = int(provider.url_scrape_priority(text))
|
||||
except Exception:
|
||||
continue
|
||||
if priority > best_priority:
|
||||
best_priority = priority
|
||||
best_provider = provider
|
||||
return best_provider
|
||||
|
||||
@@ -216,7 +216,7 @@ def _suppress_aioslsk_noise() -> Any:
|
||||
class Soulseek(Provider):
|
||||
|
||||
TABLE_AUTO_STAGES = {
|
||||
"soulseek": ["download-file", "-provider", "soulseek"],
|
||||
"soulseek": ["download-file", "-plugin", "soulseek"],
|
||||
}
|
||||
"""Search provider for Soulseek P2P network."""
|
||||
|
||||
@@ -623,7 +623,7 @@ class Soulseek(Provider):
|
||||
media_kind="audio",
|
||||
size_bytes=item["size"],
|
||||
columns=columns,
|
||||
selection_action=["download-file", "-provider", "soulseek"],
|
||||
selection_action=["download-file", "-plugin", "soulseek"],
|
||||
full_metadata={
|
||||
"username": item["username"],
|
||||
"filename": item["filename"],
|
||||
|
||||
+10
-10
@@ -37,15 +37,15 @@ class Vimm(TableProviderMixin, Provider):
|
||||
2) Each row carries explicit selection args: `['-url', '<full-url>']`.
|
||||
Using an explicit `-url` flag avoids ambiguity during argument
|
||||
parsing (some cmdlets accept positional URLs, others accept flags).
|
||||
3) The CLI's expansion logic places selection args *before* provider
|
||||
source args (e.g., `-provider vimm`) so the first positional token is
|
||||
the intended URL (not an unknown flag like `-provider`).
|
||||
3) The CLI's expansion logic places selection args *before* plugin
|
||||
source args (e.g., `-plugin vimm`) so the first positional token is
|
||||
the intended URL (not an unknown flag like `-plugin`).
|
||||
|
||||
- Why this approach? Argument parsing treats the *first* unrecognized token
|
||||
as a positional value (commonly interpreted as a URL). If a provider
|
||||
injects hints like `-provider vimm` *before* a bare URL, the parser can
|
||||
misinterpret `-provider` as the URL, causing confusing attempts to
|
||||
download `-provider`. By using `-url` and ensuring the URL appears first
|
||||
as a positional value (commonly interpreted as a URL). If a plugin
|
||||
injects hints like `-plugin vimm` *before* a bare URL, the parser can
|
||||
misinterpret `-plugin` as the URL, causing confusing attempts to
|
||||
download `-plugin`. By using `-url` and ensuring the URL appears first
|
||||
we avoid that class of bugs and make `@N` -> `download-file`/`add-file`
|
||||
flows reliable.
|
||||
|
||||
@@ -56,7 +56,7 @@ class Vimm(TableProviderMixin, Provider):
|
||||
URL_DOMAINS = ("vimm.net",)
|
||||
|
||||
def get_source_command(self, args_list: List[str]) -> Tuple[str, List[str]]:
|
||||
return "search-file", ["-provider", self.name]
|
||||
return "search-file", ["-plugin", self.name]
|
||||
|
||||
REGION_CHOICES = [
|
||||
{"value": "1", "text": "Argentina"},
|
||||
@@ -807,7 +807,7 @@ class Vimm(TableProviderMixin, Provider):
|
||||
|
||||
# Minimal provider registration
|
||||
try:
|
||||
from SYS.result_table_adapters import register_provider
|
||||
from SYS.result_table_adapters import register_plugin
|
||||
from SYS.result_table_api import ResultModel, title_column, metadata_column
|
||||
|
||||
def _convert_search_result_to_model(sr):
|
||||
@@ -857,7 +857,7 @@ try:
|
||||
return ["-title", row.title or ""]
|
||||
|
||||
|
||||
register_provider(
|
||||
register_plugin(
|
||||
"vimm",
|
||||
_adapter,
|
||||
columns=_columns_factory,
|
||||
|
||||
+3
-3
@@ -21,7 +21,7 @@ class YouTube(TableProviderMixin, Provider):
|
||||
- _selection_args: For @N expansion control and download-file routing
|
||||
|
||||
SELECTION FLOW:
|
||||
1. User runs: search-file -provider youtube "linux tutorial"
|
||||
1. User runs: search-file -plugin youtube "linux tutorial"
|
||||
2. Results show video rows with uploader, duration, views
|
||||
3. User selects a video: @1
|
||||
4. Selection metadata routes to download-file with the YouTube URL
|
||||
@@ -121,7 +121,7 @@ class YouTube(TableProviderMixin, Provider):
|
||||
|
||||
# Minimal provider registration for the new table system
|
||||
try:
|
||||
from SYS.result_table_adapters import register_provider
|
||||
from SYS.result_table_adapters import register_plugin
|
||||
from SYS.result_table_api import ResultModel, ColumnSpec, metadata_column, title_column
|
||||
|
||||
def _convert_search_result_to_model(sr: Any) -> ResultModel:
|
||||
@@ -206,7 +206,7 @@ try:
|
||||
|
||||
return ["-title", row.title or ""]
|
||||
|
||||
register_provider(
|
||||
register_plugin(
|
||||
"youtube",
|
||||
_adapter,
|
||||
columns=_columns_factory,
|
||||
|
||||
+1205
-151
File diff suppressed because it is too large
Load Diff
@@ -11,8 +11,8 @@ from SYS.logger import log
|
||||
class ZeroXZero(Provider):
|
||||
"""File provider for 0x0.st."""
|
||||
|
||||
NAME = "0x0"
|
||||
PROVIDER_ALIASES = ("zeroxzero",)
|
||||
PLUGIN_NAME = "0x0"
|
||||
PLUGIN_ALIASES = ("zeroxzero",)
|
||||
|
||||
def upload(self, file_path: str, **kwargs: Any) -> str:
|
||||
from API.HTTP import HTTPClient
|
||||
|
||||
Reference in New Issue
Block a user