goyimnose/Medios-Macina
1
0
Fork 0
Code Issues Pull Requests Packages Releases 1 Wiki Activity

updated panel display

Browse Source
This commit is contained in:
Nose
2026-04-16 17:18:50 -07:00
parent 97e310be70
commit 343a7b37a0
14 changed files with 711 additions and 264 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Store/HydrusNetwork.py
+111 -39
View File
@@ -8,7 +8,7 @@ from collections import deque
from pathlib import Path
from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple
from urllib.parse import quote
from urllib.parse import quote, parse_qsl, urlencode, urlsplit, urlunsplit
import httpx
from API.httpx_shared import get_shared_httpx_client
@@ -417,8 +417,6 @@ class HydrusNetwork(Store):
if not file_hash:
file_hash = sha256_file(file_path)
debug(f"{self._log_prefix()} file hash: {file_hash}")
# Use persistent client with session key
client = self._client
if client is None:
@@ -564,7 +562,6 @@ class HydrusNetwork(Store):
raise Exception(f"Hydrus response missing file hash: {response}")
file_hash = hydrus_hash
debug(f"{self._log_prefix()} hash: {file_hash}")
# Add tags if provided (both for new and existing files)
if tag_list:
@@ -575,13 +572,7 @@ class HydrusNetwork(Store):
service_name = "my tags"
try:
debug(
f"{self._log_prefix()} Adding {len(tag_list)} tag(s): {tag_list}"
)
client.add_tag(file_hash, tag_list, service_name)
debug(
f"{self._log_prefix()} Tags added via '{service_name}'"
)
except Exception as exc:
log(
f"{self._log_prefix()} ⚠️ Failed to add tags: {exc}",
@@ -590,14 +581,10 @@ class HydrusNetwork(Store):
# Associate url if provided (both for new and existing files)
if url:
debug(
f"{self._log_prefix()} Associating {len(url)} URL(s) with file"
)
for url in url:
if url:
try:
client.associate_url(file_hash, str(url))
debug(f"{self._log_prefix()} Associated URL: {url}")
except Exception as exc:
log(
f"{self._log_prefix()} ⚠️ Failed to associate URL {url}: {exc}",
@@ -634,7 +621,6 @@ class HydrusNetwork(Store):
raise Exception("Hydrus client unavailable")
prefix = self._log_prefix()
debug(f"{prefix} Searching for: {query}")
def _extract_urls(meta_obj: Any) -> list[str]:
if not isinstance(meta_obj, dict):
@@ -720,6 +706,70 @@ class HydrusNetwork(Store):
) -> list[dict[str, Any]]:
"""Best-effort URL search by scanning Hydrus metadata with include_file_url=True."""
try:
from API.HydrusNetwork import _generate_hydrus_url_variants
except Exception:
_generate_hydrus_url_variants = None # type: ignore[assignment]
def _normalize_url_match_token(value: str | None) -> str:
token = str(value or "").strip()
if not token:
return ""
token = token.split("#", 1)[0]
try:
parsed_url = urlsplit(token)
except Exception:
return token.lower()
if parsed_url.scheme and parsed_url.scheme.lower() not in {"http", "https"}:
return token.lower()
netloc = str(parsed_url.netloc or "").strip().lower()
if netloc.startswith("www."):
netloc = netloc[4:]
try:
query_pairs = parse_qsl(parsed_url.query, keep_blank_values=True)
except Exception:
query_pairs = []
filtered_pairs = []
for key, val in query_pairs:
key_norm = str(key or "").lower()
if key_norm in {"t", "start", "time_continue", "timestamp", "time", "begin"}:
continue
if key_norm.startswith("utm_"):
continue
filtered_pairs.append((key, val))
normalized_query = urlencode(filtered_pairs, doseq=True) if filtered_pairs else ""
normalized = urlunsplit(("", netloc, parsed_url.path or "", normalized_query, ""))
return str(normalized or token).lstrip("/").lower()
def _append_url_needles(output: list[str], candidate: str | None) -> None:
raw_candidate = str(candidate or "").strip()
if not raw_candidate:
return
expanded_candidates = [raw_candidate]
if callable(_generate_hydrus_url_variants):
try:
expanded_candidates.extend(_generate_hydrus_url_variants(raw_candidate) or [])
except Exception:
pass
for expanded in expanded_candidates:
expanded_text = str(expanded or "").strip()
if not expanded_text:
continue
lowered = expanded_text.lower()
if lowered and lowered not in output:
output.append(lowered)
normalized = _normalize_url_match_token(expanded_text)
if normalized and normalized not in output:
output.append(normalized)
candidate_file_ids: list[int] = []
candidate_hashes: list[str] = []
seen_file_ids: set[int] = set()
@@ -775,13 +825,9 @@ class HydrusNetwork(Store):
needle_list: list[str] = []
if isinstance(needles, (list, tuple, set)):
for item in needles:
text = str(item or "").strip().lower()
if text and text not in needle_list:
needle_list.append(text)
_append_url_needles(needle_list, str(item or ""))
if not needle_list:
needle = (url_value or "").strip().lower()
if needle:
needle_list = [needle]
_append_url_needles(needle_list, url_value)
chunk_size = 200
out: list[dict[str, Any]] = []
if scan_limit is None:
@@ -855,7 +901,14 @@ class HydrusNetwork(Store):
continue
if not needle_list:
continue
if any(any(n in u.lower() for n in needle_list) for u in urls):
normalized_urls = [_normalize_url_match_token(u) for u in urls]
if any(
any(
n in str(u or "").lower() or (normalized_urls[idx] and n in normalized_urls[idx])
for idx, u in enumerate(urls)
)
for n in needle_list
):
out.append(meta)
continue
@@ -1001,7 +1054,8 @@ class HydrusNetwork(Store):
return ids_out, hashes_out
query_lower = query.lower().strip()
raw_query = str(query or "").strip()
query_lower = raw_query.lower().strip()
# Support `ext:<value>` anywhere in the query. We filter results by the
# Hydrus metadata extension field.
@@ -1055,10 +1109,10 @@ class HydrusNetwork(Store):
hashes: list[str] = []
file_ids: list[int] = []
if ":" in query_lower and not query_lower.startswith(":"):
namespace, pattern = query_lower.split(":", 1)
namespace = namespace.strip().lower()
pattern = pattern.strip()
if ":" in raw_query and not raw_query.startswith(":"):
namespace_raw, pattern_raw = raw_query.split(":", 1)
namespace = namespace_raw.strip().lower()
pattern = pattern_raw.strip()
if namespace == "url":
try:
fetch_limit_raw = int(limit) if limit else 100
@@ -1066,7 +1120,7 @@ class HydrusNetwork(Store):
fetch_limit_raw = 100
if url_only:
metadata_list = _search_url_query_metadata(
query_lower,
f"url:{pattern}",
fetch_limit_raw,
minimal=minimal,
)
@@ -1092,12 +1146,13 @@ class HydrusNetwork(Store):
token = str(value or "").strip().lower()
if not token:
return ""
return token.replace("*", "").replace("?", "")
return token.replace("*", "")
# Fast-path: exact URL via /add_urls/get_url_files when a full URL is provided.
exact_url_attempted = False
try:
if pattern.startswith("http://") or pattern.startswith("https://"):
has_wildcards = ("*" in pattern)
if (pattern.startswith("http://") or pattern.startswith("https://")) and not has_wildcards:
exact_url_attempted = True
metadata_list = self.lookup_url_metadata(pattern, minimal=minimal)
except Exception:
@@ -1123,7 +1178,7 @@ class HydrusNetwork(Store):
minimal=minimal,
)
elif namespace == "system":
normalized_system_predicate = pattern.strip()
normalized_system_predicate = pattern.strip().lower()
if normalized_system_predicate == "has url":
try:
fetch_limit = int(limit) if limit else 100
@@ -1173,7 +1228,6 @@ class HydrusNetwork(Store):
if freeform_union_search:
if not title_predicates and not freeform_predicates:
debug(f"{prefix} 0 result(s)")
return []
payloads: list[Any] = []
@@ -1223,7 +1277,6 @@ class HydrusNetwork(Store):
hashes = []
else:
if not tags:
debug(f"{prefix} 0 result(s)")
return []
search_result = client.search_files(
@@ -1239,7 +1292,6 @@ class HydrusNetwork(Store):
if ext_only and ext_filter:
results = []
if not file_ids and not hashes:
debug(f"{prefix} 0 result(s)")
return []
# Prefer file_ids if available.
@@ -1301,14 +1353,11 @@ class HydrusNetwork(Store):
"ext": _resolve_ext_from_meta(meta, mime_type),
}
)
debug(f"{prefix} {len(results)} result(s)")
return results[:limit]
# If we only got hashes, fall back to the normal flow below.
if not file_ids and not hashes:
debug(f"{prefix} 0 result(s)")
return []
file_ids, hashes = _cap_metadata_candidates(
@@ -1457,8 +1506,6 @@ class HydrusNetwork(Store):
"ext": ext,
}
)
debug(f"{prefix} {len(results)} result(s)")
if ext_filter:
wanted = ext_filter
filtered: list[dict[str, Any]] = []
@@ -2110,6 +2157,31 @@ class HydrusNetwork(Store):
seen_ids.add(file_id)
file_ids.append(file_id)
statuses = response.get("url_file_statuses")
if isinstance(statuses, list):
for entry in statuses:
if not isinstance(entry, dict):
continue
status_hash = entry.get("hash") or entry.get("file_hash")
try:
normalized_hash = str(status_hash or "").strip().lower()
except Exception:
normalized_hash = ""
if normalized_hash and normalized_hash not in seen_hashes:
seen_hashes.add(normalized_hash)
hashes.append(normalized_hash)
status_id = entry.get("file_id") or entry.get("fileid")
try:
file_id = int(status_id) if status_id is not None else None
except (TypeError, ValueError):
file_id = None
if file_id is None or file_id in seen_ids:
continue
seen_ids.add(file_id)
file_ids.append(file_id)
for key in ("normalized_url", "redirect_url", "url"):
value = response.get(key)
if isinstance(value, str):