update refactoring and add new features
This commit is contained in:
@@ -0,0 +1,686 @@
|
||||
"""PipeObject utilities for chainable cmdlet and multi-action pipelines.
|
||||
|
||||
Contains helpers for creating, normalizing, mutating, and extracting data from
|
||||
PipeObject instances and result dicts. Also includes the canonical
|
||||
``merge_sequences`` dedup utility used across cmdlets.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable as IterableABC
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
from SYS import models
|
||||
from SYS.item_accessors import get_field as _item_accessor_get_field
|
||||
from SYS.payload_builders import build_file_result_payload
|
||||
from ._store_utils import resolve_hash_for_cmdlet
|
||||
|
||||
__all__ = [
|
||||
"get_field",
|
||||
"merge_sequences",
|
||||
"normalize_result_input",
|
||||
"normalize_result_items",
|
||||
"value_has_content",
|
||||
"filter_results_by_temp",
|
||||
"create_pipe_object_result",
|
||||
"mark_as_temp",
|
||||
"set_parent_hash",
|
||||
"get_pipe_object_path",
|
||||
"get_pipe_object_hash",
|
||||
"coerce_to_pipe_object",
|
||||
"resolve_item_store_hash",
|
||||
"propagate_metadata",
|
||||
"extract_relationships",
|
||||
"extract_duration",
|
||||
"pipeline_item_local_path",
|
||||
]
|
||||
|
||||
|
||||
def get_field(obj: Any, field: str, default: Optional[Any] = None) -> Any:
|
||||
"""Extract a field from either a dict or object with fallback default.
|
||||
|
||||
Handles both dict.get(field) and getattr(obj, field) access patterns.
|
||||
Also handles lists by accessing the first element.
|
||||
For PipeObjects, checks the extra field as well.
|
||||
Used throughout cmdlet to uniformly access fields from mixed types.
|
||||
|
||||
Args:
|
||||
obj: Dict, object, or list to extract from
|
||||
field: Field name to retrieve
|
||||
default: Value to return if field not found (default: None)
|
||||
|
||||
Returns:
|
||||
Field value if found, otherwise the default value
|
||||
|
||||
Examples:
|
||||
get_field(result, "hash") # From dict or object
|
||||
get_field(result, "table", "unknown") # With default
|
||||
"""
|
||||
return _item_accessor_get_field(obj, field, default)
|
||||
|
||||
|
||||
def merge_sequences(*sources: Optional[Iterable[Any]],
|
||||
case_sensitive: bool = True) -> list[str]:
|
||||
"""Merge iterable sources while preserving order and removing duplicates."""
|
||||
seen: set[str] = set()
|
||||
merged: list[str] = []
|
||||
for source in sources:
|
||||
if not source:
|
||||
continue
|
||||
if isinstance(source, str) or not isinstance(source, IterableABC):
|
||||
iterable = [source]
|
||||
else:
|
||||
iterable = source
|
||||
for value in iterable:
|
||||
if value is None:
|
||||
continue
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
continue
|
||||
key = text if case_sensitive else text.lower()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
merged.append(text)
|
||||
return merged
|
||||
|
||||
|
||||
def normalize_result_input(result: Any) -> List[Dict[str, Any]]:
|
||||
"""Normalize input result to a list of dicts.
|
||||
|
||||
Handles:
|
||||
- None -> []
|
||||
- Dict -> [dict]
|
||||
- List of dicts -> list as-is
|
||||
- PipeObject -> [dict]
|
||||
- List of PipeObjects -> list of dicts
|
||||
|
||||
Args:
|
||||
result: Result from piped input
|
||||
|
||||
Returns:
|
||||
List of result dicts (may be empty)
|
||||
"""
|
||||
if result is None:
|
||||
return []
|
||||
|
||||
if isinstance(result, dict):
|
||||
return [result]
|
||||
|
||||
if isinstance(result, list):
|
||||
output = []
|
||||
for item in result:
|
||||
if isinstance(item, dict):
|
||||
output.append(item)
|
||||
elif hasattr(item, "to_dict"):
|
||||
output.append(item.to_dict())
|
||||
else:
|
||||
output.append(item)
|
||||
return output
|
||||
|
||||
if hasattr(result, "to_dict"):
|
||||
return [result.to_dict()]
|
||||
|
||||
if isinstance(result, dict):
|
||||
return [result]
|
||||
|
||||
return []
|
||||
|
||||
|
||||
def normalize_result_items(
|
||||
result: Any,
|
||||
*,
|
||||
include_falsey_single: bool = False,
|
||||
) -> List[Any]:
|
||||
"""Normalize piped input to a raw item list without converting item types."""
|
||||
if isinstance(result, list):
|
||||
return list(result)
|
||||
if result is None:
|
||||
return []
|
||||
if include_falsey_single or result:
|
||||
return [result]
|
||||
return []
|
||||
|
||||
|
||||
def value_has_content(value: Any) -> bool:
|
||||
"""Return True when a value should be treated as present for payload building."""
|
||||
if value is None:
|
||||
return False
|
||||
if isinstance(value, str):
|
||||
return bool(value.strip())
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
return len(value) > 0
|
||||
return True
|
||||
|
||||
|
||||
def filter_results_by_temp(results: List[Any], include_temp: bool = False) -> List[Any]:
|
||||
"""Filter results by temporary status.
|
||||
|
||||
Args:
|
||||
results: List of result dicts or PipeObjects
|
||||
include_temp: If True, keep temp files; if False, exclude them
|
||||
|
||||
Returns:
|
||||
Filtered list
|
||||
"""
|
||||
if include_temp:
|
||||
return results
|
||||
|
||||
filtered = []
|
||||
for result in results:
|
||||
is_temp = False
|
||||
|
||||
if hasattr(result, "is_temp"):
|
||||
is_temp = result.is_temp
|
||||
elif isinstance(result, dict):
|
||||
is_temp = result.get("is_temp", False)
|
||||
|
||||
if not is_temp:
|
||||
filtered.append(result)
|
||||
|
||||
return filtered
|
||||
|
||||
|
||||
def create_pipe_object_result(
|
||||
source: str,
|
||||
identifier: str,
|
||||
file_path: str,
|
||||
cmdlet_name: str,
|
||||
title: Optional[str] = None,
|
||||
hash_value: Optional[str] = None,
|
||||
is_temp: bool = False,
|
||||
parent_hash: Optional[str] = None,
|
||||
tag: Optional[List[str]] = None,
|
||||
**extra: Any,
|
||||
) -> Dict[str, Any]:
|
||||
"""Create a PipeObject-compatible result dict for pipeline chaining.
|
||||
|
||||
This is a helper to emit results in the standard format that downstream
|
||||
cmdlet can process (filter, tag, cleanup, etc.).
|
||||
|
||||
Args:
|
||||
source: Source system (e.g., 'local', 'hydrus', 'download')
|
||||
identifier: Unique ID from source
|
||||
file_path: Path to the file
|
||||
cmdlet_name: Name of the cmdlet that created this (e.g., 'download-data', 'screen-shot')
|
||||
title: Human-readable title
|
||||
hash_value: SHA-256 hash of file (for integrity)
|
||||
is_temp: If True, this is a temporary/intermediate artifact
|
||||
parent_hash: Hash of the parent file in the chain (for provenance)
|
||||
tag: List of tag values to apply
|
||||
**extra: Additional fields
|
||||
|
||||
Returns:
|
||||
Dict with all PipeObject fields for emission
|
||||
"""
|
||||
store_override = extra.pop("store", None)
|
||||
|
||||
result = build_file_result_payload(
|
||||
title=title,
|
||||
path=file_path,
|
||||
hash_value=hash_value,
|
||||
store=store_override if store_override is not None else source,
|
||||
tag=tag,
|
||||
source=source,
|
||||
id=identifier,
|
||||
action=f"cmdlet:{cmdlet_name}",
|
||||
**extra,
|
||||
)
|
||||
|
||||
if is_temp:
|
||||
result["is_temp"] = True
|
||||
if parent_hash:
|
||||
result["parent_hash"] = parent_hash
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def mark_as_temp(pipe_object: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Mark a PipeObject dict as temporary (intermediate artifact).
|
||||
|
||||
Args:
|
||||
pipe_object: Result dict from cmdlet emission
|
||||
|
||||
Returns:
|
||||
Modified dict with is_temp=True
|
||||
"""
|
||||
pipe_object["is_temp"] = True
|
||||
return pipe_object
|
||||
|
||||
|
||||
def set_parent_hash(pipe_object: Dict[str, Any], parent_hash: str) -> Dict[str, Any]:
|
||||
"""Set the parent_hash for provenance tracking.
|
||||
|
||||
Args:
|
||||
pipe_object: Result dict
|
||||
parent_hash: Parent file's hash
|
||||
|
||||
Returns:
|
||||
Modified dict with parent_hash set to the hash
|
||||
"""
|
||||
pipe_object["parent_hash"] = parent_hash
|
||||
return pipe_object
|
||||
|
||||
|
||||
def get_pipe_object_path(pipe_object: Any) -> Optional[str]:
|
||||
"""Extract file path from PipeObject, dict, or pipeline-friendly object."""
|
||||
if pipe_object is None:
|
||||
return None
|
||||
for attr in ("path", "target"):
|
||||
if hasattr(pipe_object, attr):
|
||||
value = getattr(pipe_object, attr)
|
||||
if value:
|
||||
return value
|
||||
if isinstance(pipe_object, dict):
|
||||
for key in ("path", "target"):
|
||||
value = pipe_object.get(key)
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def get_pipe_object_hash(pipe_object: Any) -> Optional[str]:
|
||||
"""Extract file hash from PipeObject, dict, or pipeline-friendly object."""
|
||||
if pipe_object is None:
|
||||
return None
|
||||
for attr in ("hash",):
|
||||
if hasattr(pipe_object, attr):
|
||||
value = getattr(pipe_object, attr)
|
||||
if value:
|
||||
return value
|
||||
if isinstance(pipe_object, dict):
|
||||
for key in ("hash",):
|
||||
value = pipe_object.get(key)
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def resolve_item_store_hash(
|
||||
item: Any,
|
||||
*,
|
||||
override_store: Optional[str] = None,
|
||||
override_hash: Optional[str] = None,
|
||||
hash_field: str = "hash",
|
||||
store_field: str = "store",
|
||||
path_fields: Sequence[str] = ("path", "target"),
|
||||
) -> Tuple[str, Optional[str]]:
|
||||
"""Resolve store name and normalized hash from a result item."""
|
||||
store_name = str(override_store or get_field(item, store_field) or "").strip()
|
||||
raw_hash = get_field(item, hash_field)
|
||||
|
||||
raw_path = None
|
||||
for field_name in path_fields:
|
||||
candidate = get_field(item, field_name)
|
||||
if candidate:
|
||||
raw_path = candidate
|
||||
break
|
||||
|
||||
resolved_hash = resolve_hash_for_cmdlet(
|
||||
str(raw_hash) if raw_hash else None,
|
||||
str(raw_path) if raw_path else None,
|
||||
str(override_hash) if override_hash else None,
|
||||
)
|
||||
return store_name, resolved_hash
|
||||
|
||||
|
||||
def coerce_to_pipe_object(
|
||||
value: Any,
|
||||
default_path: Optional[str] = None,
|
||||
) -> models.PipeObject:
|
||||
"""Normalize any incoming result to a PipeObject for single-source-of-truth state.
|
||||
|
||||
Uses hash+store canonical pattern.
|
||||
"""
|
||||
if isinstance(value, models.PipeObject):
|
||||
return value
|
||||
|
||||
known_keys = {
|
||||
"hash",
|
||||
"store",
|
||||
"tag",
|
||||
"title",
|
||||
"url",
|
||||
"source_url",
|
||||
"duration",
|
||||
"metadata",
|
||||
"warnings",
|
||||
"path",
|
||||
"relationships",
|
||||
"is_temp",
|
||||
"action",
|
||||
"parent_hash",
|
||||
}
|
||||
|
||||
if hasattr(value, "to_dict"):
|
||||
value = value.to_dict()
|
||||
elif not isinstance(value, dict):
|
||||
try:
|
||||
obj_map: Dict[str, Any] = {}
|
||||
for k in (
|
||||
"hash",
|
||||
"store",
|
||||
"provider",
|
||||
"prov",
|
||||
"tag",
|
||||
"title",
|
||||
"url",
|
||||
"source_url",
|
||||
"duration",
|
||||
"duration_seconds",
|
||||
"metadata",
|
||||
"full_metadata",
|
||||
"warnings",
|
||||
"path",
|
||||
"target",
|
||||
"relationships",
|
||||
"is_temp",
|
||||
"action",
|
||||
"parent_hash",
|
||||
"extra",
|
||||
"media_kind",
|
||||
):
|
||||
if hasattr(value, k):
|
||||
obj_map[k] = getattr(value, k)
|
||||
if obj_map:
|
||||
value = obj_map
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if isinstance(value, dict):
|
||||
hash_val = value.get("hash")
|
||||
store_val = value.get("store") or "PATH"
|
||||
if not store_val or store_val == "PATH":
|
||||
try:
|
||||
extra_store = value.get("extra", {}).get("store")
|
||||
except Exception:
|
||||
extra_store = None
|
||||
if extra_store:
|
||||
store_val = extra_store
|
||||
|
||||
if not hash_val:
|
||||
path_val = value.get("path")
|
||||
if path_val:
|
||||
try:
|
||||
from SYS.utils import sha256_file
|
||||
|
||||
hash_val = sha256_file(Path(path_val))
|
||||
except Exception:
|
||||
hash_val = "unknown"
|
||||
else:
|
||||
hash_val = "unknown"
|
||||
|
||||
title_val = value.get("title")
|
||||
if not title_val:
|
||||
path_val = value.get("path")
|
||||
if path_val:
|
||||
try:
|
||||
title_val = Path(path_val).stem
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
extra = {
|
||||
k: v
|
||||
for k, v in value.items() if k not in known_keys
|
||||
}
|
||||
|
||||
from SYS.metadata import normalize_urls
|
||||
|
||||
url_list = normalize_urls(value.get("url"))
|
||||
url_val = url_list[0] if url_list else None
|
||||
if len(url_list) > 1:
|
||||
extra["url"] = url_list
|
||||
|
||||
rels = value.get("relationships") or {}
|
||||
|
||||
tag_val: list[str] = []
|
||||
if "tag" in value:
|
||||
raw_tag = value["tag"]
|
||||
if isinstance(raw_tag, list):
|
||||
tag_val = [str(t) for t in raw_tag if t is not None]
|
||||
elif isinstance(raw_tag, str):
|
||||
tag_val = [raw_tag]
|
||||
|
||||
path_val = value.get("path")
|
||||
if not path_val and "target" in value:
|
||||
target = value["target"]
|
||||
if target and not (isinstance(target, str) and (target.startswith("http://")
|
||||
or target.startswith("https://"))):
|
||||
path_val = target
|
||||
|
||||
try:
|
||||
if isinstance(path_val, str) and (path_val.startswith("http://")
|
||||
or path_val.startswith("https://")):
|
||||
if not url_val:
|
||||
url_val = path_val
|
||||
path_val = None
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if "media_kind" in value:
|
||||
extra["media_kind"] = value["media_kind"]
|
||||
|
||||
pipe_obj = models.PipeObject(
|
||||
hash=hash_val,
|
||||
store=store_val,
|
||||
plugin=str(
|
||||
value.get("plugin")
|
||||
or value.get("prov")
|
||||
or value.get("source")
|
||||
or extra.get("plugin")
|
||||
or extra.get("source")
|
||||
or ""
|
||||
).strip() or None,
|
||||
tag=tag_val,
|
||||
title=title_val,
|
||||
url=url_val,
|
||||
source_url=value.get("source_url"),
|
||||
duration=value.get("duration") or value.get("duration_seconds"),
|
||||
metadata=value.get("metadata") or value.get("full_metadata") or {},
|
||||
warnings=list(value.get("warnings") or []),
|
||||
path=path_val,
|
||||
relationships=rels,
|
||||
is_temp=bool(value.get("is_temp", False)),
|
||||
action=value.get("action"),
|
||||
parent_hash=value.get("parent_hash"),
|
||||
extra=extra,
|
||||
)
|
||||
|
||||
return pipe_obj
|
||||
|
||||
hash_val = "unknown"
|
||||
path_val = default_path or getattr(value, "path", None)
|
||||
url_val: Optional[str] = None
|
||||
title_val = None
|
||||
|
||||
if isinstance(value, str):
|
||||
s = value.strip()
|
||||
if s.lower().startswith(("http://", "https://")):
|
||||
url_val = s
|
||||
path_val = None
|
||||
else:
|
||||
path_val = s
|
||||
|
||||
if path_val and path_val != "unknown":
|
||||
try:
|
||||
from SYS.utils import sha256_file
|
||||
|
||||
path_obj = Path(path_val)
|
||||
hash_val = sha256_file(path_obj)
|
||||
title_val = path_obj.stem
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
store_val = "URL" if url_val else "PATH"
|
||||
|
||||
pipe_obj = models.PipeObject(
|
||||
hash=hash_val,
|
||||
store=store_val,
|
||||
plugin=None,
|
||||
path=str(path_val) if path_val and path_val != "unknown" else None,
|
||||
title=title_val,
|
||||
url=url_val,
|
||||
source_url=url_val,
|
||||
tag=[],
|
||||
extra={},
|
||||
)
|
||||
|
||||
return pipe_obj
|
||||
|
||||
|
||||
def propagate_metadata(
|
||||
previous_items: Sequence[Any],
|
||||
new_items: Sequence[Any],
|
||||
) -> List[Any]:
|
||||
"""Merge metadata/tags from previous pipeline stage into new items.
|
||||
|
||||
Implements "sticky metadata": items generated by a transformation (download, convert)
|
||||
should inherit rich info (lyrics, art, tags) from their source.
|
||||
|
||||
Strategies:
|
||||
A. Hash Match: If inputs/outputs share a hash, they are the same item.
|
||||
B. Index Match: If lists are same length, assume 1:1 mapping (heuristic).
|
||||
C. Explicit Parent: If output has `parent_hash`, link to input with that hash.
|
||||
"""
|
||||
if not previous_items or not new_items:
|
||||
return list(new_items)
|
||||
|
||||
try:
|
||||
prev_normalized = [coerce_to_pipe_object(p) for p in previous_items]
|
||||
except Exception:
|
||||
return list(new_items)
|
||||
|
||||
prev_by_hash: Dict[str, models.PipeObject] = {}
|
||||
for p_obj in prev_normalized:
|
||||
if p_obj.hash and p_obj.hash != "unknown":
|
||||
prev_by_hash[p_obj.hash] = p_obj
|
||||
|
||||
normalized: List[Any] = []
|
||||
|
||||
is_same_length = len(new_items) == len(prev_normalized)
|
||||
|
||||
for i, item in enumerate(new_items):
|
||||
if isinstance(item, dict) and item.get("_skip_metadata_propagation"):
|
||||
normalized.append(item)
|
||||
continue
|
||||
try:
|
||||
obj = coerce_to_pipe_object(item)
|
||||
except Exception:
|
||||
normalized.append(item)
|
||||
continue
|
||||
|
||||
parent: Optional[models.PipeObject] = None
|
||||
|
||||
if obj.hash in prev_by_hash:
|
||||
parent = prev_by_hash[obj.hash]
|
||||
|
||||
if not parent and is_same_length:
|
||||
parent = prev_normalized[i]
|
||||
|
||||
if not parent and obj.parent_hash and obj.parent_hash in prev_by_hash:
|
||||
parent = prev_by_hash[obj.parent_hash]
|
||||
|
||||
if parent:
|
||||
if parent.tag:
|
||||
if not obj.tag:
|
||||
obj.tag = list(parent.tag)
|
||||
else:
|
||||
curr_tags = {str(t).lower() for t in obj.tag}
|
||||
for pt in parent.tag:
|
||||
if str(pt).lower() not in curr_tags:
|
||||
obj.tag.append(pt)
|
||||
|
||||
if parent.metadata:
|
||||
if not obj.metadata:
|
||||
obj.metadata = parent.metadata.copy()
|
||||
else:
|
||||
for mk, mv in parent.metadata.items():
|
||||
if mk not in obj.metadata:
|
||||
obj.metadata[mk] = mv
|
||||
|
||||
if parent.source_url and not obj.source_url:
|
||||
obj.source_url = parent.source_url
|
||||
elif parent.url and not obj.source_url and not obj.url:
|
||||
obj.source_url = parent.url
|
||||
|
||||
if parent.relationships:
|
||||
if not obj.relationships:
|
||||
obj.relationships = parent.relationships.copy()
|
||||
else:
|
||||
for rk, rv in parent.relationships.items():
|
||||
if rk not in obj.relationships:
|
||||
obj.relationships[rk] = rv
|
||||
|
||||
if parent.extra:
|
||||
if not obj.extra:
|
||||
obj.extra = parent.extra.copy()
|
||||
else:
|
||||
for ek, ev in parent.extra.items():
|
||||
if ek not in obj.extra:
|
||||
obj.extra[ek] = ev
|
||||
elif ek == "notes" and isinstance(ev, dict) and isinstance(obj.extra[ek], dict):
|
||||
for nk, nv in ev.items():
|
||||
if nk not in obj.extra[ek]:
|
||||
obj.extra[ek][nk] = nv
|
||||
|
||||
normalized.append(obj)
|
||||
|
||||
return normalized
|
||||
|
||||
|
||||
def pipeline_item_local_path(item: Any) -> Optional[str]:
|
||||
"""Extract local file path from a pipeline item.
|
||||
|
||||
Supports both dataclass objects with .path attribute and dicts.
|
||||
Returns None for HTTP/HTTPS url.
|
||||
|
||||
Args:
|
||||
item: Pipeline item (PipelineItem dataclass, dict, or other)
|
||||
|
||||
Returns:
|
||||
Local file path string, or None if item is not a local file
|
||||
"""
|
||||
path_value: Optional[str] = None
|
||||
if hasattr(item, "path"):
|
||||
path_value = getattr(item, "path", None)
|
||||
elif isinstance(item, dict):
|
||||
raw = item.get("path") or item.get("url")
|
||||
path_value = str(raw) if raw is not None else None
|
||||
if not isinstance(path_value, str):
|
||||
return None
|
||||
text = path_value.strip()
|
||||
if not text:
|
||||
return None
|
||||
if text.lower().startswith(("http://", "https://")):
|
||||
return None
|
||||
return text
|
||||
|
||||
|
||||
def extract_relationships(result: Any) -> Optional[Dict[str, Any]]:
|
||||
if isinstance(result, models.PipeObject):
|
||||
relationships = result.get_relationships()
|
||||
return relationships or None
|
||||
if isinstance(result, dict):
|
||||
relationships = result.get("relationships")
|
||||
if isinstance(relationships, dict) and relationships:
|
||||
return relationships
|
||||
return None
|
||||
|
||||
|
||||
def extract_duration(result: Any) -> Optional[float]:
|
||||
duration = None
|
||||
if isinstance(result, models.PipeObject):
|
||||
duration = result.duration
|
||||
elif isinstance(result, dict):
|
||||
duration = result.get("duration")
|
||||
if duration is None:
|
||||
metadata = result.get("metadata")
|
||||
if isinstance(metadata, dict):
|
||||
duration = metadata.get("duration")
|
||||
if duration is None:
|
||||
return None
|
||||
try:
|
||||
return float(duration)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
Reference in New Issue
Block a user