style: apply ruff auto-fixes
This commit is contained in:
+1
-1
@@ -15,7 +15,7 @@ import time
|
||||
import traceback
|
||||
import re
|
||||
import os
|
||||
from typing import Optional, Dict, Any, Callable, BinaryIO, List, Iterable, Set, Union
|
||||
from typing import Optional, Dict, Any, Callable, List, Union
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote, urlparse, parse_qs
|
||||
import logging
|
||||
|
||||
+1
-2
@@ -12,7 +12,6 @@ import sys
|
||||
import time
|
||||
|
||||
from typing import Any, Dict, Optional, Set, List, Sequence, Tuple
|
||||
import time
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from SYS.logger import log, debug
|
||||
@@ -1124,7 +1123,7 @@ def unlock_link_cmdlet(result: Any, args: Sequence[str], config: Dict[str, Any])
|
||||
# Note: The cmdlet wrapper will handle emitting to pipeline
|
||||
return 0
|
||||
else:
|
||||
log(f"❌ Failed to unlock link or already unrestricted", file=sys.stderr)
|
||||
log("❌ Failed to unlock link or already unrestricted", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from .HTTP import HTTPClient
|
||||
|
||||
@@ -92,7 +92,7 @@
|
||||
"(hitfile\\.net/[a-z0-9A-Z]{4,9})"
|
||||
],
|
||||
"regexp": "(hitf\\.(to|cc)/([a-z0-9A-Z]{4,9}))|(htfl\\.(net|to|cc)/([a-z0-9A-Z]{4,9}))|(hitfile\\.(net)/download/free/([a-z0-9A-Z]{4,9}))|((hitfile\\.net/[a-z0-9A-Z]{4,9}))",
|
||||
"status": false
|
||||
"status": true
|
||||
},
|
||||
"mega": {
|
||||
"name": "mega",
|
||||
@@ -353,7 +353,7 @@
|
||||
"filedot\\.(xyz|to|top)/([0-9a-zA-Z]{12})"
|
||||
],
|
||||
"regexp": "filedot\\.(xyz|to|top)/([0-9a-zA-Z]{12})",
|
||||
"status": true
|
||||
"status": false
|
||||
},
|
||||
"filefactory": {
|
||||
"name": "filefactory",
|
||||
@@ -622,7 +622,7 @@
|
||||
"(simfileshare\\.net/download/[0-9]+/)"
|
||||
],
|
||||
"regexp": "(simfileshare\\.net/download/[0-9]+/)",
|
||||
"status": true
|
||||
"status": false
|
||||
},
|
||||
"streamtape": {
|
||||
"name": "streamtape",
|
||||
|
||||
+15
-4
@@ -13,8 +13,6 @@ from __future__ import annotations
|
||||
import sqlite3
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
import shutil
|
||||
import time
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
@@ -303,8 +301,21 @@ class API_folder_store:
|
||||
|
||||
if should_check_empty:
|
||||
# Check if there are any files or directories in the library root (excluding the DB itself if it was just created)
|
||||
# We use a generator and next() for efficiency.
|
||||
existing_items = [item for item in self.library_root.iterdir() if item.name != self.DB_NAME]
|
||||
|
||||
# Allow an empty 'incoming' directory created by upload flow to exist
|
||||
# (this prevents a false-positive safety check when an upload endpoint
|
||||
# creates the incoming dir before DB initialization).
|
||||
if existing_items:
|
||||
if len(existing_items) == 1 and existing_items[0].name == "incoming" and existing_items[0].is_dir():
|
||||
try:
|
||||
# If the incoming directory is empty, treat it as harmless.
|
||||
if not any(existing_items[0].iterdir()):
|
||||
existing_items = []
|
||||
except Exception:
|
||||
# If we can't inspect it safely, leave the original items in place
|
||||
pass
|
||||
|
||||
if existing_items:
|
||||
# Log the items found for debugging
|
||||
item_names = [i.name for i in existing_items[:5]]
|
||||
@@ -1378,7 +1389,7 @@ class API_folder_store:
|
||||
(file_hash,
|
||||
existing_title[0]),
|
||||
)
|
||||
logger.debug(f"[save_tags] Preserved existing title tag")
|
||||
logger.debug("[save_tags] Preserved existing title tag")
|
||||
elif not existing_title and not new_title_provided:
|
||||
filename_without_ext = abs_path.stem
|
||||
if filename_without_ext:
|
||||
|
||||
@@ -12,7 +12,6 @@ The LoC JSON API does not require an API key.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from .base import API, ApiError
|
||||
|
||||
@@ -12,7 +12,6 @@ Authentication headers required for most endpoints:
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
+1
-1
@@ -32,7 +32,7 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from SYS.logger import debug, log
|
||||
from SYS.logger import debug
|
||||
|
||||
# Optional Python ZeroTier bindings - prefer them when available
|
||||
_HAVE_PY_ZEROTIER = False
|
||||
|
||||
+3
-3
@@ -351,10 +351,10 @@ class MPV:
|
||||
pipeline += f" | add-file -path {_q(path or '')}"
|
||||
|
||||
try:
|
||||
from TUI.pipeline_runner import PipelineExecutor # noqa: WPS433
|
||||
from TUI.pipeline_runner import PipelineRunner # noqa: WPS433
|
||||
|
||||
executor = PipelineExecutor()
|
||||
result = executor.run_pipeline(pipeline)
|
||||
runner = PipelineRunner()
|
||||
result = runner.run_pipeline(pipeline)
|
||||
return {
|
||||
"success": bool(getattr(result,
|
||||
"success",
|
||||
|
||||
@@ -74,7 +74,7 @@ OBS_ID_REQUEST = 1001
|
||||
|
||||
def _run_pipeline(pipeline_text: str, *, seeds: Any = None) -> Dict[str, Any]:
|
||||
# Import after sys.path fix.
|
||||
from TUI.pipeline_runner import PipelineExecutor # noqa: WPS433
|
||||
from TUI.pipeline_runner import PipelineRunner # noqa: WPS433
|
||||
|
||||
def _table_to_payload(table: Any) -> Optional[Dict[str, Any]]:
|
||||
if table is None:
|
||||
@@ -133,8 +133,8 @@ def _run_pipeline(pipeline_text: str, *, seeds: Any = None) -> Dict[str, Any]:
|
||||
"rows": rows_payload
|
||||
}
|
||||
|
||||
executor = PipelineExecutor()
|
||||
result = executor.run_pipeline(pipeline_text, seeds=seeds)
|
||||
runner = PipelineRunner()
|
||||
result = runner.run_pipeline(pipeline_text, seeds=seeds)
|
||||
|
||||
table_payload = None
|
||||
try:
|
||||
@@ -905,7 +905,7 @@ def main(argv: Optional[list[str]] = None) -> int:
|
||||
]
|
||||
)
|
||||
_append_helper_log(
|
||||
f"[helper] published store-choices to user-data/medeia-store-choices-cached"
|
||||
"[helper] published store-choices to user-data/medeia-store-choices-cached"
|
||||
)
|
||||
except Exception as exc:
|
||||
_append_helper_log(
|
||||
|
||||
+1
-5
@@ -1,15 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import shutil
|
||||
import string
|
||||
import subprocess
|
||||
import time
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from API.Tidal import (
|
||||
@@ -20,7 +17,6 @@ from API.Tidal import (
|
||||
stringify,
|
||||
)
|
||||
from ProviderCore.base import Provider, SearchResult, parse_inline_query_arguments
|
||||
from ProviderCore.inline_utils import collect_choice
|
||||
from cmdlet._shared import get_field
|
||||
from SYS import pipeline as pipeline_context
|
||||
from SYS.logger import debug, log
|
||||
|
||||
+1
-4
@@ -1,15 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import shutil
|
||||
import string
|
||||
import subprocess
|
||||
import time
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from API.Tidal import (
|
||||
|
||||
+1
-1
@@ -1265,7 +1265,7 @@ class LibgenSearch:
|
||||
_call(log_info, f"[libgen] Using mirror: {mirror}")
|
||||
return results
|
||||
else:
|
||||
_call(log_info, f"[libgen] Mirror returned 0 results; stopping mirror fallback")
|
||||
_call(log_info, "[libgen] Mirror returned 0 results; stopping mirror fallback")
|
||||
break
|
||||
except requests.exceptions.Timeout:
|
||||
_call(log_info, f"[libgen] Mirror timed out: {mirror}")
|
||||
|
||||
@@ -11,7 +11,7 @@ import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
@@ -20,7 +20,7 @@ from API.HTTP import HTTPClient, get_requests_verify_value
|
||||
from ProviderCore.base import Provider, SearchResult
|
||||
from SYS.utils import sanitize_filename
|
||||
from SYS.cli_syntax import get_field, get_free_text, parse_query
|
||||
from SYS.logger import debug, log
|
||||
from SYS.logger import log
|
||||
from Provider.metadata_provider import (
|
||||
archive_item_metadata_to_tags,
|
||||
fetch_archive_item_metadata,
|
||||
|
||||
@@ -109,7 +109,6 @@ class YouTube(TableProviderMixin, Provider):
|
||||
|
||||
def validate(self) -> bool:
|
||||
try:
|
||||
import yt_dlp # type: ignore
|
||||
|
||||
return True
|
||||
except Exception:
|
||||
|
||||
+2
-5
@@ -9,13 +9,11 @@ This keeps format selection logic in ytdlp and leaves add-file plug-and-play.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
|
||||
from ProviderCore.base import Provider, SearchResult
|
||||
from SYS.provider_helpers import TableProviderMixin
|
||||
from SYS.logger import log, debug
|
||||
from tool.ytdlp import list_formats, is_url_supported_by_ytdlp
|
||||
from SYS.logger import debug
|
||||
|
||||
|
||||
class ytdlp(TableProviderMixin, Provider):
|
||||
@@ -196,7 +194,6 @@ class ytdlp(TableProviderMixin, Provider):
|
||||
def validate(self) -> bool:
|
||||
"""Validate yt-dlp availability."""
|
||||
try:
|
||||
import yt_dlp # type: ignore
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
@@ -295,7 +292,7 @@ try:
|
||||
debug(f"[ytdlp] Selection routed with format_id: {format_id}")
|
||||
return result_args
|
||||
|
||||
debug(f"[ytdlp] Warning: No selection args or format_id found in row")
|
||||
debug("[ytdlp] Warning: No selection args or format_id found in row")
|
||||
return []
|
||||
|
||||
register_provider(
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from abc import ABC
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Sequence, Tuple, Callable
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import atexit
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
|
||||
+1
-4
@@ -4,13 +4,10 @@ import subprocess
|
||||
import sys
|
||||
import shutil
|
||||
from SYS.logger import log, debug
|
||||
from urllib.parse import urlsplit, urlunsplit, unquote
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional, Sequence, Set, Tuple
|
||||
|
||||
from API.HydrusNetwork import apply_hydrus_tag_mutation, fetch_hydrus_metadata, fetch_hydrus_metadata_by_url
|
||||
from SYS.models import FileRelationshipTracker
|
||||
|
||||
try: # Optional; used when available for richer metadata fetches
|
||||
import yt_dlp
|
||||
@@ -2585,7 +2582,7 @@ def scrape_url_metadata(
|
||||
)
|
||||
except json_module.JSONDecodeError:
|
||||
pass
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
pass # Silently ignore if we can't get playlist entries
|
||||
|
||||
# Fallback: if still no tags detected, get from first item
|
||||
|
||||
@@ -4,7 +4,7 @@ import importlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
from SYS.logger import log
|
||||
from SYS.rich_display import stdout_console
|
||||
|
||||
+1690
-1
File diff suppressed because it is too large
Load Diff
@@ -18,8 +18,7 @@ so authors don't have to install pandas/bs4 unless they want to.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
from urllib.parse import quote_plus
|
||||
from typing import List, Optional
|
||||
|
||||
from API.HTTP import HTTPClient
|
||||
from ProviderCore.base import SearchResult
|
||||
|
||||
+1
-2
@@ -16,7 +16,6 @@ from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional, Callable, Set
|
||||
from pathlib import Path
|
||||
import json
|
||||
import shutil
|
||||
|
||||
from rich.box import SIMPLE
|
||||
from rich.console import Group
|
||||
@@ -1678,7 +1677,7 @@ class Table:
|
||||
try:
|
||||
int(value)
|
||||
except ValueError:
|
||||
print(f"Must be an integer")
|
||||
print("Must be an integer")
|
||||
continue
|
||||
|
||||
return value
|
||||
|
||||
+1
-4
@@ -11,7 +11,7 @@ from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import sys
|
||||
from typing import Any, Iterator, Sequence, TextIO
|
||||
from typing import Any, Iterator, TextIO
|
||||
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
@@ -81,7 +81,6 @@ def show_provider_config_panel(
|
||||
) -> None:
|
||||
"""Show a Rich panel explaining how to configure providers."""
|
||||
from rich.table import Table as RichTable
|
||||
from rich.text import Text
|
||||
from rich.console import Group
|
||||
|
||||
if isinstance(provider_names, str):
|
||||
@@ -117,7 +116,6 @@ def show_store_config_panel(
|
||||
) -> None:
|
||||
"""Show a Rich panel explaining how to configure storage backends."""
|
||||
from rich.table import Table as RichTable
|
||||
from rich.text import Text
|
||||
from rich.console import Group
|
||||
|
||||
if isinstance(store_names, str):
|
||||
@@ -152,7 +150,6 @@ def show_available_providers_panel(provider_names: List[str]) -> None:
|
||||
"""Show a Rich panel listing available/configured providers."""
|
||||
from rich.columns import Columns
|
||||
from rich.console import Group
|
||||
from rich.text import Text
|
||||
|
||||
if not provider_names:
|
||||
return
|
||||
|
||||
+1
-2
@@ -14,9 +14,8 @@ except Exception:
|
||||
import os
|
||||
import base64
|
||||
import logging
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Optional
|
||||
from typing import Any, Iterable
|
||||
from datetime import datetime
|
||||
from dataclasses import dataclass, field
|
||||
from fnmatch import fnmatch
|
||||
|
||||
+349
@@ -0,0 +1,349 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import io
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional, Set, TextIO
|
||||
|
||||
from SYS.config import get_local_storage_path
|
||||
from SYS.worker_manager import WorkerManager
|
||||
|
||||
|
||||
class WorkerOutputMirror(io.TextIOBase):
|
||||
"""Mirror stdout/stderr to worker manager while preserving console output."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
original: TextIO,
|
||||
manager: WorkerManager,
|
||||
worker_id: str,
|
||||
channel: str,
|
||||
):
|
||||
self._original = original
|
||||
self._manager = manager
|
||||
self._worker_id = worker_id
|
||||
self._channel = channel
|
||||
self._pending: str = ""
|
||||
|
||||
def write(self, data: str) -> int: # type: ignore[override]
|
||||
if not data:
|
||||
return 0
|
||||
self._original.write(data)
|
||||
self._buffer_text(data)
|
||||
return len(data)
|
||||
|
||||
def flush(self) -> None: # type: ignore[override]
|
||||
self._original.flush()
|
||||
self._flush_pending(force=True)
|
||||
|
||||
def isatty(self) -> bool: # pragma: no cover
|
||||
return bool(getattr(self._original, "isatty", lambda: False)())
|
||||
|
||||
def _buffer_text(self, data: str) -> None:
|
||||
combined = self._pending + data
|
||||
lines = combined.splitlines(keepends=True)
|
||||
if not lines:
|
||||
self._pending = combined
|
||||
return
|
||||
|
||||
if lines[-1].endswith(("\n", "\r")):
|
||||
complete = lines
|
||||
self._pending = ""
|
||||
else:
|
||||
complete = lines[:-1]
|
||||
self._pending = lines[-1]
|
||||
|
||||
for chunk in complete:
|
||||
self._emit(chunk)
|
||||
|
||||
def _flush_pending(self, *, force: bool = False) -> None:
|
||||
if self._pending and force:
|
||||
self._emit(self._pending)
|
||||
self._pending = ""
|
||||
|
||||
def _emit(self, text: str) -> None:
|
||||
if not text:
|
||||
return
|
||||
try:
|
||||
self._manager.append_stdout(self._worker_id, text, channel=self._channel)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@property
|
||||
def encoding(self) -> str: # type: ignore[override]
|
||||
return getattr(self._original, "encoding", "utf-8")
|
||||
|
||||
|
||||
class WorkerStageSession:
|
||||
"""Lifecycle helper for wrapping a CLI cmdlet execution in a worker record."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
manager: WorkerManager,
|
||||
worker_id: str,
|
||||
orig_stdout: TextIO,
|
||||
orig_stderr: TextIO,
|
||||
stdout_proxy: WorkerOutputMirror,
|
||||
stderr_proxy: WorkerOutputMirror,
|
||||
config: Optional[Dict[str, Any]],
|
||||
logging_enabled: bool,
|
||||
completion_label: str,
|
||||
error_label: str,
|
||||
) -> None:
|
||||
self.manager = manager
|
||||
self.worker_id = worker_id
|
||||
self.orig_stdout = orig_stdout
|
||||
self.orig_stderr = orig_stderr
|
||||
self.stdout_proxy = stdout_proxy
|
||||
self.stderr_proxy = stderr_proxy
|
||||
self.config = config
|
||||
self.logging_enabled = logging_enabled
|
||||
self.closed = False
|
||||
self._completion_label = completion_label
|
||||
self._error_label = error_label
|
||||
|
||||
def close(self, *, status: str = "completed", error_msg: str = "") -> None:
|
||||
if self.closed:
|
||||
return
|
||||
try:
|
||||
self.stdout_proxy.flush()
|
||||
self.stderr_proxy.flush()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
sys.stdout = self.orig_stdout
|
||||
sys.stderr = self.orig_stderr
|
||||
|
||||
if self.logging_enabled:
|
||||
try:
|
||||
self.manager.disable_logging_for_worker(self.worker_id)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
if status == "completed":
|
||||
self.manager.log_step(self.worker_id, self._completion_label)
|
||||
else:
|
||||
self.manager.log_step(
|
||||
self.worker_id, f"{self._error_label}: {error_msg or status}"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
self.manager.finish_worker(
|
||||
self.worker_id, result=status or "completed", error_msg=error_msg or ""
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if self.config and self.config.get("_current_worker_id") == self.worker_id:
|
||||
self.config.pop("_current_worker_id", None)
|
||||
self.closed = True
|
||||
|
||||
|
||||
class WorkerManagerRegistry:
|
||||
"""Process-wide WorkerManager cache keyed by library_root."""
|
||||
|
||||
_manager: Optional[WorkerManager] = None
|
||||
_manager_root: Optional[Path] = None
|
||||
_orphan_cleanup_done: bool = False
|
||||
_registered: bool = False
|
||||
|
||||
@classmethod
|
||||
def ensure(cls, config: Dict[str, Any]) -> Optional[WorkerManager]:
|
||||
if not isinstance(config, dict):
|
||||
return None
|
||||
|
||||
existing = config.get("_worker_manager")
|
||||
if isinstance(existing, WorkerManager):
|
||||
return existing
|
||||
|
||||
library_root = get_local_storage_path(config)
|
||||
if not library_root:
|
||||
return None
|
||||
|
||||
try:
|
||||
resolved_root = Path(library_root).resolve()
|
||||
except Exception:
|
||||
resolved_root = Path(library_root)
|
||||
|
||||
try:
|
||||
if cls._manager is None or cls._manager_root != resolved_root:
|
||||
if cls._manager is not None:
|
||||
try:
|
||||
cls._manager.close()
|
||||
except Exception:
|
||||
pass
|
||||
cls._manager = WorkerManager(resolved_root, auto_refresh_interval=0.5)
|
||||
cls._manager_root = resolved_root
|
||||
|
||||
manager = cls._manager
|
||||
config["_worker_manager"] = manager
|
||||
|
||||
if manager is not None and not cls._orphan_cleanup_done:
|
||||
try:
|
||||
manager.expire_running_workers(
|
||||
older_than_seconds=120,
|
||||
worker_id_prefix="cli_%",
|
||||
reason=(
|
||||
"CLI session ended unexpectedly; marking worker as failed",
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
cls._orphan_cleanup_done = True
|
||||
|
||||
if not cls._registered:
|
||||
atexit.register(cls.close)
|
||||
cls._registered = True
|
||||
|
||||
return manager
|
||||
except Exception as exc:
|
||||
print(f"[worker] Could not initialize worker manager: {exc}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def close(cls) -> None:
|
||||
if cls._manager is None:
|
||||
return
|
||||
try:
|
||||
cls._manager.close()
|
||||
except Exception:
|
||||
pass
|
||||
cls._manager = None
|
||||
cls._manager_root = None
|
||||
cls._orphan_cleanup_done = False
|
||||
|
||||
|
||||
class WorkerStages:
|
||||
"""Factory methods for stage/pipeline worker sessions."""
|
||||
|
||||
@staticmethod
|
||||
def _start_worker_session(
|
||||
worker_manager: Optional[WorkerManager],
|
||||
*,
|
||||
worker_type: str,
|
||||
title: str,
|
||||
description: str,
|
||||
pipe_text: str,
|
||||
config: Optional[Dict[str, Any]],
|
||||
completion_label: str,
|
||||
error_label: str,
|
||||
skip_logging_for: Optional[Set[str]] = None,
|
||||
session_worker_ids: Optional[Set[str]] = None,
|
||||
) -> Optional[WorkerStageSession]:
|
||||
if worker_manager is None:
|
||||
return None
|
||||
if skip_logging_for and worker_type in skip_logging_for:
|
||||
return None
|
||||
|
||||
safe_type = worker_type or "cmd"
|
||||
worker_id = f"cli_{safe_type[:8]}_{uuid.uuid4().hex[:6]}"
|
||||
|
||||
try:
|
||||
tracked = worker_manager.track_worker(
|
||||
worker_id,
|
||||
worker_type=worker_type,
|
||||
title=title,
|
||||
description=description or "(no args)",
|
||||
pipe=pipe_text,
|
||||
)
|
||||
if not tracked:
|
||||
return None
|
||||
except Exception as exc:
|
||||
print(f"[worker] Failed to track {worker_type}: {exc}", file=sys.stderr)
|
||||
return None
|
||||
|
||||
if session_worker_ids is not None:
|
||||
session_worker_ids.add(worker_id)
|
||||
|
||||
logging_enabled = False
|
||||
try:
|
||||
handler = worker_manager.enable_logging_for_worker(worker_id)
|
||||
logging_enabled = handler is not None
|
||||
except Exception:
|
||||
logging_enabled = False
|
||||
|
||||
orig_stdout = sys.stdout
|
||||
orig_stderr = sys.stderr
|
||||
stdout_proxy = WorkerOutputMirror(orig_stdout, worker_manager, worker_id, "stdout")
|
||||
stderr_proxy = WorkerOutputMirror(orig_stderr, worker_manager, worker_id, "stderr")
|
||||
sys.stdout = stdout_proxy
|
||||
sys.stderr = stderr_proxy
|
||||
if isinstance(config, dict):
|
||||
config["_current_worker_id"] = worker_id
|
||||
|
||||
try:
|
||||
worker_manager.log_step(worker_id, f"Started {worker_type}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return WorkerStageSession(
|
||||
manager=worker_manager,
|
||||
worker_id=worker_id,
|
||||
orig_stdout=orig_stdout,
|
||||
orig_stderr=orig_stderr,
|
||||
stdout_proxy=stdout_proxy,
|
||||
stderr_proxy=stderr_proxy,
|
||||
config=config,
|
||||
logging_enabled=logging_enabled,
|
||||
completion_label=completion_label,
|
||||
error_label=error_label,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def begin_stage(
|
||||
cls,
|
||||
worker_manager: Optional[WorkerManager],
|
||||
*,
|
||||
cmd_name: str,
|
||||
stage_tokens: Sequence[str],
|
||||
config: Optional[Dict[str, Any]],
|
||||
command_text: str,
|
||||
) -> Optional[WorkerStageSession]:
|
||||
description = " ".join(stage_tokens[1:]) if len(stage_tokens) > 1 else "(no args)"
|
||||
session_worker_ids = None
|
||||
if isinstance(config, dict):
|
||||
session_worker_ids = config.get("_session_worker_ids")
|
||||
|
||||
return cls._start_worker_session(
|
||||
worker_manager,
|
||||
worker_type=cmd_name,
|
||||
title=f"{cmd_name} stage",
|
||||
description=description,
|
||||
pipe_text=command_text,
|
||||
config=config,
|
||||
completion_label="Stage completed",
|
||||
error_label="Stage error",
|
||||
skip_logging_for={".worker", "worker", "workers"},
|
||||
session_worker_ids=session_worker_ids,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def begin_pipeline(
|
||||
cls,
|
||||
worker_manager: Optional[WorkerManager],
|
||||
*,
|
||||
pipeline_text: str,
|
||||
config: Optional[Dict[str, Any]],
|
||||
) -> Optional[WorkerStageSession]:
|
||||
session_worker_ids: Set[str] = set()
|
||||
if isinstance(config, dict):
|
||||
config["_session_worker_ids"] = session_worker_ids
|
||||
|
||||
return cls._start_worker_session(
|
||||
worker_manager,
|
||||
worker_type="pipeline",
|
||||
title="Pipeline run",
|
||||
description=pipeline_text,
|
||||
pipe_text=pipeline_text,
|
||||
config=config,
|
||||
completion_label="Pipeline completed",
|
||||
error_label="Pipeline error",
|
||||
session_worker_ids=session_worker_ids,
|
||||
)
|
||||
+2
-2
@@ -4,7 +4,7 @@ import json
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from fnmatch import fnmatch, translate
|
||||
from fnmatch import fnmatch
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
@@ -177,7 +177,7 @@ class Folder(Store):
|
||||
Checks for sidecars (.metadata, .tag) and imports them before renaming.
|
||||
Also ensures all files have a title: tag.
|
||||
"""
|
||||
from API.folder import API_folder_store, read_sidecar, write_sidecar, find_sidecar
|
||||
from API.folder import API_folder_store, read_sidecar, find_sidecar
|
||||
|
||||
try:
|
||||
with API_folder_store(location_path) as db:
|
||||
|
||||
@@ -1894,6 +1894,61 @@ class HydrusNetwork(Store):
|
||||
debug(f"{self._log_prefix()} add_url_bulk failed: {exc}")
|
||||
return False
|
||||
|
||||
def add_tags_bulk(self, items: List[tuple[str, List[str]]], *, service_name: str | None = None) -> bool:
|
||||
"""Bulk add tags to multiple Hydrus files.
|
||||
|
||||
Groups files by identical tag-sets and uses the Hydrus `mutate_tags_by_key`
|
||||
call (when a service key is available) to reduce the number of API calls.
|
||||
Falls back to per-hash `add_tag` calls if necessary.
|
||||
"""
|
||||
try:
|
||||
client = self._client
|
||||
if client is None:
|
||||
debug(f"{self._log_prefix()} add_tags_bulk: client unavailable")
|
||||
return False
|
||||
|
||||
# Group by canonical tag set (sorted tuple) to batch identical additions
|
||||
buckets: dict[tuple[str, ...], list[str]] = {}
|
||||
for file_identifier, tags in items or []:
|
||||
h = str(file_identifier or "").strip().lower()
|
||||
if len(h) != 64:
|
||||
continue
|
||||
tlist = [str(t).strip().lower() for t in (tags or []) if isinstance(t, str) and str(t).strip()]
|
||||
if not tlist:
|
||||
continue
|
||||
key = tuple(sorted(tlist))
|
||||
buckets.setdefault(key, []).append(h)
|
||||
|
||||
if not buckets:
|
||||
return False
|
||||
|
||||
svc = service_name or "my tags"
|
||||
service_key = self._get_service_key(svc)
|
||||
any_success = False
|
||||
|
||||
for tag_tuple, hashes in buckets.items():
|
||||
try:
|
||||
if service_key:
|
||||
# Mutate tags for many hashes in a single request
|
||||
client.mutate_tags_by_key(hashes=hashes, service_key=service_key, add_tags=list(tag_tuple))
|
||||
any_success = True
|
||||
continue
|
||||
except Exception as exc:
|
||||
debug(f"{self._log_prefix()} add_tags_bulk mutate failed for tags {tag_tuple}: {exc}")
|
||||
|
||||
# Fallback: apply per-hash add_tag
|
||||
for h in hashes:
|
||||
try:
|
||||
client.add_tag(h, list(tag_tuple), svc)
|
||||
any_success = True
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return any_success
|
||||
except Exception as exc:
|
||||
debug(f"{self._log_prefix()} add_tags_bulk failed: {exc}")
|
||||
return False
|
||||
|
||||
def delete_url(self, file_identifier: str, url: List[str], **kwargs: Any) -> bool:
|
||||
"""Delete one or more url from a Hydrus file."""
|
||||
try:
|
||||
|
||||
+52
-13
@@ -20,9 +20,6 @@ Notes:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
@@ -355,7 +352,6 @@ class ZeroTier(Store):
|
||||
|
||||
Returns the file hash on success, or None on failure.
|
||||
"""
|
||||
from SYS.utils import sha256_file
|
||||
|
||||
p = Path(file_path)
|
||||
if not p.exists():
|
||||
@@ -404,17 +400,60 @@ class ZeroTier(Store):
|
||||
data.append(("url", u))
|
||||
|
||||
files = {"file": (p.name, fh, "application/octet-stream")}
|
||||
resp = httpx.post(url, headers=headers, files=files, data=data, timeout=self._timeout)
|
||||
resp.raise_for_status()
|
||||
if resp.status_code in (200, 201):
|
||||
# Prefer `requests` for local testing / WSGI servers which may not accept
|
||||
# chunked uploads reliably with httpx/httpcore. Fall back to httpx otherwise.
|
||||
try:
|
||||
try:
|
||||
payload = resp.json()
|
||||
file_hash = payload.get("hash") or payload.get("file_hash")
|
||||
return file_hash
|
||||
except Exception:
|
||||
import requests
|
||||
# Convert data list-of-tuples to dict for requests (acceptable for repeated fields)
|
||||
data_dict = {}
|
||||
for k, v in data:
|
||||
if k in data_dict:
|
||||
existing = data_dict[k]
|
||||
if not isinstance(existing, list):
|
||||
data_dict[k] = [existing]
|
||||
data_dict[k].append(v)
|
||||
else:
|
||||
data_dict[k] = v
|
||||
r = requests.post(url, headers=headers, files=files, data=data_dict or None, timeout=self._timeout)
|
||||
if r.status_code in (200, 201):
|
||||
try:
|
||||
payload = r.json()
|
||||
file_hash = payload.get("hash") or payload.get("file_hash")
|
||||
return file_hash
|
||||
except Exception:
|
||||
return None
|
||||
try:
|
||||
debug(f"[zerotier-debug] upload failed (requests) status={r.status_code} body={r.text}")
|
||||
except Exception:
|
||||
pass
|
||||
debug(f"ZeroTier add_file failed (requests): status {r.status_code} body={getattr(r, 'text', '')}")
|
||||
return None
|
||||
debug(f"ZeroTier add_file failed: status {resp.status_code}")
|
||||
return None
|
||||
except Exception:
|
||||
import httpx
|
||||
resp = httpx.post(url, headers=headers, files=files, data=data, timeout=self._timeout)
|
||||
# Note: some environments may not create request.files correctly; capture body for debugging
|
||||
try:
|
||||
if resp.status_code in (200, 201):
|
||||
try:
|
||||
payload = resp.json()
|
||||
file_hash = payload.get("hash") or payload.get("file_hash")
|
||||
return file_hash
|
||||
except Exception:
|
||||
return None
|
||||
# Debug output to help tests capture server response
|
||||
try:
|
||||
debug(f"[zerotier-debug] upload failed status={resp.status_code} body={resp.text}")
|
||||
except Exception:
|
||||
pass
|
||||
debug(f"ZeroTier add_file failed: status {resp.status_code} body={getattr(resp, 'text', '')}")
|
||||
return None
|
||||
except Exception as exc:
|
||||
debug(f"ZeroTier add_file exception: {exc}")
|
||||
return None
|
||||
except Exception as exc:
|
||||
debug(f"ZeroTier add_file exception: {exc}")
|
||||
return None
|
||||
except Exception as exc:
|
||||
debug(f"ZeroTier add_file exception: {exc}")
|
||||
return None
|
||||
|
||||
+1
-2
@@ -15,8 +15,7 @@ import importlib
|
||||
import inspect
|
||||
import pkgutil
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, Optional, Type
|
||||
from typing import Any, Dict, Optional, Type
|
||||
|
||||
from SYS.logger import debug
|
||||
from SYS.utils import expand_path
|
||||
|
||||
@@ -6,7 +6,6 @@ import json
|
||||
import re
|
||||
import sys
|
||||
import subprocess
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional, Sequence
|
||||
from typing import Dict, Iterable, List, Sequence
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent
|
||||
ROOT_DIR = BASE_DIR.parent
|
||||
|
||||
@@ -1,12 +1,9 @@
|
||||
from textual.app import ComposeResult
|
||||
from textual.screen import ModalScreen
|
||||
from textual.containers import Container, Horizontal, Vertical, ScrollableContainer
|
||||
from textual.widgets import Static, Button, Input, Label, ListView, ListItem, Rule, OptionList, Footer, Select
|
||||
from textual.widgets import Static, Button, Input, Label, ListView, ListItem, Rule, Select
|
||||
from textual import on, work
|
||||
from textual.message import Message
|
||||
from typing import Dict, Any, List, Optional
|
||||
import os
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from pathlib import Path
|
||||
from SYS.config import load_config, save_config, global_config
|
||||
|
||||
@@ -9,11 +9,10 @@ This modal allows users to specify:
|
||||
|
||||
from textual.app import ComposeResult
|
||||
from textual.screen import ModalScreen
|
||||
from textual.containers import Container, Horizontal, Vertical, ScrollableContainer
|
||||
from textual.containers import Container, Horizontal, Vertical
|
||||
from textual.widgets import (
|
||||
Static,
|
||||
Button,
|
||||
Label,
|
||||
Select,
|
||||
Checkbox,
|
||||
TextArea,
|
||||
@@ -448,8 +447,6 @@ class DownloadModal(ModalScreen):
|
||||
|
||||
try:
|
||||
# Capture output from the cmdlet using temp files (more reliable than redirect)
|
||||
import tempfile
|
||||
import subprocess
|
||||
|
||||
# Try normal redirect first
|
||||
import io
|
||||
@@ -461,7 +458,7 @@ class DownloadModal(ModalScreen):
|
||||
# Always capture output
|
||||
try:
|
||||
with redirect_stdout(stdout_buf), redirect_stderr(stderr_buf):
|
||||
logger.info(f"Calling download_cmdlet...")
|
||||
logger.info("Calling download_cmdlet...")
|
||||
cmd_config = (
|
||||
dict(self.config)
|
||||
if isinstance(self.config,
|
||||
@@ -637,7 +634,7 @@ class DownloadModal(ModalScreen):
|
||||
|
||||
# Also append detailed error info to worker stdout for visibility
|
||||
if worker:
|
||||
worker.append_stdout(f"\n❌ DOWNLOAD FAILED\n")
|
||||
worker.append_stdout("\n❌ DOWNLOAD FAILED\n")
|
||||
worker.append_stdout(f"Reason: {error_reason}\n")
|
||||
if stderr_text and stderr_text.strip():
|
||||
worker.append_stdout(
|
||||
@@ -1169,7 +1166,7 @@ class DownloadModal(ModalScreen):
|
||||
url.endswith(".pdf") or "pdf" in url.lower() for url in url
|
||||
)
|
||||
if all_pdfs:
|
||||
logger.info(f"All url are PDFs - creating pseudo-playlist")
|
||||
logger.info("All url are PDFs - creating pseudo-playlist")
|
||||
self._handle_pdf_playlist(url)
|
||||
return
|
||||
|
||||
@@ -1646,7 +1643,7 @@ class DownloadModal(ModalScreen):
|
||||
break
|
||||
|
||||
if not json_line:
|
||||
logger.error(f"No JSON found in get-tag output")
|
||||
logger.error("No JSON found in get-tag output")
|
||||
logger.debug(f"Raw output: {output}")
|
||||
try:
|
||||
self.app.call_from_thread(
|
||||
|
||||
@@ -3,20 +3,16 @@
|
||||
from textual.app import ComposeResult
|
||||
from textual.screen import ModalScreen
|
||||
from textual.containers import Container, Horizontal, Vertical
|
||||
from textual.widgets import Static, Button, Input, TextArea, Tree, Select
|
||||
from textual.widgets import Static, Button, Input, TextArea, Select
|
||||
from textual.binding import Binding
|
||||
import logging
|
||||
from typing import Optional, Any
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
import json
|
||||
import sys
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
from SYS.utils import format_metadata_value
|
||||
from SYS.config import load_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -147,7 +143,7 @@ class ExportModal(ModalScreen):
|
||||
|
||||
if not metadata:
|
||||
logger.info(
|
||||
f"_get_metadata_text - No metadata found, returning 'No metadata available'"
|
||||
"_get_metadata_text - No metadata found, returning 'No metadata available'"
|
||||
)
|
||||
return "No metadata available"
|
||||
|
||||
@@ -184,7 +180,7 @@ class ExportModal(ModalScreen):
|
||||
)
|
||||
return "\n".join(lines)
|
||||
else:
|
||||
logger.info(f"_get_metadata_text - No matching fields found in metadata")
|
||||
logger.info("_get_metadata_text - No matching fields found in metadata")
|
||||
return "No metadata available"
|
||||
|
||||
def compose(self) -> ComposeResult:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
from textual.app import ComposeResult
|
||||
from textual.screen import ModalScreen
|
||||
from textual.containers import Container, Horizontal, Vertical
|
||||
from textual.containers import Horizontal, Vertical
|
||||
from textual.widgets import Static, Button, Input, Select, DataTable, TextArea
|
||||
from textual.binding import Binding
|
||||
from textual.message import Message
|
||||
@@ -363,7 +363,7 @@ class SearchModal(ModalScreen):
|
||||
tags_text = "\n".join(tags)
|
||||
|
||||
self.tags_textarea.text = tags_text
|
||||
logger.info(f"[search-modal] Populated tags textarea from result")
|
||||
logger.info("[search-modal] Populated tags textarea from result")
|
||||
|
||||
async def _download_book(self, result: Any) -> None:
|
||||
"""Download a book from OpenLibrary using the provider."""
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
from textual.app import ComposeResult
|
||||
from textual.screen import ModalScreen
|
||||
from textual.containers import Container, ScrollableContainer
|
||||
from textual.widgets import Static, Button, Label
|
||||
from typing import List, Callable
|
||||
from textual.widgets import Static, Button
|
||||
from typing import List
|
||||
|
||||
class SelectionModal(ModalScreen[str]):
|
||||
"""A modal for selecting a type from a list of strings."""
|
||||
|
||||
@@ -238,7 +238,7 @@ class WorkersModal(ModalScreen):
|
||||
"---",
|
||||
"No workers running"
|
||||
)
|
||||
logger.debug(f"[workers-modal] No running workers to display")
|
||||
logger.debug("[workers-modal] No running workers to display")
|
||||
return
|
||||
|
||||
logger.debug(
|
||||
@@ -319,7 +319,7 @@ class WorkersModal(ModalScreen):
|
||||
"---",
|
||||
"No finished workers"
|
||||
)
|
||||
logger.debug(f"[workers-modal] No finished workers to display")
|
||||
logger.debug("[workers-modal] No finished workers to display")
|
||||
return
|
||||
|
||||
logger.info(
|
||||
@@ -399,7 +399,7 @@ class WorkersModal(ModalScreen):
|
||||
workers_list = None
|
||||
if event.control == self.running_table:
|
||||
workers_list = self.running_workers
|
||||
logger.debug(f"[workers-modal] Highlighted in running table")
|
||||
logger.debug("[workers-modal] Highlighted in running table")
|
||||
elif event.control == self.finished_table:
|
||||
workers_list = self.finished_workers
|
||||
logger.debug(
|
||||
@@ -442,7 +442,7 @@ class WorkersModal(ModalScreen):
|
||||
workers_list = None
|
||||
if event.data_table == self.running_table:
|
||||
workers_list = self.running_workers
|
||||
logger.debug(f"[workers-modal] Cell highlighted in running table")
|
||||
logger.debug("[workers-modal] Cell highlighted in running table")
|
||||
elif event.data_table == self.finished_table:
|
||||
workers_list = self.finished_workers
|
||||
logger.debug(
|
||||
@@ -502,7 +502,7 @@ class WorkersModal(ModalScreen):
|
||||
self.stdout_display.cursor_location = (len(combined_text) - 1, 0)
|
||||
except Exception:
|
||||
pass
|
||||
logger.info(f"[workers-modal] Updated stdout display successfully")
|
||||
logger.info("[workers-modal] Updated stdout display successfully")
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"[workers-modal] Error updating stdout display: {e}",
|
||||
|
||||
@@ -22,13 +22,9 @@ for path in (ROOT_DIR, BASE_DIR):
|
||||
sys.path.insert(0, str_path)
|
||||
|
||||
from SYS import pipeline as ctx
|
||||
# Lazily import CLI dependencies to avoid import-time failures in test environments
|
||||
try:
|
||||
from CLI import ConfigLoader, PipelineExecutor as CLIPipelineExecutor, WorkerManagerRegistry
|
||||
except Exception:
|
||||
ConfigLoader = None
|
||||
CLIPipelineExecutor = None
|
||||
WorkerManagerRegistry = None
|
||||
from CLI import ConfigLoader
|
||||
from SYS.pipeline import PipelineExecutor
|
||||
from SYS.worker import WorkerManagerRegistry
|
||||
from SYS.logger import set_debug
|
||||
from SYS.rich_display import capture_rich_output
|
||||
from SYS.result_table import Table
|
||||
@@ -89,7 +85,7 @@ class PipelineRunner:
|
||||
if executor is not None:
|
||||
self._executor = executor
|
||||
else:
|
||||
self._executor = CLIPipelineExecutor(config_loader=self._config_loader) if CLIPipelineExecutor else None
|
||||
self._executor = PipelineExecutor(config_loader=self._config_loader)
|
||||
self._worker_manager = None
|
||||
|
||||
@property
|
||||
|
||||
+77
-9
@@ -499,6 +499,9 @@ class Add_File(Cmdlet):
|
||||
pending_url_associations: Dict[str,
|
||||
List[tuple[str,
|
||||
List[str]]]] = {}
|
||||
pending_tag_associations: Dict[str,
|
||||
List[tuple[str,
|
||||
List[str]]]] = {}
|
||||
successes = 0
|
||||
failures = 0
|
||||
|
||||
@@ -612,6 +615,8 @@ class Add_File(Cmdlet):
|
||||
collect_relationship_pairs=pending_relationship_pairs,
|
||||
defer_url_association=defer_url_association,
|
||||
pending_url_associations=pending_url_associations,
|
||||
defer_tag_association=defer_url_association,
|
||||
pending_tag_associations=pending_tag_associations,
|
||||
suppress_last_stage_overlay=want_final_search_file,
|
||||
auto_search_file=auto_search_file_after_add,
|
||||
store_instance=storage_registry,
|
||||
@@ -664,6 +669,17 @@ class Add_File(Cmdlet):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Apply deferred tag associations (bulk) if collected
|
||||
if pending_tag_associations:
|
||||
try:
|
||||
Add_File._apply_pending_tag_associations(
|
||||
pending_tag_associations,
|
||||
config,
|
||||
store_instance=storage_registry
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Always end add-file -store (when last stage) by showing item detail panels.
|
||||
# Legacy search-file refresh is no longer used for final display.
|
||||
if want_final_search_file and collected_payloads:
|
||||
@@ -1854,6 +1870,10 @@ class Add_File(Cmdlet):
|
||||
pending_url_associations: Optional[Dict[str,
|
||||
List[tuple[str,
|
||||
List[str]]]]] = None,
|
||||
defer_tag_association: bool = False,
|
||||
pending_tag_associations: Optional[Dict[str,
|
||||
List[tuple[str,
|
||||
List[str]]]]] = None,
|
||||
suppress_last_stage_overlay: bool = False,
|
||||
auto_search_file: bool = True,
|
||||
store_instance: Optional[Store] = None,
|
||||
@@ -2072,15 +2092,22 @@ class Add_File(Cmdlet):
|
||||
resolved_hash = chosen_hash
|
||||
|
||||
if hydrus_like_backend and tags:
|
||||
try:
|
||||
adder = getattr(backend, "add_tag", None)
|
||||
if callable(adder):
|
||||
debug(
|
||||
f"[add-file] Applying {len(tags)} tag(s) post-upload to Hydrus"
|
||||
)
|
||||
adder(resolved_hash, list(tags))
|
||||
except Exception as exc:
|
||||
log(f"[add-file] Hydrus post-upload tagging failed: {exc}", file=sys.stderr)
|
||||
# Support deferring tag application for batching bulk operations
|
||||
if defer_tag_association and pending_tag_associations is not None:
|
||||
try:
|
||||
pending_tag_associations.setdefault(str(backend_name), []).append((str(resolved_hash), list(tags)))
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
adder = getattr(backend, "add_tag", None)
|
||||
if callable(adder):
|
||||
debug(
|
||||
f"[add-file] Applying {len(tags)} tag(s) post-upload to Hydrus"
|
||||
)
|
||||
adder(resolved_hash, list(tags))
|
||||
except Exception as exc:
|
||||
log(f"[add-file] Hydrus post-upload tagging failed: {exc}", file=sys.stderr)
|
||||
|
||||
# If we have url(s), ensure they get associated with the destination file.
|
||||
# This mirrors `add-url` behavior but avoids emitting extra pipeline noise.
|
||||
@@ -2322,6 +2349,47 @@ class Add_File(Cmdlet):
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
@staticmethod
|
||||
def _apply_pending_tag_associations(
|
||||
pending: Dict[str,
|
||||
List[tuple[str,
|
||||
List[str]]]],
|
||||
config: Dict[str,
|
||||
Any],
|
||||
store_instance: Optional[Store] = None,
|
||||
) -> None:
|
||||
"""Apply deferred tag associations in bulk, grouped per backend."""
|
||||
|
||||
try:
|
||||
store = store_instance if store_instance is not None else Store(config)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
for backend_name, pairs in (pending or {}).items():
|
||||
if not pairs:
|
||||
continue
|
||||
try:
|
||||
backend = store[backend_name]
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Try bulk variant first
|
||||
bulk = getattr(backend, "add_tags_bulk", None)
|
||||
if callable(bulk):
|
||||
try:
|
||||
bulk([(h, t) for h, t in pairs])
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
single = getattr(backend, "add_tag", None)
|
||||
if callable(single):
|
||||
for h, t in pairs:
|
||||
try:
|
||||
single(h, t)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
@staticmethod
|
||||
def _load_sidecar_bundle(
|
||||
media_path: Path,
|
||||
|
||||
@@ -1097,7 +1097,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
]
|
||||
|
||||
if not relationship_tags:
|
||||
log(f"No relationship tags found in sidecar", file=sys.stderr)
|
||||
log("No relationship tags found in sidecar", file=sys.stderr)
|
||||
return 0 # Not an error, just nothing to do
|
||||
|
||||
# Get the file hash from result (should have been set by add-file)
|
||||
@@ -1166,7 +1166,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
)
|
||||
return 0
|
||||
elif error_count == 0:
|
||||
log(f"No relationships to set", file=sys.stderr)
|
||||
log("No relationships to set", file=sys.stderr)
|
||||
return 0 # Success with nothing to do
|
||||
else:
|
||||
log(f"Failed with {error_count} error(s)", file=sys.stderr)
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
||||
from typing import Any, Dict, List, Sequence, Tuple
|
||||
import sys
|
||||
|
||||
from SYS import pipeline as ctx
|
||||
|
||||
@@ -7,7 +7,6 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
from SYS.logger import debug, log
|
||||
from SYS.utils import format_bytes
|
||||
from Store.Folder import Folder
|
||||
from Store import Store
|
||||
from . import _shared as sh
|
||||
|
||||
@@ -2,10 +2,8 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, Sequence
|
||||
from pathlib import Path
|
||||
import json
|
||||
import sys
|
||||
|
||||
from SYS import models
|
||||
from SYS import pipeline as ctx
|
||||
from . import _shared as sh
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
||||
from typing import Any, Dict, List, Sequence, Tuple
|
||||
import sys
|
||||
|
||||
from SYS import pipeline as ctx
|
||||
|
||||
@@ -15,7 +15,6 @@ from typing import Any, Dict, List, Optional, Sequence
|
||||
from urllib.parse import urlparse
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
|
||||
import requests
|
||||
|
||||
from API.HTTP import _download_direct_file
|
||||
from SYS.models import DownloadError, DownloadOptions, DownloadMediaResult
|
||||
@@ -26,7 +25,6 @@ from SYS.rich_display import stderr_console as get_stderr_console
|
||||
from SYS import pipeline as pipeline_context
|
||||
from SYS.utils import sha256_file
|
||||
from SYS.metadata import normalize_urls as normalize_url_list
|
||||
from rich.prompt import Confirm
|
||||
|
||||
from tool.ytdlp import (
|
||||
YtDlpTool,
|
||||
@@ -948,7 +946,7 @@ class Download_File(Cmdlet):
|
||||
from Store import Store
|
||||
from API.HydrusNetwork import is_hydrus_available
|
||||
|
||||
debug(f"[download-file] Initializing storage interface...")
|
||||
debug("[download-file] Initializing storage interface...")
|
||||
storage = Store(config=config or {}, suppress_debug=True)
|
||||
hydrus_available = bool(is_hydrus_available(config or {}))
|
||||
|
||||
@@ -1338,7 +1336,7 @@ class Download_File(Cmdlet):
|
||||
table.set_source_command("download-file", [url])
|
||||
|
||||
debug(f"[ytdlp.formatlist] Displaying format selection table for {url}")
|
||||
debug(f"[ytdlp.formatlist] Provider: ytdlp (routing to download-file via TABLE_AUTO_STAGES)")
|
||||
debug("[ytdlp.formatlist] Provider: ytdlp (routing to download-file via TABLE_AUTO_STAGES)")
|
||||
|
||||
results_list: List[Dict[str, Any]] = []
|
||||
for idx, fmt in enumerate(filtered_formats, 1):
|
||||
@@ -1420,7 +1418,7 @@ class Download_File(Cmdlet):
|
||||
f"[ytdlp.formatlist] When user selects @N, will invoke: download-file {url} -query 'format:<format_id>'"
|
||||
)
|
||||
|
||||
log(f"", file=sys.stderr)
|
||||
log("", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
return None
|
||||
@@ -2054,7 +2052,7 @@ class Download_File(Cmdlet):
|
||||
forced_single_format_id = None
|
||||
forced_single_format_for_batch = False
|
||||
|
||||
debug(f"[download-file] Checking if format table should be shown...")
|
||||
debug("[download-file] Checking if format table should be shown...")
|
||||
early_ret = self._maybe_show_format_table_for_single_url(
|
||||
mode=mode,
|
||||
clip_spec=clip_spec,
|
||||
@@ -2763,7 +2761,7 @@ class Download_File(Cmdlet):
|
||||
debug(f"[download-file] Processing {total_selection} selected item(s) from table...")
|
||||
for idx, run_args in enumerate(selection_runs, 1):
|
||||
debug(f"[download-file] Item {idx}/{total_selection}: {run_args}")
|
||||
debug(f"[download-file] Re-invoking download-file for selected item...")
|
||||
debug("[download-file] Re-invoking download-file for selected item...")
|
||||
exit_code = self._run_impl(None, run_args, config)
|
||||
if exit_code == 0:
|
||||
successes += 1
|
||||
|
||||
+2
-2
@@ -92,7 +92,7 @@ class Get_File(sh.Cmdlet):
|
||||
debug(f"[get-file] Backend retrieved: {type(backend).__name__}")
|
||||
|
||||
# Get file metadata to determine name and extension
|
||||
debug(f"[get-file] Getting metadata for hash...")
|
||||
debug("[get-file] Getting metadata for hash...")
|
||||
metadata = backend.get_metadata(file_hash)
|
||||
if not metadata:
|
||||
log(f"Error: File metadata not found for hash {file_hash}")
|
||||
@@ -228,7 +228,7 @@ class Get_File(sh.Cmdlet):
|
||||
}
|
||||
)
|
||||
|
||||
debug(f"[get-file] Completed successfully")
|
||||
debug("[get-file] Completed successfully")
|
||||
return 0
|
||||
|
||||
def _open_file_default(self, path: Path) -> None:
|
||||
|
||||
@@ -5,7 +5,6 @@ import json
|
||||
import sys
|
||||
|
||||
from SYS.logger import log
|
||||
from pathlib import Path
|
||||
|
||||
from . import _shared as sh
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ import sys
|
||||
from SYS.logger import log
|
||||
|
||||
from SYS import pipeline as ctx
|
||||
from SYS.result_table import Table
|
||||
from . import _shared as sh
|
||||
|
||||
Cmdlet = sh.Cmdlet
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, Sequence, List, Optional
|
||||
import json
|
||||
from typing import Any, Dict, Sequence, Optional
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from SYS.logger import log
|
||||
|
||||
from SYS import models
|
||||
from SYS import pipeline as ctx
|
||||
from API import HydrusNetwork as hydrus_wrapper
|
||||
from . import _shared as sh
|
||||
@@ -22,8 +20,6 @@ fetch_hydrus_metadata = sh.fetch_hydrus_metadata
|
||||
should_show_help = sh.should_show_help
|
||||
get_field = sh.get_field
|
||||
from API.folder import API_folder_store
|
||||
from SYS.config import get_local_storage_path
|
||||
from SYS.result_table import Table
|
||||
from Store import Store
|
||||
|
||||
CMDLET = Cmdlet(
|
||||
@@ -512,7 +508,7 @@ def _run(result: Any, _args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
if source_title and source_title != "Unknown":
|
||||
metadata["Title"] = source_title
|
||||
|
||||
table = ItemDetailView(f"Relationships", item_metadata=metadata
|
||||
table = ItemDetailView("Relationships", item_metadata=metadata
|
||||
).init_command("get-relationship",
|
||||
[])
|
||||
|
||||
|
||||
+2
-4
@@ -25,8 +25,7 @@ from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
from SYS import pipeline as ctx
|
||||
from API import HydrusNetwork
|
||||
from API.folder import read_sidecar, write_sidecar, find_sidecar, API_folder_store
|
||||
from API.folder import read_sidecar, write_sidecar
|
||||
from . import _shared as sh
|
||||
|
||||
normalize_hash = sh.normalize_hash
|
||||
@@ -36,7 +35,6 @@ CmdletArg = sh.CmdletArg
|
||||
SharedArgs = sh.SharedArgs
|
||||
parse_cmdlet_args = sh.parse_cmdlet_args
|
||||
get_field = sh.get_field
|
||||
from SYS.config import get_local_storage_path
|
||||
|
||||
try:
|
||||
from SYS.metadata import extract_title
|
||||
@@ -944,7 +942,7 @@ def _scrape_url_metadata(
|
||||
)
|
||||
except json_module.JSONDecodeError:
|
||||
pass
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
pass # Silently ignore if we can't get playlist entries
|
||||
|
||||
# Fallback: if still no tags detected, get from first item
|
||||
|
||||
+21
-21
@@ -320,7 +320,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
f"Mixed file types detected: {', '.join(sorted(file_types))}",
|
||||
file=sys.stderr
|
||||
)
|
||||
log(f"Can only merge files of the same type", file=sys.stderr)
|
||||
log("Can only merge files of the same type", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
file_kind = list(file_types)[0] if file_types else "other"
|
||||
@@ -524,7 +524,7 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool:
|
||||
current_time_ms = 0
|
||||
|
||||
log(f"Analyzing {len(files)} files for chapter information...", file=sys.stderr)
|
||||
logger.info(f"[merge-file] Analyzing files for chapters")
|
||||
logger.info("[merge-file] Analyzing files for chapters")
|
||||
|
||||
for file_path in files:
|
||||
# Get duration using ffprobe
|
||||
@@ -767,14 +767,14 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool:
|
||||
logger.exception(f"[merge-file] ffmpeg process error: {e}")
|
||||
raise
|
||||
|
||||
log(f"Merge successful, adding chapters metadata...", file=sys.stderr)
|
||||
log("Merge successful, adding chapters metadata...", file=sys.stderr)
|
||||
|
||||
# Step 5: Embed chapters into container (MKA, MP4/M4A, or note limitation)
|
||||
if output_format == "mka" or output.suffix.lower() == ".mka":
|
||||
# MKA/MKV format has native chapter support via FFMetadata
|
||||
# Re-mux the file with chapters embedded (copy streams, no re-encode)
|
||||
log(f"Embedding chapters into Matroska container...", file=sys.stderr)
|
||||
logger.info(f"[merge-file] Adding chapters to MKA file via FFMetadata")
|
||||
log("Embedding chapters into Matroska container...", file=sys.stderr)
|
||||
logger.info("[merge-file] Adding chapters to MKA file via FFMetadata")
|
||||
|
||||
temp_output = output.parent / f".temp_{output.stem}.mka"
|
||||
|
||||
@@ -783,7 +783,7 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool:
|
||||
|
||||
if mkvmerge_path:
|
||||
# mkvmerge is the best tool for embedding chapters in Matroska files
|
||||
log(f"Using mkvmerge for optimal chapter embedding...", file=sys.stderr)
|
||||
log("Using mkvmerge for optimal chapter embedding...", file=sys.stderr)
|
||||
cmd2 = [
|
||||
mkvmerge_path,
|
||||
"-o",
|
||||
@@ -795,7 +795,7 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool:
|
||||
else:
|
||||
# Fallback to ffmpeg with proper chapter embedding for Matroska
|
||||
log(
|
||||
f"Using ffmpeg for chapter embedding (install mkvtoolnix for better quality)...",
|
||||
"Using ffmpeg for chapter embedding (install mkvtoolnix for better quality)...",
|
||||
file=sys.stderr,
|
||||
)
|
||||
# For Matroska files, the metadata must be provided via -f ffmetadata input
|
||||
@@ -838,12 +838,12 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool:
|
||||
if output.exists():
|
||||
output.unlink()
|
||||
shutil.move(str(temp_output), str(output))
|
||||
log(f"✓ Chapters successfully embedded!", file=sys.stderr)
|
||||
logger.info(f"[merge-file] Chapters embedded successfully")
|
||||
log("✓ Chapters successfully embedded!", file=sys.stderr)
|
||||
logger.info("[merge-file] Chapters embedded successfully")
|
||||
except Exception as e:
|
||||
logger.warning(f"[merge-file] Could not replace file: {e}")
|
||||
log(
|
||||
f"Warning: Could not embed chapters, using merge without chapters",
|
||||
"Warning: Could not embed chapters, using merge without chapters",
|
||||
file=sys.stderr,
|
||||
)
|
||||
try:
|
||||
@@ -852,12 +852,12 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool:
|
||||
pass
|
||||
else:
|
||||
logger.warning(
|
||||
f"[merge-file] Chapter embedding did not create output"
|
||||
"[merge-file] Chapter embedding did not create output"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception(f"[merge-file] Chapter embedding failed: {e}")
|
||||
log(
|
||||
f"Warning: Chapter embedding failed, using merge without chapters",
|
||||
"Warning: Chapter embedding failed, using merge without chapters",
|
||||
file=sys.stderr,
|
||||
)
|
||||
elif output_format in {"m4a",
|
||||
@@ -865,15 +865,15 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool:
|
||||
".m4b",
|
||||
".mp4"]:
|
||||
# MP4/M4A format has native chapter support via iTunes metadata atoms
|
||||
log(f"Embedding chapters into MP4 container...", file=sys.stderr)
|
||||
log("Embedding chapters into MP4 container...", file=sys.stderr)
|
||||
logger.info(
|
||||
f"[merge-file] Adding chapters to M4A/MP4 file via iTunes metadata"
|
||||
"[merge-file] Adding chapters to M4A/MP4 file via iTunes metadata"
|
||||
)
|
||||
|
||||
temp_output = output.parent / f".temp_{output.stem}{output.suffix}"
|
||||
|
||||
# ffmpeg embeds chapters in MP4 using -map_metadata and -map_chapters
|
||||
log(f"Using ffmpeg for MP4 chapter embedding...", file=sys.stderr)
|
||||
log("Using ffmpeg for MP4 chapter embedding...", file=sys.stderr)
|
||||
cmd2 = [
|
||||
ffmpeg_path,
|
||||
"-y",
|
||||
@@ -916,14 +916,14 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool:
|
||||
output.unlink()
|
||||
shutil.move(str(temp_output), str(output))
|
||||
log(
|
||||
f"✓ Chapters successfully embedded in MP4!",
|
||||
"✓ Chapters successfully embedded in MP4!",
|
||||
file=sys.stderr
|
||||
)
|
||||
logger.info(f"[merge-file] MP4 chapters embedded successfully")
|
||||
logger.info("[merge-file] MP4 chapters embedded successfully")
|
||||
except Exception as e:
|
||||
logger.warning(f"[merge-file] Could not replace file: {e}")
|
||||
log(
|
||||
f"Warning: Could not embed chapters, using merge without chapters",
|
||||
"Warning: Could not embed chapters, using merge without chapters",
|
||||
file=sys.stderr,
|
||||
)
|
||||
try:
|
||||
@@ -932,12 +932,12 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool:
|
||||
pass
|
||||
else:
|
||||
logger.warning(
|
||||
f"[merge-file] MP4 chapter embedding did not create output"
|
||||
"[merge-file] MP4 chapter embedding did not create output"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception(f"[merge-file] MP4 chapter embedding failed: {e}")
|
||||
log(
|
||||
f"Warning: MP4 chapter embedding failed, using merge without chapters",
|
||||
"Warning: MP4 chapter embedding failed, using merge without chapters",
|
||||
file=sys.stderr,
|
||||
)
|
||||
else:
|
||||
@@ -945,7 +945,7 @@ def _merge_audio(files: List[Path], output: Path, output_format: str) -> bool:
|
||||
logger.info(
|
||||
f"[merge-file] Format {output_format} does not have native chapter support"
|
||||
)
|
||||
log(f"Note: For chapter support, use MKA or M4A format", file=sys.stderr)
|
||||
log("Note: For chapter support, use MKA or M4A format", file=sys.stderr)
|
||||
|
||||
# Clean up temp files
|
||||
try:
|
||||
|
||||
@@ -4,7 +4,7 @@ import sys
|
||||
from typing import Any, Dict, Iterable, Sequence
|
||||
|
||||
from . import _shared as sh
|
||||
from SYS.logger import log, debug
|
||||
from SYS.logger import log
|
||||
from SYS import pipeline as ctx
|
||||
|
||||
from SYS.result_table_adapters import get_provider
|
||||
@@ -43,7 +43,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
|
||||
try:
|
||||
provider = get_provider(provider_name)
|
||||
except Exception as exc:
|
||||
except Exception:
|
||||
log(f"Unknown provider: {provider_name}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
@@ -656,7 +656,7 @@ def _capture(
|
||||
# Attempt platform-specific target capture if requested (and not PDF)
|
||||
element_captured = False
|
||||
if options.prefer_platform_target and format_name != "pdf":
|
||||
debug(f"[_capture] Target capture enabled")
|
||||
debug("[_capture] Target capture enabled")
|
||||
debug("Attempting platform-specific content capture...")
|
||||
progress.step("capturing locating target")
|
||||
try:
|
||||
@@ -913,7 +913,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
url_to_process.append((str(url), item))
|
||||
|
||||
if not url_to_process:
|
||||
log(f"No url to process for screen-shot cmdlet", file=sys.stderr)
|
||||
log("No url to process for screen-shot cmdlet", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
debug(f"[_run] url to process: {[u for u, _ in url_to_process]}")
|
||||
@@ -1157,7 +1157,7 @@ def _run(result: Any, args: Sequence[str], config: Dict[str, Any]) -> int:
|
||||
progress.close_local_ui(force_complete=True)
|
||||
|
||||
if not all_emitted:
|
||||
log(f"No screenshots were successfully captured", file=sys.stderr)
|
||||
log("No screenshots were successfully captured", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Log completion message (keep this as normal output)
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, Sequence, List, Optional
|
||||
import importlib
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
+2
-2
@@ -1,8 +1,8 @@
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from typing import List, Dict, Any, Optional, Sequence
|
||||
from cmdlet._shared import Cmdlet, CmdletArg, parse_cmdlet_args
|
||||
from typing import List, Dict, Any, Sequence
|
||||
from cmdlet._shared import Cmdlet, CmdletArg
|
||||
from SYS.logger import log
|
||||
from SYS.result_table import Table
|
||||
from SYS import pipeline as ctx
|
||||
|
||||
+1
-1
@@ -213,7 +213,7 @@ def _run(piped_result: Any, args: List[str], config: Dict[str, Any]) -> int:
|
||||
# Check if we're in an interactive terminal and can launch a Textual modal
|
||||
if sys.stdin.isatty() and not piped_result:
|
||||
try:
|
||||
from textual.app import App, ComposeResult
|
||||
from textual.app import App
|
||||
from TUI.modalscreen.config_modal import ConfigModal
|
||||
|
||||
class ConfigApp(App):
|
||||
|
||||
@@ -4,7 +4,6 @@ import sys
|
||||
import json
|
||||
import socket
|
||||
import re
|
||||
import subprocess
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
from pathlib import Path
|
||||
from cmdlet._shared import Cmdlet, CmdletArg, parse_cmdlet_args, resolve_tidal_manifest_path
|
||||
|
||||
+3
-6
@@ -1,15 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import shutil
|
||||
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from cmdlet._shared import Cmdlet, CmdletArg
|
||||
from cmdlet._shared import Cmdlet
|
||||
from SYS import pipeline as ctx
|
||||
from SYS.result_table import Table
|
||||
from SYS.logger import log, set_debug, debug
|
||||
from SYS.rich_display import stdout_console
|
||||
from SYS.logger import set_debug, debug
|
||||
|
||||
CMDLET = Cmdlet(
|
||||
name=".status",
|
||||
|
||||
+2
-3
@@ -1,15 +1,14 @@
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
from typing import Any, Dict, Sequence
|
||||
|
||||
# Add project root to sys.path
|
||||
root = Path(__file__).resolve().parent.parent
|
||||
if str(root) not in sys.path:
|
||||
sys.path.insert(0, str(root))
|
||||
|
||||
from cmdlet._shared import Cmdlet, CmdletArg
|
||||
from cmdlet._shared import Cmdlet
|
||||
from SYS.config import load_config
|
||||
from SYS.result_table import Table
|
||||
from API import zerotier as zt
|
||||
|
||||
+23
-24
@@ -203,7 +203,6 @@ def run_platform_bootstrap(repo_root: Path) -> int:
|
||||
|
||||
def playwright_package_installed() -> bool:
|
||||
try:
|
||||
import playwright # type: ignore
|
||||
|
||||
return True
|
||||
except Exception:
|
||||
@@ -751,7 +750,7 @@ def main() -> int:
|
||||
user_bin = Path(os.environ.get("USERPROFILE", str(home))) / "bin"
|
||||
mm_bat = user_bin / "mm.bat"
|
||||
|
||||
print(f"Checking for shim files:")
|
||||
print("Checking for shim files:")
|
||||
print(f" mm.bat: {'✓' if mm_bat.exists() else '✗'} ({mm_bat})")
|
||||
print()
|
||||
|
||||
@@ -760,14 +759,14 @@ def main() -> int:
|
||||
if "REPO=" in bat_content or "ENTRY=" in bat_content:
|
||||
print(f" mm.bat content looks valid ({len(bat_content)} bytes)")
|
||||
else:
|
||||
print(f" ⚠️ mm.bat content may be corrupted")
|
||||
print(" ⚠️ mm.bat content may be corrupted")
|
||||
print()
|
||||
|
||||
# Check PATH
|
||||
path = os.environ.get("PATH", "")
|
||||
user_bin_str = str(user_bin)
|
||||
in_path = user_bin_str in path
|
||||
print(f"Checking PATH environment variable:")
|
||||
print("Checking PATH environment variable:")
|
||||
print(f" {user_bin_str} in current session PATH: {'✓' if in_path else '✗'}")
|
||||
|
||||
# Check registry
|
||||
@@ -792,7 +791,7 @@ def main() -> int:
|
||||
try:
|
||||
result = subprocess.run(["mm", "--help"], capture_output=True, text=True, timeout=5)
|
||||
if result.returncode == 0:
|
||||
print(f" ✓ 'mm --help' works!")
|
||||
print(" ✓ 'mm --help' works!")
|
||||
print(f" Output (first line): {result.stdout.split(chr(10))[0]}")
|
||||
else:
|
||||
print(f" ✗ 'mm --help' failed with exit code {result.returncode}")
|
||||
@@ -800,8 +799,8 @@ def main() -> int:
|
||||
print(f" Error: {result.stderr.strip()}")
|
||||
except FileNotFoundError:
|
||||
# mm not found via PATH, try calling the .ps1 directly
|
||||
print(f" ✗ 'mm' command not found in PATH")
|
||||
print(f" Shims exist but command is not accessible via PATH")
|
||||
print(" ✗ 'mm' command not found in PATH")
|
||||
print(" Shims exist but command is not accessible via PATH")
|
||||
print()
|
||||
print("Attempting to call shim directly...")
|
||||
try:
|
||||
@@ -810,23 +809,23 @@ def main() -> int:
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
if result.returncode == 0:
|
||||
print(f" ✓ Direct shim call works!")
|
||||
print(f" The shim files are valid and functional.")
|
||||
print(" ✓ Direct shim call works!")
|
||||
print(" The shim files are valid and functional.")
|
||||
print()
|
||||
print("⚠️ 'mm' is not in PATH, but the shims are working correctly.")
|
||||
print()
|
||||
print("Possible causes and fixes:")
|
||||
print(f" 1. Terminal needs restart: Close and reopen your terminal/PowerShell")
|
||||
print(f" 2. PATH reload: Run: $env:Path = [Environment]::GetEnvironmentVariable('PATH', 'User') + ';' + [Environment]::GetEnvironmentVariable('PATH', 'Machine')")
|
||||
print(" 1. Terminal needs restart: Close and reopen your terminal/PowerShell")
|
||||
print(" 2. PATH reload: Run: $env:Path = [Environment]::GetEnvironmentVariable('PATH', 'User') + ';' + [Environment]::GetEnvironmentVariable('PATH', 'Machine')")
|
||||
print(f" 3. Manual PATH: Add {user_bin} to your system PATH manually")
|
||||
else:
|
||||
print(f" ✗ Direct shim call failed")
|
||||
print(" ✗ Direct shim call failed")
|
||||
if result.stderr:
|
||||
print(f" Error: {result.stderr.strip()}")
|
||||
except Exception as e:
|
||||
print(f" ✗ Could not test direct shim: {e}")
|
||||
except subprocess.TimeoutExpired:
|
||||
print(f" ✗ 'mm' command timed out")
|
||||
print(" ✗ 'mm' command timed out")
|
||||
except Exception as e:
|
||||
print(f" ✗ Error testing 'mm': {e}")
|
||||
else:
|
||||
@@ -835,7 +834,7 @@ def main() -> int:
|
||||
locations = [home / ".local" / "bin" / "mm", Path("/usr/local/bin/mm"), Path("/usr/bin/mm")]
|
||||
found_shims = [p for p in locations if p.exists()]
|
||||
|
||||
print(f"Checking for shim files:")
|
||||
print("Checking for shim files:")
|
||||
for p in locations:
|
||||
if p.exists():
|
||||
print(f" mm: ✓ ({p})")
|
||||
@@ -844,23 +843,23 @@ def main() -> int:
|
||||
print(f" mm: ✗ ({p})")
|
||||
|
||||
if not found_shims:
|
||||
print(f" mm: ✗ (No shim found in standard locations)")
|
||||
print(" mm: ✗ (No shim found in standard locations)")
|
||||
print()
|
||||
|
||||
path = os.environ.get("PATH", "")
|
||||
|
||||
# Find which 'mm' is actually being run
|
||||
actual_mm = shutil.which("mm")
|
||||
print(f"Checking PATH environment variable:")
|
||||
print("Checking PATH environment variable:")
|
||||
if actual_mm:
|
||||
print(f" 'mm' resolved to: {actual_mm}")
|
||||
# Check if it's in a directory on the PATH
|
||||
if any(str(Path(actual_mm).parent) in p for p in path.split(os.pathsep)):
|
||||
print(f" Command is accessible via current session PATH: ✓")
|
||||
print(" Command is accessible via current session PATH: ✓")
|
||||
else:
|
||||
print(f" Command is found but directory may not be in current PATH: ⚠️")
|
||||
print(" Command is found but directory may not be in current PATH: ⚠️")
|
||||
else:
|
||||
print(f" 'mm' not found in current session PATH: ✗")
|
||||
print(" 'mm' not found in current session PATH: ✗")
|
||||
print()
|
||||
|
||||
# Test if mm command works
|
||||
@@ -868,14 +867,14 @@ def main() -> int:
|
||||
try:
|
||||
result = subprocess.run(["mm", "--help"], capture_output=True, text=True, timeout=5)
|
||||
if result.returncode == 0:
|
||||
print(f" ✓ 'mm --help' works!")
|
||||
print(" ✓ 'mm --help' works!")
|
||||
print(f" Output (first line): {result.stdout.split(chr(10))[0]}")
|
||||
else:
|
||||
print(f" ✗ 'mm --help' failed with exit code {result.returncode}")
|
||||
if result.stderr:
|
||||
print(f" Error: {result.stderr.strip()}")
|
||||
except FileNotFoundError:
|
||||
print(f" ✗ 'mm' command not found in PATH")
|
||||
print(" ✗ 'mm' command not found in PATH")
|
||||
except Exception as e:
|
||||
print(f" ✗ Error testing 'mm': {e}")
|
||||
|
||||
@@ -1002,7 +1001,7 @@ def main() -> int:
|
||||
|
||||
try:
|
||||
_run_cmd([str(python_path), "-m", "ensurepip", "--upgrade"])
|
||||
except subprocess.CalledProcessError as exc:
|
||||
except subprocess.CalledProcessError:
|
||||
print(
|
||||
"Failed to install pip inside the local virtualenv via ensurepip; ensure your Python build includes ensurepip and retry.",
|
||||
file=sys.stderr,
|
||||
@@ -1326,10 +1325,10 @@ if (Test-Path (Join-Path $repo 'CLI.py')) {
|
||||
|
||||
if not args.quiet:
|
||||
print(f"Installed global launcher to: {user_bin}")
|
||||
print(f"✓ mm.bat (Command Prompt and PowerShell)")
|
||||
print("✓ mm.bat (Command Prompt and PowerShell)")
|
||||
print()
|
||||
print("You can now run 'mm' from any terminal window.")
|
||||
print(f"If 'mm' is not found, restart your terminal or reload PATH:")
|
||||
print("If 'mm' is not found, restart your terminal or reload PATH:")
|
||||
print(" PowerShell: $env:PATH = [Environment]::GetEnvironmentVariable('PATH','User') + ';' + [Environment]::GetEnvironmentVariable('PATH','Machine')")
|
||||
print(" CMD: path %PATH%")
|
||||
|
||||
|
||||
@@ -5,6 +5,6 @@ import traceback
|
||||
try:
|
||||
importlib.import_module("CLI")
|
||||
print("CLI imported OK")
|
||||
except Exception as e:
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import re
|
||||
from pathlib import Path
|
||||
p = Path(r'c:\Forgejo\Medios-Macina\CLI.py')
|
||||
s = p.read_text(encoding='utf-8')
|
||||
pattern = re.compile(r'(?s)if False:\s*class _OldPipelineExecutor:.*?from rich\\.markdown import Markdown\\s*')
|
||||
m = pattern.search(s)
|
||||
print('found', bool(m))
|
||||
if m:
|
||||
print('start', m.start(), 'end', m.end())
|
||||
print('snippet:', s[m.start():m.start()+120])
|
||||
else:
|
||||
# print a slice around the if False for debugging
|
||||
i = s.find('if False:')
|
||||
print('if False index', i)
|
||||
print('around if False:', s[max(0,i-50):i+200])
|
||||
j = s.find('from rich.markdown import Markdown', i)
|
||||
print('next from rich index after if False', j)
|
||||
if j!=-1:
|
||||
print('around that:', s[j-50:j+80])
|
||||
@@ -0,0 +1,35 @@
|
||||
from pathlib import Path
|
||||
p=Path('SYS/pipeline.py')
|
||||
s=p.read_text(encoding='utf-8')
|
||||
lines=s.splitlines()
|
||||
stack=[]
|
||||
for i,l in enumerate(lines,1):
|
||||
stripped=l.strip()
|
||||
# Skip commented lines
|
||||
if stripped.startswith('#'):
|
||||
continue
|
||||
# compute indent as leading spaces (tabs are converted)
|
||||
indent = len(l) - len(l.lstrip(' '))
|
||||
if stripped.startswith('try:'):
|
||||
stack.append((indent, i))
|
||||
if stripped.startswith('except ') or stripped=='except:' or stripped.startswith('finally:'):
|
||||
# find the most recent try with same indent
|
||||
for idx in range(len(stack)-1, -1, -1):
|
||||
if stack[idx][0] == indent:
|
||||
stack.pop(idx)
|
||||
break
|
||||
else:
|
||||
# no matching try at same indent
|
||||
print(f"Found {stripped.split()[0]} at line {i} with no matching try at same indent")
|
||||
|
||||
print('Unmatched try count', len(stack))
|
||||
if stack:
|
||||
print('Unmatched try positions (indent, line):', stack)
|
||||
for indent, lineno in stack:
|
||||
start = max(1, lineno - 10)
|
||||
end = min(len(lines), lineno + 10)
|
||||
print(f"Context around line {lineno}:")
|
||||
for i in range(start, end + 1):
|
||||
print(f"{i:5d}: {lines[i-1]}")
|
||||
else:
|
||||
print("All try statements appear matched")
|
||||
@@ -1,4 +1,5 @@
|
||||
import importlib, traceback
|
||||
import importlib
|
||||
import traceback
|
||||
|
||||
try:
|
||||
m = importlib.import_module('Provider.vimm')
|
||||
|
||||
@@ -28,7 +28,6 @@ import sys
|
||||
import tempfile
|
||||
import urllib.request
|
||||
import zipfile
|
||||
import shlex
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Optional, Tuple
|
||||
@@ -870,7 +869,7 @@ def main(argv: Optional[list[str]] = None) -> int:
|
||||
args.root = str(default_root)
|
||||
|
||||
# Ask for destination folder name
|
||||
dest_input = input(f"Enter folder name for Hydrus [default: hydrusnetwork]: ").strip()
|
||||
dest_input = input("Enter folder name for Hydrus [default: hydrusnetwork]: ").strip()
|
||||
if dest_input:
|
||||
args.dest_name = dest_input
|
||||
except (EOFError, KeyboardInterrupt):
|
||||
|
||||
@@ -41,7 +41,6 @@ from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import argparse
|
||||
import logging
|
||||
import threading
|
||||
@@ -54,7 +53,6 @@ from functools import wraps
|
||||
# Add parent directory to path for imports
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from SYS.logger import log
|
||||
|
||||
# ============================================================================
|
||||
# CONFIGURATION
|
||||
@@ -419,29 +417,32 @@ def create_app():
|
||||
|
||||
filename = sanitize_filename(file_storage.filename or "upload")
|
||||
incoming_dir = STORAGE_PATH / "incoming"
|
||||
ensure_directory(incoming_dir)
|
||||
target_path = incoming_dir / filename
|
||||
target_path = unique_path(target_path)
|
||||
|
||||
try:
|
||||
# Save uploaded file to storage
|
||||
file_storage.save(str(target_path))
|
||||
|
||||
# Extract optional metadata
|
||||
tags = []
|
||||
if 'tag' in request.form:
|
||||
# Support repeated form fields or comma-separated list
|
||||
tags = request.form.getlist('tag') or []
|
||||
if not tags and request.form.get('tag'):
|
||||
tags = [t.strip() for t in str(request.form.get('tag') or "").split(",") if t.strip()]
|
||||
|
||||
urls = []
|
||||
if 'url' in request.form:
|
||||
urls = request.form.getlist('url') or []
|
||||
if not urls and request.form.get('url'):
|
||||
urls = [u.strip() for u in str(request.form.get('url') or "").split(",") if u.strip()]
|
||||
|
||||
# Initialize the DB first (run safety checks) before creating any files.
|
||||
with API_folder_store(STORAGE_PATH) as db:
|
||||
# Ensure the incoming directory exists only after DB safety checks pass.
|
||||
ensure_directory(incoming_dir)
|
||||
|
||||
# Save uploaded file to storage
|
||||
file_storage.save(str(target_path))
|
||||
|
||||
# Extract optional metadata
|
||||
tags = []
|
||||
if 'tag' in request.form:
|
||||
# Support repeated form fields or comma-separated list
|
||||
tags = request.form.getlist('tag') or []
|
||||
if not tags and request.form.get('tag'):
|
||||
tags = [t.strip() for t in str(request.form.get('tag') or "").split(",") if t.strip()]
|
||||
|
||||
urls = []
|
||||
if 'url' in request.form:
|
||||
urls = request.form.getlist('url') or []
|
||||
if not urls and request.form.get('url'):
|
||||
urls = [u.strip() for u in str(request.form.get('url') or "").split(",") if u.strip()]
|
||||
|
||||
db.get_or_create_file_entry(target_path)
|
||||
|
||||
if tags:
|
||||
@@ -723,7 +724,7 @@ def main():
|
||||
local_ip = "127.0.0.1"
|
||||
|
||||
print(f"\n{'='*70}")
|
||||
print(f"Remote Storage Server - Medios-Macina")
|
||||
print("Remote Storage Server - Medios-Macina")
|
||||
print(f"{'='*70}")
|
||||
print(f"Storage Path: {STORAGE_PATH}")
|
||||
print(f"Local IP: {local_ip}")
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/usr/bin/env python3
|
||||
from pathlib import Path
|
||||
p = Path(r"c:\Forgejo\Medios-Macina\CLI.py")
|
||||
s = p.read_text(encoding='utf-8')
|
||||
start = s.find('\nif False:')
|
||||
if start == -1:
|
||||
print('No if False found')
|
||||
else:
|
||||
after = s[start+1:]
|
||||
idx = after.find('\nfrom rich.markdown import Markdown')
|
||||
if idx == -1:
|
||||
print('No subsequent import found')
|
||||
else:
|
||||
before = s[:start]
|
||||
rest = after[idx+1:]
|
||||
new = before + '\nfrom rich.markdown import Markdown\n' + rest
|
||||
p.write_text(new, encoding='utf-8')
|
||||
print('Removed legacy block')
|
||||
@@ -14,10 +14,9 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from typing import Any
|
||||
from pathlib import Path
|
||||
|
||||
from SYS.logger import log, debug
|
||||
from SYS.logger import log
|
||||
|
||||
try:
|
||||
from API import zerotier
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
|
||||
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
from SYS.logger import debug
|
||||
|
||||
|
||||
@@ -11,7 +11,6 @@ import sys
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Iterator, List, Optional, Sequence, cast
|
||||
|
||||
Reference in New Issue
Block a user