This commit is contained in:
2026-02-19 20:38:54 -08:00
parent 615a4fd1a4
commit 39a84b3274
5 changed files with 1475 additions and 69 deletions

View File

@@ -92,7 +92,7 @@
"(hitfile\\.net/[a-z0-9A-Z]{4,9})" "(hitfile\\.net/[a-z0-9A-Z]{4,9})"
], ],
"regexp": "(hitf\\.(to|cc)/([a-z0-9A-Z]{4,9}))|(htfl\\.(net|to|cc)/([a-z0-9A-Z]{4,9}))|(hitfile\\.(net)/download/free/([a-z0-9A-Z]{4,9}))|((hitfile\\.net/[a-z0-9A-Z]{4,9}))", "regexp": "(hitf\\.(to|cc)/([a-z0-9A-Z]{4,9}))|(htfl\\.(net|to|cc)/([a-z0-9A-Z]{4,9}))|(hitfile\\.(net)/download/free/([a-z0-9A-Z]{4,9}))|((hitfile\\.net/[a-z0-9A-Z]{4,9}))",
"status": true "status": false
}, },
"mega": { "mega": {
"name": "mega", "name": "mega",
@@ -474,13 +474,14 @@
"domains": [ "domains": [
"katfile.com", "katfile.com",
"katfile.cloud", "katfile.cloud",
"katfile.online" "katfile.online",
"katfile.vip"
], ],
"regexps": [ "regexps": [
"katfile\\.(cloud|online)/([0-9a-zA-Z]{12})", "katfile\\.(cloud|online|vip)/([0-9a-zA-Z]{12})",
"(katfile\\.com/[0-9a-zA-Z]{12})" "(katfile\\.com/[0-9a-zA-Z]{12})"
], ],
"regexp": "(katfile\\.(cloud|online)/([0-9a-zA-Z]{12}))|((katfile\\.com/[0-9a-zA-Z]{12}))", "regexp": "(katfile\\.(cloud|online|vip)/([0-9a-zA-Z]{12}))|((katfile\\.com/[0-9a-zA-Z]{12}))",
"status": false "status": false
}, },
"mediafire": { "mediafire": {
@@ -774,7 +775,7 @@
"(worldbytez\\.(net|com)/[a-zA-Z0-9]{12})" "(worldbytez\\.(net|com)/[a-zA-Z0-9]{12})"
], ],
"regexp": "(worldbytez\\.(net|com)/[a-zA-Z0-9]{12})", "regexp": "(worldbytez\\.(net|com)/[a-zA-Z0-9]{12})",
"status": true "status": false
} }
}, },
"streams": { "streams": {

View File

@@ -4,6 +4,7 @@ import importlib
import os import os
import re import re
import sys import sys
import requests
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
@@ -11,8 +12,9 @@ from urllib.parse import quote, unquote, urlparse
from API.HTTP import _download_direct_file from API.HTTP import _download_direct_file
from ProviderCore.base import Provider, SearchResult from ProviderCore.base import Provider, SearchResult
from SYS.utils import sanitize_filename from SYS.utils import sanitize_filename, unique_path
from SYS.logger import log from SYS.logger import log
from SYS.config import get_provider_block
# Helper for download-file: render selectable formats for a details URL. # Helper for download-file: render selectable formats for a details URL.
def maybe_show_formats_table( def maybe_show_formats_table(
@@ -184,6 +186,96 @@ def _pick_provider_config(config: Any) -> Dict[str, Any]:
return {} return {}
def _pick_archive_credentials(config: Any) -> tuple[Optional[str], Optional[str]]:
"""Resolve Archive.org credentials.
Preference order:
1) provider.internetarchive (email/username + password)
2) provider.openlibrary (email + password)
"""
if not isinstance(config, dict):
return None, None
ia_block = get_provider_block(config, "internetarchive")
if isinstance(ia_block, dict):
email = (
ia_block.get("email")
or ia_block.get("username")
or ia_block.get("user")
)
password = ia_block.get("password")
email_text = str(email).strip() if email else ""
password_text = str(password).strip() if password else ""
if email_text and password_text:
return email_text, password_text
ol_block = get_provider_block(config, "openlibrary")
if isinstance(ol_block, dict):
email = ol_block.get("email")
password = ol_block.get("password")
email_text = str(email).strip() if email else ""
password_text = str(password).strip() if password else ""
if email_text and password_text:
return email_text, password_text
return None, None
def _filename_from_response(url: str, response: requests.Response, suggested_filename: Optional[str] = None) -> str:
suggested = str(suggested_filename or "").strip()
if suggested:
guessed_ext = Path(str(_extract_download_filename_from_url(url) or "")).suffix
if Path(suggested).suffix:
return sanitize_filename(suggested)
merged = f"{suggested}{guessed_ext}" if guessed_ext else suggested
return sanitize_filename(merged)
content_disposition = ""
try:
content_disposition = str(response.headers.get("content-disposition", "") or "")
except Exception:
content_disposition = ""
if content_disposition:
m = re.search(r'filename\*?=(?:"([^"]+)"|([^;\s]+))', content_disposition)
if m:
candidate = (m.group(1) or m.group(2) or "").strip().strip('"')
if candidate:
return sanitize_filename(unquote(candidate))
extracted = _extract_download_filename_from_url(url)
if extracted:
return sanitize_filename(extracted)
fallback = Path(urlparse(url).path).name or "download.bin"
return sanitize_filename(unquote(fallback))
def _download_with_requests_session(
*,
session: requests.Session,
url: str,
output_dir: Path,
suggested_filename: Optional[str] = None,
) -> Path:
headers = {
"Referer": "https://archive.org/",
"Accept": "*/*",
}
response = session.get(url, headers=headers, stream=True, allow_redirects=True, timeout=120)
response.raise_for_status()
filename = _filename_from_response(url, response, suggested_filename=suggested_filename)
out_path = unique_path(Path(output_dir) / filename)
with open(out_path, "wb") as handle:
for chunk in response.iter_content(chunk_size=1024 * 256):
if chunk:
handle.write(chunk)
return out_path
def _looks_fielded_query(q: str) -> bool: def _looks_fielded_query(q: str) -> bool:
low = (q or "").lower() low = (q or "").lower()
return (":" in low) or (" and " in low) or (" or " return (":" in low) or (" and " in low) or (" or "
@@ -476,6 +568,17 @@ class InternetArchive(Provider):
@classmethod @classmethod
def config_schema(cls) -> List[Dict[str, Any]]: def config_schema(cls) -> List[Dict[str, Any]]:
return [ return [
{
"key": "email",
"label": "Archive.org Email (restricted downloads)",
"default": ""
},
{
"key": "password",
"label": "Archive.org Password (restricted downloads)",
"default": "",
"secret": True
},
{ {
"key": "access_key", "key": "access_key",
"label": "Access Key (for uploads)", "label": "Access Key (for uploads)",
@@ -542,6 +645,73 @@ class InternetArchive(Provider):
except Exception: except Exception:
return False return False
def _download_with_archive_auth(
self,
*,
url: str,
output_dir: Path,
suggested_filename: Optional[str] = None,
) -> Optional[Path]:
email, password = _pick_archive_credentials(self.config or {})
if not email or not password:
return None
try:
from Provider.openlibrary import OpenLibrary
except Exception as exc:
log(f"[internetarchive] OpenLibrary auth helper unavailable: {exc}", file=sys.stderr)
return None
identifier = _extract_identifier_from_any(url)
session: Optional[requests.Session] = None
loaned = False
try:
session = OpenLibrary._archive_login(email, password)
if identifier:
try:
session.get(
f"https://archive.org/details/{identifier}",
timeout=30,
allow_redirects=True,
)
except Exception:
pass
try:
session.get(
f"https://archive.org/download/{identifier}",
timeout=30,
allow_redirects=True,
)
except Exception:
pass
try:
session = OpenLibrary._archive_loan(session, identifier, verbose=False)
loaned = True
except Exception:
loaned = False
return _download_with_requests_session(
session=session,
url=url,
output_dir=output_dir,
suggested_filename=suggested_filename,
)
except Exception as exc:
log(f"[internetarchive] authenticated download failed: {exc}", file=sys.stderr)
return None
finally:
if session is not None:
if loaned and identifier:
try:
OpenLibrary._archive_return_loan(session, identifier)
except Exception:
pass
try:
OpenLibrary._archive_logout(session)
except Exception:
pass
@staticmethod @staticmethod
def _media_kind_from_mediatype(mediatype: str) -> str: def _media_kind_from_mediatype(mediatype: str) -> str:
mt = str(mediatype or "").strip().lower() mt = str(mediatype or "").strip().lower()
@@ -715,6 +885,13 @@ class InternetArchive(Provider):
return None return None
except Exception as exc: except Exception as exc:
log(f"[internetarchive] direct file download failed, falling back to IA API: {exc}", file=sys.stderr) log(f"[internetarchive] direct file download failed, falling back to IA API: {exc}", file=sys.stderr)
auth_path = self._download_with_archive_auth(
url=raw_path,
output_dir=output_dir,
suggested_filename=suggested_filename,
)
if auth_path is not None:
return auth_path
ia = _ia() ia = _ia()
get_item = getattr(ia, "get_item", None) get_item = getattr(ia, "get_item", None)

1246
TUI.py

File diff suppressed because it is too large Load Diff

View File

@@ -99,6 +99,7 @@ class PipelineRunner:
pipeline_text: str, pipeline_text: str,
*, *,
seeds: Optional[Any] = None, seeds: Optional[Any] = None,
seed_table: Optional[Any] = None,
isolate: bool = False, isolate: bool = False,
on_log: Optional[Callable[[str], on_log: Optional[Callable[[str],
None]] = None, None]] = None,
@@ -158,6 +159,12 @@ class PipelineRunner:
except Exception: except Exception:
debug(traceback.format_exc()) debug(traceback.format_exc())
if seed_table is not None:
try:
ctx.set_current_stage_table(seed_table)
except Exception:
debug(traceback.format_exc())
stdout_buffer = io.StringIO() stdout_buffer = io.StringIO()
stderr_buffer = io.StringIO() stderr_buffer = io.StringIO()

View File

@@ -55,12 +55,42 @@
#results-pane { #results-pane {
width: 100%; width: 100%;
height: 2fr; height: 2fr;
padding: 1; padding: 0 1 1 1;
background: $panel; background: $panel;
border: round $panel-darken-2; border: round $panel-darken-2;
margin-top: 1; margin-top: 1;
} }
#results-pane .section-title {
margin-top: 0;
margin-bottom: 0;
}
#results-layout {
width: 100%;
height: 1fr;
}
#results-list-pane {
width: 2fr;
height: 1fr;
padding-right: 1;
}
#results-tags-pane {
width: 1fr;
height: 1fr;
padding: 0 1;
border-left: solid $panel-darken-2;
}
#results-meta-pane {
width: 1fr;
height: 1fr;
padding-left: 1;
border-left: solid $panel-darken-2;
}
#store-select { #store-select {
width: 24; width: 24;
margin-right: 2; margin-right: 2;
@@ -117,6 +147,9 @@
#results-table { #results-table {
height: 1fr; height: 1fr;
border: solid #ffffff; border: solid #ffffff;
background: #ffffff;
color: #000000;
padding: 0;
} }
#results-table > .datatable--header { #results-table > .datatable--header {
@@ -125,6 +158,20 @@
text-style: bold; text-style: bold;
} }
#inline-tags-output {
height: 1fr;
border: solid #ffffff;
background: #ffffff;
color: #000000;
}
#metadata-tree {
height: 1fr;
border: solid #ffffff;
background: #ffffff;
color: #000000;
}
.status-info { .status-info {
@@ -149,6 +196,7 @@
} }
#tags-button, #tags-button,
#actions-button,
#metadata-button, #metadata-button,
#relationships-button { #relationships-button {
width: auto; width: auto;
@@ -177,6 +225,23 @@
margin-top: 1; margin-top: 1;
} }
#actions-list {
width: 100%;
height: auto;
margin-top: 1;
}
#actions-list Button {
width: 100%;
margin-bottom: 1;
}
#actions-footer {
width: 100%;
height: auto;
margin-top: 1;
}
#tags-status { #tags-status {
width: 1fr; width: 1fr;
height: 3; height: 3;