diff --git a/API/HydrusNetwork.py b/API/HydrusNetwork.py index 2408044..20bd566 100644 --- a/API/HydrusNetwork.py +++ b/API/HydrusNetwork.py @@ -207,42 +207,44 @@ class HydrusNetwork: # Keep the PipelineLiveProgress transfer line clean: show the file name. # (The hydrus instance/service is already visible in the logs above.) label = str(getattr(file_path, "name", None) or "upload") - start_t = time.time() - last_render_t = [start_t] - sent = [0] - - def _render_progress(final: bool = False) -> None: - if file_size <= 0: - return - now = time.time() - if not final and (now - float(last_render_t[0])) < 0.25: - return - last_render_t[0] = now - bar.update( - downloaded=int(sent[0]), - total=int(file_size), - label=str(label), - file=sys.stderr, - ) - if final: - bar.finish() - - def file_gen(): - try: - with file_path.open("rb") as handle: - while True: - chunk = handle.read(chunk_size) - if not chunk: - break - sent[0] += len(chunk) - _render_progress(final=False) - yield chunk - finally: - _render_progress(final=True) + traffic_sent_total = [0] upload_attempts = 3 last_upload_exc: Exception | None = None for attempt in range(upload_attempts): + sent_this_attempt = [0] + last_render_t = [time.time()] + + def _render_progress(final: bool = False) -> None: + if file_size <= 0: + return + now = time.time() + if not final and (now - float(last_render_t[0])) < 0.25: + return + last_render_t[0] = now + bar.update( + downloaded=int(sent_this_attempt[0]), + total=int(file_size), + label=str(label), + file=sys.stderr, + ) + if final: + bar.finish() + + def file_gen(): + try: + with file_path.open("rb") as handle: + while True: + chunk = handle.read(chunk_size) + if not chunk: + break + sent_this_attempt[0] += len(chunk) + traffic_sent_total[0] += len(chunk) + _render_progress(final=False) + yield chunk + finally: + _render_progress(final=True) + try: response = client.request( spec.method, @@ -253,14 +255,23 @@ class HydrusNetwork: raise_for_status=False, log_http_errors=False, ) + logger.debug( + f"{self._log_prefix()} Upload completed on attempt {attempt + 1}/{upload_attempts}; " + f"attempt_bytes={int(sent_this_attempt[0])}, total_traffic_bytes={int(traffic_sent_total[0])}" + ) last_upload_exc = None break except (httpx.TimeoutException, httpx.RequestError) as exc: last_upload_exc = exc logger.warning( - f"{self._log_prefix()} Upload timeout/error on attempt {attempt + 1}/{upload_attempts}: {exc}" + f"{self._log_prefix()} Upload timeout/error on attempt {attempt + 1}/{upload_attempts}: {exc} " + f"(attempt_bytes={int(sent_this_attempt[0])}, total_traffic_bytes={int(traffic_sent_total[0])})" ) if attempt < upload_attempts - 1: + try: + time.sleep(0.75 * (attempt + 1)) + except Exception: + pass continue raise if response is None and last_upload_exc is not None: