diff --git a/tests/mock_vws/fixtures/api_contract.py b/tests/mock_vws/fixtures/api_contract.py index f31a319b1..bf208a285 100644 --- a/tests/mock_vws/fixtures/api_contract.py +++ b/tests/mock_vws/fixtures/api_contract.py @@ -24,8 +24,8 @@ from tests.mock_vws.utils import Endpoint, ModelTargetEndpoint from tests.mock_vws.utils.retries import ( RETRY_ON_TARGET_NOT_YET_VISIBLE, - RETRY_ON_TRANSIENT_VWS_FAILURE, TRANSIENT_VWS_EXCEPTIONS, + retry_transient_vws_failures, ) # The number of targets to add before giving up on getting one which @@ -91,7 +91,7 @@ def inactive_cloud_reco_client( @beartype -@RETRY_ON_TRANSIENT_VWS_FAILURE +@retry_transient_vws_failures def _add_target(*, vws_client: VWS, image: io.BytesIO) -> str: """Add a target, which is then in the processing state. @@ -114,7 +114,7 @@ def _add_target(*, vws_client: VWS, image: io.BytesIO) -> str: @beartype -@RETRY_ON_TRANSIENT_VWS_FAILURE +@retry_transient_vws_failures @RETRY_ON_TARGET_NOT_YET_VISIBLE def _get_processed_target( *, vws_client: VWS, target_id_: str diff --git a/tests/mock_vws/fixtures/real_vws_retries.py b/tests/mock_vws/fixtures/real_vws_retries.py new file mode 100644 index 000000000..3200e5a64 --- /dev/null +++ b/tests/mock_vws/fixtures/real_vws_retries.py @@ -0,0 +1,91 @@ +"""Bounded retry scheduling for real Vuforia fixture operations. + +At most ten attempts are scheduled within 120 seconds of the first attempt. +A retry cannot interrupt an in-flight client request or its processing poll. +Valid Retry-After values are honored, even when that means giving up because +no retry fits the budget. Without one, full jitter spreads retries over an +exponentially increasing interval, starting at one second and capped at 30. +""" + +import logging +from http import HTTPStatus + +from beartype import beartype +from tenacity import RetryCallState, Retrying +from tenacity.retry import retry_if_exception_type +from tenacity.stop import stop_after_attempt, stop_before_delay +from tenacity.wait import wait_random_exponential +from urllib3.exceptions import InvalidHeader +from urllib3.util import Retry +from vws.exceptions.vws_exceptions import TooManyRequestsError + +from tests.mock_vws.utils.retries import ( + TRANSIENT_VWS_EXCEPTIONS, + TRANSIENT_VWS_RETRY_ATTEMPTS, +) + +RETRY_BUDGET_SECONDS = 120 +_LOGGER = logging.getLogger(name=__name__) +_FALLBACK_WAIT = wait_random_exponential(multiplier=1, max=30) +# Capped server delays must still exceed the budget to prevent early retries. +_RETRY_AFTER_PARSER = Retry(retry_after_max=RETRY_BUDGET_SECONDS + 1) + + +# Live-service throttling responses cannot be forced by the API tests. +@beartype +def wait_for_real_vws( + retry_state: RetryCallState, +) -> float: # pragma: no cover + """Wait for the server's delay, otherwise use capped full jitter.""" + assert retry_state.outcome is not None + error = retry_state.outcome.exception() + if isinstance(error, TooManyRequestsError): + for name, value in error.response.headers.items(): + if name.casefold() == "retry-after": + try: + return float( + _RETRY_AFTER_PARSER.parse_retry_after( + retry_after=value + ) + ) + except InvalidHeader: + break + except ValueError, OverflowError: + # A delay too large to parse must not cause an early retry. + return float(RETRY_BUDGET_SECONDS + 1) + return _FALLBACK_WAIT(retry_state=retry_state) + + +@beartype +def _log_retry(retry_state: RetryCallState) -> None: # pragma: no cover + """Log timing and error type without request headers or + credentials. + """ + assert retry_state.outcome is not None + assert retry_state.next_action is not None + error = retry_state.outcome.exception() + status = ( + HTTPStatus.TOO_MANY_REQUESTS + if isinstance(error, TooManyRequestsError) + else None + ) + _LOGGER.warning( + "Retrying real Vuforia operation after attempt %d: " + "%s (status %s), delay %.3fs", + retry_state.attempt_number, + type(error).__name__, + status, + retry_state.next_action.sleep, + ) + + +REAL_VWS_RETRYING = Retrying( + retry=retry_if_exception_type(exception_types=TRANSIENT_VWS_EXCEPTIONS), + stop=( + stop_after_attempt(max_attempt_number=TRANSIENT_VWS_RETRY_ATTEMPTS) + | stop_before_delay(max_delay=RETRY_BUDGET_SECONDS) + ), + wait=wait_for_real_vws, + before_sleep=_log_retry, + reraise=True, +) diff --git a/tests/mock_vws/fixtures/vuforia_backends.py b/tests/mock_vws/fixtures/vuforia_backends.py index f4d805be4..85244f0fb 100644 --- a/tests/mock_vws/fixtures/vuforia_backends.py +++ b/tests/mock_vws/fixtures/vuforia_backends.py @@ -46,6 +46,7 @@ from tests.mock_vws.fixtures.real_model_target_retries import ( REAL_MODEL_TARGET_RETRY_POLICY, ) +from tests.mock_vws.fixtures.real_vws_retries import REAL_VWS_RETRYING from tests.mock_vws.utils.backend_capabilities import BackendCapabilities from tests.mock_vws.utils.cloud_databases import AddCloudDatabase from tests.mock_vws.utils.model_target_response_policy import ( @@ -60,7 +61,8 @@ ) from tests.mock_vws.utils.retries import ( RETRY_ON_TARGET_STILL_PROCESSING, - RETRY_ON_TRANSIENT_VWS_FAILURE, + retry_transient_vws_failures, + retrying_vws_requests, ) from tests.mock_vws.utils.unrouted_requests import ( UnroutedRequests, @@ -103,7 +105,7 @@ def running_in_memory_mock() -> MockVWS: @beartype -@RETRY_ON_TRANSIENT_VWS_FAILURE +@retry_transient_vws_failures def _delete_all_targets( *, database_keys: CloudDatabase, base_vws_url: str ) -> None: @@ -200,10 +202,13 @@ def _enable_use_real_vuforia( assert bool(vumark_vuforia_database) assert bool(inactive_vumark_database) del base_vwq_url - _delete_all_targets( - database_keys=working_database, base_vws_url=base_vws_url - ) - with handling_request_timeouts(handler=_handle_real_request_timeout): + with ( + retrying_vws_requests(retrying=REAL_VWS_RETRYING), + handling_request_timeouts(handler=_handle_real_request_timeout), + ): + _delete_all_targets( + database_keys=working_database, base_vws_url=base_vws_url + ) yield diff --git a/tests/mock_vws/utils/retries.py b/tests/mock_vws/utils/retries.py index c6e4826bd..2ebd3d5a7 100644 --- a/tests/mock_vws/utils/retries.py +++ b/tests/mock_vws/utils/retries.py @@ -1,10 +1,16 @@ """Helpers for retrying requests to VWS.""" +import contextlib +from collections.abc import Callable, Generator +from contextvars import ContextVar +from functools import wraps + +from beartype import beartype from requests.exceptions import ConnectionError as RequestsConnectionError from requests.exceptions import ConnectTimeout as RequestsConnectTimeout from requests.exceptions import ReadTimeout as RequestsReadTimeout from requests.exceptions import Timeout as RequestsTimeout -from tenacity import retry +from tenacity import Retrying, retry from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_attempt from tenacity.wait import wait_fixed @@ -31,17 +37,50 @@ ) TRANSIENT_VWS_RETRY_ATTEMPTS = 10 -# We rely on pytest-retry for exceptions *during* tests. -# We use tenacity for exceptions *before* tests. -# See https://github.com/str0zzapreti/pytest-retry/issues/33. -RETRY_ON_TRANSIENT_VWS_FAILURE = retry( - retry=retry_if_exception_type(exception_types=TRANSIENT_VWS_EXCEPTIONS), - stop=stop_after_attempt(max_attempt_number=TRANSIENT_VWS_RETRY_ATTEMPTS), - wait=wait_fixed(wait=10), - reraise=True, +# Shared helpers send once unless an adapter supplies a retry policy. +# ContextVar requires a positional name at runtime. +_VWS_RETRYING: ContextVar[Retrying | None] = ContextVar( # type: ignore[call-arg] + "vws_retrying", + default=None, ) +@contextlib.contextmanager +@beartype +def retrying_vws_requests(*, retrying: Retrying) -> Generator[None]: + """Install an adapter's fixture retry policy for this scope. + + Yields: + ``None``, with the policy active and its predecessor restored on exit. + """ + token = _VWS_RETRYING.set(retrying) + try: + yield + finally: + _VWS_RETRYING.reset(token) + + +@beartype +def retry_transient_vws_failures[**P, T]( + operation: Callable[P, T], +) -> Callable[P, T]: + """Apply the adapter's policy without multiplying nested retry budgets.""" + + @wraps(wrapped=operation) + def wrapped(*args: P.args, **kwargs: P.kwargs) -> T: + """Run an operation under at most one transient retry loop.""" + retrying = _VWS_RETRYING.get() + if retrying is None: + return operation(*args, **kwargs) + token = _VWS_RETRYING.set(None) + try: + return retrying.copy()(operation, *args, **kwargs) + finally: + _VWS_RETRYING.reset(token) + + return wrapped + + # Real Vuforia sometimes reports a target which it has just created as # unknown, because the target is not yet visible to the read path. #