From e1819ff310e11f71aa3aafd4a9c9559a261e0f69 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Wed, 14 Oct 2026 12:26:03 +0100 Subject: [PATCH] Keep Model Target allowance policy in the real adapter --- tests/mock_vws/fixtures/vuforia_backends.py | 29 +++- .../errors/test_errors.py | 25 ++- .../test_model_target_response_policy.py | 160 ++++++++++++++++++ tests/mock_vws/utils/assertions.py | 78 ++------- .../utils/model_target_response_policy.py | 49 ++++++ 5 files changed, 256 insertions(+), 85 deletions(-) create mode 100644 tests/mock_vws/requests_mock/test_model_target_response_policy.py create mode 100644 tests/mock_vws/utils/model_target_response_policy.py diff --git a/tests/mock_vws/fixtures/vuforia_backends.py b/tests/mock_vws/fixtures/vuforia_backends.py index d6c77aaaf..f936f4103 100644 --- a/tests/mock_vws/fixtures/vuforia_backends.py +++ b/tests/mock_vws/fixtures/vuforia_backends.py @@ -19,6 +19,7 @@ from vws.exceptions.vws_exceptions import ( TargetStatusNotSuccessError, ) +from vws.response import Response from mock_vws import MockVWS from mock_vws._flask_server.target_manager import TARGET_MANAGER_FLASK_APP @@ -42,6 +43,9 @@ ) from tests.mock_vws.utils.backend_capabilities import BackendCapabilities from tests.mock_vws.utils.cloud_databases import AddCloudDatabase +from tests.mock_vws.utils.model_target_response_policy import ( + handling_unexpected_model_target_responses, +) from tests.mock_vws.utils.model_target_retries import ( retrying_transient_real_backend_failures, ) @@ -361,6 +365,21 @@ def _enable_use_flask_in_process( yield +@beartype +def _handle_real_model_target_response( + *, + response: Response | requests.Response, +) -> None: + """Skip an unexpected rejection by the shared real account's allowance.""" + if "TRAINING_ALLOWANCE_EXCEEDED" in response.text: + pytest.skip( + reason=( + "The Vuforia account is out of Model Target training " + "allowance - this is not a failure of the code under test." + ), + ) + + @beartype def _enable_use_real_model_target_vuforia( *, @@ -375,7 +394,12 @@ def _enable_use_real_model_target_vuforia( """ del base_vws_url assert bool(monkeypatch) - with retrying_transient_real_backend_failures(): + with ( + retrying_transient_real_backend_failures(), + handling_unexpected_model_target_responses( + handler=_handle_real_model_target_response, + ), + ): yield @@ -997,8 +1021,7 @@ def fixture_mock_model_target_credentials( # standard dataset, or an advanced dataset without a state-based # configuration) are far cheaper and stay enabled, though with enough # traffic they can also hit the allowance; an unexpected allowance -# rejection is reported as an expected failure by -# ``assert_model_target_status`` rather than failing the run. +# rejection is skipped by the real adapter rather than failing the run. _SIGNED_REQUEST_SKIP_REASON = ( "Signed Model Target requests consume the real Vuforia account's " "small, shared, non-resettable training allowance, so they are not " diff --git a/tests/mock_vws/model_target_web_api/errors/test_errors.py b/tests/mock_vws/model_target_web_api/errors/test_errors.py index ca3f934d9..13cd6cee1 100644 --- a/tests/mock_vws/model_target_web_api/errors/test_errors.py +++ b/tests/mock_vws/model_target_web_api/errors/test_errors.py @@ -78,31 +78,28 @@ def test_multiple_expected_statuses(*, base_vws_url: str) -> None: assert "Expected 200 OK or 204 NO_CONTENT from " in str(object=exc.value) -def test_training_allowance_exceeded() -> None: - """An exhausted account allowance is an expected failure, called - out as such in the first line so that a truncated CI summary - still shows it. - """ +@pytest.mark.parametrize( + argnames="base_url", + argvalues=["http://example.com", "https://vws.vuforia.com"], +) +def test_training_allowance_exceeded(*, base_url: str) -> None: + """An unexpected allowance response fails without a backend policy.""" response = fake_response( status_code=HTTPStatus.UNPROCESSABLE_ENTITY, text=( '{"error":{"code":"TRAINING_ALLOWANCE_EXCEEDED",' '"message":"Signing quota reached","target":"7635391"}}' ), - url="http://example.com/modeltargets/datasets", + url=f"{base_url}/modeltargets/datasets", ) - with pytest.raises(expected_exception=pytest.xfail.Exception) as exc: + with pytest.raises(expected_exception=AssertionError) as exc: assert_model_target_status( response=response, status_codes=HTTPStatus.CREATED, ) message = str(object=exc.value) - first_line = message.splitlines()[0] - assert first_line == ( - "The Vuforia account is out of Model Target training allowance - " - "this is not a failure of the code under test." + assert message == ( + f"Expected 201 CREATED from {base_url}/modeltargets/datasets, " + f"got 422.\n\nResponse body:\n{response.text}" ) - assert "MODEL_TARGET_VUFORIA_CLIENT_ID" in message - assert "has to be raised, or reset, on the Vuforia account" in message - assert "expected failure" in message diff --git a/tests/mock_vws/requests_mock/test_model_target_response_policy.py b/tests/mock_vws/requests_mock/test_model_target_response_policy.py new file mode 100644 index 000000000..e7a0b1e45 --- /dev/null +++ b/tests/mock_vws/requests_mock/test_model_target_response_policy.py @@ -0,0 +1,160 @@ +"""Tests for backend handling of unexpected Model Target responses.""" + +from collections.abc import Generator +from http import HTTPStatus + +import pytest +import requests +from vws.response import Response + +from tests.mock_vws.fixtures.vuforia_backends import VuforiaBackend +from tests.mock_vws.model_target_web_api.errors.helpers import fake_response +from tests.mock_vws.utils.assertions import assert_model_target_status +from tests.mock_vws.utils.model_target_response_policy import ( + handling_unexpected_model_target_responses, +) + +_ALLOWANCE_BODY = '{"error":{"code":"TRAINING_ALLOWANCE_EXCEEDED"}}' + + +@pytest.fixture(name="_model_target_backend") +def fixture_model_target_backend() -> VuforiaBackend: + """Select the real policy without credentials or service requests.""" + return VuforiaBackend.REAL + + +@pytest.fixture( + name="base_vws_url", + params=["https://vws.vuforia.com", "http://consumer.example"], +) +def fixture_base_vws_url(*, request: pytest.FixtureRequest) -> str: + """Supply a URL for the adapter and its synthetic responses.""" + url = request.param + assert isinstance(url, str) + return url + + +@pytest.fixture(autouse=True) +def fixture_check_response_policy_cleanup() -> Generator[None]: + """Check the default before setup and after backend fixtures finish. + + Yields: + ``None``, while the test and its backend fixtures run. + """ + response = fake_response( + status_code=HTTPStatus.UNPROCESSABLE_ENTITY, + text=_ALLOWANCE_BODY, + url="https://vws.vuforia.com/modeltargets/datasets", + ) + with pytest.raises(expected_exception=AssertionError): + assert_model_target_status( + response=response, + status_codes=HTTPStatus.CREATED, + ) + yield + with pytest.raises(expected_exception=AssertionError): + assert_model_target_status( + response=response, + status_codes=HTTPStatus.CREATED, + ) + + +@pytest.mark.usefixtures("verify_model_target_mock_vuforia") +def test_real_adapter_handles_allowance(*, base_vws_url: str) -> None: + """The real adapter handles allowance rejections at its supplied + URL. + """ + response = fake_response( + status_code=HTTPStatus.UNPROCESSABLE_ENTITY, + text=_ALLOWANCE_BODY, + url=f"{base_vws_url}/modeltargets/datasets", + ) + with pytest.raises( + expected_exception=pytest.skip.Exception, + match="out of Model Target training allowance", + ): + assert_model_target_status( + response=response, + status_codes=HTTPStatus.CREATED, + ) + + +@pytest.mark.usefixtures("verify_model_target_mock_vuforia") +def test_expected_allowance_status(*, base_vws_url: str) -> None: + """An expected allowance response remains an ordinary assertion.""" + response = fake_response( + status_code=HTTPStatus.UNPROCESSABLE_ENTITY, + text=_ALLOWANCE_BODY, + url=f"{base_vws_url}/modeltargets/datasets", + ) + assert_model_target_status( + response=response, + status_codes=HTTPStatus.UNPROCESSABLE_ENTITY, + ) + + +@pytest.mark.usefixtures("verify_model_target_mock_vuforia") +def test_other_unexpected_status(*, base_vws_url: str) -> None: + """The real adapter does not hide unrelated response failures.""" + response = fake_response( + status_code=HTTPStatus.BAD_REQUEST, + text='{"error":{"code":"VALIDATION_ERROR"}}', + url=f"{base_vws_url}/modeltargets/datasets", + ) + with pytest.raises( + expected_exception=AssertionError, + match="VALIDATION_ERROR", + ): + assert_model_target_status( + response=response, + status_codes=HTTPStatus.CREATED, + ) + + +def test_nested_response_handlers() -> None: + """Nested handlers restore the outer handler and finally the + default. + """ + calls: list[str] = [] + response = fake_response( + status_code=HTTPStatus.BAD_REQUEST, + text="{}", + url="http://consumer.example/modeltargets/datasets", + ) + + def outer_handler(*, response: Response | requests.Response) -> None: + """Record a response handled by the outer context.""" + assert response.status_code == HTTPStatus.BAD_REQUEST + calls.append("outer") + + def inner_handler(*, response: Response | requests.Response) -> None: + """Record a response handled by the inner context.""" + assert response.status_code == HTTPStatus.BAD_REQUEST + calls.append("inner") + + with handling_unexpected_model_target_responses(handler=outer_handler): + with pytest.raises(expected_exception=AssertionError): + assert_model_target_status( + response=response, + status_codes=HTTPStatus.CREATED, + ) + with ( + handling_unexpected_model_target_responses(handler=inner_handler), + pytest.raises(expected_exception=AssertionError), + ): + assert_model_target_status( + response=response, + status_codes=HTTPStatus.CREATED, + ) + with pytest.raises(expected_exception=AssertionError): + assert_model_target_status( + response=response, + status_codes=HTTPStatus.CREATED, + ) + + with pytest.raises(expected_exception=AssertionError): + assert_model_target_status( + response=response, + status_codes=HTTPStatus.CREATED, + ) + assert calls == ["outer", "inner", "outer"] diff --git a/tests/mock_vws/utils/assertions.py b/tests/mock_vws/utils/assertions.py index 4a47348cd..1bceac1f0 100644 --- a/tests/mock_vws/utils/assertions.py +++ b/tests/mock_vws/utils/assertions.py @@ -4,19 +4,19 @@ import datetime import email.utils import json -import textwrap from collections.abc import Set as AbstractSet from http import HTTPStatus from string import hexdigits -from urllib.parse import urlparse from zoneinfo import ZoneInfo -import pytest import requests from beartype import beartype from vws.response import Response from mock_vws._constants import ResultCodes +from tests.mock_vws.utils.model_target_response_policy import ( + handle_unexpected_model_target_response, +) @beartype @@ -218,34 +218,6 @@ def assert_vws_response( assert_valid_date_header(response=response) -_TRAINING_ALLOWANCE_EXCEEDED = "TRAINING_ALLOWANCE_EXCEEDED" - -# The first line of the failure message, because a truncated failure summary -# in a CI log has to be enough to tell this apart from a real regression. -_TRAINING_ALLOWANCE_EXCEEDED_HEADLINE = ( - "The Vuforia account is out of Model Target training allowance - " - "this is not a failure of the code under test." -) - -_TRAINING_ALLOWANCE_EXCEEDED_EXPLANATION = textwrap.dedent( - text="""\ - ``TRAINING_ALLOWANCE_EXCEEDED`` is a quota on the Vuforia account behind - ``MODEL_TARGET_VUFORIA_CLIENT_ID``, not a fault in the code under test or - in this request. Every CI job shares one set of Model Target credentials, - so the allowance is consumed across all jobs and all concurrent runs. - - The allowance has to be raised, or reset, on the Vuforia account for this - request to succeed against the real backend again. Signed dataset types, - such as an advanced dataset with a state-based configuration, exhaust it - first, but with enough traffic unsigned dataset creation is rejected too. - - An unexpected allowance rejection is reported as an expected failure - (xfail) rather than a test failure, so that the shared account running - dry does not turn CI red: the test passes again automatically once the - allowance recovers.""", -) - - @beartype def assert_model_target_status( *, @@ -254,11 +226,8 @@ def assert_model_target_status( ) -> None: """Assert the status code of a Model Target Web API response. - A bare ``assert response.status_code == ...`` hides the response body, - which is where the real Model Target Web API says *why* it rejected a - request. That matters most when the reason has nothing to do with the - request, such as an exhausted account allowance, so this helper always - reports the body and calls out that case by name. + An unexpected status reports the URL and body. A backend may supply + a response handler for restrictions of the service it runs against. Args: response: The response returned by a request to the Model Target Web @@ -268,12 +237,7 @@ def assert_model_target_status( Raises: AssertionError: The response does not have one of the expected status - codes. - Exception: The response unexpectedly reports a - ``TRAINING_ALLOWANCE_EXCEEDED`` rejection. That is the shared - Vuforia account running out of Model Target training allowance, - not a fault in the code under test, so it is an expected failure - rather than a test failure. + codes and the backend handler has not handled the rejection. """ expected = ( frozenset({status_codes}) @@ -283,36 +247,14 @@ def assert_model_target_status( if response.status_code in expected: return + handle_unexpected_model_target_response(response=response) expected_description = " or ".join( sorted(f"{item} {item.name}" for item in expected) ) - allowance_exceeded = _TRAINING_ALLOWANCE_EXCEEDED in response.text - if ( - allowance_exceeded - and urlparse(url=response.url).hostname == "vws.vuforia.com" - ): - pytest.skip(reason=_TRAINING_ALLOWANCE_EXCEEDED_HEADLINE) - headline = ( - [_TRAINING_ALLOWANCE_EXCEEDED_HEADLINE] if allowance_exceeded else [] - ) - explanation = ( - [_TRAINING_ALLOWANCE_EXCEEDED_EXPLANATION] - if allowance_exceeded - else [] - ) - message = "\n\n".join( - [ - *headline, - ( - f"Expected {expected_description} from {response.url}, " - f"got {response.status_code}." - ), - f"Response body:\n{response.text}", - *explanation, - ], + message = ( + f"Expected {expected_description} from {response.url}, " + f"got {response.status_code}.\n\nResponse body:\n{response.text}" ) - if allowance_exceeded: - pytest.xfail(reason=message) raise AssertionError(message) diff --git a/tests/mock_vws/utils/model_target_response_policy.py b/tests/mock_vws/utils/model_target_response_policy.py new file mode 100644 index 000000000..caa013ec7 --- /dev/null +++ b/tests/mock_vws/utils/model_target_response_policy.py @@ -0,0 +1,49 @@ +"""Backend-supplied handling of unexpected Model Target responses.""" + +import contextlib +from collections.abc import Generator +from typing import Protocol, runtime_checkable + +import requests +from beartype import beartype +from vws.response import Response + + +@runtime_checkable +class ModelTargetResponseHandler(Protocol): + """A backend operation for an unexpected Model Target response.""" + + def __call__(self, *, response: Response | requests.Response) -> None: + """Handle the response before the shared assertion reports it.""" + ... + + +_RESPONSE_HANDLERS: list[ModelTargetResponseHandler] = [] + + +@contextlib.contextmanager +@beartype +def handling_unexpected_model_target_responses( + *, + handler: ModelTargetResponseHandler, +) -> Generator[None]: + """Use the supplied response handler within this context. + + Yields: + ``None``, with the supplied handler active. + """ + _RESPONSE_HANDLERS.append(handler) + try: + yield + finally: + _ = _RESPONSE_HANDLERS.pop() + + +@beartype +def handle_unexpected_model_target_response( + *, + response: Response | requests.Response, +) -> None: + """Notify the backend before asserting an unexpected status.""" + if bool(_RESPONSE_HANDLERS): + _RESPONSE_HANDLERS[-1](response=response)