Keep Model Target allowance policy in the real adapter

tests/mock_vws/fixtures/vuforia_backends.py

@@ -19,6 +19,7 @@
 from vws.exceptions.vws_exceptions import (
     TargetStatusNotSuccessError,
 )
+from vws.response import Response
 
 from mock_vws import MockVWS
 from mock_vws._flask_server.target_manager import TARGET_MANAGER_FLASK_APP
@@ -42,6 +43,9 @@
 )
 from tests.mock_vws.utils.backend_capabilities import BackendCapabilities
 from tests.mock_vws.utils.cloud_databases import AddCloudDatabase
+from tests.mock_vws.utils.model_target_response_policy import (
+    handling_unexpected_model_target_responses,
+)
 from tests.mock_vws.utils.model_target_retries import (
     retrying_transient_real_backend_failures,
 )
@@ -361,6 +365,21 @@ def _enable_use_flask_in_process(
         yield
 
 
+@beartype
+def _handle_real_model_target_response(
+    *,
+    response: Response | requests.Response,
+) -> None:
+    """Skip an unexpected rejection by the shared real account's allowance."""
+    if "TRAINING_ALLOWANCE_EXCEEDED" in response.text:
+        pytest.skip(
+            reason=(
+                "The Vuforia account is out of Model Target training "
+                "allowance - this is not a failure of the code under test."
+            ),
+        )
+
+
 @beartype
 def _enable_use_real_model_target_vuforia(
     *,
@@ -375,7 +394,12 @@ def _enable_use_real_model_target_vuforia(
     """
     del base_vws_url
     assert bool(monkeypatch)
-    with retrying_transient_real_backend_failures():
+    with (
+        retrying_transient_real_backend_failures(),
+        handling_unexpected_model_target_responses(
+            handler=_handle_real_model_target_response,
+        ),
+    ):
         yield
 
 
@@ -997,8 +1021,7 @@ def fixture_mock_model_target_credentials(
 # standard dataset, or an advanced dataset without a state-based
 # configuration) are far cheaper and stay enabled, though with enough
 # traffic they can also hit the allowance; an unexpected allowance
-# rejection is reported as an expected failure by
-# ``assert_model_target_status`` rather than failing the run.
+# rejection is skipped by the real adapter rather than failing the run.
 _SIGNED_REQUEST_SKIP_REASON = (
     "Signed Model Target requests consume the real Vuforia account's "
     "small, shared, non-resettable training allowance, so they are not "

tests/mock_vws/model_target_web_api/errors/test_errors.py

@@ -78,31 +78,28 @@ def test_multiple_expected_statuses(*, base_vws_url: str) -> None:
     assert "Expected 200 OK or 204 NO_CONTENT from " in str(object=exc.value)
 
 
-def test_training_allowance_exceeded() -> None:
-    """An exhausted account allowance is an expected failure, called
-    out as such in the first line so that a truncated CI summary
-    still shows it.
-    """
+@pytest.mark.parametrize(
+    argnames="base_url",
+    argvalues=["http://example.com", "https://vws.vuforia.com"],
+)
+def test_training_allowance_exceeded(*, base_url: str) -> None:
+    """An unexpected allowance response fails without a backend policy."""
     response = fake_response(
         status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
         text=(
             '{"error":{"code":"TRAINING_ALLOWANCE_EXCEEDED",'
             '"message":"Signing quota reached","target":"7635391"}}'
         ),
-        url="http://example.com/modeltargets/datasets",
+        url=f"{base_url}/modeltargets/datasets",
     )
-    with pytest.raises(expected_exception=pytest.xfail.Exception) as exc:
+    with pytest.raises(expected_exception=AssertionError) as exc:
         assert_model_target_status(
             response=response,
             status_codes=HTTPStatus.CREATED,
         )
 
     message = str(object=exc.value)
-    first_line = message.splitlines()[0]
-    assert first_line == (
-        "The Vuforia account is out of Model Target training allowance - "
-        "this is not a failure of the code under test."
+    assert message == (
+        f"Expected 201 CREATED from {base_url}/modeltargets/datasets, "
+        f"got 422.\n\nResponse body:\n{response.text}"
     )
-    assert "MODEL_TARGET_VUFORIA_CLIENT_ID" in message
-    assert "has to be raised, or reset, on the Vuforia account" in message
-    assert "expected failure" in message

tests/mock_vws/requests_mock/test_model_target_response_policy.py

@@ -0,0 +1,160 @@
+"""Tests for backend handling of unexpected Model Target responses."""
+
+from collections.abc import Generator
+from http import HTTPStatus
+
+import pytest
+import requests
+from vws.response import Response
+
+from tests.mock_vws.fixtures.vuforia_backends import VuforiaBackend
+from tests.mock_vws.model_target_web_api.errors.helpers import fake_response
+from tests.mock_vws.utils.assertions import assert_model_target_status
+from tests.mock_vws.utils.model_target_response_policy import (
+    handling_unexpected_model_target_responses,
+)
+
+_ALLOWANCE_BODY = '{"error":{"code":"TRAINING_ALLOWANCE_EXCEEDED"}}'
+
+
+@pytest.fixture(name="_model_target_backend")
+def fixture_model_target_backend() -> VuforiaBackend:
+    """Select the real policy without credentials or service requests."""
+    return VuforiaBackend.REAL
+
+
+@pytest.fixture(
+    name="base_vws_url",
+    params=["https://vws.vuforia.com", "http://consumer.example"],
+)
+def fixture_base_vws_url(*, request: pytest.FixtureRequest) -> str:
+    """Supply a URL for the adapter and its synthetic responses."""
+    url = request.param
+    assert isinstance(url, str)
+    return url
+
+
+@pytest.fixture(autouse=True)
+def fixture_check_response_policy_cleanup() -> Generator[None]:
+    """Check the default before setup and after backend fixtures finish.
+
+    Yields:
+        ``None``, while the test and its backend fixtures run.
+    """
+    response = fake_response(
+        status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+        text=_ALLOWANCE_BODY,
+        url="https://vws.vuforia.com/modeltargets/datasets",
+    )
+    with pytest.raises(expected_exception=AssertionError):
+        assert_model_target_status(
+            response=response,
+            status_codes=HTTPStatus.CREATED,
+        )
+    yield
+    with pytest.raises(expected_exception=AssertionError):
+        assert_model_target_status(
+            response=response,
+            status_codes=HTTPStatus.CREATED,
+        )
+
+
+@pytest.mark.usefixtures("verify_model_target_mock_vuforia")
+def test_real_adapter_handles_allowance(*, base_vws_url: str) -> None:
+    """The real adapter handles allowance rejections at its supplied
+    URL.
+    """
+    response = fake_response(
+        status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+        text=_ALLOWANCE_BODY,
+        url=f"{base_vws_url}/modeltargets/datasets",
+    )
+    with pytest.raises(
+        expected_exception=pytest.skip.Exception,
+        match="out of Model Target training allowance",
+    ):
+        assert_model_target_status(
+            response=response,
+            status_codes=HTTPStatus.CREATED,
+        )
+
+
+@pytest.mark.usefixtures("verify_model_target_mock_vuforia")
+def test_expected_allowance_status(*, base_vws_url: str) -> None:
+    """An expected allowance response remains an ordinary assertion."""
+    response = fake_response(
+        status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+        text=_ALLOWANCE_BODY,
+        url=f"{base_vws_url}/modeltargets/datasets",
+    )
+    assert_model_target_status(
+        response=response,
+        status_codes=HTTPStatus.UNPROCESSABLE_ENTITY,
+    )
+
+
+@pytest.mark.usefixtures("verify_model_target_mock_vuforia")
+def test_other_unexpected_status(*, base_vws_url: str) -> None:
+    """The real adapter does not hide unrelated response failures."""
+    response = fake_response(
+        status_code=HTTPStatus.BAD_REQUEST,
+        text='{"error":{"code":"VALIDATION_ERROR"}}',
+        url=f"{base_vws_url}/modeltargets/datasets",
+    )
+    with pytest.raises(
+        expected_exception=AssertionError,
+        match="VALIDATION_ERROR",
+    ):
+        assert_model_target_status(
+            response=response,
+            status_codes=HTTPStatus.CREATED,
+        )
+
+
+def test_nested_response_handlers() -> None:
+    """Nested handlers restore the outer handler and finally the
+    default.
+    """
+    calls: list[str] = []
+    response = fake_response(
+        status_code=HTTPStatus.BAD_REQUEST,
+        text="{}",
+        url="http://consumer.example/modeltargets/datasets",
+    )
+
+    def outer_handler(*, response: Response | requests.Response) -> None:
+        """Record a response handled by the outer context."""
+        assert response.status_code == HTTPStatus.BAD_REQUEST
+        calls.append("outer")
+
+    def inner_handler(*, response: Response | requests.Response) -> None:
+        """Record a response handled by the inner context."""
+        assert response.status_code == HTTPStatus.BAD_REQUEST
+        calls.append("inner")
+
+    with handling_unexpected_model_target_responses(handler=outer_handler):
+        with pytest.raises(expected_exception=AssertionError):
+            assert_model_target_status(
+                response=response,
+                status_codes=HTTPStatus.CREATED,
+            )
+        with (
+            handling_unexpected_model_target_responses(handler=inner_handler),
+            pytest.raises(expected_exception=AssertionError),
+        ):
+            assert_model_target_status(
+                response=response,
+                status_codes=HTTPStatus.CREATED,
+            )
+        with pytest.raises(expected_exception=AssertionError):
+            assert_model_target_status(
+                response=response,
+                status_codes=HTTPStatus.CREATED,
+            )
+
+    with pytest.raises(expected_exception=AssertionError):
+        assert_model_target_status(
+            response=response,
+            status_codes=HTTPStatus.CREATED,
+        )
+    assert calls == ["outer", "inner", "outer"]

tests/mock_vws/utils/assertions.py

@@ -4,19 +4,19 @@
 import datetime
 import email.utils
 import json
-import textwrap
 from collections.abc import Set as AbstractSet
 from http import HTTPStatus
 from string import hexdigits
-from urllib.parse import urlparse
 from zoneinfo import ZoneInfo
 
-import pytest
 import requests
 from beartype import beartype
 from vws.response import Response
 
 from mock_vws._constants import ResultCodes
+from tests.mock_vws.utils.model_target_response_policy import (
+    handle_unexpected_model_target_response,
+)
 
 
 @beartype
@@ -218,34 +218,6 @@ def assert_vws_response(
     assert_valid_date_header(response=response)
 
 
-_TRAINING_ALLOWANCE_EXCEEDED = "TRAINING_ALLOWANCE_EXCEEDED"
-
-# The first line of the failure message, because a truncated failure summary
-# in a CI log has to be enough to tell this apart from a real regression.
-_TRAINING_ALLOWANCE_EXCEEDED_HEADLINE = (
-    "The Vuforia account is out of Model Target training allowance - "
-    "this is not a failure of the code under test."
-)
-
-_TRAINING_ALLOWANCE_EXCEEDED_EXPLANATION = textwrap.dedent(
-    text="""\
-    ``TRAINING_ALLOWANCE_EXCEEDED`` is a quota on the Vuforia account behind
-    ``MODEL_TARGET_VUFORIA_CLIENT_ID``, not a fault in the code under test or
-    in this request. Every CI job shares one set of Model Target credentials,
-    so the allowance is consumed across all jobs and all concurrent runs.
-
-    The allowance has to be raised, or reset, on the Vuforia account for this
-    request to succeed against the real backend again. Signed dataset types,
-    such as an advanced dataset with a state-based configuration, exhaust it
-    first, but with enough traffic unsigned dataset creation is rejected too.
-
-    An unexpected allowance rejection is reported as an expected failure
-    (xfail) rather than a test failure, so that the shared account running
-    dry does not turn CI red: the test passes again automatically once the
-    allowance recovers.""",
-)
-
-
 @beartype
 def assert_model_target_status(
     *,
@@ -254,11 +226,8 @@ def assert_model_target_status(
 ) -> None:
     """Assert the status code of a Model Target Web API response.
 
-    A bare ``assert response.status_code == ...`` hides the response body,
-    which is where the real Model Target Web API says *why* it rejected a
-    request. That matters most when the reason has nothing to do with the
-    request, such as an exhausted account allowance, so this helper always
-    reports the body and calls out that case by name.
+    An unexpected status reports the URL and body. A backend may supply
+    a response handler for restrictions of the service it runs against.
 
     Args:
         response: The response returned by a request to the Model Target Web
@@ -268,12 +237,7 @@ def assert_model_target_status(
 
     Raises:
         AssertionError: The response does not have one of the expected status
-            codes.
-        Exception: The response unexpectedly reports a
-            ``TRAINING_ALLOWANCE_EXCEEDED`` rejection. That is the shared
-            Vuforia account running out of Model Target training allowance,
-            not a fault in the code under test, so it is an expected failure
-            rather than a test failure.
+            codes and the backend handler has not handled the rejection.
     """
     expected = (
         frozenset({status_codes})
@@ -283,36 +247,14 @@ def assert_model_target_status(
     if response.status_code in expected:
         return
 
+    handle_unexpected_model_target_response(response=response)
     expected_description = " or ".join(
         sorted(f"{item} {item.name}" for item in expected)
     )
-    allowance_exceeded = _TRAINING_ALLOWANCE_EXCEEDED in response.text
-    if (
-        allowance_exceeded
-        and urlparse(url=response.url).hostname == "vws.vuforia.com"
-    ):
-        pytest.skip(reason=_TRAINING_ALLOWANCE_EXCEEDED_HEADLINE)
-    headline = (
-        [_TRAINING_ALLOWANCE_EXCEEDED_HEADLINE] if allowance_exceeded else []
-    )
-    explanation = (
-        [_TRAINING_ALLOWANCE_EXCEEDED_EXPLANATION]
-        if allowance_exceeded
-        else []
-    )
-    message = "\n\n".join(
-        [
-            *headline,
-            (
-                f"Expected {expected_description} from {response.url}, "
-                f"got {response.status_code}."
-            ),
-            f"Response body:\n{response.text}",
-            *explanation,
-        ],
+    message = (
+        f"Expected {expected_description} from {response.url}, "
+        f"got {response.status_code}.\n\nResponse body:\n{response.text}"
     )
-    if allowance_exceeded:
-        pytest.xfail(reason=message)
     raise AssertionError(message)