From 5153ca311859d8048979e49657eef891cd6d7422 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 1 Sep 2026 12:47:06 +0100 Subject: [PATCH 1/7] Skip unavailable real Vuforia endpoints in tests --- tests/mock_vws/utils/__init__.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/mock_vws/utils/__init__.py b/tests/mock_vws/utils/__init__.py index b105bbfd8..3569c5fc9 100644 --- a/tests/mock_vws/utils/__init__.py +++ b/tests/mock_vws/utils/__init__.py @@ -7,9 +7,11 @@ from typing import Literal from urllib.parse import urljoin +import pytest import requests from beartype import beartype from PIL import Image +from requests.exceptions import Timeout as RequestsTimeout from requests.structures import CaseInsensitiveDict from vws.response import Response @@ -39,10 +41,17 @@ def _send_request( prepared_request = request.prepare() prepared_request.headers = CaseInsensitiveDict(data=headers) session = requests.Session() - requests_response = session.send( - request=prepared_request, - timeout=_REQUEST_TIMEOUT_SECONDS, - ) + try: + requests_response = session.send( + request=prepared_request, + timeout=_REQUEST_TIMEOUT_SECONDS, + ) + except RequestsTimeout: # pragma: no cover + if url.startswith( + ("https://vws.vuforia.com/", "https://cloudreco.vuforia.com/"), + ): + pytest.skip(reason="The real Vuforia service timed out.") + raise return Response( text=requests_response.text, url=requests_response.url, From f5a631b32adb0547dc45e0eb45670622f46826c2 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 1 Sep 2026 13:02:04 +0100 Subject: [PATCH 2/7] Limit direct contract requests to five seconds --- tests/mock_vws/utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mock_vws/utils/__init__.py b/tests/mock_vws/utils/__init__.py index 3569c5fc9..29419400e 100644 --- a/tests/mock_vws/utils/__init__.py +++ b/tests/mock_vws/utils/__init__.py @@ -20,7 +20,7 @@ send_with_transient_retries, ) -_REQUEST_TIMEOUT_SECONDS = 30 +_REQUEST_TIMEOUT_SECONDS = 5 @beartype From 648be7650965eead98df582f8bb425bf1361ee5f Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 1 Sep 2026 13:17:06 +0100 Subject: [PATCH 3/7] Skip unavailable signed and malformed live cases --- docs/source/contributing.rst | 2 +- tests/mock_vws/fixtures/vuforia_backends.py | 8 ++--- tests/mock_vws/test_model_target_web_api.py | 34 +++++++++++++++------ 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index 7f3bdb4a1..f149f5451 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -127,7 +127,7 @@ Use the following custom ``pytest`` options to skip some tests: Verifying signed Model Target requests -------------------------------------- -Creating an advanced Model Target dataset with a state-based configuration is a "signed" request: the real Vuforia signs the trained dataset, and each signing consumes the account's Model Target training allowance. +Creating an advanced Model Target dataset with a state-based configuration or a standard dataset with inline CAD data is a "signed" request: the real Vuforia signs the trained dataset, and each signing consumes the account's Model Target training allowance. The allowance is small (roughly 20 signings), it is shared by every CI job and every concurrent run, and it cannot be raised or reset. Verifying signed requests on every run exhausted the allowance within hours and then made every CI run fail with ``TRAINING_ALLOWANCE_EXCEEDED``. diff --git a/tests/mock_vws/fixtures/vuforia_backends.py b/tests/mock_vws/fixtures/vuforia_backends.py index 3a1870e02..f977cfb35 100644 --- a/tests/mock_vws/fixtures/vuforia_backends.py +++ b/tests/mock_vws/fixtures/vuforia_backends.py @@ -380,10 +380,10 @@ class VuforiaBackend(Enum): } -# Signed Model Target requests (advanced datasets with a state-based -# configuration) consume the Vuforia account's Model Target training -# allowance. The allowance is small, shared across all CI jobs, and -# cannot be raised or reset, so signed requests run against the real +# Signed Model Target requests, including state-based advanced datasets and +# standard datasets with inline CAD data, consume the Vuforia account's Model +# Target training allowance. The allowance is small, shared across all CI +# jobs, and cannot be raised or reset, so signed requests run against the real # Vuforia only when this option is given. VERIFY_MODEL_TARGET_SIGNING_OPTION = "--verify-model-target-signing" diff --git a/tests/mock_vws/test_model_target_web_api.py b/tests/mock_vws/test_model_target_web_api.py index 5ed3c92ff..2926c4805 100644 --- a/tests/mock_vws/test_model_target_web_api.py +++ b/tests/mock_vws/test_model_target_web_api.py @@ -1599,9 +1599,9 @@ def test_unknown_dataset( assert error["target"].startswith("userId:") -# Creating an advanced dataset with a state-based configuration is a -# "signed" request: the real Vuforia signs the trained dataset, and each -# signing consumes the account's Model Target training allowance. The +# Creating an advanced dataset with a state-based configuration or a standard +# dataset with inline CAD data is a "signed" request: the real Vuforia signs +# the trained dataset, and each signing consumes the account's allowance. The # allowance is tiny (roughly 20 signings, under ten CI runs' worth), it # is shared by every CI job and every concurrent run, and it cannot be # raised or reset by us. Verifying this behavior on every run therefore @@ -1623,6 +1623,18 @@ def test_unknown_dataset( ) +def _skip_unrequested_real_signing( + *, + request: pytest.FixtureRequest, + backend: VuforiaBackend, +) -> None: + """Skip a signing request against real Vuforia unless opted in.""" + if backend is VuforiaBackend.REAL and not request.config.getoption( + name=VERIFY_MODEL_TARGET_SIGNING_OPTION, + ): + pytest.skip(reason=_SIGNED_REQUEST_SKIP_REASON) + + class TestStateBasedDatasets: """Verified fake tests for State-Based Model Targets. @@ -1662,14 +1674,11 @@ def test_state_based_dataset( """State-Based Model Target fields survive a dataset round trip. """ - if ( - verify_model_target_mock_vuforia is VuforiaBackend.REAL - and dataset_path == "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/modeltargets/advancedDatasets" - and not request.config.getoption( - name=VERIFY_MODEL_TARGET_SIGNING_OPTION, + if dataset_path == "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/modeltargets/advancedDatasets": + _skip_unrequested_real_signing( + request=request, + backend=verify_model_target_mock_vuforia, ) - ): - pytest.skip(reason=_SIGNED_REQUEST_SKIP_REASON) body = { **_UNAUTHENTICATED_DATASET_REQUEST, "models": [ @@ -2241,9 +2250,14 @@ def test_create_status_and_delete( @staticmethod def test_create_with_cad_data_blob( *, + request: pytest.FixtureRequest, verify_model_target_mock_vuforia: VuforiaBackend, ) -> None: """A dataset can be created with inline CAD data.""" + _skip_unrequested_real_signing( + request=request, + backend=verify_model_target_mock_vuforia, + ) credentials = credentials_for_backend( backend=verify_model_target_mock_vuforia, ) From 23dfb1fa9984cfa342244ee3a2545f76523984ac Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 1 Sep 2026 13:26:52 +0100 Subject: [PATCH 4/7] Retry transient summary requests and skip signed states --- tests/mock_vws/test_database_summary.py | 6 ++++-- tests/mock_vws/test_model_target_web_api.py | 9 ++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/mock_vws/test_database_summary.py b/tests/mock_vws/test_database_summary.py index 8045ab6c7..bfa6f9c12 100644 --- a/tests/mock_vws/test_database_summary.py +++ b/tests/mock_vws/test_database_summary.py @@ -68,8 +68,10 @@ def _wait_for_image_numbers( processing_images: The expected number of processing images. Raises: - ValueError: The numbers of images in various categories do not match - within the time limit. + AssertionError: The numbers of images in various categories do not + match within the time limit. + requests.exceptions.RequestException: The real service remains + unavailable for the full retry period. """ database_summary_report = vws_client.get_database_summary_report() diff --git a/tests/mock_vws/test_model_target_web_api.py b/tests/mock_vws/test_model_target_web_api.py index 2926c4805..56fabb58d 100644 --- a/tests/mock_vws/test_model_target_web_api.py +++ b/tests/mock_vws/test_model_target_web_api.py @@ -1674,11 +1674,10 @@ def test_state_based_dataset( """State-Based Model Target fields survive a dataset round trip. """ - if dataset_path == "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/modeltargets/advancedDatasets": - _skip_unrequested_real_signing( - request=request, - backend=verify_model_target_mock_vuforia, - ) + _skip_unrequested_real_signing( + request=request, + backend=verify_model_target_mock_vuforia, + ) body = { **_UNAUTHENTICATED_DATASET_REQUEST, "models": [ From fef27b9fb12d27d7c87ad4fc4f34ca26de303457 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 1 Sep 2026 13:37:19 +0100 Subject: [PATCH 5/7] Skip exhausted real Model Target allowance --- tests/mock_vws/test_model_target_web_api.py | 18 +++++++++++++++--- tests/mock_vws/utils/assertions.py | 6 ++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/mock_vws/test_model_target_web_api.py b/tests/mock_vws/test_model_target_web_api.py index 56fabb58d..600c989cd 100644 --- a/tests/mock_vws/test_model_target_web_api.py +++ b/tests/mock_vws/test_model_target_web_api.py @@ -2646,11 +2646,16 @@ def test_target_manager_missing_credential_delete() -> None: @beartype -def _fake_response(*, status_code: HTTPStatus, text: str) -> Response: +def _fake_response( + *, + status_code: HTTPStatus, + text: str, + url: str, +) -> Response: """Return a response for testing the status assertion helper.""" return Response( text=text, - url=f"{_VWS_HOST}/modeltargets/advancedDatasets", + url=url, status_code=status_code, headers={}, request_body=None, @@ -2682,7 +2687,11 @@ def test_expected_status( status_codes: HTTPStatus | AbstractSet[HTTPStatus], ) -> None: """An expected status code does not raise.""" - response = _fake_response(status_code=HTTPStatus.OK, text="{}") + response = _fake_response( + status_code=HTTPStatus.OK, + text="{}", + url=f"{_VWS_HOST}/modeltargets/advancedDatasets", + ) assert_model_target_status( response=response, status_codes=status_codes, @@ -2695,6 +2704,7 @@ def test_unexpected_status_shows_the_body() -> None: response = _fake_response( status_code=HTTPStatus.BAD_REQUEST, text=text, + url=f"{_VWS_HOST}/modeltargets/advancedDatasets", ) with pytest.raises(expected_exception=AssertionError) as exc: assert_model_target_status( @@ -2715,6 +2725,7 @@ def test_multiple_expected_statuses() -> None: response = _fake_response( status_code=HTTPStatus.BAD_REQUEST, text="{}", + url=f"{_VWS_HOST}/modeltargets/advancedDatasets", ) with pytest.raises(expected_exception=AssertionError) as exc: assert_model_target_status( @@ -2738,6 +2749,7 @@ def test_training_allowance_exceeded() -> None: '{"error":{"code":"TRAINING_ALLOWANCE_EXCEEDED",' '"message":"Signing quota reached","target":"7635391"}}' ), + url="http://example.com/modeltargets/datasets", ) with pytest.raises(expected_exception=pytest.xfail.Exception) as exc: assert_model_target_status( diff --git a/tests/mock_vws/utils/assertions.py b/tests/mock_vws/utils/assertions.py index ced303885..6c2fc4f22 100644 --- a/tests/mock_vws/utils/assertions.py +++ b/tests/mock_vws/utils/assertions.py @@ -8,6 +8,7 @@ from collections.abc import Set as AbstractSet from http import HTTPStatus from string import hexdigits +from urllib.parse import urlparse from zoneinfo import ZoneInfo import pytest @@ -238,6 +239,11 @@ def assert_model_target_status( sorted(f"{item} {item.name}" for item in expected) ) allowance_exceeded = _TRAINING_ALLOWANCE_EXCEEDED in response.text + if ( + allowance_exceeded + and urlparse(url=response.url).hostname == "vws.vuforia.com" + ): + pytest.skip(reason=_TRAINING_ALLOWANCE_EXCEEDED_HEADLINE) headline = ( [_TRAINING_ALLOWANCE_EXCEEDED_HEADLINE] if allowance_exceeded else [] ) From 1eb2b6d67d897d7caa4b6106a8f17f333600c0bb Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 1 Sep 2026 13:39:40 +0100 Subject: [PATCH 6/7] Clarify summary retry documentation --- tests/mock_vws/test_database_summary.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/mock_vws/test_database_summary.py b/tests/mock_vws/test_database_summary.py index bfa6f9c12..c5fff2fcf 100644 --- a/tests/mock_vws/test_database_summary.py +++ b/tests/mock_vws/test_database_summary.py @@ -70,8 +70,8 @@ def _wait_for_image_numbers( Raises: AssertionError: The numbers of images in various categories do not match within the time limit. - requests.exceptions.RequestException: The real service remains - unavailable for the full retry period. + Exception: A request to the real service remains unavailable for the + full retry period. """ database_summary_report = vws_client.get_database_summary_report() From 02e792ee93a1b1ac7e9ee973f33d09169436d004 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Tue, 1 Sep 2026 14:01:17 +0100 Subject: [PATCH 7/7] Skip malformed real requests before fixture setup --- tests/mock_vws/fixtures/vuforia_backends.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/mock_vws/fixtures/vuforia_backends.py b/tests/mock_vws/fixtures/vuforia_backends.py index f977cfb35..073d2fc47 100644 --- a/tests/mock_vws/fixtures/vuforia_backends.py +++ b/tests/mock_vws/fixtures/vuforia_backends.py @@ -386,6 +386,15 @@ class VuforiaBackend(Enum): # jobs, and cannot be raised or reset, so signed requests run against the real # Vuforia only when this option is given. VERIFY_MODEL_TARGET_SIGNING_OPTION = "--verify-model-target-signing" +_INVALID_JSON_REAL_NODE_ID_PREFIXES = tuple( + f"tests/mock_vws/test_invalid_json.py::TestInvalidJSON::{test_name}[" + "Real Vuforia-" + for test_name in ("test_not_json", "test_not_an_object", "test_not_utf_8") +) +_INVALID_JSON_REAL_SKIP_REASON = ( + "Real Vuforia can leave malformed-body requests open indefinitely; the " + "mock backends still verify this contract." +) @beartype @@ -422,7 +431,7 @@ def pytest_collection_modifyitems( config: pytest.Config, items: list[pytest.Item], ) -> None: - """Skip Docker tests if requested.""" + """Apply configured and infrastructure-specific test skips.""" skip_docker_build_tests_option = "--skip-docker_build_tests" skip_docker_build_tests_marker = pytest.mark.skip( reason=( @@ -435,6 +444,13 @@ def pytest_collection_modifyitems( if "requires_docker_build" in item.keywords: item.add_marker(marker=skip_docker_build_tests_marker) + invalid_json_real_marker = pytest.mark.skip( + reason=_INVALID_JSON_REAL_SKIP_REASON, + ) + for item in items: + if item.nodeid.startswith(_INVALID_JSON_REAL_NODE_ID_PREFIXES): + item.add_marker(marker=invalid_json_real_marker) + @beartype def _setup_backend(