From 2d824cf3f859a33ca939f63e49ce3db7ae38203c Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 9 Oct 2026 01:54:08 +0200 Subject: [PATCH 1/6] test: set up traceable end-to-end pytest infrastructure --- .github/workflows/tests.yml | 42 ++++++++++- README.md | 1 + docs/BUILD | 5 +- docs/verification/verification_report.rst | 7 +- tests/end_to_end/testcases/BUILD | 38 ++++++++++ .../end_to_end/testcases/_blackbox_support.py | 66 +++++++++++++++++ tests/end_to_end/testcases/conftest.py | 70 +++++++++++++++++++ tests/end_to_end/testcases/test_cli.py | 40 +++++++++++ 8 files changed, 265 insertions(+), 4 deletions(-) create mode 100644 tests/end_to_end/testcases/BUILD create mode 100644 tests/end_to_end/testcases/_blackbox_support.py create mode 100644 tests/end_to_end/testcases/conftest.py create mode 100644 tests/end_to_end/testcases/test_cli.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 068f286..a928465 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,7 +67,7 @@ jobs: echo "collected $(find tests-report -name test.xml | wc -l) test.xml files" - uses: actions/upload-artifact@v4 with: - name: tests-report + name: unit-tests-report path: tests-report if-no-files-found: error retention-days: 3 @@ -85,6 +85,24 @@ jobs: disk-cache-key: end_to_end_tests - name: Run the end-to-end pipeline test (C++ + Rust consumer workspace) run: tests/end_to_end/run_end_to_end_test.sh + - name: Run the end-to-end pytest CLI case + run: bazel test --lockfile_mode=error //tests/end_to_end/testcases:blackbox_test + - name: Collect pytest JUnit results for the documentation build + if: always() + run: | + mkdir -p tests-report/tests/end_to_end/testcases/blackbox_test + if [ -f bazel-testlogs/tests/end_to_end/testcases/blackbox_test/test.xml ]; then + cp bazel-testlogs/tests/end_to_end/testcases/blackbox_test/test.xml \ + tests-report/tests/end_to_end/testcases/blackbox_test/test.xml + fi + - name: Upload end-to-end pytest results + if: always() + uses: actions/upload-artifact@v4 + with: + name: end-to-end-tests-report + path: tests-report + if-no-files-found: ignore + retention-days: 3 - name: Upload coverage report of the end-to-end workspace if: always() uses: actions/upload-artifact@v4 @@ -93,8 +111,28 @@ jobs: path: tests/end_to_end/consumer/coverage_artifacts.zip if-no-files-found: ignore retention-days: 10 + test_reports: + needs: [unit_tests, end_to_end_tests] + runs-on: ubuntu-24.04 + steps: + - name: Download unit test results + uses: actions/download-artifact@v4 + with: + name: unit-tests-report + path: tests-report + - name: Download end-to-end test results + uses: actions/download-artifact@v4 + with: + name: end-to-end-tests-report + path: tests-report + - uses: actions/upload-artifact@v4 + with: + name: tests-report + path: tests-report + if-no-files-found: error + retention-days: 3 docs: - needs: unit_tests + needs: test_reports uses: eclipse-score/cicd-workflows/.github/workflows/docs.yml@d2083d5dac0e309643d9f495e61342dfbaf07ed5 # main, 2026-09-24 permissions: contents: read diff --git a/README.md b/README.md index 1e51cec..b20c4f8 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,7 @@ bazel build --config=lint //score_coverage/... //tools/... //tests/... # ruff, bazel coverage --combined_report=lcov //tests/unit/score_coverage:all bazel run //tools:self_coverage_gate -- --min-lines 95 --min-branches 87 tests/end_to_end/run_end_to_end_test.sh # end-to-end (downloads LLVM + Ferrocene) +bazel test //tests/end_to_end/testcases:blackbox_test # pytest CLI case pre-commit run --all-files bazel run //tools:copyright.check ``` diff --git a/docs/BUILD b/docs/BUILD index c36e3b9..d0b8b51 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -23,5 +23,8 @@ docs( source_dir = ".", # Only these test.xml files become testcase needs (JUnit results under # bazel-testlogs/ or tests-report/ at the workspace root). - test_sources = ["tests/unit/score_coverage"], + test_sources = [ + "tests/unit/score_coverage", + "tests/end_to_end/testcases", + ], ) diff --git a/docs/verification/verification_report.rst b/docs/verification/verification_report.rst index 119a643..eac432f 100644 --- a/docs/verification/verification_report.rst +++ b/docs/verification/verification_report.rst @@ -81,6 +81,9 @@ Test inventory - 17 - scope_transitive, scope_excludes, scope_baseline_objects, scope_gcno, scope_platform + * - ``//tests/end_to_end/testcases:blackbox_test`` + - 1 + - gate_no_verdict * - ``tests/end_to_end/run_end_to_end_test.sh`` (27 end-to-end checks) - 27 - validation_ground_truth, instrumentation_hint, scope_platform, report_baseline_zero, report_relative_paths, @@ -96,7 +99,9 @@ carries ``@verifies()``, which writes ``PartiallyVerifies``, ``TestType`` and ``DerivationTechnique`` into the JUnit XML of the test run, and docs-as-code turns the results into ``testcase`` needs with back-links on the requirements (``testlink`` column below, with the execution result of each -case). The links reflect the test run that preceded the documentation build. +case). End-to-end pytest functions attach the same metadata using the +``score_pytest`` decorator. The links reflect the test run that preceded the +documentation build. .. needtable:: Requirements and the tests that verify them :types: tool_req diff --git a/tests/end_to_end/testcases/BUILD b/tests/end_to_end/testcases/BUILD new file mode 100644 index 0000000..78a219b --- /dev/null +++ b/tests/end_to_end/testcases/BUILD @@ -0,0 +1,38 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("@score_tools//score_pytest:pytest.bzl", "score_pytest") + +score_pytest( + name = "blackbox_test", + size = "large", + timeout = "long", + srcs = ["test_cli.py"], + # score_pytest passes srcs to pytest as collection paths. Keep shared + # fixtures and helpers in runfiles without presenting them as test modules. + data = [ + "_blackbox_support.py", + "conftest.py", + ], + imports = ["."], + # These scenarios read a consumer workspace from the checkout and invoke + # the host Bazel installation. Those inputs are outside this target's + # runfiles, so their changes cannot invalidate Bazel's test-result cache. + # The external tag makes Bazel execute the tests on every invocation; + # local keeps the nested Bazel processes on the machine with the checkout. + tags = [ + "external", + "integration", + "local", + ], +) diff --git a/tests/end_to_end/testcases/_blackbox_support.py b/tests/end_to_end/testcases/_blackbox_support.py new file mode 100644 index 0000000..ed6a63d --- /dev/null +++ b/tests/end_to_end/testcases/_blackbox_support.py @@ -0,0 +1,66 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Local Bazel execution and requirement metadata for end-to-end pytest cases.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +from attribute_plugin import Decorator, DerivationTechnique, add_test_properties + + +def run_bazel( + workspace: Path, + args: list[str], + *, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + """Run a consumer-facing Bazel command and capture its user-visible output.""" + command = shutil.which("bazel") + assert command, "Bazel must be available on PATH to run the integration scenarios" + process_env = os.environ.copy() + process_env.pop("GITHUB_STEP_SUMMARY", None) + if env: + process_env.update(env) + # TEST_TMPDIR makes nested Bazel put its output-user root and download cache + # in the outer test's disposable directory. Use the host cache while the + # copied consumer workspace still isolates build outputs and source edits. + process_env.pop("TEST_TMPDIR", None) + return subprocess.run( + [command, *args], + cwd=workspace, + env=process_env, + capture_output=True, + check=False, + text=True, + ) + + +def assert_exit_code(result: subprocess.CompletedProcess[str], expected: int) -> None: + assert result.returncode == expected, ( + f"command: {' '.join(result.args)}\n" + f"expected exit code {expected}, got {result.returncode}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + + +def verifies(*requirements: str, derivation: DerivationTechnique = "requirements-analysis") -> Decorator: + """Attach score_pytest requirement metadata to an interface scenario.""" + return add_test_properties( + partially_verifies=list(requirements), + test_type="interface-test", + derivation_technique=derivation, + ) diff --git a/tests/end_to_end/testcases/conftest.py b/tests/end_to_end/testcases/conftest.py new file mode 100644 index 0000000..1fbb067 --- /dev/null +++ b/tests/end_to_end/testcases/conftest.py @@ -0,0 +1,70 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""An isolated consumer workspace for local end-to-end pytest cases.""" + +import shutil +from pathlib import Path + +import pytest + +# This local-only target needs symlinked runfiles: resolve() follows conftest.py +# back into the checkout. Copied runfiles and remote execution cannot supply +# the full repository tree or the host Bazel installation used by the fixtures. +_REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +_CONSUMER_WORKSPACE_DIRECTORY = "tests/end_to_end/consumer" + + +def _ignore_generated_workspace_files(_directory: str, names: list[str]) -> set[str]: + """Keep the consumer copy limited to source and test fixtures.""" + generated = { + ".git", + ".ub_cache", + ".venv", + ".vscode", + ".pytest_cache", + ".ruff_cache", + ".integration-test-venv", + "__pycache__", + "external", + "_build", + "coverage_artifact", + "coverage_artifacts.zip", + "coverage_gcov", + "coverage_linux", + "gcov_artifacts_dir", + "gcov_run.log", + "gcov_summary.md", + "lcov.dat", + "link_check", + "test-reports", + "tests-report", + "typo_dir", + "user.bazelrc", + } + return {name for name in names if name in generated or name.startswith("bazel-")} + + +@pytest.fixture(scope="session") +def consumer_workspace(tmp_path_factory: pytest.TempPathFactory) -> Path: + """Copy sources so nested Bazel outputs and source fault injection stay isolated.""" + if not (_REPOSITORY_ROOT / "MODULE.bazel").is_file(): + pytest.fail( + "Black-box tests require local execution with symlinked runfiles pointing " + f"into the source checkout; MODULE.bazel is missing at {_REPOSITORY_ROOT}", + pytrace=False, + ) + if shutil.which("bazel") is None: + pytest.fail("Black-box tests require the host Bazel installation on PATH", pytrace=False) + root = tmp_path_factory.mktemp("score-coverage-blackbox") / "repo" + shutil.copytree(_REPOSITORY_ROOT, root, ignore=_ignore_generated_workspace_files) + return root / _CONSUMER_WORKSPACE_DIRECTORY diff --git a/tests/end_to_end/testcases/test_cli.py b/tests/end_to_end/testcases/test_cli.py new file mode 100644 index 0000000..9e1d581 --- /dev/null +++ b/tests/end_to_end/testcases/test_cli.py @@ -0,0 +1,40 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""A CLI case exercising the end-to-end pytest infrastructure.""" + +from pathlib import Path + +from _blackbox_support import assert_exit_code, run_bazel, verifies + + +@verifies("tool_req__coverage_gate_no_verdict") +def test_missing_report_returns_no_verdict(consumer_workspace: Path) -> None: + """A consumer without coverage data gets exit 2 and a diagnostic explaining how to collect it.""" + report = consumer_workspace / "bazel-out/_coverage/_coverage_report.dat" + backup = report.with_name("missing-report-backup.dat") + # A report installed by another case must survive this missing-input check. + if report.is_file(): + report.rename(backup) + try: + result = run_bazel( + consumer_workspace, + ["run", "@score_coverage//:generate_coverage_html"], + env={"COVERAGE_THRESHOLD": "10"}, + ) + finally: + if backup.is_file(): + backup.replace(report) + assert_exit_code(result, 2) + output = result.stdout + result.stderr + assert "Coverage report not found" in output + assert "Run 'bazel coverage" in output From bbe8b15a4ace1c218e07e5a0bf59861249d35200 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 9 Oct 2026 16:57:34 +0200 Subject: [PATCH 2/6] fix: address feedback for PR #33 --- tests/end_to_end/testcases/BUILD | 10 +++++----- .../end_to_end/testcases/_blackbox_support.py | 18 +++++++++++++----- tests/end_to_end/testcases/conftest.py | 13 +++++++++---- tests/end_to_end/testcases/test_cli.py | 19 ++++++------------- 4 files changed, 33 insertions(+), 27 deletions(-) diff --git a/tests/end_to_end/testcases/BUILD b/tests/end_to_end/testcases/BUILD index 78a219b..7d4294a 100644 --- a/tests/end_to_end/testcases/BUILD +++ b/tests/end_to_end/testcases/BUILD @@ -25,11 +25,11 @@ score_pytest( "conftest.py", ], imports = ["."], - # These scenarios read a consumer workspace from the checkout and invoke - # the host Bazel installation. Those inputs are outside this target's - # runfiles, so their changes cannot invalidate Bazel's test-result cache. - # The external tag makes Bazel execute the tests on every invocation; - # local keeps the nested Bazel processes on the machine with the checkout. + # The checkout and host Bazel installation are runtime inputs outside this + # target's runfiles, so changes to them cannot invalidate a cached result. + # `external` disables test caching; `integration` lets callers select these + # scenarios with --test_tag_filters; `local` keeps the host Bazel process + # with the checkout instead of using sandboxed or remote execution. tags = [ "external", "integration", diff --git a/tests/end_to_end/testcases/_blackbox_support.py b/tests/end_to_end/testcases/_blackbox_support.py index ed6a63d..246a807 100644 --- a/tests/end_to_end/testcases/_blackbox_support.py +++ b/tests/end_to_end/testcases/_blackbox_support.py @@ -28,16 +28,22 @@ def run_bazel( *, env: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: - """Run a consumer-facing Bazel command and capture its user-visible output.""" + """Run a consumer-facing Bazel command and capture its user-visible output. + + ``env`` overrides selected inherited variables for a single invocation, + while preserving the rest of the host environment. + """ command = shutil.which("bazel") assert command, "Bazel must be available on PATH to run the integration scenarios" process_env = os.environ.copy() + # A nested coverage command must not append its output to the enclosing CI + # step's summary; only the outer workflow owns that summary file. process_env.pop("GITHUB_STEP_SUMMARY", None) if env: process_env.update(env) - # TEST_TMPDIR makes nested Bazel put its output-user root and download cache - # in the outer test's disposable directory. Use the host cache while the - # copied consumer workspace still isolates build outputs and source edits. + # Bazel uses TEST_TMPDIR as the nested command's default output root. Remove + # the outer test's disposable value so nested Bazel can reuse the host + # download cache; the private consumer workspace still isolates its outputs. process_env.pop("TEST_TMPDIR", None) return subprocess.run( [command, *args], @@ -58,7 +64,9 @@ def assert_exit_code(result: subprocess.CompletedProcess[str], expected: int) -> def verifies(*requirements: str, derivation: DerivationTechnique = "requirements-analysis") -> Decorator: - """Attach score_pytest requirement metadata to an interface scenario.""" + """Attach requirement metadata to a plain pytest interface scenario.""" + # Keep this adapter temporary until the project settles how score_pytest + # and sphinx-needs-test-reports should provide metadata for pytest functions. return add_test_properties( partially_verifies=list(requirements), test_type="interface-test", diff --git a/tests/end_to_end/testcases/conftest.py b/tests/end_to_end/testcases/conftest.py index 1fbb067..6618744 100644 --- a/tests/end_to_end/testcases/conftest.py +++ b/tests/end_to_end/testcases/conftest.py @@ -54,9 +54,9 @@ def _ignore_generated_workspace_files(_directory: str, names: list[str]) -> set[ return {name for name in names if name in generated or name.startswith("bazel-")} -@pytest.fixture(scope="session") -def consumer_workspace(tmp_path_factory: pytest.TempPathFactory) -> Path: - """Copy sources so nested Bazel outputs and source fault injection stay isolated.""" +@pytest.fixture +def consumer_workspace(tmp_path: Path) -> Path: + """Give each scenario a private consumer workspace and Bazel output tree.""" if not (_REPOSITORY_ROOT / "MODULE.bazel").is_file(): pytest.fail( "Black-box tests require local execution with symlinked runfiles pointing " @@ -65,6 +65,11 @@ def consumer_workspace(tmp_path_factory: pytest.TempPathFactory) -> Path: ) if shutil.which("bazel") is None: pytest.fail("Black-box tests require the host Bazel installation on PATH", pytrace=False) - root = tmp_path_factory.mktemp("score-coverage-blackbox") / "repo" + # The consumer MODULE.bazel resolves score_coverage with + # local_path_override(path = "../../.."). Keep that root-relative layout + # intact so nested Bazel uses a private copy of the repository sources. + # A fresh copy per test also prevents Bazel outputs or fault-injection edits + # from making another scenario depend on test order. + root = tmp_path / "repo" shutil.copytree(_REPOSITORY_ROOT, root, ignore=_ignore_generated_workspace_files) return root / _CONSUMER_WORKSPACE_DIRECTORY diff --git a/tests/end_to_end/testcases/test_cli.py b/tests/end_to_end/testcases/test_cli.py index 9e1d581..8c8ab20 100644 --- a/tests/end_to_end/testcases/test_cli.py +++ b/tests/end_to_end/testcases/test_cli.py @@ -21,19 +21,12 @@ def test_missing_report_returns_no_verdict(consumer_workspace: Path) -> None: """A consumer without coverage data gets exit 2 and a diagnostic explaining how to collect it.""" report = consumer_workspace / "bazel-out/_coverage/_coverage_report.dat" - backup = report.with_name("missing-report-backup.dat") - # A report installed by another case must survive this missing-input check. - if report.is_file(): - report.rename(backup) - try: - result = run_bazel( - consumer_workspace, - ["run", "@score_coverage//:generate_coverage_html"], - env={"COVERAGE_THRESHOLD": "10"}, - ) - finally: - if backup.is_file(): - backup.replace(report) + assert not report.exists(), "Each CLI case must start without a generated coverage report" + result = run_bazel( + consumer_workspace, + ["run", "@score_coverage//:generate_coverage_html"], + env={"COVERAGE_THRESHOLD": "10"}, + ) assert_exit_code(result, 2) output = result.stdout + result.stderr assert "Coverage report not found" in output From e4fd1cede1a7a0715ad1fede974d5e88807008a6 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 9 Oct 2026 17:36:10 +0200 Subject: [PATCH 3/6] test: reuse the consumer workspace and extracted toolchains --- README.md | 7 +++ .../end_to_end/testcases/_blackbox_support.py | 4 +- tests/end_to_end/testcases/conftest.py | 49 +++---------------- tests/end_to_end/testcases/test_cli.py | 19 ++++--- 4 files changed, 28 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index b20c4f8..b0c60c6 100644 --- a/README.md +++ b/README.md @@ -77,3 +77,10 @@ bazel test //tests/end_to_end/testcases:blackbox_test # pytest CLI case pre-commit run --all-files bazel run //tools:copyright.check ``` + +The end-to-end commands use the checked-in consumer workspace and reuse its +Bazel output base, including extracted toolchains. This output base is separate +from the main repository's, so nested Bazel commands do not wait on the outer +test's lock. Run the end-to-end commands sequentially because they share the +consumer's coverage report; the pytest CLI case temporarily removes an existing +report and restores it afterwards. diff --git a/tests/end_to_end/testcases/_blackbox_support.py b/tests/end_to_end/testcases/_blackbox_support.py index 246a807..08fab18 100644 --- a/tests/end_to_end/testcases/_blackbox_support.py +++ b/tests/end_to_end/testcases/_blackbox_support.py @@ -42,8 +42,8 @@ def run_bazel( if env: process_env.update(env) # Bazel uses TEST_TMPDIR as the nested command's default output root. Remove - # the outer test's disposable value so nested Bazel can reuse the host - # download cache; the private consumer workspace still isolates its outputs. + # the outer test's disposable value so nested Bazel can reuse the consumer's + # normal output base, extracted toolchains and the host download cache. process_env.pop("TEST_TMPDIR", None) return subprocess.run( [command, *args], diff --git a/tests/end_to_end/testcases/conftest.py b/tests/end_to_end/testcases/conftest.py index 6618744..bd79796 100644 --- a/tests/end_to_end/testcases/conftest.py +++ b/tests/end_to_end/testcases/conftest.py @@ -10,7 +10,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -"""An isolated consumer workspace for local end-to-end pytest cases.""" +"""The checked-in consumer workspace for local end-to-end pytest cases.""" import shutil from pathlib import Path @@ -19,44 +19,14 @@ # This local-only target needs symlinked runfiles: resolve() follows conftest.py # back into the checkout. Copied runfiles and remote execution cannot supply -# the full repository tree or the host Bazel installation used by the fixtures. +# the consumer workspace or the host Bazel installation used by the fixtures. _REPOSITORY_ROOT = Path(__file__).resolve().parents[3] _CONSUMER_WORKSPACE_DIRECTORY = "tests/end_to_end/consumer" -def _ignore_generated_workspace_files(_directory: str, names: list[str]) -> set[str]: - """Keep the consumer copy limited to source and test fixtures.""" - generated = { - ".git", - ".ub_cache", - ".venv", - ".vscode", - ".pytest_cache", - ".ruff_cache", - ".integration-test-venv", - "__pycache__", - "external", - "_build", - "coverage_artifact", - "coverage_artifacts.zip", - "coverage_gcov", - "coverage_linux", - "gcov_artifacts_dir", - "gcov_run.log", - "gcov_summary.md", - "lcov.dat", - "link_check", - "test-reports", - "tests-report", - "typo_dir", - "user.bazelrc", - } - return {name for name in names if name in generated or name.startswith("bazel-")} - - -@pytest.fixture -def consumer_workspace(tmp_path: Path) -> Path: - """Give each scenario a private consumer workspace and Bazel output tree.""" +@pytest.fixture(scope="session") +def consumer_workspace() -> Path: + """Reuse the consumer's Bazel outputs, including its extracted toolchains.""" if not (_REPOSITORY_ROOT / "MODULE.bazel").is_file(): pytest.fail( "Black-box tests require local execution with symlinked runfiles pointing " @@ -65,11 +35,4 @@ def consumer_workspace(tmp_path: Path) -> Path: ) if shutil.which("bazel") is None: pytest.fail("Black-box tests require the host Bazel installation on PATH", pytrace=False) - # The consumer MODULE.bazel resolves score_coverage with - # local_path_override(path = "../../.."). Keep that root-relative layout - # intact so nested Bazel uses a private copy of the repository sources. - # A fresh copy per test also prevents Bazel outputs or fault-injection edits - # from making another scenario depend on test order. - root = tmp_path / "repo" - shutil.copytree(_REPOSITORY_ROOT, root, ignore=_ignore_generated_workspace_files) - return root / _CONSUMER_WORKSPACE_DIRECTORY + return _REPOSITORY_ROOT / _CONSUMER_WORKSPACE_DIRECTORY diff --git a/tests/end_to_end/testcases/test_cli.py b/tests/end_to_end/testcases/test_cli.py index 8c8ab20..3fc006c 100644 --- a/tests/end_to_end/testcases/test_cli.py +++ b/tests/end_to_end/testcases/test_cli.py @@ -21,12 +21,19 @@ def test_missing_report_returns_no_verdict(consumer_workspace: Path) -> None: """A consumer without coverage data gets exit 2 and a diagnostic explaining how to collect it.""" report = consumer_workspace / "bazel-out/_coverage/_coverage_report.dat" - assert not report.exists(), "Each CLI case must start without a generated coverage report" - result = run_bazel( - consumer_workspace, - ["run", "@score_coverage//:generate_coverage_html"], - env={"COVERAGE_THRESHOLD": "10"}, - ) + backup = report.with_name("missing-report-backup.dat") + # Preserve coverage data from earlier runs in the shared consumer workspace. + if report.is_file(): + report.rename(backup) + try: + result = run_bazel( + consumer_workspace, + ["run", "@score_coverage//:generate_coverage_html"], + env={"COVERAGE_THRESHOLD": "10"}, + ) + finally: + if backup.is_file(): + backup.replace(report) assert_exit_code(result, 2) output = result.stdout + result.stderr assert "Coverage report not found" in output From 423c4ed5699bbdf850a25b61eaf97fe40f752726 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 9 Oct 2026 17:42:01 +0200 Subject: [PATCH 4/6] test: remove generated report before missing-input check --- README.md | 4 ++-- tests/end_to_end/testcases/test_cli.py | 19 ++++++------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index b0c60c6..4277495 100644 --- a/README.md +++ b/README.md @@ -82,5 +82,5 @@ The end-to-end commands use the checked-in consumer workspace and reuse its Bazel output base, including extracted toolchains. This output base is separate from the main repository's, so nested Bazel commands do not wait on the outer test's lock. Run the end-to-end commands sequentially because they share the -consumer's coverage report; the pytest CLI case temporarily removes an existing -report and restores it afterwards. +consumer's coverage report; the pytest CLI case deletes an existing report to +exercise the missing-report diagnostic. diff --git a/tests/end_to_end/testcases/test_cli.py b/tests/end_to_end/testcases/test_cli.py index 3fc006c..0326665 100644 --- a/tests/end_to_end/testcases/test_cli.py +++ b/tests/end_to_end/testcases/test_cli.py @@ -21,19 +21,12 @@ def test_missing_report_returns_no_verdict(consumer_workspace: Path) -> None: """A consumer without coverage data gets exit 2 and a diagnostic explaining how to collect it.""" report = consumer_workspace / "bazel-out/_coverage/_coverage_report.dat" - backup = report.with_name("missing-report-backup.dat") - # Preserve coverage data from earlier runs in the shared consumer workspace. - if report.is_file(): - report.rename(backup) - try: - result = run_bazel( - consumer_workspace, - ["run", "@score_coverage//:generate_coverage_html"], - env={"COVERAGE_THRESHOLD": "10"}, - ) - finally: - if backup.is_file(): - backup.replace(report) + report.unlink(missing_ok=True) + result = run_bazel( + consumer_workspace, + ["run", "@score_coverage//:generate_coverage_html"], + env={"COVERAGE_THRESHOLD": "10"}, + ) assert_exit_code(result, 2) output = result.stdout + result.stderr assert "Coverage report not found" in output From 7b9e439408040978d89dce93027be84ce1f97f9b Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 9 Oct 2026 17:45:02 +0200 Subject: [PATCH 5/6] test: remove unused integration tag --- tests/end_to_end/testcases/BUILD | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/end_to_end/testcases/BUILD b/tests/end_to_end/testcases/BUILD index 7d4294a..8eea875 100644 --- a/tests/end_to_end/testcases/BUILD +++ b/tests/end_to_end/testcases/BUILD @@ -27,12 +27,10 @@ score_pytest( imports = ["."], # The checkout and host Bazel installation are runtime inputs outside this # target's runfiles, so changes to them cannot invalidate a cached result. - # `external` disables test caching; `integration` lets callers select these - # scenarios with --test_tag_filters; `local` keeps the host Bazel process + # `external` disables test caching; `local` keeps the host Bazel process # with the checkout instead of using sandboxed or remote execution. tags = [ "external", - "integration", "local", ], ) From f6286787fa4b12b27c92a3097ae47240ce899390 Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 9 Oct 2026 17:56:15 +0200 Subject: [PATCH 6/6] ci: upload end-to-end JUnit results directly --- .github/workflows/tests.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a928465..9ba01b4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -87,20 +87,13 @@ jobs: run: tests/end_to_end/run_end_to_end_test.sh - name: Run the end-to-end pytest CLI case run: bazel test --lockfile_mode=error //tests/end_to_end/testcases:blackbox_test - - name: Collect pytest JUnit results for the documentation build - if: always() - run: | - mkdir -p tests-report/tests/end_to_end/testcases/blackbox_test - if [ -f bazel-testlogs/tests/end_to_end/testcases/blackbox_test/test.xml ]; then - cp bazel-testlogs/tests/end_to_end/testcases/blackbox_test/test.xml \ - tests-report/tests/end_to_end/testcases/blackbox_test/test.xml - fi - name: Upload end-to-end pytest results if: always() uses: actions/upload-artifact@v4 with: name: end-to-end-tests-report - path: tests-report + # Preserve paths below bazel-testlogs for the docs job's tests-report tree. + path: bazel-testlogs/**/test.xml if-no-files-found: ignore retention-days: 3 - name: Upload coverage report of the end-to-end workspace