Set up end-to-end pytest with CI and requirement traceability

.github/workflows/tests.yml

@@ -67,7 +67,7 @@ jobs:
           echo "collected $(find tests-report -name test.xml | wc -l) test.xml files"
       - uses: actions/upload-artifact@v4
         with:
-          name: tests-report
+          name: unit-tests-report
           path: tests-report
           if-no-files-found: error
           retention-days: 3
@@ -85,6 +85,17 @@ jobs:
           disk-cache-key: end_to_end_tests
       - name: Run the end-to-end pipeline test (C++ + Rust consumer workspace)
         run: tests/end_to_end/run_end_to_end_test.sh
+      - name: Run the end-to-end pytest CLI case
+        run: bazel test --lockfile_mode=error //tests/end_to_end/testcases:blackbox_test
+      - name: Upload end-to-end pytest results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: end-to-end-tests-report
+          # Preserve paths below bazel-testlogs for the docs job's tests-report tree.
+          path: bazel-testlogs/**/test.xml
+          if-no-files-found: ignore
+          retention-days: 3
       - name: Upload coverage report of the end-to-end workspace
         if: always()
         uses: actions/upload-artifact@v4
@@ -93,8 +104,28 @@ jobs:
           path: tests/end_to_end/consumer/coverage_artifacts.zip
           if-no-files-found: ignore
           retention-days: 10
+  test_reports:
+    needs: [unit_tests, end_to_end_tests]
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Download unit test results
+        uses: actions/download-artifact@v4
+        with:
+          name: unit-tests-report
+          path: tests-report
+      - name: Download end-to-end test results
+        uses: actions/download-artifact@v4
+        with:
+          name: end-to-end-tests-report
+          path: tests-report
+      - uses: actions/upload-artifact@v4
+        with:
+          name: tests-report
+          path: tests-report
+          if-no-files-found: error
+          retention-days: 3
   docs:
-    needs: unit_tests
+    needs: test_reports
     uses: eclipse-score/cicd-workflows/.github/workflows/docs.yml@d2083d5dac0e309643d9f495e61342dfbaf07ed5 # main, 2026-09-24
     permissions:
       contents: read

README.md

@@ -72,5 +72,13 @@ bazel build --config=lint //score_coverage/... //tools/... //tests/...   # ruff,
 bazel coverage --combined_report=lcov //tests/unit/score_coverage:all
 bazel run //tools:self_coverage_gate -- --min-lines 95 --min-branches 87
 tests/end_to_end/run_end_to_end_test.sh            # end-to-end (downloads LLVM + Ferrocene)
+bazel test //tests/end_to_end/testcases:blackbox_test # pytest CLI case
 pre-commit run --all-files
 ```
+
+The end-to-end commands use the checked-in consumer workspace and reuse its
+Bazel output base, including extracted toolchains. This output base is separate
+from the main repository's, so nested Bazel commands do not wait on the outer
+test's lock. Run the end-to-end commands sequentially because they share the
+consumer's coverage report; the pytest CLI case deletes an existing report to
+exercise the missing-report diagnostic.

docs/BUILD

@@ -23,5 +23,8 @@ docs(
     source_dir = ".",
     # Only these test.xml files become testcase needs (JUnit results under
     # bazel-testlogs/ or tests-report/ at the workspace root).
-    test_sources = ["tests/unit/score_coverage"],
+    test_sources = [
+        "tests/unit/score_coverage",
+        "tests/end_to_end/testcases",
+    ],
 )

docs/verification/verification_report.rst

@@ -81,6 +81,9 @@ Test inventory
      - 17
      - scope_transitive, scope_excludes, scope_baseline_objects, scope_gcno,
        scope_platform
+   * - ``//tests/end_to_end/testcases:blackbox_test``
+     - 1
+     - gate_no_verdict
    * - ``tests/end_to_end/run_end_to_end_test.sh`` (27 end-to-end checks)
      - 27
      - validation_ground_truth, instrumentation_hint, scope_platform, report_baseline_zero, report_relative_paths,
@@ -96,7 +99,9 @@ carries ``@verifies(<tool_req ids>)``, which writes ``PartiallyVerifies``,
 ``TestType`` and ``DerivationTechnique`` into the JUnit XML of the test run, and
 docs-as-code turns the results into ``testcase`` needs with back-links on the
 requirements (``testlink`` column below, with the execution result of each
-case). The links reflect the test run that preceded the documentation build.
+case). End-to-end pytest functions attach the same metadata using the
+``score_pytest`` decorator. The links reflect the test run that preceded the
+documentation build.
 
 .. needtable:: Requirements and the tests that verify them
    :types: tool_req

tests/end_to_end/testcases/BUILD

@@ -0,0 +1,36 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+
+load("@score_tools//score_pytest:pytest.bzl", "score_pytest")
+
+score_pytest(
+    name = "blackbox_test",
+    size = "large",
+    timeout = "long",
+    srcs = ["test_cli.py"],
+    # score_pytest passes srcs to pytest as collection paths. Keep shared
+    # fixtures and helpers in runfiles without presenting them as test modules.
+    data = [
+        "_blackbox_support.py",
+        "conftest.py",
+    ],
+    imports = ["."],
+    # The checkout and host Bazel installation are runtime inputs outside this
+    # target's runfiles, so changes to them cannot invalidate a cached result.
+    # `external` disables test caching; `local` keeps the host Bazel process
+    # with the checkout instead of using sandboxed or remote execution.
+    tags = [
+        "external",
+        "local",
+    ],
+)

tests/end_to_end/testcases/_blackbox_support.py

@@ -0,0 +1,74 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+"""Local Bazel execution and requirement metadata for end-to-end pytest cases."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+from attribute_plugin import Decorator, DerivationTechnique, add_test_properties
+
+
+def run_bazel(
+    workspace: Path,
+    args: list[str],
+    *,
+    env: dict[str, str] | None = None,
+) -> subprocess.CompletedProcess[str]:
+    """Run a consumer-facing Bazel command and capture its user-visible output.
+
+    ``env`` overrides selected inherited variables for a single invocation,
+    while preserving the rest of the host environment.
+    """
+    command = shutil.which("bazel")
+    assert command, "Bazel must be available on PATH to run the integration scenarios"
+    process_env = os.environ.copy()
+    # A nested coverage command must not append its output to the enclosing CI
+    # step's summary; only the outer workflow owns that summary file.
+    process_env.pop("GITHUB_STEP_SUMMARY", None)
+    if env:
+        process_env.update(env)
+    # Bazel uses TEST_TMPDIR as the nested command's default output root. Remove
+    # the outer test's disposable value so nested Bazel can reuse the consumer's
+    # normal output base, extracted toolchains and the host download cache.
+    process_env.pop("TEST_TMPDIR", None)
+    return subprocess.run(
+        [command, *args],
+        cwd=workspace,
+        env=process_env,
+        capture_output=True,
+        check=False,
+        text=True,
+    )
+
+
+def assert_exit_code(result: subprocess.CompletedProcess[str], expected: int) -> None:
+    assert result.returncode == expected, (
+        f"command: {' '.join(result.args)}\n"
+        f"expected exit code {expected}, got {result.returncode}\n"
+        f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
+    )
+
+
+def verifies(*requirements: str, derivation: DerivationTechnique = "requirements-analysis") -> Decorator:
+    """Attach requirement metadata to a plain pytest interface scenario."""
+    # Keep this adapter temporary until the project settles how score_pytest
+    # and sphinx-needs-test-reports should provide metadata for pytest functions.
+    return add_test_properties(
+        partially_verifies=list(requirements),
+        test_type="interface-test",
+        derivation_technique=derivation,
+    )

tests/end_to_end/testcases/conftest.py

@@ -0,0 +1,38 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+"""The checked-in consumer workspace for local end-to-end pytest cases."""
+
+import shutil
+from pathlib import Path
+
+import pytest
+
+# This local-only target needs symlinked runfiles: resolve() follows conftest.py
+# back into the checkout. Copied runfiles and remote execution cannot supply
+# the consumer workspace or the host Bazel installation used by the fixtures.
+_REPOSITORY_ROOT = Path(__file__).resolve().parents[3]
+_CONSUMER_WORKSPACE_DIRECTORY = "tests/end_to_end/consumer"
+
+
+@pytest.fixture(scope="session")
+def consumer_workspace() -> Path:
+    """Reuse the consumer's Bazel outputs, including its extracted toolchains."""
+    if not (_REPOSITORY_ROOT / "MODULE.bazel").is_file():
+        pytest.fail(
+            "Black-box tests require local execution with symlinked runfiles pointing "
+            f"into the source checkout; MODULE.bazel is missing at {_REPOSITORY_ROOT}",
+            pytrace=False,
+        )
+    if shutil.which("bazel") is None:
+        pytest.fail("Black-box tests require the host Bazel installation on PATH", pytrace=False)
+    return _REPOSITORY_ROOT / _CONSUMER_WORKSPACE_DIRECTORY

tests/end_to_end/testcases/test_cli.py

@@ -0,0 +1,33 @@
+# *******************************************************************************
+# Copyright (c) 2026 Contributors to the Eclipse Foundation
+#
+# See the NOTICE file(s) distributed with this work for additional
+# information regarding copyright ownership.
+#
+# This program and the accompanying materials are made available under the
+# terms of the Apache License Version 2.0 which is available at
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# SPDX-License-Identifier: Apache-2.0
+# *******************************************************************************
+"""A CLI case exercising the end-to-end pytest infrastructure."""
+
+from pathlib import Path
+
+from _blackbox_support import assert_exit_code, run_bazel, verifies
+
+
+@verifies("tool_req__coverage_gate_no_verdict")
+def test_missing_report_returns_no_verdict(consumer_workspace: Path) -> None:
+    """A consumer without coverage data gets exit 2 and a diagnostic explaining how to collect it."""
+    report = consumer_workspace / "bazel-out/_coverage/_coverage_report.dat"
+    report.unlink(missing_ok=True)
+    result = run_bazel(
+        consumer_workspace,
+        ["run", "@score_coverage//:generate_coverage_html"],
+        env={"COVERAGE_THRESHOLD": "10"},
+    )
+    assert_exit_code(result, 2)
+    output = result.stdout + result.stderr
+    assert "Coverage report not found" in output
+    assert "Run 'bazel coverage" in output