diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 068f286..9ba01b4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,7 +67,7 @@ jobs: echo "collected $(find tests-report -name test.xml | wc -l) test.xml files" - uses: actions/upload-artifact@v4 with: - name: tests-report + name: unit-tests-report path: tests-report if-no-files-found: error retention-days: 3 @@ -85,6 +85,17 @@ jobs: disk-cache-key: end_to_end_tests - name: Run the end-to-end pipeline test (C++ + Rust consumer workspace) run: tests/end_to_end/run_end_to_end_test.sh + - name: Run the end-to-end pytest CLI case + run: bazel test --lockfile_mode=error //tests/end_to_end/testcases:blackbox_test + - name: Upload end-to-end pytest results + if: always() + uses: actions/upload-artifact@v4 + with: + name: end-to-end-tests-report + # Preserve paths below bazel-testlogs for the docs job's tests-report tree. + path: bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 3 - name: Upload coverage report of the end-to-end workspace if: always() uses: actions/upload-artifact@v4 @@ -93,8 +104,28 @@ jobs: path: tests/end_to_end/consumer/coverage_artifacts.zip if-no-files-found: ignore retention-days: 10 + test_reports: + needs: [unit_tests, end_to_end_tests] + runs-on: ubuntu-24.04 + steps: + - name: Download unit test results + uses: actions/download-artifact@v4 + with: + name: unit-tests-report + path: tests-report + - name: Download end-to-end test results + uses: actions/download-artifact@v4 + with: + name: end-to-end-tests-report + path: tests-report + - uses: actions/upload-artifact@v4 + with: + name: tests-report + path: tests-report + if-no-files-found: error + retention-days: 3 docs: - needs: unit_tests + needs: test_reports uses: eclipse-score/cicd-workflows/.github/workflows/docs.yml@d2083d5dac0e309643d9f495e61342dfbaf07ed5 # main, 2026-09-24 permissions: contents: read diff --git a/README.md b/README.md index 2d16d4a..db984ae 100644 --- a/README.md +++ b/README.md @@ -72,5 +72,13 @@ bazel build --config=lint //score_coverage/... //tools/... //tests/... # ruff, bazel coverage --combined_report=lcov //tests/unit/score_coverage:all bazel run //tools:self_coverage_gate -- --min-lines 95 --min-branches 87 tests/end_to_end/run_end_to_end_test.sh # end-to-end (downloads LLVM + Ferrocene) +bazel test //tests/end_to_end/testcases:blackbox_test # pytest CLI case pre-commit run --all-files ``` + +The end-to-end commands use the checked-in consumer workspace and reuse its +Bazel output base, including extracted toolchains. This output base is separate +from the main repository's, so nested Bazel commands do not wait on the outer +test's lock. Run the end-to-end commands sequentially because they share the +consumer's coverage report; the pytest CLI case deletes an existing report to +exercise the missing-report diagnostic. diff --git a/docs/BUILD b/docs/BUILD index c36e3b9..d0b8b51 100644 --- a/docs/BUILD +++ b/docs/BUILD @@ -23,5 +23,8 @@ docs( source_dir = ".", # Only these test.xml files become testcase needs (JUnit results under # bazel-testlogs/ or tests-report/ at the workspace root). - test_sources = ["tests/unit/score_coverage"], + test_sources = [ + "tests/unit/score_coverage", + "tests/end_to_end/testcases", + ], ) diff --git a/docs/verification/verification_report.rst b/docs/verification/verification_report.rst index 119a643..eac432f 100644 --- a/docs/verification/verification_report.rst +++ b/docs/verification/verification_report.rst @@ -81,6 +81,9 @@ Test inventory - 17 - scope_transitive, scope_excludes, scope_baseline_objects, scope_gcno, scope_platform + * - ``//tests/end_to_end/testcases:blackbox_test`` + - 1 + - gate_no_verdict * - ``tests/end_to_end/run_end_to_end_test.sh`` (27 end-to-end checks) - 27 - validation_ground_truth, instrumentation_hint, scope_platform, report_baseline_zero, report_relative_paths, @@ -96,7 +99,9 @@ carries ``@verifies()``, which writes ``PartiallyVerifies``, ``TestType`` and ``DerivationTechnique`` into the JUnit XML of the test run, and docs-as-code turns the results into ``testcase`` needs with back-links on the requirements (``testlink`` column below, with the execution result of each -case). The links reflect the test run that preceded the documentation build. +case). End-to-end pytest functions attach the same metadata using the +``score_pytest`` decorator. The links reflect the test run that preceded the +documentation build. .. needtable:: Requirements and the tests that verify them :types: tool_req diff --git a/tests/end_to_end/testcases/BUILD b/tests/end_to_end/testcases/BUILD new file mode 100644 index 0000000..8eea875 --- /dev/null +++ b/tests/end_to_end/testcases/BUILD @@ -0,0 +1,36 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* + +load("@score_tools//score_pytest:pytest.bzl", "score_pytest") + +score_pytest( + name = "blackbox_test", + size = "large", + timeout = "long", + srcs = ["test_cli.py"], + # score_pytest passes srcs to pytest as collection paths. Keep shared + # fixtures and helpers in runfiles without presenting them as test modules. + data = [ + "_blackbox_support.py", + "conftest.py", + ], + imports = ["."], + # The checkout and host Bazel installation are runtime inputs outside this + # target's runfiles, so changes to them cannot invalidate a cached result. + # `external` disables test caching; `local` keeps the host Bazel process + # with the checkout instead of using sandboxed or remote execution. + tags = [ + "external", + "local", + ], +) diff --git a/tests/end_to_end/testcases/_blackbox_support.py b/tests/end_to_end/testcases/_blackbox_support.py new file mode 100644 index 0000000..08fab18 --- /dev/null +++ b/tests/end_to_end/testcases/_blackbox_support.py @@ -0,0 +1,74 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Local Bazel execution and requirement metadata for end-to-end pytest cases.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +from attribute_plugin import Decorator, DerivationTechnique, add_test_properties + + +def run_bazel( + workspace: Path, + args: list[str], + *, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + """Run a consumer-facing Bazel command and capture its user-visible output. + + ``env`` overrides selected inherited variables for a single invocation, + while preserving the rest of the host environment. + """ + command = shutil.which("bazel") + assert command, "Bazel must be available on PATH to run the integration scenarios" + process_env = os.environ.copy() + # A nested coverage command must not append its output to the enclosing CI + # step's summary; only the outer workflow owns that summary file. + process_env.pop("GITHUB_STEP_SUMMARY", None) + if env: + process_env.update(env) + # Bazel uses TEST_TMPDIR as the nested command's default output root. Remove + # the outer test's disposable value so nested Bazel can reuse the consumer's + # normal output base, extracted toolchains and the host download cache. + process_env.pop("TEST_TMPDIR", None) + return subprocess.run( + [command, *args], + cwd=workspace, + env=process_env, + capture_output=True, + check=False, + text=True, + ) + + +def assert_exit_code(result: subprocess.CompletedProcess[str], expected: int) -> None: + assert result.returncode == expected, ( + f"command: {' '.join(result.args)}\n" + f"expected exit code {expected}, got {result.returncode}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + + +def verifies(*requirements: str, derivation: DerivationTechnique = "requirements-analysis") -> Decorator: + """Attach requirement metadata to a plain pytest interface scenario.""" + # Keep this adapter temporary until the project settles how score_pytest + # and sphinx-needs-test-reports should provide metadata for pytest functions. + return add_test_properties( + partially_verifies=list(requirements), + test_type="interface-test", + derivation_technique=derivation, + ) diff --git a/tests/end_to_end/testcases/conftest.py b/tests/end_to_end/testcases/conftest.py new file mode 100644 index 0000000..bd79796 --- /dev/null +++ b/tests/end_to_end/testcases/conftest.py @@ -0,0 +1,38 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""The checked-in consumer workspace for local end-to-end pytest cases.""" + +import shutil +from pathlib import Path + +import pytest + +# This local-only target needs symlinked runfiles: resolve() follows conftest.py +# back into the checkout. Copied runfiles and remote execution cannot supply +# the consumer workspace or the host Bazel installation used by the fixtures. +_REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +_CONSUMER_WORKSPACE_DIRECTORY = "tests/end_to_end/consumer" + + +@pytest.fixture(scope="session") +def consumer_workspace() -> Path: + """Reuse the consumer's Bazel outputs, including its extracted toolchains.""" + if not (_REPOSITORY_ROOT / "MODULE.bazel").is_file(): + pytest.fail( + "Black-box tests require local execution with symlinked runfiles pointing " + f"into the source checkout; MODULE.bazel is missing at {_REPOSITORY_ROOT}", + pytrace=False, + ) + if shutil.which("bazel") is None: + pytest.fail("Black-box tests require the host Bazel installation on PATH", pytrace=False) + return _REPOSITORY_ROOT / _CONSUMER_WORKSPACE_DIRECTORY diff --git a/tests/end_to_end/testcases/test_cli.py b/tests/end_to_end/testcases/test_cli.py new file mode 100644 index 0000000..0326665 --- /dev/null +++ b/tests/end_to_end/testcases/test_cli.py @@ -0,0 +1,33 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""A CLI case exercising the end-to-end pytest infrastructure.""" + +from pathlib import Path + +from _blackbox_support import assert_exit_code, run_bazel, verifies + + +@verifies("tool_req__coverage_gate_no_verdict") +def test_missing_report_returns_no_verdict(consumer_workspace: Path) -> None: + """A consumer without coverage data gets exit 2 and a diagnostic explaining how to collect it.""" + report = consumer_workspace / "bazel-out/_coverage/_coverage_report.dat" + report.unlink(missing_ok=True) + result = run_bazel( + consumer_workspace, + ["run", "@score_coverage//:generate_coverage_html"], + env={"COVERAGE_THRESHOLD": "10"}, + ) + assert_exit_code(result, 2) + output = result.stdout + result.stderr + assert "Coverage report not found" in output + assert "Run 'bazel coverage" in output