From 07f3b113cebbaf053bcf6a7b8459058c9531c2fd Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Fri, 9 Oct 2026 01:59:03 +0200 Subject: [PATCH] test: migrate coverage assertions to end-to-end pytest cases --- .github/workflows/tests.yml | 6 +- README.md | 3 +- docs/verification/verification_report.rst | 25 +- tests/end_to_end/consumer/.bazelrc | 4 + tests/end_to_end/run_end_to_end_test.sh | 382 +----------------- tests/end_to_end/testcases/BUILD | 8 +- .../end_to_end/testcases/_blackbox_support.py | 150 ++++++- tests/end_to_end/testcases/conftest.py | 32 +- .../testcases/test_error_handling.py | 76 ++++ tests/end_to_end/testcases/test_gcov.py | 186 +++++++++ tests/end_to_end/testcases/test_llvm_gates.py | 114 ++++++ .../end_to_end/testcases/test_llvm_reports.py | 246 +++++++++++ 12 files changed, 831 insertions(+), 401 deletions(-) create mode 100644 tests/end_to_end/testcases/test_error_handling.py create mode 100644 tests/end_to_end/testcases/test_gcov.py create mode 100644 tests/end_to_end/testcases/test_llvm_gates.py create mode 100644 tests/end_to_end/testcases/test_llvm_reports.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a928465..d2714a6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -83,9 +83,7 @@ jobs: - uses: eclipse-score/cicd-actions/setup-bazel-cache@setup-bazel-cache/v0.1.0 with: disk-cache-key: end_to_end_tests - - name: Run the end-to-end pipeline test (C++ + Rust consumer workspace) - run: tests/end_to_end/run_end_to_end_test.sh - - name: Run the end-to-end pytest CLI case + - name: Run traceable pytest black-box cases run: bazel test --lockfile_mode=error //tests/end_to_end/testcases:blackbox_test - name: Collect pytest JUnit results for the documentation build if: always() @@ -108,7 +106,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: end_to_end_coverage_report - path: tests/end_to_end/consumer/coverage_artifacts.zip + path: bazel-testlogs/tests/end_to_end/testcases/blackbox_test/test.outputs/coverage_artifact if-no-files-found: ignore retention-days: 10 test_reports: diff --git a/README.md b/README.md index b20c4f8..2c7b6f0 100644 --- a/README.md +++ b/README.md @@ -72,8 +72,7 @@ bazel test //tests/unit/... //tools/... # unit + analysis tests bazel build --config=lint //score_coverage/... //tools/... //tests/... # ruff, pylint, ty bazel coverage --combined_report=lcov //tests/unit/score_coverage:all bazel run //tools:self_coverage_gate -- --min-lines 95 --min-branches 87 -tests/end_to_end/run_end_to_end_test.sh # end-to-end (downloads LLVM + Ferrocene) -bazel test //tests/end_to_end/testcases:blackbox_test # pytest CLI case +bazel test //tests/end_to_end/testcases:blackbox_test # traceable C++ + Rust end-to-end cases pre-commit run --all-files bazel run //tools:copyright.check ``` diff --git a/docs/verification/verification_report.rst b/docs/verification/verification_report.rst index eac432f..313adba 100644 --- a/docs/verification/verification_report.rst +++ b/docs/verification/verification_report.rst @@ -82,10 +82,7 @@ Test inventory - scope_transitive, scope_excludes, scope_baseline_objects, scope_gcno, scope_platform * - ``//tests/end_to_end/testcases:blackbox_test`` - - 1 - - gate_no_verdict - * - ``tests/end_to_end/run_end_to_end_test.sh`` (27 end-to-end checks) - - 27 + - 47 - validation_ground_truth, instrumentation_hint, scope_platform, report_baseline_zero, report_relative_paths, report_allowlist, report_unmapped, gcov_merge, gcov_baseline, gcov_html, gate_exit_codes, gate_no_verdict, just_unknown_id, artifacts, @@ -108,17 +105,14 @@ documentation build. :columns: id;title;testlink :style: table -Four requirements are verified outside the pytest suites and therefore carry no -generated link: +Three requirements are verified outside the pytest suites and therefore carry +no generated link: - :need:`tool_req__coverage_scope_transitive`, :need:`tool_req__coverage_scope_excludes` and :need:`tool_req__coverage_scope_baseline_objects` are verified by the seventeen Starlark analysis tests in ``tests/unit/starlark`` (rules_testing produces no test properties). -- :need:`tool_req__coverage_validation_ground_truth` is verified by the - end-to-end run ``tests/end_to_end/run_end_to_end_test.sh`` (golden LCOV - comparison, see below). .. needpie:: Test results of the linked test cases :labels: passed, failed, skipped @@ -175,10 +169,13 @@ yamlfmt the workflows; copyright headers are checked on every file. End-to-end validation --------------------- -``tests/end_to_end/run_end_to_end_test.sh`` builds a consumer workspace with a -tested and an untested C++ library, a header-only library reached through +``//tests/end_to_end/testcases:blackbox_test`` runs named pytest cases against +an isolated copy of the consumer workspace. Session fixtures collect LLVM and +gcov reports once each; the cases invoke the public report command with +different options and retain the LLVM archive in Bazel test outputs. The +consumer workspace contains a tested and an untested C++ library, a header-only library reached through ``strip_include_prefix``, a tested Rust library and an untested Rust binary, one -justified line, and asserts: +justified line. The cases verify: 1. the gate fails at 100 % and passes at 10 % (effective and raw mode); 2. the HTML, the summary and the archive tree are produced, the summary also @@ -187,7 +184,9 @@ justified line, and asserts: 4. the LCOV matches ``expected_lcov.dat``, a hand-derived ground truth, record by record; 5. the justified line raises effective above raw coverage; -6. fault injection: a corrupt report and a non-numeric threshold exit 2, and a +6. platform-specific sources and scope roots follow the host for LLVM and the + target platform for gcov; +7. fault injection: a corrupt report and a non-numeric threshold exit 2, and a misspelt justification id is reported and does not raise the effective coverage. diff --git a/tests/end_to_end/consumer/.bazelrc b/tests/end_to_end/consumer/.bazelrc index 2ce2e35..f1bf3dc 100644 --- a/tests/end_to_end/consumer/.bazelrc +++ b/tests/end_to_end/consumer/.bazelrc @@ -109,3 +109,7 @@ coverage:gcov --test_lang_filters=cc # the tests too so header-only code that only a test translation unit # instantiates is measured (the scope allowlist still drops the test sources). coverage:gcov --instrument_test_targets + +# This test configuration leaves the cross-package library uninstrumented so +# the end-to-end suite can check the reporter's instrumentation-filter hint. +coverage:gcov_narrow_filter --instrumentation_filter=^//lib/test[/:],^//src[/:] diff --git a/tests/end_to_end/run_end_to_end_test.sh b/tests/end_to_end/run_end_to_end_test.sh index 222fbcf..331173b 100755 --- a/tests/end_to_end/run_end_to_end_test.sh +++ b/tests/end_to_end/run_end_to_end_test.sh @@ -11,383 +11,7 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -# End-to-end test of the score_coverage LLVM coverage pipeline, run against -# the consumer/ workspace. Asserts the properties the pipeline -# guarantees: -# 1. Untested in-scope files (C++ AND Rust) appear at exact 0% in the LCOV. -# 2. The effective-coverage gate fails at threshold 100 and passes at a low -# threshold. -# 3. The justified line raises effective coverage above raw coverage. - +# Run the traceable end-to-end pytest suite from any working directory. set -euo pipefail -cd "$(dirname "$0")/consumer" - -# In GitHub Actions GITHUB_STEP_SUMMARY is set for THIS job; unset it so the -# many generate_coverage_html invocations below don't each append to the real -# run page. The dedicated summary test sets its own target file. -unset GITHUB_STEP_SUMMARY || true - -echo "=== Running coverage build ===" -bazel coverage --config=llvm_cov //... --build_tests_only - -YAML="tools/coverage/coverage_justifications.yaml" - -echo "=== Gate must FAIL at threshold 100 (uncovered fixtures exist) ===" -if COVERAGE_THRESHOLD=100 bazel run @score_coverage//:generate_coverage_html -- \ - --yaml "${YAML}" --archive coverage_artifacts; then - echo "ERROR: coverage gate passed at threshold 100 despite uncovered files" >&2 - exit 1 -fi -echo "OK: gate failed as expected" - -echo "=== Gate must PASS at a low threshold ===" -COVERAGE_THRESHOLD=10 bazel run @score_coverage//:generate_coverage_html -- \ - --yaml "${YAML}" -echo "OK: gate passed as expected" - -echo "=== Without --yaml: HTML still produced, gate applies to RAW coverage ===" -if COVERAGE_THRESHOLD=100 bazel run @score_coverage//:generate_coverage_html; then - echo "ERROR: raw-coverage gate passed at threshold 100" >&2 - exit 1 -fi -COVERAGE_THRESHOLD=10 bazel run @score_coverage//:generate_coverage_html -if [[ ! -f coverage_linux/index.html ]]; then - echo "ERROR: HTML report missing after no-yaml run" >&2 - exit 1 -fi -echo "OK: no-yaml mode works (HTML produced, raw gate enforced)" - -# The following sections all run, in order. Each one deletes summary.md -# before its own generate_coverage_html invocation so a stale file from the -# previous section cannot produce a false pass — in particular, the -# failing-gate section must prove the file was RE-created by THAT run. -echo "=== --summary-md must produce a markdown job summary ===" -rm -f summary.md -COVERAGE_THRESHOLD=10 bazel run @score_coverage//:generate_coverage_html -- \ - --yaml "${YAML}" --summary-md summary.md -for marker in "## Coverage summary" "| Lines |" "Raw vs effective" \ - "Coverage by directory" "Files at exact 0% (3)" \ - "| In-scope files without coverage data | 2 |" \ - "In-scope files without coverage data (2)" '- `src/unused_api.h`' '- `src/platform_dep.h`' \ - "Declaration-only headers (3)" "Compiled sources without code of their own (1)" '- `src/empty_unit.cpp`'; do - if ! grep -qF -- "${marker}" summary.md; then - echo "ERROR: '${marker}' missing from summary.md" >&2 - exit 1 - fi -done -grep -q "█" summary.md || { echo "ERROR: progress bars missing from summary.md" >&2; exit 1; } -echo "OK: --summary-md works" - -echo "=== Summary must still be written when the gate FAILS ===" -rm -f summary.md -if COVERAGE_THRESHOLD=100 bazel run @score_coverage//:generate_coverage_html -- \ - --yaml "${YAML}" --summary-md summary.md; then - echo "ERROR: gate unexpectedly passed at threshold 100" >&2 - exit 1 -fi -[[ -s summary.md ]] || { echo "ERROR: summary.md missing after failing gate" >&2; exit 1; } -echo "OK: summary survives a failing gate" - -echo "=== GITHUB_STEP_SUMMARY convenience default (no flag) ===" -rm -f step_summary.md -printf '# existing content\n' > step_summary.md -GITHUB_STEP_SUMMARY="$(pwd)/step_summary.md" COVERAGE_THRESHOLD=10 \ - bazel run @score_coverage//:generate_coverage_html -- --yaml "${YAML}" -grep -qF "# existing content" step_summary.md || { echo "ERROR: append mode overwrote the step summary" >&2; exit 1; } -grep -qF "## Coverage summary" step_summary.md || { echo "ERROR: summary not appended to GITHUB_STEP_SUMMARY" >&2; exit 1; } -rm -f summary.md step_summary.md -echo "OK: GITHUB_STEP_SUMMARY convenience works" - -echo "=== --archive-dir must produce an unzipped artifacts tree ===" -COVERAGE_THRESHOLD=10 bazel run @score_coverage//:generate_coverage_html -- \ - --yaml "${YAML}" --archive-dir artifacts_dir -for f in artifacts_dir/coverage_linux/index.html artifacts_dir/coverage_report.dat \ - artifacts_dir/justification_report/summary.txt; do - if [[ ! -f "$f" ]]; then - echo "ERROR: ${f} missing from --archive-dir output" >&2 - exit 1 - fi -done -# unused_api.h is the finding; coverable.h / uncovered.h hold declarations for -# compiled .cpp files and empty_unit.cpp is a compiled placeholder: categorised. -# platform_dep.h is no-data, not declaration-only: its sources are named per -# variant (platform_host.cpp / platform_target.cpp), not platform_dep.cpp. -EXPECTED_UNMAPPED=$'compiled-without-code\tsrc/empty_unit.cpp\ndeclaration-only\tlib/cross_pkg.h\ndeclaration-only\tsrc/coverable.h\ndeclaration-only\tsrc/uncovered.h\nno-data\tsrc/platform_dep.h\nno-data\tsrc/unused_api.h' -if [[ "$(cat artifacts_dir/unmapped_files.txt)" != "${EXPECTED_UNMAPPED}" ]]; then - echo "ERROR: unmapped_files.txt unexpected:" >&2 - cat artifacts_dir/unmapped_files.txt >&2 - exit 1 -fi -echo "OK: in-scope files without coverage data are listed and categorised in the archive" -rm -rf artifacts_dir -echo "OK: --archive-dir works" - -echo "=== Untested files must appear at exact 0% in the LCOV ===" -unzip -p coverage_artifacts.zip artifacts/coverage_report.dat > lcov.dat - -check_zero_coverage() { - local file="$1" - if ! grep -q "SF:.*${file}" lcov.dat; then - echo "ERROR: ${file} missing from LCOV (baseline mechanism broken)" >&2 - exit 1 - fi - # The record for the file must report zero lines hit. - if ! awk -v f="${file}" ' - $0 ~ "^SF:" && $0 ~ f {rec=1} - rec && /^LH:/ {print $0; exit ($0 == "LH:0") ? 0 : 1} - rec && /^end_of_record/ {exit 1}' lcov.dat; then - echo "ERROR: ${file} is present but not at 0% coverage" >&2 - exit 1 - fi - echo "OK: ${file} present at 0%" -} - -check_zero_coverage "src/uncovered.cpp" -check_zero_coverage "rust/main.rs" - -echo "=== LCOV must match the hand-verified ground truth exactly ===" -# Normalise: drop function records, keep one record per file sorted by SF, so -# the comparison is independent of record order and of symbol names. -normalise_lcov() { - grep -v '^FN' "$1" | awk ' - /^SF:/ { key = $0; rec = "" } - { rec = rec $0 "\n" } - /^end_of_record/ { records[key] = rec } - END { n = asorti(records, keys); for (i = 1; i <= n; i++) printf "%s", records[keys[i]] }' -} -normalise_lcov lcov.dat > actual_normalised.dat -grep -v '^#' ../expected/expected_lcov.dat | normalise_lcov /dev/stdin > expected_normalised.dat -if ! diff -u expected_normalised.dat actual_normalised.dat; then - echo "ERROR: coverage data differs from expected_lcov.dat (see diff above)" >&2 - exit 1 -fi -rm -f actual_normalised.dat expected_normalised.dat -echo "OK: LCOV matches the ground truth" - -echo "=== A library tested from a test/ subpackage must be measured (explicit --instrumentation_filter) ===" -# Bazel guesses the filter from the packages of the test targets; //lib is -# outside that guess and would be compiled without instrumentation. The -# config sets the filter explicitly; the golden above holds the numbers. -grep -q "^SF:lib/cross_pkg.cpp$" lcov.dat || { echo "ERROR: lib/cross_pkg.cpp missing: --instrumentation_filter not applied" >&2; exit 1; } -echo "OK: cross-package library measured on the LLVM backend" - -echo "=== The scope follows the run's platform: LLVM run (host) reports the host variant only ===" -grep -q "^SF:src/platform_host.cpp$" lcov.dat || { echo "ERROR: host variant platform_host.cpp missing from the LLVM report" >&2; exit 1; } -if grep -q "platform_target.cpp" lcov.dat; then echo "ERROR: target variant leaked into the host (LLVM) report" >&2; exit 1; fi -grep -q "^SF:src/host_root.cpp$" lcov.dat || { echo "ERROR: host root (select in the scope deps) missing from the LLVM report" >&2; exit 1; } -if grep -q "target_root.cpp" lcov.dat; then echo "ERROR: target root leaked into the host (LLVM) report" >&2; exit 1; fi -echo "OK: platform-selected library reported as the host variant on the LLVM run" - -echo "=== Every index link must point at an existing page; no machine or config paths ===" -rm -rf link_check && mkdir link_check -unzip -q coverage_artifacts.zip -d link_check -HTML_DIR="link_check/artifacts/coverage_linux" -[[ -f "${HTML_DIR}/index.html" ]] || { echo "ERROR: ${HTML_DIR}/index.html missing" >&2; exit 1; } -LINKS="$(grep -oE "href='coverage/[^']+\.html'" "${HTML_DIR}/index.html" | sed -E "s/^href='//; s/'$//")" -[[ -n "${LINKS}" ]] || { echo "ERROR: no source links in index.html" >&2; exit 1; } -while IFS= read -r link; do - if [[ ! -f "${HTML_DIR}/${link}" ]]; then - echo "ERROR: index.html links to ${link}, which was not generated" >&2 - exit 1 - fi - case "${link}" in - coverage/bazel-out/*|coverage/home/*|coverage/tmp/*|*/_virtual_includes/*) - echo "ERROR: index.html link is not a canonical workspace path: ${link}" >&2 - exit 1 ;; - esac - # The page's stylesheet link must resolve from the page's location. - page_dir="$(dirname "${HTML_DIR}/${link}")" - css="$(grep -oE "href='(\.\./)*style\.css'" "${HTML_DIR}/${link}" | head -1 | sed -E "s/^href='//; s/'$//")" - if [[ -z "${css}" || ! -f "${page_dir}/${css}" ]]; then - echo "ERROR: ${link}: stylesheet link '${css}' does not resolve" >&2 - exit 1 - fi -done <<< "${LINKS}" -for page in "coverage/src/vendored/include/vendored/inline_math.h.html" \ - "coverage/external/itest_external+/include/vext/vext.h.html"; do - grep -qF "href='${page}'" "${HTML_DIR}/index.html" || { echo "ERROR: ${page} not linked from index.html" >&2; exit 1; } -done -if grep -q "itest_external+/extlib" "${HTML_DIR}/index.html"; then - echo "ERROR: forwarded third-party library leaked into the HTML report" >&2 - exit 1 -fi -if grep -q "extlib" lcov.dat; then - echo "ERROR: forwarded third-party library leaked into the LCOV" >&2 - exit 1 -fi -rm -rf link_check -echo "OK: $(echo "${LINKS}" | wc -l) index links resolve, canonical paths only, third-party code excluded" - -echo "=== A header compiled only through a test-only twin target must be attributed to the declared file ===" -# Without the fallback the data sits under _virtual_includes/vendored_math_internal/ -# (a target outside the scope), gets excluded, and the header is listed as no-data. -grep -q "^SF:src/vendored/include/vendored/inline_math.h$" lcov.dat || { echo "ERROR: inline_math.h not attributed to its declared path" >&2; exit 1; } -if grep -q "vendored_math_internal" lcov.dat artifacts_dir/unmapped_files.txt 2>/dev/null; then - echo "ERROR: the test-only twin's virtual path leaked into the report" >&2; exit 1 -fi -echo "OK" - -echo "=== A header nothing includes must be reported as unmapped, not invented in the LCOV ===" -if grep -q "unused_api" lcov.dat; then - echo "ERROR: src/unused_api.h has no compiled code and must not have an LCOV record" >&2 - exit 1 -fi -echo "OK" - -echo "=== Covered files must be present with hits ===" -grep -q "SF:.*src/coverable.cpp" lcov.dat || { echo "ERROR: coverable.cpp missing" >&2; exit 1; } -grep -q "SF:.*rust/lib.rs" lcov.dat || { echo "ERROR: lib.rs missing" >&2; exit 1; } -echo "OK" - -echo "=== Justified line must raise effective coverage above raw ===" -SUMMARY="$(unzip -p coverage_artifacts.zip artifacts/justification_report/summary.txt)" -echo "${SUMMARY}" -JUSTIFIED="$(echo "${SUMMARY}" | grep -oP 'Justified lines:\s+\K[0-9]+')" -if [[ "${JUSTIFIED}" -lt 1 ]]; then - echo "ERROR: expected at least one justified line, got ${JUSTIFIED}" >&2 - exit 1 -fi -RAW="$(echo "${SUMMARY}" | grep -oP 'Raw line coverage:\s+\K[0-9.]+')" -EFFECTIVE="$(echo "${SUMMARY}" | grep -oP 'Effective line coverage:\s+\K[0-9.]+')" -if ! awk "BEGIN {exit (${EFFECTIVE} > ${RAW}) ? 0 : 1}"; then - echo "ERROR: effective coverage ${EFFECTIVE}% not above raw ${RAW}%" >&2 - exit 1 -fi -echo "OK: effective ${EFFECTIVE}% > raw ${RAW}%" - -echo "=== Fault injection: a broken report must yield NO verdict (exit 2), never a pass ===" -REPORT="bazel-out/_coverage/_coverage_report.dat" -cp "${REPORT}" report.backup -chmod u+w "${REPORT}" -printf 'this is not a zip archive' > "${REPORT}" -set +e -COVERAGE_THRESHOLD=0 bazel run @score_coverage//:generate_coverage_html > /dev/null 2>&1 -rc=$? -set -e -cp report.backup "${REPORT}" -rm -f report.backup -if [[ "${rc}" -ne 2 ]]; then - echo "ERROR: corrupt report gave exit code ${rc}, expected 2" >&2 - exit 1 -fi -echo "OK: corrupt report is rejected with exit 2" - -echo "=== Fault injection: a non-numeric threshold must be rejected (exit 2) ===" -set +e -COVERAGE_THRESHOLD=lenient bazel run @score_coverage//:generate_coverage_html > /dev/null 2>&1 -rc=$? -set -e -if [[ "${rc}" -ne 2 ]]; then - echo "ERROR: bad threshold gave exit code ${rc}, expected 2" >&2 - exit 1 -fi -echo "OK: invalid threshold is rejected with exit 2" - -echo "=== Fault injection: an unknown justification id must not count as covered ===" -sed -i 's/itest-positive-branch/itest-typo-branch/' src/coverable.cpp -set +e -COVERAGE_THRESHOLD=10 bazel run @score_coverage//:generate_coverage_html -- --yaml "${YAML}" --archive-dir typo_dir > typo.log 2>&1 -rc=$? -set -e -sed -i 's/itest-typo-branch/itest-positive-branch/' src/coverable.cpp -if [[ "${rc}" -ne 0 ]]; then - cat typo.log - echo "ERROR: run with an unknown marker id failed unexpectedly (${rc})" >&2 - exit 1 -fi -grep -q "references unknown ID 'itest-typo-branch'" typo.log || { echo "ERROR: unknown marker id was not reported" >&2; exit 1; } -TYPO_EFFECTIVE="$(grep -oP 'Effective line coverage:\s+\K[0-9.]+' typo_dir/justification_report/summary.txt)" -TYPO_RAW="$(grep -oP 'Raw line coverage:\s+\K[0-9.]+' typo_dir/justification_report/summary.txt)" -if [[ "${TYPO_EFFECTIVE}" != "${TYPO_RAW}" ]]; then - echo "ERROR: unknown marker id still raised effective (${TYPO_EFFECTIVE}) above raw (${TYPO_RAW})" >&2 - exit 1 -fi -rm -rf typo_dir typo.log -echo "OK: unknown justification id is reported and does not count" - -# --------------------------------------------------------------------------- -# gcov backend: GCC toolchain, Bazel's own per-test collector, score_coverage's -# gcov reporter. The same path a QNX (QCC) on-target run takes; only the QEMU -# transport of score_qnx_unit_tests differs. -# --------------------------------------------------------------------------- -echo "=== gcov backend: coverage build with the GCC toolchain ===" -bazel coverage --config=gcov //src/... //lib/... --build_tests_only - -echo "=== gcov backend: gate, HTML, archive ===" -rm -rf gcov_artifacts_dir -if COVERAGE_THRESHOLD=100 bazel run @score_coverage//:generate_coverage_html -- \ - --yaml "${YAML}" --archive-dir gcov_artifacts_dir --summary-md gcov_summary.md coverage_gcov > gcov_run.log 2>&1; then - cat gcov_run.log - echo "ERROR: gcov gate passed at threshold 100 despite uncovered files" >&2 - exit 1 -fi -grep -q "Effective coverage" gcov_run.log || { cat gcov_run.log; echo "ERROR: gcov run did not reach the gate" >&2; exit 1; } -COVERAGE_THRESHOLD=10 bazel run @score_coverage//:generate_coverage_html -- \ - --yaml "${YAML}" --archive-dir gcov_artifacts_dir --summary-md gcov_summary.md coverage_gcov -echo "OK: gcov gate fails at 100 and passes at 10" - -echo "=== gcov backend: LCOV must match the hand-verified ground truth ===" -normalise_lcov gcov_artifacts_dir/coverage_report.dat > actual_gcov.dat -grep -v '^#' ../expected/expected_lcov_gcov.dat | normalise_lcov /dev/stdin > expected_gcov.dat -if ! diff -u expected_gcov.dat actual_gcov.dat; then - echo "ERROR: gcov coverage data differs from expected_lcov_gcov.dat (see diff above)" >&2 - exit 1 -fi -rm -f actual_gcov.dat expected_gcov.dat -echo "OK: gcov LCOV matches the ground truth" - -echo "=== gcov backend: every index link opens; justification applied; categories ===" -GHTML="gcov_artifacts_dir/coverage_gcov" -[[ -f "${GHTML}/index.html" ]] || { echo "ERROR: gcovr index.html missing" >&2; exit 1; } -GLINKS="$(grep -oE 'href="index\.[^"]+\.html"' "${GHTML}/index.html" | sed -E 's/^href="//; s/"$//' | sort -u)" -[[ -n "${GLINKS}" ]] || { echo "ERROR: no per-file links in the gcovr index" >&2; exit 1; } -while IFS= read -r link; do - [[ -f "${GHTML}/${link}" ]] || { echo "ERROR: gcovr index links to ${link}, which does not exist" >&2; exit 1; } -done <<< "${GLINKS}" -for page in coverable.cpp uncovered.cpp inline_math.h cross_pkg.cpp platform_target.cpp target_root.cpp; do - ls "${GHTML}"/index."${page}".*.html > /dev/null 2>&1 || { echo "ERROR: no gcovr page for ${page}" >&2; exit 1; } -done -G_RAW="$(grep -oP 'Raw line coverage:\s+\K[0-9.]+' gcov_artifacts_dir/justification_report/summary.txt)" -G_EFF="$(grep -oP 'Effective line coverage:\s+\K[0-9.]+' gcov_artifacts_dir/justification_report/summary.txt)" -if ! awk "BEGIN {exit (${G_EFF} > ${G_RAW}) ? 0 : 1}"; then - echo "ERROR: gcov effective coverage ${G_EFF}% not above raw ${G_RAW}% (justification not applied on gcovr HTML)" >&2 - exit 1 -fi -EXPECTED_GCOV_UNMAPPED=$'compiled-without-code\tsrc/empty_unit.cpp\ndeclaration-only\tlib/cross_pkg.h\ndeclaration-only\tsrc/coverable.h\ndeclaration-only\tsrc/uncovered.h\nno-data\tsrc/platform_dep.h\nno-data\tsrc/unused_api.h\nnot-instrumented\trust/lib.rs\nnot-instrumented\trust/main.rs' -if [[ "$(cat gcov_artifacts_dir/unmapped_files.txt)" != "${EXPECTED_GCOV_UNMAPPED}" ]]; then - echo "ERROR: gcov unmapped_files.txt unexpected:" >&2 - cat gcov_artifacts_dir/unmapped_files.txt >&2 - exit 1 -fi -grep -qF "Not instrumentable by this backend (2)" gcov_summary.md || { echo "ERROR: not-instrumented section missing from the gcov summary" >&2; exit 1; } -echo "=== The scope follows the run's platform: gcov run (//platforms:gcov_target) reports the target variant only ===" -# coverage_scope_gcov carries platform = //platforms:gcov_target; without it -# the scope would be analysed for the host (exec configuration) and list -# platform_host.cpp at 0 % instead (coverage_tool#23). -grep -q "^SF:src/platform_target.cpp$" gcov_artifacts_dir/coverage_report.dat || { echo "ERROR: target variant platform_target.cpp missing from the gcov report: scope not evaluated for the run's platform" >&2; exit 1; } -if grep -q "platform_host.cpp" gcov_artifacts_dir/coverage_report.dat; then echo "ERROR: host variant leaked into the gcov report: scope evaluated for the host" >&2; exit 1; fi -# The select() in the scope's deps resolved for the gcov platform: the target -# root is in, the (incompatible) host root out, and the tests were executed. -grep -q "^SF:src/target_root.cpp$" gcov_artifacts_dir/coverage_report.dat || { echo "ERROR: target root (select in the scope deps) missing from the gcov report" >&2; exit 1; } -if grep -q "host_root.cpp" gcov_artifacts_dir/coverage_report.dat; then echo "ERROR: host root leaked into the gcov report" >&2; exit 1; fi -echo "OK: platform-selected library reported as the target variant on the gcov run" - -rm -rf gcov_artifacts_dir gcov_summary.md gcov_run.log coverage_gcov -echo "OK: gcov HTML complete (${GLINKS//$'\n'/, }), effective ${G_EFF}% > raw ${G_RAW}%, categories as expected" - -echo "=== gcov backend: without the explicit filter the reporter must point at --instrumentation_filter ===" -# Bazel's guessed filter for these targets is ^//lib/test[/:],^//src[/:]; pass -# it explicitly to reproduce a consumer config that forgot the flag. Bazel's -# collector then drops lib/cross_pkg.cpp's counters; the file falls back to -# the 0 % baseline and the reporter must name the cause. -bazel coverage --config=gcov '--instrumentation_filter=^//lib/test[/:],^//src[/:]' //src/... //lib/... --build_tests_only > gcov_narrow.log 2>&1 || { cat gcov_narrow.log; exit 1; } -grep -q "WARNING: 1 in-scope files have no test data although their directory is tested from a test/ or tests/ subdirectory" gcov_narrow.log \ - || { cat gcov_narrow.log; echo "ERROR: the reporter did not warn about the narrow --instrumentation_filter" >&2; exit 1; } -grep -q -- "--instrumentation_filter=\^//\[/:\]" gcov_narrow.log || { echo "ERROR: the warning does not name the flag to set" >&2; exit 1; } -unzip -p bazel-out/_coverage/_coverage_report.dat lcov_report/lcov.dat | awk '/^SF:lib\/cross_pkg.cpp$/{p=1} p&&/^LH:/{print; exit}' | grep -q "^LH:0$" \ - || { echo "ERROR: expected lib/cross_pkg.cpp at 0 % under the narrow filter" >&2; exit 1; } -rm -f gcov_narrow.log -echo "OK: narrow --instrumentation_filter is detected and reported" - -echo "" -echo "=== All integration checks passed ===" +cd "$(dirname "$0")/../.." +exec bazel test --lockfile_mode=error //tests/end_to_end/testcases:blackbox_test "$@" diff --git a/tests/end_to_end/testcases/BUILD b/tests/end_to_end/testcases/BUILD index 78a219b..7daacb8 100644 --- a/tests/end_to_end/testcases/BUILD +++ b/tests/end_to_end/testcases/BUILD @@ -17,7 +17,13 @@ score_pytest( name = "blackbox_test", size = "large", timeout = "long", - srcs = ["test_cli.py"], + srcs = [ + "test_cli.py", + "test_error_handling.py", + "test_gcov.py", + "test_llvm_gates.py", + "test_llvm_reports.py", + ], # score_pytest passes srcs to pytest as collection paths. Keep shared # fixtures and helpers in runfiles without presenting them as test modules. data = [ diff --git a/tests/end_to_end/testcases/_blackbox_support.py b/tests/end_to_end/testcases/_blackbox_support.py index ed6a63d..cc09e0e 100644 --- a/tests/end_to_end/testcases/_blackbox_support.py +++ b/tests/end_to_end/testcases/_blackbox_support.py @@ -10,17 +10,53 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -"""Local Bazel execution and requirement metadata for end-to-end pytest cases.""" +"""Shared helpers for black-box coverage scenarios.""" from __future__ import annotations import os +import re import shutil import subprocess +import zipfile +from dataclasses import dataclass from pathlib import Path +from typing import NamedTuple from attribute_plugin import Decorator, DerivationTechnique, add_test_properties +_COVERAGE_REPORT = Path("bazel-out/_coverage/_coverage_report.dat") +JUSTIFICATIONS = "tools/coverage/coverage_justifications.yaml" +EXPECTED_UNMAPPED_LLVM = ( + "compiled-without-code\tsrc/empty_unit.cpp\n" + "declaration-only\tlib/cross_pkg.h\n" + "declaration-only\tsrc/coverable.h\n" + "declaration-only\tsrc/uncovered.h\n" + "no-data\tsrc/platform_dep.h\n" + "no-data\tsrc/unused_api.h\n" +) +EXPECTED_UNMAPPED_GCOV = EXPECTED_UNMAPPED_LLVM + "not-instrumented\trust/lib.rs\nnot-instrumented\trust/main.rs\n" + + +@dataclass(frozen=True) +class CoverageReport: + """A consumer workspace and a saved report produced by one backend setup.""" + + workspace: Path + backend: str + archive: Path + collection_output: str + + def install(self) -> Path: + """Put this scenario's collected report at the documented Bazel path.""" + target = self.workspace / _COVERAGE_REPORT + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists(): + target.chmod(target.stat().st_mode | 0o200) + shutil.copyfile(self.archive, target) + target.chmod(target.stat().st_mode | 0o200) + return target + def run_bazel( workspace: Path, @@ -64,3 +100,115 @@ def verifies(*requirements: str, derivation: DerivationTechnique = "requirements test_type="interface-test", derivation_technique=derivation, ) + + +def fault_injection(*requirements: str, derivation: DerivationTechnique = "error-guessing") -> Decorator: + """Attach score_pytest requirement metadata to a fault-injection scenario.""" + return add_test_properties( + partially_verifies=list(requirements), + test_type="fault-injection", + derivation_technique=derivation, + ) + + +def collect_report( + workspace: Path, + backend: str, + targets: list[str], + destination: Path, + extra_args: list[str] | None = None, +) -> CoverageReport: + """Collect one backend report for a consumer-workspace scenario.""" + command = ["coverage", f"--config={backend}"] + if extra_args: + command.extend(extra_args) + command.extend(targets) + command.append("--build_tests_only") + result = run_bazel(workspace, command) + assert_exit_code(result, 0) + collected = workspace / _COVERAGE_REPORT + assert collected.is_file(), f"Bazel did not produce the expected {backend} coverage archive" + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(collected, destination) + return CoverageReport(workspace, backend, destination, result.stdout + result.stderr) + + +def generate_report( + report: CoverageReport, + *args: str, + threshold: str = "10", + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[str]: + """Run the public HTML-generation command against a saved backend report.""" + report.install() + command = ["run", "@score_coverage//:generate_coverage_html", "--", *args] + process_env = {"COVERAGE_THRESHOLD": threshold} + if env: + process_env.update(env) + return run_bazel(report.workspace, command, env=process_env) + + +def lcov_records(text: str) -> dict[str, list[str]]: + """Split LCOV data into source records, dropping function metadata.""" + records: dict[str, list[str]] = {} + current: list[str] = [] + source = "" + for line in text.splitlines(): + if line.startswith("#"): + continue + if line.startswith("SF:"): + source = line[3:] + if line.startswith("FN"): + continue + current.append(line) + if line == "end_of_record": + records[source] = current + current = [] + source = "" + return records + + +def normalise_lcov(text: str) -> str: + """Ignore function metadata and record order when comparing the goldens.""" + records = lcov_records(text) + return "".join("\n".join(records[source]) + "\n" for source in sorted(records)) + + +def lcov_from_artifacts(path: Path) -> str: + """Read LCOV data from either a generated archive directory or ZIP file.""" + if path.is_dir(): + return (path / "coverage_report.dat").read_text(encoding="utf-8") + with zipfile.ZipFile(path) as archive: + member = next( + name + for name in archive.namelist() + if name.endswith("lcov_report/lcov.dat") or name.endswith("coverage_report.dat") + ) + return archive.read(member).decode("utf-8") + + +class LineCounts(NamedTuple): + """Covered and found line totals from one LCOV source record.""" + + hits: int + found: int + + +def line_counts(records: dict[str, list[str]], source: str) -> LineCounts: + """Return a source record's covered-line and total-line counts.""" + record = records[source] + lines_found = next(int(line[3:]) for line in record if line.startswith("LF:")) + lines_hit = next(int(line[3:]) for line in record if line.startswith("LH:")) + return LineCounts(hits=lines_hit, found=lines_found) + + +def summary_metric(summary: str, name: str) -> float: + """Read a named percentage from the human-readable justification summary.""" + match = re.search(rf"{re.escape(name)}:\s+([0-9.]+)%", summary) + assert match, f"{name} missing from report summary:\n{summary}" + return float(match.group(1)) + + +def index_links(index: Path) -> list[str]: + """Return coverage-page links in an HTML index page.""" + return re.findall(r"href=['\"](coverage/[^'\"]+\.html)['\"]", index.read_text(encoding="utf-8")) diff --git a/tests/end_to_end/testcases/conftest.py b/tests/end_to_end/testcases/conftest.py index 1fbb067..3393f37 100644 --- a/tests/end_to_end/testcases/conftest.py +++ b/tests/end_to_end/testcases/conftest.py @@ -10,12 +10,20 @@ # # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* -"""An isolated consumer workspace for local end-to-end pytest cases.""" +"""Shared consumer workspaces and backend reports for black-box scenarios. + +The consumer copy is created once per pytest run. LLVM and gcov reports are +also collected once each, then individual scenarios pass those saved reports +to the public report-generation command with their own options. Scenarios +must run serially: they share the consumer workspace's report path and output +directories, and fault injection temporarily modifies its source files. +""" import shutil from pathlib import Path import pytest +from _blackbox_support import CoverageReport, collect_report # This local-only target needs symlinked runfiles: resolve() follows conftest.py # back into the checkout. Copied runfiles and remote execution cannot supply @@ -68,3 +76,25 @@ def consumer_workspace(tmp_path_factory: pytest.TempPathFactory) -> Path: root = tmp_path_factory.mktemp("score-coverage-blackbox") / "repo" shutil.copytree(_REPOSITORY_ROOT, root, ignore=_ignore_generated_workspace_files) return root / _CONSUMER_WORKSPACE_DIRECTORY + + +@pytest.fixture(scope="session") +def llvm_report(consumer_workspace: Path, tmp_path_factory: pytest.TempPathFactory) -> CoverageReport: + """Collect LLVM coverage for every target, including C++ and Rust tests.""" + return collect_report( + consumer_workspace, + "llvm_cov", + ["//..."], + tmp_path_factory.mktemp("llvm-report") / "coverage_report.dat", + ) + + +@pytest.fixture(scope="session") +def gcov_report(consumer_workspace: Path, tmp_path_factory: pytest.TempPathFactory) -> CoverageReport: + """Collect gcov for //src/... and //lib/...; LLVM covers Rust separately.""" + return collect_report( + consumer_workspace, + "gcov", + ["//src/...", "//lib/..."], + tmp_path_factory.mktemp("gcov-report") / "coverage_report.dat", + ) diff --git a/tests/end_to_end/testcases/test_error_handling.py b/tests/end_to_end/testcases/test_error_handling.py new file mode 100644 index 0000000..1fba8a8 --- /dev/null +++ b/tests/end_to_end/testcases/test_error_handling.py @@ -0,0 +1,76 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Black-box fault-injection scenarios for malformed coverage inputs.""" + +from pathlib import Path + +from _blackbox_support import ( + JUSTIFICATIONS, + CoverageReport, + assert_exit_code, + fault_injection, + generate_report, + run_bazel, + summary_metric, +) + + +@fault_injection("tool_req__coverage_gate_no_verdict") +def test_corrupt_llvm_report_returns_no_verdict(llvm_report: CoverageReport) -> None: + """A corrupt coverage archive returns exit 2 even with a permissive threshold.""" + report_path = llvm_report.install() + report_path.write_text("this is not a zip archive", encoding="utf-8") + try: + result = run_bazel( + llvm_report.workspace, + ["run", "@score_coverage//:generate_coverage_html"], + env={"COVERAGE_THRESHOLD": "0"}, + ) + finally: + # Restore the shared session report so later scenarios see valid input. + llvm_report.install() + assert_exit_code(result, 2) + + +@fault_injection("tool_req__coverage_gate_no_verdict", derivation="boundary-values") +def test_non_numeric_threshold_returns_no_verdict(llvm_report: CoverageReport) -> None: + """An invalid threshold is rejected with exit 2 instead of passing the gate.""" + result = generate_report(llvm_report, threshold="lenient") + assert_exit_code(result, 2) + + +@fault_injection("tool_req__coverage_just_unknown_id") +def test_unknown_justification_marker_is_reported_and_not_credited(llvm_report: CoverageReport, tmp_path: Path) -> None: + """An unrecognized code marker is reported and leaves effective equal to raw.""" + source = llvm_report.workspace / "src/coverable.cpp" + original = source.read_text(encoding="utf-8") + valid_marker = "COV_JUSTIFIED itest-positive-branch" + unknown_marker = "COV_JUSTIFIED itest-typo-branch" + assert original.count(valid_marker) == 1, "fault injection expects one known marker in coverable.cpp" + source.write_text(original.replace(valid_marker, unknown_marker), encoding="utf-8") + output = tmp_path / "typo-artifacts" + try: + result = generate_report( + llvm_report, + "--yaml", + JUSTIFICATIONS, + "--archive-dir", + str(output), + ) + finally: + # The consumer workspace is shared by the other LLVM scenarios. + source.write_text(original, encoding="utf-8") + assert_exit_code(result, 0) + assert "references unknown ID 'itest-typo-branch'" in result.stdout + result.stderr + summary = (output / "justification_report" / "summary.txt").read_text(encoding="utf-8") + assert summary_metric(summary, "Effective line coverage") == summary_metric(summary, "Raw line coverage") diff --git a/tests/end_to_end/testcases/test_gcov.py b/tests/end_to_end/testcases/test_gcov.py new file mode 100644 index 0000000..8fb2fa5 --- /dev/null +++ b/tests/end_to_end/testcases/test_gcov.py @@ -0,0 +1,186 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Black-box scenarios for the gcov coverage backend.""" + +import re +import zipfile +from pathlib import Path + +import pytest +from _blackbox_support import ( + EXPECTED_UNMAPPED_GCOV, + JUSTIFICATIONS, + CoverageReport, + assert_exit_code, + collect_report, + generate_report, + lcov_from_artifacts, + lcov_records, + line_counts, + normalise_lcov, + summary_metric, + verifies, +) + + +@pytest.fixture(name="gcov_lcov_archive", scope="module") +def create_gcov_lcov_archive(gcov_report: CoverageReport, tmp_path_factory: pytest.TempPathFactory) -> Path: + """Generate the gcov LCOV archive used by the golden-report scenarios.""" + archive = tmp_path_factory.mktemp("gcov-lcov") / "coverage-artifacts" + result = generate_report(gcov_report, "--yaml", JUSTIFICATIONS, "--archive-dir", str(archive)) + assert_exit_code(result, 0) + return archive + + +@pytest.fixture(name="gcov_html_archive", scope="module") +def create_gcov_html_archive(gcov_report: CoverageReport, tmp_path_factory: pytest.TempPathFactory) -> Path: + """Generate gcov HTML and its Markdown summary for the archive scenarios.""" + output_root = tmp_path_factory.mktemp("gcov-html") + archive = output_root / "coverage-artifacts" + markdown_summary = output_root / "gcov-summary.md" + result = generate_report( + gcov_report, + "--yaml", + JUSTIFICATIONS, + "--archive-dir", + str(archive), + "--summary-md", + str(markdown_summary), + "coverage_gcov", + ) + assert_exit_code(result, 0) + return archive + + +@pytest.mark.parametrize( + ("threshold", "expected_exit"), + [("100", 1), ("10", 0)], + ids=["fails-at-100-percent", "passes-at-10-percent"], +) +@verifies("tool_req__coverage_gate_exit_codes", derivation="boundary-values") +def test_gcov_effective_gate_uses_reviewed_justifications( + gcov_report: CoverageReport, threshold: str, expected_exit: int +) -> None: + """Reviewed justifications make the gcov 10% gate pass and 100% gate fail.""" + result = generate_report(gcov_report, "--yaml", JUSTIFICATIONS, threshold=threshold) + assert_exit_code(result, expected_exit) + + +@verifies("tool_req__coverage_validation_ground_truth") +def test_gcov_lcov_matches_hand_derived_golden(gcov_report: CoverageReport, gcov_lcov_archive: Path) -> None: + """The generated gcov LCOV report matches its hand-derived golden.""" + actual = normalise_lcov(lcov_from_artifacts(gcov_lcov_archive)) + golden_path = gcov_report.workspace.parent / "expected" / "expected_lcov_gcov.dat" + expected = normalise_lcov(golden_path.read_text(encoding="utf-8")) + + assert actual == expected + + +@verifies("tool_req__coverage_gcov_merge", "tool_req__coverage_gcov_baseline") +def test_gcov_lcov_measures_vendored_header(gcov_lcov_archive: Path) -> None: + """The gcov LCOV report includes hits from the vendored C++ header.""" + records = lcov_records(lcov_from_artifacts(gcov_lcov_archive)) + header = "external/itest_external+/include/vext/vext.h" + + assert line_counts(records, header).hits > 0 + + +@verifies("tool_req__coverage_gcov_merge") +def test_gcov_lcov_omits_rust_sources(gcov_lcov_archive: Path) -> None: + """The gcov LCOV report contains no Rust source records.""" + records = lcov_records(lcov_from_artifacts(gcov_lcov_archive)) + assert not any(path.endswith(".rs") for path in records) + + +@verifies("tool_req__coverage_gcov_html") +def test_gcov_html_links_resolve_to_generated_pages(gcov_html_archive: Path) -> None: + """Every linked gcov HTML page exists in the consumer archive.""" + html_root = gcov_html_archive / "coverage_gcov" + index = (html_root / "index.html").read_text(encoding="utf-8") + page_links = re.findall(r'href="(index\.[^"]+\.html)"', index) + assert page_links + + for link in page_links: + assert (html_root / link).is_file(), f"gcov report page does not exist: {link}" + + +@pytest.mark.parametrize( + "source", + ["coverable.cpp", "uncovered.cpp", "inline_math.h", "cross_pkg.cpp", "platform_target.cpp", "target_root.cpp"], +) +@verifies("tool_req__coverage_gcov_html") +def test_gcov_html_contains_each_expected_source_page(gcov_html_archive: Path, source: str) -> None: + """Covered, uncovered, header-only and cross-package sources each have a page.""" + html_root = gcov_html_archive / "coverage_gcov" + assert any(page.is_file() for page in html_root.glob(f"index.{source}.*.html")), ( + f"gcov report has no source page for {source}" + ) + + +@verifies("tool_req__coverage_just_markers") +def test_gcov_summary_credits_reviewed_justifications(gcov_html_archive: Path) -> None: + """The gcov justification summary reports higher effective than raw coverage.""" + summary_path = gcov_html_archive / "justification_report" / "summary.txt" + summary = summary_path.read_text(encoding="utf-8") + raw_coverage = summary_metric(summary, "Raw line coverage") + effective_coverage = summary_metric(summary, "Effective line coverage") + + assert effective_coverage > raw_coverage + + +@verifies("tool_req__coverage_report_unmapped") +def test_gcov_archive_lists_unmapped_files(gcov_html_archive: Path) -> None: + """The gcov archive categorizes every source omitted from its report.""" + unmapped = (gcov_html_archive / "unmapped_files.txt").read_text(encoding="utf-8") + assert unmapped == EXPECTED_UNMAPPED_GCOV + + +@verifies("tool_req__coverage_report_unmapped") +def test_gcov_markdown_summary_explains_uninstrumented_rust(gcov_html_archive: Path) -> None: + """The Markdown summary explains that gcov cannot instrument the Rust files.""" + markdown_path = gcov_html_archive.parent / "gcov-summary.md" + markdown_summary = markdown_path.read_text(encoding="utf-8") + assert "Not instrumentable by this backend (2)" in markdown_summary + + +@verifies("tool_req__coverage_instrumentation_hint", derivation="error-guessing") +def test_gcov_narrow_filter_warns_and_leaves_excluded_library_at_zero(consumer_workspace: Path, tmp_path: Path) -> None: + """A narrowed instrumentation filter warns and leaves the omitted library at zero hits.""" + # This config builds both packages but instruments only //src/ and //lib/test/. + report = collect_report( + consumer_workspace, + "gcov", + ["//src/...", "//lib/..."], + tmp_path / "gcov-narrow-filter.dat", + extra_args=["--config=gcov_narrow_filter"], + ) + output = report.collection_output + + assert "in-scope files have no test data" in output + assert "WARNING: 1 in-scope files have no test data" in output + assert "--instrumentation_filter=^//[/:]" in output + + with zipfile.ZipFile(report.archive) as archive: + lcov_member = next(name for name in archive.namelist() if name.endswith("lcov_report/lcov.dat")) + records = lcov_records(archive.read(lcov_member).decode("utf-8")) + assert line_counts(records, "lib/cross_pkg.cpp").hits == 0 + + +@verifies("tool_req__coverage_scope_platform") +def test_gcov_report_selects_target_platform_sources(gcov_lcov_archive: Path) -> None: + """The gcov scope follows the target platform, including select() in its roots.""" + records = lcov_records(lcov_from_artifacts(gcov_lcov_archive)) + assert "src/platform_target.cpp" in records + assert "src/target_root.cpp" in records + assert "src/platform_host.cpp" not in records + assert "src/host_root.cpp" not in records diff --git a/tests/end_to_end/testcases/test_llvm_gates.py b/tests/end_to_end/testcases/test_llvm_gates.py new file mode 100644 index 0000000..9bb66dc --- /dev/null +++ b/tests/end_to_end/testcases/test_llvm_gates.py @@ -0,0 +1,114 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Black-box scenarios for LLVM coverage gate and summary behavior.""" + +from pathlib import Path + +import pytest +from _blackbox_support import ( + JUSTIFICATIONS, + CoverageReport, + assert_exit_code, + generate_report, + verifies, +) + + +@pytest.mark.parametrize( + ("threshold", "expected_exit"), + [("100", 1), ("10", 0)], + ids=["fails-at-100-percent", "passes-at-10-percent"], +) +@verifies("tool_req__coverage_gate_exit_codes", derivation="boundary-values") +def test_llvm_effective_gate_uses_reviewed_justifications( + llvm_report: CoverageReport, threshold: str, expected_exit: int +) -> None: + """Reviewed justifications make the 10% gate pass and the 100% gate fail.""" + result = generate_report(llvm_report, "--yaml", JUSTIFICATIONS, threshold=threshold) + assert_exit_code(result, expected_exit) + + +@pytest.mark.parametrize( + ("threshold", "expected_exit"), + [("100", 1), ("10", 0)], + ids=["raw-fails-at-100-percent", "raw-passes-at-10-percent"], +) +@verifies("tool_req__coverage_gate_metric", "tool_req__coverage_gate_exit_codes") +def test_llvm_without_yaml_gates_on_raw_coverage( + llvm_report: CoverageReport, threshold: str, expected_exit: int +) -> None: + """Without a justification file, raw coverage determines both gate outcomes.""" + result = generate_report(llvm_report, threshold=threshold) + assert_exit_code(result, expected_exit) + + +@pytest.mark.parametrize( + ("threshold", "expected_exit"), + [("100", 1), ("10", 0)], + ids=["summary-written-on-failure", "summary-written-on-pass"], +) +@verifies("tool_req__coverage_summary_first") +def test_llvm_markdown_summary_is_written_before_the_gate_verdict( + llvm_report: CoverageReport, + tmp_path: Path, + threshold: str, + expected_exit: int, +) -> None: + """The requested Markdown summary is complete on both pass and failure.""" + summary = tmp_path / "coverage-summary.md" + result = generate_report( + llvm_report, + "--yaml", + JUSTIFICATIONS, + "--summary-md", + str(summary), + threshold=threshold, + ) + assert_exit_code(result, expected_exit) + assert summary.is_file(), "summary should be written even when the gate fails" + content = summary.read_text(encoding="utf-8") + for marker in ( + "## Coverage summary", + "| Lines |", + "Raw vs effective", + "Coverage by directory", + "Files at exact 0% (3)", + "| In-scope files without coverage data | 2 |", + "In-scope files without coverage data (2)", + "- `src/unused_api.h`", + "- `src/platform_dep.h`", + "Declaration-only headers (3)", + "Compiled sources without code of their own (1)", + "- `src/empty_unit.cpp`", + "█", + ): + assert marker in content, f"coverage summary is missing {marker!r}" + + +@verifies("tool_req__coverage_summary_first") +def test_llvm_github_step_summary_appends_without_overwriting_existing_content( + llvm_report: CoverageReport, tmp_path: Path +) -> None: + """The CI summary appends coverage while preserving earlier step content.""" + summary = tmp_path / "step-summary.md" + summary.write_text("# Earlier step\n", encoding="utf-8") + result = generate_report( + llvm_report, + "--yaml", + JUSTIFICATIONS, + env={"GITHUB_STEP_SUMMARY": str(summary)}, + ) + assert_exit_code(result, 0) + content = summary.read_text(encoding="utf-8") + assert "# Earlier step" in content + assert "## Coverage summary" in content diff --git a/tests/end_to_end/testcases/test_llvm_reports.py b/tests/end_to_end/testcases/test_llvm_reports.py new file mode 100644 index 0000000..d360ac7 --- /dev/null +++ b/tests/end_to_end/testcases/test_llvm_reports.py @@ -0,0 +1,246 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Black-box scenarios for LLVM reports, artifacts, and source mapping.""" + +# Keep backend-specific command setup and expected artifacts visible in each +# module; pylint otherwise flags this intentional cross-backend test symmetry. +# pylint: disable=duplicate-code + +import os +import re +import shutil +import zipfile +from pathlib import Path + +import pytest +from _blackbox_support import ( + EXPECTED_UNMAPPED_LLVM, + JUSTIFICATIONS, + CoverageReport, + assert_exit_code, + generate_report, + index_links, + lcov_from_artifacts, + lcov_records, + line_counts, + normalise_lcov, + summary_metric, + verifies, +) + + +@pytest.fixture(name="llvm_effective_archive", scope="module") +def create_llvm_effective_archive(llvm_report: CoverageReport, tmp_path_factory: pytest.TempPathFactory) -> Path: + """Generate one effective-coverage archive for the artifact scenarios.""" + output_directory = os.environ.get("TEST_UNDECLARED_OUTPUTS_DIR") + output_root = Path(output_directory) if output_directory else tmp_path_factory.mktemp("llvm-archive") + archive = output_root / "coverage_artifact" + # Bazel may reuse its undeclared-output directory between runs. + shutil.rmtree(archive, ignore_errors=True) + result = generate_report(llvm_report, "--yaml", JUSTIFICATIONS, "--archive-dir", str(archive)) + assert_exit_code(result, 0) + return archive + + +@pytest.fixture(name="llvm_raw_html_archive", scope="module") +def create_llvm_raw_html_archive(llvm_report: CoverageReport, tmp_path_factory: pytest.TempPathFactory) -> Path: + """Generate raw-coverage HTML at 100%, where the coverage gate should fail.""" + archive = tmp_path_factory.mktemp("llvm-html") / "coverage-artifacts" + result = generate_report(llvm_report, "--archive-dir", str(archive), threshold="100") + assert_exit_code(result, 1) + return archive + + +@pytest.fixture( + name="llvm_html_archive", + scope="module", + params=["llvm_raw_html_archive", "llvm_effective_archive"], + ids=["raw", "effective"], +) +def select_llvm_html_archive(request: pytest.FixtureRequest) -> Path: + """Check both collected HTML and HTML rewritten by justification processing.""" + return request.getfixturevalue(request.param) + + +@pytest.mark.parametrize( + ("threshold", "expected_exit"), + [("100", 1), ("10", 0)], + ids=["zip-written-on-failure", "zip-written-on-pass"], +) +@verifies("tool_req__coverage_artifacts") +def test_llvm_zip_contains_reports_and_consumer_test_results( + llvm_report: CoverageReport, + llvm_effective_archive: Path, + tmp_path: Path, + threshold: str, + expected_exit: int, +) -> None: + """A ZIP preserves the artifacts layout and data even when the gate fails.""" + archive_name = tmp_path / "coverage-artifacts" + result = generate_report(llvm_report, "--yaml", JUSTIFICATIONS, "--archive", str(archive_name), threshold=threshold) + assert_exit_code(result, expected_exit) + with zipfile.ZipFile(archive_name.with_suffix(".zip")) as archive: + members = set(archive.namelist()) + for relative_path in ( + "coverage_linux/index.html", + "coverage_report.dat", + "justification_report/summary.txt", + "unmapped_files.txt", + ): + member = f"artifacts/{relative_path}" + assert member in members, f"ZIP is missing {member}" + assert archive.read(member) == (llvm_effective_archive / relative_path).read_bytes() + # Source pages and their stylesheets must survive packaging, not just + # the index that points to them. + for page in (llvm_effective_archive / "coverage_linux").rglob("*"): + if page.is_file(): + member = f"artifacts/{page.relative_to(llvm_effective_archive).as_posix()}" + assert member in members, f"ZIP is missing HTML asset {member}" + assert archive.read(member) == page.read_bytes() + test_results = list((llvm_report.workspace / "bazel-testlogs").rglob("test.xml")) + assert test_results, "consumer coverage collection must produce test results to archive" + for test_result in test_results: + member = f"artifacts/{test_result.relative_to(llvm_report.workspace).as_posix()}" + assert member in members, f"ZIP is missing consumer test result {member}" + assert archive.read(member) == test_result.read_bytes() + + +@verifies("tool_req__coverage_artifacts") +def test_effective_archive_contains_html_lcov_and_justification_summary(llvm_effective_archive: Path) -> None: + """The consumer archive contains its HTML report, LCOV data, and justification summary.""" + assert (llvm_effective_archive / "coverage_linux" / "index.html").is_file() + assert (llvm_effective_archive / "coverage_report.dat").is_file() + assert (llvm_effective_archive / "justification_report" / "summary.txt").is_file() + + +@verifies("tool_req__coverage_report_unmapped") +def test_effective_archive_lists_unmapped_files(llvm_effective_archive: Path) -> None: + """The archive categorizes every file omitted from the LLVM LCOV report.""" + unmapped = (llvm_effective_archive / "unmapped_files.txt").read_text(encoding="utf-8") + assert unmapped == EXPECTED_UNMAPPED_LLVM + + +@verifies("tool_req__coverage_just_markers") +def test_effective_summary_credits_reviewed_justifications(llvm_effective_archive: Path) -> None: + """Reviewed markers raise effective coverage above raw coverage.""" + summary_path = llvm_effective_archive / "justification_report" / "summary.txt" + summary = summary_path.read_text(encoding="utf-8") + raw_coverage = summary_metric(summary, "Raw line coverage") + effective_coverage = summary_metric(summary, "Effective line coverage") + + assert re.search(r"Justified lines:\s+[1-9][0-9]*", summary) + assert effective_coverage > raw_coverage + + +@verifies("tool_req__coverage_validation_ground_truth") +def test_llvm_report_matches_hand_derived_lcov_golden( + llvm_report: CoverageReport, llvm_effective_archive: Path +) -> None: + """The public archive matches the hand-derived C++ and Rust LCOV golden.""" + actual = normalise_lcov(lcov_from_artifacts(llvm_effective_archive)) + golden_path = llvm_report.workspace.parent / "expected" / "expected_lcov.dat" + expected = normalise_lcov(golden_path.read_text(encoding="utf-8")) + + assert actual == expected + + +@verifies("tool_req__coverage_report_baseline_zero") +def test_lcov_keeps_zero_hit_records_for_uncovered_cpp_and_rust(llvm_effective_archive: Path) -> None: + """Uncovered C++ and Rust sources remain present as zero-hit LCOV records.""" + records = lcov_records(lcov_from_artifacts(llvm_effective_archive)) + for source in ("src/uncovered.cpp", "rust/main.rs"): + counts = line_counts(records, source) + assert counts.found > 0, f"{source} has no executable lines in the LCOV report" + assert counts.hits == 0, f"{source} should have zero covered lines" + + +@verifies("tool_req__coverage_report_unmapped") +def test_uncompiled_header_is_listed_without_an_lcov_record(llvm_effective_archive: Path) -> None: + """An in-scope header that was never compiled is reported separately from LCOV.""" + records = lcov_records(lcov_from_artifacts(llvm_effective_archive)) + unmapped = (llvm_effective_archive / "unmapped_files.txt").read_text(encoding="utf-8") + + assert "src/unused_api.h" not in records + assert "no-data\tsrc/unused_api.h" in unmapped + + +@verifies("tool_req__coverage_report_relative_paths") +def test_html_links_resolve_to_generated_pages(llvm_html_archive: Path) -> None: + """Every link in the HTML index points to a generated report page.""" + html_root = llvm_html_archive / "coverage_linux" + links = index_links(html_root / "index.html") + assert links + + for link in links: + assert (html_root / link).is_file(), f"report page does not exist: {link}" + + +@verifies("tool_req__coverage_report_relative_paths") +def test_html_links_contain_no_machine_specific_paths(llvm_html_archive: Path) -> None: + """Generated links stay workspace-relative instead of exposing local paths.""" + html_root = llvm_html_archive / "coverage_linux" + links = index_links(html_root / "index.html") + assert links + + machine_specific_paths = ("bazel-out", "_virtual_includes", "/home/", "/tmp/") + for link in links: + for path_fragment in machine_specific_paths: + assert path_fragment not in link, f"report link contains {path_fragment}: {link}" + + +@verifies("tool_req__coverage_report_allowlist") +def test_html_includes_declared_external_and_vendored_headers(llvm_html_archive: Path) -> None: + """The source allowlist includes declared external and vendored headers.""" + html_root = llvm_html_archive / "coverage_linux" + links = index_links(html_root / "index.html") + external_header = "coverage/external/itest_external+/include/vext/vext.h.html" + vendored_header = "coverage/src/vendored/include/vendored/inline_math.h.html" + assert external_header in links, f"missing external header page: {links}" + assert vendored_header in links, f"missing vendored header page: {links}" + + +@verifies("tool_req__coverage_report_allowlist") +def test_html_excludes_forwarded_external_library(llvm_html_archive: Path) -> None: + """Forwarded external library code is absent from the HTML and LCOV outputs.""" + html_root = llvm_html_archive / "coverage_linux" + index = (html_root / "index.html").read_text(encoding="utf-8") + lcov = lcov_from_artifacts(llvm_html_archive) + + assert "itest_external+/extlib" not in index + assert "extlib" not in lcov + + +@verifies("tool_req__coverage_report_relative_paths") +def test_html_pages_link_to_existing_stylesheets(llvm_html_archive: Path) -> None: + """Every HTML source page links to a stylesheet that exists beside it.""" + html_root = llvm_html_archive / "coverage_linux" + links = index_links(html_root / "index.html") + + for link in links: + page = html_root / link + stylesheet = re.search( + r"href=['\"]((?:\.\./)*style\.css)['\"]", + page.read_text(encoding="utf-8"), + ) + assert stylesheet, f"{link} does not reference its stylesheet" + assert (page.parent / stylesheet.group(1)).is_file(), f"{link} stylesheet does not resolve" + + +@verifies("tool_req__coverage_scope_platform") +def test_llvm_report_selects_host_platform_sources(llvm_effective_archive: Path) -> None: + """Without a target override, the LLVM scope follows the host platform.""" + records = lcov_records(lcov_from_artifacts(llvm_effective_archive)) + assert "src/platform_host.cpp" in records + assert "src/host_root.cpp" in records + assert "src/platform_target.cpp" not in records + assert "src/target_root.cpp" not in records