Migrate coverage assertions to end-to-end pytest cases

.github/workflows/tests.yml

@@ -83,9 +83,7 @@ jobs:
       - uses: eclipse-score/cicd-actions/setup-bazel-cache@setup-bazel-cache/v0.1.0
         with:
           disk-cache-key: end_to_end_tests
-      - name: Run the end-to-end pipeline test (C++ + Rust consumer workspace)
-        run: tests/end_to_end/run_end_to_end_test.sh
-      - name: Run the end-to-end pytest CLI case
+      - name: Run traceable pytest black-box cases
         run: bazel test --lockfile_mode=error //tests/end_to_end/testcases:blackbox_test
       - name: Collect pytest JUnit results for the documentation build
         if: always()
@@ -108,7 +106,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: end_to_end_coverage_report
-          path: tests/end_to_end/consumer/coverage_artifacts.zip
+          path: bazel-testlogs/tests/end_to_end/testcases/blackbox_test/test.outputs/coverage_artifact
           if-no-files-found: ignore
           retention-days: 10
   test_reports:

README.md

@@ -72,8 +72,7 @@ bazel test //tests/unit/... //tools/...          # unit + analysis tests
 bazel build --config=lint //score_coverage/... //tools/... //tests/...   # ruff, pylint, ty
 bazel coverage --combined_report=lcov //tests/unit/score_coverage:all
 bazel run //tools:self_coverage_gate -- --min-lines 95 --min-branches 87
-tests/end_to_end/run_end_to_end_test.sh            # end-to-end (downloads LLVM + Ferrocene)
-bazel test //tests/end_to_end/testcases:blackbox_test # pytest CLI case
+bazel test //tests/end_to_end/testcases:blackbox_test # traceable C++ + Rust end-to-end cases
 pre-commit run --all-files
 bazel run //tools:copyright.check
 ```

docs/verification/verification_report.rst

@@ -82,10 +82,7 @@ Test inventory
      - scope_transitive, scope_excludes, scope_baseline_objects, scope_gcno,
        scope_platform
    * - ``//tests/end_to_end/testcases:blackbox_test``
-     - 1
-     - gate_no_verdict
-   * - ``tests/end_to_end/run_end_to_end_test.sh`` (27 end-to-end checks)
-     - 27
+     - 47
      - validation_ground_truth, instrumentation_hint, scope_platform, report_baseline_zero, report_relative_paths,
        report_allowlist, report_unmapped, gcov_merge, gcov_baseline, gcov_html,
        gate_exit_codes, gate_no_verdict, just_unknown_id, artifacts,
@@ -108,17 +105,14 @@ documentation build.
    :columns: id;title;testlink
    :style: table
 
-Four requirements are verified outside the pytest suites and therefore carry no
-generated link:
+Three requirements are verified outside the pytest suites and therefore carry
+no generated link:
 
 - :need:`tool_req__coverage_scope_transitive`,
   :need:`tool_req__coverage_scope_excludes` and
   :need:`tool_req__coverage_scope_baseline_objects` are verified by the seventeen
   Starlark analysis tests in ``tests/unit/starlark`` (rules_testing
   produces no test properties).
-- :need:`tool_req__coverage_validation_ground_truth` is verified by the
-  end-to-end run ``tests/end_to_end/run_end_to_end_test.sh`` (golden LCOV
-  comparison, see below).
 
 .. needpie:: Test results of the linked test cases
    :labels: passed, failed, skipped
@@ -175,10 +169,13 @@ yamlfmt the workflows; copyright headers are checked on every file.
 End-to-end validation
 ---------------------
 
-``tests/end_to_end/run_end_to_end_test.sh`` builds a consumer workspace with a
-tested and an untested C++ library, a header-only library reached through
+``//tests/end_to_end/testcases:blackbox_test`` runs named pytest cases against
+an isolated copy of the consumer workspace. Session fixtures collect LLVM and
+gcov reports once each; the cases invoke the public report command with
+different options and retain the LLVM archive in Bazel test outputs. The
+consumer workspace contains a tested and an untested C++ library, a header-only library reached through
 ``strip_include_prefix``, a tested Rust library and an untested Rust binary, one
-justified line, and asserts:
+justified line. The cases verify:
 
 1. the gate fails at 100 % and passes at 10 % (effective and raw mode);
 2. the HTML, the summary and the archive tree are produced, the summary also
@@ -187,7 +184,9 @@ justified line, and asserts:
 4. the LCOV matches ``expected_lcov.dat``, a hand-derived ground truth, record
    by record;
 5. the justified line raises effective above raw coverage;
-6. fault injection: a corrupt report and a non-numeric threshold exit 2, and a
+6. platform-specific sources and scope roots follow the host for LLVM and the
+   target platform for gcov;
+7. fault injection: a corrupt report and a non-numeric threshold exit 2, and a
    misspelt justification id is reported and does not raise the effective
    coverage.
 

tests/end_to_end/consumer/.bazelrc

@@ -109,3 +109,7 @@ coverage:gcov --test_lang_filters=cc
 # the tests too so header-only code that only a test translation unit
 # instantiates is measured (the scope allowlist still drops the test sources).
 coverage:gcov --instrument_test_targets
+
+# This test configuration leaves the cross-package library uninstrumented so
+# the end-to-end suite can check the reporter's instrumentation-filter hint.
+coverage:gcov_narrow_filter --instrumentation_filter=^//lib/test[/:],^//src[/:]