Shard coverage across two runners

.github/workflows/ci.yml

@@ -588,7 +588,17 @@ jobs:
     # numbers are report-only so contributors get visibility without flaky
     # CI when coverage shifts naturally with test edits. A failing TEST
     # fails the job: this is the Linux leg of `test`.
+    #
+    # Two shards (scripts/ci-test-shard.py, as `test` and `test-release`):
+    # the instrumented build takes ~110 s and running the tests ~220 s, so
+    # one leg was the queue's critical path (~6 min). Each shard compiles
+    # and runs only its targets and uploads its own LCOV; coverage-merge
+    # unions them. `ci-ok` waits for every shard.
     runs-on: ${{ vars.DISABLE_DEPOT_RUNNERS == 'true' && 'ubuntu-latest' || 'depot-ubuntu-24.04-16' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2]
     timeout-minutes: 35
     permissions:
       contents: read
@@ -616,12 +626,14 @@ jobs:
       - name: Cache cargo
         # Swatinem/rust-cache, main-only saves: see the first `Cache cargo`
         # step in this file.
+        # The shards share one cache key and shard 1 alone writes it.
+        # rust-cache keeps only dependencies, which every shard builds.
         uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
         with:
-          save-if: ${{ github.ref == 'refs/heads/main' }}
+          save-if: ${{ github.ref == 'refs/heads/main' && matrix.shard == 1 }}
 
       - name: Warm the coverage cache after merge-queue validation
-        if: needs.release-readiness.outputs.reuse == 'true'
+        if: needs.release-readiness.outputs.reuse == 'true' && matrix.shard == 1
         run: |
           cargo llvm-cov show-env --sh > "$RUNNER_TEMP/coverage-env.sh"
           source "$RUNNER_TEMP/coverage-env.sh"
@@ -672,6 +684,9 @@ jobs:
         env:
           SOCKET_PATCH_GO_E2E_REQUIRED: '1'
           SOCKET_PATCH_GO_E2E_VERSION: '1.24'
+          CI_TEST_COVERAGE: '1'
+          TEST_SHARD: ${{ matrix.shard }}
+          TEST_SHARD_COUNT: ${{ strategy.job-total }}
         #
         # The test output is also teed to a file so that a failing test's
         # panic becomes an annotation (scripts/ci-test-failures.py): in a
@@ -680,27 +695,29 @@ jobs:
         # can't be diagnosed once the re-run has passed.
         run: |
           set -o pipefail
-          cargo llvm-cov --workspace --no-fail-fast \
-            --no-report 2>&1 | tee "$RUNNER_TEMP/coverage-tests.log" || {
+          python3 scripts/ci-test-shard.py "$TEST_SHARD" "$TEST_SHARD_COUNT" \
+            2>&1 | tee "$RUNNER_TEMP/coverage-tests.log" || {
             status=$?
             python3 scripts/ci-test-failures.py "$RUNNER_TEMP/coverage-tests.log" || true
             exit "$status"
           }
           set +o pipefail
-          cargo llvm-cov report --lcov --output-path coverage-host.lcov
-          python3 scripts/ci-lcov-summary.py coverage-host.lcov | tee coverage-summary.txt
+          cargo llvm-cov report --lcov --output-path "coverage-host-$TEST_SHARD.lcov"
+          python3 scripts/ci-lcov-summary.py "coverage-host-$TEST_SHARD.lcov" | tee coverage-summary.txt
 
       - name: Publish coverage summary to job summary
         if: needs.release-readiness.outputs.reuse != 'true'
         # Render the per-file LCOV line/function/branch totals as a fenced
         # block in the GitHub Actions job summary so reviewers don't
         # need to crack open the artifact for a quick look.
+        env:
+          TEST_SHARD: ${{ matrix.shard }}
         run: |
           {
-            echo "## Host coverage summary"
+            echo "## Host coverage summary (shard $TEST_SHARD)"
             echo ""
-            echo "(In-process tests only. See coverage-merge for the"
-            echo "full picture including docker-e2e binary coverage.)"
+            echo "(This shard's in-process tests only. See coverage-merge"
+            echo "for the full picture including docker-e2e binary coverage.)"
             echo ""
             echo '```'
             cat coverage-summary.txt
@@ -711,8 +728,8 @@ jobs:
         if: needs.release-readiness.outputs.reuse != 'true'
         uses: ./.github/actions/upload-artifact
         with:
-          name: coverage-host
-          path: coverage-host.lcov
+          name: coverage-host-${{ matrix.shard }}
+          path: coverage-host-${{ matrix.shard }}.lcov
           if-no-files-found: error
           retention-days: 30
 

scripts/ci-test-shard.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Run one shard of ci.yml's `test` or `test-release` job:
+"""Run one shard of ci.yml's `test`, `test-release` or `coverage` job:
 `cargo test --workspace` split over `COUNT` runners. Each runner compiles
 and executes only its assigned integration targets, shortening both the
 debug and release-mode jobs without changing their compilation profiles.
@@ -14,7 +14,12 @@
 Those shards balance measured test time plus linking work, with shard 1's
 unit tests reserved up front. Other runners retain the count-based split.
 
-Extra arguments after `SHARD COUNT` go to every `cargo test` invocation.
+CI_TEST_COVERAGE=1 runs the shard under cargo-llvm-cov for ci.yml's
+`coverage` job: `cargo llvm-cov --no-report` in place of `cargo test`, and
+no doctests, which cargo-llvm-cov leaves out without `--doctests`. A later
+`cargo llvm-cov report` exports the shard's profiles.
+
+Extra arguments after `SHARD COUNT` go to every `cargo` invocation.
 Each invocation runs with `--no-fail-fast`; the exit status is non-zero if
 any of them failed.
 
@@ -68,16 +73,18 @@ def cost(name):
     return shards
 
 
-def invocations(shard, count, names, extra=(), timings=None):
+def invocations(shard, count, names, extra=(), timings=None, coverage=False):
     """The `cargo test` argument lists shard `shard` (1-based) runs."""
     if not 1 <= shard <= count:
         raise ValueError(f"shard {shard} not in 1..{count}")
-    base = ["cargo", "test", "--workspace", "--no-fail-fast", *extra]
+    runner = ["llvm-cov", "--no-report"] if coverage else ["test"]
+    base = ["cargo", *runner, "--workspace", "--no-fail-fast", *extra]
     mine = partition(names, count, timings)[shard - 1]
     runs = []
     if shard == 1:
         runs.append(base + ["--lib", "--bins"] + [a for n in mine for a in ("--test", n)])
-        runs.append(base + ["--doc"])
+        if not coverage:
+            runs.append(base + ["--doc"])
     elif mine:
         runs.append(base + [a for n in mine for a in ("--test", n)])
     return runs
@@ -94,10 +101,11 @@ def main(argv):
     names = integration_targets(metadata)
     timing_path = os.environ.get("CI_TEST_TIMINGS")
     timings = json.loads(Path(timing_path).read_text(encoding="utf-8")) if timing_path else None
+    coverage = os.environ.get("CI_TEST_COVERAGE") == "1"
     print(f"ci-test-shard: shard {shard}/{count}: "
           f"{len(partition(names, count, timings)[shard - 1])} of {len(names)} integration targets", flush=True)
     status = 0
-    for args in invocations(shard, count, names, argv[2:], timings):
+    for args in invocations(shard, count, names, argv[2:], timings, coverage):
         print("+ " + " ".join(args), flush=True)
         if subprocess.run(args).returncode != 0:
             status = 1

scripts/tests/test_ci_test_shard.py

@@ -71,6 +71,31 @@ def test_failed_unit_or_integration_run_still_runs_docs_and_fails_the_shard(self
         self.assertEqual(run.call_count, 3)
         self.assertIn("--doc", run.call_args_list[-1].args[0])
 
+    def test_coverage_shards_run_llvm_cov_without_doctests(self):
+        # The unsharded `cargo llvm-cov --workspace` ran no doctests; the
+        # shards must neither add them nor drop a unit or integration target.
+        for count in (1, 2):
+            runs = [args for k in range(1, count + 1)
+                    for args in shard.invocations(k, count, self.NAMES, coverage=True)]
+            self.assertTrue(all(a[:5] == ["cargo", "llvm-cov", "--no-report", "--workspace", "--no-fail-fast"]
+                                for a in runs))
+            self.assertFalse(any("--doc" in a for a in runs))
+            self.assertEqual(sum("--lib" in a and "--bins" in a for a in runs), 1)
+            self.assertEqual(sorted(selected(runs)), self.NAMES)
+
+    def test_coverage_mode_comes_from_the_environment(self):
+        metadata = {"workspace_members": ["a"], "packages": [
+            {"id": "a", "targets": [{"name": "integration", "kind": ["test"]}]}]}
+        env = {k: v for k, v in shard.os.environ.items() if k != "CI_TEST_TIMINGS"}
+        with patch.dict(shard.os.environ, {**env, "CI_TEST_COVERAGE": "1"}, clear=True), \
+                patch.object(shard.subprocess, "run", side_effect=[
+                    subprocess.CompletedProcess([], 0, stdout=json.dumps(metadata)),
+                    subprocess.CompletedProcess([], 0),
+                ]) as run:
+            self.assertEqual(shard.main(["1", "2"]), 0)
+        self.assertEqual(run.call_count, 2)
+        self.assertEqual(run.call_args_list[-1].args[0][:3], ["cargo", "llvm-cov", "--no-report"])
+
     def test_negative_bad_shard(self):
         with self.assertRaises(ValueError):
             shard.invocations(3, 2, self.NAMES)
@@ -134,6 +159,24 @@ def test_all_release_shards_are_required_and_use_the_matrix_size(self):
         self.assertIn("if: always()", verdict)
         self.assertIn('if v["result"] not in ("success", "skipped")', verdict)
 
+    def test_all_coverage_shards_are_required_and_upload_distinct_lcov(self):
+        # coverage is the gating Linux test run: every shard must run under
+        # cargo-llvm-cov, gate ci-ok, and upload an LCOV coverage-merge picks
+        # up (its `coverage-{host*,docker-*}` pattern) without name clashes.
+        workflow = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8")
+        cov = workflow.split("\n  coverage:\n")[1].split("\n  docker-base:\n")[0]
+        count = json.loads(re.search(r"^        shard: (\[.*\])$", cov, re.M)[1])
+        self.assertEqual(count, list(range(1, len(count) + 1)))
+        self.assertIn("fail-fast: false", cov)
+        self.assertIn("CI_TEST_COVERAGE: '1'", cov)
+        self.assertIn('python3 scripts/ci-test-shard.py "$TEST_SHARD" "$TEST_SHARD_COUNT"', cov)
+        self.assertIn("TEST_SHARD_COUNT: ${{ strategy.job-total }}", cov)
+        self.assertIn('--output-path "coverage-host-$TEST_SHARD.lcov"', cov)
+        self.assertIn("name: coverage-host-${{ matrix.shard }}", cov)
+        self.assertIn("pattern: coverage-{host*,docker-*}", workflow)
+        verdict = workflow.split("\n  ci-ok:\n")[1]
+        self.assertIn("coverage", re.search(r"^    needs: \[(.*)\]$", verdict, re.M)[1].split(", "))
+
 
 @unittest.skipUnless(shutil.which("cargo"), "cargo not available")
 class CargoSelection(unittest.TestCase):