ci: add a kurtosis + assertoor battery for the beacon node

.github/workflows/kurtosis_assertoor.yml

@@ -0,0 +1,203 @@
+name: Kurtosis Assertoor
+
+# Starts a devnet with ethlambda beacon and validator clients next to
+# lighthouse, then runs assertoor's stability, block-proposal and sync checks
+# against every client pair. See tooling/kurtosis-assertoor/README.md.
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    paths:
+      - "tooling/kurtosis-assertoor/**"
+      - ".github/workflows/kurtosis_assertoor.yml"
+  workflow_dispatch:
+    inputs:
+      image:
+        description: "ethlambda image to test (default: ghcr.io/lambdaclass/ethlambda:unstable)"
+        required: false
+        type: string
+
+permissions:
+  contents: read
+
+# A newer push to main (or to the PR) makes this run's result stale.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  IMAGE_REPO: ghcr.io/lambdaclass/ethlambda
+  KURTOSIS_VERSION: "1.20.0"
+  # The fork of ethpandaops/ethereum-package that adds `cl_type: ethlambda` and
+  # `vc_type: ethlambda`, pinned to a commit. Bump it deliberately.
+  ETHEREUM_PACKAGE_REPO: https://github.com/lambdaclass/ethereum-package.git
+  ETHEREUM_PACKAGE_REF: 67cb0e02dc0bd49ec9461cbf2ef7ac4a3232755f
+  ENCLAVE: ethlambda-assertoor
+
+jobs:
+  image:
+    name: Select ethlambda image
+    runs-on: ubuntu-latest
+    timeout-minutes: 80
+    outputs:
+      image: ${{ steps.select.outputs.image }}
+    steps:
+      - name: Select image
+        id: select
+        env:
+          EVENT: ${{ github.event_name }}
+          INPUT_IMAGE: ${{ inputs.image }}
+          SHA: ${{ github.sha }}
+        run: |
+          case "$EVENT" in
+            push)              image="$IMAGE_REPO:sha-${SHA::7}";;
+            workflow_dispatch) image="${INPUT_IMAGE:-$IMAGE_REPO:unstable}";;
+            *)                 image="$IMAGE_REPO:unstable";;
+          esac
+          echo "image=$image" >> "$GITHUB_OUTPUT"
+          echo "Testing $image"
+
+      # docker_publish.yaml pushes sha-<7> a while after the push lands, so
+      # wait for it. Giving up after 75 minutes is a failure, not a pass.
+      - name: Wait for the image of this commit
+        if: github.event_name == 'push'
+        env:
+          IMAGE: ${{ steps.select.outputs.image }}
+        run: |
+          deadline=$((SECONDS + 75 * 60))
+          until docker manifest inspect "$IMAGE" > /dev/null 2>&1; do
+            if [ "$SECONDS" -ge "$deadline" ]; then
+              echo "::error::$IMAGE was not published within 75 minutes"
+              exit 1
+            fi
+            echo "Waiting for $IMAGE"
+            sleep 60
+          done
+          echo "$IMAGE is available"
+
+  assertoor:
+    name: Assertoor
+    needs: image
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    outputs:
+      failed: ${{ steps.collect.outputs.failed }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Free up disk space
+        uses: ./.github/actions/free-disk
+
+      - name: Install kurtosis
+        run: |
+          echo "deb [trusted=yes] https://sdk.kurtosis.com/kurtosis-cli-release-artifacts/ /" \
+            | sudo tee /etc/apt/sources.list.d/kurtosis.list
+          sudo apt-get update
+          sudo apt-get install -y "kurtosis-cli=$KURTOSIS_VERSION"
+          kurtosis analytics disable
+          kurtosis version
+
+      # Run the package from a clone, not as `github.com/...@<ref>`: kurtosis
+      # refuses a remote package whose kurtosis.yml names another repository,
+      # and the fork's still says ethpandaops/ethereum-package.
+      - name: Fetch ethereum-package at the pinned commit
+        run: |
+          git init "$RUNNER_TEMP/ethereum-package"
+          cd "$RUNNER_TEMP/ethereum-package"
+          git fetch --depth 1 "$ETHEREUM_PACKAGE_REPO" "$ETHEREUM_PACKAGE_REF"
+          git checkout --detach FETCH_HEAD
+          test "$(git rev-parse HEAD)" = "$ETHEREUM_PACKAGE_REF"
+
+      - name: Point the args at the image under test
+        env:
+          IMAGE: ${{ needs.image.outputs.image }}
+        run: |
+          cp tooling/kurtosis-assertoor/network_params.yaml "$RUNNER_TEMP/args.yaml"
+          yq -i '
+            (.participants[] | select(.cl_type == "ethlambda") | .cl_image) = strenv(IMAGE) |
+            (.participants[] | select(.vc_type == "ethlambda") | .vc_image) = strenv(IMAGE)
+          ' "$RUNNER_TEMP/args.yaml"
+          cat "$RUNNER_TEMP/args.yaml"
+
+      - name: Start the devnet
+        run: |
+          kurtosis run "$RUNNER_TEMP/ethereum-package" \
+            --enclave "$ENCLAVE" --args-file "$RUNNER_TEMP/args.yaml" \
+            | tee "$RUNNER_TEMP/kurtosis_output.txt"
+
+      - name: Wait for the assertoor tests
+        id: tests
+        run: |
+          url=$(kurtosis port print "$ENCLAVE" assertoor http)
+          expected=$(yq '.assertoor_params.tests | length' "$RUNNER_TEMP/args.yaml")
+          tooling/kurtosis-assertoor/scripts/wait-assertoor.sh \
+            "$url" "$expected" 95 "$RUNNER_TEMP/assertoor-results"
+
+      # Runs after a failure too: the notify job names the failed tests.
+      - name: Collect failed tests
+        id: collect
+        if: always()
+        run: |
+          failed="$RUNNER_TEMP/assertoor-results/failed.txt"
+          {
+            echo "failed<<EOF"
+            if [ -s "$failed" ]; then cat "$failed"; else echo "assertoor did not produce a result (the devnet did not start?)"; fi
+            echo "EOF"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Dump the enclave
+        if: always()
+        run: |
+          mkdir -p "$RUNNER_TEMP/dump"
+          cp "$RUNNER_TEMP/args.yaml" "$RUNNER_TEMP/dump/args.yaml"
+          cp "$RUNNER_TEMP/kurtosis_output.txt" "$RUNNER_TEMP/dump/" || true
+          cp -r "$RUNNER_TEMP/assertoor-results" "$RUNNER_TEMP/dump/" || true
+          cd "$RUNNER_TEMP/dump"
+          kurtosis enclave dump "$ENCLAVE" || true
+
+      - name: Upload the dump and results
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: enclave-dump-${{ github.run_id }}
+          path: ${{ runner.temp }}/dump
+          if-no-files-found: warn
+
+      - name: Remove the enclave
+        if: always()
+        run: kurtosis enclave rm -f "$ENCLAVE" || true
+
+  notify:
+    name: Post failure to Slack
+    needs: [image, assertoor]
+    if: failure() && github.event_name == 'push'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Build Slack payload
+        # Values go through the environment and the JSON is built with jq, as in
+        # daily_ci.yml, so a failed-test name cannot break the payload.
+        env:
+          SHA: ${{ github.sha }}
+          FAILED: ${{ needs.assertoor.outputs.failed }}
+          IMAGE_RESULT: ${{ needs.image.result }}
+          COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}
+          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        run: |
+          if [ "$IMAGE_RESULT" != "success" ]; then
+            FAILED="no image: the sha-${SHA::7} tag never appeared"
+          fi
+          # The backticks are literal Slack mrkdwn, not command substitution.
+          # shellcheck disable=SC2016
+          text=$(printf '❌ *Kurtosis Assertoor* failed on <%s|`%s`>\nFailed: %s\n<%s|View run>' \
+            "$COMMIT_URL" "${SHA::7}" "$(echo "$FAILED" | paste -sd, - | sed 's/,/, /g')" "$RUN_URL")
+          jq -n --arg text "$text" \
+            '{blocks: [{type: "section", text: {type: "mrkdwn", text: $text}}]}' \
+            > assertoor_slack.json
+
+      - name: Post to Slack
+        env:
+          SLACK_WEBHOOK: ${{ secrets.ETHLAMBDA_GENERAL_SLACK_WEBHOOK }}
+        run: bash .github/scripts/publish_slack.sh assertoor_slack.json

CLAUDE.md

@@ -117,6 +117,9 @@ See `.claude/skills/devnet-runner/SKILL.md` for running a local multi-client dev
 (node roster, image tags, pause/unpause instability testing) and
 `.claude/skills/devnet-log-review/SKILL.md` for analyzing the dumped logs.
 
+The kurtosis + assertoor battery (ethlambda beacon and validator next to lighthouse, run on
+every push to `main`) is in `tooling/kurtosis-assertoor/README.md`.
+
 ## Important Patterns & Idioms
 
 ### Trait Implementations

tooling/kurtosis-assertoor/README.md

@@ -0,0 +1,70 @@
+# Kurtosis + assertoor battery
+
+Starts a mainnet-preset devnet in kurtosis and runs [assertoor](https://github.com/ethpandaops/assertoor)
+checks against every client pair, ethlambda included.
+
+```
+participant 1  lighthouse + geth                       lighthouse validator   (bootnode)
+participant 2  lighthouse + geth                       ethlambda validator
+participant 3  ethlambda beacon + geth                 ethlambda validator
+```
+
+| Check | Playbook | What passing shows |
+|---|---|---|
+| Stability | `stable/stability-check` | Finality, participation, no reorg or fork churn, all nodes synced |
+| Block proposals | `stable/block-proposal-check` | Every client pair that holds validators proposed a block |
+| Sync | `dev/synchronized-check` | Every CL and EL, `cl-3-ethlambda-geth` included, reports synced |
+
+The devnet comes from `lambdaclass/ethereum-package`, a fork of ethpandaops'
+package that adds `cl_type: ethlambda` and `vc_type: ethlambda`. It is pinned to a
+commit in `.github/workflows/kurtosis_assertoor.yml` (`ETHEREUM_PACKAGE_REF`).
+Assertoor's image and the three playbook URLs are pinned to one assertoor commit
+in `network_params.yaml`; bump them together.
+
+## Run it locally
+
+Needs Docker and the kurtosis CLI (CI uses 1.20.0). From the repository root:
+
+```bash
+git clone https://github.com/lambdaclass/ethereum-package.git /tmp/ethereum-package
+git -C /tmp/ethereum-package checkout 67cb0e02dc0bd49ec9461cbf2ef7ac4a3232755f
+
+kurtosis run /tmp/ethereum-package --enclave ethlambda-assertoor \
+  --args-file tooling/kurtosis-assertoor/network_params.yaml
+
+url=$(kurtosis port print ethlambda-assertoor assertoor http)
+tooling/kurtosis-assertoor/scripts/wait-assertoor.sh "$url" 3
+open "$url"                                   # assertoor web UI
+
+kurtosis enclave rm -f ethlambda-assertoor
+```
+
+It runs from a clone because kurtosis refuses a remote package whose
+`kurtosis.yml` names another repository, and the fork's still says
+`ethpandaops/ethereum-package`. To test a local build, set the ethlambda entries'
+`cl_image` and `vc_image` in `network_params.yaml` (default `ghcr.io/lambdaclass/ethlambda:unstable`).
+Expect about an hour: the stability check waits for finality at 12 s slots.
+
+## In CI
+
+`.github/workflows/kurtosis_assertoor.yml`:
+
+| Trigger | Image under test |
+|---|---|
+| push to `main` | `ghcr.io/lambdaclass/ethlambda:sha-<7>`, waited for up to 75 min until `docker_publish.yaml` pushes it |
+| pull request touching this directory or the workflow | `ghcr.io/lambdaclass/ethlambda:unstable` |
+| manual dispatch | the `image` input, else `unstable` |
+
+The workflow rewrites the ethlambda images with `yq`, runs the battery, and always
+uploads the enclave dump and assertoor results (`enclave-dump-<run id>`) before
+removing the enclave. A failure on `main` posts the commit, the failed tests and
+the run URL to Slack.
+
+## Known failure
+
+Until ethlambda's `/eth/v1/config/spec` reports `GLOAS_FORK_EPOCH`, every test
+fails after its 5-minute client health check. Assertoor compares each CL's spec
+field by field and marks a node whose spec differs offline (`invalid node
+specs: spec mismatch: GloasForkEpoch`); lighthouse reports the far-future
+value and ethlambda omits the key. The chain itself is healthy (all nodes share
+a head and finalize), so this is a spec-endpoint gap, not a harness problem.

tooling/kurtosis-assertoor/network_params.yaml

@@ -0,0 +1,60 @@
+# The assertoor battery for ethlambda: a mainnet-preset devnet from
+# lambdaclass/ethereum-package (the fork that adds `cl_type: ethlambda` and
+# `vc_type: ethlambda`), with ethereum-package's built-in assertoor checking
+# every client pair. See README.md ("Run it locally") for the steps. The package
+# comes from a clone of lambdaclass/ethereum-package at the commit pinned in
+# .github/workflows/kurtosis_assertoor.yml (ETHEREUM_PACKAGE_REF), since kurtosis
+# refuses the fork as a remote package.
+#
+# Participants, in the order ethereum-package launches them:
+#   1. lighthouse + geth, lighthouse validator. It has to come first: the first
+#      participant is the network's bootnode, and ethlambda does not report an
+#      ENR over the beacon API.
+#   2. lighthouse + geth, validated by the ethlambda validator client, so the
+#      validator client is checked against a beacon node that is not ethlambda.
+#   3. ethlambda beacon node + geth, validated by the ethlambda validator client.
+#
+# Every participant holds validators, so block-proposal-check can require each
+# client pair to propose. The workflow rewrites the `cl_image` and `vc_image`
+# of the ethlambda entries (selected by type, not position) to the image under
+# test; the values below are what a local run uses.
+participants:
+  - el_type: geth
+    cl_type: lighthouse
+  - el_type: geth
+    cl_type: lighthouse
+    vc_type: ethlambda
+    vc_image: ghcr.io/lambdaclass/ethlambda:unstable
+  - el_type: geth
+    cl_type: ethlambda
+    cl_image: ghcr.io/lambdaclass/ethlambda:unstable
+    vc_type: ethlambda
+    vc_image: ghcr.io/lambdaclass/ethlambda:unstable
+network_params:
+  # ethlambda only supports the mainnet preset: 32-slot epochs of 12 s.
+  preset: mainnet
+  # Fulu is the last fork ethlambda processes; gloas stays unscheduled.
+  electra_fork_epoch: 0
+  fulu_fork_epoch: 0
+additional_services:
+  - assertoor
+# The image and the playbooks come from one assertoor commit, so a playbook
+# never names a task the image lacks. Bump the three together. The built-in
+# `run_*_check` flags are off because they run the fork's older copies of the
+# playbooks.
+assertoor_params:
+  image: ethpandaops/assertoor:master-87bbbd6
+  tests:
+    # Finality and participation. Every validator is held by a lighthouse or an
+    # ethlambda validator client, so passing means ethlambda beacon and its
+    # validator client attest and finalize with everyone else.
+    - file: https://raw.githubusercontent.com/ethpandaops/assertoor/87bbbd6c4efbe88be2cd2d0f40ca828d9e8f7088/playbooks/stable/stability-check.yaml
+      timeout: 75m
+    # Every client pair holding validators proposes at least one block.
+    - file: https://raw.githubusercontent.com/ethpandaops/assertoor/87bbbd6c4efbe88be2cd2d0f40ca828d9e8f7088/playbooks/stable/block-proposal-check.yaml
+      timeout: 45m
+    # Every CL and EL client reports synced. The playbook's own timeout is 12 h
+    # (it is meant to be left running); the test-level timeout bounds it, and it
+    # returns as soon as every pair is synced.
+    - file: https://raw.githubusercontent.com/ethpandaops/assertoor/87bbbd6c4efbe88be2cd2d0f40ca828d9e8f7088/playbooks/dev/synchronized-check.yaml
+      timeout: 20m