diff --git a/.github/workflows/kurtosis_assertoor.yml b/.github/workflows/kurtosis_assertoor.yml new file mode 100644 index 000000000..2e53fc8e4 --- /dev/null +++ b/.github/workflows/kurtosis_assertoor.yml @@ -0,0 +1,203 @@ +name: Kurtosis Assertoor + +# Starts a devnet with ethlambda beacon and validator clients next to +# lighthouse, then runs assertoor's stability, block-proposal and sync checks +# against every client pair. See tooling/kurtosis-assertoor/README.md. + +on: + push: + branches: [main] + pull_request: + paths: + - "tooling/kurtosis-assertoor/**" + - ".github/workflows/kurtosis_assertoor.yml" + workflow_dispatch: + inputs: + image: + description: "ethlambda image to test (default: ghcr.io/lambdaclass/ethlambda:unstable)" + required: false + type: string + +permissions: + contents: read + +# A newer push to main (or to the PR) makes this run's result stale. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + IMAGE_REPO: ghcr.io/lambdaclass/ethlambda + KURTOSIS_VERSION: "1.20.0" + # The fork of ethpandaops/ethereum-package that adds `cl_type: ethlambda` and + # `vc_type: ethlambda`, pinned to a commit. Bump it deliberately. + ETHEREUM_PACKAGE_REPO: https://github.com/lambdaclass/ethereum-package.git + ETHEREUM_PACKAGE_REF: 67cb0e02dc0bd49ec9461cbf2ef7ac4a3232755f + ENCLAVE: ethlambda-assertoor + +jobs: + image: + name: Select ethlambda image + runs-on: ubuntu-latest + timeout-minutes: 80 + outputs: + image: ${{ steps.select.outputs.image }} + steps: + - name: Select image + id: select + env: + EVENT: ${{ github.event_name }} + INPUT_IMAGE: ${{ inputs.image }} + SHA: ${{ github.sha }} + run: | + case "$EVENT" in + push) image="$IMAGE_REPO:sha-${SHA::7}";; + workflow_dispatch) image="${INPUT_IMAGE:-$IMAGE_REPO:unstable}";; + *) image="$IMAGE_REPO:unstable";; + esac + echo "image=$image" >> "$GITHUB_OUTPUT" + echo "Testing $image" + + # docker_publish.yaml pushes sha-<7> a while after the push lands, so + # wait for it. Giving up after 75 minutes is a failure, not a pass. + - name: Wait for the image of this commit + if: github.event_name == 'push' + env: + IMAGE: ${{ steps.select.outputs.image }} + run: | + deadline=$((SECONDS + 75 * 60)) + until docker manifest inspect "$IMAGE" > /dev/null 2>&1; do + if [ "$SECONDS" -ge "$deadline" ]; then + echo "::error::$IMAGE was not published within 75 minutes" + exit 1 + fi + echo "Waiting for $IMAGE" + sleep 60 + done + echo "$IMAGE is available" + + assertoor: + name: Assertoor + needs: image + runs-on: ubuntu-latest + timeout-minutes: 120 + outputs: + failed: ${{ steps.collect.outputs.failed }} + steps: + - uses: actions/checkout@v6 + + - name: Free up disk space + uses: ./.github/actions/free-disk + + - name: Install kurtosis + run: | + echo "deb [trusted=yes] https://sdk.kurtosis.com/kurtosis-cli-release-artifacts/ /" \ + | sudo tee /etc/apt/sources.list.d/kurtosis.list + sudo apt-get update + sudo apt-get install -y "kurtosis-cli=$KURTOSIS_VERSION" + kurtosis analytics disable + kurtosis version + + # Run the package from a clone, not as `github.com/...@`: kurtosis + # refuses a remote package whose kurtosis.yml names another repository, + # and the fork's still says ethpandaops/ethereum-package. + - name: Fetch ethereum-package at the pinned commit + run: | + git init "$RUNNER_TEMP/ethereum-package" + cd "$RUNNER_TEMP/ethereum-package" + git fetch --depth 1 "$ETHEREUM_PACKAGE_REPO" "$ETHEREUM_PACKAGE_REF" + git checkout --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$ETHEREUM_PACKAGE_REF" + + - name: Point the args at the image under test + env: + IMAGE: ${{ needs.image.outputs.image }} + run: | + cp tooling/kurtosis-assertoor/network_params.yaml "$RUNNER_TEMP/args.yaml" + yq -i ' + (.participants[] | select(.cl_type == "ethlambda") | .cl_image) = strenv(IMAGE) | + (.participants[] | select(.vc_type == "ethlambda") | .vc_image) = strenv(IMAGE) + ' "$RUNNER_TEMP/args.yaml" + cat "$RUNNER_TEMP/args.yaml" + + - name: Start the devnet + run: | + kurtosis run "$RUNNER_TEMP/ethereum-package" \ + --enclave "$ENCLAVE" --args-file "$RUNNER_TEMP/args.yaml" \ + | tee "$RUNNER_TEMP/kurtosis_output.txt" + + - name: Wait for the assertoor tests + id: tests + run: | + url=$(kurtosis port print "$ENCLAVE" assertoor http) + expected=$(yq '.assertoor_params.tests | length' "$RUNNER_TEMP/args.yaml") + tooling/kurtosis-assertoor/scripts/wait-assertoor.sh \ + "$url" "$expected" 95 "$RUNNER_TEMP/assertoor-results" + + # Runs after a failure too: the notify job names the failed tests. + - name: Collect failed tests + id: collect + if: always() + run: | + failed="$RUNNER_TEMP/assertoor-results/failed.txt" + { + echo "failed<> "$GITHUB_OUTPUT" + + - name: Dump the enclave + if: always() + run: | + mkdir -p "$RUNNER_TEMP/dump" + cp "$RUNNER_TEMP/args.yaml" "$RUNNER_TEMP/dump/args.yaml" + cp "$RUNNER_TEMP/kurtosis_output.txt" "$RUNNER_TEMP/dump/" || true + cp -r "$RUNNER_TEMP/assertoor-results" "$RUNNER_TEMP/dump/" || true + cd "$RUNNER_TEMP/dump" + kurtosis enclave dump "$ENCLAVE" || true + + - name: Upload the dump and results + if: always() + uses: actions/upload-artifact@v7 + with: + name: enclave-dump-${{ github.run_id }} + path: ${{ runner.temp }}/dump + if-no-files-found: warn + + - name: Remove the enclave + if: always() + run: kurtosis enclave rm -f "$ENCLAVE" || true + + notify: + name: Post failure to Slack + needs: [image, assertoor] + if: failure() && github.event_name == 'push' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Build Slack payload + # Values go through the environment and the JSON is built with jq, as in + # daily_ci.yml, so a failed-test name cannot break the payload. + env: + SHA: ${{ github.sha }} + FAILED: ${{ needs.assertoor.outputs.failed }} + IMAGE_RESULT: ${{ needs.image.result }} + COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + if [ "$IMAGE_RESULT" != "success" ]; then + FAILED="no image: the sha-${SHA::7} tag never appeared" + fi + # The backticks are literal Slack mrkdwn, not command substitution. + # shellcheck disable=SC2016 + text=$(printf '❌ *Kurtosis Assertoor* failed on <%s|`%s`>\nFailed: %s\n<%s|View run>' \ + "$COMMIT_URL" "${SHA::7}" "$(echo "$FAILED" | paste -sd, - | sed 's/,/, /g')" "$RUN_URL") + jq -n --arg text "$text" \ + '{blocks: [{type: "section", text: {type: "mrkdwn", text: $text}}]}' \ + > assertoor_slack.json + + - name: Post to Slack + env: + SLACK_WEBHOOK: ${{ secrets.ETHLAMBDA_GENERAL_SLACK_WEBHOOK }} + run: bash .github/scripts/publish_slack.sh assertoor_slack.json diff --git a/CLAUDE.md b/CLAUDE.md index 147599c7a..d6b1133f8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -117,6 +117,9 @@ See `.claude/skills/devnet-runner/SKILL.md` for running a local multi-client dev (node roster, image tags, pause/unpause instability testing) and `.claude/skills/devnet-log-review/SKILL.md` for analyzing the dumped logs. +The kurtosis + assertoor battery (ethlambda beacon and validator next to lighthouse, run on +every push to `main`) is in `tooling/kurtosis-assertoor/README.md`. + ## Important Patterns & Idioms ### Trait Implementations diff --git a/tooling/kurtosis-assertoor/README.md b/tooling/kurtosis-assertoor/README.md new file mode 100644 index 000000000..84faa773a --- /dev/null +++ b/tooling/kurtosis-assertoor/README.md @@ -0,0 +1,70 @@ +# Kurtosis + assertoor battery + +Starts a mainnet-preset devnet in kurtosis and runs [assertoor](https://github.com/ethpandaops/assertoor) +checks against every client pair, ethlambda included. + +``` +participant 1 lighthouse + geth lighthouse validator (bootnode) +participant 2 lighthouse + geth ethlambda validator +participant 3 ethlambda beacon + geth ethlambda validator +``` + +| Check | Playbook | What passing shows | +|---|---|---| +| Stability | `stable/stability-check` | Finality, participation, no reorg or fork churn, all nodes synced | +| Block proposals | `stable/block-proposal-check` | Every client pair that holds validators proposed a block | +| Sync | `dev/synchronized-check` | Every CL and EL, `cl-3-ethlambda-geth` included, reports synced | + +The devnet comes from `lambdaclass/ethereum-package`, a fork of ethpandaops' +package that adds `cl_type: ethlambda` and `vc_type: ethlambda`. It is pinned to a +commit in `.github/workflows/kurtosis_assertoor.yml` (`ETHEREUM_PACKAGE_REF`). +Assertoor's image and the three playbook URLs are pinned to one assertoor commit +in `network_params.yaml`; bump them together. + +## Run it locally + +Needs Docker and the kurtosis CLI (CI uses 1.20.0). From the repository root: + +```bash +git clone https://github.com/lambdaclass/ethereum-package.git /tmp/ethereum-package +git -C /tmp/ethereum-package checkout 67cb0e02dc0bd49ec9461cbf2ef7ac4a3232755f + +kurtosis run /tmp/ethereum-package --enclave ethlambda-assertoor \ + --args-file tooling/kurtosis-assertoor/network_params.yaml + +url=$(kurtosis port print ethlambda-assertoor assertoor http) +tooling/kurtosis-assertoor/scripts/wait-assertoor.sh "$url" 3 +open "$url" # assertoor web UI + +kurtosis enclave rm -f ethlambda-assertoor +``` + +It runs from a clone because kurtosis refuses a remote package whose +`kurtosis.yml` names another repository, and the fork's still says +`ethpandaops/ethereum-package`. To test a local build, set the ethlambda entries' +`cl_image` and `vc_image` in `network_params.yaml` (default `ghcr.io/lambdaclass/ethlambda:unstable`). +Expect about an hour: the stability check waits for finality at 12 s slots. + +## In CI + +`.github/workflows/kurtosis_assertoor.yml`: + +| Trigger | Image under test | +|---|---| +| push to `main` | `ghcr.io/lambdaclass/ethlambda:sha-<7>`, waited for up to 75 min until `docker_publish.yaml` pushes it | +| pull request touching this directory or the workflow | `ghcr.io/lambdaclass/ethlambda:unstable` | +| manual dispatch | the `image` input, else `unstable` | + +The workflow rewrites the ethlambda images with `yq`, runs the battery, and always +uploads the enclave dump and assertoor results (`enclave-dump-`) before +removing the enclave. A failure on `main` posts the commit, the failed tests and +the run URL to Slack. + +## Known failure + +Until ethlambda's `/eth/v1/config/spec` reports `GLOAS_FORK_EPOCH`, every test +fails after its 5-minute client health check. Assertoor compares each CL's spec +field by field and marks a node whose spec differs offline (`invalid node +specs: spec mismatch: GloasForkEpoch`); lighthouse reports the far-future +value and ethlambda omits the key. The chain itself is healthy (all nodes share +a head and finalize), so this is a spec-endpoint gap, not a harness problem. diff --git a/tooling/kurtosis-assertoor/network_params.yaml b/tooling/kurtosis-assertoor/network_params.yaml new file mode 100644 index 000000000..03aec974b --- /dev/null +++ b/tooling/kurtosis-assertoor/network_params.yaml @@ -0,0 +1,60 @@ +# The assertoor battery for ethlambda: a mainnet-preset devnet from +# lambdaclass/ethereum-package (the fork that adds `cl_type: ethlambda` and +# `vc_type: ethlambda`), with ethereum-package's built-in assertoor checking +# every client pair. See README.md ("Run it locally") for the steps. The package +# comes from a clone of lambdaclass/ethereum-package at the commit pinned in +# .github/workflows/kurtosis_assertoor.yml (ETHEREUM_PACKAGE_REF), since kurtosis +# refuses the fork as a remote package. +# +# Participants, in the order ethereum-package launches them: +# 1. lighthouse + geth, lighthouse validator. It has to come first: the first +# participant is the network's bootnode, and ethlambda does not report an +# ENR over the beacon API. +# 2. lighthouse + geth, validated by the ethlambda validator client, so the +# validator client is checked against a beacon node that is not ethlambda. +# 3. ethlambda beacon node + geth, validated by the ethlambda validator client. +# +# Every participant holds validators, so block-proposal-check can require each +# client pair to propose. The workflow rewrites the `cl_image` and `vc_image` +# of the ethlambda entries (selected by type, not position) to the image under +# test; the values below are what a local run uses. +participants: + - el_type: geth + cl_type: lighthouse + - el_type: geth + cl_type: lighthouse + vc_type: ethlambda + vc_image: ghcr.io/lambdaclass/ethlambda:unstable + - el_type: geth + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:unstable + vc_type: ethlambda + vc_image: ghcr.io/lambdaclass/ethlambda:unstable +network_params: + # ethlambda only supports the mainnet preset: 32-slot epochs of 12 s. + preset: mainnet + # Fulu is the last fork ethlambda processes; gloas stays unscheduled. + electra_fork_epoch: 0 + fulu_fork_epoch: 0 +additional_services: + - assertoor +# The image and the playbooks come from one assertoor commit, so a playbook +# never names a task the image lacks. Bump the three together. The built-in +# `run_*_check` flags are off because they run the fork's older copies of the +# playbooks. +assertoor_params: + image: ethpandaops/assertoor:master-87bbbd6 + tests: + # Finality and participation. Every validator is held by a lighthouse or an + # ethlambda validator client, so passing means ethlambda beacon and its + # validator client attest and finalize with everyone else. + - file: https://raw.githubusercontent.com/ethpandaops/assertoor/87bbbd6c4efbe88be2cd2d0f40ca828d9e8f7088/playbooks/stable/stability-check.yaml + timeout: 75m + # Every client pair holding validators proposes at least one block. + - file: https://raw.githubusercontent.com/ethpandaops/assertoor/87bbbd6c4efbe88be2cd2d0f40ca828d9e8f7088/playbooks/stable/block-proposal-check.yaml + timeout: 45m + # Every CL and EL client reports synced. The playbook's own timeout is 12 h + # (it is meant to be left running); the test-level timeout bounds it, and it + # returns as soon as every pair is synced. + - file: https://raw.githubusercontent.com/ethpandaops/assertoor/87bbbd6c4efbe88be2cd2d0f40ca828d9e8f7088/playbooks/dev/synchronized-check.yaml + timeout: 20m diff --git a/tooling/kurtosis-assertoor/scripts/wait-assertoor.sh b/tooling/kurtosis-assertoor/scripts/wait-assertoor.sh new file mode 100755 index 000000000..4f808d812 --- /dev/null +++ b/tooling/kurtosis-assertoor/scripts/wait-assertoor.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# +# Waits for every assertoor test run to finish and fails unless all of them +# succeeded. +# +# Usage: wait-assertoor.sh [timeout-minutes] [out-dir] +# +# The URL is what `kurtosis port print assertoor http` prints. Writes +# /test_runs.json, one run-.json per run (with per-task results), +# and /failed.txt naming every run that did not succeed. A run that +# is still going at the deadline counts as failed, as does a run that never +# showed up. + +set -euo pipefail + +URL="${1:?assertoor url required}" +EXPECTED="${2:?expected test count required}" +TIMEOUT_MIN="${3:-100}" +OUT="${4:-assertoor-results}" + +mkdir -p "$OUT" +: > "$OUT/failed.txt" +deadline=$((SECONDS + TIMEOUT_MIN * 60)) +runs='[]' + +while :; do + # Assertoor takes a moment to listen after kurtosis reports it started. + if fetched=$(curl -sf --max-time 10 "$URL/api/v1/test_runs" | jq -c '.data'); then + runs="$fetched" + total=$(jq 'length' <<<"$runs") + active=$(jq '[.[] | select(.status == "pending" or .status == "running")] | length' <<<"$runs") + echo "[$(date -u +%H:%M:%S)] $total/$EXPECTED runs seen, $active active:" \ + "$(jq -r '[.[] | "\(.name)=\(.status)"] | join(", ")' <<<"$runs")" + if [ "$total" -ge "$EXPECTED" ] && [ "$active" -eq 0 ]; then + break + fi + else + echo "[$(date -u +%H:%M:%S)] assertoor not answering yet" + fi + if [ "$SECONDS" -ge "$deadline" ]; then + echo "::error::timed out after ${TIMEOUT_MIN} minutes waiting for assertoor tests" + break + fi + sleep 30 +done + +echo "$runs" > "$OUT/test_runs.json" +for id in $(jq -r '.[].run_id' <<<"$runs"); do + curl -sf --max-time 30 "$URL/api/v1/test_run/$id/details" > "$OUT/run-$id.json" || true +done + +jq -r '.[] | select(.status != "success") | "\(.name) (\(.status))"' <<<"$runs" > "$OUT/failed.txt" +total=$(jq 'length' <<<"$runs") +if [ "$total" -lt "$EXPECTED" ]; then + echo "only $total of $EXPECTED test runs appeared" >> "$OUT/failed.txt" +fi + +echo +jq -r '.[] | "\(.status)\t\(.name)\t\((.stop_time - .start_time))s"' <<<"$runs" +if [ -s "$OUT/failed.txt" ]; then + echo "Failed:" + cat "$OUT/failed.txt" + exit 1 +fi +echo "All $total assertoor tests passed"