diff --git a/config.example.toml b/config.example.toml index 4ce750cb..1c6354e6 100644 --- a/config.example.toml +++ b/config.example.toml @@ -31,7 +31,7 @@ wait_all_registrations = true # this should be lower than that, leaving some margin for overhead # OPTIONAL, DEFAULT: 950 timeout_get_header_ms = 950 -# (unreleased, from v0.12.0-rc1) ePBS bids only: ms kept back from the beacon node's X-Timeout-Ms deadline, must be under one slot, 12000 on mainnet (https://commit-boost.github.io/commit-boost-client/get_started/epbs#timing) +# (unreleased, from v0.12.0-rc1) ePBS bids only: ms kept back from the beacon node's X-Timeout-Ms deadline, must be above 0 and under one slot, 12000 on mainnet (https://commit-boost.github.io/commit-boost-client/get_started/epbs#timing) # OPTIONAL, DEFAULT: 50 # proposer_deadline_buffer_ms = 50 # Timeout in milliseconds for the `submit_blinded_block` call to relays. diff --git a/crates/common/src/config/pbs.rs b/crates/common/src/config/pbs.rs index 22f7a1ed..294c5b53 100644 --- a/crates/common/src/config/pbs.rs +++ b/crates/common/src/config/pbs.rs @@ -250,13 +250,12 @@ impl PbsConfig { ); ensure!(self.late_in_slot_time_ms > 0, "late_in_slot_time_ms must be greater than 0"); - // The buffer comes out of the proposer's deadline, which is clamped to - // one slot, so a buffer of a slot or more leaves no time for the bid - // request. 0 is allowed (no reserve). + // 0 leaves the bid no time to get back to the beacon node, and a slot or + // more leaves the builder none, as the proposer's deadline is clamped to a slot let slot_time_ms = chain.slot_time_sec().saturating_mul(1000); ensure!( - self.proposer_deadline_buffer_ms < slot_time_ms, - "proposer_deadline_buffer_ms must be less than one slot ({slot_time_ms} ms)" + (1..slot_time_ms).contains(&self.proposer_deadline_buffer_ms), + "proposer_deadline_buffer_ms must be greater than 0 and less than one slot ({slot_time_ms} ms)" ); ensure!( @@ -557,6 +556,42 @@ mod tests { use super::*; use crate::config::test_env::{RELAY_URL, with_env}; + fn config_with_buffer(buffer: u64) -> String { + format!( + "chain = \"Holesky\"\n[pbs]\nproposer_deadline_buffer_ms = {buffer}\n\ + [[relays]]\nurl = \"{RELAY_URL}\"\n" + ) + } + + #[tokio::test] + async fn test_deadline_buffer_range() { + for (buffer, accepted) in [(0, false), (1, true), (11999, true), (12000, false)] { + let config: CommitBoostConfig = toml::from_str(&config_with_buffer(buffer)).unwrap(); + assert_eq!(config.validate().await.is_ok(), accepted, "{buffer}"); + } + } + + // The service and a custom PBS module load [pbs] through different functions + #[test] + fn test_loaders_reject_zero_deadline_buffer() { + #[derive(Deserialize)] + struct NoExtra {} + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cb-config.toml"); + std::fs::write(&path, config_with_buffer(0)).unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + let service = runtime.block_on(load_pbs_config(Some(path.clone()))).map(|_| ()); + let custom = with_env(&[(CONFIG_ENV, path.to_str())], || { + runtime.block_on(load_pbs_custom_config::()).map(|_| ()) + }); + for result in [service, custom] { + let err = result.expect_err("a zero deadline buffer loaded"); + let expected = + "proposer_deadline_buffer_ms must be greater than 0 and less than one slot"; + assert!(format!("{err:#}").contains(expected), "{err:#}"); + } + } + fn relay_with_headers(headers: &str) -> Result { toml::from_str(&format!("url = \"{RELAY_URL}\"\nheaders = {headers}\n")) } diff --git a/crates/common/src/pbs/relay.rs b/crates/common/src/pbs/relay.rs index ae7f4b9a..4dd48e40 100644 --- a/crates/common/src/pbs/relay.rs +++ b/crates/common/src/pbs/relay.rs @@ -97,6 +97,14 @@ pub struct RelayClient { impl RelayClient { pub fn new(config: RelayConfig) -> eyre::Result { + Self::with_client_builder(config, reqwest::Client::builder()) + } + + /// [`Self::new`] on a client builder that already holds extra settings + pub fn with_client_builder( + config: RelayConfig, + builder: reqwest::ClientBuilder, + ) -> eyre::Result { let stream_url = match config.get_header { GetHeaderTransport::Http => None, GetHeaderTransport::Stream => Some(stream_url(&config.entry.url)?), @@ -131,10 +139,8 @@ impl RelayClient { } } - let client = reqwest::Client::builder() - .default_headers(headers.clone()) - .timeout(DEFAULT_REQUEST_TIMEOUT) - .build()?; + let client = + builder.default_headers(headers.clone()).timeout(DEFAULT_REQUEST_TIMEOUT).build()?; Ok(Self { id: Arc::new(config.id().to_owned()), @@ -255,6 +261,13 @@ impl RelayClient { } } +/// `address`, the auth data before any `?`, as a builder URL. Only `http` and +/// `https` count, since `builder-a:prod` parses as a URL too. +pub fn decode_auth_data_url(address: &[u8]) -> Option { + let url = Url::parse(std::str::from_utf8(address).ok()?).ok()?; + matches!(url.scheme(), "http" | "https").then_some(url) +} + /// First 4 bytes of the value's SHA-256: enough to see a rotation /// across reloads, too short to identify a value on its own fn value_fingerprint(value: &str) -> String { @@ -268,12 +281,31 @@ mod tests { use alloy::primitives::B256; - use super::{GetHeaderRequest, RelayClient, RelayEntry, value_fingerprint}; + use super::{ + GetHeaderRequest, RelayClient, RelayEntry, decode_auth_data_url, value_fingerprint, + }; use crate::{ config::{GetHeaderTransport, RelayConfig, test_env::RELAY_URL}, utils::bls_pubkey_from_hex_unchecked, }; + #[test] + fn decode_auth_data_url_accepts_only_http_urls() { + let cases: [(&[u8], Option<&str>); 6] = [ + (b"https://Builder-A.example.com:443/eth", Some("https://builder-a.example.com/eth")), + (b"http://10.0.0.1:18550", Some("http://10.0.0.1:18550/")), + // Opaque data and host:port parse as URLs with their own scheme + (b"builder-a:prod", None), + (b"localhost:18550", None), + (b"builder-a.example.com", None), + (b"http://1.1.1.1/\xff", None), + ]; + for (data, expected) in cases { + let decoded = decode_auth_data_url(data).map(|url| url.to_string()); + assert_eq!(decoded.as_deref(), expected, "{}", String::from_utf8_lossy(data)); + } + } + #[test] fn test_relay_entry() { let pubkey = bls_pubkey_from_hex_unchecked( diff --git a/crates/pbs/Cargo.toml b/crates/pbs/Cargo.toml index 7497aa93..7037a566 100644 --- a/crates/pbs/Cargo.toml +++ b/crates/pbs/Cargo.toml @@ -5,6 +5,9 @@ publish = false rust-version.workspace = true version.workspace = true +[features] +testing-flags = [] + [dependencies] alloy.workspace = true async-trait.workspace = true @@ -35,3 +38,6 @@ url.workspace = true uuid.workspace = true webpki-roots.workspace = true thiserror.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/pbs/src/dial.rs b/crates/pbs/src/dial.rs new file mode 100644 index 00000000..62be496e --- /dev/null +++ b/crates/pbs/src/dial.rs @@ -0,0 +1,644 @@ +use std::{ + collections::HashMap, + io, + net::{IpAddr, Ipv4Addr, SocketAddr, ToSocketAddrs}, + sync::{Arc, LazyLock, OnceLock}, + time::{Duration, Instant}, +}; + +use cb_common::{ + config::{GetHeaderTransport, RelayConfig}, + pbs::{RelayClient, RelayEntry}, + types::BlsPublicKey, + utils::bls_pubkey_from_hex, +}; +use parking_lot::Mutex; +use reqwest::dns::{Addrs, Name, Resolve, Resolving}; +use tokio::{sync::Semaphore, task::JoinHandle}; +use tracing::{info, warn}; +use url::Url; + +use crate::error::PbsClientError; + +/// The `relay_id` of every dial: the target comes from request data, so a +/// per-host metric label would be unbounded +const DIAL_RELAY_ID: &str = "dial"; + +/// The BLS12-381 G1 generator, filling the pubkey `RelayEntry` requires and +/// only PBS reads +static DIAL_PUBKEY: LazyLock = LazyLock::new(|| { + bls_pubkey_from_hex( + "0x97f1d3a73197d7942695638c4fa9ac0fc3688c4f9774b905a14e3a3f171bac586c55e83ff97a1aeffb3af00adb22c6bb", + ) + .expect("the G1 generator is a valid pubkey") +}); + +/// The address in auth data, before the first `?`. Parameters after it are +/// for the builder, which gets them inside the signed auth +pub(crate) fn auth_data_address(data: &[u8]) -> &[u8] { + data.iter().position(|&byte| byte == b'?').map_or(data, |end| &data[..end]) +} + +/// A relay for the builder an unmatched address names, checked before the dial +/// and, for a hostname, again on each new connection +pub(crate) async fn dial_relay( + data_url: Option, + address: &[u8], + lookup_timeout: Duration, +) -> Result { + let url = dial_target(data_url, address)?; + let origin = url.origin().ascii_serialization(); + let addrs = resolve_dial_target(&url, &origin, lookup_timeout).await?; + info!(%origin, ?addrs, "auth data names a builder outside the config, dialing it"); + dial_client(url) +} + +/// `https:///`, whose `host_str` is the authority's hostname in the +/// form of default auth data +pub(crate) fn host_url(authority: &str) -> Option { + Url::parse(&format!("https://{authority}/")).ok() +} + +/// `data_url`, else `https:///` for the builder-specs default auth +/// data, a bare hostname +fn dial_target(data_url: Option, address: &[u8]) -> Result { + data_url + .or_else(|| { + let url = host_url(std::str::from_utf8(address).ok()?)?; + // Parsing also takes a port, userinfo, a path, upper case and + // numeric IPv4, so only a host already in canonical form passes + (url.host_str()?.as_bytes() == address).then_some(url) + }) + .ok_or_else(|| { + warn!( + address = ?String::from_utf8_lossy(address), + "auth data matches no configured relay and is neither a hostname nor a URL" + ); + PbsClientError::AuthDataMismatch + }) +} + +/// A lookup that times out keeps its blocking thread until the resolver +/// returns, and relays' own lookups share that pool, so few run at once +static DIAL_LOOKUPS: Semaphore = Semaphore::const_new(32); + +async fn resolve_dial_target( + url: &Url, + origin: &str, + lookup_timeout: Duration, +) -> Result, PbsClientError> { + let host = url.host_str().unwrap_or_default(); + let host_port = format!("{host}:{}", url.port_or_known_default().unwrap_or_default()); + // An IP literal needs no lookup + if let Ok(addr) = host_port.parse::() { + vet_dial_addrs(origin, &[addr])?; + return Ok(vec![addr]); + } + let Some(lookup) = spawn_lookup(host_port) else { + warn!(%origin, "too many dial lookups in flight, not dialing"); + return Err(PbsClientError::NoBuilderResponse); + }; + let addrs = match tokio::time::timeout(lookup_timeout, lookup).await { + Ok(Ok(Ok(addrs))) => addrs, + Ok(Ok(Err(err))) => { + warn!(%origin, %err, "dial target lookup failed, not dialing"); + return Err(PbsClientError::DialTargetBlocked); + } + Ok(Err(err)) => { + warn!(%origin, %err, "dial target lookup did not finish"); + return Err(PbsClientError::Internal); + } + Err(_) => { + warn!(%origin, ?lookup_timeout, "dial target lookup timed out, not dialing"); + return Err(PbsClientError::NoBuilderResponse); + } + }; + vet_dial_addrs(origin, &addrs)?; + remember_checked(host, &addrs); + Ok(addrs) +} + +/// How long a dial check's lookup serves new connections to its host, so the +/// resolver does not repeat it. Short, so a later connection sees a DNS change +const CHECKED_ADDRS_TTL: Duration = Duration::from_secs(2); + +type CheckedAddrs = HashMap)>; + +static CHECKED_ADDRS: LazyLock> = LazyLock::new(Default::default); + +/// Kept with port 0, as the resolver's own lookup returns them, so another +/// dial to the host on another port connects to its own. Drops expired entries +fn remember_checked(host: &str, addrs: &[SocketAddr]) { + let addrs = addrs.iter().map(|addr| SocketAddr::new(addr.ip(), 0)).collect(); + let mut checked = CHECKED_ADDRS.lock(); + checked.retain(|_, (at, _)| at.elapsed() < CHECKED_ADDRS_TTL); + checked.insert(host.to_owned(), (Instant::now(), addrs)); +} + +fn recently_checked(host: &str) -> Option> { + let checked = CHECKED_ADDRS.lock(); + let (at, addrs) = checked.get(host)?; + (at.elapsed() < CHECKED_ADDRS_TTL).then(|| addrs.clone()) +} + +/// `None` when every lookup slot is taken +fn spawn_lookup(host_port: String) -> Option>>> { + let permit = DIAL_LOOKUPS.try_acquire().ok()?; + Some(tokio::task::spawn_blocking(move || { + // Held until the lookup returns, which a timeout does not stop + let _permit = permit; + host_port.to_socket_addrs().map(Iterator::collect::>) + })) +} + +/// Every address is checked, since the dial may connect to any of them +fn vet_dial_addrs(origin: &str, addrs: &[SocketAddr]) -> Result<(), PbsClientError> { + if dial_target_check_enabled() && addrs.iter().any(|addr| ip_is_disallowed(addr.ip())) { + warn!(%origin, "dial target resolves to a disallowed address, not dialing"); + return Err(PbsClientError::DialTargetBlocked); + } + Ok(()) +} + +/// Dial hosts are chosen by the request, so an idle connection is kept for one +/// mainnet slot. The pool sweeps at this interval, so it closes within two. +const DIAL_IDLE_TIMEOUT: Duration = Duration::from_secs(12); + +/// Lets a builder dialed again reuse its connection. A failed build is not +/// kept, so the next dial tries again. +static DIAL_CLIENT: OnceLock = OnceLock::new(); + +/// No proxy, which would resolve the host itself, and no redirects, since one +/// to an IP address would skip the resolver +fn build_dial_client() -> eyre::Result { + RelayClient::with_client_builder( + dial_config(Url::parse("https://dial.invalid/").expect("a valid URL")), + reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .dns_resolver(DialResolver) + .pool_idle_timeout(DIAL_IDLE_TIMEOUT) + .pool_max_idle_per_host(1), + ) +} + +/// The shared dial client, addressed to `url` +fn dial_client(url: Url) -> Result { + let shared = match DIAL_CLIENT.get() { + Some(shared) => shared, + None => { + let built = build_dial_client().map_err(|err| { + warn!(%err, "failed to build the dial client"); + PbsClientError::Internal + })?; + DIAL_CLIENT.get_or_init(|| built) + } + }; + let mut relay = shared.clone(); + relay.config = Arc::new(dial_config(url)); + Ok(relay) +} + +fn dial_config(url: Url) -> RelayConfig { + RelayConfig { + entry: RelayEntry { id: DIAL_RELAY_ID.to_string(), pubkey: DIAL_PUBKEY.clone(), url }, + id: None, + headers: None, + get_params: None, + get_header: GetHeaderTransport::Http, + enable_timing_games: false, + target_first_request_ms: None, + frequency_get_header_ms: None, + validator_registration_batch_size: None, + } +} + +/// Run for each new connection to a hostname (an IP address skips it): takes +/// the dial check's recent lookup or its own, and refuses a disallowed address +struct DialResolver; + +impl Resolve for DialResolver { + fn resolve(&self, name: Name) -> Resolving { + Box::pin(async move { + let addrs = match recently_checked(name.as_str()) { + Some(addrs) => addrs, + None => { + spawn_lookup(format!("{}:0", name.as_str())) + .ok_or("too many dial lookups in flight")? + .await?? + } + }; + vet_dial_addrs(name.as_str(), &addrs)?; + Ok(Box::new(addrs.into_iter()) as Addrs) + }) + } +} + +#[cfg(feature = "testing-flags")] +thread_local! { + static SKIP_DIAL_TARGET_CHECK: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// Test-only: skips the dial target's address check on this thread, so a +/// test can dial a mock builder on loopback +#[cfg(feature = "testing-flags")] +pub fn set_skip_dial_target_check(skip: bool) { + SKIP_DIAL_TARGET_CHECK.with(|flag| flag.set(skip)); +} + +fn dial_target_check_enabled() -> bool { + #[cfg(feature = "testing-flags")] + { + !SKIP_DIAL_TARGET_CHECK.with(|flag| flag.get()) + } + #[cfg(not(feature = "testing-flags"))] + { + true + } +} + +/// Anything but public unicast: the special-purpose IPv4 ranges, also as the +/// IPv4 inside a v4-mapped, v4-compatible or NAT64 address, and IPv6 outside +/// 2000::/3 or in its documentation, Teredo and 6to4 ranges +fn ip_is_disallowed(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => { + let [a, b, c, _] = v4.octets(); + v4.is_private() || + v4.is_loopback() || + v4.is_link_local() || + v4.is_multicast() || + v4.is_documentation() || + // std checks these only on nightly: 0.0.0.0/8, 100.64.0.0/10, + // 192.0.0.0/24, 198.18.0.0/15 and 240.0.0.0/4 + a == 0 || + (a == 100 && b & 0xc0 == 0x40) || + (a == 192 && b == 0 && c == 0) || + (a == 198 && b & 0xfe == 18) || + a >= 240 + } + IpAddr::V6(v6) => { + let s = v6.segments(); + let nat64 = s[..6] == [0x64, 0xff9b, 0, 0, 0, 0]; + let embedded = + if nat64 { Some(Ipv4Addr::from_bits(v6.to_bits() as u32)) } else { v6.to_ipv4() }; + match embedded { + // :: and ::1 land here as 0.0.0.0 and 0.0.0.1 + Some(v4) => ip_is_disallowed(IpAddr::V4(v4)), + None => { + s[0] & 0xe000 != 0x2000 || + (s[0] == 0x2001 && (s[1] == 0 || s[1] == 0x0db8)) || + s[0] == 0x2002 + } + } + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use axum::http::StatusCode; + use cb_common::pbs::decode_auth_data_url; + + use super::*; + + // Literal IPs keep the lookup off the network + #[tokio::test] + async fn dial_relay_targets() { + const MISMATCH: &str = "auth data does not match a configured builder"; + const BLOCKED: &str = "dial target does not resolve or resolves to a disallowed address"; + let cases: [(&[u8], Result<&str, &str>); 9] = [ + (b"http://1.1.1.1:8551", Ok("http://1.1.1.1:8551/")), + (b"1.1.1.1", Ok("https://1.1.1.1/")), + (b"1.1.1.1?ofac=1?v=2", Ok("https://1.1.1.1/")), + (b"[2606:4700:4700::1111]", Ok("https://[2606:4700:4700::1111]/")), + (b"10.0.0.1", Err(BLOCKED)), + (b"1.1.1.1:8551", Err(MISMATCH)), + (b"Builder.example.com", Err(MISMATCH)), + (b"ftp://1.1.1.1", Err(MISMATCH)), + (&[0xde, 0xad], Err(MISMATCH)), + ]; + for (data, expected) in cases { + let address = auth_data_address(data); + let res = + dial_relay(decode_auth_data_url(address), address, Duration::from_secs(5)).await; + let case = String::from_utf8_lossy(data); + match (&res, expected) { + (Ok(relay), Ok(url)) => { + assert_eq!(relay.config.entry.url.as_str(), url, "{case}"); + assert_eq!(relay.id.as_str(), "dial", "{case}"); + } + (Err(err), Err(message)) => assert_eq!(err.to_string(), message, "{case}"), + _ => panic!("{case}: expected {expected:?}, got {:?}", res.map(|_| ())), + } + } + } + + /// Taken by every test that holds or takes a lookup slot, since the slots + /// are shared by the whole test binary + static LOOKUP_SLOTS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + + #[tokio::test] + async fn lookup_cap_refuses_hostnames_not_ip_literals() { + let _slots = LOOKUP_SLOTS.lock().await; + let _held = DIAL_LOOKUPS.try_acquire_many(32).unwrap(); + let dial = |address: &'static [u8]| dial_relay(None, address, Duration::from_secs(1)); + assert!(dial(b"1.1.1.1").await.is_ok()); + assert!(matches!(dial(b"builder.invalid").await, Err(PbsClientError::NoBuilderResponse))); + } + + // A label over 63 octets fails in the stub resolver, without a query + #[tokio::test] + async fn dial_target_that_does_not_resolve_is_blocked() { + let _slots = LOOKUP_SLOTS.lock().await; + let host = format!("{}.invalid", "a".repeat(64)); + let res = dial_relay(None, host.as_bytes(), Duration::from_secs(5)).await; + assert!(matches!(res, Err(PbsClientError::DialTargetBlocked)), "{:?}", res.map(|_| ())); + } + + // One blocking thread, so the lookup queues behind another task + #[test] + fn lookup_holds_its_slot_until_it_returns() { + let _slots = LOOKUP_SLOTS.blocking_lock(); + let runtime = + tokio::runtime::Builder::new_current_thread().max_blocking_threads(1).build().unwrap(); + runtime.block_on(async { + let (release, blocked) = std::sync::mpsc::channel::<()>(); + let busy = tokio::task::spawn_blocking(move || blocked.recv()); + let lookup = spawn_lookup("1.1.1.1:443".to_owned()).unwrap(); + assert_eq!(DIAL_LOOKUPS.available_permits(), 31); + release.send(()).unwrap(); + busy.await.unwrap().unwrap(); + lookup.await.unwrap().unwrap(); + assert_eq!(DIAL_LOOKUPS.available_permits(), 32); + }); + } + + // One blocking thread, so the lookup waits behind another task, and that + // wait is taken from the time left for the builder + #[cfg(feature = "testing-flags")] + #[test] + fn dial_lookup_time_is_taken_from_the_timeout() { + let _slots = LOOKUP_SLOTS.blocking_lock(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_time() + .max_blocking_threads(1) + .build() + .unwrap(); + let left_ms = runtime.block_on(async { + tokio::task::spawn_blocking(|| std::thread::sleep(Duration::from_millis(50))); + set_skip_dial_target_check(true); + let res = crate::utils::resolve_addressed_relay( + &[], + &[], + b"localhost", + &Default::default(), + 1000, + ) + .await; + set_skip_dial_target_check(false); + res.map(|(_, left_ms)| left_ms) + }); + // dial_client_checks_the_addresses_it_resolves needs a lookup + CHECKED_ADDRS.lock().remove("localhost"); + assert!(left_ms.unwrap() <= 950); + } + + #[tokio::test] + async fn bid_dial_without_a_lookup_slot_is_no_bid() -> eyre::Result<()> { + use cb_common::{ + config::PbsModuleConfig, + pbs::{HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS}, + types::Chain, + utils::utcnow_ms, + }; + use parking_lot::RwLock; + + use crate::{DefaultBuilderApi, PbsState, routes::create_app_router}; + + let _slots = LOOKUP_SLOTS.lock().await; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let state = PbsState::new( + PbsModuleConfig { + chain: Chain::Hoodi, + endpoint: addr, + pbs_config: Arc::new(serde_json::from_str("{}")?), + relays: vec![], + all_relays: vec![], + signer_client: None, + registry_muxes: None, + mux_lookup: None, + }, + Default::default(), + ); + let app = create_app_router::<(), DefaultBuilderApi>(Arc::new(RwLock::new(state))); + tokio::spawn(async move { axum::serve(listener, app).await }); + + let _held = DIAL_LOOKUPS.try_acquire_many(32)?; + let zero = format!("0x{}", "0".repeat(64)); + let url = format!( + "http://{addr}/eth/v1/builder/execution_payload_bid/1/{zero}/{zero}/{}", + *DIAL_PUBKEY + ); + let auth = serde_json::json!({ + // "builder.invalid" + "message": { "data": "0x6275696c6465722e696e76616c6964", "slot": "1" }, + "signature": format!("0x{}", "0".repeat(192)), + }); + let res = reqwest::Client::new() + .post(url) + .header("Eth-Consensus-Version", "gloas") + .header(HEADER_START_TIME_UNIX_MS, utcnow_ms()) + .header(HEADER_TIMEOUT_MS, 1000) + .json(&auth) + .send() + .await?; + assert_eq!(res.status(), StatusCode::NO_CONTENT); + Ok(()) + } + + #[test] + fn vet_dial_addrs_checks_every_address() { + let addr = |s: &str| s.parse::().unwrap(); + assert!(matches!( + vet_dial_addrs("https://builder.example.com", &[ + addr("1.1.1.1:443"), + addr("10.0.0.1:443") + ]), + Err(PbsClientError::DialTargetBlocked) + )); + } + + /// A builder on loopback whose `/bid` redirects to `/internal` and whose + /// `/slow` answers after 50 ms, and its count of accepted connections + async fn mock_builder() -> eyre::Result<(u16, Arc)> { + use axum::{Router, http::header::LOCATION, routing::post, serve::ListenerExt}; + + let connections = Arc::new(AtomicUsize::new(0)); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let port = listener.local_addr()?.port(); + let accepted = connections.clone(); + let listener = listener.tap_io(move |_| { + accepted.fetch_add(1, Ordering::Relaxed); + }); + let builder = Router::new() + .route( + "/bid", + post(|| async { (StatusCode::TEMPORARY_REDIRECT, [(LOCATION, "/internal")]) }), + ) + .route("/internal", post(|| async { StatusCode::OK })) + .route( + "/slow", + post(|| async { + tokio::time::sleep(Duration::from_millis(50)).await; + StatusCode::OK + }), + ); + tokio::spawn(async move { axum::serve(listener, builder).await }); + Ok((port, connections)) + } + + // An IP literal skips the resolver, which would refuse loopback + #[tokio::test] + async fn dials_share_connections_and_refuse_redirects() -> eyre::Result<()> { + let (port, connections) = mock_builder().await?; + let url = Url::parse(&format!("http://127.0.0.1:{port}/bid"))?; + for _ in 0..2 { + let res = dial_client(url.clone())?.client.post(url.clone()).send().await?; + assert_eq!(res.status(), StatusCode::TEMPORARY_REDIRECT); + } + assert_eq!(connections.load(Ordering::Relaxed), 1); + + tokio::time::pause(); + tokio::time::advance(DIAL_IDLE_TIMEOUT + Duration::from_secs(1)).await; + tokio::time::resume(); + dial_client(url.clone())?.client.post(url).send().await?; + assert_eq!(connections.load(Ordering::Relaxed), 2); + Ok(()) + } + + // Two requests at once need two connections, and only one stays idle + #[tokio::test] + async fn dial_client_keeps_one_idle_connection_per_host() -> eyre::Result<()> { + let (port, connections) = mock_builder().await?; + let url = Url::parse(&format!("http://127.0.0.1:{port}/slow"))?; + let client = dial_client(url.clone())?.client; + for _ in 0..2 { + tokio::try_join!(client.post(url.clone()).send(), client.post(url.clone()).send())?; + } + assert_eq!(connections.load(Ordering::Relaxed), 3); + Ok(()) + } + + // No check runs before these dials, so only the resolver refuses + // `localhost`, which resolves to loopback + #[cfg(feature = "testing-flags")] + #[tokio::test] + async fn dial_client_checks_the_addresses_it_resolves() -> eyre::Result<()> { + let _slots = LOOKUP_SLOTS.lock().await; + let (port, _) = mock_builder().await?; + let url = Url::parse(&format!("http://localhost:{port}/bid"))?; + let dial = || dial_client(url.clone()).map(|relay| relay.client.post(url.clone()).send()); + assert!(dial()?.await.is_err()); + set_skip_dial_target_check(true); + let held = DIAL_LOOKUPS.try_acquire_many(32)?; + assert!(dial()?.await.is_err()); + drop(held); + let res = dial()?.await; + set_skip_dial_target_check(false); + assert_eq!(res?.status(), StatusCode::TEMPORARY_REDIRECT); + Ok(()) + } + + // With every lookup slot held, the connection can only use the addresses + // the dial's check resolved + #[cfg(feature = "testing-flags")] + #[tokio::test] + async fn dial_connects_to_the_addresses_its_check_resolved() -> eyre::Result<()> { + let _slots = LOOKUP_SLOTS.lock().await; + let (port, _) = mock_builder().await?; + let url = Url::parse(&format!("http://localhost:{port}/bid"))?; + set_skip_dial_target_check(true); + let relay = dial_relay(Some(url.clone()), b"", Duration::from_secs(5)).await?; + let held = DIAL_LOOKUPS.try_acquire_many(32)?; + let res = relay.client.post(url).send().await; + drop(held); + set_skip_dial_target_check(false); + // dial_client_checks_the_addresses_it_resolves needs a lookup + CHECKED_ADDRS.lock().remove("localhost"); + assert_eq!(res?.status(), StatusCode::TEMPORARY_REDIRECT); + Ok(()) + } + + // A remembered address is checked again before a connection uses it. With + // every slot held, ignoring the entry would fail on the cap instead + #[tokio::test] + async fn dial_resolver_checks_remembered_addresses() { + let _slots = LOOKUP_SLOTS.lock().await; + let _held = DIAL_LOOKUPS.try_acquire_many(32).unwrap(); + remember_checked("remembered.invalid", &["127.0.0.1:0".parse().unwrap()]); + let res = DialResolver.resolve("remembered.invalid".parse().unwrap()).await; + CHECKED_ADDRS.lock().remove("remembered.invalid"); + let err = res.err().expect("a loopback address is refused").to_string(); + assert_eq!(err, PbsClientError::DialTargetBlocked.to_string()); + } + + #[test] + fn checked_addrs_drop_their_port_expire_and_are_swept() { + let addrs = ["1.1.1.1:443".parse().unwrap()]; + let expired = Instant::now().checked_sub(CHECKED_ADDRS_TTL).unwrap(); + CHECKED_ADDRS.lock().insert("expired.invalid".into(), (expired, addrs.to_vec())); + assert_eq!(recently_checked("expired.invalid"), None); + remember_checked("fresh.invalid", &addrs); + assert_eq!(recently_checked("fresh.invalid"), Some(vec!["1.1.1.1:0".parse().unwrap()])); + let mut checked = CHECKED_ADDRS.lock(); + assert!(!checked.contains_key("expired.invalid")); + checked.remove("fresh.invalid"); + } + + #[test] + fn ip_is_disallowed_table() { + for (ip, disallowed) in [ + ("127.0.0.1", true), + ("0.0.0.0", true), + ("::1", true), + ("10.0.0.1", true), + ("169.254.1.1", true), + ("255.255.255.255", true), + ("100.64.0.1", true), + ("100.127.255.254", true), + ("100.63.0.1", false), + ("100.128.0.1", false), + ("fd12:3456::1", true), + ("fe80::1", true), + ("::ffff:127.0.0.1", true), + ("::10.0.0.1", true), + ("64:ff9b::a00:5", true), + ("::1.1.1.1", false), + ("64:ff9b::1.1.1.1", false), + ("::ffff:1.1.1.1", false), + ("0.1.2.3", true), + ("192.0.1.1", false), + ("198.19.255.255", true), + ("198.20.0.1", false), + ("224.0.0.1", true), + ("240.0.0.1", true), + ("192.0.0.192", true), + ("198.18.0.1", true), + ("203.0.113.1", true), + ("fec0::1", true), + ("ff02::1", true), + ("2001:db8::1", true), + ("64:ff9b:1::101:101", true), + ("2002:a00:1::1", true), + ("2001::1", true), + ("1.1.1.1", false), + ("2606:4700:4700::1111", false), + ] { + assert_eq!(ip_is_disallowed(ip.parse().unwrap()), disallowed, "{ip}"); + } + } +} diff --git a/crates/pbs/src/error.rs b/crates/pbs/src/error.rs index 9b9aaf07..29f8dc4c 100644 --- a/crates/pbs/src/error.rs +++ b/crates/pbs/src/error.rs @@ -20,12 +20,14 @@ struct ErrorResponse { pub enum PbsClientError { #[error("no response from relays")] NoResponse, - /// 500, not 502: 502 is in neither endpoint's builder-specs response set. - /// Legacy routes keep `NoResponse` -> 502. + /// 500, not 502: 502 is not in the preferences endpoint's builder-specs + /// response set. Legacy routes keep `NoResponse` -> 502. #[error("no builder accepted the submission")] NoBuilderResponse, #[error("auth data does not match a configured builder")] AuthDataMismatch, + #[error("dial target does not resolve or resolves to a disallowed address")] + DialTargetBlocked, #[error("missing or invalid timing headers")] MissingTimingHeader, #[error("auth slot does not match the request path")] @@ -50,6 +52,7 @@ impl PbsClientError { PbsClientError::NoResponse => StatusCode::BAD_GATEWAY, PbsClientError::NoBuilderResponse => StatusCode::INTERNAL_SERVER_ERROR, PbsClientError::AuthDataMismatch => StatusCode::BAD_REQUEST, + PbsClientError::DialTargetBlocked => StatusCode::BAD_REQUEST, PbsClientError::MissingTimingHeader => StatusCode::BAD_REQUEST, PbsClientError::AuthSlotMismatch => StatusCode::BAD_REQUEST, PbsClientError::BuilderRejected(code) => *code, @@ -73,6 +76,9 @@ impl IntoResponse for PbsClientError { PbsClientError::AuthDataMismatch => { "Invalid SignedBuilderRequestAuth: auth.message.data does not match any configured builder".to_string() } + PbsClientError::DialTargetBlocked => { + "Invalid SignedBuilderRequestAuth: the addressed builder's host does not resolve or resolves to a disallowed address".to_string() + } PbsClientError::MissingTimingHeader => { "Invalid request: Date-Milliseconds and X-Timeout-Ms headers are required".to_string() } diff --git a/crates/pbs/src/lib.rs b/crates/pbs/src/lib.rs index 8b4afdcf..9c04224a 100644 --- a/crates/pbs/src/lib.rs +++ b/crates/pbs/src/lib.rs @@ -1,5 +1,6 @@ mod api; mod constants; +mod dial; mod error; mod metrics; mod mev_boost; @@ -10,6 +11,8 @@ mod utils; pub use api::*; pub use constants::*; +#[cfg(feature = "testing-flags")] +pub use dial::set_skip_dial_target_check; pub use mev_boost::*; pub use service::PbsService; pub use state::{BuilderApiState, PbsState, PbsStateGuard}; diff --git a/crates/pbs/src/routes/builder_preferences.rs b/crates/pbs/src/routes/builder_preferences.rs index dc8596e7..cdbeb4e4 100644 --- a/crates/pbs/src/routes/builder_preferences.rs +++ b/crates/pbs/src/routes/builder_preferences.rs @@ -6,7 +6,7 @@ use axum::{ }; use cb_common::{ pbs::{BuilderPreferencesRequest, SubmitBuilderPreferencesParams}, - wire::{decode_versioned_request_body, get_user_agent}, + wire::{EncodingType, decode_versioned_request_body, get_user_agent}, }; use reqwest::StatusCode; use ssz::Encode; @@ -18,7 +18,7 @@ use crate::{ error::PbsClientError, state::{BuilderApiState, PbsState}, utils::{ - builder_rejection, epbs_base_send_headers, log_mux_selection, post_ssz_expect_accepted, + builder_rejection, epbs_base_send_headers, log_mux_selection, post_expect_accepted, record_beacon_status, record_request_failure, resolve_addressed_relay, }, }; @@ -65,24 +65,30 @@ pub async fn submit_builder_preferences( log_mux_selection(maybe_mux_id, relays.len(), ¶ms.proposer_pubkey); - let relay = resolve_addressed_relay(relays, request.auth.message.data.as_ref())?; + // Preferences are submitted an epoch ahead, so they share the registration + // timeout rather than the block-production one + let (relay, timeout_ms) = resolve_addressed_relay( + relays, + state.default_relays(), + request.auth.message.data.as_ref(), + &req_headers, + pbs_config.timeout_register_validator_ms, + ) + .await?; let send_headers = epbs_base_send_headers(&req_headers)?; // SSZ on the relay hop let body = Bytes::from(request.as_ssz_bytes()); - // Preferences are submitted an epoch ahead, so they share the registration - // timeout rather than the block-production one - let timeout_ms = pbs_config.timeout_register_validator_ms; - let relay_id = relay.id.as_ref(); let sent = match relay.submit_builder_preferences_url(¶ms.proposer_pubkey) { Ok(url) => { - post_ssz_expect_accepted( + post_expect_accepted( &relay, url, body, + EncodingType::Ssz, send_headers, timeout_ms, SUBMIT_BUILDER_PREFERENCES_ENDPOINT_TAG, diff --git a/crates/pbs/src/routes/execution_payload_bid.rs b/crates/pbs/src/routes/execution_payload_bid.rs index 9e1b9e36..f4b57bd7 100644 --- a/crates/pbs/src/routes/execution_payload_bid.rs +++ b/crates/pbs/src/routes/execution_payload_bid.rs @@ -28,13 +28,16 @@ use tracing::{debug, error, info, warn}; use crate::{ PbsStateGuard, - constants::{GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG, MAX_SIZE_GET_HEADER_RESPONSE}, + constants::{ + GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG, MAX_SIZE_DEFAULT, MAX_SIZE_GET_HEADER_RESPONSE, + }, error::PbsClientError, metrics::{RELAY_HEADER_VALUE, RELAY_LAST_SLOT}, state::{BuilderApiState, PbsState}, utils::{ builder_rejection, epbs_base_send_headers, format_gwei_as_eth, log_mux_selection, - record_beacon_status, record_request_failure, resolve_addressed_relay, send_to_relay, + record_beacon_status, record_request_failure, relay_response_error, + resolve_addressed_relay, send_to_relay, }, }; @@ -125,8 +128,6 @@ pub async fn get_execution_payload_bid( return Err(PbsClientError::AuthSlotMismatch); } - let relay = resolve_addressed_relay(relays, auth.message.data.as_ref())?; - // The beacon node's deadline, not timeout_get_header_ms, bounds the request let slot_ms = state.config.chain.slot_time_sec().saturating_mul(1000); let budget_ms = request_budget_ms(&req_headers, utcnow_ms(), slot_ms)?; @@ -144,6 +145,21 @@ pub async fn get_execution_payload_bid( return Ok(None); } + let (relay, max_timeout_ms) = match resolve_addressed_relay( + relays, + state.default_relays(), + auth.message.data.as_ref(), + &req_headers, + max_timeout_ms, + ) + .await + { + // A dial that runs out of time or lookup slots is a builder that did not + // answer + Err(PbsClientError::NoBuilderResponse) => return Ok(None), + resolved => resolved?, + }; + let mut send_headers = epbs_base_send_headers(&req_headers)?; // SSZ is smaller and reaches a beacon node that asks for SSZ unchanged @@ -169,7 +185,7 @@ pub async fn get_execution_payload_bid( Ok(None) } Err(err) => { - error!(%err, %relay_id); + error!(?err, %relay_id); builder_rejection(&err).map_or(Ok(None), Err) } } @@ -229,17 +245,11 @@ async fn send_get_execution_payload_bid( let (res, request_latency) = send_to_relay(request, &relay, GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG).await?; let code = res.status(); + if !code.is_success() { + return Err(relay_response_error(res, "expected 200 or 204").await); + } - // Parse the negotiated Content-Type (and optional fork) before the body is - // consumed. Only successful responses carry a meaningful encoding; on - // non-success we fall through to safe_read_http_response's NonSuccess error, - // so these values are never consumed. - let (content_type, fork) = if code.is_success() { - parse_response_encoding_and_fork(res.headers(), code.as_u16())? - } else { - (EncodingType::Json, None) - }; - + let (content_type, fork) = parse_response_encoding_and_fork(res.headers(), code.as_u16())?; let response_bytes = safe_read_http_response(res, MAX_SIZE_GET_HEADER_RESPONSE).await?; if code == StatusCode::NO_CONTENT { debug!( @@ -253,9 +263,15 @@ async fn send_get_execution_payload_bid( } let bid = match content_type { - EncodingType::Json => serde_json::from_slice(&response_bytes).map_err(|err| { - PbsError::JsonDecode { err, raw: String::from_utf8_lossy(&response_bytes).into_owned() } - })?, + EncodingType::Json => { + serde_json::from_slice(&response_bytes).map_err(|err| PbsError::JsonDecode { + err, + raw: String::from_utf8_lossy( + &response_bytes[..response_bytes.len().min(MAX_SIZE_DEFAULT)], + ) + .into_owned(), + })? + } EncodingType::Ssz => { // SSZ requires the fork from Eth-Consensus-Version; its absence is a // relay protocol violation. diff --git a/crates/pbs/src/routes/submit_signed_beacon_block.rs b/crates/pbs/src/routes/submit_signed_beacon_block.rs index 75bb3554..43344c42 100644 --- a/crates/pbs/src/routes/submit_signed_beacon_block.rs +++ b/crates/pbs/src/routes/submit_signed_beacon_block.rs @@ -1,7 +1,6 @@ use axum::{body::Bytes, extract::State, http::HeaderMap, response::IntoResponse}; use cb_common::wire::{ - BodyDeserializeError, EncodingType, content_type_encoding, get_user_agent, - require_consensus_version_header, + BodyDeserializeError, content_type_encoding, get_user_agent, require_consensus_version_header, }; use reqwest::StatusCode; use tracing::{debug, info}; @@ -12,8 +11,8 @@ use crate::{ error::PbsClientError, state::{BuilderApiState, PbsState}, utils::{ - epbs_base_send_headers, join_detached_sends, post_ssz_expect_accepted, - record_beacon_status, record_request_failure, + epbs_base_send_headers, join_detached_sends, post_expect_accepted, record_beacon_status, + record_request_failure, }, }; @@ -34,8 +33,8 @@ pub async fn handle_submit_signed_beacon_block( } /// Implements https://ethereum.github.io/builder-specs/?urls.primaryName=dev#/Builder/submitSignedBeaconBlock -/// Forwards the block bytes, undecoded, to every configured builder, since CB -/// keeps no auction state to know which bid won. Returns 202 if one accepts +/// Forwards the block, undecoded, to every configured builder, as CB does not +/// know which bid won, and answers 202 whatever they answer pub async fn submit_signed_beacon_block( body: Bytes, req_headers: HeaderMap, @@ -48,11 +47,8 @@ pub async fn submit_signed_beacon_block( if body.is_empty() { return Err(BodyDeserializeError::MissingBody.into()); } - // Forwarded undecoded as SSZ, so anything else, an unlabelled (JSON) body - // included, is a 415 - if content_type_encoding(&req_headers)? != EncodingType::Ssz { - return Err(BodyDeserializeError::UnsupportedMediaType.into()); - } + // An unlabelled block is JSON, the builder-specs default + let encoding = content_type_encoding(&req_headers)?; info!(ua, "new request"); @@ -63,10 +59,11 @@ pub async fn submit_signed_beacon_block( let (relay, body, headers) = (relay.clone(), body.clone(), send_headers.clone()); async move { let url = relay.submit_signed_beacon_block_url()?; - post_ssz_expect_accepted( + post_expect_accepted( &relay, url, body, + encoding, headers, timeout_ms, SUBMIT_SIGNED_BEACON_BLOCK_ENDPOINT_TAG, @@ -81,16 +78,11 @@ pub async fn submit_signed_beacon_block( .filter(|(res, relay)| match res { Ok(()) => true, Err(err) => { - debug!(relay_id = relay.id.as_ref(), %err, "builder did not accept the block; only the winner accepts"); + debug!(relay_id = relay.id.as_ref(), %err, "builder did not accept the block"); false } }) .count(); - - // Only the winner accepts, so one 202 across the broadcast is success - if accepted == 0 { - return Err(PbsClientError::NoBuilderResponse); - } - info!(accepted, addressed = relays.len(), "signed beacon block submitted"); + info!(accepted, relays = relays.len(), "signed beacon block forwarded"); Ok(()) } diff --git a/crates/pbs/src/state.rs b/crates/pbs/src/state.rs index bd683e5f..1ca181bf 100644 --- a/crates/pbs/src/state.rs +++ b/crates/pbs/src/state.rs @@ -51,6 +51,11 @@ where &self.config.all_relays } + /// Returns the default relays, `[[relays]]`, in config order + pub fn default_relays(&self) -> &[RelayClient] { + &self.config.relays + } + /// Returns the PBS config and relay clients for the given validator pubkey. /// If the pubkey is not found in any mux, the default configs are /// returned diff --git a/crates/pbs/src/utils.rs b/crates/pbs/src/utils.rs index 023dc6bb..cd416f33 100644 --- a/crates/pbs/src/utils.rs +++ b/crates/pbs/src/utils.rs @@ -6,7 +6,7 @@ use std::{ use alloy::primitives::utils::{ParseUnits, Unit}; use axum::body::Bytes; use cb_common::{ - pbs::{RelayClient, error::PbsError}, + pbs::{HEADER_VERSION_KEY, RelayClient, decode_auth_data_url, error::PbsError}, types::BlsPublicKey, wire::{ CONSENSUS_VERSION_HEADER, EncodingType, GLOAS_CONSENSUS_VERSION, @@ -15,14 +15,15 @@ use cb_common::{ }; use futures::future::join_all; use reqwest::{ - StatusCode, - header::{CONTENT_TYPE, HeaderMap, USER_AGENT}, + Response, StatusCode, + header::{CONTENT_TYPE, HOST, HeaderMap, USER_AGENT}, }; use tracing::{Instrument, debug, error, warn}; use url::Url; use crate::{ constants::{MAX_SIZE_DEFAULT, TIMEOUT_ERROR_CODE_STR}, + dial::{auth_data_address, dial_relay, host_url}, error::PbsClientError, metrics::{BEACON_NODE_STATUS, RELAY_LATENCY, RELAY_STATUS_CODE}, }; @@ -103,13 +104,13 @@ pub(crate) fn log_mux_selection( } } -/// POSTs an SSZ body to a builder; 202 Accepted is the only success. A failed -/// response's body, capped at `MAX_SIZE_DEFAULT`, becomes the error message. -/// Returns the request latency. -pub(crate) async fn post_ssz_expect_accepted( +/// POSTs `body` in `encoding` to a builder and returns the latency; 202 is the +/// only success +pub(crate) async fn post_expect_accepted( relay: &RelayClient, url: Url, body: Bytes, + encoding: EncodingType, headers: HeaderMap, timeout_ms: u64, tag: &str, @@ -119,22 +120,26 @@ pub(crate) async fn post_ssz_expect_accepted( .post(url) .timeout(Duration::from_millis(timeout_ms)) .headers(headers) - .header(CONTENT_TYPE, EncodingType::Ssz.content_type_header().clone()) + .header(CONTENT_TYPE, encoding.content_type_header().clone()) .body(body); let (res, latency) = send_to_relay(req, relay, tag).await?; - let code = res.status(); - if code == StatusCode::ACCEPTED { + if res.status() == StatusCode::ACCEPTED { return Ok(latency); } - // Read after the status check, so a body over the cap still reports the - // builder's status + Err(relay_response_error(res, "expected 202").await) +} + +/// A failed builder response as an error; a body over the cap still reports the +/// status +pub(crate) async fn relay_response_error(res: Response, if_empty: &str) -> PbsError { + let code = res.status().as_u16(); let url = res.url().to_string(); let error_msg = match read_chunked_body_with_max(res, MAX_SIZE_DEFAULT, &url).await { Ok(body) if !body.is_empty() => String::from_utf8_lossy(&body).into_owned(), - Ok(_) => "expected 202".to_string(), + Ok(_) => if_empty.to_string(), Err(err) => err.to_string(), }; - Err(PbsError::RelayResponse { error_msg, code: code.as_u16() }) + PbsError::RelayResponse { error_msg, code } } /// A gwei amount in ETH, for logs @@ -186,27 +191,77 @@ pub fn check_gas_limit(gas_limit: u64, parent_gas_limit: u64) -> bool { true } -/// The relay an ePBS request addresses: the first whose URL hostname equals -/// `auth_data`. No match is a 400. -pub(crate) fn resolve_addressed_relay( +/// The relay `auth_data` names among `relays`, then `default_relays`, else a +/// dial to it, with what the dial's setup left of `timeout_ms` +pub(crate) async fn resolve_addressed_relay( relays: &[RelayClient], + default_relays: &[RelayClient], auth_data: &[u8], -) -> Result { - let addressed = relays.iter().find(|relay| { - relay.config.entry.url.host_str().is_some_and(|host| host.as_bytes() == auth_data) - }); - match addressed { - Some(relay) => Ok(relay.clone()), - None => { - warn!( - auth_data = %String::from_utf8_lossy(auth_data), - "auth data matches no configured relay" - ); - Err(PbsClientError::AuthDataMismatch) + req_headers: &HeaderMap, + timeout_ms: u64, +) -> Result<(RelayClient, u64), PbsClientError> { + let address = auth_data_address(auth_data); + let data_url = decode_auth_data_url(address); + // A relay configured for other keys is reached with its own settings rather + // than dialed without them + if let Some(relay) = relays + .iter() + .chain(default_relays) + .find(|relay| names_relay(relay, address, data_url.as_ref())) + { + return Ok((relay.clone(), timeout_ms)); + } + // Every Commit-Boost dial carries this header, so a request dialed back into + // a Commit-Boost, this one included, goes no further + if req_headers.contains_key(HEADER_VERSION_KEY) { + warn!( + auth_data = ?String::from_utf8_lossy(auth_data), + "auth data matches no configured relay and the request came from a Commit-Boost, not dialing on" + ); + return Err(PbsClientError::AuthDataMismatch); + } + // A key without builder config sends this Commit-Boost's host, which a dial + // would reach on port 443, not here + if is_request_host(req_headers, address) { + warn!( + auth_data = ?String::from_utf8_lossy(auth_data), + "auth data names the host this request was sent to, not dialing" + ); + return Err(PbsClientError::AuthDataMismatch); + } + let started = Instant::now(); + let relay = dial_relay(data_url, address, Duration::from_millis(timeout_ms)).await?; + let left_ms = timeout_ms.saturating_sub(started.elapsed().as_millis() as u64); + // A request with no time left would still reach the builder + if left_ms == 0 { + return Err(PbsClientError::NoBuilderResponse); + } + Ok((relay, left_ms)) +} + +/// Whether `address` is the relay's hostname or, as a URL, its scheme, host and +/// port +fn names_relay(relay: &RelayClient, address: &[u8], data_url: Option<&Url>) -> bool { + let url = &relay.config.entry.url; + match data_url { + None => url.host_str().is_some_and(|host| host.as_bytes() == address), + Some(data_url) => { + url.scheme() == data_url.scheme() && + url.host() == data_url.host() && + url.port_or_known_default() == data_url.port_or_known_default() } } } +/// Whether `address` is the host in the request's `Host` header, in the form of +/// default auth data +fn is_request_host(req_headers: &HeaderMap, address: &[u8]) -> bool { + req_headers + .get(HOST) + .and_then(|host| host_url(host.to_str().ok()?)) + .is_some_and(|url| url.host_str().is_some_and(|host| host.as_bytes() == address)) +} + #[cfg(test)] mod tests { use cb_common::{ @@ -237,24 +292,106 @@ mod tests { RelayClient::new(config).unwrap() } - #[test] - fn resolve_relay_by_hostname() { + /// Headers of a request from a Commit-Boost, so a miss is an error, not a + /// dial + fn from_cb() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert(HEADER_VERSION_KEY, reqwest::header::HeaderValue::from_static("test")); + headers + } + + #[tokio::test] + async fn resolve_relay_by_auth_data() { let relays = vec![ test_relay("https://0xdeadbeef@builder-a.example.com"), test_relay("https://builder-b.example.com:8443/eth"), test_relay("http://[::1]:18550"), ]; - let host = |data: &[u8]| -> Option { - resolve_addressed_relay(&relays, data) + let from_cb = from_cb(); + let host = async |data: &[u8]| -> Option { + resolve_addressed_relay(&relays, &[], data, &from_cb, 0) + .await + .ok() + .map(|(relay, _)| relay.config.entry.url.host_str().unwrap().to_string()) + }; + assert_eq!(host(b"builder-a.example.com").await.as_deref(), Some("builder-a.example.com")); + assert_eq!(host(b"builder-b.example.com").await.as_deref(), Some("builder-b.example.com")); + assert_eq!(host(b"[::1]").await.as_deref(), Some("[::1]")); + assert!(host(b"Builder-A.example.com").await.is_none()); + assert!(host(b"builder-a.example.com:443").await.is_none()); + // A URL matches on scheme, host and port + assert_eq!( + host(b"https://builder-a.example.com:443/eth").await.as_deref(), + Some("builder-a.example.com") + ); + assert!(host(b"http://builder-a.example.com:443").await.is_none()); + assert!(host(b"https://builder-b.example.com").await.is_none()); + // Parameters after `?` are for the builder and do not affect routing + assert_eq!( + host(b"builder-a.example.com?ofac=1").await.as_deref(), + Some("builder-a.example.com") + ); + } + + #[tokio::test] + async fn resolve_falls_back_to_default_relays_in_config_order() { + let own = vec![test_relay("https://builder-a.example.com/mux")]; + let default_relays = vec![ + test_relay("https://builder-a.example.com:8443"), + test_relay("https://builder-b.example.com:8443/eth"), + test_relay("https://builder-b.example.com:9000"), + test_relay("https://builder-b.example.com:9000/second"), + ]; + let from_cb = from_cb(); + let url = async |data: &[u8]| { + resolve_addressed_relay(&own, &default_relays, data, &from_cb, 0) + .await .ok() - .map(|relay| relay.config.entry.url.host_str().unwrap().to_string()) + .map(|(relay, _)| relay.config.entry.url.to_string()) }; - assert_eq!(host(b"builder-a.example.com").as_deref(), Some("builder-a.example.com")); - assert_eq!(host(b"builder-b.example.com").as_deref(), Some("builder-b.example.com")); - assert_eq!(host(b"[::1]").as_deref(), Some("[::1]")); - assert!(host(b"Builder-A.example.com").is_none()); - assert!(host(b"builder-a.example.com:443").is_none()); - // a URL is not a hostname - assert!(host(b"https://builder-a.example.com").is_none()); + assert_eq!( + url(b"builder-a.example.com").await.as_deref(), + Some("https://builder-a.example.com/mux") + ); + assert_eq!( + url(b"builder-b.example.com").await.as_deref(), + Some("https://builder-b.example.com:8443/eth") + ); + assert_eq!( + url(b"https://builder-b.example.com:9000").await.as_deref(), + Some("https://builder-b.example.com:9000/") + ); + } + + #[tokio::test] + async fn resolve_refuses_the_host_the_request_was_sent_to() { + let resolve = async |host: &'static str, data: &[u8]| { + let mut headers = HeaderMap::new(); + headers.insert(HOST, reqwest::header::HeaderValue::from_static(host)); + resolve_addressed_relay(&[], &[], data, &headers, 1000) + .await + .map(|(relay, _)| relay.config.entry.url.to_string()) + }; + for (host, data) in [ + ("1.1.1.1:18550", &b"1.1.1.1"[..]), + ("1.1.1.1", b"1.1.1.1"), + ("[2606:4700:4700:0:0:0:0:1111]:18550", b"[2606:4700:4700::1111]"), + ] { + let res = resolve(host, data).await; + assert!(matches!(res, Err(PbsClientError::AuthDataMismatch)), "{host}"); + } + assert_eq!(resolve("2.2.2.2:18550", b"1.1.1.1").await.unwrap(), "https://1.1.1.1/"); + } + + // Not from a Commit-Boost, so a miss is dialed at the address before `?`, + // and refused once the dial's setup leaves no time + #[tokio::test] + async fn resolve_dials_the_address_before_the_parameters() { + let headers = HeaderMap::new(); + let dial = + |timeout_ms| resolve_addressed_relay(&[], &[], b"1.1.1.1?ofac=1", &headers, timeout_ms); + let (relay, _) = dial(1000).await.unwrap(); + assert_eq!(relay.config.entry.url.as_str(), "https://1.1.1.1/"); + assert!(matches!(dial(0).await, Err(PbsClientError::NoBuilderResponse))); } } diff --git a/docs/docs/get_started/epbs.md b/docs/docs/get_started/epbs.md index 27cc0f75..0f04c4f9 100644 --- a/docs/docs/get_started/epbs.md +++ b/docs/docs/get_started/epbs.md @@ -18,7 +18,7 @@ Normal PBS paths are unaffected. From the Gloas fork, apart from the `status` en |---|---|---| | `POST /eth/v1/builder/execution_payload_bid/{slot}/{parent_hash}/{parent_root}/{proposer_pubkey}` | A bid request for the proposal slot | Asks the [addressed](#routing-by-auth-data) builder for its bid | | `POST /eth/v1/builder/builder_preferences/{proposer_pubkey}` | The proposer's `max_execution_payment` for one builder, ahead of the slot | Forwards it to the addressed builder | -| `POST /eth/v1/builder/beacon_blocks` | The signed beacon block that commits to the winning bid | Forwards it to every builder in `[[relays]]` and `[[mux.relays]]`; only the winning builder accepts it | +| `POST /eth/v1/builder/beacon_blocks` | The signed beacon block that commits to the winning bid | Forwards it to every builder in `[[relays]]` and `[[mux.relays]]` | With ePBS the beacon node validates each bid and weighs it against your builder config (`min_bid`, `builder_boost_factor`) and its local block, so Commit-Boost adds no redundant verification on the hot path. @@ -33,17 +33,17 @@ A `get_header = "stream"` relay is asked for ePBS bids over plain HTTP. From the fork, each validator key has a builder config: a list of entries, each a `url` and an `auth_data`. For every entry the beacon node sends a bid request to the entry's `url`, carrying its `auth_data` in a `SignedBuilderRequestAuth` that the validator signs. `auth_data` tells a builder that a request was meant for it, so a request signed for one builder is rejected by another. Unless the proposer and builder agree on another value, it is the builder's hostname. -To go through Commit-Boost, every entry's `url` is Commit-Boost's own URL, and its `auth_data` is the hostname of the builder it stands for. Commit-Boost reads the `auth_data` of each request, finds the relay entry with that hostname and forwards the request there. Builder preferences are routed the same way. The signed block goes to every builder, since only the one whose bid won accepts it. +To go through Commit-Boost, every entry's `url` is Commit-Boost's own URL, and its `auth_data` is the hostname of the builder it stands for. Commit-Boost reads the `auth_data` of each request, finds the relay entry with that hostname and forwards the request there. If no relay entry has it, Commit-Boost dials the builder the `auth_data` names ([builders outside your config](#builders-outside-your-config)). Builder preferences are routed the same way. The signed block goes to every builder in your config, since Commit-Boost does not track which bid won. ## Setup ### 1. Add the builders to Commit-Boost -List each builder as a relay entry, as for PBS: in `[[relays]]`, or in the `[[mux.relays]]` of the mux that lists the proposer's key. ePBS adds one option, `[pbs] proposer_deadline_buffer_ms` (default `50`): the time kept back from the beacon node's deadline (see [Timing](#timing)). It must be under one slot, `12000` on mainnet. +List each builder as a relay entry, as for PBS: in `[[relays]]`, or in the `[[mux.relays]]` of the mux that lists the proposer's key. ePBS adds one option, `[pbs] proposer_deadline_buffer_ms` (default `50`): the time kept back from the beacon node's deadline (see [Timing](#timing)). It must be above `0` and under one slot, `12000` on mainnet. ### 2. Point each validator key at Commit-Boost {#validator-builder-config} -Write each key's builder config through its validator client's keymanager API, which must implement the builder config endpoint ([keymanager-APIs #88](https://github.com/ethereum/keymanager-APIs/pull/88)). Add one entry per builder: `url` is Commit-Boost's URL, and `auth_data` is the hex of the builder's hostname. A key without builder config sends the default auth data of Commit-Boost's own URL, which matches no builder, so every bid request gets `400`. +Write each key's builder config through its validator client's keymanager API, which must implement the builder config endpoint ([keymanager-APIs #88](https://github.com/ethereum/keymanager-APIs/pull/88)). Add one entry per builder: `url` is Commit-Boost's URL, and `auth_data` is the hex of the builder's hostname. A key without builder config gets no bids through Commit-Boost ([builders outside your config](#builders-outside-your-config)). With the relay entry `url = "https://0xa1ce...@builder-a.example.com"`, Commit-Boost listening at `http://cb.example.com:18550`, the keymanager API at `$KEYMANAGER_URL`, its token in `$TOKEN` and the validator key in `$PUBKEY`: @@ -66,19 +66,29 @@ curl -X POST "$KEYMANAGER_URL/eth/v1/validator/$PUBKEY/builder_config" \ The call is `POST /eth/v1/validator/{pubkey}/builder_config`, authenticated with the keymanager API's bearer token. The body replaces the key's config in full, and the validator client answers `202` once it is stored. Besides `url` and `auth_data`, an entry or the top level can set: -- `min_bid` and `builder_boost_factor` are optional. At the top level they apply to every entry, and to p2p bids, unless an entry sets its own. A top-level `min_bid` also floors the bids that come through Commit-Boost, so a value above your builders' bids sends every proposal to p2p bids or a local block. -- `max_execution_payment`, when an entry omits it, gets the validator client's default, which differs between clients (Lodestar stores `0`, Nimbus the maximum). Lodestar accepts a nonzero value only when started with `--allowDangerousTrustedPayments`. +- `min_bid` and `builder_boost_factor` are optional. At the top level they apply to p2p bids and to every entry that does not set its own, so a top-level `min_bid` above your builders' bids sends every proposal to a p2p bid or a local block, unless an entry sets a lower one of its own. +- `max_execution_payment`, when an entry omits it, gets the validator client's default, which differs between clients and can be `0`, so no execution payment counts. Lodestar accepts a nonzero value only when started with `--allowDangerousTrustedPayments`. - `builder_pubkeys` limits which builder keys' bids the beacon node accepts; leave it empty to accept any. Don't copy the pubkey from the relay URL: that is the relay's key, not the builder's bid-signing key. ## Routing by auth data -Commit-Boost sends a bid or preferences request to the first builder serving the proposer's key (the relays of its `[[mux]]`, otherwise `[[relays]]`) whose URL hostname equals the request's `auth_data`, byte for byte. That is the [builder specs](https://github.com/ethereum/builder-specs/blob/main/specs/gloas/validator.md#default-auth-data) default auth data: the lowercase hostname, without scheme, port or path, with an IPv6 address in brackets, such as `[::1]`. Builders on one host share it, so only the first is asked. Commit-Boost routes by hostname only, so a value agreed with a builder works through it only if it is that hostname. When no builder matches, the request gets `400`. +Commit-Boost sends a bid or preferences request to the first builder whose URL hostname equals the request's `auth_data`, byte for byte. That is the [builder specs](https://github.com/ethereum/builder-specs/blob/main/specs/gloas/validator.md#default-auth-data) default auth data: the lowercase ASCII hostname (punycode for an internationalized name), without scheme, port or path, with an IPv6 address compressed and in brackets, such as `[::1]`. Builders on one host share it, so only the first is asked. Commit-Boost looks first among the relays serving the proposer's key (those of its `[[mux]]`, otherwise `[[relays]]`), then among `[[relays]]` in config order, so a key in a mux still reaches a `[[relays]]` builder through that entry, with its headers, path and port. A builder listed only in another mux's `[[mux.relays]]` is not used for this key: it is dialed like a builder outside your config. + +`auth_data` can also be an `http` or `https` URL agreed with a builder. It then matches the first builder whose `url` has the same scheme, host and port; the path and the pubkey in the relay URL do not count, so builders on one origin share it. Any other value agreed with a builder does not route through Commit-Boost. When no builder matches, Commit-Boost [dials the builder](#builders-outside-your-config) the `auth_data` names, and answers `400` when the `auth_data` is neither in that hostname form nor an `http(s)` URL. + +Either form may end in `?` and form-encoded parameters for the builder, such as `builder-a.example.com?ofac=1`. Commit-Boost routes on the part before `?`. The parameters reach the builder inside the signed auth, so set them only for a builder that accepts them; one that does not answers `400`. + +## Builders outside your config {#builders-outside-your-config} + +A key's builder config may name a builder you have not added as a relay entry. Commit-Boost dials that builder anyway: a hostname at `https://` on the default port, and a URL at its scheme, host and port. It answers `400` instead when the host does not resolve, or resolves to any address that is not public unicast, such as a loopback, private, link-local or `100.64.0.0/10` address, so a builder on your own network has to be a relay entry. These requests connect directly, ignoring `HTTP_PROXY`, `HTTPS_PROXY` and `ALL_PROXY`, follow no redirects and do not carry your relay `headers`. A builder reached this way gets the signed block over gossip, unless it is also a relay entry in `[[relays]]` or a mux. Neither `[[relays]]` nor a mux's relays limit which builders a key can reach this way, and anyone who can reach Commit-Boost's port can make it dial a public host, so keep that port private. Commit-Boost looks up at most 32 of these hosts at once and answers further requests as if the builder had not responded, as it does when a lookup runs out of the request's time. + +On a validator client whose builder URL is Commit-Boost, a key without builder config sends Commit-Boost's own hostname as its `auth_data`, so it gets no bids: unless a relay entry has that hostname, which then gets the request, Commit-Boost answers `400` without dialing. A client that sends Commit-Boost's full URL instead also gets `400`. Commit-Boost never dials for a request that came from another Commit-Boost, so chained Commit-Boosts route only through relay entries. ## Timing -The builder specs require each bid request to carry `Date-Milliseconds` and `X-Timeout-Ms`, which together say how long the beacon node will wait for a bid. Commit-Boost subtracts your `proposer_deadline_buffer_ms` from the time remaining and sends the result to the builder as its `X-Timeout-Ms`, so the builder knows how long it has to answer. +The builder specs require each bid request to carry `Date-Milliseconds` and `X-Timeout-Ms`, which together say how long the beacon node will wait for a bid. Commit-Boost subtracts your `proposer_deadline_buffer_ms` from the time remaining and sends the result to the builder as its `X-Timeout-Ms`, so the builder knows how long it has to answer. For a builder outside your config, the time its lookup took is subtracted too. -Think of `proposer_deadline_buffer_ms` as the slack you keep from the beacon node's remaining time: enough for the bid to get back from Commit-Boost to the beacon node and for the beacon node to process it. If no time is left, Commit-Boost does not ask the builder. +Think of `proposer_deadline_buffer_ms` as the slack you keep from the beacon node's remaining time: enough for the bid to get back from Commit-Boost to the beacon node and for the beacon node to process it. If no time is left, Commit-Boost does not ask the builder. With `0`, Commit-Boost refuses to start or reload: the builder gets the whole deadline, so a bid it sends at the deadline reaches the beacon node late. Builder preferences use `timeout_register_validator_ms`, and the signed block `timeout_get_payload_ms`. @@ -88,8 +98,8 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | Question | Series | |---|---| -| What did Commit-Boost answer the beacon node? | `cb_pbs_beacon_node_status_code_total`: `200` or `204` for a bid request, `202` for preferences and the signed block, `4xx` for a rejected request, `500` when no builder accepted preferences or the signed block | -| What did each builder answer? | `cb_pbs_relay_status_code_total`, by `relay_id`: the builder's HTTP status, or `555` when no response arrived (timeout, DNS or connection failure) | +| What did Commit-Boost answer the beacon node? | `cb_pbs_beacon_node_status_code_total`: `200` or `204` for a bid request, `202` for preferences, `202` for the signed block whatever the builders answer, `4xx` for a rejected request, `500` when no builder accepted the preferences or on an internal error | +| What did each builder answer? | `cb_pbs_relay_status_code_total`, by `relay_id`: the builder's HTTP status, or `555` when no response arrived (timeout, DNS or connection failure). Requests to builders outside your config count under `relay_id="dial"`; a dial Commit-Boost [refuses](#builders-outside-your-config), or whose lookup fails, times out or finds no free slot, sends no request and is not counted | | How fast are builders? | `cb_pbs_relay_latency`, by `relay_id` | | What are builders bidding? | `cb_pbs_relay_header_value` (the bid's `value` in Gwei, without the execution payment) and `cb_pbs_relay_last_slot`, by `relay_id`, from each bid a builder serves | @@ -97,7 +107,8 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | Symptom | Cause | Fix | |---|---|---| -| Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data matches no builder serving the key. Usually the key has no builder config, so its auth data is Commit-Boost's own hostname | Write the key's [builder config](#validator-builder-config), or add the builder it names as a relay entry for the key | -| A builder answers bid requests with `400` (`cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid",http_status_code="400"}`) | The builder compares auth data byte for byte and expects something other than its hostname | Have the builder accept its hostname, the only auth data Commit-Boost [routes by](#routing-by-auth-data) | -| The signed block gets `415` | The beacon node sends it as JSON | Configure the beacon node to send SSZ | +| Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data is neither a lowercase hostname without a port nor an `http(s)` URL, names the host the request was sent to (a key without builder config), or the request came from another Commit-Boost and matches none of this one's relay entries | Correct the `auth_data` in the key's [builder config](#validator-builder-config), or add the builder as a relay entry on the Commit-Boost the request reaches | +| Bid requests get `400` "the addressed builder's host does not resolve or resolves to a disallowed address" | The auth data names a builder outside your config whose host does not resolve or resolves to an [internal address](#builders-outside-your-config) | Add the builder as a relay entry for the key, or write or correct the key's [builder config](#validator-builder-config) | +| A builder answers bid requests with `400` (`cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid",http_status_code="400"}`) | The builder compares auth data byte for byte and expects something other than what the key sends, such as its hostname without `?` parameters | Send the auth data the builder expects: its hostname or its URL, the forms Commit-Boost [routes by](#routing-by-auth-data), with `?` parameters only if it accepts them | +| Commit-Boost does not start or reload, or `commit-boost init` fails, with "proposer_deadline_buffer_ms must be greater than 0 and less than one slot" | `proposer_deadline_buffer_ms` is `0`, or one slot or more | Set it above `0` and under one slot, or remove it to use the default `50` | | Commit-Boost returns `200` but the beacon node builds locally | The beacon node rejected the bid, or valued its local block higher after `builder_boost_factor` | Compare the bid with the key's `min_bid` and `builder_boost_factor` | diff --git a/docs/docs/get_started/running/metrics-catalog.md b/docs/docs/get_started/running/metrics-catalog.md index fe44cc3f..74aba597 100644 --- a/docs/docs/get_started/running/metrics-catalog.md +++ b/docs/docs/get_started/running/metrics-catalog.md @@ -10,19 +10,19 @@ Every metric emitted by the Commit-Boost PBS and Signer services, together with ## PBS metrics -PBS metrics use a custom Prometheus registry with namespace prefix `cb_pbs_`. The registry is created via `Registry::new_custom(Some("cb_pbs"), None)` in `crates/pbs/src/metrics.rs`. All wire names shown below include this prefix. +PBS metrics use a custom Prometheus registry with namespace prefix `cb_pbs_`. The registry is created via `Registry::new_custom(Some("cb_pbs"), None)` in `crates/pbs/src/metrics.rs`. All wire names shown below include this prefix. (unreleased, from v0.12.0-rc1) ePBS requests to a builder outside your config are labeled `relay_id="dial"`. | Metric name (wire) | Type | Labels | Description | |---|---|---|---| -| `cb_pbs_relay_status_code_total` | Counter | `http_status_code`, `endpoint`, `relay_id` | HTTP status code received by relay. Incremented once per relay response. Two synthetic codes stand in for outcomes that never reach a status line: `"555"` when the request hit its deadline, and `"556"` for a bid-stream transport failure (connect failed, the stream broke mid-window, or the handshake was answered with a success code instead of the upgrade). Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`. | -| `cb_pbs_relay_latency` | Histogram | `endpoint`, `relay_id` | HTTP latency (duration in seconds) by relay. Records the duration of relay HTTP requests. Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`. Under `get_header_stream` the observation is the time from opening the websocket to the first bid update, not a round trip, and a window that received no bid records nothing. | -| `cb_pbs_relay_last_slot` | Gauge | `relay_id` | Latest slot for which a relay delivered a header. Only updated in the `get_header` handler. Set to the current slot on each successful header from that relay. | -| `cb_pbs_relay_header_value` | Gauge | `relay_id` | Header value in gwei delivered by a relay. Converted from raw wei (÷ 1e9) in the `get_header` handler. | +| `cb_pbs_relay_status_code_total` | Counter | `http_status_code`, `endpoint`, `relay_id` | HTTP status code received by relay. Incremented once per relay response. Two synthetic codes stand in for outcomes that never reach a status line: `"555"` when no response arrived (timeout, DNS or connection failure; on `get_header_stream`, only the deadline), and `"556"` for a bid-stream transport failure (connect failed, the stream broke mid-window, or the handshake was answered with a success code instead of the upgrade). Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `submit_builder_preferences`, `submit_signed_beacon_block`. | +| `cb_pbs_relay_latency` | Histogram | `endpoint`, `relay_id` | HTTP latency (duration in seconds) by relay. Records the duration of relay HTTP requests. Endpoint values: `get_header`, `get_header_stream`, `register_validator`, `submit_blinded_block`, `status`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `submit_builder_preferences`, `submit_signed_beacon_block`. Under `get_header_stream` the observation is the time from opening the websocket to the first bid update, not a round trip, and a window that received no bid records nothing. | +| `cb_pbs_relay_last_slot` | Gauge | `relay_id` | Latest slot for which a relay delivered a bid. Set to the current slot on each successful `get_header` bid and (unreleased, from v0.12.0-rc1) each ePBS bid served from that relay. | +| `cb_pbs_relay_header_value` | Gauge | `relay_id` | Value in gwei of the bid a relay delivered: for `get_header`, the bid's value converted from wei (÷ 1e9). (unreleased, from v0.12.0-rc1) For ePBS, the returned bid's `value`, without its execution payment. | | `cb_pbs_relay_stream_connect_latency` | Histogram | `relay_id` | Websocket handshake latency in seconds, for a relay with `get_header = "stream"`. Observed once per successful handshake; a handshake that fails or runs out the bid window records nothing here. Custom buckets from 5 ms to 1 s. | | `cb_pbs_relay_stream_updates` | Histogram | `relay_id` | Bid updates received per stream window. Observed once per `get_header` served by the stream, including windows that received nothing. Buckets: `0, 1, 2, 3, 5, 10, 20, 50, 100`. | | `cb_pbs_relay_stream_invalid_frames_total` | Counter | `relay_id` | Websocket frames that could not be parsed as a bid. Incremented only for windows that saw at least one, so the series is absent while zero. | | `cb_pbs_relay_stream_fallback_total` | Counter | `relay_id` | Stream attempts that fell back to a plain HTTP `get_header`. Only a handshake failure with bid window left to retry counts here; one that consumed the window shows on the status series alone. The fallback's own result is recorded under `endpoint="get_header"`. | -| `cb_pbs_beacon_node_status_code_total` | Counter | `http_status_code`, `endpoint` | HTTP status code returned to the beacon node. Tracks what status codes the PBS returns for beacon node-facing requests. Endpoint values: `get_header`, `register_validator`, `submit_blinded_block`, `status`, `reload`. Error status codes (`502` for `NoResponse`/`NoPayload`, `500` for `Internal`) are set via `PbsClientError`. The handlers also record `406` directly when the request's `Accept` header offers no supported encoding, `204` when no bid is available on `get_header`, and `202` for accepted v2 `submit_blinded_block` requests. | +| `cb_pbs_beacon_node_status_code_total` | Counter | `http_status_code`, `endpoint` | HTTP status code returned to the beacon node. Tracks what status codes the PBS returns for beacon node-facing requests. Endpoint values: `get_header`, `register_validator`, `submit_blinded_block`, `status`, `reload`, and (unreleased, from v0.12.0-rc1) `get_execution_payload_bid`, `submit_builder_preferences`, `submit_signed_beacon_block`. Error status codes (`502` for `NoResponse`/`NoPayload`, `500` for `Internal`) are set via `PbsClientError`. The handlers also record `406` directly when the request's `Accept` header offers no supported encoding, `204` when no bid is available on `get_header`, and `202` for accepted v2 `submit_blinded_block` requests. (unreleased, from v0.12.0-rc1) The ePBS endpoints record `200` for a bid, `204` for none, `202` for accepted preferences and for every signed block it forwards, `400` for a request Commit-Boost refuses, a builder's own `400` or `401`, `406` for a bid request whose `Accept` offers no supported encoding, `415` for an unsupported `Content-Type`, and `500` when no builder accepts the preferences. | | `cb_pbs_pbs_submit_block_v2_unsupported_total` | Counter | `relay_id` | Count of v2 `submit_blinded_block` requests a relay could not serve because it returned 404 on the v2 endpoint. A non-zero value means the relay does not support `submitBlindedBlockV2` and those blocks were not submitted via that relay. The double `pbs` in the wire name comes from the registry prefix plus the metric name `pbs_submit_block_v2_unsupported_total`. | --- diff --git a/tests/Cargo.toml b/tests/Cargo.toml index c8503378..3e6dfdb4 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -34,3 +34,4 @@ url.workspace = true [dev-dependencies] tracing-test.workspace = true cb-common = { path = "../crates/common", features = ["testing-flags"] } +cb-pbs = { path = "../crates/pbs", features = ["testing-flags"] } diff --git a/tests/src/mock_relay.rs b/tests/src/mock_relay.rs index 5826a1d8..e9785d2e 100644 --- a/tests/src/mock_relay.rs +++ b/tests/src/mock_relay.rs @@ -41,8 +41,9 @@ use cb_common::{ }, }; use cb_pbs::{ - GET_HEADER_ENDPOINT_TAG, MAX_SIZE_SUBMIT_BLOCK_RESPONSE, REGISTER_VALIDATOR_ENDPOINT_TAG, - STATUS_ENDPOINT_TAG, SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG, + GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG, GET_HEADER_ENDPOINT_TAG, + MAX_SIZE_SUBMIT_BLOCK_RESPONSE, REGISTER_VALIDATOR_ENDPOINT_TAG, STATUS_ENDPOINT_TAG, + SUBMIT_BLINDED_BLOCK_ENDPOINT_TAG, }; use lh_types::{KzgProof, Slot}; use reqwest::header::{ACCEPT, CONTENT_TYPE}; @@ -103,6 +104,8 @@ pub struct MockRelayState { /// `slot` of the last signed beacon block forwarded, decoded from the SSZ /// body PBS sent, so a test can assert the block survived the hop received_block_slot: RwLock>, + /// Content-Type and body of the last signed beacon block forwarded + received_block_request: RwLock, axum::body::Bytes)>>, /// The last `BuilderPreferencesRequest` submitted, so a test can assert /// both the preferences and the auth were forwarded unchanged received_preferences: RwLock>, @@ -129,6 +132,11 @@ pub struct MockRelayState { trustless_bid_gwei: u64, // default 10 /// When true, an SSZ bid is served without `Eth-Consensus-Version` epbs_omit_consensus_version: bool, + /// When true, a JSON bid is pretty-printed, so it differs from the compact + /// form a re-encode produces + pretty_json_bid: bool, + /// The body of the last bid served + served_bid: RwLock>, } impl MockRelayState { @@ -156,6 +164,11 @@ impl MockRelayState { pub fn received_signed_beacon_block(&self) -> u64 { self.received_signed_beacon_block.load(Ordering::Relaxed) } + /// The last forwarded block's Content-Type and body + pub fn received_block_request(&self) -> Option<(Option, axum::body::Bytes)> { + self.received_block_request.read().unwrap().clone() + } + /// `slot` of the last signed beacon block PBS forwarded pub fn received_block_slot(&self) -> Option { *self.received_block_slot.read().unwrap() @@ -189,6 +202,9 @@ impl MockRelayState { pub fn received_auth_data(&self) -> Option> { self.received_auth.read().unwrap().as_ref().map(|a| a.message.data.to_vec()) } + pub fn served_bid(&self) -> Option { + self.served_bid.read().unwrap().clone() + } pub fn large_body(&self) -> bool { self.large_body } @@ -240,6 +256,7 @@ impl MockRelayState { received_builder_preferences: Default::default(), received_signed_beacon_block: Default::default(), received_block_slot: RwLock::new(None), + received_block_request: RwLock::new(None), received_preferences: RwLock::new(None), received_preferences_pubkey: RwLock::new(None), bid_delay_ms: None, @@ -252,6 +269,8 @@ impl MockRelayState { api_keys_seen: RwLock::new(HashMap::new()), trustless_bid_gwei: 10, epbs_omit_consensus_version: false, + pretty_json_bid: false, + served_bid: RwLock::new(None), supported_content_types: Arc::new( [EncodingType::Json, EncodingType::Ssz].iter().cloned().collect(), ), @@ -329,6 +348,10 @@ impl MockRelayState { pub fn with_epbs_omit_consensus_version(self) -> Self { Self { epbs_omit_consensus_version: true, ..self } } + + pub fn with_pretty_json_bid(self) -> Self { + Self { pretty_json_bid: true, ..self } + } } pub fn mock_relay_app_router(state: Arc) -> Router { @@ -387,6 +410,7 @@ async fn handle_get_execution_payload_bid( body: axum::body::Bytes, ) -> Response { state.received_execution_payload_bid.fetch_add(1, Ordering::Relaxed); + state.record_api_key(GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG, &headers); // Builders MUST 400 a bid request without both timing headers let header_u64 = |name| headers.get(name).and_then(|v| v.to_str().ok()).and_then(|v| v.parse::().ok()); @@ -427,6 +451,10 @@ async fn handle_get_execution_payload_bid( // fail on the bid endpoint (its bid is then dropped by PBS). The request was // already counted above. if let Some(status) = *state.response_override.read().unwrap() { + // An error body over PBS's 1 KiB read cap + if state.large_body() { + return (status, "x".repeat(2048)).into_response(); + } return status.into_response(); } @@ -485,9 +513,15 @@ async fn handle_get_execution_payload_bid( data, metadata: Default::default(), }; - serde_json::to_vec(&versioned).unwrap() + if state.pretty_json_bid { + serde_json::to_vec_pretty(&versioned).unwrap() + } else { + serde_json::to_vec(&versioned).unwrap() + } } }; + let response_body = axum::body::Bytes::from(response_body); + *state.served_bid.write().unwrap() = Some(response_body.clone()); let mut response = (StatusCode::OK, response_body).into_response(); // A real builder tags the 200 with the fork so a client can decode the @@ -629,15 +663,16 @@ async fn handle_submit_builder_preferences( StatusCode::ACCEPTED.into_response() } -/// Decodes the forwarded block (PBS always sends SSZ with -/// `Eth-Consensus-Version`), records its slot, and 202s -/// unless the test overrode the response. +/// Records the forwarded block's Content-Type and body, and its slot when it is +/// SSZ, and 202s unless the test overrode the response async fn handle_submit_signed_beacon_block( headers: HeaderMap, State(state): State>, body: axum::body::Bytes, ) -> Response { state.received_signed_beacon_block.fetch_add(1, Ordering::Relaxed); + let content_type = headers.get(CONTENT_TYPE).and_then(|v| v.to_str().ok()).map(String::from); + *state.received_block_request.write().unwrap() = Some((content_type, body.clone())); if let Some(fork) = get_consensus_version_header(&headers) && let Ok(block) = SignedBeaconBlock::from_ssz_bytes_by_fork(&body, fork) diff --git a/tests/src/mock_validator.rs b/tests/src/mock_validator.rs index 2059a245..d3d587df 100644 --- a/tests/src/mock_validator.rs +++ b/tests/src/mock_validator.rs @@ -11,11 +11,11 @@ use cb_common::{ use lh_types::ForkName; use reqwest::{ Response, - header::{ACCEPT, CONTENT_TYPE}, + header::{ACCEPT, CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue}, }; use ssz::Encode; -use crate::utils::{TEST_PROPOSER_PUBKEY, generate_mock_relay}; +use crate::utils::{API_KEY, HEADER_API_KEY, TEST_PROPOSER_PUBKEY, generate_mock_relay}; /// Timeout a test beacon node advertises on bid requests; long enough that the /// deadline never bites in tests. @@ -30,7 +30,15 @@ impl MockValidator { let pubkey = bls_pubkey_from_hex( "0xac6e77dfe25ecd6110b8e780608cce0dab71fdd5ebea22a16c0205200f2f8e2e3ad3b71d3499c54ad14d6c21b41a37ae", )?; - Ok(Self { comm_boost: generate_mock_relay(port, pubkey)? }) + let mut comm_boost = generate_mock_relay(port, pubkey)?; + // Like a beacon node: no X-CommitBoost-Version, which stops a dial. The + // API key differs from the relays', so one forwarded from here shows + let headers = HeaderMap::from_iter([( + HeaderName::from_static(HEADER_API_KEY), + HeaderValue::from_static(API_KEY), + )]); + comm_boost.client = reqwest::Client::builder().default_headers(headers).build()?; + Ok(Self { comm_boost }) } pub async fn do_get_header( diff --git a/tests/src/utils.rs b/tests/src/utils.rs index d3c92416..09412d2a 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -11,7 +11,7 @@ use cb_common::{ config::{ COMMIT_BOOST_IMAGE_DEFAULT, CommitBoostConfig, GetHeaderTransport, HeaderSource, LogsSettings, ModuleKind, ModuleSigningConfig, PbsConfig, PbsModuleConfig, RelayConfig, - ReverseProxyHeaderSetup, SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, + ReverseProxyHeaderSetup, RuntimeMuxConfig, SIGNER_JWT_AUTH_FAIL_LIMIT_DEFAULT, SIGNER_JWT_AUTH_FAIL_TIMEOUT_SECONDS_DEFAULT, SIGNER_PORT_DEFAULT, SignerConfig, SignerType, StartSignerConfig, StaticModuleConfig, StaticPbsConfig, TlsMode, }, @@ -68,7 +68,7 @@ fn mock_relay_config(port: u16, pubkey: BlsPublicKey) -> Result { id: None, headers: Some(HashMap::from([( HEADER_API_KEY.into(), - HeaderSource::Literal(API_KEY.into()), + HeaderSource::Literal(RELAY_API_KEY.into()), )])), get_params: None, get_header: GetHeaderTransport::Http, @@ -138,7 +138,7 @@ pub fn get_pbs_config(port: u16) -> PbsConfig { skip_sigverify: false, min_bid_wei: U256::ZERO, late_in_slot_time_ms: u64::MAX, - proposer_deadline_buffer_ms: 0, + proposer_deadline_buffer_ms: 50, extra_validation_enabled: false, ssv_node_api_url: Url::parse("http://localhost:0").unwrap(), @@ -347,6 +347,42 @@ pub async fn setup_relays_on_hosts( /// Poll /status until PBS and its relays are up. relay_check makes a 200 mean /// the whole chain is ready; the fixed 100ms sleep used elsewhere flakes under /// parallel suite load. +/// A PBS with a `[[relays]]` relay and a relay only in the `[[mux]]` of +/// `muxed_key`, both on host 0.0.0.0 +pub struct MuxedRelays { + pub validator: MockValidator, + pub muxed_key: BlsPublicKey, + pub default_state: Arc, + pub default_port: u16, + pub mux_state: Arc, + pub mux_port: u16, +} + +pub async fn setup_mux_relays(chain: Chain) -> Result { + let (default_state, default_port) = + spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + let (mux_state, mux_port) = + spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + let default_relay = generate_mock_relay(default_port, default_state.signer.public_key())?; + let mux_relay = generate_mock_relay(mux_port, mux_state.signer.public_key())?; + + let pbs_listener = get_free_listener().await; + let pbs_port = pbs_listener.local_addr()?.port(); + let mut config = to_pbs_config(chain, get_pbs_config(pbs_port), vec![default_relay.clone()]); + config.all_relays = vec![mux_relay.clone(), default_relay]; + let muxed_key = random_secret().public_key(); + config.mux_lookup = Some(HashMap::from([(muxed_key.clone(), RuntimeMuxConfig { + id: "test".to_string(), + config: config.pbs_config.clone(), + relays: vec![mux_relay], + })])); + let state = PbsState::new(config, PathBuf::new()); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); + let validator = MockValidator::new(pbs_port)?; + wait_for_ready(&validator).await?; + Ok(MuxedRelays { validator, muxed_key, default_state, default_port, mux_state, mux_port }) +} + pub async fn wait_for_ready(mock_validator: &MockValidator) -> Result<()> { for _ in 0..100 { if let Ok(res) = mock_validator.do_get_status().await && diff --git a/tests/tests/pbs_cfg_file_update.rs b/tests/tests/pbs_cfg_file_update.rs index 23d59e94..c4c02ce1 100644 --- a/tests/tests/pbs_cfg_file_update.rs +++ b/tests/tests/pbs_cfg_file_update.rs @@ -66,7 +66,7 @@ async fn test_cfg_file_update() -> Result<()> { min_bid_wei: U256::ZERO, late_in_slot_time_ms: u64::MAX / 2, /* serde gets very upset about serializing u64::MAX * or anything close to it */ - proposer_deadline_buffer_ms: 0, + proposer_deadline_buffer_ms: 50, extra_validation_enabled: false, rpc_url: None, ssv_node_api_url: Url::parse("http://example.com").unwrap(), diff --git a/tests/tests/pbs_get_execution_payload_bid.rs b/tests/tests/pbs_get_execution_payload_bid.rs index 3eca13e2..79d04f7e 100644 --- a/tests/tests/pbs_get_execution_payload_bid.rs +++ b/tests/tests/pbs_get_execution_payload_bid.rs @@ -1,25 +1,21 @@ -use std::{collections::HashMap, path::PathBuf}; - use alloy::primitives::{B256, U256}; use cb_common::{ - config::RuntimeMuxConfig, pbs::{ - GetExecutionPayloadBidResponse, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, + GetExecutionPayloadBidResponse, HEADER_START_TIME_UNIX_MS, HEADER_TIMEOUT_MS, RelayClient, SignedBuilderRequestAuth, SignedExecutionPayloadBid, }, signer::random_secret, - types::Chain, + types::{BlsPublicKey, Chain}, utils::utcnow_ms, wire::{CONSENSUS_VERSION_HEADER, EncodingType}, }; -use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; +use cb_pbs::GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG; use cb_tests::{ mock_relay::MockRelayState, mock_validator::MockValidator, utils::{ - TEST_AUTH_DATA, TEST_PROPOSER_PUBKEY, generate_mock_relay, get_free_listener, - get_pbs_config, opaque_auth, setup_relay, setup_relays, setup_relays_on_hosts, - spawn_mock_relay, to_pbs_config, wait_for_ready, + RELAY_API_KEY, TEST_AUTH_DATA, TEST_PROPOSER_PUBKEY, generate_mock_relay, opaque_auth, + setup_mux_relays, setup_relay, setup_relays, setup_relays_on_hosts, spawn_mock_relay, }, }; use eyre::Result; @@ -121,9 +117,8 @@ async fn test_get_execution_payload_bid_shared_auth_data_asks_the_first() -> Res } /// The spec default auth data, the builder's hostname, routes to the relay on -/// that host, and the auth reaches the relay unchanged. A hostname no relay -/// has is a 400 with the builder's data-mismatch message, and no relay is -/// contacted. +/// that host, and the auth reaches the relay unchanged. An unconfigured +/// `localhost` resolves to loopback, so it is refused with 400 and not dialed. #[tokio::test] async fn test_get_execution_payload_bid_demux_by_hostname() -> Result<()> { let chain = Chain::Hoodi; @@ -148,52 +143,88 @@ async fn test_get_execution_payload_bid_demux_by_hostname() -> Result<()> { assert_eq!(body["code"], 400); assert_eq!( body["message"], - "Invalid SignedBuilderRequestAuth: auth.message.data does not match any configured builder" + "Invalid SignedBuilderRequestAuth: the addressed builder's host does not resolve or resolves to a disallowed address" ); Ok(()) } -/// A key listed in a `[[mux]]` is served by that mux's relays: its bid request -/// reaches the mux relay and not the default `[[relays]]` one. +/// Auth data that matches no relay entry is dialed, and the builder gets the +/// auth, `?` parameters included, unchanged. A request from a Commit-Boost is +/// not dialed, though a configured relay still serves it. #[tokio::test] -async fn test_get_execution_payload_bid_mux_routes_to_mux_relays() -> Result<()> { +async fn test_get_execution_payload_bid_dial() -> Result<()> { + // The mock builder listens on an address the dial check refuses + cb_pbs::set_skip_dial_target_check(true); let chain = Chain::Hoodi; - let (default_state, default_port) = + let (mut mock_validator, cfg_state) = setup_relay(chain, |_| {}, generate_mock_relay).await?; + let (dial_state, dial_port) = spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; - let (mux_state, mux_port) = - spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; - let default_relay = generate_mock_relay(default_port, default_state.signer.public_key())?; - let mux_relay = generate_mock_relay(mux_port, mux_state.signer.public_key())?; - - let pbs_listener = get_free_listener().await; - let pbs_port = pbs_listener.local_addr()?.port(); - let mut config = to_pbs_config(chain, get_pbs_config(pbs_port), vec![default_relay.clone()]); - config.all_relays = vec![mux_relay.clone(), default_relay]; - let muxed_key = random_secret().public_key(); - config.mux_lookup = Some(HashMap::from([(muxed_key.clone(), RuntimeMuxConfig { - id: "test".to_string(), - config: config.pbs_config.clone(), - relays: vec![mux_relay], - })])); - let state = PbsState::new(config, PathBuf::new()); - tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); - let mock_validator = MockValidator::new(pbs_port)?; - wait_for_ready(&mock_validator).await?; + let dial_auth_data = format!("http://0.0.0.0:{dial_port}/?ofac=1").into_bytes(); - let auth = opaque_auth(TEST_AUTH_DATA, TEST_SLOT); - let res = mock_validator - .do_get_execution_payload_bid( - TEST_SLOT, - B256::ZERO, - B256::ZERO, - Some(muxed_key), - Some(&auth), - vec![EncodingType::Json], - ) - .await?; + let res = get_json_bid(&mock_validator, &opaque_auth(&dial_auth_data, TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::OK); + assert_eq!(dial_state.received_auth_data(), Some(dial_auth_data.clone())); + // A dial carries none of the relays' configured headers, the API key included + assert_eq!(dial_state.api_key_seen(GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG), None); + + // RelayClient::new's client sends the Commit-Boost version header + mock_validator.comm_boost = + RelayClient::new(mock_validator.comm_boost.config.as_ref().clone())?; + let res = get_json_bid(&mock_validator, &opaque_auth(&dial_auth_data, TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + let body: serde_json::Value = serde_json::from_slice(&res.bytes().await?)?; + assert_eq!( + body["message"], + "Invalid SignedBuilderRequestAuth: auth.message.data does not match any configured builder" + ); + let res = get_json_bid(&mock_validator, &opaque_auth(TEST_AUTH_DATA, TEST_SLOT)).await?; assert_eq!(res.status(), StatusCode::OK); - assert_eq!(mux_state.received_execution_payload_bid(), 1); - assert_eq!(default_state.received_execution_payload_bid(), 0); + assert_eq!(dial_state.received_execution_payload_bid(), 1); + assert_eq!(cfg_state.received_execution_payload_bid(), 1); + assert_eq!( + cfg_state.api_key_seen(GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG).as_deref(), + Some(RELAY_API_KEY) + ); + Ok(()) +} + +/// A key in a `[[mux]]` is served by that mux's relays, and reaches a relay +/// configured only in `[[relays]]` through its entry, its API key included. A +/// relay only in a mux is not reached so by other keys: it is dialed instead +#[tokio::test] +async fn test_get_execution_payload_bid_mux_routes_to_mux_relays() -> Result<()> { + let relays = setup_mux_relays(Chain::Hoodi).await?; + let bid = async |key: Option, auth_data: &[u8]| { + relays + .validator + .do_get_execution_payload_bid( + TEST_SLOT, + B256::ZERO, + B256::ZERO, + key, + Some(&opaque_auth(auth_data, TEST_SLOT)), + vec![EncodingType::Json], + ) + .await + }; + let muxed = Some(relays.muxed_key.clone()); + + assert_eq!(bid(muxed.clone(), TEST_AUTH_DATA).await?.status(), StatusCode::OK); + assert_eq!(relays.mux_state.received_execution_payload_bid(), 1); + assert_eq!(relays.default_state.received_execution_payload_bid(), 0); + + let to_default = format!("http://0.0.0.0:{}", relays.default_port); + assert_eq!(bid(muxed, to_default.as_bytes()).await?.status(), StatusCode::OK); + assert_eq!(relays.default_state.received_execution_payload_bid(), 1); + assert_eq!( + relays.default_state.api_key_seen(GET_EXECUTION_PAYLOAD_BID_ENDPOINT_TAG).as_deref(), + Some(RELAY_API_KEY) + ); + + // Dialed, and the dial check refuses 0.0.0.0 + let to_mux = format!("http://0.0.0.0:{}", relays.mux_port); + assert_eq!(bid(None, to_mux.as_bytes()).await?.status(), StatusCode::BAD_REQUEST); + assert_eq!(relays.mux_state.received_execution_payload_bid(), 1); Ok(()) } @@ -274,7 +305,8 @@ async fn test_get_execution_payload_bid_rejected_before_relays() -> Result<()> { } /// A deadline that has already passed means there is no time to serve the -/// request: CB returns 204 rather than calling a relay it cannot beat. +/// request: CB returns 204 rather than calling a relay it cannot beat, or +/// looking up a dial target (`localhost` would be refused with 400). #[tokio::test] async fn test_get_execution_payload_bid_expired_deadline_204() -> Result<()> { let (mock_validator, mock_state) = @@ -285,18 +317,20 @@ async fn test_get_execution_payload_bid_expired_deadline_204() -> Result<()> { &B256::ZERO, &random_secret().public_key(), )?; - let res = mock_validator - .comm_boost - .client - .post(url) - .header(HEADER_START_TIME_UNIX_MS, utcnow_ms() - 5_000) - .header(HEADER_TIMEOUT_MS, 1_000u64) - .header("Eth-Consensus-Version", "gloas") - .header(CONTENT_TYPE, EncodingType::Ssz.content_type_header().clone()) - .body(opaque_auth(TEST_AUTH_DATA, TEST_SLOT).as_ssz_bytes()) - .send() - .await?; - assert_eq!(res.status(), StatusCode::NO_CONTENT); + for auth_data in [TEST_AUTH_DATA, b"localhost"] { + let res = mock_validator + .comm_boost + .client + .post(url.clone()) + .header(HEADER_START_TIME_UNIX_MS, utcnow_ms() - 5_000) + .header(HEADER_TIMEOUT_MS, 1_000u64) + .header("Eth-Consensus-Version", "gloas") + .header(CONTENT_TYPE, EncodingType::Ssz.content_type_header().clone()) + .body(opaque_auth(auth_data, TEST_SLOT).as_ssz_bytes()) + .send() + .await?; + assert_eq!(res.status(), StatusCode::NO_CONTENT); + } assert_eq!(mock_state.received_execution_payload_bid(), 0, "no relay call past the deadline"); Ok(()) } @@ -408,6 +442,22 @@ async fn test_get_execution_payload_bid_response_encoding() -> Result<()> { Ok(()) } +/// A beacon node that asks for the relay's encoding gets the relay's body byte +/// for byte; a re-encode would drop the pretty-printing +#[tokio::test] +async fn test_get_execution_payload_bid_json_passthrough() -> Result<()> { + let chain = Chain::Hoodi; + let relay = MockRelayState::new(chain, random_secret()) + .with_json_only_response() + .with_pretty_json_bid(); + let (mock_validator, states) = setup_relays(chain, vec![relay]).await?; + + let res = get_json_bid(&mock_validator, &opaque_auth(TEST_AUTH_DATA, TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::OK); + assert_eq!(Some(res.bytes().await?), states[0].served_bid()); + Ok(()) +} + /// An SSZ bid without `Eth-Consensus-Version` cannot be forwarded, since CB's /// 200 must name the fork: that relay contributes no bid, so the request is a /// 204. The relay is still contacted, so the 204 proves the drop. @@ -445,6 +495,21 @@ async fn test_get_execution_payload_bid_relay_status() -> Result<()> { Ok(()) } +/// An error body over PBS's 1 KiB read cap still reports the builder's status. +#[tokio::test] +async fn test_get_execution_payload_bid_large_error_body() -> Result<()> { + let chain = Chain::Hoodi; + let (mock_validator, states) = + setup_relays(chain, vec![MockRelayState::new(chain, random_secret()).with_large_body()]) + .await?; + for status in [StatusCode::BAD_REQUEST, StatusCode::UNAUTHORIZED] { + states[0].set_response_override(status); + let res = get_json_bid(&mock_validator, &opaque_auth(TEST_AUTH_DATA, TEST_SLOT)).await?; + assert_eq!(res.status(), status); + } + Ok(()) +} + /// A relay slower than the proposer's `X-Timeout-Ms` is dropped, so the request /// degrades to 204 rather than waiting the relay out. #[tokio::test] @@ -506,3 +571,22 @@ async fn test_get_execution_payload_bid_relay_timing_headers() -> Result<()> { assert!(0 < timeout_ms && timeout_ms <= 11_900, "relay saw X-Timeout-Ms {timeout_ms}"); Ok(()) } + +/// Auth data naming Commit-Boost's own URL is dialed once: the second hop, from +/// a Commit-Boost, answers 400 and the beacon node gets it +#[tokio::test] +async fn test_get_execution_payload_bid_dial_self_400() -> Result<()> { + // As if Commit-Boost's own address were public + cb_pbs::set_skip_dial_target_check(true); + let (mock_validator, cfg_state) = + setup_relay(Chain::Hoodi, |_| {}, generate_mock_relay).await?; + let own = &mock_validator.comm_boost.config.entry.url; + let own = format!("http://{}:{}/", own.host_str().unwrap(), own.port().unwrap()); + + let res = get_json_bid(&mock_validator, &opaque_auth(own.as_bytes(), TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + let body: serde_json::Value = serde_json::from_slice(&res.bytes().await?)?; + assert_eq!(body["message"], "The addressed builder rejected the request with status 400"); + assert_eq!(cfg_state.received_execution_payload_bid(), 0); + Ok(()) +} diff --git a/tests/tests/pbs_get_header_ws.rs b/tests/tests/pbs_get_header_ws.rs index e132a70d..abf9a7f4 100644 --- a/tests/tests/pbs_get_header_ws.rs +++ b/tests/tests/pbs_get_header_ws.rs @@ -19,7 +19,7 @@ use cb_tests::{ mock_validator::MockValidator, mock_ws_relay::{MockWsRelayState, start_mock_ws_relay_service}, utils::{ - API_KEY, generate_mock_relay, generate_mock_stream_relay, + RELAY_API_KEY, generate_mock_relay, generate_mock_stream_relay, generate_mock_stream_relay_with_timing_games, get_free_listener, get_pbs_config, setup_test_env, to_pbs_config, }, @@ -177,7 +177,7 @@ async fn test_get_header_ws_sends_api_key() -> Result<()> { assert_eq!(code, StatusCode::OK); let request = relay_state.last_request().expect("relay saw no request"); - assert_eq!(request.api_key.as_deref(), Some(API_KEY)); + assert_eq!(request.api_key.as_deref(), Some(RELAY_API_KEY)); Ok(()) } diff --git a/tests/tests/pbs_submit_builder_preferences.rs b/tests/tests/pbs_submit_builder_preferences.rs index b1079e7e..53a5208d 100644 --- a/tests/tests/pbs_submit_builder_preferences.rs +++ b/tests/tests/pbs_submit_builder_preferences.rs @@ -8,8 +8,8 @@ use cb_common::{ use cb_tests::{ mock_relay::MockRelayState, utils::{ - TEST_AUTH_DATA, generate_mock_relay, opaque_auth, setup_relay, setup_relays, - setup_relays_on_hosts, + TEST_AUTH_DATA, generate_mock_relay, opaque_auth, setup_mux_relays, setup_relay, + setup_relays, setup_relays_on_hosts, spawn_mock_relay, }, }; use eyre::Result; @@ -173,10 +173,8 @@ async fn test_submit_builder_preferences_past_slot_forwarded() -> Result<()> { Ok(()) } -/// Auth data matching no relay's hostname is a 400 with the builder's -/// data-mismatch message: unmatched means CB has no builder to proxy to, and -/// proposer-private preferences must never broadcast to builders the proposer -/// did not address. +/// Auth data that is neither a hostname nor a URL is a 400, and no builder is +/// asked: preferences go only to the builder the proposer addressed #[tokio::test] async fn test_submit_builder_preferences_unmatched_auth_data_400() -> Result<()> { let chain = Chain::Hoodi; @@ -270,3 +268,45 @@ async fn test_submit_builder_preferences_two_relays_addressed_one_only() -> Resu assert_eq!(states[1].received_builder_preferences(), 1, "the addressed builder is asked"); Ok(()) } + +/// Preferences whose auth data names a builder outside the config are sent +/// to it by a dial and accepted with its 202 +#[tokio::test] +async fn test_submit_builder_preferences_dial() -> Result<()> { + // The mock builder listens on an address the dial check refuses + cb_pbs::set_skip_dial_target_check(true); + let chain = Chain::Hoodi; + let (mock_validator, _) = setup_relay(chain, |_| {}, generate_mock_relay).await?; + let (dial_state, dial_port) = + spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + + let auth = opaque_auth(format!("http://0.0.0.0:{dial_port}/").as_bytes(), TEST_SLOT); + let request = preferences(auth, TEST_MAX_EXECUTION_PAYMENT); + let res = + mock_validator.do_submit_builder_preferences(None, &request, EncodingType::Ssz).await?; + assert_eq!(res.status(), StatusCode::ACCEPTED); + assert_eq!(dial_state.received_builder_preferences(), 1); + Ok(()) +} + +/// A key in a `[[mux]]` sends preferences to a builder configured only in +/// `[[relays]]` through that entry. A builder only in the mux is not reached so +/// by other keys: it is dialed, and the dial check refuses 0.0.0.0 +#[tokio::test] +async fn test_submit_builder_preferences_mux_key_reaches_a_relays_entry() -> Result<()> { + let relays = setup_mux_relays(Chain::Hoodi).await?; + let mock_validator = &relays.validator; + let (muxed_key, default_port, mux_port) = + (relays.muxed_key.clone(), relays.default_port, relays.mux_port); + let (default_state, mux_state) = (&relays.default_state, &relays.mux_state); + let submit = async |key, port| { + let auth = opaque_auth(format!("http://0.0.0.0:{port}").as_bytes(), TEST_SLOT); + let request = preferences(auth, TEST_MAX_EXECUTION_PAYMENT); + mock_validator.do_submit_builder_preferences(key, &request, EncodingType::Ssz).await + }; + assert_eq!(submit(Some(muxed_key), default_port).await?.status(), StatusCode::ACCEPTED); + assert_eq!(submit(None, mux_port).await?.status(), StatusCode::BAD_REQUEST); + assert_eq!(default_state.received_builder_preferences(), 1); + assert_eq!(mux_state.received_builder_preferences(), 0); + Ok(()) +} diff --git a/tests/tests/pbs_submit_signed_beacon_block.rs b/tests/tests/pbs_submit_signed_beacon_block.rs index a5e474a1..a9462db3 100644 --- a/tests/tests/pbs_submit_signed_beacon_block.rs +++ b/tests/tests/pbs_submit_signed_beacon_block.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use cb_common::{ pbs::SignedBeaconBlock, signer::random_secret, @@ -5,9 +7,14 @@ use cb_common::{ utils::TestRandomSeed, wire::{CONSENSUS_VERSION_HEADER, EncodingType}, }; +use cb_pbs::{DefaultBuilderApi, PbsService, PbsState}; use cb_tests::{ mock_relay::MockRelayState, - utils::{generate_mock_relay, setup_relay, setup_relays}, + mock_validator::MockValidator, + utils::{ + generate_mock_relay, get_free_listener, get_pbs_config, setup_relay, setup_relays, + spawn_mock_relay, to_pbs_config, wait_for_ready, + }, }; use eyre::Result; use lh_types::{MainnetEthSpec, SignedBeaconBlockGloas, Slot}; @@ -56,18 +63,44 @@ async fn test_submit_signed_beacon_block_broadcasts_to_all_relays() -> Result<() Ok(()) } -/// The block is broadcast; if every builder rejects it, PBS maps the all-reject -/// outcome to a 500 (no builder accepted). +/// A relay listed only in a `[[mux]]` gets the block too +#[tokio::test] +async fn test_submit_signed_beacon_block_reaches_mux_relays() -> Result<()> { + let chain = Chain::Hoodi; + let (default_state, default_port) = + spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + let (mux_state, mux_port) = + spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + let default_relay = generate_mock_relay(default_port, default_state.signer.public_key())?; + let mux_relay = generate_mock_relay(mux_port, mux_state.signer.public_key())?; + + let pbs_listener = get_free_listener().await; + let pbs_port = pbs_listener.local_addr()?.port(); + let mut config = to_pbs_config(chain, get_pbs_config(pbs_port), vec![default_relay.clone()]); + config.all_relays = vec![default_relay, mux_relay]; + let state = PbsState::new(config, PathBuf::new()); + tokio::spawn(PbsService::run_with_listener::<(), DefaultBuilderApi>(state, pbs_listener)); + let mock_validator = MockValidator::new(pbs_port)?; + wait_for_ready(&mock_validator).await?; + + let block = gloas_block(TEST_SLOT); + let res = mock_validator.do_submit_signed_beacon_block(&block, EncodingType::Ssz).await?; + assert_eq!(res.status(), StatusCode::ACCEPTED); + assert_eq!(default_state.received_signed_beacon_block(), 1); + assert_eq!(mux_state.received_signed_beacon_block(), 1); + Ok(()) +} + +/// Every builder rejecting the block is still a 202: the beacon node gossips +/// the block anyway, so a builder's answer is not a failure. Each is asked. #[tokio::test] -async fn test_submit_signed_beacon_block_broadcast_all_reject_500() -> Result<()> { +async fn test_submit_signed_beacon_block_broadcast_all_reject_202() -> Result<()> { let chain = Chain::Hoodi; let (mock_validator, states) = setup_relays(chain, vec![ MockRelayState::new(chain, random_secret()), MockRelayState::new(chain, random_secret()), ]) .await?; - - // Make every builder reject the broadcast for state in &states { state.set_response_override(StatusCode::INTERNAL_SERVER_ERROR); } @@ -75,47 +108,48 @@ async fn test_submit_signed_beacon_block_broadcast_all_reject_500() -> Result<() let block = gloas_block(TEST_SLOT); let res = mock_validator.do_submit_signed_beacon_block(&block, EncodingType::Ssz).await?; - assert_eq!(res.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(res.status(), StatusCode::ACCEPTED); assert_eq!(states[0].received_signed_beacon_block(), 1, "every builder is asked"); assert_eq!(states[1].received_signed_beacon_block(), 1, "every builder is asked"); Ok(()) } -/// A broadcast where one builder accepts and the other rejects is still a 202: -/// one acceptance across the broadcast is success. This is the core of the -/// stateless broadcast model, distinct from the all-accept and all-reject -/// extremes the other tests cover. +/// The block reaches the builder undecoded, in the encoding it arrived in: +/// JSON, the builder-specs default when unlabelled, or SSZ #[tokio::test] -async fn test_submit_signed_beacon_block_broadcast_one_accepts_202() -> Result<()> { +async fn test_submit_signed_beacon_block_forwards_its_encoding() -> Result<()> { let chain = Chain::Hoodi; - let (mock_validator, states) = setup_relays(chain, vec![ - MockRelayState::new(chain, random_secret()), - MockRelayState::new(chain, random_secret()), - ]) - .await?; - - // Only the second builder accepts; the first rejects. PBS must still 202. - states[0].set_response_override(StatusCode::INTERNAL_SERVER_ERROR); - + let (mock_validator, state) = setup_relay(chain, |_| {}, generate_mock_relay).await?; let block = gloas_block(TEST_SLOT); - let res = mock_validator.do_submit_signed_beacon_block(&block, EncodingType::Ssz).await?; + let (json_body, ssz_body) = (serde_json::to_vec(&block)?, block.as_ssz_bytes()); + let url = mock_validator.comm_boost.submit_signed_beacon_block_url()?; - assert_eq!( - res.status(), - StatusCode::ACCEPTED, - "one accepting builder makes the broadcast a success" - ); - assert_eq!(states[0].received_signed_beacon_block(), 1, "every builder is asked"); - assert_eq!(states[1].received_signed_beacon_block(), 1, "every builder is asked"); + // (Content-Type sent, body, Content-Type the builder gets) + let cases = [ + // Neighbours differ, so each case's own assertions can fail + (None, &json_body, "application/json"), + (Some("application/octet-stream"), &ssz_body, "application/octet-stream"), + (Some("application/json"), &json_body, "application/json"), + ]; + for (sent, body, forwarded) in cases { + let mut req = mock_validator.comm_boost.client.post(url.clone()); + if let Some(content_type) = sent { + req = req.header(CONTENT_TYPE, content_type); + } + let res = req.header(CONSENSUS_VERSION_HEADER, "gloas").body(body.clone()).send().await?; + assert_eq!(res.status(), StatusCode::ACCEPTED, "{sent:?}"); + let (content_type, received) = state.received_block_request().expect("forwarded"); + assert_eq!(content_type.as_deref(), Some(forwarded), "{sent:?}"); + assert_eq!(&received, body, "{sent:?}: forwarded byte for byte"); + } + assert_eq!(state.received_signed_beacon_block(), 3); Ok(()) } /// Requests refused before anything is broadcast, each with an ErrorMessage /// body. `Eth-Consensus-Version` is required (builder-specs #165) and checked -/// first, and a missing body is named as such rather than as an undecodable -/// one. The reveal must be SSZ: a JSON reveal, and an unlabeled one -/// (builder-specs defaults it to JSON), is a 415 rather than forwarded -/// mislabeled to fail opaquely at the builder. +/// first, a missing body is named as such, and a type other than JSON or SSZ +/// is a 415. #[tokio::test] async fn test_submit_signed_beacon_block_rejected_before_broadcast() -> Result<()> { let chain = Chain::Hoodi; @@ -134,14 +168,7 @@ async fn test_submit_signed_beacon_block_rejected_before_broadcast() -> Result<( "missing consensus version", ), (None, Some("gloas"), vec![], StatusCode::BAD_REQUEST, "missing request body"), - ( - Some("application/json"), - Some("gloas"), - json_body, - StatusCode::UNSUPPORTED_MEDIA_TYPE, - "", - ), - (None, Some("gloas"), ssz_body, StatusCode::UNSUPPORTED_MEDIA_TYPE, ""), + (Some("text/plain"), Some("gloas"), ssz_body, StatusCode::UNSUPPORTED_MEDIA_TYPE, ""), ]; for (content_type, version, body, status, message) in cases { let mut req = mock_validator.comm_boost.client.post(url.clone());