diff --git a/Cargo.lock b/Cargo.lock index 32e64a92..55888f4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6143,9 +6143,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", @@ -7002,7 +7002,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", diff --git a/config.example.toml b/config.example.toml index 430050a1..b011f631 100644 --- a/config.example.toml +++ b/config.example.toml @@ -31,9 +31,14 @@ wait_all_registrations = true # this should be lower than that, leaving some margin for overhead # OPTIONAL, DEFAULT: 950 timeout_get_header_ms = 950 -# (unreleased, from v0.12.0-rc1) ePBS bids only: ms kept back from the beacon node's X-Timeout-Ms deadline, must be above 0 and under one slot, 12000 on mainnet (https://commit-boost.github.io/commit-boost-client/get_started/epbs#timing) -# OPTIONAL, DEFAULT: 50 -# proposer_deadline_buffer_ms = 50 +# (unreleased, from v0.12.0-rc1) ePBS bids only: ms kept back from the beacon node's X-Timeout-Ms deadline for +# Commit-Boost's answer to reach the beacon node. The default suits a beacon node on the same host; raise it when they +# run on different hosts. Must be above 0 and under one slot, 12000 on mainnet (https://commit-boost.github.io/commit-boost-client/get_started/epbs#timing) +# OPTIONAL, DEFAULT: 20 +# proposer_deadline_buffer_ms = 20 +# (unreleased, from v0.12.0-rc1) ePBS only: whether to dial a builder the auth data names when none of the key's relay entries match it, else answer 400 (https://commit-boost.github.io/commit-boost-client/get_started/epbs#builders-outside-your-config) +# OPTIONAL, DEFAULT: true +# dial_unconfigured_builders = true # Timeout in milliseconds for the `submit_blinded_block` call to relays. # OPTIONAL, DEFAULT: 4000 timeout_get_payload_ms = 4000 diff --git a/crates/common/src/config/pbs.rs b/crates/common/src/config/pbs.rs index 294c5b53..3e8fe01c 100644 --- a/crates/common/src/config/pbs.rs +++ b/crates/common/src/config/pbs.rs @@ -210,6 +210,10 @@ pub struct PbsConfig { /// asks the builder for `deadline - this`. #[serde(default = "default_u64::")] pub proposer_deadline_buffer_ms: u64, + /// ePBS only: whether to dial a builder the auth data names when no relay + /// entry matches it + #[serde(default = "default_bool::")] + pub dial_unconfigured_builders: bool, /// Enable extra validation of get_header responses #[serde(default = "default_bool::")] pub extra_validation_enabled: bool, diff --git a/crates/common/src/config/signer.rs b/crates/common/src/config/signer.rs index 6d948adc..8384e377 100644 --- a/crates/common/src/config/signer.rs +++ b/crates/common/src/config/signer.rs @@ -477,6 +477,7 @@ mod tests { min_bid_wei: Uint::<256, 4>::from(0), late_in_slot_time_ms: 0, proposer_deadline_buffer_ms: 0, + dial_unconfigured_builders: true, extra_validation_enabled: false, rpc_url: None, http_timeout_seconds: 30, diff --git a/crates/common/src/pbs/constants.rs b/crates/common/src/pbs/constants.rs index 111d77e9..52b99ec4 100644 --- a/crates/common/src/pbs/constants.rs +++ b/crates/common/src/pbs/constants.rs @@ -46,10 +46,9 @@ impl DefaultTimeout { pub const LATE_IN_SLOT_TIME_MS: u64 = 2000; /// ePBS bid path: ms reserved before the proposer's own deadline -/// (Date-Milliseconds + X-Timeout-Ms) for the winning bid's return trip to the -/// beacon node and the beacon node's own selection/assembly. CB asks the -/// builder for `proposer_deadline - this`. -pub const PROPOSER_DEADLINE_BUFFER_MS: u64 = 50; +/// (Date-Milliseconds + X-Timeout-Ms) for CB's answer to reach the beacon node. +/// CB asks the builder for `proposer_deadline - this`. +pub const PROPOSER_DEADLINE_BUFFER_MS: u64 = 20; // Maximum number of retries for validator registration request per relay pub const REGISTER_VALIDATOR_RETRY_LIMIT: u32 = 3; diff --git a/crates/pbs/src/dial.rs b/crates/pbs/src/dial.rs index 55aac192..11aed762 100644 --- a/crates/pbs/src/dial.rs +++ b/crates/pbs/src/dial.rs @@ -395,6 +395,7 @@ mod tests { let res = crate::utils::resolve_addressed_relay( &[], &[], + true, b"localhost", &Default::default(), 1000, diff --git a/crates/pbs/src/routes/builder_preferences.rs b/crates/pbs/src/routes/builder_preferences.rs index cdbeb4e4..8ed0a561 100644 --- a/crates/pbs/src/routes/builder_preferences.rs +++ b/crates/pbs/src/routes/builder_preferences.rs @@ -70,6 +70,7 @@ pub async fn submit_builder_preferences( let (relay, timeout_ms) = resolve_addressed_relay( relays, state.default_relays(), + state.pbs_config().dial_unconfigured_builders, request.auth.message.data.as_ref(), &req_headers, pbs_config.timeout_register_validator_ms, diff --git a/crates/pbs/src/routes/execution_payload_bid.rs b/crates/pbs/src/routes/execution_payload_bid.rs index 444ec5b8..bf6d7411 100644 --- a/crates/pbs/src/routes/execution_payload_bid.rs +++ b/crates/pbs/src/routes/execution_payload_bid.rs @@ -151,6 +151,7 @@ pub async fn get_execution_payload_bid( let (relay, max_timeout_ms) = match resolve_addressed_relay( relays, state.default_relays(), + state.pbs_config().dial_unconfigured_builders, auth.message.data.as_ref(), &req_headers, max_timeout_ms, diff --git a/crates/pbs/src/utils.rs b/crates/pbs/src/utils.rs index cd416f33..4d78f5c7 100644 --- a/crates/pbs/src/utils.rs +++ b/crates/pbs/src/utils.rs @@ -192,10 +192,12 @@ pub fn check_gas_limit(gas_limit: u64, parent_gas_limit: u64) -> bool { } /// The relay `auth_data` names among `relays`, then `default_relays`, else a -/// dial to it, with what the dial's setup left of `timeout_ms` +/// dial to it if `dial_unconfigured_builders`, with what the dial's setup left +/// of `timeout_ms` pub(crate) async fn resolve_addressed_relay( relays: &[RelayClient], default_relays: &[RelayClient], + dial_unconfigured_builders: bool, auth_data: &[u8], req_headers: &HeaderMap, timeout_ms: u64, @@ -211,6 +213,13 @@ pub(crate) async fn resolve_addressed_relay( { return Ok((relay.clone(), timeout_ms)); } + if !dial_unconfigured_builders { + warn!( + auth_data = ?String::from_utf8_lossy(auth_data), + "auth data matches no configured relay and dial_unconfigured_builders is off, not dialing" + ); + return Err(PbsClientError::AuthDataMismatch); + } // Every Commit-Boost dial carries this header, so a request dialed back into // a Commit-Boost, this one included, goes no further if req_headers.contains_key(HEADER_VERSION_KEY) { @@ -309,7 +318,7 @@ mod tests { ]; let from_cb = from_cb(); let host = async |data: &[u8]| -> Option { - resolve_addressed_relay(&relays, &[], data, &from_cb, 0) + resolve_addressed_relay(&relays, &[], true, data, &from_cb, 0) .await .ok() .map(|(relay, _)| relay.config.entry.url.host_str().unwrap().to_string()) @@ -344,7 +353,7 @@ mod tests { ]; let from_cb = from_cb(); let url = async |data: &[u8]| { - resolve_addressed_relay(&own, &default_relays, data, &from_cb, 0) + resolve_addressed_relay(&own, &default_relays, true, data, &from_cb, 0) .await .ok() .map(|(relay, _)| relay.config.entry.url.to_string()) @@ -368,7 +377,7 @@ mod tests { let resolve = async |host: &'static str, data: &[u8]| { let mut headers = HeaderMap::new(); headers.insert(HOST, reqwest::header::HeaderValue::from_static(host)); - resolve_addressed_relay(&[], &[], data, &headers, 1000) + resolve_addressed_relay(&[], &[], true, data, &headers, 1000) .await .map(|(relay, _)| relay.config.entry.url.to_string()) }; @@ -388,10 +397,25 @@ mod tests { #[tokio::test] async fn resolve_dials_the_address_before_the_parameters() { let headers = HeaderMap::new(); - let dial = - |timeout_ms| resolve_addressed_relay(&[], &[], b"1.1.1.1?ofac=1", &headers, timeout_ms); + let dial = |timeout_ms| { + resolve_addressed_relay(&[], &[], true, b"1.1.1.1?ofac=1", &headers, timeout_ms) + }; let (relay, _) = dial(1000).await.unwrap(); assert_eq!(relay.config.entry.url.as_str(), "https://1.1.1.1/"); assert!(matches!(dial(0).await, Err(PbsClientError::NoBuilderResponse))); } + + // Not from a Commit-Boost, so only the switch refuses the miss + #[tokio::test] + async fn resolve_without_dialing_refuses_a_miss() { + let default_relays = vec![test_relay("https://builder.example.com")]; + let headers = HeaderMap::new(); + let resolve = async |data: &[u8]| { + resolve_addressed_relay(&[], &default_relays, false, data, &headers, 1000) + .await + .map(|(relay, _)| relay.config.entry.url.to_string()) + }; + assert_eq!(resolve(b"builder.example.com").await.unwrap(), "https://builder.example.com/"); + assert!(matches!(resolve(b"1.1.1.1").await, Err(PbsClientError::AuthDataMismatch))); + } } diff --git a/docs/docs/get_started/epbs.md b/docs/docs/get_started/epbs.md index 92ca6ac5..6ce6e372 100644 --- a/docs/docs/get_started/epbs.md +++ b/docs/docs/get_started/epbs.md @@ -39,7 +39,7 @@ To go through Commit-Boost, every entry's `url` is Commit-Boost's own URL, and i ### 1. Add the builders to Commit-Boost -List each builder as a relay entry, as for PBS: in `[[relays]]`, or in the `[[mux.relays]]` of the mux that lists the proposer's key. ePBS adds one option, `[pbs] proposer_deadline_buffer_ms` (default `50`): the time kept back from the beacon node's deadline (see [Timing](#timing)). It must be above `0` and under one slot, `12000` on mainnet. +List each builder as a relay entry, as for PBS: in `[[relays]]`, or in the `[[mux.relays]]` of the mux that lists the proposer's key. A key can still reach a builder you do not list, unless you set `[pbs] dial_unconfigured_builders = false` (see [Builders outside your config](#builders-outside-your-config)). ### 2. Point each validator key at Commit-Boost {#validator-builder-config} @@ -82,13 +82,15 @@ Either form may end in `?` and form-encoded parameters for the builder, such as A key's builder config may name a builder you have not added as a relay entry. Commit-Boost dials that builder anyway: a hostname at `https://` on the default port, and a URL at its scheme, host and port. It answers `400` instead when the host does not resolve, or resolves to any address that is not public unicast, such as a loopback, private, link-local or `100.64.0.0/10` address, so a builder on your own network has to be a relay entry. These requests connect directly, ignoring `HTTP_PROXY`, `HTTPS_PROXY` and `ALL_PROXY`, follow no redirects and do not carry your relay `headers`. A builder reached this way gets the signed block over gossip, unless it is also a relay entry in `[[relays]]` or a mux. Neither `[[relays]]` nor a mux's relays limit which builders a key can reach this way, and anyone who can reach Commit-Boost's port can make it dial a public host, so keep that port private. Commit-Boost looks up at most 32 of these hosts at once and answers further requests as if the builder had not responded, as it does when a lookup runs out of the request's time. +To turn dialing off, set `dial_unconfigured_builders = false` in `[pbs]`. Commit-Boost then answers `400` instead of dialing. + On a validator client whose builder URL is Commit-Boost, a key without builder config sends Commit-Boost's own hostname as its `auth_data`, so it gets no bids: unless a relay entry has that hostname, which then gets the request, Commit-Boost answers `400` without dialing. A client that sends Commit-Boost's full URL instead also gets `400`. Commit-Boost never dials for a request that came from another Commit-Boost, so chained Commit-Boosts route only through relay entries. ## Timing -The builder specs require each bid request to carry `Date-Milliseconds` and `X-Timeout-Ms`, which together say how long the beacon node will wait for a bid. Commit-Boost subtracts your `proposer_deadline_buffer_ms` from the time remaining and sends the result to the builder as its `X-Timeout-Ms`, so the builder knows how long it has to answer. For a builder outside your config, the time its lookup took is subtracted too. +The builder specs require each bid request to carry `Date-Milliseconds` and `X-Timeout-Ms`, which together say how long the beacon node will wait for a bid. Commit-Boost subtracts `[pbs] proposer_deadline_buffer_ms` (default `20`) from the time remaining and sends the result to the builder as its `X-Timeout-Ms`, so the builder knows how long it has to answer. For a builder outside your config, the time its lookup took is subtracted too. -Think of `proposer_deadline_buffer_ms` as the slack you keep from the beacon node's remaining time: enough for the bid to get back from Commit-Boost to the beacon node and for the beacon node to process it. If no time is left, Commit-Boost does not ask the builder. With `0`, Commit-Boost refuses to start or reload: the builder gets the whole deadline, so a bid it sends at the deadline reaches the beacon node late. +The buffer is the time Commit-Boost's answer needs to reach the beacon node, which discards answers that arrive after its deadline. Commit-Boost's own share is a few milliseconds, so the default suits a beacon node on the same host. When Commit-Boost and the beacon node run on different hosts, raise it by the time a response takes to travel from Commit-Boost to the beacon node, plus any difference between the two hosts' clocks, since Commit-Boost reads the beacon node's `Date-Milliseconds` against its own clock. It must be above `0` and under one slot, `12000` on mainnet. If no time is left, Commit-Boost does not ask the builder. With `0`, Commit-Boost refuses to start or reload: the builder gets the whole deadline, so a bid it sends at the deadline reaches the beacon node late. Builder preferences use `timeout_register_validator_ms`, and the signed block `timeout_get_payload_ms`. @@ -120,10 +122,10 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi | Symptom | Cause | Fix | |---|---|---| -| Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data is neither a lowercase hostname without a port nor an `http(s)` URL, names the host the request was sent to (a key without builder config), or the request came from another Commit-Boost and matches none of this one's relay entries | Correct the `auth_data` in the key's [builder config](#validator-builder-config), or add the builder as a relay entry on the Commit-Boost the request reaches | +| Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data is neither a lowercase hostname without a port nor an `http(s)` URL, names the host the request was sent to (a key without builder config), the request came from another Commit-Boost and matches none of this one's relay entries, or `dial_unconfigured_builders` is `false` | Correct the `auth_data` in the key's [builder config](#validator-builder-config), or add the builder as a relay entry for the key on the Commit-Boost the request reaches | | Bid requests get `400` "the addressed builder's host does not resolve or resolves to a disallowed address" | The auth data names a builder outside your config whose host does not resolve or resolves to an [internal address](#builders-outside-your-config) | Add the builder as a relay entry for the key, or write or correct the key's [builder config](#validator-builder-config) | | A builder answers bid requests with `400` (`cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid",http_status_code="400"}`) | The builder compares auth data byte for byte and expects something other than what the key sends, such as its hostname without `?` parameters | Send the auth data the builder expects: its hostname or its URL, the forms Commit-Boost [routes by](#routing-by-auth-data), with `?` parameters only if it accepts them | | Bid requests to a streaming builder log "stream failed" with "rejected with 404", and `cb_pbs_relay_epbs_stream_fallback_total` rises | The builder does not serve the ePBS bid stream. Commit-Boost asks it over HTTP instead | Set `get_header = "http"` on its relay entry | -| Commit-Boost does not start or reload, or `commit-boost init` fails, with "proposer_deadline_buffer_ms must be greater than 0 and less than one slot" | `proposer_deadline_buffer_ms` is `0`, or one slot or more | Set it above `0` and under one slot, or remove it to use the default `50` | +| Commit-Boost does not start or reload, or `commit-boost init` fails, with "proposer_deadline_buffer_ms must be greater than 0 and less than one slot" | `proposer_deadline_buffer_ms` is `0`, or one slot or more | Set it above `0` and under one slot, or remove it to use the default `20` | | Bid requests to a streaming builder log "stream failed" with "websocket timed out" or a connection error | Commit-Boost could not open the stream, for example because your network allows only proxied connections, which the stream does not use. A connection error with time left is retried over HTTP; a handshake that times out is no bid | Set `get_header = "http"` on its relay entry, or allow direct connections to it | | Commit-Boost returns `200` but the beacon node builds locally | The beacon node rejected the bid, or valued its local block higher after `builder_boost_factor` | Compare the bid with the key's `min_bid` and `builder_boost_factor` | diff --git a/tests/src/utils.rs b/tests/src/utils.rs index 09412d2a..664c6e2b 100644 --- a/tests/src/utils.rs +++ b/tests/src/utils.rs @@ -139,6 +139,7 @@ pub fn get_pbs_config(port: u16) -> PbsConfig { min_bid_wei: U256::ZERO, late_in_slot_time_ms: u64::MAX, proposer_deadline_buffer_ms: 50, + dial_unconfigured_builders: true, extra_validation_enabled: false, ssv_node_api_url: Url::parse("http://localhost:0").unwrap(), diff --git a/tests/tests/pbs_cfg_file_update.rs b/tests/tests/pbs_cfg_file_update.rs index c4c02ce1..46b5d124 100644 --- a/tests/tests/pbs_cfg_file_update.rs +++ b/tests/tests/pbs_cfg_file_update.rs @@ -67,6 +67,7 @@ async fn test_cfg_file_update() -> Result<()> { late_in_slot_time_ms: u64::MAX / 2, /* serde gets very upset about serializing u64::MAX * or anything close to it */ proposer_deadline_buffer_ms: 50, + dial_unconfigured_builders: true, extra_validation_enabled: false, rpc_url: None, ssv_node_api_url: Url::parse("http://example.com").unwrap(), diff --git a/tests/tests/pbs_get_execution_payload_bid.rs b/tests/tests/pbs_get_execution_payload_bid.rs index 79d04f7e..4c2cf2cd 100644 --- a/tests/tests/pbs_get_execution_payload_bid.rs +++ b/tests/tests/pbs_get_execution_payload_bid.rs @@ -188,6 +188,35 @@ async fn test_get_execution_payload_bid_dial() -> Result<()> { Ok(()) } +/// With `dial_unconfigured_builders` off, auth data that matches no relay entry +/// gets a 400 and no builder is asked, while a configured relay still serves +#[tokio::test] +async fn test_get_execution_payload_bid_dial_off_400() -> Result<()> { + // So only the switch, not the address check, can stop the dial + cb_pbs::set_skip_dial_target_check(true); + let chain = Chain::Hoodi; + let (mock_validator, cfg_state) = + setup_relay(chain, |cfg| cfg.dial_unconfigured_builders = false, generate_mock_relay) + .await?; + let (dial_state, dial_port) = + spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + let dial_auth_data = format!("http://0.0.0.0:{dial_port}/").into_bytes(); + + let res = get_json_bid(&mock_validator, &opaque_auth(&dial_auth_data, TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + let body: serde_json::Value = serde_json::from_slice(&res.bytes().await?)?; + assert_eq!( + body["message"], + "Invalid SignedBuilderRequestAuth: auth.message.data does not match any configured builder" + ); + assert_eq!(dial_state.received_execution_payload_bid(), 0); + + let res = get_json_bid(&mock_validator, &opaque_auth(TEST_AUTH_DATA, TEST_SLOT)).await?; + assert_eq!(res.status(), StatusCode::OK); + assert_eq!(cfg_state.received_execution_payload_bid(), 1); + Ok(()) +} + /// A key in a `[[mux]]` is served by that mux's relays, and reaches a relay /// configured only in `[[relays]]` through its entry, its API key included. A /// relay only in a mux is not reached so by other keys: it is dialed instead diff --git a/tests/tests/pbs_submit_builder_preferences.rs b/tests/tests/pbs_submit_builder_preferences.rs index 53a5208d..184465c9 100644 --- a/tests/tests/pbs_submit_builder_preferences.rs +++ b/tests/tests/pbs_submit_builder_preferences.rs @@ -289,6 +289,28 @@ async fn test_submit_builder_preferences_dial() -> Result<()> { Ok(()) } +/// With `dial_unconfigured_builders` off, preferences for a builder outside the +/// config get a 400 and are not sent +#[tokio::test] +async fn test_submit_builder_preferences_dial_off_400() -> Result<()> { + // So only the switch, not the address check, can stop the dial + cb_pbs::set_skip_dial_target_check(true); + let chain = Chain::Hoodi; + let (mock_validator, _) = + setup_relay(chain, |cfg| cfg.dial_unconfigured_builders = false, generate_mock_relay) + .await?; + let (dial_state, dial_port) = + spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?; + + let auth = opaque_auth(format!("http://0.0.0.0:{dial_port}/").as_bytes(), TEST_SLOT); + let request = preferences(auth, TEST_MAX_EXECUTION_PAYMENT); + let res = + mock_validator.do_submit_builder_preferences(None, &request, EncodingType::Ssz).await?; + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + assert_eq!(dial_state.received_builder_preferences(), 0); + Ok(()) +} + /// A key in a `[[mux]]` sends preferences to a builder configured only in /// `[[relays]]` through that entry. A builder only in the mux is not reached so /// by other keys: it is dialed, and the dial check refuses 0.0.0.0