feat(pbs): a switch to turn off dialing, and a 20 ms ePBS buffer default

config.example.toml

@@ -31,9 +31,14 @@ wait_all_registrations = true
 # this should be lower than that, leaving some margin for overhead
 # OPTIONAL, DEFAULT: 950
 timeout_get_header_ms = 950
-# (unreleased, from v0.12.0-rc1) ePBS bids only: ms kept back from the beacon node's X-Timeout-Ms deadline, must be above 0 and under one slot, 12000 on mainnet (https://commit-boost.github.io/commit-boost-client/get_started/epbs#timing)
-# OPTIONAL, DEFAULT: 50
-# proposer_deadline_buffer_ms = 50
+# (unreleased, from v0.12.0-rc1) ePBS bids only: ms kept back from the beacon node's X-Timeout-Ms deadline for
+# Commit-Boost's answer to reach the beacon node. The default suits a beacon node on the same host; raise it when they
+# run on different hosts. Must be above 0 and under one slot, 12000 on mainnet (https://commit-boost.github.io/commit-boost-client/get_started/epbs#timing)
+# OPTIONAL, DEFAULT: 20
+# proposer_deadline_buffer_ms = 20
+# (unreleased, from v0.12.0-rc1) ePBS only: whether to dial a builder the auth data names when none of the key's relay entries match it, else answer 400 (https://commit-boost.github.io/commit-boost-client/get_started/epbs#builders-outside-your-config)
+# OPTIONAL, DEFAULT: true
+# dial_unconfigured_builders = true
 # Timeout in milliseconds for the `submit_blinded_block` call to relays.
 # OPTIONAL, DEFAULT: 4000
 timeout_get_payload_ms = 4000

crates/common/src/config/pbs.rs

@@ -210,6 +210,10 @@ pub struct PbsConfig {
     /// asks the builder for `deadline - this`.
     #[serde(default = "default_u64::<PROPOSER_DEADLINE_BUFFER_MS>")]
     pub proposer_deadline_buffer_ms: u64,
+    /// ePBS only: whether to dial a builder the auth data names when no relay
+    /// entry matches it
+    #[serde(default = "default_bool::<true>")]
+    pub dial_unconfigured_builders: bool,
     /// Enable extra validation of get_header responses
     #[serde(default = "default_bool::<false>")]
     pub extra_validation_enabled: bool,

crates/common/src/config/signer.rs

@@ -477,6 +477,7 @@ mod tests {
                     min_bid_wei: Uint::<256, 4>::from(0),
                     late_in_slot_time_ms: 0,
                     proposer_deadline_buffer_ms: 0,
+                    dial_unconfigured_builders: true,
                     extra_validation_enabled: false,
                     rpc_url: None,
                     http_timeout_seconds: 30,

crates/common/src/pbs/constants.rs

@@ -46,10 +46,9 @@ impl DefaultTimeout {
 pub const LATE_IN_SLOT_TIME_MS: u64 = 2000;
 
 /// ePBS bid path: ms reserved before the proposer's own deadline
-/// (Date-Milliseconds + X-Timeout-Ms) for the winning bid's return trip to the
-/// beacon node and the beacon node's own selection/assembly. CB asks the
-/// builder for `proposer_deadline - this`.
-pub const PROPOSER_DEADLINE_BUFFER_MS: u64 = 50;
+/// (Date-Milliseconds + X-Timeout-Ms) for CB's answer to reach the beacon node.
+/// CB asks the builder for `proposer_deadline - this`.
+pub const PROPOSER_DEADLINE_BUFFER_MS: u64 = 20;
 
 // Maximum number of retries for validator registration request per relay
 pub const REGISTER_VALIDATOR_RETRY_LIMIT: u32 = 3;

crates/pbs/src/dial.rs

@@ -395,6 +395,7 @@ mod tests {
             let res = crate::utils::resolve_addressed_relay(
                 &[],
                 &[],
+                true,
                 b"localhost",
                 &Default::default(),
                 1000,

crates/pbs/src/routes/builder_preferences.rs

@@ -70,6 +70,7 @@ pub async fn submit_builder_preferences<S: BuilderApiState>(
     let (relay, timeout_ms) = resolve_addressed_relay(
         relays,
         state.default_relays(),
+        state.pbs_config().dial_unconfigured_builders,
         request.auth.message.data.as_ref(),
         &req_headers,
         pbs_config.timeout_register_validator_ms,

crates/pbs/src/routes/execution_payload_bid.rs

@@ -151,6 +151,7 @@ pub async fn get_execution_payload_bid<S: BuilderApiState>(
     let (relay, max_timeout_ms) = match resolve_addressed_relay(
         relays,
         state.default_relays(),
+        state.pbs_config().dial_unconfigured_builders,
         auth.message.data.as_ref(),
         &req_headers,
         max_timeout_ms,

crates/pbs/src/utils.rs

@@ -192,10 +192,12 @@ pub fn check_gas_limit(gas_limit: u64, parent_gas_limit: u64) -> bool {
 }
 
 /// The relay `auth_data` names among `relays`, then `default_relays`, else a
-/// dial to it, with what the dial's setup left of `timeout_ms`
+/// dial to it if `dial_unconfigured_builders`, with what the dial's setup left
+/// of `timeout_ms`
 pub(crate) async fn resolve_addressed_relay(
     relays: &[RelayClient],
     default_relays: &[RelayClient],
+    dial_unconfigured_builders: bool,
     auth_data: &[u8],
     req_headers: &HeaderMap,
     timeout_ms: u64,
@@ -211,6 +213,13 @@ pub(crate) async fn resolve_addressed_relay(
     {
         return Ok((relay.clone(), timeout_ms));
     }
+    if !dial_unconfigured_builders {
+        warn!(
+            auth_data = ?String::from_utf8_lossy(auth_data),
+            "auth data matches no configured relay and dial_unconfigured_builders is off, not dialing"
+        );
+        return Err(PbsClientError::AuthDataMismatch);
+    }
     // Every Commit-Boost dial carries this header, so a request dialed back into
     // a Commit-Boost, this one included, goes no further
     if req_headers.contains_key(HEADER_VERSION_KEY) {
@@ -309,7 +318,7 @@ mod tests {
         ];
         let from_cb = from_cb();
         let host = async |data: &[u8]| -> Option<String> {
-            resolve_addressed_relay(&relays, &[], data, &from_cb, 0)
+            resolve_addressed_relay(&relays, &[], true, data, &from_cb, 0)
                 .await
                 .ok()
                 .map(|(relay, _)| relay.config.entry.url.host_str().unwrap().to_string())
@@ -344,7 +353,7 @@ mod tests {
         ];
         let from_cb = from_cb();
         let url = async |data: &[u8]| {
-            resolve_addressed_relay(&own, &default_relays, data, &from_cb, 0)
+            resolve_addressed_relay(&own, &default_relays, true, data, &from_cb, 0)
                 .await
                 .ok()
                 .map(|(relay, _)| relay.config.entry.url.to_string())
@@ -368,7 +377,7 @@ mod tests {
         let resolve = async |host: &'static str, data: &[u8]| {
             let mut headers = HeaderMap::new();
             headers.insert(HOST, reqwest::header::HeaderValue::from_static(host));
-            resolve_addressed_relay(&[], &[], data, &headers, 1000)
+            resolve_addressed_relay(&[], &[], true, data, &headers, 1000)
                 .await
                 .map(|(relay, _)| relay.config.entry.url.to_string())
         };
@@ -388,10 +397,25 @@ mod tests {
     #[tokio::test]
     async fn resolve_dials_the_address_before_the_parameters() {
         let headers = HeaderMap::new();
-        let dial =
-            |timeout_ms| resolve_addressed_relay(&[], &[], b"1.1.1.1?ofac=1", &headers, timeout_ms);
+        let dial = |timeout_ms| {
+            resolve_addressed_relay(&[], &[], true, b"1.1.1.1?ofac=1", &headers, timeout_ms)
+        };
         let (relay, _) = dial(1000).await.unwrap();
         assert_eq!(relay.config.entry.url.as_str(), "https://1.1.1.1/");
         assert!(matches!(dial(0).await, Err(PbsClientError::NoBuilderResponse)));
     }
+
+    // Not from a Commit-Boost, so only the switch refuses the miss
+    #[tokio::test]
+    async fn resolve_without_dialing_refuses_a_miss() {
+        let default_relays = vec![test_relay("https://builder.example.com")];
+        let headers = HeaderMap::new();
+        let resolve = async |data: &[u8]| {
+            resolve_addressed_relay(&[], &default_relays, false, data, &headers, 1000)
+                .await
+                .map(|(relay, _)| relay.config.entry.url.to_string())
+        };
+        assert_eq!(resolve(b"builder.example.com").await.unwrap(), "https://builder.example.com/");
+        assert!(matches!(resolve(b"1.1.1.1").await, Err(PbsClientError::AuthDataMismatch)));
+    }
 }

docs/docs/get_started/epbs.md

@@ -39,7 +39,7 @@ To go through Commit-Boost, every entry's `url` is Commit-Boost's own URL, and i
 
 ### 1. Add the builders to Commit-Boost
 
-List each builder as a relay entry, as for PBS: in `[[relays]]`, or in the `[[mux.relays]]` of the mux that lists the proposer's key. ePBS adds one option, `[pbs] proposer_deadline_buffer_ms` (default `50`): the time kept back from the beacon node's deadline (see [Timing](#timing)). It must be above `0` and under one slot, `12000` on mainnet.
+List each builder as a relay entry, as for PBS: in `[[relays]]`, or in the `[[mux.relays]]` of the mux that lists the proposer's key. A key can still reach a builder you do not list, unless you set `[pbs] dial_unconfigured_builders = false` (see [Builders outside your config](#builders-outside-your-config)).
 
 ### 2. Point each validator key at Commit-Boost {#validator-builder-config}
 
@@ -82,13 +82,15 @@ Either form may end in `?` and form-encoded parameters for the builder, such as
 
 A key's builder config may name a builder you have not added as a relay entry. Commit-Boost dials that builder anyway: a hostname at `https://<hostname>` on the default port, and a URL at its scheme, host and port. It answers `400` instead when the host does not resolve, or resolves to any address that is not public unicast, such as a loopback, private, link-local or `100.64.0.0/10` address, so a builder on your own network has to be a relay entry. These requests connect directly, ignoring `HTTP_PROXY`, `HTTPS_PROXY` and `ALL_PROXY`, follow no redirects and do not carry your relay `headers`. A builder reached this way gets the signed block over gossip, unless it is also a relay entry in `[[relays]]` or a mux. Neither `[[relays]]` nor a mux's relays limit which builders a key can reach this way, and anyone who can reach Commit-Boost's port can make it dial a public host, so keep that port private. Commit-Boost looks up at most 32 of these hosts at once and answers further requests as if the builder had not responded, as it does when a lookup runs out of the request's time.
 
+To turn dialing off, set `dial_unconfigured_builders = false` in `[pbs]`. Commit-Boost then answers `400` instead of dialing.
+
 On a validator client whose builder URL is Commit-Boost, a key without builder config sends Commit-Boost's own hostname as its `auth_data`, so it gets no bids: unless a relay entry has that hostname, which then gets the request, Commit-Boost answers `400` without dialing. A client that sends Commit-Boost's full URL instead also gets `400`. Commit-Boost never dials for a request that came from another Commit-Boost, so chained Commit-Boosts route only through relay entries.
 
 ## Timing
 
-The builder specs require each bid request to carry `Date-Milliseconds` and `X-Timeout-Ms`, which together say how long the beacon node will wait for a bid. Commit-Boost subtracts your `proposer_deadline_buffer_ms` from the time remaining and sends the result to the builder as its `X-Timeout-Ms`, so the builder knows how long it has to answer. For a builder outside your config, the time its lookup took is subtracted too.
+The builder specs require each bid request to carry `Date-Milliseconds` and `X-Timeout-Ms`, which together say how long the beacon node will wait for a bid. Commit-Boost subtracts `[pbs] proposer_deadline_buffer_ms` (default `20`) from the time remaining and sends the result to the builder as its `X-Timeout-Ms`, so the builder knows how long it has to answer. For a builder outside your config, the time its lookup took is subtracted too.
 
-Think of `proposer_deadline_buffer_ms` as the slack you keep from the beacon node's remaining time: enough for the bid to get back from Commit-Boost to the beacon node and for the beacon node to process it. If no time is left, Commit-Boost does not ask the builder. With `0`, Commit-Boost refuses to start or reload: the builder gets the whole deadline, so a bid it sends at the deadline reaches the beacon node late.
+The buffer is the time Commit-Boost's answer needs to reach the beacon node, which discards answers that arrive after its deadline. Commit-Boost's own share is a few milliseconds, so the default suits a beacon node on the same host. When Commit-Boost and the beacon node run on different hosts, raise it by the time a response takes to travel from Commit-Boost to the beacon node, plus any difference between the two hosts' clocks, since Commit-Boost reads the beacon node's `Date-Milliseconds` against its own clock. It must be above `0` and under one slot, `12000` on mainnet. If no time is left, Commit-Boost does not ask the builder. With `0`, Commit-Boost refuses to start or reload: the builder gets the whole deadline, so a bid it sends at the deadline reaches the beacon node late.
 
 Builder preferences use `timeout_register_validator_ms`, and the signed block `timeout_get_payload_ms`.
 
@@ -120,10 +122,10 @@ With [metrics](./running/metrics.md) enabled, the ePBS endpoints use the `endpoi
 
 | Symptom | Cause | Fix |
 |---|---|---|
-| Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data is neither a lowercase hostname without a port nor an `http(s)` URL, names the host the request was sent to (a key without builder config), or the request came from another Commit-Boost and matches none of this one's relay entries | Correct the `auth_data` in the key's [builder config](#validator-builder-config), or add the builder as a relay entry on the Commit-Boost the request reaches |
+| Bid requests get `400` "auth.message.data does not match any configured builder" | The auth data is neither a lowercase hostname without a port nor an `http(s)` URL, names the host the request was sent to (a key without builder config), the request came from another Commit-Boost and matches none of this one's relay entries, or `dial_unconfigured_builders` is `false` | Correct the `auth_data` in the key's [builder config](#validator-builder-config), or add the builder as a relay entry for the key on the Commit-Boost the request reaches |
 | Bid requests get `400` "the addressed builder's host does not resolve or resolves to a disallowed address" | The auth data names a builder outside your config whose host does not resolve or resolves to an [internal address](#builders-outside-your-config) | Add the builder as a relay entry for the key, or write or correct the key's [builder config](#validator-builder-config) |
 | A builder answers bid requests with `400` (`cb_pbs_relay_status_code_total{endpoint="get_execution_payload_bid",http_status_code="400"}`) | The builder compares auth data byte for byte and expects something other than what the key sends, such as its hostname without `?` parameters | Send the auth data the builder expects: its hostname or its URL, the forms Commit-Boost [routes by](#routing-by-auth-data), with `?` parameters only if it accepts them |
 | Bid requests to a streaming builder log "stream failed" with "rejected with 404", and `cb_pbs_relay_epbs_stream_fallback_total` rises | The builder does not serve the ePBS bid stream. Commit-Boost asks it over HTTP instead | Set `get_header = "http"` on its relay entry |
-| Commit-Boost does not start or reload, or `commit-boost init` fails, with "proposer_deadline_buffer_ms must be greater than 0 and less than one slot" | `proposer_deadline_buffer_ms` is `0`, or one slot or more | Set it above `0` and under one slot, or remove it to use the default `50` |
+| Commit-Boost does not start or reload, or `commit-boost init` fails, with "proposer_deadline_buffer_ms must be greater than 0 and less than one slot" | `proposer_deadline_buffer_ms` is `0`, or one slot or more | Set it above `0` and under one slot, or remove it to use the default `20` |
 | Bid requests to a streaming builder log "stream failed" with "websocket timed out" or a connection error | Commit-Boost could not open the stream, for example because your network allows only proxied connections, which the stream does not use. A connection error with time left is retried over HTTP; a handshake that times out is no bid | Set `get_header = "http"` on its relay entry, or allow direct connections to it |
 | Commit-Boost returns `200` but the beacon node builds locally | The beacon node rejected the bid, or valued its local block higher after `builder_boost_factor` | Compare the bid with the key's `min_bid` and `builder_boost_factor` |

tests/src/utils.rs

@@ -139,6 +139,7 @@ pub fn get_pbs_config(port: u16) -> PbsConfig {
         min_bid_wei: U256::ZERO,
         late_in_slot_time_ms: u64::MAX,
         proposer_deadline_buffer_ms: 50,
+        dial_unconfigured_builders: true,
         extra_validation_enabled: false,
 
         ssv_node_api_url: Url::parse("http://localhost:0").unwrap(),

tests/tests/pbs_cfg_file_update.rs

@@ -67,6 +67,7 @@ async fn test_cfg_file_update() -> Result<()> {
         late_in_slot_time_ms: u64::MAX / 2, /* serde gets very upset about serializing u64::MAX
                                              * or anything close to it */
         proposer_deadline_buffer_ms: 50,
+        dial_unconfigured_builders: true,
         extra_validation_enabled: false,
         rpc_url: None,
         ssv_node_api_url: Url::parse("http://example.com").unwrap(),

tests/tests/pbs_get_execution_payload_bid.rs

@@ -188,6 +188,35 @@ async fn test_get_execution_payload_bid_dial() -> Result<()> {
     Ok(())
 }
 
+/// With `dial_unconfigured_builders` off, auth data that matches no relay entry
+/// gets a 400 and no builder is asked, while a configured relay still serves
+#[tokio::test]
+async fn test_get_execution_payload_bid_dial_off_400() -> Result<()> {
+    // So only the switch, not the address check, can stop the dial
+    cb_pbs::set_skip_dial_target_check(true);
+    let chain = Chain::Hoodi;
+    let (mock_validator, cfg_state) =
+        setup_relay(chain, |cfg| cfg.dial_unconfigured_builders = false, generate_mock_relay)
+            .await?;
+    let (dial_state, dial_port) =
+        spawn_mock_relay(MockRelayState::new(chain, random_secret())).await?;
+    let dial_auth_data = format!("http://0.0.0.0:{dial_port}/").into_bytes();
+
+    let res = get_json_bid(&mock_validator, &opaque_auth(&dial_auth_data, TEST_SLOT)).await?;
+    assert_eq!(res.status(), StatusCode::BAD_REQUEST);
+    let body: serde_json::Value = serde_json::from_slice(&res.bytes().await?)?;
+    assert_eq!(
+        body["message"],
+        "Invalid SignedBuilderRequestAuth: auth.message.data does not match any configured builder"
+    );
+    assert_eq!(dial_state.received_execution_payload_bid(), 0);
+
+    let res = get_json_bid(&mock_validator, &opaque_auth(TEST_AUTH_DATA, TEST_SLOT)).await?;
+    assert_eq!(res.status(), StatusCode::OK);
+    assert_eq!(cfg_state.received_execution_payload_bid(), 1);
+    Ok(())
+}
+
 /// A key in a `[[mux]]` is served by that mux's relays, and reaches a relay
 /// configured only in `[[relays]]` through its entry, its API key included. A
 /// relay only in a mux is not reached so by other keys: it is dialed instead