From 565625279a0ee96472b61e6ad37f960082121e7c Mon Sep 17 00:00:00 2001 From: Tumas Date: Thu, 22 Oct 2026 16:31:51 +0300 Subject: [PATCH] Run the validator client against remote beacon nodes with the new vc and bn commands `grandine vc --beacon-node-urls ...` runs the validator client alone against remote beacon nodes, `grandine bn` runs the beacon node alone, and the bare command runs the beacon node with the local validator when one is configured. Duties come from exactly one source: the built-in node or the remote nodes. vc refuses options that only the built-in beacon node acts on. The validator no longer holds the fork choice controller. It works through a ChainSource, and every duty goes through the BeaconNodeApi trait, which the built-in node and the remote nodes both implement: attester, aggregation, sync committee, PTC, proposer, payload attestation, voluntary exit and doppelganger liveness duties, plus prepared proposers, validator registrations and proposer preferences. Attestation data returned by a remote node is validated for the reported index and target epoch before it is signed. Proposer preferences are published for the dependent root the duties are held under. Remote nodes are polled for health and ranked by it, then by the newest head; an unreachable node is kept as the last choice. Requests have per-endpoint timeouts, a head stream tracks reorgs, and a slot head fallback covers a node that falls behind. Duties are published to every node by default, returning once one accepts; the payload envelope goes to the node that produced the block. In remote mode the validator is driven by clock ticks directly, and genesis is agreed with the nodes before it starts. Signing no longer needs a beacon state: the signer takes the fork information and the current epoch, and slashing protection takes the epoch. Own validator indices are resolved from the finalized state, driven by finalization events and by changes to the signer's keys. The chain is identified by its genesis validators root without loading the state; vc and the interchange commands use the predefined network's root, read from the embedded mainnet state or a constant for the testnets. Doppelganger protection no longer reads the local beacon state or the liveness tracker directly: it takes validator indices and a liveness query from the caller, so it works against remote beacon nodes, where only synced nodes are asked and a validator seen live by any of them counts as live. The check runs late in the slot, at the aggregate tick, when every node has ticked into the epoch and seen the slot's attestations. A key is activated only after a successful check, a failed query postpones activation instead of panicking, and a key whose index is unknown is not activated until its index resolves and a full check window has passed from that point. A key removed during its check window starts a new one when it returns. A detected doppelganger stops the application through a typed error rather than a panic, so it is not restarted. --detect-doppelgangers needs --track-liveness only with the built-in beacon node. Sync committee messages no longer count towards validator liveness. Register the validator client with the beacon nodes' builder via --use-builder pre-Gloas, skipping validators without a fee recipient. Also: - Leave the peer counts out of validator client logs - Run data column recovery on its own executor - Accept a slot's worth of aggregates in one publish request - Keep peers on subscribed sync committee subnets until the subscription expires - Serve next epoch attester duties from the state at the current slot when the head lags, and accept next epoch duty requests sent as the clock enters a new epoch - Let the builder publish a blinded block from Fulu on instead of unblinding it - Reject and don't publish single attestations from validators outside the named committee - Limit concurrent Web3Signer requests for whole-epoch batches - Send the block event only after fork choice exposes the block's head - Keep finalized blocks in the store until they are written to storage - Fix the network dashboard --- Cargo.lock | 37 +- Cargo.toml | 1 + ad_hoc_bench/src/main.rs | 2 +- binary_utils/src/lib.rs | 94 +- bindings/c/src/lib.rs | 18 +- block_producer/src/block_producer.rs | 75 +- block_producer/src/lib.rs | 2 +- block_producer/src/misc.rs | 22 +- builder_api/src/gloas/builder_url.rs | 7 + builder_api/src/unphased/containers.rs | 2 +- clock/src/lib.rs | 7 +- directories/src/lib.rs | 13 +- doppelganger_protection/Cargo.toml | 2 - .../src/doppelganger_protection.rs | 435 ++- doppelganger_protection/src/lib.rs | 2 +- eth1_api/src/auth.rs | 2 +- fork_choice_control/Cargo.toml | 1 + fork_choice_control/src/controller.rs | 4 - fork_choice_control/src/extra_tests.rs | 54 +- fork_choice_control/src/helpers.rs | 62 +- fork_choice_control/src/messages.rs | 8 +- fork_choice_control/src/mutator.rs | 225 +- fork_choice_store/src/store.rs | 35 +- fork_choice_store/src/store_config.rs | 2 +- grandine-snapshot-tests | 2 +- helper_functions/src/accessors.rs | 79 +- helper_functions/src/predicates.rs | 25 +- helper_functions/src/signing.rs | 158 +- http_api/Cargo.toml | 5 +- http_api/src/context.rs | 74 +- http_api/src/lib.rs | 1 - http_api/src/misc.rs | 55 +- http_api/src/routing.rs | 9 +- http_api/src/standard.rs | 447 +-- http_api/src/test_endpoints.rs | 8 +- http_api_utils/Cargo.toml | 8 + http_api_utils/src/debug_tracing.rs | 88 + http_api_utils/src/error.rs | 49 +- http_api_utils/src/lib.rs | 13 +- http_api_utils/src/logging.rs | 18 +- http_api_utils/src/misc.rs | 70 + {http_api => http_api_utils}/src/response.rs | 78 +- http_api_utils/src/traits.rs | 4 +- keymanager/Cargo.toml | 2 +- keymanager/src/keystores.rs | 2 - keymanager/src/lib.rs | 32 +- keymanager/src/proposer_configs.rs | 105 +- keymanager/src/remote_keys.rs | 2 - keymanager/src/validator_definitions.rs | 6 + liveness_tracker/Cargo.toml | 1 - liveness_tracker/src/lib.rs | 34 - logging/src/lib.rs | 2 +- metrics/src/beaconchain.rs | 49 +- metrics/src/lib.rs | 2 +- metrics/src/service.rs | 127 +- .../src/blob_reconstruction_pool/manager.rs | 14 +- operation_pools/src/lib.rs | 2 +- operation_pools/src/messages.rs | 18 +- .../src/sync_committee_agg_pool/manager.rs | 7 +- .../src/sync_committee_agg_pool/tasks.rs | 13 +- p2p/src/misc.rs | 10 +- p2p/src/network.rs | 31 +- p2p/src/sync_committee_subnets.rs | 47 +- predefined_chains/src/lib.rs | 25 +- prometheus_metrics/dashboards/network.json | 16 +- prometheus_metrics/src/metrics.rs | 63 +- runtime/src/consts.rs | 5 - runtime/src/grandine_args.rs | 2257 +++++++---- runtime/src/grandine_config.rs | 394 +- runtime/src/lib.rs | 3 +- runtime/src/predefined_network.rs | 35 +- runtime/src/runtime.rs | 1950 ++++++---- runtime/src/validators.rs | 2 +- signer/src/signer.rs | 138 +- signer/src/types.rs | 26 +- signer/src/web3signer/api.rs | 4 +- signer/src/web3signer/types.rs | 4 +- slasher/src/slasher_config.rs | 2 +- slashing_protection/Cargo.toml | 2 - slashing_protection/src/lib.rs | 30 +- ssz_derive/src/ssz_type.rs | 2 +- types/src/combined.rs | 29 + types/src/config.rs | 169 +- types/src/nonstandard.rs | 51 +- types/src/preset.rs | 5 + validator/Cargo.toml | 11 +- validator/src/api.rs | 132 +- validator/src/beacon_node_api.rs | 262 ++ validator/src/beacon_nodes.rs | 1619 ++++++++ validator/src/chain_events.rs | 565 +++ validator/src/gossip_spec_tests.rs | 16 +- validator/src/health.rs | 118 + validator/src/lib.rs | 22 +- validator/src/local_beacon_node.rs | 1159 ++++++ validator/src/messages.rs | 17 +- validator/src/misc.rs | 312 +- validator/src/own_beacon_committee_members.rs | 771 +++- validator/src/own_proposer_duties.rs | 280 ++ validator/src/own_ptc_members.rs | 156 +- validator/src/own_sync_committee_members.rs | 331 ++ .../src/own_sync_committee_subscriptions.rs | 26 +- validator/src/own_validator_indices.rs | 143 + validator/src/remote_beacon_node.rs | 3048 +++++++++++++++ validator/src/remote_beacon_nodes.rs | 319 ++ validator/src/slot_head.rs | 107 +- validator/src/tasks.rs | 322 +- validator/src/validator.rs | 3395 ++++++++++------- validator/src/validator_config.rs | 7 +- 108 files changed, 16686 insertions(+), 4473 deletions(-) create mode 100644 http_api_utils/src/debug_tracing.rs rename {http_api => http_api_utils}/src/response.rs (87%) delete mode 100644 runtime/src/consts.rs create mode 100644 validator/src/beacon_node_api.rs create mode 100644 validator/src/beacon_nodes.rs create mode 100644 validator/src/chain_events.rs create mode 100644 validator/src/health.rs create mode 100644 validator/src/local_beacon_node.rs create mode 100644 validator/src/own_proposer_duties.rs create mode 100644 validator/src/own_sync_committee_members.rs create mode 100644 validator/src/own_validator_indices.rs create mode 100644 validator/src/remote_beacon_node.rs create mode 100644 validator/src/remote_beacon_nodes.rs diff --git a/Cargo.lock b/Cargo.lock index 30f95c798..448f7a84c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4871,9 +4871,7 @@ dependencies = [ "arc-swap", "bls", "factory", - "futures", "helper_functions", - "liveness_tracker", "logging", "pubkey_cache", "thiserror 2.0.18", @@ -6168,6 +6166,7 @@ dependencies = [ "genesis", "helper_functions", "http_api_utils", + "interop", "itertools 0.14.0", "logging", "mime", @@ -7304,7 +7303,6 @@ dependencies = [ "kzg_utils", "liveness_tracker", "logging", - "mediatype", "mime", "num_cpus", "once_cell", @@ -7351,17 +7349,25 @@ version = "0.0.0" dependencies = [ "anyhow", "axum 0.8.9", + "binary_utils", + "bls", "features", "hex-literal 1.1.0", "http 1.4.0", "http-body-util", "itertools 0.14.0", "logging", + "mediatype", "mime", "parse-display", "prometheus_metrics", + "serde", + "serde_utils", + "ssz", + "tap", "test-case", "thiserror 2.0.18", + "tokio", "tower 0.5.3", "tower-http", "tracing", @@ -8945,7 +8951,6 @@ dependencies = [ "helper_functions", "itertools 0.14.0", "logging", - "operation_pools", "prometheus_metrics", "tracing", "tynm", @@ -14725,14 +14730,12 @@ dependencies = [ "chrono", "derivative", "duplicate", - "factory", "fs-err", "glob", "helper_functions", "hex-literal 1.1.0", "itertools 0.14.0", "logging", - "pubkey_cache", "refinery", "rusqlite", "serde", @@ -15940,6 +15943,19 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "sse-stream" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c123f296ade4ec4b8b0f6162116e6629f5146922ca5ab40ca9d3c2e73ab4761e" +dependencies = [ + "bytes", + "futures-util", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", +] + [[package]] name = "ssz" version = "0.0.0" @@ -17721,8 +17737,11 @@ name = "validator" version = "0.0.0" dependencies = [ "anyhow", + "arc-swap", + "arithmetic", "axum 0.8.9", "axum-extra", + "binary_utils", "block_producer", "bls", "builder_api", @@ -17731,6 +17750,7 @@ dependencies = [ "debug_info", "dedicated_executor", "derivative", + "derive_more 2.1.1", "directories", "doppelganger_protection", "eip_7594", @@ -17746,19 +17766,21 @@ dependencies = [ "helper_functions", "hex", "hex-literal 1.1.0", + "http-body-util", "http_api_utils", + "httpmock", "interop", "itertools 0.14.0", "keymanager", "liveness_tracker", "logging", + "mime", "once_cell", "operation_pools", "p2p", "prometheus_metrics", "pubkey_cache", "rand 0.8.6", - "rayon", "reqwest 0.13.2", "scc 3.7.0", "serde", @@ -17767,6 +17789,7 @@ dependencies = [ "signer", "slasher", "slashing_protection", + "sse-stream", "ssz", "std_ext", "tap", diff --git a/Cargo.toml b/Cargo.toml index 7247fd7c2..2b6a33a37 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -434,6 +434,7 @@ snap = '1' sp1-helper = '6.1.0' sp1-sdk = '6.1.0' sp1-zkvm = { version = '6.1.0' } +sse-stream = '0.2' static_assertions = '1' strum = { version = '0.28', features = ['derive'] } syn = { version = '2', features = ['full'] } diff --git a/ad_hoc_bench/src/main.rs b/ad_hoc_bench/src/main.rs index 9dea5e19e..d239228c0 100644 --- a/ad_hoc_bench/src/main.rs +++ b/ad_hoc_bench/src/main.rs @@ -284,7 +284,7 @@ fn main() -> Result<()> { .tempdir()? .keep(); - binary_utils::initialize_tracing_logger(module_path!(), Some(&data_dir), None, false)?; + binary_utils::initialize_tracing_logger(module_path!(), Some(&data_dir), None, false, true)?; binary_utils::initialize_rayon()?; #[cfg(not(target_os = "windows"))] print_jemalloc_stats()?; diff --git a/binary_utils/src/lib.rs b/binary_utils/src/lib.rs index f8fdedaa6..03897c0bf 100644 --- a/binary_utils/src/lib.rs +++ b/binary_utils/src/lib.rs @@ -1,5 +1,6 @@ use anyhow::Result; use chrono::{Local, SecondsFormat}; +use core::{error::Error, fmt::Debug}; use fs_err as fs; use logging::{debug_with_peers, exception}; use logroller::{Compression, LogRollerBuilder, Rotation, RotationSize}; @@ -12,12 +13,20 @@ use std::{ sync::{Arc, OnceLock}, }; use tokio::runtime::Runtime; -use tracing::Level; +use tracing::{ + Level, + field::{Field, Visit}, +}; use tracing_appender::non_blocking::WorkerGuard; use tracing_subscriber::{ EnvFilter, Registry, + field::{MakeVisitor as _, RecordFields, VisitOutput as _}, filter::LevelFilter, - fmt::{self, format::Writer, time::FormatTime}, + fmt::{ + self, + format::{DefaultFields, DefaultVisitor, FormatFields, Writer}, + time::FormatTime, + }, prelude::*, reload::{self, Handle}, }; @@ -25,7 +34,7 @@ use types::redacting_url::RedactingUrl; static LOG_GUARD: OnceLock = OnceLock::new(); -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct TelemetryConfig { pub url: RedactingUrl, pub service_name: String, @@ -80,6 +89,7 @@ pub fn initialize_tracing_logger( data_dir: Option<&Path>, telemetry_config: Option, always_write_style: bool, + with_peers: bool, ) -> Result { let mut filter = EnvFilter::default() .add_directive(LevelFilter::OFF.into()) @@ -132,6 +142,7 @@ pub fn initialize_tracing_logger( let enable_ansi = always_write_style || io::stdout().is_terminal(); let stdout_layer = fmt::layer::() + .fmt_fields(Fields { with_peers }) .compact() .with_thread_ids(false) .with_target(true) @@ -256,6 +267,7 @@ pub fn initialize_tracing_logger( .add_directive("exception=error".parse()?); let file_layer = fmt::layer() + .fmt_fields(Fields { with_peers }) .compact() .with_ansi(false) .with_writer(non_blocking) @@ -313,6 +325,80 @@ pub fn initialize_rayon() -> Result<()> { .map_err(Into::into) } +/// The default field format, minus the peer counts of a process without a network. +struct Fields { + with_peers: bool, +} + +impl<'writer> FormatFields<'writer> for Fields { + fn format_fields( + &self, + writer: Writer<'writer>, + fields: R, + ) -> core::fmt::Result { + let mut visitor = DefaultFields::new().make_visitor(writer); + + if self.with_peers { + fields.record(&mut visitor); + } else { + fields.record(&mut WithoutPeers(&mut visitor)); + } + + visitor.finish() + } +} + +// The `*_with_peers!` macros always add a `peers` field, and `tracing` offers no way to drop a +// field short of the visitor that writes it. The validator client has no peers to report. +struct WithoutPeers<'visitor, 'writer>(&'visitor mut DefaultVisitor<'writer>); + +// Forwards every method, not only the ones `DefaultVisitor` overrides today, so a future +// override in `tracing-subscriber` is not silently bypassed here. +impl Visit for WithoutPeers<'_, '_> { + fn record_f64(&mut self, field: &Field, value: f64) { + self.0.record_f64(field, value); + } + + fn record_i64(&mut self, field: &Field, value: i64) { + self.0.record_i64(field, value); + } + + fn record_u64(&mut self, field: &Field, value: u64) { + self.0.record_u64(field, value); + } + + fn record_i128(&mut self, field: &Field, value: i128) { + self.0.record_i128(field, value); + } + + fn record_u128(&mut self, field: &Field, value: u128) { + self.0.record_u128(field, value); + } + + fn record_bool(&mut self, field: &Field, value: bool) { + self.0.record_bool(field, value); + } + + fn record_str(&mut self, field: &Field, value: &str) { + self.0.record_str(field, value); + } + + fn record_bytes(&mut self, field: &Field, value: &[u8]) { + self.0.record_bytes(field, value); + } + + fn record_error(&mut self, field: &Field, value: &(dyn Error + 'static)) { + self.0.record_error(field, value); + } + + fn record_debug(&mut self, field: &Field, value: &dyn Debug) { + // The macros record the counts through `Display`, which arrives here. + if field.name() != "peers" { + self.0.record_debug(field, value); + } + } +} + #[cfg(test)] mod tests { use std::sync::{LazyLock, Mutex}; @@ -334,7 +420,7 @@ mod tests { let mut lock = LOGGER.lock().expect("Failed to acquire LOGGER mutex lock"); if lock.is_none() { let handle = - initialize_tracing_logger(module_path!(), Some(data_dir.path()), None, false) + initialize_tracing_logger(module_path!(), Some(data_dir.path()), None, false, true) .expect("Failed to initialize tracing logger"); *lock = Some(LoggerWithTempDir { diff --git a/bindings/c/src/lib.rs b/bindings/c/src/lib.rs index a05cedd23..eff524cd5 100644 --- a/bindings/c/src/lib.rs +++ b/bindings/c/src/lib.rs @@ -1,11 +1,11 @@ use std::{ alloc::{self, Layout}, - ffi::{CStr, CString, c_char, c_void}, + ffi::{CStr, CString, OsString, c_char, c_void}, }; use allocator as _; use anyhow::{Context, Result, bail}; -use clap::{Error as ClapError, Parser}; +use clap::Error as ClapError; use eth1_api::ClientVersionV1; use execution_engine::{ BlobAndProofV1, EngineGetPayloadV1Response, EngineGetPayloadV2Response, @@ -515,13 +515,15 @@ pub extern "C" fn grandine_shutdown() { #[unsafe(no_mangle)] pub unsafe extern "C" fn grandine_run(argc: u64, argv: *const *const c_char) -> u64 { unsafe fn try_run(argc: u64, argv: *const *const c_char) -> Result<()> { - let args = std::iter::once("").chain(unsafe { - std::slice::from_raw_parts(argv, argc as usize) - .into_iter() - .filter_map(|it| CStr::from_ptr(*it).to_str().ok()) - }); + let args = std::iter::once("") + .chain(unsafe { + std::slice::from_raw_parts(argv, argc as usize) + .into_iter() + .filter_map(|it| CStr::from_ptr(*it).to_str().ok()) + }) + .map(OsString::from); - let args = GrandineArgs::try_parse_from(args)?; + let args = GrandineArgs::parse_strictly(args)?; run(args) } diff --git a/block_producer/src/block_producer.rs b/block_producer/src/block_producer.rs index 771675fb9..55eb62ea8 100644 --- a/block_producer/src/block_producer.rs +++ b/block_producer/src/block_producer.rs @@ -109,7 +109,9 @@ use types::{ traits::{BeaconState as _, PostBellatrixBeaconState, PostGloasBeaconState}, }; -use crate::misc::{PayloadIdEntry, ProposerData, ValidatorBlindedBlock, build_graffiti}; +use crate::misc::{ + BlindedBlockPublication, PayloadIdEntry, ProposerData, ValidatorBlindedBlock, build_graffiti, +}; const PAYLOAD_CACHE_SIZE: usize = 20; const PAYLOAD_ID_CACHE_SIZE: usize = 10; @@ -587,24 +589,21 @@ impl BlockProducer { pub async fn publish_signed_blinded_block( &self, block: &SignedBlindedBeaconBlock

, - ) -> Option, P>>> { + ) -> Option> { let header_root = block.execution_payload_header().hash_tree_root(); let mut payload_cache = self.producer_context.payload_cache.lock().await; let local_payload = payload_cache.cache_get(&header_root); match local_payload { - Some(payload) => Some(payload.clone()), - None => self - .publish_signed_blinded_block_using_builder(block) - .await - .map(WithClientVersions::none), + Some(payload) => Some(BlindedBlockPublication::Payload(Box::new(payload.clone()))), + None => self.publish_signed_blinded_block_using_builder(block).await, } } async fn publish_signed_blinded_block_using_builder( &self, block: &SignedBlindedBeaconBlock

, - ) -> Option, P>> { + ) -> Option> { let controller = &self.producer_context.controller; let builder_api = self.producer_context.builder_api.as_deref()?; let current_slot = controller.slot(); @@ -618,6 +617,35 @@ impl BlockProducer { return None; } + if block.phase() >= Phase::Fulu { + match builder_api + .post_blinded_block_post_fulu( + &self.producer_context.chain_config, + controller.genesis_time(), + block, + ) + .await + { + Ok(()) => return Some(BlindedBlockPublication::PublishedByBuilder), + // The builder may still publish the block, and a v1 retry would wait another interval. + Err(error) + if error + .downcast_ref::() + .is_some_and(reqwest::Error::is_timeout) => + { + debug_with_peers!( + "failed to post blinded block to the builder node: {error:?}" + ); + + return None; + } + Err(error) => warn_with_peers!( + "builder did not accept the blinded block for publishing, \ + requesting its payload instead: {error:?}" + ), + } + } + let execution_payload = match builder_api .post_blinded_block( &self.producer_context.chain_config, @@ -628,16 +656,17 @@ impl BlockProducer { { Ok(execution_payload) => execution_payload, Err(error) => { - if let Some(reqwest_error) = error.downcast_ref::() { - if reqwest_error.is_timeout() { - // If posting signed blinded beacon block to builder fails, don't print a warning, - // because builder should publish the block anyway, just cannot respond in a timely - // manner. We cannot do anything else here either, but exit early from propose, due - // to the risk of slashing. - log_with_feature(format_args!( - "failed to post blinded block to the builder node: {error:?}", - )); - } + if error + .downcast_ref::() + .is_some_and(reqwest::Error::is_timeout) + { + // If posting signed blinded beacon block to builder fails, don't print a warning, + // because builder should publish the block anyway, just cannot respond in a timely + // manner. We cannot do anything else here either, but exit early from propose, due + // to the risk of slashing. + debug_with_peers!( + "failed to post blinded block to the builder node: {error:?}" + ); return None; } @@ -652,7 +681,9 @@ impl BlockProducer { execution_payload.value )); - Some(execution_payload) + Some(BlindedBlockPublication::Payload(Box::new( + WithClientVersions::none(execution_payload), + ))) } pub async fn cache_self_built_block_parent( @@ -1245,9 +1276,9 @@ impl BlockBuildContext { let parent_execution_requests = if snapshot.should_build_on_full(slot) { // TODO(gloas): the block root needs to be checked if it is gloas or not. - // the current behavior erronously returns empty execution requests when - // payload envelope is not found, though block may be from gloas phase and - // envelope is missing (so it should error out). + // the current behavior erroneously returns empty execution requests + // when payload envelope is not found, though block may be from gloas + // phase and envelope is missing (so it should error out). snapshot .cached_execution_payload_envelope_by_root(parent_root) .ok_or_else(|| anyhow!("no cached payload envelope")) diff --git a/block_producer/src/lib.rs b/block_producer/src/lib.rs index f0bc7abe9..7fcd7a43c 100644 --- a/block_producer/src/lib.rs +++ b/block_producer/src/lib.rs @@ -2,7 +2,7 @@ pub use block_producer::{ BidWeighting, BlockBuildOptions, BlockProducer, BuilderApiBid, BuilderApiBidsJoinHandle, Options, }; -pub use misc::{ProposerData, ValidatorBlindedBlock}; +pub use misc::{BlindedBlockPublication, ProposerData, ValidatorBlindedBlock}; mod block_producer; mod misc; diff --git a/block_producer/src/misc.rs b/block_producer/src/misc.rs index 1eaaf58ad..f1e5934e3 100644 --- a/block_producer/src/misc.rs +++ b/block_producer/src/misc.rs @@ -1,15 +1,15 @@ use std::sync::Arc; use anyhow::Result; -use eth1_api::{ClientVersionV1, ClientVersions}; +use eth1_api::{ClientVersionV1, ClientVersions, WithClientVersions}; use execution_engine::PayloadId; use grandine_version::{APPLICATION_NAME_WITH_VERSION, APPLICATION_NAME_WITH_VERSION_AND_COMMIT}; use serde::{Deserialize, Serialize}; use ssz::{Size, SszHash, SszSize, SszWrite, WriteError}; use typenum::U1; use types::{ - combined::{BeaconBlock, BlindedBeaconBlock}, - nonstandard::Phase, + combined::{BeaconBlock, BlindedBeaconBlock, ExecutionPayload}, + nonstandard::{Phase, WithBlobsAndMev}, phase0::primitives::{ExecutionAddress, H256, ValidatorIndex}, preset::Preset, traits::BeaconBlock as _, @@ -21,6 +21,12 @@ pub enum PayloadIdEntry { Live(PayloadId), } +pub enum BlindedBlockPublication { + // Boxed to pass `clippy::large_enum_variant`. + Payload(Box, P>>>), + PublishedByBuilder, +} + impl PayloadIdEntry { #[must_use] pub const fn id(self) -> PayloadId { @@ -30,7 +36,7 @@ impl PayloadIdEntry { } } -#[derive(Deserialize)] +#[derive(Clone, Copy, PartialEq, Eq, Deserialize, Serialize)] pub struct ProposerData { #[serde(with = "serde_utils::string_or_native")] pub validator_index: ValidatorIndex, @@ -101,6 +107,14 @@ impl ValidatorBlindedBlock

{ pub const fn is_blinded(&self) -> bool { matches!(self, Self::BlindedBeaconBlock(_)) } + + #[must_use] + pub fn parent_root(&self) -> H256 { + match self { + Self::BlindedBeaconBlock(block) => block.parent_root(), + Self::BeaconBlock(block) => block.parent_root(), + } + } } pub fn build_graffiti( diff --git a/builder_api/src/gloas/builder_url.rs b/builder_api/src/gloas/builder_url.rs index 215273578..57f08f796 100644 --- a/builder_api/src/gloas/builder_url.rs +++ b/builder_api/src/gloas/builder_url.rs @@ -106,6 +106,13 @@ impl Debug for BuilderUrl { } } +// `Display` redacts the URL, so code that needs all of it can borrow it without naming the type. +impl AsRef for BuilderUrl { + fn as_ref(&self) -> &str { + self.as_str() + } +} + impl Display for BuilderUrl { fn fmt(&self, formatter: &mut Formatter) -> FmtResult { match self.origin() { diff --git a/builder_api/src/unphased/containers.rs b/builder_api/src/unphased/containers.rs index 76ba7ce58..2800769d2 100644 --- a/builder_api/src/unphased/containers.rs +++ b/builder_api/src/unphased/containers.rs @@ -20,7 +20,7 @@ pub struct ValidatorRegistrationV1 { pub pubkey: PublicKeyBytes, } -#[derive(Debug, Deserialize, Serialize, Ssz)] +#[derive(Clone, Copy, Debug, Deserialize, Serialize, Ssz)] #[ssz(derive_hash = false, derive_read = false)] pub struct SignedValidatorRegistrationV1 { pub message: ValidatorRegistrationV1, diff --git a/clock/src/lib.rs b/clock/src/lib.rs index 36533e441..4a3700565 100644 --- a/clock/src/lib.rs +++ b/clock/src/lib.rs @@ -169,6 +169,11 @@ impl Tick { ) } + #[must_use] + pub fn is_before_aggregating_interval(self) -> bool { + self.kind < TickKind::Aggregate + } + #[must_use] pub fn is_before_payload_attesting_interval(self) -> bool { self.kind < TickKind::PayloadAttest @@ -451,7 +456,7 @@ fn next_tick_with_instant( // Tick now operates in milliseconds, so to prevent precision loss by rounding down to unix seconds, // We pass genesis time in milliseconds to prevent rounding error and precision calculation and - // comparision. + // comparison. let unix_epoch_to_genesis = Duration::from_millis(genesis_time_in_ms); // Some platforms do not support negative `Instant`s. Operations that would produce an `Instant` diff --git a/directories/src/lib.rs b/directories/src/lib.rs index e85eb6477..8af36eb71 100644 --- a/directories/src/lib.rs +++ b/directories/src/lib.rs @@ -59,15 +59,20 @@ impl Directories { self } - // TODO: This does not include validator_dir and other files outside store and network directories + // TODO: This does not include files outside the store, network and validator directories pub fn disk_usage(&self) -> Result { - let dir_usage = - |dir: Option<&PathBuf>| dir.as_ref().map(fs_extra::dir::get_size).transpose(); + // The validator client never creates the store and network directories. + let dir_usage = |dir: Option<&PathBuf>| { + dir.filter(|dir| dir.exists()) + .map(fs_extra::dir::get_size) + .transpose() + }; let store_usage = dir_usage(self.store_directory.as_ref())?.unwrap_or_default(); let network_usage = dir_usage(self.network_dir.as_ref())?.unwrap_or_default(); + let validator_usage = dir_usage(self.validator_dir.as_ref())?.unwrap_or_default(); - Ok(store_usage + network_usage) + Ok(store_usage + network_usage + validator_usage) } } diff --git a/doppelganger_protection/Cargo.toml b/doppelganger_protection/Cargo.toml index ecd118aee..77a2a8875 100644 --- a/doppelganger_protection/Cargo.toml +++ b/doppelganger_protection/Cargo.toml @@ -7,9 +7,7 @@ edition.workspace = true anyhow = { workspace = true } arc-swap = { workspace = true } bls = { workspace = true } -futures = { workspace = true } helper_functions = { workspace = true } -liveness_tracker = { workspace = true } logging = { workspace = true } thiserror = { workspace = true } tracing ={ workspace = true } diff --git a/doppelganger_protection/src/doppelganger_protection.rs b/doppelganger_protection/src/doppelganger_protection.rs index c5079f178..0e0f187f0 100644 --- a/doppelganger_protection/src/doppelganger_protection.rs +++ b/doppelganger_protection/src/doppelganger_protection.rs @@ -1,3 +1,4 @@ +use core::future::Future; use std::{ collections::{HashMap, HashSet}, sync::Arc, @@ -6,13 +7,10 @@ use std::{ use anyhow::{Result, ensure}; use arc_swap::{ArcSwap, Guard}; use bls::PublicKeyBytes; -use futures::channel::mpsc::UnboundedSender; -use helper_functions::{accessors, misc}; -use liveness_tracker::ApiToLiveness; +use helper_functions::misc; use logging::warn_with_peers; use typenum::Unsigned as _; use types::{ - combined::BeaconState, phase0::{ consts::{GENESIS_EPOCH, GENESIS_SLOT}, primitives::{Epoch, Slot, ValidatorIndex}, @@ -24,41 +22,38 @@ use crate::error::Error; const DOPPELGANGER_CHECK_DURATION_IN_EPOCHS: Epoch = 2; -#[derive(Clone, Copy)] -struct TrackedValidator { - added_in_slot: Slot, - validator_index: ValidatorIndex, -} - +#[derive(Default)] pub struct DoppelgangerProtection { snapshot: ArcSwap, - liveness_checker: LivenessChecker, } impl DoppelgangerProtection { #[must_use] - pub fn new(liveness_tx: UnboundedSender) -> Self { - Self { - snapshot: ArcSwap::from_pointee(Snapshot::default()), - liveness_checker: LivenessChecker::Live { liveness_tx }, - } + pub fn new() -> Self { + Self::default() } - pub fn activate_validators_that_pass_checks(&self, current_slot: Slot) { + fn activate_validators_that_pass_checks( + &self, + current_slot: Slot, + checked: &HashMap, + ) { let check_duration_in_slots = DOPPELGANGER_CHECK_DURATION_IN_EPOCHS.saturating_mul(P::SlotsPerEpoch::U64); - let (validators_to_activate, validators_to_track): (HashMap<_, _>, HashMap<_, _>) = self + let validators_to_activate = self .load() .tracked_validators .iter() - .map(|(public_key, validator)| (*public_key, *validator)) - .partition(|(_, validator)| { - validator - .added_in_slot - .saturating_add(check_duration_in_slots) - <= current_slot - }); + .map(|(public_key, added_in_slot)| (*public_key, *added_in_slot)) + .filter(|(public_key, added_in_slot)| { + checked.contains_key(public_key) + && added_in_slot.is_some_and(|added_in_slot| { + added_in_slot.saturating_add(check_duration_in_slots) <= current_slot + }) + }) + .map(|(public_key, _)| public_key) + .collect::>(); if validators_to_activate.is_empty() { return; @@ -67,20 +62,20 @@ impl DoppelgangerProtection { self.update(|snapshot| { let mut snapshot = snapshot.as_ref().clone(); - snapshot - .active_validators - .extend(validators_to_activate.keys()); + snapshot.active_validators.extend(&validators_to_activate); - snapshot.tracked_validators.clone_from(&validators_to_track); + // Only the activated keys leave, so a key tracked since the load above stays. + snapshot + .tracked_validators + .retain(|public_key, _| !validators_to_activate.contains(public_key)); snapshot }); } - pub fn add_tracked_validators( + pub fn add_tracked_validators( &self, public_keys: impl IntoIterator, - beacon_state: &BeaconState

, current_slot: Slot, ) { let snapshot = self.load(); @@ -110,69 +105,110 @@ impl DoppelgangerProtection { let mut snapshot = snapshot.as_ref().clone(); for public_key in &filtered_public_keys { - match accessors::index_of_public_key(beacon_state, public_key) { - Some(validator_index) => { - snapshot.tracked_validators - .entry(*public_key) - .or_insert(TrackedValidator { - added_in_slot: current_slot, - validator_index, - }); - } - None => { - warn_with_peers!( - "validator with public key {public_key:?} was not found in beacon state!", - ); - } - } + snapshot + .tracked_validators + .entry(*public_key) + .or_insert(Some(current_slot)); } snapshot }); } - pub async fn detect_doppelgangers(&self, current_slot: Slot) -> Result<()> { - self.activate_validators_that_pass_checks::

(current_slot); + pub async fn detect_doppelgangers( + &self, + current_slot: Slot, + indices_by_pubkey: &HashMap>, + check_liveness: F, + ) -> Result<()> + where + F: Fn(Epoch, Vec) -> Fut + Send + Sync, + Fut: Future>> + Send, + { + let mut checked = HashMap::new(); + let mut validator_indices_with_pubkeys = HashMap::new(); + let mut restamped = HashMap::new(); + let mut removed = HashSet::new(); + + for (public_key, added_in_slot) in &self.load().tracked_validators { + match indices_by_pubkey.get(public_key) { + Some(Some(validator_index)) => { + checked.insert(*public_key, *validator_index); + validator_indices_with_pubkeys.insert(*validator_index, *public_key); + + // The checks only cover the key from here on, so the window starts anew. + if added_in_slot.is_none() { + restamped.insert(*public_key, Some(current_slot)); + } + } + Some(None) => { + if added_in_slot.is_some() { + warn_with_peers!( + "liveness of validator with public key {public_key:?} cannot be \ + checked until its index is known; it will not perform duties", + ); - let validator_indices_with_pubkeys = self - .load() - .tracked_validators - .iter() - .map(|(pubkey, validator)| (validator.validator_index, *pubkey)) - .collect::>(); + restamped.insert(*public_key, None); + } + } + // A key the signer no longer holds has no duties left to guard. + None => { + removed.insert(*public_key); + } + } + } - if validator_indices_with_pubkeys.is_empty() { - return Ok(()); + if !restamped.is_empty() || !removed.is_empty() { + self.update(|snapshot| { + let mut snapshot = snapshot.as_ref().clone(); + snapshot.tracked_validators.extend(&restamped); + snapshot + .tracked_validators + .retain(|public_key, _| !removed.contains(public_key)); + snapshot + }); } - let current_epoch = misc::compute_epoch_at_slot::

(current_slot); + if !validator_indices_with_pubkeys.is_empty() { + let current_epoch = misc::compute_epoch_at_slot::

(current_slot); + + if current_epoch > GENESIS_EPOCH { + Self::detect_doppelgangers_in_epoch( + current_epoch.saturating_sub(1), + &validator_indices_with_pubkeys, + &check_liveness, + ) + .await?; + } - if current_epoch > GENESIS_EPOCH { - self.detect_doppelgangers_in_epoch( - current_epoch.saturating_sub(1), + Self::detect_doppelgangers_in_epoch( + current_epoch, &validator_indices_with_pubkeys, + &check_liveness, ) .await?; } - self.detect_doppelgangers_in_epoch(current_epoch, &validator_indices_with_pubkeys) - .await?; + // Activation comes after the checks so that a failed liveness query postpones it. + self.activate_validators_that_pass_checks::

(current_slot, &checked); Ok(()) } - async fn detect_doppelgangers_in_epoch( - &self, + async fn detect_doppelgangers_in_epoch( epoch: Epoch, validator_indices_with_pubkeys: &HashMap, - ) -> Result<()> { - let liveness = self - .liveness_checker - .check_liveness( - epoch, - validator_indices_with_pubkeys.keys().copied().collect(), - ) - .await?; + check_liveness: &F, + ) -> Result<()> + where + F: Fn(Epoch, Vec) -> Fut + Send + Sync, + Fut: Future>> + Send, + { + let liveness = check_liveness( + epoch, + validator_indices_with_pubkeys.keys().copied().collect(), + ) + .await?; let public_keys = liveness .into_iter() @@ -201,83 +237,86 @@ impl DoppelgangerProtection { { self.snapshot.rcu(f) } - - #[cfg(test)] - #[must_use] - pub fn new_with_mock_liveness(liveness: HashMap>) -> Self { - Self { - snapshot: ArcSwap::from_pointee(Snapshot::default()), - liveness_checker: LivenessChecker::Mock { liveness }, - } - } } #[derive(Clone, Default)] pub struct Snapshot { // Validators that are already active and have passed doppelganger protection checks active_validators: HashSet, - // Validators that are tracked by doppelganger protection - tracked_validators: HashMap, + // Validators that are tracked by doppelganger protection, by the slot their check window + // started in, or `None` while their index is unknown and no check can cover them + tracked_validators: HashMap>, } impl Snapshot { + #[must_use] pub fn is_validator_active(&self, public_key: PublicKeyBytes) -> bool { self.active_validators.contains(&public_key) } - pub fn tracking_end_slot(&self, public_key: PublicKeyBytes) -> Slot { + #[must_use] + pub fn tracking_end_slot(&self, public_key: PublicKeyBytes) -> Option { self.tracked_validators .get(&public_key) - .map(|validator| { - validator.added_in_slot.saturating_add( + .copied() + .flatten() + .map(|added_in_slot| { + added_in_slot.saturating_add( DOPPELGANGER_CHECK_DURATION_IN_EPOCHS.saturating_mul(P::SlotsPerEpoch::U64), ) }) - .unwrap_or(Slot::MAX) - } -} - -enum LivenessChecker { - Live { - liveness_tx: UnboundedSender, - }, - #[cfg(test)] - Mock { - liveness: HashMap>, - }, -} - -impl LivenessChecker { - pub async fn check_liveness( - &self, - epoch: Epoch, - validator_indices: Vec, - ) -> Result> { - match self { - Self::Live { liveness_tx } => { - let (sender, receiver) = futures::channel::oneshot::channel(); - - ApiToLiveness::CheckLiveness(sender, epoch, validator_indices).send(liveness_tx); - - receiver.await? - } - #[cfg(test)] - Self::Mock { liveness } => Ok(liveness.get(&epoch).cloned().unwrap_or_default()), - } } } #[cfg(test)] mod tests { + use core::future::{Ready, ready}; + + use anyhow::anyhow; + use helper_functions::accessors; use pubkey_cache::PubkeyCache; - use types::{config::Config, preset::Minimal}; + use types::{combined::BeaconState, config::Config, preset::Minimal}; use super::*; fn doppelganger_protection() -> DoppelgangerProtection { - let liveness = [(GENESIS_EPOCH, vec![(0, false), (1, true)])].into(); + DoppelgangerProtection::new() + } - DoppelgangerProtection::new_with_mock_liveness(liveness) + fn mock_liveness( + epoch: Epoch, + _validator_indices: Vec, + ) -> Ready>> { + let liveness: HashMap> = + [(GENESIS_EPOCH, vec![(0, false), (1, true)])].into(); + + ready(Ok(liveness.get(&epoch).cloned().unwrap_or_default())) + } + + fn failing_liveness( + _epoch: Epoch, + _validator_indices: Vec, + ) -> Ready>> { + ready(Err(anyhow!("liveness query failed"))) + } + + fn unresolved(pubkey: PublicKeyBytes) -> HashMap> { + HashMap::from([(pubkey, None)]) + } + + fn indices_by_pubkey( + state: &BeaconState, + validator_indices: impl IntoIterator, + ) -> HashMap> { + validator_indices + .into_iter() + .map(|validator_index| { + ( + validator_pubkey(state, validator_index), + Some(validator_index), + ) + }) + .collect() } fn minimal_beacon_state() -> Arc> { @@ -294,46 +333,181 @@ mod tests { .unwrap_or_else(|_| panic!("validator at position {validator_index} should exist")) } + // Nothing can run before genesis, so a key added at the genesis slot needs no check. #[test] fn test_is_validator_active_added_at_genesis_slot() { let doppelganger_protection = doppelganger_protection(); let state = minimal_beacon_state(); let pubkey = validator_pubkey(&state, 0); - doppelganger_protection.add_tracked_validators([pubkey], &state, GENESIS_SLOT); + doppelganger_protection.add_tracked_validators([pubkey], GENESIS_SLOT); let is_active = || doppelganger_protection.load().is_validator_active(pubkey); assert!(is_active()); } + // A key added after genesis is activated only once two epochs of checks have passed. #[tokio::test] async fn test_is_validator_active_added_later_than_genesis_slot() -> Result<()> { let doppelganger_protection = doppelganger_protection(); let state = minimal_beacon_state(); let pubkey = validator_pubkey(&state, 0); + let indices = indices_by_pubkey(&state, [0]); let added_at_slot = GENESIS_SLOT + 1; - doppelganger_protection.add_tracked_validators([pubkey], &state, added_at_slot); + doppelganger_protection.add_tracked_validators([pubkey], added_at_slot); let is_active = || doppelganger_protection.load().is_validator_active(pubkey); assert!(!is_active()); doppelganger_protection - .detect_doppelgangers::(added_at_slot + 1) + .detect_doppelgangers::(added_at_slot + 1, &indices, mock_liveness) + .await?; + + assert!(!is_active()); + + doppelganger_protection + .detect_doppelgangers::(added_at_slot + 15, &indices, mock_liveness) .await?; assert!(!is_active()); doppelganger_protection - .detect_doppelgangers::(added_at_slot + 15) + .detect_doppelgangers::(added_at_slot + 16, &indices, mock_liveness) + .await?; + + assert!(is_active()); + + Ok(()) + } + + // A validator whose index is not resolved yet is neither checked nor activated on time alone, + // and its check window only starts once the index is known. + #[tokio::test] + async fn test_validator_with_unknown_index_is_not_activated() -> Result<()> { + let doppelganger_protection = doppelganger_protection(); + let state = minimal_beacon_state(); + let pubkey = validator_pubkey(&state, 0); + let added_at_slot = GENESIS_SLOT + 1; + + doppelganger_protection.add_tracked_validators([pubkey], added_at_slot); + + let is_active = || doppelganger_protection.load().is_validator_active(pubkey); + + doppelganger_protection + .detect_doppelgangers::( + added_at_slot + 16, + &unresolved(pubkey), + mock_liveness, + ) + .await?; + + assert!(!is_active()); + + let indices = indices_by_pubkey(&state, [0]); + + let resolved_at_slot = added_at_slot + 17; + + doppelganger_protection + .detect_doppelgangers::(resolved_at_slot, &indices, mock_liveness) + .await?; + + assert!(!is_active()); + + doppelganger_protection + .detect_doppelgangers::(resolved_at_slot + 15, &indices, mock_liveness) + .await?; + + assert!(!is_active()); + + doppelganger_protection + .detect_doppelgangers::(resolved_at_slot + 16, &indices, mock_liveness) + .await?; + + assert!(is_active()); + + Ok(()) + } + + // A validator whose window has elapsed stays inactive until a liveness query succeeds. + #[tokio::test] + async fn test_failed_liveness_query_postpones_activation() -> Result<()> { + let doppelganger_protection = doppelganger_protection(); + let state = minimal_beacon_state(); + let pubkey = validator_pubkey(&state, 0); + let indices = indices_by_pubkey(&state, [0]); + let added_at_slot = GENESIS_SLOT + 1; + + doppelganger_protection.add_tracked_validators([pubkey], added_at_slot); + + let is_active = || doppelganger_protection.load().is_validator_active(pubkey); + + doppelganger_protection + .detect_doppelgangers::(added_at_slot + 16, &indices, failing_liveness) + .await + .expect_err("the liveness query fails"); + + assert!(!is_active()); + + doppelganger_protection + .detect_doppelgangers::(added_at_slot + 17, &indices, mock_liveness) + .await?; + + assert!(is_active()); + + Ok(()) + } + + // A key deleted during its window is no longer tracked, and a re-import starts a new window. + #[tokio::test] + async fn test_validator_removed_during_its_window_restarts_it() -> Result<()> { + let doppelganger_protection = doppelganger_protection(); + let state = minimal_beacon_state(); + let pubkey = validator_pubkey(&state, 0); + let indices = indices_by_pubkey(&state, [0]); + let added_at_slot = GENESIS_SLOT + 1; + + doppelganger_protection.add_tracked_validators([pubkey], added_at_slot); + + let is_active = || doppelganger_protection.load().is_validator_active(pubkey); + + let tracking_end_slot = || { + doppelganger_protection + .load() + .tracking_end_slot::(pubkey) + }; + + doppelganger_protection + .detect_doppelgangers::(added_at_slot + 1, &indices, mock_liveness) + .await?; + + assert_eq!(tracking_end_slot(), Some(added_at_slot + 16)); + + doppelganger_protection + .detect_doppelgangers::( + added_at_slot + 2, + &HashMap::new(), + mock_liveness, + ) + .await?; + + assert_eq!(tracking_end_slot(), None); + + let resolved_at_slot = added_at_slot + 17; + + doppelganger_protection.add_tracked_validators([pubkey], resolved_at_slot); + + doppelganger_protection + .detect_doppelgangers::(resolved_at_slot, &indices, mock_liveness) .await?; assert!(!is_active()); + assert_eq!(tracking_end_slot(), Some(resolved_at_slot + 16)); doppelganger_protection - .detect_doppelgangers::(added_at_slot + 16) + .detect_doppelgangers::(resolved_at_slot + 16, &indices, mock_liveness) .await?; assert!(is_active()); @@ -341,6 +515,7 @@ mod tests { Ok(()) } + // Adding an active key again does not put it back under a check. #[test] fn test_adding_validators_multiple_times() { let doppelganger_protection = doppelganger_protection(); @@ -349,27 +524,29 @@ mod tests { let is_active = || doppelganger_protection.load().is_validator_active(pubkey); - doppelganger_protection.add_tracked_validators([pubkey], &state, GENESIS_SLOT); + doppelganger_protection.add_tracked_validators([pubkey], GENESIS_SLOT); assert!(is_active()); - doppelganger_protection.add_tracked_validators([pubkey], &state, GENESIS_SLOT + 40); + doppelganger_protection.add_tracked_validators([pubkey], GENESIS_SLOT + 40); assert!(is_active()); } + // A validator seen live in a checked epoch is reported, and nothing is activated. #[tokio::test] async fn test_doppelganger_detection() -> Result<()> { let doppelganger_protection = doppelganger_protection(); let state = minimal_beacon_state(); let pubkey = validator_pubkey(&state, 1); + let indices = indices_by_pubkey(&state, [1]); let added_at_slot = GENESIS_SLOT + 1; - doppelganger_protection.add_tracked_validators([pubkey], &state, added_at_slot); + doppelganger_protection.add_tracked_validators([pubkey], added_at_slot); assert_eq!( doppelganger_protection - .detect_doppelgangers::(added_at_slot + 1) + .detect_doppelgangers::(added_at_slot + 1, &indices, mock_liveness) .await .expect_err("a doppelganger should be detected") .downcast::()?, diff --git a/doppelganger_protection/src/lib.rs b/doppelganger_protection/src/lib.rs index a8b958ba8..4a30f1337 100644 --- a/doppelganger_protection/src/lib.rs +++ b/doppelganger_protection/src/lib.rs @@ -1,4 +1,4 @@ -pub use doppelganger_protection::DoppelgangerProtection; +pub use doppelganger_protection::{DoppelgangerProtection, Snapshot}; pub use error::Error; mod doppelganger_protection; diff --git a/eth1_api/src/auth.rs b/eth1_api/src/auth.rs index 03fcdd433..b6396f0ec 100644 --- a/eth1_api/src/auth.rs +++ b/eth1_api/src/auth.rs @@ -22,7 +22,7 @@ use derive_more::Debug; const JWT_SECRET_SIZE_MIN_BYTES: usize = 32; -#[derive(Debug)] +#[derive(Clone, Debug)] #[cfg_attr(test, derive(Default))] pub struct Options { pub secrets_path: Option, diff --git a/fork_choice_control/Cargo.toml b/fork_choice_control/Cargo.toml index 5c71f37d8..4396877b7 100644 --- a/fork_choice_control/Cargo.toml +++ b/fork_choice_control/Cargo.toml @@ -63,6 +63,7 @@ duplicate = { workspace = true } eth2_cache_utils = { workspace = true } factory = { workspace = true } fs-err = { workspace = true } +interop = { workspace = true } serde-aux = { workspace = true } spec_test_utils = { workspace = true } tap = { workspace = true } diff --git a/fork_choice_control/src/controller.rs b/fork_choice_control/src/controller.rs index c45640b59..ddfa66855 100644 --- a/fork_choice_control/src/controller.rs +++ b/fork_choice_control/src/controller.rs @@ -276,10 +276,6 @@ where self.spawn(StateAtSlotCacheFlushTask { state_at_slot_cache: self.state_at_slot_cache.clone_arc(), }); - - if let Some(metrics) = self.metrics.as_ref() { - metrics.set_beacon_clock_slot(tick.slot); - } } } diff --git a/fork_choice_control/src/extra_tests.rs b/fork_choice_control/src/extra_tests.rs index 7d50389a4..b4c8266b2 100644 --- a/fork_choice_control/src/extra_tests.rs +++ b/fork_choice_control/src/extra_tests.rs @@ -17,13 +17,13 @@ use std::sync::Arc; use eth2_cache_utils::medalla; #[cfg(feature = "eth2-cache")] use eth2_libp2p::GossipId; -use helper_functions::misc; +use helper_functions::{accessors, misc}; +use itertools::Itertools as _; use spec_test_utils::BlsSetting; #[cfg(feature = "eth2-cache")] use std_ext::ArcExt as _; -#[cfg(feature = "eth2-cache")] -use types::{config::Config, preset::Medalla}; use types::{ + combined::Attestation, gloas::consts::{PAYLOAD_STATUS_EMPTY, PAYLOAD_STATUS_FULL}, nonstandard::{PayloadStatus, Phase}, phase0::{ @@ -35,13 +35,16 @@ use types::{ primitives::H256, }, preset::Minimal, - traits::SignedBeaconBlock as _, + traits::{BeaconState as _, SignedBeaconBlock as _}, }; +#[cfg(feature = "eth2-cache")] +use types::{config::Config, preset::Medalla}; use fork_choice_store::PayloadPresence; use crate::{ helpers::{Context, Status, epoch_at_slot, is_at_start_of_epoch, start_of_epoch}, + messages::P2pMessage, storage::StoredBlock, }; @@ -3675,3 +3678,46 @@ fn gloas_block_events_are_optimistic_if_the_payload_they_rest_on_is() { context.assert_block_events(&[(root_3, false)]); } + +// A single attestation correctly signed by a validator outside the committee it names is rejected +// at validation, before it would be forwarded, published or handed to the attestation pool. +#[test] +fn rejects_single_attestation_from_validator_outside_committee() { + let mut context = Context::electra_minimal(); + + let (_, state_0) = context.genesis(); + + context.on_slot(1); + + let member = 0; + let (attestation, subnet_id) = + context.single_attestation_signed_by(&state_0, GENESIS_EPOCH, member, member); + + let Attestation::Single(single_attestation) = &attestation else { + unreachable!("single_attestation_signed_by builds a single attestation"); + }; + + let committee = accessors::beacon_committee( + &state_0, + single_attestation.data.slot, + single_attestation.committee_index, + ) + .expect("the committee the attestation names exists in the genesis state"); + + let outsider = (0..state_0.validators().len_u64()) + .find(|index| !committee.into_iter().contains(index)) + .expect("the minimal genesis has more validators than one committee"); + + assert!(matches!( + context.on_gossip_attestation(attestation, subnet_id), + Some(P2pMessage::Accept(_)), + )); + + let (forged, subnet_id) = + context.single_attestation_signed_by(&state_0, GENESIS_EPOCH, member, outsider); + + assert!(matches!( + context.on_gossip_attestation(forged, subnet_id), + Some(P2pMessage::Reject(_, _)), + )); +} diff --git a/fork_choice_control/src/helpers.rs b/fork_choice_control/src/helpers.rs index 0ed409e35..b7858f847 100644 --- a/fork_choice_control/src/helpers.rs +++ b/fork_choice_control/src/helpers.rs @@ -11,7 +11,7 @@ use execution_engine::{ }; use fork_choice_store::{AttestationItem, AttestationOrigin, PayloadPresence}; use futures::channel::mpsc::UnboundedReceiver; -use helper_functions::misc; +use helper_functions::{misc, signing::SignForSingleFork as _}; use pubkey_cache::PubkeyCache; use spec_test_utils::BlsSetting; use ssz::BitVector; @@ -22,6 +22,7 @@ use types::{ combined::{Attestation, AttesterSlashing, BeaconState, DataColumnSidecar, SignedBeaconBlock}, config::Config, deneb::containers::{BlobIdentifier, BlobSidecar}, + electra::containers::SingleAttestation, gloas::{ containers::{ CombinedPayloadAttestation, PayloadAttestationMessage, SignedExecutionPayloadEnvelope, @@ -31,7 +32,9 @@ use types::{ nonstandard::{PayloadStatus, Phase, TimedPowBlock}, phase0::{ containers::Checkpoint, - primitives::{Epoch, ExecutionBlockHash, H256, Slot, UnixSeconds, ValidatorIndex}, + primitives::{ + Epoch, ExecutionBlockHash, H256, Slot, SubnetId, UnixSeconds, ValidatorIndex, + }, }, preset::{Minimal, Preset}, traits::SignedBeaconBlock as _, @@ -1451,6 +1454,14 @@ impl Context

{ ) .expect("attestation should be constructed successfully"); + self.on_gossip_attestation(attestation, subnet_id) + } + + pub fn on_gossip_attestation( + &mut self, + attestation: Attestation

, + subnet_id: SubnetId, + ) -> Option> { self.controller().on_singular_attestation( self.controller().owned_wait_group(), AttestationItem::unverified( @@ -1463,6 +1474,48 @@ impl Context

{ self.next_p2p_message() } + // The single attestation `member` would send in `epoch`, signed by `signer`. `state` must + // already be in `epoch`, as the genesis state is for epoch 0. + pub fn single_attestation_signed_by( + &self, + state: &Arc>, + epoch: Epoch, + member: ValidatorIndex, + signer: ValidatorIndex, + ) -> (Attestation

, SubnetId) { + let (attestation, subnet_id) = factory::singular_attestation( + self.config(), + &self.pubkey_cache, + state.clone_arc(), + epoch, + member, + false, + ) + .expect("attestation should be constructed successfully"); + + let Attestation::Electra(attestation) = attestation else { + panic!("single attestations exist from Electra on"); + }; + + let committee_index = misc::get_committee_indices::

(attestation.committee_bits) + .next() + .expect("a singular attestation has exactly one committee bit set"); + + let signature = attestation + .data + .sign(self.config(), state, &interop::secret_key(signer)) + .into(); + + let attestation = Attestation::Single(SingleAttestation { + committee_index, + attester_index: signer, + data: attestation.data, + signature, + }); + + (attestation, subnet_id) + } + fn next_execution_service_message(&mut self) -> Option> { self.service_rx.try_recv().ok() } @@ -1526,6 +1579,11 @@ impl Context { .expect("minimal configuration modified to start in Bellatrix is valid") } + pub fn electra_minimal() -> Self { + Self::with_config(Config::minimal().start_and_stay_in(Phase::Electra)) + .expect("minimal configuration modified to start in Electra is valid") + } + #[must_use] pub fn gloas_minimal() -> Self { Self::with_config(Config::minimal().start_and_stay_in(Phase::Gloas)) diff --git a/fork_choice_control/src/messages.rs b/fork_choice_control/src/messages.rs index c3f062e81..8be1a4b2f 100644 --- a/fork_choice_control/src/messages.rs +++ b/fork_choice_control/src/messages.rs @@ -170,6 +170,11 @@ pub enum MutatorMessage { wait_group: W, persisted_block_roots: Vec, }, + FinishedArchivingFinalized { + wait_group: W, + new_anchor_index: usize, + saved: bool, + }, PayloadBid { result: Result>, origin: ExecutionPayloadBidOrigin, @@ -292,6 +297,7 @@ pub enum ValidatorMessage { Tick(W, Tick), Head(W, ChainLink

), PayloadStatusUpdated(W, ChainLink

), + FinalizedCheckpoint(Checkpoint), ValidAttestation(W, Arc>), ValidPayloadAttestation(W, Arc), PrepareExecutionPayload(Slot, ExecutionBlockHash, ExecutionBlockHash), @@ -299,7 +305,7 @@ pub enum ValidatorMessage { } impl ValidatorMessage { - pub(crate) fn send(self, tx: &impl UnboundedSink) { + pub fn send(self, tx: &impl UnboundedSink) { // Don't log the value because it can contain entire `BeaconState`s. if tx.unbounded_send(self).is_err() { debug_with_peers!("send to validator failed because the receiver was dropped"); diff --git a/fork_choice_control/src/mutator.rs b/fork_choice_control/src/mutator.rs index d90a9f832..cf39e084d 100644 --- a/fork_choice_control/src/mutator.rs +++ b/fork_choice_control/src/mutator.rs @@ -14,6 +14,7 @@ // tasks as well, but that would complicate code and would most likely not improve performance much // (in fact, the opposite may be true because `p2p_tx` would have to be cloned for each task). +use core::panic::AssertUnwindSafe; use std::{ collections::{BTreeMap, HashMap, HashSet}, sync::{ @@ -108,6 +109,8 @@ use crate::{ const DATA_COLUMN_RETAIN_DURATION_IN_SLOTS: Slot = 2; const MAX_DELAYED_FUTURE_BLOCKS_PER_SLOT: usize = 16; const MAX_DELAYED_BLOCKS_UNTIL_PARENT: usize = 64; +// Failed archives keep their blocks in memory, so a persistent failure must not keep them forever. +const MAX_CONSECUTIVE_FAILED_ARCHIVES: u8 = 2; // Room for requested envelopes waiting for their blocks. // Forward sync runs many 32-slot batches ahead of the head. // 64 was too small: envelopes near the head got dropped and sync stalled. @@ -150,6 +153,8 @@ pub struct Mutator { metrics: Option>, finished_loading_from_storage: bool, has_pending_archive_tasks: bool, + is_archiving_finalized: bool, + consecutive_failed_archives: u8, pending_state_cache_prune: Option, mutator_tx: Sender>, mutator_rx: Receiver>, @@ -218,6 +223,8 @@ where metrics, finished_loading_from_storage: false, has_pending_archive_tasks: false, + is_archiving_finalized: false, + consecutive_failed_archives: 0, pending_state_cache_prune: None, mutator_tx, mutator_rx, @@ -347,6 +354,13 @@ where persisted_block_roots, ); } + MutatorMessage::FinishedArchivingFinalized { + wait_group, + new_anchor_index, + saved, + } => { + self.handle_finish_archiving_finalized(&wait_group, new_anchor_index, saved)? + } MutatorMessage::ExecutionPayloadEnvelope { wait_group, result, @@ -3018,6 +3032,35 @@ where self.update_store_snapshot(); } + fn handle_finish_archiving_finalized( + &mut self, + wait_group: &W, + new_anchor_index: usize, + saved: bool, + ) -> Result<()> { + self.is_archiving_finalized = false; + + if !saved { + self.consecutive_failed_archives = self.consecutive_failed_archives.saturating_add(1); + + // The blocks stay for the next finalization to retry. + if self.consecutive_failed_archives < MAX_CONSECUTIVE_FAILED_ARCHIVES { + return Ok(()); + } + + error_with_peers!( + "dropping finalized blocks that failed to be archived \ + {MAX_CONSECUTIVE_FAILED_ARCHIVES} times in a row", + ); + } + + self.consecutive_failed_archives = 0; + + self.store_mut().archive_finalized(new_anchor_index); + self.update_store_snapshot(); + self.archive_finalized(wait_group) + } + fn handle_finish_persisting_execution_payload_envelopes( &mut self, wait_group: W, @@ -3505,25 +3548,6 @@ where } } - // Pre-Gloas, do not send API events about optimistic blocks. - // Vouch treats all head events as non-optimistic. - // Post-Gloas they go out regardless, flagged like head events. - // A new block has no payload of its own yet. - // It is optimistic if the payload it rests on is. - if is_valid || is_post_gloas { - let execution_optimistic = self - .store - .node_payload_status(block_root, false) - .is_some_and(PayloadStatus::is_optimistic); - - self.event_channels.send_block_event( - block_slot, - block_root, - execution_optimistic, - block.message().payload_bid(), - ); - } - let from_test_block = matches!(origin, BlockOrigin::Test(_)); // TODO(Grandine Team): Performing the validation here results in the block being added to the @@ -3622,8 +3646,6 @@ where } } - // Call `Store::apply_attester_slashing` after `Store::archive_finalized` to reduce the - // amount of work needed to update balances. let slashable_indices = attester_slashing_results .into_iter() .flat_map(|result| { @@ -3644,12 +3666,30 @@ where .apply_attester_slashing(slashable_indices)?; } - // The snapshot should be updated: - // - After calling `Mutator::archive_finalized` because it mutates the store. - // - Before spawning tasks to retry delayed objects or notifying other components to ensure - // that they cannot observe the store in an inconsistent state. + // The snapshot should be updated before spawning tasks to retry delayed objects or + // notifying other components to ensure that they cannot observe the store in an + // inconsistent state. self.update_store_snapshot(); + // Pre-Gloas, do not send API events about optimistic blocks. + // Clients treat all head events as non-optimistic. + // Post-Gloas they go out regardless, flagged like head events. + // A new block has no payload of its own yet. + // It is optimistic if the payload it rests on is. + if is_valid || is_post_gloas { + let execution_optimistic = self + .store + .node_payload_status(block_root, false) + .is_some_and(PayloadStatus::is_optimistic); + + self.event_channels.send_block_event( + block_slot, + block_root, + execution_optimistic, + block.message().payload_bid(), + ); + } + if let Some(objects) = self.take_delayed_until_block(block_root) { debug_with_peers!("retrying objects delayed until block {block_root:?}"); debug_with_peers!( @@ -3934,6 +3974,7 @@ where ); self.send_to_p2p(P2pMessage::FinalizedCheckpoint(finalized_checkpoint)); + self.send_to_validator(ValidatorMessage::FinalizedCheckpoint(finalized_checkpoint)); if let Some(metrics) = self.metrics.as_ref() { let state = head.state(&self.store); @@ -5343,6 +5384,10 @@ where } fn archive_finalized(&mut self, wait_group: &W) -> Result<()> { + if self.is_archiving_finalized { + return Ok(()); + } + if let Some(latest_archivable_index) = self.store.latest_archivable_index() { debug_with_peers!("archiving finalized blocks and anchor state…"); @@ -5350,85 +5395,109 @@ where let store = self.owned_store(); let storage = self.storage.clone_arc(); let sync_tx = self.sync_tx.clone(); + let mutator_tx = self.owned_mutator_tx(); let wait_group = wait_group.clone(); let finished_loading_from_storage = self.finished_loading_from_storage; - let mut archived = self.store_mut().archive_finalized(latest_archivable_index); - archived.push_back(self.store.anchor().clone()); + // The blocks stay in the store until they are in storage, so queries never miss them. + let archived = self.store.archivable_finalized(latest_archivable_index); let last_finalized_slot = self.store.last_finalized().slot(); Builder::new() .name("store-archiver".to_owned()) .spawn(move || { - debug_with_peers!("saving finalized blocks and anchor state…"); + // Storage writes are transactional and a panic counts as a failed save, so the + // blocks stay in the store and the next archive writes them again. + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + debug_with_peers!("saving finalized blocks and anchor state…"); - match storage.append(core::iter::empty(), archived.iter(), &store) { - Ok(slots) => { - if let Some(chain_link) = archived.back() { - let finalized_block = chain_link.block.clone_arc(); - let anchor_epoch = misc::compute_epoch_at_slot::

(chain_link.slot()); - - for candidate in &archived { - let candidate_slot = candidate.slot(); - let candidate_epoch = misc::compute_epoch_at_slot::

(candidate_slot); - - let is_last_block = candidate_slot.saturating_add(1) == chain_link.slot(); - let is_first_block = candidate_epoch == anchor_epoch.saturating_sub(1) - && misc::is_epoch_start::

(candidate_slot); - - // preload first and the last block of previous epoch into state_at_slot_cache - // to handle some of the beacon API use cases - if is_first_block || is_last_block { - debug_with_peers!( - "preloading state_at_slot_cache with chain_link: \ - slot: {candidate_slot}, epoch: {candidate_epoch}, \ - is_first_block: {is_first_block}, is_last_block: {is_last_block}", - ); + let saved = match storage.append(core::iter::empty(), archived.iter(), &store) { + Ok(slots) => { + if let Some(chain_link) = archived.back() { + let finalized_block = chain_link.block.clone_arc(); + let anchor_epoch = misc::compute_epoch_at_slot::

(chain_link.slot()); + + for candidate in &archived { + let candidate_slot = candidate.slot(); + let candidate_epoch = misc::compute_epoch_at_slot::

(candidate_slot); - if let Err(error) = state_at_slot_cache - .get_or_try_init(candidate.slot(), || { - Ok(Some(candidate.state(&store))) - }) - { + let is_last_block = candidate_slot.saturating_add(1) == chain_link.slot(); + let is_first_block = candidate_epoch == anchor_epoch.saturating_sub(1) + && misc::is_epoch_start::

(candidate_slot); + + // preload first and the last block of previous epoch into state_at_slot_cache + // to handle some of the beacon API use cases + if is_first_block || is_last_block { debug_with_peers!( - "failed to preload state_at_slot_cache with \ - a chainlink: {error:?}", + "preloading state_at_slot_cache with chain_link: \ + slot: {candidate_slot}, epoch: {candidate_epoch}, \ + is_first_block: {is_first_block}, is_last_block: {is_last_block}", ); + + if let Err(error) = state_at_slot_cache + .get_or_try_init(candidate.slot(), || { + Ok(Some(candidate.state(&store))) + }) + { + debug_with_peers!( + "failed to preload state_at_slot_cache with \ + a chainlink: {error:?}", + ); + } } } - } - if finished_loading_from_storage { - SyncMessage::Finalized(finalized_block).send(&sync_tx); + if finished_loading_from_storage { + SyncMessage::Finalized(finalized_block).send(&sync_tx); + } } - } - debug_with_peers!( - "finalized blocks and anchor state saved \ - (appended block slots: {slots:?})", - ) - } - Err(error) => error_with_peers!("saving to storage failed: {error:?}"), - } - - debug_with_peers!("removing unfinalized blocks"); - - if !storage.archive_storage_enabled() { - match storage.prune_unfinalized_blocks(last_finalized_slot) { - Ok(slots) => { debug_with_peers!( - "unfinalized block pruning complete: pruned slots: {slots:?}" + "finalized blocks and anchor state saved \ + (appended block slots: {slots:?})", ); + + true } Err(error) => { - error_with_peers!("unfinalized block pruning failed: {error:?}") + error_with_peers!("saving to storage failed: {error:?}"); + false + } + }; + + debug_with_peers!("removing unfinalized blocks"); + + if !storage.archive_storage_enabled() { + match storage.prune_unfinalized_blocks(last_finalized_slot) { + Ok(slots) => { + debug_with_peers!( + "unfinalized block pruning complete: pruned slots: {slots:?}" + ); + } + Err(error) => { + error_with_peers!("unfinalized block pruning failed: {error:?}") + } } } - } - drop(wait_group); + saved + })); + + let saved = result.unwrap_or_else(|payload| { + panics::log(payload); + false + }); + + MutatorMessage::FinishedArchivingFinalized { + wait_group, + new_anchor_index: latest_archivable_index, + saved, + } + .send(&mutator_tx); })?; + + self.is_archiving_finalized = true; } Ok(()) diff --git a/fork_choice_store/src/store.rs b/fork_choice_store/src/store.rs index bc20dcb0a..347fe89e3 100644 --- a/fork_choice_store/src/store.rs +++ b/fork_choice_store/src/store.rs @@ -56,7 +56,10 @@ use types::{ containers::{BlobIdentifier, BlobSidecar}, primitives::{BlobIndex, KzgCommitment}, }, - electra::containers::IndexedAttestation as ElectraIndexedAttestation, + electra::{ + containers::IndexedAttestation as ElectraIndexedAttestation, + error::AttestationConversionError, + }, fulu::{containers::DataColumnIdentifier, primitives::ColumnIndex}, gloas::{ consts::{ @@ -3327,6 +3330,25 @@ impl> Store { Ok(indexed_attestation.attesting_indices) } Attestation::Single(attestation) => { + let committee = accessors::beacon_committee( + target_state, + attestation.data.slot, + attestation.committee_index, + )?; + + // > The attester is a member of the committee -- i.e. + // > attestation.attester_index in + // > get_beacon_committee(state, attestation.data.slot, index) + ensure!( + committee.into_iter().contains(&attestation.attester_index), + AttestationConversionError::AttesterNotInCommittee { + attester_index: attestation.attester_index, + committee_index: attestation.committee_index, + attestation_data: attestation.data, + committee: committee.into_iter().collect(), + }, + ); + let indexed_attestation: ElectraIndexedAttestation

= (*attestation).try_into()?; @@ -6367,8 +6389,13 @@ impl> Store { .map(|(index, _)| index) } - pub fn archive_finalized(&mut self, new_anchor_index: usize) -> Vector> { - let archived = self.finalized.slice(..new_anchor_index); + #[must_use] + pub fn archivable_finalized(&self, new_anchor_index: usize) -> Vector> { + self.finalized.take(new_anchor_index.saturating_add(1)) + } + + pub fn archive_finalized(&mut self, new_anchor_index: usize) { + self.finalized = self.finalized.skip(new_anchor_index); self.finalized_indices = self .finalized @@ -6377,8 +6404,6 @@ impl> Store { .enumerate() .map(|(index, block_root)| (block_root, index)) .collect(); - - archived } // Return whether the current tick is before the deadline defined by `due_bps` diff --git a/fork_choice_store/src/store_config.rs b/fork_choice_store/src/store_config.rs index 2e261e4a7..0bee6acef 100644 --- a/fork_choice_store/src/store_config.rs +++ b/fork_choice_store/src/store_config.rs @@ -8,7 +8,7 @@ use crate::builder_circuit_breaker::BuilderCircuitBreakerConfig; pub const DEFAULT_CACHE_LOCK_TIMEOUT_MILLIS: u64 = 1500; -#[derive(Clone, Copy, Derivative)] +#[derive(Clone, Copy, Debug, Derivative)] #[derivative(Default)] pub struct StoreConfig { #[derivative(Default(value = "32"))] diff --git a/grandine-snapshot-tests b/grandine-snapshot-tests index 45db60b65..ef133f238 160000 --- a/grandine-snapshot-tests +++ b/grandine-snapshot-tests @@ -1 +1 @@ -Subproject commit 45db60b654af6bfce596c41281e089e2929d32cf +Subproject commit ef133f238b16b9a6189595271e160d2a6ecfaa17 diff --git a/helper_functions/src/accessors.rs b/helper_functions/src/accessors.rs index a3e127bc1..7baffb107 100644 --- a/helper_functions/src/accessors.rs +++ b/helper_functions/src/accessors.rs @@ -35,7 +35,9 @@ use types::{ containers::{IndexedPayloadAttestation, PayloadAttestation, ProposerPreferences}, primitives::BuilderIndex, }, - nonstandard::{AttestationEpoch, PartialValidator, Participation, Phase, RelativeEpoch}, + nonstandard::{ + AttestationEpoch, ForkInfo, PartialValidator, Participation, Phase, RelativeEpoch, + }, phase0::{ consts::{DOMAIN_BEACON_ATTESTER, DOMAIN_BEACON_PROPOSER}, containers::AttestationData, @@ -561,7 +563,22 @@ pub fn get_domain( epoch: Option, ) -> H256 { let epoch = epoch.unwrap_or_else(|| get_current_epoch(state)); - let fork = state.fork(); + + get_domain_from_fork_info(config, ForkInfo::from_state(state), domain_type, epoch) +} + +/// [`get_domain`] for a fork that is already known, as it is to a validator without a state. +#[must_use] +pub fn get_domain_from_fork_info( + config: &Config, + fork_info: ForkInfo, + domain_type: DomainType, + epoch: Epoch, +) -> H256 { + let ForkInfo { + fork, + genesis_validators_root, + } = fork_info; let fork_version = if epoch < fork.epoch { fork.previous_version @@ -573,7 +590,7 @@ pub fn get_domain( config, domain_type, Some(fork_version), - Some(state.genesis_validators_root()), + Some(genesis_validators_root), ) } @@ -1319,8 +1336,10 @@ mod tests { ProposerLookahead, fulu::beacon_state::BeaconState as FuluBeaconState, phase0::{ - beacon_state::BeaconState as Phase0BeaconState, consts::GENESIS_EPOCH, - containers::Validator, + beacon_state::BeaconState as Phase0BeaconState, + consts::GENESIS_EPOCH, + containers::{Fork, Validator}, + primitives::Version, }, preset::Minimal, }; @@ -1344,6 +1363,56 @@ mod tests { assert_eq!(get_current_epoch(&state), 4); } + // The version switches at the fork epoch, and a state and its fork info give the same domain. + #[test] + fn test_get_domain_from_fork_info_at_the_fork_boundary() { + let config = Config::minimal(); + + let fork = Fork { + previous_version: Version::from([0, 0, 0, 1]), + current_version: Version::from([0, 0, 0, 2]), + epoch: 5, + }; + + let genesis_validators_root = H256::repeat_byte(3); + + let fork_info = ForkInfo { + fork, + genesis_validators_root, + }; + + let domain_at = + |epoch| get_domain_from_fork_info(&config, fork_info, DOMAIN_BEACON_ATTESTER, epoch); + + let expected = |version| { + misc::compute_domain( + &config, + DOMAIN_BEACON_ATTESTER, + Some(version), + Some(genesis_validators_root), + ) + }; + + assert_eq!(domain_at(4), expected(fork.previous_version)); + assert_eq!(domain_at(5), expected(fork.current_version)); + assert_ne!(domain_at(4), domain_at(5)); + + let state = Phase0BeaconState:: { + fork, + genesis_validators_root, + ..Phase0BeaconState::default() + }; + + assert_eq!( + get_domain(&config, &state, DOMAIN_BEACON_ATTESTER, Some(4)), + domain_at(4), + ); + assert_eq!( + get_domain(&config, &state, DOMAIN_BEACON_ATTESTER, Some(5)), + domain_at(5), + ); + } + #[test] fn test_get_previous_epoch_genesis() { let state = Phase0BeaconState::::default(); diff --git a/helper_functions/src/predicates.rs b/helper_functions/src/predicates.rs index 4eb31f8af..9967cbf08 100644 --- a/helper_functions/src/predicates.rs +++ b/helper_functions/src/predicates.rs @@ -27,7 +27,7 @@ use types::{ }, primitives::BuilderIndex, }, - nonstandard::PartialValidator, + nonstandard::{PartialValidator, Phase}, phase0::{ consts::{ETH1_ADDRESS_WITHDRAWAL_PREFIX, FAR_FUTURE_EPOCH, TargetAggregatorsPerCommittee}, containers::{AttestationData, DepositMessage}, @@ -202,14 +202,33 @@ pub fn is_aggregator( ) -> Result { let committee = accessors::beacon_committee(state, slot, committee_index)?; + is_aggregator_from_committee_length(committee.len(), slot_signature) +} + +#[must_use] +pub fn is_valid_attestation_data_index(phase: Phase, index: u64) -> bool { + if phase < Phase::Electra { + true + } else if phase < Phase::Gloas { + index == 0 + } else { + // Gloas repurposes the index as the payload vote. + index < 2 + } +} + +/// [`is_aggregator`] for a committee whose length is already known, as it is in attester duties. +pub fn is_aggregator_from_committee_length( + committee_length: usize, + slot_signature: SignatureBytes, +) -> Result { let dividend = hashing::hash_768(slot_signature) .index(..size_of::()) .try_into() .map(u64::from_le_bytes) .expect("slice has the same length as u64"); - let modulo = committee - .len() + let modulo = committee_length .try_conv::()? .div(TargetAggregatorsPerCommittee::U64) .try_into() diff --git a/helper_functions/src/signing.rs b/helper_functions/src/signing.rs index 20176a176..3390dc924 100644 --- a/helper_functions/src/signing.rs +++ b/helper_functions/src/signing.rs @@ -40,6 +40,7 @@ use types::{ PayloadAttestationData, ProposerPreferences, }, }, + nonstandard::ForkInfo, phase0::{ consts::{ DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_BEACON_ATTESTER, DOMAIN_BEACON_PROPOSER, @@ -143,8 +144,18 @@ pub trait SignForSingleFork: SszHash { fn epoch(&self) -> Epoch; fn signing_root(&self, config: &Config, beacon_state: &(impl BeaconState

+ ?Sized)) -> H256 { - let epoch = Some(self.epoch()); - let domain = accessors::get_domain(config, beacon_state, Self::DOMAIN_TYPE, epoch); + self.signing_root_from_fork_info(config, ForkInfo::from_state(beacon_state)) + } + + /// [`Self::signing_root`] for a fork that is already known. + fn signing_root_from_fork_info(&self, config: &Config, fork_info: ForkInfo) -> H256 { + let domain = accessors::get_domain_from_fork_info( + config, + fork_info, + Self::DOMAIN_TYPE, + self.epoch(), + ); + misc::compute_signing_root(self, domain) } @@ -182,9 +193,21 @@ pub trait SignForSingleForkAtSlot: SszHash { config: &Config, beacon_state: &(impl BeaconState

+ ?Sized), slot: Slot, + ) -> H256 { + self.signing_root_from_fork_info(config, ForkInfo::from_state(beacon_state), slot) + } + + /// [`Self::signing_root`] for a fork that is already known. + fn signing_root_from_fork_info( + &self, + config: &Config, + fork_info: ForkInfo, + slot: Slot, ) -> H256 { let epoch = misc::compute_epoch_at_slot::

(slot); - let domain = accessors::get_domain(config, beacon_state, Self::DOMAIN_TYPE, Some(epoch)); + let domain = + accessors::get_domain_from_fork_info(config, fork_info, Self::DOMAIN_TYPE, epoch); + misc::compute_signing_root(self, domain) } @@ -440,21 +463,22 @@ impl SignForSingleFork

for VoluntaryExit { self.epoch } - fn signing_root(&self, config: &Config, beacon_state: &(impl BeaconState

+ ?Sized)) -> H256 { + fn signing_root_from_fork_info(&self, config: &Config, fork_info: ForkInfo) -> H256 { let domain_type = >::DOMAIN_TYPE; - let current_fork_version = beacon_state.fork().current_version; + let current_fork_version = fork_info.fork.current_version; + // EIP-7044 pins the domain to Capella, keyed on the fork as in `process_voluntary_exit`. let domain = if current_fork_version == config.deneb_fork_version || current_fork_version == config.electra_fork_version || current_fork_version == config.fulu_fork_version || current_fork_version == config.gloas_fork_version { let fork_version = Some(config.capella_fork_version); - let genesis_validators_root = Some(beacon_state.genesis_validators_root()); + let genesis_validators_root = Some(fork_info.genesis_validators_root); misc::compute_domain(config, domain_type, fork_version, genesis_validators_root) } else { let epoch = >::epoch(self); - accessors::get_domain(config, beacon_state, domain_type, Some(epoch)) + accessors::get_domain_from_fork_info(config, fork_info, domain_type, epoch) }; misc::compute_signing_root(self, domain) @@ -505,15 +529,16 @@ impl SignForSingleFork

for ProposerPreferences { misc::compute_epoch_at_slot::

(self.proposal_slot) } - fn signing_root(&self, config: &Config, beacon_state: &(impl BeaconState

+ ?Sized)) -> H256 { + fn signing_root_from_fork_info(&self, config: &Config, fork_info: ForkInfo) -> H256 { let epoch = >::epoch(self); let domain_type = >::DOMAIN_TYPE; + // The proposal epoch's fork, even when signed from a state before the fork. let fork_version = config.version_at_epoch(epoch); let domain = misc::compute_domain( config, domain_type, Some(fork_version), - Some(beacon_state.genesis_validators_root()), + Some(fork_info.genesis_validators_root), ); misc::compute_signing_root(self, domain) } @@ -524,3 +549,118 @@ impl SignForAllForks for BuilderDepositMessage { const DOMAIN_TYPE: DomainType = DOMAIN_BUILDER_DEPOSIT; const SIGNATURE_KIND: SignatureKind = SignatureKind::BuilderDeposit; } + +#[cfg(test)] +mod tests { + use types::{ + phase0::{ + containers::Fork, + primitives::{ExecutionAddress, Version}, + }, + preset::Mainnet, + }; + + use super::*; + + fn fork_info(previous_version: Version, current_version: Version) -> ForkInfo { + ForkInfo { + fork: Fork { + previous_version, + current_version, + epoch: 0, + }, + genesis_validators_root: H256::repeat_byte(1), + } + } + + #[test] + fn voluntary_exit_domain_is_pinned_to_capella_from_deneb_on() { + let config = Config::mainnet(); + + let exit = VoluntaryExit { + epoch: 0, + validator_index: 1, + }; + + let capella = fork_info(config.bellatrix_fork_version, config.capella_fork_version); + + let capella_root = + SignForSingleFork::::signing_root_from_fork_info(&exit, &config, capella); + + for current_version in [ + config.deneb_fork_version, + config.electra_fork_version, + config.fulu_fork_version, + config.gloas_fork_version, + ] { + let later = fork_info(config.capella_fork_version, current_version); + + assert_eq!( + SignForSingleFork::::signing_root_from_fork_info(&exit, &config, later), + capella_root, + "exit domain under {current_version:?} should be pinned to Capella", + ); + } + } + + #[test] + fn voluntary_exit_domain_follows_the_fork_before_deneb() { + let config = Config::mainnet(); + + let exit = VoluntaryExit { + epoch: 0, + validator_index: 1, + }; + + let bellatrix = fork_info(config.altair_fork_version, config.bellatrix_fork_version); + let capella = fork_info(config.bellatrix_fork_version, config.capella_fork_version); + + assert_ne!( + SignForSingleFork::::signing_root_from_fork_info(&exit, &config, bellatrix), + SignForSingleFork::::signing_root_from_fork_info(&exit, &config, capella), + ); + } + + // Preferences for the first Gloas epoch are signed an epoch early, from a Fulu fork. + #[test] + fn proposer_preferences_domain_follows_the_proposal_epoch() { + let mut config = Config::mainnet(); + config.gloas_fork_epoch = 10; + + let preferences = |proposal_slot| ProposerPreferences { + dependent_root: H256::repeat_byte(2), + proposal_slot, + validator_index: 1, + fee_recipient: ExecutionAddress::zero(), + target_gas_limit: 0, + }; + + let gloas_slot = misc::compute_start_slot_at_epoch::(10); + let fulu = fork_info(config.electra_fork_version, config.fulu_fork_version); + let gloas = fork_info(config.fulu_fork_version, config.gloas_fork_version); + + let root_from_fulu = SignForSingleFork::::signing_root_from_fork_info( + &preferences(gloas_slot), + &config, + fulu, + ); + + assert_eq!( + root_from_fulu, + SignForSingleFork::::signing_root_from_fork_info( + &preferences(gloas_slot), + &config, + gloas, + ), + ); + + assert_ne!( + root_from_fulu, + SignForSingleFork::::signing_root_from_fork_info( + &preferences(gloas_slot - 1), + &config, + fulu, + ), + ); + } +} diff --git a/http_api/Cargo.toml b/http_api/Cargo.toml index 958ad7012..439799302 100644 --- a/http_api/Cargo.toml +++ b/http_api/Cargo.toml @@ -16,6 +16,7 @@ block_producer = { workspace = true } bls = { workspace = true } builder_api = { workspace = true } byteorder = { workspace = true } +clock = { workspace = true } dedicated_executor = { workspace = true } derive_more = { workspace = true } enum-iterator = { workspace = true } @@ -33,8 +34,6 @@ itertools = { workspace = true } kzg_utils = { workspace = true } liveness_tracker = { workspace = true } logging ={ workspace = true } -mediatype = { workspace = true } -mime = { workspace = true } operation_pools = { workspace = true } p2p = { workspace = true } parse-display = { workspace = true } @@ -64,7 +63,6 @@ validator_statistics = { workspace = true } [dev-dependencies] attestation_verifier = { workspace = true } -clock = { workspace = true } crossbeam-utils = { workspace = true } database = { workspace = true } dedicated_executor = { workspace = true } @@ -77,6 +75,7 @@ fork_choice_store = { workspace = true } hex-literal = { workspace = true } interop = { workspace = true } keymanager = { workspace = true } +mime = { workspace = true } num_cpus = { workspace = true } once_cell = { workspace = true } predefined_chains = { workspace = true } diff --git a/http_api/src/context.rs b/http_api/src/context.rs index e25e0f1be..88eed7e43 100644 --- a/http_api/src/context.rs +++ b/http_api/src/context.rs @@ -23,7 +23,7 @@ use fork_choice_control::{ use fork_choice_store::StoreConfig; use futures::{future::FutureExt as _, lock::Mutex, select_biased}; use genesis::AnchorCheckpointProvider; -use keymanager::KeyManager; +use keymanager::{KeyManager, ProposerConfigs}; use liveness_tracker::LivenessTracker; use once_cell::sync::OnceCell; use operation_pools::{ @@ -43,11 +43,14 @@ use types::{ combined::{BeaconState, SignedBeaconBlock}, config::Config as ChainConfig, nonstandard::{FinalizedCheckpoint, Phase, StorageMode}, - phase0::primitives::{H256, NodeId}, + phase0::primitives::{ExecutionAddress, H256, NodeId}, preset::{Mainnet, Minimal, Preset}, traits::BeaconState as _, }; -use validator::{Validator, ValidatorChannels, ValidatorConfig}; +use validator::{ + Chain, ChainSource, LocalChain, LocalValidatorChannels, OwnValidatorIndices, Validator, + ValidatorChannels, ValidatorConfig, +}; use crate::{ http_api_config::HttpApiConfig, @@ -102,7 +105,6 @@ impl Context

{ let (fc_to_sync_tx, fc_to_sync_rx) = futures::channel::mpsc::unbounded(); let (fc_to_validator_tx, fc_to_validator_rx) = futures::channel::mpsc::unbounded(); let (_, p2p_to_validator_rx) = futures::channel::mpsc::unbounded(); - let (pool_to_liveness_tx, pool_to_liveness_rx) = futures::channel::mpsc::unbounded(); let (pool_to_p2p_tx, pool_to_p2p_rx) = futures::channel::mpsc::unbounded(); let (subnet_service_to_p2p_tx, _subnet_service_to_p2p_rx) = futures::channel::mpsc::unbounded(); @@ -229,7 +231,6 @@ impl Context

{ let signer = Arc::new(Signer::new( validator_keys, - client.clone(), client, Web3SignerConfig::default(), None, @@ -248,13 +249,12 @@ impl Context

{ let validator_config = Arc::new(ValidatorConfig { disable_blockprint_graffiti: true, + // The snapshots were recorded with a fee recipient, without which none are published. + suggested_fee_recipient: Some(ExecutionAddress::zero()), ..Default::default() }); - let keymanager = Arc::new(KeyManager::new_in_memory( - signer.clone_arc(), - slashing_protector.clone_arc(), - anchor_state.genesis_validators_root(), + let proposer_configs = Arc::new(ProposerConfigs::new( validator_config.suggested_fee_recipient, validator_config.default_gas_limit, H256::default(), @@ -262,6 +262,14 @@ impl Context

{ validator_config.validator_definitions.clone_arc(), )); + let keymanager = Arc::new(KeyManager::new_in_memory( + signer.clone_arc(), + slashing_protector.clone_arc(), + anchor_state.genesis_validators_root(), + proposer_configs, + validator_config.validator_definitions.clone_arc(), + )); + let attestation_verifier = AttestationVerifier::new( controller.clone_arc(), dedicated_executor.clone_arc(), @@ -279,7 +287,6 @@ impl Context

{ let sync_committee_agg_pool = SyncCommitteeAggPool::new( dedicated_executor.clone_arc(), controller.clone_arc(), - Some(pool_to_liveness_tx), pool_to_p2p_tx.clone(), None, None, @@ -303,7 +310,6 @@ impl Context

{ controller.clone_arc(), None, api_to_liveness_rx, - pool_to_liveness_rx, validator_to_liveness_rx, ); @@ -325,37 +331,49 @@ impl Context

{ )); let validator_channels = ValidatorChannels { - api_to_validator_rx, - fork_choice_rx: fc_to_validator_rx, - p2p_tx: validator_to_p2p_tx, - p2p_to_validator_rx, - slasher_to_validator_rx: None, - subnet_service_tx: subnet_service_tx.clone(), - validator_to_liveness_tx: Some(validator_to_liveness_tx), - validator_to_slasher_tx: None, + validator_rx: fc_to_validator_rx, + local: Some(LocalValidatorChannels { + api_to_validator_rx, + p2p_tx: validator_to_p2p_tx, + p2p_to_validator_rx, + slasher_to_validator_rx: None, + subnet_service_tx: subnet_service_tx.clone(), + api_to_liveness_tx: None, + validator_to_liveness_tx: Some(validator_to_liveness_tx), + validator_to_slasher_tx: None, + }), }; let mut network_config = NetworkConfig::default(); network_config.identify_agent_version = Some(IDENTIFY_AGENT_VERSION.to_owned()); let network_config = Arc::new(network_config); + let chain_source = Arc::new(ChainSource { + chain_config: controller.chain_config().clone_arc(), + genesis_time: controller.genesis_time(), + genesis_validators_root: anchor_state.genesis_validators_root(), + own_validator_indices: Arc::new(OwnValidatorIndices::new(signer.clone_arc())), + chain: Chain::Local(Arc::new(LocalChain { + controller: controller.clone_arc(), + block_producer: block_producer.clone_arc(), + attestation_agg_pool: attestation_agg_pool.clone_arc(), + sync_committee_agg_pool: sync_committee_agg_pool.clone_arc(), + payload_attestation_agg_pool: payload_attestation_agg_pool.clone_arc(), + event_channels: event_channels.clone_arc(), + builder_api: None, + })), + }); + let validator = Validator::new( validator_config.clone_arc(), - block_producer.clone_arc(), - controller.clone_arc(), - attestation_agg_pool.clone_arc(), + chain_source, None, - None, - event_channels.clone_arc(), keymanager.proposer_configs().clone_arc(), - signer, + signer.clone_arc(), slashing_protector, - payload_attestation_agg_pool.clone_arc(), - sync_committee_agg_pool.clone_arc(), None, None, validator_channels, - network_config.network_dir.as_deref(), dedicated_executor.clone_arc(), dedicated_executor.clone_arc(), None, diff --git a/http_api/src/lib.rs b/http_api/src/lib.rs index 13170c59c..763193191 100644 --- a/http_api/src/lib.rs +++ b/http_api/src/lib.rs @@ -13,7 +13,6 @@ mod gui; mod http_api_config; mod middleware; mod misc; -mod response; mod routing; mod standard; mod state_id; diff --git a/http_api/src/misc.rs b/http_api/src/misc.rs index 95c762d65..28ba6f28b 100644 --- a/http_api/src/misc.rs +++ b/http_api/src/misc.rs @@ -57,7 +57,6 @@ use types::{ }, nonstandard::{KzgProofs, Phase, WithBlobsAndMev}, phase0::{ - consts::TargetAggregatorsPerCommittee, containers::{ Attestation as Phase0Attestation, SignedAggregateAndProof as Phase0SignedAggregateAndProof, @@ -260,6 +259,9 @@ impl From, P>> } } +pub type SignedAggregateAndProofs

= + ContiguousList>,

::MaxAggregatesPerSlot>; + pub struct SignedAggregateAndProofListFromPhaseDeserializer { phase: Phase, phantom: PhantomData

, @@ -275,7 +277,7 @@ impl From for SignedAggregateAndProofListFromPhaseDeserializer } impl<'de, P: Preset> DeserializeSeed<'de> for SignedAggregateAndProofListFromPhaseDeserializer

{ - type Value = ContiguousList>, TargetAggregatorsPerCommittee>; + type Value = SignedAggregateAndProofs

; fn deserialize(self, deserializer: D) -> Result where @@ -289,19 +291,19 @@ impl<'de, P: Preset> DeserializeSeed<'de> for SignedAggregateAndProofListFromPha | Phase::Capella | Phase::Deneb => ContiguousList::< Phase0SignedAggregateAndProof

, - TargetAggregatorsPerCommittee, + P::MaxAggregatesPerSlot, >::deserialize(deserializer)? .map(Into::into) .map(Arc::new), Phase::Electra | Phase::Fulu => ContiguousList::< ElectraSignedAggregateAndProof

, - TargetAggregatorsPerCommittee, + P::MaxAggregatesPerSlot, >::deserialize(deserializer)? .map(Into::into) .map(Arc::new), Phase::Gloas => ContiguousList::< GloasSignedAggregateAndProof

, - TargetAggregatorsPerCommittee, + P::MaxAggregatesPerSlot, >::deserialize(deserializer)? .map(Into::into) .map(Arc::new), @@ -732,3 +734,46 @@ pub enum BroadcastValidation { Consensus, ConsensusAndEquivocation, } + +#[cfg(test)] +mod tests { + use serde_json::{Value, json}; + use types::preset::Minimal; + + use super::*; + + #[test] + fn aggregate_request_may_hold_more_than_one_committee_of_aggregators() + -> Result<(), serde_json::Error> { + let root = format!("0x{}", "00".repeat(32)); + let signature = format!("0x{}", "00".repeat(96)); + let checkpoint = json!({ "epoch": "0", "root": root }); + + let aggregate_and_proof = json!({ + "message": { + "aggregator_index": "0", + "aggregate": { + "aggregation_bits": "0x01", + "data": { + "slot": "0", + "index": "0", + "beacon_block_root": root, + "source": checkpoint, + "target": checkpoint, + }, + "signature": signature, + }, + "selection_proof": signature, + }, + "signature": signature, + }); + + let aggregates = + SignedAggregateAndProofListFromPhaseDeserializer::::from(Phase::Phase0) + .deserialize(Value::Array(vec![aggregate_and_proof; 17]))?; + + assert_eq!(aggregates.len(), 17); + + Ok(()) + } +} diff --git a/http_api/src/routing.rs b/http_api/src/routing.rs index ac2aa8b2e..7116ed3c8 100644 --- a/http_api/src/routing.rs +++ b/http_api/src/routing.rs @@ -16,6 +16,7 @@ use features::Feature; use fork_choice_control::{EventChannels, Wait}; use futures::channel::mpsc::UnboundedSender; use genesis::AnchorCheckpointProvider; +use http_api_utils::{post_log_level, post_trace_level}; use liveness_tracker::ApiToLiveness; use operation_pools::{ AttestationAggPool, BlsToExecutionChangePool, PayloadAttestationAggPool, SyncCommitteeAggPool, @@ -40,10 +41,10 @@ use crate::{ node_peer_count, node_peers, node_syncing_status, node_version, node_version_v2, pool_attestations, pool_attestations_v2, pool_attester_slashings, pool_attester_slashings_v2, pool_bls_to_execution_changes, pool_payload_attestations, - pool_proposer_slashings, pool_voluntary_exits, post_log_level, post_state_builders, - post_state_validator_balances, post_state_validators, post_trace_level, - proposer_preferences, publish_blinded_block, publish_blinded_block_v2, publish_block, - publish_block_v2, publish_execution_payload_bid, publish_execution_payload_envelope, + pool_proposer_slashings, pool_voluntary_exits, post_state_builders, + post_state_validator_balances, post_state_validators, proposer_preferences, + publish_blinded_block, publish_blinded_block_v2, publish_block, publish_block_v2, + publish_execution_payload_bid, publish_execution_payload_envelope, state_builder_pending_payments, state_builder_pending_withdrawals, state_committees, state_finality_checkpoints, state_fork, state_pending_consolidations, state_pending_deposits, state_pending_partial_withdrawals, state_proposer_lookahead, diff --git a/http_api/src/standard.rs b/http_api/src/standard.rs index 7d6c9a66d..1d0a6bdf2 100644 --- a/http_api/src/standard.rs +++ b/http_api/src/standard.rs @@ -6,6 +6,7 @@ use core::time::Duration; use std::{ collections::{BTreeMap, BTreeSet, HashSet}, sync::Arc, + time::SystemTime, }; use anyhow::{Error as AnyhowError, Result, anyhow, ensure}; @@ -19,8 +20,9 @@ use axum::{ sse::{Event as ServerSentEvent, KeepAlive}, }, }; -use binary_utils::TracingHandle; -use block_producer::{BlockBuildOptions, BlockProducer, ProposerData, ValidatorBlindedBlock}; +use block_producer::{ + BlindedBlockPublication, BlockBuildOptions, BlockProducer, ProposerData, ValidatorBlindedBlock, +}; use bls::{PublicKeyBytes, SignatureBytes, traits::SignatureBytes as _}; use builder_api::{ PayloadBuilderApi, @@ -28,6 +30,7 @@ use builder_api::{ gloas::containers::{BuilderConfig as GloasBuilderConfig, BuilderPreferencesEntry, BuilderUrl}, unphased::containers::SignedValidatorRegistrationV1, }; +use clock::Tick; use dedicated_executor::DedicatedExecutor; use enum_iterator::Sequence as _; use eth1_api::{ApiController, ClientVersionV1, Eth1Api}; @@ -44,7 +47,11 @@ use futures::{ }; use genesis::AnchorCheckpointProvider; use helper_functions::{accessors, error::Error as AccessorError, misc}; -use http_api_utils::{BlockId, StateId}; +use http_api_utils::{ + BlockHeadersResponse, BlockId, ETH_BLOB_DATA_INCLUDED, ETH_BUILDER_URL, EthResponse, JsonOrSsz, + StateId, ValidatorAttesterDutyResponse, ValidatorLivenessResponse, ValidatorPTCDutyResponse, + ValidatorProposerDutyResponse, ValidatorSyncDutyResponse, +}; use itertools::{Either, Itertools as _, izip}; use kzg_utils::eip_4844::compute_blob_kzg_proof; use liveness_tracker::ApiToLiveness; @@ -85,7 +92,7 @@ use types::{ capella::containers::{SignedBlsToExecutionChange, Withdrawal}, combined::{ Attestation, AttesterSlashing, BeaconBlock, BeaconState, DataColumnSidecar, - SignedAggregateAndProof, SignedBeaconBlock, SignedBlindedBeaconBlock, + SignedBeaconBlock, SignedBlindedBeaconBlock, }, config::Config as ChainConfig, deneb::{ @@ -110,10 +117,10 @@ use types::{ ValidationOutcomeWithReason, WEI_IN_GWEI, WithBlobsAndMev, WithStatus, }, phase0::{ - consts::{GENESIS_SLOT, TargetAggregatorsPerCommittee}, + consts::GENESIS_SLOT, containers::{ AttestationData, AttesterSlashing as Phase0AttesterSlashing, Checkpoint, Fork, - ProposerSlashing, SignedBeaconBlockHeader, SignedVoluntaryExit, Validator, + ProposerSlashing, SignedVoluntaryExit, Validator, }, primitives::{ ChainId, CommitteeIndex, Epoch, ExecutionAddress, Gwei, H256, Slot, SubnetId, Uint256, @@ -126,7 +133,10 @@ use types::{ BlockBodyWithPayloadBid, PostFuluBeaconState, PostGloasBeaconState, SignedBeaconBlock as _, }, }; -use validator::{ApiToValidator, ValidatorConfig}; +use validator::{ + ApiToValidator, ValidatorConfig, attester_duties_at_slot, proposer_duties_at_epoch, + proposer_lookahead_applies, ptc_duties_at_epoch, +}; use crate::{ block_id, @@ -139,11 +149,11 @@ use crate::{ BuilderPreferencesEntryListPhaseDeserializer, PayloadAttestationMessageListPhaseDeserializer, SignedAPIBlock, SignedAPIBlockPhaseDeserializer, SignedAggregateAndProofListFromPhaseDeserializer, - SignedBlindedBeaconPhaseDeserializer, SignedExecutionPayloadBidPhaseDeserializer, + SignedAggregateAndProofs, SignedBlindedBeaconPhaseDeserializer, + SignedExecutionPayloadBidPhaseDeserializer, SignedProposerPreferencesListFromPhaseDeserializer, SingleApiAttestation, SingleApiAttestationListPhaseDeserializer, SyncedStatus, }, - response::{ETH_BLOB_DATA_INCLUDED, ETH_BUILDER_URL, EthResponse, JsonOrSsz}, state_id, validator_status::{ ValidatorId, ValidatorIdQuery, ValidatorIdsAndStatuses, ValidatorIdsAndStatusesBody, @@ -315,18 +325,6 @@ pub struct PublishBlockQuery { broadcast_validation: Option, } -#[derive(Deserialize)] -pub struct LogLevelRequest { - target: String, - level: String, -} - -#[derive(Deserialize)] -pub struct TraceLevelRequest { - target: Option, - level: String, -} - #[expect(clippy::struct_field_names)] #[derive(Serialize)] pub struct GetGenesisResponse { @@ -438,13 +436,6 @@ pub struct StateValidatorBalanceResponse { balance: Gwei, } -#[derive(Serialize)] -pub struct BlockHeadersResponse { - root: H256, - canonical: bool, - header: SignedBeaconBlockHeader, -} - #[derive(Serialize)] pub struct DepositContractResponse { address: ExecutionAddress, @@ -512,57 +503,6 @@ pub struct NodeSyncingResponse { el_offline: bool, } -#[derive(Serialize)] -pub struct ValidatorAttesterDutyResponse { - #[serde(with = "serde_utils::string_or_native")] - committee_index: CommitteeIndex, - #[serde(with = "serde_utils::string_or_native")] - committee_length: usize, - #[serde(with = "serde_utils::string_or_native")] - committees_at_slot: u64, - pubkey: PublicKeyBytes, - #[serde(with = "serde_utils::string_or_native")] - slot: Slot, - #[serde(with = "serde_utils::string_or_native")] - validator_committee_index: usize, - #[serde(with = "serde_utils::string_or_native")] - validator_index: ValidatorIndex, -} - -#[derive(Serialize)] -pub struct ValidatorPTCDutyResponse { - pubkey: PublicKeyBytes, - #[serde(with = "serde_utils::string_or_native")] - validator_index: ValidatorIndex, - #[serde(with = "serde_utils::string_or_native")] - slot: Slot, -} - -#[derive(Serialize)] -pub struct ValidatorProposerDutyResponse { - pubkey: PublicKeyBytes, - #[serde(with = "serde_utils::string_or_native")] - validator_index: ValidatorIndex, - #[serde(with = "serde_utils::string_or_native")] - slot: Slot, -} - -#[derive(Serialize)] -pub struct ValidatorSyncDutyResponse { - pubkey: PublicKeyBytes, - #[serde(with = "serde_utils::string_or_native")] - validator_index: ValidatorIndex, - #[serde(with = "As::>")] - validator_sync_committee_indices: Vec, -} - -#[derive(Serialize)] -pub struct ValidatorLivenessResponse { - #[serde(with = "serde_utils::string_or_native")] - index: ValidatorIndex, - is_live: bool, -} - /// `GET /eth/v1/beacon/genesis` #[instrument(skip_all, level = "debug", name = "http_api::genesis")] pub async fn genesis( @@ -1846,18 +1786,22 @@ pub async fn publish_blinded_block( SignedBlindedBeaconPhaseDeserializer

, >, ) -> Result { - let execution_payload = block_producer + let publication = block_producer .publish_signed_blinded_block(&signed_blinded_block) - .await; + .await + .ok_or(Error::ExecutionPayloadNotAvailable)?; + + let execution_payload = match publication { + BlindedBlockPublication::Payload(execution_payload) => *execution_payload, + BlindedBlockPublication::PublishedByBuilder => return Ok(StatusCode::OK), + }; let WithBlobsAndMev { value: execution_payload, proofs, blobs, .. - } = execution_payload - .ok_or(Error::ExecutionPayloadNotAvailable)? - .result; + } = execution_payload.result; let (message, signature) = signed_blinded_block.split(); let signed_beacon_block = message @@ -1933,6 +1877,7 @@ fn builder_url_from_headers(headers: &HeaderMap) -> Option { } /// `POST /eth/v2/beacon/blinded_blocks` +#[expect(clippy::too_many_arguments)] #[instrument( skip_all, level = "debug", @@ -1943,25 +1888,31 @@ pub async fn publish_blinded_block_v2( State(block_producer): State>>, State(controller): State>, State(event_channels): State>>, + State(metrics): State>>, State(api_to_p2p_tx): State>>, + State(dedicated_executor): State>, EthQuery(query): EthQuery, EthJsonOrSsz(signed_blinded_block, _): EthJsonOrSsz< Box>, SignedBlindedBeaconPhaseDeserializer

, >, ) -> Result { - let execution_payload = block_producer + let publication = block_producer .publish_signed_blinded_block(&signed_blinded_block) - .await; + .await + .ok_or(Error::ExecutionPayloadNotAvailable)?; + + let execution_payload = match publication { + BlindedBlockPublication::Payload(execution_payload) => *execution_payload, + BlindedBlockPublication::PublishedByBuilder => return Ok(StatusCode::OK), + }; let WithBlobsAndMev { value: execution_payload, proofs, blobs, .. - } = execution_payload - .ok_or(Error::ExecutionPayloadNotAvailable)? - .result; + } = execution_payload.result; let (message, signature) = signed_blinded_block.split(); @@ -1971,21 +1922,45 @@ pub async fn publish_blinded_block_v2( .with_signature(signature) .pipe(Arc::new); - let blob_sidecars = misc::construct_blob_sidecars( - &signed_beacon_block, - blobs.unwrap_or_default().into_iter(), - proofs.unwrap_or_else(KzgProofs::empty_deneb).into_iter(), - )?; + let broadcast_validation = query.broadcast_validation.unwrap_or_default(); - publish_signed_block_v2( - signed_beacon_block, - blob_sidecars, - query.broadcast_validation.unwrap_or_default(), - controller, - event_channels, - api_to_p2p_tx, - ) - .await + if signed_beacon_block.phase() == Phase::Fulu { + let data_column_sidecars = construct_data_column_sidecars_from_blobs( + controller.clone_arc(), + signed_beacon_block.clone_arc(), + blobs, + proofs, + metrics, + dedicated_executor, + ) + .await?; + + publish_signed_block_v2_with_data_column_sidecar( + signed_beacon_block, + data_column_sidecars, + broadcast_validation, + controller, + event_channels, + api_to_p2p_tx, + ) + .await + } else { + let blob_sidecars = misc::construct_blob_sidecars( + &signed_beacon_block, + blobs.unwrap_or_default().into_iter(), + proofs.unwrap_or_else(KzgProofs::empty_deneb).into_iter(), + )?; + + publish_signed_block_v2( + signed_beacon_block, + blob_sidecars, + broadcast_validation, + controller, + event_channels, + api_to_p2p_tx, + ) + .await + } } /// `POST /eth/v2/beacon/blocks` @@ -3101,24 +3076,34 @@ pub async fn node_health(State(is_synced): State>) -> StatusCo name = "http_api::validator_attester_duties" )] pub async fn validator_attester_duties( + State(chain_config): State>, State(controller): State>, State(anchor_checkpoint_provider): State>, EthPath(epoch): EthPath, EthJson(validator_indices): EthJson>, ) -> Result>, Error> { + let current_epoch = current_epoch_with_clock_disparity(&controller, &chain_config)?; + + if epoch > current_epoch.saturating_add(1) { + return Err(Error::InvalidEpoch(anyhow!( + "epoch {epoch} is after the next epoch" + ))); + } + let head_state = controller.head_state(); - let (state, relative_epoch) = match accessors::relative_epoch(&head_state.value, epoch) { - Ok(relative_epoch) => (head_state, relative_epoch), + let state = match accessors::relative_epoch(&head_state.value, epoch) { + Ok(_) => head_state, Err(_) => { - let start_slot = misc::compute_start_slot_at_epoch::

(epoch); - let state_with_status = state_id::state( + // Committees are known an epoch ahead, so the state at the start of the previous + // epoch serves them without advancing through that whole epoch. + let start_slot = misc::compute_start_slot_at_epoch::

(misc::previous_epoch(epoch)); + + state_id::state( &StateId::Slot(start_slot), &controller, &anchor_checkpoint_provider, - )?; - - (state_with_status, RelativeEpoch::Current) + )? } }; @@ -3138,35 +3123,8 @@ pub async fn validator_attester_duties( .into_iter() .collect::>(); - let committees_at_slot = accessors::get_committee_count_per_slot(&state, relative_epoch)?; - let response = misc::slots_in_epoch::

(epoch)? - .map(|slot| { - accessors::beacon_committees(&state, slot)? - .zip(0..) - .flat_map(|(committee, committee_index)| { - let state = &state; - - committee - .into_iter() - .enumerate() - .filter(|(_, validator_index)| indices.contains(validator_index)) - .map(move |(validator_committee_index, validator_index)| { - let pubkey = *accessors::public_key(state, validator_index)?; - - Ok(ValidatorAttesterDutyResponse { - committee_index, - committee_length: committee.len(), - committees_at_slot, - pubkey, - slot, - validator_committee_index, - validator_index, - }) - }) - }) - .collect::>>() - }) + .map(|slot| attester_duties_at_slot(&state, slot, &indices)) .flatten_ok() .try_collect() .map_err(Error::Internal)?; @@ -3176,14 +3134,34 @@ pub async fn validator_attester_duties( .execution_optimistic(status.is_optimistic())) } +fn current_epoch_with_clock_disparity( + controller: &ApiController, + chain_config: &ChainConfig, +) -> Result { + // Fork choice enters a slot only once it handles the slot's tick, which a request sent at + // the epoch boundary can beat by a few milliseconds. + let now = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?; + let time = now.saturating_add(chain_config.maximum_gossip_clock_disparity); + let tick = Tick::at_time_in_ms::

( + chain_config, + u64::try_from(time.as_millis())?, + controller.genesis_time(), + )?; + + Ok(tick + .epoch::

() + .max(misc::compute_epoch_at_slot::

(controller.slot()))) +} + /// `POST /eth/v1/validator/duties/ptc/{epoch}` pub async fn validator_ptc_duties( + State(chain_config): State>, State(controller): State>, State(anchor_checkpoint_provider): State>, EthPath(epoch): EthPath, EthJson(validator_indices): EthJson>, ) -> Result>, Error> { - let current_epoch = misc::compute_epoch_at_slot::

(controller.slot()); + let current_epoch = current_epoch_with_clock_disparity(&controller, &chain_config)?; if epoch > current_epoch.saturating_add(1) { return Err(Error::InvalidEpoch(anyhow!( @@ -3241,24 +3219,7 @@ pub async fn validator_ptc_duties( .into_iter() .collect::>(); - let response = misc::slots_in_epoch::

(epoch)? - .map(|slot| { - accessors::get_ptc(controller.chain_config(), &state, slot)? - .into_iter() - .filter(|validator_index| indices.contains(validator_index)) - .map(|validator_index| { - let pubkey = *accessors::public_key(&state, validator_index)?; - - Ok(ValidatorPTCDutyResponse { - pubkey, - validator_index, - slot, - }) - }) - .collect::>>() - }) - .flatten_ok() - .try_collect() + let response = ptc_duties_at_epoch(controller.chain_config(), &state, epoch, &indices) .map_err(Error::Internal)?; Ok(EthResponse::json(response) @@ -3340,7 +3301,7 @@ async fn get_proposer_duties( (state, status) }; - let dependent_root = if post_fulu_check && chain_config.phase_at_epoch(epoch) >= Phase::Fulu { + let dependent_root = if post_fulu_check && proposer_lookahead_applies(&chain_config, epoch) { // Post-Fulu get dependent root for the previous epoch because proposer suffling is done at // the last slot of epoch N - 2 with proposer lookahead. We should note that `dependent_root` // itself will return block root of the last slot in epoch `epoch` - 1. @@ -3351,21 +3312,8 @@ async fn get_proposer_duties( controller.dependent_root(&state, epoch)? }; - let response = misc::slots_in_epoch::

(epoch)? - .map(|slot| { - let validator_index = - accessors::get_beacon_proposer_index_at_slot(&chain_config, &state, slot)?; - - let pubkey = *accessors::public_key(&state, validator_index)?; - - Ok(ValidatorProposerDutyResponse { - pubkey, - validator_index, - slot, - }) - }) - .try_collect() - .map_err(Error::Internal)?; + let response = + proposer_duties_at_epoch(&chain_config, &state, epoch).map_err(Error::Internal)?; Ok(EthResponse::json(response) .dependent_root(dependent_root) @@ -3409,7 +3357,10 @@ pub async fn validator_sync_committee_duties( ) = if let Some(committee) = state_sync_committee(&head_state.value, requested_period) { (head_state, committee) } else { - let start_slot = misc::compute_start_slot_at_epoch::

(epoch); + // A committee is known a period ahead, so the state at the current epoch serves the next + // period even while fork choice is still finishing the previous one. + let current_epoch = current_epoch_with_clock_disparity(&controller, &chain_config)?; + let start_slot = misc::compute_start_slot_at_epoch::

(epoch.min(current_epoch)); let state = state_id::state( &StateId::Slot(start_slot), @@ -4169,7 +4120,7 @@ pub async fn validator_attestation_data( target, }; - EthResponse::json_or_ssz(attestation_data, &headers) + EthResponse::json_or_ssz(attestation_data, &headers).map_err(Into::into) } /// `POST /eth/v1/validator/beacon_committee_subscriptions` @@ -4355,6 +4306,11 @@ pub async fn validator_payload_attestation_data( }); } + // No block for the slot has been seen; the validator must not attest. + if head_slot < slot { + return Err(Error::BlockNotSeen); + } + let requested_epoch = misc::compute_epoch_at_slot::

(slot); let previous_epoch = misc::previous_epoch(misc::compute_epoch_at_slot::

(head_slot)); @@ -4430,7 +4386,7 @@ pub async fn validator_publish_aggregate_and_proofs_v1( State(controller): State>, State(api_to_p2p_tx): State>>, EthJsonOrSszWithOptionalPhase(aggregate_and_proofs, _): EthJsonOrSszWithOptionalPhase< - ContiguousList>, TargetAggregatorsPerCommittee>, + SignedAggregateAndProofs

, SignedAggregateAndProofListFromPhaseDeserializer

, >, ) -> Result<(), Error> { @@ -4454,7 +4410,7 @@ pub async fn validator_publish_aggregate_and_proofs_v2( State(controller): State>, State(api_to_p2p_tx): State>>, EthJsonOrSsz(aggregate_and_proofs, _): EthJsonOrSsz< - ContiguousList>, TargetAggregatorsPerCommittee>, + SignedAggregateAndProofs

, SignedAggregateAndProofListFromPhaseDeserializer

, >, ) -> Result<(), Error> { @@ -4464,10 +4420,7 @@ pub async fn validator_publish_aggregate_and_proofs_v2( async fn validator_publish_aggregate_and_proofs( controller: &ApiController, api_to_p2p_tx: &UnboundedSender>, - aggregate_and_proofs: ContiguousList< - Arc>, - TargetAggregatorsPerCommittee, - >, + aggregate_and_proofs: SignedAggregateAndProofs

, ) -> Result<(), Error> { let (successes, failures): (Vec<_>, Vec<_>) = aggregate_and_proofs .into_iter() @@ -4737,78 +4690,6 @@ pub async fn validator_sync_committee_selections() -> Error { Error::EndpointNotImplemented } -/// `POST /eth/v2/debug/tracing/log_level` -#[instrument(skip_all, level = "debug", name = "http_api::post_log_level")] -pub async fn post_log_level( - State(handle): State>, - Json(req): Json, -) -> impl IntoResponse { - let directive_str = format!("{}={}", req.target, req.level); - - match directive_str.parse() { - Ok(directive) => { - if let Some(handle) = handle { - if let Err(e) = handle.modify_log(|filter| { - let old = core::mem::take(filter); - *filter = old.add_directive(directive); - }) { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("failed to reload filter: {e}"), - ); - } - (StatusCode::OK, "log level updated".to_owned()) - } else { - ( - StatusCode::SERVICE_UNAVAILABLE, - "tracing not available".to_owned(), - ) - } - } - Err(e) => (StatusCode::BAD_REQUEST, format!("invalid directive: {e}")), - } -} - -/// `POST /eth/v2/debug/tracing/trace_level` -#[instrument(skip_all, level = "debug", name = "http_api::post_trace_level")] -pub async fn post_trace_level( - State(handle): State>, - Json(req): Json, -) -> impl IntoResponse { - let TraceLevelRequest { target, level } = req; - - let directive_str = target - .map(|target| format!("{target}={level}")) - .unwrap_or(level); - - match directive_str.parse() { - Ok(directive) => { - if let Some(handle) = handle { - match handle.modify_trace(|filter| { - let old = core::mem::take(filter); - *filter = old.add_directive(directive); - }) { - Ok(Some(())) => (StatusCode::OK, "trace level updated".to_owned()), - Ok(None) => ( - StatusCode::SERVICE_UNAVAILABLE, - "telemetry not available".to_owned(), - ), - Err(e) => ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("failed to reload filter: {e}"), - ), - } - } else { - ( - StatusCode::SERVICE_UNAVAILABLE, - "tracing not available".to_owned(), - ) - } - } - Err(e) => (StatusCode::BAD_REQUEST, format!("invalid directive: {e}")), - } -} - fn state_validators( controller: &ApiController, anchor_checkpoint_provider: &AnchorCheckpointProvider

, @@ -6064,18 +5945,42 @@ async fn publish_signed_execution_payload_envelope( signed_envelope: Arc>, broadcast_validation: BroadcastValidation, ) -> Result { - let (sender, mut receiver) = futures::channel::mpsc::channel(1); - if broadcast_validation == BroadcastValidation::Gossip { + let (sender, mut receiver) = futures::channel::mpsc::channel(1); + controller .on_api_execution_payload_envelope_for_gossip(signed_envelope.clone_arc(), sender); - } else { - controller.on_api_execution_payload_envelope(signed_envelope.clone_arc(), sender); + + // The gossip checks only answer; publishing and importing are still to be done. + match receiver.next().await.transpose() { + Ok(Some(ValidationOutcome::Accept)) => { + ApiToP2p::PublishExecutionPayloadEnvelope(signed_envelope.clone_arc()) + .send(api_to_p2p_tx); + } + Ok(Some(ValidationOutcome::Ignore(publishable))) => { + if publishable { + ApiToP2p::PublishExecutionPayloadEnvelope(signed_envelope.clone_arc()) + .send(api_to_p2p_tx); + } + + return Ok(StatusCode::ACCEPTED); + } + Ok(None) => { + return Err(Error::InvalidPayloadEnvelope(anyhow!( + "received no envelope validation response", + ))); + } + Err(error) => return Err(Error::InvalidPayloadEnvelope(error)), + } } + let (sender, mut receiver) = futures::channel::mpsc::channel(1); + + controller.on_api_execution_payload_envelope(signed_envelope.clone_arc(), sender); + let status_code = match receiver.next().await.transpose() { Ok(Some(ValidationOutcome::Accept)) => match broadcast_validation { - // The envelope was already published by the gossip-checks path above. + // Published above once the gossip checks passed. BroadcastValidation::Gossip => StatusCode::OK, BroadcastValidation::Consensus => { ApiToP2p::PublishExecutionPayloadEnvelope(signed_envelope).send(api_to_p2p_tx); @@ -6101,7 +6006,13 @@ async fn publish_signed_execution_payload_envelope( StatusCode::ACCEPTED } Ok(None) => { + // Under gossip validation the envelope is already on the wire. if broadcast_validation == BroadcastValidation::Gossip { + warn_with_peers!( + "received no envelope validation response for HTTP API \ + (envelope: {signed_envelope:?})" + ); + StatusCode::ACCEPTED } else { return Err(Error::InvalidPayloadEnvelope(anyhow!( @@ -6111,6 +6022,11 @@ async fn publish_signed_execution_payload_envelope( } Err(error) => { if broadcast_validation == BroadcastValidation::Gossip { + warn_with_peers!( + "received invalid envelope through HTTP API \ + (envelope: {signed_envelope:?}, error: {error})" + ); + StatusCode::ACCEPTED } else { return Err(Error::InvalidPayloadEnvelope(error)); @@ -6201,6 +6117,21 @@ mod tests { use super::*; + // `produceBlockV4` takes the boost factor in the body, not the query. + #[tokio::test] + async fn parses_produce_block_v4_query_without_a_boost_factor() -> Result<()> { + let query = extract_query::(format!( + "randao_reveal={:?}&include_payload=true", + SignatureBytes::default(), + )) + .await?; + + assert!(query.include_payload); + assert!(query.graffiti.is_none()); + + Ok(()) + } + #[tokio::test] async fn test_deserialize_for_attestation() -> Result<()> { let attestations = [Phase0Attestation { diff --git a/http_api/src/test_endpoints.rs b/http_api/src/test_endpoints.rs index 62dba7bd5..3b5955480 100644 --- a/http_api/src/test_endpoints.rs +++ b/http_api/src/test_endpoints.rs @@ -66,11 +66,17 @@ pub async fn post_take_messages( validator_to_p2p_rx, } = test_state; + let mut validator_to_p2p = take(validator_to_p2p_rx).await; + + // Sort proposer preferences to the end, as their own task races the block proposal. + validator_to_p2p + .sort_by_key(|message| matches!(message, ValidatorToP2p::PublishProposerPreferences(_))); + Json(TakeMessagesResponse { api_to_p2p: take(api_to_p2p_rx).await, fc_to_p2p: take(fc_to_p2p_rx).await, fc_to_sync: take(fc_to_sync_rx).await, pool_to_p2p: take(pool_to_p2p_rx).await, - validator_to_p2p: take(validator_to_p2p_rx).await, + validator_to_p2p, }) } diff --git a/http_api_utils/Cargo.toml b/http_api_utils/Cargo.toml index 420cfb721..dae9f62b5 100644 --- a/http_api_utils/Cargo.toml +++ b/http_api_utils/Cargo.toml @@ -9,15 +9,22 @@ workspace = true [dependencies] anyhow = { workspace = true } axum = { workspace = true } +binary_utils = { workspace = true } +bls = { workspace = true } features = { workspace = true } http = { workspace = true } http-body-util = { workspace = true } itertools = { workspace = true } tracing = { workspace = true } logging = { workspace = true } +mediatype = { workspace = true } mime = { workspace = true } parse-display = { workspace = true } prometheus_metrics = { workspace = true } +serde = { workspace = true } +serde_utils = { workspace = true } +ssz = { workspace = true } +tap = { workspace = true } thiserror = { workspace = true } tower = { workspace = true } tower-http = { workspace = true } @@ -26,3 +33,4 @@ types = { workspace = true } [dev-dependencies] hex-literal = { workspace = true } test-case = { workspace = true } +tokio = { workspace = true } diff --git a/http_api_utils/src/debug_tracing.rs b/http_api_utils/src/debug_tracing.rs new file mode 100644 index 000000000..70d57c9e7 --- /dev/null +++ b/http_api_utils/src/debug_tracing.rs @@ -0,0 +1,88 @@ +use axum::{Json, extract::State, http::StatusCode, response::IntoResponse}; +use binary_utils::TracingHandle; +use serde::Deserialize; +use tracing::instrument; + +#[derive(Deserialize)] +pub struct LogLevelRequest { + target: String, + level: String, +} + +#[derive(Deserialize)] +pub struct TraceLevelRequest { + target: Option, + level: String, +} + +/// `POST /eth/v2/debug/tracing/log_level` +#[instrument(skip_all, level = "debug", name = "http_api::post_log_level")] +pub async fn post_log_level( + State(handle): State>, + Json(req): Json, +) -> impl IntoResponse { + let directive_str = format!("{}={}", req.target, req.level); + + match directive_str.parse() { + Ok(directive) => { + if let Some(handle) = handle { + if let Err(e) = handle.modify_log(|filter| { + let old = core::mem::take(filter); + *filter = old.add_directive(directive); + }) { + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("failed to reload filter: {e}"), + ); + } + (StatusCode::OK, "log level updated".to_owned()) + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + "tracing not available".to_owned(), + ) + } + } + Err(e) => (StatusCode::BAD_REQUEST, format!("invalid directive: {e}")), + } +} + +/// `POST /eth/v2/debug/tracing/trace_level` +#[instrument(skip_all, level = "debug", name = "http_api::post_trace_level")] +pub async fn post_trace_level( + State(handle): State>, + Json(req): Json, +) -> impl IntoResponse { + let TraceLevelRequest { target, level } = req; + + let directive_str = target + .map(|target| format!("{target}={level}")) + .unwrap_or(level); + + match directive_str.parse() { + Ok(directive) => { + if let Some(handle) = handle { + match handle.modify_trace(|filter| { + let old = core::mem::take(filter); + *filter = old.add_directive(directive); + }) { + Ok(Some(())) => (StatusCode::OK, "trace level updated".to_owned()), + Ok(None) => ( + StatusCode::SERVICE_UNAVAILABLE, + "telemetry not available".to_owned(), + ), + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("failed to reload filter: {e}"), + ), + } + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + "tracing not available".to_owned(), + ) + } + } + Err(e) => (StatusCode::BAD_REQUEST, format!("invalid directive: {e}")), + } +} diff --git a/http_api_utils/src/error.rs b/http_api_utils/src/error.rs index cbe510bcd..0ea1f46ce 100644 --- a/http_api_utils/src/error.rs +++ b/http_api_utils/src/error.rs @@ -3,10 +3,11 @@ use std::sync::Arc; use anyhow::Error as AnyhowError; use axum::{ - Extension, + Extension, Json, http::{StatusCode, Uri}, response::{IntoResponse, Response}, }; +use serde::Serialize; use thiserror::Error; use tracing::instrument; @@ -20,6 +21,8 @@ pub enum Error { uri: Uri, source: AnyhowError, }, + #[error("internal error")] + Internal(#[from] AnyhowError), } impl ApiError for Error { @@ -37,8 +40,15 @@ impl IntoResponse for Error { #[instrument(skip_all, level = "debug")] fn into_response(self) -> Response { let status_code = self.status_code(); + + let body = Json(EthErrorResponse { + code: status_code.as_u16(), + message: self.format_sources(), + }) + .into_response(); + let extension = Extension(Arc::new(self)); - (status_code, extension).into_response() + (status_code, extension, body).into_response() } } @@ -46,6 +56,41 @@ impl Error { const fn status_code(&self) -> StatusCode { match self { Self::InvalidBody { .. } => StatusCode::BAD_REQUEST, + Self::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } + +#[derive(Serialize)] +struct EthErrorResponse { + // The absence of `#[serde(with = "serde_utils::string_or_native")]` is intentional. + // The `code` field is supposed to contain a number. + code: u16, + message: String, +} + +#[cfg(test)] +mod tests { + use anyhow::anyhow; + use http_body_util::BodyExt as _; + + use super::*; + + // The body is what a beacon-API client expects of any error, so a failed response is not + // left bodiless. + #[tokio::test] + async fn an_internal_error_carries_the_standard_error_body() -> Result<(), AnyhowError> { + let response = Error::Internal(anyhow!("boom")).into_response(); + + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + + let body = response.into_body().collect().await?.to_bytes(); + + assert_eq!( + body.as_ref(), + br#"{"code":500,"message":"internal error: boom"}"#, + ); + + Ok(()) + } +} diff --git a/http_api_utils/src/lib.rs b/http_api_utils/src/lib.rs index 3901ab68e..68c4e3863 100644 --- a/http_api_utils/src/lib.rs +++ b/http_api_utils/src/lib.rs @@ -1,9 +1,18 @@ pub use block_id::BlockId; +pub use debug_tracing::{post_log_level, post_trace_level}; pub use helpers::{ PhaseHeaderError, extend_router_with_middleware, extract_phase_from_headers, try_extract_phase_from_headers, }; -pub use misc::{ApiMetrics, Direction, ETH_CONSENSUS_VERSION}; +pub use misc::{ + ApiMetrics, BlockHeadersResponse, Direction, ETH_CONSENSUS_VERSION, + ValidatorAttesterDutyResponse, ValidatorLivenessResponse, ValidatorPTCDutyResponse, + ValidatorProposerDutyResponse, ValidatorSyncDutyResponse, +}; +pub use response::{ + AlwaysJson, ETH_BLOB_DATA_INCLUDED, ETH_BUILDER_URL, ETH_EXECUTION_PAYLOAD_BLINDED, + ETH_EXECUTION_PAYLOAD_INCLUDED, EthResponse, JsonOrSsz, +}; pub use state_id::StateId; pub use traits::ApiError; @@ -11,8 +20,10 @@ pub mod logging; pub mod middleware; mod block_id; +mod debug_tracing; mod error; mod helpers; mod misc; +mod response; mod state_id; mod traits; diff --git a/http_api_utils/src/logging.rs b/http_api_utils/src/logging.rs index 4622ed20a..4e29ba8a3 100644 --- a/http_api_utils/src/logging.rs +++ b/http_api_utils/src/logging.rs @@ -11,7 +11,7 @@ use features::Feature; use logging::{info_with_peers, warn_with_peers}; use tracing::Span; -use crate::{ApiError, misc::ApiMetrics}; +use crate::{ApiError, error::Error, misc::ApiMetrics}; // By default, `TraceLayer` emits events at `DEBUG` with the default target. // Our application filters them out. @@ -67,11 +67,19 @@ pub fn log_response( to ({method} {original_uri} {version:?}) \ for {remote} in {latency:?}", ), - response.extensions().get::>(), + response + .extensions() + .get::>() + .map(|error| error.format_sources()) + // Responses that fail to build carry this crate's error rather than `E`. + .or_else(|| { + response + .extensions() + .get::>() + .map(|error| error.format_sources()) + }), ) { - (shared, Some(error)) => { - info_with_peers!("{shared} (error: {})", error.format_sources()) - } + (shared, Some(error)) => info_with_peers!("{shared} (error: {error})"), (shared, None) => info_with_peers!("{shared}"), } } diff --git a/http_api_utils/src/misc.rs b/http_api_utils/src/misc.rs index 65ecdc3bb..812d35c69 100644 --- a/http_api_utils/src/misc.rs +++ b/http_api_utils/src/misc.rs @@ -1,11 +1,81 @@ use core::time::Duration; use std::sync::Arc; +use bls::PublicKeyBytes; use parse_display::Display; use prometheus_metrics::Metrics; +use serde::{Deserialize, Serialize}; +use types::phase0::{ + containers::SignedBeaconBlockHeader, + primitives::{CommitteeIndex, H256, Slot, ValidatorIndex}, +}; pub const ETH_CONSENSUS_VERSION: &str = "eth-consensus-version"; +/// +#[derive(Clone, Copy, PartialEq, Eq, Debug, Deserialize, Serialize)] +pub struct ValidatorAttesterDutyResponse { + #[serde(with = "serde_utils::string_or_native")] + pub committee_index: CommitteeIndex, + #[serde(with = "serde_utils::string_or_native")] + pub committee_length: usize, + #[serde(with = "serde_utils::string_or_native")] + pub committees_at_slot: u64, + pub pubkey: PublicKeyBytes, + #[serde(with = "serde_utils::string_or_native")] + pub slot: Slot, + #[serde(with = "serde_utils::string_or_native")] + pub validator_committee_index: usize, + #[serde(with = "serde_utils::string_or_native")] + pub validator_index: ValidatorIndex, +} + +/// +#[derive(Clone, Copy, PartialEq, Eq, Debug, Deserialize, Serialize)] +pub struct ValidatorProposerDutyResponse { + pub pubkey: PublicKeyBytes, + #[serde(with = "serde_utils::string_or_native")] + pub validator_index: ValidatorIndex, + #[serde(with = "serde_utils::string_or_native")] + pub slot: Slot, +} + +/// +#[derive(Clone, PartialEq, Eq, Debug, Deserialize, Serialize)] +pub struct ValidatorSyncDutyResponse { + pub pubkey: PublicKeyBytes, + #[serde(with = "serde_utils::string_or_native")] + pub validator_index: ValidatorIndex, + #[serde(with = "serde_utils::string_or_native_sequence")] + pub validator_sync_committee_indices: Vec, +} + +/// +#[derive(Clone, Copy, PartialEq, Eq, Debug, Deserialize, Serialize)] +pub struct ValidatorPTCDutyResponse { + pub pubkey: PublicKeyBytes, + #[serde(with = "serde_utils::string_or_native")] + pub validator_index: ValidatorIndex, + #[serde(with = "serde_utils::string_or_native")] + pub slot: Slot, +} + +/// +#[derive(Clone, Copy, PartialEq, Eq, Debug, Deserialize, Serialize)] +pub struct ValidatorLivenessResponse { + #[serde(with = "serde_utils::string_or_native")] + pub index: ValidatorIndex, + pub is_live: bool, +} + +/// +#[derive(Clone, Copy, PartialEq, Eq, Debug, Deserialize, Serialize)] +pub struct BlockHeadersResponse { + pub root: H256, + pub canonical: bool, + pub header: SignedBeaconBlockHeader, +} + #[derive(Clone, Copy)] enum ApiType { Http, diff --git a/http_api/src/response.rs b/http_api_utils/src/response.rs similarity index 87% rename from http_api/src/response.rs rename to http_api_utils/src/response.rs index 08c44c22f..16412a429 100644 --- a/http_api/src/response.rs +++ b/http_api_utils/src/response.rs @@ -1,37 +1,40 @@ +use core::fmt::Display; + use anyhow::Result; use axum::{ Json, http::{HeaderMap, HeaderValue, header::ACCEPT}, response::{IntoResponse, Response}, }; -use builder_api::gloas::containers::BuilderUrl; -use http_api_utils::ETH_CONSENSUS_VERSION; use logging::warn_with_peers; use mediatype::{MediaType, MediaTypeList}; use mime::APPLICATION_OCTET_STREAM; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use ssz::SszWrite; use tap::Pipe as _; use tracing::instrument; use types::{bellatrix::primitives::Wei, nonstandard::Phase, phase0::primitives::H256}; -use crate::error::Error; +use crate::{error::Error, misc::ETH_CONSENSUS_VERSION}; const ETH_CONSENSUS_BLOCK_VALUE: &str = "eth-consensus-block-value"; pub const ETH_BLOB_DATA_INCLUDED: &str = "eth-blob-data-included"; -const ETH_EXECUTION_PAYLOAD_BLINDED: &str = "eth-execution-payload-blinded"; +pub const ETH_EXECUTION_PAYLOAD_BLINDED: &str = "eth-execution-payload-blinded"; const ETH_EXECUTION_PAYLOAD_VALUE: &str = "eth-execution-payload-value"; -const ETH_EXECUTION_PAYLOAD_INCLUDED: &str = "eth-execution-payload-included"; +pub const ETH_EXECUTION_PAYLOAD_INCLUDED: &str = "eth-execution-payload-included"; pub const ETH_BUILDER_URL: &str = "eth-builder-url"; +#[derive(Default)] pub struct AlwaysJson; +#[derive(Default)] pub enum JsonOrSsz { + #[default] Json, Ssz, } -#[derive(Serialize)] +#[derive(Serialize, Deserialize)] pub struct EthResponse { data: T, @@ -59,7 +62,7 @@ pub struct EthResponse { // These are returned only in headers. #[serde(skip)] - builder_url: Option, + builder_url: Option, #[serde(skip)] format: F, @@ -114,48 +117,81 @@ impl EthResponse { } } + #[must_use] + pub fn into_data(self) -> T { + self.data + } + + #[must_use] + pub fn into_data_and_dependent_root(self) -> (T, Option) { + (self.data, self.dependent_root) + } + + #[must_use] + pub fn into_data_and_execution_optimistic(self) -> (T, Option) { + (self.data, self.execution_optimistic) + } + + #[must_use] pub const fn version(mut self, phase: Phase) -> Self { self.version = Some(phase); self } + #[must_use] pub const fn consensus_block_value(mut self, consensus_block_value: Option) -> Self { self.consensus_block_value = consensus_block_value; self } + #[must_use] pub const fn execution_payload_blinded(mut self, execution_payload_blinded: bool) -> Self { self.execution_payload_blinded = Some(execution_payload_blinded); self } + #[must_use] pub const fn execution_payload_value(mut self, execution_payload_value: Wei) -> Self { self.execution_payload_value = Some(execution_payload_value); self } + #[must_use] pub const fn execution_payload_included(mut self, execution_payload_included: bool) -> Self { self.execution_payload_included = Some(execution_payload_included); self } + #[must_use] pub const fn dependent_root(mut self, dependent_root: H256) -> Self { self.dependent_root = Some(dependent_root); self } + #[must_use] pub const fn execution_optimistic(mut self, execution_optimistic: bool) -> Self { self.execution_optimistic = Some(execution_optimistic); self } + #[must_use] pub const fn finalized(mut self, finalized: bool) -> Self { self.finalized = Some(finalized); self } - pub fn builder_url(mut self, builder_url: Option) -> Self { - self.builder_url = builder_url; + // `Display` is only used for logging, so a URL can be redacted there. + #[must_use] + pub fn builder_url(mut self, builder_url: Option + Display>) -> Self { + // The block is more important than the header, so an unencodable URL is left out. + self.builder_url = builder_url.and_then(|builder_url| { + HeaderValue::from_str(builder_url.as_ref()) + .inspect_err(|error| { + warn_with_peers!("builder URL {builder_url} is not a valid header: {error}"); + }) + .ok() + }); + self } @@ -187,16 +223,8 @@ impl EthResponse { response_headers.insert(ETH_EXECUTION_PAYLOAD_INCLUDED, header_value); } - // The block is more important than the header, so an unencodable URL is left out. - if let Some(builder_url) = &self.builder_url { - match HeaderValue::from_str(builder_url.as_str()) { - Ok(header_value) => { - response_headers.insert(ETH_BUILDER_URL, header_value); - } - Err(error) => { - warn_with_peers!("builder URL {builder_url} is not a valid header: {error}"); - } - } + if let Some(header_value) = self.builder_url.clone() { + response_headers.insert(ETH_BUILDER_URL, header_value); } Ok(response_headers) @@ -238,6 +266,7 @@ impl EthResponse { } impl EthResponse { + #[must_use] pub fn meta(self, meta: M) -> EthResponse { let Self { data, @@ -281,14 +310,9 @@ impl EthResponse { // `axum` recommends using `axum::TypedHeader` instead of extracting all headers, // but the `headers` crate does not provide a type for the `Accept` header. // See . - pub fn json_or_ssz(data: T, request_headers: &HeaderMap) -> Result { + pub fn json_or_ssz(data: T, request_headers: &HeaderMap) -> Result { if let Some(accept_header) = request_headers.get(ACCEPT) { - let accepted_types = accept_content_type_list( - accept_header - .to_str() - .map_err(Into::into) - .map_err(Error::Internal)?, - )?; + let accepted_types = accept_content_type_list(accept_header.to_str()?)?; for accept in &accepted_types { if accept == APPLICATION_OCTET_STREAM.as_ref() { diff --git a/http_api_utils/src/traits.rs b/http_api_utils/src/traits.rs index 48a0973db..f7474e9e9 100644 --- a/http_api_utils/src/traits.rs +++ b/http_api_utils/src/traits.rs @@ -1,4 +1,4 @@ -use core::{error::Error as StdError, fmt::Display}; +use core::error::Error as StdError; use itertools::Itertools as _; @@ -8,7 +8,7 @@ pub trait ApiError { // - // - // - - fn format_sources(&self) -> impl Display + '_ { + fn format_sources(&self) -> String { self.sources().join(": ") } diff --git a/keymanager/Cargo.toml b/keymanager/Cargo.toml index fc34696dc..baf23a7bd 100644 --- a/keymanager/Cargo.toml +++ b/keymanager/Cargo.toml @@ -15,6 +15,7 @@ eip_2335 = { workspace = true } fs-err = { workspace = true } futures = { workspace = true } helper_functions = { workspace = true } +hex-literal = { workspace = true } itertools = { workspace = true } logging = { workspace = true } serde = { workspace = true } @@ -37,7 +38,6 @@ zeroize = { workspace = true } [dev-dependencies] fs-err = { workspace = true } -hex-literal = { workspace = true } reqwest = { workspace = true } tempfile = { workspace = true } diff --git a/keymanager/src/keystores.rs b/keymanager/src/keystores.rs index 43c0440fd..faeaefa8c 100644 --- a/keymanager/src/keystores.rs +++ b/keymanager/src/keystores.rs @@ -857,7 +857,6 @@ mod tests { let signer = Arc::new(Signer::new( vec![], Client::new(), - Client::new(), Web3SignerConfig::default(), None, )); @@ -1149,7 +1148,6 @@ mod tests { KeyOrigin::External, )], Client::new(), - Client::new(), Web3SignerConfig::default(), None, )); diff --git a/keymanager/src/lib.rs b/keymanager/src/lib.rs index c47b31d58..b62e2ec64 100644 --- a/keymanager/src/lib.rs +++ b/keymanager/src/lib.rs @@ -9,7 +9,8 @@ pub use crate::{ BuilderSettings, ResolvedBuilderConfig, ResolvedBuilderEntry, resolve_builder_config, }, proposer_configs::{ - LegacyMigration, ProposerConfigs, migrate_legacy_database, prune_legacy_database, + GRANDINE_DONATION_ADDRESS, LegacyMigration, ProposerConfigs, migrate_legacy_database, + prune_legacy_database, }, validator_definitions::{ BuilderOptions, DefinitionsStorage, KeystorePassword, SigningMethod, ValidatorDefinition, @@ -27,7 +28,7 @@ use futures::lock::Mutex; use signer::Signer; use slashing_protection::SlashingProtector; use std_ext::ArcExt as _; -use types::phase0::primitives::{ExecutionAddress, H256}; +use types::phase0::primitives::H256; use crate::{keystores::KeystoreManager, remote_keys::RemoteKeyManager}; @@ -45,25 +46,13 @@ pub struct KeyManager { } impl KeyManager { - #[expect(clippy::too_many_arguments)] pub fn new_in_memory( signer: Arc, slashing_protector: Arc>, genesis_validators_root: H256, - default_fee_recipient: ExecutionAddress, - default_gas_limit: Option, - default_graffiti: H256, - builder_settings: BuilderSettings, + proposer_configs: Arc, validator_definitions: Arc, ) -> Self { - let proposer_configs = Arc::new(ProposerConfigs::new( - default_fee_recipient, - default_gas_limit, - default_graffiti, - builder_settings, - validator_definitions.clone_arc(), - )); - let keystore_manager = KeystoreManager::new_in_memory( signer.clone_arc(), slashing_protector.clone_arc(), @@ -89,20 +78,9 @@ impl KeyManager { validator_directory: PathBuf, secrets_directory: PathBuf, keystore_storage_password_path: Option<&Path>, - default_fee_recipient: ExecutionAddress, - default_gas_limit: Option, - default_graffiti: H256, - builder_settings: BuilderSettings, + proposer_configs: Arc, validator_definitions: Arc, ) -> Result { - let proposer_configs = Arc::new(ProposerConfigs::new( - default_fee_recipient, - default_gas_limit, - default_graffiti, - builder_settings, - validator_definitions.clone_arc(), - )); - let keystore_manager = KeystoreManager::new_persistent( signer.clone_arc(), slashing_protector.clone_arc(), diff --git a/keymanager/src/proposer_configs.rs b/keymanager/src/proposer_configs.rs index 6885cbc68..03b80a121 100644 --- a/keymanager/src/proposer_configs.rs +++ b/keymanager/src/proposer_configs.rs @@ -1,4 +1,4 @@ -use std::{io::ErrorKind, path::Path, str, sync::Arc}; +use std::{collections::HashMap, io::ErrorKind, path::Path, str, sync::Arc}; use anyhow::Result; use bls::PublicKeyBytes; @@ -6,11 +6,12 @@ use bytesize::ByteSize; use database::{Database, DatabaseMode}; use derive_more::Display; use helper_functions::misc; +use hex_literal::hex; use serde::de::DeserializeOwned; use ssz::Uint256; use types::{ bellatrix::primitives::Gas, - phase0::primitives::{ExecutionAddress, Gwei, H256}, + phase0::primitives::{ExecutionAddress, Gwei, H160, H256}, }; use crate::{ @@ -26,8 +27,11 @@ use crate::{ const DB_MAX_SIZE: ByteSize = ByteSize::gib(1); +pub const GRANDINE_DONATION_ADDRESS: ExecutionAddress = + H160(hex!("e7cf7C3BA875Dd3884Ed6a9082d342cb4FBb1f1b")); + pub struct ProposerConfigs { - default_fee_recipient: ExecutionAddress, + default_fee_recipient: Option, default_gas_limit: Option, default_graffiti: H256, builder_settings: BuilderSettings, @@ -37,7 +41,7 @@ pub struct ProposerConfigs { impl ProposerConfigs { #[must_use] pub const fn new( - default_fee_recipient: ExecutionAddress, + default_fee_recipient: Option, default_gas_limit: Option, default_graffiti: H256, builder_settings: BuilderSettings, @@ -54,11 +58,36 @@ impl ProposerConfigs { #[must_use] pub fn fee_recipient(&self, pubkey: PublicKeyBytes) -> ExecutionAddress { + self.configured_fee_recipient(pubkey) + .unwrap_or(GRANDINE_DONATION_ADDRESS) + } + + /// [`Self::fee_recipient`] without the fallback, for what is told to other beacon nodes. + #[must_use] + pub fn configured_fee_recipient(&self, pubkey: PublicKeyBytes) -> Option { self.validator_definitions .read() .get(pubkey) .and_then(|definition| definition.fee_recipient) - .unwrap_or(self.default_fee_recipient) + .or(self.default_fee_recipient) + } + + /// [`Self::configured_fee_recipient`] for many keys, with the definitions read only once. + pub fn configured_fee_recipients(&self) -> impl Fn(PublicKeyBytes) -> Option { + let fee_recipients = self + .validator_definitions + .read() + .fee_recipients() + .collect::>(); + + let default_fee_recipient = self.default_fee_recipient; + + move |pubkey| { + fee_recipients + .get(&pubkey) + .copied() + .or(default_fee_recipient) + } } pub fn set_fee_recipient( @@ -502,7 +531,7 @@ mod tests { )); Ok(ProposerConfigs::new( - DEFAULT_FEE_RECIPIENT, + Some(DEFAULT_FEE_RECIPIENT), Some(DEFAULT_GAS_LIMIT), graffiti_bytes, BuilderSettings::default(), @@ -522,6 +551,66 @@ mod tests { Ok(()) } + // Only a configured fee recipient may be told to other beacon nodes. + #[test] + fn configured_fee_recipient_has_no_fallback() -> Result<()> { + let validator_definitions = Arc::new(ValidatorDefinitionsWithStorage::new( + Arc::new(RwLock::new(definitions_with_pubkey())), + DefinitionsStorage::InMemory, + )); + + let proposer_configs = ProposerConfigs::new( + None, + Some(DEFAULT_GAS_LIMIT), + H256::default(), + BuilderSettings::default(), + validator_definitions, + ); + + assert_eq!(proposer_configs.configured_fee_recipient(PUBKEY), None); + assert_eq!( + proposer_configs.fee_recipient(PUBKEY), + GRANDINE_DONATION_ADDRESS + ); + + proposer_configs.set_fee_recipient(PUBKEY, TEST_FEE_RECIPIENT)?; + + assert_eq!( + proposer_configs.configured_fee_recipient(PUBKEY), + Some(TEST_FEE_RECIPIENT), + ); + + Ok(()) + } + + // Proposers are prepared from the lookup for many keys, which must answer as the one for a + // single key does: the definition's own recipient, else the default. + #[test] + fn configured_fee_recipients_match_the_lookup_for_one_key() -> Result<()> { + let proposer_configs = in_memory()?; + let pubkey_without_definition = PublicKeyBytes::repeat_byte(2); + + proposer_configs.set_fee_recipient(PUBKEY, TEST_FEE_RECIPIENT)?; + + let configured_fee_recipient = proposer_configs.configured_fee_recipients(); + + assert_eq!(configured_fee_recipient(PUBKEY), Some(TEST_FEE_RECIPIENT)); + + assert_eq!( + configured_fee_recipient(pubkey_without_definition), + Some(DEFAULT_FEE_RECIPIENT), + ); + + for pubkey in [PUBKEY, pubkey_without_definition] { + assert_eq!( + configured_fee_recipient(pubkey), + proposer_configs.configured_fee_recipient(pubkey), + ); + } + + Ok(()) + } + #[test] fn test_set_and_get_fee_recipient() -> Result<()> { let proposer_configs = in_memory()?; @@ -703,7 +792,7 @@ mod tests { )); let proposer_configs = ProposerConfigs::new( - DEFAULT_FEE_RECIPIENT, + Some(DEFAULT_FEE_RECIPIENT), Some(DEFAULT_GAS_LIMIT), graffiti_bytes, BuilderSettings::default(), @@ -720,7 +809,7 @@ mod tests { )); let reloaded = ProposerConfigs::new( - DEFAULT_FEE_RECIPIENT, + Some(DEFAULT_FEE_RECIPIENT), Some(DEFAULT_GAS_LIMIT), graffiti_bytes, BuilderSettings::default(), diff --git a/keymanager/src/remote_keys.rs b/keymanager/src/remote_keys.rs index 6a488d0f2..0fb192dd5 100644 --- a/keymanager/src/remote_keys.rs +++ b/keymanager/src/remote_keys.rs @@ -195,7 +195,6 @@ mod tests { KeyOrigin::External, )], Client::new(), - Client::new(), Web3SignerConfig::default(), None, )) @@ -365,7 +364,6 @@ mod tests { let signer = Arc::new(Signer::new( core::iter::empty(), Client::new(), - Client::new(), Web3SignerConfig::default(), None, )); diff --git a/keymanager/src/validator_definitions.rs b/keymanager/src/validator_definitions.rs index 42bef8710..d07376e93 100644 --- a/keymanager/src/validator_definitions.rs +++ b/keymanager/src/validator_definitions.rs @@ -344,6 +344,12 @@ impl ValidatorDefinitions { .map(|definition| definition.pubkey) } + pub fn fee_recipients(&self) -> impl Iterator + '_ { + self.0 + .iter() + .filter_map(|definition| Some((definition.pubkey, definition.fee_recipient?))) + } + /// The `(url, pubkey)` of every enabled `web3signer` entry. pub fn web3signer_definitions(&self) -> impl Iterator { self.0 diff --git a/liveness_tracker/Cargo.toml b/liveness_tracker/Cargo.toml index dc695f3e8..12766bae3 100644 --- a/liveness_tracker/Cargo.toml +++ b/liveness_tracker/Cargo.toml @@ -16,7 +16,6 @@ futures = { workspace = true } helper_functions = { workspace = true } itertools = { workspace = true } logging = { workspace = true } -operation_pools = { workspace = true } prometheus_metrics = { workspace = true } tracing = { workspace = true } tynm = { workspace = true } diff --git a/liveness_tracker/src/lib.rs b/liveness_tracker/src/lib.rs index 7670a19b6..c68aebf95 100644 --- a/liveness_tracker/src/lib.rs +++ b/liveness_tracker/src/lib.rs @@ -10,10 +10,8 @@ use futures::{StreamExt as _, channel::mpsc::UnboundedReceiver, select}; use helper_functions::{electra, gloas, misc, phase0}; use itertools::Itertools as _; use logging::{debug_with_peers, warn_with_peers}; -use operation_pools::PoolToLivenessMessage; use prometheus_metrics::Metrics; use types::{ - altair::containers::SyncCommitteeMessage, combined::{Attestation, BeaconState, SignedBeaconBlock}, electra::containers::IndexedAttestation as ElectraIndexedAttestation, phase0::primitives::{Epoch, ValidatorIndex}, @@ -34,7 +32,6 @@ pub struct LivenessTracker { metrics: Option>, too_many_empty_slots_message_shown_at: Option, api_to_liveness_rx: UnboundedReceiver, - pool_to_liveness_rx: UnboundedReceiver, validator_to_liveness_rx: UnboundedReceiver>, } @@ -44,7 +41,6 @@ impl LivenessTracker { controller: ApiController, metrics: Option>, api_to_liveness_rx: UnboundedReceiver, - pool_to_liveness_rx: UnboundedReceiver, validator_to_liveness_rx: UnboundedReceiver>, ) -> Self { Self { @@ -53,7 +49,6 @@ impl LivenessTracker { metrics, too_many_empty_slots_message_shown_at: None, api_to_liveness_rx, - pool_to_liveness_rx, validator_to_liveness_rx, } } @@ -71,16 +66,6 @@ impl LivenessTracker { } }, - pool_message = self.pool_to_liveness_rx.select_next_some() => { - match pool_message { - PoolToLivenessMessage::SyncCommitteeMessage(sync_committee_message) => { - if let Err(error) = self.process_sync_committee_message(sync_committee_message) { - warn_with_peers!("Error while tracking liveness from sync committee message: {error:?}"); - } - }, - } - }, - validator_message = self.validator_to_liveness_rx.select_next_some() => { match validator_message { ValidatorToLiveness::Epoch(current_epoch) => { @@ -214,25 +199,6 @@ impl LivenessTracker { Ok(()) } - fn process_sync_committee_message( - &mut self, - sync_committee_message: SyncCommitteeMessage, - ) -> Result<()> { - let SyncCommitteeMessage { - slot, - validator_index, - .. - } = sync_committee_message; - - let epoch = misc::compute_epoch_at_slot::

(slot); - - if self.is_epoch_allowed(epoch) { - self.set(epoch, validator_index)?; - } - - Ok(()) - } - fn prune(&mut self, current_epoch: Epoch) { if let Some(epoch_boundary) = current_epoch.checked_sub(EPOCHS_TO_KEEP_LIVE_VALIDATORS) { self.live_validators = self.live_validators.split_off(&epoch_boundary); diff --git a/logging/src/lib.rs b/logging/src/lib.rs index 00c48452f..9d66bb3f6 100644 --- a/logging/src/lib.rs +++ b/logging/src/lib.rs @@ -115,7 +115,7 @@ mod tests { let mut lock = LOGGER.lock().expect("Failed to acquire LOGGER mutex lock"); if lock.is_none() { let handle = - initialize_tracing_logger(module_path!(), Some(data_dir.path()), None, false) + initialize_tracing_logger(module_path!(), Some(data_dir.path()), None, false, true) .expect("Failed to initialize tracing logger"); *lock = Some(LoggerWithTempDir { diff --git a/metrics/src/beaconchain.rs b/metrics/src/beaconchain.rs index b73d10b84..673f54619 100644 --- a/metrics/src/beaconchain.rs +++ b/metrics/src/beaconchain.rs @@ -19,7 +19,7 @@ use serde::Serialize; use sysinfo::{Disks, System}; use types::{preset::Preset, traits::BeaconState}; -use crate::{MetricsService, helpers}; +use crate::{MetricsService, NodeMetrics, helpers}; #[cfg(target_os = "linux")] use psutil::{cpu::os::linux::CpuTimesExt, memory::os::linux::VirtualMemoryExt}; @@ -134,15 +134,16 @@ pub struct BeaconNodeMetrics { } impl BeaconNodeMetrics { - pub fn get(service: &MetricsService

) -> Self { + pub fn get(service: &MetricsService

, node: &NodeMetrics

) -> Self { let MetricsService { + config, is_synced, .. + } = service; + + let NodeMetrics { controller, - config, - is_synced, eth1_metrics, slasher_active, - .. - } = service; + } = node; let Eth1Metrics { eth1_connection_data, @@ -195,26 +196,32 @@ pub struct ValidatorMetrics { impl ValidatorMetrics { pub fn get(service: &MetricsService

) -> Self { let MetricsService { - controller, + node, validator_keys, .. } = service; - let state = controller.head_state().value; - let current_epoch = accessors::get_current_epoch(&state); - - let validator_active = validator_keys - .iter() - .filter(|pubkey| { - accessors::index_of_public_key(&state, pubkey) - .and_then(|validator_index| { - state.validators().partial_validator(validator_index).ok() - }) - .is_some_and(|validator| { - predicates::is_active_validator(validator, current_epoch) + // Without a state to check, the loaded keys are the closest thing to active validators. + let validator_active = match node { + Some(NodeMetrics { controller, .. }) => { + let state = controller.head_state().value; + let current_epoch = accessors::get_current_epoch(&state); + + validator_keys + .iter() + .filter(|pubkey| { + accessors::index_of_public_key(&state, pubkey) + .and_then(|validator_index| { + state.validators().partial_validator(validator_index).ok() + }) + .is_some_and(|validator| { + predicates::is_active_validator(validator, current_epoch) + }) }) - }) - .count(); + .count() + } + None => validator_keys.len(), + }; Self { validator_active, diff --git a/metrics/src/lib.rs b/metrics/src/lib.rs index 286d09b0b..f504272a3 100644 --- a/metrics/src/lib.rs +++ b/metrics/src/lib.rs @@ -1,6 +1,6 @@ pub use crate::{ server::{MetricsServerConfig, run_metrics_server}, - service::{MetricsChannels, MetricsService, MetricsServiceConfig}, + service::{MetricsChannels, MetricsService, MetricsServiceConfig, NodeMetrics}, }; mod beaconchain; diff --git a/metrics/src/service.rs b/metrics/src/service.rs index 9098079a3..f677e2c88 100644 --- a/metrics/src/service.rs +++ b/metrics/src/service.rs @@ -35,7 +35,14 @@ const REMOTE_METRICS_UPDATE_REQUEST_TIMEOUT: Duration = Duration::from_secs(10); pub struct MetricsChannels { pub eth1_api_to_metrics_rx: Option>, - pub sync_to_metrics_rx: UnboundedReceiver, + pub sync_to_metrics_rx: Option>, +} + +/// What only the built-in beacon node can report. +pub struct NodeMetrics { + pub controller: RealController

, + pub eth1_metrics: Eth1Metrics, + pub slasher_active: bool, } #[derive(Clone, Debug)] @@ -47,11 +54,9 @@ pub struct MetricsServiceConfig { pub struct MetricsService { pub(crate) config: MetricsServiceConfig, - pub(crate) controller: RealController

, - pub(crate) metrics: Arc, - pub(crate) eth1_metrics: Eth1Metrics, + pub(crate) node: Option>, + pub(crate) metrics: Option>, pub(crate) is_synced: bool, - pub(crate) slasher_active: bool, // For simplicity's sake assume that validator keys won't change at runtime. pub(crate) validator_keys: Arc>, channels: MetricsChannels, @@ -61,20 +66,16 @@ impl MetricsService

{ #[must_use] pub const fn new( config: MetricsServiceConfig, - controller: RealController

, - eth1_metrics: Eth1Metrics, - metrics: Arc, - slasher_active: bool, + node: Option>, + metrics: Option>, validator_keys: Arc>, channels: MetricsChannels, ) -> Self { Self { config, - controller, - eth1_metrics, + node, metrics, is_synced: false, - slasher_active, validator_keys, channels, } @@ -114,6 +115,11 @@ impl MetricsService

{ None => Either::Right(futures::stream::pending()), }; + let mut sync_to_metrics_rx = match self.channels.sync_to_metrics_rx.take() { + Some(rx) => Either::Left(rx), + None => Either::Right(futures::stream::pending()), + }; + let client = Client::builder() .timeout(REMOTE_METRICS_UPDATE_REQUEST_TIMEOUT) .connection_verbose(true) @@ -122,12 +128,17 @@ impl MetricsService

{ loop { select! { _ = system_stats_refresh_interval.select_next_some() => { + // Without --metrics there is no registry, only the remote metrics to send. + let Some(metrics) = &self.metrics else { + continue; + }; + // skip system metrics update if no one is hitting /metrics - if self.metrics.metrics_requests_since_last_update.get() == 0 { + if metrics.metrics_requests_since_last_update.get() == 0 { continue; } - self.metrics.metrics_requests_since_last_update.reset(); + metrics.metrics_requests_since_last_update.reset(); tokio::task::block_in_place(|| { refresh_system_stats(&mut system, &mut system_refresh_time, false, grandine_pid); @@ -138,22 +149,22 @@ impl MetricsService

{ let (rx_bytes, tx_bytes) = helpers::get_network_bytes(); - self.metrics.set_cores(system.cpus().len()); - self.metrics.set_used_memory(grandine.memory()); - self.metrics.set_rx_bytes(rx_bytes); - self.metrics.set_tx_bytes(tx_bytes); - self.metrics.set_total_cpu_percentage(grandine.cpu_usage()); - self.metrics.set_system_cpu_percentage(system.global_cpu_usage()); - self.metrics.set_system_total_memory(system.total_memory()); - self.metrics.set_system_used_memory(system.used_memory()); + metrics.set_cores(system.cpus().len()); + metrics.set_used_memory(grandine.memory()); + metrics.set_rx_bytes(rx_bytes); + metrics.set_tx_bytes(tx_bytes); + metrics.set_total_cpu_percentage(grandine.cpu_usage()); + metrics.set_system_cpu_percentage(system.global_cpu_usage()); + metrics.set_system_total_memory(system.total_memory()); + metrics.set_system_used_memory(system.used_memory()); let process_metrics = ProcessMetrics::get(); - self.metrics.set_grandine_thread_count(process_metrics.thread_count); - self.metrics.set_total_cpu_seconds(process_metrics.cpu_process_seconds_total); + metrics.set_grandine_thread_count(process_metrics.thread_count); + metrics.set_total_cpu_seconds(process_metrics.cpu_process_seconds_total); // Update disk usage - self.metrics.set_disk_usage( + metrics.set_disk_usage( directories .disk_usage() .map_err(|error| { @@ -164,13 +175,17 @@ impl MetricsService

{ ); #[cfg(not(target_os = "windows"))] - if let Err(error) = update_jemalloc_metrics(&self.metrics) { + if let Err(error) = update_jemalloc_metrics(metrics) { warn_with_peers!("unable to update jemalloc metrics: {error:?}"); } - let head_slot = self.controller.head().value.slot(); - let store_slot = self.controller.slot(); - let max_empty_slots = self.controller.store_config().max_empty_slots; + let Some(NodeMetrics { controller, .. }) = &self.node else { + return; + }; + + let head_slot = controller.head().value.slot(); + let store_slot = controller.slot(); + let max_empty_slots = controller.store_config().max_empty_slots; if head_slot.saturating_add(max_empty_slots) >= store_slot { let epoch = misc::compute_epoch_at_slot::

(head_slot); @@ -183,7 +198,7 @@ impl MetricsService

{ // Take state at last slot in epoch let slot = misc::compute_start_slot_at_epoch::

(epoch).saturating_sub(1); - let state_opt = match self.controller.state_at_slot_blocking(slot) { + let state_opt = match controller.state_at_slot_blocking(slot) { Ok(state_opt) => state_opt, Err(error) => { warn_with_peers!("unable to update epoch metrics: {error:?}"); @@ -192,7 +207,7 @@ impl MetricsService

{ }; if let Some(state) = state_opt { - match self.update_epoch_metrics(&state.value) { + match Self::update_epoch_metrics(metrics, &state.value) { Ok(()) => epoch_with_metrics = Some(epoch), Err(error) => warn_with_peers!("unable to update epoch metrics: {error:?}"), } @@ -210,13 +225,20 @@ impl MetricsService

{ let process_metrics = ProcessMetrics::get(); - let response = client - .post(url.into_url()) - .json(&[ - self.beacon_node_metrics(process_metrics), + let payload = self + .node + .as_ref() + .map(|node| self.beacon_node_metrics(node, process_metrics)) + .into_iter() + .chain([ self.validator_metrics(process_metrics), Self::system_metrics(&system), ]) + .collect::>(); + + let response = client + .post(url.into_url()) + .json(&payload) .send() .await; @@ -243,11 +265,15 @@ impl MetricsService

{ eth1_api_message = eth1_api_to_metrics_rx.select_next_some() => { match eth1_api_message { - Eth1ApiToMetrics::Eth1Connection(eth1_connection_data) => self.eth1_metrics.eth1_connection_data = eth1_connection_data, + Eth1ApiToMetrics::Eth1Connection(eth1_connection_data) => { + if let Some(node) = &mut self.node { + node.eth1_metrics.eth1_connection_data = eth1_connection_data; + } + } } }, - sync_message = self.channels.sync_to_metrics_rx.select_next_some() => { + sync_message = sync_to_metrics_rx.select_next_some() => { match sync_message { SyncToMetrics::SyncStatus(is_synced) => self.is_synced = is_synced, SyncToMetrics::Stop => break Ok(()), @@ -259,38 +285,39 @@ impl MetricsService

{ } } - fn update_epoch_metrics(&self, state: &Arc>) -> Result<()> { + fn update_epoch_metrics(metrics: &Metrics, state: &Arc>) -> Result<()> { let statistics = transition_functions::combined::statistics(state)?; if let Some(value) = state.cache().total_active_balance[RelativeEpoch::Previous].get() { - self.metrics - .set_beacon_participation_prev_epoch_active_gwei_total(value.get()); + metrics.set_beacon_participation_prev_epoch_active_gwei_total(value.get()); } match statistics { Statistics::Phase0(statistics) => { - self.metrics - .set_beacon_participation_prev_epoch_target_attesting_gwei_total( - statistics.previous_epoch_target_attesting_balance, - ); + metrics.set_beacon_participation_prev_epoch_target_attesting_gwei_total( + statistics.previous_epoch_target_attesting_balance, + ); } Statistics::Altair(statistics) => { - self.metrics - .set_beacon_participation_prev_epoch_target_attesting_gwei_total( - statistics.previous_epoch_target_participating_balance, - ); + metrics.set_beacon_participation_prev_epoch_target_attesting_gwei_total( + statistics.previous_epoch_target_participating_balance, + ); } } Ok(()) } - fn beacon_node_metrics(&self, general: ProcessMetrics) -> BeaconChainMetrics { + fn beacon_node_metrics( + &self, + node: &NodeMetrics

, + general: ProcessMetrics, + ) -> BeaconChainMetrics { BeaconChainMetrics { meta: Meta::new(), metrics: MetricsContent::BeaconNode { general, - additional: BeaconNodeMetrics::get(self), + additional: BeaconNodeMetrics::get(self, node), }, } } diff --git a/operation_pools/src/blob_reconstruction_pool/manager.rs b/operation_pools/src/blob_reconstruction_pool/manager.rs index 00f7b5f0d..1486279b5 100644 --- a/operation_pools/src/blob_reconstruction_pool/manager.rs +++ b/operation_pools/src/blob_reconstruction_pool/manager.rs @@ -18,8 +18,8 @@ pub type ReconstructionParams = (W, H256, Arc>, Slot) pub struct Manager { controller: ApiController, - dedicated_executor: DedicatedExecutor, - low_priority_executor: Arc, + dedicated_executor: Arc, + recovery_executor: Arc, metrics: Option>, scheduled_reconstructions: BTreeMap>>, } @@ -28,16 +28,16 @@ impl Manager { #[must_use] pub const fn new( controller: ApiController, - dedicated_executor: DedicatedExecutor, + dedicated_executor: Arc, metrics: Option>, - low_priority_executor: Arc, + recovery_executor: Arc, ) -> Self { Self { controller, dedicated_executor, + recovery_executor, metrics, scheduled_reconstructions: BTreeMap::new(), - low_priority_executor, } } @@ -84,7 +84,9 @@ impl Manager { block_root, block, metrics: self.metrics.clone(), - dedicated_executor: self.low_priority_executor.clone_arc(), + // Recovery is heavy and has no deadline, so it must not share a thread with the + // attestation verifier, which runs on the low-priority executor. + dedicated_executor: self.recovery_executor.clone_arc(), }) } diff --git a/operation_pools/src/lib.rs b/operation_pools/src/lib.rs index 3a520dd12..1e1de1082 100644 --- a/operation_pools/src/lib.rs +++ b/operation_pools/src/lib.rs @@ -8,7 +8,7 @@ pub use crate::{ BlsToExecutionChangePool, Service as BlsToExecutionChangePoolService, }, manager::Manager, - messages::{PoolToLivenessMessage, PoolToP2pMessage}, + messages::PoolToP2pMessage, misc::{Origin, PoolAdditionOutcome, PoolRejectionReason}, payload_attestation_agg_pool::Manager as PayloadAttestationAggPool, sync_committee_agg_pool::Manager as SyncCommitteeAggPool, diff --git a/operation_pools/src/messages.rs b/operation_pools/src/messages.rs index b0fb4bbbc..fc181f02b 100644 --- a/operation_pools/src/messages.rs +++ b/operation_pools/src/messages.rs @@ -2,9 +2,7 @@ use eth2_libp2p::GossipId; use futures::channel::mpsc::UnboundedSender; use logging::debug_with_peers; use serde::Serialize; -use types::{ - altair::containers::SyncCommitteeMessage, capella::containers::SignedBlsToExecutionChange, -}; +use types::capella::containers::SignedBlsToExecutionChange; use crate::misc::PoolRejectionReason; @@ -23,17 +21,3 @@ impl PoolToP2pMessage { } } } - -pub enum PoolToLivenessMessage { - SyncCommitteeMessage(SyncCommitteeMessage), -} - -impl PoolToLivenessMessage { - pub fn send(self, tx: &UnboundedSender) { - if let Err(message) = tx.unbounded_send(self) { - debug_with_peers!( - "send to liveness tracker failed because the receiver was dropped: {message:?}" - ); - } - } -} diff --git a/operation_pools/src/sync_committee_agg_pool/manager.rs b/operation_pools/src/sync_committee_agg_pool/manager.rs index 034dcdc9b..6dc40b27f 100644 --- a/operation_pools/src/sync_committee_agg_pool/manager.rs +++ b/operation_pools/src/sync_committee_agg_pool/manager.rs @@ -20,7 +20,7 @@ use types::{ use validator_statistics::ValidatorStatistics; use crate::{ - messages::{PoolToLivenessMessage, PoolToP2pMessage}, + messages::PoolToP2pMessage, misc::{Origin, PoolTask}, sync_committee_agg_pool::{ pool::Pool, @@ -57,7 +57,6 @@ pub struct Manager { dedicated_executor: Arc, controller: ApiController, pool: Arc>, - pool_to_liveness_tx: Option>, pool_to_p2p_tx: UnboundedSender, metrics: Option>, } @@ -67,7 +66,6 @@ impl Manager { pub fn new( dedicated_executor: Arc, controller: ApiController, - pool_to_liveness_tx: Option>, pool_to_p2p_tx: UnboundedSender, metrics: Option>, validator_statistics: Option>, @@ -76,7 +74,6 @@ impl Manager { dedicated_executor, controller, pool: Arc::new(Pool::new(validator_statistics)), - pool_to_liveness_tx, pool_to_p2p_tx, metrics, }) @@ -182,7 +179,6 @@ impl Manager { message, subnet_id, origin, - pool_to_liveness_tx: self.pool_to_liveness_tx.clone(), pool_to_p2p_tx: self.pool_to_p2p_tx.clone(), metrics: self.metrics.clone(), }) @@ -201,7 +197,6 @@ impl Manager { message, subnet_id, origin, - pool_to_liveness_tx: self.pool_to_liveness_tx.clone(), pool_to_p2p_tx: self.pool_to_p2p_tx.clone(), metrics: self.metrics.clone(), }) diff --git a/operation_pools/src/sync_committee_agg_pool/tasks.rs b/operation_pools/src/sync_committee_agg_pool/tasks.rs index 949c34850..1ce10e5b9 100644 --- a/operation_pools/src/sync_committee_agg_pool/tasks.rs +++ b/operation_pools/src/sync_committee_agg_pool/tasks.rs @@ -33,7 +33,7 @@ use types::{ }; use crate::{ - messages::{PoolToLivenessMessage, PoolToP2pMessage}, + messages::PoolToP2pMessage, misc::{Origin, PoolRejectionReason, PoolTask}, sync_committee_agg_pool::{pool::Pool, types::ContributionData}, }; @@ -234,7 +234,6 @@ pub struct HandleExternalMessageTask { pub subnet_id: SubnetId, pub origin: Origin, pub pool_to_p2p_tx: UnboundedSender, - pub pool_to_liveness_tx: Option>, pub metrics: Option>, } @@ -254,21 +253,13 @@ impl PoolTask for HandleExternalMessageTask { message, subnet_id, origin, - ref pool_to_liveness_tx, ref pool_to_p2p_tx, .. } = self; if let Origin::Gossip(gossip_id) = origin { let message = match &result { - Ok(ValidationOutcome::Accept) => { - if let Some(pool_to_liveness_tx) = pool_to_liveness_tx { - PoolToLivenessMessage::SyncCommitteeMessage(message) - .send(pool_to_liveness_tx); - } - - PoolToP2pMessage::Accept(gossip_id) - } + Ok(ValidationOutcome::Accept) => PoolToP2pMessage::Accept(gossip_id), Ok(ValidationOutcome::Ignore(_)) => PoolToP2pMessage::Ignore(gossip_id), Err(error) => { debug_with_peers!( diff --git a/p2p/src/misc.rs b/p2p/src/misc.rs index 5d539c3a8..627a2ed04 100644 --- a/p2p/src/misc.rs +++ b/p2p/src/misc.rs @@ -37,9 +37,13 @@ pub struct SubnetPeerDiscovery { #[derive(Debug, Serialize)] pub enum SyncCommitteeSubnetAction { /// Subscribe and discover peers. - Subscribe, + Subscribe { + expiration: Epoch, + }, /// Discover peers but do not subscribe. - DiscoverPeers, + DiscoverPeers { + expiration: Epoch, + }, Unsubscribe, } @@ -62,7 +66,7 @@ pub enum PeerReportReason { ExpiredSyncBatch, } -#[derive(PartialEq, Eq, Debug, Deserialize, Serialize)] +#[derive(Clone, PartialEq, Eq, Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct SyncCommitteeSubscription { #[serde(with = "serde_utils::string_or_native")] diff --git a/p2p/src/network.rs b/p2p/src/network.rs index 0e195543a..7882224c9 100644 --- a/p2p/src/network.rs +++ b/p2p/src/network.rs @@ -1166,17 +1166,26 @@ impl Network { &self, actions: BTreeMap, ) { + let chain_config = self.controller.chain_config(); + let current_slot = self.controller.slot(); + let subnet_discoveries = actions .iter() - .filter(|(_, action)| { - matches!( - action, - SyncCommitteeSubnetAction::Subscribe | SyncCommitteeSubnetAction::DiscoverPeers, - ) - }) - .map(|(subnet_id, _)| SubnetDiscovery { - subnet: Subnet::SyncCommittee(*subnet_id), - min_ttl: None, + .filter_map(|(subnet_id, action)| match action { + SyncCommitteeSubnetAction::Subscribe { expiration } + | SyncCommitteeSubnetAction::DiscoverPeers { expiration } => { + // Without a lifetime the peer manager neither keeps the peers it finds on the + // subnet nor looks for more once they leave. + let time_diff = misc::compute_start_slot_at_epoch::

(*expiration) + .saturating_sub(current_slot) + .saturating_mul(chain_config.slot_duration_ms.as_secs()); + + Some(SubnetDiscovery { + subnet: Subnet::SyncCommittee(*subnet_id), + min_ttl: Instant::now().checked_add(Duration::from_secs(time_diff)), + }) + } + SyncCommitteeSubnetAction::Unsubscribe => None, }) .collect(); @@ -1187,7 +1196,7 @@ impl Network { let subnet = Subnet::SyncCommittee(subnet_id); match action { - SyncCommitteeSubnetAction::Subscribe => { + SyncCommitteeSubnetAction::Subscribe { .. } => { debug_with_peers!("subscribing to sync committee subnet {subnet_id}"); for topic in self.subnet_gossip_topics(subnet) { @@ -1197,7 +1206,7 @@ impl Network { ServiceInboundMessage::UpdateEnrSubnet(subnet, true) .send(&self.network_to_service_tx); } - SyncCommitteeSubnetAction::DiscoverPeers => { + SyncCommitteeSubnetAction::DiscoverPeers { .. } => { debug_with_peers!("discovering peers in sync committee subnet {subnet_id}"); } SyncCommitteeSubnetAction::Unsubscribe => { diff --git a/p2p/src/sync_committee_subnets.rs b/p2p/src/sync_committee_subnets.rs index a42ebe68f..44c0f64ae 100644 --- a/p2p/src/sync_committee_subnets.rs +++ b/p2p/src/sync_committee_subnets.rs @@ -116,8 +116,8 @@ impl SyncCommitteeSubnets

{ izip!(0.., old.states, new.states) .filter_map(|(subnet_id, old_state, new_state)| { let action = match (old_state, new_state) { - (Subscribed { .. }, Subscribed { .. }) => DiscoverPeers, - (_, Subscribed { .. }) => Subscribe, + (Subscribed { .. }, Subscribed { expiration }) => DiscoverPeers { expiration }, + (_, Subscribed { expiration }) => Subscribe { expiration }, (Subscribed { .. }, _) => Unsubscribe, _ => return None, }; @@ -139,3 +139,46 @@ impl SyncCommitteeSubnets

{ true } } + +#[cfg(test)] +mod tests { + use types::preset::Mainnet; + + use super::*; + + fn subscription(sync_committee_index: usize, until_epoch: Epoch) -> SyncCommitteeSubscription { + SyncCommitteeSubscription { + validator_index: 0, + sync_committee_indices: vec![sync_committee_index], + until_epoch, + } + } + + // The peer manager keeps and replenishes peers on a subnet only for as long as the action + // says, so the actions carry the subscription's expiration. + #[test] + fn actions_carry_the_expiration_of_the_subscription() { + let mut subnets = SyncCommitteeSubnets::::default(); + + // Sync committee index 0 lies in subnet 0 under the mainnet subcommittee size of 128. + let actions = subnets.update(1, [subscription(0, 8)]); + + assert!(matches!(actions.get(&0), Some(Subscribe { expiration: 8 }))); + + // A later subscription to the same subnet extends it and only asks for more peers. + let actions = subnets.update(1, [subscription(0, 16)]); + + assert!(matches!( + actions.get(&0), + Some(DiscoverPeers { expiration: 16 }) + )); + + // An earlier one does not shorten it. + let actions = subnets.update(1, [subscription(0, 4)]); + + assert!(matches!( + actions.get(&0), + Some(DiscoverPeers { expiration: 16 }) + )); + } +} diff --git a/predefined_chains/src/lib.rs b/predefined_chains/src/lib.rs index aabe13088..3a34caacd 100644 --- a/predefined_chains/src/lib.rs +++ b/predefined_chains/src/lib.rs @@ -13,6 +13,8 @@ use types::{ preset::Preset, }; +const MAINNET_GENESIS_STATE: &[u8] = include_bytes!("../assets/mainnet_genesis_state.ssz"); + // TODO(Grandine Team): Try refactoring `grandine` to make the type parameter hack unnecessary. // The type parameter is needed to make `PredefinedNetwork::anchor_checkpoint_provider` in `grandine` work. // Ultimately this is because Medalla uses a different preset. @@ -20,11 +22,7 @@ use types::{ // This should not happen as long as all built-in `Config`s have correct values in `preset_base`. #[must_use] pub fn mainnet() -> AnchorCheckpointProvider

{ - let state = state_from_ssz( - &Config::mainnet(), - include_bytes!("../assets/mainnet_genesis_state.ssz"), - ); - + let state = state_from_ssz(&Config::mainnet(), MAINNET_GENESIS_STATE); let block = Arc::new(genesis::beacon_block(&state)); AnchorCheckpointProvider::Predefined( @@ -35,6 +33,12 @@ pub fn mainnet() -> AnchorCheckpointProvider

{ ) } +#[must_use] +pub fn mainnet_genesis_validators_root() -> H256 { + // The field follows the 8-byte `genesis_time`; read directly so that no state is decoded. + H256::from_slice(&MAINNET_GENESIS_STATE[8..40]) +} + // Deserialize to the combined `BeaconState` to support testnets that start in later phases. fn state_from_ssz(config: &Config, bytes: &[u8]) -> Arc> { Arc::from_ssz(config, bytes).expect("bytes should encode a valid BeaconState") @@ -71,4 +75,15 @@ mod tests { assert_eq!(state.hash_tree_root(), state_root); } + + // Pins the field offset the byte read relies on. + #[test] + fn genesis_validators_root_is_read_from_the_right_bytes() { + let state = mainnet::().checkpoint().value.state; + + assert_eq!( + state.genesis_validators_root(), + mainnet_genesis_validators_root(), + ); + } } diff --git a/prometheus_metrics/dashboards/network.json b/prometheus_metrics/dashboards/network.json index b93feaf10..4698b6d2b 100644 --- a/prometheus_metrics/dashboards/network.json +++ b/prometheus_metrics/dashboards/network.json @@ -3000,7 +3000,7 @@ "targets": [ { "editorMode": "code", - "expr": "sum by(instance) (gossipsub_mesh_peer_counts{hash=~\".*$Topic.*\", instance=~\"$Instance\"})", + "expr": "sum by(instance) (gossipsub_mesh_peer_counts{topic_hash=~\".*$Topic.*\", instance=~\"$Instance\"})", "hide": false, "instant": false, "legendFormat": "total for {{instance}}", @@ -3009,7 +3009,7 @@ }, { "editorMode": "code", - "expr": "sum by(instance) (gossipsub_mesh_peer_counts{hash=~\".*$Topic.*\", hash=~\".*beacon_attestation.*\", instance=~\"$Instance\"})", + "expr": "sum by(instance) (gossipsub_mesh_peer_counts{topic_hash=~\".*$Topic.*\", topic_hash=~\".*beacon_attestation.*\", instance=~\"$Instance\"})", "hide": false, "instant": false, "legendFormat": "beacon attestation total for {{instance}}", @@ -3018,7 +3018,7 @@ }, { "editorMode": "code", - "expr": "sum by(instance) (gossipsub_mesh_peer_counts{hash=~\".*$Topic.*\", hash=~\".*blob_sidecar.*\", instance=~\"$Instance\"})", + "expr": "sum by(instance) (gossipsub_mesh_peer_counts{topic_hash=~\".*$Topic.*\", topic_hash=~\".*blob_sidecar.*\", instance=~\"$Instance\"})", "hide": false, "instant": false, "legendFormat": "blob sidecar total for {{instance}}", @@ -3027,7 +3027,7 @@ }, { "editorMode": "code", - "expr": "sum by(instance) (gossipsub_mesh_peer_counts{hash=~\".*$Topic.*\", hash=~\".*sync_committee.*\", hash!~\".*sync_committee_contribution_and_proof.*\", instance=~\"$Instance\"})", + "expr": "sum by(instance) (gossipsub_mesh_peer_counts{topic_hash=~\".*$Topic.*\", topic_hash=~\".*sync_committee.*\", topic_hash!~\".*sync_committee_contribution_and_proof.*\", instance=~\"$Instance\"})", "hide": false, "instant": false, "legendFormat": "sync committee total for {{instance}}", @@ -3037,11 +3037,11 @@ { "editorMode": "code", "exemplar": true, - "expr": "gossipsub_mesh_peer_counts{hash=~\".*$Topic.*\", instance=~\"$Instance\"}", + "expr": "sum by(instance, topic_hash) (gossipsub_mesh_peer_counts{topic_hash=~\".*$Topic.*\", instance=~\"$Instance\"})", "hide": false, "instant": false, "interval": "", - "legendFormat": "{{hash}} for {{instance}}", + "legendFormat": "{{topic_hash}} for {{instance}}", "refId": "A" } ], @@ -6390,14 +6390,14 @@ "$__all" ] }, - "definition": "label_values(gossipsub_mesh_peer_counts,hash)", + "definition": "label_values(gossipsub_topic_peers_counts,hash)", "includeAll": true, "multi": true, "name": "Fork", "options": [], "query": { "qryType": 1, - "query": "label_values(gossipsub_mesh_peer_counts,hash)", + "query": "label_values(gossipsub_topic_peers_counts,hash)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, diff --git a/prometheus_metrics/src/metrics.rs b/prometheus_metrics/src/metrics.rs index 051d51ba2..097792759 100644 --- a/prometheus_metrics/src/metrics.rs +++ b/prometheus_metrics/src/metrics.rs @@ -1051,11 +1051,44 @@ impl Metrics { default_registry.register(Box::new(self.total_cpu_seconds.clone()))?; default_registry.register(Box::new(self.grandine_thread_count.clone()))?; default_registry.register(Box::new(self.collection_lengths.clone()))?; - default_registry.register(Box::new(self.http_api_response_times.clone()))?; default_registry.register(Box::new(self.metrics_api_response_times.clone()))?; default_registry.register(Box::new(self.validator_api_response_times.clone()))?; default_registry.register(Box::new(self.dedicated_executor_task_count.clone()))?; default_registry.register(Box::new(self.dedicated_executor_thread_count.clone()))?; + default_registry.register(Box::new(self.validator_propose_tick_times.clone()))?; + default_registry.register(Box::new(self.validator_attest_tick_times.clone()))?; + default_registry.register(Box::new(self.validator_aggregate_tick_times.clone()))?; + default_registry.register(Box::new(self.validator_epoch_processing_times.clone()))?; + default_registry.register(Box::new(self.validator_own_attestations_init_times.clone()))?; + default_registry.register(Box::new(self.validator_attest_times.clone()))?; + default_registry.register(Box::new( + self.validator_attest_slashing_protector_times.clone(), + ))?; + default_registry.register(Box::new( + self.validator_own_payload_attestations_init_times.clone(), + ))?; + default_registry.register(Box::new(self.validator_attest_payload_times.clone()))?; + default_registry.register(Box::new(self.validator_propose_times.clone()))?; + default_registry.register(Box::new(self.validator_propose_successes.clone()))?; + default_registry.register(Box::new( + self.validator_proposal_slashing_protector_times.clone(), + ))?; + default_registry.register(Box::new(self.web3signer_load_keys_times.clone()))?; + default_registry.register(Box::new(self.web3signer_sign_times.clone()))?; + default_registry.register(Box::new(self.jemalloc_bytes_allocated.clone()))?; + default_registry.register(Box::new(self.jemalloc_bytes_active.clone()))?; + default_registry.register(Box::new(self.jemalloc_bytes_metadata.clone()))?; + default_registry.register(Box::new(self.jemalloc_bytes_resident.clone()))?; + default_registry.register(Box::new(self.jemalloc_bytes_mapped.clone()))?; + default_registry.register(Box::new(self.jemalloc_bytes_retained.clone()))?; + default_registry.register(Box::new(self.tick_delay_times.clone()))?; + Ok(()) + } + + pub fn register_beacon_node_metrics(&self) -> Result<()> { + let default_registry = prometheus::default_registry(); + + default_registry.register(Box::new(self.http_api_response_times.clone()))?; default_registry.register(Box::new(self.gossip_objects.clone()))?; default_registry.register(Box::new(self.received_sync_contribution_subsets.clone()))?; default_registry.register(Box::new( @@ -1117,28 +1150,10 @@ impl Metrics { self.payload_attestation_verifier_process_attestation_batch_times .clone(), ))?; - default_registry.register(Box::new(self.validator_propose_tick_times.clone()))?; - default_registry.register(Box::new(self.validator_attest_tick_times.clone()))?; - default_registry.register(Box::new(self.validator_aggregate_tick_times.clone()))?; - default_registry.register(Box::new(self.validator_epoch_processing_times.clone()))?; - default_registry.register(Box::new(self.validator_own_attestations_init_times.clone()))?; - default_registry.register(Box::new(self.validator_attest_times.clone()))?; - default_registry.register(Box::new( - self.validator_attest_slashing_protector_times.clone(), - ))?; - default_registry.register(Box::new( - self.validator_own_payload_attestations_init_times.clone(), - ))?; - default_registry.register(Box::new(self.validator_attest_payload_times.clone()))?; default_registry.register(Box::new(self.validator_api_attestation_data_times.clone()))?; default_registry.register(Box::new( self.validator_api_payload_attestation_data_times.clone(), ))?; - default_registry.register(Box::new(self.validator_propose_times.clone()))?; - default_registry.register(Box::new(self.validator_propose_successes.clone()))?; - default_registry.register(Box::new( - self.validator_proposal_slashing_protector_times.clone(), - ))?; default_registry.register(Box::new(self.build_beacon_block_times.clone()))?; default_registry.register(Box::new(self.local_execution_payload_times.clone()))?; default_registry.register(Box::new( @@ -1256,19 +1271,9 @@ impl Metrics { default_registry.register(Box::new( self.builder_submit_signed_beacon_block_times.clone(), ))?; - default_registry.register(Box::new(self.web3signer_load_keys_times.clone()))?; - default_registry.register(Box::new(self.web3signer_sign_times.clone()))?; default_registry.register(Box::new(self.eth1_api_request_times.clone()))?; default_registry.register(Box::new(self.eth1_api_errors_count.clone()))?; default_registry.register(Box::new(self.eth1_api_reset_count.clone()))?; - default_registry.register(Box::new(self.jemalloc_bytes_allocated.clone()))?; - default_registry.register(Box::new(self.jemalloc_bytes_active.clone()))?; - default_registry.register(Box::new(self.jemalloc_bytes_metadata.clone()))?; - default_registry.register(Box::new(self.jemalloc_bytes_resident.clone()))?; - default_registry.register(Box::new(self.jemalloc_bytes_mapped.clone()))?; - default_registry.register(Box::new(self.jemalloc_bytes_retained.clone()))?; - default_registry.register(Box::new(self.tick_delay_times.clone()))?; - Ok(()) } diff --git a/runtime/src/consts.rs b/runtime/src/consts.rs deleted file mode 100644 index d1efe0f91..000000000 --- a/runtime/src/consts.rs +++ /dev/null @@ -1,5 +0,0 @@ -use hex_literal::hex; -use types::phase0::primitives::{ExecutionAddress, H160}; - -pub const GRANDINE_DONATION_ADDRESS: ExecutionAddress = - H160(hex!("e7cf7C3BA875Dd3884Ed6a9082d342cb4FBb1f1b")); diff --git a/runtime/src/grandine_args.rs b/runtime/src/grandine_args.rs index 9e63346b4..ab8b42d23 100644 --- a/runtime/src/grandine_args.rs +++ b/runtime/src/grandine_args.rs @@ -10,7 +10,12 @@ use core::{ ops::{Not as _, RangeInclusive}, time::Duration, }; -use std::{collections::HashSet, ffi::OsString, path::PathBuf, sync::Arc}; +use std::{ + collections::HashSet, + ffi::OsString, + path::{Path, PathBuf}, + sync::Arc, +}; use anyhow::{Context as _, Result, anyhow, bail, ensure}; use binary_utils::TelemetryConfig; @@ -18,7 +23,10 @@ use bls::PublicKeyBytes; use builder_api::{BuilderApiFormat, BuilderConfig, gloas::containers::BuilderUrl}; use bytesize::ByteSize; use clap::{ - Arg, Args, CommandFactory as _, Error as ClapError, Parser, ValueEnum, error::ErrorKind, + Arg, ArgMatches, Args, Command as ClapCommand, CommandFactory as _, Error as ClapError, + FromArgMatches as _, Parser, Subcommand, ValueEnum, + builder::{PossibleValuesParser, TypedValueParser}, + error::ErrorKind, parser::ValueSource, }; use derivative::Derivative; @@ -55,6 +63,7 @@ use slasher::SlasherConfig; use slashing_protection::DEFAULT_SLASHING_PROTECTION_HISTORY_LIMIT; use ssz::Uint256; use std_ext::ArcExt as _; +use strum::VariantNames as _; use thiserror::Error; use tower_http::cors::AllowOrigin; use tracing::Level; @@ -63,7 +72,7 @@ use types::{ config::Config as ChainConfig, nonstandard::{ CustodyMode, DEFAULT_BUILDER_MAX_SKIPPED_SLOTS, - DEFAULT_BUILDER_MAX_SKIPPED_SLOTS_PER_EPOCH, Phase, StorageMode, + DEFAULT_BUILDER_MAX_SKIPPED_SLOTS_PER_EPOCH, Phase, PublishedDuty, StorageMode, }, phase0::primitives::{ Epoch, ExecutionAddress, ExecutionBlockHash, ExecutionBlockNumber, Gwei, H256, Slot, @@ -82,66 +91,590 @@ use crate::{ config_dir::{ self, CONFIG_FILE, DEPOSIT_CONTRACT_BLOCK_FILE, GENESIS_STATE_FILE, PLAIN_BOOTNODES_FILE, }, - consts::GRANDINE_DONATION_ADDRESS, default_network_config, defaults::DEFAULT_RECONSTRUCTION_DELAY_MS, - grandine_config::GrandineConfig, + grandine_config::{BeaconNodeConfig, GrandineConfig, Mode, ValidatorClientConfig}, predefined_network::PredefinedNetwork, validators::Validators, }; +const BN_COMMAND: &str = "bn"; +const VC_COMMAND: &str = "vc"; + /// Grandine Team /// High performance Ethereum consensus layer client #[derive(Debug, Parser)] #[clap(display_name = APPLICATION_NAME, verbatim_doc_comment, version = APPLICATION_VERSION)] pub struct GrandineArgs { - /// Load command-line options from a YAML file. Keys are long option names (e.g. `http-port`). - #[clap(long, value_name = "YAML_FILE")] - args_file: Option, - #[clap(flatten)] - chain_options: ChainOptions, + common_options: CommonOptions, #[clap(flatten)] beacon_node_options: BeaconNodeOptions, - #[clap(flatten)] - http_api_options: HttpApiOptions, + #[clap(flatten)] + validator_options: ValidatorOptions, + + #[clap(subcommand)] + command: Option, +} + +#[expect( + clippy::large_enum_variant, + reason = "Parsed once and consumed at once; boxing would only add indirection." +)] +#[derive(Debug, Subcommand)] +enum Command { + /// Run only the beacon node, without a validator client + /// (example: grandine bn --network holesky) + Bn { + #[command(flatten)] + common_options: CommonOptions, + + #[command(flatten)] + beacon_node_options: BeaconNodeOptions, + }, + + /// Run only the validator client against remote beacon nodes + /// (example: grandine vc --beacon-node-urls http://localhost:5052) + Vc { + #[command(flatten)] + common_options: CommonOptions, + + #[command(flatten)] + validator_options: ValidatorOptions, + + #[command(flatten)] + remote_validator_options: RemoteValidatorOptions, + }, + + #[clap(flatten)] + Other(GrandineCommand), +} + +/// Options of the built-in beacon node, which `vc` runs without. +#[expect( + clippy::struct_excessive_bools, + reason = "False positive. The `bool`s are independent." +)] +#[derive(Debug, Args)] +#[command(next_help_heading = "Beacon node")] +pub struct BeaconNodeOptions { + /// Enable validator liveness tracking + /// [default: disabled] + #[clap(long)] + track_liveness: bool, + + /// KZG backend + #[clap(long, default_value_t = DEFAULT_KZG_BACKEND)] + kzg_backend: KzgBackend, + + /// List of Eth1 RPC URLs + #[clap(long, num_args = 1..)] + eth1_rpc_urls: Vec, + + /// Max size of the Eth1 database + #[clap(help_heading = "Storage", long, default_value_t = DEFAULT_ETH1_DB_SIZE)] + eth1_database_size: ByteSize, + + /// Optional CL unique identifier to send to EL in the JWT token claim + /// [default: None] + #[clap(long)] + jwt_id: Option, + + /// Path to a file containing the hex-encoded 256 bit secret key to be used for verifying/generating JWT tokens + #[clap(long)] + jwt_secret: Option, + + /// Optional CL node type/version to send to EL in the JWT token claim + /// [default: None] + #[clap(long)] + jwt_version: Option, + + /// Start tracking deposit contract from BLOCK_NUMBER + #[clap(long, value_name = "BLOCK_NUMBER")] + deposit_contract_starting_block: Option, + + /// Beacon node API URL to load recent finalized checkpoint and sync from it + /// [default: None] + #[clap(long)] + checkpoint_sync_url: Option, + + /// Download genesis state from specified URL + #[clap(help_heading = "Chain", long, value_name = "URL")] + genesis_state_download_url: Option, + + /// Override TERMINAL_TOTAL_DIFFICULTY + #[clap(help_heading = "Chain", long, value_name = "DIFFICULTY")] + terminal_total_difficulty_override: Option, + + /// Override TERMINAL_BLOCK_HASH + #[clap(help_heading = "Chain", long, value_name = "BLOCK_HASH")] + terminal_block_hash_override: Option, + + /// Override TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH + #[clap(help_heading = "Chain", long, value_name = "EPOCH")] + terminal_block_hash_activation_epoch_override: Option, + + /// Disable additional 1 second wait before attesting for late blocks that are being processed at the start of the attest duty + #[clap(help_heading = "Block production", long)] + disable_wait_for_late_blocks: bool, + + /// Disable appending blockprint graffiti. If specified, no blockprint graffiti will be appended. + #[clap(help_heading = "Block production", long)] + disable_blockprint_graffiti: bool, + + /// Force checkpoint sync. Requires --checkpoint-sync-url + /// [default: disabled] + #[clap(long, requires = "checkpoint_sync_url")] + force_checkpoint_sync: bool, + + /// State slot + /// [default: None] + #[clap(long)] + state_slot: Option, + + /// [DEPRECATED] Enable syncing historical data + /// [default: disabled] + #[clap(long = "back_sync")] + back_sync: bool, + + /// Enable syncing historical data. + /// When used with --archive-storage, it will back-sync to genesis and reconstruct historical states. + /// When used without --archive-storage, it will back-sync blocks to the `Config::min_epochs_for_block_requests` epoch. + /// [default: disabled] + #[clap(long = "back-sync", conflicts_with("prune_storage"))] + back_sync_enabled: bool, + + /// Forcefully deletes the existing local beacon node databases on startup, allowing a fresh sync. + /// WARNING: This is destructive and will remove local eth1, beacon_fork_choice, sync, pubkey_cache databases. + /// [default: disabled] + #[clap(help_heading = "Storage", long)] + force_reset_beacon_db: bool, + + /// Disable reconstruction while syncing the chain + /// [default: disabled] + #[clap(long)] + sync_without_reconstruction: bool, + + /// Default data column reconstruction delay in milliseconds for nodes serving more than half of the available data columns. + #[clap(long, default_value_t = DEFAULT_RECONSTRUCTION_DELAY_MS)] + reconstruction_delay: u64, + + /// A list beacon block roots that beacon node rejects unconditionally + #[clap(long)] + blacklisted_blocks: Vec, + + /// Data format for communication with the builder API + #[clap(help_heading = "Block production", long = "builder-format", default_value_t = BuilderApiFormat::default())] + builder_api_format: BuilderApiFormat, + + /// [DEPRECATED] External block builder API URL + #[clap(help_heading = "Block production", long)] + builder_api_url: Option, + + /// External block builder URL + #[clap(help_heading = "Block production", long)] + builder_url: Option, + + /// Always use specified external block builder without checking for circuit breaker conditions + #[clap(help_heading = "Block production", long)] + builder_disable_checks: bool, + + /// Max allowed consecutive missing blocks (missing payloads post-Gloas) to trigger circuit breaker condition and switch to local execution engine for payload construction + #[clap(help_heading = "Block production", long, default_value_t = DEFAULT_BUILDER_MAX_SKIPPED_SLOTS)] + builder_max_skipped_slots: u64, + + /// Max allowed missing blocks (missing payloads post-Gloas) in the last rolling epoch to trigger circuit breaker condition and switch to local execution engine for payload construction + #[clap(help_heading = "Block production", long, default_value_t = DEFAULT_BUILDER_MAX_SKIPPED_SLOTS_PER_EPOCH)] + builder_max_skipped_slots_per_epoch: u64, + + /// Directory to store application data files + /// [default: {data_dir}/beacon] + #[clap(help_heading = "Storage", long)] + store_directory: Option, + + /// Directory to store application network files + /// [default: {data_dir}/network] + #[clap(help_heading = "Storage", long)] + network_dir: Option, + + /// Enable archival storage mode, where all blocks, states (every --archival-epoch-interval epochs) and blobs are stored in the database + /// [default: disabled] + #[clap(help_heading = "Storage", long, conflicts_with("prune_storage"))] + archive_storage: bool, + + /// Enable prune storage mode, where only a single checkpoint state and block are stored in the database + /// [default: disabled] + #[clap(help_heading = "Storage", long, conflicts_with("archive_storage"))] + prune_storage: bool, + + /// Store execution payloads of finalized blocks in the database + /// [default: disabled] + #[clap(help_heading = "Storage", long)] + store_payloads: bool, + + /// Compression level for zstd-compressed data in the database + #[clap(help_heading = "Storage", long, allow_negative_numbers = true, default_value_t = DEFAULT_ZSTD_COMPRESSION_LEVEL)] + zstd_compression_level: i32, + + /// Archival epoch interval + #[clap(help_heading = "Storage", long, default_value_t = DEFAULT_ARCHIVAL_EPOCH_INTERVAL)] + archival_epoch_interval: NonZeroU64, + + /// Max size of the Eth2 database + #[clap(help_heading = "Storage", long, default_value_t = DEFAULT_ETH2_DB_SIZE)] + database_size: ByteSize, + + /// Number of unfinalized states to keep in memory. + #[clap(help_heading = "Storage", long, default_value_t = StoreConfig::default().unfinalized_states_in_memory)] + unfinalized_states_in_memory: u64, + + /// Max amount of epochs to retain beacon states in state cache + #[clap(help_heading = "Storage", long, default_value_t = StoreConfig::default().max_epochs_to_retain_states_in_cache)] + max_epochs_to_retain_states_in_cache: u64, + + /// Default state cache lock timeout in milliseconds + #[clap(help_heading = "Storage", long, default_value_t = DEFAULT_CACHE_LOCK_TIMEOUT_MILLIS)] + state_cache_lock_timeout: u64, + + /// Number of epochs to keep blob or data column sidecars available for peer requests. + /// Overrides `Config::min_epochs_for_blob_sidecars_requests` and + /// `Config::min_epochs_for_data_column_sidecars_requests`. + /// Intended primarily for testing. Use with caution. + #[clap( + help_heading = "Storage", + long, + conflicts_with("archive_storage"), + conflicts_with("prune_storage") + )] + data_availability_window: Option, + + /// Run in supernode mode, subscribing to all data column subnets + #[clap( + long, + conflicts_with("semi_supernode"), + visible_alias("subscribe-all-data-column-subnets") + )] + supernode: bool, + + /// Run in semi-supernode mode, subscribing to half of the data column subnets + #[clap( + long, + conflicts_with("supernode"), + visible_alias("subscribe-half-data-column-subnets") + )] + semi_supernode: bool, + + /// Subscribe to all attestation and sync committee subnets. + /// This option does not include data column subnets. + #[clap(long)] + subscribe_all_subnets: bool, + + /// Backfill custody groups + #[clap(long)] + no_custody_groups_backfill: bool, + + #[clap(flatten)] + http_api_options: HttpApiOptions, + + #[clap(flatten)] + network_config_options: NetworkConfigOptions, + + // TODO(Grandine Team): The slasher is not working properly and should not be used. + #[clap(skip)] + slasher_options: SlasherOptions, +} + +impl BeaconNodeOptions { + fn builder_config(&self) -> Option { + let builder_url = if self.builder_url.is_none() && self.builder_api_url.is_some() { + warn_with_peers!("--builder-api-url option is deprecated. Use --builder-url instead."); + self.builder_api_url.clone() + } else { + self.builder_url.clone() + }; + + let circuit_breaker = self.builder_circuit_breaker(); + + builder_url.map(|url| BuilderConfig { + builder_api_format: self.builder_api_format, + builder_api_url: url, + builder_disable_checks: circuit_breaker.disabled, + builder_max_skipped_slots: circuit_breaker.max_skipped_slots, + builder_max_skipped_slots_per_epoch: circuit_breaker.max_skipped_slots_per_epoch, + }) + } + + #[must_use] + const fn builder_circuit_breaker(&self) -> BuilderCircuitBreakerConfig { + BuilderCircuitBreakerConfig { + disabled: self.builder_disable_checks, + max_skipped_slots: self.builder_max_skipped_slots, + max_skipped_slots_per_epoch: self.builder_max_skipped_slots_per_epoch, + } + } + + #[expect(clippy::too_many_arguments, clippy::too_many_lines)] + fn into_config( + self, + network: Network, + configuration_directory: Option<&Path>, + chain_config: &ChainConfig, + directories: &Arc, + predefined_network: Option, + genesis_state_file_given: bool, + metrics_enabled: bool, + in_memory: bool, + max_empty_slots: u64, + ) -> Result { + let builder_circuit_breaker = self.builder_circuit_breaker(); + let builder_config = self.builder_config(); + + let Self { + track_liveness, + kzg_backend, + eth1_rpc_urls, + eth1_database_size, + jwt_id, + jwt_secret, + jwt_version, + mut deposit_contract_starting_block, + checkpoint_sync_url, + genesis_state_download_url, + terminal_total_difficulty_override: _, + terminal_block_hash_override: _, + terminal_block_hash_activation_epoch_override: _, + disable_wait_for_late_blocks: _, + disable_blockprint_graffiti: _, + force_checkpoint_sync, + state_slot, + back_sync, + mut back_sync_enabled, + force_reset_beacon_db, + sync_without_reconstruction, + reconstruction_delay, + blacklisted_blocks, + builder_api_format: _, + builder_api_url: _, + builder_url: _, + builder_disable_checks: _, + builder_max_skipped_slots: _, + builder_max_skipped_slots_per_epoch: _, + store_directory: _, + network_dir: _, + archive_storage, + prune_storage, + store_payloads, + zstd_compression_level, + archival_epoch_interval, + database_size, + unfinalized_states_in_memory, + max_epochs_to_retain_states_in_cache, + state_cache_lock_timeout, + data_availability_window, + supernode, + semi_supernode, + subscribe_all_subnets, + no_custody_groups_backfill, + http_api_options, + mut network_config_options, + slasher_options, + } = self; + + // `clap(skip)` on the field keeps these at their defaults. + let SlasherOptions { + slashing_enabled, + slashing_history_limit, + } = slasher_options; + + if let Some(directory) = configuration_directory { + deposit_contract_starting_block = match deposit_contract_starting_block { + Some(number) => { + warn_with_peers!( + "both --configuration-directory and --deposit-contract-starting-block specified; \ + --deposit-contract-starting-block will take precedence", + ); + Some(number) + } + None => { + let bytes = fs_err::read(directory.join(DEPOSIT_CONTRACT_BLOCK_FILE))?; + Some(serde_yaml::from_slice(bytes.as_slice())?) + } + }; + + if network_config_options.boot_nodes.is_empty() { + let bytes = fs_err::read_to_string(directory.join(PLAIN_BOOTNODES_FILE))?; + network_config_options.boot_nodes = + config_dir::parse_plain_bootnodes(bytes.as_str())?; + } else { + warn_with_peers!( + "both --configuration-directory and --boot-nodes specified; \ + --boot-nodes will take precedence", + ); + } + } + + // A custom network with no execution layer needs a genesis state it cannot build. + if cfg!(not(feature = "embed")) && predefined_network.is_none() && eth1_rpc_urls.is_empty() + { + ensure!( + genesis_state_file_given, + Error::MissingEth1RpcUrlsForCustomWithoutGenesisState, + ); + } + + let minimum = StoreConfig::min_unfinalized_states_in_memory(chain_config); + + ensure!( + unfinalized_states_in_memory >= minimum, + Error::UnfinalizedStatesInMemoryTooLow { minimum }, + ); + + // The network config conversion below reads these. + let features = subscribe_all_subnets + .then_some(Feature::SubscribeToAllAttestationSubnets) + .into_iter() + .chain(subscribe_all_subnets.then_some(Feature::SubscribeToAllSyncCommitteeSubnets)) + .chain(supernode.then_some(Feature::SubscribeToAllDataColumnSubnets)); + + for feature in features { + feature.enable(); + } + + let auth_options = AuthOptions { + secrets_path: jwt_secret, + id: jwt_id, + version: jwt_version, + }; + + if back_sync { + warn_with_peers!("--back_sync option is deprecated. Use --back-sync instead."); + back_sync_enabled = true; + } + + let custody_mode = if supernode { + CustodyMode::Super + } else if semi_supernode { + CustodyMode::Semi + } else { + CustodyMode::Minimal + }; + + let storage_mode = if prune_storage { + StorageMode::Prune + } else if archive_storage { + StorageMode::Archive + } else { + StorageMode::Standard { + custom_data_availability_window: data_availability_window, + } + }; + + let range = zstd::compression_level_range(); + + ensure!( + range.contains(&zstd_compression_level), + Error::ZstdCompressionLevelOutOfRange { range }, + ); - #[clap(flatten)] - network_config_options: NetworkConfigOptions, + let storage_config = StorageConfig { + in_memory, + db_size: database_size, + directories: directories.clone_arc(), + eth1_db_size: eth1_database_size, + archival_epoch_interval, + reset_databases: force_reset_beacon_db, + storage_mode, + store_execution_payloads: store_payloads, + zstd_compression_level, + }; - #[expect( - dead_code, - reason = "TODO(Grandine Team): The slasher is not working properly and should not be used." - )] - #[clap(skip)] - slasher_options: SlasherOptions, + network_config_options.print_upnp_warning(); - #[clap(flatten)] - validator_options: ValidatorOptions, + let max_events = http_api_options.max_events; - #[clap(flatten)] - validator_api_options: ValidatorApiOptions, + let default_deposit_tree = predefined_network.map(PredefinedNetwork::genesis_deposit_tree); - /// Default block graffiti. Blockprint graffiti will be appended when sufficient space is available. - /// See `--disable-blockprint-graffiti` to disable this behavior. - #[clap(long, value_parser = misc::parse_graffiti)] - graffiti: Vec, + let deposit_contract_starting_block = deposit_contract_starting_block.or_else(|| { + default_deposit_tree + .as_ref() + .map(|tree| tree.last_added_block_number.saturating_add(1)) + }); - /// Disable appending blockprint graffiti. If specified, no blockprint graffiti will be appended. - #[clap(long)] - disable_blockprint_graffiti: bool, + Ok(BeaconNodeConfig { + deposit_contract_starting_block, + checkpoint_sync_url, + genesis_state_download_url, + force_checkpoint_sync, + back_sync_enabled, + eth1_rpc_urls, + auth_options, + network_config: network_config_options.into_config( + network, + directories.network_dir.clone().unwrap_or_default(), + metrics_enabled, + in_memory, + ), + storage_config, + store_config: StoreConfig { + max_empty_slots, + max_epochs_to_retain_states_in_cache, + state_cache_lock_timeout: Duration::from_millis(state_cache_lock_timeout), + unfinalized_states_in_memory, + kzg_backend, + sync_without_reconstruction, + builder_circuit_breaker, + }, + reconstruction_delay: Duration::from_millis(reconstruction_delay), + slasher_config: slashing_enabled.then_some(SlasherConfig { + slashing_history_limit, + }), + state_slot, + http_api_config: Option::::from(http_api_options), + max_events, + track_liveness, + blacklisted_blocks: blacklisted_blocks.into_iter().collect(), + backfill_custody_groups: !no_custody_groups_backfill, + custody_mode, + builder_config, + }) + } +} - /// List of optional runtime features to enable - #[clap(long, value_delimiter = ',')] - features: Vec, +/// The parsed options sorted by the process they configure: the built-in beacon node, with the +/// validator client unless `bn` was given, or under `vc` the validator client alone. Each variant +/// becomes the matching [`Mode`]. +#[expect( + clippy::large_enum_variant, + reason = "Built once and consumed at once; boxing would only add indirection." +)] +enum ModeOptions { + Local { + beacon_node_options: BeaconNodeOptions, + validator_options: Option, + command: Option, + }, + Remote { + validator_options: ValidatorOptions, + remote_validator_options: RemoteValidatorOptions, + }, +} - #[clap(subcommand)] - command: Option, +impl ModeOptions { + const fn beacon_node_options(&self) -> Option<&BeaconNodeOptions> { + match self { + Self::Local { + beacon_node_options, + .. + } => Some(beacon_node_options), + Self::Remote { .. } => None, + } + } } #[derive(Debug, Args)] +#[command(next_help_heading = "Chain")] struct ChainOptions { /// Name of the Eth2 network to connect to #[clap(long, value_enum, default_value_t = Network::default())] @@ -187,37 +720,22 @@ struct ChainOptions { #[clap(long, value_name = "YAML_FILE")] verify_configuration_file: Option, - /// Override TERMINAL_TOTAL_DIFFICULTY - #[clap(long, value_name = "DIFFICULTY")] - terminal_total_difficulty_override: Option, - - /// Override TERMINAL_BLOCK_HASH - #[clap(long, value_name = "BLOCK_HASH")] - terminal_block_hash_override: Option, - - /// Override TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH - #[clap(long, value_name = "EPOCH")] - terminal_block_hash_activation_epoch_override: Option, - - /// Start tracking deposit contract from BLOCK_NUMBER - #[clap(long, value_name = "BLOCK_NUMBER")] - deposit_contract_starting_block: Option, - /// Load genesis state from SSZ_FILE #[clap(long, value_name = "SSZ_FILE")] genesis_state_file: Option, - - /// Download genesis state from specified URL - #[clap(long, value_name = "URL")] - genesis_state_download_url: Option, } #[derive(Debug, Args)] +#[command(next_help_heading = "HTTP API")] struct HttpApiOptions { /// Run Grandine without HTTP API server. #[clap(long, default_value_t = false)] disable_http_api: bool, + /// Max number of events stored in a single channel for HTTP API /events api call + #[clap(long, default_value_t = DEFAULT_MAX_EVENTS)] + max_events: usize, + /// HTTP API address #[clap(long, default_value_t = HttpApiConfig::default().address.ip())] http_address: IpAddr, @@ -240,6 +758,7 @@ impl From for Option { fn from(http_api_options: HttpApiOptions) -> Self { let HttpApiOptions { disable_http_api, + max_events: _, http_address, http_port, http_allowed_origins, @@ -275,193 +794,91 @@ impl HttpApiOptions { } } -#[expect( - clippy::struct_excessive_bools, - reason = "False positive. The `bool`s are independent." -)] +/// Options shared by the beacon node and the validator client, so by every command. #[derive(Debug, Args)] -struct BeaconNodeOptions { - /// Max empty slots - #[clap(long, default_value_t = ValidatorConfig::default().max_empty_slots)] - max_empty_slots: u64, - - /// Max number of events stored in a single channel for HTTP API /events api call - #[clap(long, default_value_t = DEFAULT_MAX_EVENTS)] - max_events: usize, - - /// Beacon node API URL to load recent finalized checkpoint and sync from it - /// [default: None] - #[clap(long)] - checkpoint_sync_url: Option, - - /// Number of epochs to keep blob or data column sidecars available for peer requests. - /// Overrides `Config::min_epochs_for_blob_sidecars_requests` and - /// `Config::min_epochs_for_data_column_sidecars_requests`. - /// Intended primarily for testing. Use with caution. - #[clap( - long, - conflicts_with("archive_storage"), - conflicts_with("prune_storage") - )] - data_availability_window: Option, - - /// Force checkpoint sync. Requires --checkpoint-sync-url - /// [default: disabled] - #[clap(long, requires = "checkpoint_sync_url")] - force_checkpoint_sync: bool, - - /// Forcefully deletes the existing local beacon node databases on startup, allowing a fresh sync. - /// WARNING: This is destructive and will remove local eth1, beacon_fork_choice, sync, pubkey_cache databases. - /// [default: disabled] - #[clap(long)] - force_reset_beacon_db: bool, +#[command(next_help_heading = "General")] +struct CommonOptions { + /// Load command-line options from a YAML file. Keys are long option names (e.g. `network`). + #[clap(long, value_name = "YAML_FILE")] + args_file: Option, - /// List of Eth1 RPC URLs - #[clap(long, num_args = 1..)] - eth1_rpc_urls: Vec, + /// List of optional runtime features to enable + #[clap(long, value_delimiter = ',')] + features: Vec, /// Parent directory for application data files /// [default: $HOME/.grandine/{network}] #[clap(long)] data_dir: Option, - /// Directory to store application data files - /// [default: {data_dir}/beacon] - #[clap(long)] - store_directory: Option, - - /// Directory to store application network files - /// [default: {data_dir}/network] - #[clap(long)] - network_dir: Option, - - /// Archival epoch interval - #[clap(long, default_value_t = DEFAULT_ARCHIVAL_EPOCH_INTERVAL)] - archival_epoch_interval: NonZeroU64, - - /// Enable archival storage mode, where all blocks, states (every --archival-epoch-interval epochs) and blobs are stored in the database - /// [default: disabled] - #[clap(long, conflicts_with("prune_storage"))] - archive_storage: bool, - - /// Enable prune storage mode, where only a single checkpoint state and block are stored in the database - /// [default: disabled] - #[clap(long, conflicts_with("archive_storage"))] - prune_storage: bool, - - /// Store execution payloads of finalized blocks in the database - /// [default: disabled] - #[clap(long)] - store_payloads: bool, - - /// Compression level for zstd-compressed data in the database - #[clap(long, allow_negative_numbers = true, default_value_t = DEFAULT_ZSTD_COMPRESSION_LEVEL)] - zstd_compression_level: i32, - - /// Number of unfinalized states to keep in memory. - #[clap(long, default_value_t = StoreConfig::default().unfinalized_states_in_memory)] - unfinalized_states_in_memory: u64, - - /// Max size of the Eth2 database - #[clap(long, default_value_t = DEFAULT_ETH2_DB_SIZE)] - database_size: ByteSize, - - /// Max size of the Eth1 database - #[clap(long, default_value_t = DEFAULT_ETH1_DB_SIZE)] - eth1_database_size: ByteSize, - - /// Default data column reconstruction delay in milliseconds for nodes serving more than half of the available data columns. - #[clap(long, default_value_t = DEFAULT_RECONSTRUCTION_DELAY_MS)] - reconstruction_delay: u64, - /// Default global request timeout for various services in milliseconds #[clap(long, default_value_t = DEFAULT_REQUEST_TIMEOUT)] request_timeout: u64, - /// Max amount of epochs to retain beacon states in state cache - #[clap(long, default_value_t = StoreConfig::default().max_epochs_to_retain_states_in_cache)] - max_epochs_to_retain_states_in_cache: u64, + /// Enable in-memory mode. + /// No data will be stored in data-dir. + /// [default: disabled] + #[clap(long)] + in_memory: bool, - /// Default state cache lock timeout in milliseconds - #[clap(long, default_value_t = DEFAULT_CACHE_LOCK_TIMEOUT_MILLIS)] - state_cache_lock_timeout: u64, + #[clap(flatten)] + chain_options: ChainOptions, - /// State slot - /// [default: None] - #[clap(long)] - state_slot: Option, + #[clap(flatten)] + metrics_options: MetricsOptions, - /// Run in semi-supernode mode, subscribing to half of the data column subnets - #[clap( - long, - conflicts_with("supernode"), - visible_alias("subscribe-half-data-column-subnets") - )] - semi_supernode: bool, + #[clap(flatten)] + telemetry_options: TelemetryOptions, - /// Run in supernode mode, subscribing to all data column subnets - #[clap( - long, - conflicts_with("semi_supernode"), - visible_alias("subscribe-all-data-column-subnets") - )] - supernode: bool, + #[clap(flatten)] + block_production_options: BlockProductionOptions, +} - /// Subscribe to all attestation and sync committee subnets. - /// This option does not include data column subnets. - #[clap(long)] - subscribe_all_subnets: bool, +/// Block production settings the beacon node and the validator client both use. +#[derive(Debug, Args)] +#[command(next_help_heading = "Block production")] +struct BlockProductionOptions { + /// Max empty slots + #[clap(long, default_value_t = ValidatorConfig::default().max_empty_slots)] + max_empty_slots: u64, /// Suggested value for the feeRecipient field of the new payload #[clap(long, value_name = "EXECUTION_ADDRESS")] suggested_fee_recipient: Option, - /// Optional CL unique identifier to send to EL in the JWT token claim - /// [default: None] - #[clap(long)] - jwt_id: Option, - - /// Path to a file containing the hex-encoded 256 bit secret key to be used for verifying/generating JWT tokens - #[clap(long)] - jwt_secret: Option, - - /// Optional CL node type/version to send to EL in the JWT token claim - /// [default: None] - #[clap(long)] - jwt_version: Option, - - /// [DEPRECATED] Enable syncing historical data - /// [default: disabled] - #[clap(long = "back_sync")] - back_sync: bool, - - /// Enable syncing historical data. - /// When used with --archive-storage, it will back-sync to genesis and reconstruct historical states. - /// When used without --archive-storage, it will back-sync blocks to the `Config::min_epochs_for_block_requests` epoch. - /// [default: disabled] - #[clap(long = "back-sync", conflicts_with("prune_storage"))] - back_sync_enabled: bool, + /// Default block graffiti. Blockprint graffiti will be appended when sufficient space is available. + /// See `--disable-blockprint-graffiti` to disable this behavior. + #[clap(long, value_parser = misc::parse_graffiti)] + graffiti: Vec, - /// Collect Prometheus metrics - #[clap(long = "metrics")] - metrics_enabled: bool, + /// Percentage multiplier to apply to the builder's payload value when choosing between a builder payload header and payload from the paired execution node + #[clap(long, default_value_t = BuilderSettings::default().default_builder_boost_factor)] + default_builder_boost_factor: Uint256, - /// Metrics address for metrics endpoint - #[clap(long, default_value_t = IpAddr::V4(Ipv4Addr::LOCALHOST))] - metrics_address: IpAddr, + /// Default execution gas limit for all validators [default: scheduled gas limit, or 60000000] + #[clap(long)] + default_gas_limit: Option, - /// Listen port for metrics endpoint - #[clap(long, default_value_t = DEFAULT_METRICS_PORT)] - metrics_port: u16, + /// Minimum total payment in Gwei that proposer are willing to accept from a builder + #[clap(long, default_value_t = BuilderSettings::default().default_builder_min_bid)] + default_builder_min_bid: Gwei, - /// Update system metrics every n seconds - #[clap(long, default_value_t = DEFAULT_METRICS_UPDATE_INTERVAL_SECONDS)] - metrics_update_interval: u64, + /// Maximum execution layer payment in Gwei counted toward a builder bid. Values above 0 require --allow-trusted-payments + #[clap(long, default_value_t = BuilderSettings::default().default_builder_max_execution_payment)] + default_builder_max_execution_payment: Gwei, - /// Optional remote metrics (beaconcha.in metrics) URL that Grandine will periodically send metrics to + /// Allow counting execution layer payments toward builder bids. #[clap(long)] - remote_metrics_url: Option, + allow_trusted_payments: bool, + /// External builder URLs to request execution payload bids from post-Gloas + #[clap(long, num_args = 1.., value_delimiter = ',')] + payload_builder_urls: Vec, +} + +#[derive(Debug, Args)] +#[command(next_help_heading = "Telemetry")] +struct TelemetryOptions { /// The default tracing level controlling how detailed telemetry output will be. #[clap(long, requires("telemetry_metrics_url"), default_value_t = Level::INFO)] telemetry_level: Level, @@ -474,35 +891,45 @@ struct BeaconNodeOptions { /// Optional OTLP service name. #[clap(long, requires("telemetry_metrics_url"), default_value_t = APPLICATION_NAME.to_string())] telemetry_service_name: String, +} - /// Enable validator liveness tracking - /// [default: disabled] - #[clap(long)] - track_liveness: bool, +impl TelemetryOptions { + #[must_use] + fn telemetry_config(&self) -> Option { + if let Some(url) = self.telemetry_metrics_url.clone() { + return Some(TelemetryConfig { + url, + service_name: self.telemetry_service_name.clone(), + trace_level: self.telemetry_level, + }); + } - /// Enable doppelganger protection (liveness tracking must be enabled for this feature) - /// [default: disabled] - #[clap(long, requires("track_liveness"))] - detect_doppelgangers: bool, + None + } +} - /// Enable in-memory mode. - /// No data will be stored in data-dir. - /// [default: disabled] - #[clap(long)] - in_memory: bool, +#[derive(Debug, Args)] +#[command(next_help_heading = "Metrics")] +struct MetricsOptions { + /// Collect Prometheus metrics + #[clap(long = "metrics")] + metrics_enabled: bool, - /// KZG backend - #[clap(long, default_value_t = DEFAULT_KZG_BACKEND)] - kzg_backend: KzgBackend, + /// Metrics address for metrics endpoint + #[clap(long, default_value_t = IpAddr::V4(Ipv4Addr::LOCALHOST))] + metrics_address: IpAddr, - /// A list beacon block roots that beacon node rejects unconditionally - #[clap(long)] - blacklisted_blocks: Vec, + /// Listen port for metrics endpoint + #[clap(long, default_value_t = DEFAULT_METRICS_PORT)] + metrics_port: u16, - /// Disable reconstruction while syncing the chain - /// [default: disabled] + /// Update system metrics every n seconds + #[clap(long, default_value_t = DEFAULT_METRICS_UPDATE_INTERVAL_SECONDS)] + metrics_update_interval: u64, + + /// Optional remote metrics (beaconcha.in metrics) URL that Grandine will periodically send metrics to #[clap(long)] - sync_without_reconstruction: bool, + remote_metrics_url: Option, } #[expect( @@ -510,6 +937,7 @@ struct BeaconNodeOptions { reason = "False positive. The `bool`s are independent." )] #[derive(Debug, Args)] +#[command(next_help_heading = "Networking")] struct NetworkConfigOptions { /// Listen IPv4 address /// [default: 0.0.0.0, unless --disable-ipv4 is set] @@ -644,20 +1072,6 @@ struct NetworkConfigOptions { trusted_peers: Vec, } -impl BeaconNodeOptions { - pub fn telemetry_config(&self) -> Option { - if let Some(url) = self.telemetry_metrics_url.clone() { - return Some(TelemetryConfig { - url, - service_name: self.telemetry_service_name.clone(), - trace_level: self.telemetry_level, - }); - } - - None - } -} - impl NetworkConfigOptions { #[expect(clippy::too_many_lines)] fn into_config( @@ -865,13 +1279,28 @@ struct SlasherOptions { reason = "False positive. The `bool`s are independent." )] #[derive(Debug, Args)] +#[command(next_help_heading = "Validator")] struct ValidatorOptions { + /// Enable doppelganger protection (with the built-in beacon node, liveness tracking must be enabled) + /// [default: disabled] + #[clap(long)] + detect_doppelgangers: bool, + + /// Print reports about built-in validator performance + #[clap(long)] + report_validator_performance: bool, + /// Path to a directory containing EIP-2335 keystore files - #[clap(long, requires("keystore_password_file"))] + #[clap( + help_heading = "Validator keys", + long, + requires("keystore_password_file") + )] keystore_dir: Option, /// Path to a directory containing passwords for keystore files #[clap( + help_heading = "Validator keys", long, requires("keystore_dir"), conflicts_with("keystore_password_file") @@ -880,6 +1309,7 @@ struct ValidatorOptions { /// Path to a file containing password for keystore files #[clap( + help_heading = "Validator keys", long, requires("keystore_dir"), conflicts_with("keystore_password_dir") @@ -887,95 +1317,162 @@ struct ValidatorOptions { keystore_password_file: Option, /// Path to a file containing password for decrypting imported keystores from API - #[clap(long)] + #[clap(help_heading = "Validator keys", long)] keystore_storage_password_file: Option, - /// Data format for communication with the builder API - #[clap(long = "builder-format", default_value_t = BuilderApiFormat::default())] - builder_api_format: BuilderApiFormat, - - /// [DEPRECATED] External block builder API URL - #[clap(long)] - builder_api_url: Option, - - /// External block builder URL - #[clap(long)] - builder_url: Option, - - /// Always use specified external block builder without checking for circuit breaker conditions - #[clap(long)] - builder_disable_checks: bool, - - /// Max allowed consecutive missing blocks (missing payloads post-Gloas) to trigger circuit breaker condition and switch to local execution engine for payload construction - #[clap(long, default_value_t = DEFAULT_BUILDER_MAX_SKIPPED_SLOTS)] - builder_max_skipped_slots: u64, - - /// Max allowed missing blocks (missing payloads post-Gloas) in the last rolling epoch to trigger circuit breaker condition and switch to local execution engine for payload construction - #[clap(long, default_value_t = DEFAULT_BUILDER_MAX_SKIPPED_SLOTS_PER_EPOCH)] - builder_max_skipped_slots_per_epoch: u64, - - /// Percentage multiplier to apply to the builder's payload value when choosing between a builder payload header and payload from the paired execution node - #[clap(long, default_value_t = BuilderSettings::default().default_builder_boost_factor)] - default_builder_boost_factor: Uint256, - - /// Default execution gas limit for all validators [default: scheduled gas limit, or 60000000] - #[clap(long)] - default_gas_limit: Option, - - /// Minimum total payment in Gwei that proposer are willing to accept from a builder - #[clap(long, default_value_t = BuilderSettings::default().default_builder_min_bid)] - default_builder_min_bid: Gwei, - - /// Maximum execution layer payment in Gwei counted toward a builder bid. Values above 0 require --allow-trusted-payments - #[clap(long, default_value_t = BuilderSettings::default().default_builder_max_execution_payment)] - default_builder_max_execution_payment: Gwei, - - /// Allow counting execution layer payments toward builder bids. - #[clap(long)] - allow_trusted_payments: bool, - - /// External builder URLs to request execution payload bids from post-Gloas - #[clap(long, num_args = 1.., value_delimiter = ',')] - payload_builder_urls: Vec, - /// List of public keys to use from Web3Signer - #[clap(long, num_args = 1.., value_delimiter = ',')] + #[clap(help_heading = "Validator keys", long, num_args = 1.., value_delimiter = ',')] web3signer_public_keys: Vec, /// Refetches keys from Web3Signer once every epoch. This overwrites changes done via Keymanager API for remote keys - #[clap(long)] + #[clap(help_heading = "Validator keys", long)] web3signer_refresh_keys_every_epoch: bool, /// [DEPRECATED] List of Web3Signer API URLs - #[clap(long, num_args = 1..)] + #[clap(help_heading = "Validator keys", long, num_args = 1..)] web3signer_api_urls: Vec, /// List of Web3Signer URLs - #[clap(long, num_args = 1..)] + #[clap(help_heading = "Validator keys", long, num_args = 1..)] web3signer_urls: Vec, /// Use validator key cache for faster startup - #[clap(long)] + #[clap(help_heading = "Validator keys", long)] use_validator_key_cache: bool, /// Number of epochs to keep slashing protection data for #[clap(long, default_value_t = DEFAULT_SLASHING_PROTECTION_HISTORY_LIMIT)] slashing_protection_history_limit: u64, - /// Print reports about validator performance - #[clap(long)] - report_validator_performance: bool, + #[clap(flatten)] + validator_api_options: ValidatorApiOptions, +} - /// Backfill custody groups - #[clap(long)] - no_custody_groups_backfill: bool, +impl ValidatorOptions { + fn into_config( + self, + beacon_node_options: Option<&BeaconNodeOptions>, + ) -> Result { + let Self { + detect_doppelgangers, + report_validator_performance, + keystore_dir, + keystore_password_dir, + keystore_password_file, + keystore_storage_password_file, + web3signer_public_keys, + web3signer_refresh_keys_every_epoch, + web3signer_api_urls, + web3signer_urls, + use_validator_key_cache, + slashing_protection_history_limit, + validator_api_options, + } = self; - /// Disable additional 1 second wait before attesting for late blocks that are being processed at the start of the attest duty + // Without the built-in node liveness comes from the remote beacon nodes. + if detect_doppelgangers && let Some(options) = beacon_node_options { + ensure!( + options.track_liveness, + Error::DoppelgangerDetectionRequiresLivenessTracking, + ); + } + + if beacon_node_options.is_none() { + ensure!( + !report_validator_performance, + Error::PerformanceReportsRequireBuiltInNode, + ); + } + + let validators = keystore_dir + .zip(keystore_password_file.or(keystore_password_dir)) + .map(|(keystore_dir, keystore_password_file)| Validators { + keystore_dir, + keystore_password_file, + }); + + let web3signer_urls = if web3signer_urls.is_empty() && !web3signer_api_urls.is_empty() { + warn_with_peers!( + "--web3signer-api-urls option is deprecated. Use --web3signer-urls instead." + ); + web3signer_api_urls + } else { + web3signer_urls + }; + + // Each CLI URL loads every key it serves; `--web3signer-public-keys` is a global allow-list + // narrowing that. `validators.yml` entries add their own per-URL keys on top (see + // `runtime`), and only those entries — not the global list — warn if a key is not served. + let web3signer_config = Web3SignerConfig { + allow_to_reload_keys: web3signer_refresh_keys_every_epoch, + public_keys: web3signer_public_keys.into_iter().collect(), + urls: web3signer_urls + .into_iter() + .map(|url| { + ( + url, + Web3SignerUrlPolicy { + load_all: true, + required: HashSet::new(), + }, + ) + }) + .collect(), + }; + + let validator_api_config = validator_api_options + .enable_validator_api + .then(|| ValidatorApiConfig::from(validator_api_options)); + + Ok(ValidatorClientConfig { + validators, + keystore_storage_password_file, + web3signer_config, + detect_doppelgangers, + use_validator_key_cache, + slashing_protection_history_limit, + validator_api_config, + report_validator_performance, + }) + } +} + +#[derive(Debug, Args)] +#[command(next_help_heading = "Beacon nodes")] +struct RemoteValidatorOptions { + /// List of beacon node API URLs to perform validator duties against. + /// Among the healthiest nodes, duties go to the one with the newest head, then to the first + /// listed. + #[clap(long, num_args = 1.., required = true, value_delimiter = ',')] + beacon_node_urls: Vec, + + /// List of duties to publish to every beacon node rather than the first one; `all` covers + /// every duty, and the flag given without a value publishes everything to the first node only + #[clap( + long, + num_args = 0.., + value_delimiter = ',', + default_value = "all", + value_parser = published_duty_parser() + )] + publish_to_every_node: Vec, + + /// Use the builder behind the beacon nodes: register validators with it and accept its + /// payloads. Without it only local payloads are requested and builder boost factors are + /// ignored. Relevant only until Gloas, where builders bid over gossip and need no registration #[clap(long)] - disable_wait_for_late_blocks: bool, + use_builder: bool, +} + +fn published_duty_parser() -> impl TypedValueParser { + PossibleValuesParser::new(PublishedDuty::VARIANTS).map(|duty| { + duty.parse() + .expect("value parser only accepts PublishedDuty variants") + }) } #[derive(Debug, Args)] +#[command(next_help_heading = "Validator API")] struct ValidatorApiOptions { /// Enable validator API #[clap(long)] @@ -1070,18 +1567,61 @@ impl GrandineArgs { #[expect(clippy::too_many_lines)] pub fn try_into_config(self) -> Result { let Self { - chain_options, + common_options, beacon_node_options, - http_api_options, - mut network_config_options, validator_options, - validator_api_options, - graffiti, - disable_blockprint_graffiti, + command, + } = self; + + let local = |command| { + ( + common_options, + ModeOptions::Local { + beacon_node_options, + validator_options: Some(validator_options), + command, + }, + ) + }; + + let (common_options, mode_options) = match command { + Some(Command::Bn { + common_options, + beacon_node_options, + }) => ( + common_options, + ModeOptions::Local { + beacon_node_options, + validator_options: None, + command: None, + }, + ), + Some(Command::Vc { + common_options, + validator_options, + remote_validator_options, + }) => ( + common_options, + ModeOptions::Remote { + validator_options, + remote_validator_options, + }, + ), + Some(Command::Other(command)) => local(Some(command)), + None => local(None), + }; + + let CommonOptions { mut features, - command, + data_dir, + request_timeout, + in_memory, + chain_options, + metrics_options, + telemetry_options, + block_production_options, .. - } = self; + } = common_options; let ChainOptions { network, @@ -1095,98 +1635,38 @@ impl GrandineArgs { verify_electra_preset_file, verify_fulu_preset_file, verify_configuration_file, - terminal_total_difficulty_override, - terminal_block_hash_override, - terminal_block_hash_activation_epoch_override, - mut deposit_contract_starting_block, mut genesis_state_file, - genesis_state_download_url, } = chain_options; - let telemetry_config = beacon_node_options.telemetry_config(); - - let BeaconNodeOptions { - max_empty_slots, - max_events, - checkpoint_sync_url, - data_availability_window, - eth1_rpc_urls, - force_checkpoint_sync, - force_reset_beacon_db, - data_dir, - store_directory, - network_dir, - database_size, - eth1_database_size, - archival_epoch_interval, - archive_storage, - prune_storage, - store_payloads, - zstd_compression_level, - unfinalized_states_in_memory, - reconstruction_delay, - request_timeout, - max_epochs_to_retain_states_in_cache, - state_cache_lock_timeout, - state_slot, - semi_supernode, - supernode, - subscribe_all_subnets, - suggested_fee_recipient, - jwt_id, - jwt_secret, - jwt_version, - back_sync, - mut back_sync_enabled, + let MetricsOptions { metrics_enabled, metrics_address, metrics_port, metrics_update_interval, remote_metrics_url, - track_liveness, - detect_doppelgangers, - in_memory, - kzg_backend, - blacklisted_blocks, - sync_without_reconstruction, - .. - } = beacon_node_options; - - // let SlasherOptions { - // slashing_enabled, - // slashing_history_limit, - // } = slasher_options; - - let slashing_enabled = false; - let slashing_history_limit = SlasherConfig::default().slashing_history_limit; + } = metrics_options; - let ValidatorOptions { - keystore_dir, - keystore_password_dir, - keystore_password_file, - keystore_storage_password_file, - builder_api_format, - builder_api_url, - builder_url, - builder_disable_checks, - builder_max_skipped_slots, - builder_max_skipped_slots_per_epoch, + let BlockProductionOptions { + max_empty_slots, + suggested_fee_recipient, + graffiti, default_builder_boost_factor, default_builder_min_bid, default_builder_max_execution_payment, allow_trusted_payments, payload_builder_urls, default_gas_limit, - use_validator_key_cache, - web3signer_public_keys, - web3signer_refresh_keys_every_epoch, - web3signer_api_urls, - web3signer_urls, - slashing_protection_history_limit, - report_validator_performance, - no_custody_groups_backfill, - disable_wait_for_late_blocks, - } = validator_options; + } = block_production_options; + + ensure!( + default_builder_max_execution_payment == 0 || allow_trusted_payments, + "--default-builder-max-execution-payment above 0 requires --allow-trusted-payments", + ); + + for url in &payload_builder_urls { + url.http_url() + .with_context(|| format!("invalid --payload-builder-urls entry {url}"))?; + } if in_memory { warn_with_peers!( @@ -1201,14 +1681,7 @@ impl GrandineArgs { warn_with_peers!("both --configuration-file and --verify-configuration-file specified"); } - if remote_metrics_url.is_some() && !metrics_enabled { - warn_with_peers!( - "remote metrics enabled without ---metrics. \ - Network, system and process metrics will not be available" - ); - } - - if let Some(directory) = configuration_directory { + if let Some(directory) = configuration_directory.as_ref() { configuration_file = configuration_file .inspect(|_| { warn_with_peers!( @@ -1218,20 +1691,6 @@ impl GrandineArgs { }) .or_else(|| Some(directory.join(CONFIG_FILE))); - deposit_contract_starting_block = match deposit_contract_starting_block { - Some(number) => { - warn_with_peers!( - "both --configuration-directory and --deposit-contract-starting-block specified; \ - --deposit-contract-starting-block will take precedence", - ); - Some(number) - } - None => { - let bytes = fs_err::read(directory.join(DEPOSIT_CONTRACT_BLOCK_FILE))?; - Some(serde_yaml::from_slice(bytes.as_slice())?) - } - }; - genesis_state_file = genesis_state_file .inspect(|_| { warn_with_peers!( @@ -1240,30 +1699,10 @@ impl GrandineArgs { ); }) .or_else(|| Some(directory.join(GENESIS_STATE_FILE))); - - if network_config_options.boot_nodes.is_empty() { - let bytes = fs_err::read_to_string(directory.join(PLAIN_BOOTNODES_FILE))?; - - network_config_options.boot_nodes = - config_dir::parse_plain_bootnodes(bytes.as_str())?; - } else { - warn_with_peers!( - "both --configuration-directory and --boot-nodes specified; \ - --boot-nodes will take precedence", - ); - } } let predefined_network = network.predefined_network(); - if cfg!(not(feature = "embed")) && predefined_network.is_none() && eth1_rpc_urls.is_empty() - { - ensure!( - genesis_state_file.is_some(), - Error::MissingEth1RpcUrlsForCustomWithoutGenesisState, - ); - } - let mut chain_config = match configuration_file { Some(path) => { let bytes = fs_err::read(path)?; @@ -1336,16 +1775,18 @@ impl GrandineArgs { verify_config(&chain_config, verify_configuration_file)?; // Overriding after verifying seems more useful, though neither is strictly better. - if let Some(value) = terminal_total_difficulty_override { - chain_config.terminal_total_difficulty = value; - } + if let Some(options) = mode_options.beacon_node_options() { + if let Some(value) = options.terminal_total_difficulty_override { + chain_config.terminal_total_difficulty = value; + } - if let Some(value) = terminal_block_hash_override { - chain_config.terminal_block_hash = value; - } + if let Some(value) = options.terminal_block_hash_override { + chain_config.terminal_block_hash = value; + } - if let Some(value) = terminal_block_hash_activation_epoch_override { - chain_config.terminal_block_hash_activation_epoch = value; + if let Some(value) = options.terminal_block_hash_activation_epoch_override { + chain_config.terminal_block_hash_activation_epoch = value; + } } chain_config.validate()?; @@ -1353,8 +1794,12 @@ impl GrandineArgs { let directories = Arc::new( Directories { data_dir, - store_directory, - network_dir, + store_directory: mode_options + .beacon_node_options() + .and_then(|options| options.store_directory.clone()), + network_dir: mode_options + .beacon_node_options() + .and_then(|options| options.network_dir.clone()), validator_dir: None, secrets_dir: None, } @@ -1385,248 +1830,139 @@ impl GrandineArgs { timeout: request_timeout, }); - let mut services = vec![]; - - let validator_api_config = validator_api_options - .enable_validator_api - .then(|| ValidatorApiConfig::from(validator_api_options)); - - let http_api_config = Option::::from(http_api_options); - - if let Some(http_api_config) = http_api_config.as_ref() { - services.push((http_api_config.address, "HTTP API")); - } - - if let Some(metrics_server_config) = metrics_server_config.as_ref() { - services.push((SocketAddr::from(metrics_server_config), "Metrics API")); - } - - if let Some(validator_api_config) = validator_api_config.as_ref() { - services.push((validator_api_config.address, "Validator API")); - } - - for ((address1, service1), (address2, service2)) in - services.into_iter().tuple_combinations() - { - ensure!( - address1 != address2, - Error::IdenticalAddresses { service1, service2 }, - ); - } - - let metrics = if metrics_enabled { - let metrics = Metrics::new()?; - metrics.register_with_default_metrics()?; - let metrics = Arc::new(metrics); - METRICS.get_or_init(|| metrics.clone_arc()); - Some(metrics) - } else { - None - }; - - let metrics_config = MetricsConfig { - metrics, - metrics_server_config, - metrics_service_config, - }; - - let validators = keystore_dir - .zip(keystore_password_file.or(keystore_password_dir)) - .map(|(keystore_dir, keystore_password_file)| Validators { - keystore_dir, - keystore_password_file, - }); - - let minimum = StoreConfig::min_unfinalized_states_in_memory(&chain_config); - - ensure!( - unfinalized_states_in_memory >= minimum, - Error::UnfinalizedStatesInMemoryTooLow { minimum }, - ); - - let range = zstd::compression_level_range(); - - ensure!( - range.contains(&zstd_compression_level), - Error::ZstdCompressionLevelOutOfRange { range }, - ); - - let features = features - .into_iter() - .chain(subscribe_all_subnets.then_some(Feature::SubscribeToAllAttestationSubnets)) - .chain(subscribe_all_subnets.then_some(Feature::SubscribeToAllSyncCommitteeSubnets)) - .chain(supernode.then_some(Feature::SubscribeToAllDataColumnSubnets)) - .collect::>(); - - // enabling these features here, because it being used in below network config conversion for feature in &features { feature.enable(); } - let auth_options = AuthOptions { - secrets_path: jwt_secret, - id: jwt_id, - version: jwt_version, - }; - - if back_sync { - warn_with_peers!("--back_sync option is deprecated. Use --back-sync instead."); - back_sync_enabled = true; - } - - let builder_url = if builder_url.is_none() && builder_api_url.is_some() { - warn_with_peers!("--builder-api-url option is deprecated. Use --builder-url instead."); - builder_api_url - } else { - builder_url - }; - - ensure!( - default_builder_max_execution_payment == 0 || allow_trusted_payments, - "--default-builder-max-execution-payment above 0 requires --allow-trusted-payments", - ); - - for url in &payload_builder_urls { - url.http_url() - .with_context(|| format!("invalid --payload-builder-urls entry {url}"))?; - } - - let builder_config = builder_url.map(|url| BuilderConfig { - builder_api_format, - builder_api_url: url, - builder_disable_checks, - builder_max_skipped_slots, - builder_max_skipped_slots_per_epoch, - }); - - let builder_circuit_breaker = BuilderCircuitBreakerConfig { - disabled: builder_disable_checks, - max_skipped_slots: builder_max_skipped_slots, - max_skipped_slots_per_epoch: builder_max_skipped_slots_per_epoch, - }; - - let web3signer_urls = if web3signer_urls.is_empty() && !web3signer_api_urls.is_empty() { - warn_with_peers!( - "--web3signer-api-urls option is deprecated. Use --web3signer-urls instead." - ); - web3signer_api_urls - } else { - web3signer_urls - }; - - // Each CLI URL loads every key it serves; `--web3signer-public-keys` is a global allow-list - // narrowing that. `validators.yml` entries add their own per-URL keys on top (see - // `runtime`), and only those entries — not the global list — warn if a key is not served. - let web3signer_config = Web3SignerConfig { - allow_to_reload_keys: web3signer_refresh_keys_every_epoch, - public_keys: web3signer_public_keys.into_iter().collect(), - urls: web3signer_urls - .into_iter() - .map(|url| { - ( - url, - Web3SignerUrlPolicy { - load_all: true, - required: HashSet::new(), - }, - ) - }) - .collect(), - }; - - let custody_mode = if supernode { - CustodyMode::Super - } else if semi_supernode { - CustodyMode::Semi - } else { - CustodyMode::Minimal - }; + let metrics = if metrics_enabled { + let metrics = Metrics::new()?; + metrics.register_with_default_metrics()?; - let storage_mode = if prune_storage { - StorageMode::Prune - } else if archive_storage { - StorageMode::Archive - } else { - StorageMode::Standard { - custom_data_availability_window: data_availability_window, + if mode_options.beacon_node_options().is_some() { + metrics.register_beacon_node_metrics()?; } + + let metrics = Arc::new(metrics); + METRICS.get_or_init(|| metrics.clone_arc()); + Some(metrics) + } else { + None }; - let storage_config = StorageConfig { - in_memory, - db_size: database_size, - directories: directories.clone_arc(), - eth1_db_size: eth1_database_size, - archival_epoch_interval, - reset_databases: force_reset_beacon_db, - storage_mode, - store_execution_payloads: store_payloads, - zstd_compression_level, + let metrics_config = MetricsConfig { + metrics, + metrics_server_config, + metrics_service_config, }; - network_config_options.print_upnp_warning(); + let chain_config = Arc::new(chain_config); + + let disable_wait_for_late_blocks = mode_options + .beacon_node_options() + .is_some_and(|options| options.disable_wait_for_late_blocks); + + let disable_blockprint_graffiti = mode_options + .beacon_node_options() + .is_some_and(|options| options.disable_blockprint_graffiti); + + let mode = match mode_options { + ModeOptions::Local { + beacon_node_options, + validator_options, + command, + } => { + let validator_client_config = validator_options + .map(|options| options.into_config(Some(&beacon_node_options))) + .transpose()?; + + let beacon_node_config = beacon_node_options.into_config( + network, + configuration_directory.as_deref(), + &chain_config, + &directories, + predefined_network, + genesis_state_file.is_some(), + metrics_enabled, + in_memory, + max_empty_slots, + )?; + + Mode::Local { + beacon_node_config, + validator_client_config, + command, + } + } + ModeOptions::Remote { + validator_options, + remote_validator_options: + RemoteValidatorOptions { + beacon_node_urls, + publish_to_every_node, + use_builder, + }, + } => Mode::Remote { + validator_client_config: validator_options.into_config(None)?, + beacon_node_urls, + publish_to_every_node, + use_builder, + }, + }; - Ok(GrandineConfig { + let config = GrandineConfig { predefined_network, - chain_config: Arc::new(chain_config), - deposit_contract_starting_block, + chain_config, genesis_state_file, - genesis_state_download_url, - checkpoint_sync_url, - force_checkpoint_sync, - back_sync_enabled, - eth1_rpc_urls, data_dir: directories.data_dir.clone().unwrap_or_default(), - validators, - keystore_storage_password_file, - graffiti, + directories, + in_memory, + request_timeout: Duration::from_millis(request_timeout), + metrics_config, disable_blockprint_graffiti, + graffiti, max_empty_slots, - suggested_fee_recipient: suggested_fee_recipient.unwrap_or(GRANDINE_DONATION_ADDRESS), + suggested_fee_recipient, default_builder_boost_factor, default_builder_min_bid, default_builder_max_execution_payment, allow_trusted_payments, payload_builder_urls, default_gas_limit, - network_config: network_config_options.into_config( - network, - directories.network_dir.clone().unwrap_or_default(), - metrics_enabled, - in_memory, - ), - storage_config, - unfinalized_states_in_memory, - reconstruction_delay: Duration::from_millis(reconstruction_delay), - request_timeout: Duration::from_millis(request_timeout), - max_epochs_to_retain_states_in_cache, - state_cache_lock_timeout: Duration::from_millis(state_cache_lock_timeout), - command, - slashing_enabled, - slashing_history_limit, - state_slot, - auth_options, - builder_config, - builder_circuit_breaker, - web3signer_config, - http_api_config, - max_events, - metrics_config, - telemetry_config, - track_liveness, - detect_doppelgangers, - use_validator_key_cache, - slashing_protection_history_limit, - in_memory, - validator_api_config, - kzg_backend, - blacklisted_blocks: blacklisted_blocks.into_iter().collect(), - report_validator_performance, - backfill_custody_groups: !no_custody_groups_backfill, - sync_without_reconstruction, - custody_mode, disable_wait_for_late_blocks, - }) + telemetry_config: telemetry_options.telemetry_config(), + mode, + }; + + let mut services = vec![]; + + if let Some(http_api_config) = config + .beacon_node_config() + .and_then(|node| node.http_api_config.as_ref()) + { + services.push((http_api_config.address, "HTTP API")); + } + + if let Some(metrics_server_config) = config.metrics_config.metrics_server_config.as_ref() { + services.push((SocketAddr::from(metrics_server_config), "Metrics API")); + } + + if let Some(validator_api_config) = config + .validator_client_config() + .and_then(|validator_client| validator_client.validator_api_config.as_ref()) + { + services.push((validator_api_config.address, "Validator API")); + } + + for ((address1, service1), (address2, service2)) in + services.into_iter().tuple_combinations() + { + ensure!( + address1 != address2, + Error::IdenticalAddresses { service1, service2 }, + ); + } + + Ok(config) } #[must_use] @@ -1639,7 +1975,8 @@ impl GrandineArgs { /// /// The command line always wins: any option it set is dropped from the file /// (scalars and lists alike), and the remaining file-derived tokens are - /// spliced in right after the program name so subcommands are preserved. + /// spliced in right after the program name so subcommands are preserved, or + /// appended for `bn` and `vc`, whose options the file then holds. /// `cli_args` must start with the program name, as `std::env::args_os` does. pub fn parse_and_merge_args_file(cli_args: impl IntoIterator) -> Result { let cli_args = cli_args.into_iter().collect::>(); @@ -1655,10 +1992,23 @@ impl GrandineArgs { .ignore_errors(true) .try_get_matches_from(cli_args.iter().cloned())?; + // `bn` and `vc` carry their own options, so the file is read and merged at their level. + let (command, matches, append) = match matches.subcommand() { + Some((name @ (BN_COMMAND | VC_COMMAND), sub_matches)) => { + let subcommand = command + .find_subcommand(name) + .expect("subcommand names come from the command itself") + .clone(); + + (subcommand, sub_matches.clone(), true) + } + _ => (command, matches, false), + }; + let Some(path) = matches.get_one::("args_file").cloned() else { // No file — hand off to the normal strict parse so errors and help // are produced exactly as without `--args-file`. - return Self::try_parse_from(cli_args).map_err(Into::into); + return Self::parse_strictly(cli_args); }; let yaml = fs_err::read_to_string(&path).map_err(Self::clap_error)?; @@ -1667,12 +2017,7 @@ impl GrandineArgs { // Long names of options the command line set. The args file may not // override these, which gives lists the same override semantics as // scalars (the file's value is dropped, not appended). - let cli_set_longs = command - .get_arguments() - .filter(|arg| { - matches.value_source(arg.get_id().as_str()) == Some(ValueSource::CommandLine) - }) - .filter_map(Arg::get_long) + let cli_set_longs = longs_set_on_command_line(&command, &matches) .map(str::to_owned) .collect::>(); @@ -1682,25 +2027,69 @@ impl GrandineArgs { .flat_map(|(_, tokens)| tokens) .map(OsString::from); - let mut cli_args = cli_args.into_iter(); - let program = cli_args - .next() - .unwrap_or_else(|| APPLICATION_NAME.to_owned().into()); + let mut cli_args = cli_args; + + if cli_args.is_empty() { + cli_args.push(APPLICATION_NAME.to_owned().into()); + } + + let splice_after = if append { cli_args.len() } else { 1 }; + let rest = cli_args.split_off(splice_after); + let args = cli_args.into_iter().chain(file_tokens).chain(rest); + + Self::parse_strictly(args) + } + + pub fn parse_strictly(args: impl IntoIterator) -> Result { + let command = Self::command(); + let matches = command.clone().try_get_matches_from(args)?; - let args = core::iter::once(program).chain(file_tokens).chain(cli_args); + if let Some((name @ (BN_COMMAND | VC_COMMAND), _)) = matches.subcommand() + && let Some(option) = longs_set_on_command_line(&command, &matches).next() + { + // Such an option would be parsed for the base node and silently dropped. + bail!(Self::clap_error(Error::OptionBeforeCommand { + option: option.to_owned(), + command: name.to_owned(), + })); + } - Self::try_parse_from(args).map_err(Into::into) + Self::from_arg_matches(&matches).map_err(Into::into) } #[must_use] pub fn data_dir(&self) -> Option { - (!self.beacon_node_options.in_memory) - .then(|| directories::data_directory(self.beacon_node_options.data_dir.as_ref())) + let CommonOptions { + data_dir, + in_memory, + .. + } = self.common_options(); + + (!in_memory).then(|| directories::data_directory(data_dir.as_ref())) + } + + const fn common_options(&self) -> &CommonOptions { + match &self.command { + Some(Command::Bn { common_options, .. } | Command::Vc { common_options, .. }) => { + common_options + } + _ => &self.common_options, + } } #[must_use] pub fn telemetry_config(&self) -> Option { - self.beacon_node_options.telemetry_config() + match &self.command { + Some(Command::Bn { common_options, .. } | Command::Vc { common_options, .. }) => { + common_options.telemetry_options.telemetry_config() + } + _ => self.common_options.telemetry_options.telemetry_config(), + } + } + + #[must_use] + pub const fn has_network(&self) -> bool { + !matches!(self.command, Some(Command::Vc { .. })) } } @@ -1722,6 +2111,13 @@ enum Error { to custom network without --genesis-state-file" )] MissingEth1RpcUrlsForCustomWithoutGenesisState, + #[error( + "--track-liveness must be specified for --detect-doppelgangers \ + with the built-in beacon node" + )] + DoppelgangerDetectionRequiresLivenessTracking, + #[error("--report-validator-performance reads blocks from the built-in beacon node")] + PerformanceReportsRequireBuiltInNode, #[error( "{phase} variables in {preset_name} preset do not match file ({})", differences.iter().join(", "), @@ -1745,6 +2141,18 @@ enum Error { service1: &'static str, service2: &'static str, }, + #[error("--{option} must be specified after {command}")] + OptionBeforeCommand { option: String, command: String }, +} + +fn longs_set_on_command_line<'a>( + command: &'a ClapCommand, + matches: &'a ArgMatches, +) -> impl Iterator { + command + .get_arguments() + .filter(|arg| matches.value_source(arg.get_id().as_str()) == Some(ValueSource::CommandLine)) + .filter_map(Arg::get_long) } fn verify_preset( @@ -1945,27 +2353,26 @@ mod tests { #[test] fn network_config_options() { let config = config_from_args(["--discovery-port", "8888"]); + let node = beacon_node(&config); - assert_eq!(config.network_config.libp2p_nodes, []); + assert_eq!(node.network_config.libp2p_nodes, []); assert_eq!( - config - .network_config + node.network_config .listen_addrs() .v4() .map(|addr| addr.disc_port), Some(8888), ); assert_eq!( - config - .network_config + node.network_config .listen_addrs() .v4() .map(|addr| addr.tcp_port), Some(DEFAULT_LIBP2P_IPV4_PORT.into()), ); - assert_eq!(config.network_config.enr_udp4_port, None); + assert_eq!(node.network_config.enr_udp4_port, None); assert_eq!( - config.network_config.network_dir, + node.network_config.network_dir, Some( dirs::home_dir() .expect("home directory should be accessible") @@ -1977,7 +2384,8 @@ mod tests { #[test] fn listen_address_defaults_to_unspecified_ipv4() { let config = config_from_args([]); - let listen_addrs = config.network_config.listen_addrs(); + let node = beacon_node(&config); + let listen_addrs = node.network_config.listen_addrs(); assert_eq!( listen_addrs.v4().map(|addr| addr.addr), @@ -1990,7 +2398,8 @@ mod tests { #[test] fn listen_address_ipv4_only_uses_given_address() { let config = config_from_args(["--listen-address", "127.0.0.1"]); - let listen_addrs = config.network_config.listen_addrs(); + let node = beacon_node(&config); + let listen_addrs = node.network_config.listen_addrs(); let v4 = listen_addrs .v4() @@ -2006,7 +2415,8 @@ mod tests { #[test] fn listen_address_ipv6_defaults_to_dual_stack() { let config = config_from_args(["--listen-address-ipv6", "::1"]); - let listen_addrs = config.network_config.listen_addrs(); + let node = beacon_node(&config); + let listen_addrs = node.network_config.listen_addrs(); assert_eq!( listen_addrs.v4().map(|addr| addr.addr), @@ -2022,7 +2432,8 @@ mod tests { #[test] fn disable_ipv4_produces_ipv6_only() { let config = config_from_args(["--disable-ipv4", "--listen-address-ipv6", "::1"]); - let listen_addrs = config.network_config.listen_addrs(); + let node = beacon_node(&config); + let listen_addrs = node.network_config.listen_addrs(); assert_eq!(listen_addrs.v4().map(|addr| addr.addr), None); @@ -2036,7 +2447,7 @@ mod tests { assert_eq!(v6.quic_port, DEFAULT_LIBP2P_QUIC_IPV6_PORT.get()); // The ENR must not advertise an IPv4 TCP port when IPv4 is disabled. - assert_eq!(config.network_config.enr_tcp4_port, None); + assert_eq!(node.network_config.enr_tcp4_port, None); } #[test] @@ -2069,6 +2480,238 @@ mod tests { .expect_err("--disable-ipv4 conflicts with --enr-tcp-port"); } + // Options the built-in beacon node alone acts on are refused rather than silently dropped. + #[test] + fn validator_client_refuses_node_only_options() { + try_config_from_args([ + "vc", + "--beacon-node-urls", + "http://localhost:5052", + "--checkpoint-sync-url", + "http://localhost:5053", + ]) + .expect_err("vc does not accept --checkpoint-sync-url"); + + try_config_from_args([ + "vc", + "--beacon-node-urls", + "http://localhost:5052", + "--target-peers", + "10", + ]) + .expect_err("vc does not accept --target-peers"); + + try_config_from_args([ + "vc", + "--beacon-node-urls", + "http://localhost:5052", + "--report-validator-performance", + ]) + .expect_err("performance reports come from the built-in node"); + + try_config_from_args([ + "vc", + "--beacon-node-urls", + "http://localhost:5052", + "--builder-disable-checks", + ]) + .expect_err("the builder circuit breaker is the built-in node's"); + + try_config_from_args([ + "vc", + "--beacon-node-urls", + "http://localhost:5052", + "--builder-url", + "http://localhost:18550", + ]) + .expect_err("the builder is the built-in node's"); + + try_config_from_args([ + "vc", + "--beacon-node-urls", + "http://localhost:5052", + "--disable-wait-for-late-blocks", + ]) + .expect_err("the wait is for blocks the built-in node is processing"); + + try_config_from_args([ + "vc", + "--beacon-node-urls", + "http://localhost:5052", + "--disable-blockprint-graffiti", + ]) + .expect_err("blockprint graffiti is appended by the built-in node"); + + try_config_from_args([ + "vc", + "--beacon-node-urls", + "http://localhost:5052", + "--terminal-block-hash-activation-epoch-override", + "1", + ]) + .expect_err("the terminal block is the built-in node's concern"); + + let config = config_from_args(["bn", "--builder-url", "http://localhost:18550"]); + assert!( + config + .beacon_node_config() + .is_some_and(|node| node.builder_config.is_some()) + ); + + let config = config_from_args([ + "vc", + "--beacon-node-urls", + "http://localhost:5052", + "--max-empty-slots", + "5", + ]); + + assert!(matches!(config.mode, Mode::Remote { .. })); + assert_eq!(config.max_empty_slots, 5); + } + + #[test] + fn beacon_node_urls_accept_a_comma_separated_list() { + let urls = |config: GrandineConfig| match config.mode { + Mode::Remote { + beacon_node_urls, .. + } => beacon_node_urls.len(), + Mode::Local { .. } => 0, + }; + + let separated = + config_from_args(["vc", "--beacon-node-urls", "http://a:5052", "http://b:5052"]); + let joined = config_from_args(["vc", "--beacon-node-urls", "http://a:5052,http://b:5052"]); + + assert_eq!(urls(separated), 2); + assert_eq!(urls(joined), 2); + } + + // Remote beacon nodes are the validator client's concern. + #[test] + fn base_node_refuses_remote_beacon_nodes() { + try_config_from_args(["--beacon-node-urls", "http://localhost:5052"]) + .expect_err("remote beacon nodes are only given to vc"); + try_config_from_args(["--publish-to-every-node", "all"]) + .expect_err("publishing to every node is only given to vc"); + try_config_from_args(["--use-builder"]) + .expect_err("the builder behind the beacon nodes is only given to vc"); + } + + // Without a builder URL of its own the validator client can still register through the nodes. + #[test] + fn validator_client_opts_into_the_beacon_nodes_builder() { + let config = config_from_args(["vc", "--beacon-node-urls", "http://localhost:5052"]); + assert!(matches!( + config.mode, + Mode::Remote { + use_builder: false, + .. + } + )); + + let config = config_from_args([ + "vc", + "--beacon-node-urls", + "http://localhost:5052", + "--use-builder", + ]); + assert!(matches!( + config.mode, + Mode::Remote { + use_builder: true, + .. + } + )); + } + + // Publishing everywhere is the safe default; a single node is what the flag alone selects. + #[test] + fn every_duty_is_published_to_every_node_by_default() { + let vc = ["vc", "--beacon-node-urls", "http://localhost:5052"]; + + let published = + |arguments: &[&str]| match config_from_args(vc.iter().chain(arguments).copied()).mode { + Mode::Remote { + publish_to_every_node, + .. + } => publish_to_every_node, + Mode::Local { .. } => unreachable!("vc runs against remote nodes"), + }; + + assert_eq!(published(&[]), [PublishedDuty::All]); + assert_eq!(published(&["--publish-to-every-node"]), []); + assert_eq!( + published(&["--publish-to-every-node", "attestations,blocks"]), + [PublishedDuty::Attestations, PublishedDuty::Blocks], + ); + } + + #[test] + fn validator_client_requires_beacon_node_urls() { + try_config_from_args(["vc"]).expect_err("the validator client has no built-in node"); + } + + // `bn` takes node options only; anything for a validator is refused. + #[test] + fn bn_command_refuses_validator_options() { + let config = config_from_args(["bn", "--target-peers", "10"]); + let node = beacon_node(&config); + + assert_eq!(node.network_config.target_peers, 10); + assert!(matches!( + config.mode, + Mode::Local { + validator_client_config: None, + .. + } + )); + + try_config_from_args([ + "bn", + "--keystore-dir", + "dir", + "--keystore-password-file", + "pass", + ]) + .expect_err("bn holds no validator keys"); + try_config_from_args(["bn", "--beacon-node-urls", "http://localhost:5052"]) + .expect_err("bn talks to no remote beacon node"); + try_config_from_args(["bn", "--enable-validator-api"]) + .expect_err("bn serves no validator API"); + try_config_from_args(["bn", "--report-validator-performance"]) + .expect_err("bn holds no validator keys to report on"); + } + + // An option before the command would be parsed for the base node and dropped. + #[test] + fn options_before_bn_or_vc_are_refused() { + try_config_from_args(["--max-empty-slots", "5", "bn"]) + .expect_err("--max-empty-slots was given before bn"); + + try_config_from_args([ + "--max-empty-slots", + "5", + "vc", + "--beacon-node-urls", + "http://localhost:5052", + ]) + .expect_err("--max-empty-slots was given before vc"); + } + + #[test] + fn args_file_applies_to_the_validator_client() -> Result<()> { + let config = try_config_from_args_and_file( + ["vc", "--beacon-node-urls", "http://localhost:5052"], + "max-empty-slots: 5\n", + )?; + + assert!(matches!(config.mode, Mode::Remote { .. })); + assert_eq!(config.max_empty_slots, 5); + + Ok(()) + } + #[test] fn listen_address_dual_stack_uses_both_addresses() { let config = config_from_args([ @@ -2077,8 +2720,9 @@ mod tests { "--listen-address-ipv6", "::1", ]); + let node = beacon_node(&config); - let listen_addrs = config.network_config.listen_addrs(); + let listen_addrs = node.network_config.listen_addrs(); assert_eq!( listen_addrs.v4().map(|addr| addr.addr), @@ -2094,15 +2738,17 @@ mod tests { #[test] fn back_sync_disabled_by_default() { let config = config_from_args([]); - assert!(!config.back_sync_enabled); + let node = beacon_node(&config); + assert!(!node.back_sync_enabled); } #[test] fn default_builder_circuit_breaker_settings() { let config = config_from_args([]); + let node = beacon_node(&config); assert_eq!( - config.builder_circuit_breaker, + node.store_config.builder_circuit_breaker, BuilderCircuitBreakerConfig::default(), ); } @@ -2110,8 +2756,9 @@ mod tests { #[test] fn builder_disable_checks_disables_the_gloas_circuit_breaker() { let config = config_from_args(["--builder-disable-checks"]); + let node = beacon_node(&config); - assert!(config.builder_circuit_breaker.disabled); + assert!(node.store_config.builder_circuit_breaker.disabled); } #[test] @@ -2122,10 +2769,16 @@ mod tests { "--builder-max-skipped-slots-per-epoch", "9", ]); + let node = beacon_node(&config); - assert_eq!(config.builder_circuit_breaker.max_skipped_slots, 5); assert_eq!( - config.builder_circuit_breaker.max_skipped_slots_per_epoch, + node.store_config.builder_circuit_breaker.max_skipped_slots, + 5 + ); + assert_eq!( + node.store_config + .builder_circuit_breaker + .max_skipped_slots_per_epoch, 9, ); } @@ -2170,21 +2823,24 @@ mod tests { #[test] fn supports_back_sync_flag() { let config = config_from_args(["--back-sync"]); - assert!(config.back_sync_enabled); + let node = beacon_node(&config); + assert!(node.back_sync_enabled); } #[test] fn supports_deprecated_back_sync_flag() { let config = config_from_args(["--back_sync"]); - assert!(config.back_sync_enabled); + let node = beacon_node(&config); + assert!(node.back_sync_enabled); } #[test] fn eth1_rpc_urls_single_value() { let config = config_from_args(["--eth1-rpc-urls", "http://localhost:8545"]); + let node = beacon_node(&config); itertools::assert_equal( - config.eth1_rpc_urls.iter().map(RedactingUrl::to_string), + node.eth1_rpc_urls.iter().map(RedactingUrl::to_string), ["http://localhost:8545/"], ); } @@ -2196,9 +2852,10 @@ mod tests { "http://localhost:8545", "http://example.com:8545", ]); + let node = beacon_node(&config); itertools::assert_equal( - config.eth1_rpc_urls.iter().map(RedactingUrl::to_string), + node.eth1_rpc_urls.iter().map(RedactingUrl::to_string), ["http://localhost:8545/", "http://example.com:8545/"], ); } @@ -2211,9 +2868,10 @@ mod tests { "--eth1-rpc-urls", "http://example.com:8545", ]); + let node = beacon_node(&config); itertools::assert_equal( - config.eth1_rpc_urls.iter().map(RedactingUrl::to_string), + node.eth1_rpc_urls.iter().map(RedactingUrl::to_string), ["http://localhost:8545/", "http://example.com:8545/"], ); } @@ -2230,9 +2888,10 @@ mod tests { #[test] fn default_store_directory() { let config = config_from_args([]); + let node = beacon_node(&config); assert_eq!( - config.storage_config.directories.store_directory, + node.storage_config.directories.store_directory, Some( dirs::home_dir() .expect("home directory should be accessible") @@ -2244,13 +2903,14 @@ mod tests { #[test] fn data_dir_option() { let config = config_from_args(["--data-dir", "/tmp"]); + let node = beacon_node(&config); assert_eq!( - config.storage_config.directories.store_directory, + node.storage_config.directories.store_directory, Some(PathBuf::from("/tmp/mainnet/beacon")), ); assert_eq!( - config.network_config.network_dir, + node.network_config.network_dir, Some(PathBuf::from("/tmp/mainnet/network")), ); } @@ -2266,15 +2926,17 @@ mod tests { #[test] fn http_api_disabled() { let config = config_from_args(["--http-port", "1234", "--disable-http-api"]); - assert!(config.http_api_config.is_none()); + let node = beacon_node(&config); + assert!(node.http_api_config.is_none()); } #[test] fn http_port_option() { let config = config_from_args(["--http-port", "1234"]); + let node = beacon_node(&config); assert_eq!( - config.http_api_config.map(|config| config.address), + node.http_api_config.as_ref().map(|config| config.address), Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 1234)), ); } @@ -2282,12 +2944,15 @@ mod tests { #[test] fn http_allowed_origins_default() { let config = config_from_args([]); + let node = beacon_node(&config); // `Debug` is the only way to inspect the contents of `AllowOrigin`. assert_eq!( format!( "{:?}", - config.http_api_config.map(|config| config.allow_origin) + node.http_api_config + .as_ref() + .map(|config| &config.allow_origin) ), "Some(List([\"http://127.0.0.1:5052\"]))", ); @@ -2296,12 +2961,15 @@ mod tests { #[test] fn http_allowed_origins_option_single_occurrence() { let config = config_from_args(["--http-allowed-origins", "*"]); + let node = beacon_node(&config); // `Debug` is the only way to inspect the contents of `AllowOrigin`. assert_eq!( format!( "{:?}", - config.http_api_config.map(|config| config.allow_origin) + node.http_api_config + .as_ref() + .map(|config| &config.allow_origin) ), "Some(Const(\"*\"))", ); @@ -2315,12 +2983,15 @@ mod tests { "--http-allowed-origins", "http://example.com", ]); + let node = beacon_node(&config); // `Debug` is the only way to inspect the contents of `AllowOrigin`. assert_eq!( format!( "{:?}", - config.http_api_config.map(|config| config.allow_origin) + node.http_api_config + .as_ref() + .map(|config| &config.allow_origin) ), "Some(List([\"http://localhost\", \"http://example.com\"]))", ); @@ -2336,12 +3007,15 @@ mod tests { "--http-allowed-origins", "*", ]); + let node = beacon_node(&config); // `Debug` is the only way to inspect the contents of `AllowOrigin`. assert_eq!( format!( "{:?}", - config.http_api_config.map(|config| config.allow_origin) + node.http_api_config + .as_ref() + .map(|config| &config.allow_origin) ), "Some(Const(\"*\"))", ); @@ -2356,7 +3030,7 @@ mod tests { #[test] fn telemetry_config_options() { let config = config_from_args(["--telemetry-metrics-url", "http://localhost:4317"]); - let telemetry_config = config.telemetry_config; + let telemetry_config = &config.telemetry_config; assert_eq!( telemetry_config @@ -2371,7 +3045,9 @@ mod tests { ); assert_eq!( - telemetry_config.map(|config| config.service_name), + telemetry_config + .as_ref() + .map(|config| config.service_name.clone()), Some("Grandine".to_owned()), ); } @@ -2386,8 +3062,7 @@ mod tests { "--telemetry-level", "debug", ]); - - let telemetry_config = config.telemetry_config; + let telemetry_config = &config.telemetry_config; assert_eq!( telemetry_config @@ -2402,11 +3077,27 @@ mod tests { ); assert_eq!( - telemetry_config.map(|config| config.service_name), + telemetry_config + .as_ref() + .map(|config| config.service_name.clone()), Some("grandine-bn".to_owned()), ); } + // The exporter is the process's, so the validator client can use it as the node does. + #[test] + fn telemetry_config_applies_to_the_validator_client() { + let config = config_from_args([ + "vc", + "--beacon-node-urls", + "http://localhost:5052", + "--telemetry-metrics-url", + "http://localhost:4317", + ]); + + assert!(config.telemetry_config.is_some()); + } + #[test] fn telemetry_config_service_name_without_url() { try_config_from_args(["--telemetry-service-name", "grandine-bn"]).expect_err( @@ -2432,8 +3123,8 @@ mod tests { assert_eq!( config - .validator_api_config - .as_ref() + .validator_client_config() + .and_then(|config| config.validator_api_config.as_ref()) .map(|config| config.address), Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 1234)), ); @@ -2450,17 +3141,29 @@ mod tests { assert_eq!( config - .validator_api_config - .as_ref() + .validator_client_config() + .and_then(|config| config.validator_api_config.as_ref()) .map(|config| config.address), None, ); } + fn beacon_node(config: &GrandineConfig) -> &BeaconNodeConfig { + config + .beacon_node_config() + .expect("the built-in beacon node is enabled") + } + + fn validators(config: &GrandineConfig) -> Option<&Validators> { + config + .validator_client_config() + .and_then(|config| config.validators.as_ref()) + } + #[test] fn validators_from_no_keystore_paths() { let config = config_from_args([]); - assert_eq!(config.validators, None); + assert_eq!(validators(&config), None); } #[test] @@ -2473,8 +3176,8 @@ mod tests { ]); assert_eq!( - config.validators, - Some(Validators { + validators(&config), + Some(&Validators { keystore_dir: PathBuf::from("dir_value"), keystore_password_file: PathBuf::from("pass_file"), }), @@ -2491,8 +3194,8 @@ mod tests { ]); assert_eq!( - config.validators, - Some(Validators { + validators(&config), + Some(&Validators { keystore_dir: PathBuf::from("dir_value"), keystore_password_file: PathBuf::from("pass_dir"), }), @@ -2524,9 +3227,10 @@ mod tests { "--genesis-state-file", "custom.ssz", ]); + let node = beacon_node(&config); assert_eq!(config.predefined_network, Some(PredefinedNetwork::Sepolia)); - assert_eq!(config.deposit_contract_starting_block, Some(0)); + assert_eq!(node.deposit_contract_starting_block, Some(0)); assert_eq!(config.genesis_state_file, Some(PathBuf::from("custom.ssz"))); } @@ -2637,8 +3341,8 @@ mod tests { let config = config_from_args(["interchange", "import", "test.json"]); assert_eq!( - config.command, - Some(GrandineCommand::Interchange(InterchangeCommand::Import { + config.command(), + Some(&GrandineCommand::Interchange(InterchangeCommand::Import { file_path: PathBuf::from("test.json"), })), ); @@ -2649,8 +3353,8 @@ mod tests { let config = config_from_args(["interchange", "export", "test.json"]); assert_eq!( - config.command, - Some(GrandineCommand::Interchange(InterchangeCommand::Export { + config.command(), + Some(&GrandineCommand::Interchange(InterchangeCommand::Export { file_path: PathBuf::from("test.json"), })), ); @@ -2669,8 +3373,8 @@ mod tests { ]); assert_eq!( - config.command, - Some(GrandineCommand::Export { + config.command(), + Some(&GrandineCommand::Export { from: 0, to: 20, output_dir: Some(PathBuf::from("export")), @@ -2684,8 +3388,8 @@ mod tests { config_from_args(["replay", "--from", "0", "--to", "20", "--input-dir", "data"]); assert_eq!( - config.command, - Some(GrandineCommand::Replay { + config.command(), + Some(&GrandineCommand::Replay { from: 0, to: 20, input_dir: Some(PathBuf::from("data")), @@ -2796,20 +3500,20 @@ mod tests { ", ) .expect("config should be built from --args-file"); + let node = beacon_node(&config); assert_eq!( - config - .network_config + node.network_config .listen_addrs() .v4() .map(|addr| addr.disc_port), Some(8888), ); - assert!(config.back_sync_enabled); + assert!(node.back_sync_enabled); itertools::assert_equal( - config.eth1_rpc_urls.iter().map(RedactingUrl::to_string), + node.eth1_rpc_urls.iter().map(RedactingUrl::to_string), ["http://localhost:8545/", "http://example.com:8545/"], ); } @@ -2841,11 +3545,12 @@ mod tests { "checkpoint-sync-url: https://checkpoint.example\n", ) .expect("a requirement split across the command line and args file should be satisfied"); + let node = beacon_node(&config); - assert!(config.force_checkpoint_sync); + assert!(node.force_checkpoint_sync); assert_eq!( - config.checkpoint_sync_url.map(|url| url.to_string()), + node.checkpoint_sync_url.as_ref().map(ToString::to_string), Some("https://checkpoint.example/".to_owned()), ); } @@ -2854,8 +3559,9 @@ mod tests { fn args_file_empty_sequence_keeps_default() { let config = try_config_from_args_file("eth1-rpc-urls: []") .expect("an empty sequence should be treated as unset"); + let node = beacon_node(&config); - assert!(config.eth1_rpc_urls.is_empty()); + assert!(node.eth1_rpc_urls.is_empty()); } #[test] @@ -2864,10 +3570,10 @@ mod tests { "libp2p-nodes: /ip4/127.0.0.1/tcp/9000,/ip4/127.0.0.2/tcp/9001", ) .expect("a comma-delimited scalar should split into multiple values"); + let node = beacon_node(&config); itertools::assert_equal( - config - .network_config + node.network_config .libp2p_nodes .iter() .map(ToString::to_string), @@ -2899,18 +3605,19 @@ mod tests { ", ) .expect("a full node configuration should be built from --args-file"); + let node = beacon_node(&config); assert_eq!(config.predefined_network, Some(PredefinedNetwork::Sepolia)); assert_eq!( - config.checkpoint_sync_url.map(|url| url.to_string()), + node.checkpoint_sync_url.as_ref().map(ToString::to_string), Some("https://checkpoint.sepolia.example/".to_owned()), ); - assert!(config.force_checkpoint_sync); - assert!(config.back_sync_enabled); + assert!(node.force_checkpoint_sync); + assert!(node.back_sync_enabled); itertools::assert_equal( - config.eth1_rpc_urls.iter().map(RedactingUrl::to_string), + node.eth1_rpc_urls.iter().map(RedactingUrl::to_string), ["http://localhost:8545/", "http://localhost:8546/"], ); @@ -2922,8 +3629,7 @@ mod tests { assert_eq!(config.default_builder_boost_factor, Uint256::from_u64(90)); assert_eq!(config.max_empty_slots, 5); assert_eq!( - config - .network_config + node.network_config .listen_addrs() .v4() .map(|addr| addr.disc_port), @@ -2931,7 +3637,7 @@ mod tests { ); assert_eq!( - config.http_api_config.map(|config| config.address), + node.http_api_config.as_ref().map(|config| config.address), Some(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 6000)), ); @@ -2939,12 +3645,17 @@ mod tests { config .metrics_config .metrics_server_config + .as_ref() .map(|config| config.metrics_port), Some(9000), ); - assert!(config.track_liveness); - assert!(config.detect_doppelgangers); + assert!(node.track_liveness); + assert!( + config + .validator_client_config() + .is_some_and(|config| config.detect_doppelgangers) + ); } #[test] @@ -2952,10 +3663,10 @@ mod tests { let config = try_config_from_args_and_file(["--discovery-port", "7000"], "discovery-port: 8888") .expect("a command-line scalar should override the args file"); + let node = beacon_node(&config); assert_eq!( - config - .network_config + node.network_config .listen_addrs() .v4() .map(|addr| addr.disc_port), @@ -2973,9 +3684,10 @@ mod tests { ", ) .expect("a command-line list should override the args file list"); + let node = beacon_node(&config); itertools::assert_equal( - config.eth1_rpc_urls.iter().map(RedactingUrl::to_string), + node.eth1_rpc_urls.iter().map(RedactingUrl::to_string), ["http://cli:8545/"], ); } @@ -2990,9 +3702,10 @@ mod tests { ", ) .expect("the args file list should be used when the command line omits it"); + let node = beacon_node(&config); itertools::assert_equal( - config.eth1_rpc_urls.iter().map(RedactingUrl::to_string), + node.eth1_rpc_urls.iter().map(RedactingUrl::to_string), ["http://file:8545/"], ); } @@ -3004,17 +3717,18 @@ mod tests { "back-sync: true", ) .expect("a subcommand should survive args file merging"); + let node = beacon_node(&config); assert_eq!( - config.command, - Some(GrandineCommand::Export { + config.command(), + Some(&GrandineCommand::Export { from: 0, to: 20, output_dir: None, }), ); - assert!(config.back_sync_enabled); + assert!(node.back_sync_enabled); } fn config_from_args<'a>(arguments: impl IntoIterator) -> GrandineConfig { @@ -3041,8 +3755,15 @@ mod tests { .expect("temporary path should be UTF-8") .to_owned(); - let mut argv = vec!["--args-file".to_owned(), path]; - argv.extend(arguments.into_iter().map(str::to_owned)); + let mut argv = arguments.into_iter().map(str::to_owned).collect::>(); + + // `bn` and `vc` take the option themselves; the other commands leave it to the root. + let position = usize::from(matches!( + argv.first().map(String::as_str), + Some(BN_COMMAND | VC_COMMAND) + )); + + argv.splice(position..position, ["--args-file".to_owned(), path]); try_config_from_args(argv.iter().map(String::as_str)) } diff --git a/runtime/src/grandine_config.rs b/runtime/src/grandine_config.rs index 568db8706..f2da2c9db 100644 --- a/runtime/src/grandine_config.rs +++ b/runtime/src/grandine_config.rs @@ -3,127 +3,251 @@ use std::{collections::HashSet, path::PathBuf, sync::Arc}; use binary_utils::TelemetryConfig; use builder_api::{BuilderConfig, gloas::containers::BuilderUrl}; +use directories::Directories; use eth1_api::AuthOptions; -use fork_choice_store::BuilderCircuitBreakerConfig; +use fork_choice_store::StoreConfig; +use grandine_version::APPLICATION_VERSION_WITH_COMMIT_AND_PLATFORM; use http_api::HttpApiConfig; use itertools::Itertools as _; -use kzg_utils::KzgBackend; +use keymanager::{ + BuilderSettings, GRANDINE_DONATION_ADDRESS, ValidatorDefinitions, + ValidatorDefinitionsWithStorage, +}; use p2p::NetworkConfig; use signer::Web3SignerConfig; +use slasher::SlasherConfig; use ssz::Uint256; use tracing::info; use types::{ bellatrix::primitives::Gas, config::Config as ChainConfig, - nonstandard::CustodyMode, + nonstandard::{CustodyMode, PublishedDuty}, phase0::primitives::{ExecutionAddress, ExecutionBlockNumber, Gwei, H256, Slot}, redacting_url::RedactingUrl, }; -use validator::ValidatorApiConfig; +use validator::{ValidatorApiConfig, ValidatorConfig}; use crate::{ MetricsConfig, StorageConfig, commands::GrandineCommand, predefined_network::PredefinedNetwork, validators::Validators, }; +/// Where duties are performed: the built-in beacon node, or under `vc` remote beacon nodes. +#[expect( + clippy::large_enum_variant, + reason = "Built once and consumed at once; boxing would only add indirection." +)] +#[derive(Clone)] +#[cfg_attr(test, derive(Debug))] +pub enum Mode { + /// The built-in beacon node, with the validator client unless `bn` was given. + Local { + beacon_node_config: BeaconNodeConfig, + validator_client_config: Option, + command: Option, + }, + /// The validator client alone, against the nodes given with `--beacon-node-urls`. + Remote { + validator_client_config: ValidatorClientConfig, + beacon_node_urls: Vec, + publish_to_every_node: Vec, + use_builder: bool, + }, +} + #[expect( clippy::struct_excessive_bools, reason = "False positive. The `bool`s are independent." )] +#[derive(Clone)] #[cfg_attr(test, derive(Debug))] pub struct GrandineConfig { pub predefined_network: Option, pub chain_config: Arc, - pub deposit_contract_starting_block: Option, pub genesis_state_file: Option, - pub genesis_state_download_url: Option, - pub checkpoint_sync_url: Option, - pub force_checkpoint_sync: bool, - pub back_sync_enabled: bool, - pub eth1_rpc_urls: Vec, pub data_dir: PathBuf, - pub validators: Option, - pub keystore_storage_password_file: Option, + pub directories: Arc, + pub in_memory: bool, + pub request_timeout: Duration, + pub metrics_config: MetricsConfig, pub disable_blockprint_graffiti: bool, pub graffiti: Vec, pub max_empty_slots: u64, - pub suggested_fee_recipient: ExecutionAddress, + pub suggested_fee_recipient: Option, pub default_builder_boost_factor: Uint256, pub default_gas_limit: Option, pub default_builder_min_bid: Gwei, pub default_builder_max_execution_payment: Gwei, pub allow_trusted_payments: bool, pub payload_builder_urls: Vec, + pub disable_wait_for_late_blocks: bool, + pub telemetry_config: Option, + pub mode: Mode, +} + +/// What only the built-in beacon node needs; `vc` has none. +#[expect( + clippy::struct_excessive_bools, + reason = "False positive. The `bool`s are independent." +)] +#[derive(Clone)] +#[cfg_attr(test, derive(Debug))] +pub struct BeaconNodeConfig { + pub deposit_contract_starting_block: Option, + pub checkpoint_sync_url: Option, + pub genesis_state_download_url: Option, + pub force_checkpoint_sync: bool, + pub back_sync_enabled: bool, + pub eth1_rpc_urls: Vec, + pub auth_options: AuthOptions, pub network_config: NetworkConfig, pub storage_config: StorageConfig, - pub unfinalized_states_in_memory: u64, + pub store_config: StoreConfig, pub reconstruction_delay: Duration, - pub request_timeout: Duration, - pub max_epochs_to_retain_states_in_cache: u64, - pub state_cache_lock_timeout: Duration, - pub command: Option, - pub slashing_enabled: bool, - pub slashing_history_limit: u64, + pub slasher_config: Option, pub state_slot: Option, - pub auth_options: AuthOptions, - pub builder_config: Option, - pub builder_circuit_breaker: BuilderCircuitBreakerConfig, - pub web3signer_config: Web3SignerConfig, pub http_api_config: Option, pub max_events: usize, - pub metrics_config: MetricsConfig, - pub telemetry_config: Option, pub track_liveness: bool, + pub blacklisted_blocks: HashSet, + pub backfill_custody_groups: bool, + pub custody_mode: CustodyMode, + pub builder_config: Option, +} + +/// What only a running validator client needs; `bn` has none. +#[derive(Clone)] +#[cfg_attr(test, derive(Debug))] +pub struct ValidatorClientConfig { + pub validators: Option, + pub keystore_storage_password_file: Option, + pub web3signer_config: Web3SignerConfig, pub detect_doppelgangers: bool, pub use_validator_key_cache: bool, pub slashing_protection_history_limit: u64, - pub in_memory: bool, pub validator_api_config: Option, - pub kzg_backend: KzgBackend, - pub blacklisted_blocks: HashSet, pub report_validator_performance: bool, - pub backfill_custody_groups: bool, - pub sync_without_reconstruction: bool, - pub custody_mode: CustodyMode, - pub disable_wait_for_late_blocks: bool, +} + +impl ValidatorClientConfig { + #[must_use] + pub fn expects_keys(&self, validator_definitions: &ValidatorDefinitions) -> bool { + self.validator_api_config.is_some() + || self.use_validator_key_cache + || !self.web3signer_config.is_empty() + || !validator_definitions.is_empty() + || self.keystore_storage_password_file.is_some() + } } impl GrandineConfig { - #[expect(clippy::cognitive_complexity)] + #[must_use] + pub const fn beacon_node_config(&self) -> Option<&BeaconNodeConfig> { + match &self.mode { + Mode::Local { + beacon_node_config, .. + } => Some(beacon_node_config), + Mode::Remote { .. } => None, + } + } + + #[must_use] + pub const fn beacon_node_config_mut(&mut self) -> Option<&mut BeaconNodeConfig> { + match &mut self.mode { + Mode::Local { + beacon_node_config, .. + } => Some(beacon_node_config), + Mode::Remote { .. } => None, + } + } + + #[must_use] + pub const fn validator_client_config(&self) -> Option<&ValidatorClientConfig> { + match &self.mode { + Mode::Local { + validator_client_config, + .. + } => validator_client_config.as_ref(), + Mode::Remote { + validator_client_config, + .. + } => Some(validator_client_config), + } + } + + #[must_use] + pub const fn command(&self) -> Option<&GrandineCommand> { + match &self.mode { + Mode::Local { command, .. } => command.as_ref(), + Mode::Remote { .. } => None, + } + } + + #[must_use] + pub fn default_fee_recipient(&self) -> Option { + // A remote node pays its own recipient, so only the built-in node gets the fallback. + self.suggested_fee_recipient + .or_else(|| self.beacon_node_config().map(|_| GRANDINE_DONATION_ADDRESS)) + } + + #[must_use] + pub fn validator_config( + &self, + validator_definitions: Arc, + ) -> ValidatorConfig { + let Self { + disable_blockprint_graffiti, + graffiti, + max_empty_slots, + default_builder_boost_factor, + default_gas_limit, + default_builder_min_bid, + default_builder_max_execution_payment, + allow_trusted_payments, + payload_builder_urls, + disable_wait_for_late_blocks, + .. + } = self; + + let custody_mode = self + .beacon_node_config() + .map_or_else(CustodyMode::default, |node| node.custody_mode); + + ValidatorConfig { + disable_blockprint_graffiti: *disable_blockprint_graffiti, + graffiti: graffiti.clone(), + max_empty_slots: *max_empty_slots, + suggested_fee_recipient: self.default_fee_recipient(), + default_gas_limit: *default_gas_limit, + builder_settings: BuilderSettings { + default_builder_boost_factor: *default_builder_boost_factor, + default_builder_min_bid: *default_builder_min_bid, + default_builder_max_execution_payment: *default_builder_max_execution_payment, + allow_trusted_payments: *allow_trusted_payments, + payload_builder_urls: payload_builder_urls.clone(), + }, + custody_mode, + disable_wait_for_late_blocks: *disable_wait_for_late_blocks, + validator_definitions, + } + } + #[expect(clippy::too_many_lines)] pub fn report(&self) { let Self { predefined_network, chain_config, - back_sync_enabled, - eth1_rpc_urls, data_dir, + metrics_config, disable_blockprint_graffiti, graffiti, - suggested_fee_recipient, default_builder_boost_factor, - network_config, - storage_config, - slashing_enabled, - slashing_history_limit, - state_slot, - builder_config, - web3signer_config, - http_api_config, - metrics_config, + default_gas_limit, telemetry_config, - checkpoint_sync_url, - use_validator_key_cache, - validator_api_config, - sync_without_reconstruction, - custody_mode, .. } = self; - let StorageConfig { - archival_epoch_interval, - .. - } = storage_config; + info!("client version: {APPLICATION_VERSION_WITH_COMMIT_AND_PLATFORM}"); match predefined_network { Some(network) => info!("network: {network}"), @@ -133,24 +257,25 @@ impl GrandineConfig { ), } - info!("storage mode: {:?}", storage_config.storage_mode); info!("data directory: {}", data_dir.display()); - self.storage_config.print_db_sizes(); + if let Some(config) = telemetry_config { + info!("telemetry export configured with: {config:?}"); + } else { + info!("telemetry metrics data export disabled"); + } + + match self.beacon_node_config() { + Some(node) => node.report(chain_config), + None => info!("built-in beacon node disabled"), + } - info!("Eth1 RPC URLs: [{}]", eth1_rpc_urls.iter().join(", ")); info!("graffiti: {graffiti:?}"); if *disable_blockprint_graffiti { info!("blockprint graffiti disabled"); } - if let Some(http_api_config) = http_api_config { - info!("HTTP API address: {}", http_api_config.address); - } else { - info!("HTTP API disabled"); - } - if let Some(metrics_server_config) = &metrics_config.metrics_server_config { info!( "metrics server address: {}", @@ -165,23 +290,117 @@ impl GrandineConfig { ); } - if let Some(config) = telemetry_config { - info!("telemetry export configured with: {config:?}"); - } else { - info!("telemetry metrics data export disabled"); + match self + .validator_client_config() + .map(|config| config.validator_api_config.as_ref()) + { + Some(Some(validator_api_config)) => { + info!("validator API address: {}", validator_api_config.address); + } + Some(None) => info!("validator API disabled"), + None => info!("validator client disabled"), } - if let Some(validator_api_config) = validator_api_config.as_ref() { - info!("validator API address: {}", validator_api_config.address); + if let Some(builder_config) = self + .beacon_node_config() + .and_then(|node| node.builder_config.as_ref()) + { + info!( + "using external block builder (API URL: {}, format: {}, \ + default_builder_boost_factor: {default_builder_boost_factor})", + builder_config.builder_api_url, builder_config.builder_api_format, + ); + } + + if let Mode::Remote { + beacon_node_urls, + publish_to_every_node, + use_builder, + .. + } = &self.mode + { + info!( + "performing validator duties against remote beacon nodes without the built-in \ + beacon node: [{}]", + beacon_node_urls.iter().join(", "), + ); + + if !publish_to_every_node.is_empty() { + info!( + "publishing to every remote beacon node: [{}]", + publish_to_every_node.iter().join(", "), + ); + } + + if *use_builder { + info!( + "registering validators with the builder behind the remote beacon nodes \ + (default_builder_boost_factor: {default_builder_boost_factor}, \ + default gas limit: {default_gas_limit:?})", + ); + } + } + + if let Some(validator_client) = self.validator_client_config() { + if !validator_client.web3signer_config.urls.is_empty() { + info!( + "using Web3Signer API to sign validator messages (API URLs: [{}])", + validator_client.web3signer_config.urls.keys().join(", "), + ); + } + + if validator_client.use_validator_key_cache { + info!("using validator key cache"); + } + + if validator_client.detect_doppelgangers { + info!("doppelganger protection enabled"); + } + } + + if let Some(fee_recipient) = self.default_fee_recipient() { + info!("suggested fee recipient: {fee_recipient:?}"); + } + } +} + +impl BeaconNodeConfig { + fn report(&self, chain_config: &ChainConfig) { + let Self { + checkpoint_sync_url, + back_sync_enabled, + eth1_rpc_urls, + network_config, + storage_config, + store_config, + slasher_config, + state_slot, + http_api_config, + track_liveness, + custody_mode, + .. + } = self; + + info!("storage mode: {:?}", storage_config.storage_mode); + + storage_config.print_db_sizes(); + + info!("Eth1 RPC URLs: [{}]", eth1_rpc_urls.iter().join(", ")); + + if let Some(http_api_config) = http_api_config { + info!("HTTP API address: {}", http_api_config.address); } else { - info!("validator API disabled"); + info!("HTTP API disabled"); } - info!("archival interval: {archival_epoch_interval} epochs"); - info!("slasher enabled: {slashing_enabled}"); + info!( + "archival interval: {} epochs", + storage_config.archival_epoch_interval + ); + info!("slasher enabled: {}", slasher_config.is_some()); - if let Some(client_version) = &network_config.identify_agent_version { - info!("client version: {client_version}"); + if *track_liveness { + info!("validator liveness tracking enabled"); } if !network_config.trusted_peers.is_empty() { @@ -192,37 +411,20 @@ impl GrandineConfig { info!("force state slot: {slot}"); } - if let Some(builder_config) = builder_config { - info!( - "using external block builder (API URL: {}, format: {}, \ - default_builder_boost_factor: {default_builder_boost_factor})", - builder_config.builder_api_url, builder_config.builder_api_format, - ); - } - if let Some(checkpoint_sync_url) = checkpoint_sync_url { info!("checkpoint sync url: {checkpoint_sync_url}"); } - if !web3signer_config.urls.is_empty() { + if let Some(slasher_config) = slasher_config { info!( - "using Web3Signer API to sign validator messages (API URLs: [{}])", - web3signer_config.urls.keys().join(", "), + "slasher history limit: {}", + slasher_config.slashing_history_limit ); } - if *slashing_enabled { - info!("slasher history limit: {slashing_history_limit}"); - } - - info!("suggested fee recipient: {suggested_fee_recipient}"); info!("back-sync enabled: {back_sync_enabled}"); - if *use_validator_key_cache { - info!("using validator key cache"); - } - - if *sync_without_reconstruction { + if store_config.sync_without_reconstruction { info!("sync with reconstruction disabled"); } diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index a9fd7349a..99a94c6d6 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -11,7 +11,7 @@ pub use crate::{ DEFAULT_TARGET_SUBNET_PEERS, DEFAULT_TIMEOUT, default_network_config, }, misc::{MetricsConfig, StorageConfig}, - runtime::{RuntimeConfig, run, run_after_genesis}, + runtime::run, schema::initialize as initialize_schema, }; @@ -20,7 +20,6 @@ pub use crate::runtime::shutdown; pub mod commands; mod config_dir; -mod consts; mod db_info; pub mod db_stats; mod defaults; diff --git a/runtime/src/predefined_network.rs b/runtime/src/predefined_network.rs index e48227f9e..ed8ddf00c 100644 --- a/runtime/src/predefined_network.rs +++ b/runtime/src/predefined_network.rs @@ -1,7 +1,7 @@ use core::time::Duration; use std::{io::ErrorKind, path::Path, sync::Arc}; -use anyhow::{Context as _, Result, bail}; +use anyhow::{Context as _, Result, bail, ensure}; use deposit_tree::DepositTree; use genesis::AnchorCheckpointProvider; use logging::info_with_peers; @@ -20,7 +20,6 @@ use types::{ use crate::default_network_config; -#[cfg(any(feature = "network-mainnet", test))] use ::{hex_literal::hex, types::phase0::primitives::H256}; #[derive(Clone, Copy, Display)] @@ -155,6 +154,7 @@ impl PredefinedNetwork { .parse() .expect("hard-coded genesis state download URL should be valid") }), + self.genesis_validators_root(), ) }; @@ -186,6 +186,26 @@ impl PredefinedNetwork { .pipe(Ok) } + #[must_use] + pub fn genesis_validators_root(self) -> H256 { + match self { + #[cfg(any(feature = "network-mainnet", test))] + Self::Mainnet => predefined_chains::mainnet_genesis_validators_root(), + #[cfg(any(feature = "network-sepolia", test))] + Self::Sepolia => H256(hex!( + "d8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078" + )), + #[cfg(any(feature = "network-holesky", test))] + Self::Holesky => H256(hex!( + "9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1" + )), + #[cfg(any(feature = "network-hoodi", test))] + Self::Hoodi => H256(hex!( + "212f13fc4df078b6cb7db228f1c8307566dcecf900867401a92023d7ba99cb5f" + )), + } + } + #[must_use] pub fn genesis_deposit_tree(self) -> DepositTree { match self { @@ -277,6 +297,7 @@ async fn load_or_download_genesis_checkpoint( client: &Client, store_directory: impl AsRef + Send, download_url: RedactingUrl, + genesis_validators_root: H256, ) -> Result>> { let genesis_state_path = store_directory.as_ref().join("genesis_state.ssz"); @@ -300,7 +321,7 @@ async fn load_or_download_genesis_checkpoint( .await?; fs_err::create_dir_all(&store_directory)?; - fs_err::tokio::write(genesis_state_path, &bytes).await?; + fs_err::tokio::write(genesis_state_path.as_path(), &bytes).await?; bytes } @@ -310,6 +331,14 @@ async fn load_or_download_genesis_checkpoint( let state = Arc::from_ssz(config, ssz_bytes)?; let block = Arc::new(genesis::beacon_block(&state)); + ensure!( + state.genesis_validators_root() == genesis_validators_root, + "genesis state at {} has genesis validators root {:?} where {genesis_validators_root:?} \ + was expected", + genesis_state_path.display(), + state.genesis_validators_root(), + ); + info_with_peers!("genesis state loaded at slot: {}", state.slot()); Ok(WithOrigin::new_from_genesis(FinalizedCheckpoint { diff --git a/runtime/src/runtime.rs b/runtime/src/runtime.rs index ffb3ad049..fb565682b 100644 --- a/runtime/src/runtime.rs +++ b/runtime/src/runtime.rs @@ -8,7 +8,7 @@ use std::{ collections::HashSet, io::ErrorKind, net::{TcpListener, UdpSocket}, - path::{Path, PathBuf}, + path::Path, sync::{Arc, RwLock}, }; @@ -17,27 +17,32 @@ use anyhow::{Context as _, Result, bail, ensure}; use attestation_verifier::AttestationVerifier; use binary_utils::TracingHandle; use block_producer::BlockProducer; -use builder_api::{BuilderApi, BuilderConfig, PayloadBuilderApi}; +use bls::PublicKeyBytes; +use builder_api::{BuilderApi, PayloadBuilderApi}; use bytesize::ByteSize; use clock::Tick; use data_dumper::DataDumper; use database::{Database, DatabaseMode, RestartMessage}; use dedicated_executor::DedicatedExecutor; +use directories::Directories; use doppelganger_protection::DoppelgangerProtection; use eth1::{Eth1Chain, Eth1Config}; use eth1_api::{ Auth, Eth1Api, Eth1ApiToMetrics, Eth1ConnectionData, Eth1ExecutionEngine, Eth1Metrics, ExecutionBlobFetcher, ExecutionService, RealController, }; +use eth2_libp2p::NetworkGlobals; use features::Feature; -use fork_choice_control::{Controller, EventChannels, StateLoadStrategy, Storage}; -use fork_choice_store::StoreConfig; +use fork_choice_control::{ + Controller, EventChannels, StateLoadStrategy, Storage, ValidatorMessage, +}; use futures::{ + FutureExt as _, channel::{ mpsc::{self, UnboundedReceiver, UnboundedSender}, oneshot, }, - future::Either, + future::{BoxFuture, Either}, lock::Mutex, stream::StreamExt as _, }; @@ -46,16 +51,18 @@ use grandine_version::{ APPLICATION_NAME_WITH_VERSION_AND_COMMIT, APPLICATION_VERSION_WITH_COMMIT_AND_PLATFORM, }; use helper_functions::misc; -use http_api::{Channels as HttpApiChannels, HttpApi, HttpApiConfig}; +use http_api::{Channels as HttpApiChannels, HttpApi}; use keymanager::{ - BuilderSettings, DefinitionsStorage, KeyManager, LegacyMigration, ValidatorDefinitions, + DefinitionsStorage, KeyManager, LegacyMigration, ProposerConfigs, ValidatorDefinitions, ValidatorDefinitionsWithStorage, }; use liveness_tracker::LivenessTracker; use logging::{ PEER_LOG_METRICS, debug_with_peers, error_with_peers, info_with_peers, warn_with_peers, }; -use metrics::{MetricsChannels, MetricsServerConfig, MetricsService, run_metrics_server}; +use metrics::{ + MetricsChannels, MetricsServerConfig, MetricsService, NodeMetrics, run_metrics_server, +}; use operation_pools::{ AttestationAggPool, BlobReconstructionPool, BlsToExecutionChangePool, Manager, PayloadAttestationAggPool, SyncCommitteeAggPool, @@ -64,31 +71,37 @@ use p2p::{ BlockSyncService, BlockSyncServiceChannels, Channels, ListenAddr, Network, NetworkConfig, SubnetService, }; +use prometheus_client::registry::Registry; use pubkey_cache::PubkeyCache; use reqwest::{Client, ClientBuilder, redirect::Policy}; use scc::HashMap as SccHashMap; -use signer::{KeyOrigin, Signer, Web3SignerClientOptions, build_web3signer_client}; -use slasher::{Databases, Slasher, SlasherConfig}; -use slashing_protection::{SlashingProtector, interchange_format::InterchangeData}; +use signer::{ + KeyOrigin, Signer, Web3SignerClientOptions, Web3SignerConfig, build_web3signer_client, +}; +use slasher::{Databases, Slasher}; +use slashing_protection::{ + DEFAULT_SLASHING_PROTECTION_HISTORY_LIMIT, SlashingProtector, + interchange_format::InterchangeData, +}; use ssz::SszRead as _; use std_ext::ArcExt as _; use thiserror::Error; -use tokio::{runtime::Builder, select}; +use tokio::{runtime::Builder, select, time::sleep}; #[cfg(feature = "embed")] use tokio_util::sync::CancellationToken; use tracing::{error, info, warn}; use types::{ config::Config as ChainConfig, - phase0::{ - consts::GENESIS_SLOT, - primitives::{ExecutionBlockNumber, H256, Slot}, - }, + nonstandard::PublishedDuty, + phase0::{consts::GENESIS_SLOT, primitives::H256}, preset::{Preset, PresetName}, redacting_url::RedactingUrl, traits::{BeaconState as _, SignedBeaconBlock as _}, }; use validator::{ - Validator, ValidatorApiConfig, ValidatorChannels, ValidatorConfig, run_validator_api, + Chain, ChainSource, Genesis, LocalChain, LocalValidatorChannels, OwnValidatorIndices, + RemoteBeaconNode, RemoteBeaconNodes, Validator, ValidatorApiConfig, ValidatorChannels, + ValidatorConfig, ValidatorStartupError, run_validator_api, }; use validator_key_cache::ValidatorKeyCache; use validator_statistics::ValidatorStatistics; @@ -97,11 +110,11 @@ use crate::{ commands::{GrandineCommand, InterchangeCommand}, db_info, db_stats, grandine_args::GrandineArgs, - grandine_config::GrandineConfig, + grandine_config::{BeaconNodeConfig, GrandineConfig, Mode, ValidatorClientConfig}, initialize_schema, misc::{MetricsConfig, StorageConfig}, predefined_network::PredefinedNetwork, - validators::normalize_definitions, + validators::{Validators, normalize_definitions}, }; #[cfg(any(feature = "preset-mainnet", test))] @@ -112,59 +125,166 @@ use types::preset::Minimal; #[cfg(all(unix, not(feature = "embed")))] use tokio::signal::unix::SignalKind; -#[expect(clippy::struct_excessive_bools)] -pub struct RuntimeConfig { - pub back_sync_enabled: bool, - pub detect_doppelgangers: bool, - pub max_events: usize, - pub reconstruction_delay: Duration, - pub slashing_protection_history_limit: u64, - pub track_liveness: bool, - pub validator_enabled: bool, -} - -#[expect(clippy::too_many_arguments)] -#[expect(clippy::too_many_lines)] -pub async fn run_after_genesis( - chain_config: Arc, +/// What one run builds the built-in beacon node from. +struct LocalNodeConfig { + beacon_node: BeaconNodeConfig, pubkey_cache: Arc, - runtime_config: RuntimeConfig, - store_config: StoreConfig, - validator_api_config: Option, - validator_config: Arc, - network_config: NetworkConfig, - anchor_checkpoint_provider: AnchorCheckpointProvider

, - state_load_strategy: StateLoadStrategy

, + client: Client, eth1_config: Arc, - storage_config: StorageConfig, - builder_config: Option, - signer: Arc, - slasher_config: Option, - http_api_config: Option, - metrics_config: MetricsConfig, - blacklisted_blocks: HashSet, + anchor_checkpoint_provider: AnchorCheckpointProvider

, report_validator_performance: bool, tracing_handle: Option, eth1_api_to_metrics_tx: Option>, eth1_api_to_metrics_rx: Option>, restart_tx: UnboundedSender, restart_rx: UnboundedReceiver, -) -> Result<()> { - let RuntimeConfig { - back_sync_enabled, - detect_doppelgangers, - max_events, - reconstruction_delay, - slashing_protection_history_limit, - track_liveness, - validator_enabled, - } = runtime_config; +} - let MetricsConfig { - metrics, - metrics_server_config, - metrics_service_config, - } = metrics_config; +/// What the validator and the built-in beacon node share. +struct SharedConfig { + chain_config: Arc, + validator_config: Arc, + genesis: Genesis, + signer: Arc, + validator_keys: Arc>, + proposer_configs: Arc, + metrics_config: MetricsConfig, + dedicated_executor_normal_priority: Arc, + dedicated_executor_low_priority: Arc, + tracing_handle: Option, +} + +/// What the clock drives: fork choice when the built-in node runs, otherwise the validator. +#[derive(Clone)] +enum ClockTarget { + ForkChoice(RealController

), + Validator(UnboundedSender>), +} + +impl ClockTarget

{ + fn on_tick(&self, tick: Tick) { + match self { + Self::ForkChoice(controller) => controller.on_tick(tick), + Self::Validator(validator_tx) => ValidatorMessage::Tick((), tick).send(validator_tx), + } + } + + fn stop(&self) { + match self { + Self::ForkChoice(controller) => { + controller.stop(); + info_with_peers!("saving current chain before exit…"); + } + Self::Validator(validator_tx) => ValidatorMessage::Stop.send(validator_tx), + } + } +} + +/// What runs alongside the validator and feeds it. +struct NodeServices { + validator_channels: ValidatorChannels, + clock_target: ClockTarget

, + metrics_registry: Option, + node_metrics: Option>, + metrics_channels: MetricsChannels, + validator_statistics: Option>, + run_node: BoxFuture<'static, Result<()>>, + network_globals: Option>, +} + +#[expect(clippy::too_many_lines)] +async fn build_local_node( + config: LocalNodeConfig

, + shared: &SharedConfig, +) -> Result<(Arc>, NodeServices

)> { + let LocalNodeConfig { + beacon_node: + BeaconNodeConfig { + checkpoint_sync_url, + force_checkpoint_sync, + back_sync_enabled, + network_config, + storage_config, + store_config, + reconstruction_delay, + slasher_config, + state_slot, + http_api_config, + max_events, + track_liveness, + blacklisted_blocks, + backfill_custody_groups, + builder_config, + .. + }, + pubkey_cache, + client, + eth1_config, + anchor_checkpoint_provider, + report_validator_performance, + tracing_handle, + eth1_api_to_metrics_tx, + eth1_api_to_metrics_rx, + restart_tx, + restart_rx, + } = config; + + let state_load_strategy = if force_checkpoint_sync { + StateLoadStrategy::Remote { + checkpoint_sync_url: checkpoint_sync_url.expect( + "the requires attribute for force_checkpoint_sync \ + ensures checkpoint_sync_url is present", + ), + } + } else { + StateLoadStrategy::Auto { + state_slot, + checkpoint_sync_url, + anchor_checkpoint_provider: anchor_checkpoint_provider.clone(), + } + }; + + Feature::DebugAttestationPacker.enable(); + Feature::LogBlockProcessingTime.enable(); + + let SharedConfig { + chain_config, + validator_config, + validator_keys, + proposer_configs, + dedicated_executor_normal_priority, + dedicated_executor_low_priority, + .. + } = shared; + + let metrics = &shared.metrics_config.metrics; + let genesis_time = shared.genesis.genesis_time; + let metrics_service_enabled = shared.metrics_config.metrics_service_config.is_some(); + + // Builder URLs come from API callers, so redirects must not be followed (SSRF). + let payload_builder_client = ClientBuilder::new() + .user_agent(APPLICATION_VERSION_WITH_COMMIT_AND_PLATFORM) + .redirect(Policy::none()) + .build()?; + + let payload_builder_api = Arc::new(PayloadBuilderApi::new( + chain_config.clone_arc(), + payload_builder_client, + builder_config + .as_ref() + .map(|builder_config| builder_config.builder_api_format) + .unwrap_or_default(), + metrics.clone(), + )); + + let builder_api = builder_config.map(|builder_config| { + Arc::new(BuilderApi::new( + builder_config, + pubkey_cache.clone_arc(), + client.clone(), + metrics.clone(), + )) + }); let StorageConfig { in_memory, @@ -176,14 +296,6 @@ pub async fn run_after_genesis( .. } = storage_config; - let signer_snapshot = signer.load(); - - if !signer_snapshot.is_empty() { - info_with_peers!("loaded {} validator key(s)", signer_snapshot.keys().len()); - } else if validator_enabled { - warn_with_peers!("failed to load validator keys"); - } - let (blob_fetcher_to_p2p_tx, blob_fetcher_to_p2p_rx) = mpsc::unbounded(); let (execution_service_to_blob_fetcher_tx, execution_service_to_blob_fetcher_rx) = mpsc::unbounded(); @@ -210,34 +322,27 @@ pub async fn run_after_genesis( let mut api_to_liveness_tx = None; let mut network_to_slasher_tx = None; - let mut pool_to_liveness_tx = None; let mut slasher_to_validator_rx = None; - let mut sync_to_metrics_tx = None; let mut validator_to_slasher_tx = None; let mut validator_to_liveness_tx = None; - let num_of_cpus = num_cpus::get(); - - let dedicated_executor_low_priority = Arc::new(DedicatedExecutor::new( - "de-low", - (num_of_cpus / 4).max(1), + let dedicated_executor_for_recovery = Arc::new(DedicatedExecutor::new( + "de-recover", + (num_cpus::get() / 4).max(1), Some(19), metrics.clone(), )); - let dedicated_executor_normal_priority = Arc::new(DedicatedExecutor::new( - "de-normal", - num_of_cpus, - None, + let dedicated_executor_for_reconstruction = Arc::new(DedicatedExecutor::new( + "de-reconstruct", + 1, + Some(19), metrics.clone(), )); - let dedicated_executor_for_reconstruction = - DedicatedExecutor::new("de-reconstruct", 1, Some(19), metrics.clone()); - let eth1_api = Arc::new(Eth1Api::new( chain_config.clone_arc(), - signer_snapshot.client().clone(), + client.clone(), eth1_config.eth1_auth.clone_arc(), eth1_config.eth1_rpc_urls.clone(), eth1_api_to_metrics_tx, @@ -246,7 +351,7 @@ pub async fn run_after_genesis( eth1_api::spawn_exchange_capabilities_and_versions_task( eth1_api.clone_arc(), - &dedicated_executor_low_priority, + dedicated_executor_low_priority, ); let execution_engine = Arc::new(Eth1ExecutionEngine::new( @@ -275,41 +380,12 @@ pub async fn run_after_genesis( zstd_compression_level, )); - let ((anchor_state, anchor_block, unfinalized_blocks), loaded_from_remote) = storage - .load(signer_snapshot.client(), state_load_strategy) - .await?; + let ((anchor_state, anchor_block, unfinalized_blocks), loaded_from_remote) = + storage.load(&client, state_load_strategy).await?; let is_anchor_genesis = anchor_block.message().slot() == GENESIS_SLOT; - - let mut slashing_protector = if in_memory { - SlashingProtector::in_memory(slashing_protection_history_limit)? - } else { - let genesis_validators_root = anchor_state.genesis_validators_root(); - - SlashingProtector::persistent( - directories - .store_directory - .clone() - .unwrap_or_default() - .as_path(), - directories - .validator_dir - .clone() - .unwrap_or_default() - .as_path(), - slashing_protection_history_limit, - genesis_validators_root, - )? - }; - - slashing_protector.register_validators(signer_snapshot.keys().copied())?; - - let slashing_protector = Arc::new(Mutex::new(slashing_protector)); - - let current_tick = Tick::current::

(&chain_config, anchor_state.genesis_time())?; - + let current_tick = Tick::current::

(chain_config, genesis_time)?; let event_channels = Arc::new(EventChannels::new(max_events)); - let sidecars_construction_started = Arc::new(SccHashMap::new()); let (controller, mutator_handle) = Controller::new( @@ -317,7 +393,7 @@ pub async fn run_after_genesis( pubkey_cache.clone_arc(), store_config, anchor_block, - anchor_state.clone_arc(), + anchor_state, current_tick, event_channels.clone_arc(), execution_engine.clone_arc(), @@ -332,7 +408,7 @@ pub async fn run_after_genesis( unfinalized_blocks, !back_sync_enabled || is_anchor_genesis, blacklisted_blocks, - sidecars_construction_started.clone_arc(), + sidecars_construction_started, None, )?; @@ -358,7 +434,7 @@ pub async fn run_after_genesis( dedicated_executor_normal_priority.clone_arc(), ); - let validator_keys = Arc::new(signer_snapshot.keys().copied().collect::>()); + let validator_keys = validator_keys.clone_arc(); let attestation_verifier = AttestationVerifier::new( controller.clone_arc(), @@ -367,103 +443,50 @@ pub async fn run_after_genesis( fork_choice_to_attestation_verifier_rx, ); - let metrics_service = metrics_service_config.map(|metrics_config| { - let (sync_tx, sync_to_metrics_rx) = mpsc::unbounded(); - - sync_to_metrics_tx = Some(sync_tx); + let (sync_to_metrics_tx, sync_to_metrics_rx) = + metrics_service_enabled.then(mpsc::unbounded).unzip(); + let node_metrics = metrics_service_enabled.then(|| { let eth1_connection_data = Eth1ConnectionData { sync_eth1_connected: cfg!(feature = "embed") || !eth1_config.eth1_rpc_urls.is_empty(), sync_eth1_fallback_connected: false, }; - let eth1_metrics = Eth1Metrics { - eth1_connection_data, - sync_eth1_fallback_configured: eth1_config.eth1_rpc_urls.len() > 1, - }; - - let channels = MetricsChannels { - eth1_api_to_metrics_rx, - sync_to_metrics_rx, - }; - - MetricsService::new( - metrics_config, - controller.clone_arc(), - eth1_metrics, - metrics - .clone() - .expect("metrics registry must be present for metrics service"), - slasher_config.is_some(), - validator_keys.clone_arc(), - channels, - ) + NodeMetrics { + controller: controller.clone_arc(), + eth1_metrics: Eth1Metrics { + eth1_connection_data, + sync_eth1_fallback_configured: eth1_config.eth1_rpc_urls.len() > 1, + }, + slasher_active: slasher_config.is_some(), + } }); - let (liveness_tracker, doppelganger_protection) = if track_liveness { + let metrics_channels = MetricsChannels { + eth1_api_to_metrics_rx, + sync_to_metrics_rx, + }; + + let liveness_tracker = track_liveness.then(|| { let (api_tx, api_to_liveness_rx) = mpsc::unbounded(); - let (pool_tx, pool_to_liveness_rx) = mpsc::unbounded(); let (validator_tx, validator_to_liveness_rx) = mpsc::unbounded(); - api_to_liveness_tx = Some(api_tx.clone()); - pool_to_liveness_tx = Some(pool_tx); + api_to_liveness_tx = Some(api_tx); validator_to_liveness_tx = Some(validator_tx); - let liveness_tracker = Some(LivenessTracker::new( + LivenessTracker::new( controller.clone_arc(), metrics.clone(), api_to_liveness_rx, - pool_to_liveness_rx, validator_to_liveness_rx, - )); - - let doppelganger_protection = - detect_doppelgangers.then(|| Arc::new(DoppelgangerProtection::new(api_tx))); - - (liveness_tracker, doppelganger_protection) - } else { - (None, None) - }; - - if let Some(doppelganger_protection) = doppelganger_protection.as_ref() { - signer.enable_doppelganger_protection(doppelganger_protection); - - signer.update_doppelganger_protection_pubkeys( - &controller.head_state().value, - controller.slot(), - ); - } + ) + }); let data_dumper = Arc::new(DataDumper::new(&controller.chain_config().config_name)?); let validator_statistics = report_validator_performance.then(|| Arc::new(ValidatorStatistics::new(metrics.clone()))); - // Builder URLs come from API callers, so redirects must not be followed (SSRF). - let payload_builder_client = ClientBuilder::new() - .user_agent(APPLICATION_VERSION_WITH_COMMIT_AND_PLATFORM) - .redirect(Policy::none()) - .build()?; - - let payload_builder_api = Arc::new(PayloadBuilderApi::new( - chain_config.clone_arc(), - payload_builder_client, - builder_config - .as_ref() - .map(|builder_config| builder_config.builder_api_format) - .unwrap_or_default(), - metrics.clone(), - )); - - let builder_api = builder_config.map(|builder_config| { - Arc::new(BuilderApi::new( - builder_config, - pubkey_cache, - signer_snapshot.client().clone(), - metrics.clone(), - )) - }); - let slasher = slasher_config .map(|slasher_config| -> Result<_> { let fork_version = chain_config.genesis_fork_version; @@ -558,45 +581,6 @@ pub async fn run_after_genesis( }) .transpose()?; - let graffiti = validator_config - .graffiti - .first() - .copied() - .unwrap_or_else(|| { - if validator_config.disable_blockprint_graffiti { - H256::default() - } else { - misc::parse_graffiti(APPLICATION_NAME_WITH_VERSION_AND_COMMIT).unwrap_or_default() - } - }); - - let keymanager = if in_memory { - Arc::new(KeyManager::new_in_memory( - signer.clone_arc(), - slashing_protector.clone_arc(), - anchor_state.genesis_validators_root(), - validator_config.suggested_fee_recipient, - validator_config.default_gas_limit, - graffiti, - validator_config.builder_settings.clone(), - validator_config.validator_definitions.clone_arc(), - )) - } else { - Arc::new(KeyManager::new_persistent( - signer.clone_arc(), - slashing_protector.clone_arc(), - anchor_state.genesis_validators_root(), - directories.validator_dir.clone().unwrap_or_default(), - directories.secrets_dir.clone().unwrap_or_default(), - validator_config.keystore_storage_password_file.as_deref(), - validator_config.suggested_fee_recipient, - validator_config.default_gas_limit, - graffiti, - validator_config.builder_settings.clone(), - validator_config.validator_definitions.clone_arc(), - )?) - }; - let attestation_agg_pool = AttestationAggPool::new( controller.clone_arc(), dedicated_executor_normal_priority.clone_arc(), @@ -608,7 +592,7 @@ pub async fn run_after_genesis( controller.clone_arc(), dedicated_executor_for_reconstruction, metrics.clone(), - dedicated_executor_low_priority.clone_arc(), + dedicated_executor_for_recovery, ); let payload_attestation_agg_pool = PayloadAttestationAggPool::new( @@ -620,7 +604,6 @@ pub async fn run_after_genesis( let sync_committee_agg_pool = SyncCommitteeAggPool::new( dedicated_executor_normal_priority.clone_arc(), controller.clone_arc(), - pool_to_liveness_tx, pool_to_p2p_tx.clone(), metrics.clone(), validator_statistics.clone(), @@ -645,7 +628,7 @@ pub async fn run_after_genesis( ); let block_producer = Arc::new(BlockProducer::new( - keymanager.proposer_configs().clone_arc(), + proposer_configs.clone_arc(), builder_api.clone(), payload_builder_api.clone_arc(), controller.clone_arc(), @@ -660,14 +643,17 @@ pub async fn run_after_genesis( )); let validator_channels = ValidatorChannels { - api_to_validator_rx, - fork_choice_rx: fork_choice_to_validator_rx, - p2p_tx: validator_to_p2p_tx, - p2p_to_validator_rx, - slasher_to_validator_rx, - subnet_service_tx: subnet_service_tx.clone(), - validator_to_liveness_tx, - validator_to_slasher_tx, + validator_rx: fork_choice_to_validator_rx, + local: Some(LocalValidatorChannels { + api_to_validator_rx, + p2p_tx: validator_to_p2p_tx, + p2p_to_validator_rx, + slasher_to_validator_rx, + subnet_service_tx: subnet_service_tx.clone(), + api_to_liveness_tx: api_to_liveness_tx.clone(), + validator_to_liveness_tx, + validator_to_slasher_tx, + }), }; let p2p_channels = Channels { @@ -686,8 +672,8 @@ pub async fn run_after_genesis( // Prometheus registry for deep gossipsub protocol metrics. // This has to be passed to both `libp2p` to collect metrics // and the metrics server to convert collected metrics to an HTTP response for Prometheus. - let gossip_registry = prometheus_client::registry::Registry::default(); - let mut registry = network_config.metrics_enabled.then_some(gossip_registry); + let gossip_registry = Registry::default(); + let mut metrics_registry = network_config.metrics_enabled.then_some(gossip_registry); let network_config = Arc::new(network_config); if let Some(metrics) = metrics.as_ref() @@ -705,36 +691,14 @@ pub async fn run_after_genesis( sync_committee_agg_pool.clone_arc(), bls_to_execution_change_pool.clone_arc(), metrics.clone(), - registry.as_mut(), + metrics_registry.as_mut(), data_dumper.clone_arc(), - validator_config.backfill_custody_groups, + backfill_custody_groups, validator_config.custody_mode, storage_mode, ) .await?; - let validator = Validator::new( - validator_config.clone_arc(), - block_producer.clone_arc(), - controller.clone_arc(), - attestation_agg_pool.clone_arc(), - builder_api, - doppelganger_protection, - event_channels.clone_arc(), - keymanager.proposer_configs().clone_arc(), - signer.clone_arc(), - slashing_protector, - payload_attestation_agg_pool.clone_arc(), - sync_committee_agg_pool.clone_arc(), - metrics.clone(), - validator_statistics.clone(), - validator_channels, - network_config.network_dir.as_deref(), - dedicated_executor_normal_priority.clone_arc(), - dedicated_executor_low_priority.clone_arc(), - Some(network.network_globals().clone_arc()), - ); - let block_sync_service_channels = BlockSyncServiceChannels { fork_choice_to_sync_rx, p2p_to_sync_rx, @@ -755,11 +719,11 @@ pub async fn run_after_genesis( anchor_checkpoint_provider.clone(), controller.clone_arc(), metrics.clone(), - validator_statistics, + validator_statistics.clone(), block_sync_service_channels, back_sync_enabled, loaded_from_remote, - storage_config.storage_mode, + storage_mode, network_config.target_peers, received_blob_sidecars, received_data_column_sidecars, @@ -787,137 +751,390 @@ pub async fn run_after_genesis( sync_to_api_rx, }; - let run_http_api = match http_api_config { - Some(http_api_config) => { - let http_api = HttpApi { - block_producer, - payload_builder_api, - controller: controller.clone_arc(), - anchor_checkpoint_provider, - eth1_api, - event_channels, - validator_keys, - validator_config, - network_config, - http_api_config, - attestation_agg_pool, - sync_committee_agg_pool, - bls_to_execution_change_pool, - payload_attestation_agg_pool, - channels: http_api_channels, - metrics: metrics.clone(), - tracing_handle, - dedicated_executor: dedicated_executor_low_priority, - }; - - Either::Left(http_api.run()) - } - None => Either::Right(core::future::pending()), - }; + let http_api = http_api_config.map(|http_api_config| HttpApi { + block_producer: block_producer.clone_arc(), + payload_builder_api, + controller: controller.clone_arc(), + anchor_checkpoint_provider, + eth1_api, + event_channels: event_channels.clone_arc(), + validator_keys, + validator_config: validator_config.clone_arc(), + network_config, + http_api_config, + attestation_agg_pool: attestation_agg_pool.clone_arc(), + sync_committee_agg_pool: sync_committee_agg_pool.clone_arc(), + bls_to_execution_change_pool, + payload_attestation_agg_pool: payload_attestation_agg_pool.clone_arc(), + channels: http_api_channels, + metrics: metrics.clone(), + tracing_handle, + dedicated_executor: dedicated_executor_low_priority.clone_arc(), + }); - let join_mutator = async { tokio::task::spawn_blocking(|| mutator_handle.join()).await? }; + let network_globals = network.network_globals().clone_arc(); - let (stop_clock_tx, stop_clock_rx) = oneshot::channel(); - let run_clock = run_clock(controller.clone_arc(), stop_clock_rx); + let run_node = async move { + let join_mutator = async { tokio::task::spawn_blocking(|| mutator_handle.join()).await? }; - let run_slasher = match slasher { - Some(slasher) => Either::Left(slasher.run()), - None => Either::Right(core::future::pending()), - }; + let run_http_api = match http_api { + Some(http_api) => Either::Left(http_api.run()), + None => Either::Right(core::future::pending()), + }; - let run_metrics_server = match metrics_server_config { - Some(config) => Either::Left(run_metrics_server( - config, - registry.take(), - metrics - .clone() - .expect("metrics registry must be present for metrics server"), - )), - None => Either::Right(core::future::pending()), - }; + let run_slasher = match slasher { + Some(slasher) => Either::Left(slasher.run()), + None => Either::Right(core::future::pending()), + }; - let run_metrics_service = match metrics_service { - Some(service) => Either::Left(service.run()), - None => Either::Right(core::future::pending()), - }; + let run_liveness_tracker = match liveness_tracker { + Some(service) => Either::Left(service.run()), + None => Either::Right(core::future::pending()), + }; - let run_liveness_tracker = match liveness_tracker { - Some(service) => Either::Left(service.run()), - None => Either::Right(core::future::pending()), + select! { + result = join_mutator => result, + result = spawn_fallible(execution_service.run()) => result, + result = spawn_fallible(execution_blob_fetcher.run()) => result, + result = spawn_fallible(attestation_verifier.run()) => result, + result = spawn_fallible(block_sync_service.run()) => result, + result = spawn_fallible(network.run()) => result, + result = spawn_fallible(run_http_api) => result, + result = spawn_fallible(run_slasher) => result.map(from_never), + result = spawn_fallible(bls_to_execution_change_pool_service.run()) => result, + result = spawn_fallible(pool_manager.run()) => result, + result = spawn_fallible(run_liveness_tracker) => result, + result = spawn_fallible(subnet_service.run()) => result, + result = wait_for_restart(restart_rx) => result, + } + } + .boxed(); + + let services = NodeServices { + validator_channels, + clock_target: ClockTarget::ForkChoice(controller.clone_arc()), + metrics_registry, + node_metrics, + metrics_channels, + validator_statistics, + run_node, + network_globals: Some(network_globals), + }; + + let chain = Arc::new(LocalChain { + controller, + block_producer, + attestation_agg_pool, + sync_committee_agg_pool, + payload_attestation_agg_pool, + event_channels, + builder_api, + }); + + Ok((chain, services)) +} + +fn shared_config( + config: &GrandineConfig, + validator_config: Arc, + signer: Arc, + genesis: Genesis, + tracing_handle: Option, +) -> SharedConfig { + let metrics_config = config.metrics_config.clone(); + let metrics = &metrics_config.metrics; + + let num_of_cpus = num_cpus::get(); + + let dedicated_executor_low_priority = Arc::new(DedicatedExecutor::new( + "de-low", + (num_of_cpus / 4).max(1), + Some(19), + metrics.clone(), + )); + + let dedicated_executor_normal_priority = Arc::new(DedicatedExecutor::new( + "de-normal", + num_of_cpus, + None, + metrics.clone(), + )); + + let graffiti = validator_config + .graffiti + .first() + .copied() + .unwrap_or_else(|| { + if validator_config.disable_blockprint_graffiti { + H256::default() + } else { + misc::parse_graffiti(APPLICATION_NAME_WITH_VERSION_AND_COMMIT).unwrap_or_default() + } + }); + + // Kept without a keymanager too: the built-in node produces blocks for remote validator + // clients. + let proposer_configs = Arc::new(ProposerConfigs::new( + validator_config.suggested_fee_recipient, + validator_config.default_gas_limit, + graffiti, + validator_config.builder_settings.clone(), + validator_config.validator_definitions.clone_arc(), + )); + + let validator_keys = Arc::new(signer.load().keys().copied().collect::>()); + + SharedConfig { + chain_config: config.chain_config.clone_arc(), + validator_config, + genesis, + signer, + validator_keys, + proposer_configs, + metrics_config, + dedicated_executor_normal_priority, + dedicated_executor_low_priority, + tracing_handle, + } +} + +#[expect(clippy::too_many_lines)] +async fn run_services( + shared: SharedConfig, + config: &GrandineConfig, + chain: Chain, + services: NodeServices

, +) -> Result<()> { + let validator_client = config.validator_client_config(); + + let SharedConfig { + chain_config, + validator_config, + genesis, + signer, + validator_keys, + proposer_configs, + metrics_config, + dedicated_executor_normal_priority, + dedicated_executor_low_priority, + tracing_handle, + } = shared; + + let MetricsConfig { + metrics, + metrics_server_config, + metrics_service_config, + } = metrics_config; + + let in_memory = config.in_memory; + let directories = config.directories.clone_arc(); + let genesis_validators_root = genesis.genesis_validators_root; + + let validator_api = + validator_client.and_then(|validator_client| validator_client.validator_api_config.clone()); + + // The validator service also feeds the node's pools and liveness tracking from gossip and + // the API, so without a validator client it still runs, with no keys. + let (expects_keys, detect_doppelgangers, slashing_protection_history_limit) = + match validator_client { + Some(validator_client) => { + let expects_keys = + validator_client.expects_keys(&validator_config.validator_definitions.read()); + + ( + expects_keys, + validator_client.detect_doppelgangers, + validator_client.slashing_protection_history_limit, + ) + } + None => (false, false, DEFAULT_SLASHING_PROTECTION_HISTORY_LIMIT), + }; + + if !validator_keys.is_empty() { + info_with_peers!("loaded {} validator key(s)", validator_keys.len()); + } else if expects_keys { + warn_with_peers!("failed to load validator keys"); + } + + let mut slashing_protector = if in_memory || validator_client.is_none() { + SlashingProtector::in_memory(slashing_protection_history_limit)? + } else { + SlashingProtector::persistent( + directories + .store_directory + .clone() + .unwrap_or_default() + .as_path(), + directories + .validator_dir + .clone() + .unwrap_or_default() + .as_path(), + slashing_protection_history_limit, + genesis_validators_root, + )? }; - let run_validator_api = match validator_api_config { - Some(validator_api_config) => Either::Left(run_validator_api( + slashing_protector.register_validators(validator_keys.iter().copied())?; + + let slashing_protector = Arc::new(Mutex::new(slashing_protector)); + + let keymanager = validator_client + .map(|validator_client| -> Result<_> { + if in_memory { + Ok(Arc::new(KeyManager::new_in_memory( + signer.clone_arc(), + slashing_protector.clone_arc(), + genesis_validators_root, + proposer_configs.clone_arc(), + validator_config.validator_definitions.clone_arc(), + ))) + } else { + Ok(Arc::new(KeyManager::new_persistent( + signer.clone_arc(), + slashing_protector.clone_arc(), + genesis_validators_root, + directories.validator_dir.clone().unwrap_or_default(), + directories.secrets_dir.clone().unwrap_or_default(), + validator_client.keystore_storage_password_file.as_deref(), + proposer_configs.clone_arc(), + validator_config.validator_definitions.clone_arc(), + )?)) + } + }) + .transpose()?; + + let doppelganger_protection = + detect_doppelgangers.then(|| Arc::new(DoppelgangerProtection::new())); + + if let Some(doppelganger_protection) = doppelganger_protection.as_ref() { + signer.enable_doppelganger_protection(doppelganger_protection); + + let current_slot = Tick::current::

(&chain_config, genesis.genesis_time)?.slot; + + signer.update_doppelganger_protection_pubkeys(current_slot); + } + + let own_validator_indices = Arc::new(OwnValidatorIndices::new(signer.clone_arc())); + + let chain_source = Arc::new(ChainSource { + chain_config: chain_config.clone_arc(), + genesis_time: genesis.genesis_time, + genesis_validators_root, + own_validator_indices, + chain, + }); + + let NodeServices { + validator_channels, + clock_target, + metrics_registry, + node_metrics, + metrics_channels, + validator_statistics, + run_node, + network_globals, + } = services; + + let validator = Validator::new( + validator_config, + chain_source.clone_arc(), + doppelganger_protection, + proposer_configs, + signer.clone_arc(), + slashing_protector, + metrics.clone(), + validator_statistics, + validator_channels, + dedicated_executor_normal_priority, + dedicated_executor_low_priority, + network_globals, + ); + + let run_validator_api = match keymanager.zip(validator_api) { + Some((keymanager, validator_api_config)) => Either::Left(run_validator_api( validator_api_config, - controller.clone_arc(), + chain_source, directories.clone_arc(), keymanager, signer, - metrics, + metrics.clone(), + tracing_handle, + )), + None => Either::Right(core::future::pending()), + }; + + let (stop_clock_tx, stop_clock_rx) = oneshot::channel(); + + let run_clock = run_clock::

( + chain_config, + genesis.genesis_time.saturating_mul(1000), + clock_target.clone(), + stop_clock_rx, + ); + + let run_metrics_server = match metrics_server_config { + Some(config) => Either::Left(run_metrics_server( + config, + metrics_registry, + metrics + .clone() + .context("the metrics server needs --metrics")?, )), None => Either::Right(core::future::pending()), }; + let run_metrics_service = match metrics_service_config { + Some(config) => Either::Left( + MetricsService::new( + config, + node_metrics, + metrics, + validator_keys, + metrics_channels, + ) + .run(), + ), + None => Either::Right(core::future::pending()), + }; + select! { - result = join_mutator => result, - result = spawn_fallible(execution_service.run()) => result, - result = spawn_fallible(execution_blob_fetcher.run()) => result, result = spawn_fallible(validator.run()) => result, - result = spawn_fallible(attestation_verifier.run()) => result, - result = spawn_fallible(block_sync_service.run()) => result, - result = spawn_fallible(network.run()) => result, - result = spawn_fallible(run_http_api) => result, result = spawn_fallible(run_clock) => result, - result = spawn_fallible(run_slasher) => result.map(from_never), - result = spawn_fallible(bls_to_execution_change_pool_service.run()) => result, - result = spawn_fallible(pool_manager.run()) => result, + result = spawn_fallible(run_validator_api) => result, result = spawn_fallible(run_metrics_server) => result, result = spawn_fallible(run_metrics_service) => result, - result = spawn_fallible(run_liveness_tracker) => result, - result = spawn_fallible(run_validator_api) => result, - result = spawn_fallible(subnet_service.run()) => result, - result = wait_for_signal_or_restart(restart_rx) => result, + result = spawn_fallible(run_node) => result, + result = wait_for_signal() => result, }?; if stop_clock_tx.send(()).is_err() { warn_with_peers!("failed to send the message to stop the clock"); } - controller.stop(); - - info_with_peers!("saving current chain before exit…"); + clock_target.stop(); Ok(()) } async fn run_clock( - controller: RealController

, + chain_config: Arc, + genesis_time_in_ms: u64, + clock_target: ClockTarget

, mut stop_clock_rx: oneshot::Receiver<()>, ) -> Result<()> { - let mut ticks = - clock::ticks::

(controller.chain_config(), controller.genesis_time_in_ms())?.fuse(); + let mut ticks = clock::ticks::

(&chain_config, genesis_time_in_ms)?.fuse(); loop { select! { - tick = ticks.select_next_some() => { - controller.on_tick(tick?); - } - _ = &mut stop_clock_rx => { - break; - } + tick = ticks.select_next_some() => clock_target.on_tick(tick?), + _ = &mut stop_clock_rx => break, } } Ok(()) } -async fn wait_for_signal_or_restart(error_rx: UnboundedReceiver) -> Result<()> { - select! { - result = wait_for_restart(error_rx) => result, - result = wait_for_signal() => result, - } -} - async fn wait_for_restart(mut rx: UnboundedReceiver) -> Result<()> { if let Some(message) = rx.next().await { match message { @@ -925,7 +1142,8 @@ async fn wait_for_restart(mut rx: UnboundedReceiver) -> Result<( } } - Ok(()) + // In-memory databases hold no sender, so the channel closes without a restart being asked. + core::future::pending().await } #[cfg(feature = "embed")] @@ -973,43 +1191,15 @@ const fn from_never(never: Never) -> T { match never {} } -#[expect( - clippy::struct_excessive_bools, - reason = "False positive. The `bool`s are independent." -)] +/// [`GrandineConfig`] with the client, the validator's settings and the signer built. #[derive(Clone)] struct Context { - predefined_network: Option, - chain_config: Arc, - store_config: StoreConfig, - deposit_contract_starting_block: Option, - genesis_state_file: Option, - genesis_state_download_url: Option, - validator_api_config: Option, + config: GrandineConfig, + client: Client, + eth1_auth: Arc, validator_config: Arc, - checkpoint_sync_url: Option, - force_checkpoint_sync: bool, - back_sync_enabled: bool, - eth1_rpc_urls: Vec, - network_config: NetworkConfig, - storage_config: StorageConfig, - command: Option, - builder_config: Option, signer: Arc, - slasher_config: Option, - state_slot: Option, - eth1_auth: Arc, - http_api_config: Option, - max_events: usize, - metrics_config: MetricsConfig, - track_liveness: bool, tracing_handle: Option, - detect_doppelgangers: bool, - slashing_protection_history_limit: u64, - validator_enabled: bool, - blacklisted_blocks: HashSet, - reconstruction_delay: Duration, - report_validator_performance: bool, } impl Context { @@ -1024,14 +1214,17 @@ impl Context { let result = std::panic::catch_unwind(AssertUnwindSafe(|| { let mut context = self.clone(); - context.storage_config = context - .storage_config - .with_increased_db_sizes(db_size_modifier); + if let Some(node) = context.config.beacon_node_config_mut() { + node.storage_config = node + .storage_config + .clone() + .with_increased_db_sizes(db_size_modifier); - if db_size_modifier > 1 { - context.storage_config.print_db_sizes(); - // don't reset database on restart - context.storage_config.reset_databases = false; + if db_size_modifier > 1 { + node.storage_config.print_db_sizes(); + // don't reset database on restart + node.storage_config.reset_databases = false; + } } let run = context.run::

(); @@ -1056,7 +1249,8 @@ impl Context { if matches!( error.downcast_ref::(), Some(&doppelganger_protection::Error::DoppelgangersDetected { .. }) - ) { + ) || error.downcast_ref::().is_some() + { break Err(error); } } @@ -1065,43 +1259,13 @@ impl Context { } } - #[expect(clippy::too_many_lines)] async fn run(self) -> Result<()> { - let Self { - predefined_network, - chain_config, - store_config, - mut deposit_contract_starting_block, - genesis_state_file, - genesis_state_download_url, - validator_api_config, - validator_config, - checkpoint_sync_url, - force_checkpoint_sync, - back_sync_enabled, - eth1_rpc_urls, - network_config, - storage_config, - command, - builder_config, - signer, - slasher_config, - state_slot, - eth1_auth, - http_api_config, - max_events, - metrics_config, - track_liveness, - tracing_handle, - detect_doppelgangers, - slashing_protection_history_limit, - validator_enabled, - blacklisted_blocks, - reconstruction_delay, - report_validator_performance, - } = self; + // The databases are the built-in node's; without it there is nothing to remove. + if let Some(beacon_node_config) = self.config.beacon_node_config() + && beacon_node_config.storage_config.reset_databases + { + let storage_config = &beacon_node_config.storage_config; - if storage_config.reset_databases { match remove_database_dir(storage_config.eth1_database_path().as_path()) { Ok(()) => info!("successfully removed eth1 database"), Err(error) => warn!("failed to remove eth1 database: {error:?}"), @@ -1124,262 +1288,318 @@ impl Context { } // Load keys early so we can validate `eth1_rpc_urls`. - signer.load_keys_from_web3signer().await; - - let signer_snapshot = signer.load(); + self.signer.load_keys_from_web3signer().await; - if cfg!(not(feature = "embed")) && eth1_rpc_urls.is_empty() { + // An execution layer is for the built-in node's blocks; a validator without it has none. + if let Some(node) = self.config.beacon_node_config() + && cfg!(not(feature = "embed")) + && node.eth1_rpc_urls.is_empty() + { ensure!( - signer_snapshot.no_keys(), + self.signer.load().no_keys(), Error::MissingEth1RpcUrlsWithValidators, ); } - let default_deposit_tree = predefined_network.map(PredefinedNetwork::genesis_deposit_tree); + let Self { + config, + client, + eth1_auth, + validator_config, + signer, + tracing_handle, + } = self; - if let Some(deposit_tree) = default_deposit_tree { - deposit_contract_starting_block - .get_or_insert_with(|| deposit_tree.last_added_block_number.saturating_add(1)); + match &config.mode { + Mode::Local { + beacon_node_config, + command, + .. + } => { + run_beacon_node::

( + &config, + client, + eth1_auth, + validator_config, + signer, + beacon_node_config.clone(), + command.clone(), + tracing_handle, + ) + .await + } + Mode::Remote { + beacon_node_urls, + publish_to_every_node, + use_builder, + .. + } => { + run_remote_validator::

( + &config, + client, + validator_config, + signer, + beacon_node_urls.clone(), + publish_to_every_node.clone(), + *use_builder, + tracing_handle, + ) + .await + } } + } +} - let eth1_config = Arc::new(Eth1Config { - eth1_auth, - eth1_rpc_urls, - deposit_contract_starting_block, - default_deposit_tree, - }); +#[expect(clippy::too_many_arguments)] +async fn run_beacon_node( + config: &GrandineConfig, + client: Client, + eth1_auth: Arc, + validator_config: Arc, + signer: Arc, + beacon_node: BeaconNodeConfig, + command: Option, + tracing_handle: Option, +) -> Result<()> { + let validator_client = config.validator_client_config(); - let (eth1_api_to_metrics_tx, eth1_api_to_metrics_rx) = metrics_config - .metrics_service_config - .as_ref() - .and_then(|metrics_config| metrics_config.remote_metrics_url.as_ref()) - .is_some() - .then(mpsc::unbounded) - .unzip(); - - let (restart_tx, restart_rx) = mpsc::unbounded(); - - let pubkey_cache_database = if storage_config.in_memory { - Database::in_memory() - } else { - storage_config.pubkey_cache_database( - None, - DatabaseMode::ReadWrite, - Some(restart_tx.clone()), - )? - }; + let (eth1_api_to_metrics_tx, eth1_api_to_metrics_rx) = config + .metrics_config + .metrics_service_config + .as_ref() + .and_then(|metrics_config| metrics_config.remote_metrics_url.as_ref()) + .is_some() + .then(mpsc::unbounded) + .unzip(); - let pubkey_cache = Arc::new(PubkeyCache::load(pubkey_cache_database)); + let (restart_tx, restart_rx) = mpsc::unbounded(); - let anchor_checkpoint_provider = genesis_checkpoint_provider::

( - &chain_config, - ð1_config, - &pubkey_cache, - &storage_config, - genesis_state_file, - predefined_network, - signer_snapshot.client(), - genesis_state_download_url, - &metrics_config, - eth1_api_to_metrics_tx.as_ref(), - &restart_tx, - ) - .await?; + let pubkey_cache_database = if config.in_memory { + Database::in_memory() + } else { + beacon_node.storage_config.pubkey_cache_database( + None, + DatabaseMode::ReadWrite, + Some(restart_tx.clone()), + )? + }; - if let Some(command) = command { - return handle_command( - chain_config, - &pubkey_cache, - &storage_config, - command, - &anchor_checkpoint_provider, - slashing_protection_history_limit, - ) - .inspect_err(|error| error!("error occurred while executing command: {error:?}")); - } + let pubkey_cache = Arc::new(PubkeyCache::load(pubkey_cache_database)); - let state_load_strategy = if force_checkpoint_sync { - StateLoadStrategy::Remote { - checkpoint_sync_url: checkpoint_sync_url.expect( - "the requires attribute for force_checkpoint_sync \ - ensures checkpoint_sync_url is present", - ), - } - } else { - StateLoadStrategy::Auto { - state_slot, - checkpoint_sync_url, - anchor_checkpoint_provider: anchor_checkpoint_provider.clone(), - } - }; + let eth1_config = Arc::new(Eth1Config { + eth1_auth, + eth1_rpc_urls: beacon_node.eth1_rpc_urls.clone(), + deposit_contract_starting_block: beacon_node.deposit_contract_starting_block, + default_deposit_tree: config + .predefined_network + .map(PredefinedNetwork::genesis_deposit_tree), + }); - Feature::DebugAttestationPacker.enable(); - Feature::LogBlockProcessingTime.enable(); + let slashing_protection_history_limit = validator_client.map_or( + DEFAULT_SLASHING_PROTECTION_HISTORY_LIMIT, + |validator_client| validator_client.slashing_protection_history_limit, + ); - run_after_genesis( - chain_config, - pubkey_cache, - RuntimeConfig { - back_sync_enabled, - detect_doppelgangers, - max_events, - reconstruction_delay, - slashing_protection_history_limit, - track_liveness, - validator_enabled, - }, - store_config, - validator_api_config, - validator_config, - network_config, - anchor_checkpoint_provider, - state_load_strategy, - eth1_config, - storage_config, - builder_config, - signer, - slasher_config, - http_api_config, - metrics_config, - blacklisted_blocks, - report_validator_performance, - tracing_handle, - eth1_api_to_metrics_tx, - eth1_api_to_metrics_rx, - restart_tx, - restart_rx, + // The root of a predefined network is known without its state. + let known_genesis_validators_root = config + .genesis_state_file + .is_none() + .then_some(config.predefined_network) + .flatten() + .map(PredefinedNetwork::genesis_validators_root); + + // Loaded on demand, as downloading or deriving the state is the slowest part of starting. + let load_anchor_checkpoint_provider = { + let client = &client; + let pubkey_cache = &pubkey_cache; + let storage_config = &beacon_node.storage_config; + let eth1_config = ð1_config; + let eth1_api_to_metrics_tx = eth1_api_to_metrics_tx.as_ref(); + let restart_tx = &restart_tx; + + move || async move { + match genesis_checkpoint_provider::

(config, client).await? { + Some(provider) => Ok(provider), + None => { + wait_for_genesis_by_deposits::

( + config, + client, + pubkey_cache, + storage_config, + eth1_config, + eth1_api_to_metrics_tx, + restart_tx, + ) + .await + } + } + } + }; + + if let Some(command) = command { + return handle_command( + config.chain_config.clone_arc(), + &pubkey_cache, + &beacon_node.storage_config, + command, + known_genesis_validators_root, + load_anchor_checkpoint_provider, + slashing_protection_history_limit, ) .await + .inspect_err(|error| error!("error occurred while executing command: {error:?}")); } -} -#[derive(Debug, Error)] -enum Error { - #[error("{preset_name} preset is not included in this executable")] - PresetNotIncluded { preset_name: PresetName }, - #[error("--eth1-rpc-urls must be specified when validators are present")] - MissingEth1RpcUrlsWithValidators, - #[error( - "{service} port ({port}) is unavailable; \ - make sure no other instance of the application is running \ - or specify a different port with {option} (error: {error:?})" - )] - PortInUse { - port: u16, - service: &'static str, - option: &'static str, - error: anyhow::Error, - }, + let anchor_checkpoint_provider = load_anchor_checkpoint_provider().await?; + let genesis = anchor_genesis(&config.chain_config, &anchor_checkpoint_provider); + + let shared = shared_config(config, validator_config, signer, genesis, tracing_handle); + + let (chain, services) = build_local_node( + LocalNodeConfig { + beacon_node, + pubkey_cache, + client, + eth1_config, + anchor_checkpoint_provider, + report_validator_performance: validator_client + .is_some_and(|validator_client| validator_client.report_validator_performance), + tracing_handle: shared.tracing_handle.clone(), + eth1_api_to_metrics_tx, + eth1_api_to_metrics_rx, + restart_tx, + restart_rx, + }, + &shared, + ) + .await?; + + run_services(shared, config, Chain::Local(chain), services).await } -#[expect(clippy::too_many_lines)] -pub fn run(parsed_args: GrandineArgs) -> Result<()> { - let data_dir = parsed_args.data_dir(); +#[expect(clippy::too_many_arguments)] +async fn run_remote_validator( + config: &GrandineConfig, + client: Client, + validator_config: Arc, + signer: Arc, + beacon_node_urls: Vec, + publish_to_every_node: Vec, + use_builder: bool, + tracing_handle: Option, +) -> Result<()> { + const GENESIS_RETRY_DELAY: Duration = Duration::from_secs(2); + + // The chain is identified by its genesis validators root, which a predefined network knows + // without the state; the nodes' word is taken for the rest. + let expected_genesis_validators_root = if config.genesis_state_file.is_some() { + genesis_checkpoint_provider::

(config, &client) + .await? + .map(|provider| provider.checkpoint().value.state.genesis_validators_root()) + } else { + config + .predefined_network + .map(PredefinedNetwork::genesis_validators_root) + }; - let log_handle = binary_utils::initialize_tracing_logger( - module_path!(), - data_dir.as_deref(), - parsed_args.telemetry_config(), - cfg!(feature = "logger-always-write-style"), - )?; + let remote_beacon_nodes = Arc::new(RemoteBeaconNodes::new( + beacon_node_urls + .into_iter() + .map(|url| { + Arc::new(RemoteBeaconNode::new( + config.chain_config.clone_arc(), + client.clone(), + url, + validator_config.max_empty_slots, + )) + }) + .collect(), + publish_to_every_node, + use_builder, + )); - binary_utils::initialize_rayon()?; + let genesis = loop { + match remote_beacon_nodes.agreed_genesis().await? { + Some(genesis) => break genesis, + None => { + warn_with_peers!( + "no beacon node given with --beacon-node-urls reported genesis; \ + retrying in {} s", + GENESIS_RETRY_DELAY.as_secs(), + ); + + sleep(GENESIS_RETRY_DELAY).await; + } + } + }; - debug_with_peers!("grandine args - {parsed_args:?}"); + if let Some(expected) = expected_genesis_validators_root { + ensure!( + genesis.genesis_validators_root == expected, + ValidatorStartupError::UnexpectedChain { + expected, + actual: genesis.genesis_validators_root, + }, + ); + } - let config = parsed_args - .try_into_config() - .map_err(GrandineArgs::clap_error)?; + info_with_peers!( + "genesis agreed with the beacon nodes given with --beacon-node-urls \ + (genesis validators root: {:?}, genesis time: {})", + genesis.genesis_validators_root, + genesis.genesis_time, + ); - info_with_peers!("starting beacon node"); + // Nodes unreachable now are held to the agreed root on their first poll. + remote_beacon_nodes.seed_genesis_validators_root(genesis.genesis_validators_root); - config.report(); + let shared = shared_config(config, validator_config, signer, genesis, tracing_handle); - let GrandineConfig { - predefined_network, - chain_config, - deposit_contract_starting_block, - genesis_state_file, - genesis_state_download_url, - checkpoint_sync_url, - force_checkpoint_sync, - back_sync_enabled, - eth1_rpc_urls, - data_dir, - validators, - keystore_storage_password_file, - disable_blockprint_graffiti, - graffiti, - max_empty_slots, - suggested_fee_recipient, - default_builder_boost_factor, - default_builder_min_bid, - default_builder_max_execution_payment, - allow_trusted_payments, - payload_builder_urls, - default_gas_limit, - network_config, - storage_config, - reconstruction_delay, - request_timeout, - max_epochs_to_retain_states_in_cache, - state_cache_lock_timeout, - unfinalized_states_in_memory, - command, - slashing_enabled, - slashing_history_limit, - state_slot, - auth_options, - builder_config, - builder_circuit_breaker, - mut web3signer_config, - http_api_config, - max_events, - metrics_config, - track_liveness, - detect_doppelgangers, - use_validator_key_cache, - slashing_protection_history_limit, - in_memory, - validator_api_config, - kzg_backend, - blacklisted_blocks, - report_validator_performance, - backfill_custody_groups, - sync_without_reconstruction, - custody_mode, - disable_wait_for_late_blocks, - .. - } = config; + // Without the built-in node's fork choice the validator receives ticks straight from the clock. + let (runtime_tx, runtime_rx) = mpsc::unbounded(); - PEER_LOG_METRICS.set_target_peer_count(network_config.target_peers); + let services = NodeServices { + validator_channels: ValidatorChannels { + validator_rx: runtime_rx, + local: None, + }, + clock_target: ClockTarget::Validator(runtime_tx), + metrics_registry: None, + node_metrics: None, + metrics_channels: MetricsChannels { + eth1_api_to_metrics_rx: None, + sync_to_metrics_rx: None, + }, + validator_statistics: None, + run_node: core::future::pending().boxed(), + network_globals: None, + }; - let MetricsConfig { - metrics, - metrics_server_config, - .. - } = &metrics_config; + run_services::

(shared, config, Chain::Remote(remote_beacon_nodes), services).await +} - // Don't check ports for command runs. None of the commands need a network connection. - // Check ports before `Context::run_with_restart` to avoid logging an error repeatedly. - // The ports could in theory be freed or taken between restarts, but it's not likely. - if command.is_none() { - ensure_ports_not_in_use( - http_api_config.as_ref().map(|config| config.address), - &network_config, - metrics_server_config.as_ref(), - validator_api_config.as_ref(), - ) - .map_err(GrandineArgs::clap_error)?; - } +fn anchor_genesis( + chain_config: &ChainConfig, + provider: &AnchorCheckpointProvider

, +) -> Genesis { + let genesis_state = provider.checkpoint().value.state; - if !in_memory { - initialize_schema(data_dir)?; + Genesis { + genesis_time: genesis_state.genesis_time(), + genesis_fork_version: chain_config.genesis_fork_version, + genesis_validators_root: genesis_state.genesis_validators_root(), } +} - // Build the validator definitions: discover keystores, migrate the legacy database, and persist. +fn load_validator_definitions( + validators: Option<&Validators>, + keystore_storage_password_file: Option<&Path>, + directories: &Directories, + in_memory: bool, + allow_trusted_payments: bool, +) -> Result<(Arc, ValidatorKeyCache)> { let validators_config_path = (!in_memory) - .then(|| storage_config.directories.validator_dir.clone()) + .then(|| directories.validator_dir.clone()) .flatten() .map(|validator_dir| ValidatorDefinitions::file_path(&validator_dir)); @@ -1390,23 +1610,19 @@ pub fn run(parsed_args: GrandineArgs) -> Result<()> { validator_definitions.ensure_trusted_payments_allowed(allow_trusted_payments)?; - if let Some(validators) = validators.as_ref() { + if let Some(validators) = validators { validators .discover(&mut validator_definitions) .context("unable to discover validator keystores")?; } - let keystore_storage = match &keystore_storage_password_file { + let keystore_storage = match keystore_storage_password_file { Some(password_path) => { let password = keymanager::load_key_storage_password(password_path)?; keymanager::load_key_storage( &password, - storage_config - .directories - .validator_dir - .clone() - .unwrap_or_default(), + directories.validator_dir.clone().unwrap_or_default(), )? } None => ValidatorKeyCache::default(), @@ -1416,7 +1632,7 @@ pub fn run(parsed_args: GrandineArgs) -> Result<()> { .add_storage_keystores(keystore_storage.keypairs().map(|(pubkey, _)| pubkey)); let legacy_database_dir = (!in_memory) - .then(|| storage_config.directories.validator_dir.as_deref()) + .then_some(directories.validator_dir.as_deref()) .flatten(); let legacy_migration = legacy_database_dir @@ -1434,7 +1650,8 @@ pub fn run(parsed_args: GrandineArgs) -> Result<()> { validator_definitions.save(path)?; } - // Prune only after the save, so an earlier failure retries the migration instead of losing settings. + // Prune only after the save, so an earlier failure retries the migration instead of + // losing settings. if let Some(LegacyMigration { migrated, skipped }) = legacy_migration && let Some(validator_dir) = legacy_database_dir { @@ -1444,17 +1661,14 @@ pub fn run(parsed_args: GrandineArgs) -> Result<()> { } Ok(false) => warn!( "keeping the legacy proposer configs database: it still holds settings for \ - validators with no entry in validators.yml ({skipped:?})", + validators with no entry in validators.yml ({skipped:?})", ), - Err(error) => warn!("unable to prune legacy proposer configs database: {error}"), + Err(error) => { + warn!("unable to prune legacy proposer configs database: {error}") + } } } - // Fold the `web3signer` entries into the per-URL key policy: each requires its key from its URL. - for (url, pubkey) in validator_definitions.web3signer_definitions() { - web3signer_config.require_key(url.parse::()?, pubkey); - } - for pubkey in validator_definitions.disabled_pubkeys() { info_with_peers!( "validator {pubkey:?} is disabled in validators.yml and will not be loaded" @@ -1469,76 +1683,64 @@ pub fn run(parsed_args: GrandineArgs) -> Result<()> { }, )); - let validator_config = Arc::new(ValidatorConfig { - disable_blockprint_graffiti, - graffiti, - max_empty_slots, - suggested_fee_recipient, - default_gas_limit, - builder_settings: BuilderSettings { - default_builder_boost_factor, - default_builder_min_bid, - default_builder_max_execution_payment, - allow_trusted_payments, - payload_builder_urls, - }, + Ok((validator_definitions, keystore_storage)) +} + +fn load_validator_keys( + validator_client: &ValidatorClientConfig, + config: &GrandineConfig, + client: &Client, +) -> Result<(Arc, Arc)> { + let ValidatorClientConfig { + validators, keystore_storage_password_file, - backfill_custody_groups, - custody_mode, - disable_wait_for_late_blocks, - validator_definitions: validator_definitions.clone_arc(), - }); + web3signer_config, + use_validator_key_cache, + .. + } = validator_client; - let store_config = StoreConfig { - max_empty_slots, - max_epochs_to_retain_states_in_cache, - state_cache_lock_timeout, - unfinalized_states_in_memory, - kzg_backend, - sync_without_reconstruction, - builder_circuit_breaker, - }; + let mut web3signer_config = web3signer_config.clone(); - let eth1_auth = Arc::new(Auth::new(auth_options)?); + let GrandineConfig { + directories, + metrics_config, + .. + } = config; - // Creating multiple `reqwest::Client`s seems to leak memory. - // See . - // Create a single one for the whole application and reuse it through `Signer::client`. - let client = ClientBuilder::new() - .timeout(request_timeout) - .user_agent(APPLICATION_VERSION_WITH_COMMIT_AND_PLATFORM) - .connection_verbose(true) - .build()?; + let in_memory = config.in_memory; + let request_timeout = config.request_timeout; - let mut validator_key_cache = use_validator_key_cache.then(|| { - ValidatorKeyCache::new( - storage_config - .directories - .validator_dir - .clone() - .unwrap_or_default(), - ) - }); + let (validator_definitions, keystore_storage) = load_validator_definitions( + validators.as_ref(), + keystore_storage_password_file.as_deref(), + directories, + in_memory, + config.allow_trusted_payments, + )?; + + // Fold the `web3signer` entries into the per-URL key policy: each requires its key + // from its URL. + for (url, pubkey) in validator_definitions.read().web3signer_definitions() { + web3signer_config.require_key(url.parse::()?, pubkey); + } - let validator_enabled = validator_api_config.is_some() - || validator_key_cache.is_some() - || !web3signer_config.is_empty() - || !validator_definitions.read().is_empty() - || validator_config.keystore_storage_password_file.is_some(); + let mut validator_key_cache = use_validator_key_cache + .then(|| ValidatorKeyCache::new(directories.validator_dir.clone().unwrap_or_default())); - if validator_enabled { + if validator_client.expects_keys(&validator_definitions.read()) { info_with_peers!("started loading validator keys"); } let mut validator_keys = normalize_definitions( &validator_definitions.read(), - storage_config.directories.validator_dir.as_deref(), + directories.validator_dir.as_deref(), validator_key_cache.as_mut(), ) .context("unable to load validator keys")?; { - // Take only the blob keys still declared as `keystore_storage`; anything else is stale. + // Take only the blob keys still declared as `keystore_storage`; anything else is + // stale. let definitions = validator_definitions.read(); validator_keys.extend( @@ -1549,8 +1751,9 @@ pub fn run(parsed_args: GrandineArgs) -> Result<()> { ); } - // A single Web3Signer client, separate from `client` so its TLS material never reaches other - // requests. `web3signer_options` hard-errors on conflicting definitions in `validators.yml`. + // A single Web3Signer client, separate from `client` so its TLS material never reaches + // other requests. `web3signer_options` hard-errors on conflicting definitions in + // `validators.yml`. let web3signer_options = validator_definitions.read().web3signer_options()?; let web3signer_client = match web3signer_options { @@ -1574,10 +1777,9 @@ pub fn run(parsed_args: GrandineArgs) -> Result<()> { let signer = Arc::new(Signer::new( validator_keys, - client, web3signer_client, web3signer_config, - metrics.clone(), + metrics_config.metrics.clone(), )); if let Some(cache) = validator_key_cache @@ -1586,45 +1788,132 @@ pub fn run(parsed_args: GrandineArgs) -> Result<()> { warn_with_peers!("Unable to save validator key cache: {error:?}"); } - let slasher_config = slashing_enabled.then_some(SlasherConfig { - slashing_history_limit, + Ok((validator_definitions, signer)) +} + +#[derive(Debug, Error)] +enum Error { + #[error("{preset_name} preset is not included in this executable")] + PresetNotIncluded { preset_name: PresetName }, + #[error("--eth1-rpc-urls must be specified when validators are present")] + MissingEth1RpcUrlsWithValidators, + #[error( + "{service} port ({port}) is unavailable; \ + make sure no other instance of the application is running \ + or specify a different port with {option} (error: {error:?})" + )] + PortInUse { + port: u16, + service: &'static str, + option: &'static str, + error: anyhow::Error, + }, +} + +pub fn run(parsed_args: GrandineArgs) -> Result<()> { + let data_dir = parsed_args.data_dir(); + + let log_handle = binary_utils::initialize_tracing_logger( + module_path!(), + data_dir.as_deref(), + parsed_args.telemetry_config(), + cfg!(feature = "logger-always-write-style"), + parsed_args.has_network(), + )?; + + binary_utils::initialize_rayon()?; + + debug_with_peers!("grandine args - {parsed_args:?}"); + + let config = parsed_args + .try_into_config() + .map_err(GrandineArgs::clap_error)?; + + info_with_peers!( + "starting {}", + match &config.mode { + Mode::Local { + validator_client_config: Some(_), + .. + } => "beacon node with validator client", + Mode::Local { .. } => "beacon node", + Mode::Remote { .. } => "validator client", + } + ); + + config.report(); + + if let Some(node) = config.beacon_node_config() { + PEER_LOG_METRICS.set_target_peer_count(node.network_config.target_peers); + } + + // Don't check ports for command runs. None of the commands need a network connection. + // Check ports before `Context::run_with_restart` to avoid logging an error repeatedly. + // The ports could in theory be freed or taken between restarts, but it's not likely. + if config.command().is_none() { + ensure_ports_not_in_use( + config + .beacon_node_config() + .and_then(|node| node.http_api_config.as_ref()) + .map(|config| config.address), + config.beacon_node_config().map(|node| &node.network_config), + config.metrics_config.metrics_server_config.as_ref(), + config + .validator_client_config() + .and_then(|validator_client| validator_client.validator_api_config.as_ref()), + ) + .map_err(GrandineArgs::clap_error)?; + } + + if !config.in_memory { + initialize_schema(config.data_dir.clone())?; + } + + // Creating multiple `reqwest::Client`s seems to leak memory. + // See . + // Create a single one for the whole application. + let client = ClientBuilder::new() + .timeout(config.request_timeout) + .user_agent(APPLICATION_VERSION_WITH_COMMIT_AND_PLATFORM) + .connection_verbose(true) + .build()?; + + let (validator_definitions, signer) = match config.validator_client_config() { + Some(validator_client_config) => { + load_validator_keys(validator_client_config, &config, &client)? + } + // The validator service still runs under `bn`, with no keys. + None => ( + Arc::new(ValidatorDefinitionsWithStorage::new( + Arc::new(RwLock::new(ValidatorDefinitions::default())), + DefinitionsStorage::InMemory, + )), + Arc::new(Signer::new( + core::iter::empty(), + client.clone(), + Web3SignerConfig::default(), + config.metrics_config.metrics.clone(), + )), + ), + }; + + let validator_config = Arc::new(config.validator_config(validator_definitions)); + + let eth1_auth = Arc::new(match config.beacon_node_config() { + Some(node) => Auth::new(node.auth_options.clone())?, + None => Auth::default(), }); let context = Context { - predefined_network, - chain_config, - store_config, - deposit_contract_starting_block, - genesis_state_file, - genesis_state_download_url, - validator_api_config, + config, + client, + eth1_auth, validator_config, - checkpoint_sync_url, - force_checkpoint_sync, - back_sync_enabled, - eth1_rpc_urls, - network_config, - storage_config, - command, - builder_config, signer, - slasher_config, - state_slot, - eth1_auth, - http_api_config, - max_events, - metrics_config, - track_liveness, tracing_handle: Some(log_handle), - detect_doppelgangers, - slashing_protection_history_limit, - validator_enabled, - blacklisted_blocks, - reconstruction_delay, - report_validator_performance, }; - match context.chain_config.preset_base { + match context.config.chain_config.preset_base { #[cfg(any(feature = "preset-mainnet", test))] PresetName::Mainnet => context.run_with_restart::(), #[cfg(any(feature = "preset-minimal", test))] @@ -1635,21 +1924,7 @@ pub fn run(parsed_args: GrandineArgs) -> Result<()> { // Ports are checked before binding them for actual use. // This is a TOCTOU race condition, but the only consequence of it is slightly worse error messages. -fn ensure_ports_not_in_use( - http_address: Option, - network_config: &NetworkConfig, - metrics_server_config: Option<&MetricsServerConfig>, - validator_api_config: Option<&ValidatorApiConfig>, -) -> Result<()> { - if let Some(http_address) = http_address { - TcpListener::bind(http_address).map_err(|error| Error::PortInUse { - port: http_address.port(), - service: "HTTP API", - option: "--http-port", - error: error.into(), - })?; - } - +fn ensure_network_ports_not_in_use(network_config: &NetworkConfig) -> Result<()> { if let Some(listen_addr) = network_config.listen_addrs().v4() { let ListenAddr { addr, @@ -1720,6 +1995,28 @@ fn ensure_ports_not_in_use( // Port numbers in ENR fields are not used to open any sockets. + Ok(()) +} + +fn ensure_ports_not_in_use( + http_address: Option, + network_config: Option<&NetworkConfig>, + metrics_server_config: Option<&MetricsServerConfig>, + validator_api_config: Option<&ValidatorApiConfig>, +) -> Result<()> { + if let Some(network_config) = network_config { + ensure_network_ports_not_in_use(network_config)?; + } + + if let Some(http_address) = http_address { + TcpListener::bind(http_address).map_err(|error| Error::PortInUse { + port: http_address.port(), + service: "HTTP API", + option: "--http-port", + error: error.into(), + })?; + } + if let Some(config) = metrics_server_config { let metrics_port = config.metrics_port; @@ -1744,14 +2041,19 @@ fn ensure_ports_not_in_use( } #[expect(clippy::too_many_lines)] -fn handle_command( +async fn handle_command( chain_config: Arc, pubkey_cache: &Arc, storage_config: &StorageConfig, command: GrandineCommand, - anchor_checkpoint_provider: &AnchorCheckpointProvider

, + known_genesis_validators_root: Option, + load_anchor_checkpoint_provider: F, slashing_protection_history_limit: u64, -) -> Result<()> { +) -> Result<()> +where + F: Fn() -> Fut, + Fut: Future>>, +{ Feature::InhibitApplicationRestart.enable(); let StorageConfig { @@ -1786,6 +2088,7 @@ fn handle_command( ); let output_dir = output_dir.unwrap_or(std::env::current_dir()?); + let anchor_checkpoint_provider = load_anchor_checkpoint_provider().await?; fork_choice_control::export_state_and_blocks( pubkey_cache, @@ -1793,7 +2096,7 @@ fn handle_command( from, to, &output_dir, - anchor_checkpoint_provider, + &anchor_checkpoint_provider, )?; info_with_peers!("state and blocks exported to {}", output_dir.display()); @@ -1813,11 +2116,15 @@ fn handle_command( )?; } GrandineCommand::Interchange(interchange_command) => { - let genesis_validators_root = anchor_checkpoint_provider - .checkpoint() - .value - .state - .genesis_validators_root(); + let genesis_validators_root = match known_genesis_validators_root { + Some(genesis_validators_root) => genesis_validators_root, + None => load_anchor_checkpoint_provider() + .await? + .checkpoint() + .value + .state + .genesis_validators_root(), + }; let mut slashing_protector = SlashingProtector::persistent( storage_config @@ -1883,44 +2190,55 @@ fn handle_command( Ok(()) } -#[expect(clippy::too_many_arguments)] async fn genesis_checkpoint_provider( - chain_config: &Arc, - eth1_config: &Arc, - pubkey_cache: &PubkeyCache, - storage_config: &StorageConfig, - genesis_state_file: Option, - predefined_network: Option, + config: &GrandineConfig, client: &Client, - genesis_state_download_url: Option, - metrics_config: &MetricsConfig, - eth1_api_to_metrics_tx: Option<&UnboundedSender>, - restart_tx: &UnboundedSender, -) -> Result> { - if let Some(file_path) = genesis_state_file { +) -> Result>> { + if let Some(file_path) = config.genesis_state_file.as_deref() { let bytes = fs_err::read(file_path)?; - let genesis_state = Arc::from_ssz(chain_config.as_ref(), bytes)?; - return Ok(AnchorCheckpointProvider::custom_from_genesis(genesis_state)); + let genesis_state = Arc::from_ssz(config.chain_config.as_ref(), bytes)?; + return Ok(Some(AnchorCheckpointProvider::custom_from_genesis( + genesis_state, + ))); } - let store_directory = storage_config + let Some(predefined_network) = config.predefined_network else { + return Ok(None); + }; + + let store_directory = config .directories .store_directory .clone() .unwrap_or_default(); - if let Some(predefined_network) = predefined_network { - return predefined_network - .genesis_checkpoint_provider::

( - client, - store_directory.as_path(), - genesis_state_download_url, - ) - .await; - } + predefined_network + .genesis_checkpoint_provider::

( + client, + store_directory.as_path(), + config + .beacon_node_config() + .and_then(|node| node.genesis_state_download_url.clone()), + ) + .await + .map(Some) +} - // Code that waits for genesis by tracking deposits starts here - // (may be removed in the future) +// Code that waits for genesis by tracking deposits (may be removed in the future). +async fn wait_for_genesis_by_deposits( + config: &GrandineConfig, + client: &Client, + pubkey_cache: &PubkeyCache, + storage_config: &StorageConfig, + eth1_config: &Arc, + eth1_api_to_metrics_tx: Option<&UnboundedSender>, + restart_tx: &UnboundedSender, +) -> Result> { + let store_directory = config + .directories + .store_directory + .clone() + .unwrap_or_default(); let eth1_database = if storage_config.in_memory { Database::in_memory() @@ -1929,18 +2247,18 @@ async fn genesis_checkpoint_provider( }; let eth1_chain = Eth1Chain::new( - chain_config.clone_arc(), + config.chain_config.clone_arc(), eth1_config.clone_arc(), client.clone(), eth1_database, eth1_api_to_metrics_tx.cloned(), - metrics_config.metrics.clone(), + config.metrics_config.metrics.clone(), )?; let eth1_block_stream = pin!(eth1_chain.stream_blocks()?); let genesis_state = eth1::wait_for_genesis( - chain_config, + &config.chain_config, pubkey_cache, store_directory, eth1_block_stream, diff --git a/runtime/src/validators.rs b/runtime/src/validators.rs index 68e60b33c..b5fc89814 100644 --- a/runtime/src/validators.rs +++ b/runtime/src/validators.rs @@ -16,7 +16,7 @@ use tap::{Pipe as _, TryConv as _}; use validator_key_cache::ValidatorKeyCache; use zeroize::Zeroizing; -#[derive(Default)] +#[derive(Clone, Default)] #[cfg_attr(test, derive(PartialEq, Eq, Debug))] pub struct Validators { pub keystore_dir: PathBuf, diff --git a/signer/src/signer.rs b/signer/src/signer.rs index be77accaf..21d3ef08c 100644 --- a/signer/src/signer.rs +++ b/signer/src/signer.rs @@ -8,9 +8,9 @@ use arc_swap::{ArcSwap, Guard}; use bls::{PublicKeyBytes, SecretKey, Signature, traits::SecretKey as _}; use doppelganger_protection::DoppelgangerProtection; use futures::{ - TryFutureExt as _, + StreamExt as _, TryFutureExt as _, lock::Mutex, - stream::{FuturesUnordered, TryStreamExt as _}, + stream::{self, TryStreamExt as _}, try_join, }; use helper_functions::misc; @@ -22,10 +22,10 @@ use reqwest::Client; use slashing_protection::{Attestation, BlockProposal, SlashingProtector}; use std_ext::ArcExt as _; use thiserror::Error; +use tokio::sync::Notify; use tracing::instrument; use types::{ - combined::BeaconState, - phase0::primitives::{H256, Slot}, + phase0::primitives::{Epoch, H256, Slot}, preset::Preset, redacting_url::RedactingUrl, }; @@ -36,6 +36,10 @@ use crate::{ web3signer::{FetchedKeys, Web3Signer}, }; +// Whole epochs of selection proofs and registrations are signed at once, which would otherwise +// open thousands of requests to a Web3Signer. +const MAX_CONCURRENT_WEB3SIGNER_REQUESTS: usize = 32; + #[derive(Debug, Error)] enum Error { #[error("Cannot sign due to missing credentials for a public key: {public_key:?}")] @@ -61,12 +65,12 @@ enum SignMethod { pub struct Signer { snapshot: ArcSwap, + keys_changed: Notify, } impl Signer { pub fn new( validator_keys: impl IntoIterator, KeyOrigin)>, - client: Client, web3signer_client: Client, web3signer_config: Web3SignerConfig, metrics: Option>, @@ -80,12 +84,14 @@ impl Signer { let snapshot = ArcSwap::from_pointee(Snapshot { sign_methods, - client, web3signer: Web3Signer::new(web3signer_client, web3signer_config, metrics), doppelganger_protection: None, }); - Self { snapshot } + Self { + snapshot, + keys_changed: Notify::new(), + } } pub fn enable_doppelganger_protection( @@ -140,19 +146,34 @@ impl Signer { F: FnMut(&Arc) -> R, R: Into>, { - self.snapshot.rcu(f) + let previous = self.snapshot.rcu(f); + let current = self.snapshot.load(); + + let keys_changed = previous.sign_methods.len() != current.sign_methods.len() + || previous + .sign_methods + .keys() + .any(|public_key| !current.sign_methods.contains_key(public_key)); + + if keys_changed { + // Unlike `notify_waiters`, this keeps a permit for a waiter that comes later. + self.keys_changed.notify_one(); + } + + previous } - pub fn update_doppelganger_protection_pubkeys( - &self, - beacon_state: &BeaconState

, - current_slot: Slot, - ) { + #[must_use] + pub const fn keys_changed(&self) -> &Notify { + &self.keys_changed + } + + pub fn update_doppelganger_protection_pubkeys(&self, current_slot: Slot) { let snapshot = self.load(); let public_keys = snapshot.keys().copied(); if let Some(doppelganger_protection) = &snapshot.doppelganger_protection { - doppelganger_protection.add_tracked_validators(public_keys, beacon_state, current_slot); + doppelganger_protection.add_tracked_validators(public_keys, current_slot); } } } @@ -160,7 +181,6 @@ impl Signer { #[derive(Clone)] pub struct Snapshot { sign_methods: HashMap, - client: Client, web3signer: Web3Signer, doppelganger_protection: Option>, } @@ -214,11 +234,6 @@ impl Snapshot { }) } - #[must_use] - pub const fn client(&self) -> &Client { - &self.client - } - pub fn append_keys( &mut self, keys: impl IntoIterator)>, @@ -271,7 +286,7 @@ impl Snapshot { &self, message: SigningMessage<'_, P>, signing_root: H256, - fork_info: Option>, + fork_info: Option, public_key: PublicKeyBytes, ) -> Result { let signature = match self.sign_method(public_key)? { @@ -291,7 +306,8 @@ impl Snapshot { pub async fn sign_triples( &self, triples: impl IntoIterator> + Send, - beacon_state: &BeaconState

, + fork_info: ForkInfo, + current_epoch: Epoch, slashing_protector: Arc>, ) -> Result>> { let mut message_indices = vec![]; @@ -303,7 +319,6 @@ impl Snapshot { let mut block_proposals = vec![]; let mut signable_messages = vec![]; - let fork_info = ForkInfo::from(beacon_state); let mut signing_triples_count: usize = 0; let doppelganger_protection = self @@ -385,7 +400,7 @@ impl Snapshot { tokio::task::block_in_place(|| { let slashing_outcome = - protector.validate_and_store_own_attestations(beacon_state, attestations)?; + protector.validate_and_store_own_attestations(current_epoch, attestations)?; for (outcome, data, index) in izip!( slashing_outcome.iter(), @@ -445,7 +460,26 @@ impl Snapshot { pub async fn sign_triples_without_slashing_protection( &self, triples: impl IntoIterator> + Send, - fork_info: Option>, + fork_info: Option, + ) -> Result> { + self.sign_triples_with_request_limit(triples, fork_info, usize::MAX) + .await + } + + pub async fn sign_triples_throttled_without_slashing_protection( + &self, + triples: impl IntoIterator> + Send, + fork_info: Option, + ) -> Result> { + self.sign_triples_with_request_limit(triples, fork_info, MAX_CONCURRENT_WEB3SIGNER_REQUESTS) + .await + } + + async fn sign_triples_with_request_limit( + &self, + triples: impl IntoIterator> + Send, + fork_info: Option, + max_concurrent_requests: usize, ) -> Result> { let mut sign_locally = vec![]; let mut sign_remotely = vec![]; @@ -479,7 +513,7 @@ impl Snapshot { .map_err(Into::into); let sign_remotely_future = async { - sign_remotely + let requests = sign_remotely .into_iter() .map(|(index, message, signing_root, public_key)| async move { self.sign_without_slashing_protection( @@ -491,7 +525,10 @@ impl Snapshot { .await .map(|signature| (index, signature)) }) - .collect::>() + .collect_vec(); + + stream::iter(requests) + .buffer_unordered(max_concurrent_requests) .try_collect::>() .await }; @@ -512,3 +549,50 @@ impl Snapshot { .map_err(Into::into) } } + +#[cfg(test)] +mod tests { + use bls::SecretKeyBytes; + use futures::FutureExt as _; + use hex_literal::hex; + use std_ext::ArcExt; + + use super::*; + + const PUBLIC_KEY: PublicKeyBytes = PublicKeyBytes(hex!( + "b301803f8b5ac4a1133581fc676dfedc60d891dd5fa99028805e5ea5b08d3491af75d0707adab3b70c6a6a580217bf81" + )); + + const SECRET_KEY: [u8; 32] = + hex!("47b8192d77bf871b62e87859d653922725724a5c031afeabc60bcef5ff665138"); + + // The index resolver waits on the notification, so it must fire on a key change and only then. + #[test] + fn update_notifies_only_when_the_key_set_changes() -> Result<()> { + let signer = Signer::new([], Client::new(), Web3SignerConfig::default(), None); + + let secret_key: Arc = Arc::new(SecretKeyBytes::from(SECRET_KEY).try_into()?); + + signer.update(ArcExt::clone_arc); + assert!(signer.keys_changed().notified().now_or_never().is_none()); + + signer.update(|snapshot| { + let mut snapshot = snapshot.as_ref().clone(); + snapshot.append_keys([(PUBLIC_KEY, secret_key.clone_arc())]); + snapshot + }); + assert!(signer.keys_changed().notified().now_or_never().is_some()); + + signer.update(ArcExt::clone_arc); + assert!(signer.keys_changed().notified().now_or_never().is_none()); + + signer.update(|snapshot| { + let mut snapshot = snapshot.as_ref().clone(); + snapshot.delete_key(PUBLIC_KEY); + snapshot + }); + assert!(signer.keys_changed().notified().now_or_never().is_some()); + + Ok(()) + } +} diff --git a/signer/src/types.rs b/signer/src/types.rs index 7f86f4cc4..939363778 100644 --- a/signer/src/types.rs +++ b/signer/src/types.rs @@ -1,11 +1,10 @@ -use core::marker::PhantomData; - use bls::PublicKeyBytes; use builder_api::{ gloas::containers::BuilderRequestAuth, unphased::containers::ValidatorRegistrationV1, }; use serde::Serialize; use ssz::Hc; +pub use types::nonstandard::ForkInfo; use types::{ altair::containers::{ BeaconBlock as AltairBeaconBlock, ContributionAndProof, SyncAggregatorSelectionData, @@ -31,33 +30,14 @@ use types::{ }, phase0::{ containers::{ - AttestationData, BeaconBlock as Phase0BeaconBlock, BeaconBlockHeader, Fork, - VoluntaryExit, + AttestationData, BeaconBlock as Phase0BeaconBlock, BeaconBlockHeader, VoluntaryExit, }, primitives::{Epoch, H256, Slot}, }, preset::Preset, - traits::{BeaconBlock as _, BeaconState}, + traits::BeaconBlock as _, }; -#[derive(Clone, Copy, Debug, Serialize)] -pub struct ForkInfo { - pub fork: Fork, - pub genesis_validators_root: H256, - #[serde(skip)] - pub phantom: PhantomData

, -} - -impl> From<&BS> for ForkInfo

{ - fn from(state: &BS) -> Self { - Self { - fork: state.fork(), - genesis_validators_root: state.genesis_validators_root(), - phantom: PhantomData, - } - } -} - #[derive(Debug)] pub struct SigningTriple<'block, P: Preset> { pub message: SigningMessage<'block, P>, diff --git a/signer/src/web3signer/api.rs b/signer/src/web3signer/api.rs index 2184382da..edfa2bb57 100644 --- a/signer/src/web3signer/api.rs +++ b/signer/src/web3signer/api.rs @@ -179,7 +179,7 @@ impl Web3Signer { api_url: &RedactingUrl, message: SigningMessage<'_, P>, signing_root: H256, - fork_info: Option>, + fork_info: Option, public_key: PublicKeyBytes, ) -> Result { let _timer = self @@ -447,7 +447,7 @@ mod tests { let altair_block = Hc::new(AltairBeaconBlock::default()); let message = SigningMessage::from(&altair_block); let signing_root = altair_block.signing_root(&Config::minimal(), &beacon_state); - let fork_info = ForkInfo::from(&beacon_state); + let fork_info = ForkInfo::from_state(&beacon_state); let response = web3signer .sign(&url, message, signing_root, Some(fork_info), SAMPLE_PUBKEY) diff --git a/signer/src/web3signer/types.rs b/signer/src/web3signer/types.rs index d76d0c416..21fefd923 100644 --- a/signer/src/web3signer/types.rs +++ b/signer/src/web3signer/types.rs @@ -11,7 +11,7 @@ pub struct SigningRequest<'block, P: Preset> { #[serde(rename = "type")] message_type: MessageType, #[serde(skip_serializing_if = "Option::is_none")] - fork_info: Option>, + fork_info: Option, #[serde(rename = "signingRoot")] signing_root: H256, #[serde(flatten)] @@ -22,7 +22,7 @@ impl<'block, P: Preset> SigningRequest<'block, P> { pub const fn new( message: SigningMessage<'block, P>, signing_root: H256, - fork_info: Option>, + fork_info: Option, ) -> Self { let message_type = match message { SigningMessage::AggregationSlot { .. } => MessageType::AggregationSlot, diff --git a/slasher/src/slasher_config.rs b/slasher/src/slasher_config.rs index d1d1833e9..adb985830 100644 --- a/slasher/src/slasher_config.rs +++ b/slasher/src/slasher_config.rs @@ -1,6 +1,6 @@ use derivative::Derivative; -#[derive(Clone, Copy, Derivative)] +#[derive(Clone, Copy, Debug, Derivative)] #[derivative(Default)] pub struct SlasherConfig { #[derivative(Default(value = "54000"))] diff --git a/slashing_protection/Cargo.toml b/slashing_protection/Cargo.toml index dff8e1546..bfc62b400 100644 --- a/slashing_protection/Cargo.toml +++ b/slashing_protection/Cargo.toml @@ -28,9 +28,7 @@ types = { workspace = true } [dev-dependencies] duplicate = { workspace = true } -factory = { workspace = true } hex-literal = { workspace = true } -pubkey_cache = { workspace = true } tempfile = { workspace = true } test-case = { workspace = true } test-generator = { workspace = true } diff --git a/slashing_protection/src/lib.rs b/slashing_protection/src/lib.rs index 946a8e904..98e858e62 100644 --- a/slashing_protection/src/lib.rs +++ b/slashing_protection/src/lib.rs @@ -5,14 +5,13 @@ use anyhow::Result; use bls::PublicKeyBytes; use derivative::Derivative; use fs_err::File; -use helper_functions::{accessors, misc}; +use helper_functions::misc; use itertools::Itertools as _; use logging::{debug_with_peers, info_with_peers, warn_with_peers}; use rusqlite::{Connection, OptionalExtension, Rows, Transaction, TransactionBehavior}; use ssz::{SszReadDefault as _, SszWrite as _}; use thiserror::Error; use types::{ - combined::BeaconState, phase0::primitives::{Epoch, H256, Slot}, preset::Preset, }; @@ -619,13 +618,11 @@ impl SlashingProtector { Ok(result) } - pub fn validate_and_store_own_attestations( + pub fn validate_and_store_own_attestations( &mut self, - state: &BeaconState

, + current_epoch: Epoch, attestations: impl IntoIterator + Clone, ) -> Result>> { - let current_epoch = accessors::get_current_epoch(state); - if self.validate_current_epoch(current_epoch)?.is_some() { return Ok(vec![]); } @@ -1026,12 +1023,11 @@ fn move_slashing_protection_db_to_validator_dir( mod tests { use duplicate::duplicate_item; use hex_literal::hex; - use pubkey_cache::PubkeyCache; use serde::{Deserialize, de::IgnoredAny}; use tempfile::{Builder, TempDir}; use test_case::test_case; use test_generator::test_resources; - use types::{config::Config, preset::Minimal, traits::BeaconState as _}; + use types::preset::Minimal; use super::*; @@ -1221,9 +1217,6 @@ mod tests { #[test_case(build_persistent_slashing_protector)] #[test_case(build_in_memory_slashing_protector)] fn test_slashing_protection_current_epoch(constructor: Constructor) -> Result<()> { - let config = Config::minimal(); - let pubkey_cache = PubkeyCache::default(); - let (mut slashing_protector, _store_dir, _validator_dir) = constructor()?; assert_eq!(slashing_protector.stored_current_epoch()?, None); @@ -1244,12 +1237,10 @@ mod tests { slashing_protector.register_validators(core::iter::once(PUBKEY))?; - let (mut state, _) = factory::min_genesis_state::(&config, &pubkey_cache)?; - let attestation = build_own_attestation(2, 32); let accepted_attestations = slashing_protector - .validate_and_store_own_attestations(&state, core::iter::once((attestation, PUBKEY)))?; + .validate_and_store_own_attestations(32, core::iter::once((attestation, PUBKEY)))?; assert_eq!(count_some(&accepted_attestations), 0); assert_eq!( @@ -1262,10 +1253,8 @@ mod tests { )), ); - *state.slot_mut() = misc::compute_start_slot_at_epoch::(1024); - let accepted_attestations = slashing_protector - .validate_and_store_own_attestations(&state, core::iter::once((attestation, PUBKEY)))?; + .validate_and_store_own_attestations(1024, core::iter::once((attestation, PUBKEY)))?; assert_eq!(slashing_protector.validate_current_epoch(1024)?, None); assert_eq!(count_some(&accepted_attestations), 1); @@ -1294,21 +1283,16 @@ mod tests { #[test_case(build_persistent_slashing_protector)] #[test_case(build_in_memory_slashing_protector)] fn test_slashing_protection_attestation_pruning(constructor: Constructor) -> Result<()> { - let config = Config::minimal(); - let pubkey_cache = PubkeyCache::default(); - let (mut slashing_protector, _store_dir, _validator_dir) = constructor()?; slashing_protector.register_validators(core::iter::once(PUBKEY))?; - let (state, _) = factory::min_genesis_state::(&config, &pubkey_cache)?; - let attestation_1 = build_own_attestation(2, 32); let attestation_2 = build_own_attestation(34, 64); let attestation_3 = build_own_attestation(64, 66); let accepted_attestations = slashing_protector.validate_and_store_own_attestations( - &state, + 0, [ (attestation_1, PUBKEY), (attestation_2, PUBKEY), diff --git a/ssz_derive/src/ssz_type.rs b/ssz_derive/src/ssz_type.rs index 672a87525..2f233e973 100644 --- a/ssz_derive/src/ssz_type.rs +++ b/ssz_derive/src/ssz_type.rs @@ -535,7 +535,7 @@ impl SszType { let chunks = stable.active.0.iter().map(|&active| { if active { - let (member, _) = fields.next().expect("active field shoudl be validated"); + let (member, _) = fields.next().expect("active field should be validated"); quote! { #ssz::SszHash::hash_tree_root(&self.#member) } } else { diff --git a/types/src/combined.rs b/types/src/combined.rs index fefbe5791..ef0d1f438 100644 --- a/types/src/combined.rs +++ b/types/src/combined.rs @@ -1453,6 +1453,35 @@ impl SszSize for BlindedBeaconBlock

{ ]); } +impl SszRead for BlindedBeaconBlock

{ + fn from_ssz_unchecked(phase: &Phase, bytes: &[u8]) -> Result { + let block = match phase { + Phase::Phase0 => { + return Err(ReadError::Custom { + message: "blinded block has slot in Phase 0", + }); + } + Phase::Altair => { + return Err(ReadError::Custom { + message: "blinded block has slot in Altair", + }); + } + Phase::Bellatrix => Self::Bellatrix(SszReadDefault::from_ssz_default(bytes)?), + Phase::Capella => Self::Capella(SszReadDefault::from_ssz_default(bytes)?), + Phase::Deneb => Self::Deneb(SszReadDefault::from_ssz_default(bytes)?), + Phase::Electra => Self::Electra(SszReadDefault::from_ssz_default(bytes)?), + Phase::Fulu => Self::Fulu(SszReadDefault::from_ssz_default(bytes)?), + Phase::Gloas => { + return Err(ReadError::Custom { + message: "blinded block has slot in Gloas", + }); + } + }; + + Ok(block) + } +} + impl SszWrite for BlindedBeaconBlock

{ fn write_variable(&self, bytes: &mut Vec) -> Result<(), WriteError> { match self { diff --git a/types/src/config.rs b/types/src/config.rs index 269abdbeb..412c5c91c 100644 --- a/types/src/config.rs +++ b/types/src/config.rs @@ -20,7 +20,8 @@ use crate::{ gloas::containers::DataColumnSidecar as GloasDataColumnSidecar, nonstandard::{CustodyMode, Phase, Toption}, phase0::{ - consts::{AttestationSubnetCount, FAR_FUTURE_EPOCH, GENESIS_EPOCH}, + consts::{AttestationSubnetCount, BASIS_POINTS, FAR_FUTURE_EPOCH, GENESIS_EPOCH}, + containers::Fork, primitives::{ ChainId, DomainType, Epoch, ExecutionAddress, ExecutionBlockHash, Gwei, H32, H160, H256, NetworkId, Slot, UnixSeconds, Version, @@ -992,6 +993,67 @@ impl Config { self.version(phase) } + #[must_use] + pub fn fork_at_epoch(&self, epoch: Epoch) -> Fork { + let phase = self.phase_at_epoch(epoch); + + Fork { + previous_version: self.version(phase.previous().unwrap_or(phase)), + current_version: self.version(phase), + epoch: self.fork_epoch(phase), + } + } + + #[must_use] + pub fn attestation_due_bps_at(&self, phase: Phase) -> u64 { + if phase < Phase::Gloas { + self.attestation_due_bps + } else { + self.attestation_due_bps_gloas + } + } + + #[must_use] + pub fn aggregate_due_bps_at(&self, phase: Phase) -> u64 { + if phase < Phase::Gloas { + self.aggregate_due_bps + } else { + self.aggregate_due_bps_gloas + } + } + + #[must_use] + pub fn sync_message_due_bps_at(&self, phase: Phase) -> u64 { + if phase < Phase::Gloas { + self.sync_message_due_bps + } else { + self.sync_message_due_bps_gloas + } + } + + #[must_use] + pub fn contribution_due_bps_at(&self, phase: Phase) -> u64 { + if phase < Phase::Gloas { + self.contribution_due_bps + } else { + self.contribution_due_bps_gloas + } + } + + /// + #[must_use] + pub fn slot_component_duration(&self, bps: u64) -> Duration { + let nanos = self + .slot_duration_ms + .as_nanos() + .saturating_mul(bps.into()) + .checked_div(BASIS_POINTS.into()) + .and_then(|nanos| nanos.try_into().ok()) + .unwrap_or(u64::MAX); + + Duration::from_nanos(nanos) + } + #[inline] #[must_use] pub const fn fork_epoch(&self, phase: Phase) -> Epoch { @@ -1419,4 +1481,109 @@ mod tests { config.gas_limit(configured, epoch) } + + #[test_case(Config::mainnet())] + #[test_case(Config::minimal())] + #[test_case(Config::medalla())] + #[test_case(Config::goerli())] + #[test_case(Config::sepolia())] + #[test_case(Config::holesky())] + #[test_case(Config::holesky_devnet())] + fn config_fork_at_epoch_agrees_with_version_at_epoch(config: Config) { + let boundaries = config + .fork_epochs() + .map(|(_, fork_epoch)| fork_epoch) + .filter(|fork_epoch| *fork_epoch != FAR_FUTURE_EPOCH) + .flat_map(|fork_epoch| { + [ + fork_epoch.saturating_sub(1), + fork_epoch, + fork_epoch.saturating_add(1), + ] + }) + .chain([GENESIS_EPOCH]) + .collect_vec(); + + for epoch in boundaries { + assert_eq!( + config.fork_at_epoch(epoch).current_version, + config.version_at_epoch(epoch), + "fork at epoch {epoch} names the wrong current version", + ); + } + } + + fn fork_schedule() -> Config { + Config { + altair_fork_epoch: 2, + bellatrix_fork_epoch: 4, + // Never scheduled, so nothing may fall back to their versions. + capella_fork_epoch: FAR_FUTURE_EPOCH, + deneb_fork_epoch: FAR_FUTURE_EPOCH, + electra_fork_epoch: FAR_FUTURE_EPOCH, + fulu_fork_epoch: FAR_FUTURE_EPOCH, + gloas_fork_epoch: FAR_FUTURE_EPOCH, + ..Config::mainnet() + } + } + + #[test] + fn config_fork_at_epoch_is_genesis_before_the_first_fork() { + let config = fork_schedule(); + + for epoch in [GENESIS_EPOCH, 1] { + let fork = config.fork_at_epoch(epoch); + + assert_eq!(fork.previous_version, config.genesis_fork_version); + assert_eq!(fork.current_version, config.genesis_fork_version); + assert_eq!(fork.epoch, GENESIS_EPOCH); + } + } + + #[test] + fn config_fork_at_epoch_switches_in_the_epoch_of_the_fork() { + let config = fork_schedule(); + let fork = config.fork_at_epoch(2); + + assert_eq!(fork.previous_version, config.genesis_fork_version); + assert_eq!(fork.current_version, config.altair_fork_version); + assert_eq!(fork.epoch, 2); + } + + #[test] + fn config_fork_at_epoch_carries_the_version_of_the_fork_before() { + let config = fork_schedule(); + let fork = config.fork_at_epoch(4); + + assert_eq!(fork.previous_version, config.altair_fork_version); + assert_eq!(fork.current_version, config.bellatrix_fork_version); + assert_eq!(fork.epoch, 4); + } + + // `phase_at_epoch` stops at the first phase that is not scheduled, so the one before the + // current phase is always scheduled too. + #[test] + fn config_fork_at_epoch_ignores_phases_that_are_never_scheduled() { + let config = fork_schedule(); + let fork = config.fork_at_epoch(FAR_FUTURE_EPOCH - 1); + + assert_eq!(fork.previous_version, config.altair_fork_version); + assert_eq!(fork.current_version, config.bellatrix_fork_version); + assert_eq!(fork.epoch, 4); + } + + // A custom config's slot duration or bps saturates the duration rather than panicking. + #[test_case(Duration::from_secs(12), 2_500 => Duration::from_secs(3))] + #[test_case(Duration::MAX, BASIS_POINTS => Duration::from_nanos(u64::MAX))] + #[test_case(Duration::from_secs(12), u64::MAX => Duration::from_nanos(u64::MAX))] + fn slot_component_duration_scales_and_saturates( + slot_duration_ms: Duration, + bps: u64, + ) -> Duration { + Config { + slot_duration_ms, + ..Config::mainnet() + } + .slot_component_duration(bps) + } } diff --git a/types/src/nonstandard.rs b/types/src/nonstandard.rs index 039531fec..455c2d518 100644 --- a/types/src/nonstandard.rs +++ b/types/src/nonstandard.rs @@ -21,7 +21,7 @@ use ssz::{ }; use static_assertions::assert_eq_size; use std_ext::CopyExt as _; -use strum::{AsRefStr, Display, EnumString}; +use strum::{AsRefStr, Display, EnumString, VariantNames}; use try_from_iterator::TryFromIterator; use crate::{ @@ -31,7 +31,10 @@ use crate::{ }, bellatrix::{containers::PowBlock, primitives::Wei}, capella::containers::Withdrawal, - combined::{Attestation, BeaconState, DataColumnSidecar, ExecutionRequests, SignedBeaconBlock}, + combined::{ + Attestation, BeaconState as CombinedBeaconState, DataColumnSidecar, ExecutionRequests, + SignedBeaconBlock, + }, config::Config, deneb::{ containers::{BlobIdentifier, BlobSidecar}, @@ -40,11 +43,11 @@ use crate::{ fulu::containers::DataColumnIdentifier, gloas::primitives::{BlockAccessList, Transaction as GloasTransaction}, phase0::{ - containers::{SignedBeaconBlockHeader, Validator}, + containers::{Fork, SignedBeaconBlockHeader, Validator}, primitives::{Epoch, Gwei, Slot, Uint256, UnixSeconds, ValidatorIndex}, }, preset::Preset, - traits::{BlockBodyWithBlobKzgCommitments, SignedBeaconBlock as _}, + traits::{BeaconState, BlockBodyWithBlobKzgCommitments, SignedBeaconBlock as _}, }; pub use smallvec::smallvec; @@ -264,6 +267,44 @@ pub struct BlockRewards { pub attester_slashings: Gwei, } +#[derive(Clone, Copy, Debug, Serialize)] +pub struct ForkInfo { + pub fork: Fork, + pub genesis_validators_root: H256, +} + +impl ForkInfo { + #[must_use] + pub fn from_state(state: &(impl BeaconState

+ ?Sized)) -> Self { + Self { + fork: state.fork(), + genesis_validators_root: state.genesis_validators_root(), + } + } + + /// [`Self::from_state`] for an epoch whose state is not at hand, as in next-epoch duties. + #[must_use] + pub fn at_epoch(config: &Config, epoch: Epoch, genesis_validators_root: H256) -> Self { + Self { + fork: config.fork_at_epoch(epoch), + genesis_validators_root, + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Display, EnumString, VariantNames)] +#[strum(serialize_all = "kebab-case", ascii_case_insensitive)] +pub enum PublishedDuty { + All, + Aggregates, + Attestations, + Blocks, + PayloadAttestations, + ProposerPreferences, + SyncCommitteeContributions, + SyncCommitteeMessages, +} + #[derive(Clone, Copy, PartialEq, Eq, Default, Debug)] pub enum CustodyMode { #[default] @@ -690,7 +731,7 @@ pub struct SystemStats { #[derive(Clone)] pub struct FinalizedCheckpoint { pub block: Arc>, - pub state: Arc>, + pub state: Arc>, } #[derive(Clone, Copy)] diff --git a/types/src/preset.rs b/types/src/preset.rs index 495ede8b5..3947cbab5 100644 --- a/types/src/preset.rs +++ b/types/src/preset.rs @@ -48,6 +48,7 @@ use crate::{ }, gloas::primitives::Transaction as GloasTransaction, phase0::{ + consts::TargetAggregatorsPerCommittee, containers::{ Attestation, AttesterSlashing, Deposit, ProposerSlashing, SignedVoluntaryExit, }, @@ -288,6 +289,7 @@ pub trait Preset: Copy + Eq + Ord + Hash + Default + Debug + Send + Sync + 'stat + Debug + Send + Sync; + type MaxAggregatesPerSlot: Unsigned + Send + Sync; type CellsPerExtBlob: ContiguousVectorElements + ContiguousVectorElements> + ArrayLength @@ -433,6 +435,7 @@ impl Preset for Mainnet { // Derived type-level variables type MaxAttestersPerSlot = Prod; + type MaxAggregatesPerSlot = Prod; type MaxCellProofsPerBlock = Prod; type CellsPerExtBlob = Quot; @@ -540,6 +543,7 @@ impl Preset for Minimal { // Derived type-level variables type MaxAttestersPerSlot = Prod; + type MaxAggregatesPerSlot = Prod; type MaxCellProofsPerBlock = Prod; type CellsPerExtBlob = Quot; @@ -642,6 +646,7 @@ impl Preset for Medalla { // Derived type-level variables type MaxAttestersPerSlot; + type MaxAggregatesPerSlot; type MaxCellProofsPerBlock; type CellsPerExtBlob; } diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 5312cefc6..9696c614e 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -8,8 +8,11 @@ workspace = true [dependencies] anyhow = { workspace = true } +arc-swap = { workspace = true } +arithmetic = { workspace = true } axum = { workspace = true } axum-extra = { workspace = true } +binary_utils = { workspace = true } block_producer = { workspace = true } bls = { workspace = true } builder_api = { workspace = true } @@ -18,6 +21,7 @@ constant_time_eq = { workspace = true } debug_info = { workspace = true } dedicated_executor = { workspace = true } derivative = { workspace = true } +derive_more = { workspace = true } directories = { workspace = true } doppelganger_protection = { workspace = true } eip_7594 = { workspace = true } @@ -30,21 +34,24 @@ fs-err = { workspace = true } futures = { workspace = true } helper_functions = { workspace = true } hex = { workspace = true } +http-body-util = { workspace = true } http_api_utils = { workspace = true } itertools = { workspace = true } keymanager = { workspace = true } liveness_tracker = { workspace = true } logging = { workspace = true } +mime = { workspace = true } once_cell = { workspace = true } operation_pools = { workspace = true } p2p = { workspace = true } prometheus_metrics = { workspace = true } rand = { workspace = true } -rayon = { workspace = true } reqwest = { workspace = true } scc = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } serde_utils = { workspace = true } +sse-stream = { workspace = true } signer = { workspace = true } slasher = { workspace = true } slashing_protection = { workspace = true } @@ -73,11 +80,11 @@ eth2_cache_utils = { workspace = true } factory = { workspace = true } #genesis = { workspace = true } hex-literal = { workspace = true } +httpmock = { workspace = true } interop = { workspace = true } #num_cpus = { workspace = true } pubkey_cache = { workspace = true } reqwest = { workspace = true } -serde_json = { workspace = true } #spec_test_utils = { workspace = true } tempfile = { workspace = true } test-case = { workspace = true } diff --git a/validator/src/api.rs b/validator/src/api.rs index cfe4373c9..b869ec241 100644 --- a/validator/src/api.rs +++ b/validator/src/api.rs @@ -9,7 +9,7 @@ use std::{ sync::Arc, }; -use anyhow::{Error as AnyhowError, Result, ensure}; +use anyhow::{Error as AnyhowError, Result, bail, ensure}; use axum::{ Json, RequestExt as _, RequestPartsExt as _, Router, body::Body, @@ -27,13 +27,14 @@ use axum_extra::{ extract::{Query, QueryRejection}, headers::{Authorization, authorization::Bearer}, }; +use binary_utils::TracingHandle; use bls::PublicKeyBytes; +use clock::Tick; use constant_time_eq::constant_time_eq; use directories::Directories; -use eth1_api::ApiController; use fork_choice_control::Wait; use helper_functions::{accessors, error::Error as HelperError, misc, signing::SignForSingleFork}; -use http_api_utils::{ApiError, ApiMetrics}; +use http_api_utils::{ApiError, ApiMetrics, post_log_level, post_trace_level}; use keymanager::{ BuilderConfigError, BuilderConfigOptions, KeyManager, KeymanagerError, KeymanagerOperationStatus, ListedRemoteKey, RemoteKey, ResolvedBuilderConfig, ValidatingPubkey, @@ -49,14 +50,18 @@ use tower_http::cors::AllowOrigin; use tracing::instrument; use types::{ bellatrix::primitives::Gas, + nonstandard::ForkInfo, phase0::{ containers::{SignedVoluntaryExit, VoluntaryExit}, primitives::{Epoch, ExecutionAddress}, }, preset::Preset, }; + use zeroize::Zeroizing; +use crate::misc::{Chain, ChainSource}; + const VALIDATOR_API_TOKEN_PATH: &str = "api-token.txt"; const VALIDATOR_API_DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); @@ -94,6 +99,8 @@ impl ValidatorApiConfig { #[derive(Debug, Error)] enum Error { + #[error("fee recipient is not set for validator {pubkey:?}")] + FeeRecipientNotSet { pubkey: PublicKeyBytes }, #[error("internal error")] Internal(#[from] AnyhowError), #[error("invalid builder config")] @@ -155,9 +162,9 @@ impl Error { | Self::InvalidGraffiti(_) | Self::InvalidPublicKey(_) | Self::InvalidQuery(_) => StatusCode::BAD_REQUEST, - Self::ValidatorNotFound { pubkey: _ } | Self::ValidatorNotOwned { pubkey: _ } => { - StatusCode::NOT_FOUND - } + Self::FeeRecipientNotSet { pubkey: _ } + | Self::ValidatorNotFound { pubkey: _ } + | Self::ValidatorNotOwned { pubkey: _ } => StatusCode::NOT_FOUND, Self::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, Self::Unauthorized => StatusCode::UNAUTHORIZED, } @@ -353,15 +360,16 @@ impl FromRequestParts for EthQuery #[derive(Clone)] struct ValidatorApiState { - controller: ApiController, + chain_source: Arc>, keymanager: Arc, signer: Arc, token: Arc, + tracing_handle: Option, } -impl FromRef> for ApiController { +impl FromRef> for Arc> { fn from_ref(state: &ValidatorApiState) -> Self { - state.controller.clone_arc() + state.chain_source.clone_arc() } } @@ -383,6 +391,12 @@ impl FromRef> for Arc { } } +impl FromRef> for Option { + fn from_ref(state: &ValidatorApiState) -> Self { + state.tracing_handle.clone() + } +} + #[derive(Deserialize)] struct SetFeeRecipientQuery { ethaddress: ExecutionAddress, @@ -466,7 +480,10 @@ async fn keymanager_list_fee_recipient( State(keymanager): State>, EthPath(pubkey): EthPath, ) -> Result, Error> { - let fee_recipient = keymanager.proposer_configs().fee_recipient(pubkey); + let fee_recipient = keymanager + .proposer_configs() + .configured_fee_recipient(pubkey) + .ok_or(Error::FeeRecipientNotSet { pubkey })?; let response = ProposerConfigResponse { pubkey, @@ -509,13 +526,13 @@ async fn keymanager_delete_fee_recipient( /// `GET /eth/v1/validator/{pubkey}/gas_limit` async fn keymanager_get_gas_limit( - State(controller): State>, + State(chain_source): State>>, State(keymanager): State>, EthPath(pubkey): EthPath, ) -> Result, Error> { - let epoch = misc::compute_epoch_at_slot::

(controller.slot()); + let epoch = misc::compute_epoch_at_slot::

(chain_source.slot()?); let configured = keymanager.proposer_configs().gas_limit(pubkey); - let gas_limit = controller.chain_config().gas_limit(configured, epoch); + let gas_limit = chain_source.chain_config.gas_limit(configured, epoch); let response = ProposerConfigResponse { pubkey, @@ -717,25 +734,66 @@ async fn keymanager_delete_remote_keys( /// `POST /eth/v1/validator/{pubkey}/voluntary_exit` async fn keymanager_create_voluntary_exit( - State(controller): State>, + State(chain_source): State>>, State(signer): State>, EthPath(pubkey): EthPath, EthQuery(query): EthQuery, ) -> Result, Error> { - let state = controller.preprocessed_state_at_current_slot().await?; - - let epoch = query - .epoch - .unwrap_or_else(|| accessors::get_current_epoch(&state)); - let signer_snapshot = signer.load(); if !signer_snapshot.has_key(pubkey) { return Err(Error::ValidatorNotOwned { pubkey }); } - let validator_index = accessors::index_of_public_key(&state, &pubkey) - .ok_or(Error::ValidatorNotFound { pubkey })?; + let chain_config = &chain_source.chain_config; + + let (epoch, validator_index, fork_info) = match &chain_source.chain { + Chain::Local(local) => { + let state = local + .controller + .preprocessed_state_at_current_slot() + .await?; + let epoch = query + .epoch + .unwrap_or_else(|| accessors::get_current_epoch(&state)); + + let validator_index = accessors::index_of_public_key(&state, &pubkey) + .ok_or(Error::ValidatorNotFound { pubkey })?; + + (epoch, validator_index, ForkInfo::from_state(state.as_ref())) + } + Chain::Remote(remote_beacon_nodes) => { + let current_epoch = + Tick::current::

(chain_config, chain_source.genesis_time)?.epoch::

(); + let epoch = query.epoch.unwrap_or(current_epoch); + let known_index = chain_source + .own_validator_indices + .load() + .get(&pubkey) + .copied() + .flatten(); + + // A key the duty loop has not resolved yet, such as one just imported, is asked about. + let validator_index = match known_index { + Some(validator_index) => validator_index, + None => remote_beacon_nodes + .validator_indices(&[pubkey]) + .await? + .get(&pubkey) + .copied() + .ok_or(Error::ValidatorNotFound { pubkey })?, + }; + + // Verifiers use the state's fork, not the exit's epoch. + let fork_info = ForkInfo::at_epoch( + chain_config, + current_epoch, + chain_source.genesis_validators_root, + ); + + (epoch, validator_index, fork_info) + } + }; let voluntary_exit = VoluntaryExit { epoch, @@ -744,9 +802,13 @@ async fn keymanager_create_voluntary_exit( let signature = signer_snapshot .sign_without_slashing_protection( - SigningMessage::VoluntaryExit(voluntary_exit), - voluntary_exit.signing_root(controller.chain_config(), &state), - Some(state.as_ref().into()), + SigningMessage::

::VoluntaryExit(voluntary_exit), + SignForSingleFork::

::signing_root_from_fork_info( + &voluntary_exit, + chain_config, + fork_info, + ), + Some(fork_info), pubkey, ) .await?; @@ -773,11 +835,12 @@ async fn authorize_token( pub async fn run_validator_api( validator_api_config: ValidatorApiConfig, - controller: ApiController, + chain_source: Arc>, directories: Arc, keymanager: Arc, signer: Arc, metrics: Option>, + tracing_handle: Option, ) -> Result<()> { let ValidatorApiConfig { address, @@ -796,26 +859,29 @@ pub async fn run_validator_api( ) }); - let token = ApiToken::load_or_build_from(&token_file_path).map_err(|error| { - TokenLoadError::UnableToLoad { + let token = match ApiToken::load_or_build_from(&token_file_path) { + Ok(token) => token, + Err(error) => bail!(TokenLoadError::UnableToLoad { error, token_file_path, - } - })?; + }), + }; info_with_peers!( - "Validator API is listening on {address}, authorization token: {:?}", - *Zeroizing::new(hex::encode(&token.bytes)), + "Validator API is listening on {address}, authorization token file: {token_file_path:?}", ); let state = ValidatorApiState { - controller, + chain_source, keymanager, token: Arc::new(token), signer, + tracing_handle, }; let router = eth_v1_keymanager_routes() + .route("/eth/v2/debug/tracing/log_level", post(post_log_level)) + .route("/eth/v2/debug/tracing/trace_level", post(post_trace_level)) .layer(axum::middleware::from_fn_with_state( state.clone(), authorize_token, diff --git a/validator/src/beacon_node_api.rs b/validator/src/beacon_node_api.rs new file mode 100644 index 000000000..a7543df68 --- /dev/null +++ b/validator/src/beacon_node_api.rs @@ -0,0 +1,262 @@ +use core::future::Future; +use std::{collections::BTreeMap, sync::Arc}; + +use anyhow::Result; +use block_producer::{ProposerData, ValidatorBlindedBlock}; +use bls::SignatureBytes; +use builder_api::{ + consts::MaxBuilderEntries, + gloas::containers::{BuilderEntry, BuilderPreferencesEntry, BuilderUrl}, + unphased::containers::SignedValidatorRegistrationV1, +}; +use http_api_utils::{ + ValidatorAttesterDutyResponse, ValidatorLivenessResponse, ValidatorPTCDutyResponse, + ValidatorProposerDutyResponse, ValidatorSyncDutyResponse, +}; +use p2p::{BeaconCommitteeSubscription, SyncCommitteeSubscription}; +use ssz::ContiguousList; +use types::{ + altair::{ + containers::{SignedContributionAndProof, SyncCommitteeContribution, SyncCommitteeMessage}, + primitives::SubcommitteeIndex, + }, + combined::{ + Attestation, BeaconBlock, SignedAggregateAndProof, SignedBeaconBlock, + SignedBlindedBeaconBlock, + }, + deneb::primitives::{Blob, KzgProof}, + gloas::containers::{ + ExecutionPayloadEnvelope, PayloadAttestationData, PayloadAttestationMessage, + SignedExecutionPayloadEnvelope, SignedProposerPreferences, + }, + nonstandard::{KzgProofs, OwnAttestation}, + phase0::{ + containers::AttestationData, + primitives::{CommitteeIndex, Epoch, Gwei, H256, Slot, Uint256, ValidatorIndex}, + }, + preset::Preset, +}; + +/// Duties of an epoch with the root they depend on, which a reorg can change. +pub struct Duties { + pub dependent_root: H256, + pub duties: Vec, +} + +pub type AttesterDuties = Duties; +pub type PtcDuties = Duties; +pub type ProposerDuties = Duties; + +pub struct ProducedBlock { + pub block: ValidatorBlindedBlock

, + pub kzg_proofs: Option>, + pub blobs: Option, P::MaxBlobCommitmentsPerBlock>>, + /// The payload of a self-built Gloas block, which its proposer reveals after the block. + pub envelope_contents: Option>, + /// The builder whose bid the block takes, which is sent the signed block when published. + pub builder_url: Option, +} + +impl ProducedBlock

{ + #[must_use] + pub const fn without_blobs(block: ValidatorBlindedBlock

) -> Self { + Self { + block, + kzg_proofs: None, + blobs: None, + envelope_contents: None, + builder_url: None, + } + } + + #[must_use] + pub const fn with_blobs( + block: BeaconBlock

, + kzg_proofs: KzgProofs

, + blobs: ContiguousList, P::MaxBlobCommitmentsPerBlock>, + ) -> Self { + Self { + block: ValidatorBlindedBlock::BeaconBlock(block), + kzg_proofs: Some(kzg_proofs), + blobs: Some(blobs), + envelope_contents: None, + builder_url: None, + } + } +} + +pub struct EnvelopeContents { + pub envelope: ExecutionPayloadEnvelope

, + pub kzg_proofs: ContiguousList, + pub blobs: ContiguousList, P::MaxBlobCommitmentsPerBlock>, +} + +/// A beacon node the validator can perform duties against. +pub trait BeaconNodeApi { + /// + fn liveness( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> impl Future>> + Send; + + fn dependent_root( + &self, + epoch: Epoch, + validator_index: ValidatorIndex, + ) -> impl Future> + Send; + + /// + fn attestation_data( + &self, + slot: Slot, + committee_index: CommitteeIndex, + ) -> impl Future> + Send; + + /// + fn aggregate_attestation( + &self, + data: AttestationData, + committee_index: CommitteeIndex, + ) -> impl Future>> + Send; + + /// + fn publish_singular_attestations( + &self, + attestations: &[OwnAttestation

], + ) -> impl Future> + Send; + + /// + fn publish_aggregates_and_proofs( + &self, + aggregates_and_proofs: &[Arc>], + ) -> impl Future> + Send; + + /// + fn subscribe_to_beacon_committees( + &self, + current_slot: Slot, + subscriptions: &[BeaconCommitteeSubscription], + ) -> impl Future> + Send; + + /// + fn sync_committee_duties( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> impl Future>> + Send; + + // Grouped by subcommittee, as the built-in node gossips on each subcommittee's subnet. + /// + fn publish_sync_committee_messages( + &self, + messages: &BTreeMap>, + ) -> impl Future> + Send; + + /// + fn subscribe_to_sync_committees( + &self, + current_epoch: Epoch, + subscriptions: &[SyncCommitteeSubscription], + ) -> impl Future> + Send; + + /// + fn sync_committee_contribution( + &self, + slot: Slot, + subcommittee_index: SubcommitteeIndex, + beacon_block_root: H256, + ) -> impl Future>> + Send; + + /// + fn publish_contributions_and_proofs( + &self, + contributions_and_proofs: &[SignedContributionAndProof

], + ) -> impl Future> + Send; + + /// + fn ptc_duties( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> impl Future> + Send; + + /// + fn proposer_duties(&self, epoch: Epoch) -> impl Future> + Send; + + /// before Gloas and + /// from it on. + /// + /// `min_bid` and `builders` only apply from Gloas on, when bids are requested from `builders`. + fn produce_block( + &self, + slot: Slot, + randao_reveal: SignatureBytes, + graffiti: Option, + builder_boost_factor: Uint256, + min_bid: Gwei, + builders: &ContiguousList, + ) -> impl Future>> + Send; + + /// + /// + /// `builder_url` is [`ProducedBlock::builder_url`], to which the signed block is forwarded. + fn publish_block( + &self, + signed_block: &Arc>, + kzg_proofs: Option<&KzgProofs

>, + blobs: Option<&ContiguousList, P::MaxBlobCommitmentsPerBlock>>, + builder_url: Option<&BuilderUrl>, + ) -> impl Future> + Send; + + /// + fn publish_blinded_block( + &self, + signed_block: &SignedBlindedBeaconBlock

, + ) -> impl Future> + Send; + + /// + fn publish_execution_payload_envelope( + &self, + signed_envelope: &Arc>, + kzg_proofs: &ContiguousList, + blobs: &ContiguousList, P::MaxBlobCommitmentsPerBlock>, + ) -> impl Future> + Send; + + /// + fn payload_attestation_data( + &self, + slot: Slot, + ) -> impl Future>> + Send; + + /// + fn publish_payload_attestations( + &self, + messages: &[Arc], + ) -> impl Future> + Send; + + /// + fn prepare_beacon_proposer( + &self, + current_slot: Slot, + proposers: &[ProposerData], + ) -> impl Future> + Send; + + /// + fn register_validators( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> impl Future> + Send; + + /// + fn publish_proposer_preferences( + &self, + preferences: &[Arc], + ) -> impl Future> + Send; + + /// + fn submit_builder_preferences( + &self, + entries: &[BuilderPreferencesEntry], + ) -> impl Future> + Send; +} diff --git a/validator/src/beacon_nodes.rs b/validator/src/beacon_nodes.rs new file mode 100644 index 000000000..345c1fa5b --- /dev/null +++ b/validator/src/beacon_nodes.rs @@ -0,0 +1,1619 @@ +use core::{fmt::Display, future::Future, ops::Range, ptr}; +use std::{ + collections::BTreeMap, + sync::{Arc, OnceLock}, +}; + +use anyhow::{Error, Result, anyhow, bail, ensure}; +use block_producer::ProposerData; +use bls::SignatureBytes; +use builder_api::{ + consts::MaxBuilderEntries, + gloas::containers::{BuilderEntry, BuilderPreferencesEntry, BuilderUrl}, + unphased::containers::SignedValidatorRegistrationV1, +}; +use fork_choice_control::Wait; +use futures::{StreamExt as _, future::join_all, stream::FuturesUnordered}; +use http_api_utils::{ValidatorLivenessResponse, ValidatorSyncDutyResponse}; +use logging::{debug_with_peers, warn_with_peers}; +use p2p::{BeaconCommitteeSubscription, SyncCommitteeSubscription}; +use ssz::ContiguousList; +use std_ext::ArcExt; +use types::{ + altair::{ + containers::{SignedContributionAndProof, SyncCommitteeContribution, SyncCommitteeMessage}, + primitives::SubcommitteeIndex, + }, + combined::{ + Attestation, BeaconState, SignedAggregateAndProof, SignedBeaconBlock, + SignedBlindedBeaconBlock, + }, + deneb::primitives::{Blob, KzgProof}, + gloas::containers::{ + PayloadAttestationData, PayloadAttestationMessage, SignedExecutionPayloadEnvelope, + SignedProposerPreferences, + }, + nonstandard::{KzgProofs, OwnAttestation, PublishedDuty}, + phase0::{ + containers::AttestationData, + primitives::{CommitteeIndex, Epoch, Gwei, H256, Slot, Uint256, ValidatorIndex}, + }, + preset::Preset, +}; + +use crate::{ + beacon_node_api::{AttesterDuties, BeaconNodeApi, ProducedBlock, ProposerDuties, PtcDuties}, + local_beacon_node::LocalBeaconNode, + misc, + remote_beacon_node::RemoteBeaconNode, + remote_beacon_nodes::RemoteBeaconNodes, + slot_head::SlotHead, +}; + +/// The beacon nodes a slot's duties are performed against, at the head the slot's duties are +/// performed on: the built-in node with its state, or the serving remote ones when there is none. +#[derive(Clone)] +pub enum BeaconNodes { + Local(LocalBeaconNode), + Remote { + nodes: RemoteNodes, + slot_head: SlotHead

, + }, +} + +/// The remote beacon nodes serving at a slot, in health order. +#[derive(Clone)] +pub struct RemoteNodes { + nodes: Vec>, + current_slot: Slot, + max_empty_slots: u64, + publish_to_every_node: Vec, + /// The node that produced the block of the slot, which alone holds its payload. + producer: OnceLock>, +} + +impl BeaconNodes { + #[must_use] + pub const fn has_local_node(&self) -> bool { + matches!(self, Self::Local(_)) + } + + #[must_use] + pub const fn head(&self) -> &SlotHead

{ + match self { + Self::Local(node) => node.head(), + Self::Remote { slot_head, .. } => slot_head, + } + } + + #[must_use] + pub const fn state(&self) -> Option<&Arc>> { + match self { + Self::Local(node) => Some(node.state()), + Self::Remote { .. } => None, + } + } + + #[must_use] + pub fn prefetch_slots(&self, current_slot: Slot) -> Range { + match self { + Self::Local(_) => misc::slots_to_compute_in_advance(current_slot), + // A remote node answers a whole epoch per request, so nothing is gained by asking + // for less than the epochs in flight. + Self::Remote { .. } => { + let current_epoch = + helper_functions::misc::compute_epoch_at_slot::

(current_slot); + + helper_functions::misc::compute_start_slot_at_epoch::

(current_epoch) + ..helper_functions::misc::compute_start_slot_at_epoch::

( + current_epoch.saturating_add(2), + ) + } + } + } + + pub async fn attester_duties_at_slots( + &self, + slots: Range, + validator_indices: &[ValidatorIndex], + ) -> Result { + match self { + Self::Local(node) => node.attester_duties_at_slots(slots, validator_indices), + Self::Remote { nodes: remotes, .. } => { + remotes + .attester_duties_at_slots::

(slots, validator_indices) + .await + } + } + } + + pub async fn head_block_root(&self) -> Result { + match self { + Self::Local(node) => Ok(node.head_block_root()), + Self::Remote { nodes: remotes, .. } => remotes.head_block_root().await, + } + } +} + +impl RemoteNodes { + pub fn new(remotes: &RemoteBeaconNodes, current_slot: Slot, max_empty_slots: u64) -> Self { + Self { + nodes: remotes.serving().map(ArcExt::clone_arc).collect(), + current_slot, + max_empty_slots, + publish_to_every_node: remotes.publish_to_every_node().to_vec(), + producer: OnceLock::new(), + } + } + + pub fn led_by(mut self, node: &Arc) -> Option { + let position = self + .nodes + .iter() + .position(|serving| Arc::ptr_eq(serving, node))?; + + self.nodes[..=position].rotate_right(1); + Some(self) + } + + fn serving_nodes(&self) -> Vec> { + self.nodes + .iter() + .filter(|node| node.health().can_serve()) + .map(ArcExt::clone_arc) + .collect() + } + + async fn attester_duties_at_slots( + &self, + slots: Range, + validator_indices: &[ValidatorIndex], + ) -> Result { + // `slots` must lie within one epoch, as the duties carry a single dependent root. + let epoch = helper_functions::misc::compute_epoch_at_slot::

(slots.start); + let operation = format!("produce attester duties for epoch {epoch}"); + + let (node, duties) = self + .first_success(&operation, |node| { + node.attester_duties(epoch, validator_indices) + }) + .await?; + + debug_with_peers!( + "{node} beacon node produced {} attester duties for epoch {epoch} \ + under dependent root {:?}", + duties.duties.len(), + duties.dependent_root, + ); + + Ok(duties) + } + + async fn first_success<'this, T, Fut>( + &'this self, + operation: &str, + attempt: impl Fn(&'this RemoteBeaconNode) -> Fut, + ) -> Result<(&'this Arc, T)> + where + Fut: Future>, + { + let mut attempts = Vec::with_capacity(self.nodes.len()); + + for node in &self.nodes { + attempts.push((node, attempt(node.as_ref()))); + } + + first_success(operation, attempts).await + } + + async fn head_block_root(&self) -> Result { + let operation = "produce the head block root"; + + // The head of the earliest node holding one, so that the root signed for is one it knows. + for node in &self.nodes { + if let Ok(Some(block_root)) = node + .chain_head() + .get(self.current_slot, self.max_empty_slots) + { + return Ok(block_root); + } + } + + let (node, root) = self + .first_success(operation, |node| { + node.fresh_head_block_root(self.current_slot) + }) + .await?; + + debug_with_peers!("{node} beacon node reported head block root {root:?}"); + + Ok(root) + } + + pub fn slot_head(&self, slot: Slot) -> Option> { + for node in &self.nodes { + match node.slot_head::

(slot) { + Ok(Some(slot_head)) => return Some(slot_head), + Ok(None) => {} + Err(error) => { + warn_with_peers!("{node} beacon node reported an unusable head: {error:?}"); + } + } + } + + None + } + + fn producer_first(&self) -> Vec> { + let producer = self.producer.get(); + + // The producer goes first, as it is known to be able to import what it produced. + producer + .map(ArcExt::clone_arc) + .into_iter() + .chain( + self.serving_nodes() + .into_iter() + .filter(|node| producer.is_none_or(|producer| !Arc::ptr_eq(node, producer))), + ) + .collect() + } + + fn should_publish_to_every_node(&self, duty: Option) -> bool { + duty.is_none_or(|duty| should_publish_to_every_node(&self.publish_to_every_node, duty)) + } + + fn spawn_publish( + &self, + operation: &'static str, + duty: Option, + items: &T, + attempt: F, + ) where + T: ToOwned + ?Sized + 'static, + T::Owned: Send + Sync + 'static, + F: Fn(Arc, Arc) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + let publishing = self.publish(operation, duty, items, attempt); + + tokio::spawn(async move { + if let Err(error) = publishing.await { + warn_with_peers!("{error:?}"); + } + }); + } + + fn publish( + &self, + operation: &'static str, + duty: Option, + items: &T, + attempt: F, + ) -> impl Future> + Send + use + where + T: ToOwned + ?Sized + 'static, + T::Owned: Send + Sync + 'static, + F: Fn(Arc, Arc) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + let publish_to = self.serving_nodes(); + let to_every_node = self.should_publish_to_every_node(duty); + let items = Arc::new(items.to_owned()); + let attempt = move |node| attempt(node, items.clone_arc()); + + publish(operation, publish_to, to_every_node, attempt) + } +} + +impl BeaconNodeApi

for RemoteNodes { + async fn liveness( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> Result> { + // A validator seen live by any node is live. + let mut liveness = validator_indices + .iter() + .map(|validator_index| (*validator_index, false)) + .collect::>(); + + let mut any_success = false; + + // A node that is behind has not seen the attestations and reports nobody live. + let ready_nodes = self.nodes.iter().filter(|node| node.health().is_ready()); + + let responses = join_all(ready_nodes.map(|node| async move { + let result = BeaconNodeApi::

::liveness(node.as_ref(), epoch, validator_indices); + (result.await, node) + })) + .await; + + for (result, node) in responses { + match result { + Ok(response) => { + any_success = true; + + for entry in response { + if entry.is_live + && let Some(live) = liveness.get_mut(&entry.index) + { + *live = true; + } + } + } + Err(error) => warn_with_peers!( + "{node} beacon node failed to report liveness for epoch {epoch}: {error:?}", + ), + } + } + + ensure!( + any_success, + "no synced beacon node could report validator liveness for epoch {epoch}", + ); + + Ok(liveness + .into_iter() + .map(|(index, is_live)| ValidatorLivenessResponse { index, is_live }) + .collect()) + } + + async fn dependent_root(&self, epoch: Epoch, validator_index: ValidatorIndex) -> Result { + // A head event already reported the root, sparing the request it would take to ask. + for node in &self.nodes { + if let Some(dependent_root) = node.chain_head().dependent_root_for(epoch) { + return Ok(dependent_root); + } + } + + let operation = format!("produce the dependent root of epoch {epoch}"); + + self.first_success(&operation, |node| { + BeaconNodeApi::

::dependent_root(node, epoch, validator_index) + }) + .await + .map(|(_, dependent_root)| dependent_root) + } + + async fn attestation_data( + &self, + slot: Slot, + committee_index: CommitteeIndex, + ) -> Result { + let operation = format!("produce attestation data for slot {slot}"); + + let (node, data) = self + .first_success(&operation, |node| { + BeaconNodeApi::

::attestation_data(node, slot, committee_index) + }) + .await?; + + if !node.health().is_ready() { + debug_with_peers!( + "attestation data for slot {slot} came from {node}, which is not fully synced", + ); + } + + debug_with_peers!("{node} beacon node produced attestation data for slot {slot}: {data:?}"); + + Ok(data) + } + + async fn aggregate_attestation( + &self, + data: AttestationData, + committee_index: CommitteeIndex, + ) -> Result> { + let operation = format!( + "produce an aggregate attestation for committee {committee_index} in slot {}", + data.slot, + ); + + self.first_success(&operation, |node| { + BeaconNodeApi::

::aggregate_attestation(node, data, committee_index) + }) + .await + .map(|(_, aggregate)| aggregate) + } + + async fn publish_singular_attestations( + &self, + attestations: &[OwnAttestation

], + ) -> Result<()> { + self.spawn_publish( + "publish attestations", + Some(PublishedDuty::Attestations), + attestations, + move |node, attestations| async move { + BeaconNodeApi::

::publish_singular_attestations(node.as_ref(), &attestations) + .await + }, + ); + + Ok(()) + } + + async fn publish_aggregates_and_proofs( + &self, + aggregates_and_proofs: &[Arc>], + ) -> Result<()> { + self.spawn_publish( + "publish aggregates and proofs", + Some(PublishedDuty::Aggregates), + aggregates_and_proofs, + move |node, aggregates_and_proofs| async move { + BeaconNodeApi::

::publish_aggregates_and_proofs( + node.as_ref(), + &aggregates_and_proofs, + ) + .await + }, + ); + + Ok(()) + } + + async fn subscribe_to_beacon_committees( + &self, + current_slot: Slot, + subscriptions: &[BeaconCommitteeSubscription], + ) -> Result<()> { + self.spawn_publish( + "update beacon committee subscriptions", + None, + subscriptions, + move |node, subscriptions| async move { + BeaconNodeApi::

::subscribe_to_beacon_committees( + node.as_ref(), + current_slot, + &subscriptions, + ) + .await + }, + ); + + Ok(()) + } + + async fn sync_committee_duties( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> Result> { + let operation = format!("produce sync committee duties for epoch {epoch}"); + + let (node, duties) = self + .first_success(&operation, |node| { + BeaconNodeApi::

::sync_committee_duties(node, epoch, validator_indices) + }) + .await?; + + debug_with_peers!( + "{node} beacon node produced {} sync committee duties for epoch {epoch}", + duties.len(), + ); + + Ok(duties) + } + + async fn publish_sync_committee_messages( + &self, + messages: &BTreeMap>, + ) -> Result<()> { + self.spawn_publish( + "publish sync committee messages", + Some(PublishedDuty::SyncCommitteeMessages), + messages, + move |node, messages| async move { + BeaconNodeApi::

::publish_sync_committee_messages(node.as_ref(), &messages).await + }, + ); + + Ok(()) + } + + async fn subscribe_to_sync_committees( + &self, + current_epoch: Epoch, + subscriptions: &[SyncCommitteeSubscription], + ) -> Result<()> { + // Every node is told, as any of them may be asked to contribute later. + self.publish( + "update sync committee subscriptions", + None, + subscriptions, + move |node, subscriptions| async move { + BeaconNodeApi::

::subscribe_to_sync_committees( + node.as_ref(), + current_epoch, + &subscriptions, + ) + .await + }, + ) + .await + } + + async fn sync_committee_contribution( + &self, + slot: Slot, + subcommittee_index: SubcommitteeIndex, + beacon_block_root: H256, + ) -> Result> { + let operation = format!( + "produce a sync committee contribution for subcommittee {subcommittee_index} \ + in slot {slot}", + ); + + self.first_success(&operation, |node| { + BeaconNodeApi::

::sync_committee_contribution( + node, + slot, + subcommittee_index, + beacon_block_root, + ) + }) + .await + .map(|(_, contribution)| contribution) + } + + async fn publish_contributions_and_proofs( + &self, + contributions_and_proofs: &[SignedContributionAndProof

], + ) -> Result<()> { + self.spawn_publish( + "publish contributions and proofs", + Some(PublishedDuty::SyncCommitteeContributions), + contributions_and_proofs, + move |node, contributions_and_proofs| async move { + BeaconNodeApi::

::publish_contributions_and_proofs( + node.as_ref(), + &contributions_and_proofs, + ) + .await + }, + ); + + Ok(()) + } + + async fn ptc_duties( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> Result { + let operation = format!("produce PTC duties for epoch {epoch}"); + + let (node, duties) = self + .first_success(&operation, |node| { + BeaconNodeApi::

::ptc_duties(node, epoch, validator_indices) + }) + .await?; + + debug_with_peers!( + "{node} beacon node produced {} PTC duties for epoch {epoch} \ + under dependent root {:?}", + duties.duties.len(), + duties.dependent_root, + ); + + Ok(duties) + } + + async fn proposer_duties(&self, epoch: Epoch) -> Result { + let operation = format!("produce proposer duties for epoch {epoch}"); + + let (node, duties) = self + .first_success(&operation, |node| { + BeaconNodeApi::

::proposer_duties(node, epoch) + }) + .await?; + + debug_with_peers!( + "{node} beacon node produced {} proposer duties for epoch {epoch} \ + under dependent root {:?}", + duties.duties.len(), + duties.dependent_root, + ); + + Ok(duties) + } + + async fn produce_block( + &self, + slot: Slot, + randao_reveal: SignatureBytes, + graffiti: Option, + builder_boost_factor: Uint256, + min_bid: Gwei, + builders: &ContiguousList, + ) -> Result> { + // A node whose view differs from the fleet's would build on a parent the others reject; + // without a known head there is nothing to judge by, and any block beats a missed slot. + let parent_root = match self.head_block_root().await { + Ok(parent_root) => Some(parent_root), + Err(error) => { + warn_with_peers!("producing a block without a known head: {error:?}"); + None + } + }; + + // The cached head can lag a block every node already has; then no block matches it and + // the first one built is still better than a missed slot. + let fallback = OnceLock::new(); + let stash = &fallback; + + let attempt = self + .first_success("produce a block", |node| async move { + let produced_block = BeaconNodeApi::

::produce_block( + node, + slot, + randao_reveal, + graffiti, + builder_boost_factor, + min_bid, + builders, + ) + .await?; + + let built_on = produced_block.block.parent_root(); + + // A node sharing the fleet's head still builds on the grandparent when it reorgs + // a late parent; only a node whose head differs is passed over. + let shares_head = |parent_root| { + node.chain_head() + .cached() + .is_some_and(|(_, head)| head == parent_root) + }; + + if let Some(parent_root) = parent_root + && built_on != parent_root + && !shares_head(parent_root) + { + if stash.set((node, produced_block)).is_err() { + debug_with_peers!( + "the block {node} built on {built_on:?} is not kept as the fallback; \ + an earlier node's block already is", + ); + } + + bail!("{node} built on {built_on:?} rather than the head {parent_root:?}"); + } + + Ok(produced_block) + }) + .await; + + let (producer, block) = match attempt { + Ok((producer, block)) => (Some(producer), block), + Err(error) => { + let (Some(parent_root), Some((node, block))) = (parent_root, fallback.into_inner()) + else { + return Err(error); + }; + + warn_with_peers!( + "no beacon node built on the head {parent_root:?}; proposing the block \ + {node} built on {:?}", + block.block.parent_root(), + ); + + let producer = self + .nodes + .iter() + .find(|candidate| ptr::eq(candidate.as_ref(), node)); + + (producer, block) + } + }; + + if let Some(producer) = producer { + self.producer.get_or_init(|| producer.clone_arc()); + } + + Ok(block) + } + + async fn publish_block( + &self, + signed_block: &Arc>, + kzg_proofs: Option<&KzgProofs

>, + blobs: Option<&ContiguousList, P::MaxBlobCommitmentsPerBlock>>, + builder_url: Option<&BuilderUrl>, + ) -> Result<()> { + let signed_block = signed_block.clone_arc(); + let kzg_proofs = Arc::new(kzg_proofs.cloned()); + let blobs = Arc::new(blobs.cloned()); + let builder_url = Arc::new(builder_url.cloned()); + + let operation = "publish a block"; + let to_every_node = self.should_publish_to_every_node(Some(PublishedDuty::Blocks)); + + publish( + operation, + self.producer_first(), + to_every_node, + move |node| { + let signed_block = signed_block.clone_arc(); + let kzg_proofs = kzg_proofs.clone_arc(); + let blobs = blobs.clone_arc(); + let builder_url = builder_url.clone_arc(); + + async move { + BeaconNodeApi::

::publish_block( + node.as_ref(), + &signed_block, + kzg_proofs.as_ref().as_ref(), + blobs.as_ref().as_ref(), + builder_url.as_ref().as_ref(), + ) + .await + } + }, + ) + .await + } + + async fn publish_blinded_block( + &self, + signed_block: &SignedBlindedBeaconBlock

, + ) -> Result<()> { + let signed_block = Arc::new(signed_block.clone()); + + // Only the producer holds the header to unblind it, so no other node is asked. + let producer = self + .producer + .get() + .map(ArcExt::clone_arc) + .into_iter() + .collect(); + + publish("publish a blinded block", producer, false, move |node| { + let signed_block = signed_block.clone_arc(); + + async move { + BeaconNodeApi::

::publish_blinded_block(node.as_ref(), &signed_block).await + } + }) + .await + } + + async fn publish_execution_payload_envelope( + &self, + signed_envelope: &Arc>, + kzg_proofs: &ContiguousList, + blobs: &ContiguousList, P::MaxBlobCommitmentsPerBlock>, + ) -> Result<()> { + let signed_envelope = signed_envelope.clone_arc(); + let kzg_proofs = Arc::new(kzg_proofs.clone()); + let blobs = Arc::new(blobs.clone()); + + let operation = "publish an execution payload envelope"; + + // Only the producer is sure to hold the block by now; the others may still be importing + // it and would refuse its envelope, so they are left to gossip unless the producer fails. + publish(operation, self.producer_first(), false, move |node| { + let signed_envelope = signed_envelope.clone_arc(); + let kzg_proofs = kzg_proofs.clone_arc(); + let blobs = blobs.clone_arc(); + + async move { + BeaconNodeApi::

::publish_execution_payload_envelope( + node.as_ref(), + &signed_envelope, + &kzg_proofs, + &blobs, + ) + .await + } + }) + .await + } + + async fn payload_attestation_data(&self, slot: Slot) -> Result> { + let operation = format!("produce payload attestation data for slot {slot}"); + + // A node that has seen no block only settles the slot once every node agrees; a node that + // has fallen behind never answers with data. + let mut none_seen = false; + + for node in &self.nodes { + match BeaconNodeApi::

::payload_attestation_data(node.as_ref(), slot).await { + Ok(Some(data)) => { + debug_with_peers!( + "{node} beacon node produced payload attestation data \ + for slot {slot}: {data:?}", + ); + return Ok(Some(data)); + } + Ok(None) => none_seen = true, + Err(error) => { + warn_with_peers!("{node} beacon node failed to {operation}: {error:?}"); + } + } + } + + if none_seen { + return Ok(None); + } + + bail!("no beacon node was able to {operation}") + } + + async fn publish_payload_attestations( + &self, + messages: &[Arc], + ) -> Result<()> { + self.spawn_publish( + "publish payload attestations", + Some(PublishedDuty::PayloadAttestations), + messages, + move |node, messages| async move { + BeaconNodeApi::

::publish_payload_attestations(node.as_ref(), &messages).await + }, + ); + + Ok(()) + } + + async fn prepare_beacon_proposer( + &self, + current_slot: Slot, + proposers: &[ProposerData], + ) -> Result<()> { + self.spawn_publish( + "prepare beacon proposers", + None, + proposers, + move |node, proposers| async move { + BeaconNodeApi::

::prepare_beacon_proposer(node.as_ref(), current_slot, &proposers) + .await + }, + ); + + Ok(()) + } + + async fn register_validators( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> Result<()> { + // Awaited so that the caller's chunks reach the relay one at a time. + self.publish( + "register validators", + None, + registrations, + move |node, registrations| async move { + BeaconNodeApi::

::register_validators(node.as_ref(), ®istrations).await + }, + ) + .await + } + + async fn publish_proposer_preferences( + &self, + preferences: &[Arc], + ) -> Result<()> { + self.publish( + "publish proposer preferences", + Some(PublishedDuty::ProposerPreferences), + preferences, + move |node, preferences| async move { + BeaconNodeApi::

::publish_proposer_preferences(node.as_ref(), &preferences).await + }, + ) + .await + } + + async fn submit_builder_preferences(&self, entries: &[BuilderPreferencesEntry]) -> Result<()> { + // The node passes the preferences on to the builders, so one node taking them is enough. + self.first_success("submit builder preferences", |node| async move { + BeaconNodeApi::

::submit_builder_preferences(node, entries).await + }) + .await + .map(|_| ()) + } +} + +impl BeaconNodeApi

for BeaconNodes { + async fn liveness( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> Result> { + match self { + Self::Local(node) => node.liveness(epoch, validator_indices).await, + Self::Remote { nodes: remotes, .. } => { + BeaconNodeApi::

::liveness(remotes, epoch, validator_indices).await + } + } + } + + async fn dependent_root(&self, epoch: Epoch, validator_index: ValidatorIndex) -> Result { + match self { + Self::Local(node) => node.dependent_root(epoch, validator_index).await, + Self::Remote { nodes: remotes, .. } => { + BeaconNodeApi::

::dependent_root(remotes, epoch, validator_index).await + } + } + } + + async fn attestation_data( + &self, + slot: Slot, + committee_index: CommitteeIndex, + ) -> Result { + match self { + Self::Local(node) => node.attestation_data(slot, committee_index).await, + Self::Remote { nodes: remotes, .. } => { + BeaconNodeApi::

::attestation_data(remotes, slot, committee_index).await + } + } + } + + async fn aggregate_attestation( + &self, + data: AttestationData, + committee_index: CommitteeIndex, + ) -> Result> { + match self { + Self::Local(node) => node.aggregate_attestation(data, committee_index).await, + Self::Remote { nodes: remotes, .. } => { + remotes.aggregate_attestation(data, committee_index).await + } + } + } + + async fn publish_singular_attestations( + &self, + attestations: &[OwnAttestation

], + ) -> Result<()> { + match self { + Self::Local(node) => node.publish_singular_attestations(attestations).await, + Self::Remote { nodes: remotes, .. } => { + remotes.publish_singular_attestations(attestations).await + } + } + } + + async fn publish_aggregates_and_proofs( + &self, + aggregates_and_proofs: &[Arc>], + ) -> Result<()> { + match self { + Self::Local(node) => { + node.publish_aggregates_and_proofs(aggregates_and_proofs) + .await + } + Self::Remote { nodes: remotes, .. } => { + remotes + .publish_aggregates_and_proofs(aggregates_and_proofs) + .await + } + } + } + + async fn subscribe_to_beacon_committees( + &self, + current_slot: Slot, + subscriptions: &[BeaconCommitteeSubscription], + ) -> Result<()> { + if subscriptions.is_empty() { + return Ok(()); + } + + match self { + Self::Local(node) => { + node.subscribe_to_beacon_committees(current_slot, subscriptions) + .await + } + Self::Remote { nodes: remotes, .. } => { + BeaconNodeApi::

::subscribe_to_beacon_committees( + remotes, + current_slot, + subscriptions, + ) + .await + } + } + } + + async fn sync_committee_duties( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> Result> { + match self { + Self::Local(node) => node.sync_committee_duties(epoch, validator_indices).await, + Self::Remote { nodes: remotes, .. } => { + BeaconNodeApi::

::sync_committee_duties(remotes, epoch, validator_indices).await + } + } + } + + async fn publish_sync_committee_messages( + &self, + messages: &BTreeMap>, + ) -> Result<()> { + match self { + Self::Local(node) => node.publish_sync_committee_messages(messages).await, + Self::Remote { nodes: remotes, .. } => { + BeaconNodeApi::

::publish_sync_committee_messages(remotes, messages).await + } + } + } + + async fn subscribe_to_sync_committees( + &self, + current_epoch: Epoch, + subscriptions: &[SyncCommitteeSubscription], + ) -> Result<()> { + if subscriptions.is_empty() { + return Ok(()); + } + + match self { + Self::Local(node) => { + node.subscribe_to_sync_committees(current_epoch, subscriptions) + .await + } + Self::Remote { nodes: remotes, .. } => { + BeaconNodeApi::

::subscribe_to_sync_committees( + remotes, + current_epoch, + subscriptions, + ) + .await + } + } + } + + async fn sync_committee_contribution( + &self, + slot: Slot, + subcommittee_index: SubcommitteeIndex, + beacon_block_root: H256, + ) -> Result> { + match self { + Self::Local(node) => { + node.sync_committee_contribution(slot, subcommittee_index, beacon_block_root) + .await + } + Self::Remote { nodes: remotes, .. } => { + remotes + .sync_committee_contribution(slot, subcommittee_index, beacon_block_root) + .await + } + } + } + + async fn publish_contributions_and_proofs( + &self, + contributions_and_proofs: &[SignedContributionAndProof

], + ) -> Result<()> { + match self { + Self::Local(node) => { + node.publish_contributions_and_proofs(contributions_and_proofs) + .await + } + Self::Remote { nodes: remotes, .. } => { + remotes + .publish_contributions_and_proofs(contributions_and_proofs) + .await + } + } + } + + async fn ptc_duties( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> Result { + match self { + Self::Local(node) => node.ptc_duties(epoch, validator_indices).await, + Self::Remote { nodes: remotes, .. } => { + BeaconNodeApi::

::ptc_duties(remotes, epoch, validator_indices).await + } + } + } + + async fn proposer_duties(&self, epoch: Epoch) -> Result { + match self { + Self::Local(node) => node.proposer_duties(epoch).await, + Self::Remote { nodes: remotes, .. } => { + BeaconNodeApi::

::proposer_duties(remotes, epoch).await + } + } + } + + async fn produce_block( + &self, + slot: Slot, + randao_reveal: SignatureBytes, + graffiti: Option, + builder_boost_factor: Uint256, + min_bid: Gwei, + builders: &ContiguousList, + ) -> Result> { + match self { + Self::Local(node) => { + node.produce_block( + slot, + randao_reveal, + graffiti, + builder_boost_factor, + min_bid, + builders, + ) + .await + } + Self::Remote { nodes: remotes, .. } => { + remotes + .produce_block( + slot, + randao_reveal, + graffiti, + builder_boost_factor, + min_bid, + builders, + ) + .await + } + } + } + + async fn publish_block( + &self, + signed_block: &Arc>, + kzg_proofs: Option<&KzgProofs

>, + blobs: Option<&ContiguousList, P::MaxBlobCommitmentsPerBlock>>, + builder_url: Option<&BuilderUrl>, + ) -> Result<()> { + match self { + Self::Local(node) => { + node.publish_block(signed_block, kzg_proofs, blobs, builder_url) + .await + } + Self::Remote { nodes: remotes, .. } => { + remotes + .publish_block(signed_block, kzg_proofs, blobs, builder_url) + .await + } + } + } + + async fn publish_blinded_block( + &self, + signed_block: &SignedBlindedBeaconBlock

, + ) -> Result<()> { + match self { + Self::Local(node) => node.publish_blinded_block(signed_block).await, + Self::Remote { nodes: remotes, .. } => { + remotes.publish_blinded_block(signed_block).await + } + } + } + + async fn publish_execution_payload_envelope( + &self, + signed_envelope: &Arc>, + kzg_proofs: &ContiguousList, + blobs: &ContiguousList, P::MaxBlobCommitmentsPerBlock>, + ) -> Result<()> { + match self { + Self::Local(node) => { + node.publish_execution_payload_envelope(signed_envelope, kzg_proofs, blobs) + .await + } + Self::Remote { nodes: remotes, .. } => { + remotes + .publish_execution_payload_envelope(signed_envelope, kzg_proofs, blobs) + .await + } + } + } + + async fn payload_attestation_data(&self, slot: Slot) -> Result> { + match self { + Self::Local(node) => node.payload_attestation_data(slot).await, + Self::Remote { nodes: remotes, .. } => { + BeaconNodeApi::

::payload_attestation_data(remotes, slot).await + } + } + } + + async fn publish_payload_attestations( + &self, + messages: &[Arc], + ) -> Result<()> { + match self { + Self::Local(node) => node.publish_payload_attestations(messages).await, + Self::Remote { nodes: remotes, .. } => { + BeaconNodeApi::

::publish_payload_attestations(remotes, messages).await + } + } + } + + async fn prepare_beacon_proposer( + &self, + current_slot: Slot, + proposers: &[ProposerData], + ) -> Result<()> { + if proposers.is_empty() { + return Ok(()); + } + + match self { + Self::Local(node) => node.prepare_beacon_proposer(current_slot, proposers).await, + Self::Remote { nodes: remotes, .. } => { + BeaconNodeApi::

::prepare_beacon_proposer(remotes, current_slot, proposers).await + } + } + } + + async fn register_validators( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> Result<()> { + if registrations.is_empty() { + return Ok(()); + } + + match self { + Self::Local(node) => node.register_validators(registrations).await, + Self::Remote { nodes: remotes, .. } => { + BeaconNodeApi::

::register_validators(remotes, registrations).await + } + } + } + + async fn publish_proposer_preferences( + &self, + preferences: &[Arc], + ) -> Result<()> { + match self { + Self::Local(node) => node.publish_proposer_preferences(preferences).await, + Self::Remote { nodes: remotes, .. } => { + BeaconNodeApi::

::publish_proposer_preferences(remotes, preferences).await + } + } + } + + async fn submit_builder_preferences(&self, entries: &[BuilderPreferencesEntry]) -> Result<()> { + match self { + Self::Local(node) => node.submit_builder_preferences(entries).await, + Self::Remote { nodes: remotes, .. } => { + BeaconNodeApi::

::submit_builder_preferences(remotes, entries).await + } + } + } +} + +async fn publish( + operation: &'static str, + remotes: Vec>, + to_every_node: bool, + attempt: F, +) -> Result<()> +where + F: Fn(Arc) -> Fut, + Fut: Future> + Send + 'static, +{ + if to_every_node { + return broadcast(operation, remotes, attempt).await; + } + + let mut failures = Vec::new(); + + for node in remotes { + match attempt(node.clone_arc()).await { + Ok(()) => return Ok(()), + Err(error) => { + log_publish_failure(&node, operation, &error); + failures.push((node, error)); + } + } + } + + Err(publish_error(operation, &failures)) +} + +async fn broadcast( + operation: &'static str, + remotes: Vec>, + attempt: F, +) -> Result<()> +where + F: Fn(Arc) -> Fut, + Fut: Future> + Send + 'static, +{ + // Spawned so that a node slow to answer is still published to after another has accepted. + let mut attempts = remotes + .into_iter() + .map(|node| { + let attempt = attempt(node.clone_arc()); + + tokio::spawn(async move { + attempt.await.map_err(|error| { + log_publish_failure(&node, operation, &error); + (node, error) + }) + }) + }) + .collect::>(); + + let mut failures = Vec::new(); + + while let Some(result) = attempts.next().await { + match result { + Ok(Ok(())) => return Ok(()), + Ok(Err(failure)) => failures.push(failure), + Err(error) => warn_with_peers!("task to {operation} failed: {error:?}"), + } + } + + Err(publish_error(operation, &failures)) +} + +fn log_publish_failure(node: &RemoteBeaconNode, operation: &str, error: &Error) { + debug_with_peers!("{node} beacon node failed to {operation}: {error:?}"); +} + +fn publish_error(operation: &str, failures: &[(Arc, Error)]) -> Error { + let reasons = failures + .iter() + .map(|(node, error)| format!("{node}: {error}")) + .collect::>() + .join("; "); + + anyhow!("no remote beacon node was able to {operation} ({reasons})") +} + +pub async fn first_success>>( + operation: &str, + attempts: impl IntoIterator, +) -> Result<(N, T)> { + for (name, attempt) in attempts { + match attempt.await { + Ok(value) => return Ok((name, value)), + Err(error) => { + warn_with_peers!("{name} beacon node failed to {operation}: {error:?}"); + } + } + } + + bail!("no beacon node was able to {operation}") +} + +fn should_publish_to_every_node(configured: &[PublishedDuty], duty: PublishedDuty) -> bool { + configured + .iter() + .any(|configured| matches!(configured, PublishedDuty::All) || *configured == duty) +} + +#[cfg(test)] +mod tests { + use core::{ + iter::once, + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, + }; + + use anyhow::ensure; + use axum::{Json, Router}; + use reqwest::Client; + use serde_json::json; + use tokio::{net::TcpListener, time::sleep}; + use types::{config::Config as ChainConfig, preset::Mainnet}; + + use super::*; + use crate::health::Health; + + // A head streamed by a node that cannot serve duties is not acted on. + #[tokio::test] + async fn a_head_from_a_node_that_is_not_serving_is_ignored() -> Result<()> { + let node = Arc::new(RemoteBeaconNode::new( + Arc::new(ChainConfig::mainnet()), + Client::new(), + "http://unusable".parse()?, + 32, + )); + + node.set_health(Health::Incompatible); + + let remotes = RemoteBeaconNodes::new(vec![node.clone_arc()], vec![], false); + + assert!(RemoteNodes::new(&remotes, 0, 32).led_by(&node).is_none()); + + Ok(()) + } + + // The streaming node leads, and the others keep their health order behind it. + #[test] + fn led_by_moves_only_the_leading_node() -> Result<()> { + let node = |url: &str| -> Result<_> { + Ok(Arc::new(RemoteBeaconNode::new( + Arc::new(ChainConfig::mainnet()), + Client::new(), + url.parse()?, + 32, + ))) + }; + + let (first, second, third) = (node("http://a")?, node("http://b")?, node("http://c")?); + let remotes = RemoteBeaconNodes::new( + vec![first.clone_arc(), second.clone_arc(), third.clone_arc()], + vec![], + false, + ); + + let nodes = RemoteNodes::new(&remotes, 0, 32) + .led_by(&third) + .expect("a serving node can lead"); + + let expected = [third, first, second]; + + assert!( + nodes + .nodes + .iter() + .zip(&expected) + .all(|(actual, expected)| Arc::ptr_eq(actual, expected)) + ); + + Ok(()) + } + + // Only a synced node has seen the epoch's attestations, so a lagging one is never asked. + #[tokio::test] + async fn liveness_is_not_asked_of_a_node_that_is_not_ready() -> Result<()> { + let requests = Arc::new(AtomicUsize::new(0)); + let counted_requests = requests.clone_arc(); + + // A lagging node answers that nobody is live, which would pass for a completed check. + let router = Router::new().fallback(move || async move { + counted_requests.fetch_add(1, Ordering::Relaxed); + Json(json!({ "data": [{ "index": "0", "is_live": false }] })) + }); + + let listener = TcpListener::bind("127.0.0.1:0").await?; + let address = listener.local_addr()?; + + tokio::spawn(async move { axum::serve(listener, router).await }); + + let node = Arc::new(RemoteBeaconNode::new( + Arc::new(ChainConfig::mainnet()), + Client::new(), + format!("http://{address}").parse()?, + 32, + )); + + node.set_health(Health::Unusable); + + let remotes = RemoteBeaconNodes::new(vec![node], vec![], false); + let nodes = RemoteNodes::new(&remotes, 0, 32); + + BeaconNodeApi::::liveness(&nodes, 0, &[0]) + .await + .expect_err("a node that is not ready cannot report liveness"); + + assert_eq!(requests.load(Ordering::Relaxed), 0); + + Ok(()) + } + + const SLOW: Duration = Duration::from_millis(50); + const FAST: Duration = Duration::ZERO; + + #[test] + fn all_covers_every_published_duty() { + for duty in [ + PublishedDuty::Aggregates, + PublishedDuty::Attestations, + PublishedDuty::Blocks, + PublishedDuty::PayloadAttestations, + PublishedDuty::ProposerPreferences, + PublishedDuty::SyncCommitteeContributions, + PublishedDuty::SyncCommitteeMessages, + ] { + assert!(should_publish_to_every_node(&[PublishedDuty::All], duty)); + } + + assert!(should_publish_to_every_node( + &[PublishedDuty::Attestations], + PublishedDuty::Attestations, + )); + + assert!(!should_publish_to_every_node( + &[PublishedDuty::Attestations], + PublishedDuty::Aggregates, + )); + + assert!(!should_publish_to_every_node( + &[], + PublishedDuty::Aggregates + )); + } + + // Attempts are ordered, not raced, so the earlier node wins even when a later one is faster. + #[tokio::test] + async fn first_success_prefers_the_earlier_node() -> Result<()> { + let (name, value) = first_success( + "answer", + [ + ("slow", answer(1, SLOW, false)), + ("fast", answer(2, FAST, false)), + ], + ) + .await?; + + assert_eq!(name, "slow"); + assert_eq!(value, 1); + + Ok(()) + } + + #[tokio::test] + async fn first_success_ignores_a_failing_node() -> Result<()> { + let (name, value) = first_success( + "answer", + [ + ("broken", answer(2, FAST, true)), + ("slow", answer(3, SLOW, false)), + ], + ) + .await?; + + assert_eq!(name, "slow"); + assert_eq!(value, 3); + + Ok(()) + } + + #[tokio::test] + async fn first_success_fails_without_any_node() { + // `take(0)` only to give the empty iterator a concrete future type. + let attempts = once(("unused", answer(1, FAST, false))).take(0); + + first_success("answer", attempts) + .await + .expect_err("there is no node that could produce an answer"); + } + + #[tokio::test] + async fn first_success_fails_when_every_node_fails() { + first_success( + "answer", + [ + ("broken", answer(1, FAST, true)), + ("also broken", answer(2, SLOW, true)), + ], + ) + .await + .expect_err("every attempt fails, so no answer can be produced"); + } + + // Each node's failure is logged as it happens; the error names no node, so that the last + // one tried is not taken for the cause. + #[tokio::test] + async fn first_success_reports_the_operation_rather_than_a_node() { + let error = first_success( + "answer", + [ + ("best", answer(1, FAST, true)), + ("worst", answer(2, FAST, true)), + ], + ) + .await + .expect_err("every attempt fails, so no answer can be produced"); + + assert_eq!(error.to_string(), "no beacon node was able to answer"); + } + + async fn answer(value: u64, delay: Duration, fails: bool) -> Result { + sleep(delay).await; + ensure!(!fails, anyhow!("no")); + Ok(value) + } +} diff --git a/validator/src/chain_events.rs b/validator/src/chain_events.rs new file mode 100644 index 000000000..0806fde76 --- /dev/null +++ b/validator/src/chain_events.rs @@ -0,0 +1,565 @@ +use core::{pin::pin, time::Duration}; +use std::{ + sync::{Arc, RwLock}, + time::Instant, +}; + +use anyhow::Result; +use futures::{StreamExt as _, channel::mpsc::UnboundedSender, future::join_all}; +use logging::{debug_with_peers, info_with_peers, warn_with_peers}; +use std_ext::ArcExt as _; +use tap::Pipe as _; +use thiserror::Error; +use tokio::time::sleep; +use types::{ + phase0::primitives::{Epoch, H256, Slot}, + preset::Preset, +}; + +use crate::{health::Health, messages::InternalMessage, remote_beacon_node::RemoteBeaconNode}; + +const RECONNECT_DELAY: Duration = Duration::from_secs(1); + +const MAX_RECONNECT_DELAY: Duration = Duration::from_mins(1); + +#[derive(Debug, Error)] +enum Error { + #[error("cached head is at slot {head_slot}, past slot {at_slot} being signed for")] + BeyondSlot { head_slot: Slot, at_slot: Slot }, +} + +#[derive(Clone, Copy)] +struct Head { + slot: Slot, + block_root: H256, + /// When the node was seen holding this head. + seen_at: Instant, +} + +/// The roots the duties of an epoch and the next depend on, as reported with a head. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct DependentRoots { + /// The epoch of the head the roots were reported with. + pub epoch: Epoch, + pub current: H256, + pub next: H256, +} + +/// A head reported over the event stream, in either version of the event. +#[derive(Clone, Copy, Debug)] +pub struct HeadUpdate { + pub slot: Slot, + pub block: H256, + pub execution_optimistic: bool, + pub dependent_roots: DependentRoots, +} + +/// What the event stream of a node delivers. +pub enum StreamEvent { + Head(HeadUpdate), + Finalized(Epoch), +} + +/// What a cached head is worth at a slot. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum HeadStatus { + /// Nothing has been reported yet. + Unknown, + /// The head to sign for. + Usable(H256), + /// Older than the empty slot limit allows. + Stale { head_slot: Slot }, + /// Past the slot, which asking a node would not fix. + Future { head_slot: Slot }, +} + +/// The head a beacon node last reported, over the event stream or when asked. +pub struct ChainHead { + head: RwLock>, + dependent_roots: RwLock>, +} + +impl ChainHead { + #[must_use] + pub const fn new() -> Self { + Self { + head: RwLock::new(None), + dependent_roots: RwLock::new(None), + } + } + + pub fn dependent_root_for(&self, epoch: Epoch) -> Option { + let roots = (*self + .dependent_roots + .read() + .expect("dependent roots lock is never poisoned"))?; + + if epoch == roots.epoch { + Some(roots.current) + } else if epoch == roots.epoch.checked_add(1)? { + Some(roots.next) + } else { + None + } + } + + pub fn record_dependent_roots(&self, roots: DependentRoots) { + *self + .dependent_roots + .write() + .expect("dependent roots lock is never poisoned") = Some(roots); + } + + pub fn status(&self, at_slot: Slot, max_empty_slots: u64) -> HeadStatus { + let Some(head) = *self.head.read().expect("chain head lock is never poisoned") else { + return HeadStatus::Unknown; + }; + + if head.slot > at_slot { + return HeadStatus::Future { + head_slot: head.slot, + }; + } + + // A head that predates a run of missed slots is still the one to vote for, held to the + // same limit as the head of the built-in beacon node. + if at_slot.saturating_sub(head.slot) > max_empty_slots { + return HeadStatus::Stale { + head_slot: head.slot, + }; + } + + HeadStatus::Usable(head.block_root) + } + + pub fn get(&self, at_slot: Slot, max_empty_slots: u64) -> Result> { + match self.status(at_slot, max_empty_slots) { + HeadStatus::Usable(block_root) => Ok(Some(block_root)), + HeadStatus::Unknown => Ok(None), + HeadStatus::Stale { head_slot } => { + debug_with_peers!( + "cached head at slot {head_slot} is too old to sign for slot {at_slot}", + ); + + Ok(None) + } + // Asking a beacon node would not fix a head past the slot being signed for. + HeadStatus::Future { head_slot } => { + Err(Error::BeyondSlot { head_slot, at_slot }.into()) + } + } + } + + pub fn cached(&self) -> Option<(Slot, H256)> { + self.head + .read() + .expect("chain head lock is never poisoned") + .map(|head| (head.slot, head.block_root)) + } + + pub fn update(&self, slot: Slot, block_root: H256) { + // The latest report is the node's head whatever its slot; a reorg can move it back. + *self + .head + .write() + .expect("chain head lock is never poisoned") = Some(Head { + slot, + block_root, + seen_at: Instant::now(), + }); + } + + pub fn update_seen_at(&self, asked_at: Instant, slot: Slot, block_root: H256) -> bool { + let mut head = self + .head + .write() + .expect("chain head lock is never poisoned"); + + // A head delivered since the poll is the newer one and stays. + if head.is_some_and(|head| head.seen_at >= asked_at) { + return false; + } + + // The roots came with a streamed head; a polled head on another block may follow a reorg + // the stream never reported, so they are asked for again. + if head.is_none_or(|head| head.block_root != block_root) { + *self + .dependent_roots + .write() + .expect("dependent roots lock is never poisoned") = None; + } + + *head = Some(Head { + slot, + block_root, + seen_at: asked_at, + }); + true + } +} + +pub async fn stream_events( + nodes: Vec>, + internal_tx: UnboundedSender, +) { + nodes + .into_iter() + .map(|node| follow::

(node, internal_tx.clone())) + .pipe(join_all) + .await; +} + +async fn follow( + node: Arc, + internal_tx: UnboundedSender, +) { + let mut delay = RECONNECT_DELAY; + let mut subscribed_before = false; + + loop { + let delivered = match node.events::

().await { + Ok(events) => { + let mut events = pin!(events); + + if subscribed_before { + info_with_peers!("resubscribed to events from {node}"); + } else { + info_with_peers!("subscribed to events from {node}"); + subscribed_before = true; + } + + let mut delivered = false; + + while let Some(event) = events.next().await { + match event { + Ok(event) => { + delivered = true; + + // A node on a different network would otherwise report a head from + // another chain. + if node.health() == Health::Incompatible { + warn_with_peers!( + "received an event from an incompatible node: {node}" + ); + + continue; + } + + match event { + StreamEvent::Head(update) => { + accept(&node, update, &internal_tx); + } + StreamEvent::Finalized(epoch) => { + InternalMessage::FinalizedCheckpoint(epoch).send(&internal_tx); + } + } + } + Err(error) => { + warn_with_peers!("event stream from {node} failed: {error:?}"); + break; + } + } + } + + // Answering without streaming anything is indistinguishable from never + // having been asked. + if !delivered { + warn_with_peers!("event stream from {node} ended without any event"); + } + + delivered + } + Err(error) => { + warn_with_peers!("unable to stream events from {node}: {error:?}"); + false + } + }; + + // Connecting to a node that streams nothing fails as surely as not connecting at all, + // so only events that arrive shorten the wait again. + if delivered { + delay = RECONNECT_DELAY; + } + + sleep(delay).await; + + delay = delay.saturating_mul(2).min(MAX_RECONNECT_DELAY); + } +} + +fn accept( + node: &Arc, + event: HeadUpdate, + internal_tx: &UnboundedSender, +) { + // The roots only decide whether duties are fetched again, so an optimistic head may report + // them; nothing is signed by them. + node.chain_head() + .record_dependent_roots(event.dependent_roots); + + // An optimistic validator must not sign across the sync committee domains. + if event.execution_optimistic { + return; + } + + node.chain_head().update(event.slot, event.block); + + InternalMessage::Head { + node: node.clone_arc(), + slot: event.slot, + block_root: event.block, + } + .send(internal_tx); +} + +#[cfg(test)] +mod tests { + use futures::channel::mpsc::unbounded; + use reqwest::Client; + use types::config::Config; + + use super::*; + + const MAX_EMPTY_SLOTS: u64 = 8; + + // Only a head the node reports as not optimistic is offered for signing. + #[test] + fn an_optimistic_head_records_its_roots_but_is_not_offered_for_signing() -> Result<()> { + let node = Arc::new(RemoteBeaconNode::new( + Arc::new(Config::mainnet()), + Client::new(), + "http://streaming".parse()?, + MAX_EMPTY_SLOTS, + )); + let (tx, mut rx) = unbounded(); + let roots = DependentRoots { + epoch: 1, + current: H256::repeat_byte(1), + next: H256::repeat_byte(2), + }; + + let mut update = HeadUpdate { + slot: 40, + block: H256::repeat_byte(3), + execution_optimistic: true, + dependent_roots: roots, + }; + + accept(&node, update, &tx); + + assert_eq!(node.chain_head().dependent_root_for(1), Some(roots.current)); + assert_eq!(node.chain_head().cached(), None); + assert!(rx.try_recv().is_err()); + + update.execution_optimistic = false; + + accept(&node, update, &tx); + + assert_eq!(node.chain_head().cached(), Some((40, update.block))); + assert!(matches!( + rx.try_recv()?, + InternalMessage::Head { slot: 40, block_root, .. } if block_root == update.block + )); + + Ok(()) + } + + #[test] + fn a_head_at_or_before_the_slot_is_signed_for() -> Result<()> { + let chain_head = ChainHead::new(); + + assert_eq!(chain_head.get(6, MAX_EMPTY_SLOTS)?, None); + + chain_head.update(6, H256::repeat_byte(1)); + + assert_eq!( + chain_head.get(6, MAX_EMPTY_SLOTS)?, + Some(H256::repeat_byte(1)) + ); + assert_eq!( + chain_head.get(7, MAX_EMPTY_SLOTS)?, + Some(H256::repeat_byte(1)) + ); + + chain_head + .get(5, MAX_EMPTY_SLOTS) + .expect_err("the tracker has moved past slot 5"); + + Ok(()) + } + + #[test] + fn a_head_further_behind_than_max_empty_slots_is_not_signed_for() -> Result<()> { + let chain_head = ChainHead::new(); + + chain_head.update(6, H256::repeat_byte(1)); + + assert!(chain_head.get(14, MAX_EMPTY_SLOTS)?.is_some()); + assert_eq!(chain_head.get(15, MAX_EMPTY_SLOTS)?, None); + + Ok(()) + } + + #[test] + fn a_polled_head_replaces_a_cached_one_from_the_future() -> Result<()> { + let chain_head = ChainHead::new(); + + chain_head.update(u64::MAX, H256::repeat_byte(1)); + + chain_head + .get(6, MAX_EMPTY_SLOTS) + .expect_err("the cached head is past slot 6"); + + chain_head.update(5, H256::repeat_byte(2)); + + assert_eq!( + chain_head.get(6, MAX_EMPTY_SLOTS)?, + Some(H256::repeat_byte(2)) + ); + + Ok(()) + } + + // A poll answered after the stream moved the head must not move it back. + #[test] + fn a_head_delivered_since_a_poll_was_sent_outlives_the_answer() -> Result<()> { + let chain_head = ChainHead::new(); + chain_head.update(5, H256::repeat_byte(1)); + + let asked_at = Instant::now(); + chain_head.update(6, H256::repeat_byte(2)); + + assert!(!chain_head.update_seen_at(asked_at, 5, H256::repeat_byte(1))); + assert_eq!( + chain_head.get(6, MAX_EMPTY_SLOTS)?, + Some(H256::repeat_byte(2)) + ); + + // Asked after the last delivery, the answer stands, even for an earlier slot. + let asked_after = Instant::now() + .checked_add(Duration::from_millis(1)) + .expect("an instant a millisecond away is representable"); + + assert!(chain_head.update_seen_at(asked_after, 5, H256::repeat_byte(3))); + assert_eq!( + chain_head.get(6, MAX_EMPTY_SLOTS)?, + Some(H256::repeat_byte(3)) + ); + + Ok(()) + } + + #[test] + fn dependent_roots_serve_the_reported_epoch_and_the_next() { + let chain_head = ChainHead::new(); + + assert_eq!(chain_head.dependent_root_for(5), None); + + chain_head.record_dependent_roots(DependentRoots { + epoch: 5, + current: H256::repeat_byte(1), + next: H256::repeat_byte(2), + }); + + assert_eq!(chain_head.dependent_root_for(4), None); + assert_eq!(chain_head.dependent_root_for(5), Some(H256::repeat_byte(1))); + assert_eq!(chain_head.dependent_root_for(6), Some(H256::repeat_byte(2))); + assert_eq!(chain_head.dependent_root_for(7), None); + } + + // A polled head on another block may follow a reorg the stream missed; its roots are void. + #[test] + fn a_polled_head_on_another_block_drops_the_streamed_dependent_roots() { + let chain_head = ChainHead::new(); + + chain_head.record_dependent_roots(DependentRoots { + epoch: 5, + current: H256::repeat_byte(1), + next: H256::repeat_byte(2), + }); + + chain_head.update(40, H256::repeat_byte(3)); + + // Polls are stamped later than the delivery, which a coarse clock could otherwise tie. + let first_poll = Instant::now() + .checked_add(Duration::from_millis(1)) + .expect("an instant a millisecond away is representable"); + + let second_poll = first_poll + .checked_add(Duration::from_millis(1)) + .expect("an instant a millisecond away is representable"); + + // Polled on the same block, the roots still hold. + assert!(chain_head.update_seen_at(first_poll, 40, H256::repeat_byte(3))); + assert_eq!(chain_head.dependent_root_for(5), Some(H256::repeat_byte(1))); + + assert!(chain_head.update_seen_at(second_poll, 41, H256::repeat_byte(4))); + assert_eq!(chain_head.dependent_root_for(5), None); + } + + // A reorg into the previous epoch replaces the orphaned branch's roots, so the latest report + // wins even when it is for an earlier epoch. + #[test] + fn the_latest_dependent_roots_win_regardless_of_epoch() { + let chain_head = ChainHead::new(); + + chain_head.record_dependent_roots(DependentRoots { + epoch: 5, + current: H256::repeat_byte(1), + next: H256::repeat_byte(2), + }); + + chain_head.record_dependent_roots(DependentRoots { + epoch: 4, + current: H256::repeat_byte(3), + next: H256::repeat_byte(4), + }); + + assert_eq!(chain_head.dependent_root_for(4), Some(H256::repeat_byte(3))); + assert_eq!(chain_head.dependent_root_for(5), Some(H256::repeat_byte(4))); + assert_eq!(chain_head.dependent_root_for(6), None); + } + + // A node reorging to a shorter branch reports a lower head, which must not be left behind. + #[test] + fn a_reorg_to_a_lower_slot_replaces_the_cached_head() -> Result<()> { + let chain_head = ChainHead::new(); + + chain_head.update(6, H256::repeat_byte(1)); + chain_head.update(5, H256::repeat_byte(2)); + + assert_eq!( + chain_head.get(6, MAX_EMPTY_SLOTS)?, + Some(H256::repeat_byte(2)) + ); + + Ok(()) + } + + #[test] + fn an_empty_cache_is_unknown_rather_than_stale() { + assert_eq!(ChainHead::new().status(100, 5), HeadStatus::Unknown); + } + + #[test] + fn a_head_within_the_empty_slot_limit_is_not_stale() { + let chain_head = ChainHead::new(); + chain_head.update(95, H256::repeat_byte(1)); + + assert_eq!( + chain_head.status(100, 5), + HeadStatus::Usable(H256::repeat_byte(1)) + ); + } + + #[test] + fn a_head_beyond_the_empty_slot_limit_is_stale() { + let chain_head = ChainHead::new(); + chain_head.update(61, H256::repeat_byte(1)); + + assert_eq!( + chain_head.status(143, 32), + HeadStatus::Stale { head_slot: 61 } + ); + } +} diff --git a/validator/src/gossip_spec_tests.rs b/validator/src/gossip_spec_tests.rs index c06839634..a1083a994 100644 --- a/validator/src/gossip_spec_tests.rs +++ b/validator/src/gossip_spec_tests.rs @@ -22,7 +22,7 @@ use futures::{ channel::mpsc::{UnboundedReceiver, UnboundedSender}, lock::Mutex, }; -use keymanager::KeyManager; +use keymanager::{KeyManager, ProposerConfigs}; use operation_pools::{ AttestationAggPool, BlsToExecutionChangePool, PayloadAttestationAggPool, SyncCommitteeAggPool, }; @@ -163,7 +163,6 @@ impl Context

{ let signer = Arc::new(Signer::new( core::iter::empty(), - client.clone(), client, Web3SignerConfig::default(), None, @@ -179,10 +178,7 @@ impl Context

{ ..Default::default() }); - let keymanager = Arc::new(KeyManager::new_in_memory( - signer.clone_arc(), - slashing_protector.clone_arc(), - genesis_validators_root, + let proposer_configs = Arc::new(ProposerConfigs::new( validator_config.suggested_fee_recipient, validator_config.default_gas_limit, H256::default(), @@ -190,6 +186,14 @@ impl Context

{ validator_config.validator_definitions.clone_arc(), )); + let keymanager = Arc::new(KeyManager::new_in_memory( + signer.clone_arc(), + slashing_protector.clone_arc(), + genesis_validators_root, + proposer_configs, + validator_config.validator_definitions.clone_arc(), + )); + let attestation_verifier = AttestationVerifier::new( controller.clone_arc(), dedicated_executor.clone_arc(), diff --git a/validator/src/health.rs b/validator/src/health.rs new file mode 100644 index 000000000..80bdfcd20 --- /dev/null +++ b/validator/src/health.rs @@ -0,0 +1,118 @@ +/// Variants are ordered worst first. Nodes are ranked against each other by this order. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +#[repr(u8)] +pub enum Health { + Incompatible, + Unreachable, + Unusable, + Unknown, + Ready, +} + +impl Health { + #[must_use] + pub const fn is_ready(self) -> bool { + matches!(self, Self::Ready) + } + + #[must_use] + pub const fn can_serve(self) -> bool { + // A failed poll does not mean the next request fails, so only a node on the wrong + // network is never asked. + !matches!(self, Self::Incompatible) + } + + pub(crate) const fn from_syncing_status( + el_offline: bool, + is_syncing: bool, + is_optimistic: bool, + ) -> Self { + // An optimistic head has not been verified by an execution engine. + if el_offline || is_syncing || is_optimistic { + return Self::Unusable; + } + + Self::Ready + } + + pub(crate) const fn as_u8(self) -> u8 { + self as u8 + } + + pub(crate) const fn from_u8(byte: u8) -> Self { + match byte { + 0 => Self::Incompatible, + 1 => Self::Unreachable, + 2 => Self::Unusable, + 4 => Self::Ready, + _ => Self::Unknown, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_a_ready_node_reports_ready() { + assert!(Health::Ready.is_ready()); + assert!(!Health::Unknown.is_ready()); + assert!(!Health::Unusable.is_ready()); + assert!(!Health::Unreachable.is_ready()); + assert!(!Health::Incompatible.is_ready()); + } + + #[test] + fn any_node_may_serve_but_one_on_the_wrong_network() { + assert!(Health::Ready.can_serve()); + assert!(Health::Unknown.can_serve()); + assert!(Health::Unusable.can_serve()); + assert!(Health::Unreachable.can_serve()); + assert!(!Health::Incompatible.can_serve()); + } + + #[test] + fn health_is_ordered_worst_first() { + assert!(Health::Incompatible < Health::Unreachable); + assert!(Health::Unreachable < Health::Unusable); + assert!(Health::Unusable < Health::Unknown); + assert!(Health::Unknown < Health::Ready); + } + + #[test] + fn any_reported_degradation_makes_a_node_unusable() { + assert_eq!( + Health::from_syncing_status(true, false, false), + Health::Unusable, + ); + + assert_eq!( + Health::from_syncing_status(false, true, false), + Health::Unusable, + ); + + assert_eq!( + Health::from_syncing_status(false, false, true), + Health::Unusable, + ); + + assert_eq!( + Health::from_syncing_status(false, false, false), + Health::Ready, + ); + } + + #[test] + fn health_survives_a_round_trip_through_a_byte() { + for health in [ + Health::Incompatible, + Health::Unreachable, + Health::Unusable, + Health::Unknown, + Health::Ready, + ] { + assert_eq!(Health::from_u8(health.as_u8()), health); + } + } +} diff --git a/validator/src/lib.rs b/validator/src/lib.rs index 696c19066..5087fa13c 100644 --- a/validator/src/lib.rs +++ b/validator/src/lib.rs @@ -1,16 +1,36 @@ pub use crate::{ api::{ValidatorApiConfig, run_validator_api}, + local_beacon_node::{ + attester_duties_at_slot, proposer_duties_at_epoch, proposer_lookahead_applies, + ptc_duties_at_epoch, + }, messages::ApiToValidator, - validator::{Channels as ValidatorChannels, Validator}, + misc::{Chain, ChainSource, LocalChain}, + own_validator_indices::OwnValidatorIndices, + remote_beacon_node::{Genesis, RemoteBeaconNode}, + remote_beacon_nodes::{RemoteBeaconNodes, StartupError as ValidatorStartupError}, + validator::{ + Channels as ValidatorChannels, LocalChannels as LocalValidatorChannels, Validator, + }, validator_config::ValidatorConfig, }; mod api; +mod beacon_node_api; +mod beacon_nodes; +mod chain_events; +mod health; +mod local_beacon_node; mod messages; mod misc; mod own_beacon_committee_members; +mod own_proposer_duties; mod own_ptc_members; +mod own_sync_committee_members; mod own_sync_committee_subscriptions; +mod own_validator_indices; +mod remote_beacon_node; +mod remote_beacon_nodes; mod slot_head; mod tasks; mod validator; diff --git a/validator/src/local_beacon_node.rs b/validator/src/local_beacon_node.rs new file mode 100644 index 000000000..cada37275 --- /dev/null +++ b/validator/src/local_beacon_node.rs @@ -0,0 +1,1159 @@ +use core::ops::Range; +use std::{ + collections::{BTreeMap, HashSet}, + sync::Arc, +}; + +use anyhow::{Error as AnyhowError, Result, anyhow, bail, ensure}; +use block_producer::{ + BlindedBlockPublication, BlockBuildOptions, ProposerData, ValidatorBlindedBlock, +}; +use bls::{PublicKeyBytes, SignatureBytes}; +use builder_api::{ + consts::MaxBuilderEntries, + gloas::containers::{BuilderEntry, BuilderPreferencesEntry, BuilderUrl}, + unphased::containers::SignedValidatorRegistrationV1, +}; +use dedicated_executor::DedicatedExecutor; +use fork_choice_control::Wait; +use fork_choice_store::{ + AttestationItem, AttestationOrigin, PayloadAttestationItem, PayloadAttestationOrigin, +}; +use futures::{ + channel::{mpsc::UnboundedSender, oneshot}, + future::join_all, +}; +use helper_functions::{accessors, misc}; +use http_api_utils::{ + ValidatorAttesterDutyResponse, ValidatorLivenessResponse, ValidatorPTCDutyResponse, + ValidatorProposerDutyResponse, ValidatorSyncDutyResponse, +}; +use itertools::Itertools as _; +use liveness_tracker::ApiToLiveness; +use logging::{debug_with_peers, warn_with_peers}; +use operation_pools::{AttestationKey, convert_to_electra_attestation}; +use p2p::{ + BeaconCommitteeSubscription, SyncCommitteeSubscription, ToSubnetService, ValidatorToP2p, +}; +use prometheus_metrics::Metrics; +use signer::Signer; +use ssz::{ContiguousList, SszHash as _}; +use std_ext::ArcExt as _; +use tap::Pipe as _; +use thiserror::Error; +use try_from_iterator::TryFromIterator as _; +use types::{ + altair::{ + containers::{SignedContributionAndProof, SyncCommitteeContribution, SyncCommitteeMessage}, + primitives::SubcommitteeIndex, + }, + combined::{ + Attestation, BeaconState, DataColumnSidecar, SignedAggregateAndProof, SignedBeaconBlock, + SignedBlindedBeaconBlock, + }, + config::Config, + deneb::primitives::{Blob, KzgProof}, + gloas::{ + consts::BUILDER_INDEX_SELF_BUILD, + containers::{ + PayloadAttestationData, PayloadAttestationMessage, SignedExecutionPayloadEnvelope, + SignedProposerPreferences, + }, + }, + nonstandard::{ + KzgProofs, OwnAttestation, PayloadStatus, Phase, RelativeEpoch, WithBlobsAndMev, WithStatus, + }, + phase0::{ + containers::{AttestationData, Checkpoint}, + primitives::{CommitteeIndex, Epoch, Gwei, H256, Slot, SubnetId, Uint256, ValidatorIndex}, + }, + preset::Preset, + traits::{BeaconBlock as _, BeaconState as _, PostAltairBeaconState}, +}; + +use crate::{ + beacon_node_api::{ + AttesterDuties, BeaconNodeApi, EnvelopeContents, ProducedBlock, ProposerDuties, PtcDuties, + }, + misc::LocalChain, + slot_head::SlotHead, + validator_config::ValidatorConfig, +}; + +#[derive(Debug, Error)] +#[error("the builder did not publish the blinded block")] +pub struct BlindedBlockNotPublished; + +/// What duties against the built-in beacon node are performed with, the same in every slot. +pub struct LocalContext { + pub chain: Arc>, + pub validator_config: Arc, + pub signer: Arc, + pub p2p_tx: UnboundedSender>, + pub subnet_service_tx: UnboundedSender, + pub liveness_tx: Option>, + pub metrics: Option>, + pub dedicated_executor: Arc, +} + +#[derive(Clone)] +pub struct LocalBeaconNode { + context: Arc>, + slot_head: SlotHead

, + beacon_state: Arc>, + wait_group: W, + // Whether attestations vote for the full node of the head. + // It must be the node the payload status was checked on. + full: bool, +} + +impl LocalBeaconNode { + #[must_use] + pub const fn head(&self) -> &SlotHead

{ + &self.slot_head + } + + #[must_use] + pub const fn state(&self) -> &Arc> { + &self.beacon_state + } + + pub const fn new( + context: Arc>, + slot_head: SlotHead

, + beacon_state: Arc>, + wait_group: W, + ) -> Self { + Self { + context, + slot_head, + beacon_state, + wait_group, + full: false, + } + } + + #[must_use] + pub const fn with_full(mut self, full: bool) -> Self { + self.full = full; + self + } + + pub fn attester_duties_at_slots( + &self, + slots: Range, + validator_indices: &[ValidatorIndex], + ) -> Result { + let state = self.beacon_state.as_ref(); + // `slots` must lie within one epoch, as the duties carry a single dependent root. + let epoch = misc::compute_epoch_at_slot::

(slots.start); + let dependent_root = self.dependent_root_at(epoch)?; + let indices = validator_indices.iter().copied().collect::>(); + + let duties = slots + .map(|slot| attester_duties_at_slot(state, slot, &indices)) + .flatten_ok() + .try_collect()?; + + Ok(AttesterDuties { + dependent_root, + duties, + }) + } + + pub const fn head_block_root(&self) -> H256 { + self.slot_head.beacon_block_root + } + + async fn publish_blob_data( + &self, + block: &Arc>, + blobs: ContiguousList, P::MaxBlobCommitmentsPerBlock>, + kzg_proofs: Option>, + ) -> Result<()> { + if self.slot_head.phase().is_peerdas_activated() { + let data_column_sidecars = eip_7594::construct_data_column_sidecars_from_blobs( + block.clone_arc().into(), + blobs.into_iter(), + kzg_proofs + .unwrap_or_else(KzgProofs::empty_fulu) + .into_iter() + .collect_vec(), + self.context.chain.controller.store_config().kzg_backend, + self.context.metrics.clone(), + self.context.dedicated_executor.clone_arc(), + ) + .await?; + + self.publish_data_column_sidecars(data_column_sidecars) + .await; + } else { + for blob_sidecar in misc::construct_blob_sidecars( + block, + blobs, + kzg_proofs.unwrap_or_else(KzgProofs::empty_deneb), + )? { + let blob_sidecar = Arc::new(blob_sidecar); + + self.context + .chain + .controller + .on_own_blob_sidecar(self.wait_group.clone(), blob_sidecar.clone_arc()); + } + } + + Ok(()) + } + + async fn publish_data_column_sidecars( + &self, + data_column_sidecars: Vec>>, + ) { + for data_column_sidecar in data_column_sidecars { + if self + .context + .chain + .controller + .sampling_columns() + .into_iter() + .contains(&data_column_sidecar.index()) + { + self.context + .chain + .controller + .on_own_data_column_sidecar( + self.wait_group.clone(), + data_column_sidecar.clone_arc(), + ) + .await; + } + + ValidatorToP2p::PublishDataColumnSidecar(data_column_sidecar) + .send(&self.context.p2p_tx); + } + } + + fn dependent_root_at(&self, epoch: Epoch) -> Result { + self.context + .chain + .controller + .dependent_root(&self.beacon_state, misc::previous_epoch(epoch)) + } +} + +impl BeaconNodeApi

for LocalBeaconNode { + async fn liveness( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> Result> { + let liveness_tx = self.context.liveness_tx.as_ref().ok_or_else(|| { + anyhow!("liveness tracking is disabled; enable it with --track-liveness") + })?; + + let (sender, receiver) = oneshot::channel(); + + ApiToLiveness::CheckLiveness(sender, epoch, validator_indices.to_vec()).send(liveness_tx); + + let liveness = receiver.await??; + + Ok(liveness + .into_iter() + .map(|(index, is_live)| ValidatorLivenessResponse { index, is_live }) + .collect()) + } + + async fn dependent_root(&self, epoch: Epoch, _validator_index: ValidatorIndex) -> Result { + self.dependent_root_at(epoch) + } + + async fn attestation_data( + &self, + slot: Slot, + committee_index: CommitteeIndex, + ) -> Result { + let slot_head = &self.slot_head; + let phase = slot_head.phase(); + + let index = if phase >= Phase::Gloas { + // `full` is the node the payload status was checked on. + // Choosing it again here could vote for a payload nobody checked. + // See + u64::from(self.full) + } else if phase >= Phase::Electra { + 0 + } else { + committee_index + }; + + let target = tokio::task::block_in_place(|| Checkpoint { + epoch: slot_head.current_epoch(), + root: accessors::epoch_boundary_block_root( + &self.beacon_state, + slot_head.beacon_block_root, + ), + }); + + Ok(AttestationData { + slot, + index, + beacon_block_root: slot_head.beacon_block_root, + source: self.beacon_state.current_justified_checkpoint(), + target, + }) + } + + async fn aggregate_attestation( + &self, + data: AttestationData, + committee_index: CommitteeIndex, + ) -> Result> { + let aggregate = self + .context + .chain + .attestation_agg_pool + .best_aggregate_attestation(AttestationKey { + data, + committee_index, + }) + .await + .ok_or_else(|| { + anyhow!( + "no aggregate attestation for committee {committee_index} in slot {}", + data.slot, + ) + })?; + + let phase = self.slot_head.config.phase_at_slot::

(data.slot); + + if phase < Phase::Electra { + Ok(Attestation::Phase0(aggregate.into_phase0_attestation())) + } else if phase < Phase::Gloas { + convert_to_electra_attestation(aggregate).map(Attestation::Electra) + } else { + convert_to_electra_attestation(aggregate) + .map(|aggregate| Attestation::Gloas(aggregate.into())) + } + } + + async fn publish_singular_attestations( + &self, + attestations: &[OwnAttestation

], + ) -> Result<()> { + for own_attestation in attestations { + let OwnAttestation { + validator_index, + attestation, + .. + } = own_attestation; + + let committee_index = misc::committee_index(attestation); + let attestation = Arc::new(attestation.clone()); + let subnet_id = + subnet_id::

(&self.beacon_state, attestation.data().slot, committee_index)?; + + self.context.chain.controller.on_singular_attestation( + self.wait_group.clone(), + AttestationItem::unverified( + attestation.clone_arc(), + AttestationOrigin::Own(subnet_id), + ), + ); + + ValidatorToP2p::PublishSingularAttestation(attestation.clone_arc(), subnet_id) + .send(&self.context.p2p_tx); + + self.context.chain.attestation_agg_pool.insert_attestation( + self.wait_group.clone(), + attestation, + Some(*validator_index), + ); + } + + Ok(()) + } + + async fn publish_aggregates_and_proofs( + &self, + aggregates_and_proofs: &[Arc>], + ) -> Result<()> { + for aggregate_and_proof in aggregates_and_proofs { + let attestation = Arc::new(aggregate_and_proof.aggregate()); + + self.context.chain.attestation_agg_pool.insert_attestation( + self.wait_group.clone(), + attestation, + None, + ); + + ValidatorToP2p::PublishAggregateAndProof(aggregate_and_proof.clone_arc()) + .send(&self.context.p2p_tx); + } + + Ok(()) + } + + async fn subscribe_to_beacon_committees( + &self, + current_slot: Slot, + subscriptions: &[BeaconCommitteeSubscription], + ) -> Result<()> { + let (sender, receiver) = oneshot::channel(); + + ToSubnetService::UpdateBeaconCommitteeSubscriptions( + current_slot, + subscriptions.to_vec(), + sender, + ) + .send(&self.context.subnet_service_tx); + + receiver.await? + } + + async fn sync_committee_duties( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> Result> { + let Some(state) = self.beacon_state.post_altair() else { + // Erring rather than answering with no duties keeps the answer out of the cache. + ensure!( + epoch + < self + .context + .chain + .controller + .chain_config() + .altair_fork_epoch, + "sync committee duties for epoch {epoch} are not known to a pre-Altair state", + ); + + return Ok(vec![]); + }; + + let indices = validator_indices.iter().copied().collect::>(); + + sync_duties_at_epoch(state, epoch, &indices) + } + + async fn publish_sync_committee_messages( + &self, + messages: &BTreeMap>, + ) -> Result<()> { + for (subcommittee_index, messages) in messages { + for message in messages { + ValidatorToP2p::PublishSyncCommitteeMessage(Box::new(( + *subcommittee_index, + *message, + ))) + .send(&self.context.p2p_tx); + } + + self.context + .chain + .sync_committee_agg_pool + .aggregate_own_messages( + self.wait_group.clone(), + messages.clone(), + *subcommittee_index, + self.beacon_state.clone_arc(), + ); + } + + Ok(()) + } + + async fn subscribe_to_sync_committees( + &self, + current_epoch: Epoch, + subscriptions: &[SyncCommitteeSubscription], + ) -> Result<()> { + ToSubnetService::UpdateSyncCommitteeSubscriptions(current_epoch, subscriptions.to_vec()) + .send(&self.context.subnet_service_tx); + + Ok(()) + } + + async fn sync_committee_contribution( + &self, + slot: Slot, + subcommittee_index: SubcommitteeIndex, + beacon_block_root: H256, + ) -> Result> { + Ok(self + .context + .chain + .sync_committee_agg_pool + .best_subcommittee_contribution(slot, beacon_block_root, subcommittee_index) + .await) + } + + async fn publish_contributions_and_proofs( + &self, + contributions_and_proofs: &[SignedContributionAndProof

], + ) -> Result<()> { + for contribution_and_proof in contributions_and_proofs { + ValidatorToP2p::PublishContributionAndProof(Box::new(*contribution_and_proof)) + .send(&self.context.p2p_tx); + + self.context + .chain + .sync_committee_agg_pool + .add_own_contribution( + contribution_and_proof.message.aggregator_index, + contribution_and_proof.message.contribution, + self.beacon_state.clone_arc(), + ); + } + + Ok(()) + } + + async fn ptc_duties( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> Result { + let dependent_root = self.dependent_root_at(epoch)?; + let indices = validator_indices.iter().copied().collect::>(); + + let duties = tokio::task::block_in_place(|| { + ptc_duties_at_epoch( + &self.slot_head.config, + self.beacon_state.as_ref(), + epoch, + &indices, + ) + })?; + + Ok(PtcDuties { + dependent_root, + duties, + }) + } + + async fn proposer_duties(&self, epoch: Epoch) -> Result { + let state = self.beacon_state.as_ref(); + + let dependent_root = + self.dependent_root_at(proposer_dependent_epoch(&self.slot_head.config, epoch))?; + + let duties = proposer_duties_at_epoch(&self.slot_head.config, state, epoch)?; + + Ok(ProposerDuties { + dependent_root, + duties, + }) + } + + async fn payload_attestation_data(&self, slot: Slot) -> Result> { + let Some(block_with_root) = self + .context + .chain + .controller + .stored_block_by_slot(slot)? + .map(WithStatus::value) + else { + return Ok(None); + }; + + let beacon_block_root = block_with_root.root; + + let blob_data_available = self + .context + .chain + .controller + .indices_of_missing_data_columns(&block_with_root.block) + .is_empty(); + + Ok(Some(PayloadAttestationData { + beacon_block_root, + slot, + payload_present: self + .context + .chain + .controller + .is_payload_present_timely(beacon_block_root), + blob_data_available, + })) + } + + async fn publish_payload_attestations( + &self, + messages: &[Arc], + ) -> Result<()> { + for message in messages { + self.context.chain.controller.on_payload_attestation( + self.wait_group.clone(), + PayloadAttestationItem::unverified( + Arc::new(message.clone_arc().into()), + PayloadAttestationOrigin::Own, + ), + ); + + ValidatorToP2p::PublishPayloadAttestation(message.clone_arc()) + .send(&self.context.p2p_tx); + } + + let beacon_state = &self.beacon_state; + + let next_proposer_index = + tokio::task::block_in_place(|| self.slot_head.next_proposer_index(beacon_state))?; + + let public_key = accessors::public_key(beacon_state.as_ref(), next_proposer_index)?; + + // The messages are only aggregated when an own validator proposes next. + if self.context.signer.load().has_key(*public_key) { + self.context + .chain + .payload_attestation_agg_pool + .aggregate_own_messages( + self.wait_group.clone(), + messages.iter().map(|message| **message).collect(), + beacon_state.clone_arc(), + ); + } + + Ok(()) + } + + #[expect(clippy::too_many_lines)] + async fn produce_block( + &self, + slot: Slot, + randao_reveal: SignatureBytes, + graffiti: Option, + builder_boost_factor: Uint256, + min_bid: Gwei, + builders: &ContiguousList, + ) -> Result> { + ensure!( + slot == self.slot_head.slot(), + "the built-in beacon node produces blocks for slot {} only, not slot {slot}", + self.slot_head.slot(), + ); + + let beacon_state = &self.beacon_state; + let proposer_index = + tokio::task::block_in_place(|| self.slot_head.proposer_index(beacon_state))?; + let public_key = *accessors::public_key(beacon_state.as_ref(), proposer_index)?; + + let block_build_context = self.context.chain.block_producer.new_build_context( + beacon_state.clone_arc(), + self.slot_head.beacon_block_root, + proposer_index, + BlockBuildOptions { + graffiti, + disable_blockprint_graffiti: self + .context + .validator_config + .disable_blockprint_graffiti, + builder_boost_factor, + min_bid, + ..BlockBuildOptions::default() + }, + ); + + let execution_payload_header_handle = + block_build_context.get_execution_payload_header(public_key); + + let local_execution_payload_handle = block_build_context.get_local_execution_payload(); + + let builder_api_bids_handle = if self.slot_head.phase() >= Phase::Gloas { + block_build_context.get_builder_api_bids(builders.clone()) + } else { + None + }; + + let Some(( + WithBlobsAndMev { + value: block, + proofs, + blobs, + .. + }, + _block_rewards, + builder_url, + )) = block_build_context + .build_blinded_beacon_block( + randao_reveal, + execution_payload_header_handle, + local_execution_payload_handle, + builder_api_bids_handle, + ) + .await? + else { + bail!("no block could be built for slot {slot}"); + }; + + // The wait checked the payload the proposal was going to extend. + // Production picks the parent's node again, so it can extend a different payload. + // Its envelope may have arrived in between, or the PTC vote may have flipped the node. + // Never sign a block that extends a payload the execution engine has not validated. + let controller = &self.context.chain.controller; + + let extended_payload_status = match &block { + ValidatorBlindedBlock::BlindedBeaconBlock(block) => { + controller.wait_for_valid_extended_payload(block).await + } + ValidatorBlindedBlock::BeaconBlock(block) => { + controller.wait_for_valid_extended_payload(block).await + } + }; + + ensure!( + extended_payload_status.is_some_and(PayloadStatus::is_valid), + "the block extends a payload that is not fully validated \ + (status: {extended_payload_status:?})", + ); + + let envelope_contents = match &block { + ValidatorBlindedBlock::BeaconBlock(block) + if block + .payload_bid() + .is_some_and(|bid| bid.builder_index == BUILDER_INDEX_SELF_BUILD) => + { + match self + .context + .chain + .block_producer + .build_local_execution_payload_envelope_contents( + block.hash_tree_root(), + block.parent_root(), + ) + .await + { + Some((envelope, blobs, kzg_proofs)) => Some(EnvelopeContents { + envelope, + kzg_proofs: ContiguousList::try_from_iter(kzg_proofs)?, + blobs, + }), + None => { + warn_with_peers!( + "the payload of the self-built block for slot {slot} is missing, \ + so the block is published without it", + ); + None + } + } + } + _ => None, + }; + + // The blobs of a self-built payload travel with its envelope rather than the block. + Ok(match envelope_contents { + Some(_) => ProducedBlock { + block, + kzg_proofs: None, + blobs: None, + envelope_contents, + builder_url, + }, + // From Gloas on a block without its own payload commits to a builder's blobs. + None if self.slot_head.phase() >= Phase::Gloas => ProducedBlock { + builder_url, + ..ProducedBlock::without_blobs(block) + }, + None => ProducedBlock { + block, + kzg_proofs: proofs, + blobs, + envelope_contents: None, + builder_url, + }, + }) + } + + async fn publish_block( + &self, + signed_block: &Arc>, + kzg_proofs: Option<&KzgProofs

>, + blobs: Option<&ContiguousList, P::MaxBlobCommitmentsPerBlock>>, + builder_url: Option<&BuilderUrl>, + ) -> Result<()> { + let import_and_publish_block = || { + self.context + .chain + .controller + .on_own_block(self.wait_group.clone(), signed_block.clone_arc()); + + if let Some(builder_url) = builder_url.cloned() { + let payload_builder_api = self.context.chain.block_producer.payload_builder_api(); + let payload_builder_api = payload_builder_api.clone_arc(); + let block = signed_block.clone_arc(); + + tokio::spawn(async move { + if let Err(error) = payload_builder_api + .submit_signed_beacon_block(&builder_url, &block) + .await + { + warn_with_peers!( + "failed to forward block to builder {builder_url}: {error:?}" + ); + } + }); + } + + ValidatorToP2p::PublishBeaconBlock(signed_block.clone_arc()).send(&self.context.p2p_tx); + }; + + // From Gloas on the block goes out before its data column sidecars. + let post_gloas = signed_block.phase() >= Phase::Gloas; + + if post_gloas { + import_and_publish_block(); + } + + if let Some(blobs) = blobs + && !blobs.is_empty() + { + self.publish_blob_data(signed_block, blobs.clone(), kzg_proofs.cloned()) + .await?; + } + + if !post_gloas { + import_and_publish_block(); + } + + Ok(()) + } + + async fn publish_blinded_block( + &self, + signed_block: &SignedBlindedBeaconBlock

, + ) -> Result<()> { + let Some(publication) = self + .context + .chain + .block_producer + .publish_signed_blinded_block(signed_block) + .await + else { + bail!(BlindedBlockNotPublished); + }; + + let WithBlobsAndMev { + value: execution_payload, + proofs, + blobs, + .. + } = match publication { + BlindedBlockPublication::Payload(execution_payload) => execution_payload.result, + BlindedBlockPublication::PublishedByBuilder => return Ok(()), + }; + + let (message, signature) = signed_block.clone().split(); + + let signed_block = message + .with_execution_payload(execution_payload)? + .with_signature(signature) + .pipe(Arc::new); + + // Only a Gloas bid names a builder to forward the block to. + self.publish_block(&signed_block, proofs.as_ref(), blobs.as_ref(), None) + .await + } + + async fn publish_execution_payload_envelope( + &self, + signed_envelope: &Arc>, + kzg_proofs: &ContiguousList, + blobs: &ContiguousList, P::MaxBlobCommitmentsPerBlock>, + ) -> Result<()> { + let envelope = &signed_envelope.message; + + if !blobs.is_empty() { + let data_column_sidecars = eip_7594::construct_data_column_sidecars_from_blobs( + (envelope.beacon_block_root, envelope.payload.slot_number).into(), + blobs.clone().into_iter(), + kzg_proofs.iter().copied().collect_vec(), + self.context.chain.controller.store_config().kzg_backend, + self.context.metrics.clone(), + self.context.dedicated_executor.clone_arc(), + ) + .await?; + + self.publish_data_column_sidecars(data_column_sidecars) + .await; + } + + self.context + .chain + .controller + .on_own_execution_payload_envelope(signed_envelope.clone_arc()); + + ValidatorToP2p::PublishExecutionPayloadEnvelope(signed_envelope.clone_arc()) + .send(&self.context.p2p_tx); + + Ok(()) + } + + async fn prepare_beacon_proposer( + &self, + _current_slot: Slot, + _proposers: &[ProposerData], + ) -> Result<()> { + Ok(()) + } + + async fn register_validators( + &self, + _registrations: &[SignedValidatorRegistrationV1], + ) -> Result<()> { + Ok(()) + } + + async fn publish_proposer_preferences( + &self, + preferences: &[Arc], + ) -> Result<()> { + for signed_preferences in preferences { + // Pass into own fork-choice store so bids for this slot pass the + // `accepted_proposer_preferences` gate in validate_execution_payload_bid. + self.context + .chain + .controller + .on_own_proposer_preferences(signed_preferences.clone_arc()); + + ValidatorToP2p::PublishProposerPreferences(signed_preferences.clone_arc()) + .send(&self.context.p2p_tx); + } + + Ok(()) + } + + async fn submit_builder_preferences(&self, entries: &[BuilderPreferencesEntry]) -> Result<()> { + let payload_builder_api = self.context.chain.block_producer.payload_builder_api(); + + let submissions = entries.iter().map(|entry| async move { + let BuilderPreferencesEntry { + proposer_pubkey, + url, + auth, + max_execution_payment, + } = entry; + + let result = payload_builder_api + .submit_builder_preferences::

( + url, + auth.clone(), + *proposer_pubkey, + *max_execution_payment, + ) + .await; + + (url, proposer_pubkey, result) + }); + + let mut failures = Vec::<(&BuilderUrl, usize, AnyhowError)>::new(); + + for (url, pubkey, result) in join_all(submissions).await { + let Err(error) = result else { + continue; + }; + + debug_with_peers!( + "failed to submit builder preferences to {url} for validator {pubkey:?}: \ + {error:?}", + ); + + match failures + .iter_mut() + .find(|(failed_url, _, _)| *failed_url == url) + { + Some((_, count, _)) => *count = count.saturating_add(1), + None => failures.push((url, 1, error)), + } + } + + ensure!( + failures.is_empty(), + "failed to submit builder preferences: {}", + failures + .iter() + .map(|(url, count, error)| format!("to {url} for {count} proposals ({error:#})")) + .join(", "), + ); + + Ok(()) + } +} + +pub fn ptc_duties_at_epoch( + config: &Config, + state: &BeaconState

, + epoch: Epoch, + indices: &HashSet, +) -> Result> { + // The state carries the committees an epoch ahead through the seed lookahead. + misc::slots_in_epoch::

(epoch)? + .map(|slot| { + accessors::get_ptc(config, state, slot)? + .into_iter() + .filter(|validator_index| indices.contains(validator_index)) + // A validator drawn into several positions has a single duty. + .unique() + .map(|validator_index| { + let pubkey = *accessors::public_key(state, validator_index)?; + + Ok(ValidatorPTCDutyResponse { + pubkey, + validator_index, + slot, + }) + }) + .collect::>>() + }) + .flatten_ok() + .try_collect() +} + +/// The Fulu lookahead fixes proposers an epoch earlier, but it is first filled as the fork epoch +/// begins, so the fork epoch itself keeps the old dependency. +#[must_use] +pub fn proposer_lookahead_applies(config: &Config, epoch: Epoch) -> bool { + config.phase_at_epoch(epoch.saturating_sub(1)) >= Phase::Fulu +} + +#[must_use] +pub fn proposer_dependent_epoch(config: &Config, epoch: Epoch) -> Epoch { + if proposer_lookahead_applies(config, epoch) { + epoch + } else { + epoch.saturating_add(1) + } +} + +pub fn proposer_duties_at_epoch( + config: &Config, + state: &BeaconState

, + epoch: Epoch, +) -> Result> { + misc::slots_in_epoch::

(epoch)? + .map(|slot| { + let validator_index = + accessors::get_beacon_proposer_index_at_slot(config, state, slot)?; + + Ok(ValidatorProposerDutyResponse { + pubkey: *accessors::public_key(state, validator_index)?, + validator_index, + slot, + }) + }) + .collect() +} + +fn sync_duties_at_epoch( + state: &(impl PostAltairBeaconState

+ ?Sized), + epoch: Epoch, + indices: &HashSet, +) -> Result> { + let period = misc::sync_committee_period::

(epoch); + let current_period = misc::sync_committee_period::

(accessors::get_current_epoch(state)); + + // The state carries only the current period's committee and the next. + let committee = if period == current_period { + state.current_sync_committee() + } else if period == current_period.saturating_add(1) { + state.next_sync_committee() + } else { + return Err(anyhow!( + "sync committee of period {period} is not known to a state in period {current_period}", + )); + }; + + let mut duties = BTreeMap::)>::new(); + + for (position, public_key) in committee.pubkeys.iter().enumerate() { + let Some(validator_index) = accessors::index_of_public_key(state, public_key) else { + continue; + }; + + if !indices.contains(&validator_index) { + continue; + } + + duties + .entry(validator_index) + .or_insert_with(|| (*public_key, vec![])) + .1 + .push(position); + } + + Ok(duties + .into_iter() + .map( + |(validator_index, (pubkey, validator_sync_committee_indices))| { + ValidatorSyncDutyResponse { + pubkey, + validator_index, + validator_sync_committee_indices, + } + }, + ) + .collect()) +} + +fn subnet_id( + beacon_state: &BeaconState

, + slot: Slot, + committee_index: CommitteeIndex, +) -> Result { + let committees_per_slot = + accessors::get_committee_count_per_slot(beacon_state, RelativeEpoch::Current)?; + + misc::compute_subnet_for_attestation::

(committees_per_slot, slot, committee_index) +} + +pub fn attester_duties_at_slot( + state: &BeaconState

, + slot: Slot, + indices: &HashSet, +) -> Result> { + let epoch = misc::compute_epoch_at_slot::

(slot); + let relative_epoch = accessors::relative_epoch(state, epoch)?; + let committees_at_slot = accessors::get_committee_count_per_slot(state, relative_epoch)?; + + accessors::beacon_committees(state, slot)? + .zip(0..) + .flat_map(|(committee, committee_index)| { + let committee_length = committee.len(); + + committee + .into_iter() + .enumerate() + .filter(|(_, validator_index)| indices.contains(validator_index)) + .map(move |(validator_committee_index, validator_index)| { + let pubkey = *accessors::public_key(state, validator_index)?; + + Ok(ValidatorAttesterDutyResponse { + committee_index, + committee_length, + committees_at_slot, + pubkey, + slot, + validator_committee_index, + validator_index, + }) + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + // The Fulu fork epoch's proposers are computed as it begins, so they depend on the previous + // epoch's last block like before Fulu; the lookahead moves the root back only after it. + #[test] + fn proposer_duties_share_the_attester_root_of_the_same_epoch_after_the_fulu_fork_epoch() { + let config = Config::mainnet(); + let fulu = config.fulu_fork_epoch; + + assert_eq!(proposer_dependent_epoch(&config, fulu - 1), fulu); + assert_eq!(proposer_dependent_epoch(&config, fulu), fulu + 1); + assert_eq!(proposer_dependent_epoch(&config, fulu + 1), fulu + 1); + assert_eq!(proposer_dependent_epoch(&config, fulu + 2), fulu + 2); + } +} diff --git a/validator/src/messages.rs b/validator/src/messages.rs index d40ade99c..438dd36e6 100644 --- a/validator/src/messages.rs +++ b/validator/src/messages.rs @@ -1,11 +1,17 @@ -use std::collections::HashSet; +use std::{collections::HashSet, sync::Arc}; use anyhow::{Error, Result}; use bls::{PublicKeyBytes, Signature}; use builder_api::unphased::containers::ValidatorRegistrationV1; use futures::channel::{mpsc::UnboundedSender, oneshot::Sender}; use logging::warn_with_peers; -use types::{altair::containers::SignedContributionAndProof, preset::Preset}; +use types::{ + altair::containers::SignedContributionAndProof, + phase0::primitives::{Epoch, H256, Slot}, + preset::Preset, +}; + +use crate::remote_beacon_node::RemoteBeaconNode; pub enum ApiToValidator { RegisteredValidators(Sender>), @@ -26,6 +32,13 @@ impl ApiToValidator

{ pub enum InternalMessage { DoppelgangerProtectionResult(Result<()>), + FinalizedCheckpoint(Epoch), + /// A head one node streamed and reports as not optimistic. + Head { + node: Arc, + slot: Slot, + block_root: H256, + }, } impl InternalMessage { diff --git a/validator/src/misc.rs b/validator/src/misc.rs index 1f4edfa12..9255e0257 100644 --- a/validator/src/misc.rs +++ b/validator/src/misc.rs @@ -1,10 +1,228 @@ +use core::ops::Range; +use std::{collections::HashMap, sync::Arc}; + +use anyhow::{Result, ensure}; +use arithmetic::UsizeExt as _; +use block_producer::BlockProducer; use bls::{PublicKeyBytes, SignatureBytes}; -use ssz::{BitVector, H256}; +use builder_api::BuilderApi; +use clock::Tick; +use eth1_api::ApiController; +use fork_choice_control::{EventChannels, Wait}; +use helper_functions::{ + accessors, + misc::{compute_epoch_at_slot, compute_start_slot_at_epoch}, +}; +use operation_pools::{AttestationAggPool, PayloadAttestationAggPool, SyncCommitteeAggPool}; +use rand::Rng; +use ssz::BitVector; +use tokio::sync::Mutex; +use typenum::{True, U1, U8, Unsigned as _, assert_type, op}; use types::{ - altair::consts::SyncCommitteeSubnetCount, combined::SignedBeaconBlock, - phase0::primitives::ValidatorIndex, preset::Preset, + altair::consts::SyncCommitteeSubnetCount, + config::Config as ChainConfig, + phase0::primitives::{Epoch, H256, Slot, UnixSeconds, ValidatorIndex}, + preset::{Preset, SyncSubcommitteeSize}, +}; + +use crate::{ + own_beacon_committee_members::OwnBeaconCommitteeMembers, + own_proposer_duties::OwnProposerDuties, own_ptc_members::OwnPTCMembers, + own_sync_committee_members::OwnSyncCommitteeMembers, + own_validator_indices::OwnValidatorIndices, remote_beacon_nodes::RemoteBeaconNodes, }; +type ComputeInAdvanceSlots = U8; + +#[must_use] +pub const fn slots_to_compute_in_advance(current_slot: Slot) -> Range { + current_slot..current_slot.saturating_add(ComputeInAdvanceSlots::U64) +} + +pub fn slots_by_epoch(slots: Range) -> impl Iterator)> { + let first = compute_epoch_at_slot::

(slots.start); + let last = compute_epoch_at_slot::

(slots.end.saturating_sub(1)); + + (first..=last).filter_map(move |epoch| { + let start = compute_start_slot_at_epoch::

(epoch).max(slots.start); + let end = compute_start_slot_at_epoch::

(epoch.saturating_add(1)).min(slots.end); + + (start < end).then_some((epoch, start..end)) + }) +} + +/// The validator indices a cache was filled for; a key imported at runtime changes the set. +#[derive(Default)] +pub struct RequestedIndices(Mutex>); + +impl RequestedIndices { + pub async fn changed(&self, validator_indices: &[ValidatorIndex]) -> bool { + let mut requested = self.0.lock().await; + + if **requested == *validator_indices { + return false; + } + + *requested = validator_indices.into(); + true + } +} + +pub fn sync_subnet_subscription_epoch(period_start: Epoch, rng: &mut impl Rng) -> Epoch { + // From the [Altair Honest Validator specification]: + // > To join a sync committee subnet, select a random number of epochs before the end of the + // > current sync committee period between 1 and `SYNC_COMMITTEE_SUBNET_COUNT`, inclusive. + // > Validators should join their member subnet at the beginning of the epoch they have + // > randomly selected. + // + // [Altair Honest Validator specification]: https://github.com/ethereum/consensus-specs/blob/0b76c8367ed19014d104e3fbd4718e73f459a748/specs/altair/validator.md#sync-committee-subnet-stability + period_start.saturating_sub(rng.gen_range(1..=SyncCommitteeSubnetCount::U64)) +} + +pub fn subnets_from_sync_committee_indices( + indices: impl IntoIterator, +) -> Result> { + let mut subnets = BitVector::default(); + + for index in indices { + // Positions come from a remote beacon node and `BitVector::set` panics outside the vector. + ensure!( + index < P::SyncCommitteeSize::USIZE, + "beacon node reported sync committee position {index}, \ + which is outside a committee of {}", + P::SyncCommitteeSize::USIZE, + ); + + subnets.set(index.div_typenum::>(), true); + } + + Ok(subnets) +} + +/// The validator's own duties, cached across slots and refreshed by the tasks it spawns. +pub struct OwnDuties { + /// Shared with the Validator API through [`ChainSource`]. + pub validator_indices: Arc, + pub beacon_committee_members: OwnBeaconCommitteeMembers, + pub proposer_duties: OwnProposerDuties, + pub ptc_members: OwnPTCMembers, + pub sync_committee_members: OwnSyncCommitteeMembers, +} + +/// Where duties are performed and chain facts are read from. +pub struct ChainSource { + pub chain_config: Arc, + pub genesis_time: UnixSeconds, + pub genesis_validators_root: H256, + /// Shared with the Validator API. + pub own_validator_indices: Arc, + pub chain: Chain, +} + +/// The built-in beacon node, or under `vc` the nodes given with `--beacon-node-urls`. +pub enum Chain { + Local(Arc>), + Remote(Arc), +} + +/// The built-in beacon node's handles duties are performed through. +pub struct LocalChain { + pub controller: ApiController, + pub block_producer: Arc>, + pub attestation_agg_pool: Arc>, + pub sync_committee_agg_pool: Arc>, + pub payload_attestation_agg_pool: Arc>, + pub event_channels: Arc>, + pub builder_api: Option>, +} + +impl ChainSource { + pub async fn validator_indices( + &self, + public_keys: &[PublicKeyBytes], + ) -> Result> { + // From the finalized state, so that a reorg cannot change an index once resolved. + match &self.chain { + Chain::Local(local) => { + let state = local.controller.last_finalized_state().value; + + Ok(public_keys + .iter() + .filter_map(|public_key| { + let validator_index = accessors::index_of_public_key(&state, public_key)?; + Some((*public_key, validator_index)) + }) + .collect()) + } + Chain::Remote(remote_beacon_nodes) => { + remote_beacon_nodes.validator_indices(public_keys).await + } + } + } + + pub fn slot(&self) -> Result { + match &self.chain { + Chain::Local(local) => Ok(local.controller.slot()), + Chain::Remote(_) => Ok(Tick::current::

(&self.chain_config, self.genesis_time)?.slot), + } + } + + #[must_use] + pub const fn local(&self) -> Option<&Arc>> { + match &self.chain { + Chain::Local(local) => Some(local), + Chain::Remote(_) => None, + } + } + + #[must_use] + pub fn controller(&self) -> Option<&ApiController> { + self.local().map(|local| &local.controller) + } + + #[must_use] + pub fn block_producer(&self) -> Option<&Arc>> { + self.local().map(|local| &local.block_producer) + } + + #[must_use] + pub fn attestation_agg_pool(&self) -> Option<&Arc>> { + self.local().map(|local| &local.attestation_agg_pool) + } + + #[must_use] + pub fn payload_attestation_agg_pool(&self) -> Option<&Arc>> { + self.local() + .map(|local| &local.payload_attestation_agg_pool) + } + + #[must_use] + pub fn event_channels(&self) -> Option<&Arc>> { + self.local().map(|local| &local.event_channels) + } + + #[must_use] + pub fn builder_api(&self) -> Option<&Arc> { + self.local().and_then(|local| local.builder_api.as_ref()) + } + + #[must_use] + pub fn registers_with_builder(&self) -> bool { + match &self.chain { + Chain::Local(local) => local.builder_api.is_some(), + Chain::Remote(remote_beacon_nodes) => remote_beacon_nodes.use_builder(), + } + } + + #[must_use] + pub const fn remote_beacon_nodes(&self) -> Option<&Arc> { + match &self.chain { + Chain::Local(_) => None, + Chain::Remote(remote_beacon_nodes) => Some(remote_beacon_nodes), + } + } +} + #[expect(clippy::struct_field_names)] pub struct Aggregator { pub aggregator_index: ValidatorIndex, @@ -13,13 +231,95 @@ pub struct Aggregator { pub selection_proof: SignatureBytes, } +#[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct SyncCommitteeMember { pub validator_index: ValidatorIndex, pub public_key: PublicKeyBytes, pub subnets: BitVector, } -pub enum SignedBeaconBlockOrBlockRoot { - Block(Box>), - Root(H256), +#[cfg(target_pointer_width = "32")] +use typenum::U32; + +#[cfg(target_pointer_width = "64")] +use typenum::U64; + +#[cfg(target_pointer_width = "32")] +assert_type!(op!(ComputeInAdvanceSlots < U1 << U32)); + +#[cfg(target_pointer_width = "64")] +assert_type!(op!(ComputeInAdvanceSlots < U1 << U64)); + +#[cfg(test)] +mod tests { + use types::preset::Minimal; + + use super::*; + + #[test] + fn slots_by_epoch_yields_nothing_for_an_empty_range() { + assert!(grouped(0..0).is_empty()); + assert!(grouped(9..9).is_empty()); + } + + #[test] + fn slots_by_epoch_keeps_a_range_inside_one_epoch_whole() { + assert_eq!(grouped(3..7), [(0, 3..7)]); + assert_eq!(grouped(8..16), [(1, 8..16)]); + } + + #[test] + fn slots_by_epoch_splits_a_range_at_epoch_boundaries() { + assert_eq!(grouped(6..19), [(0, 6..8), (1, 8..16), (2, 16..19)]); + } + + #[test] + fn slots_by_epoch_groups_the_slots_computed_in_advance() { + assert_eq!( + grouped(slots_to_compute_in_advance(5)), + [(0, 5..8), (1, 8..13)] + ); + } + + // Under the minimal preset a sync committee holds 32 members across 4 subnets, so each + // subcommittee spans 8 positions. + #[test] + fn a_position_maps_to_the_subnet_of_its_subcommittee() -> Result<()> { + assert_eq!(subnets(&[0])?, [true, false, false, false]); + assert_eq!(subnets(&[7])?, [true, false, false, false]); + assert_eq!(subnets(&[8])?, [false, true, false, false]); + assert_eq!(subnets(&[31])?, [false, false, false, true]); + + Ok(()) + } + + #[test] + fn a_validator_in_several_subcommittees_joins_every_subnet() -> Result<()> { + assert_eq!(subnets(&[3, 3])?, [true, false, false, false]); + assert_eq!(subnets(&[3, 24])?, [true, false, false, true]); + + Ok(()) + } + + #[test] + fn no_position_means_no_subnet() -> Result<()> { + assert_eq!(subnets(&[])?, [false, false, false, false]); + + Ok(()) + } + + #[test] + fn a_position_outside_the_committee_is_rejected() { + subnets(&[32]).expect_err("position 32 is outside a committee of 32"); + subnets(&[usize::MAX]).expect_err("position usize::MAX is outside a committee of 32"); + } + + fn subnets(indices: &[usize]) -> Result> { + subnets_from_sync_committee_indices::(indices.iter().copied()) + .map(|subnets| subnets.into_iter().collect()) + } + + fn grouped(slots: Range) -> Vec<(Epoch, Range)> { + slots_by_epoch::(slots).collect() + } } diff --git a/validator/src/own_beacon_committee_members.rs b/validator/src/own_beacon_committee_members.rs index 8e5a21d23..3a70d57c9 100644 --- a/validator/src/own_beacon_committee_members.rs +++ b/validator/src/own_beacon_committee_members.rs @@ -1,25 +1,39 @@ -use std::{collections::HashMap, sync::Arc}; +use core::ops::Range; +use std::{ + collections::{BTreeMap, HashSet}, + sync::Arc, +}; use anyhow::Result; use bls::{PublicKeyBytes, SignatureBytes}; -use helper_functions::{accessors, misc, predicates, signing::SignForSingleFork as _}; -use logging::warn_with_peers; +use eth1_api::ApiController; +use fork_choice_control::Wait; +use helper_functions::{misc, predicates, signing::SignForSingleFork}; +use http_api_utils::ValidatorAttesterDutyResponse; +use logging::{debug_with_peers, warn_with_peers}; use p2p::BeaconCommitteeSubscription; -use scc::HashMap as SccHashMap; +use scc::{HashMap as SccHashMap, hash_map::Entry}; use signer::{Signer, SigningMessage, SigningTriple}; use ssz::H256; use std_ext::ArcExt; -use tap::{Conv as _, Pipe as _}; use tracing::instrument; -use typenum::{True, U1, U8, Unsigned as _, assert_type, op}; +use typenum::Unsigned as _; use types::{ combined::BeaconState, config::Config as ChainConfig, - phase0::primitives::{CommitteeIndex, Slot, ValidatorIndex}, + nonstandard::ForkInfo, + phase0::primitives::{CommitteeIndex, Epoch, Slot, ValidatorIndex}, preset::Preset, }; -type ComputeInAdvanceSlots = U8; +use crate::{ + beacon_node_api::{AttesterDuties, BeaconNodeApi as _}, + beacon_nodes::BeaconNodes, + local_beacon_node::attester_duties_at_slot, + misc::{RequestedIndices, slots_by_epoch}, +}; + +type MembersBySlot = BTreeMap>; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct BeaconCommitteeMember { @@ -31,7 +45,7 @@ pub struct BeaconCommitteeMember { pub position_in_committee: usize, pub slot: Slot, pub is_aggregator: bool, - pub selection_proof: Option, + pub selection_proof: SignatureBytes, } impl From for BeaconCommitteeSubscription { @@ -55,10 +69,29 @@ impl From for BeaconCommitteeSubscription { } } +/// One epoch's members under one shuffling. +struct EpochMembers { + dependent_root: H256, + members_by_slot: MembersBySlot, + /// Set when the key set changes; kept until replaced. + stale_for_other_keys: bool, +} + +impl EpochMembers { + const fn new(dependent_root: H256, members_by_slot: MembersBySlot) -> Self { + Self { + dependent_root, + members_by_slot, + stale_for_other_keys: false, + } + } +} + pub struct OwnBeaconCommitteeMembers { config: Arc, signer: Arc, - members: SccHashMap<(H256, Slot), Arc<[BeaconCommitteeMember]>>, + members: SccHashMap, + requested: RequestedIndices, } impl OwnBeaconCommitteeMembers { @@ -67,6 +100,20 @@ impl OwnBeaconCommitteeMembers { config, signer, members: SccHashMap::new(), + requested: RequestedIndices::default(), + } + } + + async fn mark_stale_for_other_keys(&self, validator_indices: &[ValidatorIndex]) { + if self.requested.changed(validator_indices).await { + // Kept until replaced: the duties of the keys still held remain valid, and signing + // an epoch's selection proofs can outlast the attestation deadline. + self.members + .iter_mut_async(|mut entry| { + entry.stale_for_other_keys = true; + true + }) + .await; } } @@ -74,203 +121,369 @@ impl OwnBeaconCommitteeMembers { self.members.len() } - #[instrument(skip_all, level = "debug", fields(slot = slot, dependent_root = ?dependent_root))] - pub async fn get_or_init_at_slot( + #[instrument(skip_all, level = "debug", fields(slot = slot))] + pub async fn get_or_init_at_slot( &self, - state: &BeaconState

, - dependent_root: H256, + controller: Option<&ApiController>, + beacon_state: Option<&Arc>>, slot: Slot, - ) -> Option> { - if let Some(members) = self.members.get_async(&(dependent_root, slot)).await { - return Some(members.clone_arc()); - } + validator_indices: &[ValidatorIndex], + ) -> Result, bool)>> { + let local = controller.zip(beacon_state); - match self.compute_members_at_slot(state, slot).await { - Ok(members) => { - if let Some(members) = members { - self.members - .upsert_async((dependent_root, slot), members.clone_arc()) - .await; + let dependent_root = match local { + Some((controller, state)) => { + // The prefetch's finalized indices, not the head's: a validator only in the head + // cannot have duties yet, and a differing list would recompute them every slot. + self.mark_stale_for_other_keys(validator_indices).await; - Some(members) - } else { - None - } + controller.attestation_committee_dependent_root_for_slot(state, slot)? } - Err(error) => { - warn_with_peers!( - "failed to compute own beacon committee members at slot {slot}: {error:?}" - ); - None + None => { + let epoch = misc::compute_epoch_at_slot::

(slot); + + match self.cached_dependent_root(epoch).await { + Some(dependent_root) => dependent_root, + None => { + // Without a validator index there are no duties to prefetch. + if !validator_indices.is_empty() { + warn_with_peers!( + "no attester duties were prefetched for slot {slot}, so no \ + attestation will be produced", + ); + } + + return Ok(None); + } + } } + }; + + let needs_to_compute = self + .needs_to_compute_members_at_slot::

(dependent_root, slot) + .await; + + if needs_to_compute && let Some((_, state)) = local { + self.init_at_slot(state, dependent_root, slot, validator_indices) + .await?; } + + // A concurrent computation under another root may have replaced the epoch's entry since + // the check above; members of another shuffling are not returned. + let epoch = misc::compute_epoch_at_slot::

(slot); + + Ok(self + .members + .get_async(&epoch) + .await + .filter(|entry| entry.dependent_root == dependent_root) + .and_then(|entry| entry.members_by_slot.get(&slot).map(ArcExt::clone_arc)) + .map(|members| (members, needs_to_compute))) } - pub fn slots_to_compute_in_advance(current_slot: Slot) -> impl Iterator { - current_slot..current_slot.saturating_add(ComputeInAdvanceSlots::U64) + async fn init_at_slot( + &self, + state: &BeaconState

, + dependent_root: H256, + slot: Slot, + own_validator_indices: &[ValidatorIndex], + ) -> Result<()> { + let indices = own_validator_indices + .iter() + .copied() + .collect::>(); + + let duties = + tokio::task::block_in_place(|| attester_duties_at_slot(state, slot, &indices))?; + + self.init_from_duties::

( + ForkInfo::from_state(state), + dependent_root, + slot..slot.saturating_add(1), + duties, + ) + .await } - pub async fn needs_to_compute_members_at_slot(&self, dependent_root: H256, slot: Slot) -> bool { - !self.members.contains_async(&(dependent_root, slot)).await + pub async fn cached_dependent_root(&self, epoch: Epoch) -> Option { + self.members + .get_async(&epoch) + .await + .map(|entry| entry.dependent_root) } - pub async fn prune(&self, up_to_slot: Slot) { + pub async fn get_at_slot(&self, slot: Slot) -> Option> { + let epoch = misc::compute_epoch_at_slot::

(slot); + self.members - .retain_async(|(_, slot), _| *slot >= up_to_slot) + .get_async(&epoch) .await + .and_then(|entry| entry.members_by_slot.get(&slot).map(ArcExt::clone_arc)) } - #[instrument(skip_all, level = "debug", fields(slot = slot))] - async fn compute_members_at_slot( + pub async fn init_at_slots( &self, - state: &BeaconState

, - slot: Slot, - ) -> Result>> { - let signer_snapshot = self.signer.load(); + beacon_nodes: &BeaconNodes, + fork_info: ForkInfo, + slots: Range, + validator_indices: &[ValidatorIndex], + ) -> Result<()> { + self.mark_stale_for_other_keys(validator_indices).await; + + // Split per epoch because a shuffling, and so a dependent root, covers exactly one. + for (_, slots) in slots_by_epoch::

(slots) { + let AttesterDuties { + dependent_root, + duties, + } = beacon_nodes + .attester_duties_at_slots(slots.clone(), validator_indices) + .await?; + + // A fetch that ran long can land after a reorg moved the root; the nodes' current one + // decides, so that a slower task from an earlier slot cannot replace newer duties. + let epoch = misc::compute_epoch_at_slot::

(slots.start); + + if let Some(validator_index) = validator_indices.first().copied() + && beacon_nodes.dependent_root(epoch, validator_index).await? != dependent_root + { + debug_with_peers!( + "discarding attester duties for epoch {epoch} under {dependent_root:?}, \ + which is no longer the dependent root", + ); - let own_public_keys = signer_snapshot - .keys() - .copied() - .filter_map(|public_key| { - let validator_index = accessors::index_of_public_key(state, &public_key)?; - Some((validator_index, public_key)) - }) - .collect::>(); + continue; + } - if own_public_keys.is_empty() { - return Ok(None); + self.init_from_duties::

(fork_info, dependent_root, slots, duties) + .await?; } - let mut members = vec![]; - let mut triples = vec![]; + Ok(()) + } - let beacon_committees = (0..).zip(accessors::beacon_committees(state, slot)?); + async fn needs_to_compute_members_at_slot( + &self, + dependent_root: H256, + slot: Slot, + ) -> bool { let epoch = misc::compute_epoch_at_slot::

(slot); - let relative_epoch = accessors::relative_epoch(state, epoch)?; - let committees_at_slot = accessors::get_committee_count_per_slot(state, relative_epoch)?; - for (committee_index, committee) in beacon_committees { - let committee_size = committee.len(); + self.members.get_async(&epoch).await.is_none_or(|entry| { + entry.stale_for_other_keys + || entry.dependent_root != dependent_root + || !entry.members_by_slot.contains_key(&slot) + }) + } + + async fn slots_to_compute( + &self, + dependent_root: H256, + slots: Range, + ) -> Option> { + let mut first = None; + let mut last = None; + + for slot in slots { + if self + .needs_to_compute_members_at_slot::

(dependent_root, slot) + .await + { + first.get_or_insert(slot); + last = Some(slot); + } + } + + Some(first?..last?.saturating_add(1)) + } - for (position_in_committee, validator_index) in committee.into_iter().enumerate() { - let Some(public_key) = own_public_keys.get(&validator_index).copied() else { - continue; - }; + pub async fn slots_to_compute_at_epoch( + &self, + beacon_nodes: &BeaconNodes, + epoch: Epoch, + slots: Range, + validator_indices: &[ValidatorIndex], + ) -> Result>> { + self.mark_stale_for_other_keys(validator_indices).await; + + // The request that fetches the duties reports the dependent root anyway. + if self.cached_dependent_root(epoch).await.is_none() { + return Ok(Some(slots)); + } - members.push(BeaconCommitteeMember { - public_key, - validator_index, - committee_index, - committee_size, - committees_at_slot, - position_in_committee, - slot, - is_aggregator: false, - selection_proof: None, - }); + let Some(validator_index) = validator_indices.first().copied() else { + return Ok(None); + }; - triples.push(SigningTriple::

{ - message: SigningMessage::AggregationSlot { slot }, - signing_root: slot.signing_root(&self.config, state), - public_key, - }); + let dependent_root = beacon_nodes.dependent_root(epoch, validator_index).await?; + + Ok(self.slots_to_compute::

(dependent_root, slots).await) + } + + pub async fn prune(&self, up_to_slot: Slot) { + let up_to_epoch = misc::compute_epoch_at_slot::

(up_to_slot); + + self.members + .retain_async(|epoch, _| *epoch >= up_to_epoch) + .await; + } + + #[instrument(skip_all, level = "debug", fields(dependent_root = ?dependent_root))] + async fn init_from_duties( + &self, + fork_info: ForkInfo, + dependent_root: H256, + slots: Range, + duties: Vec, + ) -> Result<()> { + if slots.is_empty() { + return Ok(()); + } + + let epoch = misc::compute_epoch_at_slot::

(slots.start); + + let mut members_by_slot = BTreeMap::>::new(); + + for slot in slots { + if self + .needs_to_compute_members_at_slot::

(dependent_root, slot) + .await + { + // Cached as empty so that a later pass does not mistake "nothing to do" for + // "not computed yet". + members_by_slot.insert(slot, vec![]); } } + let signer_snapshot = self.signer.load(); + + // A duty for a key we do not hold would fail the whole signing batch. + let mut duties = duties + .into_iter() + .filter(|duty| { + signer_snapshot.has_key(duty.pubkey) && members_by_slot.contains_key(&duty.slot) + }) + .collect::>(); + + // Members keep the order the committees are laid out in: by committee, then by position + // within it. + duties.sort_unstable_by_key(|duty| { + ( + duty.slot, + duty.committee_index, + duty.validator_committee_index, + ) + }); + + let triples = duties + .iter() + .map(|duty| SigningTriple::

{ + message: SigningMessage::AggregationSlot { slot: duty.slot }, + signing_root: SignForSingleFork::

::signing_root_from_fork_info( + &duty.slot, + &self.config, + fork_info, + ), + public_key: duty.pubkey, + }) + .collect::>(); + + // A whole epoch of proofs is signed at once. let selection_proofs = signer_snapshot - .sign_triples_without_slashing_protection(triples, Some(state.into())) + .sign_triples_throttled_without_slashing_protection(triples, Some(fork_info)) .await?; - members - .into_iter() - .zip(selection_proofs) - .map(|(member, selection_proof)| { - let BeaconCommitteeMember { - public_key, - validator_index, - committee_index, - committee_size, - committees_at_slot, - position_in_committee, - slot, - .. - } = member; + for (duty, selection_proof) in duties.into_iter().zip(selection_proofs) { + let ValidatorAttesterDutyResponse { + committee_index, + committee_length, + committees_at_slot, + pubkey, + slot, + validator_committee_index, + validator_index, + } = duty; + + // `own_singular_attestations` sizes and indexes a `BitList` by these pre-Electra, and + // `BitList` panics outside its bounds. + if validator_committee_index >= committee_length + || committee_length > P::MaxValidatorsPerCommittee::USIZE + { + warn_with_peers!( + "skipping attester duty of validator {validator_index} in slot {slot}: \ + beacon node reported position {validator_committee_index} \ + in a committee of {committee_length}", + ); + continue; + } - let is_aggregator = predicates::is_aggregator( - state, - slot, - committee_index, - selection_proof.into(), - )?; + let selection_proof = SignatureBytes::from(selection_proof); - Ok(BeaconCommitteeMember { - public_key, + let is_aggregator = + predicates::is_aggregator_from_committee_length(committee_length, selection_proof)?; + + members_by_slot + .entry(slot) + .or_default() + .push(BeaconCommitteeMember { + public_key: pubkey, validator_index, committee_index, - committee_size, + committee_size: committee_length, committees_at_slot, - position_in_committee, + position_in_committee: validator_committee_index, slot, is_aggregator, - selection_proof: Some(selection_proof.into()), - }) - }) - .collect::>>()? - .conv::>() - .pipe(Some) - .pipe(Ok) - } -} - -#[cfg(target_pointer_width = "32")] -use typenum::U32; - -#[cfg(target_pointer_width = "64")] -use typenum::U64; + selection_proof, + }); + } -#[cfg(target_pointer_width = "32")] -assert_type!(op!(ComputeInAdvanceSlots < U1 << U32)); + let members_by_slot = members_by_slot + .into_iter() + .map(|(slot, members)| (slot, members.into())) + .collect::>>(); + + // A different root means a different shuffling, so what was cached under the old one no + // longer applies. Neither does what was cached for another set of keys. + match self.members.entry_async(epoch).await { + Entry::Occupied(mut occupied) => { + if occupied.dependent_root == dependent_root && !occupied.stale_for_other_keys { + occupied.members_by_slot.extend(members_by_slot); + } else { + *occupied.get_mut() = EpochMembers::new(dependent_root, members_by_slot); + } + } + Entry::Vacant(vacant) => { + vacant.insert_entry(EpochMembers::new(dependent_root, members_by_slot)); + } + } -#[cfg(target_pointer_width = "64")] -assert_type!(op!(ComputeInAdvanceSlots < U1 << U64)); + Ok(()) + } +} #[cfg(test)] mod tests { use bls::traits::SecretKey as _; + use itertools::Itertools as _; use pubkey_cache::PubkeyCache; use reqwest::Client; use signer::{KeyOrigin, Web3SignerConfig}; use types::preset::Minimal; use super::*; + use crate::misc::slots_to_compute_in_advance; - #[tokio::test] - async fn test_computing_own_beacon_committee_members() -> Result<()> { - // beacon committees at slot 1: - // - // committee 0: validators 41, 45, 0, 24 - // committee 1: validators 17, 47, 40, 29 - // - // beacon committees at slot 2: - // - // committee 0: validators 6, 50, 33, 11 - // committee 1: validators 2, 54, 22, 63 - // - let secret_key_1 = Arc::new(interop::secret_key(40)); - let secret_key_2 = Arc::new(interop::secret_key(41)); - let secret_key_3 = Arc::new(interop::secret_key(42)); - - let public_key_1 = secret_key_1.to_public_key().into(); - let public_key_2 = secret_key_2.to_public_key().into(); - let public_key_3 = secret_key_3.to_public_key().into(); + fn own_members_with_keys() -> Result<(OwnBeaconCommitteeMembers, Arc>)> { + let secret_keys = [40, 41, 42].map(|index| Arc::new(interop::secret_key(index))); let signer = Arc::new(Signer::new( - [ - (public_key_1, secret_key_1, KeyOrigin::External), - (public_key_2, secret_key_2, KeyOrigin::External), - (public_key_3, secret_key_3, KeyOrigin::External), - ], - Client::new(), + secret_keys.map(|secret_key| { + ( + secret_key.to_public_key().into(), + secret_key, + KeyOrigin::External, + ) + }), Client::new(), Web3SignerConfig::default(), None, @@ -280,17 +493,77 @@ mod tests { let pubkey_cache = PubkeyCache::default(); let (state, _) = factory::min_genesis_state::(&config, &pubkey_cache)?; - let own_members = OwnBeaconCommitteeMembers::new(config, signer); - let dependent_root = H256::zero(); + Ok((OwnBeaconCommitteeMembers::new(config, signer), state)) + } + fn public_key(validator_index: ValidatorIndex) -> PublicKeyBytes { + interop::secret_key(validator_index).to_public_key().into() + } + + // The duties the built-in node derives from a state carry the committee layout the remote + // nodes report, so the two sources feed the cache alike. + #[test] + fn attester_duties_at_slot_match_the_committees_of_the_state() -> Result<()> { + let (_, state) = own_members_with_keys()?; + let indices = HashSet::from([40, 41]); + + let duties = attester_duties_at_slot(&state, 1, &indices)?; + + let positions = duties + .iter() + .map(|duty| { + ( + duty.validator_index, + duty.committee_index, + duty.validator_committee_index, + ) + }) + .sorted() + .collect::>(); + + assert_eq!(positions, [(40, 1, 2), (41, 0, 0)]); + assert!(duties.iter().all(|duty| duty.committees_at_slot == 2)); + assert!(duties.iter().all(|duty| duty.committee_length == 4)); assert!( - own_members - .needs_to_compute_members_at_slot(dependent_root, 1) - .await + duties + .iter() + .all(|duty| duty.pubkey == public_key(duty.validator_index)) ); + Ok(()) + } + + // Members are ordered by committee and position, whatever order the duties arrive in. + #[tokio::test] + async fn test_computing_own_beacon_committee_members() -> Result<()> { + let (own_members, state) = own_members_with_keys()?; + let public_key_1 = public_key(40); + let public_key_2 = public_key(41); + let dependent_root = H256::zero(); + + for slot in slots_to_compute_in_advance(1) { + assert!( + own_members + .needs_to_compute_members_at_slot::(dependent_root, slot) + .await + ); + } + + // Deliberately out of order, to check that members come back committee-major. + own_members + .init_from_duties::( + ForkInfo::from_state(state.as_ref()), + dependent_root, + misc::slots_in_epoch::(0)?, + vec![ + duty(public_key_1, 40, 1, 2, 1), + duty(public_key_2, 41, 0, 0, 1), + ], + ) + .await?; + let members_at_slot_1 = own_members - .get_or_init_at_slot(&state, dependent_root, 1) + .get_at_slot::(1) .await .expect("there should be 2 own beacon committee members at slot 1"); @@ -328,35 +601,189 @@ mod tests { ], ); + // One request covers the epoch, so a slot without duties is cached as empty rather than + // left looking uncomputed. + assert_eq!( + *own_members + .get_at_slot::(2) + .await + .expect("slots without duties are cached as empty"), + vec![], + ); + + assert_eq!( + own_members + .slots_to_compute::(dependent_root, misc::slots_in_epoch::(0)?) + .await, + None, + ); + + // Nothing was said about the next epoch. assert!( - !own_members - .needs_to_compute_members_at_slot(dependent_root, 1) + own_members + .needs_to_compute_members_at_slot::( + dependent_root, + misc::compute_start_slot_at_epoch::(1), + ) .await ); - for slot in OwnBeaconCommitteeMembers::slots_to_compute_in_advance(2) { - assert!( - own_members - .needs_to_compute_members_at_slot(dependent_root, slot) - .await - ); + Ok(()) + } + + // A different root is a different shuffling, so what was cached under the old one goes. + #[tokio::test] + async fn a_changed_dependent_root_replaces_the_epoch() -> Result<()> { + let (own_members, state) = own_members_with_keys()?; + let public_key = public_key(41); + let old_root = H256::repeat_byte(1); + let new_root = H256::repeat_byte(2); + + for (dependent_root, slot) in [(old_root, 1), (new_root, 2)] { + own_members + .init_from_duties::( + ForkInfo::from_state(state.as_ref()), + dependent_root, + slot..slot.saturating_add(1), + vec![duty(public_key, 41, 0, 0, slot)], + ) + .await?; } - let members_at_slot_2 = own_members - .get_or_init_at_slot(&state, dependent_root, 2) - .await; + assert!(own_members.get_at_slot::(1).await.is_none()); + assert!(own_members.get_at_slot::(2).await.is_some()); - assert_eq!( - *members_at_slot_2.expect("should return empty collection"), - vec![], + assert!( + own_members + .needs_to_compute_members_at_slot::(old_root, 2) + .await ); assert!( !own_members - .needs_to_compute_members_at_slot(dependent_root, 2) + .needs_to_compute_members_at_slot::(new_root, 2) .await ); Ok(()) } + + // Duties stay served while those of a changed key set are fetched, and the fetch then + // replaces the epoch rather than adding to it. + #[tokio::test] + async fn a_changed_key_set_keeps_the_epoch_until_it_is_replaced() -> Result<()> { + let (own_members, state) = own_members_with_keys()?; + let fork_info = ForkInfo::from_state(state.as_ref()); + let dependent_root = H256::zero(); + let slots = misc::slots_in_epoch::(0)?; + + own_members + .init_from_duties::( + fork_info, + dependent_root, + slots.clone(), + vec![duty(public_key(40), 40, 1, 2, 1)], + ) + .await?; + + own_members.mark_stale_for_other_keys(&[40, 41]).await; + + let members_at_slot_1 = own_members + .get_at_slot::(1) + .await + .expect("the duties are kept while their replacement is fetched"); + + assert_eq!(members_at_slot_1.len(), 1); + + assert_eq!( + own_members.cached_dependent_root(0).await, + Some(dependent_root), + ); + + assert_eq!( + own_members + .slots_to_compute::(dependent_root, slots.clone()) + .await, + Some(slots.clone()), + ); + + own_members + .init_from_duties::( + fork_info, + dependent_root, + slots.clone(), + vec![duty(public_key(41), 41, 0, 0, 2)], + ) + .await?; + + let members_at_slot_1 = own_members + .get_at_slot::(1) + .await + .expect("slots without duties are cached as empty"); + + assert!(members_at_slot_1.is_empty()); + + let members_at_slot_2 = own_members + .get_at_slot::(2) + .await + .expect("the new duties should be cached"); + + assert_eq!(members_at_slot_2.len(), 1); + assert_eq!(members_at_slot_2[0].validator_index, 41); + + assert_eq!( + own_members + .slots_to_compute::(dependent_root, slots) + .await, + None, + ); + + Ok(()) + } + + // A position at or past the committee length would panic when the attestation bitlist is set. + #[tokio::test] + async fn skips_a_duty_whose_position_is_outside_the_committee() -> Result<()> { + let (own_members, state) = own_members_with_keys()?; + + own_members + .init_from_duties::( + ForkInfo::from_state(state.as_ref()), + H256::zero(), + misc::slots_in_epoch::(0)?, + vec![ + duty(public_key(40), 40, 1, 4, 1), + duty(public_key(41), 41, 0, 0, 1), + ], + ) + .await?; + + let members_at_slot_1 = own_members + .get_at_slot::(1) + .await + .expect("the valid duty should still be kept"); + + assert_eq!(members_at_slot_1.len(), 1); + assert_eq!(members_at_slot_1[0].validator_index, 41); + + Ok(()) + } + + fn duty( + pubkey: PublicKeyBytes, + validator_index: ValidatorIndex, + committee_index: CommitteeIndex, + validator_committee_index: usize, + slot: Slot, + ) -> ValidatorAttesterDutyResponse { + ValidatorAttesterDutyResponse { + committee_index, + committee_length: 4, + committees_at_slot: 2, + pubkey, + slot, + validator_committee_index, + validator_index, + } + } } diff --git a/validator/src/own_proposer_duties.rs b/validator/src/own_proposer_duties.rs new file mode 100644 index 000000000..82c41e352 --- /dev/null +++ b/validator/src/own_proposer_duties.rs @@ -0,0 +1,280 @@ +use std::sync::Arc; + +use anyhow::{Result, ensure}; +use bls::PublicKeyBytes; +use fork_choice_control::Wait; +use helper_functions::misc; +use http_api_utils::ValidatorProposerDutyResponse; +use logging::debug_with_peers; +use scc::{HashMap as SccHashMap, HashSet as SccHashSet}; +use signer::Signer; +use ssz::H256; +use types::{ + phase0::primitives::{Epoch, Slot, ValidatorIndex}, + preset::Preset, +}; + +use crate::{ + beacon_node_api::{BeaconNodeApi as _, ProposerDuties}, + beacon_nodes::BeaconNodes, + misc::RequestedIndices, +}; + +/// Own proposals of the epochs in flight, from duties. +pub struct OwnProposerDuties { + signer: Arc, + proposers: SccHashMap<(H256, Slot), (ValidatorIndex, PublicKeyBytes)>, + /// Most slots have no entry, so a fetched epoch has to be recorded on its own. + fetched: SccHashSet<(H256, Epoch)>, + requested: RequestedIndices, +} + +impl OwnProposerDuties { + pub fn new(signer: Arc) -> Self { + Self { + signer, + proposers: SccHashMap::new(), + fetched: SccHashSet::new(), + requested: RequestedIndices::default(), + } + } + + async fn discard_for_other_keys(&self, validator_indices: &[ValidatorIndex]) { + if self.requested.changed(validator_indices).await { + self.proposers.clear_async().await; + self.fetched.clear_async().await; + } + } + + pub fn len(&self) -> usize { + self.proposers.len() + } + + pub async fn get_at_slot( + &self, + dependent_root: H256, + slot: Slot, + ) -> Option<(ValidatorIndex, PublicKeyBytes)> { + self.proposers + .read_async(&(dependent_root, slot), |_, proposer| *proposer) + .await + } + + pub async fn init_at_epoch( + &self, + beacon_nodes: &BeaconNodes, + epoch: Epoch, + dependent_root: H256, + validator_indices: &[ValidatorIndex], + ) -> Result<()> { + self.discard_for_other_keys(validator_indices).await; + + if self.is_fetched(dependent_root, epoch).await { + return Ok(()); + } + + let ProposerDuties { + dependent_root: reported_root, + duties, + } = beacon_nodes.proposer_duties(epoch).await?; + + // Another root is another shuffling; the duties cannot stand in for the ones asked for. + ensure!( + reported_root == dependent_root, + "proposer duties for epoch {epoch} were reported under dependent root \ + {reported_root:?} rather than {dependent_root:?}", + ); + + self.record(dependent_root, epoch, duties).await; + + Ok(()) + } + + async fn record( + &self, + dependent_root: H256, + epoch: Epoch, + duties: Vec, + ) { + let signer_snapshot = self.signer.load(); + + for duty in duties { + if !signer_snapshot.has_key(duty.pubkey) { + continue; + } + + self.proposers + .upsert_async( + (dependent_root, duty.slot), + (duty.validator_index, duty.pubkey), + ) + .await; + } + + if self + .fetched + .insert_async((dependent_root, epoch)) + .await + .is_err() + { + debug_with_peers!( + "proposer duties for epoch {epoch} were already fetched under {dependent_root:?}", + ); + } + } + + pub async fn upcoming(&self, slot: Slot) -> Vec<(H256, Slot, ValidatorIndex, PublicKeyBytes)> { + let mut proposals = vec![]; + + self.proposers + .iter_async( + |(dependent_root, proposal_slot), (validator_index, pubkey)| { + if *proposal_slot > slot { + proposals.push(( + *dependent_root, + *proposal_slot, + *validator_index, + *pubkey, + )); + } + + true + }, + ) + .await; + + proposals.sort_unstable(); + + proposals + } + + async fn is_fetched(&self, dependent_root: H256, epoch: Epoch) -> bool { + self.fetched.contains_async(&(dependent_root, epoch)).await + } + + pub async fn prune(&self, up_to_slot: Slot) { + let up_to_epoch = misc::compute_epoch_at_slot::

(up_to_slot); + + self.proposers + .retain_async(|(_, slot), _| *slot >= up_to_slot) + .await; + + self.fetched + .retain_async(|(_, epoch)| *epoch >= up_to_epoch) + .await; + } +} + +#[cfg(test)] +mod tests { + use bls::traits::SecretKey as _; + use reqwest::Client; + use signer::{KeyOrigin, Web3SignerConfig}; + use types::preset::Minimal; + + use super::*; + + fn own_duties_with_keys(indices: [ValidatorIndex; 2]) -> OwnProposerDuties { + let secret_keys = indices.map(|index| Arc::new(interop::secret_key(index))); + + let signer = Arc::new(Signer::new( + secret_keys.map(|secret_key| { + ( + secret_key.to_public_key().into(), + secret_key, + KeyOrigin::External, + ) + }), + Client::new(), + Web3SignerConfig::default(), + None, + )); + + OwnProposerDuties::new(signer) + } + + fn duty(validator_index: ValidatorIndex, slot: Slot) -> ValidatorProposerDutyResponse { + ValidatorProposerDutyResponse { + pubkey: interop::secret_key(validator_index).to_public_key().into(), + validator_index, + slot, + } + } + + // The endpoint lists every proposer of the epoch; only the keys held are worth keeping, and a + // fetched epoch must be told apart from one with no own proposals. + #[tokio::test] + async fn only_own_proposals_are_kept_and_the_epoch_is_recorded() { + let own_duties = own_duties_with_keys([40, 41]); + let root = H256::repeat_byte(1); + + own_duties + .record(root, 1, vec![duty(40, 8), duty(7, 9), duty(41, 15)]) + .await; + + assert_eq!( + own_duties.get_at_slot(root, 8).await, + Some((40, duty(40, 8).pubkey)), + ); + assert_eq!(own_duties.get_at_slot(root, 9).await, None); + assert_eq!(own_duties.len(), 2); + assert!(own_duties.is_fetched(root, 1).await); + assert!(!own_duties.is_fetched(root, 2).await); + } + + // Duties of another shuffling must never answer for the one asked about. + #[tokio::test] + async fn duties_are_only_found_under_the_root_they_were_fetched_under() { + let own_duties = own_duties_with_keys([40, 41]); + + own_duties + .record(H256::repeat_byte(1), 1, vec![duty(40, 8)]) + .await; + + assert_eq!(own_duties.get_at_slot(H256::repeat_byte(2), 8).await, None); + assert!(!own_duties.is_fetched(H256::repeat_byte(2), 1).await); + } + + // Preferences are signed for proposals still ahead, under every root they are known for. + #[tokio::test] + async fn upcoming_lists_later_proposals_under_each_root() { + let own_duties = own_duties_with_keys([40, 41]); + let old_root = H256::repeat_byte(1); + let new_root = H256::repeat_byte(2); + + own_duties + .record(old_root, 1, vec![duty(40, 8), duty(41, 12)]) + .await; + own_duties + .record(new_root, 1, vec![duty(41, 13), duty(7, 14)]) + .await; + + assert_eq!( + own_duties.upcoming(8).await, + vec![ + (old_root, 12, 41, duty(41, 12).pubkey), + (new_root, 13, 41, duty(41, 13).pubkey), + ], + ); + assert_eq!(own_duties.upcoming(13).await, vec![]); + } + + #[tokio::test] + async fn pruning_drops_past_epochs_only() { + let own_duties = own_duties_with_keys([40, 41]); + let root = H256::repeat_byte(1); + + own_duties.record(root, 0, vec![duty(40, 3)]).await; + own_duties.record(root, 1, vec![duty(41, 12)]).await; + + own_duties.prune::(8).await; + + assert_eq!(own_duties.get_at_slot(root, 3).await, None); + assert!(!own_duties.is_fetched(root, 0).await); + assert_eq!( + own_duties.get_at_slot(root, 12).await, + Some((41, duty(41, 12).pubkey)), + ); + assert!(own_duties.is_fetched(root, 1).await); + } +} diff --git a/validator/src/own_ptc_members.rs b/validator/src/own_ptc_members.rs index e983bbe23..e6bd35a1f 100644 --- a/validator/src/own_ptc_members.rs +++ b/validator/src/own_ptc_members.rs @@ -1,8 +1,13 @@ -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; -use anyhow::Result; +use anyhow::{Result, ensure}; use bls::PublicKeyBytes; -use helper_functions::accessors; +use fork_choice_control::Wait; +use helper_functions::{accessors, misc}; +use itertools::Itertools as _; use logging::warn_with_peers; use scc::HashMap as SccHashMap; use signer::Signer; @@ -13,11 +18,17 @@ use tracing::instrument; use types::{ combined::BeaconState, config::Config, - phase0::primitives::{Slot, ValidatorIndex}, + phase0::primitives::{Epoch, Slot, ValidatorIndex}, preset::Preset, }; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +use crate::{ + beacon_node_api::{BeaconNodeApi as _, PtcDuties}, + beacon_nodes::BeaconNodes, + misc::RequestedIndices, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct PTCMember { pub public_key: PublicKeyBytes, pub validator_index: ValidatorIndex, @@ -26,6 +37,7 @@ pub struct PTCMember { pub struct OwnPTCMembers { signer: Arc, members: SccHashMap<(H256, Slot), Arc<[PTCMember]>>, + requested: RequestedIndices, } impl OwnPTCMembers { @@ -33,6 +45,13 @@ impl OwnPTCMembers { Self { signer, members: SccHashMap::new(), + requested: RequestedIndices::default(), + } + } + + async fn discard_for_other_keys(&self, validator_indices: &[ValidatorIndex]) { + if self.requested.changed(validator_indices).await { + self.members.clear_async().await; } } @@ -47,12 +66,15 @@ impl OwnPTCMembers { state: &BeaconState

, dependent_root: H256, slot: Slot, + validator_indices: &[ValidatorIndex], ) -> Option> { + self.discard_for_other_keys(validator_indices).await; + if let Some(members) = self.members.get_async(&(dependent_root, slot)).await { return Some(members.clone_arc()); } - match self.compute_members_at_slot(config, state, slot) { + match Self::compute_members_at_slot(config, state, slot, validator_indices) { Ok(members) => { if let Some(members) = members { self.members @@ -65,14 +87,95 @@ impl OwnPTCMembers { } } Err(error) => { - warn_with_peers!( - "failed to compute own beacon committee members at slot {slot}: {error:?}" - ); + warn_with_peers!("failed to compute own PTC members at slot {slot}: {error:?}"); None } } } + pub async fn get_at_slot(&self, dependent_root: H256, slot: Slot) -> Option> { + self.members + .get_async(&(dependent_root, slot)) + .await + .map(|members| members.clone_arc()) + } + + pub async fn init_at_epoch( + &self, + config: &Config, + beacon_nodes: &BeaconNodes, + epoch: Epoch, + dependent_root: H256, + validator_indices: &[ValidatorIndex], + ) -> Result<()> { + self.discard_for_other_keys(validator_indices).await; + + let slots = misc::slots_in_epoch::

(epoch)?; + + // The built-in node caches one slot at a time, so only a complete epoch needs no fetch. + let mut cached = true; + + for slot in slots.clone() { + if self.get_at_slot(dependent_root, slot).await.is_none() { + cached = false; + break; + } + } + + if cached { + return Ok(()); + } + + let PtcDuties { + dependent_root: reported_root, + duties, + } = beacon_nodes.ptc_duties(epoch, validator_indices).await?; + + // Another root is another shuffling; the duties cannot stand in for the ones asked for. + // Nodes disagree on the root they report for the Gloas fork epoch, so it is let through. + ensure!( + reported_root == dependent_root || epoch == config.gloas_fork_epoch, + "PTC duties for epoch {epoch} were reported under dependent root {reported_root:?} \ + rather than {dependent_root:?}", + ); + + let signer_snapshot = self.signer.load(); + let mut members_by_slot = slots + .map(|slot| (slot, Vec::::new())) + .collect::>(); + + for duty in duties { + // A duty for a key we do not hold would fail the whole signing batch. + if !signer_snapshot.has_key(duty.pubkey) { + continue; + } + + let Some(members) = members_by_slot.get_mut(&duty.slot) else { + continue; + }; + + let member = PTCMember { + public_key: duty.pubkey, + validator_index: duty.validator_index, + }; + + // A validator drawn into several positions still sends a single message. + if !members.contains(&member) { + members.push(member); + } + } + + for (slot, mut members) in members_by_slot { + members.sort_unstable_by_key(|member| member.validator_index); + + self.members + .upsert_async((dependent_root, slot), members.into()) + .await; + } + + Ok(()) + } + pub async fn prune(&self, up_to_slot: Slot) { self.members .retain_async(|(_, slot), _| *slot >= up_to_slot) @@ -81,38 +184,29 @@ impl OwnPTCMembers { #[instrument(skip_all, level = "debug", fields(slot = slot))] fn compute_members_at_slot( - &self, config: &Config, state: &BeaconState

, slot: Slot, + validator_indices: &[ValidatorIndex], ) -> Result>> { - let signer_snapshot = self.signer.load(); - - let own_public_keys = signer_snapshot - .keys() - .copied() - .filter_map(|public_key| { - let validator_index = accessors::index_of_public_key(state, &public_key)?; - Some((validator_index, public_key)) - }) - .collect::>(); - - if own_public_keys.is_empty() { + if validator_indices.is_empty() { return Ok(None); } + let indices = validator_indices.iter().copied().collect::>(); + accessors::get_ptc(config, state, slot)? .into_iter() - .filter_map(|validator_index| { - own_public_keys - .get(&validator_index) - .copied() - .map(|public_key| PTCMember { - public_key, - validator_index, - }) + .filter(|validator_index| indices.contains(validator_index)) + // A validator drawn into several positions still sends a single message. + .unique() + .map(|validator_index| { + Ok(PTCMember { + public_key: *accessors::public_key(state, validator_index)?, + validator_index, + }) }) - .collect::>() + .collect::>>()? .conv::>() .pipe(Some) .pipe(Ok) diff --git a/validator/src/own_sync_committee_members.rs b/validator/src/own_sync_committee_members.rs new file mode 100644 index 000000000..2b43de006 --- /dev/null +++ b/validator/src/own_sync_committee_members.rs @@ -0,0 +1,331 @@ +use core::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use anyhow::Result; +use fork_choice_control::Wait; +use helper_functions::misc; +use itertools::Itertools as _; +use p2p::SyncCommitteeSubscription; +use scc::HashMap as SccHashMap; +use std_ext::ArcExt as _; +use typenum::Unsigned as _; +use types::{ + altair::consts::SyncCommitteeSubnetCount, + config::Config as ChainConfig, + phase0::primitives::{Epoch, Slot, ValidatorIndex}, + preset::Preset, +}; + +use crate::{ + beacon_node_api::BeaconNodeApi as _, + beacon_nodes::BeaconNodes, + misc::{ + SyncCommitteeMember, subnets_from_sync_committee_indices, sync_subnet_subscription_epoch, + }, +}; + +/// Sync committee members of the periods in flight. +pub struct OwnSyncCommitteeMembers { + periods: SccHashMap, + /// The epoch subscriptions were last sent in; resent each epoch for restarted nodes. + subscriptions_sent_at: AtomicU64, +} + +/// One period's answer from one duties fetch. +struct PeriodDuties { + /// The indices the duties were requested for; a key imported at runtime changes the set. + requested: Arc<[ValidatorIndex]>, + members: Arc<[SyncCommitteeMember]>, + /// Each with the epoch it is due at, drawn when the period was cached. + subscriptions: Arc<[(Epoch, SyncCommitteeSubscription)]>, +} + +impl OwnSyncCommitteeMembers { + #[must_use] + pub fn new() -> Self { + Self { + periods: SccHashMap::new(), + subscriptions_sent_at: AtomicU64::new(u64::MAX), + } + } + + pub async fn get_at_slot(&self, slot: Slot) -> Option> { + let period = period_at_slot::

(slot); + + self.periods + .get_async(&period) + .await + .map(|entry| entry.get().members.clone_arc()) + } + + pub async fn get_or_init_at_slot( + &self, + chain_config: &ChainConfig, + beacon_nodes: &BeaconNodes, + slot: Slot, + validator_indices: &[ValidatorIndex], + ) -> Result>> { + self.init_at_period( + chain_config, + beacon_nodes, + period_at_slot::

(slot), + validator_indices, + ) + .await?; + + Ok(self.get_at_slot::

(slot).await) + } + + pub async fn init_at_period( + &self, + chain_config: &ChainConfig, + beacon_nodes: &BeaconNodes, + period: u64, + validator_indices: &[ValidatorIndex], + ) -> Result<()> { + let cached = self + .periods + .get_async(&period) + .await + .is_some_and(|entry| *entry.get().requested == *validator_indices); + + if cached { + return Ok(()); + } + + // A period reaching back before Altair only has committees from the fork on. + let epoch = + misc::start_of_sync_committee_period::

(period)?.max(chain_config.altair_fork_epoch); + + let duties = beacon_nodes + .sync_committee_duties(epoch, validator_indices) + .await?; + + let period_start = misc::start_of_sync_committee_period::

(period)?; + let until_epoch = misc::start_of_sync_committee_period::

(period.saturating_add(1))?; + + // Scoped because the generator is not `Send`, and the cache update below awaits. + let subscriptions = { + let mut rng = rand::thread_rng(); + + duties + .iter() + .map(|duty| { + let subscription = SyncCommitteeSubscription { + validator_index: duty.validator_index, + sync_committee_indices: duty.validator_sync_committee_indices.clone(), + until_epoch, + }; + + ( + sync_subnet_subscription_epoch(period_start, &mut rng), + subscription, + ) + }) + .collect::>() + }; + + let members = duties + .into_iter() + .map(|duty| { + Ok(SyncCommitteeMember { + validator_index: duty.validator_index, + public_key: duty.pubkey, + subnets: subnets_from_sync_committee_indices::

( + duty.validator_sync_committee_indices, + )?, + }) + }) + .collect::>>()? + .into_iter() + .sorted_by_key(|member| member.validator_index) + .collect::>(); + + // Cached even when empty, so the same question is not asked every slot of the period. + self.periods + .upsert_async( + period, + PeriodDuties { + requested: validator_indices.into(), + members, + subscriptions, + }, + ) + .await; + + Ok(()) + } + + pub async fn subscriptions_to_send( + &self, + current_epoch: Epoch, + ) -> Option> { + // Sent every epoch, so that a restarted or newly reachable node still learns them. + if self.subscriptions_sent_at.load(Ordering::Relaxed) == current_epoch { + return None; + } + + let current_period = misc::sync_committee_period::

(current_epoch); + let next_period = current_period.saturating_add(1); + // The earliest draw is `SYNC_COMMITTEE_SUBNET_COUNT` epochs before the period. + let next_period_may_be_due = misc::sync_committee_period::

( + current_epoch.saturating_add(SyncCommitteeSubnetCount::U64), + ) == next_period; + + let mut subscriptions = vec![]; + let mut found_current = false; + let mut found_next = false; + + self.periods + .iter_async(|period, entry| { + found_current |= *period == current_period; + found_next |= *period == next_period; + + subscriptions.extend( + entry + .subscriptions + .iter() + .filter(|(due_epoch, _)| *due_epoch <= current_epoch) + .map(|(_, subscription)| subscription.clone()), + ); + + true + }) + .await; + + // A period whose fetch failed is not marked sent, so the next slot retries it. + let complete = found_current && (found_next || !next_period_may_be_due); + + (complete && !subscriptions.is_empty()).then_some(subscriptions) + } + + pub fn mark_subscriptions_sent(&self, current_epoch: Epoch) { + self.subscriptions_sent_at + .store(current_epoch, Ordering::Relaxed); + } + + #[must_use] + pub fn periods_to_prefetch(current_epoch: Epoch) -> [u64; 2] { + let current_period = misc::sync_committee_period::

(current_epoch); + + [current_period, current_period.saturating_add(1)] + } + + pub async fn prune(&self, current_epoch: Epoch) { + let current_period = misc::sync_committee_period::

(current_epoch); + + self.periods + .retain_async(|period, _| *period >= current_period) + .await; + } +} + +impl Default for OwnSyncCommitteeMembers { + fn default() -> Self { + Self::new() + } +} + +fn period_at_slot(slot: Slot) -> u64 { + // A message signed in `slot` is verified against the committee of the block at `slot + 1`. + misc::sync_committee_period::

(misc::compute_epoch_at_slot::

(slot.saturating_add(1))) +} + +#[cfg(test)] +mod tests { + use types::preset::Minimal; + + use super::*; + + // `EPOCHS_PER_SYNC_COMMITTEE_PERIOD` is 8 under the minimal preset, with 8 slots per epoch. + #[test] + fn a_slot_maps_to_the_period_of_the_next_slots_epoch() { + assert_eq!(period_at_slot::(0), 0); + assert_eq!(period_at_slot::(62), 0); + assert_eq!(period_at_slot::(63), 1); + assert_eq!(period_at_slot::(64), 1); + } + + #[test] + fn prefetching_covers_the_current_period_and_the_next() { + assert_eq!( + OwnSyncCommitteeMembers::periods_to_prefetch::(0), + [0, 1], + ); + + assert_eq!( + OwnSyncCommitteeMembers::periods_to_prefetch::(8), + [1, 2], + ); + } + + fn subscription( + validator_index: ValidatorIndex, + until_epoch: Epoch, + ) -> SyncCommitteeSubscription { + SyncCommitteeSubscription { + validator_index, + sync_committee_indices: vec![0], + until_epoch, + } + } + + async fn cache_period( + members: &OwnSyncCommitteeMembers, + period: u64, + subscriptions: impl IntoIterator, + ) { + let inserted = members + .periods + .insert_async( + period, + PeriodDuties { + requested: Arc::from([0]), + members: Arc::from([]), + subscriptions: subscriptions.into_iter().collect(), + }, + ) + .await; + + assert!(inserted.is_ok()); + } + + // Within `SYNC_COMMITTEE_SUBNET_COUNT` epochs of the next period some of its subscriptions may + // be due, so an answer without that period cached is not marked as sent. + #[tokio::test] + async fn subscriptions_are_not_sent_while_the_next_period_is_missing() { + let members = OwnSyncCommitteeMembers::new(); + + cache_period(&members, 0, [(0, subscription(0, 8))]).await; + + // A period is 8 epochs and the subnet count 4 under the minimal preset. + assert!(members.subscriptions_to_send::(3).await.is_some()); + assert!(members.subscriptions_to_send::(4).await.is_none()); + assert!(members.subscriptions_to_send::(7).await.is_none()); + } + + // A subscription is held back until the epoch drawn for it. + #[tokio::test] + async fn a_subscription_is_sent_from_its_drawn_epoch() { + let members = OwnSyncCommitteeMembers::new(); + + cache_period(&members, 0, [(0, subscription(0, 8))]).await; + cache_period(&members, 1, [(6, subscription(1, 16))]).await; + + let members = &members; + + let indices = |epoch| async move { + members + .subscriptions_to_send::(epoch) + .await + .into_iter() + .flatten() + .map(|subscription| subscription.validator_index) + .sorted() + .collect::>() + }; + + assert_eq!(indices(5).await, [0]); + assert_eq!(indices(6).await, [0, 1]); + } +} diff --git a/validator/src/own_sync_committee_subscriptions.rs b/validator/src/own_sync_committee_subscriptions.rs index 4435d08f6..315bd47a3 100644 --- a/validator/src/own_sync_committee_subscriptions.rs +++ b/validator/src/own_sync_committee_subscriptions.rs @@ -6,15 +6,13 @@ use bls::PublicKeyBytes; use helper_functions::{accessors, misc}; use itertools::Itertools as _; use p2p::SyncCommitteeSubscription; -use rand::Rng as _; -use typenum::Unsigned as _; use types::{ - altair::{consts::SyncCommitteeSubnetCount, containers::SyncCommittee}, - phase0::primitives::Epoch, - preset::Preset, + altair::containers::SyncCommittee, phase0::primitives::Epoch, preset::Preset, traits::PostAltairBeaconState, }; +use crate::misc::sync_subnet_subscription_epoch; + #[derive(Default)] pub struct OwnSyncCommitteeSubscriptions { subscriptions: BTreeMap>>, @@ -56,20 +54,10 @@ impl OwnSyncCommitteeSubscriptions

{ next_period_expiration, ) .map(|subscription| { - // From the [Altair Honest Validator specification]: - // > To join a sync committee subnet, select a random number of epochs before the - // > end of the current sync committee period between 1 and - // > `SYNC_COMMITTEE_SUBNET_COUNT`, inclusive. Validators should join their member - // > subnet at the beginning of the epoch they have randomly selected. For example, - // > if the next sync committee period starts at epoch `853,248` and the validator - // > randomly selects an offset of `3`, they should join the subnet at the beginning - // > of epoch `853,245`. - // - // [Altair Honest Validator specification]: https://github.com/ethereum/consensus-specs/blob/0b76c8367ed19014d104e3fbd4718e73f459a748/specs/altair/validator.md#sync-committee-subnet-stability - let epoch_to_subscribe_at = next_period_start - .saturating_sub(rng.gen_range(1..=SyncCommitteeSubnetCount::U64)); - - (epoch_to_subscribe_at, subscription) + ( + sync_subnet_subscription_epoch(next_period_start, &mut rng), + subscription, + ) }) .into_group_map() }); diff --git a/validator/src/own_validator_indices.rs b/validator/src/own_validator_indices.rs new file mode 100644 index 000000000..35e994cd8 --- /dev/null +++ b/validator/src/own_validator_indices.rs @@ -0,0 +1,143 @@ +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::{collections::HashMap, sync::Arc}; + +use arc_swap::{ArcSwap, Guard}; +use bls::PublicKeyBytes; +use fork_choice_control::Wait; +use itertools::Itertools as _; +use logging::{debug_with_peers, warn_with_peers}; +use signer::Signer; +use tokio::sync::Mutex; +use types::{ + phase0::primitives::{Epoch, ValidatorIndex}, + preset::Preset, +}; + +use crate::misc::ChainSource; + +/// Validator indices of the configured keys. +pub struct OwnValidatorIndices { + signer: Arc, + // Every key the signer holds, with `None` until its index is known. Swapped out whole, as the + // duty paths read it every slot and a client may hold tens of thousands of keys. + indices: ArcSwap>>, + // Every node reports each finalization, and one request per finalization is enough. + finalized_epoch: AtomicU64, + // A failed request is retried every slot; an answered one waits for a finalization. + last_resolve_failed: AtomicBool, + // Triggers land together, and a resolve that waits for another finds nothing left to ask. + resolving: Mutex<()>, +} + +impl OwnValidatorIndices { + #[must_use] + pub fn new(signer: Arc) -> Self { + Self { + signer, + indices: ArcSwap::from_pointee(HashMap::new()), + finalized_epoch: AtomicU64::new(0), + last_resolve_failed: AtomicBool::new(false), + resolving: Mutex::new(()), + } + } + + #[must_use] + pub fn load(&self) -> Guard>>> { + self.indices.load() + } + + #[must_use] + pub fn last_resolve_failed(&self) -> bool { + self.last_resolve_failed.load(Ordering::Relaxed) + } + + #[must_use] + pub fn sorted(&self) -> Vec { + self.load() + .values() + .flatten() + .copied() + // Ascending, so that caches keyed by the list see the same list each slot. + .sorted() + .collect_vec() + } + + pub async fn sync_keys(&self, chain_source: &ChainSource) { + // Loaded inside, so a swap that lost the race rebuilds from the keys the signer has now. + self.indices.rcu(|indices| { + self.signer + .load() + .keys() + .map(|public_key| (*public_key, indices.get(public_key).copied().flatten())) + .collect::>() + }); + + self.resolve(chain_source).await; + } + + pub async fn resolve_at_finalized_epoch( + &self, + chain_source: &ChainSource, + finalized_epoch: Epoch, + ) { + // Only a finalization can make a deposit's validator appear in the finalized state. + if self + .finalized_epoch + .fetch_max(finalized_epoch, Ordering::Relaxed) + >= finalized_epoch + { + return; + } + + self.resolve(chain_source).await; + } + + pub async fn resolve(&self, chain_source: &ChainSource) { + let _resolving = self.resolving.lock().await; + + let to_resolve = self + .load() + .iter() + // Indices come from the finalized state, so a resolved key is never asked again. + .filter(|(_, validator_index)| validator_index.is_none()) + .map(|(public_key, _)| *public_key) + .collect_vec(); + + // Cleared first, so a key resolved by another trigger also ends the per-slot retries. + self.last_resolve_failed.store(false, Ordering::Relaxed); + + if to_resolve.is_empty() { + return; + } + + let resolved = match chain_source.validator_indices(&to_resolve).await { + Ok(resolved) => resolved, + Err(error) => { + warn_with_peers!("failed to resolve validator indices: {error:?}"); + self.last_resolve_failed.store(true, Ordering::Relaxed); + return; + } + }; + + if resolved.is_empty() { + return; + } + + for (public_key, validator_index) in &resolved { + debug_with_peers!("resolved validator {public_key:?} to index {validator_index}"); + } + + // Only keys still held take an index; a key removed meanwhile stays out. + self.indices.rcu(|indices| { + let mut indices = (**indices).clone(); + + for (public_key, validator_index) in &resolved { + if let Some(entry) = indices.get_mut(public_key) { + *entry = Some(*validator_index); + } + } + + indices + }); + } +} diff --git a/validator/src/remote_beacon_node.rs b/validator/src/remote_beacon_node.rs new file mode 100644 index 000000000..da7be7f9a --- /dev/null +++ b/validator/src/remote_beacon_node.rs @@ -0,0 +1,3048 @@ +use core::{ + error::Error as StdError, + fmt::Write as _, + marker::PhantomData, + str::FromStr, + sync::atomic::{AtomicU8, Ordering}, + time::Duration, +}; +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + sync::{Arc, OnceLock}, + time::Instant, +}; + +use anyhow::{Error as AnyhowError, Result, anyhow, bail, ensure}; +use block_producer::{ProposerData, ValidatorBlindedBlock}; +use bls::{PublicKeyBytes, SignatureBytes}; +use builder_api::{ + consts::{MaxBuilderEntries, MaxBuilderPreferencesEntries}, + gloas::containers::{BuilderConfig, BuilderEntry, BuilderPreferencesEntry, BuilderUrl}, + unphased::containers::SignedValidatorRegistrationV1, +}; +use derive_more::Display; +use futures::{Stream, StreamExt as _, future, stream}; +use helper_functions::{misc, predicates}; +use http_api_utils::{ + BlockHeadersResponse, ETH_BLOB_DATA_INCLUDED, ETH_BUILDER_URL, ETH_CONSENSUS_VERSION, + ETH_EXECUTION_PAYLOAD_BLINDED, ETH_EXECUTION_PAYLOAD_INCLUDED, EthResponse, + ValidatorLivenessResponse, ValidatorSyncDutyResponse, +}; +use http_body_util::BodyDataStream; +use itertools::Itertools as _; +use logging::{debug_with_peers, info_with_peers, warn_with_peers}; +use mime::APPLICATION_OCTET_STREAM; +use p2p::{BeaconCommitteeSubscription, SyncCommitteeSubscription}; +use reqwest::{ + Body, Client, Response, StatusCode, + header::{ACCEPT, CONTENT_TYPE, HeaderMap}, +}; +use serde::{Deserialize, Serialize, Serializer, de::DeserializeOwned}; +use sse_stream::SseStream; +use ssz::{ + ContiguousList, Hc, Ssz, SszHash as _, SszRead, SszReadDefault as _, SszSize, SszWrite as _, +}; +use std_ext::ArcExt as _; +use thiserror::Error; +use tokio::{sync::Mutex, time::timeout}; +use typenum::Unsigned; +use types::{ + altair::{ + containers::{SignedContributionAndProof, SyncCommitteeContribution, SyncCommitteeMessage}, + primitives::SubcommitteeIndex, + }, + combined::{ + Attestation, BeaconBlock, BlindedBeaconBlock, SignedAggregateAndProof, SignedBeaconBlock, + SignedBlindedBeaconBlock, + }, + config::Config as ChainConfig, + deneb::{ + containers::BeaconBlock as DenebBeaconBlock, + primitives::{Blob, KzgProof}, + }, + electra::containers::{Attestation as ElectraAttestation, BeaconBlock as ElectraBeaconBlock}, + fulu::containers::BeaconBlock as FuluBeaconBlock, + gloas::containers::{ + Attestation as GloasAttestation, BeaconBlock as GloasBeaconBlock, ExecutionPayloadEnvelope, + PayloadAttestationData, PayloadAttestationMessage, SignedExecutionPayloadEnvelope, + SignedProposerPreferences, + }, + nonstandard::{ForkInfo, KzgProofs, OwnAttestation, Phase}, + phase0::containers::Attestation as Phase0Attestation, + phase0::{ + consts::BASIS_POINTS, + containers::AttestationData, + primitives::{ + CommitteeIndex, Epoch, Gwei, H256, Slot, Uint256, UnixSeconds, ValidatorIndex, Version, + }, + }, + preset::Preset, + redacting_url::RedactingUrl, + traits::SignedBeaconBlock as _, +}; + +use crate::{ + beacon_node_api::{ + AttesterDuties, BeaconNodeApi, Duties, EnvelopeContents, ProducedBlock, ProposerDuties, + PtcDuties, + }, + chain_events::{ChainHead, DependentRoots, HeadStatus, HeadUpdate, StreamEvent}, + health::Health, + slot_head::SlotHead, +}; + +/// The end of the slot as a due point, at 100% of the slot's basis points. +const SLOT_END_BPS: u64 = BASIS_POINTS; +/// No timeout falls below this; short slots must not shrink requests below real network latency. +const MIN_TIMEOUT: Duration = Duration::from_secs(1); +/// Most beacon nodes send keep-alive comments at least every 30 seconds, so a stream this quiet +/// is a dead connection; one that sends none merely resubscribes after five empty slots. +const EVENT_STREAM_IDLE_TIMEOUT: Duration = Duration::from_mins(1); + +const VALIDATOR_IDS_PER_REQUEST: usize = 1024; + +#[derive(Debug, Error)] +enum EventStreamError { + #[error("no bytes received in {EVENT_STREAM_IDLE_TIMEOUT:?}")] + Idle, + #[error(transparent)] + Transport(reqwest::Error), +} + +#[derive(Debug, Error)] +enum Error { + #[error( + "beacon node at {url} is on a different network \ + (expected genesis fork version {expected:?}, received {actual:?})" + )] + NetworkMismatch { + url: String, + expected: Version, + actual: Version, + }, + #[error( + "beacon node at {url} is on a chain with genesis validators root {actual:?} \ + where {expected:?} was expected" + )] + GenesisValidatorsRootMismatch { + url: String, + expected: H256, + actual: H256, + }, + #[error("beacon node at {url} reported an optimistic head")] + OptimisticHead { url: String }, + #[error("beacon node at {url} did not report whether its head is optimistic")] + UnknownOptimisticStatus { url: String }, + #[error( + "beacon node at {url} returned payload attestation data for slot {actual} \ + where slot {expected} was requested" + )] + UnexpectedPayloadAttestationSlot { + url: String, + expected: Slot, + actual: Slot, + }, + #[error("beacon node at {url} did not report whether the produced block is blinded")] + UnknownBlindedStatus { url: String }, + #[error("beacon node at {url} did not report whether the produced block has its payload")] + UnknownPayloadStatus { url: String }, + #[error( + "beacon node at {url} answered with content type {content_type:?} where SSZ was requested" + )] + UnexpectedContentType { url: String, content_type: String }, + #[error( + "beacon node produced attestation data with index {index} \ + for committee {committee_index} in {phase}" + )] + UnexpectedAttestationIndex { + phase: Phase, + committee_index: CommitteeIndex, + index: u64, + }, + #[error( + "beacon node produced attestation data with target epoch {target_epoch} \ + for slot {slot} in epoch {epoch}" + )] + UnexpectedAttestationTarget { + slot: Slot, + epoch: Epoch, + target_epoch: Epoch, + }, + #[error( + "beacon node produced attestation data with source epoch {source_epoch} \ + past target epoch {target_epoch}" + )] + UnexpectedAttestationSource { + source_epoch: Epoch, + target_epoch: Epoch, + }, + #[error("request to beacon node at {url} failed with status {status}: {body}")] + Response { + url: String, + status: StatusCode, + body: String, + }, +} + +/// The response of `getGenesis`. +#[derive(Deserialize)] +pub struct Genesis { + #[serde(with = "serde_utils::string_or_native")] + pub genesis_time: UnixSeconds, + pub genesis_fork_version: Version, + pub genesis_validators_root: H256, +} + +/// The part of a `finalized_checkpoint` event the validator client uses. +#[derive(Deserialize)] +struct FinalizedCheckpointEvent { + #[serde(with = "serde_utils::string_or_native")] + epoch: Epoch, +} + +/// The request body of `postStateValidators`. +#[derive(Serialize)] +struct ValidatorIds<'keys> { + ids: &'keys [PublicKeyBytes], +} + +/// An entry of the `postStateValidators` response, of which only the key and index are needed. +#[derive(Deserialize)] +struct StateValidator { + #[serde(with = "serde_utils::string_or_native")] + index: ValidatorIndex, + validator: ValidatorPublicKey, +} + +#[derive(Deserialize)] +struct ValidatorPublicKey { + pubkey: PublicKeyBytes, +} + +/// The deprecated `head` event, whose dependent roots are named by duty period. +#[derive(Deserialize)] +struct HeadEvent { + #[serde(with = "serde_utils::string_or_native")] + slot: Slot, + block: H256, + previous_duty_dependent_root: H256, + current_duty_dependent_root: H256, + execution_optimistic: bool, +} + +/// The `head_v2` event, whose dependent roots are named by the epoch they determine. +#[derive(Deserialize)] +struct HeadV2Event { + data: HeadV2EventData, +} + +#[derive(Deserialize)] +struct HeadV2EventData { + #[serde(with = "serde_utils::string_or_native")] + slot: Slot, + block: H256, + current_epoch_dependent_root: H256, + next_epoch_dependent_root: H256, + execution_optimistic: bool, +} + +/// The `produceBlockV3` response from Deneb on, in which the blobs travel with the block. +/// Fulu raises the proof bound from one per blob to one per cell. +#[derive(Ssz)] +#[ssz( + derive_hash = false, + derive_write = false, + bound_for_read = "B: SszRead" +)] +struct BlockContents { + block: B, + kzg_proofs: ContiguousList, + blobs: ContiguousList, P::MaxBlobCommitmentsPerBlock>, +} + +type DenebBlockContents

= + BlockContents,

::MaxBlobCommitmentsPerBlock, P>; + +type ElectraBlockContents

= + BlockContents,

::MaxBlobCommitmentsPerBlock, P>; + +type FuluBlockContents

= + BlockContents,

::MaxCellProofsPerBlock, P>; + +/// The `produceBlockV4` response with the payload included, for a self-built block. +#[derive(Ssz)] +#[ssz(derive_hash = false)] +struct GloasBlockContents { + block: GloasBeaconBlock

, + execution_payload_envelope: ExecutionPayloadEnvelope

, + kzg_proofs: ContiguousList, + blobs: ContiguousList, P::MaxBlobCommitmentsPerBlock>, +} + +/// The request body of `publishBlockV2` from Deneb on. +#[derive(Ssz)] +#[ssz(derive_hash = false, derive_read = false)] +struct SignedBlockContents<'block, P: Preset> { + signed_block: &'block SignedBeaconBlock

, + kzg_proofs: &'block KzgProofs

, + blobs: &'block ContiguousList, P::MaxBlobCommitmentsPerBlock>, +} + +/// The request body of `publishExecutionPayloadEnvelope` with the blob data included. +#[derive(Ssz)] +#[ssz(derive_hash = false, derive_read = false)] +struct SignedEnvelopeContents<'envelope, P: Preset> { + signed_execution_payload_envelope: &'envelope SignedExecutionPayloadEnvelope

, + kzg_proofs: &'envelope ContiguousList, + blobs: &'envelope ContiguousList, P::MaxBlobCommitmentsPerBlock>, +} + +const HEAD_EVENT: &str = "head"; +const HEAD_V2_EVENT: &str = "head_v2"; +const FINALIZED_CHECKPOINT_EVENT: &str = "finalized_checkpoint"; + +/// A `head` or `head_v2` event, told apart by shape: only `head_v2` nests under `data`. +#[derive(Deserialize)] +#[serde(untagged)] +enum AnyHeadEvent { + V2(HeadV2Event), + V1(HeadEvent), +} + +impl AnyHeadEvent { + fn parse(data: &str) -> Result { + serde_json::from_str::(data) + .map(Self::into_update::

) + .map_err(Into::into) + } + + fn into_update(self) -> HeadUpdate { + let (slot, block, current, next, execution_optimistic) = match self { + Self::V2(HeadV2Event { data }) => ( + data.slot, + data.block, + data.current_epoch_dependent_root, + data.next_epoch_dependent_root, + data.execution_optimistic, + ), + // The old names label the duty period rather than the epoch; the values are the same. + Self::V1(event) => ( + event.slot, + event.block, + event.previous_duty_dependent_root, + event.current_duty_dependent_root, + event.execution_optimistic, + ), + }; + + HeadUpdate { + slot, + block, + execution_optimistic, + dependent_roots: DependentRoots { + epoch: misc::compute_epoch_at_slot::

(slot), + current, + next, + }, + } + } +} + +/// The request body of the duty and liveness endpoints, whose indices are quoted in JSON. +struct ValidatorIndices<'indices>(&'indices [ValidatorIndex]); + +impl Serialize for ValidatorIndices<'_> { + fn serialize(&self, serializer: S) -> Result { + serde_utils::string_or_native_sequence::serialize(self.0, serializer) + } +} + +// `head_slot` and `sync_distance` are ignored: neither separates a node that has fallen behind +// from a chain with missed slots. +#[derive(Deserialize)] +struct SyncingStatus { + is_syncing: bool, + #[serde(default)] + is_optimistic: bool, + #[serde(default)] + el_offline: bool, +} + +/// A beacon node reached over . +#[derive(Display)] +#[display("{url}")] +pub struct RemoteBeaconNode { + chain_config: Arc, + client: Client, + url: RedactingUrl, + max_empty_slots: u64, + /// A [`Health`] discriminant. Atomic because it is read on the duty path. + health: AtomicU8, + chain_head: ChainHead, + /// Seeded at startup, before the node is polled. + genesis_validators_root: OnceLock, + /// The proposers last prepared on the node and the epoch they were sent in. + prepared_proposers: Mutex)>>, + /// Committee subscriptions already sent to the node for slots still ahead. + sent_subscriptions: Mutex>, +} + +impl RemoteBeaconNode { + #[must_use] + pub fn new( + chain_config: Arc, + client: Client, + url: RedactingUrl, + max_empty_slots: u64, + ) -> Self { + Self { + chain_config, + client, + url, + max_empty_slots, + health: AtomicU8::new(Health::Unknown.as_u8()), + chain_head: ChainHead::new(), + genesis_validators_root: OnceLock::new(), + prepared_proposers: Mutex::const_new(None), + sent_subscriptions: Mutex::const_new(HashSet::new()), + } + } + + #[must_use] + pub const fn chain_head(&self) -> &ChainHead { + &self.chain_head + } + + pub fn seed_genesis_validators_root(&self, genesis_validators_root: H256) { + if self + .genesis_validators_root + .set(genesis_validators_root) + .is_err() + { + debug_with_peers!( + "{} keeps the genesis validators root it was seeded with", + self.url + ); + } + } + + fn genesis_validators_root(&self) -> Result { + self.genesis_validators_root + .get() + .copied() + .ok_or_else(|| anyhow!("the genesis validators root has not been seeded")) + } + + #[must_use] + pub fn health(&self) -> Health { + Health::from_u8(self.health.load(Ordering::Relaxed)) + } + + #[cfg(test)] + pub(crate) fn set_health(&self, health: Health) { + self.health.store(health.as_u8(), Ordering::Relaxed); + } + + pub async fn events( + &self, + ) -> Result> + Send + use

> { + // Older nodes reject an unknown topic outright rather than streaming nothing; a node + // that cannot be reached at all is no more reachable on the old topic. + let response = match self + .subscribe(&format!("{HEAD_V2_EVENT},{FINALIZED_CHECKPOINT_EVENT}")) + .await + { + Ok(response) => response, + Err(error) if error.downcast_ref::().is_some() => { + debug_with_peers!( + "{} does not stream {HEAD_V2_EVENT} events, falling back to {HEAD_EVENT}: \ + {error:?}", + self.url, + ); + + self.subscribe(&format!("{HEAD_EVENT},{FINALIZED_CHECKPOINT_EVENT}")) + .await? + } + Err(error) => return Err(error), + }; + + // Timed below the parser, where keep-alive comments still count as traffic, so that + // only a dead connection goes quiet this long. + let chunks = stream::unfold( + BodyDataStream::new(Body::from(response)), + |mut chunks| async move { + match timeout(EVENT_STREAM_IDLE_TIMEOUT, chunks.next()).await { + Ok(chunk) => { + chunk.map(|chunk| (chunk.map_err(EventStreamError::Transport), chunks)) + } + Err(_) => Some((Err(EventStreamError::Idle), chunks)), + } + }, + ); + + // An event without data carries no head, as a keep-alive does, and is not a failure. + let events = SseStream::from_bytes_stream(chunks).filter_map(|event| { + let stream_event = match event { + Ok(event) => match (event.event.as_deref(), event.data) { + (Some(HEAD_V2_EVENT | HEAD_EVENT), Some(data)) => { + Some(AnyHeadEvent::parse::

(&data).map(StreamEvent::Head)) + } + (Some(FINALIZED_CHECKPOINT_EVENT), Some(data)) => Some( + serde_json::from_str::(&data) + .map(|event| StreamEvent::Finalized(event.epoch)) + .map_err(Into::into), + ), + _ => None, + }, + Err(error) => Some(Err(error.into())), + }; + + future::ready(stream_event) + }); + + Ok(events) + } + + async fn subscribe(&self, topic: &str) -> Result { + let url = self.endpoint(&format!("/eth/v1/events?topics={topic}"))?; + + let request = self + .client + .get(url.into_url()) + .header(ACCEPT, "text/event-stream") + .timeout(Duration::MAX) + .send(); + + // Only the headers are waited for; a node that accepts the connection and never answers + // would otherwise hold the follower forever. + let response = timeout(self.background_timeout(), request).await??; + + self.check_status(response).await + } + + pub async fn refresh_head(&self, slot: Slot) { + // A node on a different network would otherwise cache a head from another chain, + // mirroring the filter on the event stream. + if self.health() == Health::Incompatible { + return; + } + + // Polled when the head is not from this slot, as the stream may be silently dead; a healthy + // stream makes this a no-op except in a slot whose block is late or missed. + if matches!(self.chain_head.status(slot, 0), HeadStatus::Usable(_)) { + return; + } + + if let Err(error) = self.poll_head().await { + debug_with_peers!("{} did not report its head: {error:?}", self.url); + } + } + + pub(crate) async fn fresh_head_block_root(&self, slot: Slot) -> Result { + self.poll_head().await?; + + self.chain_head + .get(slot, self.max_empty_slots)? + .ok_or_else(|| { + anyhow!( + "head of beacon node at {} is too old to sign for slot {slot}", + self.url, + ) + }) + } + + async fn poll_head(&self) -> Result<()> { + let asked_at = Instant::now(); + let (head_slot, block_root) = self.head_header().await?; + + // Recorded at the head's own slot, so that a stale head is cached as stale rather than + // appearing fresh for another `max_empty_slots` slots. + if !self + .chain_head + .update_seen_at(asked_at, head_slot, block_root) + { + debug_with_peers!( + "{} answered with head {block_root:?} at slot {head_slot} after the stream \ + delivered a newer one", + self.url, + ); + } + + Ok(()) + } + + async fn head_header(&self) -> Result<(Slot, H256)> { + let url = self.endpoint("/eth/v1/beacon/headers/head")?; + + let response = self + .client + .get(url.into_url()) + .timeout(self.head_timeout()) + .send() + .await?; + + let response = self.check_status(response).await?; + + let (block_header, execution_optimistic) = response + .json::>() + .await? + .into_data_and_execution_optimistic(); + + check_not_optimistic(&self.url, execution_optimistic)?; + + Ok((block_header.header.message.slot, block_header.root)) + } + + pub async fn refresh_health(&self, slot: Slot) { + let previous = self.health(); + let health = self.poll_health(slot).await; + + if health != previous { + info_with_peers!("beacon node at {} is now {health:?}", self.url); + + // A node that just became ready may have restarted and forgotten what it was sent. + if health.is_ready() { + *self.prepared_proposers.lock().await = None; + self.sent_subscriptions.lock().await.clear(); + } + } + + self.health.store(health.as_u8(), Ordering::Relaxed); + } + + async fn needs_proposer_preparation(&self, epoch: Epoch, proposers: &[ProposerData]) -> bool { + match self.prepared_proposers.lock().await.as_ref() { + Some((prepared_epoch, prepared)) => *prepared_epoch != epoch || prepared != proposers, + None => true, + } + } + + async fn unsent_subscriptions( + &self, + subscriptions: &[BeaconCommitteeSubscription], + ) -> Vec { + let sent = self.sent_subscriptions.lock().await; + + subscriptions + .iter() + .filter(|subscription| !sent.contains(subscription)) + .copied() + .collect() + } + + async fn record_sent_subscriptions( + &self, + current_slot: Slot, + subscriptions: impl IntoIterator, + ) { + let mut sent = self.sent_subscriptions.lock().await; + + // Past slots leave the set, so it never holds more than the lookahead window. + sent.retain(|subscription| subscription.slot >= current_slot); + sent.extend(subscriptions); + } + + async fn poll_health(&self, slot: Slot) -> Health { + match self.ensure_same_network().await { + Ok(()) => {} + Err(error) + if matches!( + error.downcast_ref(), + Some( + Error::NetworkMismatch { .. } | Error::GenesisValidatorsRootMismatch { .. } + ) + ) => + { + warn_with_peers!( + "beacon node at {} is on a different network: {error:?}", + self.url, + ); + + return Health::Incompatible; + } + Err(error) => { + debug_with_peers!( + "beacon node at {} could not be reached: {error:?}", + self.url + ); + + return Health::Unreachable; + } + } + + let health = match self.syncing_status().await { + Ok(status) => Health::from_syncing_status( + status.el_offline, + status.is_syncing, + status.is_optimistic, + ), + Err(error) => { + debug_with_peers!( + "beacon node at {} did not report its sync status: {error:?}", + self.url, + ); + + Health::Unreachable + } + }; + + // A node that has fallen behind keeps serving stale duty data while calling itself + // synced; its own head, held to the wall clock, shows what its sync status cannot. + if health == Health::Ready + && matches!( + self.chain_head.status(slot, self.max_empty_slots), + HeadStatus::Stale { .. } + ) + { + debug_with_peers!( + "beacon node at {} reports itself synced but its head is stale", + self.url, + ); + + return Health::Unusable; + } + + health + } + + async fn syncing_status(&self) -> Result { + let url = self.endpoint("/eth/v1/node/syncing")?; + + let response = self + .client + .get(url.into_url()) + .timeout(self.background_timeout()) + .send() + .await?; + + self.parse_data(response).await + } + + fn attestation_timeout(&self, phase: Phase) -> Duration { + let config = &self.chain_config; + + self.window_timeout( + config.attestation_due_bps_at(phase), + config.aggregate_due_bps_at(phase), + ) + } + + fn aggregate_timeout(&self, phase: Phase) -> Duration { + self.window_timeout(self.chain_config.aggregate_due_bps_at(phase), SLOT_END_BPS) + } + + fn publish_timeout(&self, phase: Phase) -> Duration { + // The rest of the slot: what arrives late can still count, and a node busy verifying a + // block may need longer than the aggregate window to accept it. + self.window_timeout( + self.chain_config.attestation_due_bps_at(phase), + SLOT_END_BPS, + ) + } + + fn sync_message_timeout(&self, phase: Phase) -> Duration { + let config = &self.chain_config; + + self.window_timeout( + config.sync_message_due_bps_at(phase), + config.contribution_due_bps_at(phase), + ) + } + + fn contribution_timeout(&self, phase: Phase) -> Duration { + self.window_timeout( + self.chain_config.contribution_due_bps_at(phase), + SLOT_END_BPS, + ) + } + + fn block_timeout(&self, phase: Phase) -> Duration { + self.window_timeout(0, self.chain_config.attestation_due_bps_at(phase)) + } + + fn payload_attestation_timeout(&self) -> Duration { + // Gossip only accepts the vote until the slot ends. + self.window_timeout(self.chain_config.payload_attestation_due_bps, SLOT_END_BPS) + } + + fn head_timeout(&self) -> Duration { + // Polled without a preset in scope, so the tighter of the two schedules applies. + self.sync_message_timeout(Phase::Phase0) + .min(self.sync_message_timeout(Phase::Gloas)) + } + + fn background_timeout(&self) -> Duration { + self.window_timeout(BASIS_POINTS / 2, SLOT_END_BPS) + } + + fn window_timeout(&self, from_bps: u64, until_bps: u64) -> Duration { + // A due point is the tick at which a duty runs, as the spec names it, not a deadline. + self.chain_config + .slot_component_duration(until_bps.saturating_sub(from_bps)) + .max(MIN_TIMEOUT) + } + + fn endpoint(&self, path: &str) -> Result { + // Appended rather than joined, so a node behind a proxy keeps its path prefix and query. + let mut url = self.url.clone().into_url(); + let (path, query) = match path.split_once('?') { + Some((path, query)) => (path, Some(query)), + None => (path, None), + }; + + url.set_path(&format!("{}{path}", url.path().trim_end_matches('/'))); + + let query = url.query().into_iter().chain(query).join("&"); + + url.set_query((!query.is_empty()).then_some(query.as_str())); + url.as_str().parse().map_err(Into::into) + } + + pub async fn genesis(&self) -> Result { + let url = self.endpoint("/eth/v1/beacon/genesis")?; + + let response = self + .client + .get(url.into_url()) + .timeout(self.background_timeout()) + .send() + .await?; + + self.parse_data(response).await + } + + async fn ensure_same_network(&self) -> Result<()> { + let genesis = self.genesis().await?; + let expected = self.chain_config.genesis_fork_version; + + // Prevents producing attestations signed under the wrong domain. + ensure!( + genesis.genesis_fork_version == expected, + Error::NetworkMismatch { + url: self.url.to_string(), + expected, + actual: genesis.genesis_fork_version, + }, + ); + + let expected = self.genesis_validators_root()?; + + ensure!( + expected == genesis.genesis_validators_root, + Error::GenesisValidatorsRootMismatch { + url: self.url.to_string(), + expected, + actual: genesis.genesis_validators_root, + }, + ); + + Ok(()) + } + + async fn parse_data(&self, response: Response) -> Result { + let response = self.check_status(response).await?; + Ok(response.json::>().await?.into_data()) + } + + async fn parse_duties( + &self, + response: Response, + epoch: Epoch, + kind: &str, + ) -> Result> { + let response = self.check_status(response).await?; + + let (duties, dependent_root) = response + .json::>>() + .await? + .into_data_and_dependent_root(); + + let dependent_root = dependent_root.ok_or_else(|| { + anyhow!( + "beacon node at {} did not report a dependent root for {kind} duties", + self.url, + ) + })?; + + debug_with_peers!( + "{} produced {} {kind} duties for epoch {epoch}", + self.url, + duties.len(), + ); + + Ok(Duties { + dependent_root, + duties, + }) + } + + async fn check_status(&self, response: Response) -> Result { + let status = response.status(); + + if status.is_success() { + return Ok(response); + } + + let body = response.text().await.unwrap_or_default(); + + bail!(Error::Response { + url: self.url.to_string(), + status, + body, + }) + } + + pub fn slot_head(&self, slot: Slot) -> Result>> { + let Some(beacon_block_root) = self.chain_head.get(slot, self.max_empty_slots)? else { + return Ok(None); + }; + + let epoch = misc::compute_epoch_at_slot::

(slot); + + let slot_head = SlotHead { + config: self.chain_config.clone_arc(), + slot, + beacon_block_root, + fork_info: ForkInfo::at_epoch( + &self.chain_config, + epoch, + self.genesis_validators_root()?, + ), + phantom: PhantomData, + }; + + Ok(Some(slot_head)) + } + + pub async fn validator_indices( + &self, + public_keys: &[PublicKeyBytes], + ) -> Result> { + let mut indices = HashMap::new(); + + // Split up because the whole key set of a large validator client does not belong in a + // single request body. + for keys in public_keys.chunks(VALIDATOR_IDS_PER_REQUEST) { + // Activation needs a final deposit, so the finalized state holds every index that + // matters, and a reorg cannot move one. + let url = self.endpoint("/eth/v1/beacon/states/finalized/validators")?; + + let response = self + .client + .post(url.into_url()) + .json(&ValidatorIds { ids: keys }) + .timeout(self.background_timeout()) + .send() + .await?; + + let validators = self.parse_data::>(response).await?; + + indices.extend( + validators + .into_iter() + .map(|validator| (validator.validator.pubkey, validator.index)), + ); + } + + debug_with_peers!( + "{} resolved {} of {} validator indices", + self.url, + indices.len(), + public_keys.len(), + ); + + Ok(indices) + } + + pub async fn attester_duties( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> Result { + // Epoch-shaped because `getAttesterDuties` is; the built-in node answers per slot. + let url = self.endpoint(format!("/eth/v1/validator/duties/attester/{epoch}").as_str())?; + + let response = self + .client + .post(url.into_url()) + .json(&ValidatorIndices(validator_indices)) + .timeout(self.background_timeout()) + .send() + .await?; + + self.parse_duties(response, epoch, "attester").await + } + + async fn produce_block_v3( + &self, + phase: Phase, + slot: Slot, + randao_reveal: SignatureBytes, + graffiti: Option, + builder_boost_factor: Uint256, + ) -> Result> { + let url = self.endpoint(&produce_block_path( + slot, + randao_reveal, + graffiti, + saturating_boost_factor(builder_boost_factor), + ))?; + + let response = self + .client + .get(url.into_url()) + .header(ACCEPT, APPLICATION_OCTET_STREAM.as_ref()) + .timeout(self.block_timeout(phase)) + .send() + .await?; + + let response = self.check_status(response).await?; + let headers = response.headers(); + + let blinded = parse_header(headers, ETH_EXECUTION_PAYLOAD_BLINDED)?.ok_or_else(|| { + Error::UnknownBlindedStatus { + url: self.url.to_string(), + } + })?; + + self.ensure_ssz(headers)?; + + let bytes = response.bytes().await?; + + // Before Bellatrix a blinded block is the block itself. + let block = if blinded && phase >= Phase::Bellatrix { + ProducedBlock::without_blobs(ValidatorBlindedBlock::BlindedBeaconBlock( + BlindedBeaconBlock::from_ssz(&phase, bytes)?, + )) + } else { + match phase { + Phase::Deneb => { + let DenebBlockContents { + block, + kzg_proofs, + blobs, + } = DenebBlockContents::from_ssz_default(bytes)?; + + ProducedBlock::with_blobs( + BeaconBlock::Deneb(Hc::from(block)), + KzgProofs::Deneb(kzg_proofs), + blobs, + ) + } + Phase::Electra => { + let ElectraBlockContents { + block, + kzg_proofs, + blobs, + } = ElectraBlockContents::from_ssz_default(bytes)?; + + ProducedBlock::with_blobs( + BeaconBlock::Electra(Hc::from(block)), + KzgProofs::Deneb(kzg_proofs), + blobs, + ) + } + Phase::Fulu => { + let FuluBlockContents { + block, + kzg_proofs, + blobs, + } = FuluBlockContents::from_ssz_default(bytes)?; + + ProducedBlock::with_blobs( + BeaconBlock::Fulu(Hc::from(block)), + KzgProofs::Fulu(kzg_proofs), + blobs, + ) + } + _ => ProducedBlock::without_blobs(ValidatorBlindedBlock::BeaconBlock( + BeaconBlock::from_ssz(&self.chain_config, bytes)?, + )), + } + }; + + debug_with_peers!( + "{} produced a {} block for slot {slot}", + self.url, + if blinded { "blinded" } else { "full" }, + ); + + Ok(block) + } + + #[expect(clippy::too_many_arguments)] + async fn produce_block_v4( + &self, + phase: Phase, + slot: Slot, + randao_reveal: SignatureBytes, + graffiti: Option, + builder_boost_factor: Uint256, + min_bid: Gwei, + builders: &ContiguousList, + ) -> Result> { + let url = self.endpoint(&produce_block_v4_path(slot, randao_reveal, graffiti))?; + + let builder_config = BuilderConfig { + min_bid, + builder_boost_factor: saturating_boost_factor(builder_boost_factor), + builders: builders.clone(), + }; + + let response = self + .client + .post(url.into_url()) + .header(ACCEPT, APPLICATION_OCTET_STREAM.as_ref()) + .header(ETH_CONSENSUS_VERSION, phase.as_ref()) + .json(&builder_config) + .timeout(self.block_timeout(phase)) + .send() + .await?; + + let response = self.check_status(response).await?; + let headers = response.headers(); + + let payload_included = + parse_header(headers, ETH_EXECUTION_PAYLOAD_INCLUDED)?.ok_or_else(|| { + Error::UnknownPayloadStatus { + url: self.url.to_string(), + } + })?; + + // The block is worth more than the builder learning of it early, so a bad header is + // dropped rather than failing the proposal. + let builder_url = parse_header::(headers, ETH_BUILDER_URL) + .inspect_err(|error| { + warn_with_peers!( + "ignoring malformed {ETH_BUILDER_URL} header from {}: {error}", + self.url + ); + }) + .ok() + .flatten(); + + self.ensure_ssz(headers)?; + + let bytes = response.bytes().await?; + + let block = if payload_included { + let GloasBlockContents { + block, + execution_payload_envelope, + kzg_proofs, + blobs, + } = GloasBlockContents::from_ssz_default(bytes)?; + + ProducedBlock { + block: ValidatorBlindedBlock::BeaconBlock(BeaconBlock::Gloas(Hc::from(block))), + kzg_proofs: None, + blobs: None, + envelope_contents: Some(EnvelopeContents { + envelope: execution_payload_envelope, + kzg_proofs, + blobs, + }), + builder_url, + } + } else { + ProducedBlock { + builder_url, + ..ProducedBlock::without_blobs(ValidatorBlindedBlock::BeaconBlock( + BeaconBlock::from_ssz(&self.chain_config, bytes)?, + )) + } + }; + + debug_with_peers!( + "{} produced a block for slot {slot} {} its payload", + self.url, + if payload_included { "with" } else { "without" }, + ); + + Ok(block) + } + + fn ensure_ssz(&self, headers: &HeaderMap) -> Result<()> { + let content_type = headers + .get(CONTENT_TYPE) + .map(|value| value.to_str().map(str::to_owned)) + .transpose()? + .unwrap_or_default(); + + ensure!( + content_type.starts_with(APPLICATION_OCTET_STREAM.as_ref()), + Error::UnexpectedContentType { + url: self.url.to_string(), + content_type, + }, + ); + + Ok(()) + } + + async fn publish_ssz( + &self, + path: &str, + phase: Phase, + extra_headers: &[(&str, &str)], + body: Vec, + ) -> Result<()> { + let url = self.endpoint(path)?; + + let mut request = self + .client + .post(url.into_url()) + .header(ETH_CONSENSUS_VERSION, phase.as_ref()) + .header(CONTENT_TYPE, APPLICATION_OCTET_STREAM.as_ref()); + + for (name, value) in extra_headers { + request = request.header(*name, *value); + } + + let response = request + .body(body) + .timeout(self.publish_timeout(phase)) + .send() + .await?; + + self.check_status(response).await?; + + Ok(()) + } +} + +impl BeaconNodeApi

for RemoteBeaconNode { + async fn liveness( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> Result> { + let url = self.endpoint(format!("/eth/v1/validator/liveness/{epoch}").as_str())?; + + let response = self + .client + .post(url.into_url()) + .json(&ValidatorIndices(validator_indices)) + .timeout(self.background_timeout()) + .send() + .await?; + + self.parse_data(response).await + } + + async fn dependent_root(&self, epoch: Epoch, validator_index: ValidatorIndex) -> Result { + self.attester_duties(epoch, &[validator_index]) + .await + .map(|duties| duties.dependent_root) + } + + async fn attestation_data( + &self, + slot: Slot, + committee_index: CommitteeIndex, + ) -> Result { + let phase = self.chain_config.phase_at_slot::

(slot); + + let url = self.endpoint( + format!( + "/eth/v1/validator/attestation_data?slot={slot}&committee_index={committee_index}" + ) + .as_str(), + )?; + + let response = self + .client + .get(url.into_url()) + .timeout(self.attestation_timeout(phase)) + .send() + .await?; + + let data = self.parse_data::(response).await?; + + check_attestation_index(phase, committee_index, data.index)?; + check_attestation_target::

(slot, data.target.epoch)?; + check_attestation_source(data.source.epoch, data.target.epoch)?; + + Ok(data) + } + + async fn aggregate_attestation( + &self, + data: AttestationData, + committee_index: CommitteeIndex, + ) -> Result> { + let phase = self.chain_config.phase_at_slot::

(data.slot); + let url = self.endpoint(&aggregate_attestation_path(data, committee_index))?; + + let response = self + .client + .get(url.into_url()) + .timeout(self.aggregate_timeout(phase)) + .send() + .await?; + + // Electra and Gloas attestations have the same fields, so untagged deserialization would + // always read a Gloas one as Electra. + let attestation = if phase < Phase::Electra { + self.parse_data::>(response) + .await + .map(Attestation::Phase0)? + } else if phase < Phase::Gloas { + self.parse_data::>(response) + .await + .map(Attestation::Electra)? + } else { + self.parse_data::>(response) + .await + .map(Attestation::Gloas)? + }; + + Ok(attestation) + } + + async fn publish_singular_attestations( + &self, + attestations: &[OwnAttestation

], + ) -> Result<()> { + let Some(first) = attestations.first() else { + return Ok(()); + }; + + let phase = self + .chain_config + .phase_at_slot::

(first.attestation.data().slot); + + // `combined::Attestation` serializes untagged, so this is a `SingleAttestation` from + // Electra on and a phase 0 `Attestation` before it, exactly as the endpoint expects. + let bodies = attestations + .iter() + .map(|own_attestation| &own_attestation.attestation) + .collect::>(); + + let url = self.endpoint("/eth/v2/beacon/pool/attestations")?; + + let response = self + .client + .post(url.into_url()) + .header(ETH_CONSENSUS_VERSION, phase.as_ref()) + .json(&bodies) + .timeout(self.publish_timeout(phase)) + .send() + .await?; + + self.check_status(response).await?; + + debug_with_peers!("published {} attestations to {}", bodies.len(), self.url); + + Ok(()) + } + + async fn publish_aggregates_and_proofs( + &self, + aggregates_and_proofs: &[Arc>], + ) -> Result<()> { + let Some(first) = aggregates_and_proofs.first() else { + return Ok(()); + }; + + let phase = self.chain_config.phase_at_slot::

(first.slot()); + let url = self.endpoint("/eth/v2/validator/aggregate_and_proofs")?; + + let response = self + .client + .post(url.into_url()) + .header(ETH_CONSENSUS_VERSION, phase.as_ref()) + .json(&aggregates_and_proofs) + .timeout(self.aggregate_timeout(phase)) + .send() + .await?; + + self.check_status(response).await?; + + debug_with_peers!( + "published {} aggregates and proofs to {}", + aggregates_and_proofs.len(), + self.url, + ); + + Ok(()) + } + + async fn subscribe_to_beacon_committees( + &self, + current_slot: Slot, + subscriptions: &[BeaconCommitteeSubscription], + ) -> Result<()> { + // The caller offers the whole lookahead window every slot. + let subscriptions = self.unsent_subscriptions(subscriptions).await; + + if subscriptions.is_empty() { + return Ok(()); + } + + let url = self.endpoint("/eth/v1/validator/beacon_committee_subscriptions")?; + + let response = self + .client + .post(url.into_url()) + .json(&subscriptions) + .timeout(self.background_timeout()) + .send() + .await?; + + self.check_status(response).await?; + + self.record_sent_subscriptions(current_slot, subscriptions.iter().copied()) + .await; + + debug_with_peers!( + "subscribed to {} beacon committees on {}", + subscriptions.len(), + self.url, + ); + + Ok(()) + } + + async fn sync_committee_duties( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> Result> { + let url = self.endpoint(format!("/eth/v1/validator/duties/sync/{epoch}").as_str())?; + + let response = self + .client + .post(url.into_url()) + .json(&ValidatorIndices(validator_indices)) + .timeout(self.background_timeout()) + .send() + .await?; + + let duties = self + .parse_data::>(response) + .await?; + + debug_with_peers!( + "{} produced {} sync committee duties for epoch {epoch}", + self.url, + duties.len(), + ); + + Ok(duties) + } + + async fn publish_sync_committee_messages( + &self, + messages: &BTreeMap>, + ) -> Result<()> { + // The endpoint derives the subnets itself, so a validator in several subcommittees is + // submitted only once. + let bodies = messages + .values() + .flatten() + .unique_by(|message| message.validator_index) + .collect::>(); + + let Some(first) = bodies.first() else { + return Ok(()); + }; + + let phase = self.chain_config.phase_at_slot::

(first.slot); + let url = self.endpoint("/eth/v1/beacon/pool/sync_committees")?; + + let response = self + .client + .post(url.into_url()) + .json(&bodies) + .timeout(self.sync_message_timeout(phase)) + .send() + .await?; + + self.check_status(response).await?; + + debug_with_peers!( + "published {} sync committee messages to {}", + bodies.len(), + self.url, + ); + + Ok(()) + } + + async fn subscribe_to_sync_committees( + &self, + _current_epoch: Epoch, + subscriptions: &[SyncCommitteeSubscription], + ) -> Result<()> { + let url = self.endpoint("/eth/v1/validator/sync_committee_subscriptions")?; + + let response = self + .client + .post(url.into_url()) + .json(&subscriptions) + .timeout(self.background_timeout()) + .send() + .await?; + + self.check_status(response).await?; + + debug_with_peers!( + "subscribed to {} sync committees on {}", + subscriptions.len(), + self.url, + ); + + Ok(()) + } + + async fn sync_committee_contribution( + &self, + slot: Slot, + subcommittee_index: SubcommitteeIndex, + beacon_block_root: H256, + ) -> Result> { + let phase = self.chain_config.phase_at_slot::

(slot); + + let url = self.endpoint(&sync_committee_contribution_path( + slot, + subcommittee_index, + beacon_block_root, + ))?; + + let response = self + .client + .get(url.into_url()) + .timeout(self.contribution_timeout(phase)) + .send() + .await?; + + let contribution = self + .parse_data::>(response) + .await?; + + Ok(contribution) + } + + async fn publish_contributions_and_proofs( + &self, + contributions_and_proofs: &[SignedContributionAndProof

], + ) -> Result<()> { + let Some(first) = contributions_and_proofs.first() else { + return Ok(()); + }; + + let phase = self + .chain_config + .phase_at_slot::

(first.message.contribution.slot); + + let url = self.endpoint("/eth/v1/validator/contribution_and_proofs")?; + + let response = self + .client + .post(url.into_url()) + .json(&contributions_and_proofs) + .timeout(self.contribution_timeout(phase)) + .send() + .await?; + + self.check_status(response).await?; + + debug_with_peers!( + "published {} contributions and proofs to {}", + contributions_and_proofs.len(), + self.url, + ); + + Ok(()) + } + + async fn ptc_duties( + &self, + epoch: Epoch, + validator_indices: &[ValidatorIndex], + ) -> Result { + let url = self.endpoint(format!("/eth/v1/validator/duties/ptc/{epoch}").as_str())?; + + let response = self + .client + .post(url.into_url()) + .json(&ValidatorIndices(validator_indices)) + .timeout(self.background_timeout()) + .send() + .await?; + + self.parse_duties(response, epoch, "PTC").await + } + + async fn proposer_duties(&self, epoch: Epoch) -> Result { + // Only v2 reports the root the Fulu proposer lookahead makes duties depend on. + let version = if self.chain_config.phase_at_epoch(epoch) >= Phase::Fulu { + "v2" + } else { + "v1" + }; + + let url = self.endpoint(&format!("/eth/{version}/validator/duties/proposer/{epoch}"))?; + + let response = self + .client + .get(url.into_url()) + .timeout(self.background_timeout()) + .send() + .await?; + + self.parse_duties(response, epoch, "proposer").await + } + + async fn produce_block( + &self, + slot: Slot, + randao_reveal: SignatureBytes, + graffiti: Option, + builder_boost_factor: Uint256, + min_bid: Gwei, + builders: &ContiguousList, + ) -> Result> { + let phase = self.chain_config.phase_at_slot::

(slot); + + if phase >= Phase::Gloas { + self.produce_block_v4( + phase, + slot, + randao_reveal, + graffiti, + builder_boost_factor, + min_bid, + builders, + ) + .await + } else { + self.produce_block_v3(phase, slot, randao_reveal, graffiti, builder_boost_factor) + .await + } + } + + async fn publish_block( + &self, + signed_block: &Arc>, + kzg_proofs: Option<&KzgProofs

>, + blobs: Option<&ContiguousList, P::MaxBlobCommitmentsPerBlock>>, + builder_url: Option<&BuilderUrl>, + ) -> Result<()> { + let phase = signed_block.phase(); + + let body = if (Phase::Deneb..Phase::Gloas).contains(&phase) { + let empty_proofs = if phase >= Phase::Fulu { + KzgProofs::empty_fulu() + } else { + KzgProofs::empty_deneb() + }; + + let empty_blobs = ContiguousList::default(); + + SignedBlockContents { + signed_block: signed_block.as_ref(), + kzg_proofs: kzg_proofs.unwrap_or(&empty_proofs), + blobs: blobs.unwrap_or(&empty_blobs), + } + .to_ssz()? + } else { + signed_block.to_ssz()? + }; + + // Passed back so the node forwards the block to the builder whose bid it takes. + let builder_url_header = builder_url.map(|url| (ETH_BUILDER_URL, url.as_str())); + + self.publish_ssz( + "/eth/v2/beacon/blocks", + phase, + builder_url_header.as_slice(), + body, + ) + .await?; + + debug_with_peers!( + "published block for slot {} to {}", + signed_block.message().slot(), + self.url, + ); + + Ok(()) + } + + async fn publish_blinded_block( + &self, + signed_block: &SignedBlindedBeaconBlock

, + ) -> Result<()> { + self.publish_ssz( + "/eth/v2/beacon/blinded_blocks", + signed_block.phase(), + &[], + signed_block.to_ssz()?, + ) + .await?; + + debug_with_peers!( + "published blinded block for slot {} to {}", + signed_block.message().slot(), + self.url, + ); + + Ok(()) + } + + async fn publish_execution_payload_envelope( + &self, + signed_envelope: &Arc>, + kzg_proofs: &ContiguousList, + blobs: &ContiguousList, P::MaxBlobCommitmentsPerBlock>, + ) -> Result<()> { + let envelope = &signed_envelope.message; + let phase = self + .chain_config + .phase_at_slot::

(envelope.payload.slot_number); + + let body = SignedEnvelopeContents { + signed_execution_payload_envelope: signed_envelope.as_ref(), + kzg_proofs, + blobs, + } + .to_ssz()?; + + self.publish_ssz( + "/eth/v1/beacon/execution_payload_envelopes", + phase, + &[(ETH_BLOB_DATA_INCLUDED, "true")], + body, + ) + .await?; + + debug_with_peers!( + "published execution payload envelope for block {:?} to {}", + envelope.beacon_block_root, + self.url, + ); + + Ok(()) + } + + async fn payload_attestation_data(&self, slot: Slot) -> Result> { + let url = self.endpoint(&format!( + "/eth/v1/validator/payload_attestation_data?slot={slot}" + ))?; + + let response = self + .client + .get(url.into_url()) + .timeout(self.payload_attestation_timeout()) + .send() + .await?; + + // The endpoint answers with no content when it has seen no block for the slot. + if response.status() == StatusCode::NO_CONTENT { + return Ok(None); + } + + let data = self.parse_data::(response).await?; + + ensure!( + data.slot == slot, + Error::UnexpectedPayloadAttestationSlot { + url: self.url.to_string(), + expected: slot, + actual: data.slot, + }, + ); + + Ok(Some(data)) + } + + async fn publish_payload_attestations( + &self, + messages: &[Arc], + ) -> Result<()> { + let Some(first) = messages.first() else { + return Ok(()); + }; + + let phase = self.chain_config.phase_at_slot::

(first.data.slot); + let url = self.endpoint("/eth/v1/beacon/pool/payload_attestations")?; + + let response = self + .client + .post(url.into_url()) + .header(ETH_CONSENSUS_VERSION, phase.as_ref()) + .json(&messages) + .timeout(self.payload_attestation_timeout()) + .send() + .await?; + + self.check_status(response).await?; + + debug_with_peers!( + "published {} payload attestations to {}", + messages.len(), + self.url, + ); + + Ok(()) + } + + async fn prepare_beacon_proposer( + &self, + current_slot: Slot, + proposers: &[ProposerData], + ) -> Result<()> { + let epoch = misc::compute_epoch_at_slot::

(current_slot); + + // Resent each epoch because a beacon node forgets proposers it has not heard about. + if !self.needs_proposer_preparation(epoch, proposers).await { + return Ok(()); + } + + let url = self.endpoint("/eth/v1/validator/prepare_beacon_proposer")?; + + let response = self + .client + .post(url.into_url()) + .json(&proposers) + .timeout(self.background_timeout()) + .send() + .await?; + + self.check_status(response).await?; + + *self.prepared_proposers.lock().await = Some((epoch, proposers.to_vec())); + + debug_with_peers!( + "prepared {} beacon proposers on {}", + proposers.len(), + self.url, + ); + + Ok(()) + } + + async fn register_validators( + &self, + registrations: &[SignedValidatorRegistrationV1], + ) -> Result<()> { + let url = self.endpoint("/eth/v1/validator/register_validator")?; + + let response = self + .client + .post(url.into_url()) + .json(®istrations) + .timeout(self.background_timeout()) + .send() + .await?; + + self.check_status(response).await?; + + debug_with_peers!( + "registered {} validators on {}", + registrations.len(), + self.url, + ); + + Ok(()) + } + + async fn publish_proposer_preferences( + &self, + preferences: &[Arc], + ) -> Result<()> { + let Some(first) = preferences.first() else { + return Ok(()); + }; + + let phase = self + .chain_config + .phase_at_slot::

(first.message.proposal_slot); + let url = self.endpoint("/eth/v1/validator/proposer_preferences")?; + + let response = self + .client + .post(url.into_url()) + .header(ETH_CONSENSUS_VERSION, phase.as_ref()) + .json(&preferences) + .timeout(self.background_timeout()) + .send() + .await?; + + self.check_status(response).await?; + + debug_with_peers!( + "published {} proposer preferences to {}", + preferences.len(), + self.url, + ); + + Ok(()) + } + + async fn submit_builder_preferences(&self, entries: &[BuilderPreferencesEntry]) -> Result<()> { + // A request names one fork and holds a limited number of entries. + let by_phase = entries.iter().cloned().into_group_map_by(|entry| { + self.chain_config + .phase_at_slot::

(entry.auth.message.slot) + }); + + for (phase, entries) in by_phase { + for chunk in entries.chunks(MaxBuilderPreferencesEntries::USIZE) { + let url = self.endpoint("/eth/v1/validator/builder_preferences")?; + + let response = self + .client + .post(url.into_url()) + .header(ETH_CONSENSUS_VERSION, phase.as_ref()) + .json(chunk) + .timeout(self.background_timeout()) + .send() + .await?; + + self.check_status(response).await?; + } + } + + debug_with_peers!( + "submitted {} builder preferences to {}", + entries.len(), + self.url, + ); + + Ok(()) + } +} + +fn check_attestation_index( + phase: Phase, + committee_index: CommitteeIndex, + index: u64, +) -> Result<()> { + let valid = if phase < Phase::Electra { + // Before Electra `index` names the committee, so it must be the one requested. + index == committee_index + } else { + predicates::is_valid_attestation_data_index(phase, index) + }; + + ensure!( + valid, + Error::UnexpectedAttestationIndex { + phase, + committee_index, + index, + }, + ); + + Ok(()) +} + +fn check_attestation_target(slot: Slot, target_epoch: Epoch) -> Result<()> { + let epoch = misc::compute_epoch_at_slot::

(slot); + + // Once signed, a target from the future surrounds every honest vote until it is reached. + ensure!( + target_epoch <= epoch, + Error::UnexpectedAttestationTarget { + slot, + epoch, + target_epoch, + }, + ); + + Ok(()) +} + +fn check_attestation_source(source_epoch: Epoch, target_epoch: Epoch) -> Result<()> { + // Once signed, every honest vote surrounds a source past the target until it is justified. + ensure!( + source_epoch <= target_epoch, + Error::UnexpectedAttestationSource { + source_epoch, + target_epoch, + }, + ); + + Ok(()) +} + +fn check_not_optimistic(url: &RedactingUrl, execution_optimistic: Option) -> Result<()> { + match execution_optimistic { + Some(false) => Ok(()), + Some(true) => bail!(Error::OptimisticHead { + url: url.to_string(), + }), + // Not saying is not the same as saying no, and an optimistic validator must not sign + // across the sync committee domains. + None => bail!(Error::UnknownOptimisticStatus { + url: url.to_string(), + }), + } +} + +fn parse_header(headers: &HeaderMap, name: &str) -> Result> +where + T::Err: StdError + Send + Sync + 'static, +{ + headers + .get(name) + .map(|value| value.to_str()?.parse().map_err(AnyhowError::from)) + .transpose() +} + +fn saturating_boost_factor(builder_boost_factor: Uint256) -> u64 { + // Saturates at the value the spec reserves for preferring builders outright. + u64::try_from(builder_boost_factor.into_raw()).unwrap_or(u64::MAX) +} + +fn produce_block_path( + slot: Slot, + randao_reveal: SignatureBytes, + graffiti: Option, + builder_boost_factor: u64, +) -> String { + let mut path = format!("/eth/v3/validator/blocks/{slot}?randao_reveal={randao_reveal:?}"); + + if let Some(graffiti) = graffiti { + write!(path, "&graffiti={graffiti:?}").expect("writing to a string cannot fail"); + } + + write!(path, "&builder_boost_factor={builder_boost_factor}") + .expect("writing to a string cannot fail"); + + path +} + +fn produce_block_v4_path( + slot: Slot, + randao_reveal: SignatureBytes, + graffiti: Option, +) -> String { + let mut path = format!("/eth/v4/validator/blocks/{slot}?randao_reveal={randao_reveal:?}"); + + if let Some(graffiti) = graffiti { + write!(path, "&graffiti={graffiti:?}").expect("writing to a string cannot fail"); + } + + // Always requested, so that a self-built block can be published through any node. + path.push_str("&include_payload=true"); + + path +} + +fn aggregate_attestation_path(data: AttestationData, committee_index: CommitteeIndex) -> String { + format!( + "/eth/v2/validator/aggregate_attestation?attestation_data_root={:?}\ + &slot={}&committee_index={committee_index}", + data.hash_tree_root(), + data.slot, + ) +} + +fn sync_committee_contribution_path( + slot: Slot, + subcommittee_index: SubcommitteeIndex, + beacon_block_root: H256, +) -> String { + format!( + "/eth/v1/validator/sync_committee_contribution?slot={slot}\ + &subcommittee_index={subcommittee_index}&beacon_block_root={beacon_block_root:?}", + ) +} + +#[cfg(test)] +mod tests { + use bls::{ + AggregateSignatureBytes, PublicKeyBytes, SignatureBytes, traits::SignatureBytes as _, + }; + use http_api_utils::{ValidatorAttesterDutyResponse, ValidatorPTCDutyResponse}; + use serde_json::json; + use test_case::test_case; + use types::{ + combined::Attestation, + deneb::containers::SignedBeaconBlock as DenebSignedBeaconBlock, + electra::containers::{ + AggregateAndProof as ElectraAggregateAndProof, Attestation as ElectraAttestation, + SignedAggregateAndProof as ElectraSignedAggregateAndProof, SingleAttestation, + }, + phase0::primitives::{ExecutionAddress, H256}, + preset::Mainnet, + }; + + use builder_api::gloas::containers::{BuilderRequestAuth, SignedBuilderRequestAuth}; + use httpmock::{Method, MockServer}; + use types::gloas::containers::{ + BeaconBlock as GloasBeaconBlock, SignedBeaconBlock as GloasSignedBeaconBlock, + }; + + use super::*; + + // The duty and liveness endpoints take a bare array of quoted validator indices. + #[test] + fn serializes_validator_indices_request_body() -> Result<()> { + let body = serde_json::to_value(ValidatorIndices(&[1, 2]))?; + + assert_eq!(body, json!(["1", "2"])); + + Ok(()) + } + + // The dependent root lives in the response envelope rather than the duties themselves. + #[test] + fn parses_attester_duties_response() -> Result<()> { + let response = json!({ + "dependent_root": + "0x0404040404040404040404040404040404040404040404040404040404040404", + "execution_optimistic": false, + "data": [{ + "pubkey": PublicKeyBytes::zero(), + "validator_index": "1", + "committee_index": "2", + "committee_length": "3", + "committees_at_slot": "4", + "validator_committee_index": "5", + "slot": "6", + }], + }); + + let (duties, dependent_root) = + serde_json::from_value::>>(response)? + .into_data_and_dependent_root(); + + assert_eq!(dependent_root, Some(H256::repeat_byte(4))); + + assert_eq!( + duties, + [ValidatorAttesterDutyResponse { + committee_index: 2, + committee_length: 3, + committees_at_slot: 4, + pubkey: PublicKeyBytes::zero(), + slot: 6, + validator_committee_index: 5, + validator_index: 1, + }], + ); + + Ok(()) + } + + // PTC duties carry a dependent root in the envelope like attester duties do. + #[test] + fn parses_ptc_duties_response() -> Result<()> { + let response = json!({ + "dependent_root": + "0x0404040404040404040404040404040404040404040404040404040404040404", + "execution_optimistic": false, + "data": [{ + "pubkey": PublicKeyBytes::zero(), + "validator_index": "1", + "slot": "6", + }], + }); + + let (duties, dependent_root) = + serde_json::from_value::>>(response)? + .into_data_and_dependent_root(); + + assert_eq!(dependent_root, Some(H256::repeat_byte(4))); + + assert_eq!( + duties, + [ValidatorPTCDutyResponse { + pubkey: PublicKeyBytes::zero(), + validator_index: 1, + slot: 6, + }], + ); + + Ok(()) + } + + #[test] + fn parses_payload_attestation_data_response() -> Result<()> { + let response = json!({ + "version": "gloas", + "data": { + "beacon_block_root": + "0x0101010101010101010101010101010101010101010101010101010101010101", + "slot": "6", + "payload_present": true, + "blob_data_available": false, + }, + }); + + let data = + serde_json::from_value::>(response)?.into_data(); + + assert_eq!( + data, + PayloadAttestationData { + beacon_block_root: H256::repeat_byte(1), + slot: 6, + payload_present: true, + blob_data_available: false, + }, + ); + + Ok(()) + } + + // The publication body is a bare array of messages, not `Arc`-wrapped ones. + #[test] + fn serializes_payload_attestation_message_body() -> Result<()> { + let message = Arc::new(PayloadAttestationMessage { + validator_index: 1, + data: PayloadAttestationData { + beacon_block_root: H256::repeat_byte(1), + slot: 6, + payload_present: true, + blob_data_available: true, + }, + signature: SignatureBytes::empty(), + }); + + let body = serde_json::to_value([message])?; + + assert_eq!( + body, + json!([{ + "validator_index": "1", + "data": { + "beacon_block_root": + "0x0101010101010101010101010101010101010101010101010101010101010101", + "slot": "6", + "payload_present": true, + "blob_data_available": true, + }, + "signature": SignatureBytes::empty(), + }]), + ); + + Ok(()) + } + + // A head_v2 event carries both dependent roots and the payload status. + #[test] + fn parses_head_v2_event() -> Result<()> { + let data = json!({ + "version": "gloas", + "data": { + "slot": "70", + "block": "0x0101010101010101010101010101010101010101010101010101010101010101", + "state": "0x0202020202020202020202020202020202020202020202020202020202020202", + "payload_status": "full", + "epoch_transition": false, + "current_epoch_dependent_root": + "0x0303030303030303030303030303030303030303030303030303030303030303", + "next_epoch_dependent_root": + "0x0404040404040404040404040404040404040404040404040404040404040404", + "execution_optimistic": false, + }, + }); + + let update = AnyHeadEvent::parse::(&data.to_string())?; + + assert_eq!(update.slot, 70); + assert_eq!(update.block, H256::repeat_byte(1)); + assert!(!update.execution_optimistic); + assert_eq!( + update.dependent_roots, + DependentRoots { + epoch: 2, + current: H256::repeat_byte(3), + next: H256::repeat_byte(4), + }, + ); + + Ok(()) + } + + // The epoch arrives as a string, and a number-only parse would drop every finalization. + #[test] + fn parses_finalized_checkpoint_event() -> Result<()> { + let data = json!({ + "block": "0x0101010101010101010101010101010101010101010101010101010101010101", + "state": "0x0202020202020202020202020202020202020202020202020202020202020202", + "epoch": "2", + "execution_optimistic": false, + }); + + let event = serde_json::from_str::(&data.to_string())?; + + assert_eq!(event.epoch, 2); + + Ok(()) + } + + // The deprecated event names the roots by duty period, which reads backwards. + #[test] + fn parses_head_event_with_roots_renamed() -> Result<()> { + let data = json!({ + "slot": "70", + "block": "0x0101010101010101010101010101010101010101010101010101010101010101", + "state": "0x0202020202020202020202020202020202020202020202020202020202020202", + "epoch_transition": false, + "previous_duty_dependent_root": + "0x0303030303030303030303030303030303030303030303030303030303030303", + "current_duty_dependent_root": + "0x0404040404040404040404040404040404040404040404040404040404040404", + "execution_optimistic": true, + }); + + let update = AnyHeadEvent::parse::(&data.to_string())?; + + assert!(update.execution_optimistic); + assert_eq!( + update.dependent_roots, + DependentRoots { + epoch: 2, + current: H256::repeat_byte(3), + next: H256::repeat_byte(4), + }, + ); + + Ok(()) + } + + #[test] + fn attestation_index_names_the_committee_before_electra() -> Result<()> { + check_attestation_index(Phase::Deneb, 3, 3)?; + check_attestation_index(Phase::Deneb, 3, 0).expect_err("index 0 is not committee 3"); + + Ok(()) + } + + #[test] + fn attestation_index_is_checked_against_the_phase_from_electra_on() -> Result<()> { + check_attestation_index(Phase::Electra, 3, 0)?; + check_attestation_index(Phase::Gloas, 3, 1)?; + check_attestation_index(Phase::Fulu, 3, 3).expect_err("index is unused until Gloas"); + check_attestation_index(Phase::Gloas, 3, 2).expect_err("the payload vote is 0 or 1"); + + Ok(()) + } + + // A target past the slot's epoch would lock the validator out through slashing protection. + #[test] + fn attestation_target_is_not_past_the_epoch_of_the_slot() -> Result<()> { + check_attestation_target::(65, 2)?; + check_attestation_target::(65, 1)?; + check_attestation_target::(65, 3).expect_err("epoch 3 has not begun in slot 65"); + + Ok(()) + } + + // A source past the target would lock the validator out through slashing protection. + #[test] + fn attestation_source_is_not_past_the_target() -> Result<()> { + check_attestation_source(1, 2)?; + check_attestation_source(0, 0)?; + check_attestation_source(3, 2).expect_err("epoch 3 is past target epoch 2"); + + Ok(()) + } + + // The head's true slot lives in the header, so a stale head is cached as stale. + #[test] + fn parses_block_header_response() -> Result<()> { + let response = json!({ + "execution_optimistic": false, + "finalized": false, + "data": { + "root": H256::repeat_byte(1), + "canonical": true, + "header": { + "message": { + "slot": "6", + "proposer_index": "2", + "parent_root": H256::repeat_byte(3), + "state_root": H256::repeat_byte(4), + "body_root": H256::repeat_byte(5), + }, + "signature": SignatureBytes::empty(), + }, + }, + }); + + let (block_header, execution_optimistic) = + serde_json::from_value::>(response)? + .into_data_and_execution_optimistic(); + + assert_eq!(execution_optimistic, Some(false)); + assert_eq!(block_header.root, H256::repeat_byte(1)); + assert_eq!(block_header.header.message.slot, 6); + + Ok(()) + } + + fn test_node(chain_config: ChainConfig, url: &str) -> Result { + Ok(RemoteBeaconNode::new( + Arc::new(chain_config), + Client::new(), + url.parse()?, + 32, + )) + } + + // Proposers are resent on a new epoch and on a changed list, and otherwise not, so that a + // 5000-entry body does not go out every slot. + #[tokio::test] + async fn proposers_are_prepared_once_per_epoch_unless_they_change() -> Result<()> { + let node = test_node(ChainConfig::mainnet(), "http://localhost:5052")?; + + let proposers = vec![ProposerData { + validator_index: 1, + fee_recipient: ExecutionAddress::repeat_byte(1), + }]; + + assert!(node.needs_proposer_preparation(3, &proposers).await); + + *node.prepared_proposers.lock().await = Some((3, proposers.clone())); + + assert!(!node.needs_proposer_preparation(3, &proposers).await); + assert!(node.needs_proposer_preparation(4, &proposers).await); + + let changed = vec![ProposerData { + fee_recipient: ExecutionAddress::repeat_byte(2), + ..proposers[0] + }]; + + assert!(node.needs_proposer_preparation(3, &changed).await); + + Ok(()) + } + + // A subscription goes to a node once; the sliding window offered every slot must not repeat + // it, while a changed entry for the same duty is sent again. + #[tokio::test] + async fn subscriptions_are_sent_to_a_node_once() -> Result<()> { + let node = test_node(ChainConfig::mainnet(), "http://localhost:5052")?; + + let subscription = BeaconCommitteeSubscription { + validator_index: 1, + committee_index: 2, + committees_at_slot: 3, + slot: 10, + is_aggregator: false, + }; + + assert_eq!( + node.unsent_subscriptions(&[subscription]).await, + [subscription] + ); + + node.record_sent_subscriptions(9, [subscription]).await; + + assert!(node.unsent_subscriptions(&[subscription]).await.is_empty()); + + let changed = BeaconCommitteeSubscription { + is_aggregator: true, + ..subscription + }; + + assert_eq!(node.unsent_subscriptions(&[changed]).await, [changed]); + + // Once the slot has passed, the entry is forgotten. + node.record_sent_subscriptions(11, []).await; + + assert_eq!( + node.unsent_subscriptions(&[subscription]).await, + [subscription] + ); + + Ok(()) + } + + // A node behind a proxy is reached under its path prefix, with its own query kept. + #[test_case( + "http://localhost:5052", + "/eth/v1/node/syncing", + "http://localhost:5052/eth/v1/node/syncing"; + "bare host" + )] + #[test_case( + "https://proxy/beacon", + "/eth/v1/node/syncing", + "https://proxy/beacon/eth/v1/node/syncing"; + "prefix" + )] + #[test_case( + "https://proxy/beacon/", + "/eth/v1/node/syncing", + "https://proxy/beacon/eth/v1/node/syncing"; + "prefix with trailing slash" + )] + #[test_case( + "https://proxy/beacon/?key=k", + "/eth/v1/events?topics=head", + "https://proxy/beacon/eth/v1/events?key=k&topics=head"; + "prefix with query" + )] + fn endpoint_keeps_the_prefix_and_query_of_the_node_url( + url: &str, + path: &str, + expected: &str, + ) -> Result<()> { + let node = test_node(ChainConfig::mainnet(), url)?; + + assert_eq!(node.endpoint(path)?.into_url().as_str(), expected); + + Ok(()) + } + + // The node names itself by URL, so logging it must not leak credentials. + #[test] + fn display_is_the_redacted_url() -> Result<()> { + let node = test_node( + ChainConfig::mainnet(), + "http://user:password@localhost:5052/", + )?; + + assert_eq!(node.to_string(), "http://*:*@localhost:5052/"); + + Ok(()) + } + + // A request may use the whole window between the duty's due points. + #[test] + fn timeouts_follow_the_due_points_of_the_phase() -> Result<()> { + let node = test_node(ChainConfig::mainnet(), "http://localhost:5052/")?; + + // The pre-Gloas windows of the mainnet configuration are 3334 and 3333 basis points. + assert_eq!( + node.attestation_timeout(Phase::Electra), + Duration::from_micros(4_000_800), + ); + assert_eq!( + node.aggregate_timeout(Phase::Electra), + Duration::from_micros(3_999_600), + ); + assert_eq!( + node.attestation_timeout(Phase::Gloas), + Duration::from_secs(3) + ); + assert_eq!(node.aggregate_timeout(Phase::Gloas), Duration::from_secs(6)); + assert_eq!(node.head_timeout(), Duration::from_secs(3)); + + // Publishing an attestation may take the rest of the slot, not just until aggregation. + assert_eq!( + node.publish_timeout(Phase::Electra), + Duration::from_micros(8_000_400), + ); + assert_eq!(node.publish_timeout(Phase::Gloas), Duration::from_secs(9)); + assert_eq!(node.background_timeout(), Duration::from_secs(6)); + + Ok(()) + } + + // Short slots must not shrink timeouts below what real networks can answer within. + #[test] + fn timeouts_do_not_fall_below_the_minimum() -> Result<()> { + let config = ChainConfig { + slot_duration_ms: Duration::from_secs(2), + ..ChainConfig::mainnet() + }; + + let node = test_node(config, "http://localhost:5052/")?; + + assert_eq!(node.attestation_timeout(Phase::Electra), MIN_TIMEOUT); + assert_eq!(node.background_timeout(), MIN_TIMEOUT); + + Ok(()) + } + + // A block is produced until attesters vote and published until the slot ends. + #[test] + fn block_production_and_publishing_split_the_slot_at_the_attestation_deadline() -> Result<()> { + let node = test_node(ChainConfig::mainnet(), "http://localhost:5052/")?; + + assert_eq!( + node.block_timeout(Phase::Electra), + Duration::from_micros(3_999_600), + ); + assert_eq!(node.block_timeout(Phase::Gloas), Duration::from_secs(3)); + assert_eq!( + node.publish_timeout(Phase::Electra), + Duration::from_micros(8_000_400), + ); + assert_eq!(node.publish_timeout(Phase::Gloas), Duration::from_secs(9)); + + Ok(()) + } + + // The genesis fork version is decoded, as the health poll compares it. + #[test] + fn parses_genesis_response() -> Result<()> { + let response = json!({ + "data": { + "genesis_time": "1590832934", + "genesis_validators_root": + "0x0000000000000000000000000000000000000000000000000000000000000000", + "genesis_fork_version": "0x00000001", + }, + }); + + let data = serde_json::from_value::>(response)?.into_data(); + + assert_eq!(data.genesis_fork_version, Version::from([0, 0, 0, 1])); + + Ok(()) + } + + // Slots, indices and epochs are quoted in JSON and decode to numbers. + #[test] + fn parses_attestation_data_response() -> Result<()> { + let response = json!({ + "data": { + "slot": "1", + "index": "1", + "beacon_block_root": + "0x0101010101010101010101010101010101010101010101010101010101010101", + "source": { + "epoch": "2", + "root": "0x0202020202020202020202020202020202020202020202020202020202020202", + }, + "target": { + "epoch": "3", + "root": "0x0303030303030303030303030303030303030303030303030303030303030303", + }, + }, + }); + + let data = serde_json::from_value::>(response)?.into_data(); + + assert_eq!(data.slot, 1); + assert_eq!(data.index, 1); + assert_eq!(data.beacon_block_root, H256::repeat_byte(1)); + assert_eq!(data.source.epoch, 2); + assert_eq!(data.target.epoch, 3); + + Ok(()) + } + + // The publication body is the bare attestation, not an enveloped or tagged one. + #[test] + fn serializes_single_attestation_body() -> Result<()> { + let attestation = Attestation::::from(SingleAttestation { + committee_index: 4, + attester_index: 5, + data: AttestationData::default(), + signature: SignatureBytes::empty(), + }); + + let body = serde_json::to_value([&attestation])?; + + assert_eq!(body[0]["committee_index"], json!("4")); + assert_eq!(body[0]["attester_index"], json!("5")); + assert_eq!(body[0]["data"]["slot"], json!("0")); + + Ok(()) + } + + fn electra_aggregate_body() -> serde_json::Value { + json!({ + "aggregation_bits": "0x07", + "data": { + "slot": "6", + "index": "0", + "beacon_block_root": H256::zero(), + "source": { "epoch": "0", "root": H256::zero() }, + "target": { "epoch": "0", "root": H256::zero() }, + }, + "signature": AggregateSignatureBytes::empty(), + "committee_bits": "0x0100000000000000", + }) + } + + // Why `aggregate_attestation` parses by phase instead of letting serde choose: Electra and + // Gloas attestations have the same fields, so an untagged read of either yields Electra. + #[test] + fn a_gloas_attestation_is_untagged_as_electra() -> Result<()> { + let attestation = serde_json::from_value::>(electra_aggregate_body())?; + + assert!(matches!(attestation, Attestation::Electra(_))); + + Ok(()) + } + + // The phases that `deny_unknown_fields` does separate stay separated. + #[test] + fn an_electra_aggregate_is_not_a_phase0_one() -> Result<()> { + let body = electra_aggregate_body(); + + serde_json::from_value::>(body.clone())?; + serde_json::from_value::>(body.clone())?; + + serde_json::from_value::>(body) + .expect_err("committee bits should not fit a phase 0 attestation"); + + Ok(()) + } + + // Signing across the sync committee domains requires a head the node has verified. + #[test] + fn check_not_optimistic_accepts_only_a_verified_head() -> Result<()> { + let url = "http://localhost:5052/".parse()?; + + check_not_optimistic(&url, Some(false))?; + check_not_optimistic(&url, Some(true)).expect_err("an optimistic head is not verified"); + check_not_optimistic(&url, None).expect_err("an unreported head is not verified"); + + Ok(()) + } + + // The node verifies the reveal, so it and the graffiti must go into the query in full. + #[test] + fn produce_block_path_spells_out_the_reveal_and_graffiti() { + let path = produce_block_path( + 6, + SignatureBytes::default(), + Some(H256::repeat_byte(1)), + 100, + ); + + assert!(path.starts_with(&format!( + "/eth/v3/validator/blocks/6?randao_reveal=0x{}", + "00".repeat(96), + ))); + assert!(path.contains(&format!("&graffiti=0x{}", "01".repeat(32)))); + assert!(path.ends_with("&builder_boost_factor=100")); + } + + // Nodes take the factor as a `uint64`; `prefer_builder_proposals` configures `Uint256::MAX`. + #[test] + fn boost_factor_saturates_at_u64_max() { + assert_eq!(saturating_boost_factor(Uint256::from_u64(100)), 100); + assert_eq!(saturating_boost_factor(Uint256::MAX), u64::MAX); + } + + // From Gloas on the boost factor travels in the body and the payload is always asked for. + #[test] + fn produce_block_v4_path_asks_for_the_payload() { + let path = produce_block_v4_path(6, SignatureBytes::default(), Some(H256::repeat_byte(1))); + + assert!(path.starts_with(&format!( + "/eth/v4/validator/blocks/6?randao_reveal=0x{}", + "00".repeat(96), + ))); + assert!(path.contains(&format!("&graffiti=0x{}", "01".repeat(32)))); + assert!(path.ends_with("&include_payload=true")); + } + + // The node decodes the body as JSON with quoted integers, as elsewhere in the Beacon API. + #[test] + fn serializes_builder_config_request_body() -> Result<()> { + let body = serde_json::to_value(BuilderConfig { + min_bid: 0, + builder_boost_factor: 100, + builders: ContiguousList::default(), + })?; + + assert_eq!( + body, + json!({ "min_bid": "0", "builder_boost_factor": "100", "builders": [] }), + ); + + Ok(()) + } + + // `produceBlockV4` with the payload puts the block first and the envelope right after it. + #[test] + fn gloas_block_contents_put_the_block_before_the_envelope() -> Result<()> { + let contents = GloasBlockContents:: { + block: GloasBeaconBlock::default(), + execution_payload_envelope: ExecutionPayloadEnvelope::default(), + kzg_proofs: ContiguousList::default(), + blobs: ContiguousList::default(), + }; + + let block_bytes = contents.block.to_ssz()?; + let envelope_bytes = contents.execution_payload_envelope.to_ssz()?; + let bytes = contents.to_ssz()?; + + // Four offsets precede the block; the empty lists after the envelope add nothing. + let block_offset = 16; + let envelope_offset = block_offset + block_bytes.len(); + + assert_eq!(&bytes[..4], 16_u32.to_le_bytes()); + assert_eq!(&bytes[block_offset..envelope_offset], block_bytes); + assert_eq!(&bytes[envelope_offset..], envelope_bytes); + + let read_back = GloasBlockContents::::from_ssz_default(bytes)?; + + assert_eq!(read_back.block, contents.block); + assert_eq!( + read_back.execution_payload_envelope, + contents.execution_payload_envelope, + ); + + Ok(()) + } + + // `publishBlockV2` from Deneb on takes one container with the block ahead of the blob data. + #[test] + fn signed_block_contents_put_the_block_first() -> Result<()> { + let signed_block = SignedBeaconBlock::::Deneb(DenebSignedBeaconBlock { + message: Hc::from(DenebBeaconBlock::default()), + signature: SignatureBytes::default(), + }); + + let block_bytes = signed_block.to_ssz()?; + + let contents = SignedBlockContents { + signed_block: &signed_block, + kzg_proofs: &KzgProofs::empty_deneb(), + blobs: &ContiguousList::default(), + } + .to_ssz()?; + + // Three offsets precede the block; the empty lists after it add nothing. + let block_offset = 12; + + assert_eq!(&contents[..4], 12_u32.to_le_bytes()); + assert_eq!(&contents[block_offset..], block_bytes); + + Ok(()) + } + + // The root goes into the query in full; `Display` would abbreviate it. + #[test] + fn aggregate_attestation_path_spells_out_the_root() { + let data = AttestationData { + slot: 6, + ..AttestationData::default() + }; + + let path = aggregate_attestation_path(data, 3); + + let root = path + .split("attestation_data_root=") + .nth(1) + .and_then(|rest| rest.split('&').next()) + .expect("path contains the attestation data root"); + + assert!(root.starts_with("0x")); + assert_eq!(root.len(), 66); + assert!(path.ends_with("&slot=6&committee_index=3")); + } + + // The root goes into the query in full, for the same reason. + #[test] + fn sync_committee_contribution_path_spells_out_the_root() { + let path = sync_committee_contribution_path(6, 3, H256::repeat_byte(1)); + + assert_eq!( + path, + "/eth/v1/validator/sync_committee_contribution?slot=6&subcommittee_index=3\ + &beacon_block_root=0x0101010101010101010101010101010101010101010101010101010101010101", + ); + } + + // The publication body is the bare aggregate and proof, not an `Arc`-wrapped one. + #[test] + fn serializes_aggregate_and_proof_body() -> Result<()> { + let aggregate_and_proof = Arc::new(SignedAggregateAndProof::::from( + ElectraSignedAggregateAndProof { + message: ElectraAggregateAndProof { + aggregator_index: 6, + aggregate: ElectraAttestation::default(), + selection_proof: SignatureBytes::empty(), + }, + signature: SignatureBytes::empty(), + }, + )); + + let body = serde_json::to_value([&aggregate_and_proof])?; + + assert_eq!(body[0]["message"]["aggregator_index"], json!("6")); + assert_eq!(body[0]["message"]["aggregate"]["data"]["slot"], json!("0")); + + Ok(()) + } + + // The endpoint takes a bare array of subscriptions with quoted numbers. + #[test] + fn serializes_beacon_committee_subscription_body() -> Result<()> { + let body = serde_json::to_value([BeaconCommitteeSubscription { + validator_index: 1, + committee_index: 2, + committees_at_slot: 3, + slot: 4, + is_aggregator: true, + }])?; + + assert_eq!(body[0]["validator_index"], json!("1")); + assert_eq!(body[0]["committee_index"], json!("2")); + assert_eq!(body[0]["committees_at_slot"], json!("3")); + assert_eq!(body[0]["slot"], json!("4")); + assert_eq!(body[0]["is_aggregator"], json!(true)); + + Ok(()) + } + + const BUILDER_URL: &str = "https://builder.example.com"; + + fn gloas_config() -> ChainConfig { + ChainConfig::mainnet().rapid_upgrade() + } + + fn gloas_slot(chain_config: &ChainConfig) -> Slot { + misc::compute_start_slot_at_epoch::(chain_config.gloas_fork_epoch) + } + + fn builder_entry() -> Result { + Ok(BuilderEntry { + url: BUILDER_URL.try_into()?, + auth: SignedBuilderRequestAuth::default(), + builder_pubkeys: ContiguousList::default(), + max_execution_payment: 1, + min_bid: 2, + builder_boost_factor: 3, + }) + } + + // The builders and the minimum bid go to the node in the body, and the builder whose bid the + // node took comes back in a header. + #[tokio::test] + async fn produce_block_v4_sends_builders_and_reads_the_builder_url() -> Result<()> { + let server = MockServer::start_async().await; + let chain_config = gloas_config(); + let slot = gloas_slot(&chain_config); + + let block = GloasBeaconBlock:: { + slot, + ..GloasBeaconBlock::default() + }; + + let builders = ContiguousList::try_from(vec![builder_entry()?])?; + + let expected_body = serde_json::to_value(BuilderConfig { + min_bid: 10, + builder_boost_factor: 100, + builders: builders.clone(), + })?; + + let mock = server.mock(|when, then| { + when.method(Method::POST) + .path(format!("/eth/v4/validator/blocks/{slot}")) + .header(ETH_CONSENSUS_VERSION, "gloas") + .json_body(expected_body); + + then.status(200) + .header(CONTENT_TYPE.as_str(), APPLICATION_OCTET_STREAM.as_ref()) + .header(ETH_EXECUTION_PAYLOAD_INCLUDED, "false") + .header(ETH_BUILDER_URL, BUILDER_URL) + .body(block.to_ssz().expect("a block can be serialized")); + }); + + let node = test_node(chain_config, &server.base_url())?; + + let produced = BeaconNodeApi::::produce_block( + &node, + slot, + SignatureBytes::default(), + None, + Uint256::from_u64(100), + 10, + &builders, + ) + .await?; + + mock.assert_async().await; + + assert_eq!(produced.builder_url, Some(BUILDER_URL.try_into()?)); + + Ok(()) + } + + // The node is only told which builder to forward the block to when one won the slot. + #[tokio::test] + async fn publish_block_passes_the_builder_url_back() -> Result<()> { + let server = MockServer::start_async().await; + let chain_config = gloas_config(); + + let signed_block = Arc::new(SignedBeaconBlock::from(GloasSignedBeaconBlock:: { + message: Hc::from(GloasBeaconBlock { + slot: gloas_slot(&chain_config), + ..GloasBeaconBlock::default() + }), + signature: SignatureBytes::default(), + })); + + let with_builder = server.mock(|when, then| { + when.method(Method::POST) + .path("/eth/v2/beacon/blocks") + .header(ETH_BUILDER_URL, BUILDER_URL); + + then.status(200); + }); + + let without_builder = server.mock(|when, then| { + when.method(Method::POST) + .path("/eth/v2/beacon/blocks") + .header_missing(ETH_BUILDER_URL); + + then.status(200); + }); + + let node = test_node(chain_config, &server.base_url())?; + let builder_url = BuilderUrl::try_from(BUILDER_URL)?; + + BeaconNodeApi::::publish_block( + &node, + &signed_block, + None, + None, + Some(&builder_url), + ) + .await?; + + BeaconNodeApi::::publish_block(&node, &signed_block, None, None, None).await?; + + with_builder.assert_async().await; + without_builder.assert_async().await; + + Ok(()) + } + + // Builder preferences are submitted under the fork of the slots they are for. + #[tokio::test] + async fn submits_builder_preferences_under_their_fork() -> Result<()> { + let server = MockServer::start_async().await; + let chain_config = gloas_config(); + + let entries = [BuilderPreferencesEntry { + proposer_pubkey: PublicKeyBytes::zero(), + url: BUILDER_URL.try_into()?, + auth: SignedBuilderRequestAuth { + message: BuilderRequestAuth { + slot: gloas_slot(&chain_config), + ..BuilderRequestAuth::default() + }, + signature: SignatureBytes::default(), + }, + max_execution_payment: 1, + }]; + + let mock = server.mock(|when, then| { + when.method(Method::POST) + .path("/eth/v1/validator/builder_preferences") + .header(ETH_CONSENSUS_VERSION, "gloas") + .json_body(serde_json::to_value(&entries).expect("entries can be serialized")); + + then.status(200); + }); + + let node = test_node(chain_config, &server.base_url())?; + + BeaconNodeApi::::submit_builder_preferences(&node, &entries).await?; + + mock.assert_async().await; + + Ok(()) + } +} diff --git a/validator/src/remote_beacon_nodes.rs b/validator/src/remote_beacon_nodes.rs new file mode 100644 index 000000000..653cea9cb --- /dev/null +++ b/validator/src/remote_beacon_nodes.rs @@ -0,0 +1,319 @@ +use core::cmp::Reverse; +use std::{collections::HashMap, sync::Arc}; + +use anyhow::{Result, bail, ensure}; +use bls::PublicKeyBytes; +use futures::{channel::mpsc::UnboundedSender, future::join_all}; +use logging::{info_with_peers, warn_with_peers}; +use std_ext::ArcExt; +use thiserror::Error; +use types::{ + nonstandard::PublishedDuty, + phase0::primitives::{H256, Slot, ValidatorIndex}, + preset::Preset, +}; + +use crate::{ + beacon_nodes::first_success, + chain_events::stream_events, + health::Health, + messages::InternalMessage, + remote_beacon_node::{Genesis, RemoteBeaconNode}, +}; + +/// Only possible on startup and not fixed by restarting, so the restart loop exits on these. +#[derive(Debug, Error)] +pub enum StartupError { + #[error("beacon node at {node} is on a different network")] + DifferentNetwork { node: String }, + #[error( + "beacon nodes at {first} and {second} are on different chains: \ + genesis validators roots {first_root:?} and {second_root:?}" + )] + DifferentChains { + first: String, + second: String, + first_root: H256, + second_root: H256, + }, + #[error( + "beacon nodes given with --beacon-node-urls are on a chain with genesis validators root \ + {actual:?} where {expected:?} was expected" + )] + UnexpectedChain { expected: H256, actual: H256 }, +} + +pub struct RemoteBeaconNodes { + nodes: Vec>, + publish_to_every_node: Vec, + use_builder: bool, +} + +impl RemoteBeaconNodes { + #[must_use] + pub const fn new( + nodes: Vec>, + publish_to_every_node: Vec, + use_builder: bool, + ) -> Self { + Self { + nodes, + publish_to_every_node, + use_builder, + } + } + + #[must_use] + pub fn publish_to_every_node(&self) -> &[PublishedDuty] { + &self.publish_to_every_node + } + + #[must_use] + pub const fn use_builder(&self) -> bool { + self.use_builder + } + + pub fn spawn_event_streams(&self, internal_tx: UnboundedSender) { + tokio::spawn(stream_events::

( + self.nodes.iter().map(ArcExt::clone_arc).collect(), + internal_tx, + )); + } + + pub async fn agreed_genesis(&self) -> Result> { + let responses = join_all( + self.nodes + .iter() + .map(|node| async move { (node, node.genesis().await) }), + ) + .await; + + let mut agreed: Option<(&Arc, Genesis)> = None; + + for (node, result) in responses { + match result { + Ok(genesis) => match &agreed { + Some((first, first_genesis)) => ensure!( + first_genesis.genesis_validators_root == genesis.genesis_validators_root, + StartupError::DifferentChains { + first: first.to_string(), + second: node.to_string(), + first_root: first_genesis.genesis_validators_root, + second_root: genesis.genesis_validators_root, + }, + ), + None => agreed = Some((node, genesis)), + }, + Err(error) => { + warn_with_peers!("{node} beacon node failed to report genesis: {error:?}"); + } + } + } + + Ok(agreed.map(|(_, genesis)| genesis)) + } + + pub fn seed_genesis_validators_root(&self, genesis_validators_root: H256) { + for node in &self.nodes { + node.seed_genesis_validators_root(genesis_validators_root); + } + } + + pub async fn validator_indices( + &self, + public_keys: &[PublicKeyBytes], + ) -> Result> { + let mut attempts = vec![]; + + for node in self.serving() { + let attempt = node.validator_indices(public_keys); + attempts.push((node, attempt)); + } + + first_success("resolve validator indices", attempts) + .await + .map(|(_, indices)| indices) + } + + pub fn serving(&self) -> impl Iterator> { + let mut nodes = self + .nodes + .iter() + .filter(|node| node.health().can_serve()) + .collect::>(); + + // Among equally healthy nodes the newest head goes first, then the configured order. + nodes.sort_by_cached_key(|node| { + let head_slot = node.chain_head().cached().map_or(0, |(slot, _)| slot); + + Reverse((node.health(), head_slot)) + }); + + nodes.into_iter() + } + + pub async fn refresh(&self, slot: Slot) { + join_all(self.nodes.iter().map(|node| node.refresh_health(slot))).await; + join_all(self.nodes.iter().map(|node| node.refresh_head(slot))).await; + + if self.nodes.iter().any(|node| node.health().is_ready()) { + return; + } + + if self + .nodes + .iter() + .any(|node| node.health() > Health::Unreachable) + { + warn_with_peers!("no remote beacon node is fully synced"); + } else { + warn_with_peers!("no remote beacon node can serve duties"); + } + } + + pub async fn check_on_startup(&self, slot: Slot) -> Result<()> { + info_with_peers!("checking remote beacon nodes"); + + self.refresh(slot).await; + + let ready = self + .nodes + .iter() + .filter(|node| node.health().is_ready()) + .count(); + + info_with_peers!( + "{ready} of {} remote beacon nodes are ready to serve duties", + self.nodes.len(), + ); + + // Only a wrong network is fatal; anything else may be fine by the next poll. + if let Some(node) = self + .nodes + .iter() + .find(|node| node.health() == Health::Incompatible) + { + bail!(StartupError::DifferentNetwork { + node: node.to_string(), + }); + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use reqwest::Client; + use types::config::Config as ChainConfig; + + use super::*; + + fn node(url: &str) -> Result> { + Ok(Arc::new(RemoteBeaconNode::new( + Arc::new(ChainConfig::mainnet()), + Client::new(), + url.parse()?, + 32, + ))) + } + + fn order(nodes: &RemoteBeaconNodes) -> Vec> { + nodes.serving().map(ArcExt::clone_arc).collect() + } + + // Nodes stuck on the same block must not outvote the one that follows the chain. + #[test] + fn the_newest_head_outranks_a_head_more_nodes_hold() -> Result<()> { + let first = node("http://first")?; + let second = node("http://second")?; + let ahead = node("http://ahead")?; + + first.chain_head().update(10, H256::repeat_byte(2)); + second.chain_head().update(10, H256::repeat_byte(2)); + ahead.chain_head().update(11, H256::repeat_byte(1)); + + let nodes = RemoteBeaconNodes::new( + vec![first.clone_arc(), second.clone_arc(), ahead.clone_arc()], + vec![], + false, + ); + + let order = order(&nodes); + + assert!(Arc::ptr_eq(&order[0], &ahead)); + assert!(Arc::ptr_eq(&order[1], &first)); + assert!(Arc::ptr_eq(&order[2], &second)); + + Ok(()) + } + + // The newer head wins over the configured order. + #[test] + fn a_newer_head_outranks_the_configured_order() -> Result<()> { + let behind = node("http://behind")?; + let ahead = node("http://ahead")?; + + behind.chain_head().update(10, H256::repeat_byte(1)); + ahead.chain_head().update(11, H256::repeat_byte(2)); + + let nodes = + RemoteBeaconNodes::new(vec![behind.clone_arc(), ahead.clone_arc()], vec![], false); + let order = order(&nodes); + + assert!(Arc::ptr_eq(&order[0], &ahead)); + assert!(Arc::ptr_eq(&order[1], &behind)); + + Ok(()) + } + + // Health decides first; the head only orders nodes of equal health. + #[test] + fn a_ready_node_outranks_a_fresher_head_on_a_degraded_one() -> Result<()> { + let ready = node("http://ready")?; + let unusable = node("http://unusable")?; + + ready.set_health(Health::Ready); + unusable.set_health(Health::Unusable); + ready.chain_head().update(10, H256::repeat_byte(1)); + unusable.chain_head().update(11, H256::repeat_byte(2)); + + let nodes = + RemoteBeaconNodes::new(vec![unusable.clone_arc(), ready.clone_arc()], vec![], false); + let order = order(&nodes); + + assert!(Arc::ptr_eq(&order[0], &ready)); + assert!(Arc::ptr_eq(&order[1], &unusable)); + + Ok(()) + } + + // An unreachable node is kept as the last choice; one on another network is never asked. + #[test] + fn an_unreachable_node_serves_last_and_an_incompatible_one_not_at_all() -> Result<()> { + let ready = node("http://ready")?; + let unreachable = node("http://unreachable")?; + let incompatible = node("http://incompatible")?; + + ready.set_health(Health::Ready); + unreachable.set_health(Health::Unreachable); + incompatible.set_health(Health::Incompatible); + + let nodes = RemoteBeaconNodes::new( + vec![ + incompatible.clone_arc(), + unreachable.clone_arc(), + ready.clone_arc(), + ], + vec![], + false, + ); + let order = order(&nodes); + + assert_eq!(order.len(), 2); + assert!(Arc::ptr_eq(&order[0], &ready)); + assert!(Arc::ptr_eq(&order[1], &unreachable)); + + Ok(()) + } +} diff --git a/validator/src/slot_head.rs b/validator/src/slot_head.rs index 3d5cd7276..76b997fc5 100644 --- a/validator/src/slot_head.rs +++ b/validator/src/slot_head.rs @@ -1,4 +1,4 @@ -use core::fmt::Debug; +use core::{fmt::Debug, marker::PhantomData}; use std::sync::Arc; use anyhow::Result; @@ -6,7 +6,7 @@ use bls::{PublicKeyBytes, SignatureBytes}; use futures::lock::Mutex; use helper_functions::{ accessors, misc, predicates, - signing::{SignForSingleFork, SignForSingleForkAtSlot as _}, + signing::{SignForSingleFork, SignForSingleForkAtSlot}, }; use itertools::Itertools as _; use logging::warn_with_peers; @@ -17,97 +17,97 @@ use types::{ containers::{SyncAggregatorSelectionData, SyncCommitteeMessage}, primitives::SubcommitteeIndex, }, - cache::IndexSlice, combined::BeaconState, config::Config, - nonstandard::{Phase, RelativeEpoch}, - phase0::primitives::{CommitteeIndex, Epoch, H256, Slot, SubnetId, ValidatorIndex}, + nonstandard::{ForkInfo, Phase}, + phase0::primitives::{Epoch, H256, Slot, ValidatorIndex}, preset::Preset, - traits::BeaconState as _, }; +#[derive(Clone)] pub struct SlotHead { pub config: Arc, + pub slot: Slot, pub beacon_block_root: H256, - pub beacon_state: Arc>, + /// All of a state that signing needs, so that it does not depend on holding one. + pub fork_info: ForkInfo, + /// The preset the slot's epoch and phase are computed with; no field depends on it. + pub phantom: PhantomData

, } impl SlotHead

{ #[must_use] - pub fn slot(&self) -> Slot { - self.beacon_state.slot() + pub fn from_state( + config: Arc, + slot: Slot, + beacon_block_root: H256, + state: &BeaconState

, + ) -> Self { + Self { + config, + slot, + beacon_block_root, + fork_info: ForkInfo::from_state(state), + phantom: PhantomData, + } } #[must_use] - pub fn phase(&self) -> Phase { - self.beacon_state.phase() + pub const fn slot(&self) -> Slot { + self.slot } #[must_use] - pub fn current_epoch(&self) -> Epoch { - accessors::get_current_epoch(&self.beacon_state) + pub fn phase(&self) -> Phase { + self.config.phase_at_slot::

(self.slot) } #[must_use] - pub fn public_key(&self, validator_index: ValidatorIndex) -> &PublicKeyBytes { - self.beacon_state - .validators() - .pubkey(validator_index) - .expect( - "SlotHead::public_key should only be called with \ - indices of validators in SlotHead.beacon_state", - ) + pub fn current_epoch(&self) -> Epoch { + misc::compute_epoch_at_slot::

(self.slot) } - pub fn proposer_index(&self) -> Result { - accessors::get_beacon_proposer_index(&self.config, &self.beacon_state) + pub fn proposer_index(&self, beacon_state: &BeaconState

) -> Result { + accessors::get_beacon_proposer_index(&self.config, beacon_state) } - pub fn next_proposer_index(&self) -> Result { + pub fn next_proposer_index(&self, beacon_state: &BeaconState

) -> Result { accessors::get_beacon_proposer_index_at_slot( &self.config, - &self.beacon_state, + beacon_state, self.slot().saturating_add(1), ) } - pub fn beacon_committee(&self, committee_index: CommitteeIndex) -> Result> { - accessors::beacon_committee(&self.beacon_state, self.slot(), committee_index) - } - #[must_use] pub fn has_sync_committee(&self) -> bool { - self.beacon_state.phase() >= Phase::Altair - } - - pub fn subnet_id(&self, slot: Slot, committee_index: CommitteeIndex) -> Result { - let committees_per_slot = - accessors::get_committee_count_per_slot(&self.beacon_state, RelativeEpoch::Current)?; - - misc::compute_subnet_for_attestation::

(committees_per_slot, slot, committee_index) + self.phase() >= Phase::Altair } /// pub async fn sync_committee_messages( &self, slot: Slot, + beacon_block_root: H256, validator_indices_with_pubkeys: I, signer: &Signer, ) -> Result> where I: IntoIterator + Send, { + // The head of the node the duty is performed against, not necessarily `self`'s. let (triples, validator_indices): (Vec<_>, Vec<_>) = validator_indices_with_pubkeys .into_iter() .map(|(validator_index, public_key)| { - let triple = SigningTriple { + let triple = SigningTriple::

{ message: SigningMessage::SyncCommitteeMessage { - beacon_block_root: self.beacon_block_root, + beacon_block_root, slot, }, - signing_root: self.beacon_block_root.signing_root( + signing_root: SignForSingleForkAtSlot::

::signing_root_from_fork_info( + &beacon_block_root, &self.config, - &self.beacon_state, + self.fork_info, self.slot(), ), public_key, @@ -120,15 +120,12 @@ impl SlotHead

{ let signer_snapshot = signer.load(); let messages = signer_snapshot - .sign_triples_without_slashing_protection( - triples, - Some(self.beacon_state.as_ref().into()), - ) + .sign_triples_without_slashing_protection(triples, Some(self.fork_info)) .await? .zip(validator_indices) .map(move |(signature, validator_index)| SyncCommitteeMessage { slot, - beacon_block_root: self.beacon_block_root, + beacon_block_root, validator_index, signature: signature.into(), }) @@ -150,19 +147,20 @@ impl SlotHead

{ subcommittee_index, }; - SigningTriple { + SigningTriple::

{ message: SigningMessage::SyncAggregatorSelectionData(selection_data), - signing_root: selection_data.signing_root(&self.config, &self.beacon_state), + signing_root: SignForSingleFork::

::signing_root_from_fork_info( + &selection_data, + &self.config, + self.fork_info, + ), public_key, } }); signer .load() - .sign_triples_without_slashing_protection( - triples, - Some(self.beacon_state.as_ref().into()), - ) + .sign_triples_without_slashing_protection(triples, Some(self.fork_info)) .await? .map(|signature| { let selection_proof = signature.into(); @@ -185,10 +183,11 @@ impl SlotHead

{ .sign_triples( core::iter::once(SigningTriple { message, - signing_root: block.signing_root(&self.config, &self.beacon_state), + signing_root: block.signing_root_from_fork_info(&self.config, self.fork_info), public_key, }), - self.beacon_state.as_ref(), + self.fork_info, + self.current_epoch(), slashing_protector, ) .await diff --git a/validator/src/tasks.rs b/validator/src/tasks.rs index ba17acfad..ad03a42f9 100644 --- a/validator/src/tasks.rs +++ b/validator/src/tasks.rs @@ -1,121 +1,313 @@ use std::sync::Arc; -use eth1_api::ApiController; +use block_producer::ProposerData; use fork_choice_control::Wait; -use futures::channel::mpsc::UnboundedSender; +use helper_functions::misc; +use itertools::Itertools as _; +use keymanager::ProposerConfigs; use logging::{debug_with_peers, warn_with_peers}; -use p2p::ToSubnetService; +use p2p::BeaconCommitteeSubscription; use tracing::instrument; use types::{ - combined::BeaconState, config::Config as ChainConfig, phase0::primitives::Slot, preset::Preset, - traits::BeaconState as _, + config::Config as ChainConfig, + nonstandard::{ForkInfo, Phase}, + phase0::primitives::{Epoch, Slot, ValidatorIndex}, + preset::Preset, }; -use crate::own_beacon_committee_members::{BeaconCommitteeMember, OwnBeaconCommitteeMembers}; +use crate::{ + beacon_node_api::BeaconNodeApi as _, + beacon_nodes::BeaconNodes, + local_beacon_node::proposer_dependent_epoch, + misc::{OwnDuties, slots_by_epoch, slots_to_compute_in_advance}, + own_beacon_committee_members::OwnBeaconCommitteeMembers, + own_proposer_duties::OwnProposerDuties, + own_ptc_members::OwnPTCMembers, + own_sync_committee_members::OwnSyncCommitteeMembers, +}; pub struct UpdateBeaconCommitteeSubscriptionsTask { pub chain_config: Arc, - pub controller: ApiController, - pub beacon_state: Arc>, - pub own_beacon_committee_members: Arc, - pub subnet_service_tx: UnboundedSender, + pub own_duties: Arc, + pub proposer_configs: Arc, + pub beacon_nodes: BeaconNodes, pub wait_group: W, } impl UpdateBeaconCommitteeSubscriptionsTask { + #[expect(clippy::too_many_lines)] #[instrument( skip_all, - fields(slot = %self.beacon_state.slot()), + fields(slot = %self.beacon_nodes.head().slot()), name="UpdateBeaconCommitteeSubscriptionsTask::run", )] pub async fn run(self) { let Self { chain_config, - controller, - mut beacon_state, - own_beacon_committee_members, - subnet_service_tx, + own_duties, + proposer_configs, + beacon_nodes, wait_group, } = self; - let current_slot = beacon_state.slot(); + let OwnDuties { + validator_indices: own_validator_indices, + beacon_committee_members: own_beacon_committee_members, + proposer_duties: own_proposer_duties, + ptc_members: own_ptc_members, + .. + } = own_duties.as_ref(); + + let current_slot = beacon_nodes.head().slot(); + let fork_info = beacon_nodes.head().fork_info; + let current_epoch = misc::compute_epoch_at_slot::

(current_slot); + let validator_indices = own_validator_indices.sorted(); + + // The built-in node looks the fee recipient up itself when it produces a block. + if !beacon_nodes.has_local_node() { + let configured_fee_recipient = proposer_configs.configured_fee_recipients(); + + // Built every slot; each node sends it on only when it has something new to say. + let proposers = own_validator_indices + .load() + .iter() + .filter_map(|(pubkey, validator_index)| { + Some(ProposerData { + validator_index: (*validator_index)?, + fee_recipient: configured_fee_recipient(*pubkey)?, + }) + }) + .collect_vec(); + + if let Err(error) = beacon_nodes + .prepare_beacon_proposer(current_slot, &proposers) + .await + { + warn_with_peers!("failed to prepare beacon proposers: {error:?}"); + } + } + + let mut subscriptions = vec![]; + + let prefetch_slots = beacon_nodes.prefetch_slots(current_slot); - for slot in OwnBeaconCommitteeMembers::slots_to_compute_in_advance(current_slot) { - let dependent_root = match controller - .attestation_committee_dependent_root_for_slot(&beacon_state, slot) + for (epoch, slots) in slots_by_epoch::

(prefetch_slots) { + if validator_indices.is_empty() { + continue; + } + + let slots = match own_beacon_committee_members + .slots_to_compute_at_epoch(&beacon_nodes, epoch, slots, &validator_indices) + .await { + Ok(slots) => slots, Err(error) => { warn_with_peers!( - "unable to find dependent root for slot: {slot} against state with \ - slot: {}: {error:?}", - beacon_state.slot(), + "unable to find dependent root for epoch: {epoch} in slot \ + {current_slot}: {error:?}", ); continue; } - Ok(dependent_root) => dependent_root, }; - if own_beacon_committee_members - .needs_to_compute_members_at_slot(dependent_root, slot) - .await - { - let phase_at_slot = chain_config.phase_at_slot::

(slot); - - if chain_config.phase_at_slot::

(current_slot) != phase_at_slot { - beacon_state = match controller - .preprocessed_state_at_epoch(chain_config.fork_epoch(phase_at_slot)) - .await - { - Ok(with_status) => with_status.value, - Err(error) => { - warn_with_peers!( - "failed to preprocess next fork beacon state for beacon committee \ - subscriptions: {error:?}", - ); - break; - } - } - } + // Attester duties may be cached from an earlier slot while PTC duties are not yet. + if let Some(slots) = slots { + debug_with_peers!("updating beacon committee subscriptions {epoch} {current_slot}"); - debug_with_peers!("updating beacon committee subscriptions {slot} {current_slot}"); + let fork_info = + ForkInfo::at_epoch(&chain_config, epoch, fork_info.genesis_validators_root); - if let Some(members) = own_beacon_committee_members - .get_or_init_at_slot(&beacon_state, dependent_root, slot) + if let Err(error) = own_beacon_committee_members + .init_at_slots(&beacon_nodes, fork_info, slots, &validator_indices) .await { - update_beacon_committee_subscriptions( - current_slot, - &members, - &subnet_service_tx, - ) - .await; + warn_with_peers!( + "failed to obtain attester duties for epoch {epoch}: {error:?}" + ); + continue; } } + + prefetch_ptc_duties( + &chain_config, + own_beacon_committee_members, + own_ptc_members, + &beacon_nodes, + current_epoch, + epoch, + &validator_indices, + ) + .await; + + prefetch_proposer_duties( + &chain_config, + own_beacon_committee_members, + own_proposer_duties, + &beacon_nodes, + epoch, + &validator_indices, + ) + .await; + } + + for slot in slots_to_compute_in_advance(current_slot) { + if let Some(members) = own_beacon_committee_members.get_at_slot::

(slot).await { + subscriptions.extend( + members + .iter() + .copied() + .map(BeaconCommitteeSubscription::from), + ); + } + } + + if !subscriptions.is_empty() + && let Err(error) = beacon_nodes + .subscribe_to_beacon_committees(current_slot, &subscriptions) + .await + { + warn_with_peers!("failed to update beacon committee subscriptions: {error:?}"); } drop(wait_group); } } -#[instrument(level = "debug", fields(slot = current_slot))] -pub async fn update_beacon_committee_subscriptions( - current_slot: Slot, - members: &[BeaconCommitteeMember], - subnet_service_tx: &UnboundedSender, +async fn prefetch_ptc_duties( + chain_config: &ChainConfig, + own_beacon_committee_members: &OwnBeaconCommitteeMembers, + own_ptc_members: &OwnPTCMembers, + beacon_nodes: &BeaconNodes, + current_epoch: Epoch, + epoch: Epoch, + validator_indices: &[ValidatorIndex], +) { + // No beacon node serves PTC duties from a pre-Gloas state, even for the fork epoch. + if chain_config.phase_at_epoch(current_epoch) < Phase::Gloas { + return; + } + + // PTC duties share the attester shuffling, so they are refetched with the dependent root. + let Some(dependent_root) = own_beacon_committee_members + .cached_dependent_root(epoch) + .await + else { + return; + }; + + if let Err(error) = own_ptc_members + .init_at_epoch( + chain_config, + beacon_nodes, + epoch, + dependent_root, + validator_indices, + ) + .await + { + warn_with_peers!("failed to obtain PTC duties for epoch {epoch}: {error:?}"); + } +} + +async fn prefetch_proposer_duties( + chain_config: &ChainConfig, + own_beacon_committee_members: &OwnBeaconCommitteeMembers, + own_proposer_duties: &OwnProposerDuties, + beacon_nodes: &BeaconNodes, + epoch: Epoch, + validator_indices: &[ValidatorIndex], ) { - if members.is_empty() { + let Some(dependent_root) = own_beacon_committee_members + .cached_dependent_root(proposer_dependent_epoch(chain_config, epoch)) + .await + else { return; + }; + + if let Err(error) = own_proposer_duties + .init_at_epoch(beacon_nodes, epoch, dependent_root, validator_indices) + .await + { + warn_with_peers!("failed to obtain proposer duties for epoch {epoch}: {error:?}"); } +} + +/// Keeps sync committee duties for the periods in flight cached ahead of the deadline. +pub struct PrefetchSyncCommitteeDutiesTask { + pub chain_config: Arc, + pub current_slot: Slot, + pub own_duties: Arc, + pub beacon_nodes: BeaconNodes, + pub wait_group: W, +} + +impl PrefetchSyncCommitteeDutiesTask { + #[instrument( + skip_all, + fields(slot = %self.current_slot), + name = "PrefetchSyncCommitteeDutiesTask::run", + )] + pub async fn run(self) { + let Self { + chain_config, + current_slot, + own_duties, + beacon_nodes, + wait_group, + } = self; + + let OwnDuties { + validator_indices: own_validator_indices, + sync_committee_members: own_sync_committee_members, + .. + } = own_duties.as_ref(); - let subscriptions = members.iter().copied().map(Into::into).collect(); + // Sync committees only exist from Altair on. + if chain_config.phase_at_slot::

(current_slot) < Phase::Altair { + drop(wait_group); + return; + } + + let current_epoch = misc::compute_epoch_at_slot::

(current_slot); + + let validator_indices = own_validator_indices.sorted(); + + if validator_indices.is_empty() { + drop(wait_group); + return; + } + + for period in OwnSyncCommitteeMembers::periods_to_prefetch::

(current_epoch) { + if let Err(error) = own_sync_committee_members + .init_at_period(&chain_config, &beacon_nodes, period, &validator_indices) + .await + { + warn_with_peers!( + "failed to obtain sync committee duties for period {period}: {error:?}", + ); + } + } - let (sender, receiver) = futures::channel::oneshot::channel(); + own_sync_committee_members.prune::

(current_epoch).await; - ToSubnetService::UpdateBeaconCommitteeSubscriptions(current_slot, subscriptions, sender) - .send(subnet_service_tx); + // With a local node the subscriptions are sent from the state-based path instead. + if !beacon_nodes.has_local_node() + && let Some(subscriptions) = own_sync_committee_members + .subscriptions_to_send::

(current_epoch) + .await + { + match beacon_nodes + .subscribe_to_sync_committees(current_epoch, &subscriptions) + .await + { + Ok(()) => own_sync_committee_members.mark_subscriptions_sent(current_epoch), + Err(error) => { + warn_with_peers!("failed to update sync committee subscriptions: {error:?}"); + } + } + } - if let Err(error) = receiver.await { - warn_with_peers!("failed to update beacon committee subscriptions: {error:?}"); + drop(wait_group); } } diff --git a/validator/src/validator.rs b/validator/src/validator.rs index 9d872b2f4..4e897b0d2 100644 --- a/validator/src/validator.rs +++ b/validator/src/validator.rs @@ -3,35 +3,33 @@ use core::time::Duration; use std::{ collections::{BTreeMap, HashMap, HashSet}, - path::Path, sync::Arc, time::SystemTime, }; -use anyhow::{Error as AnyhowError, Result}; -use block_producer::{BlockBuildOptions, BlockProducer, ValidatorBlindedBlock}; +use anyhow::{Error as AnyhowError, Result, anyhow, bail, ensure}; +use block_producer::{BlockBuildOptions, ValidatorBlindedBlock}; use bls::{PublicKeyBytes, Signature, SignatureBytes}; use builder_api::{ - BuilderApi, consts::{EPOCHS_PER_VALIDATOR_REGISTRATION_SUBMISSION, MaxBuilderEntries}, - gloas::containers::{BuilderEntry, BuilderRequestAuth, BuilderUrl, SignedBuilderRequestAuth}, + gloas::containers::{ + BuilderEntry, BuilderPreferencesEntry, BuilderRequestAuth, SignedBuilderRequestAuth, + }, unphased::containers::{SignedValidatorRegistrationV1, ValidatorRegistrationV1}, }; use clock::{Tick, TickKind}; use debug_info::HealthCheck; use dedicated_executor::DedicatedExecutor; -use doppelganger_protection::DoppelgangerProtection; +use doppelganger_protection::{ + DoppelgangerProtection, Error as DoppelgangerProtectionError, Snapshot as DoppelgangerSnapshot, +}; use eth1_api::ApiController; use eth2_libp2p::{GossipId, NetworkGlobals, SyncStatus}; use features::Feature; -use fork_choice_control::{ - BlockWithRoot, EventChannels, StoredBlock, Topic, ValidatorMessage, Wait, -}; -use fork_choice_store::{ - AttestationItem, AttestationOrigin, ChainLink, PayloadAttestationItem, - PayloadAttestationOrigin, StateCacheError, -}; +use fork_choice_control::{Topic, ValidatorMessage, Wait}; +use fork_choice_store::{ChainLink, StateCacheError}; use futures::{ + FutureExt as _, channel::{ mpsc::{UnboundedReceiver, UnboundedSender}, oneshot::Sender, @@ -47,21 +45,17 @@ use helper_functions::{ }; use itertools::Itertools as _; use keymanager::ProposerConfigs; -use liveness_tracker::ValidatorToLiveness; +use liveness_tracker::{ApiToLiveness, ValidatorToLiveness}; use logging::{debug_with_peers, error_with_peers, info_with_peers, warn_with_peers}; use once_cell::sync::OnceCell; -use operation_pools::{ - AttestationAggPool, AttestationKey, Origin, PayloadAttestationAggPool, PoolAdditionOutcome, - SyncCommitteeAggPool, -}; +use operation_pools::{AttestationKey, Origin, PoolAdditionOutcome, SyncCommitteeAggPool}; use p2p::{P2pToValidator, ToSubnetService, ValidatorToP2p}; use prometheus_metrics::Metrics; -use rayon::iter::{IntoParallelIterator as _, ParallelIterator as _}; use signer::{Signer, SigningMessage, SigningTriple, Snapshot}; use slasher::{SlasherToValidator, ValidatorToSlasher}; use slashing_protection::SlashingProtector; -use ssz::{BitList, ContiguousList, ReadError}; -use std_ext::ArcExt as _; +use ssz::{BitList, ContiguousList, ReadError, SszHash as _}; +use std_ext::ArcExt; use tap::{Conv as _, Pipe as _}; use thiserror::Error; use tokio::time::{interval, timeout}; @@ -74,10 +68,8 @@ use types::{ }, combined::{ AggregateAndProof, Attestation, AttesterSlashing, BeaconState, SignedAggregateAndProof, - SignedBeaconBlock, SignedBlindedBeaconBlock, }, - config::Config as ChainConfig, - deneb::primitives::Blob, + config::Config, electra::containers::{ AggregateAndProof as ElectraAggregateAndProof, SignedAggregateAndProof as ElectraSignedAggregateAndProof, SingleAttestation, @@ -91,32 +83,39 @@ use types::{ SignedExecutionPayloadEnvelope, SignedProposerPreferences, }, }, - nonstandard::{ - CustodyMode, KzgProofs, OwnAttestation, PayloadStatus, Phase, SyncCommitteeEpoch, - WithBlobsAndMev, WithStatus, - }, + nonstandard::{CustodyMode, OwnAttestation, PayloadStatus, Phase}, phase0::{ consts::GENESIS_SLOT, containers::{ AggregateAndProof as Phase0AggregateAndProof, Attestation as Phase0Attestation, - AttestationData, Checkpoint, ProposerSlashing, - SignedAggregateAndProof as Phase0SignedAggregateAndProof, SignedVoluntaryExit, + ProposerSlashing, SignedAggregateAndProof as Phase0SignedAggregateAndProof, + SignedVoluntaryExit, }, - primitives::{Epoch, ExecutionBlockHash, H256, Slot, ValidatorIndex}, + primitives::{Epoch, ExecutionBlockHash, H256, Slot, Uint256, ValidatorIndex}, }, preset::Preset, - traits::{BeaconState as _, PostAltairBeaconState, SignedBeaconBlock as _}, + traits::{BeaconBlock as _, BeaconState as _}, }; use validator_statistics::ValidatorStatistics; use crate::{ + beacon_node_api::{BeaconNodeApi, EnvelopeContents, ProducedBlock}, + beacon_nodes::{BeaconNodes, RemoteNodes}, + local_beacon_node::{ + BlindedBlockNotPublished, LocalBeaconNode, LocalContext, proposer_dependent_epoch, + }, messages::{ApiToValidator, InternalMessage}, - misc::{Aggregator, SignedBeaconBlockOrBlockRoot, SyncCommitteeMember}, + misc::{Aggregator, Chain, ChainSource, LocalChain, OwnDuties, SyncCommitteeMember}, own_beacon_committee_members::{BeaconCommitteeMember, OwnBeaconCommitteeMembers}, + own_proposer_duties::OwnProposerDuties, own_ptc_members::{OwnPTCMembers, PTCMember}, + own_sync_committee_members::OwnSyncCommitteeMembers, own_sync_committee_subscriptions::OwnSyncCommitteeSubscriptions, + own_validator_indices::OwnValidatorIndices, + remote_beacon_node::RemoteBeaconNode, + remote_beacon_nodes::RemoteBeaconNodes, slot_head::SlotHead, - tasks::{UpdateBeaconCommitteeSubscriptionsTask, update_beacon_committee_subscriptions}, + tasks::{PrefetchSyncCommitteeDutiesTask, UpdateBeaconCommitteeSubscriptionsTask}, validator_config::ValidatorConfig, }; @@ -130,6 +129,9 @@ const EPOCHS_TO_KEEP_REGISTERED_VALIDATORS: u64 = 2; // which happens to be the default timeout for validator registration requests in `mev-boost`. const MAX_VALIDATORS_PER_REGISTRATION: usize = 500; +/// The head at a slot and the state the built-in beacon node holds for it. +type LocalHead

= (SlotHead

, Arc>); + #[derive(Debug, Error)] enum HeadFarBehind { #[error( @@ -152,64 +154,82 @@ enum HeadFarBehind { }, } +/// The validator's channels: what drives it, and with the built-in beacon node the node's own. pub struct Channels { + /// Ticks from the clock, and the built-in node's fork choice messages when it runs. + pub validator_rx: UnboundedReceiver>, + pub local: Option>, +} + +/// The built-in beacon node's channels; a validator without the node has none. +pub struct LocalChannels { pub api_to_validator_rx: UnboundedReceiver>, - pub fork_choice_rx: UnboundedReceiver>, pub p2p_tx: UnboundedSender>, pub p2p_to_validator_rx: UnboundedReceiver>, pub slasher_to_validator_rx: Option>>, pub subnet_service_tx: UnboundedSender, + pub api_to_liveness_tx: Option>, pub validator_to_liveness_tx: Option>>, pub validator_to_slasher_tx: Option>, } +impl Channels { + fn p2p_tx(&self) -> Option<&UnboundedSender>> { + self.local.as_ref().map(|local| &local.p2p_tx) + } + + fn subnet_service_tx(&self) -> Option<&UnboundedSender> { + self.local.as_ref().map(|local| &local.subnet_service_tx) + } + + fn validator_to_liveness_tx(&self) -> Option<&UnboundedSender>> { + self.local + .as_ref() + .and_then(|local| local.validator_to_liveness_tx.as_ref()) + } + + fn validator_to_slasher_tx(&self) -> Option<&UnboundedSender> { + self.local + .as_ref() + .and_then(|local| local.validator_to_slasher_tx.as_ref()) + } +} + #[expect(clippy::struct_field_names)] pub struct Validator { - chain_config: Arc, validator_config: Arc, - block_producer: Arc>, - controller: ApiController, - api_to_validator_rx: UnboundedReceiver>, - fork_choice_rx: UnboundedReceiver>, - p2p_tx: UnboundedSender>, - p2p_to_validator_rx: UnboundedReceiver>, + chain_source: Arc>, + channels: Channels, + local_context: Option>>, last_tick: Option, next_graffiti_index: usize, - attestation_agg_pool: Arc>, - own_beacon_committee_members: Arc, + own_duties: Arc, own_singular_attestations: OnceCell>>, - own_sync_committee_members: OnceCell>, own_sync_committee_subscriptions: OwnSyncCommitteeSubscriptions

, - own_ptc_members: Arc, own_payload_attestations: OnceCell>, published_own_sync_committee_messages_for: Option>, + // Contributions must aggregate the root the messages voted for, which is the head of the + // beacon node the duty was performed against rather than the head this validator sees. + own_sync_committee_message_root: Option, own_aggregators: BTreeMap>, - builder_api: Option>, doppelganger_protection: Option>, - event_channels: Arc>, last_registration_epoch: Option, proposer_configs: Arc, signer: Arc, slashing_protector: Arc>, - slasher_to_validator_rx: Option>>, - subnet_service_tx: UnboundedSender, registered_validators: BTreeMap>, - payload_attestation_agg_pool: Arc>, - sync_committee_agg_pool: Arc>, metrics: Option>, validator_statistics: Option>, internal_tx: UnboundedSender, internal_rx: UnboundedReceiver, - validator_to_liveness_tx: Option>>, - validator_to_slasher_tx: Option>, last_cgc_update_epoch: Option, dedicated_executor_normal_priority: Arc, dedicated_executor_low_priority: Arc, - last_proposer_preferences_epoch: Option, - published_proposer_preferences: HashSet<(H256, Slot, ValidatorIndex)>, + /// Marked once a node accepts, so a failed publish is retried at the next check. + published_proposer_preferences: Arc>>, network_globals: Option>, - submitted_builder_preferences: HashSet<(Slot, ValidatorIndex)>, + submitted_builder_preferences: Arc>>, signed_builder_request_auths: Arc>, } @@ -221,124 +241,203 @@ impl Validator { #[must_use] pub fn new( validator_config: Arc, - block_producer: Arc>, - controller: ApiController, - attestation_agg_pool: Arc>, - builder_api: Option>, + chain_source: Arc>, doppelganger_protection: Option>, - event_channels: Arc>, proposer_configs: Arc, signer: Arc, slashing_protector: Arc>, - payload_attestation_agg_pool: Arc>, - sync_committee_agg_pool: Arc>, metrics: Option>, validator_statistics: Option>, channels: Channels, - _network_dir: Option<&Path>, dedicated_executor_normal_priority: Arc, dedicated_executor_low_priority: Arc, network_globals: Option>, ) -> Self { - let Channels { - api_to_validator_rx, - fork_choice_rx, - p2p_tx, - p2p_to_validator_rx, - slasher_to_validator_rx, - subnet_service_tx, - validator_to_liveness_tx, - validator_to_slasher_tx, - } = channels; - let (internal_tx, internal_rx) = futures::channel::mpsc::unbounded(); - let own_beacon_committee_members = Arc::new(OwnBeaconCommitteeMembers::new( - controller.chain_config().clone_arc(), - signer.clone_arc(), - )); + let own_duties = Arc::new(OwnDuties { + validator_indices: chain_source.own_validator_indices.clone_arc(), + beacon_committee_members: OwnBeaconCommitteeMembers::new( + chain_source.chain_config.clone_arc(), + signer.clone_arc(), + ), + proposer_duties: OwnProposerDuties::new(signer.clone_arc()), + ptc_members: OwnPTCMembers::new(signer.clone_arc()), + sync_committee_members: OwnSyncCommitteeMembers::new(), + }); - let own_ptc_members = Arc::new(OwnPTCMembers::new(signer.clone_arc())); + let local_context = + chain_source + .local() + .zip(channels.local.as_ref()) + .map(|(chain, local)| { + Arc::new(LocalContext { + chain: chain.clone_arc(), + validator_config: validator_config.clone_arc(), + signer: signer.clone_arc(), + p2p_tx: local.p2p_tx.clone(), + subnet_service_tx: local.subnet_service_tx.clone(), + liveness_tx: local.api_to_liveness_tx.clone(), + metrics: metrics.clone(), + dedicated_executor: dedicated_executor_normal_priority.clone_arc(), + }) + }); Self { - chain_config: controller.chain_config().clone_arc(), validator_config, - block_producer, - controller, - api_to_validator_rx, - fork_choice_rx, - p2p_tx, - p2p_to_validator_rx, + chain_source, + channels, + local_context, last_tick: None, next_graffiti_index: 0, - attestation_agg_pool, - own_beacon_committee_members, + own_duties, own_singular_attestations: OnceCell::new(), - own_sync_committee_members: OnceCell::new(), own_sync_committee_subscriptions: OwnSyncCommitteeSubscriptions::default(), - own_ptc_members, own_payload_attestations: OnceCell::new(), published_own_sync_committee_messages_for: None, + own_sync_committee_message_root: None, own_aggregators: BTreeMap::new(), - builder_api, doppelganger_protection, - event_channels, last_registration_epoch: None, proposer_configs, signer, slashing_protector, - payload_attestation_agg_pool, - sync_committee_agg_pool, - slasher_to_validator_rx, - subnet_service_tx, registered_validators: BTreeMap::new(), metrics, validator_statistics, internal_rx, internal_tx, - validator_to_liveness_tx, - validator_to_slasher_tx, last_cgc_update_epoch: None, dedicated_executor_normal_priority, dedicated_executor_low_priority, - last_proposer_preferences_epoch: None, - published_proposer_preferences: HashSet::new(), + published_proposer_preferences: Arc::default(), network_globals, - submitted_builder_preferences: HashSet::new(), + submitted_builder_preferences: Arc::default(), signed_builder_request_auths: Arc::default(), } } + fn local_beacon_node( + &self, + slot_head: SlotHead

, + beacon_state: Arc>, + wait_group: &W, + ) -> Result> { + let context = self + .local_context + .as_ref() + .map(ArcExt::clone_arc) + .ok_or_else(|| anyhow!("the built-in node's duties need its handles and channels"))?; + + Ok(BeaconNodes::Local(LocalBeaconNode::new( + context, + slot_head, + beacon_state, + wait_group.clone(), + ))) + } + + fn remote_nodes_at(&self, remote_beacon_nodes: &RemoteBeaconNodes, slot: Slot) -> RemoteNodes { + RemoteNodes::new( + remote_beacon_nodes, + slot, + self.validator_config.max_empty_slots, + ) + } + pub async fn run(self) -> Result<()> { - self.run_internal().await; + if let Some(remote_beacon_nodes) = self.chain_source.remote_beacon_nodes() { + remote_beacon_nodes + .check_on_startup(self.chain_source.slot()?) + .await?; - Ok(()) + remote_beacon_nodes.spawn_event_streams::

(self.internal_tx.clone()); + } + + // Awaited so that the first tick already has the indices of the keys loaded at startup. + self.own_duties + .validator_indices + .sync_keys(&self.chain_source) + .await; + + self.run_internal().await } - async fn run_internal(mut self) { + fn spawn_own_validator_index_resolution(&self, resolve: F) + where + F: FnOnce(Arc, Arc>) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + { + let indices = self.own_duties.validator_indices.clone_arc(); + let chain_source = self.chain_source.clone_arc(); + + tokio::spawn(resolve(indices, chain_source)); + } + + #[expect(clippy::too_many_lines)] + async fn run_internal(mut self) -> Result<()> { let mut health_check = HealthCheck::new("validator"); loop { - let mut slasher_to_validator_rx = self - .slasher_to_validator_rx - .as_mut() - .map(EitherFuture::Left) - .unwrap_or_else(|| EitherFuture::Right(futures::stream::pending())); + let fork_choice_rx = &mut self.channels.validator_rx; + + let (mut slasher_to_validator_rx, mut api_to_validator_rx, mut p2p_to_validator_rx) = + match &mut self.channels.local { + Some(local) => ( + local + .slasher_to_validator_rx + .as_mut() + .map(EitherFuture::Left) + .unwrap_or_else(|| EitherFuture::Right(futures::stream::pending())), + EitherFuture::Left(&mut local.api_to_validator_rx), + EitherFuture::Left(&mut local.p2p_to_validator_rx), + ), + None => ( + EitherFuture::Right(futures::stream::pending()), + EitherFuture::Right(futures::stream::pending()), + EitherFuture::Right(futures::stream::pending()), + ), + }; select! { _ = health_check.interval.select_next_some() => { health_check.check(); }, + () = self.signer.keys_changed().notified().fuse() => { + self.spawn_own_validator_index_resolution(|indices, chain_source| async move { + indices.sync_keys(&chain_source).await; + }); + } + message = self.internal_rx.select_next_some() => match message { + InternalMessage::Head { node, slot, block_root } => { + self.attest_remote_head(node, slot, block_root).await; + } + InternalMessage::FinalizedCheckpoint(finalized_epoch) => { + self.spawn_own_validator_index_resolution( + move |indices, chain_source| async move { + indices + .resolve_at_finalized_epoch(&chain_source, finalized_epoch) + .await; + }, + ); + } InternalMessage::DoppelgangerProtectionResult(result) => { if let Err(error) = result { - panic!("Doppelganger protection error: {error}"); + // The typed error must reach the application restart loop intact: + // it exits for good on `DoppelgangersDetected` instead of retrying. + if error.downcast_ref::().is_some() { + return Err(error); + } + + // A failed liveness query postpones activation instead. + warn_with_peers!("doppelganger liveness check failed: {error:?}"); } } }, - message = self.fork_choice_rx.select_next_some() => match message { + message = fork_choice_rx.select_next_some() => match message { ValidatorMessage::Tick(wait_group, tick) => { if let Err(error) = self.handle_tick(wait_group, tick).await { panic!("error while handling tick: {error:?}"); @@ -350,34 +449,56 @@ impl Validator { ValidatorMessage::PayloadStatusUpdated(wait_group, head) => { self.attest_payload_early(&wait_group, head).await } + ValidatorMessage::FinalizedCheckpoint(finalized_checkpoint) => { + InternalMessage::FinalizedCheckpoint(finalized_checkpoint.epoch) + .send(&self.internal_tx); + } ValidatorMessage::ValidAttestation(wait_group, attestation) => { - self.attestation_agg_pool - .insert_attestation(wait_group, attestation.clone_arc(), None); + if let Some(attestation_agg_pool) = self.chain_source.attestation_agg_pool() { + attestation_agg_pool.insert_attestation( + wait_group, + attestation.clone_arc(), + None, + ); + } - if let Some(validator_to_liveness_tx) = &self.validator_to_liveness_tx { + if let Some(validator_to_liveness_tx) = self.channels.validator_to_liveness_tx() { ValidatorToLiveness::ValidAttestation(attestation) .send(validator_to_liveness_tx); } - }, + } ValidatorMessage::ValidPayloadAttestation(wait_group, payload_attestation) => { - let span = tracing::debug_span!("ValidatorMessage::ValidPayloadAttestation", service = "validator"); + let span = tracing::debug_span!( + "ValidatorMessage::ValidPayloadAttestation", + service = "validator" + ); let _enter = span.enter(); - self.payload_attestation_agg_pool - .insert_payload_attestation(wait_group, payload_attestation); + if let Some(pool) = self.chain_source.payload_attestation_agg_pool() { + pool.insert_payload_attestation(wait_group, payload_attestation); + } // TODO(gloas): apply payload attestation to liveness_tracker // - // if let Some(validator_to_liveness_tx) = &self.validator_to_liveness_tx { + // if let Some(validator_to_liveness_tx) = self.channels.validator_to_liveness_tx() { // ValidatorToLiveness::ValidPayloadAttestation(payload_attestation) // .send(validator_to_liveness_tx); // } - }, - ValidatorMessage::PrepareExecutionPayload(slot, safe_execution_payload_hash, finalized_execution_payload_hash) => { - self.prepare_execution_payload(slot, safe_execution_payload_hash, finalized_execution_payload_hash).await + } + ValidatorMessage::PrepareExecutionPayload( + slot, + safe_execution_payload_hash, + finalized_execution_payload_hash, + ) => { + self.prepare_execution_payload( + slot, + safe_execution_payload_hash, + finalized_execution_payload_hash, + ) + .await } ValidatorMessage::Stop => { - if let Some(validator_to_liveness_tx) = &self.validator_to_liveness_tx { + if let Some(validator_to_liveness_tx) = self.channels.validator_to_liveness_tx() { ValidatorToLiveness::Stop.send(validator_to_liveness_tx); } @@ -387,26 +508,30 @@ impl Validator { slashing = slasher_to_validator_rx.select_next_some() => match slashing { SlasherToValidator::AttesterSlashing(attester_slashing) => { - self.block_producer.add_new_attester_slashing(AttesterSlashing::Phase0(attester_slashing)).await; + if let Some(block_producer) = self.chain_source.block_producer() { + block_producer.add_new_attester_slashing(AttesterSlashing::Phase0(attester_slashing)).await; + } } SlasherToValidator::ProposerSlashing(proposer_slashing) => { - self.block_producer.add_new_proposer_slashing(proposer_slashing).await; + if let Some(block_producer) = self.chain_source.block_producer() { + block_producer.add_new_proposer_slashing(proposer_slashing).await; + } } }, - gossip_message = self.p2p_to_validator_rx.select_next_some() => match gossip_message { + gossip_message = p2p_to_validator_rx.select_next_some() => match gossip_message { P2pToValidator::AttesterSlashing(slashing, gossip_id, wait_group) => { self.handle_attester_slashing(slashing, gossip_id, wait_group).await } P2pToValidator::ProposerSlashing(slashing, gossip_id, wait_group) => { - self.handle_propser_slashing(*slashing, gossip_id, wait_group).await + self.handle_proposer_slashing(*slashing, gossip_id, wait_group).await } P2pToValidator::VoluntaryExit(voluntary_exit, gossip_id, wait_group) => { self.handle_voluntary_exit(*voluntary_exit, gossip_id, wait_group).await } }, - api_message = self.api_to_validator_rx.select_next_some() => { + api_message = api_to_validator_rx.select_next_some() => { let success = match api_message { ApiToValidator::RegisteredValidators(sender) => { self.handle_registered_validators(sender) @@ -427,6 +552,8 @@ impl Validator { complete => break, } } + + Ok(()) } #[instrument(parent = None, level = "debug", fields(service = "validator"), skip_all)] @@ -446,7 +573,14 @@ impl Validator { &mut self, validator_registrations: Vec<(ValidatorRegistrationV1, Signature)>, ) -> bool { - let current_slot = self.controller.slot(); + let current_slot = match self.chain_source.slot() { + Ok(current_slot) => current_slot, + Err(error) => { + warn_with_peers!("failed to handle validator registrations: {error}"); + return true; + } + }; + let current_epoch = misc::compute_epoch_at_slot::

(current_slot); let registrations = validator_registrations @@ -468,12 +602,21 @@ impl Validator { sender: Sender>>, contributions_and_proofs: Vec>, ) -> bool { - let current_slot = self.controller.slot(); - let slot_head = self.safe_slot_head(current_slot).await; + // Contributions are validated against the built-in node's state, which a validator + // performing duties elsewhere has no use for. + let Some(local) = self.chain_source.local() else { + return sender.send(None).is_ok(); + }; - let failures = slot_head - .map(|slot_head| { - self.handle_external_contributions_and_proofs(slot_head, contributions_and_proofs) + let failures = self + .safe_slot_head(local.controller.slot()) + .await + .map(|(slot_head, _)| { + self.handle_external_contributions_and_proofs( + &local.sync_committee_agg_pool, + slot_head, + contributions_and_proofs, + ) }) .conv::>() .await; @@ -482,14 +625,17 @@ impl Validator { } #[instrument(parent = None, level = "debug", fields(service = "validator"), skip_all)] - async fn handle_propser_slashing( + async fn handle_proposer_slashing( &self, slashing: ProposerSlashing, gossip_id: GossipId, _wait_group: W, ) { - let outcome = match self - .block_producer + let Some(block_producer) = self.chain_source.block_producer() else { + return; + }; + + let outcome = match block_producer .handle_external_proposer_slashing(slashing) .await { @@ -500,8 +646,10 @@ impl Validator { } }; - if matches!(outcome, PoolAdditionOutcome::Accept) { - self.event_channels.send_proposer_slashing_event(slashing); + if matches!(outcome, PoolAdditionOutcome::Accept) + && let Some(event_channels) = self.chain_source.event_channels() + { + event_channels.send_proposer_slashing_event(slashing); } self.handle_pool_addition_outcome_for_p2p(outcome, gossip_id); @@ -514,8 +662,11 @@ impl Validator { gossip_id: GossipId, _wait_group: W, ) { - let outcome = match self - .block_producer + let Some(block_producer) = self.chain_source.block_producer() else { + return; + }; + + let outcome = match block_producer .handle_external_attester_slashing(*slashing.clone()) .await { @@ -526,8 +677,10 @@ impl Validator { } }; - if matches!(outcome, PoolAdditionOutcome::Accept) { - self.event_channels.send_attester_slashing_event(slashing); + if matches!(outcome, PoolAdditionOutcome::Accept) + && let Some(event_channels) = self.chain_source.event_channels() + { + event_channels.send_attester_slashing_event(slashing); } self.handle_pool_addition_outcome_for_p2p(outcome, gossip_id); @@ -540,8 +693,11 @@ impl Validator { gossip_id: GossipId, _wait_group: W, ) { - let outcome = match self - .block_producer + let Some(block_producer) = self.chain_source.block_producer() else { + return; + }; + + let outcome = match block_producer .handle_external_voluntary_exit(voluntary_exit) .await { @@ -552,9 +708,10 @@ impl Validator { } }; - if matches!(outcome, PoolAdditionOutcome::Accept) { - self.event_channels - .send_voluntary_exit_event(voluntary_exit); + if matches!(outcome, PoolAdditionOutcome::Accept) + && let Some(event_channels) = self.chain_source.event_channels() + { + event_channels.send_voluntary_exit_event(voluntary_exit); } self.handle_pool_addition_outcome_for_p2p(outcome, gossip_id); @@ -562,8 +719,13 @@ impl Validator { #[instrument(parent = None, level = "debug", fields(service = "validator"), skip_all)] async fn handle_head_message(&mut self, wait_group: W, head: ChainLink

) { - if let Some(validator_to_liveness_tx) = &self.validator_to_liveness_tx { - let state = self.controller.state_by_chain_link(&head); + // Only the built-in node's fork choice sends heads. + let Some(controller) = self.chain_source.controller() else { + return; + }; + + if let Some(validator_to_liveness_tx) = self.channels.validator_to_liveness_tx() { + let state = controller.state_by_chain_link(&head); ValidatorToLiveness::Head(head.block.clone_arc(), state).send(validator_to_liveness_tx); } @@ -585,10 +747,21 @@ impl Validator { safe_execution_payload_hash: ExecutionBlockHash, finalized_execution_payload_hash: ExecutionBlockHash, ) { + let Some(local) = self.chain_source.local() else { + return; + }; + + let LocalChain { + block_producer, + attestation_agg_pool, + event_channels, + .. + } = local.as_ref(); + let slot_head = self.safe_slot_head(slot).await; - if let Some(slot_head) = slot_head { - let proposer_index = match slot_head.proposer_index() { + if let Some((slot_head, beacon_state)) = slot_head { + let proposer_index = match slot_head.proposer_index(&beacon_state) { Ok(proposer_index) => proposer_index, Err(error) => { error_with_peers!( @@ -600,8 +773,7 @@ impl Validator { let should_prepare_execution_payload = Feature::AlwaysPrepareExecutionPayload .is_enabled() - || self - .attestation_agg_pool + || attestation_agg_pool .is_registered_validator(proposer_index) .await; @@ -609,8 +781,8 @@ impl Validator { return; } - let block_build_context = self.block_producer.new_build_context( - slot_head.beacon_state.clone_arc(), + let block_build_context = block_producer.new_build_context( + beacon_state.clone_arc(), slot_head.beacon_block_root, proposer_index, BlockBuildOptions::default(), @@ -634,8 +806,8 @@ impl Validator { if let Some((parent_block_number, parent_block_hash)) = block_build_context.execution_payload_parent() { - self.event_channels.send_payload_attributes_event( - slot_head.beacon_state.phase(), + event_channels.send_payload_attributes_event( + slot_head.phase(), slot, proposer_index, slot_head.beacon_block_root, @@ -671,7 +843,13 @@ impl Validator { } }; - message.send(&self.p2p_tx); + self.publish(message); + } + + fn publish(&self, message: ValidatorToP2p

) { + if let Some(p2p_tx) = self.channels.p2p_tx() { + message.send(p2p_tx); + } } #[expect(clippy::too_many_lines)] @@ -688,48 +866,71 @@ impl Validator { if let Some(metrics) = self.metrics.as_ref() && tick.is_start_of_interval() { - let tick_delay = tick.delay::

(&self.chain_config, self.controller.genesis_time())?; + let tick_delay = tick.delay::

( + &self.chain_source.chain_config, + self.chain_source.genesis_time, + )?; debug_with_peers!("tick_delay: {tick_delay:?} for {tick:?}"); metrics.set_tick_delay(tick.kind.as_ref(), tick_delay); } + if let Some(metrics) = self.metrics.as_ref() + && tick.is_start_of_slot() + { + metrics.set_beacon_clock_slot(tick.slot); + } + let Tick { slot, kind } = tick; let current_epoch = misc::compute_epoch_at_slot::

(slot); if self.last_registration_epoch != Some(current_epoch) { - self.register_validators(current_epoch).await; + self.register_validators(slot, current_epoch).await; } let no_validators = self.signer.load().no_keys() && self.registered_validators.is_empty() - && self.block_producer.no_prepared_proposers().await; + && match self.chain_source.block_producer() { + Some(block_producer) => block_producer.no_prepared_proposers().await, + None => true, + }; debug_with_peers!("{kind:?} tick in slot {slot}"); + if kind == TickKind::ProposeFourth + && let Some(remote_beacon_nodes) = self.chain_source.remote_beacon_nodes() + { + let remote_beacon_nodes = remote_beacon_nodes.clone_arc(); + tokio::spawn(async move { remote_beacon_nodes.refresh(slot).await }); + } + if tick.is_start_of_epoch::

() { let _timer = self .metrics .as_ref() .map(|metrics| metrics.validator_epoch_processing_times.start_timer()); - if let Some(validator_to_slasher_tx) = &self.validator_to_slasher_tx { + if let Some(validator_to_slasher_tx) = self.channels.validator_to_slasher_tx() { ValidatorToSlasher::Epoch(current_epoch).send(validator_to_slasher_tx); } - if let Some(validator_to_liveness_tx) = &self.validator_to_liveness_tx { + if let Some(validator_to_liveness_tx) = self.channels.validator_to_liveness_tx() { ValidatorToLiveness::Epoch(current_epoch).send(validator_to_liveness_tx); } - self.discard_old_proposer_preferences(current_epoch); + self.discard_old_proposer_preferences(current_epoch).await; self.discard_old_registered_validators(current_epoch); - self.block_producer.discard_old_data(current_epoch).await; + if let Some(block_producer) = self.chain_source.block_producer() { + block_producer.discard_old_data(current_epoch).await; + } self.own_sync_committee_subscriptions .discard_old_subscriptions(current_epoch); - if let Some(validator_statistics) = self.validator_statistics.as_ref() { + if let Some(validator_statistics) = self.validator_statistics.as_ref() + && let Some(controller) = self.chain_source.controller() + { let validator_statistics = validator_statistics.clone_arc(); - let controller = self.controller.clone_arc(); + let controller = controller.clone_arc(); tokio::spawn(async move { if controller.is_forward_synced() { @@ -743,84 +944,142 @@ impl Validator { } } - let own_validator_indices = self - .attestation_agg_pool - .registered_validator_indices() - .await; - - if self.last_cgc_update_epoch != Some(current_epoch) + // Custody is the built-in node's concern, and only while it serves duties. + if let Some(attestation_agg_pool) = self.chain_source.attestation_agg_pool() + && self.last_cgc_update_epoch != Some(current_epoch) && self.validator_config.custody_mode != CustodyMode::Super - && self.chain_config.is_peerdas_scheduled() - && !own_validator_indices.is_empty() + && self.chain_source.chain_config.is_peerdas_scheduled() { - self.handle_custody_requirements_update(slot, &own_validator_indices); + let own_validator_indices = attestation_agg_pool.registered_validator_indices().await; + + if !own_validator_indices.is_empty() { + self.handle_custody_requirements_update(slot, &own_validator_indices); + } } self.track_collection_metrics().await; - let slot_head = if no_validators { + let beacon_nodes = if no_validators { None } else { - self.slot_head(slot) - .await? - .map_err(|head_far_behind| warn_with_peers!("{head_far_behind}")) - .ok() + match &self.chain_source.chain { + Chain::Local(_) => match self.slot_head(slot).await? { + Ok((slot_head, beacon_state)) => { + Some(self.local_beacon_node(slot_head, beacon_state, &wait_group)?) + } + Err(head_far_behind) => { + warn_with_peers!("{head_far_behind}"); + None + } + }, + Chain::Remote(remote_beacon_nodes) => { + let nodes = self.remote_nodes_at(remote_beacon_nodes, slot); + let slot_head = nodes.slot_head::

(slot); + + if slot_head.is_none() { + warn_with_peers!( + "no beacon node given with --beacon-node-urls reported a head recent \ + enough to perform duties for slot {slot}", + ); + } + + slot_head.map(|slot_head| BeaconNodes::Remote { nodes, slot_head }) + } + } }; if let Err(error) = self - .update_subnet_subscriptions(&wait_group, slot_head.as_ref()) + .update_subnet_subscriptions(&wait_group, tick, beacon_nodes.as_ref()) .await { warn_with_peers!("failed to update subnet subscriptions: {error:?}"); } if misc::is_epoch_start::

(slot) && kind == TickKind::AggregateFourth { - self.refresh_signer_keys(); + self.refresh_signer_keys(slot); + } + + // The fallback for a stream that is not delivering finalizations; it costs nothing while + // every key is known. + if tick.is_start_of_slot() + && (misc::is_epoch_start::

(slot) + || self.own_duties.validator_indices.last_resolve_failed()) + { + self.spawn_own_validator_index_resolution(|indices, chain_source| async move { + indices.resolve(&chain_source).await; + }); + } + + if tick.is_start_of_slot() { + self.discard_previous_slot_attestations(); + self.discard_previous_slot_payload_attestations(); } - let Some(slot_head) = slot_head else { + let Some(beacon_nodes) = beacon_nodes else { return Ok(()); }; - if tick.is_start_of_slot() + // Late in the slot every node has ticked into the epoch and seen the slot's attestations. + if tick.kind == TickKind::Aggregate && let Some(doppelganger_protection) = &self.doppelganger_protection { let doppelganger_protection = doppelganger_protection.clone_arc(); let internal_tx = self.internal_tx.clone(); + let beacon_nodes = beacon_nodes.clone(); + let own_validator_indices = self.own_duties.validator_indices.clone_arc(); + let signer = self.signer.clone_arc(); tokio::spawn(async move { + let own_validator_indices = own_validator_indices.load().clone_arc(); + + // The index map follows the signer asynchronously, so a key missing from it may + // still be held and must not be dropped from tracking. + let indices = signer + .load() + .keys() + .map(|public_key| { + ( + *public_key, + own_validator_indices.get(public_key).copied().flatten(), + ) + }) + .collect::>(); + let result = doppelganger_protection - .detect_doppelgangers::

(slot) + .detect_doppelgangers::(slot, &indices, |epoch, validator_indices| { + let beacon_nodes = &beacon_nodes; + + async move { + let liveness = beacon_nodes.liveness(epoch, &validator_indices).await?; + + Ok(liveness + .into_iter() + .map(|response| (response.index, response.is_live)) + .collect()) + } + }) .await; InternalMessage::DoppelgangerProtectionResult(result).send(&internal_tx); }); } - self.attestation_agg_pool - .compute_proposer_indices(slot_head.beacon_state.clone_arc()); - - if let Some(state) = slot_head.beacon_state.post_altair() { - if misc::is_epoch_start::

(state.slot().saturating_add(1)) { - self.own_sync_committee_members.take(); - - self.own_sync_committee_members.get_or_try_init(|| { - self.own_sync_committee_members_for_epoch(SyncCommitteeEpoch::Next, state) - })?; - } - - self.own_sync_committee_members.get_or_try_init(|| { - self.own_sync_committee_members_for_epoch(SyncCommitteeEpoch::Current, state) - })?; + // Only the built-in beacon node has a state to compute these from. + if let Some(beacon_state) = beacon_nodes.state() + && let Some(attestation_agg_pool) = self.chain_source.attestation_agg_pool() + { + attestation_agg_pool.compute_proposer_indices(beacon_state.clone_arc()); } - if self.last_proposer_preferences_epoch != Some(current_epoch) - && (slot_head.beacon_state.post_gloas().is_some() - || self.chain_config.gloas_fork_epoch == current_epoch.saturating_add(1)) + // The dependent root can change with a reorg and remote duties land in the cache + // asynchronously, so the check runs every slot. + if tick.is_start_of_slot() + && (self.chain_source.chain_config.phase_at_epoch(current_epoch) >= Phase::Gloas + || self.chain_source.chain_config.gloas_fork_epoch + == current_epoch.saturating_add(1)) { - self.publish_proposer_preferences(&slot_head); - self.submit_builder_preferences(&slot_head); - self.last_proposer_preferences_epoch = Some(current_epoch); + self.publish_proposer_preferences(&beacon_nodes).await; + self.submit_builder_preferences(&beacon_nodes).await; } match kind { @@ -830,15 +1089,21 @@ impl Validator { .as_ref() .map(|metrics| metrics.validator_propose_tick_times.start_timer()); - let slot_head = self - .wait_for_fully_validated_head(slot_head, HeadDuty::Propose) + if let Some(beacon_nodes) = self + .wait_for_fully_validated_head( + beacon_nodes, + &wait_group, + tick, + HeadDuty::Propose, + ) .await - .map(|(slot_head, _)| slot_head); + && let Err(error) = self.propose(&beacon_nodes).await + { + error_with_peers!("failed to produce and publish own block: {error:?}"); + } - self.discard_previous_slot_attestations(); - self.discard_previous_slot_payload_attestations(); - self.propose(wait_group, slot_head.as_ref()).await?; self.published_own_sync_committee_messages_for = None; + self.own_sync_committee_message_root = None; } TickKind::Attest => { let _timer = self @@ -846,27 +1111,26 @@ impl Validator { .as_ref() .map(|metrics| metrics.validator_attest_tick_times.start_timer()); - let settled_head = self - .wait_for_fully_validated_head(slot_head, HeadDuty::Attest) - .await; - - let full = settled_head.as_ref().is_some_and(|(_, full)| *full); - let slot_head = settled_head.map(|(slot_head, _)| slot_head); - - if let Err(error) = self - .attest_and_start_aggregating(&wait_group, slot_head.as_ref(), full) + if let Some(beacon_nodes) = self + .wait_for_fully_validated_head( + beacon_nodes, + &wait_group, + tick, + HeadDuty::Attest, + ) .await { - error_with_peers!("failed to produce and publish own attestations: {error:?}"); - } + if let Err(error) = self.attest_and_start_aggregating(&beacon_nodes).await { + error_with_peers!( + "failed to produce and publish own attestations: {error:?}" + ); + } - if let Err(error) = self - .publish_sync_committee_messages(&wait_group, slot_head) - .await - { - error_with_peers!( - "failed to produce and publish own sync_committee messages: {error:?}" - ); + if let Err(error) = self.publish_sync_committee_messages(&beacon_nodes).await { + error_with_peers!( + "failed to produce and publish own sync_committee messages: {error:?}" + ); + } } } TickKind::Aggregate => { @@ -875,25 +1139,31 @@ impl Validator { .as_ref() .map(|metrics| metrics.validator_aggregate_tick_times.start_timer()); - let slot_head = self - .wait_for_fully_validated_head(slot_head, HeadDuty::Attest) + match self + .wait_for_fully_validated_head( + beacon_nodes.clone(), + &wait_group, + tick, + HeadDuty::Attest, + ) .await - .map(|(slot_head, _)| slot_head); - - self.publish_aggregates_and_proofs(&wait_group, slot_head.as_ref()) - .await; - - self.publish_contributions_and_proofs( - self.published_own_sync_committee_messages_for - .as_ref() - .or(slot_head.as_ref()), - ) - .await; + { + Some(beacon_nodes) => { + self.publish_aggregates_and_proofs(&beacon_nodes).await; + self.publish_contributions_and_proofs(&beacon_nodes).await; + } + // The messages were signed for a head that was validated, and contributions + // aggregate those rather than vote for the head of this tick. + None if self.published_own_sync_committee_messages_for.is_some() => { + self.publish_contributions_and_proofs(&beacon_nodes).await; + } + None => {} + } } TickKind::PayloadAttest => { // Payload attestations report timeliness, not validity. // An optimistic head does not stop them. - if let Err(error) = self.attest_payload(&wait_group, &slot_head, false).await { + if let Err(error) = self.attest_payload(&beacon_nodes).await { error_with_peers!( "failed to produce and publish own payload attestations: {error:?}" ); @@ -902,7 +1172,9 @@ impl Validator { _ => {} } - if misc::is_epoch_start::

(slot) && tick.is_end_of_slot::

(&self.chain_config) { + if misc::is_epoch_start::

(slot) + && tick.is_end_of_slot::

(&self.chain_source.chain_config) + { self.spawn_pruning(slot); } @@ -912,7 +1184,7 @@ impl Validator { } #[instrument(level = "debug", skip_all)] - async fn safe_slot_head(&self, slot: Slot) -> Option> { + async fn safe_slot_head(&self, slot: Slot) -> Option> { self.slot_head(slot) .await .map(Result::ok) @@ -923,8 +1195,12 @@ impl Validator { // The nested `Result` is inspired by `sled`: // #[instrument(level = "debug", skip_all, fields(slot = slot))] - async fn slot_head(&self, slot: Slot) -> Result, HeadFarBehind>> { - let head = self.controller.head().value; + async fn slot_head(&self, slot: Slot) -> Result, HeadFarBehind>> { + let Some(controller) = self.chain_source.controller() else { + bail!("the built-in beacon node does not serve duties"); + }; + + let head = controller.head().value; let head_slot = head.slot(); let head_epoch = misc::compute_epoch_at_slot::

(head_slot); @@ -933,7 +1209,7 @@ impl Validator { if head_epoch.saturating_add(1) < current_epoch && self.any_peer_ahead_of(head_slot).unwrap_or(true) { - let error = if self.controller.finished_initial_forward_sync() { + let error = if controller.finished_initial_forward_sync() { HeadFarBehind::OutOfSync { head_slot, head_epoch, @@ -952,7 +1228,37 @@ impl Validator { return Ok(Err(error)); } - self.slot_head_at(&head, slot).await.map(Ok) + self.slot_head_at(controller, &head, slot).await.map(Ok) + } + + // `LocalHead` for `head` with its state advanced to `slot`. + // The built-in beacon node computes duties from the state, so an older head must be advanced. + async fn slot_head_at( + &self, + controller: &ApiController, + head: &ChainLink

, + slot: Slot, + ) -> Result> { + let beacon_state = if head.slot() < slot { + let controller = controller.clone_arc(); + let block_root = head.block_root; + + tokio::task::spawn_blocking(move || { + controller.preprocessed_state_post_block_blocking(block_root, slot) + }) + .await?? + } else { + controller.state_by_chain_link(head) + }; + + let slot_head = SlotHead::from_state( + self.chain_source.chain_config.clone_arc(), + slot, + head.block_root, + beacon_state.as_ref(), + ); + + Ok((slot_head, beacon_state)) } fn any_peer_ahead_of(&self, head_slot: Slot) -> Option { @@ -985,14 +1291,8 @@ impl Validator { /// #[expect(clippy::too_many_lines)] #[instrument(level = "debug", skip_all)] - async fn propose(&mut self, wait_group: W, slot_head: Option<&SlotHead

>) -> Result<()> { - let Some(slot_head) = slot_head else { - warn_with_peers!( - "validator cannot produce a block because \ - chain head has not been fully verified by an execution engine", - ); - return Ok(()); - }; + async fn propose(&mut self, beacon_nodes: &BeaconNodes) -> Result<()> { + let slot_head = beacon_nodes.head(); if slot_head.slot() == GENESIS_SLOT { // All peers should already have the genesis block. @@ -1000,31 +1300,44 @@ impl Validator { return Ok(()); } - let proposer_index = tokio::task::block_in_place(|| slot_head.proposer_index())?; - let public_key = slot_head.public_key(proposer_index); + let (proposer_index, public_key) = match beacon_nodes.state() { + Some(beacon_state) => { + let proposer_index = + tokio::task::block_in_place(|| slot_head.proposer_index(beacon_state))?; + + let public_key = accessors::public_key(beacon_state.as_ref(), proposer_index)?; + + (proposer_index, *public_key) + } + None => { + let Some(proposer) = self + .own_proposer_from_duties(beacon_nodes, slot_head) + .await? + else { + return Ok(()); + }; + + proposer + } + }; + let signer_snapshot = self.signer.load(); - if !signer_snapshot.has_key(*public_key) { + if !signer_snapshot.has_key(public_key) { return Ok(()); } - let doppelganger_protection = self + if let Some(doppelganger_protection) = self .doppelganger_protection .as_deref() - .map(DoppelgangerProtection::load); - - if let Some(doppelganger_protection) = &doppelganger_protection - && !doppelganger_protection.is_validator_active(*public_key) - { - info_with_peers!( - "Validator {public_key:?} skipping proposer duty in slot {} \ - since not enough time has passed to ensure there are \ - no doppelganger validators participating on network. \ - Validator will start performing duties on slot {}.", + .map(DoppelgangerProtection::load) + && !doppelganger_protection_permits::

( + &doppelganger_protection, + "proposer", slot_head.slot(), - doppelganger_protection.tracking_end_slot::

(*public_key), - ); - + public_key, + ) + { return Ok(()); } @@ -1040,58 +1353,35 @@ impl Validator { let graffiti = self .proposer_configs - .graffiti_bytes(*public_key)? + .graffiti_bytes(public_key)? .or_else(|| self.next_graffiti()); - let block_build_context = self.block_producer.new_build_context( - slot_head.beacon_state.clone_arc(), - slot_head.beacon_block_root, - proposer_index, - BlockBuildOptions { - graffiti, - disable_blockprint_graffiti: self.validator_config.disable_blockprint_graffiti, - builder_boost_factor: self.proposer_configs.builder_boost_factor(*public_key), - min_bid: self.proposer_configs.builder_min_bid(*public_key), - ..BlockBuildOptions::default() - }, - ); - - let execution_payload_header_handle = - block_build_context.get_execution_payload_header(*public_key); - - let local_execution_payload_handle = block_build_context.get_local_execution_payload(); - - let builder_api_bids_handle = if slot_head.phase() >= Phase::Gloas { - let builders = self - .builder_entries(&signer_snapshot, *public_key, slot_head.slot()) - .await; + let Some(randao_reveal) = self + .sign_randao_reveal(&signer_snapshot, slot_head, public_key) + .await + else { + return Ok(()); + }; - block_build_context.get_builder_api_bids(builders) + // Before Gloas a bid needs a registration, which is only sent with a builder, so only + // local payloads are wanted from a remote node without one. + let builder_boost_factor = if slot_head.phase() >= Phase::Gloas + || beacon_nodes.has_local_node() + || self.chain_source.registers_with_builder() + { + self.proposer_configs.builder_boost_factor(public_key) } else { - None + Uint256::ZERO }; - let epoch = slot_head.current_epoch(); - - let result = signer_snapshot - .sign_without_slashing_protection( - SigningMessage::RandaoReveal { epoch }, - RandaoEpoch::from(epoch).signing_root(&self.chain_config, &slot_head.beacon_state), - Some(slot_head.beacon_state.as_ref().into()), - *public_key, - ) - .await; - - let randao_reveal = match result { - Ok(signature) => signature.into(), - Err(error) => { - warn_with_peers!( - "failed to sign RANDAO reveal (epoch: {epoch}, public_key: {public_key}): \ - {error:?}", - ); + let min_bid = self.proposer_configs.builder_min_bid(public_key); - return Ok(()); - } + // Bids are requested from the configured builders, under request auths signed here. + let builders = if slot_head.phase() >= Phase::Gloas { + self.builder_entries(&signer_snapshot, public_key, slot_head.slot()) + .await + } else { + ContiguousList::default() }; info_with_peers!( @@ -1099,78 +1389,64 @@ impl Validator { slot_head.slot() ); - let beacon_block_option = match block_build_context - .build_blinded_beacon_block( + let ProducedBlock { + block: validator_blinded_block, + kzg_proofs: block_proofs, + blobs: block_blobs, + envelope_contents, + builder_url, + } = match beacon_nodes + .produce_block( + slot_head.slot(), randao_reveal, - execution_payload_header_handle, - local_execution_payload_handle, - builder_api_bids_handle, + graffiti, + builder_boost_factor, + min_bid, + &builders, ) .await { - Ok(block_opt) => block_opt, + Ok(produced) => produced, Err(error) => { - warn_with_peers!("failed to produce beacon block: {error}"); - return Ok(()); + warn_with_peers!( + "validator {proposer_index} skipping beacon block proposal in slot {}: {error}", + slot_head.slot(), + ); + return Ok(()); } }; - let Some(( - WithBlobsAndMev { - value: validator_blinded_block, - proofs: mut block_proofs, - blobs: mut block_blobs, - .. - }, - _block_rewards, - builder_url, - )) = beacon_block_option - else { - warn_with_peers!( - "validator {} skipping beacon block proposal in slot {}", - proposer_index, - slot_head.slot(), - ); - return Ok(()); - }; - info_with_peers!( "block producer finished block production task at slot {}", slot_head.slot() ); - // The wait checked the payload the proposal was going to extend. - // Production picks the parent's node again, so it can extend a different payload. - // Its envelope may have arrived in between, or the PTC vote may have flipped the node. - // Never sign a block that extends a payload the execution engine has not validated. - let extended_payload_status = match &validator_blinded_block { + let (block_slot, block_proposer_index) = match &validator_blinded_block { ValidatorBlindedBlock::BlindedBeaconBlock(block) => { - self.controller.wait_for_valid_extended_payload(block).await - } - ValidatorBlindedBlock::BeaconBlock(block) => { - self.controller.wait_for_valid_extended_payload(block).await + (block.slot(), block.proposer_index()) } + ValidatorBlindedBlock::BeaconBlock(block) => (block.slot(), block.proposer_index()), }; - if !extended_payload_status.is_some_and(PayloadStatus::is_valid) { + // The head may have changed since the duty was looked up. + if (block_slot, block_proposer_index) != (slot_head.slot(), proposer_index) { warn_with_peers!( - "validator {proposer_index} skipping beacon block proposal in slot {} \ - because the block extends a payload that is not fully validated \ - (status: {extended_payload_status:?})", + "beacon node produced a block for slot {block_slot} and proposer \ + {block_proposer_index} where slot {} and proposer {proposer_index} were expected", slot_head.slot(), ); return Ok(()); } - let beacon_block_or_root = match validator_blinded_block { + let published = match validator_blinded_block { ValidatorBlindedBlock::BlindedBeaconBlock(blinded_block) => { let Some(signature) = slot_head .sign_beacon_block( &self.signer, &blinded_block, (&blinded_block).into(), - *public_key, + public_key, self.slashing_protector.clone_arc(), ) .await @@ -1178,129 +1454,102 @@ impl Validator { return Ok(()); }; - let signed_blinded_block = blinded_block.with_signature(signature); + info_with_peers!( + "validator {proposer_index} proposing beacon block with root {:?} \ + in slot {} using the builder API", + blinded_block.hash_tree_root(), + slot_head.slot(), + ); - match self - .post_blinded_block( - slot_head, - signed_blinded_block, - &mut block_blobs, - &mut block_proofs, - ) - .await? - { - Some(signed_block) => signed_block, - None => return Ok(()), - } + beacon_nodes + .publish_blinded_block(&blinded_block.with_signature(signature)) + .await } ValidatorBlindedBlock::BeaconBlock(block) => { - match slot_head + let Some(signature) = slot_head .sign_beacon_block( &self.signer, &block, (&block).into(), - *public_key, + public_key, self.slashing_protector.clone_arc(), ) .await - { - Some(signature) => SignedBeaconBlockOrBlockRoot::Block(Box::new( - block.with_signature(signature), - )), - None => return Ok(()), - } - } - }; + else { + return Ok(()); + }; - match beacon_block_or_root { - SignedBeaconBlockOrBlockRoot::Root(block_root) => info_with_peers!( - "validator {} proposing beacon block with root {:?} in slot {} using builder", - proposer_index, - block_root, - slot_head.slot(), - ), - SignedBeaconBlockOrBlockRoot::Block(beacon_block) => { - let beacon_block_root = beacon_block.message().hash_tree_root(); - let parent_beacon_block_root = beacon_block.message().parent_root(); + let block_root = block.hash_tree_root(); - if let Some(payload_bid) = beacon_block.payload_bid() + if let Some(payload_bid) = block.payload_bid() && payload_bid.builder_index != BUILDER_INDEX_SELF_BUILD { info_with_peers!( - "validator {} proposing beacon block with root {:?} in slot {} using builder {}", - proposer_index, - beacon_block_root, + "validator {proposer_index} proposing beacon block with root {block_root:?} \ + in slot {} using builder {}", slot_head.slot(), - payload_bid.builder_index + payload_bid.builder_index, ); } else { info_with_peers!( - "validator {} proposing beacon block with root {:?} in slot {}", - proposer_index, - beacon_block_root, + "validator {proposer_index} proposing beacon block with root {block_root:?} \ + in slot {}", slot_head.slot(), ); } - debug_with_peers!("beacon block: {beacon_block:?}"); - - let block = Arc::new(*beacon_block); - - let self_built = block - .payload_bid() - .is_some_and(|bid| bid.builder_index == BUILDER_INDEX_SELF_BUILD); + let signed_block = Arc::new(block.with_signature(signature)); - let post_gloas = slot_head.phase() >= Phase::Gloas; + debug_with_peers!("beacon block: {signed_block:?}"); - let import_and_publish_block = |block: Arc>| { - self.controller - .on_own_block(wait_group.clone(), block.clone_arc()); - - if let Some(builder_url) = &builder_url { - self.forward_block_to_builder(builder_url.clone(), block.clone_arc()); - } + beacon_nodes + .publish_block( + &signed_block, + block_proofs.as_ref(), + block_blobs.as_ref(), + builder_url.as_ref(), + ) + .await + } + }; - ValidatorToP2p::PublishBeaconBlock(block).send(&self.p2p_tx); - }; + if let Err(error) = published { + // The block producer has already logged why the builder did not publish the block. + if error.downcast_ref::().is_none() { + warn_with_peers!("failed to publish beacon block: {error:?}"); + } - if post_gloas { - import_and_publish_block(block.clone_arc()); - } + return Ok(()); + } - if (!post_gloas || self_built) - && let Some(blobs) = block_blobs - && !blobs.is_empty() - { - self.publish_blob_data(&wait_group, slot_head, &block, blobs, block_proofs) - .await?; - } + // A self-built payload is revealed after the block, so the payload timeliness committee + // has seen the bid before the payload arrives. + if let Some(EnvelopeContents { + envelope, + kzg_proofs, + blobs, + }) = envelope_contents + { + let Some(signed_envelope) = self + .sign_execution_payload_envelope(&signer_snapshot, slot_head, envelope, public_key) + .await + else { + return Ok(()); + }; - if !post_gloas { - import_and_publish_block(block); - } + debug_with_peers!( + "validator {proposer_index} publishing execution payload envelope for block {:?} \ + in slot {}", + signed_envelope.message.beacon_block_root, + slot_head.slot(), + ); - // If self-building: - // Publish the execution payload envelope after the beacon block so PTC members - // have seen the block (and its SignedExecutionPayloadBid) before attesting - if self_built - && let Some((envelope, ..)) = self - .block_producer - .build_local_execution_payload_envelope_contents( - beacon_block_root, - parent_beacon_block_root, - ) - .await - { - self.publish_execution_payload_envelope( - slot_head, - beacon_block_root, - envelope, - proposer_index, - &signer_snapshot, - public_key, - ) - .await?; - } + if let Err(error) = beacon_nodes + .publish_execution_payload_envelope(&Arc::new(signed_envelope), &kzg_proofs, &blobs) + .await + { + warn_with_peers!("failed to publish execution payload envelope: {error:?}"); + return Ok(()); } } @@ -1311,165 +1560,118 @@ impl Validator { Ok(()) } - async fn post_blinded_block( + async fn sign_randao_reveal( &self, + signer_snapshot: &Snapshot, slot_head: &SlotHead

, - signed_blinded_block: SignedBlindedBeaconBlock

, - block_blobs: &mut Option, P::MaxBlobCommitmentsPerBlock>>, - block_proofs: &mut Option>, - ) -> Result>> { - let builder_api = self - .builder_api - .as_ref() - .expect("Builder API should be present as it was used to query ExecutionPayloadHeader"); - - let signed_block = if slot_head.phase() >= Phase::Fulu - && builder_api - .post_blinded_block_post_fulu( - &self.chain_config, - self.controller.genesis_time(), - &signed_blinded_block, - ) - .await - .is_ok() - { - debug_with_peers!("submitted blinded block to the builder node"); + public_key: PublicKeyBytes, + ) -> Option { + let epoch = slot_head.current_epoch(); - Some(SignedBeaconBlockOrBlockRoot::Root( - signed_blinded_block.message().hash_tree_root(), - )) - } else { - let WithBlobsAndMev { - value: execution_payload, - proofs, - blobs, - .. - } = match builder_api - .post_blinded_block( - &self.chain_config, - self.controller.genesis_time(), - &signed_blinded_block, - ) - .await - { - Ok(response) => response, - Err(error) => { - if let Some(reqwest_error) = error.downcast_ref::() { - if reqwest_error.is_timeout() { - // If posting signed blinded beacon block to builder fails, don't print a warning, - // because builder should publish the block anyway, just cannot respond in a timely - // manner. We cannot do anything else here either, but exit early from propose, due - // to the risk of slashing. - debug_with_peers!( - "failed to post blinded block to the builder node: {error:?}" - ); - } + let result = signer_snapshot + .sign_without_slashing_protection::

( + SigningMessage::RandaoReveal { epoch }, + SignForSingleFork::

::signing_root_from_fork_info( + &RandaoEpoch::from(epoch), + &self.chain_source.chain_config, + slot_head.fork_info, + ), + Some(slot_head.fork_info), + public_key, + ) + .await; - return Ok(None); - } + match result { + Ok(signature) => Some(signature.into()), + Err(error) => { + warn_with_peers!( + "failed to sign RANDAO reveal (epoch: {epoch}, public_key: {public_key}): \ + {error:?}", + ); - warn_with_peers!("failed to post blinded block to the builder node: {error:?}"); + None + } + } + } - return Ok(None); - } - }; + async fn own_proposer_from_duties( + &self, + beacon_nodes: &BeaconNodes, + slot_head: &SlotHead

, + ) -> Result> { + let slot = slot_head.slot(); + let epoch = slot_head.current_epoch(); - *block_proofs = proofs; - *block_blobs = blobs; + let validator_indices = self.own_duties.validator_indices.sorted(); - debug_with_peers!( - "received execution payload from the builder node: {execution_payload:?}" - ); + let Some(first_index) = validator_indices.first().copied() else { + return Ok(None); + }; - let (message, signature) = signed_blinded_block.split(); + let dependent_epoch = proposer_dependent_epoch(&self.chain_source.chain_config, epoch); - Some(SignedBeaconBlockOrBlockRoot::Block(Box::new( - message - .with_execution_payload(execution_payload)? - .with_signature(signature), - ))) + // The attester cache shares the root, but nodes unreachable at the epoch's start leave it + // empty; the head events or a request report it either way. + let dependent_root = match self + .own_duties + .beacon_committee_members + .cached_dependent_root(dependent_epoch) + .await + { + Some(dependent_root) => dependent_root, + None => { + beacon_nodes + .dependent_root(dependent_epoch, first_index) + .await? + } }; - Ok(signed_block) - } + if let Some(proposer) = self + .own_duties + .proposer_duties + .get_at_slot(dependent_root, slot) + .await + { + return Ok(Some(proposer)); + } - async fn publish_blob_data( - &self, - wait_group: &W, - slot_head: &SlotHead

, - block: &Arc>, - blobs: ContiguousList, P::MaxBlobCommitmentsPerBlock>, - block_proofs: Option>, - ) -> Result<()> { - if slot_head.phase().is_peerdas_activated() { - let data_column_sidecars = eip_7594::construct_data_column_sidecars_from_blobs( - block.clone_arc().into(), - blobs.into_iter(), - block_proofs - .unwrap_or_else(KzgProofs::empty_fulu) - .into_iter() - .collect_vec(), - self.controller.store_config().kzg_backend, - self.metrics.clone(), - self.dedicated_executor_normal_priority.clone_arc(), - ) + // A no-op once the epoch is fetched under this root; a reorg at the slot boundary is not. + self.own_duties + .proposer_duties + .init_at_epoch(beacon_nodes, epoch, dependent_root, &validator_indices) .await?; - for data_column_sidecar in data_column_sidecars { - if self - .controller - .sampling_columns() - .into_iter() - .contains(&data_column_sidecar.index()) - { - self.controller - .on_own_data_column_sidecar( - wait_group.clone(), - data_column_sidecar.clone_arc(), - ) - .await; - } - - ValidatorToP2p::PublishDataColumnSidecar(data_column_sidecar).send(&self.p2p_tx); - } - } else { - for blob_sidecar in misc::construct_blob_sidecars( - block, - blobs, - block_proofs.unwrap_or_else(KzgProofs::empty_deneb), - )? { - let blob_sidecar = Arc::new(blob_sidecar); - - self.controller - .on_own_blob_sidecar(wait_group.clone(), blob_sidecar.clone_arc()); - } - } - - Ok(()) + Ok(self + .own_duties + .proposer_duties + .get_at_slot(dependent_root, slot) + .await) } - /// Create and sign Gloas execution payload envelope for self-build proposers. - /// Returns None if envelope data is not available (i.e., not self-building). - async fn publish_execution_payload_envelope( + async fn sign_execution_payload_envelope( &self, + signer_snapshot: &Snapshot, slot_head: &SlotHead

, - beacon_block_root: H256, envelope: ExecutionPayloadEnvelope

, - proposer_index: ValidatorIndex, - signer_snapshot: &Snapshot, - public_key: &PublicKeyBytes, - ) -> Result<()> { - // Sign the envelope - let envelope_sig = match signer_snapshot + public_key: PublicKeyBytes, + ) -> Option> { + let result = signer_snapshot .sign_without_slashing_protection( SigningMessage::ExecutionPayloadEnvelope(&envelope), - envelope.signing_root(&self.chain_config, &slot_head.beacon_state), - Some(slot_head.beacon_state.as_ref().into()), - *public_key, + envelope.signing_root_from_fork_info( + &self.chain_source.chain_config, + slot_head.fork_info, + ), + Some(slot_head.fork_info), + public_key, ) - .await - { - Ok(signature) => signature.into(), + .await; + + match result { + Ok(signature) => Some(SignedExecutionPayloadEnvelope { + message: envelope, + signature: signature.into(), + }), Err(error) => { warn_with_peers!( "failed to sign execution payload envelope (slot: {}, public_key: {public_key}): \ @@ -1477,29 +1679,9 @@ impl Validator { slot_head.slot(), ); - return Ok(()); + None } - }; - - let signed_envelope = Arc::new(SignedExecutionPayloadEnvelope { - message: envelope, - signature: envelope_sig, - }); - - debug_with_peers!( - "validator {} publishing execution payload envelope for block {:?} in slot {}", - proposer_index, - beacon_block_root, - slot_head.slot(), - ); - - // Publish envelope to controller and P2P - self.controller - .on_own_execution_payload_envelope(signed_envelope.clone_arc()); - - ValidatorToP2p::PublishExecutionPayloadEnvelope(signed_envelope).send(&self.p2p_tx); - - Ok(()) + } } /// See: @@ -1507,22 +1689,11 @@ impl Validator { /// - #[expect(clippy::too_many_lines)] #[instrument(level = "debug", skip_all)] - // - // `full` is whether the attestations vote for the full node of `slot_head`. - // It must be the node the payload status was checked on. async fn attest_and_start_aggregating( &mut self, - wait_group: &W, - slot_head: Option<&SlotHead

>, - full: bool, + beacon_nodes: &BeaconNodes, ) -> Result<()> { - let Some(slot_head) = slot_head else { - warn_with_peers!( - "validator cannot participate in attestation because \ - chain head has not been fully verified by an execution engine", - ); - return Ok(()); - }; + let slot_head = beacon_nodes.head(); // Skip attesting if validators already attested at slot if self.attested_in_current_slot() { @@ -1534,38 +1705,38 @@ impl Validator { .as_ref() .map(|metrics| metrics.validator_attest_times.start_timer()); - let dependent_root = self - .controller - .attestation_committee_dependent_root_for_slot( - &slot_head.beacon_state, + let Some((own_members, needs_to_update_subscriptions)) = self + .own_duties + .beacon_committee_members + .get_or_init_at_slot( + self.chain_source.controller(), + beacon_nodes.state(), slot_head.slot(), - )?; - - let needs_to_update_subscriptions = self - .own_beacon_committee_members - .needs_to_compute_members_at_slot(dependent_root, slot_head.slot()) - .await; - - let Some(own_members) = self - .own_beacon_committee_members - .get_or_init_at_slot(&slot_head.beacon_state, dependent_root, slot_head.slot()) - .await + &self.own_duties.validator_indices.sorted(), + ) + .await? else { return Ok(()); }; - if needs_to_update_subscriptions { - update_beacon_committee_subscriptions( - slot_head.slot(), - &own_members, - &self.subnet_service_tx, - ) - .await; + if needs_to_update_subscriptions + && let Err(error) = beacon_nodes + .subscribe_to_beacon_committees( + slot_head.slot(), + &own_members + .iter() + .copied() + .map(Into::into) + .collect::>(), + ) + .await + { + warn_with_peers!("failed to update beacon committee subscriptions: {error:?}"); } let own_singular_attestations = self - .own_singular_attestations(slot_head, full, &own_members) - .await?; + .own_singular_attestations(slot_head, &own_members, beacon_nodes) + .await; if own_singular_attestations.is_empty() { prometheus_metrics::stop_and_discard(timer); @@ -1591,35 +1762,19 @@ impl Validator { let committee_index = misc::committee_index(attestation); debug_with_peers!( - "validator {} of committee {} ({:?}) attesting in slot {}: {:?}", + "validator {} of committee {} attesting in slot {}: {:?}", validator_index, committee_index, - slot_head - .beacon_committee(committee_index) - .expect("committee was already used to construct attestation"), slot_head.slot(), attestation, ); + } - let attestation = Arc::new(attestation.clone()); - let subnet_id = slot_head.subnet_id(attestation.data().slot, committee_index)?; - - self.controller.on_singular_attestation( - wait_group.clone(), - AttestationItem::unverified( - attestation.clone_arc(), - AttestationOrigin::Own(subnet_id), - ), - ); - - ValidatorToP2p::PublishSingularAttestation(attestation.clone_arc(), subnet_id) - .send(&self.p2p_tx); - - self.attestation_agg_pool.insert_attestation( - wait_group.clone(), - attestation, - Some(*validator_index), - ); + if let Err(error) = beacon_nodes + .publish_singular_attestations(own_singular_attestations) + .await + { + warn_with_peers!("failed to publish attestations: {error:?}"); } prometheus_metrics::stop_and_record(timer); @@ -1650,12 +1805,12 @@ impl Validator { return None; } - let aggregator = selection_proof.map(|selection_proof| Aggregator { + let aggregator = Aggregator { aggregator_index: validator_index, position_in_committee, public_key, selection_proof, - })?; + }; let data = own_attestation.attestation.data(); let key = AttestationKey { @@ -1672,28 +1827,35 @@ impl Validator { #[expect(clippy::too_many_lines)] #[instrument(level = "debug", skip_all)] - async fn publish_aggregates_and_proofs(&self, wait_group: &W, slot_head: Option<&SlotHead

>) { - let Some(slot_head) = slot_head else { - warn_with_peers!( - "validators cannot participate in aggregation because \ - chain head has not been fully verified by an execution engine", - ); - return; - }; - - let config = &self.chain_config; - let phase = slot_head.phase(); - + async fn publish_aggregates_and_proofs(&self, beacon_nodes: &BeaconNodes) { + let slot_head = beacon_nodes.head(); + let config = &self.chain_source.chain_config; let (triples, proofs): (Vec<_>, Vec<_>) = self .own_aggregators .iter() - .map(|(key, aggregators)| async { - self.attestation_agg_pool - .best_aggregate_attestation(*key) - .await - .into_iter() - .flat_map(|aggregate| { - aggregators.iter().filter_map(move |aggregator| { + .map(|(key, aggregators)| { + let beacon_nodes = &beacon_nodes; + + async move { + let aggregate = match beacon_nodes + .aggregate_attestation(key.data, key.committee_index) + .await + { + Ok(aggregate) => aggregate, + Err(error) => { + warn_with_peers!( + "failed to obtain an aggregate attestation for committee {} \ + in slot {}: {error:?}", + key.committee_index, + key.data.slot, + ); + return vec![]; + } + }; + + aggregators + .iter() + .filter_map(|aggregator| { let Aggregator { aggregator_index, position_in_committee, @@ -1701,39 +1863,33 @@ impl Validator { selection_proof, } = *aggregator; - if !*aggregate.aggregation_bits.get(position_in_committee)? { + if !aggregated_by(&aggregate, position_in_committee)? { return None; } - let aggregate_and_proof = if phase < Phase::Electra { - AggregateAndProof::from(Phase0AggregateAndProof { - aggregator_index, - aggregate: aggregate.clone().into_phase0_attestation(), - selection_proof, - }) - } else if phase < Phase::Gloas { - let aggregate = operation_pools::convert_to_electra_attestation( - aggregate.clone(), - ) - .ok()?; - - AggregateAndProof::from(ElectraAggregateAndProof { - aggregator_index, - aggregate, - selection_proof, - }) - } else { - let aggregate = operation_pools::convert_to_electra_attestation( - aggregate.clone(), - ) - .ok()? - .into(); - - AggregateAndProof::from(GloasAggregateAndProof { - aggregator_index, - aggregate, - selection_proof, - }) + let aggregate_and_proof = match aggregate.clone() { + Attestation::Phase0(aggregate) => { + AggregateAndProof::from(Phase0AggregateAndProof { + aggregator_index, + aggregate, + selection_proof, + }) + } + Attestation::Electra(aggregate) => { + AggregateAndProof::from(ElectraAggregateAndProof { + aggregator_index, + aggregate, + selection_proof, + }) + } + Attestation::Gloas(aggregate) => { + AggregateAndProof::from(GloasAggregateAndProof { + aggregator_index, + aggregate, + selection_proof, + }) + } + Attestation::Single(_) => return None, }; let triple = SigningTriple { @@ -1741,14 +1897,14 @@ impl Validator { aggregate_and_proof.clone(), )), signing_root: aggregate_and_proof - .signing_root(config, &slot_head.beacon_state), + .signing_root_from_fork_info(config, slot_head.fork_info), public_key, }; Some((triple, aggregate_and_proof)) }) - }) - .collect_vec() + .collect_vec() + } }) .collect::>() .collect::>() @@ -1760,10 +1916,7 @@ impl Validator { let signer_snapshot = self.signer.load(); let sign_result = signer_snapshot - .sign_triples_without_slashing_protection( - triples, - Some(slot_head.beacon_state.as_ref().into()), - ) + .sign_triples_without_slashing_protection(triples, Some(slot_head.fork_info)) .await; let signatures = match sign_result { @@ -1817,14 +1970,16 @@ impl Validator { aggregate_and_proof.slot(), ); - for aggregate_and_proof in aggregates_and_proofs { - let attestation = Arc::new(aggregate_and_proof.aggregate()); - let aggregate_and_proof = Arc::new(aggregate_and_proof); - - self.attestation_agg_pool - .insert_attestation(wait_group.clone(), attestation, None); + let aggregates_and_proofs = aggregates_and_proofs + .into_iter() + .map(Arc::new) + .collect_vec(); - ValidatorToP2p::PublishAggregateAndProof(aggregate_and_proof).send(&self.p2p_tx); + if let Err(error) = beacon_nodes + .publish_aggregates_and_proofs(&aggregates_and_proofs) + .await + { + warn_with_peers!("failed to publish aggregates and proofs: {error:?}"); } } @@ -1832,16 +1987,9 @@ impl Validator { #[instrument(level = "debug", skip_all)] async fn publish_sync_committee_messages( &mut self, - wait_group: &W, - slot_head: Option>, + beacon_nodes: &BeaconNodes, ) -> Result<()> { - let Some(slot_head) = slot_head else { - warn_with_peers!( - "validator cannot participate in sync committees because \ - chain head has not been fully verified by an execution engine", - ); - return Ok(()); - }; + let slot_head = beacon_nodes.head(); // > To reduce complexity during the Altair fork, sync committees are not expected to // > produce signatures for `compute_epoch_at_slot(ALTAIR_FORK_EPOCH) - 1`. @@ -1857,7 +2005,42 @@ impl Validator { return Ok(()); } - let own_messages = self.own_sync_committee_messages(&slot_head).await?; + let Some(members) = self + .sync_committee_members(beacon_nodes, slot_head.slot()) + .await + else { + // Without a validator index there are no duties to prefetch and nothing to sign. + if !self.own_duties.validator_indices.sorted().is_empty() { + warn_with_peers!( + "no sync committee duties were prefetched for slot {}, so no sync committee \ + message will be produced", + slot_head.slot(), + ); + } + + return Ok(()); + }; + + if members.is_empty() { + self.published_own_sync_committee_messages_for = Some(slot_head.clone()); + return Ok(()); + } + + let beacon_block_root = match beacon_nodes.head_block_root().await { + Ok(beacon_block_root) => beacon_block_root, + Err(error) => { + warn_with_peers!( + "failed to obtain a head block root for sync committee messages in slot {}: \ + {error:?}", + slot_head.slot(), + ); + return Ok(()); + } + }; + + let own_messages = self + .own_sync_committee_messages(slot_head, beacon_block_root, &members) + .await; for (subcommittee_index, messages) in &own_messages { if !messages.is_empty() { @@ -1867,64 +2050,131 @@ impl Validator { slot_head.slot(), ); } - } - for (sync_subnet_id, messages) in own_messages { - for sync_committee_message in &messages { + for sync_committee_message in messages { debug_with_peers!( "validator {} publishing sync committee message (subnet_id: {}): {:?}", sync_committee_message.validator_index, - sync_subnet_id, + subcommittee_index, sync_committee_message, ); - - ValidatorToP2p::PublishSyncCommitteeMessage(Box::new(( - sync_subnet_id, - *sync_committee_message, - ))) - .send(&self.p2p_tx); } + } - self.sync_committee_agg_pool.aggregate_own_messages( - wait_group.clone(), - messages, - sync_subnet_id, - slot_head.beacon_state.clone_arc(), - ); + if let Err(error) = beacon_nodes + .publish_sync_committee_messages(&own_messages) + .await + { + warn_with_peers!("failed to publish sync committee messages: {error:?}"); } - self.published_own_sync_committee_messages_for = Some(slot_head); + self.published_own_sync_committee_messages_for = Some(slot_head.clone()); + self.own_sync_committee_message_root = Some(beacon_block_root); Ok(()) } + async fn sync_committee_members( + &self, + beacon_nodes: &BeaconNodes, + slot: Slot, + ) -> Option> { + self.cached_sync_committee_members(beacon_nodes, slot) + .await + .map(|members| { + // A key removed at runtime must not fail the whole signing batch. + let signer_snapshot = self.signer.load(); + + members + .iter() + .filter(|member| signer_snapshot.has_key(member.public_key)) + .copied() + .collect() + }) + } + + async fn cached_sync_committee_members( + &self, + beacon_nodes: &BeaconNodes, + slot: Slot, + ) -> Option> { + if let Some(members) = self + .own_duties + .sync_committee_members + .get_at_slot::

(slot) + .await + { + return Some(members); + } + + // A restart may reach the deadline before the prefetch; only the built-in beacon node + // answers from its state in time. + if !beacon_nodes.has_local_node() { + return None; + } + + let validator_indices = self.own_duties.validator_indices.sorted(); + + match self + .own_duties + .sync_committee_members + .get_or_init_at_slot( + &self.chain_source.chain_config, + beacon_nodes, + slot, + &validator_indices, + ) + .await + { + Ok(members) => members, + Err(error) => { + warn_with_peers!( + "failed to obtain sync committee duties for slot {slot}: {error:?}" + ); + None + } + } + } + /// #[instrument(level = "debug", skip_all)] - async fn publish_contributions_and_proofs(&self, slot_head: Option<&SlotHead

>) { - let Some(slot_head) = slot_head else { - warn_with_peers!( - "validator cannot participate in sync committees because \ - chain head has not been fully verified by an execution engine", - ); - return; - }; - - if !self.controller.is_forward_synced() { + async fn publish_contributions_and_proofs(&self, beacon_nodes: &BeaconNodes) { + if let Some(controller) = self.chain_source.controller() + && !controller.is_forward_synced() + { return; } + // The messages voted for the head of the slot they were signed in, which the + // contributions must aggregate. + let slot_head = self + .published_own_sync_committee_messages_for + .as_ref() + .unwrap_or_else(|| beacon_nodes.head()); + if !slot_head.has_sync_committee() { return; } - let contributions = match self.own_contributions_and_proofs(slot_head).await { - Ok(contributions) => contributions, - Err(error) => { - error_with_peers!("error while producing own contributions and proofs: {error:?}"); - return; - } + let Some(members) = self + .sync_committee_members(beacon_nodes, slot_head.slot()) + .await + else { + return; }; + if members.is_empty() { + return; + } + + let beacon_block_root = self + .own_sync_committee_message_root + .unwrap_or(slot_head.beacon_block_root); + + let contributions = self + .own_contributions_and_proofs(beacon_nodes, slot_head, beacon_block_root, &members) + .await; + if contributions.is_empty() { return; } @@ -1938,53 +2188,33 @@ impl Validator { slot_head.slot(), ); - for contribution_and_proof in contributions { + for contribution_and_proof in &contributions { debug_with_peers!( "validator {} publishing sync committee contribution and proof: {:?}", contribution_and_proof.message.aggregator_index, contribution_and_proof, ); + } - ValidatorToP2p::PublishContributionAndProof(Box::new(contribution_and_proof)) - .send(&self.p2p_tx); - - self.sync_committee_agg_pool.add_own_contribution( - contribution_and_proof.message.aggregator_index, - contribution_and_proof.message.contribution, - slot_head.beacon_state.clone_arc(), - ); + if let Err(error) = beacon_nodes + .publish_contributions_and_proofs(&contributions) + .await + { + warn_with_peers!("failed to publish contributions and proofs: {error:?}"); } } #[instrument(level = "debug", skip_all)] - async fn attest_payload( - &self, - wait_group: &W, - slot_head: &SlotHead

, - early: bool, - ) -> Result<()> { - // Skip attesting if validators already attested at slot - if self.payload_attested_in_current_slot() { - return Ok(()); - } + async fn attest_payload(&self, beacon_nodes: &BeaconNodes) -> Result<()> { + let slot_head = beacon_nodes.head(); + let slot = slot_head.slot(); - // Skip attesting if validators has not seen any beacon block for the assigned slot - let Some(block_with_root) = self - .controller - .stored_block_by_slot(slot_head.slot())? - .map(WithStatus::value) - else { + if slot_head.phase() < Phase::Gloas { return Ok(()); - }; + } - // Before the deadline the vote is only cast once both the envelope and the blob data have been seen. - if early - && !(self.controller.is_payload_verified(block_with_root.root) - && self - .controller - .indices_of_missing_data_columns(&block_with_root.block) - .is_empty()) - { + // Skip attesting if validators already attested at slot + if self.payload_attested_in_current_slot() { return Ok(()); } @@ -1993,71 +2223,129 @@ impl Validator { .as_ref() .map(|metrics| metrics.validator_attest_payload_times.start_timer()); - let dependent_root = self - .controller - .attestation_committee_dependent_root_for_slot( - &slot_head.beacon_state, - slot_head.slot(), - )?; + let own_members = match self.chain_source.controller().zip(beacon_nodes.state()) { + Some((controller, beacon_state)) => { + let dependent_root = + controller.attestation_committee_dependent_root_for_slot(beacon_state, slot)?; + + self.own_duties + .ptc_members + .get_or_init_at_slot( + &self.chain_source.chain_config, + beacon_state, + dependent_root, + slot, + &self.own_duties.validator_indices.sorted(), + ) + .await + } + None => { + self.own_ptc_members_from_duties(beacon_nodes, slot_head) + .await? + } + }; + + let Some(own_members) = own_members else { + return Ok(()); + }; + + if own_members.is_empty() { + return Ok(()); + } - let Some(own_members) = self - .own_ptc_members - .get_or_init_at_slot( - self.controller.chain_config(), - &slot_head.beacon_state, - dependent_root, - slot_head.slot(), - ) - .await - else { + // Skip attesting if no beacon node has seen a beacon block for the assigned slot + let Some(data) = beacon_nodes.payload_attestation_data(slot).await? else { + debug_with_peers!("no beacon block seen for slot {slot}; skipping payload attestation"); return Ok(()); }; let own_payload_attestations = self - .own_payload_attestations(slot_head, &block_with_root, &own_members) - .await?; + .own_payload_attestations(slot_head, data, &own_members) + .await; if own_payload_attestations.is_empty() { return Ok(()); } debug_with_peers!( - "validators [{}] attesting to payload in slot {}", + "validators [{}] attesting to payload in slot {slot}", own_payload_attestations .iter() .map(|a| a.validator_index) .join(", "), - slot_head.slot(), ); - for own_payload_attestation in own_payload_attestations.iter().copied() { - let payload_attestation = Arc::new(own_payload_attestation); + let messages = own_payload_attestations + .iter() + .copied() + .map(Arc::new) + .collect::>(); + + if let Err(error) = beacon_nodes.publish_payload_attestations(&messages).await { + warn_with_peers!("failed to publish payload attestations: {error:?}"); + } - self.controller.on_payload_attestation( - wait_group.clone(), - PayloadAttestationItem::unverified( - Arc::new(payload_attestation.clone_arc().into()), - PayloadAttestationOrigin::Own, - ), - ); + Ok(()) + } + + async fn own_ptc_members_from_duties( + &self, + beacon_nodes: &BeaconNodes, + slot_head: &SlotHead

, + ) -> Result>> { + let slot = slot_head.slot(); + let epoch = slot_head.current_epoch(); + + // The same dependent root attester duties were fetched under, as the committees are drawn + // from the same shuffling. + let Some(dependent_root) = self + .own_duties + .beacon_committee_members + .cached_dependent_root(epoch) + .await + else { + // Without a validator index there are no duties to prefetch. + if !self.own_duties.validator_indices.sorted().is_empty() { + warn_with_peers!( + "no attester duties were prefetched for slot {slot}, so no payload \ + attestation will be produced", + ); + } + + return Ok(None); + }; - ValidatorToP2p::PublishPayloadAttestation(payload_attestation).send(&self.p2p_tx); + if let Some(members) = self + .own_duties + .ptc_members + .get_at_slot(dependent_root, slot) + .await + { + return Ok(Some(members)); } - let next_proposer_index = tokio::task::block_in_place(|| slot_head.next_proposer_index())?; - let public_key = slot_head.public_key(next_proposer_index); - let signer_snapshot = self.signer.load(); + let validator_indices = self.own_duties.validator_indices.sorted(); - // Only add the messages into the pool if any attached validators is the next proposer - if signer_snapshot.has_key(*public_key) { - self.payload_attestation_agg_pool.aggregate_own_messages( - wait_group.clone(), - own_payload_attestations.to_vec(), - slot_head.beacon_state.clone_arc(), - ); + if validator_indices.is_empty() { + return Ok(None); } - Ok(()) + self.own_duties + .ptc_members + .init_at_epoch( + &self.chain_source.chain_config, + beacon_nodes, + epoch, + dependent_root, + &validator_indices, + ) + .await?; + + Ok(self + .own_duties + .ptc_members + .get_at_slot(dependent_root, slot) + .await) } #[instrument(level = "debug", skip_all)] @@ -2074,13 +2362,38 @@ impl Validator { return; } - let slot_head = SlotHead { - config: self.chain_config.clone_arc(), - beacon_block_root: head.block_root, - beacon_state: self.controller.state_by_chain_link(&head), + let Some(controller) = self.chain_source.controller() else { + return; + }; + + // Before the deadline the vote is only cast once both the envelope and the blob data have + // been seen. + if !(controller.is_payload_verified(head.block_root) + && controller + .indices_of_missing_data_columns(&head.block) + .is_empty()) + { + return; + } + + let beacon_state = controller.state_by_chain_link(&head); + + let slot_head = SlotHead::from_state( + self.chain_source.chain_config.clone_arc(), + head.slot(), + head.block_root, + beacon_state.as_ref(), + ); + + let beacon_nodes = match self.local_beacon_node(slot_head, beacon_state, wait_group) { + Ok(beacon_nodes) => beacon_nodes, + Err(error) => { + warn_with_peers!("failed to attest to a payload early: {error}"); + return; + } }; - if let Err(error) = self.attest_payload(wait_group, &slot_head, true).await { + if let Err(error) = self.attest_payload(&beacon_nodes).await { error_with_peers!("failed to produce and publish own payload attestations: {error:?}"); } } @@ -2095,29 +2408,91 @@ impl Validator { return; } + // The early-publish paths act on the head of the built-in beacon node. + let Some(controller) = self.chain_source.controller() else { + return; + }; + // Pre-Gloas only fully validated heads are sent here. Post-Gloas every head is. // Leave an unsettled one to the attesting tick, which waits for it. - let full = - self.duty_uses_full_node(head.block_root, head.slot(), head.slot(), HeadDuty::Attest); + let full = duty_uses_full_node( + controller, + head.block_root, + head.slot(), + head.slot(), + HeadDuty::Attest, + ); - if !self - .controller + if !controller .node_payload_status(head.block_root, full) .is_some_and(PayloadStatus::is_valid) { return; } - let slot_head = SlotHead { - config: self.chain_config.clone_arc(), - beacon_block_root: head.block_root, - beacon_state: self.controller.state_by_chain_link(&head), + let beacon_state = controller.state_by_chain_link(&head); + + let slot_head = SlotHead::from_state( + self.chain_source.chain_config.clone_arc(), + head.slot(), + head.block_root, + beacon_state.as_ref(), + ); + + let beacon_nodes = match self.local_beacon_node(slot_head, beacon_state, wait_group) { + Ok(BeaconNodes::Local(node)) => BeaconNodes::Local(node.with_full(full)), + Ok(beacon_nodes) => beacon_nodes, + Err(error) => { + warn_with_peers!("failed to attest to a gossip block: {error}"); + return; + } + }; + + self.attest_before_tick(&beacon_nodes).await; + } + + async fn attest_remote_head( + &mut self, + node: Arc, + slot: Slot, + block_root: H256, + ) { + let Some(last_tick) = self.last_tick else { + return; + }; + + if !(last_tick.slot == slot && last_tick.is_before_attesting_interval()) { + return; + } + + let Some(remote_beacon_nodes) = self.chain_source.remote_beacon_nodes() else { + return; }; - if let Err(error) = self - .attest_and_start_aggregating(wait_group, Some(&slot_head), full) - .await - { + // The node that streamed the head is the one known to hold its block, so it is asked first. + let Some(nodes) = self + .remote_nodes_at(remote_beacon_nodes, slot) + .led_by(&node) + else { + return; + }; + + let Some(slot_head) = nodes.slot_head::

(slot) else { + return; + }; + + // The cache has moved past the event; the newer head arrives with an event of its own. + if slot_head.beacon_block_root != block_root { + return; + } + + let beacon_nodes = BeaconNodes::Remote { nodes, slot_head }; + + self.attest_before_tick(&beacon_nodes).await; + } + + async fn attest_before_tick(&mut self, beacon_nodes: &BeaconNodes) { + if let Err(error) = self.attest_and_start_aggregating(beacon_nodes).await { error_with_peers!("failed to produce and publish own attestations: {error:?}"); } @@ -2125,9 +2500,7 @@ impl Validator { // This noticeably improves rewards in Goerli. // This is a deviation from the Honest Validator specification. if Feature::PublishSyncCommitteeMessagesEarly.is_enabled() - && let Err(error) = self - .publish_sync_committee_messages(wait_group, Some(slot_head)) - .await + && let Err(error) = self.publish_sync_committee_messages(beacon_nodes).await { error_with_peers!( "failed to produce and publish own sync_committee messages: {error:?}" @@ -2163,29 +2536,71 @@ impl Validator { async fn own_singular_attestations( &self, slot_head: &SlotHead

, - full: bool, own_members: &[BeaconCommitteeMember], - ) -> Result<&[OwnAttestation

]> { + beacon_nodes: &BeaconNodes, + ) -> &[OwnAttestation

] { if let Some(own_attestations) = self.own_singular_attestations.get() { - return Ok(own_attestations); + return own_attestations; + } + + if own_members.is_empty() { + return &[]; } let phase = slot_head.phase(); + let slot = slot_head.slot(); - // `full` is the node the payload status was checked on. - // Choosing it again here could vote for a payload nobody checked. - // See: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.5/specs/gloas/validator.md#attestation - let gloas_index = (phase >= Phase::Gloas).then_some(u64::from(full)); + // From Electra on the committee index is not part of the produced data, + // so a single request covers every member. + let committee_indices = if phase >= Phase::Electra { + vec![0] + } else { + own_members + .iter() + .map(|member| member.committee_index) + .unique() + .collect() + }; - let (triples, other_data): (Vec<_>, Vec<_>) = tokio::task::block_in_place(|| { - let target = Checkpoint { - epoch: slot_head.current_epoch(), - root: accessors::epoch_boundary_block_root( - &slot_head.beacon_state, - slot_head.beacon_block_root, - ), - }; + // Fetched concurrently, and each committee stands on its own: one that cannot be served + // only costs its own members their attestation, rather than the whole slot's. + let attestation_data = committee_indices + .into_iter() + .map(|committee_index| async move { + let data = beacon_nodes.attestation_data(slot, committee_index).await?; + let produced_for = data.slot; + + ensure!( + produced_for == slot, + "beacon node produced attestation data for slot {produced_for}, \ + expected {slot}", + ); + + Ok::<_, AnyhowError>((committee_index, data)) + }) + .pipe(join_all) + .await + .into_iter() + .filter_map(|result| { + result + .inspect_err(|error| { + warn_with_peers!( + "unable to produce attestation data in slot {slot}: {error:?}", + ); + }) + .ok() + }) + .collect::>(); + + // Nothing fetched must not count as having attested, or the attestation tick skips the + // slot after an early attempt that found no data. + if attestation_data.is_empty() { + return &[]; + } + + let snapshot = self.signer.load(); + let (triples, other_data): (Vec<_>, Vec<_>) = tokio::task::block_in_place(|| { let doppelganger_protection = self .doppelganger_protection .as_deref() @@ -2193,40 +2608,35 @@ impl Validator { own_members .iter() + // A key removed at runtime must not fail the whole signing batch. + .filter(|member| snapshot.has_key(member.public_key)) .filter_map(|member| { if let Some(doppelganger_protection) = &doppelganger_protection - && !doppelganger_protection.is_validator_active(member.public_key) - { - info_with_peers!( - "Validator {:?} skipping attesting duty in slot {} \ - since not enough time has passed to ensure there are \ - no doppelganger validators participating on network. \ - Validator will start performing duties on slot {}.", - member.public_key, + && !doppelganger_protection_permits::

( + doppelganger_protection, + "attesting", slot_head.slot(), - doppelganger_protection.tracking_end_slot::

(member.public_key), - ); + member.public_key, + ) + { return None; } - let mut data = AttestationData { - slot: slot_head.slot(), - index: member.committee_index, - beacon_block_root: slot_head.beacon_block_root, - source: slot_head.beacon_state.current_justified_checkpoint(), - target, + let committee_index = if phase >= Phase::Electra { + 0 + } else { + member.committee_index }; - if let Some(idx) = gloas_index { - data.index = idx; - } else if phase >= Phase::Electra { - data.index = 0; - } + let data = *attestation_data.get(&committee_index)?; let triple = SigningTriple { message: SigningMessage::

::Attestation(data), - signing_root: data - .signing_root(&self.chain_config, &slot_head.beacon_state), + signing_root: SignForSingleFork::

::signing_root_from_fork_info( + &data, + &self.chain_source.chain_config, + slot_head.fork_info, + ), public_key: member.public_key, }; @@ -2235,12 +2645,11 @@ impl Validator { .unzip() }); - let snapshot = self.signer.load(); - let result = snapshot .sign_triples( triples, - slot_head.beacon_state.as_ref(), + slot_head.fork_info, + slot_head.current_epoch(), self.slashing_protector.clone_arc(), ) .await; @@ -2249,18 +2658,18 @@ impl Validator { Ok(signatures) => signatures, Err(error) => { warn_with_peers!("failed to sign attestations: {error:?}"); - return Ok(&[]); + return &[]; } }; self.own_singular_attestations - .get_or_try_init(|| { + .get_or_init(|| { let _timer = self .metrics .as_ref() .map(|metrics| metrics.validator_own_attestations_init_times.start_timer()); - let own_attestations = signatures + signatures .zip(other_data) .filter_map(|(signature, (data, member))| { signature.and_then(|signature| { @@ -2291,73 +2700,29 @@ impl Validator { }) }) }) - .collect(); - - Ok(own_attestations) + .collect() }) - .map(Vec::as_slice) - } - - #[instrument(level = "debug", skip_all)] - fn own_sync_committee_members_for_epoch( - &self, - relative_epoch: SyncCommitteeEpoch, - state: &(impl PostAltairBeaconState

+ ?Sized), - ) -> Result> { - let own_public_keys = self.own_public_keys(); - - tokio::task::block_in_place(|| { - let sync_committee = match relative_epoch { - SyncCommitteeEpoch::Current => state.current_sync_committee(), - SyncCommitteeEpoch::Next => { - if misc::sync_committee_period::

(accessors::get_current_epoch(state)) - == misc::sync_committee_period::

(accessors::get_next_epoch(state)?) - { - state.current_sync_committee() - } else { - state.next_sync_committee() - } - } - }; - - sync_committee - .pubkeys - .iter() - .filter_map(|public_key| { - if !own_public_keys.contains(public_key) { - return None; - } - - let validator_index = accessors::index_of_public_key(state, public_key)?; - Some((validator_index, public_key)) - }) - .sorted_by_key(|(validator_index, _)| *validator_index) - .collect_vec() - .into_par_iter() - .map(|(validator_index, public_key)| { - let subnets = misc::compute_subnets_for_sync_committee(state, validator_index)?; - - Ok(SyncCommitteeMember { - validator_index, - public_key: *public_key, - subnets, - }) - }) - .collect() - }) + .as_slice() } #[instrument(level = "debug", skip_all)] async fn own_sync_committee_messages( &self, slot_head: &SlotHead

, - ) -> Result>> { - let indices_with_pubkeys = self - .own_sync_committee_members() + beacon_block_root: H256, + members: &[SyncCommitteeMember], + ) -> BTreeMap> { + let indices_with_pubkeys = members + .iter() .map(|member| (member.validator_index, member.public_key)); let messages = match slot_head - .sync_committee_messages(slot_head.slot(), indices_with_pubkeys, &self.signer) + .sync_committee_messages( + slot_head.slot(), + beacon_block_root, + indices_with_pubkeys, + &self.signer, + ) .await { Ok(messages) => messages, @@ -2367,151 +2732,144 @@ impl Validator { slot_head.slot(), error, ); - return Ok(BTreeMap::new()); + return BTreeMap::new(); } }; - Ok(messages + messages .into_iter() - .zip(self.own_sync_committee_members()) + .zip(members) .flat_map(|(message, member)| { core::iter::zip(member.subnets, 0..) .filter(|(in_subnet, _)| *in_subnet) .map(move |(_, subcommittee_index)| (subcommittee_index, message)) }) - .pipe(group_into_btreemap)) + .pipe(group_into_btreemap) } async fn own_contributions_and_proofs( &self, + beacon_nodes: &BeaconNodes, slot_head: &SlotHead

, - ) -> Result>> { - let subcommittee_aggregators = self.own_subcommittee_aggregators(slot_head).await?; - - // TODO(Grandine Team): Parallelize. - // This used `into_par_iter` before, however, `build_sync_committee_contribution` - // uses blocking calls from `tokio`, and mixing `tokio` with `rayon` is tricky - // and invites all kinds of trouble. - let (triples, proofs): (Vec<_>, Vec<_>) = futures::stream::iter(subcommittee_aggregators) - .then(|(subcommittee_index, aggregators)| async move { - let contribution = self - .sync_committee_agg_pool - .best_subcommittee_contribution( + beacon_block_root: H256, + members: &[SyncCommitteeMember], + ) -> Vec> { + let subcommittee_aggregators = self.own_subcommittee_aggregators(slot_head, members).await; + + let fetches = subcommittee_aggregators.into_iter().map( + |(subcommittee_index, aggregators)| async move { + let contribution = beacon_nodes + .sync_committee_contribution( slot_head.slot(), - slot_head.beacon_block_root, subcommittee_index, + beacon_block_root, ) - .await; + .await + .inspect_err(|error| { + warn_with_peers!( + "failed to obtain a contribution for sync subcommittee \ + {subcommittee_index} in slot {}: {error:?}", + slot_head.slot(), + ); + }) + .ok(); (contribution, aggregators) - }) + }, + ); + + // Concurrent, so that one slow node does not spend the deadline of every subcommittee. + let (triples, proofs): (Vec<_>, Vec<_>) = join_all(fetches) + .await + .into_iter() .flat_map(|(contribution, aggregators)| { aggregators .into_iter() - .map(move |(aggregator, selection_proof)| { + .filter_map(move |(aggregator, selection_proof)| { let contribution_and_proof = ContributionAndProof { aggregator_index: aggregator.validator_index, - contribution, + contribution: contribution?, selection_proof, }; let triple = SigningTriple { message: SigningMessage::ContributionAndProof(contribution_and_proof), - signing_root: contribution_and_proof - .signing_root(&self.chain_config, &slot_head.beacon_state), + signing_root: contribution_and_proof.signing_root_from_fork_info( + &self.chain_source.chain_config, + slot_head.fork_info, + ), public_key: aggregator.public_key, }; - (triple, contribution_and_proof) + Some((triple, contribution_and_proof)) }) - .pipe(futures::stream::iter) }) - .unzip() - .await; + .unzip(); let signer_snapshot = self.signer.load(); let result = signer_snapshot - .sign_triples_without_slashing_protection( - triples, - Some(slot_head.beacon_state.as_ref().into()), - ) + .sign_triples_without_slashing_protection(triples, Some(slot_head.fork_info)) .await; let signatures = match result { Ok(signatures) => signatures, Err(error) => { warn_with_peers!("failed to sign contributions and proofs: {error:?}"); - return Ok(vec![]); + return vec![]; } }; - Ok(signatures + signatures .zip(proofs) .map(|(signature, message)| SignedContributionAndProof { message, signature: signature.into(), }) - .collect()) + .collect() } async fn own_payload_attestations( &self, slot_head: &SlotHead

, - block_with_root: &BlockWithRoot>, + data: PayloadAttestationData, own_members: &[PTCMember], - ) -> Result<&[PayloadAttestationMessage]> { - let beacon_block_root = block_with_root.root; - + ) -> &[PayloadAttestationMessage] { if let Some(own_payload_attestations) = self.own_payload_attestations.get() { - return Ok(own_payload_attestations); + return own_payload_attestations; } - let payload_present = self.controller.is_payload_present_timely(beacon_block_root); - - let blob_data_available = self - .controller - .indices_of_missing_data_columns(&block_with_root.block) - .is_empty(); + let snapshot = self.signer.load(); let (triples, other_data): (Vec<_>, Vec<_>) = tokio::task::block_in_place(|| { - let data = PayloadAttestationData { - slot: slot_head.slot(), - beacon_block_root, - payload_present, - blob_data_available, - }; - let doppelganger_protection = self .doppelganger_protection .as_deref() .map(DoppelgangerProtection::load); - // A validator may hold several PTC positions. `Pool::aggregate_messages` already duplicate one - // message into every position that validator holds, so sign only once per validator. own_members .iter() - .unique_by(|member| member.validator_index) + // A key removed at runtime must not fail the whole signing batch. + .filter(|member| snapshot.has_key(member.public_key)) .filter_map(|member| { if let Some(doppelganger_protection) = &doppelganger_protection - && !doppelganger_protection.is_validator_active(member.public_key) - { - info_with_peers!( - "Validator {:?} skipping attesting duty in slot {} \ - since not enough time has passed to ensure there are \ - no doppelganger validators participating on network. \ - Validator will start performing duties on slot {}.", - member.public_key, + && !doppelganger_protection_permits::

( + doppelganger_protection, + "payload attesting", slot_head.slot(), - doppelganger_protection.tracking_end_slot::

(member.public_key), - ); + member.public_key, + ) + { return None; } let triple = SigningTriple { message: SigningMessage::

::PayloadAttestation(data), - signing_root: data - .signing_root(&self.chain_config, &slot_head.beacon_state), + signing_root: SignForSingleFork::

::signing_root_from_fork_info( + &data, + &self.chain_source.chain_config, + slot_head.fork_info, + ), public_key: member.public_key, }; @@ -2520,58 +2878,45 @@ impl Validator { .unzip() }); - let snapshot = self.signer.load(); - let result = snapshot - .sign_triples( - triples, - slot_head.beacon_state.as_ref(), - self.slashing_protector.clone_arc(), - ) + .sign_triples_without_slashing_protection(triples, Some(slot_head.fork_info)) .await; let signatures = match result { Ok(signatures) => signatures, Err(error) => { warn_with_peers!("failed to sign payload attestations: {error:?}"); - return Ok(&[]); + return &[]; } }; self.own_payload_attestations - .get_or_try_init(|| { + .get_or_init(|| { let _timer = self.metrics.as_ref().map(|metrics| { metrics .validator_own_payload_attestations_init_times .start_timer() }); - let own_payload_attestations = signatures + signatures .zip(other_data) - .filter_map(|(signature, (data, member))| { - signature.map(|signature| PayloadAttestationMessage { - validator_index: member.validator_index, - data, - signature: signature.into(), - }) + .map(|(signature, (data, member))| PayloadAttestationMessage { + validator_index: member.validator_index, + data, + signature: signature.into(), }) - .collect(); - - Ok(own_payload_attestations) + .collect() }) - .map(Vec::as_slice) + .as_slice() } - fn own_sync_committee_members(&self) -> impl Iterator { - self.own_sync_committee_members.get().into_iter().flatten() - } - - async fn own_subcommittee_aggregators( + async fn own_subcommittee_aggregators<'members>( &self, slot_head: &SlotHead

, - ) -> Result>> { - let subcommittee_members = self - .own_sync_committee_members() + members: &'members [SyncCommitteeMember], + ) -> BTreeMap> { + let subcommittee_members = members + .iter() .flat_map(|member| { core::iter::zip(member.subnets, 0..) .filter(|(in_subnet, _)| *in_subnet) @@ -2595,18 +2940,18 @@ impl Validator { slot_head.slot(), error, ); - return Ok(BTreeMap::new()); + return BTreeMap::new(); } }; - Ok(proofs + proofs .into_iter() .zip(subcommittee_members) .filter_map(|(selection_proof, (subcommittee_index, member))| { let selection_proof = selection_proof?; Some((subcommittee_index, (member, selection_proof))) }) - .pipe(group_into_btreemap)) + .pipe(group_into_btreemap) } fn attested_in_current_slot(&self) -> bool { @@ -2615,6 +2960,8 @@ impl Validator { fn discard_previous_slot_attestations(&mut self) { self.own_singular_attestations.take(); + // Aggregation duties are per-slot; a leftover entry would request a stale aggregate. + self.own_aggregators.clear(); } fn payload_attested_in_current_slot(&self) -> bool { @@ -2625,13 +2972,17 @@ impl Validator { self.own_payload_attestations.take(); } - fn discard_old_proposer_preferences(&mut self, current_epoch: Epoch) { + async fn discard_old_proposer_preferences(&self, current_epoch: Epoch) { let current_epoch_start = misc::compute_start_slot_at_epoch::

(current_epoch); self.published_proposer_preferences + .lock() + .await .retain(|(_, proposal_slot, _)| *proposal_slot >= current_epoch_start); self.submitted_builder_preferences + .lock() + .await .retain(|(proposal_slot, _)| *proposal_slot >= current_epoch_start); } @@ -2645,8 +2996,7 @@ impl Validator { fn spawn_pruning(&self, current_slot: Slot) { let current_epoch = misc::compute_epoch_at_slot::

(current_slot); - let own_members = self.own_beacon_committee_members.clone_arc(); - let own_ptc_members = self.own_ptc_members.clone_arc(); + let own_duties = self.own_duties.clone_arc(); let slashing_protector = self.slashing_protector.clone_arc(); self.dedicated_executor_low_priority @@ -2656,8 +3006,12 @@ impl Validator { } let up_to_slot = misc::compute_start_slot_at_epoch::

(current_epoch); - own_members.prune(up_to_slot).await; - own_ptc_members.prune(up_to_slot).await; + own_duties + .beacon_committee_members + .prune::

(up_to_slot) + .await; + own_duties.proposer_duties.prune::

(up_to_slot).await; + own_duties.ptc_members.prune(up_to_slot).await; Ok::<_, AnyhowError>(()) }) .detach() @@ -2666,14 +3020,31 @@ impl Validator { fn spawn_update_beacon_committee_subscriptions( &self, wait_group: W, - beacon_state: Arc>, + beacon_nodes: &BeaconNodes, ) { let task = UpdateBeaconCommitteeSubscriptionsTask { - chain_config: self.chain_config.clone_arc(), - controller: self.controller.clone_arc(), - beacon_state, - own_beacon_committee_members: self.own_beacon_committee_members.clone_arc(), - subnet_service_tx: self.subnet_service_tx.clone(), + chain_config: self.chain_source.chain_config.clone_arc(), + own_duties: self.own_duties.clone_arc(), + proposer_configs: self.proposer_configs.clone_arc(), + beacon_nodes: beacon_nodes.clone(), + wait_group, + }; + + self.dedicated_executor_normal_priority + .spawn(task.run()) + .detach() + } + + fn spawn_prefetch_sync_committee_duties( + &self, + wait_group: W, + beacon_nodes: &BeaconNodes, + ) { + let task = PrefetchSyncCommitteeDutiesTask { + chain_config: self.chain_source.chain_config.clone_arc(), + current_slot: beacon_nodes.head().slot(), + own_duties: self.own_duties.clone_arc(), + beacon_nodes: beacon_nodes.clone(), wait_group, }; @@ -2682,22 +3053,38 @@ impl Validator { .detach() } - fn update_sync_committee_subscriptions(&mut self, beacon_state: &BeaconState

) -> Result<()> { - if let Some(post_altair_state) = beacon_state.post_altair() { - let own_public_keys = self.own_public_keys(); + async fn update_sync_committee_subscriptions( + &mut self, + beacon_nodes: &BeaconNodes, + ) -> Result<()> { + // Subscriptions are built from the sync committees a state carries. + let Some(beacon_state) = beacon_nodes.state() else { + return Ok(()); + }; - self.own_sync_committee_subscriptions - .build(post_altair_state, &own_public_keys)?; + let Some(post_altair_state) = beacon_state.post_altair() else { + return Ok(()); + }; - let current_epoch = accessors::get_current_epoch(beacon_state); + let own_public_keys = self.own_public_keys(); - if let Some(subscriptions) = self - .own_sync_committee_subscriptions - .take_epoch_subscriptions(current_epoch) - { - ToSubnetService::UpdateSyncCommitteeSubscriptions(current_epoch, subscriptions) - .send(&self.subnet_service_tx); - } + self.own_sync_committee_subscriptions + .build(post_altair_state, &own_public_keys)?; + + let current_epoch = accessors::get_current_epoch(beacon_state); + + let Some(subscriptions) = self + .own_sync_committee_subscriptions + .take_epoch_subscriptions(current_epoch) + else { + return Ok(()); + }; + + if let Err(error) = beacon_nodes + .subscribe_to_sync_committees(current_epoch, &subscriptions) + .await + { + warn_with_peers!("failed to update sync committee subscriptions: {error:?}"); } Ok(()) @@ -2706,37 +3093,61 @@ impl Validator { async fn update_subnet_subscriptions( &mut self, wait_group: &W, - slot_head: Option<&SlotHead

>, + tick: Tick, + beacon_nodes: Option<&BeaconNodes>, ) -> Result<()> { - if !self.controller.is_forward_synced() { + if let Some(controller) = self.chain_source.controller() + && !controller.is_forward_synced() + { return Ok(()); } - let beacon_state = match slot_head.map(|sh| sh.beacon_state.clone_arc()) { - Some(state) => state, - None => match self.controller.preprocessed_state_at_current_slot().await { - Ok(state) => state, - Err(error) => { - let is_too_many_empty_slots = matches!( - error.downcast_ref(), - Some(StateCacheError::StateFarBehind { .. }) - ); + let beacon_nodes = match beacon_nodes { + Some(beacon_nodes) => beacon_nodes.clone(), + None => { + // Duties must not fall back to the state of the disabled built-in node. + let Some(controller) = self.chain_source.controller().map(ArcExt::clone_arc) else { + return Ok(()); + }; + + let beacon_state = match controller.preprocessed_state_at_current_slot().await { + Ok(state) => state, + Err(error) => { + let is_too_many_empty_slots = matches!( + error.downcast_ref(), + Some(StateCacheError::StateFarBehind { .. }) + ); + + if !is_too_many_empty_slots { + warn_with_peers!( + "failed to obtain beacon state for current slot: {error}" + ); + } - if !is_too_many_empty_slots { - warn_with_peers!("failed to obtain beacon state for current slot: {error}"); + return Ok(()); } + }; - return Ok(()); - } - }, + let head = controller.head_block_root(); + + let slot_head = SlotHead::from_state( + self.chain_source.chain_config.clone_arc(), + beacon_state.slot(), + head.value, + beacon_state.as_ref(), + ); + + self.local_beacon_node(slot_head, beacon_state, wait_group)? + } }; - self.spawn_update_beacon_committee_subscriptions( - wait_group.clone(), - beacon_state.clone_arc(), - ); + if tick.is_start_of_slot() { + self.spawn_update_beacon_committee_subscriptions(wait_group.clone(), &beacon_nodes); + self.spawn_prefetch_sync_committee_duties(wait_group.clone(), &beacon_nodes); + } - self.update_sync_committee_subscriptions(&beacon_state) + self.update_sync_committee_subscriptions(&beacon_nodes) + .await } #[instrument(level = "debug", skip_all)] @@ -2745,22 +3156,25 @@ impl Validator { current_slot: Slot, own_validator_indices: &HashSet, ) { + let Some(controller) = self.chain_source.controller() else { + return; + }; + let current_epoch = misc::compute_epoch_at_slot::

(current_slot); - let last_finalized_state = self.controller.last_finalized_state().value; + let last_finalized_state = controller.last_finalized_state().value; let validator_custody_requirement = eip_7594::get_validator_custody_requirement( - &self.chain_config, + &self.chain_source.chain_config, &last_finalized_state, own_validator_indices, ); - let current_sampling_size: u64 = self - .controller + let current_sampling_size: u64 = controller .sampling_columns_count() .try_into() .expect("sampling size should be able to fit into u64"); - let Some(current_custody_requirements) = - current_sampling_size.checked_div(self.chain_config.columns_per_group::

()) + let Some(current_custody_requirements) = current_sampling_size + .checked_div(self.chain_source.chain_config.columns_per_group::

()) else { return; }; @@ -2769,10 +3183,9 @@ impl Validator { || self.last_cgc_update_epoch.is_none() { // Refresh data column subnets subscriptions in network globals and sampling columns fork choice store - ValidatorToP2p::UpdateDataColumnSubnets( + self.publish(ValidatorToP2p::UpdateDataColumnSubnets( validator_custody_requirement.max(current_custody_requirements), - ) - .send(&self.p2p_tx); + )); } self.last_cgc_update_epoch = Some(current_epoch); @@ -2780,6 +3193,7 @@ impl Validator { async fn handle_external_contributions_and_proofs( &self, + sync_committee_agg_pool: &SyncCommitteeAggPool, slot_head: SlotHead

, contributions_and_proofs: Vec>, ) -> Vec<(usize, AnyhowError)> { @@ -2790,8 +3204,7 @@ impl Validator { contribution_and_proof.message.contribution.slot == slot_head.slot() }) .map(|(index, contribution_and_proof)| async move { - let result = self - .sync_committee_agg_pool + let result = sync_committee_agg_pool .handle_external_contribution_and_proof(contribution_and_proof, Origin::Api) .await; @@ -2801,13 +3214,14 @@ impl Validator { .filter_map(|(index, contribution_and_proof, result)| async move { match result { Ok(_) => { - self.event_channels - .send_contribution_and_proof_event(contribution_and_proof); + if let Some(event_channels) = self.chain_source.event_channels() { + event_channels + .send_contribution_and_proof_event(contribution_and_proof); + } - ValidatorToP2p::PublishContributionAndProof(Box::new( + self.publish(ValidatorToP2p::PublishContributionAndProof(Box::new( contribution_and_proof, - )) - .send(&self.p2p_tx); + ))); None } @@ -2819,19 +3233,18 @@ impl Validator { } #[instrument(level = "debug", skip_all)] - fn refresh_signer_keys(&self) { + fn refresh_signer_keys(&self, current_slot: Slot) { let signer = self.signer.clone_arc(); - let head_state = self.controller.head_state().value; - let current_slot = self.controller.slot(); tokio::spawn(async move { signer.load_keys_from_web3signer().await; - signer.update_doppelganger_protection_pubkeys(&head_state, current_slot); + signer.update_doppelganger_protection_pubkeys(current_slot); }); } + #[expect(clippy::too_many_lines)] #[instrument(level = "debug", skip_all)] - async fn register_validators(&mut self, current_epoch: Epoch) { + async fn register_validators(&mut self, slot: Slot, current_epoch: Epoch) { if let Some(last_registration_epoch) = self.last_registration_epoch { let next_registration_epoch = last_registration_epoch .saturating_add(EPOCHS_PER_VALIDATOR_REGISTRATION_SUBMISSION); @@ -2841,32 +3254,47 @@ impl Validator { } } - let builder_api = self.builder_api.clone(); - let chain_config = self.chain_config.clone_arc(); + let builder_api = self.chain_source.builder_api().map(ArcExt::clone_arc); + let chain_config = self.chain_source.chain_config.clone_arc(); + // Builders bid over gossip from Gloas on, so no registration is wanted. + let use_builder = self.chain_source.registers_with_builder() + && chain_config.phase_at_epoch(current_epoch) < Phase::Gloas; + // Nodes without a builder reject relayed registrations, so registering is opted into. + let remote_nodes = match &self.chain_source.chain { + Chain::Remote(remote_beacon_nodes) if use_builder => { + Some(self.remote_nodes_at(remote_beacon_nodes, slot)) + } + _ => None, + }; let proposer_configs = self.proposer_configs.clone_arc(); let signer = self.signer.clone_arc(); - let prepared_proposer_indices = self.block_producer.get_prepared_proposer_indices().await; + let prepared_proposer_indices = match self.chain_source.block_producer() { + Some(block_producer) => block_producer.get_prepared_proposer_indices().await, + None => Vec::new(), + }; let registered_validators = self.registered_validators.clone(); - let subnet_service_tx = self.subnet_service_tx.clone(); + let subnet_service_tx = self.channels.subnet_service_tx().cloned(); tokio::spawn(async move { let signer_snapshot = signer.load(); let pubkeys = signer_snapshot.keys().copied().collect_vec(); - ToSubnetService::SetRegisteredValidators( - registered_validators - .values() - .flat_map(BTreeMap::keys) - .copied() - .chain(pubkeys.iter().copied()) - .collect(), - prepared_proposer_indices, - ) - .send(&subnet_service_tx); + if let Some(subnet_service_tx) = subnet_service_tx { + ToSubnetService::SetRegisteredValidators( + registered_validators + .values() + .flat_map(BTreeMap::keys) + .copied() + .chain(pubkeys.iter().copied()) + .collect(), + prepared_proposer_indices, + ) + .send(&subnet_service_tx); + } - let Some(builder_api) = builder_api.clone() else { + if !use_builder { return Ok(()); - }; + } // `ValidatorRegistrationV1` is deprecated in Gloas in favor of `ProposerPreferences`. if chain_config.phase_at_epoch(current_epoch) >= Phase::Gloas { @@ -2875,9 +3303,23 @@ impl Validator { let registrations = pubkeys .into_iter() - .map(|pubkey| { + .filter_map(|pubkey| { + // A remote node pays its own recipient for a local payload, so one made up + // here would only ever be paid by a builder. + let Some(fee_recipient) = proposer_configs.configured_fee_recipient(pubkey) + else { + debug_with_peers!( + "validator {pubkey:?} has no fee recipient configured; \ + it will not be registered with the builder", + ); + return None; + }; + + Some((pubkey, fee_recipient)) + }) + .map(|(pubkey, fee_recipient)| { Ok(ValidatorRegistrationV1 { - fee_recipient: proposer_configs.fee_recipient(pubkey), + fee_recipient, gas_limit: chain_config .gas_limit(proposer_configs.gas_limit(pubkey), current_epoch), timestamp: SystemTime::now() @@ -2910,8 +3352,9 @@ impl Validator { }) .collect_vec(); + // Every validator is registered at once. let signatures = signer_snapshot - .sign_triples_without_slashing_protection(triples, None) + .sign_triples_throttled_without_slashing_protection(triples, None) .await?; let signed_registrations = registrations @@ -2936,7 +3379,19 @@ impl Validator { // Do not submit requests in parallel. Doing so causes all of them to be timed out. for registrations in signed_registrations { - if let Err(error) = builder_api.register_validators::

(registrations).await { + if let Some(remote_nodes) = &remote_nodes + && let Err(error) = BeaconNodeApi::

::register_validators( + remote_nodes, + registrations.as_ref(), + ) + .await + { + warn_with_peers!("failed to relay validator batch: {error:?}"); + } + + if let Some(builder_api) = &builder_api + && let Err(error) = builder_api.register_validators::

(registrations).await + { warn_with_peers!("failed to register validator batch: {error}"); } } @@ -2992,7 +3447,11 @@ impl Validator { .map(|(index, auth)| { ( index, - builder_request_auth_triple::

(&self.chain_config, public_key, auth), + builder_request_auth_triple::

( + &self.chain_source.chain_config, + public_key, + auth, + ), ) }) .unzip(); @@ -3030,26 +3489,29 @@ impl Validator { .unwrap_or_default() } - #[expect(clippy::too_many_lines)] + // Builders learn the proposer's preferences ahead of its proposals. + // The request auths signed for them are reused when the block is produced. #[instrument(level = "debug", skip_all)] - fn submit_builder_preferences(&mut self, slot_head: &SlotHead

) { + async fn submit_builder_preferences(&self, beacon_nodes: &BeaconNodes) { let signer_snapshot = self.signer.load().clone_arc(); if signer_snapshot.no_keys() { return; } - let preferences = self - .own_upcoming_proposals(&signer_snapshot, &slot_head.beacon_state) - .into_iter() - .filter_map(|(proposal_slot, validator_index, pubkey)| { - if !self - .submitted_builder_preferences - .insert((proposal_slot, validator_index)) - { - return None; - } + let chain_config = self.chain_source.chain_config.clone_arc(); + let proposals = self.upcoming_proposals(beacon_nodes).await; + let submitted = self.submitted_builder_preferences.lock().await; + let preferences = proposals + .into_iter() + .filter(|(_, proposal_slot, _, _)| { + chain_config.phase_at_slot::

(*proposal_slot) >= Phase::Gloas + }) + .filter(|(_, proposal_slot, validator_index, _)| { + !submitted.contains(&(*proposal_slot, *validator_index)) + }) + .filter_map(|(_, proposal_slot, validator_index, pubkey)| { match self.proposer_configs.resolved_builder_config(pubkey) { Ok(builder_config) => { Some(builder_config.builders.into_iter().map(move |entry| { @@ -3058,7 +3520,7 @@ impl Validator { slot: proposal_slot, }; - (pubkey, entry, auth) + (validator_index, pubkey, entry, auth) })) } Err(error) => { @@ -3072,19 +3534,21 @@ impl Validator { .flatten() .collect_vec(); + drop(submitted); + if preferences.is_empty() { return; } - let chain_config = self.chain_config.clone_arc(); - let payload_builder_api = self.block_producer.payload_builder_api().clone_arc(); + let current_slot = beacon_nodes.head().slot(); + let beacon_nodes = beacon_nodes.clone(); let signed_auths = self.signed_builder_request_auths.clone_arc(); - let state_slot = slot_head.slot(); + let submitted = self.submitted_builder_preferences.clone_arc(); tokio::spawn(async move { let triples = preferences .iter() - .map(|(pubkey, _, auth)| { + .map(|(_, pubkey, _, auth)| { builder_request_auth_triple::

(&chain_config, *pubkey, auth) }) .collect_vec(); @@ -3103,9 +3567,9 @@ impl Validator { { let mut signed_auths = signed_auths.lock().await; - signed_auths.retain(|(_, proposal_slot, _), _| *proposal_slot > state_slot); + signed_auths.retain(|(_, proposal_slot, _), _| *proposal_slot > current_slot); - for ((pubkey, _, auth), signature) in preferences.iter().zip(&signatures) { + for ((_, pubkey, _, auth), signature) in preferences.iter().zip(&signatures) { signed_auths.insert( (*pubkey, auth.slot, auth.data.as_bytes().to_vec()), *signature, @@ -3113,135 +3577,72 @@ impl Validator { } } - let payload_builder_api = &payload_builder_api; - - let submissions = preferences.into_iter().zip(signatures).map( - |((pubkey, entry, message), signature)| async move { - let auth = SignedBuilderRequestAuth { message, signature }; - - let result = payload_builder_api - .submit_builder_preferences::

( - &entry.url, - auth, - pubkey, - entry.max_execution_payment, - ) - .await; - - (entry.url, pubkey, result) - }, - ); - - let mut failures = Vec::<(BuilderUrl, usize, AnyhowError)>::new(); - - for (url, pubkey, result) in join_all(submissions).await { - let Err(error) = result else { - continue; - }; - - debug_with_peers!( - "failed to submit builder preferences to {url} for validator {pubkey:?}: \ - {error:?}", - ); - - match failures - .iter_mut() - .find(|(failed_url, _, _)| *failed_url == url) - { - Some((_, count, _)) => *count = count.saturating_add(1), - None => failures.push((url, 1, error)), - } - } + let entries = preferences + .iter() + .zip(signatures) + .map( + |((_, pubkey, entry, message), signature)| BuilderPreferencesEntry { + proposer_pubkey: *pubkey, + url: entry.url.clone(), + auth: SignedBuilderRequestAuth { + message: message.clone(), + signature, + }, + max_execution_payment: entry.max_execution_payment, + }, + ) + .collect_vec(); - for (url, count, error) in failures { - warn_with_peers!( - "failed to submit builder preferences to {url} for {count} proposals: \ - {error:?}", - ); + // Only accepted preferences are left alone; the rest are tried again next slot. + if let Err(error) = beacon_nodes.submit_builder_preferences(&entries).await { + warn_with_peers!("failed to submit builder preferences: {error:?}"); + return; } - }); - } - - fn own_upcoming_proposals( - &self, - signer: &Snapshot, - beacon_state: &BeaconState

, - ) -> Vec<(Slot, ValidatorIndex, PublicKeyBytes)> { - let pubkeys_by_index = signer - .keys() - .filter_map(|pubkey| { - let validator_index = accessors::index_of_public_key(beacon_state, pubkey)?; - Some((validator_index, *pubkey)) - }) - .collect::>(); - - let validator_indices = pubkeys_by_index.keys().copied().collect::>(); - - accessors::get_upcoming_proposal_slots(&self.chain_config, beacon_state, &validator_indices) - .filter_map(|(proposal_slot, validator_index)| { - let pubkey = pubkeys_by_index.get(&validator_index).copied()?; - Some((proposal_slot, validator_index, pubkey)) - }) - .collect() - } - - fn forward_block_to_builder(&self, builder_url: BuilderUrl, block: Arc>) { - let payload_builder_api = self.block_producer.payload_builder_api().clone_arc(); - tokio::spawn(async move { - if let Err(error) = payload_builder_api - .submit_signed_beacon_block(&builder_url, &block) - .await - { - warn_with_peers!("failed to forward block to builder {builder_url}: {error:?}"); - } + submitted.lock().await.extend( + preferences + .iter() + .map(|(validator_index, _, _, auth)| (auth.slot, *validator_index)), + ); }); } + #[expect(clippy::too_many_lines)] #[instrument(level = "debug", skip_all)] - fn publish_proposer_preferences(&mut self, slot_head: &SlotHead

) { + async fn publish_proposer_preferences(&self, beacon_nodes: &BeaconNodes) { let signer_snapshot = self.signer.load().clone_arc(); if signer_snapshot.no_keys() { return; } - let beacon_state = slot_head.beacon_state.clone_arc(); - let beacon_block_root = slot_head.beacon_block_root; - let mut dependent_roots = HashMap::>::new(); + let chain_config = self.chain_source.chain_config.clone_arc(); - let preferences = self - .own_upcoming_proposals(&signer_snapshot, &beacon_state) - .into_iter() - .filter_map(|(proposal_slot, validator_index, pubkey)| { - let proposal_epoch = misc::compute_epoch_at_slot::

(proposal_slot); + let slot_head = beacon_nodes.head(); - let dependent_root = dependent_roots - .entry(proposal_epoch) - .or_insert_with(|| { - let dependent_root = self - .controller - .shuffling_dependent_root(beacon_block_root, proposal_epoch); + let proposals = self.upcoming_proposals(beacon_nodes).await; - if dependent_root.is_none() { - warn_with_peers!( - "failed to compute shuffling dependent root for proposer \ - preferences (epoch {proposal_epoch})" - ); - } + let published = self.published_proposer_preferences.lock().await; - dependent_root - }) - .as_ref() - .copied()?; + let preferences = proposals + .into_iter() + .filter(|(_, proposal_slot, _, _)| { + chain_config.phase_at_slot::

(*proposal_slot) >= Phase::Gloas + }) + .filter(|(dependent_root, proposal_slot, validator_index, _)| { + !published.contains(&(*dependent_root, *proposal_slot, *validator_index)) + }) + .filter_map(|(dependent_root, proposal_slot, validator_index, pubkey)| { + let proposal_epoch = misc::compute_epoch_at_slot::

(proposal_slot); - if self.published_proposer_preferences.contains(&( - dependent_root, - proposal_slot, - validator_index, - )) { + let Some(fee_recipient) = self.proposer_configs.configured_fee_recipient(pubkey) + else { + debug_with_peers!( + "validator {validator_index} has no fee recipient configured; \ + no proposer preferences will be published for slot {proposal_slot}", + ); return None; - } + }; Some(( pubkey, @@ -3249,46 +3650,40 @@ impl Validator { dependent_root, proposal_slot, validator_index, - fee_recipient: self.proposer_configs.fee_recipient(pubkey), - target_gas_limit: self - .chain_config + fee_recipient, + target_gas_limit: chain_config .gas_limit(self.proposer_configs.gas_limit(pubkey), proposal_epoch), }, )) }) .collect_vec(); + drop(published); + if preferences.is_empty() { return; } - self.published_proposer_preferences - .extend(preferences.iter().map(|(_, pref)| { - ( - pref.dependent_root, - pref.proposal_slot, - pref.validator_index, - ) - })); - - let chain_config = self.chain_config.clone_arc(); - let controller = self.controller.clone_arc(); - let p2p_tx = self.p2p_tx.clone(); + let fork_info = slot_head.fork_info; + let beacon_nodes = beacon_nodes.clone(); + let published = self.published_proposer_preferences.clone_arc(); tokio::spawn(async move { let triples = preferences .iter() - .map(|(pubkey, pref)| SigningTriple { + .map(|(pubkey, pref)| SigningTriple::

{ message: SigningMessage::ProposerPreferences(*pref), - signing_root: pref.signing_root(&chain_config, beacon_state.as_ref()), + signing_root: SignForSingleFork::

::signing_root_from_fork_info( + pref, + &chain_config, + fork_info, + ), public_key: *pubkey, }) .collect_vec(); - let fork_info = Some(beacon_state.as_ref().into()); - let signatures = match signer_snapshot - .sign_triples_without_slashing_protection(triples, fork_info) + .sign_triples_without_slashing_protection(triples, Some(fork_info)) .await { Ok(signatures) => signatures, @@ -3298,28 +3693,145 @@ impl Validator { } }; - // Publish each signed preference - for ((_, pref), signature) in preferences.into_iter().zip(signatures) { - debug_with_peers!( - "publishing proposer preferences for (validator: {}, slot: {})", - pref.validator_index, - pref.proposal_slot - ); - - let signed_preferences = Arc::new(SignedProposerPreferences { - message: pref, - signature: signature.into(), - }); + let signed_preferences = preferences + .into_iter() + .zip(signatures) + .map(|((_, pref), signature)| { + debug_with_peers!( + "publishing proposer preferences for (validator: {}, slot: {})", + pref.validator_index, + pref.proposal_slot + ); - // Pass into own fork-choice store so bids for this slot pass the - // `accepted_proposer_preferences` gate in validate_execution_payload_bid. - controller.on_own_proposer_preferences(signed_preferences.clone_arc()); + Arc::new(SignedProposerPreferences { + message: pref, + signature: signature.into(), + }) + }) + .collect_vec(); - ValidatorToP2p::PublishProposerPreferences(signed_preferences).send(&p2p_tx); + if let Err(error) = beacon_nodes + .publish_proposer_preferences(&signed_preferences) + .await + { + warn_with_peers!("failed to publish proposer preferences: {error:?}"); + return; } + + published + .lock() + .await + .extend(signed_preferences.iter().map(|signed| { + let ProposerPreferences { + dependent_root, + proposal_slot, + validator_index, + .. + } = signed.message; + + (dependent_root, proposal_slot, validator_index) + })); }); } + // Own proposals ahead, each with the dependent root its duty was looked up under. + async fn upcoming_proposals( + &self, + beacon_nodes: &BeaconNodes, + ) -> Vec<(H256, Slot, ValidatorIndex, PublicKeyBytes)> { + let chain_config = &self.chain_source.chain_config; + let slot_head = beacon_nodes.head(); + + match beacon_nodes.state() { + Some(beacon_state) => self.upcoming_proposals_from_state(slot_head, beacon_state), + None => { + let upcoming = self + .own_duties + .proposer_duties + .upcoming(slot_head.slot()) + .await; + + let mut proposals = Vec::with_capacity(upcoming.len()); + + // Only the root the duties are now fetched under is worth signing for; another + // one is left from before a reorg, and the nodes would not take it. + for (dependent_root, proposal_slot, validator_index, pubkey) in upcoming { + let epoch = misc::compute_epoch_at_slot::

(proposal_slot); + let dependent_epoch = proposer_dependent_epoch(chain_config, epoch); + + let current = self + .own_duties + .beacon_committee_members + .cached_dependent_root(dependent_epoch) + .await; + + if current == Some(dependent_root) { + proposals.push((dependent_root, proposal_slot, validator_index, pubkey)); + } + } + + proposals + } + } + } + + fn upcoming_proposals_from_state( + &self, + slot_head: &SlotHead

, + beacon_state: &Arc>, + ) -> Vec<(H256, Slot, ValidatorIndex, PublicKeyBytes)> { + let Some(controller) = self.chain_source.controller() else { + return vec![]; + }; + + let beacon_block_root = slot_head.beacon_block_root; + + let pubkeys_by_index = self + .signer + .load() + .keys() + .filter_map(|pubkey| { + let validator_index = accessors::index_of_public_key(beacon_state, pubkey)?; + Some((validator_index, *pubkey)) + }) + .collect::>(); + + let validator_indices = pubkeys_by_index.keys().copied().collect::>(); + let mut dependent_roots = HashMap::>::new(); + + accessors::get_upcoming_proposal_slots( + &self.chain_source.chain_config, + beacon_state, + &validator_indices, + ) + .filter_map(|(proposal_slot, validator_index)| { + let proposal_epoch = misc::compute_epoch_at_slot::

(proposal_slot); + + let dependent_root = dependent_roots + .entry(proposal_epoch) + .or_insert_with(|| { + let dependent_root = + controller.shuffling_dependent_root(beacon_block_root, proposal_epoch); + + if dependent_root.is_none() { + warn_with_peers!( + "failed to compute shuffling dependent root for proposer \ + preferences (epoch {proposal_epoch})" + ); + } + + dependent_root + }) + .as_ref() + .copied()?; + + let pubkey = pubkeys_by_index.get(&validator_index).copied()?; + + Some((dependent_root, proposal_slot, validator_index, pubkey)) + }) + .collect_vec() + } + async fn track_collection_metrics(&self) { if let Some(metrics) = self.metrics.as_ref() { let type_name = tynm::type_name::(); @@ -3338,7 +3850,7 @@ impl Validator { module_path!(), &type_name, "own_beacon_committee_members", - self.own_beacon_committee_members.len(), + self.own_duties.beacon_committee_members.len(), ); metrics.set_collection_length( @@ -3351,14 +3863,23 @@ impl Validator { .unwrap_or(0), ); + metrics.set_collection_length( + module_path!(), + &type_name, + "own_proposer_duties", + self.own_duties.proposer_duties.len(), + ); + metrics.set_collection_length( module_path!(), &type_name, "own_ptc_members", - self.own_ptc_members.len(), + self.own_duties.ptc_members.len(), ); - self.block_producer.track_collection_metrics().await; + if let Some(block_producer) = self.chain_source.block_producer() { + block_producer.track_collection_metrics().await; + } } if let Some(validator_statistics) = self.validator_statistics.as_ref() { @@ -3366,62 +3887,51 @@ impl Validator { } } - // `SlotHead` for `head` with its state advanced to `slot`. - // `SlotHead::slot` comes from the state, so an older head must be advanced. - async fn slot_head_at(&self, head: &ChainLink

, slot: Slot) -> Result> { - let beacon_state = if head.slot() < slot { - let controller = self.controller.clone_arc(); - let block_root = head.block_root; - - tokio::task::spawn_blocking(move || { - controller.preprocessed_state_post_block_blocking(block_root, slot) - }) - .await?? - } else { - self.controller.state_by_chain_link(head) - }; - - Ok(SlotHead { - config: self.chain_config.clone_arc(), - beacon_block_root: head.block_root, - beacon_state, - }) - } - // Waits for the head to become fully validated or for a late current slot block. - // Returns the head and whether the duty uses its full node. + // For the built-in beacon node it also settles whether the duty uses the head's full node. // That is the node whose payload status the wait checked. // Returns `None` if the head is still optimistic when the wait times out, // or if the block itself turns out to be invalid. + #[expect(clippy::too_many_lines)] #[instrument(level = "debug", skip_all)] async fn wait_for_fully_validated_head( &self, - slot_head: SlotHead

, + beacon_nodes: BeaconNodes, + wait_group: &W, + tick: Tick, duty: HeadDuty, - ) -> Option<(SlotHead

, bool)> { + ) -> Option> { const BLOCK_EVENT_WAIT_TIMEOUT: Duration = Duration::from_secs(1); const PAYLOAD_STATUS_POLL_INTERVAL: Duration = Duration::from_millis(50); + // A remote node serves its head only once it reports it as not optimistic. + let BeaconNodes::Local(node) = beacon_nodes else { + return Some(beacon_nodes); + }; + + let controller = self.chain_source.controller()?; + let event_channels = self.chain_source.event_channels()?; + let context = self.local_context.as_ref().map(ArcExt::clone_arc)?; + // The duty's own slot, not fork choice's. // Fork choice may already be a slot ahead if this tick ran late. - let slot = slot_head.slot(); - let head_block_slot = slot_head.beacon_state.latest_block_header().slot; + let slot = node.head().slot(); + let head_root = node.head().beacon_block_root; + let head_block_slot = node.state().latest_block_header().slot; - // Decided once: if a late block replaces `slot_head` as fork choice's head, - // `slot_head` must still be checked on the node the duty was going to use. - let mut full = - self.duty_uses_full_node(slot_head.beacon_block_root, head_block_slot, slot, duty); + // Decided once: if a late block replaces the head as fork choice's head, + // the old head must still be checked on the node the duty was going to use. + let mut full = duty_uses_full_node(controller, head_root, head_block_slot, slot, duty); - let is_optimistic = self - .controller - .node_payload_status(slot_head.beacon_block_root, full) + let is_optimistic = controller + .node_payload_status(head_root, full) .is_some_and(PayloadStatus::is_optimistic); - if !is_optimistic && !self.should_wait_for_late_block() { - return Some((slot_head, full)); + if !is_optimistic && !self.should_wait_for_late_block(tick) { + return Some(BeaconNodes::Local(node.with_full(full))); } - let mut block_events = self.event_channels.receiver_for(Topic::Block); + let mut block_events = event_channels.receiver_for(Topic::Block); // Post-Gloas blocks do not get a block event when they become fully validated. // Polling catches that. @@ -3434,7 +3944,7 @@ impl Validator { _ = poll.tick() => {} } - let head = self.controller.head().value; + let head = controller.head().value; // An attestation may move to an older head, like the Beacon API does. // Fork choice falls back to one when the payload under the head is rejected. @@ -3443,23 +3953,29 @@ impl Validator { HeadDuty::Attest => head.slot() <= slot, }; - if head.block_root != slot_head.beacon_block_root && head_usable { + if head.block_root != head_root && head_usable { let head_full = - self.duty_uses_full_node(head.block_root, head.slot(), slot, duty); + duty_uses_full_node(controller, head.block_root, head.slot(), slot, duty); - if self - .controller + if controller .node_payload_status(head.block_root, head_full) .is_some_and(PayloadStatus::is_valid) { - match self.slot_head_at(&head, slot).await { - Ok(new_slot_head) => { + match self.slot_head_at(controller, &head, slot).await { + Ok((slot_head, beacon_state)) => { debug_with_peers!( "fork choice head changed (new head: {:?})", head.block_root, ); - break HeadWait::Changed(new_slot_head, head_full); + let new_node = LocalBeaconNode::new( + context.clone_arc(), + slot_head, + beacon_state, + wait_group.clone(), + ); + + break HeadWait::Changed(new_node.with_full(head_full)); } Err(error) => warn_with_peers!( "failed to advance new head state to slot {slot}: {error:?}", @@ -3472,14 +3988,12 @@ impl Validator { continue; } - let root = slot_head.beacon_block_root; - // A rejected payload of its own only moves the duty to its empty node. // `Invalid` there means the block itself is dead. - let status = match self.controller.node_payload_status(root, full) { + let status = match controller.node_payload_status(head_root, full) { Some(status) if full && status.is_invalid() => { full = false; - self.controller.node_payload_status(root, false) + controller.node_payload_status(head_root, false) } status => status, }; @@ -3500,49 +4014,35 @@ impl Validator { .await; match result { - Ok(HeadWait::Changed(new_slot_head, new_full)) => Some((new_slot_head, new_full)), - Ok(HeadWait::Validated) => Some((slot_head, full)), + Ok(HeadWait::Changed(new_node)) => Some(BeaconNodes::Local(new_node)), + Ok(HeadWait::Validated) => Some(BeaconNodes::Local(node.with_full(full))), Ok(HeadWait::Invalidated) => None, Err(error) => { if is_optimistic { - debug_with_peers!( - "timed out while waiting for chain head to become fully validated: {error:?}", + warn_with_peers!( + "validator cannot perform its duties because \ + chain head has not been fully verified by an execution engine", ); None } else { debug_with_peers!( "timed out while waiting for current slot block to be processed: {error:?}", ); - Some((slot_head, full)) + Some(BeaconNodes::Local(node.with_full(full))) } } } } - // Whether a duty at `slot` uses the full node of this block. - // That decides which payload the duty relies on. - // Proposals use the node fork choice builds on, the same choice as the block producer. - // Attestations use the node fork choice resolves the head to. - // A block from `slot` itself is attested on its empty node. - fn duty_uses_full_node( - &self, - block_root: H256, - block_slot: Slot, - slot: Slot, - duty: HeadDuty, - ) -> bool { - let (head_root, head_payload_status) = self.controller.head_root_with_payload_status(); - - head_root == block_root - && match duty { - HeadDuty::Propose => self.controller.snapshot().should_build_on_full(slot), - HeadDuty::Attest => block_slot < slot && head_payload_status == PAYLOAD_STATUS_FULL, - } - } - - fn should_wait_for_late_block(&self) -> bool { - !self.validator_config.disable_wait_for_late_blocks - && self.controller.has_current_slot_blocks_in_processing() + fn should_wait_for_late_block(&self, tick: Tick) -> bool { + // From the aggregate on, the slot's attestations are already signed, which a late + // block cannot change. + tick.is_before_aggregating_interval() + && !self.validator_config.disable_wait_for_late_blocks + && self + .chain_source + .controller() + .is_some_and(|controller| controller.has_current_slot_blocks_in_processing()) } } @@ -3554,13 +4054,76 @@ enum HeadDuty { } // Outcome of `Validator::wait_for_fully_validated_head`. -enum HeadWait { - // The new head and whether the duty uses its full node. - Changed(SlotHead

, bool), +enum HeadWait { + // The new head, with whether the duty uses its full node. + Changed(LocalBeaconNode), Validated, Invalidated, } +// Whether a duty at `slot` uses the full node of this block. +// That decides which payload the duty relies on. +// Proposals use the node fork choice builds on, the same choice as the block producer. +// Attestations use the node fork choice resolves the head to. +// A block from `slot` itself is attested on its empty node. +fn duty_uses_full_node( + controller: &ApiController, + block_root: H256, + block_slot: Slot, + slot: Slot, + duty: HeadDuty, +) -> bool { + let (head_root, head_payload_status) = controller.head_root_with_payload_status(); + + head_root == block_root + && match duty { + HeadDuty::Propose => controller.snapshot().should_build_on_full(slot), + HeadDuty::Attest => block_slot < slot && head_payload_status == PAYLOAD_STATUS_FULL, + } +} + +fn doppelganger_protection_permits( + doppelganger_protection: &DoppelgangerSnapshot, + duty: &str, + slot: Slot, + public_key: PublicKeyBytes, +) -> bool { + if doppelganger_protection.is_validator_active(public_key) { + return true; + } + + let Some(tracking_end_slot) = doppelganger_protection.tracking_end_slot::

(public_key) else { + info_with_peers!( + "Validator {public_key:?} skipping {duty} duty in slot {slot} \ + since doppelganger checks for it have not started yet.", + ); + + return false; + }; + + info_with_peers!( + "Validator {public_key:?} skipping {duty} duty in slot {slot} \ + since not enough time has passed to ensure there are \ + no doppelganger validators participating on network. \ + Validator will start performing duties on slot {tracking_end_slot}.", + ); + + false +} + +fn aggregated_by( + aggregate: &Attestation

, + position_in_committee: usize, +) -> Option { + match aggregate { + Attestation::Phase0(aggregate) => aggregate.aggregation_bits.get(position_in_committee), + Attestation::Electra(aggregate) => aggregate.aggregation_bits.get(position_in_committee), + Attestation::Gloas(aggregate) => aggregate.aggregation_bits.get(position_in_committee), + Attestation::Single(_) => None, + } + .map(|bit| *bit) +} + // Use `BTreeMap` to make grouping deterministic for snapshot testing. // There is no equivalent of `Itertools::into_group_map` that collects into a `BTreeMap`. // See . @@ -3575,7 +4138,7 @@ fn group_into_btreemap(pairs: impl IntoIterator) -> BT } fn builder_request_auth_triple( - chain_config: &ChainConfig, + chain_config: &Config, public_key: PublicKeyBytes, auth: &BuilderRequestAuth, ) -> SigningTriple<'static, P> { diff --git a/validator/src/validator_config.rs b/validator/src/validator_config.rs index 2dc43d177..0bbff7556 100644 --- a/validator/src/validator_config.rs +++ b/validator/src/validator_config.rs @@ -1,4 +1,4 @@ -use std::{path::PathBuf, sync::Arc}; +use std::sync::Arc; use derivative::Derivative; use keymanager::{BuilderSettings, ValidatorDefinitionsWithStorage}; @@ -15,12 +15,9 @@ pub struct ValidatorConfig { pub graffiti: Vec, #[derivative(Default(value = "32"))] pub max_empty_slots: u64, - pub suggested_fee_recipient: ExecutionAddress, + pub suggested_fee_recipient: Option, pub default_gas_limit: Option, pub builder_settings: BuilderSettings, - pub keystore_storage_password_file: Option, - #[derivative(Default(value = "true"))] - pub backfill_custody_groups: bool, pub custody_mode: CustodyMode, pub disable_wait_for_late_blocks: bool, /// The `validators.yml` definitions, shared with the Keymanager API so runtime settings changes