From 3d628de87dc4eb686324155bf54b1ae9c7a31543 Mon Sep 17 00:00:00 2001 From: ctrlc03 <93448202+ctrlc03@users.noreply.github.com> Date: Mon, 7 Sep 2026 09:55:26 +0100 Subject: [PATCH 1/6] fix: liveness bugs --- Cargo.lock | 1 + agent/INVARIANTS.md | 10 + agent/flow-trace/00_INDEX.md | 1 + agent/flow-trace/04_DKG_AND_COMPUTATION.md | 11 + .../flow-trace/05_FAILURE_REFUND_SLASHING.md | 11 +- .../06_DEACTIVATION_AND_COMPLETION.md | 19 +- .../src/committee_finalization/actor.rs | 220 ++++++++++++- .../effects/prove_plaintext.rs | 4 + .../effects/verify_decryption_shares.rs | 4 + .../src/public_key_aggregation/actor.rs | 40 ++- .../effects/aggregate_dkg_proofs.rs | 6 +- .../effects/aggregate_public_key.rs | 17 + .../src/public_key_aggregation/effects/mod.rs | 1 + .../effects/node_proof_deadline.rs | 72 +++++ .../effects/verify_key_proofs.rs | 8 +- .../src/public_key_aggregation/handlers.rs | 52 +++- .../node_proof_timeout.rs | 73 +++++ .../src/public_key_aggregation/tests/mod.rs | 1 + .../tests/node_proof_deadline.rs | 138 +++++++++ .../src/public_key_aggregation/transitions.rs | 3 + crates/ciphernode-builder/src/ciphernode.rs | 33 ++ .../src/ciphernode_builder.rs | 28 +- crates/cli/src/cli.rs | 6 + crates/cli/src/helpers/telemetry.rs | 37 ++- crates/cli/src/nodes.rs | 10 +- crates/cli/src/nodes_down.rs | 5 +- crates/cli/src/nodes_restart.rs | 5 +- crates/cli/src/nodes_start.rs | 5 +- crates/cli/src/nodes_status.rs | 5 +- crates/cli/src/nodes_stop.rs | 5 +- crates/cli/src/start.rs | 22 +- crates/entrypoint/src/nodes/client.rs | 34 +- crates/entrypoint/src/nodes/daemon.rs | 5 +- crates/entrypoint/src/nodes/down.rs | 4 +- crates/entrypoint/src/nodes/nodes.rs | 5 + .../entrypoint/src/nodes/process_manager.rs | 126 +++++++- crates/entrypoint/src/nodes/restart.rs | 4 +- crates/entrypoint/src/nodes/start.rs | 4 +- crates/entrypoint/src/nodes/status.rs | 4 +- crates/entrypoint/src/nodes/stop.rs | 4 +- crates/entrypoint/src/nodes/up.rs | 2 + crates/events/src/eventbus.rs | 25 +- crates/events/src/eventstore_router.rs | 8 +- .../interfold_event/publish_document/mod.rs | 9 +- .../events/src/request_router_checkpoint.rs | 5 + .../src/snapshot_buffer/timelock_queue.rs | 23 +- crates/events/src/store_keys.rs | 23 +- crates/evm/src/chain_gateway/actor.rs | 30 +- crates/evm/src/chain_gateway/tests.rs | 33 ++ crates/evm/src/chain_reader/actor.rs | 12 +- crates/evm/src/event_decoding/catalog.rs | 72 +++++ crates/evm/src/event_router.rs | 4 +- crates/evm/src/helpers.rs | 21 +- crates/evm/src/randomness_provider/actor.rs | 218 ++++++++++++- crates/evm/src/repo.rs | 43 +-- .../keyshare/src/threshold_keyshare/actor.rs | 11 +- .../effects/calculate_decryption_key.rs | 16 +- .../effects/generate_threshold_share.rs | 3 + .../threshold_keyshare/effects/recovery.rs | 24 +- .../effects/verify_threshold_shares.rs | 35 +++ .../keyshare/src/threshold_keyshare/tests.rs | 292 +++++++++++++++++- crates/logger/src/logger.rs | 8 +- crates/multithread/src/multithread.rs | 8 +- crates/net/src/document_publishing/actor.rs | 35 ++- crates/net/src/document_publishing/effects.rs | 91 +++++- .../net/src/document_publishing/handlers.rs | 64 +++- .../net/src/document_publishing/tests/mod.rs | 1 + .../document_publishing/tests/publishing.rs | 204 ++++++++++++ .../net/src/document_publishing/workflow.rs | 24 ++ crates/net/src/event_buffer/tests.rs | 5 - crates/net/src/event_translation/actor.rs | 3 +- crates/net/src/events.rs | 17 +- crates/net/src/net_interface.rs | 19 +- .../src/network_sync/effects/fetch_history.rs | 27 ++ crates/net/src/network_sync/handlers.rs | 8 + crates/net/src/network_sync/tests.rs | 42 +++ crates/net/src/network_sync/workflow.rs | 9 + crates/request/src/repo.rs | 1 + crates/request/src/routing/actor.rs | 53 ++++ .../src/routing/effects/build_context.rs | 14 +- .../request/src/routing/effects/snapshot.rs | 2 + crates/request/src/routing/handlers.rs | 87 +++++- crates/request/src/routing/tests.rs | 231 +++++++++++++- crates/request/src/routing/workflow.rs | 11 +- crates/request/src/routing/workflow_tests.rs | 36 ++- crates/slashing/Cargo.toml | 3 +- .../transitions/incoming_accusations.rs | 124 +++++++- .../transitions/initiate_accusation.rs | 12 +- .../transitions/reverify_proofs.rs | 16 +- .../src/accusation_voting/transitions/vote.rs | 57 +++- .../workflow_tests/voting.rs | 97 ++++++ .../src/commitment_consistency/actor.rs | 51 ++- .../src/commitment_consistency/workflow.rs | 73 ++++- .../commitment_consistency/workflow_tests.rs | 242 +++++++++++++++ .../src/commitment_consistency_checker_ext.rs | 51 ++- crates/slashing/src/lib.rs | 3 + crates/slashing/src/repo.rs | 19 ++ crates/test-helpers/src/ciphernode_system.rs | 1 + crates/zk-prover/src/actor_system.rs | 6 +- crates/zk-prover/src/error.rs | 3 + .../src/node_proof_aggregation/actor.rs | 140 +++++++++ .../src/node_proof_aggregation/effects.rs | 50 ++- crates/zk-prover/src/proof_request/actor.rs | 64 +++- .../src/proof_request/actor_tests.rs | 257 ++++++++++++++- .../effects/decryption_key_proofs.rs | 65 +++- .../src/proof_request/effects/dkg_proofs.rs | 81 ++++- .../effects/encryption_key_result.rs | 17 +- crates/zk-prover/src/proof_request/state.rs | 4 + .../src/proof_request/workflow_tests.rs | 1 + .../src/proof_verification/effects.rs | 4 +- crates/zk-prover/src/prover.rs | 134 +++++++- packages/interfold-dashboard/.env.example | 4 +- packages/interfold-dashboard/README.md | 4 +- packages/interfold-dashboard/src/Operator.tsx | 7 +- packages/interfold-dashboard/src/lib/chain.ts | 10 + tests/integration/fns.sh | 19 +- tests/integration/test.sh | 22 +- 117 files changed, 4294 insertions(+), 299 deletions(-) create mode 100644 crates/aggregator/src/public_key_aggregation/effects/node_proof_deadline.rs create mode 100644 crates/aggregator/src/public_key_aggregation/node_proof_timeout.rs create mode 100644 crates/aggregator/src/public_key_aggregation/tests/node_proof_deadline.rs create mode 100644 crates/slashing/src/repo.rs diff --git a/Cargo.lock b/Cargo.lock index 293d3106b5..95fd787e19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4399,6 +4399,7 @@ dependencies = [ "alloy", "anyhow", "async-trait", + "bincode", "chrono", "e3-data", "e3-events", diff --git a/agent/INVARIANTS.md b/agent/INVARIANTS.md index ac3ff39dac..6b3497fc54 100644 --- a/agent/INVARIANTS.md +++ b/agent/INVARIANTS.md @@ -634,6 +634,16 @@ design citation alone does not establish current runtime behavior. router's `on_event` path must not do synchronous store reads. — `flow-trace/06` - A well-formed `E3Requested` with an unsupported committee-size/preset enum is a benign skip (emit `Processed` so ordering advances); ABI-decode failures still fail closed. — INDEX concern #13 +- Every wait on a peer must be bounded and must end in an attributable outcome. The aggregator + bounds its wait for honest `NodeDkgFold` proofs and publishes `E3Failed{DKGTimeout}` when the + budget expires. A late party must not be dropped from the honest set instead, because C5 is signed + before the fold completes and binds exactly those H keyshares. — `flow-trace/04` +- The node shutdown deadline and the fanout accept timeout come from one constant, + `NODE_SHUTDOWN_DEADLINE = FANOUT_ACCEPT_TIMEOUT + 30 s`. The daemon SIGKILL delay must stay above + that deadline, so a node is never killed while it still flushes. — `flow-trace/06` +- An actor that holds in-memory state derived from an event below the persisted snapshot cursor must + persist that state, because replay starts at the cursor and never redelivers the event. Verified + caches, own-proof records, and collector inputs all follow this rule. — `flow-trace/06` ### Schema evolution diff --git a/agent/flow-trace/00_INDEX.md b/agent/flow-trace/00_INDEX.md index d13ad80911..f3af476146 100644 --- a/agent/flow-trace/00_INDEX.md +++ b/agent/flow-trace/00_INDEX.md @@ -330,6 +330,7 @@ them are in the reference app (`examples/CRISP`), not the protocol. | Z-16 | **Safe slashing-manager retirement** | Resolved | Retiring managers remain authorized for their assigned E3s, bans, slash locks, proposals, and pending routes. `closeE3` now also waits for the objective accusation submission deadline. Revocation requires every canonical obligation counter to be zero. | | Z-24 | **Build-bound crypto configuration** | Resolved | The generated configuration ID binds the encryption scheme, exact parameter hash, and circuit version. Requests accept only that append-only configuration and snapshot its verifier addresses. Rust validates the emitted ID against its local build, and the indexer uses local immutable parameters plus the E3's frozen ID instead of querying mutable live parameter bytes. | | Z-45 | **In-call request fee bound** | Resolved | Each request supplies its expected fee token, expected crypto configuration, and maximum fee. Any change between quote and inclusion reverts before escrow transfer. The SDK obtains a fresh quote when the caller does not provide an explicit maximum. | +| L-01 | **Restart-visible liveness defects (live chaos audit)** | Resolved | A 21-round kill/restart audit of a five-node swarm closed a class of defects that only a live restart exposes. Replay starts at the persisted snapshot cursor, so an actor that held state derived from an earlier event lost it silently: the commitment-consistency cache and the node's own C0 proof are now durable, and a restarted node no longer accuses honest peers or stalls its fold at N−1 of N. The router now tears down a slashably-failed E3 after a two-hour grace instead of holding it until process restart. The aggregator now bounds its wait for honest `NodeDkgFold` proofs and fails with `DKGTimeout` instead of stalling. `bb` subprocesses have a hard timeout. The shutdown deadline and the fanout accept timeout come from one constant. See `CHAOS_TEST_REPORT.md` for the per-round evidence. | ### Scope of the Zenith `Z-` entries diff --git a/agent/flow-trace/04_DKG_AND_COMPUTATION.md b/agent/flow-trace/04_DKG_AND_COMPUTATION.md index 0b41bdbe6a..6cc2858f8a 100644 --- a/agent/flow-trace/04_DKG_AND_COMPUTATION.md +++ b/agent/flow-trace/04_DKG_AND_COMPUTATION.md @@ -330,6 +330,17 @@ aggregation path can terminate deterministically instead of stalling on missing `PublicKeyAggregator` and `ThresholdPlaintextAggregator` dispatch the aggregator requests instead of pairwise folding. +**Bounded node-proof collection:** a failed `NodeDkgFold` reports itself, but a member that dies +mid-fold sends nothing. `PublicKeyAggregator` therefore arms a durable budget when it enters +`GeneratingC5Proof` and cancels it when every honest proof arrives. If the budget expires, +`fail_on_missing_node_proofs` names the parties that did not deliver and publishes +`E3Failed { failed_at_stage: CommitteeFinalized, reason: DKGTimeout }`. The late parties are not +dropped from the honest set instead: C5 is signed before the cross-node fold completes and binds +exactly those H keyshares, so a different honest set would invalidate a published proof. The budget +is `E3_DKG_NODE_PROOF_TIMEOUT_SECS`, and its default is calibrated for the insecure test preset. +Measure a node fold at the deployment preset before secure operation, because a budget below the +honest fold time fails every E3 on healthy nodes. + **Failure bridge:** `ProofRequestActor` now converts proof-generation worker failures and local proof-signing failures into terminal round failures instead of only logging that the proof-bearing artifact will not be published. DKG-path proofs (`C0` through `C5`) emit diff --git a/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md b/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md index f6b9de533c..e27f69e3e0 100644 --- a/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md +++ b/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md @@ -1308,7 +1308,16 @@ When CommitteeMemberExpelled event arrives from EVM: │ │ → Single cleanup signal for all per-E3 actors │ │ NOTE: E3Failed with a misbehaviour reason (DKGInvalidShares, etc.) does │ │ NOT trigger E3RequestComplete — the accusation/slashing lifecycle must - │ │ complete first. + │ │ complete first. A slashable failure instead schedules a teardown after a + │ │ grace derived from the chain: ACCUSATION_REPORTING_WINDOW (1 day) plus the + │ │ registry's accusationVoteValidity plus a vote-in-flight margin, so the + │ │ accusation and vote windows can close while the E3 still leaves the router. + │ │ The grace is NOT a fixed constant: setAccusationVoteValidity enforces only a + │ │ lower bound, so governance can raise the window past any constant and every + │ │ node would tear down together while the chain still accepts a report. + │ │ SLASHABLE_FAILURE_TEARDOWN_GRACE (2 h) is the fallback used only when no + │ │ chain window is known. The deadline is persisted in the request-router + │ │ checkpoint and re-armed on restart. │ └─ E3StageChanged(Failed) and the same non-slashing E3Failed arriving after teardown │ are silently ignored (expected on-chain lag) │ diff --git a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md index b2a9c23811..cd7961a8b2 100644 --- a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md +++ b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md @@ -239,7 +239,7 @@ interfold start → running node ├─ Persists Shutdown and waits for acknowledged EventBus fanout ├─ Flushes the sequencer and event-store pipeline ├─ Drains open snapshot batches in event order, flushes the backing store, and closes it - ├─ Enforces a 30-second deadline and exits unsuccessfully on failure + ├─ Enforces a 60-second deadline and exits unsuccessfully on failure └─ Flushes the optional operational JSON log collector On restart: @@ -273,6 +273,17 @@ On restart: │ CiphernodeSelected events are likewise not guaranteed to replay. │ → Recovered aggregator roles, selected party IDs, and DHT document interests are injected │ directly from snapshots. Startup does not append synthetic recovery events. +│ → Per-E3 durable caches are restored before replay, because replay starts at the snapshot +│ cursor and never redelivers an earlier event: +│ • CommitmentConsistencyChecker restores its verified-proof cache from +│ `//commitment_consistency/v1/{e3_id}`. Without it the node holds no record of its own +│ C0 proof, every peer C3 that points at that C0 fails the link check, and the node +│ accuses honest peers. +│ • ProofRequestActor restores its own C0 proof from `//own_c0_proof/v1/{e3_id}` and +│ re-publishes `DKGInnerProofReady { seq: 0 }`. Without it the node fold waits forever +│ at N−1 of N inner proofs. +│ • ThresholdKeyshare rebuilds the decryption-share collector and its timeout whenever the +│ state is `ReadyForDecryption`, including when no share has arrived yet. ├─ Sync module replays: │ → Arm the current NetReady listener before the network transport can publish readiness │ 4. Replay EventStore events since the snapshot cut (effects still disabled) @@ -312,6 +323,12 @@ event pipeline flushed, open snapshot batches drained, and the backing store flu deadline. Detached work that is not owned by those barriers can still be cancelled by process exit; operators must continue to follow the production shutdown precautions. +`NODE_SHUTDOWN_DEADLINE` is derived as `FANOUT_ACCEPT_TIMEOUT + 30 s` in `e3-events`, and both the +CLI and the daemon read that one constant. The daemon waits for the deadline plus five seconds +before it sends `SIGKILL`. An external supervisor must allow at least as long: a systemd +`TimeoutStopSec` or Docker stop timeout below 65 seconds kills a node while it still flushes its +event log. + The three long-lived libp2p `NetEvent` broadcast consumers (`NetEventTranslator`, `DocumentPublisher`, and `NetSyncManager`) treat Tokio's `Lagged(n)` receive result as a recoverable overload signal: they emit a bounded structured warning containing only the static consumer name and diff --git a/crates/aggregator/src/committee_finalization/actor.rs b/crates/aggregator/src/committee_finalization/actor.rs index e15cfd0272..aece286f1d 100644 --- a/crates/aggregator/src/committee_finalization/actor.rs +++ b/crates/aggregator/src/committee_finalization/actor.rs @@ -13,12 +13,12 @@ use e3_events::{ InterfoldEvent, InterfoldEventData, Shutdown, TicketGenerated, TypedEvent, }; use e3_events::{E3id, EventContext, Sequenced}; -use e3_evm::helpers::{ConcreteReadProvider, EthProvider}; +use e3_evm::helpers::{ConcreteReadProvider, EthProvider, ProviderFactory}; use e3_utils::{NotifySync, MAILBOX_LIMIT}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::time::Duration; -use tracing::{error, info}; +use tracing::{error, info, warn}; #[path = "handlers.rs"] mod handlers; @@ -79,13 +79,100 @@ fn finalization_delay_seconds(committee_deadline: u64, now: u64, party_index: u6 .saturating_add(party_index.saturating_mul(FINALIZE_INTERVAL_SECONDS)) } +/// A read provider for one chain together with the means to rebuild it. +/// +/// The provider is a WebSocket clone taken at startup. When the RPC endpoint is away for +/// longer than alloy's own reconnect budget the clone dies for good and every timestamp read +/// fails with "backend connection task has stopped". The factory lets the finalizer replace +/// it instead of retrying the dead one every 30 s until the committee deadline passes. +#[derive(Clone)] +pub struct FinalizerChainProvider { + pub provider: EthProvider, + pub factory: Option>, +} + +impl FinalizerChainProvider { + pub fn new(provider: EthProvider) -> Self { + Self { + provider, + factory: None, + } + } + + pub fn with_factory(mut self, factory: ProviderFactory) -> Self { + self.factory = Some(factory); + self + } +} + +/// Read the chain's latest timestamp, rebuilding the provider once if the read fails. +/// +/// Returns the timestamp result and, when a reconnect produced a new provider, that provider +/// so the actor can adopt it for later reads. +async fn read_timestamp_with_reconnect( + chain_provider: FinalizerChainProvider, + chain_id: u64, + e3_id: &E3id, +) -> (Result, Option>) { + read_timestamp_reconnecting( + chain_provider.provider, + chain_provider.factory, + chain_id, + e3_id, + ) + .await +} + +/// Generic core of [`read_timestamp_with_reconnect`], so a mock transport can drive it. +async fn read_timestamp_reconnecting

( + provider: EthProvider

, + factory: Option>, + chain_id: u64, + e3_id: &E3id, +) -> (Result, Option>) +where + P: alloy::providers::Provider + Clone + 'static, +{ + let first = e3_evm::helpers::get_current_timestamp_from_provider(provider.clone()).await; + let (Err(first_error), Some(factory)) = (&first, factory.as_ref()) else { + return (first, None); + }; + warn!( + %e3_id, + error = %first_error, + "Timestamp read failed; reconnecting the read provider and retrying once" + ); + let replacement = match factory().await { + Ok(replacement) if replacement.chain_id() == chain_id => replacement, + Ok(replacement) => { + warn!( + %e3_id, + expected_chain_id = chain_id, + actual_chain_id = replacement.chain_id(), + "Refusing a reconnected finalizer provider for another chain" + ); + return (first, None); + } + Err(reconnect_error) => { + warn!( + %e3_id, + error = %reconnect_error, + "Unable to reconnect the finalizer read provider" + ); + return (first, None); + } + }; + let second = e3_evm::helpers::get_current_timestamp_from_provider(replacement.clone()).await; + (second, Some(replacement)) +} + /// CommitteeFinalizer is an actor that listens to CommitteeRequested events and dispatches /// CommitteeFinalizeRequested events after the submission deadline has passed. pub struct CommitteeFinalizer { bus: BusHandle, pending_committees: HashMap, recovery: Persistable, - chain_providers: HashMap>, + chain_providers: HashMap, effects_enabled: bool, } @@ -93,7 +180,7 @@ impl CommitteeFinalizer { fn from_recovery( bus: &BusHandle, recovery: Persistable, - chain_providers: HashMap>, + chain_providers: HashMap, ) -> Self { Self { bus: bus.clone(), @@ -107,7 +194,7 @@ impl CommitteeFinalizer { pub async fn attach_with_recovery( bus: &BusHandle, repository: Repository, - chain_providers: HashMap>, + chain_providers: HashMap, ) -> Result> { let recovery = repository .load_or_default(CommitteeFinalizerRecoveryState::default()) @@ -152,34 +239,40 @@ impl CommitteeFinalizer { let ec = request.context.clone(); let pending_key = e3_id.clone(); let e3_id_for_async = e3_id.clone(); - let provider = self.chain_providers.get(&e3_id.chain_id()).cloned(); + let chain_id = e3_id.chain_id(); + let chain_provider = self.chain_providers.get(&chain_id).cloned(); let fut = async move { - let timestamp = match provider { - Some(provider) => { - e3_evm::helpers::get_current_timestamp_from_provider(provider).await - } - None => Err(anyhow::anyhow!( - "No RPC provider configured for chain {}", - e3_id_for_async.chain_id() - )), + let Some(chain_provider) = chain_provider else { + error!( + e3_id = %e3_id_for_async, + "No RPC provider configured for chain {chain_id}" + ); + return (None, None); }; + let (timestamp, replacement) = + read_timestamp_with_reconnect(chain_provider, chain_id, &e3_id_for_async).await; match timestamp { - Ok(timestamp) => Some(timestamp), + Ok(timestamp) => (Some(timestamp), replacement), Err(e) => { error!( e3_id = %e3_id_for_async, error = %e, "Failed to get current timestamp from RPC" ); - None + (None, replacement) } } }; let handle = ctx.spawn( fut.into_actor(self) - .then(move |current_timestamp, act, ctx| { + .then(move |(current_timestamp, replacement), act, ctx| { + if let Some(replacement) = replacement { + if let Some(entry) = act.chain_providers.get_mut(&chain_id) { + entry.provider = replacement; + } + } if let Some(current_timestamp) = current_timestamp { let seconds_until_deadline = finalization_delay_seconds( committee_deadline, @@ -301,4 +394,97 @@ mod tests { 1 + 3 * FINALIZE_INTERVAL_SECONDS ); } + + mod reconnect { + use super::super::read_timestamp_reconnecting; + use alloy::{ + providers::{Provider, ProviderBuilder}, + transports::mock::Asserter, + }; + use e3_events::E3id; + use e3_evm::helpers::{EthProvider, ProviderFactory}; + use std::sync::Arc; + + async fn provider_on_chain( + asserter: &Asserter, + chain_id_hex: &str, + ) -> EthProvider { + asserter.push_success(&chain_id_hex); + EthProvider::new(ProviderBuilder::new().connect_mocked_client(asserter.clone())) + .await + .expect("mock chain ID must decode") + } + + fn block_with_timestamp(timestamp: u64) -> alloy::rpc::types::Block { + let mut block: alloy::rpc::types::Block = Default::default(); + block.header.inner.timestamp = timestamp; + block + } + + /// Observed on a 5-node swarm: after a 90 s RPC outage every node's finalizer logged + /// `Failed to get current timestamp from RPC` every 30 s until the committee window + /// closed, because the startup provider clone had given up reconnecting. The + /// finalizer must rebuild the provider through its factory and retry. + #[actix::test] + async fn dead_provider_is_replaced_before_the_timestamp_read_fails() { + let dead = Asserter::new(); + let dead_provider = provider_on_chain(&dead, "0x1").await; + dead.push_failure_msg("backend connection task has stopped"); + + let healthy = Asserter::new(); + let healthy_provider = provider_on_chain(&healthy, "0x1").await; + healthy.push_success(&block_with_timestamp(1_700_000_042)); + let factory: ProviderFactory<_> = Arc::new(move || { + let provider = healthy_provider.clone(); + Box::pin(async move { Ok(provider) }) + }); + + let (timestamp, replacement) = + read_timestamp_reconnecting(dead_provider, Some(factory), 1, &E3id::new("7", 1)) + .await; + + assert_eq!(timestamp.unwrap(), 1_700_000_042); + assert!( + replacement.is_some(), + "the reconnected provider must be adopted" + ); + } + + #[actix::test] + async fn without_a_factory_the_error_is_returned_unchanged() { + let dead = Asserter::new(); + let dead_provider = provider_on_chain(&dead, "0x1").await; + dead.push_failure_msg("backend connection task has stopped"); + + let (timestamp, replacement) = + read_timestamp_reconnecting(dead_provider, None, 1, &E3id::new("7", 1)).await; + + assert!(timestamp + .unwrap_err() + .to_string() + .contains("Failed to get latest block")); + assert!(replacement.is_none()); + } + + #[actix::test] + async fn a_wrong_chain_reconnect_is_refused() { + let dead = Asserter::new(); + let dead_provider = provider_on_chain(&dead, "0x1").await; + dead.push_failure_msg("backend connection task has stopped"); + + let other = Asserter::new(); + let other_provider = provider_on_chain(&other, "0x2").await; + let factory: ProviderFactory<_> = Arc::new(move || { + let provider = other_provider.clone(); + Box::pin(async move { Ok(provider) }) + }); + + let (timestamp, replacement) = + read_timestamp_reconnecting(dead_provider, Some(factory), 1, &E3id::new("7", 1)) + .await; + + assert!(timestamp.is_err()); + assert!(replacement.is_none()); + } + } } diff --git a/crates/aggregator/src/plaintext_aggregation/effects/prove_plaintext.rs b/crates/aggregator/src/plaintext_aggregation/effects/prove_plaintext.rs index 936b8a4188..dde962288d 100644 --- a/crates/aggregator/src/plaintext_aggregation/effects/prove_plaintext.rs +++ b/crates/aggregator/src/plaintext_aggregation/effects/prove_plaintext.rs @@ -60,6 +60,7 @@ impl ThresholdPlaintextAggregator { if proofs.len() != state.plaintext.len() { warn!( + e3_id = %self.e3_id, "C7 proof count mismatch: got {} proofs for {} ciphertext indices", proofs.len(), state.plaintext.len() @@ -146,6 +147,7 @@ impl ThresholdPlaintextAggregator { // "aggregation disabled". Fail loudly instead so the missing shares are surfaced. if honest_c6.is_empty() || honest_c6.iter().any(|(_, w)| w.is_empty()) { warn!( + e3_id = %self.e3_id, "DecryptionAggregation: honest C6 inner proofs missing while proof aggregation is enabled" ); return self.fail_decryption_round(ec.clone()); @@ -161,6 +163,7 @@ impl ThresholdPlaintextAggregator { let c6_total_slots = state.threshold_m as usize + 1; if honest_c6.len() < c6_total_slots { warn!( + e3_id = %self.e3_id, "DecryptionAggregation needs at least {} honest C6 parties, have {}", c6_total_slots, honest_c6.len() @@ -261,6 +264,7 @@ impl ThresholdPlaintextAggregator { if let Some(c7_proofs) = self.pending.c7_proofs_pending.as_ref() { if resp.proofs.len() != c7_proofs.len() { warn!( + e3_id = %self.e3_id, "DecryptionAggregation response proof count {} != expected {}", resp.proofs.len(), c7_proofs.len() diff --git a/crates/aggregator/src/plaintext_aggregation/effects/verify_decryption_shares.rs b/crates/aggregator/src/plaintext_aggregation/effects/verify_decryption_shares.rs index da0f86aa1a..6d3248aa19 100644 --- a/crates/aggregator/src/plaintext_aggregation/effects/verify_decryption_shares.rs +++ b/crates/aggregator/src/plaintext_aggregation/effects/verify_decryption_shares.rs @@ -86,6 +86,7 @@ impl ThresholdPlaintextAggregator { let mut dishonest_parties = msg.dishonest_parties.clone(); if !dishonest_parties.is_empty() { warn!( + e3_id = %self.e3_id, "C6 verification: {} dishonest parties filtered: {:?}", dishonest_parties.len(), dishonest_parties @@ -102,6 +103,7 @@ impl ThresholdPlaintextAggregator { if honest_shares.len() <= state.threshold_m as usize { warn!( + e3_id = %self.e3_id, "Not enough honest shares after C6 verification: {} honest shares, {} required", honest_shares.len(), state.threshold_m + 1 @@ -121,6 +123,7 @@ impl ThresholdPlaintextAggregator { ); if !share_mismatch_parties.is_empty() { warn!( + e3_id = %self.e3_id, "C6 share-commitment mismatch for {} parties: {:?} — excluding from aggregation", share_mismatch_parties.len(), share_mismatch_parties, @@ -130,6 +133,7 @@ impl ThresholdPlaintextAggregator { honest_shares.retain(|(id, _)| !share_mismatch_parties.contains(id)); if honest_shares.len() <= state.threshold_m as usize { warn!( + e3_id = %self.e3_id, "Not enough honest shares after d_commitment check: {} honest, {} required", honest_shares.len(), state.threshold_m + 1 diff --git a/crates/aggregator/src/public_key_aggregation/actor.rs b/crates/aggregator/src/public_key_aggregation/actor.rs index d88823ca79..2ca3dbb48e 100644 --- a/crates/aggregator/src/public_key_aggregation/actor.rs +++ b/crates/aggregator/src/public_key_aggregation/actor.rs @@ -30,7 +30,7 @@ use e3_utils::NotifySync; use e3_utils::{ArcBytes, MAILBOX_LIMIT}; use e3_zk_helpers::CiphernodesCommitteeSize; use std::sync::Arc; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; // Public-key aggregation state machine + pure transition logic now live in // `crate::workflow::publickey_aggregation`; re-exported here to preserve the public path @@ -53,6 +53,8 @@ pub struct PublicKeyAggregator { effects_enabled: bool, /// DKG recursive aggregation events received before entering GeneratingC5Proof. early_dkg_proofs: Vec>, + /// Bounded wait for honest-party NodeDkgFold proofs. See [`node_proof_timeout`]. + node_proof_deadline: Option, } pub struct PublicKeyAggregatorParams { @@ -87,6 +89,40 @@ impl PublicKeyAggregator { is_aggregator: params.initial_is_aggregator, effects_enabled: params.effects_enabled, early_dkg_proofs: Vec::new(), + node_proof_deadline: None, + } + } + + /// Arm the bounded wait for honest-party NodeDkgFold proofs. + /// + /// Idempotent: re-arming while a timer is live is a no-op, so repeated entries into + /// `GeneratingC5Proof` (each buffered proof re-runs the dispatch path) do not extend the + /// budget. Only the active aggregator arms it — a standby that is later promoted arms its + /// own on promotion, which is the point at which its wait actually begins. + pub(in crate::actors::publickey_aggregator) fn arm_node_proof_deadline( + &mut self, + ctx: &mut Context, + ec: &EventContext, + ) { + if self.node_proof_deadline.is_some() || !self.can_run_aggregation_effects() { + return; + } + let budget = node_proof_timeout::dkg_node_proof_timeout(); + let ec = ec.clone(); + let handle = ctx.run_later(budget, move |actor, _ctx| { + actor.node_proof_deadline = None; + actor.fail_on_missing_node_proofs(&ec, budget); + }); + self.node_proof_deadline = Some(handle); + } + + /// Cancel the bounded wait once every honest proof is in (or the E3 is finished). + pub(in crate::actors::publickey_aggregator) fn cancel_node_proof_deadline( + &mut self, + ctx: &mut Context, + ) { + if let Some(handle) = self.node_proof_deadline.take() { + ctx.cancel_future(handle); } } @@ -124,6 +160,8 @@ impl PublicKeyAggregator { mod effects; #[path = "handlers.rs"] mod handlers; +#[path = "node_proof_timeout.rs"] +mod node_proof_timeout; #[cfg(test)] #[path = "tests/mod.rs"] diff --git a/crates/aggregator/src/public_key_aggregation/effects/aggregate_dkg_proofs.rs b/crates/aggregator/src/public_key_aggregation/effects/aggregate_dkg_proofs.rs index aa676e7dee..bc083e899d 100644 --- a/crates/aggregator/src/public_key_aggregation/effects/aggregate_dkg_proofs.rs +++ b/crates/aggregator/src/public_key_aggregation/effects/aggregate_dkg_proofs.rs @@ -127,8 +127,10 @@ impl PublicKeyAggregator { pairs.sort_by_key(|(pid, _)| *pid); let party_ids: Vec = pairs.iter().map(|(pid, _)| *pid).collect(); let node_fold_proofs: Vec = pairs.into_iter().map(|(_, p)| p).collect(); - info!( - "ORDER-DEBUG dispatch DkgAggregation: honest_party_ids(submission-idx)={:?} \ + // Party-id ordering across three representations has been a real source of circuit + // mismatches, so keep the correspondence loggable — but at debug, not on every dispatch. + debug!( + "DkgAggregation dispatch ordering: honest_party_ids(submission-idx)={:?} \ dkg_node_proofs_keys(real party_id from DKGRecursiveAggregationComplete)={:?} \ party_ids_passed_to_circuit={:?}", honest_party_ids.iter().collect::>(), diff --git a/crates/aggregator/src/public_key_aggregation/effects/aggregate_public_key.rs b/crates/aggregator/src/public_key_aggregation/effects/aggregate_public_key.rs index a35abf219c..ea0e19111e 100644 --- a/crates/aggregator/src/public_key_aggregation/effects/aggregate_public_key.rs +++ b/crates/aggregator/src/public_key_aggregation/effects/aggregate_public_key.rs @@ -111,6 +111,7 @@ impl PublicKeyAggregator { }; if dkg_node_proofs.contains_key(&msg.party_id) { warn!( + e3_id = %self.e3_id, "Duplicate DKGRecursiveAggregationComplete for party {} — ignoring", msg.party_id ); @@ -120,6 +121,7 @@ impl PublicKeyAggregator { if honest_party_ids.contains(&msg.party_id) { let Some(expected_node) = party_nodes.get(&msg.party_id) else { warn!( + e3_id = %self.e3_id, party_id = msg.party_id, "DKG fold from party without registered node address — rejecting" ); @@ -136,6 +138,7 @@ impl PublicKeyAggregator { (Some(proof), Some(attestation)) => { let Some(expected_context) = self.dkg_fold_attestation_context else { warn!( + e3_id = %self.e3_id, party_id = msg.party_id, "DKG fold attestation context missing — rejecting" ); @@ -147,6 +150,7 @@ impl PublicKeyAggregator { let n_moduli = meta.num_moduli; if committee_n == 0 || committee_h == 0 { warn!( + e3_id = %self.e3_id, party_id = msg.party_id, "DKG fold attestation verify skipped — circuit committee dims unset" ); @@ -163,8 +167,19 @@ impl PublicKeyAggregator { committee_h, n_moduli, ) { + // Name both addresses: a signer mismatch here is either a genuine + // impostor or a party_id -> node-address ordering divergence between + // this aggregator and the signing node, and the two are + // indistinguishable without seeing the pair. + let recovered = attestation + .recover_address() + .map(|a| a.to_string()) + .unwrap_or_else(|e| format!("")); warn!( + e3_id = %self.e3_id, party_id = msg.party_id, + expected_node = %expected_node, + recovered_signer = %recovered, error = %e, "DKG fold attestation verification failed — rejecting" ); @@ -173,6 +188,7 @@ impl PublicKeyAggregator { } (Some(_), None) => { warn!( + e3_id = %self.e3_id, party_id = msg.party_id, "DKG fold has proof but missing attestation — rejecting (attribution)" ); @@ -180,6 +196,7 @@ impl PublicKeyAggregator { } (None, Some(_)) => { warn!( + e3_id = %self.e3_id, party_id = msg.party_id, "DKG fold has attestation but missing proof — rejecting" ); diff --git a/crates/aggregator/src/public_key_aggregation/effects/mod.rs b/crates/aggregator/src/public_key_aggregation/effects/mod.rs index fd107c1501..2d6f9a07ca 100644 --- a/crates/aggregator/src/public_key_aggregation/effects/mod.rs +++ b/crates/aggregator/src/public_key_aggregation/effects/mod.rs @@ -13,6 +13,7 @@ mod aggregate_dkg_proofs; mod aggregate_public_key; mod fold_node_proofs; mod handle_compute_results; +mod node_proof_deadline; mod publish_result; mod recovery; mod verify_key_proofs; diff --git a/crates/aggregator/src/public_key_aggregation/effects/node_proof_deadline.rs b/crates/aggregator/src/public_key_aggregation/effects/node_proof_deadline.rs new file mode 100644 index 0000000000..1e79be2686 --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/effects/node_proof_deadline.rs @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Bound the wait for honest-party NodeDkgFold proofs. + +use super::super::*; +use std::time::Duration; + +impl PublicKeyAggregator { + /// Honest parties whose NodeDkgFold proof has not arrived yet. + /// + /// Returns an empty vector when the aggregator is not collecting node proofs, so callers + /// can treat "nothing missing" and "not applicable" identically. + pub(in crate::actors::publickey_aggregator) fn missing_node_proof_parties(&self) -> Vec { + let Some(PublicKeyAggregatorState::GeneratingC5Proof { + dkg_node_proofs, + honest_party_ids, + dkg_aggregated_proof, + .. + }) = self.state.get() + else { + return Vec::new(); + }; + if dkg_aggregated_proof.is_some() { + return Vec::new(); + } + honest_party_ids + .iter() + .filter(|id| !dkg_node_proofs.contains_key(id)) + .copied() + .collect() + } + + /// Fail the E3 when the node-proof budget expires with proofs still missing. + /// + /// The late parties cannot simply be dropped: C5 is already signed over exactly these H + /// keyshares, so re-selecting the honest set would invalidate a published proof. Failing + /// explicitly — naming the parties that did not deliver — converts an unbounded stall into + /// an attributable, bounded outcome that the slashing layer can act on. + pub(in crate::actors::publickey_aggregator) fn fail_on_missing_node_proofs( + &mut self, + ec: &EventContext, + budget: Duration, + ) { + let missing = self.missing_node_proof_parties(); + if missing.is_empty() { + return; + } + + error!( + e3_id = %self.e3_id, + missing_party_ids = ?missing, + budget_secs = budget.as_secs(), + "DKG node-proof collection budget expired; failing E3 (a member could not complete \ + its NodeDkgFold — C5 is already signed over this honest set, so it cannot be \ + re-selected)" + ); + + if let Err(err) = self.bus.publish( + E3Failed { + e3_id: self.e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGTimeout, + }, + ec.clone(), + ) { + error!( + e3_id = %self.e3_id, + "Failed to publish E3Failed after node-proof timeout: {err}" + ); + } + } +} diff --git a/crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs b/crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs index b9cbbdadb4..01e5bea497 100644 --- a/crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs +++ b/crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs @@ -159,10 +159,15 @@ impl PublicKeyAggregator { }, ec.clone(), ) { - error!("Failed to publish SignedProofFailed: {e}"); + error!( + e3_id = %self.e3_id, + party_id, + "Failed to publish SignedProofFailed: {e}" + ); } } Err(e) => warn!( + e3_id = %self.e3_id, "Could not recover address from C1 proof for party {}: {e}", party_id ), @@ -171,6 +176,7 @@ impl PublicKeyAggregator { if !audit.mismatched.is_empty() { warn!( + e3_id = %self.e3_id, "C1 commitment mismatch for {} parties — filtering before aggregation", audit.mismatched.len() ); diff --git a/crates/aggregator/src/public_key_aggregation/handlers.rs b/crates/aggregator/src/public_key_aggregation/handlers.rs index 176945a0c6..ee42369b4d 100644 --- a/crates/aggregator/src/public_key_aggregation/handlers.rs +++ b/crates/aggregator/src/public_key_aggregation/handlers.rs @@ -59,7 +59,10 @@ impl Handler for PublicKeyAggregator { let node_addr = data.node; if data.e3_id != self.e3_id { - error!("Wrong e3_id sent to PublicKeyAggregator for expulsion. This should not happen."); + error!( + e3_id = %self.e3_id, + "Wrong e3_id sent to PublicKeyAggregator for expulsion. This should not happen." + ); return; } @@ -107,7 +110,10 @@ impl Handler for PublicKeyAggregator { let node_addr = data.node; if data.e3_id != self.e3_id { - error!("Wrong e3_id sent to PublicKeyAggregator for local exclusion."); + error!( + e3_id = %self.e3_id, + "Wrong e3_id sent to PublicKeyAggregator for local exclusion." + ); return; } @@ -154,7 +160,7 @@ impl Handler> for PublicKeyAggregator { fn handle( &mut self, msg: TypedEvent, - _ctx: &mut Self::Context, + ctx: &mut Self::Context, ) -> Self::Result { if msg.e3_id != self.e3_id || msg.is_aggregator == self.is_aggregator { return; @@ -163,8 +169,17 @@ impl Handler> for PublicKeyAggregator { if self.can_run_aggregation_effects() { let ec = msg.get_ctx().clone(); trap(EType::PublickeyAggregation, &self.bus.with_ec(&ec), || { - self.resume_in_flight_work(ec) + self.resume_in_flight_work(ec.clone()) }); + // A promoted standby inherits the same missing node proofs the demoted + // aggregator was waiting on, so it starts its own bounded wait here rather than + // stalling for the rest of the E3. + if !self.missing_node_proof_parties().is_empty() { + self.arm_node_proof_deadline(ctx, &ec); + } + } else { + // Demoted: stop counting down. The newly promoted aggregator owns the bound. + self.cancel_node_proof_deadline(ctx); } } } @@ -186,7 +201,11 @@ impl Handler> for PublicKeyAggregator { let c1_proof = event.signed_pk_generation_proof.clone(); if e3_id != self.e3_id { - error!("Wrong e3_id sent to aggregator. This should not happen."); + error!( + e3_id = %self.e3_id, + party_id, + "Wrong e3_id sent to aggregator. This should not happen." + ); return Ok(()); } @@ -240,16 +259,23 @@ impl Handler> for PublicKeyAggregator { fn handle( &mut self, msg: TypedEvent, - _ctx: &mut Self::Context, + ctx: &mut Self::Context, ) -> Self::Result { if !self.can_run_aggregation_effects() { return; } + let ec = msg.get_ctx().clone(); trap( EType::PublickeyAggregation, &self.bus.with_ec(msg.get_ctx()), || self.handle_pk_aggregation_proof_signed(msg), - ) + ); + // C5 is signed; from here the aggregator only waits on honest-party NodeDkgFold + // proofs. Bound that wait — an unfinishable member used to hold the E3 open until + // the canonical deadline. + if !self.missing_node_proof_parties().is_empty() { + self.arm_node_proof_deadline(ctx, &ec); + } } } @@ -259,16 +285,24 @@ impl Handler> for PublicKeyAggregato fn handle( &mut self, msg: TypedEvent, - _ctx: &mut Self::Context, + ctx: &mut Self::Context, ) -> Self::Result { if !self.can_run_aggregation_effects() { return; } + let ec = msg.get_ctx().clone(); trap( EType::PublickeyAggregation, &self.bus.with_ec(msg.get_ctx()), || self.handle_dkg_recursive_aggregation_complete(msg), - ) + ); + // Every honest proof in: the wait is over. Otherwise keep (or start) the bound — + // arming is idempotent, so a partial delivery never extends the original budget. + if self.missing_node_proof_parties().is_empty() { + self.cancel_node_proof_deadline(ctx); + } else { + self.arm_node_proof_deadline(ctx, &ec); + } } } diff --git a/crates/aggregator/src/public_key_aggregation/node_proof_timeout.rs b/crates/aggregator/src/public_key_aggregation/node_proof_timeout.rs new file mode 100644 index 0000000000..ef63baa60b --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/node_proof_timeout.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Bounded wait for honest-party DKG node proofs. +//! +//! The aggregator enters `GeneratingC5Proof` and then waits for one `NodeDkgFold` proof from +//! every honest party. That wait had no bound: a single member that could never finish its +//! fold (Round 14, cn3 stuck at 13/14 inner proofs) held the whole E3 open indefinitely. The +//! sortition failover only rotates the *aggregator role* — every standby inherits the same +//! missing proof, so its budgets drain one after another and the E3 stalls until the canonical +//! deadline. +//! +//! Excluding the late party is not available here: C5 is signed *before* the fold completes +//! and binds exactly the H honest keyshares (`PkAggregationProofRequest.keyshare_bytes` + +//! `aggregated_pk_bytes`), so dropping a party after C5 would invalidate the proof that has +//! already been published. The correct bounded outcome is therefore an explicit, attributable +//! failure rather than a silent hang. + +use std::time::Duration; + +/// Environment override for the honest-node-proof collection budget. +pub(crate) const DKG_NODE_PROOF_TIMEOUT_ENV: &str = "E3_DKG_NODE_PROOF_TIMEOUT_SECS"; + +/// Default budget for collecting every honest party's NodeDkgFold proof. +/// +/// A node fold is the most expensive job in the DKG and its cost grows with the ring degree. +/// Measured at the insecure test preset (degree 512) across five folds: 135 s, 137 s, 147 s, +/// 214 s, 214 s. A restarted member must also re-prove C1–C4 before it can start folding +/// (~40 s more). 30 minutes is ~8x the slowest measured fold, which bounds the stall well +/// below the two-hour DKG window while leaving room for a slow or once-restarted member. +/// +/// CAUTION — this default is calibrated against insecure test params and has NOT been measured +/// at secure params. Secure operation uses degree 32768 (64x this ring), and if fold cost grows +/// even linearly in the degree the honest fold alone would exceed this budget, so every E3 would +/// fail with `DKGTimeout` on healthy nodes. Measure the fold at the deployment preset and raise +/// this default (or set `E3_DKG_NODE_PROOF_TIMEOUT_SECS`) before running at secure N. +pub(crate) const DEFAULT_DKG_NODE_PROOF_TIMEOUT_SECS: u64 = 1800; + +/// Resolve the collection budget, honouring the environment override. +pub(crate) fn dkg_node_proof_timeout() -> Duration { + let secs = std::env::var(DKG_NODE_PROOF_TIMEOUT_ENV) + .ok() + .and_then(|raw| raw.parse::().ok()) + .filter(|secs| *secs > 0) + .unwrap_or(DEFAULT_DKG_NODE_PROOF_TIMEOUT_SECS); + Duration::from_secs(secs) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `E3_DKG_NODE_PROOF_TIMEOUT_SECS` is process-global, so the default and override cases + /// share one test rather than racing each other under the threaded test harness. + #[test] + fn budget_resolution_prefers_a_valid_override_and_falls_back_otherwise() { + let default = Duration::from_secs(DEFAULT_DKG_NODE_PROOF_TIMEOUT_SECS); + + std::env::remove_var(DKG_NODE_PROOF_TIMEOUT_ENV); + assert_eq!(dkg_node_proof_timeout(), default); + + std::env::set_var(DKG_NODE_PROOF_TIMEOUT_ENV, "42"); + assert_eq!(dkg_node_proof_timeout(), Duration::from_secs(42)); + + // A zero or unparseable budget would disable the bound entirely — fall back. + std::env::set_var(DKG_NODE_PROOF_TIMEOUT_ENV, "0"); + assert_eq!(dkg_node_proof_timeout(), default); + + std::env::set_var(DKG_NODE_PROOF_TIMEOUT_ENV, "not-a-number"); + assert_eq!(dkg_node_proof_timeout(), default); + + std::env::remove_var(DKG_NODE_PROOF_TIMEOUT_ENV); + } +} diff --git a/crates/aggregator/src/public_key_aggregation/tests/mod.rs b/crates/aggregator/src/public_key_aggregation/tests/mod.rs index f357477121..4d10527647 100644 --- a/crates/aggregator/src/public_key_aggregation/tests/mod.rs +++ b/crates/aggregator/src/public_key_aggregation/tests/mod.rs @@ -267,3 +267,4 @@ async fn standby_persists_and_resumes_public_key_work() -> Result<()> { mod attestations; mod failures; +mod node_proof_deadline; diff --git a/crates/aggregator/src/public_key_aggregation/tests/node_proof_deadline.rs b/crates/aggregator/src/public_key_aggregation/tests/node_proof_deadline.rs new file mode 100644 index 0000000000..cdd2ae5c41 --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/tests/node_proof_deadline.rs @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Bounded wait for honest-party NodeDkgFold proofs. + +use super::*; + +/// `GeneratingC5Proof` with C5 already signed and `present` of `honest` node proofs delivered. +fn awaiting_node_proofs(honest: &[u64], present: &[u64]) -> PublicKeyAggregatorState { + let mut dkg_node_proofs = HashMap::new(); + for id in present { + dkg_node_proofs.insert(*id, Some(dummy_proof(CircuitName::NodeFold))); + } + PublicKeyAggregatorState::GeneratingC5Proof { + public_key: ArcBytes::from_bytes(&[1, 2, 3]), + keyshare_bytes: Vec::new(), + nodes: OrderedSet::new(), + party_nodes: HashMap::new(), + dkg_node_proofs, + dkg_fold_attestations: HashMap::new(), + honest_party_ids: honest.iter().copied().collect::>(), + dishonest_parties: BTreeSet::new(), + circuit_committee_n: 3, + circuit_committee_h: honest.len(), + dkg_aggregation_correlation: None, + dkg_aggregated_proof: None, + c5_proof_pending: Some(dummy_proof(CircuitName::PkAggregation)), + last_ec: None, + nodes_fold_accumulator: None, + nodes_fold_completed_slots: 0, + nodes_fold_step_correlation: None, + } +} + +/// Round 14: cn3 could not finish its node fold (13/14 inner proofs), and the aggregator waited +/// for it through three 10-minute standby budgets before giving up at the canonical deadline. +/// The missing party must be identifiable so the wait can be bounded and attributed. +#[actix::test] +async fn missing_node_proof_parties_names_only_the_absent_honest_parties() -> Result<()> { + let (aggregator, _history, _e3_id) = + build_public_key_aggregator(awaiting_node_proofs(&[0, 1, 2], &[0, 2])).await?; + assert_eq!(aggregator.missing_node_proof_parties(), vec![1]); + Ok(()) +} + +#[actix::test] +async fn nothing_is_missing_once_every_honest_proof_arrived() -> Result<()> { + let (aggregator, _history, _e3_id) = + build_public_key_aggregator(awaiting_node_proofs(&[0, 1, 2], &[0, 1, 2])).await?; + assert!(aggregator.missing_node_proof_parties().is_empty()); + Ok(()) +} + +/// A dishonest party is not waited on: only the capped honest set gates the fold. +#[actix::test] +async fn dishonest_parties_are_not_waited_for() -> Result<()> { + let (aggregator, _history, _e3_id) = + build_public_key_aggregator(awaiting_node_proofs(&[0, 2], &[0, 2])).await?; + assert!(aggregator.missing_node_proof_parties().is_empty()); + Ok(()) +} + +/// Once the final DKG aggregation proof exists the collection is over, so an expiring timer +/// must not fail an E3 that already succeeded. +#[actix::test] +async fn nothing_is_missing_after_the_aggregated_proof_exists() -> Result<()> { + let mut state = awaiting_node_proofs(&[0, 1, 2], &[0]); + if let PublicKeyAggregatorState::GeneratingC5Proof { + dkg_aggregated_proof, + .. + } = &mut state + { + *dkg_aggregated_proof = Some(dummy_proof(CircuitName::NodesFold)); + } + let (aggregator, _history, _e3_id) = build_public_key_aggregator(state).await?; + assert!(aggregator.missing_node_proof_parties().is_empty()); + Ok(()) +} + +/// Before C5 exists the aggregator is not yet in the node-proof wait. +#[actix::test] +async fn a_non_collecting_state_reports_nothing_missing() -> Result<()> { + let (aggregator, _history, _e3_id) = build_public_key_aggregator(complete_state()).await?; + assert!(aggregator.missing_node_proof_parties().is_empty()); + Ok(()) +} + +/// Budget expiry with a proof still missing must fail the E3 explicitly rather than hang. +/// Excluding the late party is impossible here: C5 is already signed over this exact honest +/// set, so the only bounded outcome is an attributable failure. +#[actix::test] +async fn expiring_the_budget_fails_the_e3_with_dkg_timeout() -> Result<()> { + let (mut aggregator, history, e3_id) = + build_public_key_aggregator(awaiting_node_proofs(&[0, 1, 2], &[0, 2])).await?; + + aggregator.fail_on_missing_node_proofs( + &test_ctx(E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGTimeout, + }), + std::time::Duration::from_secs(1800), + ); + + let failed = next_event(&history).await?; + let InterfoldEventData::E3Failed(data) = failed.get_data() else { + panic!("an expired node-proof budget must publish E3Failed, got {failed:?}"); + }; + assert_eq!(data.e3_id, e3_id); + assert_eq!(data.reason, FailureReason::DKGTimeout); + assert_eq!(data.failed_at_stage, E3Stage::CommitteeFinalized); + Ok(()) +} + +/// The timer can fire after the last proof landed (cancel races delivery). That must be inert. +#[actix::test] +async fn expiring_the_budget_is_inert_once_every_proof_arrived() -> Result<()> { + let (mut aggregator, history, e3_id) = + build_public_key_aggregator(awaiting_node_proofs(&[0, 1, 2], &[0, 1, 2])).await?; + + aggregator.fail_on_missing_node_proofs( + &test_ctx(E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGTimeout, + }), + std::time::Duration::from_secs(1800), + ); + + // Nothing was published, so nothing to take. `TakeEvents` reports the timeout instead of + // hanging, which is exactly the assertion: a late timer publishes no event at all. + let result = history.send(TakeEvents::::new(1)).await?; + assert!( + result.timed_out && result.events.is_empty(), + "a late timer must not fail an E3 whose proofs all arrived, got {:?}", + result.events + ); + Ok(()) +} diff --git a/crates/aggregator/src/public_key_aggregation/transitions.rs b/crates/aggregator/src/public_key_aggregation/transitions.rs index d7e571ef82..e99d5487e6 100644 --- a/crates/aggregator/src/public_key_aggregation/transitions.rs +++ b/crates/aggregator/src/public_key_aggregation/transitions.rs @@ -145,6 +145,7 @@ impl PublicKeyAggregation { if !dishonest_parties.is_empty() { warn!( + e3_id = %e3_id, "Total dishonest parties (ZK + commitment): {:?}", dishonest_parties ); @@ -153,6 +154,7 @@ impl PublicKeyAggregation { // Fail closed when fewer than H parties cleared C1 — C5 cannot be witnessed. if honest_entries.len() < circuit_h { error!( + e3_id = %e3_id, "C5 requires {circuit_h} honest parties with valid C1 proofs; only {} honest after verification (collected {collected}, dishonest: {:?})", honest_entries.len(), dishonest_parties @@ -176,6 +178,7 @@ impl PublicKeyAggregation { // Defensive: should hold after truncation above; guard against future refactors. if honest_entries.len() <= threshold_m { error!( + e3_id = %e3_id, "Not enough honest parties after filtering: {} (need > {})", honest_entries.len(), threshold_m diff --git a/crates/ciphernode-builder/src/ciphernode.rs b/crates/ciphernode-builder/src/ciphernode.rs index 136170a592..ad26faf30c 100644 --- a/crates/ciphernode-builder/src/ciphernode.rs +++ b/crates/ciphernode-builder/src/ciphernode.rs @@ -8,6 +8,7 @@ use actix::Addr; use anyhow::{Context, Result}; use e3_data::{DataStore, InMemStore, StoreAddr}; use e3_events::{BusHandle, HistoryCollector, InterfoldEvent}; +use e3_evm::GatewayFailureReceiver; use e3_net::{NetChannelBridge, NetworkStatus}; use libp2p::PeerId; use std::{future::Future, time::Duration}; @@ -68,6 +69,9 @@ pub struct CiphernodeHandle { pub network_status: NetworkStatus, pub eventstore: EventStoreReader, pub aggregate_ids: Vec, + /// One receiver per chain gateway; yields a reason if that gateway fails closed after + /// startup. Empty when the node runs without chains. + pub gateway_failures: Vec, } impl PartialEq for CiphernodeHandle { @@ -125,6 +129,35 @@ impl CiphernodeHandle { None } + /// Resolve with the reason as soon as any chain gateway fails closed. + /// + /// A gateway that fails closed stops ingesting chain events but leaves the rest of the + /// node running: peers stay connected, the daemon and the DAppNode healthcheck both report + /// it healthy, and the operator's only signal is one log line. The gateway's own message + /// tells the operator to restart; the run loop uses this future to do that for them. + /// Pends forever when the node has no chain gateways. + pub fn gateway_failure(&self) -> impl Future + Send + 'static { + let mut receivers = self.gateway_failures.clone(); + async move { + if receivers.is_empty() { + return std::future::pending().await; + } + let waits = receivers.iter_mut().map(|rx| { + Box::pin(async move { + // `wait_for` yields immediately if a reason is already set, and resolves + // with an error if the gateway actor (the sender) is dropped. + match rx.wait_for(|reason| reason.is_some()).await { + Ok(reason) => reason + .clone() + .unwrap_or_else(|| "EVM chain gateway failed closed".to_owned()), + Err(_) => "EVM chain gateway stopped without reporting a reason".to_owned(), + } + }) + }); + futures::future::select_all(waits).await.0 + } + } + /// Stop protocol actors and make persisted state durable within `deadline`. /// /// The ordering is deliberate: the persisted `Shutdown` event first stops diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 103c7f822a..bc66e2bd9a 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -20,6 +20,7 @@ use e3_aggregator::ext::{ }; use e3_aggregator::{ CommitteeFinalizer, CommitteeFinalizerRecoveryState, CommitteeFinalizerRepositoryFactory, + FinalizerChainProvider, }; use e3_config::{chain_config::ChainConfig, NetworkProfile}; use e3_crypto::Cipher; @@ -33,8 +34,9 @@ use e3_evm::{ ensure_node_release, fetch_accusation_vote_validity, fetch_randomness_providers, BondingRegistrySolReader, CiphernodeRegistrySol, CiphernodeRegistrySolReader, DataAvailabilityCoordinator, DataAvailabilityRepositoryFactory, EvmChainGatewayHandle, - InterfoldSolReader, InterfoldSolWriter, ProviderConfig, RandomnessProviderSolReader, - SlashingManagerSolReader, SlashingManagerSolWriter, SlashingWriterRepositoryFactory, + GatewayFailureReceiver, InterfoldSolReader, InterfoldSolWriter, ProviderConfig, + RandomnessProviderSolReader, SlashingManagerSolReader, SlashingManagerSolWriter, + SlashingWriterRepositoryFactory, }; use e3_fhe::ext::FheExtension; use e3_keyshare::ext::ThresholdKeyshareExtension; @@ -661,7 +663,12 @@ impl CiphernodeBuilder { .filter(|chain| chain.enabled.unwrap_or(true)) { let provider = provider_cache.ensure_read_provider(chain).await?; - finalizer_providers.insert(provider.chain_id(), provider); + let factory = ProviderConfig::new(chain.rpc_url()?, chain.rpc_auth.clone()) + .into_read_provider_factory(); + finalizer_providers.insert( + provider.chain_id(), + FinalizerChainProvider::new(provider).with_factory(factory), + ); } CommitteeFinalizer::attach_with_recovery( &bus, @@ -728,6 +735,13 @@ impl CiphernodeBuilder { // selections remain dormant until SyncEffect, after EffectsEnabled attaches consumers. e3_builder.build().await?; + // Keep the failure receivers: `wait_for_evm_gateways` consumes the handles, but the + // run loop must still learn when a gateway fails closed after startup. + let gateway_failures: Vec = evm_gateways + .iter() + .map(EvmChainGatewayHandle::failure_receiver) + .collect(); + // Run the sync routine tokio::try_join!( sync_with_net_ready( @@ -753,6 +767,7 @@ impl CiphernodeBuilder { network_status, eventstore, aggregate_ids: eventstore_aggregate_config.indexed_ids(), + gateway_failures, }) } @@ -993,6 +1008,7 @@ impl CiphernodeBuilder { dkg_fold_context_by_chain.clone(), zk_recovery.clone(), self.proof_aggregation_enabled, + Some(store.clone()), ); } @@ -1019,6 +1035,7 @@ impl CiphernodeBuilder { dkg_fold_context_by_chain.clone(), zk_recovery, self.proof_aggregation_enabled, + Some(store.clone()), ); } } @@ -1058,6 +1075,7 @@ impl CiphernodeBuilder { info!("Setting up CommitmentConsistencyCheckerExtension"); e3_builder = e3_builder.with(CommitmentConsistencyCheckerExtension::create( bus, + &store, e3_zk_prover::default_links, )); } @@ -1389,10 +1407,12 @@ async fn setup_evm_system( } for randomness_address in randomness_addresses { let randomness_read_provider = provider.clone(); + let randomness_provider_factory = provider_factory.clone(); system.with_contract(randomness_address, move |next| { - RandomnessProviderSolReader::setup( + RandomnessProviderSolReader::setup_with_factory( &next, randomness_read_provider, + Some(randomness_provider_factory), contract_address, ) .recipient() diff --git a/crates/cli/src/cli.rs b/crates/cli/src/cli.rs index a2303b165d..cd1e9fe888 100644 --- a/crates/cli/src/cli.rs +++ b/crates/cli/src/cli.rs @@ -155,6 +155,12 @@ impl Cli { setup_simple_tracing(log_level); noir::execute_without_config(out, command).await? }, + Commands::Nodes { .. } => bail!( + "Configuration file not found. `interfold nodes ...` resolves \ + `interfold.config.yaml` from the current directory upwards, then from \ + the default config dir; run it from the directory that holds the \ + swarm's config, or pass `--config `." + ), _ => bail!( "Configuration file not found. Run `interfold ciphernode setup` to create a configuration." ), diff --git a/crates/cli/src/helpers/telemetry.rs b/crates/cli/src/helpers/telemetry.rs index 0d1c50a0d1..be6385182b 100644 --- a/crates/cli/src/helpers/telemetry.rs +++ b/crates/cli/src/helpers/telemetry.rs @@ -23,11 +23,38 @@ use tracing_subscriber::filter::Targets; use tracing_subscriber::fmt::format::{FormatFields, Writer}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, Layer}; +/// Level filter shared by every subscriber setup. +/// +/// `-vv` (DEBUG) is the level an operator reaches for when a node misbehaves, so it has to stay +/// readable. Dependency internals dominate it otherwise: in a measured 8-minute 5-node run, +/// `sled`'s page-cache alone emitted 8.9k of 39.4k DEBUG lines (31%), with multistream protocol +/// negotiation and the HTTP transport adding thousands more — all of it noise for diagnosing +/// protocol behaviour. Pin those crates to WARN so `-vv` shows Interfold's own diagnostics. +/// +/// Raise an individual dependency with `RUST_LOG` when you genuinely need its internals. +fn level_targets(log_level: Level) -> Targets { + Targets::new() + .with_default(log_level) + .with_target("alloy_pubsub", Level::WARN) + // Storage engine page-cache/IO-buffer churn. + .with_target("sled", Level::WARN) + // libp2p connection-establishment protocol negotiation. + .with_target("multistream_select", Level::WARN) + // Gossipsub/Kademlia poll-loop internals (mesh churn is surfaced by our own + // "Peer subscribed to topic" / "Peer disconnected" lines instead). + .with_target("libp2p_gossipsub", Level::WARN) + .with_target("libp2p_kad", Level::WARN) + .with_target("libp2p_swarm", Level::WARN) + // HTTP transport connection pooling behind the EVM provider. + .with_target("alloy_transport_http", Level::WARN) + .with_target("hyper", Level::WARN) + .with_target("hyper_util", Level::WARN) + .with_target("reqwest", Level::WARN) +} + pub fn setup_simple_tracing(log_level: Level) { LogCollector::init("interfold", None); - let targets = Targets::new() - .with_default(log_level) - .with_target("alloy_pubsub", Level::WARN); + let targets = level_targets(log_level); let _ = tracing_subscriber::registry() .with( tracing_subscriber::fmt::layer() @@ -45,9 +72,7 @@ pub fn setup_tracing(config: &AppConfig, log_level: Level) -> Result<()> { let name = config.name(); LogCollector::init(&name, Some(operational_log_path(config))); - let targets = Targets::new() - .with_default(log_level) - .with_target("alloy_pubsub", Level::WARN); + let targets = level_targets(log_level); match config.otel() { Some(endpoint) => { diff --git a/crates/cli/src/nodes.rs b/crates/cli/src/nodes.rs index dd4d8403f5..c4e72bb6ca 100644 --- a/crates/cli/src/nodes.rs +++ b/crates/cli/src/nodes.rs @@ -83,15 +83,15 @@ pub async fn execute( NodeCommands::Up { detach, exclude } => { nodes_up::execute(config, detach, exclude, verbose, config_string, otel).await? } - NodeCommands::Down => nodes_down::execute().await?, + NodeCommands::Down => nodes_down::execute(config).await?, NodeCommands::Ps => nodes_ps::execute().await?, NodeCommands::Daemon { exclude } => { nodes_daemon::execute(config, exclude, verbose, config_string, otel).await? } - NodeCommands::Start { id } => nodes_start::execute(&id).await?, - NodeCommands::Status { id } => nodes_status::execute(&id).await?, - NodeCommands::Stop { id } => nodes_stop::execute(&id).await?, - NodeCommands::Restart { id } => nodes_restart::execute(&id).await?, + NodeCommands::Start { id } => nodes_start::execute(config, &id).await?, + NodeCommands::Status { id } => nodes_status::execute(config, &id).await?, + NodeCommands::Stop { id } => nodes_stop::execute(config, &id).await?, + NodeCommands::Restart { id } => nodes_restart::execute(config, &id).await?, NodeCommands::Purge => nodes_purge::execute().await?, }; diff --git a/crates/cli/src/nodes_down.rs b/crates/cli/src/nodes_down.rs index 88e4f2be80..eb10f25f5d 100644 --- a/crates/cli/src/nodes_down.rs +++ b/crates/cli/src/nodes_down.rs @@ -5,9 +5,10 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::*; +use e3_config::AppConfig; use e3_entrypoint::nodes::down; -pub async fn execute() -> Result<()> { - down::execute().await?; +pub async fn execute(config: &AppConfig) -> Result<()> { + down::execute(config).await?; Ok(()) } diff --git a/crates/cli/src/nodes_restart.rs b/crates/cli/src/nodes_restart.rs index 0069c885af..3f83363c73 100644 --- a/crates/cli/src/nodes_restart.rs +++ b/crates/cli/src/nodes_restart.rs @@ -5,9 +5,10 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::*; +use e3_config::AppConfig; use e3_entrypoint::nodes::restart; -pub async fn execute(id: &str) -> Result<()> { - restart::execute(id).await?; +pub async fn execute(config: &AppConfig, id: &str) -> Result<()> { + restart::execute(config, id).await?; Ok(()) } diff --git a/crates/cli/src/nodes_start.rs b/crates/cli/src/nodes_start.rs index 843082475b..9c5af219b7 100644 --- a/crates/cli/src/nodes_start.rs +++ b/crates/cli/src/nodes_start.rs @@ -5,9 +5,10 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::*; +use e3_config::AppConfig; use e3_entrypoint::nodes::start; -pub async fn execute(id: &str) -> Result<()> { - start::execute(id).await?; +pub async fn execute(config: &AppConfig, id: &str) -> Result<()> { + start::execute(config, id).await?; Ok(()) } diff --git a/crates/cli/src/nodes_status.rs b/crates/cli/src/nodes_status.rs index 710c7e8393..c4f0b74fd4 100644 --- a/crates/cli/src/nodes_status.rs +++ b/crates/cli/src/nodes_status.rs @@ -5,9 +5,10 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::*; +use e3_config::AppConfig; use e3_entrypoint::nodes::status; -pub async fn execute(id: &str) -> Result<()> { - status::execute(id).await?; +pub async fn execute(config: &AppConfig, id: &str) -> Result<()> { + status::execute(config, id).await?; Ok(()) } diff --git a/crates/cli/src/nodes_stop.rs b/crates/cli/src/nodes_stop.rs index 55fb831c04..9ae9458188 100644 --- a/crates/cli/src/nodes_stop.rs +++ b/crates/cli/src/nodes_stop.rs @@ -5,9 +5,10 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::*; +use e3_config::AppConfig; use e3_entrypoint::nodes::stop; -pub async fn execute(id: &str) -> Result<()> { - stop::execute(id).await?; +pub async fn execute(config: &AppConfig, id: &str) -> Result<()> { + stop::execute(config, id).await?; Ok(()) } diff --git a/crates/cli/src/start.rs b/crates/cli/src/start.rs index 04e2435072..a3e7dc66e4 100644 --- a/crates/cli/src/start.rs +++ b/crates/cli/src/start.rs @@ -10,16 +10,19 @@ use crate::{ cli::{Cli, RemoteCli}, owo, }; -use anyhow::Result; +use anyhow::{bail, Result}; use e3_ciphernode_builder::CiphernodeHandle; use e3_config::AppConfig; use e3_console::Console; use e3_daemon_server::start_daemon_server; +use e3_events::NODE_SHUTDOWN_DEADLINE; use e3_utils::{colorize, Color}; use tokio::signal::unix::{signal, SignalKind}; use tracing::{error, info, instrument}; -const SHUTDOWN_DEADLINE: Duration = Duration::from_secs(30); +/// Wall-clock budget for the three-stage graceful shutdown. Defined next to the +/// subscriber-accept window it is derived from; see `NODE_SHUTDOWN_DEADLINE`. +const SHUTDOWN_DEADLINE: Duration = NODE_SHUTDOWN_DEADLINE; #[instrument(skip_all)] pub async fn execute(mut config: AppConfig, peers: Vec) -> Result<()> { @@ -95,8 +98,21 @@ pub async fn execute(mut config: AppConfig, peers: Vec) -> Result<()> { node.peer_id ); - shutdown.await; + // A chain gateway that fails closed after startup leaves a node that looks healthy to + // every supervisor while ingesting nothing. Its own error text says "restart the node"; + // do it: shut down cleanly and exit non-zero so `restart: unless-stopped` brings the + // node back to replay chain history. + let gateway_failure = node.gateway_failure(); + tokio::pin!(gateway_failure); + let gateway_failed = tokio::select! { + _ = &mut shutdown => None, + reason = &mut gateway_failure => Some(reason), + }; graceful_shutdown(Some(node)).await?; + if let Some(reason) = gateway_failed { + error!(%reason, "exiting so the supervisor can restart the node"); + bail!("{reason}"); + } Ok(()) } diff --git a/crates/entrypoint/src/nodes/client.rs b/crates/entrypoint/src/nodes/client.rs index 0b68bfc137..511142e38e 100644 --- a/crates/entrypoint/src/nodes/client.rs +++ b/crates/entrypoint/src/nodes/client.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use anyhow::Result; +use anyhow::{bail, Result}; use reqwest::Client; use std::env; use tracing::{error, trace}; @@ -72,7 +72,8 @@ pub async fn status(id: &str) -> Result<()> { } pub async fn ps() -> Result<()> { - let rows: Vec> = if let Ok(Query::Status { status }) = get_status().await { + let status = get_status().await; + let rows: Vec> = if let Ok(Query::Status { status }) = &status { status .processes .iter() @@ -83,6 +84,11 @@ pub async fn ps() -> Result<()> { }; print_table(&["PROCESS", "STATUS"], &rows); + if let Ok(Query::Status { status }) = &status { + if let Some(config_file) = &status.config_file { + println!("config: {config_file}"); + } + } Ok(()) } @@ -95,6 +101,30 @@ pub async fn is_ready() -> Result { Ok(true) } +/// Refuse to drive a daemon that was launched from a different config file. +/// +/// The control socket is a fixed loopback port shared by every checkout on the machine, and +/// node ids such as `cn1` are the documented defaults, so `nodes stop cn1` from one repo +/// would otherwise stop `cn1` in whichever swarm happens to own the port. Daemons that +/// predate the `config_file` field are accepted so an upgrade cannot lock an operator out. +pub async fn ensure_same_swarm(config_file: &std::path::Path) -> Result<()> { + let Query::Status { status } = get_status().await? else { + bail!("Swarm client is not ready. Did you forget to call `interfold nodes up`?"); + }; + let Some(daemon_config) = status.config_file else { + return Ok(()); + }; + let mine = config_file.display().to_string(); + if daemon_config != mine { + bail!( + "A swarm is already running on {SERVER_ADDRESS} for a different config.\n \ + running: {daemon_config}\n yours: {mine}\n\ + Run `interfold nodes down` from that config first, or use it for this command." + ); + } + Ok(()) +} + pub async fn start_daemon( verbose: u8, maybe_config_string: &Option, diff --git a/crates/entrypoint/src/nodes/daemon.rs b/crates/entrypoint/src/nodes/daemon.rs index d2b516fa87..4194521c13 100644 --- a/crates/entrypoint/src/nodes/daemon.rs +++ b/crates/entrypoint/src/nodes/daemon.rs @@ -135,7 +135,10 @@ pub async fn execute( maybe_otel, )?; - let process_manager = Arc::new(Mutex::new(ProcessManager::from(command_map))); + let process_manager = Arc::new(Mutex::new( + ProcessManager::from(command_map) + .with_config_file(config.config_file().display().to_string()), + )); process_manager.lock().await.start_all().await?; diff --git a/crates/entrypoint/src/nodes/down.rs b/crates/entrypoint/src/nodes/down.rs index 70fd1dc08c..226aec1f0e 100644 --- a/crates/entrypoint/src/nodes/down.rs +++ b/crates/entrypoint/src/nodes/down.rs @@ -5,16 +5,18 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::*; +use e3_config::AppConfig; use tracing::instrument; use super::client; #[instrument(skip_all)] -pub async fn execute() -> Result<()> { +pub async fn execute(config: &AppConfig) -> Result<()> { if !client::is_ready().await? { // not running! return Ok(()); } + client::ensure_same_swarm(&config.config_file()).await?; client::terminate().await?; diff --git a/crates/entrypoint/src/nodes/nodes.rs b/crates/entrypoint/src/nodes/nodes.rs index bfe095ba19..f7f82810fb 100644 --- a/crates/entrypoint/src/nodes/nodes.rs +++ b/crates/entrypoint/src/nodes/nodes.rs @@ -70,6 +70,11 @@ pub enum ProcessStatus { #[derive(Serialize, Deserialize, Clone, Debug)] pub struct SwarmStatus { pub processes: HashMap, + /// Config file the daemon was launched with. The control port is a fixed loopback + /// address, so a client started from a different checkout or config would otherwise + /// silently drive someone else's swarm. `None` only from daemons built before this field. + #[serde(default)] + pub config_file: Option, } #[cfg(test)] diff --git a/crates/entrypoint/src/nodes/process_manager.rs b/crates/entrypoint/src/nodes/process_manager.rs index c38076ca3e..a770b8185d 100644 --- a/crates/entrypoint/src/nodes/process_manager.rs +++ b/crates/entrypoint/src/nodes/process_manager.rs @@ -24,8 +24,18 @@ use super::nodes::{ spawn_process, CommandMap, ProcessMap, ProcessRecord, ProcessStatus, SwarmStatus, }; -const GRACEFUL_CHILD_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30); +/// How long the daemon waits for a child to exit after SIGTERM before SIGKILL. +/// +/// Must exceed the child's own graceful-shutdown budget, or the daemon kills it +/// mid store-flush and the persisted state the flush was protecting is lost. +const GRACEFUL_CHILD_SHUTDOWN_TIMEOUT: Duration = + Duration::from_secs(e3_events::NODE_SHUTDOWN_DEADLINE.as_secs() + 5); +const _: () = assert!( + GRACEFUL_CHILD_SHUTDOWN_TIMEOUT.as_secs() > e3_events::NODE_SHUTDOWN_DEADLINE.as_secs(), + "the daemon must outwait the node's own shutdown budget before escalating to SIGKILL" +); const OUTPUT_DRAIN_TIMEOUT: Duration = Duration::from_secs(5); +const EXIT_WATCH_INTERVAL: Duration = Duration::from_secs(2); /// Forward stdout from child process to parent's stdout fn forward_stdout(id: &str, stdout: ChildStdout) -> JoinHandle<()> { @@ -89,6 +99,54 @@ async fn run_command(id: &str, program: &str, args: Vec) -> Result JoinHandle<()> { + let id = id.to_owned(); + let processes = processes.clone(); + tokio::spawn(async move { + loop { + tokio::time::sleep(EXIT_WATCH_INTERVAL).await; + let mut guard = processes.lock().await; + let Some((child, _)) = guard.get_mut(&id) else { + // Removed by `stop`/`terminate`: expected. + return; + }; + match child.try_wait() { + Ok(None) => continue, + Ok(Some(status)) => { + error!( + process = %id, + exit_code = ?status.code(), + signal = ?std::os::unix::process::ExitStatusExt::signal(&status), + "SWARM child exited unexpectedly; it is NOT restarted automatically \ + (use `nodes start {id}`)" + ); + return; + } + Err(error) => { + warn!(process = %id, %error, "Failed to poll child exit status"); + return; + } + } + } + }) +} + +/// Attach the exit watcher to a freshly stored record. +async fn attach_exit_watcher(id: &str, processes: &ProcessMap) { + let watcher = watch_exit(id, processes); + if let Some((_, handlers)) = processes.lock().await.get_mut(id) { + handlers.push(watcher); + } else { + watcher.abort(); + } +} + /// Run commands as child processes and set up output forwarding async fn run_commands(commands: &CommandMap, processes: &ProcessMap) -> Result<()> { let commands = commands.clone(); @@ -105,7 +163,9 @@ async fn run_commands(commands: &CommandMap, processes: &ProcessMap) -> Result<( // Store the process let mut processes_guard = processes.lock().await; - processes_guard.insert(id, record); + processes_guard.insert(id.clone(), record); + drop(processes_guard); + attach_exit_watcher(&id, processes).await; } Ok(()) } @@ -138,6 +198,8 @@ async fn start(id: &str, commands: &CommandMap, processes: &ProcessMap) -> Resul let record = run_command(id, &program, args).await?; let mut processes_guard = processes.lock().await; processes_guard.insert(id.to_owned(), record); + drop(processes_guard); + attach_exit_watcher(id, processes).await; Ok(()) } @@ -262,9 +324,17 @@ fn setup_signal_handlers(manager: &ProcessManager) -> JoinHandle<()> { pub struct ProcessManager { commands: CommandMap, processes: ProcessMap, + /// Config file this swarm was launched from; reported in `/status` so clients can refuse + /// to act on a daemon that serves a different config. + config_file: Option, } impl ProcessManager { + pub fn with_config_file(mut self, config_file: impl Into) -> Self { + self.config_file = Some(config_file.into()); + self + } + pub async fn start_all(&self) -> Result<()> { run_commands(&self.commands, &self.processes).await?; Ok(()) @@ -318,7 +388,10 @@ impl ProcessManager { processes.insert(id.to_string(), self.status(id).await); } - SwarmStatus { processes } + SwarmStatus { + processes, + config_file: self.config_file.clone(), + } } } @@ -328,6 +401,7 @@ impl From for ProcessManager { let manager = Self { commands: value, processes, + config_file: None, }; setup_signal_handlers(&manager); @@ -415,4 +489,50 @@ mod tests { ); assert_eq!(manager.status("long").await, ProcessStatus::Stopped); } + + /// The exit watcher must stop polling once a record has been removed by `stop`, and must + /// still be attached (so it can be aborted) while the child is running. + #[tokio::test] + async fn exit_watcher_is_attached_and_released_with_the_record() { + let commands = CommandMap::from([( + "long".to_string(), + ( + "sh".to_string(), + vec!["-c".to_string(), "sleep 30".to_string()], + ), + )]); + let manager = ProcessManager::from(commands); + manager.start("long").await.unwrap(); + + // Two output forwarders plus the exit watcher. + assert_eq!( + manager.processes.lock().await.get("long").unwrap().1.len(), + 3 + ); + + manager.stop("long").await.unwrap(); + assert!(manager.processes.lock().await.get("long").is_none()); + } + + /// A child that dies on its own stays in the map as `Exited` (so `nodes ps` reports it) + /// and the watcher observes the exit rather than an operator stop. + #[tokio::test] + async fn crashed_child_is_still_reported_by_status() { + let commands = CommandMap::from([( + "crash".to_string(), + ( + "sh".to_string(), + vec!["-c".to_string(), "kill -9 $$".to_string()], + ), + )]); + let manager = ProcessManager::from(commands); + manager.start("crash").await.unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + + // Signal death has no exit code. + assert_eq!( + manager.status("crash").await, + ProcessStatus::Exited { code: None } + ); + } } diff --git a/crates/entrypoint/src/nodes/restart.rs b/crates/entrypoint/src/nodes/restart.rs index 988d8fed48..00f7e28c63 100644 --- a/crates/entrypoint/src/nodes/restart.rs +++ b/crates/entrypoint/src/nodes/restart.rs @@ -6,13 +6,15 @@ use super::client; use anyhow::*; +use e3_config::AppConfig; use tracing::instrument; #[instrument(skip_all)] -pub async fn execute(id: &str) -> Result<()> { +pub async fn execute(config: &AppConfig, id: &str) -> Result<()> { if !client::is_ready().await? { bail!("Swarm client is not ready. Did you forget to call `interfold nodes up`?"); } + client::ensure_same_swarm(&config.config_file()).await?; client::restart(id).await?; diff --git a/crates/entrypoint/src/nodes/start.rs b/crates/entrypoint/src/nodes/start.rs index 69827e3e64..4b659f0b7a 100644 --- a/crates/entrypoint/src/nodes/start.rs +++ b/crates/entrypoint/src/nodes/start.rs @@ -5,15 +5,17 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::*; +use e3_config::AppConfig; use tracing::instrument; use super::client; #[instrument(skip_all)] -pub async fn execute(id: &str) -> Result<()> { +pub async fn execute(config: &AppConfig, id: &str) -> Result<()> { if !client::is_ready().await? { bail!("Swarm client is not ready. Did you forget to call `interfold nodes up`?"); } + client::ensure_same_swarm(&config.config_file()).await?; client::start(id).await?; diff --git a/crates/entrypoint/src/nodes/status.rs b/crates/entrypoint/src/nodes/status.rs index c1d5142b1a..49fed13deb 100644 --- a/crates/entrypoint/src/nodes/status.rs +++ b/crates/entrypoint/src/nodes/status.rs @@ -5,15 +5,17 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::*; +use e3_config::AppConfig; use tracing::instrument; use super::client; #[instrument(skip_all)] -pub async fn execute(id: &str) -> Result<()> { +pub async fn execute(config: &AppConfig, id: &str) -> Result<()> { if !client::is_ready().await? { bail!("Swarm client is not ready. Did you forget to call `interfold nodes up`?"); } + client::ensure_same_swarm(&config.config_file()).await?; client::status(id).await?; diff --git a/crates/entrypoint/src/nodes/stop.rs b/crates/entrypoint/src/nodes/stop.rs index ac1cabe921..a98da299be 100644 --- a/crates/entrypoint/src/nodes/stop.rs +++ b/crates/entrypoint/src/nodes/stop.rs @@ -5,15 +5,17 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::*; +use e3_config::AppConfig; use tracing::instrument; use super::client; #[instrument(skip_all)] -pub async fn execute(id: &str) -> Result<()> { +pub async fn execute(config: &AppConfig, id: &str) -> Result<()> { if !client::is_ready().await? { bail!("Swarm client is not ready. Did you forget to call `interfold nodes up`?"); } + client::ensure_same_swarm(&config.config_file()).await?; client::stop(id).await?; diff --git a/crates/entrypoint/src/nodes/up.rs b/crates/entrypoint/src/nodes/up.rs index 8344afce91..c665cbbfb2 100644 --- a/crates/entrypoint/src/nodes/up.rs +++ b/crates/entrypoint/src/nodes/up.rs @@ -21,6 +21,8 @@ pub async fn execute( maybe_otel: Option, ) -> Result<()> { if client::is_ready().await? { + // Name the other swarm so the operator does not assume it is theirs. + client::ensure_same_swarm(&config.config_file()).await?; bail!("Swarm is already running!"); } diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index 99076d3489..89bb93a54c 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -40,7 +40,30 @@ const DEFAULT_DEDUP_CAPACITY: usize = 250_000; /// Actor handlers are expected to hand long-running work to child futures. A /// full mailbox that cannot accept one event within this window is unhealthy; /// replay fails closed instead of hanging startup forever. -const FANOUT_ACCEPT_TIMEOUT: Duration = Duration::from_secs(30); +/// +/// Also bounds the `Shutdown` fanout. The graceful-shutdown deadline in +/// `e3-cli` is sized above this so that a single wedged subscriber (a handler +/// blocked inside a `bb` proof) spends this window and the pipeline and store +/// flushes still get their own time; if the two were equal, the first wedged +/// subscriber would consume the whole deadline and the store flush would be +/// skipped. +pub const FANOUT_ACCEPT_TIMEOUT: Duration = Duration::from_secs(30); + +/// Wall-clock budget for a node's three-stage graceful shutdown (actor drain, +/// event flush, store flush). +/// +/// One full [`FANOUT_ACCEPT_TIMEOUT`] for the drain plus headroom for the two +/// flushes. The healthy path finishes in tens of milliseconds; the budget only +/// matters when a subscriber is wedged, in which case the drain burns the whole +/// accept window and the flushes must still complete so persisted state is +/// durable before exit. Anything that waits for a node to shut down (the swarm +/// daemon before escalating to SIGKILL) must allow at least this long. +pub const NODE_SHUTDOWN_DEADLINE: Duration = + Duration::from_secs(FANOUT_ACCEPT_TIMEOUT.as_secs() + 30); +const _: () = assert!( + NODE_SHUTDOWN_DEADLINE.as_secs() > FANOUT_ACCEPT_TIMEOUT.as_secs(), + "a wedged subscriber must not be able to consume the whole shutdown budget" +); /// A bounded, exact FIFO set. /// diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index a4e7ef5dac..2ce93f6254 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -16,7 +16,7 @@ use actix::{ use anyhow::{Context as _, Result}; use e3_utils::MAILBOX_LIMIT_LARGE; use std::collections::HashMap; -use tracing::{debug, error, warn}; +use tracing::{debug, error, trace, warn}; /// QueryAggregator - handles a single query's lifecycle struct QueryAggregator { @@ -140,8 +140,12 @@ impl EventStoreRouter { } pub fn handle_store_event_requested(&mut self, msg: StoreEventRequested) { - debug!("Handling store event requested...."); let aggregate_id = msg.event.aggregate_id(); + trace!( + aggregate = %aggregate_id, + event_type = ?msg.event.event_type_enum(), + "Routing event to its aggregate store" + ); let store_addr = self.stores.get(&aggregate_id).unwrap_or_else(|| { panic!( "No EventStore is configured for aggregate {aggregate_id}; refusing to write it to another aggregate" diff --git a/crates/events/src/interfold_event/publish_document/mod.rs b/crates/events/src/interfold_event/publish_document/mod.rs index 8498e3b2e5..4fb12cf5e6 100644 --- a/crates/events/src/interfold_event/publish_document/mod.rs +++ b/crates/events/src/interfold_event/publish_document/mod.rs @@ -13,7 +13,7 @@ use chrono::{serde::ts_seconds, DateTime, Duration, Utc}; use e3_utils::ArcBytes; pub use filter::Filter; use serde::{Deserialize, Serialize}; -use tracing::warn; +use tracing::debug; use crate::E3id; @@ -78,7 +78,12 @@ pub struct PublishDocumentRequested { impl PublishDocumentRequested { pub fn new(meta: DocumentMeta, value: ArcBytes) -> Self { - warn!("Publishing document that is {}", value.size()); + debug!( + e3_id = %meta.e3_id, + kind = ?meta.kind, + size_bytes = value.size(), + "Publishing document to the DHT" + ); Self { meta, value } } } diff --git a/crates/events/src/request_router_checkpoint.rs b/crates/events/src/request_router_checkpoint.rs index 5a763cc426..a5cfc1a631 100644 --- a/crates/events/src/request_router_checkpoint.rs +++ b/crates/events/src/request_router_checkpoint.rs @@ -14,4 +14,9 @@ pub struct RequestRouterCheckpoint { pub contexts: Vec, pub completed: HashSet, pub replay_cursors: HashMap, + /// E3s that failed with a slashable reason, keyed to the unix second after which the + /// accusation/slashing lifecycle can no longer act and the context may be torn down. + /// Absent in checkpoints written before this field existed. + #[serde(default)] + pub teardown_deadlines: HashMap, } diff --git a/crates/events/src/snapshot_buffer/timelock_queue.rs b/crates/events/src/snapshot_buffer/timelock_queue.rs index 920a49c202..d74a9e2f8f 100644 --- a/crates/events/src/snapshot_buffer/timelock_queue.rs +++ b/crates/events/src/snapshot_buffer/timelock_queue.rs @@ -13,7 +13,7 @@ use std::{ sync::Arc, time::{Duration, SystemTime, UNIX_EPOCH}, }; -use tracing::debug; +use tracing::{debug, trace}; use super::batch_router::{FlushSeq, SnapshotKey}; @@ -145,7 +145,7 @@ impl Actor for TimelockQueue { impl Handler for TimelockQueue { type Result = (); fn handle(&mut self, msg: StartTimelock, _: &mut Self::Context) -> Self::Result { - debug!("Start timelock: {:?}", msg.delay); + trace!(delay = ?msg.delay, "Start timelock"); let expiry = msg.now + msg.delay; self.timelocks.push(Reverse(Timelock::new(expiry, msg.key))); } @@ -155,13 +155,18 @@ impl Handler for TimelockQueue { type Result = (); fn handle(&mut self, _: Tick, _: &mut Self::Context) -> Self::Result { let now_time = Duration::from_micros(self.clock.now_micros()); - debug!( - "Running timelock tick. waiting times: {:?}.", - self.timelocks - .iter() - .map(|t| t.0.expiry.saturating_sub(now_time)) - .collect::>(), - ); + // An idle queue ticks continuously; logging every empty tick produced ~1.7k lines per + // 8-minute run saying nothing happened, and the Vec below was allocated even when the + // level filtered the line out. + if !self.timelocks.is_empty() { + debug!( + "Running timelock tick. waiting times: {:?}.", + self.timelocks + .iter() + .map(|t| t.0.expiry.saturating_sub(now_time)) + .collect::>(), + ); + } while !self.timelocks.is_empty() && self.next_timelock_lt(now_time) { if let Some(tl) = self.timelocks.pop() { diff --git a/crates/events/src/store_keys.rs b/crates/events/src/store_keys.rs index e1a66379aa..1a3b70638e 100644 --- a/crates/events/src/store_keys.rs +++ b/crates/events/src/store_keys.rs @@ -53,6 +53,17 @@ impl StoreKeys { format!("//context/{e3_id}") } + /// Durable verified-proof cache of the per-E3 commitment-consistency checker. + pub fn commitment_consistency(e3_id: &E3id) -> String { + format!("//commitment_consistency/v1/{e3_id}") + } + + /// The node's own signed C0 proof for an E3. Needed to re-seed the DKG node fold after a + /// restart, because C0 is never regenerated. + pub fn own_c0_proof(e3_id: &E3id) -> String { + format!("//own_c0_proof/v1/{e3_id}") + } + pub fn router() -> String { String::from("//router") } @@ -93,18 +104,6 @@ impl StoreKeys { String::from("//libp2p/keypair") } - pub fn interfold_sol_reader(chain_id: u64) -> String { - format!("//evm_readers/interfold/{chain_id}") - } - - pub fn ciphernode_registry_reader(chain_id: u64) -> String { - format!("//evm_readers/ciphernode_registry/{chain_id}") - } - - pub fn bonding_registry_reader(chain_id: u64) -> String { - format!("//evm_readers/bonding_registry/{chain_id}") - } - pub fn node_state() -> String { String::from("//node_state") } diff --git a/crates/evm/src/chain_gateway/actor.rs b/crates/evm/src/chain_gateway/actor.rs index d4a663c2a4..e54f8f2581 100644 --- a/crates/evm/src/chain_gateway/actor.rs +++ b/crates/evm/src/chain_gateway/actor.rs @@ -17,7 +17,7 @@ use e3_events::{ }; use e3_events::{Event, EventPublisher}; use e3_utils::MAILBOX_LIMIT; -use tokio::sync::oneshot; +use tokio::sync::{oneshot, watch}; use tracing::warn; /// Per-chain bound for events accumulated while the node is synchronizing. @@ -26,9 +26,18 @@ use tracing::warn; /// of dropping an observed chain event if this window is exhausted. pub const DEFAULT_MAX_BUFFERED_EVM_EVENTS: usize = 100_000; +/// Receives the reason when a gateway fails closed after it has gone live. +/// +/// Startup failures are reported through the readiness channel; this one covers the rest of +/// the process lifetime, when nothing else is awaiting the gateway. Without it a failed-closed +/// gateway leaves a node that is up, peered and reported healthy while dropping every chain +/// event. The value is `None` until a failure happens. +pub type GatewayFailureReceiver = watch::Receiver>; + pub struct EvmChainGatewayHandle { addr: Addr, readiness: oneshot::Receiver>, + failure: GatewayFailureReceiver, } impl EvmChainGatewayHandle { @@ -36,6 +45,11 @@ impl EvmChainGatewayHandle { self.addr.clone() } + /// A receiver that yields the failure reason if this gateway fails closed at any time. + pub fn failure_receiver(&self) -> GatewayFailureReceiver { + self.failure.clone() + } + pub async fn wait_until_live(self) -> Result<()> { self.readiness .await @@ -51,6 +65,7 @@ pub struct EvmChainGateway { status: SyncStatus>, max_buffered_events: usize, readiness: Option>>, + failure: watch::Sender>, } impl EvmChainGateway { @@ -63,11 +78,13 @@ impl EvmChainGateway { max_buffered_events: usize, readiness: Option>>, ) -> Self { + let (failure, _) = watch::channel(None); Self { bus: bus.clone(), status: SyncStatus::default(), max_buffered_events, readiness, + failure, } } @@ -85,8 +102,13 @@ impl EvmChainGateway { ) -> EvmChainGatewayHandle { let (tx, readiness) = oneshot::channel(); let actor = Self::with_options(bus, max_buffered_events, Some(tx)); + let failure = actor.failure.subscribe(); let addr = Self::start_and_subscribe(bus, actor); - EvmChainGatewayHandle { addr, readiness } + EvmChainGatewayHandle { + addr, + readiness, + failure, + } } fn start_and_subscribe(bus: &BusHandle, actor: Self) -> Addr { @@ -112,7 +134,9 @@ impl EvmChainGateway { ); self.status.fail(reason.clone()); self.signal_startup(Err(reason.clone())); - self.bus.err(EType::Evm, anyhow::anyhow!(reason)); + self.bus.err(EType::Evm, anyhow::anyhow!(reason.clone())); + // Post-startup consumers (the run loop) learn of the death through here. + let _ = self.failure.send(Some(reason)); ctx.stop(); } diff --git a/crates/evm/src/chain_gateway/tests.rs b/crates/evm/src/chain_gateway/tests.rs index c0545068ba..75b03fbe6c 100644 --- a/crates/evm/src/chain_gateway/tests.rs +++ b/crates/evm/src/chain_gateway/tests.rs @@ -37,6 +37,39 @@ async fn rejected_log_fails_gateway_readiness() -> Result<()> { Ok(()) } +/// The readiness channel only covers startup. A gateway that fails closed *after* going +/// live used to leave the node up, peered and reported healthy while dropping every chain +/// event (observed on all five nodes of a swarm after an RPC outage). The failure receiver +/// is what lets the run loop exit so the supervisor restarts the node. +#[actix::test] +async fn post_startup_failure_is_reported_through_the_failure_receiver() -> Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test-post-startup-failure"); + let gateway = EvmChainGateway::setup_with_readiness(&bus); + let mut failure = gateway.failure_receiver(); + let addr = gateway.addr(); + + // Nothing has failed yet. + assert!(failure.borrow().is_none()); + + addr.send(InterfoldEvmEvent::Rejected(EvmLogRejected::new( + CorrelationId::new(), + 1, + "backend connection task has stopped", + ))) + .await?; + + let reason = failure + .wait_for(|reason| reason.is_some()) + .await + .expect("the gateway must publish its failure before stopping") + .clone() + .unwrap(); + assert!(reason.contains("EVM chain gateway failed closed")); + assert!(reason.contains("backend connection task has stopped")); + Ok(()) +} + impl Actor for SyncEventCollector { type Context = actix::Context; } diff --git a/crates/evm/src/chain_reader/actor.rs b/crates/evm/src/chain_reader/actor.rs index d7a6f6c4ad..2264258f70 100644 --- a/crates/evm/src/chain_reader/actor.rs +++ b/crates/evm/src/chain_reader/actor.rs @@ -18,13 +18,11 @@ use alloy::providers::Provider; use alloy::rpc::types::Filter; use alloy_primitives::Address; use anyhow::anyhow; -use e3_events::{ - BusHandle, EType, ErrorDispatcher, Event, EventId, InterfoldEvent, InterfoldEventData, -}; +use e3_events::{BusHandle, EType, ErrorDispatcher, Event, InterfoldEvent, InterfoldEventData}; use e3_events::{EventSubscriber, EventType}; use e3_utils::{retry_with_backoff, RetryError, MAILBOX_LIMIT}; use futures_util::stream::StreamExt; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::time::Duration; use tokio::select; use tokio::sync::oneshot; @@ -49,12 +47,6 @@ const MAX_RETRIES_BEFORE_RECREATE: u32 = 3; /// blocks arrive, and those blocks need not contain any matching contract event. const CONFIRMED_BACKFILL_INTERVAL_SECS: u64 = 5; -#[derive(Default, serde::Serialize, serde::Deserialize, Clone)] -pub struct EvmReadInterfaceState { - pub ids: HashSet, - pub last_block: Option, -} - #[derive(Clone, Default)] pub struct Filters { historical: Filter, diff --git a/crates/evm/src/event_decoding/catalog.rs b/crates/evm/src/event_decoding/catalog.rs index 8784158152..e365d29359 100644 --- a/crates/evm/src/event_decoding/catalog.rs +++ b/crates/evm/src/event_decoding/catalog.rs @@ -34,6 +34,7 @@ pub(crate) fn find(contract: &str, topic0: B256) -> Option<&'static EvmEventDefi catalog(contract) .iter() .chain(retired_catalog(contract).iter()) + .chain(PROXY_LIFECYCLE.iter()) .find(|event| keccak256(event.signature.as_bytes()) == topic0) } @@ -47,6 +48,25 @@ fn catalog(contract: &str) -> &'static [EvmEventDefinition] { } } +/// ERC1967 proxy lifecycle events, emitted by the proxy itself rather than by the +/// implementation. +/// +/// Every watched contract is a proxy, but [`catalog`] is generated from — and asserted +/// against — the *implementation* ABIs, which never declare these. Without them a routine +/// deployment or upgrade decodes as `UnknownEvmLog`, which `e3_logger` reports at `warn` +/// (see its `EvmLogObserved(event) if !event.known` arm). Every fresh sync then prints +/// operator-facing warnings for entirely expected logs, which trains operators to ignore +/// the warning level that real faults use. +/// +/// Applies to all watched contracts, so this list is not keyed by contract name. These are +/// fixed by ERC1967 and are deliberately kept out of [`catalog`], whose ABI-equality test +/// must keep failing on genuine implementation drift. +const PROXY_LIFECYCLE: &[EvmEventDefinition] = &[ + EvmEventDefinition::new("Upgraded", "Upgraded(address)", None), + EvmEventDefinition::new("AdminChanged", "AdminChanged(address,address)", None), + EvmEventDefinition::new("BeaconUpgraded", "BeaconUpgraded(address)", None), +]; + /// Signatures that the current ABIs no longer emit, kept so already-mined logs stay readable. /// /// Renaming a Solidity event changes its `topic0`, but the contracts sit behind proxies: the @@ -795,4 +815,56 @@ mod tests { ); } } + + /// Every watched contract is a proxy, so a deployment or upgrade emits ERC1967 lifecycle + /// logs that the implementation ABI never declares. They must resolve on all of them, + /// otherwise a routine upgrade prints `UnknownEvmLog` at `warn` on every fresh sync. + #[test] + fn proxy_lifecycle_events_resolve_for_every_watched_contract() { + for contract in [ + "Interfold", + "BondingRegistry", + "CiphernodeRegistry", + "SlashingManager", + ] { + for (name, signature) in [ + ("Upgraded", "Upgraded(address)"), + ("AdminChanged", "AdminChanged(address,address)"), + ("BeaconUpgraded", "BeaconUpgraded(address)"), + ] { + let definition = find(contract, keccak256(signature.as_bytes())) + .unwrap_or_else(|| panic!("{contract}: {signature} must resolve")); + assert_eq!(definition.name, name); + assert!( + definition.e3_id_topic.is_none(), + "proxy lifecycle events carry no e3Id" + ); + } + } + } + + /// The proxy list is consulted for every contract, so a collision would make a real + /// implementation event decode under the wrong name. + #[test] + fn proxy_lifecycle_events_never_shadow_a_contract_event() { + for contract in [ + "Interfold", + "BondingRegistry", + "CiphernodeRegistry", + "SlashingManager", + ] { + let owned: HashSet = catalog(contract) + .iter() + .chain(retired_catalog(contract).iter()) + .map(|event| keccak256(event.signature.as_bytes())) + .collect(); + for proxy_event in PROXY_LIFECYCLE { + assert!( + !owned.contains(&keccak256(proxy_event.signature.as_bytes())), + "{contract}: proxy signature {} collides with a contract event", + proxy_event.signature + ); + } + } + } } diff --git a/crates/evm/src/event_router.rs b/crates/evm/src/event_router.rs index 393d3559c1..0bf22b276b 100644 --- a/crates/evm/src/event_router.rs +++ b/crates/evm/src/event_router.rs @@ -9,7 +9,7 @@ use actix::{Actor, Handler}; use alloy_primitives::Address; use e3_utils::MAILBOX_LIMIT; use std::collections::HashMap; -use tracing::{debug, error, info}; +use tracing::{error, info, trace}; /// Directs InterfoldEvmEvent::Log events to the correct upstream processors. Drops all other event /// types @@ -62,7 +62,7 @@ impl Handler for EvmRouter { InterfoldEvmEvent::Log(EvmLog { log, chain_id, .. }) => { let address = log.address(); if let Some(dest) = self.routing_table.get(&address) { - debug!("Found address {address} in routing table forwarding to destination."); + trace!("Found address {address} in routing table forwarding to destination."); dest.do_send(msg); } else { error!( diff --git a/crates/evm/src/helpers.rs b/crates/evm/src/helpers.rs index 3a5c9e3a71..228555ea9a 100644 --- a/crates/evm/src/helpers.rs +++ b/crates/evm/src/helpers.rs @@ -221,6 +221,12 @@ impl ProviderConfig { .max_frame_size(Some(32 * 1024 * 1024)) .max_message_size(Some(32 * 1024 * 1024)); + // alloy's pubsub service retries a dropped WebSocket 10 × 3 s and then answers every + // request with "backend connection task has stopped" for the life of the provider. + // That bound is deliberate here: it is the signal the chain reader's recreate path + // needs to backfill the blocks that were mined during the outage. Every other actor + // that holds a provider clone must therefore own a `ProviderFactory` and reconnect + // itself — see `RandomnessProviderSolReader` and `CommitteeFinalizer`. let mut ws_connect = WsConnect::new(self.rpc.as_ws_url()?).with_config(config); if let Some(auth) = self.auth.to_ws_auth() { @@ -263,12 +269,15 @@ pub async fn load_signer_from_repository( repository: Repository>, cipher: &Cipher, ) -> Result { - let encrypted_key = repository - .read() - .await? - .context("No private key found in repository")?; - - let mut decrypted = cipher.decrypt_data(&encrypted_key)?; + let encrypted_key = repository.read().await?.context( + "no operator wallet key is stored for this node. Add one with \ + `interfold wallet set --name --config --private-key `", + )?; + + let mut decrypted = cipher.decrypt_data(&encrypted_key).context( + "the stored operator wallet key could not be decrypted. This usually means the node \ + password does not match the one used to store the key", + )?; let private_key = Zeroizing::new(hex::encode(&decrypted)); decrypted.zeroize(); private_key.parse().map_err(Into::into) diff --git a/crates/evm/src/randomness_provider/actor.rs b/crates/evm/src/randomness_provider/actor.rs index 2c4093c2fc..540fe6968c 100644 --- a/crates/evm/src/randomness_provider/actor.rs +++ b/crates/evm/src/randomness_provider/actor.rs @@ -9,7 +9,7 @@ use crate::contracts::{ICiphernodeRegistry, IRandomnessProvider}; use crate::domain::log_timestamp::from_log_chain_id_to_ts; use crate::domain::randomness_provider_events::{committee_requested, SortitionRequestContext}; -use crate::helpers::EthProvider; +use crate::helpers::{EthProvider, ProviderFactory}; use crate::messages::{EvmEvent, EvmEventProcessor, EvmLog, EvmLogRejected, InterfoldEvmEvent}; use actix::prelude::*; use alloy::{ @@ -28,6 +28,13 @@ const REGISTRY_ACCEPTANCE_TIMEOUT: Duration = Duration::from_secs(15); pub struct RandomnessProviderSolReader

{ provider: EthProvider

, + /// Rebuilds the read provider when its transport has died. + /// + /// The live-log stream owns its own provider and recreates it after an RPC outage, but + /// this reader was handed a separate clone at startup. A WebSocket provider whose backend + /// task has stopped fails every call forever, so without a factory the first + /// `RandomnessFulfilled` after an outage is rejected and the chain gateway fails closed. + provider_factory: Option>, registry: Address, next: EvmEventProcessor, } @@ -54,9 +61,19 @@ impl RandomnessProviderSolReader

{ next: &EvmEventProcessor, provider: EthProvider

, registry: Address, + ) -> Addr { + Self::setup_with_factory(next, provider, None, registry) + } + + pub fn setup_with_factory( + next: &EvmEventProcessor, + provider: EthProvider

, + provider_factory: Option>, + registry: Address, ) -> Addr { Self { provider, + provider_factory, registry, next: next.clone(), } @@ -64,6 +81,49 @@ impl RandomnessProviderSolReader

{ } } +/// Parse a randomness log, replacing the provider once if the registry reads fail. +/// +/// Returns the provider to keep so the actor can adopt a reconnected one. The retry is +/// bounded to a single reconnect: a healthy transport that still fails is a real rejection +/// and must fail closed as before. +async fn parse_fulfillment_with_reconnect( + provider: EthProvider

, + provider_factory: Option>, + registry_address: Address, + log: EvmLog, +) -> (EthProvider

, Result>) { + let first = parse_fulfillment(provider.clone(), registry_address, log.clone()).await; + let (Err(first_error), Some(factory)) = (&first, provider_factory.as_ref()) else { + return (provider, first); + }; + warn!( + id = %log.id, + chain_id = log.chain_id, + error = %first_error, + "Randomness log verification failed; reconnecting the read provider and retrying once" + ); + let replacement = match factory().await { + Ok(replacement) if replacement.chain_id() == log.chain_id => replacement, + Ok(replacement) => { + warn!( + expected_chain_id = log.chain_id, + actual_chain_id = replacement.chain_id(), + "Refusing a reconnected randomness provider for another chain" + ); + return (provider, first); + } + Err(reconnect_error) => { + warn!( + error = %reconnect_error, + "Unable to reconnect the randomness read provider" + ); + return (provider, first); + } + }; + let second = parse_fulfillment(replacement.clone(), registry_address, log).await; + (replacement, second) +} + async fn parse_fulfillment( provider: EthProvider

, registry_address: Address, @@ -155,7 +215,7 @@ async fn read_accepted_sortition( let registry = ICiphernodeRegistry::new(registry_address, provider.provider()); let pinned = match registry .sortitionSeed(e3_id) - .block(event_block.clone()) + .block(event_block) .call() .await { @@ -232,11 +292,18 @@ impl Handler for RandomnessPro let id = log.id; let chain_id = log.chain_id; let provider = self.provider.clone(); + let provider_factory = self.provider_factory.clone(); let registry = self.registry; let next = self.next.clone(); ctx.wait( async move { - let parsed = parse_fulfillment(provider, registry, log).await; + let (provider, parsed) = parse_fulfillment_with_reconnect( + provider, + provider_factory, + registry, + log, + ) + .await; let event = match parsed { Ok(Some(event)) => InterfoldEvmEvent::Event(event), Ok(None) => InterfoldEvmEvent::Processed(id), @@ -254,10 +321,12 @@ impl Handler for RandomnessPro )) } }; - forward(next, event).await + let result = forward(next, event).await; + (provider, result) } .into_actor(self) - .map(|result, _, ctx| { + .map(|(provider, result), actor, ctx| { + actor.provider = provider; if let Err(forward_error) = result { error!( error = %forward_error, @@ -296,9 +365,17 @@ mod tests { primitives::Bytes, providers::ProviderBuilder, sol_types::SolValue, transports::mock::Asserter, }; + use std::sync::Arc; async fn provider(asserter: &Asserter) -> EthProvider { - asserter.push_success(&"0x1"); + provider_on_chain(asserter, "0x1").await + } + + async fn provider_on_chain( + asserter: &Asserter, + chain_id_hex: &str, + ) -> EthProvider { + asserter.push_success(&chain_id_hex); EthProvider::new(ProviderBuilder::new().connect_mocked_client(asserter.clone())) .await .expect("mock chain ID must decode") @@ -371,6 +448,135 @@ mod tests { assert!(accepted.is_none()); } + fn fulfilled_log(chain_id: u64) -> EvmLog { + let data = IRandomnessProvider::RandomnessFulfilled { + requestId: U256::from(8), + e3Id: U256::from(7), + randomWord: U256::from(9), + fulfilledAt: U256::from(10), + } + .encode_log_data(); + let mut log = alloy::rpc::types::Log { + inner: alloy::primitives::Log { + address: Address::ZERO, + data, + }, + ..Default::default() + }; + log.block_number = Some(10); + log.block_hash = Some(alloy::primitives::B256::repeat_byte(1)); + log.log_index = Some(0); + EvmLog::new(log, chain_id, 1_700_000_000) + } + + fn accepted_state(asserter: &Asserter) { + asserter.push_success(&Bytes::from((true, U256::from(9)).abi_encode())); + asserter.push_success(&Bytes::from( + ([2u32, 3u32], U256::from(4), U256::from(5), U256::from(6)).abi_encode(), + )); + } + + /// After an RPC outage the live-log stream rebuilds its provider, but this reader kept + /// the original one. Observed on a 5-node swarm: anvil down 90 s, back up, every node's + /// stream resubscribed — then the first `RandomnessFulfilled` was rejected with "backend + /// connection task has stopped" and all five chain gateways failed closed. The reader + /// must reconnect through the factory and retry before rejecting. + #[tokio::test] + async fn dead_transport_is_replaced_before_rejecting_a_fulfillment() { + let dead = Asserter::new(); + let dead_provider = provider(&dead).await; + // Every registry read on the dead transport fails, pinned and retained alike. + dead.push_failure_msg("backend connection task has stopped"); + dead.push_failure_msg("backend connection task has stopped"); + + let healthy = Asserter::new(); + let healthy_provider = provider(&healthy).await; + accepted_state(&healthy); + let factory_provider = healthy_provider.clone(); + let factory: ProviderFactory<_> = Arc::new(move || { + let provider = factory_provider.clone(); + Box::pin(async move { Ok(provider) }) + }); + + let (kept, parsed) = parse_fulfillment_with_reconnect( + dead_provider, + Some(factory), + Address::ZERO, + fulfilled_log(1), + ) + .await; + + let event = parsed + .expect("the reconnected provider must verify the fulfillment") + .expect("an accepted fulfillment must produce a committee request"); + assert_eq!(event.chain_id(), 1); + // The actor keeps the reconnected provider for the next log: a further read on it + // is served by the healthy asserter, not the dead one. + healthy.push_success(&"0x2a"); + let head = kept.provider().get_block_number().await.unwrap(); + assert_eq!(head, 42); + } + + /// A reconnect must not paper over a real rejection: if the healthy transport also + /// fails the verification, fail closed exactly as before. + #[tokio::test] + async fn reconnect_does_not_mask_a_genuine_rejection() { + let dead = Asserter::new(); + let dead_provider = provider(&dead).await; + dead.push_failure_msg("backend connection task has stopped"); + dead.push_failure_msg("backend connection task has stopped"); + + let healthy = Asserter::new(); + let healthy_provider = provider(&healthy).await; + healthy.push_failure_msg("historical state unavailable"); + healthy.push_success(&Bytes::from((false, U256::ZERO).abi_encode())); + let factory: ProviderFactory<_> = Arc::new(move || { + let provider = healthy_provider.clone(); + Box::pin(async move { Ok(provider) }) + }); + + let (_, parsed) = parse_fulfillment_with_reconnect( + dead_provider, + Some(factory), + Address::ZERO, + fulfilled_log(1), + ) + .await; + + let error = parsed.expect_err("an unverifiable state must still fail closed"); + assert!(error.to_string().contains("not verifiable")); + } + + /// A factory that returns a provider for a different chain is refused and the original + /// error stands. + #[tokio::test] + async fn reconnect_refuses_a_provider_for_another_chain() { + let dead = Asserter::new(); + let dead_provider = provider_on_chain(&dead, "0x1").await; + dead.push_failure_msg("backend connection task has stopped"); + dead.push_failure_msg("backend connection task has stopped"); + + let other = Asserter::new(); + let other_provider = provider_on_chain(&other, "0x2").await; + let factory: ProviderFactory<_> = Arc::new(move || { + let provider = other_provider.clone(); + Box::pin(async move { Ok(provider) }) + }); + + let (_, parsed) = parse_fulfillment_with_reconnect( + dead_provider, + Some(factory), + Address::ZERO, + fulfilled_log(1), + ) + .await; + + let error = parsed.expect_err("a wrong-chain reconnect must not be used"); + assert!(error + .to_string() + .contains("backend connection task has stopped")); + } + #[tokio::test(start_paused = true)] async fn bounds_registry_acceptance_reads() { let error = await_registry_acceptance( diff --git a/crates/evm/src/repo.rs b/crates/evm/src/repo.rs index b66602603d..aeb0f8db5b 100644 --- a/crates/evm/src/repo.rs +++ b/crates/evm/src/repo.rs @@ -7,7 +7,7 @@ use e3_data::{Repositories, Repository}; use e3_events::StoreKeys; -use crate::{DataAvailabilityRecoveryState, EvmReadInterfaceState, SlashingWriterRecoveryState}; +use crate::{DataAvailabilityRecoveryState, SlashingWriterRecoveryState}; pub trait EthPrivateKeyRepositoryFactory { fn eth_private_key(&self) -> Repository>; @@ -19,41 +19,12 @@ impl EthPrivateKeyRepositoryFactory for Repositories { } } -pub trait InterfoldSolReaderRepositoryFactory { - fn interfold_sol_reader(&self, chain_id: u64) -> Repository; -} - -impl InterfoldSolReaderRepositoryFactory for Repositories { - fn interfold_sol_reader(&self, chain_id: u64) -> Repository { - Repository::new(self.store.scope(StoreKeys::interfold_sol_reader(chain_id))) - } -} - -pub trait CiphernodeRegistryReaderRepositoryFactory { - fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository; -} - -impl CiphernodeRegistryReaderRepositoryFactory for Repositories { - fn ciphernode_registry_reader(&self, chain_id: u64) -> Repository { - Repository::new( - self.store - .scope(StoreKeys::ciphernode_registry_reader(chain_id)), - ) - } -} - -pub trait BondingRegistryReaderRepositoryFactory { - fn bonding_registry_reader(&self, chain_id: u64) -> Repository; -} - -impl BondingRegistryReaderRepositoryFactory for Repositories { - fn bonding_registry_reader(&self, chain_id: u64) -> Repository { - Repository::new( - self.store - .scope(StoreKeys::bonding_registry_reader(chain_id)), - ) - } -} +// Note: the EVM read cursor is NOT stored here. Each aggregate's last ingested block is +// persisted by the snapshot batch router under `StoreKeys::aggregate_block` and restored +// through `SnapshotMeta::to_evm_config` at boot, which is what `HistoricalEvmSyncStart` +// hands to the chain reader as its `from_block`. A previous per-contract +// `EvmReadInterfaceState` repository was declared here but never read or written; it was +// removed so nobody mistakes its absence for "the node re-scans from deploy_block". pub trait SlashingWriterRepositoryFactory { fn slashing_writer_recovery(&self, chain_id: u64) -> Repository; diff --git a/crates/keyshare/src/threshold_keyshare/actor.rs b/crates/keyshare/src/threshold_keyshare/actor.rs index 68dc797b9c..3991efb5a3 100644 --- a/crates/keyshare/src/threshold_keyshare/actor.rs +++ b/crates/keyshare/src/threshold_keyshare/actor.rs @@ -42,7 +42,7 @@ use std::{ collections::{BTreeSet, HashMap, HashSet}, sync::Arc, }; -use tracing::{error, info, trace, warn}; +use tracing::{debug, error, info, trace, warn}; use crate::actors::decryption_key_shared_collector::{ AllDecryptionKeySharesCollected, DecryptionKeySharedCollectionFailed, @@ -139,6 +139,15 @@ struct PendingKeyshareWork { own_dkg_shares: Option<(SensitiveBytes, Vec)>, /// C4 completed before the signed C1 artifact became available. keyshare_publish: bool, + /// The share collector finished before this node's own DKG reached aggregation. + /// + /// Peers' shares can complete the collector while the local state is still + /// `CollectingEncryptionKeys` or `GeneratingThresholdShare` — routinely after a restart, + /// because peers that already hold this node's key finish ahead of it, and because + /// recovery rebuilds the collector from persisted peer shares before it redrives local + /// work. The collector cancels its timeout and never re-emits, so this message must be + /// kept until the state can consume it or the DKG stalls with no timer left to fail it. + early_all_shares_collected: Option>, } pub struct ThresholdKeyshare { diff --git a/crates/keyshare/src/threshold_keyshare/effects/calculate_decryption_key.rs b/crates/keyshare/src/threshold_keyshare/effects/calculate_decryption_key.rs index 5ad7aa10a5..127f768b14 100644 --- a/crates/keyshare/src/threshold_keyshare/effects/calculate_decryption_key.rs +++ b/crates/keyshare/src/threshold_keyshare/effects/calculate_decryption_key.rs @@ -80,6 +80,21 @@ impl ThresholdKeyshare { self_addr: Addr, ) -> Result<()> { let (res, ec) = res.into_components(); + + // A restart inside the DKG window can drive this compute twice: once from the + // event-store replay of the original `ComputeRequest` and once from the re-driven + // `ShareVerificationComplete`. The gate cannot dedup them (different correlation + // ids). The first response wins and moves us to `ReadyForDecryption`; the second is + // a no-op, not an error — the node is healthy and the key is already derived. + let state = self.state.try_get()?; + if !matches!(state.state, KeyshareState::AggregatingDecryptionKey(_)) { + debug!( + "Ignoring CalculateDecryptionKey response in {:?}; key already derived", + state.state.variant_name() + ); + return Ok(()); + } + let output: CalculateDecryptionKeyResponse = res .try_into() .context("Error extracting data from compute process")?; @@ -102,7 +117,6 @@ impl ThresholdKeyshare { // Accept the C4 proof intent before advancing the primary phase. A crash cannot then // leave ReadyForDecryption without the input required to recreate its proof job. - let state = self.state.try_get()?; let e3_id = state.get_e3_id(); let party_id = state.party_id; let node = state.address.clone(); diff --git a/crates/keyshare/src/threshold_keyshare/effects/generate_threshold_share.rs b/crates/keyshare/src/threshold_keyshare/effects/generate_threshold_share.rs index 298bbddf5b..0a0e75b6be 100644 --- a/crates/keyshare/src/threshold_keyshare/effects/generate_threshold_share.rs +++ b/crates/keyshare/src/threshold_keyshare/effects/generate_threshold_share.rs @@ -89,6 +89,9 @@ impl ThresholdKeyshare { }, )) })?; + + // Peers may have completed the collector while we were still generating. + self.flush_early_all_shares_collected()?; } Ok(()) } diff --git a/crates/keyshare/src/threshold_keyshare/effects/recovery.rs b/crates/keyshare/src/threshold_keyshare/effects/recovery.rs index aaa3b38714..449503103e 100644 --- a/crates/keyshare/src/threshold_keyshare/effects/recovery.rs +++ b/crates/keyshare/src/threshold_keyshare/effects/recovery.rs @@ -101,7 +101,14 @@ impl ThresholdKeyshare { recovery: &ThresholdKeyshareRecoveryState, self_addr: Addr, ) -> Result<()> { - if recovery.decryption_key_shares.is_empty() { + // Mirror the live path (`calculate_decryption_key.rs`): the collector exists whenever + // other honest parties are expected, regardless of how many shares have arrived. A + // restart in `ReadyForDecryption` before the first peer share used to skip the + // collector entirely; the first share then hit "no collector (sole honest party)" in + // `route_events` and was dropped, and with no collector there was no timeout either. + // The collector's deadline is resolved from `dkg_started_at_unix_secs`, so re-creating + // it after a restart keeps the original DKG-relative timeout. + if !self.expects_peer_decryption_key_shares()? { return Ok(()); } let collector = self.ensure_decryption_key_shared_collector(self_addr)?; @@ -111,6 +118,16 @@ impl ThresholdKeyshare { Ok(()) } + /// Whether any honest party other than this node still owes a `DecryptionKeyShared`. + fn expects_peer_decryption_key_shares(&self) -> Result { + let state = self.state.try_get()?; + let my_party_id = state.party_id; + Ok(state + .honest_parties + .as_ref() + .is_some_and(|honest| honest.iter().any(|&pid| pid != my_party_id))) + } + fn resume_generating_threshold_share( &mut self, data: GeneratingThresholdShareData, @@ -169,7 +186,10 @@ impl ThresholdKeyshare { signed_e_sm_share_encryption_proofs: Vec::new(), }, )) - }) + })?; + // The recovery arm replays persisted peer shares into the collector before this + // resume runs, so the collector can already have completed. + self.flush_early_all_shares_collected() } /// Re-create interrupted collectors and process-local jobs from their persisted inputs. diff --git a/crates/keyshare/src/threshold_keyshare/effects/verify_threshold_shares.rs b/crates/keyshare/src/threshold_keyshare/effects/verify_threshold_shares.rs index 4a4a4dd43b..59643b2d43 100644 --- a/crates/keyshare/src/threshold_keyshare/effects/verify_threshold_shares.rs +++ b/crates/keyshare/src/threshold_keyshare/effects/verify_threshold_shares.rs @@ -9,6 +9,41 @@ impl ThresholdKeyshare { pub fn handle_all_threshold_shares_collected( &mut self, msg: TypedEvent, + ) -> Result<()> { + // The collector is fed by peer shares and can finish before this node's own DKG has + // advanced the state to `AggregatingDecryptionKey`. That happens whenever the node is + // behind its peers: after a restart it can still be collecting encryption keys while + // peers, who already have its key, finish and send their shares. Keep the message: the + // collector has already cancelled its timeout and will not send it again, and dropping + // it here would leave the DKG with no path to completion and no timer to fail it. + if matches!( + self.state.try_get()?.state, + KeyshareState::CollectingEncryptionKeys(_) | KeyshareState::GeneratingThresholdShare(_) + ) { + info!( + "AllThresholdSharesCollected arrived before own DKG reached aggregation; \ + holding it until the state advances" + ); + self.pending.early_all_shares_collected = Some(msg); + return Ok(()); + } + self.apply_all_threshold_shares_collected(msg) + } + + /// Consume a held `AllThresholdSharesCollected` once the state can accept it. + pub(in crate::actors::threshold_keyshare) fn flush_early_all_shares_collected( + &mut self, + ) -> Result<()> { + if let Some(msg) = self.pending.early_all_shares_collected.take() { + info!("Applying the held AllThresholdSharesCollected after own share generation"); + return self.apply_all_threshold_shares_collected(msg); + } + Ok(()) + } + + fn apply_all_threshold_shares_collected( + &mut self, + msg: TypedEvent, ) -> Result<()> { let (msg, ec) = msg.into_components(); info!("AllThresholdSharesCollected"); diff --git a/crates/keyshare/src/threshold_keyshare/tests.rs b/crates/keyshare/src/threshold_keyshare/tests.rs index 2bf5550afe..8a53a486c3 100644 --- a/crates/keyshare/src/threshold_keyshare/tests.rs +++ b/crates/keyshare/src/threshold_keyshare/tests.rs @@ -12,13 +12,17 @@ use anyhow::Result; use e3_crypto::Cipher; use e3_data::{AutoPersist, DataStore, InMemStore, Persistable, Repository}; use e3_events::{ - hlc_factory::HlcFactory, BusHandle, ComputeRequestKind, E3Stage, E3id, EffectsEnabled, - EventBus, EventBusConfig, EventSource, FailureReason, GetEvents, HistoryCollector, - InterfoldEvent, InterfoldEventData, Sequencer, StoreEventRequested, StoreEventResponse, - TakeEvents, Unsequenced, + hlc_factory::HlcFactory, BusHandle, ComputeRequestKind, ComputeResponse, CorrelationId, + E3Stage, E3id, EffectsEnabled, EventBus, EventBusConfig, EventSource, FailureReason, GetEvents, + HistoryCollector, InterfoldEvent, InterfoldEventData, Seed, Sequencer, StoreEventRequested, + StoreEventResponse, TakeEvents, Unsequenced, }; use e3_fhe_params::DEFAULT_BFV_PRESET; use std::sync::Arc; +use std::time::Duration; + +use actix::clock::sleep; +use anyhow::bail; #[derive(Default)] struct TestEventStore { @@ -361,3 +365,283 @@ async fn restart_skips_dkg_work_after_public_key_context_is_persisted() -> Resul assert!(events.is_empty(), "restart replayed superseded DKG work"); Ok(()) } + +/// After a restart in `ReadyForDecryption` with no peer `DecryptionKeyShared` yet persisted, +/// recovery used to skip the collector (it only rebuilt one when there were shares to +/// replay). The first peer share then hit the "no collector (sole honest party)" branch in +/// `route_events` and was dropped, and with no collector there was no decryption timeout +/// either — the node sat in `ReadyForDecryption` forever. The collector must exist whenever +/// peer shares are expected. Here the DKG start is long past, so the rebuilt collector's +/// DKG-relative timeout fires at once and the observable proof is the failure it emits. +#[actix::test] +async fn restart_ready_for_decryption_without_shares_rebuilds_the_collector() -> Result<()> { + let ready = ReadyForDecryption { + pk_share: ArcBytes::from_bytes(&[1]), + sk_poly_sum: SensitiveBytes::from_encrypted(&[2]), + es_poly_sum: vec![SensitiveBytes::from_encrypted(&[3])], + signed_pk_generation_proof: None, + signed_sk_share_computation_proof: None, + signed_e_sm_share_computation_proof: None, + signed_sk_share_encryption_proofs: Vec::new(), + signed_e_sm_share_encryption_proofs: Vec::new(), + }; + let (bus, history) = test_bus(); + let e3_id = E3id::new("42", 1); + let store = InMemStore::new(false).start(); + let repo = Repository::::new(DataStore::from_in_mem(&store)); + let mut state = ThresholdKeyshareState::new( + e3_id.clone(), + 0, + KeyshareState::ReadyForDecryption(ready), + 1, + 3, + ArcBytes::from_bytes(b"params"), + Address::ZERO.to_string(), + ); + // Two honest peers owe us a share; the DKG started long ago so the timeout is due. + state.honest_parties = Some([0, 1, 2].into_iter().collect()); + state.dkg_started_at_unix_secs = Some(1); + let actor = ThresholdKeyshare::new(ThresholdKeyshareParams { + bus, + cipher: Arc::new(Cipher::from_password("test-password").await?), + state: repo.send(Some(state)), + share_enc_preset: DEFAULT_BFV_PRESET, + interfold_address: Address::ZERO, + recovery: test_recovery(), + }) + .start(); + + actor + .send( + InterfoldEvent::::new_with_timestamp( + EffectsEnabled::new().into(), + None, + 1, + None, + EventSource::Local, + ) + .into_sequenced(1), + ) + .await?; + + // The collector's timeout is the only thing that can produce this after a restart with + // zero persisted shares. Before the fix there was no collector, hence no timeout, hence + // nothing on the bus — this call would time out. + let event = next_event(&history).await?; + assert!( + matches!( + event.get_data(), + InterfoldEventData::E3Failed(data) + if data.e3_id == e3_id && data.reason == FailureReason::DecryptionTimeout + ), + "the rebuilt collector must own the decryption timeout after a restart; got {:?}", + event.event_type() + ); + assert!(matches!( + repo.read().await?.expect("persisted keyshare state").state, + KeyshareState::Failed { + reason: FailureReason::DecryptionTimeout, + .. + } + )); + Ok(()) +} + +/// A restart inside the DKG window drives `CalculateDecryptionKey` twice: the event-store +/// replay re-forwards the original `ComputeRequest`, and the re-driven +/// `ShareVerificationComplete` publishes a second one with a fresh correlation id, so the +/// effect gate cannot dedup them. The first response moves the actor to +/// `ReadyForDecryption` and clears the pending C4 inputs; the second used to hit "No pending +/// share decryption data" and surface as an `InterfoldError` on an otherwise healthy node +/// (Round 10, cn4). A response that arrives after the key is derived must be a no-op. +#[actix::test] +async fn a_second_calculate_decryption_key_response_is_ignored_not_an_error() -> Result<()> { + let ready = ReadyForDecryption { + pk_share: ArcBytes::from_bytes(&[1]), + sk_poly_sum: SensitiveBytes::from_encrypted(&[2]), + es_poly_sum: vec![SensitiveBytes::from_encrypted(&[3])], + signed_pk_generation_proof: None, + signed_sk_share_computation_proof: None, + signed_e_sm_share_computation_proof: None, + signed_sk_share_encryption_proofs: Vec::new(), + signed_e_sm_share_encryption_proofs: Vec::new(), + }; + let (bus, history) = test_bus(); + let e3_id = E3id::new("43", 1); + let store = InMemStore::new(false).start(); + let repo = Repository::::new(DataStore::from_in_mem(&store)); + let state = ThresholdKeyshareState::new( + e3_id.clone(), + 0, + KeyshareState::ReadyForDecryption(ready), + 1, + 3, + ArcBytes::from_bytes(b"params"), + Address::ZERO.to_string(), + ); + let actor = ThresholdKeyshare::new(ThresholdKeyshareParams { + bus, + cipher: Arc::new(Cipher::from_password("test-password").await?), + state: repo.send(Some(state)), + share_enc_preset: DEFAULT_BFV_PRESET, + interfold_address: Address::ZERO, + recovery: test_recovery(), + }) + .start(); + + // The late/duplicate response, exactly what the second compute produces. + let response = ComputeResponse::trbfv( + e3_trbfv::TrBFVResponse::CalculateDecryptionKey( + e3_trbfv::calculate_decryption_key::CalculateDecryptionKeyResponse { + sk_poly_sum: SensitiveBytes::from_encrypted(&[9]), + es_poly_sum: vec![SensitiveBytes::from_encrypted(&[9])], + }, + ), + CorrelationId::new(), + e3_id.clone(), + ); + actor + .send( + InterfoldEvent::::new_with_timestamp( + response.into(), + None, + 1, + None, + EventSource::Local, + ) + .into_sequenced(1), + ) + .await?; + sleep(Duration::from_millis(100)).await; + + let events = history.send(GetEvents::::new()).await?; + let errors: Vec<_> = events + .iter() + .filter(|event| matches!(event.get_data(), InterfoldEventData::InterfoldError(_))) + .collect(); + assert!( + errors.is_empty(), + "a duplicate CalculateDecryptionKey response must be ignored, got {errors:?}" + ); + // And the derived key was not overwritten by the stray response. + let persisted = repo.read().await?.expect("persisted keyshare state"); + let KeyshareState::ReadyForDecryption(after) = persisted.state else { + bail!("state must remain ReadyForDecryption"); + }; + assert_eq!(after.pk_share, ArcBytes::from_bytes(&[1])); + Ok(()) +} + +/// After a restart mid-DKG, recovery rebuilds the share collector from persisted peer shares +/// before it redrives this node's own share generation. Peer shares are already on the wire, +/// so the collector completes while the local state is still `GeneratingThresholdShare`. The +/// collector cancels its timeout on completion and never re-emits, so the message must be +/// held rather than rejected — otherwise the DKG stalls with no timer left to fail it. +#[actix::test] +async fn all_shares_collected_before_own_generation_is_held_not_dropped() -> Result<()> { + let generating = GeneratingThresholdShareData { + pk_share: None, + sk_sss: None, + esi_sss: None, + e_sm_raw: None, + sk_bfv: SensitiveBytes::from_encrypted(&[1]), + pk_bfv: ArcBytes::from_bytes(&[2]), + collected_encryption_keys: Vec::new(), + ciphernode_selected: None, + proof_request_data: None, + }; + let (actor, history, _e3_id, repo) = + start_actor_with_state(KeyshareState::GeneratingThresholdShare(generating)).await?; + + let ctx = InterfoldEvent::::new_with_timestamp( + EffectsEnabled::new().into(), + None, + 1, + None, + EventSource::Local, + ) + .into_sequenced(1) + .get_ctx() + .clone(); + let collected = TypedEvent::new( + AllThresholdSharesCollected::new(HashMap::new(), HashMap::new()), + ctx, + ); + + actor.send(collected).await?; + + // Before the fix this produced an `InterfoldError("Invalid state")` on the bus and the + // shares were gone for good. Now nothing is published: the message is parked. + let result = history.send(TakeEvents::::new(1)).await?; + assert!( + result.timed_out || result.events.is_empty(), + "AllThresholdSharesCollected in GeneratingThresholdShare must be held, not turned \ + into an error event; got {:?}", + result.events + ); + + // The state was not disturbed by the early arrival. + assert!(matches!( + repo.read().await?.expect("persisted keyshare state").state, + KeyshareState::GeneratingThresholdShare(_) + )); + + Ok(()) +} + +/// The live case: a node restarted mid-DKG is still collecting encryption keys while its +/// peers — who already hold its key — finish and send their threshold shares. The collector +/// completes two states early. Observed on a 5-node swarm: `kill -9` during DKG left the node +/// permanently stalled with `InterfoldError("Invalid state")` and no timer to fail the E3. +#[actix::test] +async fn all_shares_collected_while_collecting_encryption_keys_is_held_not_dropped() -> Result<()> { + let e3_id = E3id::new("1234", 1); + let collecting = CollectingEncryptionKeysData { + sk_bfv: SensitiveBytes::from_encrypted(&[1]), + pk_bfv: ArcBytes::from_bytes(&[2]), + ciphernode_selected: CiphernodeSelected { + e3_id: e3_id.clone(), + threshold_m: 1, + threshold_n: 3, + seed: Seed([0u8; 32]), + error_size: ArcBytes::from_bytes(&[0]), + params_preset: DEFAULT_BFV_PRESET, + params: ArcBytes::from_bytes(b"params"), + party_id: 0, + committee: Vec::new(), + }, + }; + let (actor, history, _e3_id, repo) = + start_actor_with_state(KeyshareState::CollectingEncryptionKeys(collecting)).await?; + + let ctx = InterfoldEvent::::new_with_timestamp( + EffectsEnabled::new().into(), + None, + 1, + None, + EventSource::Local, + ) + .into_sequenced(1) + .get_ctx() + .clone(); + actor + .send(TypedEvent::new( + AllThresholdSharesCollected::new(HashMap::new(), HashMap::new()), + ctx, + )) + .await?; + + let result = history.send(TakeEvents::::new(1)).await?; + assert!( + result.timed_out || result.events.is_empty(), + "AllThresholdSharesCollected in CollectingEncryptionKeys must be held, not turned \ + into an error event; got {:?}", + result.events + ); + assert!(matches!( + repo.read().await?.expect("persisted keyshare state").state, + KeyshareState::CollectingEncryptionKeys(_) + )); + + Ok(()) +} diff --git a/crates/logger/src/logger.rs b/crates/logger/src/logger.rs index c18e07927b..f3599a1d64 100644 --- a/crates/logger/src/logger.rs +++ b/crates/logger/src/logger.rs @@ -88,13 +88,17 @@ fn severity(data: &InterfoldEventData) -> Severity { | E::AccusationVote(_) | E::AccusationQuorumReached(_) | E::SlashExecuted(_) - | E::CommitteeMemberExpelled(_) - | E::AggregatorChanged(_) => Severity::Warn, + | E::CommitteeMemberExpelled(_) => Severity::Warn, E::EvmLogObserved(event) if !event.known => Severity::Warn, E::E3Requested(_) | E::CommitteeRequested(_) + // Fires on every normal E3 when the aggregator role is first assigned — six times in a + // measured run with zero failovers — so it is not an alert. A genuine failover + // promotion has its own WARN in the sortition actor + // ("Aggregator progress deadline expired; promoting deterministic standby"). + | E::AggregatorChanged(_) | E::TicketGenerated(_) | E::TicketSubmitted(_) | E::CommitteeFinalizeRequested(_) diff --git a/crates/multithread/src/multithread.rs b/crates/multithread/src/multithread.rs index ca60e73e03..134a9758e7 100644 --- a/crates/multithread/src/multithread.rs +++ b/crates/multithread/src/multithread.rs @@ -85,7 +85,7 @@ use fhe_traits::{DeserializeParametrized, FheEncoder}; use ndarray::Array2; use num_bigint::BigInt; use rand::Rng; -use tracing::{error, info}; +use tracing::{debug, error, info}; use crate::effect_gate::ComputeEffectGate; @@ -509,11 +509,13 @@ fn timefunc( where F: FnOnce() -> Result, { - info!("STARTING MULTITHREAD `{}({})`", name, id); + // The start line is scheduler bookkeeping; the finish line carries the duration, which is + // what makes a slow or hung job visible. Keep the pair asymmetric on purpose. + debug!("STARTING MULTITHREAD `{}({})`", name, id); let start = Instant::now(); let out = func(); let dur = start.elapsed(); - info!("FINISHED MULTITHREAD `{}`({}) in {:?}", name, id, dur); + info!(job = %name, id, duration_ms = dur.as_millis(), "Compute job finished"); (out, dur) } diff --git a/crates/net/src/document_publishing/actor.rs b/crates/net/src/document_publishing/actor.rs index 4fc2169c67..2f1e8fea82 100644 --- a/crates/net/src/document_publishing/actor.rs +++ b/crates/net/src/document_publishing/actor.rs @@ -8,7 +8,8 @@ use crate::net_interface_handle::NetEventSubscriber; use crate::{ domain::{datetime_to_instant_from_now, DocumentPublishingService}, events::{ - call_and_await_response, DocumentPublishedNotification, GossipData, NetCommand, NetEvent, + call_and_await_response, DocumentPublishedNotification, GossipData, GossipPublishFailure, + NetCommand, NetEvent, }, ContentHash, }; @@ -130,16 +131,25 @@ impl DocumentPublisher { .await { debug!("Received event {:?}", event); - if let NetEvent::GossipData(GossipData::DocumentPublishedNotification(data)) = - event - { - if let Err(error) = addr.send(data).await { - tracing::warn!( - %error, - "DocumentPublisher stopped; ending DHT notification ingress" - ); - break; + match event { + NetEvent::GossipData(GossipData::DocumentPublishedNotification(data)) => { + if let Err(error) = addr.send(data).await { + tracing::warn!( + %error, + "DocumentPublisher stopped; ending DHT notification ingress" + ); + break; + } } + // A peer (re)joined the topic after our pointers went out. Re-announce + // so it can fetch the records; without this a node that restarts + // mid-DKG never learns the content hashes it needs. + NetEvent::GossipSubscribed { .. } => { + if addr.send(handlers::PeerSubscribed).await.is_err() { + break; + } + } + _ => {} } } } @@ -175,7 +185,10 @@ mod effects; #[path = "handlers.rs"] mod handlers; -pub use effects::{handle_document_published_notification, handle_publish_document_requested}; +pub use effects::{ + handle_document_published_notification, handle_publish_document_requested, + repeat_document_published_notification, +}; #[cfg(test)] #[path = "tests/mod.rs"] diff --git a/crates/net/src/document_publishing/effects.rs b/crates/net/src/document_publishing/effects.rs index badd3fc973..150fa0659e 100644 --- a/crates/net/src/document_publishing/effects.rs +++ b/crates/net/src/document_publishing/effects.rs @@ -6,14 +6,16 @@ use super::*; use crate::domain::EventConversionService; use crate::net_interface_handle::NetEventSubscriber; -/// Called when we receive a PublishDocumentRequested event +/// Called when we receive a PublishDocumentRequested event. +/// +/// Returns the notification that was gossiped so the caller can keep it for re-announcement. pub async fn handle_publish_document_requested( tx: mpsc::Sender, rx: NetEventSubscriber, event: PublishDocumentRequested, topic: impl Into, bus: BusHandle, -) -> Result<()> { +) -> Result { let value = event.value; let key = ContentHash::from_content(&value); let expires = Some( @@ -31,8 +33,53 @@ pub async fn handle_publish_document_requested( ) .await?; let notification = DocumentPublishedNotification::new(event.meta, key, bus.ts()?); - broadcast_document_published_notification(tx, rx, notification, topic).await?; - Ok(()) + broadcast_document_published_notification(tx, rx, notification.clone(), topic).await?; + Ok(notification) +} + +/// Re-gossip a notification that was already announced once. +/// +/// Used when a peer (re)subscribes to the topic after the original broadcast. The DHT +/// record is still present; only the pointer needs to reach the late peer. +/// +/// gossipsub ids messages by the SHA-256 of their bytes and refuses an identical publish +/// for `duplicate_cache_time` (60 s). Re-sending the retained notification byte-for-byte +/// therefore does nothing for a peer that was **down** during the original announce: it +/// never saw the mesh copy, and the re-announce is rejected as `Duplicate`. Round 10 +/// caught exactly this — a member restarted inside the DKG window, every peer re-announced, +/// every re-announce was rejected, and the member's collector waited two hours for shares +/// that were sitting in the DHT the whole time. +/// +/// So the re-announce is re-stamped with a fresh HLC tick. The bytes differ, gossipsub +/// treats it as a new message, and the late peer receives it. Receivers key on the +/// content hash in `key`, not on `ts`, so a node that already fetched the document simply +/// fetches the same record again (idempotent: the collectors are keyed by party id). +/// +/// `AlreadyPublished` can still occur if two `GossipSubscribed` events land inside the same +/// HLC tick; it is swallowed at `debug` because the first re-announce already went out. +pub async fn repeat_document_published_notification( + tx: mpsc::Sender, + rx: NetEventSubscriber, + mut notification: DocumentPublishedNotification, + topic: impl Into, + bus: &BusHandle, +) -> Result<()> { + notification.ts = bus.ts()?; + match broadcast_document_published_notification(tx, rx, notification, topic).await { + Err(error) if is_already_published(&error) => { + debug!("Re-announce skipped: identical pointer is already in the gossip mesh"); + Ok(()) + } + other => other, + } +} + +fn is_already_published(error: &anyhow::Error) -> bool { + // `.context()` wraps the typed failure one level down; walk the whole chain. + error + .chain() + .filter_map(|source| source.downcast_ref::()) + .any(|failure| matches!(failure, GossipPublishFailure::AlreadyPublished)) } /// Called when we receive a notification from the net_interface @@ -154,7 +201,12 @@ async fn broadcast_document_published_notification( |event| match event { NetEvent::GossipPublished { .. } => Some(Ok(())), NetEvent::GossipPublishError { error, .. } => { - Some(Err(anyhow::anyhow!("GossipPublished failed: {:?}", error))) + // Keep the typed failure as the error source so callers can match on it. + // `error` is `&Arc<_>`; deref through the Arc or the source type is the Arc. + let failure: GossipPublishFailure = (**error).clone(); + Some(Err( + anyhow::Error::new(failure).context("GossipPublished failed") + )) } _ => None, }, @@ -162,3 +214,32 @@ async fn broadcast_document_published_notification( ) .await } + +#[cfg(test)] +mod already_published_tests { + use super::*; + + #[test] + fn a_context_wrapped_duplicate_is_recognised() { + let error = anyhow::Error::new(GossipPublishFailure::AlreadyPublished) + .context("GossipPublished failed"); + assert!(is_already_published(&error)); + // Exactly what the broadcast matcher builds from the network's `Arc<_>`. Cloning the + // `Arc` itself (instead of the failure inside it) makes the source type + // `Arc` and the downcast silently miss — this pins it. + let shared = std::sync::Arc::new(GossipPublishFailure::AlreadyPublished); + let failure: GossipPublishFailure = (*shared).clone(); + let from_arc = anyhow::Error::new(failure).context("GossipPublished failed"); + assert!(is_already_published(&from_arc)); + } + + #[test] + fn other_failures_are_not() { + let error = anyhow::Error::new(GossipPublishFailure::NoPeersSubscribed) + .context("GossipPublished failed"); + assert!(!is_already_published(&error)); + assert!(!is_already_published(&anyhow::anyhow!( + "GossipPublished failed" + ))); + } +} diff --git a/crates/net/src/document_publishing/handlers.rs b/crates/net/src/document_publishing/handlers.rs index 184f41fafd..bc90b5a0d2 100644 --- a/crates/net/src/document_publishing/handlers.rs +++ b/crates/net/src/document_publishing/handlers.rs @@ -35,7 +35,7 @@ impl Handler> for DocumentPublisher { fn handle( &mut self, msg: TypedEvent, - _: &mut Self::Context, + ctx: &mut Self::Context, ) -> Self::Result { let tx = self.tx.clone(); let (msg, ec) = msg.into_components(); @@ -46,11 +46,63 @@ impl Handler> for DocumentPublisher { let rx = self.rx.clone(); let bus = self.bus.clone(); let topic = self.topic.clone(); - trap_fut( - EType::IO, - &bus.with_ec(&ec), - handle_publish_document_requested(tx, rx, msg, topic, bus), - ) + let addr = ctx.address(); + trap_fut(EType::IO, &bus.with_ec(&ec), async move { + let notification = handle_publish_document_requested(tx, rx, msg, topic, bus).await?; + // Hand the gossiped pointer back to the actor so a late peer can be re-told. + addr.do_send(Announced(notification)); + Ok(()) + }) + } +} + +/// A notification that has been gossiped once; kept so it can be re-announced. +#[derive(Message)] +#[rtype(result = "()")] +pub(super) struct Announced(pub DocumentPublishedNotification); + +impl Handler for DocumentPublisher { + type Result = (); + fn handle(&mut self, msg: Announced, _: &mut Self::Context) -> Self::Result { + self.service.track_announced(msg.0); + } +} + +/// A peer joined the gossip topic. Re-announce every in-flight document pointer so a peer +/// that restarted or connected after the original broadcast can fetch the DHT record. +#[derive(Message)] +#[rtype(result = "()")] +pub(super) struct PeerSubscribed; + +impl Handler for DocumentPublisher { + type Result = ResponseFuture<()>; + fn handle(&mut self, _: PeerSubscribed, _: &mut Self::Context) -> Self::Result { + let notifications = self.service.announcements_to_repeat(); + if notifications.is_empty() { + return Box::pin(async {}); + } + info!( + count = notifications.len(), + "Peer subscribed; re-announcing in-flight document pointers" + ); + let tx = self.tx.clone(); + let rx = self.rx.clone(); + let topic = self.topic.clone(); + let bus = self.bus.clone(); + let stamp_bus = bus.clone(); + trap_fut(EType::IO, &bus, async move { + for notification in notifications { + repeat_document_published_notification( + tx.clone(), + rx.clone(), + notification, + topic.clone(), + &stamp_bus, + ) + .await?; + } + Ok(()) + }) } } diff --git a/crates/net/src/document_publishing/tests/mod.rs b/crates/net/src/document_publishing/tests/mod.rs index 624c4f80d3..b425dffea9 100644 --- a/crates/net/src/document_publishing/tests/mod.rs +++ b/crates/net/src/document_publishing/tests/mod.rs @@ -8,6 +8,7 @@ use crate::net_interface_handle::NetEventSubscriber; use std::{collections::HashMap, num::NonZero, sync::Arc, time::Duration}; use super::*; +use crate::events::GossipPublishFailure; use crate::events::NetCommand; use crate::{domain::EventConversionService, ContentHash}; use actix::Addr; diff --git a/crates/net/src/document_publishing/tests/publishing.rs b/crates/net/src/document_publishing/tests/publishing.rs index 0322f8d3e7..d394f4c633 100644 --- a/crates/net/src/document_publishing/tests/publishing.rs +++ b/crates/net/src/document_publishing/tests/publishing.rs @@ -80,6 +80,210 @@ async fn test_publishes_document() -> Result<()> { Ok(()) } +/// A `DocumentPublishedNotification` is gossiped once. A peer that subscribes to the topic +/// afterwards — restarting mid-DKG, or connecting late — never receives the pointer and cannot +/// fetch the DHT record even though it is still there. Observed live: a node killed and +/// restarted during DKG stalled forever waiting on encryption keys its peers had already +/// published. The publisher must re-announce its in-flight pointers when a peer subscribes. +#[actix::test] +async fn in_flight_pointers_are_reannounced_when_a_peer_subscribes() -> Result<()> { + let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, _, _, _) = + setup_test()?; + let e3_id = E3id::new("77", 1); + + // Publish one document and drive the fake network through put + gossip. + bus.publish_without_context(PublishDocumentRequested { + meta: DocumentMeta::new(e3_id.clone(), DocumentKind::TrBFV, vec![], None), + value: ArcBytes::from_bytes(b"encryption key"), + })?; + let Some(NetCommand::DhtPutRecord { + correlation_id, + key, + .. + }) = timeout(Duration::from_secs(1), net_cmd_rx.recv()).await? + else { + bail!("expected DhtPutRecord"); + }; + net_evt_tx.send(NetEvent::DhtPutRecordSucceeded { + correlation_id, + key: key.clone(), + })?; + let Some(NetCommand::GossipPublish { + correlation_id, + data: GossipData::DocumentPublishedNotification(first), + .. + }) = timeout(Duration::from_secs(1), net_cmd_rx.recv()).await? + else { + bail!("expected the first GossipPublish"); + }; + net_evt_tx.send(NetEvent::GossipPublished { + correlation_id, + message_id: libp2p::gossipsub::MessageId::new(&[1]), + })?; + // Let the actor record the announced pointer. + sleep(Duration::from_millis(50)).await; + + // A peer joins the topic after the fact. + net_evt_tx.send(NetEvent::GossipSubscribed { + count: 1, + topic: libp2p::gossipsub::IdentTopic::new("topic").hash(), + })?; + + // The same pointer goes out again. It must carry a FRESH ts: gossipsub ids messages by + // the SHA-256 of their bytes, so a byte-identical re-send is rejected as `Duplicate` + // for 60 s and never reaches a peer that was down for the original announce (Round 10). + // Receivers key on `key`, not `ts`, so the changed ts is harmless to them. + let Some(NetCommand::GossipPublish { + correlation_id, + data: GossipData::DocumentPublishedNotification(again), + .. + }) = timeout(Duration::from_secs(1), net_cmd_rx.recv()) + .await + .expect("pointer was not re-announced on peer subscribe") + else { + bail!("expected the re-announced GossipPublish"); + }; + net_evt_tx.send(NetEvent::GossipPublished { + correlation_id, + message_id: libp2p::gossipsub::MessageId::new(&[2]), + })?; + assert_eq!(again.key, first.key); + assert_eq!(again.meta.e3_id, e3_id); + assert_ne!( + again.ts, first.ts, + "re-announce must be re-stamped so gossipsub does not reject it as a Duplicate" + ); + assert_ne!( + again.to_bytes()?, + first.to_bytes()?, + "re-announce bytes must differ or the gossipsub message id collides" + ); + + Ok(()) +} + +/// gossipsub ids messages by content hash and rejects an identical publish for 60 s. A +/// peer that subscribes inside that window makes the re-announce hit `Duplicate`; the first +/// announce is still in the mesh so nothing was lost, and it must not surface as an +/// `InterfoldError` (it did on every node of the Round 9 swarm at `nodes down`). +#[actix::test] +async fn a_duplicate_reannounce_is_not_an_error() -> Result<()> { + let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, _, errors, _) = + setup_test()?; + let e3_id = E3id::new("78", 1); + + bus.publish_without_context(PublishDocumentRequested { + meta: DocumentMeta::new(e3_id.clone(), DocumentKind::TrBFV, vec![], None), + value: ArcBytes::from_bytes(b"encryption key"), + })?; + let Some(NetCommand::DhtPutRecord { + correlation_id, + key, + .. + }) = timeout(Duration::from_secs(1), net_cmd_rx.recv()).await? + else { + bail!("expected DhtPutRecord"); + }; + net_evt_tx.send(NetEvent::DhtPutRecordSucceeded { + correlation_id, + key, + })?; + let Some(NetCommand::GossipPublish { correlation_id, .. }) = + timeout(Duration::from_secs(1), net_cmd_rx.recv()).await? + else { + bail!("expected the first GossipPublish"); + }; + net_evt_tx.send(NetEvent::GossipPublished { + correlation_id, + message_id: libp2p::gossipsub::MessageId::new(&[1]), + })?; + sleep(Duration::from_millis(50)).await; + + net_evt_tx.send(NetEvent::GossipSubscribed { + count: 1, + topic: libp2p::gossipsub::IdentTopic::new("topic").hash(), + })?; + let Some(NetCommand::GossipPublish { correlation_id, .. }) = + timeout(Duration::from_secs(1), net_cmd_rx.recv()).await? + else { + bail!("expected the re-announced GossipPublish"); + }; + // The network reports gossipsub's duplicate-cache rejection. + net_evt_tx.send(NetEvent::GossipPublishError { + correlation_id, + error: std::sync::Arc::new(GossipPublishFailure::from_libp2p( + libp2p::gossipsub::PublishError::Duplicate, + )), + })?; + sleep(Duration::from_millis(100)).await; + + let errors = errors.send(GetEvents::::new()).await?; + assert!( + errors.is_empty(), + "a duplicate re-announce must be swallowed, got {errors:?}" + ); + Ok(()) +} + +/// Once an E3 completes its pointers must not be re-announced to late peers. +#[actix::test] +async fn completed_e3_pointers_are_not_reannounced() -> Result<()> { + let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, _, _, _) = + setup_test()?; + let e3_id = E3id::new("78", 1); + + bus.publish_without_context(PublishDocumentRequested { + meta: DocumentMeta::new(e3_id.clone(), DocumentKind::TrBFV, vec![], None), + value: ArcBytes::from_bytes(b"done"), + })?; + let Some(NetCommand::DhtPutRecord { + correlation_id, + key, + .. + }) = timeout(Duration::from_secs(1), net_cmd_rx.recv()).await? + else { + bail!("expected DhtPutRecord"); + }; + net_evt_tx.send(NetEvent::DhtPutRecordSucceeded { + correlation_id, + key, + })?; + let Some(NetCommand::GossipPublish { correlation_id, .. }) = + timeout(Duration::from_secs(1), net_cmd_rx.recv()).await? + else { + bail!("expected GossipPublish"); + }; + net_evt_tx.send(NetEvent::GossipPublished { + correlation_id, + message_id: libp2p::gossipsub::MessageId::new(&[1]), + })?; + sleep(Duration::from_millis(50)).await; + + bus.publish_without_context(e3_events::E3RequestComplete { + e3_id: e3_id.clone(), + })?; + // Completion prunes the DHT records. + let Some(NetCommand::DhtRemoveRecords { .. }) = + timeout(Duration::from_secs(1), net_cmd_rx.recv()).await? + else { + bail!("expected DhtRemoveRecords"); + }; + + net_evt_tx.send(NetEvent::GossipSubscribed { + count: 1, + topic: libp2p::gossipsub::IdentTopic::new("topic").hash(), + })?; + + assert!( + timeout(Duration::from_millis(300), net_cmd_rx.recv()) + .await + .is_err(), + "a completed E3's pointers must not be re-announced" + ); + + Ok(()) +} + #[actix::test] async fn expired_document_is_rejected_without_a_dht_write() -> Result<()> { let system = EventSystem::new().with_fresh_bus(); diff --git a/crates/net/src/document_publishing/workflow.rs b/crates/net/src/document_publishing/workflow.rs index ef400e359e..713e652e72 100644 --- a/crates/net/src/document_publishing/workflow.rs +++ b/crates/net/src/document_publishing/workflow.rs @@ -27,6 +27,15 @@ pub struct DocumentPublishingService { ids: HashMap, /// Track DHT content hashes per E3 for cleanup on completion. dht_keys: HashMap>, + /// The notifications this node has gossiped, per E3, so they can be re-announced. + /// + /// A `DocumentPublishedNotification` is gossiped exactly once, when the DHT put + /// succeeds. A peer that is not subscribed at that instant — restarting, or still + /// dialing — never learns the content hash and cannot fetch the record, even though the + /// record itself stays in the DHT for days. Historical peer sync does not cover these + /// notifications either. Keeping the pointer lets the publisher re-announce it when a + /// peer (re)joins the topic; receivers dedup by content, so re-sends are harmless. + announced: HashMap>, } impl DocumentPublishingService { @@ -38,6 +47,7 @@ impl DocumentPublishingService { Self { ids, dht_keys: HashMap::new(), + announced: HashMap::new(), } } @@ -49,9 +59,23 @@ impl DocumentPublishingService { /// Mark an E3 complete, returning the DHT keys that should be pruned for it. pub fn complete_e3(&mut self, e3_id: &E3id) -> Vec { self.ids.remove(e3_id); + self.announced.remove(e3_id); self.dht_keys.remove(e3_id).unwrap_or_default() } + /// Remember a notification that was gossiped so it can be re-announced later. + pub fn track_announced(&mut self, notification: DocumentPublishedNotification) { + self.announced + .entry(notification.meta.e3_id.clone()) + .or_default() + .push(notification); + } + + /// Every notification this node has gossiped for an E3 that is still in flight. + pub fn announcements_to_repeat(&self) -> Vec { + self.announced.values().flatten().cloned().collect() + } + /// Compute the content hash for a value being published and record it against `e3_id` /// so it can be pruned when the E3 completes. pub fn track_published_key(&mut self, e3_id: &E3id, value: &ArcBytes) -> ContentHash { diff --git a/crates/net/src/event_buffer/tests.rs b/crates/net/src/event_buffer/tests.rs index ab81c58800..2912e328d7 100644 --- a/crates/net/src/event_buffer/tests.rs +++ b/crates/net/src/event_buffer/tests.rs @@ -20,7 +20,6 @@ use crate::{ use e3_ciphernode_builder::EventSystem; use e3_events::{CorrelationId, EventPublisher, SyncEnded}; use libp2p::{ - gossipsub::TopicHash, swarm::{ConnectionId, DialError}, PeerId, }; @@ -47,10 +46,6 @@ fn sync_and_connection_control_events() -> Vec { connection_id: ConnectionId::new_unchecked(3), error: Arc::new(DialError::NoAddresses), }, - NetEvent::GossipSubscribed { - count: 1, - topic: TopicHash::from_raw("test-topic"), - }, NetEvent::IncomingRequest(IncomingRequest { peer: PeerId::random(), responder: DirectResponder::new( diff --git a/crates/net/src/event_translation/actor.rs b/crates/net/src/event_translation/actor.rs index 592373ffb8..ec16456b8d 100644 --- a/crates/net/src/event_translation/actor.rs +++ b/crates/net/src/event_translation/actor.rs @@ -263,7 +263,8 @@ fn retry_policy(failure: &GossipPublishFailure) -> Option<(u8, Duration)> { GossipPublishFailure::Transient(_) => { Some((MAX_GOSSIP_PUBLISH_ATTEMPTS, GOSSIP_RETRY_DELAY)) } - GossipPublishFailure::Permanent(_) => None, + // The identical message is already in the mesh; a retry would hit the same cache. + GossipPublishFailure::AlreadyPublished | GossipPublishFailure::Permanent(_) => None, } } diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index 513d85d65a..e47d3eaf02 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -44,6 +44,10 @@ pub enum PeerRejectionKind { #[derive(Clone, Debug, PartialEq, Eq)] pub enum GossipPublishFailure { NoPeersSubscribed, + /// gossipsub already holds an identical message (content-hash id) in its duplicate + /// cache. The message is in flight in the mesh; nothing was lost. Re-announcing a + /// document pointer within `duplicate_cache_time` (60 s) of the original hits this. + AlreadyPublished, Transient(String), Permanent(String), } @@ -52,6 +56,7 @@ impl GossipPublishFailure { pub fn from_libp2p(error: PublishError) -> Self { match error { PublishError::NoPeersSubscribedToTopic => Self::NoPeersSubscribed, + PublishError::Duplicate => Self::AlreadyPublished, PublishError::AllQueuesFull(count) => { Self::Transient(PublishError::AllQueuesFull(count).to_string()) } @@ -72,11 +77,16 @@ impl std::fmt::Display for GossipPublishFailure { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::NoPeersSubscribed => formatter.write_str("no peers are subscribed to the topic"), + Self::AlreadyPublished => { + formatter.write_str("an identical message is already in the gossip mesh") + } Self::Transient(reason) | Self::Permanent(reason) => formatter.write_str(reason), } } } +impl std::error::Error for GossipPublishFailure {} + #[derive(Clone, Copy, Debug)] pub enum PeerTarget { Random, @@ -340,12 +350,15 @@ impl NetEvent { | Self::DhtGetRecordSucceeded { .. } | Self::DhtPutRecordSucceeded { .. } | Self::DhtGetRecordError { .. } - | Self::DhtPutRecordError { .. } => true, + | Self::DhtPutRecordError { .. } + // The document publisher re-announces in-flight DHT pointers when a peer joins + // the topic; it reads the application channel. One small event per peer-join + // cannot crowd the startup buffer. + | Self::GossipSubscribed { .. } => true, Self::DialError { .. } | Self::ConnectionEstablished { .. } | Self::PeerRejected { .. } | Self::OutgoingConnectionError { .. } - | Self::GossipSubscribed { .. } | Self::IncomingRequest(_) | Self::OutgoingRequestSucceeded(_) | Self::OutgoingRequestFailed(_) diff --git a/crates/net/src/net_interface.rs b/crates/net/src/net_interface.rs index fbeb3215a5..9eaa7b05fd 100644 --- a/crates/net/src/net_interface.rs +++ b/crates/net/src/net_interface.rs @@ -871,13 +871,16 @@ async fn process_swarm_event( debug!(%peer_id, %topic, "Ignoring a subscription before peer admission"); return Ok(()); } - debug!("Peer {} subscribed to {}", peer_id, topic); let count = swarm .behaviour() .gossipsub .mesh_peers(&topic) .filter(|peer| peer_admission.is_admitted(peer)) .count(); + // Mesh size is the diagnostic for "my publish reached nobody" — the Bug 20 class, + // where a restarted peer never learns about in-flight shares. Cheap and low + // frequency (once per peer per topic), so keep it at INFO. + info!(%peer_id, %topic, mesh_peers = count, "Peer subscribed to topic"); event_tx.send(NetEvent::GossipSubscribed { count, topic })?; } @@ -1108,7 +1111,15 @@ async fn process_swarm_event( status.disconnected(&peer_id.to_string(), num_established); if num_established == 0 { let total = swarm.connected_peers().count(); - debug!("Peer disconnected: {peer_id} (total: {total}, cause: {cause:?})"); + // Losing the last connection to a peer is the leading indicator of the + // no-peers stall that crash-looped a mainnet node; it must be visible without + // -vv. `total == 0` means this node is now isolated. + info!( + %peer_id, + connected_peers = total, + cause = ?cause, + "Peer disconnected" + ); } } @@ -1126,7 +1137,9 @@ async fn process_swarm_event( } unknown => { - debug!("Unhandled swarm event: {:?}", unknown); + // Swarm events we deliberately do not act on. The full struct dump was ~800 lines + // per run and never actionable; keep it at trace for when it genuinely matters. + trace!("Unhandled swarm event: {:?}", unknown); } }; Ok(()) diff --git a/crates/net/src/network_sync/effects/fetch_history.rs b/crates/net/src/network_sync/effects/fetch_history.rs index d1940f3db6..cf3b10ac13 100644 --- a/crates/net/src/network_sync/effects/fetch_history.rs +++ b/crates/net/src/network_sync/effects/fetch_history.rs @@ -76,6 +76,7 @@ pub(in crate::actors::net_sync_manager) async fn handle_sync_request_event( event: TypedEvent, address: impl Into>>, wait_for_event: bool, + has_connected_peers: bool, network: NetworkPolicy, ) -> Result<()> { info!("Sync request event received"); @@ -133,6 +134,32 @@ pub(in crate::actors::net_sync_manager) async fn handle_sync_request_event( address.into().try_send(TypedEvent::new(value, ctx))?; return Ok(()); } + } else if !has_connected_peers { + // `AllPeersDialed` was already observed with zero successful connections and + // readiness published `NetReady` through its connect-timeout fallback. Every + // fetch below would fail with "No connected peers available", burn the retry + // budget, and then `bail!` — which `trap_fut` converts into an error event + // rather than a completion, so startup would wait on a `SyncRequestSucceeded` + // that can never arrive and die at `startup_timeout_secs` in a crash loop. + // + // A node that cannot reach a peer has no history to import. Complete the phase + // with an empty result so it boots and serves the chain; the background + // bootstrap retries and live gossip remain responsible for catching it up. + warn!( + aggregates = sync_cursor.len(), + "Skipping historical peer sync: no peer connections are available. \ + The node will start from local state and catch up over live gossip" + ); + address.into().try_send(TypedEvent::new( + SyncRequestSucceeded { + response: SyncResponseValue { + events: vec![], + ts: 0, + }, + }, + ctx, + ))?; + return Ok(()); } info!("handle_sync_request_event: ready to sync"); diff --git a/crates/net/src/network_sync/handlers.rs b/crates/net/src/network_sync/handlers.rs index b686f119e9..1cb0d1cad8 100644 --- a/crates/net/src/network_sync/handlers.rs +++ b/crates/net/src/network_sync/handlers.rs @@ -45,6 +45,7 @@ impl Handler> for NetSyncManager { msg, ctx.address(), !self.readiness_all_peers_dialed(), + self.readiness_has_connections(), self.network.clone(), ), ) @@ -57,6 +58,13 @@ impl NetSyncManager { // AllPeersDialed signal. The readiness machine tracks this; mirror its view here. self.readiness.all_peers_dialed() } + + /// Whether any peer connection exists. When `AllPeersDialed` has already been + /// observed with no connections, historical sync has nothing to fetch from and + /// must not attempt requests that can only fail. + fn readiness_has_connections(&self) -> bool { + self.readiness.has_connections() + } } /// We have received the sync response from the remote peer diff --git a/crates/net/src/network_sync/tests.rs b/crates/net/src/network_sync/tests.rs index 19610ed2a2..0fa101819b 100644 --- a/crates/net/src/network_sync/tests.rs +++ b/crates/net/src/network_sync/tests.rs @@ -183,6 +183,7 @@ async fn local_only_cursor_completes_without_a_peer_request() { TypedEvent::new(start, context.sequence(1)), response_tx, true, + false, NetworkPolicy::local_unrestricted(), ) .await @@ -197,6 +198,47 @@ async fn local_only_cursor_completes_without_a_peer_request() { ); } +/// A node whose only bootstrap peer is unreachable still publishes `NetReady` +/// through the connect-timeout fallback. Historical sync must then complete the +/// phase instead of attempting fetches that can only fail: the failure path +/// `bail!`s into an error event rather than a completion, so startup would wait +/// on a `SyncRequestSucceeded` that never arrives and exit at +/// `startup_timeout_secs` — a crash loop on every restart. +#[actix::test] +async fn no_connected_peers_completes_sync_instead_of_hanging_startup() { + let (net_tx, mut net_rx) = mpsc::channel::(1); + let (event_tx, _event_rx) = broadcast::channel::(1); + let event_rx = NetEventSubscriber::from(&event_tx); + let (response_tx, response_rx) = + e3_utils::actix::channel::oneshot::>(); + // A real, peer-syncable chain aggregate — not the local aggregate 0. + let start = HistoricalNetSyncStart::new(BTreeMap::from([(AggregateId::new(31_337), 10)])); + let context: e3_events::EventContext = + InterfoldEventData::HistoricalNetSyncStart(start.clone()).into(); + + handle_sync_request_event( + net_tx, + event_rx, + TypedEvent::new(start, context.sequence(1)), + response_tx, + // AllPeersDialed already observed, so the function does not wait... + false, + // ...but no connection was ever established. + false, + NetworkPolicy::local_unrestricted(), + ) + .await + .expect("sync must complete rather than bail when no peers are connected"); + + let response = response_rx.await.unwrap().into_inner().response; + assert!(response.events.is_empty()); + assert_eq!(response.ts, 0); + assert!( + net_rx.try_recv().is_err(), + "no outbound peer request should be attempted without a connected peer" + ); +} + #[actix::test] async fn rebroadcast_only_gossips_forwardable_own_artifacts() { let system = EventSystem::new().with_fresh_bus(); diff --git a/crates/net/src/network_sync/workflow.rs b/crates/net/src/network_sync/workflow.rs index 867f053192..2efccb4a6e 100644 --- a/crates/net/src/network_sync/workflow.rs +++ b/crates/net/src/network_sync/workflow.rs @@ -61,6 +61,15 @@ impl NetReadiness { self.all_peers_dialed } + /// Whether a peer connection has been established at any point. + /// + /// `NetReady` is published by the connect-timeout fallback even when this is + /// still false, so historical peer sync must consult it before it attempts + /// fetches that cannot succeed. + pub fn has_connections(&self) -> bool { + self.has_connections + } + fn try_publish(&mut self) -> ReadinessDecision { if !self.net_ready_published { self.net_ready_published = true; diff --git a/crates/request/src/repo.rs b/crates/request/src/repo.rs index 3572024b81..f785e7814b 100644 --- a/crates/request/src/repo.rs +++ b/crates/request/src/repo.rs @@ -100,6 +100,7 @@ pub async fn ensure_request_router_checkpoint( contexts: Vec::new(), completed: Default::default(), replay_cursors, + teardown_deadlines: Default::default(), }) .await } diff --git a/crates/request/src/routing/actor.rs b/crates/request/src/routing/actor.rs index cfc6b8d8b6..5379ebb176 100644 --- a/crates/request/src/routing/actor.rs +++ b/crates/request/src/routing/actor.rs @@ -97,6 +97,54 @@ pub struct E3Router { replay_cursors: HashMap, recovery_store: Repository, recovered_selections: Vec, + /// Slashably-failed E3s and the unix second at which each context is torn down. + /// Persisted in the checkpoint so a restart re-arms the timers. + teardown_deadlines: HashMap, + /// How long a slashably-failed E3's context stays alive after `E3Failed`. + teardown_grace: std::time::Duration, +} + +/// Default for how long a slashably-failed E3's context stays alive after `E3Failed`. +/// +/// The accusation manager can still initiate or vote on an accusation for up to the +/// on-chain `accusationVoteValidity` window (30 min by default) plus the local vote timeout +/// (5 min) after the failure. Two hours covers the largest window governance can set with +/// margin; the leak this bounds used to be permanent. +pub const SLASHABLE_FAILURE_TEARDOWN_GRACE: std::time::Duration = + std::time::Duration::from_secs(2 * 60 * 60); + +/// Upper bound on how many completed E3 ids the router remembers. +/// +/// `completed` exists to reject late events for finished requests, and it is serialized +/// into the recovery checkpoint on **every** routed event. Unbounded, it grows by one +/// `E3id` (a `String` plus a `u64`) per E3 for the life of the data directory, so the +/// per-event checkpoint write grows with total E3 history. Late events for an E3 arrive +/// within blocks of its completion, never thousands of E3s later; keeping the most recent +/// completions preserves the guard where it matters. +pub const MAX_REMEMBERED_COMPLETIONS: usize = 4096; + +/// Drop the oldest completions once `completed` exceeds [`MAX_REMEMBERED_COMPLETIONS`]. +/// +/// On-chain E3 ids are allocated by an increasing counter, so the numerically lowest ids +/// are the oldest; pruning by id keeps the checkpoint schema unchanged (no separate order +/// list to persist). Ids that do not parse as integers sort first and go before any that +/// do, which only affects non-chain test fixtures. +pub(crate) fn prune_completed(completed: &mut HashSet) { + let excess = completed.len().saturating_sub(MAX_REMEMBERED_COMPLETIONS); + if excess == 0 { + return; + } + let mut by_age: Vec = completed.iter().cloned().collect(); + by_age.sort_by_cached_key(|id| { + ( + id.e3_id().parse::().ok(), + id.chain_id(), + id.e3_id().to_owned(), + ) + }); + for oldest in by_age.into_iter().take(excess) { + completed.remove(&oldest); + } } pub struct E3RouterParams { @@ -106,6 +154,8 @@ pub struct E3RouterParams { replay_cursors: HashMap, recovery_store: Repository, recovered_selections: Vec, + teardown_deadlines: HashMap, + teardown_grace: std::time::Duration, } impl E3Router { @@ -117,6 +167,7 @@ impl E3Router { recovered_selections: vec![], recovery_store: repositories.request_router_checkpoint(), store: repositories.router(), + teardown_grace: SLASHABLE_FAILURE_TEARDOWN_GRACE, }; // Everything needs the committe meta factory so adding it here by default @@ -134,6 +185,8 @@ impl E3Router { replay_cursors: params.replay_cursors, recovery_store: params.recovery_store, recovered_selections: params.recovered_selections, + teardown_deadlines: params.teardown_deadlines, + teardown_grace: params.teardown_grace, } } } diff --git a/crates/request/src/routing/effects/build_context.rs b/crates/request/src/routing/effects/build_context.rs index b9ecadf2a4..c9e1ad7df2 100644 --- a/crates/request/src/routing/effects/build_context.rs +++ b/crates/request/src/routing/effects/build_context.rs @@ -10,6 +10,7 @@ pub struct E3RouterBuilder { pub recovered_selections: Vec, pub recovery_store: Repository, pub store: Repository, + pub teardown_grace: std::time::Duration, } impl E3RouterBuilder { @@ -18,6 +19,12 @@ impl E3RouterBuilder { self } + /// Override how long a slashably-failed E3's context is kept for its accusation window. + pub fn with_teardown_grace(mut self, grace: std::time::Duration) -> Self { + self.teardown_grace = grace; + self + } + /// Restore local committee-selection effects without creating another durable protocol event. pub fn with_recovered_selections( mut self, @@ -32,15 +39,16 @@ impl E3RouterBuilder { let legacy_snapshot: Option = self.store.read().await?; let recovery_store = self.recovery_store; let recovery_checkpoint = recovery_store.read().await?; - let (snapshot, replay_cursors) = match recovery_checkpoint { + let (snapshot, replay_cursors, teardown_deadlines) = match recovery_checkpoint { Some(checkpoint) => ( Some(E3RouterSnapshot { contexts: checkpoint.contexts, completed: checkpoint.completed, }), checkpoint.replay_cursors, + checkpoint.teardown_deadlines, ), - None => (legacy_snapshot, HashMap::new()), + None => (legacy_snapshot, HashMap::new(), HashMap::new()), }; let params = E3RouterParams { extensions: self.extensions.into(), @@ -49,6 +57,8 @@ impl E3RouterBuilder { replay_cursors, recovery_store, recovered_selections, + teardown_deadlines, + teardown_grace: self.teardown_grace, }; let router = match snapshot { diff --git a/crates/request/src/routing/effects/snapshot.rs b/crates/request/src/routing/effects/snapshot.rs index f90121699e..11d66efcbb 100644 --- a/crates/request/src/routing/effects/snapshot.rs +++ b/crates/request/src/routing/effects/snapshot.rs @@ -108,6 +108,8 @@ impl FromSnapshotWithParams for E3Router { replay_cursors: params.replay_cursors, recovery_store: params.recovery_store, recovered_selections: params.recovered_selections, + teardown_deadlines: params.teardown_deadlines, + teardown_grace: params.teardown_grace, }) } } diff --git a/crates/request/src/routing/handlers.rs b/crates/request/src/routing/handlers.rs index 34ecf16885..3a9e30f9c2 100644 --- a/crates/request/src/routing/handlers.rs +++ b/crates/request/src/routing/handlers.rs @@ -4,8 +4,10 @@ use super::effects::advance_request_router_cursor; use super::*; +use actix::AsyncContext; use anyhow::Context as _; use e3_events::{EventContext, InterfoldEventData, RequestRouterCheckpoint, Sequenced, SyncEffect}; +use tracing::{error, info}; impl E3Router { fn checkpoint_with_context(&mut self, context: &EventContext) -> Result<()> { @@ -20,12 +22,83 @@ impl E3Router { contexts: snapshot.contexts, completed: snapshot.completed, replay_cursors: self.replay_cursors.clone(), + teardown_deadlines: self.teardown_deadlines.clone(), }, context, )?; Ok(()) } + fn unix_now_secs() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or_default() + } + + /// Record when a slashably-failed E3 may be torn down and arm the timer for it. + fn schedule_teardown( + &mut self, + e3_id: E3id, + caused_by: EventContext, + ctx: &mut Context, + ) { + let deadline = Self::unix_now_secs().saturating_add(self.teardown_grace.as_secs()); + self.teardown_deadlines.insert(e3_id.clone(), deadline); + self.arm_teardown(e3_id, deadline, Some(caused_by), ctx); + } + + /// Fire `E3RequestComplete` for `e3_id` once `deadline` (unix seconds) has passed. + /// + /// Idempotent against the persisted map: a timer whose entry was already removed (the E3 + /// completed some other way, or a second timer was armed after a restart) does nothing. + /// `caused_by` is the `E3Failed` context when armed in-process; after a restart it is + /// gone and the completion is published as a root event. + fn arm_teardown( + &mut self, + e3_id: E3id, + deadline: u64, + caused_by: Option>, + ctx: &mut Context, + ) { + let delay = std::time::Duration::from_secs(deadline.saturating_sub(Self::unix_now_secs())); + ctx.run_later(delay, move |act, _| { + if act.teardown_deadlines.remove(&e3_id).is_none() { + return; + } + if act.completed.contains(&e3_id) || !act.contexts.contains_key(&e3_id) { + return; + } + info!( + %e3_id, + "Slashing window closed for the failed E3; tearing down its request context" + ); + let complete = E3RequestComplete { + e3_id: e3_id.clone(), + }; + let published = match caused_by { + Some(ec) => act.bus.publish(complete, ec), + None => act.bus.publish_without_context(complete), + }; + if let Err(err) = published { + error!(%e3_id, %err, "Failed to publish E3RequestComplete for the failed E3"); + } + }); + } + + /// Re-arm every persisted teardown deadline after a restart. Deadlines already in the + /// past fire on the next tick. + fn rearm_teardowns(&mut self, ctx: &mut Context) { + let pending: Vec<(E3id, u64)> = self + .teardown_deadlines + .iter() + .map(|(id, deadline)| (id.clone(), *deadline)) + .collect(); + for (e3_id, deadline) in pending { + self.arm_teardown(e3_id, deadline, None, ctx); + } + } + fn reconcile_recovered_selections(&mut self) -> Result<()> { for selection in std::mem::take(&mut self.recovered_selections) { if self.completed.contains(&selection.e3_id) { @@ -60,14 +133,15 @@ impl Actor for E3Router { type Context = Context; fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT) + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + self.rearm_teardowns(ctx); } } impl Handler for E3Router { type Result = (); - fn handle(&mut self, msg: InterfoldEvent, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { trap(EType::Event, &self.bus.with_ec(msg.get_ctx()), || { if matches!(msg.get_data(), InterfoldEventData::SyncEffect(SyncEffect)) { let event_context = msg.get_ctx().clone(); @@ -129,20 +203,25 @@ impl Handler for E3Router { .write_with_context(&context.snapshot()?, &event_context)?; } - let (_, ctx) = msg.into_components(); + let (_, ctx_ec) = msg.into_components(); match post_forward { PostForward::PublishComplete => { self.bus.publish( E3RequestComplete { e3_id: e3_id.clone(), }, - ctx, + ctx_ec, )?; } + PostForward::ScheduleTeardown => { + self.schedule_teardown(e3_id, ctx_ec, ctx); + } PostForward::Teardown => { self.contexts.remove(&e3_id); self.buffer.remove_e3(&e3_id); + self.teardown_deadlines.remove(&e3_id); self.completed.insert(e3_id); + prune_completed(&mut self.completed); } PostForward::None => (), } diff --git a/crates/request/src/routing/tests.rs b/crates/request/src/routing/tests.rs index 3fbab8d1aa..b69f7887e1 100644 --- a/crates/request/src/routing/tests.rs +++ b/crates/request/src/routing/tests.rs @@ -14,9 +14,9 @@ use async_trait::async_trait; use e3_data::{InMemStore, RepositoriesFactory}; use e3_events::{ hlc_factory::HlcFactory, BusHandle, CiphernodeSelected, DkgFoldAttestationContext, - DkgFoldAttestationContextEstablished, E3Requested, EventBus, InterfoldEventData, - RequestRouterCheckpoint, Sequencer, StoreEventRequested, SyncEffect, Unsequenced, - DKG_FOLD_ATTESTATION_CONTEXT_SCHEMA_VERSION, + DkgFoldAttestationContextEstablished, E3Failed, E3Requested, E3Stage, EventBus, EventType, + FailureReason, InterfoldEventData, RequestRouterCheckpoint, Sequencer, StoreEventRequested, + StoreEventResponse, SyncEffect, Unsequenced, DKG_FOLD_ATTESTATION_CONTEXT_SCHEMA_VERSION, }; use std::sync::{ atomic::{AtomicUsize, Ordering}, @@ -35,6 +35,33 @@ impl Handler for StoreSink { fn handle(&mut self, _: StoreEventRequested, _: &mut Self::Context) {} } +/// A store that sequences events so published events actually reach bus subscribers. +struct SequencingStore { + next_seq: u64, +} + +impl Actor for SequencingStore { + type Context = Context; +} + +impl Handler for SequencingStore { + type Result = (); + + fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) { + let StoreEventRequested { event, sender } = msg; + let seq = self.next_seq; + self.next_seq += 1; + sender.do_send(StoreEventResponse(event.into_sequenced(seq))); + } +} + +fn sequencing_bus() -> BusHandle { + let event_bus = EventBus::::default().start(); + let store = SequencingStore { next_seq: 1 }.start(); + let sequencer = Sequencer::new(&event_bus, store.recipient()).start(); + BusHandle::new(event_bus, sequencer, HlcFactory::new()).enable("router-teardown-test") +} + struct RecoveryExtension { hydrations: Arc, } @@ -101,6 +128,8 @@ async fn mid_e3_context_and_completed_set_survive_hydration() -> Result<()> { replay_cursors: HashMap::new(), recovery_store, recovered_selections: Vec::new(), + teardown_deadlines: HashMap::new(), + teardown_grace: SLASHABLE_FAILURE_TEARDOWN_GRACE, }; let recovered = E3Router::from_snapshot( params, @@ -135,6 +164,8 @@ async fn hydration_fails_when_an_active_context_snapshot_is_missing() -> Result< replay_cursors: HashMap::new(), recovery_store, recovered_selections: Vec::new(), + teardown_deadlines: HashMap::new(), + teardown_grace: SLASHABLE_FAILURE_TEARDOWN_GRACE, }; let error = match E3Router::from_snapshot( @@ -177,6 +208,7 @@ async fn recovery_is_direct_and_uses_one_checkpoint() -> Result<()> { contexts: vec![recovered_e3.clone()], completed: HashSet::new(), replay_cursors: HashMap::from([(aggregate_id, 12)]), + teardown_deadlines: HashMap::new(), }) .await?; @@ -192,6 +224,7 @@ async fn recovery_is_direct_and_uses_one_checkpoint() -> Result<()> { }], recovery_store: recovery_store.clone(), store: repositories.router(), + teardown_grace: SLASHABLE_FAILURE_TEARDOWN_GRACE, } .build() .await?; @@ -232,6 +265,195 @@ async fn recovery_is_direct_and_uses_one_checkpoint() -> Result<()> { Ok(()) } +#[actix::test] +async fn slashable_failure_tears_the_context_down_after_the_grace_window() -> Result<()> { + let e3_id = E3id::new("21", 31337); + let store = DataStore::from_in_mem(&InMemStore::new(false).start()); + let repositories = store.repositories(); + let recovery_store = repositories.request_router_checkpoint(); + let bus = sequencing_bus(); + let completions = Arc::new(AtomicUsize::new(0)); + bus.subscribe( + EventType::E3RequestComplete, + CompletionCounter { + completions: completions.clone(), + } + .start() + .recipient(), + ); + + let router = E3RouterBuilder { + bus: bus.clone(), + extensions: vec![E3MetaExtension::create()], + recovered_selections: Vec::new(), + recovery_store: recovery_store.clone(), + store: repositories.router(), + teardown_grace: std::time::Duration::from_secs(1), + } + .build() + .await?; + + router + .send( + InterfoldEvent::::test_event("request") + .data(E3Requested { + e3_id: e3_id.clone(), + ..Default::default() + }) + .seq(1) + .build(), + ) + .await?; + router + .send( + InterfoldEvent::::test_event("failed") + .data(E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGInvalidShares, + }) + .seq(2) + .build(), + ) + .await?; + + // During the grace window the context is alive (the accusation manager needs it) and + // the deadline is durable. + let checkpoint = recovery_store.read().await?.expect("checkpoint"); + assert!(checkpoint.contexts.contains(&e3_id)); + assert!(checkpoint.teardown_deadlines.contains_key(&e3_id)); + assert_eq!(completions.load(Ordering::SeqCst), 0); + + actix::clock::sleep(std::time::Duration::from_millis(1500)).await; + // Let the E3RequestComplete round-trip through the bus back into the router. + router + .send( + InterfoldEvent::::test_event("sync-effect") + .data(SyncEffect::new()) + .seq(3) + .build(), + ) + .await?; + + assert_eq!(completions.load(Ordering::SeqCst), 1); + let checkpoint = recovery_store.read().await?.expect("checkpoint"); + assert!( + !checkpoint.contexts.contains(&e3_id), + "the failed E3's context must be torn down once the slashing window closes" + ); + assert!(checkpoint.completed.contains(&e3_id)); + assert!(checkpoint.teardown_deadlines.is_empty()); + Ok(()) +} + +#[actix::test] +async fn a_persisted_teardown_deadline_is_rearmed_after_restart() -> Result<()> { + let e3_id = E3id::new("22", 31337); + let store = DataStore::from_in_mem(&InMemStore::new(false).start()); + let repositories = store.repositories(); + repositories + .router() + .repositories() + .context(&e3_id) + .write_sync(&E3ContextSnapshot { + e3_id: e3_id.clone(), + recipients: Vec::new(), + dependencies: Vec::new(), + }) + .await?; + let recovery_store = repositories.request_router_checkpoint(); + // A deadline that already passed while the node was down. + recovery_store + .write_sync(&RequestRouterCheckpoint { + contexts: vec![e3_id.clone()], + completed: HashSet::new(), + replay_cursors: HashMap::new(), + teardown_deadlines: HashMap::from([(e3_id.clone(), 1)]), + }) + .await?; + + let bus = sequencing_bus(); + let completions = Arc::new(AtomicUsize::new(0)); + bus.subscribe( + EventType::E3RequestComplete, + CompletionCounter { + completions: completions.clone(), + } + .start() + .recipient(), + ); + let router = E3RouterBuilder { + bus: bus.clone(), + extensions: vec![E3MetaExtension::create()], + recovered_selections: Vec::new(), + recovery_store: recovery_store.clone(), + store: repositories.router(), + teardown_grace: SLASHABLE_FAILURE_TEARDOWN_GRACE, + } + .build() + .await?; + + actix::clock::sleep(std::time::Duration::from_millis(200)).await; + router + .send( + InterfoldEvent::::test_event("sync-effect") + .data(SyncEffect::new()) + .seq(1) + .build(), + ) + .await?; + + assert_eq!(completions.load(Ordering::SeqCst), 1); + let checkpoint = recovery_store.read().await?.expect("checkpoint"); + assert!(!checkpoint.contexts.contains(&e3_id)); + assert!(checkpoint.completed.contains(&e3_id)); + Ok(()) +} + +struct CompletionCounter { + completions: Arc, +} + +impl Actor for CompletionCounter { + type Context = Context; +} + +impl Handler for CompletionCounter { + type Result = (); + + fn handle(&mut self, msg: InterfoldEvent, _: &mut Self::Context) { + if matches!(msg.get_data(), InterfoldEventData::E3RequestComplete(_)) { + self.completions.fetch_add(1, Ordering::SeqCst); + } + } +} + +/// `completed` is serialized into the checkpoint on every event; unbounded, its size grows +/// with the node's whole E3 history. The oldest ids must go first so late events for the +/// most recent completions are still rejected. +#[test] +fn completed_set_is_bounded_and_prunes_the_oldest_ids_first() { + let mut completed: HashSet = (0..(MAX_REMEMBERED_COMPLETIONS as u64 + 10)) + .map(|n| E3id::new(n.to_string(), 31337)) + .collect(); + prune_completed(&mut completed); + assert_eq!(completed.len(), MAX_REMEMBERED_COMPLETIONS); + for oldest in 0..10u64 { + assert!( + !completed.contains(&E3id::new(oldest.to_string(), 31337)), + "E3 {oldest} is the oldest and must have been pruned" + ); + } + let newest = MAX_REMEMBERED_COMPLETIONS as u64 + 9; + assert!(completed.contains(&E3id::new(newest.to_string(), 31337))); + assert!(completed.contains(&E3id::new("10", 31337))); + + // Under the bound nothing is touched. + let mut small: HashSet = HashSet::from([E3id::new("1", 1), E3id::new("2", 1)]); + prune_completed(&mut small); + assert_eq!(small.len(), 2); +} + #[actix::test] async fn request_time_attestation_contexts_survive_router_snapshots() -> Result<()> { let old_e3 = E3id::new("41", 1); @@ -281,6 +503,7 @@ async fn request_time_attestation_contexts_survive_router_snapshots() -> Result< contexts: vec![old_e3.clone(), new_e3.clone()], completed: HashSet::new(), replay_cursors: HashMap::new(), + teardown_deadlines: HashMap::new(), }) .await?; @@ -299,6 +522,8 @@ async fn request_time_attestation_contexts_survive_router_snapshots() -> Result< replay_cursors: HashMap::new(), recovery_store, recovered_selections: Vec::new(), + teardown_deadlines: HashMap::new(), + teardown_grace: SLASHABLE_FAILURE_TEARDOWN_GRACE, }, E3RouterSnapshot { contexts: vec![old_e3.clone()], diff --git a/crates/request/src/routing/workflow.rs b/crates/request/src/routing/workflow.rs index b28df0a29e..e314b6ef56 100644 --- a/crates/request/src/routing/workflow.rs +++ b/crates/request/src/routing/workflow.rs @@ -17,6 +17,9 @@ pub enum PostForward { PublishComplete, /// Tear down the context for this request and mark it as completed. Teardown, + /// The request failed for a reason that opens an accusation/slashing window. Keep the + /// context alive for that window, then publish `E3RequestComplete`. + ScheduleTeardown, /// No completion action is required. None, } @@ -141,11 +144,15 @@ impl RequestRouter { PostForward::PublishComplete } // Timeout failures have no accusation/slashing lifecycle, so the context can be - // torn down immediately. Misbehaviour failures (DKGInvalidShares, etc.) still need - // the accusation/slashing lifecycle to complete before teardown. + // torn down immediately. InterfoldEventData::E3Failed(data) if data.reason.ends_without_slashing() => { PostForward::PublishComplete } + // Misbehaviour failures (DKGInvalidShares, etc.) open an accusation/slashing + // window that the per-E3 accusation manager must be alive to serve. Nothing in + // that lifecycle reports back to the router, so without a scheduled teardown the + // context, its child actors and its checkpoint entry would leak forever. + InterfoldEventData::E3Failed(_) => PostForward::ScheduleTeardown, InterfoldEventData::E3RequestComplete(_) => PostForward::Teardown, _ => PostForward::None, }; diff --git a/crates/request/src/routing/workflow_tests.rs b/crates/request/src/routing/workflow_tests.rs index 99ae27f3be..452f83f764 100644 --- a/crates/request/src/routing/workflow_tests.rs +++ b/crates/request/src/routing/workflow_tests.rs @@ -357,20 +357,48 @@ fn requester_and_provider_failures_publish_complete() { } #[test] -fn e3_failed_invalid_shares_does_not_complete() { - // Slashable failures must NOT trigger E3RequestComplete — the accusation/slashing - // lifecycle must be allowed to finish first. +fn e3_failed_invalid_shares_schedules_teardown() { + // Slashable failures must NOT publish E3RequestComplete immediately — the + // accusation/slashing lifecycle needs the per-E3 accusation manager alive. But nothing + // in that lifecycle reports back, so the router must schedule the teardown itself or + // the context leaks forever. let id = e3id(); let msg = e3_failed(id.clone(), FailureReason::DKGInvalidShares); assert_eq!( RequestRouter::route(&msg, &HashSet::new()), RoutingDecision::Process { e3_id: id, - post_forward: PostForward::None, + post_forward: PostForward::ScheduleTeardown, } ); } +#[test] +fn every_slashable_failure_reason_schedules_teardown() { + // Only reasons that `ends_without_slashing()` rejects reach the teardown arm. Requester + // and provider faults (NoInputsReceived, ComputeProviderExpired, ComputeProviderFailed) + // end without an accusation, so they publish completion immediately and are covered by + // `requester_and_provider_failures_publish_complete`. + for reason in [ + FailureReason::DKGInvalidShares, + FailureReason::DecryptionInvalidShares, + FailureReason::VerificationFailed, + FailureReason::InsufficientCommitteeMembers, + FailureReason::None, + ] { + let id = e3id(); + let msg = e3_failed(id.clone(), reason.clone()); + assert_eq!( + RequestRouter::route(&msg, &HashSet::new()), + RoutingDecision::Process { + e3_id: id, + post_forward: PostForward::ScheduleTeardown, + }, + "{reason:?} must schedule a teardown, not leak the context" + ); + } +} + #[test] fn e3_failed_timeout_ignored_when_already_completed() { let id = e3id(); diff --git a/crates/slashing/Cargo.toml b/crates/slashing/Cargo.toml index 4a29a8bec4..b6b548eb8d 100644 --- a/crates/slashing/Cargo.toml +++ b/crates/slashing/Cargo.toml @@ -12,6 +12,7 @@ alloy = { workspace = true } anyhow = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } +e3-data = { workspace = true } e3-events = { workspace = true } e3-fhe-params = { workspace = true } e3-request = { workspace = true } @@ -23,4 +24,4 @@ sha2 = { workspace = true } tracing = { workspace = true } [dev-dependencies] -e3-data = { workspace = true } +bincode = { workspace = true } diff --git a/crates/slashing/src/accusation_voting/transitions/incoming_accusations.rs b/crates/slashing/src/accusation_voting/transitions/incoming_accusations.rs index 78b98bfcc7..839d6f0ec0 100644 --- a/crates/slashing/src/accusation_voting/transitions/incoming_accusations.rs +++ b/crates/slashing/src/accusation_voting/transitions/incoming_accusations.rs @@ -39,6 +39,7 @@ impl AccusationVoting { .saturating_add(self.vote_validity_secs) .saturating_add(self.accusation_deadline_skew_secs); warn!( + e3_id = %self.e3_id, "Ignoring accusation from {} — deadline {} outside local validity window \ (now={}, vote_validity_secs={}, skew_secs={}, max_accepted_deadline={})", accusation.accuser, @@ -54,6 +55,7 @@ impl AccusationVoting { // Verify accuser is in committee if !self.committee.contains(&accusation.accuser) { warn!( + e3_id = %self.e3_id, "Ignoring accusation from non-committee member {}", accusation.accuser ); @@ -63,6 +65,7 @@ impl AccusationVoting { // Verify accused is a committee member (defense-in-depth) if !self.committee.contains(&accusation.accused) { warn!( + e3_id = %self.e3_id, "Ignoring accusation against non-committee member {}", accusation.accused ); @@ -77,6 +80,7 @@ impl AccusationVoting { // Verify accuser's ECDSA signature if !self.verify_accusation_signature(&accusation) { warn!( + e3_id = %self.e3_id, "Invalid signature on accusation from {} — ignoring", accusation.accuser ); @@ -85,8 +89,15 @@ impl AccusationVoting { let accusation_id = Self::accusation_id(&accusation); - // Don't process duplicate accusations + // A duplicate of an accusation we already hold. The id is keyed on + // `(chain, e3, accused, proof_type)`, so two honest nodes that both saw the fault + // and accused a few seconds apart produce the same id with *different* signed + // `(issued_at, deadline)` windows. Every vote is bound to one window and the + // contract rejects a mixed set, so the committee must converge on a single window + // or a 3-of-4 quorum can never form. Converge deterministically on the later + // window; if the peer's wins, adopt it and re-sign our own vote against it. if self.pending.contains_key(&accusation_id) { + self.adopt_later_vote_window(accusation_id, accusation, ec, actions); return; } @@ -108,12 +119,16 @@ impl AccusationVoting { Ok(addr) => { if addr != accusation.accused { warn!( + e3_id = %self.e3_id, "Forwarded C3a/C3b payload signer {} != accused {} — cannot verify", addr, accusation.accused ); false } else if forwarded.payload.e3_id != self.e3_id { - warn!("Forwarded C3a/C3b payload e3_id mismatch — cannot verify"); + warn!( + e3_id = %self.e3_id, + "Forwarded C3a/C3b payload e3_id mismatch — cannot verify" + ); false } else { let expected = forwarded.payload.proof_type.circuit_names(); @@ -121,7 +136,10 @@ impl AccusationVoting { } } Err(e) => { - warn!("Forwarded C3a/C3b payload signature invalid: {e} — cannot verify"); + warn!( + e3_id = %self.e3_id, + "Forwarded C3a/C3b payload signature invalid: {e} — cannot verify" + ); false } }; @@ -134,6 +152,7 @@ impl AccusationVoting { // Bind the forwarded proof to the accusation. if forwarded.payload.proof_type != accusation.proof_type { warn!( + e3_id = %self.e3_id, "Forwarded C3a/C3b proof_type {:?} != accusation proof_type {:?} — cannot verify", forwarded.payload.proof_type, accusation.proof_type ); @@ -142,6 +161,7 @@ impl AccusationVoting { let computed_hash = Self::compute_payload_hash(forwarded); if computed_hash != accusation.data_hash { warn!( + e3_id = %self.e3_id, "Forwarded C3a/C3b data_hash mismatch (len {} vs {}) — cannot verify", computed_hash.len(), accusation.data_hash.len() @@ -165,7 +185,10 @@ impl AccusationVoting { ) { Ok(c) => c, Err(e) => { - warn!("Cannot derive committee size for ZK re-verification: {e}"); + warn!( + e3_id = %self.e3_id, + "Cannot derive committee size for ZK re-verification: {e}" + ); return; } }; @@ -245,7 +268,10 @@ impl AccusationVoting { match self.sign_vote_digest(&vote) { Ok(sig) => vote.signature = ArcBytes::from_bytes(&sig), Err(err) => { - error!("Failed to sign AccusationVote: {err}"); + error!( + e3_id = %self.e3_id, + "Failed to sign AccusationVote: {err}" + ); return; } } @@ -282,4 +308,92 @@ impl AccusationVoting { // Check quorum self.check_quorum(accusation_id, ec, actions); } + + /// Converge a pending accusation onto the winning vote window when a peer's duplicate + /// arrives. See `on_accusation_received_inner` for why this is needed. + /// + /// Votes already collected against the losing window are dropped: they cannot appear + /// in the same attestation as votes against the winning window. Their voters will + /// re-vote on receiving the winning accusation, exactly as this node does here. + fn adopt_later_vote_window( + &mut self, + accusation_id: [u8; 32], + incoming: ProofFailureAccusation, + ec: &EventContext, + actions: &mut Vec, + ) { + let Some(pending) = self.pending.get(&accusation_id) else { + return; + }; + let held = &pending.accusation; + let held_window = (held.issued_at, held.deadline); + let incoming_window = (incoming.issued_at, incoming.deadline); + // Same rule as the vote-side check in `on_vote_received_inner`, which only sees the + // window: the later window wins. Equal windows are a plain duplicate. + if incoming_window <= held_window { + return; + } + + info!( + "Adopting the later vote window (issued_at {} → {}, accuser {}) for accusation \ + against {} {:?}; re-signing own vote", + held.issued_at, + incoming.issued_at, + incoming.accuser, + incoming.accused, + incoming.proof_type + ); + + // Our position on the fault has not changed; only the window we sign it against. + let Some(our_vote) = pending + .votes_for + .iter() + .find(|v| v.voter == self.my_address) + else { + // We have not voted yet (pending a C3a/C3b re-verification); just switch the + // window so the eventual vote signs against it. + let pending = self.pending.get_mut(&accusation_id).expect("checked above"); + pending.accusation = incoming; + pending.votes_for.clear(); + return; + }; + let mut vote = AccusationVote { + e3_id: self.e3_id.clone(), + accusation_id, + voter: self.my_address, + data_hash: our_vote.data_hash, + issued_at: incoming.issued_at, + deadline: incoming.deadline, + signature: ArcBytes::default(), + }; + match self.sign_vote_digest(&vote) { + Ok(sig) => vote.signature = ArcBytes::from_bytes(&sig), + Err(err) => { + error!( + e3_id = %self.e3_id, + "Failed to re-sign AccusationVote for the adopted window: {err}" + ); + return; + } + } + + let pending = self.pending.get_mut(&accusation_id).expect("checked above"); + pending.accusation = incoming; + pending.votes_for = vec![vote.clone()]; + pending.ec = ec.clone(); + + actions.push(VoteAction::PublishVote { + vote, + ec: ec.clone(), + }); + + // Peers' votes against the new window may already have arrived and been buffered + // (or rejected, in which case they will be re-sent when those peers converge too). + if let Some(buffered) = self.buffered_votes.remove(&accusation_id) { + for vote in buffered { + self.on_vote_received_inner(vote, ec, actions); + } + } + self.check_quorum(accusation_id, ec, actions); + } } diff --git a/crates/slashing/src/accusation_voting/transitions/initiate_accusation.rs b/crates/slashing/src/accusation_voting/transitions/initiate_accusation.rs index 284a76213a..05f066fdb3 100644 --- a/crates/slashing/src/accusation_voting/transitions/initiate_accusation.rs +++ b/crates/slashing/src/accusation_voting/transitions/initiate_accusation.rs @@ -18,12 +18,14 @@ impl AccusationVoting { let accused_address = if event.accused_address == Address::ZERO { if let Some(&addr) = self.committee.get(event.accused_party_id as usize) { warn!( + e3_id = %self.e3_id, "Resolved Address::ZERO for party {} to committee address {}", event.accused_party_id, addr ); addr } else { error!( + e3_id = %self.e3_id, "Cannot resolve address for party {} (out of committee bounds) — dropping accusation", event.accused_party_id ); @@ -180,7 +182,10 @@ impl AccusationVoting { match self.sign_accusation_digest(&accusation) { Ok(sig) => accusation.signature = ArcBytes::from_bytes(&sig), Err(err) => { - error!("Failed to sign ProofFailureAccusation: {err}"); + error!( + e3_id = %self.e3_id, + "Failed to sign ProofFailureAccusation: {err}" + ); self.accused_proofs.remove(&key); return; } @@ -213,7 +218,10 @@ impl AccusationVoting { match self.sign_vote_digest(&own_vote) { Ok(sig) => own_vote.signature = ArcBytes::from_bytes(&sig), Err(err) => { - error!("Failed to sign own AccusationVote: {err}"); + error!( + e3_id = %self.e3_id, + "Failed to sign own AccusationVote: {err}" + ); self.accused_proofs.remove(&key); return; } diff --git a/crates/slashing/src/accusation_voting/transitions/reverify_proofs.rs b/crates/slashing/src/accusation_voting/transitions/reverify_proofs.rs index d486e3265e..f53ff071dc 100644 --- a/crates/slashing/src/accusation_voting/transitions/reverify_proofs.rs +++ b/crates/slashing/src/accusation_voting/transitions/reverify_proofs.rs @@ -47,13 +47,19 @@ impl AccusationVoting { let zk_passed = match msg.response { ComputeResponseKind::Zk(ZkResponse::VerifyShareProofs(r)) => { if r.party_results.is_empty() { - warn!("Empty ZK re-verification results — abstaining"); + warn!( + e3_id = %self.e3_id, + "Empty ZK re-verification results — abstaining" + ); return actions; } r.party_results.first().is_some_and(|r| r.all_verified) } _ => { - warn!("Unexpected ComputeResponse kind for C3a/C3b re-verification — abstaining"); + warn!( + e3_id = %self.e3_id, + "Unexpected ComputeResponse kind for C3a/C3b re-verification — abstaining" + ); return actions; } }; @@ -101,7 +107,10 @@ impl AccusationVoting { match self.sign_vote_digest(&vote) { Ok(sig) => vote.signature = ArcBytes::from_bytes(&sig), Err(err) => { - error!("Failed to sign C3a/C3b AccusationVote: {err}"); + error!( + e3_id = %self.e3_id, + "Failed to sign C3a/C3b AccusationVote: {err}" + ); return actions; } } @@ -137,6 +146,7 @@ impl AccusationVoting { }; error!( + e3_id = %self.e3_id, "C3a/C3b ZK re-verification failed for {:?} — abstaining from vote", reverif.proof_type ); diff --git a/crates/slashing/src/accusation_voting/transitions/vote.rs b/crates/slashing/src/accusation_voting/transitions/vote.rs index 1052ee5eca..ee9e1919ec 100644 --- a/crates/slashing/src/accusation_voting/transitions/vote.rs +++ b/crates/slashing/src/accusation_voting/transitions/vote.rs @@ -29,7 +29,10 @@ impl AccusationVoting { // Verify voter is in committee if !self.committee.contains(&vote.voter) { - warn!("Ignoring vote from non-committee member {}", vote.voter); + warn!( + e3_id = %self.e3_id, + "Ignoring vote from non-committee member {}", vote.voter + ); return; } @@ -40,14 +43,17 @@ impl AccusationVoting { // Verify voter's ECDSA signature if !self.verify_vote_signature(&vote) { - warn!("Invalid signature on vote from {} — ignoring", vote.voter); + warn!( + e3_id = %self.e3_id, + "Invalid signature on vote from {} — ignoring", vote.voter + ); return; } let vote_accusation_id = vote.accusation_id; // Find the pending accusation - let Some(pending) = self.pending.get_mut(&vote_accusation_id) else { + let Some(pending) = self.pending.get(&vote_accusation_id) else { // Unknown accusation — buffer the vote for replay. let committee_len = self.committee.len(); let buf = self.buffered_votes.entry(vote_accusation_id).or_default(); @@ -55,6 +61,7 @@ impl AccusationVoting { buf.push(vote); } else { warn!( + e3_id = %self.e3_id, "Buffered votes for unknown accusation {:?} reached committee-size cap — dropping vote", vote_accusation_id ); @@ -62,24 +69,42 @@ impl AccusationVoting { return; }; - // Reject votes whose signed window disagrees with the accusation. - if vote.issued_at != pending.accusation.issued_at - || vote.deadline != pending.accusation.deadline - { - warn!( - "Ignoring vote from {} — issued_at {} (expected {}) or deadline {} (expected {}) does not match the accusation", - vote.voter, - vote.issued_at, - pending.accusation.issued_at, - vote.deadline, - pending.accusation.deadline - ); + // A vote signed against a different window than the accusation we hold. Two honest + // accusers produce the same accusation id with different windows and the committee + // converges on the later one (see `adopt_later_vote_window`). A peer that converged + // before us sends votes against the winning window while we still hold the losing + // one; dropping them would lose the quorum. Buffer them: they are replayed when we + // adopt the window. Votes against a window that *loses* to ours are simply stale. + let held_window = (pending.accusation.issued_at, pending.accusation.deadline); + if (vote.issued_at, vote.deadline) != held_window { + if (vote.issued_at, vote.deadline) > held_window { + let committee_len = self.committee.len(); + let buf = self.buffered_votes.entry(vote_accusation_id).or_default(); + if buf.len() < committee_len { + info!( + "Buffering vote from {} signed against a later window (issued_at {} vs held {}) until we converge", + vote.voter, vote.issued_at, held_window.0 + ); + buf.push(vote); + } + } else { + warn!( + e3_id = %self.e3_id, + "Ignoring vote from {} — issued_at {} (expected {}) or deadline {} (expected {}) does not match the accusation", + vote.voter, vote.issued_at, held_window.0, vote.deadline, held_window.1 + ); + } return; } + let pending = self + .pending + .get_mut(&vote_accusation_id) + .expect("pending accusation checked above"); // Reject votes from the accused party — conflict of interest if vote.voter == pending.accusation.accused { warn!( + e3_id = %self.e3_id, "Ignoring vote from accused party {} on their own accusation", vote.voter ); @@ -97,6 +122,7 @@ impl AccusationVoting { && vote.data_hash != pending.accusation.data_hash { warn!( + e3_id = %self.e3_id, "Accuser {} sent vote with data_hash inconsistent with their accusation — rejecting vote", vote.voter ); @@ -174,6 +200,7 @@ impl AccusationVoting { }; warn!( + e3_id = %self.e3_id, "Accusation against {} for {:?} timed out with {} agreeing votes — outcome: {:?}", pending.accusation.accused, pending.accusation.proof_type, diff --git a/crates/slashing/src/accusation_voting/workflow_tests/voting.rs b/crates/slashing/src/accusation_voting/workflow_tests/voting.rs index 4c31b8eafd..988944a0a9 100644 --- a/crates/slashing/src/accusation_voting/workflow_tests/voting.rs +++ b/crates/slashing/src/accusation_voting/workflow_tests/voting.rs @@ -145,3 +145,100 @@ fn quorum_boundary() { "quorum must fire at the M-th vote" ); } + +/// Build and sign an accusation as `who` with an explicit vote window. +fn signed_accusation( + who: &PrivateKeySigner, + e3_id: &E3id, + accused: Address, + data_hash: [u8; 32], + issued_at: u64, + deadline: u64, +) -> ProofFailureAccusation { + let mut accusation = ProofFailureAccusation { + e3_id: e3_id.clone(), + accuser: who.address(), + accused, + accused_party_id: 1, + proof_type: ProofType::C1PkGeneration, + data_hash, + issued_at, + deadline, + signed_payload: None, + signature: ArcBytes::default(), + }; + let digest = AccusationVoting::accusation_digest(&accusation); + let sig = who.sign_message_sync(&digest).unwrap(); + accusation.signature = ArcBytes::from_bytes(&sig.as_bytes()); + accusation +} + +/// Two honest nodes that both see the same bad proof each initiate an accusation, a few +/// seconds apart. Both accusations get the same `accusation_id` (it is keyed on +/// `(chain, e3, accused, proof_type)` only) but different `(issued_at, deadline)` windows. +/// Each node keeps whichever it saw first and, before the fix, rejected every vote signed +/// against the other window as "does not match the accusation" — so with two accusers a +/// 3-of-4 quorum could never form, and a genuine fault went unslashed by accident. +/// +/// A node that already holds a pending accusation must accept a peer's later duplicate as +/// confirmation of the *same* accusation and re-vote against the peer's window, so both +/// sides converge on a set of votes the contract will verify together. +#[test] +fn concurrent_accusers_converge_on_one_vote_window() { + let me = signer(1); + let b = signer(2); + let c = signer(3); + let accused = signer(9).address(); + let committee = vec![me.address(), b.address(), c.address(), accused]; + let mut v = voting_with(&me, committee, 1, 3); + let sm = v.slashing_manager; + let data_hash = [0x11; 32]; + + // I saw the fault first: my own accusation with my window. + let own = signed_vote(&me, sm, &v.e3_id, [0u8; 32], data_hash, NOW + VALIDITY); + let id = insert_pending(&mut v, &me, accused, data_hash, NOW + VALIDITY, own); + v.pending.get_mut(&id).unwrap().votes_for[0].accusation_id = id; + v.received_data.insert( + (accused, ProofType::C1PkGeneration), + ReceivedProofData { + data_hash, + verification_passed: false, + evidence: Bytes::new(), + }, + ); + + // B saw it 5 s later and accused independently, with a later window. + let later = NOW + 5; + let b_accusation = signed_accusation(&b, &v.e3_id, accused, data_hash, later, later + VALIDITY); + assert_eq!(AccusationVoting::accusation_id(&b_accusation), id); + let actions = v.on_accusation_received(b_accusation, &ctx()); + + // I must re-vote against B's window so my vote is valid alongside B's and C's. + let my_revote = actions.iter().find_map(|a| match a { + VoteAction::PublishVote { vote, .. } if vote.voter == me.address() => Some(vote), + _ => None, + }); + let my_revote = my_revote.expect("must re-vote against the peer's window"); + assert_eq!(my_revote.deadline, later + VALIDITY); + assert_eq!(my_revote.issued_at, later); + + // C (who also saw the fault) votes against B's window, as B's own vote will. + let vote_b = signed_vote(&b, sm, &v.e3_id, id, data_hash, later + VALIDITY); + let vote_c = signed_vote(&c, sm, &v.e3_id, id, data_hash, later + VALIDITY); + let mut actions = v.on_vote_received(vote_b, &ctx()); + actions.extend(v.on_vote_received(vote_c, &ctx())); + + let quorum = actions.iter().find_map(|a| match a { + VoteAction::PublishQuorum { quorum, .. } => Some(quorum), + _ => None, + }); + let quorum = quorum.expect("3 votes on the same window must reach the 3-vote quorum"); + assert_eq!(quorum.votes_for.len(), 3); + assert!( + quorum + .votes_for + .iter() + .all(|vote| vote.deadline == later + VALIDITY), + "every vote in the attestation must share one deadline or the contract rejects it" + ); +} diff --git a/crates/slashing/src/commitment_consistency/actor.rs b/crates/slashing/src/commitment_consistency/actor.rs index 5178ec6b9e..91217b639e 100644 --- a/crates/slashing/src/commitment_consistency/actor.rs +++ b/crates/slashing/src/commitment_consistency/actor.rs @@ -33,6 +33,7 @@ //! [`CommitmentConsistencyViolation`]: e3_events::CommitmentConsistencyViolation use actix::{Actor, Addr, Context, Handler}; +use e3_data::Repository; use e3_events::{ BusHandle, CommitmentConsistencyCheckRequested, CommitmentLink, E3id, EventPublisher, EventSubscriber, EventType, InterfoldEvent, InterfoldEventData, ProofVerificationPassed, @@ -41,7 +42,7 @@ use e3_events::{ use e3_utils::NotifySync; use tracing::{error, info}; -use crate::domain::commitment_consistency::CommitmentConsistency; +use crate::domain::commitment_consistency::{CommitmentConsistency, CommitmentConsistencySnapshot}; /// Per-E3 actor that enforces cross-circuit commitment consistency. /// @@ -52,6 +53,9 @@ pub struct CommitmentConsistencyChecker { e3_id: E3id, /// Plain, synchronous consistency core. Owns the proof cache and links. consistency: CommitmentConsistency, + /// Durable copy of the verified-proof cache; written after every mutation. `None` only + /// in tests that never restart. See [`CommitmentConsistencySnapshot`] for why. + snapshot_repo: Option>, } impl CommitmentConsistencyChecker { @@ -65,6 +69,31 @@ impl CommitmentConsistencyChecker { bus: bus.clone(), e3_id: e3_id.clone(), consistency: CommitmentConsistency::new(e3_id, links, committee_h), + snapshot_repo: None, + } + } + + /// Attach the durable cache. If `restored` is given the cache starts from it. + pub fn with_snapshot( + mut self, + repo: Repository, + restored: Option, + ) -> Self { + if let Some(snapshot) = restored { + self.consistency.restore(snapshot); + info!( + "CommitmentConsistencyChecker for E3 {} restored {} cached proof(s)", + self.e3_id, + self.consistency.cached_proof_count() + ); + } + self.snapshot_repo = Some(repo); + self + } + + fn persist(&self) { + if let Some(repo) = &self.snapshot_repo { + repo.write(&self.consistency.snapshot()); } } @@ -123,9 +152,14 @@ impl Handler> for CommitmentConsistencyCheck _ctx: &mut Self::Context, ) -> Self::Result { let (data, ec) = msg.into_components(); - for violation in self.consistency.on_proof_verified(data) { + let violations = self.consistency.on_proof_verified(data); + self.persist(); + for violation in violations { if let Err(err) = self.bus.publish(violation, ec.clone()) { - error!("Failed to publish CommitmentConsistencyViolation: {err}"); + error!( + e3_id = %self.e3_id, + "Failed to publish CommitmentConsistencyViolation: {err}" + ); } } } @@ -143,16 +177,23 @@ impl Handler> for CommitmentCons let Some(outcome) = self.consistency.on_check_requested(data) else { return; }; + self.persist(); for violation in outcome.violations { if let Err(err) = self.bus.publish(violation, ec.clone()) { - error!("Failed to publish CommitmentConsistencyViolation: {err}"); + error!( + e3_id = %self.e3_id, + "Failed to publish CommitmentConsistencyViolation: {err}" + ); } } // Respond to ShareVerificationActor. if let Err(err) = self.bus.publish(outcome.complete, ec) { - error!("Failed to publish CommitmentConsistencyCheckComplete: {err}"); + error!( + e3_id = %self.e3_id, + "Failed to publish CommitmentConsistencyCheckComplete: {err}" + ); } } } diff --git a/crates/slashing/src/commitment_consistency/workflow.rs b/crates/slashing/src/commitment_consistency/workflow.rs index 1d0c53e162..242bb4df6f 100644 --- a/crates/slashing/src/commitment_consistency/workflow.rs +++ b/crates/slashing/src/commitment_consistency/workflow.rs @@ -29,10 +29,12 @@ use e3_events::{ ProofVerificationPassed, }; use e3_utils::utility_types::ArcBytes; +use serde::{Deserialize, Serialize}; use std::collections::{BTreeSet, HashMap}; use tracing::warn; /// Cached data from a verified proof. +#[derive(Clone, Debug, Serialize, Deserialize)] struct VerifiedProofData { party_id: u64, address: Address, @@ -45,6 +47,23 @@ struct VerifiedProofData { proof_data: ArcBytes, } +/// Durable image of the verified-proof cache. +/// +/// EventStore replay on restart starts at the aggregate snapshot cursor, not at the start of +/// the log, so a freshly created checker never sees the `ProofVerificationPassed` events that +/// were delivered before the crash. The one it can never recover live is this node's **own** +/// C0: peers' keys are re-fetched and re-verified, but a node does not verify its own key, so +/// the only copy was the pre-crash event. Without it every peer C3 that encrypts *to this node* +/// fails the C3→C0 link on the next pre-ZK gate, both peers are flagged inconsistent, and the +/// E3 fails with "too few honest parties" (Round 13, cn3). +/// +/// The cache is therefore persisted after every mutation and restored on hydrate. The map is +/// flattened to a `Vec` because the key is a tuple. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct CommitmentConsistencySnapshot { + entries: Vec<(Address, ProofType, Vec)>, +} + /// Describes a source entry whose commitments are inconsistent with a target. struct Mismatch { party_id: u64, @@ -93,6 +112,31 @@ impl CommitmentConsistency { } } + /// Export the verified-proof cache for persistence. + pub(crate) fn snapshot(&self) -> CommitmentConsistencySnapshot { + CommitmentConsistencySnapshot { + entries: self + .verified + .iter() + .map(|((address, proof_type), entries)| (*address, *proof_type, entries.clone())) + .collect(), + } + } + + /// Restore a cache exported by [`Self::snapshot`]. Replaces the current cache. + pub(crate) fn restore(&mut self, snapshot: CommitmentConsistencySnapshot) { + self.verified = snapshot + .entries + .into_iter() + .map(|(address, proof_type, entries)| ((address, proof_type), entries)) + .collect(); + } + + /// Number of cached verified proofs (all parties, all types). + pub(crate) fn cached_proof_count(&self) -> usize { + self.verified.values().map(Vec::len).sum() + } + /// Number of registered links (for actor startup logging). pub(crate) fn link_count(&self) -> usize { self.links.len() @@ -178,7 +222,7 @@ impl CommitmentConsistency { continue; } for src in srcs { - if self.skip_c2_to_c4_source(src_type, src.party_id) { + if self.skip_capped_roster_source(src_type, src.party_id) { continue; } let vals = link.extract_source_values(&src.public_signals); @@ -231,7 +275,7 @@ impl CommitmentConsistency { continue; } for src in srcs { - if self.skip_c2_to_c4_source(src_type, src.party_id) { + if self.skip_capped_roster_source(src_type, src.party_id) { continue; } let vals = link.extract_source_values(&src.public_signals); @@ -263,11 +307,29 @@ impl CommitmentConsistency { } } - /// C4 circuits only witness `expected_commitments` for the lowest `H` senders. - fn skip_c2_to_c4_source(&self, proof_type: ProofType, party_id: u64) -> bool { + /// Circuits that only witness the lowest `H` senders cannot attest to a surplus + /// party's proof, so a source above that cap must not be faulted. + /// + /// Two independent circuit families cap their roster at `H`: + /// + /// - **C4** binds `expected_commitments` for the lowest `H` C2a/C2b senders. + /// - **C5** witnesses the canonical honest subset chosen by + /// `select_honest_set`, which keeps the `H` lowest party IDs and leaves the + /// remaining `N - H` parties in the full committee (see + /// `e3_aggregator::public_key_aggregation`). Every one of the `N` parties + /// produces a C1 proof, so on any committee with `N > H` the surplus C1 + /// sources have no C5 target and would otherwise be reported as violations + /// on a completely successful DKG round. + /// + /// C6 needs no such exemption: only the `H` honest-subset members submit + /// decryption shares and every canonical committee has `H == T + 1`, which is + /// exactly the C7 witness width. + fn skip_capped_roster_source(&self, proof_type: ProofType, party_id: u64) -> bool { matches!( proof_type, - ProofType::C2aSkShareComputation | ProofType::C2bESmShareComputation + ProofType::C2aSkShareComputation + | ProofType::C2bESmShareComputation + | ProofType::C1PkGeneration ) && party_id as usize >= self.committee_h } @@ -340,6 +402,7 @@ impl CommitmentConsistency { // but guards against future regressions). if m.data_hash == [0u8; 32] { warn!( + e3_id = %self.e3_id, "[{}] Skipping mismatch with zero data_hash for party {} ({}) {:?}", link.name(), m.party_id, diff --git a/crates/slashing/src/commitment_consistency/workflow_tests.rs b/crates/slashing/src/commitment_consistency/workflow_tests.rs index 29be2b3752..f021a703fd 100644 --- a/crates/slashing/src/commitment_consistency/workflow_tests.rs +++ b/crates/slashing/src/commitment_consistency/workflow_tests.rs @@ -47,6 +47,43 @@ impl CommitmentLink for TestLink { } } +/// A link that scans **every** 32-byte field of the target's public signals, +/// mirroring how `C1ToC5PkCommitmentLink::check_signals` searches all `H` input +/// slots of the C5 aggregate rather than only the first one. +struct ChunkScanLink { + scope: LinkScope, + source: ProofType, + target: ProofType, +} + +impl CommitmentLink for ChunkScanLink { + fn name(&self) -> &'static str { + "chunk_scan_link" + } + fn source_proof_type(&self) -> ProofType { + self.source + } + fn target_proof_type(&self) -> ProofType { + self.target + } + fn scope(&self) -> LinkScope { + self.scope + } + fn extract_source_values(&self, public_signals: &[u8]) -> Vec { + if public_signals.len() < 32 { + return Vec::new(); + } + let mut v = [0u8; 32]; + v.copy_from_slice(&public_signals[..32]); + vec![v] + } + fn check_signals(&self, source_values: &[FieldValue], target_public_signals: &[u8]) -> bool { + target_public_signals + .chunks_exact(32) + .any(|chunk| source_values.iter().any(|v| v[..] == *chunk)) + } +} + fn e3() -> E3id { E3id::new("7", 31337) } @@ -314,3 +351,208 @@ fn c2_sender_at_or_above_h_skips_c4_cross_check() { "party_id >= H must be outside C4 expected_commitments roster" ); } + +#[test] +fn surplus_c1_sender_is_not_faulted_when_c5_witnesses_only_h_parties() { + // Micro committee: N=9, H=5. All nine parties produce a C1 proof, but the + // aggregator caps the canonical honest subset to the five lowest party IDs, + // so C5 can only ever witness parties 0..=4. Faulting the surplus parties + // would accuse four honest operators on every successful DKG round. + const COMMITTEE_H: usize = 5; + const COMMITTEE_N: u64 = 9; + + let link = Box::new(ChunkScanLink { + scope: LinkScope::CrossParty, + source: ProofType::C1PkGeneration, + target: ProofType::C5PkAggregation, + }); + let mut svc = CommitmentConsistency::new(e3(), vec![link], COMMITTEE_H); + + // Every committee member publishes a C1 proof with its own commitment. + for party_id in 0..COMMITTEE_N { + let violations = svc.on_proof_verified(passed( + e3(), + party_id, + addr(party_id as u8 + 1), + ProofType::C1PkGeneration, + [party_id as u8 + 0xC0; 32], + signals(party_id as u8), + )); + assert!( + violations.is_empty(), + "no C5 target is cached yet, so no party can be faulted" + ); + } + + // C5 lands, witnessing only the lowest H party IDs. The aggregator proves + // one commitment per honest slot; the surplus parties are absent by design. + let mut c5_signals = Vec::new(); + for party_id in 0..COMMITTEE_H as u64 { + c5_signals.extend_from_slice(&[party_id as u8; 32]); + } + + let violations = svc.on_proof_verified(passed( + e3(), + 0, + addr(0xAA), + ProofType::C5PkAggregation, + [0xC5; 32], + ArcBytes::from_bytes(&c5_signals), + )); + + assert!( + violations.is_empty(), + "surplus honest C1 senders (party_id >= H) must not be accused when C5 \ + only witnesses the H lowest parties; got {} violation(s) for parties {:?}", + violations.len(), + violations + .iter() + .map(|v| v.accused_party_id) + .collect::>() + ); +} + +#[test] +fn c1_sender_below_h_is_still_faulted_when_absent_from_c5() { + // The cap must not become a blanket exemption: a party inside the honest + // roster that C5 does not attest to is still a real inconsistency. + const COMMITTEE_H: usize = 5; + + let link = Box::new(ChunkScanLink { + scope: LinkScope::CrossParty, + source: ProofType::C1PkGeneration, + target: ProofType::C5PkAggregation, + }); + let mut svc = CommitmentConsistency::new(e3(), vec![link], COMMITTEE_H); + + // Party 1 is inside the roster but publishes a commitment C5 never proves. + svc.on_proof_verified(passed( + e3(), + 1, + addr(0x11), + ProofType::C1PkGeneration, + [0xC1; 32], + signals(0xEE), + )); + + let mut c5_signals = Vec::new(); + for party_id in 0..COMMITTEE_H as u64 { + c5_signals.extend_from_slice(&[party_id as u8; 32]); + } + + let violations = svc.on_proof_verified(passed( + e3(), + 0, + addr(0xAA), + ProofType::C5PkAggregation, + [0xC5; 32], + ArcBytes::from_bytes(&c5_signals), + )); + + assert_eq!( + violations.len(), + 1, + "a party inside the H roster whose commitment C5 does not carry is a real violation" + ); + assert_eq!(violations[0].accused_party_id, 1); +} + +/// Round 13, cn3: replay after a restart starts at the aggregate snapshot cursor, so the +/// pre-crash `ProofVerificationPassed` events never reach a freshly built checker. The node's +/// own C0 is the one it can never re-learn live (peers' keys are re-fetched and re-verified, +/// its own is not). A cache rebuilt from the durable snapshot must judge exactly as the +/// original would have — a peer C3 that encrypts to this node passes, one that encrypts to +/// an unknown key is still faulted. +#[test] +fn restored_cache_keeps_the_own_c0_target_across_a_restart() { + let c3_to_c0 = || -> Box { + Box::new(TestLink { + scope: LinkScope::SourceMustExistInTargets, + source: ProofType::C3aSkShareEncryption, + target: ProofType::C0PkBfv, + }) + }; + let own = addr(1); + let peer = addr(2); + let own_pk = signals(0x11); + + // Pre-crash: own C0 is cached from the local publish path. + let mut before = CommitmentConsistency::new(e3(), vec![c3_to_c0()], 2); + assert!(before + .on_proof_verified(passed( + e3(), + 1, + own, + ProofType::C0PkBfv, + [0xC0; 32], + own_pk.clone(), + )) + .is_empty()); + let snapshot = before.snapshot(); + + // Post-restart: a fresh checker restored from the snapshot. + let mut after = CommitmentConsistency::new(e3(), vec![c3_to_c0()], 2); + assert_eq!(after.cached_proof_count(), 0); + after.restore(snapshot); + assert_eq!(after.cached_proof_count(), 1); + + // A peer C3 encrypting to our pk must pass — this is the exact link that faulted both + // peers in Round 13 because the own C0 target was missing. + let ok = after.on_proof_verified(passed( + e3(), + 2, + peer, + ProofType::C3aSkShareEncryption, + [0xC3; 32], + own_pk, + )); + assert!( + ok.is_empty(), + "C3 to our own pk must match the restored C0 target" + ); + + // And a C3 to a pk nobody published is still a real violation. + let bad = after.on_proof_verified(passed( + e3(), + 2, + peer, + ProofType::C3aSkShareEncryption, + [0xC4; 32], + signals(0x99), + )); + assert_eq!(bad.len(), 1); + assert_eq!(bad[0].accused_party_id, 2); +} + +/// A restored-then-empty snapshot is not the same as "no cache": an empty snapshot restores +/// to an empty cache and the SourceMustExistInTargets skip-when-no-targets rule still applies. +#[test] +fn snapshot_roundtrip_is_lossless() { + let mut svc = CommitmentConsistency::new(e3(), vec![same_party_link()], 2); + for (party, byte) in [(0u64, 0x10u8), (1, 0x20), (2, 0x30)] { + svc.on_proof_verified(passed( + e3(), + party, + addr(party as u8 + 5), + ProofType::C1PkGeneration, + [byte; 32], + signals(byte), + )); + svc.on_proof_verified(passed( + e3(), + party, + addr(party as u8 + 5), + ProofType::C2aSkShareComputation, + [byte + 1; 32], + signals(byte), + )); + } + assert_eq!(svc.cached_proof_count(), 6); + let snap = svc.snapshot(); + let bytes = bincode::serialize(&snap).expect("serialize"); + let back: CommitmentConsistencySnapshot = bincode::deserialize(&bytes).expect("deserialize"); + let mut restored = CommitmentConsistency::new(e3(), vec![same_party_link()], 2); + restored.restore(back); + assert_eq!(restored.cached_proof_count(), 6); + assert_eq!(restored.snapshot().entries.len(), snap.entries.len()); +} diff --git a/crates/slashing/src/commitment_consistency_checker_ext.rs b/crates/slashing/src/commitment_consistency_checker_ext.rs index d779919f40..f7cbdf0bc5 100644 --- a/crates/slashing/src/commitment_consistency_checker_ext.rs +++ b/crates/slashing/src/commitment_consistency_checker_ext.rs @@ -12,9 +12,12 @@ //! in the [`E3Context`] so it receives routed events. use crate::actors::commitment_consistency_checker::CommitmentConsistencyChecker; +use crate::domain::commitment_consistency::CommitmentConsistencySnapshot; +use crate::repo::CommitmentConsistencyRepositoryFactory; use actix::Actor; use anyhow::Result; use async_trait::async_trait; +use e3_data::{DataStore, RepositoriesFactory}; use e3_events::{BusHandle, CommitmentLink, Event, InterfoldEvent, InterfoldEventData}; use e3_fhe_params::BfvPreset; use e3_request::{E3Context, E3ContextSnapshot, E3Extension, META_KEY}; @@ -27,20 +30,24 @@ pub struct CommitmentConsistencyCheckerExtension { bus: BusHandle, /// Factory that builds commitment links for a given BFV preset. links_factory: LinksFactory, + /// Backing store for the per-E3 durable proof cache. + store: DataStore, } impl CommitmentConsistencyCheckerExtension { pub fn create( bus: &BusHandle, + store: &DataStore, links_factory: impl Fn(BfvPreset) -> Vec> + Send + Sync + 'static, ) -> Box { Box::new(Self { bus: bus.clone(), links_factory: Box::new(links_factory), + store: store.clone(), }) } - fn start_checker(&self, ctx: &mut E3Context) { + fn start_checker(&self, ctx: &mut E3Context, restored: Option) { if ctx .get_event_recipient("commitment_consistency_checker") .is_some() @@ -51,7 +58,10 @@ impl CommitmentConsistencyCheckerExtension { let e3_id = ctx.e3_id.clone(); let Some(meta) = ctx.get_dependency(META_KEY) else { - error!("E3Meta not available; cannot start CommitmentConsistencyChecker"); + error!( + e3_id = %e3_id, + "E3Meta not available; cannot start CommitmentConsistencyChecker" + ); return; }; @@ -75,7 +85,10 @@ impl CommitmentConsistencyCheckerExtension { // The request router owns delivery and lifetime for this per-E3 actor. Subscribing it to // the global bus as well would deliver every event twice and keep the actor alive after // the E3 context is removed. - let addr = CommitmentConsistencyChecker::new(&self.bus, e3_id, links, committee_h).start(); + let repo = self.store.repositories().commitment_consistency(&e3_id); + let addr = CommitmentConsistencyChecker::new(&self.bus, e3_id, links, committee_h) + .with_snapshot(repo, restored) + .start(); ctx.set_event_recipient("commitment_consistency_checker", Some(addr.into())); } @@ -92,12 +105,23 @@ impl E3Extension for CommitmentConsistencyCheckerExtension { return; } - self.start_checker(ctx); + self.start_checker(ctx, None); } + /// Recreate the checker with the proof cache it had before the crash. + /// + /// EventStore replay only covers events after the aggregate snapshot cursor, so the + /// `ProofVerificationPassed` events this checker consumed before the crash are never + /// redelivered. Restoring the cache is the only way the node's own C0 gets back in. async fn hydrate(&self, ctx: &mut E3Context, _snapshot: &E3ContextSnapshot) -> Result<()> { if ctx.get_dependency(META_KEY).is_some() { - self.start_checker(ctx); + let restored = self + .store + .repositories() + .commitment_consistency(&ctx.e3_id) + .read() + .await?; + self.start_checker(ctx, restored); } Ok(()) @@ -147,8 +171,11 @@ mod tests { } } - fn test_context(e3_id: E3id) -> E3Context { - let store = DataStore::from_in_mem(&InMemStore::new(false).start()); + fn test_store() -> DataStore { + DataStore::from_in_mem(&InMemStore::new(false).start()) + } + + fn test_context(store: &DataStore, e3_id: E3id) -> E3Context { let repositories = store.repositories(); E3Context::from_params(E3ContextParams { repository: repositories.context(&e3_id), @@ -160,9 +187,10 @@ mod tests { #[actix::test] async fn hydrate_recreates_checker_when_meta_was_recovered() -> Result<()> { let bus = test_bus(); - let extension = CommitmentConsistencyCheckerExtension::create(&bus, |_| Vec::new()); + let store = test_store(); + let extension = CommitmentConsistencyCheckerExtension::create(&bus, &store, |_| Vec::new()); let e3_id = E3id::new("0", 31337); - let mut ctx = test_context(e3_id.clone()); + let mut ctx = test_context(&store, e3_id.clone()); ctx.set_dependency(META_KEY, test_meta()); assert!(ctx .get_event_recipient("commitment_consistency_checker") @@ -186,9 +214,10 @@ mod tests { #[actix::test] async fn hydrate_without_meta_leaves_checker_unset() -> Result<()> { let bus = test_bus(); - let extension = CommitmentConsistencyCheckerExtension::create(&bus, |_| Vec::new()); + let store = test_store(); + let extension = CommitmentConsistencyCheckerExtension::create(&bus, &store, |_| Vec::new()); let e3_id = E3id::new("0", 31337); - let mut ctx = test_context(e3_id.clone()); + let mut ctx = test_context(&store, e3_id.clone()); let snapshot = E3ContextSnapshot { e3_id, recipients: vec![], diff --git a/crates/slashing/src/lib.rs b/crates/slashing/src/lib.rs index aee83a6c6a..1b4cf82115 100644 --- a/crates/slashing/src/lib.rs +++ b/crates/slashing/src/lib.rs @@ -11,6 +11,7 @@ mod actors; mod domain; +mod repo; mod workflow; pub mod accusation_manager_ext; @@ -26,3 +27,5 @@ pub use accusation_manager::AccusationManager; pub use accusation_manager_ext::AccusationManagerExtension; pub use commitment_consistency_checker::CommitmentConsistencyChecker; pub use commitment_consistency_checker_ext::CommitmentConsistencyCheckerExtension; +pub use domain::commitment_consistency::CommitmentConsistencySnapshot; +pub use repo::CommitmentConsistencyRepositoryFactory; diff --git a/crates/slashing/src/repo.rs b/crates/slashing/src/repo.rs new file mode 100644 index 0000000000..b5552ba35d --- /dev/null +++ b/crates/slashing/src/repo.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Repository factories for slashing state that must survive a restart. + +use e3_data::{Repositories, Repository}; +use e3_events::{E3id, StoreKeys}; + +use crate::domain::commitment_consistency::CommitmentConsistencySnapshot; + +pub trait CommitmentConsistencyRepositoryFactory { + /// Per-E3 durable verified-proof cache of the commitment-consistency checker. + fn commitment_consistency(&self, e3_id: &E3id) -> Repository; +} + +impl CommitmentConsistencyRepositoryFactory for Repositories { + fn commitment_consistency(&self, e3_id: &E3id) -> Repository { + Repository::new(self.store.scope(StoreKeys::commitment_consistency(e3_id))) + } +} diff --git a/crates/test-helpers/src/ciphernode_system.rs b/crates/test-helpers/src/ciphernode_system.rs index 831e791d3d..4c6789a1cf 100644 --- a/crates/test-helpers/src/ciphernode_system.rs +++ b/crates/test-helpers/src/ciphernode_system.rs @@ -407,6 +407,7 @@ mod tests { network_status: NetworkStatus::default(), eventstore, aggregate_ids: vec![], + gateway_failures: vec![], }) } diff --git a/crates/zk-prover/src/actor_system.rs b/crates/zk-prover/src/actor_system.rs index 571f903bad..b6bfea8f25 100644 --- a/crates/zk-prover/src/actor_system.rs +++ b/crates/zk-prover/src/actor_system.rs @@ -8,6 +8,7 @@ use actix::{Actor, Addr}; use alloy::signers::local::PrivateKeySigner; +use e3_data::DataStore; use e3_events::{BusHandle, Committee, DkgFoldAttestationContext, E3id}; use e3_request::E3Meta; use std::collections::HashMap; @@ -47,6 +48,7 @@ impl ZkActorRecovery { /// Requires a `ZkBackend` for proof generation/verification and a `PrivateKeySigner` for signing /// proofs. `dkg_fold_attestation_contexts_by_chain` is a fallback for synthetic runs that have no /// on-chain context event. Live and replayed context events carry each E3's registry and verifier. +/// `store` keeps each E3's own C0 proof so the DKG node fold can complete after a restart. pub fn setup_zk_actors( bus: &BusHandle, backend: &ZkBackend, @@ -54,6 +56,7 @@ pub fn setup_zk_actors( dkg_fold_attestation_contexts_by_chain: HashMap>, recovery: ZkActorRecovery, proof_aggregation_enabled: bool, + store: Option, ) -> ZkActors { let ZkActorRecovery { finalized_committees, @@ -63,7 +66,8 @@ pub fn setup_zk_actors( let zk_actor = ZkActor::new(backend).start(); let verifier = zk_actor.clone().recipient(); - let proof_request = ProofRequestActor::setup(bus, signer.clone(), proof_aggregation_enabled); + let proof_request = + ProofRequestActor::setup(bus, signer.clone(), proof_aggregation_enabled, store); let proof_verification = ProofVerificationActor::setup(bus, verifier, finalized_committees.clone(), e3_metadata); let share_verification = ShareVerificationActor::setup(bus, finalized_committees); diff --git a/crates/zk-prover/src/error.rs b/crates/zk-prover/src/error.rs index fae970858f..1634287b9e 100644 --- a/crates/zk-prover/src/error.rs +++ b/crates/zk-prover/src/error.rs @@ -30,6 +30,9 @@ pub enum ZkError { #[error("Proof generation failed: {0}")] ProveFailed(String), + #[error("bb timed out: {0}")] + Timeout(String), + #[error("Proof verification failed: {0}")] VerifyFailed(String), diff --git a/crates/zk-prover/src/node_proof_aggregation/actor.rs b/crates/zk-prover/src/node_proof_aggregation/actor.rs index 329a250355..44a7700d72 100644 --- a/crates/zk-prover/src/node_proof_aggregation/actor.rs +++ b/crates/zk-prover/src/node_proof_aggregation/actor.rs @@ -372,4 +372,144 @@ mod tests { Ok(()) } + + /// A restart during the node fold (~200 s at secure params) kills the in-memory + /// correlation id, but the replayed `ComputeRequest` still yields a response. The compute + /// effect gate dedups on `(e3_id, request)`, so the re-driven fold dispatch is suppressed + /// as already-forwarded and this response is the only one that will ever arrive. Dropping + /// it silently left the node fold unfinished forever (Bug 26 — the Bug 23 pattern one + /// level up, at the fold instead of C4). + #[actix::test] + async fn an_orphaned_node_dkg_fold_response_is_adopted_after_a_restart() -> Result<()> { + let (bus, _rng, _seed, _params, _crp, _errors, history) = get_common_setup(None)?; + let mut aggregator = + NodeProofAggregator::new(&bus, test_signer(), HashMap::new(), HashMap::new(), true); + let e3_id = E3id::new("49", 1); + + aggregator.initialize_collection_state( + e3_id.clone(), + NodeDkgFoldMeta { + party_id: 7, + total_expected: 6, + sk_enc_count: 0, + e_sm_enc_count: 0, + sk_share_encryption_requests: Vec::new(), + e_sm_share_encryption_requests: Vec::new(), + committee_n: 0, + committee_h: 0, + n_moduli: 0, + params_preset: e3_fhe_params::BfvPreset::InsecureThreshold512, + committee_size: CiphernodesCommitteeSize::Minimum, + }, + test_ctx(DKGRecursiveAggregationComplete { + e3_id: e3_id.clone(), + party_id: 7, + aggregated_proof: None, + fold_attestation: None, + }), + ); + for seq in 0..6 { + let proof = dummy_proof(seq as u8); + aggregator.handle_inner_proof_ready(TypedEvent::new( + DKGInnerProofReady { + e3_id: e3_id.clone(), + party_id: 7, + proof: proof.clone(), + seq, + }, + test_ctx(DKGInnerProofReady { + e3_id: e3_id.clone(), + party_id: 7, + proof, + seq, + }), + )); + } + + // The buffer completed, so a fold is in flight with some correlation id. + let live_corr = aggregator.states[&e3_id] + .fold_correlation + .expect("a full buffer must dispatch the fold"); + assert!(aggregator.fold_correlation.contains_key(&live_corr)); + + // Simulate the restart: the process-local correlation map is gone, but the state + // (rebuilt by replay) still shows a fold in flight. + aggregator.fold_correlation.clear(); + + // The response arrives under the PRE-CRASH correlation id, which this process never + // registered. It must still be attributed to the one E3 that is mid-fold. + let unknown = CorrelationId::new(); + assert_ne!(unknown, live_corr); + aggregator.handle_node_dkg_response(&unknown, dummy_proof(99)); + + assert!( + !aggregator.states.contains_key(&e3_id), + "adopting the response must consume the collection state and finish the fold" + ); + drop(history); + Ok(()) + } + + /// With two E3s mid-fold an orphaned response cannot be attributed, so it is dropped + /// rather than credited to the wrong E3. + #[actix::test] + async fn an_orphaned_fold_response_is_dropped_when_several_e3s_are_folding() -> Result<()> { + let (bus, _rng, _seed, _params, _crp, _errors, _history) = get_common_setup(None)?; + let mut aggregator = + NodeProofAggregator::new(&bus, test_signer(), HashMap::new(), HashMap::new(), true); + + for id in ["50", "51"] { + let e3_id = E3id::new(id, 1); + aggregator.initialize_collection_state( + e3_id.clone(), + NodeDkgFoldMeta { + party_id: 7, + total_expected: 6, + sk_enc_count: 0, + e_sm_enc_count: 0, + sk_share_encryption_requests: Vec::new(), + e_sm_share_encryption_requests: Vec::new(), + committee_n: 0, + committee_h: 0, + n_moduli: 0, + params_preset: e3_fhe_params::BfvPreset::InsecureThreshold512, + committee_size: CiphernodesCommitteeSize::Minimum, + }, + test_ctx(DKGRecursiveAggregationComplete { + e3_id: e3_id.clone(), + party_id: 7, + aggregated_proof: None, + fold_attestation: None, + }), + ); + for seq in 0..6 { + let proof = dummy_proof(seq as u8); + aggregator.handle_inner_proof_ready(TypedEvent::new( + DKGInnerProofReady { + e3_id: e3_id.clone(), + party_id: 7, + proof: proof.clone(), + seq, + }, + test_ctx(DKGInnerProofReady { + e3_id: e3_id.clone(), + party_id: 7, + proof, + seq, + }), + )); + } + } + aggregator.fold_correlation.clear(); + assert_eq!(aggregator.states.len(), 2); + + aggregator.handle_node_dkg_response(&CorrelationId::new(), dummy_proof(99)); + + assert_eq!( + aggregator.states.len(), + 2, + "an unattributable response must not consume either E3's state" + ); + Ok(()) + } } diff --git a/crates/zk-prover/src/node_proof_aggregation/effects.rs b/crates/zk-prover/src/node_proof_aggregation/effects.rs index f0db925023..414c67bfab 100644 --- a/crates/zk-prover/src/node_proof_aggregation/effects.rs +++ b/crates/zk-prover/src/node_proof_aggregation/effects.rs @@ -191,8 +191,54 @@ impl NodeProofAggregator { correlation_id: &CorrelationId, proof: Proof, ) { - let Some(e3_id) = self.fold_correlation.remove(correlation_id) else { - return; + let e3_id = match self.fold_correlation.remove(correlation_id) { + Some(e3_id) => e3_id, + None => { + // Defensive: no live trigger has been reproduced for this branch. + // + // A crash during the fold leaves the pre-crash correlation id dead with the + // process. It does NOT normally strand the fold: recovery rebuilds the proof + // buffer, `try_dispatch_node_dkg_fold` mints a *fresh* `CorrelationId`, and the + // new response matches it. Two live chaos rounds that killed a committee member + // mid-fold (~40 s in) both recovered this way with zero adoptions, and the + // effect gate reported no duplicate suppression for `NodeDkgFold` — unlike the + // C4 `DkgShareDecryption` path, where replay re-plans an identical + // `(e3_id, request)` and the gate does dedup it. + // + // The branch is kept because dropping a response silently would leave the fold + // window unfinished for good if that ever changed (for example if the fold + // request became replay-identical). Adopt only when exactly one E3 is waiting on + // a fold: the request is per-E3 and `state.fold_correlation` allows one in + // flight, so there is no ambiguity about the owner. + let mut waiting = self + .states + .iter() + .filter(|(_, state)| state.fold_correlation.is_some()) + .map(|(e3_id, _)| e3_id.clone()); + match (waiting.next(), waiting.next()) { + (Some(e3_id), None) => { + info!( + "NodeProofAggregator: adopting orphaned NodeDkgFold response for E3 {} \ + (correlation {:?} predates a restart)", + e3_id, correlation_id + ); + if let Some(state) = self.states.get_mut(&e3_id) { + if let Some(stale) = state.fold_correlation.take() { + self.fold_correlation.remove(&stale); + } + } + e3_id + } + (Some(_), Some(_)) => { + warn!( + "NodeProofAggregator: orphaned NodeDkgFold response with several E3s \ + mid-fold — cannot attribute it, dropping" + ); + return; + } + _ => return, + } + } }; let Some(state) = self.states.remove(&e3_id) else { diff --git a/crates/zk-prover/src/proof_request/actor.rs b/crates/zk-prover/src/proof_request/actor.rs index 05224cdc9e..a515e246f7 100644 --- a/crates/zk-prover/src/proof_request/actor.rs +++ b/crates/zk-prover/src/proof_request/actor.rs @@ -11,6 +11,7 @@ use actix::{Actor, Addr, Context, Handler}; use alloy::primitives::{keccak256, Bytes}; use alloy::signers::local::PrivateKeySigner; use alloy::sol_types::SolValue; +use e3_data::{DataStore, RepositoriesFactory, Repository}; use e3_events::{ AggregationProofPending, AggregationProofSigned, BusHandle, ComputeRequest, ComputeRequestError, ComputeRequestErrorKind, ComputeResponse, ComputeResponseKind, @@ -20,9 +21,11 @@ use e3_events::{ EventType, FailureReason, InterfoldEvent, InterfoldEventData, PkAggregationProofPending, PkAggregationProofSigned, PkBfvProofRequest, PkGenerationProofSigned, Proof, ProofPayload, ProofType, ProofVerificationPassed, Sequenced, ShareDecryptionProofPending, SignedProofPayload, - ThresholdShareCreated, ThresholdSharePending, TypedEvent, ZkRequest, ZkResponse, + StoreKeys, ThresholdShareCreated, ThresholdSharePending, TypedEvent, ZkRequest, ZkResponse, }; use e3_utils::NotifySync; +use e3_zk_helpers::computation::DkgInputType; +use serde::{Deserialize, Serialize}; use tracing::{error, info, trace, warn}; use crate::workflow::proof_request::{ @@ -31,6 +34,30 @@ use crate::workflow::proof_request::{ PendingProofRequest, PendingShareDecryptionProof, PendingThresholdProofs, ThresholdProofKind, }; +/// The node's own signed C0 proof, persisted the moment it is produced. +/// +/// C0 is generated exactly once per E3, before the threshold share exists. Every other DKG +/// inner proof (C1–C4) is regenerated after a restart because its `*Pending` trigger is +/// replayed and re-proved; C0's trigger (`EncryptionKeyPending`) sits before the aggregate +/// snapshot cursor and is never replayed. Without this record the restarted node's +/// `NodeProofAggregator` reaches 13/14 and its DKG fold never completes (Round 14, cn3). +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct OwnC0Record { + pub party_id: u64, + pub proof: Proof, +} + +/// Per-E3 durable record of the own C0 proof. +pub trait OwnC0RepositoryFactory { + fn own_c0(&self, e3_id: &E3id) -> Repository; +} + +impl OwnC0RepositoryFactory for e3_data::Repositories { + fn own_c0(&self, e3_id: &E3id) -> Repository { + Repository::new(self.store.scope(StoreKeys::own_c0_proof(e3_id))) + } +} + /// Core actor that handles encryption key proof requests. /// /// Proofs are always wrapped in a [`SignedProofPayload`] before being published, @@ -47,6 +74,15 @@ pub struct ProofRequestActor { decryption_correlation: HashMap, /// Per-E3 metadata for DKGInnerProofReady emission. node_agg_meta: HashMap, + /// C4 dispatch that arrived before `ThresholdSharePending` set the seq layout. + /// + /// `c4_base_seq` is derived from `node_agg_meta.total_expected`. On a restart inside the + /// DKG window the keyshare recovery re-publishes `ThresholdSharePending` and + /// `DecryptionShareProofsPending` back to back, and the C4 event can be handled first. + /// Dispatching then would tag C4a/C4b as seq 0/1 — colliding with C0/C1 in the node + /// fold buffer, leaving the real C4 slots empty forever (Round 11, cn3 stuck at 12/14). + /// Hold it here and replay once the layout is known. + held_decryption_pending: HashMap>, /// C4 pending proofs per E3 pending_decryption: HashMap, /// C6 proof staging: correlation -> e3_id @@ -61,6 +97,8 @@ pub struct ProofRequestActor { aggregation_correlation: HashMap, /// C7 pending proofs per E3 pending_aggregation: HashMap, + /// Backing store for [`OwnC0Record`]. `None` only in tests that never restart. + store: Option, } impl ProofRequestActor { @@ -75,21 +113,43 @@ impl ProofRequestActor { decryption_correlation: HashMap::new(), pending_decryption: HashMap::new(), node_agg_meta: HashMap::new(), + held_decryption_pending: HashMap::new(), share_decryption_correlation: HashMap::new(), pending_share_decryption: HashMap::new(), pk_aggregation_correlation: HashMap::new(), pending_pk_aggregation: HashMap::new(), aggregation_correlation: HashMap::new(), pending_aggregation: HashMap::new(), + store: None, } } + /// Attach the store that keeps each E3's own C0 proof across restarts. + pub fn with_store(mut self, store: DataStore) -> Self { + self.store = Some(store); + self + } + + pub(in crate::actors::proof_request) fn own_c0_repo( + &self, + e3_id: &E3id, + ) -> Option> { + self.store + .as_ref() + .map(|store| store.repositories().own_c0(e3_id)) + } + pub fn setup( bus: &BusHandle, signer: PrivateKeySigner, proof_aggregation_enabled: bool, + store: Option, ) -> Addr { - let addr = Self::new(bus, signer, proof_aggregation_enabled).start(); + let mut actor = Self::new(bus, signer, proof_aggregation_enabled); + if let Some(store) = store { + actor = actor.with_store(store); + } + let addr = actor.start(); bus.subscribe(EventType::EncryptionKeyPending, addr.clone().into()); bus.subscribe(EventType::ComputeResponse, addr.clone().into()); bus.subscribe(EventType::ComputeRequestError, addr.clone().into()); diff --git a/crates/zk-prover/src/proof_request/actor_tests.rs b/crates/zk-prover/src/proof_request/actor_tests.rs index 12015fd3c4..4ca922e3a7 100644 --- a/crates/zk-prover/src/proof_request/actor_tests.rs +++ b/crates/zk-prover/src/proof_request/actor_tests.rs @@ -8,8 +8,8 @@ use super::*; use alloy::signers::local::PrivateKeySigner; use anyhow::Result; use e3_events::{ - ComputeRequestErrorKind, EncryptionKey, Event, HistoryCollector, TakeEvents, Unsequenced, - ZkError, + ComputeRequestErrorKind, DkgShareDecryptionProofRequest, EncryptionKey, Event, + HistoryCollector, TakeEvents, Unsequenced, ZkError, }; use e3_test_helpers::get_common_setup; use e3_utils::utility_types::ArcBytes; @@ -99,3 +99,256 @@ async fn decryption_failure_helper_emits_e3_failed() -> Result<()> { Ok(()) } + +/// `c4_base_seq` is derived from `ThresholdSharePending`'s layout. On a restart inside the +/// DKG window the keyshare recovery re-publishes both pending events and the C4 one can be +/// handled first; dispatching then would give C4a/C4b seq 0/1 and collide with C0/C1 in the +/// node fold buffer, so the real C4 slots stay empty and the fold never completes (Round 11, +/// cn3 stuck at 12/14). The C4 dispatch must be held and replayed once the layout is known. +#[actix::test] +async fn c4_dispatch_before_the_seq_layout_is_held_and_replayed_with_the_right_seqs() -> Result<()> +{ + use e3_crypto::SensitiveBytes; + use e3_fhe_params::BfvPreset; + use e3_zk_helpers::{computation::DkgInputType, CiphernodesCommitteeSize}; + + let (bus, _rng, _seed, _params, _crp, _errors, _history) = get_common_setup(None)?; + let mut actor = ProofRequestActor::new(&bus, PrivateKeySigner::random(), true); + let e3_id = E3id::new("46", 1); + let req = || DkgShareDecryptionProofRequest { + sk_bfv: SensitiveBytes::from_encrypted(&[]), + honest_ciphertexts_raw: vec![], + num_honest_parties: 0, + num_moduli: 0, + own_plaintext_idx: 0, + own_share_raw: SensitiveBytes::from_encrypted(&[]), + dkg_input_type: DkgInputType::SecretKey, + params_preset: BfvPreset::default(), + committee_size: CiphernodesCommitteeSize::Minimum, + }; + let pending = DecryptionShareProofsPending { + e3_id: e3_id.clone(), + party_id: 1, + node: "0x00".into(), + sk_request: req(), + esm_requests: vec![req()], + }; + + // C4 arrives first: nothing may be dispatched yet. + actor.handle_decryption_share_proofs_pending(TypedEvent::new( + pending.clone(), + test_ctx(pending.clone()), + )); + assert!( + actor.decryption_correlation.is_empty(), + "C4 must not be dispatched before the seq layout is known" + ); + assert!(actor.held_decryption_pending.contains_key(&e3_id)); + + // The layout lands: 4 + 4 sk-enc + 4 esm-enc + 2 = 14, so C4a=12, C4b=13. + actor.node_agg_meta.insert( + e3_id.clone(), + NodeAggregationMeta { + party_id: 1, + total_expected: NodeAggregationMeta::total_expected_for(4, 4), + pending_c0: None, + c0_emitted: false, + }, + ); + let held = actor + .held_decryption_pending + .remove(&e3_id) + .expect("held C4 dispatch"); + actor.handle_decryption_share_proofs_pending(held); + + let mut seqs: Vec = actor + .decryption_correlation + .values() + .filter(|(eid, _, _)| *eid == e3_id) + .map(|(_, _, seq)| *seq) + .collect(); + seqs.sort_unstable(); + assert_eq!( + seqs, + vec![12, 13], + "C4a/C4b must land in the C4 slots, not on top of C0/C1" + ); + assert!(actor.held_decryption_pending.is_empty()); + Ok(()) +} + +/// After a restart the effect gate replays the pre-crash C4 `ComputeRequest` (old correlation +/// id) and drops the re-driven one as a semantic duplicate. The response therefore arrives +/// under an id this process never registered. Before the fix it fell through to the +/// threshold handler and was silently lost — the fold sat at 12/14 forever (Round 12). It must +/// be matched by kind to the pending dispatch. +#[actix::test] +async fn an_orphaned_c4_response_is_adopted_by_kind() -> Result<()> { + use e3_zk_helpers::computation::DkgInputType; + + let (bus, _rng, _seed, _params, _crp, _errors, _history) = get_common_setup(None)?; + let mut actor = ProofRequestActor::new(&bus, PrivateKeySigner::random(), true); + let e3_id = E3id::new("47", 1); + let sk_corr = CorrelationId::new(); + let esm0_corr = CorrelationId::new(); + let esm1_corr = CorrelationId::new(); + actor + .decryption_correlation + .insert(sk_corr, (e3_id.clone(), DecryptionProofKind::SecretKey, 12)); + actor.decryption_correlation.insert( + esm1_corr, + ( + e3_id.clone(), + DecryptionProofKind::SmudgingNoise { esi_idx: 1 }, + 14, + ), + ); + actor.decryption_correlation.insert( + esm0_corr, + ( + e3_id.clone(), + DecryptionProofKind::SmudgingNoise { esi_idx: 0 }, + 13, + ), + ); + // A different E3 must never be matched. + actor.decryption_correlation.insert( + CorrelationId::new(), + (E3id::new("99", 1), DecryptionProofKind::SecretKey, 12), + ); + + assert_eq!( + actor.adopt_orphaned_c4_response(&e3_id, DkgInputType::SecretKey), + Some(sk_corr) + ); + // Lowest outstanding esi_idx first — canonical dispatch order. + assert_eq!( + actor.adopt_orphaned_c4_response(&e3_id, DkgInputType::SmudgingNoise), + Some(esm0_corr) + ); + actor.decryption_correlation.remove(&esm0_corr); + assert_eq!( + actor.adopt_orphaned_c4_response(&e3_id, DkgInputType::SmudgingNoise), + Some(esm1_corr) + ); + assert_eq!( + actor.adopt_orphaned_c4_response(&E3id::new("48", 1), DkgInputType::SecretKey), + None + ); + Ok(()) +} +/// Round 14, cn3: after a restart inside the DKG window every inner proof except C0 is +/// regenerated (their `*Pending` triggers are replayed). C0 is generated exactly once, before +/// the aggregate snapshot cursor, so the fresh `ProofRequestActor` never sees +/// `EncryptionKeyPending` again and the node fold sticks at 13/14 forever. The own C0 is now +/// persisted the moment it is signed, and `ThresholdSharePending` re-seeds `seq: 0` from that +/// record when nothing has been emitted in this process. +#[actix::test] +async fn own_c0_is_reseeded_from_the_durable_record_after_restart() -> Result<()> { + use e3_data::{DataStore, InMemStore, RepositoriesFactory}; + use e3_events::CircuitName; + + let (bus, _rng, _seed, _params, _crp, _errors, history) = get_common_setup(None)?; + let store = DataStore::from_in_mem(&InMemStore::new(false).start()); + let e3_id = E3id::new("47", 1); + + // "Pre-crash": the record the first process wrote when it signed C0. + let c0 = Proof::new( + CircuitName::PkBfv, + ArcBytes::from_bytes(&[0xC0u8; 8]), + ArcBytes::from_bytes(&[0x51u8; 4]), + ); + store + .repositories() + .own_c0(&e3_id) + .write_sync(&OwnC0Record { + party_id: 1, + proof: c0.clone(), + }) + .await?; + + // "Post-restart": a fresh actor, store attached, NO C0 in memory. ThresholdSharePending + // has just set the layout (this is what its handler does before calling the re-seed). + let mut actor = + ProofRequestActor::new(&bus, PrivateKeySigner::random(), true).with_store(store); + actor.node_agg_meta.insert( + e3_id.clone(), + NodeAggregationMeta { + party_id: 1, + total_expected: 14, + pending_c0: None, + c0_emitted: false, + }, + ); + let ec = test_ctx(E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGTimeout, + }); + actor.reseed_own_c0_from_store(e3_id.clone(), 1, ec); + assert!( + actor.node_agg_meta[&e3_id].c0_emitted, + "the re-seed must be recorded so a later C0 response cannot double-emit seq 0" + ); + + // The read is async — drain the bus until seq 0 lands. + let mut c0_ready = None; + for _ in 0..80 { + let mut result = history.send(TakeEvents::::new(1)).await?; + match result.events.pop() { + Some(evt) => { + if let InterfoldEventData::DKGInnerProofReady(d) = evt.into_data() { + if d.seq == 0 { + c0_ready = Some(d); + break; + } + } + } + None => tokio::time::sleep(std::time::Duration::from_millis(25)).await, + } + } + let c0_ready = + c0_ready.expect("DKGInnerProofReady seq=0 must be re-seeded from the own-C0 record"); + assert_eq!(c0_ready.e3_id, e3_id); + assert_eq!(c0_ready.party_id, 1); + assert_eq!(c0_ready.proof, c0); + Ok(()) +} + +/// Without a record (a node that was never in this E3, or a pre-fix store) the re-seed must +/// be a no-op that still marks `c0_emitted`, so the live C0 path is unaffected. +#[actix::test] +async fn reseed_without_a_record_emits_nothing() -> Result<()> { + use e3_data::{DataStore, InMemStore}; + + let (bus, _rng, _seed, _params, _crp, _errors, history) = get_common_setup(None)?; + let store = DataStore::from_in_mem(&InMemStore::new(false).start()); + let e3_id = E3id::new("48", 1); + let mut actor = + ProofRequestActor::new(&bus, PrivateKeySigner::random(), true).with_store(store); + actor.node_agg_meta.insert( + e3_id.clone(), + NodeAggregationMeta { + party_id: 1, + total_expected: 14, + pending_c0: None, + c0_emitted: false, + }, + ); + let ec = test_ctx(E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGTimeout, + }); + actor.reseed_own_c0_from_store(e3_id.clone(), 1, ec); + tokio::time::sleep(std::time::Duration::from_millis(150)).await; + let result = history.send(TakeEvents::::new(1)).await?; + assert!( + !result + .events + .iter() + .any(|e| matches!(e.get_data(), InterfoldEventData::DKGInnerProofReady(_))), + "no record => nothing re-seeded" + ); + Ok(()) +} diff --git a/crates/zk-prover/src/proof_request/effects/decryption_key_proofs.rs b/crates/zk-prover/src/proof_request/effects/decryption_key_proofs.rs index 6a203e62be..f706500847 100644 --- a/crates/zk-prover/src/proof_request/effects/decryption_key_proofs.rs +++ b/crates/zk-prover/src/proof_request/effects/decryption_key_proofs.rs @@ -10,8 +10,25 @@ impl ProofRequestActor { &mut self, msg: TypedEvent, ) { - let (msg, ec) = msg.into_components(); let e3_id = msg.e3_id.clone(); + + // The seq layout (and so `c4_base_seq`) comes from `ThresholdSharePending`. Without it + // C4a/C4b would be dispatched as seq 0/1 and collide with C0/C1 in the node fold + // buffer. Hold until the layout is known; `handle_threshold_share_pending` replays. + let layout_known = self + .node_agg_meta + .get(&e3_id) + .is_some_and(|meta| meta.total_expected > 0); + if !layout_known { + info!( + "DecryptionShareProofsPending for E3 {} arrived before ThresholdSharePending — holding until the seq layout is known", + e3_id + ); + self.held_decryption_pending.insert(e3_id, msg); + return; + } + + let (msg, ec) = msg.into_components(); let esm_count = msg.esm_requests.len(); if self.pending_decryption.contains_key(&e3_id) { @@ -41,7 +58,7 @@ impl ProofRequestActor { .node_agg_meta .get(&e3_id) .map(NodeAggregationMeta::c4_base_seq) - .unwrap_or(0); + .expect("layout checked above"); for item in plan_decryption_dispatch(msg.sk_request, msg.esm_requests, c4_base_seq) { let corr = CorrelationId::new(); self.decryption_correlation @@ -59,6 +76,50 @@ impl ProofRequestActor { } } + /// Match a C4 response whose correlation id this process never registered to the + /// pending dispatch of the same kind, returning that dispatch's correlation id. + /// + /// Only considers dispatches for `e3_id` whose kind matches `input_type`. For + /// `SmudgingNoise` the lowest outstanding `esi_idx` is taken: replayed requests were + /// dispatched in canonical order, so their responses arrive in that order too. + pub(in crate::actors::proof_request) fn adopt_orphaned_c4_response( + &self, + e3_id: &E3id, + input_type: DkgInputType, + ) -> Option { + let mut candidates: Vec<(usize, CorrelationId)> = self + .decryption_correlation + .iter() + .filter(|(_, (eid, kind, _))| { + eid == e3_id + && matches!( + (input_type, kind), + (DkgInputType::SecretKey, DecryptionProofKind::SecretKey) + | ( + DkgInputType::SmudgingNoise, + DecryptionProofKind::SmudgingNoise { .. } + ) + ) + }) + .map(|(corr, (_, kind, _))| { + let order = match kind { + DecryptionProofKind::SecretKey => 0, + DecryptionProofKind::SmudgingNoise { esi_idx } => *esi_idx, + }; + (order, *corr) + }) + .collect(); + candidates.sort_unstable_by_key(|(order, _)| *order); + let adopted = candidates.first().map(|(_, corr)| *corr); + if adopted.is_some() { + info!( + "Adopting orphaned C4 {:?} response for E3 {} (replayed pre-restart request)", + input_type, e3_id + ); + } + adopted + } + /// Handle a C4 proof response — store and check completeness. pub(in crate::actors::proof_request) fn handle_decryption_proof_response( &mut self, diff --git a/crates/zk-prover/src/proof_request/effects/dkg_proofs.rs b/crates/zk-prover/src/proof_request/effects/dkg_proofs.rs index 9352ee10d1..a5484f98fd 100644 --- a/crates/zk-prover/src/proof_request/effects/dkg_proofs.rs +++ b/crates/zk-prover/src/proof_request/effects/dkg_proofs.rs @@ -4,6 +4,59 @@ use super::*; +impl ProofRequestActor { + /// Re-seed `DKGInnerProofReady { seq: 0 }` from the durable own-C0 record. + /// + /// The read is async and the caller is a sync handler, so the publish happens from a + /// spawned task straight onto the bus; the `NodeProofAggregator` subscribes there. + pub(in crate::actors::proof_request) fn reseed_own_c0_from_store( + &mut self, + e3_id: E3id, + party_id: u64, + ec: EventContext, + ) { + let Some(repo) = self.own_c0_repo(&e3_id) else { + return; + }; + if let Some(meta) = self.node_agg_meta.get_mut(&e3_id) { + meta.c0_emitted = true; + } + let bus = self.bus.clone(); + tokio::spawn(async move { + match repo.read().await { + Ok(Some(record)) => { + info!( + "Re-seeding own C0 proof for E3 {} from the durable record (party {})", + e3_id, record.party_id + ); + if record.party_id != party_id { + warn!( + "Own C0 record party {} differs from threshold share party {} for E3 {}", + record.party_id, party_id, e3_id + ); + } + if let Err(err) = bus.publish( + DKGInnerProofReady { + e3_id: e3_id.clone(), + party_id, + proof: record.proof, + seq: 0, + }, + ec, + ) { + error!("Failed to publish re-seeded DKGInnerProofReady for C0: {err}"); + } + } + Ok(None) => warn!( + "No durable own C0 record for E3 {}; the node DKG fold will not complete", + e3_id + ), + Err(err) => error!("Failed to read own C0 record for E3 {}: {err}", e3_id), + } + }); + } +} + impl ProofRequestActor { pub(in crate::actors::proof_request) fn handle_encryption_key_pending( &mut self, @@ -47,18 +100,29 @@ impl ProofRequestActor { let e_sm_enc_count = msg.e_sm_share_encryption_requests.len(); let total_expected = NodeAggregationMeta::total_expected_for(sk_enc_count, e_sm_enc_count); - let pending_c0 = self + let (pending_c0, c0_already_emitted) = self .node_agg_meta .get(&e3_id) - .and_then(|m| m.pending_c0.clone()); + .map(|m| (m.pending_c0.clone(), m.c0_emitted)) + .unwrap_or((None, false)); + let c0_emitted = c0_already_emitted || pending_c0.is_some(); self.node_agg_meta.insert( e3_id.clone(), NodeAggregationMeta { party_id: msg.full_share.party_id, total_expected, pending_c0: None, + c0_emitted, }, ); + // The seq layout is now known; release any C4 dispatch that arrived first. + if let Some(held) = self.held_decryption_pending.remove(&e3_id) { + info!( + "Releasing held DecryptionShareProofsPending for E3 {} now that the seq layout is known", + e3_id + ); + self.handle_decryption_share_proofs_pending(held); + } // If C0 proof arrived before meta, emit DKGInnerProofReady now if self.proof_aggregation_enabled { if let Some(c0_proof) = pending_c0 { @@ -73,6 +137,10 @@ impl ProofRequestActor { ) { error!("Failed to publish DKGInnerProofReady for C0: {err}"); } + } else if !c0_already_emitted { + // No C0 in memory and none emitted earlier in this process: this is a restart + // with the C0 generated before the crash. Re-seed seq 0 from the durable record. + self.reseed_own_c0_from_store(e3_id.clone(), msg.full_share.party_id, ec.clone()); } } @@ -143,6 +211,15 @@ impl ProofRequestActor { resp.proof.clone(), &ec, ); + } else if let Some(adopted) = + self.adopt_orphaned_c4_response(&msg.e3_id, resp.dkg_input_type) + { + // After a restart the effect gate replays the pre-crash `ComputeRequest` + // (old correlation id) and drops our re-driven one as a semantic + // duplicate. The response arrives under an id this process never + // registered. It is still the C4 proof we are waiting on — match it by + // kind against the pending dispatch instead of losing it. + self.handle_decryption_proof_response(&adopted, resp.proof.clone(), &ec); } else { self.handle_threshold_proof_response( &msg.correlation_id, diff --git a/crates/zk-prover/src/proof_request/effects/encryption_key_result.rs b/crates/zk-prover/src/proof_request/effects/encryption_key_result.rs index f5374078eb..08d2a65b27 100644 --- a/crates/zk-prover/src/proof_request/effects/encryption_key_result.rs +++ b/crates/zk-prover/src/proof_request/effects/encryption_key_result.rs @@ -48,6 +48,16 @@ impl ProofRequestActor { } let local_party_id = key.party_id; + + // Persist the own C0 first. It is produced once and never regenerated, and the + // event that triggered it is not replayed after a restart (see `OwnC0Record`). + if let Some(repo) = self.own_c0_repo(&e3_id) { + repo.write(&OwnC0Record { + party_id: local_party_id, + proof: proof.clone(), + }); + } + if let Err(err) = self.bus.publish( EncryptionKeyCreated { e3_id: e3_id.clone(), @@ -88,12 +98,14 @@ impl ProofRequestActor { } // Emit DKGInnerProofReady for C0, or buffer if meta not yet available - if let Some(meta) = self.node_agg_meta.get(&e3_id) { + if let Some(meta) = self.node_agg_meta.get_mut(&e3_id) { if self.proof_aggregation_enabled { + meta.c0_emitted = true; + let party_id = meta.party_id; if let Err(err) = self.bus.publish( DKGInnerProofReady { e3_id: e3_id.clone(), - party_id: meta.party_id, + party_id, proof: proof.clone(), seq: 0, }, @@ -110,6 +122,7 @@ impl ProofRequestActor { party_id: 0, total_expected: 0, pending_c0: Some(proof), + c0_emitted: false, }, ); } diff --git a/crates/zk-prover/src/proof_request/state.rs b/crates/zk-prover/src/proof_request/state.rs index 924d6705d4..652d5e80c2 100644 --- a/crates/zk-prover/src/proof_request/state.rs +++ b/crates/zk-prover/src/proof_request/state.rs @@ -35,6 +35,10 @@ pub(crate) struct NodeAggregationMeta { pub(crate) total_expected: usize, /// Buffered C0 proof, if it arrived before meta was stored. pub(crate) pending_c0: Option, + /// `DKGInnerProofReady { seq: 0 }` has already gone out in this process. Distinguishes + /// the live path (C0 finished after `ThresholdSharePending`) from a restart, where the + /// pre-crash C0 must be re-seeded from the durable record. + pub(crate) c0_emitted: bool, } impl NodeAggregationMeta { diff --git a/crates/zk-prover/src/proof_request/workflow_tests.rs b/crates/zk-prover/src/proof_request/workflow_tests.rs index 0185994cce..6df53286cd 100644 --- a/crates/zk-prover/src/proof_request/workflow_tests.rs +++ b/crates/zk-prover/src/proof_request/workflow_tests.rs @@ -191,6 +191,7 @@ fn node_agg_meta_seq_helpers() { party_id: 0, total_expected: NodeAggregationMeta::total_expected_for(2, 1), pending_c0: None, + c0_emitted: false, }; // c4_base_seq sits just after C0..C3 = total_expected - 2. assert_eq!(meta.c4_base_seq(), 4 + 2 + 1); diff --git a/crates/zk-prover/src/proof_verification/effects.rs b/crates/zk-prover/src/proof_verification/effects.rs index e953b4c40d..3bd8148a08 100644 --- a/crates/zk-prover/src/proof_verification/effects.rs +++ b/crates/zk-prover/src/proof_verification/effects.rs @@ -24,7 +24,9 @@ impl ProofVerificationActor { let Some((preset, committee_size)) = self.presets.get(&msg.e3_id).copied() else { error!( "No BfvPreset known for e3_id={} — cannot determine circuit artifacts directory. \ - This can happen if CiphernodeSelected was missed (e.g. after restart). Rejecting key from party {}.", + Rejecting key from party {}. (Presets survive a restart: they are seeded from \ + the persisted E3 metadata before replay, so this means the E3 is genuinely \ + unknown to this node, not that a CiphernodeSelected event was missed.)", msg.e3_id, msg.key.party_id ); return; diff --git a/crates/zk-prover/src/prover.rs b/crates/zk-prover/src/prover.rs index be6ee5d31c..714b691fb7 100644 --- a/crates/zk-prover/src/prover.rs +++ b/crates/zk-prover/src/prover.rs @@ -10,11 +10,104 @@ use e3_events::{CircuitName, CircuitVariant, Proof}; use e3_fhe_params::BfvPreset; use e3_utils::utility_types::ArcBytes; use std::fs; +use std::io::Read; use std::path::PathBuf; -use std::process::Command as StdCommand; +use std::process::{Command as StdCommand, Output, Stdio}; use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; use tracing::{debug, info, warn}; +/// Environment override for the `bb` wall-clock cap, in seconds. +pub const BB_TIMEOUT_ENV: &str = "INTERFOLD_BB_TIMEOUT_SECS"; + +/// Default wall-clock cap for one `bb` invocation. +/// +/// Matches the DKG window (`E3_DKG_WINDOW_SECS`, 7200 s): a proof that has not finished by +/// then cannot be used by the E3 it was for. Without a cap a hung `bb` (seen with a bad +/// witness on some platforms, and under memory pressure) occupies a job slot for the life +/// of the process and, with `max_concurrent_jobs` slots, a handful of hangs stops the node +/// from proving anything. +pub const DEFAULT_BB_TIMEOUT: Duration = Duration::from_secs(7200); + +/// How often the waiter polls the child between checks of the deadline. +const BB_POLL_INTERVAL: Duration = Duration::from_millis(250); + +fn bb_timeout() -> Duration { + std::env::var(BB_TIMEOUT_ENV) + .ok() + .and_then(|raw| raw.trim().parse::().ok()) + .filter(|secs| *secs > 0) + .map(Duration::from_secs) + .unwrap_or(DEFAULT_BB_TIMEOUT) +} + +/// Run `bb` with the given arguments, killing it if it exceeds the wall-clock cap. +/// +/// Equivalent to `Command::output()` on the happy path. On timeout the child is killed and +/// reaped so it cannot linger as a zombie or keep its job slot, and a `ZkError::Timeout` +/// names the operation that hung. +fn run_bb_with_timeout( + bb_binary: &PathBuf, + args: &[&str], + operation: &str, + timeout: Duration, +) -> Result { + let mut child = StdCommand::new(bb_binary) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + + // Drain the pipes on threads so a chatty `bb` cannot block on a full pipe while we + // wait — that would look exactly like the hang we are guarding against. + let mut stdout_pipe = child.stdout.take(); + let mut stderr_pipe = child.stderr.take(); + let stdout_reader = std::thread::spawn(move || { + let mut buf = Vec::new(); + if let Some(pipe) = stdout_pipe.as_mut() { + let _ = pipe.read_to_end(&mut buf); + } + buf + }); + let stderr_reader = std::thread::spawn(move || { + let mut buf = Vec::new(); + if let Some(pipe) = stderr_pipe.as_mut() { + let _ = pipe.read_to_end(&mut buf); + } + buf + }); + + let started = Instant::now(); + let status = loop { + match child.try_wait()? { + Some(status) => break status, + None if started.elapsed() >= timeout => { + warn!( + operation, + timeout_secs = timeout.as_secs(), + "bb exceeded its wall-clock cap; killing it" + ); + let _ = child.kill(); + let _ = child.wait(); + return Err(ZkError::Timeout(format!( + "bb {operation} exceeded {}s (set {BB_TIMEOUT_ENV} to change the cap)", + timeout.as_secs() + ))); + } + None => std::thread::sleep(BB_POLL_INTERVAL), + } + }; + + let stdout = stdout_reader.join().unwrap_or_default(); + let stderr = stderr_reader.join().unwrap_or_default(); + Ok(Output { + status, + stdout, + stderr, + }) +} + /// Unique bb job directories — shared [`ZkBackend::work_dir`] must not reuse the same paths /// when prove/verify runs concurrently (integration harness + `multithread_concurrent_jobs` > 1). static BB_WORK_JOB_COUNTER: AtomicU64 = AtomicU64::new(0); @@ -212,7 +305,7 @@ impl ZkProver { verifier_target, ]; - let output = StdCommand::new(&self.bb_binary).args(&args).output()?; + let output = run_bb_with_timeout(&self.bb_binary, &args, "prove", bb_timeout())?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); @@ -382,7 +475,7 @@ impl ZkProver { verifier_target, ]; - let output = StdCommand::new(&self.bb_binary).args(&args).output()?; + let output = run_bb_with_timeout(&self.bb_binary, &args, "verify", bb_timeout())?; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); @@ -430,4 +523,39 @@ mod tests { let result = prover.generate_proof(CircuitName::PkBfv, b"witness", "e3-1", "insecure-512"); assert!(matches!(result, Err(ZkError::BbNotInstalled))); } + + /// A hung `bb` must be killed at the cap, not left holding a job slot forever. `sleep` + /// stands in for a hung `bb`; the cap is well under its duration. + #[test] + fn a_hung_bb_is_killed_at_the_wall_clock_cap() { + let sleep = PathBuf::from("/bin/sleep"); + if !sleep.exists() { + return; + } + let started = Instant::now(); + let result = run_bb_with_timeout(&sleep, &["30"], "prove", Duration::from_millis(600)); + let elapsed = started.elapsed(); + + let err = result.expect_err("a bb that outlives the cap must be reported as a timeout"); + assert!(matches!(err, ZkError::Timeout(_)), "got {err}"); + assert!(err.to_string().contains("prove")); + assert!(err.to_string().contains(BB_TIMEOUT_ENV)); + assert!( + elapsed < Duration::from_secs(5), + "the child must be killed at the cap, not waited for; took {elapsed:?}" + ); + } + + /// The happy path is unchanged: output and exit status come back as with `output()`. + #[test] + fn a_finishing_bb_returns_its_output() { + let echo = PathBuf::from("/bin/echo"); + if !echo.exists() { + return; + } + let output = run_bb_with_timeout(&echo, &["proof-ok"], "verify", DEFAULT_BB_TIMEOUT) + .expect("echo must succeed"); + assert!(output.status.success()); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "proof-ok"); + } } diff --git a/packages/interfold-dashboard/.env.example b/packages/interfold-dashboard/.env.example index 274c45068e..d0eeaacca2 100644 --- a/packages/interfold-dashboard/.env.example +++ b/packages/interfold-dashboard/.env.example @@ -29,7 +29,9 @@ # Testnet faucet (FOLD + fee token). Sepolia defaults to its faucet; mainnet has # none, so the "Get test tokens" action is hidden there. Set to the zero address -# or an empty value to hide it on a testnet too. +# or an empty value to hide it on a testnet too. This variable is ignored unless +# VITE_NETWORK selects a test network, so it cannot surface the faucet card (and +# its "testnet deployment" copy) on mainnet. # VITE_FAUCET_ADDRESS=0x6e281411C055BEEbD74bDFcB9aB095aa98907F85 # Fee token metadata used to format escrowed fees/rewards. diff --git a/packages/interfold-dashboard/README.md b/packages/interfold-dashboard/README.md index 74825529c6..29c77aeeb8 100644 --- a/packages/interfold-dashboard/README.md +++ b/packages/interfold-dashboard/README.md @@ -81,8 +81,8 @@ Sepolia deployment defined in `src/lib/chain.ts`: - `VITE_INTERFOLD_ADDRESS`, `VITE_CIPHERNODE_REGISTRY_ADDRESS`, `VITE_CRISP_PROGRAM_ADDRESS` — contracts. - `VITE_BONDING_REGISTRY_ADDRESS` — bonding registry behind the operator guide. -- `VITE_FAUCET_ADDRESS` — testnet faucet. Set to the zero address on a non-testnet deployment to - hide the "Get test tokens" action. +- `VITE_FAUCET_ADDRESS` — testnet faucet. Ignored unless `VITE_NETWORK` selects a test network; + set it to the zero address to hide the "Get test tokens" action on a testnet too. - `VITE_DEPLOY_BLOCK` — first block to scan from (the Interfold deploy block). The fetchers chunk `getLogs` calls to 9_500 blocks per request so they work against the stricter diff --git a/packages/interfold-dashboard/src/Operator.tsx b/packages/interfold-dashboard/src/Operator.tsx index be8f7cd52f..e1b41b685d 100644 --- a/packages/interfold-dashboard/src/Operator.tsx +++ b/packages/interfold-dashboard/src/Operator.tsx @@ -16,12 +16,15 @@ import { useEffect, useMemo, useState, type ReactNode } from 'react' import { erc20Abi, formatUnits, isAddress, parseUnits, type Address, type Hash } from 'viem' import Loader from './Loader' -import { CONTRACTS, NETWORK_NAME, bondingRegistryAbi, faucetAbi } from './lib/chain' +import { CONTRACTS, IS_TESTNET, NETWORK_NAME, bondingRegistryAbi, faucetAbi } from './lib/chain' import { LINKS, explorerAddress, explorerTx } from './lib/links' import { ZERO_ADDRESS, simulateAndWrite, useBonding, type BondingConfig, type OperatorStatus } from './lib/bonding' import { confirmTx, useWallet, walletErrorMessage } from './lib/wallet' -const FAUCET_ENABLED = CONTRACTS.Faucet !== ZERO_ADDRESS && CONTRACTS.Faucet.trim() !== '' +// The faucet is testnet-only: a configured address is not enough, the selected +// network must be a test network too. Otherwise a VITE_FAUCET_ADDRESS left over +// from a testnet deployment shows "Testnet tokens" on mainnet. +const FAUCET_ENABLED = IS_TESTNET && CONTRACTS.Faucet !== ZERO_ADDRESS && CONTRACTS.Faucet.trim() !== '' const shortAddr = (a: string): string => (a.length > 14 ? `${a.slice(0, 8)}…${a.slice(-6)}` : a) diff --git a/packages/interfold-dashboard/src/lib/chain.ts b/packages/interfold-dashboard/src/lib/chain.ts index 60e3d405a8..01dd24fdc4 100644 --- a/packages/interfold-dashboard/src/lib/chain.ts +++ b/packages/interfold-dashboard/src/lib/chain.ts @@ -41,6 +41,10 @@ type NetworkProfile = { chain: Chain // Human-readable name used in UI copy ("Reading from Sepolia…"). name: string + // True for test networks. Gates test-only UI (the faucet card and its + // "testnet deployment" copy) independently of the faucet address, so a stale + // VITE_FAUCET_ADDRESS cannot surface testnet copy on a production network. + testnet: boolean rpc: string explorer: string interfold: string @@ -63,6 +67,7 @@ const NETWORKS: Record = { sepolia: { chain: sepolia, name: 'Sepolia', + testnet: true, rpc: 'https://ethereum-sepolia.publicnode.com', explorer: 'https://sepolia.etherscan.io', interfold: '0x3E856E24c7a95d0e04d387f847DA6FA9f6F6c20C', @@ -82,6 +87,7 @@ const NETWORKS: Record = { mainnet: { chain: mainnet, name: 'Ethereum mainnet', + testnet: false, rpc: 'https://ethereum-rpc.publicnode.com', explorer: 'https://etherscan.io', interfold: '0x28cF63B459e6218C69EA97ea7D90541cf648c715', @@ -150,6 +156,10 @@ export const CHAIN = NET.chain // Human-readable network name for UI copy. export const NETWORK_NAME = NET.name +// Whether the selected network is a test network. Test-only UI must gate on this +// as well as on the faucet address. +export const IS_TESTNET = NET.testnet + // Block explorer base URL for the selected network. export const EXPLORER_URL = NET.explorer diff --git a/tests/integration/fns.sh b/tests/integration/fns.sh index f079d4fe36..294d8e327a 100644 --- a/tests/integration/fns.sh +++ b/tests/integration/fns.sh @@ -172,15 +172,22 @@ waiton-files() { done } + # `${var,,}` needs bash >= 4. macOS ships bash 3.2, so lowercase with `tr` + # to keep this harness runnable on a developer machine. + lowercase() { + printf '%s' "$1" | tr '[:upper:]' '[:lower:]' + } + node_name_for_address() { - local address="${1,,}" + local address + address="$(lowercase "$1")" case "$address" in - "${CIPHERNODE_ADDRESS_1,,}") echo "cn1" ;; - "${CIPHERNODE_ADDRESS_2,,}") echo "cn2" ;; - "${CIPHERNODE_ADDRESS_3,,}") echo "cn3" ;; - "${CIPHERNODE_ADDRESS_4,,}") echo "cn4" ;; - "${CIPHERNODE_ADDRESS_5,,}") echo "cn5" ;; + "$(lowercase "$CIPHERNODE_ADDRESS_1")") echo "cn1" ;; + "$(lowercase "$CIPHERNODE_ADDRESS_2")") echo "cn2" ;; + "$(lowercase "$CIPHERNODE_ADDRESS_3")") echo "cn3" ;; + "$(lowercase "$CIPHERNODE_ADDRESS_4")") echo "cn4" ;; + "$(lowercase "$CIPHERNODE_ADDRESS_5")") echo "cn5" ;; *) echo "Unknown ciphernode address: $1" >&2 return 1 diff --git a/tests/integration/test.sh b/tests/integration/test.sh index f199bd74b9..c7eb273db1 100755 --- a/tests/integration/test.sh +++ b/tests/integration/test.sh @@ -21,7 +21,7 @@ parse_integration_args() { ;; *) echo "Unknown integration argument: $1" >&2 - echo "Usage: ./test.sh [base|persist|net|restart] [--skip-proof-aggregation true|false] [--no-prebuild]" >&2 + echo "Usage: ./test.sh [base|persist|net] [--skip-proof-aggregation true|false] [--no-prebuild]" >&2 exit 1 ;; esac @@ -57,16 +57,32 @@ if [ $# -eq 0 ]; then "$THIS_DIR/persist.sh" "$THIS_DIR/base.sh" "$THIS_DIR/net.sh" - "$THIS_DIR/restart.sh" else SCRIPT_NAME="$1" shift parse_integration_args "$@" export_integration_flags + SUITE="$THIS_DIR/${SCRIPT_NAME}.sh" + # Fail loudly on an unknown suite. Without this the `set -e` shell reports the + # missing file but still exits 0 through the pipeline, so CI records a pass for + # a suite that never ran. + if [[ ! -f "$SUITE" ]]; then + echo "Unknown integration suite: ${SCRIPT_NAME}" >&2 + echo "Available suites:" >&2 + for candidate in "$THIS_DIR"/*.sh; do + name="$(basename "$candidate" .sh)" + case "$name" in + test|fns|prebuild) continue ;; + esac + echo " - $name" >&2 + done + exit 1 + fi + if [[ "$SKIP_PREBUILD" != "true" ]]; then "$THIS_DIR/lib/prebuild.sh" fi - "$THIS_DIR/${SCRIPT_NAME}.sh" + "$SUITE" fi From 2ee921021018473e969126d94589af6f4279b89c Mon Sep 17 00:00:00 2001 From: ctrlc03 <93448202+ctrlc03@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:16:06 +0100 Subject: [PATCH 2/6] chore: adjust timeout --- agent/flow-trace/04_DKG_AND_COMPUTATION.md | 8 ++-- .../node_proof_timeout.rs | 47 ++++++++++++++----- 2 files changed, 41 insertions(+), 14 deletions(-) diff --git a/agent/flow-trace/04_DKG_AND_COMPUTATION.md b/agent/flow-trace/04_DKG_AND_COMPUTATION.md index 6cc2858f8a..a50f2ebe7c 100644 --- a/agent/flow-trace/04_DKG_AND_COMPUTATION.md +++ b/agent/flow-trace/04_DKG_AND_COMPUTATION.md @@ -337,9 +337,11 @@ mid-fold sends nothing. `PublicKeyAggregator` therefore arms a durable budget wh `E3Failed { failed_at_stage: CommitteeFinalized, reason: DKGTimeout }`. The late parties are not dropped from the honest set instead: C5 is signed before the cross-node fold completes and binds exactly those H keyshares, so a different honest set would invalidate a published proof. The budget -is `E3_DKG_NODE_PROOF_TIMEOUT_SECS`, and its default is calibrated for the insecure test preset. -Measure a node fold at the deployment preset before secure operation, because a budget below the -honest fold time fails every E3 on healthy nodes. +is `E3_DKG_NODE_PROOF_TIMEOUT_SECS`, and its default matches the DKG window (7200 s) because a node +proof that arrives after the window cannot be used by its E3. Measured `ZkNodeDkgFold` at the +`secure-8192` preset is 132 s at N=3, 380 s at N=5, and 904 s at N=9, and a member that restarts +mid-DKG re-proves about 5500 s of inner circuits before it can fold again. Do not lower the budget +below that restart-inclusive worst case for the deployed committee size. **Failure bridge:** `ProofRequestActor` now converts proof-generation worker failures and local proof-signing failures into terminal round failures instead of only logging that the proof-bearing diff --git a/crates/aggregator/src/public_key_aggregation/node_proof_timeout.rs b/crates/aggregator/src/public_key_aggregation/node_proof_timeout.rs index ef63baa60b..9a5a758085 100644 --- a/crates/aggregator/src/public_key_aggregation/node_proof_timeout.rs +++ b/crates/aggregator/src/public_key_aggregation/node_proof_timeout.rs @@ -22,18 +22,21 @@ pub(crate) const DKG_NODE_PROOF_TIMEOUT_ENV: &str = "E3_DKG_NODE_PROOF_TIMEOUT_S /// Default budget for collecting every honest party's NodeDkgFold proof. /// -/// A node fold is the most expensive job in the DKG and its cost grows with the ring degree. -/// Measured at the insecure test preset (degree 512) across five folds: 135 s, 137 s, 147 s, -/// 214 s, 214 s. A restarted member must also re-prove C1–C4 before it can start folding -/// (~40 s more). 30 minutes is ~8x the slowest measured fold, which bounds the stall well -/// below the two-hour DKG window while leaving room for a slow or once-restarted member. +/// Matches the DKG window (`E3_DKG_WINDOW_SECS`, 7200 s), for the same reason the `bb` cap does: +/// a node proof that arrives after the window cannot be used by the E3 it belongs to, because +/// `Interfold.onCommitteePublished` rejects a key published after `dkgDeadline`. Failing earlier +/// than the window only converts a recoverable delay into a lost E3. /// -/// CAUTION — this default is calibrated against insecure test params and has NOT been measured -/// at secure params. Secure operation uses degree 32768 (64x this ring), and if fold cost grows -/// even linearly in the degree the honest fold alone would exceed this budget, so every E3 would -/// fail with `DKGTimeout` on healthy nodes. Measure the fold at the deployment preset and raise -/// this default (or set `E3_DKG_NODE_PROOF_TIMEOUT_SECS`) before running at secure N. -pub(crate) const DEFAULT_DKG_NODE_PROOF_TIMEOUT_SECS: u64 = 1800; +/// A shorter budget is unsafe at production parameters. Measured `ZkNodeDkgFold` at the +/// `secure-8192` preset (`circuits/benchmarks/results_secure_*`) is 132 s at N=3, 380 s at N=5, +/// and 904 s at N=9, which extrapolates to about 2180 s at N=19, the largest supported committee. +/// A member that restarts mid-DKG must also re-prove its inner circuits before it can fold again, +/// which measures about 5500 s per node at N=9. The earlier 1800 s default was therefore below +/// the honest completion time for a restarted member at N=9 and would have failed healthy E3s. +/// +/// Operators who want a tighter bound must measure a node fold at their own preset and committee +/// size first, then set `E3_DKG_NODE_PROOF_TIMEOUT_SECS` above the restart-inclusive worst case. +pub(crate) const DEFAULT_DKG_NODE_PROOF_TIMEOUT_SECS: u64 = 7200; /// Resolve the collection budget, honouring the environment override. pub(crate) fn dkg_node_proof_timeout() -> Duration { @@ -70,4 +73,26 @@ mod tests { std::env::remove_var(DKG_NODE_PROOF_TIMEOUT_ENV); } + + /// The budget must not drop below the honest completion time at production parameters. + /// + /// Measured `ZkNodeDkgFold` at `secure-8192` is 904 s per node at N=9, and a member that + /// restarts mid-DKG re-proves about 5500 s of inner circuits before it can fold again. A + /// budget under that sum fails healthy E3s, which is worse than the stall it replaces. The + /// DKG window is the natural bound: a proof that lands later cannot be used by its E3. + #[test] + fn default_budget_covers_a_restarted_member_at_secure_parameters() { + const DKG_WINDOW_SECS: u64 = 7200; + const MEASURED_RESTART_WORST_CASE_SECS: u64 = 6414; + + assert_eq!( + DEFAULT_DKG_NODE_PROOF_TIMEOUT_SECS, DKG_WINDOW_SECS, + "the budget must track the DKG window; work finishing later cannot be used" + ); + assert!( + DEFAULT_DKG_NODE_PROOF_TIMEOUT_SECS > MEASURED_RESTART_WORST_CASE_SECS, + "budget {DEFAULT_DKG_NODE_PROOF_TIMEOUT_SECS} s would fail a healthy restarted \ + member that needs {MEASURED_RESTART_WORST_CASE_SECS} s at N=9" + ); + } } From 8ef253f5cef33ba152980c49f7f6a20cf9a39488 Mon Sep 17 00:00:00 2001 From: ctrlc03 <93448202+ctrlc03@users.noreply.github.com> Date: Mon, 7 Sep 2026 11:44:02 +0100 Subject: [PATCH 3/6] chore: ci --- agent/INVARIANTS.md | 26 +++++ agent/flow-trace/04_DKG_AND_COMPUTATION.md | 21 ++-- crates/aggregator/src/ext.rs | 1 + .../src/public_key_aggregation/actor.rs | 105 +++++++++++++++++- .../effects/aggregate_dkg_proofs.rs | 4 + .../effects/aggregate_public_key.rs | 4 + .../effects/fold_node_proofs.rs | 4 + .../effects/handle_compute_results.rs | 7 ++ .../effects/verify_key_proofs.rs | 4 + .../src/public_key_aggregation/handlers.rs | 6 +- .../node_proof_timeout.rs | 4 +- .../src/public_key_aggregation/state.rs | 13 +++ .../src/public_key_aggregation/tests/mod.rs | 1 + .../tests/node_proof_deadline.rs | 62 +++++++++++ .../src/ciphernode_builder.rs | 16 ++- crates/entrypoint/src/nodes/client.rs | 41 ++++++- crates/entrypoint/src/nodes/nodes.rs | 26 +++++ .../events/src/request_router_checkpoint.rs | 7 +- crates/net/src/document_publishing/effects.rs | 28 ++++- .../net/src/document_publishing/handlers.rs | 12 +- .../document_publishing/tests/publishing.rs | 76 +++++++++++++ crates/request/src/routing/actor.rs | 37 +++++- crates/request/src/routing/tests.rs | 37 ++++++ .../src/commitment_consistency/actor.rs | 37 ++++-- .../commitment_consistency/workflow_tests.rs | 57 ++++++++++ crates/sync/src/sync/schema_version.rs | 7 +- .../effects/decryption_key_proofs.rs | 11 +- .../effects/encryption_key_result.rs | 25 ++++- dappnode/docker-compose.yml | 6 +- dappnode/tests/test-hardening.sh | 12 +- deploy/docker-compose.yml | 8 +- scripts/invariant-baselines.env | 11 +- 32 files changed, 667 insertions(+), 49 deletions(-) diff --git a/agent/INVARIANTS.md b/agent/INVARIANTS.md index 6b3497fc54..ba3164c514 100644 --- a/agent/INVARIANTS.md +++ b/agent/INVARIANTS.md @@ -641,6 +641,25 @@ design citation alone does not establish current runtime behavior. - The node shutdown deadline and the fanout accept timeout come from one constant, `NODE_SHUTDOWN_DEADLINE = FANOUT_ACCEPT_TIMEOUT + 30 s`. The daemon SIGKILL delay must stay above that deadline, so a node is never killed while it still flushes. — `flow-trace/06` +- A process that must outlive its spawner must not be started with `kill_on_drop`. The handle is + dropped as soon as the spawner returns, so the flag kills the very process it just started; + `spawn_detached_process` is the correct helper, and `spawn_process` is only for a caller that + retains the handle for the child's whole life. A detached start must then confirm readiness + through the child's own protocol, because it holds no handle to observe. — `flow-trace/06` +- Every **external** supervisor grace must also exceed `NODE_SHUTDOWN_DEADLINE`: the + `const _: () = assert!` guards cover only the in-process `nodes daemon` path, not the container + runtimes that actually run production and DAppNode nodes. `deploy/docker-compose.yml` and + `dappnode/docker-compose.yml` carry `stop_grace_period`, and the DAppNode gate must compare the + parsed value against the deadline rather than pin a literal — a pinned literal silently asserts + the opposite of this rule once the Rust constant moves. — `flow-trace/06` +- A deadline that bounds a wait must be persisted as an absolute instant and re-armed on recovery, + not held only in an actix `SpawnHandle`. The handle dies with the process, and a bound whose + arming events are not replayed is silently dropped by the restart it exists to survive. — + `flow-trace/04` +- A local teardown that ends this node's ability to act on an on-chain window must derive its + deadline from that window, not from a fixed constant. `setAccusationVoteValidity` enforces only a + lower bound, so governance can outgrow any constant; because every node shares the grace, the + whole committee would go dark together while the chain still accepts a report. — `flow-trace/05` - An actor that holds in-memory state derived from an event below the persisted snapshot cursor must persist that state, because replay starts at the cursor and never redelivers the event. Verified caches, own-proof records, and collector inputs all follow this rule. — `flow-trace/06` @@ -651,6 +670,13 @@ design citation alone does not establish current runtime behavior. explicit schema version; add/remove/reorder of fields requires a compatibility test against checked-in fixtures; version mismatch runs a tested migration or fails startup with an actionable error. — `ARCHITECTURE.md` +- `#[serde(default)]` does **not** make a new field backward compatible on a bincode-persisted type. + Bincode encodes a struct as a fixed sequence with no field names, so a record written before the + field existed fails to decode with "unexpected end of file" rather than defaulting; the attribute + only covers a value built in memory. Adding a field to any type reachable from + `Repository`/`Persistable` therefore requires bumping `SCHEMA_VERSION`, which converts the opaque + decode error into the explicit halt-and-migrate path. The JSON daemon socket is the exception: it + is field-named, so `serde(default)` behaves as expected there. ## Build / config sync diff --git a/agent/flow-trace/04_DKG_AND_COMPUTATION.md b/agent/flow-trace/04_DKG_AND_COMPUTATION.md index a50f2ebe7c..b39687c502 100644 --- a/agent/flow-trace/04_DKG_AND_COMPUTATION.md +++ b/agent/flow-trace/04_DKG_AND_COMPUTATION.md @@ -331,17 +331,24 @@ aggregation path can terminate deterministically instead of stalling on missing pairwise folding. **Bounded node-proof collection:** a failed `NodeDkgFold` reports itself, but a member that dies -mid-fold sends nothing. `PublicKeyAggregator` therefore arms a durable budget when it enters +mid-fold sends nothing. `PublicKeyAggregator` therefore arms a budget when it enters `GeneratingC5Proof` and cancels it when every honest proof arrives. If the budget expires, `fail_on_missing_node_proofs` names the parties that did not deliver and publishes `E3Failed { failed_at_stage: CommitteeFinalized, reason: DKGTimeout }`. The late parties are not dropped from the honest set instead: C5 is signed before the cross-node fold completes and binds -exactly those H keyshares, so a different honest set would invalidate a published proof. The budget -is `E3_DKG_NODE_PROOF_TIMEOUT_SECS`, and its default matches the DKG window (7200 s) because a node -proof that arrives after the window cannot be used by its E3. Measured `ZkNodeDkgFold` at the -`secure-8192` preset is 132 s at N=3, 380 s at N=5, and 904 s at N=9, and a member that restarts -mid-DKG re-proves about 5500 s of inner circuits before it can fold again. Do not lower the budget -below that restart-inclusive worst case for the deployed committee size. +exactly those H keyshares, so a different honest set would invalidate a published proof. + +The budget is durable, not only an in-process timer. `GeneratingC5Proof.node_proof_deadline_at` +holds the absolute unix second, and `EffectsEnabled` re-arms it for the time that is actually left. +The in-process handle is an actix `SpawnHandle` that dies with the process, and none of the three +events that arm it are replayed on recovery, so without the persisted instant a restart would +silently drop the bound and restore the unbounded stall it exists to prevent. + +The budget is `E3_DKG_NODE_PROOF_TIMEOUT_SECS`, and its default matches the DKG window (7200 s) +because a node proof that arrives after the window cannot be used by its E3. Measured +`ZkNodeDkgFold` at the `secure-8192` preset is 132 s at N=3, 380 s at N=5, and 904 s at N=9, and a +member that restarts mid-DKG re-proves about 5500 s of inner circuits before it can fold again. Do +not lower the budget below that restart-inclusive worst case for the deployed committee size. **Failure bridge:** `ProofRequestActor` now converts proof-generation worker failures and local proof-signing failures into terminal round failures instead of only logging that the proof-bearing diff --git a/crates/aggregator/src/ext.rs b/crates/aggregator/src/ext.rs index eecb297d22..4dfd9b6dcf 100644 --- a/crates/aggregator/src/ext.rs +++ b/crates/aggregator/src/ext.rs @@ -843,6 +843,7 @@ mod tests { nodes_fold_accumulator: None, nodes_fold_completed_slots: 0, nodes_fold_step_correlation: None, + node_proof_deadline_at: None, } } diff --git a/crates/aggregator/src/public_key_aggregation/actor.rs b/crates/aggregator/src/public_key_aggregation/actor.rs index 2ca3dbb48e..22ee15d056 100644 --- a/crates/aggregator/src/public_key_aggregation/actor.rs +++ b/crates/aggregator/src/public_key_aggregation/actor.rs @@ -30,6 +30,7 @@ use e3_utils::NotifySync; use e3_utils::{ArcBytes, MAILBOX_LIMIT}; use e3_zk_helpers::CiphernodesCommitteeSize; use std::sync::Arc; +use std::time::Duration; use tracing::{debug, error, info, warn}; // Public-key aggregation state machine + pure transition logic now live in @@ -99,6 +100,11 @@ impl PublicKeyAggregator { /// `GeneratingC5Proof` (each buffered proof re-runs the dispatch path) do not extend the /// budget. Only the active aggregator arms it — a standby that is later promoted arms its /// own on promotion, which is the point at which its wait actually begins. + /// + /// Records an absolute deadline in the persisted state as well as arming the in-process + /// timer. The handle is a [`SpawnHandle`] that dies with the process, and none of the three + /// events that arm it are replayed on recovery, so without the persisted instant a restart + /// would silently drop the bound and restore the unbounded stall this exists to prevent. pub(in crate::actors::publickey_aggregator) fn arm_node_proof_deadline( &mut self, ctx: &mut Context, @@ -107,15 +113,112 @@ impl PublicKeyAggregator { if self.node_proof_deadline.is_some() || !self.can_run_aggregation_effects() { return; } + let budget = node_proof_timeout::dkg_node_proof_timeout(); + let deadline_at = Self::unix_now_secs().saturating_add(budget.as_secs()); + if let Err(err) = self.persist_node_proof_deadline(ec, Some(deadline_at)) { + error!( + e3_id = %self.e3_id, + error = %err, + "Failed to persist the node-proof deadline; a restart would drop the bound" + ); + } + self.spawn_node_proof_deadline(ctx, ec, budget); + } + + /// Re-arm the bound after a restart from the persisted absolute deadline. + /// + /// Arms for the time that is actually left rather than a fresh full budget. An already + /// expired deadline fires on the next tick instead of being skipped. + pub(in crate::actors::publickey_aggregator) fn rearm_node_proof_deadline( + &mut self, + ctx: &mut Context, + ec: &EventContext, + ) { + if self.node_proof_deadline.is_some() || !self.can_run_aggregation_effects() { + return; + } + if self.missing_node_proof_parties().is_empty() { + return; + } + + let Some(PublicKeyAggregatorState::GeneratingC5Proof { + node_proof_deadline_at, + .. + }) = self.state.get() + else { + return; + }; + + // A checkpoint written before the deadline was persisted carries no instant. Start a + // full budget now: a bound that is too generous still terminates, whereas none does not. + let deadline_at = match node_proof_deadline_at { + Some(at) => at, + None => { + let at = Self::unix_now_secs() + .saturating_add(node_proof_timeout::dkg_node_proof_timeout().as_secs()); + if let Err(err) = self.persist_node_proof_deadline(ec, Some(at)) { + error!( + e3_id = %self.e3_id, + error = %err, + "Failed to persist a node-proof deadline during recovery" + ); + } + at + } + }; + + let remaining = Duration::from_secs(deadline_at.saturating_sub(Self::unix_now_secs())); + info!( + e3_id = %self.e3_id, + remaining_secs = remaining.as_secs(), + missing_party_ids = ?self.missing_node_proof_parties(), + "Re-armed the DKG node-proof deadline after recovery" + ); + self.spawn_node_proof_deadline(ctx, ec, remaining); + } + + fn spawn_node_proof_deadline( + &mut self, + ctx: &mut Context, + ec: &EventContext, + delay: Duration, + ) { let budget = node_proof_timeout::dkg_node_proof_timeout(); let ec = ec.clone(); - let handle = ctx.run_later(budget, move |actor, _ctx| { + let handle = ctx.run_later(delay, move |actor, _ctx| { actor.node_proof_deadline = None; + // Clear the persisted instant so a restart after the failure does not re-arm a + // deadline for an E3 that has already been failed. + let _ = actor.persist_node_proof_deadline(&ec, None); actor.fail_on_missing_node_proofs(&ec, budget); }); self.node_proof_deadline = Some(handle); } + fn persist_node_proof_deadline( + &mut self, + ec: &EventContext, + deadline_at: Option, + ) -> Result<()> { + self.state.try_mutate(ec, |mut state| { + if let PublicKeyAggregatorState::GeneratingC5Proof { + node_proof_deadline_at, + .. + } = &mut state + { + *node_proof_deadline_at = deadline_at; + } + Ok(state) + }) + } + + pub(in crate::actors::publickey_aggregator) fn unix_now_secs() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) + } + /// Cancel the bounded wait once every honest proof is in (or the E3 is finished). pub(in crate::actors::publickey_aggregator) fn cancel_node_proof_deadline( &mut self, diff --git a/crates/aggregator/src/public_key_aggregation/effects/aggregate_dkg_proofs.rs b/crates/aggregator/src/public_key_aggregation/effects/aggregate_dkg_proofs.rs index bc083e899d..8437ca535f 100644 --- a/crates/aggregator/src/public_key_aggregation/effects/aggregate_dkg_proofs.rs +++ b/crates/aggregator/src/public_key_aggregation/effects/aggregate_dkg_proofs.rs @@ -91,6 +91,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation, + node_proof_deadline_at, } = state else { return Ok(state); @@ -114,6 +115,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation, + node_proof_deadline_at, }) })?; return Ok(()); @@ -217,6 +219,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation, + node_proof_deadline_at, } = state else { return Ok(state); @@ -239,6 +242,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation, + node_proof_deadline_at, }) })?; Ok(()) diff --git a/crates/aggregator/src/public_key_aggregation/effects/aggregate_public_key.rs b/crates/aggregator/src/public_key_aggregation/effects/aggregate_public_key.rs index ea0e19111e..9d90ddaec9 100644 --- a/crates/aggregator/src/public_key_aggregation/effects/aggregate_public_key.rs +++ b/crates/aggregator/src/public_key_aggregation/effects/aggregate_public_key.rs @@ -52,6 +52,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation, + node_proof_deadline_at, .. } = state else { @@ -75,6 +76,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation, + node_proof_deadline_at, }) })?; self.try_publish_complete() @@ -230,6 +232,7 @@ impl PublicKeyAggregator { nodes_fold_completed_slots, nodes_fold_step_correlation, last_ec: _, + node_proof_deadline_at, } = state else { return Ok(state); @@ -256,6 +259,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation, + node_proof_deadline_at, }) })?; diff --git a/crates/aggregator/src/public_key_aggregation/effects/fold_node_proofs.rs b/crates/aggregator/src/public_key_aggregation/effects/fold_node_proofs.rs index 0dbb6beba4..4a77115c9f 100644 --- a/crates/aggregator/src/public_key_aggregation/effects/fold_node_proofs.rs +++ b/crates/aggregator/src/public_key_aggregation/effects/fold_node_proofs.rs @@ -94,6 +94,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation: _, + node_proof_deadline_at, } = state else { return Ok(state); @@ -116,6 +117,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation: Some(corr), + node_proof_deadline_at, }) })?; Ok(()) @@ -173,6 +175,7 @@ impl PublicKeyAggregator { c5_proof_pending, last_ec, nodes_fold_step_correlation: _, + node_proof_deadline_at, .. } = state else { @@ -196,6 +199,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator: Some(accumulator_proof), nodes_fold_completed_slots: completed, nodes_fold_step_correlation: None, + node_proof_deadline_at, }) })?; diff --git a/crates/aggregator/src/public_key_aggregation/effects/handle_compute_results.rs b/crates/aggregator/src/public_key_aggregation/effects/handle_compute_results.rs index a2b06d8d8b..6d758c1636 100644 --- a/crates/aggregator/src/public_key_aggregation/effects/handle_compute_results.rs +++ b/crates/aggregator/src/public_key_aggregation/effects/handle_compute_results.rs @@ -48,6 +48,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation, + node_proof_deadline_at, } = state else { return Ok(state); @@ -71,6 +72,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation, + node_proof_deadline_at, }); } Ok(PublicKeyAggregatorState::GeneratingC5Proof { @@ -91,6 +93,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation, + node_proof_deadline_at, }) })?; self.try_publish_complete()?; @@ -150,6 +153,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation: _, + node_proof_deadline_at, } = state else { return Ok(state); @@ -172,6 +176,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation: None, + node_proof_deadline_at, }) })?; return Ok(()); @@ -223,6 +228,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation, + node_proof_deadline_at, } = state else { return Ok(state); @@ -246,6 +252,7 @@ impl PublicKeyAggregator { nodes_fold_accumulator, nodes_fold_completed_slots, nodes_fold_step_correlation, + node_proof_deadline_at, }) })?; diff --git a/crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs b/crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs index 01e5bea497..f880cc2d00 100644 --- a/crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs +++ b/crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs @@ -292,6 +292,10 @@ impl PublicKeyAggregator { nodes_fold_accumulator: None, nodes_fold_completed_slots: 0, nodes_fold_step_correlation: None, + // Fresh transition out of VerifyingC1: the node-proof wait has not started, so + // no deadline is armed yet. `arm_node_proof_deadline` persists one once C5 is + // signed and honest proofs are actually outstanding. + node_proof_deadline_at: None, }) })?; diff --git a/crates/aggregator/src/public_key_aggregation/handlers.rs b/crates/aggregator/src/public_key_aggregation/handlers.rs index ee42369b4d..1a54b34cb1 100644 --- a/crates/aggregator/src/public_key_aggregation/handlers.rs +++ b/crates/aggregator/src/public_key_aggregation/handlers.rs @@ -46,8 +46,12 @@ impl Handler for PublicKeyAggregator { trap(EType::PublickeyAggregation, &self.bus.with_ec(&ec), || { self.effects_enabled = true; self.publish_inputs_ready(ec.clone())?; - self.resume_in_flight_work(ec) + self.resume_in_flight_work(ec.clone()) }); + // Re-arm the node-proof bound after the state is hydrated. None of the events + // that arm it in-process are replayed on recovery, so without this a restart + // while proofs are outstanding would wait forever for a member that is gone. + self.rearm_node_proof_deadline(ctx, &ec); } InterfoldEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), InterfoldEventData::CommitteeMemberExpelled(data) => { diff --git a/crates/aggregator/src/public_key_aggregation/node_proof_timeout.rs b/crates/aggregator/src/public_key_aggregation/node_proof_timeout.rs index 9a5a758085..6d8352807b 100644 --- a/crates/aggregator/src/public_key_aggregation/node_proof_timeout.rs +++ b/crates/aggregator/src/public_key_aggregation/node_proof_timeout.rs @@ -89,8 +89,10 @@ mod tests { DEFAULT_DKG_NODE_PROOF_TIMEOUT_SECS, DKG_WINDOW_SECS, "the budget must track the DKG window; work finishing later cannot be used" ); + // Compared through the resolver rather than as two constants, so the check survives + // constant folding: `assert!(CONST > CONST)` is optimized out and guards nothing. assert!( - DEFAULT_DKG_NODE_PROOF_TIMEOUT_SECS > MEASURED_RESTART_WORST_CASE_SECS, + dkg_node_proof_timeout() > Duration::from_secs(MEASURED_RESTART_WORST_CASE_SECS), "budget {DEFAULT_DKG_NODE_PROOF_TIMEOUT_SECS} s would fail a healthy restarted \ member that needs {MEASURED_RESTART_WORST_CASE_SECS} s at N=9" ); diff --git a/crates/aggregator/src/public_key_aggregation/state.rs b/crates/aggregator/src/public_key_aggregation/state.rs index 484f21dca5..38c1b46ab0 100644 --- a/crates/aggregator/src/public_key_aggregation/state.rs +++ b/crates/aggregator/src/public_key_aggregation/state.rs @@ -97,6 +97,19 @@ pub enum PublicKeyAggregatorState { nodes_fold_completed_slots: u32, /// Correlation ID of the in-flight [`ZkRequest::NodesFoldStep`], if any. nodes_fold_step_correlation: Option, + /// Absolute unix second at which the node-proof collection budget expires. + /// + /// The in-process timer is a [`SpawnHandle`], which does not survive a restart, and the + /// events that arm it are not replayed. Persisting the deadline lets a hydrated + /// aggregator re-arm for the time that is actually left, so a member that goes quiet + /// cannot convert a restart into an unbounded stall. + /// + /// `serde(default)` covers a value built in memory, NOT an old on-disk record: this + /// state is bincode-encoded as a fixed sequence, so a checkpoint written before this + /// field existed fails to decode rather than defaulting. `SCHEMA_VERSION` 3 turns that + /// into an explicit halt-and-migrate message. + #[serde(default)] + node_proof_deadline_at: Option, }, Complete { public_key: ArcBytes, diff --git a/crates/aggregator/src/public_key_aggregation/tests/mod.rs b/crates/aggregator/src/public_key_aggregation/tests/mod.rs index 4d10527647..ad3b1da69e 100644 --- a/crates/aggregator/src/public_key_aggregation/tests/mod.rs +++ b/crates/aggregator/src/public_key_aggregation/tests/mod.rs @@ -50,6 +50,7 @@ fn generating_c5_state(correlation_id: CorrelationId) -> PublicKeyAggregatorStat nodes_fold_accumulator: None, nodes_fold_completed_slots: 0, nodes_fold_step_correlation: None, + node_proof_deadline_at: None, } } diff --git a/crates/aggregator/src/public_key_aggregation/tests/node_proof_deadline.rs b/crates/aggregator/src/public_key_aggregation/tests/node_proof_deadline.rs index cdd2ae5c41..82899f8a1a 100644 --- a/crates/aggregator/src/public_key_aggregation/tests/node_proof_deadline.rs +++ b/crates/aggregator/src/public_key_aggregation/tests/node_proof_deadline.rs @@ -28,6 +28,9 @@ fn awaiting_node_proofs(honest: &[u64], present: &[u64]) -> PublicKeyAggregatorS nodes_fold_accumulator: None, nodes_fold_completed_slots: 0, nodes_fold_step_correlation: None, + // These cases drive `missing_node_proof_parties` / `fail_on_missing_node_proofs` + // directly and never read the persisted instant, so no deadline is armed. + node_proof_deadline_at: None, } } @@ -136,3 +139,62 @@ async fn expiring_the_budget_is_inert_once_every_proof_arrived() -> Result<()> { ); Ok(()) } + +/// The node-proof deadline must be an absolute instant in the persisted state, not only an +/// in-process timer. +/// +/// The `SpawnHandle` dies with the process, and none of the three events that arm it +/// (`AggregatorChanged`, `PkAggregationProofSigned`, `DKGRecursiveAggregationComplete`) are +/// replayed on recovery: `AggregatorChanged` early-returns when the role has not flipped, the +/// recovery path does not republish the signed proof, and the recursive-aggregation event only +/// arrives if the stuck party sends something, which by construction it never does. Without a +/// persisted instant a restart therefore silently drops the bound and restores the unbounded +/// stall the deadline exists to prevent. +#[actix::test] +async fn the_node_proof_deadline_survives_a_restart() { + let armed_at = 1_700_000_000_u64; + + // A state hydrated from a checkpoint that was written while the wait was in progress. + let mut state = awaiting_node_proofs(&[0, 1], &[0]); + if let PublicKeyAggregatorState::GeneratingC5Proof { + node_proof_deadline_at, + .. + } = &mut state + { + *node_proof_deadline_at = Some(armed_at); + } + + // Round-trip through the durable encoding, which is what a restart actually does. + let encoded = bincode::serialize(&state).expect("state must serialize"); + let restored: PublicKeyAggregatorState = + bincode::deserialize(&encoded).expect("state must survive the durable round trip"); + + let PublicKeyAggregatorState::GeneratingC5Proof { + node_proof_deadline_at, + dkg_node_proofs, + honest_party_ids, + .. + } = restored + else { + panic!("expected GeneratingC5Proof after the round trip"); + }; + + assert_eq!( + node_proof_deadline_at, + Some(armed_at), + "the absolute deadline must survive a restart so the wait can be re-armed for the time \ + that is actually left" + ); + // The party that has not delivered is still identifiable, so the re-armed deadline has + // something to attribute the failure to. + let missing: Vec = honest_party_ids + .iter() + .filter(|id| !dkg_node_proofs.contains_key(id)) + .copied() + .collect(); + assert_eq!( + missing, + vec![1], + "the outstanding party must still be identifiable after recovery" + ); +} diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index bc66e2bd9a..3c627008ab 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -946,8 +946,20 @@ impl CiphernodeBuilder { lifecycle_stages: &HashMap, ) -> Result { let recovered_selections = recovered_ciphernode_selections(selector_state, addr)?; - let mut e3_builder = - E3Router::builder(bus, store.clone()).with_recovered_selections(recovered_selections); + // Keep a slashably-failed E3's context alive for as long as the chain still accepts a + // report. `accusationVoteValidity` has no on-chain upper bound, so a fixed grace can be + // silently outgrown by governance; take the largest window across the configured chains. + let teardown_grace = e3_request::teardown_grace_for( + accusation_vote_validity_by_chain + .values() + .copied() + .filter(|secs| *secs > 0) + .max() + .map(std::time::Duration::from_secs), + ); + let mut e3_builder = E3Router::builder(bus, store.clone()) + .with_recovered_selections(recovered_selections) + .with_teardown_grace(teardown_grace); e3_builder = e3_builder.with(AggregatorRoleExtension::create( selector_state.is_aggregator.clone(), )); diff --git a/crates/entrypoint/src/nodes/client.rs b/crates/entrypoint/src/nodes/client.rs index 511142e38e..72cf82f496 100644 --- a/crates/entrypoint/src/nodes/client.rs +++ b/crates/entrypoint/src/nodes/client.rs @@ -7,11 +7,12 @@ use anyhow::{bail, Result}; use reqwest::Client; use std::env; +use std::time::Duration; use tracing::{error, trace}; use crate::helpers::termtable::print_table; -use super::nodes::{spawn_process, Action, ProcessStatus, Query, SERVER_ADDRESS}; +use super::nodes::{spawn_detached_process, Action, ProcessStatus, Query, SERVER_ADDRESS}; pub async fn get_status() -> Result { let client = Client::new(); @@ -154,10 +155,44 @@ pub async fn start_daemon( args.push(exclude.join(",")); } - // Start and forget - spawn_process(&interfold_bin, args).await?; + // Start and forget. The daemon must outlive this CLI process, so it is spawned detached: + // holding no handle means nothing kills it when this function returns. Readiness is + // confirmed through the socket rather than through the child handle. + let mut child = spawn_detached_process(&interfold_bin, args).await?; + + // A daemon that dies immediately (port taken, bad config, missing binary) otherwise looks + // like a success: the CLI reports "started" and exits, and the operator finds out only when + // the next command cannot reach the socket. + if tokio::time::timeout(DAEMON_START_TIMEOUT, wait_until_ready()) + .await + .is_err() + { + // Report the child's own exit status when it has one; it is the actionable detail. + if let Ok(Some(status)) = child.try_wait() { + bail!("Daemon exited during startup with {status}"); + } + let _ = child.kill().await; + bail!( + "Daemon did not become ready on {SERVER_ADDRESS} within {}s", + DAEMON_START_TIMEOUT.as_secs() + ); + } tracing::info!("Daemon started successfully"); Ok(()) } + +/// How long `nodes up --detach` waits for the daemon to answer on its socket. +const DAEMON_START_TIMEOUT: Duration = Duration::from_secs(10); + +/// Poll the daemon socket until it accepts a connection. +/// +/// `is_ready` reports a refused connection as `Ok(false)` rather than an error, so a daemon that +/// is still binding its port is indistinguishable from one that never will be. The caller bounds +/// this loop with [`DAEMON_START_TIMEOUT`]. +async fn wait_until_ready() { + while !is_ready().await.unwrap_or(false) { + tokio::time::sleep(Duration::from_millis(100)).await; + } +} diff --git a/crates/entrypoint/src/nodes/nodes.rs b/crates/entrypoint/src/nodes/nodes.rs index f7f82810fb..c27dd1e267 100644 --- a/crates/entrypoint/src/nodes/nodes.rs +++ b/crates/entrypoint/src/nodes/nodes.rs @@ -25,6 +25,11 @@ pub type ProcessRecord = (Child, Vec>); pub type ProcessMap = Arc>>; /// Spawn a child process and return the Child handle +/// +/// The returned handle owns the child: `kill_on_drop` means dropping it kills the process. Use +/// this only when the caller retains the handle for the child's whole life, as `ProcessManager` +/// does in its `ProcessMap`. For a process that must outlive this one, use +/// [`spawn_detached_process`]. pub async fn spawn_process(program: &str, args: Vec) -> Result { let child = Command::new(program) .args(args) @@ -39,6 +44,27 @@ pub async fn spawn_process(program: &str, args: Vec) -> Result { Ok(child) } +/// Spawn a child process that outlives this one. +/// +/// Unlike [`spawn_process`] this does NOT set `kill_on_drop`: the caller starts the child and +/// returns, so the handle is dropped immediately and `kill_on_drop` would kill the very process +/// it just started. The child is also detached from this process's pipes, because the read ends +/// close when the caller exits and the child would then fail on every write to a broken pipe. +/// +/// The caller keeps no handle, so the child is reached through its own protocol from then on +/// (for the daemon, the socket at [`SERVER_ADDRESS`]). +pub async fn spawn_detached_process(program: &str, args: Vec) -> Result { + let child = Command::new(program) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(false) + .spawn()?; + + Ok(child) +} + #[derive(Serialize, Deserialize, Debug)] #[serde(tag = "type", content = "data")] pub enum Action { diff --git a/crates/events/src/request_router_checkpoint.rs b/crates/events/src/request_router_checkpoint.rs index a5cfc1a631..4e6673da2e 100644 --- a/crates/events/src/request_router_checkpoint.rs +++ b/crates/events/src/request_router_checkpoint.rs @@ -16,7 +16,12 @@ pub struct RequestRouterCheckpoint { pub replay_cursors: HashMap, /// E3s that failed with a slashable reason, keyed to the unix second after which the /// accusation/slashing lifecycle can no longer act and the context may be torn down. - /// Absent in checkpoints written before this field existed. + /// + /// `serde(default)` covers a value built in memory, NOT an old on-disk record: bincode + /// encodes this struct as a fixed sequence, so decoding a version-2 checkpoint that + /// predates this field fails with "unexpected end of file" rather than defaulting. + /// `SCHEMA_VERSION` was bumped to 3 so such a store halts with a migration message + /// instead of an opaque decode error. #[serde(default)] pub teardown_deadlines: HashMap, } diff --git a/crates/net/src/document_publishing/effects.rs b/crates/net/src/document_publishing/effects.rs index 150fa0659e..a759081f63 100644 --- a/crates/net/src/document_publishing/effects.rs +++ b/crates/net/src/document_publishing/effects.rs @@ -15,7 +15,7 @@ pub async fn handle_publish_document_requested( event: PublishDocumentRequested, topic: impl Into, bus: BusHandle, -) -> Result { +) -> Result { let value = event.value; let key = ContentHash::from_content(&value); let expires = Some( @@ -33,8 +33,30 @@ pub async fn handle_publish_document_requested( ) .await?; let notification = DocumentPublishedNotification::new(event.meta, key, bus.ts()?); - broadcast_document_published_notification(tx, rx, notification.clone(), topic).await?; - Ok(notification) + + // The DHT record is durable from here on, so the pointer is worth retaining even if the + // gossip publish fails. `NoPeersSubscribed` is the normal outcome when this node is the + // first to join the topic, and treating it as fatal would drop the announcement from + // `announcements_to_repeat` — the late peer would then never learn about a record that is + // sitting in the DHT. Report the outcome and let the caller retain the pointer either way. + let broadcast = broadcast_document_published_notification(tx, rx, notification.clone(), topic) + .await + .map(|_| ()); + Ok(PublishOutcome { + notification, + broadcast, + }) +} + +/// The result of publishing a document: the pointer to retain, and whether the initial gossip +/// broadcast reached the mesh. +/// +/// The two are separate because the DHT put and the gossip publish fail independently: the +/// record can be durable while the broadcast finds no subscribed peers. +#[derive(Debug)] +pub(super) struct PublishOutcome { + pub notification: DocumentPublishedNotification, + pub broadcast: Result<()>, } /// Re-gossip a notification that was already announced once. diff --git a/crates/net/src/document_publishing/handlers.rs b/crates/net/src/document_publishing/handlers.rs index bc90b5a0d2..5d9a8f36bd 100644 --- a/crates/net/src/document_publishing/handlers.rs +++ b/crates/net/src/document_publishing/handlers.rs @@ -48,9 +48,17 @@ impl Handler> for DocumentPublisher { let topic = self.topic.clone(); let addr = ctx.address(); trap_fut(EType::IO, &bus.with_ec(&ec), async move { - let notification = handle_publish_document_requested(tx, rx, msg, topic, bus).await?; + let outcome = handle_publish_document_requested(tx, rx, msg, topic, bus).await?; // Hand the gossiped pointer back to the actor so a late peer can be re-told. - addr.do_send(Announced(notification)); + // This is correctness-critical, not telemetry: a dropped `Announced` leaves the + // pointer out of `announcements_to_repeat`, so a peer that subscribes later never + // learns the DHT record and its E3 stalls. Await the send rather than fire it. + // + // Retain the pointer even when the initial broadcast failed. The DHT put already + // succeeded, so a later `GossipSubscribed` can still deliver it; failing here would + // strand a record that is present and fetchable. + addr.send(Announced(outcome.notification)).await?; + outcome.broadcast?; Ok(()) }) } diff --git a/crates/net/src/document_publishing/tests/publishing.rs b/crates/net/src/document_publishing/tests/publishing.rs index d394f4c633..c8b82b9dcf 100644 --- a/crates/net/src/document_publishing/tests/publishing.rs +++ b/crates/net/src/document_publishing/tests/publishing.rs @@ -419,3 +419,79 @@ async fn test_publishes_document_fails_with_exponential_backoff() -> Result<()> Ok(()) } + +/// A pointer whose DHT record is durable must survive an initial gossip publish that found no +/// peers, so a peer that subscribes later still learns about the document. +/// +/// `put_record` runs before the gossip publish. When this node is the first to join the topic +/// the publish fails with `NoPeersSubscribed`, which used to abort the handler before the +/// pointer was recorded — leaving a live DHT record that no late subscriber was ever told +/// about. Every other test in this file drives the initial publish to success, so this path +/// was uncovered. +#[actix::test] +async fn a_pointer_survives_an_initial_publish_with_no_peers() -> Result<()> { + let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, _, _, _) = + setup_test()?; + let e3_id = E3id::new("91", 1); + + bus.publish_without_context(PublishDocumentRequested { + meta: DocumentMeta::new(e3_id.clone(), DocumentKind::TrBFV, vec![], None), + value: ArcBytes::from_bytes(b"encryption key"), + })?; + + // The DHT put succeeds: the record is durable and fetchable from here on. + let Some(NetCommand::DhtPutRecord { + correlation_id, + key, + .. + }) = timeout(Duration::from_secs(1), net_cmd_rx.recv()).await? + else { + bail!("expected DhtPutRecord"); + }; + net_evt_tx.send(NetEvent::DhtPutRecordSucceeded { + correlation_id, + key: key.clone(), + })?; + + // The gossip publish then fails because nobody is subscribed yet. + let Some(NetCommand::GossipPublish { + correlation_id, + data: GossipData::DocumentPublishedNotification(first), + .. + }) = timeout(Duration::from_secs(1), net_cmd_rx.recv()).await? + else { + bail!("expected the first GossipPublish"); + }; + net_evt_tx.send(NetEvent::GossipPublishError { + correlation_id, + error: std::sync::Arc::new(GossipPublishFailure::from_libp2p( + libp2p::gossipsub::PublishError::NoPeersSubscribedToTopic, + )), + })?; + sleep(Duration::from_millis(50)).await; + + // A peer joins the topic afterwards. The pointer must still be re-announced. + net_evt_tx.send(NetEvent::GossipSubscribed { + count: 1, + topic: libp2p::gossipsub::IdentTopic::new("topic").hash(), + })?; + + let Some(NetCommand::GossipPublish { + data: GossipData::DocumentPublishedNotification(again), + .. + }) = timeout(Duration::from_secs(1), net_cmd_rx.recv()) + .await + .expect( + "pointer was dropped after a failed initial publish, so the late peer was never told", + ) + else { + bail!("expected the re-announced GossipPublish"); + }; + assert_eq!( + again.key, first.key, + "the re-announced pointer must reference the same durable DHT record" + ); + assert_eq!(again.meta.e3_id, e3_id); + + Ok(()) +} diff --git a/crates/request/src/routing/actor.rs b/crates/request/src/routing/actor.rs index 5379ebb176..19b60f524b 100644 --- a/crates/request/src/routing/actor.rs +++ b/crates/request/src/routing/actor.rs @@ -106,13 +106,42 @@ pub struct E3Router { /// Default for how long a slashably-failed E3's context stays alive after `E3Failed`. /// -/// The accusation manager can still initiate or vote on an accusation for up to the -/// on-chain `accusationVoteValidity` window (30 min by default) plus the local vote timeout -/// (5 min) after the failure. Two hours covers the largest window governance can set with -/// margin; the leak this bounds used to be permanent. +/// Used only when the on-chain windows are unknown (no chain configured, or the registry read +/// failed). Prefer [`teardown_grace_for`], which derives the value from the chain. pub const SLASHABLE_FAILURE_TEARDOWN_GRACE: std::time::Duration = std::time::Duration::from_secs(2 * 60 * 60); +/// The on-chain window during which a slashing report is still accepted, counted from the E3's +/// lifecycle deadline: `SlashingManager.ACCUSATION_REPORTING_WINDOW`. +pub const ACCUSATION_REPORTING_WINDOW: std::time::Duration = + std::time::Duration::from_secs(24 * 60 * 60); + +/// Local margin for a vote that is in flight when the on-chain window closes. +const LOCAL_VOTE_TIMEOUT_MARGIN: std::time::Duration = std::time::Duration::from_secs(5 * 60); + +/// How long to keep a slashably-failed E3's context, given the chain's vote-validity window. +/// +/// `SlashingManager.submitSlashProposal` accepts a report until the E3's snapshotted lifecycle +/// deadline plus `ACCUSATION_REPORTING_WINDOW` (1 day), and votes stay valid for +/// `CiphernodeRegistry.accusationVoteValidity()`, which governance can raise with no upper +/// bound (`setAccusationVoteValidity` enforces only `>=`). A context that is torn down while +/// the chain still accepts evidence takes this node out of the accusation quorum; because every +/// node uses the same grace, the whole committee goes dark at once and the H-of-N attestation +/// quorum becomes unreachable for a report the chain would still have accepted. +/// +/// So the grace covers the reporting window plus the actual vote validity plus a margin for a +/// vote already in flight, rather than a fixed constant that governance can silently outgrow. +pub fn teardown_grace_for( + accusation_vote_validity: Option, +) -> std::time::Duration { + let Some(validity) = accusation_vote_validity else { + return SLASHABLE_FAILURE_TEARDOWN_GRACE; + }; + ACCUSATION_REPORTING_WINDOW + .saturating_add(validity) + .saturating_add(LOCAL_VOTE_TIMEOUT_MARGIN) +} + /// Upper bound on how many completed E3 ids the router remembers. /// /// `completed` exists to reject late events for finished requests, and it is serialized diff --git a/crates/request/src/routing/tests.rs b/crates/request/src/routing/tests.rs index b69f7887e1..d26e86a830 100644 --- a/crates/request/src/routing/tests.rs +++ b/crates/request/src/routing/tests.rs @@ -541,3 +541,40 @@ async fn request_time_attestation_contexts_survive_router_snapshots() -> Result< ); Ok(()) } + +/// The teardown grace must cover the window during which the chain still accepts a report. +/// +/// `SlashingManager.submitSlashProposal` accepts a report until the E3's lifecycle deadline plus +/// `ACCUSATION_REPORTING_WINDOW` (1 day), and `setAccusationVoteValidity` enforces only a lower +/// bound, so governance can raise the vote window past any fixed constant. Tearing a context down +/// early removes this node from the accusation quorum while the evidence is still admissible. +#[actix::test] +async fn teardown_grace_covers_the_on_chain_accusation_window() { + // A vote validity far larger than the old fixed two-hour grace. + let validity = std::time::Duration::from_secs(7 * 24 * 60 * 60); + let grace = teardown_grace_for(Some(validity)); + + assert!( + grace > validity, + "the grace must outlast the vote-validity window it is meant to cover: \ + grace={grace:?} validity={validity:?}" + ); + assert!( + grace >= ACCUSATION_REPORTING_WINDOW + validity, + "the grace must cover the reporting window plus the vote validity: grace={grace:?}" + ); + assert!( + grace > SLASHABLE_FAILURE_TEARDOWN_GRACE, + "a governance-raised window must widen the grace beyond the fixed default" + ); +} + +/// Without a chain value the grace falls back to the fixed default rather than to zero. +#[actix::test] +async fn teardown_grace_falls_back_when_the_chain_window_is_unknown() { + assert_eq!( + teardown_grace_for(None), + SLASHABLE_FAILURE_TEARDOWN_GRACE, + "an unknown on-chain window must not shorten the grace" + ); +} diff --git a/crates/slashing/src/commitment_consistency/actor.rs b/crates/slashing/src/commitment_consistency/actor.rs index 91217b639e..170869392a 100644 --- a/crates/slashing/src/commitment_consistency/actor.rs +++ b/crates/slashing/src/commitment_consistency/actor.rs @@ -35,9 +35,9 @@ use actix::{Actor, Addr, Context, Handler}; use e3_data::Repository; use e3_events::{ - BusHandle, CommitmentConsistencyCheckRequested, CommitmentLink, E3id, EventPublisher, - EventSubscriber, EventType, InterfoldEvent, InterfoldEventData, ProofVerificationPassed, - TypedEvent, + BusHandle, CommitmentConsistencyCheckRequested, CommitmentLink, E3id, EventContext, + EventPublisher, EventSubscriber, EventType, InterfoldEvent, InterfoldEventData, + ProofVerificationPassed, Sequenced, TypedEvent, }; use e3_utils::NotifySync; use tracing::{error, info}; @@ -91,12 +91,33 @@ impl CommitmentConsistencyChecker { self } - fn persist(&self) { - if let Some(repo) = &self.snapshot_repo { - repo.write(&self.consistency.snapshot()); + /// Persist the verified-proof cache in the same atomic snapshot batch as the event that + /// changed it. + /// + /// Plain `Repository::write` is a `do_send`: the causing event can reach the event log and + /// advance its snapshot cursor while this write is still queued, and a crash in that window + /// loses the cache while replay skips the event that would rebuild it. That is exactly the + /// failure this cache exists to prevent, so the write must ride the event's batch. + fn persist(&self, context: &EventContext) { + let Some(repo) = &self.snapshot_repo else { + return; + }; + if let Err(err) = repo.write_with_context(&self.consistency.snapshot(), context) { + error!( + e3_id = %self.e3_id, + error = %err, + "Failed to persist the commitment-consistency cache; a restart would drop it" + ); } } + /// Size of the verified-proof cache. Lets a test assert that the actor actually restored + /// its durable snapshot, rather than only that the underlying service can restore one. + #[cfg(test)] + pub(crate) fn cached_proof_count(&self) -> usize { + self.consistency.cached_proof_count() + } + pub fn setup( bus: &BusHandle, e3_id: E3id, @@ -153,7 +174,7 @@ impl Handler> for CommitmentConsistencyCheck ) -> Self::Result { let (data, ec) = msg.into_components(); let violations = self.consistency.on_proof_verified(data); - self.persist(); + self.persist(&ec); for violation in violations { if let Err(err) = self.bus.publish(violation, ec.clone()) { error!( @@ -177,7 +198,7 @@ impl Handler> for CommitmentCons let Some(outcome) = self.consistency.on_check_requested(data) else { return; }; - self.persist(); + self.persist(&ec); for violation in outcome.violations { if let Err(err) = self.bus.publish(violation, ec.clone()) { diff --git a/crates/slashing/src/commitment_consistency/workflow_tests.rs b/crates/slashing/src/commitment_consistency/workflow_tests.rs index f021a703fd..eada420920 100644 --- a/crates/slashing/src/commitment_consistency/workflow_tests.rs +++ b/crates/slashing/src/commitment_consistency/workflow_tests.rs @@ -556,3 +556,60 @@ fn snapshot_roundtrip_is_lossless() { assert_eq!(restored.cached_proof_count(), 6); assert_eq!(restored.snapshot().entries.len(), snap.entries.len()); } + +/// The tests above drive `restore` directly, so they pass even when the actor never calls it. +/// This one goes through `with_snapshot`, the wiring the extension actually uses +/// (`commitment_consistency_checker_ext.rs`), so deleting the restore in the actor fails here. +#[actix::test] +async fn the_actor_restores_its_cache_from_the_durable_repository() { + use crate::actors::commitment_consistency_checker::CommitmentConsistencyChecker; + use crate::repo::CommitmentConsistencyRepositoryFactory; + use actix::{Actor, Context as ActixContext, Handler}; + use e3_data::{DataStore, InMemStore, RepositoriesFactory}; + use e3_events::{ + hlc_factory::HlcFactory, BusHandle, EventBus, EventBusConfig, Sequencer, + StoreEventRequested, + }; + + struct StoreSink; + impl Actor for StoreSink { + type Context = ActixContext; + } + impl Handler for StoreSink { + type Result = (); + fn handle(&mut self, _: StoreEventRequested, _: &mut Self::Context) {} + } + + let event_bus = EventBus::new(EventBusConfig { deduplicate: true }).start(); + let sequencer = Sequencer::new(&event_bus, StoreSink.start().recipient()).start(); + let bus = BusHandle::new(event_bus, sequencer, HlcFactory::new()) + .enable("commitment-consistency-restore-test"); + let store = DataStore::from_in_mem(&InMemStore::new(false).start()); + let repo = store.repositories().commitment_consistency(&e3()); + + // "Pre-crash": a checker caches its own C0 and persists the snapshot. + let mut before = CommitmentConsistency::new(e3(), vec![same_party_link()], 2); + before.on_proof_verified(passed( + e3(), + 1, + addr(1), + ProofType::C1PkGeneration, + [0x10; 32], + signals(0x10), + )); + assert_eq!(before.cached_proof_count(), 1); + repo.write(&before.snapshot()); + + // "Post-restart": the extension rebuilds the actor and hands it the persisted snapshot. + let restored = repo.read().await.expect("read snapshot"); + assert!(restored.is_some(), "the snapshot must survive in the store"); + let checker = CommitmentConsistencyChecker::new(&bus, e3(), vec![same_party_link()], 2) + .with_snapshot(repo, restored); + + assert_eq!( + checker.cached_proof_count(), + 1, + "the actor must start from the durable cache; without the restore the node forgets \ + its own C0 and faults honest peers" + ); +} diff --git a/crates/sync/src/sync/schema_version.rs b/crates/sync/src/sync/schema_version.rs index 8a9ab69452..54b5215e07 100644 --- a/crates/sync/src/sync/schema_version.rs +++ b/crates/sync/src/sync/schema_version.rs @@ -12,7 +12,12 @@ /// marker is the guardrail: bump it whenever a persisted format changes in a /// non-additive way. On boot the persisted value is compared against this /// constant (see `decide_schema_version`). -pub const SCHEMA_VERSION: u32 = 2; +/// +/// Version 3 adds `RequestRouterCheckpoint::teardown_deadlines`. Bincode encodes a struct as +/// a fixed sequence with no field names, so `#[serde(default)]` cannot recover the missing +/// field from a version-2 record: the decode fails with "unexpected end of file". The bump +/// converts that opaque decode error into the explicit halt-and-migrate message above. +pub const SCHEMA_VERSION: u32 = 3; /// The action a node should take after reading the persisted schema version. #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/zk-prover/src/proof_request/effects/decryption_key_proofs.rs b/crates/zk-prover/src/proof_request/effects/decryption_key_proofs.rs index f706500847..993f45725b 100644 --- a/crates/zk-prover/src/proof_request/effects/decryption_key_proofs.rs +++ b/crates/zk-prover/src/proof_request/effects/decryption_key_proofs.rs @@ -80,8 +80,15 @@ impl ProofRequestActor { /// pending dispatch of the same kind, returning that dispatch's correlation id. /// /// Only considers dispatches for `e3_id` whose kind matches `input_type`. For - /// `SmudgingNoise` the lowest outstanding `esi_idx` is taken: replayed requests were - /// dispatched in canonical order, so their responses arrive in that order too. + /// `SmudgingNoise` the lowest outstanding `esi_idx` is taken. + /// + /// SAFE ONLY WHILE THERE IS ONE ESI SLOT. `DkgShareDecryptionProofResponse` carries no + /// ESI index, so this infers the slot from arrival order. These jobs run concurrently and + /// can finish out of order, which would put a proof in another slot and produce a C4b + /// vector that does not match its witness. Today every preset generates exactly one ESI + /// share (`vec![SharedSecret::from(..)]`, `trbfv/src/gen_esi_sss.rs:91`), so there is only + /// one candidate and the order cannot be wrong. Before `num_esi > 1`, carry the index on + /// the response and match on it instead of inferring it here. pub(in crate::actors::proof_request) fn adopt_orphaned_c4_response( &self, e3_id: &E3id, diff --git a/crates/zk-prover/src/proof_request/effects/encryption_key_result.rs b/crates/zk-prover/src/proof_request/effects/encryption_key_result.rs index 08d2a65b27..788d22838a 100644 --- a/crates/zk-prover/src/proof_request/effects/encryption_key_result.rs +++ b/crates/zk-prover/src/proof_request/effects/encryption_key_result.rs @@ -49,13 +49,30 @@ impl ProofRequestActor { let local_party_id = key.party_id; - // Persist the own C0 first. It is produced once and never regenerated, and the - // event that triggered it is not replayed after a restart (see `OwnC0Record`). + // Persist the own C0 before publishing the event that depends on it. It is produced + // once and never regenerated, and the event that triggered it is not replayed after a + // restart (see `OwnC0Record`). + // + // This must ride the causing event's atomic batch: plain `write` is a `do_send`, so + // `EncryptionKeyCreated` below could be logged and its snapshot cursor advanced while + // the record is still queued. A crash in that window loses the record permanently and + // the node faults honest peers for a C0 it can no longer produce. if let Some(repo) = self.own_c0_repo(&e3_id) { - repo.write(&OwnC0Record { + let record = OwnC0Record { party_id: local_party_id, proof: proof.clone(), - }); + }; + if let Err(err) = repo.write_with_context(&record, &ec) { + error!( + e3_id = %e3_id, + party_id = local_party_id, + error = %err, + "Failed to persist the own C0 record — failing the DKG round rather than \ + continuing without the durable proof" + ); + self.fail_dkg_round(e3_id, ec, "own C0 persistence error"); + return; + } } if let Err(err) = self.bus.publish( diff --git a/dappnode/docker-compose.yml b/dappnode/docker-compose.yml index 8b00b8b32c..64ce1d0bb1 100644 --- a/dappnode/docker-compose.yml +++ b/dappnode/docker-compose.yml @@ -7,8 +7,10 @@ services: UPSTREAM_VERSION: 0.14.0 image: 'ciphernode.interfold-ciphernode.public.dappnode.eth:0.5.0' restart: unless-stopped - # Allow the node's 30-second durability barrier to finish before Docker sends SIGKILL. - stop_grace_period: 45s + # Must exceed NODE_SHUTDOWN_DEADLINE (60s = FANOUT_ACCEPT_TIMEOUT + 30s, see + # crates/events/src/eventbus.rs). A shorter grace sends SIGKILL while the node is still + # flushing its event log, losing the state the barrier exists to protect. + stop_grace_period: 65s volumes: - 'ciphernode_data:/data' ports: diff --git a/dappnode/tests/test-hardening.sh b/dappnode/tests/test-hardening.sh index 82cce582f9..c0044acebb 100644 --- a/dappnode/tests/test-hardening.sh +++ b/dappnode/tests/test-hardening.sh @@ -12,8 +12,16 @@ fail() { exit 1 } -grep -Fq 'stop_grace_period: 45s' "$ROOT_DIR/docker-compose.yml" \ - || fail "Docker stop grace period must exceed the node shutdown deadline" +# The grace must exceed NODE_SHUTDOWN_DEADLINE (60s = FANOUT_ACCEPT_TIMEOUT + 30s, see +# crates/events/src/eventbus.rs), or Docker sends SIGKILL mid store-flush. Compare the value +# instead of pinning a literal: a pinned literal silently asserts the opposite of this rule +# once the Rust constant moves. +NODE_SHUTDOWN_DEADLINE_SECS=60 +grace_line="$(grep -oE 'stop_grace_period: [0-9]+s' "$ROOT_DIR/docker-compose.yml" | head -1)" +[ -n "$grace_line" ] || fail "docker-compose.yml must declare stop_grace_period" +grace_secs="$(printf '%s' "$grace_line" | grep -oE '[0-9]+')" +[ "$grace_secs" -gt "$NODE_SHUTDOWN_DEADLINE_SECS" ] \ + || fail "Docker stop grace period (${grace_secs}s) must exceed the node shutdown deadline (${NODE_SHUTDOWN_DEADLINE_SECS}s)" assert_contains() { grep -Fq -- "$2" "$1" || fail "expected '$2' in $1" diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 07b6904be6..a8e6876c7d 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -1,7 +1,7 @@ services: cn1: image: '{{IMAGE}}' - stop_grace_period: 45s + stop_grace_period: 65s volumes: - ./cn1.yaml:/home/ciphernode/.config/interfold/config.yaml:ro - cn1-data:/home/ciphernode/.local/share/interfold @@ -20,7 +20,7 @@ services: cn2: image: '{{IMAGE}}' - stop_grace_period: 45s + stop_grace_period: 65s volumes: - ./cn2.yaml:/home/ciphernode/.config/interfold/config.yaml:ro - cn2-data:/home/ciphernode/.local/share/interfold @@ -39,7 +39,7 @@ services: cn3: image: '{{IMAGE}}' - stop_grace_period: 45s + stop_grace_period: 65s volumes: - ./cn3.yaml:/home/ciphernode/.config/interfold/config.yaml:ro - cn3-data:/home/ciphernode/.local/share/interfold @@ -58,7 +58,7 @@ services: cn4: image: '{{IMAGE}}' - stop_grace_period: 45s + stop_grace_period: 65s depends_on: - cn1 volumes: diff --git a/scripts/invariant-baselines.env b/scripts/invariant-baselines.env index 101919f3ab..49f11f9b95 100644 --- a/scripts/invariant-baselines.env +++ b/scripts/invariant-baselines.env @@ -4,4 +4,13 @@ # Call sites of `.do_send(` in crates/**/*.rs — fire-and-forget sends, allowed only for # best-effort telemetry (agent/ARCHITECTURE.md §Routing, agent/INVARIANTS.md). -DO_SEND_BASELINE=88 +# +# Raised 88 -> 89 for `crates/request/src/routing/tests.rs`: a `SequencingStore` test double +# that stands in for the real event store. Its `Handler` is synchronous, +# so it cannot await a reply, and it mirrors the production store, which answers the same +# message with `do_send` (`crates/events/src/eventstore.rs:255`). Making the double +# acknowledge a send the real store fires would test a sequencing path that does not exist. +# No production call site was added: the one correctness-critical `do_send` introduced with +# the document re-announce fix is now an awaited `send` in +# `crates/net/src/document_publishing/handlers.rs`. +DO_SEND_BASELINE=89 From d92d4070446d67551727f1766d6474c068ffa899 Mon Sep 17 00:00:00 2001 From: ctrlc03 <93448202+ctrlc03@users.noreply.github.com> Date: Wed, 9 Sep 2026 08:08:26 +0100 Subject: [PATCH 4/6] fix: reduce restart repetition work --- .../effects/verify_key_proofs.rs | 20 +- .../src/public_key_aggregation/tests/mod.rs | 67 +++++ .../keyshare/src/threshold_keyshare/actor.rs | 35 +++ .../effects/route_events.rs | 15 +- .../src/threshold_keyshare/handlers.rs | 1 + .../keyshare/src/threshold_keyshare/tests.rs | 65 +++++ .../IBondingRegistry.json | 2 +- .../ICiphernodeRegistry.json | 2 +- .../interfaces/IInterfold.sol/IInterfold.json | 2 +- .../ISlashingManager.json | 2 +- .../deployed_contracts.json | 228 +++++++++++++++++- tests/integration/interfold.config.yaml | 22 +- 12 files changed, 438 insertions(+), 23 deletions(-) diff --git a/crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs b/crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs index f880cc2d00..af030391de 100644 --- a/crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs +++ b/crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs @@ -14,11 +14,25 @@ impl PublicKeyAggregator { c1_proof: Option, ec: &EventContext, ) -> Result<()> { - if matches!( + // Collection closes at `VerifyingC1`, not at `Complete`. A keyshare that arrives after + // that is a duplicate of one already collected, not a fault: peers re-announce their + // in-flight document pointers whenever a node (re)subscribes to the gossip topic, so a + // restart anywhere in the committee re-delivers every keyshare pointer to nodes that + // have long since moved on. Treating those as errors raised `InterfoldError` on healthy + // aggregators — observed on three nodes at once in a single restart, which is exactly + // the noise that hides a real fault. + // + // Ignoring is safe because the share was already recorded: `add_keyshare` is idempotent + // per `party_id`, and the honest set is fixed once C1 verification starts. + if !matches!( self.state.get().as_ref(), - Some(PublicKeyAggregatorState::Complete { .. }) + Some(PublicKeyAggregatorState::Collecting { .. }) ) { - info!("Ignoring replayed keyshare after public-key aggregation completed"); + debug!( + e3_id = %self.e3_id, + party_id, + "Ignoring a keyshare that arrived after collection closed" + ); return Ok(()); } self.state.try_mutate(ec, |state| { diff --git a/crates/aggregator/src/public_key_aggregation/tests/mod.rs b/crates/aggregator/src/public_key_aggregation/tests/mod.rs index ad3b1da69e..aa354d9812 100644 --- a/crates/aggregator/src/public_key_aggregation/tests/mod.rs +++ b/crates/aggregator/src/public_key_aggregation/tests/mod.rs @@ -269,3 +269,70 @@ async fn standby_persists_and_resumes_public_key_work() -> Result<()> { mod attestations; mod failures; mod node_proof_deadline; + +/// A keyshare that arrives after collection closed must be ignored, not raised as an error. +/// +/// Peers re-announce their in-flight document pointers whenever a node (re)subscribes to the +/// gossip topic, so a restart anywhere in the committee re-delivers every keyshare pointer to +/// aggregators that already moved to `VerifyingC1`. Round 22 killed one committee member and +/// three healthy nodes published `InterfoldError("Can only add keyshare in Collecting state")` +/// within 100 s of its restart. The guard used to cover only `Complete`, which is reached far +/// too late to catch this. +#[actix::test] +async fn a_keyshare_after_collection_closed_is_ignored_not_an_error() -> Result<()> { + let e3_id = E3id::new("42", 1); + let committee = CiphernodesCommitteeSize::Minimum.values(); + let nodes = (0..committee.n as u64) + .map(|party_id| (party_id, format!("0x{:040x}", party_id + 1))) + .collect::>(); + let state = PublicKeyAggregatorState::init( + committee.n, + committee.threshold, + Seed([0; 32]), + nodes.clone(), + ); + let (mut aggregator, _history, _) = build_public_key_aggregator(state).await?; + let ec = test_ctx(EffectsEnabled::new()); + + // Fill the committee so collection closes and the actor moves to VerifyingC1. + for (party_id, node) in &nodes { + aggregator.add_keyshare( + ArcBytes::from_bytes(&[*party_id as u8]), + node.clone(), + *party_id, + Some(c1_proof_with_pk_commitment(&e3_id, [7; 32])), + &ec, + )?; + } + assert!( + matches!( + aggregator.state.get(), + Some(PublicKeyAggregatorState::VerifyingC1 { .. }) + ), + "collection must be closed before the late re-announce arrives" + ); + + // The re-announced duplicate. Before the fix this returned Err and surfaced as an + // InterfoldError on a node doing nothing wrong. + let (party_id, node) = nodes.iter().next().expect("committee is not empty"); + let result = aggregator.add_keyshare( + ArcBytes::from_bytes(&[*party_id as u8]), + node.clone(), + *party_id, + Some(c1_proof_with_pk_commitment(&e3_id, [7; 32])), + &ec, + ); + assert!( + result.is_ok(), + "a re-announced keyshare must be ignored after collection closed, not raised as an \ + error: {result:?}" + ); + assert!( + matches!( + aggregator.state.get(), + Some(PublicKeyAggregatorState::VerifyingC1 { .. }) + ), + "the late keyshare must not disturb the state" + ); + Ok(()) +} diff --git a/crates/keyshare/src/threshold_keyshare/actor.rs b/crates/keyshare/src/threshold_keyshare/actor.rs index 3991efb5a3..f1b1528bc1 100644 --- a/crates/keyshare/src/threshold_keyshare/actor.rs +++ b/crates/keyshare/src/threshold_keyshare/actor.rs @@ -150,12 +150,32 @@ struct PendingKeyshareWork { early_all_shares_collected: Option>, } +/// The reason a `DecryptionKeyShared` found no collector. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum UncollectedShare { + /// Collection finished and the collector was dropped; this is a late duplicate. Peers + /// re-announce their in-flight document pointers on every (re)subscribe, so a restart + /// anywhere in the committee re-delivers shares minutes after they were consumed. + AlreadyCollected, + /// No collector was ever needed because this node is the only honest party. + SoleHonestParty, +} + pub struct ThresholdKeyshare { bus: BusHandle, cipher: Arc, decryption_key_collector: Option>, encryption_key_collector: Option>, decryption_key_shared_collector: Option>, + /// Set once `AllDecryptionKeySharesCollected` fires, so a share that arrives afterwards is + /// recognised as a late duplicate rather than reported as a missing collector. + /// + /// The collector is dropped the moment collection completes, which makes "no collector" + /// ambiguous: it means either "this node is the sole honest party and never needed one" or + /// "collection already finished". Peers re-announce their in-flight document pointers on + /// every (re)subscribe, so a restart anywhere in the committee re-delivers shares long after + /// the fact and used to log the misleading second case as the first. + decryption_key_shares_collected: bool, state: Persistable, recovery: Persistable, share_enc_preset: BfvPreset, @@ -164,6 +184,20 @@ pub struct ThresholdKeyshare { } impl ThresholdKeyshare { + /// Why a `DecryptionKeyShared` arrived with no collector to receive it. + /// + /// The collector is dropped the instant collection completes, so its absence alone is + /// ambiguous. Distinguishing the two cases keeps the operator-facing warning meaningful: + /// only [`UncollectedShare::SoleHonestParty`] is unusual, and reporting a routine + /// re-announce as that hides the case an operator should act on. + pub(crate) fn classify_uncollected_share(&self) -> UncollectedShare { + if self.decryption_key_shares_collected { + UncollectedShare::AlreadyCollected + } else { + UncollectedShare::SoleHonestParty + } + } + pub fn new(params: ThresholdKeyshareParams) -> Self { let recovered = params.recovery.get().unwrap_or_default(); let own_party_id = params.state.get().map(|state| state.party_id); @@ -190,6 +224,7 @@ impl ThresholdKeyshare { decryption_key_collector: None, encryption_key_collector: None, decryption_key_shared_collector: None, + decryption_key_shares_collected: false, state: params.state, recovery: params.recovery, share_enc_preset: params.share_enc_preset, diff --git a/crates/keyshare/src/threshold_keyshare/effects/route_events.rs b/crates/keyshare/src/threshold_keyshare/effects/route_events.rs index 4554d9f560..4e0ca7c2bf 100644 --- a/crates/keyshare/src/threshold_keyshare/effects/route_events.rs +++ b/crates/keyshare/src/threshold_keyshare/effects/route_events.rs @@ -113,10 +113,17 @@ impl Handler for ThresholdKeyshare { collector.do_send(TypedEvent::new(data, ec)); Ok(()) } else { - warn!( - "DecryptionKeyShared from party {} dropped — no collector (sole honest party)", - data.party_id - ); + match self.classify_uncollected_share() { + UncollectedShare::AlreadyCollected => debug!( + "DecryptionKeyShared from party {} ignored — \ + collection already complete", + data.party_id + ), + UncollectedShare::SoleHonestParty => warn!( + "DecryptionKeyShared from party {} dropped — no collector (sole honest party)", + data.party_id + ), + } Ok(()) } } diff --git a/crates/keyshare/src/threshold_keyshare/handlers.rs b/crates/keyshare/src/threshold_keyshare/handlers.rs index a4d69420f9..5da12e52dd 100644 --- a/crates/keyshare/src/threshold_keyshare/handlers.rs +++ b/crates/keyshare/src/threshold_keyshare/handlers.rs @@ -211,6 +211,7 @@ impl Handler> for ThresholdKeyshare || { let (msg, ec) = msg.into_components(); self.decryption_key_shared_collector = None; + self.decryption_key_shares_collected = true; self.dispatch_c4_verification(msg.shares, ec) }, ) diff --git a/crates/keyshare/src/threshold_keyshare/tests.rs b/crates/keyshare/src/threshold_keyshare/tests.rs index 8a53a486c3..4bbfc2b7f3 100644 --- a/crates/keyshare/src/threshold_keyshare/tests.rs +++ b/crates/keyshare/src/threshold_keyshare/tests.rs @@ -645,3 +645,68 @@ async fn all_shares_collected_while_collecting_encryption_keys_is_held_not_dropp Ok(()) } + +/// A `DecryptionKeyShared` arriving with no collector must be classified by WHY the collector is +/// absent, so the operator-facing warning stays meaningful. +/// +/// The collector is dropped the instant `AllDecryptionKeySharesCollected` fires, so its absence +/// alone cannot tell "sole honest party, none needed" apart from "collection already finished". +/// Peers re-announce their in-flight document pointers on every (re)subscribe, so a restart +/// anywhere in the committee re-delivers shares minutes later. Round 22 logged +/// `DecryptionKeyShared from party 1 dropped — no collector (sole honest party)` on two nodes +/// that had each already collected and consumed that exact share — a routine duplicate reported +/// as the unusual case. +#[actix::test] +async fn a_share_after_collection_is_classified_as_a_duplicate_not_sole_honest_party() -> Result<()> +{ + let ready = ReadyForDecryption { + pk_share: ArcBytes::from_bytes(&[1]), + sk_poly_sum: SensitiveBytes::from_encrypted(&[2]), + es_poly_sum: vec![SensitiveBytes::from_encrypted(&[3])], + signed_pk_generation_proof: None, + signed_sk_share_computation_proof: None, + signed_e_sm_share_computation_proof: None, + signed_sk_share_encryption_proofs: Vec::new(), + signed_e_sm_share_encryption_proofs: Vec::new(), + }; + let (bus, _history) = test_bus(); + let e3_id = E3id::new("44", 1); + let store = InMemStore::new(false).start(); + let repo = Repository::::new(DataStore::from_in_mem(&store)); + let state = ThresholdKeyshareState::new( + e3_id.clone(), + 0, + KeyshareState::ReadyForDecryption(ready), + 1, + 3, + ArcBytes::from_bytes(b"params"), + Address::ZERO.to_string(), + ); + let mut actor = ThresholdKeyshare::new(ThresholdKeyshareParams { + bus, + cipher: Arc::new(Cipher::from_password("test-password").await?), + state: repo.send(Some(state)), + share_enc_preset: DEFAULT_BFV_PRESET, + interfold_address: Address::ZERO, + recovery: test_recovery(), + }); + + // Before collection completes, no collector genuinely means sole honest party. + assert_eq!( + actor.classify_uncollected_share(), + UncollectedShare::SoleHonestParty, + "before collection, a missing collector is the sole-honest-party case" + ); + + // `AllDecryptionKeySharesCollected` drops the collector and records completion. + actor.decryption_key_shared_collector = None; + actor.decryption_key_shares_collected = true; + + assert_eq!( + actor.classify_uncollected_share(), + UncollectedShare::AlreadyCollected, + "after collection completes, a re-announced share is a late duplicate, not a node \ + operating alone" + ); + Ok(()) +} diff --git a/packages/interfold-contracts/artifacts/contracts/interfaces/IBondingRegistry.sol/IBondingRegistry.json b/packages/interfold-contracts/artifacts/contracts/interfaces/IBondingRegistry.sol/IBondingRegistry.json index 33be415ab4..eb89f1db40 100644 --- a/packages/interfold-contracts/artifacts/contracts/interfaces/IBondingRegistry.sol/IBondingRegistry.json +++ b/packages/interfold-contracts/artifacts/contracts/interfaces/IBondingRegistry.sol/IBondingRegistry.json @@ -2427,5 +2427,5 @@ "deployedLinkReferences": {}, "immutableReferences": {}, "inputSourceName": "project/contracts/interfaces/IBondingRegistry.sol", - "buildInfoId": "solc-0_8_28-b95d25dbcdea5354ca4c3fb8a90c5c3f029dfa00" + "buildInfoId": "solc-0_8_28-e467bb56b612419615d50f61867526de69828bc3" } \ No newline at end of file diff --git a/packages/interfold-contracts/artifacts/contracts/interfaces/ICiphernodeRegistry.sol/ICiphernodeRegistry.json b/packages/interfold-contracts/artifacts/contracts/interfaces/ICiphernodeRegistry.sol/ICiphernodeRegistry.json index c7537a851e..bce493c1c9 100644 --- a/packages/interfold-contracts/artifacts/contracts/interfaces/ICiphernodeRegistry.sol/ICiphernodeRegistry.json +++ b/packages/interfold-contracts/artifacts/contracts/interfaces/ICiphernodeRegistry.sol/ICiphernodeRegistry.json @@ -1962,5 +1962,5 @@ "deployedLinkReferences": {}, "immutableReferences": {}, "inputSourceName": "project/contracts/interfaces/ICiphernodeRegistry.sol", - "buildInfoId": "solc-0_8_28-b95d25dbcdea5354ca4c3fb8a90c5c3f029dfa00" + "buildInfoId": "solc-0_8_28-e467bb56b612419615d50f61867526de69828bc3" } \ No newline at end of file diff --git a/packages/interfold-contracts/artifacts/contracts/interfaces/IInterfold.sol/IInterfold.json b/packages/interfold-contracts/artifacts/contracts/interfaces/IInterfold.sol/IInterfold.json index d8f5fe936b..c84ae71492 100644 --- a/packages/interfold-contracts/artifacts/contracts/interfaces/IInterfold.sol/IInterfold.json +++ b/packages/interfold-contracts/artifacts/contracts/interfaces/IInterfold.sol/IInterfold.json @@ -2912,5 +2912,5 @@ "deployedLinkReferences": {}, "immutableReferences": {}, "inputSourceName": "project/contracts/interfaces/IInterfold.sol", - "buildInfoId": "solc-0_8_28-b95d25dbcdea5354ca4c3fb8a90c5c3f029dfa00" + "buildInfoId": "solc-0_8_28-e467bb56b612419615d50f61867526de69828bc3" } \ No newline at end of file diff --git a/packages/interfold-contracts/artifacts/contracts/interfaces/ISlashingManager.sol/ISlashingManager.json b/packages/interfold-contracts/artifacts/contracts/interfaces/ISlashingManager.sol/ISlashingManager.json index 600e4140a4..55416dcbbc 100644 --- a/packages/interfold-contracts/artifacts/contracts/interfaces/ISlashingManager.sol/ISlashingManager.json +++ b/packages/interfold-contracts/artifacts/contracts/interfaces/ISlashingManager.sol/ISlashingManager.json @@ -1597,5 +1597,5 @@ "deployedLinkReferences": {}, "immutableReferences": {}, "inputSourceName": "project/contracts/interfaces/ISlashingManager.sol", - "buildInfoId": "solc-0_8_28-b95d25dbcdea5354ca4c3fb8a90c5c3f029dfa00" + "buildInfoId": "solc-0_8_28-e467bb56b612419615d50f61867526de69828bc3" } \ No newline at end of file diff --git a/packages/interfold-contracts/deployed_contracts.json b/packages/interfold-contracts/deployed_contracts.json index 86b6cf8db8..f3429c4407 100644 --- a/packages/interfold-contracts/deployed_contracts.json +++ b/packages/interfold-contracts/deployed_contracts.json @@ -384,5 +384,231 @@ "implementationAddress": "0x4FF6e77A10E8f06C11a4DD2A71b6AB55394640e4" } } + }, + "localhost": { + "PoseidonT3": { + "blockNumber": 7, + "address": "0x3333333C0A88F9BE4fd23ed0536F9B6c427e3B93" + }, + "MockUSDC": { + "constructorArgs": { + "initialSupply": "1000000" + }, + "blockNumber": 8, + "address": "0xe7f1725E7734CE288F8367e1Bb143E90bb3F0512" + }, + "InterfoldTicketToken": { + "constructorArgs": { + "baseToken": "0xe7f1725E7734CE288F8367e1Bb143E90bb3F0512", + "registry": "0x0000000000000000000000000000000000000001", + "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" + }, + "blockNumber": 9, + "address": "0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0" + }, + "SlashingEvidenceLib": { + "blockNumber": 10, + "address": "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" + }, + "SlashingManager": { + "constructorArgs": { + "initialDelay": "172800", + "admin": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" + }, + "libraries": { + "SlashingEvidenceLib": "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" + }, + "blockNumber": 11, + "address": "0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9" + }, + "CiphernodeRegistryOwnable": { + "constructorArgs": { + "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "submissionWindow": "60" + }, + "proxyRecords": { + "initData": "0xcd6dc687000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb92266000000000000000000000000000000000000000000000000000000000000003c", + "initialOwner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "proxyAddress": "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853", + "proxyAdminAddress": "0x9bd03768a7DCc129555dE410FF8E85528A4F88b5", + "implementationAddress": "0x0165878A594ca255338adfa4d48449f69242Eb8F" + }, + "libraries": { + "PoseidonT3": "0x3333333C0A88F9BE4fd23ed0536F9B6c427e3B93", + "RegistrySortitionLib": "0x5FC8d32690cc91D4c39d9d3abcBD16989F875707" + }, + "blockNumber": 13, + "address": "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853" + }, + "RegistrySortitionLib": { + "address": "0x5FC8d32690cc91D4c39d9d3abcBD16989F875707", + "blockNumber": 13 + }, + "MockRandomnessProvider": { + "constructorArgs": { + "requester": "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853" + }, + "address": "0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6", + "blockNumber": 15 + }, + "BondingAssetLib": { + "address": "0x8A791620dd6260079BF849Dc5567aDC3F2FdC318", + "blockNumber": 15 + }, + "BondingEligibilityLib": { + "address": "0x610178dA211FEF7D417bC0e6FeD39F05609AD788", + "blockNumber": 15 + }, + "BondingSlashingLib": { + "address": "0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e", + "blockNumber": 15 + }, + "BondingRegistrationLib": { + "address": "0xA51c1fc2f0D1a1b8494Ed1FE312d7C3a78Ed91C0", + "blockNumber": 15 + }, + "BondingOwnershipLib": { + "address": "0x0DCd1Bf9A1b36cE34237eEaFef220932846BCD82", + "blockNumber": 15 + }, + "BondingRegistry": { + "constructorArgs": { + "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "ticketToken": "0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0", + "ciphernodeBondToken": "0x0000000000000000000000000000000000000000", + "registry": "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853", + "slashedFundsTreasury": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "ticketPrice": "10000000", + "requiredCiphernodeBond": "100000000000000000000", + "ticketTokenDecimals": "6", + "ciphernodeBondTokenDecimals": "0", + "minTicketBalance": "1", + "exitDelay": "604800" + }, + "libraries": { + "BondingAssetLib": "0x8A791620dd6260079BF849Dc5567aDC3F2FdC318", + "BondingEligibilityLib": "0x610178dA211FEF7D417bC0e6FeD39F05609AD788", + "BondingSlashingLib": "0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e", + "BondingRegistrationLib": "0xA51c1fc2f0D1a1b8494Ed1FE312d7C3a78Ed91C0", + "BondingOwnershipLib": "0x0DCd1Bf9A1b36cE34237eEaFef220932846BCD82" + }, + "proxyRecords": { + "initData": "0x571f8259000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb922660000000000000000000000009fe46736679d2d9a65f0992f2272de9f3c7fa6e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009896800000000000000000000000000000000000000000000000056bc75e2d6310000000000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a513e6e4b8f2a923d98304ec87f64353c4d5c853000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb9226600000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000093a80", + "initialOwner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "proxyAddress": "0x0B306BF915C4d645ff596e518fAf3F9669b97016", + "proxyAdminAddress": "0x524F04724632eED237cbA3c37272e018b3A7967e", + "implementationAddress": "0x9A676e781A523b5d0C0e43731313A708CB607508" + }, + "blockNumber": 15, + "address": "0x0B306BF915C4d645ff596e518fAf3F9669b97016" + }, + "InterfoldToken": { + "constructorArgs": { + "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "ccaStart": "1788908936", + "ccaEnd": "1789513736", + "noMoreLocks": "1921705736", + "bondingRegistry": "0x0B306BF915C4d645ff596e518fAf3F9669b97016" + }, + "blockNumber": 24, + "address": "0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE" + }, + "BondedCheckpoints": { + "constructorArgs": { + "registry": "0x0B306BF915C4d645ff596e518fAf3F9669b97016" + }, + "blockNumber": 27, + "address": "0x59b670e9fA9D0A427751Af201D676719a970857b" + }, + "BondedVotes": { + "constructorArgs": { + "token": "0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE", + "votesSource": "0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE", + "checkpoints": "0x59b670e9fA9D0A427751Af201D676719a970857b" + }, + "blockNumber": 28, + "address": "0x4ed7c70F96B99c776995fB64377f0d4aB3B0e1C1" + }, + "MockComputeProvider": { + "blockNumber": 30, + "address": "0xa85233C63b9Ee964Add6F2cffe00Fd84eb32338f" + }, + "MockDecryptionVerifier": { + "blockNumber": 31, + "address": "0x4A679253410272dd5232B3Ff7cF5dbB88f295319" + }, + "MockCiphertextVerifier": { + "address": "0x7a2088a1bFc9d81c55368AE168C2C02570cB814F", + "blockNumber": 32 + }, + "MockPkVerifier": { + "blockNumber": 33, + "address": "0x09635F643e140090A9A8Dcd712eD6285858ceBef" + }, + "MockE3Program": { + "blockNumber": 34, + "address": "0xc5a5C42992dECbae36851359345FE25997F5C42d" + }, + "InterfoldPricing": { + "address": "0x67d269191c92Caf3cD7723F116c85e6E9bf55933", + "blockNumber": 37 + }, + "InterfoldLifecycle": { + "address": "0xE6E340D132b5f46d1e472DebcD681B2aBc16e57E", + "blockNumber": 37 + }, + "Interfold": { + "constructorArgs": { + "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "registry": "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853", + "bondingRegistry": "0x0B306BF915C4d645ff596e518fAf3F9669b97016", + "e3RefundManager": "0x0000000000000000000000000000000000000001", + "feeToken": "0xe7f1725E7734CE288F8367e1Bb143E90bb3F0512", + "feeTokenDecimals": "6", + "maxDuration": "2592000", + "timeoutConfig": "{\"dkgWindow\":7200,\"computeWindow\":86400,\"decryptionWindow\":3600}", + "randomnessFlatFee": "1000000", + "pricingConfig": "{\"keyGenFixedPerNode\":\"100000\",\"keyGenPerEncryptionProof\":\"50000\",\"coordinationPerPair\":\"10000\",\"availabilityPerNodePerSec\":\"50\",\"decryptionPerNode\":\"300000\",\"publicationBase\":\"1000000\",\"verificationPerProof\":\"5000\",\"protocolTreasury\":\"0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266\",\"marginBps\":\"1000\",\"protocolShareBps\":\"0\",\"dkgUtilizationBps\":\"2500\",\"computeUtilizationBps\":\"5000\",\"decryptUtilizationBps\":\"2500\",\"minCommitteeSize\":\"0\",\"minThreshold\":\"0\",\"randomnessFlatFee\":\"1000000\"}", + "initialE3Program": "0xc5a5C42992dECbae36851359345FE25997F5C42d" + }, + "libraries": { + "InterfoldLifecycle": "0xE6E340D132b5f46d1e472DebcD681B2aBc16e57E", + "InterfoldPricing": "0x67d269191c92Caf3cD7723F116c85e6E9bf55933" + }, + "proxyRecords": { + "initData": "0x9de48cc5000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb92266000000000000000000000000a513e6e4b8f2a923d98304ec87f64353c4d5c8530000000000000000000000000b306bf915c4d645ff596e518faf3f9669b970160000000000000000000000000000000000000000000000000000000000000001000000000000000000000000e7f1725e7734ce288f8367e1bb143e90bb3f0512000000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000186a0000000000000000000000000000000000000000000000000000000000000c3500000000000000000000000000000000000000000000000000000000000002710000000000000000000000000000000000000000000000000000000000000003200000000000000000000000000000000000000000000000000000000000493e000000000000000000000000000000000000000000000000000000000000f42400000000000000000000000000000000000000000000000000000000000001388000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb9226600000000000000000000000000000000000000000000000000000000000003e8000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009c4000000000000000000000000000000000000000000000000000000000000138800000000000000000000000000000000000000000000000000000000000009c40000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f42400000000000000000000000000000000000000000000000000000000000278d000000000000000000000000000000000000000000000000000000000000001c2000000000000000000000000000000000000000000000000000000000000151800000000000000000000000000000000000000000000000000000000000000e10000000000000000000000000c5a5c42992decbae36851359345fe25997f5c42d", + "initialOwner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "proxyAddress": "0x84eA74d481Ee0A5332c457a4d796187F6Ba67fEB", + "proxyAdminAddress": "0x212fdfCfCC22db97DeB3AC3260414909282BB4EE", + "implementationAddress": "0xc3e53F4d16Ae77Db1c982e75a937B9f60FE63690" + }, + "blockNumber": 37, + "address": "0x84eA74d481Ee0A5332c457a4d796187F6Ba67fEB" + }, + "E3RefundManager": { + "constructorArgs": { + "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "interfold": "0x84eA74d481Ee0A5332c457a4d796187F6Ba67fEB", + "treasury": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" + }, + "proxyRecords": { + "initData": "0xc0c53b8b000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb9226600000000000000000000000084ea74d481ee0a5332c457a4d796187f6ba67feb000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb92266", + "initialOwner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "proxyAddress": "0xa82fF9aFd8f496c3d6ac40E2a0F282E47488CFc9", + "proxyAdminAddress": "0x6A358FD7B7700887b0cd974202CdF93208F793E2", + "implementationAddress": "0x9E545E3C0baAB3E08CdfD552C960A1050f373042" + }, + "blockNumber": 39, + "address": "0xa82fF9aFd8f496c3d6ac40E2a0F282E47488CFc9" + }, + "NodeReleaseRegistry": { + "address": "0x4826533B4897376654Bb4d4AD88B7faFD0C98528", + "blockNumber": 47, + "constructorArgs": { + "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", + "bondingRegistry": "0x0B306BF915C4d645ff596e518fAf3F9669b97016", + "ciphernodeRegistry": "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853" + } + } } -} +} \ No newline at end of file diff --git a/tests/integration/interfold.config.yaml b/tests/integration/interfold.config.yaml index 636edd0c1a..1f13c909d2 100644 --- a/tests/integration/interfold.config.yaml +++ b/tests/integration/interfold.config.yaml @@ -7,23 +7,23 @@ chains: rpc_url: http://127.0.0.1:4000/availability contracts: e3_program: - address: "0xb7278A61aa25c888815aFC32Ad3cC52fF24fE575" - deploy_block: 37 + address: "0xc5a5C42992dECbae36851359345FE25997F5C42d" + deploy_block: 34 interfold: - address: "0x9A676e781A523b5d0C0e43731313A708CB607508" - deploy_block: 18 + address: "0x84eA74d481Ee0A5332c457a4d796187F6Ba67fEB" + deploy_block: 37 ciphernode_registry: - address: "0x5FC8d32690cc91D4c39d9d3abcBD16989F875707" - deploy_block: 9 - bonding_registry: address: "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853" - deploy_block: 10 + deploy_block: 13 + bonding_registry: + address: "0x0B306BF915C4d645ff596e518fAf3F9669b97016" + deploy_block: 15 slashing_manager: - address: "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" - deploy_block: 8 + address: "0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9" + deploy_block: 11 fee_token: address: "0xe7f1725E7734CE288F8367e1Bb143E90bb3F0512" - deploy_block: 6 + deploy_block: 8 program: dev: true From f59e449c9333477c3f16b4a529f2e1b5615409e5 Mon Sep 17 00:00:00 2001 From: ctrlc03 <93448202+ctrlc03@users.noreply.github.com> Date: Wed, 9 Sep 2026 08:41:32 +0100 Subject: [PATCH 5/6] fix: threshold keyshare restart --- .../effects/verify_decryption_shares.rs | 17 ++++ .../src/plaintext_aggregation/tests/mod.rs | 56 ++++++++++++ .../keyshare/src/threshold_keyshare/actor.rs | 7 +- .../keyshare/src/threshold_keyshare/tests.rs | 91 +++++++++++++++++++ .../deployed_contracts.json | 60 ++++++------ tests/integration/interfold.config.yaml | 12 +-- 6 files changed, 206 insertions(+), 37 deletions(-) diff --git a/crates/aggregator/src/plaintext_aggregation/effects/verify_decryption_shares.rs b/crates/aggregator/src/plaintext_aggregation/effects/verify_decryption_shares.rs index 6d3248aa19..f1cff06593 100644 --- a/crates/aggregator/src/plaintext_aggregation/effects/verify_decryption_shares.rs +++ b/crates/aggregator/src/plaintext_aggregation/effects/verify_decryption_shares.rs @@ -12,6 +12,23 @@ impl ThresholdPlaintextAggregator { signed_decryption_proofs: Vec, ec: &EventContext, ) -> Result<()> { + // Collection closes at `VerifyingC6`. A share that arrives after that is a duplicate of + // one already collected, not a fault: peers re-announce their in-flight document + // pointers whenever a node (re)subscribes to the gossip topic, so a restart anywhere in + // the committee re-delivers every decryption share to aggregators that have moved on. + // Without this guard `TryInto` fails and the duplicate surfaces as + // `InterfoldError("PlaintextState was expected to be Collecting but it was not.")` on a + // node doing nothing wrong — the same defect that hit `add_keyshare` on the DKG side. + if !matches!( + self.state.get().as_ref(), + Some(ThresholdPlaintextAggregatorState::Collecting(_)) + ) { + debug!( + party_id, + "Ignoring a decryption share that arrived after collection closed" + ); + return Ok(()); + } let required_shares = self.aggregated_committee_n(); ensure!( required_shares > 0, diff --git a/crates/aggregator/src/plaintext_aggregation/tests/mod.rs b/crates/aggregator/src/plaintext_aggregation/tests/mod.rs index bfeffd5339..f2fffbd260 100644 --- a/crates/aggregator/src/plaintext_aggregation/tests/mod.rs +++ b/crates/aggregator/src/plaintext_aggregation/tests/mod.rs @@ -271,3 +271,59 @@ async fn standby_persists_and_resumes_plaintext_work() -> Result<()> { mod completion; mod failures; + +/// A decryption share that arrives after collection closed must be ignored, not raised as an +/// error. +/// +/// This is the decryption-phase twin of the DKG-side `add_keyshare` defect. Peers re-announce +/// their in-flight document pointers whenever a node (re)subscribes to the gossip topic, so a +/// restart anywhere in the committee re-delivers every decryption share to aggregators that have +/// already moved to `VerifyingC6`. Without a state guard `TryInto` fails and the +/// duplicate surfaces as `InterfoldError("PlaintextState was expected to be Collecting but it +/// was not.")` on a node doing nothing wrong. +#[actix::test] +async fn a_decryption_share_after_collection_closed_is_ignored_not_an_error() -> Result<()> { + // Start already past collection, which is what a late re-announce finds. + let state = ThresholdPlaintextAggregatorState::VerifyingC6(VerifyingC6 { + threshold_m: 1, + threshold_n: 2, + shares: BTreeMap::new(), + c6_proofs: BTreeMap::new(), + ciphertext_output: vec![ArcBytes::from_bytes(&[9])], + params: test_params(), + }); + let (mut aggregator, _history, _e3_id) = build_plaintext_aggregator(state, false).await?; + let ec = test_ctx(EffectsEnabled::new()); + + let result = aggregator.add_share( + 0, + vec![ArcBytes::from_bytes(&[1])], + vec![SignedProofPayload { + payload: ProofPayload { + e3_id: E3id::new("42", 1), + proof_type: ProofType::C6ThresholdShareDecryption, + proof: Proof::new( + CircuitName::ThresholdShareDecryption, + ArcBytes::from_bytes(&[1]), + ArcBytes::from_bytes(&[0u8; 32]), + ), + }, + signature: ArcBytes::from_bytes(&[0u8; 65]), + }], + &ec, + ); + + assert!( + result.is_ok(), + "a re-announced decryption share must be ignored after collection closed, not raised \ + as an error: {result:?}" + ); + assert!( + matches!( + aggregator.state.get(), + Some(ThresholdPlaintextAggregatorState::VerifyingC6(_)) + ), + "the late share must not disturb the state" + ); + Ok(()) +} diff --git a/crates/keyshare/src/threshold_keyshare/actor.rs b/crates/keyshare/src/threshold_keyshare/actor.rs index f1b1528bc1..86c6f5c19d 100644 --- a/crates/keyshare/src/threshold_keyshare/actor.rs +++ b/crates/keyshare/src/threshold_keyshare/actor.rs @@ -175,6 +175,11 @@ pub struct ThresholdKeyshare { /// "collection already finished". Peers re-announce their in-flight document pointers on /// every (re)subscribe, so a restart anywhere in the committee re-delivers shares long after /// the fact and used to log the misleading second case as the first. + /// + /// Not persisted directly: it is derived at construction from the durable + /// `decryption_key_shares` map, which recovery already carries. A restart after collection + /// completed therefore still classifies a re-announced share as a duplicate instead of + /// reporting a missing collector. decryption_key_shares_collected: bool, state: Persistable, recovery: Persistable, @@ -224,7 +229,7 @@ impl ThresholdKeyshare { decryption_key_collector: None, encryption_key_collector: None, decryption_key_shared_collector: None, - decryption_key_shares_collected: false, + decryption_key_shares_collected: !recovered.decryption_key_shares.is_empty(), state: params.state, recovery: params.recovery, share_enc_preset: params.share_enc_preset, diff --git a/crates/keyshare/src/threshold_keyshare/tests.rs b/crates/keyshare/src/threshold_keyshare/tests.rs index 4bbfc2b7f3..952b64b74e 100644 --- a/crates/keyshare/src/threshold_keyshare/tests.rs +++ b/crates/keyshare/src/threshold_keyshare/tests.rs @@ -710,3 +710,94 @@ async fn a_share_after_collection_is_classified_as_a_duplicate_not_sole_honest_p ); Ok(()) } + +/// The duplicate classification must survive a restart. +/// +/// R24 caught this: the restarted node had collected its shares before the crash, but the +/// in-memory flag reset to false and the very next re-announced share logged +/// `no collector (sole honest party)` on a node that was not alone at all. The durable +/// `decryption_key_shares` map already records what was collected, so the flag is derived from +/// it at construction rather than persisted separately. +#[actix::test] +async fn the_duplicate_classification_survives_a_restart() -> Result<()> { + let ready = ReadyForDecryption { + pk_share: ArcBytes::from_bytes(&[1]), + sk_poly_sum: SensitiveBytes::from_encrypted(&[2]), + es_poly_sum: vec![SensitiveBytes::from_encrypted(&[3])], + signed_pk_generation_proof: None, + signed_sk_share_computation_proof: None, + signed_e_sm_share_computation_proof: None, + signed_sk_share_encryption_proofs: Vec::new(), + signed_e_sm_share_encryption_proofs: Vec::new(), + }; + let (bus, _history) = test_bus(); + let e3_id = E3id::new("45", 1); + let store = InMemStore::new(false).start(); + let repo = Repository::::new(DataStore::from_in_mem(&store)); + let state = ThresholdKeyshareState::new( + e3_id.clone(), + 0, + KeyshareState::ReadyForDecryption(ready), + 1, + 3, + ArcBytes::from_bytes(b"params"), + Address::ZERO.to_string(), + ); + + // A recovery snapshot from a node that already collected a peer share before it crashed. + let recovery_store = InMemStore::new(false).start(); + let recovery_repo = + Repository::::new(DataStore::from_in_mem(&recovery_store)); + let mut recovered = ThresholdKeyshareRecoveryState::default(); + recovered.decryption_key_shares.insert( + 1, + TypedEvent::new( + DecryptionKeyShared { + e3_id: e3_id.clone(), + party_id: 1, + node: format!("0x{:040x}", 2), + signed_sk_decryption_proof: SignedProofPayload { + payload: e3_events::ProofPayload { + e3_id: e3_id.clone(), + proof_type: e3_events::ProofType::C4aSkShareDecryption, + proof: e3_events::Proof::new( + e3_events::CircuitName::DkgShareDecryption, + ArcBytes::from_bytes(&[1]), + ArcBytes::from_bytes(&[0u8; 32]), + ), + }, + signature: ArcBytes::from_bytes(&[0u8; 65]), + }, + signed_e_sm_decryption_proofs: Vec::new(), + external: true, + }, + InterfoldEvent::::new_with_timestamp( + EffectsEnabled::new().into(), + None, + 1, + None, + EventSource::Local, + ) + .into_sequenced(1) + .get_ctx() + .clone(), + ), + ); + + let actor = ThresholdKeyshare::new(ThresholdKeyshareParams { + bus, + cipher: Arc::new(Cipher::from_password("test-password").await?), + state: repo.send(Some(state)), + share_enc_preset: DEFAULT_BFV_PRESET, + interfold_address: Address::ZERO, + recovery: recovery_repo.send(Some(recovered)), + }); + + assert_eq!( + actor.classify_uncollected_share(), + UncollectedShare::AlreadyCollected, + "a node that collected shares before the crash must still treat a re-announced share \ + as a duplicate after restart" + ); + Ok(()) +} diff --git a/packages/interfold-contracts/deployed_contracts.json b/packages/interfold-contracts/deployed_contracts.json index f3429c4407..30ed2437e0 100644 --- a/packages/interfold-contracts/deployed_contracts.json +++ b/packages/interfold-contracts/deployed_contracts.json @@ -387,14 +387,14 @@ }, "localhost": { "PoseidonT3": { - "blockNumber": 7, + "blockNumber": 6, "address": "0x3333333C0A88F9BE4fd23ed0536F9B6c427e3B93" }, "MockUSDC": { "constructorArgs": { "initialSupply": "1000000" }, - "blockNumber": 8, + "blockNumber": 7, "address": "0xe7f1725E7734CE288F8367e1Bb143E90bb3F0512" }, "InterfoldTicketToken": { @@ -403,11 +403,11 @@ "registry": "0x0000000000000000000000000000000000000001", "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" }, - "blockNumber": 9, + "blockNumber": 8, "address": "0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0" }, "SlashingEvidenceLib": { - "blockNumber": 10, + "blockNumber": 9, "address": "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" }, "SlashingManager": { @@ -418,7 +418,7 @@ "libraries": { "SlashingEvidenceLib": "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" }, - "blockNumber": 11, + "blockNumber": 10, "address": "0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9" }, "CiphernodeRegistryOwnable": { @@ -437,39 +437,39 @@ "PoseidonT3": "0x3333333C0A88F9BE4fd23ed0536F9B6c427e3B93", "RegistrySortitionLib": "0x5FC8d32690cc91D4c39d9d3abcBD16989F875707" }, - "blockNumber": 13, + "blockNumber": 12, "address": "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853" }, "RegistrySortitionLib": { "address": "0x5FC8d32690cc91D4c39d9d3abcBD16989F875707", - "blockNumber": 13 + "blockNumber": 12 }, "MockRandomnessProvider": { "constructorArgs": { "requester": "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853" }, "address": "0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6", - "blockNumber": 15 + "blockNumber": 14 }, "BondingAssetLib": { "address": "0x8A791620dd6260079BF849Dc5567aDC3F2FdC318", - "blockNumber": 15 + "blockNumber": 14 }, "BondingEligibilityLib": { "address": "0x610178dA211FEF7D417bC0e6FeD39F05609AD788", - "blockNumber": 15 + "blockNumber": 14 }, "BondingSlashingLib": { "address": "0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e", - "blockNumber": 15 + "blockNumber": 14 }, "BondingRegistrationLib": { "address": "0xA51c1fc2f0D1a1b8494Ed1FE312d7C3a78Ed91C0", - "blockNumber": 15 + "blockNumber": 14 }, "BondingOwnershipLib": { "address": "0x0DCd1Bf9A1b36cE34237eEaFef220932846BCD82", - "blockNumber": 15 + "blockNumber": 14 }, "BondingRegistry": { "constructorArgs": { @@ -499,25 +499,25 @@ "proxyAdminAddress": "0x524F04724632eED237cbA3c37272e018b3A7967e", "implementationAddress": "0x9A676e781A523b5d0C0e43731313A708CB607508" }, - "blockNumber": 15, + "blockNumber": 14, "address": "0x0B306BF915C4d645ff596e518fAf3F9669b97016" }, "InterfoldToken": { "constructorArgs": { "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", - "ccaStart": "1788908936", - "ccaEnd": "1789513736", - "noMoreLocks": "1921705736", + "ccaStart": "1788941401", + "ccaEnd": "1789546201", + "noMoreLocks": "1921738201", "bondingRegistry": "0x0B306BF915C4d645ff596e518fAf3F9669b97016" }, - "blockNumber": 24, + "blockNumber": 23, "address": "0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE" }, "BondedCheckpoints": { "constructorArgs": { "registry": "0x0B306BF915C4d645ff596e518fAf3F9669b97016" }, - "blockNumber": 27, + "blockNumber": 26, "address": "0x59b670e9fA9D0A427751Af201D676719a970857b" }, "BondedVotes": { @@ -526,36 +526,36 @@ "votesSource": "0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE", "checkpoints": "0x59b670e9fA9D0A427751Af201D676719a970857b" }, - "blockNumber": 28, + "blockNumber": 27, "address": "0x4ed7c70F96B99c776995fB64377f0d4aB3B0e1C1" }, "MockComputeProvider": { - "blockNumber": 30, + "blockNumber": 29, "address": "0xa85233C63b9Ee964Add6F2cffe00Fd84eb32338f" }, "MockDecryptionVerifier": { - "blockNumber": 31, + "blockNumber": 30, "address": "0x4A679253410272dd5232B3Ff7cF5dbB88f295319" }, "MockCiphertextVerifier": { "address": "0x7a2088a1bFc9d81c55368AE168C2C02570cB814F", - "blockNumber": 32 + "blockNumber": 31 }, "MockPkVerifier": { - "blockNumber": 33, + "blockNumber": 32, "address": "0x09635F643e140090A9A8Dcd712eD6285858ceBef" }, "MockE3Program": { - "blockNumber": 34, + "blockNumber": 33, "address": "0xc5a5C42992dECbae36851359345FE25997F5C42d" }, "InterfoldPricing": { "address": "0x67d269191c92Caf3cD7723F116c85e6E9bf55933", - "blockNumber": 37 + "blockNumber": 36 }, "InterfoldLifecycle": { "address": "0xE6E340D132b5f46d1e472DebcD681B2aBc16e57E", - "blockNumber": 37 + "blockNumber": 36 }, "Interfold": { "constructorArgs": { @@ -582,7 +582,7 @@ "proxyAdminAddress": "0x212fdfCfCC22db97DeB3AC3260414909282BB4EE", "implementationAddress": "0xc3e53F4d16Ae77Db1c982e75a937B9f60FE63690" }, - "blockNumber": 37, + "blockNumber": 36, "address": "0x84eA74d481Ee0A5332c457a4d796187F6Ba67fEB" }, "E3RefundManager": { @@ -598,12 +598,12 @@ "proxyAdminAddress": "0x6A358FD7B7700887b0cd974202CdF93208F793E2", "implementationAddress": "0x9E545E3C0baAB3E08CdfD552C960A1050f373042" }, - "blockNumber": 39, + "blockNumber": 38, "address": "0xa82fF9aFd8f496c3d6ac40E2a0F282E47488CFc9" }, "NodeReleaseRegistry": { "address": "0x4826533B4897376654Bb4d4AD88B7faFD0C98528", - "blockNumber": 47, + "blockNumber": 46, "constructorArgs": { "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", "bondingRegistry": "0x0B306BF915C4d645ff596e518fAf3F9669b97016", diff --git a/tests/integration/interfold.config.yaml b/tests/integration/interfold.config.yaml index 1f13c909d2..0b56599512 100644 --- a/tests/integration/interfold.config.yaml +++ b/tests/integration/interfold.config.yaml @@ -8,22 +8,22 @@ chains: contracts: e3_program: address: "0xc5a5C42992dECbae36851359345FE25997F5C42d" - deploy_block: 34 + deploy_block: 33 interfold: address: "0x84eA74d481Ee0A5332c457a4d796187F6Ba67fEB" - deploy_block: 37 + deploy_block: 36 ciphernode_registry: address: "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853" - deploy_block: 13 + deploy_block: 12 bonding_registry: address: "0x0B306BF915C4d645ff596e518fAf3F9669b97016" - deploy_block: 15 + deploy_block: 14 slashing_manager: address: "0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9" - deploy_block: 11 + deploy_block: 10 fee_token: address: "0xe7f1725E7734CE288F8367e1Bb143E90bb3F0512" - deploy_block: 8 + deploy_block: 7 program: dev: true From e5e327d375a3c846c754e0d709d1b151ca96c958 Mon Sep 17 00:00:00 2001 From: ctrlc03 <93448202+ctrlc03@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:53:23 +0100 Subject: [PATCH 6/6] fix: more liveness issues --- .../src/public_key_aggregation/actor.rs | 21 ++ .../src/public_key_aggregation/handlers.rs | 6 +- .../tests/node_proof_deadline.rs | 61 +++++ .../effects/create_decryption_share.rs | 14 ++ .../keyshare/src/threshold_keyshare/tests.rs | 68 ++++++ crates/net/src/document_publishing/effects.rs | 4 +- .../src/ciphernode_selection/actor.rs | 14 +- crates/sortition/src/failover.rs | 61 +++++ .../deployed_contracts.json | 228 +----------------- 9 files changed, 245 insertions(+), 232 deletions(-) diff --git a/crates/aggregator/src/public_key_aggregation/actor.rs b/crates/aggregator/src/public_key_aggregation/actor.rs index 22ee15d056..857ea3a09c 100644 --- a/crates/aggregator/src/public_key_aggregation/actor.rs +++ b/crates/aggregator/src/public_key_aggregation/actor.rs @@ -220,6 +220,11 @@ impl PublicKeyAggregator { } /// Cancel the bounded wait once every honest proof is in (or the E3 is finished). + /// + /// Also clears the persisted instant when a context is available. A demoted aggregator no + /// longer owns the bound — the promoted standby arms its own — so leaving the old deadline + /// in durable state would describe a wait this node is not performing. Re-promotion is + /// unaffected either way because `arm_node_proof_deadline` always writes a fresh instant. pub(in crate::actors::publickey_aggregator) fn cancel_node_proof_deadline( &mut self, ctx: &mut Context, @@ -229,6 +234,22 @@ impl PublicKeyAggregator { } } + /// Cancel the bounded wait and clear the persisted instant that described it. + pub(in crate::actors::publickey_aggregator) fn cancel_node_proof_deadline_with_context( + &mut self, + ctx: &mut Context, + ec: &EventContext, + ) { + self.cancel_node_proof_deadline(ctx); + if let Err(err) = self.persist_node_proof_deadline(ec, None) { + error!( + e3_id = %self.e3_id, + error = %err, + "Failed to clear the node-proof deadline after demotion" + ); + } + } + fn aggregation_inputs_ready(&self) -> bool { matches!( self.state.get(), diff --git a/crates/aggregator/src/public_key_aggregation/handlers.rs b/crates/aggregator/src/public_key_aggregation/handlers.rs index 1a54b34cb1..22ee4861b1 100644 --- a/crates/aggregator/src/public_key_aggregation/handlers.rs +++ b/crates/aggregator/src/public_key_aggregation/handlers.rs @@ -182,8 +182,10 @@ impl Handler> for PublicKeyAggregator { self.arm_node_proof_deadline(ctx, &ec); } } else { - // Demoted: stop counting down. The newly promoted aggregator owns the bound. - self.cancel_node_proof_deadline(ctx); + // Demoted: stop counting down and drop the persisted instant. The newly promoted + // aggregator owns the bound now. + let ec = msg.get_ctx().clone(); + self.cancel_node_proof_deadline_with_context(ctx, &ec); } } } diff --git a/crates/aggregator/src/public_key_aggregation/tests/node_proof_deadline.rs b/crates/aggregator/src/public_key_aggregation/tests/node_proof_deadline.rs index 82899f8a1a..9afc6de276 100644 --- a/crates/aggregator/src/public_key_aggregation/tests/node_proof_deadline.rs +++ b/crates/aggregator/src/public_key_aggregation/tests/node_proof_deadline.rs @@ -198,3 +198,64 @@ async fn the_node_proof_deadline_survives_a_restart() { "the outstanding party must still be identifiable after recovery" ); } + +/// A demoted aggregator must give up the persisted deadline, not just its in-process timer. +/// +/// Failover promotes the lowest-id standby when the active aggregator stops making progress, so +/// the E3 changes hands mid-DKG. The promoted node arms its own bound; the demoted one must +/// clear the instant it wrote, or durable state describes a wait that node is no longer +/// performing and a later restart would reason from it. `arm_node_proof_deadline` always writes +/// a fresh instant, so re-promotion stays correct either way — this is about not leaving a +/// deadline behind that nothing owns. +#[actix::test] +async fn demotion_clears_the_persisted_node_proof_deadline() -> Result<()> { + // An aggregator part-way through collection, with one honest party still outstanding. + let state = awaiting_node_proofs(&[0, 1], &[0]); + let (mut aggregator, _history, _) = build_public_key_aggregator(state).await?; + aggregator.is_aggregator = true; + let ec = test_ctx(EffectsEnabled::new()); + + // Arming records an absolute instant in the persisted state. + aggregator.persist_node_proof_deadline(&ec, Some(1_700_000_000))?; + let Some(PublicKeyAggregatorState::GeneratingC5Proof { + node_proof_deadline_at, + .. + }) = aggregator.state.get() + else { + panic!("expected GeneratingC5Proof"); + }; + assert_eq!( + node_proof_deadline_at, + Some(1_700_000_000), + "arming must record the absolute instant" + ); + + // Demotion clears it: the promoted standby owns the bound now. + aggregator.persist_node_proof_deadline(&ec, None)?; + let Some(PublicKeyAggregatorState::GeneratingC5Proof { + node_proof_deadline_at, + dkg_node_proofs, + honest_party_ids, + .. + }) = aggregator.state.get() + else { + panic!("expected GeneratingC5Proof"); + }; + assert_eq!( + node_proof_deadline_at, None, + "a demoted aggregator must not leave a deadline describing a wait it is not performing" + ); + + // The outstanding party is unchanged, so the promoted standby inherits the same work. + let missing: Vec = honest_party_ids + .iter() + .filter(|id| !dkg_node_proofs.contains_key(id)) + .copied() + .collect(); + assert_eq!( + missing, + vec![1], + "demotion must not disturb which party is still owed" + ); + Ok(()) +} diff --git a/crates/keyshare/src/threshold_keyshare/effects/create_decryption_share.rs b/crates/keyshare/src/threshold_keyshare/effects/create_decryption_share.rs index b6ec7ed05a..55e7001ce8 100644 --- a/crates/keyshare/src/threshold_keyshare/effects/create_decryption_share.rs +++ b/crates/keyshare/src/threshold_keyshare/effects/create_decryption_share.rs @@ -126,6 +126,20 @@ impl ThresholdKeyshare { let msg: CalculateDecryptionShareResponse = res.try_into()?; let state = self.state.try_get()?; let e3_id = state.e3_id.clone(); + + // A restart replays this response, and the share may already be computed: the pre-crash + // handler transitions to `GeneratingDecryptionProof` once it publishes + // `ShareDecryptionProofPending`. Recomputing is not possible from that state and is not + // needed, because the C6 proof request is already in flight. Treat the late copy as the + // duplicate it is instead of raising a fault on a node that did nothing wrong. + let KeyshareState::Decrypting(_) = state.state else { + debug!( + e3_id = %e3_id, + state = ?state.state, + "Decryption share already computed for this request; ignoring the replayed response" + ); + return Ok(()); + }; let decrypting: Decrypting = state.clone().try_into()?; let d_share_poly = msg.d_share_poly; diff --git a/crates/keyshare/src/threshold_keyshare/tests.rs b/crates/keyshare/src/threshold_keyshare/tests.rs index 952b64b74e..4fa9a9e7b7 100644 --- a/crates/keyshare/src/threshold_keyshare/tests.rs +++ b/crates/keyshare/src/threshold_keyshare/tests.rs @@ -801,3 +801,71 @@ async fn the_duplicate_classification_survives_a_restart() -> Result<()> { ); Ok(()) } + +/// A restart replays the decryption-share response after the share is already computed. +/// +/// Observed on a 5-node swarm (chaos round 28): `kill -9` during decryption, then restart. The +/// node replayed its own `CalculateDecryptionShare` compute response, but the pre-crash handler +/// had already published `ShareDecryptionProofPending` and moved to `GeneratingDecryptionProof`. +/// `TryInto` then failed and raised `InterfoldError("Invalid state")` on a node that +/// had done its job correctly. The C6 proof request was already in flight, so nothing was lost — +/// only a spurious fault was reported, which is what an operator would chase. +#[actix::test] +async fn a_replayed_decryption_share_response_after_the_transition_is_ignored() -> Result<()> { + let generating = GeneratingDecryptionProof { + pk_share: ArcBytes::from_bytes(&[1]), + decryption_share: vec![ArcBytes::from_bytes(&[2])], + signed_pk_generation_proof: None, + signed_sk_share_computation_proof: None, + signed_e_sm_share_computation_proof: None, + signed_sk_share_encryption_proofs: Vec::new(), + signed_e_sm_share_encryption_proofs: Vec::new(), + }; + let (actor, history, e3_id, repo) = + start_actor_with_state(KeyshareState::GeneratingDecryptionProof(generating)).await?; + + let ctx = InterfoldEvent::::new_with_timestamp( + EffectsEnabled::new().into(), + None, + 1, + None, + EventSource::Local, + ) + .into_sequenced(1) + .get_ctx() + .clone(); + + let replayed = TypedEvent::new( + ComputeResponse::trbfv( + TrBFVResponse::CalculateDecryptionShare(CalculateDecryptionShareResponse { + d_share_poly: vec![ArcBytes::from_bytes(&[3])], + }), + CorrelationId::new(), + e3_id.clone(), + ), + ctx, + ); + + actor.send(replayed).await?; + + // The replay must not turn into a fault event. + let result = history.send(TakeEvents::::new(1)).await?; + let errors: Vec<_> = result + .events + .iter() + .filter(|e| matches!(e.get_data(), InterfoldEventData::InterfoldError(_))) + .collect(); + assert!( + errors.is_empty(), + "a replayed decryption-share response after the transition is a duplicate, not a fault; \ + got {errors:?}" + ); + + // The already-computed share is untouched. + assert!(matches!( + repo.read().await?.expect("persisted keyshare state").state, + KeyshareState::GeneratingDecryptionProof(_) + )); + + Ok(()) +} diff --git a/crates/net/src/document_publishing/effects.rs b/crates/net/src/document_publishing/effects.rs index a759081f63..9709cacd2c 100644 --- a/crates/net/src/document_publishing/effects.rs +++ b/crates/net/src/document_publishing/effects.rs @@ -8,7 +8,7 @@ use crate::net_interface_handle::NetEventSubscriber; /// Called when we receive a PublishDocumentRequested event. /// -/// Returns the notification that was gossiped so the caller can keep it for re-announcement. +/// Returns the pointer to retain plus whether the initial gossip broadcast reached the mesh. pub async fn handle_publish_document_requested( tx: mpsc::Sender, rx: NetEventSubscriber, @@ -54,7 +54,7 @@ pub async fn handle_publish_document_requested( /// The two are separate because the DHT put and the gossip publish fail independently: the /// record can be durable while the broadcast finds no subscribed peers. #[derive(Debug)] -pub(super) struct PublishOutcome { +pub struct PublishOutcome { pub notification: DocumentPublishedNotification, pub broadcast: Result<()>, } diff --git a/crates/sortition/src/ciphernode_selection/actor.rs b/crates/sortition/src/ciphernode_selection/actor.rs index a71b97dc5e..43c6ff0ea1 100644 --- a/crates/sortition/src/ciphernode_selection/actor.rs +++ b/crates/sortition/src/ciphernode_selection/actor.rs @@ -377,6 +377,18 @@ impl CiphernodeSelector { return Ok(()); } + // Arm the round at the start of a phase as well as when aggregation inputs are ready. + // + // `ready_phases` is set by `AggregationInputsReady`, which the aggregator publishes only + // once it holds a threshold of shares (`VerifyingC6` or later for the plaintext phase). + // Arming solely on that signal leaves the collection window uncovered: an aggregator that + // dies between the phase starting and inputs becoming ready is never replaced, because no + // round exists for the timer to fire on. Observed on a 5-node swarm — the active + // aggregator was killed two seconds into the decryption phase and the E3 stalled + // permanently, with the surviving members each holding a usable decryption share. + // + // A real phase change re-arms with the full budget, so covering the collection window + // does not shorten the budget for the work that follows. let ready = phase.is_some_and(|phase| self.ready_phases.get(&e3_id) == Some(&phase)); if phase_changed || phase.is_none() || ready { let now = self.clock.now_unix_secs(); @@ -385,7 +397,7 @@ impl CiphernodeSelector { if phase_changed || phase.is_none() { reconcile_phase(&mut state, &e3_id, None, now, &policy); } - if ready { + if ready || phase_changed { reconcile_phase(&mut state, &e3_id, phase, now, &policy); } Ok(state) diff --git a/crates/sortition/src/failover.rs b/crates/sortition/src/failover.rs index 363ad9fe30..56f6dade15 100644 --- a/crates/sortition/src/failover.rs +++ b/crates/sortition/src/failover.rs @@ -459,4 +459,65 @@ mod tests { assert_eq!(state.unresponsive[&id], vec![0, 1]); assert!(state.rounds[&id].exhausted); } + + /// A phase must be covered by a failover deadline from the moment it starts. + /// + /// `CiphernodeSelector::observe_phase` decides when to create a round. It previously created + /// one only when `ready_phases` held the phase, which `AggregationInputsReady` sets after the + /// aggregator already holds a threshold of shares. The collection window was therefore + /// uncovered: an aggregator that died early in a phase left no round, so no timer could fire + /// and no standby was ever promoted. + /// + /// Observed on a 5-node swarm: the active aggregator was killed two seconds into the + /// decryption phase, and 903 seconds later — over 1.5x the 600 second budget — there had been + /// no promotion and no plaintext, while both survivors held a usable decryption share. + /// + /// This models the selector's gating expression. `reconcile_phase` itself was always correct; + /// the defect was that it was not called at phase start. + #[test] + fn a_phase_is_covered_by_a_deadline_from_the_moment_it_starts() { + let id = e3_id(); + let mut state = AggregatorFailoverState::default(); + + // The phase has just changed, and no AggregationInputsReady has arrived yet. + let phase = Some(AggregatorPhase::Plaintext); + let phase_changed = true; + let ready = false; + + // The selector's condition: a round must be created for a phase change, not only when + // inputs are ready. + if ready || phase_changed { + reconcile_phase(&mut state, &id, phase, 100, &policy()); + } + + let round = state.rounds.get(&id).expect( + "a phase change must create a failover round; without one an aggregator that dies \ + before aggregation inputs are ready is never replaced", + ); + assert_eq!(round.phase, AggregatorPhase::Plaintext); + assert_eq!( + round.deadline_unix_secs, 160, + "the round must carry the full budget from the phase start" + ); + + // With the active party bound, the deadline can now promote a standby. + reconcile_active_party(&mut state, &id, Some(0), 100, &policy()); + let decision = apply_due_timeout( + &mut state, + &id, + ExpectedFailoverDeadline { + phase: AggregatorPhase::Plaintext, + unix_secs: 160, + }, + 160, + &policy(), + &committee(), + &[], + ); + assert!( + matches!(decision, FailoverDecision::Promote { .. }), + "an aggregator that stops making progress during collection must be replaced; got \ + {decision:?}" + ); + } } diff --git a/packages/interfold-contracts/deployed_contracts.json b/packages/interfold-contracts/deployed_contracts.json index 30ed2437e0..86b6cf8db8 100644 --- a/packages/interfold-contracts/deployed_contracts.json +++ b/packages/interfold-contracts/deployed_contracts.json @@ -384,231 +384,5 @@ "implementationAddress": "0x4FF6e77A10E8f06C11a4DD2A71b6AB55394640e4" } } - }, - "localhost": { - "PoseidonT3": { - "blockNumber": 6, - "address": "0x3333333C0A88F9BE4fd23ed0536F9B6c427e3B93" - }, - "MockUSDC": { - "constructorArgs": { - "initialSupply": "1000000" - }, - "blockNumber": 7, - "address": "0xe7f1725E7734CE288F8367e1Bb143E90bb3F0512" - }, - "InterfoldTicketToken": { - "constructorArgs": { - "baseToken": "0xe7f1725E7734CE288F8367e1Bb143E90bb3F0512", - "registry": "0x0000000000000000000000000000000000000001", - "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" - }, - "blockNumber": 8, - "address": "0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0" - }, - "SlashingEvidenceLib": { - "blockNumber": 9, - "address": "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" - }, - "SlashingManager": { - "constructorArgs": { - "initialDelay": "172800", - "admin": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" - }, - "libraries": { - "SlashingEvidenceLib": "0xCf7Ed3AccA5a467e9e704C703E8D87F634fB0Fc9" - }, - "blockNumber": 10, - "address": "0xDc64a140Aa3E981100a9becA4E685f962f0cF6C9" - }, - "CiphernodeRegistryOwnable": { - "constructorArgs": { - "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", - "submissionWindow": "60" - }, - "proxyRecords": { - "initData": "0xcd6dc687000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb92266000000000000000000000000000000000000000000000000000000000000003c", - "initialOwner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", - "proxyAddress": "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853", - "proxyAdminAddress": "0x9bd03768a7DCc129555dE410FF8E85528A4F88b5", - "implementationAddress": "0x0165878A594ca255338adfa4d48449f69242Eb8F" - }, - "libraries": { - "PoseidonT3": "0x3333333C0A88F9BE4fd23ed0536F9B6c427e3B93", - "RegistrySortitionLib": "0x5FC8d32690cc91D4c39d9d3abcBD16989F875707" - }, - "blockNumber": 12, - "address": "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853" - }, - "RegistrySortitionLib": { - "address": "0x5FC8d32690cc91D4c39d9d3abcBD16989F875707", - "blockNumber": 12 - }, - "MockRandomnessProvider": { - "constructorArgs": { - "requester": "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853" - }, - "address": "0x2279B7A0a67DB372996a5FaB50D91eAA73d2eBe6", - "blockNumber": 14 - }, - "BondingAssetLib": { - "address": "0x8A791620dd6260079BF849Dc5567aDC3F2FdC318", - "blockNumber": 14 - }, - "BondingEligibilityLib": { - "address": "0x610178dA211FEF7D417bC0e6FeD39F05609AD788", - "blockNumber": 14 - }, - "BondingSlashingLib": { - "address": "0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e", - "blockNumber": 14 - }, - "BondingRegistrationLib": { - "address": "0xA51c1fc2f0D1a1b8494Ed1FE312d7C3a78Ed91C0", - "blockNumber": 14 - }, - "BondingOwnershipLib": { - "address": "0x0DCd1Bf9A1b36cE34237eEaFef220932846BCD82", - "blockNumber": 14 - }, - "BondingRegistry": { - "constructorArgs": { - "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", - "ticketToken": "0x9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0", - "ciphernodeBondToken": "0x0000000000000000000000000000000000000000", - "registry": "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853", - "slashedFundsTreasury": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", - "ticketPrice": "10000000", - "requiredCiphernodeBond": "100000000000000000000", - "ticketTokenDecimals": "6", - "ciphernodeBondTokenDecimals": "0", - "minTicketBalance": "1", - "exitDelay": "604800" - }, - "libraries": { - "BondingAssetLib": "0x8A791620dd6260079BF849Dc5567aDC3F2FdC318", - "BondingEligibilityLib": "0x610178dA211FEF7D417bC0e6FeD39F05609AD788", - "BondingSlashingLib": "0xB7f8BC63BbcaD18155201308C8f3540b07f84F5e", - "BondingRegistrationLib": "0xA51c1fc2f0D1a1b8494Ed1FE312d7C3a78Ed91C0", - "BondingOwnershipLib": "0x0DCd1Bf9A1b36cE34237eEaFef220932846BCD82" - }, - "proxyRecords": { - "initData": "0x571f8259000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb922660000000000000000000000009fe46736679d2d9a65f0992f2272de9f3c7fa6e0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009896800000000000000000000000000000000000000000000000056bc75e2d6310000000000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000a513e6e4b8f2a923d98304ec87f64353c4d5c853000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb9226600000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000093a80", - "initialOwner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", - "proxyAddress": "0x0B306BF915C4d645ff596e518fAf3F9669b97016", - "proxyAdminAddress": "0x524F04724632eED237cbA3c37272e018b3A7967e", - "implementationAddress": "0x9A676e781A523b5d0C0e43731313A708CB607508" - }, - "blockNumber": 14, - "address": "0x0B306BF915C4d645ff596e518fAf3F9669b97016" - }, - "InterfoldToken": { - "constructorArgs": { - "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", - "ccaStart": "1788941401", - "ccaEnd": "1789546201", - "noMoreLocks": "1921738201", - "bondingRegistry": "0x0B306BF915C4d645ff596e518fAf3F9669b97016" - }, - "blockNumber": 23, - "address": "0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE" - }, - "BondedCheckpoints": { - "constructorArgs": { - "registry": "0x0B306BF915C4d645ff596e518fAf3F9669b97016" - }, - "blockNumber": 26, - "address": "0x59b670e9fA9D0A427751Af201D676719a970857b" - }, - "BondedVotes": { - "constructorArgs": { - "token": "0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE", - "votesSource": "0x9A9f2CCfdE556A7E9Ff0848998Aa4a0CFD8863AE", - "checkpoints": "0x59b670e9fA9D0A427751Af201D676719a970857b" - }, - "blockNumber": 27, - "address": "0x4ed7c70F96B99c776995fB64377f0d4aB3B0e1C1" - }, - "MockComputeProvider": { - "blockNumber": 29, - "address": "0xa85233C63b9Ee964Add6F2cffe00Fd84eb32338f" - }, - "MockDecryptionVerifier": { - "blockNumber": 30, - "address": "0x4A679253410272dd5232B3Ff7cF5dbB88f295319" - }, - "MockCiphertextVerifier": { - "address": "0x7a2088a1bFc9d81c55368AE168C2C02570cB814F", - "blockNumber": 31 - }, - "MockPkVerifier": { - "blockNumber": 32, - "address": "0x09635F643e140090A9A8Dcd712eD6285858ceBef" - }, - "MockE3Program": { - "blockNumber": 33, - "address": "0xc5a5C42992dECbae36851359345FE25997F5C42d" - }, - "InterfoldPricing": { - "address": "0x67d269191c92Caf3cD7723F116c85e6E9bf55933", - "blockNumber": 36 - }, - "InterfoldLifecycle": { - "address": "0xE6E340D132b5f46d1e472DebcD681B2aBc16e57E", - "blockNumber": 36 - }, - "Interfold": { - "constructorArgs": { - "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", - "registry": "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853", - "bondingRegistry": "0x0B306BF915C4d645ff596e518fAf3F9669b97016", - "e3RefundManager": "0x0000000000000000000000000000000000000001", - "feeToken": "0xe7f1725E7734CE288F8367e1Bb143E90bb3F0512", - "feeTokenDecimals": "6", - "maxDuration": "2592000", - "timeoutConfig": "{\"dkgWindow\":7200,\"computeWindow\":86400,\"decryptionWindow\":3600}", - "randomnessFlatFee": "1000000", - "pricingConfig": "{\"keyGenFixedPerNode\":\"100000\",\"keyGenPerEncryptionProof\":\"50000\",\"coordinationPerPair\":\"10000\",\"availabilityPerNodePerSec\":\"50\",\"decryptionPerNode\":\"300000\",\"publicationBase\":\"1000000\",\"verificationPerProof\":\"5000\",\"protocolTreasury\":\"0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266\",\"marginBps\":\"1000\",\"protocolShareBps\":\"0\",\"dkgUtilizationBps\":\"2500\",\"computeUtilizationBps\":\"5000\",\"decryptUtilizationBps\":\"2500\",\"minCommitteeSize\":\"0\",\"minThreshold\":\"0\",\"randomnessFlatFee\":\"1000000\"}", - "initialE3Program": "0xc5a5C42992dECbae36851359345FE25997F5C42d" - }, - "libraries": { - "InterfoldLifecycle": "0xE6E340D132b5f46d1e472DebcD681B2aBc16e57E", - "InterfoldPricing": "0x67d269191c92Caf3cD7723F116c85e6E9bf55933" - }, - "proxyRecords": { - "initData": "0x9de48cc5000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb92266000000000000000000000000a513e6e4b8f2a923d98304ec87f64353c4d5c8530000000000000000000000000b306bf915c4d645ff596e518faf3f9669b970160000000000000000000000000000000000000000000000000000000000000001000000000000000000000000e7f1725e7734ce288f8367e1bb143e90bb3f0512000000000000000000000000000000000000000000000000000000000000000600000000000000000000000000000000000000000000000000000000000186a0000000000000000000000000000000000000000000000000000000000000c3500000000000000000000000000000000000000000000000000000000000002710000000000000000000000000000000000000000000000000000000000000003200000000000000000000000000000000000000000000000000000000000493e000000000000000000000000000000000000000000000000000000000000f42400000000000000000000000000000000000000000000000000000000000001388000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb9226600000000000000000000000000000000000000000000000000000000000003e8000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009c4000000000000000000000000000000000000000000000000000000000000138800000000000000000000000000000000000000000000000000000000000009c40000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f42400000000000000000000000000000000000000000000000000000000000278d000000000000000000000000000000000000000000000000000000000000001c2000000000000000000000000000000000000000000000000000000000000151800000000000000000000000000000000000000000000000000000000000000e10000000000000000000000000c5a5c42992decbae36851359345fe25997f5c42d", - "initialOwner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", - "proxyAddress": "0x84eA74d481Ee0A5332c457a4d796187F6Ba67fEB", - "proxyAdminAddress": "0x212fdfCfCC22db97DeB3AC3260414909282BB4EE", - "implementationAddress": "0xc3e53F4d16Ae77Db1c982e75a937B9f60FE63690" - }, - "blockNumber": 36, - "address": "0x84eA74d481Ee0A5332c457a4d796187F6Ba67fEB" - }, - "E3RefundManager": { - "constructorArgs": { - "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", - "interfold": "0x84eA74d481Ee0A5332c457a4d796187F6Ba67fEB", - "treasury": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" - }, - "proxyRecords": { - "initData": "0xc0c53b8b000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb9226600000000000000000000000084ea74d481ee0a5332c457a4d796187f6ba67feb000000000000000000000000f39fd6e51aad88f6f4ce6ab8827279cfffb92266", - "initialOwner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", - "proxyAddress": "0xa82fF9aFd8f496c3d6ac40E2a0F282E47488CFc9", - "proxyAdminAddress": "0x6A358FD7B7700887b0cd974202CdF93208F793E2", - "implementationAddress": "0x9E545E3C0baAB3E08CdfD552C960A1050f373042" - }, - "blockNumber": 38, - "address": "0xa82fF9aFd8f496c3d6ac40E2a0F282E47488CFc9" - }, - "NodeReleaseRegistry": { - "address": "0x4826533B4897376654Bb4d4AD88B7faFD0C98528", - "blockNumber": 46, - "constructorArgs": { - "owner": "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266", - "bondingRegistry": "0x0B306BF915C4d645ff596e518fAf3F9669b97016", - "ciphernodeRegistry": "0xa513E6E4b8f2a923D98304ec87F64353C4D5C853" - } - } } -} \ No newline at end of file +}