From 03e50744f2c669f85b4cbb1a3ce2ac9eda71a931 Mon Sep 17 00:00:00 2001 From: samliok Date: Thu, 13 Aug 2026 18:11:53 -0400 Subject: [PATCH 1/6] Refactor instance: extract util helpers, split BlockDeserializer from VM, rework epoch transitions Extract LastBlock and epoch/validator-set derivation into util.go with tests. Split ParseBlock into a BlockDeserializer interface and move ICM epoch computation to a Config field. Replace EpochAwareStorage with InstanceStorage, which skips Telocks by block type and exposes an onIndex hook. Signal epoch changes via EpochConfig.OnSealingBlockIndex for validators and nonvalidator.Config.TransitionToValidator for non-validators instead of the storage wrapper. Rework processEpochChange to honor Stop for validators and notifyEpochChange to keep only the newest pending change. --- adapters.go | 72 ++++--- config.go | 5 +- instance.go | 381 ++++++++++++---------------------- instance_test.go | 28 +-- nonvalidator/non_validator.go | 12 ++ simplex/epoch.go | 5 + testutil/controlled.go | 14 +- testutil/node.go | 2 +- util.go | 111 ++++++++++ util_test.go | 198 ++++++++++++++++++ 10 files changed, 518 insertions(+), 310 deletions(-) create mode 100644 util.go create mode 100644 util_test.go diff --git a/adapters.go b/adapters.go index 1f8512be..f6b6d35b 100644 --- a/adapters.go +++ b/adapters.go @@ -19,6 +19,15 @@ type Communication struct { Broadcaster } +func newCommunication(sender Sender, broadcaster Broadcaster, validators common.Nodes) *Communication { + c := &Communication{ + Sender: sender, + Broadcaster: broadcaster, + } + c.SetValidators(validators) + return c +} + func (c *Communication) SetValidators(nodes common.Nodes) { c.nodes.Store(nodes) } @@ -31,45 +40,52 @@ func (c *Communication) Validators() common.Nodes { return nodes } -// EpochAwareStorage is a wrapper around Storage that is aware of epoch changes. -// Upon an epoch change, it will ignore blocks from previous epochs -// and will call the onEpochChange callback when a new epoch is detected. -type EpochAwareStorage struct { - msm *metadata.StateMachine - onEpochChange func(seq uint64, validators common.Nodes) error +// InstanceStorage is a wrapper around Storage that skips indexing Telocks +// and delegates post-index handling to a caller-provided onIndex hook. +type InstanceStorage struct { Storage - epoch uint64 + + msm *metadata.StateMachine + + onIndex func(block *ParsedBlock) error +} + +func NewInstanceStorage(storage Storage, msm *metadata.StateMachine, onIndex func(block *ParsedBlock) error) *InstanceStorage { + return &InstanceStorage{ + Storage: storage, + msm: msm, + onIndex: onIndex, + } } -func (e *EpochAwareStorage) Retrieve(seq uint64) (common.VerifiedBlock, common.Finalization, error) { - block, finalization, err := e.GetBlock(seq) +func (s *InstanceStorage) Retrieve(seq uint64) (common.VerifiedBlock, common.Finalization, error) { + block, finalization, err := s.GetBlock(seq) if err != nil { return nil, common.Finalization{}, err } parsedBlock := &ParsedBlock{ - msm: e.msm, + msm: s.msm, StateMachineBlock: block, } return parsedBlock, *finalization, nil } -func (e *EpochAwareStorage) Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error { - if block.BlockHeader().Epoch < e.epoch { - // This is a Telock from a previous epoch, so we ignore it and do not index it. +func (s *InstanceStorage) Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error { + pb, ok := block.(*ParsedBlock) + if !ok { + return fmt.Errorf("expected ParsedBlock, got %T", block) + } + + // A Telock only extends time until the epoch transition finalizes, so we never index it. + if pb.Type() == metadata.BlockTypeTelock { return nil } - if err := e.Storage.Index(ctx, block, certificate); err != nil { + + if err := s.Storage.Index(ctx, block, certificate); err != nil { return err } - // This is a sealing block, and it is not the zero block - if block.SealingBlockInfo() != nil && block.SealingBlockInfo().PrevSealingBlockHash != [32]byte{} { - if err := e.onEpochChange(block.BlockHeader().Seq, block.SealingBlockInfo().ValidatorSet); err != nil { - return err - } - // We are now in a new epoch, so we update the epoch number to prevent indexing Telocks from the previous epoch. - e.epoch = block.BlockHeader().Seq - } - return nil + + return s.onIndex(pb) } // cachedBlock is a wrapper around ParsedBlock that caches the block in the CachedStorage upon verification. @@ -229,17 +245,17 @@ func (bw *BlockBuilderWaiter) BuildBlock(ctx context.Context, metadata common.Pr } type blockDeserializer struct { - vm VM - msm *metadata.StateMachine + deserializer BlockDeserializer + msm *metadata.StateMachine } -func (bp *blockDeserializer) DeserializeBlock(ctx context.Context, bytes []byte) (common.Block, error) { +func (bd *blockDeserializer) DeserializeBlock(ctx context.Context, bytes []byte) (common.Block, error) { var rawBlock metadata.RawBlock if err := rawBlock.UnmarshalCanoto(bytes); err != nil { return nil, err } - block, err := bp.vm.ParseBlock(ctx, rawBlock.InnerBlockBytes) + block, err := bd.deserializer.ParseBlock(ctx, rawBlock.InnerBlockBytes) if err != nil { return nil, err } @@ -248,6 +264,6 @@ func (bp *blockDeserializer) DeserializeBlock(ctx context.Context, bytes []byte) InnerBlock: block, Metadata: rawBlock.Metadata, }, - msm: bp.msm, + msm: bd.msm, }, nil } diff --git a/config.go b/config.go index a47474f5..1aba21aa 100644 --- a/config.go +++ b/config.go @@ -53,12 +53,11 @@ type VM interface { // WaitForPendingBlock returns when either the given context is cancelled, // or when the VM signals that a block should be built. WaitForPendingBlock(ctx context.Context) +} +type BlockDeserializer interface { // ParseBlock parses the given block bytes into a VMBlock. ParseBlock(context.Context, []byte) (avalanchego.VMBlock, error) - - // ComputeICMEpoch computes the ICM epoch transition given the input parameters. - ComputeICMEpoch(input metadata.ICMEpochInput) metadata.ICMEpochInfo } type Storage interface { diff --git a/instance.go b/instance.go index 7dc61abe..c0ebd0ac 100644 --- a/instance.go +++ b/instance.go @@ -5,6 +5,7 @@ package simplex import ( "context" + "errors" "fmt" "math" "sync" @@ -19,6 +20,8 @@ import ( "go.uber.org/zap" ) +var errAlreadyStarted = errors.New("instance already started") + const ( // tickInterval is the interval at which the instance will call AdvanceTime on the current epoch or non-validator. tickInterval = time.Millisecond * 100 @@ -26,7 +29,7 @@ const ( type Config struct { // LastNonSimplexInnerBlock is the last non-simplex inner block that was persisted to storage. - // This is used to determine the current epoch and validator set. + // The genesis validator state from pchain is used to determine the current epoch and validator set. Can be the genesis block LastNonSimplexInnerBlock avalanchego.VMBlock // ParameterConfig is the configuration for the simplex instance. ParameterConfig ParameterConfig @@ -34,38 +37,35 @@ type Config struct { PlatformChain PlatformChain // Broadcaster is the interface to broadcast messages to other nodes in the network. Broadcaster Broadcaster + // Sender is an interface to send messages to a specific node in the network + Sender Sender // CryptoOps is the interface to the cryptographic operations needed by the simplex instance. CryptoOps CryptoOps // WalCreator is the interface to create new write-ahead logs for the simplex instance. WalCreator wal.Creator // Storage is the interface to the block storage layer for the simplex instance. - Storage Storage - Logger common.Logger - Sender Sender - WALs []wal.DeletableWAL - VM VM - ID common.NodeID + Storage Storage + Logger common.Logger + WALs []wal.DeletableWAL + VM VM + ICMETransition metadata.ICMEpochTransition + BlockDeserializer BlockDeserializer + ID common.NodeID } -type nodeRole byte - -const ( - nonValidator nodeRole = iota - validator -) - type epochChange struct { - epochNum uint64 + epoch uint64 validators common.Nodes - nodeRole nodeRole } +func noopOnIndex(*ParsedBlock) error { return nil } type timeAdvancer interface { AdvanceTime(t time.Time) } type Instance struct { - Config Config + Config Config + lock sync.Mutex started bool cs *CachedStorage @@ -82,8 +82,8 @@ func NewInstance(config Config) *Instance { return &Instance{ Config: config, stopCh: make(chan struct{}), - cs: NewCachedStorage(config.Storage), epochChanges: make(chan epochChange, 1), + cs: NewCachedStorage(config.Storage), } } @@ -93,21 +93,14 @@ func (i *Instance) Start(ctx context.Context) error { defer i.lock.Unlock() if i.started { - return fmt.Errorf("instance already started") + return errAlreadyStarted } i.started = true context.AfterFunc(ctx, i.Stop) - lastBlock, numBlocks, err := i.lastBlock() - if err != nil { - return fmt.Errorf("error retrieving last block: %w", err) - } - - lastNonSimplexHeight := i.Config.LastNonSimplexInnerBlock.Height() - genesisValidatorSet := i.Config.PlatformChain.GenesisValidatorSet() - nodes, epochNum, err := constructEpochAndValidatorSet(i.Config.Logger, lastNonSimplexHeight, genesisValidatorSet, numBlocks, &ParsedBlock{StateMachineBlock: lastBlock}, i.Config.Storage) + nodes, epochNum, err := getLastAcceptedEpochAndValidatorSet(&i.Config) if err != nil { return fmt.Errorf("error determining latest epoch and validator set: %w", err) } @@ -122,16 +115,26 @@ func (i *Instance) Start(ctx context.Context) error { return nil } -func (i *Instance) startValidator() error { - epochConfig, err := i.createEpochConfig() +func (i *Instance) startValidator(epochNum uint64, validators common.Nodes) error { + epochConfig, err := i.createEpochConfig(epochNum, validators) if err != nil { return err } - return i.startEpoch(epochConfig) + + epoch, err := simplex.NewEpoch(epochConfig) + if err != nil { + return fmt.Errorf("error creating simplex epoch: %w", err) + } + + epoch.Epoch = epochConfig.Epoch + i.e = epoch + i.epochOrNV = epoch + + return epoch.Start() } -func (i *Instance) startNonValidator(epochNum uint64, validators common.Nodes) error { - config, err := i.createNonValidatorConfig(epochNum, validators) +func (i *Instance) startNonValidator() error { + config, err := i.createNonValidatorConfig() if err != nil { return err } @@ -146,35 +149,17 @@ func (i *Instance) startNonValidator(epochNum uint64, validators common.Nodes) e return nil } -func (i *Instance) createNonValidatorConfig(epochNum uint64, validators common.Nodes) (nonvalidator.Config, error) { +func (i *Instance) createNonValidatorConfig() (nonvalidator.Config, error) { source, err := simplex.NewRandomSource() if err != nil { return nonvalidator.Config{}, err } - comm := &Communication{Sender: i.Config.Sender, Broadcaster: i.Config.Broadcaster} - comm.SetValidators(validators) - - epochAwareStorage := &EpochAwareStorage{ - epoch: epochNum, - Storage: i.Config.Storage, - onEpochChange: func(epoch uint64, validators common.Nodes) error { - height := i.Config.PlatformChain.GetCurrentHeight() - vdrs, err := i.Config.PlatformChain.GetValidatorSet(height) - if err != nil { - i.Config.Logger.Error("error getting validator set", zap.Error(err)) - return fmt.Errorf("error getting validator set from platform chain: %w", err) - } - comm.SetValidators(validators) - if i.iAmValidator(vdrs.Nodes()) { - i.notifyEpochChange(epoch, validators, nonValidator) - } else { - i.Config.Logger.Debug("I am still a non-validator at the tip of the P-chain, skipping role change", - zap.Uint64("height", height)) - } - return nil - }, + nodes, err := GetHighestValidatorSet(i.Config.PlatformChain) + if err != nil { + return nonvalidator.Config{}, err } + comm := newCommunication(i.Config.Sender, i.Config.Broadcaster, nodes) // Plant an artificial MSM. A non-validator never verifies the state machine transition, // it only verifies the inner block (see common.OnlyVMVerifyOpt), so this MSM is only @@ -183,30 +168,41 @@ func (i *Instance) createNonValidatorConfig(epochNum uint64, validators common.N Config: &metadata.Config{}, } i.cs.msm = i.msm + instanceStorage := NewInstanceStorage(i.cs, i.msm, noopOnIndex) config := nonvalidator.Config{ ID: i.Config.ID, RandomSource: source, - Storage: epochAwareStorage, + Storage: instanceStorage, Comm: comm, Logger: i.Config.Logger, StartTime: time.Now(), SignatureAggregatorCreator: i.Config.CryptoOps.CreateSignatureAggregator, MaxSequenceWindow: simplex.DefaultMaxRoundWindow, + TransitionToValidator: i.notifyEpochChange, } return config, nil } -func (i *Instance) notifyEpochChange(epoch uint64, validators common.Nodes, role nodeRole) { - select { - case i.epochChanges <- epochChange{ - epochNum: epoch, +func (i *Instance) notifyEpochChange(epoch uint64, validators common.Nodes) { + ec := epochChange{ + epoch: epoch, validators: validators, - nodeRole: role, - }: - case <-i.stopCh: - // If the instance is stopped, we don't need to notify about epoch changes. - return + } + + for { + select { + case i.epochChanges <- ec: + return + // The slot holds a stale epoch change: take it, keep the newer of the two and retry. + case pending := <-i.epochChanges: + if pending.epoch > ec.epoch { + ec = pending + } + case <-i.stopCh: + // If the instance is stopped, we don't need to notify about epoch changes. + return + } } } @@ -248,7 +244,7 @@ func (i *Instance) Stop() { close(i.stopCh) } - i.stopValidator() + i.stopValidator(false) i.stopNonValidator() } @@ -260,9 +256,18 @@ func (i *Instance) stopNonValidator() { } } -func (i *Instance) stopValidator() { +func (i *Instance) stopValidator(garbageCollectWAL bool) { if i.e != nil { i.e.Stop() + // Wipe out the WALs from the config so we won't try to load them again + if garbageCollectWAL { + i.Config.WALs = nil + // On epoch change, garbage collect the WAL to remove all entries from previous epochs. + if err := i.wal.GarbageCollect(math.MaxUint64); err != nil { + i.Config.Logger.Error("Error garbage collecting epoch config on epoch change", zap.Error(err)) + } + } + i.e = nil i.epochOrNV = nil } @@ -362,73 +367,40 @@ func (i *Instance) listenForEpochChanges() { } func (i *Instance) processEpochChange(epochChange epochChange) { - var err error - switch epochChange.nodeRole { - case nonValidator: - err = i.transitionEpochNonValidator(epochChange) - case validator: - err = i.transitionEpochValidator(epochChange) - default: // This should never happen, but we log it just in case. - i.Config.Logger.Fatal("Unknown node role on epoch change", - zap.String("role", fmt.Sprintf("%v", epochChange.nodeRole))) - return - } - if err != nil { - i.Config.Logger.Error("Error transitioning epoch", zap.Uint8("role", uint8(epochChange.nodeRole)), zap.Error(err)) - i.Stop() - } -} + // Hold the lock so the transition cannot interleave with Stop or HandleMessage. + i.lock.Lock() -// startEpoch starts a new epoch with the given configuration. -// Must be called under the lock, and assumes that the previous epoch has been stopped (if any). -func (i *Instance) startEpoch(epochConfig simplex.EpochConfig) error { - epoch, err := simplex.NewEpoch(epochConfig) - if err != nil { - return fmt.Errorf("error creating simplex epoch: %w", err) + if i.isStopped() { + i.lock.Unlock() + i.Config.Logger.Info("instance is already stopped, skipping epoch change") + return } - epoch.Epoch = epochConfig.Epoch - i.e = epoch - i.epochOrNV = epoch - return epoch.Start() -} + var err error -func (i *Instance) lastBlock() (metadata.StateMachineBlock, uint64, error) { - numBlocks := i.Config.Storage.NumBlocks() - if numBlocks == 0 { - return metadata.StateMachineBlock{}, 0, fmt.Errorf("no genesis block found in storage") + switch { + case i.nv != nil: + // Stop the non-validator before doing anything else, so that we don't process any more messages while we are changing epochs. + i.stopNonValidator() + err = i.startAtEpoch(epochChange.validators, epochChange.epoch) + case i.e != nil: + i.stopValidator(true) + + err = i.startAtEpoch(epochChange.validators, epochChange.epoch) + default: // This should never happen, but we log it just in case. + i.lock.Unlock() + i.Config.Logger.Fatal("We are not running either a validator or non-validator") + return } + i.lock.Unlock() - lastBlock, _, err := i.Config.Storage.GetBlock(numBlocks - 1) if err != nil { - return metadata.StateMachineBlock{}, 0, fmt.Errorf("error retrieving last block from storage: %w", err) - } - - return lastBlock, numBlocks, nil -} - -func (i *Instance) iAmValidator(nodes common.Nodes) bool { - for _, node := range nodes { - if i.Config.ID.Equals(node.Id) { - return true - } + i.Config.Logger.Error("Error transitioning epoch", zap.Error(err)) + i.Stop() } - return false } -func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { - lastBlock, numBlocks, err := i.lastBlock() - if err != nil { - return simplex.EpochConfig{}, err - } - - lastNonSimplexHeight := i.Config.LastNonSimplexInnerBlock.Height() - genesisValidatorSet := i.Config.PlatformChain.GenesisValidatorSet() - nodes, epochNum, err := constructEpochAndValidatorSet(i.Config.Logger, lastNonSimplexHeight, genesisValidatorSet, numBlocks, &ParsedBlock{StateMachineBlock: lastBlock}, i.Config.Storage) - if err != nil { - return simplex.EpochConfig{}, err - } - +func (i *Instance) createEpochConfig(epoch uint64, validators common.Nodes) (simplex.EpochConfig, error) { wal, err := wal.NewGarbageCollectedWAL(i.Config.WALs, i.Config.WalCreator, &common.WALRetentionReader{}, i.Config.ParameterConfig.WALMaxEntryCount) if err != nil { return simplex.EpochConfig{}, fmt.Errorf("error creating garbage collected wal: %w", err) @@ -438,7 +410,7 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { // We might have crashed right after a sealing block was persisted to storage, // but before the WAL was garbage collected. // In that case, we need to garbage collect the WAL to remove all entries from previous epochs. - if err := i.maybeGarbageCollectWAL(lastBlock); err != nil { + if err := i.maybeGarbageCollectWAL(); err != nil { return simplex.EpochConfig{}, err } @@ -453,7 +425,7 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { MaxBlockBuildingWaitTime: i.Config.ParameterConfig.MaxNetworkDelay, Logger: i.Config.Logger, Signer: i.Config.CryptoOps, - GenesisValidatorSet: genesisValidatorSet, + GenesisValidatorSet: i.Config.PlatformChain.GenesisValidatorSet(), LastNonSimplexBlockPChainHeight: i.Config.PlatformChain.LastNonSimplexBlockPChainHeight(), SignatureAggregatorCreator: i.Config.CryptoOps.CreateSignatureAggregator, BlockBuilder: i.Config.VM, @@ -461,7 +433,7 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { GetPChainHeightForProposing: i.Config.PlatformChain.GetMinimumHeight, GetPChainHeightForVerifying: i.Config.PlatformChain.GetCurrentHeight, AuxiliaryInfoApp: &NoopAuxiliaryInfoApp{}, - ComputeICMEpoch: i.Config.VM.ComputeICMEpoch, + ComputeICMEpoch: i.Config.ICMETransition, GetBlock: i.cs.RetrieveBlock, }) if err != nil { @@ -478,26 +450,19 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { blockBuilder := &BlockBuilderWaiter{vm: i.Config.VM, msm: msm} - comm := &Communication{Sender: i.Config.Sender, Broadcaster: i.Config.Broadcaster} - comm.SetValidators(nodes) - - epochAwareStorage := &EpochAwareStorage{ - msm: msm, - epoch: epochNum, - Storage: i.cs, - onEpochChange: func(epoch uint64, validators common.Nodes) error { - blockBuilder.stop() - comm.SetValidators(validators) - i.notifyEpochChange(epoch, validators, validator) - return nil - }, - } + comm := newCommunication(i.Config.Sender, i.Config.Broadcaster, validators) + + instanceStorage := NewInstanceStorage(i.cs, msm, noopOnIndex) + onEpochChange := func(epoch uint64, validators common.Nodes) { + blockBuilder.stop() + i.notifyEpochChange(epoch, validators) + } epochConfig := simplex.EpochConfig{ - Epoch: epochNum, + Epoch: epoch, ReplicationEnabled: true, StartTime: time.Now(), - // TODO: For simpicity, we use the same value for all timeouts. If needed we can expand the config. + // TODO: For simplicity, we use the same value for all timeouts. If needed we can expand the config. MaxProposalWait: i.Config.ParameterConfig.MaxNetworkDelay * 2, // 1 proposal + 1 vote MaxRebroadcastWait: i.Config.ParameterConfig.MaxNetworkDelay * 2, FinalizeRebroadcastTimeout: i.Config.ParameterConfig.MaxNetworkDelay * 2, @@ -510,15 +475,21 @@ func (i *Instance) createEpochConfig() (simplex.EpochConfig, error) { QCDeserializer: i.Config.CryptoOps, Signer: i.Config.CryptoOps, Verifier: i.Config.CryptoOps, - Storage: epochAwareStorage, + Storage: instanceStorage, Comm: comm, BlockBuilder: blockBuilder, - BlockDeserializer: &blockDeserializer{vm: i.Config.VM, msm: msm}, + BlockDeserializer: &blockDeserializer{deserializer: i.Config.BlockDeserializer, msm: msm}, + OnSealingBlockIndex: onEpochChange, } return epochConfig, nil } -func (i *Instance) maybeGarbageCollectWAL(lastBlock metadata.StateMachineBlock) error { +func (i *Instance) maybeGarbageCollectWAL() error { + lastBlock, _, err := LastBlock(i.Config.Storage) + if err != nil { + return fmt.Errorf("error retrieving last block: %w", err) + } + if lastBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor != nil { i.Config.Logger.Info("Last block is a sealing block, garbage collecting all WALs preceding it to start a new epoch") // We figure out the round number of the latest block and garbage collect all WALs preceding it. @@ -533,119 +504,21 @@ func (i *Instance) maybeGarbageCollectWAL(lastBlock metadata.StateMachineBlock) return nil } -func (i *Instance) transitionEpochNonValidator(epochChange epochChange) error { - i.lock.Lock() - defer i.lock.Unlock() - - if i.isStopped() { - i.Config.Logger.Info("instance is already stopped, skipping epoch change") - return nil - } - - if !i.iAmValidator(epochChange.validators) { - i.Config.Logger.Debug("Skipping restarting a non-validator because I am not a validator yet") - return nil - } - - // Stop the non-validator before doing anything else, so that we don't process any more messages while we are changing epochs. - i.stopNonValidator() - - return i.startAtEpoch(epochChange.validators, epochChange.epochNum) -} - +// startAtEpoch starts either a validator or non-validator at `epoch“. func (i *Instance) startAtEpoch(validators common.Nodes, epoch uint64) error { - if i.iAmValidator(validators) { - if err := i.startValidator(); err != nil { - i.Config.Logger.Error("Error starting validator on epoch change", zap.Error(err)) - return err - } - return nil + if validators.Contains(i.Config.ID) { + return i.startValidator(epoch, validators) } - if err := i.startNonValidator(epoch, validators); err != nil { - i.Config.Logger.Error("Error starting non-validator on epoch change", zap.Error(err)) - return err - } - return nil + return i.startNonValidator() } -func (i *Instance) transitionEpochValidator(epochChange epochChange) error { - i.lock.Lock() - defer i.lock.Unlock() - - // Stop the epoch before doing anything else, so that we don't process any more messages while we are changing epochs. - i.stopValidator() - // Wipe out the WALs from the config so we won't try to load them again - i.Config.WALs = nil - // On epoch change, garbage collect the WAL to remove all entries from previous epochs. - if err := i.wal.GarbageCollect(math.MaxUint64); err != nil { - i.Config.Logger.Error("Error garbage collecting epoch config on epoch change", zap.Error(err)) - } - - return i.startAtEpoch(epochChange.validators, epochChange.epochNum) -} - -func constructEpochAndValidatorSet(logger common.Logger, lastNonSimplexInnerBlockHeight uint64, genesisValidatorSet metadata.NodeBLSMappings, numBlocks uint64, lastBlock *ParsedBlock, storage Storage) (common.Nodes, uint64, error) { - epochNum := lastBlock.BlockHeader().Epoch - - var validatorSet metadata.NodeBLSMappings - var nodes common.Nodes - - switch { - // If all we have in the ledger is non-Simplex blocks, load the validator set from genesis - case lastNonSimplexInnerBlockHeight+1 == numBlocks: - nodes = validatorSetToNodes(genesisValidatorSet) - epochNum = lastNonSimplexInnerBlockHeight + 1 - logger.Debug("Determined epoch and validator set from genesis (ledger holds only non-Simplex blocks)", - zap.Uint64("epoch", epochNum)) - // If the last block persisted is a sealing block, then we are in the next epoch. - case lastBlock.SealingBlockInfo() != nil: - epochNum = lastBlock.BlockHeader().Seq - nodes = lastBlock.SealingBlockInfo().ValidatorSet - logger.Debug("Determined epoch and validator set from sealing block at tip", - zap.Uint64("epoch", epochNum)) - // Else, we have at least one Simplex block in the ledger, and it's not a sealing block. - default: - // Therefore, the sequence of the sealing block is the epoch number. - sealingBlockSeq := lastBlock.BlockHeader().Epoch - sealingBlock, _, err := storage.GetBlock(sealingBlockSeq) - if err != nil { - return nil, 0, fmt.Errorf("error retrieving sealing block from storage: %w", err) - } - if sealingBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor == nil { - return nil, 0, fmt.Errorf("expected sealing block at seq %d, but got a non-sealing block", sealingBlockSeq) - } - validatorSet = constructValidatorSetFromSealingBlock(&ParsedBlock{StateMachineBlock: sealingBlock}) - nodes = validatorSetToNodes(validatorSet) - logger.Debug("Determined epoch and validator set from sealing block in storage", - zap.Uint64("epoch", epochNum), zap.Uint64("sealingBlockSeq", sealingBlockSeq)) +func GetHighestValidatorSet(platform PlatformChain) (common.Nodes, error) { + height := platform.GetCurrentHeight() + mappings, err := platform.GetValidatorSet(height) + if err != nil { + return nil, err } - return nodes, epochNum, nil -} - -func validatorSetToNodes(validatorSet metadata.NodeBLSMappings) common.Nodes { - var nodes common.Nodes - for i := range validatorSet { - vdr := &validatorSet[i] - nodes = append(nodes, common.Node{ - Id: vdr.NodeID[:], - Weight: vdr.Weight, - PK: vdr.BLSKey, - }) - } - return nodes -} -func constructValidatorSetFromSealingBlock(lastBlock *ParsedBlock) metadata.NodeBLSMappings { - var validatorSet metadata.NodeBLSMappings - vdrs := lastBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor.AggregatedMembership.Members - for i := range vdrs { - vdr := &vdrs[i] - validatorSet = append(validatorSet, metadata.NodeBLSMapping{ - NodeID: vdr.NodeID, - BLSKey: vdr.BLSKey, - Weight: vdr.Weight, - }) - } - return validatorSet + return mappings.Nodes(), nil } diff --git a/instance_test.go b/instance_test.go index a6092468..dfa281d4 100644 --- a/instance_test.go +++ b/instance_test.go @@ -124,9 +124,7 @@ func TestInstanceNonValidatorBootstraps(t *testing.T) { // One node is a validator and progresses the chain by building blocks, // and its weight changes while the chain progresses in 3 different P-chain epoch heights. // Then, we add another node which is a non-validator. - // The node should bootstrap the chain but without shutting down the non-validator instance, - // and the test should detect the log entry "I am still a non-validator at the tip of the P-chain, skipping role change" - // being printed several times until the non-validator node bootstraps. + // The node should bootstrap the chain but without shutting down the non-validator instance. // Later on, the non-validator becomes a validator. const ( basePChainHeight = uint64(1) @@ -179,17 +177,11 @@ func TestInstanceNonValidatorBootstraps(t *testing.T) { validatorInstance := newInstance(t, validatorNodeID, storage, net, pChain, cops, genesisBlock) nonValidatorInstance := newInstance(t, nonValidatorNodeID, storage2, net, pChain, cops, genesisBlock) - // Count how many times the non-validator reports that it is still not a validator at the - // tip of the P-chain while it replicates across the sealed epochs. - var stillNonValidatorLogs atomic.Uint64 // transitioned is closed when the node starts a Simplex epoch, i.e. becomes a validator. // The node only ever starts an epoch here as part of its non-validator -> validator // transition. transitioned := make(chan struct{}) nonValidatorInstance.Config.Logger.(*testutil.TestLogger).Intercept(func(entry zapcore.Entry) error { - if strings.Contains(entry.Message, "I am still a non-validator at the tip of the P-chain, skipping role change") { - stillNonValidatorLogs.Add(1) - } if strings.Contains(entry.Message, "Starting Simplex Epoch") { select { case <-transitioned: @@ -228,16 +220,16 @@ func TestInstanceNonValidatorBootstraps(t *testing.T) { require.NoError(t, nonValidatorInstance.Start(t.Context())) t.Cleanup(nonValidatorInstance.Stop) - // The non-validator replicates every sealed epoch. It stays a non-validator throughout, - // so on each sealing block it logs that it is still a non-validator at the tip. + // The non-validator replicates every sealed epoch and stays a non-validator throughout. bootstrapTarget := storage.NumBlocks() waitForNumBlocks(t, storage2, bootstrapTarget) - // The "still a non-validator" message was printed once per sealed epoch it replicated - // through, so once for each of the two epochs the weight changes above sealed. - require.Eventually(t, func() bool { - return stillNonValidatorLogs.Load() >= 2 - }, 20*time.Second, 100*time.Millisecond) + // It replicated through the sealed epochs without becoming a validator. + select { + case <-transitioned: + t.Fatal("non-validator transitioned to validator before joining the set") + default: + } // Now grow the validator set to include the peer at the P-chain tip. pChain.advanceTo(joinEpochP) @@ -873,12 +865,14 @@ func newInstance(t *testing.T, nodeID common.NodeID, storage *MockStorage, net * // newInstanceWithVM is like newInstance but uses a caller-supplied VM, so a test // can share one controllable VM across restarts of the same node. -func newInstanceWithVM(t *testing.T, nodeID common.NodeID, storage *MockStorage, net *inMemNetwork, pChain *testPlatformChain, cops *testCryptoOps, genesisBlock *testInnerBlock, vm VM) *Instance { +func newInstanceWithVM(t *testing.T, nodeID common.NodeID, storage *MockStorage, net *inMemNetwork, pChain *testPlatformChain, cops *testCryptoOps, genesisBlock *testInnerBlock, vm *testVM) *Instance { comm := &networkSender{net: net, self: nodeID} config := Config{ Logger: testutil.MakeLogger(t, int(nodeID[0])), ID: nodeID, VM: vm, + BlockDeserializer: vm, + ICMETransition: vm.ComputeICMEpoch, Storage: storage, Sender: comm, Broadcaster: comm, diff --git a/nonvalidator/non_validator.go b/nonvalidator/non_validator.go index bf704067..2ae53353 100644 --- a/nonvalidator/non_validator.go +++ b/nonvalidator/non_validator.go @@ -54,6 +54,10 @@ type Config struct { // RandomSource is used by the replication state to pick which nodes to // request sequences from. If nil, a cryptographically secure source is used. RandomSource *rand.Rand + + // TransitionToValidator is called when our non-validator indexes the highest known epoch + // and it is in the validator set + TransitionToValidator func(epoch uint64, validators common.Nodes) } type NonValidator struct { @@ -266,6 +270,14 @@ func (n *NonValidator) newFinalizedBlockTask(block common.Block, finalization *c return md.Digest } + if block.SealingBlockInfo() != nil { + // are we the highest validator + highestEpoch, highestValidatorSet := n.epochs.highestEpoch() + if highestValidatorSet.Contains(n.ID) && highestEpoch == md.Seq { + n.TransitionToValidator(block.BlockHeader().Seq, block.SealingBlockInfo().ValidatorSet) + } + } + n.Logger.Info("Verified and Indexed Block", zap.Uint64("Block Seq", md.Seq), zap.Stringer("Block Digest", md.Digest)) n.removeOldSequencesAndEpochs(md.Seq, md.Epoch) diff --git a/simplex/epoch.go b/simplex/epoch.go index 1377da11..845254c1 100644 --- a/simplex/epoch.go +++ b/simplex/epoch.go @@ -86,6 +86,7 @@ type EpochConfig struct { StartTime time.Time ReplicationEnabled bool RandomSource *rand.Rand + OnSealingBlockIndex func(epoch uint64, validators common.Nodes) } type Epoch struct { @@ -788,6 +789,7 @@ func (e *Epoch) Stop() { e.buildBlockScheduler.Close() e.timeoutHandler.Close() e.replicationState.Close() + e.Logger.Info("Node shutdown complete") } func (e *Epoch) isEpochSealed() bool { @@ -1490,6 +1492,9 @@ func (e *Epoch) indexFinalization(block common.VerifiedBlock, finalization commo e.broadcast(finalizationMsg) e.epochSealed.Store(true) + if e.OnSealingBlockIndex != nil { + e.OnSealingBlockIndex(block.BlockHeader().Seq, block.SealingBlockInfo().ValidatorSet) + } } // We have committed because we have collected a finalization. diff --git a/testutil/controlled.go b/testutil/controlled.go index 0dc76182..56d5b5d9 100644 --- a/testutil/controlled.go +++ b/testutil/controlled.go @@ -91,7 +91,7 @@ func (n *ControlledInMemoryNetwork) AdvanceWithoutLeader(round uint64, laggingNo type ControlledNode struct { *BasicNode - bb *testControlledBlockBuilder + bb *TestControlledBlockBuilder WAL *TestWAL Storage *InMemStorage } @@ -185,9 +185,9 @@ func (t *ControlledNode) TickUntilRoundAdvanced(round uint64, tick time.Duration } } -// testControlledBlockBuilder is a BlockBuilder that only builds a block when +// TestControlledBlockBuilder is a BlockBuilder that only builds a block when // a control signal is received. -type testControlledBlockBuilder struct { +type TestControlledBlockBuilder struct { t *testing.T control chan struct{} TestBlockBuilder @@ -195,22 +195,22 @@ type testControlledBlockBuilder struct { // NewTestControlledBlockBuilder returns a BlockBuilder that only builds a block // when triggerNewBlock is called. -func NewTestControlledBlockBuilder(t *testing.T) *testControlledBlockBuilder { - return &testControlledBlockBuilder{ +func NewTestControlledBlockBuilder(t *testing.T) *TestControlledBlockBuilder { + return &TestControlledBlockBuilder{ t: t, control: make(chan struct{}, 1), TestBlockBuilder: *NewTestBlockBuilder(), } } -func (t *testControlledBlockBuilder) TriggerNewBlock() { +func (t *TestControlledBlockBuilder) TriggerNewBlock() { select { case t.control <- struct{}{}: default: } } -func (t *testControlledBlockBuilder) BuildBlock(ctx context.Context, metadata common.ProtocolMetadata, blacklist common.Blacklist) (common.VerifiedBlock, bool) { +func (t *TestControlledBlockBuilder) BuildBlock(ctx context.Context, metadata common.ProtocolMetadata, blacklist common.Blacklist) (common.VerifiedBlock, bool) { select { case <-t.control: case <-ctx.Done(): diff --git a/testutil/node.go b/testutil/node.go index f5951bb7..fa5f8fdb 100644 --- a/testutil/node.go +++ b/testutil/node.go @@ -242,7 +242,7 @@ type TestNodeConfig struct { Comm common.Communication SigAggregatorCreator common.SignatureAggregatorCreator ReplicationEnabled bool - BlockBuilder *testControlledBlockBuilder + BlockBuilder *TestControlledBlockBuilder // Long Running Tests MaxRoundWindow uint64 diff --git a/util.go b/util.go new file mode 100644 index 00000000..49a8b9cd --- /dev/null +++ b/util.go @@ -0,0 +1,111 @@ +// Copyright (C) 2019-2025, Ava Labs, Inc. All rights reserved. +// See the file LICENSE for licensing terms. + +package simplex + +import ( + "errors" + "fmt" + + "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" + "go.uber.org/zap" +) + +var ( + errNoGenesisBlock = errors.New("no genesis block found in storage") + errNonSealingBlock = errors.New("expected sealing block, got a non-sealing block") +) + +// LastBlock returns the last block in storage along with the total number of blocks. +func LastBlock(storage Storage) (metadata.StateMachineBlock, uint64, error) { + numBlocks := storage.NumBlocks() + if numBlocks == 0 { + return metadata.StateMachineBlock{}, 0, errNoGenesisBlock + } + + lastBlock, _, err := storage.GetBlock(numBlocks - 1) + if err != nil { + return metadata.StateMachineBlock{}, 0, fmt.Errorf("error retrieving last block from storage: %w", err) + } + + return lastBlock, numBlocks, nil +} + +// getLastAcceptedEpoch determines the epoch the instance should start at based on +// the last block in storage. If the ledger only contains non-Simplex blocks, the +// epoch is the first Simplex height. If the last block is a sealing block, the +// epoch it seals has ended, so the next epoch is returned. Otherwise, the epoch +// of the last block is returned. +func getLastAcceptedEpochAndValidatorSet(config *Config) (common.Nodes, uint64, error) { + lastBlock, numBlocks, err := LastBlock(config.Storage) + if err != nil { + return nil, 0, fmt.Errorf("error retrieving last block: %w", err) + } + + lastNonSimplexHeight := config.LastNonSimplexInnerBlock.Height() + parsedLastBlock := ParsedBlock{StateMachineBlock: lastBlock} + epochNum := parsedLastBlock.BlockHeader().Epoch + genesisValidatorSet := config.PlatformChain.GenesisValidatorSet() + + var validatorSet metadata.NodeBLSMappings + var nodes common.Nodes + + switch { + // If all we have in the ledger is non-Simplex blocks, load the validator set from genesis + case lastNonSimplexHeight+1 == numBlocks: + nodes = validatorSetToNodes(genesisValidatorSet) + epochNum = lastNonSimplexHeight + 1 + config.Logger.Debug("Determined epoch and validator set from genesis (ledger holds only non-Simplex blocks)", + zap.Uint64("epoch", epochNum)) + // If the last block persisted is a sealing block, then we are in the next epoch. + case lastBlock.SealingBlockInfo() != nil: + epochNum = parsedLastBlock.BlockHeader().Seq + nodes = lastBlock.SealingBlockInfo().ValidatorSet + config.Logger.Debug("Determined epoch and validator set from sealing block at tip", + zap.Uint64("epoch", epochNum)) + // Else, we have at least one Simplex block in the ledger, and it's not a sealing block. + default: + // Therefore, the sequence of the sealing block is the epoch number. + sealingBlockSeq := parsedLastBlock.BlockHeader().Epoch + sealingBlock, _, err := config.Storage.GetBlock(sealingBlockSeq) + if err != nil { + return nil, 0, fmt.Errorf("error retrieving sealing block from storage: %w", err) + } + if sealingBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor == nil { + return nil, 0, fmt.Errorf("%w at seq %d", errNonSealingBlock, sealingBlockSeq) + } + validatorSet = constructValidatorSetFromSealingBlock(&ParsedBlock{StateMachineBlock: sealingBlock}) + nodes = validatorSetToNodes(validatorSet) + config.Logger.Debug("Determined epoch and validator set from sealing block in storage", + zap.Uint64("epoch", epochNum), zap.Uint64("sealingBlockSeq", sealingBlockSeq)) + } + return nodes, epochNum, nil +} + +func validatorSetToNodes(validatorSet metadata.NodeBLSMappings) common.Nodes { + var nodes common.Nodes + for i := range validatorSet { + vdr := &validatorSet[i] + nodes = append(nodes, common.Node{ + Id: vdr.NodeID[:], + Weight: vdr.Weight, + PK: vdr.BLSKey, + }) + } + return nodes +} + +func constructValidatorSetFromSealingBlock(lastBlock *ParsedBlock) metadata.NodeBLSMappings { + var validatorSet metadata.NodeBLSMappings + vdrs := lastBlock.Metadata.SimplexEpochInfo.BlockValidationDescriptor.AggregatedMembership.Members + for i := range vdrs { + vdr := &vdrs[i] + validatorSet = append(validatorSet, metadata.NodeBLSMapping{ + NodeID: vdr.NodeID, + BLSKey: vdr.BLSKey, + Weight: vdr.Weight, + }) + } + return validatorSet +} diff --git a/util_test.go b/util_test.go new file mode 100644 index 00000000..498f268f --- /dev/null +++ b/util_test.go @@ -0,0 +1,198 @@ +package simplex + +import ( + "context" + "errors" + "testing" + + "github.com/ava-labs/simplex/avalanchego" + "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" + "github.com/ava-labs/simplex/testutil" + "github.com/stretchr/testify/require" +) + +// stubStorage is a minimal Storage for exercising util functions. +// When err is set, GetBlock fails at seq errSeq. +type stubStorage struct { + blocks []metadata.StateMachineBlock + errSeq uint64 + err error +} + +func (s *stubStorage) NumBlocks() uint64 { + return uint64(len(s.blocks)) +} + +func (s *stubStorage) GetBlock(seq uint64) (metadata.StateMachineBlock, *common.Finalization, error) { + if s.err != nil && seq == s.errSeq { + return metadata.StateMachineBlock{}, nil, s.err + } + return s.blocks[seq], &common.Finalization{}, nil +} + +func (s *stubStorage) Index(context.Context, common.VerifiedBlock, common.Finalization) error { + return nil +} + +// nonSimplexBlock returns a pre-fork block holding only an inner block at the given height. +func nonSimplexBlock(height uint64) metadata.StateMachineBlock { + return metadata.StateMachineBlock{InnerBlock: &testInnerBlock{Height_: height}} +} + +// simplexBlock returns a non-sealing simplex block at the given epoch and seq. +func simplexBlock(epoch, seq uint64) metadata.StateMachineBlock { + return metadata.StateMachineBlock{ + Metadata: metadata.StateMachineMetadata{ + SimplexProtocolMetadata: common.ProtocolMetadata{Epoch: epoch, Seq: seq}, + }, + } +} + +// sealingBlock returns a sealing block at the given epoch and seq whose +// descriptor holds the given validator set. +func sealingBlock(epoch, seq uint64, members []metadata.NodeBLSMapping) metadata.StateMachineBlock { + block := simplexBlock(epoch, seq) + block.Metadata.SimplexEpochInfo.BlockValidationDescriptor = &metadata.BlockValidationDescriptor{ + AggregatedMembership: metadata.AggregatedMembership{Members: members}, + } + return block +} + +func testValidatorSet() metadata.NodeBLSMappings { + return metadata.NodeBLSMappings{ + {NodeID: avalanchego.NodeID{1}, BLSKey: []byte{1, 2}, Weight: 1}, + {NodeID: avalanchego.NodeID{2}, BLSKey: []byte{3, 4}, Weight: 2}, + } +} + +// epochTestConfig derives the last non-Simplex height from the leading +// blocks in storage that carry no Simplex metadata. +func epochTestConfig(t *testing.T, storage *stubStorage, genesisSet metadata.NodeBLSMappings) *Config { + var lastNonSimplexHeight uint64 + for seq, block := range storage.blocks { + if block.Metadata.SimplexProtocolMetadata.Epoch != 0 { + break + } + lastNonSimplexHeight = uint64(seq) + } + return &Config{ + Storage: storage, + PlatformChain: newTestPlatformChain(0, map[uint64]metadata.NodeBLSMappings{0: genesisSet}), + LastNonSimplexInnerBlock: &testInnerBlock{Height_: lastNonSimplexHeight}, + Logger: testutil.MakeLogger(t, 1), + } +} + +// LastBlock errors on empty storage. +func TestLastBlockEmptyStorage(t *testing.T) { + _, _, err := LastBlock(&stubStorage{}) + require.ErrorIs(t, err, errNoGenesisBlock) +} + +// LastBlock wraps GetBlock errors. +func TestLastBlockGetBlockError(t *testing.T) { + sentinel := errors.New("disk corrupted") + storage := &stubStorage{ + blocks: make([]metadata.StateMachineBlock, 3), + errSeq: 2, + err: sentinel, + } + _, _, err := LastBlock(storage) + require.ErrorIs(t, err, sentinel) +} + +// LastBlock returns the block at seq numBlocks-1 and the block count. +func TestLastBlockSuccess(t *testing.T) { + storage := &stubStorage{ + blocks: []metadata.StateMachineBlock{ + nonSimplexBlock(0), + simplexBlock(1, 1), + }, + } + got, numBlocks, err := LastBlock(storage) + require.NoError(t, err) + require.Equal(t, uint64(2), numBlocks) + require.Equal(t, storage.blocks[1], got) +} + +// Covers each branch of getLastAcceptedEpochAndValidatorSet: genesis, +// sealing block at tip, sealing block in storage, and the error paths. +func TestGetLastAcceptedEpochAndValidatorSet(t *testing.T) { + vdrSet := testValidatorSet() + + tests := []struct { + name string + blocks []metadata.StateMachineBlock + expectedEpoch uint64 + expectedNodes common.Nodes + expectedErr error + }{ + { + name: "only non-Simplex blocks starts at first Simplex height with genesis set", + blocks: []metadata.StateMachineBlock{nonSimplexBlock(0)}, + expectedEpoch: 1, + expectedNodes: vdrSet.Nodes(), + }, + { + name: "multiple non-Simplex blocks start at first Simplex height with genesis set", + blocks: []metadata.StateMachineBlock{ + nonSimplexBlock(0), + nonSimplexBlock(1), + nonSimplexBlock(2), + }, + expectedEpoch: 3, + expectedNodes: vdrSet.Nodes(), + }, + { + name: "sealing block at tip starts next epoch with its descriptor set", + blocks: []metadata.StateMachineBlock{ + simplexBlock(1, 1), + sealingBlock(1, 2, vdrSet), + }, + expectedEpoch: 2, + expectedNodes: vdrSet.Nodes(), + }, + { + name: "non-sealing tip keeps its epoch, set loaded from sealing block at seq==epoch", + blocks: []metadata.StateMachineBlock{ + nonSimplexBlock(0), + simplexBlock(1, 1), + sealingBlock(1, 2, vdrSet), + simplexBlock(2, 3), + }, + expectedEpoch: 2, + expectedNodes: vdrSet.Nodes(), + }, + { + name: "empty storage errors", + expectedErr: errNoGenesisBlock, + }, + { + name: "non-sealing block at the sealing seq errors", + blocks: []metadata.StateMachineBlock{ + nonSimplexBlock(0), + simplexBlock(1, 1), + simplexBlock(1, 2), + simplexBlock(2, 3), + }, + expectedErr: errNonSealingBlock, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + storage := &stubStorage{blocks: tt.blocks} + config := epochTestConfig(t, storage, vdrSet) + + nodes, epoch, err := getLastAcceptedEpochAndValidatorSet(config) + if tt.expectedErr != nil { + require.ErrorIs(t, err, tt.expectedErr) + return + } + require.NoError(t, err) + require.Equal(t, tt.expectedEpoch, epoch) + require.Equal(t, tt.expectedNodes, nodes) + }) + } +} From db7321769f16d7cab454b04236a398c1a2fcd4c1 Mon Sep 17 00:00:00 2001 From: samliok Date: Thu, 13 Aug 2026 18:12:35 -0400 Subject: [PATCH 2/6] Disseminate auxiliary info and approvals via network messages Add Message.AuxiliaryInfo and Message.EpochTransitionApproval along with the common.AuxiliaryInfo type. Blocks now carry an AuxiliaryInfoBatch with one entry per contributing node instead of a single locally generated entry. The builder collects legal entries from a new auxInfoStore mempool populated by HandleAuxiliaryInfo, and SignApproval is exported so callers can sign and broadcast their own approvals. GetAuxiliaryHistory exposes the aux history traversal used to decide whether to generate info or approve. Note: this changes the wire format of the metadata aux info field, and the candidate digest for an empty aux history is now the zero digest instead of sha256(nil). --- common/msg.canoto.go | 194 ++++++++++++++++++ common/msg.go | 21 ++ msm/approvals.go | 4 +- msm/auxiliary.canoto.go | 257 ++++++++++++++++++++++++ msm/auxiliary.go | 177 +++++++++++++++++ msm/auxiliary_test.go | 425 ++++++++++++++++++++++++++++++++++++++++ msm/encoding.canoto.go | 231 ++-------------------- msm/encoding.go | 57 ++---- msm/fake_node_test.go | 5 +- msm/fuzz_test.go | 18 +- msm/msm.go | 257 ++++++++++-------------- msm/msm_test.go | 299 ++++++++++++++-------------- 12 files changed, 1368 insertions(+), 577 deletions(-) create mode 100644 common/msg.canoto.go create mode 100644 msm/auxiliary.canoto.go create mode 100644 msm/auxiliary.go create mode 100644 msm/auxiliary_test.go diff --git a/common/msg.canoto.go b/common/msg.canoto.go new file mode 100644 index 00000000..f3f6a49b --- /dev/null +++ b/common/msg.canoto.go @@ -0,0 +1,194 @@ +// Code generated by canoto. DO NOT EDIT. +// versions: +// canoto v0.19.0 +// source: msg.go + +package common + +import ( + "io" + "reflect" + "sync/atomic" + + "github.com/StephenButtolph/canoto" +) + +// Ensure that the generated code is compatible with the library version. +const ( + _ uint = canoto.VersionCompatibility - 1 + _ uint = 1 - canoto.VersionCompatibility +) + +// Ensure that unused imports do not error +var _ = io.ErrUnexpectedEOF + +const ( + canotoNumber_AuxiliaryInfo__Version = 1 + canotoNumber_AuxiliaryInfo__Data = 2 + + canotoTag_AuxiliaryInfo__Version = "\x08" // canoto.Tag(canotoNumber_AuxiliaryInfo__Version, canoto.Varint) + canotoTag_AuxiliaryInfo__Data = "\x12" // canoto.Tag(canotoNumber_AuxiliaryInfo__Data, canoto.Len) +) + +type canotoData_AuxiliaryInfo struct { + size uint64 +} + +// CanotoSpec returns the specification of this canoto message. +func (*AuxiliaryInfo) CanotoSpec(...reflect.Type) *canoto.Spec { + var zero AuxiliaryInfo + s := &canoto.Spec{ + Name: "AuxiliaryInfo", + Fields: []canoto.FieldType{ + { + FieldNumber: canotoNumber_AuxiliaryInfo__Version, + Name: "Version", + OneOf: "", + TypeUint: canoto.SizeOf(zero.Version), + }, + { + FieldNumber: canotoNumber_AuxiliaryInfo__Data, + Name: "Data", + OneOf: "", + TypeBytes: true, + }, + }, + } + s.CalculateCanotoCache() + return s +} + +// UnmarshalCanoto unmarshals a Canoto-encoded byte slice into the struct. +// +// During parsing, the canoto cache is saved. +func (c *AuxiliaryInfo) UnmarshalCanoto(bytes []byte) error { + r := canoto.Reader{ + B: bytes, + } + return c.UnmarshalCanotoFrom(r) +} + +// UnmarshalCanotoFrom populates the struct from a [canoto.Reader]. Most users +// should just use UnmarshalCanoto. +// +// During parsing, the canoto cache is saved. +// +// This function enables configuration of reader options. +func (c *AuxiliaryInfo) UnmarshalCanotoFrom(r canoto.Reader) error { + // Zero the struct before unmarshaling. + *c = AuxiliaryInfo{} + atomic.StoreUint64(&c.canotoData.size, uint64(len(r.B))) + + var minField uint32 + for canoto.HasNext(&r) { + field, wireType, err := canoto.ReadTag(&r) + if err != nil { + return err + } + if field < minField { + return canoto.ErrInvalidFieldOrder + } + + switch field { + case canotoNumber_AuxiliaryInfo__Version: + if wireType != canoto.Varint { + return canoto.ErrUnexpectedWireType + } + + if err := canoto.ReadUint(&r, &c.Version); err != nil { + return err + } + if canoto.IsZero(c.Version) { + return canoto.ErrZeroValue + } + case canotoNumber_AuxiliaryInfo__Data: + if wireType != canoto.Len { + return canoto.ErrUnexpectedWireType + } + + if err := canoto.ReadBytes(&r, &c.Data); err != nil { + return err + } + if len(c.Data) == 0 { + return canoto.ErrZeroValue + } + default: + return canoto.ErrUnknownField + } + + minField = field + 1 + } + return nil +} + +// ValidCanoto validates that the struct can be correctly marshaled into the +// Canoto format. +// +// Specifically, ValidCanoto ensures: +// 1. All OneOfs are specified at most once. +// 2. All strings are valid utf-8. +// 3. All custom fields are ValidCanoto. +func (c *AuxiliaryInfo) ValidCanoto() bool { + return true +} + +// CalculateCanotoCache populates size and OneOf caches based on the current +// values in the struct. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfo) CalculateCanotoCache() { + var size uint64 + if !canoto.IsZero(c.Version) { + size += uint64(len(canotoTag_AuxiliaryInfo__Version)) + canoto.SizeUint(c.Version) + } + if len(c.Data) != 0 { + size += uint64(len(canotoTag_AuxiliaryInfo__Data)) + canoto.SizeBytes(c.Data) + } + atomic.StoreUint64(&c.canotoData.size, size) +} + +// CachedCanotoSize returns the previously calculated size of the Canoto +// representation from CalculateCanotoCache. +// +// If CalculateCanotoCache has not yet been called, it will return 0. +// +// If the struct has been modified since the last call to CalculateCanotoCache, +// the returned size may be incorrect. +func (c *AuxiliaryInfo) CachedCanotoSize() uint64 { + return atomic.LoadUint64(&c.canotoData.size) +} + +// MarshalCanoto returns the Canoto representation of this struct. +// +// It is assumed that this struct is ValidCanoto. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfo) MarshalCanoto() []byte { + c.CalculateCanotoCache() + w := canoto.Writer{ + B: make([]byte, 0, c.CachedCanotoSize()), + } + w = c.MarshalCanotoInto(w) + return w.B +} + +// MarshalCanotoInto writes the struct into a [canoto.Writer] and returns the +// resulting [canoto.Writer]. Most users should just use MarshalCanoto. +// +// It is assumed that CalculateCanotoCache has been called since the last +// modification to this struct. +// +// It is assumed that this struct is ValidCanoto. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfo) MarshalCanotoInto(w canoto.Writer) canoto.Writer { + if !canoto.IsZero(c.Version) { + canoto.Append(&w, canotoTag_AuxiliaryInfo__Version) + canoto.AppendUint(&w, c.Version) + } + if len(c.Data) != 0 { + canoto.Append(&w, canotoTag_AuxiliaryInfo__Data) + canoto.AppendBytes(&w, c.Data) + } + return w +} diff --git a/common/msg.go b/common/msg.go index 5b4e75e6..c19ebce9 100644 --- a/common/msg.go +++ b/common/msg.go @@ -30,6 +30,10 @@ type Message struct { // Verified Messages VerifiedBlockMessage *VerifiedBlockMessage VerifiedReplicationResponse *VerifiedReplicationResponse + + // Epoch Transition Messages + AuxiliaryInfo *AuxiliaryInfo + EpochTransitionApproval *ValidatorSetApproval } func (m *Message) IsReplicationMessage() bool { @@ -432,6 +436,23 @@ type BlockDigestRequest struct { // VersionID is an identifier for applications that care about epoch changes. type VersionID uint32 +//go:generate go run github.com/StephenButtolph/canoto/canoto msg.go + +// AuxiliaryInfo defines application-specific information for applications that might care about epoch change, +// such as threshold distributed public key generation. +type AuxiliaryInfo struct { + // VersionID is an identifier that identifies the application. + // Can be used for backward-compatibility and upgrade purposes. + Version VersionID `canoto:"uint,1"` + + // Info is opaque bytes that can be used by applications to encode any information that describes + // the current state for the application. + Data []byte `canoto:"bytes,2"` + + canotoData canotoData_AuxiliaryInfo +} + +// ValidatorSetApproval is an approval from a validator type ValidatorSetApproval struct { NodeID avalanchego.NodeID AuxInfoDigest [32]byte diff --git a/msm/approvals.go b/msm/approvals.go index 602331a4..44e088e5 100644 --- a/msm/approvals.go +++ b/msm/approvals.go @@ -157,7 +157,7 @@ func (as *ApprovalStore) checkApprovalSignature(approval *common.ValidatorSetApp } func (as *ApprovalStore) approvalExistsAndUpToDate(approval *common.ValidatorSetApproval, timestamp uint64) bool { - if as.approvalsByNodes[avalanchego.NodeID(approval.NodeID)] == nil { + if as.approvalsByNodes[approval.NodeID] == nil { return false } @@ -166,7 +166,7 @@ func (as *ApprovalStore) approvalExistsAndUpToDate(approval *common.ValidatorSet auxInfoDigest: approval.AuxInfoDigest, } - existingApproval := as.approvalsByNodes[avalanchego.NodeID(approval.NodeID)][key] + existingApproval := as.approvalsByNodes[approval.NodeID][key] if existingApproval == nil { return false } diff --git a/msm/auxiliary.canoto.go b/msm/auxiliary.canoto.go new file mode 100644 index 00000000..65899377 --- /dev/null +++ b/msm/auxiliary.canoto.go @@ -0,0 +1,257 @@ +// Code generated by canoto. DO NOT EDIT. +// versions: +// canoto v0.19.0 +// source: auxiliary.go + +package metadata + +import ( + "io" + "reflect" + "sync/atomic" + + "github.com/StephenButtolph/canoto" +) + +// Ensure that the generated code is compatible with the library version. +const ( + _ uint = canoto.VersionCompatibility - 1 + _ uint = 1 - canoto.VersionCompatibility +) + +// Ensure that unused imports do not error +var _ = io.ErrUnexpectedEOF + +const ( + canotoNumber_AuxiliaryInfoBatch__data = 1 + canotoNumber_AuxiliaryInfoBatch__PrevAuxInfoSeq = 2 + + canotoTag_AuxiliaryInfoBatch__data = "\x0a" // canoto.Tag(canotoNumber_AuxiliaryInfoBatch__data, canoto.Len) + canotoTag_AuxiliaryInfoBatch__PrevAuxInfoSeq = "\x10" // canoto.Tag(canotoNumber_AuxiliaryInfoBatch__PrevAuxInfoSeq, canoto.Varint) +) + +type canotoData_AuxiliaryInfoBatch struct { + size uint64 +} + +// CanotoSpec returns the specification of this canoto message. +func (*AuxiliaryInfoBatch) CanotoSpec(types ...reflect.Type) *canoto.Spec { + types = append(types, reflect.TypeFor[AuxiliaryInfoBatch]()) + var zero AuxiliaryInfoBatch + s := &canoto.Spec{ + Name: "AuxiliaryInfoBatch", + Fields: []canoto.FieldType{ + canoto.FieldTypeFromField( + /*type inference:*/ (canoto.MakeEntryNilPointer(zero.data)), + /*FieldNumber: */ canotoNumber_AuxiliaryInfoBatch__data, + /*Name: */ "data", + /*FixedLength: */ 0, + /*Repeated: */ true, + /*OneOf: */ "", + /*Pointer: */ false, + /*types: */ types, + ), + { + FieldNumber: canotoNumber_AuxiliaryInfoBatch__PrevAuxInfoSeq, + Name: "PrevAuxInfoSeq", + OneOf: "", + TypeUint: canoto.SizeOf(zero.PrevAuxInfoSeq), + }, + }, + } + s.CalculateCanotoCache() + return s +} + +// UnmarshalCanoto unmarshals a Canoto-encoded byte slice into the struct. +// +// During parsing, the canoto cache is saved. +func (c *AuxiliaryInfoBatch) UnmarshalCanoto(bytes []byte) error { + r := canoto.Reader{ + B: bytes, + } + return c.UnmarshalCanotoFrom(r) +} + +// UnmarshalCanotoFrom populates the struct from a [canoto.Reader]. Most users +// should just use UnmarshalCanoto. +// +// During parsing, the canoto cache is saved. +// +// This function enables configuration of reader options. +func (c *AuxiliaryInfoBatch) UnmarshalCanotoFrom(r canoto.Reader) error { + // Zero the struct before unmarshaling. + *c = AuxiliaryInfoBatch{} + atomic.StoreUint64(&c.canotoData.size, uint64(len(r.B))) + + var minField uint32 + for canoto.HasNext(&r) { + field, wireType, err := canoto.ReadTag(&r) + if err != nil { + return err + } + if field < minField { + return canoto.ErrInvalidFieldOrder + } + + switch field { + case canotoNumber_AuxiliaryInfoBatch__data: + if wireType != canoto.Len { + return canoto.ErrUnexpectedWireType + } + + // Read the first entry manually because the tag is already + // stripped. + originalUnsafe := r.Unsafe + r.Unsafe = true + var msgBytes []byte + if err := canoto.ReadBytes(&r, &msgBytes); err != nil { + return err + } + r.Unsafe = originalUnsafe + + // Count the number of additional entries after the first entry. + countMinus1, err := canoto.CountBytes(r.B, canotoTag_AuxiliaryInfoBatch__data) + if err != nil { + return err + } + + c.data = canoto.MakeSlice(c.data, countMinus1+1) + field := c.data + additionalField := field[1:] + if len(msgBytes) != 0 { + remainingBytes := r.B + r.B = msgBytes + if err := (&field[0]).UnmarshalCanotoFrom(r); err != nil { + return err + } + r.B = remainingBytes + } + + // Read the rest of the entries, stripping the tag each time. + for i := range additionalField { + r.B = r.B[len(canotoTag_AuxiliaryInfoBatch__data):] + r.Unsafe = true + if err := canoto.ReadBytes(&r, &msgBytes); err != nil { + return err + } + r.Unsafe = originalUnsafe + if len(msgBytes) == 0 { + continue + } + + remainingBytes := r.B + r.B = msgBytes + if err := (&additionalField[i]).UnmarshalCanotoFrom(r); err != nil { + return err + } + r.B = remainingBytes + } + case canotoNumber_AuxiliaryInfoBatch__PrevAuxInfoSeq: + if wireType != canoto.Varint { + return canoto.ErrUnexpectedWireType + } + + if err := canoto.ReadUint(&r, &c.PrevAuxInfoSeq); err != nil { + return err + } + if canoto.IsZero(c.PrevAuxInfoSeq) { + return canoto.ErrZeroValue + } + default: + return canoto.ErrUnknownField + } + + minField = field + 1 + } + return nil +} + +// ValidCanoto validates that the struct can be correctly marshaled into the +// Canoto format. +// +// Specifically, ValidCanoto ensures: +// 1. All OneOfs are specified at most once. +// 2. All strings are valid utf-8. +// 3. All custom fields are ValidCanoto. +func (c *AuxiliaryInfoBatch) ValidCanoto() bool { + { + field := c.data + for i := range field { + if !(&field[i]).ValidCanoto() { + return false + } + } + } + return true +} + +// CalculateCanotoCache populates size and OneOf caches based on the current +// values in the struct. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfoBatch) CalculateCanotoCache() { + var size uint64 + { + field := c.data + for i := range field { + (&field[i]).CalculateCanotoCache() + fieldSize := (&field[i]).CachedCanotoSize() + size += uint64(len(canotoTag_AuxiliaryInfoBatch__data)) + canoto.SizeUint(fieldSize) + fieldSize + } + } + if !canoto.IsZero(c.PrevAuxInfoSeq) { + size += uint64(len(canotoTag_AuxiliaryInfoBatch__PrevAuxInfoSeq)) + canoto.SizeUint(c.PrevAuxInfoSeq) + } + atomic.StoreUint64(&c.canotoData.size, size) +} + +// CachedCanotoSize returns the previously calculated size of the Canoto +// representation from CalculateCanotoCache. +// +// If CalculateCanotoCache has not yet been called, it will return 0. +// +// If the struct has been modified since the last call to CalculateCanotoCache, +// the returned size may be incorrect. +func (c *AuxiliaryInfoBatch) CachedCanotoSize() uint64 { + return atomic.LoadUint64(&c.canotoData.size) +} + +// MarshalCanoto returns the Canoto representation of this struct. +// +// It is assumed that this struct is ValidCanoto. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfoBatch) MarshalCanoto() []byte { + c.CalculateCanotoCache() + w := canoto.Writer{ + B: make([]byte, 0, c.CachedCanotoSize()), + } + w = c.MarshalCanotoInto(w) + return w.B +} + +// MarshalCanotoInto writes the struct into a [canoto.Writer] and returns the +// resulting [canoto.Writer]. Most users should just use MarshalCanoto. +// +// It is assumed that CalculateCanotoCache has been called since the last +// modification to this struct. +// +// It is assumed that this struct is ValidCanoto. +// +// It is not safe to copy this struct concurrently. +func (c *AuxiliaryInfoBatch) MarshalCanotoInto(w canoto.Writer) canoto.Writer { + { + field := c.data + for i := range field { + canoto.Append(&w, canotoTag_AuxiliaryInfoBatch__data) + canoto.AppendUint(&w, (&field[i]).CachedCanotoSize()) + w = (&field[i]).MarshalCanotoInto(w) + } + } + if !canoto.IsZero(c.PrevAuxInfoSeq) { + canoto.Append(&w, canotoTag_AuxiliaryInfoBatch__PrevAuxInfoSeq) + canoto.AppendUint(&w, c.PrevAuxInfoSeq) + } + return w +} diff --git a/msm/auxiliary.go b/msm/auxiliary.go new file mode 100644 index 00000000..607469c2 --- /dev/null +++ b/msm/auxiliary.go @@ -0,0 +1,177 @@ +package metadata + +import ( + "bytes" + "crypto/sha256" + "fmt" + "slices" + "sync" + + "github.com/ava-labs/simplex/avalanchego" + "github.com/ava-labs/simplex/common" +) + +//go:generate go run github.com/StephenButtolph/canoto/canoto auxiliary.go + +// AuxiliaryInfoBatch is a batch of AuxiliaryInfos to be included in a block +type AuxiliaryInfoBatch struct { + // data is how we expect the order being appended. 0 index is appended first, then data[len()-1] is last + data []common.AuxiliaryInfo `canoto:"repeated value,1"` + // PrevAuxInfoSeq is a sequence number that applications can use to find previous AuxiliaryInfo in the chain. + // It is zero if this is the first AuxiliaryInfoBatch for this epoch. + PrevAuxInfoSeq uint64 `canoto:"uint,2"` + + canotoData canotoData_AuxiliaryInfoBatch +} + +func (ai *AuxiliaryInfoBatch) IsZero() bool { + var zero AuxiliaryInfoBatch + return ai.Equal(&zero) +} + +func (ai *AuxiliaryInfoBatch) Equal(a *AuxiliaryInfoBatch) bool { + if ai == nil { + return a == nil + } + if a == nil { + return false + } + if ai.PrevAuxInfoSeq != a.PrevAuxInfoSeq || len(ai.data) != len(a.data) { + return false + } + for i := range ai.data { + if ai.data[i].Version != a.data[i].Version || !bytes.Equal(ai.data[i].Data, a.data[i].Data) { + return false + } + } + return true +} + +type AuxInfoHistory struct { + Data [][]byte + LastSeq uint64 + OldestVersionID common.VersionID // oldest version id in the histories data, or DefaultVersionID if no history +} + +func (aih *AuxInfoHistory) LastHistoryDigest() [32]byte { + if len(aih.Data) == 0 { + return [32]byte{} + } + last := aih.Data[len(aih.Data)-1] + return sha256.Sum256(last) +} + +// GetAuxiliaryHistory traverses backwards starting from the given block and returns the AuxInfoHistory of all blocks in the chain. +// It returns the collected auxiliary info ordered from oldest to newest, the sequence of the newest block it was collected from, +// and the version ID of the oldest non-empty auxiliary info entry (or defaultVersionID if there was none). +// blockSeq must be the sequence of the given block. +func GetAuxiliaryHistory(block *StateMachineBlock, blockSeq uint64, getBlock BlockRetriever, defaultVersionID common.VersionID) (AuxInfoHistory, error) { + var lastSeq *uint64 + var history [][]byte + var versionID = defaultVersionID + + // We traverse the chain of blocks backwards in the following manner: + // (1) Every block that doesn't have an AuxiliaryInfoBatch, its parents also do not have one. + // (2) Every block that has an AuxiliaryInfoBatch, its descendants also have one. + // (3) A block's AuxiliaryInfoBatch may have no entries, but its PrevAuxInfoSeq field must point + // to a block whose AuxiliaryInfoBatch isn't nil and has non-empty entries. + // (4) When a block with an empty batch is built on a parent block that has an AuxiliaryInfoBatch, + // if its parent block's batch has non-empty entries, then the block's PrevAuxInfoSeq points to its parent block. + // Else, its parent block's batch is also empty, then the block's PrevAuxInfoSeq is inherited from its parent block's PrevAuxInfoSeq. + + batch := block.Metadata.AuxiliaryInfoBatch + currentSeq := blockSeq + for batch != nil { + // Entries within a batch are ordered oldest to newest, so iterate newest-first: + // the full history is reversed once traversal completes. + for i := len(batch.data) - 1; i >= 0; i-- { + entry := batch.data[i] + if len(entry.Data) == 0 { + continue + } + history = append(history, entry.Data) + if lastSeq == nil { + lastSeq = new(uint64) + *lastSeq = currentSeq + } + versionID = entry.Version + } + if batch.PrevAuxInfoSeq == 0 { + // This is the first auxiliary info of the epoch, we can stop traversing back. + break + } + currentSeq = batch.PrevAuxInfoSeq + prevBlock, _, err := getBlock(batch.PrevAuxInfoSeq, [32]byte{}) + if err != nil { + return AuxInfoHistory{}, fmt.Errorf("%w: at sequence %d: %w", errAuxInfoBlockRetrieval, batch.PrevAuxInfoSeq, err) + } + batch = prevBlock.Metadata.AuxiliaryInfoBatch + } + + if lastSeq == nil { + lastSeq = new(uint64) + *lastSeq = 0 + } + + // Reverse so the history is ordered from oldest to newest. + slices.Reverse(history) + return AuxInfoHistory{Data: history, LastSeq: *lastSeq, OldestVersionID: versionID}, nil +} + +// auxInfoStore stores auxiliary info that has been received but not yet included in blocks +type auxInfoStore struct { + app AuxiliaryInfoGenVerifier + + lock sync.Mutex + sentInfo map[avalanchego.NodeID]common.AuxiliaryInfo +} + +func newAuxInfoStore(app AuxiliaryInfoGenVerifier) *auxInfoStore { + return &auxInfoStore{ + app: app, + sentInfo: make(map[avalanchego.NodeID]common.AuxiliaryInfo), + } +} + +func (a *auxInfoStore) HandleAuxiliaryMessage(info common.AuxiliaryInfo, from avalanchego.NodeID) { + a.lock.Lock() + defer a.lock.Unlock() + + // just set the nodes Auxiliary info to the most recent one they sent + a.sentInfo[from] = info +} + +// collectAuxInfo returns the stored entries that are legal appends to the given history. +func (a *auxInfoStore) collectAuxInfo(history AuxInfoHistory, validators NodeBLSMappings) []common.AuxiliaryInfo { + a.lock.Lock() + defer a.lock.Unlock() + + // Iterate in node ID order so the returned entries are deterministic. + nodeIDs := make([]avalanchego.NodeID, 0, len(a.sentInfo)) + for nodeID := range a.sentInfo { + nodeIDs = append(nodeIDs, nodeID) + } + slices.SortFunc(nodeIDs, func(x, y avalanchego.NodeID) int { + return bytes.Compare(x[:], y[:]) + }) + + var legalAppends []common.AuxiliaryInfo + legalHistory := append([][]byte{}, history.Data...) + + for _, nodeID := range nodeIDs { + info := a.sentInfo[nodeID] + if history.OldestVersionID != info.Version { + continue // keep consistent versions throughout epoch transition + } + + if err := a.app.IsLegalAppend(info.Version, validators, legalHistory, info.Data); err != nil { + // we don't remove this info from the mempool. maybe it can be added in a different block + continue + } + + legalAppends = append(legalAppends, info) + legalHistory = append(legalHistory, info.Data) + } + + return legalAppends +} diff --git a/msm/auxiliary_test.go b/msm/auxiliary_test.go new file mode 100644 index 00000000..49860c49 --- /dev/null +++ b/msm/auxiliary_test.go @@ -0,0 +1,425 @@ +package metadata + +import ( + "fmt" + "testing" + + "github.com/ava-labs/simplex/avalanchego" + "github.com/ava-labs/simplex/common" + + "github.com/stretchr/testify/require" +) + +func TestAuxiliaryInfoBatchEqual(t *testing.T) { + for _, tt := range []struct { + name string + a *AuxiliaryInfoBatch + b *AuxiliaryInfoBatch + expected bool + }{ + { + name: "both nil", + a: nil, + b: nil, + expected: true, + }, + { + name: "nil vs non-nil", + a: nil, + b: &AuxiliaryInfoBatch{}, + expected: false, + }, + { + name: "both zero", + a: &AuxiliaryInfoBatch{}, + b: &AuxiliaryInfoBatch{}, + expected: true, + }, + { + name: "equal with data", + a: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte{1, 2, 3}}, + {Version: 2, Data: []byte{4, 5}}, + }, + PrevAuxInfoSeq: 7, + }, + b: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte{1, 2, 3}}, + {Version: 2, Data: []byte{4, 5}}, + }, + PrevAuxInfoSeq: 7, + }, + expected: true, + }, + { + name: "nil data vs empty data", + a: &AuxiliaryInfoBatch{}, + b: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{}, + }, + expected: true, + }, + { + name: "different PrevAuxInfoSeq", + a: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte{1}}}, + PrevAuxInfoSeq: 1, + }, + b: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte{1}}}, + PrevAuxInfoSeq: 2, + }, + expected: false, + }, + { + name: "different number of entries", + a: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte{1}}}, + }, + b: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte{1}}, + {Version: 1, Data: []byte{2}}, + }, + }, + expected: false, + }, + { + name: "different entry version", + a: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte{1}}}, + }, + b: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 2, Data: []byte{1}}}, + }, + expected: false, + }, + { + name: "different entry data", + a: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte{1}}}, + }, + b: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte{2}}}, + }, + expected: false, + }, + { + name: "same entries in different order", + a: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte{1}}, + {Version: 2, Data: []byte{2}}, + }, + }, + b: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{ + {Version: 2, Data: []byte{2}}, + {Version: 1, Data: []byte{1}}, + }, + }, + expected: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expected, tt.a.Equal(tt.b)) + require.Equal(t, tt.expected, tt.b.Equal(tt.a)) + }) + } +} + +func TestAuxiliaryInfoBatchIsZero(t *testing.T) { + for _, tt := range []struct { + name string + batch *AuxiliaryInfoBatch + expected bool + }{ + { + name: "zero value", + batch: &AuxiliaryInfoBatch{}, + expected: true, + }, + { + name: "empty data slice", + batch: &AuxiliaryInfoBatch{data: []common.AuxiliaryInfo{}}, + expected: true, + }, + { + name: "non-zero PrevAuxInfoSeq", + batch: &AuxiliaryInfoBatch{PrevAuxInfoSeq: 1}, + expected: false, + }, + { + name: "non-empty data", + batch: &AuxiliaryInfoBatch{data: []common.AuxiliaryInfo{{Version: 1, Data: []byte{1}}}}, + expected: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.expected, tt.batch.IsZero()) + }) + } +} + +// batchBlock returns a StateMachineBlock whose metadata carries the given AuxiliaryInfoBatch. +func batchBlock(batch *AuxiliaryInfoBatch) StateMachineBlock { + return StateMachineBlock{ + Metadata: StateMachineMetadata{ + AuxiliaryInfoBatch: batch, + }, + } +} + +// blockRetrieverFromMap returns a BlockRetriever backed by the given seq -> block mapping, +// failing the test if a sequence outside the mapping is requested. +func blockRetrieverFromMap(t *testing.T, blocks map[uint64]StateMachineBlock) BlockRetriever { + return func(seq uint64, _ common.Digest) (StateMachineBlock, *common.Finalization, error) { + block, ok := blocks[seq] + require.True(t, ok, "requested unexpected block at seq %d", seq) + return block, nil, nil + } +} + +func TestGetAuxiliaryHistory(t *testing.T) { + const ( + defaultVersionID = common.VersionID(42) + startSeq = uint64(10) + ) + + for _, tt := range []struct { + name string + // batch of the block traversal starts from + startBatch *AuxiliaryInfoBatch + // batches of ancestor blocks by seq, reachable via PrevAuxInfoSeq links + prevBatches map[uint64]*AuxiliaryInfoBatch + expected AuxInfoHistory + }{ + { + name: "no batch", + startBatch: nil, + expected: AuxInfoHistory{ + LastSeq: 0, + OldestVersionID: defaultVersionID, + }, + }, + { + name: "batch with no entries", + startBatch: &AuxiliaryInfoBatch{}, + expected: AuxInfoHistory{ + LastSeq: 0, + OldestVersionID: defaultVersionID, + }, + }, + { + name: "single batch preserves entry order", + startBatch: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte("a")}, + {Version: 2, Data: []byte("b")}, + }, + }, + expected: AuxInfoHistory{ + Data: [][]byte{[]byte("a"), []byte("b")}, + LastSeq: startSeq, + OldestVersionID: 1, + }, + }, + { + name: "entries with empty data are skipped", + startBatch: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte("a")}, + {Version: 2, Data: nil}, + {Version: 3, Data: []byte("c")}, + }, + }, + expected: AuxInfoHistory{ + Data: [][]byte{[]byte("a"), []byte("c")}, + LastSeq: startSeq, + OldestVersionID: 1, + }, + }, + { + name: "chain of batches ordered oldest to newest", + startBatch: &AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 3, Data: []byte("d")}}, + PrevAuxInfoSeq: 5, + }, + prevBatches: map[uint64]*AuxiliaryInfoBatch{ + 5: { + data: []common.AuxiliaryInfo{ + {Version: 2, Data: []byte("b")}, + {Version: 2, Data: []byte("c")}, + }, + PrevAuxInfoSeq: 3, + }, + 3: { + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte("a")}}, + }, + }, + expected: AuxInfoHistory{ + Data: [][]byte{[]byte("a"), []byte("b"), []byte("c"), []byte("d")}, + LastSeq: startSeq, + OldestVersionID: 1, + }, + }, + { + name: "empty starting batch inherits from ancestors", + startBatch: &AuxiliaryInfoBatch{ + PrevAuxInfoSeq: 4, + }, + prevBatches: map[uint64]*AuxiliaryInfoBatch{ + 4: { + data: []common.AuxiliaryInfo{{Version: 7, Data: []byte("a")}}, + }, + }, + expected: AuxInfoHistory{ + Data: [][]byte{[]byte("a")}, + LastSeq: 4, + OldestVersionID: 7, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + blocks := make(map[uint64]StateMachineBlock, len(tt.prevBatches)) + for seq, batch := range tt.prevBatches { + blocks[seq] = batchBlock(batch) + } + + startBlock := batchBlock(tt.startBatch) + history, err := GetAuxiliaryHistory(&startBlock, startSeq, blockRetrieverFromMap(t, blocks), defaultVersionID) + require.NoError(t, err) + require.Equal(t, tt.expected, history) + }) + } +} + +func TestGetAuxiliaryHistoryRetrievalError(t *testing.T) { + startBlock := batchBlock(&AuxiliaryInfoBatch{ + data: []common.AuxiliaryInfo{{Version: 1, Data: []byte("a")}}, + PrevAuxInfoSeq: 5, + }) + + getBlock := func(seq uint64, _ common.Digest) (StateMachineBlock, *common.Finalization, error) { + return StateMachineBlock{}, nil, fmt.Errorf("no block at seq %d", seq) + } + + _, err := GetAuxiliaryHistory(&startBlock, 10, getBlock, 0) + require.ErrorIs(t, err, errAuxInfoBlockRetrieval) +} + +type sentAuxInfo struct { + from avalanchego.NodeID + info common.AuxiliaryInfo +} + +func TestCollectAuxInfo(t *testing.T) { + node1 := avalanchego.NodeID{1} + node2 := avalanchego.NodeID{2} + node3 := avalanchego.NodeID{3} + + // voteCountingAuxInfoApp rejects appends whose data is already in the history. + history := AuxInfoHistory{ + Data: [][]byte{[]byte("a")}, + OldestVersionID: 1, + } + + for _, tt := range []struct { + name string + sends []sentAuxInfo + expected []common.AuxiliaryInfo + }{ + { + name: "empty store", + sends: nil, + expected: nil, + }, + { + name: "legal entries returned sorted by node id", + sends: []sentAuxInfo{ + {from: node3, info: common.AuxiliaryInfo{Version: 1, Data: []byte("d")}}, + {from: node1, info: common.AuxiliaryInfo{Version: 1, Data: []byte("b")}}, + {from: node2, info: common.AuxiliaryInfo{Version: 1, Data: []byte("c")}}, + }, + expected: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte("b")}, + {Version: 1, Data: []byte("c")}, + {Version: 1, Data: []byte("d")}, + }, + }, + { + name: "version mismatch filtered", + sends: []sentAuxInfo{ + {from: node1, info: common.AuxiliaryInfo{Version: 2, Data: []byte("b")}}, + }, + expected: nil, + }, + { + name: "inconsistent versions only keep entries matching the history version", + sends: []sentAuxInfo{ + {from: node1, info: common.AuxiliaryInfo{Version: 2, Data: []byte("b")}}, + {from: node2, info: common.AuxiliaryInfo{Version: 1, Data: []byte("c")}}, + {from: node3, info: common.AuxiliaryInfo{Version: 3, Data: []byte("d")}}, + }, + expected: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte("c")}, + }, + }, + { + name: "entries already in history filtered", + sends: []sentAuxInfo{ + {from: node1, info: common.AuxiliaryInfo{Version: 1, Data: []byte("a")}}, + }, + expected: nil, + }, + { + name: "accepted entries extend the history for later entries", + sends: []sentAuxInfo{ + {from: node1, info: common.AuxiliaryInfo{Version: 1, Data: []byte("b")}}, + {from: node2, info: common.AuxiliaryInfo{Version: 1, Data: []byte("b")}}, + {from: node3, info: common.AuxiliaryInfo{Version: 1, Data: []byte("c")}}, + }, + expected: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte("b")}, + {Version: 1, Data: []byte("c")}, + }, + }, + { + name: "latest info from a node wins", + sends: []sentAuxInfo{ + {from: node1, info: common.AuxiliaryInfo{Version: 1, Data: []byte("b")}}, + {from: node1, info: common.AuxiliaryInfo{Version: 1, Data: []byte("c")}}, + }, + expected: []common.AuxiliaryInfo{ + {Version: 1, Data: []byte("c")}, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + store := newAuxInfoStore(&voteCountingAuxInfoApp{}) + for _, send := range tt.sends { + store.HandleAuxiliaryMessage(send.info, send.from) + } + + require.Equal(t, tt.expected, store.collectAuxInfo(history, nil)) + }) + } +} + +func TestCollectAuxInfoKeepsRejectedEntries(t *testing.T) { + store := newAuxInfoStore(&voteCountingAuxInfoApp{}) + info := common.AuxiliaryInfo{Version: 1, Data: []byte("a")} + store.HandleAuxiliaryMessage(info, avalanchego.NodeID{1}) + + // the entry duplicates the history, so it is rejected but kept in the store + history := AuxInfoHistory{ + Data: [][]byte{[]byte("a")}, + OldestVersionID: 1, + } + require.Empty(t, store.collectAuxInfo(history, nil)) + + // with a history that no longer contains the entry, it becomes legal + require.Equal(t, []common.AuxiliaryInfo{info}, store.collectAuxInfo(AuxInfoHistory{OldestVersionID: 1}, nil)) +} diff --git a/msm/encoding.canoto.go b/msm/encoding.canoto.go index a196d7d2..16664850 100644 --- a/msm/encoding.canoto.go +++ b/msm/encoding.canoto.go @@ -29,7 +29,7 @@ const ( canotoNumber_StateMachineMetadata__PChainHeight = 4 canotoNumber_StateMachineMetadata__Timestamp = 5 canotoNumber_StateMachineMetadata__ICMEpochInfo = 6 - canotoNumber_StateMachineMetadata__AuxiliaryInfo = 7 + canotoNumber_StateMachineMetadata__AuxiliaryInfoBatch = 7 canotoTag_StateMachineMetadata__SimplexEpochInfo = "\x0a" // canoto.Tag(canotoNumber_StateMachineMetadata__SimplexEpochInfo, canoto.Len) canotoTag_StateMachineMetadata__SimplexProtocolMetadata = "\x12" // canoto.Tag(canotoNumber_StateMachineMetadata__SimplexProtocolMetadata, canoto.Len) @@ -37,7 +37,7 @@ const ( canotoTag_StateMachineMetadata__PChainHeight = "\x20" // canoto.Tag(canotoNumber_StateMachineMetadata__PChainHeight, canoto.Varint) canotoTag_StateMachineMetadata__Timestamp = "\x28" // canoto.Tag(canotoNumber_StateMachineMetadata__Timestamp, canoto.Varint) canotoTag_StateMachineMetadata__ICMEpochInfo = "\x32" // canoto.Tag(canotoNumber_StateMachineMetadata__ICMEpochInfo, canoto.Len) - canotoTag_StateMachineMetadata__AuxiliaryInfo = "\x3a" // canoto.Tag(canotoNumber_StateMachineMetadata__AuxiliaryInfo, canoto.Len) + canotoTag_StateMachineMetadata__AuxiliaryInfoBatch = "\x3a" // canoto.Tag(canotoNumber_StateMachineMetadata__AuxiliaryInfoBatch, canoto.Len) ) type canotoData_StateMachineMetadata struct { @@ -104,9 +104,9 @@ func (*StateMachineMetadata) CanotoSpec(types ...reflect.Type) *canoto.Spec { /*types: */ types, ), canoto.FieldTypeFromField( - /*type inference:*/ (zero.AuxiliaryInfo), - /*FieldNumber: */ canotoNumber_StateMachineMetadata__AuxiliaryInfo, - /*Name: */ "AuxiliaryInfo", + /*type inference:*/ (zero.AuxiliaryInfoBatch), + /*FieldNumber: */ canotoNumber_StateMachineMetadata__AuxiliaryInfoBatch, + /*Name: */ "AuxiliaryInfoBatch", /*FixedLength: */ 0, /*Repeated: */ false, /*OneOf: */ "", @@ -269,7 +269,7 @@ func (c *StateMachineMetadata) UnmarshalCanotoFrom(r canoto.Reader) error { return err } r.B = remainingBytes - case canotoNumber_StateMachineMetadata__AuxiliaryInfo: + case canotoNumber_StateMachineMetadata__AuxiliaryInfoBatch: if wireType != canoto.Len { return canoto.ErrUnexpectedWireType } @@ -286,8 +286,8 @@ func (c *StateMachineMetadata) UnmarshalCanotoFrom(r canoto.Reader) error { // Unmarshal the field from the bytes. remainingBytes := r.B r.B = msgBytes - c.AuxiliaryInfo = canoto.MakePointer(c.AuxiliaryInfo) - if err := (c.AuxiliaryInfo).UnmarshalCanotoFrom(r); err != nil { + c.AuxiliaryInfoBatch = canoto.MakePointer(c.AuxiliaryInfoBatch) + if err := (c.AuxiliaryInfoBatch).UnmarshalCanotoFrom(r); err != nil { return err } r.B = remainingBytes @@ -320,7 +320,7 @@ func (c *StateMachineMetadata) ValidCanoto() bool { if !(&c.ICMEpochInfo).ValidCanoto() { return false } - if c.AuxiliaryInfo != nil && !(c.AuxiliaryInfo).ValidCanoto() { + if c.AuxiliaryInfoBatch != nil && !(c.AuxiliaryInfoBatch).ValidCanoto() { return false } return true @@ -354,10 +354,10 @@ func (c *StateMachineMetadata) CalculateCanotoCache() { if fieldSize := (&c.ICMEpochInfo).CachedCanotoSize(); fieldSize != 0 { size += uint64(len(canotoTag_StateMachineMetadata__ICMEpochInfo)) + canoto.SizeUint(fieldSize) + fieldSize } - if c.AuxiliaryInfo != nil { - (c.AuxiliaryInfo).CalculateCanotoCache() - fieldSize := (c.AuxiliaryInfo).CachedCanotoSize() - size += uint64(len(canotoTag_StateMachineMetadata__AuxiliaryInfo)) + canoto.SizeUint(fieldSize) + fieldSize + if c.AuxiliaryInfoBatch != nil { + (c.AuxiliaryInfoBatch).CalculateCanotoCache() + fieldSize := (c.AuxiliaryInfoBatch).CachedCanotoSize() + size += uint64(len(canotoTag_StateMachineMetadata__AuxiliaryInfoBatch)) + canoto.SizeUint(fieldSize) + fieldSize } atomic.StoreUint64(&c.canotoData.size, size) } @@ -425,11 +425,11 @@ func (c *StateMachineMetadata) MarshalCanotoInto(w canoto.Writer) canoto.Writer canoto.AppendUint(&w, fieldSize) w = (&c.ICMEpochInfo).MarshalCanotoInto(w) } - if c.AuxiliaryInfo != nil { - fieldSize := (c.AuxiliaryInfo).CachedCanotoSize() - canoto.Append(&w, canotoTag_StateMachineMetadata__AuxiliaryInfo) + if c.AuxiliaryInfoBatch != nil { + fieldSize := (c.AuxiliaryInfoBatch).CachedCanotoSize() + canoto.Append(&w, canotoTag_StateMachineMetadata__AuxiliaryInfoBatch) canoto.AppendUint(&w, fieldSize) - w = (c.AuxiliaryInfo).MarshalCanotoInto(w) + w = (c.AuxiliaryInfoBatch).MarshalCanotoInto(w) } return w } @@ -631,203 +631,6 @@ func (c *ICMEpochInfo) MarshalCanotoInto(w canoto.Writer) canoto.Writer { return w } -const ( - canotoNumber_AuxiliaryInfo__Info = 1 - canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq = 2 - canotoNumber_AuxiliaryInfo__VersionID = 3 - - canotoTag_AuxiliaryInfo__Info = "\x0a" // canoto.Tag(canotoNumber_AuxiliaryInfo__Info, canoto.Len) - canotoTag_AuxiliaryInfo__PrevAuxInfoSeq = "\x10" // canoto.Tag(canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq, canoto.Varint) - canotoTag_AuxiliaryInfo__VersionID = "\x18" // canoto.Tag(canotoNumber_AuxiliaryInfo__VersionID, canoto.Varint) -) - -type canotoData_AuxiliaryInfo struct { - size uint64 -} - -// CanotoSpec returns the specification of this canoto message. -func (*AuxiliaryInfo) CanotoSpec(...reflect.Type) *canoto.Spec { - var zero AuxiliaryInfo - s := &canoto.Spec{ - Name: "AuxiliaryInfo", - Fields: []canoto.FieldType{ - { - FieldNumber: canotoNumber_AuxiliaryInfo__Info, - Name: "Info", - OneOf: "", - TypeBytes: true, - }, - { - FieldNumber: canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq, - Name: "PrevAuxInfoSeq", - OneOf: "", - TypeUint: canoto.SizeOf(zero.PrevAuxInfoSeq), - }, - { - FieldNumber: canotoNumber_AuxiliaryInfo__VersionID, - Name: "VersionID", - OneOf: "", - TypeUint: canoto.SizeOf(zero.VersionID), - }, - }, - } - s.CalculateCanotoCache() - return s -} - -// UnmarshalCanoto unmarshals a Canoto-encoded byte slice into the struct. -// -// During parsing, the canoto cache is saved. -func (c *AuxiliaryInfo) UnmarshalCanoto(bytes []byte) error { - r := canoto.Reader{ - B: bytes, - } - return c.UnmarshalCanotoFrom(r) -} - -// UnmarshalCanotoFrom populates the struct from a [canoto.Reader]. Most users -// should just use UnmarshalCanoto. -// -// During parsing, the canoto cache is saved. -// -// This function enables configuration of reader options. -func (c *AuxiliaryInfo) UnmarshalCanotoFrom(r canoto.Reader) error { - // Zero the struct before unmarshaling. - *c = AuxiliaryInfo{} - atomic.StoreUint64(&c.canotoData.size, uint64(len(r.B))) - - var minField uint32 - for canoto.HasNext(&r) { - field, wireType, err := canoto.ReadTag(&r) - if err != nil { - return err - } - if field < minField { - return canoto.ErrInvalidFieldOrder - } - - switch field { - case canotoNumber_AuxiliaryInfo__Info: - if wireType != canoto.Len { - return canoto.ErrUnexpectedWireType - } - - if err := canoto.ReadBytes(&r, &c.Info); err != nil { - return err - } - if len(c.Info) == 0 { - return canoto.ErrZeroValue - } - case canotoNumber_AuxiliaryInfo__PrevAuxInfoSeq: - if wireType != canoto.Varint { - return canoto.ErrUnexpectedWireType - } - - if err := canoto.ReadUint(&r, &c.PrevAuxInfoSeq); err != nil { - return err - } - if canoto.IsZero(c.PrevAuxInfoSeq) { - return canoto.ErrZeroValue - } - case canotoNumber_AuxiliaryInfo__VersionID: - if wireType != canoto.Varint { - return canoto.ErrUnexpectedWireType - } - - if err := canoto.ReadUint(&r, &c.VersionID); err != nil { - return err - } - if canoto.IsZero(c.VersionID) { - return canoto.ErrZeroValue - } - default: - return canoto.ErrUnknownField - } - - minField = field + 1 - } - return nil -} - -// ValidCanoto validates that the struct can be correctly marshaled into the -// Canoto format. -// -// Specifically, ValidCanoto ensures: -// 1. All OneOfs are specified at most once. -// 2. All strings are valid utf-8. -// 3. All custom fields are ValidCanoto. -func (c *AuxiliaryInfo) ValidCanoto() bool { - return true -} - -// CalculateCanotoCache populates size and OneOf caches based on the current -// values in the struct. -// -// It is not safe to copy this struct concurrently. -func (c *AuxiliaryInfo) CalculateCanotoCache() { - var size uint64 - if len(c.Info) != 0 { - size += uint64(len(canotoTag_AuxiliaryInfo__Info)) + canoto.SizeBytes(c.Info) - } - if !canoto.IsZero(c.PrevAuxInfoSeq) { - size += uint64(len(canotoTag_AuxiliaryInfo__PrevAuxInfoSeq)) + canoto.SizeUint(c.PrevAuxInfoSeq) - } - if !canoto.IsZero(c.VersionID) { - size += uint64(len(canotoTag_AuxiliaryInfo__VersionID)) + canoto.SizeUint(c.VersionID) - } - atomic.StoreUint64(&c.canotoData.size, size) -} - -// CachedCanotoSize returns the previously calculated size of the Canoto -// representation from CalculateCanotoCache. -// -// If CalculateCanotoCache has not yet been called, it will return 0. -// -// If the struct has been modified since the last call to CalculateCanotoCache, -// the returned size may be incorrect. -func (c *AuxiliaryInfo) CachedCanotoSize() uint64 { - return atomic.LoadUint64(&c.canotoData.size) -} - -// MarshalCanoto returns the Canoto representation of this struct. -// -// It is assumed that this struct is ValidCanoto. -// -// It is not safe to copy this struct concurrently. -func (c *AuxiliaryInfo) MarshalCanoto() []byte { - c.CalculateCanotoCache() - w := canoto.Writer{ - B: make([]byte, 0, c.CachedCanotoSize()), - } - w = c.MarshalCanotoInto(w) - return w.B -} - -// MarshalCanotoInto writes the struct into a [canoto.Writer] and returns the -// resulting [canoto.Writer]. Most users should just use MarshalCanoto. -// -// It is assumed that CalculateCanotoCache has been called since the last -// modification to this struct. -// -// It is assumed that this struct is ValidCanoto. -// -// It is not safe to copy this struct concurrently. -func (c *AuxiliaryInfo) MarshalCanotoInto(w canoto.Writer) canoto.Writer { - if len(c.Info) != 0 { - canoto.Append(&w, canotoTag_AuxiliaryInfo__Info) - canoto.AppendBytes(&w, c.Info) - } - if !canoto.IsZero(c.PrevAuxInfoSeq) { - canoto.Append(&w, canotoTag_AuxiliaryInfo__PrevAuxInfoSeq) - canoto.AppendUint(&w, c.PrevAuxInfoSeq) - } - if !canoto.IsZero(c.VersionID) { - canoto.Append(&w, canotoTag_AuxiliaryInfo__VersionID) - canoto.AppendUint(&w, c.VersionID) - } - return w -} - const ( canotoNumber_SimplexEpochInfo__PChainReferenceHeight = 1 canotoNumber_SimplexEpochInfo__EpochNumber = 2 diff --git a/msm/encoding.go b/msm/encoding.go index 856105c6..29da324b 100644 --- a/msm/encoding.go +++ b/msm/encoding.go @@ -33,9 +33,9 @@ type StateMachineMetadata struct { Timestamp uint64 `canoto:"uint,5"` // ICMEpochInfo is the metadata that the StateMachine uses for ICM epoching. ICMEpochInfo ICMEpochInfo `canoto:"value,6"` - // AuxiliaryInfo is application-specific information that the StateMachine doesn't need to understand, + // AuxiliaryInfoBatch is application-specific information that the StateMachine doesn't need to understand, // but can be used by applications that care about epoch changes, such as threshold distributed public key generation. - AuxiliaryInfo *AuxiliaryInfo `canoto:"pointer,7"` + AuxiliaryInfoBatch *AuxiliaryInfoBatch `canoto:"pointer,7"` canotoData canotoData_StateMachineMetadata } @@ -50,7 +50,7 @@ func (smm *StateMachineMetadata) Clone() StateMachineMetadata { PChainHeight: smm.PChainHeight, Timestamp: smm.Timestamp, ICMEpochInfo: smm.ICMEpochInfo.Clone(), - AuxiliaryInfo: smm.AuxiliaryInfo.Clone(), + AuxiliaryInfoBatch: smm.AuxiliaryInfoBatch, } } @@ -88,48 +88,6 @@ func (ei *ICMEpochInfo) Equal(other *ICMEpochInfo) bool { return ei.EpochStartTime == other.EpochStartTime && ei.EpochNumber == other.EpochNumber && ei.PChainEpochHeight == other.PChainEpochHeight } -// AuxiliaryInfo defines application-specific information for applications that might care about epoch change, -// such as threshold distributed public key generation. -type AuxiliaryInfo struct { - // Info is opaque bytes that can be used by applications to encode any information that describes - // the current state for the application. - Info []byte `canoto:"bytes,1"` - // PrevAuxInfoSeq is a sequence number that applications can use to find previous AuxiliaryInfo in the chain. - // It is zero if this is the first AuxiliaryInfo for this epoch. - PrevAuxInfoSeq uint64 `canoto:"uint,2"` - // VersionID is an identifier that identifies the application. - // Can be used for backward-compatibility and upgrade purposes. - VersionID common.VersionID `canoto:"uint,3"` - - canotoData canotoData_AuxiliaryInfo -} - -func (ai *AuxiliaryInfo) Clone() *AuxiliaryInfo { - if ai == nil { - return nil - } - return &AuxiliaryInfo{ - Info: ai.Info, - PrevAuxInfoSeq: ai.PrevAuxInfoSeq, - VersionID: ai.VersionID, - } -} - -func (ai *AuxiliaryInfo) IsZero() bool { - var zero AuxiliaryInfo - return ai.Equal(&zero) -} - -func (ai *AuxiliaryInfo) Equal(a *AuxiliaryInfo) bool { - if ai == nil { - return a == nil - } - if a == nil { - return ai == nil - } - return bytes.Equal(ai.Info, a.Info) && ai.PrevAuxInfoSeq == a.PrevAuxInfoSeq && ai.VersionID == a.VersionID -} - // SimplexEpochInfo is metadata used by the StateMachine. type SimplexEpochInfo struct { // PChainReferenceHeight is the P-Chain height that the StateMachine uses as a reference for the current epoch. @@ -381,6 +339,15 @@ func (nbms NodeBLSMappings) Nodes() common.Nodes { return nodeWeights } +// NodeIDs returns the NodeIDs of the mappings. +func (nbms NodeBLSMappings) NodeIDs() []common.NodeID { + nodeIDs := make([]common.NodeID, len(nbms)) + for i := range nbms { + nodeIDs[i] = nbms[i].NodeID[:] + } + return nodeIDs +} + // IndexByNodeID returns a mapping from NodeID to the validator's index in the set, // which is the position used by approval bitmasks. func (nbms NodeBLSMappings) IndexByNodeID() map[avalanchego.NodeID]int { diff --git a/msm/fake_node_test.go b/msm/fake_node_test.go index 21042284..07c7c395 100644 --- a/msm/fake_node_test.go +++ b/msm/fake_node_test.go @@ -6,7 +6,6 @@ package metadata import ( "context" "crypto/rand" - "crypto/sha256" "fmt" "sync/atomic" "testing" @@ -18,7 +17,9 @@ import ( "github.com/stretchr/testify/require" ) -var emptyAuxInfoDigest = sha256.Sum256(nil) +// emptyAuxInfoDigest is the candidate digest approvals commit to when the auxiliary info +// history is empty: LastHistoryDigest returns the zero digest in that case. +var emptyAuxInfoDigest [32]byte func TestFakeNodeEpochChangesDespiteEmptyMempool(t *testing.T) { validatorSetRetriever := validatorSetRetriever{ diff --git a/msm/fuzz_test.go b/msm/fuzz_test.go index 3a2b84ac..c0ca37cb 100644 --- a/msm/fuzz_test.go +++ b/msm/fuzz_test.go @@ -6,7 +6,6 @@ package metadata import ( "bytes" "context" - "crypto/sha256" "testing" "time" @@ -72,7 +71,7 @@ const numBuiltBlocks = 8 // inputs (selected by index). For each input, a freshly instantiated verifier MSM first // verifies the unfuzzed block (which must succeed), then verifies a copy whose // consensus-authoritative metadata has been mutated (which must fail). -// + // The mutation is applied at the field level (rather than by flipping serialized bytes) // so the fuzzed block is always well-formed: byte-level mutations of the Canoto encoding // overwhelmingly corrupt the structure and merely exercise the decoder. Each fuzzed field @@ -116,8 +115,10 @@ func FuzzVerifyBlock(f *testing.F) { fuzzedMD := block.Metadata field.set(&fuzzedMD, value) - if fieldIdx%2 == 1 && block.Metadata.AuxiliaryInfo == nil { - fuzzedMD.AuxiliaryInfo = &AuxiliaryInfo{PrevAuxInfoSeq: value} + if fieldIdx%2 == 1 && block.Metadata.AuxiliaryInfoBatch == nil { + // value|1 forces a non-zero PrevAuxInfoSeq: collecting-approvals blocks reconstruct it + // as 0 (parent has no aux info), so value 0 would match and slip through unrejected. + fuzzedMD.AuxiliaryInfoBatch = &AuxiliaryInfoBatch{PrevAuxInfoSeq: value | 1} } if bytes.Equal(fuzzedMD.MarshalCanoto(), block.Metadata.MarshalCanoto()) { @@ -251,10 +252,11 @@ func buildEpochChain(tb testing.TB, logger common.Logger) ([]*StateMachineBlock, block3 := build(3, 2, 1, block2) addBlock(3, block3, nil) - // The noopTestAuxInfoApp is always "ready" with an empty aux info history, so the candidate - // aux info digest the builder signs over is sha256 of the empty history. Peer approvals must - // carry the same digest to survive sanitizeApprovals' digest filter. - auxInfoDigest := sha256.Sum256(nil) + // The noopTestAuxInfoApp is always "ready" with an empty aux info history, and + // LastHistoryDigest returns the zero digest for an empty history. That zero value is the + // candidate digest the builder signs over, so peer approvals must carry it to survive + // sanitizeApprovals' digest filter. + var auxInfoDigest [32]byte // block4 & block5: collecting-approvals blocks (1/3 then 2/3, not enough to seal). sm.HandleApproval(&common.ValidatorSetApproval{NodeID: node1, PChainHeight: pChainHeight2, AuxInfoDigest: auxInfoDigest, Signature: signApproval(pChainHeight2, auxInfoDigest)}, 1) diff --git a/msm/msm.go b/msm/msm.go index d149515f..aaf756cc 100644 --- a/msm/msm.go +++ b/msm/msm.go @@ -5,13 +5,11 @@ package metadata import ( "context" - "crypto/sha256" "encoding/asn1" "encoding/binary" "errors" "fmt" "math" - "slices" "sync" "time" @@ -152,7 +150,7 @@ type AuxiliaryInfoGenVerifier interface { // Generate generates an auxiliary information encoded as a byte slice based on the history of auxiliary information // for the given versionID in the current epoch so far. - // If this is the first invocation in the epoch, DefaultVersionID() should be passed as the VersionID. + // If this is the first invocation in the epoch, DefaultversionID() should be passed as the VersionID. // Otherwise, the versionID from previous blocks in the epoch should be used. // If the application deems the given history to be sufficient for the epoch change, it can return a nil byte slice, // in which case it will not be appended to the history. @@ -168,6 +166,8 @@ type StateMachine struct { lock sync.RWMutex approvalStore *ApprovalStore approvalStoreValidatorSet NodeBLSMappings + + auxInfoStore *auxInfoStore } // Config contains the dependencies and configuration parameters needed to initialize the StateMachine. @@ -235,10 +235,18 @@ func NewStateMachine(config *Config) (*StateMachine, error) { if config.TimeSkewLimit == 0 { config.TimeSkewLimit = maxSkew } - sm := StateMachine{Config: config} + sm := StateMachine{Config: config, auxInfoStore: newAuxInfoStore(config.AuxiliaryInfoApp)} return &sm, nil } +// HandleAuxiliaryMessage processes +func (sm *StateMachine) HandleAuxiliaryInfo(info common.AuxiliaryInfo, from avalanchego.NodeID) { + sm.auxInfoStore.HandleAuxiliaryMessage(info, from) +} + +// HandleApproval processes a validator set approval from a node. +// timestamp is the time the approval was received, in milliseconds +// elapsed since January 1, 1970 UTC. func (sm *StateMachine) HandleApproval(approval *common.ValidatorSetApproval, timestamp uint64) { sm.lock.Lock() approvalStore := sm.approvalStore @@ -254,6 +262,24 @@ func (sm *StateMachine) HandleApproval(approval *common.ValidatorSetApproval, ti approvalStore.HandleApproval(approval, timestamp) } +// InitializeApprovalStore initializes the approval store for the given validator set +// if it is not already initialized for it. +func (sm *StateMachine) InitializeApprovalStore(validatorSet NodeBLSMappings) { + sm.maybeInitializeApprovalStore(validatorSet) +} + +// Approvals returns the approvals accumulated in the approval store, +// or nil if the store has not been initialized. +func (sm *StateMachine) Approvals() ValidatorSetApprovals { + sm.lock.RLock() + defer sm.lock.RUnlock() + + if sm.approvalStore == nil { + return nil + } + return sm.approvalStore.Approvals() +} + func (sm *StateMachine) maybeInitializeApprovalStore(validatorSet NodeBLSMappings) *ApprovalStore { sm.lock.Lock() defer sm.lock.Unlock() @@ -289,6 +315,7 @@ func (sm *StateMachine) BuildBlock(ctx context.Context, metadata common.Protocol sm.Logger.Debug("Building block", zap.Uint64("seq", metadata.Seq), + zap.Uint64("round", metadata.Round), zap.Uint64("epoch", metadata.Epoch), zap.Stringer("prevHash", metadata.Prev)) @@ -296,6 +323,7 @@ func (sm *StateMachine) BuildBlock(ctx context.Context, metadata common.Protocol elapsed := time.Since(start) sm.Logger.Debug("Built block", zap.Uint64("seq", metadata.Seq), + zap.Uint64("round", metadata.Round), zap.Uint64("epoch", metadata.Epoch), zap.Stringer("prevHash", metadata.Prev), zap.Duration("elapsed", elapsed), @@ -502,7 +530,7 @@ func verifyAgainstExpected( nextBlock *StateMachineBlock, timestamp time.Time, expectedIcmEpochInfo ICMEpochInfo, - auxInfo *AuxiliaryInfo, + auxInfo *AuxiliaryInfoBatch, ) error { // First verify the metadata matches the expected values, only afterwards verify the inner block, if any. expectedBlock := wrapBlock( @@ -879,14 +907,19 @@ func (sm *StateMachine) buildBlockCollectingApprovals(ctx context.Context, paren return nil, err } - auxInfo, isAuxInfoReadyForEpochTransition, auxInfoDigest, err := sm.computeAuxInfo(parentBlock, prevBlockSeq, validators) + auxInfoHistory, err := GetAuxiliaryHistory(parentBlock, prevBlockSeq, sm.GetBlock, sm.AuxiliaryInfoApp.DefaultVersionID()) if err != nil { - return nil, fmt.Errorf("failed to compute auxiliary info: %w", err) + return nil, err + } + + isAuxInfoReadyForEpochTransition, err := sm.AuxiliaryInfoApp.IsSufficient(auxInfoHistory.OldestVersionID, validators, auxInfoHistory.Data) + if err != nil { + return nil, fmt.Errorf("failed to check if auxiliary info history is final: %w", err) } var newApprovals *approvals if isAuxInfoReadyForEpochTransition { - newApprovals, err = sm.computeNewApprovals(parentBlock, validators, auxInfoDigest) + newApprovals, err = sm.computeNewApprovals(parentBlock, validators, auxInfoHistory.LastHistoryDigest()) if err != nil { return nil, err } @@ -902,7 +935,10 @@ func (sm *StateMachine) buildBlockCollectingApprovals(ctx context.Context, paren now := sm.GetTime() icmEpochInfo := computeICMEpochInfo(parentBlock, sm.ComputeICMEpoch, now) - + auxInfo, err := sm.buildAuxInfoBatch(auxInfoHistory, parentBlock, validators, !isAuxInfoReadyForEpochTransition) + if err != nil { + return nil, fmt.Errorf("failed to build the auxiliary info batch: %w", err) + } // We might not have enough approvals to seal the current epoch, // in which case we just carry over the approvals we have so far to the next block, // so that eventually we'll have enough approvals to seal the epoch. @@ -1026,6 +1062,19 @@ func assembleApprovalToBeSigned(pChainHeight uint64, auxInfoDigest [32]byte) ([] return asn1.Marshal(signedMsg) } +func SignApproval(signer common.Signer, nextPChainReferenceHeight uint64, auxInfoDigest [32]byte) ([]byte, error) { + toBeSigned, err := assembleApprovalToBeSigned(nextPChainReferenceHeight, auxInfoDigest) + if err != nil { + return nil, err + } + + sig, err := signer.Sign(toBeSigned) + if err != nil { + return nil, fmt.Errorf("failed to sign approval: %w", err) + } + return sig, nil +} + func (sm *StateMachine) aggregatePubKeysForBitmask(nodeIDsBitmask []byte, validators NodeBLSMappings) ([]byte, error) { approvingNodes := avalanchego.BitmaskFromBytes(nodeIDsBitmask) publicKeys := make([][]byte, 0, len(validators)) @@ -1077,8 +1126,7 @@ func (sm *StateMachine) computeNewApprovals(parentBlock *StateMachineBlock, vali // Optimistically sign the epoch transition even if we have already did so in a previous round. // We'll just deduplicate this approval later on. - - sig, err := sm.createSelfApproval(prevBlockNextPChainReferenceHeight, auxInfoDigest) + sig, err := SignApproval(sm.Signer, prevBlockNextPChainReferenceHeight, auxInfoDigest) if err != nil { return nil, err } @@ -1090,6 +1138,8 @@ func (sm *StateMachine) computeNewApprovals(parentBlock *StateMachineBlock, vali Signature: sig, }) + sm.Logger.Debug("Retrieved approvals from peers", zap.Int("numApprovals", len(approvalsFromPeers))) + nextPChainHeight := prevBlockNextPChainReferenceHeight prevNextEpochApprovals := parentBlock.Metadata.SimplexEpochInfo.NextEpochApprovals @@ -1100,81 +1150,6 @@ func (sm *StateMachine) computeNewApprovals(parentBlock *StateMachineBlock, vali return newApprovals, nil } -func (sm *StateMachine) createSelfApproval(nextPChainReferenceHeight uint64, auxInfoDigest [32]byte) ([]byte, error) { - toBeSigned, err := assembleApprovalToBeSigned(nextPChainReferenceHeight, auxInfoDigest) - if err != nil { - return nil, err - } - - sig, err := sm.Signer.Sign(toBeSigned) - if err != nil { - return nil, fmt.Errorf("failed to sign approval: %w", err) - } - return sig, nil -} - -type auxInfoHistory struct { - data [][]byte - lastSeq uint64 -} - -func (aih *auxInfoHistory) lastHistory() []byte { - if len(aih.data) == 0 { - return nil - } - return aih.data[len(aih.data)-1] -} - -// collectAuxiliaryInfo traverses backwards starting from the given block and collects the AuxiliaryInfo of all blocks in the chain. -// returns the collected AuxiliaryInfo, the corresponding sequences of the blocks they were collected from, -// and the application ID of the oldest block that contains a non empty Info (or defaultVersionID if there was none). -func collectAuxiliaryInfo(block *StateMachineBlock, startSeq uint64, getBlock BlockRetriever, defaultVersionID common.VersionID) (auxInfoHistory, common.VersionID, error) { - var lastSeq *uint64 - var history [][]byte - var versionID = defaultVersionID - - // We traverse the chain of blocks backwards in the following manner: - // (1) Every block that doesn't have AuxiliaryInfo, its parents also do not have AuxiliaryInfo. - // (2) Every block that has AuxiliaryInfo, its descendants also have AuxiliaryInfo. - // (3) A block that has AuxiliaryInfo may have an empty Info field, but its PrevAuxInfoSeq field must point - // to a block that its AuxiliaryInfo isn't nil, and its Info field is also non-nil. - // (4) When a block with an empty Info field is built on a parent block that has AuxiliaryInfo, - // if its parent block has a non-empty Info field, then the block's PrevAuxInfoSeq points to its parent block. - // Else, its parent block has an empty Info field, then the block's PrevAuxInfoSeq is inherited from its parent block's PrevAuxInfoSeq. - - auxInfo := block.Metadata.AuxiliaryInfo - currentSeq := startSeq - for auxInfo != nil { - if len(auxInfo.Info) > 0 { - history = append(history, auxInfo.Info) - if lastSeq == nil { - lastSeq = new(uint64) - *lastSeq = currentSeq - } - versionID = auxInfo.VersionID - } - if auxInfo.PrevAuxInfoSeq == 0 { - // This is the first auxiliary info of the epoch, we can stop traversing back. - break - } - currentSeq = auxInfo.PrevAuxInfoSeq - prevBlock, _, err := getBlock(auxInfo.PrevAuxInfoSeq, [32]byte{}) - if err != nil { - return auxInfoHistory{}, 0, fmt.Errorf("%w: at sequence %d: %w", errAuxInfoBlockRetrieval, auxInfo.PrevAuxInfoSeq, err) - } - auxInfo = prevBlock.Metadata.AuxiliaryInfo - } - - if lastSeq == nil { - lastSeq = new(uint64) - *lastSeq = 0 - } - - // Reverse so the history (and the matching seqs) are ordered from oldest to newest. - slices.Reverse(history) - return auxInfoHistory{data: history, lastSeq: *lastSeq}, versionID, nil -} - // buildBlockImpatiently builds a block by waiting for the VM to build a block until MaxBlockBuildingWaitTime. // If the VM fails to build a block within that time, we build a block without an inner block, // so that we can continue making progress and not get stuck waiting for the VM. @@ -1185,7 +1160,7 @@ func (sm *StateMachine) buildBlockImpatiently(ctx context.Context, simplexEpochInfo SimplexEpochInfo, pChainHeight uint64, icmEpochInfo ICMEpochInfo, - auxInfo *AuxiliaryInfo) (*StateMachineBlock, error) { + auxInfo *AuxiliaryInfoBatch) (*StateMachineBlock, error) { impatientContext, cancel := context.WithTimeout(ctx, sm.MaxBlockBuildingWaitTime) defer cancel() @@ -1212,7 +1187,7 @@ func (sm *StateMachine) createSealingBlock(ctx context.Context, simplexEpochInfo SimplexEpochInfo, pChainHeight uint64, icmEpochInfo ICMEpochInfo, - auxInfo *AuxiliaryInfo) (*StateMachineBlock, error) { + auxInfo *AuxiliaryInfoBatch) (*StateMachineBlock, error) { simplexEpochInfo, err := sm.computeSimplexEpochInfoForSealingBlock(simplexEpochInfo) if err != nil { return nil, fmt.Errorf("failed to compute simplex epoch info for sealing block: %w", err) @@ -1253,7 +1228,7 @@ func wrapBlock( simplexBlacklist common.Blacklist, timestamp time.Time, icmEpochInfo ICMEpochInfo, - auxiliaryInfo *AuxiliaryInfo) *StateMachineBlock { + auxiliaryInfo *AuxiliaryInfoBatch) *StateMachineBlock { return &StateMachineBlock{ InnerBlock: innerBlock, @@ -1264,7 +1239,7 @@ func wrapBlock( SimplexEpochInfo: newSimplexEpochInfo, PChainHeight: pChainHeight, ICMEpochInfo: icmEpochInfo, - AuxiliaryInfo: auxiliaryInfo, + AuxiliaryInfoBatch: auxiliaryInfo, }, } } @@ -1389,19 +1364,19 @@ func (sm *StateMachine) verifyBlockEpochSealed(ctx context.Context, parentBlock // computeExpectedAuxInfoForApprovalCollection computes the expected AuxiliaryInfo that should be included in the proposed block // for approval collection, and returns the auxiliary info digest, and whether the auxiliary info history is ready for epoch transition. -func (sm *StateMachine) computeExpectedAuxInfoForApprovalCollection(parentBlock *StateMachineBlock, nextBlock *StateMachineBlock, prevBlockSeq uint64, validators NodeBLSMappings) (*AuxiliaryInfo, [32]byte, bool, error) { +func (sm *StateMachine) computeExpectedAuxInfoForApprovalCollection(parentBlock *StateMachineBlock, nextBlock *StateMachineBlock, prevBlockSeq uint64, validators NodeBLSMappings) (*AuxiliaryInfoBatch, [32]byte, bool, error) { nextMD := nextBlock.Metadata prevMD := parentBlock.Metadata - auxInfoHistory, versionID, err := collectAuxiliaryInfo(parentBlock, prevBlockSeq, sm.GetBlock, sm.AuxiliaryInfoApp.DefaultVersionID()) + auxInfoHistory, err := GetAuxiliaryHistory(parentBlock, prevBlockSeq, sm.GetBlock, sm.AuxiliaryInfoApp.DefaultVersionID()) if err != nil { return nil, [32]byte{}, false, err } - if len(auxInfoHistory.data) > 0 && nextMD.AuxiliaryInfo == nil { + if len(auxInfoHistory.Data) > 0 && nextMD.AuxiliaryInfoBatch == nil { // If we have auxiliary info history but the proposed block doesn't include any auxiliary info, // it means the block builder has dropped the auxiliary info, which is not allowed. - return nil, [32]byte{}, false, fmt.Errorf("expected auxiliary info for application %d with history length %d, but got nil", versionID, len(auxInfoHistory.data)) + return nil, [32]byte{}, false, fmt.Errorf("expected auxiliary info for application %d with history length %d, but got nil", auxInfoHistory.OldestVersionID, len(auxInfoHistory.Data)) } // Else, either len(auxInfoHistory) == 0, @@ -1409,86 +1384,66 @@ func (sm *StateMachine) computeExpectedAuxInfoForApprovalCollection(parentBlock // Both of these cases are fine, because a node doesn't have to include Auxiliary information. // We will verify the legality of the proposed auxiliary info (if any) in the next step. - var expectedAuxInfo *AuxiliaryInfo - var proposedAuxInf []byte + var expectedAuxInfo *AuxiliaryInfoBatch + var proposedAuxInfos []common.AuxiliaryInfo - if nextMD.AuxiliaryInfo != nil { - proposedAuxInf = nextMD.AuxiliaryInfo.Info - expectedAuxInfo = &AuxiliaryInfo{ - VersionID: versionID, - Info: proposedAuxInf, + if nextMD.AuxiliaryInfoBatch != nil { + proposedAuxInfos = nextMD.AuxiliaryInfoBatch.data + expectedAuxInfo = &AuxiliaryInfoBatch{ + data: proposedAuxInfos, } - if prevMD.AuxiliaryInfo != nil { - expectedAuxInfo.PrevAuxInfoSeq = auxInfoHistory.lastSeq + if prevMD.AuxiliaryInfoBatch != nil { + expectedAuxInfo.PrevAuxInfoSeq = auxInfoHistory.LastSeq } } - if err := sm.AuxiliaryInfoApp.IsLegalAppend(versionID, validators, auxInfoHistory.data, proposedAuxInf); err != nil { - return nil, [32]byte{}, false, fmt.Errorf("proposed auxiliary info is not a legal append to the history for application %d: %w", versionID, err) + // go through all the collected data and return whether proposed datum are legal + + for _, info := range proposedAuxInfos { + if auxInfoHistory.OldestVersionID != info.Version { + return nil, [32]byte{}, false, fmt.Errorf("proposed auxiliary info does not have the proper version %d: %w", auxInfoHistory.OldestVersionID, err) + } + if err := sm.AuxiliaryInfoApp.IsLegalAppend(auxInfoHistory.OldestVersionID, validators, auxInfoHistory.Data, info.Data); err != nil { + return nil, [32]byte{}, false, fmt.Errorf("proposed auxiliary info is not a legal append to the history for application %d: %w", auxInfoHistory.OldestVersionID, err) + } } - auxInfoReady, err := sm.AuxiliaryInfoApp.IsSufficient(versionID, validators, auxInfoHistory.data) + auxInfoReady, err := sm.AuxiliaryInfoApp.IsSufficient(auxInfoHistory.OldestVersionID, validators, auxInfoHistory.Data) if err != nil { - return nil, [32]byte{}, false, fmt.Errorf("failed to check if auxiliary info history is final for application %d: %w", versionID, err) + return nil, [32]byte{}, false, fmt.Errorf("failed to check if auxiliary info history is final for application %d: %w", auxInfoHistory.OldestVersionID, err) } var digest [32]byte if auxInfoReady { - digest = sha256.Sum256(auxInfoHistory.lastHistory()) + digest = auxInfoHistory.LastHistoryDigest() } return expectedAuxInfo, digest, auxInfoReady, nil } -// computeAuxInfo computes the AuxiliaryInfo that should be included in the block being built, and whether the auxiliary info history is ready for epoch transition, -func (sm *StateMachine) computeAuxInfo(parentBlock *StateMachineBlock, prevBlockSeq uint64, validators NodeBLSMappings) (*AuxiliaryInfo, bool, common.Digest, error) { - auxInfoHistory, versionID, err := collectAuxiliaryInfo(parentBlock, prevBlockSeq, sm.GetBlock, sm.AuxiliaryInfoApp.DefaultVersionID()) - if err != nil { - return nil, false, common.Digest{}, err - } - - isAuxInfoReadyForEpochTransition, err := sm.AuxiliaryInfoApp.IsSufficient(versionID, validators, auxInfoHistory.data) - if err != nil { - return nil, false, common.Digest{}, fmt.Errorf("failed to check if auxiliary info history is final: %w", err) - } - - var auxInfo *AuxiliaryInfo - parentAuxInfo := parentBlock.Metadata.AuxiliaryInfo - if parentAuxInfo != nil { - auxInfo = &AuxiliaryInfo{ - VersionID: parentAuxInfo.VersionID, - PrevAuxInfoSeq: auxInfoHistory.lastSeq, - } +// buildAuxInfoBatch builds the AuxiliaryInfoBatch that should be included in the block being built. +func (sm *StateMachine) buildAuxInfoBatch(history AuxInfoHistory, parentBlock *StateMachineBlock, validators NodeBLSMappings, shouldGenerate bool) (*AuxiliaryInfoBatch, error) { + var prevAuxInfoSeq uint64 + if parentBlock.Metadata.AuxiliaryInfoBatch != nil { + prevAuxInfoSeq = history.LastSeq } - if !isAuxInfoReadyForEpochTransition { - // If the auxiliary info isn't ready for epoch transition, - // we should focus on contributing to finalizing it before collecting approvals for the epoch transition, - // as without it being ready, we won't be able to transition epochs anyway. - auxInf, err := sm.AuxiliaryInfoApp.Generate(versionID, validators, auxInfoHistory.data) - if err != nil { - return nil, false, common.Digest{}, fmt.Errorf("failed to generate auxiliary info: %w", err) - } - if auxInfo == nil { - // This is the first auxiliary info we're generating for this epoch, - // so we need to initialize it. - auxInfo = &AuxiliaryInfo{ - VersionID: versionID, - Info: auxInf, - } - } else { - // Otherwise, we already have auxiliary info from the parent block, - // so we just update the Info field and carry over the VersionID and PrevAuxInfoSeq. - auxInfo.Info = auxInf - } + var info []common.AuxiliaryInfo + if shouldGenerate { + info = sm.auxInfoStore.collectAuxInfo(history, validators) } - var auxInfoDigest common.Digest - if isAuxInfoReadyForEpochTransition { - auxInfoDigest = sha256.Sum256(auxInfoHistory.lastHistory()) + // Only emit a batch when there's new info to record, or a prior batch in the chain + // to link back to. An empty batch with PrevAuxInfoSeq == 0 would violate the invariant + // that an empty batch points to an ancestor with non-empty entries. + if len(info) == 0 && prevAuxInfoSeq == 0 { + return nil, nil } - return auxInfo, isAuxInfoReadyForEpochTransition, auxInfoDigest, nil + return &AuxiliaryInfoBatch{ + data: info, + PrevAuxInfoSeq: prevAuxInfoSeq, + }, nil } // constructSimplexZeroBlockSimplexEpochInfo constructs the SimplexEpochInfo for the zero block, which is the first ever block built by Simplex. diff --git a/msm/msm_test.go b/msm/msm_test.go index ddbd0df9..ed1fcc98 100644 --- a/msm/msm_test.go +++ b/msm/msm_test.go @@ -7,7 +7,6 @@ import ( "context" "crypto/rand" "crypto/sha256" - "errors" "fmt" "math" "testing" @@ -1596,8 +1595,9 @@ func TestVerifyCollectingApprovalsNotReady(t *testing.T) { sm, tc, parent := newSM(t) block := build(t, sm, tc, parent) - // The builder generated auxiliary info but collected no approvals. - require.NotNil(t, block.Metadata.AuxiliaryInfo) + // No auxiliary info was received and the history isn't ready, so the builder collects + // neither auxiliary info (nil batch) nor approvals. + require.Nil(t, block.Metadata.AuxiliaryInfoBatch) require.Empty(t, block.Metadata.SimplexEpochInfo.NextEpochApprovals.NodeIDs) require.Empty(t, block.Metadata.SimplexEpochInfo.NextEpochApprovals.Signature) @@ -1650,7 +1650,7 @@ func TestCollectingApprovalsAuxInfoGating(t *testing.T) { vote1 := []byte("vote-1") vote2 := []byte("vote-2") votes := [][]byte{vote1, vote2} - sm.AuxiliaryInfoApp = &voteCountingAuxInfoApp{ + auxiliaryApp := &voteCountingAuxInfoApp{ threshold: 2, randomTape: func() []byte { next := votes[0] @@ -1658,10 +1658,9 @@ func TestCollectingApprovalsAuxInfoGating(t *testing.T) { return next }, } + sm.AuxiliaryInfoApp = auxiliaryApp - // A 3-node validator set including MyNodeID at index 0, so the optimistic self-approval - // is retained once approvals are collected, but a single approval is below quorum (the - // block stays in the collecting state rather than sealing). + // A 3-node validator set including MyNodeID at index 0 validators := NodeBLSMappings{ {NodeID: avalanchego.NodeID(sm.MyNodeID), BLSKey: []byte{1}, Weight: 1}, {NodeID: avalanchego.NodeID{0xBB}, BLSKey: []byte{2}, Weight: 1}, @@ -1686,9 +1685,9 @@ func TestCollectingApprovalsAuxInfoGating(t *testing.T) { } tc.blockStore[parentSeq] = &outerBlock{block: parent} - // build constructs the next collecting block on top of prev, stores it so it can serve + // buildAndVerify constructs the next collecting block on top of prev, stores it so it can serve // as a parent (and as a back-pointer target for the aux info history), and verifies it. - build := func(seq uint64, prev StateMachineBlock) *StateMachineBlock { + buildAndVerify := func(seq uint64, prev StateMachineBlock) *StateMachineBlock { tc.blockBuilder.Block = &testutil.InnerBlock{TS: time.Now(), BlockHeight: seq, Content: []byte{byte(seq)}} md := common.ProtocolMetadata{Seq: seq, Round: seq, Epoch: 1, Prev: prev.Digest()} block, err := sm.BuildBlock(context.Background(), md, emptyBlacklist) @@ -1702,29 +1701,45 @@ func TestCollectingApprovalsAuxInfoGating(t *testing.T) { return b.Metadata.SimplexEpochInfo.NextEpochApprovals } // requireAuxInfo compares the meaningful fields, ignoring the cached canoto size. - requireAuxInfo := func(want, got *AuxiliaryInfo) { + requireAuxInfo := func(want, got *AuxiliaryInfoBatch) { require.True(t, want.Equal(got), "expected aux info %+v, got %+v", want, got) } + auxVersionId := auxiliaryApp.DefaultVersionID() + firstAuxInfoBytes, err := auxiliaryApp.Generate(auxVersionId, validators, [][]byte{}) + require.NoError(t, err) + firstAuxInfo := common.AuxiliaryInfo{ + Version: auxVersionId, + Data: firstAuxInfoBytes, + } + sm.HandleAuxiliaryInfo(firstAuxInfo, validators[0].NodeID) + // block1: history empty, not final -> generates vote1, collects no approvals. - block1 := build(parentSeq+1, parent) - requireAuxInfo(&AuxiliaryInfo{Info: vote1, VersionID: 1}, block1.Metadata.AuxiliaryInfo) + block1 := buildAndVerify(parentSeq+1, parent) + requireAuxInfo(&AuxiliaryInfoBatch{data: []common.AuxiliaryInfo{firstAuxInfo}}, block1.Metadata.AuxiliaryInfoBatch) require.Empty(t, approvals(block1).NodeIDs) + // we get another auxiliary info sent + auxInfoHistory, err := GetAuxiliaryHistory(block1, parentSeq+1, sm.GetBlock, auxVersionId) + require.NoError(t, err) + secondAuxInfoBytes, err := auxiliaryApp.Generate(auxVersionId, validators, auxInfoHistory.Data) + require.NoError(t, err) + secondAuxInfo := common.AuxiliaryInfo{ + Version: auxVersionId, + Data: secondAuxInfoBytes, + } + sm.HandleAuxiliaryInfo(secondAuxInfo, validators[1].NodeID) + // block2: history [vote1], still not final -> generates vote2, collects no approvals. - block2 := build(parentSeq+2, *block1) - requireAuxInfo(&AuxiliaryInfo{Info: vote2, PrevAuxInfoSeq: parentSeq + 1, VersionID: 1}, block2.Metadata.AuxiliaryInfo) + block2 := buildAndVerify(parentSeq+2, *block1) + requireAuxInfo(&AuxiliaryInfoBatch{data: []common.AuxiliaryInfo{secondAuxInfo}, PrevAuxInfoSeq: parentSeq + 1}, block2.Metadata.AuxiliaryInfoBatch) require.Empty(t, approvals(block2).NodeIDs) - // block3: history [vote1, vote2] is now final -> no new vote, and approvals are - // collected (the optimistic self-approval sets MyNodeID's bit). block3 is the first - // empty-Info block; it points at block2, the last non-empty Info block. - block3 := build(parentSeq+3, *block2) - requireAuxInfo(&AuxiliaryInfo{PrevAuxInfoSeq: parentSeq + 2, VersionID: 1}, block3.Metadata.AuxiliaryInfo) - require.Equal(t, []byte{1}, approvals(block3).NodeIDs, "self-approval bit should be set once aux info is ready") + block3 := buildAndVerify(parentSeq+3, *block2) + requireAuxInfo(&AuxiliaryInfoBatch{PrevAuxInfoSeq: parentSeq + 2}, block3.Metadata.AuxiliaryInfoBatch) // The collected approval must be signed over the epoch-transition payload for the - //mnext epoch's P-chain reference height (200) and the digest + //next epoch's P-chain reference height (200) and the digest // of the final auxiliary info history, which is sha256 of the last vote (vote2). wantSigned, err := assembleApprovalToBeSigned(nextPChainRefHeight, sha256.Sum256(vote2)) require.NoError(t, err) @@ -1735,16 +1750,16 @@ func TestCollectingApprovalsAuxInfoGating(t *testing.T) { // quorum). Its PrevAuxInfoSeq must SKIP the empty block3 and point at block2 (parentSeq+2), // the most recent non-empty Info block -- not at its immediate parent block3 (parentSeq+3). // This is the case the rest of the chain never reaches and where "skip" differs from "successive". - block4 := build(parentSeq+4, *block3) - require.NotEqual(t, parentSeq+3, block4.Metadata.AuxiliaryInfo.PrevAuxInfoSeq, + block4 := buildAndVerify(parentSeq+4, *block3) + require.NotEqual(t, parentSeq+3, block4.Metadata.AuxiliaryInfoBatch.PrevAuxInfoSeq, "PrevAuxInfoSeq must not point at the empty-Info parent block3") - requireAuxInfo(&AuxiliaryInfo{PrevAuxInfoSeq: parentSeq + 2, VersionID: 1}, block4.Metadata.AuxiliaryInfo) + requireAuxInfo(&AuxiliaryInfoBatch{PrevAuxInfoSeq: parentSeq + 2}, block4.Metadata.AuxiliaryInfoBatch) // block5: another empty-Info block on top of the empty block4. The back-pointer still skips // the whole empty run and points at block2, confirming the skip persists across consecutive // empty-Info blocks (collectAuxiliaryInfo finds the same most-recent non-empty block each time). - block5 := build(parentSeq+5, *block4) - requireAuxInfo(&AuxiliaryInfo{PrevAuxInfoSeq: parentSeq + 2, VersionID: 1}, block5.Metadata.AuxiliaryInfo) + block5 := buildAndVerify(parentSeq+5, *block4) + requireAuxInfo(&AuxiliaryInfoBatch{PrevAuxInfoSeq: parentSeq + 2}, block5.Metadata.AuxiliaryInfoBatch) } func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { @@ -1752,14 +1767,16 @@ func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { // VersionID must be reused for the rest of the epoch -- for both building AND verifying // subsequent blocks -- even if the application's DefaultVersionID() later changes. // - // collectAuxiliaryInfo only consults DefaultVersionID() when the auxiliary info history is - // empty; once a block carries a VersionID, every later buildAndVerify and verify reads that VersionID - // back from the chain instead. So we seed the epoch's parent with auxiliary info stamped with - // VersionID 1, then flip DefaultVersionID() to 2 right after the first Generate(). Because the - // epoch already has a VersionID on-chain, every Generate()/IsLegalAppend()/IsSufficient() - // invocation -- on the buildAndVerify path and the verify path -- must keep using VersionID 1, never 2. - // The app asserts that internally: it requires the VersionID it receives to equal - // expectedVersionID, which we hold at 1 throughout. + // GetAuxiliaryHistory only consults DefaultVersionID() when the auxiliary info history is + // empty; once a block carries a VersionID, every later build and verify reads that VersionID + // back from the chain instead. Auxiliary info is no longer generated inside the block: it + // arrives from peers via HandleAuxiliaryInfo and is collected into the block being built. So the + // first collecting block establishes the epoch's VersionID (1) from the received vote while the + // default is still 1, then we flip DefaultVersionID() to 2. Because the epoch already carries a + // VersionID on-chain, every Generate()/IsLegalAppend()/IsSufficient() invocation -- on both the + // build and verify paths -- must keep using VersionID 1, never 2. The app asserts that + // internally: it requires the VersionID it receives to equal expectedVersionID, which we hold at + // 1 throughout. const ( pChainRefHeight = uint64(100) @@ -1771,10 +1788,11 @@ func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } - // threshold 4 so Generate() runs for the first three collecting blocks built on top of the - // pre-seeded parent (history not yet sufficient), giving us one "first" and several "later" - // Generate() invocations. defaultVersionID starts at 1 (the original default); expectedVersionID - // stays 1 for the whole test -- the app asserts every invocation uses it. + // threshold 4 so the history never becomes sufficient across the three collecting blocks we + // build: every block collects a freshly received auxiliary vote (never approvals), giving one + // "first" build under the original default and two "later" builds after the default changes. + // defaultVersionID starts at 1 (the original default); expectedVersionID stays 1 for the whole + // test -- the app asserts every invocation uses it. app := &versionRecordingAuxInfoApp{ t: t, threshold: 4, @@ -1791,8 +1809,8 @@ func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { } tc.validatorSetRetriever.result = validators - // The parent already carries auxiliary info for this epoch, stamped with VersionID 1. - // This is the backward-compatibility precondition: the epoch's VersionID is already set. + // A plain parent with no auxiliary info yet: the epoch's VersionID is established by the first + // received auxiliary vote rather than pre-seeded into the block. parent := StateMachineBlock{ InnerBlock: &testutil.InnerBlock{TS: time.Now(), BlockHeight: 1, Content: []byte{0xAA}}, Metadata: StateMachineMetadata{ @@ -1806,11 +1824,6 @@ func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { NextPChainReferenceHeight: nextPChainRefHeight, PrevVMBlockSeq: parentSeq - 1, }, - AuxiliaryInfo: &AuxiliaryInfo{ - VersionID: 1, - Info: []byte("vote-0"), - PrevAuxInfoSeq: 0, - }, }, } tc.blockStore[parentSeq] = &outerBlock{block: parent} @@ -1827,133 +1840,109 @@ func TestCollectingApprovalsAuxInfoVersionIDIsBackwardCompatible(t *testing.T) { return block } - // block1: the epoch already has VersionID 1 (from the parent), so the buildAndVerify reads 1 from the - // chain and generates vote-1 under VersionID 1. Being the first Generate(), we now flip the - // application's default to 2. Verifying block1 also reads VersionID 1 from the parent's aux - // info, so it passes despite the changed default. + // receiveAuxVote generates the next vote under VersionID 1 and delivers it as if received from + // the given validator, so the next built block collects it into its auxiliary info. + receiveAuxVote := func(from avalanchego.NodeID) { + data, err := app.Generate(app.expectedVersionID, validators, nil) + require.NoError(t, err) + sm.HandleAuxiliaryInfo(common.AuxiliaryInfo{Version: app.expectedVersionID, Data: data}, from) + } + + // block1: default is still 1 and the history is empty, so the received vote (VersionID 1) sets + // the epoch's VersionID. Building and verifying block1 both read 1 from the default. We then flip + // the default to 2; every later build/verify must keep reading 1 back from the chain. + receiveAuxVote(validators[0].NodeID) block1 := buildAndVerify(parentSeq+1, parent) - require.Equal(t, common.VersionID(1), block1.Metadata.AuxiliaryInfo.VersionID) + require.Equal(t, common.VersionID(1), block1.Metadata.AuxiliaryInfoBatch.data[0].Version) app.defaultVersionID = 2 - // block2, block3: the default is now 2, but each block's buildAndVerify and verify still read VersionID - // 1 back from the chain and ignore the changed default. + // block2, block3: the default is now 2, but each block's build and verify still read VersionID 1 + // back from the chain and ignore the changed default. + receiveAuxVote(validators[1].NodeID) block2 := buildAndVerify(parentSeq+2, *block1) - require.Equal(t, common.VersionID(1), block2.Metadata.AuxiliaryInfo.VersionID) + require.Equal(t, common.VersionID(1), block2.Metadata.AuxiliaryInfoBatch.data[0].Version) + receiveAuxVote(validators[2].NodeID) block3 := buildAndVerify(parentSeq+3, *block2) - require.Equal(t, common.VersionID(1), block3.Metadata.AuxiliaryInfo.VersionID) - - // block4: history [vote-0, vote-1, vote-2, vote-3] is now sufficient, so no further vote is - // generated and approvals are collected -- still under VersionID 1. - block4 := buildAndVerify(parentSeq+4, *block3) - require.Equal(t, common.VersionID(1), block4.Metadata.AuxiliaryInfo.VersionID) + require.Equal(t, common.VersionID(1), block3.Metadata.AuxiliaryInfoBatch.data[0].Version) } -func TestCollectAuxiliaryInfo(t *testing.T) { - const versionID = common.VersionID(7) +func TestCollectingApprovalsIncludesMultipleAuxInfoMessages(t *testing.T) { + // Multiple auxiliary info messages received from distinct validators are all collected into a + // single built block's AuxiliaryInfoBatch. - blockWithAuxInfo := func(info []byte, prevAuxInfoSeq uint64) StateMachineBlock { - return StateMachineBlock{ - Metadata: StateMachineMetadata{ - AuxiliaryInfo: &AuxiliaryInfo{ - Info: info, - PrevAuxInfoSeq: prevAuxInfoSeq, - VersionID: versionID, - }, - }, - } - } + const ( + pChainRefHeight = uint64(100) + nextPChainRefHeight = uint64(200) + parentSeq = uint64(10) + ) - errRetrieval := errors.New("retrieval failed") + sm, tc := newStateMachine(t) + sm.GetPChainHeightForProposing = func() uint64 { return nextPChainRefHeight } + sm.GetPChainHeightForVerifying = func() uint64 { return nextPChainRefHeight } - // startSeq is the sequence of tt.block itself (the block collectAuxiliaryInfo starts from). - const startSeq = uint64(10) + // A high threshold keeps the history from ever becoming sufficient, so the block stays in the + // collecting-approvals state and carries the received auxiliary info instead of sealing. + sm.AuxiliaryInfoApp = &voteCountingAuxInfoApp{threshold: 10} - tests := []struct { - name string - block StateMachineBlock - blocks map[uint64]StateMachineBlock - getBlockErr error - expectedHistory [][]byte - expectedLastSeq uint64 - expectedversionID common.VersionID - expectedErr error - }{ - { - name: "block without auxiliary info", - block: StateMachineBlock{}, - }, - { - name: "empty info, first of epoch", - block: blockWithAuxInfo(nil, 0), - }, - { - name: "non-empty info, first of epoch", - block: blockWithAuxInfo([]byte{1}, 0), - expectedHistory: [][]byte{{1}}, - expectedLastSeq: startSeq, - expectedversionID: versionID, - }, - { - name: "empty info pointing back to non-empty info", - block: blockWithAuxInfo(nil, 3), - blocks: map[uint64]StateMachineBlock{ - 3: blockWithAuxInfo([]byte{1}, 0), - }, - expectedHistory: [][]byte{{1}}, - expectedLastSeq: 3, - expectedversionID: versionID, - }, - { - name: "history is ordered from oldest to newest", - block: blockWithAuxInfo([]byte{3}, 5), - blocks: map[uint64]StateMachineBlock{ - 5: blockWithAuxInfo([]byte{2}, 2), - 2: blockWithAuxInfo([]byte{1}, 0), + validators := NodeBLSMappings{ + {NodeID: avalanchego.NodeID(sm.MyNodeID), BLSKey: []byte{1}, Weight: 1}, + {NodeID: avalanchego.NodeID{0xBB}, BLSKey: []byte{2}, Weight: 1}, + {NodeID: avalanchego.NodeID{0xCC}, BLSKey: []byte{3}, Weight: 1}, + } + tc.validatorSetRetriever.result = validators + + parent := StateMachineBlock{ + InnerBlock: &testutil.InnerBlock{TS: time.Now(), BlockHeight: 1, Content: []byte{0xAA}}, + Metadata: StateMachineMetadata{ + PChainHeight: nextPChainRefHeight, + SimplexProtocolMetadata: common.ProtocolMetadata{ + Seq: parentSeq, Round: 5, Epoch: 1, }, - expectedHistory: [][]byte{{1}, {2}, {3}}, - expectedLastSeq: startSeq, - expectedversionID: versionID, - }, - { - name: "traversal stops at a block without auxiliary info", - block: blockWithAuxInfo([]byte{2}, 4), - blocks: map[uint64]StateMachineBlock{ - 4: {}, + SimplexEpochInfo: SimplexEpochInfo{ + PChainReferenceHeight: pChainRefHeight, + EpochNumber: 1, + NextPChainReferenceHeight: nextPChainRefHeight, + PrevVMBlockSeq: parentSeq - 1, }, - expectedHistory: [][]byte{{2}}, - expectedLastSeq: startSeq, - expectedversionID: versionID, - }, - { - name: "block retrieval failure", - block: blockWithAuxInfo([]byte{2}, 4), - getBlockErr: errRetrieval, - expectedErr: errRetrieval, }, } + tc.blockStore[parentSeq] = &outerBlock{block: parent} - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - getBlock := func(seq uint64, _ common.Digest) (StateMachineBlock, *common.Finalization, error) { - if tt.getBlockErr != nil { - return StateMachineBlock{}, nil, tt.getBlockErr - } - block, ok := tt.blocks[seq] - require.True(t, ok, "unexpected retrieval of block at sequence %d", seq) - return block, nil, nil - } + version := sm.AuxiliaryInfoApp.DefaultVersionID() - history, gotversionID, err := collectAuxiliaryInfo(&tt.block, startSeq, getBlock, 0) - if tt.expectedErr != nil { - require.ErrorIs(t, err, tt.expectedErr) - require.ErrorIs(t, err, errAuxInfoBlockRetrieval) - return - } - require.NoError(t, err) - require.Equal(t, tt.expectedHistory, history.data) - require.Equal(t, tt.expectedLastSeq, history.lastSeq) - require.Equal(t, tt.expectedversionID, gotversionID) - }) + // buildWithAuxMessages delivers one distinct auxiliary message per validator, builds a block on + // top of prev, verifies and stores it, and asserts the block collected exactly those messages. + // collectAuxInfo orders entries by NodeID, so the payloads are compared as a set. + buildWithAuxMessages := func(seq uint64, prev StateMachineBlock, payloads [][]byte) *StateMachineBlock { + require.Len(t, payloads, len(validators)) + for i, payload := range payloads { + sm.HandleAuxiliaryInfo(common.AuxiliaryInfo{Version: version, Data: payload}, validators[i].NodeID) + } + + tc.blockBuilder.Block = &testutil.InnerBlock{TS: time.Now(), BlockHeight: seq, Content: []byte{byte(seq)}} + md := common.ProtocolMetadata{Seq: seq, Round: seq, Epoch: 1, Prev: prev.Digest()} + block, err := sm.BuildBlock(context.Background(), md, emptyBlacklist) + require.NoError(t, err) + require.NoError(t, sm.VerifyBlock(context.Background(), block)) + tc.blockStore[seq] = &outerBlock{block: *block} + + require.NotNil(t, block.Metadata.AuxiliaryInfoBatch) + gotPayloads := make([][]byte, 0, len(payloads)) + for _, info := range block.Metadata.AuxiliaryInfoBatch.data { + require.Equal(t, version, info.Version) + gotPayloads = append(gotPayloads, info.Data) + } + require.ElementsMatch(t, payloads, gotPayloads) + return block } + + // First batch of three messages lands in block1. + block1 := buildWithAuxMessages(parentSeq+1, parent, [][]byte{[]byte("aux-a"), []byte("aux-b"), []byte("aux-c")}) + + // A second batch of three messages lands in block2, built on top of block1. + block2 := buildWithAuxMessages(parentSeq+2, *block1, [][]byte{[]byte("aux-d"), []byte("aux-e"), []byte("aux-f")}) + + // block2 links back to block1, the most recent block carrying non-empty auxiliary info. + require.Equal(t, parentSeq+1, block2.Metadata.AuxiliaryInfoBatch.PrevAuxInfoSeq) } From 0e9405cab9d112788ff55179d2a884a27f36a054 Mon Sep 17 00:00:00 2001 From: samliok Date: Thu, 13 Aug 2026 18:15:52 -0400 Subject: [PATCH 3/6] Wire epoch transition listener into the instance Construct an epochTransitionListener in NewInstance and hook its onIndex into both the validator and non-validator storage paths, so indexing a transition block generates and broadcasts auxiliary info, or signs and broadcasts an epoch transition approval once the history is sufficient. Route incoming AuxiliaryInfo and EpochTransitionApproval messages into the MSM. --- instance.go | 57 ++++++--- transition_listener.go | 151 ++++++++++++++++++++++ transition_listener_test.go | 249 ++++++++++++++++++++++++++++++++++++ 3 files changed, 439 insertions(+), 18 deletions(-) create mode 100644 transition_listener.go create mode 100644 transition_listener_test.go diff --git a/instance.go b/instance.go index c0ebd0ac..7ec186f3 100644 --- a/instance.go +++ b/instance.go @@ -57,8 +57,6 @@ type epochChange struct { epoch uint64 validators common.Nodes } - -func noopOnIndex(*ParsedBlock) error { return nil } type timeAdvancer interface { AdvanceTime(t time.Time) } @@ -66,24 +64,40 @@ type timeAdvancer interface { type Instance struct { Config Config - lock sync.Mutex - started bool - cs *CachedStorage - wal *wal.GarbageCollectedWAL - msm *metadata.StateMachine - e *simplex.Epoch - nv *nonvalidator.NonValidator - epochOrNV timeAdvancer - epochChanges chan epochChange - stopCh chan struct{} + lock sync.Mutex + started bool + cs *CachedStorage + transitionListener *epochTransitionListener + wal *wal.GarbageCollectedWAL + msm *metadata.StateMachine + e *simplex.Epoch + nv *nonvalidator.NonValidator + epochOrNV timeAdvancer + epochChanges chan epochChange + stopCh chan struct{} } func NewInstance(config Config) *Instance { + cs := NewCachedStorage(config.Storage) + // Non-validators have no block builder, so they pass a nil approval handler: + // they broadcast approvals but do not need to record their own locally. + transitionListener := newEpochTransitionListener( + config.Logger, + config.Broadcaster, + avalanchego.NodeID(config.ID), + config.PlatformChain.GetValidatorSet, + cs.RetrieveBlock, + config.CryptoOps, + &NoopAuxiliaryInfoApp{}, // TODO: set this in the config + nil, + ) + return &Instance{ - Config: config, - stopCh: make(chan struct{}), - epochChanges: make(chan epochChange, 1), - cs: NewCachedStorage(config.Storage), + Config: config, + stopCh: make(chan struct{}), + epochChanges: make(chan epochChange, 1), + cs: cs, + transitionListener: transitionListener, } } @@ -168,7 +182,7 @@ func (i *Instance) createNonValidatorConfig() (nonvalidator.Config, error) { Config: &metadata.Config{}, } i.cs.msm = i.msm - instanceStorage := NewInstanceStorage(i.cs, i.msm, noopOnIndex) + instanceStorage := NewInstanceStorage(i.cs, i.msm, i.transitionListener.onIndex) config := nonvalidator.Config{ ID: i.Config.ID, @@ -295,6 +309,13 @@ func (i *Instance) HandleMessage(msg *common.Message, from common.NodeID) error } if i.e != nil { + switch { + case msg.AuxiliaryInfo != nil: + i.msm.HandleAuxiliaryInfo(*msg.AuxiliaryInfo, avalanchego.NodeID(from)) + case msg.EpochTransitionApproval != nil: + // TODO: pass in time.Now() rather than uint64 + i.msm.HandleApproval(msg.EpochTransitionApproval, uint64(time.Now().Unix())) + } return i.e.HandleMessage(msg, from) } @@ -452,7 +473,7 @@ func (i *Instance) createEpochConfig(epoch uint64, validators common.Nodes) (sim comm := newCommunication(i.Config.Sender, i.Config.Broadcaster, validators) - instanceStorage := NewInstanceStorage(i.cs, msm, noopOnIndex) + instanceStorage := NewInstanceStorage(i.cs, msm, i.transitionListener.onIndex) onEpochChange := func(epoch uint64, validators common.Nodes) { blockBuilder.stop() diff --git a/transition_listener.go b/transition_listener.go new file mode 100644 index 00000000..c4ebc2d5 --- /dev/null +++ b/transition_listener.go @@ -0,0 +1,151 @@ +package simplex + +import ( + "time" + + "github.com/ava-labs/simplex/avalanchego" + "github.com/ava-labs/simplex/common" + + metadata "github.com/ava-labs/simplex/msm" +) + +// epochTransitionListener reacts to blocks committed to storage. When a +// transition block is indexed, it performs any tasks required of this node to +// complete the epoch transition, such as sending out approval messages. +// Non-validators should also use this listener, since they may become +// validators after the transition. +type epochTransitionListener struct { + // broadcaster is used for broadcasting potential approvals and auxiliary information. + // It should be broadcast to the validators of the current epoch. + broadcaster Broadcaster + + myNodeID avalanchego.NodeID + + // getValidatorSet returns the validator set at a given P-chain height. + getValidatorSet metadata.ValidatorSetRetriever + // getBlock retrieves a previously finalized block, used to traverse the auxiliary info history. + getBlock metadata.BlockRetriever + // signer signs epoch transition approvals. + signer common.Signer + // auxInfoApp decides whether the auxiliary info history is sufficient and generates new entries. + auxInfoApp metadata.AuxiliaryInfoGenVerifier + // handleApproval records our own broadcast approval in the local approval store. + // It is set for validators (whose MSM builds the next blocks and must include the + // approval) and nil for non-validators, which have no block builder to feed. + handleApproval func(approval *common.ValidatorSetApproval, timestamp uint64) error + + logger common.Logger +} + +func newEpochTransitionListener( + logger common.Logger, + broadcaster Broadcaster, + myNodeID avalanchego.NodeID, + getValidatorSet metadata.ValidatorSetRetriever, + getBlock metadata.BlockRetriever, + signer common.Signer, + auxInfoApp metadata.AuxiliaryInfoGenVerifier, + handleApproval func(approval *common.ValidatorSetApproval, timestamp uint64) error, +) *epochTransitionListener { + return &epochTransitionListener{ + broadcaster: broadcaster, + myNodeID: myNodeID, + getValidatorSet: getValidatorSet, + getBlock: getBlock, + signer: signer, + auxInfoApp: auxInfoApp, + handleApproval: handleApproval, + logger: logger, + } +} + +func (a *epochTransitionListener) onIndex(block *ParsedBlock) error { + switch block.Type() { + case metadata.BlockTypeTransitioning: + return a.handleTransitionBlock(block) + } + + return nil +} + +func (a *epochTransitionListener) handleTransitionBlock(block *ParsedBlock) error { + nextEpochPChainReference := block.Metadata.SimplexEpochInfo.NextPChainReferenceHeight + + // if our node is not in the next validator set, no need to send anything. + nextEpochValidatorSet, err := a.getValidatorSet(nextEpochPChainReference) + if err != nil { + return err + } + + indexes := nextEpochValidatorSet.IndexByNodeID() + if _, ok := indexes[a.myNodeID]; !ok { + return nil // we are not in the next validator set + } + + auxInfoHistory, err := metadata.GetAuxiliaryHistory(&block.StateMachineBlock, block.BlockHeader().Seq, a.getBlock, a.auxInfoApp.DefaultVersionID()) + if err != nil { + return err + } + + isSufficient, err := a.auxInfoApp.IsSufficient(auxInfoHistory.OldestVersionID, nextEpochValidatorSet, auxInfoHistory.Data) + if err != nil { + return err + } + + if isSufficient { + // no more auxiliary info to send, maybe send our approval + lastAuxInfoDigest := auxInfoHistory.LastHistoryDigest() + return a.maybeSendApprovals(block, lastAuxInfoDigest) + } + + // we need more auxiliary information, attempt to generate + generatedAuxInfo, err := a.auxInfoApp.Generate(auxInfoHistory.OldestVersionID, nextEpochValidatorSet, auxInfoHistory.Data) + if err != nil { + return err + } + + if generatedAuxInfo == nil { + return nil + } + + auxInfoMessage := &common.Message{ + AuxiliaryInfo: &common.AuxiliaryInfo{ + Version: auxInfoHistory.OldestVersionID, + Data: generatedAuxInfo, + }, + } + + a.broadcaster.Broadcast(auxInfoMessage) + return nil +} + +// TODO: use common.Digest +func (a *epochTransitionListener) maybeSendApprovals(block *ParsedBlock, auxInfoDigest [32]byte) error { + nextEpochPChainReference := block.Metadata.SimplexEpochInfo.NextPChainReferenceHeight + + sig, err := metadata.SignApproval(a.signer, nextEpochPChainReference, auxInfoDigest) + if err != nil { + return err + } + + approval := common.ValidatorSetApproval{ + NodeID: a.myNodeID, + PChainHeight: nextEpochPChainReference, + AuxInfoDigest: auxInfoDigest, + Signature: sig, + } + + approvalMessage := common.Message{ + EpochTransitionApproval: &approval, + } + + a.broadcaster.Broadcast(&approvalMessage) + + // Validators also record their own approval locally so the next block they build + // includes it. Non-validators have no block builder, so handleApproval is nil. + if a.handleApproval == nil { + return nil + } + timestamp := uint64(time.Now().UnixMilli()) + return a.handleApproval(&approval, timestamp) +} diff --git a/transition_listener_test.go b/transition_listener_test.go new file mode 100644 index 00000000..01c6034d --- /dev/null +++ b/transition_listener_test.go @@ -0,0 +1,249 @@ +package simplex + +import ( + "testing" + + "github.com/ava-labs/simplex/avalanchego" + "github.com/ava-labs/simplex/common" + metadata "github.com/ava-labs/simplex/msm" + "github.com/ava-labs/simplex/testutil" + + "github.com/stretchr/testify/require" +) + +var testNodeID = avalanchego.NodeID{1} + +type recordingBroadcaster struct { + messages []*common.Message +} + +func (rb *recordingBroadcaster) Broadcast(msg *common.Message) { + rb.messages = append(rb.messages, msg) +} + +type stubSigner struct { + sig []byte +} + +func (s stubSigner) Sign([]byte) ([]byte, error) { + return s.sig, nil +} + +type stubAuxInfoApp struct { + sufficient bool + generated []byte +} + +func (s *stubAuxInfoApp) IsLegalAppend(common.VersionID, metadata.NodeBLSMappings, [][]byte, []byte) error { + return nil +} + +func (s *stubAuxInfoApp) IsSufficient(common.VersionID, metadata.NodeBLSMappings, [][]byte) (bool, error) { + return s.sufficient, nil +} + +func (s *stubAuxInfoApp) Generate(common.VersionID, metadata.NodeBLSMappings, [][]byte) ([]byte, error) { + return s.generated, nil +} + +func (s *stubAuxInfoApp) DefaultVersionID() common.VersionID { + return 7 +} + +type listenerTestEnv struct { + broadcaster *recordingBroadcaster + // approvals records the approvals fed back to the local store via the handleApproval + // callback. It stays empty for a non-validator listener (nil handleApproval). + approvals []common.ValidatorSetApproval + listener *epochTransitionListener +} + +// newListenerTestEnv builds a listener wired to the given next-epoch validator set and +// auxiliary info app. When isValidator is true, the listener is given a handleApproval +// callback (recording into env.approvals) as a real validator MSM would; otherwise it is +// nil, matching a non-validator that has no block builder to record its own approval. +func newListenerTestEnv(t *testing.T, validatorSet metadata.NodeBLSMappings, auxApp metadata.AuxiliaryInfoGenVerifier, isValidator bool) *listenerTestEnv { + env := &listenerTestEnv{broadcaster: &recordingBroadcaster{}} + + getValidatorSet := func(uint64) (metadata.NodeBLSMappings, error) { + return validatorSet, nil + } + getBlock := func(seq uint64, _ common.Digest) (metadata.StateMachineBlock, *common.Finalization, error) { + require.Fail(t, "unexpected getBlock call", "seq %d", seq) + return metadata.StateMachineBlock{}, nil, nil + } + + var handleApproval func(approval *common.ValidatorSetApproval, timestamp uint64) error + if isValidator { + handleApproval = func(approval *common.ValidatorSetApproval, _ uint64) error { + env.approvals = append(env.approvals, *approval) + return nil + } + } + + env.listener = newEpochTransitionListener( + testutil.MakeLogger(t, 1), + env.broadcaster, + testNodeID, + getValidatorSet, + getBlock, + stubSigner{sig: []byte("signature")}, + auxApp, + handleApproval, + ) + return env +} + +// newTransitionBlock returns a ParsedBlock of type BlockTypeTransitioning carrying the +// given next-epoch P-chain reference height. The listener supplies the validator set and +// auxiliary info app, so the block itself needs no MSM. +func newTransitionBlock(t *testing.T, nextPChainRef uint64) *ParsedBlock { + block := &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{ + Metadata: metadata.StateMachineMetadata{ + SimplexProtocolMetadata: common.ProtocolMetadata{Seq: 10}, + SimplexEpochInfo: metadata.SimplexEpochInfo{ + NextPChainReferenceHeight: nextPChainRef, + }, + }, + }, + } + require.Equal(t, metadata.BlockTypeTransitioning, block.Type()) + return block +} + +func TestListenerIgnoresSealingBlock(t *testing.T) { + const sealingSeq = uint64(42) + + env := newListenerTestEnv(t, nil, nil, true) + + validatorSet := metadata.NodeBLSMappings{{NodeID: testNodeID, BLSKey: []byte("bls-key"), Weight: 5}} + block := &ParsedBlock{ + StateMachineBlock: metadata.StateMachineBlock{ + Metadata: metadata.StateMachineMetadata{ + SimplexProtocolMetadata: (common.ProtocolMetadata{Seq: sealingSeq}), + SimplexEpochInfo: metadata.SimplexEpochInfo{ + // a non-empty PrevSealingBlockHash distinguishes a sealing block from the zero block + PrevSealingBlockHash: [32]byte{1}, + BlockValidationDescriptor: &metadata.BlockValidationDescriptor{ + AggregatedMembership: metadata.AggregatedMembership{Members: validatorSet}, + }, + }, + }, + }, + } + require.Equal(t, metadata.BlockTypeSealing, block.Type()) + + // the listener only reacts to transitioning blocks, a sealing block is ignored + require.NoError(t, env.listener.onIndex(block)) + require.Empty(t, env.broadcaster.messages) +} + +func TestTransitionNotInValidatorSet(t *testing.T) { + // the next validator set does not contain our node + otherValidator := metadata.NodeBLSMappings{{NodeID: avalanchego.NodeID{2}, Weight: 1}} + auxApp := &stubAuxInfoApp{sufficient: false, generated: []byte("more aux info")} + env := newListenerTestEnv(t, otherValidator, auxApp, true) + + block := newTransitionBlock(t, 100) + + require.NoError(t, env.listener.onIndex(block)) + require.Empty(t, env.broadcaster.messages) + require.Empty(t, env.approvals) +} + +func TestTransitionNotEnoughAuxiliary(t *testing.T) { + validatorSet := metadata.NodeBLSMappings{{NodeID: testNodeID, Weight: 1}, {NodeID: avalanchego.NodeID{2}, Weight: 1}} + auxApp := &stubAuxInfoApp{sufficient: false, generated: []byte("more aux info")} + env := newListenerTestEnv(t, validatorSet, auxApp, true) + + block := newTransitionBlock(t, 100) + + require.NoError(t, env.listener.onIndex(block)) + + // the generated auxiliary info should be broadcast instead of an approval + require.Len(t, env.broadcaster.messages, 1) + msg := env.broadcaster.messages[0] + require.Nil(t, msg.EpochTransitionApproval) + require.NotNil(t, msg.AuxiliaryInfo) + require.Equal(t, auxApp.DefaultVersionID(), msg.AuxiliaryInfo.Version) + require.Equal(t, auxApp.generated, msg.AuxiliaryInfo.Data) + require.Empty(t, env.approvals) +} + +func TestTransitionBroadcastsApproval(t *testing.T) { + const nextPChainRef = uint64(100) + + validatorSet := metadata.NodeBLSMappings{{NodeID: testNodeID, Weight: 1}} + env := newListenerTestEnv(t, validatorSet, &stubAuxInfoApp{sufficient: true}, true) + + block := newTransitionBlock(t, nextPChainRef) + + require.NoError(t, env.listener.onIndex(block)) + + require.Len(t, env.broadcaster.messages, 1) + msg := env.broadcaster.messages[0] + require.Nil(t, msg.AuxiliaryInfo) + require.NotNil(t, msg.EpochTransitionApproval) + + approval := msg.EpochTransitionApproval + require.Equal(t, testNodeID, approval.NodeID) + require.Equal(t, nextPChainRef, approval.PChainHeight) + require.Equal(t, [32]byte{}, approval.AuxInfoDigest) // no auxiliary info was collected + require.Equal(t, []byte("signature"), approval.Signature) + + // a validator also records its own approval locally so its next block includes it + require.Equal(t, []common.ValidatorSetApproval{*approval}, env.approvals) +} + +// TestNonValidatorContributesAuxiliaryInfo asserts that a node still on the outside of the +// current validator set but present in the NEXT one contributes auxiliary info during the +// transition, exactly like a validator does. Non-validators pass a nil handleApproval, so +// nothing is recorded locally, but the auxiliary info is still broadcast. +func TestNonValidatorContributesAuxiliaryInfo(t *testing.T) { + validatorSet := metadata.NodeBLSMappings{{NodeID: testNodeID, Weight: 1}, {NodeID: avalanchego.NodeID{2}, Weight: 1}} + auxApp := &stubAuxInfoApp{sufficient: false, generated: []byte("non-validator aux")} + env := newListenerTestEnv(t, validatorSet, auxApp, false /* non-validator */) + + block := newTransitionBlock(t, 100) + + require.NoError(t, env.listener.onIndex(block)) + + require.Len(t, env.broadcaster.messages, 1) + msg := env.broadcaster.messages[0] + require.Nil(t, msg.EpochTransitionApproval) + require.NotNil(t, msg.AuxiliaryInfo) + require.Equal(t, auxApp.DefaultVersionID(), msg.AuxiliaryInfo.Version) + require.Equal(t, auxApp.generated, msg.AuxiliaryInfo.Data) + + // non-validators do not record approvals locally + require.Empty(t, env.approvals) +} + +// TestNonValidatorContributesApproval asserts that once the auxiliary info history is +// sufficient, a non-validator that belongs to the next validator set broadcasts its +// epoch transition approval. It does not record the approval locally (nil handleApproval), +// since it has no block builder to include it. +func TestNonValidatorContributesApproval(t *testing.T) { + const nextPChainRef = uint64(100) + + validatorSet := metadata.NodeBLSMappings{{NodeID: testNodeID, Weight: 1}} + env := newListenerTestEnv(t, validatorSet, &stubAuxInfoApp{sufficient: true}, false /* non-validator */) + + block := newTransitionBlock(t, nextPChainRef) + + require.NoError(t, env.listener.onIndex(block)) + + require.Len(t, env.broadcaster.messages, 1) + msg := env.broadcaster.messages[0] + require.Nil(t, msg.AuxiliaryInfo) + require.NotNil(t, msg.EpochTransitionApproval) + + approval := msg.EpochTransitionApproval + require.Equal(t, testNodeID, approval.NodeID) + require.Equal(t, nextPChainRef, approval.PChainHeight) + require.Equal(t, []byte("signature"), approval.Signature) + + // non-validators broadcast but do not record their own approval locally + require.Empty(t, env.approvals) +} From 7e61e2f99ad263dadd12b3fba9d32d22b6eeeb91 Mon Sep 17 00:00:00 2001 From: samliok Date: Fri, 14 Aug 2026 12:42:20 -0400 Subject: [PATCH 4/6] gate on epoch sealed --- adapters.go | 35 +++++++++++++++++++++-------------- instance.go | 31 +++++++++++++++++++++++-------- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/adapters.go b/adapters.go index f6b6d35b..a60b6684 100644 --- a/adapters.go +++ b/adapters.go @@ -108,7 +108,8 @@ type CachedStorage struct { msm *metadata.StateMachine lock sync.RWMutex Storage - cache map[common.Digest]cachedBlock + cache map[common.Digest]cachedBlock + lastSealedEpoch uint64 } func NewCachedStorage(storage Storage) *CachedStorage { @@ -153,23 +154,29 @@ func (cs *CachedStorage) Retrieve(seq uint64, digest common.Digest) (common.Veri func (cs *CachedStorage) Index(ctx context.Context, block common.VerifiedBlock, certificate common.Finalization) error { err := cs.Storage.Index(ctx, block, certificate) + if err != nil { + return err + } - if err == nil { - // We delete the block from the cache after it has been indexed because now that it is persisted, - // we can just lookup by sequence number instead of digest. - cs.lock.Lock() - defer cs.lock.Unlock() - delete(cs.cache, block.BlockHeader().Digest) - - // We also delete all blocks that are older than the indexed block, because they are now finalized and persisted. - for digest, cachedBlock := range cs.cache { - if cachedBlock.BlockHeader().Seq < block.BlockHeader().Seq { - delete(cs.cache, digest) - } + // We delete the block from the cache after it has been indexed because now that it is persisted, + // we can just lookup by sequence number instead of digest. + cs.lock.Lock() + defer cs.lock.Unlock() + delete(cs.cache, block.BlockHeader().Digest) + + // We also delete all blocks that are older than the indexed block, because they are now finalized and persisted. + for digest, cachedBlock := range cs.cache { + if cachedBlock.BlockHeader().Seq < block.BlockHeader().Seq { + delete(cs.cache, digest) } } - return err + // remove previous epochs from map + if block.SealingBlockInfo() != nil && block.SealingBlockInfo().PrevSealingBlockHash != [32]byte{} { + cs.lastSealedEpoch = block.BlockHeader().Epoch + } + + return nil } func (cs *CachedStorage) insertBlock(block *ParsedBlock) { diff --git a/instance.go b/instance.go index 7ec186f3..ed1db71c 100644 --- a/instance.go +++ b/instance.go @@ -309,14 +309,7 @@ func (i *Instance) HandleMessage(msg *common.Message, from common.NodeID) error } if i.e != nil { - switch { - case msg.AuxiliaryInfo != nil: - i.msm.HandleAuxiliaryInfo(*msg.AuxiliaryInfo, avalanchego.NodeID(from)) - case msg.EpochTransitionApproval != nil: - // TODO: pass in time.Now() rather than uint64 - i.msm.HandleApproval(msg.EpochTransitionApproval, uint64(time.Now().Unix())) - } - return i.e.HandleMessage(msg, from) + return i.handleValidatorMessage(msg, from) } if i.nv != nil { @@ -325,6 +318,28 @@ func (i *Instance) HandleMessage(msg *common.Message, from common.NodeID) error return nil } +func (i *Instance) handleValidatorMessage(msg *common.Message, from common.NodeID) error { + // we only want to process replication requests if the epoch is sealed + if i.cs.lastSealedEpoch == i.e.Metadata().Epoch { + if msg.ReplicationRequest != nil && i.e.ReplicationEnabled { + return i.e.HandleMessage(msg, from) + } + + i.Config.Logger.Debug("Receive a message for an epoch that was sealed", zap.Uint64("epoch", i.cs.lastSealedEpoch), zap.Any("Message", msg)) + return nil + } + + switch { + case msg.AuxiliaryInfo != nil: + i.msm.HandleAuxiliaryInfo(*msg.AuxiliaryInfo, avalanchego.NodeID(from)) + case msg.EpochTransitionApproval != nil: + // TODO: pass in time.Now() rather than uint64 + i.msm.HandleApproval(msg.EpochTransitionApproval, uint64(time.Now().Unix())) + } + + return i.e.HandleMessage(msg, from) +} + func (i *Instance) wireReplicationResponse(msg *common.Message) error { resp := msg.ReplicationResponse if resp.LatestRound != nil && resp.LatestRound.Block != nil { From 4767124e400429178c9eb120d714516ed2aa7fa4 Mon Sep 17 00:00:00 2001 From: samliok Date: Fri, 14 Aug 2026 12:44:37 -0400 Subject: [PATCH 5/6] remove comment --- adapters.go | 1 - 1 file changed, 1 deletion(-) diff --git a/adapters.go b/adapters.go index a60b6684..c9616349 100644 --- a/adapters.go +++ b/adapters.go @@ -171,7 +171,6 @@ func (cs *CachedStorage) Index(ctx context.Context, block common.VerifiedBlock, } } - // remove previous epochs from map if block.SealingBlockInfo() != nil && block.SealingBlockInfo().PrevSealingBlockHash != [32]byte{} { cs.lastSealedEpoch = block.BlockHeader().Epoch } From 8f24f3c59e686a24936a70c0277ba5d08fc84328 Mon Sep 17 00:00:00 2001 From: samliok Date: Mon, 17 Aug 2026 16:57:30 -0400 Subject: [PATCH 6/6] return in handle message --- instance.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/instance.go b/instance.go index ed1db71c..cf60d4ef 100644 --- a/instance.go +++ b/instance.go @@ -332,9 +332,11 @@ func (i *Instance) handleValidatorMessage(msg *common.Message, from common.NodeI switch { case msg.AuxiliaryInfo != nil: i.msm.HandleAuxiliaryInfo(*msg.AuxiliaryInfo, avalanchego.NodeID(from)) + return nil case msg.EpochTransitionApproval != nil: // TODO: pass in time.Now() rather than uint64 i.msm.HandleApproval(msg.EpochTransitionApproval, uint64(time.Now().Unix())) + return nil } return i.e.HandleMessage(msg, from)