fix(p2p): make the inbound accept rate configurable and raise its default

sei-tendermint/config/config.go

@@ -150,6 +150,9 @@ func (cfg *Config) ValidateBasic() error {
 	if err := cfg.RPC.ValidateBasic(); err != nil {
 		return fmt.Errorf("error in [rpc] section: %w", err)
 	}
+	if err := cfg.P2P.ValidateBasic(); err != nil {
+		return fmt.Errorf("error in [p2p] section: %w", err)
+	}
 	if err := cfg.Mempool.ValidateBasic(); err != nil {
 		return fmt.Errorf("error in [mempool] section: %w", err)
 	}
@@ -724,6 +727,17 @@ type P2PConfig struct {
 	// How often node should dial a new peer.
 	DialInterval time.Duration `mapstructure:"dial-interval"`
 
+	// How often node should accept a new inbound connection. This paces the
+	// accept loop only. The number of connections being handshaked concurrently
+	// is bounded separately by RouterOptions.MaxConcurrentAccepts, which node
+	// setup derives from max-connections minus the outbound reservation.
+	// Kept in sync with the p2p router's own MaxAcceptRate default.
+	// max-incoming-connection-attempts caps concurrent connections per source IP
+	// (with a re-connect delay applied only once a source drops back to zero);
+	// note its default exceeds the inbound pool, so it does not bound a single
+	// source below that pool. A value of 0 disables the limiter.
+	AcceptInterval time.Duration `mapstructure:"accept-interval"`
+
 	// Testing params.
 	// Force dial to fail
 	TestDialFail bool `mapstructure:"test-dial-fail"`
@@ -754,6 +768,7 @@ func DefaultP2PConfig() *P2PConfig {
 		HandshakeTimeout:              10 * time.Second,
 		DialTimeout:                   3 * time.Second,
 		DialInterval:                  10 * time.Second,
+		AcceptInterval:                10 * time.Millisecond,
 		TestDialFail:                  false,
 		QueueType:                     "simple-priority",
 	}
@@ -774,6 +789,12 @@ func (cfg *P2PConfig) ValidateBasic() error {
 	if cfg.RecvRate < 0 {
 		return errors.New("recv-rate can't be negative")
 	}
+	if cfg.DialInterval < 0 {
+		return errors.New("dial-interval can't be negative")
+	}
+	if cfg.AcceptInterval < 0 {
+		return errors.New("accept-interval can't be negative")
+	}
 	return nil
 }
 

sei-tendermint/config/config_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/sei-protocol/sei-chain/sei-tendermint/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/time/rate"
 )
 
 func TestDefaultConfig(t *testing.T) {
@@ -38,6 +39,20 @@ func TestConfigValidateBasic(t *testing.T) {
 	assert.Error(t, cfg.ValidateBasic())
 }
 
+// P2PConfig.ValidateBasic was unreachable from production: Config.ValidateBasic
+// routed every other section but not [p2p], so its checks — including the
+// pre-existing send-rate/recv-rate ones — never ran outside tests, and a negative
+// accept-interval reached rate.Every as rate.Inf, silently disabling the accept
+// limiter. Assert the section is routed, not merely that its own checks work; the
+// latter passed the whole time nothing called them.
+func TestConfigValidateBasicRoutesP2P(t *testing.T) {
+	cfg := DefaultConfig()
+	require.NoError(t, cfg.ValidateBasic())
+
+	cfg.P2P.AcceptInterval = -1
+	require.Error(t, cfg.ValidateBasic())
+}
+
 func TestTLSConfiguration(t *testing.T) {
 	cfg := DefaultConfig()
 	cfg.SetRoot("/home/user")
@@ -230,6 +245,8 @@ func TestP2PConfigValidateBasic(t *testing.T) {
 		"MaxPacketMsgPayloadSize",
 		"SendRate",
 		"RecvRate",
+		"DialInterval",
+		"AcceptInterval",
 	}
 
 	for _, fieldName := range fieldsToTest {
@@ -239,6 +256,25 @@ func TestP2PConfigValidateBasic(t *testing.T) {
 	}
 }
 
+// The accept loop paces itself off AcceptInterval. 10ms (100/s) sits well above
+// the rate at which peers arrive on a public listener; a default admitting only a
+// handful per second cannot drain the kernel accept backlog, so peers queue behind
+// it, time out mid-handshake, and the node stops acquiring inbound peers while
+// still reporting healthy. Pin the value exactly, so changing it is deliberate and
+// visible in the diff.
+func TestP2PConfigAcceptInterval(t *testing.T) {
+	cfg := DefaultP2PConfig()
+	require.NoError(t, cfg.ValidateBasic())
+
+	require.Equal(t, 10*time.Millisecond, cfg.AcceptInterval)
+
+	// A zero interval is the documented escape hatch for disabling the limiter
+	// outright, and must stay valid rather than becoming a zero rate.
+	cfg.AcceptInterval = 0
+	require.NoError(t, cfg.ValidateBasic())
+	require.Equal(t, rate.Inf, rate.Every(cfg.AcceptInterval))
+}
+
 // --- WalFile legacy fallback tests ---
 
 func TestWalFile_NewDefault_NoLegacy(t *testing.T) {

sei-tendermint/config/p2p_compat_test.go

@@ -0,0 +1,75 @@
+package config_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+
+	"github.com/sei-protocol/sei-chain/sei-tendermint/cmd/tendermint/commands"
+	tmconfig "github.com/sei-protocol/sei-chain/sei-tendermint/config"
+)
+
+// This test (and TestP2PConfigPredatingPacingKnobsKeepsDefaults) mutate the
+// global viper singleton via commands.ParseConfig, so they must not run in
+// parallel with other tests in this package.
+
+// accept-interval is rendered in the template, but dial-interval is not (see
+// checkConfig in toml_test.go), so for that key "absent from the template" and
+// "not readable at all" would otherwise be indistinguishable. Cover both, since
+// an operator sets them the same way.
+func TestP2PPacingKnobsParseFromExistingConfig(t *testing.T) {
+	viper.Reset()
+	t.Cleanup(viper.Reset)
+
+	configPath := filepath.Join(t.TempDir(), "config.toml")
+	err := os.WriteFile(configPath, []byte(`
+[p2p]
+laddr = "tcp://0.0.0.0:26656"
+dial-interval = "5s"
+accept-interval = "20ms"
+`), 0600)
+	require.NoError(t, err)
+
+	viper.SetConfigFile(configPath)
+	require.NoError(t, viper.ReadInConfig())
+
+	cfg, err := commands.ParseConfig(tmconfig.DefaultConfig())
+	require.NoError(t, err)
+	require.Equal(t, 5*time.Second, cfg.P2P.DialInterval)
+	require.Equal(t, 20*time.Millisecond, cfg.P2P.AcceptInterval)
+	require.NoError(t, cfg.P2P.ValidateBasic())
+}
+
+// TestP2PConfigPredatingPacingKnobsKeepsDefaults mirrors a config.toml written
+// before these keys existed — the case every already-deployed node is in, since
+// seid does not rewrite an existing config.toml — and verifies ParseConfig still
+// produces the defaults. The failure it guards is silent: a zeroed AcceptInterval
+// means rate.Every(0) == rate.Inf, disabling accept pacing entirely, and an
+// absent key must not land there.
+func TestP2PConfigPredatingPacingKnobsKeepsDefaults(t *testing.T) {
+	viper.Reset()
+	t.Cleanup(viper.Reset)
+
+	configPath := filepath.Join(t.TempDir(), "config.toml")
+	err := os.WriteFile(configPath, []byte(`
+[p2p]
+laddr = "tcp://0.0.0.0:26656"
+`), 0600)
+	require.NoError(t, err)
+
+	viper.SetConfigFile(configPath)
+	require.NoError(t, viper.ReadInConfig())
+
+	cfg, err := commands.ParseConfig(tmconfig.DefaultConfig())
+	require.NoError(t, err)
+
+	defaults := tmconfig.DefaultP2PConfig()
+	require.Equal(t, defaults.AcceptInterval, cfg.P2P.AcceptInterval)
+	require.Equal(t, defaults.DialInterval, cfg.P2P.DialInterval)
+	require.NotZero(t, cfg.P2P.AcceptInterval, "a zero accept-interval disables accept pacing entirely")
+	require.NoError(t, cfg.P2P.ValidateBasic())
+}

sei-tendermint/config/toml.go

@@ -346,6 +346,16 @@ allow-duplicate-ip = {{ .P2P.AllowDuplicateIP }}
 handshake-timeout = "{{ .P2P.HandshakeTimeout }}"
 dial-timeout = "{{ .P2P.DialTimeout }}"
 
+# How often the node accepts a new inbound connection. This paces the accept
+# loop only: concurrent handshakes are capped at max-connections minus the
+# outbound reservation. max-incoming-connection-attempts caps concurrent
+# connections per source IP, and its default exceeds that inbound pool, so it
+# does not bound a single source below it. A larger interval paces the loop more slowly;
+# if the kernel accept backlog outpaces it, arriving peers wait past
+# handshake-timeout and the node silently stops acquiring inbound peers.
+# A value of 0 disables the limiter.
+accept-interval = "{{ .P2P.AcceptInterval }}"
+
 # Time to wait before flushing messages out on the connection
 # TODO: Remove once MConnConnection is removed.
 flush-throttle-timeout = "{{ .P2P.FlushThrottleTimeout }}"

sei-tendermint/config/toml_test.go

@@ -97,6 +97,25 @@ func checkConfig(t *testing.T, configFile string) {
 			t.Errorf("config file was not expected to contain %s", e)
 		}
 	}
+
+	// accept-interval is rendered deliberately: an accept rate too low to drain
+	// the kernel backlog silently stops a node acquiring inbound peers, and the
+	// template is where an operator looks. Keep it discoverable.
+	if !configContainsKey(configFile, "accept-interval") {
+		t.Errorf("config file was expected to contain accept-interval but did not")
+	}
+
+	// dial-interval remains an expert-only knob, left out of the generated
+	// template while still being parsed from existing config files.
+	// See TestP2PPacingKnobsParseFromExistingConfig.
+	var hiddenP2PElems = []string{
+		"dial-interval",
+	}
+	for _, e := range hiddenP2PElems {
+		if configContainsKey(configFile, e) {
+			t.Errorf("config file was not expected to contain %s", e)
+		}
+	}
 }
 
 func configContainsKey(configFile string, key string) bool {

sei-tendermint/internal/p2p/routeroptions.go

@@ -68,7 +68,13 @@ type RouterOptions struct {
 	// MaxDialRate limits the rate at which router is dialing peers. Defaults to 0.1/s.
 	MaxDialRate utils.Option[rate.Limit]
 
-	// MaxAcceptRate limits the rate at which router is accepting TCP connections. Defaults to 1/s.
+	// MaxAcceptRate limits the rate at which router is accepting TCP connections. Defaults to 100/s,
+	// kept in sync with config.DefaultP2PConfig().AcceptInterval (config cannot import this
+	// package, so the two values are deliberate copies rather than a shared constant).
+	// Node setup sets this from the p2p accept-interval config key; the default covers
+	// embedders that construct RouterOptions directly. Keep it high enough to drain the
+	// kernel accept backlog: a rate below the arrival rate leaves peers queued past
+	// handshake-timeout, so the node stops acquiring inbound peers while looking healthy.
 	MaxAcceptRate utils.Option[rate.Limit]
 
 	// ResolveTimeout is the timeout for resolving NodeAddress URLs.
@@ -165,7 +171,7 @@ func (o *RouterOptions) maxDialRate() rate.Limit {
 }
 
 func (o *RouterOptions) maxAcceptRate() rate.Limit {
-	return o.MaxAcceptRate.Or(rate.Every(time.Second))
+	return o.MaxAcceptRate.Or(rate.Every(10 * time.Millisecond))
 }
 
 func (o *RouterOptions) incomingConnectionWindow() time.Duration {

sei-tendermint/node/setup.go

@@ -454,27 +454,12 @@ func buildFullnodeGigaConfig(
 	}, nil
 }
 
-func createRouter(
-	nodeInfoProducer func() *types.NodeInfo,
-	nodeKey types.NodeKey,
-	validatorKey utils.Option[atypes.SecretKey],
-	cfg *config.Config,
-	app utils.Option[*proxy.Proxy],
-	genDoc *types.GenesisDoc,
-	dbProvider config.DBProvider,
-) (*p2p.Router, closer, utils.Option[atypes.BlockDB], error) {
-	closer := func() error { return nil }
-	noneDB := utils.None[atypes.BlockDB]()
-	gigaBlockDB := noneDB
-	ep, err := p2p.ResolveEndpoint(nodeKey.ID().AddressString(cfg.P2P.ListenAddress))
-	if err != nil {
-		return nil, closer, noneDB, err
-	}
-	var privatePeerIDs []types.NodeID
-	for _, id := range tmstrings.SplitAndTrimEmpty(cfg.P2P.PrivatePeerIDs, ",", " ") {
-		privatePeerIDs = append(privatePeerIDs, types.NodeID(id))
-	}
-
+// p2pRouterOptions derives the router's connection budget and pacing from the
+// p2p config. Split out of createRouter so the derivation is testable on its
+// own: any RouterOptions field left unset here silently falls back to a package
+// default rather than failing, which is how the accept rate stayed pinned at
+// its 1/s default while max-connections appeared to govern it.
+func p2pRouterOptions(cfg *config.Config, ep p2p.Endpoint, privatePeerIDs []types.NodeID) *p2p.RouterOptions {
 	// MaxConnections defaults to 64
 	maxConns := 64
 	if cfg.P2P.MaxConnections > 0 {
@@ -496,10 +481,11 @@ func createRouter(
 	connection.SendRate = cfg.P2P.SendRate
 	connection.RecvRate = cfg.P2P.RecvRate
 	connection.MaxPacketMsgPayloadSize = cfg.P2P.MaxPacketMsgPayloadSize
-	options := &p2p.RouterOptions{
+	return &p2p.RouterOptions{
 		Endpoint:                      ep,
 		MaxIncomingConnectionAttempts: utils.Some(cfg.P2P.MaxIncomingConnectionAttempts),
 		MaxDialRate:                   utils.Some(rate.Every(cfg.P2P.DialInterval)),
+		MaxAcceptRate:                 utils.Some(rate.Every(cfg.P2P.AcceptInterval)),
 		HandshakeTimeout:              utils.Some(cfg.P2P.HandshakeTimeout),
 		DialTimeout:                   utils.Some(cfg.P2P.DialTimeout),
 		PexOnHandshake:                cfg.P2P.PexReactor,
@@ -509,6 +495,30 @@ func createRouter(
 		MaxConcurrentAccepts:          utils.Some(maxInbound),
 		Connection:                    connection,
 	}
+}
+
+func createRouter(
+	nodeInfoProducer func() *types.NodeInfo,
+	nodeKey types.NodeKey,
+	validatorKey utils.Option[atypes.SecretKey],
+	cfg *config.Config,
+	app utils.Option[*proxy.Proxy],
+	genDoc *types.GenesisDoc,
+	dbProvider config.DBProvider,
+) (*p2p.Router, closer, utils.Option[atypes.BlockDB], error) {
+	closer := func() error { return nil }
+	noneDB := utils.None[atypes.BlockDB]()
+	gigaBlockDB := noneDB
+	ep, err := p2p.ResolveEndpoint(nodeKey.ID().AddressString(cfg.P2P.ListenAddress))
+	if err != nil {
+		return nil, closer, noneDB, err
+	}
+	var privatePeerIDs []types.NodeID
+	for _, id := range tmstrings.SplitAndTrimEmpty(cfg.P2P.PrivatePeerIDs, ",", " ") {
+		privatePeerIDs = append(privatePeerIDs, types.NodeID(id))
+	}
+
+	options := p2pRouterOptions(cfg, ep, privatePeerIDs)
 	if addr := cfg.P2P.ExternalAddress; addr != "" {
 		nodeAddr, err := p2p.ParseNodeAddress(nodeKey.ID().AddressString(addr))
 		if err != nil {