From 9dfd2d8258a8ef4dc91a0a55d8f4f6f2589f2001 Mon Sep 17 00:00:00 2001 From: Rick Staa Date: Mon, 15 Jun 2026 20:47:10 +0200 Subject: [PATCH 1/3] feat(eth): add web3signer external signer for remote Ethereum key custody Add an alternative AccountManager that delegates signing to a remote Web3Signer-compatible endpoint (eth_* JSON-RPC) instead of a local keystore, so the Ethereum private key never lives on the node host. This is aimed at pooled-wallet payment remote signers, where a host compromise could otherwise drain the deposit/reserve. Select it with -ethExternalSigner plus -ethAcctAddr for the address the signer holds. Output is byte-compatible with the keystore path: EIP-191 (accounts.TextHash) message signing, latest-signer tx signing, and recovery id normalized to {27,28}. remote_signer.go, the payment structs, and the protocol are unchanged. One protocol covers every backend: Web3Signer fronts AWS KMS, Azure Key Vault, HashiCorp Vault, and HSM with no go-livepeer change (config only), and MPC/enclave custody (Turnkey, Fireblocks) sits behind an eth_* sidecar (livepeer/external-signer). Documented in doc/external-signer.md. Co-authored-by: John | Elite Encoder Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/livepeer/starter/flags.go | 1 + cmd/livepeer/starter/starter.go | 10 +- doc/external-signer.md | 86 +++++++++++++++++ doc/remote-signer.md | 2 + eth/web3signer.go | 158 ++++++++++++++++++++++++++++++++ eth/web3signer_test.go | 95 +++++++++++++++++++ 6 files changed, 351 insertions(+), 1 deletion(-) create mode 100644 doc/external-signer.md create mode 100644 eth/web3signer.go create mode 100644 eth/web3signer_test.go diff --git a/cmd/livepeer/starter/flags.go b/cmd/livepeer/starter/flags.go index 8a8c724f2e..1e5fbf4308 100644 --- a/cmd/livepeer/starter/flags.go +++ b/cmd/livepeer/starter/flags.go @@ -83,6 +83,7 @@ func NewLivepeerConfig(fs *flag.FlagSet) LivepeerConfig { cfg.EthAcctAddr = fs.String("ethAcctAddr", *cfg.EthAcctAddr, "Existing Eth account address. For use when multiple ETH accounts exist in the keystore directory") cfg.EthPassword = fs.String("ethPassword", *cfg.EthPassword, "Password for existing Eth account address or path to file") cfg.EthKeystorePath = fs.String("ethKeystorePath", *cfg.EthKeystorePath, "Path to ETH keystore directory or keyfile. If keyfile, overrides -ethAcctAddr and uses parent directory") + cfg.EthExternalSigner = fs.String("ethExternalSigner", *cfg.EthExternalSigner, "JSON-RPC endpoint of a Web3Signer-compatible external signer (eth_* namespace; e.g. Web3Signer fronting KMS/Vault/HSM, or a Turnkey/MPC custody sidecar). When set, signing is delegated to it instead of a local keystore and -ethAcctAddr must be the address it signs for") cfg.EthOrchAddr = fs.String("ethOrchAddr", *cfg.EthOrchAddr, "ETH address of an on-chain registered orchestrator") cfg.EthUrl = fs.String("ethUrl", *cfg.EthUrl, "Ethereum node JSON-RPC URL") cfg.TxTimeout = fs.Duration("transactionTimeout", *cfg.TxTimeout, "Amount of time to wait for an Ethereum transaction to confirm before timing out") diff --git a/cmd/livepeer/starter/starter.go b/cmd/livepeer/starter/starter.go index cf3aca3dbd..5b721a8e97 100755 --- a/cmd/livepeer/starter/starter.go +++ b/cmd/livepeer/starter/starter.go @@ -124,6 +124,7 @@ type LivepeerConfig struct { EthAcctAddr *string EthPassword *string EthKeystorePath *string + EthExternalSigner *string EthOrchAddr *string EthUrl *string TxTimeout *time.Duration @@ -254,6 +255,7 @@ func DefaultLivepeerConfig() LivepeerConfig { defaultEthAcctAddr := "" defaultEthPassword := "" defaultEthKeystorePath := "" + defaultEthExternalSigner := "" defaultEthOrchAddr := "" defaultEthUrl := "" defaultTxTimeout := 5 * time.Minute @@ -379,6 +381,7 @@ func DefaultLivepeerConfig() LivepeerConfig { EthAcctAddr: &defaultEthAcctAddr, EthPassword: &defaultEthPassword, EthKeystorePath: &defaultEthKeystorePath, + EthExternalSigner: &defaultEthExternalSigner, EthOrchAddr: &defaultEthOrchAddr, EthUrl: &defaultEthUrl, TxTimeout: &defaultTxTimeout, @@ -846,7 +849,12 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) { } defer gpm.Stop() - am, err := eth.NewAccountManager(ethcommon.HexToAddress(*cfg.EthAcctAddr), keystoreDir, chainID, *cfg.EthPassword) + var am eth.AccountManager + if *cfg.EthExternalSigner != "" { + am, err = eth.NewWeb3SignerAccountManager(ethcommon.HexToAddress(*cfg.EthAcctAddr), *cfg.EthExternalSigner, chainID) + } else { + am, err = eth.NewAccountManager(ethcommon.HexToAddress(*cfg.EthAcctAddr), keystoreDir, chainID, *cfg.EthPassword) + } if err != nil { glog.Errorf("Error creating Ethereum account manager: %v", err) return diff --git a/doc/external-signer.md b/doc/external-signer.md new file mode 100644 index 0000000000..0fd4869435 --- /dev/null +++ b/doc/external-signer.md @@ -0,0 +1,86 @@ +# External signer (remote Ethereum key custody) + +By default a go-livepeer node that needs an Ethereum key (a payment [remote signer](./remote-signer.md), an orchestrator, or a gateway running onchain) holds that key as a local keystore file and unlocks it into process memory. The **external signer** lets the node delegate signing to a separate signing service instead, so the private key never lives on the node host. + +This is most valuable for a shared, pooled-wallet payment signer: the signer holds a hot key that funds on-chain deposit/reserve and signs probabilistic micropayment (PM) tickets. Moving that key behind an external signer removes the worst failure mode (host compromise drains the deposit/reserve) and, depending on the backend, lets it enforce signing policy. + +## How it works + +Signing in go-livepeer already goes through a single interface (`eth.AccountManager`: `Sign`, `SignTx`, `SignTypedData`, ...). The external signer is an alternative implementation of that interface that, instead of touching a local keystore, proxies each request over JSON-RPC to a signing service that speaks the standard Ethereum `eth_*` signing namespace (the [Web3Signer](https://docs.web3signer.consensys.io/) protocol). + +```mermaid +sequenceDiagram + participant Node as go-livepeer node + participant Signer as External signer (Web3Signer-compatible) + participant Backend as Key backend + + Node->>Signer: eth_sign / eth_signTransaction / eth_signTypedData + Signer->>Backend: sign (key never leaves the backend) + Backend-->>Signer: signature + Signer-->>Node: signature +``` + +The node still builds every transaction itself (nonce, gas) and broadcasts via its own Ethereum RPC; the external signer only returns a signature. Nothing about the protocol changes: PM tickets, deposit/reserve funding, redemption, and the gateway -> orchestrator flow stay byte-identical. Only *how* a signature is produced changes. + +Signatures are byte-compatible with the local keystore path: messages are signed over the EIP-191 personal-message hash (`accounts.TextHash`), transactions with the latest signer for the chain, and the recovery id is normalized to `{27, 28}`. + +## Usage + +Point the node at a Web3Signer-compatible endpoint and tell it which address that endpoint signs for: + +- `-ethExternalSigner `: JSON-RPC endpoint of the external signer. When set, signing is delegated to it instead of a local keystore. +- `-ethAcctAddr <0x...>`: the Ethereum address the external signer holds. Required in this mode, since there is no local keystore to default from. + +When `-ethExternalSigner` is set, the usual keystore flags (`-ethKeystorePath`, `-ethPassword`) are not used for signing. The node fails fast at startup if the endpoint is unreachable or does not respond to the `eth_*` signing API. + +Example (a payment remote signer backed by an external signer): + +```bash +./livepeer \ + -remoteSigner \ + -network mainnet \ + -httpAddr 127.0.0.1:7936 \ + -ethUrl \ + -ethAcctAddr 0xYourSignerAddress \ + -ethExternalSigner http://127.0.0.1:9000 \ + ... +``` + +## Backends + +The node speaks one protocol — Web3Signer's `eth_*` namespace (`eth_sign`, `eth_signTransaction`, `eth_signTypedData`, `eth_accounts`). Anything that exposes it works with no go-livepeer change. There are two ways to get there. + +### 1. KMS / Vault / HSM — via Web3Signer (config only) + +[Web3Signer](https://docs.web3signer.consensys.io/) is a maintained, off-the-shelf signing service with built-in key backends. Run it, point it at your key store, and point go-livepeer at Web3Signer. No code on either side. + +| Backend | How | +|---|---| +| HashiCorp Vault | Web3Signer Vault key source | +| AWS KMS / AWS Secrets Manager | Web3Signer AWS key source | +| Azure Key Vault | Web3Signer Azure key source | +| HSM (PKCS#11 / YubiHSM) | Web3Signer HSM key source | + +The choice of backend is a Web3Signer configuration detail; go-livepeer is unaware of it and never changes when you switch. + +### 2. Turnkey / MPC / enclave custody — via a sidecar + +Providers like [Turnkey](https://www.turnkey.com/) and Fireblocks use proprietary, policy-aware APIs that speak neither standard protocol. Run a thin sidecar that presents the `eth_*` API and forwards to the provider, holding the provider credentials. The provider-specific code and secrets live in the sidecar, outside go-livepeer. See [livepeer/external-signer](https://github.com/livepeer/external-signer). + +## Custody security ladder + +Backends differ in how much they protect the key. From weakest to strongest: + +| Option | Key stealable from host? | Abuse during host compromise | Ethereum spend policy | +|---|---|---|---| +| Local keystore (default) | Yes — permanent loss | Full | No | +| KMS / Vault / HSM (via Web3Signer) | No (never exported) | Possible while host creds valid; revocable + audited | No | +| Turnkey / enclave (via sidecar) | No | Bounded by policy | Yes (limits, allowlists, quorum) | + +KMS/Vault/HSM remove key exfiltration but still sign whatever digest they are asked to. Enclave/MPC custody additionally enforces signing policy, so even a fully compromised signer host cannot drain beyond the configured limits. + +## Caveats + +- **Raw-hash signing.** PM tickets are signed as a hash. `eth_sign` / `eth_signTypedData` reconstruct the digest from the message/structured input, so backends that refuse raw precomputed-hash signing still work. Verify this against your backend before production. +- **Hot-path latency and cost.** PM mints tickets frequently. Each signature is now an API round-trip rather than a local operation, which adds latency and (for hosted backends) cost. Measure against your minting rate. +- **Treat the endpoint like an internal wallet service.** Run the external signer on a private network or behind an authenticated proxy, the same as the [remote signer guidance](./remote-signer.md#operational--security-guidance). diff --git a/doc/remote-signer.md b/doc/remote-signer.md index 55334322eb..25ca7facc3 100644 --- a/doc/remote-signer.md +++ b/doc/remote-signer.md @@ -253,3 +253,5 @@ When `-remoteSignerWebhookUrl` is configured, the remote signer calls the auth w For the moment, remote signers are intended to sit behind infrastructure controls rather than being exposed directly to end-users. For example, run the remote signer on a private network or behind an authenticated proxy. Do not expose the remote signer to unauthenticated end-users. Run the remote signer close to gateways on a private network; protect it like you would an internal wallet service. If a proxy sits in front of the signer, configure it to scrub all incoming `Signer-` headers from untrusted clients before applying trusted internal headers. Remote signers are stateless, so signer nodes can operate in a redundant configuration (eg, round-robin DNS, anycasting) with no special gateway-side configuration. + +To remove the Ethereum hot key from the signer host entirely (delegating signing to a KMS/HSM via Web3Signer, or to an enclave/MPC provider such as Turnkey), see the [external signer](./external-signer.md) (`-ethExternalSigner`). diff --git a/eth/web3signer.go b/eth/web3signer.go new file mode 100644 index 0000000000..a326bd2e37 --- /dev/null +++ b/eth/web3signer.go @@ -0,0 +1,158 @@ +package eth + +import ( + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/signer/core/apitypes" + "github.com/golang/glog" +) + +// web3signerAccountManager is an AccountManager that delegates signing to a +// remote signer speaking the standard Ethereum eth_* JSON-RPC namespace +// (eth_sign, eth_signTransaction, eth_signTypedData), instead of a local +// keystore. This lets the node reach key backends fronted by Web3Signer (AWS +// KMS, Azure Key Vault, HashiCorp Vault, HSM) with no provider-specific code, +// as well as MPC/enclave custody sidecars (e.g. Turnkey) that present the same +// API. +// +// Output is byte-identical to the keystore accountManager: EIP-191 message +// signing, latest-signer transactions, and recovery id in {27,28}. +type web3signerAccountManager struct { + rpc *rpc.Client + account accounts.Account + chainID *big.Int +} + +// NewWeb3SignerAccountManager connects to a Web3Signer-compatible endpoint and +// signs on behalf of accountAddr (required, since there is no local keystore). +func NewWeb3SignerAccountManager(accountAddr ethcommon.Address, endpoint string, chainID *big.Int) (AccountManager, error) { + if (accountAddr == ethcommon.Address{}) { + return nil, fmt.Errorf("web3signer requires an explicit -ethAcctAddr") + } + + rpcClient, err := rpc.Dial(endpoint) + if err != nil { + return nil, fmt.Errorf("failed to dial web3signer at %s: %w", endpoint, err) + } + + // eth_accounts doubles as a reachability check and lets us warn if the + // remote signer does not hold the configured address. + var addrs []ethcommon.Address + if err := rpcClient.Call(&addrs, "eth_accounts"); err != nil { + return nil, fmt.Errorf("failed to reach web3signer at %s: %w", endpoint, err) + } + if !containsAddress(addrs, accountAddr) { + glog.Warningf("Web3Signer at %s did not list account %v; signing requests may be rejected", endpoint, accountAddr.Hex()) + } + + glog.Infof("Using web3signer at %s for Ethereum account: %v", endpoint, accountAddr.Hex()) + + return &web3signerAccountManager{ + rpc: rpcClient, + account: accounts.Account{Address: accountAddr}, + chainID: chainID, + }, nil +} + +func containsAddress(addrs []ethcommon.Address, want ethcommon.Address) bool { + for _, a := range addrs { + if a == want { + return true + } + } + return false +} + +func (m *web3signerAccountManager) Unlock(string) error { return nil } + +func (m *web3signerAccountManager) Lock() error { return nil } + +func (m *web3signerAccountManager) Account() accounts.Account { return m.account } + +func (m *web3signerAccountManager) CreateTransactOpts(gasLimit uint64) (*bind.TransactOpts, error) { + return &bind.TransactOpts{ + From: m.account.Address, + GasLimit: gasLimit, + Signer: func(addr ethcommon.Address, tx *types.Transaction) (*types.Transaction, error) { + if addr != m.account.Address { + return nil, fmt.Errorf("not authorized to sign for address %v", addr.Hex()) + } + return m.SignTx(tx) + }, + }, nil +} + +// Sign signs msg with the EIP-191 personal-message prefix. Web3Signer's eth_sign +// applies that prefix, matching the keystore accountManager. +func (m *web3signerAccountManager) Sign(msg []byte) ([]byte, error) { + var res hexutil.Bytes + if err := m.rpc.Call(&res, "eth_sign", m.account.Address, hexutil.Encode(msg)); err != nil { + return nil, err + } + return toEthV(res), nil +} + +func (m *web3signerAccountManager) SignTypedData(typedData apitypes.TypedData) ([]byte, error) { + var res hexutil.Bytes + if err := m.rpc.Call(&res, "eth_signTypedData", m.account.Address, typedData); err != nil { + return nil, err + } + return toEthV(res), nil +} + +// SignTx asks the remote signer to sign tx and returns the decoded signed +// transaction. go-livepeer still owns nonce/gas and broadcasts the result. +func (m *web3signerAccountManager) SignTx(tx *types.Transaction) (*types.Transaction, error) { + var raw hexutil.Bytes + if err := m.rpc.Call(&raw, "eth_signTransaction", m.toSendTxArgs(tx)); err != nil { + return nil, err + } + signed := new(types.Transaction) + if err := signed.UnmarshalBinary(raw); err != nil { + return nil, fmt.Errorf("web3signer returned an undecodable transaction: %w", err) + } + return signed, nil +} + +func (m *web3signerAccountManager) toSendTxArgs(tx *types.Transaction) *apitypes.SendTxArgs { + data := hexutil.Bytes(tx.Data()) + args := &apitypes.SendTxArgs{ + Data: &data, + Nonce: hexutil.Uint64(tx.Nonce()), + Value: hexutil.Big(*tx.Value()), + Gas: hexutil.Uint64(tx.Gas()), + From: ethcommon.NewMixedcaseAddress(m.account.Address), + } + if tx.To() != nil { + to := ethcommon.NewMixedcaseAddress(*tx.To()) + args.To = &to + } + switch tx.Type() { + case types.LegacyTxType, types.AccessListTxType: + args.GasPrice = (*hexutil.Big)(tx.GasPrice()) + case types.DynamicFeeTxType: + args.MaxFeePerGas = (*hexutil.Big)(tx.GasFeeCap()) + args.MaxPriorityFeePerGas = (*hexutil.Big)(tx.GasTipCap()) + } + if m.chainID != nil && m.chainID.Sign() != 0 { + args.ChainID = (*hexutil.Big)(m.chainID) + } + return args +} + +// toEthV normalizes the recovery id to {27,28} so output matches the keystore +// accountManager regardless of whether the remote signer returns {0,1} or +// {27,28}. +func toEthV(sig []byte) []byte { + if len(sig) == 65 && sig[64] < 27 { + sig[64] += 27 + } + return sig +} diff --git a/eth/web3signer_test.go b/eth/web3signer_test.go new file mode 100644 index 0000000000..50fe33847f --- /dev/null +++ b/eth/web3signer_test.go @@ -0,0 +1,95 @@ +package eth + +import ( + "crypto/ecdsa" + "encoding/json" + "math/big" + "net/http" + "net/http/httptest" + "testing" + + "github.com/ethereum/go-ethereum/accounts" + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" + lpcrypto "github.com/livepeer/go-livepeer/crypto" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// web3signerStub is a minimal Web3Signer-style JSON-RPC server that signs with +// key, exposing the eth_* namespace the adapter uses. +func web3signerStub(t *testing.T, key *ecdsa.PrivateKey) *httptest.Server { + addr := crypto.PubkeyToAddress(key.PublicKey) + + reply := func(w http.ResponseWriter, id json.RawMessage, result interface{}) { + raw, err := json.Marshal(result) + require.NoError(t, err) + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "jsonrpc": "2.0", "id": json.RawMessage(id), "result": json.RawMessage(raw), + }) + } + + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + ID json.RawMessage `json:"id"` + Method string `json:"method"` + Params []json.RawMessage `json:"params"` + } + require.NoError(t, json.NewDecoder(r.Body).Decode(&req)) + + switch req.Method { + case "eth_accounts": + reply(w, req.ID, []ethcommon.Address{addr}) + case "eth_sign": + // params: [address, hexData]; Web3Signer applies the EIP-191 prefix. + var hexData string + require.NoError(t, json.Unmarshal(req.Params[1], &hexData)) + data, err := hexutil.Decode(hexData) + require.NoError(t, err) + sig, err := crypto.Sign(accounts.TextHash(data), key) // V in {0,1} + require.NoError(t, err) + sig[64] += 27 // Web3Signer returns V in {27,28} + reply(w, req.ID, hexutil.Bytes(sig)) + default: + http.Error(w, "unsupported method "+req.Method, http.StatusBadRequest) + } + })) +} + +func TestWeb3Signer_SignMatchesKeystoreConvention(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + key, err := crypto.GenerateKey() + require.NoError(err) + addr := crypto.PubkeyToAddress(key.PublicKey) + + srv := web3signerStub(t, key) + defer srv.Close() + + am, err := NewWeb3SignerAccountManager(addr, srv.URL, big.NewInt(1)) + require.NoError(err) + assert.Equal(addr, am.Account().Address) + + msg := []byte("livepeer payment ticket hash") + sig, err := am.Sign(msg) + require.NoError(err) + + require.Len(sig, 65) + assert.Contains([]byte{27, 28}, sig[64]) + assert.True(lpcrypto.VerifySig(addr, msg, sig), "signature must recover to signer address") +} + +func TestWeb3Signer_RequiresExplicitAddress(t *testing.T) { + _, err := NewWeb3SignerAccountManager(ethcommon.Address{}, "http://127.0.0.1:0", big.NewInt(1)) + assert.Error(t, err) +} + +func TestWeb3Signer_UnreachableEndpoint(t *testing.T) { + key, err := crypto.GenerateKey() + require.NoError(t, err) + addr := crypto.PubkeyToAddress(key.PublicKey) + _, err = NewWeb3SignerAccountManager(addr, "http://127.0.0.1:1", big.NewInt(1)) + assert.Error(t, err) +} From 7326d40a4a86a7c8cf700e624d856c22d85cb4f5 Mon Sep 17 00:00:00 2001 From: Rick Staa Date: Mon, 15 Jun 2026 21:59:36 +0200 Subject: [PATCH 2/3] docs(external-signer): document the full stack and fix terminology Add a "Where it sits in the stack" section showing the end-to-end loop: clearinghouse control plane (auth/metering/billing) above the signer, go-livepeer owning the signing protocol, and the external signer service + backend owning key custody below it. Add a terminology section (adapter / external signer service / backend) and stop calling the standalone service a "sidecar" except as a deployment mode. Co-authored-by: John | Elite Encoder Co-Authored-By: Claude Opus 4.8 (1M context) --- doc/external-signer.md | 60 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/doc/external-signer.md b/doc/external-signer.md index 0fd4869435..87866ce5e1 100644 --- a/doc/external-signer.md +++ b/doc/external-signer.md @@ -24,6 +24,60 @@ The node still builds every transaction itself (nonce, gas) and broadcasts via i Signatures are byte-compatible with the local keystore path: messages are signed over the EIP-191 personal-message hash (`accounts.TextHash`), transactions with the latest signer for the chain, and the recovery id is normalized to `{27, 28}`. +## Terminology + +Three distinct pieces, often conflated: + +- **Adapter** — the in-process `eth.AccountManager` in go-livepeer, selected with `-ethExternalSigner`. It speaks the `eth_*` protocol. (This is the part this repo ships.) +- **External signer service** — a standalone process that presents the `eth_*` API and forwards to a custody provider. "Sidecar" describes *one way to deploy it* (co-located with the node); it can equally run as a standalone networked service. Either [Web3Signer](https://docs.web3signer.consensys.io/) or a provider bridge such as [livepeer/external-signer](https://github.com/livepeer/external-signer). +- **Backend** — the custody provider that actually holds the key (Turnkey, or a KMS/Vault/HSM behind Web3Signer). + +## Where it sits in the stack + +The external signer is the **key-custody layer beneath the signer**. It is independent of — but composes with — the payment control plane *above* the signer (a payment [clearinghouse](https://forum.livepeer.org/t/livepeer-payment-clearinghouse/3264)). End to end: + +``` + App / client + │ OIDC / API key + ┌─────▼──────────────── control plane, ABOVE the signer ───────────────┐ + │ Payment clearinghouse │ + │ • auth & identity (OIDC / JWT) │ + │ • usage metering (OpenMeter) │ + │ • fiat billing & clearing ─┼──┐ + │ • per-app wallet lifecycle │ │ management + └─────┬────────────────────────────────────────────────────────────────┘ │ path + │ /generate-live-payment │ + ┌─────▼──────────────┐ │ + │ Gateway (offchain) │──── media ────▶ Orchestrators │ + └─────┬──────────────┘ │ + ┌─────▼─────────────────────┐ │ + │ Remote signer │ builds PM tickets, needs a signature │ + │ (-remoteSigner) │ │ + └─────┬─────────────────────┘ │ + │ eth.AccountManager (Eth.Sign) │ + ┌─────▼──────────── THIS DOCUMENT, the custody layer BELOW the signer ──┐ │ + │ External signer adapter (-ethExternalSigner) [in go-livepeer] │ │ + └─────┬──────────────────────────────────────────────────────────────────┘ │ + │ eth_* JSON-RPC │ + ┌─────▼─────────────────────┐ │ + │ External signer service │ presents eth_*, forwards to the backend │ + │ (sidecar or standalone) │ [Web3Signer, or livepeer/external-signer] │ + └─────┬─────────────────────┘ │ + │ provider API │ provider API + ┌─────▼─────────────────────────────────────────────────────────────▼────┐ + │ Backend custody │ + │ Turnkey enclave · or Web3Signer → KMS / Vault / HSM │ + │ signing key (never exported) + per-app wallets (clearinghouse) │ + └─────────────────────────────────────────────────────────────────────────┘ +``` + +Two independent paths touch the backend: + +- **Signing hot path** (per ticket): `remote signer → adapter → external signer service → backend`. This is what this document covers. The clearinghouse is *not* in this path. +- **Control plane** (the clearinghouse): wraps the remote signer with auth, usage metering, and fiat billing *above* it, and — if it uses an enclave provider — manages per-app wallets directly against the backend's own API (e.g. Turnkey sub-orgs) on a separate **management path**. Key custody and the signing protocol are unchanged whether or not a clearinghouse sits on top. + +So the layers compose cleanly: the **clearinghouse** owns identity, metering, and billing; **go-livepeer** owns the signing protocol; the **external signer service + backend** own key custody. Each can change without the others. + ## Usage Point the node at a Web3Signer-compatible endpoint and tell it which address that endpoint signs for: @@ -63,9 +117,9 @@ The node speaks one protocol — Web3Signer's `eth_*` namespace (`eth_sign`, `et The choice of backend is a Web3Signer configuration detail; go-livepeer is unaware of it and never changes when you switch. -### 2. Turnkey / MPC / enclave custody — via a sidecar +### 2. Turnkey / MPC / enclave custody — via an external signer service -Providers like [Turnkey](https://www.turnkey.com/) and Fireblocks use proprietary, policy-aware APIs that speak neither standard protocol. Run a thin sidecar that presents the `eth_*` API and forwards to the provider, holding the provider credentials. The provider-specific code and secrets live in the sidecar, outside go-livepeer. See [livepeer/external-signer](https://github.com/livepeer/external-signer). +Providers like [Turnkey](https://www.turnkey.com/) and Fireblocks use proprietary, policy-aware APIs that speak neither standard protocol. Run a thin external signer service (deploy it as a sidecar or standalone) that presents the `eth_*` API and forwards to the provider, holding the provider credentials. The provider-specific code and secrets live in that service, outside go-livepeer. See [livepeer/external-signer](https://github.com/livepeer/external-signer). ## Custody security ladder @@ -75,7 +129,7 @@ Backends differ in how much they protect the key. From weakest to strongest: |---|---|---|---| | Local keystore (default) | Yes — permanent loss | Full | No | | KMS / Vault / HSM (via Web3Signer) | No (never exported) | Possible while host creds valid; revocable + audited | No | -| Turnkey / enclave (via sidecar) | No | Bounded by policy | Yes (limits, allowlists, quorum) | +| Turnkey / enclave (via external signer service) | No | Bounded by policy | Yes (limits, allowlists, quorum) | KMS/Vault/HSM remove key exfiltration but still sign whatever digest they are asked to. Enclave/MPC custody additionally enforces signing policy, so even a fully compromised signer host cannot drain beyond the configured limits. From 46d2ecc83e41dc6fa2149202f814e9912d88b5d2 Mon Sep 17 00:00:00 2001 From: Rick Staa Date: Tue, 16 Jun 2026 08:24:03 +0200 Subject: [PATCH 3/3] feat(eth): bound web3signer signing calls with a timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The web3signer adapter called the remote signer with rpc.Call, which uses context.Background() and no deadline — a hung or slow signer would block the PM ticket hot path indefinitely. Switch the signing calls and the startup reachability check to CallContext with a per-call timeout, configurable via -ethExternalSignerTimeout (default 5s). Add a test that asserts Sign returns promptly on timeout instead of waiting for a slow signer. Co-authored-by: John | Elite Encoder Co-Authored-By: Claude Opus 4.8 (1M context) --- cmd/livepeer/starter/flags.go | 1 + cmd/livepeer/starter/starter.go | 75 +++++++++++++++++---------------- doc/external-signer.md | 3 +- eth/web3signer.go | 37 +++++++++++++--- eth/web3signer_test.go | 46 ++++++++++++++++++-- 5 files changed, 116 insertions(+), 46 deletions(-) diff --git a/cmd/livepeer/starter/flags.go b/cmd/livepeer/starter/flags.go index 1e5fbf4308..b209883876 100644 --- a/cmd/livepeer/starter/flags.go +++ b/cmd/livepeer/starter/flags.go @@ -84,6 +84,7 @@ func NewLivepeerConfig(fs *flag.FlagSet) LivepeerConfig { cfg.EthPassword = fs.String("ethPassword", *cfg.EthPassword, "Password for existing Eth account address or path to file") cfg.EthKeystorePath = fs.String("ethKeystorePath", *cfg.EthKeystorePath, "Path to ETH keystore directory or keyfile. If keyfile, overrides -ethAcctAddr and uses parent directory") cfg.EthExternalSigner = fs.String("ethExternalSigner", *cfg.EthExternalSigner, "JSON-RPC endpoint of a Web3Signer-compatible external signer (eth_* namespace; e.g. Web3Signer fronting KMS/Vault/HSM, or a Turnkey/MPC custody sidecar). When set, signing is delegated to it instead of a local keystore and -ethAcctAddr must be the address it signs for") + cfg.EthExternalSignerTimeout = fs.Duration("ethExternalSignerTimeout", *cfg.EthExternalSignerTimeout, "Per-call timeout for -ethExternalSigner signing requests, bounding the PM ticket hot path if the signer is slow or unreachable") cfg.EthOrchAddr = fs.String("ethOrchAddr", *cfg.EthOrchAddr, "ETH address of an on-chain registered orchestrator") cfg.EthUrl = fs.String("ethUrl", *cfg.EthUrl, "Ethereum node JSON-RPC URL") cfg.TxTimeout = fs.Duration("transactionTimeout", *cfg.TxTimeout, "Amount of time to wait for an Ethereum transaction to confirm before timing out") diff --git a/cmd/livepeer/starter/starter.go b/cmd/livepeer/starter/starter.go index 5b721a8e97..e61f8507ec 100755 --- a/cmd/livepeer/starter/starter.go +++ b/cmd/livepeer/starter/starter.go @@ -125,6 +125,7 @@ type LivepeerConfig struct { EthPassword *string EthKeystorePath *string EthExternalSigner *string + EthExternalSignerTimeout *time.Duration EthOrchAddr *string EthUrl *string TxTimeout *time.Duration @@ -256,6 +257,7 @@ func DefaultLivepeerConfig() LivepeerConfig { defaultEthPassword := "" defaultEthKeystorePath := "" defaultEthExternalSigner := "" + defaultEthExternalSignerTimeout := 5 * time.Second defaultEthOrchAddr := "" defaultEthUrl := "" defaultTxTimeout := 5 * time.Minute @@ -378,41 +380,42 @@ func DefaultLivepeerConfig() LivepeerConfig { LiveAICapReportInterval: &defaultLiveAICapReportInterval, // Onchain: - EthAcctAddr: &defaultEthAcctAddr, - EthPassword: &defaultEthPassword, - EthKeystorePath: &defaultEthKeystorePath, - EthExternalSigner: &defaultEthExternalSigner, - EthOrchAddr: &defaultEthOrchAddr, - EthUrl: &defaultEthUrl, - TxTimeout: &defaultTxTimeout, - MaxTxReplacements: &defaultMaxTxReplacements, - GasLimit: &defaultGasLimit, - MaxGasPrice: &defaultMaxGasPrice, - EthController: &defaultEthController, - InitializeRound: &defaultInitializeRound, - InitializeRoundMaxDelay: &defaultInitializeRoundMaxDelay, - TicketEV: &defaultTicketEV, - MaxFaceValue: &defaultMaxFaceValue, - MaxTicketEV: &defaultMaxTicketEV, - MaxTotalEV: &defaultMaxTotalEV, - DepositMultiplier: &defaultDepositMultiplier, - MaxPricePerUnit: &defaultMaxPricePerUnit, - MaxPricePerCapability: &defaultMaxPricePerCapability, - IgnoreMaxPriceIfNeeded: &defaultIgnoreMaxPriceIfNeeded, - PixelsPerUnit: &defaultPixelsPerUnit, - PriceFeedAddr: &defaultPriceFeedAddr, - AutoAdjustPrice: &defaultAutoAdjustPrice, - PricePerGateway: &defaultPricePerGateway, - PricePerBroadcaster: &defaultPricePerBroadcaster, - BlockPollingInterval: &defaultBlockPollingInterval, - Redeemer: &defaultRedeemer, - RedeemerAddr: &defaultRedeemerAddr, - Monitor: &defaultMonitor, - MetricsPerStream: &defaultMetricsPerStream, - MetricsExposeClientIP: &defaultMetricsExposeClientIP, - MetadataQueueUri: &defaultMetadataQueueUri, - MetadataAmqpExchange: &defaultMetadataAmqpExchange, - MetadataPublishTimeout: &defaultMetadataPublishTimeout, + EthAcctAddr: &defaultEthAcctAddr, + EthPassword: &defaultEthPassword, + EthKeystorePath: &defaultEthKeystorePath, + EthExternalSigner: &defaultEthExternalSigner, + EthExternalSignerTimeout: &defaultEthExternalSignerTimeout, + EthOrchAddr: &defaultEthOrchAddr, + EthUrl: &defaultEthUrl, + TxTimeout: &defaultTxTimeout, + MaxTxReplacements: &defaultMaxTxReplacements, + GasLimit: &defaultGasLimit, + MaxGasPrice: &defaultMaxGasPrice, + EthController: &defaultEthController, + InitializeRound: &defaultInitializeRound, + InitializeRoundMaxDelay: &defaultInitializeRoundMaxDelay, + TicketEV: &defaultTicketEV, + MaxFaceValue: &defaultMaxFaceValue, + MaxTicketEV: &defaultMaxTicketEV, + MaxTotalEV: &defaultMaxTotalEV, + DepositMultiplier: &defaultDepositMultiplier, + MaxPricePerUnit: &defaultMaxPricePerUnit, + MaxPricePerCapability: &defaultMaxPricePerCapability, + IgnoreMaxPriceIfNeeded: &defaultIgnoreMaxPriceIfNeeded, + PixelsPerUnit: &defaultPixelsPerUnit, + PriceFeedAddr: &defaultPriceFeedAddr, + AutoAdjustPrice: &defaultAutoAdjustPrice, + PricePerGateway: &defaultPricePerGateway, + PricePerBroadcaster: &defaultPricePerBroadcaster, + BlockPollingInterval: &defaultBlockPollingInterval, + Redeemer: &defaultRedeemer, + RedeemerAddr: &defaultRedeemerAddr, + Monitor: &defaultMonitor, + MetricsPerStream: &defaultMetricsPerStream, + MetricsExposeClientIP: &defaultMetricsExposeClientIP, + MetadataQueueUri: &defaultMetadataQueueUri, + MetadataAmqpExchange: &defaultMetadataAmqpExchange, + MetadataPublishTimeout: &defaultMetadataPublishTimeout, // Ingest: HttpIngest: &defaultHttpIngest, @@ -851,7 +854,7 @@ func StartLivepeer(ctx context.Context, cfg LivepeerConfig) { var am eth.AccountManager if *cfg.EthExternalSigner != "" { - am, err = eth.NewWeb3SignerAccountManager(ethcommon.HexToAddress(*cfg.EthAcctAddr), *cfg.EthExternalSigner, chainID) + am, err = eth.NewWeb3SignerAccountManager(ethcommon.HexToAddress(*cfg.EthAcctAddr), *cfg.EthExternalSigner, chainID, *cfg.EthExternalSignerTimeout) } else { am, err = eth.NewAccountManager(ethcommon.HexToAddress(*cfg.EthAcctAddr), keystoreDir, chainID, *cfg.EthPassword) } diff --git a/doc/external-signer.md b/doc/external-signer.md index 87866ce5e1..99eb6f262d 100644 --- a/doc/external-signer.md +++ b/doc/external-signer.md @@ -84,6 +84,7 @@ Point the node at a Web3Signer-compatible endpoint and tell it which address tha - `-ethExternalSigner `: JSON-RPC endpoint of the external signer. When set, signing is delegated to it instead of a local keystore. - `-ethAcctAddr <0x...>`: the Ethereum address the external signer holds. Required in this mode, since there is no local keystore to default from. +- `-ethExternalSignerTimeout `: per-call timeout for signing requests (default `5s`). Bounds the PM ticket hot path so a slow or hung signer fails fast instead of blocking minting. When `-ethExternalSigner` is set, the usual keystore flags (`-ethKeystorePath`, `-ethPassword`) are not used for signing. The node fails fast at startup if the endpoint is unreachable or does not respond to the `eth_*` signing API. @@ -136,5 +137,5 @@ KMS/Vault/HSM remove key exfiltration but still sign whatever digest they are as ## Caveats - **Raw-hash signing.** PM tickets are signed as a hash. `eth_sign` / `eth_signTypedData` reconstruct the digest from the message/structured input, so backends that refuse raw precomputed-hash signing still work. Verify this against your backend before production. -- **Hot-path latency and cost.** PM mints tickets frequently. Each signature is now an API round-trip rather than a local operation, which adds latency and (for hosted backends) cost. Measure against your minting rate. +- **Hot-path latency and cost.** PM mints tickets frequently. Each signature is now an API round-trip rather than a local operation, which adds latency and (for hosted backends) cost. Measure against your minting rate, and tune `-ethExternalSignerTimeout` so a stalled signer fails fast rather than blocking minting. - **Treat the endpoint like an internal wallet service.** Run the external signer on a private network or behind an authenticated proxy, the same as the [remote signer guidance](./remote-signer.md#operational--security-guidance). diff --git a/eth/web3signer.go b/eth/web3signer.go index a326bd2e37..a51fbe47bc 100644 --- a/eth/web3signer.go +++ b/eth/web3signer.go @@ -1,8 +1,10 @@ package eth import ( + "context" "fmt" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -24,18 +26,27 @@ import ( // // Output is byte-identical to the keystore accountManager: EIP-191 message // signing, latest-signer transactions, and recovery id in {27,28}. +// defaultWeb3SignerTimeout bounds each signing round-trip so a hung or slow +// remote signer cannot block the PM ticket hot path indefinitely. +const defaultWeb3SignerTimeout = 5 * time.Second + type web3signerAccountManager struct { rpc *rpc.Client account accounts.Account chainID *big.Int + timeout time.Duration } // NewWeb3SignerAccountManager connects to a Web3Signer-compatible endpoint and // signs on behalf of accountAddr (required, since there is no local keystore). -func NewWeb3SignerAccountManager(accountAddr ethcommon.Address, endpoint string, chainID *big.Int) (AccountManager, error) { +// timeout bounds each signing call; values <= 0 fall back to a sane default. +func NewWeb3SignerAccountManager(accountAddr ethcommon.Address, endpoint string, chainID *big.Int, timeout time.Duration) (AccountManager, error) { if (accountAddr == ethcommon.Address{}) { return nil, fmt.Errorf("web3signer requires an explicit -ethAcctAddr") } + if timeout <= 0 { + timeout = defaultWeb3SignerTimeout + } rpcClient, err := rpc.Dial(endpoint) if err != nil { @@ -44,23 +55,31 @@ func NewWeb3SignerAccountManager(accountAddr ethcommon.Address, endpoint string, // eth_accounts doubles as a reachability check and lets us warn if the // remote signer does not hold the configured address. + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() var addrs []ethcommon.Address - if err := rpcClient.Call(&addrs, "eth_accounts"); err != nil { + if err := rpcClient.CallContext(ctx, &addrs, "eth_accounts"); err != nil { return nil, fmt.Errorf("failed to reach web3signer at %s: %w", endpoint, err) } if !containsAddress(addrs, accountAddr) { glog.Warningf("Web3Signer at %s did not list account %v; signing requests may be rejected", endpoint, accountAddr.Hex()) } - glog.Infof("Using web3signer at %s for Ethereum account: %v", endpoint, accountAddr.Hex()) + glog.Infof("Using web3signer at %s for Ethereum account: %v (timeout %s)", endpoint, accountAddr.Hex(), timeout) return &web3signerAccountManager{ rpc: rpcClient, account: accounts.Account{Address: accountAddr}, chainID: chainID, + timeout: timeout, }, nil } +// callContext returns a context bounded by the configured signing timeout. +func (m *web3signerAccountManager) callContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), m.timeout) +} + func containsAddress(addrs []ethcommon.Address, want ethcommon.Address) bool { for _, a := range addrs { if a == want { @@ -92,16 +111,20 @@ func (m *web3signerAccountManager) CreateTransactOpts(gasLimit uint64) (*bind.Tr // Sign signs msg with the EIP-191 personal-message prefix. Web3Signer's eth_sign // applies that prefix, matching the keystore accountManager. func (m *web3signerAccountManager) Sign(msg []byte) ([]byte, error) { + ctx, cancel := m.callContext() + defer cancel() var res hexutil.Bytes - if err := m.rpc.Call(&res, "eth_sign", m.account.Address, hexutil.Encode(msg)); err != nil { + if err := m.rpc.CallContext(ctx, &res, "eth_sign", m.account.Address, hexutil.Encode(msg)); err != nil { return nil, err } return toEthV(res), nil } func (m *web3signerAccountManager) SignTypedData(typedData apitypes.TypedData) ([]byte, error) { + ctx, cancel := m.callContext() + defer cancel() var res hexutil.Bytes - if err := m.rpc.Call(&res, "eth_signTypedData", m.account.Address, typedData); err != nil { + if err := m.rpc.CallContext(ctx, &res, "eth_signTypedData", m.account.Address, typedData); err != nil { return nil, err } return toEthV(res), nil @@ -110,8 +133,10 @@ func (m *web3signerAccountManager) SignTypedData(typedData apitypes.TypedData) ( // SignTx asks the remote signer to sign tx and returns the decoded signed // transaction. go-livepeer still owns nonce/gas and broadcasts the result. func (m *web3signerAccountManager) SignTx(tx *types.Transaction) (*types.Transaction, error) { + ctx, cancel := m.callContext() + defer cancel() var raw hexutil.Bytes - if err := m.rpc.Call(&raw, "eth_signTransaction", m.toSendTxArgs(tx)); err != nil { + if err := m.rpc.CallContext(ctx, &raw, "eth_signTransaction", m.toSendTxArgs(tx)); err != nil { return nil, err } signed := new(types.Transaction) diff --git a/eth/web3signer_test.go b/eth/web3signer_test.go index 50fe33847f..34d4c3c0f8 100644 --- a/eth/web3signer_test.go +++ b/eth/web3signer_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/ethereum/go-ethereum/accounts" ethcommon "github.com/ethereum/go-ethereum/common" @@ -68,7 +69,7 @@ func TestWeb3Signer_SignMatchesKeystoreConvention(t *testing.T) { srv := web3signerStub(t, key) defer srv.Close() - am, err := NewWeb3SignerAccountManager(addr, srv.URL, big.NewInt(1)) + am, err := NewWeb3SignerAccountManager(addr, srv.URL, big.NewInt(1), 2*time.Second) require.NoError(err) assert.Equal(addr, am.Account().Address) @@ -82,7 +83,7 @@ func TestWeb3Signer_SignMatchesKeystoreConvention(t *testing.T) { } func TestWeb3Signer_RequiresExplicitAddress(t *testing.T) { - _, err := NewWeb3SignerAccountManager(ethcommon.Address{}, "http://127.0.0.1:0", big.NewInt(1)) + _, err := NewWeb3SignerAccountManager(ethcommon.Address{}, "http://127.0.0.1:0", big.NewInt(1), 2*time.Second) assert.Error(t, err) } @@ -90,6 +91,45 @@ func TestWeb3Signer_UnreachableEndpoint(t *testing.T) { key, err := crypto.GenerateKey() require.NoError(t, err) addr := crypto.PubkeyToAddress(key.PublicKey) - _, err = NewWeb3SignerAccountManager(addr, "http://127.0.0.1:1", big.NewInt(1)) + _, err = NewWeb3SignerAccountManager(addr, "http://127.0.0.1:1", big.NewInt(1), 2*time.Second) assert.Error(t, err) } + +// A slow signer must not block the hot path: Sign returns an error promptly +// once the per-call timeout elapses, rather than waiting for the signer. +func TestWeb3Signer_SignTimesOut(t *testing.T) { + require := require.New(t) + + key, err := crypto.GenerateKey() + require.NoError(err) + addr := crypto.PubkeyToAddress(key.PublicKey) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var req struct { + ID json.RawMessage `json:"id"` + Method string `json:"method"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + if req.Method == "eth_accounts" { + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "jsonrpc": "2.0", "id": json.RawMessage(req.ID), "result": []ethcommon.Address{addr}, + }) + return + } + // eth_sign: stall well past the timeout, but unblock when the client + // cancels so the test server shuts down promptly. + select { + case <-time.After(2 * time.Second): + case <-r.Context().Done(): + } + })) + defer srv.Close() + + am, err := NewWeb3SignerAccountManager(addr, srv.URL, big.NewInt(1), 50*time.Millisecond) + require.NoError(err) + + start := time.Now() + _, err = am.Sign([]byte("ticket hash")) + require.Error(err, "Sign must fail when the signer exceeds the timeout") + require.Less(time.Since(start), time.Second, "Sign must return on timeout, not block on the slow signer") +}