Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,36 @@ Verify it bit (alpha can no longer reach bravo's discovery port):
docker exec disruptoor-test-alpha nc -vz 172.30.0.11 30303
```

Or isolate a single container from **everything else** without enumerating the
counterparties — the complement of `target` is computed at apply time, so the
same request keeps working when the enclave topology changes:

```bash
curl -X PUT http://localhost:7700/v1/state \
-H 'Content-Type: application/json' \
-d '{
"isolations": [
{
"name": "blackout-alpha",
"target": {"id": "alpha"},
"scope": ["cl_p2p", "el_p2p", "include_control"]
}
]
}'
```

An isolation is semantically a symmetric two-group partition of `target` vs
the rest of the enclave. `scope` defaults to `[cl_p2p, el_p2p]`; add
`include_control` to also cut RPC/engine/metrics/VC↔CL traffic (a full
blackout). The target must not match every container — there'd be nothing
left to isolate from.

A target matching multiple containers is isolated **as a group**: traffic
among its members keeps flowing (useful for "island" scenarios, e.g. cutting
all beacon nodes off from their EL/VC stacks while they still gossip with
each other). To black out several containers individually, declare one
isolation per container.

Heal everything:

```bash
Expand Down
8 changes: 8 additions & 0 deletions disruptoor.md
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,13 @@ Declarative, versioned, idempotent.
"scope": ["cl_p2p", "el_p2p"]
}
],
"isolations": [
{
"name": "blackout-node-5",
"target": { "node-index": "5" },
"scope": ["cl_p2p", "el_p2p", "include_control"]
}
],
"shaping": [
{
"name": "dial-up-node",
Expand All @@ -146,6 +153,7 @@ Declarative, versioned, idempotent.
- **Auth by network.** Controller binds only to the enclave network. No tokens. Opt-in `expose: true` to publish on the host.
- **Stable selectors.** The standalone API accepts label selectors. Higher-level package integrations can translate participant names into these selectors.
- **Validation.** Reject configs at PUT time: same node in two groups, unknown participants, contradictory shaping rules.
- **Isolations.** `isolations` cuts a target selector off from every other container in the enclave; the counterparty group is the complement of the target, computed at apply time. This covers the "isolate one node" scenario that partitions cannot express (groups must be disjoint and there is no negation selector).

## Configuration block in ethereum-package

Expand Down
13 changes: 13 additions & 0 deletions examples/disruption.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,19 @@ partitions:
# scope omitted → defaults to [cl_p2p, el_p2p]
# symmetric omitted → defaults to true

# Isolations cut a target off from every other container in the enclave.
# The counterparty group is computed at apply time as the complement of
# target, so it never needs to be enumerated (and stays correct when the
# enclave topology changes). Commented out here because it would overlap
# with the alpha-vs-bravo partition above in the 2-container smoke harness.
#
# isolations:
# - name: blackout-alpha
# target: { id: alpha }
# # scope omitted → defaults to [cl_p2p, el_p2p]; add include_control
# # to also cut RPC/engine/metrics/VC-CL traffic:
# # scope: [cl_p2p, el_p2p, include_control]

shaping:
- name: jitter-everything
target: all
Expand Down
67 changes: 67 additions & 0 deletions internal/api/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,16 @@ func cloneState(s state.State) state.State {
}
}
}
if len(s.Isolations) > 0 {
out.Isolations = make([]state.Isolation, len(s.Isolations))
for i, iso := range s.Isolations {
out.Isolations[i] = state.Isolation{
Name: iso.Name,
Target: cloneSelectorPtr(iso.Target),
Scope: cloneStrings(iso.Scope),
}
}
}
if len(s.Shaping) > 0 {
out.Shaping = make([]state.Shaping, len(s.Shaping))
for i, sh := range s.Shaping {
Expand Down Expand Up @@ -398,6 +408,12 @@ func (s *service) applyLocked(ctx context.Context, desired state.State) error {
_ = s.clearLocked(ctx)
return fmt.Errorf("resolve partitions: %w", err)
}
resolvedIsolations, err := s.resolveIsolations(ctx, desired.Isolations)
if err != nil {
_ = s.clearLocked(ctx)
return fmt.Errorf("resolve isolations: %w", err)
}
resolvedPartitions = append(resolvedPartitions, resolvedIsolations...)
resolvedShaping, err := s.resolveShaping(ctx, desired.Shaping)
if err != nil {
_ = s.clearLocked(ctx)
Expand Down Expand Up @@ -465,6 +481,40 @@ func (s *service) resolvePartitions(ctx context.Context, ps []state.Partition) (
return out, nil
}

// resolveIsolations expands each isolation into a two-group partition:
// the target selector vs the complement of its match set. Reusing
// ResolvedPartition means the conntrack and iptables backends need no
// isolation-specific code paths.
func (s *service) resolveIsolations(ctx context.Context, isos []state.Isolation) ([]backend.ResolvedPartition, error) {
defaultScope := []string{"cl_p2p", "el_p2p"}
out := make([]backend.ResolvedPartition, 0, len(isos))
for _, iso := range isos {
// Validate guarantees iso.Target is non-nil, non-empty, and not "all".
target, err := s.cfg.Discovery.Resolve(ctx, *iso.Target)
if err != nil {
return nil, fmt.Errorf("isolation %q: %w", iso.Name, err)
}
if len(target) == 0 {
return nil, fmt.Errorf("isolation %q: target matched no containers", iso.Name)
}
everyone, err := s.cfg.Discovery.Resolve(ctx, state.Selector{All: true})
if err != nil {
return nil, fmt.Errorf("isolation %q: resolve enclave containers: %w", iso.Name, err)
}
rest := subtractContainers(everyone, target)
if len(rest) == 0 {
return nil, fmt.Errorf("isolation %q: target matches every container in the enclave; nothing to isolate from", iso.Name)
}
out = append(out, backend.ResolvedPartition{
Name: iso.Name,
Groups: [][]discovery.Container{target, rest},
Scope: iso.EffectiveScope(defaultScope),
Symmetric: true,
})
}
return out, nil
}

func (s *service) resolveShaping(ctx context.Context, sh []state.Shaping) ([]backend.ResolvedShaping, error) {
out := make([]backend.ResolvedShaping, 0, len(sh))
for _, r := range sh {
Expand All @@ -486,6 +536,23 @@ func (s *service) resolveShaping(ctx context.Context, sh []state.Shaping) ([]bac
return out, nil
}

// subtractContainers returns the containers in from that are not in remove,
// keyed by container ID. Order of from is preserved.
func subtractContainers(from, remove []discovery.Container) []discovery.Container {
removeIDs := make(map[string]struct{}, len(remove))
for _, c := range remove {
removeIDs[c.ID] = struct{}{}
}
out := make([]discovery.Container, 0, len(from))
for _, c := range from {
if _, drop := removeIDs[c.ID]; drop {
continue
}
out = append(out, c)
}
return out
}

// statusRecorder wraps http.ResponseWriter to capture the status code that
// downstream handlers wrote, so the request-log middleware can include it.
// WriteHeader is the only hook needed because handlers that never call it
Expand Down
178 changes: 178 additions & 0 deletions internal/api/api_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,124 @@ func TestStatePutApplyErrorUsesStableResponse(t *testing.T) {
require.NotContains(t, string(respBody), "selector matched no containers")
}

func TestApplyIsolationResolvesComplement(t *testing.T) {
ipt := &recordingIptables{}
svc := newTestServiceWithDiscovery(inventoryDiscovery{"alpha", "bravo", "charlie"})
svc.cfg.Iptables = ipt

require.NoError(t, svc.Apply(context.Background(), state.State{
Isolations: []state.Isolation{{
Name: "blackout-alpha",
Target: &state.Selector{Match: map[string][]string{"id": {"alpha"}}},
Scope: []string{state.ScopeCLP2P, state.ScopeELP2P, state.ScopeControl},
}},
}))

require.Len(t, ipt.partitions, 1)
part := ipt.partitions[0]
require.Equal(t, "blackout-alpha", part.Name)
require.Len(t, part.Groups, 2)
require.Equal(t, []string{"alpha"}, containerNames(part.Groups[0]))
require.Equal(t, []string{"bravo", "charlie"}, containerNames(part.Groups[1]))
require.Equal(t, []string{state.ScopeCLP2P, state.ScopeELP2P, state.ScopeControl}, part.Scope)
require.True(t, part.Symmetric)
}

// A target matching multiple containers is isolated as a group: its members
// end up in the same partition group, so traffic among them is unaffected.
// This is load-bearing API semantics — callers wanting per-container
// blackouts declare one isolation each.
func TestApplyIsolationKeepsMultiMatchTargetAsOneGroup(t *testing.T) {
ipt := &recordingIptables{}
svc := newTestServiceWithDiscovery(inventoryDiscovery{"alpha", "bravo", "charlie", "delta"})
svc.cfg.Iptables = ipt

require.NoError(t, svc.Apply(context.Background(), state.State{
Isolations: []state.Isolation{{
Name: "island",
Target: &state.Selector{Match: map[string][]string{"id": {"alpha", "bravo"}}},
}},
}))

require.Len(t, ipt.partitions, 1)
part := ipt.partitions[0]
require.Len(t, part.Groups, 2)
require.Equal(t, []string{"alpha", "bravo"}, containerNames(part.Groups[0]),
"matched containers must share one group so intra-target traffic keeps flowing")
require.Equal(t, []string{"charlie", "delta"}, containerNames(part.Groups[1]))
}

func TestApplyAppendsIsolationsAfterPartitions(t *testing.T) {
ipt := &recordingIptables{}
svc := newTestServiceWithDiscovery(inventoryDiscovery{"alpha", "bravo", "charlie"})
svc.cfg.Iptables = ipt

require.NoError(t, svc.Apply(context.Background(), state.State{
Partitions: []state.Partition{{
Name: "split",
Groups: []state.Selector{
{Match: map[string][]string{"id": {"alpha"}}},
{Match: map[string][]string{"id": {"bravo"}}},
},
}},
Isolations: []state.Isolation{{
Name: "blackout-charlie",
Target: &state.Selector{Match: map[string][]string{"id": {"charlie"}}},
}},
}))

require.Len(t, ipt.partitions, 2)
require.Equal(t, "split", ipt.partitions[0].Name)
require.Equal(t, "blackout-charlie", ipt.partitions[1].Name)
// Isolation with no scope inherits the partition default.
require.Equal(t, []string{state.ScopeCLP2P, state.ScopeELP2P}, ipt.partitions[1].Scope)
}

func TestApplyIsolationTargetMatchingNothingFails(t *testing.T) {
svc := newTestServiceWithDiscovery(inventoryDiscovery{"alpha", "bravo"})

err := svc.Apply(context.Background(), state.State{
Isolations: []state.Isolation{{
Name: "ghost",
Target: &state.Selector{Match: map[string][]string{"id": {"missing"}}},
}},
})

require.ErrorContains(t, err, "target matched no containers")
}

func TestApplyIsolationTargetMatchingEverythingFails(t *testing.T) {
svc := newTestServiceWithDiscovery(inventoryDiscovery{"alpha", "bravo"})

err := svc.Apply(context.Background(), state.State{
Isolations: []state.Isolation{{
Name: "everyone",
Target: &state.Selector{Match: map[string][]string{"id": {"alpha", "bravo"}}},
}},
})

require.ErrorContains(t, err, "nothing to isolate from")
}

func TestGetStateDeepCopiesIsolations(t *testing.T) {
svc := newTestServiceWithDiscovery(inventoryDiscovery{"alpha", "bravo"})
require.NoError(t, svc.Apply(context.Background(), state.State{
Isolations: []state.Isolation{{
Name: "blackout",
Target: &state.Selector{Match: map[string][]string{"id": {"alpha"}}},
Scope: []string{state.ScopeCLP2P},
}},
}))

got := svc.GetState()
got.Isolations[0].Target.Match["id"][0] = "mutated"
got.Isolations[0].Scope[0] = state.ScopeControl

current := svc.GetState()
require.Equal(t, "alpha", current.Isolations[0].Target.Match["id"][0])
require.Equal(t, state.ScopeCLP2P, current.Isolations[0].Scope[0])
}

func TestApplyClearsPreviousStateBeforeConntrackFlush(t *testing.T) {
ops := &opLog{}
svc := newTestServiceWithOps(ops)
Expand Down Expand Up @@ -225,6 +343,66 @@ func (fakeDiscovery) ResolveGroups(_ context.Context, sels []state.Selector) ([]
return out, nil
}

// inventoryDiscovery resolves selectors against a fixed container inventory:
// the All selector matches everything, and Match selectors are honoured for
// the "id" key only (values name containers directly).
type inventoryDiscovery []string

func (inventoryDiscovery) Start(context.Context) error { return nil }
func (inventoryDiscovery) Stop() error { return nil }
func (inventoryDiscovery) EnclaveID() string { return "test" }
func (d inventoryDiscovery) Resolve(_ context.Context, sel state.Selector) ([]discovery.Container, error) {
out := make([]discovery.Container, 0, len(d))
for _, name := range d {
if sel.All || containsValue(sel.Match["id"], name) {
out = append(out, discovery.Container{ID: name, Name: name})
}
}
return out, nil
}

func (d inventoryDiscovery) ResolveGroups(ctx context.Context, sels []state.Selector) ([][]discovery.Container, error) {
out := make([][]discovery.Container, len(sels))
for i, sel := range sels {
matched, err := d.Resolve(ctx, sel)
if err != nil {
return nil, err
}
out[i] = matched
}
return out, nil
}

func containsValue(values []string, target string) bool {
for _, v := range values {
if v == target {
return true
}
}
return false
}

func containerNames(cs []discovery.Container) []string {
out := make([]string, 0, len(cs))
for _, c := range cs {
out = append(out, c.Name)
}
return out
}

// recordingIptables captures the resolved partitions passed to Apply.
type recordingIptables struct {
partitions []backend.ResolvedPartition
}

func (*recordingIptables) Start(context.Context) error { return nil }
func (*recordingIptables) Stop() error { return nil }
func (r *recordingIptables) Apply(_ context.Context, ps []backend.ResolvedPartition) error {
r.partitions = ps
return nil
}
func (*recordingIptables) Clear(context.Context) error { return nil }

type emptyGroupDiscovery struct {
fakeDiscovery
}
Expand Down
Loading