diff --git a/go/agentfield-package.yaml b/go/agentfield-package.yaml index ee449506..1b37dfd2 100644 --- a/go/agentfield-package.yaml +++ b/go/agentfield-package.yaml @@ -63,6 +63,61 @@ user_environment: description: Engine reasoning-effort variant (low | high) — unset keeps the engine default - name: SWE_PRO_MAX_COST description: Per-run USD ceiling for the engine — unset means no per-run cap + - name: SWE_FURROW_ENABLED + description: >- + Furrow workspace mirroring. Set 1/true/yes/on to force it on, 0 to force it + off. Unset, it follows FURROW_PUBLIC_ADDR: cloud deploys (where the desktop + app provisions a public sync port and sets that variable) mirror out of the + box, local installs stay off. Mirroring copies the whole build workspace — + including untracked files and any secrets the coder wrote there — into a + second encrypted store on this node. + # No default on purpose: a non-empty default is injected into the node's + # environment as an EXPLICIT value, which would defeat the unset-follows- + # FURROW_PUBLIC_ADDR behaviour that turns mirroring on in cloud deploys. + default: "" + - name: SWE_FURROW_EXPOSE_SECRETS + description: >- + Return the recovery key and transport token from get_workspace_handle instead of + redacting them. That reasoner authorizes no one — any caller with a run ID is + answered — and the key decrypts the workspace while the token grants read-write + access to its remote. Enable only on a single-tenant, trusted cluster. + default: "0" + - name: SWE_FURROW_BIN + description: Path to the furrow client binary — unset auto-resolves (/usr/local/bin/furrow, then vendored bin/furrow--) + default: "" + - name: SWE_FURROWD_BIN + description: Path to the furrow daemon binary — unset auto-resolves (/usr/local/bin/furrowd, then vendored bin/furrowd--) + default: "" + - name: SWE_FURROW_DATA_DIR + description: Furrow client data directory — unset uses $AGENTFIELD_HOME/furrow/store when AGENTFIELD_HOME is set, otherwise /.furrow-store + default: "" + - name: SWE_FURROW_REMOTES_ROOT + description: Furrow remote stores root — unset uses $AGENTFIELD_HOME/furrow/remotes when AGENTFIELD_HOME is set, otherwise /.furrow-remotes + default: "" + - name: SWE_FURROW_TTL_HOURS + description: Retention time for mirrored workspaces in hours + default: "72" + - name: SWE_FURROW_MAX_GB + description: Maximum aggregate furrow store size in GB; 0 means unlimited disk (no budget eviction) + default: "20" + - name: FURROWD_ADDR + description: Address on which furrowd listens + default: ":8802" + - name: FURROW_PUBLIC_ADDR + description: >- + Public furrowd host and port advertised in ssh:// workspace handles. The + AgentField desktop app's cloud deploy sets this on the control-plane service + (a Railway TCP proxy in front of furrowd's port 8802) and nodes inherit it; + its presence also turns mirroring on unless SWE_FURROW_ENABLED says otherwise. + default: "" + - name: FURROW_DIAL_TOKEN + description: Token from an ssh:// workspace handle used by furrow clients + - name: FURROW_DIAL_INSECURE + description: Allow the default self-signed furrowd certificate; required for ssh:// handles until certificate pinning is available + - name: FURROWD_TLS_CERT + description: Path to an operator-provided furrowd TLS certificate + - name: FURROWD_TLS_KEY + description: Path to the matching operator-provided furrowd TLS private key - name: AGENTFIELD_SERVER description: Control-plane URL default: http://localhost:8080 diff --git a/go/bin/furrow-dial-linux-amd64 b/go/bin/furrow-dial-linux-amd64 new file mode 100755 index 00000000..957a672f Binary files /dev/null and b/go/bin/furrow-dial-linux-amd64 differ diff --git a/go/bin/furrow-linux-amd64 b/go/bin/furrow-linux-amd64 new file mode 100755 index 00000000..a6d58104 Binary files /dev/null and b/go/bin/furrow-linux-amd64 differ diff --git a/go/bin/furrowd-linux-amd64 b/go/bin/furrowd-linux-amd64 new file mode 100755 index 00000000..a1e74b8a Binary files /dev/null and b/go/bin/furrowd-linux-amd64 differ diff --git a/go/cmd/furrow-dial/main.go b/go/cmd/furrow-dial/main.go new file mode 100644 index 00000000..d1d3d089 --- /dev/null +++ b/go/cmd/furrow-dial/main.go @@ -0,0 +1,103 @@ +// furrow-dial is a FURROW_SSH_COMMAND shim. With FURROW_DIAL_INSECURE=1 TLS +// certificate verification is disabled; furrow's encrypted payload remains +// the confidentiality boundary, but transport authentication is then by token. +package main + +import ( + "bufio" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "os" + "strings" +) + +func main() { + if err := dial(os.Args[1:], os.Stdin, os.Stdout); err != nil { + fmt.Fprintf(os.Stderr, "furrow-dial: %v\n", err) + os.Exit(1) + } +} + +func dial(args []string, stdin io.Reader, stdout io.Writer) error { + if len(args) == 0 { + return errors.New("missing namespace") + } + namespace := args[len(args)-1] + addr := os.Getenv("FURROW_DIAL_ADDR") + if addr == "" { + for i, arg := range args { + if arg == "--" && i+1 < len(args) { + addr = args[i+1] + break + } + } + } + if addr == "" { + return errors.New("FURROW_DIAL_ADDR is unset and argv has no host") + } + token := os.Getenv("FURROW_DIAL_TOKEN") + if token == "" || strings.ContainsAny(token, " \r\n") || strings.ContainsAny(namespace, " \r\n") { + return errors.New("missing or invalid authentication parameters") + } + insecure := os.Getenv("FURROW_DIAL_INSECURE") == "1" + if insecure { + fmt.Fprintln(os.Stderr, "furrow-dial: warning: TLS certificate verification disabled; relying on furrow payload encryption") + } + host, _, err := net.SplitHostPort(addr) + if err != nil { + return fmt.Errorf("invalid address: %w", err) + } + conn, err := tls.Dial("tcp", addr, &tls.Config{ServerName: host, InsecureSkipVerify: insecure, MinVersion: tls.VersionTLS12}) //nolint:gosec -- explicitly operator-controlled for the generated self-signed certificate. + if err != nil { + return fmt.Errorf("connect: %w", err) + } + defer conn.Close() + if _, err := fmt.Fprintf(conn, "AUTH %s %s\n", token, namespace); err != nil { + return fmt.Errorf("authenticate: %w", err) + } + reader := bufio.NewReader(conn) + line, err := reader.ReadString('\n') + if err != nil || line != "OK\n" { + return errors.New("authentication failed") + } + inputDone := make(chan error, 1) + go func() { + _, err := io.Copy(conn, stdin) + if tcp, ok := conn.NetConn().(*net.TCPConn); ok { + _ = tcp.CloseWrite() + } + inputDone <- err + }() + outputDone := make(chan error, 1) + go func() { + _, err := io.Copy(stdout, reader) + outputDone <- err + }() + var inputErr, outputErr error + select { + case outputErr = <-outputDone: + // The remote side ended first. Closing the connection makes a pending + // socket write fail; a goroutine blocked reading an interactive stdin + // is harmless because process exit releases it. + _ = conn.Close() + return copyError("receive output", outputErr) + case inputErr = <-inputDone: + // A stdin EOF is a half-close: retain the read side so the remote can + // flush its final protocol response before it exits. + outputErr = <-outputDone + } + if inputErr != nil && !errors.Is(inputErr, net.ErrClosed) { + return fmt.Errorf("send input: %w", inputErr) + } + return copyError("receive output", outputErr) +} + +func copyError(operation string, err error) error { + if err != nil && !errors.Is(err, net.ErrClosed) { + return fmt.Errorf("%s: %w", operation, err) + } + return nil +} diff --git a/go/cmd/furrow-dial/main_test.go b/go/cmd/furrow-dial/main_test.go new file mode 100644 index 00000000..6d1f5bc0 --- /dev/null +++ b/go/cmd/furrow-dial/main_test.go @@ -0,0 +1,21 @@ +package main + +import ( + "bytes" + "strings" + "testing" +) + +func TestDialRejectsMissingNamespace(t *testing.T) { + if err := dial(nil, strings.NewReader(""), &bytes.Buffer{}); err == nil { + t.Fatal("dial accepted missing namespace") + } +} + +func TestDialRejectsMissingAddress(t *testing.T) { + t.Setenv("FURROW_DIAL_ADDR", "") + t.Setenv("FURROW_DIAL_TOKEN", "token") + if err := dial([]string{"workspace"}, strings.NewReader(""), &bytes.Buffer{}); err == nil { + t.Fatal("dial accepted missing address") + } +} diff --git a/go/cmd/furrowd/main.go b/go/cmd/furrowd/main.go new file mode 100644 index 00000000..7f08b627 --- /dev/null +++ b/go/cmd/furrowd/main.go @@ -0,0 +1,398 @@ +package main + +import ( + "bufio" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/subtle" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "log" + "math/big" + "net" + "os" + "os/exec" + "os/signal" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +const ( + defaultAddr = ":8802" + defaultRemotesRoot = "/var/lib/swe-af/furrow/remotes" + defaultMaxConns = 32 +) + +type config struct { + addr, root, cert, key, furrowBin string + maxConns int +} + +type server struct { + cfg config + sem chan struct{} + wg sync.WaitGroup + + // Live connections, so shutdown can close them. Closing only the listener + // leaves handlers blocked in io.Copy on a client that sends nothing and + // never hangs up, and the wg.Wait below then never returns — SIGTERM would + // hang the daemon indefinitely. Reproduced as a test failure ("server did + // not shut down") under parallel load. + mu sync.Mutex + conns map[net.Conn]struct{} + shutdown bool +} + +// track registers a live connection, reporting false once shutdown has begun so +// a connection accepted in the race window is closed rather than served. +func (s *server) track(conn net.Conn) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.shutdown { + return false + } + if s.conns == nil { + s.conns = make(map[net.Conn]struct{}) + } + s.conns[conn] = struct{}{} + return true +} + +func (s *server) untrack(conn net.Conn) { + s.mu.Lock() + delete(s.conns, conn) + s.mu.Unlock() +} + +// closeConns unblocks every in-flight handler so the daemon can actually exit. +func (s *server) closeConns() { + s.mu.Lock() + defer s.mu.Unlock() + s.shutdown = true + for conn := range s.conns { + _ = conn.Close() + } +} + +func main() { + cfg, err := configFromEnv() + if err != nil { + log.Fatalf("furrowd: %v", err) + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := run(ctx, cfg, nil); err != nil && !errors.Is(err, context.Canceled) { + log.Fatalf("furrowd: %v", err) + } +} + +func configFromEnv() (config, error) { + c := config{addr: envOr("FURROWD_ADDR", defaultAddr), root: envOr("FURROWD_REMOTES_ROOT", defaultRemotesRoot), maxConns: defaultMaxConns} + c.cert, c.key = os.Getenv("FURROWD_TLS_CERT"), os.Getenv("FURROWD_TLS_KEY") + if (c.cert == "") != (c.key == "") { + return c, errors.New("FURROWD_TLS_CERT and FURROWD_TLS_KEY must be set together") + } + if c.cert == "" { + c.cert, c.key = filepath.Join(c.root, "furrowd.crt"), filepath.Join(c.root, "furrowd.key") + } + if value := os.Getenv("FURROWD_MAX_CONNECTIONS"); value != "" { + n, err := strconv.Atoi(value) + if err != nil || n < 1 { + return c, fmt.Errorf("invalid FURROWD_MAX_CONNECTIONS %q", value) + } + c.maxConns = n + } + var ok bool + c.furrowBin, ok = resolveFurrowBin() + if !ok { + return c, fmt.Errorf("no runnable furrow binary at %s", c.furrowBin) + } + return c, nil +} + +func resolveFurrowBin() (string, bool) { + if value := os.Getenv("SWE_FURROW_BIN"); value != "" { + return value, runnable(value) + } + const defaultBin = "/usr/local/bin/furrow" + if runnable(defaultBin) { + return defaultBin, true + } + if executable, err := os.Executable(); err == nil { + for _, name := range []string{"furrow-" + runtime.GOOS + "-" + runtime.GOARCH, "furrow"} { + path := filepath.Join(filepath.Dir(executable), name) + if runnable(path) { + return path, true + } + } + } + return defaultBin, false +} + +func runnable(path string) bool { + info, err := os.Stat(path) + return err == nil && !info.IsDir() && info.Mode().Perm()&0o111 != 0 +} + +func run(ctx context.Context, cfg config, ready chan<- net.Addr) error { + certificate, err := loadOrCreateCertificate(cfg.cert, cfg.key) + if err != nil { + return fmt.Errorf("prepare TLS certificate: %w", err) + } + listener, err := tls.Listen("tcp", cfg.addr, &tls.Config{Certificates: []tls.Certificate{certificate}, MinVersion: tls.VersionTLS12}) + if err != nil { + return fmt.Errorf("listen on %s: %w", cfg.addr, err) + } + defer listener.Close() + if ready != nil { + ready <- listener.Addr() + } + s := &server{cfg: cfg, sem: make(chan struct{}, cfg.maxConns)} + go func() { + <-ctx.Done() + _ = listener.Close() + s.closeConns() + }() + for { + conn, err := listener.Accept() + if err != nil { + if ctx.Err() != nil { + s.wg.Wait() + return ctx.Err() + } + return fmt.Errorf("accept connection: %w", err) + } + select { + case s.sem <- struct{}{}: + s.wg.Add(1) + go s.serveSafely(conn) + default: + log.Printf("furrowd: warning: connection rejected") + _ = conn.Close() + } + } +} + +func (s *server) serveSafely(conn net.Conn) { + defer s.wg.Done() + defer func() { <-s.sem }() + defer conn.Close() + if !s.track(conn) { + return + } + defer s.untrack(conn) + defer func() { + if recovered := recover(); recovered != nil { + log.Printf("furrowd: warning: recovered serving connection") + } + }() + if err := s.serve(conn); err != nil { + log.Printf("furrowd: warning: connection rejected") + } +} + +func (s *server) serve(conn net.Conn) error { + if err := conn.SetDeadline(time.Now().Add(10 * time.Second)); err != nil { + return err + } + if tlsConn, ok := conn.(*tls.Conn); ok { + if err := tlsConn.Handshake(); err != nil { + return err + } + } + reader := bufio.NewReaderSize(conn, 513) + lineBytes, err := reader.ReadSlice('\n') + if err != nil || len(lineBytes) > 512 { + return errors.New("invalid authentication") + } + line := string(lineBytes) + parts := strings.Split(strings.TrimSuffix(line, "\n"), " ") + if len(parts) != 3 || parts[0] != "AUTH" || parts[1] == "" || parts[2] == "" { + return errors.New("invalid authentication") + } + entry, ok := lookupEntry(filepath.Join(s.cfg.root, "registry.json"), parts[1]) + if !ok || !validNamespace(parts[2]) { + return errors.New("invalid authentication") + } + // The namespace on the wire is furrow's *blinded* name — a keyed BLAKE3 + // digest of the human one, which never leaves the client — so it cannot be + // compared against the registry's namespace. It does not need to be: the + // token already pins this connection to one run's data root, and the + // namespace only selects a directory beneath it. Charset validation above is + // what keeps that selection inside the root. + if err := conn.SetDeadline(time.Time{}); err != nil { + return err + } + if _, err := io.WriteString(conn, "OK\n"); err != nil { + return err + } + return s.runChild(conn, reader, entry, parts[2]) +} + +// validNamespace mirrors furrow's own namespace rule: [A-Za-z0-9._-], at most 96 +// bytes, and never a path traversal. +func validNamespace(namespace string) bool { + if namespace == "" || len(namespace) > 96 || namespace == "." || namespace == ".." { + return false + } + // The namespace is attacker-supplied text that becomes an argv element of + // `furrow __remote `. '-' is in the permitted charset, so a + // LEADING one would reach furrow looking like a flag. How furrow's parser + // treats that is not ours to assume — and a "--" separator only helps if it + // honours one — so the shape is refused here instead. No legitimate + // namespace starts with '-': the manager derives them from run IDs. + if namespace[0] == '-' { + return false + } + for i := 0; i < len(namespace); i++ { + c := namespace[i] + switch { + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9': + case c == '.', c == '_', c == '-': + default: + return false + } + } + return true +} + +// registryRow is deliberately NARROWER than furrow.Entry, and that is its whole +// reason for existing. furrowd is the only network-facing process in this +// feature, and Entry carries the run's furrow recovery key — the secret that +// decrypts the mirror. Decoding the registry into Entry pulled EVERY run's +// recovery key into this process's address space on every authentication +// attempt, including attempts from an unauthenticated stranger. The daemon +// needs a token to compare and a directory to serve; the key is not a field +// here, so it is never parsed, never held, and never available to dump. +// +// The json tags must stay in step with furrow.Entry's (types.go) — the manager +// writes that struct and this reads its file. +type registryRow struct { + Namespace string `json:"namespace"` + Token string `json:"token,omitempty"` + StoreDir string `json:"store_dir"` +} + +func lookupEntry(path, token string) (registryRow, bool) { + data, err := os.ReadFile(path) + if err != nil { + return registryRow{}, false + } + entries := make(map[string]registryRow) + if json.Unmarshal(data, &entries) != nil { + return registryRow{}, false + } + var match registryRow + found := 0 + for _, entry := range entries { + if subtle.ConstantTimeCompare([]byte(token), []byte(entry.Token)) == 1 { + match = entry + found = 1 + } + } + return match, found == 1 +} + +func (s *server) runChild(conn net.Conn, input io.Reader, entry registryRow, namespace string) error { + // The manager records the run's store under a SANITIZED directory name; + // rebuilding the path from the raw run ID here would serve the wrong + // directory for any ID sanitization alters — and hand a traversal-shaped + // ID a path outside the root. + dataDir := entry.StoreDir + if dataDir == "" { + dataDir = filepath.Join(s.cfg.root, entry.Namespace) + } + cmd := exec.Command(s.cfg.furrowBin, "__remote", namespace) + cmd.Env = append(os.Environ(), "FURROW_REMOTE_DATA_DIR="+dataDir) + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + stdin, err := cmd.StdinPipe() + if err != nil { + return fmt.Errorf("create child stdin: %w", err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("create child stdout: %w", err) + } + if err := cmd.Start(); err != nil { + return fmt.Errorf("start furrow: %w", err) + } + done := make(chan error, 1) + go func() { _, err := io.Copy(conn, stdout); done <- err }() + _, copyErr := io.Copy(stdin, input) + _ = stdin.Close() + wait := make(chan error, 1) + go func() { wait <- cmd.Wait() }() + var waitErr error + select { + case waitErr = <-wait: + case <-time.After(time.Second): + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + waitErr = <-wait + } + outputErr := <-done + if waitErr != nil { + log.Printf("furrowd: furrow child exited non-zero: %v", waitErr) + } + if copyErr != nil { + return fmt.Errorf("copy client input: %w", copyErr) + } + if outputErr != nil && !errors.Is(outputErr, net.ErrClosed) { + return fmt.Errorf("copy child output: %w", outputErr) + } + return nil +} + +func loadOrCreateCertificate(certPath, keyPath string) (tls.Certificate, error) { + certificate, err := tls.LoadX509KeyPair(certPath, keyPath) + if err == nil { + return certificate, nil + } + if !errors.Is(err, os.ErrNotExist) { + return tls.Certificate{}, err + } + if err := os.MkdirAll(filepath.Dir(certPath), 0o700); err != nil { + return tls.Certificate{}, err + } + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return tls.Certificate{}, err + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return tls.Certificate{}, err + } + now := time.Now() + template := x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: "furrowd"}, NotBefore: now.Add(-time.Minute), NotAfter: now.AddDate(10, 0, 0), KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, DNSNames: []string{"localhost"}, IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}} + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + if err != nil { + return tls.Certificate{}, err + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + if err := os.WriteFile(certPath, certPEM, 0o600); err != nil { + return tls.Certificate{}, err + } + if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil { + return tls.Certificate{}, err + } + return tls.X509KeyPair(certPEM, keyPEM) +} + +func envOr(name, fallback string) string { + if value := os.Getenv(name); value != "" { + return value + } + return fallback +} diff --git a/go/cmd/furrowd/main_test.go b/go/cmd/furrowd/main_test.go new file mode 100644 index 00000000..f8005234 --- /dev/null +++ b/go/cmd/furrowd/main_test.go @@ -0,0 +1,358 @@ +package main + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "syscall" + "testing" + "time" + + "github.com/Agent-Field/SWE-AF/go/internal/furrow" +) + +type testServer struct { + addr string + root string + cancel context.CancelFunc + done chan error +} + +func startTestServer(t *testing.T, maxConns int, script string) testServer { + t.Helper() + root := t.TempDir() + helper := filepath.Join(root, "furrow-helper") + if err := os.WriteFile(helper, []byte("#!/bin/sh\n"+script+"\n"), 0o700); err != nil { + t.Fatal(err) + } + entries := map[string]furrow.Entry{"run-1": {Token: "correct-token", Namespace: "workspace"}} + data, _ := json.Marshal(entries) + if err := os.WriteFile(filepath.Join(root, "registry.json"), data, 0o600); err != nil { + t.Fatal(err) + } + cfg := config{addr: "127.0.0.1:0", root: root, cert: filepath.Join(root, "cert.pem"), key: filepath.Join(root, "key.pem"), furrowBin: helper, maxConns: maxConns} + ctx, cancel := context.WithCancel(context.Background()) + ready := make(chan net.Addr, 1) + done := make(chan error, 1) + go func() { done <- run(ctx, cfg, ready) }() + var addr net.Addr + select { + case addr = <-ready: + case err := <-done: + t.Fatalf("server failed to start: %v", err) + case <-time.After(5 * time.Second): + t.Fatal("server did not start") + } + s := testServer{addr: addr.String(), root: root, cancel: cancel, done: done} + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(4 * time.Second): + t.Error("server did not shut down") + } + }) + return s +} + +func buildDialer(t *testing.T) string { + t.Helper() + path := filepath.Join(t.TempDir(), "furrow-dial") + if runtime.GOOS == "windows" { + path += ".exe" + } + cmd := exec.Command("go", "build", "-o", path, "../furrow-dial") + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("build dialer: %v\n%s", err, output) + } + return path +} + +func dialTLS(t *testing.T, addr string) *tls.Conn { + t.Helper() + conn, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: true}) //nolint:gosec -- test certificate. + if err != nil { + t.Fatal(err) + } + return conn +} + +func TestHappyPathRoundTripThroughDialer(t *testing.T) { + s := startTestServer(t, 32, `printf 'MARKER:'; cat`) + cmd := exec.Command(buildDialer(t), "-T", "-o", "BatchMode=yes", "--", s.addr, "furrow", "__remote", "workspace") + cmd.Env = append(os.Environ(), "FURROW_DIAL_TOKEN=correct-token", "FURROW_DIAL_INSECURE=1") + cmd.Stdin = strings.NewReader("ciphertext") + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + t.Fatalf("dialer failed: %v: %s", err, stderr.String()) + } + if got, want := stdout.String(), "MARKER:ciphertext"; got != want { + t.Fatalf("round trip = %q, want %q", got, want) + } +} + +// furrow never puts the human workspace name on the wire: it sends a keyed +// BLAKE3 digest of it, which the node has no way to predict or recompute. A +// connection therefore has to be accepted on the strength of its token alone, +// with the namespace passed through to the child untouched. Comparing it to the +// registry's namespace rejected every real clone. +func TestBlindedNamespaceIsAcceptedAndPassedThrough(t *testing.T) { + s := startTestServer(t, 32, `printf 'NS:'; cat`) + blinded := "54678c944c726f3f5e1af8a279d2a42c" // shape furrow actually sends + cmd := exec.Command(buildDialer(t), "-T", "-o", "BatchMode=yes", "--", s.addr, "furrow", "__remote", blinded) + cmd.Env = append(os.Environ(), "FURROW_DIAL_TOKEN=correct-token", "FURROW_DIAL_INSECURE=1") + cmd.Stdin = strings.NewReader("payload") + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + if err := cmd.Run(); err != nil { + t.Fatalf("dialer failed for blinded namespace: %v: %s", err, stderr.String()) + } + if got, want := stdout.String(), "NS:payload"; got != want { + t.Fatalf("round trip = %q, want %q", got, want) + } +} + +// The manager records each run's store under a SANITIZED directory; the +// daemon must serve the recorded path, not one rebuilt from the raw run ID — +// a traversal-shaped ID would otherwise name a directory outside the root. +func TestChildServesRecordedStoreDirNotRawRunID(t *testing.T) { + s := startTestServer(t, 32, `printf 'DIR:%s' "$FURROW_REMOTE_DATA_DIR"`) + storeDir := filepath.Join(s.root, "remotes", "run") + entries := map[string]furrow.Entry{"../escape": {Token: "dir-token", Namespace: "run", StoreDir: storeDir}} + data, _ := json.Marshal(entries) + if err := os.WriteFile(filepath.Join(s.root, "registry.json"), data, 0o600); err != nil { + t.Fatal(err) + } + cmd := exec.Command(buildDialer(t), "-T", "-o", "BatchMode=yes", "--", s.addr, "furrow", "__remote", "workspace") + cmd.Env = append(os.Environ(), "FURROW_DIAL_TOKEN=dir-token", "FURROW_DIAL_INSECURE=1") + cmd.Stdin = strings.NewReader("") + var stdout, stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + if err := cmd.Run(); err != nil { + t.Fatalf("dialer failed: %v: %s", err, stderr.String()) + } + if got, want := stdout.String(), "DIR:"+storeDir; got != want { + t.Fatalf("data dir = %q, want %q", got, want) + } +} + +func TestAuthenticationRejected(t *testing.T) { + s := startTestServer(t, 32, `cat`) + tests := []struct { + name, line string + }{ + {"wrong token", "AUTH wrong workspace\n"}, + {"unknown token", "AUTH absent workspace\n"}, + {"garbage", "hello\n"}, + {"oversized", strings.Repeat("x", 513) + "\n"}, + // The namespace is attacker-chosen text used to pick a directory under + // the run's data root, so anything outside furrow's own charset — + // especially a traversal — must never reach the child. + {"namespace traversal", "AUTH correct-token ../../etc\n"}, + {"namespace slash", "AUTH correct-token a/b\n"}, + {"namespace dotdot", "AUTH correct-token ..\n"}, + {"namespace too long", "AUTH correct-token " + strings.Repeat("n", 97) + "\n"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + conn := dialTLS(t, s.addr) + defer conn.Close() + if _, err := io.WriteString(conn, tc.line); err != nil { + t.Fatal(err) + } + _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + data, _ := io.ReadAll(conn) + if len(data) != 0 { + t.Fatalf("rejection disclosed %q", data) + } + }) + } +} + +func TestChildKilledWhenClientDisconnects(t *testing.T) { + pidFile := filepath.Join(t.TempDir(), "pid") + t.Setenv("CHILD_PID_FILE", pidFile) + s := startTestServer(t, 32, `echo $$ > "$CHILD_PID_FILE"; trap '' TERM; while :; do sleep 1; done`) + conn := dialTLS(t, s.addr) + if _, err := io.WriteString(conn, "AUTH correct-token workspace\n"); err != nil { + t.Fatal(err) + } + buf := make([]byte, 3) + if _, err := io.ReadFull(conn, buf); err != nil || string(buf) != "OK\n" { + t.Fatalf("auth response %q, %v", buf, err) + } + deadline := time.Now().Add(3 * time.Second) + for { + if _, err := os.Stat(pidFile); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("child did not publish pid") + } + time.Sleep(10 * time.Millisecond) + } + pids, _ := os.ReadFile(pidFile) + var pid int + if _, err := fmt.Sscanf(string(pids), "%d", &pid); err != nil { + t.Fatal(err) + } + _ = conn.Close() + deadline = time.Now().Add(4 * time.Second) + for { + err := syscall.Kill(pid, 0) + if err == syscall.ESRCH { + break + } + if time.Now().After(deadline) { + t.Fatalf("child %d remains alive", pid) + } + time.Sleep(20 * time.Millisecond) + } +} + +// A client that authenticates and then goes silent must not be able to hold +// the daemon open: cancelling has to close live connections, not just the +// listener, or SIGTERM hangs forever waiting on that handler. +func TestShutdownClosesIdleAuthenticatedConnection(t *testing.T) { + s := startTestServer(t, 32, `cat`) + conn := dialTLS(t, s.addr) + defer conn.Close() + if _, err := io.WriteString(conn, "AUTH correct-token workspace\n"); err != nil { + t.Fatal(err) + } + reply := make([]byte, 3) + if _, err := io.ReadFull(conn, reply); err != nil || string(reply) != "OK\n" { + t.Fatalf("auth reply = %q, %v", reply, err) + } + // Authenticated and now idle: the handler is blocked copying client input. + s.cancel() + select { + case err := <-s.done: + s.done <- err // put it back; the harness's cleanup reads this too + case <-time.After(10 * time.Second): + t.Fatal("server did not shut down while an idle client was connected") + } +} + +func TestConcurrencyLimitEnforced(t *testing.T) { + s := startTestServer(t, 1, `cat`) + first := dialTLS(t, s.addr) + defer first.Close() + if _, err := io.WriteString(first, "AUTH correct-token workspace\n"); err != nil { + t.Fatal(err) + } + response := make([]byte, 3) + if _, err := io.ReadFull(first, response); err != nil { + t.Fatal(err) + } + second, err := tls.Dial("tcp", s.addr, &tls.Config{InsecureSkipVerify: true}) //nolint:gosec -- test certificate. + if err != nil { + return + } + defer second.Close() + _ = second.SetDeadline(time.Now().Add(2 * time.Second)) + _, writeErr := io.WriteString(second, "AUTH correct-token workspace\n") + if writeErr == nil { + _, writeErr = io.ReadAll(second) + } + if writeErr == nil { + t.Fatal("second connection was not rejected at the concurrency limit") + } +} + +// furrowd is the only process in this feature listening on a network socket, +// and lookupEntry runs on every AUTH line an unauthenticated stranger sends. +// It used to decode the registry into furrow.Entry, whose Key field IS the +// run's recovery key, so every key on the node was resident in the daemon's +// memory during that read. The daemon needs a token to compare and a directory +// to serve; nothing here may parse the key. +func TestRegistryReadNeverDecodesRecoveryKeys(t *testing.T) { + rowType := reflect.TypeOf(registryRow{}) + for i := 0; i < rowType.NumField(); i++ { + field := rowType.Field(i) + name := strings.Split(field.Tag.Get("json"), ",")[0] + if name == "key" { + t.Fatalf("furrowd decodes the recovery key through field %s", field.Name) + } + } + // And the field really is present in the file being read, so the check + // above is about what we DECODE, not about what happens to be on disk. + entryType := reflect.TypeOf(furrow.Entry{}) + keyField, ok := entryType.FieldByName("Key") + if !ok || strings.Split(keyField.Tag.Get("json"), ",")[0] != "key" { + t.Fatal("furrow.Entry no longer writes a \"key\" field; revisit this test") + } + + root := t.TempDir() + storeDir := filepath.Join(root, "remotes", "run-1") + const recoveryKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + entries := map[string]furrow.Entry{"run-1": { + RunID: "run-1", RepoPath: "/work/repo", Namespace: "workspace", + Key: recoveryKey, Token: "correct-token", StoreDir: storeDir, + }} + data, err := json.Marshal(entries) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(root, "registry.json") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), recoveryKey) { + t.Fatal("registry fixture does not actually contain a recovery key") + } + + entry, ok := lookupEntry(path, "correct-token") + if !ok { + t.Fatal("narrowing the registry read broke authentication") + } + if entry.StoreDir != storeDir || entry.Namespace != "workspace" { + t.Fatalf("entry = %+v, want the recorded store dir and namespace", entry) + } + if rendered := fmt.Sprintf("%+v", entry); strings.Contains(rendered, recoveryKey) { + t.Fatalf("recovery key reached furrowd: %s", rendered) + } +} + +// The namespace is attacker-supplied text that becomes an argv element of +// `furrow __remote `. The permitted charset includes '-', so a +// leading one would arrive at furrow looking like a flag; nothing here knows +// how furrow's parser treats that, and no namespace the manager produces ever +// starts with '-'. +func TestValidNamespaceRejectsFlagShapedInput(t *testing.T) { + for _, tc := range []struct { + namespace string + want bool + }{ + {"workspace", true}, + {"run_2026-08-10.a", true}, + {"a-b", true}, + {"-workspace", false}, + {"--force", false}, + {"-", false}, + {"", false}, + {".", false}, + {"..", false}, + {"../escape", false}, + {"has space", false}, + {strings.Repeat("a", 96), true}, + {strings.Repeat("a", 97), false}, + } { + if got := validNamespace(tc.namespace); got != tc.want { + t.Errorf("validNamespace(%q) = %v, want %v", tc.namespace, got, tc.want) + } + } +} diff --git a/go/internal/dag/executor.go b/go/internal/dag/executor.go index 2a506fc7..bb2560d8 100644 --- a/go/internal/dag/executor.go +++ b/go/internal/dag/executor.go @@ -35,6 +35,7 @@ type ExecuteFn func(ctx context.Context, issue map[string]any, dagState *schemas type runOptions struct { executeFn ExecuteFn noteFn NoteFn + levelCompleteFn func(int) gitConfig map[string]any resume bool buildID string @@ -51,6 +52,11 @@ func WithExecuteFn(fn ExecuteFn) Option { return func(o *runOptions) { o.execute // WithNoteFn sets the observability callback (Python note_fn=app.note). func WithNoteFn(fn NoteFn) Option { return func(o *runOptions) { o.noteFn = fn } } +// WithLevelCompleteFn runs fn after a DAG level's worktree cleanup completes. +func WithLevelCompleteFn(fn func(int)) Option { + return func(o *runOptions) { o.levelCompleteFn = fn } +} + // WithGitConfig sets the git configuration from run_git_init (Python git_config). func WithGitConfig(g map[string]any) Option { return func(o *runOptions) { o.gitConfig = g } } @@ -514,6 +520,9 @@ mainLoop: if err := awaitCleanup(); err != nil { return nil, err } + if o.levelCompleteFn != nil { + o.levelCompleteFn(dagState.CurrentLevel) + } dagState.CurrentLevel++ } diff --git a/go/internal/furrow/bin.go b/go/internal/furrow/bin.go new file mode 100644 index 00000000..8236f426 --- /dev/null +++ b/go/internal/furrow/bin.go @@ -0,0 +1,144 @@ +package furrow + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" +) + +const ( + EnvBin = "SWE_FURROW_BIN" + EnvDaemonBin = "SWE_FURROWD_BIN" + // EnvEnabled gates the whole feature. Mirroring copies every byte of a + // build's workspace — including the untracked files and secrets git never + // sees — into a second on-disk store, so someone has to ask for it. A set + // value is an explicit answer in either direction; unset (or blank) defers + // to EnvPublicAddr — see enabledByEnv. + EnvEnabled = "SWE_FURROW_ENABLED" + // EnvPublicAddr is the host:port furrowd is reachable at from outside the + // box, advertised in ssh:// handles. The AgentField desktop app's cloud + // deploy sets it on the control-plane service (with a TCP proxy in front + // of furrowd's port), and agent nodes inherit the control plane's + // environment — so its presence means the platform provisioned a public + // mirror endpoint for this node. + EnvPublicAddr = "FURROW_PUBLIC_ADDR" + // EnvExposeSecrets opts a node into returning a handle's recovery key and + // transport token from get_workspace_handle. That reasoner authorizes + // nobody, so the secrets are withheld unless an operator states that every + // caller which can reach this node is already trusted with the workspace. + EnvExposeSecrets = "SWE_FURROW_EXPOSE_SECRETS" + DefaultBin = "/usr/local/bin/furrow" + DefaultDaemonBin = "/usr/local/bin/furrowd" +) + +// EnvTruthy reports whether an environment variable opts a feature in. +// "1", "true", "yes" and "on" (any case, surrounding space ignored) enable it; +// "0", "false", "no", "off", empty, unset and anything unrecognised disable it. +// Deliberately the same rule as pro.Enabled — one spelling for every SWE-AF +// feature gate — and deliberately closed by default, so a typo in a flag can +// never be what switches a feature ON. +func EnvTruthy(key string) bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv(key))) { + case "1", "true", "yes", "on": + return true + } + return false +} + +// enabledByEnv decides whether mirroring is on. An explicit SWE_FURROW_ENABLED +// wins in both directions (EnvTruthy's rule: only a recognised truthy spelling +// turns a workspace-copying feature ON). When it is unset — or blank, which is +// what "not configured" looks like after an installer pass — the decision +// follows FURROW_PUBLIC_ADDR: the desktop cloud deploy sets that exactly when +// it has provisioned a public TCP endpoint for furrowd, so its presence is the +// platform asking for a reachable mirror, and a local install that never set +// either variable stays off. That is what makes a cloud control-plane deploy +// mirror out of the box while a laptop `af run` keeps today's opt-in behaviour. +func enabledByEnv() bool { + if strings.TrimSpace(os.Getenv(EnvEnabled)) != "" { + return EnvTruthy(EnvEnabled) + } + return strings.TrimSpace(os.Getenv(EnvPublicAddr)) != "" +} + +// runnable rejects copies that exist but lost their execute bit during install. +// It is a pure probe: an operator-supplied path (SWE_FURROW_BIN, /usr/local/bin) +// is never modified, so an explicit override that is not executable still fails +// loudly instead of being silently rewritten. +func runnable(path string) bool { + info, err := os.Stat(path) + return err == nil && info.Mode().IsRegular() && info.Mode().Perm()&0o111 != 0 +} + +// vendoredRunnable is runnable for the binaries WE ship, beside our own +// executable. `af` only started preserving file modes in v0.1.121 +// (agentfield#865); every older CLI copies these into the package as 0644. +// Verified live on af 0.1.119: a checkout that is rwxr-xr-x installs as +// rw-r--r--, so the node logged "no runnable furrow binary found" and the +// feature was silently off on the one platform it ships for. Repairing is safe +// precisely here — the file is one we vendored, inside our own install tree, +// and never a path anyone else chose. When the repair fails (read-only fs, +// foreign owner) the candidate stays rejected. +func vendoredRunnable(path string) bool { + if runnable(path) { + return true + } + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() { + return false + } + return os.Chmod(path, info.Mode().Perm()|0o755) == nil +} + +// ResolveDaemonBin returns the first runnable furrowd binary. An explicit +// SWE_FURROWD_BIN is authoritative; installed and sibling layouts otherwise +// follow the same order as ResolveBin. +func ResolveDaemonBin() (string, error) { + if path := os.Getenv(EnvDaemonBin); path != "" { + if runnable(path) { + return path, nil + } + return "", fmt.Errorf("furrowd binary %q is missing or not executable", path) + } + if runnable(DefaultDaemonBin) { + return DefaultDaemonBin, nil + } + if executable, err := osExecutable(); err == nil { + dir := filepath.Dir(executable) + for _, name := range []string{"furrowd-" + runtime.GOOS + "-" + runtime.GOARCH, "furrowd"} { + path := filepath.Join(dir, name) + if vendoredRunnable(path) { + return path, nil + } + } + } + return "", fmt.Errorf("no runnable furrowd binary found") +} + +var osExecutable = os.Executable + +// ResolveBin returns the first runnable furrow binary in the supported install +// layouts. An explicit override is authoritative and never falls through. +func ResolveBin() (string, error) { + if path := os.Getenv(EnvBin); path != "" { + if runnable(path) { + return path, nil + } + return "", fmt.Errorf("furrow binary %q is missing or not executable", path) + } + if runnable(DefaultBin) { + return DefaultBin, nil + } + if executable, err := osExecutable(); err == nil { + dir := filepath.Dir(executable) + for _, name := range []string{"furrow-" + runtime.GOOS + "-" + runtime.GOARCH, "furrow"} { + path := filepath.Join(dir, name) + if vendoredRunnable(path) { + return path, nil + } + } + } + return "", fmt.Errorf("no runnable furrow binary found") +} diff --git a/go/internal/furrow/bin_test.go b/go/internal/furrow/bin_test.go new file mode 100644 index 00000000..7b201192 --- /dev/null +++ b/go/internal/furrow/bin_test.go @@ -0,0 +1,131 @@ +package furrow + +import ( + "os" + "path/filepath" + "runtime" + "testing" +) + +func TestResolveBin(t *testing.T) { + dir := t.TempDir() + runnableBin := filepath.Join(dir, "runnable") + if err := os.WriteFile(runnableBin, []byte("binary"), 0o755); err != nil { + t.Fatal(err) + } + nonExecutable := filepath.Join(dir, "non-executable") + if err := os.WriteFile(nonExecutable, []byte("binary"), 0o644); err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + name string + override string + want string + wantErr bool + }{ + {"authoritative runnable override", runnableBin, runnableBin, false}, + {"authoritative missing override", filepath.Join(dir, "missing"), "", true}, + // An operator-chosen path is never rewritten: an explicit override that + // is not executable fails loudly rather than being silently chmod'd. + {"authoritative non-executable override", nonExecutable, "", true}, + {"directory is not runnable", dir, "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(EnvBin, tc.override) + got, err := ResolveBin() + if got != tc.want || (err != nil) != tc.wantErr { + t.Fatalf("ResolveBin() = (%q, %v), want (%q, error=%v)", got, err, tc.want, tc.wantErr) + } + }) + } +} + +// The real install shape that loses the bit: a vendored sibling binary next +// to the node executable, delivered rw-r--r-- by the installer. +func TestResolveBinRepairsStrippedSibling(t *testing.T) { + dir := t.TempDir() + self := filepath.Join(dir, "swe-planner") + if err := os.WriteFile(self, []byte("self"), 0o755); err != nil { + t.Fatal(err) + } + vendored := filepath.Join(dir, "furrow-"+runtime.GOOS+"-"+runtime.GOARCH) + if err := os.WriteFile(vendored, []byte("binary"), 0o644); err != nil { + t.Fatal(err) + } + orig := osExecutable + t.Cleanup(func() { osExecutable = orig }) + osExecutable = func() (string, error) { return self, nil } + t.Setenv(EnvBin, "") + + got, err := ResolveBin() + if err != nil || got != vendored { + t.Fatalf("ResolveBin() = (%q, %v), want repaired %q", got, err, vendored) + } + info, err := os.Stat(vendored) + if err != nil || info.Mode().Perm()&0o111 == 0 { + t.Fatalf("execute bit not repaired: mode=%v err=%v", info.Mode(), err) + } +} + +func TestResolveBinSiblingOrder(t *testing.T) { + if runnable(DefaultBin) { + t.Skipf("%s exists and precedes sibling binaries", DefaultBin) + } + t.Setenv(EnvBin, "") + dir := t.TempDir() + plain := filepath.Join(dir, "furrow") + suffixed := filepath.Join(dir, "furrow-"+runtime.GOOS+"-"+runtime.GOARCH) + for _, path := range []string{plain, suffixed} { + if err := os.WriteFile(path, []byte("binary"), 0o755); err != nil { + t.Fatal(err) + } + } + original := osExecutable + osExecutable = func() (string, error) { return filepath.Join(dir, "swe-af"), nil } + t.Cleanup(func() { osExecutable = original }) + got, err := ResolveBin() + if err != nil || got != suffixed { + t.Fatalf("ResolveBin() = (%q, %v), want (%q, nil)", got, err, suffixed) + } +} + +func TestResolveDaemonBin(t *testing.T) { + dir := t.TempDir() + runnableDaemon := filepath.Join(dir, "furrowd") + if err := os.WriteFile(runnableDaemon, []byte("binary"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv(EnvDaemonBin, runnableDaemon) + if got, err := ResolveDaemonBin(); err != nil || got != runnableDaemon { + t.Fatalf("ResolveDaemonBin() = (%q, %v), want (%q, nil)", got, err, runnableDaemon) + } + + missing := filepath.Join(dir, "missing") + t.Setenv(EnvDaemonBin, missing) + if got, err := ResolveDaemonBin(); err == nil || got != "" { + t.Fatalf("ResolveDaemonBin() with authoritative missing override = (%q, %v)", got, err) + } +} + +func TestResolveDaemonBinSiblingOrder(t *testing.T) { + if runnable(DefaultDaemonBin) { + t.Skipf("%s exists and precedes sibling binaries", DefaultDaemonBin) + } + t.Setenv(EnvDaemonBin, "") + dir := t.TempDir() + plain := filepath.Join(dir, "furrowd") + suffixed := filepath.Join(dir, "furrowd-"+runtime.GOOS+"-"+runtime.GOARCH) + for _, path := range []string{plain, suffixed} { + if err := os.WriteFile(path, []byte("binary"), 0o755); err != nil { + t.Fatal(err) + } + } + original := osExecutable + osExecutable = func() (string, error) { return filepath.Join(dir, "swe-af"), nil } + t.Cleanup(func() { osExecutable = original }) + got, err := ResolveDaemonBin() + if err != nil || got != suffixed { + t.Fatalf("ResolveDaemonBin() = (%q, %v), want (%q, nil)", got, err, suffixed) + } +} diff --git a/go/internal/furrow/freespace_other.go b/go/internal/furrow/freespace_other.go new file mode 100644 index 00000000..b10ae875 --- /dev/null +++ b/go/internal/furrow/freespace_other.go @@ -0,0 +1,7 @@ +//go:build !(linux || darwin) + +package furrow + +// statfsFreeBytes has no portable implementation here; reporting ok=false +// disables the free-space floor rather than inventing an answer. +func statfsFreeBytes(string) (int64, bool) { return 0, false } diff --git a/go/internal/furrow/freespace_unix.go b/go/internal/furrow/freespace_unix.go new file mode 100644 index 00000000..5b1bef1f --- /dev/null +++ b/go/internal/furrow/freespace_unix.go @@ -0,0 +1,17 @@ +//go:build linux || darwin + +package furrow + +import "syscall" + +// statfsFreeBytes reports the space available to unprivileged writers on the +// filesystem holding path. ok is false when the probe itself fails (path +// missing, filesystem not statable) — the caller treats that as "no answer", +// not "no space". +func statfsFreeBytes(path string) (int64, bool) { + var st syscall.Statfs_t + if err := syscall.Statfs(path, &st); err != nil { + return 0, false + } + return int64(st.Bavail) * int64(st.Bsize), true +} diff --git a/go/internal/furrow/integration_test.go b/go/internal/furrow/integration_test.go new file mode 100644 index 00000000..d3725791 --- /dev/null +++ b/go/internal/furrow/integration_test.go @@ -0,0 +1,153 @@ +package furrow_test + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/Agent-Field/SWE-AF/go/internal/furrow" +) + +// The unit tests inject a fake exec and assert argv, which pins what we *mean* +// to run. This one runs the real binary, because the two ways that contract has +// actually broken — a flag that does not exist (`snap --label`) and a JSON field +// spelled differently than assumed (`key_hex`) — are both invisible to a fake. +// Skips when no furrow binary is installed, so it is free for everyone else. +func TestRealBinaryAttachPublishAndClone(t *testing.T) { + bin, err := furrow.ResolveBin() + if err != nil { + t.Skipf("no furrow binary: %v", err) + } + // Mirroring is opt-in; this test is the opt-in. + t.Setenv(furrow.EnvEnabled, "1") + + root := t.TempDir() + repo := filepath.Join(root, "myrepo-b33f") + if err := os.MkdirAll(repo, 0o755); err != nil { + t.Fatal(err) + } + git(t, "", "init", "--quiet", "--", repo) + git(t, repo, "config", "user.email", "swe@af.local") + git(t, repo, "config", "user.name", "SWE AF") + write(t, filepath.Join(repo, "solver.py"), "def solve(): return 41\n") + git(t, repo, "add", "-A") + git(t, repo, "commit", "--quiet", "-m", "init") + + // The state git would never carry: an untracked secret and a dirty edit. + // Carrying these is the entire reason for mirroring rather than pushing. + write(t, filepath.Join(repo, ".env"), "TOKEN=shhh\n") + write(t, filepath.Join(repo, "solver.py"), "def solve(): return 42\n") + + m := furrow.New(furrow.Options{ + Bin: bin, + StoreRoot: filepath.Join(root, "store"), + RemotesRoot: filepath.Join(root, "remotes"), + }) + if !m.Enabled() { + t.Fatal("manager disabled with a resolvable binary") + } + + handle, err := m.Attach("run-int-0001", "b33f", repo) + if err != nil { + t.Fatalf("Attach: %v", err) + } + if handle == nil { + t.Fatal("Attach returned no handle for a real git repo") + } + if len(handle.Key) != 64 { + t.Fatalf("recovery key = %q, want 64 hex chars (is the JSON field still key_hex?)", handle.Key) + } + if handle.Token == "" || handle.Namespace == "" { + t.Fatalf("incomplete handle: %+v", handle) + } + + // Attach itself must publish a HEAD. A caller receives the handle before any + // build milestone, so it must be able to pair and materialize immediately. + dest := filepath.Join(root, "clone") + if err := os.MkdirAll(dest, 0o755); err != nil { + t.Fatal(err) + } + git(t, "", "init", "--quiet", "--", dest) + cloneStore := filepath.Join(root, "store-clone") + remoteDir, ok := strings.CutPrefix(handle.Remote, "dir:") + if !ok { + t.Fatalf("remote = %q, want a dir: handle when no public address is set", handle.Remote) + } + furrowCmd(t, bin, cloneStore, dest, "watch", "--no-daemon") + furrowCmd(t, bin, cloneStore, dest, "pair", remoteDir, "--name", handle.Namespace, "--key", handle.Key) + furrowCmd(t, bin, cloneStore, dest, "sync", "--pull", "--bootstrap") + assertFileContains(t, dest, "solver.py", "return 42") + assertFileContains(t, dest, ".env", "TOKEN=shhh") + + // Work lands after the attach, exactly as a coding agent produces it. + write(t, filepath.Join(repo, "feature.py"), "print('agent wrote this')\n") + if err := m.Publish("run-int-0001", "issue-01 complete"); err != nil { + t.Fatalf("Publish: %v", err) + } + + // Materialize on a fresh store, standing in for another machine. `furrow + // clone` only accepts ssh:// and s3:// URLs, so a directory remote is + // reproduced by the sequence clone performs internally. + // Everything below comes from the handle alone — no path assembled by the + // test. A consumer only ever has the handle, so if it is not sufficient on + // its own, the mirror is unreachable however correct the rest is. + furrowCmd(t, bin, cloneStore, dest, "sync", "--pull", "--bootstrap") + + for _, tc := range []struct{ path, want string }{ + {"solver.py", "return 42"}, // the uncommitted edit + {".env", "TOKEN=shhh"}, // untracked, git-invisible + {"feature.py", "agent wrote"}, // written after attach, carried by Publish + } { + got, err := os.ReadFile(filepath.Join(dest, tc.path)) + if err != nil { + t.Errorf("%s missing from mirror: %v", tc.path, err) + continue + } + if !strings.Contains(string(got), tc.want) { + t.Errorf("%s = %q, want it to contain %q", tc.path, got, tc.want) + } + } + if _, err := os.Stat(filepath.Join(dest, ".git")); err != nil { + t.Errorf("mirror has no .git, so the caller cannot diff or commit: %v", err) + } +} + +func assertFileContains(t *testing.T, root, path, want string) { + t.Helper() + got, err := os.ReadFile(filepath.Join(root, path)) + if err != nil { + t.Fatalf("%s missing from mirror: %v", path, err) + } + if !strings.Contains(string(got), want) { + t.Fatalf("%s = %q, want it to contain %q", path, got, want) + } +} + +func furrowCmd(t *testing.T, bin, store, repo string, args ...string) { + t.Helper() + cmd := exec.Command(bin, append([]string{"--repo", repo, "--json"}, args...)...) + cmd.Env = append(os.Environ(), "FURROW_DATA_DIR="+store) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("furrow %s: %v\n%s", strings.Join(args, " "), err, out) + } +} + +func git(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + if dir != "" { + cmd.Dir = dir + } + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } +} + +func write(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} diff --git a/go/internal/furrow/manager.go b/go/internal/furrow/manager.go new file mode 100644 index 00000000..b8d53267 --- /dev/null +++ b/go/internal/furrow/manager.go @@ -0,0 +1,646 @@ +package furrow + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + "unicode" + + "github.com/Agent-Field/SWE-AF/go/internal/workspace" +) + +// Options configures one node-wide furrow manager. +type Options struct { + Bin string + StoreRoot string + RemotesRoot string + PublicAddr string + Logger *log.Logger + Now func() time.Time + Exec func(*exec.Cmd) ([]byte, error) + CmdTimeout time.Duration + MaxBytes int64 + // BudgetGrace is how long a mirror must have gone without publishing before + // budget eviction may delete it. Zero uses defaultBudgetGrace. + BudgetGrace time.Duration +} + +// defaultBudgetGrace matches the node's sweep cadence (node.sweepFurrow ticks +// hourly): a mirror that published since the previous tick is presumed to +// belong to a build that is still running. +const defaultBudgetGrace = time.Hour + +// minFreeBytes is the free-space floor under the remotes root below which +// Attach refuses to start new mirrors. It exists for the volume the budget +// cannot see: a cloud deploy's mirrors share one disk with the control plane's +// database, and SWE_FURROW_MAX_GB says nothing about how big that disk is. +const minFreeBytes = 1 << 30 // 1 GiB + +// Manager owns the node's persistent run registry and furrow content store. +// +// Locking has two levels on purpose. mu guards the registry and is only ever +// held for map access, never across a furrow invocation: a node serves several +// builds at once, and an initial capture of a large repository takes long +// enough that holding one lock across it would stall every other run's publish. +// runLocks serializes work per run instead, which is the only ordering that +// actually matters — two calls for the same run must not both pair it. +type Manager struct { + mu sync.RWMutex + bin string + storeRoot string + remotesRoot string + publicAddr string + transportHealthy func() bool + logger *log.Logger + now func() time.Time + exec func(*exec.Cmd) ([]byte, error) + cmdTimeout time.Duration + maxBytes int64 + budgetGrace time.Duration + freeBytes func(string) (int64, bool) + enabled bool + entries map[string]Entry + runLocks map[string]*sync.Mutex +} + +// SetTransportHealth supplies the public transport health gate used when +// issuing handles. Without a gate, public transport is treated as unavailable. +func (m *Manager) SetTransportHealth(healthy func() bool) { + if m == nil { + return + } + m.mu.Lock() + m.transportHealthy = healthy + m.mu.Unlock() +} + +// lockRun serializes callers working on one run and returns its unlock. +func (m *Manager) lockRun(runID string) func() { + m.mu.Lock() + if m.runLocks == nil { + m.runLocks = make(map[string]*sync.Mutex) + } + lock, ok := m.runLocks[runID] + if !ok { + lock = &sync.Mutex{} + m.runLocks[runID] = lock + } + m.mu.Unlock() + lock.Lock() + return lock.Unlock +} + +// New constructs a manager and loads its persisted registry. Mirroring has to +// be asked for: an explicit SWE_FURROW_ENABLED decides in either direction and +// an unconfigured one follows FURROW_PUBLIC_ADDR (see enabledByEnv); when the +// answer is off the manager is inert, whatever binaries are installed. Missing +// helpers and corrupt registries deliberately degrade to an inert or empty +// manager too. +func New(opts Options) *Manager { + m := &Manager{ + storeRoot: opts.StoreRoot, + remotesRoot: opts.RemotesRoot, + publicAddr: strings.TrimSpace(opts.PublicAddr), + logger: opts.Logger, + now: opts.Now, + exec: opts.Exec, + cmdTimeout: opts.CmdTimeout, + maxBytes: opts.MaxBytes, + budgetGrace: opts.BudgetGrace, + entries: make(map[string]Entry), + } + if m.budgetGrace <= 0 { + m.budgetGrace = defaultBudgetGrace + } + if m.logger == nil { + m.logger = log.Default() + } + if m.now == nil { + m.now = time.Now + } + if m.exec == nil { + m.exec = func(cmd *exec.Cmd) ([]byte, error) { return cmd.Output() } + } + if m.cmdTimeout == 0 { + m.cmdTimeout = 5 * time.Minute + } + if m.freeBytes == nil { + m.freeBytes = statfsFreeBytes + } + if m.maxBytes == 0 { + m.maxBytes = configuredMaxBytes() + } + if m.storeRoot == "" { + m.storeRoot = filepath.Join(workspace.Root(), "furrow") + } + if m.remotesRoot == "" { + m.remotesRoot = filepath.Join(m.storeRoot, "remotes") + } + if !enabledByEnv() { + return m + } + if opts.Bin != "" { + if !runnable(opts.Bin) { + m.logf("furrow disabled: binary %q is missing or not executable", opts.Bin) + return m + } + m.bin = opts.Bin + } else { + bin, err := ResolveBin() + if err != nil { + m.logf("furrow disabled: %v", err) + return m + } + m.bin = bin + } + m.enabled = true + m.loadRegistry() + m.alignStoreBudget() + return m +} + +// alignStoreBudget caps the furrow client's own content store at half the +// node's allowance. +// +// Two directories grow: the client store (furrow's, written through +// FURROW_DATA_DIR) and the per-run remotes (ours). Only the remotes can be +// reclaimed here — retire() deletes a run's remote directory, and nothing in +// this package can free store packs. So if the store were allowed to consume +// the whole allowance, aggregateSize would stay over budget with no remote +// entries left to retire, and every later Attach would refuse forever: the +// mirror would switch itself off permanently with one log line. Halving keeps +// the total inside SWE_FURROW_MAX_GB while guaranteeing the sweeper always has +// something it can actually free. furrow enforces its half itself; verified +// with `furrow budget`, whose default happened to equal our own cap exactly. +func (m *Manager) alignStoreBudget() { + if m.maxBytes <= 0 { + return + } + if err := os.MkdirAll(m.storeRoot, 0o700); err != nil { + m.logf("furrow: could not create store root %q: %v", m.storeRoot, err) + return + } + if _, err := m.command(m.storeRoot, "budget", "--max", strconv.FormatInt(m.maxBytes/2, 10)); err != nil { + m.logf("furrow: could not set client store budget: %v", err) + } +} + +func (m *Manager) logf(format string, args ...any) { + if m != nil && m.logger != nil { + m.logger.Printf(format, args...) + } +} + +func (m *Manager) Enabled() bool { + if m == nil { + return false + } + m.mu.RLock() + defer m.mu.RUnlock() + return m.enabled +} + +func (m *Manager) command(repoPath string, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), m.cmdTimeout) + defer cancel() + argv := append([]string{"--repo", repoPath, "--json"}, args...) + cmd := exec.CommandContext(ctx, m.bin, argv...) + cmd.WaitDelay = time.Second + cmd.Env = append(os.Environ(), "FURROW_DATA_DIR="+m.storeRoot) + out, err := m.exec(cmd) + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return nil, fmt.Errorf("furrow command timeout after %s: %w", m.cmdTimeout, ctx.Err()) + } + return out, err +} + +func (m *Manager) Attach(runID, buildID, repoPath string) (*Handle, error) { + if m == nil || !m.Enabled() { + return nil, nil + } + // The run ID is BOTH the registry key and (after sanitization) the remote + // directory name, so an empty one is not a missing label — it is a shared + // one. Two builds attaching without an ID would land on the same row under + // the same namespace, and the second Attach would return the first build's + // RepoPath, recovery key and transport token. Refuse instead: a caller with + // no run ID gets no mirror, which is the same nil handle it already handles + // for a node where furrow is not installed. + if runID == "" { + m.logf("WARN furrow attach: refusing to mirror %q with no run ID; a shared key would hand one build another's recovery key", repoPath) + return nil, nil + } + if info, err := os.Stat(filepath.Join(repoPath, ".git")); err != nil || !info.IsDir() { + return nil, nil + } + + unlock := m.lockRun(runID) + defer unlock() + if handle := m.Handle(runID); handle != nil { + return handle, nil + } + if m.maxBytes > 0 { + total, err := m.aggregateSize() + if err != nil { + return nil, fmt.Errorf("furrow attach %q: measure aggregate store size: %w", runID, err) + } + if total > m.maxBytes { + err := fmt.Errorf("furrow attach %q: aggregate store size %d exceeds disk budget %d", runID, total, m.maxBytes) + m.logf("WARN %v; new mirrors are disabled until space is freed", err) + return nil, err + } + } + // The budget only protects the volume when the volume is bigger than the + // budget. A cloud deploy mirrors onto the same volume that holds the + // control plane's database, so filling it takes the whole deployment down, + // not just this feature. Refuse new mirrors when the filesystem under the + // remotes root is nearly out of space; like every other unavailable path + // this degrades to a build without a handle. (MkdirAll first: the root may + // not exist before the first mirror, and a probe on a missing path answers + // ok=false, which would silently skip the floor.) + _ = os.MkdirAll(m.remotesRoot, 0o700) + if free, ok := m.freeBytes(m.remotesRoot); ok && free < minFreeBytes { + err := fmt.Errorf("furrow attach %q: %d bytes free under %s, below the %d-byte floor", runID, free, m.remotesRoot, int64(minFreeBytes)) + m.logf("WARN %v; new mirrors are disabled until space is freed", err) + return nil, err + } + // Every line must be `exclude `; furrow rejects the whole + // file otherwise and `watch` then fails, which would leave the mirror + // silently switched off for every build. + policy := []byte("exclude .obs\nexclude node_modules\n") + if err := os.WriteFile(filepath.Join(repoPath, ".furrowpolicy"), policy, 0o644); err != nil { + m.logf("furrow attach %q: write policy: %v", runID, err) + return nil, nil + } + if _, err := m.command(repoPath, "watch", "--no-daemon"); err != nil { + m.logf("furrow attach %q: watch: %v", runID, err) + return nil, nil + } + + namespace := sanitizeNamespace(runID) + storeDir := filepath.Join(m.remotesRoot, namespace) + if err := os.MkdirAll(storeDir, 0o700); err != nil { + m.logf("furrow attach %q: create remote: %v", runID, err) + return nil, nil + } + out, err := m.command(repoPath, "remote", "add", storeDir, "--name", namespace) + if err != nil { + m.logf("furrow attach %q: pair remote: %v", runID, err) + return nil, nil + } + var paired struct { + Key string `json:"key_hex"` + } + if err := json.Unmarshal(out, &paired); err != nil || len(paired.Key) != 64 { + m.logf("furrow attach %q: invalid remote response", runID) + return nil, nil + } + if _, err := hex.DecodeString(paired.Key); err != nil { + m.logf("furrow attach %q: invalid key_hex", runID) + return nil, nil + } + tokenBytes := make([]byte, 32) + if _, err := rand.Read(tokenBytes); err != nil { + return nil, fmt.Errorf("furrow attach %q: mint token: %w", runID, err) + } + now := m.now() + entry := Entry{RunID: runID, BuildID: buildID, RepoPath: repoPath, Namespace: namespace, + Key: paired.Key, Token: hex.EncodeToString(tokenBytes), StoreDir: storeDir, + CreatedAt: now, UpdatedAt: now} + m.mu.Lock() + m.entries[runID] = entry + err = m.saveRegistryLocked() + if err != nil { + delete(m.entries, runID) + } + m.mu.Unlock() + if err != nil { + return nil, fmt.Errorf("furrow attach %q: save registry: %w", runID, err) + } + // Attach already owns the run lock, so use the locked publish path directly: + // calling Publish here would try to acquire the same non-reentrant mutex. + _ = m.publishLocked(runID, "attached") + return m.handle(entry), nil +} + +func sanitizeNamespace(runID string) string { + var b strings.Builder + for _, r := range runID { + if unicode.IsLetter(r) && r <= unicode.MaxASCII || unicode.IsDigit(r) && r <= unicode.MaxASCII || strings.ContainsRune("._-", r) { + b.WriteRune(r) + } else { + b.WriteByte('-') + } + if b.Len() >= 96 { + break + } + } + if b.Len() == 0 { + return "run" + } + out := b.String()[:min(b.Len(), 96)] + // Dots survive sanitization, and "." / ".." are the two surviving names + // the filesystem treats as traversal rather than a directory of its own. + if out == "." || out == ".." { + return "run" + } + return out +} + +func (m *Manager) handle(entry Entry) *Handle { + // The run's own store, not the root that holds every run's: a caller pairs + // directly with this path, and the root is not a furrow remote at all. + // Over the network the path stays on the node — furrowd resolves it from + // the token — so the address is all the caller needs. + remote := "dir:" + entry.StoreDir + if m.publicAddr != "" { + if m.transportHealthy != nil && m.transportHealthy() { + remote = "ssh://" + m.publicAddr + } else { + m.logf("WARN furrow: public address configured but furrowd is not running or its local listen address is unreachable; using local handle") + } + } + return &Handle{Version: HandleVersion, Remote: remote, Namespace: entry.Namespace, + Key: entry.Key, Token: entry.Token, RepoPath: entry.RepoPath} +} + +func (m *Manager) Publish(runID, label string) error { + if m == nil || !m.Enabled() { + return nil + } + unlock := m.lockRun(runID) + defer unlock() + return m.publishLocked(runID, label) +} + +// publishLocked snapshots and pushes a run while its per-run lock is held. +func (m *Manager) publishLocked(runID, label string) error { + m.mu.RLock() + entry, ok := m.entries[runID] + m.mu.RUnlock() + if !ok { + return fmt.Errorf("furrow publish: unknown run ID %q", runID) + } + if _, err := m.command(entry.RepoPath, "snap", "-m", label); err != nil { + m.logf("furrow publish %q: snapshot: %v", runID, err) + return nil + } + if _, err := m.command(entry.RepoPath, "sync", "--push"); err != nil { + m.logf("furrow publish %q: sync: %v", runID, err) + return nil + } + + m.mu.Lock() + // A sweep may have retired this run while the push was in flight; recording + // a fresh timestamp then would resurrect a row whose store is already gone. + if current, ok := m.entries[runID]; ok { + current.UpdatedAt = m.now() + m.entries[runID] = current + if err := m.saveRegistryLocked(); err != nil { + m.logf("furrow publish %q: save registry: %v", runID, err) + } + } + m.mu.Unlock() + return nil +} + +func (m *Manager) Handle(runID string) *Handle { + if m == nil || !m.Enabled() { + return nil + } + m.mu.RLock() + defer m.mu.RUnlock() + entry, ok := m.entries[runID] + if !ok { + return nil + } + return m.handle(entry) +} + +func (m *Manager) Detach(runID string) error { + if m == nil || !m.Enabled() { + return nil + } + m.mu.RLock() + defer m.mu.RUnlock() + if _, ok := m.entries[runID]; !ok { + return fmt.Errorf("furrow detach: unknown run ID %q", runID) + } + return nil +} + +func (m *Manager) Sweep(maxAge time.Duration, maxBytes int64) (int, error) { + if m == nil || !m.Enabled() { + return 0, nil + } + removed := 0 + now := m.now() + + m.mu.RLock() + stale := make([]string, 0, len(m.entries)) + for runID, entry := range m.entries { + if maxAge > 0 && now.Sub(entry.UpdatedAt) > maxAge { + stale = append(stale, runID) + } + } + m.mu.RUnlock() + for _, runID := range stale { + dropped, err := m.retire(runID, m.olderThan(maxAge)) + if err != nil { + return removed, err + } + if dropped { + removed++ + } + } + + // A budget of zero or less is NO budget. Everywhere else in the manager + // already reads it that way — alignStoreBudget returns early and Attach + // skips its check — but this pass used `>= 0`, so the one configuration + // that says "do not cap me", SWE_FURROW_MAX_GB=0, made every sweep tick + // retire every mirror on the node, live ones included, once an hour. + if maxBytes > 0 { + for { + // Walking the store is I/O, so it happens with no lock held. + total, err := m.aggregateSize() + if err != nil && !errors.Is(err, os.ErrNotExist) { + return removed, fmt.Errorf("furrow sweep size: %w", err) + } + m.mu.RLock() + var oldestID string + var oldest Entry + for id, entry := range m.entries { + if oldestID == "" || entry.UpdatedAt.Before(oldest.UpdatedAt) { + oldestID, oldest = id, entry + } + } + empty := len(m.entries) == 0 + m.mu.RUnlock() + if total <= maxBytes { + break + } + if empty { + m.logf("WARN furrow sweep: aggregate store size %d exceeds disk budget %d with no remote entries; new mirrors are disabled until space is freed", total, maxBytes) + break + } + // Only mirrors that have gone quiet for a full grace window are + // eligible. A build publishes on attach, at every completed DAG + // level and at completion, so anything more recent belongs to a run + // that is still going. + dropped, err := m.retire(oldestID, m.abandonedSince(oldest.UpdatedAt)) + if err != nil { + return removed, err + } + if !dropped { + // Either something republished it while we were measuring, or + // it is too recently active to treat as abandoned. Every other + // entry is newer than this one, so there is nothing reclaimable + // left this pass; measuring again would pick the same victim + // forever. + m.logf("WARN furrow sweep: aggregate store size %d exceeds disk budget %d but the oldest mirror (%s) is still active; new mirrors are disabled until it goes quiet or space is freed", total, maxBytes, oldestID) + break + } + removed++ + } + } + return removed, nil +} + +// olderThan is the age-expiry eligibility rule: a run may be retired once its +// last publish is further back than maxAge. A non-positive maxAge disables age +// expiry, so nothing is eligible under it. +func (m *Manager) olderThan(maxAge time.Duration) func(Entry) bool { + return func(entry Entry) bool { + return maxAge > 0 && m.now().Sub(entry.UpdatedAt) > maxAge + } +} + +// abandonedSince is the budget-eviction eligibility rule. Reclaiming disk is +// worth less than a running build's mirror, so a candidate must satisfy BOTH: +// +// - its last publish is still the one the sweeper measured (observed) — +// anything newer means the run republished while we were choosing; and +// - that publish is at least budgetGrace old. A build publishes on attach, at +// every completed DAG level and at completion, so a mirror that moved +// inside the grace window belongs to a run that is still going. +// +// When nothing is eligible the store stays over budget and Attach refuses NEW +// mirrors, which is a degradation an operator can undo by raising +// SWE_FURROW_MAX_GB. Deleting a live run's mirror is not undoable. +func (m *Manager) abandonedSince(observed time.Time) func(Entry) bool { + return func(entry Entry) bool { + return entry.UpdatedAt.Equal(observed) && m.now().Sub(entry.UpdatedAt) >= m.budgetGrace + } +} + +// retire deletes one run's remote store and its registry row. It takes that +// run's lock so a publish in flight finishes first rather than pushing into a +// directory being deleted, and re-checks eligible under that lock so a run that +// became active in the meantime is left alone. Passing an eligible that ignores +// the entry is how the promise in that last clause gets quietly dropped, so +// both call sites pass a real rule. +func (m *Manager) retire(runID string, eligible func(Entry) bool) (bool, error) { + unlock := m.lockRun(runID) + defer unlock() + + m.mu.RLock() + entry, ok := m.entries[runID] + m.mu.RUnlock() + if !ok { + return false, nil + } + if !eligible(entry) { + return false, nil + } + // The path about to be handed to RemoveAll comes off disk, from a JSON file + // this process rewrites but does not own exclusively. Attach only ever + // builds StoreDir as remotesRoot/, so anything else — + // an absolute path elsewhere, an empty string (which would delete the + // process's working directory), the remotes root itself — is a corrupted or + // edited row, not something we created. Drop the row so it stops being + // counted, and delete nothing. + switch { + case !within(m.remotesRoot, entry.StoreDir) || filepath.Clean(entry.StoreDir) == filepath.Clean(m.remotesRoot): + m.logf("WARN furrow sweep %q: registry store dir %q is not inside %q; dropping the row without deleting anything", + runID, entry.StoreDir, m.remotesRoot) + default: + // Remove the files first: a failure here leaves the row in place so the + // next sweep retries, rather than orphaning a store nothing points at + // any more. + if err := os.RemoveAll(entry.StoreDir); err != nil { + return false, fmt.Errorf("furrow sweep %q: %w", runID, err) + } + } + m.mu.Lock() + delete(m.entries, runID) + // The run's lock is deliberately left behind. Dropping it here would let a + // goroutine already waiting on this mutex and one arriving afterwards end + // up holding two different mutexes for the same run, which is the one thing + // the per-run lock exists to prevent. A retired run leaves a bare mutex. + err := m.saveRegistryLocked() + m.mu.Unlock() + if err != nil { + return true, fmt.Errorf("furrow sweep: save registry: %w", err) + } + return true, nil +} + +func dirSize(root string) (int64, error) { + var size int64 + err := filepath.Walk(root, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.Mode().IsRegular() { + size += info.Size() + } + return nil + }) + return size, err +} + +func (m *Manager) aggregateSize() (int64, error) { + storeSize, err := dirSize(m.storeRoot) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return 0, err + } + remotesSize, err := dirSize(m.remotesRoot) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return 0, err + } + if within(m.storeRoot, m.remotesRoot) { + return storeSize, nil + } + if within(m.remotesRoot, m.storeRoot) { + return remotesSize, nil + } + return storeSize + remotesSize, nil +} + +func within(root, path string) bool { + rel, err := filepath.Rel(root, path) + return err == nil && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +// configuredMaxBytes reads the node's disk allowance. An explicit 0 means +// UNLIMITED and is returned as 0 — every consumer of maxBytes treats a +// non-positive budget as "no cap". Unset or unparseable falls back to the +// documented default; a negative value is nonsense and does the same. +func configuredMaxBytes() int64 { + const defaultMaxGB = 20 + maxGB, err := strconv.ParseInt(os.Getenv("SWE_FURROW_MAX_GB"), 10, 64) + if err != nil || maxGB < 0 { + maxGB = defaultMaxGB + } + return maxGB * 1024 * 1024 * 1024 +} diff --git a/go/internal/furrow/manager_test.go b/go/internal/furrow/manager_test.go new file mode 100644 index 00000000..40ff8166 --- /dev/null +++ b/go/internal/furrow/manager_test.go @@ -0,0 +1,733 @@ +package furrow + +import ( + "bytes" + "errors" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "sync" + "testing" + "time" +) + +const testKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +type fakeExec struct { + mu sync.Mutex + commands [][]string + errFor map[string]error + // onCommand runs outside the recording lock so a test can park inside a + // call and observe whether another one proceeds alongside it. + onCommand func(args []string) +} + +func (f *fakeExec) run(cmd *exec.Cmd) ([]byte, error) { + f.mu.Lock() + f.commands = append(f.commands, append([]string(nil), cmd.Args...)) + f.mu.Unlock() + if f.onCommand != nil { + f.onCommand(cmd.Args) + } + f.mu.Lock() + defer f.mu.Unlock() + if !containsEnv(cmd.Env, "FURROW_DATA_DIR=") { + return nil, errors.New("FURROW_DATA_DIR missing") + } + operation := strings.Join(cmd.Args[4:], " ") + if err := f.errFor[operation]; err != nil { + return nil, err + } + if len(cmd.Args) > 4 && cmd.Args[4] == "remote" { + return []byte(`{"remote":"local","namespace":"ns","key_hex":"` + testKey + `","machine_id":"m"}`), nil + } + return []byte(`{}`), nil +} + +func (f *fakeExec) snapshot() [][]string { + f.mu.Lock() + defer f.mu.Unlock() + result := make([][]string, len(f.commands)) + for i := range f.commands { + result[i] = append([]string(nil), f.commands[i]...) + } + return result +} + +func containsEnv(env []string, prefix string) bool { + for _, value := range env { + if strings.HasPrefix(value, prefix) { + return true + } + } + return false +} + +func testManager(t *testing.T, fake *fakeExec, now func() time.Time) (*Manager, string, string) { + t.Helper() + t.Setenv(EnvEnabled, "1") + root := t.TempDir() + bin := filepath.Join(root, "furrow") + if err := os.WriteFile(bin, []byte("binary"), 0o755); err != nil { + t.Fatal(err) + } + repo := filepath.Join(root, "repo") + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + remotes := filepath.Join(root, "remotes") + m := New(Options{Bin: bin, StoreRoot: filepath.Join(root, "store"), RemotesRoot: remotes, Now: now, Exec: fake.run, + Logger: log.New(&bytes.Buffer{}, "", 0)}) + return m, repo, remotes +} + +func TestNilManagerNoOps(t *testing.T) { + var m *Manager + if m.Enabled() || m.Handle("run") != nil { + t.Fatal("nil manager reported enabled or returned a handle") + } + if handle, err := m.Attach("run", "build", t.TempDir()); handle != nil || err != nil { + t.Fatalf("Attach() = (%v, %v), want (nil, nil)", handle, err) + } + if err := m.Publish("run", "label"); err != nil { + t.Fatal(err) + } + if err := m.Detach("run"); err != nil { + t.Fatal(err) + } + if count, err := m.Sweep(time.Hour, 1); count != 0 || err != nil { + t.Fatalf("Sweep() = (%d, %v), want (0, nil)", count, err) + } +} + +func TestManagerDisabled(t *testing.T) { + for _, tc := range []struct { + name string + env string + bin string + }{ + {"environment opt-out", "0", "unused"}, + {"unset is opt-out", "", "unused"}, + {"missing binary", "1", filepath.Join(t.TempDir(), "missing")}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(EnvEnabled, tc.env) + t.Setenv(EnvPublicAddr, "") + m := New(Options{Bin: tc.bin, Logger: log.New(&bytes.Buffer{}, "", 0)}) + if m.Enabled() { + t.Fatal("manager is enabled") + } + }) + } +} + +// Mirroring is opt-in and the flag is written by hand into a manifest, a +// compose file or a shell, so it has to survive the spellings people actually +// use — and, more importantly, an unrecognised value must never be what turns +// a workspace-copying feature ON. +func TestManagerEnableFlagSpellings(t *testing.T) { + root := t.TempDir() + bin := filepath.Join(root, "furrow") + if err := os.WriteFile(bin, []byte("binary"), 0o755); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + value string + want bool + }{ + {"1", true}, {"true", true}, {"TRUE", true}, {"yes", true}, {"On", true}, {" 1 ", true}, + {"0", false}, {"false", false}, {"NO", false}, {"off", false}, {"", false}, + {"maybe", false}, {"2", false}, {"disabled", false}, + } { + t.Run(fmt.Sprintf("%s=%q", EnvEnabled, tc.value), func(t *testing.T) { + t.Setenv(EnvEnabled, tc.value) + t.Setenv(EnvPublicAddr, "") + m := New(Options{Bin: bin, StoreRoot: filepath.Join(t.TempDir(), "store"), + RemotesRoot: filepath.Join(t.TempDir(), "remotes"), + Exec: func(*exec.Cmd) ([]byte, error) { return []byte(`{}`), nil }, + Logger: log.New(&bytes.Buffer{}, "", 0)}) + if got := m.Enabled(); got != tc.want { + t.Fatalf("Enabled() with %s=%q = %v, want %v", EnvEnabled, tc.value, got, tc.want) + } + }) + } +} + +// With SWE_FURROW_ENABLED unconfigured, mirroring follows FURROW_PUBLIC_ADDR: +// the desktop cloud deploy sets it exactly when it provisioned a public sync +// port, so a cloud control plane mirrors out of the box and a local install +// that set neither variable stays off. An explicit SWE_FURROW_ENABLED beats the +// address in both directions, and an unrecognised spelling still means OFF even +// with the address present — a typo must never be what copies a workspace. +func TestManagerAutoEnableFollowsPublicAddr(t *testing.T) { + root := t.TempDir() + bin := filepath.Join(root, "furrow") + if err := os.WriteFile(bin, []byte("binary"), 0o755); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + enabled string + addr string + want bool + }{ + {"unset follows present addr", "", "mirror.example:31427", true}, + {"blank follows present addr", " ", "mirror.example:31427", true}, + {"unset with no addr stays off", "", "", false}, + {"unset with blank addr stays off", "", " ", false}, + {"explicit off beats addr", "0", "mirror.example:31427", false}, + {"explicit on needs no addr", "1", "", true}, + {"typo means off even with addr", "maybe", "mirror.example:31427", false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(EnvEnabled, tc.enabled) + t.Setenv(EnvPublicAddr, tc.addr) + m := New(Options{Bin: bin, StoreRoot: filepath.Join(t.TempDir(), "store"), + RemotesRoot: filepath.Join(t.TempDir(), "remotes"), + Exec: func(*exec.Cmd) ([]byte, error) { return []byte(`{}`), nil }, + Logger: log.New(&bytes.Buffer{}, "", 0)}) + if got := m.Enabled(); got != tc.want { + t.Fatalf("Enabled() with %s=%q %s=%q = %v, want %v", + EnvEnabled, tc.enabled, EnvPublicAddr, tc.addr, got, tc.want) + } + }) + } +} + +// An empty run ID is a key two builds SHARE, not a label one build is missing. +// Sanitization used to turn it into the namespace "run", so a second build +// attaching without an ID got the first build's registry row back — its +// workspace path, its recovery key and its transport token. Refusing is the +// only outcome that cannot leak one build's mirror to another. +func TestAttachRefusesEmptyRunID(t *testing.T) { + fake := &fakeExec{} + m, repoA, _ := testManager(t, fake, time.Now) + repoB := t.TempDir() + if err := os.MkdirAll(filepath.Join(repoB, ".git"), 0o755); err != nil { + t.Fatal(err) + } + before := len(fake.snapshot()) + + first, err := m.Attach("", "build-a", repoA) + if first != nil || err != nil { + t.Fatalf("Attach(\"\") = (%v, %v), want (nil, nil)", first, err) + } + second, err := m.Attach("", "build-b", repoB) + if second != nil || err != nil { + t.Fatalf("second Attach(\"\") = (%v, %v), want (nil, nil)", second, err) + } + if len(m.entries) != 0 { + t.Fatalf("registry entries = %v, want none", m.entries) + } + if got := fake.snapshot()[before:]; len(got) != 0 { + t.Fatalf("furrow was invoked for a run with no ID: %v", got) + } + if handle := m.Handle(""); handle != nil { + t.Fatalf("Handle(\"\") = %v, want nil", handle) + } +} + +func TestAttachMissingGit(t *testing.T) { + fake := &fakeExec{} + m, _, _ := testManager(t, fake, time.Now) + // Construction sets the store budget; what matters here is that ATTACH + // itself runs nothing when the path is not a git repository. + before := len(fake.snapshot()) + handle, err := m.Attach("run", "build", t.TempDir()) + if err != nil || handle != nil || len(fake.snapshot()) != before { + t.Fatalf("Attach() = (%v, %v), commands=%v", handle, err, fake.snapshot()[before:]) + } +} + +func TestCommandTimesOut(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("shell script test") + } + t.Setenv(EnvEnabled, "1") + root := t.TempDir() + bin := filepath.Join(root, "furrow") + if err := os.WriteFile(bin, []byte("#!/bin/sh\nsleep 60\n"), 0o755); err != nil { + t.Fatal(err) + } + m := New(Options{Bin: bin, StoreRoot: filepath.Join(root, "store"), CmdTimeout: 200 * time.Millisecond, + Logger: log.New(&bytes.Buffer{}, "", 0)}) + started := time.Now() + _, err := m.command(root, "snap") + if err == nil || !strings.Contains(err.Error(), "timeout") { + t.Fatalf("command error = %v, want timeout", err) + } + if elapsed := time.Since(started); elapsed > 3*time.Second { + t.Fatalf("timed-out command returned after %s", elapsed) + } +} + +func TestAttachRefusesWhenAggregateStoreExceedsBudget(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("uses /bin/false") + } + t.Setenv(EnvEnabled, "1") + root := t.TempDir() + store := filepath.Join(root, "store") + repo := filepath.Join(root, "repo") + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(store, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(store, "client-data"), make([]byte, 100), 0o600); err != nil { + t.Fatal(err) + } + var logs bytes.Buffer + m := New(Options{Bin: "/bin/false", StoreRoot: store, RemotesRoot: filepath.Join(root, "remotes"), + MaxBytes: 10, Logger: log.New(&logs, "", 0)}) + handle, err := m.Attach("run", "build", repo) + if err == nil || !strings.Contains(err.Error(), "disk budget") || handle != nil { + t.Fatalf("Attach() = (%v, %v), want disk budget error", handle, err) + } + if len(m.entries) != 0 { + t.Fatalf("registered entries = %v, want none", m.entries) + } + if got := strings.Count(logs.String(), "WARN "); got != 1 { + t.Fatalf("warning count = %d, logs = %q", got, logs.String()) + } +} + +// The budget only protects the volume when the volume is bigger than the +// budget: a cloud deploy's mirrors share one disk with the control plane's +// database, so Attach also enforces an absolute free-space floor. A probe that +// cannot answer (ok=false) must NOT refuse — "no answer" is not "no space". +func TestAttachRefusesWhenDiskNearlyFull(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("uses /bin/false") + } + t.Setenv(EnvEnabled, "1") + for _, tc := range []struct { + name string + probe func(string) (int64, bool) + refused bool + }{ + {"below floor refuses", func(string) (int64, bool) { return minFreeBytes - 1, true }, true}, + {"at floor proceeds", func(string) (int64, bool) { return minFreeBytes, true }, false}, + {"probe unavailable proceeds", func(string) (int64, bool) { return 0, false }, false}, + } { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + repo := filepath.Join(root, "repo") + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + var logs bytes.Buffer + m := New(Options{Bin: "/bin/false", StoreRoot: filepath.Join(root, "store"), + RemotesRoot: filepath.Join(root, "remotes"), Logger: log.New(&logs, "", 0)}) + m.freeBytes = tc.probe + handle, err := m.Attach("run", "build", repo) + if tc.refused { + if err == nil || !strings.Contains(err.Error(), "floor") || handle != nil { + t.Fatalf("Attach() = (%v, %v), want free-space floor error", handle, err) + } + if len(m.entries) != 0 { + t.Fatalf("registered entries = %v, want none", m.entries) + } + return + } + // Past the floor the attach proceeds into the furrow invocations, + // where /bin/false fails the watch — proof the gate did not trip. + if err != nil || handle != nil { + t.Fatalf("Attach() = (%v, %v), want (nil, nil) degradation past the gate", handle, err) + } + if strings.Contains(logs.String(), "-byte floor") { + t.Fatalf("free-space floor tripped unexpectedly: %q", logs.String()) + } + }) + } +} + +func TestAttachPublishExactArgvAndIdempotence(t *testing.T) { + fake := &fakeExec{} + m, repo, remotes := testManager(t, fake, time.Now) + handle, err := m.Attach("run/one", "build-1", repo) + if err != nil || handle == nil { + t.Fatalf("Attach() = (%v, %v)", handle, err) + } + if handle.Key != testKey { + t.Fatalf("Handle.Key = %q, want key_hex value", handle.Key) + } + // The run's own store, not the root. Pairing with the root would find no + // workspace there, so a handle pointing at it is unusable. + if handle.Remote != "dir:"+filepath.Join(remotes, "run-one") || len(handle.Token) != 64 { + t.Fatalf("unexpected handle: %+v", handle) + } + second, err := m.Attach("run/one", "different", repo) + if err != nil || !reflect.DeepEqual(handle, second) { + t.Fatalf("idempotent Attach() = (%v, %v), want %v", second, err, handle) + } + if err := m.Publish("run/one", "checkpoint"); err != nil { + t.Fatal(err) + } + bin := m.bin + store := filepath.Join(filepath.Dir(remotes), "store") + want := [][]string{ + // Capping furrow's own store at half the allowance is what keeps the + // sweeper — which can only reclaim remotes — from ever being left with + // nothing to free while the budget stays exceeded. + {bin, "--repo", store, "--json", "budget", "--max", "10737418240"}, + {bin, "--repo", repo, "--json", "watch", "--no-daemon"}, + {bin, "--repo", repo, "--json", "remote", "add", filepath.Join(remotes, "run-one"), "--name", "run-one"}, + {bin, "--repo", repo, "--json", "snap", "-m", "attached"}, + {bin, "--repo", repo, "--json", "sync", "--push"}, + {bin, "--repo", repo, "--json", "snap", "-m", "checkpoint"}, + {bin, "--repo", repo, "--json", "sync", "--push"}, + } + if got := fake.snapshot(); !reflect.DeepEqual(got, want) { + t.Fatalf("argv mismatch\n got: %#v\nwant: %#v", got, want) + } + // furrow rejects a policy file whose lines are not `exclude `, and a + // rejected file fails `watch`, which switches the mirror off for every build + // with nothing but a debug line to show for it. Pin the exact bytes. + policy, err := os.ReadFile(filepath.Join(repo, ".furrowpolicy")) + if err != nil || string(policy) != "exclude .obs\nexclude node_modules\n" { + t.Fatalf("policy = %q, %v", policy, err) + } + if err := m.Publish("unknown", "label"); err == nil { + t.Fatal("Publish accepted unknown run ID") + } +} + +func TestPublicHandleRequiresHealthyTransport(t *testing.T) { + fake := &fakeExec{} + m, repo, remotes := testManager(t, fake, time.Now) + m.publicAddr = "mirror.example:8802" + + m.SetTransportHealth(func() bool { return false }) + handle, err := m.Attach("run", "build", repo) + if err != nil || handle == nil { + t.Fatalf("Attach() = (%v, %v)", handle, err) + } + if want := "dir:" + filepath.Join(remotes, "run"); handle.Remote != want { + t.Fatalf("unhealthy Remote = %q, want %q", handle.Remote, want) + } + + m.SetTransportHealth(func() bool { return true }) + if got := m.Handle("run").Remote; got != "ssh://mirror.example:8802" { + t.Fatalf("healthy Remote = %q", got) + } +} + +// SWE_FURROW_MAX_GB=0 is the operator saying "do not cap my disk". Every other +// consumer of the budget already read it that way; the sweeper read `>= 0` and +// so treated the no-cap setting as a zero-byte cap, retiring every mirror on +// the node — live builds included — on every hourly tick. +func TestSweepTreatsNonPositiveBudgetAsUnlimited(t *testing.T) { + for _, maxBytes := range []int64{0, -1} { + t.Run(fmt.Sprintf("maxBytes=%d", maxBytes), func(t *testing.T) { + fake := &fakeExec{} + m, repo, _ := testManager(t, fake, time.Now) + if _, err := m.Attach("run/live", "build", repo); err != nil { + t.Fatalf("Attach: %v", err) + } + entry := m.entries["run/live"] + + removed, err := m.Sweep(0, maxBytes) + if err != nil { + t.Fatalf("Sweep: %v", err) + } + if removed != 0 { + t.Fatalf("Sweep removed %d entries under an unlimited budget, want 0", removed) + } + if _, ok := m.entries["run/live"]; !ok { + t.Fatal("unlimited budget retired the run's registry row") + } + if _, err := os.Stat(entry.StoreDir); err != nil { + t.Fatalf("unlimited budget deleted the run's remote store: %v", err) + } + }) + } +} + +// retire promises that "a run that became active in the meantime is left +// alone". Budget eviction called it with maxAge 0, which turned that re-check +// off, so being over budget deleted the workspace of whichever run happened to +// have published least recently — including one that was mid-build. Reclaiming +// disk is worth less than a live mirror: refusing NEW mirrors is recoverable, +// deleting a running build's is not. +func TestSweepBudgetSparesRunsThatAreStillPublishing(t *testing.T) { + fake := &fakeExec{} + clock := time.Now() + m, abandonedRepo, _ := testManager(t, fake, func() time.Time { return clock }) + liveRepo := t.TempDir() + if err := os.MkdirAll(filepath.Join(liveRepo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + if _, err := m.Attach("run/abandoned", "build", abandonedRepo); err != nil { + t.Fatalf("Attach: %v", err) + } + // Well past the grace window for the first run; the second attaches (and so + // publishes) right now, which is exactly what a mid-build run looks like. + clock = clock.Add(3 * time.Hour) + if _, err := m.Attach("run/live", "build", liveRepo); err != nil { + t.Fatalf("Attach: %v", err) + } + liveStore := m.entries["run/live"].StoreDir + + // A one-byte budget: the store is over it no matter what, so eviction runs + // until it either frees enough or runs out of candidates it may touch. + removed, err := m.Sweep(0, 1) + if err != nil { + t.Fatalf("Sweep: %v", err) + } + if removed != 1 { + t.Fatalf("Sweep removed %d entries, want 1 (the abandoned run only)", removed) + } + if _, ok := m.entries["run/abandoned"]; ok { + t.Error("abandoned run survived budget eviction") + } + if _, ok := m.entries["run/live"]; !ok { + t.Error("budget eviction retired a run that published moments ago") + } + if _, err := os.Stat(liveStore); err != nil { + t.Errorf("live run's remote store was deleted: %v", err) + } +} + +// The other half of the same promise: a run that republishes between being +// chosen as the victim and the retirement taking its lock must survive. +func TestRetireSkipsAnEntryThatMovedSinceItWasChosen(t *testing.T) { + fake := &fakeExec{} + clock := time.Now() + m, repo, _ := testManager(t, fake, func() time.Time { return clock }) + if _, err := m.Attach("run/one", "build", repo); err != nil { + t.Fatalf("Attach: %v", err) + } + observed := m.entries["run/one"].UpdatedAt + clock = clock.Add(3 * time.Hour) + + // The run publishes after the sweeper measured it: same run, newer stamp. + if err := m.Publish("run/one", "checkpoint"); err != nil { + t.Fatalf("Publish: %v", err) + } + dropped, err := m.retire("run/one", m.abandonedSince(observed)) + if err != nil { + t.Fatalf("retire: %v", err) + } + if dropped { + t.Fatal("retire deleted a run that republished after it was chosen") + } + if _, ok := m.entries["run/one"]; !ok { + t.Fatal("registry row removed") + } +} + +func TestAttachSanitizesRemoteStorePath(t *testing.T) { + for _, runID := range []string{"../escape", "/absolute", "..", "."} { + t.Run(runID, func(t *testing.T) { + fake := &fakeExec{} + clock := time.Now() + m, repo, remotes := testManager(t, fake, func() time.Time { return clock }) + outside := filepath.Join(filepath.Dir(remotes), "escape") + if err := os.WriteFile(outside, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + handle, err := m.Attach(runID, "build", repo) + if err != nil || handle == nil { + t.Fatalf("Attach() = (%v, %v)", handle, err) + } + entry := m.entries[runID] + if filepath.Dir(entry.StoreDir) != remotes || filepath.Base(entry.StoreDir) != sanitizeNamespace(runID) { + t.Fatalf("StoreDir %q escaped remotes root %q", entry.StoreDir, remotes) + } + // Age the entry past the TTL and sweep by age alone: a zero budget + // means unlimited disk, so it would no longer retire anything and + // the deletion this test is about would never run. + clock = clock.Add(2 * time.Hour) + if _, err := m.Sweep(time.Hour, 0); err != nil { + t.Fatal(err) + } + if _, ok := m.entries[runID]; ok { + t.Fatalf("entry %q survived the age sweep, so nothing was deleted", runID) + } + if got, err := os.ReadFile(outside); err != nil || string(got) != "keep" { + t.Fatalf("outside sentinel = %q, %v", got, err) + } + }) + } +} + +// StoreDir is read back out of a JSON file on disk and handed straight to +// os.RemoveAll. Attach only ever writes remotesRoot/, so +// anything else is a corrupted or hand-edited row — and honouring it would let +// that file choose what the node deletes. +func TestRetireNeverDeletesOutsideTheRemotesRoot(t *testing.T) { + for _, tc := range []struct{ name, storeDir string }{ + {name: "sibling directory", storeDir: "sibling"}, + {name: "the remotes root itself", storeDir: "root"}, + {name: "empty", storeDir: ""}, + } { + t.Run(tc.name, func(t *testing.T) { + fake := &fakeExec{} + clock := time.Now() + m, repo, remotes := testManager(t, fake, func() time.Time { return clock }) + if _, err := m.Attach("run/one", "build", repo); err != nil { + t.Fatalf("Attach: %v", err) + } + sibling := filepath.Join(filepath.Dir(remotes), "not-ours") + if err := os.MkdirAll(sibling, 0o700); err != nil { + t.Fatal(err) + } + sentinel := filepath.Join(sibling, "keep") + if err := os.WriteFile(sentinel, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + rootSentinel := filepath.Join(remotes, "keep") + if err := os.WriteFile(rootSentinel, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + + // Corrupt the row the way an edited registry.json would. + entry := m.entries["run/one"] + switch tc.storeDir { + case "sibling": + entry.StoreDir = sibling + case "root": + entry.StoreDir = remotes + default: + entry.StoreDir = "" + } + m.entries["run/one"] = entry + + clock = clock.Add(2 * time.Hour) + if _, err := m.Sweep(time.Hour, -1); err != nil { + t.Fatalf("Sweep: %v", err) + } + if _, ok := m.entries["run/one"]; ok { + t.Error("bogus row survived the sweep and will be retried forever") + } + for _, path := range []string{sentinel, rootSentinel} { + if got, err := os.ReadFile(path); err != nil || string(got) != "keep" { + t.Errorf("sweep deleted %q: %q, %v", path, got, err) + } + } + }) + } +} + +// A node serves several builds at once and an initial capture of a large +// repository is slow, so work on one run must not block another. This fails if +// the manager ever goes back to holding one lock across furrow invocations: +// each publish parks inside the fake exec until both have arrived, which can +// only happen if they run concurrently. +func TestPublishesForDifferentRunsDoNotSerialize(t *testing.T) { + arrived := make(chan struct{}, 2) + release := make(chan struct{}) + block := false + blocking := &fakeExec{onCommand: func(args []string) { + if block && len(args) > 4 && args[4] == "sync" { + arrived <- struct{}{} + <-release + } + }} + m, repoA, _ := testManager(t, blocking, time.Now) + repoB := t.TempDir() + if err := os.MkdirAll(filepath.Join(repoB, ".git"), 0o755); err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ run, repo string }{{"run/a", repoA}, {"run/b", repoB}} { + if _, err := m.Attach(tc.run, "build", tc.repo); err != nil { + t.Fatalf("Attach(%s): %v", tc.run, err) + } + } + block = true + + done := make(chan struct{}, 2) + for _, run := range []string{"run/a", "run/b"} { + go func(runID string) { + _ = m.Publish(runID, "checkpoint") + done <- struct{}{} + }(run) + } + for i := 0; i < 2; i++ { + select { + case <-arrived: + case <-time.After(5 * time.Second): + t.Fatal("publishes serialized: the second never reached furrow while the first was in flight") + } + } + close(release) + for i := 0; i < 2; i++ { + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("publish did not return") + } + } +} + +func TestPublishTransportFailuresAreNonFatal(t *testing.T) { + for _, operation := range []string{"snap -m attached", "snap -m label", "sync --push"} { + t.Run(operation, func(t *testing.T) { + fake := &fakeExec{errFor: map[string]error{operation: errors.New("transport down")}} + m, repo, _ := testManager(t, fake, time.Now) + handle, err := m.Attach("run", "build", repo) + if err != nil || handle == nil { + t.Fatalf("Attach() = (%v, %v), want a handle despite publish failure", handle, err) + } + if err := m.Publish("run", "label"); err != nil { + t.Fatalf("Publish returned transport error: %v", err) + } + }) + } +} + +func TestNamespaceSanitizationAndTruncation(t *testing.T) { + cases := []struct{ input, want string }{ + {"abc.DEF_123-xy", "abc.DEF_123-xy"}, + {"run/with spaces/☃", "run-with-spaces--"}, + {"", "run"}, + {strings.Repeat("a", 100), strings.Repeat("a", 96)}, + } + for _, tc := range cases { + t.Run(fmt.Sprintf("%q", tc.input), func(t *testing.T) { + if got := sanitizeNamespace(tc.input); got != tc.want { + t.Fatalf("sanitizeNamespace(%q) = %q, want %q", tc.input, got, tc.want) + } + }) + } +} + +func TestConcurrentAttachPublish(t *testing.T) { + fake := &fakeExec{} + m, repo, _ := testManager(t, fake, time.Now) + const workers = 24 + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := m.Attach("run", "build", repo); err != nil { + t.Errorf("Attach: %v", err) + } + if err := m.Publish("run", "live"); err != nil { + t.Errorf("Publish: %v", err) + } + }() + } + wg.Wait() + watchCount := 0 + remoteCount := 0 + for _, argv := range fake.snapshot() { + if len(argv) > 4 && argv[4] == "watch" { + watchCount++ + } + if len(argv) > 4 && argv[4] == "remote" { + remoteCount++ + } + } + if watchCount != 1 || remoteCount != 1 { + t.Fatalf("pairing commands = watch:%d remote:%d, want one each", watchCount, remoteCount) + } +} diff --git a/go/internal/furrow/registry.go b/go/internal/furrow/registry.go new file mode 100644 index 00000000..b3acb62e --- /dev/null +++ b/go/internal/furrow/registry.go @@ -0,0 +1,56 @@ +package furrow + +import ( + "encoding/json" + "os" + "path/filepath" +) + +func (m *Manager) registryPath() string { return filepath.Join(m.remotesRoot, "registry.json") } + +func (m *Manager) loadRegistry() { + data, err := os.ReadFile(m.registryPath()) + if err != nil { + m.logf("furrow registry: starting empty: %v", err) + return + } + var entries map[string]Entry + if err := json.Unmarshal(data, &entries); err != nil || entries == nil { + m.logf("furrow registry: ignoring corrupt file %q", m.registryPath()) + return + } + m.entries = entries +} + +func (m *Manager) saveRegistryLocked() error { + if err := os.MkdirAll(m.remotesRoot, 0o700); err != nil { + return err + } + data, err := json.MarshalIndent(m.entries, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + tmp, err := os.CreateTemp(m.remotesRoot, ".registry-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + if err := tmp.Chmod(0o600); err != nil { + tmp.Close() + return err + } + if _, err := tmp.Write(data); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmpName, m.registryPath()) +} diff --git a/go/internal/furrow/registry_test.go b/go/internal/furrow/registry_test.go new file mode 100644 index 00000000..3d883bfe --- /dev/null +++ b/go/internal/furrow/registry_test.go @@ -0,0 +1,108 @@ +package furrow + +import ( + "bytes" + "log" + "os" + "path/filepath" + "testing" + "time" +) + +func TestRegistryRoundTripAndMode(t *testing.T) { + fake := &fakeExec{} + m, repo, remotes := testManager(t, fake, time.Now) + want, err := m.Attach("run", "build", repo) + if err != nil { + t.Fatal(err) + } + info, err := os.Stat(filepath.Join(remotes, "registry.json")) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0o600 { + t.Fatalf("registry mode = %o, want 600", got) + } + reloaded := New(Options{Bin: m.bin, StoreRoot: m.storeRoot, RemotesRoot: remotes, Exec: fake.run, + Logger: log.New(&bytes.Buffer{}, "", 0)}) + if got := reloaded.Handle("run"); got == nil || got.Key != want.Key || got.Namespace != want.Namespace { + t.Fatalf("reloaded handle = %+v, want %+v", got, want) + } +} + +func TestCorruptRegistryRecovery(t *testing.T) { + t.Setenv(EnvEnabled, "1") + root := t.TempDir() + bin := filepath.Join(root, "furrow") + remotes := filepath.Join(root, "remotes") + if err := os.WriteFile(bin, []byte("binary"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(remotes, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(remotes, "registry.json"), []byte("not json"), 0o600); err != nil { + t.Fatal(err) + } + var logs bytes.Buffer + m := New(Options{Bin: bin, StoreRoot: filepath.Join(root, "store"), RemotesRoot: remotes, + Exec: (&fakeExec{}).run, Logger: log.New(&logs, "", 0)}) + if m.Handle("run") != nil || !bytes.Contains(logs.Bytes(), []byte("corrupt")) { + t.Fatalf("corrupt registry did not recover empty with log: %q", logs.String()) + } +} + +func TestSweepByAgeAndSize(t *testing.T) { + t.Run("age", func(t *testing.T) { + fake := &fakeExec{} + now := time.Date(2026, 8, 5, 12, 0, 0, 0, time.UTC) + current := now.Add(-2 * time.Hour) + m, repo, _ := testManager(t, fake, func() time.Time { return current }) + if _, err := m.Attach("old", "build", repo); err != nil { + t.Fatal(err) + } + current = now + if _, err := m.Attach("new", "build", repo); err != nil { + t.Fatal(err) + } + removed, err := m.Sweep(time.Hour, 1<<30) + if err != nil || removed != 1 || m.Handle("old") != nil || m.Handle("new") == nil { + t.Fatalf("Sweep() = (%d, %v), old=%v new=%v", removed, err, m.Handle("old"), m.Handle("new")) + } + }) + + t.Run("size oldest first", func(t *testing.T) { + fake := &fakeExec{} + current := time.Date(2026, 8, 5, 10, 0, 0, 0, time.UTC) + m, repo, remotes := testManager(t, fake, func() time.Time { return current }) + if _, err := m.Attach("old", "build", repo); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(remotes, "old", "data"), make([]byte, 100), 0o600); err != nil { + t.Fatal(err) + } + current = current.Add(time.Hour) + if _, err := m.Attach("new", "build", repo); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(remotes, "new", "data"), make([]byte, 10), 0o600); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(m.storeRoot, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(m.storeRoot, "client-data"), make([]byte, 100), 0o600); err != nil { + t.Fatal(err) + } + // The client store and registry both count toward the total. Pick a + // ceiling just below the aggregate so exactly the oldest remote goes. + total, err := m.aggregateSize() + if err != nil { + t.Fatal(err) + } + removed, err := m.Sweep(0, total-50) + if err != nil || removed != 1 || m.Handle("old") != nil || m.Handle("new") == nil { + t.Fatalf("Sweep() = (%d, %v), old=%v new=%v", removed, err, m.Handle("old"), m.Handle("new")) + } + }) +} diff --git a/go/internal/furrow/supervisor.go b/go/internal/furrow/supervisor.go new file mode 100644 index 00000000..f5437c8e --- /dev/null +++ b/go/internal/furrow/supervisor.go @@ -0,0 +1,283 @@ +package furrow + +import ( + "bytes" + "context" + "log" + "net" + "os" + "os/exec" + "strings" + "sync" + "time" +) + +const defaultDaemonAddr = ":8802" + +// Supervisor owns the furrowd process lifecycle. A nil Supervisor is safe. +type Supervisor struct { + manager *Manager + bin string + addr string + logger *log.Logger + + backoffInitial time.Duration + backoffMax time.Duration + healthyUptime time.Duration + maxFailures int + now func() time.Time + after func(time.Duration) <-chan time.Time + + mu sync.Mutex + started bool + running bool + done chan struct{} + healthChecked time.Time + healthy bool +} + +// NewSupervisor constructs an inert supervisor unless every feature gate is +// satisfied. Gate failures are deliberately silent feature discovery. +func NewSupervisor(manager *Manager) *Supervisor { + s := &Supervisor{manager: manager} + if !s.Enabled() { + return s + } + daemon, err := ResolveDaemonBin() + if err != nil { + return s + } + s.bin = daemon + s.addr = envOrDefault("FURROWD_ADDR", defaultDaemonAddr) + s.logger = manager.logger + s.backoffInitial = time.Second + s.backoffMax = 60 * time.Second + s.healthyUptime = 60 * time.Second + s.maxFailures = 5 + s.now = time.Now + s.after = time.After + return s +} + +// Enabled reports whether the manager and advertised-address gates are open. +func (s *Supervisor) Enabled() bool { + return s != nil && s.manager != nil && s.manager.Enabled() && strings.TrimSpace(os.Getenv(EnvPublicAddr)) != "" +} + +// Available reports whether all gates, including binary resolution, are open. +func (s *Supervisor) Available() bool { return s != nil && s.Enabled() && s.bin != "" } + +// Addr returns furrowd's listen address, or empty for an inert supervisor. +func (s *Supervisor) Addr() string { + if s == nil { + return "" + } + return s.addr +} + +// Healthy reports whether the supervised process is running and accepting TCP +// connections on its local listen address. Results are briefly cached. +func (s *Supervisor) Healthy() bool { + if s == nil || s.bin == "" { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + if !s.running { + return false + } + if s.now().Sub(s.healthChecked) < 250*time.Millisecond { + return s.healthy + } + addr := s.addr + if strings.HasPrefix(addr, ":") { + addr = "127.0.0.1" + addr + } + conn, err := net.DialTimeout("tcp", addr, 200*time.Millisecond) + if err == nil { + _ = conn.Close() + } + s.healthChecked = s.now() + s.healthy = err == nil + return s.healthy +} + +// Start begins supervision and returns immediately. +func (s *Supervisor) Start(ctx context.Context) { + if s == nil || !s.Available() { + return + } + s.mu.Lock() + if s.started { + s.mu.Unlock() + return + } + s.started = true + s.done = make(chan struct{}) + s.mu.Unlock() + go s.loop(ctx) +} + +// Wait blocks until supervision ends or timeout expires. +func (s *Supervisor) Wait(timeout time.Duration) { + if s == nil { + return + } + s.mu.Lock() + done := s.done + s.mu.Unlock() + if done == nil { + return + } + select { + case <-done: + case <-time.After(timeout): + } +} + +func (s *Supervisor) loop(ctx context.Context) { + defer close(s.done) + backoff := s.backoffInitial + failures := 0 + for { + if ctx.Err() != nil { + return + } + started := s.now() + err := s.runOnce(ctx) + uptime := s.now().Sub(started) + if ctx.Err() != nil { + return + } + if uptime >= s.healthyUptime { + failures = 0 + backoff = s.backoffInitial + } else { + failures++ + if failures >= s.maxFailures { + s.logger.Printf("WARN furrowd: %d consecutive failures; giving up (last: %v)", failures, err) + return + } + } + select { + case <-ctx.Done(): + return + case <-s.after(backoff): + } + backoff *= 2 + if backoff > s.backoffMax { + backoff = s.backoffMax + } + } +} + +func (s *Supervisor) runOnce(ctx context.Context) error { + s.manager.mu.RLock() + remotesRoot, furrowBin := s.manager.remotesRoot, s.manager.bin + s.manager.mu.RUnlock() + cmd := exec.Command(s.bin) + cmd.Env = append(os.Environ(), + "FURROWD_ADDR="+s.addr, + "FURROWD_REMOTES_ROOT="+remotesRoot, + "SWE_FURROW_BIN="+furrowBin, + ) + // Without this the child's output went nowhere: a furrowd that could not + // bind its port, or could not read its TLS key, restarted every backoff + // interval and said nothing, until the supervisor gave up after five + // failures with a single line naming only the exit status. The reason was + // always on the child's stderr. Assigning an io.Writer (rather than a + // *os.File) makes exec run its own copier and wait for it in cmd.Wait, so + // there is no read racing the process teardown. + stdout := &prefixWriter{logger: s.logger, prefix: "furrowd: "} + stderr := &prefixWriter{logger: s.logger, prefix: "furrowd: "} + cmd.Stdout, cmd.Stderr = stdout, stderr + defer func() { + stdout.flush() + stderr.flush() + }() + setDaemonProcessGroup(cmd) + if err := cmd.Start(); err != nil { + return err + } + s.mu.Lock() + s.running = true + s.healthChecked = time.Time{} + s.mu.Unlock() + defer func() { + s.mu.Lock() + s.running = false + s.healthy = false + s.mu.Unlock() + }() + wait := make(chan error, 1) + go func() { wait <- cmd.Wait() }() + select { + case err := <-wait: + return err + case <-ctx.Done(): + killDaemonProcessGroup(cmd.Process.Pid) + <-wait + return ctx.Err() + } +} + +// maxPrefixLine bounds a single buffered line so a child that writes megabytes +// without a newline cannot grow the supervisor's memory without limit. +const maxPrefixLine = 64 << 10 + +// prefixWriter forwards a child process's output into the node's log one line +// at a time, tagged so it is attributable. exec.Cmd writes to it from its own +// copier goroutine — one per stream — and log.Logger is already safe for +// concurrent use; the mutex guards this writer's own partial-line buffer. +type prefixWriter struct { + logger *log.Logger + prefix string + + mu sync.Mutex + buf []byte +} + +func (w *prefixWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + w.buf = append(w.buf, p...) + for { + index := bytes.IndexByte(w.buf, '\n') + if index < 0 { + break + } + w.emitLocked(w.buf[:index]) + w.buf = w.buf[index+1:] + } + if len(w.buf) >= maxPrefixLine { + w.emitLocked(w.buf) + w.buf = w.buf[:0] + } + return len(p), nil +} + +// flush emits whatever the child left without a trailing newline. Safe to call +// once exec.Cmd's copiers have finished, which cmd.Wait guarantees. +func (w *prefixWriter) flush() { + w.mu.Lock() + defer w.mu.Unlock() + if len(w.buf) > 0 { + w.emitLocked(w.buf) + w.buf = w.buf[:0] + } +} + +func (w *prefixWriter) emitLocked(line []byte) { + text := strings.TrimRight(string(line), "\r") + if text == "" || w.logger == nil { + return + } + w.logger.Printf("%s%s", w.prefix, text) +} + +func envOrDefault(key, fallback string) string { + if value := os.Getenv(key); value != "" { + return value + } + return fallback +} diff --git a/go/internal/furrow/supervisor_linux.go b/go/internal/furrow/supervisor_linux.go new file mode 100644 index 00000000..22382ddf --- /dev/null +++ b/go/internal/furrow/supervisor_linux.go @@ -0,0 +1,14 @@ +//go:build linux + +package furrow + +import ( + "os/exec" + "syscall" +) + +func setDaemonProcessGroup(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Pdeathsig: syscall.SIGKILL} +} + +func killDaemonProcessGroup(pid int) { _ = syscall.Kill(-pid, syscall.SIGKILL) } diff --git a/go/internal/furrow/supervisor_other.go b/go/internal/furrow/supervisor_other.go new file mode 100644 index 00000000..ebd8741d --- /dev/null +++ b/go/internal/furrow/supervisor_other.go @@ -0,0 +1,16 @@ +//go:build !unix + +package furrow + +import ( + "os" + "os/exec" +) + +func setDaemonProcessGroup(*exec.Cmd) {} + +func killDaemonProcessGroup(pid int) { + if process, err := os.FindProcess(pid); err == nil { + _ = process.Kill() + } +} diff --git a/go/internal/furrow/supervisor_test.go b/go/internal/furrow/supervisor_test.go new file mode 100644 index 00000000..af1e2650 --- /dev/null +++ b/go/internal/furrow/supervisor_test.go @@ -0,0 +1,267 @@ +package furrow + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "log" + "net" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +func TestSupervisorInertWithoutPublicAddr(t *testing.T) { + // A whitespace-only address is "unset" everywhere: the enable decision + // trims it, so the supervisor must too — otherwise the one spelling turns + // mirroring off while still starting a daemon that mints "ssh:// ". + for _, addr := range []string{"", " "} { + t.Run(fmt.Sprintf("addr=%q", addr), func(t *testing.T) { + m := supervisorManager(t) + t.Setenv("FURROW_PUBLIC_ADDR", addr) + t.Setenv(EnvDaemonBin, daemonScript(t, "exit 0\n")) + s := NewSupervisor(m) + if s.Enabled() || s.Available() || s.Addr() != "" { + t.Fatalf("supervisor should be inert with FURROW_PUBLIC_ADDR=%q", addr) + } + s.Start(context.Background()) + }) + } +} + +func TestSupervisorInertWithoutDaemonBinary(t *testing.T) { + m := supervisorManager(t) + t.Setenv("FURROW_PUBLIC_ADDR", "mirror.example:8802") + t.Setenv(EnvDaemonBin, filepath.Join(t.TempDir(), "missing")) + s := NewSupervisor(m) + if !s.Enabled() || s.Available() || s.Healthy() { + t.Fatalf("gates = enabled %v, available %v; want true, false", s.Enabled(), s.Available()) + } + s.Start(context.Background()) +} + +func TestSupervisorHealthyRequiresRunningProcessAndTCPListener(t *testing.T) { + m := supervisorManager(t) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + t.Setenv("FURROW_PUBLIC_ADDR", "mirror.example:8802") + t.Setenv("FURROWD_ADDR", listener.Addr().String()) + t.Setenv(EnvDaemonBin, daemonScript(t, "while :; do sleep 1; done\n")) + ctx, cancel := context.WithCancel(context.Background()) + s := NewSupervisor(m) + s.Start(ctx) + deadline := time.Now().Add(time.Second) + for !s.Healthy() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if !s.Healthy() { + t.Fatal("Healthy() = false with a running process and TCP listener") + } + cancel() + s.Wait(time.Second) + if s.Healthy() { + t.Fatal("Healthy() = true after supervisor stopped") + } +} + +func TestNilSupervisorNoOps(t *testing.T) { + var s *Supervisor + if s.Enabled() || s.Available() || s.Addr() != "" { + t.Fatal("nil supervisor did not report inert") + } + s.Start(context.Background()) + s.Wait(time.Millisecond) +} + +func TestSupervisorSpawnsWithExpectedEnv(t *testing.T) { + m := supervisorManager(t) + out := filepath.Join(t.TempDir(), "env") + t.Setenv("FURROW_PUBLIC_ADDR", "mirror.example:8802") + t.Setenv("FURROWD_ADDR", "127.0.0.1:9912") + t.Setenv("SUPERVISOR_TEST_OUT", out) + t.Setenv("FURROWD_REMOTES_ROOT", "stale-root") + t.Setenv(EnvBin, m.bin) + t.Setenv(EnvDaemonBin, daemonScript(t, ` +printf '%s\n%s\n%s\n' "$FURROWD_ADDR" "$FURROWD_REMOTES_ROOT" "$SWE_FURROW_BIN" > "$SUPERVISOR_TEST_OUT" +exit 0 +`)) + s := NewSupervisor(m) + s.maxFailures = 1 + s.Start(context.Background()) + s.Wait(time.Second) + got, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + want := "127.0.0.1:9912\n" + m.remotesRoot + "\n" + m.bin + "\n" + if string(got) != want { + t.Fatalf("child env = %q, want %q", got, want) + } + if s.Addr() != "127.0.0.1:9912" { + t.Fatalf("Addr() = %q", s.Addr()) + } +} + +func TestSupervisorRestartsAfterUnexpectedExit(t *testing.T) { + m := supervisorManager(t) + count := filepath.Join(t.TempDir(), "count") + t.Setenv("FURROW_PUBLIC_ADDR", "mirror.example:8802") + t.Setenv("SUPERVISOR_TEST_COUNT", count) + t.Setenv(EnvDaemonBin, daemonScript(t, `echo x >> "$SUPERVISOR_TEST_COUNT"; exit 1 +`)) + s := NewSupervisor(m) + s.maxFailures = 3 + s.after = immediateTimer + s.Start(context.Background()) + s.Wait(time.Second) + if got := lineCount(t, count); got != 3 { + t.Fatalf("spawn count = %d, want 3", got) + } +} + +func TestSupervisorGivesUpAfterFailureThreshold(t *testing.T) { + m := supervisorManager(t) + count := filepath.Join(t.TempDir(), "count") + var logs bytes.Buffer + m.logger.SetOutput(&logs) + t.Setenv("FURROW_PUBLIC_ADDR", "mirror.example:8802") + t.Setenv("SUPERVISOR_TEST_COUNT", count) + t.Setenv(EnvDaemonBin, daemonScript(t, `echo x >> "$SUPERVISOR_TEST_COUNT"; exit 2 +`)) + s := NewSupervisor(m) + s.maxFailures = 2 + s.after = immediateTimer + s.Start(context.Background()) + s.Wait(time.Second) + if got := lineCount(t, count); got != 2 { + t.Fatalf("spawn count = %d, want 2", got) + } + if got := strings.Count(logs.String(), "giving up"); got != 1 { + t.Fatalf("give-up log count = %d, logs %q", got, logs.String()) + } +} + +func TestSupervisorCancellationKillsChildAndReturns(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX process-group assertion") + } + m := supervisorManager(t) + pidFile := filepath.Join(t.TempDir(), "pid") + t.Setenv("FURROW_PUBLIC_ADDR", "mirror.example:8802") + t.Setenv("SUPERVISOR_TEST_PID", pidFile) + t.Setenv(EnvDaemonBin, daemonScript(t, `echo $$ > "$SUPERVISOR_TEST_PID"; while :; do sleep 1; done +`)) + ctx, cancel := context.WithCancel(context.Background()) + s := NewSupervisor(m) + s.Start(ctx) + pid := waitForPID(t, pidFile) + cancel() + s.Wait(time.Second) + if err := syscall.Kill(pid, 0); !errors.Is(err, syscall.ESRCH) { + t.Fatalf("child pid %d survived cancellation: %v", pid, err) + } +} + +func supervisorManager(t *testing.T) *Manager { + t.Helper() + bin := daemonScript(t, "exit 0\n") + t.Setenv(EnvEnabled, "1") + t.Setenv(EnvBin, bin) + root := t.TempDir() + return New(Options{ + Bin: bin, StoreRoot: filepath.Join(root, "store"), RemotesRoot: filepath.Join(root, "remotes"), + Logger: log.New(io.Discard, "", 0), + }) +} + +func daemonScript(t *testing.T, body string) string { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("shell-script fake daemon") + } + path := filepath.Join(t.TempDir(), "furrowd") + if err := os.WriteFile(path, []byte("#!/bin/sh\n"+body), 0o755); err != nil { + t.Fatal(err) + } + return path +} + +func immediateTimer(time.Duration) <-chan time.Time { + ch := make(chan time.Time, 1) + ch <- time.Now() + return ch +} + +func lineCount(t *testing.T, path string) int { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return strings.Count(string(b), "\n") +} + +func waitForPID(t *testing.T, path string) int { + t.Helper() + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if b, err := os.ReadFile(path); err == nil { + pid, err := strconv.Atoi(strings.TrimSpace(string(b))) + if err != nil { + t.Fatal(err) + } + return pid + } + time.Sleep(time.Millisecond) + } + t.Fatal(fmt.Sprintf("timed out waiting for %s", path)) + return 0 +} + +// A furrowd that cannot bind its port or read its TLS key dies immediately and +// is restarted every backoff interval. Its stdout and stderr used to go +// nowhere, so the only trace of a restart loop was one line after five +// failures naming an exit status — the actual reason was on the child's +// stderr and was discarded. Both streams now reach the node's log, tagged. +func TestSupervisorForwardsDaemonOutputToTheNodeLog(t *testing.T) { + m := supervisorManager(t) + var logs bytes.Buffer + logger := log.New(&logs, "", 0) + t.Setenv("FURROW_PUBLIC_ADDR", "mirror.example:8802") + t.Setenv(EnvDaemonBin, daemonScript(t, + "echo 'listening on :8802'\n"+ + "echo 'furrowd: listen on :8802: address already in use' >&2\n"+ + "printf 'no trailing newline' >&2\n"+ + "exit 1\n")) + s := NewSupervisor(m) + s.logger = logger + s.maxFailures = 1 + s.after = immediateTimer + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + s.Start(ctx) + s.Wait(5 * time.Second) + + got := logs.String() + for _, want := range []string{ + "furrowd: listening on :8802", + "furrowd: furrowd: listen on :8802: address already in use", + // A final partial line must be flushed, not swallowed. + "furrowd: no trailing newline", + } { + if !strings.Contains(got, want) { + t.Errorf("supervisor log missing %q; got:\n%s", want, got) + } + } +} diff --git a/go/internal/furrow/supervisor_unix.go b/go/internal/furrow/supervisor_unix.go new file mode 100644 index 00000000..0e4d6185 --- /dev/null +++ b/go/internal/furrow/supervisor_unix.go @@ -0,0 +1,14 @@ +//go:build unix && !linux + +package furrow + +import ( + "os/exec" + "syscall" +) + +func setDaemonProcessGroup(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} +} + +func killDaemonProcessGroup(pid int) { _ = syscall.Kill(-pid, syscall.SIGKILL) } diff --git a/go/internal/furrow/types.go b/go/internal/furrow/types.go new file mode 100644 index 00000000..a4102bf9 --- /dev/null +++ b/go/internal/furrow/types.go @@ -0,0 +1,107 @@ +// Package furrow gives a run's workspace a byte-exact, encrypted mirror that a +// main harness (Claude Code, another coding agent) can clone and follow while +// the run is still going. +// +// Mirroring has to be asked for: an explicit SWE_FURROW_ENABLED decides in +// either direction, and when it is unconfigured the feature follows +// FURROW_PUBLIC_ADDR — set by the desktop app's cloud deploy exactly when a +// public sync endpoint was provisioned for this node (see enabledByEnv). A +// local install that set neither stays off. A mirror is a byte-exact second +// copy of the build workspace, untracked files and all, so turning it on means +// accepting the disk it costs. +// +// The contract with the rest of SWE-AF is deliberately one-way: orchestration +// calls Attach when a workspace exists and Publish when something worth seeing +// has landed, and never has to care whether furrow is installed. Every entry +// point is a no-op that returns a nil handle when the binary is missing, the +// feature is switched off, or the workspace cannot be attached — availability is +// discovered by the caller finding a handle in the result, never by asking. +package furrow + +import "time" + +// HandleVersion is the schema version of the handle embedded in reasoner +// results. Bump it only for a breaking change to the wire shape; consumers are +// expected to ignore a handle whose version they do not recognise. +const HandleVersion = 1 + +// Handle is what a caller needs to materialize this run's workspace on another +// machine. It travels inside the reasoner result under the key "workspace_handle". +// +// Remote carries its own transport: +// +// dir: same box; pair with the directory and bootstrap-pull +// ssh://[:] furrowd (or sshd) reachable over the network +// +// Key is a 64-hex furrow recovery key scoped to this run alone: it decrypts this +// run's namespace and nothing else. Token authenticates the transport hop to +// furrowd and is empty for dir: handles, which are guarded by filesystem +// permissions instead. +type Handle struct { + Version int `json:"v"` + Remote string `json:"remote"` + Namespace string `json:"namespace"` + Key string `json:"key"` + Token string `json:"token,omitempty"` + // RepoPath is the workspace's absolute path on the node. It is advisory — + // useful when the caller shares the filesystem, meaningless otherwise. + RepoPath string `json:"repo_path,omitempty"` + // Deliberately no Ref: every run gets its OWN remote directory (the + // namespace IS the directory), so there is nothing on a remote to + // disambiguate, and publishing under a named ref would break the pull the + // agentfield-use skill documents — `sync --pull --bootstrap`, which reads + // the default HEAD. Verified live: publishing to a named ref makes that + // exact command fail with "no such file or directory". If remotes are ever + // shared between runs, add the ref to the publish, the handle, and the + // skill's recipe together — never one without the others. +} + +// Entry is one run's row in the registry: the mapping from a control-plane run +// ID to the workspace on disk that SWE-AF only ever knew by its own build ID. +type Entry struct { + RunID string `json:"run_id"` + BuildID string `json:"build_id,omitempty"` + RepoPath string `json:"repo_path"` + Namespace string `json:"namespace"` + Key string `json:"key"` + Token string `json:"token,omitempty"` + StoreDir string `json:"store_dir"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// Attacher is the surface orchestration code depends on. +// +// Every method on *Manager tolerates a nil RECEIVER, but that is not the same +// as callers never needing a nil check: they hold this interface, and a nil +// Attacher interface value has no method set to dispatch to — calling through +// it panics. node.buildFurrowManager returns a nil interface when furrow is off +// or unavailable, so orch.Deps.furrowAttach and furrowPublish nil-check the +// field before every call. Anything else holding an Attacher must do the same. +type Attacher interface { + // Enabled reports whether this manager will do anything at all. + Enabled() bool + // Attach begins mirroring repoPath for runID and returns the handle a caller + // needs to reach it. It is idempotent per runID, and an empty runID is + // refused outright — it would be a key two builds share, not a missing + // label. A nil handle with a nil error means furrow is simply unavailable — + // never an error worth failing a build over. + Attach(runID, buildID, repoPath string) (*Handle, error) + // Publish seals current state and pushes it to the run's remote. Safe to + // call often; cheap when nothing changed. + Publish(runID, label string) error + // Handle returns a previously attached run's handle, or nil if unknown. + Handle(runID string) *Handle + // Detach reports whether the run is still known, returning an error when it + // is not. It does NOT stop mirroring: `furrow watch --no-daemon` leaves + // nothing running to stop, and the mirror's remote and registry row are + // reclaimed by Sweep on age or budget instead. The name is kept because + // callers use it as an "is this run still attached" probe. + Detach(runID string) error + // Sweep removes registry entries and remote stores older than maxAge, and + // trims the store root to maxBytes (oldest first). Returns entries removed. + // Each limit is independently opt-out: a maxAge of zero or less disables + // age expiry, and a maxBytes of zero or less means UNLIMITED disk and skips + // budget eviction entirely — it never means "evict everything". + Sweep(maxAge time.Duration, maxBytes int64) (int, error) +} diff --git a/go/internal/node/discovery_surface_test.go b/go/internal/node/discovery_surface_test.go index 6b44e22f..9bd41617 100644 --- a/go/internal/node/discovery_surface_test.go +++ b/go/internal/node/discovery_surface_test.go @@ -57,6 +57,8 @@ var wantEntrypoints = []string{"build", "implement_issue", "plan", "resolve", "r // has no orchestrator context. func TestRoleReasonersAreMarkedInternal(t *testing.T) { t.Setenv("SWE_PRO_ENGINE", "") + t.Setenv("SWE_FURROW_ENABLED", "") + t.Setenv("FURROW_PUBLIC_ADDR", "") n, err := BuildAgent("swe-planner", "8005", "Autonomous SWE planning pipeline") if err != nil { t.Fatalf("BuildAgent: %v", err) @@ -100,6 +102,8 @@ func TestRoleReasonersAreMarkedInternal(t *testing.T) { // leaking in, no real entry point missing. func TestEntrypointTagIsExactSet(t *testing.T) { t.Setenv("SWE_PRO_ENGINE", "") + t.Setenv("SWE_FURROW_ENABLED", "") + t.Setenv("FURROW_PUBLIC_ADDR", "") n, err := BuildAgent("swe-planner", "8005", "Autonomous SWE planning pipeline") if err != nil { t.Fatalf("BuildAgent: %v", err) @@ -131,6 +135,8 @@ func TestEntrypointTagIsExactSet(t *testing.T) { // internal and must not appear as an entry point. func TestProExecuteIsInternal(t *testing.T) { t.Setenv(pro.EnvEnabled, "1") + t.Setenv("SWE_FURROW_ENABLED", "") + t.Setenv("FURROW_PUBLIC_ADDR", "") fakeEngineBin(t) n, err := BuildAgent("swe-planner", "8005", "Autonomous SWE planning pipeline") @@ -160,6 +166,8 @@ func TestProExecuteIsInternal(t *testing.T) { // since execute is (correctly) not tagged as an entry point but is still visible. func TestExecuteDescribesItsPlanResultInput(t *testing.T) { t.Setenv("SWE_PRO_ENGINE", "") + t.Setenv("SWE_FURROW_ENABLED", "") + t.Setenv("FURROW_PUBLIC_ADDR", "") n, err := BuildAgent("swe-planner", "8005", "Autonomous SWE planning pipeline") if err != nil { t.Fatalf("BuildAgent: %v", err) diff --git a/go/internal/node/node.go b/go/internal/node/node.go index b253dc70..3a8805ee 100644 --- a/go/internal/node/node.go +++ b/go/internal/node/node.go @@ -12,15 +12,21 @@ package node import ( "context" "fmt" + "log" "os" + "path/filepath" + "strconv" "strings" + "time" "github.com/Agent-Field/agentfield/sdk/go/agent" "github.com/Agent-Field/agentfield/sdk/go/ai" "github.com/Agent-Field/SWE-AF/go/internal/envelope" + "github.com/Agent-Field/SWE-AF/go/internal/furrow" "github.com/Agent-Field/SWE-AF/go/internal/hitl" "github.com/Agent-Field/SWE-AF/go/internal/orch" + "github.com/Agent-Field/SWE-AF/go/internal/workspace" ) // Node bundles the constructed agent with the resolved environment config and @@ -43,6 +49,10 @@ type Node struct { // way the agent does. Token string + // Furrow is the node-wide workspace mirror and registry. It is nil when the + // helper binary is unavailable, which is a normal feature-discovery result. + Furrow furrow.Attacher + // hax is the hax REST client, nil when HAX_API_KEY is unset (HITL disabled, // mirroring build_hax_client_from_env() returning None). hax *hitl.HaxClient @@ -149,6 +159,7 @@ func BuildAgent(defaultNodeID, defaultPort, description string) (*Node, error) { Token: token, hax: hitl.BuildHaxClientFromEnv(), } + n.Furrow = buildFurrowManager() // Wire the orchestrator pause seam once: the plan-approval gate pauses the // execution through this provider. The *agent.Agent satisfies hitl.Pauser @@ -162,6 +173,70 @@ func BuildAgent(defaultNodeID, defaultPort, description string) (*Node, error) { return n, nil } +func buildFurrowManager() furrow.Attacher { + bin, err := furrow.ResolveBin() + if err != nil { + log.Printf("DEBUG furrow unavailable: %v", err) + return nil + } + storeRoot, remotesRoot := furrowRoots() + if err := os.Setenv("FURROW_DATA_DIR", storeRoot); err != nil { + log.Printf("DEBUG furrow unavailable: set FURROW_DATA_DIR: %v", err) + return nil + } + m := furrow.New(furrow.Options{ + Bin: bin, + StoreRoot: storeRoot, + RemotesRoot: remotesRoot, + PublicAddr: os.Getenv(furrow.EnvPublicAddr), + }) + if !m.Enabled() { + return nil + } + // furrowd is a best-effort sidecar. Its supervisor is silent unless the + // manager, public-address, and binary gates are all open. + supervisor := furrow.NewSupervisor(m) + m.SetTransportHealth(supervisor.Healthy) + supervisor.Start(context.Background()) + maxAge := time.Duration(envInt64("SWE_FURROW_TTL_HOURS", 72)) * time.Hour + maxBytes := envInt64("SWE_FURROW_MAX_GB", 20) * 1024 * 1024 * 1024 + go sweepFurrow(m, maxAge, maxBytes) + return m +} + +func furrowRoots() (string, string) { + storeDefault := filepath.Join(workspace.Root(), ".furrow-store") + remotesDefault := filepath.Join(workspace.Root(), ".furrow-remotes") + if home := os.Getenv("AGENTFIELD_HOME"); home != "" { + storeDefault = filepath.Join(home, "furrow", "store") + remotesDefault = filepath.Join(home, "furrow", "remotes") + } + return envOr("SWE_FURROW_DATA_DIR", storeDefault), envOr("SWE_FURROW_REMOTES_ROOT", remotesDefault) +} + +func sweepFurrow(m *furrow.Manager, maxAge time.Duration, maxBytes int64) { + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + for range ticker.C { + removed, err := m.Sweep(maxAge, maxBytes) + if err != nil { + log.Printf("furrow sweep: %v", err) + continue + } + if removed > 0 { + log.Printf("furrow sweep removed %d workspace(s)", removed) + } + } +} + +func envInt64(key string, def int64) int64 { + value, err := strconv.ParseInt(os.Getenv(key), 10, 64) + if err != nil || value < 0 { + return def + } + return value +} + // newCallFn returns the app.Call + envelope-unwrap closure injected into the // coding loop, the DAG executor and the fast pipeline. It is structurally // identical to coding.CallFn and fast.CallFn (both func(ctx, target, kwargs) diff --git a/go/internal/node/node_test.go b/go/internal/node/node_test.go index 2ff8a066..b0d7c00d 100644 --- a/go/internal/node/node_test.go +++ b/go/internal/node/node_test.go @@ -2,14 +2,44 @@ package node import ( "context" + "encoding/json" + "path/filepath" "sort" + "strings" "testing" + "time" "github.com/Agent-Field/agentfield/sdk/go/agent" "github.com/Agent-Field/SWE-AF/go/internal/fast" + "github.com/Agent-Field/SWE-AF/go/internal/furrow" + "github.com/Agent-Field/SWE-AF/go/internal/workspace" ) +func TestFurrowRootResolutionPrecedence(t *testing.T) { + t.Setenv("AGENTFIELD_HOME", "") + t.Setenv("SWE_FURROW_DATA_DIR", "") + t.Setenv("SWE_FURROW_REMOTES_ROOT", "") + store, remotes := furrowRoots() + if store != filepath.Join(workspace.Root(), ".furrow-store") || remotes != filepath.Join(workspace.Root(), ".furrow-remotes") { + t.Fatalf("legacy roots = (%q, %q)", store, remotes) + } + + home := t.TempDir() + t.Setenv("AGENTFIELD_HOME", home) + store, remotes = furrowRoots() + if store != filepath.Join(home, "furrow", "store") || remotes != filepath.Join(home, "furrow", "remotes") { + t.Fatalf("AGENTFIELD_HOME roots = (%q, %q)", store, remotes) + } + + t.Setenv("SWE_FURROW_DATA_DIR", "/override/store") + t.Setenv("SWE_FURROW_REMOTES_ROOT", "/override/remotes") + store, remotes = furrowRoots() + if store != "/override/store" || remotes != "/override/remotes" { + t.Fatalf("override roots = (%q, %q)", store, remotes) + } +} + // pythonRoleSurface is the independent parity checklist: the exact 25 role // reasoner names the Python swe_af.reasoners.router registers (pipeline.py's 5 // planning roles + execution_agents.py's 20 execution roles). It is written from @@ -50,6 +80,8 @@ var pythonRoleSurface = []string{ // pythonOrchestrators is the 5 orchestrator reasoners defined on swe_af.app // (app.py @app.reasoner()): build, plan, execute, resolve, resume_build. +// get_workspace_handle is deliberately NOT here: it is gated on furrow being +// switched on, and TestWorkspaceHandleReasonerIsGatedOnFurrow owns it. var pythonOrchestrators = []string{"build", "plan", "execute", "resolve", "resume_build"} // pythonFastReasoners is the 4 first-class fast reasoners: fast/app.py's build @@ -61,9 +93,13 @@ var pythonFastReasoners = []string{"build", "fast_plan_tasks", "fast_execute_tas var pythonIssueReasoners = []string{"implement_issue"} func TestRegisterPlannerExactSurface(t *testing.T) { - // Pin the pro engine off so an inherited SWE_PRO_ENGINE cannot - // widen the surface under test (the gated surface has its own test). + // Pin the pro engine and furrow off so an inherited SWE_PRO_ENGINE, + // SWE_FURROW_ENABLED or FURROW_PUBLIC_ADDR (which auto-enables mirroring + // when the enable flag is unconfigured) cannot widen the surface under + // test (each gated surface has its own test). t.Setenv("SWE_PRO_ENGINE", "") + t.Setenv(furrow.EnvEnabled, "") + t.Setenv(furrow.EnvPublicAddr, "") n, err := BuildAgent("swe-planner", "8005", "Autonomous SWE planning pipeline") if err != nil { t.Fatalf("BuildAgent: %v", err) @@ -77,6 +113,99 @@ func TestRegisterPlannerExactSurface(t *testing.T) { assertSurface(t, "swe-planner", n.RegisteredNames(), want) } +// stubAttacher is an enabled furrow that never mirrors anything: enough to +// open the registration gate, nothing more. +type stubAttacher struct{ enabled bool } + +func (s stubAttacher) Enabled() bool { return s.enabled } +func (s stubAttacher) Attach(string, string, string) (*furrow.Handle, error) { return nil, nil } +func (s stubAttacher) Publish(string, string) error { return nil } +func (s stubAttacher) Handle(string) *furrow.Handle { return nil } +func (s stubAttacher) Detach(string) error { return nil } +func (s stubAttacher) Sweep(time.Duration, int64) (int, error) { return 0, nil } + +// get_workspace_handle hands out the connection details for a live workspace +// mirror. Mirroring is opt-in, so on a node that never makes a mirror the +// reasoner must not be advertised at all — an entrypoint-tagged surface that +// can only ever answer {"available": false} is an invitation to route to it. +func TestWorkspaceHandleReasonerIsGatedOnFurrow(t *testing.T) { + const name = "get_workspace_handle" + for _, tc := range []struct { + label string + attacher furrow.Attacher + want bool + }{ + {label: "furrow absent", attacher: nil, want: false}, + {label: "furrow present but disabled", attacher: stubAttacher{}, want: false}, + {label: "furrow mirroring", attacher: stubAttacher{enabled: true}, want: true}, + } { + t.Run(tc.label, func(t *testing.T) { + t.Setenv("SWE_PRO_ENGINE", "") + t.Setenv(furrow.EnvEnabled, "") + t.Setenv(furrow.EnvPublicAddr, "") + n, err := BuildAgent("swe-planner", "8005", "Autonomous SWE planning pipeline") + if err != nil { + t.Fatalf("BuildAgent: %v", err) + } + n.Furrow = tc.attacher + n.RegisterPlanner() + if got := toSet(n.RegisteredNames())[name]; got != tc.want { + t.Fatalf("%s registered = %v, want %v", name, got, tc.want) + } + }) + } +} + +// get_workspace_handle answers anyone who can reach the node and name a run — +// there is no per-caller authorization anywhere on that path. The handle's Key +// decrypts the workspace and its Token authenticates to furrowd read-write, so +// neither may be the default answer to an unauthenticated question. +func TestWorkspaceHandleRedactsSecretsUnlessOperatorOptsIn(t *testing.T) { + handle := &furrow.Handle{ + Version: furrow.HandleVersion, Remote: "ssh://node.internal:8802", Namespace: "run-1", + Key: "0123456789abcdef", Token: "transport-token", RepoPath: "/work/repo", + } + for _, tc := range []struct { + env string + want bool // secrets present + }{ + {env: "", want: false}, + {env: "0", want: false}, + {env: "no", want: false}, + {env: "1", want: true}, + {env: "true", want: true}, + } { + t.Run("SWE_FURROW_EXPOSE_SECRETS="+tc.env, func(t *testing.T) { + t.Setenv(furrow.EnvExposeSecrets, tc.env) + result := workspaceHandleResult(handle) + + _, gotKey := result["key"] + _, gotToken := result["token"] + if gotKey != tc.want || gotToken != tc.want { + t.Fatalf("key present = %v, token present = %v, want both %v", gotKey, gotToken, tc.want) + } + if result["secrets_redacted"] != !tc.want { + t.Errorf("secrets_redacted = %v, want %v", result["secrets_redacted"], !tc.want) + } + // Redacting must not blind the caller: what a mirror IS stays. + for _, key := range []string{"v", "remote", "namespace", "repo_path"} { + if _, ok := result[key]; !ok { + t.Errorf("result dropped %q, which carries no secret", key) + } + } + // Nothing may smuggle the secrets back through another field. + rendered, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + if leaked := strings.Contains(string(rendered), handle.Key) || + strings.Contains(string(rendered), handle.Token); leaked != tc.want { + t.Fatalf("secret material in payload = %v, want %v: %s", leaked, tc.want, rendered) + } + }) + } +} + func TestRegisterFastExactSurface(t *testing.T) { n, err := BuildAgent("swe-fast", "8006", "fast desc") if err != nil { diff --git a/go/internal/node/register.go b/go/internal/node/register.go index 598ab4aa..eefe374f 100644 --- a/go/internal/node/register.go +++ b/go/internal/node/register.go @@ -43,6 +43,7 @@ import ( "github.com/Agent-Field/agentfield/sdk/go/agent" + "github.com/Agent-Field/SWE-AF/go/internal/furrow" "github.com/Agent-Field/SWE-AF/go/internal/hitl" "github.com/Agent-Field/SWE-AF/go/internal/orch" "github.com/Agent-Field/SWE-AF/go/internal/roles/advisor" @@ -68,6 +69,9 @@ const ( func (n *Node) RegisterPlanner() { n.registerRoles() n.registerOrchestrators() + if n.furrowEnabled() { + n.registerWorkspaceHandleReasoner() + } n.registerIssueReasoner() if pro.Available() { n.registerProReasoners() @@ -166,6 +170,7 @@ func (n *Node) registerOrchestrators() { AgentFieldServer: n.AgentFieldServer, CIGate: orch.RunCIGate, ApprovalGate: orch.PlanApprovalGate, + Furrow: n.Furrow, } // Engine default routing (seamless path): with the flag truthy AND the // binary present, builds and execute calls that name no execute_fn_target @@ -203,6 +208,67 @@ func (n *Node) registerOrchestrators() { } } +// furrowEnabled reports whether this node actually mirrors workspaces. It is +// the same shape as the pro.Available() gate next to it: a surface that exists +// only to reach a live mirror has no business being advertised on a node that +// never makes one. Mirroring must be asked for — explicitly via +// SWE_FURROW_ENABLED, or by the platform having provisioned a public mirror +// endpoint (FURROW_PUBLIC_ADDR, set by the desktop app's cloud deploy) — so on +// a local install that configured neither this is false and +// get_workspace_handle is simply not registered. +func (n *Node) furrowEnabled() bool { + return n != nil && n.Furrow != nil && n.Furrow.Enabled() +} + +// workspaceHandleResult renders a handle for the wire. +// +// The trust boundary matters here. This reasoner has NO per-caller +// authorization: anything that can reach the node and guess or observe a run ID +// gets an answer. A furrow handle's Key is the run's recovery key — it decrypts +// that workspace, secrets and untracked files included — and Token authenticates +// to furrowd, which serves the run's remote read-write, so a leaked token buys +// push and delete as well as pull. +// +// So both are withheld by default and the result says so, leaving Remote, +// Namespace and RepoPath: enough for a caller that already shares the +// filesystem, and enough for a human to see a mirror exists. An operator on a +// single-tenant, trusted cluster opts back in with SWE_FURROW_EXPOSE_SECRETS. +func workspaceHandleResult(handle *furrow.Handle) map[string]any { + data, _ := json.Marshal(handle) + result := map[string]any{} + _ = json.Unmarshal(data, &result) + expose := furrow.EnvTruthy(furrow.EnvExposeSecrets) + if !expose { + delete(result, "key") + delete(result, "token") + } + result["secrets_redacted"] = !expose + return result +} + +// registerWorkspaceHandleReasoner exposes connection details for a workspace +// only when furrow discovered and attached one for the requested run. +func (n *Node) registerWorkspaceHandleReasoner() { + name := "get_workspace_handle" + n.registered = append(n.registered, name) + n.App.RegisterReasoner(name, func(_ context.Context, input map[string]any) (any, error) { + runID, _ := input["run_id"].(string) + if runID == "" || n.Furrow == nil { + return map[string]any{"available": false}, nil + } + handle := n.Furrow.Handle(runID) + if handle == nil { + return map[string]any{"available": false}, nil + } + return workspaceHandleResult(handle), nil + }, agent.WithReasonerTags(tagEntrypoint), agent.WithDescription( + "Returns connection details for cloning a run's live workspace. Route here when a caller needs to clone or follow an active build workspace. "+ + "The reasoner performs NO authorization: any caller holding a run ID gets an answer, so the recovery key and transport token are redacted "+ + "(secrets_redacted=true) and the response carries only the remote, namespace and on-node path. Set SWE_FURROW_EXPOSE_SECRETS=1 to return them "+ + "in full — appropriate only on a single-tenant cluster where every caller is already trusted with the workspace contents."), + agent.WithInputSchema(schema(`{"type":"object","additionalProperties":true,"required":["run_id"],"properties":{"run_id":{"type":"string"}}}`))) +} + // orchestratorEntrypoints is the set of orchestrators a caller may start a run // from, and therefore the ones tagged "entrypoint" for discovery. plan, resolve // and resume_build are advanced but legitimate entries (a goal, a PR URL and a diff --git a/go/internal/orch/build.go b/go/internal/orch/build.go index c25ad0b5..28ebee2b 100644 --- a/go/internal/orch/build.go +++ b/go/internal/orch/build.go @@ -93,9 +93,10 @@ func Build(ctx context.Context, deps *Deps, input map[string]any) (any, error) { deps.Note(ctx, fmt.Sprintf("Build starting (build_id=%s)", buildID), "build", "start") - // Scope key for the credentials store; cleared in the deferred finally so - // even an error leaves no secrets in process memory. - scopeID := runIDFromCtx(ctx) + // Scope key for the credentials store AND for this build's workspace mirror; + // cleared in the deferred finally so even an error leaves no secrets in + // process memory. Both consumers below guard against an empty scope. + scopeID := scopeIDFromCtx(ctx) defer func() { if scopeID != "" { hitl.ClearScopedCredentials(scopeID) @@ -148,6 +149,12 @@ func Build(ctx context.Context, deps *Deps, input map[string]any) (any, error) { } manifestMap := dumpToMap(manifest) + workspaceHandle := deps.furrowAttach(scopeID, buildID, repoPath) + if workspaceHandle != nil { + deps.Note(ctx, fmt.Sprintf("Workspace mirror ready (namespace=%s)", workspaceHandle.Namespace), + "build", "furrow") + } + // 1. PLAN + GIT INIT (concurrent — no data dependency). deps.Note(ctx, "Phase 1: Planning + Git init (parallel)", "build", "parallel") @@ -488,6 +495,10 @@ func Build(ctx context.Context, deps *Deps, input map[string]any) (any, error) { CIGateResults: ciGateResults, } buildResultMap := dumpToMap(buildResult) + deps.furrowPublish(scopeID, "build complete") + if workspaceHandle != nil { + buildResultMap["workspace_handle"] = dumpToMap(workspaceHandle) + } // Empty-build guard: nothing shipped AND verification failed → report failed. // Return the SDK's result-carrying &agent.ReasonerFailed so the async handler diff --git a/go/internal/orch/build_test.go b/go/internal/orch/build_test.go index f6ff04e2..8dd93a43 100644 --- a/go/internal/orch/build_test.go +++ b/go/internal/orch/build_test.go @@ -8,12 +8,37 @@ import ( "strings" "sync" "testing" + "time" "github.com/Agent-Field/agentfield/sdk/go/agent" + "github.com/Agent-Field/SWE-AF/go/internal/furrow" "github.com/Agent-Field/SWE-AF/go/internal/workspace" ) +type fakeFurrow struct { + handle *furrow.Handle + attachErr error + attachedPath string + attachedRunID string + publishedRuns []string + publishes []string +} + +func (f *fakeFurrow) Enabled() bool { return true } +func (f *fakeFurrow) Attach(runID, _, repoPath string) (*furrow.Handle, error) { + f.attachedRunID, f.attachedPath = runID, repoPath + return f.handle, f.attachErr +} +func (f *fakeFurrow) Publish(runID string, label string) error { + f.publishedRuns = append(f.publishedRuns, runID) + f.publishes = append(f.publishes, label) + return errors.New("ignored publish failure") +} +func (f *fakeFurrow) Handle(string) *furrow.Handle { return f.handle } +func (f *fakeFurrow) Detach(string) error { return nil } +func (f *fakeFurrow) Sweep(time.Duration, int64) (int, error) { return 0, nil } + // buildHandler routes mock reasoner responses by target suffix. Overridable // per-reasoner via the exec/verify hooks. func buildHandler(execResp, verifyResp func(input map[string]any) map[string]any) func(context.Context, string, map[string]any) (map[string]any, error) { @@ -147,6 +172,111 @@ func TestBuildVerifiedSuccess(t *testing.T) { } } +func TestBuildWorkspaceHandleAvailability(t *testing.T) { + defer withExecCtx("run-furrow", "exec-furrow")() + exec := func(map[string]any) map[string]any { + return map[string]any{ + "completed_issues": []any{map[string]any{"name": "i1"}}, + "merged_branches": []any{"issue/x"}, + "all_issues": []any{map[string]any{"name": "i1"}}, + "failed_issues": []any{}, "skipped_issues": []any{}, "accumulated_debt": []any{}, + } + } + verify := func(map[string]any) map[string]any { + return map[string]any{"passed": true, "criteria_results": []any{}, "summary": "ok"} + } + + tests := []struct { + name string + attacher *fakeFurrow + wantHandle bool + }{ + {name: "available", attacher: &fakeFurrow{handle: &furrow.Handle{ + Version: 1, Remote: "dir:/mirror", Namespace: "run-furrow", Key: "secret-key", + }}, wantHandle: true}, + {name: "unavailable", attacher: &fakeFurrow{}}, + {name: "attach error", attacher: &fakeFurrow{attachErr: errors.New("attach failed")}}, + {name: "nil dependency"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + app := &mockApp{handler: buildHandler(exec, verify)} + deps := &Deps{App: app, NodeID: "swe-planner"} + if tc.attacher != nil { + deps.Furrow = tc.attacher + } + out, err := Build(context.Background(), deps, map[string]any{ + "goal": "thing", "repo_path": t.TempDir(), + "config": map[string]any{"git_init_max_retries": 1}, + }) + if err != nil { + t.Fatalf("Build: %v", err) + } + result := out.(map[string]any) + _, gotHandle := result["workspace_handle"] + if gotHandle != tc.wantHandle { + t.Fatalf("workspace_handle present = %v, want %v", gotHandle, tc.wantHandle) + } + if tc.attacher != nil && len(tc.attacher.publishes) != 1 { + t.Fatalf("publishes = %v, want build completion", tc.attacher.publishes) + } + for _, note := range app.notes { + if strings.Contains(note, "secret-key") { + t.Fatalf("note leaked workspace key: %q", note) + } + } + }) + } +} + +// A build whose execution context carries no run ID still has a root workflow +// ID, and that is what everything per-run in the build must be filed under — +// planning.Scout already stores scoped credentials that way. Handing furrow an +// empty scope instead made every such build share one registry row, so build B +// received build A's workspace path, recovery key and token. +func TestBuildScopesWorkspaceMirrorByRootWorkflowID(t *testing.T) { + exec := func(map[string]any) map[string]any { + return map[string]any{ + "completed_issues": []any{map[string]any{"name": "i1"}}, + "merged_branches": []any{"issue/x"}, + "all_issues": []any{map[string]any{"name": "i1"}}, + "failed_issues": []any{}, "skipped_issues": []any{}, "accumulated_debt": []any{}, + } + } + verify := func(map[string]any) map[string]any { + return map[string]any{"passed": true, "criteria_results": []any{}, "summary": "ok"} + } + for _, tc := range []struct { + name string + runID string + rootWorkflowID string + wantScope string + }{ + {name: "run id wins", runID: "run-1", rootWorkflowID: "wf-1", wantScope: "run-1"}, + {name: "root workflow id fallback", rootWorkflowID: "wf-2", wantScope: "wf-2"}, + } { + t.Run(tc.name, func(t *testing.T) { + defer withExecCtxRoot(tc.runID, "exec", tc.rootWorkflowID)() + f := &fakeFurrow{handle: &furrow.Handle{Version: 1, Remote: "dir:/mirror", Namespace: "ns"}} + deps := &Deps{App: &mockApp{handler: buildHandler(exec, verify)}, NodeID: "swe-planner", Furrow: f} + if _, err := Build(context.Background(), deps, map[string]any{ + "goal": "thing", "repo_path": t.TempDir(), + "config": map[string]any{"git_init_max_retries": 1}, + }); err != nil { + t.Fatalf("Build: %v", err) + } + if f.attachedRunID != tc.wantScope { + t.Errorf("Attach run ID = %q, want %q", f.attachedRunID, tc.wantScope) + } + // Publish must reach the same row Attach created, or the mirror + // stops updating the moment the run ID is absent. + if len(f.publishedRuns) != 1 || f.publishedRuns[0] != tc.wantScope { + t.Errorf("Publish run IDs = %v, want [%q]", f.publishedRuns, tc.wantScope) + } + }) + } +} + // TestBuildRequiresRepoPathOrURL maps to the ValueError branch. func TestBuildRequiresRepoPathOrURL(t *testing.T) { defer withExecCtx("r", "e")() diff --git a/go/internal/orch/common.go b/go/internal/orch/common.go index 422bdd65..bbe83937 100644 --- a/go/internal/orch/common.go +++ b/go/internal/orch/common.go @@ -25,6 +25,7 @@ import ( "github.com/Agent-Field/SWE-AF/go/internal/coding" "github.com/Agent-Field/SWE-AF/go/internal/config" "github.com/Agent-Field/SWE-AF/go/internal/envelope" + "github.com/Agent-Field/SWE-AF/go/internal/furrow" "github.com/Agent-Field/SWE-AF/go/internal/schemas" ) @@ -64,6 +65,11 @@ type Deps struct { CIGate CIGateRunner ApprovalGate ApprovalGate + // Furrow mirrors a build's workspace to a per-run encrypted remote so a + // caller can clone and follow it while the build runs. A nil Attacher (the + // default) disables the whole feature silently. + Furrow furrow.Attacher + // DefaultExecuteFnTarget, when non-empty, is the external coder target // applied by the execute path whenever a request does not name one — the // node-level engine opt-in seam. A caller-supplied execute_fn_target @@ -94,7 +100,25 @@ var sleepFn = func(ctx context.Context, d time.Duration) { } } -func runIDFromCtx(ctx context.Context) string { return executionContextFrom(ctx).RunID } +// scopeIDFromCtx is the key every per-run store in a build files its state +// under: the control-plane run ID when there is one, and the root workflow ID +// when the run ID is absent. planning.Scout already stashes scoped credentials +// behind exactly this fallback, so anything keyed differently would look at a +// row Scout never wrote. +// +// The fallback is not cosmetic. An empty key is a SHARED key: the furrow +// registry would file two unrelated builds under the same row, and the second +// build's Attach would hand back the first build's workspace path, recovery key +// and transport token. Whatever this returns must either identify one build or +// be empty, and callers must treat empty as "no scoped state at all". +func scopeIDFromCtx(ctx context.Context) string { + ec := executionContextFrom(ctx) + if ec.RunID != "" { + return ec.RunID + } + return ec.RootWorkflowID +} + func executionIDFromCtx(ctx context.Context) string { return executionContextFrom(ctx).ExecutionID } // --------------------------------------------------------------------------- @@ -108,6 +132,26 @@ func (d *Deps) Note(ctx context.Context, message string, tags ...string) { } } +// furrowAttach forwards to the optional workspace mirror without allowing an +// unavailable or failed attachment to affect the build. +func (d *Deps) furrowAttach(runID, buildID, repoPath string) *furrow.Handle { + if d == nil || d.Furrow == nil { + return nil + } + handle, err := d.Furrow.Attach(runID, buildID, repoPath) + if err != nil { + return nil + } + return handle +} + +// furrowPublish publishes a best-effort workspace snapshot. +func (d *Deps) furrowPublish(runID, label string) { + if d != nil && d.Furrow != nil { + _ = d.Furrow.Publish(runID, label) + } +} + // Call invokes the local reasoner name (addressed as ".") and // unwraps the envelope, mirroring Python `_unwrap(await app.call(...), label)`. // label defaults to name when empty. diff --git a/go/internal/orch/common_test.go b/go/internal/orch/common_test.go index 8bb6b9e0..86138c38 100644 --- a/go/internal/orch/common_test.go +++ b/go/internal/orch/common_test.go @@ -31,9 +31,15 @@ func (m *mockApp) Note(ctx context.Context, message string, tags ...string) { // withExecCtx overrides the execution-context seam for a test. func withExecCtx(runID, execID string) func() { + return withExecCtxRoot(runID, execID, "") +} + +// withExecCtxRoot is withExecCtx for the case that matters to scoping: a +// context whose RunID is empty but which still names a root workflow. +func withExecCtxRoot(runID, execID, rootWorkflowID string) func() { prev := executionContextFrom executionContextFrom = func(context.Context) agent.ExecutionContext { - return agent.ExecutionContext{RunID: runID, ExecutionID: execID} + return agent.ExecutionContext{RunID: runID, ExecutionID: execID, RootWorkflowID: rootWorkflowID} } return func() { executionContextFrom = prev } } diff --git a/go/internal/orch/execute.go b/go/internal/orch/execute.go index 654b6a72..7df07a59 100644 --- a/go/internal/orch/execute.go +++ b/go/internal/orch/execute.go @@ -93,6 +93,9 @@ func ExecuteHandler(ctx context.Context, deps *Deps, input map[string]any) (any, opts := []dag.Option{ dag.WithNoteFn(deps.NewNoteFn(ctx)), + dag.WithLevelCompleteFn(func(level int) { + deps.furrowPublish(scopeIDFromCtx(ctx), fmt.Sprintf("level %d complete", level)) + }), dag.WithGitConfig(in.GitConfig), dag.WithResume(in.Resume), dag.WithBuildID(in.BuildID), diff --git a/go/internal/orch/execute_test.go b/go/internal/orch/execute_test.go index 58530e30..b1181e65 100644 --- a/go/internal/orch/execute_test.go +++ b/go/internal/orch/execute_test.go @@ -43,6 +43,29 @@ func minimalPlan() map[string]any { } } +func TestExecutePublishesCompletedLevels(t *testing.T) { + defer withExecCtx("run-levels", "exec-levels")() + f := &fakeFurrow{} + deps := &Deps{App: &mockApp{handler: func(context.Context, string, map[string]any) (map[string]any, error) { + return map[string]any{}, nil + }}, NodeID: "swe-planner", Furrow: f} + plan := minimalPlan() + plan["issues"] = []any{map[string]any{"name": "issue-1", "sequence_number": 1}} + plan["levels"] = []any{[]any{"issue-1"}} + + _, err := ExecuteHandler(context.Background(), deps, map[string]any{ + "plan_result": plan, + "repo_path": t.TempDir(), + "execute_fn_target": "coder.execute", + }) + if err != nil { + t.Fatalf("ExecuteHandler: %v", err) + } + if !reflect.DeepEqual(f.publishes, []string{"level 0 complete"}) { + t.Fatalf("publishes = %v, want level boundary", f.publishes) + } +} + // --------------------------------------------------------------------------- // Contract: config dict → ExecutionConfig with model resolution; call_fn wired; // plan_result / repo_path / node_id forwarded unchanged. diff --git a/go/internal/orch/repohygiene.go b/go/internal/orch/repohygiene.go index 368fff1d..e60f5d4f 100644 --- a/go/internal/orch/repohygiene.go +++ b/go/internal/orch/repohygiene.go @@ -7,10 +7,14 @@ import ( "strings" ) -// harnessMetadataPatterns are the directories the build harness writes INTO the -// target repository: the plan/issue/checkpoint artifacts and the per-issue -// worktrees. They are ours, not the user's work. -var harnessMetadataPatterns = []string{".artifacts/", ".worktrees/"} +// harnessMetadataPatterns are the files and directories the build harness +// writes INTO the target repository: the plan/issue/checkpoint artifacts, the +// per-issue worktrees, and furrow's capture policy and its workspace metadata +// directory. They are ours, not the user's work. .furrow/ matters as much as +// the policy file: `furrow watch` creates it inside the repo, so without this +// every furrow-enabled build leaves a permanent `?? .furrow/` in the user's +// own git status — and a `git add -A` would commit it. +var harnessMetadataPatterns = []string{".artifacts/", ".worktrees/", ".furrowpolicy", ".furrow/"} // excludeHarnessMetadata keeps the harness's own bookkeeping out of the target // repository's git view.