build, cli, rust: ThinLTO multisig-release + --rayon-threads + leanMultisig 5eba3b1 bump

.github/workflows/auto-release.yml

@@ -179,7 +179,9 @@ jobs:
       uses: Swatinem/rust-cache@v2
       with:
         workspaces: "rust -> target"
-        key: ${{ runner.os }}-cargo-docker-${{ hashFiles('**/Cargo.lock') }}
+        # See cache-key rationale in ci.yml `lint` job (#903): include the
+        # rust workspace Cargo.toml so profile changes bust the cache.
+        key: ${{ runner.os }}-cargo-docker-${{ hashFiles('**/Cargo.lock', 'rust/Cargo.toml') }}
 
     - name: Build zeam natively
       run: |

.github/workflows/ci.yml

@@ -69,7 +69,13 @@ jobs:
       with:
         workspaces: "rust -> target"
         cache-bin: false
-        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+        # PR #903 introduced a `multisig-release` cargo profile and pointed the
+        # default (dummy) prover at `rust/target/multisig-release/libzeam_glue.a`.
+        # Hashing `rust/Cargo.toml` here (in addition to Cargo.lock) busts the
+        # cache automatically on any future profile change, avoiding the
+        # "restored cache lacks the new profile dir → confusing linker error"
+        # failure mode flagged in #903 review.
+        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', 'rust/Cargo.toml') }}
 
     - name: Check Rust formatting
       run: |
@@ -152,7 +158,8 @@ jobs:
       with:
         cache-bin: false
         workspaces: "rust -> target"
-        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+        # See cache-key rationale in the `lint` job.
+        key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', 'rust/Cargo.toml') }}
 
     - name: Fetch Zig dependencies
       run: |
@@ -255,7 +262,8 @@ jobs:
       with:
         cache-bin: false
         workspaces: "rust -> target"
-        key: ${{ runner.os }}-cargo-all-provers-${{ hashFiles('**/Cargo.lock') }}
+        # See cache-key rationale in the `lint` job.
+        key: ${{ runner.os }}-cargo-all-provers-${{ hashFiles('**/Cargo.lock', 'rust/Cargo.toml') }}
 
     - name: Fetch Zig dependencies
       run: |
@@ -366,7 +374,9 @@ jobs:
       with:
         cache-bin: false
         workspaces: "rust -> target"
-        key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }}
+        # See cache-key rationale in the `lint` job. Also shared with `dummy-prove`
+        # (same key prefix) — busting via `rust/Cargo.toml` keeps both in sync.
+        key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock', 'rust/Cargo.toml') }}
 
     - name: Fetch Zig dependencies
       run: |
@@ -497,7 +507,10 @@ jobs:
       with:
         cache-bin: false
         workspaces: "rust -> target"
-        key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock') }}
+        # See cache-key rationale in the `lint` job. Shared key prefix with
+        # `test`/`simtest` so all three jobs invalidate together when the rust
+        # workspace profile set changes (e.g. PR #903's `multisig-release`).
+        key: ${{ runner.os }}-cargo-test-${{ hashFiles('**/Cargo.lock', 'rust/Cargo.toml') }}
 
     - name: Fetch Zig dependencies
       run: |
@@ -571,7 +584,8 @@ jobs:
       with:
         cache-bin: false
         workspaces: "rust -> target"
-        key: ${{ runner.os }}-cargo-docker-${{ hashFiles('**/Cargo.lock') }}
+        # See cache-key rationale in the `lint` job.
+        key: ${{ runner.os }}-cargo-docker-${{ hashFiles('**/Cargo.lock', 'rust/Cargo.toml') }}
 
     - name: Fetch Zig dependencies
       run: |

build.zig

@@ -50,7 +50,11 @@ fn defaultSimpleTestRunner(b: *Builder) std.Build.Step.Compile.TestRunner {
 // See blockblaz/zeam#773.
 fn addRustGlueLib(b: *Builder, comp: *Builder.Step.Compile, target: Builder.ResolvedTarget, prover: ProverChoice) void {
     const glue_path = switch (prover) {
-        .dummy, .all => "rust/target/release/libzeam_glue.a",
+        // `.dummy` uses the dedicated multisig-only Cargo profile (ThinLTO,
+        // codegen-units=1) to give the leanMultisig prover the same level of
+        // cross-crate inlining that single-prover builds already get.
+        .dummy => "rust/target/multisig-release/libzeam_glue.a",
+        .all => "rust/target/release/libzeam_glue.a",
         .risc0 => "rust/target/risc0-release/libzeam_glue.a",
         .openvm => "rust/target/openvm-release/libzeam_glue.a",
     };
@@ -881,10 +885,10 @@ fn build_rust_project(b: *Builder, path: []const u8, prover: ProverChoice) *Buil
     // any more than it satisfied the previous `cargo +nightly` shape.
     const cargo_build = switch (prover) {
         .dummy => b.addSystemCommand(&.{
-            "rustup",                "run",        "nightly",                 "cargo",
-            "-C",                    path,         "-Z",                      "unstable-options",
-            "build",                 "--release",  "-p",                      "zeam-glue",
-            "--no-default-features", "--features", "libp2p,hashsig,multisig",
+            "rustup",    "run",                   "nightly",          "cargo",
+            "-C",        path,                    "-Z",               "unstable-options",
+            "build",     "--profile",             "multisig-release", "-p",
+            "zeam-glue", "--no-default-features", "--features",       "libp2p,hashsig,multisig",
         }),
         .risc0 => b.addSystemCommand(&.{
             "rustup",    "run",                   "nightly",       "cargo",

pkgs/cli/src/main.zig

@@ -82,6 +82,13 @@ pub const NodeCommand = struct {
     /// worker path is the supported prod path; the synchronous path
     /// stays in place as a kill-switch via `--chain-worker false`.
     @"chain-worker": bool = true,
+    /// Override the rayon worker count used by the multisig (XMSS) aggregate
+    /// prover. `null` (the default, surfaced as omitted on the CLI) keeps the
+    /// existing post-system-thread split that gives roughly half of the
+    /// remaining cores to the Zig pool and half to rayon — fine for non-
+    /// aggregator nodes. Aggregators on CPU-rich hosts can pass a value here
+    /// to give the prover more parallelism without rebuilding (#899).
+    @"rayon-threads": ?u32 = null,
 
     pub const __shorts__ = .{
         .help = .h,
@@ -107,6 +114,7 @@ pub const NodeCommand = struct {
         .@"db-backend" = "Database backend to use for on-disk state: 'rocksdb' (default) or 'lmdb'",
         .@"chain-spec" = "Path to the chain specification file, if unspecified falls back to the default setting",
         .@"chain-worker" = "Route gossip block + attestation handlers through the dedicated chain-worker thread. On by default; pass `--chain-worker false` to fall back to the legacy synchronous path as a kill-switch.",
+        .@"rayon-threads" = "Override the rayon worker count used by the multisig aggregate prover. If unset, half of the post-system-thread budget goes to the Zig pool and half to rayon. Aggregators in CPU-rich environments benefit from a higher value (e.g. 12 on a 16-vCPU host); non-aggregators can leave it unset.",
         .help = "Show help information for the node command",
     };
 };
@@ -842,6 +850,7 @@ fn mainInner(init: std.process.Init) !void {
                 .hash_sig_key_dir = &.{}, // Initialize to empty slice to avoid segfault in deinit
                 .node_registry = node_registry,
                 .db_backend = leancmd.@"db-backend",
+                .rayon_threads = leancmd.@"rayon-threads",
             };
 
             defer start_options.deinit(allocator);

pkgs/cli/src/node.zig

@@ -103,6 +103,13 @@ pub const NodeOptions = struct {
     /// `--chain-worker false` as the kill-switch for the legacy
     /// synchronous path.
     chain_worker_enabled: bool = true,
+    /// Override the rayon worker count for the multisig aggregate prover.
+    /// `null` keeps the existing automatic split (half of the post-system-
+    /// thread budget to Zig workers, half to rayon). Aggregators in
+    /// CPU-rich environments can set this higher to give the prover more
+    /// parallelism without rebuilding (#899). Surfaced as `--rayon-threads`
+    /// on the `zeam node` CLI.
+    rayon_threads: ?u32 = null,
 
     pub fn deinit(self: *NodeOptions, allocator: std.mem.Allocator) void {
         for (self.bootnodes) |b| allocator.free(b);
@@ -417,9 +424,43 @@ pub const Node = struct {
         // the extra worker on odd counts since aggregate verification enters
         // rayon from Zig workers. Both pools still keep a minimum of one worker
         // so tiny/cgroup-limited systems remain functional.
+        //
+        // Operators can override the rayon worker count with `--rayon-threads`
+        // (#899). The automatic split is conservative — it deliberately leaves
+        // half of the post-system-thread budget to the Zig pool because
+        // verification and gossip work also enter rayon. On a CPU-rich
+        // aggregator that bottleneck is the produce path instead, so giving
+        // rayon more cores measurably shortens the per-pass build time.
+        //
         // Must be called before setupProver/setupVerifier since rayon’s global
         // pool is initialized lazily on first use.
-        const rayon_threads = @max(@as(usize, 1), desired_workers -| worker_count);
+        const rayon_threads = if (options.rayon_threads) |override|
+            @max(@as(usize, 1), @as(usize, override))
+        else
+            @max(@as(usize, 1), desired_workers -| worker_count);
+        self.logger.info(
+            "thread pools: cpu_count={d} zig_workers={d} rayon_threads={d}{s}",
+            .{
+                cpu_count,
+                worker_count,
+                rayon_threads,
+                if (options.rayon_threads != null) " (rayon override via --rayon-threads)" else "",
+            },
+        );
+        // Operator-typo guard for --rayon-threads (review feedback on #903).
+        // Rayon tolerates over-subscription, but values like `--rayon-threads 160`
+        // on a 4-vCPU box silently degrade throughput. Warn (don't reject) so the
+        // operator notices in startup logs without blocking deliberate edge cases
+        // (e.g. fractional cgroup quotas where `getCpuCount` reports more CPUs
+        // than the container can actually use).
+        if (options.rayon_threads) |override| {
+            if (@as(usize, override) > cpu_count) {
+                self.logger.warn(
+                    "--rayon-threads {d} exceeds detected cpu_count={d}; rayon over-subscription typically reduces throughput. Verify this is intentional.",
+                    .{ override, cpu_count },
+                );
+            }
+        }
         xmss.setRayonThreads(rayon_threads);
 
         // Pre-warm the XMSS verifier on the main thread before any worker can