bench-verify.yml

name: Bench verifier

# Manual-only (/bench-verify, or workflow_dispatch to test workflow changes on a
# branch — issue_comment always runs main's copy of this file, see profile-recursion.yml);
# separate from /bench so they never share the bench server.
on:
  workflow_dispatch:
    inputs:
      pairs:
        description: "ABBA pair count (2-40)"
        required: false
        default: "20"
  issue_comment:
    types: [created]

concurrency:
  # Serialization is provided by the single self-hosted bench runner (jobs queue
  # for it). Never cancel a running bench: real /bench-verify comments share the
  # per-PR group and queue; any other comment (this workflow fires on every
  # issue_comment) gets a unique throwaway group so it can't sit in — or evict —
  # the real queue.
  group: ${{ startsWith(github.event.comment.body, '/bench-verify') && format('bench-verify-{0}', github.event.issue.number) || format('bench-verify-ignore-{0}', github.run_id) }}
  cancel-in-progress: false

permissions:
  contents: read
  pull-requests: write
  issues: write

jobs:
  verify:
    if: >-
      github.event_name == 'workflow_dispatch' ||
      (github.event_name == 'issue_comment' &&
       github.event.issue.pull_request &&
       startsWith(github.event.comment.body, '/bench-verify') &&
       contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association))
    runs-on: [self-hosted, bench]
    # Job cap. On a cold runner the recursion BUILDS dominate: per ref a guest build, a
    # prover-test build, and a measuring-CLI build (the CLI is built FROM each ref so it
    # understands that ref's own guest syscalls). Cached in /tmp; build-std and the host
    # cargo target are both per ref (see the recursion step's env), so a ref's second and
    # later presets reuse its own compiled deps — but the two refs never share a dir.
    # 125 rather than 90 to absorb the continuation arms: a 20-tx continuation prove per
    # ref on each side of the verifier bench, plus a 20-tx (was 4-tx) block for the cycle
    # comparison. Keep this ABOVE the sum of the step caps below (50 + 70 = 120) so a
    # runaway step always trips its OWN timeout first: a step timeout still runs the
    # `always()` Post result step, whereas hitting the job cap is a cancellation and is far
    # less dependable about doing so — which would lose the comment entirely.
    timeout-minutes: 125
    steps:
      - name: Acknowledge (react + occupancy notice)
        if: github.event_name == 'issue_comment'
        uses: actions/github-script@v7
        with:
          script: |
            await github.rest.reactions.createForIssueComment({
              owner: context.repo.owner, repo: context.repo.repo,
              comment_id: context.payload.comment.id, content: 'eyes'
            });
            await github.rest.issues.createComment({
              owner: context.repo.owner, repo: context.repo.repo,
              issue_number: context.issue.number,
              body: '⏳ **Benchmark started** on the bench server. Two verifier arms (monolithic + continuations over an ethrex 20-tx block), then the recursion-guest cycle comparison, which adds guest builds on top — longer on a cold runner. The bench server is occupied until it finishes.'
            });

      - name: Resolve PR head + pair count
        id: cfg
        env:
          GH_TOKEN: ${{ github.token }}
          PR_NUM: ${{ github.event.issue.number }}
          COMMENT_BODY: ${{ github.event.comment.body }}
          DISPATCH_PAIRS: ${{ github.event.inputs.pairs }}
        run: |
          if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then
            # Testing this workflow's own changes: bench the dispatched branch vs main.
            HEAD_SHA="$GITHUB_SHA"
            N="${DISPATCH_PAIRS:-20}"
          else
            # Head SHA (not branch name) so fork PRs resolve and a mid-run force-push can't race.
            HEAD_SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid)
            # Optional pair count "/bench-verify 32"; default 20.
            N=$(echo "$COMMENT_BODY" | sed -n 's|^/bench-verify[[:space:]]*\([0-9]\+\).*|\1|p')
            N=${N:-20}
          fi
          echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"
          if ! [[ "$N" =~ ^[0-9]+$ ]]; then
            echo "::warning::pair count '$N' is not a number; using 20"
            N=20
          fi
          if [ "$N" -lt 2 ] || [ "$N" -gt 40 ]; then
            echo "::warning::pair count $N out of range [2,40]; using 20"
            N=20
          fi
          echo "pairs=$N" >> "$GITHUB_OUTPUT"

      - name: Checkout (full history for ref resolution)
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Fetch PR head commit (works for fork PRs)
        if: github.event_name == 'issue_comment'
        env:
          PR_NUM: ${{ github.event.issue.number }}
        run: git fetch origin "pull/$PR_NUM/head" --quiet

      - name: Add cargo to PATH
        run: echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"

      - name: Run verifier benchmark
        id: run
        # Own cap so a hung continuation prove/verify can't eat the whole job budget and
        # starve the recursion step. On timeout `always()` still posts the failure tail.
        timeout-minutes: 50
        env:
          HEAD_SHA: ${{ steps.cfg.outputs.head_sha }}
          PAIRS: ${{ steps.cfg.outputs.pairs }}
          # The script defaults to the real block; this job pins the synthetic one.
          # /bench-verify reports TWO arms and the real path runs only the continuation
          # one (a real block does not fit monolithically), so inheriting the default
          # would silently drop an arm and break comparability with every verify number
          # recorded so far.
          WORKLOAD: synthetic
        run: |
          export SYSROOT_DIR="$HOME/.lambda-vm-sysroot"
          set -o pipefail
          scripts/bench_verify.sh "$HEAD_SHA" origin/main "$PAIRS" 2>&1 | tee /tmp/verify_out.txt
          sed -n '/<!-- verify-abba-report -->/,$p' /tmp/verify_out.txt > /tmp/verify_result.txt

      # Additive: deterministic recursion-guest cycle+accelerator diff (PR vs main), in
      # two regimes: `min` (cheap canary over the empty diagnostic program) and
      # `blowup2-block` (the same verifier over a REAL ethrex 20-tx block proved with
      # continuations, via the `continuation` guest). One exact `execute --cycles`
      # reading per ref (no ABBA); blowup2-block's dumped blob is cached by ref SHA
      # (bench_recursion_cycles.sh), so a repeat run skips re-proving.
      # GUEST_TARGET_DIR and HOST_TARGET_DIR are BASE paths — bench_recursion_cycles.sh
      # appends the ref SHA so each worktree owns its target dirs. Sharing one dir across
      # refs made cargo mix rlibs from both worktrees: loudly on the guest side (broke
      # every regime after the first) and SILENTLY on the host side (cargo declared the
      # second ref fresh and the run measured the first ref's binary).
      # continue-on-error + `!cancelled()` isolate this from the verifier bench above.
      - name: Run recursion guest cycle benchmark
        id: recursion
        if: ${{ !cancelled() }}
        continue-on-error: true
        # Fail-fast under the job cap so a runaway build can't burn the whole job
        # (continue-on-error absorbs the timeout; the verifier verdict still posts).
        timeout-minutes: 70
        env:
          HEAD_SHA: ${{ steps.cfg.outputs.head_sha }}
          # Base paths for the per-ref guest and host target dirs (`<base>_<sha8>`), both
          # rooted under the script's /tmp cache dir: build-std and native deps survive
          # across runs (and across presets within a run) instead of being rebuilt cold.
          GUEST_TARGET_DIR: /tmp/recursion_cycles_run/guest_target
          HOST_TARGET_DIR: /tmp/recursion_cycles_run/host_target
        run: |
          export SYSROOT_DIR="$HOME/.lambda-vm-sysroot"
          .github/scripts/run_recursion_bench.sh "$HEAD_SHA"

      - name: Post result
        if: always()
        uses: actions/github-script@v7
        env:
          HEAD_SHA: ${{ steps.cfg.outputs.head_sha }}
          PAIRS: ${{ steps.cfg.outputs.pairs }}
          OUTCOME: ${{ steps.run.outcome }}
          RECURSION_OUTCOME: ${{ steps.recursion.outcome }}
        with:
          script: |
            const fs = require('fs');
            const read = (p) => { try { return fs.readFileSync(p, 'utf8').trim(); } catch { return ''; } };
            // Bound any raw-log fallback so a future header rename (empty *_result.txt)
            // can't dump the entire build log into the PR comment.
            const tail = (s, n) => s.split('\n').slice(-n).join('\n');
            const head = (process.env.HEAD_SHA || '').slice(0, 10), pairs = process.env.PAIRS;
            let body = `## Verifier benchmark — \`${head}\` vs \`main\` (${pairs} pairs, monolithic + continuations)\n\n`;
            if (process.env.OUTCOME === 'success') {
              const res = read('/tmp/verify_result.txt') || tail(read('/tmp/verify_out.txt'), 30);
              body += res + '\n';
              // Scope this to the rows it actually describes: only the Verify-time rows are
              // ABBA. It used to be a blanket claim, which was wrong for the proof sizes here
              // and for every guest-cycle number in the section below.
              body += '\n<sub>Verify-time rows only: drift-free interleaved A/B/B/A, with paired-t ';
              body += 'and exact Wilcoxon — trust the verdict when the two agree. Proof sizes are ';
              body += 'single exact readings (no averaging). - = PR faster.</sub>\n';
            } else {
              // A step timeout or OOM kill takes the whole script down, so the graceful
              // CONT_SKIP path never runs. bench_verify.sh renders the monolithic report as
              // soon as that arm finishes, so post it rather than throwing away a verdict
              // that was already measured. Path is $WORK/result_mono.txt in that script.
              const mono = read('/tmp/verify_run/result_mono.txt');
              if (mono) {
                body += '⚠️ Run did not complete — the monolithic arm had already finished, ';
                body += 'so its result is below. The continuation arm is missing.\n\n' + mono + '\n';
              }
              body += `❌ Run failed. Last log lines:\n\n` + '```\n' + tail(read('/tmp/verify_out.txt'), 30) + '\n```\n';
            }
            // Additive recursion-guest cycle section, kept clearly separated from the
            // verifier verdict above so a failure here can't change how the bench reads.
            body += '\n---\n\n## Recursion guest cycles — verifier running INSIDE the VM (main vs PR)\n\n';
            if (process.env.RECURSION_OUTCOME === 'success') {
              const rec = read('/tmp/recursion_result.txt') || tail(read('/tmp/recursion_out.txt'), 20);
              if (rec) {
                body += rec + '\n';
              } else {
                body += '_No recursion comparison output was captured._\n';
              }
            } else {
              const rtail = tail(read('/tmp/recursion_out.txt'), 20);
              body += '⚠️ Recursion cycle bench did not complete (does not affect the verifier verdict above).';
              body += rtail ? ' Last log lines:\n\n' + '```\n' + rtail + '\n```\n' : '\n';
            }
            // workflow_dispatch has no PR to comment on; write to the job summary instead.
            if (context.eventName !== 'issue_comment') {
              await core.summary.addRaw(body).write();
              return;
            }
            const { data: comments } = await github.rest.issues.listComments({
              owner: context.repo.owner, repo: context.repo.repo,
              issue_number: context.issue.number,
            });
            const existing = comments.find(c =>
              c.user.type === 'Bot' && c.body.includes('Verifier benchmark —'));
            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner, repo: context.repo.repo,
                comment_id: existing.id, body
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner, repo: context.repo.repo,
                issue_number: context.issue.number, body
              });
            }

      # continue-on-error above protects the posted verifier result, not the failure itself.
      - name: Fail if recursion cycle bench didn't complete
        if: always() && steps.recursion.outcome != 'success'
        run: |
          echo "::error::Recursion cycle bench step did not complete (outcome=${{ steps.recursion.outcome }}) — see its log and the posted result above."
          exit 1