Add explorer NATS publishing and backfill service

.github/actions/setup-zeko-ci/action.yaml

@@ -0,0 +1,93 @@
+name: Setup Zeko CI
+description: Install the shared Zeko CI toolchain and restore dependency/build caches.
+inputs:
+  opam-cache-key:
+    description: Cache key for the opam switch cache.
+    required: true
+  build-cache-key:
+    description: Cache key for the dune build cache.
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: 🐹 Setup go
+      uses: actions/setup-go@v6
+      with:
+        go-version: "1.19.11"
+    - name: 🐹 Build and install capnpc-go
+      shell: bash
+      run: |
+        tmpdir=$(mktemp -d)
+        cd "$tmpdir"
+        go mod init local/build
+        go get capnproto.org/go/capnp/v3@v3.0.0-alpha.5
+        go build -o capnpc-go capnproto.org/go/capnp/v3/capnpc-go
+        sudo cp capnpc-go /usr/local/bin/
+        cd -
+    - uses: dtolnay/rust-toolchain@stable
+    - name: 📥 Install OPAM
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y opam ocaml \
+          libboost-dev \
+          libboost-program-options-dev \
+          libbz2-dev \
+          libcap-dev \
+          libffi-dev \
+          libgflags-dev \
+          libgmp-dev \
+          libgmp3-dev \
+          libjemalloc-dev \
+          liblmdb-dev \
+          liblmdb0 \
+          libpq-dev \
+          libsodium-dev \
+          libssl-dev \
+          pkg-config zlib1g-dev \
+          capnproto \
+          netcat-traditional
+        for i in 1 2 3 4 5; do
+          rm -rf ./opam-repository
+          git clone https://github.com/ocaml/opam-repository.git --depth 1 ./opam-repository && break
+          echo "clone failed, retry $i"
+          getent hosts github.com || true
+          sleep 5
+        done
+        git -C ./opam-repository fetch origin 08d8c16c16dc6b23a5278b06dff0ac6c7a217356
+        git -C ./opam-repository checkout 08d8c16c16dc6b23a5278b06dff0ac6c7a217356
+        opam init --disable-sandboxing -k git -a ./opam-repository --bare
+        for i in 1 2 3 4 5; do
+          opam repository add --yes --all --set-default o1-labs https://github.com/o1-labs/opam-repository.git && break
+          echo "opam repo add failed, retry $i"
+          sleep 5
+        done
+        for i in 1 2 3 4 5; do
+          opam repository add --yes --all --set-default --rank=-1 opam-latest https://github.com/ocaml/opam-repository.git && break
+          echo "opam repo add failed, retry $i"
+          sleep 5
+        done
+        opam switch create "4.14.0"
+        opam switch "4.14.0"
+        opam repository set-repos --yes --this-switch o1-labs default opam-latest
+        export OPAMERRLOGLEN=20
+        export OPAMYES=1
+    - name: 🧩 Get opam dependency cache
+      uses: WarpBuilds/cache@v1
+      id: opam_dependencies
+      with:
+        path: ~/.opam
+        key: ${{ inputs.opam-cache-key }}
+    - name: 🏗️  Install opam dependencies
+      shell: bash
+      run: |
+        opam switch import opam.export --yes
+        eval "$(opam env)"
+        ./scripts/pin-external-packages.sh
+        opam clean --logs -cs --quiet
+    - name: 🧩 Get zeko build cache
+      uses: WarpBuilds/cache@v1
+      id: zeko_build
+      with:
+        path: ./_build
+        key: ${{ inputs.build-cache-key }}

.github/workflows/ci.yaml

@@ -1,3 +1,4 @@
+# Builds and tests Zeko in CI using the exported opam toolchain.
 ---
 name: CI
 
@@ -17,87 +18,17 @@ jobs:
         uses: actions/checkout@v5
         with:
           submodules: recursive
-      # Prepare build environment(Go/OCaml/Rust)
-      - name: 🐹 Setup go
-        uses: actions/setup-go@v6
+      - name: 🧰 Setup Zeko CI
+        uses: ./.github/actions/setup-zeko-ci
+        timeout-minutes: 20
         with:
-          go-version: "1.19.11"
-      - name: 🐹 Build and install capnpc-go
-        run: |
-          tmpdir=$(mktemp -d)
-          cd "$tmpdir"
-          go mod init local/build
-          go get capnproto.org/go/capnp/v3@v3.0.0-alpha.5
-          go build -o capnpc-go capnproto.org/go/capnp/v3/capnpc-go
-          sudo cp capnpc-go /usr/local/bin/
-          cd -
-      - uses: dtolnay/rust-toolchain@stable
-      # Install OPAM and needed dependencies
-      - name: 📥 Install OPAM
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y opam ocaml \
-            libboost-dev \
-            libboost-program-options-dev \
-            libbz2-dev \
-            libcap-dev \
-            libffi-dev \
-            libgflags-dev \
-            libgmp-dev \
-            libgmp3-dev \
-            libjemalloc-dev \
-            liblmdb-dev \
-            liblmdb0 \
-            libpq-dev \
-            libsodium-dev \
-            libssl-dev \
-            pkg-config zlib1g-dev \
-            capnproto \
-            netcat-traditional
-          for i in 1 2 3 4 5; do
-            rm -rf ./opam-repository
-            git clone https://github.com/ocaml/opam-repository.git --depth 1 ./opam-repository && break
-            echo "clone failed, retry $i"
-            getent hosts github.com || true
-            sleep 5
-          done
-          git -C ./opam-repository fetch origin 08d8c16c16dc6b23a5278b06dff0ac6c7a217356
-          git -C ./opam-repository checkout 08d8c16c16dc6b23a5278b06dff0ac6c7a217356
-          opam init --disable-sandboxing -k git -a ./opam-repository --bare
-          for i in 1 2 3 4 5; do
-            opam repository add --yes --all --set-default o1-labs https://github.com/o1-labs/opam-repository.git && break
-            echo "opam repo add failed, retry $i"
-            sleep 5
-          done
-          opam switch create "4.14.0"
-          opam switch "4.14.0"
-          export OPAMERRLOGLEN=20
-          export OPAMYES=1
-      # Retrieve .opam from cache
-      - name: 🧩 Get opam dependency cache
-        uses: WarpBuilds/cache@v1
-        id: opam_dependencies
-        with:
-          path: ~/.opam
-          key: zeko_opam_dependencies-${{ hashFiles('./opam.export') }}
-      # Install opam dependencies
-      - name: 🏗️  Install opam dependencies
-        run: |
-          opam switch import opam.export --yes
-          eval "$(opam env)"
-          ./scripts/pin-external-packages.sh
-          opam clean --logs -cs --quiet
-      # Retrieve _build dir from cache before initiating a build
-      - name: 🧩 Get zeko build cache
-        uses: WarpBuilds/cache@v1
-        id: zeko_build
-        with:
-          path: ./_build
-          key: zeko_build-${{ hashFiles('./opam.export') }}
+          opam-cache-key: zeko_opam_dependencies-${{ hashFiles('./opam.export') }}
+          build-cache-key: zeko_build-${{ hashFiles('./opam.export') }}
       # Build!
       - name: 📦 Build zeko packages
         run: |
           opam exec -- dune build --profile=devnet src/app/zeko/sequencer
+          opam exec -- dune build --profile=devnet src/app/zeko/sequencer/explorer
           opam exec -- dune build --profile=devnet src/app/zeko/da_layer
           opam exec -- dune build --profile=devnet src/app/zeko/signer
       - name: 🧪 Run sequencer tests

.github/workflows/explorer-e2e.yaml

@@ -0,0 +1,45 @@
+# Runs the slow explorer end-to-end Gherkin scenarios independently from the
+# faster explorer integration suite and the existing sequencer integration run.
+---
+name: Explorer E2E
+
+on:
+  push:
+
+concurrency:
+  group: "${{ github.workflow }} @ ${{ github.head_ref || github.ref }}"
+  cancel-in-progress: true
+
+jobs:
+  explorer-e2e:
+    name: "Explorer E2E"
+    runs-on: warp-ubuntu-latest-x64-16x
+    services:
+      nats:
+        image: nats:2-alpine
+        command: -js
+        ports:
+          - 4222:4222
+    steps:
+      - name: 📥 Checkout
+        uses: actions/checkout@v5
+        with:
+          submodules: recursive
+      - name: 🧰 Setup Zeko CI
+        uses: ./.github/actions/setup-zeko-ci
+        timeout-minutes: 20
+        with:
+          opam-cache-key: zeko_opam_dependencies-${{ hashFiles('./opam.export') }}
+          build-cache-key: zeko_build-${{ hashFiles('./opam.export') }}
+      - name: 🧪 Run explorer E2E Gherkin tests
+        run: |
+          for i in $(seq 1 30); do
+            nc -z 127.0.0.1 4222 && break
+            if [ "$i" -eq 30 ]; then
+              echo "Timed out waiting for NATS" >&2
+              exit 1
+            fi
+            sleep 1
+          done
+          NATS_URL="nats://127.0.0.1:4222" \
+            ./src/app/zeko/sequencer/tests/run-sequencer-test.sh real 1 explorer-e2e

.github/workflows/explorer-gherkin.yaml

@@ -0,0 +1,46 @@
+# Runs the explorer Gherkin acceptance tests independently from the sequencer
+# integration suite so PR feedback is not blocked on the longer sequencer flow.
+---
+name: Explorer Gherkin
+
+on:
+  push:
+
+concurrency:
+  group: "${{ github.workflow }} @ ${{ github.head_ref || github.ref }}"
+  cancel-in-progress: true
+
+jobs:
+  explorer-gherkin:
+    name: "Explorer Gherkin"
+    runs-on: warp-ubuntu-latest-x64-16x
+    services:
+      nats:
+        image: nats:2-alpine
+        command: -js
+        ports:
+          - 4222:4222
+    steps:
+      - name: 📥 Checkout
+        uses: actions/checkout@v5
+        with:
+          submodules: recursive
+      - name: 🧰 Setup Zeko CI
+        uses: ./.github/actions/setup-zeko-ci
+        timeout-minutes: 20
+        with:
+          opam-cache-key: zeko_opam_dependencies-${{ hashFiles('./opam.export') }}
+          build-cache-key: zeko_build-${{ hashFiles('./opam.export') }}
+      - name: 🧪 Run explorer Gherkin tests
+        run: |
+          for i in $(seq 1 30); do
+            nc -z 127.0.0.1 4222 && break
+            if [ "$i" -eq 30 ]; then
+              echo "Timed out waiting for NATS" >&2
+              exit 1
+            fi
+            sleep 1
+          done
+          NATS_URL="nats://127.0.0.1:4222" \
+            opam exec -- env -u DUNE_RPC dune runtest --profile=devnet \
+            src/app/zeko/sequencer/explorer/tests

.prototools

@@ -0,0 +1,4 @@
+ocaml = "4.14.0"
+
+[plugins.tools]
+ocaml = "github://Hebilicious/proto-ocaml@v0.1.2"

opam.export

@@ -19,17 +19,19 @@ roots: [
   "dispatch.0.5.0"
   "extlib.1.7.8"
   "fileutils.0.6.4"
-  "graphql-async.0.13.0"
-  "graphql-cohttp.0.13.0"
-  "graphql-lwt.0.13.0"
+  "graphql-async.0.14.0"
+  "graphql-cohttp.0.14.0"
+  "graphql-lwt.0.14.0"
   "graphql_ppx.1.2.2"
   "http.6.0.0~alpha2"
   "js_of_ocaml.4.0.0"
   "js_of_ocaml-ppx.4.0.0"
   "js_of_ocaml-toplevel.4.0.0"
   "lmdb.1.0"
   "menhir.20210419"
-  "merlin.4.5-414"
+  "merlin.4.18-414"
+  "nats-client.0.0.10"
+  "nats-client-async.0.0.10"
   "ocaml-base-compiler.4.14.0"
   "ocamlformat.0.20.1"
   "ocamlgraph.1.8.8"
@@ -52,7 +54,7 @@ roots: [
   "utop.2.9.1"
   "websocket.2.14"
   "websocket-async.2.14"
-  "yojson.1.7.0"
+  "yojson.2.0.0"
 ]
 installed: [
   "alcotest.1.1.0"
@@ -127,7 +129,7 @@ installed: [
   "digestif.0.9.0"
   "dispatch.0.5.0"
   "domain-name.0.3.0"
-  "dot-merlin-reader.4.2"
+  "dot-merlin-reader.4.18-414"
   "dune.3.19.0"
   "dune-build-info.3.1.1"
   "dune-configurator.2.8.2"
@@ -145,10 +147,10 @@ installed: [
   "fix.20201120"
   "fmt.0.8.6"
   "fpath.0.7.3"
-  "graphql.0.13.0"
-  "graphql-async.0.13.0"
-  "graphql-cohttp.0.13.0"
-  "graphql-lwt.0.13.0"
+  "graphql.0.14.0"
+  "graphql-async.0.14.0"
+  "graphql-cohttp.0.14.0"
+  "graphql-lwt.0.14.0"
   "graphql_parser.0.12.2"
   "graphql_ppx.1.2.2"
   "http.6.0.0~alpha2"
@@ -177,8 +179,9 @@ installed: [
   "menhir.20210419"
   "menhirLib.20210419"
   "menhirSdk.20210419"
-  "merlin.4.5-414"
+  "merlin.4.18-414"
   "merlin-extend.0.6.1"
+  "merlin-lib.4.18-414"
   "mew.0.1.0"
   "mew_vi.0.5.0"
   "minicli.5.0.2"
@@ -187,6 +190,8 @@ installed: [
   "mirage-crypto-rng.0.11.0"
   "mirage-crypto-rng-async.0.11.0"
   "mmap.1.1.0"
+  "nats-client.0.0.10"
+  "nats-client-async.0.0.10"
   "num.1.1"
   "ocaml.4.14.0"
   "ocaml-base-compiler.4.14.0"
@@ -302,7 +307,7 @@ installed: [
   "websocket.2.14"
   "websocket-async.2.14"
   "xdg.3.5.0"
-  "yojson.1.7.0"
+  "yojson.2.0.0"
   "zarith.1.7"
   "zarith_stubs_js.v0.14.1"
   "zed.3.1.0"

src/app/cli/src/init/graphql_internal.ml

@@ -138,7 +138,7 @@ module Make
     (Io : Cohttp.S.IO with type 'a t = 'a Schema.Io.t)
     (Body : HttpBody with type +'a io := 'a Schema.Io.t) =
 struct
-  module Ws = Websocket.Connection.Make (Io)
+  module Ws = Graphql_websocket.Connection.Make (Io)
   module Websocket_transport = Websocket_handler.Make (Schema.Io) (Ws)
 
   let ( >>= ) = Io.( >>= )

src/app/zeko/da_layer/lib/core.ml

@@ -12,7 +12,7 @@ module Field = Snark_params.Tick.Field
     6. Check that after applying all the receipts of the command, the receipt chain hashes match the target ledger.
     7. Check that new accounts in ledger openings are in same order as in acc set openings.
     8. Attach timestamp and acc set root.
-    9. Store the diff under the [target_ledger_hash]. 
+    9. Store the diff under the [target_ledger_hash].
     10. Sign [target_ledger_hash]. *)
 let post_diff ~logger ~proof_cache_db ~kvdb ~network_id
     ~(signer : Signer_service.Signer.t) ~ledger_openings ~acc_set_openings