Add deterministic Callgrind benchmarking harness + periodic CI workflow for libgraphql-parser

.github/workflows/parser-benchmarks.yml

@@ -0,0 +1,75 @@
+run-name: "Parser benchmarks (${{ github.event_name == 'schedule' && 'scheduled' || format('@{0}', github.actor) }})"
+
+# Deterministic benchmarks for libgraphql-parser, run under Valgrind's
+# Callgrind (via iai-callgrind). Callgrind executes the benchmarks on
+# a simulated CPU and counts exact instructions + simulated cache
+# hits/misses, so results are reproducible even on noisy shared CI
+# runners — unlike wall-clock benchmarks, which are meaningless there.
+#
+# Runs on demand (workflow_dispatch) and on a monthly schedule to
+# check how libgraphql-parser compares against the latest releases of
+# other GraphQL parsers on crates.io. The markdown report lands in
+# the workflow run's summary page; raw iai-callgrind output is
+# uploaded as an artifact.
+
+name: "Parser benchmarks"
+
+on:
+  workflow_dispatch:
+    inputs:
+      use_latest_competitor_parsers:
+        description: "Benchmark against the latest crates.io releases of graphql-parser and apollo-parser (instead of the versions pinned in Cargo.lock)"
+        type: "boolean"
+        default: true
+  schedule:
+    # 06:23 UTC on the 2nd of every month. Odd minute chosen to avoid
+    # the on-the-hour thundering herd on GitHub's shared runners.
+    - cron: "23 6 2 * *"
+
+env:
+  CARGO_TERM_COLOR: "always"
+
+jobs:
+  callgrind-benchmarks:
+    runs-on: "ubuntu-latest"
+    timeout-minutes: 90
+    steps:
+      - uses: "actions/checkout@v4"
+
+      - name: "Install valgrind"
+        run: |
+          sudo apt-get update
+          sudo apt-get install --yes valgrind
+          valgrind --version
+
+      - name: "Install iai-callgrind-runner (version-matched to Cargo.lock)"
+        run: |
+          version="$(
+            cargo metadata --format-version 1 \
+              | jq -r '.packages[] | select(.name == "iai-callgrind") | .version'
+          )"
+          echo "Installing iai-callgrind-runner ${version}"
+          cargo install iai-callgrind-runner --version "${version}" --locked
+
+      - name: "Fetch Shopify Admin schema fixture"
+        run: "${GITHUB_WORKSPACE}/crates/libgraphql-parser/scripts/fetch-shopify-admin-graphql-schema-fixture.sh"
+
+      - name: "Update competitor parsers to latest crates.io releases"
+        if: "github.event_name == 'schedule' || inputs.use_latest_competitor_parsers"
+        run: "${GITHUB_WORKSPACE}/crates/libgraphql-parser/scripts/ci/parser-benchmarks.use-latest-competitor-parsers.sh"
+
+      - name: "Run Callgrind benchmarks"
+        run: "${GITHUB_WORKSPACE}/crates/libgraphql-parser/scripts/run-callgrind-benchmarks.sh"
+
+      - name: "Publish report to workflow summary"
+        run: "cat ${GITHUB_WORKSPACE}/target/iai/REPORT.md >> ${GITHUB_STEP_SUMMARY}"
+
+      - name: "Upload raw iai-callgrind results"
+        uses: "actions/upload-artifact@v4"
+        with:
+          if-no-files-found: "error"
+          name: "callgrind-benchmark-results"
+          path: |
+            target/iai/**/summary.json
+            target/iai/REPORT.md
+          retention-days: 90

Cargo.toml

@@ -13,11 +13,12 @@ apollo-parser = "0.8"
 bincode = { version = "2.0.1", features = ["serde"] }
 criterion = { version = "0.5", features = ["html_reports"] }
 graphql-parser = "0.4.0"
-proptest = "1.6"
+iai-callgrind = "0.16.1"
 indexmap = { version = "2.10.0", features = ["serde"] }
 inherent = "1.0.12"
 memchr = "2.7"
 proc-macro2 = "1.0.101"
+proptest = "1.6"
 quote = "1.0.40"
 rayon = "1.10"
 serde = { version = "1.0.226", features = ["derive"] }

crates/libgraphql-parser/Cargo.toml

@@ -21,6 +21,15 @@ apollo-parser.workspace = true
 criterion.workspace = true
 proptest.workspace = true
 
+# Valgrind (and therefore iai-callgrind) only supports Linux. The
+# callgrind benchmark target compiles to a no-op on other platforms.
+[target.'cfg(target_os = "linux")'.dev-dependencies]
+iai-callgrind.workspace = true
+
 [[bench]]
 name = "parse_benchmarks"
 harness = false
+
+[[bench]]
+name = "callgrind_benchmarks"
+harness = false

crates/libgraphql-parser/README.md

@@ -276,6 +276,36 @@ script which will also aggregate results in a table like below:
 | github (~1.2 MB)         | 2.06 ms  | ~568 MiB/s  |
 | shopify_admin (~3.1 MB)  | 3.68 ms  | ~840 MiB/s  |
 
+### Deterministic (Callgrind) Benchmarks
+
+Wall-clock benchmarks are meaningful only on quiet, dedicated
+hardware, so a second benchmark suite runs the same fixtures and
+comparison parsers under [Valgrind's Callgrind](https://valgrind.org/docs/manual/cl-manual.html)
+via [iai-callgrind](https://github.com/iai-callgrind/iai-callgrind).
+Callgrind executes each benchmark on a simulated CPU and counts the
+exact number of instructions executed plus simulated cache
+hits/misses (from which an "estimated cycles" metric is derived).
+Repeated runs of the same binary produce identical counts, making
+this suite suitable for noisy shared CI runners and for tracking
+small regressions that wall-clock noise would swallow.
+
+Run it locally (Linux only — Valgrind does not support macOS on
+Apple Silicon) with:
+
+```sh
+cargo install iai-callgrind-runner --version <iai-callgrind version in Cargo.lock>
+./crates/libgraphql-parser/scripts/run-callgrind-benchmarks.sh
+```
+
+The same suite runs in CI via the "Parser benchmarks" GitHub Actions
+workflow (`.github/workflows/parser-benchmarks.yml`), on demand and
+on a monthly schedule. By default the CI run first updates
+`graphql-parser` and `apollo-parser` to their latest crates.io
+releases, so each report reflects how `libgraphql-parser` compares
+against the current state of the ecosystem. The markdown report is
+published to the workflow run's summary page and the raw
+iai-callgrind data is uploaded as an artifact.
+
 ## Core Types
 
 | Type                      | Description                                                                                                                         |