chore(ci): lower self-hosted earthly cache_size_pct 50 → 20

.earthly/config.selfhosted.yml

@@ -20,4 +20,4 @@ global:
     tls_enabled: false
     disable_analytics: true
     buildkit_max_parallelism: 16
-    cache_size_pct: 50
+    cache_size_pct: 20

.github/actions/local-environment-tests/action.yml

@@ -98,14 +98,18 @@ runs:
         WALLET_INDEXER_IMAGE: ${{ inputs.wallet-indexer-image }}
         DOCKERHUB_USER: ${{ inputs.dockerhub-username }}
         DOCKERHUB_TOKEN: ${{ inputs.dockerhub-password }}
+      # Wrapped in retry-transient-ci.sh: retries only on a known-transient CI
+      # failure (buildkit recycled by a concurrent job, or a dropped ghcr push);
+      # genuine failures fail fast. See that script.
       run: |
         . ./.envrc
-        earthly -P \
-          --secret DOCKERHUB_USER="$DOCKERHUB_USER" \
-          --secret DOCKERHUB_TOKEN="$DOCKERHUB_TOKEN" \
-          +local-env-ci \
-          --NODE_IMAGE="$NODE_IMAGE" \
-          --TOOLKIT_IMAGE="$TOOLKIT_IMAGE" \
-          --INDEXER_API_IMAGE="$INDEXER_API_IMAGE" \
-          --CHAIN_INDEXER_IMAGE="$CHAIN_INDEXER_IMAGE" \
-          --WALLET_INDEXER_IMAGE="$WALLET_INDEXER_IMAGE"
+        ./.github/scripts/retry-transient-ci.sh \
+          earthly -P \
+            --secret DOCKERHUB_USER="$DOCKERHUB_USER" \
+            --secret DOCKERHUB_TOKEN="$DOCKERHUB_TOKEN" \
+            +local-env-ci \
+            --NODE_IMAGE="$NODE_IMAGE" \
+            --TOOLKIT_IMAGE="$TOOLKIT_IMAGE" \
+            --INDEXER_API_IMAGE="$INDEXER_API_IMAGE" \
+            --CHAIN_INDEXER_IMAGE="$CHAIN_INDEXER_IMAGE" \
+            --WALLET_INDEXER_IMAGE="$WALLET_INDEXER_IMAGE"

.github/actions/reusable-e2e-tests/action.yml

@@ -29,21 +29,25 @@ runs:
         password: ${{ inputs.ghcr-password }}
 
     - name: Run JUST command
+      env:
+        JUST_COMMAND: ${{ inputs.just-command }}
+        NODE_IMAGE: ${{ inputs.node-image }}
+        TOOLKIT_IMAGE: ${{ inputs.toolkit-image }}
+      shell: bash
       run: |
-        echo "🧪 Running E2E command: ${{ inputs.just-command }}"
+        echo "🧪 Running E2E command: $JUST_COMMAND"
 
-        case "${{ inputs.just-command }}" in
+        # Resolve the just invocation (which positional images each command wants),
+        # then run it through retry-transient-ci.sh: retries only on a known-transient
+        # CI failure (buildkit recycled by a concurrent job, or a dropped ghcr push);
+        # a genuine E2E failure fails fast.
+        case "$JUST_COMMAND" in
           indexer-api-e2e)
-            just "${{ inputs.just-command }}"
-            ;;
-          startup-dev-e2e | startup-qanet-e2e)
-            just "${{ inputs.just-command }}" "${{ inputs.node-image }}"
-            ;;
+            args=(just "$JUST_COMMAND") ;;
           toolkit-e2e | toolkit-update-ledger-parameters-e2e | toolkit-maintenance-e2e | toolkit-mint-e2e | toolkit-tokens-minter-e2e | toolkit-contracts-e2e | genesis-wallets-undeployed-e2e | genesis-wallets-devnet-e2e)
-            just "${{ inputs.just-command }}" "${{ inputs.node-image }}" "${{ inputs.toolkit-image }}"
-            ;;
+            args=(just "$JUST_COMMAND" "$NODE_IMAGE" "$TOOLKIT_IMAGE") ;;
           *)
-            just "${{ inputs.just-command }}" "${{ inputs.node-image }}"
-            ;;
+            args=(just "$JUST_COMMAND" "$NODE_IMAGE") ;;
         esac
-      shell: bash
+
+        ./.github/scripts/retry-transient-ci.sh "${args[@]}"

.github/scripts/retry-transient-ci.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Run "$@" up to 3 times, retrying ONLY on a known-transient CI failure:
+#
+#   * shared self-hosted buildkitd recycled out from under the build — a
+#     concurrent job connected with a different earthly settings hash and
+#     SIGTERMed the daemon, taking down every in-flight build on the host;
+#   * a dropped ghcr.io layer push ("timeout awaiting response headers" and
+#     friends) — buildkit's registry response-header timeout isn't tunable.
+#
+# Earthly's cache makes the retry cheap (work that already succeeded is
+# skipped). A genuine build/test failure does NOT match TRANSIENT_RE and fails
+# fast on attempt 1, so a real regression is never masked by a blind retry.
+#
+# Only "$1" (the command name) is echoed — never "$*", which would leak
+# --secret values passed as arguments into the CI log.
+set -uo pipefail
+
+TRANSIENT_RE='Shutdown signal received|killing buildkit pid|transport is closing|rpc error: code = Unavailable|connection reset by peer|failed to connect to buildkit|timeout awaiting response headers|TLS handshake timeout|i/o timeout'
+LOG="${RUNNER_TEMP:-/tmp}/retry-transient-ci.$$.log"
+
+for attempt in 1 2 3; do
+  echo "----- $1: attempt $attempt/3 -----"
+  "$@" 2>&1 | tee "$LOG"
+  rc=${PIPESTATUS[0]}
+  [ "$rc" -eq 0 ] && exit 0
+  if [ "$attempt" -lt 3 ] && grep -qE "$TRANSIENT_RE" "$LOG"; then
+    echo "::warning::$1: transient CI failure (buildkit recycle / registry push) — retrying in 30s (attempt $attempt/3)"
+    sleep 30
+    continue
+  fi
+  echo "$1 failed (rc=$rc); not a transient signature or attempts exhausted — not retrying"
+  exit "$rc"
+done

.github/workflows/continuous-integration.yml

@@ -171,30 +171,15 @@ jobs:
           echo "[net]" >> "$HOME"/.cargo/config
           echo "git-fetch-with-cli = true" >> "$HOME"/.cargo/config
 
-          # Retry the earthly invocation up to 3 times. Buildkit's default
-          # response-header timeout for registry uploads (~30s in the
-          # underlying containerd resolver) isn't tunable via buildkitd.toml,
-          # and the self-hosted -> ghcr.io path occasionally drops a layer push
-          # with "timeout awaiting response headers". Earthly's cache means
-          # retries skip everything that already succeeded — they essentially
-          # only retry the failing push.
-          #
-          # No ::group:: wrapping — failures need to be visible by default in
-          # the GHA log, not collapsed.
+          # Retry the earthly invocation via the shared retry-transient-ci.sh.
+          # It absorbs the two transient CI failure classes — buildkitd recycled
+          # by a concurrent job, and dropped ghcr.io layer pushes ("timeout
+          # awaiting response headers"; buildkit's registry response-header
+          # timeout isn't tunable) — while failing fast on a genuine build error
+          # (earthly's cache means a retry only re-does the failing push anyway).
           . ./.envrc
-          for attempt in 1 2 3; do
-            echo "----- earthly attempt $attempt/3 -----"
-            if earthly --secret GITHUB_TOKEN="$GITHUB_TOKEN" -P --push +images; then
-              exit 0
-            fi
-            echo "----- earthly attempt $attempt failed -----"
-            if [ $attempt -lt 3 ]; then
-              echo "sleeping 30s before retry"
-              sleep 30
-            fi
-          done
-          echo "earthly failed after 3 attempts"
-          exit 1
+          ./.github/scripts/retry-transient-ci.sh \
+            earthly --secret GITHUB_TOKEN="$GITHUB_TOKEN" -P --push +images
 
       - name: Upload build artifacts
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1