refactor: add in-memory chainstate backends to speed up tests

.github/actions/artifacts/nextest-archive/action.yml

@@ -9,10 +9,26 @@ inputs:
   archive-file:
     description: "The nextest archive file"
     required: true
-  debug-binary-path:
-    description: "Debug binary path"
-    required: true
-
+  artifact-name:
+    description: "Artifact containing the nextest archive and bundled tools"
+    required: false
+    default: "nextest-archive"
+  test-list-artifact-name:
+    description: "Artifact containing the nextest test manifest"
+    required: false
+    default: "nextest-test-list"
+  package:
+    description: "Optional workspace package to archive instead of the full workspace"
+    required: false
+    default: ""
+  exclude-packages:
+    description: "Space-separated workspace packages to exclude from the archive"
+    required: false
+    default: ""
+  features:
+    description: "Comma-separated Cargo features enabled while building the archive"
+    required: false
+    default: "monitoring_prom"
 runs:
   using: "composite"
   steps:
@@ -21,20 +37,19 @@ runs:
         inputs.action == 'restore'
       uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # 8.0.1
       with:
-        name: "nextest-archive"
+        name: ${{ inputs.artifact-name }}
 
     # Install the tool binaries bundled in the artifact (see the staging step
-    # in the create path below)
+    # in the create path below).
     - name: Install Bundled Tools
       if: |
         inputs.action == 'restore'
       shell: bash
       run: |
-        # upload-artifact does not preserve file permissions, so restore the
-        # executable bit before putting the tools on $PATH
-        chmod +x ci-tools/*
         mkdir -p "$HOME/.cargo/bin"
-        mv -f ci-tools/* "$HOME/.cargo/bin/"
+        tar --extract --zstd --file ci-tools.tar.zst --directory "$HOME/.cargo/bin"
+        chmod +x "$HOME/.cargo/bin"/{cargo-nextest,grcov}
+        echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
 
     # Checkout the code
     - name: Checkout the latest code
@@ -49,8 +64,13 @@ runs:
       if: |
         inputs.action == 'create'
       uses: ./.github/actions/setup-rust-toolchain
-      with:
-        components: llvm-tools-preview
+
+    # Cache compiler outputs by content in addition to restoring Cargo's target
+    # tree; this still helps when changed sources invalidate a local crate.
+    - name: Setup sccache
+      if: |
+        inputs.action == 'create'
+      uses: ./.github/actions/setup-sccache
 
     #  Install nextest and grcov
     #    grcov is not needed to build the archive, but it is bundled into the
@@ -62,25 +82,29 @@ runs:
       with:
         tools: nextest, grcov
 
-    # Stage the tool binaries so they are uploaded with the artifact. The many
-    # parallel test jobs restore them from there instead of each downloading
-    # from GitHub Releases, which rate-limits the runners' shared IPs (429)
+    # Stage the tools that do not ship with the Rust toolchain. Coverage jobs
+    # install llvm-tools-preview because its binaries depend on matching shared
+    # libraries from the toolchain sysroot.
     - name: Stage Tool Binaries for Upload
       if: |
         inputs.action == 'create'
       shell: bash
       run: |
         mkdir -p ci-tools
-        cp "$(command -v cargo-nextest)" "$(command -v grcov)" ci-tools/
+        cp \
+          "$(command -v cargo-nextest)" \
+          "$(command -v grcov)" \
+          ci-tools/
+        tar --create --zstd --file ci-tools.tar.zst --directory ci-tools .
 
-    # Install mold linker and clang (faster than default)      
-    - name: Install mold Linker (and clang)
+    # Clang is preinstalled on GitHub's Ubuntu runners. Downloading mold
+    # directly avoids refreshing apt metadata in every archive job.
+    - name: Install mold Linker
       if: |
         inputs.action == 'create'
-      shell: bash
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y mold clang
+      uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1, mold 2.42.0
+      with:
+        make-default: false
 
     # Restore the cargo cache
     - name: Restore Cargo Cache
@@ -96,39 +120,70 @@ runs:
         inputs.action == 'create'
       shell: bash
       env:
+        ARCHIVE_EXCLUDE_PACKAGES: ${{ inputs.exclude-packages }}
+        ARCHIVE_FEATURES: ${{ inputs.features }}
+        ARCHIVE_PACKAGE: ${{ inputs.package }}
         RUSTFLAGS: "-C instrument-coverage -C linker=clang -C link-arg=-fuse-ld=mold -A warnings"
         CARGO_INCREMENTAL: 0
-      # This command should be kept in sync with the same command in cache/cargo action
+      # Keep these package and feature combinations represented in the
+      # default-branch builds in cache/cargo.
       run: |
         # Capture the archive file input, then expand the leading ~ to $HOME
         INPUT_FILE="${{ inputs.archive-file }}"
         ARCHIVE_FILE="${INPUT_FILE/#\~/$HOME}"
 
+        PACKAGE_ARGS=(--workspace)
+        if [[ -n "$ARCHIVE_PACKAGE" ]]; then
+          if [[ -n "$ARCHIVE_EXCLUDE_PACKAGES" ]]; then
+            echo "package and exclude-packages cannot be used together" >&2
+            exit 1
+          fi
+          PACKAGE_ARGS=(--package "$ARCHIVE_PACKAGE")
+        elif [[ -n "$ARCHIVE_EXCLUDE_PACKAGES" ]]; then
+          read -r -a EXCLUDED_PACKAGES <<< "$ARCHIVE_EXCLUDE_PACKAGES"
+          for excluded_package in "${EXCLUDED_PACKAGES[@]}"; do
+            PACKAGE_ARGS+=(--exclude "$excluded_package")
+          done
+        fi
+
         cargo nextest archive \
-          --workspace \
+          "${PACKAGE_ARGS[@]}" \
           --tests \
           --locked \
-          --features monitoring_prom \
+          --features "$ARCHIVE_FEATURES" \
           --archive-file "$ARCHIVE_FILE"
 
+        cargo nextest list \
+          --archive-file "$ARCHIVE_FILE" \
+          --message-format json \
+          > nextest-test-list.json
+
+    # Matrix-generation jobs only need the test manifest. Keep it separate so
+    # they do not download the much larger archive and debug binaries.
+    - name: Create Nextest Test List Artifact
+      if: |
+        inputs.action == 'create'
+      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # 7.0.1
+      with:
+        name: ${{ inputs.test-list-artifact-name }}
+        path: nextest-test-list.json
+        if-no-files-found: error
+        retention-days: 1
+        compression-level: 9
+        overwrite: true
+
     - name: Create Nextest Archive Artifact
       if: |
         inputs.action == 'create'
       uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # 7.0.1
       with:
-        name: "nextest-archive"
+        name: ${{ inputs.artifact-name }}
         path: |
           ${{ inputs.archive-file }}
-          ci-tools/
-          ${{ inputs.debug-binary-path }}
-          !${{ inputs.debug-binary-path }}/examples/**
-          !${{ inputs.debug-binary-path }}/incremental/**
-          !${{ inputs.debug-binary-path }}/.fingerprint/**
-          !${{ inputs.debug-binary-path }}/build/**
-          !${{ inputs.debug-binary-path }}/deps/**/*.d
-          !${{ inputs.debug-binary-path }}/deps/**/*.rlib
-          !${{ inputs.debug-binary-path }}/deps/**/*.rmeta
+          ci-tools.tar.zst
         if-no-files-found: error
         retention-days: 1
-        compression-level: 9 # 0 (none) - 9 (most)
+        # The nextest archive and tool binaries are already compressed or
+        # incompressible; recompressing them only delays every test job.
+        compression-level: 0
         overwrite: true  

.github/actions/cache/cargo/action.yml

@@ -60,6 +60,15 @@ runs:
         restore-keys: ${{ steps.generate-keys.outputs.cargo-restore-key }}
         fail-on-cache-miss: true
 
+    # Cache extraction makes target files newer than the checkout. Refresh only
+    # sources whose Git blob changed so Cargo can safely reuse unchanged local
+    # crates and test harnesses from the default-branch cache.
+    - name: Restore Workspace Source Freshness
+      if: |
+        inputs.action == 'restore'
+      shell: bash
+      run: bash ./.github/scripts/restore_cargo_freshness.sh
+
     # Checkout the code
     - name: Checkout the latest code
       if: |
@@ -73,8 +82,13 @@ runs:
       if: |
         inputs.action == 'create'
       uses: ./.github/actions/setup-rust-toolchain
-      with:
-        components: llvm-tools-preview
+
+    # Populate the shared compiler cache while creating the default-branch
+    # Cargo cache so pull-request builds can reuse unchanged workspace crates.
+    - name: Setup sccache
+      if: |
+        inputs.action == 'create'
+      uses: ./.github/actions/setup-sccache
 
     #  Install nextest
     - name: Install nextest
@@ -84,50 +98,55 @@ runs:
       with:
         tools: nextest
 
-    # Install mold linker and clang (faster than default)      
-    - name: Install mold Linker (and clang)
+    # Clang is preinstalled on GitHub's Ubuntu runners. Downloading mold
+    # directly avoids refreshing apt metadata in the cache-generation job.
+    - name: Install mold Linker
       if: |
         inputs.action == 'create'
-      shell: bash
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y mold clang
+      uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1, mold 2.42.0
+      with:
+        make-default: false
 
-    # Build the cargo crates
+    # Build the exact archive topologies consumed by pull-request CI. Cargo
+    # keeps their distinct feature fingerprints side-by-side in target/.
     - name: Build Cargo Crates
       if: |
         inputs.action == 'create'
       shell: bash
       env:
         RUSTFLAGS: "-C instrument-coverage -C linker=clang -C link-arg=-fuse-ld=mold -A warnings"
         CARGO_INCREMENTAL: 0
-      # This command should be kept in sync with the same command in artifacts/nextest-archive action
       run: |
-        cargo nextest archive \
+        build_cache_archive() {
+          local archive_name="$1"
+          local features="$2"
+          shift 2
+
+          cargo nextest archive \
+            "$@" \
+            --tests \
+            --locked \
+            --features "$features" \
+            --archive-file "$RUNNER_TEMP/$archive_name.tar.zst"
+        }
+
+        build_cache_archive remaining monitoring_prom \
           --workspace \
-          --tests \
-          --locked \
-          --features monitoring_prom \
-          --archive-file "not_used.tar.zst"
+          --exclude stacks-node \
+          --exclude stackslib
+        build_cache_archive bitcoin monitoring_prom,stacks-signer/monitoring_prom \
+          --package stacks-node
+        build_cache_archive stackslib monitoring_prom \
+          --package stackslib
 
-    # Dynamic cleanup of local workspace crates right after building. We never want them to exist in the produced cache.
-    # These are things like stacks-signer, stacks-inspect, etc. built locally from our codebase.
-    # We must always compile them on-the-fly from current code, as they aren't bumped in version when they are changed.
-    - name: Clean Local Workspace Crates
+    # Record content identities so restore jobs can invalidate changed sources
+    # without throwing away every reusable workspace artifact.
+    - name: Record Workspace Source Index
       if: |
         inputs.action == 'create'
       shell: bash
       run: |
-        if [ -f Cargo.toml ]; then
-          echo "Identifying local workspace crates to exclude from restored cache..."
-          cargo metadata --format-version 1 --no-deps | jq -r '.packages[].name' | while read -r crate; do
-            echo "Wiping build artifacts for local crate: $crate"
-            cargo clean -p "$crate"
-          done
-        else
-          echo "ERROR: No Cargo.toml found in the current working directory."
-          exit 1
-        fi
+        git ls-files -s -z > target/.ci-workspace-source-index
 
     # Save cache data
     - name: Save Cache

.github/actions/coveralls-upload/action.yml

@@ -6,7 +6,8 @@ description: |
 inputs:
   files:
     description: Space-separated list of coverage report files to upload (supports globs)
-    required: true
+    required: false
+    default: ''
   build-number:
     description: Overrides the build number Coveralls autodetects from CI
     required: true
@@ -21,6 +22,18 @@ inputs:
   git-commit:
     description: When provided, attributes the coverage to a specific commit
     required: false
+  flag-name:
+    description: Unique report name for a parallel Coveralls build
+    required: false
+    default: ''
+  parallel:
+    description: Whether this report belongs to a parallel Coveralls build
+    required: false
+    default: 'false'
+  parallel-finished:
+    description: Whether to finalize a parallel Coveralls build
+    required: false
+    default: 'false'
   coverage-reporter-version:
     description: Coveralls reporter binary version (the action SHA is pinned but downloads the reporter at 'latest' unless pinned)
     required: false
@@ -42,6 +55,9 @@ runs:
         compare-ref: ${{ inputs.compare-ref }}
         git-branch: ${{ inputs.git-branch }}
         git-commit: ${{ inputs.git-commit }}
+        flag-name: ${{ inputs.flag-name }}
+        parallel: ${{ inputs.parallel }}
+        parallel-finished: ${{ inputs.parallel-finished }}
         fail-on-error: true
         coverage-reporter-version: ${{ inputs.coverage-reporter-version }}
 
@@ -61,6 +77,9 @@ runs:
         compare-ref: ${{ inputs.compare-ref }}
         git-branch: ${{ inputs.git-branch }}
         git-commit: ${{ inputs.git-commit }}
+        flag-name: ${{ inputs.flag-name }}
+        parallel: ${{ inputs.parallel }}
+        parallel-finished: ${{ inputs.parallel-finished }}
         fail-on-error: true
         coverage-reporter-version: ${{ inputs.coverage-reporter-version }}
 
@@ -80,6 +99,9 @@ runs:
         compare-ref: ${{ inputs.compare-ref }}
         git-branch: ${{ inputs.git-branch }}
         git-commit: ${{ inputs.git-commit }}
+        flag-name: ${{ inputs.flag-name }}
+        parallel: ${{ inputs.parallel }}
+        parallel-finished: ${{ inputs.parallel-finished }}
         fail-on-error: true
         coverage-reporter-version: ${{ inputs.coverage-reporter-version }}