From 307662def63911d2b2cb17008ed9b5cdbdc4bc08 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 9 May 2026 22:27:37 +0000 Subject: [PATCH 1/8] build: move duplicate policy into spack concretizer settings Agent-Logs-Url: https://github.com/eic/containers/sessions/d3ab981b-e2cf-4c22-a919-9eb07c60f3b0 Co-authored-by: wdconinc <4656391+wdconinc@users.noreply.github.com> --- .github/copilot-instructions.md | 2 +- .github/workflows/build-push.yml | 9 ---- .gitlab-ci.yml | 28 ++++------- AGENTS.md | 2 +- containers/eic/Dockerfile | 49 +------------------- docs/spack-environment.md | 11 +++++ spack-environment/ci/spack.yaml | 7 ++- spack-environment/ci_without_acts/spack.yaml | 7 ++- spack-environment/concretizer.yaml | 5 +- spack-environment/cuda/epic/spack.yaml | 7 ++- spack-environment/cvmfs/spack.yaml | 7 ++- spack-environment/tf/epic/spack.yaml | 12 ++++- spack-environment/tf/spack.yaml | 12 ++++- spack-environment/xl/epic/spack.yaml | 7 ++- 14 files changed, 77 insertions(+), 88 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 94bf5e675..7bd3b0b44 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -63,7 +63,7 @@ When updating package versions: Before submitting: 1. Test concretization with builder stage builds -2. Verify no unexpected duplicate packages +2. Verify no unexpected duplicate packages beyond each environment's `concretizer.duplicates.max_dupes` settings 3. **Check BOTH CI workflows pass** (GitHub Actions AND GitLab/EICweb) 4. Update documentation if architecture changes diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 093caee8b..d616b86fe 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -369,7 +369,6 @@ jobs: runner: ubuntu-latest PLATFORM: linux/amd64 target: final - SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3" - BUILD_IMAGE: eic_ BUILD_TYPE: default BUILDER_IMAGE: debian_stable_base @@ -379,7 +378,6 @@ jobs: runner: ubuntu-latest PLATFORM: linux/amd64/v3 target: final - SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3" - BUILD_IMAGE: eic_ BUILD_TYPE: default BUILDER_IMAGE: debian_stable_base @@ -389,7 +387,6 @@ jobs: runner: ubuntu-24.04-arm PLATFORM: linux/arm64 target: final - SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3" - BUILD_IMAGE: eic_ BUILD_TYPE: default BUILDER_IMAGE: debian_stable_base @@ -399,7 +396,6 @@ jobs: runner: ubuntu-latest PLATFORM: linux/amd64 target: final - SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3|py-dask|py-dask-awkward|py-dask-histogram|py-distributed|py-requests" - BUILD_IMAGE: eic_ BUILD_TYPE: default BUILDER_IMAGE: debian_stable_base @@ -409,7 +405,6 @@ jobs: runner: ubuntu-latest PLATFORM: linux/amd64/v3 target: final - SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3|py-dask|py-dask-awkward|py-dask-histogram|py-distributed|py-requests" - BUILD_IMAGE: eic_ BUILD_TYPE: default BUILDER_IMAGE: debian_stable_base @@ -419,7 +414,6 @@ jobs: runner: ubuntu-24.04-arm PLATFORM: linux/arm64 target: final - SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3|py-dask|py-dask-awkward|py-dask-histogram|py-distributed|py-requests" - BUILD_IMAGE: eic_ BUILD_TYPE: default BUILDER_IMAGE: cuda_devel @@ -429,7 +423,6 @@ jobs: runner: ubuntu-latest PLATFORM: linux/amd64 target: builder_concretization_default - SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3" - BUILD_IMAGE: eic_ BUILD_TYPE: default BUILDER_IMAGE: cuda_devel @@ -439,7 +432,6 @@ jobs: runner: ubuntu-latest PLATFORM: linux/amd64 target: builder_concretization_default - SPACK_DUPLICATE_ALLOWLIST: "epic|llvm|py-setuptools|py-urllib3|py-dask|py-dask-awkward|py-dask-histogram|py-distributed|py-requests" fail-fast: false steps: - name: Free Disk Space (Ubuntu) @@ -559,7 +551,6 @@ jobs: RUNTIME_IMAGE=${{ matrix.RUNTIME_IMAGE }} INTERNAL_TAG=${{ env.INTERNAL_TAG }} ENV=${{ matrix.ENV }} - SPACK_DUPLICATE_ALLOWLIST=${{ matrix.SPACK_DUPLICATE_ALLOWLIST }} cache-from: | type=registry,ref=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/buildcache:${{ matrix.BUILD_IMAGE }}${{ matrix.ENV }}-${{ matrix.BUILD_TYPE }}-${{ env.GITHUB_REF_POINT_SLUG }}-${{ matrix.arch }} type=registry,ref=${{ env.GH_REGISTRY }}/${{ env.GH_REGISTRY_USER }}/buildcache:${{ matrix.BUILD_IMAGE }}${{ matrix.ENV }}-${{ matrix.BUILD_TYPE }}-${{ env.GITHUB_BASE_REF_SLUG }}-${{ matrix.arch }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e86ec08c2..180d9da9a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -472,25 +472,15 @@ eic: ${EPIC_VERSION:+--build-arg EPIC_SHA=$(sh .ci/resolve_git_ref eic/epic ${EPIC_VERSION})} ${JUGGLER_VERSION:+--build-arg JUGGLER_SHA=$(sh .ci/resolve_git_ref eic/juggler ${JUGGLER_VERSION})} } - ${IF_BUILD_NIGHTLY+ - --build-arg EDM4EIC_SHA=$(sh .ci/resolve_git_ref eic/EDM4eic ${EDM4EIC_VERSION:-main}) - --build-arg EICRECON_SHA=$(sh .ci/resolve_git_ref eic/EICrecon ${EICRECON_VERSION:-main}) - --build-arg EPIC_SHA=$(sh .ci/resolve_git_ref eic/epic ${EPIC_VERSION:-main}) - --build-arg JUGGLER_SHA=$(sh .ci/resolve_git_ref eic/juggler ${JUGGLER_VERSION:-main}) - } - --build-arg ENV=${ENV} - --build-arg SPACK_DUPLICATE_ALLOWLIST=$( - case "${ENV}" in - ci|ci_without_acts|cuda|dbg|jl|prod) - echo "epic|llvm|py-setuptools|py-urllib3" ;; - xl|tf) - echo "epic|llvm|py-setuptools|py-urllib3|py-dask|py-dask-awkward|py-dask-histogram|py-distributed|py-requests" ;; - *) - echo "epic|llvm|py-setuptools|py-urllib3" ;; - esac - ) - --build-arg jobs=${JOBS} - --build-context spack-environment=spack-environment + ${IF_BUILD_NIGHTLY+ + --build-arg EDM4EIC_SHA=$(sh .ci/resolve_git_ref eic/EDM4eic ${EDM4EIC_VERSION:-main}) + --build-arg EICRECON_SHA=$(sh .ci/resolve_git_ref eic/EICrecon ${EICRECON_VERSION:-main}) + --build-arg EPIC_SHA=$(sh .ci/resolve_git_ref eic/epic ${EPIC_VERSION:-main}) + --build-arg JUGGLER_SHA=$(sh .ci/resolve_git_ref eic/juggler ${JUGGLER_VERSION:-main}) + } + --build-arg ENV=${ENV} + --build-arg jobs=${JOBS} + --build-context spack-environment=spack-environment --secret id=mirrors,src=mirrors.yaml --secret type=env,id=CI_REGISTRY_USER,env=CI_REGISTRY_USER --secret type=env,id=CI_REGISTRY_PASSWORD,env=CI_REGISTRY_PASSWORD diff --git a/AGENTS.md b/AGENTS.md index badfe9df3..e8365e2c3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -214,7 +214,7 @@ The GitLab CI configuration (`.gitlab-ci.yml`) runs similar build jobs but in th When modifying `spack-environment/packages.yaml`: 1. Test concretization: Build `builder_concretization_default` stage -2. Check for duplicate packages (intentionally allowed: `epic`, `llvm`, `py-setuptools`, `py-urllib3`) +2. Check for duplicate packages against the environment's `concretizer.duplicates.max_dupes` settings in `spack-environment/*/spack.yaml` 3. Verify no unexpected duplicates appear in concretization output ### Build Testing diff --git a/containers/eic/Dockerfile b/containers/eic/Dockerfile index ccb2a59f3..922b02368 100644 --- a/containers/eic/Dockerfile +++ b/containers/eic/Dockerfile @@ -46,7 +46,6 @@ ARG ENV=xl ENV SPACK_ENV=/opt/spack-environment/${ENV} ARG SPACK_FLAGS="--backtrace" ARG SPACK_INSTALL_FLAGS="--no-check-signature --show-log-on-error --yes-to-all" -ARG SPACK_DUPLICATE_ALLOWLIST="" ENV SPACK_COLOR="always" ENV GIT_TERMINAL_PROMPT=0 @@ -55,29 +54,6 @@ RUN < Date: Sat, 9 May 2026 22:31:35 +0000 Subject: [PATCH 2/8] fix: restore formatting after duplicate-policy migration Agent-Logs-Url: https://github.com/eic/containers/sessions/d3ab981b-e2cf-4c22-a919-9eb07c60f3b0 Co-authored-by: wdconinc <4656391+wdconinc@users.noreply.github.com> --- .gitlab-ci.yml | 18 +++++++++--------- containers/eic/Dockerfile | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 180d9da9a..71e9993c0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -472,15 +472,15 @@ eic: ${EPIC_VERSION:+--build-arg EPIC_SHA=$(sh .ci/resolve_git_ref eic/epic ${EPIC_VERSION})} ${JUGGLER_VERSION:+--build-arg JUGGLER_SHA=$(sh .ci/resolve_git_ref eic/juggler ${JUGGLER_VERSION})} } - ${IF_BUILD_NIGHTLY+ - --build-arg EDM4EIC_SHA=$(sh .ci/resolve_git_ref eic/EDM4eic ${EDM4EIC_VERSION:-main}) - --build-arg EICRECON_SHA=$(sh .ci/resolve_git_ref eic/EICrecon ${EICRECON_VERSION:-main}) - --build-arg EPIC_SHA=$(sh .ci/resolve_git_ref eic/epic ${EPIC_VERSION:-main}) - --build-arg JUGGLER_SHA=$(sh .ci/resolve_git_ref eic/juggler ${JUGGLER_VERSION:-main}) - } - --build-arg ENV=${ENV} - --build-arg jobs=${JOBS} - --build-context spack-environment=spack-environment + ${IF_BUILD_NIGHTLY+ + --build-arg EDM4EIC_SHA=$(sh .ci/resolve_git_ref eic/EDM4eic ${EDM4EIC_VERSION:-main}) + --build-arg EICRECON_SHA=$(sh .ci/resolve_git_ref eic/EICrecon ${EICRECON_VERSION:-main}) + --build-arg EPIC_SHA=$(sh .ci/resolve_git_ref eic/epic ${EPIC_VERSION:-main}) + --build-arg JUGGLER_SHA=$(sh .ci/resolve_git_ref eic/juggler ${JUGGLER_VERSION:-main}) + } + --build-arg ENV=${ENV} + --build-arg jobs=${JOBS} + --build-context spack-environment=spack-environment --secret id=mirrors,src=mirrors.yaml --secret type=env,id=CI_REGISTRY_USER,env=CI_REGISTRY_USER --secret type=env,id=CI_REGISTRY_PASSWORD,env=CI_REGISTRY_PASSWORD diff --git a/containers/eic/Dockerfile b/containers/eic/Dockerfile index 922b02368..ed22cc384 100644 --- a/containers/eic/Dockerfile +++ b/containers/eic/Dockerfile @@ -171,7 +171,7 @@ if [ -n "${EPIC_SHA}" ] ; then fi if [ -n "${JUGGLER_SHA}" ] ; then sed -i "/# JUGGLER_VERSION$/ s/@[^' ]*/@git.${JUGGLER_SHA}=main/" /opt/spack-environment/packages.yaml -spack deconcretize -y --all juggler + spack deconcretize -y --all juggler fi spack concretize --force EOF From e47d30657a0fc114610bc966f85e79dd8b9792c5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 9 May 2026 22:35:48 +0000 Subject: [PATCH 3/8] fix: keep when_possible with max_dupes concretizer settings Agent-Logs-Url: https://github.com/eic/containers/sessions/d3ab981b-e2cf-4c22-a919-9eb07c60f3b0 Co-authored-by: wdconinc <4656391+wdconinc@users.noreply.github.com> --- spack-environment/ci/spack.yaml | 1 + spack-environment/ci_without_acts/spack.yaml | 1 + spack-environment/concretizer.yaml | 1 + spack-environment/cuda/epic/spack.yaml | 1 + spack-environment/cvmfs/spack.yaml | 1 + spack-environment/tf/epic/spack.yaml | 1 + spack-environment/tf/spack.yaml | 1 + spack-environment/xl/epic/spack.yaml | 1 + 8 files changed, 8 insertions(+) diff --git a/spack-environment/ci/spack.yaml b/spack-environment/ci/spack.yaml index b4c1fcdfa..d9d480d82 100644 --- a/spack-environment/ci/spack.yaml +++ b/spack-environment/ci/spack.yaml @@ -6,6 +6,7 @@ spack: - ../packages_root_without_opengl.yaml - ../view.yaml concretizer: + unify: when_possible duplicates: max_dupes: epic: 10 diff --git a/spack-environment/ci_without_acts/spack.yaml b/spack-environment/ci_without_acts/spack.yaml index 036f1d6d4..d6319f50b 100644 --- a/spack-environment/ci_without_acts/spack.yaml +++ b/spack-environment/ci_without_acts/spack.yaml @@ -6,6 +6,7 @@ spack: - ../packages_root_without_opengl.yaml - ../view.yaml concretizer: + unify: when_possible duplicates: max_dupes: epic: 10 diff --git a/spack-environment/concretizer.yaml b/spack-environment/concretizer.yaml index ae43ae8d9..8f558feda 100644 --- a/spack-environment/concretizer.yaml +++ b/spack-environment/concretizer.yaml @@ -4,6 +4,7 @@ # - We want unified environments that can be installed in a simple view, # with package-specific duplicate allowances configured per environment # through concretizer.duplicates.max_dupes in each spack.yaml. +# Some environments still override unify to when_possible where needed. # concretizer: reuse: diff --git a/spack-environment/cuda/epic/spack.yaml b/spack-environment/cuda/epic/spack.yaml index 7d8e24499..282817f9b 100644 --- a/spack-environment/cuda/epic/spack.yaml +++ b/spack-environment/cuda/epic/spack.yaml @@ -7,6 +7,7 @@ spack: include_concrete: - /opt/spack-environment/cuda concretizer: + unify: when_possible # multiple epic versions duplicates: max_dupes: epic: 10 diff --git a/spack-environment/cvmfs/spack.yaml b/spack-environment/cvmfs/spack.yaml index b4014ec7e..f4b8a8f47 100644 --- a/spack-environment/cvmfs/spack.yaml +++ b/spack-environment/cvmfs/spack.yaml @@ -7,6 +7,7 @@ spack: - ../packages_root_without_opengl.yaml - ../view.yaml concretizer: + unify: when_possible duplicates: max_dupes: epic: 10 diff --git a/spack-environment/tf/epic/spack.yaml b/spack-environment/tf/epic/spack.yaml index 0ce5fe3c1..41534fe96 100644 --- a/spack-environment/tf/epic/spack.yaml +++ b/spack-environment/tf/epic/spack.yaml @@ -5,6 +5,7 @@ spack: - ../../packages.yaml - ../../view.yaml concretizer: + unify: when_possible # included environment: py-numba needs a different llvm than the one provided as external duplicates: max_dupes: epic: 10 diff --git a/spack-environment/tf/spack.yaml b/spack-environment/tf/spack.yaml index a73befc98..f7354f0eb 100644 --- a/spack-environment/tf/spack.yaml +++ b/spack-environment/tf/spack.yaml @@ -5,6 +5,7 @@ spack: - ../packages.yaml - ../view.yaml concretizer: + unify: when_possible # py-numba needs a different llvm than the one provided as external duplicates: max_dupes: epic: 10 diff --git a/spack-environment/xl/epic/spack.yaml b/spack-environment/xl/epic/spack.yaml index 44cf1dc7b..ce252d431 100644 --- a/spack-environment/xl/epic/spack.yaml +++ b/spack-environment/xl/epic/spack.yaml @@ -7,6 +7,7 @@ spack: include_concrete: - /opt/spack-environment/xl concretizer: + unify: when_possible # multiple epic versions duplicates: max_dupes: epic: 10 From 2c2feb2259bf2e65aea4f8afa6843d385905e147 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 9 May 2026 17:45:07 -0500 Subject: [PATCH 4/8] fix: ci: unify: true; max_dupes: rm epic Co-authored-by: Wouter Deconinck --- spack-environment/ci/spack.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/spack-environment/ci/spack.yaml b/spack-environment/ci/spack.yaml index d9d480d82..fb624b526 100644 --- a/spack-environment/ci/spack.yaml +++ b/spack-environment/ci/spack.yaml @@ -6,10 +6,8 @@ spack: - ../packages_root_without_opengl.yaml - ../view.yaml concretizer: - unify: when_possible duplicates: max_dupes: - epic: 10 llvm: 2 py-setuptools: 2 py-urllib3: 2 From b87ea288f40d838ea69d5906ed96527745ebcb54 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 9 May 2026 17:51:38 -0500 Subject: [PATCH 5/8] fix: tf: concretizer: unify: true; max_dupes: rm epic Co-authored-by: Wouter Deconinck --- spack-environment/tf/epic/spack.yaml | 2 -- spack-environment/tf/spack.yaml | 2 -- 2 files changed, 4 deletions(-) diff --git a/spack-environment/tf/epic/spack.yaml b/spack-environment/tf/epic/spack.yaml index 41534fe96..c5ff5ac31 100644 --- a/spack-environment/tf/epic/spack.yaml +++ b/spack-environment/tf/epic/spack.yaml @@ -5,10 +5,8 @@ spack: - ../../packages.yaml - ../../view.yaml concretizer: - unify: when_possible # included environment: py-numba needs a different llvm than the one provided as external duplicates: max_dupes: - epic: 10 llvm: 2 py-setuptools: 2 py-urllib3: 2 diff --git a/spack-environment/tf/spack.yaml b/spack-environment/tf/spack.yaml index f7354f0eb..cb454d3a2 100644 --- a/spack-environment/tf/spack.yaml +++ b/spack-environment/tf/spack.yaml @@ -5,10 +5,8 @@ spack: - ../packages.yaml - ../view.yaml concretizer: - unify: when_possible # py-numba needs a different llvm than the one provided as external duplicates: max_dupes: - epic: 10 llvm: 2 py-setuptools: 2 py-urllib3: 2 From ce165b0ea3f0bc181d4adea01a6e62f58820b9c7 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 9 May 2026 22:35:11 -0500 Subject: [PATCH 6/8] fix: xl: concretizer: unify: true Co-authored-by: Wouter Deconinck --- spack-environment/xl/epic/spack.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/spack-environment/xl/epic/spack.yaml b/spack-environment/xl/epic/spack.yaml index ce252d431..44cf1dc7b 100644 --- a/spack-environment/xl/epic/spack.yaml +++ b/spack-environment/xl/epic/spack.yaml @@ -7,7 +7,6 @@ spack: include_concrete: - /opt/spack-environment/xl concretizer: - unify: when_possible # multiple epic versions duplicates: max_dupes: epic: 10 From 4ece869a38ec64adf7cb2195d4144150c9dae080 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sat, 9 May 2026 22:46:41 -0500 Subject: [PATCH 7/8] fix: *: concretizer: unify: true; max_dupes: rm epic Co-authored-by: Wouter Deconinck --- spack-environment/ci_without_acts/spack.yaml | 2 -- spack-environment/cuda/epic/spack.yaml | 1 - spack-environment/cvmfs/spack.yaml | 1 - 3 files changed, 4 deletions(-) diff --git a/spack-environment/ci_without_acts/spack.yaml b/spack-environment/ci_without_acts/spack.yaml index d6319f50b..daae356c1 100644 --- a/spack-environment/ci_without_acts/spack.yaml +++ b/spack-environment/ci_without_acts/spack.yaml @@ -6,10 +6,8 @@ spack: - ../packages_root_without_opengl.yaml - ../view.yaml concretizer: - unify: when_possible duplicates: max_dupes: - epic: 10 llvm: 2 py-setuptools: 2 py-urllib3: 2 diff --git a/spack-environment/cuda/epic/spack.yaml b/spack-environment/cuda/epic/spack.yaml index 282817f9b..7d8e24499 100644 --- a/spack-environment/cuda/epic/spack.yaml +++ b/spack-environment/cuda/epic/spack.yaml @@ -7,7 +7,6 @@ spack: include_concrete: - /opt/spack-environment/cuda concretizer: - unify: when_possible # multiple epic versions duplicates: max_dupes: epic: 10 diff --git a/spack-environment/cvmfs/spack.yaml b/spack-environment/cvmfs/spack.yaml index f4b8a8f47..b4014ec7e 100644 --- a/spack-environment/cvmfs/spack.yaml +++ b/spack-environment/cvmfs/spack.yaml @@ -7,7 +7,6 @@ spack: - ../packages_root_without_opengl.yaml - ../view.yaml concretizer: - unify: when_possible duplicates: max_dupes: epic: 10 From 5cc89800e5f23af09b168ec03c631ddb4500a500 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Sun, 10 May 2026 08:27:53 -0500 Subject: [PATCH 8/8] fix: comments Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- spack-environment/concretizer.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/spack-environment/concretizer.yaml b/spack-environment/concretizer.yaml index 8f558feda..ae43ae8d9 100644 --- a/spack-environment/concretizer.yaml +++ b/spack-environment/concretizer.yaml @@ -4,7 +4,6 @@ # - We want unified environments that can be installed in a simple view, # with package-specific duplicate allowances configured per environment # through concretizer.duplicates.max_dupes in each spack.yaml. -# Some environments still override unify to when_possible where needed. # concretizer: reuse: