From ffa5b2ae95dc7251609de7091247b509208b2681 Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Tue, 25 Aug 2026 12:22:39 -0700 Subject: [PATCH 1/2] Build and publish toolchain images in a single job A version bump used to run three full pipelines. The pull request built all five images across every architecture, the branch push built them again and published nothing, and the tag push built them a third time before each promotion rebuilt the 13 images it was about to push. Seventy builds ran per release-branch bump; thirteen produced an image anyone could pull. Publishing rebuilt because the build and the push happened in different jobs on different VMs. Split each image into a pull-request block that builds and tests, and a branch block that builds, tests and pushes in one job. Exactly one of the pair runs per event, so the push happens on the machine that already holds the image and calico-*-cd no longer has calico-*-image as a prerequisite. Only the branch blocks carry the docker push secret, and they are gated on `pull_request !~ '.*'` as well as the branch name. Semaphore's `branch` resolves to a pull request's *target* branch, so `branch = 'master'` alone is true on a pull request into master. The promotions this replaces hinted at that: each was gated on `branch = 'master' OR tag =~ ...` and *also* wrapped its command in `if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]`, a second guard that only makes sense if the first condition can be true on a pull request. Gate every block with change_in, so a Kubernetes bump no longer rebuilds base, binfmt, rust-build and tinygo. Let release branches publish their branch tag the way master already does, and publish the release tag from the same job by reading the tag create-tag-on-version-change.yml has created by then, which leaves the tag pipeline with nothing to build. The promotions are folded into the main pipeline: manifests need no local image, and one pipeline is easier to follow than six. That takes a release-branch bump from 70 builds to 8. Publishing also now happens strictly after the tests pass, which the promotions did not guarantee -- their auto_promote had no result = 'passed' condition. Assemble semaphore.yml from fragments under .semaphore/semaphore.yml.d the way projectcalico/calico does, so each image's blocks can be edited on their own. Block fragments sit at column zero and the generator indents them, so each one is valid YAML by itself. The numeric prefix groups blocks by what the image is, not by build order; no toolchain image is built FROM another. A new Checks block verifies the generated file matches its fragments and runs shellcheck, and every image block depends on it, so a stale pipeline file stops the builds before any emulated build starts. Raise execution_time_limit to 120 minutes: a publish job now does the work that the build pipeline and the publish promotion each had 60 minutes for, and the ppc64le and s390x builds run under QEMU. --- .semaphore/promotions/calico-base.yml | 34 -- .semaphore/promotions/calico-binfmt.yml | 31 -- .semaphore/promotions/calico-go-build.yml | 65 --- .semaphore/promotions/calico-rust-build.yml | 45 -- .semaphore/promotions/calico-tinygo.yml | 45 -- .semaphore/semaphore.yml | 403 +++++++++++++++--- .semaphore/semaphore.yml.d/01-preamble.yml | 31 ++ .../semaphore.yml.d/02-global_job_config.yml | 11 + .semaphore/semaphore.yml.d/09-blocks.yml | 12 + .../semaphore.yml.d/blocks/00-checks.yml | 19 + .../semaphore.yml.d/blocks/10-go-build.yml | 100 +++++ .../semaphore.yml.d/blocks/10-rust-build.yml | 62 +++ .../semaphore.yml.d/blocks/10-tinygo.yml | 168 ++++++++ .semaphore/semaphore.yml.d/blocks/20-base.yml | 38 ++ .../semaphore.yml.d/blocks/30-binfmt.yml | 31 ++ Makefile | 13 + hack/gen-semaphore-yaml.sh | 62 +++ images/Makefile | 39 +- 18 files changed, 926 insertions(+), 283 deletions(-) delete mode 100644 .semaphore/promotions/calico-base.yml delete mode 100644 .semaphore/promotions/calico-binfmt.yml delete mode 100644 .semaphore/promotions/calico-go-build.yml delete mode 100644 .semaphore/promotions/calico-rust-build.yml delete mode 100644 .semaphore/promotions/calico-tinygo.yml create mode 100644 .semaphore/semaphore.yml.d/01-preamble.yml create mode 100644 .semaphore/semaphore.yml.d/02-global_job_config.yml create mode 100644 .semaphore/semaphore.yml.d/09-blocks.yml create mode 100644 .semaphore/semaphore.yml.d/blocks/00-checks.yml create mode 100644 .semaphore/semaphore.yml.d/blocks/10-go-build.yml create mode 100644 .semaphore/semaphore.yml.d/blocks/10-rust-build.yml create mode 100644 .semaphore/semaphore.yml.d/blocks/10-tinygo.yml create mode 100644 .semaphore/semaphore.yml.d/blocks/20-base.yml create mode 100644 .semaphore/semaphore.yml.d/blocks/30-binfmt.yml create mode 100755 hack/gen-semaphore-yaml.sh diff --git a/.semaphore/promotions/calico-base.yml b/.semaphore/promotions/calico-base.yml deleted file mode 100644 index 5a12b8ab..00000000 --- a/.semaphore/promotions/calico-base.yml +++ /dev/null @@ -1,34 +0,0 @@ -version: v1.0 -name: Publish calico/base images -agent: - machine: - type: f1-standard-2 - os_image: ubuntu2204 - -execution_time_limit: - minutes: 30 - -global_job_config: - env_vars: - - name: DEV_REGISTRIES - value: calico - secrets: - - name: docker - prologue: - commands: - - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin - - checkout - -blocks: - - name: Publish calico/base multi-arch images - dependencies: [] - run: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - task: - env_vars: - - name: BRANCH_NAME - value: ${SEMAPHORE_JOB_CREATION_TIME} - jobs: - - name: Linux multi-arch - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images calico-base-cd CONFIRM=true; fi diff --git a/.semaphore/promotions/calico-binfmt.yml b/.semaphore/promotions/calico-binfmt.yml deleted file mode 100644 index 846d94bf..00000000 --- a/.semaphore/promotions/calico-binfmt.yml +++ /dev/null @@ -1,31 +0,0 @@ -version: v1.0 -name: Publish calico/binfmt images -agent: - machine: - type: f1-standard-2 - os_image: ubuntu2204 - -execution_time_limit: - minutes: 30 - -global_job_config: - env_vars: - - name: DEV_REGISTRIES - value: calico - secrets: - - name: docker - prologue: - commands: - - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin - - checkout - -blocks: - - name: Publish calico/binfmt amd64 images - dependencies: [] - run: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - task: - jobs: - - name: Linux amd64 - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images calico-binfmt-cd CONFIRM=true; fi diff --git a/.semaphore/promotions/calico-go-build.yml b/.semaphore/promotions/calico-go-build.yml deleted file mode 100644 index ebd38ebc..00000000 --- a/.semaphore/promotions/calico-go-build.yml +++ /dev/null @@ -1,65 +0,0 @@ -version: v1.0 -name: Publish calico/go-build images -agent: - machine: - type: f1-standard-2 - os_image: ubuntu2204 - -execution_time_limit: - minutes: 60 - -global_job_config: - env_vars: - - name: DEV_REGISTRIES - value: calico - secrets: - - name: docker - prologue: - commands: - - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin - - checkout - # If an image build is triggered from a git tag, use the git tag name set by semaphore. - # For utility changes that don't affect calico/go-build versions, we will append a release - # number in the tag. This is handled in create-tag-on-version-change github workflow. - - | - if [ "${SEMAPHORE_GIT_REF_TYPE}" = "tag" ]; then - export BRANCH_NAME=${SEMAPHORE_GIT_TAG_NAME} - else - export BRANCH_NAME=${SEMAPHORE_GIT_WORKING_BRANCH} - fi - -blocks: - - name: Publish calico/go-build images - dependencies: [] - run: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - task: - jobs: - - name: Linux multi-arch - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images calico-go-build-cd VALIDARCHES=$ARCH CONFIRM=true; fi - matrix: - - env_var: ARCH - values: ["amd64", "arm64", "ppc64le", "s390x"] - - name: Publish calico/go-build multi-arch manifests - dependencies: - - Publish calico/go-build images - run: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - task: - jobs: - - name: Linux multi-arch manifests - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images push-calico-go-build-manifests CONFIRM=true; fi - - name: Trigger calico/go-build pin updates - dependencies: - - Publish calico/go-build multi-arch manifests - run: - when: "tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - task: - secrets: - - name: semaphore-api - jobs: - - name: Auto calico/go-build update - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make update-go-build-pins CONFIRM=true; fi diff --git a/.semaphore/promotions/calico-rust-build.yml b/.semaphore/promotions/calico-rust-build.yml deleted file mode 100644 index cf0b404b..00000000 --- a/.semaphore/promotions/calico-rust-build.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: v1.0 -name: Publish calico/rust-build images -agent: - machine: - type: f1-standard-2 - os_image: ubuntu2204 - -execution_time_limit: - minutes: 60 - -global_job_config: - env_vars: - - name: DEV_REGISTRIES - value: calico - secrets: - - name: docker - prologue: - commands: - - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin - - checkout - - export BRANCH_NAME=$(yq -r '.rust.version' images/calico-rust-build/versions.yaml) - -blocks: - - name: Publish calico/rust-build images - dependencies: [] - run: - when: "branch = 'master' AND change_in('/images/calico-rust-build/')" - task: - jobs: - - name: Linux multi-arch - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images calico-rust-build-cd VALIDARCHES=$ARCH CONFIRM=true; fi - matrix: - - env_var: ARCH - values: ["amd64", "arm64", "ppc64le", "s390x"] - - name: Publish calico/rust-build multi-arch manifests - dependencies: - - Publish calico/rust-build images - run: - when: "branch = 'master' AND change_in('/images/calico-rust-build/')" - task: - jobs: - - name: Linux multi-arch manifests - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images push-calico-rust-build-manifests CONFIRM=true; fi diff --git a/.semaphore/promotions/calico-tinygo.yml b/.semaphore/promotions/calico-tinygo.yml deleted file mode 100644 index a71ef13c..00000000 --- a/.semaphore/promotions/calico-tinygo.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: v1.0 -name: Publish calico/tinygo images -agent: - machine: - type: f1-standard-2 - os_image: ubuntu2204 - -execution_time_limit: - minutes: 60 - -global_job_config: - env_vars: - - name: DEV_REGISTRIES - value: calico - secrets: - - name: docker - prologue: - commands: - - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin - - checkout - - export BRANCH_NAME=$(yq -r '.tinygo.version' images/calico-tinygo/versions.yaml) - -blocks: - - name: Publish calico/tinygo image - dependencies: [] - run: - when: "branch = 'master' AND change_in('/images/calico-tinygo/')" - task: - jobs: - - name: Publish per-arch - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images calico-tinygo-cd ARCH=$ARCH CONFIRM=true; fi - matrix: - - env_var: ARCH - values: ["amd64", "arm64"] - - name: Publish calico/tinygo multi-arch manifest - dependencies: - - Publish calico/tinygo image - run: - when: "branch = 'master' AND change_in('/images/calico-tinygo/')" - task: - jobs: - - name: Manifest - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images push-calico-tinygo-manifests CONFIRM=true; fi diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index a4a7db92..ed9f0b22 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -1,3 +1,6 @@ +# !! WARNING, DO NOT EDIT !! This file is generated from the fragments +# in /.semaphore/semaphore.yml.d. To update, modify the relevant +# fragment and then run 'make gen-semaphore-yaml'. version: v1.0 name: go-build agent: @@ -10,9 +13,25 @@ auto_cancel: queued: when: "branch != 'master'" +# A publish job builds, tests and pushes in one go, where the build and the +# push used to have a 60-minute budget each in separate pipelines. The +# ppc64le and s390x builds run under QEMU emulation and are the long pole. execution_time_limit: - minutes: 60 + minutes: 120 +# Each image has two blocks: one for pull requests that builds and tests, and +# one for master and release branches that builds, tests and publishes in a +# single job. Exactly one of the pair runs per event, so an image is built once +# per commit instead of once per pipeline, and the push happens on the machine +# that already holds the image. +# +# Only the publish blocks carry the `docker` push secret. Semaphore evaluates +# `run: when:` before scheduling a block, so that secret is never injected into +# a pull request run, forked or otherwise. +# +# Git tags do not build. A push to a release branch publishes both the branch +# tag and the release tag that create-tag-on-version-change.yml just created, +# so the tag pipeline has nothing left to do and every block below skips it. global_job_config: secrets: - name: docker-hub @@ -23,35 +42,44 @@ global_job_config: # Semaphore is doing shallow clone on a commit without tags. # unshallow it for GIT_VERSION:=$(shell git describe --tags --dirty --always) @ Makefile.common - git fetch --unshallow - -promotions: - # Publish base, binfmt, and go-build images for master or release tags (example: 1.23.3-llvm18.1.8-k8s1.30.5). - - name: Publish calico/base images - pipeline_file: promotions/calico-base.yml - auto_promote: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - - name: Publish calico/binfmt images - pipeline_file: promotions/calico-binfmt.yml - auto_promote: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - - name: Publish calico/go-build images - pipeline_file: promotions/calico-go-build.yml - auto_promote: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - # Publish rust-build images for master and files under images/calico-rust-build have changed. - - name: Publish calico/rust-build images - pipeline_file: promotions/calico-rust-build.yml - auto_promote: - when: "branch = 'master' AND change_in('/images/calico-rust-build/')" - # Publish tinygo images for master and files under images/calico-tinygo have changed. - - name: Publish calico/tinygo images - pipeline_file: promotions/calico-tinygo.yml - auto_promote: - when: "branch = 'master' AND change_in('/images/calico-tinygo/')" - + - touch ~/publish.env +# Block fragments live in semaphore.yml.d/blocks and are appended here in +# filename order. The numeric prefix groups blocks by what the image is, which +# is durable, rather than by how it currently publishes, which is not: +# +# 00 repository checks - builds nothing, and every image block waits on it +# 10 compiler toolchains, the images this repo exists to produce +# 20 runtime base, shipped inside Calico components +# 30 multi-arch CI helper +# +# The numbers are not build order. No toolchain image is built FROM another; +# all five build in parallel once Checks has passed. blocks: - - name: calico/go-build image + # Repository checks + # + # Every image block depends on this one, so a failure here stops the builds + # before any QEMU-emulated build starts or anything is published. The block is + # change-gated and often skips; Semaphore counts a skipped dependency as + # passed, and this block produces no artifacts, so skipping it is safe. + - name: Checks dependencies: [] + run: + when: "change_in(['/.semaphore/', '/hack/', '/Makefile'], {pipeline_file: 'track'})" + task: + jobs: + - name: Check generated semaphore.yml + commands: + - make check-semaphore-yaml + - name: Shellcheck + commands: + - command -v shellcheck >/dev/null || sudo apt-get install -y shellcheck + - shellcheck hack/*.sh + # calico/go-build + - name: calico/go-build image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-go-build/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" task: env_vars: # The branch to test the current go-build against @@ -59,14 +87,9 @@ blocks: value: master prologue: commands: - - | - if [ "${SEMAPHORE_GIT_REF_TYPE}" = "tag" ]; then - export CALICO_GO_BUILD_IMAGETAG=${SEMAPHORE_GIT_TAG_NAME} - else - export CALICO_GO_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} - fi + - export CALICO_GO_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} jobs: - - name: Build calico/go-build image + - name: Build and test calico/go-build image commands: - make -C images calico-go-build-image ARCH=$ARCH - git clone -b "${CALICO_BRANCH}" --depth 1 git@github.com:projectcalico/calico.git calico @@ -77,17 +100,90 @@ blocks: - env_var: ARCH values: ["amd64", "arm64", "ppc64le", "s390x"] + - name: calico/go-build image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-go-build/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + - name: CALICO_BRANCH + value: master + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - export CALICO_GO_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + jobs: + - name: Build, test and publish calico/go-build image + commands: + - make -C images calico-go-build-image ARCH=$ARCH + - git clone -b "${CALICO_BRANCH}" --depth 1 git@github.com:projectcalico/calico.git calico + - sed -i 's/^GO_BUILD_VER=.*$/GO_BUILD_VER=${CALICO_GO_BUILD_IMAGETAG}/' calico/metadata.mk + - if [ "${ARCH}" == "amd64" ]; then ( cd calico/felix && make ut ); fi + # The release tag is created by create-tag-on-version-change.yml + # within a minute of the push; this job reaches here ~20 minutes + # later. Fetch now rather than in the prologue so the tag is visible. + - git fetch --tags --force + - | + export RELEASE_TAG=$(git tag --points-at HEAD | grep -E '^[0-9]+\.[0-9]+\.[0-9]+-llvm' | sort -V | tail -1) + if [ -z "${RELEASE_TAG}" ] && [ "${SEMAPHORE_GIT_WORKING_BRANCH}" != "master" ]; then + echo >&2 "FAIL: no release tag points at HEAD on ${SEMAPHORE_GIT_WORKING_BRANCH}." + echo >&2 " create-tag-on-version-change.yml should have created one." + exit 1 + fi + echo "RELEASE_TAG=${RELEASE_TAG}" >> ~/publish.env + # Publish only after the tests above have passed. + - make -C images calico-go-build-push VALIDARCHES=$ARCH ARCH=$ARCH BRANCH_NAME=${SEMAPHORE_GIT_WORKING_BRANCH} CONFIRM=true + - | + . ~/publish.env + if [ -n "${RELEASE_TAG}" ]; then + make -C images calico-go-build-push VALIDARCHES=$ARCH ARCH=$ARCH BRANCH_NAME=${RELEASE_TAG} CONFIRM=true + fi + matrix: + - env_var: ARCH + values: ["amd64", "arm64", "ppc64le", "s390x"] + + - name: calico/go-build manifests + dependencies: + - calico/go-build image and publish + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-go-build/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + - name: semaphore-api + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Push calico/go-build multi-arch manifests + commands: + - git fetch --tags --force + - make -C images push-calico-go-build-manifests BRANCH_NAME=${SEMAPHORE_GIT_WORKING_BRANCH} CONFIRM=true + - | + RELEASE_TAG=$(git tag --points-at HEAD | grep -E '^[0-9]+\.[0-9]+\.[0-9]+-llvm' | sort -V | tail -1) + if [ -n "${RELEASE_TAG}" ]; then + make -C images push-calico-go-build-manifests BRANCH_NAME=${RELEASE_TAG} CONFIRM=true + # Tell downstream repos to pin the tag we just published. + make update-go-build-pins CONFIRM=true + fi + # calico/rust-build - name: calico/rust-build image - dependencies: [] + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-rust-build/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" task: prologue: commands: - - | - if [ "${SEMAPHORE_GIT_REF_TYPE}" = "tag" ]; then - export CALICO_RUST_BUILD_IMAGETAG=${SEMAPHORE_GIT_TAG_NAME} - else - export CALICO_RUST_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} - fi + - export CALICO_RUST_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} jobs: - name: Build calico/rust-build image commands: @@ -96,36 +192,60 @@ blocks: - env_var: ARCH values: ["amd64", "arm64", "ppc64le", "s390x"] - - name: calico/base image - dependencies: [] + - name: calico/rust-build image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-rust-build/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - export CALICO_RUST_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + # rust-build is versioned by its own toolchain version, not by the branch. + - export RUST_VERSION=$(yq -r '.rust.version' images/calico-rust-build/versions.yaml) jobs: - - name: Build calico/base image + - name: Build and publish calico/rust-build image commands: - - make -C images calico-base-image ARCH=$ARCH + - make -C images calico-rust-build-image ARCH=$ARCH + - make -C images calico-rust-build-push VALIDARCHES=$ARCH ARCH=$ARCH BRANCH_NAME=${RUST_VERSION} CONFIRM=true matrix: - env_var: ARCH values: ["amd64", "arm64", "ppc64le", "s390x"] - - name: calico/binfmt image - dependencies: [] + - name: calico/rust-build manifests + dependencies: + - calico/rust-build image and publish + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-rust-build/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin jobs: - - name: Build calico/binfmt amd64 image + - name: Push calico/rust-build multi-arch manifests commands: - - make -C images calico-binfmt-image ARCH=amd64 - + - make -C images push-calico-rust-build-manifests BRANCH_NAME=$(yq -r '.rust.version' images/calico-rust-build/versions.yaml) CONFIRM=true + # calico/tinygo - name: calico/tinygo image - dependencies: [] + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-tinygo/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" task: prologue: commands: - - | - if [ "${SEMAPHORE_GIT_REF_TYPE}" = "tag" ]; then - export CALICO_TINYGO_IMAGETAG=${SEMAPHORE_GIT_TAG_NAME} - else - export CALICO_TINYGO_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} - fi + - export CALICO_TINYGO_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} jobs: - name: Build and test calico/tinygo commands: @@ -186,3 +306,170 @@ blocks: matrix: - env_var: ARCH values: ["amd64", "arm64"] + + - name: calico/tinygo image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-tinygo/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - export CALICO_TINYGO_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + # tinygo is versioned by its own toolchain version, not by the branch. + - export TINYGO_VERSION=$(yq -r '.tinygo.version' images/calico-tinygo/versions.yaml) + jobs: + - name: Build, test and publish calico/tinygo + commands: + - make -C images calico-tinygo-image ARCH=$ARCH + # Compile the test fixture against the just-built image, then + # inspect the produced wasm's import section for two failure + # classes a downstream consumer (gateway/coraza-wasm) would + # otherwise hit: + # + # 1. wasm-ld undefined symbols at link time. The fixture + # imports github.com/wasilibs/{go-re2, go-libinjection} + # and uses each via os.Args-derived input so TinyGo's + # `-opt=2` optimizer cannot prove the calls dead. If a + # TinyGo or wasi-libc bump breaks the prebuilt-archive + # ABI, the build fails here. + # + # 2. Runtime `missing import: env.cre2_new` on Envoy. The + # assertion below counts `<- env.cre2_*` imports in the + # wasm; with the TinyGo-compatible go-re2 (currently + # v1.6.0) cre2 resolves internally to the bundled + # static archive and the count is zero. wasilibs/go-re2 + # v1.8.0+ removed TinyGo support (libre2 now needs + # Abseil with thread sync, which TinyGo's libc++ shim + # does not provide); a bump to such a version produces + # external env.cre2_* imports and the assertion fails + # here, at toolchain-image build time, not during a + # downstream WAF rebuild. + # + # The Makefile builds the image as `tinygo:latest-amd64` + # locally before retagging it as `calico/tinygo:${branch}-amd64` + # for the registry. Use the local tag here — the calico/- + # prefixed tag uses the branch, not "latest". + - | + docker run --rm \ + -v "$(pwd)/images/calico-tinygo/test/fixture:/src" \ + -v /tmp:/out \ + -w /src \ + tinygo:latest-${ARCH} \ + sh -c 'go mod download && \ + tinygo build \ + -target=wasip1 \ + -gc=precise -opt=2 -scheduler=none \ + -tags="nottinygc_envoy memoize_builders" \ + -o /out/fixture.wasm . && \ + file /out/fixture.wasm | grep -q "WebAssembly"' + - | + count=$(docker run --rm -v /tmp:/host ghcr.io/webassembly/wabt:1.0.36 \ + wasm-objdump -x /host/fixture.wasm | grep -c "<- env.cre2_" || true) + if [ "${count:-0}" -ne 0 ]; then + echo >&2 "FAIL: fixture wasm has ${count} env.cre2_* imports (expected 0)" + echo >&2 " A wasilibs/go-re2 bump likely lost TinyGo's bundled libcre2.a;" + echo >&2 " this wasm would fail to load in Envoy with 'missing import: env.cre2_new'." + docker run --rm -v /tmp:/host ghcr.io/webassembly/wabt:1.0.36 \ + wasm-objdump -x /host/fixture.wasm | grep "<- env.cre2_" >&2 || true + exit 1 + fi + echo "OK: fixture wasm resolves cre2_* internally (no env.cre2_* imports)" + - make -C images calico-tinygo-push ARCH=$ARCH VALIDARCHES=$ARCH BRANCH_NAME=${TINYGO_VERSION} CONFIRM=true + matrix: + - env_var: ARCH + values: ["amd64", "arm64"] + + - name: calico/tinygo manifests + dependencies: + - calico/tinygo image and publish + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-tinygo/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Push calico/tinygo multi-arch manifest + commands: + - make -C images push-calico-tinygo-manifests BRANCH_NAME=$(yq -r '.tinygo.version' images/calico-tinygo/versions.yaml) CONFIRM=true + # calico/base + - name: calico/base image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-base/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + jobs: + - name: Build calico/base image + commands: + - make -C images calico-base-image ARCH=$ARCH + matrix: + - env_var: ARCH + values: ["amd64", "arm64", "ppc64le", "s390x"] + + - name: calico/base image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-base/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + # Every architecture and both UBI versions must share one tag, so base + # builds and publishes in a single job rather than a per-arch matrix. + - name: BRANCH_NAME + value: ${SEMAPHORE_JOB_CREATION_TIME} + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Build and publish calico/base multi-arch images + commands: + - make -C images calico-base-image-all + - make -C images calico-base-push CONFIRM=true + # calico/binfmt + - name: calico/binfmt image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-binfmt/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + jobs: + - name: Build calico/binfmt amd64 image + commands: + - make -C images calico-binfmt-image ARCH=amd64 + + - name: calico/binfmt image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-binfmt/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Build and publish calico/binfmt amd64 image + commands: + - make -C images calico-binfmt-image ARCH=amd64 + - make -C images calico-binfmt-push CONFIRM=true diff --git a/.semaphore/semaphore.yml.d/01-preamble.yml b/.semaphore/semaphore.yml.d/01-preamble.yml new file mode 100644 index 00000000..c2cdef79 --- /dev/null +++ b/.semaphore/semaphore.yml.d/01-preamble.yml @@ -0,0 +1,31 @@ +version: v1.0 +name: go-build +agent: + machine: + type: f1-standard-2 + os_image: ubuntu2204 +auto_cancel: + running: + when: "branch != 'master'" + queued: + when: "branch != 'master'" + +# A publish job builds, tests and pushes in one go, where the build and the +# push used to have a 60-minute budget each in separate pipelines. The +# ppc64le and s390x builds run under QEMU emulation and are the long pole. +execution_time_limit: + minutes: 120 + +# Each image has two blocks: one for pull requests that builds and tests, and +# one for master and release branches that builds, tests and publishes in a +# single job. Exactly one of the pair runs per event, so an image is built once +# per commit instead of once per pipeline, and the push happens on the machine +# that already holds the image. +# +# Only the publish blocks carry the `docker` push secret. Semaphore evaluates +# `run: when:` before scheduling a block, so that secret is never injected into +# a pull request run, forked or otherwise. +# +# Git tags do not build. A push to a release branch publishes both the branch +# tag and the release tag that create-tag-on-version-change.yml just created, +# so the tag pipeline has nothing left to do and every block below skips it. diff --git a/.semaphore/semaphore.yml.d/02-global_job_config.yml b/.semaphore/semaphore.yml.d/02-global_job_config.yml new file mode 100644 index 00000000..ac3c933c --- /dev/null +++ b/.semaphore/semaphore.yml.d/02-global_job_config.yml @@ -0,0 +1,11 @@ +global_job_config: + secrets: + - name: docker-hub + prologue: + commands: + - echo $DOCKERHUB_PASSWORD | docker login --username "$DOCKERHUB_USERNAME" --password-stdin + - checkout + # Semaphore is doing shallow clone on a commit without tags. + # unshallow it for GIT_VERSION:=$(shell git describe --tags --dirty --always) @ Makefile.common + - git fetch --unshallow + - touch ~/publish.env diff --git a/.semaphore/semaphore.yml.d/09-blocks.yml b/.semaphore/semaphore.yml.d/09-blocks.yml new file mode 100644 index 00000000..13677f2c --- /dev/null +++ b/.semaphore/semaphore.yml.d/09-blocks.yml @@ -0,0 +1,12 @@ +# Block fragments live in semaphore.yml.d/blocks and are appended here in +# filename order. The numeric prefix groups blocks by what the image is, which +# is durable, rather than by how it currently publishes, which is not: +# +# 00 repository checks - builds nothing, and every image block waits on it +# 10 compiler toolchains, the images this repo exists to produce +# 20 runtime base, shipped inside Calico components +# 30 multi-arch CI helper +# +# The numbers are not build order. No toolchain image is built FROM another; +# all five build in parallel once Checks has passed. +blocks: diff --git a/.semaphore/semaphore.yml.d/blocks/00-checks.yml b/.semaphore/semaphore.yml.d/blocks/00-checks.yml new file mode 100644 index 00000000..c0366ae3 --- /dev/null +++ b/.semaphore/semaphore.yml.d/blocks/00-checks.yml @@ -0,0 +1,19 @@ +# Repository checks +# +# Every image block depends on this one, so a failure here stops the builds +# before any QEMU-emulated build starts or anything is published. The block is +# change-gated and often skips; Semaphore counts a skipped dependency as +# passed, and this block produces no artifacts, so skipping it is safe. +- name: Checks + dependencies: [] + run: + when: "change_in(['/.semaphore/', '/hack/', '/Makefile'], {pipeline_file: 'track'})" + task: + jobs: + - name: Check generated semaphore.yml + commands: + - make check-semaphore-yaml + - name: Shellcheck + commands: + - command -v shellcheck >/dev/null || sudo apt-get install -y shellcheck + - shellcheck hack/*.sh diff --git a/.semaphore/semaphore.yml.d/blocks/10-go-build.yml b/.semaphore/semaphore.yml.d/blocks/10-go-build.yml new file mode 100644 index 00000000..fe734fa9 --- /dev/null +++ b/.semaphore/semaphore.yml.d/blocks/10-go-build.yml @@ -0,0 +1,100 @@ +# calico/go-build +- name: calico/go-build image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-go-build/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + env_vars: + # The branch to test the current go-build against + - name: CALICO_BRANCH + value: master + prologue: + commands: + - export CALICO_GO_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + jobs: + - name: Build and test calico/go-build image + commands: + - make -C images calico-go-build-image ARCH=$ARCH + - git clone -b "${CALICO_BRANCH}" --depth 1 git@github.com:projectcalico/calico.git calico + - cd calico + - sed -i 's/^GO_BUILD_VER=.*$/GO_BUILD_VER=${CALICO_GO_BUILD_IMAGETAG}/' metadata.mk + - if [ "${ARCH}" == "amd64" ]; then cd felix && make ut; fi + matrix: + - env_var: ARCH + values: ["amd64", "arm64", "ppc64le", "s390x"] + +- name: calico/go-build image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-go-build/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + - name: CALICO_BRANCH + value: master + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - export CALICO_GO_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + jobs: + - name: Build, test and publish calico/go-build image + commands: + - make -C images calico-go-build-image ARCH=$ARCH + - git clone -b "${CALICO_BRANCH}" --depth 1 git@github.com:projectcalico/calico.git calico + - sed -i 's/^GO_BUILD_VER=.*$/GO_BUILD_VER=${CALICO_GO_BUILD_IMAGETAG}/' calico/metadata.mk + - if [ "${ARCH}" == "amd64" ]; then ( cd calico/felix && make ut ); fi + # The release tag is created by create-tag-on-version-change.yml + # within a minute of the push; this job reaches here ~20 minutes + # later. Fetch now rather than in the prologue so the tag is visible. + - git fetch --tags --force + - | + export RELEASE_TAG=$(git tag --points-at HEAD | grep -E '^[0-9]+\.[0-9]+\.[0-9]+-llvm' | sort -V | tail -1) + if [ -z "${RELEASE_TAG}" ] && [ "${SEMAPHORE_GIT_WORKING_BRANCH}" != "master" ]; then + echo >&2 "FAIL: no release tag points at HEAD on ${SEMAPHORE_GIT_WORKING_BRANCH}." + echo >&2 " create-tag-on-version-change.yml should have created one." + exit 1 + fi + echo "RELEASE_TAG=${RELEASE_TAG}" >> ~/publish.env + # Publish only after the tests above have passed. + - make -C images calico-go-build-push VALIDARCHES=$ARCH ARCH=$ARCH BRANCH_NAME=${SEMAPHORE_GIT_WORKING_BRANCH} CONFIRM=true + - | + . ~/publish.env + if [ -n "${RELEASE_TAG}" ]; then + make -C images calico-go-build-push VALIDARCHES=$ARCH ARCH=$ARCH BRANCH_NAME=${RELEASE_TAG} CONFIRM=true + fi + matrix: + - env_var: ARCH + values: ["amd64", "arm64", "ppc64le", "s390x"] + +- name: calico/go-build manifests + dependencies: + - calico/go-build image and publish + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-go-build/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + - name: semaphore-api + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Push calico/go-build multi-arch manifests + commands: + - git fetch --tags --force + - make -C images push-calico-go-build-manifests BRANCH_NAME=${SEMAPHORE_GIT_WORKING_BRANCH} CONFIRM=true + - | + RELEASE_TAG=$(git tag --points-at HEAD | grep -E '^[0-9]+\.[0-9]+\.[0-9]+-llvm' | sort -V | tail -1) + if [ -n "${RELEASE_TAG}" ]; then + make -C images push-calico-go-build-manifests BRANCH_NAME=${RELEASE_TAG} CONFIRM=true + # Tell downstream repos to pin the tag we just published. + make update-go-build-pins CONFIRM=true + fi diff --git a/.semaphore/semaphore.yml.d/blocks/10-rust-build.yml b/.semaphore/semaphore.yml.d/blocks/10-rust-build.yml new file mode 100644 index 00000000..025c1673 --- /dev/null +++ b/.semaphore/semaphore.yml.d/blocks/10-rust-build.yml @@ -0,0 +1,62 @@ +# calico/rust-build +- name: calico/rust-build image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-rust-build/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + prologue: + commands: + - export CALICO_RUST_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + jobs: + - name: Build calico/rust-build image + commands: + - make -C images calico-rust-build-image ARCH=$ARCH + matrix: + - env_var: ARCH + values: ["amd64", "arm64", "ppc64le", "s390x"] + +- name: calico/rust-build image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-rust-build/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - export CALICO_RUST_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + # rust-build is versioned by its own toolchain version, not by the branch. + - export RUST_VERSION=$(yq -r '.rust.version' images/calico-rust-build/versions.yaml) + jobs: + - name: Build and publish calico/rust-build image + commands: + - make -C images calico-rust-build-image ARCH=$ARCH + - make -C images calico-rust-build-push VALIDARCHES=$ARCH ARCH=$ARCH BRANCH_NAME=${RUST_VERSION} CONFIRM=true + matrix: + - env_var: ARCH + values: ["amd64", "arm64", "ppc64le", "s390x"] + +- name: calico/rust-build manifests + dependencies: + - calico/rust-build image and publish + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-rust-build/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Push calico/rust-build multi-arch manifests + commands: + - make -C images push-calico-rust-build-manifests BRANCH_NAME=$(yq -r '.rust.version' images/calico-rust-build/versions.yaml) CONFIRM=true diff --git a/.semaphore/semaphore.yml.d/blocks/10-tinygo.yml b/.semaphore/semaphore.yml.d/blocks/10-tinygo.yml new file mode 100644 index 00000000..1762495b --- /dev/null +++ b/.semaphore/semaphore.yml.d/blocks/10-tinygo.yml @@ -0,0 +1,168 @@ +# calico/tinygo +- name: calico/tinygo image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-tinygo/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + prologue: + commands: + - export CALICO_TINYGO_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + jobs: + - name: Build and test calico/tinygo + commands: + - make -C images calico-tinygo-image ARCH=$ARCH + # Compile the test fixture against the just-built image, then + # inspect the produced wasm's import section for two failure + # classes a downstream consumer (gateway/coraza-wasm) would + # otherwise hit: + # + # 1. wasm-ld undefined symbols at link time. The fixture + # imports github.com/wasilibs/{go-re2, go-libinjection} + # and uses each via os.Args-derived input so TinyGo's + # `-opt=2` optimizer cannot prove the calls dead. If a + # TinyGo or wasi-libc bump breaks the prebuilt-archive + # ABI, the build fails here. + # + # 2. Runtime `missing import: env.cre2_new` on Envoy. The + # assertion below counts `<- env.cre2_*` imports in the + # wasm; with the TinyGo-compatible go-re2 (currently + # v1.6.0) cre2 resolves internally to the bundled + # static archive and the count is zero. wasilibs/go-re2 + # v1.8.0+ removed TinyGo support (libre2 now needs + # Abseil with thread sync, which TinyGo's libc++ shim + # does not provide); a bump to such a version produces + # external env.cre2_* imports and the assertion fails + # here, at toolchain-image build time, not during a + # downstream WAF rebuild. + # + # The Makefile builds the image as `tinygo:latest-amd64` + # locally before retagging it as `calico/tinygo:${branch}-amd64` + # for the registry. Use the local tag here — the calico/- + # prefixed tag uses the branch, not "latest". + - | + docker run --rm \ + -v "$(pwd)/images/calico-tinygo/test/fixture:/src" \ + -v /tmp:/out \ + -w /src \ + tinygo:latest-${ARCH} \ + sh -c 'go mod download && \ + tinygo build \ + -target=wasip1 \ + -gc=precise -opt=2 -scheduler=none \ + -tags="nottinygc_envoy memoize_builders" \ + -o /out/fixture.wasm . && \ + file /out/fixture.wasm | grep -q "WebAssembly"' + - | + count=$(docker run --rm -v /tmp:/host ghcr.io/webassembly/wabt:1.0.36 \ + wasm-objdump -x /host/fixture.wasm | grep -c "<- env.cre2_" || true) + if [ "${count:-0}" -ne 0 ]; then + echo >&2 "FAIL: fixture wasm has ${count} env.cre2_* imports (expected 0)" + echo >&2 " A wasilibs/go-re2 bump likely lost TinyGo's bundled libcre2.a;" + echo >&2 " this wasm would fail to load in Envoy with 'missing import: env.cre2_new'." + docker run --rm -v /tmp:/host ghcr.io/webassembly/wabt:1.0.36 \ + wasm-objdump -x /host/fixture.wasm | grep "<- env.cre2_" >&2 || true + exit 1 + fi + echo "OK: fixture wasm resolves cre2_* internally (no env.cre2_* imports)" + matrix: + - env_var: ARCH + values: ["amd64", "arm64"] + +- name: calico/tinygo image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-tinygo/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - export CALICO_TINYGO_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + # tinygo is versioned by its own toolchain version, not by the branch. + - export TINYGO_VERSION=$(yq -r '.tinygo.version' images/calico-tinygo/versions.yaml) + jobs: + - name: Build, test and publish calico/tinygo + commands: + - make -C images calico-tinygo-image ARCH=$ARCH + # Compile the test fixture against the just-built image, then + # inspect the produced wasm's import section for two failure + # classes a downstream consumer (gateway/coraza-wasm) would + # otherwise hit: + # + # 1. wasm-ld undefined symbols at link time. The fixture + # imports github.com/wasilibs/{go-re2, go-libinjection} + # and uses each via os.Args-derived input so TinyGo's + # `-opt=2` optimizer cannot prove the calls dead. If a + # TinyGo or wasi-libc bump breaks the prebuilt-archive + # ABI, the build fails here. + # + # 2. Runtime `missing import: env.cre2_new` on Envoy. The + # assertion below counts `<- env.cre2_*` imports in the + # wasm; with the TinyGo-compatible go-re2 (currently + # v1.6.0) cre2 resolves internally to the bundled + # static archive and the count is zero. wasilibs/go-re2 + # v1.8.0+ removed TinyGo support (libre2 now needs + # Abseil with thread sync, which TinyGo's libc++ shim + # does not provide); a bump to such a version produces + # external env.cre2_* imports and the assertion fails + # here, at toolchain-image build time, not during a + # downstream WAF rebuild. + # + # The Makefile builds the image as `tinygo:latest-amd64` + # locally before retagging it as `calico/tinygo:${branch}-amd64` + # for the registry. Use the local tag here — the calico/- + # prefixed tag uses the branch, not "latest". + - | + docker run --rm \ + -v "$(pwd)/images/calico-tinygo/test/fixture:/src" \ + -v /tmp:/out \ + -w /src \ + tinygo:latest-${ARCH} \ + sh -c 'go mod download && \ + tinygo build \ + -target=wasip1 \ + -gc=precise -opt=2 -scheduler=none \ + -tags="nottinygc_envoy memoize_builders" \ + -o /out/fixture.wasm . && \ + file /out/fixture.wasm | grep -q "WebAssembly"' + - | + count=$(docker run --rm -v /tmp:/host ghcr.io/webassembly/wabt:1.0.36 \ + wasm-objdump -x /host/fixture.wasm | grep -c "<- env.cre2_" || true) + if [ "${count:-0}" -ne 0 ]; then + echo >&2 "FAIL: fixture wasm has ${count} env.cre2_* imports (expected 0)" + echo >&2 " A wasilibs/go-re2 bump likely lost TinyGo's bundled libcre2.a;" + echo >&2 " this wasm would fail to load in Envoy with 'missing import: env.cre2_new'." + docker run --rm -v /tmp:/host ghcr.io/webassembly/wabt:1.0.36 \ + wasm-objdump -x /host/fixture.wasm | grep "<- env.cre2_" >&2 || true + exit 1 + fi + echo "OK: fixture wasm resolves cre2_* internally (no env.cre2_* imports)" + - make -C images calico-tinygo-push ARCH=$ARCH VALIDARCHES=$ARCH BRANCH_NAME=${TINYGO_VERSION} CONFIRM=true + matrix: + - env_var: ARCH + values: ["amd64", "arm64"] + +- name: calico/tinygo manifests + dependencies: + - calico/tinygo image and publish + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-tinygo/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Push calico/tinygo multi-arch manifest + commands: + - make -C images push-calico-tinygo-manifests BRANCH_NAME=$(yq -r '.tinygo.version' images/calico-tinygo/versions.yaml) CONFIRM=true diff --git a/.semaphore/semaphore.yml.d/blocks/20-base.yml b/.semaphore/semaphore.yml.d/blocks/20-base.yml new file mode 100644 index 00000000..4d19d3f2 --- /dev/null +++ b/.semaphore/semaphore.yml.d/blocks/20-base.yml @@ -0,0 +1,38 @@ +# calico/base +- name: calico/base image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-base/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + jobs: + - name: Build calico/base image + commands: + - make -C images calico-base-image ARCH=$ARCH + matrix: + - env_var: ARCH + values: ["amd64", "arm64", "ppc64le", "s390x"] + +- name: calico/base image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-base/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + # Every architecture and both UBI versions must share one tag, so base + # builds and publishes in a single job rather than a per-arch matrix. + - name: BRANCH_NAME + value: ${SEMAPHORE_JOB_CREATION_TIME} + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Build and publish calico/base multi-arch images + commands: + - make -C images calico-base-image-all + - make -C images calico-base-push CONFIRM=true diff --git a/.semaphore/semaphore.yml.d/blocks/30-binfmt.yml b/.semaphore/semaphore.yml.d/blocks/30-binfmt.yml new file mode 100644 index 00000000..cbcbf7ec --- /dev/null +++ b/.semaphore/semaphore.yml.d/blocks/30-binfmt.yml @@ -0,0 +1,31 @@ +# calico/binfmt +- name: calico/binfmt image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-binfmt/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + jobs: + - name: Build calico/binfmt amd64 image + commands: + - make -C images calico-binfmt-image ARCH=amd64 + +- name: calico/binfmt image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-binfmt/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Build and publish calico/binfmt amd64 image + commands: + - make -C images calico-binfmt-image ARCH=amd64 + - make -C images calico-binfmt-push CONFIRM=true diff --git a/Makefile b/Makefile index 9f03f142..a0432ac2 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,19 @@ clean: $(MAKE) -C cmd clean $(MAKE) -C images clean +# The Semaphore pipeline is assembled from fragments so each image's blocks can +# be edited on their own. Run this after changing anything under +# .semaphore/semaphore.yml.d and commit the regenerated file. +.PHONY: gen-semaphore-yaml +gen-semaphore-yaml: + hack/gen-semaphore-yaml.sh + +# check-semaphore-yaml fails when the committed pipeline does not match its +# fragments, which is how an edit to the generated file gets caught. +.PHONY: check-semaphore-yaml +check-semaphore-yaml: + hack/gen-semaphore-yaml.sh --check + .PHONY: update-go-build-pins update-go-build-pins: SEMAPHORE_AUTO_PIN_UPDATE_PROJECT_IDS=$(SEMAPHORE_CALICO_PROJECT_ID) \ diff --git a/hack/gen-semaphore-yaml.sh b/hack/gen-semaphore-yaml.sh new file mode 100755 index 00000000..5183075a --- /dev/null +++ b/hack/gen-semaphore-yaml.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Assemble .semaphore/semaphore.yml from the fragments in +# .semaphore/semaphore.yml.d, the way projectcalico/calico does. +# +# Top-level fragments are concatenated in filename order, so the numeric +# prefixes set the order. When 09-blocks.yml is reached (it contains just the +# `blocks:` key) every file under semaphore.yml.d/blocks/ follows it, also in +# filename order, indented by two spaces. Block fragments are written at column +# zero so each one is valid YAML on its own and an editor can lint it. + +set -eu + +# --check reports whether the committed file matches its fragments instead of +# rewriting it. It compares generated output against the file on disk, so it is +# independent of what git happens to have staged. +check=false +if [ "${1:-}" = "--check" ]; then + check=true +fi + +cd "$(git rev-parse --show-toplevel)" + +d=.semaphore/semaphore.yml.d +out=.semaphore/semaphore.yml +tmp=$(mktemp) +trap 'rm -f "$tmp"' EXIT + +cat >"$tmp" <<'HEADER' +# !! WARNING, DO NOT EDIT !! This file is generated from the fragments +# in /.semaphore/semaphore.yml.d. To update, modify the relevant +# fragment and then run 'make gen-semaphore-yaml'. +HEADER + +for f in "$d"/*.yml; do + cat "$f" >>"$tmp" + if [ "$(basename "$f")" = "09-blocks.yml" ]; then + for b in "$d"/blocks/*.yml; do + # Indent to sit under `blocks:`, leaving blank lines empty rather + # than filling them with trailing spaces. + sed 's/^\(.\)/ \1/' "$b" >>"$tmp" + done + fi +done + +# A fragment with a typo must not land in the tree. +if command -v python3 >/dev/null 2>&1; then + python3 -c 'import sys,yaml; yaml.safe_load(open(sys.argv[1]))' "$tmp" +fi + +if [ "$check" = true ]; then + if ! diff -u "$out" "$tmp"; then + echo >&2 "ERROR: $out does not match its fragments. Run 'make gen-semaphore-yaml'." + exit 1 + fi + echo "$out is up to date" + exit 0 +fi + +mv "$tmp" "$out" +trap - EXIT +echo "generated $out" diff --git a/images/Makefile b/images/Makefile index 1a54d6f5..6ff82e66 100644 --- a/images/Makefile +++ b/images/Makefile @@ -52,7 +52,14 @@ sub-calico-base-image-%: $(MAKE) calico-base-image ARCH=$* .PHONY: calico-base-cd -calico-base-cd: calico-base-image-all var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME +calico-base-cd: calico-base-image-all calico-base-push + +# calico-base-push publishes images that are already in the local daemon. CI +# builds and publishes in one job, so a push never rebuilds. All architectures +# and both UBI versions share a single BRANCH_NAME, so they are pushed together +# rather than per-arch. +.PHONY: calico-base-push +calico-base-push: var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME $(foreach version,$(UBI_VERSIONS),$(MAKE) BUILD_IMAGES=$(CALICO_BASE) retag-build-images-with-registries push-images-to-registries push-manifests LATEST_IMAGE_TAG=$(version)-latest IMAGETAG=$(version)-$(BRANCH_NAME) EXCLUDEARCH="$(EXCLUDEARCH)";) ################################################################################ @@ -64,6 +71,12 @@ CALICO_GO_BUILD_IMAGETAG ?= latest .PHONY: calico-go-build-image calico-go-build-image: register build $(DOCKER_BUILD) -t $(CALICO_GO_BUILD):latest-$(ARCH) -f calico-go-build/Dockerfile calico-go-build/ + $(MAKE) calico-go-build-retag + +# calico-go-build-retag applies the registry tags to an image that is already in +# the local daemon, whether it was just built or loaded from the image cache. +.PHONY: calico-go-build-retag +calico-go-build-retag: $(MAKE) BUILD_IMAGES=$(CALICO_GO_BUILD) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=$(CALICO_GO_BUILD_IMAGETAG) .PHONY: calico-go-build-image-all @@ -72,7 +85,11 @@ sub-calico-go-build-image-%: $(MAKE) calico-go-build-image ARCH=$* .PHONY: calico-go-build-cd -calico-go-build-cd: calico-go-build-image var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME +calico-go-build-cd: calico-go-build-image calico-go-build-push + +# calico-go-build-push publishes an image that is already in the local daemon. +.PHONY: calico-go-build-push +calico-go-build-push: var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME $(MAKE) BUILD_IMAGES=$(CALICO_GO_BUILD) retag-build-images-with-registries push-images-to-registries IMAGETAG=$(BRANCH_NAME) EXCLUDEARCH="$(EXCLUDEARCH)" ifeq ($(BRANCH_NAME),master) $(MAKE) BUILD_IMAGES=$(CALICO_GO_BUILD) retag-build-images-with-registries push-images-to-registries IMAGETAG=latest EXCLUDEARCH="$(EXCLUDEARCH)" @@ -100,7 +117,11 @@ calico-binfmt-image: build $(MAKE) BUILD_IMAGES=$(CALICO_BINFMT) retag-build-images-with-registries VALIDARCHES=amd64 LATEST_IMAGE_TAG=$(CALICO_BINFMT_IMAGETAG) IMAGETAG=$(CALICO_BINFMT_IMAGETAG) .PHONY: calico-binfmt-cd -calico-binfmt-cd: calico-binfmt-image var-require-one-of-CONFIRM-DRYRUN +calico-binfmt-cd: calico-binfmt-image calico-binfmt-push + +# calico-binfmt-push publishes an image that is already in the local daemon. +.PHONY: calico-binfmt-push +calico-binfmt-push: var-require-one-of-CONFIRM-DRYRUN $(MAKE) BUILD_IMAGES=$(CALICO_BINFMT) retag-build-images-with-registries push-images-to-registries IMAGETAG=$(CALICO_BINFMT_IMAGETAG) LATEST_IMAGE_TAG=$(CALICO_BINFMT_IMAGETAG) VALIDARCHES=amd64 EXCLUDEARCH="$(EXCLUDEARCH)" ################################################################################ @@ -120,7 +141,11 @@ sub-calico-rust-build-image-%: $(MAKE) calico-rust-build-image ARCH=$* .PHONY: calico-rust-build-cd -calico-rust-build-cd: calico-rust-build-image var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME +calico-rust-build-cd: calico-rust-build-image calico-rust-build-push + +# calico-rust-build-push publishes an image that is already in the local daemon. +.PHONY: calico-rust-build-push +calico-rust-build-push: var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME $(MAKE) BUILD_IMAGES=$(CALICO_RUST_BUILD) retag-build-images-with-registries push-images-to-registries IMAGETAG=$(BRANCH_NAME) EXCLUDEARCH="$(EXCLUDEARCH)" .PHONY: push-calico-rust-build-manifests @@ -145,7 +170,11 @@ sub-calico-tinygo-image-%: $(MAKE) calico-tinygo-image ARCH=$* .PHONY: calico-tinygo-cd -calico-tinygo-cd: calico-tinygo-image var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME +calico-tinygo-cd: calico-tinygo-image calico-tinygo-push + +# calico-tinygo-push publishes an image that is already in the local daemon. +.PHONY: calico-tinygo-push +calico-tinygo-push: var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME $(MAKE) BUILD_IMAGES=$(CALICO_TINYGO) retag-build-images-with-registries push-images-to-registries IMAGETAG=$(BRANCH_NAME) VALIDARCHES=$(ARCH) .PHONY: push-calico-tinygo-manifests From f89d5602aa2c309c9707266d596c2008a9a06f93 Mon Sep 17 00:00:00 2001 From: Jiawei Huang Date: Tue, 25 Aug 2026 12:22:39 -0700 Subject: [PATCH 2/2] Reuse built images from a GCS tarball cache After the previous commit a release-branch bump builds each image twice: once in the pull request, once again on the merge. Nothing carries an image between the two, because Semaphore jobs each start on a clean VM. Cache every built image as a zstd tarball in GCS so a merge reuses what its pull request built, which matters most for arm64, ppc64le and s390x: those build under QEMU emulation. An entry is keyed by two things, because the source tree alone does not determine the image: 1. A hash of the committed files that can change that image, so a pull request and the merge that follows it share an entry. 2. The digests of the base images it is built FROM. Those are floating tags, and a new release means a rebuild would produce a different image. UBI is rebuilt roughly daily; almalinux:9 and fedora:44 roughly quarterly. That covers new base releases but not everything, because dnf and microdnf pull from repositories that ship updates between base image retags -- almalinux:9 can sit unchanged for months while its repositories do not. A 72-hour expiry is the backstop for that drift, and it is also the staleness bound on anything published, which before this cache was always built fresh. A cache that cannot be reached never fails a build: a restore reports a miss so the caller builds, and a store quietly does nothing. Forks cannot write to the cache at all -- a fork could otherwise upload an image that does not match the tree its build ID names, and a later merge of that innocuous-looking tree would publish it. Leave calico/base's publish ungated and on master only, as its promotion was. base exists to carry current UBI packages: its Dockerfile barely changes, but every rebuild picks up microdnf upgrade, and it was republished on every master push, roughly every three days. Change-gating it would have cut that to whenever the Dockerfile moved. Keying on the UBI digest means it rebuilds when UBI does. A restored image still needs its registry tags, so each image gains a -retag target that applies them to whatever is in the local daemon, and the build targets call it instead of inlining the retag. base builds every architecture in one job because they share a single BRANCH_NAME, so its publish block loops rather than using a matrix. The bucket is this repo's own. calico-transient-build-artifacts-europe-west3 belongs to projectcalico/calico, where the images/ prefix holds a Docker Hub fallback copy of the published calico/go-build keyed by GO_BUILD_VER -- a consumer-side pull-through cache, not a place to put build output. Create calico-toolchain in europe-west3, where the Semaphore runners are, and grant it to the service account calico already uses. Semaphore's own cache was measured and rejected: it caps at 9.6 GB per project and one commit's images come to 8.7 GB compressed. --- .semaphore/semaphore.yml | 103 ++++++++++-- .../semaphore.yml.d/02-global_job_config.yml | 12 ++ .../semaphore.yml.d/blocks/10-go-build.yml | 17 +- .../semaphore.yml.d/blocks/10-rust-build.yml | 16 +- .../semaphore.yml.d/blocks/10-tinygo.yml | 16 +- .semaphore/semaphore.yml.d/blocks/20-base.yml | 26 ++- .../semaphore.yml.d/blocks/30-binfmt.yml | 16 +- hack/image-build-id.sh | 50 ++++++ hack/image-cache.sh | 151 ++++++++++++++++++ images/Makefile | 19 +++ 10 files changed, 404 insertions(+), 22 deletions(-) create mode 100755 hack/image-build-id.sh create mode 100755 hack/image-cache.sh diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index ed9f0b22..7bdaedf6 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -35,6 +35,14 @@ execution_time_limit: global_job_config: secrets: - name: docker-hub + # Built images are cached as tarballs in GCS. The service account is the + # one projectcalico/calico already uses; the bucket is this repo's own. + - name: google-service-account-for-gce + env_vars: + - name: GOOGLE_PROJECT + value: unique-caldron-775 + - name: GCS_IMAGE_CACHE_BUCKET + value: calico-toolchain prologue: commands: - echo $DOCKERHUB_PASSWORD | docker login --username "$DOCKERHUB_USERNAME" --password-stdin @@ -43,6 +51,10 @@ global_job_config: # unshallow it for GIT_VERSION:=$(shell git describe --tags --dirty --always) @ Makefile.common - git fetch --unshallow - touch ~/publish.env + - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/secret.google-service-account-key.json + - gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" || true + - gcloud config set project ${GOOGLE_PROJECT} || true + - sudo apt-get install -y zstd || true # Block fragments live in semaphore.yml.d/blocks and are appended here in # filename order. The numeric prefix groups blocks by what the image is, which # is durable, rather than by how it currently publishes, which is not: @@ -91,7 +103,13 @@ blocks: jobs: - name: Build and test calico/go-build image commands: - - make -C images calico-go-build-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-go-build $ARCH; then + make -C images calico-go-build-retag ARCH=$ARCH + else + make -C images calico-go-build-image ARCH=$ARCH + hack/image-cache.sh store calico-go-build $ARCH + fi - git clone -b "${CALICO_BRANCH}" --depth 1 git@github.com:projectcalico/calico.git calico - cd calico - sed -i 's/^GO_BUILD_VER=.*$/GO_BUILD_VER=${CALICO_GO_BUILD_IMAGETAG}/' metadata.mk @@ -120,7 +138,14 @@ blocks: jobs: - name: Build, test and publish calico/go-build image commands: - - make -C images calico-go-build-image ARCH=$ARCH + # A merge normally finds the image its pull request already built. + - | + if hack/image-cache.sh restore calico-go-build $ARCH; then + make -C images calico-go-build-retag ARCH=$ARCH + else + make -C images calico-go-build-image ARCH=$ARCH + hack/image-cache.sh store calico-go-build $ARCH + fi - git clone -b "${CALICO_BRANCH}" --depth 1 git@github.com:projectcalico/calico.git calico - sed -i 's/^GO_BUILD_VER=.*$/GO_BUILD_VER=${CALICO_GO_BUILD_IMAGETAG}/' calico/metadata.mk - if [ "${ARCH}" == "amd64" ]; then ( cd calico/felix && make ut ); fi @@ -187,7 +212,13 @@ blocks: jobs: - name: Build calico/rust-build image commands: - - make -C images calico-rust-build-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-rust-build ARCH; then + make -C images calico-rust-build-retag ARCH=$ARCH + else + make -C images calico-rust-build-image ARCH=$ARCH + hack/image-cache.sh store calico-rust-build ARCH + fi matrix: - env_var: ARCH values: ["amd64", "arm64", "ppc64le", "s390x"] @@ -212,7 +243,13 @@ blocks: jobs: - name: Build and publish calico/rust-build image commands: - - make -C images calico-rust-build-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-rust-build ARCH; then + make -C images calico-rust-build-retag ARCH=$ARCH + else + make -C images calico-rust-build-image ARCH=$ARCH + hack/image-cache.sh store calico-rust-build ARCH + fi - make -C images calico-rust-build-push VALIDARCHES=$ARCH ARCH=$ARCH BRANCH_NAME=${RUST_VERSION} CONFIRM=true matrix: - env_var: ARCH @@ -249,7 +286,13 @@ blocks: jobs: - name: Build and test calico/tinygo commands: - - make -C images calico-tinygo-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-tinygo ARCH; then + make -C images calico-tinygo-retag ARCH=$ARCH + else + make -C images calico-tinygo-image ARCH=$ARCH + hack/image-cache.sh store calico-tinygo ARCH + fi # Compile the test fixture against the just-built image, then # inspect the produced wasm's import section for two failure # classes a downstream consumer (gateway/coraza-wasm) would @@ -327,7 +370,13 @@ blocks: jobs: - name: Build, test and publish calico/tinygo commands: - - make -C images calico-tinygo-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-tinygo ARCH; then + make -C images calico-tinygo-retag ARCH=$ARCH + else + make -C images calico-tinygo-image ARCH=$ARCH + hack/image-cache.sh store calico-tinygo ARCH + fi # Compile the test fixture against the just-built image, then # inspect the produced wasm's import section for two failure # classes a downstream consumer (gateway/coraza-wasm) would @@ -414,7 +463,13 @@ blocks: jobs: - name: Build calico/base image commands: - - make -C images calico-base-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-base $ARCH; then + make -C images calico-base-retag ARCH=$ARCH + else + make -C images calico-base-image ARCH=$ARCH + hack/image-cache.sh store calico-base $ARCH + fi matrix: - env_var: ARCH values: ["amd64", "arm64", "ppc64le", "s390x"] @@ -422,8 +477,14 @@ blocks: - name: calico/base image and publish dependencies: - Checks + # Deliberately not change-gated, and master only, which is what the promotion + # it replaces did. base exists to carry current UBI packages: its Dockerfile + # barely changes, but every rebuild picks up `microdnf upgrade`. Gating it on + # its own sources would republish it only when the Dockerfile moved, which is + # rarely. The image cache still applies, so repeated master pushes inside the + # cache window reuse one build rather than producing near-identical tags. run: - when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-base/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + when: "pull_request !~ '.*' and branch = 'master'" task: secrets: - name: docker @@ -440,7 +501,15 @@ blocks: jobs: - name: Build and publish calico/base multi-arch images commands: - - make -C images calico-base-image-all + - | + for arch in amd64 arm64 ppc64le s390x; do + if hack/image-cache.sh restore calico-base "$arch"; then + make -C images calico-base-retag ARCH="$arch" + else + make -C images calico-base-image ARCH="$arch" + hack/image-cache.sh store calico-base "$arch" + fi + done - make -C images calico-base-push CONFIRM=true # calico/binfmt - name: calico/binfmt image @@ -452,7 +521,13 @@ blocks: jobs: - name: Build calico/binfmt amd64 image commands: - - make -C images calico-binfmt-image ARCH=amd64 + - | + if hack/image-cache.sh restore calico-binfmt amd64; then + make -C images calico-binfmt-retag + else + make -C images calico-binfmt-image ARCH=amd64 + hack/image-cache.sh store calico-binfmt amd64 + fi - name: calico/binfmt image and publish dependencies: @@ -471,5 +546,11 @@ blocks: jobs: - name: Build and publish calico/binfmt amd64 image commands: - - make -C images calico-binfmt-image ARCH=amd64 + - | + if hack/image-cache.sh restore calico-binfmt amd64; then + make -C images calico-binfmt-retag + else + make -C images calico-binfmt-image ARCH=amd64 + hack/image-cache.sh store calico-binfmt amd64 + fi - make -C images calico-binfmt-push CONFIRM=true diff --git a/.semaphore/semaphore.yml.d/02-global_job_config.yml b/.semaphore/semaphore.yml.d/02-global_job_config.yml index ac3c933c..5746113c 100644 --- a/.semaphore/semaphore.yml.d/02-global_job_config.yml +++ b/.semaphore/semaphore.yml.d/02-global_job_config.yml @@ -1,6 +1,14 @@ global_job_config: secrets: - name: docker-hub + # Built images are cached as tarballs in GCS. The service account is the + # one projectcalico/calico already uses; the bucket is this repo's own. + - name: google-service-account-for-gce + env_vars: + - name: GOOGLE_PROJECT + value: unique-caldron-775 + - name: GCS_IMAGE_CACHE_BUCKET + value: calico-toolchain prologue: commands: - echo $DOCKERHUB_PASSWORD | docker login --username "$DOCKERHUB_USERNAME" --password-stdin @@ -9,3 +17,7 @@ global_job_config: # unshallow it for GIT_VERSION:=$(shell git describe --tags --dirty --always) @ Makefile.common - git fetch --unshallow - touch ~/publish.env + - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/secret.google-service-account-key.json + - gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" || true + - gcloud config set project ${GOOGLE_PROJECT} || true + - sudo apt-get install -y zstd || true diff --git a/.semaphore/semaphore.yml.d/blocks/10-go-build.yml b/.semaphore/semaphore.yml.d/blocks/10-go-build.yml index fe734fa9..0aed6ef3 100644 --- a/.semaphore/semaphore.yml.d/blocks/10-go-build.yml +++ b/.semaphore/semaphore.yml.d/blocks/10-go-build.yml @@ -15,7 +15,13 @@ jobs: - name: Build and test calico/go-build image commands: - - make -C images calico-go-build-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-go-build $ARCH; then + make -C images calico-go-build-retag ARCH=$ARCH + else + make -C images calico-go-build-image ARCH=$ARCH + hack/image-cache.sh store calico-go-build $ARCH + fi - git clone -b "${CALICO_BRANCH}" --depth 1 git@github.com:projectcalico/calico.git calico - cd calico - sed -i 's/^GO_BUILD_VER=.*$/GO_BUILD_VER=${CALICO_GO_BUILD_IMAGETAG}/' metadata.mk @@ -44,7 +50,14 @@ jobs: - name: Build, test and publish calico/go-build image commands: - - make -C images calico-go-build-image ARCH=$ARCH + # A merge normally finds the image its pull request already built. + - | + if hack/image-cache.sh restore calico-go-build $ARCH; then + make -C images calico-go-build-retag ARCH=$ARCH + else + make -C images calico-go-build-image ARCH=$ARCH + hack/image-cache.sh store calico-go-build $ARCH + fi - git clone -b "${CALICO_BRANCH}" --depth 1 git@github.com:projectcalico/calico.git calico - sed -i 's/^GO_BUILD_VER=.*$/GO_BUILD_VER=${CALICO_GO_BUILD_IMAGETAG}/' calico/metadata.mk - if [ "${ARCH}" == "amd64" ]; then ( cd calico/felix && make ut ); fi diff --git a/.semaphore/semaphore.yml.d/blocks/10-rust-build.yml b/.semaphore/semaphore.yml.d/blocks/10-rust-build.yml index 025c1673..d7dbf06f 100644 --- a/.semaphore/semaphore.yml.d/blocks/10-rust-build.yml +++ b/.semaphore/semaphore.yml.d/blocks/10-rust-build.yml @@ -11,7 +11,13 @@ jobs: - name: Build calico/rust-build image commands: - - make -C images calico-rust-build-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-rust-build ARCH; then + make -C images calico-rust-build-retag ARCH=$ARCH + else + make -C images calico-rust-build-image ARCH=$ARCH + hack/image-cache.sh store calico-rust-build ARCH + fi matrix: - env_var: ARCH values: ["amd64", "arm64", "ppc64le", "s390x"] @@ -36,7 +42,13 @@ jobs: - name: Build and publish calico/rust-build image commands: - - make -C images calico-rust-build-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-rust-build ARCH; then + make -C images calico-rust-build-retag ARCH=$ARCH + else + make -C images calico-rust-build-image ARCH=$ARCH + hack/image-cache.sh store calico-rust-build ARCH + fi - make -C images calico-rust-build-push VALIDARCHES=$ARCH ARCH=$ARCH BRANCH_NAME=${RUST_VERSION} CONFIRM=true matrix: - env_var: ARCH diff --git a/.semaphore/semaphore.yml.d/blocks/10-tinygo.yml b/.semaphore/semaphore.yml.d/blocks/10-tinygo.yml index 1762495b..9cb8f7b9 100644 --- a/.semaphore/semaphore.yml.d/blocks/10-tinygo.yml +++ b/.semaphore/semaphore.yml.d/blocks/10-tinygo.yml @@ -11,7 +11,13 @@ jobs: - name: Build and test calico/tinygo commands: - - make -C images calico-tinygo-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-tinygo ARCH; then + make -C images calico-tinygo-retag ARCH=$ARCH + else + make -C images calico-tinygo-image ARCH=$ARCH + hack/image-cache.sh store calico-tinygo ARCH + fi # Compile the test fixture against the just-built image, then # inspect the produced wasm's import section for two failure # classes a downstream consumer (gateway/coraza-wasm) would @@ -89,7 +95,13 @@ jobs: - name: Build, test and publish calico/tinygo commands: - - make -C images calico-tinygo-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-tinygo ARCH; then + make -C images calico-tinygo-retag ARCH=$ARCH + else + make -C images calico-tinygo-image ARCH=$ARCH + hack/image-cache.sh store calico-tinygo ARCH + fi # Compile the test fixture against the just-built image, then # inspect the produced wasm's import section for two failure # classes a downstream consumer (gateway/coraza-wasm) would diff --git a/.semaphore/semaphore.yml.d/blocks/20-base.yml b/.semaphore/semaphore.yml.d/blocks/20-base.yml index 4d19d3f2..7eb98f40 100644 --- a/.semaphore/semaphore.yml.d/blocks/20-base.yml +++ b/.semaphore/semaphore.yml.d/blocks/20-base.yml @@ -8,7 +8,13 @@ jobs: - name: Build calico/base image commands: - - make -C images calico-base-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-base $ARCH; then + make -C images calico-base-retag ARCH=$ARCH + else + make -C images calico-base-image ARCH=$ARCH + hack/image-cache.sh store calico-base $ARCH + fi matrix: - env_var: ARCH values: ["amd64", "arm64", "ppc64le", "s390x"] @@ -16,8 +22,14 @@ - name: calico/base image and publish dependencies: - Checks + # Deliberately not change-gated, and master only, which is what the promotion + # it replaces did. base exists to carry current UBI packages: its Dockerfile + # barely changes, but every rebuild picks up `microdnf upgrade`. Gating it on + # its own sources would republish it only when the Dockerfile moved, which is + # rarely. The image cache still applies, so repeated master pushes inside the + # cache window reuse one build rather than producing near-identical tags. run: - when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-base/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + when: "pull_request !~ '.*' and branch = 'master'" task: secrets: - name: docker @@ -34,5 +46,13 @@ jobs: - name: Build and publish calico/base multi-arch images commands: - - make -C images calico-base-image-all + - | + for arch in amd64 arm64 ppc64le s390x; do + if hack/image-cache.sh restore calico-base "$arch"; then + make -C images calico-base-retag ARCH="$arch" + else + make -C images calico-base-image ARCH="$arch" + hack/image-cache.sh store calico-base "$arch" + fi + done - make -C images calico-base-push CONFIRM=true diff --git a/.semaphore/semaphore.yml.d/blocks/30-binfmt.yml b/.semaphore/semaphore.yml.d/blocks/30-binfmt.yml index cbcbf7ec..4cedf48f 100644 --- a/.semaphore/semaphore.yml.d/blocks/30-binfmt.yml +++ b/.semaphore/semaphore.yml.d/blocks/30-binfmt.yml @@ -8,7 +8,13 @@ jobs: - name: Build calico/binfmt amd64 image commands: - - make -C images calico-binfmt-image ARCH=amd64 + - | + if hack/image-cache.sh restore calico-binfmt amd64; then + make -C images calico-binfmt-retag + else + make -C images calico-binfmt-image ARCH=amd64 + hack/image-cache.sh store calico-binfmt amd64 + fi - name: calico/binfmt image and publish dependencies: @@ -27,5 +33,11 @@ jobs: - name: Build and publish calico/binfmt amd64 image commands: - - make -C images calico-binfmt-image ARCH=amd64 + - | + if hack/image-cache.sh restore calico-binfmt amd64; then + make -C images calico-binfmt-retag + else + make -C images calico-binfmt-image ARCH=amd64 + hack/image-cache.sh store calico-binfmt amd64 + fi - make -C images calico-binfmt-push CONFIRM=true diff --git a/hack/image-build-id.sh b/hack/image-build-id.sh new file mode 100755 index 00000000..b4b879d1 --- /dev/null +++ b/hack/image-build-id.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Print a stable content hash ("build ID") for one toolchain image. +# +# The hash covers every committed file that can change the image, so the same +# source tree always maps to the same ID. CI addresses its staged per-arch +# builds by this ID, which lets a pipeline skip a rebuild when the registry +# already holds an image for the tree it was handed. +# +# The hash reads the committed tree, not the working directory, so it ignores +# generated files under bin/ but also ignores uncommitted edits. + +set -eu + +usage() { + echo "usage: $0 " >&2 + exit 1 +} + +[ $# -eq 1 ] || usage + +case "$1" in +calico-base) + deps="images/calico-base" + ;; +calico-binfmt) + # The binfmt binary is built from cmd/ and copied into the image. + deps="images/calico-binfmt cmd" + ;; +calico-go-build) + # The semvalidator binary is built from cmd/ and copied into the image. + deps="images/calico-go-build cmd" + ;; +calico-rust-build) + deps="images/calico-rust-build" + ;; +calico-tinygo) + deps="images/calico-tinygo" + ;; +*) + usage + ;; +esac + +# These carry the build args and docker invocation shared by every image. +deps="$deps images/Makefile lib.Makefile" + +for dep in $deps; do + git rev-parse "HEAD:$dep" +done | sha256sum | cut -c1-12 diff --git a/hack/image-cache.sh b/hack/image-cache.sh new file mode 100755 index 00000000..f821f203 --- /dev/null +++ b/hack/image-cache.sh @@ -0,0 +1,151 @@ +#!/bin/bash + +# Restore or store a built toolchain image as a compressed tarball in GCS. +# +# The cache carries one change from its pull request to the merge that follows: +# the two share a source tree, so they share a cache entry, and the merge does +# not repeat a build the pull request already did. That matters most for arm64, +# ppc64le and s390x, which build under QEMU emulation. +# +# An entry is keyed by two things, because the source tree alone does not +# determine the image: +# +# 1. The build ID: a hash of the committed files that can change this image. +# 2. The digests of the base images it is built FROM. Those are floating tags, +# and a new release means a rebuild would produce a different image. UBI is +# rebuilt roughly daily; almalinux:9 and fedora:44 roughly quarterly. +# +# That covers new base releases but not everything, because dnf and microdnf +# pull from repositories that ship updates between base image retags -- +# almalinux:9 can sit unchanged for months while its repositories do not. +# MAX_AGE_HOURS is the backstop for that drift, and it is also the staleness +# bound on anything published, which before this cache was always built fresh. +# +# Usage: +# image-cache.sh restore # exit 0 on hit, 1 on miss +# image-cache.sh store + +set -eu + +usage() { + echo "usage: $0 " >&2 + exit 2 +} + +[ $# -eq 3 ] || usage +action=$1 +image=$2 +arch=$3 + +case "$action" in +restore | store) ;; +*) usage ;; +esac + +cd "$(git rev-parse --show-toplevel)" + +MAX_AGE_HOURS=${MAX_AGE_HOURS:-72} + +# A cache that cannot be reached must never fail a build. A restore reports a +# miss so the caller builds instead; a store quietly does nothing. +unavailable() { + echo "$1, skipping image cache" >&2 + if [ "$action" = restore ]; then + exit 1 + fi + exit 0 +} + +[ -n "${GCS_IMAGE_CACHE_BUCKET:-}" ] || unavailable "GCS_IMAGE_CACHE_BUCKET is not set" + +# Local tags produced by the images/Makefile build targets. calico-base builds +# one image per UBI version, so it carries two tags per architecture. +case "$image" in +calico-base) + tags="base:ubi9-latest-${arch} base:ubi10-latest-${arch}" + ;; +calico-binfmt) + qemu_version=$(yq -r '.qemu.version' images/calico-binfmt/versions.yaml) + tags="binfmt:qemu-v${qemu_version}-amd64" + ;; +calico-go-build) + tags="go-build:latest-${arch}" + ;; +calico-rust-build) + tags="rust-build:latest-${arch}" + ;; +calico-tinygo) + tags="tinygo:latest-${arch}" + ;; +*) + usage + ;; +esac + +build_id=$(hack/image-build-id.sh "$image") + +# calico-base substitutes UBI_VERSION at build time; expand it to the versions +# the Makefile actually builds. +bases=$(grep '^FROM' "images/${image}/Dockerfile" | awk '{print $2}' | grep -v '^scratch$' | sort -u) +if [ "$image" = calico-base ]; then + bases="${bases//\$\{UBI_VERSION\}/ubi9} ${bases//\$\{UBI_VERSION\}/ubi10}" +fi + +digests="" +for base in $bases; do + digest=$(docker buildx imagetools inspect "$base" --format '{{.Manifest.Digest}}' 2>/dev/null || true) + [ -n "$digest" ] || unavailable "cannot resolve the digest of $base" + digests="${digests}${digest}" +done +base_id=$(printf '%s' "$digests" | sha256sum | cut -c1-8) + +object="gs://${GCS_IMAGE_CACHE_BUCKET}/images/${image}-${build_id}-${base_id}-${arch}.tar.zst" +tarball="/tmp/${image}-${arch}.tar" + +case "$action" in +restore) + # Read timeCreated from the object metadata rather than parsing the + # human-formatted `ls` output. A missing or unreadable timestamp is a miss + # too: rebuilding is always safe. + created=$(gcloud storage objects describe "$object" --format="value(timeCreated)" 2>/dev/null || true) + if [ -z "$created" ]; then + echo "image cache miss: $object" + exit 1 + fi + created_epoch=$(date -u -d "$created" +%s 2>/dev/null || echo 0) + if [ "$created_epoch" -eq 0 ]; then + echo "image cache timestamp unreadable, rebuilding: $object" + exit 1 + fi + age_hours=$((($(date -u +%s) - created_epoch) / 3600)) + if [ "$age_hours" -ge "$MAX_AGE_HOURS" ]; then + echo "image cache stale (${age_hours}h >= ${MAX_AGE_HOURS}h), rebuilding: $object" + exit 1 + fi + echo "image cache hit (${age_hours}h old): $object" + gcloud storage cp "$object" "${tarball}.zst" + zstd -d --rm -o "$tarball" "${tarball}.zst" + docker load -i "$tarball" + rm -f "$tarball" + ;; +store) + # Never let a fork populate the cache. A fork could otherwise upload an + # image that does not match the tree its build ID names, and a later merge + # of that innocuous-looking tree would publish it. + if [ -n "${SEMAPHORE_GIT_PR_SLUG:-}" ] && + [ "${SEMAPHORE_GIT_PR_SLUG}" != "${SEMAPHORE_GIT_REPO_SLUG:-}" ]; then + echo "forked pull request, not storing to the image cache" + exit 0 + fi + if gcloud storage ls "$object" >/dev/null 2>&1; then + echo "image cache already populated: $object" + exit 0 + fi + # shellcheck disable=SC2086 + docker save $tags -o "$tarball" + zstd -3 --rm "$tarball" + gcloud storage cp "${tarball}.zst" "$object" + rm -f "${tarball}.zst" + echo "image cache stored: $object" + ;; +esac diff --git a/images/Makefile b/images/Makefile index 6ff82e66..ad5beaa3 100644 --- a/images/Makefile +++ b/images/Makefile @@ -46,6 +46,13 @@ calico-base-image-%: register $(DOCKER_BUILD) --build-arg LDSONAME=$(LDSONAME) --build-arg=UBI_VERSION=$* -t $(CALICO_BASE):$*-latest-$(ARCH) -f calico-base/Dockerfile calico-base/ $(MAKE) BUILD_IMAGES=$(CALICO_BASE) retag-build-images-with-registries VALIDARCHES=$(ARCH) LATEST_IMAGE_TAG=$*-latest IMAGETAG=$*-latest +# calico-base-retag applies the registry tags to images already in the local +# daemon, whether just built or loaded from the image cache. +.PHONY: calico-base-retag +calico-base-retag: $(addprefix calico-base-retag-,$(UBI_VERSIONS)) +calico-base-retag-%: + $(MAKE) BUILD_IMAGES=$(CALICO_BASE) retag-build-images-with-registries VALIDARCHES=$(ARCH) LATEST_IMAGE_TAG=$*-latest IMAGETAG=$*-latest + .PHONY: calico-base-image-all calico-base-image-all: $(addprefix sub-calico-base-image-,$(VALIDARCHES)) sub-calico-base-image-%: @@ -116,6 +123,10 @@ calico-binfmt-image: build $(DOCKER_BUILD) --build-arg QEMU_VERSION=$(QEMU_VERSION) -t $(CALICO_BINFMT):$(CALICO_BINFMT_IMAGETAG)-amd64 -f calico-binfmt/Dockerfile calico-binfmt/ $(MAKE) BUILD_IMAGES=$(CALICO_BINFMT) retag-build-images-with-registries VALIDARCHES=amd64 LATEST_IMAGE_TAG=$(CALICO_BINFMT_IMAGETAG) IMAGETAG=$(CALICO_BINFMT_IMAGETAG) +.PHONY: calico-binfmt-retag +calico-binfmt-retag: + $(MAKE) BUILD_IMAGES=$(CALICO_BINFMT) retag-build-images-with-registries VALIDARCHES=amd64 LATEST_IMAGE_TAG=$(CALICO_BINFMT_IMAGETAG) IMAGETAG=$(CALICO_BINFMT_IMAGETAG) + .PHONY: calico-binfmt-cd calico-binfmt-cd: calico-binfmt-image calico-binfmt-push @@ -135,6 +146,10 @@ calico-rust-build-image: register $(DOCKER_BUILD) -t $(CALICO_RUST_BUILD):latest-$(ARCH) -f calico-rust-build/Dockerfile calico-rust-build/ $(MAKE) BUILD_IMAGES=$(CALICO_RUST_BUILD) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=$(CALICO_RUST_BUILD_IMAGETAG) +.PHONY: calico-rust-build-retag +calico-rust-build-retag: + $(MAKE) BUILD_IMAGES=$(CALICO_RUST_BUILD) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=$(CALICO_RUST_BUILD_IMAGETAG) + .PHONY: calico-rust-build-image-all calico-rust-build-image-all: $(addprefix sub-calico-rust-build-image-,$(VALIDARCHES)) sub-calico-rust-build-image-%: @@ -164,6 +179,10 @@ calico-tinygo-image: register $(DOCKER_BUILD) -t $(CALICO_TINYGO):latest-$(ARCH) -f calico-tinygo/Dockerfile calico-tinygo/ $(MAKE) BUILD_IMAGES=$(CALICO_TINYGO) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=$(CALICO_TINYGO_IMAGETAG) +.PHONY: calico-tinygo-retag +calico-tinygo-retag: + $(MAKE) BUILD_IMAGES=$(CALICO_TINYGO) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=$(CALICO_TINYGO_IMAGETAG) + .PHONY: calico-tinygo-image-all calico-tinygo-image-all: $(addprefix sub-calico-tinygo-image-,amd64 arm64) sub-calico-tinygo-image-%: