diff --git a/.semaphore/promotions/calico-base.yml b/.semaphore/promotions/calico-base.yml deleted file mode 100644 index 5a12b8ab..00000000 --- a/.semaphore/promotions/calico-base.yml +++ /dev/null @@ -1,34 +0,0 @@ -version: v1.0 -name: Publish calico/base images -agent: - machine: - type: f1-standard-2 - os_image: ubuntu2204 - -execution_time_limit: - minutes: 30 - -global_job_config: - env_vars: - - name: DEV_REGISTRIES - value: calico - secrets: - - name: docker - prologue: - commands: - - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin - - checkout - -blocks: - - name: Publish calico/base multi-arch images - dependencies: [] - run: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - task: - env_vars: - - name: BRANCH_NAME - value: ${SEMAPHORE_JOB_CREATION_TIME} - jobs: - - name: Linux multi-arch - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images calico-base-cd CONFIRM=true; fi diff --git a/.semaphore/promotions/calico-binfmt.yml b/.semaphore/promotions/calico-binfmt.yml deleted file mode 100644 index 846d94bf..00000000 --- a/.semaphore/promotions/calico-binfmt.yml +++ /dev/null @@ -1,31 +0,0 @@ -version: v1.0 -name: Publish calico/binfmt images -agent: - machine: - type: f1-standard-2 - os_image: ubuntu2204 - -execution_time_limit: - minutes: 30 - -global_job_config: - env_vars: - - name: DEV_REGISTRIES - value: calico - secrets: - - name: docker - prologue: - commands: - - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin - - checkout - -blocks: - - name: Publish calico/binfmt amd64 images - dependencies: [] - run: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - task: - jobs: - - name: Linux amd64 - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images calico-binfmt-cd CONFIRM=true; fi diff --git a/.semaphore/promotions/calico-go-build.yml b/.semaphore/promotions/calico-go-build.yml deleted file mode 100644 index ebd38ebc..00000000 --- a/.semaphore/promotions/calico-go-build.yml +++ /dev/null @@ -1,65 +0,0 @@ -version: v1.0 -name: Publish calico/go-build images -agent: - machine: - type: f1-standard-2 - os_image: ubuntu2204 - -execution_time_limit: - minutes: 60 - -global_job_config: - env_vars: - - name: DEV_REGISTRIES - value: calico - secrets: - - name: docker - prologue: - commands: - - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin - - checkout - # If an image build is triggered from a git tag, use the git tag name set by semaphore. - # For utility changes that don't affect calico/go-build versions, we will append a release - # number in the tag. This is handled in create-tag-on-version-change github workflow. - - | - if [ "${SEMAPHORE_GIT_REF_TYPE}" = "tag" ]; then - export BRANCH_NAME=${SEMAPHORE_GIT_TAG_NAME} - else - export BRANCH_NAME=${SEMAPHORE_GIT_WORKING_BRANCH} - fi - -blocks: - - name: Publish calico/go-build images - dependencies: [] - run: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - task: - jobs: - - name: Linux multi-arch - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images calico-go-build-cd VALIDARCHES=$ARCH CONFIRM=true; fi - matrix: - - env_var: ARCH - values: ["amd64", "arm64", "ppc64le", "s390x"] - - name: Publish calico/go-build multi-arch manifests - dependencies: - - Publish calico/go-build images - run: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - task: - jobs: - - name: Linux multi-arch manifests - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images push-calico-go-build-manifests CONFIRM=true; fi - - name: Trigger calico/go-build pin updates - dependencies: - - Publish calico/go-build multi-arch manifests - run: - when: "tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - task: - secrets: - - name: semaphore-api - jobs: - - name: Auto calico/go-build update - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make update-go-build-pins CONFIRM=true; fi diff --git a/.semaphore/promotions/calico-rust-build.yml b/.semaphore/promotions/calico-rust-build.yml deleted file mode 100644 index cf0b404b..00000000 --- a/.semaphore/promotions/calico-rust-build.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: v1.0 -name: Publish calico/rust-build images -agent: - machine: - type: f1-standard-2 - os_image: ubuntu2204 - -execution_time_limit: - minutes: 60 - -global_job_config: - env_vars: - - name: DEV_REGISTRIES - value: calico - secrets: - - name: docker - prologue: - commands: - - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin - - checkout - - export BRANCH_NAME=$(yq -r '.rust.version' images/calico-rust-build/versions.yaml) - -blocks: - - name: Publish calico/rust-build images - dependencies: [] - run: - when: "branch = 'master' AND change_in('/images/calico-rust-build/')" - task: - jobs: - - name: Linux multi-arch - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images calico-rust-build-cd VALIDARCHES=$ARCH CONFIRM=true; fi - matrix: - - env_var: ARCH - values: ["amd64", "arm64", "ppc64le", "s390x"] - - name: Publish calico/rust-build multi-arch manifests - dependencies: - - Publish calico/rust-build images - run: - when: "branch = 'master' AND change_in('/images/calico-rust-build/')" - task: - jobs: - - name: Linux multi-arch manifests - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images push-calico-rust-build-manifests CONFIRM=true; fi diff --git a/.semaphore/promotions/calico-tinygo.yml b/.semaphore/promotions/calico-tinygo.yml deleted file mode 100644 index a71ef13c..00000000 --- a/.semaphore/promotions/calico-tinygo.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: v1.0 -name: Publish calico/tinygo images -agent: - machine: - type: f1-standard-2 - os_image: ubuntu2204 - -execution_time_limit: - minutes: 60 - -global_job_config: - env_vars: - - name: DEV_REGISTRIES - value: calico - secrets: - - name: docker - prologue: - commands: - - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin - - checkout - - export BRANCH_NAME=$(yq -r '.tinygo.version' images/calico-tinygo/versions.yaml) - -blocks: - - name: Publish calico/tinygo image - dependencies: [] - run: - when: "branch = 'master' AND change_in('/images/calico-tinygo/')" - task: - jobs: - - name: Publish per-arch - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images calico-tinygo-cd ARCH=$ARCH CONFIRM=true; fi - matrix: - - env_var: ARCH - values: ["amd64", "arm64"] - - name: Publish calico/tinygo multi-arch manifest - dependencies: - - Publish calico/tinygo image - run: - when: "branch = 'master' AND change_in('/images/calico-tinygo/')" - task: - jobs: - - name: Manifest - commands: - - if [ -z "${SEMAPHORE_GIT_PR_NUMBER}" ]; then make -C images push-calico-tinygo-manifests CONFIRM=true; fi diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index a4a7db92..7bdaedf6 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -1,3 +1,6 @@ +# !! WARNING, DO NOT EDIT !! This file is generated from the fragments +# in /.semaphore/semaphore.yml.d. To update, modify the relevant +# fragment and then run 'make gen-semaphore-yaml'. version: v1.0 name: go-build agent: @@ -10,12 +13,36 @@ auto_cancel: queued: when: "branch != 'master'" +# A publish job builds, tests and pushes in one go, where the build and the +# push used to have a 60-minute budget each in separate pipelines. The +# ppc64le and s390x builds run under QEMU emulation and are the long pole. execution_time_limit: - minutes: 60 + minutes: 120 +# Each image has two blocks: one for pull requests that builds and tests, and +# one for master and release branches that builds, tests and publishes in a +# single job. Exactly one of the pair runs per event, so an image is built once +# per commit instead of once per pipeline, and the push happens on the machine +# that already holds the image. +# +# Only the publish blocks carry the `docker` push secret. Semaphore evaluates +# `run: when:` before scheduling a block, so that secret is never injected into +# a pull request run, forked or otherwise. +# +# Git tags do not build. A push to a release branch publishes both the branch +# tag and the release tag that create-tag-on-version-change.yml just created, +# so the tag pipeline has nothing left to do and every block below skips it. global_job_config: secrets: - name: docker-hub + # Built images are cached as tarballs in GCS. The service account is the + # one projectcalico/calico already uses; the bucket is this repo's own. + - name: google-service-account-for-gce + env_vars: + - name: GOOGLE_PROJECT + value: unique-caldron-775 + - name: GCS_IMAGE_CACHE_BUCKET + value: calico-toolchain prologue: commands: - echo $DOCKERHUB_PASSWORD | docker login --username "$DOCKERHUB_USERNAME" --password-stdin @@ -23,35 +50,48 @@ global_job_config: # Semaphore is doing shallow clone on a commit without tags. # unshallow it for GIT_VERSION:=$(shell git describe --tags --dirty --always) @ Makefile.common - git fetch --unshallow - -promotions: - # Publish base, binfmt, and go-build images for master or release tags (example: 1.23.3-llvm18.1.8-k8s1.30.5). - - name: Publish calico/base images - pipeline_file: promotions/calico-base.yml - auto_promote: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - - name: Publish calico/binfmt images - pipeline_file: promotions/calico-binfmt.yml - auto_promote: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - - name: Publish calico/go-build images - pipeline_file: promotions/calico-go-build.yml - auto_promote: - when: "branch = 'master' OR tag =~ '^1\\.\\d+\\.\\d+-llvm\\d+\\.\\d\\.\\d-k8s1\\.\\d+\\.\\d+'" - # Publish rust-build images for master and files under images/calico-rust-build have changed. - - name: Publish calico/rust-build images - pipeline_file: promotions/calico-rust-build.yml - auto_promote: - when: "branch = 'master' AND change_in('/images/calico-rust-build/')" - # Publish tinygo images for master and files under images/calico-tinygo have changed. - - name: Publish calico/tinygo images - pipeline_file: promotions/calico-tinygo.yml - auto_promote: - when: "branch = 'master' AND change_in('/images/calico-tinygo/')" - + - touch ~/publish.env + - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/secret.google-service-account-key.json + - gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" || true + - gcloud config set project ${GOOGLE_PROJECT} || true + - sudo apt-get install -y zstd || true +# Block fragments live in semaphore.yml.d/blocks and are appended here in +# filename order. The numeric prefix groups blocks by what the image is, which +# is durable, rather than by how it currently publishes, which is not: +# +# 00 repository checks - builds nothing, and every image block waits on it +# 10 compiler toolchains, the images this repo exists to produce +# 20 runtime base, shipped inside Calico components +# 30 multi-arch CI helper +# +# The numbers are not build order. No toolchain image is built FROM another; +# all five build in parallel once Checks has passed. blocks: - - name: calico/go-build image + # Repository checks + # + # Every image block depends on this one, so a failure here stops the builds + # before any QEMU-emulated build starts or anything is published. The block is + # change-gated and often skips; Semaphore counts a skipped dependency as + # passed, and this block produces no artifacts, so skipping it is safe. + - name: Checks dependencies: [] + run: + when: "change_in(['/.semaphore/', '/hack/', '/Makefile'], {pipeline_file: 'track'})" + task: + jobs: + - name: Check generated semaphore.yml + commands: + - make check-semaphore-yaml + - name: Shellcheck + commands: + - command -v shellcheck >/dev/null || sudo apt-get install -y shellcheck + - shellcheck hack/*.sh + # calico/go-build + - name: calico/go-build image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-go-build/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" task: env_vars: # The branch to test the current go-build against @@ -59,16 +99,17 @@ blocks: value: master prologue: commands: - - | - if [ "${SEMAPHORE_GIT_REF_TYPE}" = "tag" ]; then - export CALICO_GO_BUILD_IMAGETAG=${SEMAPHORE_GIT_TAG_NAME} - else - export CALICO_GO_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} - fi + - export CALICO_GO_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} jobs: - - name: Build calico/go-build image + - name: Build and test calico/go-build image commands: - - make -C images calico-go-build-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-go-build $ARCH; then + make -C images calico-go-build-retag ARCH=$ARCH + else + make -C images calico-go-build-image ARCH=$ARCH + hack/image-cache.sh store calico-go-build $ARCH + fi - git clone -b "${CALICO_BRANCH}" --depth 1 git@github.com:projectcalico/calico.git calico - cd calico - sed -i 's/^GO_BUILD_VER=.*$/GO_BUILD_VER=${CALICO_GO_BUILD_IMAGETAG}/' metadata.mk @@ -77,59 +118,181 @@ blocks: - env_var: ARCH values: ["amd64", "arm64", "ppc64le", "s390x"] + - name: calico/go-build image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-go-build/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + - name: CALICO_BRANCH + value: master + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - export CALICO_GO_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + jobs: + - name: Build, test and publish calico/go-build image + commands: + # A merge normally finds the image its pull request already built. + - | + if hack/image-cache.sh restore calico-go-build $ARCH; then + make -C images calico-go-build-retag ARCH=$ARCH + else + make -C images calico-go-build-image ARCH=$ARCH + hack/image-cache.sh store calico-go-build $ARCH + fi + - git clone -b "${CALICO_BRANCH}" --depth 1 git@github.com:projectcalico/calico.git calico + - sed -i 's/^GO_BUILD_VER=.*$/GO_BUILD_VER=${CALICO_GO_BUILD_IMAGETAG}/' calico/metadata.mk + - if [ "${ARCH}" == "amd64" ]; then ( cd calico/felix && make ut ); fi + # The release tag is created by create-tag-on-version-change.yml + # within a minute of the push; this job reaches here ~20 minutes + # later. Fetch now rather than in the prologue so the tag is visible. + - git fetch --tags --force + - | + export RELEASE_TAG=$(git tag --points-at HEAD | grep -E '^[0-9]+\.[0-9]+\.[0-9]+-llvm' | sort -V | tail -1) + if [ -z "${RELEASE_TAG}" ] && [ "${SEMAPHORE_GIT_WORKING_BRANCH}" != "master" ]; then + echo >&2 "FAIL: no release tag points at HEAD on ${SEMAPHORE_GIT_WORKING_BRANCH}." + echo >&2 " create-tag-on-version-change.yml should have created one." + exit 1 + fi + echo "RELEASE_TAG=${RELEASE_TAG}" >> ~/publish.env + # Publish only after the tests above have passed. + - make -C images calico-go-build-push VALIDARCHES=$ARCH ARCH=$ARCH BRANCH_NAME=${SEMAPHORE_GIT_WORKING_BRANCH} CONFIRM=true + - | + . ~/publish.env + if [ -n "${RELEASE_TAG}" ]; then + make -C images calico-go-build-push VALIDARCHES=$ARCH ARCH=$ARCH BRANCH_NAME=${RELEASE_TAG} CONFIRM=true + fi + matrix: + - env_var: ARCH + values: ["amd64", "arm64", "ppc64le", "s390x"] + + - name: calico/go-build manifests + dependencies: + - calico/go-build image and publish + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-go-build/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + - name: semaphore-api + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Push calico/go-build multi-arch manifests + commands: + - git fetch --tags --force + - make -C images push-calico-go-build-manifests BRANCH_NAME=${SEMAPHORE_GIT_WORKING_BRANCH} CONFIRM=true + - | + RELEASE_TAG=$(git tag --points-at HEAD | grep -E '^[0-9]+\.[0-9]+\.[0-9]+-llvm' | sort -V | tail -1) + if [ -n "${RELEASE_TAG}" ]; then + make -C images push-calico-go-build-manifests BRANCH_NAME=${RELEASE_TAG} CONFIRM=true + # Tell downstream repos to pin the tag we just published. + make update-go-build-pins CONFIRM=true + fi + # calico/rust-build - name: calico/rust-build image - dependencies: [] + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-rust-build/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" task: prologue: commands: - - | - if [ "${SEMAPHORE_GIT_REF_TYPE}" = "tag" ]; then - export CALICO_RUST_BUILD_IMAGETAG=${SEMAPHORE_GIT_TAG_NAME} - else - export CALICO_RUST_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} - fi + - export CALICO_RUST_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} jobs: - name: Build calico/rust-build image commands: - - make -C images calico-rust-build-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-rust-build ARCH; then + make -C images calico-rust-build-retag ARCH=$ARCH + else + make -C images calico-rust-build-image ARCH=$ARCH + hack/image-cache.sh store calico-rust-build ARCH + fi matrix: - env_var: ARCH values: ["amd64", "arm64", "ppc64le", "s390x"] - - name: calico/base image - dependencies: [] + - name: calico/rust-build image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-rust-build/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - export CALICO_RUST_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + # rust-build is versioned by its own toolchain version, not by the branch. + - export RUST_VERSION=$(yq -r '.rust.version' images/calico-rust-build/versions.yaml) jobs: - - name: Build calico/base image + - name: Build and publish calico/rust-build image commands: - - make -C images calico-base-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-rust-build ARCH; then + make -C images calico-rust-build-retag ARCH=$ARCH + else + make -C images calico-rust-build-image ARCH=$ARCH + hack/image-cache.sh store calico-rust-build ARCH + fi + - make -C images calico-rust-build-push VALIDARCHES=$ARCH ARCH=$ARCH BRANCH_NAME=${RUST_VERSION} CONFIRM=true matrix: - env_var: ARCH values: ["amd64", "arm64", "ppc64le", "s390x"] - - name: calico/binfmt image - dependencies: [] + - name: calico/rust-build manifests + dependencies: + - calico/rust-build image and publish + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-rust-build/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin jobs: - - name: Build calico/binfmt amd64 image + - name: Push calico/rust-build multi-arch manifests commands: - - make -C images calico-binfmt-image ARCH=amd64 - + - make -C images push-calico-rust-build-manifests BRANCH_NAME=$(yq -r '.rust.version' images/calico-rust-build/versions.yaml) CONFIRM=true + # calico/tinygo - name: calico/tinygo image - dependencies: [] + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-tinygo/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" task: prologue: commands: - - | - if [ "${SEMAPHORE_GIT_REF_TYPE}" = "tag" ]; then - export CALICO_TINYGO_IMAGETAG=${SEMAPHORE_GIT_TAG_NAME} - else - export CALICO_TINYGO_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} - fi + - export CALICO_TINYGO_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} jobs: - name: Build and test calico/tinygo commands: - - make -C images calico-tinygo-image ARCH=$ARCH + - | + if hack/image-cache.sh restore calico-tinygo ARCH; then + make -C images calico-tinygo-retag ARCH=$ARCH + else + make -C images calico-tinygo-image ARCH=$ARCH + hack/image-cache.sh store calico-tinygo ARCH + fi # Compile the test fixture against the just-built image, then # inspect the produced wasm's import section for two failure # classes a downstream consumer (gateway/coraza-wasm) would @@ -186,3 +349,208 @@ blocks: matrix: - env_var: ARCH values: ["amd64", "arm64"] + + - name: calico/tinygo image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-tinygo/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - export CALICO_TINYGO_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + # tinygo is versioned by its own toolchain version, not by the branch. + - export TINYGO_VERSION=$(yq -r '.tinygo.version' images/calico-tinygo/versions.yaml) + jobs: + - name: Build, test and publish calico/tinygo + commands: + - | + if hack/image-cache.sh restore calico-tinygo ARCH; then + make -C images calico-tinygo-retag ARCH=$ARCH + else + make -C images calico-tinygo-image ARCH=$ARCH + hack/image-cache.sh store calico-tinygo ARCH + fi + # Compile the test fixture against the just-built image, then + # inspect the produced wasm's import section for two failure + # classes a downstream consumer (gateway/coraza-wasm) would + # otherwise hit: + # + # 1. wasm-ld undefined symbols at link time. The fixture + # imports github.com/wasilibs/{go-re2, go-libinjection} + # and uses each via os.Args-derived input so TinyGo's + # `-opt=2` optimizer cannot prove the calls dead. If a + # TinyGo or wasi-libc bump breaks the prebuilt-archive + # ABI, the build fails here. + # + # 2. Runtime `missing import: env.cre2_new` on Envoy. The + # assertion below counts `<- env.cre2_*` imports in the + # wasm; with the TinyGo-compatible go-re2 (currently + # v1.6.0) cre2 resolves internally to the bundled + # static archive and the count is zero. wasilibs/go-re2 + # v1.8.0+ removed TinyGo support (libre2 now needs + # Abseil with thread sync, which TinyGo's libc++ shim + # does not provide); a bump to such a version produces + # external env.cre2_* imports and the assertion fails + # here, at toolchain-image build time, not during a + # downstream WAF rebuild. + # + # The Makefile builds the image as `tinygo:latest-amd64` + # locally before retagging it as `calico/tinygo:${branch}-amd64` + # for the registry. Use the local tag here — the calico/- + # prefixed tag uses the branch, not "latest". + - | + docker run --rm \ + -v "$(pwd)/images/calico-tinygo/test/fixture:/src" \ + -v /tmp:/out \ + -w /src \ + tinygo:latest-${ARCH} \ + sh -c 'go mod download && \ + tinygo build \ + -target=wasip1 \ + -gc=precise -opt=2 -scheduler=none \ + -tags="nottinygc_envoy memoize_builders" \ + -o /out/fixture.wasm . && \ + file /out/fixture.wasm | grep -q "WebAssembly"' + - | + count=$(docker run --rm -v /tmp:/host ghcr.io/webassembly/wabt:1.0.36 \ + wasm-objdump -x /host/fixture.wasm | grep -c "<- env.cre2_" || true) + if [ "${count:-0}" -ne 0 ]; then + echo >&2 "FAIL: fixture wasm has ${count} env.cre2_* imports (expected 0)" + echo >&2 " A wasilibs/go-re2 bump likely lost TinyGo's bundled libcre2.a;" + echo >&2 " this wasm would fail to load in Envoy with 'missing import: env.cre2_new'." + docker run --rm -v /tmp:/host ghcr.io/webassembly/wabt:1.0.36 \ + wasm-objdump -x /host/fixture.wasm | grep "<- env.cre2_" >&2 || true + exit 1 + fi + echo "OK: fixture wasm resolves cre2_* internally (no env.cre2_* imports)" + - make -C images calico-tinygo-push ARCH=$ARCH VALIDARCHES=$ARCH BRANCH_NAME=${TINYGO_VERSION} CONFIRM=true + matrix: + - env_var: ARCH + values: ["amd64", "arm64"] + + - name: calico/tinygo manifests + dependencies: + - calico/tinygo image and publish + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-tinygo/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Push calico/tinygo multi-arch manifest + commands: + - make -C images push-calico-tinygo-manifests BRANCH_NAME=$(yq -r '.tinygo.version' images/calico-tinygo/versions.yaml) CONFIRM=true + # calico/base + - name: calico/base image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-base/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + jobs: + - name: Build calico/base image + commands: + - | + if hack/image-cache.sh restore calico-base $ARCH; then + make -C images calico-base-retag ARCH=$ARCH + else + make -C images calico-base-image ARCH=$ARCH + hack/image-cache.sh store calico-base $ARCH + fi + matrix: + - env_var: ARCH + values: ["amd64", "arm64", "ppc64le", "s390x"] + + - name: calico/base image and publish + dependencies: + - Checks + # Deliberately not change-gated, and master only, which is what the promotion + # it replaces did. base exists to carry current UBI packages: its Dockerfile + # barely changes, but every rebuild picks up `microdnf upgrade`. Gating it on + # its own sources would republish it only when the Dockerfile moved, which is + # rarely. The image cache still applies, so repeated master pushes inside the + # cache window reuse one build rather than producing near-identical tags. + run: + when: "pull_request !~ '.*' and branch = 'master'" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + # Every architecture and both UBI versions must share one tag, so base + # builds and publishes in a single job rather than a per-arch matrix. + - name: BRANCH_NAME + value: ${SEMAPHORE_JOB_CREATION_TIME} + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Build and publish calico/base multi-arch images + commands: + - | + for arch in amd64 arm64 ppc64le s390x; do + if hack/image-cache.sh restore calico-base "$arch"; then + make -C images calico-base-retag ARCH="$arch" + else + make -C images calico-base-image ARCH="$arch" + hack/image-cache.sh store calico-base "$arch" + fi + done + - make -C images calico-base-push CONFIRM=true + # calico/binfmt + - name: calico/binfmt image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-binfmt/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + jobs: + - name: Build calico/binfmt amd64 image + commands: + - | + if hack/image-cache.sh restore calico-binfmt amd64; then + make -C images calico-binfmt-retag + else + make -C images calico-binfmt-image ARCH=amd64 + hack/image-cache.sh store calico-binfmt amd64 + fi + + - name: calico/binfmt image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-binfmt/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Build and publish calico/binfmt amd64 image + commands: + - | + if hack/image-cache.sh restore calico-binfmt amd64; then + make -C images calico-binfmt-retag + else + make -C images calico-binfmt-image ARCH=amd64 + hack/image-cache.sh store calico-binfmt amd64 + fi + - make -C images calico-binfmt-push CONFIRM=true diff --git a/.semaphore/semaphore.yml.d/01-preamble.yml b/.semaphore/semaphore.yml.d/01-preamble.yml new file mode 100644 index 00000000..c2cdef79 --- /dev/null +++ b/.semaphore/semaphore.yml.d/01-preamble.yml @@ -0,0 +1,31 @@ +version: v1.0 +name: go-build +agent: + machine: + type: f1-standard-2 + os_image: ubuntu2204 +auto_cancel: + running: + when: "branch != 'master'" + queued: + when: "branch != 'master'" + +# A publish job builds, tests and pushes in one go, where the build and the +# push used to have a 60-minute budget each in separate pipelines. The +# ppc64le and s390x builds run under QEMU emulation and are the long pole. +execution_time_limit: + minutes: 120 + +# Each image has two blocks: one for pull requests that builds and tests, and +# one for master and release branches that builds, tests and publishes in a +# single job. Exactly one of the pair runs per event, so an image is built once +# per commit instead of once per pipeline, and the push happens on the machine +# that already holds the image. +# +# Only the publish blocks carry the `docker` push secret. Semaphore evaluates +# `run: when:` before scheduling a block, so that secret is never injected into +# a pull request run, forked or otherwise. +# +# Git tags do not build. A push to a release branch publishes both the branch +# tag and the release tag that create-tag-on-version-change.yml just created, +# so the tag pipeline has nothing left to do and every block below skips it. diff --git a/.semaphore/semaphore.yml.d/02-global_job_config.yml b/.semaphore/semaphore.yml.d/02-global_job_config.yml new file mode 100644 index 00000000..5746113c --- /dev/null +++ b/.semaphore/semaphore.yml.d/02-global_job_config.yml @@ -0,0 +1,23 @@ +global_job_config: + secrets: + - name: docker-hub + # Built images are cached as tarballs in GCS. The service account is the + # one projectcalico/calico already uses; the bucket is this repo's own. + - name: google-service-account-for-gce + env_vars: + - name: GOOGLE_PROJECT + value: unique-caldron-775 + - name: GCS_IMAGE_CACHE_BUCKET + value: calico-toolchain + prologue: + commands: + - echo $DOCKERHUB_PASSWORD | docker login --username "$DOCKERHUB_USERNAME" --password-stdin + - checkout + # Semaphore is doing shallow clone on a commit without tags. + # unshallow it for GIT_VERSION:=$(shell git describe --tags --dirty --always) @ Makefile.common + - git fetch --unshallow + - touch ~/publish.env + - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/secret.google-service-account-key.json + - gcloud auth activate-service-account --key-file="$GOOGLE_APPLICATION_CREDENTIALS" || true + - gcloud config set project ${GOOGLE_PROJECT} || true + - sudo apt-get install -y zstd || true diff --git a/.semaphore/semaphore.yml.d/09-blocks.yml b/.semaphore/semaphore.yml.d/09-blocks.yml new file mode 100644 index 00000000..13677f2c --- /dev/null +++ b/.semaphore/semaphore.yml.d/09-blocks.yml @@ -0,0 +1,12 @@ +# Block fragments live in semaphore.yml.d/blocks and are appended here in +# filename order. The numeric prefix groups blocks by what the image is, which +# is durable, rather than by how it currently publishes, which is not: +# +# 00 repository checks - builds nothing, and every image block waits on it +# 10 compiler toolchains, the images this repo exists to produce +# 20 runtime base, shipped inside Calico components +# 30 multi-arch CI helper +# +# The numbers are not build order. No toolchain image is built FROM another; +# all five build in parallel once Checks has passed. +blocks: diff --git a/.semaphore/semaphore.yml.d/blocks/00-checks.yml b/.semaphore/semaphore.yml.d/blocks/00-checks.yml new file mode 100644 index 00000000..c0366ae3 --- /dev/null +++ b/.semaphore/semaphore.yml.d/blocks/00-checks.yml @@ -0,0 +1,19 @@ +# Repository checks +# +# Every image block depends on this one, so a failure here stops the builds +# before any QEMU-emulated build starts or anything is published. The block is +# change-gated and often skips; Semaphore counts a skipped dependency as +# passed, and this block produces no artifacts, so skipping it is safe. +- name: Checks + dependencies: [] + run: + when: "change_in(['/.semaphore/', '/hack/', '/Makefile'], {pipeline_file: 'track'})" + task: + jobs: + - name: Check generated semaphore.yml + commands: + - make check-semaphore-yaml + - name: Shellcheck + commands: + - command -v shellcheck >/dev/null || sudo apt-get install -y shellcheck + - shellcheck hack/*.sh diff --git a/.semaphore/semaphore.yml.d/blocks/10-go-build.yml b/.semaphore/semaphore.yml.d/blocks/10-go-build.yml new file mode 100644 index 00000000..0aed6ef3 --- /dev/null +++ b/.semaphore/semaphore.yml.d/blocks/10-go-build.yml @@ -0,0 +1,113 @@ +# calico/go-build +- name: calico/go-build image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-go-build/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + env_vars: + # The branch to test the current go-build against + - name: CALICO_BRANCH + value: master + prologue: + commands: + - export CALICO_GO_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + jobs: + - name: Build and test calico/go-build image + commands: + - | + if hack/image-cache.sh restore calico-go-build $ARCH; then + make -C images calico-go-build-retag ARCH=$ARCH + else + make -C images calico-go-build-image ARCH=$ARCH + hack/image-cache.sh store calico-go-build $ARCH + fi + - git clone -b "${CALICO_BRANCH}" --depth 1 git@github.com:projectcalico/calico.git calico + - cd calico + - sed -i 's/^GO_BUILD_VER=.*$/GO_BUILD_VER=${CALICO_GO_BUILD_IMAGETAG}/' metadata.mk + - if [ "${ARCH}" == "amd64" ]; then cd felix && make ut; fi + matrix: + - env_var: ARCH + values: ["amd64", "arm64", "ppc64le", "s390x"] + +- name: calico/go-build image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-go-build/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + - name: CALICO_BRANCH + value: master + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - export CALICO_GO_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + jobs: + - name: Build, test and publish calico/go-build image + commands: + # A merge normally finds the image its pull request already built. + - | + if hack/image-cache.sh restore calico-go-build $ARCH; then + make -C images calico-go-build-retag ARCH=$ARCH + else + make -C images calico-go-build-image ARCH=$ARCH + hack/image-cache.sh store calico-go-build $ARCH + fi + - git clone -b "${CALICO_BRANCH}" --depth 1 git@github.com:projectcalico/calico.git calico + - sed -i 's/^GO_BUILD_VER=.*$/GO_BUILD_VER=${CALICO_GO_BUILD_IMAGETAG}/' calico/metadata.mk + - if [ "${ARCH}" == "amd64" ]; then ( cd calico/felix && make ut ); fi + # The release tag is created by create-tag-on-version-change.yml + # within a minute of the push; this job reaches here ~20 minutes + # later. Fetch now rather than in the prologue so the tag is visible. + - git fetch --tags --force + - | + export RELEASE_TAG=$(git tag --points-at HEAD | grep -E '^[0-9]+\.[0-9]+\.[0-9]+-llvm' | sort -V | tail -1) + if [ -z "${RELEASE_TAG}" ] && [ "${SEMAPHORE_GIT_WORKING_BRANCH}" != "master" ]; then + echo >&2 "FAIL: no release tag points at HEAD on ${SEMAPHORE_GIT_WORKING_BRANCH}." + echo >&2 " create-tag-on-version-change.yml should have created one." + exit 1 + fi + echo "RELEASE_TAG=${RELEASE_TAG}" >> ~/publish.env + # Publish only after the tests above have passed. + - make -C images calico-go-build-push VALIDARCHES=$ARCH ARCH=$ARCH BRANCH_NAME=${SEMAPHORE_GIT_WORKING_BRANCH} CONFIRM=true + - | + . ~/publish.env + if [ -n "${RELEASE_TAG}" ]; then + make -C images calico-go-build-push VALIDARCHES=$ARCH ARCH=$ARCH BRANCH_NAME=${RELEASE_TAG} CONFIRM=true + fi + matrix: + - env_var: ARCH + values: ["amd64", "arm64", "ppc64le", "s390x"] + +- name: calico/go-build manifests + dependencies: + - calico/go-build image and publish + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-go-build/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + - name: semaphore-api + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Push calico/go-build multi-arch manifests + commands: + - git fetch --tags --force + - make -C images push-calico-go-build-manifests BRANCH_NAME=${SEMAPHORE_GIT_WORKING_BRANCH} CONFIRM=true + - | + RELEASE_TAG=$(git tag --points-at HEAD | grep -E '^[0-9]+\.[0-9]+\.[0-9]+-llvm' | sort -V | tail -1) + if [ -n "${RELEASE_TAG}" ]; then + make -C images push-calico-go-build-manifests BRANCH_NAME=${RELEASE_TAG} CONFIRM=true + # Tell downstream repos to pin the tag we just published. + make update-go-build-pins CONFIRM=true + fi diff --git a/.semaphore/semaphore.yml.d/blocks/10-rust-build.yml b/.semaphore/semaphore.yml.d/blocks/10-rust-build.yml new file mode 100644 index 00000000..d7dbf06f --- /dev/null +++ b/.semaphore/semaphore.yml.d/blocks/10-rust-build.yml @@ -0,0 +1,74 @@ +# calico/rust-build +- name: calico/rust-build image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-rust-build/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + prologue: + commands: + - export CALICO_RUST_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + jobs: + - name: Build calico/rust-build image + commands: + - | + if hack/image-cache.sh restore calico-rust-build ARCH; then + make -C images calico-rust-build-retag ARCH=$ARCH + else + make -C images calico-rust-build-image ARCH=$ARCH + hack/image-cache.sh store calico-rust-build ARCH + fi + matrix: + - env_var: ARCH + values: ["amd64", "arm64", "ppc64le", "s390x"] + +- name: calico/rust-build image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-rust-build/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - export CALICO_RUST_BUILD_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + # rust-build is versioned by its own toolchain version, not by the branch. + - export RUST_VERSION=$(yq -r '.rust.version' images/calico-rust-build/versions.yaml) + jobs: + - name: Build and publish calico/rust-build image + commands: + - | + if hack/image-cache.sh restore calico-rust-build ARCH; then + make -C images calico-rust-build-retag ARCH=$ARCH + else + make -C images calico-rust-build-image ARCH=$ARCH + hack/image-cache.sh store calico-rust-build ARCH + fi + - make -C images calico-rust-build-push VALIDARCHES=$ARCH ARCH=$ARCH BRANCH_NAME=${RUST_VERSION} CONFIRM=true + matrix: + - env_var: ARCH + values: ["amd64", "arm64", "ppc64le", "s390x"] + +- name: calico/rust-build manifests + dependencies: + - calico/rust-build image and publish + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-rust-build/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Push calico/rust-build multi-arch manifests + commands: + - make -C images push-calico-rust-build-manifests BRANCH_NAME=$(yq -r '.rust.version' images/calico-rust-build/versions.yaml) CONFIRM=true diff --git a/.semaphore/semaphore.yml.d/blocks/10-tinygo.yml b/.semaphore/semaphore.yml.d/blocks/10-tinygo.yml new file mode 100644 index 00000000..9cb8f7b9 --- /dev/null +++ b/.semaphore/semaphore.yml.d/blocks/10-tinygo.yml @@ -0,0 +1,180 @@ +# calico/tinygo +- name: calico/tinygo image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-tinygo/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + prologue: + commands: + - export CALICO_TINYGO_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + jobs: + - name: Build and test calico/tinygo + commands: + - | + if hack/image-cache.sh restore calico-tinygo ARCH; then + make -C images calico-tinygo-retag ARCH=$ARCH + else + make -C images calico-tinygo-image ARCH=$ARCH + hack/image-cache.sh store calico-tinygo ARCH + fi + # Compile the test fixture against the just-built image, then + # inspect the produced wasm's import section for two failure + # classes a downstream consumer (gateway/coraza-wasm) would + # otherwise hit: + # + # 1. wasm-ld undefined symbols at link time. The fixture + # imports github.com/wasilibs/{go-re2, go-libinjection} + # and uses each via os.Args-derived input so TinyGo's + # `-opt=2` optimizer cannot prove the calls dead. If a + # TinyGo or wasi-libc bump breaks the prebuilt-archive + # ABI, the build fails here. + # + # 2. Runtime `missing import: env.cre2_new` on Envoy. The + # assertion below counts `<- env.cre2_*` imports in the + # wasm; with the TinyGo-compatible go-re2 (currently + # v1.6.0) cre2 resolves internally to the bundled + # static archive and the count is zero. wasilibs/go-re2 + # v1.8.0+ removed TinyGo support (libre2 now needs + # Abseil with thread sync, which TinyGo's libc++ shim + # does not provide); a bump to such a version produces + # external env.cre2_* imports and the assertion fails + # here, at toolchain-image build time, not during a + # downstream WAF rebuild. + # + # The Makefile builds the image as `tinygo:latest-amd64` + # locally before retagging it as `calico/tinygo:${branch}-amd64` + # for the registry. Use the local tag here — the calico/- + # prefixed tag uses the branch, not "latest". + - | + docker run --rm \ + -v "$(pwd)/images/calico-tinygo/test/fixture:/src" \ + -v /tmp:/out \ + -w /src \ + tinygo:latest-${ARCH} \ + sh -c 'go mod download && \ + tinygo build \ + -target=wasip1 \ + -gc=precise -opt=2 -scheduler=none \ + -tags="nottinygc_envoy memoize_builders" \ + -o /out/fixture.wasm . && \ + file /out/fixture.wasm | grep -q "WebAssembly"' + - | + count=$(docker run --rm -v /tmp:/host ghcr.io/webassembly/wabt:1.0.36 \ + wasm-objdump -x /host/fixture.wasm | grep -c "<- env.cre2_" || true) + if [ "${count:-0}" -ne 0 ]; then + echo >&2 "FAIL: fixture wasm has ${count} env.cre2_* imports (expected 0)" + echo >&2 " A wasilibs/go-re2 bump likely lost TinyGo's bundled libcre2.a;" + echo >&2 " this wasm would fail to load in Envoy with 'missing import: env.cre2_new'." + docker run --rm -v /tmp:/host ghcr.io/webassembly/wabt:1.0.36 \ + wasm-objdump -x /host/fixture.wasm | grep "<- env.cre2_" >&2 || true + exit 1 + fi + echo "OK: fixture wasm resolves cre2_* internally (no env.cre2_* imports)" + matrix: + - env_var: ARCH + values: ["amd64", "arm64"] + +- name: calico/tinygo image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-tinygo/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + - export CALICO_TINYGO_IMAGETAG=${SEMAPHORE_GIT_WORKING_BRANCH} + # tinygo is versioned by its own toolchain version, not by the branch. + - export TINYGO_VERSION=$(yq -r '.tinygo.version' images/calico-tinygo/versions.yaml) + jobs: + - name: Build, test and publish calico/tinygo + commands: + - | + if hack/image-cache.sh restore calico-tinygo ARCH; then + make -C images calico-tinygo-retag ARCH=$ARCH + else + make -C images calico-tinygo-image ARCH=$ARCH + hack/image-cache.sh store calico-tinygo ARCH + fi + # Compile the test fixture against the just-built image, then + # inspect the produced wasm's import section for two failure + # classes a downstream consumer (gateway/coraza-wasm) would + # otherwise hit: + # + # 1. wasm-ld undefined symbols at link time. The fixture + # imports github.com/wasilibs/{go-re2, go-libinjection} + # and uses each via os.Args-derived input so TinyGo's + # `-opt=2` optimizer cannot prove the calls dead. If a + # TinyGo or wasi-libc bump breaks the prebuilt-archive + # ABI, the build fails here. + # + # 2. Runtime `missing import: env.cre2_new` on Envoy. The + # assertion below counts `<- env.cre2_*` imports in the + # wasm; with the TinyGo-compatible go-re2 (currently + # v1.6.0) cre2 resolves internally to the bundled + # static archive and the count is zero. wasilibs/go-re2 + # v1.8.0+ removed TinyGo support (libre2 now needs + # Abseil with thread sync, which TinyGo's libc++ shim + # does not provide); a bump to such a version produces + # external env.cre2_* imports and the assertion fails + # here, at toolchain-image build time, not during a + # downstream WAF rebuild. + # + # The Makefile builds the image as `tinygo:latest-amd64` + # locally before retagging it as `calico/tinygo:${branch}-amd64` + # for the registry. Use the local tag here — the calico/- + # prefixed tag uses the branch, not "latest". + - | + docker run --rm \ + -v "$(pwd)/images/calico-tinygo/test/fixture:/src" \ + -v /tmp:/out \ + -w /src \ + tinygo:latest-${ARCH} \ + sh -c 'go mod download && \ + tinygo build \ + -target=wasip1 \ + -gc=precise -opt=2 -scheduler=none \ + -tags="nottinygc_envoy memoize_builders" \ + -o /out/fixture.wasm . && \ + file /out/fixture.wasm | grep -q "WebAssembly"' + - | + count=$(docker run --rm -v /tmp:/host ghcr.io/webassembly/wabt:1.0.36 \ + wasm-objdump -x /host/fixture.wasm | grep -c "<- env.cre2_" || true) + if [ "${count:-0}" -ne 0 ]; then + echo >&2 "FAIL: fixture wasm has ${count} env.cre2_* imports (expected 0)" + echo >&2 " A wasilibs/go-re2 bump likely lost TinyGo's bundled libcre2.a;" + echo >&2 " this wasm would fail to load in Envoy with 'missing import: env.cre2_new'." + docker run --rm -v /tmp:/host ghcr.io/webassembly/wabt:1.0.36 \ + wasm-objdump -x /host/fixture.wasm | grep "<- env.cre2_" >&2 || true + exit 1 + fi + echo "OK: fixture wasm resolves cre2_* internally (no env.cre2_* imports)" + - make -C images calico-tinygo-push ARCH=$ARCH VALIDARCHES=$ARCH BRANCH_NAME=${TINYGO_VERSION} CONFIRM=true + matrix: + - env_var: ARCH + values: ["amd64", "arm64"] + +- name: calico/tinygo manifests + dependencies: + - calico/tinygo image and publish + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-tinygo/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Push calico/tinygo multi-arch manifest + commands: + - make -C images push-calico-tinygo-manifests BRANCH_NAME=$(yq -r '.tinygo.version' images/calico-tinygo/versions.yaml) CONFIRM=true diff --git a/.semaphore/semaphore.yml.d/blocks/20-base.yml b/.semaphore/semaphore.yml.d/blocks/20-base.yml new file mode 100644 index 00000000..7eb98f40 --- /dev/null +++ b/.semaphore/semaphore.yml.d/blocks/20-base.yml @@ -0,0 +1,58 @@ +# calico/base +- name: calico/base image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-base/', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + jobs: + - name: Build calico/base image + commands: + - | + if hack/image-cache.sh restore calico-base $ARCH; then + make -C images calico-base-retag ARCH=$ARCH + else + make -C images calico-base-image ARCH=$ARCH + hack/image-cache.sh store calico-base $ARCH + fi + matrix: + - env_var: ARCH + values: ["amd64", "arm64", "ppc64le", "s390x"] + +- name: calico/base image and publish + dependencies: + - Checks + # Deliberately not change-gated, and master only, which is what the promotion + # it replaces did. base exists to carry current UBI packages: its Dockerfile + # barely changes, but every rebuild picks up `microdnf upgrade`. Gating it on + # its own sources would republish it only when the Dockerfile moved, which is + # rarely. The image cache still applies, so repeated master pushes inside the + # cache window reuse one build rather than producing near-identical tags. + run: + when: "pull_request !~ '.*' and branch = 'master'" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + # Every architecture and both UBI versions must share one tag, so base + # builds and publishes in a single job rather than a per-arch matrix. + - name: BRANCH_NAME + value: ${SEMAPHORE_JOB_CREATION_TIME} + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Build and publish calico/base multi-arch images + commands: + - | + for arch in amd64 arm64 ppc64le s390x; do + if hack/image-cache.sh restore calico-base "$arch"; then + make -C images calico-base-retag ARCH="$arch" + else + make -C images calico-base-image ARCH="$arch" + hack/image-cache.sh store calico-base "$arch" + fi + done + - make -C images calico-base-push CONFIRM=true diff --git a/.semaphore/semaphore.yml.d/blocks/30-binfmt.yml b/.semaphore/semaphore.yml.d/blocks/30-binfmt.yml new file mode 100644 index 00000000..4cedf48f --- /dev/null +++ b/.semaphore/semaphore.yml.d/blocks/30-binfmt.yml @@ -0,0 +1,43 @@ +# calico/binfmt +- name: calico/binfmt image + dependencies: + - Checks + run: + when: "pull_request =~ '.*' and change_in(['/images/calico-binfmt/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + jobs: + - name: Build calico/binfmt amd64 image + commands: + - | + if hack/image-cache.sh restore calico-binfmt amd64; then + make -C images calico-binfmt-retag + else + make -C images calico-binfmt-image ARCH=amd64 + hack/image-cache.sh store calico-binfmt amd64 + fi + +- name: calico/binfmt image and publish + dependencies: + - Checks + run: + when: "pull_request !~ '.*' and (branch = 'master' or branch =~ '^go1\\.') and change_in(['/images/calico-binfmt/', '/cmd/', '/go.mod', '/go.sum', '/images/Makefile', '/lib.Makefile', '/Makefile.common'])" + task: + secrets: + - name: docker + env_vars: + - name: DEV_REGISTRIES + value: calico + prologue: + commands: + - echo $DOCKER_TOKEN | docker login --username "$DOCKER_USER" --password-stdin + jobs: + - name: Build and publish calico/binfmt amd64 image + commands: + - | + if hack/image-cache.sh restore calico-binfmt amd64; then + make -C images calico-binfmt-retag + else + make -C images calico-binfmt-image ARCH=amd64 + hack/image-cache.sh store calico-binfmt amd64 + fi + - make -C images calico-binfmt-push CONFIRM=true diff --git a/Makefile b/Makefile index 9f03f142..a0432ac2 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,19 @@ clean: $(MAKE) -C cmd clean $(MAKE) -C images clean +# The Semaphore pipeline is assembled from fragments so each image's blocks can +# be edited on their own. Run this after changing anything under +# .semaphore/semaphore.yml.d and commit the regenerated file. +.PHONY: gen-semaphore-yaml +gen-semaphore-yaml: + hack/gen-semaphore-yaml.sh + +# check-semaphore-yaml fails when the committed pipeline does not match its +# fragments, which is how an edit to the generated file gets caught. +.PHONY: check-semaphore-yaml +check-semaphore-yaml: + hack/gen-semaphore-yaml.sh --check + .PHONY: update-go-build-pins update-go-build-pins: SEMAPHORE_AUTO_PIN_UPDATE_PROJECT_IDS=$(SEMAPHORE_CALICO_PROJECT_ID) \ diff --git a/hack/gen-semaphore-yaml.sh b/hack/gen-semaphore-yaml.sh new file mode 100755 index 00000000..5183075a --- /dev/null +++ b/hack/gen-semaphore-yaml.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# Assemble .semaphore/semaphore.yml from the fragments in +# .semaphore/semaphore.yml.d, the way projectcalico/calico does. +# +# Top-level fragments are concatenated in filename order, so the numeric +# prefixes set the order. When 09-blocks.yml is reached (it contains just the +# `blocks:` key) every file under semaphore.yml.d/blocks/ follows it, also in +# filename order, indented by two spaces. Block fragments are written at column +# zero so each one is valid YAML on its own and an editor can lint it. + +set -eu + +# --check reports whether the committed file matches its fragments instead of +# rewriting it. It compares generated output against the file on disk, so it is +# independent of what git happens to have staged. +check=false +if [ "${1:-}" = "--check" ]; then + check=true +fi + +cd "$(git rev-parse --show-toplevel)" + +d=.semaphore/semaphore.yml.d +out=.semaphore/semaphore.yml +tmp=$(mktemp) +trap 'rm -f "$tmp"' EXIT + +cat >"$tmp" <<'HEADER' +# !! WARNING, DO NOT EDIT !! This file is generated from the fragments +# in /.semaphore/semaphore.yml.d. To update, modify the relevant +# fragment and then run 'make gen-semaphore-yaml'. +HEADER + +for f in "$d"/*.yml; do + cat "$f" >>"$tmp" + if [ "$(basename "$f")" = "09-blocks.yml" ]; then + for b in "$d"/blocks/*.yml; do + # Indent to sit under `blocks:`, leaving blank lines empty rather + # than filling them with trailing spaces. + sed 's/^\(.\)/ \1/' "$b" >>"$tmp" + done + fi +done + +# A fragment with a typo must not land in the tree. +if command -v python3 >/dev/null 2>&1; then + python3 -c 'import sys,yaml; yaml.safe_load(open(sys.argv[1]))' "$tmp" +fi + +if [ "$check" = true ]; then + if ! diff -u "$out" "$tmp"; then + echo >&2 "ERROR: $out does not match its fragments. Run 'make gen-semaphore-yaml'." + exit 1 + fi + echo "$out is up to date" + exit 0 +fi + +mv "$tmp" "$out" +trap - EXIT +echo "generated $out" diff --git a/hack/image-build-id.sh b/hack/image-build-id.sh new file mode 100755 index 00000000..b4b879d1 --- /dev/null +++ b/hack/image-build-id.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Print a stable content hash ("build ID") for one toolchain image. +# +# The hash covers every committed file that can change the image, so the same +# source tree always maps to the same ID. CI addresses its staged per-arch +# builds by this ID, which lets a pipeline skip a rebuild when the registry +# already holds an image for the tree it was handed. +# +# The hash reads the committed tree, not the working directory, so it ignores +# generated files under bin/ but also ignores uncommitted edits. + +set -eu + +usage() { + echo "usage: $0 " >&2 + exit 1 +} + +[ $# -eq 1 ] || usage + +case "$1" in +calico-base) + deps="images/calico-base" + ;; +calico-binfmt) + # The binfmt binary is built from cmd/ and copied into the image. + deps="images/calico-binfmt cmd" + ;; +calico-go-build) + # The semvalidator binary is built from cmd/ and copied into the image. + deps="images/calico-go-build cmd" + ;; +calico-rust-build) + deps="images/calico-rust-build" + ;; +calico-tinygo) + deps="images/calico-tinygo" + ;; +*) + usage + ;; +esac + +# These carry the build args and docker invocation shared by every image. +deps="$deps images/Makefile lib.Makefile" + +for dep in $deps; do + git rev-parse "HEAD:$dep" +done | sha256sum | cut -c1-12 diff --git a/hack/image-cache.sh b/hack/image-cache.sh new file mode 100755 index 00000000..f821f203 --- /dev/null +++ b/hack/image-cache.sh @@ -0,0 +1,151 @@ +#!/bin/bash + +# Restore or store a built toolchain image as a compressed tarball in GCS. +# +# The cache carries one change from its pull request to the merge that follows: +# the two share a source tree, so they share a cache entry, and the merge does +# not repeat a build the pull request already did. That matters most for arm64, +# ppc64le and s390x, which build under QEMU emulation. +# +# An entry is keyed by two things, because the source tree alone does not +# determine the image: +# +# 1. The build ID: a hash of the committed files that can change this image. +# 2. The digests of the base images it is built FROM. Those are floating tags, +# and a new release means a rebuild would produce a different image. UBI is +# rebuilt roughly daily; almalinux:9 and fedora:44 roughly quarterly. +# +# That covers new base releases but not everything, because dnf and microdnf +# pull from repositories that ship updates between base image retags -- +# almalinux:9 can sit unchanged for months while its repositories do not. +# MAX_AGE_HOURS is the backstop for that drift, and it is also the staleness +# bound on anything published, which before this cache was always built fresh. +# +# Usage: +# image-cache.sh restore # exit 0 on hit, 1 on miss +# image-cache.sh store + +set -eu + +usage() { + echo "usage: $0 " >&2 + exit 2 +} + +[ $# -eq 3 ] || usage +action=$1 +image=$2 +arch=$3 + +case "$action" in +restore | store) ;; +*) usage ;; +esac + +cd "$(git rev-parse --show-toplevel)" + +MAX_AGE_HOURS=${MAX_AGE_HOURS:-72} + +# A cache that cannot be reached must never fail a build. A restore reports a +# miss so the caller builds instead; a store quietly does nothing. +unavailable() { + echo "$1, skipping image cache" >&2 + if [ "$action" = restore ]; then + exit 1 + fi + exit 0 +} + +[ -n "${GCS_IMAGE_CACHE_BUCKET:-}" ] || unavailable "GCS_IMAGE_CACHE_BUCKET is not set" + +# Local tags produced by the images/Makefile build targets. calico-base builds +# one image per UBI version, so it carries two tags per architecture. +case "$image" in +calico-base) + tags="base:ubi9-latest-${arch} base:ubi10-latest-${arch}" + ;; +calico-binfmt) + qemu_version=$(yq -r '.qemu.version' images/calico-binfmt/versions.yaml) + tags="binfmt:qemu-v${qemu_version}-amd64" + ;; +calico-go-build) + tags="go-build:latest-${arch}" + ;; +calico-rust-build) + tags="rust-build:latest-${arch}" + ;; +calico-tinygo) + tags="tinygo:latest-${arch}" + ;; +*) + usage + ;; +esac + +build_id=$(hack/image-build-id.sh "$image") + +# calico-base substitutes UBI_VERSION at build time; expand it to the versions +# the Makefile actually builds. +bases=$(grep '^FROM' "images/${image}/Dockerfile" | awk '{print $2}' | grep -v '^scratch$' | sort -u) +if [ "$image" = calico-base ]; then + bases="${bases//\$\{UBI_VERSION\}/ubi9} ${bases//\$\{UBI_VERSION\}/ubi10}" +fi + +digests="" +for base in $bases; do + digest=$(docker buildx imagetools inspect "$base" --format '{{.Manifest.Digest}}' 2>/dev/null || true) + [ -n "$digest" ] || unavailable "cannot resolve the digest of $base" + digests="${digests}${digest}" +done +base_id=$(printf '%s' "$digests" | sha256sum | cut -c1-8) + +object="gs://${GCS_IMAGE_CACHE_BUCKET}/images/${image}-${build_id}-${base_id}-${arch}.tar.zst" +tarball="/tmp/${image}-${arch}.tar" + +case "$action" in +restore) + # Read timeCreated from the object metadata rather than parsing the + # human-formatted `ls` output. A missing or unreadable timestamp is a miss + # too: rebuilding is always safe. + created=$(gcloud storage objects describe "$object" --format="value(timeCreated)" 2>/dev/null || true) + if [ -z "$created" ]; then + echo "image cache miss: $object" + exit 1 + fi + created_epoch=$(date -u -d "$created" +%s 2>/dev/null || echo 0) + if [ "$created_epoch" -eq 0 ]; then + echo "image cache timestamp unreadable, rebuilding: $object" + exit 1 + fi + age_hours=$((($(date -u +%s) - created_epoch) / 3600)) + if [ "$age_hours" -ge "$MAX_AGE_HOURS" ]; then + echo "image cache stale (${age_hours}h >= ${MAX_AGE_HOURS}h), rebuilding: $object" + exit 1 + fi + echo "image cache hit (${age_hours}h old): $object" + gcloud storage cp "$object" "${tarball}.zst" + zstd -d --rm -o "$tarball" "${tarball}.zst" + docker load -i "$tarball" + rm -f "$tarball" + ;; +store) + # Never let a fork populate the cache. A fork could otherwise upload an + # image that does not match the tree its build ID names, and a later merge + # of that innocuous-looking tree would publish it. + if [ -n "${SEMAPHORE_GIT_PR_SLUG:-}" ] && + [ "${SEMAPHORE_GIT_PR_SLUG}" != "${SEMAPHORE_GIT_REPO_SLUG:-}" ]; then + echo "forked pull request, not storing to the image cache" + exit 0 + fi + if gcloud storage ls "$object" >/dev/null 2>&1; then + echo "image cache already populated: $object" + exit 0 + fi + # shellcheck disable=SC2086 + docker save $tags -o "$tarball" + zstd -3 --rm "$tarball" + gcloud storage cp "${tarball}.zst" "$object" + rm -f "${tarball}.zst" + echo "image cache stored: $object" + ;; +esac diff --git a/images/Makefile b/images/Makefile index 1a54d6f5..ad5beaa3 100644 --- a/images/Makefile +++ b/images/Makefile @@ -46,13 +46,27 @@ calico-base-image-%: register $(DOCKER_BUILD) --build-arg LDSONAME=$(LDSONAME) --build-arg=UBI_VERSION=$* -t $(CALICO_BASE):$*-latest-$(ARCH) -f calico-base/Dockerfile calico-base/ $(MAKE) BUILD_IMAGES=$(CALICO_BASE) retag-build-images-with-registries VALIDARCHES=$(ARCH) LATEST_IMAGE_TAG=$*-latest IMAGETAG=$*-latest +# calico-base-retag applies the registry tags to images already in the local +# daemon, whether just built or loaded from the image cache. +.PHONY: calico-base-retag +calico-base-retag: $(addprefix calico-base-retag-,$(UBI_VERSIONS)) +calico-base-retag-%: + $(MAKE) BUILD_IMAGES=$(CALICO_BASE) retag-build-images-with-registries VALIDARCHES=$(ARCH) LATEST_IMAGE_TAG=$*-latest IMAGETAG=$*-latest + .PHONY: calico-base-image-all calico-base-image-all: $(addprefix sub-calico-base-image-,$(VALIDARCHES)) sub-calico-base-image-%: $(MAKE) calico-base-image ARCH=$* .PHONY: calico-base-cd -calico-base-cd: calico-base-image-all var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME +calico-base-cd: calico-base-image-all calico-base-push + +# calico-base-push publishes images that are already in the local daemon. CI +# builds and publishes in one job, so a push never rebuilds. All architectures +# and both UBI versions share a single BRANCH_NAME, so they are pushed together +# rather than per-arch. +.PHONY: calico-base-push +calico-base-push: var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME $(foreach version,$(UBI_VERSIONS),$(MAKE) BUILD_IMAGES=$(CALICO_BASE) retag-build-images-with-registries push-images-to-registries push-manifests LATEST_IMAGE_TAG=$(version)-latest IMAGETAG=$(version)-$(BRANCH_NAME) EXCLUDEARCH="$(EXCLUDEARCH)";) ################################################################################ @@ -64,6 +78,12 @@ CALICO_GO_BUILD_IMAGETAG ?= latest .PHONY: calico-go-build-image calico-go-build-image: register build $(DOCKER_BUILD) -t $(CALICO_GO_BUILD):latest-$(ARCH) -f calico-go-build/Dockerfile calico-go-build/ + $(MAKE) calico-go-build-retag + +# calico-go-build-retag applies the registry tags to an image that is already in +# the local daemon, whether it was just built or loaded from the image cache. +.PHONY: calico-go-build-retag +calico-go-build-retag: $(MAKE) BUILD_IMAGES=$(CALICO_GO_BUILD) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=$(CALICO_GO_BUILD_IMAGETAG) .PHONY: calico-go-build-image-all @@ -72,7 +92,11 @@ sub-calico-go-build-image-%: $(MAKE) calico-go-build-image ARCH=$* .PHONY: calico-go-build-cd -calico-go-build-cd: calico-go-build-image var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME +calico-go-build-cd: calico-go-build-image calico-go-build-push + +# calico-go-build-push publishes an image that is already in the local daemon. +.PHONY: calico-go-build-push +calico-go-build-push: var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME $(MAKE) BUILD_IMAGES=$(CALICO_GO_BUILD) retag-build-images-with-registries push-images-to-registries IMAGETAG=$(BRANCH_NAME) EXCLUDEARCH="$(EXCLUDEARCH)" ifeq ($(BRANCH_NAME),master) $(MAKE) BUILD_IMAGES=$(CALICO_GO_BUILD) retag-build-images-with-registries push-images-to-registries IMAGETAG=latest EXCLUDEARCH="$(EXCLUDEARCH)" @@ -99,8 +123,16 @@ calico-binfmt-image: build $(DOCKER_BUILD) --build-arg QEMU_VERSION=$(QEMU_VERSION) -t $(CALICO_BINFMT):$(CALICO_BINFMT_IMAGETAG)-amd64 -f calico-binfmt/Dockerfile calico-binfmt/ $(MAKE) BUILD_IMAGES=$(CALICO_BINFMT) retag-build-images-with-registries VALIDARCHES=amd64 LATEST_IMAGE_TAG=$(CALICO_BINFMT_IMAGETAG) IMAGETAG=$(CALICO_BINFMT_IMAGETAG) +.PHONY: calico-binfmt-retag +calico-binfmt-retag: + $(MAKE) BUILD_IMAGES=$(CALICO_BINFMT) retag-build-images-with-registries VALIDARCHES=amd64 LATEST_IMAGE_TAG=$(CALICO_BINFMT_IMAGETAG) IMAGETAG=$(CALICO_BINFMT_IMAGETAG) + .PHONY: calico-binfmt-cd -calico-binfmt-cd: calico-binfmt-image var-require-one-of-CONFIRM-DRYRUN +calico-binfmt-cd: calico-binfmt-image calico-binfmt-push + +# calico-binfmt-push publishes an image that is already in the local daemon. +.PHONY: calico-binfmt-push +calico-binfmt-push: var-require-one-of-CONFIRM-DRYRUN $(MAKE) BUILD_IMAGES=$(CALICO_BINFMT) retag-build-images-with-registries push-images-to-registries IMAGETAG=$(CALICO_BINFMT_IMAGETAG) LATEST_IMAGE_TAG=$(CALICO_BINFMT_IMAGETAG) VALIDARCHES=amd64 EXCLUDEARCH="$(EXCLUDEARCH)" ################################################################################ @@ -114,13 +146,21 @@ calico-rust-build-image: register $(DOCKER_BUILD) -t $(CALICO_RUST_BUILD):latest-$(ARCH) -f calico-rust-build/Dockerfile calico-rust-build/ $(MAKE) BUILD_IMAGES=$(CALICO_RUST_BUILD) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=$(CALICO_RUST_BUILD_IMAGETAG) +.PHONY: calico-rust-build-retag +calico-rust-build-retag: + $(MAKE) BUILD_IMAGES=$(CALICO_RUST_BUILD) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=$(CALICO_RUST_BUILD_IMAGETAG) + .PHONY: calico-rust-build-image-all calico-rust-build-image-all: $(addprefix sub-calico-rust-build-image-,$(VALIDARCHES)) sub-calico-rust-build-image-%: $(MAKE) calico-rust-build-image ARCH=$* .PHONY: calico-rust-build-cd -calico-rust-build-cd: calico-rust-build-image var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME +calico-rust-build-cd: calico-rust-build-image calico-rust-build-push + +# calico-rust-build-push publishes an image that is already in the local daemon. +.PHONY: calico-rust-build-push +calico-rust-build-push: var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME $(MAKE) BUILD_IMAGES=$(CALICO_RUST_BUILD) retag-build-images-with-registries push-images-to-registries IMAGETAG=$(BRANCH_NAME) EXCLUDEARCH="$(EXCLUDEARCH)" .PHONY: push-calico-rust-build-manifests @@ -139,13 +179,21 @@ calico-tinygo-image: register $(DOCKER_BUILD) -t $(CALICO_TINYGO):latest-$(ARCH) -f calico-tinygo/Dockerfile calico-tinygo/ $(MAKE) BUILD_IMAGES=$(CALICO_TINYGO) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=$(CALICO_TINYGO_IMAGETAG) +.PHONY: calico-tinygo-retag +calico-tinygo-retag: + $(MAKE) BUILD_IMAGES=$(CALICO_TINYGO) retag-build-images-with-registries VALIDARCHES=$(ARCH) IMAGETAG=$(CALICO_TINYGO_IMAGETAG) + .PHONY: calico-tinygo-image-all calico-tinygo-image-all: $(addprefix sub-calico-tinygo-image-,amd64 arm64) sub-calico-tinygo-image-%: $(MAKE) calico-tinygo-image ARCH=$* .PHONY: calico-tinygo-cd -calico-tinygo-cd: calico-tinygo-image var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME +calico-tinygo-cd: calico-tinygo-image calico-tinygo-push + +# calico-tinygo-push publishes an image that is already in the local daemon. +.PHONY: calico-tinygo-push +calico-tinygo-push: var-require-one-of-CONFIRM-DRYRUN var-require-all-BRANCH_NAME $(MAKE) BUILD_IMAGES=$(CALICO_TINYGO) retag-build-images-with-registries push-images-to-registries IMAGETAG=$(BRANCH_NAME) VALIDARCHES=$(ARCH) .PHONY: push-calico-tinygo-manifests