From 124b2b1c6498a38d0929ab753bc7ef14c8ce7015 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Tue, 21 Jul 2026 15:03:37 -0700 Subject: [PATCH 01/10] feat(packaging): Add a busybox init-container installer image as a fourth distribution channel. Builds a small busybox image bundling both plugins under `/opt/clp-plugin-presto-connector`; its entrypoint copies the coordinator JAR and/or native worker `.so` into mounted volumes named by `COORDINATOR_PLUGIN_INSTALL_PATH` / `WORKER_PLUGIN_INSTALL_PATH` (at least one required, else it errors), so a coordinator pod and a worker pod run the same image with different config. `build-installer-init-image.sh` builds the image host-side from a package tarball (docker isn't available inside the build-env container). `task package` now also builds and loads it locally, and the CI workflow builds it per architecture on native runners and publishes a multi-arch `:` manifest from the default branch and tags. Moves `image_repo_from_origin` into `dependency-image/utils.sh` so both `build-dependency-image.sh` and the new script share one GHCR-repo derivation. --- .github/workflows/build-packages.yaml | 109 +++++++++++++++ tools/build-packages/README.md | 16 ++- .../build-packages/build-dependency-image.sh | 18 --- .../build-installer-init-image.sh | 126 ++++++++++++++++++ tools/build-packages/build-packages.sh | 13 ++ .../build-packages/dependency-image/utils.sh | 18 +++ tools/build-packages/image/Dockerfile | 18 +++ tools/build-packages/image/entrypoint.sh | 49 +++++++ 8 files changed, 347 insertions(+), 20 deletions(-) create mode 100755 tools/build-packages/build-installer-init-image.sh create mode 100644 tools/build-packages/image/Dockerfile create mode 100755 tools/build-packages/image/entrypoint.sh diff --git a/.github/workflows/build-packages.yaml b/.github/workflows/build-packages.yaml index c20fe59..313a9e0 100644 --- a/.github/workflows/build-packages.yaml +++ b/.github/workflows/build-packages.yaml @@ -4,6 +4,8 @@ # resolve-version use the input version or derive it from pom.xml # build-dependency-image ensure the hash-tagged build-env image exists # build build .deb / .rpm / .tar.gz artifacts per architecture +# image build the busybox installer image per arch (push on main/tags) +# manifest combine per-arch images into a multi-arch tag (main/tags only) # # See tools/build-packages/README.md for package-build details. @@ -58,6 +60,7 @@ jobs: runs-on: "ubuntu-24.04" outputs: version: "${{steps.version.outputs.version}}" + publish: "${{steps.publish.outputs.publish}}" steps: - name: "Check out repository" uses: "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" # v7.0.0 @@ -88,6 +91,19 @@ jobs: echo "version=${version}" >> "$GITHUB_OUTPUT" echo "::notice::Package version: ${version}" + - name: "Decide whether to publish images" + id: "publish" + env: + DEFAULT_BRANCH: "${{github.event.repository.default_branch}}" + run: |- + # Publish images only from the default branch and version tags so feature + # branches don't overwrite shared tags. + if [[ ${GITHUB_REF_NAME} == "${DEFAULT_BRANCH}" || ${GITHUB_REF_TYPE} == tag ]]; then + echo "publish=true" >> "$GITHUB_OUTPUT" + else + echo "publish=false" >> "$GITHUB_OUTPUT" + fi + build-dependency-image: uses: "./.github/workflows/build-dependency-image.yaml" permissions: @@ -182,3 +198,96 @@ jobs: path: "packages/${{steps.packages.outputs.tar_gz_filename}}" if-no-files-found: "error" retention-days: 14 + + # Build the busybox installer image per architecture from the tarball artifact. This runs + # on the native runner (not inside the build-env container) so docker/buildx is available. + image: + needs: ["build", "resolve-version"] + permissions: + contents: "read" + packages: "write" + strategy: + fail-fast: false + matrix: + include: + - arch: "amd64" + runner: "ubuntu-24.04" + - arch: "arm64" + runner: "ubuntu-24.04-arm" + runs-on: "${{matrix.runner}}" + timeout-minutes: 30 + env: + ARCH: "${{matrix.arch}}" + steps: + - name: "Check out repository" + uses: "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" # v7.0.0 + with: + persist-credentials: false + + - name: "Download tarball artifact" + uses: "actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53" # v6.0.0 + with: + pattern: "*-linux-${{matrix.arch}}.tar.gz" + path: "dist" + merge-multiple: true + + - name: "Log in to GHCR" + if: "${{needs.resolve-version.outputs.publish == 'true'}}" + uses: "docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0" # v4.4.0 + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Set up Docker Buildx" + uses: "docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c" # v4.2.0 + + - name: "Build the installer image (and push when publishing)" + env: + PUBLISH: "${{needs.resolve-version.outputs.publish}}" + run: |- + image_repo="ghcr.io/$(printf '%s' "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]')" + shopt -s nullglob + tarballs=(dist/*-linux-"${ARCH}".tar.gz) + if (( ${#tarballs[@]} != 1 )); then + echo "::error::Expected exactly one tarball for ${ARCH}, found ${#tarballs[@]}" + exit 1 + fi + # Push from the default branch / tags; otherwise just build to validate. + output="--load" + if [[ "${PUBLISH}" == "true" ]]; then + output="--push" + fi + bash tools/build-packages/build-installer-init-image.sh \ + --tarball "${tarballs[0]}" \ + --arch "${ARCH}" \ + --repo "${image_repo}" \ + "${output}" + + # Combine the per-architecture installer images into one multi-arch version tag. + manifest: + needs: ["image", "resolve-version"] + if: "${{needs.resolve-version.outputs.publish == 'true'}}" + permissions: + packages: "write" + runs-on: "ubuntu-24.04" + timeout-minutes: 15 + env: + VERSION: "${{needs.resolve-version.outputs.version}}" + steps: + - name: "Log in to GHCR" + uses: "docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0" # v4.4.0 + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Create multi-arch manifest" + run: |- + image_repo="ghcr.io/$(printf '%s' "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]')" + # Docker tags allow only [A-Za-z0-9_.-]; sanitize to match the build script. + tag_version="${VERSION//[^A-Za-z0-9_.-]/_}" + docker buildx imagetools create \ + --tag "${image_repo}:${tag_version}" \ + "${image_repo}:${tag_version}-amd64" \ + "${image_repo}:${tag_version}-arm64" diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md index d3abc22..5767ec3 100644 --- a/tools/build-packages/README.md +++ b/tools/build-packages/README.md @@ -1,7 +1,8 @@ # CLP Presto connector packaging -This directory builds installable `.deb`, `.rpm`, and `.tar.gz` artifacts for the CLP -Presto connector (coordinator + worker) on `amd64` and `arm64`. +This directory builds installable `.deb`, `.rpm`, and `.tar.gz` artifacts, plus a busybox +init-container installer image, for the CLP Presto connector (coordinator + worker) on +`amd64` and `arm64`. CI packaging runs `tools/build-packages/internal/container/build-artifacts.sh` through `.github/workflows/build-packages.yaml`. Local builds use @@ -25,6 +26,17 @@ task package A thin wrapper over `./tools/build-packages/build-packages.sh` (call that directly if `go-task` isn't installed). Both accept `--output DIR`, `--version VER`, and `--with-ca-certs`; with the task, put `--` before the flags: `task package -- --output DIR`. +### Installer image + +`task package` also builds and loads a busybox init-container image that bundles both plugins. Its entrypoint copies each component into a mounted volume named by `COORDINATOR_PLUGIN_INSTALL_PATH` / `WORKER_PLUGIN_INSTALL_PATH` (set either or both): + +```bash +docker run --rm -e WORKER_PLUGIN_INSTALL_PATH=/plugins -v "$(pwd)/plugins:/plugins" \ + ghcr.io/y-scope/clp-plugin-presto-connector:- +``` + +Run `build-installer-init-image.sh --help` to build it standalone from any package tarball. + The build runs inside a hash-tagged **build-env image** (`env-`) based on `manylinux_2_28`. `build-dependency-image.sh` resolves it from the local Docker cache, this repository's GHCR package, or a local build, reusing the cached diff --git a/tools/build-packages/build-dependency-image.sh b/tools/build-packages/build-dependency-image.sh index 13c36d4..e00d51b 100755 --- a/tools/build-packages/build-dependency-image.sh +++ b/tools/build-packages/build-dependency-image.sh @@ -19,24 +19,6 @@ set -o pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" source "${script_dir}/dependency-image/utils.sh" -# Derive this repo's GHCR namespace from its GitHub origin remote. -image_repo_from_origin() { - local remote_url owner_repo - remote_url="$(git -C "${_REPO_ROOT}" remote get-url origin)" - case "${remote_url}" in - https://github.com/*) owner_repo="${remote_url#https://github.com/}" ;; - git@github.com:*) owner_repo="${remote_url#git@github.com:}" ;; - ssh://git@github.com/*) owner_repo="${remote_url#ssh://git@github.com/}" ;; - *) - echo >&2 "ERROR: can't derive GHCR image repo from origin remote: ${remote_url}" - echo >&2 " Expected a github.com remote." - exit 1 - ;; - esac - owner_repo="${owner_repo%.git}" - printf 'ghcr.io/%s\n' "$(printf '%s' "${owner_repo}" | tr '[:upper:]' '[:lower:]')" -} - host_platform() { case "$(uname -m)" in x86_64) printf 'linux/amd64\n' ;; diff --git a/tools/build-packages/build-installer-init-image.sh b/tools/build-packages/build-installer-init-image.sh new file mode 100755 index 0000000..d0d375e --- /dev/null +++ b/tools/build-packages/build-installer-init-image.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash + +# Builds the busybox init-container installer image from a connector package tarball. +# +# The image bundles both plugins (coordinator JAR + native worker .so and lib/) and, when +# run, installs each into a mounted target directory. See tools/build-packages/README.md. +# +# Reusable by local builds (build-packages.sh, --load) and CI (--push). Prints the built +# image reference to stdout. +# +# Requires: docker (with buildx), git, tar. + +set -o errexit +set -o nounset +set -o pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +image_dir="${script_dir}/image" + +# Shared helpers: image_repo_from_origin (GHCR repo derivation) and _REPO_ROOT. +source "${script_dir}/dependency-image/utils.sh" + +show_help() { + cat <<'EOF' +Usage: ./tools/build-packages/build-installer-init-image.sh --tarball FILE [OPTIONS] + +Builds the busybox init-container installer image from a connector package tarball +(clp-plugin-presto-connector--linux-.tar.gz). + +Options: + --tarball FILE Package tarball to build the image from (required) + --version VER Image version tag (default: parsed from the tarball name) + --arch ARCH amd64 or arm64 (default: parsed from the tarball name) + --repo REPO Image repository (default: derived from the git origin remote, + e.g. ghcr.io/y-scope/clp-plugin-presto-connector) + --push Push the image to the registry (default: --load into local docker) + --load Load the image into the local docker daemon (default) + --help Show this help + +See tools/build-packages/README.md for details. +EOF +} + +die() { + echo >&2 "ERROR: $*" + exit 1 +} + +require_value() { + [[ -n "${2:-}" ]] || die "$1 requires a value" +} + +# ── Parse arguments ─────────────────────────────────────────────────────────── + +tarball="" +version="" +arch="" +repo="" +output="--load" + +while [[ $# -gt 0 ]]; do + case $1 in + --tarball) require_value "$1" "${2:-}"; tarball="$2"; shift 2 ;; + --version) require_value "$1" "${2:-}"; version="$2"; shift 2 ;; + --arch) require_value "$1" "${2:-}"; arch="$2"; shift 2 ;; + --repo) require_value "$1" "${2:-}"; repo="$2"; shift 2 ;; + --push) output="--push"; shift ;; + --load) output="--load"; shift ;; + --help) show_help; exit 0 ;; + *) die "unknown option: $1 (use --help for usage)" ;; + esac +done + +[[ -n "${tarball}" ]] || die "--tarball is required (use --help for usage)" +[[ -f "${tarball}" ]] || die "tarball not found: ${tarball}" + +command -v docker &>/dev/null || die "docker is required" +docker buildx version &>/dev/null || die "docker buildx is required" + +# ── Resolve version and arch from the tarball name when not given ────────────── + +# Tarball name format: clp-plugin-presto-connector--linux-.tar.gz +tar_base="$(basename "${tarball}")" +tar_base="${tar_base%.tar.gz}" +name_rest="${tar_base#clp-plugin-presto-connector-}" +if [[ "${name_rest}" == "${tar_base}" || "${name_rest}" != *-linux-* ]]; then + die "cannot parse tarball name '${tar_base}'; pass --version and --arch explicitly" +fi +[[ -n "${arch}" ]] || arch="${name_rest##*-linux-}" +[[ -n "${version}" ]] || version="${name_rest%-linux-"${arch}"}" + +case "${arch}" in + amd64) platform="linux/amd64" ;; + arm64) platform="linux/arm64" ;; + *) die "unsupported arch: ${arch} (expected amd64 or arm64)" ;; +esac + +[[ -n "${repo}" ]] || repo="$(image_repo_from_origin)" + +# Docker tags allow only [A-Za-z0-9_.-]; sanitize any other version characters (e.g. '+'). +tag_version="${version//[^A-Za-z0-9_.-]/_}" +image="${repo}:${tag_version}-${arch}" + +# ── Assemble a self-contained build context and build ───────────────────────── + +context_dir="$(mktemp -d)" +trap 'rm -rf "${context_dir}"' EXIT + +# Extract the install tree so coordinator/ and worker/ sit at the context root, matching the +# Dockerfile's COPY paths. --strip-components=1 drops the versioned top-level directory. +tar -xzf "${tarball}" -C "${context_dir}" --strip-components=1 +[[ -d "${context_dir}/coordinator" && -d "${context_dir}/worker" ]] \ + || die "tarball did not contain coordinator/ and worker/ trees" + +cp "${image_dir}/Dockerfile" "${image_dir}/entrypoint.sh" "${context_dir}/" + +echo >&2 "==> Building installer image ${image} (${platform})..." +docker buildx build \ + --platform "${platform}" \ + --tag "${image}" \ + "${output}" \ + -f "${context_dir}/Dockerfile" \ + "${context_dir}" + +echo >&2 "==> Built ${image}" +echo "${image}" diff --git a/tools/build-packages/build-packages.sh b/tools/build-packages/build-packages.sh index 6eca16d..a1ce64c 100755 --- a/tools/build-packages/build-packages.sh +++ b/tools/build-packages/build-packages.sh @@ -164,3 +164,16 @@ if ! compgen -G "${artifact_stage}/*" > /dev/null; then exit 1 fi cp -f "${artifact_stage}"/* "${output_dir}/" + +# Build the busybox installer image as a fourth distribution channel, from the tarball this +# run just produced. Source from artifact_stage (this run's fresh staging) rather than +# output_dir, which may hold tarballs from earlier or other-arch builds. +echo "==> Building busybox installer image..." +shopt -s nullglob +tarballs=("${artifact_stage}"/*.tar.gz) +shopt -u nullglob +if (( ${#tarballs[@]} != 1 )); then + echo >&2 "ERROR: expected exactly one .tar.gz in staging, found ${#tarballs[@]}" + exit 1 +fi +"${src}/tools/build-packages/build-installer-init-image.sh" --tarball "${tarballs[0]}" --load diff --git a/tools/build-packages/dependency-image/utils.sh b/tools/build-packages/dependency-image/utils.sh index 15cf825..66bb900 100644 --- a/tools/build-packages/dependency-image/utils.sh +++ b/tools/build-packages/dependency-image/utils.sh @@ -19,6 +19,24 @@ image_ref() { echo "$1/$2:env-$3" } +# Derives this repo's GHCR namespace from its GitHub origin remote. +image_repo_from_origin() { + local remote_url owner_repo + remote_url="$(git -C "${_REPO_ROOT}" remote get-url origin)" + case "${remote_url}" in + https://github.com/*) owner_repo="${remote_url#https://github.com/}" ;; + git@github.com:*) owner_repo="${remote_url#git@github.com:}" ;; + ssh://git@github.com/*) owner_repo="${remote_url#ssh://git@github.com/}" ;; + *) + echo >&2 "ERROR: can't derive GHCR image repo from origin remote: ${remote_url}" + echo >&2 " Expected a github.com remote." + exit 1 + ;; + esac + owner_repo="${owner_repo%.git}" + printf 'ghcr.io/%s\n' "$(printf '%s' "${owner_repo}" | tr '[:upper:]' '[:lower:]')" +} + # Inputs that should change the build-env image tag. _BUILD_ENV_HASH_INPUTS=( ".dockerignore" diff --git a/tools/build-packages/image/Dockerfile b/tools/build-packages/image/Dockerfile new file mode 100644 index 0000000..5d6d334 --- /dev/null +++ b/tools/build-packages/image/Dockerfile @@ -0,0 +1,18 @@ +# syntax=docker/dockerfile:1 + +# Init-container installer image for the CLP Presto connector. +# +# Bundles both plugins and, when run, copies each into a mounted target directory. The +# build context is an extracted package install tree (coordinator/ + worker/), assembled by +# build-installer-init-image.sh. See tools/build-packages/README.md. + +# The installer only runs busybox's own sh/cp/mkdir to copy files; it never executes the +# plugin, so the libc flavor is irrelevant. Pin the small musl variant for a tiny image. +FROM busybox:1.37.0-musl + +# Match the .deb/.rpm install root (PLUGIN_ROOT) so all channels share one layout. +COPY coordinator /opt/clp-plugin-presto-connector/coordinator +COPY worker /opt/clp-plugin-presto-connector/worker +COPY --chmod=0755 entrypoint.sh /usr/local/bin/install-clp-plugin + +ENTRYPOINT ["/usr/local/bin/install-clp-plugin"] diff --git a/tools/build-packages/image/entrypoint.sh b/tools/build-packages/image/entrypoint.sh new file mode 100755 index 0000000..42cf808 --- /dev/null +++ b/tools/build-packages/image/entrypoint.sh @@ -0,0 +1,49 @@ +#!/bin/sh + +# Init-container installer for the CLP Presto connector plugins. +# +# The image bundles both plugins under /opt/clp-plugin-presto-connector. Because the +# coordinator JAR and the native worker .so install into different locations, each is +# selected independently by its own target env var. Set whichever the pod needs; a +# coordinator pod sets COORDINATOR_PLUGIN_INSTALL_PATH, a worker pod sets WORKER_PLUGIN_INSTALL_PATH, and +# either or both may be set in a single run. + +set -eu + +readonly PLUGIN_ROOT="/opt/clp-plugin-presto-connector" + +usage() { + cat >&2 < +EOF + exit 1 +} + +# install_component +install_component() { + src="${PLUGIN_ROOT}/$1" + dest="$2" + mkdir -p "${dest}" + # Copy contents (not the subdir itself) so the target holds the plugin files directly. + cp -a "${src}/." "${dest}/" + echo "Installed $1 plugin -> ${dest}" +} + +installed=0 +if [ -n "${COORDINATOR_PLUGIN_INSTALL_PATH:-}" ]; then + install_component "coordinator" "${COORDINATOR_PLUGIN_INSTALL_PATH}" + installed=1 +fi +if [ -n "${WORKER_PLUGIN_INSTALL_PATH:-}" ]; then + install_component "worker" "${WORKER_PLUGIN_INSTALL_PATH}" + installed=1 +fi + +[ "${installed}" -eq 1 ] || usage From fac30a9304e4aeefb7e710dad8c4ccd67f5c6a31 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Fri, 31 Jul 2026 10:29:45 -0400 Subject: [PATCH 02/10] fix(packaging): Don't preserve source ownership when installing plugins, so a non-root init container works. --- tools/build-packages/image/entrypoint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/build-packages/image/entrypoint.sh b/tools/build-packages/image/entrypoint.sh index 42cf808..1fb3a77 100755 --- a/tools/build-packages/image/entrypoint.sh +++ b/tools/build-packages/image/entrypoint.sh @@ -32,7 +32,9 @@ install_component() { dest="$2" mkdir -p "${dest}" # Copy contents (not the subdir itself) so the target holds the plugin files directly. - cp -a "${src}/." "${dest}/" + # -r without -p: preserve permission bits but not ownership, so installed files belong + # to whatever user the container runs as (root by default, or a pod's runAsUser). + cp -r "${src}/." "${dest}/" echo "Installed $1 plugin -> ${dest}" } From 65737b07225130f8e8f9861e698efac9b613868c Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Fri, 31 Jul 2026 13:00:58 -0400 Subject: [PATCH 03/10] docs(build-packages): Document the installer image's CI tagging and push policy. --- tools/build-packages/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md index 5767ec3..f65251e 100644 --- a/tools/build-packages/README.md +++ b/tools/build-packages/README.md @@ -37,6 +37,12 @@ docker run --rm -e WORKER_PLUGIN_INSTALL_PATH=/plugins -v "$(pwd)/plugins:/plugi Run `build-installer-init-image.sh --help` to build it standalone from any package tarball. +In CI, `build-packages.yaml` builds the image per architecture on every run and +combines them into a multi-arch `:` tag; pushes to GHCR happen only +from the default branch and version tags. The multi-arch tag exists only on the +registry (a manifest can't be loaded into a local daemon) — local builds always +load `:-`. + The build runs inside a hash-tagged **build-env image** (`env-`) based on `manylinux_2_28`. `build-dependency-image.sh` resolves it from the local Docker cache, this repository's GHCR package, or a local build, reusing the cached From b57f9ecc730e2b161dbea2efc72494a8ff10791f Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Fri, 31 Jul 2026 19:57:07 -0400 Subject: [PATCH 04/10] Update tools/build-packages/build-installer-init-image.sh Co-authored-by: Junhao Liao --- tools/build-packages/build-installer-init-image.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/build-packages/build-installer-init-image.sh b/tools/build-packages/build-installer-init-image.sh index d0d375e..b151ee8 100755 --- a/tools/build-packages/build-installer-init-image.sh +++ b/tools/build-packages/build-installer-init-image.sh @@ -14,6 +14,8 @@ set -o errexit set -o nounset set -o pipefail +umask 0022 + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" image_dir="${script_dir}/image" From 5ae71ac3c01984a3478a67292a84fdf237b6e2e6 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Fri, 31 Jul 2026 19:57:40 -0400 Subject: [PATCH 05/10] Update .github/workflows/build-packages.yaml Co-authored-by: Junhao Liao --- .github/workflows/build-packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-packages.yaml b/.github/workflows/build-packages.yaml index 313a9e0..49c3173 100644 --- a/.github/workflows/build-packages.yaml +++ b/.github/workflows/build-packages.yaml @@ -225,7 +225,7 @@ jobs: persist-credentials: false - name: "Download tarball artifact" - uses: "actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53" # v6.0.0 + uses: "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" # v8.0.1 with: pattern: "*-linux-${{matrix.arch}}.tar.gz" path: "dist" From 192de1cc373424d51d9e308a4513ae9eae98ae84 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Fri, 31 Jul 2026 19:57:53 -0400 Subject: [PATCH 06/10] Update tools/build-packages/README.md Co-authored-by: Junhao Liao --- tools/build-packages/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md index f65251e..230c15a 100644 --- a/tools/build-packages/README.md +++ b/tools/build-packages/README.md @@ -35,7 +35,7 @@ docker run --rm -e WORKER_PLUGIN_INSTALL_PATH=/plugins -v "$(pwd)/plugins:/plugi ghcr.io/y-scope/clp-plugin-presto-connector:- ``` -Run `build-installer-init-image.sh --help` to build it standalone from any package tarball. +Run `./tools/build-packages/build-installer-init-image.sh --help` to build it standalone from any package tarball. In CI, `build-packages.yaml` builds the image per architecture on every run and combines them into a multi-arch `:` tag; pushes to GHCR happen only From 2b50e412a9472a4bee04bc4688da5ff57edaa90c Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Fri, 31 Jul 2026 20:02:19 -0400 Subject: [PATCH 07/10] style(build-packages): Rename die() to panic() in build-installer-init-image.sh. Co-authored-by: Junhao Liao --- .../build-installer-init-image.sh | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/build-packages/build-installer-init-image.sh b/tools/build-packages/build-installer-init-image.sh index b151ee8..17ebd23 100755 --- a/tools/build-packages/build-installer-init-image.sh +++ b/tools/build-packages/build-installer-init-image.sh @@ -43,13 +43,13 @@ See tools/build-packages/README.md for details. EOF } -die() { +panic() { echo >&2 "ERROR: $*" exit 1 } require_value() { - [[ -n "${2:-}" ]] || die "$1 requires a value" + [[ -n "${2:-}" ]] || panic "$1 requires a value" } # ── Parse arguments ─────────────────────────────────────────────────────────── @@ -69,15 +69,15 @@ while [[ $# -gt 0 ]]; do --push) output="--push"; shift ;; --load) output="--load"; shift ;; --help) show_help; exit 0 ;; - *) die "unknown option: $1 (use --help for usage)" ;; + *) panic "unknown option: $1 (use --help for usage)" ;; esac done -[[ -n "${tarball}" ]] || die "--tarball is required (use --help for usage)" -[[ -f "${tarball}" ]] || die "tarball not found: ${tarball}" +[[ -n "${tarball}" ]] || panic "--tarball is required (use --help for usage)" +[[ -f "${tarball}" ]] || panic "tarball not found: ${tarball}" -command -v docker &>/dev/null || die "docker is required" -docker buildx version &>/dev/null || die "docker buildx is required" +command -v docker &>/dev/null || panic "docker is required" +docker buildx version &>/dev/null || panic "docker buildx is required" # ── Resolve version and arch from the tarball name when not given ────────────── @@ -86,7 +86,7 @@ tar_base="$(basename "${tarball}")" tar_base="${tar_base%.tar.gz}" name_rest="${tar_base#clp-plugin-presto-connector-}" if [[ "${name_rest}" == "${tar_base}" || "${name_rest}" != *-linux-* ]]; then - die "cannot parse tarball name '${tar_base}'; pass --version and --arch explicitly" + panic "cannot parse tarball name '${tar_base}'; pass --version and --arch explicitly" fi [[ -n "${arch}" ]] || arch="${name_rest##*-linux-}" [[ -n "${version}" ]] || version="${name_rest%-linux-"${arch}"}" @@ -94,7 +94,7 @@ fi case "${arch}" in amd64) platform="linux/amd64" ;; arm64) platform="linux/arm64" ;; - *) die "unsupported arch: ${arch} (expected amd64 or arm64)" ;; + *) panic "unsupported arch: ${arch} (expected amd64 or arm64)" ;; esac [[ -n "${repo}" ]] || repo="$(image_repo_from_origin)" @@ -112,7 +112,7 @@ trap 'rm -rf "${context_dir}"' EXIT # Dockerfile's COPY paths. --strip-components=1 drops the versioned top-level directory. tar -xzf "${tarball}" -C "${context_dir}" --strip-components=1 [[ -d "${context_dir}/coordinator" && -d "${context_dir}/worker" ]] \ - || die "tarball did not contain coordinator/ and worker/ trees" + || panic "tarball did not contain coordinator/ and worker/ trees" cp "${image_dir}/Dockerfile" "${image_dir}/entrypoint.sh" "${context_dir}/" From 801b5fb984b1689ff9b28000ddcf5242d711f535 Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Fri, 31 Jul 2026 20:12:48 -0400 Subject: [PATCH 08/10] tools: reject package versions unusable as Docker image tags instead of mangling them The old inline sanitize (${VERSION//[^A-Za-z0-9_.-]/_}) was lossy: distinct package versions like '1.0+rc' and '1.0~rc' both mapped to the same '1.0_rc' tag. Replace it with a shared package_version_to_image_tag helper in dependency-image/utils.sh that validates the version and fails loudly when it contains characters Docker tags can't represent ('+', '~'). Both the local build script and the CI manifest job now use the same helper, so the manifest tag always matches the per-arch tags. Co-authored-by: Junhao Liao --- .github/workflows/build-packages.yaml | 15 +++++++++++++-- tools/build-packages/README.md | 3 ++- .../build-packages/build-installer-init-image.sh | 7 +++++-- tools/build-packages/dependency-image/utils.sh | 14 ++++++++++++++ 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-packages.yaml b/.github/workflows/build-packages.yaml index 49c3173..55a9b14 100644 --- a/.github/workflows/build-packages.yaml +++ b/.github/workflows/build-packages.yaml @@ -275,6 +275,11 @@ jobs: env: VERSION: "${{needs.resolve-version.outputs.version}}" steps: + - name: "Check out repository" + uses: "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" # v7.0.0 + with: + persist-credentials: false + - name: "Log in to GHCR" uses: "docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0" # v4.4.0 with: @@ -285,8 +290,14 @@ jobs: - name: "Create multi-arch manifest" run: |- image_repo="ghcr.io/$(printf '%s' "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]')" - # Docker tags allow only [A-Za-z0-9_.-]; sanitize to match the build script. - tag_version="${VERSION//[^A-Za-z0-9_.-]/_}" + # Validate the version with the same shared helper the build script uses, so the + # manifest tag always matches the per-arch tags (versions with characters Docker + # tags can't represent, like '+' or '~', fail loudly instead of being mangled). + source tools/build-packages/dependency-image/utils.sh + if ! tag_version="$(package_version_to_image_tag "${VERSION}")"; then + echo "::error::Version '${VERSION}' can't be used as an image tag" + exit 1 + fi docker buildx imagetools create \ --tag "${image_repo}:${tag_version}" \ "${image_repo}:${tag_version}-amd64" \ diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md index 230c15a..7ded63a 100644 --- a/tools/build-packages/README.md +++ b/tools/build-packages/README.md @@ -10,7 +10,8 @@ through `.github/workflows/build-packages.yaml`. Local builds use container-side script. Supported package version format: must start with a digit and use only -`[0-9A-Za-z.+~-]`. +`[0-9A-Za-z.+~-]`. The installer image additionally rejects versions containing +`+` or `~` (Docker tags can't represent them). For command options, run `--help` on the relevant entry point. diff --git a/tools/build-packages/build-installer-init-image.sh b/tools/build-packages/build-installer-init-image.sh index 17ebd23..dc83381 100755 --- a/tools/build-packages/build-installer-init-image.sh +++ b/tools/build-packages/build-installer-init-image.sh @@ -99,8 +99,11 @@ esac [[ -n "${repo}" ]] || repo="$(image_repo_from_origin)" -# Docker tags allow only [A-Za-z0-9_.-]; sanitize any other version characters (e.g. '+'). -tag_version="${version//[^A-Za-z0-9_.-]/_}" +# Docker tags forbid '+' and '~'; the shared helper rejects versions image tags can't +# represent losslessly. +tag_version="$(package_version_to_image_tag "${version}")" \ + || panic "version '${version}' can't be used as an image tag" \ + "(only letters, digits, '.', '_', and '-' are allowed)" image="${repo}:${tag_version}-${arch}" # ── Assemble a self-contained build context and build ───────────────────────── diff --git a/tools/build-packages/dependency-image/utils.sh b/tools/build-packages/dependency-image/utils.sh index 66bb900..08e3167 100644 --- a/tools/build-packages/dependency-image/utils.sh +++ b/tools/build-packages/dependency-image/utils.sh @@ -37,6 +37,20 @@ image_repo_from_origin() { printf 'ghcr.io/%s\n' "$(printf '%s' "${owner_repo}" | tr '[:upper:]' '[:lower:]')" } +# Validates a package version for use as a Docker tag and prints it unchanged. Docker tags +# allow only [A-Za-z0-9_.-], so versions containing '+' or '~' (valid in packages) are +# rejected rather than encoded — a lossy substitution would collide distinct versions +# (e.g. '1.0+rc' and '1.0~rc'), and an encoding would produce unreadable tags. The single +# shared definition keeps local builds and the CI manifest job producing identical tags. +# +# Args: +# Fails when the version isn't a digit followed by letters, digits, '.', '_', or '-'. +package_version_to_image_tag() { + local version="$1" + [[ "${version}" =~ ^[0-9][0-9A-Za-z._-]*$ ]] || return 1 + printf '%s\n' "${version}" +} + # Inputs that should change the build-env image tag. _BUILD_ENV_HASH_INPUTS=( ".dockerignore" From 60313450fd72f56adb4a705fc71cc35b4e24abfb Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Fri, 31 Jul 2026 20:27:10 -0400 Subject: [PATCH 09/10] tools: build the multi-arch manifest from digests; drop the local - tag suffix Manifest by digest: the manifest job previously combined the per-arch images by their mutable :- tags, so a concurrent publish run could move a tag between the image jobs and the manifest job, silently mixing images from different runs. Each image leg now captures the pushed image's registry digest (build-installer-init-image.sh --digest-file, via buildx --metadata-file) and exposes it as a job output; the manifest job references the per-arch images by those immutable digests, guaranteeing the manifest combines exactly the images this run built. Per-arch tags are still pushed for debugging convenience but no longer carry correctness. Local tag without arch suffix: --load now tags the image : instead of :-, the conventional Docker pattern where a locally-built image and the published multi-arch one share a name and whatever is in the daemon wins. The suffix remains only on pushed images, where the two CI legs need distinct registry names below the manifest. Co-authored-by: Junhao Liao --- .github/workflows/build-packages.yaml | 35 ++++++++++++--- tools/build-packages/README.md | 9 ++-- .../build-installer-init-image.sh | 43 ++++++++++++++++--- 3 files changed, 70 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build-packages.yaml b/.github/workflows/build-packages.yaml index 55a9b14..6771352 100644 --- a/.github/workflows/build-packages.yaml +++ b/.github/workflows/build-packages.yaml @@ -218,6 +218,11 @@ jobs: timeout-minutes: 30 env: ARCH: "${{matrix.arch}}" + # Matrix legs share one output namespace, so each leg fills only its own arch's key + # (the other resolves to empty and is ignored by the merge below). + outputs: + digest_amd64: "${{steps.image.outputs.digest_amd64}}" + digest_arm64: "${{steps.image.outputs.digest_arm64}}" steps: - name: "Check out repository" uses: "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" # v7.0.0 @@ -243,6 +248,7 @@ jobs: uses: "docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c" # v4.2.0 - name: "Build the installer image (and push when publishing)" + id: "image" env: PUBLISH: "${{needs.resolve-version.outputs.publish}}" run: |- @@ -253,16 +259,22 @@ jobs: echo "::error::Expected exactly one tarball for ${ARCH}, found ${#tarballs[@]}" exit 1 fi - # Push from the default branch / tags; otherwise just build to validate. - output="--load" + # Push from the default branch / tags; otherwise just build to validate. When + # pushing, capture the image digest so the manifest job can combine the exact + # images this run built (per-arch tags are mutable and a concurrent run could + # move them between this job and the manifest job). + args=(--load) if [[ "${PUBLISH}" == "true" ]]; then - output="--push" + args=(--push --digest-file digest.txt) fi bash tools/build-packages/build-installer-init-image.sh \ --tarball "${tarballs[0]}" \ --arch "${ARCH}" \ --repo "${image_repo}" \ - "${output}" + "${args[@]}" + if [[ "${PUBLISH}" == "true" ]]; then + echo "digest_${ARCH}=$(cat digest.txt)" >> "$GITHUB_OUTPUT" + fi # Combine the per-architecture installer images into one multi-arch version tag. manifest: @@ -274,6 +286,8 @@ jobs: timeout-minutes: 15 env: VERSION: "${{needs.resolve-version.outputs.version}}" + DIGEST_AMD64: "${{needs.image.outputs.digest_amd64}}" + DIGEST_ARM64: "${{needs.image.outputs.digest_arm64}}" steps: - name: "Check out repository" uses: "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" # v7.0.0 @@ -298,7 +312,16 @@ jobs: echo "::error::Version '${VERSION}' can't be used as an image tag" exit 1 fi + for digest in "${DIGEST_AMD64}" "${DIGEST_ARM64}"; do + if [[ ! "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "::error::Missing or malformed image digest: '${digest}'" + exit 1 + fi + done + # Reference the per-arch images by immutable digest rather than by tag: the + # per-arch tags are mutable, so a concurrent publish run could move them between + # the image jobs and this one, silently mixing images from different runs. docker buildx imagetools create \ --tag "${image_repo}:${tag_version}" \ - "${image_repo}:${tag_version}-amd64" \ - "${image_repo}:${tag_version}-arm64" + "${image_repo}@${DIGEST_AMD64}" \ + "${image_repo}@${DIGEST_ARM64}" diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md index 7ded63a..e081821 100644 --- a/tools/build-packages/README.md +++ b/tools/build-packages/README.md @@ -33,16 +33,17 @@ A thin wrapper over `./tools/build-packages/build-packages.sh` (call that direct ```bash docker run --rm -e WORKER_PLUGIN_INSTALL_PATH=/plugins -v "$(pwd)/plugins:/plugins" \ - ghcr.io/y-scope/clp-plugin-presto-connector:- + ghcr.io/y-scope/clp-plugin-presto-connector: ``` Run `./tools/build-packages/build-installer-init-image.sh --help` to build it standalone from any package tarball. In CI, `build-packages.yaml` builds the image per architecture on every run and combines them into a multi-arch `:` tag; pushes to GHCR happen only -from the default branch and version tags. The multi-arch tag exists only on the -registry (a manifest can't be loaded into a local daemon) — local builds always -load `:-`. +from the default branch and version tags. Local builds load the same +`:` tag (single-arch, for the build host) — a locally-built image +therefore shadows the published one in your Docker daemon until you +`docker pull` it. The build runs inside a hash-tagged **build-env image** (`env-`) based on `manylinux_2_28`. `build-dependency-image.sh` resolves it from the local Docker diff --git a/tools/build-packages/build-installer-init-image.sh b/tools/build-packages/build-installer-init-image.sh index dc83381..29cadd9 100755 --- a/tools/build-packages/build-installer-init-image.sh +++ b/tools/build-packages/build-installer-init-image.sh @@ -37,6 +37,9 @@ Options: e.g. ghcr.io/y-scope/clp-plugin-presto-connector) --push Push the image to the registry (default: --load into local docker) --load Load the image into the local docker daemon (default) + --digest-file F Write the pushed image's registry digest (sha256:...) to F + (requires --push; used by CI to build the multi-arch manifest + from immutable digests instead of mutable per-arch tags) --help Show this help See tools/build-packages/README.md for details. @@ -59,6 +62,7 @@ version="" arch="" repo="" output="--load" +digest_file="" while [[ $# -gt 0 ]]; do case $1 in @@ -68,6 +72,7 @@ while [[ $# -gt 0 ]]; do --repo) require_value "$1" "${2:-}"; repo="$2"; shift 2 ;; --push) output="--push"; shift ;; --load) output="--load"; shift ;; + --digest-file) require_value "$1" "${2:-}"; digest_file="$2"; shift 2 ;; --help) show_help; exit 0 ;; *) panic "unknown option: $1 (use --help for usage)" ;; esac @@ -75,6 +80,8 @@ done [[ -n "${tarball}" ]] || panic "--tarball is required (use --help for usage)" [[ -f "${tarball}" ]] || panic "tarball not found: ${tarball}" +[[ -z "${digest_file}" || "${output}" == "--push" ]] \ + || panic "--digest-file requires --push (only pushed images have a registry digest)" command -v docker &>/dev/null || panic "docker is required" docker buildx version &>/dev/null || panic "docker buildx is required" @@ -104,7 +111,16 @@ esac tag_version="$(package_version_to_image_tag "${version}")" \ || panic "version '${version}' can't be used as an image tag" \ "(only letters, digits, '.', '_', and '-' are allowed)" -image="${repo}:${tag_version}-${arch}" + +# Pushed images get an arch suffix because the two CI legs need distinct registry names +# (the bare tag is the multi-arch manifest combining them). Local loads use the bare tag — +# the conventional Docker pattern where a locally-built image and the published one share +# a name and whatever is in the daemon wins. +if [[ "${output}" == "--push" ]]; then + image="${repo}:${tag_version}-${arch}" +else + image="${repo}:${tag_version}" +fi # ── Assemble a self-contained build context and build ───────────────────────── @@ -120,12 +136,25 @@ tar -xzf "${tarball}" -C "${context_dir}" --strip-components=1 cp "${image_dir}/Dockerfile" "${image_dir}/entrypoint.sh" "${context_dir}/" echo >&2 "==> Building installer image ${image} (${platform})..." -docker buildx build \ - --platform "${platform}" \ - --tag "${image}" \ - "${output}" \ - -f "${context_dir}/Dockerfile" \ - "${context_dir}" +buildx_args=( + --platform "${platform}" + --tag "${image}" + "${output}" + -f "${context_dir}/Dockerfile" +) +metadata_file="${context_dir}/buildx-metadata.json" +[[ -z "${digest_file}" ]] || buildx_args+=(--metadata-file "${metadata_file}") +docker buildx build "${buildx_args[@]}" "${context_dir}" + +if [[ -n "${digest_file}" ]]; then + # Extract the registry digest from the buildx metadata JSON. sed keeps the dependency + # footprint small (no jq); the strict format check makes a parse failure loud. + digest="$(sed -n 's/.*"containerimage\.digest": *"\([^"]*\)".*/\1/p' "${metadata_file}")" + [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]] \ + || panic "failed to extract image digest from buildx metadata" + printf '%s\n' "${digest}" > "${digest_file}" + echo >&2 "==> Pushed digest ${digest}" +fi echo >&2 "==> Built ${image}" echo "${image}" From 4ec3ed27fa07acb1fd8da54516ab0bedf2d55cba Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sat, 1 Aug 2026 14:17:36 -0400 Subject: [PATCH 10/10] docs(packaging): document tar prerequisite --- tools/build-packages/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md index e081821..7a0d486 100644 --- a/tools/build-packages/README.md +++ b/tools/build-packages/README.md @@ -62,7 +62,7 @@ while `packages/` is owned by the invoking user. ### Prerequisites -Docker with buildx (usable without `sudo`), git, `sha256sum` or `shasum`, and +Docker with buildx (usable without `sudo`), git, `tar`, `sha256sum` or `shasum`, and ~10 GB free disk for the build-env image. ## Target-CPU flags