diff --git a/.github/workflows/build-packages.yaml b/.github/workflows/build-packages.yaml index c20fe59..6771352 100644 --- a/.github/workflows/build-packages.yaml +++ b/.github/workflows/build-packages.yaml @@ -4,6 +4,8 @@ # resolve-version use the input version or derive it from pom.xml # build-dependency-image ensure the hash-tagged build-env image exists # build build .deb / .rpm / .tar.gz artifacts per architecture +# image build the busybox installer image per arch (push on main/tags) +# manifest combine per-arch images into a multi-arch tag (main/tags only) # # See tools/build-packages/README.md for package-build details. @@ -58,6 +60,7 @@ jobs: runs-on: "ubuntu-24.04" outputs: version: "${{steps.version.outputs.version}}" + publish: "${{steps.publish.outputs.publish}}" steps: - name: "Check out repository" uses: "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" # v7.0.0 @@ -88,6 +91,19 @@ jobs: echo "version=${version}" >> "$GITHUB_OUTPUT" echo "::notice::Package version: ${version}" + - name: "Decide whether to publish images" + id: "publish" + env: + DEFAULT_BRANCH: "${{github.event.repository.default_branch}}" + run: |- + # Publish images only from the default branch and version tags so feature + # branches don't overwrite shared tags. + if [[ ${GITHUB_REF_NAME} == "${DEFAULT_BRANCH}" || ${GITHUB_REF_TYPE} == tag ]]; then + echo "publish=true" >> "$GITHUB_OUTPUT" + else + echo "publish=false" >> "$GITHUB_OUTPUT" + fi + build-dependency-image: uses: "./.github/workflows/build-dependency-image.yaml" permissions: @@ -182,3 +198,130 @@ jobs: path: "packages/${{steps.packages.outputs.tar_gz_filename}}" if-no-files-found: "error" retention-days: 14 + + # Build the busybox installer image per architecture from the tarball artifact. This runs + # on the native runner (not inside the build-env container) so docker/buildx is available. + image: + needs: ["build", "resolve-version"] + permissions: + contents: "read" + packages: "write" + strategy: + fail-fast: false + matrix: + include: + - arch: "amd64" + runner: "ubuntu-24.04" + - arch: "arm64" + runner: "ubuntu-24.04-arm" + runs-on: "${{matrix.runner}}" + timeout-minutes: 30 + env: + ARCH: "${{matrix.arch}}" + # Matrix legs share one output namespace, so each leg fills only its own arch's key + # (the other resolves to empty and is ignored by the merge below). + outputs: + digest_amd64: "${{steps.image.outputs.digest_amd64}}" + digest_arm64: "${{steps.image.outputs.digest_arm64}}" + steps: + - name: "Check out repository" + uses: "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" # v7.0.0 + with: + persist-credentials: false + + - name: "Download tarball artifact" + uses: "actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" # v8.0.1 + with: + pattern: "*-linux-${{matrix.arch}}.tar.gz" + path: "dist" + merge-multiple: true + + - name: "Log in to GHCR" + if: "${{needs.resolve-version.outputs.publish == 'true'}}" + uses: "docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0" # v4.4.0 + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Set up Docker Buildx" + uses: "docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c" # v4.2.0 + + - name: "Build the installer image (and push when publishing)" + id: "image" + env: + PUBLISH: "${{needs.resolve-version.outputs.publish}}" + run: |- + image_repo="ghcr.io/$(printf '%s' "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]')" + shopt -s nullglob + tarballs=(dist/*-linux-"${ARCH}".tar.gz) + if (( ${#tarballs[@]} != 1 )); then + echo "::error::Expected exactly one tarball for ${ARCH}, found ${#tarballs[@]}" + exit 1 + fi + # Push from the default branch / tags; otherwise just build to validate. When + # pushing, capture the image digest so the manifest job can combine the exact + # images this run built (per-arch tags are mutable and a concurrent run could + # move them between this job and the manifest job). + args=(--load) + if [[ "${PUBLISH}" == "true" ]]; then + args=(--push --digest-file digest.txt) + fi + bash tools/build-packages/build-installer-init-image.sh \ + --tarball "${tarballs[0]}" \ + --arch "${ARCH}" \ + --repo "${image_repo}" \ + "${args[@]}" + if [[ "${PUBLISH}" == "true" ]]; then + echo "digest_${ARCH}=$(cat digest.txt)" >> "$GITHUB_OUTPUT" + fi + + # Combine the per-architecture installer images into one multi-arch version tag. + manifest: + needs: ["image", "resolve-version"] + if: "${{needs.resolve-version.outputs.publish == 'true'}}" + permissions: + packages: "write" + runs-on: "ubuntu-24.04" + timeout-minutes: 15 + env: + VERSION: "${{needs.resolve-version.outputs.version}}" + DIGEST_AMD64: "${{needs.image.outputs.digest_amd64}}" + DIGEST_ARM64: "${{needs.image.outputs.digest_arm64}}" + steps: + - name: "Check out repository" + uses: "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" # v7.0.0 + with: + persist-credentials: false + + - name: "Log in to GHCR" + uses: "docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0" # v4.4.0 + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Create multi-arch manifest" + run: |- + image_repo="ghcr.io/$(printf '%s' "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]')" + # Validate the version with the same shared helper the build script uses, so the + # manifest tag always matches the per-arch tags (versions with characters Docker + # tags can't represent, like '+' or '~', fail loudly instead of being mangled). + source tools/build-packages/dependency-image/utils.sh + if ! tag_version="$(package_version_to_image_tag "${VERSION}")"; then + echo "::error::Version '${VERSION}' can't be used as an image tag" + exit 1 + fi + for digest in "${DIGEST_AMD64}" "${DIGEST_ARM64}"; do + if [[ ! "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]]; then + echo "::error::Missing or malformed image digest: '${digest}'" + exit 1 + fi + done + # Reference the per-arch images by immutable digest rather than by tag: the + # per-arch tags are mutable, so a concurrent publish run could move them between + # the image jobs and this one, silently mixing images from different runs. + docker buildx imagetools create \ + --tag "${image_repo}:${tag_version}" \ + "${image_repo}@${DIGEST_AMD64}" \ + "${image_repo}@${DIGEST_ARM64}" diff --git a/tools/build-packages/README.md b/tools/build-packages/README.md index d3abc22..7a0d486 100644 --- a/tools/build-packages/README.md +++ b/tools/build-packages/README.md @@ -1,7 +1,8 @@ # CLP Presto connector packaging -This directory builds installable `.deb`, `.rpm`, and `.tar.gz` artifacts for the CLP -Presto connector (coordinator + worker) on `amd64` and `arm64`. +This directory builds installable `.deb`, `.rpm`, and `.tar.gz` artifacts, plus a busybox +init-container installer image, for the CLP Presto connector (coordinator + worker) on +`amd64` and `arm64`. CI packaging runs `tools/build-packages/internal/container/build-artifacts.sh` through `.github/workflows/build-packages.yaml`. Local builds use @@ -9,7 +10,8 @@ through `.github/workflows/build-packages.yaml`. Local builds use container-side script. Supported package version format: must start with a digit and use only -`[0-9A-Za-z.+~-]`. +`[0-9A-Za-z.+~-]`. The installer image additionally rejects versions containing +`+` or `~` (Docker tags can't represent them). For command options, run `--help` on the relevant entry point. @@ -25,6 +27,24 @@ task package A thin wrapper over `./tools/build-packages/build-packages.sh` (call that directly if `go-task` isn't installed). Both accept `--output DIR`, `--version VER`, and `--with-ca-certs`; with the task, put `--` before the flags: `task package -- --output DIR`. +### Installer image + +`task package` also builds and loads a busybox init-container image that bundles both plugins. Its entrypoint copies each component into a mounted volume named by `COORDINATOR_PLUGIN_INSTALL_PATH` / `WORKER_PLUGIN_INSTALL_PATH` (set either or both): + +```bash +docker run --rm -e WORKER_PLUGIN_INSTALL_PATH=/plugins -v "$(pwd)/plugins:/plugins" \ + ghcr.io/y-scope/clp-plugin-presto-connector: +``` + +Run `./tools/build-packages/build-installer-init-image.sh --help` to build it standalone from any package tarball. + +In CI, `build-packages.yaml` builds the image per architecture on every run and +combines them into a multi-arch `:` tag; pushes to GHCR happen only +from the default branch and version tags. Local builds load the same +`:` tag (single-arch, for the build host) — a locally-built image +therefore shadows the published one in your Docker daemon until you +`docker pull` it. + The build runs inside a hash-tagged **build-env image** (`env-`) based on `manylinux_2_28`. `build-dependency-image.sh` resolves it from the local Docker cache, this repository's GHCR package, or a local build, reusing the cached @@ -42,7 +62,7 @@ while `packages/` is owned by the invoking user. ### Prerequisites -Docker with buildx (usable without `sudo`), git, `sha256sum` or `shasum`, and +Docker with buildx (usable without `sudo`), git, `tar`, `sha256sum` or `shasum`, and ~10 GB free disk for the build-env image. ## Target-CPU flags diff --git a/tools/build-packages/build-dependency-image.sh b/tools/build-packages/build-dependency-image.sh index 13c36d4..e00d51b 100755 --- a/tools/build-packages/build-dependency-image.sh +++ b/tools/build-packages/build-dependency-image.sh @@ -19,24 +19,6 @@ set -o pipefail script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" source "${script_dir}/dependency-image/utils.sh" -# Derive this repo's GHCR namespace from its GitHub origin remote. -image_repo_from_origin() { - local remote_url owner_repo - remote_url="$(git -C "${_REPO_ROOT}" remote get-url origin)" - case "${remote_url}" in - https://github.com/*) owner_repo="${remote_url#https://github.com/}" ;; - git@github.com:*) owner_repo="${remote_url#git@github.com:}" ;; - ssh://git@github.com/*) owner_repo="${remote_url#ssh://git@github.com/}" ;; - *) - echo >&2 "ERROR: can't derive GHCR image repo from origin remote: ${remote_url}" - echo >&2 " Expected a github.com remote." - exit 1 - ;; - esac - owner_repo="${owner_repo%.git}" - printf 'ghcr.io/%s\n' "$(printf '%s' "${owner_repo}" | tr '[:upper:]' '[:lower:]')" -} - host_platform() { case "$(uname -m)" in x86_64) printf 'linux/amd64\n' ;; diff --git a/tools/build-packages/build-installer-init-image.sh b/tools/build-packages/build-installer-init-image.sh new file mode 100755 index 0000000..29cadd9 --- /dev/null +++ b/tools/build-packages/build-installer-init-image.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash + +# Builds the busybox init-container installer image from a connector package tarball. +# +# The image bundles both plugins (coordinator JAR + native worker .so and lib/) and, when +# run, installs each into a mounted target directory. See tools/build-packages/README.md. +# +# Reusable by local builds (build-packages.sh, --load) and CI (--push). Prints the built +# image reference to stdout. +# +# Requires: docker (with buildx), git, tar. + +set -o errexit +set -o nounset +set -o pipefail + +umask 0022 + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +image_dir="${script_dir}/image" + +# Shared helpers: image_repo_from_origin (GHCR repo derivation) and _REPO_ROOT. +source "${script_dir}/dependency-image/utils.sh" + +show_help() { + cat <<'EOF' +Usage: ./tools/build-packages/build-installer-init-image.sh --tarball FILE [OPTIONS] + +Builds the busybox init-container installer image from a connector package tarball +(clp-plugin-presto-connector--linux-.tar.gz). + +Options: + --tarball FILE Package tarball to build the image from (required) + --version VER Image version tag (default: parsed from the tarball name) + --arch ARCH amd64 or arm64 (default: parsed from the tarball name) + --repo REPO Image repository (default: derived from the git origin remote, + e.g. ghcr.io/y-scope/clp-plugin-presto-connector) + --push Push the image to the registry (default: --load into local docker) + --load Load the image into the local docker daemon (default) + --digest-file F Write the pushed image's registry digest (sha256:...) to F + (requires --push; used by CI to build the multi-arch manifest + from immutable digests instead of mutable per-arch tags) + --help Show this help + +See tools/build-packages/README.md for details. +EOF +} + +panic() { + echo >&2 "ERROR: $*" + exit 1 +} + +require_value() { + [[ -n "${2:-}" ]] || panic "$1 requires a value" +} + +# ── Parse arguments ─────────────────────────────────────────────────────────── + +tarball="" +version="" +arch="" +repo="" +output="--load" +digest_file="" + +while [[ $# -gt 0 ]]; do + case $1 in + --tarball) require_value "$1" "${2:-}"; tarball="$2"; shift 2 ;; + --version) require_value "$1" "${2:-}"; version="$2"; shift 2 ;; + --arch) require_value "$1" "${2:-}"; arch="$2"; shift 2 ;; + --repo) require_value "$1" "${2:-}"; repo="$2"; shift 2 ;; + --push) output="--push"; shift ;; + --load) output="--load"; shift ;; + --digest-file) require_value "$1" "${2:-}"; digest_file="$2"; shift 2 ;; + --help) show_help; exit 0 ;; + *) panic "unknown option: $1 (use --help for usage)" ;; + esac +done + +[[ -n "${tarball}" ]] || panic "--tarball is required (use --help for usage)" +[[ -f "${tarball}" ]] || panic "tarball not found: ${tarball}" +[[ -z "${digest_file}" || "${output}" == "--push" ]] \ + || panic "--digest-file requires --push (only pushed images have a registry digest)" + +command -v docker &>/dev/null || panic "docker is required" +docker buildx version &>/dev/null || panic "docker buildx is required" + +# ── Resolve version and arch from the tarball name when not given ────────────── + +# Tarball name format: clp-plugin-presto-connector--linux-.tar.gz +tar_base="$(basename "${tarball}")" +tar_base="${tar_base%.tar.gz}" +name_rest="${tar_base#clp-plugin-presto-connector-}" +if [[ "${name_rest}" == "${tar_base}" || "${name_rest}" != *-linux-* ]]; then + panic "cannot parse tarball name '${tar_base}'; pass --version and --arch explicitly" +fi +[[ -n "${arch}" ]] || arch="${name_rest##*-linux-}" +[[ -n "${version}" ]] || version="${name_rest%-linux-"${arch}"}" + +case "${arch}" in + amd64) platform="linux/amd64" ;; + arm64) platform="linux/arm64" ;; + *) panic "unsupported arch: ${arch} (expected amd64 or arm64)" ;; +esac + +[[ -n "${repo}" ]] || repo="$(image_repo_from_origin)" + +# Docker tags forbid '+' and '~'; the shared helper rejects versions image tags can't +# represent losslessly. +tag_version="$(package_version_to_image_tag "${version}")" \ + || panic "version '${version}' can't be used as an image tag" \ + "(only letters, digits, '.', '_', and '-' are allowed)" + +# Pushed images get an arch suffix because the two CI legs need distinct registry names +# (the bare tag is the multi-arch manifest combining them). Local loads use the bare tag — +# the conventional Docker pattern where a locally-built image and the published one share +# a name and whatever is in the daemon wins. +if [[ "${output}" == "--push" ]]; then + image="${repo}:${tag_version}-${arch}" +else + image="${repo}:${tag_version}" +fi + +# ── Assemble a self-contained build context and build ───────────────────────── + +context_dir="$(mktemp -d)" +trap 'rm -rf "${context_dir}"' EXIT + +# Extract the install tree so coordinator/ and worker/ sit at the context root, matching the +# Dockerfile's COPY paths. --strip-components=1 drops the versioned top-level directory. +tar -xzf "${tarball}" -C "${context_dir}" --strip-components=1 +[[ -d "${context_dir}/coordinator" && -d "${context_dir}/worker" ]] \ + || panic "tarball did not contain coordinator/ and worker/ trees" + +cp "${image_dir}/Dockerfile" "${image_dir}/entrypoint.sh" "${context_dir}/" + +echo >&2 "==> Building installer image ${image} (${platform})..." +buildx_args=( + --platform "${platform}" + --tag "${image}" + "${output}" + -f "${context_dir}/Dockerfile" +) +metadata_file="${context_dir}/buildx-metadata.json" +[[ -z "${digest_file}" ]] || buildx_args+=(--metadata-file "${metadata_file}") +docker buildx build "${buildx_args[@]}" "${context_dir}" + +if [[ -n "${digest_file}" ]]; then + # Extract the registry digest from the buildx metadata JSON. sed keeps the dependency + # footprint small (no jq); the strict format check makes a parse failure loud. + digest="$(sed -n 's/.*"containerimage\.digest": *"\([^"]*\)".*/\1/p' "${metadata_file}")" + [[ "${digest}" =~ ^sha256:[0-9a-f]{64}$ ]] \ + || panic "failed to extract image digest from buildx metadata" + printf '%s\n' "${digest}" > "${digest_file}" + echo >&2 "==> Pushed digest ${digest}" +fi + +echo >&2 "==> Built ${image}" +echo "${image}" diff --git a/tools/build-packages/build-packages.sh b/tools/build-packages/build-packages.sh index 6eca16d..a1ce64c 100755 --- a/tools/build-packages/build-packages.sh +++ b/tools/build-packages/build-packages.sh @@ -164,3 +164,16 @@ if ! compgen -G "${artifact_stage}/*" > /dev/null; then exit 1 fi cp -f "${artifact_stage}"/* "${output_dir}/" + +# Build the busybox installer image as a fourth distribution channel, from the tarball this +# run just produced. Source from artifact_stage (this run's fresh staging) rather than +# output_dir, which may hold tarballs from earlier or other-arch builds. +echo "==> Building busybox installer image..." +shopt -s nullglob +tarballs=("${artifact_stage}"/*.tar.gz) +shopt -u nullglob +if (( ${#tarballs[@]} != 1 )); then + echo >&2 "ERROR: expected exactly one .tar.gz in staging, found ${#tarballs[@]}" + exit 1 +fi +"${src}/tools/build-packages/build-installer-init-image.sh" --tarball "${tarballs[0]}" --load diff --git a/tools/build-packages/dependency-image/utils.sh b/tools/build-packages/dependency-image/utils.sh index 15cf825..08e3167 100644 --- a/tools/build-packages/dependency-image/utils.sh +++ b/tools/build-packages/dependency-image/utils.sh @@ -19,6 +19,38 @@ image_ref() { echo "$1/$2:env-$3" } +# Derives this repo's GHCR namespace from its GitHub origin remote. +image_repo_from_origin() { + local remote_url owner_repo + remote_url="$(git -C "${_REPO_ROOT}" remote get-url origin)" + case "${remote_url}" in + https://github.com/*) owner_repo="${remote_url#https://github.com/}" ;; + git@github.com:*) owner_repo="${remote_url#git@github.com:}" ;; + ssh://git@github.com/*) owner_repo="${remote_url#ssh://git@github.com/}" ;; + *) + echo >&2 "ERROR: can't derive GHCR image repo from origin remote: ${remote_url}" + echo >&2 " Expected a github.com remote." + exit 1 + ;; + esac + owner_repo="${owner_repo%.git}" + printf 'ghcr.io/%s\n' "$(printf '%s' "${owner_repo}" | tr '[:upper:]' '[:lower:]')" +} + +# Validates a package version for use as a Docker tag and prints it unchanged. Docker tags +# allow only [A-Za-z0-9_.-], so versions containing '+' or '~' (valid in packages) are +# rejected rather than encoded — a lossy substitution would collide distinct versions +# (e.g. '1.0+rc' and '1.0~rc'), and an encoding would produce unreadable tags. The single +# shared definition keeps local builds and the CI manifest job producing identical tags. +# +# Args: +# Fails when the version isn't a digit followed by letters, digits, '.', '_', or '-'. +package_version_to_image_tag() { + local version="$1" + [[ "${version}" =~ ^[0-9][0-9A-Za-z._-]*$ ]] || return 1 + printf '%s\n' "${version}" +} + # Inputs that should change the build-env image tag. _BUILD_ENV_HASH_INPUTS=( ".dockerignore" diff --git a/tools/build-packages/image/Dockerfile b/tools/build-packages/image/Dockerfile new file mode 100644 index 0000000..5d6d334 --- /dev/null +++ b/tools/build-packages/image/Dockerfile @@ -0,0 +1,18 @@ +# syntax=docker/dockerfile:1 + +# Init-container installer image for the CLP Presto connector. +# +# Bundles both plugins and, when run, copies each into a mounted target directory. The +# build context is an extracted package install tree (coordinator/ + worker/), assembled by +# build-installer-init-image.sh. See tools/build-packages/README.md. + +# The installer only runs busybox's own sh/cp/mkdir to copy files; it never executes the +# plugin, so the libc flavor is irrelevant. Pin the small musl variant for a tiny image. +FROM busybox:1.37.0-musl + +# Match the .deb/.rpm install root (PLUGIN_ROOT) so all channels share one layout. +COPY coordinator /opt/clp-plugin-presto-connector/coordinator +COPY worker /opt/clp-plugin-presto-connector/worker +COPY --chmod=0755 entrypoint.sh /usr/local/bin/install-clp-plugin + +ENTRYPOINT ["/usr/local/bin/install-clp-plugin"] diff --git a/tools/build-packages/image/entrypoint.sh b/tools/build-packages/image/entrypoint.sh new file mode 100755 index 0000000..1fb3a77 --- /dev/null +++ b/tools/build-packages/image/entrypoint.sh @@ -0,0 +1,51 @@ +#!/bin/sh + +# Init-container installer for the CLP Presto connector plugins. +# +# The image bundles both plugins under /opt/clp-plugin-presto-connector. Because the +# coordinator JAR and the native worker .so install into different locations, each is +# selected independently by its own target env var. Set whichever the pod needs; a +# coordinator pod sets COORDINATOR_PLUGIN_INSTALL_PATH, a worker pod sets WORKER_PLUGIN_INSTALL_PATH, and +# either or both may be set in a single run. + +set -eu + +readonly PLUGIN_ROOT="/opt/clp-plugin-presto-connector" + +usage() { + cat >&2 < +EOF + exit 1 +} + +# install_component +install_component() { + src="${PLUGIN_ROOT}/$1" + dest="$2" + mkdir -p "${dest}" + # Copy contents (not the subdir itself) so the target holds the plugin files directly. + # -r without -p: preserve permission bits but not ownership, so installed files belong + # to whatever user the container runs as (root by default, or a pod's runAsUser). + cp -r "${src}/." "${dest}/" + echo "Installed $1 plugin -> ${dest}" +} + +installed=0 +if [ -n "${COORDINATOR_PLUGIN_INSTALL_PATH:-}" ]; then + install_component "coordinator" "${COORDINATOR_PLUGIN_INSTALL_PATH}" + installed=1 +fi +if [ -n "${WORKER_PLUGIN_INSTALL_PATH:-}" ]; then + install_component "worker" "${WORKER_PLUGIN_INSTALL_PATH}" + installed=1 +fi + +[ "${installed}" -eq 1 ] || usage