diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-package-image/action.yaml similarity index 69% rename from .github/actions/clp-build-runtime-image/action.yaml rename to .github/actions/clp-build-package-image/action.yaml index 454dd19394..959386e112 100644 --- a/.github/actions/clp-build-runtime-image/action.yaml +++ b/.github/actions/clp-build-package-image/action.yaml @@ -1,4 +1,4 @@ -name: "clp-build-runtime-image" +name: "clp-build-package-image" description: "Builds a container image to be used for running CLP." inputs: @@ -22,6 +22,14 @@ inputs: description: >- Platform version codename (e.g. jammy, noble) used as the runtime image base version. required: true + push_image: + description: "Whether to publish the package image" + required: true + +outputs: + image_ref: + description: "The tagged reference of the built container image." + value: "${{steps.compute-image-ref.outputs.IMAGE_REF}}" runs: using: "composite" @@ -30,6 +38,7 @@ runs: uses: "docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2" - name: "Login to Image Registry" + if: "inputs.push_image == 'true'" uses: "docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121" # v4.1.0 with: registry: "${{inputs.image_registry}}" @@ -65,6 +74,13 @@ runs: flavor: | suffix=-${{inputs.arch}} + - name: "Compute image reference" + id: "compute-image-ref" + shell: "bash" + run: | + echo "IMAGE_REF=$(head -n 1 <<< '${{steps.extract-gh-meta.outputs.tags}}')" \ + >> "$GITHUB_OUTPUT" + - name: "Map arch to Docker platform" id: "map-arch" shell: "bash" @@ -78,8 +94,21 @@ runs: exit 1 fi + - name: "Compute image output properties" + id: "output-properties" + shell: "bash" + run: | + image_name="${{steps.compute-meta.outputs.IMAGE_NAME}}" + if [[ "${{inputs.push_image}}" == "true" ]]; then + echo "image_output=type=registry" >> "$GITHUB_OUTPUT" + else + image_path="/tmp/${image_name}-image-${{inputs.arch}}.tar" + echo "image_path=${image_path}" >> "$GITHUB_OUTPUT" + echo "image_output=type=docker,dest=${image_path}" >> "$GITHUB_OUTPUT" + fi + - name: "Build and Push" - if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" + id: "build-and-push" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: context: "./" @@ -88,8 +117,16 @@ runs: # Disable provenance to create a simple image instead of a manifest list. # This allows `docker manifest create` to combine the per-arch images later. provenance: false - push: true + outputs: "${{steps.output-properties.outputs.image_output}}" tags: "${{steps.extract-gh-meta.outputs.tags}}" labels: "${{steps.extract-gh-meta.outputs.labels}}" build-args: | UBUNTU_VERSION_CODENAME=${{inputs.platform_version_codename}} + + - name: "Upload package image as artifact" + if: "inputs.push_image == 'false'" + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "clp-package-image-${{inputs.arch}}" + path: "${{steps.output-properties.outputs.image_path}}" + retention-days: 1 diff --git a/.github/workflows/clp-artifact-build.yaml b/.github/workflows/clp-artifact-build.yaml index 235be77650..750a510ab4 100644 --- a/.github/workflows/clp-artifact-build.yaml +++ b/.github/workflows/clp-artifact-build.yaml @@ -17,6 +17,8 @@ on: env: DEPS_IMAGE_NAME_PREFIX_AARCH64: "clp-core-dependencies-aarch64-" BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" + PACKAGE_ARTIFACT_NAME_PREFIX: "clp-package-" + PACKAGE_IMAGE_ARTIFACT_NAME_PREFIX: "clp-package-image-" PYTHON_WHEELS_ARTIFACT_NAME_PREFIX: "clp-python-wheels-" DEPS_IMAGE_NAME_PREFIX_X86: "clp-core-dependencies-x86-" # Bare (no arch) prefix for multi-arch dependency images; the arch is carried in the tag @@ -724,8 +726,8 @@ jobs: 'build/lint-clang-tidy/**/*' )}} - package-image: - name: "package-image-${{matrix.arch}}" + package: + name: "package-${{matrix.arch}}" # NOTE: The aarch64 job is skipped on PRs, so we accept both 'success' and 'skipped'. if: >- !cancelled() && !failure() && ( @@ -740,11 +742,11 @@ jobs: - "ubuntu-jammy-aarch64-deps-image" strategy: matrix: - # Only build arm64 images on push to main to save CI resources on PRs. - arch: >- + # arm64 packages are only built on main to save CI resources on PRs. + arch: &package_matrix_arch >- ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main' && fromJSON('["amd64", "arm64"]') || fromJSON('["amd64"]')}} - runs-on: >- + runs-on: &package_arch_runner >- ${{ matrix.arch == 'amd64' && (github.repository_owner == 'y-scope' @@ -780,21 +782,150 @@ jobs: HOME=/tmp task package-build-deps - - uses: "./.github/actions/clp-build-runtime-image" + - id: "build-package-image" + uses: "./.github/actions/clp-build-package-image" with: image_registry: "ghcr.io" image_registry_username: "${{github.actor}}" image_registry_password: "${{secrets.GITHUB_TOKEN}}" arch: "${{matrix.arch}}" platform_version_codename: "jammy" + push_image: &package_image_published >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + + - name: "Assemble the package directory structure" + uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: >- + ${{format('{0}ubuntu-jammy', + matrix.arch == 'amd64' && env.DEPS_IMAGE_NAME_PREFIX_X86 + || env.DEPS_IMAGE_NAME_PREFIX_AARCH64)}} + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CPP_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) + HOME=/tmp + task package-assemble OUTPUT_DIR=build/clp-package + + - name: "Write the package image reference" + shell: "bash" + run: >- + echo "${{steps.build-package-image.outputs.image_ref}}" + > "build/clp-package/clp-package-image.id" + + - name: "Tar the package directories" + shell: "bash" + run: >- + tar --create --file "build/clp-package.tar" + --directory "build/clp-package" . + + - name: "Upload the package tarball artifact" + uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 + with: + name: "${{env.PACKAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}" + path: "build/clp-package.tar" + retention-days: 1 + + package-integration-tests: + name: "package-integration-tests-${{matrix.arch}}" + if: "!cancelled() && !failure() && needs.package.result == 'success'" + needs: + - "package" + strategy: + matrix: + # arm64 packages are only tested on main to save CI resources on PRs. + arch: *package_matrix_arch + runs-on: *package_arch_runner + env: + # `package` publishes the image to the registry on pushes to main and scheduled runs. + PACKAGE_IMAGE_PUBLISHED: *package_image_published + BUILD_DIR: "build" + steps: + - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 + with: + persist-credentials: false + submodules: "recursive" + + - name: "Download the package artifact" + uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + with: + name: "${{env.PACKAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}" + path: "${{env.BUILD_DIR}}/clp-package" + + - name: "Untar the package" + working-directory: "${{env.BUILD_DIR}}/clp-package" + run: |- + tar xf clp-package.tar + rm clp-package.tar + + - if: "env.PACKAGE_IMAGE_PUBLISHED == 'false'" + name: "Download the package image artifact" + uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + with: + name: "${{env.PACKAGE_IMAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}" + path: "${{env.BUILD_DIR}}/clp-package-image" + + - if: "env.PACKAGE_IMAGE_PUBLISHED == 'false'" + name: "Load the package image" + shell: "bash" + env: + PACKAGE_IMAGE_DIR: "${{env.BUILD_DIR}}/clp-package-image" + PACKAGE_IMAGE_TAR_NAME: "${{env.PACKAGE_IMAGE_ARTIFACT_NAME_PREFIX}}${{matrix.arch}}.tar" + run: >- + docker load + --input + "${PACKAGE_IMAGE_DIR}/${PACKAGE_IMAGE_TAR_NAME}" + + - if: "env.PACKAGE_IMAGE_PUBLISHED == 'true'" + name: "Login to Image Registry" + uses: "docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121" # v4.1.0 + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - if: "env.PACKAGE_IMAGE_PUBLISHED == 'true'" + name: "Pull the package image" + working-directory: "${{env.BUILD_DIR}}/clp-package" + shell: "bash" + run: >- + docker pull "$(cat clp-package-image.id)" + + - uses: "./tools/yscope-dev-utils/exports/github/actions/install-uv" + + - name: "Install MariaDB connector" + run: "sudo apt-get update && sudo apt-get install -y libmariadb-dev" + + - name: "Run the integration tests for the CLP package" + working-directory: "integration-tests" + env: + # Tell pytest where the CLP package is, and where it should store the test logs. + CLP_BUILD_DIR: "../${{env.BUILD_DIR}}" + CLP_PACKAGE_DIR: "../${{env.BUILD_DIR}}/clp-package" + run: >- + uv run pytest -m 'smoke or package' + + - name: "Upload the integration test logs" + if: "always()" + uses: "actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f" # v6.0.0 + with: + name: "clp-integration-test-package-logs-${{matrix.arch}}" + path: "${{env.BUILD_DIR}}/integration_tests/test_logs" + retention-days: 3 package-image-multiarch-manifest: name: "package-image-multiarch-manifest" if: >- github.event_name != 'pull_request' && github.ref == 'refs/heads/main' - && needs.package-image.result == 'success' - needs: "package-image" + && needs.package.result == 'success' + && needs.package-integration-tests.result == 'success' + needs: + - "package" + - "package-integration-tests" runs-on: *runner steps: - name: "Login to Image Registry" diff --git a/docs/src/dev-docs/tooling-gh-workflows.md b/docs/src/dev-docs/tooling-gh-workflows.md index 981da79537..838a301735 100644 --- a/docs/src/dev-docs/tooling-gh-workflows.md +++ b/docs/src/dev-docs/tooling-gh-workflows.md @@ -7,9 +7,9 @@ builds, testing, and linting. We briefly describe each workflow below. This workflow is responsible for: -1. building (Linux) container images containing CLP-core's dependencies, -2. building CLP-core and running its unit tests, and -3. building a container image containing CLP's package components. +1. building (Linux) container images containing CLP-core's dependencies. +2. building CLP-core and running its unit tests. +3. building CLP's package and package image, and running package integration tests. To minimize build times, the jobs in the workflow are organized in the directed acyclic graph (DAG) shown below. @@ -45,8 +45,9 @@ flowchart LR musllinux_1_2-deps-image --> musllinux_1_2-x86_64-binaries musllinux_1_2-deps-image-merge --> musllinux_1_2-x86_64-binaries ubuntu-jammy-deps-image --> ubuntu-jammy-binaries - ubuntu-jammy-deps-image --> package-image + ubuntu-jammy-deps-image --> package ubuntu-jammy-binaries --> ubuntu-jammy-binaries-image + package --> package-integration-tests ::: Arrows between jobs indicate a dependency. The jobs are as follows: @@ -75,7 +76,10 @@ Arrows between jobs indicate a dependency. The jobs are as follows: container and runs core's unit tests. * `musllinux_1_2-x86_64-binaries`: Builds the CLP-core binaries in the built musllinux_1_2 container and runs core's unit tests. -* `package-image`: Builds a container image containing CLP's package components. +* `package`: Builds CLP's package, along with a container image containing the package's components, + then uploads the package as a tarball artifact. +* `package-integration-tests`: Runs the package's integration tests against the package built by the + `package` job, and then uploads the logs from the tests. * `ubuntu-jammy-binaries`: Builds the CLP-core binaries in the built Ubuntu Jammy container and runs core's unit tests. * `ubuntu-jammy-binaries-image`: Builds an Ubuntu Jammy container image containing CLP-core's diff --git a/integration-tests/.pytest.ini b/integration-tests/.pytest.ini index 0cf24cf095..e4a42810c0 100644 --- a/integration-tests/.pytest.ini +++ b/integration-tests/.pytest.ini @@ -30,4 +30,5 @@ markers = core: mark tests that test the CLP core binaries package: mark tests that use the CLP package search: mark tests that test search + smoke: mark tests that are smoke tests startstop: mark tests that test start-stop diff --git a/integration-tests/tests/test_clp_native_py_project_imports.py b/integration-tests/tests/test_clp_native_py_project_imports.py index b3809843e0..b119452f42 100644 --- a/integration-tests/tests/test_clp_native_py_project_imports.py +++ b/integration-tests/tests/test_clp_native_py_project_imports.py @@ -1,11 +1,13 @@ """Smoke tests to validate that CLP Python projects can be imported without errors.""" +import pytest from clp_mcp_server.constants import QueryJobType from clp_package_utils.general import JobType from clp_py_utils.clp_config import StorageEngine from job_orchestration.scheduler.constants import CompressionJobStatus +@pytest.mark.smoke def test_clp_native_py_project_enum_classes() -> None: """ Verifies that the following CLP Python projects can be imported successfully by testing diff --git a/integration-tests/tests/utils/classes.py b/integration-tests/tests/utils/classes.py index 89c847945f..390d239e49 100644 --- a/integration-tests/tests/utils/classes.py +++ b/integration-tests/tests/utils/classes.py @@ -16,7 +16,7 @@ from tests.conftest import get_test_log_dir from tests.utils.utils import validate_dir_exists, validate_file_exists -DEFAULT_CMD_TIMEOUT_SECONDS = 120.0 +DEFAULT_CMD_TIMEOUT_SECONDS = 300.0 logger = logging.getLogger(__name__) diff --git a/taskfile.yaml b/taskfile.yaml index 385d57bd0b..551f0a9370 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -136,6 +136,26 @@ tasks: vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + cmds: + - task: "package-assemble" + vars: + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + - >- + rsync --archive + "{{.G_BUILD_DIR}}/clp-package-image.id" + "{{.OUTPUT_DIR}}" + - "echo '{{.G_PACKAGE_VERSION}}' > '{{.OUTPUT_DIR}}/VERSION'" + # This command must be last + - task: "utils:checksum:compute" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + + package-assemble: + requires: + vars: ["OUTPUT_DIR"] + deps: + - "generate-config-schemas" cmds: - "rm -rf '{{.OUTPUT_DIR}}'" - >- @@ -161,16 +181,6 @@ tasks: rsync --archive "tools/deployment/package/" "{{.OUTPUT_DIR}}" - - >- - rsync --archive - "{{.G_BUILD_DIR}}/clp-package-image.id" - "{{.OUTPUT_DIR}}" - - "echo '{{.G_PACKAGE_VERSION}}' > '{{.OUTPUT_DIR}}/VERSION'" - # This command must be last - - task: "utils:checksum:compute" - vars: - CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" - INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] core: cmds: @@ -246,6 +256,7 @@ tasks: COMPONENT: "{{.TASK}}" generate-config-schemas: + run: "once" vars: OUTPUT_DIR: "{{.G_CONFIG_SCHEMA_BUILD_DIR}}" cmds: