diff --git a/deployment-configuration/helm/templates/auto-database-postgres-backup.yaml b/deployment-configuration/helm/templates/auto-database-postgres-backup.yaml index d2ff59802..a514c8f85 100644 --- a/deployment-configuration/helm/templates/auto-database-postgres-backup.yaml +++ b/deployment-configuration/helm/templates/auto-database-postgres-backup.yaml @@ -27,7 +27,7 @@ spec: containers: - name: "{{ .app.harness.database.name }}-backup" imagePullPolicy: IfNotPresent - image: prodrigestivill/postgres-backup-local + image: {{ .root.Values.backup.image | default "prodrigestivill/postgres-backup-local" }} command: ["/bin/bash"] args: ["-c" , "./backup.sh"] env: diff --git a/deployment-configuration/helm/templates/auto-database.yaml b/deployment-configuration/helm/templates/auto-database.yaml index 40e48ac60..91ed94e82 100644 --- a/deployment-configuration/helm/templates/auto-database.yaml +++ b/deployment-configuration/helm/templates/auto-database.yaml @@ -105,7 +105,7 @@ spec: # Holds the pod until the volume-migration job has copied the legacy volume data into # the statefulset volume. Delete the legacy PVC after verifying the migration. - name: volume-migration - image: busybox:1.36 + image: {{ .root.Values.volumeMigration.wait.image | default "busybox:1.36" }} command: ['sh', '-c', 'until [ -f /migration/target/.cloudharness-volume-migrated ]; do echo "waiting for the volume migration job"; sleep 5; done'] volumeMounts: - name: {{ .app.harness.database.name | quote }} diff --git a/deployment-configuration/helm/templates/auto-deployments.yaml b/deployment-configuration/helm/templates/auto-deployments.yaml index d12201612..17efddd63 100644 --- a/deployment-configuration/helm/templates/auto-deployments.yaml +++ b/deployment-configuration/helm/templates/auto-deployments.yaml @@ -108,7 +108,7 @@ spec: # Holds the pod until the volume-migration job has copied the legacy volume data into # this pod's volume. Delete the legacy PVC after verifying the migration to drop this gate. - name: volume-migration - image: busybox:1.36 + image: {{ .root.Values.volumeMigration.wait.image | default "busybox:1.36" }} command: ['sh', '-c', 'until [ -f /migration/target/.cloudharness-volume-migrated ]; do echo "waiting for the volume migration job"; sleep 5; done'] volumeMounts: - name: {{ $volume.name }} diff --git a/deployment-configuration/helm/templates/auto-gatekeepers.yaml b/deployment-configuration/helm/templates/auto-gatekeepers.yaml index 6f4494848..540156a03 100644 --- a/deployment-configuration/helm/templates/auto-gatekeepers.yaml +++ b/deployment-configuration/helm/templates/auto-gatekeepers.yaml @@ -181,7 +181,7 @@ spec: {{ include "deploy_utils.etcHosts" .root | indent 6 }} containers: - name: {{ .app.harness.service.name | quote }} - image: {{ .app.harness.proxy.gatekeeper.image | default .root.Values.proxy.gatekeeper.image | default "quay.io/gogatekeeper/gatekeeper:4.6.0" }} + image: {{ .app.harness.proxy.gatekeeper.image | default .root.Values.source_images.GATEKEEPER | default "quay.io/gogatekeeper/gatekeeper:4.6.0" }} imagePullPolicy: IfNotPresent {{ if .root.Values.local }} securityContext: diff --git a/deployment-configuration/helm/templates/auto-volume-migration.yaml b/deployment-configuration/helm/templates/auto-volume-migration.yaml index 2b9d8b93c..690dd5170 100644 --- a/deployment-configuration/helm/templates/auto-volume-migration.yaml +++ b/deployment-configuration/helm/templates/auto-volume-migration.yaml @@ -66,7 +66,7 @@ spec: - name: volume-migration # Fully open source kubectl image (alpine-based: provides sh and tar), see # https://github.com/alpine-docker/k8s - image: alpine/k8s:1.34.9 + image: {{ .root.Values.volumeMigration.image | default "alpine/k8s:1.34.9" }} command: - sh - -c diff --git a/deployment-configuration/helm/values.yaml b/deployment-configuration/helm/values.yaml index 207bea565..5966c02e1 100644 --- a/deployment-configuration/helm/values.yaml +++ b/deployment-configuration/helm/values.yaml @@ -110,6 +110,8 @@ secretmanagers: {} # # -- How often the value is refreshed from AWS. # refreshInterval: 1h backup: + # -- Image running the database backup cronjob. Not built by CloudHarness. + image: prodrigestivill/postgres-backup-local # -- Flag to enable/disable backups. active: false # -- Number of days to keep backups. @@ -137,6 +139,14 @@ backup: memory: "64Mi" # -- K8s cpu resource definition. cpu: "50m" +# -- Images used by the generated volume migration resources. Not built by CloudHarness. +volumeMigration: + # -- Image of the job copying a legacy volume into its replacement. + image: alpine/k8s:1.34.9 + wait: + # -- Image of the init container holding pods until the migration job has completed. + image: busybox:1.36 + proxy: # -- Set to false to hide remote client headers. Will hide the client IPs in all logs forwardedHeaders: true @@ -150,9 +160,8 @@ proxy: # -- Maximum size of payload in MB max: 250 gatekeeper: - # -- Default gatekeeper image - image: "quay.io/gogatekeeper/gatekeeper:4.6.0" - # -- Default number of gatekeeper replicas + # -- Default number of gatekeeper replicas. The gatekeeper image is not set here: it is + # configured once, at source_images.GATEKEEPER. replicas: 1 secret: "" resources: diff --git a/deployment-configuration/value-template.yaml b/deployment-configuration/value-template.yaml index 6b85846d7..8a6b60904 100644 --- a/deployment-configuration/value-template.yaml +++ b/deployment-configuration/value-template.yaml @@ -202,8 +202,9 @@ harness: # -- Maximum size of payload in MB max: gatekeeper: - # -- Default gatekeeper image - image: "quay.io/gogatekeeper/gatekeeper:4.6.0" + # -- Overrides source_images.GATEKEEPER for this application only. Left unset so that the + # gatekeeper image is configured once, at source_images.GATEKEEPER. + image: # -- Default number of gatekeeper replicas replicas: 1 # -- Native Gatekeeper proxy.yml settings. Kebab-case keys follow the diff --git a/deployment-configuration/values-template.yaml b/deployment-configuration/values-template.yaml index 4dee15469..fb6aceb7b 100644 --- a/deployment-configuration/values-template.yaml +++ b/deployment-configuration/values-template.yaml @@ -1 +1,6 @@ debug: false + +source_images: + # -- Gatekeeper image, pulled by the gatekeeper deployment generated for every secured + # application. Set here rather than per application so that it is configured in one place. + GATEKEEPER: quay.io/gogatekeeper/gatekeeper:4.6.0 diff --git a/docs/README.md b/docs/README.md index 3f669ef49..ec92b2356 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,6 +7,8 @@ - [Work with local deployments](./build-deploy/local-deploy/README.md) - [Debug your applications](./build-deploy/local-deploy/debug.md) - [Setting up Minikube](./build-deploy/local-deploy/minikube.md) +- [Define image sources, e.g. a private registry](./image-sources.md) +- [Define and use base and common images](./base-common-images.md) - [Create and configure applications](./applications) - [Generate a new application](./applications/harness-application.md) - [Override applications values and files](./applications/overridings.md) diff --git a/docs/applications/development/backend-development.md b/docs/applications/development/backend-development.md index 169f495e0..f909a651d 100644 --- a/docs/applications/development/backend-development.md +++ b/docs/applications/development/backend-development.md @@ -55,7 +55,7 @@ harness: Every image defined as a base image or a common image can be used as a build dependency. -For more details about how to define your custom image and the available images, see [here](../../base-common-images.md). +For more details about how to define your custom image and the available images, see [base and common images](../../base-common-images.md). For more info about dependencies, see [here](../dependencies.md) diff --git a/docs/base-common-images.md b/docs/base-common-images.md index d1c7bdeda..4d891a98e 100644 --- a/docs/base-common-images.md +++ b/docs/base-common-images.md @@ -1,5 +1,13 @@ # Define and use base and common images +Base and common images are the images CloudHarness builds for your applications to inherit from as build dependencies. + +Some default base and common images are provided by Cloud Harness and can be customized in your project; following the same pattern, can also define other images that can be reused across your project applications and tasks. +Define them when you want your applications to share libraries, tooling or a common stack. + +> Note: If you are interested in customizing where external base images are pulled from, +> see [image sources](image-sources.md). + ## Relevant files and directory structure - `base-images`: base Docker images. Those images can used as base images in CloudHarness apps and tasks. @@ -62,73 +70,8 @@ To override cloudharness-base, create a directory `MY_SOLUTION/infrastructure/ba then run `harness-deployment cloudharness MY_SOLUTION` -## Change the base image of any application (`FROM [xxx]`) - -It's possible to change the image each dockerfile inherits by using the `FROM` directive. -Depending on the kind of application, changing the base image is done in two different ways. -There is basically two main situations: - -. your application has a `Dockerfile` (e.g: Keycloak, the app you're building); -. your application doesn't have a `Dockerfile` (e.g: gogatekeeper, Kafka, etc), and the image is directly injected inside the helm chart when the helm chart is generated. - -In both situations, the modification is done in the `value-template.yaml` of your CloudHarness projects, but the YAML path to modify depends on the app your targetting then. - -### Change the base image for applications with a `Dockerfile` - -Changing the base image for your application or applications which have a `Dockerfile` is done through the `source_images` entry of your `value-template.yaml` file. -This entry defines a mapping between the `ARG` of your `Dockerfile` and the value you want to inject. -Here is an example of a declared mapping to change the base image for Python based apps and Node base apps: - -```yaml -# value-template.yaml -source_images: - NODE: "mybaseimg:14.5" - PYTHON: "myotherimage:15.1" -``` - -The list of the base images variables and the value they resolve to is automatically generated in the `helm/values.yaml` file which is generated by the `harness-deployment` command under the `source_images` key. You can then copy/paste the entries you need in your `value-template.yaml` file and tweak the values there. - - -### Change the base image for applications which inject their image in the helm chart - -Those applications are not providing a Dockerfile, but directly an image which is injcted in the helm chart from the helm template. -Each of those applications considers a specific path from the helm configuration (i.e: `values.yaml` or `value-template.yaml`). -To change the base image for those, you need to know this path and change it in the `value-template.yaml` by redefining it. -The redefinition will override the base value from the sub-`values.yaml` files on configuration merging. -Here is an example for the gatekeeper and the Argo controller: - -```yaml -# value-template.yaml -harness: - proxy: - gatekeeper: - image: YOUR_NEW_IMG - -# or, depending on your application -argo: - controller: - image: - registry: YOUR_REG - tag: YOUR TAG -``` +## Change the image a Dockerfile inherits from -Here is a table of the identified applications inside of CloudHarness which do not own a `Dockerfile`. - -| Application | Image path | -| --- | --- | -| gatekeeper | harness.proxy.gatekeeper.image | -| Argo controller | argo.controller.image.{repository, tag} | -| Argo executor | argo.executor.image.{repository, tag} | -| Argo server | argo.server.image.{repository, tag} | -| Elasticsearch | elasticsearch.image | -| Events | events.image.{repository, tag, nullPolicy} | -| Events kafka | events.kafka.image | -| JupyterHub | jupyterhub.singleuser.image.{pullSecrets, name, tag} | -| JupyterHub proxy | jupyterhub.proxy.chp.image.{name, tag, pullPolicy} | -| JupyterHub scheduling | jupyterhub.scheduling.userPlaceHolder.image.{name, pullPolicy, tag} | -| NFS server | nfsserver.harness.deployment.image | -| Sentry redis | sentry.redis.image | -| MongoDB | harness.database.mongo.image | -| Neo4J | harness.database.neo4j.image | -| Neo4J reverseProxy | neo4j.reverseProxy.image | -| Postgres | harness.database.postgres.image | +Defining a base image is not the only way to change what an application's `Dockerfile` inherits +from. The image each `FROM` resolves to is itself configurable, per deployment, and is described in +[image sources](image-sources.md). diff --git a/docs/build-deploy/helm-configuration.md b/docs/build-deploy/helm-configuration.md index f5933bd5a..5a9838423 100644 --- a/docs/build-deploy/helm-configuration.md +++ b/docs/build-deploy/helm-configuration.md @@ -1,6 +1,6 @@ ## Override Helm values -Edit `./deployment-configuration/values-template.yaml` to override any helm chart values file. Refer to the default [Helm values file](tools/cloudharness_utilities/deployment-configuration/helm/values.yaml) for documentation about each value. +Edit `./deployment-configuration/values-template.yaml` to override any helm chart values file. Refer to the default [Helm values file](../../deployment-configuration/helm/values.yaml) for documentation about each value. For environment specific configurations edit `deployment-configuration/values-template-[ENV].yaml`. @@ -9,7 +9,19 @@ For environment specific configurations edit `deployment-configuration/values-te ## Override base/common images Base images are built from the infrastructure/base-images sub folders. -See the dedicated [base images](./docs/applications.md) document for more details and examples. +See the dedicated [base and common images](../base-common-images.md) document for more details and examples. + +## Override the images the chart pulls + +Every run of `harness-deployment` writes `deployment/helm/values-overrides.yaml` alongside the +generated `values.yaml`, listing every image the chart pulls at the exact path Helm reads it from: +the `source_images` build arguments, the vendored sub-chart images keyed by sub-chart name, and the +images each application declares inline or pulls through its harness configuration. + +The file is a reference and is not applied by any deployment flow. Use it to find the path of an +image, try a replacement out by passing it explicitly (`helm ... -f +./deployment/helm/values-overrides.yaml`), then make the change permanent as described in +[image sources](../image-sources.md#make-an-override-permanent). ## Single application deployment configuration diff --git a/docs/image-sources.md b/docs/image-sources.md new file mode 100644 index 000000000..555c6a992 --- /dev/null +++ b/docs/image-sources.md @@ -0,0 +1,219 @@ +# Image sources + +An image source is the concrete image a deployment builds from or pulls, and this document is about +declaring those. It is what you need when the images have to come from somewhere other than their +default public location, typically a closed artifact registry in an on-premise installation. + +If instead you want to define or customize the base images your applications inherit from, see +[base and common images](base-common-images.md). + +A CloudHarness deployment takes its images from two places: + +- the images CloudHarness **builds** from a `Dockerfile`, whose `FROM` is resolved through the + `source_images` build arguments; +- everything it does not build but **pulls**: the images of the vendored sub-charts (Argo, Kafka UI, + Elasticsearch), the ones an application declares in its own values (JupyterHub, Kafka), the ones + in its harness configuration (databases, gatekeepers, extra containers), and the ones the + generated resources themselves run (database backup, volume migration). + +Both kinds are listed together in the generated `values-overrides.yaml`, described in +[Discover every image the deployment uses](#discover-every-image-the-deployment-uses). For mirroring +a whole deployment into your own registry, start there. + +In both cases the change is made in the deployment configuration of your own project, but the file +and the YAML path depend on which of the two kinds the image belongs to. + +## Change the base image for applications with a `Dockerfile` + +Changing the base image for your application or applications which have a `Dockerfile` is done +through the `source_images` entry of your `values-template.yaml` file. This entry defines a mapping +between the `ARG` of your `Dockerfile` and the value you want to inject. Here is an example of a +declared mapping to change the base image for Python based apps and Node based apps: + +```yaml +# deployment-configuration/values-template.yaml +source_images: + NODE: "mybaseimg:14.5" + PYTHON: "myotherimage:15.1" +``` + +The list of the base image variables and the value they resolve to is automatically generated by the +`harness-deployment` command under the `source_images` key of both the generated `helm/values.yaml` +and `helm/values-overrides.yaml` files. You can then copy/paste the entries you need in your +`values-template.yaml` file and tweak the values there. + +Note that this goes in `values-template.yaml` (plural *values*), which is merged at the root of the +generated configuration. A `source_images` entry placed in `value-template.yaml` (singular) is +silently ignored, because that file holds per-application defaults and the per-application +`source_images` are overwritten by the `ARG` values read from the application's own `Dockerfile`. + +These variables are passed to the builds as `--build-arg`, so they only affect the images that +CloudHarness builds itself. Images that are merely pulled at runtime are listed separately, as +described below. + +## Change the image of applications which inject their image in the helm chart + +Those applications do not provide a Dockerfile, but directly an image which is injected in the helm +chart from the helm template, either by a vendored sub-chart or by the application's own values. +Rather than looking such a path up by hand, read it off the generated file described next, then +redefine it as explained in [Make an override permanent](#make-an-override-permanent). + +## Discover every image the deployment uses + +You do not need to look up any path by hand. Every time `harness-deployment` runs it writes +`deployment/helm/values-overrides.yaml` next to the generated `values.yaml`, listing every image the +chart pulls, at the exact path Helm reads it from: + +```yaml +# deployment/helm/values-overrides.yaml (generated) +source_images: # base images injected as Dockerfile build arguments + NODE: 'node:22-alpine' + PYTHON: 'python:3.12-slim-trixie' +argo-workflows: # vendored sub-chart, keyed by its own chart name + controller: + image: {registry: quay.io, repository: argoproj/workflow-controller, tag: v3.1.15} +kafka-ui: + image: {registry: docker.io, repository: provectuslabs/kafka-ui, tag: v0.7.2} +elasticsearch: {image: docker.elastic.co/elasticsearch/elasticsearch, imageTag: 8.17.0} +apps: # images declared inline by an application + events: + kafka: {image: 'docker.io/apache/kafka:4.0.2'} + jupyterhub: + hub: + image: {name: quay.io/jupyterhub/k8s-hub, tag: '3.2.1'} +``` + +The file is a reference, not part of any deployment: no pipeline applies it, since it is regenerated +on every run and would only ever restate the values already in effect. It is a real Helm values +file though, so you can pass it explicitly to try an image out: + +```bash +harness-deployment cloudharness . -i events -i argo +# edit deployment/helm/values-overrides.yaml, then +helm template deployment/helm -f deployment/helm/values-overrides.yaml +``` + +Because the file is regenerated on every run, use it to *discover* paths and to try an image out. +Make the change permanent by redefining the same path in your own solution, as described below. + +The only images left out are the ones CloudHarness builds itself, which are not an image source to +redirect: an application's own `image` and `harness.deployment.image`. An application that declares +a prebuilt image instead of being built (`build: false`) does not build anything, so its +`harness.deployment.image` is listed like any other pulled image. + +Every listed path is one that genuinely takes effect: a value shadowed by something of higher +precedence is not reported, so editing any entry in the file changes what gets deployed. An +application declaring `harness.database.image_ref`, for instance, runs the task image built under +that reference, so its `harness.database..image` is left out rather than shown as an override +that would do nothing. This is what makes the file usable for repointing a deployment wholesale, +for instance with `yq`. An image shared by many applications, such as the gatekeeper, is listed +once rather than repeated on each of them. + +## Make an override permanent + +Which file to edit depends on where the image sits in the generated configuration. The two cases are +distinct, and using the wrong one fails silently. + +**Vendored sub-chart images** go at the root of `deployment-configuration/values-template.yaml`, +under the sub-chart key, using the same paths as the generated file. Only the fields you list are +overridden, the rest keep the vendored defaults: + +```yaml +# deployment-configuration/values-template.yaml +argo-workflows: + controller: + image: + tag: v3.7.2 +kafka-ui: + image: + registry: myregistry.io + repository: mirror/kafka-ui +``` + +**Images an application declares inline** (everything the generated file lists under `apps.`) +are overridden by [overriding the application](applications/overridings.md): create the same values +file in your own solution and redefine the path, without the `apps.` prefix. + +```yaml +# applications/jupyterhub/deploy/values.yaml, in your own solution +singleuser: + image: + name: myregistry.io/my/singleuser + tag: "9.9.9" +``` + +Do not put an `apps:` block in `values-template.yaml` to this end: the generator resets that key +before loading the applications, so such an override is silently discarded even though the generated +file displays the image under `apps.`. + +A vendored sub-chart is addressed by the `name` in its own `Chart.yaml`, which is not always the +CloudHarness application name. This is how Helm passes parent values down to a sub-chart, so using +the application directory name instead (`argo:` rather than `argo-workflows:`) silently does +nothing. The current mapping is: + +| Application | Sub-chart key | +| --- | --- | +| argo | `argo-workflows` | +| events | `kafka-ui` | +| elasticsearch | `elasticsearch` | + +Here are the identified images inside of CloudHarness which do not come from a `Dockerfile`, grouped +by how each one is overridden. The paths are the ones shown by `values-overrides.yaml`, i.e. +root-relative in the generated `values.yaml`. + +Vendored sub-charts, overridden at the root of `values-template.yaml`: + +| Application | Image path | +| --- | --- | +| Argo controller | `argo-workflows.controller.image.{registry, repository, tag}` | +| Argo executor | `argo-workflows.executor.image.{registry, repository, tag}` | +| Argo server | `argo-workflows.server.image.{registry, repository, tag}` | +| Events UI | `kafka-ui.image.{registry, repository, tag}` | +| Elasticsearch | `elasticsearch.image` and `elasticsearch.imageTag` | + +Images declared inline by an application, overridden by redefining the path in that application's +`deploy/values.yaml` in your own solution (drop the `apps.` prefix): + +| Application | Image path | +| --- | --- | +| Events kafka | `apps.events.kafka.image` | +| JupyterHub hub | `apps.jupyterhub.hub.image.{name, tag}` | +| JupyterHub singleuser | `apps.jupyterhub.singleuser.image.{name, tag}` | +| JupyterHub proxy | `apps.jupyterhub.proxy.chp.image.{name, tag}` | +| JupyterHub scheduling | `apps.jupyterhub.scheduling.userScheduler.image.{name, tag}` | +| JupyterHub prepuller | `apps.jupyterhub.prePuller.hook.image.{name, tag}` | +| Neo4J reverseProxy | `apps.neo4j.reverseProxy.image` | +| Sentry redis | `apps.sentry.redis.image` | + +Images an application pulls as part of its harness configuration. These are set per application the +same way, or for every application at once through `value-template.yaml`: + +| Image | Image path | +| --- | --- | +| MongoDB | `apps..harness.database.mongo.image` | +| Neo4J | `apps..harness.database.neo4j.image` | +| Postgres | `apps..harness.database.postgres.image` | +| Database built by CloudHarness | not an image source: see `harness.database.image_ref` | +| extra containers | `apps..harness.deployment.extraContainers..image` | +| prebuilt application image (`build: false` only) | `apps..harness.deployment.image` | + +Images the generated resources run themselves, overridden at the root of `values-template.yaml`: + +| Image | Image path | +| --- | --- | +| Gatekeeper | `source_images.GATEKEEPER` | +| Database backup cronjob | `backup.image` | +| Volume migration job | `volumeMigration.image` | +| Volume migration wait init container | `volumeMigration.wait.image` | + +A gatekeeper is generated for every secured application, so its image is configured once, at +`source_images.GATEKEEPER`, instead of being repeated on each application. A single application can +still opt out through `apps..harness.proxy.gatekeeper.image`, which is unset by default and so +does not show up in `values-overrides.yaml` until you set it. + +> The root `proxy.gatekeeper.image` used to override the gatekeeper of every application. It is no +> longer read: set `source_images.GATEKEEPER` instead. `harness-deployment` warns if it is still set. + +The JupyterHub hub image is a special case: JupyterHub's own template renders the hub pod from the +image CloudHarness builds for the application, so `apps.jupyterhub.hub.image` is listed and can be +set but does not change the container that runs. diff --git a/libraries/cloudharness-utils/cloudharness_utils/constants.py b/libraries/cloudharness-utils/cloudharness_utils/constants.py index b4540072c..00050e23e 100644 --- a/libraries/cloudharness-utils/cloudharness_utils/constants.py +++ b/libraries/cloudharness-utils/cloudharness_utils/constants.py @@ -28,6 +28,11 @@ CF_TEMPLATE_PUBLISH_PATH = f'{DEPLOYMENT_CONFIGURATION_PATH}/codefresh-publish-template.yaml' VALUES_MANUAL_PATH = 'values.yaml' +# Generated next to values.yaml as a reference: every image the chart pulls (source_images for +# builds, vendored sub-chart images keyed by sub-chart name, application images under apps.*), in +# the structure Helm consumes. No pipeline applies it; it is passed to helm explicitly when trying +# an image out. +VALUES_OVERRIDES_PATH = 'values-overrides.yaml' VALUE_TEMPLATE_PATH = f'{DEPLOYMENT_CONFIGURATION_PATH}/value-template.yaml' CH_BASE_IMAGES = {'cloudharness-base': 'python:3.9.10'} diff --git a/tools/deployment-cli-tools/ch_cli_tools/helm.py b/tools/deployment-cli-tools/ch_cli_tools/helm.py index 989fcbcdf..0e742755a 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/helm.py +++ b/tools/deployment-cli-tools/ch_cli_tools/helm.py @@ -9,10 +9,10 @@ from hashlib import sha1 import subprocess -from cloudharness_utils.constants import VALUES_MANUAL_PATH, HELM_CHART_PATH +from cloudharness_utils.constants import VALUES_MANUAL_PATH, VALUES_OVERRIDES_PATH, HELM_CHART_PATH from .utils import get_cluster_ip, get_dockerfile_baseimg_args, get_git_commit_hash, get_image_name, image_name_from_dockerfile_path, \ get_template, merge_to_yaml_file, dict_merge, app_name_from_path, \ - find_dockerfiles_paths + find_dockerfiles_paths, write_values_overrides from .models import HarnessMainConfig @@ -147,6 +147,9 @@ def process_values(self) -> HarnessMainConfig: collect_apps_helm_templates(root_path, exclude=self.exclude, include=self.include, dest_helm_chart_path=self.dest_deployment_path, envs=self.env) + # Collect every image the chart pulls into values-overrides.yaml, in Helm-native structure + write_values_overrides(helm_values, self.dest_deployment_path) + # Save values file for manual helm chart merged_values = merge_to_yaml_file(helm_values, os.path.join( self.dest_deployment_path, VALUES_MANUAL_PATH)) diff --git a/tools/deployment-cli-tools/ch_cli_tools/utils.py b/tools/deployment-cli-tools/ch_cli_tools/utils.py index 765d8e9b8..61c6d8f43 100644 --- a/tools/deployment-cli-tools/ch_cli_tools/utils.py +++ b/tools/deployment-cli-tools/ch_cli_tools/utils.py @@ -4,6 +4,7 @@ import socket import glob import subprocess +from dataclasses import dataclass from typing import Any, Union import requests import os @@ -13,6 +14,7 @@ import collections import re from ruamel.yaml import YAML +from ruamel.yaml.comments import CommentedMap import shutil import logging import fileinput @@ -20,7 +22,7 @@ from pathlib import Path from cloudharness_utils.constants import NEUTRAL_PATHS, DEPLOYMENT_CONFIGURATION_PATH, BASE_IMAGES_PATH, STATIC_IMAGES_PATH, \ - APPS_PATH, EXCLUDE_PATHS + APPS_PATH, EXCLUDE_PATHS, VALUES_OVERRIDES_PATH from . import CH_ROOT yaml = YAML(typ='safe') @@ -594,6 +596,191 @@ def get_image_source(helm_values) -> dict[str, str]: return helm_values.get("source_images", {}) +@dataclass(frozen=True) +class ChartImageRef: + """A runtime (non-built) image reference discovered inside a values dict. + + `path` locates the node inside the scanned dict; `value` is what a values file must + hold there to select the image: the flat string, or a dict of only the identifying + fields (registry/repository/name/tag), with an empty tag resolved to the chart appVersion. + """ + path: tuple + value: Union[str, dict] + + +def nested_get(doc: dict, path: tuple): + for segment in path: + if not isinstance(doc, dict) or segment not in doc: + return None + doc = doc[segment] + return doc + + +def nested_set(doc: dict, path: tuple, value) -> dict: + node = doc + for segment in path[:-1]: + node = node.setdefault(segment, {}) + node[path[-1]] = value + return doc + + +def find_chart_images(node: dict, *, skip_paths: frozenset = frozenset(), + app_version: Union[str, None] = None, + _path: tuple = ()) -> list: + """Heuristically finds references to images that CloudHarness does not build itself. + + Matches only an exact key named "image" - never by substring, which would also catch + imagePullSecrets/pullPolicy - holding one of these shapes: + - a flat string: image: "docker.io/x/y:1" + (a sibling `imageTag` key, as in the elasticsearch chart, is reported as well) + - a {repository, registry?, tag?} dict (registry is optional) + - a {name, tag?} dict + + A dict under "image" matching neither shape (e.g. {pullPolicy, pullSecrets}) is not + recorded and not recursed into - its sub-keys are never image references. + + `skip_paths` holds paths (as tuples, relative to the scanned dict) that are neither + recorded nor recursed into. Callers use it to prune the images CloudHarness builds + itself, which are not overridable image sources: an application's own `image` and + `harness.deployment.image`, and the `apps` subtree when it is scanned separately. + + `app_version` resolves an empty/missing "tag": vendored charts ship tag: "" and rely on + Helm's `tag | default .Chart.AppVersion`, so pass the sub-chart appVersion for the + reported value to state what is actually deployed. + """ + found = [] + for key, value in node.items(): + path = _path + (key,) + if path in skip_paths: + continue + if key == 'image': + if isinstance(value, str) and value: + found.append(ChartImageRef(path=path, value=value)) + if 'imageTag' in node: + found.append(ChartImageRef(path=_path + ('imageTag',), value=node['imageTag'])) + elif isinstance(value, dict): + identity = {k: value[k] for k in ('registry', 'repository', 'name') if k in value} + if 'repository' in identity or 'name' in identity: + tag = value.get('tag') or app_version + if tag: + identity['tag'] = tag + found.append(ChartImageRef(path=path, value=identity)) + continue + if isinstance(value, dict): + found.extend(find_chart_images(value, skip_paths=skip_paths, + app_version=app_version, _path=path)) + return found + + +def write_values_overrides(helm_values: dict, dest_deployment_path: pathlib.Path) -> None: + """Writes values-overrides.yaml next to values.yaml: every image the chart pulls but does + not build, in the structure Helm consumes. It is a reference for looking a path up and + trying an image out by passing the file to helm explicitly; no pipeline applies it, since it + is regenerated on every run and would only restate the values already in effect. + + - source_images: the Dockerfile base images (build arguments) aggregated at the root + - : the images of each vendored sub-chart copied under charts/, keyed + by the sub-chart's own Chart.yaml name - the key Helm uses to pass parent values down. + Empty tags are resolved from the sub-chart appVersion so the file states what is deployed. + - apps.: the images an application pulls, both the ones declared inline in its own + values (e.g. jupyterhub) and the ones under its harness configuration (database, + gatekeeper, extra containers) + - the remaining root paths: images the generated resources themselves run, such as the + database backup and the volume migration containers + + The images CloudHarness builds are left out, not being an overridable image source: an + application's own `image` and `harness.deployment.image` are pruned, unless the application + declares a prebuilt image instead of being built (`build: false`). + + Values already set at those paths (e.g. from values-template-.yaml) win over the + vendored defaults, so the file always reflects the effective configuration. + """ + apps_key, harness_key, database_key, deployment_key = 'apps', 'harness', 'database', 'deployment' + + overrides = {} + key_comments = {} + if helm_values.get("source_images"): + overrides["source_images"] = dict(helm_values["source_images"]) + key_comments["source_images"] = ( + "Dockerfile base images, injected as build arguments.\n" + "Origin: the ARG defaults declared in each application's own Dockerfile (and\n" + "CloudHarness's base/static image Dockerfiles), aggregated here at build time." + ) + + # proxy.gatekeeper.image used to override the gatekeeper image for every application. + # The gatekeeper is now configured at source_images.GATEKEEPER, so no template reads it: + # reporting it would advertise a path that overrides nothing. + legacy_gatekeeper_path = ('proxy', 'gatekeeper', 'image') + if nested_get(helm_values, legacy_gatekeeper_path): + logging.warning( + "proxy.gatekeeper.image is set but no longer used: set source_images.GATEKEEPER " + "to change the gatekeeper image for every application, or " + "apps..harness.proxy.gatekeeper.image for a single one") + + # Images the generated resources run, declared at the root of the chart values + for ref in find_chart_images(helm_values, skip_paths=frozenset({(apps_key,), legacy_gatekeeper_path})): + nested_set(overrides, ref.path, ref.value) + key_comments.setdefault(ref.path[0], ( + "Image run by a resource generated by the chart itself.\n" + "Origin: CloudHarness's own chart defaults, deployment-configuration/helm/values.yaml." + )) + + for app_name, app_values in helm_values[apps_key].items(): + skip_paths = set() + # A built application owns its image; everything else it merely pulls + if app_values.get('build', False): + skip_paths |= {('image',), (harness_key, deployment_key, 'image')} + # An application declaring database.image_ref runs the task image built under that + # reference, which shadows the database type's own image: that one is then not the + # value to override, so it is not reported as one + harness_values = app_values.get(harness_key) or {} + if (harness_values.get(database_key) or {}).get('image_ref'): + skip_paths.add((harness_key, database_key)) + for ref in find_chart_images(app_values, skip_paths=frozenset(skip_paths)): + nested_set(overrides, (apps_key, app_name) + ref.path, ref.value) + key_comments.setdefault(apps_key, ( + "Images declared inline by an application, or pulled through its harness\n" + "configuration (database, gatekeeper, extra containers).\n" + "Origin: declared inline in the application's own deploy/values.yaml, or\n" + "inherited from the harness-wide defaults in deployment-configuration/value-template.yaml\n" + "(singular)." + )) + + chart_dir = dest_deployment_path / 'charts' / app_name + chart_values_path = chart_dir / 'values.yaml' + if not chart_values_path.exists(): + continue + chart_meta_path = chart_dir / 'Chart.yaml' + chart_meta = (load_yaml(chart_meta_path) if chart_meta_path.exists() else None) or {} + chart_name = chart_meta.get('name') or app_name + for ref in find_chart_images(load_yaml(chart_values_path) or {}, app_version=chart_meta.get('appVersion')): + value = ref.value + existing = nested_get(helm_values, (chart_name,) + ref.path) + if existing is not None: + value = dict_merge(value, existing) if isinstance(value, dict) and isinstance(existing, dict) else existing + nested_set(overrides, (chart_name,) + ref.path, value) + key_comments.setdefault(chart_name, ( + f"Images of the vendored '{chart_name}' sub-chart (from the {app_name} application),\n" + "keyed by its own Chart.yaml name - the key Helm uses to pass parent values down.\n" + f"Origin: the sub-chart's own default values, vendored (checked in as-is from the\n" + f"upstream chart) at applications/{app_name}/deploy/charts/values.yaml." + )) + + header = ( + "This file is a reference, auto-generated by `harness-deployment` and overwritten on\n" + "every run. No pipeline applies it: it lists every image the chart pulls but does not\n" + "build, at the exact path Helm reads it from, so you can look a path up here or try an\n" + "image out with `helm ... -f `. The comment on each key below states where\n" + "that value originates. To make a change permanent instead, redefine the same path in\n" + "your own deployment configuration - which file depends on where the image sits, as\n" + "explained in docs/image-sources.md (e.g. a per-environment\n" + "deployment-configuration/values-template-.yaml for most root-level keys, but each\n" + "application's own deploy/values.yaml for anything under apps.)." + ) + save_yaml_with_comments(dest_deployment_path / VALUES_OVERRIDES_PATH, overrides, + header=header, key_comments=key_comments) + + def check_response_200(endpoint_url, headers=None): resp = requests.get(endpoint_url, headers=headers, timeout=5) return resp.status_code == 200 @@ -679,6 +866,31 @@ def save_yaml(yaml_file: pathlib.Path, data: dict) -> None: yaml.dump(data, file) +def _sorted_deep(value): + if isinstance(value, dict): + return {key: _sorted_deep(value[key]) for key in sorted(value)} + if isinstance(value, list): + return [_sorted_deep(item) for item in value] + return value + + +def save_yaml_with_comments(yaml_file: pathlib.Path, data: dict, header: str = None, + key_comments: dict[str, str] = None) -> None: + """Writes data as YAML (keys sorted at every level, like `save_yaml`), with a comment block + before the document and, optionally, a comment before each named top-level key.""" + commented = CommentedMap(_sorted_deep(data)) + if header: + commented.yaml_set_start_comment(header) + for key, comment in (key_comments or {}).items(): + if key in commented: + # Leading blank line separates each key's comment from the previous entry + commented.yaml_set_comment_before_after_key(key, before='\n' + comment) + handler = YAML() + handler.default_flow_style = False + with yaml_file.open('w') as file: + handler.dump(commented, file) + + def get_apps_paths(root, app_name) -> tuple[str]: apps_path = [] diff --git a/tools/deployment-cli-tools/tests/resources/applications/chartimgapp/Dockerfile b/tools/deployment-cli-tools/tests/resources/applications/chartimgapp/Dockerfile new file mode 100644 index 000000000..c35f1b5f5 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/applications/chartimgapp/Dockerfile @@ -0,0 +1 @@ +FROM scratch diff --git a/tools/deployment-cli-tools/tests/resources/applications/chartimgapp/deploy/charts/Chart.yaml b/tools/deployment-cli-tools/tests/resources/applications/chartimgapp/deploy/charts/Chart.yaml new file mode 100644 index 000000000..a2d84fde3 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/applications/chartimgapp/deploy/charts/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +name: vendored-thing +version: 0.1.0 +appVersion: "2.0.0" diff --git a/tools/deployment-cli-tools/tests/resources/applications/chartimgapp/deploy/charts/templates/configmap.yaml b/tools/deployment-cli-tools/tests/resources/applications/chartimgapp/deploy/charts/templates/configmap.yaml new file mode 100644 index 000000000..7835a106b --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/applications/chartimgapp/deploy/charts/templates/configmap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: vendored-thing-image +data: + image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" diff --git a/tools/deployment-cli-tools/tests/resources/applications/chartimgapp/deploy/charts/values.yaml b/tools/deployment-cli-tools/tests/resources/applications/chartimgapp/deploy/charts/values.yaml new file mode 100644 index 000000000..821bb0b60 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/applications/chartimgapp/deploy/charts/values.yaml @@ -0,0 +1,4 @@ +image: + registry: docker.io + repository: someorg/somerepo + tag: "" diff --git a/tools/deployment-cli-tools/tests/resources/applications/chartimgapp/deploy/values.yaml b/tools/deployment-cli-tools/tests/resources/applications/chartimgapp/deploy/values.yaml new file mode 100644 index 000000000..dcb5f45d0 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/applications/chartimgapp/deploy/values.yaml @@ -0,0 +1,10 @@ +harness: + subdomain: chartimgapp + +worker: + image: docker.io/baz/qux:9 + +sidecar: + image: + name: quay.io/foo/bar + tag: "1.2.3" diff --git a/tools/deployment-cli-tools/tests/resources/deployment-configuration/values-template-chartimages.yaml b/tools/deployment-cli-tools/tests/resources/deployment-configuration/values-template-chartimages.yaml new file mode 100644 index 000000000..990ef5200 --- /dev/null +++ b/tools/deployment-cli-tools/tests/resources/deployment-configuration/values-template-chartimages.yaml @@ -0,0 +1,5 @@ +# Helm-native sub-chart override: keyed by the vendored sub-chart's Chart.yaml name. +# Only the tag is overridden; registry/repository come from the vendored defaults. +vendored-thing: + image: + tag: "9.9.9" diff --git a/tools/deployment-cli-tools/tests/test_codefresh.py b/tools/deployment-cli-tools/tests/test_codefresh.py index 93f6d2dd8..716c91186 100644 --- a/tools/deployment-cli-tools/tests/test_codefresh.py +++ b/tools/deployment-cli-tools/tests/test_codefresh.py @@ -506,6 +506,9 @@ def test_codefresh_db_connect_string_secret(): envs=['dev'], base_image_name=values['name'], helm_values=values, save=False) + # values-overrides.yaml is a reference for discovering image paths, not applied automatically + assert cf['steps']['deployment']['arguments']['custom_value_files'] == [ + './deployment/helm/values.yaml'] custom_values = cf['steps']['deployment']['arguments']['custom_values'] expected = "apps_myapp_harness_database_connect__string=\"${{MYAPP_DB_CONNECT_STRING}}\"" assert expected in custom_values, \ diff --git a/tools/deployment-cli-tools/tests/test_helm.py b/tools/deployment-cli-tools/tests/test_helm.py index 928dc3804..a106e3d27 100644 --- a/tools/deployment-cli-tools/tests/test_helm.py +++ b/tools/deployment-cli-tools/tests/test_helm.py @@ -1,20 +1,17 @@ -from ch_cli_tools.helm import * -from ch_cli_tools.configurationgenerator import * -from ch_cli_tools import configurationgenerator -from ch_cli_tools.preprocessing import preprocess_build_overrides, generate_hash_based_image_tags + import logging import pytest import shutil import subprocess -import pytest -from ch_cli_tools import configurationgenerator -from ch_cli_tools.configurationgenerator import * from ch_cli_tools.helm import * +from ch_cli_tools.configurationgenerator import * +from ch_cli_tools import configurationgenerator from ch_cli_tools.preprocessing import ( generate_hash_based_image_tags, preprocess_build_overrides, ) +from ch_cli_tools.utils import find_chart_images, ChartImageRef HERE = os.path.dirname(os.path.realpath(__file__)) RESOURCES = os.path.join(HERE, 'resources') @@ -25,9 +22,11 @@ def exists(path): return path.exists() -def render_helm_chart(chart_path): +def render_helm_chart(chart_path, values_files=(), set_values=()): completed = subprocess.run( - ["helm", "template", str(chart_path)], + ["helm", "template", str(chart_path), + *(arg for f in values_files for arg in ("-f", str(f))), + *(arg for v in set_values for arg in ("--set", v))], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -202,6 +201,8 @@ def test_collect_helm_values_noreg_noinclude(tmp_path): "mybase": "foo:bar", "mybase2": "spam:egg", "NODE": "node:22-alpine", + # Declared by CloudHarness itself: the gatekeeper is configured in one place + "GATEKEEPER": "quay.io/gogatekeeper/gatekeeper:4.6.0", } @@ -1426,3 +1427,273 @@ def test_collect_helm_values_source_images_merge_no_include(tmp_path): source_images = values.get("source_images") assert source_images["KEYCLOAK"] == "myregistry.myapp:15.3" assert "NODE" in source_images + + +def test_values_overrides_helm_native_structure(tmp_path): + out_path = tmp_path / 'test_values_overrides_helm_native_structure' + values = create_helm_chart([CLOUDHARNESS_ROOT, RESOURCES], output_path=out_path, include=['chartimgapp'], + domain="my.local", namespace='test', env='dev', local=False, tag=1, registry='reg') + + helm_path = out_path / HELM_CHART_PATH + with open(helm_path / VALUES_OVERRIDES_PATH) as f: + overrides = yaml.safe_load(f) + + # Build-argument base images are listed too, as aggregated at the root + assert overrides["source_images"] == values["source_images"] + + # The vendored sub-chart's images are keyed by its Chart.yaml name (how Helm passes parent + # values to a sub-chart), not by the app/dir name; the empty tag resolves to the appVersion + assert overrides["vendored-thing"]["image"] == {"registry": "docker.io", "repository": "someorg/somerepo", "tag": "2.0.0"} + assert "chartimgapp" not in overrides + + # Inline images declared in the app's own values keep their path under apps. + assert overrides[KEY_APPS]["chartimgapp"]["worker"]["image"] == "docker.io/baz/qux:9" + assert overrides[KEY_APPS]["chartimgapp"]["sidecar"]["image"] == {"name": "quay.io/foo/bar", "tag": "1.2.3"} + # The CloudHarness-built image (root image / harness.deployment.image) is not an image source + assert "image" not in overrides[KEY_APPS]["chartimgapp"] + assert "image" not in overrides[KEY_APPS]["chartimgapp"].get(KEY_HARNESS, {}).get(KEY_DEPLOYMENT, {}) + # The gatekeeper image is configured once at source_images.GATEKEEPER, so it is not repeated + # under every application + assert overrides["source_images"]["GATEKEEPER"].startswith("quay.io/gogatekeeper") + assert "proxy" not in overrides[KEY_APPS]["chartimgapp"].get(KEY_HARNESS, {}) + + # No inert aggregated listing in values.yaml + assert "chart_images" not in values + + # Rendering the chart with the overrides file selects exactly the listed image + manifests = render_helm_chart(helm_path, values_files=[helm_path / VALUES_OVERRIDES_PATH]) + assert find_manifest(manifests, "ConfigMap", "vendored-thing-image")["data"]["image"] == "docker.io/someorg/somerepo:2.0.0" + + +def test_values_overrides_edit_overrides_subchart_image(tmp_path): + out_path = tmp_path / 'test_values_overrides_edit' + create_helm_chart([CLOUDHARNESS_ROOT, RESOURCES], output_path=out_path, include=['chartimgapp'], + domain="my.local", namespace='test', env='dev', local=False, tag=1, registry='reg') + helm_path = out_path / HELM_CHART_PATH + overrides_path = helm_path / VALUES_OVERRIDES_PATH + + # Editing the generated file is enough to change the deployed image: Helm applies it after values.yaml + with open(overrides_path) as f: + overrides = yaml.safe_load(f) + overrides["vendored-thing"]["image"] = {"registry": "myregistry.io", "repository": "other/repo", "tag": "7"} + with open(overrides_path, "w") as f: + yaml.safe_dump(overrides, f) + + manifests = render_helm_chart(helm_path, values_files=[overrides_path]) + assert find_manifest(manifests, "ConfigMap", "vendored-thing-image")["data"]["image"] == "myregistry.io/other/repo:7" + + +def test_values_overrides_from_values_template(tmp_path): + out_path = tmp_path / 'test_values_overrides_from_values_template' + values = create_helm_chart([CLOUDHARNESS_ROOT, RESOURCES], output_path=out_path, include=['chartimgapp'], + domain="my.local", namespace='test', env='chartimages', local=False, tag=1, registry='reg') + helm_path = out_path / HELM_CHART_PATH + + # values-template-chartimages.yaml overrides only the tag: the overrides file shows the merged result + with open(helm_path / VALUES_OVERRIDES_PATH) as f: + overrides = yaml.safe_load(f) + assert overrides["vendored-thing"]["image"] == {"registry": "docker.io", "repository": "someorg/somerepo", "tag": "9.9.9"} + + # ...and, being Helm-native, the override in values.yaml alone already reaches the sub-chart + assert values["vendored-thing"]["image"]["tag"] == "9.9.9" + manifests = render_helm_chart(helm_path) + assert find_manifest(manifests, "ConfigMap", "vendored-thing-image")["data"]["image"] == "docker.io/someorg/somerepo:9.9.9" + + +def test_values_overrides_no_include_lists_vendored_charts(tmp_path): + out_path = tmp_path / 'test_values_overrides_no_include' + values = create_helm_chart([CLOUDHARNESS_ROOT, RESOURCES], output_path=out_path, domain="my.local", + namespace='test', env='dev', local=False, tag=1) + with open(out_path / HELM_CHART_PATH / VALUES_OVERRIDES_PATH) as f: + overrides = yaml.safe_load(f) + + assert overrides["vendored-thing"]["image"]["repository"] == "someorg/somerepo" + # Real vendored sub-charts, keyed by their Chart.yaml names + assert overrides["argo-workflows"]["controller"]["image"]["repository"] == "argoproj/workflow-controller" + assert overrides["argo-workflows"]["controller"]["image"]["tag"] # resolved from the sub-chart appVersion + assert overrides["kafka-ui"]["image"]["repository"] == "provectuslabs/kafka-ui" + # Inline images of real applications + assert overrides[KEY_APPS]["events"]["kafka"]["image"].startswith("docker.io/apache/kafka") + assert overrides[KEY_APPS]["jupyterhub"]["hub"]["image"]["name"] == "quay.io/jupyterhub/k8s-hub" + + # Database images, which live under the application's harness configuration + assert overrides[KEY_APPS]["accounts"][KEY_HARNESS]["database"]["postgres"]["image"] == "postgres:17" + assert overrides[KEY_APPS]["neo4j"][KEY_HARNESS]["database"]["neo4j"]["image"] == "neo4j:5" + # The gatekeeper image is configured once at source_images.GATEKEEPER, so it is listed there + # and not repeated on the applications that rely on it, nor at the root proxy configuration + assert overrides["source_images"]["GATEKEEPER"].startswith("quay.io/gogatekeeper") + assert "gatekeeper" not in overrides.get("proxy", {}) + # An application only appears here when it explicitly overrides its own gatekeeper, in which + # case that value is the effective one; the rest carry nothing + repeated = {app_name for app_name, app in overrides[KEY_APPS].items() + if "gatekeeper" in app.get(KEY_HARNESS, {}).get("proxy", {})} + explicit = {app_name for app_name, app in values[KEY_APPS].items() + if (((app.get(KEY_HARNESS) or {}).get("proxy") or {}).get("gatekeeper") or {}).get("image")} + assert repeated == explicit, f"gatekeeper image reported for {repeated - explicit}" + assert len(repeated) < len(overrides[KEY_APPS]), "gatekeeper image repeated on every application" + # Images the generated resources themselves run + assert overrides["backup"]["image"] == "prodrigestivill/postgres-backup-local" + assert overrides["volumeMigration"]["image"].startswith("alpine/k8s") + assert overrides["volumeMigration"]["wait"]["image"].startswith("busybox") + + # The images CloudHarness builds are not image sources and must not be listed + for app_name, app in overrides[KEY_APPS].items(): + assert "image" not in app, f"built image of {app_name} leaked into the overrides" + assert "image" not in app.get(KEY_HARNESS, {}).get(KEY_DEPLOYMENT, {}), app_name + + +def test_gatekeeper_image_comes_from_source_images(tmp_path): + out_path = tmp_path / 'test_gatekeeper_source_images' + create_helm_chart([CLOUDHARNESS_ROOT, RESOURCES], output_path=out_path, include=['samples', 'argo'], + domain="my.local", namespace='test', env='dev', local=False, tag=1, secured=True) + helm_path = out_path / HELM_CHART_PATH + + def gatekeeper_images(*set_values): + found = {} + for manifest in render_helm_chart(helm_path, set_values=set_values): + name = manifest.get("metadata", {}).get("name", "") + if manifest.get("kind") != "Deployment" or not name.endswith("-gk"): + continue + for container in (manifest["spec"]["template"]["spec"].get("containers") or []): + found[name] = container.get("image") + return found + + # A gatekeeper without an application-specific override takes the single source_images entry + default_image = "quay.io/gogatekeeper/gatekeeper:4.6.0" + baseline = gatekeeper_images() + assert baseline, "no gatekeeper deployment was rendered" + shared = {name for name, image in baseline.items() if image == default_image} + assert shared, f"no gatekeeper used source_images.GATEKEEPER, got {baseline}" + + # Overriding that one entry moves every gatekeeper that relies on it, and only those + overridden = gatekeeper_images(f"source_images.GATEKEEPER=myreg.io/gk:9.9.9") + assert {name for name, image in overridden.items() if image == "myreg.io/gk:9.9.9"} == shared + assert {name: image for name, image in overridden.items() if name not in shared} == \ + {name: image for name, image in baseline.items() if name not in shared} + + # A single application can still opt out, without affecting the others + opted_out = next(iter(shared)) + per_app = gatekeeper_images("source_images.GATEKEEPER=myreg.io/gk:9.9.9", + f"apps.argo.harness.proxy.gatekeeper.image=argoonly/gk:1") + assert per_app["argo-gk"] == "argoonly/gk:1" + assert all(per_app[name] == "myreg.io/gk:9.9.9" for name in shared if name != "argo-gk") + assert opted_out + + +def test_values_overrides_omits_shadowed_gatekeeper_path(tmp_path): + """proxy.gatekeeper.image no longer overrides anything, so it must not be advertised.""" + solution = tmp_path / 'solution' + (solution / 'deployment-configuration').mkdir(parents=True) + (solution / 'deployment-configuration' / 'values-template.yaml').write_text( + 'proxy:\n gatekeeper:\n image: "legacyroot/gk:1"\n') + + out_path = tmp_path / 'out' + create_helm_chart([CLOUDHARNESS_ROOT, RESOURCES, str(solution)], output_path=out_path, + include=['samples'], domain="my.local", namespace='test', local=False, + tag=1, secured=True) + helm_path = out_path / HELM_CHART_PATH + overrides = yaml.safe_load(open(helm_path / VALUES_OVERRIDES_PATH)) + + # The inert path is not reported, the one that works is + assert "gatekeeper" not in overrides.get("proxy", {}) + assert overrides["source_images"]["GATEKEEPER"].startswith("quay.io/gogatekeeper") + + # ...and no gatekeeper container actually uses the inert value + images = {manifest["metadata"]["name"]: container.get("image") + for manifest in render_helm_chart(helm_path) + if manifest.get("kind") == "Deployment" and manifest["metadata"]["name"].endswith("-gk") + for container in (manifest["spec"]["template"]["spec"].get("containers") or [])} + assert images, "no gatekeeper deployment was rendered" + assert "legacyroot/gk:1" not in images.values() + + +def test_values_overrides_omits_database_image_shadowed_by_image_ref(tmp_path): + """A database built from an image_ref ignores the type's image, so it is not an override.""" + def overrides_for(app_values, name): + solution = tmp_path / name + (solution / 'applications' / 'myapp' / 'deploy').mkdir(parents=True) + (solution / 'applications' / 'myapp' / 'deploy' / 'values.yaml').write_text(app_values) + out_path = tmp_path / f'out-{name}' + create_helm_chart([CLOUDHARNESS_ROOT, RESOURCES, str(solution)], output_path=out_path, + include=['myapp'], exclude=['events'], domain="my.local", + namespace='test', local=False, tag=1) + loaded = yaml.safe_load(open(out_path / HELM_CHART_PATH / VALUES_OVERRIDES_PATH)) + return (loaded.get(KEY_APPS, {}).get('myapp', {}).get(KEY_HARNESS) or {}) + + database = 'harness:\n database:\n auto: true\n type: postgres\n postgres:\n image: "postgres:17"\n' + + # Without image_ref the database image is the effective one, so it is reported + assert overrides_for(database, 'plain')['database']['postgres']['image'] == 'postgres:17' + + # With image_ref the built task image wins, so the shadowed value is not reported + with_ref = database.replace(' type: postgres\n', ' type: postgres\n image_ref: myapp-mytask\n') + assert 'database' not in overrides_for(with_ref, 'withref') + + +def test_values_overrides_includes_prebuilt_deployment_image(tmp_path): + out_path = tmp_path / 'test_values_overrides_prebuilt' + values = create_helm_chart([RESOURCES], output_path=out_path, include=['myapp'], exclude=['events'], + domain="my.local", namespace='test', env='nobuild', local=False, tag=1) + + # An application declaring a prebuilt image is not built, so that image IS an overridable + # source, unlike the image CloudHarness would have built for it + assert values[KEY_APPS]['myapp']['build'] is False + with open(out_path / HELM_CHART_PATH / VALUES_OVERRIDES_PATH) as f: + overrides = yaml.safe_load(f) + assert overrides[KEY_APPS]['myapp'][KEY_HARNESS][KEY_DEPLOYMENT]['image'] == 'custom-image' + + +def test_find_chart_images_shapes(): + refs = {ref.path: ref.value for ref in find_chart_images({ + 'flat': {'image': 'docker.io/x/y:1'}, + 'flat_with_tag': {'image': 'docker.elastic.co/es/es', 'imageTag': '8.17.0', 'imagePullPolicy': 'IfNotPresent'}, + 'registry_shape': {'image': {'registry': 'quay.io', 'repository': 'a/b', 'tag': '2', 'pullPolicy': 'Always'}}, + 'name_shape': {'image': {'name': 'quay.io/a/b', 'tag': '3', 'pullPolicy': 'Always'}}, + })} + assert refs == { + ('flat', 'image'): 'docker.io/x/y:1', + ('flat_with_tag', 'image'): 'docker.elastic.co/es/es', + ('flat_with_tag', 'imageTag'): '8.17.0', + ('registry_shape', 'image'): {'registry': 'quay.io', 'repository': 'a/b', 'tag': '2'}, + ('name_shape', 'image'): {'name': 'quay.io/a/b', 'tag': '3'}, + } + + +def test_find_chart_images_excludes_pull_policy_traps(): + # Plural "images" key (argo's shape) must never match + assert find_chart_images({'images': {'pullPolicy': 'Always', 'pullSecrets': []}}) == [] + # Singular "image" with neither "repository" nor "name" must never match + assert find_chart_images({'component': {'image': {'pullPolicy': 'Always', 'pullSecrets': []}}}) == [] + assert find_chart_images({'imagePullSecrets': [{'name': 'x'}], 'imagePullSecret': {'registry': 'r'}}) == [] + + +def test_find_chart_images_prunes_skip_paths(): + node = { + 'harness': { + 'deployment': {'image': 'the-built-image:1'}, + 'database': {'postgres': {'image': 'postgres:17'}}, + 'proxy': {'gatekeeper': {'image': 'quay.io/gogatekeeper/gatekeeper:4.6.0'}}, + }, + 'image': 'the-built-image:1', + 'worker': {'image': 'a-real-runtime-image:1'}, + } + # A built application: only its own image is not an overridable source, while the images it + # merely pulls under harness (database, gatekeeper) are + skip = frozenset({('image',), ('harness', 'deployment', 'image')}) + assert sorted(find_chart_images(node, skip_paths=skip), key=lambda r: r.path) == [ + ChartImageRef(path=('harness', 'database', 'postgres', 'image'), value='postgres:17'), + ChartImageRef(path=('harness', 'proxy', 'gatekeeper', 'image'), value='quay.io/gogatekeeper/gatekeeper:4.6.0'), + ChartImageRef(path=('worker', 'image'), value='a-real-runtime-image:1'), + ] + # Pruning nothing: a prebuilt application (build: false), or a sub-chart's own values.yaml + assert ChartImageRef(path=('image',), value='the-built-image:1') in find_chart_images(node) + + +def test_find_chart_images_resolves_empty_tag_from_app_version(): + assert find_chart_images({'image': {'repository': 'a/b', 'tag': ''}}, app_version='9.9.9') == [ + ChartImageRef(path=('image',), value={'repository': 'a/b', 'tag': '9.9.9'}), + ] + # Without an appVersion the tag is simply left out rather than emitted empty + assert find_chart_images({'image': {'repository': 'a/b', 'tag': ''}}) == [ + ChartImageRef(path=('image',), value={'repository': 'a/b'}), + ] diff --git a/tools/deployment-cli-tools/tests/test_skaffold.py b/tools/deployment-cli-tools/tests/test_skaffold.py index c3006bf0b..cc15fcf15 100644 --- a/tools/deployment-cli-tools/tests/test_skaffold.py +++ b/tools/deployment-cli-tools/tests/test_skaffold.py @@ -39,6 +39,8 @@ def test_create_skaffold_configuration(tmp_path): output_path=tmp_path ) assert os.path.exists(os.path.join(tmp_path, 'skaffold.yaml')) + # values-overrides.yaml is a reference for discovering image paths, not applied automatically + assert 'valuesFiles' not in sk['deploy']['helm']['releases'][0] exp_apps = ('accounts', 'samples', 'workflows', 'myapp', 'common') assert len(sk['build']['artifacts']) == len( exp_apps) + len(values[KEY_TASK_IMAGES]) @@ -384,12 +386,15 @@ def test_skaffold_imgarg_retrieval(tmp_path): # Ensure in the test that the Helm is well formed source_images = values.get("source_images") - assert len(source_images) == 2 + assert len(source_images) == 3 assert source_images["KEYCLOAK"] == "myregistry.myapp:15.3" assert source_images["NODE"] == "node:22-alpine" assert get_source_images(values) == { "KEYCLOAK": "myregistry.myapp:15.3", "NODE": "node:22-alpine", + # Not a build argument of any Dockerfile: declared by CloudHarness so that the + # gatekeeper image is configured in a single place + "GATEKEEPER": "quay.io/gogatekeeper/gatekeeper:4.6.0", }