Complete source images

deployment-configuration/helm/templates/auto-database-postgres-backup.yaml

@@ -27,7 +27,7 @@ spec:
           containers:
             - name: "{{ .app.harness.database.name }}-backup"
               imagePullPolicy: IfNotPresent
-              image: prodrigestivill/postgres-backup-local
+              image: {{ .root.Values.backup.image | default "prodrigestivill/postgres-backup-local" }}
               command: ["/bin/bash"]
               args: ["-c" , "./backup.sh"]
               env:

deployment-configuration/helm/templates/auto-database.yaml

@@ -105,7 +105,7 @@ spec:
       # Holds the pod until the volume-migration job has copied the legacy volume data into
       # the statefulset volume. Delete the legacy PVC after verifying the migration.
       - name: volume-migration
-        image: busybox:1.36
+        image: {{ .root.Values.volumeMigration.wait.image | default "busybox:1.36" }}
         command: ['sh', '-c', 'until [ -f /migration/target/.cloudharness-volume-migrated ]; do echo "waiting for the volume migration job"; sleep 5; done']
         volumeMounts:
           - name: {{ .app.harness.database.name | quote }}

deployment-configuration/helm/templates/auto-deployments.yaml

@@ -108,7 +108,7 @@ spec:
       # Holds the pod until the volume-migration job has copied the legacy volume data into
       # this pod's volume. Delete the legacy PVC after verifying the migration to drop this gate.
       - name: volume-migration
-        image: busybox:1.36
+        image: {{ .root.Values.volumeMigration.wait.image | default "busybox:1.36" }}
         command: ['sh', '-c', 'until [ -f /migration/target/.cloudharness-volume-migrated ]; do echo "waiting for the volume migration job"; sleep 5; done']
         volumeMounts:
           - name: {{ $volume.name }}

deployment-configuration/helm/templates/auto-gatekeepers.yaml

@@ -181,7 +181,7 @@ spec:
 {{ include "deploy_utils.etcHosts" .root | indent 6 }}
       containers:
       - name: {{ .app.harness.service.name | quote }}
-        image: {{ .app.harness.proxy.gatekeeper.image | default .root.Values.proxy.gatekeeper.image | default "quay.io/gogatekeeper/gatekeeper:4.6.0" }}
+        image: {{ .app.harness.proxy.gatekeeper.image | default .root.Values.source_images.GATEKEEPER | default "quay.io/gogatekeeper/gatekeeper:4.6.0" }}
         imagePullPolicy: IfNotPresent
         {{ if .root.Values.local }}
         securityContext:

deployment-configuration/helm/templates/auto-volume-migration.yaml

@@ -66,7 +66,7 @@ spec:
         - name: volume-migration
           # Fully open source kubectl image (alpine-based: provides sh and tar), see
           # https://github.com/alpine-docker/k8s
-          image: alpine/k8s:1.34.9
+          image: {{ .root.Values.volumeMigration.image | default "alpine/k8s:1.34.9" }}
           command:
             - sh
             - -c

deployment-configuration/helm/values.yaml

@@ -110,6 +110,8 @@ secretmanagers: {}
   #   # -- How often the value is refreshed from AWS.
   #   refreshInterval: 1h
 backup:
+  # -- Image running the database backup cronjob. Not built by CloudHarness.
+  image: prodrigestivill/postgres-backup-local
   # -- Flag to enable/disable backups.
   active: false
   # -- Number of days to keep backups.
@@ -137,6 +139,14 @@ backup:
       memory: "64Mi"
       # -- K8s cpu resource definition.
       cpu: "50m"
+# -- Images used by the generated volume migration resources. Not built by CloudHarness.
+volumeMigration:
+  # -- Image of the job copying a legacy volume into its replacement.
+  image: alpine/k8s:1.34.9
+  wait:
+    # -- Image of the init container holding pods until the migration job has completed.
+    image: busybox:1.36
+
 proxy:
   # -- Set to false to hide remote client headers. Will hide the client IPs in all logs
   forwardedHeaders: true
@@ -150,9 +160,8 @@ proxy:
     # -- Maximum size of payload in MB
     max: 250
   gatekeeper:
-    # -- Default gatekeeper image
-    image: "quay.io/gogatekeeper/gatekeeper:4.6.0"
-    # -- Default number of gatekeeper replicas
+    # -- Default number of gatekeeper replicas. The gatekeeper image is not set here: it is
+    # configured once, at source_images.GATEKEEPER.
     replicas: 1
     secret: ""
     resources:

deployment-configuration/value-template.yaml

@@ -202,8 +202,9 @@ harness:
       # -- Maximum size of payload in MB
       max:
     gatekeeper:
-      # -- Default gatekeeper image
-      image: "quay.io/gogatekeeper/gatekeeper:4.6.0"
+      # -- Overrides source_images.GATEKEEPER for this application only. Left unset so that the
+      # gatekeeper image is configured once, at source_images.GATEKEEPER.
+      image:
       # -- Default number of gatekeeper replicas
       replicas: 1
       # -- Native Gatekeeper proxy.yml settings. Kebab-case keys follow the

deployment-configuration/values-template.yaml

@@ -1 +1,6 @@
 debug: false
+
+source_images:
+  # -- Gatekeeper image, pulled by the gatekeeper deployment generated for every secured
+  # application. Set here rather than per application so that it is configured in one place.
+  GATEKEEPER: quay.io/gogatekeeper/gatekeeper:4.6.0

docs/README.md

@@ -7,6 +7,8 @@
   - [Work with local deployments](./build-deploy/local-deploy/README.md)
     - [Debug your applications](./build-deploy/local-deploy/debug.md)
     - [Setting up Minikube](./build-deploy/local-deploy/minikube.md)
+- [Define image sources, e.g. a private registry](./image-sources.md)
+- [Define and use base and common images](./base-common-images.md)
 - [Create and configure applications](./applications)
   - [Generate a new application](./applications/harness-application.md)
   - [Override applications values and files](./applications/overridings.md)

docs/applications/development/backend-development.md

@@ -55,7 +55,7 @@ harness:
 Every image defined as a base image or a common image can be used as a
 build dependency.
 
-For more details about how to define your custom image and the available images, see [here](../../base-common-images.md).
+For more details about how to define your custom image and the available images, see [base and common images](../../base-common-images.md).
 
 For more info about dependencies, see [here](../dependencies.md)
 

docs/base-common-images.md

@@ -1,5 +1,13 @@
 # Define and use base and common images
 
+Base and common images are the images CloudHarness builds for your applications to inherit from as build dependencies.
+
+Some default base and common images are provided by Cloud Harness and can be customized in your project; following the same pattern, can also define other images that can be reused across your project applications and tasks.
+Define them when you want your applications to share libraries, tooling or a common stack.
+
+> Note: If you are interested in customizing where external base images are pulled from,
+> see [image sources](image-sources.md).
+
 ## Relevant files and directory structure
 
 - `base-images`: base Docker images. Those images can used as base images in CloudHarness apps and tasks.
@@ -62,73 +70,8 @@ To override cloudharness-base, create a directory `MY_SOLUTION/infrastructure/ba
 then run `harness-deployment cloudharness MY_SOLUTION`
 
 
-## Change the base image of any application (`FROM [xxx]`)
-
-It's possible to change the image each dockerfile inherits by using the `FROM` directive.
-Depending on the kind of application, changing the base image is done in two different ways.
-There is basically two main situations:
-
-. your application has a `Dockerfile` (e.g: Keycloak, the app you're building);
-. your application doesn't have a `Dockerfile` (e.g: gogatekeeper, Kafka, etc), and the image is directly injected inside the helm chart when the helm chart is generated.
-
-In both situations, the modification is done in the `value-template.yaml` of your CloudHarness projects, but the YAML path to modify depends on the app your targetting then.
-
-### Change the base image for applications with a `Dockerfile`
-
-Changing the base image for your application or applications which have a `Dockerfile` is done through the `source_images` entry of your `value-template.yaml` file. 
-This entry defines a mapping between the `ARG` of your `Dockerfile` and the value you want to inject. 
-Here is an example of a declared mapping to change the base image for Python based apps and Node base apps:
-
-```yaml
-# value-template.yaml
-source_images:
-  NODE: "mybaseimg:14.5"
-  PYTHON: "myotherimage:15.1"
-```
-
-The list of the base images variables and the value they resolve to is automatically generated in the `helm/values.yaml` file which is generated by the `harness-deployment` command under the `source_images` key. You can then copy/paste the entries you need in your `value-template.yaml` file and tweak the values there.
-
-
-### Change the base image for applications which inject their image in the helm chart
-
-Those applications are not providing a Dockerfile, but directly an image which is injcted in the helm chart from the helm template.
-Each of those applications considers a specific path from the helm configuration (i.e: `values.yaml` or `value-template.yaml`).
-To change the base image for those, you need to know this path and change it in the `value-template.yaml` by redefining it.
-The redefinition will override the base value from the sub-`values.yaml` files on configuration merging.
-Here is an example for the gatekeeper and the Argo controller:
-
-```yaml
-# value-template.yaml
-harness:
-  proxy:
-    gatekeeper:
-      image: YOUR_NEW_IMG
-
-# or, depending on your application
-argo:
-  controller:
-    image:
-      registry: YOUR_REG
-      tag: YOUR TAG
-```
+## Change the image a Dockerfile inherits from
 
-Here is a table of the identified applications inside of CloudHarness which do not own a `Dockerfile`.
-
-| Application | Image path |
-| --- | --- |
-| gatekeeper | harness.proxy.gatekeeper.image |
-| Argo controller | argo.controller.image.{repository, tag} |
-| Argo executor | argo.executor.image.{repository, tag} |
-| Argo server | argo.server.image.{repository, tag} |
-| Elasticsearch | elasticsearch.image |
-| Events | events.image.{repository, tag, nullPolicy} |
-| Events kafka | events.kafka.image |
-| JupyterHub | jupyterhub.singleuser.image.{pullSecrets, name, tag} |
-| JupyterHub proxy | jupyterhub.proxy.chp.image.{name, tag, pullPolicy} |
-| JupyterHub scheduling | jupyterhub.scheduling.userPlaceHolder.image.{name, pullPolicy, tag} |
-| NFS server | nfsserver.harness.deployment.image |
-| Sentry redis | sentry.redis.image |
-| MongoDB | harness.database.mongo.image |
-| Neo4J | harness.database.neo4j.image |
-| Neo4J reverseProxy | neo4j.reverseProxy.image |
-| Postgres | harness.database.postgres.image |
+Defining a base image is not the only way to change what an application's `Dockerfile` inherits
+from. The image each `FROM` resolves to is itself configurable, per deployment, and is described in
+[image sources](image-sources.md).

docs/build-deploy/helm-configuration.md

@@ -1,6 +1,6 @@
 
 ## Override Helm values
-Edit `./deployment-configuration/values-template.yaml` to override any helm chart values file. Refer to the default [Helm values file](tools/cloudharness_utilities/deployment-configuration/helm/values.yaml) for documentation about each value. 
+Edit `./deployment-configuration/values-template.yaml` to override any helm chart values file. Refer to the default [Helm values file](../../deployment-configuration/helm/values.yaml) for documentation about each value. 
 
 For environment specific configurations edit `deployment-configuration/values-template-[ENV].yaml`.
 
@@ -9,7 +9,19 @@ For environment specific configurations edit `deployment-configuration/values-te
 ## Override base/common images
 
 Base images are built from the infrastructure/base-images sub folders.
-See the dedicated [base images](./docs/applications.md) document for more details and examples.
+See the dedicated [base and common images](../base-common-images.md) document for more details and examples.
+
+## Override the images the chart pulls
+
+Every run of `harness-deployment` writes `deployment/helm/values-overrides.yaml` alongside the
+generated `values.yaml`, listing every image the chart pulls at the exact path Helm reads it from:
+the `source_images` build arguments, the vendored sub-chart images keyed by sub-chart name, and the
+images each application declares inline or pulls through its harness configuration.
+
+The file is a reference and is not applied by any deployment flow. Use it to find the path of an
+image, try a replacement out by passing it explicitly (`helm ... -f
+./deployment/helm/values-overrides.yaml`), then make the change permanent as described in
+[image sources](../image-sources.md#make-an-override-permanent).
 
 ## Single application deployment configuration