CH-288 add support for RWX volumes

deployment-configuration/helm/templates/_helpers.tpl

@@ -94,6 +94,56 @@ heritage: {{ $.Release.Service | quote }}
 {{- end }}
 
 
+{{/*
+Tells whether a harness.deployment.volume is ReadWriteMany: nfs volumes (legacy `usenfs` flag)
+always are, otherwise `writeMany` decides. Renders "true" or nothing, so the result can be used
+directly in a condition. Accepts a nil volume.
+Usage: {{ if include "deploy_utils.volumeWriteMany" $volume }}
+*/}}
+{{- define "deploy_utils.volumeWriteMany" -}}
+{{- if . }}{{ if .usenfs }}true{{ else if .writeMany }}true{{ end }}{{ end }}
+{{- end -}}
+
+{{/*
+Storage class of a harness.deployment.volume claim: nfs volumes always use the class created by
+the nfsserver application, otherwise the volume `storageClass` wins on the deployment default
+(harness.deployment.storageClass). A null default renders nothing, leaving the claim to the
+cluster default storage class; `standard` is used when the deployment does not declare the key
+at all (values generated before the setting existed).
+Usage: {{ include "deploy_utils.volumeStorageClass" (dict "root" .root "deployment" $deployment) }}
+*/}}
+{{- define "deploy_utils.volumeStorageClass" -}}
+{{- $volume := .deployment.volume -}}
+{{- if $volume.usenfs }}{{ printf "%s-%s" .root.Values.namespace .root.Values.apps.nfsserver.storageClass.name }}{{ else if $volume.storageClass }}{{ $volume.storageClass }}{{ else if .deployment.storageClass }}{{ .deployment.storageClass }}{{ else if not (hasKey .deployment "storageClass") }}standard{{ end }}
+{{- end -}}
+
+{{/*
+Storage class of a database volume claim: harness.database.storageClass. A null value renders
+nothing, leaving the claim to the cluster default storage class; `standard` is used when the
+database does not declare the key at all (values generated before the setting existed).
+Usage: {{ include "deploy_utils.databaseStorageClass" .app.harness.database }}
+*/}}
+{{- define "deploy_utils.databaseStorageClass" -}}
+{{- if .storageClass }}{{ .storageClass }}{{ else if not (hasKey . "storageClass") }}standard{{ end }}
+{{- end -}}
+
+{{/*
+Render the spec of a claim (PersistentVolumeClaim or statefulset volumeClaimTemplate) for a
+harness.deployment.volume.
+Usage: {{ include "deploy_utils.volumeClaimSpec" (dict "root" .root "deployment" $deployment) | nindent 2 }}
+*/}}
+{{- define "deploy_utils.volumeClaimSpec" -}}
+{{- $storageClass := include "deploy_utils.volumeStorageClass" (dict "root" .root "deployment" .deployment) -}}
+accessModes:
+  - {{ if include "deploy_utils.volumeWriteMany" .deployment.volume }}ReadWriteMany{{ else }}ReadWriteOnce{{ end }}
+{{- if $storageClass }}
+storageClassName: {{ $storageClass }}
+{{- end }}
+resources:
+  requests:
+    storage: {{ .deployment.volume.size }}
+{{- end -}}
+
 {{/*
 Render volumeMounts block for a container.
 Usage: {{ include "deploy_utils.volumeMounts" (dict "app" .app "root" .root) }}

deployment-configuration/helm/templates/auto-database-postgres-operator.yaml

@@ -46,6 +46,10 @@ spec:
 
   storage:
     size: {{ .app.harness.database.size }}
+    {{- $storageClass := include "deploy_utils.databaseStorageClass" .app.harness.database }}
+    {{- if $storageClass }}
+    storageClass: {{ $storageClass }}
+    {{- end }}
 
   {{- with .app.harness.database.resources }}
   resources:

deployment-configuration/helm/templates/auto-database.yaml

@@ -42,6 +42,10 @@ metadata:
 spec:
   accessModes:
     - ReadWriteOnce
+  {{- $storageClass := include "deploy_utils.databaseStorageClass" .app.harness.database }}
+  {{- if $storageClass }}
+  storageClassName: {{ $storageClass }}
+  {{- end }}
   resources:
     requests:
       storage: {{ .app.harness.database.size }}
@@ -138,6 +142,10 @@ spec:
       spec:
         accessModes:
           - ReadWriteOnce
+        {{- $storageClass := include "deploy_utils.databaseStorageClass" .app.harness.database }}
+        {{- if $storageClass }}
+        storageClassName: {{ $storageClass }}
+        {{- end }}
         resources:
           requests:
             storage: {{ .app.harness.database.size }}

deployment-configuration/helm/templates/auto-deployments.yaml

@@ -1,15 +1,17 @@
 {{- define "deploy_utils.deployment" }}
 {{- $isStatefulSet := .app.harness.deployment.statefulset | default false }}
 {{- $volume := .app.harness.deployment.volume }}
-{{- /* $rwoVolume: a ReadWriteOnce (non-nfs) volume pins the pod to a single node — it drives both
-      the Recreate strategy and the podAffinity. $ownVolume additionally requires the PVC to be
-      managed here (auto), which a statefulset turns into a volumeClaimTemplate. The checks are
-      nested so hasKey/index never runs on a nil volume: helm < 3.10 (go < 1.18) does not
-      short-circuit 'and'/'or', so a flat expression would fail for volume-less apps. */}}
+{{- /* $rwoVolume: a ReadWriteOnce volume pins the pod to a single node — it drives both the
+      Recreate strategy and the podAffinity. ReadWriteMany volumes (`writeMany`, or the legacy
+      `usenfs` flag) attach to several nodes at once, hence need neither. $ownVolume additionally
+      requires the PVC to be managed here (auto), which a statefulset turns into a
+      volumeClaimTemplate. The checks are nested so hasKey/index never runs on a nil volume:
+      helm < 3.10 (go < 1.18) does not short-circuit 'and'/'or', so a flat expression would fail
+      for volume-less apps. */}}
 {{- $rwoVolume := false }}
 {{- $ownVolume := false }}
 {{- if $volume }}
-{{- if or (not (hasKey $volume "usenfs")) (not $volume.usenfs) }}
+{{- if not (include "deploy_utils.volumeWriteMany" $volume) }}
 {{- $rwoVolume = true }}
 {{- if or (not (hasKey $volume "auto")) $volume.auto }}
 {{- $ownVolume = true }}
@@ -47,7 +49,7 @@ spec:
   # A ReadWriteOnce volume attaches to a single node and the pod is pinned to the
   # volume's node via podAffinity (see below). Recreate terminates the old pod
   # before starting the new one, avoiding an unschedulable pod when the node can't
-  # fit both. NFS (ReadWriteMany) volumes have no pinning and can roll normally.
+  # fit both. ReadWriteMany volumes have no pinning and can roll normally.
   strategy:
     type: Recreate
   {{- end }}
@@ -210,12 +212,7 @@ spec:
     - metadata:
         name: {{ $volume.name }}
       spec:
-        accessModes:
-          - ReadWriteOnce
-        storageClassName: standard
-        resources:
-          requests:
-            storage: {{ $volume.size }}
+        {{- include "deploy_utils.volumeClaimSpec" (dict "root" .root "deployment" .app.harness.deployment) | nindent 8 }}
   {{- end }}
   {{- if $legacyClaim }}
   {{- include "deploy_utils.volumeMigration" (dict "root" .root "name" .app.harness.deployment.name "pvc" $volume.name) }}

deployment-configuration/helm/templates/auto-volumes.yaml

@@ -1,11 +1,16 @@
 {{- define "deploy_utils.pvolume" }}
 {{- $volume := .app.harness.deployment.volume }}
 {{- $isStatefulSet := .app.harness.deployment.statefulset | default false }}
+{{- /* ReadWriteMany volumes are shared by all the pods using them: they are never provisioned
+      per replica through volumeClaimTemplates, hence never owned by a statefulset. */}}
+{{- $writeMany := include "deploy_utils.volumeWriteMany" $volume }}
 {{- $ownVolume := false }}
-{{- if and (or (not (hasKey $volume "usenfs")) (not $volume.usenfs)) (or (not (hasKey $volume "auto")) $volume.auto) }}
+{{- if not $writeMany }}
+{{- if or (not (hasKey $volume "auto")) $volume.auto }}
 {{- $ownVolume = true }}
 {{- end }}
-{{- /* Statefulsets own their (non-nfs) volume through volumeClaimTemplates: the standalone PVC
+{{- end }}
+{{- /* Statefulsets own their (ReadWriteOnce) volume through volumeClaimTemplates: the standalone PVC
       is only kept while a legacy one exists, so that its data can be migrated. Delete the legacy
       PVC once migrated. */}}
 {{- $legacyClaim := false }}
@@ -27,18 +32,7 @@ metadata:
   labels:
     app: {{ .app.harness.deployment.name| quote }}
 spec:
-  resources:
-    requests:
-      storage: {{ .app.harness.deployment.volume.size }}
-
-  accessModes:
-{{- if or (not (hasKey .app.harness.deployment.volume "usenfs")) (not .app.harness.deployment.volume.usenfs) }}
-  - ReadWriteOnce
-  storageClassName: standard
-{{- else }}
-  - ReadWriteMany
-  storageClassName: {{ printf "%s-%s" .root.Values.namespace .root.Values.apps.nfsserver.storageClass.name }}
-{{- end }}
+  {{- include "deploy_utils.volumeClaimSpec" (dict "root" .root "deployment" .app.harness.deployment) | nindent 2 }}
 {{- end }}
 ---
 {{- end }}

deployment-configuration/value-template.yaml

@@ -36,8 +36,20 @@ harness:
     name:
     # -- Deployment port.
     port: 8080
-    # -- volume specification
+    # -- Default storage class of the deployment volume claim. Set to null to use the cluster default storage class.
+    storageClass: standard
+    # -- volume specification.
+    # `writeMany: true` creates and mounts the volume as ReadWriteMany: the volume attaches to several
+    # nodes at once, hence pods using it are not pinned to the volume's node. Requires a storage class
+    # supporting ReadWriteMany, set through the volume `storageClass` (which overrides the deployment default).
     volume:
+      # example:
+      #   name: my-volume
+      #   mountpath: /usr/src/app/myvolume
+      #   auto: true
+      #   size: 5Gi
+      #   writeMany: true
+      #   storageClass: efs-sc
     # -- When true, the deployment is rendered as a StatefulSet instead of a Deployment. Recommended for deployments with a (non-nfs) volume: updates terminate the old pod before creating the new one. The volume is provisioned per replica through volumeClaimTemplates; data of a pre-existing PVC named after the volume is copied into each statefulset volume by a migration job (delete the legacy PVC once migrated).
     statefulset: false
     # -- Deployment resources.
@@ -93,6 +105,8 @@ harness:
     # -- supported db types: mongo, postgres, neo4j
     type:
     size: 1Gi
+    # -- Storage class of the database volume claim. Set to null to use the cluster default storage class.
+    storageClass: standard
     # -- database username
     user: mnp
     # -- database password

docs/applications/databases.md

@@ -31,6 +31,13 @@ harness:
 
 `size`: Size of the persistent volume that the database container mounts, default is set to `1Gi`
 
+`storageClass`: Storage class of the database volume claim, default is set to `standard`. Set it to
+null to omit the storage class from the claim, so that the cluster default storage class is used.
+It applies to the plain and statefulset database volumes as well as to the storage of a
+`postgres.operator` cluster. Note that the storage class is immutable on an existing claim: on a
+cluster whose default class is not `standard`, set this value (or null) before upgrading a release
+that already has a database volume.
+
 `resources`: Set the database pod resources
 
 `image_ref`: Optional setting, used for referencing a base/static image from the build. The complete image name with tag will automagically being generated from the values.yaml file. This setting overrides the `image` setting specific for the database type (e.g. postgres/image). Note: the referenced image must be included as a build dependency in order to be built by the pipelines.

docs/applications/volumes.md

@@ -34,17 +34,79 @@ harness:
 A Volume can be mounted by one or more pods (shared Volume). Be careful: only one of the deployments
 should create the Volume, the other deployment should only mount it.
 
-Cloudharness uses the `standard` StorageClass for the volume and ReadWriteOnce
+By default Cloudharness uses the `standard` StorageClass for the volume and ReadWriteOnce
 mount strategy. 
 In order to support volume sharing affinity rules are added so that all pods using
 the same volume end up in the same node.
 This strategy works for basic use cases but can easily cause deadlocks if more than
 one node is available on the cluster and other affinity rules or taints are present.
 
-A better support for volume sharing is achieved by using a Network File System (NFS).
-In order to use the nfs, the NFS must be added to the deployment (e.g. as a dependency and `usenfs` must be set to true.
+### Storage class
 
+The storage class of the volume claim is `harness.deployment.storageClass`, which defaults to
+`standard`. Set it to null to omit the storage class from the claim, so that the cluster default
+storage class is used:
+
+```yaml
+harness:
+  ...
+  deployment:
+    # the cluster default storage class provisions the volume
+    storageClass: null
+    volume:
+      name: my-volume
+      mountpath: /usr/src/app/myvolume
+      auto: true
+      size: 5Gi
+```
+
+The volume's own `storageClass` overrides the deployment default, and takes precedence for both
+ReadWriteOnce and ReadWriteMany volumes. A null on the volume means "not set", hence inherits the
+deployment default: use the deployment `storageClass: null` to provision the volume on the cluster
+default class. The same setting is available for database volumes as
+`harness.database.storageClass` (see [databases](databases.md)).
+
+### ReadWriteMany volumes
+
+Setting `writeMany: true` creates and mounts the volume as ReadWriteMany. A ReadWriteMany volume
+attaches to several nodes at the same time, hence the pods using it are not pinned to the volume's
+node: no podAffinity is added, and deployments roll normally instead of being recreated.
+
+ReadWriteMany requires a storage class supporting it (e.g. AWS EFS, Azure Files, CephFS,
+the nfs provisioner). The class is resolved as above — `standard` is normally ReadWriteOnce only,
+so set the volume `storageClass`, or the deployment default, to a ReadWriteMany capable class
+(or to null when the cluster default one supports ReadWriteMany).
+
+```yaml
+harness:
+  ...
+  deployment:
+    ...
+    volume:
+      name: my-shared-volume
+      mountpath: /usr/src/app/myvolume
+      auto: true
+      size: 5Gi
+      writeMany: true
+      storageClass: efs-sc
 ```
+
+When a volume is shared by several deployments, declare the same `writeMany` on all of them: the
+claim is created once (by the deployment declaring `auto: true`), but each deployment decides on
+its own declaration whether its pods are pinned to the volume's node.
+
+Note that both the access mode and the storage class are immutable on an existing
+PersistentVolumeClaim: changing `writeMany` or the storage class on a live volume requires
+deleting and recreating the claim, and the data is not migrated.
+
+### Using the NFS server application
+
+Volume sharing can also be achieved by using the Network File System provided by the `nfsserver`
+application. In order to use the nfs, the nfs server must be added to the deployment (e.g. as a
+dependency) and `usenfs` must be set to true: the volume is created as ReadWriteMany on the storage
+class of the nfs provisioner.
+
+```yaml
 harness:
   ...
   dependencies:
@@ -62,10 +124,36 @@ harness:
       usenfs: true
 ```
 
+`usenfs` is equivalent to `writeMany: true` with the nfs provisioner storage class, and is kept
+for backwards compatibility: on a cluster providing a ReadWriteMany storage class, prefer
+`writeMany` with `storageClass`.
+
+The nfs server settings prevail on the volume ones: an `usenfs` volume is always created on the
+nfs provisioner storage class and mounted ReadWriteMany, whatever `storageClass` and `writeMany`
+say. `harness-deployment` logs a warning when they collide:
+
+```
+WARNING Volume my-shared-volume of application samples sets usenfs and storageClass efs-sc: the nfs server storage class prevails.
+```
+
+### Volumes mounted by Argo workflows
+
+Argo workflow pods mounting an application volume (`<volume name>:<mount path>`, see
+[Argo workflows](../argo-workflows.md)) are pinned to the volume's node in the same way
+deployments are. ReadWriteMany application volumes are recognized from the application
+configuration, so their workflows get no node pinning.
+
+Volumes that are not declared by an application can be marked as ReadWriteMany explicitly with
+the `rwx` mount mode, which also disables the pinning:
+
+```python
+operations.PipelineOperation('my-op-', tasks, shared_directory='my-claim:/mnt/shared:rwx')
+```
+
 ## Deploying as a StatefulSet
 
-By default, a deployment with a (non-nfs) volume is rendered as a Kubernetes `Deployment` with a
-`Recreate` update strategy and podAffinity pinning it to the node holding the volume, since a
+By default, a deployment with a ReadWriteOnce volume is rendered as a Kubernetes `Deployment` with
+a `Recreate` update strategy and podAffinity pinning it to the node holding the volume, since a
 ReadWriteOnce volume can only attach to one node at a time.
 
 Setting `harness.deployment.statefulset: true` renders it as a `StatefulSet` instead. StatefulSet
@@ -86,9 +174,9 @@ harness:
 ```
 
 The volume is provisioned per replica through `volumeClaimTemplates` (PVCs named
-`<volume>-<app>-<ordinal>`). Exceptions: nfs volumes (`usenfs: true`) and externally managed
-volumes (`auto: false`) keep mounting their common PVC by name — per-replica claims would
-un-share them.
+`<volume>-<app>-<ordinal>`). Exceptions: ReadWriteMany volumes (`writeMany: true` or
+`usenfs: true`) and externally managed volumes (`auto: false`) keep mounting their common PVC by
+name — per-replica claims would un-share them.
 
 **Migrating from an existing Deployment**: if a PVC named after the volume exists in the cluster
 at deploy time (left over from the pre-statefulset Deployment), it is treated as a legacy volume:

docs/model/DatabaseDeploymentConfig.md

@@ -10,6 +10,7 @@ Name | Type | Description | Notes
 **name** | **str** |  | [optional] 
 **type** | **str** | Define the database type.  One of (mongo, postgres, neo4j, sqlite3) | [optional] 
 **size** | **str** | Specify database disk size | [optional] 
+**storage_class** | **str** | Storage class of the database volume claim.  Set to null to omit the storage class from the claim, so that the cluster default storage class is used. | [optional] 
 **user** | **str** | database username | [optional] 
 **var_pass** | **str** | Database password | [optional] 
 **image_ref** | **str** | Used for referencing images from the build | [optional] 

docs/model/DeploymentAutoArtifactConfig.md

@@ -13,7 +13,8 @@ Name | Type | Description | Notes
 **image** | **str** | Image name to use in the deployment. Leave it blank to set from the application's Docker file | [optional] 
 **resources** | [**DeploymentResourcesConf**](DeploymentResourcesConf.md) |  | [optional] 
 **volume** | [**DeploymentVolumeSpec**](DeploymentVolumeSpec.md) |  | [optional] 
-**statefulset** | **bool** | When true, the workload is rendered as a Kubernetes StatefulSet instead of a Deployment. Recommended for deployments with a ReadWriteOnce volume: updates terminate the old pod before creating the new one, so no Recreate strategy or node pinning is needed. The volume, unless nfs-shared or externally managed (auto false), is provisioned per replica through volumeClaimTemplates. A pre-existing PVC named after the volume (left over from a previous Deployment) is migrated automatically: a migration job streams its data into each statefulset volume through the Kubernetes API, so the volumes are never mounted by the same pod (works on multi-zone clusters); delete the legacy PVC once migrated. | [optional] 
+**storage_class** | **str** | Default storage class of the deployment volume claim, used when the volume does not define its own `storageClass`.  Set to null to omit the storage class from the claim, so that the cluster default storage class is used. | [optional] 
+**statefulset** | **bool** | When true, the workload is rendered as a Kubernetes StatefulSet instead of a Deployment. Recommended for deployments with a ReadWriteOnce volume: updates terminate the old pod before creating the new one, so no Recreate strategy or node pinning is needed. The volume, unless ReadWriteMany or externally managed (auto false), is provisioned per replica through volumeClaimTemplates. A pre-existing PVC named after the volume (left over from a previous Deployment) is migrated automatically: a migration job streams its data into each statefulset volume through the Kubernetes API, so the volumes are never mounted by the same pod (works on multi-zone clusters); delete the legacy PVC once migrated. | [optional] 
 **network** | [**NetworkConfig**](NetworkConfig.md) |  | [optional] 
 **extra_containers** | [**Dict[str, ExtraContainerConfig]**](ExtraContainerConfig.md) | Extra containers (init containers and sidecars) for the deployment. Each key is a container name mapping to an ExtraContainerConfig. | [optional]