Add zodb-pgjsonb based storage support

Dockerfile.builder

@@ -4,7 +4,7 @@ FROM python:${PYTHON_VERSION}-slim-bookworm
 
 ARG PLONE_VERSION
 
-ENV EXTRA_PACKAGES="relstorage==4.1.1 psycopg2==2.9.10 python-ldap==3.4.4 ZEO"
+ENV EXTRA_PACKAGES="relstorage==4.1.1 psycopg2==2.9.10 python-ldap==3.4.4 zodb-pgjsonb[s3]==1.12.0 zodb-json-codec==1.6.1 ZEO"
 
 
 LABEL maintainer="Plone Community <dev@plone.org>" \

skeleton/docker-entrypoint.sh

@@ -64,6 +64,39 @@ elif  [[ -v ZEO_ADDRESS ]]; then
   [ -z ${ZEO_STORAGE+x} ] && export ZEO_STORAGE=1
   [ -z ${ZEO_CLIENT_CACHE_SIZE+x} ] && export ZEO_CLIENT_CACHE_SIZE=128MB
   [ -z ${ZEO_DROP_CACHE_RATHER_VERIFY+x} ] && export ZEO_DROP_CACHE_RATHER_VERIFY=false
+elif [[ -v ZODB_PGJSONB_DSN ]]; then
+  MSG="Using zodb-pgjsonb configuration"
+  CONF=zodb-pgjsonb.conf
+  # Check zodb-pgjsonb variables
+  [ -z ${ZODB_PGJSONB_HISTORY_PRESERVING+x} ] && export ZODB_PGJSONB_HISTORY_PRESERVING=false
+
+  if [[ -v ZODB_PGJSONB_Z3BLOBS_ENABLED && $ZODB_PGJSONB_Z3BLOBS_ENABLED == "true" ]]; then
+
+    if [[ -n "$ZODB_PGJSONB_S3BLOBS_ENDPOINT_URL" && \
+          -n "$ZODB_PGJSONB_S3BLOBS_BUCKET_NAME" && \
+          -n "$ZODB_PGJSONB_S3BLOBS_ACCESS_KEY" && \
+          -n "$ZODB_PGJSONB_S3BLOBS_SECRET_KEY" ]]; then
+
+      MSG="Using zodb-pgjsonb S3 blobs configuration"
+      CONF=zodb-pgjsonb-s3blobs.conf
+
+      [ -z ${ZODB_PGJSONB_S3BLOBS_USE_SSL+x} ] && export ZODB_PGJSONB_S3BLOBS_USE_SSL=false
+      [ -z ${ZODB_PGJSONB_S3BLOBS_REGION+x} ] && export ZODB_PGJSONB_S3BLOBS_REGION=none
+      [ -z ${ZODB_PGJSONB_S3BLOBS_PREFIX+x} ] && export ZODB_PGJSONB_S3BLOBS_PREFIX=
+      [ -z ${ZODB_PGJSONB_S3BLOBS_THRESHOLD+x} ] && export ZODB_PGJSONB_S3BLOBS_THRESHOLD=100KB
+      [ -z ${ZODB_PGJSONB_S3BLOBS_CACHE_DIR+x} ] && export ZODB_PGJSONB_S3BLOBS_CACHE_DIR=auto
+      [ -z ${ZODB_PGJSONB_S3BLOBS_CACHE_SIZE+x} ] && export ZODB_PGJSONB_S3BLOBS_CACHE_SIZE=1GB
+
+    else
+      # If the required environment vars are missing, print an error message and exit
+      echo "ERROR: You have enabled the usage of S3 blobs, but some required data is missing:"
+      echo "- Endpoint URL: ${ZODB_PGJSONB_S3BLOBS_ENDPOINT_URL:-MISSING}"
+      echo "- Bucket: ${ZODB_PGJSONB_S3BLOBS_BUCKET_NAME:-MISSING}"
+      echo "- Access Key: ${ZODB_PGJSONB_S3BLOBS_ACCESS_KEY:-MISSING}"
+      echo "- Secret Key: ${ZODB_PGJSONB_S3BLOBS_SECRET_KEY:-MISSING}"
+      exit 1
+    fi
+  fi
 else
   MSG="Using default configuration"
   CONF=zope.conf

skeleton/etc/zodb-pgjsonb-s3blobs.conf

@@ -0,0 +1,45 @@
+%define INSTANCE /app
+instancehome $INSTANCE
+
+%define CLIENTHOME $(CLIENT_HOME)
+clienthome $CLIENTHOME
+
+debug-mode $(DEBUG_MODE)
+security-policy-implementation $(SECURITY_POLICY_IMPLEMENTATION)
+verbose-security $(VERBOSE_SECURITY)
+default-zpublisher-encoding $(DEFAULT_ZPUBLISHER_ENCODING)
+
+<environment>
+    CHAMELEON_CACHE $INSTANCE/var/cache
+</environment>
+
+<dos_protection>
+  form-memory-limit $(ZOPE_FORM_MEMORY_LIMIT)
+  form-disk-limit $(ZOPE_FORM_DISK_LIMIT)
+  form-memfile-limit $(ZOPE_FORM_MEMFILE_LIMIT)
+</dos_protection>
+
+<zodb_db main>
+    # Blob-enabled FileStorage database
+    %import zodb_pgjsonb
+    <pgjsonb>
+        dsn $(ZODB_PGJSONB_DSN)
+        history-preserving $(ZODB_PGJSONB_HISTORY_PRESERVING)
+
+        s3-endpoint-url $(ZODB_PGJSONB_S3BLOBS_ENDPOINT_URL)
+        s3-use-ssl $(ZODB_PGJSONB_S3BLOBS_USE_SSL)
+        s3-bucket-name $(ZODB_PGJSONB_S3BLOBS_BUCKET_NAME)
+        s3-access-key $(ZODB_PGJSONB_S3BLOBS_ACCESS_KEY)
+        s3-secret-key $(ZODB_PGJSONB_S3BLOBS_SECRET_KEY)
+        s3-region $(ZODB_PGJSONB_S3BLOBS_REGION)
+        s3-prefix $(ZODB_PGJSONB_S3BLOBS_PREFIX)
+
+        blob-threshold $(ZODB_PGJSONB_S3BLOBS_THRESHOLD)
+        blob-cache-dir $(ZODB_PGJSONB_S3BLOBS_CACHE_DIR)
+        blob-cache-size $(ZODB_PGJSONB_S3BLOBS_CACHE_SIZE)
+
+    </pgjsonb>
+
+    mount-point /
+    cache-size $(ZODB_CACHE_SIZE)
+</zodb_db>

skeleton/etc/zodb-pgjsonb.conf

@@ -0,0 +1,32 @@
+%define INSTANCE /app
+instancehome $INSTANCE
+
+%define CLIENTHOME $(CLIENT_HOME)
+clienthome $CLIENTHOME
+
+debug-mode $(DEBUG_MODE)
+security-policy-implementation $(SECURITY_POLICY_IMPLEMENTATION)
+verbose-security $(VERBOSE_SECURITY)
+default-zpublisher-encoding $(DEFAULT_ZPUBLISHER_ENCODING)
+
+<environment>
+    CHAMELEON_CACHE $INSTANCE/var/cache
+</environment>
+
+<dos_protection>
+  form-memory-limit $(ZOPE_FORM_MEMORY_LIMIT)
+  form-disk-limit $(ZOPE_FORM_DISK_LIMIT)
+  form-memfile-limit $(ZOPE_FORM_MEMFILE_LIMIT)
+</dos_protection>
+
+<zodb_db main>
+    # Main database
+    %import zodb_pgjsonb
+    <pgjsonb>
+        dsn $(ZODB_PGJSONB_DSN)
+        history-preserving $(ZODB_PGJSONB_HISTORY_PRESERVING)
+    </pgjsonb>
+
+    mount-point /
+    cache-size $(ZODB_CACHE_SIZE)
+</zodb_db>

test/config.sh

@@ -16,6 +16,8 @@ plone-listenport
 plone-zeoclient
 plone-relstorage
 plone-shared-blob-dir
+plone-pgjsonb
+plone-pgjsonb-s3blobs
 '
 
 imageTests+=(

test/tests/plone-pgjsonb-s3blobs/run.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -eo pipefail
+
+dir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+
+image="$1"
+
+PLONE_TEST_SLEEP=10
+PLONE_TEST_TRIES=10
+
+# Start Postgres
+zname="pgjsonb-container-$RANDOM-$RANDOM"
+zpull="$(docker pull postgres:17)"
+zid="$(docker run -d --name "$zname" -e POSTGRES_USER=plone -e POSTGRES_PASSWORD=plone -e POSTGRES_DB=plone postgres:17)"
+
+# Start Plone as RelStorage Client
+pname="plone-container-$RANDOM-$RANDOM"
+pid="$(docker run -d --name "$pname" --link=$zname:db -e ZODB_PGJSONB_DSN="dbname='plone' user='plone' host='db' password='plone'" "$image")"
+
+# Tear down
+trap "docker rm -vf $pid $zid > /dev/null" EXIT
+
+get() {
+	docker run --rm -i \
+		--link "$pname":plone \
+		--entrypoint /app/bin/python \
+		"$image" \
+		-c "from urllib.request import urlopen; con = urlopen('$1'); print(con.read())"
+}
+
+. "$dir/../../retry.sh" --tries "$PLONE_TEST_TRIES" --sleep "$PLONE_TEST_SLEEP" get "http://plone:8080"
+
+# Plone is up and running
+[[ "$(get 'http://plone:8080')" == *"Welcome to Plone!"* ]]

test/tests/plone-pgjsonb/run.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -eo pipefail
+
+dir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
+
+image="$1"
+
+PLONE_TEST_SLEEP=10
+PLONE_TEST_TRIES=10
+
+# Start Postgres
+zname="pgjsonb-container-$RANDOM-$RANDOM"
+zpull="$(docker pull postgres:17)"
+zid="$(docker run -d --name "$zname" -e POSTGRES_USER=plone -e POSTGRES_PASSWORD=plone -e POSTGRES_DB=plone postgres:17)"
+
+# Start Plone as RelStorage Client
+pname="plone-container-$RANDOM-$RANDOM"
+pid="$(docker run -d --name "$pname" --link=$zname:db -e ZODB_PGJSONB_DSN="dbname='plone' user='plone' host='db' password='plone'" "$image")"
+
+# Tear down
+trap "docker rm -vf $pid $zid > /dev/null" EXIT
+
+get() {
+	docker run --rm -i \
+		--link "$pname":plone \
+		--entrypoint /app/bin/python \
+		"$image" \
+		-c "from urllib.request import urlopen; con = urlopen('$1'); print(con.read())"
+}
+
+. "$dir/../../retry.sh" --tries "$PLONE_TEST_TRIES" --sleep "$PLONE_TEST_SLEEP" get "http://plone:8080"
+
+# Plone is up and running
+[[ "$(get 'http://plone:8080')" == *"Welcome to Plone!"* ]]