feat(connectors): ship connector plugins in the iggy-connect docker image

.dockerignore

@@ -21,6 +21,7 @@
 /scripts/ci/*
 !/scripts/ci/third-party-licenses.sh
 !/scripts/ci/render-node-licenses.mjs
+!/scripts/ci/connector-plugins.sh
 **/target
 /target
 !/target/debug/iggy

.github/actions/utils/docker-buildx/action.yml

@@ -46,6 +46,10 @@ inputs:
     description: "Single platform to build (e.g., linux/amd64). If set, builds only this platform without QEMU. Leave empty for multi-arch build."
     required: false
     default: ""
+  target:
+    description: "Dockerfile stage to build. Empty builds the default (last) stage. Used to select an image flavor from a multi-flavor Dockerfile."
+    required: false
+    default: ""
   gha-cache:
     description: "Whether to use GitHub Actions cache for Docker layers (disable to save cache space)"
     required: false
@@ -342,6 +346,7 @@ runs:
       with:
         context: ${{ steps.ctx.outputs.context }}
         file: ${{ steps.config.outputs.dockerfile }}
+        target: ${{ inputs.target }}
         platforms: ${{ steps.platforms.outputs.platforms }}
         labels: ${{ steps.meta.outputs.labels }}
         cache-from: ${{ env.CACHE_FROM }}
@@ -357,6 +362,7 @@ runs:
       with:
         context: ${{ steps.ctx.outputs.context }}
         file: ${{ steps.config.outputs.dockerfile }}
+        target: ${{ inputs.target }}
         platforms: ${{ steps.platforms.outputs.platforms }}
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}

.github/actions/utils/validate-third-party-licenses/action.yml

@@ -66,9 +66,17 @@ runs:
       shell: bash
       run: ./scripts/ci/third-party-licenses.sh --validate --manifest core/bench/dashboard/server/Cargo.toml
 
-    - name: Validate iggy-connect bundle
+    - name: Validate iggy-connect bundle (runtime + connector plugins)
       shell: bash
-      run: ./scripts/ci/third-party-licenses.sh --validate --manifest core/connectors/runtime/Cargo.toml
+      run: |
+        # The fat image bundles the runtime plus every connector plugin, so
+        # validate the plugins' dependency closures too. The manifest list is
+        # derived from cargo metadata so a newly added connector is license
+        # validated on the PR that adds it, with no change here.
+        MANIFEST_FLAGS="$(scripts/ci/connector-plugins.sh --manifest-flags)"
+        ./scripts/ci/third-party-licenses.sh --validate \
+          --manifest core/connectors/runtime/Cargo.toml \
+          $MANIFEST_FLAGS
 
     - name: Validate Python wheel bundle
       shell: bash

.github/config/publish.yml

@@ -96,9 +96,24 @@ components:
     platforms: ["linux/amd64", "linux/arm64"]
     version_file: "core/connectors/runtime/Cargo.toml"
     version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"'
-    # Image ships only the runtime binary; plugin .so files are not bundled.
+    # Two published flavors from one Dockerfile. The fat flavor (empty suffix,
+    # runtime-fat stage) bundles every connector plugin and is the default;
+    # -slim (runtime stage) ships the runtime binary only. Each flavor keeps
+    # its own per-arch digest set and manifest tags. Components without a
+    # flavors list build their single default stage.
+    flavors:
+      - suffix: ""
+        target: runtime-fat
+      - suffix: "-slim"
+        target: runtime
+    # The fat image bundles the runtime plus every connector plugin, so any
+    # change under core/connectors/ must refresh :edge. crates: keeps
+    # sensitivity to shared workspace deps in the runtime's closure (e.g.
+    # iggy_common); paths: covers all plugin crates without a hardcoded
+    # per-plugin list, so a newly added connector is gated automatically.
     gate:
       crates: [iggy-connectors]
+      paths: [core/connectors]
 
   web-ui:
     tag_pattern: "^web-ui-([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$"

.github/workflows/_build_rust_artifacts.yml

@@ -46,8 +46,8 @@ on:
       connector_plugins:
         type: string
         required: false
-        default: "iggy_connector_elasticsearch_sink,iggy_connector_elasticsearch_source,iggy_connector_iceberg_sink,iggy_connector_postgres_sink,iggy_connector_postgres_source,iggy_connector_quickwit_sink,iggy_connector_random_source,iggy_connector_s3_sink,iggy_connector_stdout_sink,iggy_connector_surrealdb_sink"
-        description: "Comma-separated list of connector plugin crates to build as shared libraries"
+        default: ""
+        description: "Comma-separated list of connector plugin crates to build as shared libraries. Empty (default) derives the full cdylib set from scripts/ci/connector-plugins.sh so new connectors are picked up automatically."
     outputs:
       artifact_name:
         description: "Name of the uploaded artifact containing all artifacts"
@@ -216,12 +216,25 @@ jobs:
       - name: Add Rust target
         run: rustup target add ${{ matrix.target }}
 
+      - name: Resolve connector plugin list
+        env:
+          OVERRIDE: ${{ inputs.connector_plugins }}
+        run: |
+          if [[ -n "$OVERRIDE" ]]; then
+            plugins="$OVERRIDE"
+            echo "Using connector plugin list from workflow input (override)"
+          else
+            plugins="$(scripts/ci/connector-plugins.sh --comma-names)"
+            echo "Derived connector plugin list from cargo metadata"
+          fi
+          echo "Plugins: $plugins"
+          echo "CONNECTOR_PLUGINS=$plugins" >> "$GITHUB_ENV"
+
       - name: Build connector plugins
         run: |
-          plugins="${{ inputs.connector_plugins }}"
           pkg_flags=()
 
-          IFS=',' read -ra pkgs <<< "$plugins"
+          IFS=',' read -ra pkgs <<< "$CONNECTOR_PLUGINS"
           for pkg in "${pkgs[@]}"; do
             name="$(echo "$pkg" | xargs)"
             [[ -z "$name" ]] && continue
@@ -238,13 +251,16 @@ jobs:
           outdir="dist/${target}"
           mkdir -p "${outdir}"
 
-          plugins="${{ inputs.connector_plugins }}"
-          IFS=',' read -ra pkgs <<< "$plugins"
+          IFS=',' read -ra pkgs <<< "$CONNECTOR_PLUGINS"
           for pkg in "${pkgs[@]}"; do
             lib_name="$(echo "$pkg" | xargs)"
             [[ -z "$lib_name" ]] && continue
             so_file="target/${target}/release/lib${lib_name}.so"
-            [[ -f "$so_file" ]] && cp "$so_file" "${outdir}/"
+            if [[ ! -f "$so_file" ]]; then
+              echo "::error::expected connector plugin artifact missing: ${so_file}. The plugin crate built no cdylib, or its name does not match lib<crate>.so." >&2
+              exit 1
+            fi
+            cp "$so_file" "${outdir}/"
           done
 
           tarball="iggy-connectors-${target}-${version}.tar.gz"

.github/workflows/edge-release.yml

@@ -59,6 +59,17 @@ jobs:
           server_version=$(scripts/extract-version.sh rust-server)
           echo "server_version=${server_version}" >> "$GITHUB_OUTPUT"
 
+      - name: Render bundled connector plugin list
+        id: plugins
+        run: |
+          # Derive the plugin list from cargo metadata so the release notes
+          # stay in sync with what is actually built (see connector-plugins.sh).
+          {
+            echo "list<<PLUGINS_EOF"
+            scripts/ci/connector-plugins.sh --names | sed 's/^/- `/; s/$/`/'
+            echo "PLUGINS_EOF"
+          } >> "$GITHUB_OUTPUT"
+
       - name: Download all artifacts
         uses: actions/download-artifact@v8
         with:
@@ -101,16 +112,7 @@ jobs:
             - `iggy-connectors` - The connectors runtime
 
             ## Connector plugins included (.so)
-            - `iggy_connector_elasticsearch_sink`
-            - `iggy_connector_elasticsearch_source`
-            - `iggy_connector_iceberg_sink`
-            - `iggy_connector_postgres_sink`
-            - `iggy_connector_postgres_source`
-            - `iggy_connector_quickwit_sink`
-            - `iggy_connector_random_source`
-            - `iggy_connector_s3_sink`
-            - `iggy_connector_stdout_sink`
-            - `iggy_connector_surrealdb_sink`
+            ${{ steps.plugins.outputs.list }}
 
             ## Downloads
 

.github/workflows/publish.yml

@@ -399,21 +399,38 @@ jobs:
             // Output non-Docker, non-Rust targets (SDKs only)
             core.setOutput('non_docker_targets', JSON.stringify(nonDockerTargets.length ? { include: nonDockerTargets } : { include: [{ key: 'noop', type: 'noop' }] }));
 
-            // Build Docker matrix: components × platforms for native runner builds
+            // Build Docker matrix: components × flavors × platforms for native
+            // runner builds. A component may declare multiple image flavors
+            // (e.g. fat + slim) built from different Dockerfile stages; each
+            // flavor gets its own suffix and target stage. Components without a
+            // flavors list build a single default flavor (empty suffix, empty
+            // target = default stage). flavor_suffix/flavor_target ride on every
+            // matrix entry so the build and manifest jobs keep flavors' digest
+            // sets and tags separate.
             const platforms = [
               { platform: 'linux/amd64', arch: 'amd64', runner: 'ubuntu-latest' },
               { platform: 'linux/arm64', arch: 'arm64', runner: 'ubuntu-24.04-arm' }
             ];
 
+            const flavorsOf = (key) => {
+              const fl = cfg[key] && cfg[key].flavors;
+              return Array.isArray(fl) && fl.length ? fl : [{ suffix: '', target: '' }];
+            };
+
             const dockerMatrix = [];
+            const dockerComponents = [];
             for (const t of dockerTargets) {
-              for (const p of platforms) {
-                dockerMatrix.push({ ...t, ...p });
+              for (const f of flavorsOf(t.key)) {
+                const flavor = { flavor_suffix: f.suffix || '', flavor_target: f.target || '' };
+                dockerComponents.push({ ...t, ...flavor });
+                for (const p of platforms) {
+                  dockerMatrix.push({ ...t, ...p, ...flavor });
+                }
               }
             }
 
             core.setOutput('docker_matrix', JSON.stringify(dockerMatrix.length ? { include: dockerMatrix } : { include: [{ key: 'noop', type: 'noop' }] }));
-            core.setOutput('docker_components', JSON.stringify(dockerTargets.length ? { include: dockerTargets } : { include: [{ key: 'noop', type: 'noop' }] }));
+            core.setOutput('docker_components', JSON.stringify(dockerComponents.length ? { include: dockerComponents } : { include: [{ key: 'noop', type: 'noop' }] }));
             core.setOutput('has_docker', String(dockerTargets.length > 0));
 
             core.setOutput('count', String(targets.length));
@@ -804,6 +821,7 @@ jobs:
           task: publish
           libc: ${{ steps.libc.outputs.libc }}
           component: ${{ matrix.key }}
+          target: ${{ matrix.flavor_target }}
           version: ${{ steps.ver.outputs.version }}
           platform: ${{ matrix.platform }}
           dry_run: ${{ inputs.dry_run }}
@@ -827,7 +845,9 @@ jobs:
         if: ${{ !inputs.dry_run }}
         uses: actions/upload-artifact@v7
         with:
-          name: docker-digest-${{ matrix.key }}-${{ matrix.arch }}
+          # flavor_suffix keeps each flavor's per-arch digests in a distinct
+          # artifact so the manifest job never merges flavors into one image.
+          name: docker-digest-${{ matrix.key }}${{ matrix.flavor_suffix }}-${{ matrix.arch }}
           path: ${{ runner.temp }}/digests/*
           if-no-files-found: error
           retention-days: 1
@@ -931,13 +951,13 @@ jobs:
       - name: Download amd64 digest
         uses: actions/download-artifact@v8
         with:
-          name: docker-digest-${{ matrix.key }}-amd64
+          name: docker-digest-${{ matrix.key }}${{ matrix.flavor_suffix }}-amd64
           path: ${{ runner.temp }}/digests
 
       - name: Download arm64 digest
         uses: actions/download-artifact@v8
         with:
-          name: docker-digest-${{ matrix.key }}-arm64
+          name: docker-digest-${{ matrix.key }}${{ matrix.flavor_suffix }}-arm64
           path: ${{ runner.temp }}/digests
 
       - name: Set up Docker Buildx
@@ -956,16 +976,22 @@ jobs:
         run: |
           IMAGE="${{ steps.config.outputs.image }}"
           VERSION="${{ steps.ver.outputs.version }}"
-
-          echo "Creating manifests for $IMAGE from digests:"
+          # Per-flavor tag suffix (e.g. -slim). Empty for the default flavor,
+          # so single-flavor components keep their existing tags unchanged.
+          SUFFIX="${{ matrix.flavor_suffix }}"
+
+          # This job's digest dir holds only this flavor's per-arch digests
+          # (the upload/download names carry the same suffix), so globbing every
+          # digest here composes one manifest per flavor with no cross-merge.
+          echo "Creating manifests for ${IMAGE} (flavor '${SUFFIX:-default}') from digests:"
           ls -la
 
           if [ "${{ inputs.create_edge_docker_tag }}" = "true" ]; then
             # Auto-publish: :edge is the rolling tag and is ALWAYS refreshed.
             docker buildx imagetools create \
-              -t "${IMAGE}:edge" \
+              -t "${IMAGE}:edge${SUFFIX}" \
               $(printf "${IMAGE}@sha256:%s " *)
-            echo "✅ Pushed manifest: ${IMAGE}:edge"
+            echo "✅ Pushed manifest: ${IMAGE}:edge${SUFFIX}"
 
             # The versioned :version manifest is part of the immutable release
             # and ships only alongside a new git tag. should_tag is false when
@@ -974,26 +1000,26 @@ jobs:
             # refreshes :edge and nothing else.
             if [ "$SHOULD_TAG" = "true" ]; then
               docker buildx imagetools create \
-                -t "${IMAGE}:${VERSION}" \
+                -t "${IMAGE}:${VERSION}${SUFFIX}" \
                 $(printf "${IMAGE}@sha256:%s " *)
-              echo "✅ Pushed manifest: ${IMAGE}:${VERSION}"
+              echo "✅ Pushed manifest: ${IMAGE}:${VERSION}${SUFFIX}"
             else
-              echo "ℹ️ should_tag=false - :edge refreshed, skipping versioned :${VERSION} manifest"
+              echo "ℹ️ should_tag=false - :edge${SUFFIX} refreshed, skipping versioned :${VERSION}${SUFFIX} manifest"
             fi
           else
             # Manual publish: always push the versioned manifest, plus :latest
             # for stable (non edge/rc) releases.
             docker buildx imagetools create \
-              -t "${IMAGE}:${VERSION}" \
+              -t "${IMAGE}:${VERSION}${SUFFIX}" \
               $(printf "${IMAGE}@sha256:%s " *)
-            echo "✅ Pushed manifest: ${IMAGE}:${VERSION}"
+            echo "✅ Pushed manifest: ${IMAGE}:${VERSION}${SUFFIX}"
 
             if [[ ! "$VERSION" =~ -(edge|rc) ]]; then
-              echo "Creating 'latest' manifest"
+              echo "Creating 'latest${SUFFIX}' manifest"
               docker buildx imagetools create \
-                -t "${IMAGE}:latest" \
+                -t "${IMAGE}:latest${SUFFIX}" \
                 $(printf "${IMAGE}@sha256:%s " *)
-              echo "✅ Pushed manifest: ${IMAGE}:latest"
+              echo "✅ Pushed manifest: ${IMAGE}:latest${SUFFIX}"
             fi
           fi
 
@@ -1003,24 +1029,29 @@ jobs:
         run: |
           IMAGE="${{ steps.config.outputs.image }}"
           VERSION="${{ steps.ver.outputs.version }}"
+          SUFFIX="${{ matrix.flavor_suffix }}"
 
           # Auto-publish pushed :version only when should_tag was true; in
           # every other auto-publish case (:edge-only) inspect :edge instead.
           if [ "${{ inputs.create_edge_docker_tag }}" = "true" ] && [ "$SHOULD_TAG" != "true" ]; then
-            echo "Inspecting :edge manifest (versioned manifest was skipped: should_tag=false)"
-            docker buildx imagetools inspect "${IMAGE}:edge"
+            echo "Inspecting :edge${SUFFIX} manifest (versioned manifest was skipped: should_tag=false)"
+            docker buildx imagetools inspect "${IMAGE}:edge${SUFFIX}"
           else
-            docker buildx imagetools inspect "${IMAGE}:${VERSION}"
+            docker buildx imagetools inspect "${IMAGE}:${VERSION}${SUFFIX}"
           fi
 
       # Inline per-component tagging: tightly couple the git tag to the
       # multi-arch manifest that just shipped. should_tag was computed in the
       # version step above and already encodes the SNAPSHOT and auto-publish
       # stable-Docker skip rules. dry_run is gated at the job level.
+      # The git release tag is flavor-independent (one tag per component
+      # version), so only the default flavor creates it; other flavors ship
+      # their manifests but must not race to create the same tag.
       - name: Tag Docker release (${{ matrix.key }})
         if: |
           success() &&
           inputs.skip_tag_creation == false &&
+          matrix.flavor_suffix == '' &&
           steps.ver.outputs.should_tag == 'true'
         uses: ./.github/actions/utils/create-git-tag
         with:

core/connectors/README.md

@@ -2,7 +2,7 @@
 
 The highly performant and modular runtime for statically typed, yet dynamically loaded connectors. Ingest the data from the external sources and push it further to the Iggy streams, or fetch the data from the Iggy streams and push it further to the external sources. Create your own Rust plugins by simply implementing either the `Source` or `Sink` trait and build custom pipelines for the data processing.
 
-The [docker image](https://hub.docker.com/r/apache/iggy-connect) is available, and can be fetched via `docker pull apache/iggy-connect`.
+The [docker image](https://hub.docker.com/r/apache/iggy-connect) is available via `docker pull apache/iggy-connect`. It ships in two flavors: a default image with every connector plugin bundled, and a `-slim` runtime-only image (see [Docker image](#docker-image)).
 
 ## Features
 
@@ -18,6 +18,38 @@ The [docker image](https://hub.docker.com/r/apache/iggy-connect) is available, a
 - **Observability**: Prometheus metrics with per-stage latency histograms, plus an opt-in per-batch tracing benchmark target.
 - **Structured logging**: Selectable text or JSON log format via `[logging]` configuration.
 
+## Docker image
+
+Published to Docker Hub as [`apache/iggy-connect`](https://hub.docker.com/r/apache/iggy-connect) in two flavors:
+
+| Tag | Contents |
+| --- | --- |
+| `edge`, `x.y.z`, `latest` | Runtime plus every bundled connector plugin (default) |
+| `edge-slim`, `x.y.z-slim`, `latest-slim` | Runtime binary only |
+
+The default (fat) image bakes every connector plugin into `/usr/local/lib`, which is on the runtime's [plugin search path](#plugin-path-resolution). A connector config can therefore load a bundled plugin by library file name alone, with no directory and no extension:
+
+```toml
+# a connector definition in your config_dir
+type = "sink"
+key = "stdout"
+name = "Stdout sink"
+path = "libiggy_connector_stdout_sink"
+```
+
+Run the runtime with a config mounted in, pointing `IGGY_CONNECTORS_CONFIG_PATH` at it:
+
+```bash
+docker run --rm \
+  -v "$PWD/my_config:/config" \
+  -e IGGY_CONNECTORS_CONFIG_PATH=/config/config.toml \
+  apache/iggy-connect
+```
+
+The mounted `config.toml` must set `[connectors].config_dir` to the connector definitions inside the container, and each definition's `path` must resolve there; bundled plugins resolve by file name as shown above.
+
+Choose a `-slim` tag when you supply your own plugins and want a smaller image with a narrower dependency surface, and mount the plugin `.so` files into one of the search-path directories (for example `/usr/local/lib`).
+
 ## Quick Start
 
 1. Build the project in release mode (or debug, and update the connectors paths in the config accordingly), and make sure that the plugins specified in `core/connectors/runtime/example_config/connectors/` directory under `path` are available. The configuration must be provided in `toml` format, with files following the `{connector_name}_{type}[_v{N}].toml` naming convention.