feat(helm)!: Add the Spider subchart and compression-coordinator wiring for Spider-orchestrated compression.

taskfiles/helm.yaml

@@ -16,4 +16,5 @@ tasks:
       - "mkdir -p '{{.OUTPUT_DIR}}'"
       - |-
         . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}"
+        helm dependency build "{{.ROOT_DIR}}/tools/deployment/package-helm"
         helm package "{{.ROOT_DIR}}/tools/deployment/package-helm" --destination "{{.OUTPUT_DIR}}"

taskfiles/lint.yaml

@@ -69,6 +69,7 @@ tasks:
         . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}"
         ct lint \
           --chart-dirs="tools/deployment" \
+          --chart-repos="spider=https://github.com/y-scope/spider/raw/gh-pages" \
           --chart-yaml-schema="{{.G_HELM_TOOLCHAIN_DIR}}/etc/chart_schema.yaml" \
           --lint-conf="tools/yscope-dev-utils/exports/lint-configs/.yamllint.yml"
 

tools/deployment/package-helm/.gitignore

@@ -0,0 +1,2 @@
+# Subcharts are downloaded by `helm dependency update`.
+charts/

tools/deployment/package-helm/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: spider
+  repository: https://github.com/y-scope/spider/raw/gh-pages
+  version: 0.1.4
+digest: sha256:ef717c497d7d93913e52811b5128da41be08bdfa9dc2e586f2b7e2528eea0791
+generated: "2026-07-27T13:33:18.893480848-04:00"

tools/deployment/package-helm/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: "v2"
 name: "clp"
-version: "0.4.1-dev.1"
+version: "0.4.1-dev.2"
 description: "A Helm chart for CLP's (Compressed Log Processor) package deployment"
 type: "application"
 appVersion: "0.13.1-dev"
@@ -15,3 +15,8 @@ keywords:
   - "log-parser"
   - "logging"
   - "search"
+dependencies:
+  - name: "spider"
+    version: "0.1.4"
+    repository: "https://github.com/y-scope/spider/raw/gh-pages"
+    condition: "spider.enabled"

tools/deployment/package-helm/set-up-multi-dedicated-test.sh

@@ -138,6 +138,7 @@ helm uninstall test --ignore-not-found
 sleep 2
 # Word splitting is intentional: helper functions return multiple --set flags.
 # shellcheck disable=SC2046
+helm dependency build "${script_dir}"
 helm install test "${script_dir}" \
     --set "distributedDeployment=true" \
     --set "scheduling.compressionWorker.replicas=${COMPRESSION_WORKER_REPLICAS}" \

tools/deployment/package-helm/set-up-multi-shared-test.sh

@@ -44,6 +44,7 @@ helm uninstall test --ignore-not-found
 sleep 2
 # Word splitting is intentional: helper functions return multiple --set flags.
 # shellcheck disable=SC2046
+helm dependency build "${script_dir}"
 helm install test "${script_dir}" \
     --set "distributedDeployment=true" \
     --set "scheduling.compressionWorker.replicas=${COMPRESSION_WORKER_REPLICAS}" \

tools/deployment/package-helm/set-up-test.sh

@@ -28,6 +28,7 @@ helm uninstall test --ignore-not-found
 sleep 2
 # Word splitting is intentional: helper functions return multiple --set flags.
 # shellcheck disable=SC2046
+helm dependency build "${script_dir}"
 helm install test "${script_dir}" \
     $(get_presto_helm_args) \
     $(get_image_helm_args "${CLUSTER_NAME}" "clpPackage" "${CLP_PACKAGE_IMAGE}")

tools/deployment/package-helm/templates/_helpers.tpl

@@ -442,6 +442,26 @@ Gets the port for the Presto service.
 {{- end -}}
 {{- end }}
 
+{{/*
+Gets the host for the Spider storage service.
+
+@param {object} . Root template context
+@return {string} The Spider storage host
+*/}}
+{{- define "clp.spiderStorageHost" -}}
+{{- printf "%s-storage" .Values.spider.fullnameOverride -}}
+{{- end }}
+
+{{/*
+Gets the port for the Spider storage service.
+
+@param {object} . Root template context
+@return {string} The Spider storage port
+*/}}
+{{- define "clp.spiderStoragePort" -}}
+50051
+{{- end }}
+
 {{/*
 Gets the BROKER_URL env var for Celery workers.
 

tools/deployment/package-helm/templates/compression-coordinator-deployment.yaml

@@ -0,0 +1,75 @@
+{{- if .Values.spider.enabled }}
+apiVersion: "apps/v1"
+kind: "Deployment"
+metadata:
+  name: {{ include "clp.fullname" . }}-compression-coordinator
+  labels:
+    {{- include "clp.labels" . | nindent 4 }}
+    app.kubernetes.io/component: "compression-coordinator"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "clp.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: "compression-coordinator"
+  template:
+    metadata:
+      labels:
+        {{- include "clp.labels" . | nindent 8 }}
+        app.kubernetes.io/component: "compression-coordinator"
+    spec:
+      {{- include "clp.createSchedulingConfigs" (dict
+          "root" .
+          "component" "compressionCoordinator"
+        ) | nindent 6 }}
+      serviceAccountName: {{ include "clp.fullname" . }}-service-account
+      terminationGracePeriodSeconds: 60
+      initContainers:
+        - {{- include "clp.waitFor" (dict
+            "root" .
+            "type" "job"
+            "name" "db-table-creator"
+          ) | nindent 10 }}
+        - {{- include "clp.waitFor" (dict
+            "root" .
+            "type" "service"
+            "name" "storage"
+          ) | nindent 10 }}
+      containers:
+        - name: "compression-coordinator"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }}
+          imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
+          env:
+            {{- include "clp.telemetryEnv" . | nindent 12 }}
+            - name: "CLP_DB_PASS"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "clp.fullname" . }}-database
+                  key: "password"
+            - name: "CLP_DB_USER"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "clp.fullname" . }}-database
+                  key: "username"
+            - name: "OTEL_SERVICE_NAME"
+              value: "compression-coordinator"
+            - name: "RUST_LOG"
+              value: {{ .Values.clpConfig.compression_coordinator.logging_level | quote }}
+          {{- include "clp.createResourceLimits" (dict
+              "root" .
+              "component" "compressionCoordinator"
+            ) | nindent 10 }}
+          volumeMounts:
+            - name: "config"
+              mountPath: "/etc/clp-config.yaml"
+              subPath: "clp-config.yaml"
+              readOnly: true
+          command: [
+            "/opt/clp/bin/compression-coordinator",
+            "--config", "/etc/clp-config.yaml"
+          ]
+      volumes:
+        - name: "config"
+          configMap:
+            name: {{ include "clp.fullname" . }}-config
+{{- end }}

tools/deployment/package-helm/templates/compression-scheduler-deployment.yaml

@@ -1,3 +1,4 @@
+{{- if not .Values.spider.enabled }}
 apiVersion: "apps/v1"
 kind: "Deployment"
 metadata:
@@ -112,3 +113,4 @@ spec:
         {{- if eq .Values.clpConfig.logs_input.type "fs" }}
         - {{- include "clp.logsInputVolume" . | nindent 10 }}
         {{- end }}
+{{- end }}

tools/deployment/package-helm/templates/compression-worker-deployment.yaml

@@ -1,3 +1,4 @@
+{{- if not .Values.spider.enabled }}
 apiVersion: "apps/v1"
 kind: "Deployment"
 metadata:
@@ -132,3 +133,4 @@ spec:
         {{- if eq .Values.clpConfig.logs_input.type "fs" }}
         - {{- include "clp.logsInputVolume" . | nindent 10 }}
         {{- end }}
+{{- end }}

tools/deployment/package-helm/templates/configmap.yaml

@@ -67,6 +67,33 @@ data:
         | int }}
       target_segment_size: {{ .Values.clpConfig.archive_output.target_segment_size
         | int }}
+    {{- if .Values.spider.enabled }}
+    compression_coordinator:
+      commit_task_hard_timeout_secs: {{
+        .Values.clpConfig.compression_coordinator.commit_task_hard_timeout_secs | int }}
+      commit_task_max_retry: {{
+        .Values.clpConfig.compression_coordinator.commit_task_max_retry | int }}
+      commit_task_soft_timeout_secs: {{
+        .Values.clpConfig.compression_coordinator.commit_task_soft_timeout_secs | int }}
+      compression_task_max_retry: {{
+        .Values.clpConfig.compression_coordinator.compression_task_max_retry | int }}
+      database_connection_pool_size: {{
+        .Values.clpConfig.compression_coordinator.database_connection_pool_size | int }}
+      job_polling_interval_millisecs: {{
+        .Values.clpConfig.compression_coordinator.job_polling_interval_millisecs | int }}
+      resource_group:
+        name: {{ .Values.clpConfig.compression_coordinator.resource_group.name | quote }}
+      result_polling:
+        init_backoff_millisecs: {{
+          .Values.clpConfig.compression_coordinator.result_polling.init_backoff_millisecs | int }}
+        max_backoff_millisecs: {{
+          .Values.clpConfig.compression_coordinator.result_polling.max_backoff_millisecs | int }}
+      termination_timeout_secs: {{
+        .Values.clpConfig.compression_coordinator.termination_timeout_secs | int }}
+    spider:
+      host: "{{ include "clp.spiderStorageHost" . }}"
+      port: {{ include "clp.spiderStoragePort" . | int }}
+    {{- end }}
     compression_scheduler:
       jobs_poll_delay: {{ .Values.clpConfig.compression_scheduler.jobs_poll_delay }}
       logging_level: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }}

tools/deployment/package-helm/values.yaml

@@ -100,6 +100,7 @@ scheduling:
   resultsCache: {}
 
   # Core
+  compressionCoordinator: {}
   compressionScheduler: {}
   logIngestor: {}
   queryScheduler: {}
@@ -130,6 +131,7 @@ resources:
   resultsCache: {}
 
   # Core
+  compressionCoordinator: {}
   compressionScheduler: {}
   logIngestor: {}
   queryScheduler: {}
@@ -189,6 +191,21 @@ clpConfig:
       clp: "clp-db"
       spider: "spider-db"
 
+  compression_coordinator:
+    commit_task_hard_timeout_secs: 60
+    commit_task_max_retry: 1
+    commit_task_soft_timeout_secs: 45
+    compression_task_max_retry: 1
+    database_connection_pool_size: 10
+    job_polling_interval_millisecs: 100
+    logging_level: "INFO"
+    resource_group:
+      name: "compression-coordinator"
+    result_polling:
+      init_backoff_millisecs: 100
+      max_backoff_millisecs: 1000
+    termination_timeout_secs: 30
+
   compression_scheduler:
     jobs_poll_delay: 0.1  # seconds
     logging_level: "INFO"
@@ -359,3 +376,8 @@ serviceAccount:
   annotations:
   # For IRSA on EKS, set:
   #   eks.amazonaws.com/role-arn: "arn:aws:iam::<ACCOUNT_ID>:role/<role-name>"
+
+# Overrides for the Spider subchart's `values.yaml`
+spider:
+  enabled: false
+  fullnameOverride: "spider"