feat(helm)!: Add the Spider subchart and compression-coordinator wiring for Spider-orchestrated compression.

taskfiles/helm.yaml

@@ -16,4 +16,5 @@ tasks:
       - "mkdir -p '{{.OUTPUT_DIR}}'"
       - |-
         . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}"
+        helm dependency update "{{.ROOT_DIR}}/tools/deployment/package-helm"
         helm package "{{.ROOT_DIR}}/tools/deployment/package-helm" --destination "{{.OUTPUT_DIR}}"

taskfiles/lint.yaml

@@ -69,6 +69,7 @@ tasks:
         . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}"
         ct lint \
           --chart-dirs="tools/deployment" \
+          --chart-repos="spider=https://github.com/y-scope/spider/raw/gh-pages" \
           --chart-yaml-schema="{{.G_HELM_TOOLCHAIN_DIR}}/etc/chart_schema.yaml" \
           --lint-conf="tools/yscope-dev-utils/exports/lint-configs/.yamllint.yml"
 

tools/deployment/package-helm/.gitignore

@@ -0,0 +1,2 @@
+# Subcharts are downloaded by `helm dependency update`.
+/charts/

tools/deployment/package-helm/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: spider
+  repository: https://github.com/y-scope/spider/raw/gh-pages
+  version: 0.1.8
+digest: sha256:8f49c09d52edef214a19f9442f5275a39b39a5259657f9314a25b1ed79308785
+generated: "2026-07-30T15:54:39.26863006-04:00"

tools/deployment/package-helm/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: "v2"
 name: "clp"
-version: "0.4.1-dev.3"
+version: "0.4.1-dev.4"
 description: "A Helm chart for CLP's (Compressed Log Processor) package deployment"
 type: "application"
 appVersion: "0.13.1-dev"
@@ -15,3 +15,8 @@ keywords:
   - "log-parser"
   - "logging"
   - "search"
+dependencies:
+  - name: "spider"
+    version: "0.1.8"
+    repository: "https://github.com/y-scope/spider/raw/gh-pages"
+    condition: "spider.enabled"

tools/deployment/package-helm/set-up-multi-dedicated-test.sh

@@ -136,6 +136,7 @@ EOF
 echo "Installing Helm chart..."
 helm uninstall test --ignore-not-found
 sleep 2
+helm dependency update "${script_dir}"
 # Word splitting is intentional: helper functions return multiple --set flags.
 # shellcheck disable=SC2046
 helm install test "${script_dir}" \

tools/deployment/package-helm/set-up-multi-shared-test.sh

@@ -42,6 +42,7 @@ generate_kind_config "${NUM_WORKER_NODES}" | kind create cluster --name "${CLUST
 echo "Installing Helm chart..."
 helm uninstall test --ignore-not-found
 sleep 2
+helm dependency update "${script_dir}"
 # Word splitting is intentional: helper functions return multiple --set flags.
 # shellcheck disable=SC2046
 helm install test "${script_dir}" \

tools/deployment/package-helm/set-up-test.sh

@@ -26,6 +26,7 @@ generate_kind_config 0 | kind create cluster --name "${CLUSTER_NAME}" --config=-
 echo "Installing Helm chart..."
 helm uninstall test --ignore-not-found
 sleep 2
+helm dependency update "${script_dir}"
 # Word splitting is intentional: helper functions return multiple --set flags.
 # shellcheck disable=SC2046
 helm install test "${script_dir}" \

tools/deployment/package-helm/templates/_helpers.tpl

@@ -4,7 +4,8 @@ Expands the name of the chart.
 @return {string} The chart name (truncated to 63 characters)
 */}}
 {{- define "clp.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- $global := .Values.global | default dict }}
+{{- default .Chart.Name $global.nameOverride | trunc 63 | trimSuffix "-" }}
 {{- end }}
 
 {{/*
@@ -15,10 +16,11 @@ used as a full name.
 @return {string} The fully qualified app name (truncated to 63 characters)
 */}}
 {{- define "clp.fullname" -}}
-{{- if .Values.fullnameOverride }}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- $global := .Values.global | default dict }}
+{{- if $global.fullnameOverride }}
+{{- $global.fullnameOverride | trunc 63 | trimSuffix "-" }}
 {{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- $name := default "clp" $global.nameOverride }}
 {{- if contains $name .Release.Name }}
 {{- .Release.Name | trunc 63 | trimSuffix "-" }}
 {{- else }}
@@ -442,6 +444,27 @@ Gets the port for the Presto service.
 {{- end -}}
 {{- end }}
 
+{{/*
+Gets the host for the Spider storage service.
+
+@param {object} . Root template context
+@return {string} The Spider storage host
+*/}}
+{{- define "clp.spiderStorageHost" -}}
+{{- include "spider.componentFullname" (dict "root" (index .Subcharts "spider") "component" "storage") -}}
+{{- end }}
+
+{{/*
+Gets the port for the Spider storage service.
+
+@param {object} . Root template context
+@return {string} The Spider storage port
+*/}}
+{{- define "clp.spiderStoragePort" -}}
+{{- $spider := index .Subcharts "spider" -}}
+{{- $spider.Values.spiderConfig.storage.port -}}
+{{- end }}
+
 {{/*
 Gets the BROKER_URL env var for Celery workers.
 
@@ -545,7 +568,7 @@ command: [
   "kubectl", "wait",
   {{- if eq .type "service" }}
   "--for=condition=ready",
-  "pod", "--selector", "app.kubernetes.io/component={{ .name }}",
+  "pod", "--selector", "app.kubernetes.io/instance={{ .root.Release.Name }},app.kubernetes.io/component={{ .name }}",
   {{- else if eq .type "job" }}
   "--for=condition=complete",
   "job/{{ include "clp.fullname" .root }}-{{ .name }}",

tools/deployment/package-helm/templates/compression-coordinator-deployment.yaml

@@ -0,0 +1,75 @@
+{{- if .Values.spider.enabled }}
+apiVersion: "apps/v1"
+kind: "Deployment"
+metadata:
+  name: {{ include "clp.fullname" . }}-compression-coordinator
+  labels:
+    {{- include "clp.labels" . | nindent 4 }}
+    app.kubernetes.io/component: "compression-coordinator"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "clp.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: "compression-coordinator"
+  template:
+    metadata:
+      labels:
+        {{- include "clp.labels" . | nindent 8 }}
+        app.kubernetes.io/component: "compression-coordinator"
+    spec:
+      {{- include "clp.createSchedulingConfigs" (dict
+          "root" .
+          "component" "compressionCoordinator"
+        ) | nindent 6 }}
+      serviceAccountName: {{ include "clp.fullname" . }}-service-account
+      terminationGracePeriodSeconds: 60
+      initContainers:
+        - {{- include "clp.waitFor" (dict
+            "root" .
+            "type" "job"
+            "name" "db-table-creator"
+          ) | nindent 10 }}
+        - {{- include "clp.waitFor" (dict
+            "root" .
+            "type" "service"
+            "name" "storage"
+          ) | nindent 10 }}
+      containers:
+        - name: "compression-coordinator"
+          image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }}
+          imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}"
+          env:
+            {{- include "clp.telemetryEnv" . | nindent 12 }}
+            - name: "CLP_DB_PASS"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "clp.fullname" . }}-database
+                  key: "password"
+            - name: "CLP_DB_USER"
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "clp.fullname" . }}-database
+                  key: "username"
+            - name: "OTEL_SERVICE_NAME"
+              value: "compression-coordinator"
+            - name: "RUST_LOG"
+              value: {{ .Values.clpConfig.compression_coordinator.logging_level | quote }}
+          {{- include "clp.createResourceLimits" (dict
+              "root" .
+              "component" "compressionCoordinator"
+            ) | nindent 10 }}
+          volumeMounts:
+            - name: "config"
+              mountPath: "/etc/clp-config.yaml"
+              subPath: "clp-config.yaml"
+              readOnly: true
+          command: [
+            "/opt/clp/bin/compression-coordinator",
+            "--config", "/etc/clp-config.yaml"
+          ]
+      volumes:
+        - name: "config"
+          configMap:
+            name: {{ include "clp.fullname" . }}-config
+{{- end }}

tools/deployment/package-helm/templates/configmap.yaml

@@ -67,6 +67,33 @@ data:
         | int }}
       target_segment_size: {{ .Values.clpConfig.archive_output.target_segment_size
         | int }}
+    {{- if .Values.spider.enabled }}
+    compression_coordinator:
+      commit_task_hard_timeout_secs: {{
+        .Values.clpConfig.compression_coordinator.commit_task_hard_timeout_secs | int }}
+      commit_task_max_retry: {{
+        .Values.clpConfig.compression_coordinator.commit_task_max_retry | int }}
+      commit_task_soft_timeout_secs: {{
+        .Values.clpConfig.compression_coordinator.commit_task_soft_timeout_secs | int }}
+      compression_task_max_retry: {{
+        .Values.clpConfig.compression_coordinator.compression_task_max_retry | int }}
+      database_connection_pool_size: {{
+        .Values.clpConfig.compression_coordinator.database_connection_pool_size | int }}
+      job_polling_interval_millisecs: {{
+        .Values.clpConfig.compression_coordinator.job_polling_interval_millisecs | int }}
+      resource_group:
+        name: {{ .Values.clpConfig.compression_coordinator.resource_group.name | quote }}
+      result_polling:
+        init_backoff_millisecs: {{
+          .Values.clpConfig.compression_coordinator.result_polling.init_backoff_millisecs | int }}
+        max_backoff_millisecs: {{
+          .Values.clpConfig.compression_coordinator.result_polling.max_backoff_millisecs | int }}
+      termination_timeout_secs: {{
+        .Values.clpConfig.compression_coordinator.termination_timeout_secs | int }}
+    spider:
+      host: "{{ include "clp.spiderStorageHost" . }}"
+      port: {{ include "clp.spiderStoragePort" . | int }}
+    {{- end }}
     compression_scheduler:
       jobs_poll_delay: {{ .Values.clpConfig.compression_scheduler.jobs_poll_delay }}
       logging_level: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }}

tools/deployment/package-helm/values.yaml

@@ -1,5 +1,6 @@
-nameOverride: ""
-fullnameOverride: ""
+global:
+  fullnameOverride: ""
+  nameOverride: ""
 
 # Whether to allow scripts in `sbin` access to services on the host.
 allowHostAccessForSbinScripts: true
@@ -100,6 +101,7 @@ scheduling:
   resultsCache: {}
 
   # Core
+  compressionCoordinator: {}
   compressionScheduler: {}
   logIngestor: {}
   queryScheduler: {}
@@ -130,6 +132,7 @@ resources:
   resultsCache: {}
 
   # Core
+  compressionCoordinator: {}
   compressionScheduler: {}
   logIngestor: {}
   queryScheduler: {}
@@ -190,6 +193,21 @@ clpConfig:
       clp: "clp-db"
       spider: "spider-db"
 
+  compression_coordinator:
+    commit_task_hard_timeout_secs: 60
+    commit_task_max_retry: 1
+    commit_task_soft_timeout_secs: 45
+    compression_task_max_retry: 1
+    database_connection_pool_size: 10
+    job_polling_interval_millisecs: 100
+    logging_level: "INFO"
+    resource_group:
+      name: "compression-coordinator"
+    result_polling:
+      init_backoff_millisecs: 100
+      max_backoff_millisecs: 1000
+    termination_timeout_secs: 30
+
   compression_scheduler:
     jobs_poll_delay: 0.1  # seconds
     logging_level: "INFO"
@@ -366,3 +384,40 @@ serviceAccount:
   annotations:
   # For IRSA on EKS, set:
   #   eks.amazonaws.com/role-arn: "arn:aws:iam::<ACCOUNT_ID>:role/<role-name>"
+
+# Overrides for the Spider subchart's `values.yaml`
+spider:
+  enabled: false
+  image:
+    worker:
+      repository: "ghcr.io/y-scope/clp/clp-spider-worker"
+      tag: "main"
+  spiderConfig:
+    worker:
+      extra_envs:
+        - name: "CLP_CONFIG_PATH"
+          value: "/etc/clp-config.yaml"
+        - name: "CLP_DB_USER"
+          valueFrom:
+            secretKeyRef:
+              name: "{{ include \"clp.fullname\" . }}-database"
+              key: "username"
+        - name: "CLP_DB_PASS"
+          valueFrom:
+            secretKeyRef:
+              name: "{{ include \"clp.fullname\" . }}-database"
+              key: "password"
+      extra_volume_mounts:
+        - name: "clp-config"
+          mountPath: "/etc/clp-config.yaml"
+          subPath: "clp-config.yaml"
+          readOnly: true
+        - name: "staged-archives"
+          mountPath: "/var/data/staged-archives"
+      extra_volumes:
+        - name: "clp-config"
+          configMap:
+            name: "{{ include \"clp.fullname\" . }}-config"
+        - name: "staged-archives"
+          emptyDir: {}
+      service_account_name: "{{ include \"clp.fullname\" . }}-service-account"