diff --git a/taskfiles/helm.yaml b/taskfiles/helm.yaml index 424bc8e5a..aeddfca82 100644 --- a/taskfiles/helm.yaml +++ b/taskfiles/helm.yaml @@ -16,4 +16,5 @@ tasks: - "mkdir -p '{{.OUTPUT_DIR}}'" - |- . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" + helm dependency update "{{.ROOT_DIR}}/tools/deployment/package-helm" helm package "{{.ROOT_DIR}}/tools/deployment/package-helm" --destination "{{.OUTPUT_DIR}}" diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 477f49392..6eca4a659 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -69,6 +69,7 @@ tasks: . "{{.G_HELM_TOOLCHAIN_ENV_FILE}}" ct lint \ --chart-dirs="tools/deployment" \ + --chart-repos="spider=https://github.com/y-scope/spider/raw/gh-pages" \ --chart-yaml-schema="{{.G_HELM_TOOLCHAIN_DIR}}/etc/chart_schema.yaml" \ --lint-conf="tools/yscope-dev-utils/exports/lint-configs/.yamllint.yml" diff --git a/tools/deployment/package-helm/.gitignore b/tools/deployment/package-helm/.gitignore new file mode 100644 index 000000000..f987a667d --- /dev/null +++ b/tools/deployment/package-helm/.gitignore @@ -0,0 +1,2 @@ +# Subcharts are downloaded by `helm dependency update`. +/charts/ diff --git a/tools/deployment/package-helm/Chart.lock b/tools/deployment/package-helm/Chart.lock new file mode 100644 index 000000000..ef7f5577e --- /dev/null +++ b/tools/deployment/package-helm/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: spider + repository: https://github.com/y-scope/spider/raw/gh-pages + version: 0.1.8 +digest: sha256:8f49c09d52edef214a19f9442f5275a39b39a5259657f9314a25b1ed79308785 +generated: "2026-07-30T15:54:39.26863006-04:00" diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 5e6fef4af..f96fd11bc 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "clp" -version: "0.4.1-dev.3" +version: "0.4.1-dev.4" description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.13.1-dev" @@ -15,3 +15,8 @@ keywords: - "log-parser" - "logging" - "search" +dependencies: + - name: "spider" + version: "0.1.8" + repository: "https://github.com/y-scope/spider/raw/gh-pages" + condition: "spider.enabled" diff --git a/tools/deployment/package-helm/set-up-multi-dedicated-test.sh b/tools/deployment/package-helm/set-up-multi-dedicated-test.sh index b838b0519..5e5e185c6 100755 --- a/tools/deployment/package-helm/set-up-multi-dedicated-test.sh +++ b/tools/deployment/package-helm/set-up-multi-dedicated-test.sh @@ -136,6 +136,7 @@ EOF echo "Installing Helm chart..." helm uninstall test --ignore-not-found sleep 2 +helm dependency update "${script_dir}" # Word splitting is intentional: helper functions return multiple --set flags. # shellcheck disable=SC2046 helm install test "${script_dir}" \ diff --git a/tools/deployment/package-helm/set-up-multi-shared-test.sh b/tools/deployment/package-helm/set-up-multi-shared-test.sh index 45a498d9a..6c53b0a53 100755 --- a/tools/deployment/package-helm/set-up-multi-shared-test.sh +++ b/tools/deployment/package-helm/set-up-multi-shared-test.sh @@ -42,6 +42,7 @@ generate_kind_config "${NUM_WORKER_NODES}" | kind create cluster --name "${CLUST echo "Installing Helm chart..." helm uninstall test --ignore-not-found sleep 2 +helm dependency update "${script_dir}" # Word splitting is intentional: helper functions return multiple --set flags. # shellcheck disable=SC2046 helm install test "${script_dir}" \ diff --git a/tools/deployment/package-helm/set-up-test.sh b/tools/deployment/package-helm/set-up-test.sh index de7238363..0578dabfb 100755 --- a/tools/deployment/package-helm/set-up-test.sh +++ b/tools/deployment/package-helm/set-up-test.sh @@ -26,6 +26,7 @@ generate_kind_config 0 | kind create cluster --name "${CLUSTER_NAME}" --config=- echo "Installing Helm chart..." helm uninstall test --ignore-not-found sleep 2 +helm dependency update "${script_dir}" # Word splitting is intentional: helper functions return multiple --set flags. # shellcheck disable=SC2046 helm install test "${script_dir}" \ diff --git a/tools/deployment/package-helm/templates/_helpers.tpl b/tools/deployment/package-helm/templates/_helpers.tpl index 26d03d852..b4054e792 100644 --- a/tools/deployment/package-helm/templates/_helpers.tpl +++ b/tools/deployment/package-helm/templates/_helpers.tpl @@ -4,7 +4,8 @@ Expands the name of the chart. @return {string} The chart name (truncated to 63 characters) */}} {{- define "clp.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- $global := .Values.global | default dict }} +{{- default .Chart.Name $global.nameOverride | trunc 63 | trimSuffix "-" }} {{- end }} {{/* @@ -15,10 +16,11 @@ used as a full name. @return {string} The fully qualified app name (truncated to 63 characters) */}} {{- define "clp.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- $global := .Values.global | default dict }} +{{- if $global.fullnameOverride }} +{{- $global.fullnameOverride | trunc 63 | trimSuffix "-" }} {{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} +{{- $name := default "clp" $global.nameOverride }} {{- if contains $name .Release.Name }} {{- .Release.Name | trunc 63 | trimSuffix "-" }} {{- else }} @@ -442,6 +444,27 @@ Gets the port for the Presto service. {{- end -}} {{- end }} +{{/* +Gets the host for the Spider storage service. + +@param {object} . Root template context +@return {string} The Spider storage host +*/}} +{{- define "clp.spiderStorageHost" -}} +{{- include "spider.componentFullname" (dict "root" (index .Subcharts "spider") "component" "storage") -}} +{{- end }} + +{{/* +Gets the port for the Spider storage service. + +@param {object} . Root template context +@return {string} The Spider storage port +*/}} +{{- define "clp.spiderStoragePort" -}} +{{- $spider := index .Subcharts "spider" -}} +{{- $spider.Values.spiderConfig.storage.port -}} +{{- end }} + {{/* Gets the BROKER_URL env var for Celery workers. @@ -545,7 +568,7 @@ command: [ "kubectl", "wait", {{- if eq .type "service" }} "--for=condition=ready", - "pod", "--selector", "app.kubernetes.io/component={{ .name }}", + "pod", "--selector", "app.kubernetes.io/instance={{ .root.Release.Name }},app.kubernetes.io/component={{ .name }}", {{- else if eq .type "job" }} "--for=condition=complete", "job/{{ include "clp.fullname" .root }}-{{ .name }}", diff --git a/tools/deployment/package-helm/templates/compression-coordinator-deployment.yaml b/tools/deployment/package-helm/templates/compression-coordinator-deployment.yaml new file mode 100644 index 000000000..8378ff57f --- /dev/null +++ b/tools/deployment/package-helm/templates/compression-coordinator-deployment.yaml @@ -0,0 +1,75 @@ +{{- if .Values.spider.enabled }} +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp.fullname" . }}-compression-coordinator + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "compression-coordinator" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "compression-coordinator" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "compression-coordinator" + spec: + {{- include "clp.createSchedulingConfigs" (dict + "root" . + "component" "compressionCoordinator" + ) | nindent 6 }} + serviceAccountName: {{ include "clp.fullname" . }}-service-account + terminationGracePeriodSeconds: 60 + initContainers: + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "service" + "name" "storage" + ) | nindent 10 }} + containers: + - name: "compression-coordinator" + image: {{ include "clp.imageRef" (dict "root" . "component" "clpPackage") | quote }} + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" + env: + {{- include "clp.telemetryEnv" . | nindent 12 }} + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "password" + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "username" + - name: "OTEL_SERVICE_NAME" + value: "compression-coordinator" + - name: "RUST_LOG" + value: {{ .Values.clpConfig.compression_coordinator.logging_level | quote }} + {{- include "clp.createResourceLimits" (dict + "root" . + "component" "compressionCoordinator" + ) | nindent 10 }} + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true + command: [ + "/opt/clp/bin/compression-coordinator", + "--config", "/etc/clp-config.yaml" + ] + volumes: + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config +{{- end }} diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index ee49584a0..48d7aeef8 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -67,6 +67,33 @@ data: | int }} target_segment_size: {{ .Values.clpConfig.archive_output.target_segment_size | int }} + {{- if .Values.spider.enabled }} + compression_coordinator: + commit_task_hard_timeout_secs: {{ + .Values.clpConfig.compression_coordinator.commit_task_hard_timeout_secs | int }} + commit_task_max_retry: {{ + .Values.clpConfig.compression_coordinator.commit_task_max_retry | int }} + commit_task_soft_timeout_secs: {{ + .Values.clpConfig.compression_coordinator.commit_task_soft_timeout_secs | int }} + compression_task_max_retry: {{ + .Values.clpConfig.compression_coordinator.compression_task_max_retry | int }} + database_connection_pool_size: {{ + .Values.clpConfig.compression_coordinator.database_connection_pool_size | int }} + job_polling_interval_millisecs: {{ + .Values.clpConfig.compression_coordinator.job_polling_interval_millisecs | int }} + resource_group: + name: {{ .Values.clpConfig.compression_coordinator.resource_group.name | quote }} + result_polling: + init_backoff_millisecs: {{ + .Values.clpConfig.compression_coordinator.result_polling.init_backoff_millisecs | int }} + max_backoff_millisecs: {{ + .Values.clpConfig.compression_coordinator.result_polling.max_backoff_millisecs | int }} + termination_timeout_secs: {{ + .Values.clpConfig.compression_coordinator.termination_timeout_secs | int }} + spider: + host: "{{ include "clp.spiderStorageHost" . }}" + port: {{ include "clp.spiderStoragePort" . | int }} + {{- end }} compression_scheduler: jobs_poll_delay: {{ .Values.clpConfig.compression_scheduler.jobs_poll_delay }} logging_level: {{ .Values.clpConfig.compression_scheduler.logging_level | quote }} diff --git a/tools/deployment/package-helm/values.yaml b/tools/deployment/package-helm/values.yaml index 27412fdbb..4e38dc3ae 100644 --- a/tools/deployment/package-helm/values.yaml +++ b/tools/deployment/package-helm/values.yaml @@ -1,5 +1,6 @@ -nameOverride: "" -fullnameOverride: "" +global: + fullnameOverride: "" + nameOverride: "" # Whether to allow scripts in `sbin` access to services on the host. allowHostAccessForSbinScripts: true @@ -100,6 +101,7 @@ scheduling: resultsCache: {} # Core + compressionCoordinator: {} compressionScheduler: {} logIngestor: {} queryScheduler: {} @@ -130,6 +132,7 @@ resources: resultsCache: {} # Core + compressionCoordinator: {} compressionScheduler: {} logIngestor: {} queryScheduler: {} @@ -190,6 +193,21 @@ clpConfig: clp: "clp-db" spider: "spider-db" + compression_coordinator: + commit_task_hard_timeout_secs: 60 + commit_task_max_retry: 1 + commit_task_soft_timeout_secs: 45 + compression_task_max_retry: 1 + database_connection_pool_size: 10 + job_polling_interval_millisecs: 100 + logging_level: "INFO" + resource_group: + name: "compression-coordinator" + result_polling: + init_backoff_millisecs: 100 + max_backoff_millisecs: 1000 + termination_timeout_secs: 30 + compression_scheduler: jobs_poll_delay: 0.1 # seconds logging_level: "INFO" @@ -366,3 +384,40 @@ serviceAccount: annotations: # For IRSA on EKS, set: # eks.amazonaws.com/role-arn: "arn:aws:iam:::role/" + +# Overrides for the Spider subchart's `values.yaml` +spider: + enabled: false + image: + worker: + repository: "ghcr.io/y-scope/clp/clp-spider-worker" + tag: "main" + spiderConfig: + worker: + extra_envs: + - name: "CLP_CONFIG_PATH" + value: "/etc/clp-config.yaml" + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: "{{ include \"clp.fullname\" . }}-database" + key: "username" + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: "{{ include \"clp.fullname\" . }}-database" + key: "password" + extra_volume_mounts: + - name: "clp-config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true + - name: "staged-archives" + mountPath: "/var/data/staged-archives" + extra_volumes: + - name: "clp-config" + configMap: + name: "{{ include \"clp.fullname\" . }}-config" + - name: "staged-archives" + emptyDir: {} + service_account_name: "{{ include \"clp.fullname\" . }}-service-account"