deployment: add rules formatter makefile command to force literal style for rule summary and description

deployment/docker/Makefile

@@ -14,6 +14,8 @@ DOCKER_BUILD ?= $(DOCKER) build
 DOCKER_COMPOSE ?= $(DOCKER) compose
 DOCKER_IMAGE_LS ?= $(DOCKER) image ls --format '{{.Repository}}:{{.Tag}}'
 
+include deployment/docker/rules/Makefile
+
 package-builder:
 	($(DOCKER_IMAGE_LS) | grep -q '$(BUILDER_IMAGE)$$') \
 		|| $(DOCKER_BUILD) \

deployment/docker/rules/Makefile

@@ -0,0 +1,8 @@
+
+RULES_DIR ?= ./deployment/docker/rules
+
+format-rules:
+	find $(RULES_DIR) -type f -name "*.yml" | while IFS= read -r f; do \
+		echo "Processing $${f}"; \
+		yq -i '.groups[].rules[].annotations.summary style="literal" | .groups[].rules[].annotations.description style="literal"' "$${f}"; \
+	done

deployment/docker/rules/alerts-health.yml

@@ -11,104 +11,101 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: "{{ $labels.job }} too many restarts (instance {{ $labels.instance }})"
-          description: >
-            Job {{ $labels.job }} (instance {{ $labels.instance }}) has restarted more than twice in the last 15 minutes.
-            It might be crashlooping.
-
+          summary: |-
+            {{ $labels.job }} too many restarts (instance {{ $labels.instance }})
+          description: |-
+            Job {{ $labels.job }} (instance {{ $labels.instance }}) has restarted more than twice in the last 15 minutes. It might be crashlooping.
       - alert: ServiceDown
         expr: up{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"} == 0
         for: 2m
         labels:
           severity: critical
         annotations:
-          summary: "Service {{ $labels.job }} is down on {{ $labels.instance }}"
-          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes."
-
+          summary: |-
+            Service {{ $labels.job }} is down on {{ $labels.instance }}
+          description: |-
+            {{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.
       - alert: ProcessNearFDLimits
         expr: (process_max_fds{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"} - process_open_fds{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"}) < 100
         for: 5m
         labels:
           severity: critical
         annotations:
-          summary: "Number of free file descriptors is less than 100 for \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") for the last 5m"
-          description: | 
+          summary: |-
+            Number of free file descriptors is less than 100 for "{{ $labels.job }}"("{{ $labels.instance }}") for the last 5m
+          description: |-
             Exhausting OS file descriptors limit can cause severe degradation of the process.
             Consider to increase the limit as fast as possible.
-
       - alert: TooHighMemoryUsage
         expr: (min_over_time(process_resident_memory_anon_bytes{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"}[10m]) / vm_available_memory_bytes{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"}) > 0.8
         for: 5m
         labels:
           severity: critical
         annotations:
-          summary: "It is more than 80% of memory used by \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")"
-          description: |
+          summary: |-
+            It is more than 80% of memory used by "{{ $labels.job }}"("{{ $labels.instance }}")
+          description: |-
             Too high memory usage may result into multiple issues such as OOMs or degraded performance.
             Consider to either increase available memory or decrease the load on the process.
-
       - alert: TooHighCPUUsage
         expr: rate(process_cpu_seconds_total{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"}[5m]) / process_cpu_cores_available{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"} > 0.9
         for: 5m
         labels:
           severity: critical
         annotations:
-          summary: "More than 90% of CPU is used by \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") during the last 5m"
-          description: >
+          summary: |-
+            More than 90% of CPU is used by "{{ $labels.job }}"("{{ $labels.instance }}") during the last 5m
+          description: |-
             Too high CPU usage may be a sign of insufficient resources and make process unstable.
             Consider to either increase available CPU resources or decrease the load on the process.
-
       - alert: TooHighGoroutineSchedulingLatency
         expr: histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"}[5m])) by (le, job, instance)) > 0.1
         for: 15m
         labels:
           severity: critical
         annotations:
-          summary: "\"{{ $labels.job }}\"(\"{{ $labels.instance }}\") has insufficient CPU resources for >15m"
-          description: >
-            Go runtime is unable to schedule goroutines execution in acceptable time. This is usually a sign of
-            insufficient CPU resources or CPU throttling. Verify that service has enough CPU resources. Otherwise,
-            the service could work unreliably with delays in processing.
-
+          summary: |-
+            "{{ $labels.job }}"("{{ $labels.instance }}") has insufficient CPU resources for >15m
+          description: "Go runtime is unable to schedule goroutines execution in acceptable time. This is usually a sign of insufficient CPU resources or CPU throttling. \nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing."
       - alert: TooManyLogs
         expr: sum(increase(vm_log_messages_total{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*", level="error"}[5m])) without (app_version, location) > 0
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Too many logs printed for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
-          description: >
-            Logging rate for job \"{{ $labels.job }}\" ({{ $labels.instance }}) is {{ $value }} for last 15m.
-            Worth to check logs for specific error messages.
-
+          summary: |-
+            Too many logs printed for job "{{ $labels.job }}" ({{ $labels.instance }})
+          description: |-
+            Logging rate for job "{{ $labels.job }}" ({{ $labels.instance }}) is {{ $value }} for last 15m. Worth to check logs for specific error messages.
       - alert: RequestErrorsToAPI
         expr: increase(vl_http_errors_total[5m]) > 0
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Too many errors served for path {{ $labels.path }} (instance {{ $labels.instance }})"
-          description: "Requests to path {{ $labels.path }} are receiving errors.
-            Please verify if clients are sending correct requests."
-
+          summary: |-
+            Too many errors served for path {{ $labels.path }} (instance {{ $labels.instance }})
+          description: |-
+            Requests to path {{ $labels.path }} are receiving errors. Please verify if clients are sending correct requests.
       - alert: RowsRejectedOnIngestion
         expr: rate(vl_rows_dropped_total[5m]) > 0
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Some rows are rejected on \"{{ $labels.instance }}\" on ingestion attempt"
-          description: "VictoriaLogs is rejecting to ingest rows on \"{{ $labels.instance }}\" due to the
-            following reason: \"{{ $labels.reason }}\""
-
+          summary: |-
+            Some rows are rejected on "{{ $labels.instance }}" on ingestion attempt
+          description: |-
+            VictoriaLogs is rejecting to ingest rows on "{{ $labels.instance }}" due to the following reason: "{{ $labels.reason }}"
       - alert: ConcurrentInsertsHitTheLimit
         expr: avg_over_time(vm_concurrent_insert_current{job=~".*(victorialogs|vlstorage|vlinsert|vlagent).*"}[1m]) >= vm_concurrent_insert_capacity{job=~".*(victorialogs|vlstorage|vlinsert|vlagent).*"}
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "{{ $labels.job }} on instance {{ $labels.instance }} is constantly hitting concurrent inserts limit"
-          description: |
+          summary: |-
+            {{ $labels.job }} on instance {{ $labels.instance }} is constantly hitting concurrent inserts limit
+          description: |-
             The limit of concurrent inserts on instance {{ $labels.instance }} depends on the number of CPUs.
             Usually, when component constantly hits the limit it is likely the component is overloaded and requires more CPU.
             In some cases the alert might trigger if there are too many clients making write attempts to the component.

deployment/docker/rules/alerts-vlagent.yml

@@ -10,93 +10,91 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: "Instance {{ $labels.instance }} is dropping data from persistent queue"
-          description: "vlagent dropped {{ $value | humanize1024 }} from persistent queue
-              on instance {{ $labels.instance }} for the last 10m."
-
+          summary: |-
+            Instance {{ $labels.instance }} is dropping data from persistent queue
+          description: |-
+            vlagent dropped {{ $value | humanize1024 }} from persistent queue on instance {{ $labels.instance }} for the last 10m.
       - alert: RejectedRemoteWriteDataBlocksAreDropped
         expr: sum(increase(vlagent_remotewrite_packets_dropped_total[5m])) without (url) > 0
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "vlagent is dropping data blocks that are rejected by remote storage"
-          description: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} drops the rejected by 
-            remote-write server data blocks. Check the logs to find the reason for rejects."
-
+          summary: |-
+            vlagent is dropping data blocks that are rejected by remote storage
+          description: |-
+            Job "{{ $labels.job }}" on instance {{ $labels.instance }} drops the rejected by remote-write server data blocks. Check the logs to find the reason for rejects.
       - alert: TooManyRemoteWriteErrors
         expr: rate(vlagent_remotewrite_retries_count_total[5m]) > 0
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} fails to push to remote storage"
-          description: "vlagent fails to push data via remote write protocol to destination \"{{ $labels.url }}\"\n
-            Ensure that destination is up and reachable."
-
+          summary: |-
+            Job "{{ $labels.job }}" on instance {{ $labels.instance }} fails to push to remote storage
+          description: |-
+            vlagent fails to push data via remote write protocol to destination "{{ $labels.url }}"
+             Ensure that destination is up and reachable.
       - alert: RemoteWriteConnectionIsSaturated
         expr: rate(vlagent_remotewrite_send_duration_seconds_total[5m]) / vlagent_remotewrite_queues > 0.9
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Remote write connection from \"{{ $labels.job }}\" (instance {{ $labels.instance }}) to {{ $labels.url }} is saturated"
-          description: "The remote write connection between vlagent \"{{ $labels.job }}\" (instance {{ $labels.instance }}) and destination \"{{ $labels.url }}\"
-            is saturated by more than 90% and vlagent won't be able to keep up.\n
-            There could be the following reasons for this:\n
-             * vlagent can't send data fast enough through the existing network connections. Increase `-remoteWrite.queues` cmd-line flag value to establish more connections per destination.\n
-             * remote destination can't accept data fast enough. Check if remote destination has enough resources for processing."
-
+          summary: |-
+            Remote write connection from "{{ $labels.job }}" (instance {{ $labels.instance }}) to {{ $labels.url }} is saturated
+          description: |-
+            The remote write connection between vlagent "{{ $labels.job }}" (instance {{ $labels.instance }}) and destination "{{ $labels.url }}" is saturated by more than 90% and vlagent won't be able to keep up.
+             There could be the following reasons for this:
+             * vlagent can't send data fast enough through the existing network connections. Increase `-remoteWrite.queues` cmd-line flag value to establish more connections per destination.
+             * remote destination can't accept data fast enough. Check if remote destination has enough resources for processing.
       - alert: PersistentQueueForWritesIsSaturated
         expr: rate(vm_persistentqueue_write_duration_seconds_total[5m]) > 0.9
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Persistent queue writes for instance {{ $labels.instance }} are saturated"
-          description: "Persistent queue writes for vlagent \"{{ $labels.job }}\" (instance {{ $labels.instance }})
-            are saturated by more than 90% and vlagent won't be able to keep up with flushing data on disk. 
-            In this case, consider to decrease load on the vlagent or improve the disk throughput."
-
+          summary: |-
+            Persistent queue writes for instance {{ $labels.instance }} are saturated
+          description: |-
+            Persistent queue writes for vlagent "{{ $labels.job }}" (instance {{ $labels.instance }}) are saturated by more than 90% and vlagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vlagent or improve the disk throughput.
       - alert: PersistentQueueForReadsIsSaturated
         expr: rate(vm_persistentqueue_read_duration_seconds_total[5m]) > 0.9
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Persistent queue reads for instance {{ $labels.instance }} are saturated"
-          description: "Persistent queue reads for vlagent \"{{ $labels.job }}\" (instance {{ $labels.instance }})
-            are saturated by more than 90% and vlagent won't be able to keep up with reading data from the disk. 
-            In this case, consider to decrease load on the vlagent or improve the disk throughput."
-
+          summary: |-
+            Persistent queue reads for instance {{ $labels.instance }} are saturated
+          description: |-
+            Persistent queue reads for vlagent "{{ $labels.job }}" (instance {{ $labels.instance }}) are saturated by more than 90% and vlagent won't be able to keep up with reading data from the disk. In this case, consider to decrease load on the vlagent or improve the disk throughput.
       - alert: PersistentQueueRunsOutOfSpaceIn12Hours
-        expr: |
+        expr: |-
           (
             (sum(vm_persistentqueue_free_disk_space_bytes) by (job, instance, path)) /
             (sum(deriv(vm_persistentqueue_bytes_pending[1m])) by (job, instance, path) > 0)
-          ) * 
-          on(job, instance, path) group_left(url)
+          ) * on(job, instance, path) group_left(url)
           (vlagent_remotewrite_pending_data_bytes * 0 + 1) < 12 * 3600 >0
         for: 10m
         labels:
           severity: warning
         annotations:
-          summary: "Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 12 hours."
-          description: "RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. 
-            Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})."
-
+          summary: |-
+            Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 12 hours.
+          description: |-
+            RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }}).
       - alert: PersistentQueueRunsOutOfSpaceIn4Hours
-        expr: |
+        expr: |-
           (
             (sum(vm_persistentqueue_free_disk_space_bytes) by (job, instance, path)) /
             (sum(deriv(vm_persistentqueue_bytes_pending[1m])) by (job, instance, path) > 0)
-          ) * 
-          on(job, instance, path) group_left(url)
+          ) * on(job, instance, path) group_left(url)
           (vlagent_remotewrite_pending_data_bytes * 0 + 1) < 4 * 3600 >0
         for: 10m
         labels:
           severity: critical
         annotations:
-          summary: "Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 4 hours."
-          description: "RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. 
-            Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})."
+          summary: |-
+            Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 4 hours.
+          description: |-
+            RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }}).