deployment: add rules formatter makefile command to force literal style for rule summary and description

deployment/docker/Makefile

@@ -14,6 +14,8 @@ DOCKER_BUILD ?= $(DOCKER) build
 DOCKER_COMPOSE ?= $(DOCKER) compose
 DOCKER_IMAGE_LS ?= $(DOCKER) image ls --format '{{.Repository}}:{{.Tag}}'
 
+include deployment/docker/rules/Makefile
+
 package-builder:
 	($(DOCKER_IMAGE_LS) | grep -q '$(BUILDER_IMAGE)$$') \
 		|| $(DOCKER_BUILD) \

deployment/docker/rules/Makefile

@@ -0,0 +1,8 @@
+
+RULES_DIR ?= ./deployment/docker/rules
+
+format-rules:
+	find $(RULES_DIR) -type f -name "*.yml" | while IFS= read -r f; do \
+		echo "Processing $${f}"; \
+		yq -i '.groups[].rules[].annotations.summary style="literal" | .groups[].rules[].annotations.description style="literal"' "$${f}"; \
+	done

deployment/docker/rules/alerts-health.yml

@@ -11,103 +11,100 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: "{{ $labels.job }} too many restarts (instance {{ $labels.instance }})"
-          description: >
-            Job {{ $labels.job }} (instance {{ $labels.instance }}) has restarted more than twice in the last 15 minutes.
-            It might be crashlooping.
-
+          summary: |-
+            {{ $labels.job }} too many restarts (instance {{ $labels.instance }})
+          description: |
+            Job {{ $labels.job }} (instance {{ $labels.instance }}) has restarted more than twice in the last 15 minutes. It might be crashlooping.
       - alert: ServiceDown
         expr: up{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"} == 0
         for: 2m
         labels:
           severity: critical
         annotations:
-          summary: "Service {{ $labels.job }} is down on {{ $labels.instance }}"
-          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes."
-
+          summary: |-
+            Service {{ $labels.job }} is down on {{ $labels.instance }}
+          description: |-
+            {{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes.
       - alert: ProcessNearFDLimits
         expr: (process_max_fds{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"} - process_open_fds{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"}) < 100
         for: 5m
         labels:
           severity: critical
         annotations:
-          summary: "Number of free file descriptors is less than 100 for \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") for the last 5m"
-          description: | 
+          summary: |-
+            Number of free file descriptors is less than 100 for "{{ $labels.job }}"("{{ $labels.instance }}") for the last 5m
+          description: |
             Exhausting OS file descriptors limit can cause severe degradation of the process.
             Consider to increase the limit as fast as possible.
-
       - alert: TooHighMemoryUsage
         expr: (min_over_time(process_resident_memory_anon_bytes{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"}[10m]) / vm_available_memory_bytes{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"}) > 0.8
         for: 5m
         labels:
           severity: critical
         annotations:
-          summary: "It is more than 80% of memory used by \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")"
+          summary: |-
+            It is more than 80% of memory used by "{{ $labels.job }}"("{{ $labels.instance }}")
           description: |
             Too high memory usage may result into multiple issues such as OOMs or degraded performance.
             Consider to either increase available memory or decrease the load on the process.
-
       - alert: TooHighCPUUsage
         expr: rate(process_cpu_seconds_total{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"}[5m]) / process_cpu_cores_available{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"} > 0.9
         for: 5m
         labels:
           severity: critical
         annotations:
-          summary: "More than 90% of CPU is used by \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") during the last 5m"
-          description: >
-            Too high CPU usage may be a sign of insufficient resources and make process unstable.
-            Consider to either increase available CPU resources or decrease the load on the process.
-
+          summary: |-
+            More than 90% of CPU is used by "{{ $labels.job }}"("{{ $labels.instance }}") during the last 5m
+          description: |
+            Too high CPU usage may be a sign of insufficient resources and make process unstable. Consider to either increase available CPU resources or decrease the load on the process.
       - alert: TooHighGoroutineSchedulingLatency
         expr: histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"}[5m])) by (le, job, instance)) > 0.1
         for: 15m
         labels:
           severity: critical
         annotations:
-          summary: "\"{{ $labels.job }}\"(\"{{ $labels.instance }}\") has insufficient CPU resources for >15m"
-          description: >
-            Go runtime is unable to schedule goroutines execution in acceptable time. This is usually a sign of
-            insufficient CPU resources or CPU throttling. Verify that service has enough CPU resources. Otherwise,
-            the service could work unreliably with delays in processing.
-
+          summary: |-
+            "{{ $labels.job }}"("{{ $labels.instance }}") has insufficient CPU resources for >15m
+          description: |
+            Go runtime is unable to schedule goroutines execution in acceptable time. This is usually a sign of insufficient CPU resources or CPU throttling. Verify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing.
       - alert: TooManyLogs
         expr: sum(increase(vm_log_messages_total{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*", level="error"}[5m])) without (app_version, location) > 0
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Too many logs printed for job \"{{ $labels.job }}\" ({{ $labels.instance }})"
-          description: >
-            Logging rate for job \"{{ $labels.job }}\" ({{ $labels.instance }}) is {{ $value }} for last 15m.
-            Worth to check logs for specific error messages.
-
+          summary: |-
+            Too many logs printed for job "{{ $labels.job }}" ({{ $labels.instance }})
+          description: |
+            Logging rate for job "{{ $labels.job }}" ({{ $labels.instance }}) is {{ $value }} for last 15m. Worth to check logs for specific error messages.
       - alert: RequestErrorsToAPI
         expr: increase(vl_http_errors_total[5m]) > 0
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Too many errors served for path {{ $labels.path }} (instance {{ $labels.instance }})"
-          description: "Requests to path {{ $labels.path }} are receiving errors.
-            Please verify if clients are sending correct requests."
-
+          summary: |-
+            Too many errors served for path {{ $labels.path }} (instance {{ $labels.instance }})
+          description: |-
+            Requests to path {{ $labels.path }} are receiving errors. Please verify if clients are sending correct requests.
       - alert: RowsRejectedOnIngestion
         expr: rate(vl_rows_dropped_total[5m]) > 0
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Some rows are rejected on \"{{ $labels.instance }}\" on ingestion attempt"
-          description: "VictoriaLogs is rejecting to ingest rows on \"{{ $labels.instance }}\" due to the
-            following reason: \"{{ $labels.reason }}\""
-
+          summary: |-
+            Some rows are rejected on "{{ $labels.instance }}" on ingestion attempt
+          description: |-
+            VictoriaLogs is rejecting to ingest rows on "{{ $labels.instance }}" due to the following reason: "{{ $labels.reason }}"
       - alert: ConcurrentInsertsHitTheLimit
         expr: avg_over_time(vm_concurrent_insert_current{job=~".*(victorialogs|vlstorage|vlinsert|vlagent).*"}[1m]) >= vm_concurrent_insert_capacity{job=~".*(victorialogs|vlstorage|vlinsert|vlagent).*"}
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "{{ $labels.job }} on instance {{ $labels.instance }} is constantly hitting concurrent inserts limit"
+          summary: |-
+            {{ $labels.job }} on instance {{ $labels.instance }} is constantly hitting concurrent inserts limit
           description: |
             The limit of concurrent inserts on instance {{ $labels.instance }} depends on the number of CPUs.
             Usually, when component constantly hits the limit it is likely the component is overloaded and requires more CPU.

deployment/docker/rules/alerts-vlagent.yml

@@ -10,93 +10,91 @@ groups:
         labels:
           severity: critical
         annotations:
-          summary: "Instance {{ $labels.instance }} is dropping data from persistent queue"
-          description: "vlagent dropped {{ $value | humanize1024 }} from persistent queue
-              on instance {{ $labels.instance }} for the last 10m."
-
+          summary: |-
+            Instance {{ $labels.instance }} is dropping data from persistent queue
+          description: |-
+            vlagent dropped {{ $value | humanize1024 }} from persistent queue on instance {{ $labels.instance }} for the last 10m.
       - alert: RejectedRemoteWriteDataBlocksAreDropped
         expr: sum(increase(vlagent_remotewrite_packets_dropped_total[5m])) without (url) > 0
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "vlagent is dropping data blocks that are rejected by remote storage"
-          description: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} drops the rejected by 
-            remote-write server data blocks. Check the logs to find the reason for rejects."
-
+          summary: |-
+            vlagent is dropping data blocks that are rejected by remote storage
+          description: |-
+            Job "{{ $labels.job }}" on instance {{ $labels.instance }} drops the rejected by remote-write server data blocks. Check the logs to find the reason for rejects.
       - alert: TooManyRemoteWriteErrors
         expr: rate(vlagent_remotewrite_retries_count_total[5m]) > 0
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} fails to push to remote storage"
-          description: "vlagent fails to push data via remote write protocol to destination \"{{ $labels.url }}\"\n
-            Ensure that destination is up and reachable."
-
+          summary: |-
+            Job "{{ $labels.job }}" on instance {{ $labels.instance }} fails to push to remote storage
+          description: |-
+            vlagent fails to push data via remote write protocol to destination "{{ $labels.url }}"
+             Ensure that destination is up and reachable.
       - alert: RemoteWriteConnectionIsSaturated
         expr: rate(vlagent_remotewrite_send_duration_seconds_total[5m]) / vlagent_remotewrite_queues > 0.9
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Remote write connection from \"{{ $labels.job }}\" (instance {{ $labels.instance }}) to {{ $labels.url }} is saturated"
-          description: "The remote write connection between vlagent \"{{ $labels.job }}\" (instance {{ $labels.instance }}) and destination \"{{ $labels.url }}\"
-            is saturated by more than 90% and vlagent won't be able to keep up.\n
-            There could be the following reasons for this:\n
-             * vlagent can't send data fast enough through the existing network connections. Increase `-remoteWrite.queues` cmd-line flag value to establish more connections per destination.\n
-             * remote destination can't accept data fast enough. Check if remote destination has enough resources for processing."
-
+          summary: |-
+            Remote write connection from "{{ $labels.job }}" (instance {{ $labels.instance }}) to {{ $labels.url }} is saturated
+          description: |-
+            The remote write connection between vlagent "{{ $labels.job }}" (instance {{ $labels.instance }}) and destination "{{ $labels.url }}" is saturated by more than 90% and vlagent won't be able to keep up.
+             There could be the following reasons for this:
+             * vlagent can't send data fast enough through the existing network connections. Increase `-remoteWrite.queues` cmd-line flag value to establish more connections per destination.
+             * remote destination can't accept data fast enough. Check if remote destination has enough resources for processing.
       - alert: PersistentQueueForWritesIsSaturated
         expr: rate(vm_persistentqueue_write_duration_seconds_total[5m]) > 0.9
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Persistent queue writes for instance {{ $labels.instance }} are saturated"
-          description: "Persistent queue writes for vlagent \"{{ $labels.job }}\" (instance {{ $labels.instance }})
-            are saturated by more than 90% and vlagent won't be able to keep up with flushing data on disk. 
-            In this case, consider to decrease load on the vlagent or improve the disk throughput."
-
+          summary: |-
+            Persistent queue writes for instance {{ $labels.instance }} are saturated
+          description: |-
+            Persistent queue writes for vlagent "{{ $labels.job }}" (instance {{ $labels.instance }}) are saturated by more than 90% and vlagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vlagent or improve the disk throughput.
       - alert: PersistentQueueForReadsIsSaturated
         expr: rate(vm_persistentqueue_read_duration_seconds_total[5m]) > 0.9
         for: 15m
         labels:
           severity: warning
         annotations:
-          summary: "Persistent queue reads for instance {{ $labels.instance }} are saturated"
-          description: "Persistent queue reads for vlagent \"{{ $labels.job }}\" (instance {{ $labels.instance }})
-            are saturated by more than 90% and vlagent won't be able to keep up with reading data from the disk. 
-            In this case, consider to decrease load on the vlagent or improve the disk throughput."
-
+          summary: |-
+            Persistent queue reads for instance {{ $labels.instance }} are saturated
+          description: |-
+            Persistent queue reads for vlagent "{{ $labels.job }}" (instance {{ $labels.instance }}) are saturated by more than 90% and vlagent won't be able to keep up with reading data from the disk. In this case, consider to decrease load on the vlagent or improve the disk throughput.
       - alert: PersistentQueueRunsOutOfSpaceIn12Hours
         expr: |
           (
             (sum(vm_persistentqueue_free_disk_space_bytes) by (job, instance, path)) /
             (sum(deriv(vm_persistentqueue_bytes_pending[1m])) by (job, instance, path) > 0)
-          ) * 
-          on(job, instance, path) group_left(url)
+          ) * on(job, instance, path) group_left(url)
           (vlagent_remotewrite_pending_data_bytes * 0 + 1) < 12 * 3600 >0
         for: 10m
         labels:
           severity: warning
         annotations:
-          summary: "Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 12 hours."
-          description: "RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. 
-            Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})."
-
+          summary: |-
+            Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 12 hours.
+          description: |-
+            RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }}).
       - alert: PersistentQueueRunsOutOfSpaceIn4Hours
-        expr: |
+        expr: |-
           (
             (sum(vm_persistentqueue_free_disk_space_bytes) by (job, instance, path)) /
             (sum(deriv(vm_persistentqueue_bytes_pending[1m])) by (job, instance, path) > 0)
-          ) * 
-          on(job, instance, path) group_left(url)
+          ) * on(job, instance, path) group_left(url)
           (vlagent_remotewrite_pending_data_bytes * 0 + 1) < 4 * 3600 >0
         for: 10m
         labels:
           severity: critical
         annotations:
-          summary: "Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 4 hours."
-          description: "RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. 
-            Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})."
+          summary: |-
+            Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 4 hours.
+          description: |-
+            RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }}).