diff --git a/deployment/docker/Makefile b/deployment/docker/Makefile index 2337db910b..3c6bd26156 100644 --- a/deployment/docker/Makefile +++ b/deployment/docker/Makefile @@ -14,6 +14,8 @@ DOCKER_BUILD ?= $(DOCKER) build DOCKER_COMPOSE ?= $(DOCKER) compose DOCKER_IMAGE_LS ?= $(DOCKER) image ls --format '{{.Repository}}:{{.Tag}}' +include deployment/docker/rules/Makefile + package-builder: ($(DOCKER_IMAGE_LS) | grep -q '$(BUILDER_IMAGE)$$') \ || $(DOCKER_BUILD) \ diff --git a/deployment/docker/rules/Makefile b/deployment/docker/rules/Makefile new file mode 100644 index 0000000000..ca8a238bed --- /dev/null +++ b/deployment/docker/rules/Makefile @@ -0,0 +1,8 @@ + +RULES_DIR ?= ./deployment/docker/rules + +format-rules: + find $(RULES_DIR) -type f -name "*.yml" | while IFS= read -r f; do \ + echo "Processing $${f}"; \ + yq -i '.groups[].rules[].annotations.summary style="literal" | .groups[].rules[].annotations.description style="literal"' "$${f}" || exit 1; \ + done diff --git a/deployment/docker/rules/README.md b/deployment/docker/rules/README.md new file mode 100644 index 0000000000..25e9f031d8 --- /dev/null +++ b/deployment/docker/rules/README.md @@ -0,0 +1,25 @@ +# Alerting rules for VictoriaLogs components + +This folder contains alerting rules for VictoriaLogs components. + +## Best practice + +`annotations.summary` and `annotations.description` should use literal style, preferably `|-` for easier synchronization across Helm templates. + +Good example: + +```yaml + summary: |- + Instance {{ $labels.instance }} (job={{ $labels.job }}) will run out of disk space soon + description: |- + Disk utilisation on instance {{ $labels.instance }} is more than 80%. +``` + +Not recommended: + +```yaml + summary: "Instance \"{{ $labels.instance }}\" (job=\"{{ $labels.job }}\") will run out of disk space soon" + description: "Disk utilisation on instance \"{{ $labels.instance }}\" is more than 80%." +``` + +Run `make format-rules` in root folder to format `annotations.summary` and `annotations.description` fields. diff --git a/deployment/docker/rules/alerts-health.yml b/deployment/docker/rules/alerts-health.yml index 7e3b6ecabc..d9637760d6 100644 --- a/deployment/docker/rules/alerts-health.yml +++ b/deployment/docker/rules/alerts-health.yml @@ -11,104 +11,101 @@ groups: labels: severity: critical annotations: - summary: "{{ $labels.job }} too many restarts (instance {{ $labels.instance }})" - description: > - Job {{ $labels.job }} (instance {{ $labels.instance }}) has restarted more than twice in the last 15 minutes. - It might be crashlooping. - + summary: |- + {{ $labels.job }} too many restarts (instance {{ $labels.instance }}) + description: |- + Job {{ $labels.job }} (instance {{ $labels.instance }}) has restarted more than twice in the last 15 minutes. It might be crashlooping. - alert: ServiceDown expr: up{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"} == 0 for: 2m labels: severity: critical annotations: - summary: "Service {{ $labels.job }} is down on {{ $labels.instance }}" - description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes." - + summary: |- + Service {{ $labels.job }} is down on {{ $labels.instance }} + description: |- + {{ $labels.instance }} of job {{ $labels.job }} has been down for more than 2 minutes. - alert: ProcessNearFDLimits expr: (process_max_fds{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"} - process_open_fds{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"}) < 100 for: 5m labels: severity: critical annotations: - summary: "Number of free file descriptors is less than 100 for \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") for the last 5m" - description: | + summary: |- + Number of free file descriptors is less than 100 for "{{ $labels.job }}"("{{ $labels.instance }}") for the last 5m + description: |- Exhausting OS file descriptors limit can cause severe degradation of the process. Consider to increase the limit as fast as possible. - - alert: TooHighMemoryUsage expr: (min_over_time(process_resident_memory_anon_bytes{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"}[10m]) / vm_available_memory_bytes{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"}) > 0.8 for: 5m labels: severity: critical annotations: - summary: "It is more than 80% of memory used by \"{{ $labels.job }}\"(\"{{ $labels.instance }}\")" - description: | + summary: |- + It is more than 80% of memory used by "{{ $labels.job }}"("{{ $labels.instance }}") + description: |- Too high memory usage may result into multiple issues such as OOMs or degraded performance. Consider to either increase available memory or decrease the load on the process. - - alert: TooHighCPUUsage expr: rate(process_cpu_seconds_total{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"}[5m]) / process_cpu_cores_available{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"} > 0.9 for: 5m labels: severity: critical annotations: - summary: "More than 90% of CPU is used by \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") during the last 5m" - description: > + summary: |- + More than 90% of CPU is used by "{{ $labels.job }}"("{{ $labels.instance }}") during the last 5m + description: |- Too high CPU usage may be a sign of insufficient resources and make process unstable. Consider to either increase available CPU resources or decrease the load on the process. - - alert: TooHighGoroutineSchedulingLatency expr: histogram_quantile(0.99, sum(rate(go_sched_latencies_seconds_bucket{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*"}[5m])) by (le, job, instance)) > 0.1 for: 15m labels: severity: critical annotations: - summary: "\"{{ $labels.job }}\"(\"{{ $labels.instance }}\") has insufficient CPU resources for >15m" - description: > - Go runtime is unable to schedule goroutines execution in acceptable time. This is usually a sign of - insufficient CPU resources or CPU throttling. Verify that service has enough CPU resources. Otherwise, - the service could work unreliably with delays in processing. - + summary: |- + "{{ $labels.job }}"("{{ $labels.instance }}") has insufficient CPU resources for >15m + description: "Go runtime is unable to schedule goroutines execution in acceptable time. This is usually a sign of insufficient CPU resources or CPU throttling. \nVerify that service has enough CPU resources. Otherwise, the service could work unreliably with delays in processing." - alert: TooManyLogs expr: sum(increase(vm_log_messages_total{job=~".*(victorialogs|vlstorage|vlselect|vlinsert|vlagent).*", level="error"}[5m])) without (app_version, location) > 0 for: 15m labels: severity: warning annotations: - summary: "Too many logs printed for job \"{{ $labels.job }}\" ({{ $labels.instance }})" - description: > - Logging rate for job \"{{ $labels.job }}\" ({{ $labels.instance }}) is {{ $value }} for last 15m. - Worth to check logs for specific error messages. - + summary: |- + Too many logs printed for job "{{ $labels.job }}" ({{ $labels.instance }}) + description: |- + Logging rate for job "{{ $labels.job }}" ({{ $labels.instance }}) is {{ $value }} for last 15m. Worth to check logs for specific error messages. - alert: RequestErrorsToAPI expr: increase(vl_http_errors_total[5m]) > 0 for: 15m labels: severity: warning annotations: - summary: "Too many errors served for path {{ $labels.path }} (instance {{ $labels.instance }})" - description: "Requests to path {{ $labels.path }} are receiving errors. - Please verify if clients are sending correct requests." - + summary: |- + Too many errors served for path {{ $labels.path }} (instance {{ $labels.instance }}) + description: |- + Requests to path {{ $labels.path }} are receiving errors. Please verify if clients are sending correct requests. - alert: RowsRejectedOnIngestion expr: rate(vl_rows_dropped_total[5m]) > 0 for: 15m labels: severity: warning annotations: - summary: "Some rows are rejected on \"{{ $labels.instance }}\" on ingestion attempt" - description: "VictoriaLogs is rejecting to ingest rows on \"{{ $labels.instance }}\" due to the - following reason: \"{{ $labels.reason }}\"" - + summary: |- + Some rows are rejected on "{{ $labels.instance }}" on ingestion attempt + description: |- + VictoriaLogs is rejecting to ingest rows on "{{ $labels.instance }}" due to the following reason: "{{ $labels.reason }}" - alert: ConcurrentInsertsHitTheLimit expr: avg_over_time(vm_concurrent_insert_current{job=~".*(victorialogs|vlstorage|vlinsert|vlagent).*"}[1m]) >= vm_concurrent_insert_capacity{job=~".*(victorialogs|vlstorage|vlinsert|vlagent).*"} for: 15m labels: severity: warning annotations: - summary: "{{ $labels.job }} on instance {{ $labels.instance }} is constantly hitting concurrent inserts limit" - description: | + summary: |- + {{ $labels.job }} on instance {{ $labels.instance }} is constantly hitting concurrent inserts limit + description: |- The limit of concurrent inserts on instance {{ $labels.instance }} depends on the number of CPUs. Usually, when component constantly hits the limit it is likely the component is overloaded and requires more CPU. In some cases the alert might trigger if there are too many clients making write attempts to the component. diff --git a/deployment/docker/rules/alerts-vlagent.yml b/deployment/docker/rules/alerts-vlagent.yml index 9187b3f3ba..cdc0d8e4f9 100644 --- a/deployment/docker/rules/alerts-vlagent.yml +++ b/deployment/docker/rules/alerts-vlagent.yml @@ -10,93 +10,91 @@ groups: labels: severity: critical annotations: - summary: "Instance {{ $labels.instance }} is dropping data from persistent queue" - description: "vlagent dropped {{ $value | humanize1024 }} from persistent queue - on instance {{ $labels.instance }} for the last 10m." - + summary: |- + Instance {{ $labels.instance }} is dropping data from persistent queue + description: |- + vlagent dropped {{ $value | humanize1024 }} from persistent queue on instance {{ $labels.instance }} for the last 10m. - alert: RejectedRemoteWriteDataBlocksAreDropped expr: sum(increase(vlagent_remotewrite_packets_dropped_total[5m])) without (url) > 0 for: 15m labels: severity: warning annotations: - summary: "vlagent is dropping data blocks that are rejected by remote storage" - description: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} drops the rejected by - remote-write server data blocks. Check the logs to find the reason for rejects." - + summary: |- + vlagent is dropping data blocks that are rejected by remote storage + description: |- + Job "{{ $labels.job }}" on instance {{ $labels.instance }} drops the rejected by remote-write server data blocks. Check the logs to find the reason for rejects. - alert: TooManyRemoteWriteErrors expr: rate(vlagent_remotewrite_retries_count_total[5m]) > 0 for: 15m labels: severity: warning annotations: - summary: "Job \"{{ $labels.job }}\" on instance {{ $labels.instance }} fails to push to remote storage" - description: "vlagent fails to push data via remote write protocol to destination \"{{ $labels.url }}\"\n - Ensure that destination is up and reachable." - + summary: |- + Job "{{ $labels.job }}" on instance {{ $labels.instance }} fails to push to remote storage + description: |- + vlagent fails to push data via remote write protocol to destination "{{ $labels.url }}" + Ensure that destination is up and reachable. - alert: RemoteWriteConnectionIsSaturated expr: rate(vlagent_remotewrite_send_duration_seconds_total[5m]) / vlagent_remotewrite_queues > 0.9 for: 15m labels: severity: warning annotations: - summary: "Remote write connection from \"{{ $labels.job }}\" (instance {{ $labels.instance }}) to {{ $labels.url }} is saturated" - description: "The remote write connection between vlagent \"{{ $labels.job }}\" (instance {{ $labels.instance }}) and destination \"{{ $labels.url }}\" - is saturated by more than 90% and vlagent won't be able to keep up.\n - There could be the following reasons for this:\n - * vlagent can't send data fast enough through the existing network connections. Increase `-remoteWrite.queues` cmd-line flag value to establish more connections per destination.\n - * remote destination can't accept data fast enough. Check if remote destination has enough resources for processing." - + summary: |- + Remote write connection from "{{ $labels.job }}" (instance {{ $labels.instance }}) to {{ $labels.url }} is saturated + description: |- + The remote write connection between vlagent "{{ $labels.job }}" (instance {{ $labels.instance }}) and destination "{{ $labels.url }}" is saturated by more than 90% and vlagent won't be able to keep up. + There could be the following reasons for this: + * vlagent can't send data fast enough through the existing network connections. Increase `-remoteWrite.queues` cmd-line flag value to establish more connections per destination. + * remote destination can't accept data fast enough. Check if remote destination has enough resources for processing. - alert: PersistentQueueForWritesIsSaturated expr: rate(vm_persistentqueue_write_duration_seconds_total[5m]) > 0.9 for: 15m labels: severity: warning annotations: - summary: "Persistent queue writes for instance {{ $labels.instance }} are saturated" - description: "Persistent queue writes for vlagent \"{{ $labels.job }}\" (instance {{ $labels.instance }}) - are saturated by more than 90% and vlagent won't be able to keep up with flushing data on disk. - In this case, consider to decrease load on the vlagent or improve the disk throughput." - + summary: |- + Persistent queue writes for instance {{ $labels.instance }} are saturated + description: |- + Persistent queue writes for vlagent "{{ $labels.job }}" (instance {{ $labels.instance }}) are saturated by more than 90% and vlagent won't be able to keep up with flushing data on disk. In this case, consider to decrease load on the vlagent or improve the disk throughput. - alert: PersistentQueueForReadsIsSaturated expr: rate(vm_persistentqueue_read_duration_seconds_total[5m]) > 0.9 for: 15m labels: severity: warning annotations: - summary: "Persistent queue reads for instance {{ $labels.instance }} are saturated" - description: "Persistent queue reads for vlagent \"{{ $labels.job }}\" (instance {{ $labels.instance }}) - are saturated by more than 90% and vlagent won't be able to keep up with reading data from the disk. - In this case, consider to decrease load on the vlagent or improve the disk throughput." - + summary: |- + Persistent queue reads for instance {{ $labels.instance }} are saturated + description: |- + Persistent queue reads for vlagent "{{ $labels.job }}" (instance {{ $labels.instance }}) are saturated by more than 90% and vlagent won't be able to keep up with reading data from the disk. In this case, consider to decrease load on the vlagent or improve the disk throughput. - alert: PersistentQueueRunsOutOfSpaceIn12Hours - expr: | + expr: |- ( (sum(vm_persistentqueue_free_disk_space_bytes) by (job, instance, path)) / (sum(deriv(vm_persistentqueue_bytes_pending[1m])) by (job, instance, path) > 0) - ) * - on(job, instance, path) group_left(url) + ) * on(job, instance, path) group_left(url) (vlagent_remotewrite_pending_data_bytes * 0 + 1) < 12 * 3600 >0 for: 10m labels: severity: warning annotations: - summary: "Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 12 hours." - description: "RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. - Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})." - + summary: |- + Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 12 hours. + description: |- + RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }}). - alert: PersistentQueueRunsOutOfSpaceIn4Hours - expr: | + expr: |- ( (sum(vm_persistentqueue_free_disk_space_bytes) by (job, instance, path)) / (sum(deriv(vm_persistentqueue_bytes_pending[1m])) by (job, instance, path) > 0) - ) * - on(job, instance, path) group_left(url) + ) * on(job, instance, path) group_left(url) (vlagent_remotewrite_pending_data_bytes * 0 + 1) < 4 * 3600 >0 for: 10m labels: severity: critical annotations: - summary: "Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 4 hours." - description: "RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. - Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }})." + summary: |- + Persistent Queue (url {{ $labels.url }}) of {{ $labels.instance }} (job:{{ $labels.job }}) will run out of space in 4 hours. + description: |- + RemoteWrite destination ({{ $labels.url }}) is unavailable or unable to receive data in a timely manner, so the persistent queue size is growing. Once the available space is exhausted, some samples will be discarded and cause incident. Please check the health of remoteWrite destination ({{ $labels.url }}). diff --git a/deployment/docker/rules/alerts-vlogs.yml b/deployment/docker/rules/alerts-vlogs.yml index 723953e290..e7c4cae32c 100644 --- a/deployment/docker/rules/alerts-vlogs.yml +++ b/deployment/docker/rules/alerts-vlogs.yml @@ -5,7 +5,7 @@ groups: - name: vlogs rules: - alert: DiskRunsOutOfSpace - expr: | + expr: |- sum(vl_data_size_bytes) by(job, instance) / ( sum(vl_free_disk_space_bytes) by(job, instance) + @@ -15,34 +15,34 @@ groups: labels: severity: critical annotations: - summary: "Instance {{ $labels.instance }} (job={{ $labels.job }}) will run out of disk space soon" - description: "Disk utilisation on instance {{ $labels.instance }} is more than 80%.\n - Having less than 20% of free disk space could cripple merge processes and overall performance. - Consider to limit the ingestion rate, decrease retention or scale the disk space if possible." - - + summary: |- + Instance {{ $labels.instance }} (job={{ $labels.job }}) will run out of disk space soon + description: |- + Disk utilisation on instance {{ $labels.instance }} is more than 80%. + Having less than 20% of free disk space could cripple merge processes and overall performance. Consider to limit the ingestion rate, decrease retention or scale the disk space if possible. - alert: ConcurrentQueriesHitTheLimit expr: avg_over_time(vl_concurrent_select_current[1m]) >= vl_concurrent_select_capacity for: 15m labels: severity: warning annotations: - summary: "{{ $labels.job }} on instance {{ $labels.instance }} is constantly hitting concurrent query limit" - description: | + summary: |- + {{ $labels.job }} on instance {{ $labels.instance }} is constantly hitting concurrent query limit + description: |- The limit of concurrent queries on instance {{ $labels.instance }} is controlled by `-search.maxConcurrentRequests`. Usually, when the component constantly hits the limit it is likely overloaded and requires more CPU or more replicas. In some cases the alert might trigger if clients send too many expensive queries to the component. If the component's CPU usage and storage latency are at normal level, then it might be worth adjusting `-search.maxConcurrentRequests`. - - alert: TooHighQueryLoad expr: increase(vl_concurrent_select_limit_timeout_total[5m]) > 0 for: 15m labels: severity: warning annotations: - summary: "Read queries fail with timeout for {{ $labels.job }} on instance {{ $labels.instance }}" - description: | + summary: |- + Read queries fail with timeout for {{ $labels.job }} on instance {{ $labels.instance }} + description: |- Instance {{ $labels.instance }} ({{ $labels.job }}) is failing to serve read queries during last 15m. Concurrency limit `-search.maxConcurrentRequests` was reached on this instance and extra queries were put into the queue for `-search.maxQueueDuration` interval. But even after waiting in the queue these queries weren't served.