Filter OTEL by environment

changes/47573-signoz-dashboards-deployment-environment

@@ -0,0 +1 @@
+* Updated the SigNoz OTEL dashboards under `tools/signoz/` to template and filter on the `deployment.environment` resource attribute, with the environment variable defaulting to `default`, so multiple Fleet environments reporting to the same SigNoz backend can be scoped per environment.

cmd/fleet/otel.go

@@ -7,6 +7,7 @@ import (
 	"github.com/fleetdm/fleet/v4/server/platform/tracing"
 	"github.com/fleetdm/fleet/v4/server/version"
 	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc"
 	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
@@ -36,12 +37,16 @@ func initOTELProviders(cfg config.FleetConfig, traceRegistry *tracing.Registry,
 	}
 
 	// Create shared resource with service identification attributes. OTEL_SERVICE_NAME and OTEL_RESOURCE_ATTRIBUTES env vars
-	// can override the defaults below.
+	// can override the defaults below. resource.WithFromEnv() runs after WithAttributes, so env-provided values win on conflict.
+	// We always emit a deployment.environment so the attribute key exists in every SigNoz instance Fleet reports to. This lets
+	// dashboards use a dynamic environment selector.
 	res, err := resource.New(context.Background(),
 		resource.WithSchemaURL(semconv.SchemaURL),
 		resource.WithAttributes(
 			semconv.ServiceName("fleet"),
 			semconv.ServiceVersion(version.Version().Version),
+			semconv.DeploymentEnvironmentName("default"),
+			attribute.String("deployment.environment", "default"), // 2026-06-14: deprecated attribute still used by SigNoz
 		),
 		resource.WithFromEnv(),
 		resource.WithTelemetrySDK(),

tools/signoz/README.md

@@ -91,3 +91,6 @@ JSON exports of Fleet-specific SigNoz dashboards live alongside this README. Imp
 
 - `database_custom_dashboard.json` — MySQL query metrics (RPS, latency, slow queries) derived from `db.sql.*` instrumentation.
 - `host_cache_dashboard.json` — Redis-backed host lookup cache (`LoadHostByNodeKey` / `LoadHostByOrbitNodeKey`). Shows hit rate over time, lookups/sec by result, errors/sec by op, and invalidations/sec by write-path reason. Requires `FLEET_REDIS_HOST_CACHE_ENABLED=true` (default on).
+- `http_errors_dashboard.json` — Fleet HTTP errors (the "Errors" signal of the RED method / Google SRE Golden Signals). Shows 4XX client errors and 5XX server errors from `fleet.http.client_errors` / `fleet.http.server_errors`, broken down by `error.type`.
+
+Each dashboard includes an `environment` selector (a dynamic dashboard variable on the `deployment.environment` resource attribute) so you can scope panels to a single deployment or view all. It defaults to ALL. Fleet always emits `deployment.environment` (default value `default`, overridable via `OTEL_RESOURCE_ATTRIBUTES`), so the selector populates on any instance Fleet reports to.