*: add keyspace observability labels

cmd/tidb-server/BUILD.bazel

@@ -19,6 +19,7 @@ go_library(
         "//pkg/keyspace",
         "//pkg/kv",
         "//pkg/metrics",
+        "//pkg/metrics/common",
         "//pkg/parser/mysql",
         "//pkg/parser/terror",
         "//pkg/parser/types",
@@ -72,6 +73,7 @@ go_library(
         "@com_github_prometheus_client_golang//prometheus/push",
         "@com_github_tikv_client_go_v2//tikv",
         "@com_github_tikv_client_go_v2//txnkv/transaction",
+        "@com_github_tikv_pd_client//constants",
         "@org_uber_go_automaxprocs//maxprocs",
         "@org_uber_go_zap//:zap",
     ],
@@ -107,7 +109,7 @@ go_test(
     srcs = ["main_test.go"],
     embed = [":tidb-server_lib"],
     flaky = True,
-    shard_count = 6,
+    shard_count = 9,
     deps = [
         "//pkg/config",
         "//pkg/config/deploymode",
@@ -117,6 +119,7 @@ go_test(
         "//pkg/sessionctx/variable",
         "//pkg/testkit/testsetup",
         "@com_github_stretchr_testify//require",
+        "@com_github_tikv_pd_client//constants",
         "@io_opencensus_go//stats/view",
         "@org_uber_go_goleak//:goleak",
     ],

cmd/tidb-server/main.go

@@ -44,6 +44,7 @@ import (
 	"github.com/pingcap/tidb/pkg/keyspace"
 	"github.com/pingcap/tidb/pkg/kv"
 	"github.com/pingcap/tidb/pkg/metrics"
+	metricscommon "github.com/pingcap/tidb/pkg/metrics/common"
 	"github.com/pingcap/tidb/pkg/parser/mysql"
 	"github.com/pingcap/tidb/pkg/parser/terror"
 	parsertypes "github.com/pingcap/tidb/pkg/parser/types"
@@ -92,6 +93,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/push"
 	"github.com/tikv/client-go/v2/tikv"
 	"github.com/tikv/client-go/v2/txnkv/transaction"
+	"github.com/tikv/pd/client/constants"
 	"go.uber.org/automaxprocs/maxprocs"
 	"go.uber.org/zap"
 )
@@ -314,6 +316,8 @@ func main() {
 	}
 
 	var standbyController server.StandbyController
+	activationKeyspaceID := constants.NullKeyspaceID
+	var activationMetadata map[string]string
 	if config.GetGlobalConfig().Standby.StandByMode {
 		standbyController = standby.NewLoadKeyspaceController()
 	}
@@ -330,11 +334,17 @@ func main() {
 		defer standbyController.EndStandby(err)
 		// need to validate config again in case of config change via standby
 		terror.MustNil(config.GetGlobalConfig().Valid())
+		if c, ok := standbyController.(*standby.LoadKeyspaceController); ok {
+			activationKeyspaceID = c.ActivationKeyspaceID()
+			activationMetadata = c.ActivationMetadata()
+		}
 	}
 
 	signal.SetupUSR1Handler()
 	err = registerStores()
 	terror.MustNil(err)
+	err = prepareKeyspaceObservability(activationKeyspaceID, activationMetadata)
+	terror.MustNil(err)
 	err = metricsutil.RegisterMetrics()
 	terror.MustNil(err)
 
@@ -1146,6 +1156,47 @@ func closeStmtSummary() {
 	}
 }
 
+const (
+	keyspaceIDMetricLabel   = "keyspace_id"
+	keyspaceNameMetricLabel = "keyspace_name"
+)
+
+func prepareKeyspaceObservability(keyspaceID uint32, metadata map[string]string) error {
+	cfg := config.GetGlobalConfig()
+	if !kerneltype.IsNextGen() || cfg.Store != config.StoreTypeTiKV {
+		return nil
+	}
+	metricscommon.SetConstLabels(keyspaceNameMetricLabel, cfg.KeyspaceName)
+	return prepareKeyspaceObservabilityWithMetadata(keyspaceID, metadata, cfg.KeyspaceName, deploymode.IsStarter())
+}
+
+func prepareKeyspaceObservabilityWithMetadata(keyspaceID uint32, metadata map[string]string, keyspaceName string, includeConfiguredFields bool) error {
+	resolvedValues := config.KeyspaceObservabilityValues{
+		MetricLabels: map[string]string{
+			keyspaceNameMetricLabel: keyspaceName,
+		},
+	}
+	if keyspaceID != constants.NullKeyspaceID {
+		resolvedValues.MetricLabels[keyspaceIDMetricLabel] = fmt.Sprint(keyspaceID)
+	}
+	if includeConfiguredFields {
+		copiedConfig := *config.GetGlobalConfig()
+		if err := copiedConfig.ResolveKeyspaceObservability(metadata); err != nil {
+			return err
+		}
+		configuredValues := copiedConfig.KeyspaceObservabilityValues.Clone()
+		for k, v := range configuredValues.MetricLabels {
+			resolvedValues.MetricLabels[k] = v
+		}
+		resolvedValues.SlowLogFields = configuredValues.SlowLogFields
+		resolvedValues.StmtLogFields = configuredValues.StmtLogFields
+	}
+	config.UpdateGlobal(func(conf *config.Config) {
+		conf.KeyspaceObservabilityValues = resolvedValues
+	})
+	return nil
+}
+
 func enablePyroscope() {
 	if os.Getenv("PYROSCOPE_SERVER_ADDRESS") != "" {
 		runtime.SetMutexProfileFraction(5)

cmd/tidb-server/main_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/pingcap/tidb/pkg/sessionctx/variable"
 	"github.com/pingcap/tidb/pkg/testkit/testsetup"
 	"github.com/stretchr/testify/require"
+	"github.com/tikv/pd/client/constants"
 	"go.opencensus.io/stats/view"
 	"go.uber.org/goleak"
 )
@@ -154,3 +155,61 @@ func TestSetVersionByConfigNormalizeLegacyPlaceholderForNextGen(t *testing.T) {
 	require.Equal(t, "v26.3.0", mysql.TiDBReleaseVersion)
 	require.Equal(t, "8.0.11-TiDB-CLOUD.202603.0", mysql.ServerVersion)
 }
+
+func TestSetupKeyspaceObservabilityForStarter(t *testing.T) {
+	restore := config.RestoreFunc()
+	defer restore()
+	config.UpdateGlobal(func(conf *config.Config) {
+		conf.KeyspaceObservability = config.KeyspaceObservability{
+			Fields: []config.KeyspaceObservabilityField{{
+				Source:       "meta_a",
+				MetricLabel:  "keyspace_meta_label_a",
+				SlowLogField: "Slow_meta_a",
+				StmtLogField: "stmt_meta_a",
+				Required:     true,
+			}},
+		}
+	})
+
+	err := prepareKeyspaceObservabilityWithMetadata(42, map[string]string{
+		"meta_a": "value_a",
+	}, "ks", true)
+	require.NoError(t, err)
+
+	cfg := config.GetGlobalConfig()
+	require.Equal(t, map[string]string{"keyspace_id": "42", "keyspace_name": "ks", "keyspace_meta_label_a": "value_a"}, cfg.GetKeyspaceObservabilityMetricLabels())
+	require.Equal(t, []config.KeyspaceObservabilityFieldPair{{Key: "Slow_meta_a", Value: "value_a"}}, cfg.GetKeyspaceObservabilitySlowLogFields())
+	require.Equal(t, []config.KeyspaceObservabilityFieldPair{{Key: "stmt_meta_a", Value: "value_a"}}, cfg.GetKeyspaceObservabilityStmtLogFields())
+}
+
+func TestSetupKeyspaceObservabilityForNonStarter(t *testing.T) {
+	restore := config.RestoreFunc()
+	defer restore()
+
+	err := prepareKeyspaceObservabilityWithMetadata(42, map[string]string{
+		"meta_a": "value_a",
+	}, "ks", false)
+	require.NoError(t, err)
+
+	cfg := config.GetGlobalConfig()
+	require.Equal(t, map[string]string{"keyspace_id": "42", "keyspace_name": "ks"}, cfg.GetKeyspaceObservabilityMetricLabels())
+	require.Empty(t, cfg.GetKeyspaceObservabilitySlowLogFields())
+	require.Empty(t, cfg.GetKeyspaceObservabilityStmtLogFields())
+}
+
+func TestSetupKeyspaceObservabilityForStartSkipsClassic(t *testing.T) {
+	if !kerneltype.IsClassic() {
+		t.Skip("only verifies the classic-mode short-circuit path")
+	}
+
+	restore := config.RestoreFunc()
+	defer restore()
+	config.UpdateGlobal(func(conf *config.Config) {
+		conf.Store = config.StoreTypeTiKV
+		conf.Path = "invalid-pd-path"
+		conf.KeyspaceName = "test_keyspace"
+	})
+
+	require.NoError(t, prepareKeyspaceObservability(constants.NullKeyspaceID, nil))
+	require.Empty(t, config.GetGlobalConfig().GetKeyspaceObservabilityMetricLabels())
+}

pkg/config/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
         "config.go",
         "config_util.go",
         "const.go",
+        "keyspace_observability.go",
         "store.go",
         "tiflash.go",
     ],
@@ -42,7 +43,7 @@ go_test(
     data = glob(["**"]),
     embed = [":config"],
     flaky = True,
-    shard_count = 32,
+    shard_count = 34,
     deps = [
         "//pkg/config/deploymode",
         "//pkg/config/kerneltype",

pkg/config/config.go

@@ -262,6 +262,9 @@ type Config struct {
 	// key will be the default value of the session variable `txn_scope` for this tidb-server.
 	Labels map[string]string `toml:"labels" json:"labels"`
 
+	KeyspaceObservability       KeyspaceObservability       `toml:"keyspace-observability" json:"keyspace-observability"`
+	KeyspaceObservabilityValues KeyspaceObservabilityValues `toml:"-" json:"-"`
+
 	// EnableGlobalIndex is deprecated.
 	EnableGlobalIndex bool `toml:"enable-global-index" json:"enable-global-index"`
 
@@ -1473,12 +1476,18 @@ func (c *Config) Valid() error {
 	if !kerneltype.IsNextGen() && c.DeployMode != deploymode.Premium {
 		return fmt.Errorf("deploy-mode can only be configured for nextgen TiDB")
 	}
+	if len(c.KeyspaceObservability.Fields) > 0 && c.DeployMode != deploymode.Starter {
+		return fmt.Errorf("keyspace-observability.fields can only be configured when deploy-mode is starter")
+	}
 	if c.DXFResourceLimit < MinDXFResourceLimit || c.DXFResourceLimit > MaxDXFResourceLimit {
 		return fmt.Errorf("dxf-resource-limit should be between %d and %d", MinDXFResourceLimit, MaxDXFResourceLimit)
 	}
 	if c.DXFResourceLimit != DefDXFResourceLimit && c.DeployMode != deploymode.PremiumReserved {
 		return fmt.Errorf("dxf-resource-limit can only be configured when deploy-mode is premium_reserved")
 	}
+	if err := c.KeyspaceObservability.Valid(); err != nil {
+		return err
+	}
 	if c.Store == StoreTypeMockTiKV && !c.Instance.TiDBEnableDDL.Load() {
 		return fmt.Errorf("can't disable DDL on mocktikv")
 	}

pkg/config/config.toml.example

@@ -480,6 +480,15 @@ tikv-raftstore-store-write-trigger-wb-bytes = 0.00006100
 tikv-storage-processed-keys-batch-get = 0.00266791
 tikv-storage-processed-keys-get = 0.01416829
 
+# Map selected keyspace metadata entries to observability outputs.
+# Only valid when deploy-mode is starter.
+# [[keyspace-observability.fields]]
+# source = "meta_key"
+# metric-label = "metric_label"
+# slow-log-field = "Slow_log_field"
+# stmt-log-field = "stmt_log_field"
+# required = false
+
 # instance scope variables
 # These options are also available as a system variable for online configuration
 # changes to the system variable do not persist to the cluster. You must make changes

pkg/config/config.toml.nextgen.example

@@ -446,6 +446,15 @@ allow-expression-index = false
 # engines means allow the tidb server read data from which types of engines. options: "tikv", "tiflash", "tidb".
 engines = ["tikv", "tiflash", "tidb"]
 
+# Map selected keyspace metadata entries to observability outputs.
+# Only valid when deploy-mode is starter.
+# [[keyspace-observability.fields]]
+# source = "meta_key"
+# metric-label = "metric_label"
+# slow-log-field = "Slow_log_field"
+# stmt-log-field = "stmt_log_field"
+# required = false
+
 # instance scope variables
 # These options are also available as a system variable for online configuration
 # changes to the system variable do not persist to the cluster. You must make changes