Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions pkg/sessionctx/vardef/tidb_vars.go
Original file line number Diff line number Diff line change
Expand Up @@ -721,6 +721,11 @@ const (
// TiDBStmtSummaryMaxSQLLength indicates the max length of displayed normalized sql and sample sql.
TiDBStmtSummaryMaxSQLLength = "tidb_stmt_summary_max_sql_length"

// TiDBStmtSummaryGroupByUser, when enabled, adds the executing user to the
// statement summary grouping key so the same digest run by different users
// produces separate rows. Off by default to avoid cardinality growth.
TiDBStmtSummaryGroupByUser = "tidb_stmt_summary_group_by_user"

// TiDBIgnoreInlistPlanDigest enables TiDB to generate the same plan digest with SQL using different in-list arguments.
TiDBIgnoreInlistPlanDigest = "tidb_ignore_inlist_plan_digest"

Expand Down Expand Up @@ -1622,6 +1627,7 @@ const (
DefTiDBStmtSummaryHistorySize = 24
DefTiDBStmtSummaryMaxStmtCount = 3000
DefTiDBStmtSummaryMaxSQLLength = 32768
DefTiDBStmtSummaryGroupByUser = false
DefTiDBCapturePlanBaseline = Off
DefTiDBIgnoreInlistPlanDigest = true
DefTiDBEnableIndexMerge = true
Expand Down
4 changes: 4 additions & 0 deletions pkg/sessionctx/variable/sysvar.go
Original file line number Diff line number Diff line change
Expand Up @@ -953,6 +953,10 @@ var defaultSysVars = []*SysVar{
SetGlobal: func(_ context.Context, s *SessionVars, val string) error {
return stmtsummaryv2.SetMaxSQLLength(TidbOptInt(val, vardef.DefTiDBStmtSummaryMaxSQLLength))
}},
{Scope: vardef.ScopeGlobal, Name: vardef.TiDBStmtSummaryGroupByUser, Value: BoolToOnOff(vardef.DefTiDBStmtSummaryGroupByUser), Type: vardef.TypeBool, AllowEmpty: true,
SetGlobal: func(_ context.Context, s *SessionVars, val string) error {
return stmtsummaryv2.SetGroupByUser(TiDBOptOn(val))
}},
{Scope: vardef.ScopeGlobal, Name: vardef.TiDBCapturePlanBaseline, Value: vardef.DefTiDBCapturePlanBaseline, Type: vardef.TypeBool, AllowEmptyAll: true},
{Scope: vardef.ScopeGlobal, Name: vardef.TiDBEvolvePlanTaskMaxTime, Value: strconv.Itoa(vardef.DefTiDBEvolvePlanTaskMaxTime), Type: vardef.TypeInt, MinValue: -1, MaxValue: math.MaxInt64},
{Scope: vardef.ScopeGlobal, Name: vardef.TiDBEvolvePlanTaskStartTime, Value: vardef.DefTiDBEvolvePlanTaskStartTime, Type: vardef.TypeTime},
Expand Down
2 changes: 1 addition & 1 deletion pkg/util/stmtsummary/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ go_test(
],
embed = [":stmtsummary"],
flaky = True,
shard_count = 26,
shard_count = 28,
deps = [
"//pkg/meta/model",
"//pkg/metrics",
Expand Down
4 changes: 2 additions & 2 deletions pkg/util/stmtsummary/evicted_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func newInduceSsbde(beginTime int64, endTime int64) *stmtSummaryByDigestElement
// generate new StmtDigestKey and stmtSummaryByDigest
func generateStmtSummaryByDigestKeyValue(schema string, beginTime int64, endTime int64) (*StmtDigestKey, *stmtSummaryByDigest) {
key := &StmtDigestKey{}
key.Init(schema, "", "", "", "")
key.Init(schema, "", "", "", "", "")
value := newInduceSsbd(beginTime, endTime)
return key, value
}
Expand Down Expand Up @@ -191,7 +191,7 @@ func TestSimpleStmtSummaryByDigestEvicted(t *testing.T) {
require.Equal(t, "{begin: 8, end: 9, count: 1}, {begin: 5, end: 6, count: 1}, {begin: 2, end: 3, count: 1}", getAllEvicted(ssbde))

evictedKey = &StmtDigestKey{}
evictedKey.Init("b", "", "", "", "")
evictedKey.Init("b", "", "", "", "", "")
ssbde.AddEvicted(evictedKey, evictedValue, 4)
require.Equal(t, "{begin: 8, end: 9, count: 2}, {begin: 5, end: 6, count: 2}, {begin: 2, end: 3, count: 2}, {begin: 1, end: 2, count: 1}", getAllEvicted(ssbde))

Expand Down
59 changes: 55 additions & 4 deletions pkg/util/stmtsummary/statement_summary.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"bytes"
"cmp"
"container/list"
"encoding/binary"
"fmt"
"math"
"slices"
Expand Down Expand Up @@ -53,8 +54,16 @@ type StmtDigestKey struct {
}

// Init initialize the hash key.
func (key *StmtDigestKey) Init(schemaName, digest, prevDigest, planDigest, resourceGroupName string) {
length := len(schemaName) + len(digest) + len(prevDigest) + len(planDigest) + len(resourceGroupName)
// When user is empty (group_by_user disabled), the hash is byte-identical to
// the pre-user-dimension layout. When user is non-empty, the hash appends a
// length-prefixed user segment after resourceGroupName so the boundary is
// unambiguous and pairs like ("rg", "alice") and ("rga", "lice") cannot
// collide.
func (key *StmtDigestKey) Init(schemaName, digest, prevDigest, planDigest, resourceGroupName, user string) {
length := len(schemaName) + len(digest) + len(prevDigest) + len(planDigest) + len(resourceGroupName) + len(user)
if len(user) > 0 {
length += 4
}
if cap(key.hash) < length {
key.hash = make([]byte, 0, length)
} else {
Expand All @@ -65,6 +74,12 @@ func (key *StmtDigestKey) Init(schemaName, digest, prevDigest, planDigest, resou
key.hash = append(key.hash, hack.Slice(prevDigest)...)
key.hash = append(key.hash, hack.Slice(planDigest)...)
key.hash = append(key.hash, hack.Slice(resourceGroupName)...)
if len(user) > 0 {
var buf [4]byte
binary.BigEndian.PutUint32(buf[:], uint32(len(user)))
key.hash = append(key.hash, buf[:]...)
key.hash = append(key.hash, hack.Slice(user)...)
}
}

// Hash implements SimpleLRUCache.Key.
Expand All @@ -90,6 +105,7 @@ type stmtSummaryByDigestMap struct {
optRefreshInterval *atomic2.Int64
optHistorySize *atomic2.Int32
optMaxSQLLength *atomic2.Int32
optGroupByUser *atomic2.Bool

// other stores summary of evicted data.
other *stmtSummaryByDigestEvicted
Expand Down Expand Up @@ -322,6 +338,7 @@ func newStmtSummaryByDigestMap() *stmtSummaryByDigestMap {
optRefreshInterval: atomic2.NewInt64(1800),
optHistorySize: atomic2.NewInt32(24),
optMaxSQLLength: atomic2.NewInt32(32768),
optGroupByUser: atomic2.NewBool(false),
other: ssbde,
}
newSsMap.summaryMap.SetOnEvict(func(k kvcache.Key, v kvcache.Value) {
Expand Down Expand Up @@ -355,8 +372,6 @@ func (ssMap *stmtSummaryByDigestMap) AddStatement(sei *StmtExecInfo) {
}

key := StmtDigestKeyPool.Get().(*StmtDigestKey)
// Init hash value in advance, to reduce the time holding the lock.
key.Init(sei.SchemaName, sei.Digest, sei.PrevSQLDigest, sei.PlanDigest, sei.ResourceGroupName)

var exist bool

Expand All @@ -370,6 +385,15 @@ func (ssMap *stmtSummaryByDigestMap) AddStatement(sei *StmtExecInfo) {
ssMap.Lock()
defer ssMap.Unlock()

// Decide userForKey under the lock so SetGroupByUser's flag flip + Clear
// is atomic w.r.t. AddStatement; otherwise a post-clear insert could land
// under the wrong grouping mode.
userForKey := ""
if ssMap.optGroupByUser.Load() {
userForKey = sei.User
}
key.Init(sei.SchemaName, sei.Digest, sei.PrevSQLDigest, sei.PlanDigest, sei.ResourceGroupName, userForKey)

// Check again. Statements could be added before disabling the flag and after Clear().
if !ssMap.Enabled() {
return
Expand Down Expand Up @@ -411,6 +435,11 @@ func (ssMap *stmtSummaryByDigestMap) Clear() {
ssMap.Lock()
defer ssMap.Unlock()

ssMap.clearLocked()
}

// clearLocked removes all statement summaries. ssMap.Lock must be held.
func (ssMap *stmtSummaryByDigestMap) clearLocked() {
ssMap.summaryMap.DeleteAll()
ssMap.other.Clear()
ssMap.beginTimeForCurInterval = 0
Expand Down Expand Up @@ -518,6 +547,28 @@ func (ssMap *stmtSummaryByDigestMap) historySize() int {
return int(ssMap.optHistorySize.Load())
}

// SetGroupByUser enables or disables grouping statement summaries by the
// executing user. Switching the flag clears existing data because existing
// rows were aggregated under a different grouping key.
func (ssMap *stmtSummaryByDigestMap) SetGroupByUser(value bool) error {
// Hold ssMap.Lock across the flag flip and clear so AddStatement (which
// reads the flag under the same lock) cannot insert a record with the
// old grouping mode after Clear() completes.
ssMap.Lock()
defer ssMap.Unlock()
if ssMap.optGroupByUser.Load() == value {
return nil
}
ssMap.optGroupByUser.Store(value)
ssMap.clearLocked()
return nil
}

// GroupByUser reports whether statement summaries are grouped by user.
func (ssMap *stmtSummaryByDigestMap) GroupByUser() bool {
return ssMap.optGroupByUser.Load()
}

// SetHistorySize sets the history size for all summaries.
func (ssMap *stmtSummaryByDigestMap) SetMaxStmtCount(value uint) error {
// `optMaxStmtCount` and `ssMap` don't need to be strictly atomically updated.
Expand Down
104 changes: 91 additions & 13 deletions pkg/util/stmtsummary/statement_summary_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func TestAddStatement(t *testing.T) {
stmtExecInfo1 := generateAnyExecInfo()
stmtExecInfo1.ExecDetail.CommitDetail.Mu.PrewriteBackoffTypes = make([]string, 0)
key := &StmtDigestKey{}
key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, "", stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName)
key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, "", stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName, "")
samplePlan, _, _ := stmtExecInfo1.LazyInfo.GetEncodedPlan()
stmtExecInfo1.ExecDetail.CommitDetail.Mu.Lock()
expectedSummaryElement := stmtSummaryByDigestElement{
Expand Down Expand Up @@ -502,7 +502,7 @@ func TestAddStatement(t *testing.T) {
stmtExecInfo4.SchemaName = "schema2"
stmtExecInfo4.ExecDetail.CommitDetail = nil
key = &StmtDigestKey{}
key.Init(stmtExecInfo4.SchemaName, stmtExecInfo4.Digest, "", stmtExecInfo4.PlanDigest, stmtExecInfo4.ResourceGroupName)
key.Init(stmtExecInfo4.SchemaName, stmtExecInfo4.Digest, "", stmtExecInfo4.PlanDigest, stmtExecInfo4.ResourceGroupName, "")
ssMap.AddStatement(stmtExecInfo4)
require.Equal(t, 2, ssMap.summaryMap.Size())
_, ok = ssMap.summaryMap.Get(key)
Expand All @@ -512,7 +512,7 @@ func TestAddStatement(t *testing.T) {
stmtExecInfo5 := stmtExecInfo1
stmtExecInfo5.Digest = "digest2"
key = &StmtDigestKey{}
key.Init(stmtExecInfo5.SchemaName, stmtExecInfo5.Digest, "", stmtExecInfo5.PlanDigest, stmtExecInfo5.ResourceGroupName)
key.Init(stmtExecInfo5.SchemaName, stmtExecInfo5.Digest, "", stmtExecInfo5.PlanDigest, stmtExecInfo5.ResourceGroupName, "")
ssMap.AddStatement(stmtExecInfo5)
require.Equal(t, 3, ssMap.summaryMap.Size())
_, ok = ssMap.summaryMap.Get(key)
Expand All @@ -522,7 +522,7 @@ func TestAddStatement(t *testing.T) {
stmtExecInfo6 := stmtExecInfo1
stmtExecInfo6.PlanDigest = "plan_digest2"
key = &StmtDigestKey{}
key.Init(stmtExecInfo6.SchemaName, stmtExecInfo6.Digest, "", stmtExecInfo6.PlanDigest, stmtExecInfo6.ResourceGroupName)
key.Init(stmtExecInfo6.SchemaName, stmtExecInfo6.Digest, "", stmtExecInfo6.PlanDigest, stmtExecInfo6.ResourceGroupName, "")
ssMap.AddStatement(stmtExecInfo6)
require.Equal(t, 4, ssMap.summaryMap.Size())
_, ok = ssMap.summaryMap.Get(key)
Expand All @@ -545,7 +545,7 @@ func TestAddStatement(t *testing.T) {
bindingSQL: originalSQL,
}
key = &StmtDigestKey{}
key.Init(stmtExecInfo7.SchemaName, stmtExecInfo7.Digest, "", stmtExecInfo7.PlanDigest, stmtExecInfo7.ResourceGroupName)
key.Init(stmtExecInfo7.SchemaName, stmtExecInfo7.Digest, "", stmtExecInfo7.PlanDigest, stmtExecInfo7.ResourceGroupName, "")
ssMap.AddStatement(stmtExecInfo7)
require.Equal(t, 5, ssMap.summaryMap.Size())
v, ok := ssMap.summaryMap.Get(key)
Expand Down Expand Up @@ -1122,7 +1122,7 @@ func TestMaxStmtCount(t *testing.T) {
// LRU cache should work.
for i := loops - 10; i < loops; i++ {
key := &StmtDigestKey{}
key.Init(stmtExecInfo1.SchemaName, fmt.Sprintf("digest%d", i), "", stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName)
key.Init(stmtExecInfo1.SchemaName, fmt.Sprintf("digest%d", i), "", stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName, "")
key.Hash()
_, ok := sm.Get(key)
require.True(t, ok)
Expand Down Expand Up @@ -1166,7 +1166,7 @@ func TestMaxSQLLength(t *testing.T) {
ssMap.AddStatement(stmtExecInfo1)

key := &StmtDigestKey{}
key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, "", stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName)
key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, "", stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName, "")
value, ok := ssMap.summaryMap.Get(key)
require.True(t, ok)

Expand Down Expand Up @@ -1418,7 +1418,7 @@ func TestRefreshCurrentSummary(t *testing.T) {
ssMap.beginTimeForCurInterval = now + 10
stmtExecInfo1 := generateAnyExecInfo()
key := &StmtDigestKey{}
key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, "", stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName)
key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, "", stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName, "")
ssMap.AddStatement(stmtExecInfo1)
require.Equal(t, 1, ssMap.summaryMap.Size())
value, ok := ssMap.summaryMap.Get(key)
Expand Down Expand Up @@ -1465,7 +1465,7 @@ func TestSummaryHistory(t *testing.T) {

stmtExecInfo1 := generateAnyExecInfo()
key := &StmtDigestKey{}
key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, "", stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName)
key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, "", stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName, "")
for i := range 11 {
ssMap.beginTimeForCurInterval = now + int64(i+1)*10
ssMap.AddStatement(stmtExecInfo1)
Expand Down Expand Up @@ -1534,7 +1534,7 @@ func TestPrevSQL(t *testing.T) {
stmtExecInfo1.PrevSQLDigest = "prevSQLDigest"
ssMap.AddStatement(stmtExecInfo1)
key := &StmtDigestKey{}
key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, stmtExecInfo1.PrevSQLDigest, stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName)
key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, stmtExecInfo1.PrevSQLDigest, stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName, "")
require.Equal(t, 1, ssMap.summaryMap.Size())
_, ok := ssMap.summaryMap.Get(key)
require.True(t, ok)
Expand All @@ -1549,7 +1549,7 @@ func TestPrevSQL(t *testing.T) {
stmtExecInfo2.PrevSQLDigest = "prevSQLDigest1"
ssMap.AddStatement(stmtExecInfo2)
require.Equal(t, 2, ssMap.summaryMap.Size())
key.Init(stmtExecInfo2.SchemaName, stmtExecInfo2.Digest, stmtExecInfo2.PrevSQLDigest, stmtExecInfo2.PlanDigest, stmtExecInfo2.ResourceGroupName)
key.Init(stmtExecInfo2.SchemaName, stmtExecInfo2.Digest, stmtExecInfo2.PrevSQLDigest, stmtExecInfo2.PlanDigest, stmtExecInfo2.ResourceGroupName, "")
_, ok = ssMap.summaryMap.Get(key)
require.True(t, ok)
}
Expand All @@ -1562,7 +1562,7 @@ func TestEndTime(t *testing.T) {
stmtExecInfo1 := generateAnyExecInfo()
ssMap.AddStatement(stmtExecInfo1)
key := &StmtDigestKey{}
key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, "", stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName)
key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, "", stmtExecInfo1.PlanDigest, stmtExecInfo1.ResourceGroupName, "")
require.Equal(t, 1, ssMap.summaryMap.Size())
value, ok := ssMap.summaryMap.Get(key)
require.True(t, ok)
Expand Down Expand Up @@ -1608,7 +1608,7 @@ func TestPointGet(t *testing.T) {
stmtExecInfo1.LazyInfo.(*mockLazyInfo).plan = fakePlanDigestGenerator()
ssMap.AddStatement(stmtExecInfo1)
key := &StmtDigestKey{}
key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, "", "", stmtExecInfo1.ResourceGroupName)
key.Init(stmtExecInfo1.SchemaName, stmtExecInfo1.Digest, "", "", stmtExecInfo1.ResourceGroupName, "")
require.Equal(t, 1, ssMap.summaryMap.Size())
value, ok := ssMap.summaryMap.Get(key)
require.True(t, ok)
Expand Down Expand Up @@ -1687,3 +1687,81 @@ func TestAccessPrivilege(t *testing.T) {
datums = reader.GetStmtSummaryCurrentRows()
require.Len(t, datums, loops)
}

// TestAddStatementGroupByUser verifies that flipping the group-by-user flag
// splits the same digest into per-user rows and fills ssbd.user. The default
// (flag OFF) keeps legacy behavior: one row per digest regardless of user.
func TestAddStatementGroupByUser(t *testing.T) {
ssMap := newStmtSummaryByDigestMap()

info1 := generateAnyExecInfo()
info1.User = "alice"
info2 := generateAnyExecInfo()
info2.User = "bob"

// Flag off: both statements collapse into one record.
ssMap.AddStatement(info1)
ssMap.AddStatement(info2)
require.Equal(t, 1, ssMap.summaryMap.Size())

// Flipping the flag clears prior data (different grouping key).
require.NoError(t, ssMap.SetGroupByUser(true))
require.Equal(t, 0, ssMap.summaryMap.Size())

ssMap.AddStatement(info1)
ssMap.AddStatement(info2)
ssMap.AddStatement(info1)
require.Equal(t, 2, ssMap.summaryMap.Size())

// With grouping ON, each record's authUsers must hold exactly one user —
// the one that groups it — so SAMPLE_USER naturally reflects the grouping
// dimension without a dedicated column.
seen := map[string]bool{}
for _, v := range ssMap.summaryMap.Values() {
ssbd := v.(*stmtSummaryByDigest)
elem := ssbd.history.Front().Value.(*stmtSummaryByDigestElement)
require.Len(t, elem.authUsers, 1)
for u := range elem.authUsers {
seen[u] = true
}
}
require.True(t, seen["alice"])
require.True(t, seen["bob"])

// Flipping back off clears again, and re-emitted records merge users.
require.NoError(t, ssMap.SetGroupByUser(false))
require.Equal(t, 0, ssMap.summaryMap.Size())
ssMap.AddStatement(info1)
ssMap.AddStatement(info2)
require.Equal(t, 1, ssMap.summaryMap.Size())
for _, v := range ssMap.summaryMap.Values() {
ssbd := v.(*stmtSummaryByDigest)
elem := ssbd.history.Front().Value.(*stmtSummaryByDigestElement)
require.Len(t, elem.authUsers, 2)
}
}

// TestStmtDigestKeyBoundary guards against two regressions:
// 1. Adjacent string fields must not collide across boundary, e.g.
// (resourceGroupName, user) = ("rg", "alice") vs ("rga", "lice"); without
// a boundary marker on user, both produce the same hash.
// 2. With user empty (group_by_user OFF), the hash must stay byte-identical
// to the pre-user-dimension encoding so persisted/in-memory rows from
// older versions match.
func TestStmtDigestKeyBoundary(t *testing.T) {
k1 := &StmtDigestKey{}
k1.Init("schema", "digest", "prev", "plan", "rg", "alice")
k2 := &StmtDigestKey{}
k2.Init("schema", "digest", "prev", "plan", "rga", "lice")
require.NotEqual(t, k1.Hash(), k2.Hash(), "user segment must have an unambiguous boundary")

// user="" leaves the hash equal to the legacy 5-field layout.
off := &StmtDigestKey{}
off.Init("schema", "digest", "prev", "plan", "rg", "")
legacy := append([]byte{}, hack.Slice("digest")...)
legacy = append(legacy, hack.Slice("schema")...)
legacy = append(legacy, hack.Slice("prev")...)
legacy = append(legacy, hack.Slice("plan")...)
legacy = append(legacy, hack.Slice("rg")...)
require.Equal(t, legacy, off.Hash())
}
2 changes: 1 addition & 1 deletion pkg/util/stmtsummary/v2/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ go_test(
],
embed = [":stmtsummary"],
flaky = True,
shard_count = 15,
shard_count = 16,
deps = [
"//pkg/meta/model",
"//pkg/metrics",
Expand Down
Loading
Loading