Skip to content

metrics: expose IA remote-read scan details #69606

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Connor1996 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

metrics: expose IA remote-read scan details #69606

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions pkg/executor/slow_query.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1124,7 +1124,8 @@ func getColumnValueFactoryByName(colName string, columnIdx int) (slowQueryColumn
execdetails.RequestCountStr, execdetails.TotalKeysStr, execdetails.ProcessKeysStr,
execdetails.RocksdbDeleteSkippedCountStr, execdetails.RocksdbKeySkippedCountStr,
execdetails.RocksdbBlockCacheHitCountStr, execdetails.RocksdbBlockReadCountStr,
variable.SlowLogTxnStartTSStr, execdetails.RocksdbBlockReadByteStr:
variable.SlowLogTxnStartTSStr, execdetails.RocksdbBlockReadByteStr,
execdetails.IARemoteReadSegmentCountStr, execdetails.IARemoteReadSegmentSizeStr:
return func(row []types.Datum, value string, _ *time.Location, _ *slowLogChecker) (valid bool, err error) {
v, err := strconv.ParseUint(value, 10, 64)
if err != nil {
Expand All @@ -1143,7 +1144,7 @@ func getColumnValueFactoryByName(colName string, columnIdx int) (slowQueryColumn
variable.SlowLogCopWaitAvg, variable.SlowLogCopWaitP90, variable.SlowLogCopWaitMax, variable.SlowLogKVTotal,
variable.SlowLogPDTotal, variable.SlowLogBackoffTotal, variable.SlowLogWriteSQLRespTotal, variable.SlowLogRRU,
variable.SlowLogWRU, variable.SlowLogWaitRUDuration, variable.SlowLogTidbCPUUsageDuration, variable.SlowLogTikvCPUUsageDuration,
variable.SlowLogMemArbitration, variable.SlowLogRequestUnitV2:
variable.SlowLogMemArbitration, execdetails.IARemoteReadSegmentWaitTimeStr, variable.SlowLogRequestUnitV2:
return func(row []types.Datum, value string, _ *time.Location, _ *slowLogChecker) (valid bool, err error) {
v, err := strconv.ParseFloat(value, 64)
if err != nil {
Expand Down
11 changes: 7 additions & 4 deletions pkg/executor/slow_query_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,9 @@ func TestParseSlowLogFile(t *testing.T) {
# Query_time: 0.216905
# Cop_time: 0.38 Process_time: 0.021 Request_count: 1 Total_keys: 637 Processed_keys: 436
# Rocksdb_delete_skipped_count: 10 Rocksdb_key_skipped_count: 10 Rocksdb_block_cache_hit_count: 10 Rocksdb_block_read_count: 10 Rocksdb_block_read_byte: 100
# IA_remote_read_segment_count: 4
# IA_remote_read_segment_size: 4096
# IA_remote_read_segment_wait_time: 0.015
# Is_internal: true
# Digest: 42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772
# Stats: t1:1,t2:2
Expand Down Expand Up @@ -191,7 +194,7 @@ select * from t;`
}
expectRecordString := `2019-04-28 15:24:04.309074,` +
`405888132465033227,root,localhost,0,alias123,57,0.12,0.216905,` +
`0,0,0,0,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0.38,0.021,0,0,0,1,637,0,10,10,10,10,100,,,1,42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772,t1:1,t2:2,` +
`0,0,0,0,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0.38,0.021,0,0,0,1,637,0,10,10,10,10,100,4,4096,0.015,,,1,42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772,t1:1,t2:2,` +
`0.1,0.2,0.03,127.0.0.1:20160,0.05,0.6,0.8,0.0.0.0:20160,70724,23333,65536,0,0,0,30000,3000,10000,1000,500000,500005,300000,300005,0,0,,` +
`Cop_backoff_regionMiss_total_times: 200 Cop_backoff_regionMiss_total_time: 0.2 Cop_backoff_regionMiss_max_time: 0.2 Cop_backoff_regionMiss_max_addr: 127.0.0.1 Cop_backoff_regionMiss_avg_time: 0.2 Cop_backoff_regionMiss_p90_time: 0.2 Cop_backoff_rpcPD_total_times: 200 Cop_backoff_rpcPD_total_time: 0.2 Cop_backoff_rpcPD_max_time: 0.2 Cop_backoff_rpcPD_max_addr: 127.0.0.1 Cop_backoff_rpcPD_avg_time: 0.2 Cop_backoff_rpcPD_p90_time: 0.2 Cop_backoff_rpcTiKV_total_times: 200 Cop_backoff_rpcTiKV_total_time: 0.2 Cop_backoff_rpcTiKV_max_time: 0.2 Cop_backoff_rpcTiKV_max_addr: 127.0.0.1 Cop_backoff_rpcTiKV_avg_time: 0.2 Cop_backoff_rpcTiKV_p90_time: 0.2,` +
`0,0,1,0,1,1,0,default,2.158,2.123,0.05,0.01,0.021,1,1,150,total_ru:150.00, tidb_ru:0.00, tikv_ru:100.00, tiflash_ru:50.00,,60e9378c746d9a2be1c791047e008967cf252eb6de9167ad3aa6098fa2d523f4,` +
Expand All @@ -214,7 +217,7 @@ select * from t;`
}
expectRecordString = `2019-04-28 15:24:04.309074,` +
`405888132465033227,root,localhost,0,alias123,57,0.12,0.216905,` +
`0,0,0,0,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0.38,0.021,0,0,0,1,637,0,10,10,10,10,100,,,1,42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772,t1:1,t2:2,` +
`0,0,0,0,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0.38,0.021,0,0,0,1,637,0,10,10,10,10,100,4,4096,0.015,,,1,42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772,t1:1,t2:2,` +
`0.1,0.2,0.03,127.0.0.1:20160,0.05,0.6,0.8,0.0.0.0:20160,70724,23333,65536,0,0,0,30000,3000,10000,1000,500000,500005,300000,300005,0,0,,` +
`Cop_backoff_regionMiss_total_times: 200 Cop_backoff_regionMiss_total_time: 0.2 Cop_backoff_regionMiss_max_time: 0.2 Cop_backoff_regionMiss_max_addr: 127.0.0.1 Cop_backoff_regionMiss_avg_time: 0.2 Cop_backoff_regionMiss_p90_time: 0.2 Cop_backoff_rpcPD_total_times: 200 Cop_backoff_rpcPD_total_time: 0.2 Cop_backoff_rpcPD_max_time: 0.2 Cop_backoff_rpcPD_max_addr: 127.0.0.1 Cop_backoff_rpcPD_avg_time: 0.2 Cop_backoff_rpcPD_p90_time: 0.2 Cop_backoff_rpcTiKV_total_times: 200 Cop_backoff_rpcTiKV_total_time: 0.2 Cop_backoff_rpcTiKV_max_time: 0.2 Cop_backoff_rpcTiKV_max_addr: 127.0.0.1 Cop_backoff_rpcTiKV_avg_time: 0.2 Cop_backoff_rpcTiKV_p90_time: 0.2,` +
`0,0,1,0,1,1,0,default,2.158,2.123,0.05,0.01,0.021,1,1,150,total_ru:150.00, tidb_ru:0.00, tikv_ru:100.00, tiflash_ru:50.00,,60e9378c746d9a2be1c791047e008967cf252eb6de9167ad3aa6098fa2d523f4,` +
Expand Down Expand Up @@ -282,9 +285,9 @@ select * from t;
rows, err = parseSlowLog(ctx, reader)
require.NoError(t, err)
require.Len(t, rows, 1)
value, _ := rows[0][41].ToString()
value, _ := rows[0][44].ToString()
require.Equal(t, value, "a: b")
value, _ = rows[0][42].ToString()
value, _ = rows[0][45].ToString()
require.Equal(t, value, "[t:i: a]")
}

Expand Down
15 changes: 15 additions & 0 deletions pkg/infoschema/metric_table_def.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -889,6 +889,21 @@ var MetricTableMap = map[string]MetricTableDef{
Labels: []string{"instance", "type", "db"},
Comment: "The storage size per TiKV instance",
},
"tidb_ia_remote_read_segment_count": {
PromQL: `sum(increase(tidb_server_ia_remote_read_segment_count{$LABEL_CONDITIONS}[$RANGE_DURATION])) by (instance)`,
Labels: []string{"instance"},
Comment: "The total count of IA remote read segments observed by each TiDB instance",
},
"tidb_ia_remote_read_segment_size": {
PromQL: `sum(increase(tidb_server_ia_remote_read_segment_size_bytes{$LABEL_CONDITIONS}[$RANGE_DURATION])) by (instance)`,
Labels: []string{"instance"},
Comment: "The total bytes of IA remote read segments observed by each TiDB instance",
},
"tidb_ia_remote_read_segment_wait_time_histogram": {
PromQL: `sum(rate(tidb_server_ia_remote_read_segment_wait_duration_seconds_bucket{$LABEL_CONDITIONS}[$RANGE_DURATION])) by (instance,le)`,
Labels: []string{"instance", "le"},
Comment: "The histogram of IA remote read segment wait time observed by each TiDB instance",
},
"tikv_store_size": {
PromQL: `sum(tikv_store_size_bytes{$LABEL_CONDITIONS}) by (instance,type)`,
Labels: []string{"instance", "type"},
Expand Down
9 changes: 9 additions & 0 deletions pkg/infoschema/tables.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -933,6 +933,9 @@ var slowQueryCols = []columnInfo{
{name: execdetails.RocksdbBlockCacheHitCountStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.UnsignedFlag},
{name: execdetails.RocksdbBlockReadCountStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.UnsignedFlag},
{name: execdetails.RocksdbBlockReadByteStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.UnsignedFlag},
{name: execdetails.IARemoteReadSegmentCountStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.UnsignedFlag},
{name: execdetails.IARemoteReadSegmentSizeStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.UnsignedFlag},
{name: execdetails.IARemoteReadSegmentWaitTimeStr, tp: mysql.TypeDouble, size: 22},
{name: variable.SlowLogDBStr, tp: mysql.TypeVarchar, size: 64},
{name: variable.SlowLogIndexNamesStr, tp: mysql.TypeVarchar, size: 100},
{name: variable.SlowLogIsInternalStr, tp: mysql.TypeTiny, size: 1},
Expand Down Expand Up @@ -1365,6 +1368,12 @@ var tableStatementsSummaryCols = []columnInfo{
{name: stmtsummary.MaxRocksdbBlockReadCountStr, tp: mysql.TypeLong, size: 11, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Max number of rocksdb block read count"},
{name: stmtsummary.AvgRocksdbBlockReadByteStr, tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average number of rocksdb block read byte"},
{name: stmtsummary.MaxRocksdbBlockReadByteStr, tp: mysql.TypeLong, size: 11, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Max number of rocksdb block read byte"},
{name: stmtsummary.AvgIAReadSegmentCountStr, tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average number of IA segments read"},
{name: stmtsummary.MaxIAReadSegmentCountStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Max number of IA segments read"},
{name: stmtsummary.AvgIARemoteReadSegmentSizeStr, tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average number of bytes returned from IA remote segment reads"},
{name: stmtsummary.MaxIARemoteReadSegmentSizeStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Max number of bytes returned from IA remote segment reads"},
{name: stmtsummary.AvgIARemoteReadSegmentWaitTimeStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average time spent waiting for IA remote segment reads"},
{name: stmtsummary.MaxIARemoteReadSegmentWaitTimeStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Max time spent waiting for IA remote segment reads"},
{name: stmtsummary.AvgPrewriteTimeStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average time of prewrite phase"},
{name: stmtsummary.MaxPrewriteTimeStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Max time of prewrite phase"},
{name: stmtsummary.AvgCommitTimeStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average time of commit phase"},
Expand Down
6 changes: 6 additions & 0 deletions pkg/infoschema/test/clustertablestest/tables_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,9 @@ func TestSlowQuery(t *testing.T) {
"10",
"10",
"100",
"0",
"0",
"0",
"test",
"",
"0",
Expand Down Expand Up @@ -546,6 +549,9 @@ func TestSlowQuery(t *testing.T) {
"0",
"0",
"0",
"0",
"0",
"0",
"rtdb",
"",
"0",
Expand Down
3 changes: 3 additions & 0 deletions pkg/metrics/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,9 @@ func RegisterMetrics() {
prometheus.MustRegister(QueryDurationHistogram)
prometheus.MustRegister(QueryRPCHistogram)
prometheus.MustRegister(QueryProcessedKeyHistogram)
prometheus.MustRegister(IARemoteReadSegmentCount)
prometheus.MustRegister(IARemoteReadSegmentSize)
prometheus.MustRegister(IARemoteReadSegmentWaitDuration)
prometheus.MustRegister(QueryTotalCounter)
prometheus.MustRegister(AffectedRowsCounter)
prometheus.MustRegister(SchemaLeaseErrorCounter)
Expand Down
48 changes: 38 additions & 10 deletions pkg/metrics/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,19 @@ var (

// Metrics
var (
PacketIOCounter *prometheus.CounterVec
QueryDurationHistogram *prometheus.HistogramVec
QueryRPCHistogram *prometheus.HistogramVec
QueryProcessedKeyHistogram *prometheus.HistogramVec
QueryTotalCounter *prometheus.CounterVec
ConnGauge *prometheus.GaugeVec
DisconnectionCounter *prometheus.CounterVec
PreparedStmtGauge prometheus.Gauge
ExecuteErrorCounter *prometheus.CounterVec
CriticalErrorCounter prometheus.Counter
PacketIOCounter *prometheus.CounterVec
QueryDurationHistogram *prometheus.HistogramVec
QueryRPCHistogram *prometheus.HistogramVec
QueryProcessedKeyHistogram *prometheus.HistogramVec
IARemoteReadSegmentCount *prometheus.CounterVec
IARemoteReadSegmentSize *prometheus.CounterVec
IARemoteReadSegmentWaitDuration *prometheus.HistogramVec
QueryTotalCounter *prometheus.CounterVec
ConnGauge *prometheus.GaugeVec
DisconnectionCounter *prometheus.CounterVec
PreparedStmtGauge prometheus.Gauge
ExecuteErrorCounter *prometheus.CounterVec
CriticalErrorCounter prometheus.Counter

ServerStart = "server-start"
ServerStop = "server-stop"
Expand Down Expand Up @@ -120,6 +123,31 @@ func InitServerMetrics() {
Buckets: prometheus.ExponentialBuckets(1, 2, 32),
}, []string{LblSQLType, LblDb})

IARemoteReadSegmentCount = metricscommon.NewCounterVec(
prometheus.CounterOpts{
Namespace: "tidb",
Subsystem: "server",
Name: "ia_remote_read_segment_count",
Help: "Counter of IA remote read segments observed by TiDB.",
}, []string{LblSQLType, LblDb})

IARemoteReadSegmentSize = metricscommon.NewCounterVec(
prometheus.CounterOpts{
Namespace: "tidb",
Subsystem: "server",
Name: "ia_remote_read_segment_size_bytes",
Help: "Counter of IA remote read segment bytes observed by TiDB.",
}, []string{LblSQLType, LblDb})

IARemoteReadSegmentWaitDuration = metricscommon.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: "tidb",
Subsystem: "server",
Name: "ia_remote_read_segment_wait_duration_seconds",
Help: "Bucketed histogram of IA remote read segment wait time observed by TiDB.",
Buckets: prometheus.ExponentialBuckets(0.00005, 2, 20), // 50us ~ 26s
}, []string{LblSQLType, LblDb})

QueryTotalCounter = metricscommon.NewCounterVec(
prometheus.CounterOpts{
Namespace: "tidb",
Expand Down
13 changes: 10 additions & 3 deletions pkg/server/conn.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1414,12 +1414,19 @@ func (cc *clientConn) addQueryMetrics(cmd byte, startTime time.Time, err error)
if stmtType != "" {
sqlType = stmtType
}
execDetails := vars.StmtCtx.GetExecDetails()

for _, dbName := range session.GetDBNames(vars) {
metrics.QueryDurationHistogram.WithLabelValues(sqlType, dbName, vars.StmtCtx.ResourceGroupName).Observe(cost.Seconds())
metrics.QueryRPCHistogram.WithLabelValues(sqlType, dbName).Observe(float64(vars.StmtCtx.GetExecDetails().RequestCount))
if vars.StmtCtx.GetExecDetails().ScanDetail != nil {
metrics.QueryProcessedKeyHistogram.WithLabelValues(sqlType, dbName).Observe(float64(vars.StmtCtx.GetExecDetails().ScanDetail.ProcessedKeys))
metrics.QueryRPCHistogram.WithLabelValues(sqlType, dbName).Observe(float64(execDetails.RequestCount))
if execDetails.ScanDetail != nil {
metrics.QueryProcessedKeyHistogram.WithLabelValues(sqlType, dbName).Observe(float64(execDetails.ScanDetail.ProcessedKeys))
iaStats := execdetails.GetIARemoteReadSegmentStats(execDetails.ScanDetail)
metrics.IARemoteReadSegmentCount.WithLabelValues(sqlType, dbName).Add(float64(iaStats.Count))
metrics.IARemoteReadSegmentSize.WithLabelValues(sqlType, dbName).Add(float64(iaStats.Bytes))
if iaStats.WaitTime > 0 {
metrics.IARemoteReadSegmentWaitDuration.WithLabelValues(sqlType, dbName).Observe(iaStats.WaitTime.Seconds())
}
}
}
}
Expand Down
10 changes: 10 additions & 0 deletions pkg/sessionctx/variable/slow_log.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,16 @@ func (s *SessionVars) SlowLogFormat(logItems *SlowQueryLogItems) string {
if execDetailStr := logItems.ExecDetail.String(); len(execDetailStr) > 0 {
buf.WriteString(SlowLogRowPrefixStr + execDetailStr + "\n")
}
iaStats := execdetails.GetIARemoteReadSegmentStats(logItems.ExecDetail.ScanDetail)
if iaStats.Count > 0 {
writeSlowLogItem(&buf, execdetails.IARemoteReadSegmentCountStr, strconv.FormatUint(iaStats.Count, 10))
}
if iaStats.Bytes > 0 {
writeSlowLogItem(&buf, execdetails.IARemoteReadSegmentSizeStr, strconv.FormatUint(iaStats.Bytes, 10))
}
if iaStats.WaitTime > 0 {
writeSlowLogItem(&buf, execdetails.IARemoteReadSegmentWaitTimeStr, strconv.FormatFloat(iaStats.WaitTime.Seconds(), 'f', -1, 64))
}

if len(s.CurrentDB) > 0 {
writeSlowLogItem(&buf, SlowLogDBStr, strings.ToLower(s.CurrentDB))
Expand Down
11 changes: 9 additions & 2 deletions pkg/sessionctx/variable/tests/session_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,8 +166,12 @@ func TestSlowLogFormat(t *testing.T) {
CopExecDetails: execdetails.CopExecDetails{
BackoffTime: time.Millisecond,
ScanDetail: &util.ScanDetail{
ProcessedKeys: 20001,
TotalKeys: 10000,
ProcessedKeys: 20001,
TotalKeys: 10000,
IaCacheHitCount: 2,
IaRemoteReadSegmentCount: 4,
IaRemoteReadSegmentBytes: 4096,
IaRemoteReadSegmentDuration: 15 * time.Millisecond,
},
TimeDetail: util.TimeDetail{
ProcessTime: time.Second * time.Duration(2),
Expand Down Expand Up @@ -241,6 +245,9 @@ func TestSlowLogFormat(t *testing.T) {
# Optimize_time: 0.00000001 Opt_logical: 0.00000001 Opt_physical: 0.00000001 Opt_binding_match: 0.00000001 Opt_stats_sync_wait: 0.00000001 Opt_stats_derive: 0.00000001
# Wait_TS: 0.000000003
# Process_time: 2 Wait_time: 60 Backoff_time: 0.001 Request_count: 2 Process_keys: 20001 Total_keys: 10000
# IA_remote_read_segment_count: 4
# IA_remote_read_segment_size: 4096
# IA_remote_read_segment_wait_time: 0.015
# DB: test
# Index_names: [t1:a,t2:b]
# Is_internal: true
Expand Down
25 changes: 25 additions & 0 deletions pkg/util/execdetails/execdetails.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,12 @@
RocksdbBlockReadByteStr = "Rocksdb_block_read_byte"
// RocksdbBlockReadTimeStr means the time spent on rocksdb block read.
RocksdbBlockReadTimeStr = "Rocksdb_block_read_time"
// IARemoteReadSegmentCountStr means the number of IA remote segment reads.
IARemoteReadSegmentCountStr = "IA_remote_read_segment_count"
// IARemoteReadSegmentSizeStr means the number of bytes returned from IA remote segment reads.
IARemoteReadSegmentSizeStr = "IA_remote_read_segment_size"
// IARemoteReadSegmentWaitTimeStr means the total time spent waiting for IA remote segment reads.
IARemoteReadSegmentWaitTimeStr = "IA_remote_read_segment_wait_time"

// The following constants define the set of fields for SlowQueryLogItems
// that are relevant to evaluating and triggering SlowLogRules.
Expand All @@ -204,6 +210,25 @@
PrewriteRegionStr = "Prewrite_region"
)

// IARemoteReadSegmentStats contains IA remote-read scan statistics.
type IARemoteReadSegmentStats struct {
Count uint64
Bytes uint64
WaitTime time.Duration
}

// GetIARemoteReadSegmentStats reads IA remote-read scan statistics from client-go ScanDetail.
func GetIARemoteReadSegmentStats(scanDetail *util.ScanDetail) IARemoteReadSegmentStats {
if scanDetail == nil {
return IARemoteReadSegmentStats{}
}
return IARemoteReadSegmentStats{
Count: scanDetail.IaRemoteReadSegmentCount,

Check failure on line 226 in pkg/util/execdetails/execdetails.go

View workflow job for this annotation

GitHub Actions / Bazel Crossbuild (ubuntu-24.04-arm) 

scanDetail.IaRemoteReadSegmentCount undefined (type *"github.com/tikv/client-go/v2/util".ScanDetail has no field or method IaRemoteReadSegmentCount)
Bytes: scanDetail.IaRemoteReadSegmentBytes,

Check failure on line 227 in pkg/util/execdetails/execdetails.go

View workflow job for this annotation

GitHub Actions / Bazel Crossbuild (ubuntu-24.04-arm) 

scanDetail.IaRemoteReadSegmentBytes undefined (type *"github.com/tikv/client-go/v2/util".ScanDetail has no field or method IaRemoteReadSegmentBytes)
WaitTime: scanDetail.IaRemoteReadSegmentDuration,

Check failure on line 228 in pkg/util/execdetails/execdetails.go

View workflow job for this annotation

GitHub Actions / Bazel Crossbuild (ubuntu-24.04-arm) 

scanDetail.IaRemoteReadSegmentDuration undefined (type *"github.com/tikv/client-go/v2/util".ScanDetail has no field or method IaRemoteReadSegmentDuration)
}
}

Comment on lines +213 to +231

@coderabbitai coderabbitai Bot Jul 3, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🔴 Critical | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
set -euo pipefail

printf '\n== go.mod client-go dependency ==\n'
grep -nE 'tikv/client-go|client-go' go.mod || true

printf '\n== execdetails.go relevant lines ==\n'
sed -n '180,260p' pkg/util/execdetails/execdetails.go

printf '\n== ScanDetail field references in repo ==\n'
rg -n 'IaRemoteReadSegment|IARemoteReadSegment' -S .

printf '\n== client-go module cache probe (if available) ==\n'
go env GOPATH GOMODCACHE 2>/dev/null || true

Repository: pingcap/tidb

Length of output: 21160

🏁 Script executed:

#!/bin/bash
set -euo pipefail

GOMODCACHE="$(go env GOMODCACHE)"
printf 'GOMODCACHE=%s\n' "$GOMODCACHE"

printf '\n== client-go module versions ==\n'
fd -a 'client-go' "$GOMODCACHE/github.com/tikv" 2>/dev/null || true

printf '\n== ScanDetail definition ==\n'
SCANDETAIL_FILE="$(rg -l -n 'type ScanDetail struct' "$GOMODCACHE/github.com/tikv/client-go" 2>/dev/null | head -n 1 || true)"
printf 'file=%s\n' "${SCANDETAIL_FILE:-<not found>}"
if [ -n "${SCANDETAIL_FILE:-}" ]; then
  sed -n '1,260p' "$SCANDETAIL_FILE"
fi

printf '\n== IaRemoteReadSegment field references in client-go ==\n'
rg -n 'IaRemoteReadSegment(Count|Bytes|Duration)' "$GOMODCACHE/github.com/tikv/client-go" 2>/dev/null || true

Repository: pingcap/tidb

Length of output: 9764

🏁 Script executed:

#!/bin/bash
set -euo pipefail

GOMODCACHE="$(go env GOMODCACHE)"
CLIENT_GO_DIR="$GOMODCACHE/github.com/tikv/client-go/v2@v2.0.8-0.20260617030124-661db4f5f4e8"

printf '== ScanDetail struct location ==\n'
rg -n 'type ScanDetail struct' "$CLIENT_GO_DIR" -S

printf '\n== ScanDetail struct body ==\n'
SCAN_FILE="$(rg -l 'type ScanDetail struct' "$CLIENT_GO_DIR" | head -n 1)"
sed -n '/type ScanDetail struct/,/^}/p' "$SCAN_FILE"

printf '\n== IaRemoteReadSegment fields in ScanDetail body ==\n'
sed -n '/type ScanDetail struct/,/^}/p' "$SCAN_FILE" | rg -n 'IaRemoteReadSegment|IARemoteReadSegment' -n || true

Repository: pingcap/tidb

Length of output: 1852

Wait for the client-go bump before merging this change.
util.ScanDetail in the current github.com/tikv/client-go/v2 dependency still has no IaRemoteReadSegmentCount, IaRemoteReadSegmentBytes, or IaRemoteReadSegmentDuration fields, so this code and its tests will not compile until that dependency update lands.

🧰 Tools
🪛 golangci-lint (2.12.2)

[error] 226-226: : # github.com/pingcap/tidb/pkg/util/execdetails [github.com/pingcap/tidb/pkg/util/execdetails.test]
pkg/util/execdetails/execdetails.go:226:24: scanDetail.IaRemoteReadSegmentCount undefined (type *"github.com/tikv/client-go/v2/util".ScanDetail has no field or method IaRemoteReadSegmentCount)
pkg/util/execdetails/execdetails.go:227:24: scanDetail.IaRemoteReadSegmentBytes undefined (type *"github.com/tikv/client-go/v2/util".ScanDetail has no field or method IaRemoteReadSegmentBytes)
pkg/util/execdetails/execdetails.go:228:24: scanDetail.IaRemoteReadSegmentDuration undefined (type *"github.com/tikv/client-go/v2/util".ScanDetail has no field or method IaRemoteReadSegmentDuration)
pkg/util/execdetails/execdetails_test.go:1194:3: unknown field IaRemoteReadSegmentCount in struct literal of type "github.com/tikv/client-go/v2/util".ScanDetail
pkg/util/execdetails/execdetails_test.go:1195:3: unknown field IaRemoteReadSegmentBytes in struct literal of type "github.com/tikv/client-go/v2/util".ScanDetail
pkg/util/execdetails/execdetails_test.go:1196:3: unknown field IaRemoteReadSegmentDuration in struct literal of type "github.com/tikv/client-go/v2/util".ScanDetail

(typecheck)

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@pkg/util/execdetails/execdetails.go` around lines 213 - 231, The new IA
remote-read fields are not available in the current client-go version, so the
added accessor in GetIARemoteReadSegmentStats will not compile yet. Hold this
change until the github.com/tikv/client-go/v2 bump lands, or guard/remove the
references to util.ScanDetail.IaRemoteReadSegmentCount,
IaRemoteReadSegmentBytes, and IaRemoteReadSegmentDuration so this package and
its tests only use fields that exist in the current dependency.

Source: Linters/SAST tools

// String implements the fmt.Stringer interface.
func (d ExecDetails) String() string {
parts := make([]string, 0, 8)
Expand Down
16 changes: 16 additions & 0 deletions pkg/util/execdetails/execdetails_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1188,3 +1188,19 @@ func TestRURuntimeStatsMergeKeepsExistingRUVersion(t *testing.T) {
dst.Merge(src)
require.Equal(t, rmclient.RUVersionV1, dst.RUVersion)
}

func TestGetIARemoteReadSegmentStats(t *testing.T) {
stats := GetIARemoteReadSegmentStats(&util.ScanDetail{
IaRemoteReadSegmentCount: 3,
IaRemoteReadSegmentBytes: 4096,
IaRemoteReadSegmentDuration: 5 * time.Millisecond,
})
require.Equal(t, IARemoteReadSegmentStats{
Count: 3,
Bytes: 4096,
WaitTime: 5 * time.Millisecond,
}, stats)

require.Equal(t, IARemoteReadSegmentStats{}, GetIARemoteReadSegmentStats(&util.ScanDetail{}))
require.Equal(t, IARemoteReadSegmentStats{}, GetIARemoteReadSegmentStats(nil))
}
1 change: 1 addition & 0 deletions pkg/util/stmtsummary/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ go_test(
"//pkg/testkit/testsetup",
"//pkg/types",
"//pkg/util",
"//pkg/util/chunk",
"//pkg/util/execdetails",
"//pkg/util/hack",
"//pkg/util/plancodec",
Expand Down
12 changes: 12 additions & 0 deletions pkg/util/stmtsummary/evicted.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,18 @@ func addInfo(addTo *stmtSummaryByDigestElement, addWith *stmtSummaryByDigestElem
if addTo.maxRocksdbBlockReadByte < addWith.maxRocksdbBlockReadByte {
addTo.maxRocksdbBlockReadByte = addWith.maxRocksdbBlockReadByte
}
addTo.sumIAReadSegmentCount += addWith.sumIAReadSegmentCount
if addTo.maxIAReadSegmentCount < addWith.maxIAReadSegmentCount {
addTo.maxIAReadSegmentCount = addWith.maxIAReadSegmentCount
}
addTo.sumIARemoteReadSegmentSize += addWith.sumIARemoteReadSegmentSize
if addTo.maxIARemoteReadSegmentSize < addWith.maxIARemoteReadSegmentSize {
addTo.maxIARemoteReadSegmentSize = addWith.maxIARemoteReadSegmentSize
}
addTo.sumIARemoteReadSegmentWaitTime += addWith.sumIARemoteReadSegmentWaitTime
if addTo.maxIARemoteReadSegmentWaitTime < addWith.maxIARemoteReadSegmentWaitTime {
addTo.maxIARemoteReadSegmentWaitTime = addWith.maxIARemoteReadSegmentWaitTime
}

// txn
addTo.commitCount += addWith.commitCount
Expand Down
Loading
Loading