From d136bb67b34b3abbf81fe913020a339603535b44 Mon Sep 17 00:00:00 2001 From: Connor1996 Date: Thu, 2 Jul 2026 15:52:56 -0700 Subject: [PATCH] metrics: expose IA remote-read scan details Signed-off-by: Connor1996 --- pkg/executor/slow_query.go | 5 +- pkg/executor/slow_query_test.go | 11 ++- pkg/infoschema/metric_table_def.go | 15 +++ pkg/infoschema/tables.go | 9 ++ .../test/clustertablestest/tables_test.go | 6 ++ pkg/metrics/metrics.go | 3 + pkg/metrics/server.go | 48 ++++++++-- pkg/server/conn.go | 13 ++- pkg/sessionctx/variable/slow_log.go | 10 ++ pkg/sessionctx/variable/tests/session_test.go | 11 ++- pkg/util/execdetails/execdetails.go | 25 +++++ pkg/util/execdetails/execdetails_test.go | 16 ++++ pkg/util/stmtsummary/BUILD.bazel | 1 + pkg/util/stmtsummary/evicted.go | 12 +++ pkg/util/stmtsummary/reader.go | 24 +++++ pkg/util/stmtsummary/statement_summary.go | 59 ++++++++---- .../stmtsummary/statement_summary_test.go | 95 +++++++++++++++++++ pkg/util/stmtsummary/v2/BUILD.bazel | 2 + pkg/util/stmtsummary/v2/column.go | 24 +++++ pkg/util/stmtsummary/v2/column_test.go | 88 +++++++++++++++++ pkg/util/stmtsummary/v2/record.go | 71 ++++++++++---- 21 files changed, 487 insertions(+), 61 deletions(-) diff --git a/pkg/executor/slow_query.go b/pkg/executor/slow_query.go index 8cab021c79900..228604b7fe9d5 100644 --- a/pkg/executor/slow_query.go +++ b/pkg/executor/slow_query.go @@ -1124,7 +1124,8 @@ func getColumnValueFactoryByName(colName string, columnIdx int) (slowQueryColumn execdetails.RequestCountStr, execdetails.TotalKeysStr, execdetails.ProcessKeysStr, execdetails.RocksdbDeleteSkippedCountStr, execdetails.RocksdbKeySkippedCountStr, execdetails.RocksdbBlockCacheHitCountStr, execdetails.RocksdbBlockReadCountStr, - variable.SlowLogTxnStartTSStr, execdetails.RocksdbBlockReadByteStr: + variable.SlowLogTxnStartTSStr, execdetails.RocksdbBlockReadByteStr, + execdetails.IARemoteReadSegmentCountStr, execdetails.IARemoteReadSegmentSizeStr: return func(row []types.Datum, value string, _ *time.Location, _ *slowLogChecker) (valid bool, err error) { v, err := strconv.ParseUint(value, 10, 64) if err != nil { @@ -1143,7 +1144,7 @@ func getColumnValueFactoryByName(colName string, columnIdx int) (slowQueryColumn variable.SlowLogCopWaitAvg, variable.SlowLogCopWaitP90, variable.SlowLogCopWaitMax, variable.SlowLogKVTotal, variable.SlowLogPDTotal, variable.SlowLogBackoffTotal, variable.SlowLogWriteSQLRespTotal, variable.SlowLogRRU, variable.SlowLogWRU, variable.SlowLogWaitRUDuration, variable.SlowLogTidbCPUUsageDuration, variable.SlowLogTikvCPUUsageDuration, - variable.SlowLogMemArbitration, variable.SlowLogRequestUnitV2: + variable.SlowLogMemArbitration, execdetails.IARemoteReadSegmentWaitTimeStr, variable.SlowLogRequestUnitV2: return func(row []types.Datum, value string, _ *time.Location, _ *slowLogChecker) (valid bool, err error) { v, err := strconv.ParseFloat(value, 64) if err != nil { diff --git a/pkg/executor/slow_query_test.go b/pkg/executor/slow_query_test.go index c0affb5d9b1bf..acef2ff899b2a 100644 --- a/pkg/executor/slow_query_test.go +++ b/pkg/executor/slow_query_test.go @@ -135,6 +135,9 @@ func TestParseSlowLogFile(t *testing.T) { # Query_time: 0.216905 # Cop_time: 0.38 Process_time: 0.021 Request_count: 1 Total_keys: 637 Processed_keys: 436 # Rocksdb_delete_skipped_count: 10 Rocksdb_key_skipped_count: 10 Rocksdb_block_cache_hit_count: 10 Rocksdb_block_read_count: 10 Rocksdb_block_read_byte: 100 +# IA_remote_read_segment_count: 4 +# IA_remote_read_segment_size: 4096 +# IA_remote_read_segment_wait_time: 0.015 # Is_internal: true # Digest: 42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772 # Stats: t1:1,t2:2 @@ -191,7 +194,7 @@ select * from t;` } expectRecordString := `2019-04-28 15:24:04.309074,` + `405888132465033227,root,localhost,0,alias123,57,0.12,0.216905,` + - `0,0,0,0,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0.38,0.021,0,0,0,1,637,0,10,10,10,10,100,,,1,42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772,t1:1,t2:2,` + + `0,0,0,0,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0.38,0.021,0,0,0,1,637,0,10,10,10,10,100,4,4096,0.015,,,1,42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772,t1:1,t2:2,` + `0.1,0.2,0.03,127.0.0.1:20160,0.05,0.6,0.8,0.0.0.0:20160,70724,23333,65536,0,0,0,30000,3000,10000,1000,500000,500005,300000,300005,0,0,,` + `Cop_backoff_regionMiss_total_times: 200 Cop_backoff_regionMiss_total_time: 0.2 Cop_backoff_regionMiss_max_time: 0.2 Cop_backoff_regionMiss_max_addr: 127.0.0.1 Cop_backoff_regionMiss_avg_time: 0.2 Cop_backoff_regionMiss_p90_time: 0.2 Cop_backoff_rpcPD_total_times: 200 Cop_backoff_rpcPD_total_time: 0.2 Cop_backoff_rpcPD_max_time: 0.2 Cop_backoff_rpcPD_max_addr: 127.0.0.1 Cop_backoff_rpcPD_avg_time: 0.2 Cop_backoff_rpcPD_p90_time: 0.2 Cop_backoff_rpcTiKV_total_times: 200 Cop_backoff_rpcTiKV_total_time: 0.2 Cop_backoff_rpcTiKV_max_time: 0.2 Cop_backoff_rpcTiKV_max_addr: 127.0.0.1 Cop_backoff_rpcTiKV_avg_time: 0.2 Cop_backoff_rpcTiKV_p90_time: 0.2,` + `0,0,1,0,1,1,0,default,2.158,2.123,0.05,0.01,0.021,1,1,150,total_ru:150.00, tidb_ru:0.00, tikv_ru:100.00, tiflash_ru:50.00,,60e9378c746d9a2be1c791047e008967cf252eb6de9167ad3aa6098fa2d523f4,` + @@ -214,7 +217,7 @@ select * from t;` } expectRecordString = `2019-04-28 15:24:04.309074,` + `405888132465033227,root,localhost,0,alias123,57,0.12,0.216905,` + - `0,0,0,0,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0.38,0.021,0,0,0,1,637,0,10,10,10,10,100,,,1,42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772,t1:1,t2:2,` + + `0,0,0,0,0,0,0,0,0,0,0,0,,0,0,0,0,0,0,0.38,0.021,0,0,0,1,637,0,10,10,10,10,100,4,4096,0.015,,,1,42a1c8aae6f133e934d4bf0147491709a8812ea05ff8819ec522780fe657b772,t1:1,t2:2,` + `0.1,0.2,0.03,127.0.0.1:20160,0.05,0.6,0.8,0.0.0.0:20160,70724,23333,65536,0,0,0,30000,3000,10000,1000,500000,500005,300000,300005,0,0,,` + `Cop_backoff_regionMiss_total_times: 200 Cop_backoff_regionMiss_total_time: 0.2 Cop_backoff_regionMiss_max_time: 0.2 Cop_backoff_regionMiss_max_addr: 127.0.0.1 Cop_backoff_regionMiss_avg_time: 0.2 Cop_backoff_regionMiss_p90_time: 0.2 Cop_backoff_rpcPD_total_times: 200 Cop_backoff_rpcPD_total_time: 0.2 Cop_backoff_rpcPD_max_time: 0.2 Cop_backoff_rpcPD_max_addr: 127.0.0.1 Cop_backoff_rpcPD_avg_time: 0.2 Cop_backoff_rpcPD_p90_time: 0.2 Cop_backoff_rpcTiKV_total_times: 200 Cop_backoff_rpcTiKV_total_time: 0.2 Cop_backoff_rpcTiKV_max_time: 0.2 Cop_backoff_rpcTiKV_max_addr: 127.0.0.1 Cop_backoff_rpcTiKV_avg_time: 0.2 Cop_backoff_rpcTiKV_p90_time: 0.2,` + `0,0,1,0,1,1,0,default,2.158,2.123,0.05,0.01,0.021,1,1,150,total_ru:150.00, tidb_ru:0.00, tikv_ru:100.00, tiflash_ru:50.00,,60e9378c746d9a2be1c791047e008967cf252eb6de9167ad3aa6098fa2d523f4,` + @@ -282,9 +285,9 @@ select * from t; rows, err = parseSlowLog(ctx, reader) require.NoError(t, err) require.Len(t, rows, 1) - value, _ := rows[0][41].ToString() + value, _ := rows[0][44].ToString() require.Equal(t, value, "a: b") - value, _ = rows[0][42].ToString() + value, _ = rows[0][45].ToString() require.Equal(t, value, "[t:i: a]") } diff --git a/pkg/infoschema/metric_table_def.go b/pkg/infoschema/metric_table_def.go index 5cc6660368913..5fce4da0add33 100644 --- a/pkg/infoschema/metric_table_def.go +++ b/pkg/infoschema/metric_table_def.go @@ -889,6 +889,21 @@ var MetricTableMap = map[string]MetricTableDef{ Labels: []string{"instance", "type", "db"}, Comment: "The storage size per TiKV instance", }, + "tidb_ia_remote_read_segment_count": { + PromQL: `sum(increase(tidb_server_ia_remote_read_segment_count{$LABEL_CONDITIONS}[$RANGE_DURATION])) by (instance)`, + Labels: []string{"instance"}, + Comment: "The total count of IA remote read segments observed by each TiDB instance", + }, + "tidb_ia_remote_read_segment_size": { + PromQL: `sum(increase(tidb_server_ia_remote_read_segment_size_bytes{$LABEL_CONDITIONS}[$RANGE_DURATION])) by (instance)`, + Labels: []string{"instance"}, + Comment: "The total bytes of IA remote read segments observed by each TiDB instance", + }, + "tidb_ia_remote_read_segment_wait_time_histogram": { + PromQL: `sum(rate(tidb_server_ia_remote_read_segment_wait_duration_seconds_bucket{$LABEL_CONDITIONS}[$RANGE_DURATION])) by (instance,le)`, + Labels: []string{"instance", "le"}, + Comment: "The histogram of IA remote read segment wait time observed by each TiDB instance", + }, "tikv_store_size": { PromQL: `sum(tikv_store_size_bytes{$LABEL_CONDITIONS}) by (instance,type)`, Labels: []string{"instance", "type"}, diff --git a/pkg/infoschema/tables.go b/pkg/infoschema/tables.go index 3072d29e07c87..4305ec588db1d 100644 --- a/pkg/infoschema/tables.go +++ b/pkg/infoschema/tables.go @@ -933,6 +933,9 @@ var slowQueryCols = []columnInfo{ {name: execdetails.RocksdbBlockCacheHitCountStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.UnsignedFlag}, {name: execdetails.RocksdbBlockReadCountStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.UnsignedFlag}, {name: execdetails.RocksdbBlockReadByteStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.UnsignedFlag}, + {name: execdetails.IARemoteReadSegmentCountStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.UnsignedFlag}, + {name: execdetails.IARemoteReadSegmentSizeStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.UnsignedFlag}, + {name: execdetails.IARemoteReadSegmentWaitTimeStr, tp: mysql.TypeDouble, size: 22}, {name: variable.SlowLogDBStr, tp: mysql.TypeVarchar, size: 64}, {name: variable.SlowLogIndexNamesStr, tp: mysql.TypeVarchar, size: 100}, {name: variable.SlowLogIsInternalStr, tp: mysql.TypeTiny, size: 1}, @@ -1365,6 +1368,12 @@ var tableStatementsSummaryCols = []columnInfo{ {name: stmtsummary.MaxRocksdbBlockReadCountStr, tp: mysql.TypeLong, size: 11, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Max number of rocksdb block read count"}, {name: stmtsummary.AvgRocksdbBlockReadByteStr, tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average number of rocksdb block read byte"}, {name: stmtsummary.MaxRocksdbBlockReadByteStr, tp: mysql.TypeLong, size: 11, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Max number of rocksdb block read byte"}, + {name: stmtsummary.AvgIAReadSegmentCountStr, tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average number of IA segments read"}, + {name: stmtsummary.MaxIAReadSegmentCountStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Max number of IA segments read"}, + {name: stmtsummary.AvgIARemoteReadSegmentSizeStr, tp: mysql.TypeDouble, size: 22, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average number of bytes returned from IA remote segment reads"}, + {name: stmtsummary.MaxIARemoteReadSegmentSizeStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Max number of bytes returned from IA remote segment reads"}, + {name: stmtsummary.AvgIARemoteReadSegmentWaitTimeStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average time spent waiting for IA remote segment reads"}, + {name: stmtsummary.MaxIARemoteReadSegmentWaitTimeStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Max time spent waiting for IA remote segment reads"}, {name: stmtsummary.AvgPrewriteTimeStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average time of prewrite phase"}, {name: stmtsummary.MaxPrewriteTimeStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Max time of prewrite phase"}, {name: stmtsummary.AvgCommitTimeStr, tp: mysql.TypeLonglong, size: 20, flag: mysql.NotNullFlag | mysql.UnsignedFlag, comment: "Average time of commit phase"}, diff --git a/pkg/infoschema/test/clustertablestest/tables_test.go b/pkg/infoschema/test/clustertablestest/tables_test.go index 8f299182016bb..7999c4e2806e1 100644 --- a/pkg/infoschema/test/clustertablestest/tables_test.go +++ b/pkg/infoschema/test/clustertablestest/tables_test.go @@ -450,6 +450,9 @@ func TestSlowQuery(t *testing.T) { "10", "10", "100", + "0", + "0", + "0", "test", "", "0", @@ -546,6 +549,9 @@ func TestSlowQuery(t *testing.T) { "0", "0", "0", + "0", + "0", + "0", "rtdb", "", "0", diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index a71cc5637d540..3bc6342ba4581 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -197,6 +197,9 @@ func RegisterMetrics() { prometheus.MustRegister(QueryDurationHistogram) prometheus.MustRegister(QueryRPCHistogram) prometheus.MustRegister(QueryProcessedKeyHistogram) + prometheus.MustRegister(IARemoteReadSegmentCount) + prometheus.MustRegister(IARemoteReadSegmentSize) + prometheus.MustRegister(IARemoteReadSegmentWaitDuration) prometheus.MustRegister(QueryTotalCounter) prometheus.MustRegister(AffectedRowsCounter) prometheus.MustRegister(SchemaLeaseErrorCounter) diff --git a/pkg/metrics/server.go b/pkg/metrics/server.go index 655be61364e42..8a77c088a5ba2 100644 --- a/pkg/metrics/server.go +++ b/pkg/metrics/server.go @@ -28,16 +28,19 @@ var ( // Metrics var ( - PacketIOCounter *prometheus.CounterVec - QueryDurationHistogram *prometheus.HistogramVec - QueryRPCHistogram *prometheus.HistogramVec - QueryProcessedKeyHistogram *prometheus.HistogramVec - QueryTotalCounter *prometheus.CounterVec - ConnGauge *prometheus.GaugeVec - DisconnectionCounter *prometheus.CounterVec - PreparedStmtGauge prometheus.Gauge - ExecuteErrorCounter *prometheus.CounterVec - CriticalErrorCounter prometheus.Counter + PacketIOCounter *prometheus.CounterVec + QueryDurationHistogram *prometheus.HistogramVec + QueryRPCHistogram *prometheus.HistogramVec + QueryProcessedKeyHistogram *prometheus.HistogramVec + IARemoteReadSegmentCount *prometheus.CounterVec + IARemoteReadSegmentSize *prometheus.CounterVec + IARemoteReadSegmentWaitDuration *prometheus.HistogramVec + QueryTotalCounter *prometheus.CounterVec + ConnGauge *prometheus.GaugeVec + DisconnectionCounter *prometheus.CounterVec + PreparedStmtGauge prometheus.Gauge + ExecuteErrorCounter *prometheus.CounterVec + CriticalErrorCounter prometheus.Counter ServerStart = "server-start" ServerStop = "server-stop" @@ -120,6 +123,31 @@ func InitServerMetrics() { Buckets: prometheus.ExponentialBuckets(1, 2, 32), }, []string{LblSQLType, LblDb}) + IARemoteReadSegmentCount = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "server", + Name: "ia_remote_read_segment_count", + Help: "Counter of IA remote read segments observed by TiDB.", + }, []string{LblSQLType, LblDb}) + + IARemoteReadSegmentSize = metricscommon.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "tidb", + Subsystem: "server", + Name: "ia_remote_read_segment_size_bytes", + Help: "Counter of IA remote read segment bytes observed by TiDB.", + }, []string{LblSQLType, LblDb}) + + IARemoteReadSegmentWaitDuration = metricscommon.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "tidb", + Subsystem: "server", + Name: "ia_remote_read_segment_wait_duration_seconds", + Help: "Bucketed histogram of IA remote read segment wait time observed by TiDB.", + Buckets: prometheus.ExponentialBuckets(0.00005, 2, 20), // 50us ~ 26s + }, []string{LblSQLType, LblDb}) + QueryTotalCounter = metricscommon.NewCounterVec( prometheus.CounterOpts{ Namespace: "tidb", diff --git a/pkg/server/conn.go b/pkg/server/conn.go index 178a90fb10b91..d0a7c2955b39d 100644 --- a/pkg/server/conn.go +++ b/pkg/server/conn.go @@ -1414,12 +1414,19 @@ func (cc *clientConn) addQueryMetrics(cmd byte, startTime time.Time, err error) if stmtType != "" { sqlType = stmtType } + execDetails := vars.StmtCtx.GetExecDetails() for _, dbName := range session.GetDBNames(vars) { metrics.QueryDurationHistogram.WithLabelValues(sqlType, dbName, vars.StmtCtx.ResourceGroupName).Observe(cost.Seconds()) - metrics.QueryRPCHistogram.WithLabelValues(sqlType, dbName).Observe(float64(vars.StmtCtx.GetExecDetails().RequestCount)) - if vars.StmtCtx.GetExecDetails().ScanDetail != nil { - metrics.QueryProcessedKeyHistogram.WithLabelValues(sqlType, dbName).Observe(float64(vars.StmtCtx.GetExecDetails().ScanDetail.ProcessedKeys)) + metrics.QueryRPCHistogram.WithLabelValues(sqlType, dbName).Observe(float64(execDetails.RequestCount)) + if execDetails.ScanDetail != nil { + metrics.QueryProcessedKeyHistogram.WithLabelValues(sqlType, dbName).Observe(float64(execDetails.ScanDetail.ProcessedKeys)) + iaStats := execdetails.GetIARemoteReadSegmentStats(execDetails.ScanDetail) + metrics.IARemoteReadSegmentCount.WithLabelValues(sqlType, dbName).Add(float64(iaStats.Count)) + metrics.IARemoteReadSegmentSize.WithLabelValues(sqlType, dbName).Add(float64(iaStats.Bytes)) + if iaStats.WaitTime > 0 { + metrics.IARemoteReadSegmentWaitDuration.WithLabelValues(sqlType, dbName).Observe(iaStats.WaitTime.Seconds()) + } } } } diff --git a/pkg/sessionctx/variable/slow_log.go b/pkg/sessionctx/variable/slow_log.go index dfb84f0ab6cd0..79f7c7c7289b7 100644 --- a/pkg/sessionctx/variable/slow_log.go +++ b/pkg/sessionctx/variable/slow_log.go @@ -430,6 +430,16 @@ func (s *SessionVars) SlowLogFormat(logItems *SlowQueryLogItems) string { if execDetailStr := logItems.ExecDetail.String(); len(execDetailStr) > 0 { buf.WriteString(SlowLogRowPrefixStr + execDetailStr + "\n") } + iaStats := execdetails.GetIARemoteReadSegmentStats(logItems.ExecDetail.ScanDetail) + if iaStats.Count > 0 { + writeSlowLogItem(&buf, execdetails.IARemoteReadSegmentCountStr, strconv.FormatUint(iaStats.Count, 10)) + } + if iaStats.Bytes > 0 { + writeSlowLogItem(&buf, execdetails.IARemoteReadSegmentSizeStr, strconv.FormatUint(iaStats.Bytes, 10)) + } + if iaStats.WaitTime > 0 { + writeSlowLogItem(&buf, execdetails.IARemoteReadSegmentWaitTimeStr, strconv.FormatFloat(iaStats.WaitTime.Seconds(), 'f', -1, 64)) + } if len(s.CurrentDB) > 0 { writeSlowLogItem(&buf, SlowLogDBStr, strings.ToLower(s.CurrentDB)) diff --git a/pkg/sessionctx/variable/tests/session_test.go b/pkg/sessionctx/variable/tests/session_test.go index f80f733d526e4..25e5edf470bf4 100644 --- a/pkg/sessionctx/variable/tests/session_test.go +++ b/pkg/sessionctx/variable/tests/session_test.go @@ -166,8 +166,12 @@ func TestSlowLogFormat(t *testing.T) { CopExecDetails: execdetails.CopExecDetails{ BackoffTime: time.Millisecond, ScanDetail: &util.ScanDetail{ - ProcessedKeys: 20001, - TotalKeys: 10000, + ProcessedKeys: 20001, + TotalKeys: 10000, + IaCacheHitCount: 2, + IaRemoteReadSegmentCount: 4, + IaRemoteReadSegmentBytes: 4096, + IaRemoteReadSegmentDuration: 15 * time.Millisecond, }, TimeDetail: util.TimeDetail{ ProcessTime: time.Second * time.Duration(2), @@ -241,6 +245,9 @@ func TestSlowLogFormat(t *testing.T) { # Optimize_time: 0.00000001 Opt_logical: 0.00000001 Opt_physical: 0.00000001 Opt_binding_match: 0.00000001 Opt_stats_sync_wait: 0.00000001 Opt_stats_derive: 0.00000001 # Wait_TS: 0.000000003 # Process_time: 2 Wait_time: 60 Backoff_time: 0.001 Request_count: 2 Process_keys: 20001 Total_keys: 10000 +# IA_remote_read_segment_count: 4 +# IA_remote_read_segment_size: 4096 +# IA_remote_read_segment_wait_time: 0.015 # DB: test # Index_names: [t1:a,t2:b] # Is_internal: true diff --git a/pkg/util/execdetails/execdetails.go b/pkg/util/execdetails/execdetails.go index 69d2a904589e9..bbced51a3fafa 100644 --- a/pkg/util/execdetails/execdetails.go +++ b/pkg/util/execdetails/execdetails.go @@ -180,6 +180,12 @@ const ( RocksdbBlockReadByteStr = "Rocksdb_block_read_byte" // RocksdbBlockReadTimeStr means the time spent on rocksdb block read. RocksdbBlockReadTimeStr = "Rocksdb_block_read_time" + // IARemoteReadSegmentCountStr means the number of IA remote segment reads. + IARemoteReadSegmentCountStr = "IA_remote_read_segment_count" + // IARemoteReadSegmentSizeStr means the number of bytes returned from IA remote segment reads. + IARemoteReadSegmentSizeStr = "IA_remote_read_segment_size" + // IARemoteReadSegmentWaitTimeStr means the total time spent waiting for IA remote segment reads. + IARemoteReadSegmentWaitTimeStr = "IA_remote_read_segment_wait_time" // The following constants define the set of fields for SlowQueryLogItems // that are relevant to evaluating and triggering SlowLogRules. @@ -204,6 +210,25 @@ const ( PrewriteRegionStr = "Prewrite_region" ) +// IARemoteReadSegmentStats contains IA remote-read scan statistics. +type IARemoteReadSegmentStats struct { + Count uint64 + Bytes uint64 + WaitTime time.Duration +} + +// GetIARemoteReadSegmentStats reads IA remote-read scan statistics from client-go ScanDetail. +func GetIARemoteReadSegmentStats(scanDetail *util.ScanDetail) IARemoteReadSegmentStats { + if scanDetail == nil { + return IARemoteReadSegmentStats{} + } + return IARemoteReadSegmentStats{ + Count: scanDetail.IaRemoteReadSegmentCount, + Bytes: scanDetail.IaRemoteReadSegmentBytes, + WaitTime: scanDetail.IaRemoteReadSegmentDuration, + } +} + // String implements the fmt.Stringer interface. func (d ExecDetails) String() string { parts := make([]string, 0, 8) diff --git a/pkg/util/execdetails/execdetails_test.go b/pkg/util/execdetails/execdetails_test.go index 2369dda4cb37e..6f980f5f697d2 100644 --- a/pkg/util/execdetails/execdetails_test.go +++ b/pkg/util/execdetails/execdetails_test.go @@ -1188,3 +1188,19 @@ func TestRURuntimeStatsMergeKeepsExistingRUVersion(t *testing.T) { dst.Merge(src) require.Equal(t, rmclient.RUVersionV1, dst.RUVersion) } + +func TestGetIARemoteReadSegmentStats(t *testing.T) { + stats := GetIARemoteReadSegmentStats(&util.ScanDetail{ + IaRemoteReadSegmentCount: 3, + IaRemoteReadSegmentBytes: 4096, + IaRemoteReadSegmentDuration: 5 * time.Millisecond, + }) + require.Equal(t, IARemoteReadSegmentStats{ + Count: 3, + Bytes: 4096, + WaitTime: 5 * time.Millisecond, + }, stats) + + require.Equal(t, IARemoteReadSegmentStats{}, GetIARemoteReadSegmentStats(&util.ScanDetail{})) + require.Equal(t, IARemoteReadSegmentStats{}, GetIARemoteReadSegmentStats(nil)) +} diff --git a/pkg/util/stmtsummary/BUILD.bazel b/pkg/util/stmtsummary/BUILD.bazel index f98a374797061..b360664d4eb1f 100644 --- a/pkg/util/stmtsummary/BUILD.bazel +++ b/pkg/util/stmtsummary/BUILD.bazel @@ -51,6 +51,7 @@ go_test( "//pkg/testkit/testsetup", "//pkg/types", "//pkg/util", + "//pkg/util/chunk", "//pkg/util/execdetails", "//pkg/util/hack", "//pkg/util/plancodec", diff --git a/pkg/util/stmtsummary/evicted.go b/pkg/util/stmtsummary/evicted.go index fec9e5f9399a6..2aa66dac0d4b4 100644 --- a/pkg/util/stmtsummary/evicted.go +++ b/pkg/util/stmtsummary/evicted.go @@ -302,6 +302,18 @@ func addInfo(addTo *stmtSummaryByDigestElement, addWith *stmtSummaryByDigestElem if addTo.maxRocksdbBlockReadByte < addWith.maxRocksdbBlockReadByte { addTo.maxRocksdbBlockReadByte = addWith.maxRocksdbBlockReadByte } + addTo.sumIAReadSegmentCount += addWith.sumIAReadSegmentCount + if addTo.maxIAReadSegmentCount < addWith.maxIAReadSegmentCount { + addTo.maxIAReadSegmentCount = addWith.maxIAReadSegmentCount + } + addTo.sumIARemoteReadSegmentSize += addWith.sumIARemoteReadSegmentSize + if addTo.maxIARemoteReadSegmentSize < addWith.maxIARemoteReadSegmentSize { + addTo.maxIARemoteReadSegmentSize = addWith.maxIARemoteReadSegmentSize + } + addTo.sumIARemoteReadSegmentWaitTime += addWith.sumIARemoteReadSegmentWaitTime + if addTo.maxIARemoteReadSegmentWaitTime < addWith.maxIARemoteReadSegmentWaitTime { + addTo.maxIARemoteReadSegmentWaitTime = addWith.maxIARemoteReadSegmentWaitTime + } // txn addTo.commitCount += addWith.commitCount diff --git a/pkg/util/stmtsummary/reader.go b/pkg/util/stmtsummary/reader.go index 90edc89cbbb33..3aeba542eded5 100644 --- a/pkg/util/stmtsummary/reader.go +++ b/pkg/util/stmtsummary/reader.go @@ -302,6 +302,12 @@ const ( MaxRocksdbBlockReadCountStr = "MAX_ROCKSDB_BLOCK_READ_COUNT" AvgRocksdbBlockReadByteStr = "AVG_ROCKSDB_BLOCK_READ_BYTE" MaxRocksdbBlockReadByteStr = "MAX_ROCKSDB_BLOCK_READ_BYTE" + AvgIAReadSegmentCountStr = "AVG_IA_READ_SEGMENT_COUNT" + MaxIAReadSegmentCountStr = "MAX_IA_READ_SEGMENT_COUNT" + AvgIARemoteReadSegmentSizeStr = "AVG_IA_REMOTE_READ_SEGMENT_SIZE" + MaxIARemoteReadSegmentSizeStr = "MAX_IA_REMOTE_READ_SEGMENT_SIZE" + AvgIARemoteReadSegmentWaitTimeStr = "AVG_IA_REMOTE_READ_SEGMENT_WAIT_TIME" + MaxIARemoteReadSegmentWaitTimeStr = "MAX_IA_REMOTE_READ_SEGMENT_WAIT_TIME" AvgPrewriteTimeStr = "AVG_PREWRITE_TIME" MaxPrewriteTimeStr = "MAX_PREWRITE_TIME" AvgCommitTimeStr = "AVG_COMMIT_TIME" @@ -651,6 +657,24 @@ var columnValueFactoryMap = map[string]columnValueFactory{ MaxRocksdbBlockReadByteStr: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { return ssStats.maxRocksdbBlockReadByte }, + AvgIAReadSegmentCountStr: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { + return avgFloat(int64(ssStats.sumIAReadSegmentCount), ssStats.execCount) + }, + MaxIAReadSegmentCountStr: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { + return ssStats.maxIAReadSegmentCount + }, + AvgIARemoteReadSegmentSizeStr: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { + return avgFloat(int64(ssStats.sumIARemoteReadSegmentSize), ssStats.execCount) + }, + MaxIARemoteReadSegmentSizeStr: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { + return ssStats.maxIARemoteReadSegmentSize + }, + AvgIARemoteReadSegmentWaitTimeStr: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { + return avgInt(int64(ssStats.sumIARemoteReadSegmentWaitTime), ssStats.execCount) + }, + MaxIARemoteReadSegmentWaitTimeStr: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { + return int64(ssStats.maxIARemoteReadSegmentWaitTime) + }, PrewriteTimeStr: func(_ *stmtSummaryReader, _ *stmtSummaryByDigestElement, _ *stmtSummaryByDigest, ssStats *stmtSummaryStats) any { return int64(ssStats.sumPrewriteTime) }, diff --git a/pkg/util/stmtsummary/statement_summary.go b/pkg/util/stmtsummary/statement_summary.go index d81e1a72c2275..b566ea72a63e2 100644 --- a/pkg/util/stmtsummary/statement_summary.go +++ b/pkg/util/stmtsummary/statement_summary.go @@ -179,26 +179,32 @@ type stmtSummaryStats struct { maxCopWaitTime time.Duration maxCopWaitAddress string // TiKV - sumProcessTime time.Duration - maxProcessTime time.Duration - sumWaitTime time.Duration - maxWaitTime time.Duration - sumBackoffTime time.Duration - maxBackoffTime time.Duration - sumTotalKeys int64 - maxTotalKeys int64 - sumProcessedKeys int64 - maxProcessedKeys int64 - sumRocksdbDeleteSkippedCount uint64 - maxRocksdbDeleteSkippedCount uint64 - sumRocksdbKeySkippedCount uint64 - maxRocksdbKeySkippedCount uint64 - sumRocksdbBlockCacheHitCount uint64 - maxRocksdbBlockCacheHitCount uint64 - sumRocksdbBlockReadCount uint64 - maxRocksdbBlockReadCount uint64 - sumRocksdbBlockReadByte uint64 - maxRocksdbBlockReadByte uint64 + sumProcessTime time.Duration + maxProcessTime time.Duration + sumWaitTime time.Duration + maxWaitTime time.Duration + sumBackoffTime time.Duration + maxBackoffTime time.Duration + sumTotalKeys int64 + maxTotalKeys int64 + sumProcessedKeys int64 + maxProcessedKeys int64 + sumRocksdbDeleteSkippedCount uint64 + maxRocksdbDeleteSkippedCount uint64 + sumRocksdbKeySkippedCount uint64 + maxRocksdbKeySkippedCount uint64 + sumRocksdbBlockCacheHitCount uint64 + maxRocksdbBlockCacheHitCount uint64 + sumRocksdbBlockReadCount uint64 + maxRocksdbBlockReadCount uint64 + sumRocksdbBlockReadByte uint64 + maxRocksdbBlockReadByte uint64 + sumIAReadSegmentCount uint64 + maxIAReadSegmentCount uint64 + sumIARemoteReadSegmentSize uint64 + maxIARemoteReadSegmentSize uint64 + sumIARemoteReadSegmentWaitTime time.Duration + maxIARemoteReadSegmentWaitTime time.Duration // txn commitCount int64 sumGetCommitTsTime time.Duration @@ -868,6 +874,19 @@ func (ssStats *stmtSummaryStats) add(sei *StmtExecInfo, warningCount int, affect if sei.ExecDetail.ScanDetail.RocksdbBlockReadByte > ssStats.maxRocksdbBlockReadByte { ssStats.maxRocksdbBlockReadByte = sei.ExecDetail.ScanDetail.RocksdbBlockReadByte } + iaStats := execdetails.GetIARemoteReadSegmentStats(sei.ExecDetail.ScanDetail) + ssStats.sumIAReadSegmentCount += iaStats.Count + if iaStats.Count > ssStats.maxIAReadSegmentCount { + ssStats.maxIAReadSegmentCount = iaStats.Count + } + ssStats.sumIARemoteReadSegmentSize += iaStats.Bytes + if iaStats.Bytes > ssStats.maxIARemoteReadSegmentSize { + ssStats.maxIARemoteReadSegmentSize = iaStats.Bytes + } + ssStats.sumIARemoteReadSegmentWaitTime += iaStats.WaitTime + if iaStats.WaitTime > ssStats.maxIARemoteReadSegmentWaitTime { + ssStats.maxIARemoteReadSegmentWaitTime = iaStats.WaitTime + } } // txn diff --git a/pkg/util/stmtsummary/statement_summary_test.go b/pkg/util/stmtsummary/statement_summary_test.go index bdf4fac921c78..7a80d626524cc 100644 --- a/pkg/util/stmtsummary/statement_summary_test.go +++ b/pkg/util/stmtsummary/statement_summary_test.go @@ -32,6 +32,7 @@ import ( "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/types" tidbutil "github.com/pingcap/tidb/pkg/util" + "github.com/pingcap/tidb/pkg/util/chunk" "github.com/pingcap/tidb/pkg/util/execdetails" "github.com/pingcap/tidb/pkg/util/hack" "github.com/pingcap/tidb/pkg/util/plancodec" @@ -926,6 +927,10 @@ func newStmtSummaryReaderForTest(ssMap *stmtSummaryByDigestMap) *stmtSummaryRead StorageKVStr, StorageMPPStr, } + return newStmtSummaryReaderWithColumnNamesForTest(ssMap, columnNames...) +} + +func newStmtSummaryReaderWithColumnNamesForTest(ssMap *stmtSummaryByDigestMap, columnNames ...string) *stmtSummaryReader { cols := make([]*model.ColumnInfo, len(columnNames)) for i := range columnNames { cols[i] = &model.ColumnInfo{ @@ -1053,6 +1058,96 @@ func TestToDatum(t *testing.T) { match(t, datums[1], expectedEvictedDatum...) } +func TestToDatumIAColumns(t *testing.T) { + ssMap := newStmtSummaryByDigestMap() + now := time.Now().Unix() + ssMap.beginTimeForCurInterval = now + 60 + + stmtExecInfo1 := generateAnyExecInfo() + stmtExecInfo1.ExecDetail.ScanDetail.IaRemoteReadSegmentCount = 3 + stmtExecInfo1.ExecDetail.ScanDetail.IaRemoteReadSegmentBytes = 4096 + stmtExecInfo1.ExecDetail.ScanDetail.IaRemoteReadSegmentDuration = 5 * time.Millisecond + + stmtExecInfo2 := generateAnyExecInfo() + stmtExecInfo2.ExecDetail.ScanDetail.IaRemoteReadSegmentCount = 5 + stmtExecInfo2.ExecDetail.ScanDetail.IaRemoteReadSegmentBytes = 8192 + stmtExecInfo2.ExecDetail.ScanDetail.IaRemoteReadSegmentDuration = 9 * time.Millisecond + + ssMap.AddStatement(stmtExecInfo1) + ssMap.AddStatement(stmtExecInfo2) + reader := newStmtSummaryReaderWithColumnNamesForTest( + ssMap, + AvgIAReadSegmentCountStr, + MaxIAReadSegmentCountStr, + AvgIARemoteReadSegmentSizeStr, + MaxIARemoteReadSegmentSizeStr, + AvgIARemoteReadSegmentWaitTimeStr, + MaxIARemoteReadSegmentWaitTimeStr, + ) + + rows := reader.GetStmtSummaryCurrentRows() + require.Len(t, rows, 1) + require.Equal(t, 4.0, rows[0][0].GetFloat64()) + require.Equal(t, uint64(5), rows[0][1].GetUint64()) + require.Equal(t, 6144.0, rows[0][2].GetFloat64()) + require.Equal(t, uint64(8192), rows[0][3].GetUint64()) + require.Equal(t, int64(7*time.Millisecond), rows[0][4].GetInt64()) + require.Equal(t, int64(9*time.Millisecond), rows[0][5].GetInt64()) +} + +func TestToDatumIAColumnsChunkRoundTrip(t *testing.T) { + ssMap := newStmtSummaryByDigestMap() + now := time.Now().Unix() + ssMap.beginTimeForCurInterval = now + 60 + + stmtExecInfo1 := generateAnyExecInfo() + stmtExecInfo1.ExecDetail.ScanDetail.IaRemoteReadSegmentCount = 3 + stmtExecInfo1.ExecDetail.ScanDetail.IaRemoteReadSegmentBytes = 4096 + stmtExecInfo1.ExecDetail.ScanDetail.IaRemoteReadSegmentDuration = 5 * time.Millisecond + + stmtExecInfo2 := generateAnyExecInfo() + stmtExecInfo2.ExecDetail.ScanDetail.IaRemoteReadSegmentCount = 5 + stmtExecInfo2.ExecDetail.ScanDetail.IaRemoteReadSegmentBytes = 8192 + stmtExecInfo2.ExecDetail.ScanDetail.IaRemoteReadSegmentDuration = 9 * time.Millisecond + + ssMap.AddStatement(stmtExecInfo1) + ssMap.AddStatement(stmtExecInfo2) + + reader := newStmtSummaryReaderWithColumnNamesForTest( + ssMap, + AvgIAReadSegmentCountStr, + MaxIAReadSegmentCountStr, + AvgIARemoteReadSegmentSizeStr, + MaxIARemoteReadSegmentSizeStr, + AvgIARemoteReadSegmentWaitTimeStr, + MaxIARemoteReadSegmentWaitTimeStr, + ) + + rows := reader.GetStmtSummaryCurrentRows() + require.Len(t, rows, 1) + + maxUnsignedType := types.NewFieldType(mysql.TypeLonglong) + maxUnsignedType.SetFlag(mysql.UnsignedFlag) + retTypes := []*types.FieldType{ + types.NewFieldType(mysql.TypeDouble), + maxUnsignedType, + types.NewFieldType(mysql.TypeDouble), + maxUnsignedType.Clone(), + types.NewFieldType(mysql.TypeLonglong), + types.NewFieldType(mysql.TypeLonglong), + } + mutRow := chunk.MutRowFromTypes(retTypes) + mutRow.SetDatums(rows[0]...) + row := mutRow.ToRow() + + require.Equal(t, 4.0, row.GetFloat64(0)) + require.Equal(t, uint64(5), row.GetUint64(1)) + require.Equal(t, 6144.0, row.GetFloat64(2)) + require.Equal(t, uint64(8192), row.GetUint64(3)) + require.Equal(t, int64(7*time.Millisecond), row.GetInt64(4)) + require.Equal(t, int64(9*time.Millisecond), row.GetInt64(5)) +} + // Test AddStatement and ToDatum parallel. func TestAddStatementParallel(t *testing.T) { ssMap := newStmtSummaryByDigestMap() diff --git a/pkg/util/stmtsummary/v2/BUILD.bazel b/pkg/util/stmtsummary/v2/BUILD.bazel index 2ad5781b7b0b9..6cc0cac2ca4c4 100644 --- a/pkg/util/stmtsummary/v2/BUILD.bazel +++ b/pkg/util/stmtsummary/v2/BUILD.bazel @@ -56,9 +56,11 @@ go_test( "//pkg/metrics", "//pkg/parser/ast", "//pkg/parser/auth", + "//pkg/parser/mysql", "//pkg/testkit/testsetup", "//pkg/types", "//pkg/util", + "//pkg/util/chunk", "//pkg/util/set", "//pkg/util/stmtsummary", "@com_github_prometheus_client_golang//prometheus", diff --git a/pkg/util/stmtsummary/v2/column.go b/pkg/util/stmtsummary/v2/column.go index 68ec4e170a88e..e09044c37c6e6 100644 --- a/pkg/util/stmtsummary/v2/column.go +++ b/pkg/util/stmtsummary/v2/column.go @@ -78,6 +78,12 @@ const ( MaxRocksdbBlockReadCountStr = "MAX_ROCKSDB_BLOCK_READ_COUNT" AvgRocksdbBlockReadByteStr = "AVG_ROCKSDB_BLOCK_READ_BYTE" MaxRocksdbBlockReadByteStr = "MAX_ROCKSDB_BLOCK_READ_BYTE" + AvgIAReadSegmentCountStr = "AVG_IA_READ_SEGMENT_COUNT" + MaxIAReadSegmentCountStr = "MAX_IA_READ_SEGMENT_COUNT" + AvgIARemoteReadSegmentSizeStr = "AVG_IA_REMOTE_READ_SEGMENT_SIZE" + MaxIARemoteReadSegmentSizeStr = "MAX_IA_REMOTE_READ_SEGMENT_SIZE" + AvgIARemoteReadSegmentWaitTimeStr = "AVG_IA_REMOTE_READ_SEGMENT_WAIT_TIME" + MaxIARemoteReadSegmentWaitTimeStr = "MAX_IA_REMOTE_READ_SEGMENT_WAIT_TIME" AvgPrewriteTimeStr = "AVG_PREWRITE_TIME" MaxPrewriteTimeStr = "MAX_PREWRITE_TIME" AvgCommitTimeStr = "AVG_COMMIT_TIME" @@ -322,6 +328,24 @@ var columnFactoryMap = map[string]columnFactory{ MaxRocksdbBlockReadByteStr: func(_ columnInfo, record *StmtRecord) any { return record.MaxRocksdbBlockReadByte }, + AvgIAReadSegmentCountStr: func(_ columnInfo, record *StmtRecord) any { + return avgFloat(int64(record.SumIAReadSegmentCount), record.ExecCount) + }, + MaxIAReadSegmentCountStr: func(_ columnInfo, record *StmtRecord) any { + return record.MaxIAReadSegmentCount + }, + AvgIARemoteReadSegmentSizeStr: func(_ columnInfo, record *StmtRecord) any { + return avgFloat(int64(record.SumIARemoteReadSegmentSize), record.ExecCount) + }, + MaxIARemoteReadSegmentSizeStr: func(_ columnInfo, record *StmtRecord) any { + return record.MaxIARemoteReadSegmentSize + }, + AvgIARemoteReadSegmentWaitTimeStr: func(_ columnInfo, record *StmtRecord) any { + return avgInt(int64(record.SumIARemoteReadSegmentWaitTime), record.ExecCount) + }, + MaxIARemoteReadSegmentWaitTimeStr: func(_ columnInfo, record *StmtRecord) any { + return int64(record.MaxIARemoteReadSegmentWaitTime) + }, AvgPrewriteTimeStr: func(_ columnInfo, record *StmtRecord) any { return avgInt(int64(record.SumPrewriteTime), record.CommitCount) }, diff --git a/pkg/util/stmtsummary/v2/column_test.go b/pkg/util/stmtsummary/v2/column_test.go index dd07fd5e22077..8508b42b39ee8 100644 --- a/pkg/util/stmtsummary/v2/column_test.go +++ b/pkg/util/stmtsummary/v2/column_test.go @@ -21,6 +21,9 @@ import ( "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/chunk" "github.com/stretchr/testify/require" ) @@ -77,6 +80,91 @@ func TestColumn(t *testing.T) { } } +func TestIAAvgColumns(t *testing.T) { + columns := []*model.ColumnInfo{ + {Name: ast.NewCIStr(AvgIAReadSegmentCountStr)}, + {Name: ast.NewCIStr(MaxIAReadSegmentCountStr)}, + {Name: ast.NewCIStr(AvgIARemoteReadSegmentSizeStr)}, + {Name: ast.NewCIStr(MaxIARemoteReadSegmentSizeStr)}, + {Name: ast.NewCIStr(AvgIARemoteReadSegmentWaitTimeStr)}, + {Name: ast.NewCIStr(MaxIARemoteReadSegmentWaitTimeStr)}, + } + factories := makeColumnFactories(columns) + + info1 := GenerateStmtExecInfo4Test("digest") + info1.ExecDetail.ScanDetail.IaRemoteReadSegmentCount = 3 + info1.ExecDetail.ScanDetail.IaRemoteReadSegmentBytes = 4096 + info1.ExecDetail.ScanDetail.IaRemoteReadSegmentDuration = 5 * time.Millisecond + + info2 := GenerateStmtExecInfo4Test("digest") + info2.ExecDetail.ScanDetail.IaRemoteReadSegmentCount = 5 + info2.ExecDetail.ScanDetail.IaRemoteReadSegmentBytes = 8192 + info2.ExecDetail.ScanDetail.IaRemoteReadSegmentDuration = 9 * time.Millisecond + + record := NewStmtRecord(info1) + record.Add(info1) + record.Add(info2) + + require.Equal(t, 4.0, factories[0](mockColumnInfo{}, record)) + require.Equal(t, uint64(5), factories[1](mockColumnInfo{}, record)) + require.Equal(t, 6144.0, factories[2](mockColumnInfo{}, record)) + require.Equal(t, uint64(8192), factories[3](mockColumnInfo{}, record)) + require.Equal(t, int64(7*time.Millisecond), factories[4](mockColumnInfo{}, record)) + require.Equal(t, int64(9*time.Millisecond), factories[5](mockColumnInfo{}, record)) +} + +func TestIAAvgColumnsChunkRoundTrip(t *testing.T) { + columns := []*model.ColumnInfo{ + {Name: ast.NewCIStr(AvgIAReadSegmentCountStr)}, + {Name: ast.NewCIStr(MaxIAReadSegmentCountStr)}, + {Name: ast.NewCIStr(AvgIARemoteReadSegmentSizeStr)}, + {Name: ast.NewCIStr(MaxIARemoteReadSegmentSizeStr)}, + {Name: ast.NewCIStr(AvgIARemoteReadSegmentWaitTimeStr)}, + {Name: ast.NewCIStr(MaxIARemoteReadSegmentWaitTimeStr)}, + } + factories := makeColumnFactories(columns) + + info1 := GenerateStmtExecInfo4Test("digest") + info1.ExecDetail.ScanDetail.IaRemoteReadSegmentCount = 3 + info1.ExecDetail.ScanDetail.IaRemoteReadSegmentBytes = 4096 + info1.ExecDetail.ScanDetail.IaRemoteReadSegmentDuration = 5 * time.Millisecond + + info2 := GenerateStmtExecInfo4Test("digest") + info2.ExecDetail.ScanDetail.IaRemoteReadSegmentCount = 5 + info2.ExecDetail.ScanDetail.IaRemoteReadSegmentBytes = 8192 + info2.ExecDetail.ScanDetail.IaRemoteReadSegmentDuration = 9 * time.Millisecond + + record := NewStmtRecord(info1) + record.Add(info1) + record.Add(info2) + + rowDatums := make([]types.Datum, len(factories)) + for i, factory := range factories { + rowDatums[i] = types.NewDatum(factory(mockColumnInfo{}, record)) + } + + maxUnsignedType := types.NewFieldType(mysql.TypeLonglong) + maxUnsignedType.SetFlag(mysql.UnsignedFlag) + retTypes := []*types.FieldType{ + types.NewFieldType(mysql.TypeDouble), + maxUnsignedType, + types.NewFieldType(mysql.TypeDouble), + maxUnsignedType.Clone(), + types.NewFieldType(mysql.TypeLonglong), + types.NewFieldType(mysql.TypeLonglong), + } + mutRow := chunk.MutRowFromTypes(retTypes) + mutRow.SetDatums(rowDatums...) + row := mutRow.ToRow() + + require.Equal(t, 4.0, row.GetFloat64(0)) + require.Equal(t, uint64(5), row.GetUint64(1)) + require.Equal(t, 6144.0, row.GetFloat64(2)) + require.Equal(t, uint64(8192), row.GetUint64(3)) + require.Equal(t, int64(7*time.Millisecond), row.GetInt64(4)) + require.Equal(t, int64(9*time.Millisecond), row.GetInt64(5)) +} + type mockColumnInfo struct{} func (mockColumnInfo) getInstanceAddr() string { diff --git a/pkg/util/stmtsummary/v2/record.go b/pkg/util/stmtsummary/v2/record.go index fdd9a2de1d471..e54f8bd4308d9 100644 --- a/pkg/util/stmtsummary/v2/record.go +++ b/pkg/util/stmtsummary/v2/record.go @@ -77,26 +77,32 @@ type StmtRecord struct { MaxCopWaitTime time.Duration `json:"max_cop_wait_time"` MaxCopWaitAddress string `json:"max_cop_wait_address"` // TiKV - SumProcessTime time.Duration `json:"sum_process_time"` - MaxProcessTime time.Duration `json:"max_process_time"` - SumWaitTime time.Duration `json:"sum_wait_time"` - MaxWaitTime time.Duration `json:"max_wait_time"` - SumBackoffTime time.Duration `json:"sum_backoff_time"` - MaxBackoffTime time.Duration `json:"max_backoff_time"` - SumTotalKeys int64 `json:"sum_total_keys"` - MaxTotalKeys int64 `json:"max_total_keys"` - SumProcessedKeys int64 `json:"sum_processed_keys"` - MaxProcessedKeys int64 `json:"max_processed_keys"` - SumRocksdbDeleteSkippedCount uint64 `json:"sum_rocksdb_delete_skipped_count"` - MaxRocksdbDeleteSkippedCount uint64 `json:"max_rocksdb_delete_skipped_count"` - SumRocksdbKeySkippedCount uint64 `json:"sum_rocksdb_key_skipped_count"` - MaxRocksdbKeySkippedCount uint64 `json:"max_rocksdb_key_skipped_count"` - SumRocksdbBlockCacheHitCount uint64 `json:"sum_rocksdb_block_cache_hit_count"` - MaxRocksdbBlockCacheHitCount uint64 `json:"max_rocksdb_block_cache_hit_count"` - SumRocksdbBlockReadCount uint64 `json:"sum_rocksdb_block_read_count"` - MaxRocksdbBlockReadCount uint64 `json:"max_rocksdb_block_read_count"` - SumRocksdbBlockReadByte uint64 `json:"sum_rocksdb_block_read_byte"` - MaxRocksdbBlockReadByte uint64 `json:"max_rocksdb_block_read_byte"` + SumProcessTime time.Duration `json:"sum_process_time"` + MaxProcessTime time.Duration `json:"max_process_time"` + SumWaitTime time.Duration `json:"sum_wait_time"` + MaxWaitTime time.Duration `json:"max_wait_time"` + SumBackoffTime time.Duration `json:"sum_backoff_time"` + MaxBackoffTime time.Duration `json:"max_backoff_time"` + SumTotalKeys int64 `json:"sum_total_keys"` + MaxTotalKeys int64 `json:"max_total_keys"` + SumProcessedKeys int64 `json:"sum_processed_keys"` + MaxProcessedKeys int64 `json:"max_processed_keys"` + SumRocksdbDeleteSkippedCount uint64 `json:"sum_rocksdb_delete_skipped_count"` + MaxRocksdbDeleteSkippedCount uint64 `json:"max_rocksdb_delete_skipped_count"` + SumRocksdbKeySkippedCount uint64 `json:"sum_rocksdb_key_skipped_count"` + MaxRocksdbKeySkippedCount uint64 `json:"max_rocksdb_key_skipped_count"` + SumRocksdbBlockCacheHitCount uint64 `json:"sum_rocksdb_block_cache_hit_count"` + MaxRocksdbBlockCacheHitCount uint64 `json:"max_rocksdb_block_cache_hit_count"` + SumRocksdbBlockReadCount uint64 `json:"sum_rocksdb_block_read_count"` + MaxRocksdbBlockReadCount uint64 `json:"max_rocksdb_block_read_count"` + SumRocksdbBlockReadByte uint64 `json:"sum_rocksdb_block_read_byte"` + MaxRocksdbBlockReadByte uint64 `json:"max_rocksdb_block_read_byte"` + SumIAReadSegmentCount uint64 `json:"sum_ia_read_segment_count"` + MaxIAReadSegmentCount uint64 `json:"max_ia_read_segment_count"` + SumIARemoteReadSegmentSize uint64 `json:"sum_ia_remote_read_segment_size"` + MaxIARemoteReadSegmentSize uint64 `json:"max_ia_remote_read_segment_size"` + SumIARemoteReadSegmentWaitTime time.Duration `json:"sum_ia_remote_read_segment_wait_time"` + MaxIARemoteReadSegmentWaitTime time.Duration `json:"max_ia_remote_read_segment_wait_time"` // Txn CommitCount int64 `json:"commit_count"` SumGetCommitTsTime time.Duration `json:"sum_get_commit_ts_time"` @@ -320,6 +326,19 @@ func (r *StmtRecord) Add(info *stmtsummary.StmtExecInfo) { if info.ExecDetail.ScanDetail.RocksdbBlockReadByte > r.MaxRocksdbBlockReadByte { r.MaxRocksdbBlockReadByte = info.ExecDetail.ScanDetail.RocksdbBlockReadByte } + iaStats := execdetails.GetIARemoteReadSegmentStats(info.ExecDetail.ScanDetail) + r.SumIAReadSegmentCount += iaStats.Count + if iaStats.Count > r.MaxIAReadSegmentCount { + r.MaxIAReadSegmentCount = iaStats.Count + } + r.SumIARemoteReadSegmentSize += iaStats.Bytes + if iaStats.Bytes > r.MaxIARemoteReadSegmentSize { + r.MaxIARemoteReadSegmentSize = iaStats.Bytes + } + r.SumIARemoteReadSegmentWaitTime += iaStats.WaitTime + if iaStats.WaitTime > r.MaxIARemoteReadSegmentWaitTime { + r.MaxIARemoteReadSegmentWaitTime = iaStats.WaitTime + } } // Txn commitDetails := info.ExecDetail.CommitDetail @@ -525,6 +544,18 @@ func (r *StmtRecord) Merge(other *StmtRecord) { if r.MaxRocksdbBlockReadByte < other.MaxRocksdbBlockReadByte { r.MaxRocksdbBlockReadByte = other.MaxRocksdbBlockReadByte } + r.SumIAReadSegmentCount += other.SumIAReadSegmentCount + if r.MaxIAReadSegmentCount < other.MaxIAReadSegmentCount { + r.MaxIAReadSegmentCount = other.MaxIAReadSegmentCount + } + r.SumIARemoteReadSegmentSize += other.SumIARemoteReadSegmentSize + if r.MaxIARemoteReadSegmentSize < other.MaxIARemoteReadSegmentSize { + r.MaxIARemoteReadSegmentSize = other.MaxIARemoteReadSegmentSize + } + r.SumIARemoteReadSegmentWaitTime += other.SumIARemoteReadSegmentWaitTime + if r.MaxIARemoteReadSegmentWaitTime < other.MaxIARemoteReadSegmentWaitTime { + r.MaxIARemoteReadSegmentWaitTime = other.MaxIARemoteReadSegmentWaitTime + } // Txn r.CommitCount += other.CommitCount r.SumPrewriteTime += other.SumPrewriteTime