From c84b8a795982ba2389486af649731aae7fe5933f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Eeden?= Date: Tue, 16 Jun 2026 09:17:12 +0200 Subject: [PATCH 1/3] Modernize dm (excluding dm/pkg) --- cmd/dm-worker/main_test.go | 2 +- dm/chaos/cases/db.go | 2 +- dm/chaos/cases/diff.go | 2 +- dm/chaos/cases/generator.go | 2 +- dm/chaos/cases/member.go | 2 +- dm/chaos/cases/task.go | 4 +- dm/checker/checker.go | 10 +- dm/config/checking_item.go | 5 +- dm/config/checking_item_test.go | 5 +- dm/config/dbconfig/config.go | 5 +- dm/config/security_test.go | 2 +- dm/config/source_config.go | 34 +-- dm/config/source_config_test.go | 2 +- dm/config/subtask_test.go | 22 +- dm/config/task.go | 8 +- dm/config/task_test.go | 15 +- dm/ctl/common/config.go | 4 +- dm/ctl/common/util.go | 14 +- dm/ctl/master/operate_task.go | 6 +- dm/dumpling/dumpling.go | 4 +- dm/loader/checkpoint.go | 4 +- dm/loader/lightning.go | 11 +- dm/master/agent_pool.go | 4 +- dm/master/agent_pool_test.go | 14 +- dm/master/bootstrap_test.go | 2 +- dm/master/etcd_test.go | 4 +- dm/master/openapi_view_test.go | 2 +- dm/master/scheduler/latch_test.go | 16 +- dm/master/scheduler/scheduler.go | 21 +- dm/master/scheduler/scheduler_test.go | 78 ++--- dm/master/server.go | 52 ++-- dm/master/server_test.go | 34 +-- dm/master/shardddl/optimist.go | 6 +- dm/master/shardddl/pessimist.go | 6 +- dm/master/shardddl/pessimist_test.go | 24 +- dm/relay/binlog_writer.go | 6 +- dm/relay/file.go | 2 +- dm/relay/file_util_test.go | 4 +- dm/relay/local_reader.go | 18 +- dm/relay/local_reader_test.go | 10 +- dm/relay/meta_test.go | 2 +- dm/relay/purge_strategy.go | 20 +- dm/relay/purger.go | 6 +- dm/relay/relay.go | 8 +- dm/relay/relay_test.go | 8 +- dm/relay/relay_writer_test.go | 4 +- dm/relay/remote_retry_test.go | 4 +- dm/simulator/mcp/mcp_test.go | 18 +- dm/simulator/mcp/uk.go | 33 +-- dm/simulator/mcp/uk_test.go | 20 +- dm/simulator/sqlgen/impl.go | 4 +- dm/simulator/sqlgen/impl_test.go | 18 +- .../binlogstream/binlog_locations_test.go | 24 +- dm/syncer/binlogstream/stream_modifier.go | 6 +- .../binlogstream/streamer_controller_test.go | 2 +- dm/syncer/causality_test.go | 18 +- dm/syncer/checkpoint.go | 31 +- dm/syncer/checkpoint_flush_worker.go | 2 +- dm/syncer/checkpoint_test.go | 14 +- dm/syncer/compactor_test.go | 55 ++-- dm/syncer/data_validator.go | 10 +- dm/syncer/data_validator_test.go | 26 +- dm/syncer/dbconn/db.go | 16 +- dm/syncer/dml.go | 28 +- dm/syncer/dml_test.go | 76 ++--- dm/syncer/dml_worker.go | 8 +- dm/syncer/dml_worker_test.go | 66 ++--- dm/syncer/expr_filter_group.go | 2 +- dm/syncer/expr_filter_group_test.go | 108 +++---- dm/syncer/job_test.go | 2 +- dm/syncer/online-ddl-tools/online_ddl.go | 6 +- dm/syncer/shardddl/optimist.go | 2 +- dm/syncer/shardddl/pessimist.go | 2 +- dm/syncer/sharding-meta/shardmeta.go | 20 +- dm/syncer/sharding-meta/shardmeta_test.go | 14 +- dm/syncer/sharding_group.go | 15 +- dm/syncer/status.go | 2 +- dm/syncer/status_test.go | 8 +- dm/syncer/syncer.go | 37 +-- dm/syncer/syncer_test.go | 274 +++++++++--------- dm/syncer/validate_worker.go | 16 +- dm/syncer/validate_worker_test.go | 34 +-- dm/syncer/validator_checkpoint.go | 38 +-- dm/syncer/validator_checkpoint_test.go | 6 +- dm/syncer/validator_cond.go | 4 +- dm/syncer/validator_cond_test.go | 6 +- dm/unit/unit.go | 2 +- dm/worker/join.go | 2 +- dm/worker/relay.go | 12 +- dm/worker/relay_test.go | 4 +- dm/worker/server.go | 6 +- dm/worker/server_test.go | 14 +- dm/worker/source_worker.go | 18 +- dm/worker/source_worker_test.go | 18 +- dm/worker/status.go | 2 +- dm/worker/subtask.go | 19 +- dm/worker/subtask_holder.go | 5 +- dm/worker/subtask_test.go | 12 +- dm/worker/task_checker.go | 6 +- dm/worker/task_checker_test.go | 4 +- 100 files changed, 781 insertions(+), 934 deletions(-) diff --git a/cmd/dm-worker/main_test.go b/cmd/dm-worker/main_test.go index 1e3f7dcee0..29caa27b8d 100644 --- a/cmd/dm-worker/main_test.go +++ b/cmd/dm-worker/main_test.go @@ -32,7 +32,7 @@ func TestRunMain(_ *testing.T) { var ( args []string exit = make(chan int) - waitCh = make(chan interface{}, 1) + waitCh = make(chan any, 1) ) for _, arg := range os.Args { switch { diff --git a/dm/chaos/cases/db.go b/dm/chaos/cases/db.go index daf80cb985..6e0cc2a6e8 100644 --- a/dm/chaos/cases/db.go +++ b/dm/chaos/cases/db.go @@ -86,7 +86,7 @@ func (c *dbConn) execSQLs(ctx context.Context, queries ...string) error { } _, _, err := c.baseConn.ApplyRetryStrategy(tcontext.NewContext(ctx, log.L()), params, - func(tctx *tcontext.Context) (interface{}, error) { + func(tctx *tcontext.Context) (any, error) { ret, err2 := c.baseConn.ExecuteSQLWithIgnoreError(tctx, nil, "chaos-cases", ignoreExecSQLError, queries) return ret, err2 }) diff --git a/dm/chaos/cases/diff.go b/dm/chaos/cases/diff.go index cbf217ab4d..d98b061b71 100644 --- a/dm/chaos/cases/diff.go +++ b/dm/chaos/cases/diff.go @@ -28,7 +28,7 @@ import ( // diffDataLoop checks whether target has the same data with source via `sync-diff-inspector` multiple times. func diffDataLoop(ctx context.Context, count int, interval time.Duration, schema string, tables []string, targetDB *sql.DB, sourceDBs ...*sql.DB) (err error) { - for i := 0; i < count; i++ { + for i := range count { select { case <-ctx.Done(): return nil diff --git a/dm/chaos/cases/generator.go b/dm/chaos/cases/generator.go index 3c50a6d1e0..f5a2921dbb 100644 --- a/dm/chaos/cases/generator.go +++ b/dm/chaos/cases/generator.go @@ -222,7 +222,7 @@ func (g *CaseGenerator) genSQLs(ctx context.Context) { for _, table := range g.tables { rand.Shuffle(len(g.testCases), func(i, j int) { g.testCases[i], g.testCases[j] = g.testCases[j], g.testCases[i] }) casesNum := rand.Intn(len(g.testCases) + 1) - for i := 0; i < casesNum; i++ { + for i := range casesNum { for _, sqls := range g.testCases[i] { fullSqls := make(SQLs, len(sqls)) copy(fullSqls, sqls) diff --git a/dm/chaos/cases/member.go b/dm/chaos/cases/member.go index d7b3e9b621..91154a5b0c 100644 --- a/dm/chaos/cases/member.go +++ b/dm/chaos/cases/member.go @@ -30,7 +30,7 @@ const ( // checkMembersReadyLoop checks whether all DM-master and DM-worker members have been ready. // NOTE: in this chaos case, we ensure 3 DM-master and 3 DM-worker started. func checkMembersReadyLoop(ctx context.Context, cli pb.MasterClient, masterCount, workerCount int) (err error) { - for i := 0; i < checkMemberTimes; i++ { + for range checkMemberTimes { select { case <-ctx.Done(): return nil diff --git a/dm/chaos/cases/task.go b/dm/chaos/cases/task.go index 24cece74e3..88fa6109db 100644 --- a/dm/chaos/cases/task.go +++ b/dm/chaos/cases/task.go @@ -224,7 +224,7 @@ func (t *task) genFullData() error { ) // generate `CREATE TABLE` statements. - for i := 0; i < tableCount; i++ { + for range tableCount { query, name, err := t.ss[0].CreateTableStmt() if err != nil { return err @@ -258,7 +258,7 @@ func (t *task) genFullData() error { for _, conn := range t.sourceConns { conn2 := conn eg.Go(func() error { - for i := 0; i < fullInsertCount; i++ { + for range fullInsertCount { query, _, err2 := t.ss[0].InsertStmt(false) if err2 != nil { return err2 diff --git a/dm/checker/checker.go b/dm/checker/checker.go index 69e6eb076a..851b5ce79f 100644 --- a/dm/checker/checker.go +++ b/dm/checker/checker.go @@ -235,7 +235,7 @@ func (c *Checker) getTablePairInfo(ctx context.Context) (info *tablePairInfo, er pool := checker.NewWorkerPoolWithContext[job, int64](ctx, func(result int64) { info.totalDataSize.Add(result) }) - for i := 0; i < concurrency; i++ { + for range concurrency { pool.Go(func(ctx context.Context, job job) (int64, error) { return conn.FetchTableEstimatedBytes( ctx, @@ -644,7 +644,7 @@ func (c *Checker) Process(ctx context.Context, pr chan pb.ProcessResult) { if result.Summary.Successful != result.Summary.Total { rawResult, err = json.MarshalIndent(result, "\t", "\t") if err != nil { - rawResult = []byte(fmt.Sprintf("marshal error %v", err)) + rawResult = fmt.Appendf(nil, "marshal error %v", err) } } c.result.Lock() @@ -830,14 +830,14 @@ func (c *Checker) IsFreshTask() (bool, error) { } // Status implements Unit interface. -func (c *Checker) Status(_ *binlog.SourceStatus) interface{} { +func (c *Checker) Status(_ *binlog.SourceStatus) any { c.result.RLock() res := c.result.detail c.result.RUnlock() rawResult, err := json.Marshal(res) if err != nil { - rawResult = []byte(fmt.Sprintf("marshal %+v error %v", res, err)) + rawResult = fmt.Appendf(nil, "marshal %+v error %v", res, err) } return &pb.CheckStatus{ @@ -851,7 +851,7 @@ func (c *Checker) Status(_ *binlog.SourceStatus) interface{} { } // Error implements Unit interface. -func (c *Checker) Error() interface{} { +func (c *Checker) Error() any { return &pb.CheckError{} } diff --git a/dm/config/checking_item.go b/dm/config/checking_item.go index 777548b67c..f2821ceb16 100644 --- a/dm/config/checking_item.go +++ b/dm/config/checking_item.go @@ -16,6 +16,7 @@ package config import ( "bytes" "fmt" + "maps" "github.com/pingcap/tiflow/dm/pkg/terror" ) @@ -130,9 +131,7 @@ func SupportCheckingItems() string { // FilterCheckingItems filters ignored items from all checking items. func FilterCheckingItems(ignoredItems []string) map[string]string { checkingItems := make(map[string]string) - for item, desc := range AllCheckingItems { - checkingItems[item] = desc - } + maps.Copy(checkingItems, AllCheckingItems) delete(checkingItems, AllChecking) for _, item := range ignoredItems { diff --git a/dm/config/checking_item_test.go b/dm/config/checking_item_test.go index 1de7b91e16..2eb6dab2b8 100644 --- a/dm/config/checking_item_test.go +++ b/dm/config/checking_item_test.go @@ -14,6 +14,7 @@ package config import ( + "maps" "testing" "github.com/stretchr/testify/require" @@ -45,9 +46,7 @@ func TestCheckingItems(t *testing.T) { // ignore shard checking items checkingItems := make(map[string]string) - for item, desc := range AllCheckingItems { - checkingItems[item] = desc - } + maps.Copy(checkingItems, AllCheckingItems) delete(checkingItems, AllChecking) require.Equal(t, checkingItems, FilterCheckingItems(ignoredCheckingItems[:0])) diff --git a/dm/config/dbconfig/config.go b/dm/config/dbconfig/config.go index da57eb98b2..64c76ec420 100644 --- a/dm/config/dbconfig/config.go +++ b/dm/config/dbconfig/config.go @@ -16,6 +16,7 @@ package dbconfig import ( "bytes" "encoding/json" + "maps" "strings" "github.com/BurntSushi/toml" @@ -150,9 +151,7 @@ func (db *DBConfig) Clone() *DBConfig { if db.Session != nil { clone.Session = make(map[string]string, len(db.Session)) - for k, v := range db.Session { - clone.Session[k] = v - } + maps.Copy(clone.Session, db.Session) } clone.Security = db.Security.Clone() diff --git a/dm/config/security_test.go b/dm/config/security_test.go index ce763f9802..cb702e1842 100644 --- a/dm/config/security_test.go +++ b/dm/config/security_test.go @@ -165,7 +165,7 @@ func (c *testTLSConfig) TestClone() { } // When add new fields, also update this value // TODO: check it - c.Require().Equal(10, reflect.TypeOf(*s).NumField()) + c.Require().Equal(10, reflect.TypeFor[security.Security]().NumField()) clone := s.Clone() c.Require().Equal(s, clone) clone.CertAllowedCN[0] = "g" diff --git a/dm/config/source_config.go b/dm/config/source_config.go index 9c0ab5e7e1..96a63e6cfc 100644 --- a/dm/config/source_config.go +++ b/dm/config/source_config.go @@ -98,7 +98,7 @@ type SourceConfig struct { ServerID uint32 `yaml:"server-id" toml:"server-id" json:"server-id"` // deprecated tracer, to keep compatibility with older version - Tracer map[string]interface{} `yaml:"tracer" toml:"tracer" json:"-"` + Tracer map[string]any `yaml:"tracer" toml:"tracer" json:"-"` CaseSensitive bool `yaml:"case-sensitive" toml:"case-sensitive" json:"case-sensitive"` Filters []*bf.BinlogEventRule `yaml:"filters" toml:"filters" json:"filters"` @@ -366,7 +366,7 @@ func (c *SourceConfig) AdjustServerID(ctx context.Context, db *conn.BaseDB) erro } rand.Seed(time.Now().UnixNano()) - for i := 0; i < 5; i++ { + for range 5 { randomValue := uint32(rand.Intn(100000)) randomServerID := defaultBaseServerID + randomValue if _, ok := serverIDs[randomServerID]; ok { @@ -403,21 +403,21 @@ func (c *SourceConfig) YamlForDowngrade() (string, error) { // This config is used for downgrade(config export) from a higher dmctl version. // When we add any new config item into SourceConfig, we should update it also. type SourceConfigForDowngrade struct { - Enable bool `yaml:"enable,omitempty"` - EnableGTID bool `yaml:"enable-gtid"` - RelayDir string `yaml:"relay-dir"` - Flavor string `yaml:"flavor"` - Charset string `yaml:"charset"` - EnableRelay bool `yaml:"enable-relay"` - RelayBinLogName string `yaml:"relay-binlog-name"` - RelayBinlogGTID string `yaml:"relay-binlog-gtid"` - UUIDSuffix int `yaml:"-"` - SourceID string `yaml:"source-id"` - From dbconfig.DBConfig `yaml:"from"` - Purge PurgeConfig `yaml:"purge"` - Checker CheckerConfig `yaml:"checker"` - ServerID uint32 `yaml:"server-id"` - Tracer map[string]interface{} `yaml:"tracer"` + Enable bool `yaml:"enable,omitempty"` + EnableGTID bool `yaml:"enable-gtid"` + RelayDir string `yaml:"relay-dir"` + Flavor string `yaml:"flavor"` + Charset string `yaml:"charset"` + EnableRelay bool `yaml:"enable-relay"` + RelayBinLogName string `yaml:"relay-binlog-name"` + RelayBinlogGTID string `yaml:"relay-binlog-gtid"` + UUIDSuffix int `yaml:"-"` + SourceID string `yaml:"source-id"` + From dbconfig.DBConfig `yaml:"from"` + Purge PurgeConfig `yaml:"purge"` + Checker CheckerConfig `yaml:"checker"` + ServerID uint32 `yaml:"server-id"` + Tracer map[string]any `yaml:"tracer"` // any new config item, we mark it omitempty CaseSensitive bool `yaml:"case-sensitive,omitempty"` Filters []*bf.BinlogEventRule `yaml:"filters,omitempty"` diff --git a/dm/config/source_config_test.go b/dm/config/source_config_test.go index 27f459bcff..e454707195 100644 --- a/dm/config/source_config_test.go +++ b/dm/config/source_config_test.go @@ -71,7 +71,7 @@ func TestConfigFunctions(t *testing.T) { require.NoError(t, err) require.Equal(t, uint32(100), cfg1.ServerID) cfg.Filters = []*bf.BinlogEventRule{} - cfg.Tracer = map[string]interface{}{} + cfg.Tracer = map[string]any{} var cfg2 SourceConfig require.NoError(t, cfg2.FromToml(originCfgStr)) diff --git a/dm/config/subtask_test.go b/dm/config/subtask_test.go index d91fc2ea03..5d40d6e4e6 100644 --- a/dm/config/subtask_test.go +++ b/dm/config/subtask_test.go @@ -462,14 +462,12 @@ func TestSubTaskConfigMarshalAtomic(t *testing.T) { require.Equal(t, cfg.DumpIOTotalBytes.Load(), uint64(200)) var wg sync.WaitGroup - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() + for range 10 { + wg.Go(func() { data, err := json.Marshal(cfg) require.NoError(t, err) - jsonMap := make(map[string]interface{}) + jsonMap := make(map[string]any) err = json.Unmarshal(data, &jsonMap) require.NoError(t, err) @@ -487,11 +485,9 @@ func TestSubTaskConfigMarshalAtomic(t *testing.T) { _, hasDumpUUID := jsonMap["dump-uuid"] require.False(t, hasUUID, "UUID should not be in JSON") require.False(t, hasDumpUUID, "DumpUUID should not be in JSON") - }() + }) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { newCfg, err := cfg.Clone() require.NoError(t, err) @@ -501,14 +497,12 @@ func TestSubTaskConfigMarshalAtomic(t *testing.T) { require.GreaterOrEqual(t, newCfg.DumpIOTotalBytes.Load(), uint64(200)) require.Equal(t, newCfg.UUID, uuid) require.Equal(t, newCfg.DumpUUID, dumpUUID) - }() + }) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { cfg.IOTotalBytes.Add(1) cfg.DumpIOTotalBytes.Add(1) - }() + }) } wg.Wait() diff --git a/dm/config/task.go b/dm/config/task.go index acea793b62..413dd36076 100644 --- a/dm/config/task.go +++ b/dm/config/task.go @@ -230,7 +230,7 @@ func DefaultMydumperConfig() MydumperConfig { type rawMydumperConfig MydumperConfig // UnmarshalYAML implements Unmarshaler.UnmarshalYAML. -func (m *MydumperConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (m *MydumperConfig) UnmarshalYAML(unmarshal func(any) error) error { raw := rawMydumperConfig(DefaultMydumperConfig()) if err := unmarshal(&raw); err != nil { return terror.ErrConfigYamlTransform.Delegate(err, "unmarshal mydumper config") @@ -324,7 +324,7 @@ func DefaultLoaderConfig() LoaderConfig { type rawLoaderConfig LoaderConfig // UnmarshalYAML implements Unmarshaler.UnmarshalYAML. -func (m *LoaderConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (m *LoaderConfig) UnmarshalYAML(unmarshal func(any) error) error { raw := rawLoaderConfig(DefaultLoaderConfig()) if err := unmarshal(&raw); err != nil { return terror.ErrConfigYamlTransform.Delegate(err, "unmarshal loader config") @@ -461,7 +461,7 @@ func DefaultSyncerConfig() SyncerConfig { type rawSyncerConfig SyncerConfig // UnmarshalYAML implements Unmarshaler.UnmarshalYAML. -func (m *SyncerConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (m *SyncerConfig) UnmarshalYAML(unmarshal func(any) error) error { raw := rawSyncerConfig(DefaultSyncerConfig()) if err := unmarshal(&raw); err != nil { return terror.ErrConfigYamlTransform.Delegate(err, "unmarshal syncer config") @@ -1050,7 +1050,7 @@ func (c *TaskConfig) adjust() error { // getGenerateName generates name by rule or gets name from nameMap // if it's a new name, increase nameIdx // otherwise return current nameIdx. -func getGenerateName(rule interface{}, nameIdx int, namePrefix string, nameMap map[string]string) (string, int) { +func getGenerateName(rule any, nameIdx int, namePrefix string, nameMap map[string]string) (string, int) { // use json as key since no DeepEqual for rules now. ruleByte, err := json.Marshal(rule) if err != nil { diff --git a/dm/config/task_test.go b/dm/config/task_test.go index ffb034a180..72624cd8b0 100644 --- a/dm/config/task_test.go +++ b/dm/config/task_test.go @@ -53,7 +53,6 @@ func TestIsForeignKeyChecksEnabled(t *testing.T) { } for _, c := range cases { - c := c t.Run(c.name, func(t *testing.T) { t.Parallel() require.Equal(t, c.expected, IsForeignKeyChecksEnabled(c.session)) @@ -1114,7 +1113,7 @@ func TestTaskConfigForDowngrade(t *testing.T) { } // Clone clones src to dest. -func Clone(dest, src interface{}) { +func Clone(dest, src any) { cloneValues(reflect.ValueOf(dest), reflect.ValueOf(src)) } @@ -1123,17 +1122,17 @@ func Clone(dest, src interface{}) { func cloneValues(dest, src reflect.Value) { destType := dest.Type() srcType := src.Type() - if destType.Kind() == reflect.Ptr { + if destType.Kind() == reflect.Pointer { destType = destType.Elem() } - if srcType.Kind() == reflect.Ptr { + if srcType.Kind() == reflect.Pointer { srcType = srcType.Elem() } if destType.Kind() == reflect.Map { destMap := reflect.MakeMap(destType) for _, k := range src.MapKeys() { - if src.MapIndex(k).Type().Kind() == reflect.Ptr { + if src.MapIndex(k).Type().Kind() == reflect.Pointer { newVal := reflect.New(destType.Elem().Elem()) cloneValues(newVal, src.MapIndex(k)) destMap.SetMapIndex(k, newVal) @@ -1148,7 +1147,7 @@ func cloneValues(dest, src reflect.Value) { if destType.Kind() == reflect.Slice { slice := reflect.MakeSlice(destType, src.Len(), src.Cap()) for i := 0; i < src.Len(); i++ { - if slice.Index(i).Type().Kind() == reflect.Ptr { + if slice.Index(i).Type().Kind() == reflect.Pointer { newVal := reflect.New(slice.Index(i).Type().Elem()) cloneValues(newVal, src.Index(i)) slice.Index(i).Set(newVal) @@ -1170,10 +1169,10 @@ func cloneValues(dest, src reflect.Value) { srcField := src.Elem().Field(i) destFieldType := destField.Type() srcFieldType := srcField.Type() - if destFieldType.Kind() == reflect.Ptr { + if destFieldType.Kind() == reflect.Pointer { destFieldType = destFieldType.Elem() } - if srcFieldType.Kind() == reflect.Ptr { + if srcFieldType.Kind() == reflect.Pointer { srcFieldType = srcFieldType.Elem() } if destFieldType != srcFieldType { diff --git a/dm/ctl/common/config.go b/dm/ctl/common/config.go index 70a3225b6b..0d3a4aa05d 100644 --- a/dm/ctl/common/config.go +++ b/dm/ctl/common/config.go @@ -200,8 +200,8 @@ func (c *Config) adjust() error { // validate host:port format address. func validateAddr(addr string) error { - endpoints := strings.Split(addr, ",") - for _, endpoint := range endpoints { + endpoints := strings.SplitSeq(addr, ",") + for endpoint := range endpoints { if _, _, err := net.SplitHostPort(utils.UnwrapScheme(endpoint)); err != nil { return errors.Trace(err) } diff --git a/dm/ctl/common/util.go b/dm/ctl/common/util.go index daf593841b..5940857cfe 100644 --- a/dm/ctl/common/util.go +++ b/dm/ctl/common/util.go @@ -98,9 +98,9 @@ func (c *CtlClient) updateMasterClient() error { func (c *CtlClient) sendRequest( ctx context.Context, reqName string, - req interface{}, - respPointer interface{}, - opts ...interface{}, + req any, + respPointer any, + opts ...any, ) error { c.mu.RLock() defer c.mu.RUnlock() @@ -121,12 +121,12 @@ func (c *CtlClient) sendRequest( } // SendRequest send request to master. -func SendRequest(ctx context.Context, reqName string, req interface{}, respPointer interface{}) error { +func SendRequest(ctx context.Context, reqName string, req any, respPointer any) error { err := GlobalCtlClient.sendRequest(ctx, reqName, req, respPointer) if err == nil { return nil } - var opts []interface{} + var opts []any switch status.Code(err) { case codes.ResourceExhausted: matches := re.FindStringSubmatch(err.Error()) @@ -199,7 +199,7 @@ func GlobalConfig() *Config { } // PrintLinesf adds a wrap to support `\n` within `chzyer/readline`. -func PrintLinesf(format string, a ...interface{}) { +func PrintLinesf(format string, a ...any) { fmt.Println(fmt.Sprintf(format, a...)) } @@ -214,7 +214,7 @@ func PrettyPrintResponse(resp proto.Message) { } // PrettyPrintInterface prints an interface through encoding/json prettily. -func PrettyPrintInterface(resp interface{}) { +func PrettyPrintInterface(resp any) { s, err := json.MarshalIndent(resp, "", " ") if err != nil { PrintLinesf("%v", err) diff --git a/dm/ctl/master/operate_task.go b/dm/ctl/master/operate_task.go index 4fb37366ca..e72e62abd4 100644 --- a/dm/ctl/master/operate_task.go +++ b/dm/ctl/master/operate_task.go @@ -126,9 +126,7 @@ func batchOperateTask(taskOp pb.TaskOp, batchSize int, sources []string, subTask var wg sync.WaitGroup resultCh := make(chan *operateTaskResult, 1) for i := 0; i < batchSize; i++ { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { for name := range workCh { taskResult := operateTaskResult{Task: name, Op: taskOp.String()} @@ -143,7 +141,7 @@ func batchOperateTask(taskOp pb.TaskOp, batchSize int, sources []string, subTask } resultCh <- &taskResult } - }() + }) } go func() { diff --git a/dm/dumpling/dumpling.go b/dm/dumpling/dumpling.go index 5d8540a5af..4ac0c86db3 100644 --- a/dm/dumpling/dumpling.go +++ b/dm/dumpling/dumpling.go @@ -270,7 +270,7 @@ func (m *Dumpling) Update(context.Context, *config.SubTaskConfig) error { } // Status implements Unit.Status. -func (m *Dumpling) Status(_ *binlog.SourceStatus) interface{} { +func (m *Dumpling) Status(_ *binlog.SourceStatus) any { // NOTE: try to add some status, like dumped file count m.mu.RLock() defer m.mu.RUnlock() @@ -369,7 +369,7 @@ func (m *Dumpling) constructArgs(ctx context.Context) (*export.Config, error) { return nil, err1 } } - dumpConfig.SessionParams = map[string]interface{}{ + dumpConfig.SessionParams = map[string]any{ "time_zone": tz, } diff --git a/dm/loader/checkpoint.go b/dm/loader/checkpoint.go index a103d8609a..18abbbbbfa 100644 --- a/dm/loader/checkpoint.go +++ b/dm/loader/checkpoint.go @@ -123,7 +123,7 @@ func (cp *LightningCheckpointList) RegisterCheckPoint(ctx context.Context) error cp.logger.Info("initial checkpoint record", zap.String("task", cp.taskName), zap.String("source", cp.sourceName)) - args := []interface{}{cp.taskName, cp.sourceName} + args := []any{cp.taskName, cp.sourceName} tctx := tcontext.NewContext(ctx, log.With(zap.String("job", "lightning-checkpoint"))) _, err = connection.ExecuteSQL(tctx, nil, "lightning-checkpoint", []string{sql}, args) if err != nil { @@ -145,7 +145,7 @@ func (cp *LightningCheckpointList) UpdateStatus(ctx context.Context, status ligh zap.Stringer("status", status)) tctx := tcontext.NewContext(ctx, log.With(zap.String("job", "lightning-checkpoint"))) _, err = connection.ExecuteSQL(tctx, nil, "lightning-checkpoint", []string{sql}, - []interface{}{status.String(), cp.taskName, cp.sourceName}) + []any{status.String(), cp.taskName, cp.sourceName}) if err != nil { return terror.WithScope(terror.Annotate(err, "update lightning status"), terror.ScopeDownstream) } diff --git a/dm/loader/lightning.go b/dm/loader/lightning.go index 500b9ebf45..49cbb19e34 100644 --- a/dm/loader/lightning.go +++ b/dm/loader/lightning.go @@ -16,6 +16,7 @@ package loader import ( "context" "fmt" + "maps" "path/filepath" "regexp" "strings" @@ -307,8 +308,8 @@ func (l *LightningLoader) runLightning(ctx context.Context, cfg *lcfg.Config) (e failpoint.Inject("LoadDataSlowDown", nil) failpoint.Inject("LoadDataSlowDownByTask", func(val failpoint.Value) { tasks := val.(string) - taskNames := strings.Split(tasks, ",") - for _, taskName := range taskNames { + taskNames := strings.SplitSeq(tasks, ",") + for taskName := range taskNames { if l.cfg.Name == taskName { l.logger.Info("inject failpoint LoadDataSlowDownByTask in lightning loader", zap.String("task", taskName)) <-taskCtx.Done() @@ -463,9 +464,7 @@ func GetLightningConfig(globalCfg *lcfg.GlobalConfig, subtaskCfg *config.SubTask cfg.TiDB.Vars = make(map[string]string) cfg.Routes = subtaskCfg.RouteRules if subtaskCfg.To.Session != nil { - for k, v := range subtaskCfg.To.Session { - cfg.TiDB.Vars[k] = v - } + maps.Copy(cfg.TiDB.Vars, subtaskCfg.To.Session) } if subtaskCfg.RangeConcurrency > 0 { @@ -727,7 +726,7 @@ func (l *LightningLoader) status() *pb.LoadStatus { } // Status returns the unit's current status. -func (l *LightningLoader) Status(_ *binlog.SourceStatus) interface{} { +func (l *LightningLoader) Status(_ *binlog.SourceStatus) any { return l.status() } diff --git a/dm/master/agent_pool.go b/dm/master/agent_pool.go index 3b21e22acb..10ef0e4b03 100644 --- a/dm/master/agent_pool.go +++ b/dm/master/agent_pool.go @@ -29,7 +29,7 @@ const ( ErrorNoEmitToken = "fail to get emit opportunity for %s" ) -type emitFunc func(args ...interface{}) +type emitFunc func(args ...any) // AgentPool is a pool to control communication with dm-workers // It provides rate limit control for agent acquire, including dispatch rate r @@ -108,7 +108,7 @@ func (ap *AgentPool) Start(ctx context.Context) { } // Emit applies for an agent to communicates with dm-worker. -func (ap *AgentPool) Emit(ctx context.Context, id int, fn emitFunc, errFn emitFunc, args ...interface{}) { +func (ap *AgentPool) Emit(ctx context.Context, id int, fn emitFunc, errFn emitFunc, args ...any) { if agent := ap.Apply(ctx, id); agent == nil { errFn(args...) } else { diff --git a/dm/master/agent_pool_test.go b/dm/master/agent_pool_test.go index b4d1d2bb57..dc86b6009a 100644 --- a/dm/master/agent_pool_test.go +++ b/dm/master/agent_pool_test.go @@ -41,7 +41,7 @@ func (t *testMaster) testPool(c *check.C) { } }() - for i := 0; i < burst; i++ { + for i := range burst { agent := <-pc c.Assert(agent.ID, check.Equals, i) } @@ -51,7 +51,7 @@ func (t *testMaster) testPool(c *check.C) { default: } - for i := 0; i < rate; i++ { + for i := range rate { select { case agent := <-pc: c.Assert(agent.ID, check.Equals, i+burst) @@ -73,7 +73,7 @@ func (t *testMaster) testEmit(c *check.C) { ap := NewAgentPool(&RateLimitConfig{rate: DefaultRate, burst: DefaultBurst}) go ap.Start(context.Background()) - ap.Emit(context.Background(), 1, func(args ...interface{}) { + ap.Emit(context.Background(), 1, func(args ...any) { if len(args) != 2 { c.Fatalf("args count is not 2, args %v", args) } @@ -93,14 +93,14 @@ func (t *testMaster) testEmit(c *check.C) { if worker1 != worker { c.Fatalf("args[1] is not expected worker, args[1] %v vs %v", worker1, worker) } - }, func(args ...interface{}) {}, []interface{}{id, worker}...) + }, func(args ...any) {}, []any{id, worker}...) counter := 0 ctx, cancel := context.WithCancel(context.Background()) cancel() - ap.Emit(ctx, 1, func(args ...interface{}) { + ap.Emit(ctx, 1, func(args ...any) { c.FailNow() - }, func(args ...interface{}) { + }, func(args ...any) { if len(args) != 1 { c.Fatalf("args count is not 1, args %v", args) } @@ -109,6 +109,6 @@ func (t *testMaster) testEmit(c *check.C) { c.Fatalf("args[0] is not *int, args %+v", args) } *pCounter++ - }, []interface{}{&counter}...) + }, []any{&counter}...) c.Assert(counter, check.Equals, 1) } diff --git a/dm/master/bootstrap_test.go b/dm/master/bootstrap_test.go index eedc1cfb4e..a4f05b6f65 100644 --- a/dm/master/bootstrap_test.go +++ b/dm/master/bootstrap_test.go @@ -142,7 +142,7 @@ func (t *testMaster) TestCollectSourceConfigFilesV1Import(c *check.C) { c.Assert(err, check.IsNil) // fix empty map after marshal/unmarshal becomes nil cfg1.From.Adjust() - cfg1.Tracer = map[string]interface{}{} + cfg1.Tracer = map[string]any{} cfg1.Filters = []*filter.BinlogEventRule{} cfg1.From.Host = host cfg1.From.Port = port diff --git a/dm/master/etcd_test.go b/dm/master/etcd_test.go index 2987612e33..cc771ff3b6 100644 --- a/dm/master/etcd_test.go +++ b/dm/master/etcd_test.go @@ -188,7 +188,7 @@ func (t *testEtcdSuite) TestPrepareJoinEtcd(c *check.C) { defer e2.Close() // try join again - for i := 0; i < 20; i++ { + for range 20 { err = prepareJoinEtcd(cfgAfter2) if err == nil { break @@ -242,7 +242,7 @@ func (t *testEtcdSuite) TestEtcdAutoCompaction(c *check.C) { }) c.Assert(err, check.IsNil) - for i := 0; i < 100; i++ { + for i := range 100 { _, err = etcdCli.Put(ctx, "key", fmt.Sprintf("%03d", i)) c.Assert(err, check.IsNil) } diff --git a/dm/master/openapi_view_test.go b/dm/master/openapi_view_test.go index 330ec908f3..c22f1d80d1 100644 --- a/dm/master/openapi_view_test.go +++ b/dm/master/openapi_view_test.go @@ -252,7 +252,7 @@ func (s *OpenAPIViewSuite) TestClusterAPI() { // offline master-2 with retry // operate etcd cluster may met `etcdserver: unhealthy cluster`, add some retry - for i := 0; i < 20; i++ { + for range 20 { result = testutil.NewRequest().Delete(fmt.Sprintf("%s/%s", masterURL, s2.cfg.Name)).GoWithHTTPHandler(s.T(), s1.openapiHandles) if result.Code() == http.StatusBadRequest { s.Equal(http.StatusBadRequest, result.Code()) diff --git a/dm/master/scheduler/latch_test.go b/dm/master/scheduler/latch_test.go index 6ed0366ddb..58aa29ffa3 100644 --- a/dm/master/scheduler/latch_test.go +++ b/dm/master/scheduler/latch_test.go @@ -35,7 +35,7 @@ func TestOneAcquireSuccess(t *testing.T) { wg sync.WaitGroup ) - for i := 0; i < 10; i++ { + for i := range 10 { var group string if i < 5 { group = group1 @@ -56,7 +56,7 @@ func TestOneAcquireSuccess(t *testing.T) { }(group) } - for i := 0; i < 10; i++ { + for range 10 { fire <- struct{}{} } wg.Wait() @@ -81,10 +81,8 @@ func TestAcquireAfterRelease(t *testing.T) { wg sync.WaitGroup ) - for i := 0; i < 5; i++ { - wg.Add(1) - go func() { - defer wg.Done() + for range 5 { + wg.Go(func() { <-fire for { @@ -98,10 +96,10 @@ func TestAcquireAfterRelease(t *testing.T) { return } } - }() + }) } - for i := 0; i < 5; i++ { + for range 5 { fire <- struct{}{} } @@ -119,7 +117,7 @@ func TestMultiRelease(t *testing.T) { wg sync.WaitGroup ) - for repeat := 0; repeat < 3; repeat++ { + for range 3 { for i := range names { wg.Add(1) go func(name string) { diff --git a/dm/master/scheduler/scheduler.go b/dm/master/scheduler/scheduler.go index c53a5fbb1c..68a790148f 100644 --- a/dm/master/scheduler/scheduler.go +++ b/dm/master/scheduler/scheduler.go @@ -15,6 +15,7 @@ package scheduler import ( "context" + "maps" "sort" "sync" "time" @@ -456,7 +457,7 @@ func (s *Scheduler) RemoveSourceCfg(source string) error { // 2. check whether any subtask or relay config exists for the source. existingSubtasksM := make(map[string]struct{}) - s.subTaskCfgs.Range(func(k, v interface{}) bool { + s.subTaskCfgs.Range(func(k, v any) bool { task := k.(string) cfg := v.(map[string]config.SubTaskConfig) for source2 := range cfg { @@ -785,7 +786,7 @@ func (s *Scheduler) BatchOperateTaskOnWorker( } // wait all tasks are in expected stage before actually starting scheduling WaitLoop: - for retry := 0; retry < maxQueryWorkerRetryNum; retry++ { + for retry := range maxQueryWorkerRetryNum { resp, err := worker.queryStatus(ctx) if err != nil { return terror.Annotatef(err, "failed to query worker: %s status", worker.baseInfo.Name) @@ -1173,7 +1174,7 @@ func (s *Scheduler) GetSubTaskCfgsByTaskAndSource(taskName string, sources []str } // filter the source that we don't want if len(sources) > 0 { - filterSource := map[string]interface{}{} + filterSource := map[string]any{} for _, source := range sources { filterSource[source] = true // the source we want } @@ -1195,7 +1196,7 @@ func (s *Scheduler) GetSubTaskCfgsByTaskAndSource(taskName string, sources []str func (s *Scheduler) GetSubTaskCfgs() map[string]map[string]config.SubTaskConfig { // taskName -> sourceName -> SubTaskConfig clone := make(map[string]map[string]config.SubTaskConfig) - s.subTaskCfgs.Range(func(k, v interface{}) bool { + s.subTaskCfgs.Range(func(k, v any) bool { task := k.(string) m := v.(map[string]config.SubTaskConfig) clone2 := make(map[string]config.SubTaskConfig, len(m)) @@ -1218,7 +1219,7 @@ func (s *Scheduler) GetALlSubTaskCfgs() map[string]map[string]*config.SubTaskCon defer s.mu.RUnlock() // taskName -> sourceName -> SubTaskConfig clone := make(map[string]map[string]*config.SubTaskConfig) - s.subTaskCfgs.Range(func(k, v interface{}) bool { + s.subTaskCfgs.Range(func(k, v any) bool { task := k.(string) m := v.(map[string]config.SubTaskConfig) clone2 := make(map[string]*config.SubTaskConfig, len(m)) @@ -1238,7 +1239,7 @@ func (s *Scheduler) GetALlSubTaskCfgs() map[string]map[string]*config.SubTaskCon // GetTaskNameListBySourceName gets task name list by source name. func (s *Scheduler) GetTaskNameListBySourceName(sourceName string, expectStage *pb.Stage) []string { var taskNameList []string - s.expectSubTaskStages.Range(func(k, v interface{}) bool { + s.expectSubTaskStages.Range(func(k, v any) bool { subtaskM := v.(map[string]ha.Stage) subtaskStage, ok2 := subtaskM[sourceName] if !ok2 { @@ -1805,12 +1806,8 @@ func (s *Scheduler) recoverSources() error { } // recover in-memory data. - for source, cfg := range cfgM { - s.sourceCfgs[source] = cfg - } - for source, stage := range stageM { - s.expectRelayStages[source] = stage - } + maps.Copy(s.sourceCfgs, cfgM) + maps.Copy(s.expectRelayStages, stageM) return nil } diff --git a/dm/master/scheduler/scheduler_test.go b/dm/master/scheduler/scheduler_test.go index 3a95da06f2..aff7750e5d 100644 --- a/dm/master/scheduler/scheduler_test.go +++ b/dm/master/scheduler/scheduler_test.go @@ -207,11 +207,9 @@ func (t *testSchedulerSuite) testSchedulerProgress(restart int) { // do keep-alive for worker1. ctx1, cancel1 := context.WithCancel(ctx) var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { require.NoError(t.T(), ha.KeepAlive(ctx1, t.etcdTestCli, workerName1, keepAliveTTL)) - }() + }) // wait for source1 being bound to worker1. require.Eventually(t.T(), func() bool { bounds := s.BoundSources() @@ -334,11 +332,9 @@ func (t *testSchedulerSuite) testSchedulerProgress(restart int) { // CASE 3.2: start worker1 again. // do keep-alive for worker1 again. ctx1, cancel1 = context.WithCancel(ctx) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { require.NoError(t.T(), ha.KeepAlive(ctx1, t.etcdTestCli, workerName1, keepAliveTTL)) - }() + }) // wait for source1 bound to worker1. require.Eventually(t.T(), func() bool { bounds := s.BoundSources() @@ -379,11 +375,9 @@ func (t *testSchedulerSuite) testSchedulerProgress(restart int) { // CASE 4.3: the worker2 become online. // do keep-alive for worker2. ctx2, cancel2 := context.WithCancel(ctx) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { require.NoError(t.T(), ha.KeepAlive(ctx2, t.etcdTestCli, workerName2, keepAliveTTL)) - }() + }) // wait for worker2 become Free. require.Eventually(t.T(), func() bool { w := s.GetWorkerByName(workerName2) @@ -859,11 +853,9 @@ func (t *testSchedulerSuite) TestRestartScheduler() { }() // step 2.2: worker start keepAlive ctx1, cancel1 := context.WithCancel(ctx) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { require.NoError(t.T(), ha.KeepAlive(ctx1, t.etcdTestCli, workerName1, keepAliveTTL)) - }() + }) // step 2.3: scheduler should bound source to worker // wait for source1 bound to worker1. require.Eventually(t.T(), func() bool { @@ -898,11 +890,9 @@ func (t *testSchedulerSuite) TestRestartScheduler() { require.Len(t.T(), sourceBoundCh, 0) ctx2, cancel2 := context.WithCancel(ctx) // trigger same keepalive event again, just for test - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { require.NoError(t.T(), ha.KeepAlive(ctx2, t.etcdTestCli, workerName1, keepAliveTTL)) - }() + }) checkSourceBoundCh() // case 3: scheduler is restarted, but worker also broke after scheduler is down // step 5: stop scheduler -> stop worker keepalive -> restart scheduler @@ -934,11 +924,9 @@ func (t *testSchedulerSuite) TestRestartScheduler() { // first let the source bound again ctx4, cancel4 := context.WithCancel(ctx) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { require.NoError(t.T(), ha.KeepAlive(ctx4, t.etcdTestCli, workerName1, keepAliveTTL)) - }() + }) sourceBound1.Source = sourceID1 sourceBound1.IsDeleted = false checkSourceBoundCh() @@ -1026,16 +1014,12 @@ func (t *testSchedulerSuite) TestWatchWorkerEventEtcdCompact() { // step 3: add two workers, and then cancel them to simulate they have lost connection var wg sync.WaitGroup ctx1, cancel1 := context.WithCancel(ctx) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { require.NoError(t.T(), ha.KeepAlive(ctx1, t.etcdTestCli, workerName1, keepAliveTTL)) - }() - wg.Add(1) - go func() { - defer wg.Done() + }) + wg.Go(func() { require.NoError(t.T(), ha.KeepAlive(ctx1, t.etcdTestCli, workerName2, keepAliveTTL)) - }() + }) require.Eventually(t.T(), func() bool { kam, _, e := ha.GetKeepAliveWorkers(t.etcdTestCli) return e == nil && len(kam) == 2 @@ -1068,11 +1052,9 @@ func (t *testSchedulerSuite) TestWatchWorkerEventEtcdCompact() { // step 5: scheduler start to handle workerEvent from compact revision, should handle worker keepalive events correctly ctx2, cancel2 := context.WithCancel(ctx) // step 5.1: start one worker before scheduler start to handle workerEvent - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { require.NoError(t.T(), ha.KeepAlive(ctx2, t.etcdTestCli, workerName3, keepAliveTTL)) - }() + }) require.Eventually(t.T(), func() bool { kam, _, err := ha.GetKeepAliveWorkers(t.etcdTestCli) if err == nil { @@ -1083,18 +1065,14 @@ func (t *testSchedulerSuite) TestWatchWorkerEventEtcdCompact() { return false }, 3*time.Second, 100*time.Millisecond) // step 5.2: scheduler start to handle workerEvent - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { require.NoError(t.T(), s.observeWorkerEvent(ctx2, startRev)) - }() + }) // step 5.3: wait for scheduler to restart handleWorkerEvent, then start a new worker time.Sleep(time.Second) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { require.NoError(t.T(), ha.KeepAlive(ctx2, t.etcdTestCli, workerName4, keepAliveTTL)) - }() + }) require.Eventually(t.T(), func() bool { unbounds := s.UnboundSources() return len(unbounds) == 0 @@ -1105,11 +1083,9 @@ func (t *testSchedulerSuite) TestWatchWorkerEventEtcdCompact() { // step 6: restart to observe workerEvents, should unbound all sources ctx3, cancel3 := context.WithCancel(ctx) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { require.NoError(t.T(), s.observeWorkerEvent(ctx3, startRev)) - }() + }) require.Eventually(t.T(), func() bool { bounds := s.BoundSources() return len(bounds) == 0 @@ -1804,11 +1780,9 @@ func (t *testSchedulerSuite) TestWatchLoadTask() { require.True(t.T(), s.hasLoadTaskByWorkerAndSource(workerName4, sourceID2)) // observer load tasks - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { require.NoError(t.T(), s.observeLoadTask(ctx1, startRev)) - }() + }) // put task2, source1, worker1 _, err = ha.PutLoadTask(t.etcdTestCli, task2, sourceID1, workerName1) diff --git a/dm/master/server.go b/dm/master/server.go index 9c0950798d..2d8cb6590f 100644 --- a/dm/master/server.go +++ b/dm/master/server.go @@ -22,6 +22,7 @@ import ( "net/http" "reflect" "runtime" + "slices" "sort" "strings" "sync" @@ -254,24 +255,18 @@ func (s *Server) Start(ctx context.Context) (err error) { s.closed.Store(false) // the server started now. - s.bgFunWg.Add(1) - go func() { - defer s.bgFunWg.Done() + s.bgFunWg.Go(func() { s.ap.Start(ctx) - }() + }) - s.bgFunWg.Add(1) - go func() { - defer s.bgFunWg.Done() + s.bgFunWg.Go(func() { s.electionNotify(ctx) - }() + }) runBackgroundOnce.Do(func() { - s.bgFunWg.Add(1) - go func() { - defer s.bgFunWg.Done() + s.bgFunWg.Go(func() { metrics.RunBackgroundJob(ctx) - }() + }) }) failpoint.Inject("FailToElect", func(val failpoint.Value) { @@ -484,7 +479,7 @@ func (s *Server) StartTask(ctx context.Context, req *pb.StartTaskRequest) (*pb.S var b strings.Builder size := 5 * 1024 * 1024 b.Grow(size) - for i := 0; i < size; i++ { + for range size { b.WriteByte(0) } resp2 = &pb.StartTaskResponse{Msg: b.String()} @@ -845,12 +840,7 @@ func (s *Server) QueryStatus(ctx context.Context, req *pb.QueryStatusListRequest resps := s.getStatusFromWorkers(ctx, sources, req.Name, specifiedSource) workerRespMap := make(map[string][]*pb.QueryStatusResponse, len(sources)) // sourceName -> worker QueryStatusResponse inSlice := func(s []string, e string) bool { - for _, v := range s { - if v == e { - return true - } - } - return false + return slices.Contains(s, e) } for _, workerResp := range resps { workerRespMap[workerResp.SourceStatus.Source] = append(workerRespMap[workerResp.SourceStatus.Source], workerResp) @@ -1190,7 +1180,7 @@ func (s *Server) getStatusFromWorkers( for _, worker := range workers { wg.Add(1) - go s.ap.Emit(ctx, 0, func(args ...interface{}) { + go s.ap.Emit(ctx, 0, func(args ...any) { defer wg.Done() sourceID := args[0].(string) w, _ := args[1].(*scheduler.Worker) @@ -1214,7 +1204,7 @@ func (s *Server) getStatusFromWorkers( } workerStatus.SourceStatus.Source = sourceID setWorkerResp(workerStatus) - }, func(args ...interface{}) { + }, func(args ...any) { defer wg.Done() sourceID, _ := args[0].(string) w, _ := args[1].(*scheduler.Worker) @@ -1932,7 +1922,7 @@ func (s *Server) waitOperationOk( cli *scheduler.Worker, taskName string, sourceID string, - masterReq interface{}, + masterReq any, ) (bool, string, *pb.QueryStatusResponse, error) { var expect pb.Stage switch req := masterReq.(type) { @@ -2121,7 +2111,7 @@ func (s *Server) waitOperationOk( return false, "", nil, terror.ErrMasterFailToGetExpectResult } -func (s *Server) handleOperationResult(ctx context.Context, cli *scheduler.Worker, taskName, sourceID string, req interface{}) *pb.CommonWorkerResponse { +func (s *Server) handleOperationResult(ctx context.Context, cli *scheduler.Worker, taskName, sourceID string, req any) *pb.CommonWorkerResponse { if cli == nil { return errorCommonWorkerResponse(sourceID+" relevant worker-client not found", sourceID, "") } @@ -2152,7 +2142,7 @@ func sortCommonWorkerResults(sourceRespCh chan *pb.CommonWorkerResponse) []*pb.C return sourceResps } -func (s *Server) getSourceRespsAfterOperation(ctx context.Context, taskName string, sources, workers []string, req interface{}) []*pb.CommonWorkerResponse { +func (s *Server) getSourceRespsAfterOperation(ctx context.Context, taskName string, sources, workers []string, req any) []*pb.CommonWorkerResponse { sourceRespCh := make(chan *pb.CommonWorkerResponse, len(sources)) var wg sync.WaitGroup for i, source := range sources { @@ -2161,7 +2151,7 @@ func (s *Server) getSourceRespsAfterOperation(ctx context.Context, taskName stri if i < len(workers) { worker = workers[i] } - go s.ap.Emit(ctx, 0, func(args ...interface{}) { + go s.ap.Emit(ctx, 0, func(args ...any) { defer wg.Done() source1, _ := args[0].(string) worker1, _ := args[1].(string) @@ -2175,7 +2165,7 @@ func (s *Server) getSourceRespsAfterOperation(ctx context.Context, taskName stri sourceResp := s.handleOperationResult(ctx, workerCli, taskName, source1, req) sourceResp.Source = source1 // may return other source's ID during stop worker sourceRespCh <- sourceResp - }, func(args ...interface{}) { + }, func(args ...any) { defer wg.Done() source1, _ := args[0].(string) worker1, _ := args[1].(string) @@ -2795,7 +2785,7 @@ func (s *Server) OperateRelay(ctx context.Context, req *pb.OperateRelayRequest) // sharedLogic does some shared logic for each RPC implementation // arguments with `Pointer` suffix should be pointer to that variable its name indicated // return `true` means caller should return with variable that `xxPointer` modified. -func (s *Server) sharedLogic(ctx context.Context, req interface{}, respPointer interface{}, errPointer *error) bool { +func (s *Server) sharedLogic(ctx context.Context, req any, respPointer any, errPointer *error) bool { // nolint:dogsled pc, _, _, _ := runtime.Caller(1) fullMethodName := runtime.FuncForPC(pc).Name() @@ -3234,7 +3224,7 @@ func sendValidationRequest[T any]( return } wg.Add(1) - go s.ap.Emit(ctx, 0, func(args ...interface{}) { + go s.ap.Emit(ctx, 0, func(args ...any) { // send request in parallel defer wg.Done() workerResp, err := worker.SendRequest(ctx, req, s.cfg.RPCTimeout) @@ -3245,7 +3235,7 @@ func sendValidationRequest[T any]( resp := getValidationWorkerResp(req, workerResp) appendWorkerResp(workerRespMu, workerResps, resp.(T)) } - }, func(args ...interface{}) { + }, func(args ...any) { defer wg.Done() err := terror.ErrMasterNoEmitToken.Generate(sourceID) resp := genValidationWorkerErrorResp(req, err, logMsg, worker.BaseInfo().Name, sourceID) @@ -3253,7 +3243,7 @@ func sendValidationRequest[T any]( }) } -func getValidationWorkerResp(req *workerrpc.Request, resp *workerrpc.Response) interface{} { +func getValidationWorkerResp(req *workerrpc.Request, resp *workerrpc.Response) any { switch req.Type { case workerrpc.CmdGetValidationStatus: return resp.GetValidationStatus @@ -3266,7 +3256,7 @@ func getValidationWorkerResp(req *workerrpc.Request, resp *workerrpc.Response) i } } -func genValidationWorkerErrorResp(req *workerrpc.Request, err error, logMsg, workerID, sourceID string) interface{} { +func genValidationWorkerErrorResp(req *workerrpc.Request, err error, logMsg, workerID, sourceID string) any { log.L().Error(logMsg, zap.Error(err), zap.String("source", sourceID), zap.String("worker", workerID)) switch req.Type { case workerrpc.CmdGetValidationStatus: diff --git a/dm/master/server_test.go b/dm/master/server_test.go index 741f36c38f..a432a4cde7 100644 --- a/dm/master/server_test.go +++ b/dm/master/server_test.go @@ -213,7 +213,7 @@ func stageDeepEqualExcludeRev(t *testing.T, stage, expectStage ha.Stage) { require.Equal(t, expectStage, stage) } -func mockRevelantWorkerClient(mockWorkerClient *pbmock.MockWorkerClient, taskName, sourceID string, masterReq interface{}) { +func mockRevelantWorkerClient(mockWorkerClient *pbmock.MockWorkerClient, taskName, sourceID string, masterReq any) { var expect pb.Stage switch req := masterReq.(type) { case *pb.OperateSourceRequest: @@ -313,7 +313,7 @@ func makeNilWorkerClients(workers []string) map[string]workerrpc.Client { return nilWorkerClients } -func makeWorkerClientsForHandle(ctrl *gomock.Controller, taskName string, sources []string, workers []string, reqs ...interface{}) map[string]workerrpc.Client { +func makeWorkerClientsForHandle(ctrl *gomock.Controller, taskName string, sources []string, workers []string, reqs ...any) map[string]workerrpc.Client { workerClients := make(map[string]workerrpc.Client, len(workers)) for i := range workers { mockWorkerClient := pbmock.NewMockWorkerClient(ctrl) @@ -517,7 +517,7 @@ func (t *testMasterSuite) TestQueryStatus() { func (t *testMasterSuite) TestWaitOperationOkRightResult() { cases := []struct { - req interface{} + req any resp *pb.QueryStatusResponse expectedOK bool expectedEmptyMsg bool @@ -1004,9 +1004,7 @@ func (t *testMasterSuite) TestStartTaskWithRemoveMeta() { require.Greater(t.T(), len(server.pessimist.Locks()), 0) resp, err := server.StartTask(context.Background(), req) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { time.Sleep(10 * time.Microsecond) // start another same task at the same time, should get err verMock2 := conn.InitVersionDB() @@ -1017,7 +1015,7 @@ func (t *testMasterSuite) TestStartTaskWithRemoveMeta() { require.False(t.T(), resp1.Result) require.Equal(t.T(), terror.Annotate(terror.ErrSchedulerSubTaskExist.Generate(cfg.Name, sources), "while remove-meta is true").Error(), resp1.Msg) - }() + }) require.NoError(t.T(), err) require.True(t.T(), resp.Result, "start task failed: %s", resp.Msg) for _, source := range sources { @@ -1099,9 +1097,7 @@ func (t *testMasterSuite) TestStartTaskWithRemoveMeta() { require.Greater(t.T(), len(server.optimist.Locks()), 0) resp, err = server.StartTask(context.Background(), req) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { time.Sleep(10 * time.Microsecond) // start another same task at the same time, should get err vermock2 := conn.InitVersionDB() @@ -1112,7 +1108,7 @@ func (t *testMasterSuite) TestStartTaskWithRemoveMeta() { require.False(t.T(), resp1.Result) require.Equal(t.T(), terror.Annotate(terror.ErrSchedulerSubTaskExist.Generate(cfg.Name, sources), "while remove-meta is true").Error(), resp1.Msg) - }() + }) require.NoError(t.T(), err) require.True(t.T(), resp.Result) for _, source := range sources { @@ -1248,12 +1244,12 @@ func (t *testMasterSuite) TestPurgeWorkerRelay() { // mock PurgeRelay request mockPurgeRelay := func(rpcSuccess bool) { for i, worker := range workers { - rets := []interface{}{ + rets := []any{ nil, errors.New(errGRPCFailed), } if rpcSuccess { - rets = []interface{}{ + rets = []any{ &pb.CommonWorkerResponse{ Result: true, Source: sources[i], @@ -1926,22 +1922,18 @@ func (t *testMasterSuite) TestOfflineMember() { cancel() s1.Close() }() - wg.Add(1) - go func() { + wg.Go(func() { require.NoError(t.T(), s1.Start(ctx)) - wg.Done() - }() + }) s2 := NewServer(cfg2) defer func() { cancel() s2.Close() }() - wg.Add(1) - go func() { + wg.Go(func() { require.NoError(t.T(), s2.Start(ctx)) - wg.Done() - }() + }) ctx3, cancel3 := context.WithCancel(ctx) s3 := NewServer(cfg3) diff --git a/dm/master/shardddl/optimist.go b/dm/master/shardddl/optimist.go index cad7b258f2..47605d334f 100644 --- a/dm/master/shardddl/optimist.go +++ b/dm/master/shardddl/optimist.go @@ -81,13 +81,11 @@ func (o *Optimist) Start(pCtx context.Context, etcdCli *clientv3.Client) error { ctx, cancel := context.WithCancel(pCtx) - o.wg.Add(1) - go func() { - defer o.wg.Done() + o.wg.Go(func() { // TODO: handle fatal error from run //nolint:errcheck o.run(ctx, revSource, revInfo, revOperation) - }() + }) o.closed = false // started now, no error will interrupt the start process. o.cancel = cancel diff --git a/dm/master/shardddl/pessimist.go b/dm/master/shardddl/pessimist.go index f89b376eed..2a06f11b9b 100644 --- a/dm/master/shardddl/pessimist.go +++ b/dm/master/shardddl/pessimist.go @@ -83,13 +83,11 @@ func (p *Pessimist) Start(pCtx context.Context, etcdCli *clientv3.Client) error return err } ctx, cancel := context.WithCancel(pCtx) - p.wg.Add(1) - go func() { - defer p.wg.Done() + p.wg.Go(func() { // TODO: handle fatal error from run //nolint:errcheck p.run(ctx, etcdCli, rev1, rev2) - }() + }) p.closed = false // started now. p.cancel = cancel diff --git a/dm/master/shardddl/pessimist_test.go b/dm/master/shardddl/pessimist_test.go index 17ccdeca1c..e4d912e7e3 100644 --- a/dm/master/shardddl/pessimist_test.go +++ b/dm/master/shardddl/pessimist_test.go @@ -404,9 +404,7 @@ func (t *testPessimistSuite) TestSourceReEntrant() { // 4. wait exec operation for the owner become available. var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { opCh := make(chan pessimism.Operation, 10) errCh := make(chan error, 10) ctx2, cancel2 := context.WithTimeout(ctx, watchTimeout) @@ -419,7 +417,7 @@ func (t *testPessimistSuite) TestSourceReEntrant() { op := <-opCh require.True(t.T(), op.Exec) require.False(t.T(), op.Done) - }() + }) // 5. put i13, the lock will become synced, then an operation PUT for the owner will be triggered. _, err = pessimism.PutInfo(t.etcdTestCli, i13) @@ -445,9 +443,7 @@ func (t *testPessimistSuite) TestSourceReEntrant() { require.False(t.T(), op11.Done) // 9. wait exec operation for the non-owner become available. - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { opCh = make(chan pessimism.Operation, 10) errCh = make(chan error, 10) ctx2, cancel2 = context.WithTimeout(ctx, watchTimeout) @@ -460,7 +456,7 @@ func (t *testPessimistSuite) TestSourceReEntrant() { op := <-opCh require.False(t.T(), op.Exec) require.False(t.T(), op.Done) - }() + }) // 10. mark exec operation for the owner as `done` (and delete the info). op11c := op11 @@ -815,11 +811,9 @@ func (t *testPessimistSuite) TestUnlockSourceOwnerRemoved() { // 4. put done for the replace owner then can unlock the lock. var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { t.putDoneForSource(ctx, task, source2, i11, true, rev1+1, watchTimeout) - }() + }) require.NoError(t.T(), p.UnlockLock(ctx, ID, source2, false)) wg.Wait() @@ -908,15 +902,13 @@ func (t *testPessimistSuite) TestMeetEtcdCompactError() { // step 2: start running, i11 and i12 should be handled successfully ctx2, cancel2 := context.WithCancel(ctx) var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { rev1, rev2 := revCompacted, revThreshold if i == 1 { rev1, rev2 = rev2, rev1 } require.NoError(t.T(), p.run(ctx2, t.etcdTestCli, rev1, rev2)) - }() + }) // PUT i11, will create a lock but not synced. require.Eventually(t.T(), func() bool { return len(p.Locks()) == 1 diff --git a/dm/relay/binlog_writer.go b/dm/relay/binlog_writer.go index b383bda0fc..7146dee66e 100644 --- a/dm/relay/binlog_writer.go +++ b/dm/relay/binlog_writer.go @@ -144,11 +144,9 @@ func (w *BinlogWriter) Open(uuid, filename string) error { w.err.Store(nilErr) w.input = make(chan []byte, chanSize) - w.wg.Add(1) - go func() { - defer w.wg.Done() + w.wg.Go(func() { w.run() - }() + }) return nil } diff --git a/dm/relay/file.go b/dm/relay/file.go index 933a6adcd5..3aede2faee 100644 --- a/dm/relay/file.go +++ b/dm/relay/file.go @@ -39,7 +39,7 @@ const ( // EventNotifier notifies whether there is new binlog event written to the file. type EventNotifier interface { // Notified returns a channel used to check whether there is new binlog event written to the file - Notified() chan interface{} + Notified() chan any } // CollectAllBinlogFiles collects all valid binlog files in dir, and returns filenames in binlog ascending order. diff --git a/dm/relay/file_util_test.go b/dm/relay/file_util_test.go index 46a4ca301d..74d3c65b49 100644 --- a/dm/relay/file_util_test.go +++ b/dm/relay/file_util_test.go @@ -335,8 +335,8 @@ func (t *testFileUtilSuite) testGetTxnPosGTIDs(c *check.C, filename, flavor, pre schema = "db" table = "tbl2" ) - updateRows := make([][]interface{}, 0, 2) - updateRows = append(updateRows, []interface{}{int32(1)}, []interface{}{int32(2)}) + updateRows := make([][]any, 0, 2) + updateRows = append(updateRows, []any{int32(1)}, []any{int32(2)}) dmlData := []*event.DMLData{ { TableID: tableID, diff --git a/dm/relay/local_reader.go b/dm/relay/local_reader.go index 2f23dd3c34..27eaba88cd 100644 --- a/dm/relay/local_reader.go +++ b/dm/relay/local_reader.go @@ -68,7 +68,7 @@ type BinlogReader struct { usingGTID bool prevGset, currGset mysql.GTIDSet // ch with size = 1, we only need to be notified whether binlog file of relay changed, not how many times - notifyCh chan interface{} + notifyCh chan any relay Process currentSubDir string // current UUID(with suffix) @@ -96,7 +96,7 @@ func newBinlogReader(logger log.Logger, cfg *BinlogReaderConfig, relay Process) indexPath: path.Join(cfg.RelayDir, utils.UUIDIndexFilename), cancel: cancel, tctx: newtctx, - notifyCh: make(chan interface{}, 1), + notifyCh: make(chan any, 1), relay: relay, lastFileGracefulEnd: true, } @@ -237,9 +237,7 @@ func (r *BinlogReader) StartSyncByPos(pos mysql.Position) (reader.Streamer, erro r.running = true s := newLocalStreamer() - r.wg.Add(1) - go func() { - defer r.wg.Done() + r.wg.Go(func() { r.tctx.L().Info("start reading", zap.Stringer("position", pos)) err = r.parseRelay(r.tctx.Context(), s, pos) if errors.Cause(err) == r.tctx.Context().Err() { @@ -248,7 +246,7 @@ func (r *BinlogReader) StartSyncByPos(pos mysql.Position) (reader.Streamer, erro s.closeWithError(err) r.tctx.L().Error("parse relay stopped", zap.Error(err)) } - }() + }) return s, nil } @@ -279,9 +277,7 @@ func (r *BinlogReader) StartSyncByGTID(gset mysql.GTIDSet) (reader.Streamer, err r.running = true s := newLocalStreamer() - r.wg.Add(1) - go func() { - defer r.wg.Done() + r.wg.Go(func() { r.tctx.L().Info("start reading", zap.Stringer("position", pos)) err = r.parseRelay(r.tctx.Context(), s, *pos) if errors.Cause(err) == r.tctx.Context().Err() { @@ -290,7 +286,7 @@ func (r *BinlogReader) StartSyncByGTID(gset mysql.GTIDSet) (reader.Streamer, err s.closeWithError(err) r.tctx.L().Error("parse relay stopped", zap.Error(err)) } - }() + }) return s, nil } @@ -817,7 +813,7 @@ func (r *BinlogReader) advanceCurrentGtidSet(gtid string) (bool, error) { return false, err } -func (r *BinlogReader) Notified() chan interface{} { +func (r *BinlogReader) Notified() chan any { return r.notifyCh } diff --git a/dm/relay/local_reader_test.go b/dm/relay/local_reader_test.go index addf7595fa..ea8a0e8f39 100644 --- a/dm/relay/local_reader_test.go +++ b/dm/relay/local_reader_test.go @@ -464,7 +464,7 @@ func (t *testReaderSuite) TestStartSyncByPos(c *check.C) { // 1. generate relay log files // 1 for the first sub directory, 2 for the second directory and 3 for the third directory // so, write the same events data into (1+2+3) files. - for i := 0; i < 3; i++ { + for i := range 3 { for j := 1; j < i+2; j++ { filename := filepath.Join(baseDir, UUIDs[i], filenamePrefix+strconv.Itoa(j)) var content []byte @@ -789,7 +789,7 @@ func (t *testReaderSuite) TestStartSyncByGTID(c *check.C) { // exclude event except for first server includeServerUUID := testCase[0].serverUUID includeUUID := testCase[0].uuid - for _, s := range strings.Split(preGset.String(), ",") { + for s := range strings.SplitSeq(preGset.String(), ",") { if !strings.Contains(s, includeServerUUID) { excludeStrs = append(excludeStrs, s) } @@ -1070,7 +1070,7 @@ func (t *testReaderSuite) genBinlogEvents(c *check.C, latestPos uint32, latestGT // for these tests, generates some DDL events is enough count := 5 + rand.Intn(5) - for i := 0; i < count; i++ { + for i := range count { evs, err := event.GenDDLEvents(gmysql.MySQLFlavor, 1, latestPos, latestGTID, fmt.Sprintf("db_%d", i), fmt.Sprintf("CREATE TABLE %d (c1 INT)", i), true, false, 0) c.Assert(err, check.IsNil) events = append(events, evs.Events...) @@ -1125,7 +1125,7 @@ func (t *testReaderSuite) genEvents( Schema: fmt.Sprintf("db_%d", i), Table: strconv.Itoa(i), ColumnType: []byte{gmysql.MYSQL_TYPE_INT24}, - Rows: [][]interface{}{{int32(1)}, {int32(2)}}, + Rows: [][]any{{int32(1)}, {int32(2)}}, }, } evs, err := event.GenDMLEvents(gmysql.MySQLFlavor, 1, latestPos, latestGTID, replication.WRITE_ROWS_EVENTv2, 10, insertDMLData, true, false, 0) @@ -1478,7 +1478,7 @@ func (t *testReaderSuite) TestwaitBinlogChanged(c *check.C) { { cfg := &BinlogReaderConfig{RelayDir: relayDir, Flavor: gmysql.MySQLFlavor} r := newBinlogReaderForTest(log.L(), cfg, true, "xxx.000001") - r.notifyCh = make(chan interface{}, 2) + r.notifyCh = make(chan any, 2) r.notifyCh <- struct{}{} r.notifyCh <- struct{}{} relay := r.relay.(*Relay) diff --git a/dm/relay/meta_test.go b/dm/relay/meta_test.go index b90a5031eb..6510016730 100644 --- a/dm/relay/meta_test.go +++ b/dm/relay/meta_test.go @@ -281,7 +281,7 @@ func (r *testMetaSuite) TestLocalMetaPotentialDataRace(c *check.C) { defer func() { ch2 <- err }() - for i := 0; i < 100; i++ { + for range 100 { _, currentGTID := lm.GTID() gtidString = currentGTID.String() } diff --git a/dm/relay/purge_strategy.go b/dm/relay/purge_strategy.go index 0eab37f784..a74c0a5e0e 100644 --- a/dm/relay/purge_strategy.go +++ b/dm/relay/purge_strategy.go @@ -59,10 +59,10 @@ func (s strategyType) String() string { // a strategy can support both or one of them. type PurgeStrategy interface { // Check checks whether need to do the purge in the background automatically - Check(args interface{}) (bool, error) + Check(args any) (bool, error) // Do does the purge process one time - Do(args interface{}) error + Do(args any) error // Purging indicates whether is doing purge Purging() bool @@ -142,12 +142,12 @@ func newFilenameStrategy() PurgeStrategy { } } -func (s *filenameStrategy) Check(args interface{}) (bool, error) { +func (s *filenameStrategy) Check(args any) (bool, error) { // do not support purge in the background return false, nil } -func (s *filenameStrategy) Do(args interface{}) error { +func (s *filenameStrategy) Do(args any) error { if !s.purging.CAS(false, true) { return terror.ErrRelayThisStrategyIsPurging.Generate() } @@ -202,12 +202,12 @@ func newInactiveStrategy() PurgeStrategy { } } -func (s *inactiveStrategy) Check(args interface{}) (bool, error) { +func (s *inactiveStrategy) Check(args any) (bool, error) { // do not support purge in the background return false, nil } -func (s *inactiveStrategy) Do(args interface{}) error { +func (s *inactiveStrategy) Do(args any) error { if !s.purging.CAS(false, true) { return terror.ErrRelayThisStrategyIsPurging.Generate() } @@ -259,7 +259,7 @@ func newSpaceStrategy() PurgeStrategy { } } -func (s *spaceStrategy) Check(args interface{}) (bool, error) { +func (s *spaceStrategy) Check(args any) (bool, error) { sa, ok := args.(*spaceArgs) if !ok { return false, terror.ErrRelayPurgeArgsNotValid.Generate(args, args) @@ -274,7 +274,7 @@ func (s *spaceStrategy) Check(args interface{}) (bool, error) { return storageSize.Available < requiredBytes, nil } -func (s *spaceStrategy) Do(args interface{}) error { +func (s *spaceStrategy) Do(args any) error { if !s.purging.CAS(false, true) { return terror.ErrRelayThisStrategyIsPurging.Generate() } @@ -329,7 +329,7 @@ func newTimeStrategy() PurgeStrategy { } } -func (s *timeStrategy) Check(args interface{}) (bool, error) { +func (s *timeStrategy) Check(args any) (bool, error) { // for time strategy, we always try to do the purging return true, nil } @@ -337,7 +337,7 @@ func (s *timeStrategy) Check(args interface{}) (bool, error) { func (s *timeStrategy) Stop() { } -func (s *timeStrategy) Do(args interface{}) error { +func (s *timeStrategy) Do(args any) error { if !s.purging.CAS(false, true) { return terror.ErrRelayThisStrategyIsPurging.Generate() } diff --git a/dm/relay/purger.go b/dm/relay/purger.go index d69f6f4e4a..6c7830a6ef 100644 --- a/dm/relay/purger.go +++ b/dm/relay/purger.go @@ -114,11 +114,9 @@ func (p *relayPurger) Start() { p.logger.Info("starting relay log purger", zap.Reflect("config", p.cfg)) // Close will wait process to return - p.wg.Add(1) - go func() { - defer p.wg.Done() + p.wg.Go(func() { p.run() - }() + }) } // run starts running the process diff --git a/dm/relay/relay.go b/dm/relay/relay.go index ee783bcf52..ddb0cf894f 100644 --- a/dm/relay/relay.go +++ b/dm/relay/relay.go @@ -89,9 +89,9 @@ type Process interface { // Pause pauses a running relay log process unit Pause() // Error returns error message if having one - Error() interface{} + Error() any // Status returns status of relay log process unit. - Status(sourceStatus *binlog.SourceStatus) interface{} + Status(sourceStatus *binlog.SourceStatus) any // Close does some clean works Close() // IsClosed returns whether relay log process unit was closed @@ -1013,7 +1013,7 @@ func (r *Relay) IsActive(uuid, filename string) (bool, int64) { } // Status implements the dm.Unit interface. -func (r *Relay) Status(sourceStatus *binlog.SourceStatus) interface{} { +func (r *Relay) Status(sourceStatus *binlog.SourceStatus) any { r.RLock() defer r.RUnlock() uuid, relayPos := r.meta.Pos() @@ -1045,7 +1045,7 @@ func (r *Relay) Status(sourceStatus *binlog.SourceStatus) interface{} { } // Error implements the dm.Unit interface. -func (r *Relay) Error() interface{} { +func (r *Relay) Error() any { return &pb.RelayError{} } diff --git a/dm/relay/relay_test.go b/dm/relay/relay_test.go index ec6f3ffbc2..6035b5c65e 100644 --- a/dm/relay/relay_test.go +++ b/dm/relay/relay_test.go @@ -380,9 +380,9 @@ func genBinlogEventsWithGTIDs(c *check.C, flavor string, previousGTIDSet, latest schema = "db" table = "tbl1" ) - for i := 0; i < 10; i++ { - insertRows := make([][]interface{}, 0, 1) - insertRows = append(insertRows, []interface{}{int32(i)}) + for i := range 10 { + insertRows := make([][]any, 0, 1) + insertRows = append(insertRows, []any{int32(i)}) dmlData := []*event.DMLData{ { TableID: tableID, @@ -898,7 +898,7 @@ func (t *testRelaySuite) TestRecoverMySQL(c *check.C) { // write an completed transaction f, err = os.OpenFile(fullName, os.O_WRONLY|os.O_APPEND, 0o644) c.Assert(err, check.IsNil) - for i := 0; i < len(extraEvents); i++ { + for i := range extraEvents { _, err = f.Write(extraEvents[i].RawData) c.Assert(err, check.IsNil) } diff --git a/dm/relay/relay_writer_test.go b/dm/relay/relay_writer_test.go index bd686bab91..07f644f2e0 100644 --- a/dm/relay/relay_writer_test.go +++ b/dm/relay/relay_writer_test.go @@ -380,9 +380,9 @@ func TestWriteMultiEvents(t *testing.T) { var ( tableID uint64 = 8 columnType = []byte{gmysql.MYSQL_TYPE_LONG} - insertRows = make([][]interface{}, 1) + insertRows = make([][]any, 1) ) - insertRows[0] = []interface{}{int32(1)} + insertRows[0] = []any{int32(1)} events, data, err = g.GenDMLEvents(replication.WRITE_ROWS_EVENTv2, []*event.DMLData{ {TableID: tableID, Schema: "db", Table: "tbl", ColumnType: columnType, Rows: insertRows}, }, 0) diff --git a/dm/relay/remote_retry_test.go b/dm/relay/remote_retry_test.go index a0611b132a..a606c9714c 100644 --- a/dm/relay/remote_retry_test.go +++ b/dm/relay/remote_retry_test.go @@ -41,13 +41,13 @@ func (t *testReaderRetrySuite) TestRetry(c *check.C) { ctx := context.Background() // check some times - for i := 0; i < 3; i++ { + for range 3 { c.Assert(rr.Check(ctx, retryableErr), check.IsTrue) } c.Assert(rr.bf.Current(), check.Equals, 8*time.Millisecond) // check more times, until reach Max - for i := 0; i < 10; i++ { + for range 10 { c.Assert(rr.Check(ctx, retryableErr), check.IsTrue) } c.Assert(rr.bf.Current(), check.Equals, rr.cfg.BackoffMax) diff --git a/dm/simulator/mcp/mcp_test.go b/dm/simulator/mcp/mcp_test.go index 20f6a9016f..0bf58caa31 100644 --- a/dm/simulator/mcp/mcp_test.go +++ b/dm/simulator/mcp/mcp_test.go @@ -33,10 +33,10 @@ func (s *testMCPSuite) SetupSuite() { func (s *testMCPSuite) SetupTest() { mcp := NewModificationCandidatePool(8192) - for i := 0; i < 4096; i++ { + for i := range 4096 { mcp.keyPool = append(mcp.keyPool, &UniqueKey{ rowID: i, - value: map[string]interface{}{ + value: map[string]any{ "id": i, }, }) @@ -47,7 +47,7 @@ func (s *testMCPSuite) SetupTest() { func (s *testMCPSuite) TestNextUK() { allHitRowIDs := map[int]int{} repeatCnt := 20 - for i := 0; i < repeatCnt; i++ { + for range repeatCnt { theUK := s.mcp.NextUK() s.Require().NotNil(theUK, "the picked UK should not be nil") theRowID := theUK.GetRowID() @@ -74,10 +74,10 @@ func (s *testMCPSuite) TestParallelNextUK() { rowIDCh := make(chan int, workerCnt) var wg sync.WaitGroup wg.Add(workerCnt) - for i := 0; i < workerCnt; i++ { + for range workerCnt { go func() { defer wg.Done() - for i := 0; i < repeatCnt; i++ { + for range repeatCnt { theUK := s.mcp.NextUK() if theUK != nil { rowIDCh <- theUK.GetRowID() @@ -122,7 +122,7 @@ func (s *testMCPSuite) TestMCPAddDeleteBasic() { startPoolSize := curPoolSize repeatCnt = 5 for i := 0; i < repeatCnt; i++ { - theUK := NewUniqueKey(-1, map[string]interface{}{ + theUK := NewUniqueKey(-1, map[string]any{ "id": rand.Int(), }) err = s.mcp.AddUK(theUK) @@ -188,8 +188,8 @@ func (s *testMCPSuite) TestMCPAddDeleteInParallel() { defer func() { ch <- err }() - for i := 0; i < 5; i++ { - theUK := NewUniqueKey(-1, map[string]interface{}{ + for range 5 { + theUK := NewUniqueKey(-1, map[string]any{ "id": rand.Int(), }) err = s.mcp.AddUK(theUK) @@ -209,7 +209,7 @@ func (s *testMCPSuite) TestMCPAddDeleteInParallel() { defer func() { ch <- err }() - for i := 0; i < 5; i++ { + for range 5 { theDelUK := s.mcp.NextUK() deletedRowID := theDelUK.rowID err = s.mcp.DeleteUK(theDelUK) diff --git a/dm/simulator/mcp/uk.go b/dm/simulator/mcp/uk.go index 73f300a149..ac3ba3e17e 100644 --- a/dm/simulator/mcp/uk.go +++ b/dm/simulator/mcp/uk.go @@ -15,6 +15,7 @@ package mcp import ( "fmt" + "maps" "sort" "strings" "sync" @@ -30,20 +31,18 @@ type UniqueKey struct { rowID int // value is the real value of all the UK columns. // The key is the column name, the value is the real value. - value map[string]interface{} + value map[string]any } // NewUniqueKey creates a new unique key instance. // the map values are cloned into the new UK instance, // so that the further changes in the value map won't affect the values inside the UK. -func NewUniqueKey(rowID int, value map[string]interface{}) *UniqueKey { +func NewUniqueKey(rowID int, value map[string]any) *UniqueKey { result := &UniqueKey{ rowID: rowID, - value: make(map[string]interface{}), - } - for k, v := range value { - result.value[k] = v + value: make(map[string]any), } + maps.Copy(result.value, value) return result } @@ -65,13 +64,11 @@ func (uk *UniqueKey) SetRowID(rowID int) { // GetValue gets the UK value map of a unique key. // The returned value is cloned, so that further modifications won't affect the value inside the UK. -func (uk *UniqueKey) GetValue() map[string]interface{} { +func (uk *UniqueKey) GetValue() map[string]any { uk.RLock() defer uk.RUnlock() - result := make(map[string]interface{}) - for k, v := range uk.value { - result[k] = v - } + result := make(map[string]any) + maps.Copy(result, uk.value) return result } @@ -96,13 +93,11 @@ func (uk *UniqueKey) GetValueHash() string { // SetValue sets the UK value map. // The input values are cloned into the UK, // and further modifications on the input map won't affect the values inside the UK. -func (uk *UniqueKey) SetValue(value map[string]interface{}) { +func (uk *UniqueKey) SetValue(value map[string]any) { uk.Lock() defer uk.Unlock() - uk.value = make(map[string]interface{}) - for k, v := range value { - uk.value[k] = v - } + uk.value = make(map[string]any) + maps.Copy(uk.value, value) } // Clone is to clone a UK into a new one. @@ -112,11 +107,9 @@ func (uk *UniqueKey) Clone() *UniqueKey { defer uk.RUnlock() result := &UniqueKey{ rowID: uk.rowID, - value: map[string]interface{}{}, - } - for k, v := range uk.value { - result.value[k] = v + value: map[string]any{}, } + maps.Copy(result.value, uk.value) return result } diff --git a/dm/simulator/mcp/uk_test.go b/dm/simulator/mcp/uk_test.go index 4fdfae4233..9a69e5a640 100644 --- a/dm/simulator/mcp/uk_test.go +++ b/dm/simulator/mcp/uk_test.go @@ -34,7 +34,7 @@ func (s *testUniqueKeySuite) TestUKClone() { origUKCol2Value := "COL1" originalUK := &UniqueKey{ rowID: -1, - value: map[string]interface{}{ + value: map[string]any{ "col1": origUKCol1Value, "col2": origUKCol2Value, }, @@ -56,7 +56,7 @@ func (s *testUniqueKeySuite) TestUKClone() { func (s *testUniqueKeySuite) TestUKChangeBasic() { col1Value := 111 col2Value := "aaa" - theValueMap := map[string]interface{}{ + theValueMap := map[string]any{ "col1": col1Value, "col2": col2Value, } @@ -79,7 +79,7 @@ func (s *testUniqueKeySuite) TestUKChangeBasic() { newCol1Value := 333 newCol2Value := "ccc" - newValueMap := map[string]interface{}{ + newValueMap := map[string]any{ "col1": newCol1Value, "col2": newCol2Value, } @@ -102,13 +102,13 @@ func (s *testUniqueKeySuite) TestUKParallelChange() { targetID := 100 workerCnt := 10 wg.Add(workerCnt) - for i := 0; i < workerCnt; i++ { + for range workerCnt { go func() { defer wg.Done() <-pendingCh for i := 1; i <= targetID; i++ { theUK.SetRowID(i) - theUK.SetValue(map[string]interface{}{ + theUK.SetValue(map[string]any{ "id": i, }) } @@ -127,14 +127,14 @@ func (s *testUniqueKeySuite) TestUKValueEqual() { col2Value := "aaa" uk1 := &UniqueKey{ rowID: -1, - value: map[string]interface{}{ + value: map[string]any{ "col1": col1Value, "col2": col2Value, }, } uk2 := &UniqueKey{ rowID: 100, - value: map[string]interface{}{ + value: map[string]any{ "col1": col1Value, "col2": col2Value, }, @@ -143,7 +143,7 @@ func (s *testUniqueKeySuite) TestUKValueEqual() { s.Equal(true, uk2.IsValueEqual(uk1), "uk2 should equal uk1 on value") uk3 := &UniqueKey{ rowID: 100, - value: map[string]interface{}{ + value: map[string]any{ "col1": col1Value, "col2": "bbb", }, @@ -152,7 +152,7 @@ func (s *testUniqueKeySuite) TestUKValueEqual() { s.Equal(false, uk3.IsValueEqual(uk1), "uk3 should not equal uk1 on value") uk4 := &UniqueKey{ rowID: 100, - value: map[string]interface{}{ + value: map[string]any{ "col3": 321, }, } @@ -160,7 +160,7 @@ func (s *testUniqueKeySuite) TestUKValueEqual() { s.Equal(false, uk4.IsValueEqual(uk1), "uk4 should not equal uk1 on value") uk5 := &UniqueKey{ rowID: 100, - value: map[string]interface{}{ + value: map[string]any{ "col3": 321, "col1": col1Value, "col2": col2Value, diff --git a/dm/simulator/sqlgen/impl.go b/dm/simulator/sqlgen/impl.go index d9f88247b8..9bad1e092c 100644 --- a/dm/simulator/sqlgen/impl.go +++ b/dm/simulator/sqlgen/impl.go @@ -75,7 +75,7 @@ func (g *sqlGeneratorImpl) GenTruncateTable() (string, error) { return outputString(truncateTree) } -func (g *sqlGeneratorImpl) generateWhereClause(theUK map[string]interface{}) (ast.ExprNode, error) { +func (g *sqlGeneratorImpl) generateWhereClause(theUK map[string]any) (ast.ExprNode, error) { compareExprs := make([]ast.ExprNode, 0) // iterate the existing UKs, to make sure all the uk columns has values for ukColName := range g.ukMap { @@ -172,7 +172,7 @@ func (g *sqlGeneratorImpl) GenUpdateRow(theUK *mcp.UniqueKey) (string, error) { // The new row's unique key is also provided, // so that it can be further added into an MCP. func (g *sqlGeneratorImpl) GenInsertRow() (string, *mcp.UniqueKey, error) { - ukValues := make(map[string]interface{}) + ukValues := make(map[string]any) columnNames := []*ast.ColumnName{} values := []ast.ExprNode{} for _, col := range g.columnMap { diff --git a/dm/simulator/sqlgen/impl_test.go b/dm/simulator/sqlgen/impl_test.go index 775ebdaa90..b9468f4c00 100644 --- a/dm/simulator/sqlgen/impl_test.go +++ b/dm/simulator/sqlgen/impl_test.go @@ -225,14 +225,14 @@ func (s *testSQLGenImplSuite) TestDMLBasic() { s.checkTruncateSQL(sql) theMCP := mcp.NewModificationCandidatePool(8192) - for i := 0; i < 4096; i++ { + for i := range 4096 { s.Nil( - theMCP.AddUK(mcp.NewUniqueKey(i, map[string]interface{}{ + theMCP.AddUK(mcp.NewUniqueKey(i, map[string]any{ "id": i, })), ) } - for i := 0; i < 10; i++ { + for range 10 { uk = theMCP.NextUK() sql, err = g.GenUpdateRow(uk) s.Nil(err, "generate update sql error") @@ -264,7 +264,7 @@ func (s *testSQLGenImplSuite) TestWhereNULL() { UniqueKeyColumnNames: []string{"name", "team_id"}, } g := NewSQLGeneratorImpl(theTableConfig) - theUK := mcp.NewUniqueKey(-1, map[string]interface{}{ + theUK := mcp.NewUniqueKey(-1, map[string]any{ "name": "ABCDEFG", "team_id": nil, }) @@ -286,7 +286,7 @@ func (s *testSQLGenImplSuite) TestDMLAbnormalUK() { uk *mcp.UniqueKey ) g := NewSQLGeneratorImpl(s.tableConfig) - uk = mcp.NewUniqueKey(-1, map[string]interface{}{ + uk = mcp.NewUniqueKey(-1, map[string]any{ "abcdefg": 123, }) _, err = g.GenUpdateRow(uk) @@ -294,7 +294,7 @@ func (s *testSQLGenImplSuite) TestDMLAbnormalUK() { _, err = g.GenDeleteRow(uk) s.NotNil(err) - uk = mcp.NewUniqueKey(-1, map[string]interface{}{ + uk = mcp.NewUniqueKey(-1, map[string]any{ "id": 123, "abcdefg": 321, }) @@ -308,7 +308,7 @@ func (s *testSQLGenImplSuite) TestDMLAbnormalUK() { s.T().Logf("Generated SQL: %s\n", sql) s.checkDeleteSQL(sql, s.tableConfig.UniqueKeyColumnNames) - uk = mcp.NewUniqueKey(-1, map[string]interface{}{}) + uk = mcp.NewUniqueKey(-1, map[string]any{}) _, err = g.GenUpdateRow(uk) s.NotNil(err) } @@ -332,13 +332,13 @@ func (s *testSQLGenImplSuite) TestDMLWithNoUK() { s.T().Logf("Generated SQL: %s\n; Unique key: %v\n", sql, theUK) s.checkInsertSQL(sql) - theUK = mcp.NewUniqueKey(-1, map[string]interface{}{}) + theUK = mcp.NewUniqueKey(-1, map[string]any{}) _, err = g.GenUpdateRow(theUK) s.NotNil(err) _, err = g.GenDeleteRow(theUK) s.NotNil(err) - theUK = mcp.NewUniqueKey(-1, map[string]interface{}{ + theUK = mcp.NewUniqueKey(-1, map[string]any{ "id": 123, // the column is filtered out by the UK configs }) _, err = g.GenUpdateRow(theUK) diff --git a/dm/syncer/binlogstream/binlog_locations_test.go b/dm/syncer/binlogstream/binlog_locations_test.go index 81b7981e7d..3d338acfa1 100644 --- a/dm/syncer/binlogstream/binlog_locations_test.go +++ b/dm/syncer/binlogstream/binlog_locations_test.go @@ -31,7 +31,7 @@ import ( type ( mockBinlogEvent struct { typ int - args []interface{} + args []any } ) @@ -118,7 +118,7 @@ func (s *testLocationSuite) generateEvents(binlogEvents []mockBinlogEvent) []*re Schema: e.args[1].(string), Table: e.args[2].(string), ColumnType: e.args[3].([]byte), - Rows: e.args[4].([][]interface{}), + Rows: e.args[4].([][]any), }, } eventType := replication.WRITE_ROWS_EVENTv2 @@ -177,8 +177,8 @@ func (s *testLocationSuite) updateLastEventGSet(events []*replication.BinlogEven func (s *testLocationSuite) generateDMLEvents() []*replication.BinlogEvent { events := s.generateEvents([]mockBinlogEvent{ - {Headers, []interface{}{s.binlogFile}}, - {Write, []interface{}{uint64(8), "foo", "bar", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(1)}}}}, + {Headers, []any{s.binlogFile}}, + {Write, []any{uint64(8), "foo", "bar", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(1)}}}}, }) s.updateLastEventGSet(events) @@ -187,8 +187,8 @@ func (s *testLocationSuite) generateDMLEvents() []*replication.BinlogEvent { func (s *testLocationSuite) generateDDLEvents() []*replication.BinlogEvent { events := s.generateEvents([]mockBinlogEvent{ - {Headers, []interface{}{s.binlogFile}}, - {DBCreate, []interface{}{"foo1"}}, + {Headers, []any{s.binlogFile}}, + {DBCreate, []any{"foo1"}}, }) s.updateLastEventGSet(events) @@ -442,8 +442,8 @@ func (s *testLocationSuite) generateDMLQueryEvents() []*replication.BinlogEvent s.eventsGenerator, err = event.NewGenerator(s.flavor, s.serverID, s.binlogPos, s.lastGTID, s.prevGSet, 0) s.Require().NoError(err) events := s.generateEvents([]mockBinlogEvent{ - {Headers, []interface{}{s.binlogFile}}, - {DMLQuery, []interface{}{"foo", "INSERT INTO v VALUES(1)"}}, + {Headers, []any{s.binlogFile}}, + {DMLQuery, []any{"foo", "INSERT INTO v VALUES(1)"}}, }) s.updateLastEventGSet(events) @@ -502,10 +502,10 @@ func (s *testLocationSuite) generateRotateAndDMLEvents() []*replication.BinlogEv s.eventsGenerator, err = event.NewGenerator(s.flavor, s.serverID, s.binlogPos, s.lastGTID, s.prevGSet, 0) s.Require().NoError(err) events := s.generateEvents([]mockBinlogEvent{ - {Headers, []interface{}{s.binlogFile}}, - {Rotate, []interface{}{s.nextBinlogFile}}, - {Headers, []interface{}{s.nextBinlogFile}}, - {Write, []interface{}{uint64(8), "foo", "bar", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(1)}}}}, + {Headers, []any{s.binlogFile}}, + {Rotate, []any{s.nextBinlogFile}}, + {Headers, []any{s.nextBinlogFile}}, + {Write, []any{uint64(8), "foo", "bar", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(1)}}}}, }) s.updateLastEventGSet(events) diff --git a/dm/syncer/binlogstream/stream_modifier.go b/dm/syncer/binlogstream/stream_modifier.go index 7cd6aec8bc..053dc75a4e 100644 --- a/dm/syncer/binlogstream/stream_modifier.go +++ b/dm/syncer/binlogstream/stream_modifier.go @@ -199,11 +199,7 @@ func (m *streamModifier) ListEqualAndAfter(posStr string) []*pb.HandleWorkerErro // the argument. // RemoveOutdated will not remove the operator equals or after the `front`. func (m *streamModifier) RemoveOutdated(pos mysql.Position) { - newStartIdx := m.minIdxLargerOrEqual(pos) - - if newStartIdx > m.nextOp { - newStartIdx = m.nextOp - } + newStartIdx := min(m.minIdxLargerOrEqual(pos), m.nextOp) m.ops = m.ops[newStartIdx:] m.nextOp -= newStartIdx diff --git a/dm/syncer/binlogstream/streamer_controller_test.go b/dm/syncer/binlogstream/streamer_controller_test.go index 597be52f0d..2a1e8d3910 100644 --- a/dm/syncer/binlogstream/streamer_controller_test.go +++ b/dm/syncer/binlogstream/streamer_controller_test.go @@ -69,7 +69,7 @@ func TestCanErrorRetry(t *testing.T) { mockErr := errors.New("test") // local binlog puller can always retry - for i := 0; i < 5; i++ { + for range 5 { require.True(t, controller.CanRetry(mockErr)) } diff --git a/dm/syncer/causality_test.go b/dm/syncer/causality_test.go index db7390c4d3..6c74b087f8 100644 --- a/dm/syncer/causality_test.go +++ b/dm/syncer/causality_test.go @@ -86,24 +86,24 @@ func TestCausality(t *testing.T) { syncer.metricsProxies = metrics.DefaultMetricsProxies.CacheForOneTask("task", "worker", "source") causalityCh := causalityWrap(jobCh, syncer) testCases := []struct { - preVals []interface{} - postVals []interface{} + preVals []any + postVals []any }{ { - postVals: []interface{}{1, 2}, + postVals: []any{1, 2}, }, { - postVals: []interface{}{2, 3}, + postVals: []any{2, 3}, }, { - preVals: []interface{}{2, 3}, - postVals: []interface{}{3, 4}, + preVals: []any{2, 3}, + postVals: []any{3, 4}, }, { - preVals: []interface{}{1, 2}, + preVals: []any{1, 2}, }, { - postVals: []interface{}{1, 3}, + postVals: []any{1, 3}, }, } results := []opType{dml, dml, dml, dml, conflict, dml} @@ -195,7 +195,7 @@ func (s *testSyncerSuite) TestCasualityRelation(c *check.C) { c.Assert(rmMap.prevFlushJobSeq, check.Not(check.Equals), int64(index)) } - for ti := 0; ti < index; ti++ { + for ti := range index { _, ok := rm.get(testCases[ti].key) c.Assert(ok, check.Equals, false) } diff --git a/dm/syncer/checkpoint.go b/dm/syncer/checkpoint.go index d227c64c0e..bf7b9b9f59 100644 --- a/dm/syncer/checkpoint.go +++ b/dm/syncer/checkpoint.go @@ -277,7 +277,7 @@ type CheckPoint interface { // by extraSQLs and extraArgs. Currently extraSQLs contain shard meta only. // @exceptTables: [[schema, table]... ] // corresponding to Meta.Flush - FlushPointsExcept(tctx *tcontext.Context, snapshotID int, exceptTables []*filter.Table, extraSQLs []string, extraArgs [][]interface{}) error + FlushPointsExcept(tctx *tcontext.Context, snapshotID int, exceptTables []*filter.Table, extraSQLs []string, extraArgs [][]any) error // FlushPointsWithTableInfos flushed the table points with given table infos FlushPointsWithTableInfos(tctx *tcontext.Context, tables []*filter.Table, tis []*model.TableInfo) error @@ -518,7 +518,7 @@ func (cp *RemoteCheckPoint) Clear(tctx *tcontext.Context) error { tctx2, cp.metricProxies, []string{`DELETE FROM ` + cp.tableName + ` WHERE id = ?`}, - []interface{}{cp.id}, + []any{cp.id}, ) if err != nil { return err @@ -599,7 +599,7 @@ func (cp *RemoteCheckPoint) DeleteTablePoint(tctx *tcontext.Context, table *filt tctx2, cp.metricProxies, []string{`DELETE FROM ` + cp.tableName + ` WHERE id = ? AND cp_schema = ? AND cp_table = ?`}, - []interface{}{cp.id, sourceSchema, sourceTable}, + []any{cp.id, sourceSchema, sourceTable}, ) if err != nil { return err @@ -620,7 +620,7 @@ func (cp *RemoteCheckPoint) DeleteAllTablePoint(tctx *tcontext.Context) error { tctx2, cp.metricProxies, []string{`DELETE FROM ` + cp.tableName + ` WHERE id = ? AND is_global = ?`}, - []interface{}{cp.id, false}, + []any{cp.id, false}, ) if err != nil { return err @@ -645,7 +645,7 @@ func (cp *RemoteCheckPoint) DeleteSchemaPoint(tctx *tcontext.Context, sourceSche tctx2, cp.metricProxies, []string{`DELETE FROM ` + cp.tableName + ` WHERE id = ? AND cp_schema = ?`}, - []interface{}{cp.id, sourceSchema}, + []any{cp.id, sourceSchema}, ) if err != nil { return err @@ -707,7 +707,7 @@ func (cp *RemoteCheckPoint) FlushPointsExcept( snapshotID int, exceptTables []*filter.Table, extraSQLs []string, - extraArgs [][]interface{}, + extraArgs [][]any, ) error { cp.Lock() @@ -730,7 +730,7 @@ func (cp *RemoteCheckPoint) FlushPointsExcept( } sqls := make([]string, 0, 100) - args := make([][]interface{}, 0, 100) + args := make([][]any, 0, 100) type tableCpSnapshotTuple struct { tableCp *binlogPoint // current table checkpoint location @@ -813,13 +813,10 @@ func (cp *RemoteCheckPoint) FlushPointsWithTableInfos(tctx *tcontext.Context, ta } for i := 0; i < len(tables); i += batchFlushPoints { - end := i + batchFlushPoints - if end > len(tables) { - end = len(tables) - } + end := min(i+batchFlushPoints, len(tables)) sqls := make([]string, 0, batchFlushPoints) - args := make([][]interface{}, 0, batchFlushPoints) + args := make([][]any, 0, batchFlushPoints) points := make([]*binlogPoint, 0, batchFlushPoints) for j := i; j < end; j++ { table := tables[j] @@ -872,7 +869,7 @@ func (cp *RemoteCheckPoint) FlushSafeModeExitPoint(tctx *tcontext.Context) error defer cp.RUnlock() sqls := make([]string, 1) - args := make([][]interface{}, 1) + args := make([][]any, 1) // use FlushedGlobalPoint here to avoid update global checkpoint locationG := cp.FlushedGlobalPoint() @@ -995,8 +992,8 @@ func (cp *RemoteCheckPoint) prepare(tctx *tcontext.Context) error { func (cp *RemoteCheckPoint) createSchema(tctx *tcontext.Context) error { // TODO(lance6716): change ColumnName to IdentName or something sql2 := fmt.Sprintf("CREATE SCHEMA IF NOT EXISTS %s", dbutil.ColumnName(cp.cfg.MetaSchema)) - args := make([]interface{}, 0) - _, err := cp.dbConn.ExecuteSQL(tctx, cp.metricProxies, []string{sql2}, [][]interface{}{args}...) + args := make([]any, 0) + _, err := cp.dbConn.ExecuteSQL(tctx, cp.metricProxies, []string{sql2}, [][]any{args}...) cp.logCtx.L().Info("create checkpoint schema", zap.String("statement", sql2)) return err } @@ -1254,7 +1251,7 @@ func (cp *RemoteCheckPoint) LoadMeta(ctx context.Context) error { } // genUpdateSQL generates SQL and arguments for update checkpoint. -func (cp *RemoteCheckPoint) genUpdateSQL(cpSchema, cpTable string, location binlog.Location, safeModeExitLoc *binlog.Location, tiBytes []byte, isGlobal bool) (string, []interface{}) { +func (cp *RemoteCheckPoint) genUpdateSQL(cpSchema, cpTable string, location binlog.Location, safeModeExitLoc *binlog.Location, tiBytes []byte, isGlobal bool) (string, []any) { // use `INSERT INTO ... ON DUPLICATE KEY UPDATE` rather than `REPLACE INTO` // to keep `create_time`, `update_time` correctly sql2 := `INSERT INTO ` + cp.tableName + ` @@ -1292,7 +1289,7 @@ func (cp *RemoteCheckPoint) genUpdateSQL(cpSchema, cpTable string, location binl } // convert tiBytes to string to get a readable log - args := []interface{}{ + args := []any{ cp.id, cpSchema, cpTable, location.Position.Name, location.Position.Pos, location.GTIDSetStr(), exitSafeName, exitSafePos, exitSafeGTIDStr, string(tiBytes), isGlobal, } diff --git a/dm/syncer/checkpoint_flush_worker.go b/dm/syncer/checkpoint_flush_worker.go index 403424782d..2380d810ad 100644 --- a/dm/syncer/checkpoint_flush_worker.go +++ b/dm/syncer/checkpoint_flush_worker.go @@ -30,7 +30,7 @@ type checkpointFlushTask struct { // extra sharding ddl sqls exceptTables []*filter.Table shardMetaSQLs []string - shardMetaArgs [][]interface{} + shardMetaArgs [][]any // async flush job asyncflushJob *job // error chan for sync flush diff --git a/dm/syncer/checkpoint_test.go b/dm/syncer/checkpoint_test.go index 22159d2570..a8ec200b7a 100644 --- a/dm/syncer/checkpoint_test.go +++ b/dm/syncer/checkpoint_test.go @@ -260,8 +260,8 @@ func (s *testCheckpointSuite) testGlobalCheckPoint(c *check.C, cp CheckPoint) { dir := c.MkDir() filename := filepath.Join(dir, "metadata") - err = os.WriteFile(filename, []byte( - fmt.Sprintf("SHOW MASTER STATUS:\n\tLog: %s\n\tPos: %d\n\tGTID:\n\nSHOW SLAVE STATUS:\n\tHost: %s\n\tLog: %s\n\tPos: %d\n\tGTID:\n\n", pos1.Name, pos1.Pos, "slave_host", pos1.Name, pos1.Pos+1000)), + err = os.WriteFile(filename, + fmt.Appendf(nil, "SHOW MASTER STATUS:\n\tLog: %s\n\tPos: %d\n\tGTID:\n\nSHOW SLAVE STATUS:\n\tHost: %s\n\tLog: %s\n\tPos: %d\n\tGTID:\n\n", pos1.Name, pos1.Pos, "slave_host", pos1.Name, pos1.Pos+1000), 0o644) c.Assert(err, check.IsNil) s.cfg.Mode = config.ModeAll @@ -285,8 +285,8 @@ func (s *testCheckpointSuite) testGlobalCheckPoint(c *check.C, cp CheckPoint) { c.Assert(err, check.IsNil) // check dumpling write exitSafeModeLocation in metadata - err = os.WriteFile(filename, []byte( - fmt.Sprintf(`SHOW MASTER STATUS: + err = os.WriteFile(filename, + fmt.Appendf(nil, `SHOW MASTER STATUS: Log: %s Pos: %d GTID: @@ -301,7 +301,7 @@ SHOW MASTER STATUS: /* AFTER CONNECTION POOL ESTABLISHED */ Log: %s Pos: %d GTID: -`, pos1.Name, pos1.Pos, "slave_host", pos1.Name, pos1.Pos+1000, pos2.Name, pos2.Pos)), 0o644) +`, pos1.Name, pos1.Pos, "slave_host", pos1.Name, pos1.Pos+1000, pos2.Name, pos2.Pos), 0o644) c.Assert(err, check.IsNil) c.Assert(cp.LoadMeta(tctx.Ctx), check.IsNil) @@ -539,14 +539,14 @@ func TestRemoteCheckPointLoadIntoSchemaTracker(t *testing.T) { // create 100K comment string comment := make([]byte, 0, 100000) - for i := 0; i < 100000; i++ { + for range 100000 { comment = append(comment, 'A') } ti.Comment = string(comment) tp1 = tablePoint{ti: ti} amount := 100 - for i := 0; i < amount; i++ { + for i := range amount { tableName := fmt.Sprintf("tbl_%d", i) checkpoint.points[tbl1.Schema][tableName] = &binlogPoint{flushedPoint: tp1} } diff --git a/dm/syncer/compactor_test.go b/dm/syncer/compactor_test.go index 406f0db206..404d24851a 100644 --- a/dm/syncer/compactor_test.go +++ b/dm/syncer/compactor_test.go @@ -34,7 +34,7 @@ import ( ) // mockExecute mock a kv store. -func mockExecute(kv map[interface{}][]interface{}, dmls []*sqlmodel.RowChange) map[interface{}][]interface{} { +func mockExecute(kv map[any][]any, dmls []*sqlmodel.RowChange) map[any][]any { for _, dml := range dmls { switch dml.Type() { case sqlmodel.RowChangeInsert: @@ -86,12 +86,12 @@ func (s *testSyncerSuite) TestCompactJob(c *check.C) { updateIdentifyProbability := 0.1 // generate DMLs - kv := make(map[interface{}][]interface{}) - for i := 0; i < dmlNum; i++ { + kv := make(map[any][]any) + for range dmlNum { newID := rand.Intn(maxID) newCol1 := rand.Intn(maxID * 10) newName := randString(rand.Intn(20)) - values := []interface{}{newID, newCol1, newName} + values := []any{newID, newCol1, newName} oldValues, ok := kv[newID] if !ok { // insert @@ -101,7 +101,7 @@ func (s *testSyncerSuite) TestCompactJob(c *check.C) { // update // check whether to update ID if rand.Float64() < updateIdentifyProbability { - for try := 0; try < 10; try++ { + for range 10 { newID := rand.Intn(maxID) if _, ok := kv[newID]; !ok { values[0] = newID @@ -120,15 +120,12 @@ func (s *testSyncerSuite) TestCompactJob(c *check.C) { dmls = append(dmls, dml) } - kv = make(map[interface{}][]interface{}) - compactKV := make(map[interface{}][]interface{}) + kv = make(map[any][]any) + compactKV := make(map[any][]any) // mock compactJob for i := 0; i < len(dmls); i += batch { - end := i + batch - if end > len(dmls) { - end = len(dmls) - } + end := min(i+batch, len(dmls)) kv = mockExecute(kv, dmls[i:end]) for _, dml := range dmls[i:end] { @@ -184,37 +181,37 @@ func (s *testSyncerSuite) TestCompactorSafeMode(c *check.C) { { input: []*job{ newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, nil, []interface{}{1, 1, "a"}, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, nil, []any{1, 1, "a"}, ti, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, nil, []interface{}{2, 2, "b"}, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, nil, []any{2, 2, "b"}, ti, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, []interface{}{1, 1, "a"}, []interface{}{3, 3, "c"}, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, []any{1, 1, "a"}, []any{3, 3, "c"}, ti, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, []interface{}{2, 2, "b"}, nil, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, []any{2, 2, "b"}, nil, ti, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, nil, []interface{}{1, 1, "a"}, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, nil, []any{1, 1, "a"}, ti, nil, nil), ec, ), }, output: []*job{ newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, nil, []interface{}{3, 3, "c"}, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, nil, []any{3, 3, "c"}, ti, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, []interface{}{2, 2, "b"}, nil, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, []any{2, 2, "b"}, nil, ti, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, nil, []interface{}{1, 1, "a"}, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, nil, []any{1, 1, "a"}, ti, nil, nil), ecWithSafeMode, ), }, @@ -223,45 +220,45 @@ func (s *testSyncerSuite) TestCompactorSafeMode(c *check.C) { { input: []*job{ newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, nil, []interface{}{1, 1, "a"}, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, nil, []any{1, 1, "a"}, ti, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, nil, []interface{}{2, 2, "b"}, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, nil, []any{2, 2, "b"}, ti, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, []interface{}{1, 1, "a"}, []interface{}{3, 3, "c"}, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, []any{1, 1, "a"}, []any{3, 3, "c"}, ti, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, []interface{}{2, 2, "b"}, nil, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, []any{2, 2, "b"}, nil, ti, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, nil, []interface{}{1, 1, "a"}, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, nil, []any{1, 1, "a"}, ti, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, nil, []interface{}{2, 2, "b"}, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, nil, []any{2, 2, "b"}, ti, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, []interface{}{2, 2, "b"}, []interface{}{2, 2, "c"}, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, []any{2, 2, "b"}, []any{2, 2, "c"}, ti, nil, nil), ec, ), }, output: []*job{ newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, nil, []interface{}{3, 3, "c"}, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, nil, []any{3, 3, "c"}, ti, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, nil, []interface{}{1, 1, "a"}, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, nil, []any{1, 1, "a"}, ti, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable, nil, nil, []interface{}{2, 2, "c"}, ti, nil, nil), + sqlmodel.NewRowChange(sourceTable, nil, nil, []any{2, 2, "c"}, ti, nil, nil), ecWithSafeMode, ), }, diff --git a/dm/syncer/data_validator.go b/dm/syncer/data_validator.go index aea64576e4..59548d3ff8 100644 --- a/dm/syncer/data_validator.go +++ b/dm/syncer/data_validator.go @@ -470,11 +470,9 @@ func (v *DataValidator) errorProcessRoutine() { if errors.Cause(err) != context.Canceled && !stopped { stopped = true - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { v.stopInner() - }() + }) } } wg.Wait() @@ -915,7 +913,7 @@ func (v *DataValidator) processRowsEvent(header *replication.EventHeader, ev *re } estimatedRowSize := int32(header.EventSize) / int32(len(ev.Rows)) for i := 0; i < len(ev.Rows); i += step { - var beforeImage, afterImage []interface{} + var beforeImage, afterImage []any switch changeType { case rowInsert: afterImage = ev.Rows[i] @@ -1043,7 +1041,7 @@ func (v *DataValidator) loadPersistedData() error { // aggregate using target table just as worker did. pendingChanges[validateTbl.targetTable.String()] = pendingTblChange for _, row := range tblChange.rows { - var beforeImage, afterImage []interface{} + var beforeImage, afterImage []any switch row.Tp { case rowInsert: afterImage = row.Data diff --git a/dm/syncer/data_validator_test.go b/dm/syncer/data_validator_test.go index 27dfc3e72d..a9a466406d 100644 --- a/dm/syncer/data_validator_test.go +++ b/dm/syncer/data_validator_test.go @@ -225,7 +225,7 @@ func TestValidatorDeadLock(t *testing.T) { // nolint:errcheck recover() }() - for i := 0; i < 100; i++ { + for range 100 { validator.sendError(context.Canceled) // prevent from stopping the validator } }() @@ -383,7 +383,7 @@ func TestValidatorProcessRowsEventAdjustUnsignedData(t *testing.T) { Table: []byte(tableName), }, ColumnCount: 2, - Rows: [][]interface{}{ + Rows: [][]any{ {int32(-2061521730), "v1"}, }, }, @@ -489,7 +489,7 @@ func TestValidatorDoValidate(t *testing.T) { Schema: schemaName, Table: tableName, ColumnType: []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, - Rows: [][]interface{}{ + Rows: [][]any{ {int32(1), "a"}, {int32(2), "b"}, {int32(3), "c"}, @@ -501,7 +501,7 @@ func TestValidatorDoValidate(t *testing.T) { Schema: schemaName, Table: tableName2, ColumnType: []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, - Rows: [][]interface{}{ + Rows: [][]any{ {int32(1), "a"}, {int32(2), "b"}, {int32(3), "c"}, @@ -513,7 +513,7 @@ func TestValidatorDoValidate(t *testing.T) { Schema: schemaName, Table: tableName3, ColumnType: []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, - Rows: [][]interface{}{ + Rows: [][]any{ {int32(1), "a"}, {int32(2), "b"}, {int32(3), "c"}, @@ -525,7 +525,7 @@ func TestValidatorDoValidate(t *testing.T) { Schema: schemaName, Table: tableName3, ColumnType: []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, - Rows: [][]interface{}{ + Rows: [][]any{ {int32(4), "a"}, }, }, @@ -536,7 +536,7 @@ func TestValidatorDoValidate(t *testing.T) { Schema: schemaName, Table: tableName, ColumnType: []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, - Rows: [][]interface{}{ + Rows: [][]any{ // update non-primary column {int32(3), "c"}, {int32(3), "d"}, @@ -552,7 +552,7 @@ func TestValidatorDoValidate(t *testing.T) { Schema: schemaName, Table: tableName, ColumnType: []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, - Rows: [][]interface{}{ + Rows: [][]any{ {int32(3), "c"}, }, }, @@ -562,7 +562,7 @@ func TestValidatorDoValidate(t *testing.T) { Schema: schemaName, Table: tableName4, ColumnType: []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, - Rows: [][]interface{}{ + Rows: [][]any{ {int32(4), "c"}, }, }, @@ -648,7 +648,7 @@ func TestValidatorGenRowKey(t *testing.T) { require.Equal(t, "a\tb", genRowKeyByString([]string{"a", "b"})) require.Equal(t, "a\tb\tc", genRowKeyByString([]string{"a", "b", "c"})) var bytes []byte - for i := 0; i < 100; i++ { + for range 100 { bytes = append(bytes, 'a') } { @@ -886,13 +886,11 @@ func TestValidatorErrorProcessRoutineDeadlock(t *testing.T) { finishedCh := make(chan struct{}) // simulate a worker - validator.wg.Add(1) - go func() { - defer validator.wg.Done() + validator.wg.Go(func() { validator.sendError(errors.New("test error 1")) validator.sendError(errors.New("test error 2")) validator.sendError(errors.New("test error 3")) - }() + }) validator.errProcessWg.Add(1) go func() { diff --git a/dm/syncer/dbconn/db.go b/dm/syncer/dbconn/db.go index 7b5f26b5ea..4477b567fe 100644 --- a/dm/syncer/dbconn/db.go +++ b/dm/syncer/dbconn/db.go @@ -88,7 +88,7 @@ func (conn *DBConn) QuerySQL( tctx *tcontext.Context, metricProxies *metrics.Proxies, query string, - args ...interface{}, + args ...any, ) (*sql.Rows, error) { if conn == nil || conn.baseConn == nil { return nil, terror.ErrDBUnExpect.Generate("database base connection not valid") @@ -104,7 +104,7 @@ func (conn *DBConn) QuerySQL( ret, _, err := conn.baseConn.ApplyRetryStrategy( tctx, params, - func(ctx *tcontext.Context) (interface{}, error) { + func(ctx *tcontext.Context) (any, error) { startTime := time.Now() ret, err := conn.baseConn.QuerySQL(ctx, query, args...) if err == nil { @@ -143,7 +143,7 @@ func (conn *DBConn) ExecuteSQLWithIgnore( metricProxies *metrics.Proxies, ignoreError func(error) bool, queries []string, - args ...[]interface{}, + args ...[]any, ) (int, error) { failpoint.Inject("ExecuteSQLWithIgnoreFailed", func(val failpoint.Value) { queryPattern := val.(string) @@ -173,7 +173,7 @@ func (conn *DBConn) ExecuteSQLWithIgnore( ret, numRetries, err := conn.baseConn.ApplyRetryStrategy( tctx, params, - func(ctx *tcontext.Context) (interface{}, error) { + func(ctx *tcontext.Context) (any, error) { startTime := time.Now() var histProxy *prometheus.HistogramVec if metricProxies != nil { @@ -218,7 +218,7 @@ func (conn *DBConn) ExecuteSQL( tctx *tcontext.Context, metricProxies *metrics.Proxies, queries []string, - args ...[]interface{}, + args ...[]any, ) (int, error) { return conn.ExecuteSQLWithIgnore(tctx, metricProxies, nil, queries, args...) } @@ -229,7 +229,7 @@ func (conn *DBConn) ExecuteSQLAutoSplit( tctx *tcontext.Context, metricProxies *metrics.Proxies, queries []string, - args ...[]interface{}, + args ...[]any, ) error { if conn == nil { // only happens in test @@ -261,7 +261,7 @@ func (conn *DBConn) retryableFn(tctx *tcontext.Context, queries, args any) func( _, resetRetries, resetErr := conn.baseConn.ApplyRetryStrategy( tctx, resetRetryParams, - func(ctx *tcontext.Context) (interface{}, error) { + func(ctx *tcontext.Context) (any, error) { err = conn.ResetConn(tctx) return nil, err }) @@ -330,7 +330,7 @@ func CreateConns(tctx *tcontext.Context, cfg *config.SubTaskConfig, dbCfg conn.S if err != nil { return nil, nil, err } - for i := 0; i < count; i++ { + for range count { baseConn, err := baseDB.GetBaseConn(tctx.Context()) if err != nil { CloseBaseDB(tctx, baseDB) diff --git a/dm/syncer/dml.go b/dm/syncer/dml.go index 8a20915796..0f3e9e9705 100644 --- a/dm/syncer/dml.go +++ b/dm/syncer/dml.go @@ -38,9 +38,9 @@ type genDMLParam struct { sourceTable *filter.Table // origin table targetTable *filter.Table safeMode bool // only used in update - originalData [][]interface{} // all data + originalData [][]any // all data sourceTableInfo *model.TableInfo // all table info - extendData [][]interface{} // all data include extend data + extendData [][]any // all data include extend data } // latin1Decider is not usually ISO8859_1 in MySQL. @@ -51,10 +51,10 @@ var latin1Decoder = charmap.Windows1252.NewDecoder() // - the values can be correctly converted to TiDB datum // - the values are in the correct type that go-sql-driver/mysql uses. func adjustValueFromBinlogData( - data []interface{}, + data []any, sourceTI *model.TableInfo, -) ([]interface{}, error) { - value := make([]interface{}, 0, len(data)) +) ([]any, error) { + value := make([]any, 0, len(data)) var err error columns := make([]*model.ColumnInfo, 0, len(sourceTI.Columns)) @@ -329,7 +329,7 @@ func (s *Syncer) causalityKeySourceTableNameForRowChange(sourceTable *filter.Tab } } -func castUnsigned(data interface{}, ft *types.FieldType) interface{} { +func castUnsigned(data any, ft *types.FieldType) any { if !mysql.HasUnsignedFlag(ft.GetFlag()) { return data } @@ -371,7 +371,7 @@ func checkLogColumns(skipped [][]int) error { } // genSQLMultipleRows generates multiple rows SQL with different dmlOpType. -func genSQLMultipleRows(op sqlmodel.DMLType, dmls []*sqlmodel.RowChange) (queries string, args []interface{}) { +func genSQLMultipleRows(op sqlmodel.DMLType, dmls []*sqlmodel.RowChange) (queries string, args []any) { if len(dmls) > 1 { log.L().Debug("generate DMLs with multiple rows", zap.Stringer("op", op), zap.Stringer("original op", dmls[0].Type()), zap.Int("rows", len(dmls))) } @@ -393,12 +393,12 @@ func genSQLMultipleRows(op sqlmodel.DMLType, dmls []*sqlmodel.RowChange) (querie // insert into tb(a,b,d) values(2,2,2) // we can only combine DMLs with same column names. // all dmls should have same dmlOpType and same tableName. -func genDMLsWithSameCols(op sqlmodel.DMLType, dmls []*sqlmodel.RowChange) ([]string, [][]interface{}) { +func genDMLsWithSameCols(op sqlmodel.DMLType, dmls []*sqlmodel.RowChange) ([]string, [][]any) { queries := make([]string, 0, len(dmls)) - args := make([][]interface{}, 0, len(dmls)) + args := make([][]any, 0, len(dmls)) var lastDML *sqlmodel.RowChange var query string - var arg []interface{} + var arg []any groupDMLs := make([]*sqlmodel.RowChange, 0, len(dmls)) // group dmls by same columns @@ -426,9 +426,9 @@ func genDMLsWithSameCols(op sqlmodel.DMLType, dmls []*sqlmodel.RowChange) ([]str // genDMLsWithSameTable groups and generates dmls with same table. // all the dmls should have same dmlOpType. -func genDMLsWithSameTable(op sqlmodel.DMLType, jobs []*job) ([]string, [][]interface{}) { +func genDMLsWithSameTable(op sqlmodel.DMLType, jobs []*job) ([]string, [][]any) { queries := make([]string, 0, len(jobs)) - args := make([][]interface{}, 0, len(jobs)) + args := make([][]any, 0, len(jobs)) var lastTable string groupDMLs := make([]*sqlmodel.RowChange, 0, len(jobs)) @@ -492,9 +492,9 @@ func genDMLsWithSameTable(op sqlmodel.DMLType, jobs []*job) ([]string, [][]inter // genDMLsWithSameOp groups and generates dmls by dmlOpType. // TODO: implement a volcano iterator interface for genDMLsWithSameXXX. -func genDMLsWithSameOp(jobs []*job) ([]string, [][]interface{}) { +func genDMLsWithSameOp(jobs []*job) ([]string, [][]any) { queries := make([]string, 0, len(jobs)) - args := make([][]interface{}, 0, len(jobs)) + args := make([][]any, 0, len(jobs)) var lastOp sqlmodel.DMLType jobsWithSameOp := make([]*job, 0, len(jobs)) diff --git a/dm/syncer/dml_test.go b/dm/syncer/dml_test.go index c15cc1899e..fea22a10f8 100644 --- a/dm/syncer/dml_test.go +++ b/dm/syncer/dml_test.go @@ -44,10 +44,10 @@ func TestCastUnsigned(t *testing.T) { // ref: https://dev.mysql.com/doc/refman/5.7/en/integer-types.html cases := []struct { - data interface{} + data any unsigned bool Type byte - expected interface{} + expected any }{ {int8(-math.Exp2(7)), false, mysql.TypeTiny, int8(-math.Exp2(7))}, // TINYINT {int8(-math.Exp2(7)), true, mysql.TypeTiny, uint8(math.Exp2(7))}, @@ -96,156 +96,156 @@ func TestGenDMLWithSameOp(t *testing.T) { dmls := []*job{ // insert newDMLJob( - sqlmodel.NewRowChange(sourceTable11, targetTable1, nil, []interface{}{1, 1, "a"}, tableInfo11, nil, nil), + sqlmodel.NewRowChange(sourceTable11, targetTable1, nil, []any{1, 1, "a"}, tableInfo11, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable11, targetTable1, nil, []interface{}{2, 2, "b"}, tableInfo11, nil, nil), + sqlmodel.NewRowChange(sourceTable11, targetTable1, nil, []any{2, 2, "b"}, tableInfo11, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable12, targetTable1, nil, []interface{}{3, 3, "c"}, tableInfo12, nil, nil), + sqlmodel.NewRowChange(sourceTable12, targetTable1, nil, []any{3, 3, "c"}, tableInfo12, nil, nil), ecWithSafeMode, ), // update no index but safemode newDMLJob( - sqlmodel.NewRowChange(sourceTable11, targetTable1, []interface{}{1, 1, "a"}, []interface{}{1, 1, "aa"}, tableInfo11, nil, nil), + sqlmodel.NewRowChange(sourceTable11, targetTable1, []any{1, 1, "a"}, []any{1, 1, "aa"}, tableInfo11, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable11, targetTable1, []interface{}{2, 2, "b"}, []interface{}{2, 2, "bb"}, tableInfo11, nil, nil), + sqlmodel.NewRowChange(sourceTable11, targetTable1, []any{2, 2, "b"}, []any{2, 2, "bb"}, tableInfo11, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable12, targetTable1, []interface{}{3, 3, "c"}, []interface{}{3, 3, "cc"}, tableInfo12, nil, nil), + sqlmodel.NewRowChange(sourceTable12, targetTable1, []any{3, 3, "c"}, []any{3, 3, "cc"}, tableInfo12, nil, nil), ecWithSafeMode, ), // update uk newDMLJob( - sqlmodel.NewRowChange(sourceTable11, targetTable1, []interface{}{1, 1, "aa"}, []interface{}{1, 4, "aa"}, tableInfo11, nil, nil), + sqlmodel.NewRowChange(sourceTable11, targetTable1, []any{1, 1, "aa"}, []any{1, 4, "aa"}, tableInfo11, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable11, targetTable1, []interface{}{2, 2, "bb"}, []interface{}{2, 5, "bb"}, tableInfo11, nil, nil), + sqlmodel.NewRowChange(sourceTable11, targetTable1, []any{2, 2, "bb"}, []any{2, 5, "bb"}, tableInfo11, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable12, targetTable1, []interface{}{3, 3, "cc"}, []interface{}{3, 6, "cc"}, tableInfo12, nil, nil), + sqlmodel.NewRowChange(sourceTable12, targetTable1, []any{3, 3, "cc"}, []any{3, 6, "cc"}, tableInfo12, nil, nil), ecWithSafeMode, ), // update pk newDMLJob( - sqlmodel.NewRowChange(sourceTable11, targetTable1, []interface{}{1, 4, "aa"}, []interface{}{4, 4, "aa"}, tableInfo11, nil, nil), + sqlmodel.NewRowChange(sourceTable11, targetTable1, []any{1, 4, "aa"}, []any{4, 4, "aa"}, tableInfo11, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable11, targetTable1, []interface{}{2, 5, "bb"}, []interface{}{5, 5, "bb"}, tableInfo11, nil, nil), + sqlmodel.NewRowChange(sourceTable11, targetTable1, []any{2, 5, "bb"}, []any{5, 5, "bb"}, tableInfo11, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable12, targetTable1, []interface{}{3, 6, "cc"}, []interface{}{6, 6, "cc"}, tableInfo12, nil, nil), + sqlmodel.NewRowChange(sourceTable12, targetTable1, []any{3, 6, "cc"}, []any{6, 6, "cc"}, tableInfo12, nil, nil), ecWithSafeMode, ), // delete newDMLJob( - sqlmodel.NewRowChange(sourceTable11, targetTable1, []interface{}{4, 4, "aa"}, nil, tableInfo11, nil, nil), + sqlmodel.NewRowChange(sourceTable11, targetTable1, []any{4, 4, "aa"}, nil, tableInfo11, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable11, targetTable1, []interface{}{5, 5, "bb"}, nil, tableInfo11, nil, nil), + sqlmodel.NewRowChange(sourceTable11, targetTable1, []any{5, 5, "bb"}, nil, tableInfo11, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable12, targetTable1, []interface{}{6, 6, "cc"}, nil, tableInfo12, nil, nil), + sqlmodel.NewRowChange(sourceTable12, targetTable1, []any{6, 6, "cc"}, nil, tableInfo12, nil, nil), ecWithSafeMode, ), // target table 2 // insert newDMLJob( - sqlmodel.NewRowChange(sourceTable21, targetTable2, nil, []interface{}{1, 1, "a"}, tableInfo21, nil, nil), + sqlmodel.NewRowChange(sourceTable21, targetTable2, nil, []any{1, 1, "a"}, tableInfo21, nil, nil), ecWithSafeMode, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable21, targetTable2, nil, []interface{}{2, 2, "b"}, tableInfo21, nil, nil), + sqlmodel.NewRowChange(sourceTable21, targetTable2, nil, []any{2, 2, "b"}, tableInfo21, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable22, targetTable2, nil, []interface{}{3, 3, "c"}, tableInfo22, nil, nil), + sqlmodel.NewRowChange(sourceTable22, targetTable2, nil, []any{3, 3, "c"}, tableInfo22, nil, nil), ec, ), // update no index newDMLJob( - sqlmodel.NewRowChange(sourceTable21, targetTable2, []interface{}{1, 1, "a"}, []interface{}{1, 1, "aa"}, tableInfo21, nil, nil), + sqlmodel.NewRowChange(sourceTable21, targetTable2, []any{1, 1, "a"}, []any{1, 1, "aa"}, tableInfo21, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable21, targetTable2, []interface{}{2, 2, "b"}, []interface{}{2, 2, "bb"}, tableInfo21, nil, nil), + sqlmodel.NewRowChange(sourceTable21, targetTable2, []any{2, 2, "b"}, []any{2, 2, "bb"}, tableInfo21, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable22, targetTable2, []interface{}{3, 3, "c"}, []interface{}{3, 3, "cc"}, tableInfo22, nil, nil), + sqlmodel.NewRowChange(sourceTable22, targetTable2, []any{3, 3, "c"}, []any{3, 3, "cc"}, tableInfo22, nil, nil), ec, ), // update uk newDMLJob( - sqlmodel.NewRowChange(sourceTable21, targetTable2, []interface{}{1, 1, "aa"}, []interface{}{1, 4, "aa"}, tableInfo21, nil, nil), + sqlmodel.NewRowChange(sourceTable21, targetTable2, []any{1, 1, "aa"}, []any{1, 4, "aa"}, tableInfo21, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable21, targetTable2, []interface{}{2, 2, "bb"}, []interface{}{2, 5, "bb"}, tableInfo21, nil, nil), + sqlmodel.NewRowChange(sourceTable21, targetTable2, []any{2, 2, "bb"}, []any{2, 5, "bb"}, tableInfo21, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable22, targetTable2, []interface{}{3, 3, "cc"}, []interface{}{3, 6, "cc"}, tableInfo22, nil, nil), + sqlmodel.NewRowChange(sourceTable22, targetTable2, []any{3, 3, "cc"}, []any{3, 6, "cc"}, tableInfo22, nil, nil), ec, ), // update pk newDMLJob( - sqlmodel.NewRowChange(sourceTable21, targetTable2, []interface{}{1, 4, "aa"}, []interface{}{4, 4, "aa"}, tableInfo21, nil, nil), + sqlmodel.NewRowChange(sourceTable21, targetTable2, []any{1, 4, "aa"}, []any{4, 4, "aa"}, tableInfo21, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable21, targetTable2, []interface{}{2, 5, "bb"}, []interface{}{5, 5, "bb"}, tableInfo21, nil, nil), + sqlmodel.NewRowChange(sourceTable21, targetTable2, []any{2, 5, "bb"}, []any{5, 5, "bb"}, tableInfo21, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable22, targetTable2, []interface{}{3, 6, "cc"}, []interface{}{6, 6, "cc"}, tableInfo22, nil, nil), + sqlmodel.NewRowChange(sourceTable22, targetTable2, []any{3, 6, "cc"}, []any{6, 6, "cc"}, tableInfo22, nil, nil), ec, ), // delete newDMLJob( - sqlmodel.NewRowChange(sourceTable21, targetTable2, []interface{}{4, 4, "aa"}, nil, tableInfo21, nil, nil), + sqlmodel.NewRowChange(sourceTable21, targetTable2, []any{4, 4, "aa"}, nil, tableInfo21, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable21, targetTable2, []interface{}{5, 5, "bb"}, nil, tableInfo21, nil, nil), + sqlmodel.NewRowChange(sourceTable21, targetTable2, []any{5, 5, "bb"}, nil, tableInfo21, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable22, targetTable2, []interface{}{6, 6, "cc"}, nil, tableInfo22, nil, nil), + sqlmodel.NewRowChange(sourceTable22, targetTable2, []any{6, 6, "cc"}, nil, tableInfo22, nil, nil), ec, ), // table1 // detele newDMLJob( - sqlmodel.NewRowChange(sourceTable11, targetTable1, []interface{}{44, 44, "aaa"}, nil, tableInfo11, nil, nil), + sqlmodel.NewRowChange(sourceTable11, targetTable1, []any{44, 44, "aaa"}, nil, tableInfo11, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable11, targetTable1, []interface{}{55, 55, "bbb"}, nil, tableInfo11, nil, nil), + sqlmodel.NewRowChange(sourceTable11, targetTable1, []any{55, 55, "bbb"}, nil, tableInfo11, nil, nil), ec, ), newDMLJob( - sqlmodel.NewRowChange(sourceTable12, targetTable1, []interface{}{66, 66, "ccc"}, nil, tableInfo12, nil, nil), + sqlmodel.NewRowChange(sourceTable12, targetTable1, []any{66, 66, "ccc"}, nil, tableInfo12, nil, nil), ec, ), } @@ -286,7 +286,7 @@ func TestGenDMLWithSameOp(t *testing.T) { "DELETE FROM `db1`.`tb1` WHERE (`id` = ?) OR (`id` = ?) OR (`id` = ?)", } - expectArgs := [][]interface{}{ + expectArgs := [][]any{ // table1 {1, 1, "a", 2, 2, "b", 3, 3, "c"}, {1, 1, "aa"}, @@ -336,8 +336,8 @@ func TestGBKExtractValueFromData(t *testing.T) { ti, err := createTableInfo(p, se, 0, table) require.NoError(t, err) - row := []interface{}{1, "\xc4\xe3\xba\xc3"} - expect := []interface{}{1, []byte("\xc4\xe3\xba\xc3")} + row := []any{1, "\xc4\xe3\xba\xc3"} + expect := []any{1, []byte("\xc4\xe3\xba\xc3")} got, err := adjustValueFromBinlogData(row, ti) require.NoError(t, err) require.Equal(t, expect, got) diff --git a/dm/syncer/dml_worker.go b/dm/syncer/dml_worker.go index 848d000459..0e770d029f 100644 --- a/dm/syncer/dml_worker.go +++ b/dm/syncer/dml_worker.go @@ -258,7 +258,7 @@ func (w *DMLWorker) executeBatchJobs(queueID int, jobs []*job, disableForeignKey var ( affect int queries []string - args [][]interface{} + args [][]any db = w.toDBConns[queueID] err error dmls = make([]*sqlmodel.RowChange, 0, len(jobs)) @@ -375,16 +375,16 @@ func (w *DMLWorker) setForeignKeyChecks(queueID int, enable bool) error { } // genSQLs generate SQLs in single row mode or multiple rows mode. -func (w *DMLWorker) genSQLs(jobs []*job) ([]string, [][]interface{}) { +func (w *DMLWorker) genSQLs(jobs []*job) ([]string, [][]any) { if w.multipleRows { return genDMLsWithSameOp(jobs) } queries := make([]string, 0, len(jobs)) - args := make([][]interface{}, 0, len(jobs)) + args := make([][]any, 0, len(jobs)) for _, j := range jobs { var query string - var arg []interface{} + var arg []any appendQueryAndArg := func() { queries = append(queries, query) args = append(args, arg) diff --git a/dm/syncer/dml_worker_test.go b/dm/syncer/dml_worker_test.go index dc3c94bab9..42131666b4 100644 --- a/dm/syncer/dml_worker_test.go +++ b/dm/syncer/dml_worker_test.go @@ -55,60 +55,60 @@ func TestGenSQL(t *testing.T) { createSQL := "create table db.tb(id int primary key, col1 int unique not null, col2 int unique, name varchar(24))" cases := []struct { - preValues []interface{} - postValues []interface{} + preValues []any + postValues []any safeMode bool expectedSQLs []string - expectedArgs [][]interface{} + expectedArgs [][]any }{ { nil, - []interface{}{1, 2, 3, "haha"}, + []any{1, 2, 3, "haha"}, false, []string{"INSERT INTO `targetSchema`.`targetTable` (`id`,`col1`,`col2`,`name`) VALUES (?,?,?,?)"}, - [][]interface{}{{1, 2, 3, "haha"}}, + [][]any{{1, 2, 3, "haha"}}, }, { nil, - []interface{}{1, 2, 3, "haha"}, + []any{1, 2, 3, "haha"}, true, []string{"REPLACE INTO `targetSchema`.`targetTable` (`id`,`col1`,`col2`,`name`) VALUES (?,?,?,?)"}, - [][]interface{}{{1, 2, 3, "haha"}}, + [][]any{{1, 2, 3, "haha"}}, }, { - []interface{}{1, 2, 3, "haha"}, + []any{1, 2, 3, "haha"}, nil, false, []string{"DELETE FROM `targetSchema`.`targetTable` WHERE `id` = ? LIMIT 1"}, - [][]interface{}{{1}}, + [][]any{{1}}, }, { - []interface{}{1, 2, 3, "haha"}, - []interface{}{4, 5, 6, "hihi"}, + []any{1, 2, 3, "haha"}, + []any{4, 5, 6, "hihi"}, false, []string{"UPDATE `targetSchema`.`targetTable` SET `id` = ?, `col1` = ?, `col2` = ?, `name` = ? WHERE `id` = ? LIMIT 1"}, - [][]interface{}{{4, 5, 6, "hihi", 1}}, + [][]any{{4, 5, 6, "hihi", 1}}, }, { - []interface{}{1, 2, 3, "haha"}, - []interface{}{1, 2, 3, "hihi"}, + []any{1, 2, 3, "haha"}, + []any{1, 2, 3, "hihi"}, true, []string{"REPLACE INTO `targetSchema`.`targetTable` (`id`,`col1`,`col2`,`name`) VALUES (?,?,?,?)"}, - [][]interface{}{{1, 2, 3, "hihi"}}, + [][]any{{1, 2, 3, "hihi"}}, }, { - []interface{}{1, 2, 3, "haha"}, - []interface{}{4, 5, 6, "hihi"}, + []any{1, 2, 3, "haha"}, + []any{4, 5, 6, "hihi"}, true, []string{"DELETE FROM `targetSchema`.`targetTable` WHERE `id` = ? LIMIT 1", "REPLACE INTO `targetSchema`.`targetTable` (`id`,`col1`,`col2`,`name`) VALUES (?,?,?,?)"}, - [][]interface{}{{1}, {4, 5, 6, "hihi"}}, + [][]any{{1}, {4, 5, 6, "hihi"}}, }, } @@ -155,18 +155,18 @@ func TestShouldDisableForeignKeyChecksForJob(t *testing.T) { target := &cdcmodel.TableName{Schema: "target", Table: "tb"} tableInfo := mockTableInfo(t, "create table db.tb(id int primary key, name varchar(10))") - insertChange := sqlmodel.NewRowChange(source, target, nil, []interface{}{1, "v"}, tableInfo, nil, nil) + insertChange := sqlmodel.NewRowChange(source, target, nil, []any{1, "v"}, tableInfo, nil, nil) insertJob := &job{tp: dml, safeMode: true, dml: insertChange} require.False(t, worker.shouldDisableForeignKeyChecksForJob(insertJob)) worker.foreignKeyChecksEnabled = true require.True(t, worker.shouldDisableForeignKeyChecksForJob(insertJob)) - updateChange := sqlmodel.NewRowChange(source, target, []interface{}{1, "a"}, []interface{}{1, "b"}, tableInfo, nil, nil) + updateChange := sqlmodel.NewRowChange(source, target, []any{1, "a"}, []any{1, "b"}, tableInfo, nil, nil) updateJob := &job{tp: dml, safeMode: true, dml: updateChange} require.True(t, worker.shouldDisableForeignKeyChecksForJob(updateJob)) - deleteChange := sqlmodel.NewRowChange(source, target, []interface{}{1, "a"}, nil, tableInfo, nil, nil) + deleteChange := sqlmodel.NewRowChange(source, target, []any{1, "a"}, nil, tableInfo, nil, nil) deleteJob := &job{tp: dml, safeMode: true, dml: deleteChange} require.False(t, worker.shouldDisableForeignKeyChecksForJob(deleteJob)) @@ -182,18 +182,18 @@ func TestShouldDisableForeignKeyChecks(t *testing.T) { target := &cdcmodel.TableName{Schema: "target", Table: "tbl"} tableInfo := mockTableInfo(t, "create table db.tb(id int primary key, col1 int unique not null, col2 int unique, name varchar(24))") - insertChange := sqlmodel.NewRowChange(source, target, nil, []interface{}{1, 2, 3, "v"}, tableInfo, nil, nil) + insertChange := sqlmodel.NewRowChange(source, target, nil, []any{1, 2, 3, "v"}, tableInfo, nil, nil) insertJob := newDMLJob(insertChange, ecWithSafeMode) require.True(t, worker.shouldDisableForeignKeyChecksForJob(insertJob)) insertJob.safeMode = false require.False(t, worker.shouldDisableForeignKeyChecksForJob(insertJob)) - updateChange := sqlmodel.NewRowChange(source, target, []interface{}{1, 2, 3, "v"}, []interface{}{1, 2, 3, "v2"}, tableInfo, nil, nil) + updateChange := sqlmodel.NewRowChange(source, target, []any{1, 2, 3, "v"}, []any{1, 2, 3, "v2"}, tableInfo, nil, nil) updateJob := newDMLJob(updateChange, ecWithSafeMode) require.True(t, worker.shouldDisableForeignKeyChecksForJob(updateJob)) - deleteChange := sqlmodel.NewRowChange(source, target, []interface{}{1, 2, 3, "v"}, nil, tableInfo, nil, nil) + deleteChange := sqlmodel.NewRowChange(source, target, []any{1, 2, 3, "v"}, nil, tableInfo, nil, nil) deleteJob := newDMLJob(deleteChange, ecWithSafeMode) require.False(t, worker.shouldDisableForeignKeyChecksForJob(deleteJob)) @@ -217,24 +217,24 @@ func TestValidateSafeModeForeignKeyUpdate(t *testing.T) { // PK change pkUpdate := sqlmodel.NewRowChange( source, target, - []interface{}{1, "a", "x"}, - []interface{}{2, "a", "x"}, + []any{1, "a", "x"}, + []any{2, "a", "x"}, tableInfo, nil, nil, ) // UK change ukUpdate := sqlmodel.NewRowChange( source, target, - []interface{}{1, "a", "x"}, - []interface{}{1, "b", "x"}, + []any{1, "a", "x"}, + []any{1, "b", "x"}, tableInfo, nil, nil, ) // Non-key change nonKeyUpdate := sqlmodel.NewRowChange( source, target, - []interface{}{1, "a", "x"}, - []interface{}{1, "a", "y"}, + []any{1, "a", "x"}, + []any{1, "a", "y"}, tableInfo, nil, nil, ) @@ -267,7 +267,7 @@ func TestExecuteBatchJobsWithForeignKey(t *testing.T) { t.Parallel() // helper: convert []interface{} -> []driver.Value - toDriverValues := func(args []interface{}) []driver.Value { + toDriverValues := func(args []any) []driver.Value { out := make([]driver.Value, len(args)) for i, v := range args { out[i] = v @@ -302,8 +302,8 @@ func TestExecuteBatchJobsWithForeignKey(t *testing.T) { createSQL := "create table db.tb(id int primary key, col1 int unique not null, col2 int unique, name varchar(24))" tableInfo := mockTableInfo(t, createSQL) - insertChange := sqlmodel.NewRowChange(source, target, nil, []interface{}{1, 2, 3, "normal"}, tableInfo, nil, nil) - replaceChange := sqlmodel.NewRowChange(source, target, nil, []interface{}{2, 3, 4, "safe"}, tableInfo, nil, nil) + insertChange := sqlmodel.NewRowChange(source, target, nil, []any{1, 2, 3, "normal"}, tableInfo, nil, nil) + replaceChange := sqlmodel.NewRowChange(source, target, nil, []any{2, 3, 4, "safe"}, tableInfo, nil, nil) insertJob := newDMLJob(insertChange, ec) replaceJob := newDMLJob(replaceChange, ecWithSafeMode) diff --git a/dm/syncer/expr_filter_group.go b/dm/syncer/expr_filter_group.go index 8335b2c873..4e418f1f6e 100644 --- a/dm/syncer/expr_filter_group.go +++ b/dm/syncer/expr_filter_group.go @@ -179,7 +179,7 @@ func (g *ExprFilterGroup) ResetExprs(table *filter.Table) { } // SkipDMLByExpression returns true when given row matches the expr, which means this row should be skipped. -func SkipDMLByExpression(ctx sessionctx.Context, row []interface{}, expr expression.Expression, upstreamCols []*model.ColumnInfo) (bool, error) { +func SkipDMLByExpression(ctx sessionctx.Context, row []any, expr expression.Expression, upstreamCols []*model.ColumnInfo) (bool, error) { // TODO: add MetricsProxies data, err := utils.AdjustBinaryProtocolForDatum(ctx, row, upstreamCols) if err != nil { diff --git a/dm/syncer/expr_filter_group_test.go b/dm/syncer/expr_filter_group_test.go index 459dc73f75..dbf13f4303 100644 --- a/dm/syncer/expr_filter_group_test.go +++ b/dm/syncer/expr_filter_group_test.go @@ -36,8 +36,8 @@ func TestSkipDMLByExpression(t *testing.T) { cases := []struct { exprStr string tableStr string - skippedRow []interface{} - passedRow []interface{} + skippedRow []any + passedRow []any }{ { "state != 1", @@ -50,8 +50,8 @@ create table t ( UNIQUE KEY uniq_id (id), KEY idx_state (state) );`, - []interface{}{100, 100, 3}, - []interface{}{100, 100, 1}, + []any{100, 100, 3}, + []any{100, 100, 1}, }, { "f > 1.23", @@ -59,8 +59,8 @@ create table t ( create table t ( f float );`, - []interface{}{float32(2.0)}, - []interface{}{float32(1.0)}, + []any{float32(2.0)}, + []any{float32(1.0)}, }, { "f > a + b", @@ -70,8 +70,8 @@ create table t ( a int, b int );`, - []interface{}{float32(123.45), 1, 2}, - []interface{}{float32(0.01), 23, 45}, + []any{float32(123.45), 1, 2}, + []any{float32(0.01), 23, 45}, }, { "id = 30", @@ -83,8 +83,8 @@ create table t ( ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (id) );`, - []interface{}{int32(30), "30", nil, "2021-06-17 10:13:05"}, - []interface{}{int32(20), "20", nil, "2021-06-17 10:13:05"}, + []any{int32(30), "30", nil, "2021-06-17 10:13:05"}, + []any{int32(20), "20", nil, "2021-06-17 10:13:05"}, }, } @@ -143,8 +143,8 @@ func TestAllBinaryProtocolTypes(t *testing.T) { cases := []struct { exprStr string tableStr string - skippedRow []interface{} - passedRow []interface{} + skippedRow []any + passedRow []any }{ // MYSQL_TYPE_NULL { @@ -153,8 +153,8 @@ func TestAllBinaryProtocolTypes(t *testing.T) { create table t ( c int );`, - []interface{}{nil}, - []interface{}{100}, + []any{nil}, + []any{100}, }, // MYSQL_TYPE_LONG { @@ -163,8 +163,8 @@ create table t ( create table t ( c int );`, - []interface{}{int32(1)}, - []interface{}{int32(100)}, + []any{int32(1)}, + []any{int32(100)}, }, // MYSQL_TYPE_TINY { @@ -173,8 +173,8 @@ create table t ( create table t ( c tinyint );`, - []interface{}{int8(2)}, - []interface{}{int8(-1)}, + []any{int8(2)}, + []any{int8(-1)}, }, // MYSQL_TYPE_SHORT { @@ -183,8 +183,8 @@ create table t ( create table t ( c smallint );`, - []interface{}{int16(8)}, - []interface{}{int16(18)}, + []any{int16(8)}, + []any{int16(18)}, }, // MYSQL_TYPE_INT24 { @@ -193,8 +193,8 @@ create table t ( create table t ( c mediumint );`, - []interface{}{int32(-8)}, - []interface{}{int32(1)}, + []any{int32(-8)}, + []any{int32(1)}, }, // MYSQL_TYPE_LONGLONG { @@ -203,8 +203,8 @@ create table t ( create table t ( c bigint );`, - []interface{}{int64(100000000)}, - []interface{}{int64(200000000)}, + []any{int64(100000000)}, + []any{int64(200000000)}, }, // MYSQL_TYPE_NEWDECIMAL { @@ -213,8 +213,8 @@ create table t ( create table t ( c decimal(5,2) );`, - []interface{}{"10.10"}, - []interface{}{"10.11"}, + []any{"10.10"}, + []any{"10.11"}, }, // MYSQL_TYPE_FLOAT { @@ -223,8 +223,8 @@ create table t ( create table t ( c float );`, - []interface{}{float32(0.08)}, - []interface{}{float32(0.18)}, + []any{float32(0.08)}, + []any{float32(0.18)}, }, // MYSQL_TYPE_DOUBLE { @@ -233,8 +233,8 @@ create table t ( create table t ( c double );`, - []interface{}{float64(0.08)}, - []interface{}{float64(0.18)}, + []any{float64(0.08)}, + []any{float64(0.18)}, }, // MYSQL_TYPE_BIT { @@ -243,8 +243,8 @@ create table t ( create table t ( c bit(4) );`, - []interface{}{int64(1)}, - []interface{}{int64(2)}, + []any{int64(1)}, + []any{int64(2)}, }, // MYSQL_TYPE_TIMESTAMP, MYSQL_TYPE_TIMESTAMP2 // DM does not set ParseTime @@ -255,8 +255,8 @@ create table t ( create table t ( c timestamp );`, - []interface{}{"2021-06-21 12:34:56"}, - []interface{}{"1970-01-01 00:00:01"}, + []any{"2021-06-21 12:34:56"}, + []any{"1970-01-01 00:00:01"}, }, // MYSQL_TYPE_DATETIME, MYSQL_TYPE_DATETIME2 { @@ -265,8 +265,8 @@ create table t ( create table t ( c datetime );`, - []interface{}{"2021-06-21 00:00:12"}, - []interface{}{"1970-01-01 00:00:01"}, + []any{"2021-06-21 00:00:12"}, + []any{"1970-01-01 00:00:01"}, }, // MYSQL_TYPE_TIME, MYSQL_TYPE_TIME2 { @@ -275,8 +275,8 @@ create table t ( create table t ( c time(6) );`, - []interface{}{"00:00:12"}, - []interface{}{"00:00:01"}, + []any{"00:00:12"}, + []any{"00:00:01"}, }, // MYSQL_TYPE_DATE { @@ -285,8 +285,8 @@ create table t ( create table t ( c date );`, - []interface{}{"2021-06-21"}, - []interface{}{"1970-01-01"}, + []any{"2021-06-21"}, + []any{"1970-01-01"}, }, // MYSQL_TYPE_YEAR { @@ -295,8 +295,8 @@ create table t ( create table t ( c year );`, - []interface{}{int(2021)}, - []interface{}{int(2020)}, + []any{int(2021)}, + []any{int(2020)}, }, // MYSQL_TYPE_ENUM { @@ -305,8 +305,8 @@ create table t ( create table t ( c ENUM('x-small', 'small', 'medium', 'large', 'x-large') );`, - []interface{}{int64(1)}, // 1-indexed - []interface{}{int64(2)}, + []any{int64(1)}, // 1-indexed + []any{int64(2)}, }, // MYSQL_TYPE_SET { @@ -315,8 +315,8 @@ create table t ( create table t ( c SET('a', 'b', 'c', 'd') );`, - []interface{}{int64(0b1100)}, // c,d - []interface{}{int64(0b1000)}, // d + []any{int64(0b1100)}, // c,d + []any{int64(0b1000)}, // d }, // MYSQL_TYPE_BLOB { @@ -325,8 +325,8 @@ create table t ( create table t ( c blob );`, - []interface{}{[]byte("\x124")}, // x'1234' - []interface{}{[]byte("Vx")}, // x'5678' + []any{[]byte("\x124")}, // x'1234' + []any{[]byte("Vx")}, // x'5678' }, // MYSQL_TYPE_VARCHAR, MYSQL_TYPE_VAR_STRING, MYSQL_TYPE_STRING { @@ -335,8 +335,8 @@ create table t ( create table t ( c varchar(20) );`, - []interface{}{"abc"}, - []interface{}{"def"}, + []any{"abc"}, + []any{"def"}, }, // MYSQL_TYPE_JSON { @@ -345,8 +345,8 @@ create table t ( create table t ( c json );`, - []interface{}{[]byte(`{"id": 1}`)}, - []interface{}{[]byte(`{"id": 2}`)}, + []any{[]byte(`{"id": 1}`)}, + []any{[]byte(`{"id": 2}`)}, }, // MYSQL_TYPE_GEOMETRY, parser not supported } @@ -445,10 +445,10 @@ create table t ( require.Equal(t, "0", expr.StringWithCtx(context2.EmptyParamValues, errors.RedactLogDisable)) // skip nothing - skip, err := SkipDMLByExpression(sessCtx, []interface{}{0}, expr, ti.Columns) + skip, err := SkipDMLByExpression(sessCtx, []any{0}, expr, ti.Columns) require.NoError(t, err) require.False(t, skip) - skip, err = SkipDMLByExpression(sessCtx, []interface{}{2}, expr, ti.Columns) + skip, err = SkipDMLByExpression(sessCtx, []any{2}, expr, ti.Columns) require.NoError(t, err) require.False(t, skip) } diff --git a/dm/syncer/job_test.go b/dm/syncer/job_test.go index 03519901e6..8839267fe8 100644 --- a/dm/syncer/job_test.go +++ b/dm/syncer/job_test.go @@ -84,7 +84,7 @@ func TestJob(t *testing.T) { schema := "create table test.tb(id int primary key, col1 int unique not null)" ti := mockTableInfo(t, schema) - exampleChange := sqlmodel.NewRowChange(table, nil, nil, []interface{}{2, 2}, ti, nil, nil) + exampleChange := sqlmodel.NewRowChange(table, nil, nil, []any{2, 2}, ti, nil, nil) testCases := []struct { job *job diff --git a/dm/syncer/online-ddl-tools/online_ddl.go b/dm/syncer/online-ddl-tools/online_ddl.go index 8902e0c1ba..3690b9e93d 100644 --- a/dm/syncer/online-ddl-tools/online_ddl.go +++ b/dm/syncer/online-ddl-tools/online_ddl.go @@ -254,7 +254,7 @@ func (s *Storage) saveToDB(tctx *tcontext.Context, ghostSchema, ghostTable strin } query := fmt.Sprintf("REPLACE INTO %s(`id`,`ghost_schema`, `ghost_table`, `ddls`) VALUES (?, ?, ?, ?)", s.tableName) - _, err = s.dbConn.ExecuteSQL(tctx, s.metricProxies, []string{query}, []interface{}{s.id, ghostSchema, ghostTable, string(ddlsBytes)}) + _, err = s.dbConn.ExecuteSQL(tctx, s.metricProxies, []string{query}, []any{s.id, ghostSchema, ghostTable, string(ddlsBytes)}) failpoint.Inject("ExitAfterSaveOnlineDDL", func() { tctx.L().Info("failpoint ExitAfterSaveOnlineDDL") panic("ExitAfterSaveOnlineDDL") @@ -277,7 +277,7 @@ func (s *Storage) delete(tctx *tcontext.Context, ghostSchema, ghostTable string) // delete all checkpoints sql := fmt.Sprintf("DELETE FROM %s WHERE `id` = ? and `ghost_schema` = ? and `ghost_table` = ?", s.tableName) - _, err := s.dbConn.ExecuteSQL(tctx, s.metricProxies, []string{sql}, []interface{}{s.id, ghostSchema, ghostTable}) + _, err := s.dbConn.ExecuteSQL(tctx, s.metricProxies, []string{sql}, []any{s.id, ghostSchema, ghostTable}) if err != nil { return terror.WithScope(err, terror.ScopeDownstream) } @@ -293,7 +293,7 @@ func (s *Storage) Clear(tctx *tcontext.Context) error { // delete all checkpoints sql := fmt.Sprintf("DELETE FROM %s WHERE `id` = ?", s.tableName) - _, err := s.dbConn.ExecuteSQL(tctx, s.metricProxies, []string{sql}, []interface{}{s.id}) + _, err := s.dbConn.ExecuteSQL(tctx, s.metricProxies, []string{sql}, []any{s.id}) if err != nil { return terror.WithScope(err, terror.ScopeDownstream) } diff --git a/dm/syncer/shardddl/optimist.go b/dm/syncer/shardddl/optimist.go index 94707693a7..01ea2acbf5 100644 --- a/dm/syncer/shardddl/optimist.go +++ b/dm/syncer/shardddl/optimist.go @@ -217,7 +217,7 @@ func (o *Optimist) GetRedirectOperation(ctx context.Context, info optimism.Info, // DoneOperation marks the shard DDL lock operation as done. func (o *Optimist) DoneOperation(op optimism.Operation) error { op.Done = true - _, _, err := etcdutil.DoTxnWithRepeatable(o.cli, func(_ *tcontext.Context, cli *clientv3.Client) (interface{}, error) { + _, _, err := etcdutil.DoTxnWithRepeatable(o.cli, func(_ *tcontext.Context, cli *clientv3.Client) (any, error) { _, _, err := optimism.PutOperation(cli, false, op, 0) return nil, err }) diff --git a/dm/syncer/shardddl/pessimist.go b/dm/syncer/shardddl/pessimist.go index 2011fdd3b4..a2dc528b51 100644 --- a/dm/syncer/shardddl/pessimist.go +++ b/dm/syncer/shardddl/pessimist.go @@ -134,7 +134,7 @@ func (p *Pessimist) GetOperation(ctx context.Context, info pessimism.Info, rev i func (p *Pessimist) DoneOperationDeleteInfo(op pessimism.Operation, info pessimism.Info) error { op.Done = true // mark the operation as `done`. - _, _, err := etcdutil.DoTxnWithRepeatable(p.cli, func(_ *tcontext.Context, cli *clientv3.Client) (interface{}, error) { + _, _, err := etcdutil.DoTxnWithRepeatable(p.cli, func(_ *tcontext.Context, cli *clientv3.Client) (any, error) { done, _, err := pessimism.PutOperationDeleteExistInfo(cli, op, info) if err != nil { return nil, err diff --git a/dm/syncer/sharding-meta/shardmeta.go b/dm/syncer/sharding-meta/shardmeta.go index 1b4246ef21..3a23839d8b 100644 --- a/dm/syncer/sharding-meta/shardmeta.go +++ b/dm/syncer/sharding-meta/shardmeta.go @@ -257,34 +257,34 @@ func (meta *ShardingMeta) ActiveDDLFirstLocation() (binlog.Location, error) { } // FlushData returns sharding meta flush SQL and args. -func (meta *ShardingMeta) FlushData(sourceID, tableID string) ([]string, [][]interface{}) { +func (meta *ShardingMeta) FlushData(sourceID, tableID string) ([]string, [][]any) { if len(meta.global.Items) == 0 { sql2 := fmt.Sprintf("DELETE FROM %s where source_id=? and target_table_id=?", meta.tableName) - args2 := []interface{}{sourceID, tableID} - return []string{sql2}, [][]interface{}{args2} + args2 := []any{sourceID, tableID} + return []string{sql2}, [][]any{args2} } var ( sqls = make([]string, 1+len(meta.sources)) - args = make([][]interface{}, 0, 1+len(meta.sources)) + args = make([][]any, 0, 1+len(meta.sources)) baseSQL = fmt.Sprintf("INSERT INTO %s (`source_id`, `target_table_id`, `source_table_id`, `active_index`, `is_global`, `data`) VALUES(?,?,?,?,?,?) ON DUPLICATE KEY UPDATE `data`=?, `active_index`=?", meta.tableName) ) for i := range sqls { sqls[i] = baseSQL } - args = append(args, []interface{}{sourceID, tableID, "", meta.activeIdx, true, meta.global.String(), meta.global.String(), meta.activeIdx}) + args = append(args, []any{sourceID, tableID, "", meta.activeIdx, true, meta.global.String(), meta.global.String(), meta.activeIdx}) for source, seq := range meta.sources { - args = append(args, []interface{}{sourceID, tableID, source, meta.activeIdx, false, seq.String(), seq.String(), meta.activeIdx}) + args = append(args, []any{sourceID, tableID, source, meta.activeIdx, false, seq.String(), seq.String(), meta.activeIdx}) } return sqls, args } -func (meta *ShardingMeta) genRemoveSQL(sourceID, tableID string) (string, []interface{}) { +func (meta *ShardingMeta) genRemoveSQL(sourceID, tableID string) (string, []any) { sql := fmt.Sprintf("DELETE FROM %s where source_id=? and target_table_id=?", meta.tableName) - return sql, []interface{}{sourceID, tableID} + return sql, []any{sourceID, tableID} } // CheckAndUpdate check and fix schema and table names for all the sharding groups. -func (meta *ShardingMeta) CheckAndUpdate(logger log.Logger, targetID string, schemaMap map[string]string, tablesMap map[string]map[string]string) ([]string, [][]interface{}, error) { +func (meta *ShardingMeta) CheckAndUpdate(logger log.Logger, targetID string, schemaMap map[string]string, tablesMap map[string]map[string]string) ([]string, [][]any, error) { if len(schemaMap) == 0 && len(tablesMap) == 0 { return nil, nil, nil } @@ -330,7 +330,7 @@ func (meta *ShardingMeta) CheckAndUpdate(logger log.Logger, targetID string, sch } var ( sqls []string - args [][]interface{} + args [][]any ) for oldID, newID := range sourceIDsMap { if oldID != newID { diff --git a/dm/syncer/sharding-meta/shardmeta_test.go b/dm/syncer/sharding-meta/shardmeta_test.go index a662e904d7..c9018a2f1d 100644 --- a/dm/syncer/sharding-meta/shardmeta_test.go +++ b/dm/syncer/sharding-meta/shardmeta_test.go @@ -35,7 +35,7 @@ func (t *testShardMetaSuite) TestShardingMeta(c *check.C) { active bool err error sqls []string - args [][]interface{} + args [][]any location binlog.Location filename = "mysql-bin.000001" table1 = "table1" @@ -60,7 +60,7 @@ func (t *testShardMetaSuite) TestShardingMeta(c *check.C) { ) // 1st round sharding DDL sync - for i := 0; i < 7; i++ { + for i := range 7 { active, err = meta.AddItem(items[i]) c.Assert(err, check.IsNil) if i%3 == 0 { @@ -104,7 +104,7 @@ func (t *testShardMetaSuite) TestShardingMeta(c *check.C) { } // 2nd round sharding DDL sync - for i := 0; i < 8; i++ { + for i := range 8 { if i%3 == 0 { continue } @@ -150,7 +150,7 @@ func (t *testShardMetaSuite) TestShardingMeta(c *check.C) { } // 3rd round sharding DDL sync - for i := 0; i < 9; i++ { + for i := range 9 { if i%3 != 2 { continue } @@ -186,7 +186,7 @@ func (t *testShardMetaSuite) TestShardingMeta(c *check.C) { c.Assert(sqls, check.HasLen, 1) c.Assert(args, check.HasLen, 1) c.Assert(sqls[0], check.Matches, "DELETE FROM .*") - c.Assert(args[0], check.DeepEquals, []interface{}{sourceID, tableID}) + c.Assert(args[0], check.DeepEquals, []any{sourceID, tableID}) } func (t *testShardMetaSuite) TestShardingMetaWrongSequence(c *check.C) { @@ -208,7 +208,7 @@ func (t *testShardMetaSuite) TestShardingMetaWrongSequence(c *check.C) { ) // 1st round sharding DDL sync - for i := 0; i < 4; i++ { + for i := range 4 { active, err = meta.AddItem(items[i]) c.Assert(err, check.IsNil) if i%3 == 0 { @@ -221,7 +221,7 @@ func (t *testShardMetaSuite) TestShardingMetaWrongSequence(c *check.C) { c.Assert(meta.ResolveShardingDDL(), check.IsFalse) // 2nd round sharding DDL sync - for i := 0; i < 4; i++ { + for i := range 4 { if i%3 == 0 { continue } diff --git a/dm/syncer/sharding_group.go b/dm/syncer/sharding_group.go index 9151fca609..d1e3d794ad 100644 --- a/dm/syncer/sharding_group.go +++ b/dm/syncer/sharding_group.go @@ -72,6 +72,7 @@ package syncer import ( "fmt" + "maps" "sync" "github.com/pingcap/tidb/pkg/util/dbutil" @@ -279,9 +280,7 @@ func (sg *ShardingGroup) Sources() map[string]bool { sg.RLock() defer sg.RUnlock() ret := make(map[string]bool, len(sg.sources)) - for k, v := range sg.sources { - ret[k] = v - } + maps.Copy(ret, sg.sources) return ret } @@ -366,7 +365,7 @@ func (sg *ShardingGroup) ActiveDDLFirstLocation() (binlog.Location, error) { } // FlushData returns sharding meta flush SQLs and args. -func (sg *ShardingGroup) FlushData(targetTableID string) ([]string, [][]interface{}) { +func (sg *ShardingGroup) FlushData(targetTableID string) ([]string, [][]any) { sg.RLock() defer sg.RUnlock() return sg.meta.FlushData(sg.sourceID, targetTableID) @@ -619,9 +618,7 @@ func (k *ShardingGroupKeeper) Groups() map[string]*ShardingGroup { // do a copy groups := make(map[string]*ShardingGroup, len(k.groups)) - for key, value := range k.groups { - groups[key] = value - } + maps.Copy(groups, k.groups) return groups } @@ -662,12 +659,12 @@ func (k *ShardingGroupKeeper) ActiveDDLFirstLocation(targetTable *filter.Table) } // PrepareFlushSQLs returns all sharding meta flushed SQLs except for given table IDs. -func (k *ShardingGroupKeeper) PrepareFlushSQLs(exceptTableIDs map[string]bool) ([]string, [][]interface{}) { +func (k *ShardingGroupKeeper) PrepareFlushSQLs(exceptTableIDs map[string]bool) ([]string, [][]any) { k.RLock() defer k.RUnlock() var ( sqls = make([]string, 0, len(k.groups)) - args = make([][]interface{}, 0, len(k.groups)) + args = make([][]any, 0, len(k.groups)) ) for id, group := range k.groups { if group.IsSchemaOnly { diff --git a/dm/syncer/status.go b/dm/syncer/status.go index e2687ab7e4..02729372ca 100644 --- a/dm/syncer/status.go +++ b/dm/syncer/status.go @@ -27,7 +27,7 @@ import ( ) // Status implements Unit.Status. -func (s *Syncer) Status(sourceStatus *binlog.SourceStatus) interface{} { +func (s *Syncer) Status(sourceStatus *binlog.SourceStatus) any { syncerLocation := s.checkpoint.FlushedGlobalPoint() st := &pb.SyncStatus{ TotalEvents: s.count.Load(), diff --git a/dm/syncer/status_test.go b/dm/syncer/status_test.go index c346654be3..b0553e8821 100644 --- a/dm/syncer/status_test.go +++ b/dm/syncer/status_test.go @@ -56,15 +56,13 @@ func (t *statusSuite) TestStatusRace(c *check.C) { } wg := sync.WaitGroup{} - for i := 0; i < 10; i++ { - wg.Add(1) - go func() { - defer wg.Done() + for range 10 { + wg.Go(func() { ret := s.Status(sourceStatus) status := ret.(*pb.SyncStatus) c.Assert(status.MasterBinlog, check.Equals, "(mysql-bin.000123, 223)") c.Assert(status.SyncerBinlog, check.Equals, "(mysql-bin.000123, 123)") - }() + }) } wg.Wait() } diff --git a/dm/syncer/syncer.go b/dm/syncer/syncer.go index 1013bf2a39..f35d6f2ab3 100644 --- a/dm/syncer/syncer.go +++ b/dm/syncer/syncer.go @@ -361,7 +361,7 @@ func (s *Syncer) closeJobChans() { s.jobsClosed.Store(true) } -func (s *Syncer) recordUnhandledEvent(message string, ev interface{}) { +func (s *Syncer) recordUnhandledEvent(message string, ev any) { s.unhandledEventLogger.Warn(message, zap.String("type", fmt.Sprintf("%T", ev))) } @@ -758,9 +758,7 @@ func (s *Syncer) Process(ctx context.Context, pr chan pb.ProcessResult) { ) var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { for { err, ok := <-runFatalChan if !ok { @@ -771,13 +769,11 @@ func (s *Syncer) Process(ctx context.Context, pr chan pb.ProcessResult) { errs = append(errs, err) errsMu.Unlock() } - }() + }) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { <-newCtx.Done() // ctx or newCtx - }() + }) err := s.Run(newCtx) if err != nil { @@ -1362,7 +1358,7 @@ func (s *Syncer) flushCheckPointsAsync(asyncFlushJob *job) { s.checkpointFlushWorker.Add(task) } -func (s *Syncer) createCheckpointSnapshot(isSyncFlush bool) (*SnapshotInfo, []*filter.Table, []string, [][]interface{}) { +func (s *Syncer) createCheckpointSnapshot(isSyncFlush bool) (*SnapshotInfo, []*filter.Table, []string, [][]any) { snapshotInfo := s.checkpoint.Snapshot(isSyncFlush) if snapshotInfo == nil { return nil, nil, nil, nil @@ -1372,7 +1368,7 @@ func (s *Syncer) createCheckpointSnapshot(isSyncFlush bool) (*SnapshotInfo, []*f exceptTableIDs map[string]bool exceptTables []*filter.Table shardMetaSQLs []string - shardMetaArgs [][]interface{} + shardMetaArgs [][]any ) if s.cfg.ShardMode == config.ShardPessimistic { // flush all checkpoints except tables which are unresolved for sharding DDL for the pessimistic mode. @@ -1836,12 +1832,10 @@ func (s *Syncer) Run(ctx context.Context) (err error) { s.runWg.Add(1) go s.syncDML() - s.runWg.Add(1) - go func() { - defer s.runWg.Done() + s.runWg.Go(func() { // also need to use a different ctx. checkpointFlushWorker worker will be closed in the first defer s.checkpointFlushWorker.Run(s.tctx) - }() + }) s.runWg.Add(1) go s.syncDDL(adminQueueName, s.ddlDBConn, s.ddlJobCh) s.runWg.Add(1) @@ -2805,13 +2799,13 @@ func (qec *queryEventContext) String() string { } // generateExtendColumn generate extended columns by extractor. -func generateExtendColumn(data [][]interface{}, r *regexprrouter.RouteTable, table *filter.Table, sourceID string) [][]interface{} { +func generateExtendColumn(data [][]any, r *regexprrouter.RouteTable, table *filter.Table, sourceID string) [][]any { extendCol, extendVal := r.FetchExtendColumn(table.Schema, table.Name, sourceID) if len(extendCol) == 0 { return nil } - rows := make([][]interface{}, len(data)) + rows := make([][]any, len(data)) for i := range data { rows[i] = data[i] for _, v := range extendVal { @@ -2913,10 +2907,7 @@ func (s *Syncer) trackDDL(usedSchema string, trackInfo *ddlInfo, ec *eventContex } } // skip getTable before in above loop - start := 1 - if shouldTableExistNum > start { - start = shouldTableExistNum - } + start := max(shouldTableExistNum, 1) for i := start; i < shouldRefTableExistNum; i++ { if err := s.schemaTracker.CreateSchemaIfNotExists(srcTables[i].Schema); err != nil { return terror.ErrSchemaTrackerCannotCreateSchema.Delegate(err, srcTables[i].Schema) @@ -3100,8 +3091,8 @@ func (s *Syncer) loadTableStructureFromDump(ctx context.Context) error { setFirstErr(err2) continue } - stmts := bytes.Split(content, []byte(";\n")) - for _, stmt := range stmts { + stmts := bytes.SplitSeq(content, []byte(";\n")) + for stmt := range stmts { stmt = bytes.TrimSpace(stmt) if len(stmt) == 0 || bytes.HasPrefix(stmt, []byte("/*")) { continue diff --git a/dm/syncer/syncer_test.go b/dm/syncer/syncer_test.go index 1366cd9bc2..7864769c35 100644 --- a/dm/syncer/syncer_test.go +++ b/dm/syncer/syncer_test.go @@ -76,7 +76,7 @@ type ( mockBinlogEvents []mockBinlogEvent mockBinlogEvent struct { typ int - args []interface{} + args []any } ) @@ -180,7 +180,7 @@ func (s *testSyncerSuite) generateEvents(binlogEvents mockBinlogEvents, c *check Schema: e.args[1].(string), Table: e.args[2].(string), ColumnType: e.args[3].([]byte), - Rows: e.args[4].([][]interface{}), + Rows: e.args[4].([][]any), }, } var eventType replication.EventType @@ -237,7 +237,7 @@ func (s *testSyncerSuite) TestSampleUnhandledEvents(c *check.C) { entries := logs.All() c.Assert(entries, check.HasLen, 2) - seen := make(map[string][]map[string]interface{}, len(entries)) + seen := make(map[string][]map[string]any, len(entries)) for _, entry := range entries { seen[entry.Message] = append(seen[entry.Message], entry.ContextMap()) } @@ -337,33 +337,33 @@ func (s *testSyncerSuite) TestSelectTable(c *check.C) { } s.resetEventsGenerator(c) events := mockBinlogEvents{ - mockBinlogEvent{typ: DBCreate, args: []interface{}{"s1"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"s1", "create table s1.log(id int)"}}, - mockBinlogEvent{typ: DBDrop, args: []interface{}{"s1"}}, - - mockBinlogEvent{typ: TableCreate, args: []interface{}{"mysql", "create table mysql.test(id int)"}}, - mockBinlogEvent{typ: TableDrop, args: []interface{}{"mysql", "test"}}, - mockBinlogEvent{typ: DBCreate, args: []interface{}{"stest"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"stest", "create table stest.log(id int)"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"stest", "create table stest.t(id int)"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"stest", "create table stest.log2(id int)"}}, - mockBinlogEvent{typ: Write, args: []interface{}{uint64(8), "stest", "t", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(10)}}}}, - mockBinlogEvent{typ: Write, args: []interface{}{uint64(9), "stest", "log", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(10)}}}}, - mockBinlogEvent{typ: Write, args: []interface{}{uint64(10), "stest", "log2", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(10)}}}}, - mockBinlogEvent{typ: TableDrop, args: []interface{}{"stest", "log"}}, - mockBinlogEvent{typ: TableDrop, args: []interface{}{"stest", "t"}}, - mockBinlogEvent{typ: TableDrop, args: []interface{}{"stest", "log2"}}, - mockBinlogEvent{typ: DBDrop, args: []interface{}{"stest"}}, - - mockBinlogEvent{typ: DBCreate, args: []interface{}{"t2"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"t2", "create table t2.log(id int)"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"t2", "create table t2.log1(id int)"}}, - mockBinlogEvent{typ: TableDrop, args: []interface{}{"t2", "log"}}, - mockBinlogEvent{typ: DBDrop, args: []interface{}{"t2"}}, - - mockBinlogEvent{typ: DBCreate, args: []interface{}{"ptest1"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"ptest1", "create table ptest1.t1(id int)"}}, - mockBinlogEvent{typ: DBDrop, args: []interface{}{"ptest1"}}, + mockBinlogEvent{typ: DBCreate, args: []any{"s1"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"s1", "create table s1.log(id int)"}}, + mockBinlogEvent{typ: DBDrop, args: []any{"s1"}}, + + mockBinlogEvent{typ: TableCreate, args: []any{"mysql", "create table mysql.test(id int)"}}, + mockBinlogEvent{typ: TableDrop, args: []any{"mysql", "test"}}, + mockBinlogEvent{typ: DBCreate, args: []any{"stest"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"stest", "create table stest.log(id int)"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"stest", "create table stest.t(id int)"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"stest", "create table stest.log2(id int)"}}, + mockBinlogEvent{typ: Write, args: []any{uint64(8), "stest", "t", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(10)}}}}, + mockBinlogEvent{typ: Write, args: []any{uint64(9), "stest", "log", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(10)}}}}, + mockBinlogEvent{typ: Write, args: []any{uint64(10), "stest", "log2", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(10)}}}}, + mockBinlogEvent{typ: TableDrop, args: []any{"stest", "log"}}, + mockBinlogEvent{typ: TableDrop, args: []any{"stest", "t"}}, + mockBinlogEvent{typ: TableDrop, args: []any{"stest", "log2"}}, + mockBinlogEvent{typ: DBDrop, args: []any{"stest"}}, + + mockBinlogEvent{typ: DBCreate, args: []any{"t2"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"t2", "create table t2.log(id int)"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"t2", "create table t2.log1(id int)"}}, + mockBinlogEvent{typ: TableDrop, args: []any{"t2", "log"}}, + mockBinlogEvent{typ: DBDrop, args: []any{"t2"}}, + + mockBinlogEvent{typ: DBCreate, args: []any{"ptest1"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"ptest1", "create table ptest1.t1(id int)"}}, + mockBinlogEvent{typ: DBDrop, args: []any{"ptest1"}}, } allEvents := s.generateEvents(events, c) @@ -417,18 +417,18 @@ func (s *testSyncerSuite) TestIgnoreDB(c *check.C) { s.resetEventsGenerator(c) events := mockBinlogEvents{ - mockBinlogEvent{typ: DBCreate, args: []interface{}{"s1"}}, - mockBinlogEvent{typ: DBDrop, args: []interface{}{"s1"}}, - mockBinlogEvent{typ: DBCreate, args: []interface{}{"s2"}}, - mockBinlogEvent{typ: DBDrop, args: []interface{}{"s2"}}, - mockBinlogEvent{typ: DBCreate, args: []interface{}{"btest"}}, - mockBinlogEvent{typ: DBDrop, args: []interface{}{"btest"}}, - mockBinlogEvent{typ: DBCreate, args: []interface{}{"b1"}}, - mockBinlogEvent{typ: DBDrop, args: []interface{}{"b1"}}, - mockBinlogEvent{typ: DBCreate, args: []interface{}{"stest"}}, - mockBinlogEvent{typ: DBDrop, args: []interface{}{"stest"}}, - mockBinlogEvent{typ: DBCreate, args: []interface{}{"st"}}, - mockBinlogEvent{typ: DBDrop, args: []interface{}{"st"}}, + mockBinlogEvent{typ: DBCreate, args: []any{"s1"}}, + mockBinlogEvent{typ: DBDrop, args: []any{"s1"}}, + mockBinlogEvent{typ: DBCreate, args: []any{"s2"}}, + mockBinlogEvent{typ: DBDrop, args: []any{"s2"}}, + mockBinlogEvent{typ: DBCreate, args: []any{"btest"}}, + mockBinlogEvent{typ: DBDrop, args: []any{"btest"}}, + mockBinlogEvent{typ: DBCreate, args: []any{"b1"}}, + mockBinlogEvent{typ: DBDrop, args: []any{"b1"}}, + mockBinlogEvent{typ: DBCreate, args: []any{"stest"}}, + mockBinlogEvent{typ: DBDrop, args: []any{"stest"}}, + mockBinlogEvent{typ: DBCreate, args: []any{"st"}}, + mockBinlogEvent{typ: DBDrop, args: []any{"st"}}, } allEvents := s.generateEvents(events, c) @@ -480,31 +480,31 @@ func (s *testSyncerSuite) TestIgnoreTable(c *check.C) { s.resetEventsGenerator(c) events := mockBinlogEvents{ - mockBinlogEvent{typ: DBCreate, args: []interface{}{"s1"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"s1", "create table s1.log(id int)"}}, - mockBinlogEvent{typ: DBDrop, args: []interface{}{"s1"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"mysql", "create table mysql.test(id int)"}}, - mockBinlogEvent{typ: TableDrop, args: []interface{}{"mysql", "test"}}, - - mockBinlogEvent{typ: DBCreate, args: []interface{}{"stest"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"stest", "create table stest.log(id int)"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"stest", "create table stest.t(id int)"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"stest", "create table stest.log2(id int)"}}, - - mockBinlogEvent{typ: Write, args: []interface{}{uint64(8), "stest", "t", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(10)}}}}, - mockBinlogEvent{typ: Write, args: []interface{}{uint64(9), "stest", "log", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(10)}}}}, - mockBinlogEvent{typ: Write, args: []interface{}{uint64(10), "stest", "log2", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(10)}}}}, + mockBinlogEvent{typ: DBCreate, args: []any{"s1"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"s1", "create table s1.log(id int)"}}, + mockBinlogEvent{typ: DBDrop, args: []any{"s1"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"mysql", "create table mysql.test(id int)"}}, + mockBinlogEvent{typ: TableDrop, args: []any{"mysql", "test"}}, + + mockBinlogEvent{typ: DBCreate, args: []any{"stest"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"stest", "create table stest.log(id int)"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"stest", "create table stest.t(id int)"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"stest", "create table stest.log2(id int)"}}, + + mockBinlogEvent{typ: Write, args: []any{uint64(8), "stest", "t", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(10)}}}}, + mockBinlogEvent{typ: Write, args: []any{uint64(9), "stest", "log", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(10)}}}}, + mockBinlogEvent{typ: Write, args: []any{uint64(10), "stest", "log2", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(10)}}}}, // TODO event generator support generate an event with multiple tables DDL - mockBinlogEvent{typ: TableDrop, args: []interface{}{"stest", "log"}}, - mockBinlogEvent{typ: TableDrop, args: []interface{}{"stest", "t"}}, - mockBinlogEvent{typ: TableDrop, args: []interface{}{"stest", "log2"}}, - mockBinlogEvent{typ: DBDrop, args: []interface{}{"stest"}}, + mockBinlogEvent{typ: TableDrop, args: []any{"stest", "log"}}, + mockBinlogEvent{typ: TableDrop, args: []any{"stest", "t"}}, + mockBinlogEvent{typ: TableDrop, args: []any{"stest", "log2"}}, + mockBinlogEvent{typ: DBDrop, args: []any{"stest"}}, - mockBinlogEvent{typ: DBCreate, args: []interface{}{"t2"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"t2", "create table t2.log(id int)"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"t2", "create table t2.log1(id int)"}}, - mockBinlogEvent{typ: TableDrop, args: []interface{}{"t2", "log"}}, - mockBinlogEvent{typ: DBDrop, args: []interface{}{"t2"}}, + mockBinlogEvent{typ: DBCreate, args: []any{"t2"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"t2", "create table t2.log(id int)"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"t2", "create table t2.log1(id int)"}}, + mockBinlogEvent{typ: TableDrop, args: []any{"t2", "log"}}, + mockBinlogEvent{typ: DBDrop, args: []any{"t2"}}, } allEvents := s.generateEvents(events, c) @@ -584,49 +584,49 @@ func (s *testSyncerSuite) TestSkipDML(c *check.C) { sqls := make([]SQLChecker, 0, 16) - evs := s.generateEvents([]mockBinlogEvent{{DBCreate, []interface{}{"foo"}}}, c) + evs := s.generateEvents([]mockBinlogEvent{{DBCreate, []any{"foo"}}}, c) sqls = append(sqls, SQLChecker{events: evs, isDML: false, expected: false}) - evs = s.generateEvents([]mockBinlogEvent{{TableCreate, []interface{}{"foo", "create table foo.bar(id int)"}}}, c) + evs = s.generateEvents([]mockBinlogEvent{{TableCreate, []any{"foo", "create table foo.bar(id int)"}}}, c) sqls = append(sqls, SQLChecker{events: evs, isDML: false, expected: false}) - evs = s.generateEvents([]mockBinlogEvent{{Write, []interface{}{uint64(8), "foo", "bar", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(1)}}}}}, c) + evs = s.generateEvents([]mockBinlogEvent{{Write, []any{uint64(8), "foo", "bar", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(1)}}}}}, c) sqls = append(sqls, SQLChecker{events: evs, isDML: true, expected: false}) - evs = s.generateEvents([]mockBinlogEvent{{Update, []interface{}{uint64(8), "foo", "bar", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(2)}, {int32(1)}}}}}, c) + evs = s.generateEvents([]mockBinlogEvent{{Update, []any{uint64(8), "foo", "bar", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(2)}, {int32(1)}}}}}, c) sqls = append(sqls, SQLChecker{events: evs, isDML: true, expected: true}) - evs = s.generateEvents([]mockBinlogEvent{{Delete, []interface{}{uint64(8), "foo", "bar", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(2)}}}}}, c) + evs = s.generateEvents([]mockBinlogEvent{{Delete, []any{uint64(8), "foo", "bar", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(2)}}}}}, c) sqls = append(sqls, SQLChecker{events: evs, isDML: true, expected: true}) - evs = s.generateEvents([]mockBinlogEvent{{DBDrop, []interface{}{"foo1"}}}, c) + evs = s.generateEvents([]mockBinlogEvent{{DBDrop, []any{"foo1"}}}, c) sqls = append(sqls, SQLChecker{events: evs, isDML: false, expected: false}) - evs = s.generateEvents([]mockBinlogEvent{{DBCreate, []interface{}{"foo1"}}}, c) + evs = s.generateEvents([]mockBinlogEvent{{DBCreate, []any{"foo1"}}}, c) sqls = append(sqls, SQLChecker{events: evs, isDML: false, expected: false}) - evs = s.generateEvents([]mockBinlogEvent{{TableCreate, []interface{}{"foo1", "create table foo1.bar1(id int)"}}}, c) + evs = s.generateEvents([]mockBinlogEvent{{TableCreate, []any{"foo1", "create table foo1.bar1(id int)"}}}, c) sqls = append(sqls, SQLChecker{events: evs, isDML: false, expected: false}) - evs = s.generateEvents([]mockBinlogEvent{{Write, []interface{}{uint64(9), "foo1", "bar1", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(1)}}}}}, c) + evs = s.generateEvents([]mockBinlogEvent{{Write, []any{uint64(9), "foo1", "bar1", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(1)}}}}}, c) sqls = append(sqls, SQLChecker{events: evs, isDML: true, expected: false}) - evs = s.generateEvents([]mockBinlogEvent{{Update, []interface{}{uint64(9), "foo1", "bar1", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(2)}, {int32(1)}}}}}, c) + evs = s.generateEvents([]mockBinlogEvent{{Update, []any{uint64(9), "foo1", "bar1", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(2)}, {int32(1)}}}}}, c) sqls = append(sqls, SQLChecker{events: evs, isDML: true, expected: true}) - evs = s.generateEvents([]mockBinlogEvent{{Delete, []interface{}{uint64(9), "foo1", "bar1", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(2)}}}}}, c) + evs = s.generateEvents([]mockBinlogEvent{{Delete, []any{uint64(9), "foo1", "bar1", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(2)}}}}}, c) sqls = append(sqls, SQLChecker{events: evs, isDML: true, expected: true}) - evs = s.generateEvents([]mockBinlogEvent{{TableCreate, []interface{}{"foo1", "create table foo1.bar2(id int)"}}}, c) + evs = s.generateEvents([]mockBinlogEvent{{TableCreate, []any{"foo1", "create table foo1.bar2(id int)"}}}, c) sqls = append(sqls, SQLChecker{events: evs, isDML: false, expected: false}) - evs = s.generateEvents([]mockBinlogEvent{{Write, []interface{}{uint64(10), "foo1", "bar2", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(1)}}}}}, c) + evs = s.generateEvents([]mockBinlogEvent{{Write, []any{uint64(10), "foo1", "bar2", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(1)}}}}}, c) sqls = append(sqls, SQLChecker{events: evs, isDML: true, expected: false}) - evs = s.generateEvents([]mockBinlogEvent{{Update, []interface{}{uint64(10), "foo1", "bar2", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(2)}, {int32(1)}}}}}, c) + evs = s.generateEvents([]mockBinlogEvent{{Update, []any{uint64(10), "foo1", "bar2", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(2)}, {int32(1)}}}}}, c) sqls = append(sqls, SQLChecker{events: evs, isDML: true, expected: true}) - evs = s.generateEvents([]mockBinlogEvent{{Delete, []interface{}{uint64(10), "foo1", "bar2", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(2)}}}}}, c) + evs = s.generateEvents([]mockBinlogEvent{{Delete, []any{uint64(10), "foo1", "bar2", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(2)}}}}}, c) sqls = append(sqls, SQLChecker{events: evs, isDML: true, expected: true}) p := parser.New() @@ -684,10 +684,10 @@ func (s *testSyncerSuite) TestColumnMapping(c *check.C) { // create db and tables events := mockBinlogEvents{ - mockBinlogEvent{typ: DBCreate, args: []interface{}{"stest_3"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"stest_3", "create table stest_3.log(id varchar(45))"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"stest_3", "create table stest_3.t_2(name varchar(45), id bigint)"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"stest_3", "create table stest_3.a(id int)"}}, + mockBinlogEvent{typ: DBCreate, args: []any{"stest_3"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"stest_3", "create table stest_3.log(id varchar(45))"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"stest_3", "create table stest_3.t_2(name varchar(45), id bigint)"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"stest_3", "create table stest_3.a(id int)"}}, } createEvents := s.generateEvents(events, c) @@ -696,19 +696,19 @@ func (s *testSyncerSuite) TestColumnMapping(c *check.C) { type dml struct { events []*replication.BinlogEvent column []string - data []interface{} + data []any } dmls := make([]dml, 0, 3) - evs := s.generateEvents([]mockBinlogEvent{{typ: Write, args: []interface{}{uint64(8), "stest_3", "t_2", []byte{mysql.MYSQL_TYPE_STRING, mysql.MYSQL_TYPE_LONG}, [][]interface{}{{"ian", int32(10)}}}}}, c) - dmls = append(dmls, dml{events: evs, column: []string{"name", "id"}, data: []interface{}{"ian", int64(1<<59 | 3<<52 | 2<<44 | 10)}}) + evs := s.generateEvents([]mockBinlogEvent{{typ: Write, args: []any{uint64(8), "stest_3", "t_2", []byte{mysql.MYSQL_TYPE_STRING, mysql.MYSQL_TYPE_LONG}, [][]any{{"ian", int32(10)}}}}}, c) + dmls = append(dmls, dml{events: evs, column: []string{"name", "id"}, data: []any{"ian", int64(1<<59 | 3<<52 | 2<<44 | 10)}}) - evs = s.generateEvents([]mockBinlogEvent{{typ: Write, args: []interface{}{uint64(9), "stest_3", "log", []byte{mysql.MYSQL_TYPE_STRING}, [][]interface{}{{"10"}}}}}, c) - dmls = append(dmls, dml{events: evs, column: []string{"id"}, data: []interface{}{"test:10"}}) + evs = s.generateEvents([]mockBinlogEvent{{typ: Write, args: []any{uint64(9), "stest_3", "log", []byte{mysql.MYSQL_TYPE_STRING}, [][]any{{"10"}}}}}, c) + dmls = append(dmls, dml{events: evs, column: []string{"id"}, data: []any{"test:10"}}) - evs = s.generateEvents([]mockBinlogEvent{{typ: Write, args: []interface{}{uint64(10), "stest_3", "a", []byte{mysql.MYSQL_TYPE_LONG}, [][]interface{}{{int32(10)}}}}}, c) - dmls = append(dmls, dml{events: evs, column: []string{"id"}, data: []interface{}{int32(10)}}) + evs = s.generateEvents([]mockBinlogEvent{{typ: Write, args: []any{uint64(10), "stest_3", "a", []byte{mysql.MYSQL_TYPE_LONG}, [][]any{{int32(10)}}}}}, c) + dmls = append(dmls, dml{events: evs, column: []string{"id"}, data: []any{int32(10)}}) dmlEvents := make([]*replication.BinlogEvent, 0, 15) for _, dml := range dmls { @@ -718,10 +718,10 @@ func (s *testSyncerSuite) TestColumnMapping(c *check.C) { // drop tables and db events = mockBinlogEvents{ // TODO event generator support generate an event with multiple tables DDL - mockBinlogEvent{typ: TableDrop, args: []interface{}{"stest_3", "log"}}, - mockBinlogEvent{typ: TableDrop, args: []interface{}{"stest_3", "t_2"}}, - mockBinlogEvent{typ: TableDrop, args: []interface{}{"stest_3", "a"}}, - mockBinlogEvent{typ: DBDrop, args: []interface{}{"stest_3"}}, + mockBinlogEvent{typ: TableDrop, args: []any{"stest_3", "log"}}, + mockBinlogEvent{typ: TableDrop, args: []any{"stest_3", "t_2"}}, + mockBinlogEvent{typ: TableDrop, args: []any{"stest_3", "a"}}, + mockBinlogEvent{typ: DBDrop, args: []any{"stest_3"}}, } dropEvents := s.generateEvents(events, c) @@ -832,19 +832,19 @@ func (s *testSyncerSuite) TestRun(c *check.C) { syncer.reset() events1 := mockBinlogEvents{ - mockBinlogEvent{typ: DBCreate, args: []interface{}{"test_1"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"test_1", "create table test_1.t_1(id int primary key, name varchar(24))"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"test_1", "create table test_1.t_2(id int primary key, name varchar(24))"}}, + mockBinlogEvent{typ: DBCreate, args: []any{"test_1"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"test_1", "create table test_1.t_1(id int primary key, name varchar(24))"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"test_1", "create table test_1.t_2(id int primary key, name varchar(24))"}}, - mockBinlogEvent{typ: Write, args: []interface{}{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]interface{}{{int32(1), "a"}}}}, - mockBinlogEvent{typ: DDL, args: []interface{}{"test_1", "alter table test_1.t_1 add index index1(name)"}}, - mockBinlogEvent{typ: Write, args: []interface{}{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]interface{}{{int32(2), "b"}}}}, - mockBinlogEvent{typ: Delete, args: []interface{}{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]interface{}{{int32(1), "a"}}}}, - mockBinlogEvent{typ: Update, args: []interface{}{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]interface{}{{int32(2), "b"}, {int32(1), "b"}}}}, + mockBinlogEvent{typ: Write, args: []any{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]any{{int32(1), "a"}}}}, + mockBinlogEvent{typ: DDL, args: []any{"test_1", "alter table test_1.t_1 add index index1(name)"}}, + mockBinlogEvent{typ: Write, args: []any{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]any{{int32(2), "b"}}}}, + mockBinlogEvent{typ: Delete, args: []any{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]any{{int32(1), "a"}}}}, + mockBinlogEvent{typ: Update, args: []any{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]any{{int32(2), "b"}, {int32(1), "b"}}}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"test_1", "create table test_1.t_3(id int primary key, name varchar(24))"}}, - mockBinlogEvent{typ: DDL, args: []interface{}{"test_1", "alter table test_1.t_3 drop primary key"}}, - mockBinlogEvent{typ: DDL, args: []interface{}{"test_1", "alter table test_1.t_3 add primary key(id, name)"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"test_1", "create table test_1.t_3(id int primary key, name varchar(24))"}}, + mockBinlogEvent{typ: DDL, args: []any{"test_1", "alter table test_1.t_3 drop primary key"}}, + mockBinlogEvent{typ: DDL, args: []any{"test_1", "alter table test_1.t_3 add primary key(id, name)"}}, } mockStreamerProducer := &MockStreamProducer{s.generateEvents(events1, c)} @@ -899,7 +899,7 @@ func (s *testSyncerSuite) TestRun(c *check.C) { }, { dml, []string{"REPLACE INTO `test_1`.`t_1` (`id`,`name`) VALUES (?,?)"}, - [][]interface{}{{int64(1), "a"}}, + [][]any{{int64(1), "a"}}, }, { flush, nil, @@ -911,16 +911,16 @@ func (s *testSyncerSuite) TestRun(c *check.C) { }, { dml, []string{"REPLACE INTO `test_1`.`t_1` (`id`,`name`) VALUES (?,?)"}, - [][]interface{}{{int64(2), "b"}}, + [][]any{{int64(2), "b"}}, }, { dml, []string{"DELETE FROM `test_1`.`t_1` WHERE `id` = ? LIMIT 1"}, - [][]interface{}{{int64(1)}}, + [][]any{{int64(1)}}, }, { // safe mode is true, will split update to delete + replace dml, []string{"DELETE FROM `test_1`.`t_1` WHERE `id` = ? LIMIT 1", "REPLACE INTO `test_1`.`t_1` (`id`,`name`) VALUES (?,?)"}, - [][]interface{}{{int64(2)}, {int64(1), "b"}}, + [][]any{{int64(2)}, {int64(1), "b"}}, }, { flush, nil, @@ -991,8 +991,8 @@ func (s *testSyncerSuite) TestRun(c *check.C) { c.Assert(syncer.timezone.String(), check.Equals, "+01:00") events2 := mockBinlogEvents{ - mockBinlogEvent{typ: Write, args: []interface{}{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]interface{}{{int32(3), "c"}}}}, - mockBinlogEvent{typ: Delete, args: []interface{}{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]interface{}{{int32(3), "c"}}}}, + mockBinlogEvent{typ: Write, args: []any{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]any{{int32(3), "c"}}}}, + mockBinlogEvent{typ: Delete, args: []any{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]any{{int32(3), "c"}}}}, } ctx, cancel = context.WithCancel(context.Background()) @@ -1029,11 +1029,11 @@ func (s *testSyncerSuite) TestRun(c *check.C) { { dml, []string{"INSERT INTO `test_1`.`t_2` (`id`,`name`) VALUES (?,?)"}, - [][]interface{}{{int64(3), "c"}}, + [][]any{{int64(3), "c"}}, }, { dml, []string{"DELETE FROM `test_1`.`t_2` WHERE `id` = ? LIMIT 1"}, - [][]interface{}{{int64(3)}}, + [][]any{{int64(3)}}, }, { flush, nil, @@ -1165,12 +1165,12 @@ func (s *testSyncerSuite) TestExitSafeModeByConfig(c *check.C) { syncer.reset() events1 := mockBinlogEvents{ - mockBinlogEvent{typ: DBCreate, args: []interface{}{"test_1"}}, - mockBinlogEvent{typ: TableCreate, args: []interface{}{"test_1", "create table test_1.t_1(id int primary key, name varchar(24))"}}, + mockBinlogEvent{typ: DBCreate, args: []any{"test_1"}}, + mockBinlogEvent{typ: TableCreate, args: []any{"test_1", "create table test_1.t_1(id int primary key, name varchar(24))"}}, - mockBinlogEvent{typ: Write, args: []interface{}{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]interface{}{{int32(1), "a"}}}}, - mockBinlogEvent{typ: Delete, args: []interface{}{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]interface{}{{int32(1), "a"}}}}, - mockBinlogEvent{typ: Update, args: []interface{}{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]interface{}{{int32(2), "b"}, {int32(1), "b"}}}}, + mockBinlogEvent{typ: Write, args: []any{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]any{{int32(1), "a"}}}}, + mockBinlogEvent{typ: Delete, args: []any{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]any{{int32(1), "a"}}}}, + mockBinlogEvent{typ: Update, args: []any{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]any{{int32(2), "b"}, {int32(1), "b"}}}}, } generatedEvents1 := s.generateEvents(events1, c) @@ -1182,9 +1182,9 @@ func (s *testSyncerSuite) TestExitSafeModeByConfig(c *check.C) { // check after safeModeExitLocation, safe mode is turned off events2 := mockBinlogEvents{ - mockBinlogEvent{typ: Write, args: []interface{}{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]interface{}{{int32(1), "a"}}}}, - mockBinlogEvent{typ: Delete, args: []interface{}{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]interface{}{{int32(1), "a"}}}}, - mockBinlogEvent{typ: Update, args: []interface{}{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]interface{}{{int32(2), "b"}, {int32(1), "b"}}}}, + mockBinlogEvent{typ: Write, args: []any{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]any{{int32(1), "a"}}}}, + mockBinlogEvent{typ: Delete, args: []any{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]any{{int32(1), "a"}}}}, + mockBinlogEvent{typ: Update, args: []any{uint64(8), "test_1", "t_1", []byte{mysql.MYSQL_TYPE_LONG, mysql.MYSQL_TYPE_STRING}, [][]any{{int32(2), "b"}, {int32(1), "b"}}}}, } generatedEvents2 := s.generateEvents(events2, c) @@ -1248,28 +1248,28 @@ func (s *testSyncerSuite) TestExitSafeModeByConfig(c *check.C) { }, { dml, []string{"REPLACE INTO `test_1`.`t_1` (`id`,`name`) VALUES (?,?)"}, - [][]interface{}{{int64(1), "a"}}, + [][]any{{int64(1), "a"}}, }, { dml, []string{"DELETE FROM `test_1`.`t_1` WHERE `id` = ? LIMIT 1"}, - [][]interface{}{{int64(1)}}, + [][]any{{int64(1)}}, }, { dml, []string{"DELETE FROM `test_1`.`t_1` WHERE `id` = ? LIMIT 1", "REPLACE INTO `test_1`.`t_1` (`id`,`name`) VALUES (?,?)"}, - [][]interface{}{{int64(2)}, {int64(1), "b"}}, + [][]any{{int64(2)}, {int64(1), "b"}}, }, { // start from this event, location passes safeModeExitLocation and safe mode should exit dml, []string{"INSERT INTO `test_1`.`t_1` (`id`,`name`) VALUES (?,?)"}, - [][]interface{}{{int64(1), "a"}}, + [][]any{{int64(1), "a"}}, }, { dml, []string{"DELETE FROM `test_1`.`t_1` WHERE `id` = ? LIMIT 1"}, - [][]interface{}{{int64(1)}}, + [][]any{{int64(1)}}, }, { dml, []string{"UPDATE `test_1`.`t_1` SET `id` = ?, `name` = ? WHERE `id` = ? LIMIT 1"}, - [][]interface{}{{int64(1), "b", int64(2)}}, + [][]any{{int64(1), "b", int64(2)}}, }, { flush, nil, @@ -1515,7 +1515,7 @@ func checkEventWithTableResult(c *check.C, syncer *Syncer, allEvents []*replicat } func executeSQLAndWait(expectJobNum int) { - for i := 0; i < 10; i++ { + for range 10 { time.Sleep(time.Second) testJobs.RLock() @@ -1531,7 +1531,7 @@ func executeSQLAndWait(expectJobNum int) { type expectJob struct { tp opType sqlInJob []string - args [][]interface{} + args [][]any } var defaultDMLType = map[sqlmodel.RowChangeType]sqlmodel.DMLType{ @@ -1554,7 +1554,7 @@ func checkJobs(c *check.C, jobs []*job, expectJobs []*expectJob) { if !job.safeMode { sql, args := job.dml.GenSQL(defaultDMLType[job.dml.Type()]) c.Assert([]string{sql}, check.DeepEquals, expectJobs[i].sqlInJob) - c.Assert([][]interface{}{args}, check.DeepEquals, expectJobs[i].args) + c.Assert([][]any{args}, check.DeepEquals, expectJobs[i].args) continue } @@ -1563,16 +1563,16 @@ func checkJobs(c *check.C, jobs []*job, expectJobs []*expectJob) { case sqlmodel.RowChangeInsert: sql, args := job.dml.GenSQL(sqlmodel.DMLReplace) c.Assert([]string{sql}, check.DeepEquals, expectJobs[i].sqlInJob) - c.Assert([][]interface{}{args}, check.DeepEquals, expectJobs[i].args) + c.Assert([][]any{args}, check.DeepEquals, expectJobs[i].args) case sqlmodel.RowChangeUpdate: sql, args := job.dml.GenSQL(sqlmodel.DMLDelete) sql2, args2 := job.dml.GenSQL(sqlmodel.DMLReplace) c.Assert([]string{sql, sql2}, check.DeepEquals, expectJobs[i].sqlInJob) - c.Assert([][]interface{}{args, args2}, check.DeepEquals, expectJobs[i].args) + c.Assert([][]any{args, args2}, check.DeepEquals, expectJobs[i].args) case sqlmodel.RowChangeDelete: sql, args := job.dml.GenSQL(sqlmodel.DMLDelete) c.Assert([]string{sql}, check.DeepEquals, expectJobs[i].sqlInJob) - c.Assert([][]interface{}{args}, check.DeepEquals, expectJobs[i].args) + c.Assert([][]any{args}, check.DeepEquals, expectJobs[i].args) } } } @@ -1934,11 +1934,9 @@ func TestWaitBeforeRunExit(t *testing.T) { wg := &sync.WaitGroup{} errCh := make(chan error, 1) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { errCh <- syncer.Run(ctx) - }() + }) time.Sleep(time.Second) // wait s.Run start // test s.Run will not exit unit caller cancel ctx or call s.runCancel diff --git a/dm/syncer/validate_worker.go b/dm/syncer/validate_worker.go index 3e7082085f..8a2ccd0e10 100644 --- a/dm/syncer/validate_worker.go +++ b/dm/syncer/validate_worker.go @@ -17,6 +17,7 @@ import ( "context" "database/sql" "fmt" + "maps" "math" "strconv" "strings" @@ -201,9 +202,7 @@ func (vw *validateWorker) validateTableChange() { if err2 != nil { return err2 } - for key, val := range failedRows { - allFailedRows[key] = val - } + maps.Copy(allFailedRows, failedRows) return nil } if err = validateFunc(insertUpdateChanges, false); err != nil { @@ -282,18 +281,13 @@ func (vw *validateWorker) updatePendingAndErrorRows(failedChanges map[string]map func (vw *validateWorker) validateRowChanges(rows []*rowValidationJob, deleteChange bool) (map[string]*validateFailedRow, error) { res := make(map[string]*validateFailedRow) for start := 0; start < len(rows); start += vw.batchSize { - end := start + vw.batchSize - if end > len(rows) { - end = len(rows) - } + end := min(start+vw.batchSize, len(rows)) batch := rows[start:end] failedRows, err := vw.batchValidateRowChanges(batch, deleteChange) if err != nil { return nil, err } - for k, v := range failedRows { - res[k] = v - } + maps.Copy(res, failedRows) } return res, nil } @@ -528,7 +522,7 @@ func scanRow(rows *sql.Rows) ([]*sql.NullString, error) { } colVals := make([][]byte, len(cols)) - colValsI := make([]interface{}, len(colVals)) + colValsI := make([]any, len(colVals)) for i := range colValsI { colValsI[i] = &colVals[i] } diff --git a/dm/syncer/validate_worker_test.go b/dm/syncer/validate_worker_test.go index 34161d0409..da750b9405 100644 --- a/dm/syncer/validate_worker_test.go +++ b/dm/syncer/validate_worker_test.go @@ -36,8 +36,8 @@ import ( "github.com/stretchr/testify/require" ) -func genRowChangeJob(tbl filter.Table, tblInfo *model.TableInfo, key string, tp rowChangeJobType, data []interface{}) *rowValidationJob { - var beforeImage, afterImage []interface{} +func genRowChangeJob(tbl filter.Table, tblInfo *model.TableInfo, key string, tp rowChangeJobType, data []any) *rowValidationJob { + var beforeImage, afterImage []any switch tp { case rowInsert: afterImage = data @@ -104,9 +104,9 @@ func TestValidatorWorkerValidateTableChanges(t *testing.T) { checkInitStatus() // insert & update same table, both row are validated failed - worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "1", rowInsert, []interface{}{1, "a"})) - worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "1", rowUpdated, []interface{}{1, "b"})) - worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "2", rowInsert, []interface{}{2, "2b"})) + worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "1", rowInsert, []any{1, "a"})) + worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "1", rowUpdated, []any{1, "b"})) + worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "2", rowInsert, []any{2, "2b"})) mock.ExpectQuery("SELECT .* FROM .*tbl1.* WHERE .*").WillReturnRows( sqlmock.NewRows([]string{"a", "b"}).AddRow(2, "incorrect data")) @@ -163,8 +163,8 @@ func TestValidatorWorkerValidateTableChanges(t *testing.T) { // // add 2 delete row of tbl2 and tbl3 - worker.updateRowChange(genRowChangeJob(tbl2, tableInfo2, "a", rowDeleted, []interface{}{"a", "b"})) - worker.updateRowChange(genRowChangeJob(tbl3, tableInfo3, "aa", rowDeleted, []interface{}{"aa", "b"})) + worker.updateRowChange(genRowChangeJob(tbl2, tableInfo2, "a", rowDeleted, []any{"a", "b"})) + worker.updateRowChange(genRowChangeJob(tbl3, tableInfo3, "aa", rowDeleted, []any{"aa", "b"})) mock.ExpectQuery("SELECT .* FROM .*tbl1.* WHERE .*").WillReturnRows( sqlmock.NewRows([]string{"a", "b"})) @@ -208,9 +208,9 @@ func TestValidatorWorkerValidateTableChanges(t *testing.T) { // // validate with batch size = 2 worker.batchSize = 2 - worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "1", rowInsert, []interface{}{1, "a"})) - worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "2", rowInsert, []interface{}{2, "2b"})) - worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "3", rowInsert, []interface{}{3, "3c"})) + worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "1", rowInsert, []any{1, "a"})) + worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "2", rowInsert, []any{2, "2b"})) + worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "3", rowInsert, []any{3, "3c"})) mock.ExpectQuery("SELECT .* FROM .*tbl1.* WHERE .*").WillReturnRows( sqlmock.NewRows([]string{"a", "b"}).AddRow(1, "a").AddRow(2, "2b")) @@ -273,7 +273,7 @@ func TestValidatorWorkerValidateTableChanges(t *testing.T) { // set markErrorStarted = false, there should not be any errors and failedCount=0 validator.markErrorStarted.Store(false) - worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "1", rowInsert, []interface{}{1, "a"})) + worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "1", rowInsert, []any{1, "a"})) mock.ExpectQuery("SELECT .* FROM .*tbl1.* WHERE .*").WillReturnRows( sqlmock.NewRows([]string{"a", "b"})) @@ -304,7 +304,7 @@ func TestValidatorWorkerValidateTableChanges(t *testing.T) { // set markErrorStarted=true, rowErrorDelayInSec = 0, failed rows became error directly validator.markErrorStarted.Store(true) worker.rowErrorDelayInSec = 0 - worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "1", rowInsert, []interface{}{1, "a"})) + worker.updateRowChange(genRowChangeJob(tbl1, tableInfo1, "1", rowInsert, []any{1, "a"})) mock.ExpectQuery("SELECT .* FROM .*tbl1.* WHERE .*").WillReturnRows( sqlmock.NewRows([]string{"a", "b"})) worker.validateTableChange() @@ -396,7 +396,7 @@ func TestValidatorWorkerGetTargetRows(t *testing.T) { creatSQL string pkValues [][]string allCols []string - rowData [][]interface{} + rowData [][]any querySQL string } testCases := []testCase{ @@ -413,7 +413,7 @@ func TestValidatorWorkerGetTargetRows(t *testing.T) { {"1", "2"}, {"3", "4"}, {"5", "6"}, }, allCols: []string{"a", "b", "c"}, - rowData: [][]interface{}{ + rowData: [][]any{ {"1", "2", "3"}, {"3", "4", "5"}, {"5", "6", "7"}, }, querySQL: "SELECT .* FROM .*test1.*", @@ -432,7 +432,7 @@ func TestValidatorWorkerGetTargetRows(t *testing.T) { {"a"}, {"b"}, {"c"}, }, allCols: []string{"a", "other", "b", "c"}, - rowData: [][]interface{}{ + rowData: [][]any{ {"a", "aaa", "\xdd\xcc", "1"}, {"b", "bbb", nil, "2"}, {"c", nil, nil, "3"}, }, querySQL: "SELECT .* FROM .*test2.*", @@ -513,8 +513,8 @@ func TestValidatorWorkerGetSourceRowsForCompare(t *testing.T) { tbl1 := filter.Table{Schema: "test", Name: "tbl1"} tableInfo1 := genValidateTableInfo(t, "create table tbl1(a varchar(10) primary key, b int)") rows := getSourceRowsForCompare([]*rowValidationJob{ - genRowChangeJob(tbl1, tableInfo1, "a", rowInsert, []interface{}{nil, 1}), - genRowChangeJob(tbl1, tableInfo1, "b", rowInsert, []interface{}{1, 2}), + genRowChangeJob(tbl1, tableInfo1, "a", rowInsert, []any{nil, 1}), + genRowChangeJob(tbl1, tableInfo1, "b", rowInsert, []any{1, 2}), }) require.Len(t, rows, 2) require.Len(t, rows["a"], 2) diff --git a/dm/syncer/validator_checkpoint.go b/dm/syncer/validator_checkpoint.go index bc1ab1e211..8e788038a4 100644 --- a/dm/syncer/validator_checkpoint.go +++ b/dm/syncer/validator_checkpoint.go @@ -99,7 +99,7 @@ func (c *validatorPersistHelper) init(tctx *tcontext.Context) error { c.db = c.validator.toDB if !c.schemaInitialized.Load() { - workFunc := func(tctx *tcontext.Context) (interface{}, error) { + workFunc := func(tctx *tcontext.Context) (any, error) { return nil, c.createSchemaAndTables(tctx) } if _, cnt, err := c.retryer.Apply(tctx, workFunc); err != nil { @@ -207,14 +207,14 @@ type rowChangeDataForPersist struct { Key string `json:"key"` Tp rowChangeJobType `json:"tp"` Size int32 `json:"size"` - Data []interface{} `json:"data"` + Data []any `json:"data"` FailedCnt int `json:"failed-cnt"` // failed count } var triggeredFailOnPersistForIntegrationTest bool -func (c *validatorPersistHelper) execQueriesWithRetry(tctx *tcontext.Context, queries []string, args [][]interface{}) error { - workFunc := func(tctx *tcontext.Context) (interface{}, error) { +func (c *validatorPersistHelper) execQueriesWithRetry(tctx *tcontext.Context, queries []string, args [][]any) error { + workFunc := func(tctx *tcontext.Context) (any, error) { for i, q := range queries { failpoint.Inject("ValidatorFailOnPersist", func() { // on persist pending row changes, the queries would be [delete, insert...] @@ -246,7 +246,7 @@ func (c *validatorPersistHelper) persistTableStatusAndErrors(tctx *tcontext.Cont tableStatus := c.validator.getTableStatusMap() count := len(tableStatus) + int(c.validator.getNewErrorRowCount()) queries := make([]string, 0, count) - args := make([][]interface{}, 0, count) + args := make([][]any, 0, count) // upsert table status for _, state := range tableStatus { @@ -262,7 +262,7 @@ func (c *validatorPersistHelper) persistTableStatusAndErrors(tctx *tcontext.Cont message = VALUES(message) ` queries = append(queries, query) - args = append(args, []interface{}{ + args = append(args, []any{ c.cfg.SourceID, state.source.Schema, state.source.Name, state.target.Schema, state.target.Name, int(state.stage), state.message, }, @@ -292,7 +292,7 @@ func (c *validatorPersistHelper) persistTableStatusAndErrors(tctx *tcontext.Cont if err != nil { return err } - dstData := make([]interface{}, len(r.dstData)) + dstData := make([]any, len(r.dstData)) for i, d := range r.dstData { if d.Valid { dstData[i] = d.String @@ -304,7 +304,7 @@ func (c *validatorPersistHelper) persistTableStatusAndErrors(tctx *tcontext.Cont } sourceTable := row.GetSourceTable() targetTable := row.GetTargetTable() - args = append(args, []interface{}{ + args = append(args, []any{ c.cfg.SourceID, sourceTable.Schema, sourceTable.Table, r.srcJob.Key, targetTable.Schema, targetTable.Table, string(srcDataBytes), string(dstDataBytes), r.tp, pb.ValidateErrorState_NewErr, @@ -318,11 +318,11 @@ func (c *validatorPersistHelper) persistTableStatusAndErrors(tctx *tcontext.Cont func (c *validatorPersistHelper) persistPendingRows(tctx *tcontext.Context, rev int64) error { count := int(c.validator.getAllPendingRowCount()) + 1 queries := make([]string, 0, count) - args := make([][]interface{}, 0, count) + args := make([][]any, 0, count) // delete pending rows left by previous failed call of "persist" queries = append(queries, `DELETE FROM `+c.pendingChangeTableName+` WHERE source = ? and revision = ?`) - args = append(args, []interface{}{c.cfg.SourceID, rev}) + args = append(args, []any{c.cfg.SourceID, rev}) // insert pending row changes with revision=rev for _, worker := range c.validator.getWorkers() { for _, tblChange := range worker.getPendingChangesMap() { @@ -344,7 +344,7 @@ func (c *validatorPersistHelper) persistPendingRows(tctx *tcontext.Context, rev (source, schema_name, table_name, row_pk, data, revision) VALUES (?, ?, ?, ?, ?, ?)` queries = append(queries, query) sourceTable := row.GetSourceTable() - args = append(args, []interface{}{ + args = append(args, []any{ c.cfg.SourceID, sourceTable.Schema, sourceTable.Table, @@ -393,12 +393,12 @@ func (c *validatorPersistHelper) persist(tctx *tcontext.Context, loc binlog.Loca revision = VALUES(revision) ` rowCounts := c.validator.getProcessedRowCounts() - args := []interface{}{ + args := []any{ c.cfg.SourceID, loc.Position.Name, loc.Position.Pos, loc.GTIDSetStr(), rowCounts[rowInsert], rowCounts[rowUpdated], rowCounts[rowDeleted], nextRevision, } - if err := c.execQueriesWithRetry(newCtx, []string{query}, [][]interface{}{args}); err != nil { + if err := c.execQueriesWithRetry(newCtx, []string{query}, [][]any{args}); err != nil { return err } @@ -406,8 +406,8 @@ func (c *validatorPersistHelper) persist(tctx *tcontext.Context, loc binlog.Loca // but we need to clean up previous pending row changes, i.e. rows with different revision. // it's ok to fail here, next persist will try to delete again, so just log it. query = `DELETE FROM ` + c.pendingChangeTableName + ` WHERE source = ? and revision != ?` - args = []interface{}{c.cfg.SourceID, nextRevision} - if err := c.execQueriesWithRetry(newCtx, []string{query}, [][]interface{}{args}); err != nil { + args = []any{c.cfg.SourceID, nextRevision} + if err := c.execQueriesWithRetry(newCtx, []string{query}, [][]any{args}); err != nil { c.L.Warn("failed to delete previous pending row changes", zap.Error(err), zap.Reflect("args", args)) // nolint:nilerr } @@ -430,7 +430,7 @@ func (c *validatorPersistHelper) loadPersistedDataRetry(tctx *tcontext.Context) start := time.Now() newCtx, cancelFunc := tctx.WithTimeout(validationDBTimeout) defer cancelFunc() - workFunc := func(tctx *tcontext.Context) (interface{}, error) { + workFunc := func(tctx *tcontext.Context) (any, error) { return c.loadPersistedData(tctx) } ret, i, err := c.retryer.Apply(newCtx, workFunc) @@ -637,7 +637,7 @@ func (c *validatorPersistHelper) loadError(tctx *tcontext.Context, db *conn.Base err error ) res := make([]*pb.ValidationError, 0) - args := []interface{}{ + args := []any{ c.cfg.SourceID, } query := "SELECT id, source, src_schema_name, src_table_name, dst_schema_name, dst_table_name, data, dst_data, error_type, status, update_time " + @@ -696,7 +696,7 @@ func (c *validatorPersistHelper) operateError(tctx *tcontext.Context, db *conn.B c.L.Warn("unsupported validator error operation", zap.Reflect("op", validateOp)) return nil } - args := []interface{}{ + args := []any{ int(setStatus), c.cfg.SourceID, } @@ -710,7 +710,7 @@ func (c *validatorPersistHelper) operateError(tctx *tcontext.Context, db *conn.B } func (c *validatorPersistHelper) deleteError(tctx *tcontext.Context, db *conn.BaseDB, errID uint64, isAll bool) error { - args := []interface{}{ + args := []any{ c.cfg.SourceID, } query := "DELETE FROM " + c.errorChangeTableName + " WHERE source=?" diff --git a/dm/syncer/validator_checkpoint_test.go b/dm/syncer/validator_checkpoint_test.go index 6f4f843ac7..00f93289cf 100644 --- a/dm/syncer/validator_checkpoint_test.go +++ b/dm/syncer/validator_checkpoint_test.go @@ -125,9 +125,9 @@ func TestValidatorCheckpointPersist(t *testing.T) { validator.workers[0].errorRows = append(validator.workers[0].errorRows, &validateFailedRow{ tp: deletedRowExists, dstData: []*sql.NullString{{String: "1", Valid: true}, {String: "a", Valid: true}}, - srcJob: genRowChangeJob(tbl, tblInfo, "1", rowDeleted, []interface{}{1, "a"}), + srcJob: genRowChangeJob(tbl, tblInfo, "1", rowDeleted, []any{1, "a"}), }) - validator.dispatchRowChange("1", genRowChangeJob(tbl, tblInfo, "1", rowInsert, []interface{}{1, "a"})) + validator.dispatchRowChange("1", genRowChangeJob(tbl, tblInfo, "1", rowInsert, []any{1, "a"})) validator.newErrorRowCount.Store(1) // fail on first persist @@ -179,7 +179,7 @@ func TestValidatorCheckpointPersist(t *testing.T) { validator.workers[0].errorRows = append(validator.workers[0].errorRows, &validateFailedRow{ tp: deletedRowExists, dstData: []*sql.NullString{{String: "1", Valid: true}, {String: "a", Valid: true}}, - srcJob: genRowChangeJob(tbl, tblInfo, "1", rowDeleted, []interface{}{1, "a"}), + srcJob: genRowChangeJob(tbl, tblInfo, "1", rowDeleted, []any{1, "a"}), }) validator.newErrorRowCount.Store(1) validator.flushedLoc = nil diff --git a/dm/syncer/validator_cond.go b/dm/syncer/validator_cond.go index 4c4d0e04a6..edff8d5890 100644 --- a/dm/syncer/validator_cond.go +++ b/dm/syncer/validator_cond.go @@ -26,8 +26,8 @@ type Cond struct { PkValues [][]string } -func (c *Cond) GetArgs() []interface{} { - var res []interface{} +func (c *Cond) GetArgs() []any { + var res []any for _, v := range c.PkValues { for _, val := range v { res = append(res, val) diff --git a/dm/syncer/validator_cond_test.go b/dm/syncer/validator_cond_test.go index c553f28051..73903927b7 100644 --- a/dm/syncer/validator_cond_test.go +++ b/dm/syncer/validator_cond_test.go @@ -66,7 +66,7 @@ func TestValidatorCondSelectMultiKey(t *testing.T) { ");" // get table diff pkValues := make([][]string, 0) - for i := 0; i < 3; i++ { + for i := range 3 { // 3 primary key key1, key2 := strconv.Itoa(i+1), strconv.Itoa(i+2) pkValues = append(pkValues, []string{key1, key2}) @@ -141,11 +141,11 @@ func TestValidatorCondGetWhereArgs(t *testing.T) { }, }, } - for i := 0; i < len(cases); i++ { + for i := range cases { cond := genValidationCond(t, cases[i].schemaName, cases[i].tblName, cases[i].creatTbl, cases[i].pks) require.Equal(t, cases[i].where, cond.GetWhere()) rawArgs := cond.GetArgs() - for j := 0; j < 3; j++ { + for j := range 3 { curData := fmt.Sprintf("%v", rawArgs[j]) require.Equal(t, cases[i].args[j], curData) } diff --git a/dm/unit/unit.go b/dm/unit/unit.go index 5198109371..cb5e7b2124 100644 --- a/dm/unit/unit.go +++ b/dm/unit/unit.go @@ -62,7 +62,7 @@ type Unit interface { // Status returns the unit's current status. The result may need calculation with source status, like estimated time // to catch up. If sourceStatus is nil, the calculation should be skipped. - Status(sourceStatus *binlog.SourceStatus) interface{} + Status(sourceStatus *binlog.SourceStatus) any // Type returns the unit's type Type() pb.UnitType // IsFreshTask return whether is a fresh task (not processed before) diff --git a/dm/worker/join.go b/dm/worker/join.go index 476e007723..582f2e92a3 100644 --- a/dm/worker/join.go +++ b/dm/worker/join.go @@ -66,7 +66,7 @@ func (s *Server) JoinMaster(endpoints []string) error { var errorStr string // retry to connect master - for i := 0; i < retryTimes; i++ { + for i := range retryTimes { for _, endpoint := range endpoints { ctx1, cancel1 := context.WithTimeout(ctx, 3*time.Second) //nolint:staticcheck diff --git a/dm/worker/relay.go b/dm/worker/relay.go index 5515368ff0..ecae2fde20 100644 --- a/dm/worker/relay.go +++ b/dm/worker/relay.go @@ -108,11 +108,9 @@ func (h *realRelayHolder) Init(ctx context.Context, interceptors []relay.PurgeIn // Start starts run the relay. func (h *realRelayHolder) Start() { - h.wg.Add(1) - go func() { - defer h.wg.Done() + h.wg.Go(func() { h.run() - }() + }) } // Close closes the holder. @@ -213,11 +211,9 @@ func (h *realRelayHolder) resumeRelay(_ context.Context, op pb.RelayOp) error { return terror.ErrWorkerRelayStageNotValid.Generate(h.stage, pb.Stage_Paused, op) } - h.wg.Add(1) - go func() { - defer h.wg.Done() + h.wg.Go(func() { h.run() - }() + }) return nil } diff --git a/dm/worker/relay_test.go b/dm/worker/relay_test.go index 88232e42cf..5f50af4f0b 100644 --- a/dm/worker/relay_test.go +++ b/dm/worker/relay_test.go @@ -113,12 +113,12 @@ func (d *DummyRelay) Resume(ctx context.Context, pr chan pb.ProcessResult) {} func (d *DummyRelay) Pause() {} // Error implements Process interface. -func (d *DummyRelay) Error() interface{} { +func (d *DummyRelay) Error() any { return d.errorInfo } // Status implements Process interface. -func (d *DummyRelay) Status(sourceStatus *binlog.SourceStatus) interface{} { +func (d *DummyRelay) Status(sourceStatus *binlog.SourceStatus) any { return &pb.RelayStatus{ Stage: pb.Stage_New, } diff --git a/dm/worker/server.go b/dm/worker/server.go index 5d3f813fa6..5c70928fba 100644 --- a/dm/worker/server.go +++ b/dm/worker/server.go @@ -164,11 +164,9 @@ func (s *Server) Start() error { s.setWorker(nil, true) - s.runWg.Add(1) - go func() { + s.runWg.Go(func() { s.runBackgroundJob(s.runCtx) - s.runWg.Done() - }() + }) s.startKeepAlive() diff --git a/dm/worker/server_test.go b/dm/worker/server_test.go index 8740df6c79..3cfc8add91 100644 --- a/dm/worker/server_test.go +++ b/dm/worker/server_test.go @@ -494,11 +494,9 @@ func (t *testServer) TestWatchSourceBoundEtcdCompact(c *check.C) { // step 4: watch source bound from startRev var wg sync.WaitGroup ctx1, cancel1 := context.WithCancel(ctx) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { c.Assert(s.observeSourceBound(ctx1, startRev), check.IsNil) - }() + }) // step 4.1: should stop the running worker, source bound has been deleted, should stop this worker c.Assert(utils.WaitSomething(20, 100*time.Millisecond, func() bool { return s.getSourceWorker(true) == nil @@ -516,11 +514,9 @@ func (t *testServer) TestWatchSourceBoundEtcdCompact(c *check.C) { c.Assert(s.stopSourceWorker(sourceCfg.SourceID, true, true), check.IsNil) // step 5: start observeSourceBound from compacted revision again, should start worker ctx2, cancel2 := context.WithCancel(ctx) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { c.Assert(s.observeSourceBound(ctx2, startRev), check.IsNil) - }() + }) c.Assert(utils.WaitSomething(30, 100*time.Millisecond, func() bool { return s.getSourceWorker(true) != nil }), check.IsTrue) @@ -751,7 +747,7 @@ func (t *testServer) TestServerDataRace(c *check.C) { defer s.Close() var wg sync.WaitGroup - for i := 0; i < 20; i++ { + for range 20 { wg.Add(2) go func() { defer wg.Done() diff --git a/dm/worker/source_worker.go b/dm/worker/source_worker.go index 4af53f8ca7..9436edd9ba 100644 --- a/dm/worker/source_worker.go +++ b/dm/worker/source_worker.go @@ -466,13 +466,11 @@ func (w *SourceWorker) EnableRelay(startBySourceCfg bool) (err error) { } // 4. watch relay stage - w.relayWg.Add(1) - go func() { - defer w.relayWg.Done() + w.relayWg.Go(func() { // TODO: handle fatal error from observeRelayStage //nolint:errcheck w.observeRelayStage(w.relayCtx, w.etcdClient, revRelay) - }() + }) w.relayEnabled.Store(true) w.l.Info("relay enabled") @@ -566,20 +564,16 @@ func (w *SourceWorker) EnableHandleSubtasks() error { } } - w.subTaskWg.Add(1) - go func() { - defer w.subTaskWg.Done() + w.subTaskWg.Go(func() { // TODO: handle fatal error from observeSubtaskStage //nolint:errcheck w.observeSubtaskStage(w.subTaskCtx, w.etcdClient, revSubTask) - }() - w.subTaskWg.Add(1) - go func() { - defer w.subTaskWg.Done() + }) + w.subTaskWg.Go(func() { // TODO: handle fatal error from observeValidatorStage //nolint:errcheck w.observeValidatorStage(w.subTaskCtx, revSubTask) - }() + }) w.subTaskEnabled.Store(true) w.l.Info("handling subtask enabled") diff --git a/dm/worker/source_worker_test.go b/dm/worker/source_worker_test.go index fe6ad63dce..bca1c341e3 100644 --- a/dm/worker/source_worker_test.go +++ b/dm/worker/source_worker_test.go @@ -448,11 +448,9 @@ func (t *testWorkerEtcdCompact) TestWatchSubtaskStageEtcdCompact(c *check.C) { c.Assert(w.subTaskHolder.findSubTask(subtaskCfg.Name), check.NotNil) var wg sync.WaitGroup ctx1, cancel1 := context.WithCancel(ctx) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { c.Assert(w.observeSubtaskStage(ctx1, etcdCli, startRev), check.IsNil) - }() + }) time.Sleep(time.Second) // step 4.1: after observe, invalid subtask should be removed c.Assert(utils.WaitSomething(30, 100*time.Millisecond, func() bool { @@ -476,11 +474,9 @@ func (t *testWorkerEtcdCompact) TestWatchSubtaskStageEtcdCompact(c *check.C) { w.subTaskHolder.closeAllSubTasks() // step 5: restart observe and start from startRev, this subtask should be added ctx2, cancel2 := context.WithCancel(ctx) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { c.Assert(w.observeSubtaskStage(ctx2, etcdCli, startRev), check.IsNil) - }() + }) time.Sleep(time.Second) c.Assert(utils.WaitSomething(30, 100*time.Millisecond, func() bool { return w.subTaskHolder.findSubTask(subtaskCfg.Name) != nil @@ -584,11 +580,9 @@ func (t *testWorkerEtcdCompact) TestWatchValidatorStageEtcdCompact(c *check.C) { c.Assert(getValidator(), check.IsNil) var wg sync.WaitGroup ctx1, cancel1 := context.WithCancel(ctx) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { c.Assert(w.observeValidatorStage(ctx1, startRev), check.IsNil) - }() + }) time.Sleep(time.Second) subtaskCfg.ValidatorCfg = config.ValidatorConfig{Mode: config.ValidationFast} diff --git a/dm/worker/status.go b/dm/worker/status.go index cfb83c1ac8..6b70fce81e 100644 --- a/dm/worker/status.go +++ b/dm/worker/status.go @@ -25,7 +25,7 @@ import ( ) // Status returns the status of the current sub task. -func (st *SubTask) Status() interface{} { +func (st *SubTask) Status() any { if cu := st.CurrUnit(); cu != nil { return cu.Status(nil) } diff --git a/dm/worker/subtask.go b/dm/worker/subtask.go index 971e04c4f5..56644bdb65 100644 --- a/dm/worker/subtask.go +++ b/dm/worker/subtask.go @@ -15,6 +15,7 @@ package worker import ( "context" + "slices" "sync" "time" @@ -165,7 +166,7 @@ func (st *SubTask) initUnits(relay relay.Process) error { if err != nil { initializeUnitSuccess = false // when init fail, other units initialized before should be closed - for j := 0; j < i; j++ { + for j := range i { needCloseUnits = append(needCloseUnits, st.units[j]) } return terror.Annotatef(err, "fail to initialize unit %s of subtask %s ", u.Type(), st.cfg.Name) @@ -491,10 +492,8 @@ func (st *SubTask) stageCAS(oldStage, newStage pb.Stage) bool { func (st *SubTask) setStageIfNotIn(oldStages []pb.Stage, newStage pb.Stage) bool { st.Lock() defer st.Unlock() - for _, s := range oldStages { - if st.stage == s { - return false - } + if slices.Contains(oldStages, st.stage) { + return false } st.stage = newStage updateTaskMetric(st.cfg.Name, st.cfg.SourceID, st.stage, st.workerName) @@ -505,12 +504,10 @@ func (st *SubTask) setStageIfNotIn(oldStages []pb.Stage, newStage pb.Stage) bool func (st *SubTask) setStageIfIn(oldStages []pb.Stage, newStage pb.Stage) bool { st.Lock() defer st.Unlock() - for _, s := range oldStages { - if st.stage == s { - st.stage = newStage - updateTaskMetric(st.cfg.Name, st.cfg.SourceID, st.stage, st.workerName) - return true - } + if slices.Contains(oldStages, st.stage) { + st.stage = newStage + updateTaskMetric(st.cfg.Name, st.cfg.SourceID, st.stage, st.workerName) + return true } return false } diff --git a/dm/worker/subtask_holder.go b/dm/worker/subtask_holder.go index 7781216c11..7fb82aeac7 100644 --- a/dm/worker/subtask_holder.go +++ b/dm/worker/subtask_holder.go @@ -15,6 +15,7 @@ package worker import ( "context" + "maps" "sync" "github.com/pingcap/tiflow/dm/relay" @@ -105,8 +106,6 @@ func (h *subTaskHolder) getAllSubTasks() map[string]*SubTask { h.mu.RLock() defer h.mu.RUnlock() result := make(map[string]*SubTask, len(h.subTasks)) - for name, st := range h.subTasks { - result[name] = st - } + maps.Copy(result, h.subTasks) return result } diff --git a/dm/worker/subtask_test.go b/dm/worker/subtask_test.go index cc1f2e0ef5..3a6fed0ff6 100644 --- a/dm/worker/subtask_test.go +++ b/dm/worker/subtask_test.go @@ -132,7 +132,7 @@ func (m *MockUnit) Update(context.Context, *config.SubTaskConfig) error { return m.errUpdate } -func (m *MockUnit) Status(_ *binlog.SourceStatus) interface{} { +func (m *MockUnit) Status(_ *binlog.SourceStatus) any { switch m.typ { case pb.UnitType_Check: return &pb.CheckStatus{} @@ -211,7 +211,7 @@ func (u *KillOrderUnit) Resume(ctx context.Context, pr chan pb.ProcessResult) { func (u *KillOrderUnit) Update(context.Context, *config.SubTaskConfig) error { return nil } -func (u *KillOrderUnit) Status(*binlog.SourceStatus) interface{} { return &pb.SyncStatus{} } +func (u *KillOrderUnit) Status(*binlog.SourceStatus) any { return &pb.SyncStatus{} } func (u *KillOrderUnit) Type() pb.UnitType { return u.typ } @@ -250,7 +250,7 @@ func (t *testSubTask) TestSubTaskNormalUsage(c *check.C) { // finish dump c.Assert(mockDumper.InjectProcessError(context.Background(), nil), check.IsNil) - for i := 0; i < 10; i++ { + for range 10 { if st.CurrUnit().Type() == pb.UnitType_Load { break } @@ -262,7 +262,7 @@ func (t *testSubTask) TestSubTaskNormalUsage(c *check.C) { // fail loader c.Assert(mockLoader.InjectProcessError(context.Background(), errors.New("loader process error")), check.IsNil) - for i := 0; i < 10; i++ { + for range 10 { res := st.Result() if res != nil && st.Stage() == pb.Stage_Paused { break @@ -325,7 +325,7 @@ func (t *testSubTask) TestSubTaskNormalUsage(c *check.C) { // finish loader c.Assert(mockLoader.InjectProcessError(context.Background(), nil), check.IsNil) - for i := 0; i < 1000; i++ { + for range 1000 { if st.Stage() == pb.Stage_Finished { break } @@ -655,7 +655,7 @@ func TestSubtaskRace(t *testing.T) { tempQueryStatusResponse.SubTaskStatus[0] = &tempSubTaskStatus st.result.IsCanceled = false go func() { - for i := 0; i < 10; i++ { + for range 10 { _, _ = tempQueryStatusResponse.Marshal() } }() diff --git a/dm/worker/task_checker.go b/dm/worker/task_checker.go index 81c874d9b6..bb8e4f8a8e 100644 --- a/dm/worker/task_checker.go +++ b/dm/worker/task_checker.go @@ -153,11 +153,9 @@ func (tsc *realTaskStatusChecker) Init() error { // Start implements TaskStatusChecker.Start. func (tsc *realTaskStatusChecker) Start() { - tsc.wg.Add(1) - go func() { - defer tsc.wg.Done() + tsc.wg.Go(func() { tsc.run() - }() + }) } // Close implements TaskStatusChecker.Close. diff --git a/dm/worker/task_checker_test.go b/dm/worker/task_checker_test.go index 867d54f7e2..abb0c97265 100644 --- a/dm/worker/task_checker_test.go +++ b/dm/worker/task_checker_test.go @@ -186,7 +186,7 @@ func TestCheck(t *testing.T) { latestResumeTime = rtsc.subtaskAutoResume[taskName].LatestResumeTime latestPausedTime = rtsc.subtaskAutoResume[taskName].LatestPausedTime require.Equal(t, 10*time.Second, bf.Current()) - for i := 0; i < 10; i++ { + for range 10 { rtsc.check() require.Equal(t, latestResumeTime, rtsc.subtaskAutoResume[taskName].LatestResumeTime) require.True(t, latestPausedTime.Before(rtsc.subtaskAutoResume[taskName].LatestPausedTime)) @@ -269,7 +269,7 @@ func TestCheckTaskIndependent(t *testing.T) { task1LatestResumeTime = rtsc.subtaskAutoResume[task1].LatestResumeTime task2LatestResumeTime = rtsc.subtaskAutoResume[task2].LatestResumeTime - for i := 0; i < 10; i++ { + for range 10 { time.Sleep(backoffMin) rtsc.check() require.Equal(t, task1LatestResumeTime, rtsc.subtaskAutoResume[task1].LatestResumeTime) From 7b14ec0c2e0f612866e59944e0f9080955c5ed50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Eeden?= Date: Tue, 16 Jun 2026 09:33:36 +0200 Subject: [PATCH 2/3] make fmt --- dm/config/subtask_test.go | 2 -- dm/ctl/master/operate_task.go | 1 - 2 files changed, 3 deletions(-) diff --git a/dm/config/subtask_test.go b/dm/config/subtask_test.go index 5d40d6e4e6..3efb320be0 100644 --- a/dm/config/subtask_test.go +++ b/dm/config/subtask_test.go @@ -464,7 +464,6 @@ func TestSubTaskConfigMarshalAtomic(t *testing.T) { var wg sync.WaitGroup for range 10 { wg.Go(func() { - data, err := json.Marshal(cfg) require.NoError(t, err) jsonMap := make(map[string]any) @@ -488,7 +487,6 @@ func TestSubTaskConfigMarshalAtomic(t *testing.T) { }) wg.Go(func() { - newCfg, err := cfg.Clone() require.NoError(t, err) diff --git a/dm/ctl/master/operate_task.go b/dm/ctl/master/operate_task.go index e72e62abd4..7a97bbadf9 100644 --- a/dm/ctl/master/operate_task.go +++ b/dm/ctl/master/operate_task.go @@ -127,7 +127,6 @@ func batchOperateTask(taskOp pb.TaskOp, batchSize int, sources []string, subTask resultCh := make(chan *operateTaskResult, 1) for i := 0; i < batchSize; i++ { wg.Go(func() { - for name := range workCh { taskResult := operateTaskResult{Task: name, Op: taskOp.String()} taskOpResp, err := common.OperateTask(taskOp, name, sources) From d2d746b65b49c267aeb9e7b3b7f230e59ab99db0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Eeden?= Date: Thu, 25 Jun 2026 09:40:42 +0200 Subject: [PATCH 3/3] Update based on review --- dm/config/checking_item.go | 3 +-- dm/config/checking_item_test.go | 3 +-- dm/config/dbconfig/config.go | 5 +---- dm/master/server.go | 5 +---- dm/simulator/mcp/uk.go | 19 ++++++------------- dm/syncer/sharding_group.go | 8 ++------ dm/worker/subtask_holder.go | 4 +--- 7 files changed, 13 insertions(+), 34 deletions(-) diff --git a/dm/config/checking_item.go b/dm/config/checking_item.go index f2821ceb16..44670cf591 100644 --- a/dm/config/checking_item.go +++ b/dm/config/checking_item.go @@ -130,8 +130,7 @@ func SupportCheckingItems() string { // FilterCheckingItems filters ignored items from all checking items. func FilterCheckingItems(ignoredItems []string) map[string]string { - checkingItems := make(map[string]string) - maps.Copy(checkingItems, AllCheckingItems) + checkingItems := maps.Clone(AllCheckingItems) delete(checkingItems, AllChecking) for _, item := range ignoredItems { diff --git a/dm/config/checking_item_test.go b/dm/config/checking_item_test.go index 2eb6dab2b8..1786b349ad 100644 --- a/dm/config/checking_item_test.go +++ b/dm/config/checking_item_test.go @@ -45,8 +45,7 @@ func TestCheckingItems(t *testing.T) { require.Empty(t, FilterCheckingItems(ignoredCheckingItems)) // ignore shard checking items - checkingItems := make(map[string]string) - maps.Copy(checkingItems, AllCheckingItems) + checkingItems := maps.Clone(AllCheckingItems) delete(checkingItems, AllChecking) require.Equal(t, checkingItems, FilterCheckingItems(ignoredCheckingItems[:0])) diff --git a/dm/config/dbconfig/config.go b/dm/config/dbconfig/config.go index 64c76ec420..bff9220d78 100644 --- a/dm/config/dbconfig/config.go +++ b/dm/config/dbconfig/config.go @@ -149,10 +149,7 @@ func (db *DBConfig) Clone() *DBConfig { clone.MaxAllowedPacket = &packet } - if db.Session != nil { - clone.Session = make(map[string]string, len(db.Session)) - maps.Copy(clone.Session, db.Session) - } + clone.Session = maps.Clone(db.Session) clone.Security = db.Security.Clone() diff --git a/dm/master/server.go b/dm/master/server.go index 2d8cb6590f..35d81da8bd 100644 --- a/dm/master/server.go +++ b/dm/master/server.go @@ -839,13 +839,10 @@ func (s *Server) QueryStatus(ctx context.Context, req *pb.QueryStatusListRequest } resps := s.getStatusFromWorkers(ctx, sources, req.Name, specifiedSource) workerRespMap := make(map[string][]*pb.QueryStatusResponse, len(sources)) // sourceName -> worker QueryStatusResponse - inSlice := func(s []string, e string) bool { - return slices.Contains(s, e) - } for _, workerResp := range resps { workerRespMap[workerResp.SourceStatus.Source] = append(workerRespMap[workerResp.SourceStatus.Source], workerResp) // append some offline worker responses - if !inSlice(sources, workerResp.SourceStatus.Source) { + if !slices.Contains(sources, workerResp.SourceStatus.Source) { sources = append(sources, workerResp.SourceStatus.Source) } } diff --git a/dm/simulator/mcp/uk.go b/dm/simulator/mcp/uk.go index ac3ba3e17e..56afb9af21 100644 --- a/dm/simulator/mcp/uk.go +++ b/dm/simulator/mcp/uk.go @@ -38,12 +38,10 @@ type UniqueKey struct { // the map values are cloned into the new UK instance, // so that the further changes in the value map won't affect the values inside the UK. func NewUniqueKey(rowID int, value map[string]any) *UniqueKey { - result := &UniqueKey{ + return &UniqueKey{ rowID: rowID, - value: make(map[string]any), + value: maps.Clone(value), } - maps.Copy(result.value, value) - return result } // GetRowID gets the row ID of the unique key. @@ -67,9 +65,7 @@ func (uk *UniqueKey) SetRowID(rowID int) { func (uk *UniqueKey) GetValue() map[string]any { uk.RLock() defer uk.RUnlock() - result := make(map[string]any) - maps.Copy(result, uk.value) - return result + return maps.Clone(uk.value) } // GetValueHash return hash for values. @@ -96,8 +92,7 @@ func (uk *UniqueKey) GetValueHash() string { func (uk *UniqueKey) SetValue(value map[string]any) { uk.Lock() defer uk.Unlock() - uk.value = make(map[string]any) - maps.Copy(uk.value, value) + uk.value = maps.Clone(value) } // Clone is to clone a UK into a new one. @@ -105,12 +100,10 @@ func (uk *UniqueKey) SetValue(value map[string]any) { func (uk *UniqueKey) Clone() *UniqueKey { uk.RLock() defer uk.RUnlock() - result := &UniqueKey{ + return &UniqueKey{ rowID: uk.rowID, - value: map[string]any{}, + value: maps.Clone(uk.value), } - maps.Copy(result.value, uk.value) - return result } // String returns the string representation of a UK. diff --git a/dm/syncer/sharding_group.go b/dm/syncer/sharding_group.go index d1e3d794ad..f9a30fb3ca 100644 --- a/dm/syncer/sharding_group.go +++ b/dm/syncer/sharding_group.go @@ -279,9 +279,7 @@ func (sg *ShardingGroup) UnresolvedGroupInfo() *pb.ShardingGroup { func (sg *ShardingGroup) Sources() map[string]bool { sg.RLock() defer sg.RUnlock() - ret := make(map[string]bool, len(sg.sources)) - maps.Copy(ret, sg.sources) - return ret + return maps.Clone(sg.sources) } // Tables returns all source tables' pair. @@ -617,9 +615,7 @@ func (k *ShardingGroupKeeper) Groups() map[string]*ShardingGroup { defer k.RUnlock() // do a copy - groups := make(map[string]*ShardingGroup, len(k.groups)) - maps.Copy(groups, k.groups) - return groups + return maps.Clone(k.groups) } // UnresolvedGroups returns sharding groups which are un-resolved diff --git a/dm/worker/subtask_holder.go b/dm/worker/subtask_holder.go index 7fb82aeac7..775579e95d 100644 --- a/dm/worker/subtask_holder.go +++ b/dm/worker/subtask_holder.go @@ -105,7 +105,5 @@ func (h *subTaskHolder) findSubTask(name string) *SubTask { func (h *subTaskHolder) getAllSubTasks() map[string]*SubTask { h.mu.RLock() defer h.mu.RUnlock() - result := make(map[string]*SubTask, len(h.subTasks)) - maps.Copy(result, h.subTasks) - return result + return maps.Clone(h.subTasks) }