From 97267de8021ed2d9f7033ed0c03c8a93755649ba Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 19 May 2026 17:02:48 +0200 Subject: [PATCH 01/11] Optimize split-aware parallelism planning --- README.md | 11 +- internal/runner/distribution.go | 54 +----- internal/runner/parallelism.go | 46 ++++-- internal/runner/parallelism_test.go | 246 ++++++++++------------------ internal/runner/runner.go | 2 +- internal/runner/runner_test.go | 69 +++++++- internal/runner/split.go | 128 +++++++++++++++ 7 files changed, 337 insertions(+), 219 deletions(-) create mode 100644 internal/runner/split.go diff --git a/README.md b/README.md index 230dbf2..7cf8e13 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,13 @@ The folder contents are: config # (GHA only) matrix JSON with ci_node_index entries ``` +DDTest chooses parallelism by estimating the runnable duration of each test file, +then trying worker counts between `--min-parallelism` and `--max-parallelism`. +Duration estimates come from Datadog test suite p50 timings when available and +fall back to local discovery weights otherwise. DDTest chooses the split with +the lowest slowest-worker time, then the most even load, so it avoids launching +workers that would sit idle or would not shorten the run. + DDTest may also write compatibility files at legacy root paths for existing integrations. New integrations should read runner files from `.testoptimization/runner/*`. You can use `runner/test-files.txt` or `runner/tests-split/runner-X` files to feed @@ -158,8 +165,8 @@ DDTest automatically sets `DD_TEST_SESSION_NAME` for each worker to ` files[j].count - }) - - // loads tracks current test duration assigned to each bin - loads := make([]int, parallelRunners) // result tracks files assigned to each bin (can be returned directly) result := make([][]string, parallelRunners) for i := range result { result[i] = []string{} } - // First Fit Decreasing algorithm for bin packing - // On each step take the file in decreasing order of load - // and put it into the bin with minimum load - // - // Time complexity is N * M where - // N - number of bins (estimated about 10^2) - // M - number of test files (estimated about 10^4) - for _, file := range files { - minBin := 0 - for i := 1; i < len(loads); i++ { - if loads[i] < loads[minBin] { - minBin = i - } - } - - loads[minBin] += file.count - result[minBin] = append(result[minBin], file.path) - } - + scheduleSortedFiles(files, parallelRunners, result) return result } // CreateTestSplits creates test split files for parallel runners -// For multiple runners: distributes files using bin packing and writes to separate runner files +// For multiple runners: distributes files using weighted list scheduling and writes to separate runner files // For single runner: copies test-files.txt content to runner-0 func CreateTestSplits(testFiles map[string]int, parallelRunners int, testFilesOutputPath string) error { testsSplitDirs := []string{constants.TestsSplitDir, constants.LegacyTestsSplitDir} if parallelRunners > 1 { - // Distribute test files across parallel runners using bin packing + // Distribute test files across parallel runners using weighted list scheduling. distribution := DistributeTestFiles(testFiles, parallelRunners) for _, testsSplitDir := range testsSplitDirs { if err := writeDistributedTestSplits(distribution, testsSplitDir); err != nil { diff --git a/internal/runner/parallelism.go b/internal/runner/parallelism.go index f32fc07..15a0910 100644 --- a/internal/runner/parallelism.go +++ b/internal/runner/parallelism.go @@ -2,18 +2,17 @@ package runner import ( "log/slog" - "math" "github.com/DataDog/ddtest/internal/settings" ) -// calculateParallelRunners determines the number of parallel runners based on skippable percentage -// and parallelism configuration -func calculateParallelRunners(skippablePercentage float64) int { - return calculateParallelRunnersWithParams(skippablePercentage, settings.GetMinParallelism(), settings.GetMaxParallelism()) +// calculateParallelRunners determines the number of parallel runners by +// estimating splits between the configured min and max parallelism. +func calculateParallelRunners(testFileWeights map[string]int) int { + return calculateParallelRunnersWithParams(testFileWeights, settings.GetMinParallelism(), settings.GetMaxParallelism()) } -func calculateParallelRunnersWithParams(skippablePercentage float64, minParallelism, maxParallelism int) int { +func calculateParallelRunnersWithParams(testFileWeights map[string]int, minParallelism, maxParallelism int) int { // maxParallelism could be 0 or negative! if maxParallelism <= 1 { return 1 @@ -21,7 +20,7 @@ func calculateParallelRunnersWithParams(skippablePercentage float64, minParallel if minParallelism < 1 { slog.Warn("min_parallelism is less than 1, setting to 1", "min_parallelism", minParallelism) - return 1 + minParallelism = 1 } if maxParallelism < minParallelism { @@ -30,8 +29,35 @@ func calculateParallelRunnersWithParams(skippablePercentage float64, minParallel minParallelism = maxParallelism } - percentage := math.Max(0.0, math.Min(100.0, skippablePercentage)) // Clamp to [0, 100] - runners := float64(maxParallelism) - (percentage/100.0)*float64(maxParallelism-minParallelism) + files := sortedWeightedTestFiles(testFileWeights) + if len(files) == 0 { + return minParallelism + } + + candidateMax := maxUsefulParallelism(minParallelism, maxParallelism, len(files)) + + best := scoreSortedSplit(files, minParallelism) + for parallelRunners := minParallelism + 1; parallelRunners <= candidateMax; parallelRunners++ { + score := scoreSortedSplit(files, parallelRunners) + if betterSplit(score, best) { + best = score + } + } + + return best.parallelRunners +} + +func maxUsefulParallelism(minParallelism, maxParallelism, filesCount int) int { + if filesCount < minParallelism { + return minParallelism + } + if filesCount < maxParallelism { + return filesCount + } + return maxParallelism +} - return int(math.Round(runners)) +func betterSplit(candidate, currentBest splitScore) bool { + return candidate.wallTime < currentBest.wallTime || + (candidate.wallTime == currentBest.wallTime && candidate.imbalance < currentBest.imbalance) } diff --git a/internal/runner/parallelism_test.go b/internal/runner/parallelism_test.go index 645122f..058c09d 100644 --- a/internal/runner/parallelism_test.go +++ b/internal/runner/parallelism_test.go @@ -1,192 +1,126 @@ package runner -import "testing" +import ( + "fmt" + "testing" +) -// Helper function to run calculateParallelRunnersWithParams tests -func testCalculateParallelRunners(skippablePercentage float64, minParallelism, maxParallelism int) int { - return calculateParallelRunnersWithParams(skippablePercentage, minParallelism, maxParallelism) +func testCalculateParallelRunners(testFileWeights map[string]int, minParallelism, maxParallelism int) int { + return calculateParallelRunnersWithParams(testFileWeights, minParallelism, maxParallelism) } func TestCalculateParallelRunners_MaxParallelismIsOne(t *testing.T) { - tests := []struct { - name string - skippablePercentage float64 - expected int - }{ - {"0% skippable", 0.0, 1}, - {"50% skippable", 50.0, 1}, - {"100% skippable", 100.0, 1}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := testCalculateParallelRunners(tt.skippablePercentage, 1, 1) - if result != tt.expected { - t.Errorf("calculateParallelRunners(%f) = %d, expected %d", tt.skippablePercentage, result, tt.expected) + testFileWeights := map[string]int{ + "test1.rb": 10, + "test2.rb": 10, + } + + for _, maxParallelism := range []int{1, 0, -1, -100} { + t.Run(fmt.Sprintf("maxParallelism=%d", maxParallelism), func(t *testing.T) { + result := testCalculateParallelRunners(testFileWeights, 1, maxParallelism) + if result != 1 { + t.Errorf("calculateParallelRunners() with maxParallelism=%d = %d, expected 1", maxParallelism, result) } }) } } -func TestCalculateParallelRunners_MaxParallelismZeroOrNegative(t *testing.T) { - tests := []struct { - name string - maxParallelism int - }{ - {"maxParallelism is 0", 0}, - {"maxParallelism is -1", -1}, - {"maxParallelism is -100", -100}, +func TestCalculateParallelRunners_MinParallelismLessThanOne(t *testing.T) { + testFileWeights := map[string]int{ + "test1.rb": 10, + "test2.rb": 10, + "test3.rb": 10, + "test4.rb": 10, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - // Should always return 1 regardless of skippable percentage - result := testCalculateParallelRunners(0.0, 1, tt.maxParallelism) - if result != 1 { - t.Errorf("calculateParallelRunners(0.0) with maxParallelism=%d = %d, expected 1", tt.maxParallelism, result) - } + result := testCalculateParallelRunners(testFileWeights, 0, 4) + if result != 4 { + t.Errorf("calculateParallelRunners() = %d, expected 4 when min_parallelism is normalized to 1", result) + } +} - result = testCalculateParallelRunners(50.0, 1, tt.maxParallelism) - if result != 1 { - t.Errorf("calculateParallelRunners(50.0) with maxParallelism=%d = %d, expected 1", tt.maxParallelism, result) - } +func TestCalculateParallelRunners_MaxLessThanMin(t *testing.T) { + testFileWeights := map[string]int{ + "test1.rb": 10, + "test2.rb": 10, + "test3.rb": 10, + "test4.rb": 10, + } - result = testCalculateParallelRunners(100.0, 1, tt.maxParallelism) - if result != 1 { - t.Errorf("calculateParallelRunners(100.0) with maxParallelism=%d = %d, expected 1", tt.maxParallelism, result) - } - }) + result := testCalculateParallelRunners(testFileWeights, 5, 3) + if result != 3 { + t.Errorf("calculateParallelRunners() = %d, expected 3 when max_parallelism is clamped", result) } } -func TestCalculateParallelRunners_MinParallelismLessThanOne(t *testing.T) { - tests := []struct { - name string - skippablePercentage float64 - expected int - }{ - {"0% skippable with min<1", 0.0, 1}, - {"50% skippable with min<1", 50.0, 1}, - {"100% skippable with min<1", 100.0, 1}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := testCalculateParallelRunners(tt.skippablePercentage, 0, 5) - if result != tt.expected { - t.Errorf("calculateParallelRunners(%f) = %d, expected %d", tt.skippablePercentage, result, tt.expected) - } - }) +func TestCalculateParallelRunners_MinEqualsMax(t *testing.T) { + testFileWeights := map[string]int{ + "test1.rb": 10, + "test2.rb": 10, + "test3.rb": 10, + } + + result := testCalculateParallelRunners(testFileWeights, 4, 4) + if result != 4 { + t.Errorf("calculateParallelRunners() = %d, expected 4 when min and max match", result) } } -func TestCalculateParallelRunners_MaxLessThanMin(t *testing.T) { - // When max < min, min is clamped to max. This ensures that a user who only - // sets --max-parallelism to a lower value gets the expected behavior. - result := testCalculateParallelRunners(50.0, 5, 3) // max < min - expected := 3 // Should clamp min to max and return max - if result != expected { - t.Errorf("calculateParallelRunners(50.0) = %d, expected %d when max < min", result, expected) +func TestCalculateParallelRunners_EmptyTestFiles(t *testing.T) { + result := testCalculateParallelRunners(map[string]int{}, 2, 8) + if result != 2 { + t.Errorf("calculateParallelRunners() = %d, expected normalized min parallelism 2 for empty tests", result) } } -func TestCalculateParallelRunners_LinearInterpolation(t *testing.T) { - tests := []struct { - name string - skippablePercentage float64 - expected int - }{ - {"0% skippable -> max parallelism", 0.0, 8}, - {"25% skippable", 25.0, 7}, // 8 - 0.25 * (8-2) = 8 - 1.5 = 6.5 -> 7 - {"50% skippable", 50.0, 5}, // 8 - 0.5 * (8-2) = 8 - 3 = 5 - {"75% skippable", 75.0, 4}, // 8 - 0.75 * (8-2) = 8 - 4.5 = 3.5 -> 4 - {"100% skippable -> min parallelism", 100.0, 2}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := testCalculateParallelRunners(tt.skippablePercentage, 2, 8) - if result != tt.expected { - t.Errorf("calculateParallelRunners(%f) = %d, expected %d", tt.skippablePercentage, result, tt.expected) - } - }) +func TestCalculateParallelRunners_WallTimeWins(t *testing.T) { + testFileWeights := map[string]int{ + "test1.rb": 10, + "test2.rb": 10, + "test3.rb": 10, + "test4.rb": 10, + } + + result := testCalculateParallelRunners(testFileWeights, 1, 4) + if result != 4 { + t.Errorf("calculateParallelRunners() = %d, expected 4 to minimize slowest runner time", result) } } -func TestCalculateParallelRunners_EdgeCases(t *testing.T) { - tests := []struct { - name string - skippablePercentage float64 - expected int - }{ - {"Negative percentage", -10.0, 10}, // Should clamp to 0% - {"Over 100%", 150.0, 3}, // Should clamp to 100% - {"Exact boundary 0%", 0.0, 10}, - {"Exact boundary 100%", 100.0, 3}, - {"Fractional result rounds", 33.33, 8}, // 10 - 0.3333 * 7 = 7.67 -> 8 - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := testCalculateParallelRunners(tt.skippablePercentage, 3, 10) - if result != tt.expected { - t.Errorf("calculateParallelRunners(%f) = %d, expected %d", tt.skippablePercentage, result, tt.expected) - } - }) +func TestCalculateParallelRunners_ImbalanceBreaksWallTimeTie(t *testing.T) { + testFileWeights := map[string]int{ + "test1.rb": 10, + "test2.rb": 6, + "test3.rb": 6, + "test4.rb": 6, + "test5.rb": 6, + } + + result := testCalculateParallelRunners(testFileWeights, 3, 4) + if result != 3 { + t.Errorf("calculateParallelRunners() = %d, expected 3 because it keeps the same wall time with lower imbalance", result) } } -func TestCalculateParallelRunners_MinEqualsMax(t *testing.T) { - tests := []struct { - name string - skippablePercentage float64 - expected int - }{ - {"0% skippable", 0.0, 4}, - {"50% skippable", 50.0, 4}, - {"100% skippable", 100.0, 4}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := testCalculateParallelRunners(tt.skippablePercentage, 4, 4) - if result != tt.expected { - t.Errorf("calculateParallelRunners(%f) = %d, expected %d", tt.skippablePercentage, result, tt.expected) - } - }) +func TestCalculateParallelRunners_OverParallelizedInputsPreserveMinimum(t *testing.T) { + testFileWeights := map[string]int{ + "test1.rb": 10, + } + + result := testCalculateParallelRunners(testFileWeights, 2, 20) + if result != 2 { + t.Errorf("calculateParallelRunners() = %d, expected 2 because min_parallelism is unavoidable", result) } } -func TestCalculateParallelRunners_RealWorldScenarios(t *testing.T) { - tests := []struct { - name string - minParallelism int - maxParallelism int - skippablePercentage float64 - expected int - description string - }{ - {"Small project", 1, 4, 25.0, 3, "25% skippable in small project"}, - {"Medium project", 2, 12, 60.0, 6, "60% skippable in medium project"}, - {"Large project", 4, 32, 80.0, 10, "80% skippable in large project"}, - {"CI with high parallelism", 8, 64, 90.0, 14, "90% skippable with high parallelism"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := testCalculateParallelRunners(tt.skippablePercentage, tt.minParallelism, tt.maxParallelism) - if result != tt.expected { - t.Errorf("%s: calculateParallelRunners(%f) = %d, expected %d", - tt.description, tt.skippablePercentage, result, tt.expected) - } +func BenchmarkCalculateParallelRunners20000TestFiles(b *testing.B) { + testFileWeights := make(map[string]int, 20000) + for i := range 20000 { + testFileWeights[fmt.Sprintf("test/%05d_test.rb", i)] = (i % 1000) + 1 + } - // Verify result is within bounds - if result < tt.minParallelism { - t.Errorf("%s: result %d is less than min_parallelism %d", tt.description, result, tt.minParallelism) - } - if result > tt.maxParallelism { - t.Errorf("%s: result %d is greater than max_parallelism %d", tt.description, result, tt.maxParallelism) - } - }) + b.ResetTimer() + for range b.N { + _ = calculateParallelRunnersWithParams(testFileWeights, 1, 256) } } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index eeffac3..58980d8 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -105,7 +105,7 @@ func (tr *TestRunner) Plan(ctx context.Context) error { } // Calculate and write parallel runners count - parallelRunners := calculateParallelRunners(tr.skippablePercentage) + parallelRunners := calculateParallelRunners(tr.testFileWeights) runnersContent := fmt.Sprintf("%d", parallelRunners) if err := writePlanFileCopies([]byte(runnersContent), constants.ParallelRunnersOutputPath, constants.LegacyParallelRunnersOutputPath); err != nil { return fmt.Errorf("failed to write parallel runners: %w", err) diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 4692619..ffa5d3e 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -421,6 +421,64 @@ func TestTestRunner_Plan_WritesManifestAndRunnerLayout(t *testing.T) { assertFileContent(t, filepath.Join(constants.LegacyTestsSplitDir, "runner-0"), expectedTestFiles) } +func TestTestRunner_Plan_ChoosesParallelismFromSplitNotSkippablePercentage(t *testing.T) { + tempDir := t.TempDir() + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "2") + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "4") + defer func() { + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") + }() + settings.Init() + + var tests []testoptimization.Test + skippableTests := map[string]bool{} + for suiteIndex := range 4 { + suite := fmt.Sprintf("TestSuite%d", suiteIndex) + sourceFile := fmt.Sprintf("test/file%d_test.rb", suiteIndex) + for testIndex := range 10 { + name := fmt.Sprintf("test%d", testIndex) + tests = append(tests, testoptimization.Test{ + Suite: suite, + Name: name, + Parameters: "", + SuiteSourceFile: sourceFile, + }) + if testIndex > 0 { + skippableTests[fmt.Sprintf("%s.%s.", suite, name)] = true + } + } + } + + mockFramework := &MockFramework{ + FrameworkName: "rspec", + Tests: tests, + } + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{"platform": "ruby"}, + Framework: mockFramework, + } + + runner := NewWithDependencies( + &MockPlatformDetector{Platform: mockPlatform}, + &MockTestOptimizationClient{SkippableTests: skippableTests}, + &MockTestSuiteDurationsClient{}, + newDefaultMockCIProviderDetector(), + ) + + if err := runner.Plan(context.Background()); err != nil { + t.Fatalf("Plan() should not return error, got: %v", err) + } + + assertFileContent(t, constants.SkippablePercentageOutputPath, "90.00") + assertFileContent(t, constants.ParallelRunnersOutputPath, "4") +} + func TestTestRunner_Setup_WithCIProvider(t *testing.T) { tempDir := t.TempDir() @@ -702,10 +760,11 @@ func TestTestRunner_Setup_WithTestSplit(t *testing.T) { SkippableTests: map[string]bool{}, // No tests skipped } - expectedParallelRunnersCount := 4 + expectedParallelRunnersCount := 2 + maxParallelism := 4 // Set environment variables to force multiple parallel runners _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "2") - _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", strconv.Itoa(expectedParallelRunnersCount)) + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", strconv.Itoa(maxParallelism)) defer func() { _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") @@ -727,7 +786,7 @@ func TestTestRunner_Setup_WithTestSplit(t *testing.T) { t.Error("Expected tests-split directory to be created") } - // With min=2 and 0% skippable tests, we should get 4 parallel runners + // With this split, 2 runners are as fast as 3 and more balanced. // Verify runner files exist for i := range expectedParallelRunnersCount { runnerPath := filepath.Join(constants.TestsSplitDir, fmt.Sprintf("runner-%d", i)) @@ -737,8 +796,8 @@ func TestTestRunner_Setup_WithTestSplit(t *testing.T) { } // Verify content of runner files - // With the test distribution (file1: 2 tests, file2: 1 test, file3: 1 test) - // and 4 runners, expected: Runner 0 gets file1 (2 tests), others get 1 test each + // With the test distribution (file1: 2 tests, file2: 1 test, file3: 1 test), + // expected: runner 0 gets file1 (2 tests), runner 1 gets file2+file3 (2 tests). runner0Content, err := os.ReadFile(filepath.Join(constants.TestsSplitDir, "runner-0")) if err != nil { t.Fatalf("Failed to read runner-0 file: %v", err) diff --git a/internal/runner/split.go b/internal/runner/split.go new file mode 100644 index 0000000..2c061ab --- /dev/null +++ b/internal/runner/split.go @@ -0,0 +1,128 @@ +package runner + +import ( + "container/heap" + "slices" +) + +type weightedTestFile struct { + path string + weight int +} + +type runnerLoad struct { + index int + load int +} + +type minLoadHeap []runnerLoad + +func (h minLoadHeap) Len() int { + return len(h) +} + +func (h minLoadHeap) Less(i, j int) bool { + if h[i].load == h[j].load { + return h[i].index < h[j].index + } + return h[i].load < h[j].load +} + +func (h minLoadHeap) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +func (h *minLoadHeap) Push(x any) { + *h = append(*h, x.(runnerLoad)) +} + +func (h *minLoadHeap) Pop() any { + old := *h + n := len(old) + x := old[n-1] + *h = old[:n-1] + return x +} + +type splitScore struct { + parallelRunners int + wallTime int + imbalance int +} + +func sortedWeightedTestFiles(testFiles map[string]int) []weightedTestFile { + files := make([]weightedTestFile, 0, len(testFiles)) + for path, weight := range testFiles { + files = append(files, weightedTestFile{path: path, weight: weight}) + } + + slices.SortFunc(files, func(a, b weightedTestFile) int { + if a.weight > b.weight { + return -1 + } + if a.weight < b.weight { + return 1 + } + if a.path < b.path { + return -1 + } + if a.path > b.path { + return 1 + } + return 0 + }) + + return files +} + +func scoreSortedSplit(files []weightedTestFile, parallelRunners int) splitScore { + return scheduleSortedFiles(files, parallelRunners, nil) +} + +// scheduleSortedFiles uses longest-processing-time list scheduling: assign the +// heaviest remaining file to the currently lightest runner. +func scheduleSortedFiles(files []weightedTestFile, parallelRunners int, result [][]string) splitScore { + if parallelRunners <= 0 { + parallelRunners = 1 + } + + loads := makeMinLoadHeap(parallelRunners) + for _, file := range files { + lightestRunner := heap.Pop(&loads).(runnerLoad) + lightestRunner.load += file.weight + if result != nil { + result[lightestRunner.index] = append(result[lightestRunner.index], file.path) + } + heap.Push(&loads, lightestRunner) + } + + minLoad, maxLoad := minMaxLoad(loads) + return splitScore{ + parallelRunners: parallelRunners, + wallTime: maxLoad, + imbalance: maxLoad - minLoad, + } +} + +func makeMinLoadHeap(parallelRunners int) minLoadHeap { + loads := make(minLoadHeap, parallelRunners) + for i := range loads { + loads[i] = runnerLoad{index: i} + } + heap.Init(&loads) + return loads +} + +func minMaxLoad(loads []runnerLoad) (int, int) { + minLoad := loads[0].load + maxLoad := loads[0].load + for _, load := range loads[1:] { + if load.load < minLoad { + minLoad = load.load + } + if load.load > maxLoad { + maxLoad = load.load + } + } + return minLoad, maxLoad +} From 0fc9fecf27806f912c49062ee32908d999b46762 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 19 May 2026 21:47:22 +0200 Subject: [PATCH 02/11] Address split planning review feedback --- internal/runner/distribution.go | 19 +++++-------------- internal/runner/parallelism.go | 12 ++---------- internal/runner/parallelism_test.go | 4 ++-- internal/runner/runner.go | 6 +++++- 4 files changed, 14 insertions(+), 27 deletions(-) diff --git a/internal/runner/distribution.go b/internal/runner/distribution.go index 3c0c408..ad5e1cd 100644 --- a/internal/runner/distribution.go +++ b/internal/runner/distribution.go @@ -11,29 +11,20 @@ import ( // DistributeTestFiles distributes test files across parallel runners using weighted list scheduling. func DistributeTestFiles(testFiles map[string]int, parallelRunners int) [][]string { - return distributeSortedTestFiles(sortedWeightedTestFiles(testFiles), parallelRunners) -} - -func distributeSortedTestFiles(files []weightedTestFile, parallelRunners int) [][]string { if parallelRunners <= 0 { parallelRunners = 1 } - if len(files) == 0 { - result := make([][]string, parallelRunners) - for i := range result { - result[i] = []string{} - } - return result - } - - // result tracks files assigned to each bin (can be returned directly) result := make([][]string, parallelRunners) for i := range result { result[i] = []string{} } - scheduleSortedFiles(files, parallelRunners, result) + files := sortedWeightedTestFiles(testFiles) + if len(files) > 0 { + scheduleSortedFiles(files, parallelRunners, result) + } + return result } diff --git a/internal/runner/parallelism.go b/internal/runner/parallelism.go index 15a0910..f549c76 100644 --- a/internal/runner/parallelism.go +++ b/internal/runner/parallelism.go @@ -1,18 +1,10 @@ package runner -import ( - "log/slog" - - "github.com/DataDog/ddtest/internal/settings" -) +import "log/slog" // calculateParallelRunners determines the number of parallel runners by // estimating splits between the configured min and max parallelism. -func calculateParallelRunners(testFileWeights map[string]int) int { - return calculateParallelRunnersWithParams(testFileWeights, settings.GetMinParallelism(), settings.GetMaxParallelism()) -} - -func calculateParallelRunnersWithParams(testFileWeights map[string]int, minParallelism, maxParallelism int) int { +func calculateParallelRunners(testFileWeights map[string]int, minParallelism, maxParallelism int) int { // maxParallelism could be 0 or negative! if maxParallelism <= 1 { return 1 diff --git a/internal/runner/parallelism_test.go b/internal/runner/parallelism_test.go index 058c09d..6821a3b 100644 --- a/internal/runner/parallelism_test.go +++ b/internal/runner/parallelism_test.go @@ -6,7 +6,7 @@ import ( ) func testCalculateParallelRunners(testFileWeights map[string]int, minParallelism, maxParallelism int) int { - return calculateParallelRunnersWithParams(testFileWeights, minParallelism, maxParallelism) + return calculateParallelRunners(testFileWeights, minParallelism, maxParallelism) } func TestCalculateParallelRunners_MaxParallelismIsOne(t *testing.T) { @@ -121,6 +121,6 @@ func BenchmarkCalculateParallelRunners20000TestFiles(b *testing.B) { b.ResetTimer() for range b.N { - _ = calculateParallelRunnersWithParams(testFileWeights, 1, 256) + _ = calculateParallelRunners(testFileWeights, 1, 256) } } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 58980d8..631e739 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -105,7 +105,11 @@ func (tr *TestRunner) Plan(ctx context.Context) error { } // Calculate and write parallel runners count - parallelRunners := calculateParallelRunners(tr.testFileWeights) + parallelRunners := calculateParallelRunners( + tr.testFileWeights, + settings.GetMinParallelism(), + settings.GetMaxParallelism(), + ) runnersContent := fmt.Sprintf("%d", parallelRunners) if err := writePlanFileCopies([]byte(runnersContent), constants.ParallelRunnersOutputPath, constants.LegacyParallelRunnersOutputPath); err != nil { return fmt.Errorf("failed to write parallel runners: %w", err) From 3554a3263118d6e870807bdcd750b2f3b1d38563 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 19 May 2026 21:53:37 +0200 Subject: [PATCH 03/11] Remove optional split scheduling output --- internal/runner/distribution.go | 12 ++++------ internal/runner/parallelism.go | 4 ++-- internal/runner/split.go | 41 +++++++++++++++++++++------------ 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/internal/runner/distribution.go b/internal/runner/distribution.go index ad5e1cd..ec634c8 100644 --- a/internal/runner/distribution.go +++ b/internal/runner/distribution.go @@ -11,18 +11,16 @@ import ( // DistributeTestFiles distributes test files across parallel runners using weighted list scheduling. func DistributeTestFiles(testFiles map[string]int, parallelRunners int) [][]string { - if parallelRunners <= 0 { - parallelRunners = 1 - } + split := newWeightedRunnerSplit(parallelRunners) - result := make([][]string, parallelRunners) + result := make([][]string, split.parallelRunners) for i := range result { result[i] = []string{} } - files := sortedWeightedTestFiles(testFiles) - if len(files) > 0 { - scheduleSortedFiles(files, parallelRunners, result) + for _, file := range sortedWeightedTestFiles(testFiles) { + runnerIndex := split.addFile(file.weight) + result[runnerIndex] = append(result[runnerIndex], file.path) } return result diff --git a/internal/runner/parallelism.go b/internal/runner/parallelism.go index f549c76..c3dc49b 100644 --- a/internal/runner/parallelism.go +++ b/internal/runner/parallelism.go @@ -28,9 +28,9 @@ func calculateParallelRunners(testFileWeights map[string]int, minParallelism, ma candidateMax := maxUsefulParallelism(minParallelism, maxParallelism, len(files)) - best := scoreSortedSplit(files, minParallelism) + best := scoreSortedWeightedRunnerSplit(files, minParallelism) for parallelRunners := minParallelism + 1; parallelRunners <= candidateMax; parallelRunners++ { - score := scoreSortedSplit(files, parallelRunners) + score := scoreSortedWeightedRunnerSplit(files, parallelRunners) if betterSplit(score, best) { best = score } diff --git a/internal/runner/split.go b/internal/runner/split.go index 2c061ab..fcbc583 100644 --- a/internal/runner/split.go +++ b/internal/runner/split.go @@ -15,6 +15,11 @@ type runnerLoad struct { load int } +type weightedRunnerSplit struct { + parallelRunners int + loads minLoadHeap +} + type minLoadHeap []runnerLoad func (h minLoadHeap) Len() int { @@ -75,30 +80,36 @@ func sortedWeightedTestFiles(testFiles map[string]int) []weightedTestFile { return files } -func scoreSortedSplit(files []weightedTestFile, parallelRunners int) splitScore { - return scheduleSortedFiles(files, parallelRunners, nil) +func scoreSortedWeightedRunnerSplit(files []weightedTestFile, parallelRunners int) splitScore { + split := newWeightedRunnerSplit(parallelRunners) + for _, file := range files { + split.addFile(file.weight) + } + return split.score() } -// scheduleSortedFiles uses longest-processing-time list scheduling: assign the -// heaviest remaining file to the currently lightest runner. -func scheduleSortedFiles(files []weightedTestFile, parallelRunners int, result [][]string) splitScore { +func newWeightedRunnerSplit(parallelRunners int) weightedRunnerSplit { if parallelRunners <= 0 { parallelRunners = 1 } - loads := makeMinLoadHeap(parallelRunners) - for _, file := range files { - lightestRunner := heap.Pop(&loads).(runnerLoad) - lightestRunner.load += file.weight - if result != nil { - result[lightestRunner.index] = append(result[lightestRunner.index], file.path) - } - heap.Push(&loads, lightestRunner) + return weightedRunnerSplit{ + parallelRunners: parallelRunners, + loads: makeMinLoadHeap(parallelRunners), } +} - minLoad, maxLoad := minMaxLoad(loads) +func (s *weightedRunnerSplit) addFile(weight int) int { + lightestRunner := heap.Pop(&s.loads).(runnerLoad) + lightestRunner.load += weight + heap.Push(&s.loads, lightestRunner) + return lightestRunner.index +} + +func (s weightedRunnerSplit) score() splitScore { + minLoad, maxLoad := minMaxLoad(s.loads) return splitScore{ - parallelRunners: parallelRunners, + parallelRunners: s.parallelRunners, wallTime: maxLoad, imbalance: maxLoad - minLoad, } From 7f933b18266215d87759fa78f704be5a1a7309c7 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 19 May 2026 21:56:52 +0200 Subject: [PATCH 04/11] Add split scheduling test coverage --- internal/runner/distribution_test.go | 30 +++++++++++-- internal/runner/split_test.go | 64 ++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 internal/runner/split_test.go diff --git a/internal/runner/distribution_test.go b/internal/runner/distribution_test.go index 3dddfa8..f066d1e 100644 --- a/internal/runner/distribution_test.go +++ b/internal/runner/distribution_test.go @@ -81,9 +81,9 @@ func TestDistributeTestFiles(t *testing.T) { t.Errorf("Expected 3 runners, got %d", len(result)) } - // With First Fit Decreasing algorithm, files are sorted by count (descending): + // With weighted list scheduling, files are sorted by weight descending: // test2.rb: 10, test4.rb: 8, test3.rb: 6, test5.rb: 4, test1.rb: 2 - // Expected distribution (always picking bin with minimum load): + // Expected distribution (always picking the runner with minimum load): // Runner 0: test2.rb (10) // Runner 1: test4.rb (8) + test1.rb (2) = 10 // Runner 2: test3.rb (6) + test5.rb (4) = 10 @@ -198,6 +198,30 @@ func TestDistributeTestFiles(t *testing.T) { } }) + t.Run("ties sort by path and lower runner index", func(t *testing.T) { + testFiles := map[string]int{ + "b.rb": 1, + "a.rb": 1, + "c.rb": 1, + } + + result := DistributeTestFiles(testFiles, 2) + expected := [][]string{ + {"a.rb", "c.rb"}, + {"b.rb"}, + } + + if len(result) != len(expected) { + t.Fatalf("Expected %d runners, got %d", len(expected), len(result)) + } + + for i := range expected { + if !slices.Equal(result[i], expected[i]) { + t.Errorf("Runner %d = %v, expected %v", i, result[i], expected[i]) + } + } + }) + t.Run("deterministic output", func(t *testing.T) { testFiles := map[string]int{ "test1.rb": 5, @@ -341,7 +365,7 @@ func TestCreateTestSplits(t *testing.T) { } // Verify content distribution - // With bin packing: runner-0 gets file1 (10), runner-1 gets file2+file3 (8) + // With weighted list scheduling: runner-0 gets file1 (10), runner-1 gets file2+file3 (8) runner0Content, _ := os.ReadFile(filepath.Join(constants.TestsSplitDir, "runner-0")) runner1Content, _ := os.ReadFile(filepath.Join(constants.TestsSplitDir, "runner-1")) legacyRunner0Content, _ := os.ReadFile(filepath.Join(constants.LegacyTestsSplitDir, "runner-0")) diff --git a/internal/runner/split_test.go b/internal/runner/split_test.go new file mode 100644 index 0000000..4fbd41f --- /dev/null +++ b/internal/runner/split_test.go @@ -0,0 +1,64 @@ +package runner + +import ( + "slices" + "testing" +) + +func TestSortedWeightedTestFiles(t *testing.T) { + testFiles := map[string]int{ + "small.rb": 1, + "same-b.rb": 5, + "large.rb": 10, + "same-a.rb": 5, + } + + result := sortedWeightedTestFiles(testFiles) + expected := []weightedTestFile{ + {path: "large.rb", weight: 10}, + {path: "same-a.rb", weight: 5}, + {path: "same-b.rb", weight: 5}, + {path: "small.rb", weight: 1}, + } + + if !slices.Equal(result, expected) { + t.Fatalf("sortedWeightedTestFiles() = %v, expected %v", result, expected) + } +} + +func TestScoreSortedWeightedRunnerSplit(t *testing.T) { + files := []weightedTestFile{ + {path: "slow.rb", weight: 10}, + {path: "medium.rb", weight: 6}, + {path: "fast.rb", weight: 4}, + {path: "tiny.rb", weight: 2}, + } + + result := scoreSortedWeightedRunnerSplit(files, 2) + expected := splitScore{ + parallelRunners: 2, + wallTime: 12, + imbalance: 2, + } + + if result != expected { + t.Fatalf("scoreSortedWeightedRunnerSplit() = %+v, expected %+v", result, expected) + } +} + +func TestScoreSortedWeightedRunnerSplit_UnavoidableEmptyRunners(t *testing.T) { + files := []weightedTestFile{ + {path: "slow.rb", weight: 10}, + } + + result := scoreSortedWeightedRunnerSplit(files, 3) + expected := splitScore{ + parallelRunners: 3, + wallTime: 10, + imbalance: 10, + } + + if result != expected { + t.Fatalf("scoreSortedWeightedRunnerSplit() = %+v, expected %+v", result, expected) + } +} From 56170d16e320241f507282048d7c4c231218926b Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 19 May 2026 22:00:13 +0200 Subject: [PATCH 05/11] Reorder split helpers by call surface --- internal/runner/split.go | 90 ++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/internal/runner/split.go b/internal/runner/split.go index fcbc583..808df8e 100644 --- a/internal/runner/split.go +++ b/internal/runner/split.go @@ -10,51 +10,6 @@ type weightedTestFile struct { weight int } -type runnerLoad struct { - index int - load int -} - -type weightedRunnerSplit struct { - parallelRunners int - loads minLoadHeap -} - -type minLoadHeap []runnerLoad - -func (h minLoadHeap) Len() int { - return len(h) -} - -func (h minLoadHeap) Less(i, j int) bool { - if h[i].load == h[j].load { - return h[i].index < h[j].index - } - return h[i].load < h[j].load -} - -func (h minLoadHeap) Swap(i, j int) { - h[i], h[j] = h[j], h[i] -} - -func (h *minLoadHeap) Push(x any) { - *h = append(*h, x.(runnerLoad)) -} - -func (h *minLoadHeap) Pop() any { - old := *h - n := len(old) - x := old[n-1] - *h = old[:n-1] - return x -} - -type splitScore struct { - parallelRunners int - wallTime int - imbalance int -} - func sortedWeightedTestFiles(testFiles map[string]int) []weightedTestFile { files := make([]weightedTestFile, 0, len(testFiles)) for path, weight := range testFiles { @@ -80,6 +35,12 @@ func sortedWeightedTestFiles(testFiles map[string]int) []weightedTestFile { return files } +type splitScore struct { + parallelRunners int + wallTime int + imbalance int +} + func scoreSortedWeightedRunnerSplit(files []weightedTestFile, parallelRunners int) splitScore { split := newWeightedRunnerSplit(parallelRunners) for _, file := range files { @@ -88,6 +49,11 @@ func scoreSortedWeightedRunnerSplit(files []weightedTestFile, parallelRunners in return split.score() } +type weightedRunnerSplit struct { + parallelRunners int + loads minLoadHeap +} + func newWeightedRunnerSplit(parallelRunners int) weightedRunnerSplit { if parallelRunners <= 0 { parallelRunners = 1 @@ -115,6 +81,40 @@ func (s weightedRunnerSplit) score() splitScore { } } +type runnerLoad struct { + index int + load int +} + +type minLoadHeap []runnerLoad + +func (h minLoadHeap) Len() int { + return len(h) +} + +func (h minLoadHeap) Less(i, j int) bool { + if h[i].load == h[j].load { + return h[i].index < h[j].index + } + return h[i].load < h[j].load +} + +func (h minLoadHeap) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +func (h *minLoadHeap) Push(x any) { + *h = append(*h, x.(runnerLoad)) +} + +func (h *minLoadHeap) Pop() any { + old := *h + n := len(old) + x := old[n-1] + *h = old[:n-1] + return x +} + func makeMinLoadHeap(parallelRunners int) minLoadHeap { loads := make(minLoadHeap, parallelRunners) for i := range loads { From 57336565bc1ca2668d32054a67aa92a11a004dfb Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 19 May 2026 22:02:40 +0200 Subject: [PATCH 06/11] Rename weighted runner split builder --- internal/runner/distribution.go | 6 +++--- internal/runner/split.go | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/internal/runner/distribution.go b/internal/runner/distribution.go index ec634c8..fbb2832 100644 --- a/internal/runner/distribution.go +++ b/internal/runner/distribution.go @@ -11,15 +11,15 @@ import ( // DistributeTestFiles distributes test files across parallel runners using weighted list scheduling. func DistributeTestFiles(testFiles map[string]int, parallelRunners int) [][]string { - split := newWeightedRunnerSplit(parallelRunners) + builder := newTestSplitBuilder(parallelRunners) - result := make([][]string, split.parallelRunners) + result := make([][]string, builder.parallelRunners) for i := range result { result[i] = []string{} } for _, file := range sortedWeightedTestFiles(testFiles) { - runnerIndex := split.addFile(file.weight) + runnerIndex := builder.addFile(file.weight) result[runnerIndex] = append(result[runnerIndex], file.path) } diff --git a/internal/runner/split.go b/internal/runner/split.go index 808df8e..07c0d60 100644 --- a/internal/runner/split.go +++ b/internal/runner/split.go @@ -42,40 +42,40 @@ type splitScore struct { } func scoreSortedWeightedRunnerSplit(files []weightedTestFile, parallelRunners int) splitScore { - split := newWeightedRunnerSplit(parallelRunners) + builder := newTestSplitBuilder(parallelRunners) for _, file := range files { - split.addFile(file.weight) + builder.addFile(file.weight) } - return split.score() + return builder.score() } -type weightedRunnerSplit struct { +type testSplitBuilder struct { parallelRunners int loads minLoadHeap } -func newWeightedRunnerSplit(parallelRunners int) weightedRunnerSplit { +func newTestSplitBuilder(parallelRunners int) testSplitBuilder { if parallelRunners <= 0 { parallelRunners = 1 } - return weightedRunnerSplit{ + return testSplitBuilder{ parallelRunners: parallelRunners, loads: makeMinLoadHeap(parallelRunners), } } -func (s *weightedRunnerSplit) addFile(weight int) int { - lightestRunner := heap.Pop(&s.loads).(runnerLoad) +func (b *testSplitBuilder) addFile(weight int) int { + lightestRunner := heap.Pop(&b.loads).(runnerLoad) lightestRunner.load += weight - heap.Push(&s.loads, lightestRunner) + heap.Push(&b.loads, lightestRunner) return lightestRunner.index } -func (s weightedRunnerSplit) score() splitScore { - minLoad, maxLoad := minMaxLoad(s.loads) +func (b testSplitBuilder) score() splitScore { + minLoad, maxLoad := minMaxLoad(b.loads) return splitScore{ - parallelRunners: s.parallelRunners, + parallelRunners: b.parallelRunners, wallTime: maxLoad, imbalance: maxLoad - minLoad, } From 76be21922d5bf11cca46856228a2dfb4d4f243db Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 19 May 2026 22:07:02 +0200 Subject: [PATCH 07/11] Move distribution into split builder --- internal/runner/distribution.go | 14 +--------- internal/runner/split.go | 28 ++++++++++++++++++++ internal/runner/split_test.go | 46 +++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 13 deletions(-) diff --git a/internal/runner/distribution.go b/internal/runner/distribution.go index fbb2832..69ecdce 100644 --- a/internal/runner/distribution.go +++ b/internal/runner/distribution.go @@ -11,19 +11,7 @@ import ( // DistributeTestFiles distributes test files across parallel runners using weighted list scheduling. func DistributeTestFiles(testFiles map[string]int, parallelRunners int) [][]string { - builder := newTestSplitBuilder(parallelRunners) - - result := make([][]string, builder.parallelRunners) - for i := range result { - result[i] = []string{} - } - - for _, file := range sortedWeightedTestFiles(testFiles) { - runnerIndex := builder.addFile(file.weight) - result[runnerIndex] = append(result[runnerIndex], file.path) - } - - return result + return distributeWeightedTestFiles(testFiles, parallelRunners) } // CreateTestSplits creates test split files for parallel runners diff --git a/internal/runner/split.go b/internal/runner/split.go index 07c0d60..0506a09 100644 --- a/internal/runner/split.go +++ b/internal/runner/split.go @@ -35,6 +35,16 @@ func sortedWeightedTestFiles(testFiles map[string]int) []weightedTestFile { return files } +func distributeWeightedTestFiles(testFiles map[string]int, parallelRunners int) [][]string { + builder := newTestSplitBuilder(parallelRunners) + return builder.distributeFiles(testFiles) +} + +func distributeSortedWeightedTestFiles(files []weightedTestFile, parallelRunners int) [][]string { + builder := newTestSplitBuilder(parallelRunners) + return builder.distributeSortedFiles(files) +} + type splitScore struct { parallelRunners int wallTime int @@ -72,6 +82,24 @@ func (b *testSplitBuilder) addFile(weight int) int { return lightestRunner.index } +func (b *testSplitBuilder) distributeFiles(testFiles map[string]int) [][]string { + return b.distributeSortedFiles(sortedWeightedTestFiles(testFiles)) +} + +func (b *testSplitBuilder) distributeSortedFiles(files []weightedTestFile) [][]string { + result := make([][]string, b.parallelRunners) + for i := range result { + result[i] = []string{} + } + + for _, file := range files { + runnerIndex := b.addFile(file.weight) + result[runnerIndex] = append(result[runnerIndex], file.path) + } + + return result +} + func (b testSplitBuilder) score() splitScore { minLoad, maxLoad := minMaxLoad(b.loads) return splitScore{ diff --git a/internal/runner/split_test.go b/internal/runner/split_test.go index 4fbd41f..456d0c5 100644 --- a/internal/runner/split_test.go +++ b/internal/runner/split_test.go @@ -62,3 +62,49 @@ func TestScoreSortedWeightedRunnerSplit_UnavoidableEmptyRunners(t *testing.T) { t.Fatalf("scoreSortedWeightedRunnerSplit() = %+v, expected %+v", result, expected) } } + +func TestDistributeWeightedTestFiles(t *testing.T) { + testFiles := map[string]int{ + "fast.rb": 1, + "medium.rb": 2, + "slow.rb": 3, + } + + result := distributeWeightedTestFiles(testFiles, 2) + expected := [][]string{ + {"slow.rb"}, + {"medium.rb", "fast.rb"}, + } + + assertDistribution(t, result, expected) +} + +func TestDistributeSortedWeightedTestFiles(t *testing.T) { + files := []weightedTestFile{ + {path: "slow.rb", weight: 3}, + {path: "medium.rb", weight: 2}, + {path: "fast.rb", weight: 1}, + } + + result := distributeSortedWeightedTestFiles(files, 2) + expected := [][]string{ + {"slow.rb"}, + {"medium.rb", "fast.rb"}, + } + + assertDistribution(t, result, expected) +} + +func assertDistribution(t *testing.T, result, expected [][]string) { + t.Helper() + + if len(result) != len(expected) { + t.Fatalf("distribution has %d runners, expected %d", len(result), len(expected)) + } + + for i := range expected { + if !slices.Equal(result[i], expected[i]) { + t.Errorf("runner %d = %v, expected %v", i, result[i], expected[i]) + } + } +} From 77fa8da89eadba46cc752310ec76b14a475cf921 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 19 May 2026 22:09:59 +0200 Subject: [PATCH 08/11] Merge split helpers into distribution --- internal/runner/distribution.go | 163 ++++++++++++++++++++++++++ internal/runner/distribution_test.go | 104 +++++++++++++++++ internal/runner/split.go | 167 --------------------------- internal/runner/split_test.go | 110 ------------------ 4 files changed, 267 insertions(+), 277 deletions(-) delete mode 100644 internal/runner/split.go delete mode 100644 internal/runner/split_test.go diff --git a/internal/runner/distribution.go b/internal/runner/distribution.go index 69ecdce..d4ac02b 100644 --- a/internal/runner/distribution.go +++ b/internal/runner/distribution.go @@ -1,9 +1,11 @@ package runner import ( + "container/heap" "fmt" "os" "path/filepath" + "slices" "strings" "github.com/DataDog/ddtest/internal/constants" @@ -45,6 +47,167 @@ func CreateTestSplits(testFiles map[string]int, parallelRunners int, testFilesOu return nil } +type weightedTestFile struct { + path string + weight int +} + +func sortedWeightedTestFiles(testFiles map[string]int) []weightedTestFile { + files := make([]weightedTestFile, 0, len(testFiles)) + for path, weight := range testFiles { + files = append(files, weightedTestFile{path: path, weight: weight}) + } + + slices.SortFunc(files, func(a, b weightedTestFile) int { + if a.weight > b.weight { + return -1 + } + if a.weight < b.weight { + return 1 + } + if a.path < b.path { + return -1 + } + if a.path > b.path { + return 1 + } + return 0 + }) + + return files +} + +func distributeWeightedTestFiles(testFiles map[string]int, parallelRunners int) [][]string { + builder := newTestSplitBuilder(parallelRunners) + return builder.distributeFiles(testFiles) +} + +func distributeSortedWeightedTestFiles(files []weightedTestFile, parallelRunners int) [][]string { + builder := newTestSplitBuilder(parallelRunners) + return builder.distributeSortedFiles(files) +} + +type splitScore struct { + parallelRunners int + wallTime int + imbalance int +} + +func scoreSortedWeightedRunnerSplit(files []weightedTestFile, parallelRunners int) splitScore { + builder := newTestSplitBuilder(parallelRunners) + for _, file := range files { + builder.addFile(file.weight) + } + return builder.score() +} + +type testSplitBuilder struct { + parallelRunners int + loads minLoadHeap +} + +func newTestSplitBuilder(parallelRunners int) testSplitBuilder { + if parallelRunners <= 0 { + parallelRunners = 1 + } + + return testSplitBuilder{ + parallelRunners: parallelRunners, + loads: makeMinLoadHeap(parallelRunners), + } +} + +func (b *testSplitBuilder) addFile(weight int) int { + lightestRunner := heap.Pop(&b.loads).(runnerLoad) + lightestRunner.load += weight + heap.Push(&b.loads, lightestRunner) + return lightestRunner.index +} + +func (b *testSplitBuilder) distributeFiles(testFiles map[string]int) [][]string { + return b.distributeSortedFiles(sortedWeightedTestFiles(testFiles)) +} + +func (b *testSplitBuilder) distributeSortedFiles(files []weightedTestFile) [][]string { + result := make([][]string, b.parallelRunners) + for i := range result { + result[i] = []string{} + } + + for _, file := range files { + runnerIndex := b.addFile(file.weight) + result[runnerIndex] = append(result[runnerIndex], file.path) + } + + return result +} + +func (b testSplitBuilder) score() splitScore { + minLoad, maxLoad := minMaxLoad(b.loads) + return splitScore{ + parallelRunners: b.parallelRunners, + wallTime: maxLoad, + imbalance: maxLoad - minLoad, + } +} + +type runnerLoad struct { + index int + load int +} + +type minLoadHeap []runnerLoad + +func (h minLoadHeap) Len() int { + return len(h) +} + +func (h minLoadHeap) Less(i, j int) bool { + if h[i].load == h[j].load { + return h[i].index < h[j].index + } + return h[i].load < h[j].load +} + +func (h minLoadHeap) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +func (h *minLoadHeap) Push(x any) { + *h = append(*h, x.(runnerLoad)) +} + +func (h *minLoadHeap) Pop() any { + old := *h + n := len(old) + x := old[n-1] + *h = old[:n-1] + return x +} + +func makeMinLoadHeap(parallelRunners int) minLoadHeap { + loads := make(minLoadHeap, parallelRunners) + for i := range loads { + loads[i] = runnerLoad{index: i} + } + heap.Init(&loads) + return loads +} + +func minMaxLoad(loads []runnerLoad) (int, int) { + minLoad := loads[0].load + maxLoad := loads[0].load + for _, load := range loads[1:] { + if load.load < minLoad { + minLoad = load.load + } + if load.load > maxLoad { + maxLoad = load.load + } + } + return minLoad, maxLoad +} + func writeDistributedTestSplits(distribution [][]string, testsSplitDir string) error { for i, runnerFiles := range distribution { runnerContent := strings.Join(runnerFiles, "\n") diff --git a/internal/runner/distribution_test.go b/internal/runner/distribution_test.go index f066d1e..c2f1660 100644 --- a/internal/runner/distribution_test.go +++ b/internal/runner/distribution_test.go @@ -268,6 +268,96 @@ func TestDistributeTestFiles(t *testing.T) { }) } +func TestSortedWeightedTestFiles(t *testing.T) { + testFiles := map[string]int{ + "small.rb": 1, + "same-b.rb": 5, + "large.rb": 10, + "same-a.rb": 5, + } + + result := sortedWeightedTestFiles(testFiles) + expected := []weightedTestFile{ + {path: "large.rb", weight: 10}, + {path: "same-a.rb", weight: 5}, + {path: "same-b.rb", weight: 5}, + {path: "small.rb", weight: 1}, + } + + if !slices.Equal(result, expected) { + t.Fatalf("sortedWeightedTestFiles() = %v, expected %v", result, expected) + } +} + +func TestScoreSortedWeightedRunnerSplit(t *testing.T) { + files := []weightedTestFile{ + {path: "slow.rb", weight: 10}, + {path: "medium.rb", weight: 6}, + {path: "fast.rb", weight: 4}, + {path: "tiny.rb", weight: 2}, + } + + result := scoreSortedWeightedRunnerSplit(files, 2) + expected := splitScore{ + parallelRunners: 2, + wallTime: 12, + imbalance: 2, + } + + if result != expected { + t.Fatalf("scoreSortedWeightedRunnerSplit() = %+v, expected %+v", result, expected) + } +} + +func TestScoreSortedWeightedRunnerSplit_UnavoidableEmptyRunners(t *testing.T) { + files := []weightedTestFile{ + {path: "slow.rb", weight: 10}, + } + + result := scoreSortedWeightedRunnerSplit(files, 3) + expected := splitScore{ + parallelRunners: 3, + wallTime: 10, + imbalance: 10, + } + + if result != expected { + t.Fatalf("scoreSortedWeightedRunnerSplit() = %+v, expected %+v", result, expected) + } +} + +func TestDistributeWeightedTestFiles(t *testing.T) { + testFiles := map[string]int{ + "fast.rb": 1, + "medium.rb": 2, + "slow.rb": 3, + } + + result := distributeWeightedTestFiles(testFiles, 2) + expected := [][]string{ + {"slow.rb"}, + {"medium.rb", "fast.rb"}, + } + + assertDistribution(t, result, expected) +} + +func TestDistributeSortedWeightedTestFiles(t *testing.T) { + files := []weightedTestFile{ + {path: "slow.rb", weight: 3}, + {path: "medium.rb", weight: 2}, + {path: "fast.rb", weight: 1}, + } + + result := distributeSortedWeightedTestFiles(files, 2) + expected := [][]string{ + {"slow.rb"}, + {"medium.rb", "fast.rb"}, + } + + assertDistribution(t, result, expected) +} + func TestCreateTestSplits(t *testing.T) { t.Run("single runner - copies test-files.txt to runner-0", func(t *testing.T) { tempDir := t.TempDir() @@ -444,3 +534,17 @@ func TestCreateTestSplits(t *testing.T) { } }) } + +func assertDistribution(t *testing.T, result, expected [][]string) { + t.Helper() + + if len(result) != len(expected) { + t.Fatalf("distribution has %d runners, expected %d", len(result), len(expected)) + } + + for i := range expected { + if !slices.Equal(result[i], expected[i]) { + t.Errorf("runner %d = %v, expected %v", i, result[i], expected[i]) + } + } +} diff --git a/internal/runner/split.go b/internal/runner/split.go deleted file mode 100644 index 0506a09..0000000 --- a/internal/runner/split.go +++ /dev/null @@ -1,167 +0,0 @@ -package runner - -import ( - "container/heap" - "slices" -) - -type weightedTestFile struct { - path string - weight int -} - -func sortedWeightedTestFiles(testFiles map[string]int) []weightedTestFile { - files := make([]weightedTestFile, 0, len(testFiles)) - for path, weight := range testFiles { - files = append(files, weightedTestFile{path: path, weight: weight}) - } - - slices.SortFunc(files, func(a, b weightedTestFile) int { - if a.weight > b.weight { - return -1 - } - if a.weight < b.weight { - return 1 - } - if a.path < b.path { - return -1 - } - if a.path > b.path { - return 1 - } - return 0 - }) - - return files -} - -func distributeWeightedTestFiles(testFiles map[string]int, parallelRunners int) [][]string { - builder := newTestSplitBuilder(parallelRunners) - return builder.distributeFiles(testFiles) -} - -func distributeSortedWeightedTestFiles(files []weightedTestFile, parallelRunners int) [][]string { - builder := newTestSplitBuilder(parallelRunners) - return builder.distributeSortedFiles(files) -} - -type splitScore struct { - parallelRunners int - wallTime int - imbalance int -} - -func scoreSortedWeightedRunnerSplit(files []weightedTestFile, parallelRunners int) splitScore { - builder := newTestSplitBuilder(parallelRunners) - for _, file := range files { - builder.addFile(file.weight) - } - return builder.score() -} - -type testSplitBuilder struct { - parallelRunners int - loads minLoadHeap -} - -func newTestSplitBuilder(parallelRunners int) testSplitBuilder { - if parallelRunners <= 0 { - parallelRunners = 1 - } - - return testSplitBuilder{ - parallelRunners: parallelRunners, - loads: makeMinLoadHeap(parallelRunners), - } -} - -func (b *testSplitBuilder) addFile(weight int) int { - lightestRunner := heap.Pop(&b.loads).(runnerLoad) - lightestRunner.load += weight - heap.Push(&b.loads, lightestRunner) - return lightestRunner.index -} - -func (b *testSplitBuilder) distributeFiles(testFiles map[string]int) [][]string { - return b.distributeSortedFiles(sortedWeightedTestFiles(testFiles)) -} - -func (b *testSplitBuilder) distributeSortedFiles(files []weightedTestFile) [][]string { - result := make([][]string, b.parallelRunners) - for i := range result { - result[i] = []string{} - } - - for _, file := range files { - runnerIndex := b.addFile(file.weight) - result[runnerIndex] = append(result[runnerIndex], file.path) - } - - return result -} - -func (b testSplitBuilder) score() splitScore { - minLoad, maxLoad := minMaxLoad(b.loads) - return splitScore{ - parallelRunners: b.parallelRunners, - wallTime: maxLoad, - imbalance: maxLoad - minLoad, - } -} - -type runnerLoad struct { - index int - load int -} - -type minLoadHeap []runnerLoad - -func (h minLoadHeap) Len() int { - return len(h) -} - -func (h minLoadHeap) Less(i, j int) bool { - if h[i].load == h[j].load { - return h[i].index < h[j].index - } - return h[i].load < h[j].load -} - -func (h minLoadHeap) Swap(i, j int) { - h[i], h[j] = h[j], h[i] -} - -func (h *minLoadHeap) Push(x any) { - *h = append(*h, x.(runnerLoad)) -} - -func (h *minLoadHeap) Pop() any { - old := *h - n := len(old) - x := old[n-1] - *h = old[:n-1] - return x -} - -func makeMinLoadHeap(parallelRunners int) minLoadHeap { - loads := make(minLoadHeap, parallelRunners) - for i := range loads { - loads[i] = runnerLoad{index: i} - } - heap.Init(&loads) - return loads -} - -func minMaxLoad(loads []runnerLoad) (int, int) { - minLoad := loads[0].load - maxLoad := loads[0].load - for _, load := range loads[1:] { - if load.load < minLoad { - minLoad = load.load - } - if load.load > maxLoad { - maxLoad = load.load - } - } - return minLoad, maxLoad -} diff --git a/internal/runner/split_test.go b/internal/runner/split_test.go deleted file mode 100644 index 456d0c5..0000000 --- a/internal/runner/split_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package runner - -import ( - "slices" - "testing" -) - -func TestSortedWeightedTestFiles(t *testing.T) { - testFiles := map[string]int{ - "small.rb": 1, - "same-b.rb": 5, - "large.rb": 10, - "same-a.rb": 5, - } - - result := sortedWeightedTestFiles(testFiles) - expected := []weightedTestFile{ - {path: "large.rb", weight: 10}, - {path: "same-a.rb", weight: 5}, - {path: "same-b.rb", weight: 5}, - {path: "small.rb", weight: 1}, - } - - if !slices.Equal(result, expected) { - t.Fatalf("sortedWeightedTestFiles() = %v, expected %v", result, expected) - } -} - -func TestScoreSortedWeightedRunnerSplit(t *testing.T) { - files := []weightedTestFile{ - {path: "slow.rb", weight: 10}, - {path: "medium.rb", weight: 6}, - {path: "fast.rb", weight: 4}, - {path: "tiny.rb", weight: 2}, - } - - result := scoreSortedWeightedRunnerSplit(files, 2) - expected := splitScore{ - parallelRunners: 2, - wallTime: 12, - imbalance: 2, - } - - if result != expected { - t.Fatalf("scoreSortedWeightedRunnerSplit() = %+v, expected %+v", result, expected) - } -} - -func TestScoreSortedWeightedRunnerSplit_UnavoidableEmptyRunners(t *testing.T) { - files := []weightedTestFile{ - {path: "slow.rb", weight: 10}, - } - - result := scoreSortedWeightedRunnerSplit(files, 3) - expected := splitScore{ - parallelRunners: 3, - wallTime: 10, - imbalance: 10, - } - - if result != expected { - t.Fatalf("scoreSortedWeightedRunnerSplit() = %+v, expected %+v", result, expected) - } -} - -func TestDistributeWeightedTestFiles(t *testing.T) { - testFiles := map[string]int{ - "fast.rb": 1, - "medium.rb": 2, - "slow.rb": 3, - } - - result := distributeWeightedTestFiles(testFiles, 2) - expected := [][]string{ - {"slow.rb"}, - {"medium.rb", "fast.rb"}, - } - - assertDistribution(t, result, expected) -} - -func TestDistributeSortedWeightedTestFiles(t *testing.T) { - files := []weightedTestFile{ - {path: "slow.rb", weight: 3}, - {path: "medium.rb", weight: 2}, - {path: "fast.rb", weight: 1}, - } - - result := distributeSortedWeightedTestFiles(files, 2) - expected := [][]string{ - {"slow.rb"}, - {"medium.rb", "fast.rb"}, - } - - assertDistribution(t, result, expected) -} - -func assertDistribution(t *testing.T, result, expected [][]string) { - t.Helper() - - if len(result) != len(expected) { - t.Fatalf("distribution has %d runners, expected %d", len(result), len(expected)) - } - - for i := range expected { - if !slices.Equal(result[i], expected[i]) { - t.Errorf("runner %d = %v, expected %v", i, result[i], expected[i]) - } - } -} From e933d7292ff412a0394cb11534e4f7909da494e4 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 19 May 2026 22:11:55 +0200 Subject: [PATCH 09/11] Inline distribution helper wrappers --- internal/runner/distribution.go | 13 ++----------- internal/runner/distribution_test.go | 10 ++++++---- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/internal/runner/distribution.go b/internal/runner/distribution.go index d4ac02b..3bb1bfc 100644 --- a/internal/runner/distribution.go +++ b/internal/runner/distribution.go @@ -13,7 +13,8 @@ import ( // DistributeTestFiles distributes test files across parallel runners using weighted list scheduling. func DistributeTestFiles(testFiles map[string]int, parallelRunners int) [][]string { - return distributeWeightedTestFiles(testFiles, parallelRunners) + builder := newTestSplitBuilder(parallelRunners) + return builder.distributeFiles(testFiles) } // CreateTestSplits creates test split files for parallel runners @@ -77,16 +78,6 @@ func sortedWeightedTestFiles(testFiles map[string]int) []weightedTestFile { return files } -func distributeWeightedTestFiles(testFiles map[string]int, parallelRunners int) [][]string { - builder := newTestSplitBuilder(parallelRunners) - return builder.distributeFiles(testFiles) -} - -func distributeSortedWeightedTestFiles(files []weightedTestFile, parallelRunners int) [][]string { - builder := newTestSplitBuilder(parallelRunners) - return builder.distributeSortedFiles(files) -} - type splitScore struct { parallelRunners int wallTime int diff --git a/internal/runner/distribution_test.go b/internal/runner/distribution_test.go index c2f1660..5a47dc5 100644 --- a/internal/runner/distribution_test.go +++ b/internal/runner/distribution_test.go @@ -326,14 +326,15 @@ func TestScoreSortedWeightedRunnerSplit_UnavoidableEmptyRunners(t *testing.T) { } } -func TestDistributeWeightedTestFiles(t *testing.T) { +func TestTestSplitBuilderDistributeFiles(t *testing.T) { testFiles := map[string]int{ "fast.rb": 1, "medium.rb": 2, "slow.rb": 3, } - result := distributeWeightedTestFiles(testFiles, 2) + builder := newTestSplitBuilder(2) + result := builder.distributeFiles(testFiles) expected := [][]string{ {"slow.rb"}, {"medium.rb", "fast.rb"}, @@ -342,14 +343,15 @@ func TestDistributeWeightedTestFiles(t *testing.T) { assertDistribution(t, result, expected) } -func TestDistributeSortedWeightedTestFiles(t *testing.T) { +func TestTestSplitBuilderDistributeSortedFiles(t *testing.T) { files := []weightedTestFile{ {path: "slow.rb", weight: 3}, {path: "medium.rb", weight: 2}, {path: "fast.rb", weight: 1}, } - result := distributeSortedWeightedTestFiles(files, 2) + builder := newTestSplitBuilder(2) + result := builder.distributeSortedFiles(files) expected := [][]string{ {"slow.rb"}, {"medium.rb", "fast.rb"}, From 37838c8bee5ed6b8dc99555ab7d374cfde85cc20 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 20 May 2026 12:30:23 +0200 Subject: [PATCH 10/11] Log selected parallel split metrics --- internal/runner/distribution.go | 23 +++++++++++-- internal/runner/distribution_test.go | 2 ++ internal/runner/parallelism.go | 29 ++++++++++++---- internal/runner/parallelism_test.go | 49 ++++++++++++++++++++++++++-- internal/runner/runner.go | 10 ++++-- internal/runner/runner_test.go | 10 ++++++ 6 files changed, 109 insertions(+), 14 deletions(-) diff --git a/internal/runner/distribution.go b/internal/runner/distribution.go index 3bb1bfc..44085cd 100644 --- a/internal/runner/distribution.go +++ b/internal/runner/distribution.go @@ -7,6 +7,7 @@ import ( "path/filepath" "slices" "strings" + "time" "github.com/DataDog/ddtest/internal/constants" ) @@ -82,6 +83,19 @@ type splitScore struct { parallelRunners int wallTime int imbalance int + totalRuntime int +} + +func (s splitScore) wallTimeDuration() time.Duration { + return time.Duration(s.wallTime) * time.Millisecond +} + +func (s splitScore) imbalanceDuration() time.Duration { + return time.Duration(s.imbalance) * time.Millisecond +} + +func (s splitScore) totalRuntimeDuration() time.Duration { + return time.Duration(s.totalRuntime) * time.Millisecond } func scoreSortedWeightedRunnerSplit(files []weightedTestFile, parallelRunners int) splitScore { @@ -134,11 +148,12 @@ func (b *testSplitBuilder) distributeSortedFiles(files []weightedTestFile) [][]s } func (b testSplitBuilder) score() splitScore { - minLoad, maxLoad := minMaxLoad(b.loads) + minLoad, maxLoad, totalLoad := loadStats(b.loads) return splitScore{ parallelRunners: b.parallelRunners, wallTime: maxLoad, imbalance: maxLoad - minLoad, + totalRuntime: totalLoad, } } @@ -185,9 +200,10 @@ func makeMinLoadHeap(parallelRunners int) minLoadHeap { return loads } -func minMaxLoad(loads []runnerLoad) (int, int) { +func loadStats(loads []runnerLoad) (int, int, int) { minLoad := loads[0].load maxLoad := loads[0].load + totalLoad := loads[0].load for _, load := range loads[1:] { if load.load < minLoad { minLoad = load.load @@ -195,8 +211,9 @@ func minMaxLoad(loads []runnerLoad) (int, int) { if load.load > maxLoad { maxLoad = load.load } + totalLoad += load.load } - return minLoad, maxLoad + return minLoad, maxLoad, totalLoad } func writeDistributedTestSplits(distribution [][]string, testsSplitDir string) error { diff --git a/internal/runner/distribution_test.go b/internal/runner/distribution_test.go index 5a47dc5..01fdd84 100644 --- a/internal/runner/distribution_test.go +++ b/internal/runner/distribution_test.go @@ -302,6 +302,7 @@ func TestScoreSortedWeightedRunnerSplit(t *testing.T) { parallelRunners: 2, wallTime: 12, imbalance: 2, + totalRuntime: 22, } if result != expected { @@ -319,6 +320,7 @@ func TestScoreSortedWeightedRunnerSplit_UnavoidableEmptyRunners(t *testing.T) { parallelRunners: 3, wallTime: 10, imbalance: 10, + totalRuntime: 10, } if result != expected { diff --git a/internal/runner/parallelism.go b/internal/runner/parallelism.go index c3dc49b..b7a89c1 100644 --- a/internal/runner/parallelism.go +++ b/internal/runner/parallelism.go @@ -2,12 +2,16 @@ package runner import "log/slog" -// calculateParallelRunners determines the number of parallel runners by -// estimating splits between the configured min and max parallelism. -func calculateParallelRunners(testFileWeights map[string]int, minParallelism, maxParallelism int) int { +// calculateParallelRunnerSplit determines the selected runner split by +// estimating candidates between the configured min and max parallelism. +func calculateParallelRunnerSplit(testFileWeights map[string]int, minParallelism, maxParallelism int) splitScore { + files := sortedWeightedTestFiles(testFileWeights) + // maxParallelism could be 0 or negative! if maxParallelism <= 1 { - return 1 + score := scoreSortedWeightedRunnerSplit(files, 1) + logCandidateSplit(score) + return score } if minParallelism < 1 { @@ -21,22 +25,25 @@ func calculateParallelRunners(testFileWeights map[string]int, minParallelism, ma minParallelism = maxParallelism } - files := sortedWeightedTestFiles(testFileWeights) if len(files) == 0 { - return minParallelism + score := scoreSortedWeightedRunnerSplit(files, minParallelism) + logCandidateSplit(score) + return score } candidateMax := maxUsefulParallelism(minParallelism, maxParallelism, len(files)) best := scoreSortedWeightedRunnerSplit(files, minParallelism) + logCandidateSplit(best) for parallelRunners := minParallelism + 1; parallelRunners <= candidateMax; parallelRunners++ { score := scoreSortedWeightedRunnerSplit(files, parallelRunners) + logCandidateSplit(score) if betterSplit(score, best) { best = score } } - return best.parallelRunners + return best } func maxUsefulParallelism(minParallelism, maxParallelism, filesCount int) int { @@ -53,3 +60,11 @@ func betterSplit(candidate, currentBest splitScore) bool { return candidate.wallTime < currentBest.wallTime || (candidate.wallTime == currentBest.wallTime && candidate.imbalance < currentBest.imbalance) } + +func logCandidateSplit(score splitScore) { + slog.Debug("Considered parallel runner split", + "parallelRunners", score.parallelRunners, + "expectedWallTime", score.wallTimeDuration(), + "imbalance", score.imbalanceDuration(), + "expectedTotalRuntime", score.totalRuntimeDuration()) +} diff --git a/internal/runner/parallelism_test.go b/internal/runner/parallelism_test.go index 6821a3b..07a9c11 100644 --- a/internal/runner/parallelism_test.go +++ b/internal/runner/parallelism_test.go @@ -2,11 +2,12 @@ package runner import ( "fmt" + "strings" "testing" ) func testCalculateParallelRunners(testFileWeights map[string]int, minParallelism, maxParallelism int) int { - return calculateParallelRunners(testFileWeights, minParallelism, maxParallelism) + return calculateParallelRunnerSplit(testFileWeights, minParallelism, maxParallelism).parallelRunners } func TestCalculateParallelRunners_MaxParallelismIsOne(t *testing.T) { @@ -113,6 +114,50 @@ func TestCalculateParallelRunners_OverParallelizedInputsPreserveMinimum(t *testi } } +func TestCalculateParallelRunnerSplit_ReturnsSelectedScore(t *testing.T) { + testFileWeights := map[string]int{ + "test1.rb": 10, + "test2.rb": 6, + "test3.rb": 6, + "test4.rb": 6, + "test5.rb": 6, + } + + result := calculateParallelRunnerSplit(testFileWeights, 3, 4) + expected := splitScore{ + parallelRunners: 3, + wallTime: 12, + imbalance: 2, + totalRuntime: 34, + } + + if result != expected { + t.Errorf("calculateParallelRunnerSplit() = %+v, expected %+v", result, expected) + } +} + +func TestCalculateParallelRunnerSplit_LogsCandidateSplits(t *testing.T) { + logs := captureLogs(t) + testFileWeights := map[string]int{ + "test1.rb": 10, + "test2.rb": 10, + "test3.rb": 10, + } + + _ = calculateParallelRunnerSplit(testFileWeights, 1, 3) + + logOutput := logs.String() + if strings.Count(logOutput, "Considered parallel runner split") != 3 || + !strings.Contains(logOutput, "parallelRunners=1") || + !strings.Contains(logOutput, "parallelRunners=2") || + !strings.Contains(logOutput, "parallelRunners=3") || + !strings.Contains(logOutput, "expectedWallTime=") || + !strings.Contains(logOutput, "imbalance=") || + !strings.Contains(logOutput, "expectedTotalRuntime=") { + t.Errorf("Expected DEBUG logs for each candidate split with score fields, got logs: %s", logOutput) + } +} + func BenchmarkCalculateParallelRunners20000TestFiles(b *testing.B) { testFileWeights := make(map[string]int, 20000) for i := range 20000 { @@ -121,6 +166,6 @@ func BenchmarkCalculateParallelRunners20000TestFiles(b *testing.B) { b.ResetTimer() for range b.N { - _ = calculateParallelRunners(testFileWeights, 1, 256) + _ = calculateParallelRunnerSplit(testFileWeights, 1, 256) } } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 631e739..15f917b 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -105,11 +105,12 @@ func (tr *TestRunner) Plan(ctx context.Context) error { } // Calculate and write parallel runners count - parallelRunners := calculateParallelRunners( + parallelRunnerSplit := calculateParallelRunnerSplit( tr.testFileWeights, settings.GetMinParallelism(), settings.GetMaxParallelism(), ) + parallelRunners := parallelRunnerSplit.parallelRunners runnersContent := fmt.Sprintf("%d", parallelRunners) if err := writePlanFileCopies([]byte(runnersContent), constants.ParallelRunnersOutputPath, constants.LegacyParallelRunnersOutputPath); err != nil { return fmt.Errorf("failed to write parallel runners: %w", err) @@ -132,7 +133,12 @@ func (tr *TestRunner) Plan(ctx context.Context) error { return fmt.Errorf("failed to create test splits: %w", err) } - slog.Info("Test execution planning completed", "parallelRunners", parallelRunners, "testFilesCount", len(tr.testFileWeights)) + slog.Info("Test execution planning completed", + "parallelRunners", parallelRunners, + "expectedWallTime", parallelRunnerSplit.wallTimeDuration(), + "imbalance", parallelRunnerSplit.imbalanceDuration(), + "expectedTotalRuntime", parallelRunnerSplit.totalRuntimeDuration(), + "testFilesCount", len(tr.testFileWeights)) return nil } diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index ffa5d3e..3184523 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -321,6 +321,7 @@ func TestTestRunner_Setup_WithParallelRunners(t *testing.T) { _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") }() settings.Init() + logs := captureLogs(t) // Setup mocks for a test with 40% skippable percentage mockFramework := &MockFramework{ @@ -366,6 +367,15 @@ func TestTestRunner_Setup_WithParallelRunners(t *testing.T) { if string(content) != expected { t.Errorf("Expected parallel runners file content '%s', got '%s'", expected, string(content)) } + + logOutput := logs.String() + if !strings.Contains(logOutput, "Test execution planning completed") || + !strings.Contains(logOutput, "parallelRunners=1") || + !strings.Contains(logOutput, "expectedWallTime=") || + !strings.Contains(logOutput, "imbalance=") || + !strings.Contains(logOutput, "expectedTotalRuntime=") { + t.Errorf("Expected planning log with selected split information, got logs: %s", logOutput) + } } func TestTestRunner_Plan_WritesManifestAndRunnerLayout(t *testing.T) { From 2c23953327364b60d9cbf8c905c77a4e22fc6e13 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Wed, 20 May 2026 14:37:43 +0200 Subject: [PATCH 11/11] Add human-readable test run reports --- README.md | 17 + internal/runner/ci_node_executor.go | 66 ++- internal/runner/ci_node_executor_test.go | 61 ++- internal/runner/dd_test_optimization.go | 87 +++- internal/runner/dd_test_optimization_test.go | 11 + internal/runner/parallel_executor.go | 27 +- internal/runner/parallel_executor_test.go | 20 +- internal/runner/report.go | 425 ++++++++++++++++++ internal/runner/report_test.go | 265 +++++++++++ internal/runner/runner.go | 113 +++-- internal/runner/runner_test.go | 68 ++- internal/runner/sequential_executor.go | 27 +- internal/runner/sequential_executor_test.go | 17 +- internal/runner/test_batch.go | 61 --- internal/runner/test_batch_executor.go | 70 +++ internal/runner/test_batch_test.go | 58 +-- .../runner/test_optimization_plan_cache.go | 93 ++++ ...o => test_optimization_plan_cache_test.go} | 44 +- internal/runner/test_suite_durations_cache.go | 85 ---- internal/runner/worker_env_test.go | 42 +- internal/settings/settings.go | 6 + internal/settings/settings_test.go | 25 ++ internal/testoptimization/cache.go | 22 +- internal/testoptimization/cache_test.go | 44 +- internal/testoptimization/client.go | 16 + 25 files changed, 1364 insertions(+), 406 deletions(-) create mode 100644 internal/runner/report.go create mode 100644 internal/runner/report_test.go delete mode 100644 internal/runner/test_batch.go create mode 100644 internal/runner/test_batch_executor.go create mode 100644 internal/runner/test_optimization_plan_cache.go rename internal/runner/{test_suite_durations_cache_test.go => test_optimization_plan_cache_test.go} (83%) delete mode 100644 internal/runner/test_suite_durations_cache.go diff --git a/README.md b/README.md index 7cf8e13..901e826 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,22 @@ In CI-node mode, DDTest uses one local worker by default so database and other p DDTest automatically sets `DD_TEST_SESSION_NAME` for each worker to `-node--worker-` when the variable is not already set. If you set `DD_TEST_SESSION_NAME` yourself, DDTest preserves it and expands the same `{{nodeIndex}}` and `{{workerIndex}}` placeholders before starting each worker. +### Reports + +DDTest prints a human-readable report to stderr after `ddtest plan` and `ddtest run`. +The plan report summarizes the run identity, Datadog feature settings, backend +data, planning quality, and selected split. `Test impact collection` is shown +from Datadog's `code_coverage` setting. The run report summarizes the worker, +file count, duration, and process result. + +Reports are aggregate-only: DDTest does not print per-test or per-file lists, +and counts may show `disabled` or `not available` when a Datadog feature is off +or its backend payload is not present. To turn reports off: + +```bash +DD_TEST_OPTIMIZATION_RUNNER_REPORT_ENABLED=false ddtest plan +``` + ### Settings (flags and environment variables) | CLI flag | Environment variable | Default | What it does | @@ -173,6 +189,7 @@ DDTest automatically sets `DD_TEST_SESSION_NAME` for each worker to ` 0 { aggregate.EstimatedDuration = p50 * float64(aggregate.NumTests-aggregate.NumTestsSkipped) / float64(aggregate.NumTests) } + if aggregate.EstimatedDuration > 0 { + aggregate.DurationSource = testFileDurationSourceKnown + } } } tr.suiteAggregates[key] = aggregate @@ -366,6 +378,7 @@ func (tr *TestRunner) addBackendTestSuites(subdirPrefix string) { SourceFile: sourceFile, TotalDuration: duration, EstimatedDuration: duration, + DurationSource: testFileDurationSourceKnown, NumTests: 1, NumTestsSkipped: 0, } @@ -448,43 +461,83 @@ func indexSuitesBySourceFile(suiteAggregates map[testSuiteKey]testSuiteAggregate return sourceFileLookup } +type testFileDurationSource string + +const ( + testFileDurationSourceKnown testFileDurationSource = "known" + testFileDurationSourceDefault testFileDurationSource = "default" +) + +type testFileWeightEstimate struct { + weight int + source testFileDurationSource +} + func (tr *TestRunner) weightedTestFiles() map[string]int { - testFileWeights := make(map[string]int, len(tr.testFiles)) - for testFile := range tr.testFiles { - weight, ok := tr.testFileWeight(testFile) + return tr.estimateTestFileWeights(tr.testFiles) +} + +func (tr *TestRunner) estimateTestFileWeights(testFiles map[string]struct{}) map[string]int { + testFileWeights := make(map[string]int, len(testFiles)) + tr.testFileDurationSources = make(map[string]testFileDurationSource, len(testFiles)) + for testFile := range testFiles { + estimate, ok := tr.estimateTestFileWeight(testFile) if ok { - testFileWeights[testFile] = weight + testFileWeights[testFile] = estimate.weight + tr.testFileDurationSources[testFile] = estimate.source } } return testFileWeights } func (tr *TestRunner) testFileWeight(testFile string) (int, bool) { + estimate, ok := tr.estimateTestFileWeight(testFile) + return estimate.weight, ok +} + +func (tr *TestRunner) estimateTestFileWeight(testFile string) (testFileWeightEstimate, bool) { suiteKeys := tr.suitesBySourceFile[testFile] if len(suiteKeys) == 0 { - return defaultTestFileWeight, true + return testFileWeightEstimate{ + weight: defaultTestFileWeight, + source: testFileDurationSourceDefault, + }, true } var duration float64 var hasRunnableSuite bool + var source testFileDurationSource for _, key := range suiteKeys { aggregate := tr.suiteAggregates[key] if aggregate.NumTests == aggregate.NumTestsSkipped { continue } hasRunnableSuite = true + source = aggregate.DurationSource duration += aggregate.EstimatedDuration } if !hasRunnableSuite { - return 0, false + return testFileWeightEstimate{}, false + } + if source == "" { + source = testFileDurationSourceDefault } if duration <= 0 { - return defaultTestFileWeight, true + return testFileWeightEstimate{ + weight: defaultTestFileWeight, + source: source, + }, true } weight := int(duration / float64(time.Millisecond)) if weight < 1 { - return 1, true - } - return weight, true + return testFileWeightEstimate{ + weight: 1, + source: source, + }, true + } + return testFileWeightEstimate{ + weight: weight, + source: source, + }, true } diff --git a/internal/runner/dd_test_optimization_test.go b/internal/runner/dd_test_optimization_test.go index cd5cfd6..abe9283 100644 --- a/internal/runner/dd_test_optimization_test.go +++ b/internal/runner/dd_test_optimization_test.go @@ -428,6 +428,17 @@ func TestTestRunner_TestFileWeight_CountFallbackForMissingSuiteDuration(t *testi if weight, ok := runner.testFileWeight("spec/unknown_test.rb"); !ok || weight != int(time.Second/time.Millisecond) { t.Errorf("Expected unknown file weight to use default 1 second, got weight=%d ok=%t", weight, ok) } + + runner.weightedTestFiles() + if source := runner.testFileDurationSources["spec/file1_test.rb"]; source != testFileDurationSourceKnown { + t.Errorf("Expected Suite1 file duration source to be known, got %q", source) + } + if source := runner.testFileDurationSources["spec/file2_test.rb"]; source != testFileDurationSourceDefault { + t.Errorf("Expected Suite2 file duration source to be default, got %q", source) + } + if source := runner.testFileDurationSources["spec/unknown_test.rb"]; source != testFileDurationSourceDefault { + t.Errorf("Expected unknown file duration source to be default, got %q", source) + } } func TestTestRunner_TestFileWeight_InvalidP50FallsBackForFullDiscoveryAggregate(t *testing.T) { diff --git a/internal/runner/parallel_executor.go b/internal/runner/parallel_executor.go index a605a9d..b577259 100644 --- a/internal/runner/parallel_executor.go +++ b/internal/runner/parallel_executor.go @@ -1,24 +1,25 @@ package runner import ( - "context" "fmt" "log/slog" "os" "path/filepath" "github.com/DataDog/ddtest/internal/constants" - "github.com/DataDog/ddtest/internal/framework" "golang.org/x/sync/errgroup" ) -// runParallelTests executes tests across multiple parallel runners on a single node. -func runParallelTests(ctx context.Context, framework framework.Framework, workerEnvMap map[string]string) error { +// runParallel executes tests across multiple parallel runners on a single node. +func (e testExecutor) runParallel() runExecutionResult { + report := runExecutionReport{ + Mode: runModeParallel, + } slog.Info("Running tests in parallel mode") entries, err := os.ReadDir(constants.TestsSplitDir) if err != nil { - return fmt.Errorf("failed to read tests split directory %s: %w", constants.TestsSplitDir, err) + return report.failure(fmt.Errorf("failed to read tests split directory %s: %w", constants.TestsSplitDir, err)) } var g errgroup.Group @@ -28,14 +29,24 @@ func runParallelTests(ctx context.Context, framework framework.Framework, worker continue } + report.LocalWorkers++ splitFilePath := filepath.Join(constants.TestsSplitDir, entry.Name()) + testFiles, err := loadTestBatch(splitFilePath) + if err != nil { + return report.failure(fmt.Errorf("failed to read test files from %s: %w", splitFilePath, err)) + } + report.TestFilesRun += len(testFiles) + if len(testFiles) == 0 { + continue + } + g.Go(func() error { - return runTestBatchFromFile(ctx, framework, splitFilePath, workerEnvMap, 0, workerIndex) + return e.runBatch(testFiles, 0, workerIndex) }) } if err := g.Wait(); err != nil { - return fmt.Errorf("failed to run parallel tests: %w", err) + return report.failure(fmt.Errorf("failed to run parallel tests: %w", err)) } - return nil + return report.success() } diff --git a/internal/runner/parallel_executor_test.go b/internal/runner/parallel_executor_test.go index e0709e6..90efd64 100644 --- a/internal/runner/parallel_executor_test.go +++ b/internal/runner/parallel_executor_test.go @@ -10,7 +10,7 @@ import ( "github.com/DataDog/ddtest/internal/constants" ) -func TestRunParallelTests_Success(t *testing.T) { +func TestRunParallel_Success(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() defer func() { _ = os.Chdir(oldWd) }() @@ -26,9 +26,16 @@ func TestRunParallelTests_Success(t *testing.T) { RunTestsCalls: []RunTestsCall{}, } - err := runParallelTests(context.Background(), mockFramework, map[string]string{}) + result := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runParallel() + report, err := result.report, result.err if err != nil { - t.Fatalf("runParallelTests() should not return error, got: %v", err) + t.Fatalf("runParallel() should not return error, got: %v", err) + } + if report.LocalWorkers != 2 { + t.Errorf("Expected report to count 2 local workers, got %d", report.LocalWorkers) + } + if report.TestFilesRun != 2 { + t.Errorf("Expected report to count 2 test files, got %d", report.TestFilesRun) } // Verify RunTests was called twice @@ -37,7 +44,7 @@ func TestRunParallelTests_Success(t *testing.T) { } } -func TestRunParallelTests_MissingSplitDirectory(t *testing.T) { +func TestRunParallel_MissingSplitDirectory(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() defer func() { _ = os.Chdir(oldWd) }() @@ -46,9 +53,10 @@ func TestRunParallelTests_MissingSplitDirectory(t *testing.T) { // Don't create tests-split directory mockFramework := &MockFramework{FrameworkName: "rspec"} - err := runParallelTests(context.Background(), mockFramework, map[string]string{}) + result := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runParallel() + err := result.err if err == nil { - t.Error("runParallelTests() should return error when tests-split directory is missing") + t.Error("runParallel() should return error when tests-split directory is missing") } expectedMsg := "failed to read tests split directory" diff --git a/internal/runner/report.go b/internal/runner/report.go new file mode 100644 index 0000000..3687663 --- /dev/null +++ b/internal/runner/report.go @@ -0,0 +1,425 @@ +package runner + +import ( + "fmt" + "io" + "strconv" + "strings" + "time" + + ciConstants "github.com/DataDog/ddtest/civisibility/constants" + "github.com/DataDog/ddtest/civisibility/utils/net" +) + +const ( + runModeSequential = "sequential" + runModeParallel = "parallel" + runModeCINode = "CI node" +) + +type runInfoReport struct { + Service string `json:"service"` + Repository string `json:"repository"` + Commit string `json:"commit"` + Branch string `json:"branch"` + Platform string `json:"platform"` + Framework string `json:"framework"` + OSTags map[string]string `json:"osTags"` + RuntimeTags map[string]string `json:"runtimeTags"` +} + +func newRunInfoReport(ciTags map[string]string, runtimeTags map[string]string, platformName, frameworkName string) runInfoReport { + repository := ciTags[ciConstants.GitRepositoryURL] + return runInfoReport{ + Service: resolveServiceName(repository), + Repository: repository, + Commit: ciTags[ciConstants.GitCommitSHA], + Branch: ciTags[ciConstants.GitBranch], + Platform: platformName, + Framework: frameworkName, + OSTags: selectTags(runtimeTags, ciConstants.OSPlatform, ciConstants.OSArchitecture, ciConstants.OSVersion), + RuntimeTags: selectTags(runtimeTags, ciConstants.RuntimeName, ciConstants.RuntimeVersion), + } +} + +func (r runInfoReport) isZero() bool { + return r.Service == "" && + r.Repository == "" && + r.Commit == "" && + r.Branch == "" && + r.Platform == "" && + r.Framework == "" && + len(r.OSTags) == 0 && + len(r.RuntimeTags) == 0 +} + +type datadogSettingsReport struct { + Available bool + TestImpactAnalysis bool + TestSkipping bool + TestImpactCollection bool + KnownTests bool + ImpactedTests bool + EarlyFlakeDetection bool + AutoTestRetries bool + FlakyTestManagement bool +} + +func newDatadogSettingsReport(settings *net.SettingsResponseData) datadogSettingsReport { + if settings == nil { + return datadogSettingsReport{} + } + return datadogSettingsReport{ + Available: true, + TestImpactAnalysis: settings.ItrEnabled, + TestSkipping: settings.TestsSkipping, + TestImpactCollection: settings.CodeCoverage, + KnownTests: settings.KnownTestsEnabled, + ImpactedTests: settings.ImpactedTestsEnabled, + EarlyFlakeDetection: settings.EarlyFlakeDetection.Enabled, + AutoTestRetries: settings.FlakyTestRetriesEnabled, + FlakyTestManagement: settings.TestManagement.Enabled, + } +} + +type knownTestsReport struct { + Available bool + Modules int + Suites int + Tests int +} + +func newKnownTestsReport(knownTests *net.KnownTestsResponseData) knownTestsReport { + if knownTests == nil { + return knownTestsReport{} + } + + report := knownTestsReport{ + Available: true, + Modules: len(knownTests.Tests), + } + for _, suites := range knownTests.Tests { + report.Suites += len(suites) + for _, tests := range suites { + report.Tests += len(tests) + } + } + return report +} + +type managedFlakyTestsReport struct { + Available bool + Total int + Quarantined int + Disabled int + AttemptToFix int +} + +func newManagedFlakyTestsReport(testManagementTests *net.TestManagementTestsResponseDataModules) managedFlakyTestsReport { + if testManagementTests == nil { + return managedFlakyTestsReport{} + } + + report := managedFlakyTestsReport{Available: true} + for _, suites := range testManagementTests.Modules { + for _, tests := range suites.Suites { + for _, test := range tests.Tests { + report.Total++ + if test.Properties.Quarantined { + report.Quarantined++ + } + if test.Properties.Disabled { + report.Disabled++ + } + if test.Properties.AttemptToFix { + report.AttemptToFix++ + } + } + } + } + return report +} + +type durationSourceReport struct { + Known int + Default int +} + +type planningReport struct { + TestFilesDiscovered int + FullySkippedFiles int + TestFilesToRun int + DurationSources durationSourceReport + EstimatedTimeSaved float64 +} + +type planReport struct { + RunInfo runInfoReport + DatadogSettings datadogSettingsReport + KnownTests knownTestsReport + SkippableTestsCount int + ManagedFlakyTests managedFlakyTestsReport + Planning planningReport + Split splitScore +} + +type runExecutionReport struct { + Mode string + CINode int + LocalWorkers int + TestFilesRun int +} + +type runReport struct { + RunInfo runInfoReport + Execution runExecutionReport + Duration time.Duration + Err error +} + +func (tr *TestRunner) newPlanningReport() planningReport { + fullySkippedFiles := len(tr.testFiles) - len(tr.testFileWeights) + if fullySkippedFiles < 0 { + fullySkippedFiles = 0 + } + + return planningReport{ + TestFilesDiscovered: len(tr.testFiles), + FullySkippedFiles: fullySkippedFiles, + TestFilesToRun: len(tr.testFileWeights), + DurationSources: tr.durationSourceReport(), + EstimatedTimeSaved: tr.skippablePercentage, + } +} + +func (tr *TestRunner) durationSourceReport() durationSourceReport { + var report durationSourceReport + for _, source := range tr.testFileDurationSources { + switch source { + case testFileDurationSourceKnown: + report.Known++ + default: + report.Default++ + } + } + return report +} + +func printPlanReport(w io.Writer, report planReport) { + reportFprintln(w, "+++ DDTest: plan report") + reportFprintln(w) + printRunInfoReport(w, report.RunInfo) + reportFprintln(w) + printDatadogSettingsReport(w, report.DatadogSettings) + reportFprintln(w) + printBackendDataReport(w, report) + reportFprintln(w) + printPlanningReport(w, report.Planning) + reportFprintln(w) + printSplitReport(w, report.Split) +} + +func printRunReport(w io.Writer, report runReport) { + reportFprintln(w, "+++ DDTest: run report") + reportFprintln(w) + printRunInfoReport(w, report.RunInfo) + reportFprintln(w) + printExecutionReport(w, report) +} + +func printRunInfoReport(w io.Writer, report runInfoReport) { + reportFprintln(w, "Run") + reportFprintf(w, " Service: %s\n", valueOrNotAvailable(report.Service)) + reportFprintf(w, " Repository: %s\n", valueOrNotAvailable(report.Repository)) + reportFprintf(w, " Commit: %s\n", valueOrNotAvailable(report.Commit)) + reportFprintf(w, " Branch: %s\n", valueOrNotAvailable(report.Branch)) + reportFprintf(w, " Platform: %s\n", formatPlatform(report.Platform, report.Framework)) + reportFprintf(w, " OS tags: %s\n", formatTagList(report.OSTags, ciConstants.OSPlatform, ciConstants.OSArchitecture, ciConstants.OSVersion)) + reportFprintf(w, " Runtime tags: %s\n", formatTagList(report.RuntimeTags, ciConstants.RuntimeName, ciConstants.RuntimeVersion)) +} + +func printDatadogSettingsReport(w io.Writer, report datadogSettingsReport) { + reportFprintln(w, "Datadog") + if !report.Available { + reportFprintln(w, " Settings: not available") + return + } + + reportFprintf(w, " Test Impact Analysis: %s\n", enabledWord(report.TestImpactAnalysis)) + reportFprintf(w, " Test skipping: %s\n", enabledWord(report.TestSkipping)) + reportFprintf(w, " Test impact collection: %s\n", enabledWord(report.TestImpactCollection)) + reportFprintf(w, " Known tests: %s\n", enabledWord(report.KnownTests)) + reportFprintf(w, " Impacted tests: %s\n", enabledWord(report.ImpactedTests)) + reportFprintf(w, " Early flake detection: %s\n", enabledWord(report.EarlyFlakeDetection)) + reportFprintf(w, " Auto test retries: %s\n", enabledWord(report.AutoTestRetries)) + reportFprintf(w, " Flaky test management: %s\n", enabledWord(report.FlakyTestManagement)) +} + +func printBackendDataReport(w io.Writer, report planReport) { + reportFprintln(w, "Backend data") + reportFprintf(w, " Known tests: %s\n", formatKnownTests(report.DatadogSettings, report.KnownTests)) + reportFprintf(w, " Skippable tests for this run: %s\n", formatSkippableTests(report.DatadogSettings, report.SkippableTestsCount)) + reportFprintf(w, " Managed flaky tests: %s\n", formatManagedFlakyTests(report.DatadogSettings, report.ManagedFlakyTests)) +} + +func printPlanningReport(w io.Writer, report planningReport) { + reportFprintln(w, "Planning") + reportFprintf(w, " Test files discovered: %s\n", formatCount(report.TestFilesDiscovered)) + reportFprintf(w, " Fully skipped files: %s\n", formatCount(report.FullySkippedFiles)) + reportFprintf(w, " Test files to run: %s\n", formatCount(report.TestFilesToRun)) + reportFprintf(w, " Duration source: %s known, %s default\n", + formatCount(report.DurationSources.Known), + formatCount(report.DurationSources.Default)) + reportFprintf(w, " Estimated time saved: %.2f%%\n", report.EstimatedTimeSaved) +} + +func printSplitReport(w io.Writer, report splitScore) { + reportFprintln(w, "Split") + reportFprintf(w, " Runners: %s\n", formatCount(report.parallelRunners)) + reportFprintf(w, " Expected wall time: %s\n", formatDuration(report.wallTimeDuration())) + reportFprintf(w, " Imbalance: %s\n", formatDuration(report.imbalanceDuration())) + reportFprintf(w, " Total estimated runtime: %s\n", formatDuration(report.totalRuntimeDuration())) +} + +func printExecutionReport(w io.Writer, report runReport) { + reportFprintln(w, "Execution") + reportFprintf(w, " Mode: %s\n", valueOrNotAvailable(report.Execution.Mode)) + if report.Execution.Mode == runModeCINode { + reportFprintf(w, " CI node: %d\n", report.Execution.CINode) + } + reportFprintf(w, " Local workers: %s\n", formatCount(report.Execution.LocalWorkers)) + reportFprintf(w, " Test files run: %s\n", formatCount(report.Execution.TestFilesRun)) + reportFprintf(w, " Duration: %s\n", formatDuration(report.Duration)) + if report.Err == nil { + reportFprintln(w, " Result: passed") + return + } + reportFprintln(w, " Result: failed") + reportFprintf(w, " Error: %s\n", report.Err) +} + +func reportFprintln(w io.Writer, args ...any) { + _, _ = fmt.Fprintln(w, args...) +} + +func reportFprintf(w io.Writer, format string, args ...any) { + _, _ = fmt.Fprintf(w, format, args...) +} + +func formatKnownTests(settings datadogSettingsReport, known knownTestsReport) string { + if settings.Available && !settings.KnownTests { + return "disabled" + } + if !known.Available { + return "not available" + } + return fmt.Sprintf("%s modules, %s suites, %s tests", + formatCount(known.Modules), + formatCount(known.Suites), + formatCount(known.Tests)) +} + +func formatSkippableTests(settings datadogSettingsReport, count int) string { + if settings.Available && !settings.TestSkipping { + return "disabled" + } + return formatCount(count) +} + +func formatManagedFlakyTests(settings datadogSettingsReport, managed managedFlakyTestsReport) string { + if settings.Available && !settings.FlakyTestManagement { + return "disabled" + } + if !managed.Available { + return "not available" + } + return fmt.Sprintf("%s total, %s quarantined, %s disabled, %s attempt-to-fix", + formatCount(managed.Total), + formatCount(managed.Quarantined), + formatCount(managed.Disabled), + formatCount(managed.AttemptToFix)) +} + +func selectTags(tags map[string]string, keys ...string) map[string]string { + selected := make(map[string]string) + for _, key := range keys { + if value := tags[key]; value != "" { + selected[key] = value + } + } + return selected +} + +func formatTagList(tags map[string]string, keys ...string) string { + parts := make([]string, 0, len(keys)) + for _, key := range keys { + if value := tags[key]; value != "" { + parts = append(parts, key+"="+value) + } + } + if len(parts) == 0 { + return "not available" + } + return strings.Join(parts, ", ") +} + +func formatPlatform(platformName, frameworkName string) string { + switch { + case platformName == "" && frameworkName == "": + return "not available" + case platformName == "": + return frameworkName + case frameworkName == "": + return platformName + default: + return platformName + " / " + frameworkName + } +} + +func valueOrNotAvailable(value string) string { + if value == "" { + return "not available" + } + return value +} + +func enabledWord(enabled bool) string { + if enabled { + return "enabled" + } + return "disabled" +} + +func formatCount(count int) string { + sign := "" + if count < 0 { + sign = "-" + count = -count + } + + value := strconv.Itoa(count) + if len(value) <= 3 { + return sign + value + } + + prefixLength := len(value) % 3 + if prefixLength == 0 { + prefixLength = 3 + } + + var builder strings.Builder + builder.WriteString(sign) + builder.WriteString(value[:prefixLength]) + for i := prefixLength; i < len(value); i += 3 { + builder.WriteByte(',') + builder.WriteString(value[i : i+3]) + } + return builder.String() +} + +func formatDuration(duration time.Duration) string { + if duration < time.Millisecond { + return duration.String() + } + return duration.Round(time.Millisecond).String() +} diff --git a/internal/runner/report_test.go b/internal/runner/report_test.go new file mode 100644 index 0000000..12008e4 --- /dev/null +++ b/internal/runner/report_test.go @@ -0,0 +1,265 @@ +package runner + +import ( + "errors" + "strings" + "testing" + "time" + + "github.com/DataDog/ddtest/civisibility/utils/net" +) + +func TestPrintPlanReport_AllData(t *testing.T) { + var output strings.Builder + + printPlanReport(&output, planReport{ + RunInfo: runInfoReport{ + Service: "checkout-api", + Repository: "https://github.com/acme/checkout.git", + Commit: "9f3a1c7d2b4e", + Branch: "feature/split-report", + Platform: "ruby", + Framework: "rspec", + OSTags: map[string]string{ + "os.platform": "linux", + "os.architecture": "amd64", + "os.version": "6.8.0", + }, + RuntimeTags: map[string]string{ + "runtime.name": "ruby", + "runtime.version": "3.3.4", + }, + }, + DatadogSettings: datadogSettingsReport{ + Available: true, + TestImpactAnalysis: true, + TestSkipping: true, + TestImpactCollection: false, + KnownTests: true, + ImpactedTests: false, + EarlyFlakeDetection: true, + AutoTestRetries: true, + FlakyTestManagement: true, + }, + KnownTests: knownTestsReport{ + Available: true, + Modules: 4, + Suites: 1284, + Tests: 18921, + }, + SkippableTestsCount: 312, + ManagedFlakyTests: managedFlakyTestsReport{ + Available: true, + Total: 26, + Quarantined: 8, + Disabled: 3, + AttemptToFix: 5, + }, + Planning: planningReport{ + TestFilesDiscovered: 642, + FullySkippedFiles: 118, + TestFilesToRun: 524, + DurationSources: durationSourceReport{ + Known: 431, + Default: 90, + }, + EstimatedTimeSaved: 38.4, + }, + Split: splitScore{ + parallelRunners: 6, + wallTime: 252000, + imbalance: 11000, + totalRuntime: 1426000, + }, + }) + + expected := `+++ DDTest: plan report + +Run + Service: checkout-api + Repository: https://github.com/acme/checkout.git + Commit: 9f3a1c7d2b4e + Branch: feature/split-report + Platform: ruby / rspec + OS tags: os.platform=linux, os.architecture=amd64, os.version=6.8.0 + Runtime tags: runtime.name=ruby, runtime.version=3.3.4 + +Datadog + Test Impact Analysis: enabled + Test skipping: enabled + Test impact collection: disabled + Known tests: enabled + Impacted tests: disabled + Early flake detection: enabled + Auto test retries: enabled + Flaky test management: enabled + +Backend data + Known tests: 4 modules, 1,284 suites, 18,921 tests + Skippable tests for this run: 312 + Managed flaky tests: 26 total, 8 quarantined, 3 disabled, 5 attempt-to-fix + +Planning + Test files discovered: 642 + Fully skipped files: 118 + Test files to run: 524 + Duration source: 431 known, 90 default + Estimated time saved: 38.40% + +Split + Runners: 6 + Expected wall time: 4m12s + Imbalance: 11s + Total estimated runtime: 23m46s +` + if output.String() != expected { + t.Errorf("unexpected plan report:\n%s", output.String()) + } +} + +func TestPrintPlanReport_MissingSettingsAndData(t *testing.T) { + var output strings.Builder + + printPlanReport(&output, planReport{}) + + report := output.String() + if !strings.Contains(report, " Settings: not available") { + t.Errorf("expected missing settings message, got:\n%s", report) + } + if !strings.Contains(report, " Known tests: not available") { + t.Errorf("expected missing known tests message, got:\n%s", report) + } + if !strings.Contains(report, " Managed flaky tests: not available") { + t.Errorf("expected missing managed flaky tests message, got:\n%s", report) + } +} + +func TestPrintPlanReport_DisabledFeatures(t *testing.T) { + var output strings.Builder + + printPlanReport(&output, planReport{ + DatadogSettings: datadogSettingsReport{ + Available: true, + }, + }) + + report := output.String() + if !strings.Contains(report, " Known tests: disabled") { + t.Errorf("expected disabled known tests, got:\n%s", report) + } + if !strings.Contains(report, " Skippable tests for this run: disabled") { + t.Errorf("expected disabled skippable tests, got:\n%s", report) + } + if !strings.Contains(report, " Managed flaky tests: disabled") { + t.Errorf("expected disabled managed flaky tests, got:\n%s", report) + } +} + +func TestPrintRunReport_Passed(t *testing.T) { + var output strings.Builder + + printRunReport(&output, runReport{ + RunInfo: runInfoReport{ + Service: "checkout-api", + Repository: "https://github.com/acme/checkout.git", + Commit: "9f3a1c7d2b4e", + Branch: "feature/split-report", + Platform: "ruby", + Framework: "rspec", + OSTags: map[string]string{ + "os.platform": "linux", + "os.architecture": "amd64", + "os.version": "6.8.0", + }, + RuntimeTags: map[string]string{ + "runtime.name": "ruby", + "runtime.version": "3.3.4", + }, + }, + Execution: runExecutionReport{ + Mode: runModeCINode, + CINode: 2, + LocalWorkers: 2, + TestFilesRun: 87, + }, + Duration: 238 * time.Second, + }) + + expected := `+++ DDTest: run report + +Run + Service: checkout-api + Repository: https://github.com/acme/checkout.git + Commit: 9f3a1c7d2b4e + Branch: feature/split-report + Platform: ruby / rspec + OS tags: os.platform=linux, os.architecture=amd64, os.version=6.8.0 + Runtime tags: runtime.name=ruby, runtime.version=3.3.4 + +Execution + Mode: CI node + CI node: 2 + Local workers: 2 + Test files run: 87 + Duration: 3m58s + Result: passed +` + if output.String() != expected { + t.Errorf("unexpected run report:\n%s", output.String()) + } +} + +func TestPrintRunReport_Failed(t *testing.T) { + var output strings.Builder + + printRunReport(&output, runReport{ + Execution: runExecutionReport{ + Mode: runModeSequential, + LocalWorkers: 1, + TestFilesRun: 2, + }, + Err: errors.New("rspec exited with status 1"), + }) + + report := output.String() + if !strings.Contains(report, " Result: failed") || + !strings.Contains(report, " Error: rspec exited with status 1") { + t.Errorf("expected failed run report, got:\n%s", report) + } +} + +func TestReportSummaries(t *testing.T) { + known := newKnownTestsReport(&net.KnownTestsResponseData{ + Tests: net.KnownTestsResponseDataModules{ + "module-a": net.KnownTestsResponseDataSuites{ + "suite-a": []string{"test-a", "test-b"}, + }, + "module-b": net.KnownTestsResponseDataSuites{ + "suite-b": []string{"test-c"}, + "suite-c": []string{"test-d", "test-e"}, + }, + }, + }) + if known.Modules != 2 || known.Suites != 3 || known.Tests != 5 { + t.Errorf("unexpected known test summary: %+v", known) + } + + managed := newManagedFlakyTestsReport(&net.TestManagementTestsResponseDataModules{ + Modules: map[string]net.TestManagementTestsResponseDataSuites{ + "module-a": { + Suites: map[string]net.TestManagementTestsResponseDataTests{ + "suite-a": { + Tests: map[string]net.TestManagementTestsResponseDataTestProperties{ + "test-a": {Properties: net.TestManagementTestsResponseDataTestPropertiesAttributes{Quarantined: true}}, + "test-b": {Properties: net.TestManagementTestsResponseDataTestPropertiesAttributes{Disabled: true}}, + "test-c": {Properties: net.TestManagementTestsResponseDataTestPropertiesAttributes{AttemptToFix: true}}, + }, + }, + }, + }, + }, + }) + if managed.Total != 3 || managed.Quarantined != 1 || managed.Disabled != 1 || managed.AttemptToFix != 1 { + t.Errorf("unexpected managed flaky test summary: %+v", managed) + } +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 15f917b..24ce9a6 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -4,11 +4,14 @@ import ( "context" "errors" "fmt" + "io" "log/slog" "os" "slices" "strings" + "time" + ciUtils "github.com/DataDog/ddtest/civisibility/utils" "github.com/DataDog/ddtest/internal/ciprovider" "github.com/DataDog/ddtest/internal/constants" "github.com/DataDog/ddtest/internal/platform" @@ -22,31 +25,29 @@ type Runner interface { } type TestRunner struct { - testFiles map[string]struct{} - suiteAggregates map[testSuiteKey]testSuiteAggregate - suitesBySourceFile map[string][]testSuiteKey - testSuiteDurations map[string]map[string]testoptimization.TestSuiteDurationInfo - testFileWeights map[string]int - skippablePercentage float64 - platformDetector platform.PlatformDetector - optimizationClient testoptimization.TestOptimizationClient - durationsClient testoptimization.TestSuiteDurationsClient - ciProviderDetector ciprovider.CIProviderDetector + testFiles map[string]struct{} + suiteAggregates map[testSuiteKey]testSuiteAggregate + suitesBySourceFile map[string][]testSuiteKey + testSuiteDurations map[string]map[string]testoptimization.TestSuiteDurationInfo + testFileWeights map[string]int + testFileDurationSources map[string]testFileDurationSource + skippablePercentage float64 + planReport planReport + runInfoReport runInfoReport + platformDetector platform.PlatformDetector + optimizationClient testoptimization.TestOptimizationClient + durationsClient testoptimization.TestSuiteDurationsClient + ciProviderDetector ciprovider.CIProviderDetector + reportWriter io.Writer } func New() *TestRunner { - return &TestRunner{ - testFiles: make(map[string]struct{}), - suiteAggregates: make(map[testSuiteKey]testSuiteAggregate), - suitesBySourceFile: make(map[string][]testSuiteKey), - testSuiteDurations: make(map[string]map[string]testoptimization.TestSuiteDurationInfo), - testFileWeights: make(map[string]int), - skippablePercentage: 0.0, - platformDetector: platform.NewPlatformDetector(), - optimizationClient: testoptimization.NewDatadogClient(), - durationsClient: testoptimization.NewDurationsClient(), - ciProviderDetector: ciprovider.NewCIProviderDetector(), - } + runner := newTestRunnerWithDefaults() + runner.platformDetector = platform.NewPlatformDetector() + runner.optimizationClient = testoptimization.NewDatadogClient() + runner.durationsClient = testoptimization.NewDurationsClient() + runner.ciProviderDetector = ciprovider.NewCIProviderDetector() + return runner } func NewWithDependencies( @@ -55,17 +56,24 @@ func NewWithDependencies( durationsClient testoptimization.TestSuiteDurationsClient, ciProviderDetector ciprovider.CIProviderDetector, ) *TestRunner { + runner := newTestRunnerWithDefaults() + runner.platformDetector = platformDetector + runner.optimizationClient = optimizationClient + runner.durationsClient = durationsClient + runner.ciProviderDetector = ciProviderDetector + return runner +} + +func newTestRunnerWithDefaults() *TestRunner { return &TestRunner{ - testFiles: make(map[string]struct{}), - suiteAggregates: make(map[testSuiteKey]testSuiteAggregate), - suitesBySourceFile: make(map[string][]testSuiteKey), - testSuiteDurations: make(map[string]map[string]testoptimization.TestSuiteDurationInfo), - testFileWeights: make(map[string]int), - skippablePercentage: 0.0, - platformDetector: platformDetector, - optimizationClient: optimizationClient, - durationsClient: durationsClient, - ciProviderDetector: ciProviderDetector, + testFiles: make(map[string]struct{}), + suiteAggregates: make(map[testSuiteKey]testSuiteAggregate), + suitesBySourceFile: make(map[string][]testSuiteKey), + testSuiteDurations: make(map[string]map[string]testoptimization.TestSuiteDurationInfo), + testFileWeights: make(map[string]int), + testFileDurationSources: make(map[string]testFileDurationSource), + skippablePercentage: 0.0, + reportWriter: os.Stderr, } } @@ -80,8 +88,8 @@ func (tr *TestRunner) Plan(ctx context.Context) error { return fmt.Errorf("failed to write test optimization manifest: %w", err) } - if err := tr.storeTestSuiteDurationsCache(); err != nil { - return fmt.Errorf("failed to store test suite durations cache: %w", err) + if err := tr.storeTestOptimizationPlanCache(); err != nil { + return fmt.Errorf("failed to store test optimization plan cache: %w", err) } testFileNames := make([]string, 0, len(tr.testFileWeights)) @@ -133,6 +141,7 @@ func (tr *TestRunner) Plan(ctx context.Context) error { return fmt.Errorf("failed to create test splits: %w", err) } + tr.planReport.Split = parallelRunnerSplit slog.Info("Test execution planning completed", "parallelRunners", parallelRunners, "expectedWallTime", parallelRunnerSplit.wallTimeDuration(), @@ -140,6 +149,10 @@ func (tr *TestRunner) Plan(ctx context.Context) error { "expectedTotalRuntime", parallelRunnerSplit.totalRuntimeDuration(), "testFilesCount", len(tr.testFileWeights)) + if settings.GetReportEnabled() { + printPlanReport(tr.reportWriter, tr.planReport) + } + return nil } @@ -156,13 +169,13 @@ func (tr *TestRunner) Run(ctx context.Context) error { return fmt.Errorf("failed to check parallel runners count at %s: %w", constants.ParallelRunnersOutputPath, err) } - if err := tr.restoreTestSuiteDurationsCache(); err != nil { + if err := tr.restoreTestOptimizationPlanCache(); err != nil { if errors.Is(err, os.ErrNotExist) { - slog.Debug("Test suite durations cache not found; CI-node subsplits will use default weights", - "file", testoptimization.TestSuiteDurationsCacheFile) + slog.Debug("Test optimization plan cache not found; CI-node subsplits will use default weights", + "file", testoptimization.TestOptimizationPlanCacheFile) } else { - slog.Warn("Failed to restore test suite durations cache; CI-node subsplits will use default weights", - "file", testoptimization.TestSuiteDurationsCacheFile, "error", err) + slog.Warn("Failed to restore test optimization plan cache; CI-node subsplits will use default weights", + "file", testoptimization.TestOptimizationPlanCacheFile, "error", err) } } @@ -195,13 +208,29 @@ func (tr *TestRunner) Run(ctx context.Context) error { return fmt.Errorf("failed to detect framework: %w", err) } slog.Info("Framework detected", "framework", framework.Name()) + if tr.runInfoReport.isZero() { + tr.runInfoReport = newRunInfoReport(ciUtils.GetCITags(), nil, detectedPlatform.Name(), framework.Name()) + } ciNode := settings.GetCiNode() + startTime := time.Now() + executor := newTestExecutor(ctx, framework, workerEnvMap) + var executionResult runExecutionResult if ciNode >= 0 { - return runCINodeTests(ctx, framework, workerEnvMap, ciNode, settings.GetCiNodeWorkers(), tr.testFileWeights) + executionResult = executor.runCINode(ciNode, settings.GetCiNodeWorkers(), tr.testFileWeights) } else if parallelRunners > 1 { - return runParallelTests(ctx, framework, workerEnvMap) + executionResult = executor.runParallel() } else { - return runSequentialTests(ctx, framework, workerEnvMap) + executionResult = executor.runSequential() + } + + if settings.GetReportEnabled() { + printRunReport(tr.reportWriter, runReport{ + RunInfo: tr.runInfoReport, + Execution: executionResult.report, + Duration: time.Since(startTime), + Err: executionResult.err, + }) } + return executionResult.err } diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 3184523..dfa1752 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -129,12 +129,14 @@ func (m *MockFramework) GetRunTestsCalls() []RunTestsCall { // MockTestOptimizationClient mocks the test optimization client type MockTestOptimizationClient struct { - InitializeCalled bool - InitializeErr error - Settings *net.SettingsResponseData - SkippableTests map[string]bool - ShutdownCalled bool - Tags map[string]string + InitializeCalled bool + InitializeErr error + Settings *net.SettingsResponseData + SkippableTests map[string]bool + KnownTests *net.KnownTestsResponseData + TestManagementTests *net.TestManagementTestsResponseDataModules + ShutdownCalled bool + Tags map[string]string } func (m *MockTestOptimizationClient) Initialize(tags map[string]string) error { @@ -154,6 +156,14 @@ func (m *MockTestOptimizationClient) GetSkippableTests() map[string]bool { return m.SkippableTests } +func (m *MockTestOptimizationClient) GetKnownTests() *net.KnownTestsResponseData { + return m.KnownTests +} + +func (m *MockTestOptimizationClient) GetTestManagementTestsData() *net.TestManagementTestsResponseDataModules { + return m.TestManagementTests +} + func (m *MockTestOptimizationClient) StoreCacheAndExit() { m.ShutdownCalled = true } @@ -431,6 +441,52 @@ func TestTestRunner_Plan_WritesManifestAndRunnerLayout(t *testing.T) { assertFileContent(t, filepath.Join(constants.LegacyTestsSplitDir, "runner-0"), expectedTestFiles) } +func TestTestRunner_Plan_DoesNotPrintReportWhenDisabled(t *testing.T) { + tempDir := t.TempDir() + oldWd, _ := os.Getwd() + defer func() { _ = os.Chdir(oldWd) }() + _ = os.Chdir(tempDir) + + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "1") + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "1") + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_REPORT_ENABLED", "false") + defer func() { + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM") + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM") + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_REPORT_ENABLED") + settings.Init() + }() + settings.Init() + + mockFramework := &MockFramework{ + FrameworkName: "rspec", + Tests: []testoptimization.Test{ + {Suite: "TestSuite1", Name: "test1", Parameters: "", SuiteSourceFile: "test/file1_test.rb"}, + }, + } + mockPlatform := &MockPlatform{ + PlatformName: "ruby", + Tags: map[string]string{"platform": "ruby"}, + Framework: mockFramework, + } + + runner := NewWithDependencies( + &MockPlatformDetector{Platform: mockPlatform}, + &MockTestOptimizationClient{SkippableTests: map[string]bool{}}, + &MockTestSuiteDurationsClient{}, + newDefaultMockCIProviderDetector(), + ) + var output strings.Builder + runner.reportWriter = &output + + if err := runner.Plan(context.Background()); err != nil { + t.Fatalf("Plan() should not return error, got: %v", err) + } + if output.Len() != 0 { + t.Errorf("Expected no report output when report is disabled, got: %s", output.String()) + } +} + func TestTestRunner_Plan_ChoosesParallelismFromSplitNotSkippablePercentage(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() diff --git a/internal/runner/sequential_executor.go b/internal/runner/sequential_executor.go index 60749f4..87a0569 100644 --- a/internal/runner/sequential_executor.go +++ b/internal/runner/sequential_executor.go @@ -1,20 +1,33 @@ package runner import ( - "context" "fmt" "log/slog" "github.com/DataDog/ddtest/internal/constants" - "github.com/DataDog/ddtest/internal/framework" ) -// runSequentialTests executes tests in a single sequential runner. -func runSequentialTests(ctx context.Context, framework framework.Framework, workerEnvMap map[string]string) error { +// runSequential executes tests in a single sequential runner. +func (e testExecutor) runSequential() runExecutionResult { + report := runExecutionReport{ + Mode: runModeSequential, + LocalWorkers: 1, + } slog.Info("Running all tests in a single process") - if err := runTestBatchFromFile(ctx, framework, constants.TestFilesOutputPath, workerEnvMap, 0, 0); err != nil { - return fmt.Errorf("failed to run tests: %w", err) + testFiles, err := loadTestBatch(constants.TestFilesOutputPath) + if err != nil { + return report.failure(fmt.Errorf("failed to read test files from %s: %w", constants.TestFilesOutputPath, err)) + } + report.TestFilesRun = len(testFiles) + + if len(testFiles) == 0 { + slog.Info("No tests to run", "nodeIndex", 0, "workerIndex", 0) + return report.success() + } + + if err := e.runBatch(testFiles, 0, 0); err != nil { + return report.failure(fmt.Errorf("failed to run tests: %w", err)) } - return nil + return report.success() } diff --git a/internal/runner/sequential_executor_test.go b/internal/runner/sequential_executor_test.go index 4251496..315d943 100644 --- a/internal/runner/sequential_executor_test.go +++ b/internal/runner/sequential_executor_test.go @@ -11,7 +11,7 @@ import ( "github.com/DataDog/ddtest/internal/constants" ) -func TestRunSequentialTests_Success(t *testing.T) { +func TestRunSequential_Success(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() defer func() { _ = os.Chdir(oldWd) }() @@ -27,9 +27,13 @@ func TestRunSequentialTests_Success(t *testing.T) { RunTestsCalls: []RunTestsCall{}, } - err := runSequentialTests(context.Background(), mockFramework, map[string]string{}) + result := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runSequential() + report, err := result.report, result.err if err != nil { - t.Fatalf("runSequentialTests() should not return error, got: %v", err) + t.Fatalf("runSequential() should not return error, got: %v", err) + } + if report.TestFilesRun != 2 { + t.Errorf("Expected report to count 2 test files, got %d", report.TestFilesRun) } // Verify RunTests was called exactly once @@ -45,7 +49,7 @@ func TestRunSequentialTests_Success(t *testing.T) { } } -func TestRunSequentialTests_DoesNotReadLegacyTestFiles(t *testing.T) { +func TestRunSequential_DoesNotReadLegacyTestFiles(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() defer func() { _ = os.Chdir(oldWd) }() @@ -59,9 +63,10 @@ func TestRunSequentialTests_DoesNotReadLegacyTestFiles(t *testing.T) { RunTestsCalls: []RunTestsCall{}, } - err := runSequentialTests(context.Background(), mockFramework, map[string]string{}) + result := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runSequential() + err := result.err if err == nil { - t.Fatal("runSequentialTests() should return error when only the legacy test files list exists") + t.Fatal("runSequential() should return error when only the legacy test files list exists") } if !strings.Contains(err.Error(), constants.TestFilesOutputPath) { diff --git a/internal/runner/test_batch.go b/internal/runner/test_batch.go deleted file mode 100644 index 61e2270..0000000 --- a/internal/runner/test_batch.go +++ /dev/null @@ -1,61 +0,0 @@ -package runner - -import ( - "context" - "fmt" - "log/slog" - "os" - "strings" - - "github.com/DataDog/ddtest/internal/framework" -) - -// runTestBatchFromFile reads a test batch from the given file path and runs it using the framework. -func runTestBatchFromFile(ctx context.Context, framework framework.Framework, filePath string, workerEnvMap map[string]string, nodeIndex int, workerIndex int) error { - slog.Info("Reading prepared files list", "filePath", filePath, "nodeIndex", nodeIndex, "workerIndex", workerIndex) - - testFiles, err := loadTestBatch(filePath) - if err != nil { - return fmt.Errorf("failed to read test files from %s: %w", filePath, err) - } - - if len(testFiles) > 0 { - return runTestBatch(ctx, framework, testFiles, workerEnvMap, nodeIndex, workerIndex) - } - - slog.Info("No tests to run", "nodeIndex", nodeIndex, "workerIndex", workerIndex) - return nil -} - -// runTestBatch executes an already selected batch of test files in one worker. -func runTestBatch(ctx context.Context, framework framework.Framework, testFiles []string, workerEnvMap map[string]string, nodeIndex int, workerIndex int) error { - workerEnv := createWorkerEnv(workerEnvMap, nodeIndex, workerIndex) - - slog.Info("Running tests in worker", "nodeIndex", nodeIndex, "workerIndex", workerIndex, "testFilesCount", len(testFiles), "workerEnv", workerEnv) - return framework.RunTests(ctx, testFiles, workerEnv) -} - -// loadTestBatch reads a file containing test file paths (one per line) -// and returns them as a slice of strings. -func loadTestBatch(filePath string) ([]string, error) { - data, err := os.ReadFile(filePath) - if err != nil { - return nil, err - } - - content := strings.TrimSpace(string(data)) - if content == "" { - return []string{}, nil - } - - lines := strings.Split(content, "\n") - testFiles := make([]string, 0, len(lines)) - for _, line := range lines { - line = strings.TrimSpace(line) - if line != "" { - testFiles = append(testFiles, line) - } - } - - return testFiles, nil -} diff --git a/internal/runner/test_batch_executor.go b/internal/runner/test_batch_executor.go new file mode 100644 index 0000000..c98a956 --- /dev/null +++ b/internal/runner/test_batch_executor.go @@ -0,0 +1,70 @@ +package runner + +import ( + "context" + "log/slog" + "os" + "strings" + + "github.com/DataDog/ddtest/internal/framework" +) + +type testExecutor struct { + ctx context.Context + framework framework.Framework + workerEnvMap map[string]string +} + +func newTestExecutor(ctx context.Context, framework framework.Framework, workerEnvMap map[string]string) testExecutor { + return testExecutor{ + ctx: ctx, + framework: framework, + workerEnvMap: workerEnvMap, + } +} + +type runExecutionResult struct { + report runExecutionReport + err error +} + +func (r runExecutionReport) success() runExecutionResult { + return runExecutionResult{report: r} +} + +func (r runExecutionReport) failure(err error) runExecutionResult { + return runExecutionResult{report: r, err: err} +} + +// runBatch executes an already selected batch of test files in one worker. +func (e testExecutor) runBatch(testFiles []string, nodeIndex int, workerIndex int) error { + workerEnv := createWorkerEnv(e.workerEnvMap, nodeIndex, workerIndex) + + slog.Info("Running tests in worker", "nodeIndex", nodeIndex, "workerIndex", workerIndex, "testFilesCount", len(testFiles), "workerEnv", workerEnv) + return e.framework.RunTests(e.ctx, testFiles, workerEnv) +} + +// loadTestBatch reads a file containing test file paths (one per line) +// and returns them as a slice of strings. +func loadTestBatch(filePath string) ([]string, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + + content := strings.TrimSpace(string(data)) + if content == "" { + return []string{}, nil + } + + lines := strings.Split(content, "\n") + testFiles := make([]string, 0, len(lines)) + for _, line := range lines { + line = strings.TrimSpace(line) + if line != "" { + testFiles = append(testFiles, line) + } + } + + return testFiles, nil +} diff --git a/internal/runner/test_batch_test.go b/internal/runner/test_batch_test.go index a9e02cc..94f1700 100644 --- a/internal/runner/test_batch_test.go +++ b/internal/runner/test_batch_test.go @@ -7,58 +7,6 @@ import ( "testing" ) -func TestRunTestBatchFromFile_WithWorkerEnv(t *testing.T) { - tempDir := t.TempDir() - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - _ = os.Chdir(tempDir) - - // Create test file - testFile := "test/file1_test.rb\n" - _ = os.WriteFile("test-list.txt", []byte(testFile), 0644) - - mockFramework := &MockFramework{ - FrameworkName: "rspec", - RunTestsCalls: []RunTestsCall{}, - } - - workerEnvMap := map[string]string{ - "NODE_INDEX": "{{nodeIndex}}", - "WORKER_INDEX": "{{workerIndex}}", - "WORKER_RESOURCES": "node_{{nodeIndex}}_worker_{{workerIndex}}", - "BUILD_ID": "123", - } - - err := runTestBatchFromFile(context.Background(), mockFramework, "test-list.txt", workerEnvMap, 4, 5) - if err != nil { - t.Fatalf("runTestBatchFromFile() should not return error, got: %v", err) - } - - // Verify RunTests was called - if mockFramework.GetRunTestsCallsCount() != 1 { - t.Fatalf("Expected RunTests to be called once, got %d calls", mockFramework.GetRunTestsCallsCount()) - } - - calls := mockFramework.GetRunTestsCalls() - call := calls[0] - - // Verify nodeIndex identifies the machine, and workerIndex identifies the process on that machine. - if call.EnvMap["NODE_INDEX"] != "4" { - t.Errorf("Expected NODE_INDEX=4, got %s", call.EnvMap["NODE_INDEX"]) - } - if call.EnvMap["WORKER_INDEX"] != "5" { - t.Errorf("Expected WORKER_INDEX=5, got %s", call.EnvMap["WORKER_INDEX"]) - } - if call.EnvMap["WORKER_RESOURCES"] != "node_4_worker_5" { - t.Errorf("Expected WORKER_RESOURCES=node_4_worker_5, got %s", call.EnvMap["WORKER_RESOURCES"]) - } - - // Verify other env vars preserved - if call.EnvMap["BUILD_ID"] != "123" { - t.Errorf("Expected BUILD_ID=123, got %s", call.EnvMap["BUILD_ID"]) - } -} - func TestLoadTestBatch_EmptyFile(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() @@ -97,7 +45,7 @@ func TestLoadTestBatch_WithContent(t *testing.T) { } } -func TestRunTestBatch(t *testing.T) { +func TestRunBatch(t *testing.T) { mockFramework := &MockFramework{ FrameworkName: "rspec", RunTestsCalls: []RunTestsCall{}, @@ -111,9 +59,9 @@ func TestRunTestBatch(t *testing.T) { "STATIC": "value", } - err := runTestBatch(context.Background(), mockFramework, testFiles, workerEnvMap, 5, 3) + err := newTestExecutor(context.Background(), mockFramework, workerEnvMap).runBatch(testFiles, 5, 3) if err != nil { - t.Fatalf("runTestBatch() should not return error, got: %v", err) + t.Fatalf("runBatch() should not return error, got: %v", err) } if mockFramework.GetRunTestsCallsCount() != 1 { diff --git a/internal/runner/test_optimization_plan_cache.go b/internal/runner/test_optimization_plan_cache.go new file mode 100644 index 0000000..322bdc8 --- /dev/null +++ b/internal/runner/test_optimization_plan_cache.go @@ -0,0 +1,93 @@ +package runner + +import ( + "log/slog" + + "github.com/DataDog/ddtest/internal/testoptimization" +) + +type testOptimizationPlanCache struct { + TestSuiteDurations map[string]map[string]testoptimization.TestSuiteDurationInfo `json:"testSuiteDurations"` + SuiteAggregates map[testSuiteKey]testSuiteAggregate `json:"suiteAggregates"` + SuitesBySourceFile map[string][]testSuiteKey `json:"suitesBySourceFile"` + TestFileWeights map[string]int `json:"testFileWeights"` + TestFileDurationSources map[string]testFileDurationSource `json:"testFileDurationSources"` + RunInfo runInfoReport `json:"runInfo"` +} + +func (tr *TestRunner) storeTestOptimizationPlanCache() error { + cache := testOptimizationPlanCache{ + TestSuiteDurations: tr.testSuiteDurations, + SuiteAggregates: tr.suiteAggregates, + SuitesBySourceFile: tr.suitesBySourceFile, + TestFileWeights: tr.testFileWeights, + TestFileDurationSources: tr.testFileDurationSources, + RunInfo: tr.runInfoReport, + } + + return testoptimization.NewCacheManager().StoreTestOptimizationPlanCache(cache) +} + +func (tr *TestRunner) restoreTestOptimizationPlanCache() error { + var cache testOptimizationPlanCache + if err := testoptimization.NewCacheManager().ReadTestOptimizationPlanCache(&cache); err != nil { + return err + } + + tr.testSuiteDurations = cache.TestSuiteDurations + if tr.testSuiteDurations == nil { + tr.testSuiteDurations = make(map[string]map[string]testoptimization.TestSuiteDurationInfo) + } + + tr.suiteAggregates = cache.SuiteAggregates + if tr.suiteAggregates == nil { + tr.suiteAggregates = make(map[testSuiteKey]testSuiteAggregate) + } + + tr.suitesBySourceFile = cache.SuitesBySourceFile + if tr.suitesBySourceFile == nil { + tr.suitesBySourceFile = indexSuitesBySourceFile(tr.suiteAggregates) + } + + tr.testFileWeights = cache.TestFileWeights + if tr.testFileWeights == nil { + tr.testFileWeights = tr.testFileWeightsFromSuites() + } + + tr.testFileDurationSources = cache.TestFileDurationSources + if tr.testFileDurationSources == nil { + tr.testFileDurationSources = make(map[string]testFileDurationSource) + } + + tr.runInfoReport = cache.RunInfo + + testSuitesCount := countTestSuites(tr.testSuiteDurations) + suiteAggregatesCount := len(tr.suiteAggregates) + suitesBySourceFileCount := len(tr.suitesBySourceFile) + testFileWeightsCount := len(tr.testFileWeights) + slog.Info("Restored test optimization plan cache", + "objectsCount", testSuitesCount+suiteAggregatesCount+suitesBySourceFileCount+testFileWeightsCount, + "modulesCount", len(tr.testSuiteDurations), + "testSuitesCount", testSuitesCount, + "suiteAggregatesCount", suiteAggregatesCount, + "suitesBySourceFileCount", suitesBySourceFileCount, + "testFileWeightsCount", testFileWeightsCount) + + return nil +} + +func countTestSuites(testSuiteDurations map[string]map[string]testoptimization.TestSuiteDurationInfo) int { + totalSuites := 0 + for _, suites := range testSuiteDurations { + totalSuites += len(suites) + } + return totalSuites +} + +func (tr *TestRunner) testFileWeightsFromSuites() map[string]int { + testFiles := make(map[string]struct{}, len(tr.suitesBySourceFile)) + for testFile := range tr.suitesBySourceFile { + testFiles[testFile] = struct{}{} + } + return tr.estimateTestFileWeights(testFiles) +} diff --git a/internal/runner/test_suite_durations_cache_test.go b/internal/runner/test_optimization_plan_cache_test.go similarity index 83% rename from internal/runner/test_suite_durations_cache_test.go rename to internal/runner/test_optimization_plan_cache_test.go index faa7cf8..dca4958 100644 --- a/internal/runner/test_suite_durations_cache_test.go +++ b/internal/runner/test_optimization_plan_cache_test.go @@ -13,7 +13,7 @@ import ( "github.com/DataDog/ddtest/internal/testoptimization" ) -func TestTestRunner_Plan_StoresTestSuiteDurationsCache(t *testing.T) { +func TestTestRunner_Plan_StoresTestOptimizationPlanCache(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() defer func() { _ = os.Chdir(oldWd) }() @@ -51,9 +51,9 @@ func TestTestRunner_Plan_StoresTestSuiteDurationsCache(t *testing.T) { t.Fatalf("Plan() should not return error, got: %v", err) } - cachePath := filepath.Join(constants.RunnerCacheDir, testoptimization.TestSuiteDurationsCacheFile) + cachePath := filepath.Join(constants.RunnerCacheDir, testoptimization.TestOptimizationPlanCacheFile) if _, err := os.Stat(cachePath); err != nil { - t.Fatalf("Expected test suite durations cache file to be written: %v", err) + t.Fatalf("Expected test optimization plan cache file to be written: %v", err) } restored := NewWithDependencies( @@ -62,8 +62,8 @@ func TestTestRunner_Plan_StoresTestSuiteDurationsCache(t *testing.T) { &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector(), ) - if err := restored.restoreTestSuiteDurationsCache(); err != nil { - t.Fatalf("restoreTestSuiteDurationsCache() should not return error, got: %v", err) + if err := restored.restoreTestOptimizationPlanCache(); err != nil { + t.Fatalf("restoreTestOptimizationPlanCache() should not return error, got: %v", err) } if !reflect.DeepEqual(restored.suiteAggregates, runner.suiteAggregates) { @@ -77,7 +77,7 @@ func TestTestRunner_Plan_StoresTestSuiteDurationsCache(t *testing.T) { } } -func TestTestRunner_StoreAndRestoreTestSuiteDurationsCache_RoundTripDurations(t *testing.T) { +func TestTestRunner_StoreAndRestoreTestOptimizationPlanCache_RoundTripDurations(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() defer func() { _ = os.Chdir(oldWd) }() @@ -115,8 +115,8 @@ func TestTestRunner_StoreAndRestoreTestSuiteDurationsCache_RoundTripDurations(t "spec/suite1_spec.rb": 2500, } - if err := runner.storeTestSuiteDurationsCache(); err != nil { - t.Fatalf("storeTestSuiteDurationsCache() should not return error, got: %v", err) + if err := runner.storeTestOptimizationPlanCache(); err != nil { + t.Fatalf("storeTestOptimizationPlanCache() should not return error, got: %v", err) } logs := captureLogs(t) @@ -126,8 +126,8 @@ func TestTestRunner_StoreAndRestoreTestSuiteDurationsCache_RoundTripDurations(t &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector(), ) - if err := restored.restoreTestSuiteDurationsCache(); err != nil { - t.Fatalf("restoreTestSuiteDurationsCache() should not return error, got: %v", err) + if err := restored.restoreTestOptimizationPlanCache(); err != nil { + t.Fatalf("restoreTestOptimizationPlanCache() should not return error, got: %v", err) } if !reflect.DeepEqual(restored.testSuiteDurations, runner.testSuiteDurations) { @@ -145,7 +145,7 @@ func TestTestRunner_StoreAndRestoreTestSuiteDurationsCache_RoundTripDurations(t logOutput := logs.String() if !strings.Contains(logOutput, "level=INFO") || - !strings.Contains(logOutput, "Restored test suite durations cache") || + !strings.Contains(logOutput, "Restored test optimization plan cache") || !strings.Contains(logOutput, "objectsCount=4") || !strings.Contains(logOutput, "modulesCount=1") || !strings.Contains(logOutput, "testSuitesCount=1") || @@ -156,19 +156,19 @@ func TestTestRunner_StoreAndRestoreTestSuiteDurationsCache_RoundTripDurations(t } } -func TestTestRunner_RestoreTestSuiteDurationsCache_ComputesWeightsForLegacyCache(t *testing.T) { +func TestTestRunner_RestoreTestOptimizationPlanCache_ComputesWeightsForLegacyCache(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() defer func() { _ = os.Chdir(oldWd) }() _ = os.Chdir(tempDir) - type legacyTestSuiteDurationsCache struct { + type legacyTestOptimizationPlanCache struct { TestSuiteDurations map[string]map[string]testoptimization.TestSuiteDurationInfo `json:"testSuiteDurations"` SuiteAggregates map[testSuiteKey]testSuiteAggregate `json:"suiteAggregates"` SuitesBySourceFile map[string][]testSuiteKey `json:"suitesBySourceFile"` } - cache := legacyTestSuiteDurationsCache{ + cache := legacyTestOptimizationPlanCache{ TestSuiteDurations: map[string]map[string]testoptimization.TestSuiteDurationInfo{ "rspec": { "Suite1": { @@ -193,8 +193,8 @@ func TestTestRunner_RestoreTestSuiteDurationsCache_ComputesWeightsForLegacyCache }, } - if err := testoptimization.NewCacheManager().StoreTestSuiteDurationsCache(cache); err != nil { - t.Fatalf("StoreTestSuiteDurationsCache() should not return error, got: %v", err) + if err := testoptimization.NewCacheManager().StoreTestOptimizationPlanCache(cache); err != nil { + t.Fatalf("StoreTestOptimizationPlanCache() should not return error, got: %v", err) } restored := NewWithDependencies( @@ -203,8 +203,8 @@ func TestTestRunner_RestoreTestSuiteDurationsCache_ComputesWeightsForLegacyCache &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector(), ) - if err := restored.restoreTestSuiteDurationsCache(); err != nil { - t.Fatalf("restoreTestSuiteDurationsCache() should not return error, got: %v", err) + if err := restored.restoreTestOptimizationPlanCache(); err != nil { + t.Fatalf("restoreTestOptimizationPlanCache() should not return error, got: %v", err) } expectedWeights := map[string]int{ @@ -240,8 +240,8 @@ func TestTestSuiteKey_JSONMapKeyRoundTrip(t *testing.T) { }, } - if err := runner.storeTestSuiteDurationsCache(); err != nil { - t.Fatalf("storeTestSuiteDurationsCache() should not return error, got: %v", err) + if err := runner.storeTestOptimizationPlanCache(); err != nil { + t.Fatalf("storeTestOptimizationPlanCache() should not return error, got: %v", err) } restored := NewWithDependencies( @@ -250,8 +250,8 @@ func TestTestSuiteKey_JSONMapKeyRoundTrip(t *testing.T) { &MockTestSuiteDurationsClient{}, newDefaultMockCIProviderDetector(), ) - if err := restored.restoreTestSuiteDurationsCache(); err != nil { - t.Fatalf("restoreTestSuiteDurationsCache() should not return error, got: %v", err) + if err := restored.restoreTestOptimizationPlanCache(); err != nil { + t.Fatalf("restoreTestOptimizationPlanCache() should not return error, got: %v", err) } if !reflect.DeepEqual(restored.suiteAggregates, runner.suiteAggregates) { diff --git a/internal/runner/test_suite_durations_cache.go b/internal/runner/test_suite_durations_cache.go deleted file mode 100644 index 7617fbd..0000000 --- a/internal/runner/test_suite_durations_cache.go +++ /dev/null @@ -1,85 +0,0 @@ -package runner - -import ( - "log/slog" - - "github.com/DataDog/ddtest/internal/testoptimization" -) - -type testSuiteDurationsCache struct { - TestSuiteDurations map[string]map[string]testoptimization.TestSuiteDurationInfo `json:"testSuiteDurations"` - SuiteAggregates map[testSuiteKey]testSuiteAggregate `json:"suiteAggregates"` - SuitesBySourceFile map[string][]testSuiteKey `json:"suitesBySourceFile"` - TestFileWeights map[string]int `json:"testFileWeights"` -} - -func (tr *TestRunner) storeTestSuiteDurationsCache() error { - cache := testSuiteDurationsCache{ - TestSuiteDurations: tr.testSuiteDurations, - SuiteAggregates: tr.suiteAggregates, - SuitesBySourceFile: tr.suitesBySourceFile, - TestFileWeights: tr.testFileWeights, - } - - return testoptimization.NewCacheManager().StoreTestSuiteDurationsCache(cache) -} - -func (tr *TestRunner) restoreTestSuiteDurationsCache() error { - var cache testSuiteDurationsCache - if err := testoptimization.NewCacheManager().ReadTestSuiteDurationsCache(&cache); err != nil { - return err - } - - tr.testSuiteDurations = cache.TestSuiteDurations - if tr.testSuiteDurations == nil { - tr.testSuiteDurations = make(map[string]map[string]testoptimization.TestSuiteDurationInfo) - } - - tr.suiteAggregates = cache.SuiteAggregates - if tr.suiteAggregates == nil { - tr.suiteAggregates = make(map[testSuiteKey]testSuiteAggregate) - } - - tr.suitesBySourceFile = cache.SuitesBySourceFile - if tr.suitesBySourceFile == nil { - tr.suitesBySourceFile = indexSuitesBySourceFile(tr.suiteAggregates) - } - - tr.testFileWeights = cache.TestFileWeights - if tr.testFileWeights == nil { - tr.testFileWeights = tr.testFileWeightsFromSuites() - } - - testSuitesCount := countTestSuites(tr.testSuiteDurations) - suiteAggregatesCount := len(tr.suiteAggregates) - suitesBySourceFileCount := len(tr.suitesBySourceFile) - testFileWeightsCount := len(tr.testFileWeights) - slog.Info("Restored test suite durations cache", - "objectsCount", testSuitesCount+suiteAggregatesCount+suitesBySourceFileCount+testFileWeightsCount, - "modulesCount", len(tr.testSuiteDurations), - "testSuitesCount", testSuitesCount, - "suiteAggregatesCount", suiteAggregatesCount, - "suitesBySourceFileCount", suitesBySourceFileCount, - "testFileWeightsCount", testFileWeightsCount) - - return nil -} - -func countTestSuites(testSuiteDurations map[string]map[string]testoptimization.TestSuiteDurationInfo) int { - totalSuites := 0 - for _, suites := range testSuiteDurations { - totalSuites += len(suites) - } - return totalSuites -} - -func (tr *TestRunner) testFileWeightsFromSuites() map[string]int { - testFileWeights := make(map[string]int, len(tr.suitesBySourceFile)) - for testFile := range tr.suitesBySourceFile { - weight, ok := tr.testFileWeight(testFile) - if ok { - testFileWeights[testFile] = weight - } - } - return testFileWeights -} diff --git a/internal/runner/worker_env_test.go b/internal/runner/worker_env_test.go index 8d3bc0e..399c278 100644 --- a/internal/runner/worker_env_test.go +++ b/internal/runner/worker_env_test.go @@ -44,7 +44,7 @@ func chdirForTest(t *testing.T, dir string) { }) } -func TestRunTestBatch_DefaultTestSessionName(t *testing.T) { +func TestRunBatch_DefaultTestSessionName(t *testing.T) { ciUtils.ResetCITags() t.Cleanup(ciUtils.ResetCITags) unsetEnvForTest(t, ciConstants.CIVisibilityTestSessionNameEnvironmentVariable) @@ -56,9 +56,9 @@ func TestRunTestBatch_DefaultTestSessionName(t *testing.T) { RunTestsCalls: []RunTestsCall{}, } - err := runTestBatch(context.Background(), mockFramework, []string{"test/file1_test.rb"}, map[string]string{}, 2, 4) + err := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runBatch([]string{"test/file1_test.rb"}, 2, 4) if err != nil { - t.Fatalf("runTestBatch() should not return error, got: %v", err) + t.Fatalf("runBatch() should not return error, got: %v", err) } call := mockFramework.GetRunTestsCalls()[0] @@ -68,7 +68,7 @@ func TestRunTestBatch_DefaultTestSessionName(t *testing.T) { } } -func TestRunTestBatch_DefaultTestSessionNameUsesDDService(t *testing.T) { +func TestRunBatch_DefaultTestSessionNameUsesDDService(t *testing.T) { ciUtils.ResetCITags() t.Cleanup(ciUtils.ResetCITags) unsetEnvForTest(t, ciConstants.CIVisibilityTestSessionNameEnvironmentVariable) @@ -80,9 +80,9 @@ func TestRunTestBatch_DefaultTestSessionNameUsesDDService(t *testing.T) { RunTestsCalls: []RunTestsCall{}, } - err := runTestBatch(context.Background(), mockFramework, []string{"test/file1_test.rb"}, map[string]string{}, 3, 7) + err := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runBatch([]string{"test/file1_test.rb"}, 3, 7) if err != nil { - t.Fatalf("runTestBatch() should not return error, got: %v", err) + t.Fatalf("runBatch() should not return error, got: %v", err) } call := mockFramework.GetRunTestsCalls()[0] @@ -92,7 +92,7 @@ func TestRunTestBatch_DefaultTestSessionNameUsesDDService(t *testing.T) { } } -func TestRunTestBatch_UserTestSessionNameSupportsPlaceholders(t *testing.T) { +func TestRunBatch_UserTestSessionNameSupportsPlaceholders(t *testing.T) { ciUtils.ResetCITags() t.Cleanup(ciUtils.ResetCITags) t.Setenv(ciConstants.CIVisibilityTestSessionNameEnvironmentVariable, "custom-node-{{nodeIndex}}-worker-{{workerIndex}}") @@ -102,9 +102,9 @@ func TestRunTestBatch_UserTestSessionNameSupportsPlaceholders(t *testing.T) { RunTestsCalls: []RunTestsCall{}, } - err := runTestBatch(context.Background(), mockFramework, []string{"test/file1_test.rb"}, map[string]string{}, 5, 8) + err := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runBatch([]string{"test/file1_test.rb"}, 5, 8) if err != nil { - t.Fatalf("runTestBatch() should not return error, got: %v", err) + t.Fatalf("runBatch() should not return error, got: %v", err) } call := mockFramework.GetRunTestsCalls()[0] @@ -114,7 +114,7 @@ func TestRunTestBatch_UserTestSessionNameSupportsPlaceholders(t *testing.T) { } } -func TestRunTestBatch_WorkerEnvTestSessionNameTakesPrecedence(t *testing.T) { +func TestRunBatch_WorkerEnvTestSessionNameTakesPrecedence(t *testing.T) { ciUtils.ResetCITags() t.Cleanup(ciUtils.ResetCITags) t.Setenv(ciConstants.CIVisibilityTestSessionNameEnvironmentVariable, "outer-session") @@ -127,9 +127,9 @@ func TestRunTestBatch_WorkerEnvTestSessionNameTakesPrecedence(t *testing.T) { ciConstants.CIVisibilityTestSessionNameEnvironmentVariable: "worker-node-{{nodeIndex}}-worker-{{workerIndex}}", } - err := runTestBatch(context.Background(), mockFramework, []string{"test/file1_test.rb"}, workerEnvMap, 9, 1) + err := newTestExecutor(context.Background(), mockFramework, workerEnvMap).runBatch([]string{"test/file1_test.rb"}, 9, 1) if err != nil { - t.Fatalf("runTestBatch() should not return error, got: %v", err) + t.Fatalf("runBatch() should not return error, got: %v", err) } call := mockFramework.GetRunTestsCalls()[0] @@ -139,7 +139,7 @@ func TestRunTestBatch_WorkerEnvTestSessionNameTakesPrecedence(t *testing.T) { } } -func TestRunTestBatch_DefaultManifestFile(t *testing.T) { +func TestRunBatch_DefaultManifestFile(t *testing.T) { chdirForTest(t, t.TempDir()) unsetEnvForTest(t, constants.TestOptimizationManifestFileEnvVar) unsetEnvForTest(t, "DD_TEST_OPTIMIZATION_PAYLOADS_IN_FILES") @@ -149,9 +149,9 @@ func TestRunTestBatch_DefaultManifestFile(t *testing.T) { RunTestsCalls: []RunTestsCall{}, } - err := runTestBatch(context.Background(), mockFramework, []string{"test/file1_test.rb"}, map[string]string{}, 2, 4) + err := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runBatch([]string{"test/file1_test.rb"}, 2, 4) if err != nil { - t.Fatalf("runTestBatch() should not return error, got: %v", err) + t.Fatalf("runBatch() should not return error, got: %v", err) } call := mockFramework.GetRunTestsCalls()[0] @@ -172,7 +172,7 @@ func TestRunTestBatch_DefaultManifestFile(t *testing.T) { } } -func TestRunTestBatch_ManifestFileUsesProcessEnv(t *testing.T) { +func TestRunBatch_ManifestFileUsesProcessEnv(t *testing.T) { t.Setenv(constants.TestOptimizationManifestFileEnvVar, "/tmp/custom-manifest.txt") mockFramework := &MockFramework{ @@ -180,9 +180,9 @@ func TestRunTestBatch_ManifestFileUsesProcessEnv(t *testing.T) { RunTestsCalls: []RunTestsCall{}, } - err := runTestBatch(context.Background(), mockFramework, []string{"test/file1_test.rb"}, map[string]string{}, 2, 4) + err := newTestExecutor(context.Background(), mockFramework, map[string]string{}).runBatch([]string{"test/file1_test.rb"}, 2, 4) if err != nil { - t.Fatalf("runTestBatch() should not return error, got: %v", err) + t.Fatalf("runBatch() should not return error, got: %v", err) } call := mockFramework.GetRunTestsCalls()[0] @@ -191,7 +191,7 @@ func TestRunTestBatch_ManifestFileUsesProcessEnv(t *testing.T) { } } -func TestRunTestBatch_WorkerEnvManifestFileTakesPrecedence(t *testing.T) { +func TestRunBatch_WorkerEnvManifestFileTakesPrecedence(t *testing.T) { t.Setenv(constants.TestOptimizationManifestFileEnvVar, "/tmp/process-manifest.txt") mockFramework := &MockFramework{ @@ -202,9 +202,9 @@ func TestRunTestBatch_WorkerEnvManifestFileTakesPrecedence(t *testing.T) { constants.TestOptimizationManifestFileEnvVar: "/tmp/worker-manifest.txt", } - err := runTestBatch(context.Background(), mockFramework, []string{"test/file1_test.rb"}, workerEnvMap, 2, 4) + err := newTestExecutor(context.Background(), mockFramework, workerEnvMap).runBatch([]string{"test/file1_test.rb"}, 2, 4) if err != nil { - t.Fatalf("runTestBatch() should not return error, got: %v", err) + t.Fatalf("runBatch() should not return error, got: %v", err) } call := mockFramework.GetRunTestsCalls()[0] diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 8428e13..064e1d4 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -81,6 +81,7 @@ type Config struct { Command string `mapstructure:"command"` TestsLocation string `mapstructure:"tests_location"` RuntimeTags string `mapstructure:"runtime_tags"` + ReportEnabled bool `mapstructure:"report_enabled"` } var ( @@ -119,6 +120,7 @@ func setDefaults() { viper.SetDefault("command", "") viper.SetDefault("tests_location", "") viper.SetDefault("runtime_tags", "") + viper.SetDefault("report_enabled", true) } // ParseCiNodeWorkers resolves the ci_node_workers setting from either a positive integer @@ -189,6 +191,10 @@ func GetRuntimeTags() string { return Get().RuntimeTags } +func GetReportEnabled() bool { + return Get().ReportEnabled +} + // GetRuntimeTagsMap parses the runtime_tags setting as JSON and returns it as a map. // Returns nil if runtime_tags is empty or not set. // Returns an error if the JSON is invalid. diff --git a/internal/settings/settings_test.go b/internal/settings/settings_test.go index c376d8c..64dd39a 100644 --- a/internal/settings/settings_test.go +++ b/internal/settings/settings_test.go @@ -129,6 +129,9 @@ func TestInit(t *testing.T) { if config.RuntimeTags != "" { t.Errorf("expected default runtime_tags to be empty, got %q", config.RuntimeTags) } + if !config.ReportEnabled { + t.Error("expected default report_enabled to be true") + } } func TestSetDefaults(t *testing.T) { @@ -167,6 +170,9 @@ func TestSetDefaults(t *testing.T) { if viper.GetString("runtime_tags") != "" { t.Errorf("expected default runtime_tags to be empty, got %q", viper.GetString("runtime_tags")) } + if !viper.GetBool("report_enabled") { + t.Error("expected default report_enabled to be true") + } } func TestGet(t *testing.T) { @@ -239,6 +245,7 @@ func TestEnvironmentVariables(t *testing.T) { _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_COMMAND", "bundle exec rspec") _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_TESTS_LOCATION", "spec/**/*_spec.rb") _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_RUNTIME_TAGS", `{"os.platform":"linux","runtime.version":"3.2.0"}`) + _ = os.Setenv("DD_TEST_OPTIMIZATION_RUNNER_REPORT_ENABLED", "false") defer func() { _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_PLATFORM") _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_FRAMEWORK") @@ -250,6 +257,7 @@ func TestEnvironmentVariables(t *testing.T) { _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_COMMAND") _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_TESTS_LOCATION") _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_RUNTIME_TAGS") + _ = os.Unsetenv("DD_TEST_OPTIMIZATION_RUNNER_REPORT_ENABLED") }() Init() @@ -284,6 +292,9 @@ func TestEnvironmentVariables(t *testing.T) { if config.RuntimeTags != `{"os.platform":"linux","runtime.version":"3.2.0"}` { t.Errorf("expected runtime_tags from env var to be JSON string, got %q", config.RuntimeTags) } + if config.ReportEnabled { + t.Error("expected report_enabled from env var to be false") + } } func TestGetMinParallelism(t *testing.T) { @@ -390,6 +401,20 @@ func TestGetRuntimeTags(t *testing.T) { } } +func TestGetReportEnabled(t *testing.T) { + config = nil + viper.Reset() + + if !GetReportEnabled() { + t.Error("expected report_enabled to be true by default") + } + + config = &Config{ReportEnabled: false} + if GetReportEnabled() { + t.Error("expected report_enabled to be false") + } +} + func TestGetRuntimeTagsMap(t *testing.T) { t.Run("empty runtime tags", func(t *testing.T) { config = &Config{RuntimeTags: ""} diff --git a/internal/testoptimization/cache.go b/internal/testoptimization/cache.go index 15624a8..2b7b2dd 100644 --- a/internal/testoptimization/cache.go +++ b/internal/testoptimization/cache.go @@ -19,8 +19,8 @@ type SkippableTestsCache struct { SkippableTests map[string]map[string][]net.SkippableResponseDataAttributes `json:"skippableTests"` } -// TestSuiteDurationsCacheFile is the cache file name for suite duration metadata. -const TestSuiteDurationsCacheFile = "test_suite_durations.json" +// TestOptimizationPlanCacheFile keeps the historical filename used for plan/run handoff data. +const TestOptimizationPlanCacheFile = "test_suite_durations.json" const ( httpSettingsCacheFile = "settings.json" @@ -198,29 +198,29 @@ func (cm *CacheManager) StoreTestManagementTestsCache(testManagementTests *net.T return nil } -// StoreTestSuiteDurationsCache stores ddtest-private duration data in the runner cache. -func (cm *CacheManager) StoreTestSuiteDurationsCache(cache any) error { +// StoreTestOptimizationPlanCache stores ddtest-private plan data in the runner cache. +func (cm *CacheManager) StoreTestOptimizationPlanCache(cache any) error { if err := cm.createRunnerCacheDirectory(); err != nil { return fmt.Errorf("failed to create runner cache directory: %w", err) } - runnerPath := filepath.Join(appConstants.RunnerCacheDir, TestSuiteDurationsCacheFile) + runnerPath := filepath.Join(appConstants.RunnerCacheDir, TestOptimizationPlanCacheFile) if err := cm.writeJSONToFile(cache, runnerPath); err != nil { - slog.Error("Failed to write test suite durations to file", "error", err, "path", runnerPath) + slog.Error("Failed to write test optimization plan to file", "error", err, "path", runnerPath) return err } - slog.Debug("Test suite durations written to file", "path", runnerPath) + slog.Debug("Test optimization plan written to file", "path", runnerPath) return nil } -// ReadTestSuiteDurationsCache reads ddtest-private duration data from the runner cache. -func (cm *CacheManager) ReadTestSuiteDurationsCache(cache any) error { - runnerPath := filepath.Join(appConstants.RunnerCacheDir, TestSuiteDurationsCacheFile) +// ReadTestOptimizationPlanCache reads ddtest-private plan data from the runner cache. +func (cm *CacheManager) ReadTestOptimizationPlanCache(cache any) error { + runnerPath := filepath.Join(appConstants.RunnerCacheDir, TestOptimizationPlanCacheFile) if err := cm.readJSONFromFile(runnerPath, cache); err != nil { return err } - slog.Debug("Test suite durations read from file", "path", runnerPath) + slog.Debug("Test optimization plan read from file", "path", runnerPath) return nil } diff --git a/internal/testoptimization/cache_test.go b/internal/testoptimization/cache_test.go index f2631ae..51adc24 100644 --- a/internal/testoptimization/cache_test.go +++ b/internal/testoptimization/cache_test.go @@ -9,7 +9,7 @@ import ( appConstants "github.com/DataDog/ddtest/internal/constants" ) -type testSuiteDurationsCacheFixture struct { +type testOptimizationPlanCacheFixture struct { TestSuiteDurations map[string]map[string]TestSuiteDurationInfo `json:"testSuiteDurations"` SuiteAggregates []testSuiteAggregateCacheFixture `json:"suiteAggregates"` SuitesBySourceFile map[string][]testSuiteCacheKeyFixture `json:"suitesBySourceFile"` @@ -31,8 +31,8 @@ type testSuiteAggregateCacheFixture struct { NumTestsSkipped int `json:"numTestsSkipped"` } -func newTestSuiteDurationsCacheFixture(sourceFile string, weight int) testSuiteDurationsCacheFixture { - return testSuiteDurationsCacheFixture{ +func newTestOptimizationPlanCacheFixture(sourceFile string, weight int) testOptimizationPlanCacheFixture { + return testOptimizationPlanCacheFixture{ TestSuiteDurations: map[string]map[string]TestSuiteDurationInfo{ "rspec": { "Suite1": { @@ -61,37 +61,37 @@ func newTestSuiteDurationsCacheFixture(sourceFile string, weight int) testSuiteD } } -func TestCacheManager_StoreAndReadTestSuiteDurationsCache(t *testing.T) { +func TestCacheManager_StoreAndReadTestOptimizationPlanCache(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() defer func() { _ = os.Chdir(oldWd) }() _ = os.Chdir(tempDir) - cache := newTestSuiteDurationsCacheFixture("spec/suite1_spec.rb", 2500) + cache := newTestOptimizationPlanCacheFixture("spec/suite1_spec.rb", 2500) cacheManager := NewCacheManager() - if err := cacheManager.StoreTestSuiteDurationsCache(cache); err != nil { - t.Fatalf("StoreTestSuiteDurationsCache() should not return error, got: %v", err) + if err := cacheManager.StoreTestOptimizationPlanCache(cache); err != nil { + t.Fatalf("StoreTestOptimizationPlanCache() should not return error, got: %v", err) } - runnerCachePath := filepath.Join(appConstants.RunnerCacheDir, TestSuiteDurationsCacheFile) + runnerCachePath := filepath.Join(appConstants.RunnerCacheDir, TestOptimizationPlanCacheFile) if _, err := os.Stat(runnerCachePath); err != nil { - t.Fatalf("Expected runner test suite durations cache file to be written: %v", err) + t.Fatalf("Expected runner test optimization plan cache file to be written: %v", err) } - legacyCachePath := filepath.Join(appConstants.CacheDir, TestSuiteDurationsCacheFile) + legacyCachePath := filepath.Join(appConstants.CacheDir, TestOptimizationPlanCacheFile) if _, err := os.Stat(legacyCachePath); !os.IsNotExist(err) { - t.Fatalf("Expected test suite durations cache to stay out of legacy cache dir, got error: %v", err) + t.Fatalf("Expected test optimization plan cache to stay out of legacy cache dir, got error: %v", err) } - httpCachePath := filepath.Join(appConstants.HTTPCacheDir, TestSuiteDurationsCacheFile) + httpCachePath := filepath.Join(appConstants.HTTPCacheDir, TestOptimizationPlanCacheFile) if _, err := os.Stat(httpCachePath); !os.IsNotExist(err) { - t.Fatalf("Expected test suite durations cache to stay out of cache/http, got error: %v", err) + t.Fatalf("Expected test optimization plan cache to stay out of cache/http, got error: %v", err) } - var restored testSuiteDurationsCacheFixture - if err := cacheManager.ReadTestSuiteDurationsCache(&restored); err != nil { - t.Fatalf("ReadTestSuiteDurationsCache() should not return error, got: %v", err) + var restored testOptimizationPlanCacheFixture + if err := cacheManager.ReadTestOptimizationPlanCache(&restored); err != nil { + t.Fatalf("ReadTestOptimizationPlanCache() should not return error, got: %v", err) } if !reflect.DeepEqual(restored, cache) { @@ -99,7 +99,7 @@ func TestCacheManager_StoreAndReadTestSuiteDurationsCache(t *testing.T) { } } -func TestCacheManager_ReadTestSuiteDurationsCache_DoesNotReadLegacyCache(t *testing.T) { +func TestCacheManager_ReadTestOptimizationPlanCache_DoesNotReadLegacyCache(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() defer func() { _ = os.Chdir(oldWd) }() @@ -110,14 +110,14 @@ func TestCacheManager_ReadTestSuiteDurationsCache_DoesNotReadLegacyCache(t *test t.Fatalf("CreateCacheDirectory() should not return error, got: %v", err) } - legacyCache := newTestSuiteDurationsCacheFixture("spec/legacy_spec.rb", 1000) - legacyCachePath := filepath.Join(appConstants.CacheDir, TestSuiteDurationsCacheFile) + legacyCache := newTestOptimizationPlanCacheFixture("spec/legacy_spec.rb", 1000) + legacyCachePath := filepath.Join(appConstants.CacheDir, TestOptimizationPlanCacheFile) if err := cacheManager.writeJSONToFile(legacyCache, legacyCachePath); err != nil { t.Fatalf("writeJSONToFile() should not return error for legacy cache, got: %v", err) } - var restored testSuiteDurationsCacheFixture - if err := cacheManager.ReadTestSuiteDurationsCache(&restored); err == nil { - t.Fatal("ReadTestSuiteDurationsCache() should not read the legacy cache path") + var restored testOptimizationPlanCacheFixture + if err := cacheManager.ReadTestOptimizationPlanCache(&restored); err == nil { + t.Fatal("ReadTestOptimizationPlanCache() should not read the legacy cache path") } } diff --git a/internal/testoptimization/client.go b/internal/testoptimization/client.go index d7a9707..39c70c0 100644 --- a/internal/testoptimization/client.go +++ b/internal/testoptimization/client.go @@ -16,6 +16,8 @@ type TestOptimizationClient interface { Initialize(tags map[string]string) error GetSettings() *net.SettingsResponseData GetSkippableTests() map[string]bool + GetKnownTests() *net.KnownTestsResponseData + GetTestManagementTestsData() *net.TestManagementTestsResponseDataModules StoreCacheAndExit() } @@ -168,6 +170,20 @@ func (c *DatadogClient) GetSkippableTests() map[string]bool { return skippedTests } +func (c *DatadogClient) GetKnownTests() *net.KnownTestsResponseData { + if c.settings == nil || !c.settings.KnownTestsEnabled { + return nil + } + return c.integrations.GetKnownTests() +} + +func (c *DatadogClient) GetTestManagementTestsData() *net.TestManagementTestsResponseDataModules { + if c.settings == nil || !c.settings.TestManagement.Enabled { + return nil + } + return c.integrations.GetTestManagementTestsData() +} + func (c *DatadogClient) StoreCacheAndExit() { // store repository settings repositorySettings := c.integrations.GetSettings()