Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions docs/docs/usage/commands/unleash/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -390,8 +390,10 @@ gremlins unleash --test-cpu=1

:material-flag: `--threshold-efficacy` · :material-sign-direction: Default: 0

When set, it makes Gremlins exit with an error (code 10) if the _test efficacy_ threshold is not met. By default it is
zero, which
When set, it makes Gremlins exit with an error (code 10) if the _test efficacy_
is below the threshold. The threshold is satisfied when the actual efficacy is
greater than or equal to the configured value (so `--threshold-efficacy 100`
is met when every reached mutant is killed). By default it is zero, which
means Gremlins never exits with an error.

The _test efficacy_ is calculated as `KILLED / (KILLED + LIVED)` and assesses how effective are the tests.
Expand All @@ -404,7 +406,9 @@ gremlins unleash --threshold-efficacy 80

:material-flag: `--threshold-mcover` · :material-sign-direction: Default: 0

When set, it makes Gremlins exit with an error (code 11) if the _mutant coverage_ threshold is not met. By default
When set, it makes Gremlins exit with an error (code 11) if the
_mutant coverage_ is below the threshold. The threshold is satisfied when the
actual coverage is greater than or equal to the configured value. By default
it is zero, which means Gremlins never exits with an error.

The _mutant coverage_ is calculated as `(KILLED + LIVED) / (KILLED + LIVED + NOT_COVERED)` and assesses how many mutants
Expand Down
4 changes: 2 additions & 2 deletions internal/report/report.go
Original file line number Diff line number Diff line change
Expand Up @@ -235,14 +235,14 @@ func (r *reportStatus) assess(tEfficacy, rCoverage float64) error {
if et == 0 {
et = float64(configuration.Get[int](configuration.UnleashThresholdEfficacyKey))
}
if et > 0 && tEfficacy <= et {
if et > 0 && tEfficacy < et {
return execution.NewExitErr(execution.EfficacyThreshold)
}
ct := configuration.Get[float64](configuration.UnleashThresholdMCoverageKey)
if ct == 0 {
ct = float64(configuration.Get[int](configuration.UnleashThresholdMCoverageKey))
}
if ct > 0 && rCoverage <= ct {
if ct > 0 && rCoverage < ct {
return execution.NewExitErr(execution.MutantCoverageThreshold)
}

Expand Down
72 changes: 62 additions & 10 deletions internal/report/report_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -199,56 +199,104 @@ func TestAssessment(t *testing.T) {
}{
// Efficacy-threshold as float64
{
name: "efficacy < efficacy-threshold",
name: "efficacy < efficacy-threshold float64",
confKey: configuration.UnleashThresholdEfficacyKey,
value: float64(51),
expectError: true,
},
{
name: "efficacy >= efficacy-threshold",
name: "efficacy == efficacy-threshold float64",
confKey: configuration.UnleashThresholdEfficacyKey,
value: float64(50),
expectError: false,
},
{
name: "efficacy-threshold == 0",
name: "efficacy > efficacy-threshold float64",
confKey: configuration.UnleashThresholdEfficacyKey,
value: float64(49),
expectError: false,
},
{
name: "efficacy-threshold == 0 float64",
confKey: configuration.UnleashThresholdEfficacyKey,
value: float64(0),
expectError: false,
},
// Efficacy-threshold as float64
// Efficacy-threshold as int
{
name: "efficacy < efficacy-threshold",
name: "efficacy < efficacy-threshold int",
confKey: configuration.UnleashThresholdEfficacyKey,
value: 51,
expectError: true,
},
// Mutator coverage-threshold as float
{
name: "coverage < coverage-threshold",
name: "efficacy == efficacy-threshold int",
confKey: configuration.UnleashThresholdEfficacyKey,
value: 50,
expectError: false,
},
{
name: "efficacy > efficacy-threshold int",
confKey: configuration.UnleashThresholdEfficacyKey,
value: 49,
expectError: false,
},
{
name: "efficacy-threshold == 0 int",
confKey: configuration.UnleashThresholdEfficacyKey,
value: 0,
expectError: false,
},
// Mutator coverage-threshold as float64
{
name: "coverage < coverage-threshold float64",
confKey: configuration.UnleashThresholdMCoverageKey,
value: float64(51),
expectError: true,
},
{
name: "coverage >= coverage-threshold",
name: "coverage == coverage-threshold float64",
confKey: configuration.UnleashThresholdMCoverageKey,
value: float64(50),
expectError: false,
},
{
name: "coverage-threshold == 0",
name: "coverage > coverage-threshold float64",
confKey: configuration.UnleashThresholdMCoverageKey,
value: float64(49),
expectError: false,
},
{
name: "coverage-threshold == 0 float64",
confKey: configuration.UnleashThresholdMCoverageKey,
value: float64(0),
expectError: false,
},
// Mutator coverage-threshold as int
{
name: "coverage < coverage-threshold",
name: "coverage < coverage-threshold int",
confKey: configuration.UnleashThresholdMCoverageKey,
value: 51,
expectError: true,
},
{
name: "coverage == coverage-threshold int",
confKey: configuration.UnleashThresholdMCoverageKey,
value: 50,
expectError: false,
},
{
name: "coverage > coverage-threshold int",
confKey: configuration.UnleashThresholdMCoverageKey,
value: 49,
expectError: false,
},
{
name: "coverage-threshold == 0 int",
confKey: configuration.UnleashThresholdMCoverageKey,
value: 0,
expectError: false,
},
}

for _, tc := range testCases {
Expand Down Expand Up @@ -278,6 +326,10 @@ func TestAssessment(t *testing.T) {
t.Fatal("expected an error")
}
if !tc.expectError {
if err != nil {
t.Fatalf("expected no error, got %v", err)
}

return
}
var exitErr *execution.ExitError
Expand Down