From 0e9a01ac6b1448302563371ac802ae22dd7e4048 Mon Sep 17 00:00:00 2001 From: Vinit Tomar Date: Thu, 17 Sep 2026 18:58:45 +0530 Subject: [PATCH 1/9] Fix CI version stamping and schema target shell The build job passed -X ldflags for github.com/saucelabs/saucectl/cli/version.*, a package that does not exist. Go ignores -X for symbols it cannot find, so CI binaries were never stamped. Point it at internal/version, as .goreleaser.yml does. The schema target used pushd/popd, which are bash builtins and fail under make's /bin/sh. A plain cd in the recipe's subshell is equivalent. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/test.yml | 4 ++-- Makefile | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d1f591703..0b0438429 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -85,8 +85,8 @@ jobs: - name: Build Project run: | SHORT_SHA=$(echo $GITHUB_SHA | cut -c1-8) - LDFLAG_VERSION="github.com/saucelabs/saucectl/cli/version.Version=v0.0.0+$SHORT_SHA" - LDFLAG_SHA="github.com/saucelabs/saucectl/cli/version.GitCommit=$GITHUB_SHA" + LDFLAG_VERSION="github.com/saucelabs/saucectl/internal/version.Version=v0.0.0+$SHORT_SHA" + LDFLAG_SHA="github.com/saucelabs/saucectl/internal/version.GitCommit=$GITHUB_SHA" go install ./... CGO_ENABLED=0 go build -ldflags="-X $LDFLAG_VERSION -X $LDFLAG_SHA" cmd/saucectl/saucectl.go GOOS=windows GOARCH=amd64 go build cmd/saucectl/saucectl.go diff --git a/Makefile b/Makefile index 6ed4c1be4..306ae8dc5 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,5 @@ coverage: schema: $(eval INPUT_SCHEMA := $(shell pwd)/api/global.schema.json) $(eval OUTPUT_SCHEMA := $(shell pwd)/api/saucectl.schema.json) - pushd scripts/json-schema-bundler/ && \ - npm run bundle -- -s $(INPUT_SCHEMA) -o $(OUTPUT_SCHEMA) && \ - popd + cd scripts/json-schema-bundler/ && \ + npm run bundle -- -s $(INPUT_SCHEMA) -o $(OUTPUT_SCHEMA) From 1e40062018c95e69d7295e7af6f1b2ea0e0a3dd4 Mon Sep 17 00:00:00 2001 From: Vinit Tomar Date: Thu, 17 Sep 2026 18:58:45 +0530 Subject: [PATCH 2/9] Derive advertised region list from region package region.Options() builds the list from sauceRegionMetas so help text and error messages cannot fall behind the table, which is what happened when asia-south-2 was added. None and the internal staging region are not advertised. Co-Authored-By: Claude Opus 5 (1M context) --- internal/region/region.go | 18 ++++++++++++++++++ internal/region/region_test.go | 20 ++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/internal/region/region.go b/internal/region/region.go index 65809f9b8..793609d19 100644 --- a/internal/region/region.go +++ b/internal/region/region.go @@ -3,6 +3,7 @@ package region import ( "os" "path/filepath" + "strings" "github.com/rs/zerolog/log" "github.com/saucelabs/saucectl/internal/credentials" @@ -162,6 +163,23 @@ func FromString(s string) Region { return None } +// Options returns the public Sauce Labs regions as a comma separated list, in +// declaration order, for use in help text and error messages. None is not a +// region and Staging is internal to Sauce Labs, so neither is advertised. +// User defined regions are omitted too, since they are not options this +// binary can suggest. +func Options() string { + var names []string + for _, m := range sauceRegionMetas { + if m.Name == None.String() || m.Name == Staging.String() { + continue + } + names = append(names, m.Name) + } + + return strings.Join(names, ", ") +} + func lookupMeta(r Region) regionMeta { m, ok := allRegionMetas(sauceRegionMetas, userRegionMetas)[r] if ok { diff --git a/internal/region/region_test.go b/internal/region/region_test.go index aa733ea7d..4250bebcb 100644 --- a/internal/region/region_test.go +++ b/internal/region/region_test.go @@ -1,6 +1,7 @@ package region import ( + "strings" "testing" "github.com/google/go-cmp/cmp" @@ -71,6 +72,25 @@ func TestString(t *testing.T) { assert.Equal(t, name, r.String()) } +// TestOptions guards the list of regions the CLI advertises. Options derives +// it from sauceRegionMetas so help text and error messages cannot fall behind +// the table, which is what happened when asia-south-2 was added. Adding a +// public region is meant to fail this test: update the want string once you +// have confirmed the region should be advertised. +func TestOptions(t *testing.T) { + // Options advertises the built-in public regions, so a user's + // ~/.sauce/regions.yml must not leak into it. + saved := userRegionMetas + defer func() { userRegionMetas = saved }() + userRegionMetas = []regionMeta{{Name: "my-own-region"}} + + got := Options() + + assert.Equal(t, "us-west-1, us-east-4, eu-central-1, asia-south-2", got) + assert.Assert(t, !strings.Contains(got, Staging.String())) + assert.Assert(t, !strings.Contains(got, "my-own-region")) +} + // TestRegionURLs guards the sauceRegionMetas table. Its entries are unkeyed // struct literals, so a field written out of order still compiles and would // silently point the CLI at the wrong host. From cd72f7a32e0259008f49bd23cbdead7b93654e14 Mon Sep 17 00:00:00 2001 From: Vinit Tomar Date: Thu, 17 Sep 2026 18:58:45 +0530 Subject: [PATCH 3/9] Fix stale region help across command groups Every group advertised "us-west-1, eu-central-1", omitting us-east-4 and asia-south-2. Each now builds the list with region.Options(). Note that internal/cmd/run/run.go carries the same change, but it lands with the authoring dispatch it also contains. Co-Authored-By: Claude Opus 5 (1M context) --- internal/cmd/apit/cmd.go | 3 ++- internal/cmd/artifacts/cmd.go | 3 ++- internal/cmd/builds/cmd.go | 3 ++- internal/cmd/devices/cmd.go | 3 ++- internal/cmd/ini/cmd.go | 3 ++- internal/cmd/jobs/cmd.go | 3 ++- internal/cmd/storage/cmd.go | 3 ++- 7 files changed, 14 insertions(+), 7 deletions(-) diff --git a/internal/cmd/apit/cmd.go b/internal/cmd/apit/cmd.go index fd5034359..4f6f20f29 100644 --- a/internal/cmd/apit/cmd.go +++ b/internal/cmd/apit/cmd.go @@ -2,6 +2,7 @@ package apit import ( "errors" + "fmt" "time" "github.com/saucelabs/saucectl/internal/usage" @@ -46,7 +47,7 @@ func Command(preRun func(cmd *cobra.Command, args []string)) *cobra.Command { } flags := cmd.PersistentFlags() - flags.StringVarP(®io, "region", "r", "us-west-1", "The Sauce Labs region. Options: us-west-1, eu-central-1.") + flags.StringVarP(®io, "region", "r", "us-west-1", fmt.Sprintf("The Sauce Labs region. Options: %s.", region.Options())) cmd.AddCommand( VaultCommand(cmd.PersistentPreRunE), diff --git a/internal/cmd/artifacts/cmd.go b/internal/cmd/artifacts/cmd.go index 4f8ef39a9..887007067 100644 --- a/internal/cmd/artifacts/cmd.go +++ b/internal/cmd/artifacts/cmd.go @@ -2,6 +2,7 @@ package artifacts import ( "errors" + "fmt" "time" "github.com/saucelabs/saucectl/internal/usage" @@ -63,7 +64,7 @@ func Command(preRun func(cmd *cobra.Command, args []string)) *cobra.Command { } flags := cmd.PersistentFlags() - flags.StringVarP(®io, "region", "r", "us-west-1", "The Sauce Labs region. Options: us-west-1, eu-central-1.") + flags.StringVarP(®io, "region", "r", "us-west-1", fmt.Sprintf("The Sauce Labs region. Options: %s.", region.Options())) cmd.AddCommand( DownloadCommand(), diff --git a/internal/cmd/builds/cmd.go b/internal/cmd/builds/cmd.go index ecf256c2f..4ba55a71d 100644 --- a/internal/cmd/builds/cmd.go +++ b/internal/cmd/builds/cmd.go @@ -2,6 +2,7 @@ package builds import ( "errors" + "fmt" "time" "github.com/saucelabs/saucectl/internal/build" @@ -54,7 +55,7 @@ func Command(preRun func(cmd *cobra.Command, args []string)) *cobra.Command { } flags := cmd.PersistentFlags() - flags.StringVarP(®io, "region", "r", "us-west-1", "The Sauce Labs region. Options: us-west-1, eu-central-1.") + flags.StringVarP(®io, "region", "r", "us-west-1", fmt.Sprintf("The Sauce Labs region. Options: %s.", region.Options())) cmd.AddCommand( GetCommand(), diff --git a/internal/cmd/devices/cmd.go b/internal/cmd/devices/cmd.go index d75fd73ce..bfd1f4509 100644 --- a/internal/cmd/devices/cmd.go +++ b/internal/cmd/devices/cmd.go @@ -2,6 +2,7 @@ package devices import ( "errors" + "fmt" "time" "github.com/saucelabs/saucectl/internal/credentials" @@ -52,7 +53,7 @@ func Command(preRun func(cmd *cobra.Command, args []string)) *cobra.Command { } flags := cmd.PersistentFlags() - flags.StringVarP(®io, "region", "r", "us-west-1", "The Sauce Labs region. Options: us-west-1, eu-central-1.") + flags.StringVarP(®io, "region", "r", "us-west-1", fmt.Sprintf("The Sauce Labs region. Options: %s.", region.Options())) cmd.AddCommand(ListCommand()) cmd.AddCommand(GetCommand()) diff --git a/internal/cmd/ini/cmd.go b/internal/cmd/ini/cmd.go index 47328fd83..911f95c4f 100644 --- a/internal/cmd/ini/cmd.go +++ b/internal/cmd/ini/cmd.go @@ -14,6 +14,7 @@ import ( "github.com/saucelabs/saucectl/internal/http" "github.com/saucelabs/saucectl/internal/msg" "github.com/saucelabs/saucectl/internal/playwright" + "github.com/saucelabs/saucectl/internal/region" "github.com/saucelabs/saucectl/internal/testcafe" "github.com/saucelabs/saucectl/internal/xctest" "github.com/saucelabs/saucectl/internal/xcuitest" @@ -96,7 +97,7 @@ func Command(preRun func(cmd *cobra.Command, args []string)) *cobra.Command { flags := cmd.PersistentFlags() flags.BoolVar(&noPrompt, "no-prompt", false, "Disable interactive prompts.") - flags.StringVarP(®ionName, "region", "r", "", "Sauce Labs region. Options: us-west-1, eu-central-1.") + flags.StringVarP(®ionName, "region", "r", "", fmt.Sprintf("Sauce Labs region. Options: %s.", region.Options())) return cmd } diff --git a/internal/cmd/jobs/cmd.go b/internal/cmd/jobs/cmd.go index a7926b4f6..041bb177e 100644 --- a/internal/cmd/jobs/cmd.go +++ b/internal/cmd/jobs/cmd.go @@ -2,6 +2,7 @@ package jobs import ( "errors" + "fmt" "time" "github.com/saucelabs/saucectl/internal/usage" @@ -55,7 +56,7 @@ func Command(preRun func(cmd *cobra.Command, args []string)) *cobra.Command { } flags := cmd.PersistentFlags() - flags.StringVarP(®io, "region", "r", "us-west-1", "The Sauce Labs region. Options: us-west-1, eu-central-1.") + flags.StringVarP(®io, "region", "r", "us-west-1", fmt.Sprintf("The Sauce Labs region. Options: %s.", region.Options())) cmd.AddCommand( GetCommand(), diff --git a/internal/cmd/storage/cmd.go b/internal/cmd/storage/cmd.go index ac37e7b93..28c58dd76 100644 --- a/internal/cmd/storage/cmd.go +++ b/internal/cmd/storage/cmd.go @@ -2,6 +2,7 @@ package storage import ( "errors" + "fmt" "time" "github.com/saucelabs/saucectl/internal/usage" @@ -46,7 +47,7 @@ func Command(preRun func(cmd *cobra.Command, args []string)) *cobra.Command { } flags := cmd.PersistentFlags() - flags.StringVarP(®io, "region", "r", "us-west-1", "The Sauce Labs region. Options: us-west-1, eu-central-1.") + flags.StringVarP(®io, "region", "r", "us-west-1", fmt.Sprintf("The Sauce Labs region. Options: %s.", region.Options())) cmd.AddCommand( ListCommand(), From 6f2581e01c38abf8977a3439017a2899b5b53474 Mon Sep 17 00:00:00 2001 From: Vinit Tomar Date: Thu, 17 Sep 2026 18:58:45 +0530 Subject: [PATCH 4/9] Add AI Test Authoring domain and HTTP client Co-Authored-By: Claude Opus 5 (1M context) --- internal/authoring/authoring.go | 174 ++++++ internal/authoring/config.go | 274 ++++++++++ internal/authoring/config_test.go | 200 +++++++ internal/authoring/entitlement.go | 38 ++ internal/authoring/errors.go | 118 ++++ internal/authoring/errors_test.go | 38 ++ internal/authoring/generate.go | 99 ++++ internal/authoring/list_test.go | 98 ++++ internal/authoring/run.go | 150 ++++++ internal/authoring/run_test.go | 111 ++++ internal/authoring/runner.go | 748 ++++++++++++++++++++++++++ internal/authoring/runner_test.go | 806 ++++++++++++++++++++++++++++ internal/authoring/schedule.go | 199 +++++++ internal/authoring/schedule_test.go | 76 +++ internal/authoring/target.go | 44 ++ internal/authoring/testcase.go | 355 ++++++++++++ internal/authoring/testsuite.go | 84 +++ internal/authoring/tool_test.go | 227 ++++++++ internal/authoring/variable.go | 138 +++++ internal/authoring/variable_test.go | 65 +++ internal/http/authoring.go | 654 ++++++++++++++++++++++ internal/http/authoring_test.go | 498 +++++++++++++++++ internal/mocks/authoring.go | 200 +++++++ 23 files changed, 5394 insertions(+) create mode 100644 internal/authoring/authoring.go create mode 100644 internal/authoring/config.go create mode 100644 internal/authoring/config_test.go create mode 100644 internal/authoring/entitlement.go create mode 100644 internal/authoring/errors.go create mode 100644 internal/authoring/errors_test.go create mode 100644 internal/authoring/generate.go create mode 100644 internal/authoring/list_test.go create mode 100644 internal/authoring/run.go create mode 100644 internal/authoring/run_test.go create mode 100644 internal/authoring/runner.go create mode 100644 internal/authoring/runner_test.go create mode 100644 internal/authoring/schedule.go create mode 100644 internal/authoring/schedule_test.go create mode 100644 internal/authoring/target.go create mode 100644 internal/authoring/testcase.go create mode 100644 internal/authoring/testsuite.go create mode 100644 internal/authoring/tool_test.go create mode 100644 internal/authoring/variable.go create mode 100644 internal/authoring/variable_test.go create mode 100644 internal/http/authoring.go create mode 100644 internal/http/authoring_test.go create mode 100644 internal/mocks/authoring.go diff --git a/internal/authoring/authoring.go b/internal/authoring/authoring.go new file mode 100644 index 000000000..622d43d36 --- /dev/null +++ b/internal/authoring/authoring.go @@ -0,0 +1,174 @@ +// Package authoring holds the domain model and service interfaces for the +// Sauce Labs AI Test Authoring service, reached over the AI Authoring API +// (/ai-authoring/v1). The HTTP implementation lives in internal/http; the +// commands live in internal/cmd/authoring; the `kind: authoring` runner is in +// this package alongside its configuration. +// +// Everything about the remote service that this package encodes was verified +// against the live API rather than taken from its published specification — +// several observed behaviours contradict that specification and are recorded +// where the affected code lives (see specs/001-ai-test-authoring/research.md). +package authoring + +import ( + "context" + "errors" + "fmt" + "io" +) + +// DefaultPageSize is the page size used when walking a listing exhaustively. +// Listings are heavy (a test case is ~13 KB because every revision and step is +// inlined), so this stays well below the service's uncapped limit. +const DefaultPageSize = 100 + +// maxListPages bounds ListAll so a service that ignored `skip` and kept +// returning full pages could not loop forever. At DefaultPageSize this allows +// 100,000 items, far beyond any organisation observed. +const maxListPages = 1000 + +// ErrListTooLong is returned by ListAll when maxListPages is exhausted before +// the service signals the end of the listing. +var ErrListTooLong = errors.New("listing did not end within the maximum number of pages") + +// List is one page of a listing endpoint. Every listing in the service returns +// the same envelope: the items on this page plus the total across all pages. +type List[T any] struct { + Items []T `json:"items"` + Total int `json:"total"` +} + +// ListOptions is the pagination shared by every listing endpoint. +type ListOptions struct { + // Skip is the offset of the first item to return. + Skip int + // Limit is the page size. It is a pointer because the service treats + // limit=0 as a meaningful count-only request ({"total": N, "items": []}), + // so "not set" and "zero" must be distinguishable: nil is not sent, + // zero is. + Limit *int +} + +// NewListOptions returns options for one page of the given size at the given +// offset. A convenience for callers that always send a limit. +func NewListOptions(skip, limit int) ListOptions { + return ListOptions{Skip: skip, Limit: &limit} +} + +// ListAll walks every page of a listing by calling fetch with successive +// offsets until the returned page is short, the reported total is reached, or +// maxListPages is exhausted. pageSize <= 0 uses DefaultPageSize. +func ListAll[T any](ctx context.Context, pageSize int, fetch func(ctx context.Context, opts ListOptions) (List[T], error)) ([]T, error) { + if pageSize <= 0 { + pageSize = DefaultPageSize + } + + var all []T + for page := 0; page < maxListPages; page++ { + l, err := fetch(ctx, NewListOptions(page*pageSize, pageSize)) + if err != nil { + return all, err + } + all = append(all, l.Items...) + + if len(l.Items) < pageSize || len(all) >= l.Total { + return all, nil + } + } + + return all, fmt.Errorf("%w (%d pages of %d)", ErrListTooLong, maxListPages, pageSize) +} + +// TestCaseService covers the test case endpoints, including runs, generation +// and code export. +type TestCaseService interface { + // ListTestCases returns one page of test cases matching opts. + ListTestCases(ctx context.Context, opts ListTestCasesOptions) (List[TestCase], error) + // GetTestCase returns a single test case with every revision and step. + GetTestCase(ctx context.Context, id string) (TestCase, error) + // DeleteTestCase removes a test case. It does not fail if runs exist. + DeleteTestCase(ctx context.Context, id string) error + // RenameTestCase changes a test case's name and returns the updated case. + RenameTestCase(ctx context.Context, id, name string) (TestCase, error) + // RunTestCase starts a run of the test case's latest revision, or of the + // given revision when revisionID is non-empty. The returned Run is not + // terminal: poll it with GetRun using the run's own TestCaseID. + RunTestCase(ctx context.Context, id, revisionID string, opts RunOptions) (Run, error) + // ListRuns returns one page of the runs that belong to testCaseID. The + // implementation must send testCaseID as a query parameter — the path + // parameter alone is ignored by the service and returns every run in the + // organisation (research R-004). + ListRuns(ctx context.Context, testCaseID string, opts ListRunsOptions) (List[Run], error) + // GetRun returns a single run. testCaseID must be the run's own + // TestCaseID; the service enforces it here even though it ignores it on + // the list endpoint. + GetRun(ctx context.Context, testCaseID, runID string) (Run, error) + // ListTags returns every distinct tag in the organisation. Tags are + // case-sensitive: "Login" and "login" are different tags. + ListTags(ctx context.Context) ([]string, error) + // Generate starts an asynchronous authoring task from a plain-language + // intent and returns its identifiers. + Generate(ctx context.Context, opts GenerateOptions) (GenerateTask, error) + // GenerationStatus returns the current state of an authoring task, + // including the steps captured so far. + GenerationStatus(ctx context.Context, taskID string) (GenerationState, error) + // Code exports the latest revision as source for the given target. Use + // CodeTargets to discover valid targets first. + Code(ctx context.Context, id, target string) (string, error) + // CodeTargets returns the export targets available for the test case. + CodeTargets(ctx context.Context, id string) ([]string, error) +} + +// TestSuiteService covers the test suite endpoints. +type TestSuiteService interface { + ListTestSuites(ctx context.Context, opts ListTestSuitesOptions) (List[TestSuite], error) + GetTestSuite(ctx context.Context, id string) (TestSuite, error) + CreateTestSuite(ctx context.Context, opts CreateTestSuiteOptions) (TestSuite, error) + UpdateTestSuite(ctx context.Context, id string, opts UpdateTestSuiteOptions) (TestSuite, error) + // DeleteTestSuite removes a suite. When deleteTestCases is true the + // service also deletes every test case in the suite. + DeleteTestSuite(ctx context.Context, id string, deleteTestCases bool) error + // RunTestSuite queues a run for every case in the suite. The response + // carries only a count and the build name — there is nothing to poll, so + // this is fire-and-forget (research R-002). + RunTestSuite(ctx context.Context, id, buildName string) (SuiteRun, error) +} + +// ScheduleService covers the test schedule endpoints. +type ScheduleService interface { + ListSchedules(ctx context.Context, opts ListSchedulesOptions) (List[TestSchedule], error) + GetSchedule(ctx context.Context, id string) (TestSchedule, error) + CreateSchedule(ctx context.Context, opts CreateScheduleOptions) (TestSchedule, error) + UpdateSchedule(ctx context.Context, id string, opts UpdateScheduleOptions) (TestSchedule, error) + DeleteSchedule(ctx context.Context, id string) error +} + +// VariableService covers the variable endpoints. +type VariableService interface { + ListVariables(ctx context.Context, opts ListVariablesOptions) (List[Variable], error) + GetVariable(ctx context.Context, id string) (Variable, error) + CreateVariable(ctx context.Context, opts CreateVariableOptions) (Variable, error) + // UpdateVariable changes a variable. opts.ExpectedLastUpdate must echo the + // LastUpdate of a prior read; a mismatch is ErrVariableVersionConflict. + UpdateVariable(ctx context.Context, id string, opts UpdateVariableOptions) (Variable, error) + // DeleteVariable removes a variable. expectedLastUpdate must echo the + // LastUpdate of a prior read; a mismatch is ErrVariableVersionConflict. + DeleteVariable(ctx context.Context, id, expectedLastUpdate string) error +} + +// ArtifactService downloads files captured during authoring, such as step +// screenshots. +type ArtifactService interface { + // DownloadArtifact streams the artifact with the given identifier. The + // identifier is the last path segment of a step's screenshot URL, not the + // URL itself. The response carries no content type, so the caller decides + // the file name. The caller must close the returned reader. + DownloadArtifact(ctx context.Context, id string) (io.ReadCloser, error) +} + +// EntitlementReader answers whether an organisation may use AI authoring. +// This is served by a platform API outside the authoring service (research +// R-009); without it every command would fail with an opaque 401/403. +type EntitlementReader interface { + IsAIAuthoringEnabled(ctx context.Context, orgID string) (bool, error) +} diff --git a/internal/authoring/config.go b/internal/authoring/config.go new file mode 100644 index 000000000..8f3e8b372 --- /dev/null +++ b/internal/authoring/config.go @@ -0,0 +1,274 @@ +package authoring + +import ( + "errors" + "fmt" + "time" + "unicode/utf8" + + "github.com/rs/zerolog/log" + + "github.com/saucelabs/saucectl/internal/config" + "github.com/saucelabs/saucectl/internal/msg" + "github.com/saucelabs/saucectl/internal/region" +) + +// Config descriptors. +var ( + // Kind is the `kind` value that selects this runner. + Kind = "authoring" + // APIVersion is the supported configuration version. + APIVersion = "v1alpha" +) + +// Limits and defaults the service and the runner impose. +const ( + // maxBuildNameLength is the per-case run endpoint's cap. The suite-run + // endpoint allows 255, but the runner never uses it. + maxBuildNameLength = 100 + // DefaultSuiteTimeout bounds a suite's wait when the configuration sets + // none. Nothing waits indefinitely (Constitution VIII); an authored run + // typically finishes within a minute, so this is generous. + DefaultSuiteTimeout = 30 * time.Minute + // defaultConcurrency matches the other kinds' default. + defaultConcurrency = 2 + // defaultTunnelTimeout matches the --tunnel-timeout flag default. + defaultTunnelTimeout = 30 * time.Second +) + +// Project is the `kind: authoring` configuration. +type Project struct { + config.TypeDef `yaml:",inline" mapstructure:",squash"` + ConfigFilePath string `yaml:"-" json:"-"` + DryRun bool `yaml:"-" json:"-"` + Sauce config.SauceConfig `yaml:"sauce,omitempty" json:"sauce"` + Defaults config.Defaults `yaml:"defaults,omitempty" json:"defaults"` + Suites []Suite `yaml:"suites,omitempty" json:"suites"` + Artifacts config.Artifacts `yaml:"artifacts,omitempty" json:"artifacts"` + Reporters config.Reporters `yaml:"reporters,omitempty" json:"-"` + + // Settings that the shared `run` flags or a copied configuration can + // populate but this kind cannot honour. They are decoded only so that + // Validate can warn about them instead of dropping them silently + // (FR-013). EnvFlag is what --env binds to; Env is the YAML form. + Env map[string]string `yaml:"env,omitempty" json:"-"` + EnvFlag map[string]string `yaml:"-" json:"-"` + ShowConsoleLog bool `yaml:"showConsoleLog,omitempty" json:"-"` + LiveLogs bool `yaml:"liveLogs,omitempty" json:"-"` +} + +// Suite selects a set of authored test cases to run and where to run them. +// Exactly one of TestSuiteID, TestSuiteName and TestCases must be set. +type Suite struct { + Name string `yaml:"name,omitempty" json:"name"` + // TestSuiteID is a suite's identifier (dashless 32-hex). + TestSuiteID string `yaml:"testSuiteId,omitempty" json:"testSuiteId,omitempty"` + // TestSuiteName is a suite's exact name, resolved at run time. The + // service's search is a substring match, so the exact comparison is made + // client-side and an ambiguous match is an error. + TestSuiteName string `yaml:"testSuiteName,omitempty" json:"testSuiteName,omitempty"` + // TestCases is an explicit list of test case identifiers (24-hex). + TestCases []string `yaml:"testCases,omitempty" json:"testCases,omitempty"` + // Tags narrows a referenced suite's cases to those carrying any of the + // tags. Ignored when TestCases is given. + Tags []string `yaml:"tags,omitempty" json:"tags,omitempty"` + // Targets overrides each case's stored run targets. When omitted the + // stored targets apply. + Targets []Target `yaml:"targets,omitempty" json:"targets,omitempty"` + // Timeout bounds how long the runner waits for each run in this suite. + // Defaults to defaults.timeout, then DefaultSuiteTimeout. + Timeout time.Duration `yaml:"timeout,omitempty" json:"timeout"` +} + +// FromFile creates a Project from the configuration file at cfgPath. +func FromFile(cfgPath string) (Project, error) { + var p Project + if err := config.Unmarshal(cfgPath, &p); err != nil { + return p, err + } + p.ConfigFilePath = cfgPath + return p, nil +} + +// SetDefaults fills in what the user left blank and normalises what the YAML +// decoder produced: nested capability maps arrive as map[interface{}] +// interface{}, which encoding/json cannot marshal, so they are converted to +// map[string]any here rather than at request time. +func SetDefaults(p *Project) { + if p.Kind == "" { + p.Kind = Kind + } + if p.APIVersion == "" { + p.APIVersion = APIVersion + } + if p.Sauce.Concurrency < 1 { + p.Sauce.Concurrency = defaultConcurrency + } + p.Sauce.Tunnel.SetDefaults() + // The other kinds inherit this from the --tunnel-timeout flag default; + // applying it here too keeps a YAML-only configuration valid. + if p.Sauce.Tunnel.Name != "" && p.Sauce.Tunnel.Timeout <= 0 { + p.Sauce.Tunnel.Timeout = defaultTunnelTimeout + } + p.Sauce.Metadata.SetDefaultBuild() + // The service's cap is on characters, so count runes: slicing bytes + // splits a multi-byte rune and puts invalid UTF-8 in the run request, + // while also dropping characters that were within the limit. + if utf8.RuneCountInString(p.Sauce.Metadata.Build) > maxBuildNameLength { + log.Warn().Msgf("Build name exceeds %d characters and will be truncated for AI authoring runs.", maxBuildNameLength) + p.Sauce.Metadata.Build = string([]rune(p.Sauce.Metadata.Build)[:maxBuildNameLength]) + } + + for i := range p.Suites { + s := &p.Suites[i] + if s.Timeout <= 0 { + s.Timeout = p.Defaults.Timeout + } + if s.Timeout <= 0 { + s.Timeout = DefaultSuiteTimeout + } + for j := range s.Targets { + s.Targets[j].Capabilities = normalizeCapabilities(s.Targets[j].Capabilities) + } + } +} + +// normalizeCapabilities converts YAML-decoded nested maps into map[string]any +// all the way down, so the capabilities marshal to JSON unchanged. +func normalizeCapabilities(caps map[string]any) map[string]any { + if caps == nil { + return nil + } + out := make(map[string]any, len(caps)) + for k, v := range caps { + out[k] = normalizeValue(v) + } + return out +} + +// normalizeValue is the recursive step of normalizeCapabilities. +func normalizeValue(v any) any { + switch t := v.(type) { + case map[string]any: + return normalizeCapabilities(t) + case map[any]any: + out := make(map[string]any, len(t)) + for k, val := range t { + out[fmt.Sprint(k)] = normalizeValue(val) + } + return out + case []any: + out := make([]any, len(t)) + for i, val := range t { + out[i] = normalizeValue(val) + } + return out + default: + return v + } +} + +// Validate enforces the configuration contract. It also warns about settings +// this kind cannot honour (FR-013) — from here rather than from the command, +// so the warnings fire whether the value came from YAML or from a flag. +func Validate(p Project) error { + if region.FromString(p.Sauce.Region) == region.None { + return errors.New(msg.MissingRegion) + } + if len(p.Suites) == 0 { + return errors.New("no suites configured: add at least one entry under 'suites'") + } + + seen := map[string]bool{} + for _, s := range p.Suites { + if err := validateSuite(s); err != nil { + return err + } + if seen[s.Name] { + return fmt.Errorf("suite name %q is used more than once", s.Name) + } + seen[s.Name] = true + } + + if p.Sauce.Retries > 0 { + log.Warn().Msg("sauce.retries is not supported for kind: authoring and will be ignored.") + } + if len(p.Sauce.Metadata.Tags) > 0 { + log.Warn().Msg("sauce.metadata.tags is not supported for kind: authoring and will be ignored.") + } + if p.Sauce.Tunnel.Owner != "" { + log.Warn().Msg("sauce.tunnel.owner is not supported for kind: authoring; runs use the tunnel by name only.") + } + if len(p.Env) > 0 || len(p.EnvFlag) > 0 { + log.Warn().Msg("env / --env is not supported for kind: authoring and will be ignored; use 'saucectl authoring variables' to pass values to authored tests.") + } + if p.ShowConsoleLog { + log.Warn().Msg("showConsoleLog / --show-console-log is not supported for kind: authoring and will be ignored.") + } + if p.LiveLogs { + log.Warn().Msg("--live-logs is not supported for kind: authoring and will be ignored.") + } + if p.Sauce.LaunchOrder != "" { + log.Warn().Msg("sauce.launchOrder / --launch-order is not supported for kind: authoring and will be ignored.") + } + if p.Sauce.Visibility != "" { + log.Warn().Msg("sauce.visibility is not supported for kind: authoring and will be ignored.") + } + if len(p.Sauce.Experiments) > 0 { + log.Warn().Msg("sauce.experiments / --experiment is not supported for kind: authoring and will be ignored.") + } + // --root-dir and --sauceignore carry non-empty defaults, so a set value + // cannot be told from the default here; they are irrelevant to this kind + // and are not warned about. + + return nil +} + +// validateSuite checks one suite entry. +func validateSuite(s Suite) error { + if s.Name == "" { + return errors.New("every suite needs a name") + } + + refs := 0 + if s.TestSuiteID != "" { + refs++ + } + if s.TestSuiteName != "" { + refs++ + } + if len(s.TestCases) > 0 { + refs++ + } + if refs != 1 { + return fmt.Errorf("suite %q must set exactly one of testSuiteId, testSuiteName or testCases", s.Name) + } + for _, id := range s.TestCases { + if id == "" { + return fmt.Errorf("suite %q has an empty test case id", s.Name) + } + } + if len(s.TestCases) > 0 && len(s.Tags) > 0 { + log.Warn().Msgf("Suite %q: tags are ignored when testCases is set.", s.Name) + } + for i, t := range s.Targets { + if len(t.Capabilities) == 0 { + return fmt.Errorf("suite %q target %d has no capabilities", s.Name, i+1) + } + } + if s.Timeout < 0 { + return fmt.Errorf("suite %q has a negative timeout", s.Name) + } + return nil +} + +// FilterSuites narrows the project to the named suite, for --select-suite. +func FilterSuites(p *Project, suiteName string) error { + for _, s := range p.Suites { + if s.Name == suiteName { + p.Suites = []Suite{s} + return nil + } + } + return fmt.Errorf(msg.SuiteNameNotFound, suiteName) +} diff --git a/internal/authoring/config_test.go b/internal/authoring/config_test.go new file mode 100644 index 000000000..e2c194975 --- /dev/null +++ b/internal/authoring/config_test.go @@ -0,0 +1,200 @@ +package authoring + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/saucelabs/saucectl/internal/config" +) + +func TestFromFile_PreservesCapabilityKeyCaseAndNesting(t *testing.T) { + // Two things this locks in: the viper fork keeps key case (a stock viper + // lowercases "browserName" to "browsername"), and nested maps decode to + // something SetDefaults can turn into JSON-marshalable capabilities. + dir := t.TempDir() + cfg := filepath.Join(dir, "authoring.yml") + yaml := `apiVersion: v1alpha +kind: authoring +sauce: + region: us-west-1 + concurrency: 3 + metadata: + build: "nightly" +defaults: + timeout: 10m +suites: + - name: "Regression on Chrome" + testSuiteName: "Checkout Regression" + targets: + - capabilities: + browserName: chrome + platformName: "Windows 11" + sauce:options: + name: x + screenResolution: 1920x1080 + - name: "Two cases" + testCases: [6a9ba94e5c93b6d220adc17d, 6a88b8ee7760d47c61f9e1fa] + timeout: 2m +artifacts: + download: + when: fail + match: ["*.mp4"] + directory: ./artifacts +reporters: + junit: + enabled: true +` + if err := os.WriteFile(cfg, []byte(yaml), 0o644); err != nil { + t.Fatal(err) + } + + p, err := FromFile(cfg) + if err != nil { + t.Fatal(err) + } + SetDefaults(&p) + if err := Validate(p); err != nil { + t.Fatal(err) + } + + if p.Kind != Kind || p.Sauce.Concurrency != 3 || p.Sauce.Metadata.Build != "nightly" { + t.Errorf("project = %+v", p) + } + if p.Suites[0].Timeout != 10*time.Minute || p.Suites[1].Timeout != 2*time.Minute { + t.Errorf("timeouts = %v, %v", p.Suites[0].Timeout, p.Suites[1].Timeout) + } + if !p.Reporters.JUnit.Enabled || p.Artifacts.Download.When != config.WhenFail { + t.Errorf("reporters/artifacts not decoded: %+v %+v", p.Reporters, p.Artifacts) + } + + caps := p.Suites[0].Targets[0].Capabilities + if caps["browserName"] != "chrome" || caps["platformName"] != "Windows 11" { + t.Errorf("capability key case not preserved: %v", caps) + } + opts, ok := caps["sauce:options"].(map[string]any) + if !ok || opts["screenResolution"] != "1920x1080" { + t.Errorf("nested capabilities not normalised: %#v", caps["sauce:options"]) + } + if _, err := json.Marshal(caps); err != nil { + t.Errorf("capabilities must marshal to JSON: %v", err) + } + if len(p.Suites[1].TestCases) != 2 { + t.Errorf("testCases = %v", p.Suites[1].TestCases) + } +} + +func TestSetDefaults(t *testing.T) { + p := Project{Suites: []Suite{{Name: "a", TestSuiteID: "x"}}} + SetDefaults(&p) + if p.Kind != Kind || p.APIVersion != APIVersion { + t.Errorf("type def defaults: %+v", p.TypeDef) + } + if p.Sauce.Concurrency != defaultConcurrency { + t.Errorf("concurrency = %d", p.Sauce.Concurrency) + } + if p.Sauce.Metadata.Build == "" { + t.Error("build name must default") + } + if p.Suites[0].Timeout != DefaultSuiteTimeout { + t.Errorf("suite timeout = %v, want %v", p.Suites[0].Timeout, DefaultSuiteTimeout) + } + + long := Project{Sauce: config.SauceConfig{Metadata: config.Metadata{Build: string(make([]byte, 150))}}} + SetDefaults(&long) + if len(long.Sauce.Metadata.Build) != maxBuildNameLength { + t.Errorf("build name not truncated to %d: %d", maxBuildNameLength, len(long.Sauce.Metadata.Build)) + } +} + +func TestValidate(t *testing.T) { + valid := func() Project { + return Project{ + Sauce: config.SauceConfig{Region: "us-west-1"}, + Suites: []Suite{{Name: "a", TestSuiteID: "x"}}, + } + } + + tests := []struct { + name string + mutate func(*Project) + wantErr bool + }{ + {name: "valid", mutate: func(*Project) {}}, + {name: "missing region", mutate: func(p *Project) { p.Sauce.Region = "" }, wantErr: true}, + {name: "no suites", mutate: func(p *Project) { p.Suites = nil }, wantErr: true}, + {name: "suite without name", mutate: func(p *Project) { p.Suites[0].Name = "" }, wantErr: true}, + {name: "no reference", mutate: func(p *Project) { p.Suites[0].TestSuiteID = "" }, wantErr: true}, + {name: "two references", mutate: func(p *Project) { p.Suites[0].TestSuiteName = "n" }, wantErr: true}, + {name: "by name", mutate: func(p *Project) { p.Suites[0].TestSuiteID = ""; p.Suites[0].TestSuiteName = "n" }}, + {name: "by cases", mutate: func(p *Project) { p.Suites[0].TestSuiteID = ""; p.Suites[0].TestCases = []string{"c"} }}, + {name: "empty case id", mutate: func(p *Project) { p.Suites[0].TestSuiteID = ""; p.Suites[0].TestCases = []string{""} }, wantErr: true}, + {name: "target without capabilities", mutate: func(p *Project) { p.Suites[0].Targets = []Target{{}} }, wantErr: true}, + {name: "duplicate suite names", mutate: func(p *Project) { p.Suites = append(p.Suites, Suite{Name: "a", TestSuiteID: "y"}) }, wantErr: true}, + {name: "negative timeout", mutate: func(p *Project) { p.Suites[0].Timeout = -1 }, wantErr: true}, + {name: "unsupported settings only warn", mutate: func(p *Project) { + p.Sauce.Retries = 2 + p.Sauce.Metadata.Tags = []string{"t"} + p.Sauce.Tunnel.Owner = "o" + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := valid() + tt.mutate(&p) + err := Validate(p) + if (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestFilterSuites(t *testing.T) { + p := Project{Suites: []Suite{{Name: "a"}, {Name: "b"}}} + if err := FilterSuites(&p, "b"); err != nil || len(p.Suites) != 1 || p.Suites[0].Name != "b" { + t.Errorf("filter: %v %+v", err, p.Suites) + } + if err := FilterSuites(&p, "zzz"); err == nil { + t.Error("expected error for unknown suite") + } +} + +func TestNormalizeValue(t *testing.T) { + in := map[string]any{ + "a": map[any]any{"b": map[any]any{"c": 1}, 2: "two"}, + "list": []any{map[any]any{"d": true}}, + } + out := normalizeCapabilities(in) + if _, err := json.Marshal(out); err != nil { + t.Fatalf("normalised capabilities must marshal: %v", err) + } + a := out["a"].(map[string]any) + if a["2"] != "two" || a["b"].(map[string]any)["c"] != 1 { + t.Errorf("nested conversion wrong: %#v", out) + } +} + +func TestSetDefaults_TruncatesBuildNameOnRuneBoundary(t *testing.T) { + // The cap is on characters. Slicing bytes split a rune, put invalid + // UTF-8 into the run request, and dropped characters that were within + // the limit. + name := strings.Repeat("日", 120) // 120 characters, 360 bytes + p := Project{Sauce: config.SauceConfig{Metadata: config.Metadata{Build: name}}} + SetDefaults(&p) + + got := p.Sauce.Metadata.Build + if n := utf8.RuneCountInString(got); n != maxBuildNameLength { + t.Errorf("truncated to %d characters, want %d", n, maxBuildNameLength) + } + if !utf8.ValidString(got) { + t.Error("truncation produced invalid UTF-8") + } + if strings.ContainsRune(got, '�') { + t.Error("truncation split a rune") + } +} diff --git a/internal/authoring/entitlement.go b/internal/authoring/entitlement.go new file mode 100644 index 000000000..ed41f755b --- /dev/null +++ b/internal/authoring/entitlement.go @@ -0,0 +1,38 @@ +package authoring + +import ( + "context" + "errors" + "fmt" + + "github.com/saucelabs/saucectl/internal/iam" +) + +// ErrNotEntitled is returned when the organisation's plan does not include AI +// authoring. It is deliberately distinct from a verification failure +// (FR-032): the two need different remedies, and a user must be able to tell +// an entitlement problem from a credentials or network problem. +var ErrNotEntitled = errors.New("AI Test Authoring is not included in your Sauce Labs plan; contact your Sauce Labs account team to enable it") + +// VerifyEntitlement resolves the caller's organisation and checks that it is +// entitled to AI authoring. It fails closed. Any error other than +// ErrNotEntitled means "could not verify". This costs two serial requests per +// invocation, accepted for the quality of the error (research R-009). +func VerifyEntitlement(ctx context.Context, users iam.UserService, ents EntitlementReader) (iam.User, error) { + user, err := users.User(ctx) + if err != nil { + return iam.User{}, fmt.Errorf("could not verify AI authoring entitlement: resolving the current user failed: %w", err) + } + if user.Organization.ID == "" { + return iam.User{}, errors.New("could not verify AI authoring entitlement: the current user has no organisation") + } + + enabled, err := ents.IsAIAuthoringEnabled(ctx, user.Organization.ID) + if err != nil { + return iam.User{}, fmt.Errorf("could not verify AI authoring entitlement: %w", err) + } + if !enabled { + return iam.User{}, ErrNotEntitled + } + return user, nil +} diff --git a/internal/authoring/errors.go b/internal/authoring/errors.go new file mode 100644 index 000000000..cb4c2413b --- /dev/null +++ b/internal/authoring/errors.go @@ -0,0 +1,118 @@ +package authoring + +import ( + "fmt" + "strings" +) + +// APIError is the decoded error envelope returned by the AI Authoring +// service. Every non-2xx response carries +// +// {"error": {"code": "...", "detail": "...", "data": [...]}} +// +// Code is the machine-readable identifier the sentinels below compare on; +// Detail is the human-readable summary. Items holds the undocumented +// error.data[] array, which is where the service puts the sentence that +// actually says what to fix (research R-007): for INVALID_QUERY the detail is +// merely "Invalid query string parameters." while data[0].message names the +// missing parameter. Dropping it would leave users with an unactionable error. +type APIError struct { + // HTTPStatus is the response status code. Zero when the error was + // constructed as a sentinel rather than decoded from a response. + HTTPStatus int `json:"-"` + // Code is the service's machine-readable error code, e.g. TEST_CASE_NOT_FOUND. + Code string `json:"code"` + // Detail is the service's human-readable summary. + Detail string `json:"detail,omitempty"` + // Items carries the per-field messages from the undocumented data[] array. + Items []APIErrorItem `json:"data,omitempty"` +} + +// APIErrorItem is one entry of the undocumented error.data[] array. Path is +// kept as raw values because the service mixes strings and integers in it +// (object keys and array indices). +type APIErrorItem struct { + Code string `json:"code,omitempty"` + Path []any `json:"path,omitempty"` + Message string `json:"message,omitempty"` +} + +// Error renders the code, the detail and every per-field message, so the one +// sentence that explains the problem is never hidden behind a generic summary. +func (e *APIError) Error() string { + var b strings.Builder + b.WriteString("ai authoring service error") + if e.HTTPStatus != 0 { + fmt.Fprintf(&b, " (HTTP %d)", e.HTTPStatus) + } + if e.Code != "" { + fmt.Fprintf(&b, " %s", e.Code) + } + if e.Detail != "" { + fmt.Fprintf(&b, ": %s", e.Detail) + } + for _, it := range e.Items { + if it.Message == "" { + continue + } + b.WriteString("; ") + if len(it.Path) > 0 { + parts := make([]string, 0, len(it.Path)) + for _, p := range it.Path { + parts = append(parts, fmt.Sprint(p)) + } + fmt.Fprintf(&b, "%s: ", strings.Join(parts, ".")) + } + b.WriteString(it.Message) + } + return b.String() +} + +// Is reports whether target is an *APIError with the same Code, which makes +// errors.Is(err, ErrTestCaseNotFound) work regardless of HTTP status or detail +// text. A sentinel with an empty Code never matches. +func (e *APIError) Is(target error) bool { + t, ok := target.(*APIError) + if !ok || t.Code == "" { + return false + } + return t.Code == e.Code +} + +// Sentinels for every error code the service declares. They carry only a +// Code, so they compare by code through errors.Is; the decoded error that +// reaches the caller still holds the status, detail and items. Named ErrXxx +// for the errname linter. +var ( + ErrAppNotFound = &APIError{Code: "APP_NOT_FOUND"} + ErrCodeGenerationTargetInvalid = &APIError{Code: "CODE_GENERATION_TARGET_INVALID"} + ErrCodeGenerationTargetNotFound = &APIError{Code: "CODE_GENERATION_TARGET_NOT_FOUND"} + ErrFileNotFound = &APIError{Code: "FILE_NOT_FOUND"} + ErrInvalidBody = &APIError{Code: "INVALID_BODY"} + ErrInvalidParams = &APIError{Code: "INVALID_PARAMS"} + ErrInvalidQuery = &APIError{Code: "INVALID_QUERY"} + ErrInvalidTestCases = &APIError{Code: "INVALID_TEST_CASES"} + ErrInvalidTestSuites = &APIError{Code: "INVALID_TEST_SUITES"} + ErrNoRunTargets = &APIError{Code: "NO_RUN_TARGETS"} + ErrRunningUserNotFound = &APIError{Code: "RUNNING_USER_NOT_FOUND"} + ErrSCTunnelNotFound = &APIError{Code: "SC_TUNNEL_NOT_FOUND"} + ErrTestCasesNotFound = &APIError{Code: "TEST_CASES_NOT_FOUND"} + ErrTestCaseEmpty = &APIError{Code: "TEST_CASE_EMPTY"} + ErrGenerationTaskNotFound = &APIError{Code: "TEST_CASE_GENERATION_TASK_NOT_FOUND"} + ErrTestCaseNotFound = &APIError{Code: "TEST_CASE_NOT_FOUND"} + ErrTestCaseRevisionNotFound = &APIError{Code: "TEST_CASE_REVISION_NOT_FOUND"} + ErrTestCaseRunNotFound = &APIError{Code: "TEST_CASE_RUN_NOT_FOUND"} + ErrTestScheduleNotFound = &APIError{Code: "TEST_SCHEDULE_NOT_FOUND"} + ErrTestSuitesNotFound = &APIError{Code: "TEST_SUITES_NOT_FOUND"} + ErrTestSuiteNotFound = &APIError{Code: "TEST_SUITE_NOT_FOUND"} + ErrTestSuiteNoRunJobs = &APIError{Code: "TEST_SUITE_NO_RUN_JOBS"} + ErrUnauthorized = &APIError{Code: "UNAUTHORIZED"} + ErrUnknownBackendType = &APIError{Code: "UNKNOWN_BACKEND_TYPE"} + ErrVariableForbidden = &APIError{Code: "VARIABLE_FORBIDDEN"} + ErrVariableNameConflict = &APIError{Code: "VARIABLE_NAME_CONFLICT"} + ErrVariableNotFound = &APIError{Code: "VARIABLE_NOT_FOUND"} + ErrVariableScopeInvalid = &APIError{Code: "VARIABLE_SCOPE_INVALID"} + // ErrVariableVersionConflict is the 412 returned when expectedLastUpdate no + // longer matches: somebody changed the variable since it was read. + ErrVariableVersionConflict = &APIError{Code: "VARIABLE_VERSION_CONFLICT"} +) diff --git a/internal/authoring/errors_test.go b/internal/authoring/errors_test.go new file mode 100644 index 000000000..418ccad92 --- /dev/null +++ b/internal/authoring/errors_test.go @@ -0,0 +1,38 @@ +package authoring + +import ( + "errors" + "fmt" + "testing" +) + +func TestAPIError_ErrorAndIs(t *testing.T) { + err := &APIError{ + HTTPStatus: 400, + Code: "INVALID_QUERY", + Detail: "Invalid query string parameters.", + Items: []APIErrorItem{{ + Code: "custom", + Path: []any{"query", "scope"}, + Message: "scope-specific id is required", + }}, + } + want := "ai authoring service error (HTTP 400) INVALID_QUERY: Invalid query string parameters.; query.scope: scope-specific id is required" + if got := err.Error(); got != want { + t.Errorf("Error() =\n%s\nwant\n%s", got, want) + } + + if !errors.Is(err, ErrInvalidQuery) { + t.Error("errors.Is must match on code") + } + if errors.Is(err, ErrTestCaseNotFound) { + t.Error("errors.Is must not match a different code") + } + wrapped := fmt.Errorf("listing variables: %w", err) + if !errors.Is(wrapped, ErrInvalidQuery) { + t.Error("errors.Is must see through wrapping") + } + if errors.Is(err, &APIError{}) { + t.Error("a sentinel with no code must never match") + } +} diff --git a/internal/authoring/generate.go b/internal/authoring/generate.go new file mode 100644 index 000000000..f084c611e --- /dev/null +++ b/internal/authoring/generate.go @@ -0,0 +1,99 @@ +package authoring + +import "fmt" + +// GenerateOptions is the request to author a new test case: the AI agent +// drives a browser against the target, following the plain-language intent, +// and saves the result as a test case. +type GenerateOptions struct { + // Name is the new test case's name, 1–255 characters. + Name string `json:"name"` + // TestSuiteID assigns the result to a suite. Dashed or dashless UUID. + TestSuiteID string `json:"testSuiteId,omitempty"` + // Tags has at most 20 entries of at most 60 characters each. + Tags []string `json:"tags,omitempty"` + RunSettings GenerateRunSettings `json:"runSettings"` + PromptSettings PromptSettings `json:"promptSettings"` + // TimeoutMillis is the service-side generation budget in milliseconds, + // 60 000 to 3 600 000 (one minute to one hour). Zero omits it and lets the + // service apply its default. This is distinct from the client-side wait + // timeout, which bounds how long saucectl watches the task. + TimeoutMillis int `json:"timeout,omitempty"` +} + +// GenerateRunSettings is where the authoring session runs. Note the single +// Target, unlike the stored RunSettings' primary + run targets. +type GenerateRunSettings struct { + Target Target `json:"target"` + // TestURL is the starting address, at most 2048 characters. + TestURL string `json:"testUrl,omitempty"` + TunnelName string `json:"scTunnelName,omitempty"` +} + +// PromptSettings is what the agent is asked to do. +type PromptSettings struct { + // Intent is the plain-language description, 1–20 000 characters. + Intent string `json:"intent"` + // MaxSteps caps the agent's actions, 1–200. Zero omits it. + MaxSteps int `json:"maxSteps,omitempty"` +} + +// GenerateTask identifies an accepted authoring task. TaskID is what +// GenerationStatus polls; SauceJobID is the browser session doing the work. +type GenerateTask struct { + TaskID string `json:"taskId"` + SauceJobID string `json:"sauceJobId"` +} + +// GenerationStatus is the lifecycle state of an authoring task. +type GenerationStatus string + +// The generation statuses. QUEUED and IN_PROGRESS are transient; the service +// recommends polling every 2–3 seconds until COMPLETED or FAILED. +const ( + GenerationQueued GenerationStatus = "QUEUED" + GenerationInProgress GenerationStatus = "IN_PROGRESS" + GenerationCompleted GenerationStatus = "COMPLETED" + GenerationFailed GenerationStatus = "FAILED" +) + +// GenerationState is the current state of an authoring task. Which fields are +// populated depends on Status: Steps and Reasoning while in progress and on +// failure, TestCaseID on completion, Error on failure. +type GenerationState struct { + Status GenerationStatus `json:"status"` + // Steps are the actions attempted so far. They are partial while the task + // runs; the saved test case holds the final steps. + Steps []GenerationStep `json:"steps,omitempty"` + Reasoning []Reasoning `json:"reasoning,omitempty"` + // TestCaseID is the saved test case, set only on completion. + TestCaseID string `json:"testCaseId,omitempty"` + // Error is set only on failure. + Error *GenerationError `json:"error,omitempty"` +} + +// Done reports whether the task has reached a terminal status. +func (s GenerationState) Done() bool { + return s.Status == GenerationCompleted || s.Status == GenerationFailed +} + +// GenerationStep is one action attempted during authoring. The wire name is +// "action" here, whereas a saved step calls the same thing "tool". +type GenerationStep struct { + Action Tool `json:"action"` + Result *StepResult `json:"result,omitempty"` +} + +// GenerationError is why an authoring task failed. +type GenerationError struct { + Code string `json:"code"` + Detail string `json:"detail"` +} + +// Error renders the code and detail. +func (e GenerationError) Error() string { + if e.Detail == "" { + return fmt.Sprintf("generation failed: %s", e.Code) + } + return fmt.Sprintf("generation failed: %s: %s", e.Code, e.Detail) +} diff --git a/internal/authoring/list_test.go b/internal/authoring/list_test.go new file mode 100644 index 000000000..f8af63a17 --- /dev/null +++ b/internal/authoring/list_test.go @@ -0,0 +1,98 @@ +package authoring + +import ( + "context" + "errors" + "testing" +) + +// pagedServer fakes a listing endpoint over a fixed set of integers. +func pagedServer(total int) func(ctx context.Context, opts ListOptions) (List[int], error) { + return func(_ context.Context, opts ListOptions) (List[int], error) { + var items []int + for i := opts.Skip; i < total && i < opts.Skip+*opts.Limit; i++ { + items = append(items, i) + } + return List[int]{Items: items, Total: total}, nil + } +} + +func TestListAll(t *testing.T) { + tests := []struct { + name string + total int + pageSize int + wantCalls int + }{ + {name: "exact multiple of the page size", total: 200, pageSize: 100, wantCalls: 2}, + {name: "partial last page", total: 150, pageSize: 100, wantCalls: 2}, + {name: "fewer than one page", total: 7, pageSize: 100, wantCalls: 1}, + {name: "zero total", total: 0, pageSize: 100, wantCalls: 1}, + {name: "non-positive page size uses the default", total: DefaultPageSize + 1, pageSize: 0, wantCalls: 2}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + calls := 0 + srv := pagedServer(tt.total) + fetch := func(ctx context.Context, opts ListOptions) (List[int], error) { + calls++ + if opts.Limit == nil { + t.Fatal("ListAll must always send a limit") + } + return srv(ctx, opts) + } + got, err := ListAll(context.Background(), tt.pageSize, fetch) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != tt.total { + t.Errorf("got %d items, want %d", len(got), tt.total) + } + for i, v := range got { + if v != i { + t.Fatalf("item %d = %d; pages were not disjoint or not ordered", i, v) + } + } + if calls != tt.wantCalls { + t.Errorf("fetch called %d times, want %d", calls, tt.wantCalls) + } + }) + } +} + +func TestListAll_NeverEndingServer(t *testing.T) { + // A server that ignores skip and always returns a full page with a huge + // total must not loop forever. + calls := 0 + fetch := func(_ context.Context, opts ListOptions) (List[int], error) { + calls++ + items := make([]int, *opts.Limit) + return List[int]{Items: items, Total: 1 << 30}, nil + } + _, err := ListAll(context.Background(), 10, fetch) + if !errors.Is(err, ErrListTooLong) { + t.Fatalf("expected ErrListTooLong, got %v", err) + } + if calls != maxListPages { + t.Errorf("fetch called %d times, want exactly %d", calls, maxListPages) + } +} + +func TestListAll_PropagatesFetchError(t *testing.T) { + boom := errors.New("boom") + calls := 0 + fetch := func(_ context.Context, opts ListOptions) (List[int], error) { + calls++ + if calls == 2 { + return List[int]{}, boom + } + return List[int]{Items: make([]int, *opts.Limit), Total: 1000}, nil + } + got, err := ListAll(context.Background(), 10, fetch) + if !errors.Is(err, boom) { + t.Fatalf("expected the fetch error, got %v", err) + } + if len(got) != 10 { + t.Errorf("expected the first page to be returned alongside the error, got %d items", len(got)) + } +} diff --git a/internal/authoring/run.go b/internal/authoring/run.go new file mode 100644 index 000000000..7ea0992a3 --- /dev/null +++ b/internal/authoring/run.go @@ -0,0 +1,150 @@ +package authoring + +import "encoding/json" + +// Run is one execution of a test case, grouped under a build and producing one +// job per target. A run has no status field: completion is inferred from its +// jobs (see Done). Observed on 2026-09-05: the start response carries jobs +// with only id, name, sauceJobId and target; isRdc appears ~15 s later and +// success ~19 s in, a few seconds *before* the underlying Sauce job reaches +// "complete". The run resource is therefore the earliest and authoritative +// completion signal. +type Run struct { + ID string `json:"id"` + OrgID string `json:"orgId,omitempty"` + TeamID string `json:"teamId,omitempty"` + UserID string `json:"userId,omitempty"` + // TestCaseID is the identifier to poll this run with. It is the run's own + // test case, which is what the detail endpoint enforces (research R-004). + TestCaseID string `json:"testCaseId"` + // Build is the build name as stored by the service. The service decorates + // the requested name (e.g. "nightly" becomes "nightly - 1"), so this is + // not byte-identical to what was sent. + Build string `json:"build,omitempty"` + Jobs []RunJob `json:"jobs"` + CreationDate string `json:"creationDate,omitempty"` + TestURL string `json:"testUrl,omitempty"` +} + +// Done reports whether every job has reported an outcome. A run with no jobs +// is not done: nothing has been reported for it. +func (r Run) Done() bool { + if len(r.Jobs) == 0 { + return false + } + for _, j := range r.Jobs { + if !j.Done() { + return false + } + } + return true +} + +// Passed reports whether the run is done and every job passed. +func (r Run) Passed() bool { + if !r.Done() { + return false + } + for _, j := range r.Jobs { + if !j.Passed() { + return false + } + } + return true +} + +// RunJob is the execution of a run against a single target. +type RunJob struct { + // ID is internal to the authoring service and is not resolvable through + // the Sauce job APIs. + ID string `json:"id"` + // SauceJobID is the real Sauce Labs job identifier: the one to link to, + // download artifacts for, and look up builds by. + SauceJobID string `json:"sauceJobId,omitempty"` + Name string `json:"name"` + Target Target `json:"target"` + // URL is unreliable — present on roughly half of observed jobs. Derive + // the link from SauceJobID instead (research R-006). + URL string `json:"url,omitempty"` + // IsRDC at the job level is absent until the job is under way; prefer + // RealDevice, which also consults the target. + IsRDC bool `json:"isRdc,omitempty"` + // Success is nil until the service reports an outcome. This pointer is + // the crux of the design: with a plain bool an in-flight job would be + // indistinguishable from a failed one (research R-005). + Success *bool `json:"success,omitempty"` + // Error carries infrastructure or assertion failure text. + Error string `json:"error,omitempty"` +} + +// Done reports whether the job has a reported outcome, either way. +func (j RunJob) Done() bool { + return j.Success != nil || j.Error != "" +} + +// Passed reports whether the job has reported success. +func (j RunJob) Passed() bool { + return j.Success != nil && *j.Success +} + +// RealDevice reports whether the job runs on a real device, consulting both +// the job-level flag (absent early on) and the target's flag (present from +// the start). +func (j RunJob) RealDevice() bool { + return j.IsRDC || j.Target.IsRDC +} + +// RunOptions is the request to start a run. +type RunOptions struct { + // BuildName groups the run's jobs under a build. The service caps it at + // 100 characters and decorates it (see Run.Build). + BuildName string + // TunnelName names an active Sauce Connect tunnel. When empty an explicit + // JSON null is sent, which clears any tunnel name stored on the test + // case. This is deliberate: some stored cases carry an empty-string tunnel + // name, and the service rejects a run of those with SC_TUNNEL_NOT_FOUND + // unless the field is explicitly nulled (research R-008, Open-3). Omitting + // the field is therefore never correct. + TunnelName string + // Targets overrides the test case's stored run targets. When nil the + // stored targets apply; a run fails with NO_RUN_TARGETS only when neither + // exists. + Targets []Target +} + +// MarshalJSON emits the request body, always including scTunnelName so that an +// empty TunnelName becomes an explicit null rather than an omission. See the +// TunnelName field for why omitting is never correct. +func (o RunOptions) MarshalJSON() ([]byte, error) { + // The documented run body carries capabilities and nothing else per + // target, so isRdc is deliberately not sent even though Target holds it + // for the configuration surface: whether a job ran on a real device is + // read back from the job the service reports, not asserted by us. + type wireTarget struct { + Capabilities map[string]any `json:"capabilities"` + } + type wire struct { + BuildName string `json:"buildName,omitempty"` + TunnelName *string `json:"scTunnelName"` + Targets []wireTarget `json:"targets,omitempty"` + } + var targets []wireTarget + for _, t := range o.Targets { + targets = append(targets, wireTarget{Capabilities: t.Capabilities}) + } + w := wire{BuildName: o.BuildName, Targets: targets} + if o.TunnelName != "" { + w.TunnelName = &o.TunnelName + } + return json.Marshal(w) +} + +// ListRunsOptions filters a run listing. The test case is a method argument, +// not an option, because it is mandatory. +type ListRunsOptions struct { + ListOptions + StartDate string + EndDate string + UserID string + TeamID string +} diff --git a/internal/authoring/run_test.go b/internal/authoring/run_test.go new file mode 100644 index 000000000..403fdd297 --- /dev/null +++ b/internal/authoring/run_test.go @@ -0,0 +1,111 @@ +package authoring + +import ( + "encoding/json" + "testing" +) + +func boolPtr(b bool) *bool { return &b } + +func TestRunOptions_MarshalJSON(t *testing.T) { + tests := []struct { + name string + opts RunOptions + want string + }{ + { + name: "no tunnel sends an explicit null, not an omission", + opts: RunOptions{BuildName: "nightly"}, + want: `{"buildName":"nightly","scTunnelName":null}`, + }, + { + name: "tunnel is sent by name", + opts: RunOptions{BuildName: "nightly", TunnelName: "my-tunnel"}, + want: `{"buildName":"nightly","scTunnelName":"my-tunnel"}`, + }, + { + name: "empty build name is omitted and targets carry capabilities only", + opts: RunOptions{Targets: []Target{{Capabilities: map[string]any{"browserName": "chrome"}}}}, + want: `{"scTunnelName":null,"targets":[{"capabilities":{"browserName":"chrome"}}]}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, err := json.Marshal(tt.opts) + if err != nil { + t.Fatal(err) + } + if string(b) != tt.want { + t.Errorf("got %s\nwant %s", b, tt.want) + } + }) + } +} + +func TestRunJob_DecodesMissingSuccessAsNil(t *testing.T) { + // The start response has no success key at all; that must decode to + // "not yet reported", never to "failed". + var j RunJob + if err := json.Unmarshal([]byte(`{"id":"j","name":"n","sauceJobId":"s","target":{"capabilities":{},"isRdc":false}}`), &j); err != nil { + t.Fatal(err) + } + if j.Success != nil { + t.Fatalf("expected nil success, got %v", *j.Success) + } + if j.Done() || j.Passed() { + t.Error("a job without an outcome must be neither done nor passed") + } +} + +func TestRun_DoneAndPassed(t *testing.T) { + tests := []struct { + name string + run Run + wantDone bool + wantPassed bool + }{ + {name: "no jobs is not done", run: Run{}}, + {name: "job without outcome", run: Run{Jobs: []RunJob{{ID: "a"}}}}, + {name: "one of two still running", run: Run{Jobs: []RunJob{{Success: boolPtr(true)}, {}}}}, + {name: "error text alone is terminal and failed", run: Run{Jobs: []RunJob{{Error: "boom"}}}, wantDone: true}, + {name: "explicit false", run: Run{Jobs: []RunJob{{Success: boolPtr(false)}}}, wantDone: true}, + {name: "all passed", run: Run{Jobs: []RunJob{{Success: boolPtr(true)}, {Success: boolPtr(true)}}}, wantDone: true, wantPassed: true}, + {name: "mixed", run: Run{Jobs: []RunJob{{Success: boolPtr(true)}, {Success: boolPtr(false)}}}, wantDone: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.run.Done(); got != tt.wantDone { + t.Errorf("Done() = %v, want %v", got, tt.wantDone) + } + if got := tt.run.Passed(); got != tt.wantPassed { + t.Errorf("Passed() = %v, want %v", got, tt.wantPassed) + } + }) + } +} + +func TestRunJob_RealDevice(t *testing.T) { + if (RunJob{}).RealDevice() { + t.Error("expected virtual by default") + } + if !(RunJob{IsRDC: true}).RealDevice() { + t.Error("job-level flag must count") + } + if !(RunJob{Target: Target{IsRDC: true}}).RealDevice() { + t.Error("target-level flag must count; it is the only one present early in a run") + } +} + +func TestRunOptions_MarshalJSONSendsCapabilitiesOnly(t *testing.T) { + // The documented run body carries capabilities per target and nothing + // else; isRdc belongs to the configuration surface only. + opts := RunOptions{Targets: []Target{{Capabilities: map[string]any{"browserName": "chrome"}, IsRDC: true}}} + b, err := json.Marshal(opts) + if err != nil { + t.Fatal(err) + } + want := `{"scTunnelName":null,"targets":[{"capabilities":{"browserName":"chrome"}}]}` + if string(b) != want { + t.Errorf("got %s\nwant %s", b, want) + } +} diff --git a/internal/authoring/runner.go b/internal/authoring/runner.go new file mode 100644 index 000000000..6403966d0 --- /dev/null +++ b/internal/authoring/runner.go @@ -0,0 +1,748 @@ +package authoring + +import ( + "context" + "errors" + "fmt" + "strings" + "sync/atomic" + "time" + + "github.com/rs/zerolog/log" + + "github.com/saucelabs/saucectl/internal/build" + "github.com/saucelabs/saucectl/internal/job" + "github.com/saucelabs/saucectl/internal/junit" + "github.com/saucelabs/saucectl/internal/region" + "github.com/saucelabs/saucectl/internal/report" + "github.com/saucelabs/saucectl/internal/tunnel" +) + +// DefaultPollInterval is how often a run is polled for completion. Observed +// runs of a short journey take ~20 s and report success on the run resource +// ~4 s before the Sauce job completes (research Open-1), so five seconds +// keeps requests modest without adding noticeable latency. +const DefaultPollInterval = 5 * time.Second + +// stopJobTimeout bounds the attempt to stop jobs after we have stopped +// waiting. It runs on a context detached from the interrupted one, so it needs +// a deadline of its own to honour the promise that nothing waits for ever. +const stopJobTimeout = 30 * time.Second + +// JobStopper stops a Sauce job. It is the part of saucecloud.JobService the +// runner needs to avoid leaving work running after we stop watching it. +type JobStopper interface { + StopJob(ctx context.Context, jobID string, realDevice bool) (job.Job, error) +} + +// ArtifactDownloader is the part of saucecloud.JobService the runner needs. +// It takes a job.Job so the shared skip rules (when: fail/pass, timed out, +// unfinished) apply exactly as they do for every other kind. +type ArtifactDownloader interface { + DownloadArtifacts(ctx context.Context, j job.Job, isLastAttempt bool) []string +} + +// Runner executes a `kind: authoring` project: it resolves suites to test +// cases, starts one run per case under a bounded worker pool, polls each run +// to completion and hands one result per job to the shared reporters. +// +// It is deliberately not a saucecloud.CloudRunner: that type is built around +// job.StartOptions (WebDriver job creation), whereas these runs are triggered +// through the authoring service. The runner also deliberately does not copy +// internal/apitest's shape, which starts every test at once and shares a +// package-level poll variable across suites (research R-012). +type Runner struct { + Project Project + TestCases TestCaseService + TestSuites TestSuiteService + // Artifacts downloads job assets after a run; nil disables downloads. + Artifacts ArtifactDownloader + // Stopper stops jobs when we stop waiting for them, so an interrupted or + // timed-out run does not keep consuming the organisation's concurrency. + // nil disables stopping. + Stopper JobStopper + // Builds resolves the build link shown under the results table; nil + // leaves it out. + Builds build.Service + // Tunnels validates tunnel readiness before anything starts. + Tunnels tunnel.Service + Region region.Region + Reporters []report.Reporter + // Async starts every run and returns without waiting. + Async bool + // PollInterval overrides DefaultPollInterval; tests set it very low. + PollInterval time.Duration +} + +// ResolvedCase pairs a configured suite with one test case it expands to. +type ResolvedCase struct { + Suite Suite + TestCase TestCase +} + +// runResult is what one worker produces for one resolved case. +type runResult struct { + Case ResolvedCase + // Run is the last state seen. Its Jobs are empty when the start failed. + Run Run + // Err is a start failure or a fatal poll failure. + Err error + // TimedOut is set when the suite timeout expired before the run ended. + TimedOut bool + // Interrupted is set when the context ended before the run did; the run + // continues on Sauce Labs. + Interrupted bool + StartTime time.Time + EndTime time.Time +} + +// RunProject runs the project and returns the process exit code: 0 only when +// every run passed (or was started, under Async), 1 otherwise. +func (r *Runner) RunProject(ctx context.Context) (int, error) { + // The owner is deliberately not passed: runCase sends only scTunnelName, + // so validating with an owner would check a colleague's tunnel and then + // report ready for a run that fails with SC_TUNNEL_NOT_FOUND. Validate + // says exactly what the run will do. Validate() warns that the owner is + // ignored. + if err := tunnel.Validate( + ctx, + r.Tunnels, + r.Project.Sauce.Tunnel.Name, + "", + tunnel.NoneFilter, + r.Project.DryRun, + r.Project.Sauce.Tunnel.Timeout, + ); err != nil { + return 1, err + } + + cases, err := r.ResolveTestCases(ctx) + if err != nil { + return 1, err + } + + if r.Project.DryRun { + printDryRun(cases) + return 0, nil + } + + if r.runCases(ctx, cases) { + return 0, nil + } + return 1, nil +} + +// ResolveTestCases expands every configured suite into concrete test cases. +// Resolution is read-only, which is what lets --dry-run show exactly what +// would execute. +func (r *Runner) ResolveTestCases(ctx context.Context) ([]ResolvedCase, error) { + var out []ResolvedCase + for _, s := range r.Project.Suites { + cases, err := r.resolveSuite(ctx, s) + if err != nil { + return nil, fmt.Errorf("suite %q: %w", s.Name, err) + } + if len(cases) == 0 { + log.Warn().Str("suite", s.Name).Msg("Suite resolved to no test cases.") + } + for _, tc := range cases { + out = append(out, ResolvedCase{Suite: s, TestCase: tc}) + } + } + if len(out) == 0 { + return nil, errors.New("no test cases to run") + } + return out, nil +} + +// resolveSuite resolves one suite entry: explicit cases are fetched one by +// one; a suite reference is expanded through the testSuiteId filter, which +// matches the suite's own count exactly (research R-003). +func (r *Runner) resolveSuite(ctx context.Context, s Suite) ([]TestCase, error) { + if len(s.TestCases) > 0 { + cases := make([]TestCase, 0, len(s.TestCases)) + for _, id := range s.TestCases { + tc, err := r.TestCases.GetTestCase(ctx, id) + if err != nil { + return nil, fmt.Errorf("test case %s: %w", id, err) + } + cases = append(cases, tc) + } + return cases, nil + } + + suiteID := s.TestSuiteID + if s.TestSuiteName != "" { + id, err := r.findSuiteByName(ctx, s.TestSuiteName) + if err != nil { + return nil, err + } + suiteID = id + } + + return ListAll(ctx, DefaultPageSize, func(ctx context.Context, lo ListOptions) (List[TestCase], error) { + return r.TestCases.ListTestCases(ctx, ListTestCasesOptions{ + ListOptions: lo, + TestSuiteIDs: []string{suiteID}, + Tags: s.Tags, + }) + }) +} + +// findSuiteByName resolves a suite by exact name. The service's search is a +// case-insensitive substring match inside words ("demo" matches "Saucedemo - +// Checkout flow"), so the exact comparison happens here, and an ambiguous +// name is an error rather than "first result wins". +func (r *Runner) findSuiteByName(ctx context.Context, name string) (string, error) { + all, err := ListAll(ctx, DefaultPageSize, func(ctx context.Context, lo ListOptions) (List[TestSuite], error) { + return r.TestSuites.ListTestSuites(ctx, ListTestSuitesOptions{ListOptions: lo, Search: name}) + }) + if err != nil { + return "", fmt.Errorf("looking up test suite %q: %w", name, err) + } + + var ids []string + for _, s := range all { + if s.Name == name { + ids = append(ids, s.ID) + } + } + switch len(ids) { + case 0: + return "", fmt.Errorf("no test suite is named %q (the match is exact and case-sensitive)", name) + case 1: + return ids[0], nil + default: + return "", fmt.Errorf("%d test suites are named %q; reference one by testSuiteId instead: %s", len(ids), name, strings.Join(ids, ", ")) + } +} + +// printDryRun lists what would run without starting anything. +func printDryRun(cases []ResolvedCase) { + fmt.Println("\nThe following test cases would have run:") + for _, c := range cases { + targets := "stored run targets" + if n := len(c.Suite.Targets); n > 0 { + targets = fmt.Sprintf("%d configured target(s)", n) + } + fmt.Printf(" - %s: %s (%s) on %s\n", c.Suite.Name, c.TestCase.Name, c.TestCase.ID, targets) + } + fmt.Println() +} + +// runCases starts one run per case under a semaphore sized from +// sauce.concurrency, then collects results. Unlike apitest's unbounded +// fan-out, no more than the configured number of runs are ever in flight +// (SC-011): an authored run consumes real VM or device capacity. +func (r *Runner) runCases(ctx context.Context, cases []ResolvedCase) bool { + ccy := r.Project.Sauce.Concurrency + if ccy < 1 { + ccy = 1 + } + totalJobs := 0 + for _, c := range cases { + totalJobs += expectedJobs(c) + } + log.Info().Int("concurrency", ccy).Int("testCases", len(cases)).Int("jobs", totalJobs). + Msg("Starting AI-authored test runs.") + + results := make(chan runResult, len(cases)) + sem := make(chan struct{}, ccy) + // gate serialises acquisition. Without it two cases each needing two of + // two slots would take one apiece and wait for ever for the other's — + // a deadlock, not merely unfair. Holding the gate while waiting means + // one case queues behind another, which is the price of a correct + // ceiling and is invisible at these sizes. + gate := make(chan struct{}, 1) + for _, c := range cases { + go func(c ResolvedCase) { + // The limit counts Sauce jobs, not runs: one run fans out to one + // job per target, so a per-run semaphore would let a suite with + // four targets put four times the configured load on the + // organisation's capacity (SC-011). Weight each case by the jobs + // it will start, capped at the whole budget so a case needing + // more than that still runs — alone — rather than never. + weight := expectedJobs(c) + if weight > ccy { + weight = ccy + } + if !acquire(ctx, gate, sem, weight) { + results <- runResult{Case: c, Err: ctx.Err(), Interrupted: true, StartTime: time.Now(), EndTime: time.Now()} + return + } + defer release(sem, weight) + results <- r.runCase(ctx, c) + }(c) + } + + return r.collectResults(ctx, results, len(cases)) +} + +// expectedJobs reports how many Sauce jobs one resolved case will start. The +// service starts one per target: the suite's targets when it overrides them, +// otherwise the case's own stored run targets, otherwise its single primary +// target. Everything needed is already in hand, so this costs no request. +func expectedJobs(c ResolvedCase) int { + if n := len(c.Suite.Targets); n > 0 { + return n + } + if n := len(c.TestCase.RunSettings.RunTargets); n > 0 { + return n + } + return 1 +} + +// acquire takes n slots, or reports false if the context ended first. +// +// The gate ensures only one caller collects slots at a time, so a caller +// asking for n <= capacity always eventually gets them as holders finish. +// Collecting slots concurrently would let two callers each hold part of the +// budget and deadlock waiting for the rest. +func acquire(ctx context.Context, gate, sem chan struct{}, n int) bool { + select { + case gate <- struct{}{}: + case <-ctx.Done(): + return false + } + defer func() { <-gate }() + + for i := 0; i < n; i++ { + select { + case sem <- struct{}{}: + case <-ctx.Done(): + release(sem, i) + return false + } + } + return true +} + +// release returns n slots. +func release(sem chan struct{}, n int) { + for i := 0; i < n; i++ { + <-sem + } +} + +// runCase starts one run and, unless Async, polls it to completion. +func (r *Runner) runCase(ctx context.Context, c ResolvedCase) runResult { + res := runResult{Case: c, StartTime: time.Now()} + + // No configured tunnel means an explicit null on the wire, which also + // clears a stored (possibly empty and therefore broken) tunnel name on + // the case (research Open-3). + opts := RunOptions{ + BuildName: r.Project.Sauce.Metadata.Build, + TunnelName: r.Project.Sauce.Tunnel.Name, + Targets: c.Suite.Targets, + } + + run, err := r.TestCases.RunTestCase(ctx, c.TestCase.ID, "", opts) + if err != nil { + res.Err = fmt.Errorf("failed to start run: %w", err) + res.EndTime = time.Now() + return res + } + // The run resource is polled with its own testCaseId (research R-004). + // If the start response omitted it, fall back to the identifier we asked + // with: polling an empty one 404s, and isFatalPollError treats 404 as + // transient, so it would burn the whole suite timeout in silence. + if run.TestCaseID == "" { + run.TestCaseID = c.TestCase.ID + } + res.Run = run + for _, j := range run.Jobs { + log.Info(). + Str("suite", c.Suite.Name). + Str("testCase", c.TestCase.Name). + Str("url", r.jobURL(j.SauceJobID)). + Msg("Run started.") + } + + if r.Async { + res.EndTime = time.Now() + return res + } + + latest, timedOut, err := r.pollRun(ctx, run, c.Suite.Timeout) + res.Run, res.TimedOut, res.EndTime = latest, timedOut, time.Now() + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + res.Interrupted = true + } + res.Err = err + } + + // We have stopped watching, so stop the work: otherwise a cancelled or + // timed-out run keeps a VM or device busy for its full duration, which is + // what every other kind avoids (internal/saucecloud/cloud.go). + if res.TimedOut || res.Interrupted { + r.stopRun(res.Run, c) + } + return res +} + +// stopRun asks the service to stop every job of a run we are no longer +// waiting for. Errors are ignored, as they are for the other kinds: a job may +// already have ended, or be in a state that cannot be stopped, and either way +// there is nothing to do about it. +func (r *Runner) stopRun(run Run, c ResolvedCase) { + if r.Stopper == nil { + return + } + // The caller's context is already cancelled on Ctrl-C, so stopping needs + // one that outlives it. This mirrors the localCtx in saucecloud, whose + // comment says it exists so jobs are not left abandoned. + stopCtx, cancel := context.WithTimeout(context.WithoutCancel(context.Background()), stopJobTimeout) + defer cancel() + + for _, j := range run.Jobs { + if j.SauceJobID == "" { + continue + } + log.Info(). + Str("suite", c.Suite.Name). + Str("testCase", c.TestCase.Name). + Str("job", j.SauceJobID). + Msg("Attempting to stop job...") + _, _ = r.Stopper.StopJob(stopCtx, j.SauceJobID, j.RealDevice()) + } +} + +// pollRun polls the run until every job has an outcome, the suite timeout +// expires, or the context ends. It polls before the first wait and returns +// at once when the start response is already terminal, so it behaves +// correctly whether the service answers asynchronously (observed) or +// synchronously. Transient failures (network, 5xx, and 404 which may be +// propagation lag) are retried until the deadline; other 4xx are fatal. The +// timeout is an argument, never package state, so one suite's setting +// cannot leak into another's (research R-012). +func (r *Runner) pollRun(ctx context.Context, run Run, timeout time.Duration) (Run, bool, error) { + if run.Done() { + return run, false, nil + } + + interval := r.PollInterval + if interval <= 0 { + interval = DefaultPollInterval + } + if timeout <= 0 { + timeout = DefaultSuiteTimeout + } + deadline := time.NewTimer(timeout) + defer deadline.Stop() + + for { + latest, err := r.TestCases.GetRun(ctx, run.TestCaseID, run.ID) + switch { + case err == nil: + run = latest + if run.Done() { + return run, false, nil + } + case ctx.Err() != nil: + return run, false, ctx.Err() + case IsFatalPollError(err): + return run, false, fmt.Errorf("failed to poll run %s: %w", run.ID, err) + default: + log.Debug().Err(err).Str("run", run.ID).Msg("Transient error while polling run; retrying.") + } + + select { + case <-ctx.Done(): + return run, false, ctx.Err() + case <-deadline.C: + return run, true, nil + case <-time.After(interval): + } + } +} + +// IsFatalPollError reports whether a poll error cannot be recovered by +// waiting: a 4xx other than 404. 404 is tolerated because a freshly started +// run — or a freshly accepted generation task — might not be readable for a +// moment; the caller's deadline still bounds the retrying. Exported so the +// generate wait applies the same classification as the run poll; two loops in +// one feature disagreeing about which errors are fatal is how a user ends up +// told to keep polling a task that does not exist. +func IsFatalPollError(err error) bool { + var apiErr *APIError + if !errors.As(err, &apiErr) { + return false + } + return apiErr.HTTPStatus >= 400 && apiErr.HTTPStatus < 500 && apiErr.HTTPStatus != 404 +} + +// collectResults drains the results, feeds the reporters and renders them. +// It returns whether everything passed. +func (r *Runner) collectResults(ctx context.Context, results <-chan runResult, expected int) bool { + passed := true + var inProgress atomic.Int32 + inProgress.Store(int32(expected)) + + done := make(chan struct{}) + go func() { + t := time.NewTicker(10 * time.Second) + defer t.Stop() + for { + select { + case <-done: + return + case <-t.C: + log.Info().Msgf("Runs in progress: %d", inProgress.Load()) + } + } + }() + + wantJUnit := report.IsArtifactRequired(r.Reporters, report.JUnitArtifact) + buildURLs := map[build.Source]string{} + + for i := 0; i < expected; i++ { + res := <-results + inProgress.Add(-1) + + r.logResult(res) + if !r.resultPassed(res) { + passed = false + } + for _, tr := range r.toTestResults(ctx, res, wantJUnit, buildURLs) { + for _, rep := range r.Reporters { + rep.Add(tr) + } + } + } + close(done) + + for _, rep := range r.Reporters { + rep.Render() + } + return passed +} + +// resultPassed decides the exit code contribution of one result. Under Async +// a started run counts as passing because its outcome is unknown by +// definition (FR-009). +func (r *Runner) resultPassed(res runResult) bool { + if res.Err != nil || res.TimedOut || res.Interrupted { + return false + } + if r.Async { + return true + } + return res.Run.Passed() +} + +// logResult writes one line per finished case, telling the user how to +// follow a run that is still going when the wait ended early. +func (r *Runner) logResult(res runResult) { + ev := log.Info() + msg := "Run finished." + checkHint := fmt.Sprintf("saucectl authoring testcases get-run %s %s", res.Run.TestCaseID, res.Run.ID) + + switch { + case res.Interrupted && res.Run.ID == "": + ev = log.Warn() + msg = "Run was not started: interrupted." + case res.Interrupted: + ev = log.Warn() + msg = "Interrupted; stopping the run on Sauce Labs. Check it with: " + checkHint + case res.Err != nil: + ev = log.Error().Err(res.Err) + msg = "Run failed." + case res.TimedOut: + ev = log.Error() + msg = "Timed out waiting; stopping the run on Sauce Labs. Check it with: " + checkHint + case r.Async: + msg = "Run started (async). Check it with: " + checkHint + case !res.Run.Passed(): + ev = log.Error() + msg = "Run finished with failures." + } + + ev.Str("suite", res.Case.Suite.Name).Str("testCase", res.Case.TestCase.Name).Str("run", res.Run.ID).Msg(msg) +} + +// toTestResults turns one run into one report.TestResult per job — one per +// browser or device, not one per suite (FR-003). A run that never produced +// jobs (start failure, interruption before start) yields a single failed or +// in-progress result so it is never silently absent from the report. +func (r *Runner) toTestResults(ctx context.Context, res runResult, wantJUnit bool, buildURLs map[build.Source]string) []report.TestResult { + name := res.Case.Suite.Name + " - " + res.Case.TestCase.Name + duration := res.EndTime.Sub(res.StartTime) + + if len(res.Run.Jobs) == 0 { + // Only a start failure is a failure. An interruption, or an accepted + // asynchronous run whose jobs the service has not reported yet, is + // still in progress — reporting it as failed would contradict the + // zero exit code resultPassed returns under Async. + status := job.StateFailed + if res.Interrupted || (r.Async && res.Err == nil) { + status = job.StateInProgress + } + tr := report.TestResult{ + Name: name, + Duration: duration, + StartTime: res.StartTime, + EndTime: res.EndTime, + Status: status, + TimedOut: res.TimedOut, + Attempts: []report.Attempt{{Duration: duration, StartTime: res.StartTime, EndTime: res.EndTime, Status: status}}, + } + if wantJUnit { + tr.Attempts[0].TestSuites = synthesizeJUnit(res.Case, RunJob{Error: errString(res.Err)}, status, duration) + } + return []report.TestResult{tr} + } + + out := make([]report.TestResult, 0, len(res.Run.Jobs)) + for _, j := range res.Run.Jobs { + status := jobState(j) + browser, platform, device := DescribeCapabilities(j.Target.Capabilities) + + tr := report.TestResult{ + Name: name, + Duration: duration, + StartTime: res.StartTime, + EndTime: res.EndTime, + Status: status, + Browser: browser, + Platform: platform, + DeviceName: device, + URL: r.jobURL(j.SauceJobID), + RunID: res.Run.ID, + // RDC routes the row into the real-device table, which has its + // own build link (internal/report/buildtable). + RDC: j.RealDevice(), + // TimedOut is what makes the table count an unfinished run as an + // error rather than merely in progress (internal/report/table). + TimedOut: res.TimedOut, + Attempts: []report.Attempt{{ + ID: j.SauceJobID, + Duration: duration, + StartTime: res.StartTime, + EndTime: res.EndTime, + Status: status, + }}, + } + + if wantJUnit { + tr.Attempts[0].TestSuites = synthesizeJUnit(res.Case, j, status, duration) + } + if !r.Async && j.SauceJobID != "" { + tr.BuildURL = r.buildURL(ctx, j, buildURLs) + } + if !r.Async && r.Artifacts != nil && j.Done() && j.SauceJobID != "" { + tr.Artifacts = r.downloadArtifacts(ctx, name, j, status, res.TimedOut) + } + + out = append(out, tr) + } + return out +} + +// jobState maps a job's inferred outcome onto the shared result states. +// There is no status on the wire; nil success is "not yet reported" +// (research R-005). +func jobState(j RunJob) string { + switch { + case j.Passed(): + return job.StatePassed + case j.Done(): + return job.StateFailed + default: + return job.StateInProgress + } +} + +// jobURL derives the dashboard link for one job in this runner's region. +func (r *Runner) jobURL(sauceJobID string) string { + return JobURL(r.Region, sauceJobID) +} + +// buildURL resolves the build link through the build service by job ID, +// caching per source: every job of this invocation shares the build. The +// service decorates the build name ("X" becomes "X - 1"), which is why the +// lookup is by job and never by name. +func (r *Runner) buildURL(ctx context.Context, j RunJob, cache map[build.Source]string) string { + if r.Builds == nil { + return "" + } + source := build.SourceVDC + if j.RealDevice() { + source = build.SourceRDC + } + if u, ok := cache[source]; ok { + return u + } + + b, err := r.Builds.GetBuild(ctx, build.GetBuildOptions{ID: j.SauceJobID, Source: source, ByJob: true}) + if err != nil { + log.Debug().Err(err).Str("job", j.SauceJobID).Msg("Unable to resolve the build link.") + return "" + } + cache[source] = b.URL + return b.URL +} + +// downloadArtifacts hands the job to the shared downloader with the fields +// its skip rules read: a done state, the pass flag and the timeout flag. +func (r *Runner) downloadArtifacts(ctx context.Context, name string, j RunJob, status string, timedOut bool) []report.Artifact { + files := r.Artifacts.DownloadArtifacts(ctx, job.Job{ + ID: j.SauceJobID, + Name: name, + Passed: j.Passed(), + Status: status, + IsRDC: j.RealDevice(), + TimedOut: timedOut, + }, true) + + arts := make([]report.Artifact, 0, len(files)) + for _, f := range files { + arts = append(arts, report.Artifact{FilePath: f}) + } + return arts +} + +// synthesizeJUnit builds the JUnit content for one job. Authored runs publish +// no junit.xml asset (research Open-2), and the JUnit reporter renders +// elements only from Attempt.TestSuites, so without this the +// report would be a set of empty containers that downstream +// consumers show as "no tests" (research R-011, FR-006, SC-008). +func synthesizeJUnit(c ResolvedCase, j RunJob, status string, duration time.Duration) junit.TestSuites { + tc := junit.TestCase{ + Name: c.TestCase.Name, + ClassName: c.Suite.Name, + Time: fmt.Sprintf("%.0f", duration.Seconds()), + Status: status, + } + ts := junit.TestSuite{ + Name: c.Suite.Name + " - " + c.TestCase.Name, + Tests: 1, + Time: tc.Time, + } + + switch status { + case job.StateFailed: + msg := j.Error + if msg == "" { + msg = "test case failed" + } + tc.Failure = &junit.Failure{Message: msg, Type: "failure", Text: msg} + ts.Failures = 1 + case job.StateInProgress: + msg := "the run did not finish before saucectl stopped waiting" + tc.Error = &junit.Error{Message: msg, Type: "timeout", Text: msg} + ts.Errors = 1 + } + + ts.TestCases = []junit.TestCase{tc} + return junit.TestSuites{TestSuites: []junit.TestSuite{ts}} +} + +// errString renders an error for a synthesized failure message. +func errString(err error) string { + if err == nil { + return "" + } + return err.Error() +} diff --git a/internal/authoring/runner_test.go b/internal/authoring/runner_test.go new file mode 100644 index 000000000..9bcc638b4 --- /dev/null +++ b/internal/authoring/runner_test.go @@ -0,0 +1,806 @@ +package authoring + +import ( + "context" + "errors" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/saucelabs/saucectl/internal/build" + "github.com/saucelabs/saucectl/internal/config" + "github.com/saucelabs/saucectl/internal/job" + "github.com/saucelabs/saucectl/internal/report" + "github.com/saucelabs/saucectl/internal/tunnel" +) + +// fakeService is a package-local fake for the two services the runner uses. +// It cannot live in internal/mocks: that package imports this one, and a +// test here importing it would form a cycle. +type fakeService struct { + TestCaseService + TestSuiteService + getTestCase func(ctx context.Context, id string) (TestCase, error) + listTestCases func(ctx context.Context, opts ListTestCasesOptions) (List[TestCase], error) + listTestSuites func(ctx context.Context, opts ListTestSuitesOptions) (List[TestSuite], error) + runTestCase func(ctx context.Context, id, revisionID string, opts RunOptions) (Run, error) + getRun func(ctx context.Context, testCaseID, runID string) (Run, error) +} + +func (f *fakeService) GetTestCase(ctx context.Context, id string) (TestCase, error) { + return f.getTestCase(ctx, id) +} + +func (f *fakeService) ListTestCases(ctx context.Context, opts ListTestCasesOptions) (List[TestCase], error) { + return f.listTestCases(ctx, opts) +} + +func (f *fakeService) ListTestSuites(ctx context.Context, opts ListTestSuitesOptions) (List[TestSuite], error) { + return f.listTestSuites(ctx, opts) +} + +func (f *fakeService) RunTestCase(ctx context.Context, id, revisionID string, opts RunOptions) (Run, error) { + return f.runTestCase(ctx, id, revisionID, opts) +} + +func (f *fakeService) GetRun(ctx context.Context, testCaseID, runID string) (Run, error) { + return f.getRun(ctx, testCaseID, runID) +} + +// captureReporter records what the runner reports. +type captureReporter struct { + mu sync.Mutex + results []report.TestResult + rendered bool + wantJUnit bool +} + +func (c *captureReporter) Add(t report.TestResult) { + c.mu.Lock() + defer c.mu.Unlock() + c.results = append(c.results, t) +} +func (c *captureReporter) Render() { c.rendered = true } +func (c *captureReporter) Reset() { c.results = nil } +func (c *captureReporter) ArtifactRequirements() []report.ArtifactType { + if c.wantJUnit { + return []report.ArtifactType{report.JUnitArtifact} + } + return nil +} + +// fakeBuilds resolves every job to one build URL. +type fakeBuilds struct{ calls int32 } + +func (f *fakeBuilds) GetBuild(_ context.Context, opts build.GetBuildOptions) (build.Build, error) { + atomic.AddInt32(&f.calls, 1) + return build.Build{ID: "b", URL: "https://app.saucelabs.com/builds/" + string(opts.Source) + "/b"}, nil +} +func (f *fakeBuilds) ListBuilds(context.Context, build.ListBuildsOptions) ([]build.Build, error) { + return nil, nil +} + +// fakeTunnels records the tunnel readiness check. +type fakeTunnels struct{ name, owner string } + +func (f *fakeTunnels) IsTunnelRunning(_ context.Context, id, owner string, _ tunnel.Filter, _ time.Duration) error { + f.name, f.owner = id, owner + return nil +} + +// fakeDownloader records the jobs handed to it. +type fakeDownloader struct { + mu sync.Mutex + jobs []job.Job +} + +func (f *fakeDownloader) DownloadArtifacts(_ context.Context, j job.Job, _ bool) []string { + f.mu.Lock() + defer f.mu.Unlock() + f.jobs = append(f.jobs, j) + return []string{"artifacts/" + j.ID + "/video.mp4"} +} + +func newProject(suites ...Suite) Project { + p := Project{ + Sauce: config.SauceConfig{Region: "us-west-1", Concurrency: 2, Metadata: config.Metadata{Build: "nightly"}}, + Suites: suites, + } + SetDefaults(&p) + return p +} + +func chromeJob(success *bool) RunJob { + return RunJob{ID: "j1", SauceJobID: "sauce-1", Name: "n", Success: success, + Target: Target{Capabilities: map[string]any{"browserName": "chrome", "browserVersion": "latest", "platformName": "Windows 11"}}} +} + +func newRunner(svc *fakeService, rep *captureReporter, p Project) *Runner { + return &Runner{Project: p, TestCases: svc, TestSuites: svc, Reporters: []report.Reporter{rep}, PollInterval: time.Millisecond} +} + +func TestRunner_SynchronousStartNeverPolls(t *testing.T) { + yes := true + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: "case"}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + return Run{ID: "run", TestCaseID: id, Jobs: []RunJob{chromeJob(&yes)}}, nil + }, + getRun: func(context.Context, string, string) (Run, error) { + t.Fatal("a terminal start response must not be polled") + return Run{}, nil + }, + } + rep := &captureReporter{} + r := newRunner(svc, rep, newProject(Suite{Name: "s", TestCases: []string{"tc1"}})) + + code, err := r.RunProject(context.Background()) + if err != nil || code != 0 { + t.Fatalf("exit %d, err %v", code, err) + } + if len(rep.results) != 1 || rep.results[0].Status != job.StatePassed || !rep.rendered { + t.Errorf("results = %+v rendered=%v", rep.results, rep.rendered) + } + res := rep.results[0] + if res.Name != "s - case" || res.Browser != "chrome latest" || res.Platform != "Windows 11" || res.URL != "/tests/sauce-1" || res.RDC { + t.Errorf("result fields = %+v", res) + } +} + +func TestRunner_PollsUntilDone(t *testing.T) { + yes := true + var polls int32 + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: "case"}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + return Run{ID: "run", TestCaseID: id, Jobs: []RunJob{chromeJob(nil)}}, nil + }, + getRun: func(_ context.Context, testCaseID, runID string) (Run, error) { + if testCaseID != "tc1" || runID != "run" { + t.Errorf("polled with %s/%s; must use the run's own testCaseId", testCaseID, runID) + } + n := atomic.AddInt32(&polls, 1) + j := chromeJob(nil) + if n >= 3 { + j.Success = &yes + } + return Run{ID: runID, TestCaseID: testCaseID, Jobs: []RunJob{j}}, nil + }, + } + rep := &captureReporter{} + r := newRunner(svc, rep, newProject(Suite{Name: "s", TestCases: []string{"tc1"}})) + + code, _ := r.RunProject(context.Background()) + if code != 0 || atomic.LoadInt32(&polls) != 3 { + t.Errorf("exit %d after %d polls", code, polls) + } +} + +func TestRunner_MultiTargetReportsOnePerJob(t *testing.T) { + yes, no := true, false + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: "case"}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + return Run{ID: "run", TestCaseID: id, Jobs: []RunJob{ + chromeJob(&yes), + {ID: "j2", SauceJobID: "sauce-2", Success: &no, Error: "assertion failed", + Target: Target{IsRDC: true, Capabilities: map[string]any{"platformName": "Android", "appium:platformVersion": "16", "appium:deviceName": "Google Pixel 9"}}}, + }}, nil + }, + } + rep := &captureReporter{} + r := newRunner(svc, rep, newProject(Suite{Name: "s", TestCases: []string{"tc1"}})) + + code, _ := r.RunProject(context.Background()) + if code != 1 { + t.Errorf("exit = %d, want 1 because one job failed", code) + } + if len(rep.results) != 2 { + t.Fatalf("got %d results, want one per job", len(rep.results)) + } + var rdc report.TestResult + for _, res := range rep.results { + if res.RDC { + rdc = res + } + } + if rdc.Status != job.StateFailed || rdc.DeviceName != "Google Pixel 9" || rdc.Platform != "Android 16" { + t.Errorf("rdc result = %+v", rdc) + } +} + +func TestRunner_TimeoutSetsTimedOut(t *testing.T) { + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: "case"}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + return Run{ID: "run", TestCaseID: id, Jobs: []RunJob{chromeJob(nil)}}, nil + }, + getRun: func(_ context.Context, tc, id string) (Run, error) { + return Run{ID: id, TestCaseID: tc, Jobs: []RunJob{chromeJob(nil)}}, nil + }, + } + rep := &captureReporter{} + r := newRunner(svc, rep, newProject(Suite{Name: "s", TestCases: []string{"tc1"}, Timeout: 30 * time.Millisecond})) + + code, _ := r.RunProject(context.Background()) + if code != 1 { + t.Errorf("exit = %d, want 1", code) + } + res := rep.results[0] + if !res.TimedOut || res.Status != job.StateInProgress { + t.Errorf("a timed-out run must be in progress AND TimedOut so the table counts it as an error: %+v", res) + } +} + +func TestRunner_StartFailure(t *testing.T) { + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: "case"}, nil }, + runTestCase: func(context.Context, string, string, RunOptions) (Run, error) { + return Run{}, &APIError{HTTPStatus: 400, Code: "SC_TUNNEL_NOT_FOUND", Detail: "Sauce Connect tunnel does not exist."} + }, + } + rep := &captureReporter{wantJUnit: true} + r := newRunner(svc, rep, newProject(Suite{Name: "s", TestCases: []string{"tc1"}})) + + code, _ := r.RunProject(context.Background()) + if code != 1 || len(rep.results) != 1 || rep.results[0].Status != job.StateFailed { + t.Fatalf("exit %d results %+v", code, rep.results) + } + suites := rep.results[0].Attempts[0].TestSuites.TestSuites + if len(suites) != 1 || len(suites[0].TestCases) != 1 || suites[0].TestCases[0].Failure == nil { + t.Fatalf("junit synthesis missing for a start failure: %+v", suites) + } + if !strings.Contains(suites[0].TestCases[0].Failure.Message, "SC_TUNNEL_NOT_FOUND") { + t.Errorf("failure message = %q", suites[0].TestCases[0].Failure.Message) + } +} + +func TestRunner_Async(t *testing.T) { + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: "case"}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + return Run{ID: "run", TestCaseID: id, Jobs: []RunJob{chromeJob(nil)}}, nil + }, + getRun: func(context.Context, string, string) (Run, error) { + t.Fatal("async must not poll") + return Run{}, nil + }, + } + rep := &captureReporter{} + r := newRunner(svc, rep, newProject(Suite{Name: "s", TestCases: []string{"tc1"}})) + r.Async = true + r.Builds = &fakeBuilds{} + + code, _ := r.RunProject(context.Background()) + if code != 0 || rep.results[0].Status != job.StateInProgress || rep.results[0].BuildURL != "" { + t.Errorf("exit %d results %+v", code, rep.results) + } +} + +func TestRunner_ConcurrencyCeiling(t *testing.T) { + yes := true + var inFlight, maxInFlight int32 + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: id}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + n := atomic.AddInt32(&inFlight, 1) + for { + m := atomic.LoadInt32(&maxInFlight) + if n <= m || atomic.CompareAndSwapInt32(&maxInFlight, m, n) { + break + } + } + time.Sleep(20 * time.Millisecond) + atomic.AddInt32(&inFlight, -1) + return Run{ID: "run-" + id, TestCaseID: id, Jobs: []RunJob{chromeJob(&yes)}}, nil + }, + } + rep := &captureReporter{} + p := newProject(Suite{Name: "s", TestCases: []string{"a", "b", "c", "d", "e"}}) + p.Sauce.Concurrency = 2 + r := newRunner(svc, rep, p) + + code, _ := r.RunProject(context.Background()) + if code != 0 || len(rep.results) != 5 { + t.Fatalf("exit %d, %d results", code, len(rep.results)) + } + if maxInFlight > 2 { + t.Errorf("max in flight = %d, want <= 2 (SC-011)", maxInFlight) + } +} + +func TestRunner_ContextCancellation(t *testing.T) { + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: "case"}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + return Run{ID: "run", TestCaseID: id, Jobs: []RunJob{chromeJob(nil)}}, nil + }, + getRun: func(_ context.Context, tc, id string) (Run, error) { + return Run{ID: id, TestCaseID: tc, Jobs: []RunJob{chromeJob(nil)}}, nil + }, + } + rep := &captureReporter{} + r := newRunner(svc, rep, newProject(Suite{Name: "s", TestCases: []string{"tc1"}, Timeout: time.Hour})) + + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(20 * time.Millisecond) + cancel() + }() + start := time.Now() + code, _ := r.RunProject(ctx) + if time.Since(start) > 2*time.Second { + t.Error("cancellation did not stop polling promptly") + } + if code != 1 { + t.Errorf("exit = %d, want 1 on interruption", code) + } + res := rep.results[0] + if res.Status != job.StateInProgress || res.TimedOut { + t.Errorf("an interrupted run is still running remotely, not timed out: %+v", res) + } +} + +func TestRunner_JUnitSynthesisAndArtifactsAndBuildLink(t *testing.T) { + no := false + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: "Login"}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + j := chromeJob(&no) + j.Error = "element not found" + return Run{ID: "run", TestCaseID: id, Jobs: []RunJob{j}}, nil + }, + } + rep := &captureReporter{wantJUnit: true} + dl := &fakeDownloader{} + r := newRunner(svc, rep, newProject(Suite{Name: "Smoke", TestCases: []string{"tc1"}})) + r.Artifacts = dl + r.Builds = &fakeBuilds{} + + code, _ := r.RunProject(context.Background()) + if code != 1 { + t.Errorf("exit = %d", code) + } + res := rep.results[0] + + ts := res.Attempts[0].TestSuites.TestSuites + if len(ts) != 1 || ts[0].Tests != 1 || ts[0].Failures != 1 || len(ts[0].TestCases) != 1 { + t.Fatalf("synthesized junit = %+v", ts) + } + tc := ts[0].TestCases[0] + if tc.Name != "Login" || tc.ClassName != "Smoke" || tc.Failure == nil || tc.Failure.Message != "element not found" { + t.Errorf("junit test case = %+v", tc) + } + + if res.BuildURL != "https://app.saucelabs.com/builds/vdc/b" { + t.Errorf("build url = %q", res.BuildURL) + } + if len(dl.jobs) != 1 || dl.jobs[0].ID != "sauce-1" || dl.jobs[0].Status != job.StateFailed || dl.jobs[0].Passed || dl.jobs[0].TimedOut { + t.Errorf("downloader job = %+v; Status/Passed/TimedOut drive skipDownload", dl.jobs) + } + if len(res.Artifacts) != 1 || res.Artifacts[0].FilePath != "artifacts/sauce-1/video.mp4" { + t.Errorf("artifacts = %+v", res.Artifacts) + } +} + +func TestRunner_RunOptionsCarryConfig(t *testing.T) { + yes := true + var got RunOptions + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id}, nil }, + runTestCase: func(_ context.Context, id, rev string, opts RunOptions) (Run, error) { + got = opts + if rev != "" { + t.Errorf("revision = %q, want latest", rev) + } + return Run{ID: "run", TestCaseID: id, Jobs: []RunJob{chromeJob(&yes)}}, nil + }, + } + p := newProject(Suite{Name: "s", TestCases: []string{"tc1"}, Targets: []Target{{Capabilities: map[string]any{"browserName": "firefox"}}}}) + p.Sauce.Tunnel.Name = "my-tunnel" + SetDefaults(&p) + tunnels := &fakeTunnels{} + r := newRunner(svc, &captureReporter{}, p) + r.Tunnels = tunnels + + code, err := r.RunProject(context.Background()) + if err != nil || code != 0 { + t.Fatalf("exit %d, err %v", code, err) + } + if tunnels.name != "my-tunnel" { + t.Errorf("tunnel readiness was not checked for %q", "my-tunnel") + } + if got.BuildName != "nightly" || got.TunnelName != "my-tunnel" || len(got.Targets) != 1 || got.Targets[0].Capabilities["browserName"] != "firefox" { + t.Errorf("run options = %+v", got) + } +} + +func TestRunner_TransientAndFatalPollErrors(t *testing.T) { + yes := true + t.Run("5xx is retried", func(t *testing.T) { + var polls int32 + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + return Run{ID: "run", TestCaseID: id, Jobs: []RunJob{chromeJob(nil)}}, nil + }, + getRun: func(_ context.Context, tc, id string) (Run, error) { + if atomic.AddInt32(&polls, 1) == 1 { + return Run{}, &APIError{HTTPStatus: 503, Detail: "unavailable"} + } + return Run{ID: id, TestCaseID: tc, Jobs: []RunJob{chromeJob(&yes)}}, nil + }, + } + r := newRunner(svc, &captureReporter{}, newProject(Suite{Name: "s", TestCases: []string{"tc1"}})) + if code, _ := r.RunProject(context.Background()); code != 0 { + t.Errorf("exit = %d", code) + } + }) + t.Run("403 is fatal", func(t *testing.T) { + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + return Run{ID: "run", TestCaseID: id, Jobs: []RunJob{chromeJob(nil)}}, nil + }, + getRun: func(context.Context, string, string) (Run, error) { + return Run{}, &APIError{HTTPStatus: 403, Code: "UNAUTHORIZED"} + }, + } + rep := &captureReporter{} + r := newRunner(svc, rep, newProject(Suite{Name: "s", TestCases: []string{"tc1"}, Timeout: time.Hour})) + start := time.Now() + code, _ := r.RunProject(context.Background()) + if code != 1 || time.Since(start) > time.Second { + t.Errorf("exit = %d after %v; a 4xx must fail fast", code, time.Since(start)) + } + }) +} + +func TestRunner_ResolveTestCases(t *testing.T) { + suites := []TestSuite{{ID: "demo", Name: "Demo"}, {ID: "demo-suite", Name: "Demo Suite"}, {ID: "dup1", Name: "Dup"}, {ID: "dup2", Name: "Dup"}} + var listedSuiteIDs []string + svc := &fakeService{ + listTestSuites: func(_ context.Context, opts ListTestSuitesOptions) (List[TestSuite], error) { + // The service's search is a substring match: everything containing + // the term comes back and the exact match must be made here. + var out []TestSuite + for _, s := range suites { + if strings.Contains(strings.ToLower(s.Name), strings.ToLower(opts.Search)) { + out = append(out, s) + } + } + return List[TestSuite]{Items: out, Total: len(out)}, nil + }, + listTestCases: func(_ context.Context, opts ListTestCasesOptions) (List[TestCase], error) { + listedSuiteIDs = opts.TestSuiteIDs + if opts.Skip > 0 { + return List[TestCase]{Total: 2}, nil + } + return List[TestCase]{Items: []TestCase{{ID: "a"}, {ID: "b"}}, Total: 2}, nil + }, + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: "n-" + id}, nil }, + } + + t.Run("exact name among substring matches", func(t *testing.T) { + r := newRunner(svc, &captureReporter{}, newProject(Suite{Name: "s", TestSuiteName: "Demo"})) + cases, err := r.ResolveTestCases(context.Background()) + if err != nil || len(cases) != 2 || listedSuiteIDs[0] != "demo" { + t.Errorf("cases=%v err=%v listed=%v", cases, err, listedSuiteIDs) + } + }) + t.Run("ambiguous name", func(t *testing.T) { + r := newRunner(svc, &captureReporter{}, newProject(Suite{Name: "s", TestSuiteName: "Dup"})) + if _, err := r.ResolveTestCases(context.Background()); err == nil || !strings.Contains(err.Error(), "2 test suites") { + t.Errorf("err = %v", err) + } + }) + t.Run("unknown name", func(t *testing.T) { + r := newRunner(svc, &captureReporter{}, newProject(Suite{Name: "s", TestSuiteName: "Nope"})) + if _, err := r.ResolveTestCases(context.Background()); err == nil { + t.Error("expected error") + } + }) + t.Run("by id and by explicit cases", func(t *testing.T) { + r := newRunner(svc, &captureReporter{}, newProject( + Suite{Name: "by-id", TestSuiteID: "xyz", Tags: []string{"smoke"}}, + Suite{Name: "explicit", TestCases: []string{"c1"}}, + )) + cases, err := r.ResolveTestCases(context.Background()) + if err != nil || len(cases) != 3 { + t.Fatalf("cases=%v err=%v", cases, err) + } + if listedSuiteIDs[0] != "xyz" || cases[2].TestCase.Name != "n-c1" || cases[2].Suite.Name != "explicit" { + t.Errorf("resolution wrong: %v %+v", listedSuiteIDs, cases[2]) + } + }) + t.Run("missing explicit case is an error", func(t *testing.T) { + bad := &fakeService{getTestCase: func(context.Context, string) (TestCase, error) { + return TestCase{}, &APIError{HTTPStatus: 404, Code: "TEST_CASE_NOT_FOUND"} + }} + r := newRunner(bad, &captureReporter{}, newProject(Suite{Name: "s", TestCases: []string{"nope"}})) + if _, err := r.ResolveTestCases(context.Background()); !errors.Is(err, ErrTestCaseNotFound) { + t.Errorf("err = %v", err) + } + }) +} + +func TestRunner_DryRunStartsNothing(t *testing.T) { + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: "case"}, nil }, + runTestCase: func(context.Context, string, string, RunOptions) (Run, error) { + t.Fatal("dry run must not start a run") + return Run{}, nil + }, + } + p := newProject(Suite{Name: "s", TestCases: []string{"tc1"}}) + p.DryRun = true + rep := &captureReporter{} + r := newRunner(svc, rep, p) + code, err := r.RunProject(context.Background()) + if err != nil || code != 0 || len(rep.results) != 0 { + t.Errorf("exit %d err %v results %d", code, err, len(rep.results)) + } +} + +func TestRunner_AsyncRunWithNoJobsIsInProgressNotFailed(t *testing.T) { + // A start response that carries no jobs yet is not a failure. Reporting + // it as failed contradicted the zero exit code Async returns, so a CI + // job gating on the report failed a launch that had worked. + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: "case"}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + return Run{ID: "run", TestCaseID: id}, nil // accepted, no jobs reported + }, + } + rep := &captureReporter{wantJUnit: true} + r := newRunner(svc, rep, newProject(Suite{Name: "s", TestCases: []string{"tc1"}})) + r.Async = true + + code, err := r.RunProject(context.Background()) + if err != nil || code != 0 { + t.Fatalf("exit %d, err %v; an accepted async launch must succeed", code, err) + } + if len(rep.results) != 1 { + t.Fatalf("got %d results", len(rep.results)) + } + res := rep.results[0] + if res.Status != job.StateInProgress { + t.Errorf("status = %q, want %q so the report agrees with the exit code", res.Status, job.StateInProgress) + } + for _, ts := range res.Attempts[0].TestSuites.TestSuites { + if ts.Failures != 0 { + t.Errorf("synthesized JUnit reports %d failure(s) for a successful async launch", ts.Failures) + } + } +} + +func TestRunner_StartFailureWithNoJobsStillFails(t *testing.T) { + // The counterpart: a genuine start failure stays failed even under Async. + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id}, nil }, + runTestCase: func(context.Context, string, string, RunOptions) (Run, error) { + return Run{}, &APIError{HTTPStatus: 400, Code: "SC_TUNNEL_NOT_FOUND"} + }, + } + rep := &captureReporter{} + r := newRunner(svc, rep, newProject(Suite{Name: "s", TestCases: []string{"tc1"}})) + r.Async = true + if code, _ := r.RunProject(context.Background()); code != 1 { + t.Errorf("exit = %d, want 1", code) + } + if rep.results[0].Status != job.StateFailed { + t.Errorf("status = %q, want failed", rep.results[0].Status) + } +} + +func TestRunner_PollsWithKnownTestCaseIDWhenStartResponseOmitsIt(t *testing.T) { + // An empty testCaseId would 404 forever: isFatalPollError treats 404 as + // transient, so the loop would burn the whole suite timeout in silence. + yes := true + var polledWith string + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: "case"}, nil }, + runTestCase: func(_ context.Context, _, _ string, _ RunOptions) (Run, error) { + return Run{ID: "run", TestCaseID: "", Jobs: []RunJob{chromeJob(nil)}}, nil + }, + getRun: func(_ context.Context, testCaseID, runID string) (Run, error) { + polledWith = testCaseID + return Run{ID: runID, TestCaseID: testCaseID, Jobs: []RunJob{chromeJob(&yes)}}, nil + }, + } + r := newRunner(svc, &captureReporter{}, newProject(Suite{Name: "s", TestCases: []string{"tc1"}, Timeout: time.Second})) + if code, _ := r.RunProject(context.Background()); code != 0 { + t.Errorf("exit = %d", code) + } + if polledWith != "tc1" { + t.Errorf("polled with %q, want the requested test case id", polledWith) + } +} + +// fakeStopper records which jobs it was asked to stop. +type fakeStopper struct { + mu sync.Mutex + stopped []string + rdc []bool +} + +func (f *fakeStopper) StopJob(_ context.Context, jobID string, realDevice bool) (job.Job, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.stopped = append(f.stopped, jobID) + f.rdc = append(f.rdc, realDevice) + return job.Job{ID: jobID}, nil +} + +func TestRunner_StopsTheRunOnTimeout(t *testing.T) { + // We have stopped waiting, so the work must stop too: otherwise a + // cancelled run keeps a VM or device busy for its full duration. + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: "case"}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + return Run{ID: "run", TestCaseID: id, Jobs: []RunJob{chromeJob(nil)}}, nil + }, + getRun: func(_ context.Context, tc, id string) (Run, error) { + return Run{ID: id, TestCaseID: tc, Jobs: []RunJob{chromeJob(nil)}}, nil + }, + } + stopper := &fakeStopper{} + r := newRunner(svc, &captureReporter{}, newProject(Suite{Name: "s", TestCases: []string{"tc1"}, Timeout: 30 * time.Millisecond})) + r.Stopper = stopper + + if code, _ := r.RunProject(context.Background()); code != 1 { + t.Errorf("exit = %d, want 1", code) + } + stopper.mu.Lock() + defer stopper.mu.Unlock() + if len(stopper.stopped) != 1 || stopper.stopped[0] != "sauce-1" { + t.Errorf("stopped %v, want the run's job", stopper.stopped) + } +} + +func TestRunner_StopsTheRunOnCancellation(t *testing.T) { + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: "case"}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + return Run{ID: "run", TestCaseID: id, Jobs: []RunJob{chromeJob(nil)}}, nil + }, + getRun: func(_ context.Context, tc, id string) (Run, error) { + return Run{ID: id, TestCaseID: tc, Jobs: []RunJob{chromeJob(nil)}}, nil + }, + } + stopper := &fakeStopper{} + r := newRunner(svc, &captureReporter{}, newProject(Suite{Name: "s", TestCases: []string{"tc1"}, Timeout: time.Hour})) + r.Stopper = stopper + + ctx, cancel := context.WithCancel(context.Background()) + go func() { time.Sleep(20 * time.Millisecond); cancel() }() + _, _ = r.RunProject(ctx) + + stopper.mu.Lock() + defer stopper.mu.Unlock() + // The stop runs on a context detached from the cancelled one, so it must + // still have happened. + if len(stopper.stopped) != 1 { + t.Errorf("stopped %v, want one job despite the cancelled context", stopper.stopped) + } +} + +func TestRunner_AsyncDoesNotStopAnything(t *testing.T) { + // Not waiting is the point of --async; stopping would defeat it. + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: "case"}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + return Run{ID: "run", TestCaseID: id, Jobs: []RunJob{chromeJob(nil)}}, nil + }, + } + stopper := &fakeStopper{} + r := newRunner(svc, &captureReporter{}, newProject(Suite{Name: "s", TestCases: []string{"tc1"}})) + r.Stopper, r.Async = stopper, true + + if code, _ := r.RunProject(context.Background()); code != 0 { + t.Errorf("exit = %d, want 0", code) + } + if len(stopper.stopped) != 0 { + t.Errorf("stopped %v under --async", stopper.stopped) + } +} + +func TestRunner_NilStopperIsSafe(t *testing.T) { + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + return Run{ID: "run", TestCaseID: id, Jobs: []RunJob{chromeJob(nil)}}, nil + }, + getRun: func(_ context.Context, tc, id string) (Run, error) { + return Run{ID: id, TestCaseID: tc, Jobs: []RunJob{chromeJob(nil)}}, nil + }, + } + r := newRunner(svc, &captureReporter{}, newProject(Suite{Name: "s", TestCases: []string{"tc1"}, Timeout: 20 * time.Millisecond})) + if code, _ := r.RunProject(context.Background()); code != 1 { + t.Errorf("exit = %d", code) + } +} + +func TestExpectedJobs(t *testing.T) { + two := []Target{{Capabilities: map[string]any{"browserName": "chrome"}}, {Capabilities: map[string]any{"browserName": "firefox"}}} + tests := []struct { + name string + c ResolvedCase + want int + }{ + {"suite targets win", ResolvedCase{Suite: Suite{Targets: two}, TestCase: TestCase{RunSettings: RunSettings{RunTargets: two}}}, 2}, + {"stored run targets", ResolvedCase{TestCase: TestCase{RunSettings: RunSettings{RunTargets: two}}}, 2}, + {"primary target only", ResolvedCase{}, 1}, + } + for _, tt := range tests { + if got := expectedJobs(tt.c); got != tt.want { + t.Errorf("%s: expectedJobs = %d, want %d", tt.name, got, tt.want) + } + } +} + +func TestRunner_ConcurrencyCountsJobsNotRuns(t *testing.T) { + // One run fans out to one job per target, so a per-run semaphore let a + // two-target suite put twice the configured load on the organisation. + yes := true + two := []Target{{Capabilities: map[string]any{"browserName": "chrome"}}, {Capabilities: map[string]any{"browserName": "firefox"}}} + var inFlight, maxInFlight int32 + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id, Name: id}, nil }, + runTestCase: func(_ context.Context, id, _ string, opts RunOptions) (Run, error) { + n := atomic.AddInt32(&inFlight, int32(len(opts.Targets))) + for { + m := atomic.LoadInt32(&maxInFlight) + if n <= m || atomic.CompareAndSwapInt32(&maxInFlight, m, n) { + break + } + } + time.Sleep(25 * time.Millisecond) + atomic.AddInt32(&inFlight, -int32(len(opts.Targets))) + jobs := []RunJob{chromeJob(&yes), chromeJob(&yes)} + return Run{ID: "run-" + id, TestCaseID: id, Jobs: jobs}, nil + }, + } + p := newProject(Suite{Name: "s", TestCases: []string{"a", "b", "c"}, Targets: two}) + p.Sauce.Concurrency = 2 + r := newRunner(svc, &captureReporter{}, p) + + if code, _ := r.RunProject(context.Background()); code != 0 { + t.Errorf("exit = %d", code) + } + if maxInFlight > 2 { + t.Errorf("max jobs in flight = %d, want <= 2 (SC-011 counts jobs, not runs)", maxInFlight) + } +} + +func TestRunner_CaseNeedingMoreJobsThanTheBudgetStillRuns(t *testing.T) { + // Capping the weight matters: without it a four-target case under + // concurrency 2 would block for ever. + yes := true + four := make([]Target, 4) + for i := range four { + four[i] = Target{Capabilities: map[string]any{"browserName": "chrome"}} + } + svc := &fakeService{ + getTestCase: func(_ context.Context, id string) (TestCase, error) { return TestCase{ID: id}, nil }, + runTestCase: func(_ context.Context, id, _ string, _ RunOptions) (Run, error) { + return Run{ID: "run", TestCaseID: id, Jobs: []RunJob{chromeJob(&yes)}}, nil + }, + } + p := newProject(Suite{Name: "s", TestCases: []string{"a"}, Targets: four}) + p.Sauce.Concurrency = 2 + r := newRunner(svc, &captureReporter{}, p) + + done := make(chan int, 1) + go func() { code, _ := r.RunProject(context.Background()); done <- code }() + select { + case code := <-done: + if code != 0 { + t.Errorf("exit = %d", code) + } + case <-time.After(5 * time.Second): + t.Fatal("a case needing more jobs than the concurrency budget never ran") + } +} diff --git a/internal/authoring/schedule.go b/internal/authoring/schedule.go new file mode 100644 index 000000000..1a4427c44 --- /dev/null +++ b/internal/authoring/schedule.go @@ -0,0 +1,199 @@ +package authoring + +import ( + "encoding/json" + "strings" +) + +// TestSchedule is a recurring trigger that runs one or more suites on a cron +// pattern in a stated timezone. +type TestSchedule struct { + ID string `json:"id"` + OrgID string `json:"orgId,omitempty"` + TeamID string `json:"teamId,omitempty"` + CreatorUserID string `json:"creatorUserId,omitempty"` + CreatorUserName string `json:"creatorUserName,omitempty"` + CreationDate string `json:"creationDate,omitempty"` + LastModifierUserID string `json:"lastModifierUserId,omitempty"` + LastModifierUserName string `json:"lastModifierUserName,omitempty"` + LastUpdateDate string `json:"lastUpdateDate,omitempty"` + Name string `json:"name"` + Settings ScheduleSettings `json:"settings"` + State ScheduleState `json:"state"` + TestSuiteIDs []string `json:"testSuiteIds"` +} + +// ScheduleSettings is when and how a schedule runs. Cron, Timezone and +// RunningUserID are required by the service; the rest are optional bounds. +type ScheduleSettings struct { + // Cron is a six-field cron expression with seconds first, as observed on + // live schedules ("0 0 10 * * *" = 10:00:00 daily). The data model's + // earlier "five-field" note was wrong. + Cron string `json:"cron"` + // Timezone is an IANA zone name such as "Europe/Berlin". + Timezone string `json:"timezone"` + // RunningUserID is the user the scheduled runs execute as. + RunningUserID string `json:"runningUserId"` + StartDate string `json:"startDate,omitempty"` + EndDate string `json:"endDate,omitempty"` + // MaxRuns is a pointer so that "no limit" (nil) is distinct from a limit + // of zero. + MaxRuns *int `json:"maxRuns,omitempty"` + TunnelName string `json:"scTunnelName,omitempty"` + BuildName string `json:"buildName,omitempty"` +} + +// ScheduleState is the schedule's observed runtime state. +type ScheduleState struct { + StateName ScheduleStateName `json:"stateName"` + LastRunError string `json:"lastRunError,omitempty"` + LastRunDate string `json:"lastRunDate,omitempty"` + NextRunDate string `json:"nextRunDate,omitempty"` + RemainingRuns *int `json:"remainingRuns,omitempty"` +} + +// ScheduleStateName is a schedule's lifecycle state. +type ScheduleStateName string + +// The schedule states. Only Enabled and Disabled can be set by a user; +// Running and Errored are observed. Reaching MaxRuns or passing EndDate +// disables a schedule. +const ( + ScheduleEnabled ScheduleStateName = "ENABLED" + ScheduleDisabled ScheduleStateName = "DISABLED" + ScheduleErrored ScheduleStateName = "ERRORED" + ScheduleRunning ScheduleStateName = "RUNNING" +) + +// SettableScheduleStates are the states a user may request. +var SettableScheduleStates = []ScheduleStateName{ScheduleEnabled, ScheduleDisabled} + +// ParseScheduleState resolves a user-supplied state case-insensitively and +// reports whether it is one a user may set. +func ParseScheduleState(s string) (ScheduleStateName, bool) { + name := ScheduleStateName(strings.ToUpper(strings.TrimSpace(s))) + for _, st := range SettableScheduleStates { + if st == name { + return st, true + } + } + return name, false +} + +// ListSchedulesOptions filters a schedule listing. +type ListSchedulesOptions struct { + ListOptions + IDs []string + Search string + StartDate string + EndDate string + UserID string + TeamID string + TestSuiteIDs []string +} + +// CreateScheduleOptions is the request to create a schedule. +type CreateScheduleOptions struct { + Name string `json:"name"` + Settings ScheduleSettings `json:"settings"` + TestSuiteIDs []string `json:"testSuiteIds"` + StateName ScheduleStateName `json:"stateName"` +} + +// UpdateScheduleOptions is the request to update a schedule. The service +// replaces the schedule and rejects a partial body with INVALID_BODY naming +// the missing fields (observed 2026-09-06, research Open-4), so callers +// perform read-modify-write and always send Name, a fully populated Settings, +// the full TestSuiteIDs and StateName. AddTestSuiteIDs / RemoveTestSuiteIDs +// mirror the documented request shape but are not relied upon. +type UpdateScheduleOptions struct { + Name string `json:"name,omitempty"` + Settings *ScheduleSettingsPatch `json:"settings,omitempty"` + TestSuiteIDs []string `json:"testSuiteIds,omitempty"` + AddTestSuiteIDs []string `json:"addTestSuiteIds,omitempty"` + RemoveTestSuiteIDs []string `json:"removeTestSuiteIds,omitempty"` + StateName ScheduleStateName `json:"stateName,omitempty"` +} + +// ScheduleSettingsPatch is the settings object of an update. Every optional +// field is tri-state, because that is how the service behaves (observed +// 2026-09-06): an omitted field keeps its stored value, an explicit null +// clears it — for maxRuns, startDate and endDate too, although the +// specification marks them non-nullable — and an empty-string date is +// rejected. So nil omits, a pointer to "" (or ClearMaxRuns) sends null, and +// any other value is sent as is. Cron, Timezone and RunningUserID are +// required by the service and always sent. +type ScheduleSettingsPatch struct { + Cron string `json:"cron"` + Timezone string `json:"timezone"` + RunningUserID string `json:"runningUserId"` + StartDate *string `json:"-"` + EndDate *string `json:"-"` + MaxRuns *int `json:"-"` + // ClearMaxRuns sends maxRuns as null. A separate flag because zero is a + // real value the service stores, not "unlimited". + ClearMaxRuns bool `json:"-"` + TunnelName *string `json:"-"` + BuildName *string `json:"-"` +} + +// PatchFromSettings converts stored settings into a patch that re-sends every +// populated field, the starting point for read-modify-write. +func PatchFromSettings(s ScheduleSettings) ScheduleSettingsPatch { + p := ScheduleSettingsPatch{ + Cron: s.Cron, + Timezone: s.Timezone, + RunningUserID: s.RunningUserID, + MaxRuns: s.MaxRuns, + } + p.StartDate = optionalString(s.StartDate) + p.EndDate = optionalString(s.EndDate) + p.TunnelName = optionalString(s.TunnelName) + p.BuildName = optionalString(s.BuildName) + return p +} + +// optionalString returns nil for "" and a pointer to the value otherwise, so +// an unset stored field is omitted rather than nulled. +func optionalString(s string) *string { + if s == "" { + return nil + } + return &s +} + +// MarshalJSON implements the tri-state encoding of the optional fields. +func (p ScheduleSettingsPatch) MarshalJSON() ([]byte, error) { + m := map[string]any{ + "cron": p.Cron, + "timezone": p.Timezone, + "runningUserId": p.RunningUserID, + } + if p.StartDate != nil { + m["startDate"] = nullable(*p.StartDate) + } + if p.EndDate != nil { + m["endDate"] = nullable(*p.EndDate) + } + switch { + case p.ClearMaxRuns: + m["maxRuns"] = nil + case p.MaxRuns != nil: + m["maxRuns"] = *p.MaxRuns + } + if p.TunnelName != nil { + m["scTunnelName"] = nullable(*p.TunnelName) + } + if p.BuildName != nil { + m["buildName"] = nullable(*p.BuildName) + } + return json.Marshal(m) +} + +// nullable maps the empty string to JSON null and any other string to itself. +func nullable(s string) any { + if s == "" { + return nil + } + return s +} diff --git a/internal/authoring/schedule_test.go b/internal/authoring/schedule_test.go new file mode 100644 index 000000000..f4631c540 --- /dev/null +++ b/internal/authoring/schedule_test.go @@ -0,0 +1,76 @@ +package authoring + +import ( + "encoding/json" + "testing" +) + +func TestScheduleSettingsPatch_MarshalJSON(t *testing.T) { + empty := "" + tunnel := "my-tunnel" + three := 3 + + tests := []struct { + name string + patch ScheduleSettingsPatch + want string + }{ + { + name: "nil pointers omit every optional field; required ones always travel", + patch: ScheduleSettingsPatch{Cron: "0 * * * *", Timezone: "Europe/Berlin", RunningUserID: "u"}, + want: `{"cron":"0 * * * *","runningUserId":"u","timezone":"Europe/Berlin"}`, + }, + { + name: "pointer to empty string sends null to clear", + patch: ScheduleSettingsPatch{Cron: "c", Timezone: "tz", RunningUserID: "u", TunnelName: &empty, BuildName: &empty, StartDate: &empty, EndDate: &empty}, + want: `{"buildName":null,"cron":"c","endDate":null,"runningUserId":"u","scTunnelName":null,"startDate":null,"timezone":"tz"}`, + }, + { + name: "ClearMaxRuns sends null, a value sends the number, zero is a real value", + patch: ScheduleSettingsPatch{Cron: "c", Timezone: "tz", RunningUserID: "u", ClearMaxRuns: true}, + want: `{"cron":"c","maxRuns":null,"runningUserId":"u","timezone":"tz"}`, + }, + { + name: "values are sent as is", + patch: ScheduleSettingsPatch{Cron: "c", Timezone: "tz", RunningUserID: "u", TunnelName: &tunnel, MaxRuns: &three}, + want: `{"cron":"c","maxRuns":3,"runningUserId":"u","scTunnelName":"my-tunnel","timezone":"tz"}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, err := json.Marshal(tt.patch) + if err != nil { + t.Fatal(err) + } + if string(b) != tt.want { + t.Errorf("got %s\nwant %s", b, tt.want) + } + }) + } +} + +func TestPatchFromSettings(t *testing.T) { + five := 5 + p := PatchFromSettings(ScheduleSettings{Cron: "c", Timezone: "tz", RunningUserID: "u", MaxRuns: &five, TunnelName: "t"}) + if p.TunnelName == nil || *p.TunnelName != "t" { + t.Errorf("tunnel not carried over: %v", p.TunnelName) + } + if p.BuildName != nil || p.StartDate != nil || p.EndDate != nil { + t.Errorf("empty stored optional fields must become nil (omitted): %+v", p) + } + if p.MaxRuns == nil || *p.MaxRuns != 5 { + t.Error("maxRuns not carried over") + } +} + +func TestParseScheduleState(t *testing.T) { + if st, ok := ParseScheduleState(" enabled "); !ok || st != ScheduleEnabled { + t.Errorf("case-insensitive parse failed: %v %v", st, ok) + } + if _, ok := ParseScheduleState("running"); ok { + t.Error("RUNNING is observed, never settable") + } + if _, ok := ParseScheduleState("bogus"); ok { + t.Error("unknown state accepted") + } +} diff --git a/internal/authoring/target.go b/internal/authoring/target.go new file mode 100644 index 000000000..5762c69b4 --- /dev/null +++ b/internal/authoring/target.go @@ -0,0 +1,44 @@ +package authoring + +import ( + "strings" + + "github.com/saucelabs/saucectl/internal/region" +) + +// JobURL derives a job's dashboard link from its Sauce job identifier. +// +// The service returns a url field on only about half of its jobs (research +// R-006), so the link is always derived instead. This lives here, rather than +// in each caller, because the runner's results table and the `authoring +// testcases run` / `get-run` tables must never disagree about where a job +// lives. +func JobURL(reg region.Region, sauceJobID string) string { + if sauceJobID == "" { + return "" + } + return reg.AppBaseURL() + "/tests/" + sauceJobID +} + +// DescribeCapabilities pulls the browser, platform and device out of a +// target's free-form W3C capabilities. +// +// Capabilities are passed through untouched, so the only way to describe a +// target is to look for the keys that carry meaning, including the appium: +// prefixed forms used for devices. Callers format the three parts however +// their surface needs; keeping the extraction in one place means a new key +// is added once rather than in every table that renders a target. +func DescribeCapabilities(caps map[string]any) (browser, platform, device string) { + str := func(keys ...string) string { + for _, k := range keys { + if s, ok := caps[k].(string); ok && s != "" { + return s + } + } + return "" + } + browser = strings.TrimSpace(str("browserName") + " " + str("browserVersion")) + platform = strings.TrimSpace(str("platformName") + " " + str("appium:platformVersion", "platformVersion")) + device = str("appium:deviceName", "deviceName") + return browser, platform, device +} diff --git a/internal/authoring/testcase.go b/internal/authoring/testcase.go new file mode 100644 index 000000000..21b715f5a --- /dev/null +++ b/internal/authoring/testcase.go @@ -0,0 +1,355 @@ +package authoring + +import ( + "encoding/json" + "fmt" + "net/url" + "path" + "strings" +) + +// TestCase is a saved, reusable test produced from a plain-language +// description. Its identifier is a 24-character hex ObjectId. +type TestCase struct { + ID string `json:"id"` + OrgID string `json:"orgId,omitempty"` + // TeamID is returned by the detail endpoint but absent from the spec. + TeamID string `json:"teamId,omitempty"` + Name string `json:"name"` + // Tags are case-sensitive: an organisation may hold both "Login" and + // "login". Never fold or deduplicate them. + Tags []string `json:"tags"` + // TestSuiteID is set when the case belongs to a suite; a case belongs to + // at most one. + TestSuiteID string `json:"testSuiteId,omitempty"` + // Revisions is the full history and is returned even by the list + // endpoint, which is why a listing weighs ~13 KB per case. + Revisions []Revision `json:"revisions"` + RunSettings RunSettings `json:"runSettings"` + CreationDate string `json:"creationDate,omitempty"` + LastUpdateDate string `json:"lastUpdateDate,omitempty"` + CreatorUserID string `json:"creatorUserId,omitempty"` + CreatorUserName string `json:"creatorUserName,omitempty"` + LastModifierUserID string `json:"lastModifierUserId,omitempty"` + LastModifierUserName string `json:"lastModifierUserName,omitempty"` +} + +// LatestRevision returns the last revision, or false when the case has none. A +// case with no revisions is legitimate and must not be exported. +func (tc TestCase) LatestRevision() (Revision, bool) { + if len(tc.Revisions) == 0 { + return Revision{}, false + } + return tc.Revisions[len(tc.Revisions)-1], true +} + +// Revision is a point-in-time version of a test case: the original intent, +// the steps derived from it, and the reasoning behind them. +type Revision struct { + ID string `json:"id"` + Intent string `json:"intent"` + Steps []Step `json:"steps"` + DiscoveredIntent string `json:"discoveredIntent,omitempty"` + Description string `json:"description,omitempty"` + Reasoning []Reasoning `json:"reasoning,omitempty"` +} + +// Reasoning is one titled paragraph of the agent's reasoning. +type Reasoning struct { + Title string `json:"title"` + Description string `json:"description"` +} + +// Step is a single action within a revision. +type Step struct { + ID string `json:"id"` + Tool Tool `json:"tool"` + // Result is absent while the outcome is unknown. + Result *StepResult `json:"result,omitempty"` + // ScreenshotURL is a fully pre-signed Google Cloud Storage URL (~1 KB, + // expiring after 24 h), not an identifier. Use ArtifactID to obtain the + // identifier the storage endpoint accepts. + ScreenshotURL string `json:"screenshotUrl,omitempty"` +} + +// ArtifactID extracts the artifact identifier from ScreenshotURL: the last +// path segment before the query string. Passing it to GET /storage/{id} +// returns the file with Sauce credentials and without expiry. Returns "" when +// there is no screenshot or the URL cannot be parsed. +func (s Step) ArtifactID() string { + return ArtifactIDFromURL(s.ScreenshotURL) +} + +// ArtifactIDFromURL is the URL-to-identifier extraction behind Step.ArtifactID, +// exposed so commands can accept either form from users. +func ArtifactIDFromURL(raw string) string { + if raw == "" { + return "" + } + u, err := url.Parse(raw) + if err != nil { + return "" + } + id := path.Base(u.Path) + if id == "." || id == "/" { + return "" + } + return id +} + +// StepResult is the outcome of a step. +type StepResult struct { + Success bool `json:"success"` + Message string `json:"message,omitempty"` +} + +// ToolType names the kind of action a step performs. +type ToolType string + +// The documented tool types. The set is expected to grow; unknown types are +// rendered by their bare name rather than rejected. +const ( + ToolGoToURL ToolType = "go_to_url" + ToolSwitchWindow ToolType = "switch_window" + ToolEnterKeySubmit ToolType = "enter_key_submit" + ToolPause ToolType = "pause" + ToolClick ToolType = "click" + ToolInputText ToolType = "input_text" + ToolScrollDocument ToolType = "scroll_document" + ToolScrollElement ToolType = "scroll_element" + ToolAssert ToolType = "assert" + ToolSelect ToolType = "select" + ToolInShadowRoot ToolType = "in_shadow_root" + ToolFinish ToolType = "finish" +) + +// Tool is the action a step performs. Args is kept as raw JSON rather than +// decoded into twelve typed variants for three reasons: args.matcher is a +// string for switch_window but an object for assert, so one flat struct would +// be incorrect; in_shadow_root nests another Tool recursively; and saucectl +// only ever reads steps — it renders one line and otherwise passes them +// through. Typed variants would silently discard data the day a thirteenth +// type ships. Summary and Reasoning cover the display need and fail soft. +type Tool struct { + Type ToolType `json:"type"` + Args json.RawMessage `json:"args,omitempty"` +} + +// Selector locates an element by CSS or XPath. +type Selector struct { + Type string `json:"type"` + Value string `json:"value"` + // Index picks one match when the selector matches several; nil means the + // first. + Index *int `json:"index,omitempty"` +} + +// String renders the selector as type=value, e.g. css=#submit, with an [n] +// suffix when an index is set. +func (s Selector) String() string { + out := s.Type + "=" + s.Value + if s.Index != nil { + out = fmt.Sprintf("%s[%d]", out, *s.Index) + } + return out +} + +// toolArgs is the union of every documented argument across all tool types. +// Decoding into the union is safe because argument names do not collide in +// meaning; the per-type accessors below only read the fields their type +// defines. Matcher stays raw because its shape depends on the type. +type toolArgs struct { + Reasoning string `json:"reasoning"` + URL string `json:"url"` + Matcher json.RawMessage `json:"matcher"` + Time float64 `json:"time"` + Selector *Selector `json:"selector"` + XPath string `json:"xpath"` + Text string `json:"text"` + Direction string `json:"direction"` + Not bool `json:"not"` + OptionValue string `json:"optionValue"` + Tool *Tool `json:"tool"` +} + +// assertMatcher is the object form of args.matcher used by assert steps. +type assertMatcher struct { + Name string `json:"name"` + Args []any `json:"args"` +} + +// decodeArgs decodes Args into the union, reporting false when Args is empty +// or malformed so callers can degrade to the bare tool name. +func (t Tool) decodeArgs() (toolArgs, bool) { + var a toolArgs + if len(t.Args) == 0 { + return a, false + } + if err := json.Unmarshal(t.Args, &a); err != nil { + return a, false + } + return a, true +} + +// Reasoning returns the agent's stated reason for the step, or "" when the +// arguments are missing or malformed. +func (t Tool) Reasoning() string { + a, ok := t.decodeArgs() + if !ok { + return "" + } + return a.Reasoning +} + +// Summary renders the step as one readable line, e.g. "click css=#submit" or +// "input_text css=#user ← standard_user". Unknown tool types and malformed +// arguments degrade to the bare type name; they never fail. +func (t Tool) Summary() string { + name := string(t.Type) + a, ok := t.decodeArgs() + if !ok { + return name + } + + target := targetOf(a) + switch t.Type { + case ToolGoToURL: + return joinNonEmpty(name, a.URL) + case ToolSwitchWindow: + return joinNonEmpty(name, rawString(a.Matcher)) + case ToolPause: + if a.Time > 0 { + return fmt.Sprintf("%s %gms", name, a.Time) + } + return name + case ToolClick: + return joinNonEmpty(name, target) + case ToolInputText: + return joinNonEmpty(name, target, "←", a.Text) + case ToolScrollDocument: + return joinNonEmpty(name, a.Direction) + case ToolScrollElement: + return joinNonEmpty(name, target, a.Direction) + case ToolAssert: + not := "" + if a.Not { + not = "not" + } + return joinNonEmpty(name, not, target, matcherString(a.Matcher)) + case ToolSelect: + return joinNonEmpty(name, target, "=", a.OptionValue) + case ToolInShadowRoot: + if a.Tool != nil { + return joinNonEmpty(name, target, ">", a.Tool.Summary()) + } + return joinNonEmpty(name, target) + case ToolEnterKeySubmit, ToolFinish: + return name + default: + return name + } +} + +// targetOf renders the element a step acts on, preferring the structured +// selector over the deprecated bare xpath. +func targetOf(a toolArgs) string { + if a.Selector != nil && a.Selector.Value != "" { + return a.Selector.String() + } + if a.XPath != "" { + return "xpath=" + a.XPath + } + return "" +} + +// rawString returns a JSON string's value, or the compact JSON for anything +// else, or "" for nothing. +func rawString(raw json.RawMessage) string { + if len(raw) == 0 { + return "" + } + var s string + if err := json.Unmarshal(raw, &s); err == nil { + return s + } + return string(raw) +} + +// matcherString renders an assert matcher as name(args), e.g. +// toHaveText("2"), degrading to the raw JSON when it is not the documented +// object shape. +func matcherString(raw json.RawMessage) string { + if len(raw) == 0 { + return "" + } + var m assertMatcher + if err := json.Unmarshal(raw, &m); err != nil || m.Name == "" { + return rawString(raw) + } + if len(m.Args) == 0 { + return m.Name + } + parts := make([]string, len(m.Args)) + for i, arg := range m.Args { + b, err := json.Marshal(arg) + if err != nil { + parts[i] = fmt.Sprint(arg) + continue + } + parts[i] = string(b) + } + return fmt.Sprintf("%s(%s)", m.Name, strings.Join(parts, ", ")) +} + +// joinNonEmpty joins the non-empty parts with single spaces. +func joinNonEmpty(parts ...string) string { + kept := make([]string, 0, len(parts)) + for _, p := range parts { + if p != "" { + kept = append(kept, p) + } + } + return strings.Join(kept, " ") +} + +// Target is one browser or device to run against: free-form W3C WebDriver +// capabilities passed through untouched, plus whether it is a real device. +// Because capabilities are free-form, the configuration schema needs no +// platform enum and avoids the four-file duplication the other frameworks +// carry. IsRDC is omitted when false so a Target can double as the request +// shape, which declares capabilities only. +type Target struct { + Capabilities map[string]any `json:"capabilities"` + IsRDC bool `json:"isRdc,omitempty"` +} + +// RunSettings are the defaults stored on a test case. A stored case carries +// PrimaryTarget plus RunTargets (plural), whereas an authoring request carries +// a single target — two different shapes that must not be conflated. +type RunSettings struct { + TestURL string `json:"testUrl,omitempty"` + // TunnelName may be the empty string, which the service treats as a real + // tunnel lookup that fails (research Open-3). It is a pointer so that a + // stored "" survives decoding distinct from an absent field. + TunnelName *string `json:"scTunnelName,omitempty"` + PrimaryTarget Target `json:"primaryTarget"` + RunTargets []Target `json:"runTargets"` + LastBuildName string `json:"lastBuildName,omitempty"` +} + +// ListTestCasesOptions filters a test case listing. +type ListTestCasesOptions struct { + ListOptions + // Search is a case-insensitive substring match inside words: "demo" + // matches "Saucedemo - Checkout flow". Exact matching must be done + // client-side. + Search string + StartDate string + EndDate string + UserID string + TeamID string + // TestSuiteIDs filters by suite; the literal "null" finds unassigned cases. + TestSuiteIDs []string + // Tags matches cases carrying at least one of the tags. + Tags []string +} diff --git a/internal/authoring/testsuite.go b/internal/authoring/testsuite.go new file mode 100644 index 000000000..5e1950ac8 --- /dev/null +++ b/internal/authoring/testsuite.go @@ -0,0 +1,84 @@ +package authoring + +import "errors" + +// TestSuite is a named collection of test cases that can be run, scheduled +// and reported on together. Its identifier is a dashless 32-character hex +// UUID — a different shape from a test case's 24-character ObjectId. +type TestSuite struct { + ID string `json:"id"` + OrgID string `json:"orgId,omitempty"` + TeamID string `json:"teamId,omitempty"` + Name string `json:"name"` + Tags []string `json:"tags"` + CreationDate string `json:"creationDate,omitempty"` + LastUpdate string `json:"lastUpdate,omitempty"` + CreatorUserID string `json:"creatorUserId,omitempty"` + CreatorUserName string `json:"creatorUserName,omitempty"` + LastModifierUserID string `json:"lastModifierUserId,omitempty"` + LastModifierUserName string `json:"lastModifierUserName,omitempty"` + // TestCaseCount matches GET /testcases?testSuiteId= exactly (research + // R-003), which is what makes client-side suite expansion viable. + TestCaseCount int `json:"testCaseCount"` +} + +// ListTestSuitesOptions filters a suite listing. +type ListTestSuitesOptions struct { + ListOptions + IDs []string + Search string + StartDate string + EndDate string + UserID string + TeamID string +} + +// CreateTestSuiteOptions is the request to create a suite. +type CreateTestSuiteOptions struct { + Name string `json:"name"` + Tags []string `json:"tags,omitempty"` + TestCases []string `json:"testCases,omitempty"` +} + +// UpdateTestSuiteOptions is the request to update a suite. Fields left at +// their zero value are omitted and therefore unchanged. TestCases replaces the +// membership wholesale and is mutually exclusive with the incremental +// AddTestCases / RemoveTestCases — enforced by Validate before any request is +// sent rather than left to the service. +type UpdateTestSuiteOptions struct { + Name string `json:"name,omitempty"` + Tags []string `json:"tags,omitempty"` + TestCases []string `json:"testCases,omitempty"` + AddTestCases []string `json:"addTestCases,omitempty"` + RemoveTestCases []string `json:"removeTestCases,omitempty"` +} + +// ErrEmptyUpdate is returned when an update carries no change at all. +var ErrEmptyUpdate = errors.New("nothing to update: specify at least one change") + +// ErrTestCasesExclusive is returned when wholesale and incremental membership +// changes are combined in one update. +var ErrTestCasesExclusive = errors.New("testCases cannot be combined with addTestCases or removeTestCases") + +// Validate enforces the update's invariants client-side. +func (o UpdateTestSuiteOptions) Validate() error { + if o.Name == "" && o.Tags == nil && o.TestCases == nil && o.AddTestCases == nil && o.RemoveTestCases == nil { + return ErrEmptyUpdate + } + if o.TestCases != nil && (o.AddTestCases != nil || o.RemoveTestCases != nil) { + return ErrTestCasesExclusive + } + return nil +} + +// SuiteRun is the response to queuing a suite run. It carries a count and the +// build name only — no per-case run identifiers — which is why the runner +// expands suites client-side instead (research R-002). +type SuiteRun struct { + ID string `json:"id"` + OrgID string `json:"orgId,omitempty"` + TeamID string `json:"teamId,omitempty"` + UserID string `json:"userId,omitempty"` + RunCount int `json:"runCount"` + BuildName string `json:"buildName"` +} diff --git a/internal/authoring/tool_test.go b/internal/authoring/tool_test.go new file mode 100644 index 000000000..c741a17c3 --- /dev/null +++ b/internal/authoring/tool_test.go @@ -0,0 +1,227 @@ +package authoring + +import ( + "encoding/json" + "testing" +) + +func TestTool_Summary(t *testing.T) { + tests := []struct { + name string + tool Tool + want string + }{ + { + name: "go_to_url", + tool: Tool{Type: ToolGoToURL, Args: json.RawMessage(`{"reasoning":"open","url":"https://saucedemo.com"}`)}, + want: "go_to_url https://saucedemo.com", + }, + { + name: "switch_window with string matcher", + tool: Tool{Type: ToolSwitchWindow, Args: json.RawMessage(`{"reasoning":"r","matcher":"Checkout"}`)}, + want: "switch_window Checkout", + }, + { + name: "switch_window with unexpected object matcher degrades to json", + tool: Tool{Type: ToolSwitchWindow, Args: json.RawMessage(`{"matcher":{"title":"x"}}`)}, + want: `switch_window {"title":"x"}`, + }, + { + name: "enter_key_submit", + tool: Tool{Type: ToolEnterKeySubmit, Args: json.RawMessage(`{"reasoning":"submit"}`)}, + want: "enter_key_submit", + }, + { + name: "pause in milliseconds", + tool: Tool{Type: ToolPause, Args: json.RawMessage(`{"reasoning":"wait","time":1500}`)}, + want: "pause 1500ms", + }, + { + name: "pause without time", + tool: Tool{Type: ToolPause, Args: json.RawMessage(`{"reasoning":"wait"}`)}, + want: "pause", + }, + { + name: "click with css selector", + tool: Tool{Type: ToolClick, Args: json.RawMessage(`{"reasoning":"r","selector":{"type":"css","value":"#submit"}}`)}, + want: "click css=#submit", + }, + { + name: "click with indexed selector", + tool: Tool{Type: ToolClick, Args: json.RawMessage(`{"selector":{"type":"css","value":".item","index":2}}`)}, + want: "click css=.item[2]", + }, + { + name: "click falls back to deprecated xpath", + tool: Tool{Type: ToolClick, Args: json.RawMessage(`{"reasoning":"r","xpath":"//a[1]"}`)}, + want: "click xpath=//a[1]", + }, + { + name: "input_text", + tool: Tool{Type: ToolInputText, Args: json.RawMessage(`{"selector":{"type":"css","value":"#user"},"text":"standard_user"}`)}, + want: "input_text css=#user ← standard_user", + }, + { + name: "scroll_document", + tool: Tool{Type: ToolScrollDocument, Args: json.RawMessage(`{"direction":"down"}`)}, + want: "scroll_document down", + }, + { + name: "scroll_element", + tool: Tool{Type: ToolScrollElement, Args: json.RawMessage(`{"selector":{"type":"xpath","value":"//ul"},"direction":"up"}`)}, + want: "scroll_element xpath=//ul up", + }, + { + name: "assert with matcher args", + tool: Tool{Type: ToolAssert, Args: json.RawMessage(`{"selector":{"type":"css","value":".badge"},"matcher":{"name":"toHaveText","args":["2"]}}`)}, + want: `assert css=.badge toHaveText("2")`, + }, + { + name: "assert negated without matcher args", + tool: Tool{Type: ToolAssert, Args: json.RawMessage(`{"not":true,"selector":{"type":"css","value":".error"},"matcher":{"name":"toBeVisible"}}`)}, + want: "assert not css=.error toBeVisible", + }, + { + name: "assert with multiple matcher args of mixed types", + tool: Tool{Type: ToolAssert, Args: json.RawMessage(`{"selector":{"type":"css","value":"#n"},"matcher":{"name":"toHaveCount","args":[3,true]}}`)}, + want: "assert css=#n toHaveCount(3, true)", + }, + { + name: "select", + tool: Tool{Type: ToolSelect, Args: json.RawMessage(`{"selector":{"type":"css","value":"#country"},"optionValue":"eu"}`)}, + want: "select css=#country = eu", + }, + { + name: "in_shadow_root recurses into the nested tool", + tool: Tool{Type: ToolInShadowRoot, Args: json.RawMessage(`{"selector":{"type":"css","value":"my-host"},"tool":{"type":"click","args":{"selector":{"type":"css","value":"#inner"}}}}`)}, + want: "in_shadow_root css=my-host > click css=#inner", + }, + { + name: "in_shadow_root without nested tool", + tool: Tool{Type: ToolInShadowRoot, Args: json.RawMessage(`{"selector":{"type":"css","value":"my-host"}}`)}, + want: "in_shadow_root css=my-host", + }, + { + name: "finish", + tool: Tool{Type: ToolFinish, Args: json.RawMessage(`{"reasoning":"done"}`)}, + want: "finish", + }, + { + name: "unknown type degrades to its name", + tool: Tool{Type: "teleport", Args: json.RawMessage(`{"where":"home"}`)}, + want: "teleport", + }, + { + name: "malformed args degrade to the type name", + tool: Tool{Type: ToolClick, Args: json.RawMessage(`{not json`)}, + want: "click", + }, + { + name: "nil args degrade to the type name", + tool: Tool{Type: ToolInputText}, + want: "input_text", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.tool.Summary(); got != tt.want { + t.Errorf("Summary() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestTool_Reasoning(t *testing.T) { + tests := []struct { + name string + tool Tool + want string + }{ + {"present", Tool{Type: ToolFinish, Args: json.RawMessage(`{"reasoning":"all items added"}`)}, "all items added"}, + {"absent", Tool{Type: ToolFinish, Args: json.RawMessage(`{}`)}, ""}, + {"malformed", Tool{Type: ToolFinish, Args: json.RawMessage(`{`)}, ""}, + {"nil", Tool{Type: ToolFinish}, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.tool.Reasoning(); got != tt.want { + t.Errorf("Reasoning() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestTool_RoundTripKeepsUnknownArgs(t *testing.T) { + // Args are raw so a thirteenth tool type, or a new argument on an existing + // one, survives decode and re-encode unchanged. + in := `{"type":"click","args":{"reasoning":"r","selector":{"type":"css","value":"#a"},"newField":{"nested":true}}}` + var tool Tool + if err := json.Unmarshal([]byte(in), &tool); err != nil { + t.Fatal(err) + } + out, err := json.Marshal(tool) + if err != nil { + t.Fatal(err) + } + var a, b map[string]any + _ = json.Unmarshal([]byte(in), &a) + _ = json.Unmarshal(out, &b) + if a["args"].(map[string]any)["newField"] == nil || b["args"].(map[string]any)["newField"] == nil { + t.Errorf("unknown argument was lost in round trip: %s", out) + } +} + +func TestArtifactIDFromURL(t *testing.T) { + tests := []struct { + name string + url string + want string + }{ + { + name: "pre-signed storage url", + url: "https://storage.googleapis.com/bucket/org/steps/3f2a9c1e4b5d6e7f8a9b0c1d2e3f4a5b?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Expires=86400&X-Goog-Signature=abc", + want: "3f2a9c1e4b5d6e7f8a9b0c1d2e3f4a5b", + }, + {name: "bare identifier passes through", url: "3f2a9c1e4b5d6e7f8a9b0c1d2e3f4a5b", want: "3f2a9c1e4b5d6e7f8a9b0c1d2e3f4a5b"}, + {name: "empty", url: "", want: ""}, + {name: "unparseable", url: "://nope", want: ""}, + {name: "host only", url: "https://example.com/", want: ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ArtifactIDFromURL(tt.url); got != tt.want { + t.Errorf("ArtifactIDFromURL(%q) = %q, want %q", tt.url, got, tt.want) + } + }) + } +} + +func TestTestCase_LatestRevision(t *testing.T) { + var empty TestCase + if _, ok := empty.LatestRevision(); ok { + t.Error("expected no revision on an empty test case") + } + tc := TestCase{Revisions: []Revision{{ID: "old"}, {ID: "new"}}} + rev, ok := tc.LatestRevision() + if !ok || rev.ID != "new" { + t.Errorf("LatestRevision() = %v, %v; want the last revision", rev, ok) + } +} + +func TestRunSettings_EmptyTunnelNameSurvivesDecoding(t *testing.T) { + // The whole reason TunnelName is a pointer: a stored "" must remain + // distinguishable from an absent field (research R-008). + var withEmpty, without RunSettings + if err := json.Unmarshal([]byte(`{"scTunnelName":"","primaryTarget":{"capabilities":{}},"runTargets":[]}`), &withEmpty); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(`{"primaryTarget":{"capabilities":{}},"runTargets":[]}`), &without); err != nil { + t.Fatal(err) + } + if withEmpty.TunnelName == nil || *withEmpty.TunnelName != "" { + t.Errorf("expected an empty-string tunnel name, got %v", withEmpty.TunnelName) + } + if without.TunnelName != nil { + t.Errorf("expected nil tunnel name, got %q", *without.TunnelName) + } +} diff --git a/internal/authoring/variable.go b/internal/authoring/variable.go new file mode 100644 index 000000000..124b7f7ea --- /dev/null +++ b/internal/authoring/variable.go @@ -0,0 +1,138 @@ +package authoring + +import ( + "errors" + "fmt" + "strings" +) + +// VariableScope is where a variable is visible. +type VariableScope string + +// The variable scopes. Org-scoped variables are visible to every team; the +// others are restricted to the owning team, suite or case. +const ( + ScopeOrg VariableScope = "org" + ScopeTeam VariableScope = "team" + ScopeTestSuite VariableScope = "testSuite" + ScopeTestCase VariableScope = "testCase" +) + +// AllVariableScopes lists every scope, in the order the service documents. +var AllVariableScopes = []VariableScope{ScopeOrg, ScopeTeam, ScopeTestSuite, ScopeTestCase} + +// ParseVariableScope resolves a user-supplied scope case-insensitively. +func ParseVariableScope(s string) (VariableScope, bool) { + for _, sc := range AllVariableScopes { + if strings.EqualFold(string(sc), strings.TrimSpace(s)) { + return sc, true + } + } + return VariableScope(s), false +} + +// ErrScopeRequiresID is returned when a suite or case scope is used without +// its identifier, and ErrScopeForbidsID when an org or team scope is given +// one. The service enforces the same pairing on listing as on creation +// (400 INVALID_QUERY), so validating client-side yields a better message. +var ( + ErrScopeRequiresID = errors.New("scope requires its identifier") + ErrScopeForbidsID = errors.New("scope does not accept an identifier") +) + +// ValidateScopePairing checks that exactly the identifier the scope requires +// is present. An empty scope is accepted (no filter). +func ValidateScopePairing(scope VariableScope, testSuiteID, testCaseID string) error { + switch scope { + case ScopeTestSuite: + if testSuiteID == "" { + return fmt.Errorf("%w: scope=testSuite requires a test suite id", ErrScopeRequiresID) + } + if testCaseID != "" { + return fmt.Errorf("%w: scope=testSuite does not accept a test case id", ErrScopeForbidsID) + } + case ScopeTestCase: + if testCaseID == "" { + return fmt.Errorf("%w: scope=testCase requires a test case id", ErrScopeRequiresID) + } + if testSuiteID != "" { + return fmt.Errorf("%w: scope=testCase does not accept a test suite id", ErrScopeForbidsID) + } + case ScopeOrg, ScopeTeam, "": + if testSuiteID != "" || testCaseID != "" { + return fmt.Errorf("%w: scope=%s does not accept a test suite or test case id", ErrScopeForbidsID, scope) + } + default: + return fmt.Errorf("unknown scope %q, options: %s", scope, joinScopes()) + } + return nil +} + +// joinScopes renders AllVariableScopes for error messages. +func joinScopes() string { + names := make([]string, len(AllVariableScopes)) + for i, s := range AllVariableScopes { + names[i] = string(s) + } + return strings.Join(names, ", ") +} + +// Variable is a named value made available to tests, scoped to an +// organisation, team, suite or single case. +type Variable struct { + ID string `json:"id"` + OrgID string `json:"orgId,omitempty"` + TeamID string `json:"teamId,omitempty"` + TestSuiteID string `json:"testSuiteId,omitempty"` + TestCaseID string `json:"testCaseId,omitempty"` + Scope VariableScope `json:"scope"` + // Name matches ^[a-z0-9_]+$ and is 1–255 characters. + Name string `json:"name"` + Description string `json:"description,omitempty"` + IsSecret bool `json:"isSecret"` + // Value is never returned for a secret variable — verified against the + // live service. Commands must nonetheless blank it before rendering when + // IsSecret is set, so a change in service behaviour cannot leak it. + Value string `json:"value,omitempty"` + CreatorUserID string `json:"creatorUserId,omitempty"` + CreatorUserName string `json:"creatorUserName,omitempty"` + LastModifierUserID string `json:"lastModifierUserId,omitempty"` + LastModifierUserName string `json:"lastModifierUserName,omitempty"` + CreationDate string `json:"creationDate,omitempty"` + // LastUpdate is the optimistic-concurrency token: it must be echoed + // byte-for-byte as expectedLastUpdate on update and delete, which is why + // it is a string and never parsed into time.Time. + LastUpdate string `json:"lastUpdate,omitempty"` +} + +// ListVariablesOptions filters a variable listing. Scope pairing is validated +// client-side via ValidateScopePairing. +type ListVariablesOptions struct { + ListOptions + Scope VariableScope + TestSuiteID string + TestCaseID string + Search string +} + +// CreateVariableOptions is the request to create a variable. +type CreateVariableOptions struct { + Scope VariableScope `json:"scope"` + TestSuiteID string `json:"testSuiteId,omitempty"` + TestCaseID string `json:"testCaseId,omitempty"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + IsSecret bool `json:"isSecret"` + Value string `json:"value"` +} + +// UpdateVariableOptions is the request to change a variable. Pointer fields +// are tri-state: nil leaves the field unchanged. ExpectedLastUpdate is +// mandatory and goes in the body here — but in the query string on delete. +type UpdateVariableOptions struct { + Name *string `json:"name,omitempty"` + Description *string `json:"description,omitempty"` + Value *string `json:"value,omitempty"` + IsSecret *bool `json:"isSecret,omitempty"` + ExpectedLastUpdate string `json:"expectedLastUpdate"` +} diff --git a/internal/authoring/variable_test.go b/internal/authoring/variable_test.go new file mode 100644 index 000000000..62bf0a55f --- /dev/null +++ b/internal/authoring/variable_test.go @@ -0,0 +1,65 @@ +package authoring + +import ( + "errors" + "testing" +) + +func TestValidateScopePairing(t *testing.T) { + tests := []struct { + name string + scope VariableScope + suite string + tc string + wantErr error + }{ + {name: "org without ids", scope: ScopeOrg}, + {name: "team without ids", scope: ScopeTeam}, + {name: "no scope, no ids", scope: ""}, + {name: "testSuite with suite id", scope: ScopeTestSuite, suite: "s"}, + {name: "testCase with case id", scope: ScopeTestCase, tc: "c"}, + {name: "testSuite missing id", scope: ScopeTestSuite, wantErr: ErrScopeRequiresID}, + {name: "testCase missing id", scope: ScopeTestCase, wantErr: ErrScopeRequiresID}, + {name: "testSuite with case id", scope: ScopeTestSuite, suite: "s", tc: "c", wantErr: ErrScopeForbidsID}, + {name: "org with suite id", scope: ScopeOrg, suite: "s", wantErr: ErrScopeForbidsID}, + {name: "no scope with case id", scope: "", tc: "c", wantErr: ErrScopeForbidsID}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidateScopePairing(tt.scope, tt.suite, tt.tc) + if tt.wantErr == nil && err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tt.wantErr != nil && !errors.Is(err, tt.wantErr) { + t.Fatalf("got %v, want %v", err, tt.wantErr) + } + }) + } + if err := ValidateScopePairing("galaxy", "", ""); err == nil { + t.Error("unknown scope must be rejected") + } +} + +func TestParseVariableScope(t *testing.T) { + if s, ok := ParseVariableScope("TESTSUITE"); !ok || s != ScopeTestSuite { + t.Errorf("got %v %v", s, ok) + } + if _, ok := ParseVariableScope("nope"); ok { + t.Error("unknown scope accepted") + } +} + +func TestUpdateTestSuiteOptions_Validate(t *testing.T) { + if err := (UpdateTestSuiteOptions{}).Validate(); !errors.Is(err, ErrEmptyUpdate) { + t.Errorf("empty update: got %v", err) + } + if err := (UpdateTestSuiteOptions{TestCases: []string{"a"}, AddTestCases: []string{"b"}}).Validate(); !errors.Is(err, ErrTestCasesExclusive) { + t.Errorf("exclusive violation: got %v", err) + } + if err := (UpdateTestSuiteOptions{Name: "x"}).Validate(); err != nil { + t.Errorf("name only should be valid: %v", err) + } + if err := (UpdateTestSuiteOptions{AddTestCases: []string{"b"}, RemoveTestCases: []string{"c"}}).Validate(); err != nil { + t.Errorf("add+remove should be valid: %v", err) + } +} diff --git a/internal/http/authoring.go b/internal/http/authoring.go new file mode 100644 index 000000000..c740262be --- /dev/null +++ b/internal/http/authoring.go @@ -0,0 +1,654 @@ +package http + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/hashicorp/go-retryablehttp" + "github.com/saucelabs/saucectl/internal/authoring" + "github.com/saucelabs/saucectl/internal/iam" + "github.com/saucelabs/saucectl/internal/region" +) + +// authoringBasePath is the AI Authoring API's prefix on the region's API host. +// The entitlement check deliberately lives outside it (see IsAIAuthoringEnabled). +const authoringBasePath = "/ai-authoring/v1" + +// aiAuthoringEntitlement is the entitlement name that gates the feature. +const aiAuthoringEntitlement = "ai_authoring.enabled" + +// AuthoringService is the HTTP implementation of the authoring service +// interfaces (authoring.TestCaseService, TestSuiteService, ScheduleService, +// VariableService, ArtifactService and EntitlementReader). +// +// Authentication is HTTP Basic with username and access key. The published +// specification declares bearer JWT auth on every operation, but Basic auth is +// what the live service accepts and is the platform-wide convention (research +// R-001). It is applied in exactly one place, newRequest, so a future change +// is a one-line edit. +type AuthoringService struct { + // Client is used for reads. It retries transport errors and 5xx responses + // but, unlike the shared client, never retries 404: this API answers 404 + // as an ordinary outcome (TEST_CASE_NOT_FOUND, VARIABLE_NOT_FOUND, ...), + // and retrying it would turn every miss into four requests and several + // seconds (research R-010). + Client *retryablehttp.Client + // OnceClient is used for mutations and never retries. A retry after a + // POST that succeeded server-side but failed to reach us would start a + // second run or authoring task, consuming a second VM (research Open-5); + // a retried PATCH or DELETE that carries a concurrency token would report + // a spurious conflict for a change that actually went through. + OnceClient *retryablehttp.Client + // URL is the region's API base URL, e.g. https://api.us-west-1.saucelabs.com. + URL string + Username string + AccessKey string +} + +// NewAuthoringService creates a client for the given region and credentials. +// timeout bounds each individual request. +func NewAuthoringService(r region.Region, creds iam.Credentials, timeout time.Duration) AuthoringService { + return AuthoringService{ + Client: newAuthoringHTTPClient(timeout), + OnceClient: newAuthoringOnceClient(timeout), + URL: r.APIBaseURL(), + Username: creds.Username, + AccessKey: creds.AccessKey, + } +} + +// newAuthoringHTTPClient is the shared retryable client with its 404 retry +// removed. It keeps the default policy's handling of transport errors and 5xx. +func newAuthoringHTTPClient(timeout time.Duration) *retryablehttp.Client { + c := NewRetryableClient(timeout) + c.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) { + return retryablehttp.DefaultRetryPolicy(ctx, resp, err) + } + return c +} + +// newAuthoringOnceClient performs exactly one attempt per request. +func newAuthoringOnceClient(timeout time.Duration) *retryablehttp.Client { + c := newAuthoringHTTPClient(timeout) + c.RetryMax = 0 + return c +} + +// envelope is the {"data": ...} wrapper around every successful body. +type envelope struct { + Data json.RawMessage `json:"data"` +} + +// errorEnvelope is the {"error": {...}} wrapper around every failure body. +type errorEnvelope struct { + Error authoring.APIError `json:"error"` +} + +// newRequest builds an authenticated request against the authoring API. +// query may be nil. body, when non-nil, is JSON-encoded. +func (c *AuthoringService) newRequest(ctx context.Context, method, path string, query url.Values, body any) (*retryablehttp.Request, error) { + var payload []byte + if body != nil { + var err error + payload, err = json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("encoding request body: %w", err) + } + } + + u := c.URL + authoringBasePath + path + if len(query) > 0 { + u += "?" + query.Encode() + } + + var rdr io.Reader + if payload != nil { + rdr = bytes.NewReader(payload) + } + req, err := NewRetryableRequestWithContext(ctx, method, u, rdr) + if err != nil { + return nil, err + } + if payload != nil { + req.Header.Set("Content-Type", "application/json") + } + req.Header.Set("Accept", "application/json") + req.SetBasicAuth(c.Username, c.AccessKey) + return req, nil +} + +// do sends the request with the given client and returns the response, or a +// decoded *authoring.APIError for any 4xx/5xx status. The caller must close the +// body of a returned response. +func (c *AuthoringService) do(client *retryablehttp.Client, req *retryablehttp.Request) (*http.Response, error) { + resp, err := client.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode >= http.StatusBadRequest { + defer resp.Body.Close() + return nil, newAuthoringError(resp) + } + return resp, nil +} + +// doJSON sends the request and decodes the "data" member of the response into +// out. out may be nil for endpoints that return 204. +func (c *AuthoringService) doJSON(client *retryablehttp.Client, req *retryablehttp.Request, out any) error { + resp, err := c.do(client, req) + if err != nil { + return err + } + defer resp.Body.Close() + + if out == nil || resp.StatusCode == http.StatusNoContent { + return nil + } + + var env envelope + if err := json.NewDecoder(resp.Body).Decode(&env); err != nil { + return fmt.Errorf("decoding response: %w", err) + } + if len(env.Data) == 0 { + return fmt.Errorf("decoding response: missing data envelope") + } + if err := json.Unmarshal(env.Data, out); err != nil { + return fmt.Errorf("decoding response data: %w", err) + } + return nil +} + +// newAuthoringError decodes the error envelope of a failed response. It always +// records the HTTP status; when the body is not the documented envelope (a +// proxy error page, say) the raw text becomes the detail so nothing is lost. +func newAuthoringError(resp *http.Response) error { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) + + var env errorEnvelope + if err := json.Unmarshal(body, &env); err == nil && env.Error.Code != "" { + apiErr := env.Error + apiErr.HTTPStatus = resp.StatusCode + return &apiErr + } + + detail := strings.TrimSpace(string(body)) + if detail == "" { + detail = http.StatusText(resp.StatusCode) + } + return &authoring.APIError{HTTPStatus: resp.StatusCode, Detail: detail} +} + +// listQuery encodes shared pagination. skip is omitted at zero; limit is sent +// whenever the caller set it, even when zero, because limit=0 is the +// service's count-only mode (research: supporting observations). +func listQuery(opts authoring.ListOptions) url.Values { + q := url.Values{} + if opts.Skip > 0 { + q.Set("skip", strconv.Itoa(opts.Skip)) + } + if opts.Limit != nil { + q.Set("limit", strconv.Itoa(*opts.Limit)) + } + return q +} + +// setNonEmpty adds a single-valued parameter when it has a value. +func setNonEmpty(q url.Values, key, value string) { + if value != "" { + q.Set(key, value) + } +} + +// addAll adds a repeatable parameter once per value. The service accepts a +// repeated key as an array. +func addAll(q url.Values, key string, values []string) { + for _, v := range values { + if v != "" { + q.Add(key, v) + } + } +} + +// ---- Test cases ----------------------------------------------------------- + +// ListTestCases implements authoring.TestCaseService. +func (c *AuthoringService) ListTestCases(ctx context.Context, opts authoring.ListTestCasesOptions) (authoring.List[authoring.TestCase], error) { + q := listQuery(opts.ListOptions) + setNonEmpty(q, "search", opts.Search) + setNonEmpty(q, "startDate", opts.StartDate) + setNonEmpty(q, "endDate", opts.EndDate) + setNonEmpty(q, "userId", opts.UserID) + setNonEmpty(q, "teamId", opts.TeamID) + addAll(q, "testSuiteId", opts.TestSuiteIDs) + addAll(q, "tags", opts.Tags) + + var out authoring.List[authoring.TestCase] + req, err := c.newRequest(ctx, http.MethodGet, "/testcases", q, nil) + if err != nil { + return out, err + } + return out, c.doJSON(c.Client, req, &out) +} + +// GetTestCase implements authoring.TestCaseService. +func (c *AuthoringService) GetTestCase(ctx context.Context, id string) (authoring.TestCase, error) { + var out authoring.TestCase + req, err := c.newRequest(ctx, http.MethodGet, "/testcases/"+url.PathEscape(id), nil, nil) + if err != nil { + return out, err + } + return out, c.doJSON(c.Client, req, &out) +} + +// DeleteTestCase implements authoring.TestCaseService. +func (c *AuthoringService) DeleteTestCase(ctx context.Context, id string) error { + req, err := c.newRequest(ctx, http.MethodDelete, "/testcases/"+url.PathEscape(id), nil, nil) + if err != nil { + return err + } + return c.doJSON(c.OnceClient, req, nil) +} + +// RenameTestCase implements authoring.TestCaseService. +func (c *AuthoringService) RenameTestCase(ctx context.Context, id, name string) (authoring.TestCase, error) { + var out authoring.TestCase + body := struct { + Name string `json:"name"` + }{Name: name} + req, err := c.newRequest(ctx, http.MethodPost, "/testcases/"+url.PathEscape(id)+"/rename", nil, body) + if err != nil { + return out, err + } + return out, c.doJSON(c.OnceClient, req, &out) +} + +// RunTestCase implements authoring.TestCaseService. The revision-scoped path +// is described by the service but not declared in its specification; it is +// only used when revisionID is given. +func (c *AuthoringService) RunTestCase(ctx context.Context, id, revisionID string, opts authoring.RunOptions) (authoring.Run, error) { + var out authoring.Run + path := "/testcases/" + url.PathEscape(id) + "/run" + if revisionID != "" { + path += "/" + url.PathEscape(revisionID) + } + req, err := c.newRequest(ctx, http.MethodPost, path, nil, opts) + if err != nil { + return out, err + } + return out, c.doJSON(c.OnceClient, req, &out) +} + +// ListRuns implements authoring.TestCaseService. testCaseID is sent as the +// testCaseId query parameter, which is the filter the service actually +// honours: the path parameter alone returns every run in the organisation +// (research R-004, SC-002). +func (c *AuthoringService) ListRuns(ctx context.Context, testCaseID string, opts authoring.ListRunsOptions) (authoring.List[authoring.Run], error) { + q := listQuery(opts.ListOptions) + q.Set("testCaseId", testCaseID) + setNonEmpty(q, "startDate", opts.StartDate) + setNonEmpty(q, "endDate", opts.EndDate) + setNonEmpty(q, "userId", opts.UserID) + setNonEmpty(q, "teamId", opts.TeamID) + + var out authoring.List[authoring.Run] + req, err := c.newRequest(ctx, http.MethodGet, "/testcases/"+url.PathEscape(testCaseID)+"/runs", q, nil) + if err != nil { + return out, err + } + return out, c.doJSON(c.Client, req, &out) +} + +// GetRun implements authoring.TestCaseService. +func (c *AuthoringService) GetRun(ctx context.Context, testCaseID, runID string) (authoring.Run, error) { + var out authoring.Run + req, err := c.newRequest(ctx, http.MethodGet, "/testcases/"+url.PathEscape(testCaseID)+"/runs/"+url.PathEscape(runID), nil, nil) + if err != nil { + return out, err + } + return out, c.doJSON(c.Client, req, &out) +} + +// ListTags implements authoring.TestCaseService. +func (c *AuthoringService) ListTags(ctx context.Context) ([]string, error) { + var out []string + req, err := c.newRequest(ctx, http.MethodGet, "/testcases/tags", nil, nil) + if err != nil { + return nil, err + } + return out, c.doJSON(c.Client, req, &out) +} + +// Generate implements authoring.TestCaseService. +func (c *AuthoringService) Generate(ctx context.Context, opts authoring.GenerateOptions) (authoring.GenerateTask, error) { + var out authoring.GenerateTask + req, err := c.newRequest(ctx, http.MethodPost, "/testcases/generate", nil, opts) + if err != nil { + return out, err + } + return out, c.doJSON(c.OnceClient, req, &out) +} + +// GenerationStatus implements authoring.TestCaseService. +func (c *AuthoringService) GenerationStatus(ctx context.Context, taskID string) (authoring.GenerationState, error) { + var out authoring.GenerationState + req, err := c.newRequest(ctx, http.MethodGet, "/testcases/generate/"+url.PathEscape(taskID), nil, nil) + if err != nil { + return out, err + } + return out, c.doJSON(c.Client, req, &out) +} + +// Code implements authoring.TestCaseService. +func (c *AuthoringService) Code(ctx context.Context, id, target string) (string, error) { + var out struct { + Code string `json:"code"` + } + q := url.Values{} + q.Set("target", target) + req, err := c.newRequest(ctx, http.MethodGet, "/testcases/"+url.PathEscape(id)+"/code", q, nil) + if err != nil { + return "", err + } + return out.Code, c.doJSON(c.Client, req, &out) +} + +// CodeTargets implements authoring.TestCaseService. +func (c *AuthoringService) CodeTargets(ctx context.Context, id string) ([]string, error) { + var out struct { + Targets []string `json:"targets"` + } + req, err := c.newRequest(ctx, http.MethodGet, "/testcases/"+url.PathEscape(id)+"/code/targets", nil, nil) + if err != nil { + return nil, err + } + return out.Targets, c.doJSON(c.Client, req, &out) +} + +// ---- Test suites ---------------------------------------------------------- + +// ListTestSuites implements authoring.TestSuiteService. +func (c *AuthoringService) ListTestSuites(ctx context.Context, opts authoring.ListTestSuitesOptions) (authoring.List[authoring.TestSuite], error) { + q := listQuery(opts.ListOptions) + addAll(q, "ids", opts.IDs) + setNonEmpty(q, "search", opts.Search) + setNonEmpty(q, "startDate", opts.StartDate) + setNonEmpty(q, "endDate", opts.EndDate) + setNonEmpty(q, "userId", opts.UserID) + setNonEmpty(q, "teamId", opts.TeamID) + + var out authoring.List[authoring.TestSuite] + req, err := c.newRequest(ctx, http.MethodGet, "/testsuites", q, nil) + if err != nil { + return out, err + } + return out, c.doJSON(c.Client, req, &out) +} + +// GetTestSuite implements authoring.TestSuiteService. +func (c *AuthoringService) GetTestSuite(ctx context.Context, id string) (authoring.TestSuite, error) { + var out authoring.TestSuite + req, err := c.newRequest(ctx, http.MethodGet, "/testsuites/"+url.PathEscape(id), nil, nil) + if err != nil { + return out, err + } + return out, c.doJSON(c.Client, req, &out) +} + +// CreateTestSuite implements authoring.TestSuiteService. +func (c *AuthoringService) CreateTestSuite(ctx context.Context, opts authoring.CreateTestSuiteOptions) (authoring.TestSuite, error) { + var out authoring.TestSuite + req, err := c.newRequest(ctx, http.MethodPost, "/testsuites", nil, opts) + if err != nil { + return out, err + } + return out, c.doJSON(c.OnceClient, req, &out) +} + +// UpdateTestSuite implements authoring.TestSuiteService. +func (c *AuthoringService) UpdateTestSuite(ctx context.Context, id string, opts authoring.UpdateTestSuiteOptions) (authoring.TestSuite, error) { + var out authoring.TestSuite + req, err := c.newRequest(ctx, http.MethodPost, "/testsuites/"+url.PathEscape(id), nil, opts) + if err != nil { + return out, err + } + return out, c.doJSON(c.OnceClient, req, &out) +} + +// DeleteTestSuite implements authoring.TestSuiteService. +func (c *AuthoringService) DeleteTestSuite(ctx context.Context, id string, deleteTestCases bool) error { + body := struct { + DeleteTestCases bool `json:"deleteTestCases"` + }{DeleteTestCases: deleteTestCases} + req, err := c.newRequest(ctx, http.MethodDelete, "/testsuites/"+url.PathEscape(id), nil, body) + if err != nil { + return err + } + return c.doJSON(c.OnceClient, req, nil) +} + +// RunTestSuite implements authoring.TestSuiteService. +func (c *AuthoringService) RunTestSuite(ctx context.Context, id, buildName string) (authoring.SuiteRun, error) { + var out authoring.SuiteRun + // Omitted when unset, matching RunOptions on the test-case run endpoint. + // Sending an explicit null here was never asked for by the API and never + // verified, unlike scTunnelName, where a null is load-bearing. + body := struct { + BuildName string `json:"buildName,omitempty"` + }{BuildName: buildName} + req, err := c.newRequest(ctx, http.MethodPost, "/testsuites/"+url.PathEscape(id)+"/run", nil, body) + if err != nil { + return out, err + } + return out, c.doJSON(c.OnceClient, req, &out) +} + +// ---- Schedules ------------------------------------------------------------ + +// ListSchedules implements authoring.ScheduleService. +func (c *AuthoringService) ListSchedules(ctx context.Context, opts authoring.ListSchedulesOptions) (authoring.List[authoring.TestSchedule], error) { + q := listQuery(opts.ListOptions) + addAll(q, "ids", opts.IDs) + setNonEmpty(q, "search", opts.Search) + setNonEmpty(q, "startDate", opts.StartDate) + setNonEmpty(q, "endDate", opts.EndDate) + setNonEmpty(q, "userId", opts.UserID) + setNonEmpty(q, "teamId", opts.TeamID) + addAll(q, "testSuiteIds", opts.TestSuiteIDs) + + var out authoring.List[authoring.TestSchedule] + req, err := c.newRequest(ctx, http.MethodGet, "/test-schedules", q, nil) + if err != nil { + return out, err + } + return out, c.doJSON(c.Client, req, &out) +} + +// GetSchedule implements authoring.ScheduleService. +func (c *AuthoringService) GetSchedule(ctx context.Context, id string) (authoring.TestSchedule, error) { + var out authoring.TestSchedule + req, err := c.newRequest(ctx, http.MethodGet, "/test-schedules/"+url.PathEscape(id), nil, nil) + if err != nil { + return out, err + } + return out, c.doJSON(c.Client, req, &out) +} + +// CreateSchedule implements authoring.ScheduleService. +func (c *AuthoringService) CreateSchedule(ctx context.Context, opts authoring.CreateScheduleOptions) (authoring.TestSchedule, error) { + var out authoring.TestSchedule + req, err := c.newRequest(ctx, http.MethodPost, "/test-schedules", nil, opts) + if err != nil { + return out, err + } + return out, c.doJSON(c.OnceClient, req, &out) +} + +// UpdateSchedule implements authoring.ScheduleService. +func (c *AuthoringService) UpdateSchedule(ctx context.Context, id string, opts authoring.UpdateScheduleOptions) (authoring.TestSchedule, error) { + var out authoring.TestSchedule + req, err := c.newRequest(ctx, http.MethodPost, "/test-schedules/"+url.PathEscape(id), nil, opts) + if err != nil { + return out, err + } + return out, c.doJSON(c.OnceClient, req, &out) +} + +// DeleteSchedule implements authoring.ScheduleService. +func (c *AuthoringService) DeleteSchedule(ctx context.Context, id string) error { + req, err := c.newRequest(ctx, http.MethodDelete, "/test-schedules/"+url.PathEscape(id), nil, nil) + if err != nil { + return err + } + return c.doJSON(c.OnceClient, req, nil) +} + +// ---- Variables ------------------------------------------------------------ + +// ListVariables implements authoring.VariableService. +func (c *AuthoringService) ListVariables(ctx context.Context, opts authoring.ListVariablesOptions) (authoring.List[authoring.Variable], error) { + q := listQuery(opts.ListOptions) + setNonEmpty(q, "scope", string(opts.Scope)) + setNonEmpty(q, "testSuiteId", opts.TestSuiteID) + setNonEmpty(q, "testCaseId", opts.TestCaseID) + setNonEmpty(q, "search", opts.Search) + + var out authoring.List[authoring.Variable] + req, err := c.newRequest(ctx, http.MethodGet, "/variables", q, nil) + if err != nil { + return out, err + } + return out, c.doJSON(c.Client, req, &out) +} + +// GetVariable implements authoring.VariableService. +func (c *AuthoringService) GetVariable(ctx context.Context, id string) (authoring.Variable, error) { + var out authoring.Variable + req, err := c.newRequest(ctx, http.MethodGet, "/variables/"+url.PathEscape(id), nil, nil) + if err != nil { + return out, err + } + return out, c.doJSON(c.Client, req, &out) +} + +// CreateVariable implements authoring.VariableService. +func (c *AuthoringService) CreateVariable(ctx context.Context, opts authoring.CreateVariableOptions) (authoring.Variable, error) { + var out authoring.Variable + req, err := c.newRequest(ctx, http.MethodPost, "/variables", nil, opts) + if err != nil { + return out, err + } + return out, c.doJSON(c.OnceClient, req, &out) +} + +// UpdateVariable implements authoring.VariableService. The concurrency token +// travels in the body here — but in the query string on delete. +func (c *AuthoringService) UpdateVariable(ctx context.Context, id string, opts authoring.UpdateVariableOptions) (authoring.Variable, error) { + var out authoring.Variable + req, err := c.newRequest(ctx, http.MethodPatch, "/variables/"+url.PathEscape(id), nil, opts) + if err != nil { + return out, err + } + return out, c.doJSON(c.OnceClient, req, &out) +} + +// DeleteVariable implements authoring.VariableService. The concurrency token +// travels in the query string here — but in the body on update. +func (c *AuthoringService) DeleteVariable(ctx context.Context, id, expectedLastUpdate string) error { + q := url.Values{} + q.Set("expectedLastUpdate", expectedLastUpdate) + req, err := c.newRequest(ctx, http.MethodDelete, "/variables/"+url.PathEscape(id), q, nil) + if err != nil { + return err + } + return c.doJSON(c.OnceClient, req, nil) +} + +// ---- Artifacts ------------------------------------------------------------ + +// DownloadArtifact implements authoring.ArtifactService. The response carries +// no Content-Type, so nothing about the file's kind can be inferred here. +func (c *AuthoringService) DownloadArtifact(ctx context.Context, id string) (io.ReadCloser, error) { + req, err := c.newRequest(ctx, http.MethodGet, "/storage/"+url.PathEscape(id), nil, nil) + if err != nil { + return nil, err + } + req.Header.Del("Accept") + resp, err := c.do(c.Client, req) + if err != nil { + return nil, err + } + return resp.Body, nil +} + +// ---- Entitlement ---------------------------------------------------------- + +// IsAIAuthoringEnabled implements authoring.EntitlementReader against the +// platform entitlements API on the same host — a different API from the +// authoring service, absent from its specification (research R-009): +// +// GET {URL}/v2/entitlements/entities/org/{orgID}?entitlements=ai_authoring.enabled +// 200 {"level":"org","uuid":"…","entitlements":[{"name":"ai_authoring.enabled","value":true,"region":"GLOBAL"}]} +// +// value was observed as a JSON boolean but is undocumented, so it is decoded +// permissively: true, "true" and 1 all count as enabled. A missing +// entitlement entry counts as disabled. Any non-200 response is an error, so +// the caller can distinguish "not in your plan" from "could not verify". +func (c *AuthoringService) IsAIAuthoringEnabled(ctx context.Context, orgID string) (bool, error) { + u := fmt.Sprintf("%s/v2/entitlements/entities/org/%s?entitlements=%s", c.URL, url.PathEscape(orgID), aiAuthoringEntitlement) + req, err := NewRetryableRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return false, err + } + req.Header.Set("Accept", "application/json") + req.SetBasicAuth(c.Username, c.AccessKey) + + resp, err := c.Client.Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10)) + return false, fmt.Errorf("entitlement check failed with HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + + var out struct { + Entitlements []struct { + Name string `json:"name"` + Value any `json:"value"` + } `json:"entitlements"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return false, fmt.Errorf("decoding entitlement response: %w", err) + } + + for _, e := range out.Entitlements { + if e.Name == aiAuthoringEntitlement { + return entitlementEnabled(e.Value), nil + } + } + return false, nil +} + +// entitlementEnabled interprets the undocumented entitlement value leniently. +func entitlementEnabled(v any) bool { + switch t := v.(type) { + case bool: + return t + case string: + return strings.EqualFold(t, "true") || t == "1" + case float64: + return t == 1 + default: + return false + } +} diff --git a/internal/http/authoring_test.go b/internal/http/authoring_test.go new file mode 100644 index 000000000..7fc00b8d4 --- /dev/null +++ b/internal/http/authoring_test.go @@ -0,0 +1,498 @@ +package http + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/saucelabs/saucectl/internal/authoring" + "github.com/saucelabs/saucectl/internal/iam" + "github.com/saucelabs/saucectl/internal/region" +) + +// newTestAuthoringService points a client at the test server with retries +// made fast, so tests that count attempts finish in milliseconds. +func newTestAuthoringService(srv *httptest.Server) AuthoringService { + c := NewAuthoringService(region.USWest1, iam.Credentials{Username: "user", AccessKey: "key"}, 5*time.Second) + c.URL = srv.URL + c.Client.RetryWaitMin = 1 * time.Millisecond + c.Client.RetryWaitMax = 1 * time.Millisecond + c.OnceClient.RetryWaitMin = 1 * time.Millisecond + c.OnceClient.RetryWaitMax = 1 * time.Millisecond + return c +} + +func writeJSON(w http.ResponseWriter, status int, body string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = io.WriteString(w, body) +} + +func TestAuthoringService_BasicAuthAndEnvelope(t *testing.T) { + var gotPath, gotAuth, gotAccept string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotAuth = r.Header.Get("Authorization") + gotAccept = r.Header.Get("Accept") + writeJSON(w, 200, `{"data":{"id":"6a882c1dc8b4482c166e96c9","name":"test","tags":[],"revisions":[],"runSettings":{"scTunnelName":"","primaryTarget":{"capabilities":{"browserName":"chrome"},"isRdc":false},"runTargets":[]}}}`) + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + tc, err := c.GetTestCase(context.Background(), "6a882c1dc8b4482c166e96c9") + if err != nil { + t.Fatal(err) + } + + if gotPath != "/ai-authoring/v1/testcases/6a882c1dc8b4482c166e96c9" { + t.Errorf("path = %q", gotPath) + } + want := "Basic " + base64.StdEncoding.EncodeToString([]byte("user:key")) + if gotAuth != want { + t.Errorf("Authorization = %q, want basic auth (research R-001)", gotAuth) + } + if gotAccept != "application/json" { + t.Errorf("Accept = %q", gotAccept) + } + if tc.Name != "test" { + t.Errorf("envelope not unwrapped: %+v", tc) + } + if tc.RunSettings.TunnelName == nil || *tc.RunSettings.TunnelName != "" { + t.Error("empty stored tunnel name lost in decoding") + } +} + +func TestAuthoringService_ErrorEnvelopeWithUndocumentedData(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, 400, `{"error":{"code":"INVALID_QUERY","detail":"Invalid query string parameters.","data":[{"code":"custom","path":[],"message":"scope-specific id is required: scope=testSuite requires testSuiteId."}]}}`) + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + _, err := c.ListVariables(context.Background(), authoring.ListVariablesOptions{Scope: authoring.ScopeTestSuite}) + if !errors.Is(err, authoring.ErrInvalidQuery) { + t.Fatalf("expected ErrInvalidQuery, got %v", err) + } + var apiErr *authoring.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("expected *APIError, got %T", err) + } + if apiErr.HTTPStatus != 400 { + t.Errorf("HTTPStatus = %d", apiErr.HTTPStatus) + } + if !strings.Contains(err.Error(), "scope-specific id is required") { + t.Errorf("the actionable message from error.data[] is missing: %s", err) + } +} + +func TestAuthoringService_NonEnvelopeErrorBody(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(502) + _, _ = io.WriteString(w, "Bad Gateway") + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + _, err := c.GetTestSuite(context.Background(), "x") + var apiErr *authoring.APIError + if !errors.As(err, &apiErr) { + t.Fatalf("expected *APIError, got %T: %v", err, err) + } + if apiErr.HTTPStatus != 502 || !strings.Contains(apiErr.Detail, "Bad Gateway") { + t.Errorf("unexpected error: %+v", apiErr) + } +} + +func TestAuthoringService_NotFoundIsAnsweredInOneRequest(t *testing.T) { + // The shared client retries 404 for APIs where it means propagation + // delay. This API returns 404 as an ordinary answer, so exactly one + // request must be made (research R-010, SC-007). + var calls int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + atomic.AddInt32(&calls, 1) + writeJSON(w, 404, `{"error":{"code":"TEST_CASE_NOT_FOUND","detail":"Test case not found."}}`) + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + _, err := c.GetTestCase(context.Background(), "000000000000000000000000") + if !errors.Is(err, authoring.ErrTestCaseNotFound) { + t.Fatalf("expected ErrTestCaseNotFound, got %v", err) + } + if n := atomic.LoadInt32(&calls); n != 1 { + t.Errorf("handler invoked %d times, want exactly 1", n) + } +} + +func TestAuthoringService_ServerErrorRetriesReadsButNeverMutations(t *testing.T) { + // Reads may be retried on 5xx. Mutations must not be: a retried run + // start could consume a second VM (research Open-5). + var calls int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + atomic.AddInt32(&calls, 1) + writeJSON(w, 503, `{"error":{"code":"UNAVAILABLE","detail":"try later"}}`) + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + + _, _ = c.ListTags(context.Background()) + if n := atomic.LoadInt32(&calls); n <= 1 { + t.Errorf("read on 5xx made %d request(s), expected retries", n) + } + + atomic.StoreInt32(&calls, 0) + _, err := c.RunTestCase(context.Background(), "id", "", authoring.RunOptions{}) + if err == nil { + t.Fatal("expected an error") + } + if n := atomic.LoadInt32(&calls); n != 1 { + t.Errorf("mutation on 5xx made %d requests, want exactly 1", n) + } +} + +func TestAuthoringService_NoContent(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + t.Errorf("method = %s", r.Method) + } + w.WriteHeader(204) + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + if err := c.DeleteTestCase(context.Background(), "abc"); err != nil { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestAuthoringService_ListRunsSendsTestCaseIDQuery(t *testing.T) { + // Without the query parameter the service returns every run in the + // organisation (research R-004, SC-002). + var gotQuery string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotQuery = r.URL.RawQuery + writeJSON(w, 200, `{"data":{"items":[],"total":0}}`) + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + limit := 5 + _, err := c.ListRuns(context.Background(), "6a882c1dc8b4482c166e96c9", authoring.ListRunsOptions{ListOptions: authoring.ListOptions{Skip: 10, Limit: &limit}}) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"testCaseId=6a882c1dc8b4482c166e96c9", "skip=10", "limit=5"} { + if !strings.Contains(gotQuery, want) { + t.Errorf("query %q lacks %q", gotQuery, want) + } + } +} + +func TestAuthoringService_LimitIsSentWhenSetEvenIfZero(t *testing.T) { + var queries []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + queries = append(queries, r.URL.RawQuery) + writeJSON(w, 200, `{"data":{"items":[],"total":187}}`) + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + zero := 0 + _, _ = c.ListTestCases(context.Background(), authoring.ListTestCasesOptions{ListOptions: authoring.ListOptions{Limit: &zero}}) + _, _ = c.ListTestCases(context.Background(), authoring.ListTestCasesOptions{Tags: []string{"Login", "login"}, TestSuiteIDs: []string{"a", "b"}}) + + if queries[0] != "limit=0" { + t.Errorf("count-only request sent %q, want limit=0", queries[0]) + } + if strings.Contains(queries[1], "limit") { + t.Errorf("unset limit must be omitted, got %q", queries[1]) + } + for _, want := range []string{"tags=Login", "tags=login", "testSuiteId=a", "testSuiteId=b"} { + if !strings.Contains(queries[1], want) { + t.Errorf("query %q lacks repeated parameter %q", queries[1], want) + } + } +} + +func TestAuthoringService_RunTestCaseBodyAndRevisionPath(t *testing.T) { + var gotPath, gotBody string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + b, _ := io.ReadAll(r.Body) + gotBody = string(b) + if r.Header.Get("Content-Type") != "application/json" { + t.Errorf("Content-Type = %q", r.Header.Get("Content-Type")) + } + writeJSON(w, 200, `{"data":{"id":"run","testCaseId":"tc","build":"b - 1","jobs":[{"id":"j","name":"n","sauceJobId":"s","target":{"capabilities":{},"isRdc":false}}]}}`) + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + run, err := c.RunTestCase(context.Background(), "tc", "rev1", authoring.RunOptions{BuildName: "b"}) + if err != nil { + t.Fatal(err) + } + if gotPath != "/ai-authoring/v1/testcases/tc/run/rev1" { + t.Errorf("path = %q", gotPath) + } + if gotBody != `{"buildName":"b","scTunnelName":null}` { + t.Errorf("body = %s; an absent tunnel must be sent as explicit null (research Open-3)", gotBody) + } + if run.Done() { + t.Error("start response has no success and must not read as done") + } + if run.Build != "b - 1" { + t.Errorf("build = %q", run.Build) + } +} + +func TestAuthoringService_VariableConcurrencyToken(t *testing.T) { + // The token goes in the body on update and in the query string on delete. + var updateBody, deleteQuery string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodPatch: + b, _ := io.ReadAll(r.Body) + updateBody = string(b) + writeJSON(w, 200, `{"data":{"id":"v","scope":"org","name":"n","isSecret":true,"lastUpdate":"2026-09-05T15:00:00.123Z"}}`) + case http.MethodDelete: + deleteQuery = r.URL.RawQuery + writeJSON(w, 412, `{"error":{"code":"VARIABLE_VERSION_CONFLICT","detail":"stale"}}`) + } + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + secret := true + v, err := c.UpdateVariable(context.Background(), "v", authoring.UpdateVariableOptions{IsSecret: &secret, ExpectedLastUpdate: "2026-09-05T14:59:59.000Z"}) + if err != nil { + t.Fatal(err) + } + if updateBody != `{"isSecret":true,"expectedLastUpdate":"2026-09-05T14:59:59.000Z"}` { + t.Errorf("update body = %s", updateBody) + } + if v.Value != "" { + t.Errorf("secret value leaked: %q", v.Value) + } + + err = c.DeleteVariable(context.Background(), "v", "2026-09-05T14:59:59.000Z") + if !errors.Is(err, authoring.ErrVariableVersionConflict) { + t.Fatalf("expected ErrVariableVersionConflict, got %v", err) + } + if deleteQuery != "expectedLastUpdate=2026-09-05T14%3A59%3A59.000Z" { + t.Errorf("delete query = %q", deleteQuery) + } +} + +func TestAuthoringService_DeleteTestSuiteCascadeFlag(t *testing.T) { + var gotBody string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + gotBody = string(b) + w.WriteHeader(204) + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + if err := c.DeleteTestSuite(context.Background(), "s", true); err != nil { + t.Fatal(err) + } + if gotBody != `{"deleteTestCases":true}` { + t.Errorf("body = %s", gotBody) + } +} + +func TestAuthoringService_CodeAndTargets(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/code/targets"): + writeJSON(w, 200, `{"data":{"targets":["typescript_playwright","python_selenium"]}}`) + case strings.HasSuffix(r.URL.Path, "/code"): + if r.URL.Query().Get("target") != "typescript_playwright" { + writeJSON(w, 404, `{"error":{"code":"CODE_GENERATION_TARGET_NOT_FOUND"}}`) + return + } + writeJSON(w, 200, `{"data":{"code":"import { test } from '@playwright/test';"}}`) + } + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + targets, err := c.CodeTargets(context.Background(), "tc") + if err != nil || len(targets) != 2 { + t.Fatalf("targets = %v, %v", targets, err) + } + code, err := c.Code(context.Background(), "tc", "typescript_playwright") + if err != nil || !strings.HasPrefix(code, "import") { + t.Fatalf("code = %q, %v", code, err) + } + _, err = c.Code(context.Background(), "tc", "cobol") + if !errors.Is(err, authoring.ErrCodeGenerationTargetNotFound) { + t.Errorf("expected ErrCodeGenerationTargetNotFound, got %v", err) + } +} + +func TestAuthoringService_GenerateAndStatus(t *testing.T) { + var gotBody string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost { + b, _ := io.ReadAll(r.Body) + gotBody = string(b) + writeJSON(w, 202, `{"data":{"taskId":"task","sauceJobId":"job"}}`) + return + } + writeJSON(w, 200, `{"data":{"status":"IN_PROGRESS","steps":[{"action":{"type":"go_to_url","args":{"reasoning":"r","url":"https://x"}},"result":{"success":true}}],"reasoning":[]}}`) + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + task, err := c.Generate(context.Background(), authoring.GenerateOptions{ + Name: "n", + RunSettings: authoring.GenerateRunSettings{Target: authoring.Target{Capabilities: map[string]any{"browserName": "chrome"}}}, + PromptSettings: authoring.PromptSettings{Intent: "do it"}, + TimeoutMillis: 120000, + }) + if err != nil || task.TaskID != "task" { + t.Fatalf("task = %+v, %v", task, err) + } + if !strings.Contains(gotBody, `"timeout":120000`) || strings.Contains(gotBody, `"isRdc"`) { + t.Errorf("body = %s", gotBody) + } + + state, err := c.GenerationStatus(context.Background(), "task") + if err != nil { + t.Fatal(err) + } + if state.Done() || len(state.Steps) != 1 || state.Steps[0].Action.Summary() != "go_to_url https://x" { + t.Errorf("state = %+v", state) + } +} + +func TestAuthoringService_DownloadArtifact(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/storage/missing") { + writeJSON(w, 404, `{"error":{"code":"FILE_NOT_FOUND"}}`) + return + } + // No Content-Type on purpose: the live service sends none. + w.WriteHeader(200) + _, _ = w.Write([]byte{0x89, 'P', 'N', 'G'}) + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + rc, err := c.DownloadArtifact(context.Background(), "abc") + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(rc) + _ = rc.Close() + if string(b) != "\x89PNG" { + t.Errorf("body = %q", b) + } + _, err = c.DownloadArtifact(context.Background(), "missing") + if !errors.Is(err, authoring.ErrFileNotFound) { + t.Errorf("expected ErrFileNotFound, got %v", err) + } +} + +func TestAuthoringService_IsAIAuthoringEnabled(t *testing.T) { + tests := []struct { + name string + status int + body string + want bool + wantErr bool + }{ + {name: "boolean true", status: 200, body: `{"level":"org","entitlements":[{"name":"ai_authoring.enabled","value":true,"region":"GLOBAL"}]}`, want: true}, + {name: "string true", status: 200, body: `{"entitlements":[{"name":"ai_authoring.enabled","value":"true"}]}`, want: true}, + {name: "numeric one", status: 200, body: `{"entitlements":[{"name":"ai_authoring.enabled","value":1}]}`, want: true}, + {name: "false", status: 200, body: `{"entitlements":[{"name":"ai_authoring.enabled","value":false}]}`, want: false}, + {name: "missing entitlement", status: 200, body: `{"entitlements":[{"name":"other","value":true}]}`, want: false}, + {name: "unauthorized is an error, not a no", status: 401, body: `{"detail":"nope"}`, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var gotPath, gotQuery string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath, gotQuery = r.URL.Path, r.URL.RawQuery + writeJSON(w, tt.status, tt.body) + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + c.Client.RetryMax = 0 + got, err := c.IsAIAuthoringEnabled(context.Background(), "org-1") + if (err != nil) != tt.wantErr { + t.Fatalf("err = %v, wantErr %v", err, tt.wantErr) + } + if got != tt.want { + t.Errorf("enabled = %v, want %v", got, tt.want) + } + if gotPath != "/v2/entitlements/entities/org/org-1" || gotQuery != "entitlements=ai_authoring.enabled" { + t.Errorf("request = %s?%s; the entitlement API is not under the authoring base path", gotPath, gotQuery) + } + }) + } +} + +func TestAuthoringService_UpdateScheduleSendsNullToClear(t *testing.T) { + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&gotBody) + writeJSON(w, 200, `{"data":{"id":"s","name":"n","settings":{"cron":"0 * * * *","timezone":"UTC","runningUserId":"u"},"state":{"stateName":"ENABLED"},"testSuiteIds":["a"]}}`) + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + empty := "" + _, err := c.UpdateSchedule(context.Background(), "s", authoring.UpdateScheduleOptions{ + Settings: &authoring.ScheduleSettingsPatch{Cron: "0 * * * *", Timezone: "UTC", RunningUserID: "u", TunnelName: &empty}, + }) + if err != nil { + t.Fatal(err) + } + settings := gotBody["settings"].(map[string]any) + if v, present := settings["scTunnelName"]; !present || v != nil { + t.Errorf("scTunnelName = %v (present=%v), want explicit null", v, present) + } + if _, present := settings["buildName"]; present { + t.Error("buildName must be omitted when not being changed") + } +} + +func TestAuthoringService_RunTestSuiteOmitsUnsetBuildName(t *testing.T) { + // Probed 2026-09-07: the endpoint accepts an omitted buildName, so the + // explicit null it used to send was unnecessary and inconsistent with + // RunOptions on the test-case run endpoint. + var bodies []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + bodies = append(bodies, string(b)) + writeJSON(w, 200, `{"data":{"id":"s","runCount":2,"buildName":"x - 1"}}`) + })) + defer srv.Close() + + c := newTestAuthoringService(srv) + if _, err := c.RunTestSuite(context.Background(), "suite", ""); err != nil { + t.Fatal(err) + } + if _, err := c.RunTestSuite(context.Background(), "suite", "nightly"); err != nil { + t.Fatal(err) + } + if bodies[0] != `{}` { + t.Errorf("unset build name sent %s, want {}", bodies[0]) + } + if bodies[1] != `{"buildName":"nightly"}` { + t.Errorf("set build name sent %s", bodies[1]) + } +} diff --git a/internal/mocks/authoring.go b/internal/mocks/authoring.go new file mode 100644 index 000000000..ecccdf2b8 --- /dev/null +++ b/internal/mocks/authoring.go @@ -0,0 +1,200 @@ +package mocks + +import ( + "context" + "io" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// AuthoringService is a hand-written fake for every authoring service +// interface. Each method delegates to its Fn field; an unset field panics, +// which makes an unexpected call in a test loud rather than silently +// successful. +type AuthoringService struct { + ListTestCasesFn func(ctx context.Context, opts authoring.ListTestCasesOptions) (authoring.List[authoring.TestCase], error) + GetTestCaseFn func(ctx context.Context, id string) (authoring.TestCase, error) + DeleteTestCaseFn func(ctx context.Context, id string) error + RenameTestCaseFn func(ctx context.Context, id, name string) (authoring.TestCase, error) + RunTestCaseFn func(ctx context.Context, id, revisionID string, opts authoring.RunOptions) (authoring.Run, error) + ListRunsFn func(ctx context.Context, testCaseID string, opts authoring.ListRunsOptions) (authoring.List[authoring.Run], error) + GetRunFn func(ctx context.Context, testCaseID, runID string) (authoring.Run, error) + ListTagsFn func(ctx context.Context) ([]string, error) + GenerateFn func(ctx context.Context, opts authoring.GenerateOptions) (authoring.GenerateTask, error) + GenerationStatusFn func(ctx context.Context, taskID string) (authoring.GenerationState, error) + CodeFn func(ctx context.Context, id, target string) (string, error) + CodeTargetsFn func(ctx context.Context, id string) ([]string, error) + + ListTestSuitesFn func(ctx context.Context, opts authoring.ListTestSuitesOptions) (authoring.List[authoring.TestSuite], error) + GetTestSuiteFn func(ctx context.Context, id string) (authoring.TestSuite, error) + CreateTestSuiteFn func(ctx context.Context, opts authoring.CreateTestSuiteOptions) (authoring.TestSuite, error) + UpdateTestSuiteFn func(ctx context.Context, id string, opts authoring.UpdateTestSuiteOptions) (authoring.TestSuite, error) + DeleteTestSuiteFn func(ctx context.Context, id string, deleteTestCases bool) error + RunTestSuiteFn func(ctx context.Context, id, buildName string) (authoring.SuiteRun, error) + + ListSchedulesFn func(ctx context.Context, opts authoring.ListSchedulesOptions) (authoring.List[authoring.TestSchedule], error) + GetScheduleFn func(ctx context.Context, id string) (authoring.TestSchedule, error) + CreateScheduleFn func(ctx context.Context, opts authoring.CreateScheduleOptions) (authoring.TestSchedule, error) + UpdateScheduleFn func(ctx context.Context, id string, opts authoring.UpdateScheduleOptions) (authoring.TestSchedule, error) + DeleteScheduleFn func(ctx context.Context, id string) error + + ListVariablesFn func(ctx context.Context, opts authoring.ListVariablesOptions) (authoring.List[authoring.Variable], error) + GetVariableFn func(ctx context.Context, id string) (authoring.Variable, error) + CreateVariableFn func(ctx context.Context, opts authoring.CreateVariableOptions) (authoring.Variable, error) + UpdateVariableFn func(ctx context.Context, id string, opts authoring.UpdateVariableOptions) (authoring.Variable, error) + DeleteVariableFn func(ctx context.Context, id, expectedLastUpdate string) error + + DownloadArtifactFn func(ctx context.Context, id string) (io.ReadCloser, error) + + IsAIAuthoringEnabledFn func(ctx context.Context, orgID string) (bool, error) +} + +// ListTestCases delegates to ListTestCasesFn. +func (s *AuthoringService) ListTestCases(ctx context.Context, opts authoring.ListTestCasesOptions) (authoring.List[authoring.TestCase], error) { + return s.ListTestCasesFn(ctx, opts) +} + +// GetTestCase delegates to GetTestCaseFn. +func (s *AuthoringService) GetTestCase(ctx context.Context, id string) (authoring.TestCase, error) { + return s.GetTestCaseFn(ctx, id) +} + +// DeleteTestCase delegates to DeleteTestCaseFn. +func (s *AuthoringService) DeleteTestCase(ctx context.Context, id string) error { + return s.DeleteTestCaseFn(ctx, id) +} + +// RenameTestCase delegates to RenameTestCaseFn. +func (s *AuthoringService) RenameTestCase(ctx context.Context, id, name string) (authoring.TestCase, error) { + return s.RenameTestCaseFn(ctx, id, name) +} + +// RunTestCase delegates to RunTestCaseFn. +func (s *AuthoringService) RunTestCase(ctx context.Context, id, revisionID string, opts authoring.RunOptions) (authoring.Run, error) { + return s.RunTestCaseFn(ctx, id, revisionID, opts) +} + +// ListRuns delegates to ListRunsFn. +func (s *AuthoringService) ListRuns(ctx context.Context, testCaseID string, opts authoring.ListRunsOptions) (authoring.List[authoring.Run], error) { + return s.ListRunsFn(ctx, testCaseID, opts) +} + +// GetRun delegates to GetRunFn. +func (s *AuthoringService) GetRun(ctx context.Context, testCaseID, runID string) (authoring.Run, error) { + return s.GetRunFn(ctx, testCaseID, runID) +} + +// ListTags delegates to ListTagsFn. +func (s *AuthoringService) ListTags(ctx context.Context) ([]string, error) { + return s.ListTagsFn(ctx) +} + +// Generate delegates to GenerateFn. +func (s *AuthoringService) Generate(ctx context.Context, opts authoring.GenerateOptions) (authoring.GenerateTask, error) { + return s.GenerateFn(ctx, opts) +} + +// GenerationStatus delegates to GenerationStatusFn. +func (s *AuthoringService) GenerationStatus(ctx context.Context, taskID string) (authoring.GenerationState, error) { + return s.GenerationStatusFn(ctx, taskID) +} + +// Code delegates to CodeFn. +func (s *AuthoringService) Code(ctx context.Context, id, target string) (string, error) { + return s.CodeFn(ctx, id, target) +} + +// CodeTargets delegates to CodeTargetsFn. +func (s *AuthoringService) CodeTargets(ctx context.Context, id string) ([]string, error) { + return s.CodeTargetsFn(ctx, id) +} + +// ListTestSuites delegates to ListTestSuitesFn. +func (s *AuthoringService) ListTestSuites(ctx context.Context, opts authoring.ListTestSuitesOptions) (authoring.List[authoring.TestSuite], error) { + return s.ListTestSuitesFn(ctx, opts) +} + +// GetTestSuite delegates to GetTestSuiteFn. +func (s *AuthoringService) GetTestSuite(ctx context.Context, id string) (authoring.TestSuite, error) { + return s.GetTestSuiteFn(ctx, id) +} + +// CreateTestSuite delegates to CreateTestSuiteFn. +func (s *AuthoringService) CreateTestSuite(ctx context.Context, opts authoring.CreateTestSuiteOptions) (authoring.TestSuite, error) { + return s.CreateTestSuiteFn(ctx, opts) +} + +// UpdateTestSuite delegates to UpdateTestSuiteFn. +func (s *AuthoringService) UpdateTestSuite(ctx context.Context, id string, opts authoring.UpdateTestSuiteOptions) (authoring.TestSuite, error) { + return s.UpdateTestSuiteFn(ctx, id, opts) +} + +// DeleteTestSuite delegates to DeleteTestSuiteFn. +func (s *AuthoringService) DeleteTestSuite(ctx context.Context, id string, deleteTestCases bool) error { + return s.DeleteTestSuiteFn(ctx, id, deleteTestCases) +} + +// RunTestSuite delegates to RunTestSuiteFn. +func (s *AuthoringService) RunTestSuite(ctx context.Context, id, buildName string) (authoring.SuiteRun, error) { + return s.RunTestSuiteFn(ctx, id, buildName) +} + +// ListSchedules delegates to ListSchedulesFn. +func (s *AuthoringService) ListSchedules(ctx context.Context, opts authoring.ListSchedulesOptions) (authoring.List[authoring.TestSchedule], error) { + return s.ListSchedulesFn(ctx, opts) +} + +// GetSchedule delegates to GetScheduleFn. +func (s *AuthoringService) GetSchedule(ctx context.Context, id string) (authoring.TestSchedule, error) { + return s.GetScheduleFn(ctx, id) +} + +// CreateSchedule delegates to CreateScheduleFn. +func (s *AuthoringService) CreateSchedule(ctx context.Context, opts authoring.CreateScheduleOptions) (authoring.TestSchedule, error) { + return s.CreateScheduleFn(ctx, opts) +} + +// UpdateSchedule delegates to UpdateScheduleFn. +func (s *AuthoringService) UpdateSchedule(ctx context.Context, id string, opts authoring.UpdateScheduleOptions) (authoring.TestSchedule, error) { + return s.UpdateScheduleFn(ctx, id, opts) +} + +// DeleteSchedule delegates to DeleteScheduleFn. +func (s *AuthoringService) DeleteSchedule(ctx context.Context, id string) error { + return s.DeleteScheduleFn(ctx, id) +} + +// ListVariables delegates to ListVariablesFn. +func (s *AuthoringService) ListVariables(ctx context.Context, opts authoring.ListVariablesOptions) (authoring.List[authoring.Variable], error) { + return s.ListVariablesFn(ctx, opts) +} + +// GetVariable delegates to GetVariableFn. +func (s *AuthoringService) GetVariable(ctx context.Context, id string) (authoring.Variable, error) { + return s.GetVariableFn(ctx, id) +} + +// CreateVariable delegates to CreateVariableFn. +func (s *AuthoringService) CreateVariable(ctx context.Context, opts authoring.CreateVariableOptions) (authoring.Variable, error) { + return s.CreateVariableFn(ctx, opts) +} + +// UpdateVariable delegates to UpdateVariableFn. +func (s *AuthoringService) UpdateVariable(ctx context.Context, id string, opts authoring.UpdateVariableOptions) (authoring.Variable, error) { + return s.UpdateVariableFn(ctx, id, opts) +} + +// DeleteVariable delegates to DeleteVariableFn. +func (s *AuthoringService) DeleteVariable(ctx context.Context, id, expectedLastUpdate string) error { + return s.DeleteVariableFn(ctx, id, expectedLastUpdate) +} + +// DownloadArtifact delegates to DownloadArtifactFn. +func (s *AuthoringService) DownloadArtifact(ctx context.Context, id string) (io.ReadCloser, error) { + return s.DownloadArtifactFn(ctx, id) +} + +// IsAIAuthoringEnabled delegates to IsAIAuthoringEnabledFn. +func (s *AuthoringService) IsAIAuthoringEnabled(ctx context.Context, orgID string) (bool, error) { + return s.IsAIAuthoringEnabledFn(ctx, orgID) +} From c42c7ad74b026d619bf4d474d459304c7a22713a Mon Sep 17 00:00:00 2001 From: Vinit Tomar Date: Thu, 17 Sep 2026 18:58:45 +0530 Subject: [PATCH 5/9] Add saucectl authoring commands and run kind Co-Authored-By: Claude Opus 5 (1M context) --- .sauce/authoring.yml | 32 ++ api/global.schema.json | 15 +- api/saucectl.schema.json | 380 ++++++++++++++- api/v1alpha/framework/authoring.schema.json | 127 +++++ cmd/saucectl/saucectl.go | 2 + internal/cmd/authoring/artifact.go | 97 ++++ internal/cmd/authoring/cmd.go | 149 ++++++ internal/cmd/authoring/confirm.go | 62 +++ internal/cmd/authoring/fixes_test.go | 294 ++++++++++++ internal/cmd/authoring/helpers_test.go | 434 +++++++++++++++++ internal/cmd/authoring/output.go | 104 +++++ internal/cmd/authoring/paging.go | 104 +++++ internal/cmd/authoring/schedules.go | 26 ++ internal/cmd/authoring/schedules_create.go | 163 +++++++ internal/cmd/authoring/schedules_list.go | 148 ++++++ internal/cmd/authoring/schedules_state.go | 117 +++++ internal/cmd/authoring/schedules_update.go | 256 ++++++++++ internal/cmd/authoring/targets.go | 171 +++++++ internal/cmd/authoring/testcases.go | 33 ++ internal/cmd/authoring/testcases_code.go | 261 +++++++++++ .../cmd/authoring/testcases_codetargets.go | 51 ++ internal/cmd/authoring/testcases_delete.go | 76 +++ internal/cmd/authoring/testcases_generate.go | 442 ++++++++++++++++++ .../authoring/testcases_generate_status.go | 81 ++++ .../cmd/authoring/testcases_generate_test.go | 184 ++++++++ internal/cmd/authoring/testcases_get.go | 177 +++++++ internal/cmd/authoring/testcases_list.go | 100 ++++ internal/cmd/authoring/testcases_rename.go | 41 ++ internal/cmd/authoring/testcases_run.go | 133 ++++++ internal/cmd/authoring/testcases_runs.go | 129 +++++ internal/cmd/authoring/testcases_tags.go | 52 +++ internal/cmd/authoring/testsuites.go | 25 + internal/cmd/authoring/testsuites_create.go | 54 +++ internal/cmd/authoring/testsuites_delete.go | 124 +++++ internal/cmd/authoring/testsuites_list.go | 127 +++++ internal/cmd/authoring/testsuites_update.go | 60 +++ internal/cmd/authoring/valuesource.go | 109 +++++ internal/cmd/authoring/variables.go | 24 + internal/cmd/authoring/variables_create.go | 108 +++++ internal/cmd/authoring/variables_delete.go | 74 +++ internal/cmd/authoring/variables_list.go | 186 ++++++++ internal/cmd/authoring/variables_update.go | 143 ++++++ internal/cmd/run/authoring.go | 110 +++++ internal/cmd/run/run.go | 8 +- 44 files changed, 5590 insertions(+), 3 deletions(-) create mode 100644 .sauce/authoring.yml create mode 100644 api/v1alpha/framework/authoring.schema.json create mode 100644 internal/cmd/authoring/artifact.go create mode 100644 internal/cmd/authoring/cmd.go create mode 100644 internal/cmd/authoring/confirm.go create mode 100644 internal/cmd/authoring/fixes_test.go create mode 100644 internal/cmd/authoring/helpers_test.go create mode 100644 internal/cmd/authoring/output.go create mode 100644 internal/cmd/authoring/paging.go create mode 100644 internal/cmd/authoring/schedules.go create mode 100644 internal/cmd/authoring/schedules_create.go create mode 100644 internal/cmd/authoring/schedules_list.go create mode 100644 internal/cmd/authoring/schedules_state.go create mode 100644 internal/cmd/authoring/schedules_update.go create mode 100644 internal/cmd/authoring/targets.go create mode 100644 internal/cmd/authoring/testcases.go create mode 100644 internal/cmd/authoring/testcases_code.go create mode 100644 internal/cmd/authoring/testcases_codetargets.go create mode 100644 internal/cmd/authoring/testcases_delete.go create mode 100644 internal/cmd/authoring/testcases_generate.go create mode 100644 internal/cmd/authoring/testcases_generate_status.go create mode 100644 internal/cmd/authoring/testcases_generate_test.go create mode 100644 internal/cmd/authoring/testcases_get.go create mode 100644 internal/cmd/authoring/testcases_list.go create mode 100644 internal/cmd/authoring/testcases_rename.go create mode 100644 internal/cmd/authoring/testcases_run.go create mode 100644 internal/cmd/authoring/testcases_runs.go create mode 100644 internal/cmd/authoring/testcases_tags.go create mode 100644 internal/cmd/authoring/testsuites.go create mode 100644 internal/cmd/authoring/testsuites_create.go create mode 100644 internal/cmd/authoring/testsuites_delete.go create mode 100644 internal/cmd/authoring/testsuites_list.go create mode 100644 internal/cmd/authoring/testsuites_update.go create mode 100644 internal/cmd/authoring/valuesource.go create mode 100644 internal/cmd/authoring/variables.go create mode 100644 internal/cmd/authoring/variables_create.go create mode 100644 internal/cmd/authoring/variables_delete.go create mode 100644 internal/cmd/authoring/variables_list.go create mode 100644 internal/cmd/authoring/variables_update.go create mode 100644 internal/cmd/run/authoring.go diff --git a/.sauce/authoring.yml b/.sauce/authoring.yml new file mode 100644 index 000000000..864e06c6c --- /dev/null +++ b/.sauce/authoring.yml @@ -0,0 +1,32 @@ +apiVersion: v1alpha +kind: authoring +sauce: + region: us-west-1 + concurrency: 2 + metadata: + build: "saucectl authoring $BUILD_ID" +defaults: + timeout: 15m +suites: + # AI-authored test cases live in the service, not in this repository, so + # there is no id that works for every reader: substitute one of your own, + # from `saucectl authoring testcases list`, or reference a suite by name + # with `testSuiteName:` instead. Omit `targets` to use the case's stored ones. + - name: "saucedemo checkout on Chrome" + testCases: [] + targets: + - capabilities: + browserName: chrome + browserVersion: latest + platformName: "Windows 11" +artifacts: + download: + when: fail + match: + - "*.mp4" + - log.json + directory: ./artifacts/ +reporters: + junit: + enabled: true + filename: saucectl-authoring-report.xml diff --git a/api/global.schema.json b/api/global.schema.json index f1bb09cc5..123bbf06d 100644 --- a/api/global.schema.json +++ b/api/global.schema.json @@ -14,7 +14,8 @@ "testcafe", "xcuitest", "xctest", - "playwright-cucumberjs" + "playwright-cucumberjs", + "authoring" ] } }, @@ -129,6 +130,18 @@ "then": { "$ref": "v1alpha/framework/playwright-cucumberjs.schema.json" } + }, + { + "if": { + "properties": { + "kind": { + "const": "authoring" + } + } + }, + "then": { + "$ref": "v1alpha/framework/authoring.schema.json" + } } ] } diff --git a/api/saucectl.schema.json b/api/saucectl.schema.json index 0f815de43..53b8aaaae 100644 --- a/api/saucectl.schema.json +++ b/api/saucectl.schema.json @@ -14,7 +14,8 @@ "testcafe", "xcuitest", "xctest", - "playwright-cucumberjs" + "playwright-cucumberjs", + "authoring" ] } }, @@ -4767,6 +4768,383 @@ ], "additionalProperties": true } + }, + { + "if": { + "properties": { + "kind": { + "const": "authoring" + } + } + }, + "then": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "saucectl AI Test Authoring runner configuration", + "description": "Configuration file for running Sauce Labs AI-authored tests using saucectl", + "type": "object", + "allOf": [ + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "saucectl artifacts schema", + "description": "Subschema for controlling artifacts", + "type": "object", + "properties": { + "artifacts": { + "description": "Manage test output, such as logs, videos, and screenshots.", + "type": "object", + "properties": { + "cleanup": { + "description": "Whether to remove all contents of artifacts directory", + "type": "boolean" + }, + "download": { + "description": "Settings related to downloading test artifacts from Sauce Labs.", + "type": "object", + "properties": { + "match": { + "description": "Specifies which artifacts to download based on whether they match the file pattern provided. Supports the wildcard character '*'.", + "type": "array" + }, + "when": { + "description": "Specifies when and under what circumstances to download artifacts.", + "enum": [ + "always", + "fail", + "never", + "pass" + ] + }, + "directory": { + "description": "Specifies the path to the folder in which to download artifacts. A separate subdirectory is generated in this location for each suite.", + "type": "string" + }, + "allAttempts": { + "description": "If true and a test is retried, artifacts for every attempt will be downloaded. Otherwise, only artifacts for the final attempt will be downloaded.", + "type": "boolean" + } + }, + "required": [ + "when", + "match", + "directory" + ], + "additionalProperties": false + }, + "retain": { + "description": "Compress folders into zip files, which can then be downloaded as artifacts.", + "type": "object", + "additionalProperties": { + "type": "string" + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": true + }, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "saucectl sauce specific schema", + "description": "Subschema for sauce specific settings", + "type": "object", + "properties": { + "sauce": { + "description": "All settings related to how tests are run and identified in the Sauce Labs platform.", + "type": "object", + "properties": { + "concurrency": { + "description": "Sets the maximum number of suites to execute at the same time. Excess suites are queued and run in order as each suite completes.", + "type": "integer", + "minimum": 1 + }, + "metadata": { + "description": "The set of properties that allows you to provide additional information about your project that helps you distinguish it in the various environments in which it is used and reviewed.", + "type": "object", + "properties": { + "build": { + "description": "Sauce Labs can aggregate all jobs under one view based on their association with a build.", + "type": "string" + }, + "tags": { + "description": "Tag your jobs so you can find them easier in Sauce Labs.", + "type": "array" + } + }, + "additionalProperties": false + }, + "region": { + "description": "Which Sauce Labs data center to target.", + "enum": [ + "us-west-1", + "us-east-4", + "eu-central-1", + "asia-south-2" + ] + }, + "sauceignore": { + "description": "Path to the .sauceignore file.", + "default": ".sauceignore" + }, + "tunnel": { + "description": "SauceCTL supports using Sauce Connect to establish a secure connection when running your tests on Sauce Labs. To do so, launch a tunnel; then provide the identifier in this property.", + "properties": { + "name": { + "description": "The tunnel name.", + "type": "string" + }, + "owner": { + "description": "The owner (username) of the tunnel. Must be specified if the user that created the tunnel differs from the user that is running the tests.", + "type": "string" + }, + "timeout": { + "description": "How long to wait for the specified tunnel to be ready. Supports duration values like '10s', '30m' etc.", + "type": "string", + "pattern": "^(?:\\d+h)?(?:\\d+m)?(?:\\d+s)?(?:\\d+ms)?$", + "examples": [ + "1m", + "30s" + ] + } + }, + "required": [ + "name" + ], + "additionalProperties": false + }, + "retries": { + "description": "The number of times to retry a failing suite.", + "type": "integer", + "minimum": 0 + }, + "visibility": { + "description": "Set the visibility level of test results for suites run on Sauce Labs.", + "default": "team", + "type": "string", + "oneOf": [ + { + "const": "public", + "title": "Accessible to everyone." + }, + { + "const": "public restricted", + "title": "Share your test's results page and video, but keeps the logs only for you." + }, + { + "const": "share", + "title": "Only accessible to people with a valid link." + }, + { + "const": "team", + "title": "Only accessible to people under the same root account as you." + }, + { + "const": "private", + "title": "Only you (the owner) will be able to view assets and test results page." + } + ] + }, + "launchOrder": { + "description": "Control starting order of suites. The default is the order in which suites are written in the config file.", + "type": "string", + "oneOf": [ + { + "const": "fail rate", + "title": "Suites that historically have the highest failure rate start first." + } + ] + } + }, + "additionalProperties": false + } + }, + "additionalProperties": true + }, + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "saucectl reporters specific schema", + "description": "Subschema for reporters specific settings", + "type": "object", + "properties": { + "reporters": { + "type": "object", + "properties": { + "junit": { + "type": "object", + "description": "The JUnit reporter merges test results from all jobs in the JUnit format into a single report.", + "properties": { + "enabled": { + "description": "Toggles the reporter on/off.", + "type": "boolean" + }, + "filename": { + "description": "Filename for the generated JUnit report.", + "type": "string", + "default": "saucectl-report.xml" + } + } + }, + "json": { + "type": "object", + "description": "The JSON reporter merges test results from all jobs in the JSON format into a single report.", + "properties": { + "enabled": { + "description": "Toggles the reporter on/off.", + "type": "boolean" + }, + "webhookURL": { + "description": "Webhook URL to pass JSON report.", + "type": "string" + }, + "filename": { + "description": "Filename for the generated JSON report.", + "type": "string", + "default": "saucectl-report.json" + } + } + }, + "spotlight": { + "type": "object", + "description": "The spotlight reporter prints an overview of failed, or otherwise interesting, jobs.", + "properties": { + "enabled": { + "description": "Toggles the reporter on/off.", + "type": "boolean" + } + } + } + }, + "additionalProperties": false + } + }, + "additionalProperties": true + } + ], + "properties": { + "apiVersion": { + "const": "v1alpha" + }, + "kind": { + "const": "authoring" + }, + "defaults": { + "description": "Settings that are applied onto every suite by default, if no value is set on a suite explicitly.", + "type": "object", + "properties": { + "timeout": { + "description": "Instructs how long (in ms, s, m, or h) saucectl should wait for a suite to complete.", + "type": "string", + "pattern": "^(?:\\d+h)?(?:\\d+m)?(?:\\d+s)?(?:\\d+ms)?$", + "examples": [ + "1h", + "10m", + "90s" + ] + } + }, + "additionalProperties": false + }, + "suites": { + "description": "The set of AI-authored test cases to run. Each entry references a test suite by ID or exact name, or lists test cases explicitly.", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "properties": { + "name": { + "description": "The name of the suite, which will be reflected in the test results.", + "type": "string" + }, + "testSuiteId": { + "description": "The ID of the AI Test Authoring test suite whose test cases to run.", + "type": "string", + "pattern": "^[0-9a-fA-F-]{32,36}$" + }, + "testSuiteName": { + "description": "The exact name of the AI Test Authoring test suite whose test cases to run. Resolved at run time; an ambiguous name is an error.", + "type": "string" + }, + "testCases": { + "description": "Explicit list of test case IDs to run.", + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "pattern": "^[0-9a-fA-F]{24}$" + } + }, + "tags": { + "description": "Only run test cases of the referenced suite that carry at least one of these tags. Ignored when testCases is set.", + "type": "array", + "items": { + "type": "string" + } + }, + "targets": { + "description": "Browsers or devices to run against, overriding each test case's stored run targets. When omitted, the stored targets apply.", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "properties": { + "capabilities": { + "description": "W3C WebDriver capabilities, passed through unchanged, e.g. browserName, platformName, browserVersion, appium:deviceName.", + "type": "object", + "additionalProperties": true + }, + "isRdc": { + "description": "Declares the target as a real device rather than a virtual one. Optional; results are routed to the real-device table from the job the service reports, so this is only a hint.", + "type": "boolean" + } + }, + "required": [ + "capabilities" + ], + "additionalProperties": false + } + }, + "timeout": { + "description": "Instructs how long (in ms, s, m, or h) saucectl should wait for a suite to complete.", + "type": "string", + "pattern": "^(?:\\d+h)?(?:\\d+m)?(?:\\d+s)?(?:\\d+ms)?$", + "examples": [ + "1h", + "10m", + "90s" + ] + } + }, + "required": [ + "name" + ], + "oneOf": [ + { + "required": [ + "testSuiteId" + ] + }, + { + "required": [ + "testSuiteName" + ] + }, + { + "required": [ + "testCases" + ] + } + ], + "additionalProperties": false + } + } + }, + "required": [ + "apiVersion", + "kind", + "suites" + ], + "additionalProperties": true + } } ] } \ No newline at end of file diff --git a/api/v1alpha/framework/authoring.schema.json b/api/v1alpha/framework/authoring.schema.json new file mode 100644 index 000000000..e983c2fc6 --- /dev/null +++ b/api/v1alpha/framework/authoring.schema.json @@ -0,0 +1,127 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "saucectl AI Test Authoring runner configuration", + "description": "Configuration file for running Sauce Labs AI-authored tests using saucectl", + "type": "object", + "allOf": [ + { + "$ref": "../subschema/artifacts.schema.json" + }, + { + "$ref": "../subschema/sauce.schema.json" + }, + { + "$ref": "../subschema/reporters.schema.json" + } + ], + "properties": { + "apiVersion": { + "const": "v1alpha" + }, + "kind": { + "const": "authoring" + }, + "defaults": { + "description": "Settings that are applied onto every suite by default, if no value is set on a suite explicitly.", + "type": "object", + "properties": { + "timeout": { + "$ref": "../subschema/common.schema.json#/definitions/timeout" + } + }, + "additionalProperties": false + }, + "suites": { + "description": "The set of AI-authored test cases to run. Each entry references a test suite by ID or exact name, or lists test cases explicitly.", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "properties": { + "name": { + "description": "The name of the suite, which will be reflected in the test results.", + "type": "string" + }, + "testSuiteId": { + "description": "The ID of the AI Test Authoring test suite whose test cases to run.", + "type": "string", + "pattern": "^[0-9a-fA-F-]{32,36}$" + }, + "testSuiteName": { + "description": "The exact name of the AI Test Authoring test suite whose test cases to run. Resolved at run time; an ambiguous name is an error.", + "type": "string" + }, + "testCases": { + "description": "Explicit list of test case IDs to run.", + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "pattern": "^[0-9a-fA-F]{24}$" + } + }, + "tags": { + "description": "Only run test cases of the referenced suite that carry at least one of these tags. Ignored when testCases is set.", + "type": "array", + "items": { + "type": "string" + } + }, + "targets": { + "description": "Browsers or devices to run against, overriding each test case's stored run targets. When omitted, the stored targets apply.", + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "properties": { + "capabilities": { + "description": "W3C WebDriver capabilities, passed through unchanged, e.g. browserName, platformName, browserVersion, appium:deviceName.", + "type": "object", + "additionalProperties": true + }, + "isRdc": { + "description": "Declares the target as a real device rather than a virtual one. Optional; results are routed to the real-device table from the job the service reports, so this is only a hint.", + "type": "boolean" + } + }, + "required": [ + "capabilities" + ], + "additionalProperties": false + } + }, + "timeout": { + "$ref": "../subschema/common.schema.json#/definitions/timeout" + } + }, + "required": [ + "name" + ], + "oneOf": [ + { + "required": [ + "testSuiteId" + ] + }, + { + "required": [ + "testSuiteName" + ] + }, + { + "required": [ + "testCases" + ] + } + ], + "additionalProperties": false + } + } + }, + "required": [ + "apiVersion", + "kind", + "suites" + ], + "additionalProperties": true +} diff --git a/cmd/saucectl/saucectl.go b/cmd/saucectl/saucectl.go index 191a4a545..226f5c957 100644 --- a/cmd/saucectl/saucectl.go +++ b/cmd/saucectl/saucectl.go @@ -14,6 +14,7 @@ import ( "github.com/rs/zerolog/log" "github.com/saucelabs/saucectl/internal/cmd/apit" "github.com/saucelabs/saucectl/internal/cmd/artifacts" + "github.com/saucelabs/saucectl/internal/cmd/authoring" "github.com/saucelabs/saucectl/internal/cmd/builds" "github.com/saucelabs/saucectl/internal/cmd/completion" "github.com/saucelabs/saucectl/internal/cmd/configure" @@ -73,6 +74,7 @@ func main() { apit.Command(cmd.PersistentPreRun), builds.Command(cmd.PersistentPreRun), devices.Command(cmd.PersistentPreRun), + authoring.Command(cmd.PersistentPreRun), ) if err := cmd.ExecuteContext(newContext()); err != nil { diff --git a/internal/cmd/authoring/artifact.go b/internal/cmd/authoring/artifact.go new file mode 100644 index 000000000..94fca048f --- /dev/null +++ b/internal/cmd/authoring/artifact.go @@ -0,0 +1,97 @@ +package authoring + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// DownloadArtifactCommand is `authoring download-artifact`: it fetches a file +// captured during authoring, such as a step screenshot, by its identifier. +func DownloadArtifactCommand() *cobra.Command { + var filename string + var force bool + + cmd := &cobra.Command{ + Use: "download-artifact -f ", + Aliases: []string{"artifact"}, + Short: "Download an artifact captured during authoring, such as a step screenshot", + Long: `Download an artifact by its identifier. + +The identifier is the last path segment of a step's screenshot URL, as shown by +'testcases get --show-steps'. A full screenshot URL is also accepted. The download +carries no content type, so the destination filename is required.`, + Example: ` saucectl authoring download-artifact 3f2a9c1e4b5d6e7f8a9b0c1d2e3f4a5b -f step-3.png`, + SilenceUsage: true, + Args: requireArgs("artifact-id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if filename == "" { + return errors.New("a destination is required: use -f/--filename") + } + id := authoring.ArtifactIDFromURL(args[0]) + if id == "" { + return fmt.Errorf("invalid artifact identifier %q", args[0]) + } + return downloadArtifact(cmd, id, filename, force) + }, + } + + flags := cmd.Flags() + flags.StringVarP(&filename, "filename", "f", "", "Destination file. Required, because the response carries no content type from which an extension could be inferred.") + flags.BoolVar(&force, "force", false, "Overwrite the destination if it exists.") + + return cmd +} + +// downloadArtifact streams the artifact to filename, refusing to overwrite +// unless forced (FR-031 applies to every file this tool writes). +func downloadArtifact(cmd *cobra.Command, id, filename string, force bool) error { + if !force { + if _, err := os.Stat(filename); err == nil { + return fmt.Errorf("%s already exists; use --force to overwrite", filename) + } + } + + rc, err := artifactService.DownloadArtifact(cmd.Context(), id) + if err != nil { + return fmt.Errorf("failed to download artifact: %w", err) + } + defer rc.Close() + + // Download to a temporary file beside the destination and rename on + // success. A transfer that dies half way would otherwise leave a + // truncated file that looks complete, and the overwrite guard above + // would then refuse the obvious retry of the same command. + dir := filepath.Dir(filename) + tmp, err := os.CreateTemp(dir, filepath.Base(filename)+".partial-*") + if err != nil { + return fmt.Errorf("failed to create a temporary file in %s: %w", dir, err) + } + tmpName := tmp.Name() + defer func() { + tmp.Close() + os.Remove(tmpName) // no-op once the rename below has succeeded + }() + + n, err := io.Copy(tmp, rc) + if err != nil { + return fmt.Errorf("failed to write %s: %w", filename, err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("failed to write %s: %w", filename, err) + } + if err := os.Rename(tmpName, filename); err != nil { + return fmt.Errorf("failed to move the download into place at %s: %w", filename, err) + } + fmt.Printf("Wrote %d bytes to %s\n", n, filename) + return nil +} diff --git a/internal/cmd/authoring/cmd.go b/internal/cmd/authoring/cmd.go new file mode 100644 index 000000000..b0b2d6ef4 --- /dev/null +++ b/internal/cmd/authoring/cmd.go @@ -0,0 +1,149 @@ +// Package authoring implements the `saucectl authoring` command group: the +// command-line surface of the Sauce Labs AI Test Authoring service — test +// cases, test suites, schedules, variables and artifacts. Running authored +// suites as part of `saucectl run` lives in internal/cmd/run and +// internal/authoring instead. +package authoring + +import ( + "errors" + "fmt" + "time" + + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" + cmds "github.com/saucelabs/saucectl/internal/cmd" + "github.com/saucelabs/saucectl/internal/credentials" + "github.com/saucelabs/saucectl/internal/http" + "github.com/saucelabs/saucectl/internal/iam" + "github.com/saucelabs/saucectl/internal/region" + "github.com/saucelabs/saucectl/internal/usage" +) + +// Service handles shared by every subcommand. They are set by the root +// command's pre-run and kept unexported so the four subgroups can share them +// without a global registry — the same choice internal/cmd/apit makes. +var ( + testCaseService authoring.TestCaseService + testSuiteService authoring.TestSuiteService + scheduleService authoring.ScheduleService + variableService authoring.VariableService + artifactService authoring.ArtifactService + entitlementReader authoring.EntitlementReader + userService iam.UserService + + // regio is the resolved region, needed to derive dashboard links. + regio region.Region + // currentUser is resolved by the entitlement gate and reused by commands + // that default to the caller's own identity. + currentUser iam.User +) + +// Request timeouts. authoringTimeout bounds one request to the authoring +// service; listings can be heavy, so it is generous. iamTimeout bounds the +// user lookup behind the entitlement gate. +var ( + authoringTimeout = 2 * time.Minute + iamTimeout = 30 * time.Second +) + +// Command creates the `authoring` command group. preRun is the root command's +// persistent pre-run (logging and usage setup), invoked first so behaviour +// matches every other group. +func Command(preRun func(cmd *cobra.Command, args []string)) *cobra.Command { + var regionFlag string + + cmd := &cobra.Command{ + Use: "authoring", + Short: "Manage and run AI-authored tests", + Long: `Manage Sauce Labs AI Test Authoring assets — test cases, test suites, schedules and +variables — and author new tests from a plain-language description. + +To run authored suites as part of a pipeline, with reporters, artifact download and CI +exit codes, use 'saucectl run' with a 'kind: authoring' configuration instead.`, + Example: ` saucectl authoring testcases list --tag smoke + saucectl authoring testcases get 6a882c1dc8b4482c166e96c9 --show-steps + saucectl authoring testsuites create --name "Checkout" --test-case 6a882c1dc8b4482c166e96c9 + saucectl authoring variables create --scope org --name password --secret --value-from-env PASSWORD`, + SilenceUsage: true, + TraverseChildren: true, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if preRun != nil { + preRun(cmd, args) + } + + reg := region.FromString(regionFlag) + if reg == region.None { + return fmt.Errorf("invalid region %q; options: %s", regionFlag, region.Options()) + } + if reg == region.Staging { + usage.DefaultClient.Enabled = false + } + regio = reg + + creds := credentials.Get() + if !creds.IsSet() { + return errors.New("no credentials set; run 'saucectl configure' or set SAUCE_USERNAME and SAUCE_ACCESS_KEY") + } + + svc := http.NewAuthoringService(reg, creds, authoringTimeout) + testCaseService = &svc + testSuiteService = &svc + scheduleService = &svc + variableService = &svc + artifactService = &svc + entitlementReader = &svc + + iamClient := http.NewUserService(reg.APIBaseURL(), creds, iamTimeout) + userService = &iamClient + + user, err := authoring.VerifyEntitlement(cmd.Context(), userService, entitlementReader) + if err != nil { + return err + } + currentUser = user + return nil + }, + } + + cmd.PersistentFlags().StringVarP(®ionFlag, "region", "r", "us-west-1", fmt.Sprintf("The Sauce Labs region. Options: %s.", region.Options())) + + cmd.AddCommand( + TestCasesCommand(), + TestSuitesCommand(), + SchedulesCommand(), + VariablesCommand(), + DownloadArtifactCommand(), + ) + + return cmd +} + +// trackUsage reports the command invocation the way every other command does. +func trackUsage(cmd *cobra.Command) { + tracker := usage.DefaultClient + go func() { + tracker.Collect( + cmds.FullName(cmd), + usage.Flags(cmd.Flags()), + ) + _ = tracker.Close() + }() +} + +// requireArgs returns a cobra argument validator demanding exactly n +// non-empty positional arguments, named for the error message. +func requireArgs(names ...string) cobra.PositionalArgs { + return func(_ *cobra.Command, args []string) error { + if len(args) != len(names) { + return fmt.Errorf("expected %d argument(s): %v", len(names), names) + } + for i, a := range args { + if a == "" { + return fmt.Errorf("argument %s must not be empty", names[i]) + } + } + return nil + } +} diff --git a/internal/cmd/authoring/confirm.go b/internal/cmd/authoring/confirm.go new file mode 100644 index 000000000..90486931d --- /dev/null +++ b/internal/cmd/authoring/confirm.go @@ -0,0 +1,62 @@ +package authoring + +import ( + "errors" + "fmt" + "os" + + "github.com/AlecAivazis/survey/v2" +) + +// ErrAborted is returned when the user declines a confirmation prompt. +var ErrAborted = errors.New("aborted") + +// ErrConfirmationRequired is returned when a destructive command runs without +// a terminal to ask on and without an explicit bypass. +var ErrConfirmationRequired = errors.New("refusing to proceed without confirmation: not running interactively; re-run with --yes to confirm") + +// promptConfirm asks the yes/no question on the terminal. It is a variable so +// tests can replace it; the default uses survey like the rest of the tool. +var promptConfirm = func(question string) (bool, error) { + var ok bool + err := survey.AskOne(&survey.Confirm{Message: question, Default: false}, &ok) + return ok, err +} + +// isInteractive reports whether a prompt can be shown. A variable for tests. +var isInteractive = interactive + +// confirmDestructive gates every removal of a shared asset (FR-037–041). The +// behaviour is identical across all four asset types so it is learned once: +// +// interactive, no --yes → print what is affected, then prompt +// interactive, --yes → proceed silently +// non-interactive, no --yes → refuse with ErrConfirmationRequired +// non-interactive, --yes → proceed +// +// This deliberately departs from `storage delete`, which acts immediately: +// these are organisation-level assets that colleagues depend on, with no +// undo. The refusal, rather than a hang or an unconfirmed proceed, is what +// makes the command safe in a pipeline (FR-040). +func confirmDestructive(yes bool, what string, affects []string) error { + if yes { + return nil + } + if !isInteractive() { + return fmt.Errorf("%w (%s)", ErrConfirmationRequired, what) + } + + fmt.Fprintf(os.Stdout, "About to delete %s.\n", what) + for _, a := range affects { + fmt.Fprintf(os.Stdout, " - %s\n", a) + } + + ok, err := promptConfirm("Proceed?") + if err != nil { + return err + } + if !ok { + return ErrAborted + } + return nil +} diff --git a/internal/cmd/authoring/fixes_test.go b/internal/cmd/authoring/fixes_test.go new file mode 100644 index 000000000..abacb118b --- /dev/null +++ b/internal/cmd/authoring/fixes_test.go @@ -0,0 +1,294 @@ +package authoring + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "strings" + "testing" + "time" + + "github.com/spf13/pflag" + + "github.com/saucelabs/saucectl/internal/authoring" + "github.com/saucelabs/saucectl/internal/mocks" +) + +func TestFetchPage_AllHonoursSkip(t *testing.T) { + // --all used to restart from offset 0, so `--all --skip 100` silently + // returned the whole listing from the beginning. + var offsets []int + fetch := func(_ context.Context, opts authoring.ListOptions) (authoring.List[int], error) { + offsets = append(offsets, opts.Skip) + if opts.Skip >= 150 { + return authoring.List[int]{Items: nil, Total: 150}, nil + } + items := make([]int, authoring.DefaultPageSize) + return authoring.List[int]{Items: items, Total: 150}, nil + } + p := pageFlags{all: true, skip: 50, limit: 20} + if _, _, err := fetchPage(context.Background(), p, "things", fetch); err != nil { + t.Fatal(err) + } + if len(offsets) == 0 || offsets[0] != 50 { + t.Errorf("first page fetched at offset %v, want the requested skip of 50", offsets) + } +} + +func TestFetchPage_WarnsWhenAllCannotHonourLimit(t *testing.T) { + // Constitution VIII: a setting the tool cannot honour warns rather than + // being dropped in silence. Assert the flag state is what drives it. + fs := pflag.NewFlagSet("t", pflag.ContinueOnError) + p := &pageFlags{} + p.bind(fs) + if err := fs.Parse([]string{"--all", "--limit", "5"}); err != nil { + t.Fatal(err) + } + p.capture(fs) + if !p.limitChanged { + t.Error("an explicitly set --limit must be recorded so the warning can fire") + } + + fs2 := pflag.NewFlagSet("t2", pflag.ContinueOnError) + p2 := &pageFlags{} + p2.bind(fs2) + _ = fs2.Parse([]string{"--all"}) + p2.capture(fs2) + if p2.limitChanged { + t.Error("an untouched --limit must not warn") + } +} + +func TestGetTestCase_JSONHonoursRevision(t *testing.T) { + // --revision was resolved and then discarded under -o json, so a script + // reading .revisions[-1] got the latest revision instead of the pinned one. + tc := authoring.TestCase{ID: "tc", Name: "n", Revisions: []authoring.Revision{ + {ID: "old", Intent: "first"}, + {ID: "new", Intent: "second"}, + }} + testCaseService = &mocks.AuthoringService{ + GetTestCaseFn: func(context.Context, string) (authoring.TestCase, error) { return tc, nil }, + } + t.Cleanup(func() { testCaseService = nil }) + + out := captureStdout(t, func() { + if err := getTestCase(context.Background(), JSONOutput, "tc", "old", false); err != nil { + t.Fatal(err) + } + }) + var got authoring.TestCase + if err := json.Unmarshal([]byte(out), &got); err != nil { + t.Fatalf("output is not valid JSON: %v\n%s", err, out) + } + if len(got.Revisions) != 1 || got.Revisions[0].ID != "old" { + t.Errorf("JSON carried %d revision(s) %v; want only the pinned one", len(got.Revisions), revisionIDs(got)) + } +} + +func revisionIDs(tc authoring.TestCase) []string { + ids := make([]string, 0, len(tc.Revisions)) + for _, r := range tc.Revisions { + ids = append(ids, r.ID) + } + return ids +} + +func TestIsAbandonedWait(t *testing.T) { + // Only our own giving up means "still running"; anything the service + // said is a real error and must not advise more polling. + if !isAbandonedWait(context.Canceled) || !isAbandonedWait(context.DeadlineExceeded) { + t.Error("interrupt and local timeout are abandoned waits") + } + if isAbandonedWait(&authoring.APIError{HTTPStatus: 404, Code: "TEST_CASE_GENERATION_TASK_NOT_FOUND"}) { + t.Error("a 404 from the service is not an abandoned wait") + } +} + +func TestWaitForGeneration_FatalErrorIsNotReportedAsStillRunning(t *testing.T) { + // A mistyped task id used to print "generation is still running on Sauce + // Labs. Check progress with: ..." and swallow the cause. + notFound := &authoring.APIError{HTTPStatus: 404, Code: "TEST_CASE_GENERATION_TASK_NOT_FOUND", Detail: "Test case generation task not found."} + testCaseService = &mocks.AuthoringService{ + GenerationStatusFn: func(context.Context, string) (authoring.GenerationState, error) { + return authoring.GenerationState{}, notFound + }, + } + t.Cleanup(func() { testCaseService = nil }) + + var buf bytes.Buffer + err := waitForGeneration(context.Background(), "nope", time.Millisecond, time.Second, TextOutput, &buf) + if err == nil { + t.Fatal("expected an error") + } + if errors.Is(err, ErrGenerationStillRunning) { + t.Error("a 404 must not be reported as still running") + } + if !errors.Is(err, notFound) { + t.Error("the cause must be wrapped with %w so callers can match the service sentinel") + } + if strings.Contains(buf.String(), "still running") { + t.Errorf("stdout advised more polling for a task that does not exist:\n%s", buf.String()) + } +} + +// captureStdout runs fn with os.Stdout redirected and returns what it wrote. +func captureStdout(t *testing.T, fn func()) string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + orig := os.Stdout + os.Stdout = w + done := make(chan string, 1) + go func() { + var b bytes.Buffer + _, _ = io.Copy(&b, r) + done <- b.String() + }() + fn() + w.Close() + os.Stdout = orig + return <-done +} + +func TestWaitForGeneration_FailedTaskExitsNonZeroInJSONMode(t *testing.T) { + // The JSON branch used to return before the status switch, so a FAILED + // task printed its payload and exited 0, breaking CI gating with -o json. + testCaseService = &mocks.AuthoringService{ + GenerationStatusFn: func(context.Context, string) (authoring.GenerationState, error) { + return authoring.GenerationState{ + Status: authoring.GenerationFailed, + Error: &authoring.GenerationError{Code: "TEST_CASE_EMPTY", Detail: "nothing was recorded"}, + }, nil + }, + } + t.Cleanup(func() { testCaseService = nil }) + + var err error + out := captureStdout(t, func() { + err = waitForGeneration(context.Background(), "task", time.Millisecond, time.Second, JSONOutput, io.Discard) + }) + if err == nil { + t.Fatal("a FAILED task must return an error so the process exits non-zero") + } + if !strings.Contains(err.Error(), "TEST_CASE_EMPTY") { + t.Errorf("error lost the service's code: %v", err) + } + // The payload still has to reach a script that asked for JSON. + var doc map[string]any + if e := json.Unmarshal([]byte(out), &doc); e != nil { + t.Fatalf("no JSON object was printed: %v\n%s", e, out) + } + if doc["status"] != "FAILED" || doc["taskId"] != "task" { + t.Errorf("JSON = %v; want the failed status and the task id", doc) + } +} + +func TestWaitForGeneration_TimeoutSurfacesTaskIDInBothFormats(t *testing.T) { + // Without the id there is nothing to reattach with. + inProgress := func(context.Context, string) (authoring.GenerationState, error) { + return authoring.GenerationState{Status: authoring.GenerationInProgress}, nil + } + testCaseService = &mocks.AuthoringService{GenerationStatusFn: inProgress} + t.Cleanup(func() { testCaseService = nil }) + + var buf bytes.Buffer + err := waitForGeneration(context.Background(), "task-42", time.Millisecond, 30*time.Millisecond, TextOutput, &buf) + if err == nil || !strings.Contains(err.Error(), "task-42") { + t.Errorf("text mode error = %v; want the task id", err) + } + if !strings.Contains(buf.String(), "task-42") { + t.Errorf("text mode printed no reattach hint:\n%s", buf.String()) + } + + out := captureStdout(t, func() { + err = waitForGeneration(context.Background(), "task-42", time.Millisecond, 30*time.Millisecond, JSONOutput, io.Discard) + }) + if err == nil || !strings.Contains(err.Error(), "task-42") { + t.Errorf("json mode error = %v; want the task id", err) + } + var doc map[string]any + if e := json.Unmarshal([]byte(out), &doc); e != nil || doc["taskId"] != "task-42" { + t.Errorf("json mode emitted %q; want an object carrying taskId", out) + } +} + +func TestBuildCreateScheduleOptions_RejectsUTCLocally(t *testing.T) { + // The service's accepted list is region/city zones only; catching these + // two saves a round trip and an opaque INVALID_BODY. + for _, tz := range []string{"UTC", "utc", "Etc/UTC"} { + _, err := buildCreateScheduleOptions(scheduleFlags{name: "n", cron: "0 0 3 * * *", timezone: tz, testSuiteIDs: []string{"s"}}, "me") + if err == nil { + t.Errorf("--timezone %q was accepted", tz) + continue + } + if !strings.Contains(err.Error(), "Atlantic/Reykjavik") { + t.Errorf("--timezone %q: error should name a usable zero-offset zone, got %v", tz, err) + } + } + if _, err := buildCreateScheduleOptions(scheduleFlags{name: "n", cron: "0 0 3 * * *", timezone: "Europe/Berlin", testSuiteIDs: []string{"s"}}, "me"); err != nil { + t.Errorf("a region/city zone must be accepted: %v", err) + } +} + +func TestBuildUpdateScheduleOptions_SetAndUnsetConflict(t *testing.T) { + // The unset loop ran last and silently won. + current := authoring.TestSchedule{ + Name: "n", + State: authoring.ScheduleState{StateName: authoring.ScheduleEnabled}, + TestSuiteIDs: []string{"s1"}, + Settings: authoring.ScheduleSettings{Cron: "c", Timezone: "Europe/Berlin", RunningUserID: "u"}, + } + cases := []struct{ flag, field string }{ + {"tunnel-name", "tunnelName"}, + {"build", "buildName"}, + {"start-date", "startDate"}, + {"end-date", "endDate"}, + {"max-runs", "maxRuns"}, + } + for _, c := range cases { + f := scheduleUpdateFlags{unset: []string{c.field}} + _, err := buildUpdateScheduleOptions(changedSet{c.flag: true}, f, current) + if err == nil { + t.Errorf("--%s with --unset %s was accepted", c.flag, c.field) + continue + } + if !strings.Contains(err.Error(), c.flag) || !strings.Contains(err.Error(), c.field) { + t.Errorf("--%s/--unset %s: error should name both, got %v", c.flag, c.field, err) + } + } + // Unsetting a field nobody set is still fine. + if _, err := buildUpdateScheduleOptions(changedSet{}, scheduleUpdateFlags{unset: []string{"buildName"}}, current); err != nil { + t.Errorf("unset alone must work: %v", err) + } +} + +func TestFetchPage_WarningsAreSuppressedUnderJSON(t *testing.T) { + // The logger writes to stdout for every command, so a warning emitted + // while rendering JSON lands inside the document and breaks `| jq`. + fs := pflag.NewFlagSet("t", pflag.ContinueOnError) + fs.StringP("out", "o", TextOutput, "") + p := &pageFlags{} + p.bind(fs) + if err := fs.Parse([]string{"--all", "--limit", "5", "-o", "json"}); err != nil { + t.Fatal(err) + } + p.capture(fs) + if !p.jsonOut { + t.Error("JSON output must be recorded so advisory warnings can be held back") + } + + fs2 := pflag.NewFlagSet("t2", pflag.ContinueOnError) + fs2.StringP("out", "o", TextOutput, "") + p2 := &pageFlags{} + p2.bind(fs2) + _ = fs2.Parse([]string{"--all", "--limit", "5"}) + p2.capture(fs2) + if p2.jsonOut { + t.Error("text output must still warn") + } +} diff --git a/internal/cmd/authoring/helpers_test.go b/internal/cmd/authoring/helpers_test.go new file mode 100644 index 000000000..d63d20039 --- /dev/null +++ b/internal/cmd/authoring/helpers_test.go @@ -0,0 +1,434 @@ +package authoring + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/saucelabs/saucectl/internal/authoring" + "github.com/saucelabs/saucectl/internal/iam" + "github.com/saucelabs/saucectl/internal/mocks" +) + +func TestParseTargets(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "target.json") + if err := os.WriteFile(file, []byte(`{"capabilities":{"platformName":"Android"},"isRdc":true}`), 0o644); err != nil { + t.Fatal(err) + } + + got, err := parseTargets( + []string{`browserName=chrome,platformName="Windows 11",browserVersion=latest,sauce:options.screenResolution=1920x1080,appium:autoGrantPermissions=true`}, + []string{`{"browserName":"firefox"}`, "@" + file}, + ) + if err != nil { + t.Fatal(err) + } + if len(got) != 3 { + t.Fatalf("got %d targets, want 3", len(got)) + } + + kv := got[0].Capabilities + if kv["browserName"] != "chrome" || kv["platformName"] != "Windows 11" || kv["browserVersion"] != "latest" { + t.Errorf("kv capabilities = %v", kv) + } + if kv["appium:autoGrantPermissions"] != true { + t.Errorf("boolean literal not coerced: %v", kv["appium:autoGrantPermissions"]) + } + nested, ok := kv["sauce:options"].(map[string]any) + if !ok || nested["screenResolution"] != "1920x1080" { + t.Errorf("dotted key not nested: %v", kv["sauce:options"]) + } + if got[1].Capabilities["browserName"] != "firefox" || got[1].IsRDC { + t.Errorf("bare json = %+v", got[1]) + } + if got[2].Capabilities["platformName"] != "Android" || !got[2].IsRDC { + t.Errorf("wrapped json from file = %+v", got[2]) + } + + for _, bad := range [][]string{{"nokey"}, {"=value"}, {""}} { + if _, err := parseTargets(bad, nil); err == nil { + t.Errorf("expected error for %v", bad) + } + } + if _, err := parseTargets(nil, []string{`{not json`}); err == nil { + t.Error("expected error for invalid json") + } + if _, err := parseTargets(nil, []string{`{}`}); err == nil { + t.Error("expected error for empty json object") + } + if _, err := parseTargets(nil, []string{`{"capabilities":"oops"}`}); err == nil { + t.Error("expected error for non-object capabilities") + } +} + +func TestDescribeTarget(t *testing.T) { + tests := []struct { + caps map[string]any + want string + }{ + {map[string]any{"browserName": "chrome", "browserVersion": "latest", "platformName": "Windows 11"}, "chrome latest / Windows 11"}, + {map[string]any{"platformName": "Android", "appium:platformVersion": "16", "appium:deviceName": "Google Pixel 9 Emulator"}, "Google Pixel 9 Emulator / Android 16"}, + {map[string]any{}, "-"}, + } + for _, tt := range tests { + if got := describeTarget(authoring.Target{Capabilities: tt.caps}); got != tt.want { + t.Errorf("describeTarget(%v) = %q, want %q", tt.caps, got, tt.want) + } + } +} + +func TestResolveValue(t *testing.T) { + t.Setenv("AUTHORING_TEST_VALUE", "from-env") + t.Setenv("AUTHORING_EMPTY_VALUE", "") + + dir := t.TempDir() + file := filepath.Join(dir, "value.txt") + if err := os.WriteFile(file, []byte("from-file\n"), 0o600); err != nil { + t.Fatal(err) + } + + origTerm, origInteractive, origPrompt := stdinIsTerminal, isInteractive, promptValue + t.Cleanup(func() { stdinIsTerminal, isInteractive, promptValue = origTerm, origInteractive, origPrompt }) + stdinIsTerminal = func() bool { return false } + isInteractive = func() bool { return false } + + tests := []struct { + name string + vs valueSource + stdin string + want string + wantErr string + }{ + {name: "env", vs: valueSource{envName: "AUTHORING_TEST_VALUE"}, want: "from-env"}, + {name: "env empty", vs: valueSource{envName: "AUTHORING_EMPTY_VALUE"}, wantErr: "not set or empty"}, + {name: "env missing", vs: valueSource{envName: "AUTHORING_MISSING_VALUE"}, wantErr: "not set or empty"}, + {name: "file strips one newline", vs: valueSource{file: file}, want: "from-file"}, + {name: "file missing", vs: valueSource{file: filepath.Join(dir, "nope")}, wantErr: "reading value file"}, + {name: "stdin", vs: valueSource{file: "-"}, stdin: "piped\r\n", want: "piped"}, + {name: "stdin keeps inner newlines", vs: valueSource{file: "-"}, stdin: "a\nb\n", want: "a\nb"}, + {name: "literal", vs: valueSource{value: "literal"}, want: "literal"}, + {name: "two sources", vs: valueSource{value: "a", envName: "AUTHORING_TEST_VALUE"}, wantErr: "only one of"}, + {name: "none, non-interactive", vs: valueSource{}, wantErr: "no value given"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := resolveValue(tt.vs, strings.NewReader(tt.stdin)) + if tt.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("err = %v, want containing %q", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Errorf("got %q, want %q", got, tt.want) + } + }) + } + + t.Run("stdin refused on a terminal", func(t *testing.T) { + stdinIsTerminal = func() bool { return true } + defer func() { stdinIsTerminal = func() bool { return false } }() + if _, err := resolveValue(valueSource{file: "-"}, strings.NewReader("x")); err == nil { + t.Error("expected refusal") + } + }) + + t.Run("none, interactive prompts", func(t *testing.T) { + isInteractive = func() bool { return true } + defer func() { isInteractive = func() bool { return false } }() + var askedSecret bool + promptValue = func(secret bool) (string, error) { askedSecret = secret; return "typed", nil } + got, err := resolveValue(valueSource{secret: true}, strings.NewReader("")) + if err != nil || got != "typed" || !askedSecret { + t.Errorf("got %q, %v, secret=%v", got, err, askedSecret) + } + }) +} + +func TestConfirmDestructive(t *testing.T) { + origInteractive, origPrompt := isInteractive, promptConfirm + t.Cleanup(func() { isInteractive, promptConfirm = origInteractive, origPrompt }) + + promptCalls := 0 + promptConfirm = func(string) (bool, error) { promptCalls++; return false, nil } + + isInteractive = func() bool { return false } + if err := confirmDestructive(true, "x", nil); err != nil { + t.Errorf("--yes non-interactive: %v", err) + } + if err := confirmDestructive(false, "x", nil); !errors.Is(err, ErrConfirmationRequired) { + t.Errorf("non-interactive without --yes must refuse, got %v", err) + } + if promptCalls != 0 { + t.Error("must never prompt when non-interactive") + } + + isInteractive = func() bool { return true } + if err := confirmDestructive(true, "x", nil); err != nil || promptCalls != 0 { + t.Errorf("--yes interactive must not prompt: %v, calls=%d", err, promptCalls) + } + if err := confirmDestructive(false, "x", []string{"a"}); !errors.Is(err, ErrAborted) || promptCalls != 1 { + t.Errorf("declined prompt: err=%v calls=%d", err, promptCalls) + } + promptConfirm = func(string) (bool, error) { return true, nil } + if err := confirmDestructive(false, "x", nil); err != nil { + t.Errorf("accepted prompt: %v", err) + } +} + +// changedSet stubs pflag.FlagSet.Changed for the overlay test. +type changedSet map[string]bool + +func (c changedSet) Changed(name string) bool { return c[name] } + +func TestBuildUpdateScheduleOptions(t *testing.T) { + five := 5 + current := authoring.TestSchedule{ + Name: "Nightly", + State: authoring.ScheduleState{StateName: authoring.ScheduleEnabled}, + TestSuiteIDs: []string{"s1", "s2"}, + Settings: authoring.ScheduleSettings{ + Cron: "0 2 * * *", Timezone: "Europe/Berlin", RunningUserID: "u", MaxRuns: &five, TunnelName: "old-tunnel", BuildName: "nightly", + }, + } + + t.Run("cron change re-sends the complete settings", func(t *testing.T) { + f := scheduleUpdateFlags{scheduleFlags: scheduleFlags{cron: "0 3 * * *"}} + opts, err := buildUpdateScheduleOptions(changedSet{"cron": true}, f, current) + if err != nil { + t.Fatal(err) + } + s := opts.Settings + if s == nil || s.Cron != "0 3 * * *" || s.Timezone != "Europe/Berlin" || s.RunningUserID != "u" || s.MaxRuns == nil || *s.MaxRuns != 5 { + t.Errorf("settings not fully re-sent: %+v", s) + } + // The service replaces the schedule, so the rest of the object must + // always travel along. + if opts.Name != "Nightly" || opts.StateName != authoring.ScheduleEnabled || len(opts.TestSuiteIDs) != 2 { + t.Errorf("complete object not re-sent: %+v", opts) + } + if s.TunnelName == nil || *s.TunnelName != "old-tunnel" || s.BuildName == nil || *s.BuildName != "nightly" { + t.Errorf("nullable fields not carried over: %+v", s) + } + }) + + t.Run("unset tunnelName sends null", func(t *testing.T) { + f := scheduleUpdateFlags{unset: []string{"tunnelName", "maxRuns"}} + opts, err := buildUpdateScheduleOptions(changedSet{}, f, current) + if err != nil { + t.Fatal(err) + } + if opts.Settings.TunnelName == nil || *opts.Settings.TunnelName != "" { + t.Errorf("tunnelName = %v, want pointer to empty string (null on the wire)", opts.Settings.TunnelName) + } + if opts.Settings.MaxRuns != nil || !opts.Settings.ClearMaxRuns { + t.Error("--unset maxRuns must send an explicit null; omitting keeps the stored value") + } + }) + + t.Run("unknown unset field", func(t *testing.T) { + _, err := buildUpdateScheduleOptions(changedSet{}, scheduleUpdateFlags{unset: []string{"cron"}}, current) + if err == nil { + t.Error("expected error") + } + }) + + t.Run("nothing to update", func(t *testing.T) { + _, err := buildUpdateScheduleOptions(changedSet{}, scheduleUpdateFlags{}, current) + if !errors.Is(err, authoring.ErrEmptyUpdate) { + t.Errorf("got %v", err) + } + }) + + t.Run("wholesale and incremental suites are exclusive", func(t *testing.T) { + f := scheduleUpdateFlags{scheduleFlags: scheduleFlags{testSuiteIDs: []string{"a"}}, addTestSuiteIDs: []string{"b"}} + if _, err := buildUpdateScheduleOptions(changedSet{}, f, current); err == nil { + t.Error("expected error") + } + }) + + t.Run("state only still sends the complete object", func(t *testing.T) { + opts, err := buildUpdateScheduleOptions(noFlagChanges{}, scheduleUpdateFlags{scheduleFlags: scheduleFlags{state: "disabled"}}, current) + if err != nil || opts.StateName != authoring.ScheduleDisabled || opts.Settings == nil || opts.Name != "Nightly" || len(opts.TestSuiteIDs) != 2 { + t.Errorf("opts = %+v, err = %v", opts, err) + } + }) + + t.Run("add and remove suites are applied client-side", func(t *testing.T) { + f := scheduleUpdateFlags{addTestSuiteIDs: []string{"s3", "s1"}, removeTestSuiteIDs: []string{"s2"}} + opts, err := buildUpdateScheduleOptions(changedSet{}, f, current) + if err != nil || strings.Join(opts.TestSuiteIDs, ",") != "s1,s3" { + t.Errorf("suites = %v, err = %v", opts.TestSuiteIDs, err) + } + }) + + t.Run("removing every suite is refused", func(t *testing.T) { + f := scheduleUpdateFlags{removeTestSuiteIDs: []string{"s1", "s2"}} + if _, err := buildUpdateScheduleOptions(changedSet{}, f, current); err == nil { + t.Error("expected error") + } + }) + + t.Run("observed state needs an explicit --state", func(t *testing.T) { + errored := current + errored.State = authoring.ScheduleState{StateName: authoring.ScheduleErrored} + if _, err := buildUpdateScheduleOptions(changedSet{"cron": true}, scheduleUpdateFlags{scheduleFlags: scheduleFlags{cron: "x"}}, errored); err == nil { + t.Error("expected error asking for --state") + } + f := scheduleUpdateFlags{scheduleFlags: scheduleFlags{cron: "x", state: "enabled"}} + if _, err := buildUpdateScheduleOptions(changedSet{"cron": true}, f, errored); err != nil { + t.Errorf("explicit state should be accepted: %v", err) + } + }) +} + +func TestBuildCreateScheduleOptions(t *testing.T) { + opts, err := buildCreateScheduleOptions(scheduleFlags{name: "n", cron: "* * * * *", timezone: "Europe/Berlin", testSuiteIDs: []string{"s"}}, "caller") + if err != nil { + t.Fatal(err) + } + if opts.Settings.Timezone != "Europe/Berlin" || opts.Settings.RunningUserID != "caller" || opts.StateName != authoring.ScheduleEnabled { + t.Errorf("defaults not applied: %+v", opts) + } + if opts.Settings.MaxRuns != nil { + t.Error("maxRuns must be nil when not given") + } + for _, f := range []scheduleFlags{ + {cron: "* * * * *", timezone: "Europe/Berlin", testSuiteIDs: []string{"s"}}, + {name: "n", timezone: "Europe/Berlin", testSuiteIDs: []string{"s"}}, + {name: "n", cron: "* * * * *", timezone: "Europe/Berlin"}, + {name: "n", cron: "* * * * *", testSuiteIDs: []string{"s"}}, + {name: "n", cron: "* * * * *", timezone: "Europe/Berlin", testSuiteIDs: []string{"s"}, state: "running"}, + } { + if _, err := buildCreateScheduleOptions(f, "caller"); err == nil { + t.Errorf("expected error for %+v", f) + } + } +} + +func TestDeriveFilename(t *testing.T) { + tests := []struct { + target, name, code string + want string + known bool + }{ + {"typescript_playwright", "Login: Add to cart!", "", "login_add_to_cart.spec.ts", true}, + {"javascript_webdriverio", "Login", "", "login.spec.js", true}, + {"python_selenium", "Checkout Flow", "", "test_checkout_flow.py", true}, + {"csharp_selenium", "checkout flow", "", "CheckoutFlow.cs", true}, + {"java_selenium", "whatever", "package x;\npublic class LoginTest {\n}", "LoginTest.java", true}, + {"java_selenium", "whatever", "public final class FinalOne {}", "FinalOne.java", true}, + {"java_selenium", "login flow", "// no class here", "LoginFlow.java", true}, + {"ruby_capybara", "Login", "", "login_spec.rb", true}, + {"kotlin_appium", "login", "", "Login.kt", true}, + {"cobol_thing", "Login", "", "login.txt", false}, + {"python_selenium", " ", "", "test_test.py", true}, + {"python_selenium", "123 go", "", "test_test_123_go.py", true}, + {"csharp_selenium", "9lives", "", "Test9lives.cs", true}, + } + for _, tt := range tests { + got, known := deriveFilename(tt.target, tt.name, tt.code) + if got != tt.want || known != tt.known { + t.Errorf("deriveFilename(%q, %q) = %q, %v; want %q, %v", tt.target, tt.name, got, known, tt.want, tt.known) + } + } +} + +func TestVerifyEntitlement(t *testing.T) { + users := &mocks.UserService{UserFn: func(context.Context) (iam.User, error) { + return iam.User{ID: "u", Organization: iam.Organization{ID: "org"}}, nil + }} + + t.Run("enabled", func(t *testing.T) { + ents := &mocks.AuthoringService{IsAIAuthoringEnabledFn: func(_ context.Context, orgID string) (bool, error) { + if orgID != "org" { + t.Errorf("orgID = %q", orgID) + } + return true, nil + }} + u, err := authoring.VerifyEntitlement(context.Background(), users, ents) + if err != nil || u.ID != "u" { + t.Errorf("got %+v, %v", u, err) + } + }) + + t.Run("not in plan", func(t *testing.T) { + ents := &mocks.AuthoringService{IsAIAuthoringEnabledFn: func(context.Context, string) (bool, error) { return false, nil }} + _, err := authoring.VerifyEntitlement(context.Background(), users, ents) + if !errors.Is(err, authoring.ErrNotEntitled) { + t.Errorf("got %v", err) + } + }) + + t.Run("could not verify is distinct", func(t *testing.T) { + ents := &mocks.AuthoringService{IsAIAuthoringEnabledFn: func(context.Context, string) (bool, error) { return false, errors.New("503") }} + _, err := authoring.VerifyEntitlement(context.Background(), users, ents) + if err == nil || errors.Is(err, authoring.ErrNotEntitled) || !strings.Contains(err.Error(), "could not verify") { + t.Errorf("got %v", err) + } + }) + + t.Run("user lookup failure", func(t *testing.T) { + badUsers := &mocks.UserService{UserFn: func(context.Context) (iam.User, error) { return iam.User{}, errors.New("401") }} + ents := &mocks.AuthoringService{IsAIAuthoringEnabledFn: func(context.Context, string) (bool, error) { + t.Error("entitlement must not be checked without an organisation") + return true, nil + }} + _, err := authoring.VerifyEntitlement(context.Background(), badUsers, ents) + if err == nil || errors.Is(err, authoring.ErrNotEntitled) { + t.Errorf("got %v", err) + } + }) +} + +func TestBuildGenerateOptions(t *testing.T) { + origTerm := stdinIsTerminal + t.Cleanup(func() { stdinIsTerminal = origTerm }) + stdinIsTerminal = func() bool { return false } + + base := generateFlags{name: "n", intent: "do it", kvTargets: []string{"browserName=chrome"}} + + opts, err := buildGenerateOptions(base, strings.NewReader("")) + if err != nil { + t.Fatal(err) + } + if opts.PromptSettings.Intent != "do it" || opts.RunSettings.Target.Capabilities["browserName"] != "chrome" || opts.TimeoutMillis != 0 { + t.Errorf("opts = %+v", opts) + } + + withTimeout := base + withTimeout.generationTimeout = 2 * 60 * 1e9 + opts, err = buildGenerateOptions(withTimeout, strings.NewReader("")) + if err != nil || opts.TimeoutMillis != 120000 { + t.Errorf("timeout ms = %d, err = %v; the service wants milliseconds", opts.TimeoutMillis, err) + } + + fromStdin := generateFlags{name: "n", intentFile: "-", kvTargets: []string{"browserName=chrome"}} + opts, err = buildGenerateOptions(fromStdin, strings.NewReader(" piped intent \n")) + if err != nil || opts.PromptSettings.Intent != "piped intent" { + t.Errorf("stdin intent = %q, err = %v", opts.PromptSettings.Intent, err) + } + + bad := []generateFlags{ + {intent: "x", kvTargets: []string{"browserName=chrome"}}, + {name: "n", kvTargets: []string{"browserName=chrome"}}, + {name: "n", intent: "x", intentFile: "f", kvTargets: []string{"browserName=chrome"}}, + {name: "n", intent: "x"}, + {name: "n", intent: "x", kvTargets: []string{"browserName=chrome", "browserName=firefox"}}, + {name: "n", intent: "x", kvTargets: []string{"browserName=chrome"}, maxSteps: 201}, + {name: "n", intent: "x", kvTargets: []string{"browserName=chrome"}, generationTimeout: 30 * 1e9}, + {name: "n", intent: "x", kvTargets: []string{"browserName=chrome"}, tags: []string{strings.Repeat("x", 61)}}, + } + for i, f := range bad { + if _, err := buildGenerateOptions(f, strings.NewReader("")); err == nil { + t.Errorf("case %d: expected error for %+v", i, f) + } + } +} diff --git a/internal/cmd/authoring/output.go b/internal/cmd/authoring/output.go new file mode 100644 index 000000000..1382d8e80 --- /dev/null +++ b/internal/cmd/authoring/output.go @@ -0,0 +1,104 @@ +package authoring + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "time" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/mattn/go-isatty" + + "github.com/saucelabs/saucectl/internal/tables" +) + +// Output formats selected with -o/--out, matching the tool's other commands. +const ( + JSONOutput = "json" + TextOutput = "text" +) + +// validateOutput rejects an unknown format before any request is made. +func validateOutput(out string) error { + if out != JSONOutput && out != TextOutput { + return fmt.Errorf("unknown output format %q; options: %s, %s", out, TextOutput, JSONOutput) + } + return nil +} + +// renderJSON writes val as one JSON document to stdout, the same way +// `builds list -o json` does. +func renderJSON(val any) error { + return json.NewEncoder(os.Stdout).Encode(val) +} + +// newTable returns a table writer in the shared saucectl style. +func newTable() table.Writer { + t := table.NewWriter() + t.SetStyle(tables.DefaultTableStyle) + t.SuppressEmptyColumns() + return t +} + +// listFooter renders the "showing N of M " footer every listing +// carries, so users always know how many results exist beyond those shown. +func listFooter(shown, total int, resource string) table.Row { + return table.Row{fmt.Sprintf("showing %d of %d %s", shown, total, resource)} +} + +// humanizeDate renders a service timestamp for people: local time without +// sub-second noise. It falls back to the raw value on anything it cannot +// parse, so an unexpected format is a display concern rather than a failure. +func humanizeDate(s string) string { + if s == "" { + return "" + } + t, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + return s + } + return t.Local().Format("2006-01-02 15:04:05") +} + +// isTerm reports whether fd is an interactive terminal, Cygwin included. +func isTerm(fd uintptr) bool { + return isatty.IsTerminal(fd) || isatty.IsCygwinTerminal(fd) +} + +// interactive reports whether both stdin and stdout are terminals, which is +// the precondition for prompting. +func interactive() bool { + return isTerm(os.Stdin.Fd()) && isTerm(os.Stdout.Fd()) +} + +// truncate shortens s to at most n runes, marking the cut with an ellipsis. +func truncate(s string, n int) string { + if n <= 0 { + return "" + } + r := []rune(s) + if len(r) <= n { + return s + } + if n == 1 { + return "…" + } + return string(r[:n-1]) + "…" +} + +// joinOrDash joins strings for a table cell, showing "-" for none. +func joinOrDash(parts []string) string { + if len(parts) == 0 { + return "-" + } + return strings.Join(parts, ", ") +} + +// orDash returns s, or "-" when empty, for table cells. +func orDash(s string) string { + if s == "" { + return "-" + } + return s +} diff --git a/internal/cmd/authoring/paging.go b/internal/cmd/authoring/paging.go new file mode 100644 index 000000000..2cef91282 --- /dev/null +++ b/internal/cmd/authoring/paging.go @@ -0,0 +1,104 @@ +package authoring + +import ( + "context" + "fmt" + + "github.com/rs/zerolog/log" + "github.com/spf13/pflag" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// largeListingThreshold is the total above which --all warns before fetching +// everything: a test case listing weighs ~13 KB per case. +const largeListingThreshold = 200 + +// pageFlags are the pagination flags every listing command shares. They mirror +// the service's skip/limit rather than `builds`' page/size because users of +// the service's own API already think in those terms. +type pageFlags struct { + skip int + limit int + all bool + // limitChanged records whether the user actually set --limit, which is + // the only way to tell an explicit value from the default when deciding + // whether to warn that --all cannot honour it. + limitChanged bool + // jsonOut suppresses advisory warnings. The logger writes to stdout for + // every saucectl command, so a warning emitted while rendering JSON + // lands in the middle of the document and breaks `| jq`. Silence is + // wrong in general (Constitution VIII), which is why this only applies + // to advice the user cannot act on mid-listing. + jsonOut bool +} + +// bind registers the flags. +func (p *pageFlags) bind(fs *pflag.FlagSet) { + fs.IntVar(&p.skip, "skip", 0, "Number of results to skip.") + fs.IntVar(&p.limit, "limit", 20, "Maximum number of results to return. 0 returns only the total count.") + fs.BoolVar(&p.all, "all", false, "Return every result, fetching all pages.") +} + +// capture records flag state that cannot be read from the values alone. +// Call it from RunE, before fetching. +func (p *pageFlags) capture(fs *pflag.FlagSet) { + p.limitChanged = fs.Changed("limit") + out, err := fs.GetString("out") + p.jsonOut = err == nil && out == JSONOutput +} + +// validate rejects negative values before any request. +func (p pageFlags) validate() error { + if p.skip < 0 { + return fmt.Errorf("--skip must not be negative") + } + if p.limit < 0 { + return fmt.Errorf("--limit must not be negative") + } + return nil +} + +// options converts the flags into list options. The limit is always sent +// because the flag always has a value; 0 is the count-only request. +func (p pageFlags) options() authoring.ListOptions { + limit := p.limit + return authoring.ListOptions{Skip: p.skip, Limit: &limit} +} + +// fetchPage fetches one page, or every page under --all, through fetch. It +// returns the items and the total the service reports, so the footer can say +// how many results exist beyond those shown (FR-035). +func fetchPage[T any](ctx context.Context, p pageFlags, resource string, fetch func(context.Context, authoring.ListOptions) (authoring.List[T], error)) ([]T, int, error) { + if !p.all { + l, err := fetch(ctx, p.options()) + return l.Items, l.Total, err + } + + // --all fetches whole pages at a fixed size, so --limit cannot be + // honoured. Say so rather than ignoring it silently (Constitution VIII). + if p.limitChanged && !p.jsonOut { + log.Warn().Msgf("--limit is ignored with --all; every %s is fetched in pages of %d.", resource, authoring.DefaultPageSize) + } + + warned := false + items, err := authoring.ListAll(ctx, authoring.DefaultPageSize, func(ctx context.Context, opts authoring.ListOptions) (authoring.List[T], error) { + // --skip still means "start here": offset every page by it, so + // `--all --skip 100` does not silently restart from the beginning. + opts.Skip += p.skip + l, err := fetch(ctx, opts) + if err == nil && !warned && !p.jsonOut && l.Total > largeListingThreshold { + warned = true + log.Warn().Msgf("Fetching all %d %s; large listings take a while and transfer several megabytes.", l.Total, resource) + } + return l, err + }) + return items, len(items), err +} + +// jobURL derives a job's dashboard link in the region resolved for this +// invocation. The derivation itself lives in internal/authoring so the run +// results table and these tables cannot drift apart. +func jobURL(sauceJobID string) string { + return authoring.JobURL(regio, sauceJobID) +} diff --git a/internal/cmd/authoring/schedules.go b/internal/cmd/authoring/schedules.go new file mode 100644 index 000000000..69e26d696 --- /dev/null +++ b/internal/cmd/authoring/schedules.go @@ -0,0 +1,26 @@ +package authoring + +import "github.com/spf13/cobra" + +// SchedulesCommand is the `authoring schedules` subgroup. No pre-run of its +// own, so the root's runs (see TestCasesCommand). +func SchedulesCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "schedules", + Aliases: []string{"schedule", "test-schedules"}, + Short: "Run test suites on a recurring schedule", + SilenceUsage: true, + } + + cmd.AddCommand( + SchedulesListCommand(), + SchedulesGetCommand(), + SchedulesCreateCommand(), + SchedulesUpdateCommand(), + SchedulesEnableCommand(), + SchedulesDisableCommand(), + SchedulesDeleteCommand(), + ) + + return cmd +} diff --git a/internal/cmd/authoring/schedules_create.go b/internal/cmd/authoring/schedules_create.go new file mode 100644 index 000000000..f57474db0 --- /dev/null +++ b/internal/cmd/authoring/schedules_create.go @@ -0,0 +1,163 @@ +package authoring + +import ( + "errors" + "fmt" + "strings" + + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// scheduleFlags are the settings flags shared by create and update. +type scheduleFlags struct { + name string + cron string + timezone string + runningUserID string + testSuiteIDs []string + state string + startDate string + endDate string + maxRuns int + tunnelName string + build string +} + +// bindScheduleSettingsFlags registers the settings flags. Defaults are left +// empty so update can tell an unset flag from an explicit one via Changed. +func bindScheduleSettingsFlags(cmd *cobra.Command, f *scheduleFlags) { + flags := cmd.Flags() + flags.StringVar(&f.name, "name", "", "Name of the schedule (1–255 characters).") + flags.StringVar(&f.cron, "cron", "", "Six-field cron expression starting with seconds, e.g. \"0 0 6 * * 1-5\" for 06:00 on weekdays.") + flags.StringVar(&f.timezone, "timezone", "", "IANA region/city timezone for the cron expression, e.g. Europe/Berlin or America/New_York. Required on create. The service does not accept \"UTC\"; use a zero-offset zone such as Atlantic/Reykjavik instead.") + flags.StringVar(&f.runningUserID, "running-user-id", "", "User the scheduled runs execute as. Default: you.") + flags.StringVar(&f.state, "state", "", "ENABLED or DISABLED (case-insensitive). Default: ENABLED.") + flags.StringVar(&f.startDate, "start-date", "", "ISO 8601 date before which the schedule does not run.") + flags.StringVar(&f.endDate, "end-date", "", "ISO 8601 date after which the schedule stops.") + flags.IntVar(&f.maxRuns, "max-runs", 0, "Maximum number of runs before the schedule disables itself.") + flags.StringVar(&f.tunnelName, "tunnel-name", "", "Sauce Connect tunnel to route scheduled runs through.") + flags.StringVar(&f.build, "build", "", "Build name for scheduled runs.") + + _ = cmd.RegisterFlagCompletionFunc("state", func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + names := make([]string, len(authoring.SettableScheduleStates)) + for i, s := range authoring.SettableScheduleStates { + names[i] = string(s) + } + return names, cobra.ShellCompDirectiveNoFileComp + }) +} + +// SchedulesCreateCommand is `authoring schedules create`. +func SchedulesCreateCommand() *cobra.Command { + var out string + var f scheduleFlags + + cmd := &cobra.Command{ + Use: "create", + Short: "Create a test schedule", + Example: ` saucectl authoring schedules create --name "Nightly" --cron "0 0 2 * * *" --timezone Europe/Berlin \ + --test-suite-id 3f2a9c1e4b5d6e7f8a9b0c1d2e3f4a5b --build nightly + saucectl authoring schedules create --name "Smoke" --cron "0 */30 * * * *" --timezone America/New_York --test-suite-id 3f2a… --max-runs 10 --state disabled`, + SilenceUsage: true, + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, _ []string) error { + if err := validateOutput(out); err != nil { + return err + } + opts, err := buildCreateScheduleOptions(f, currentUser.ID) + if err != nil { + return err + } + s, err := scheduleService.CreateSchedule(cmd.Context(), opts) + if err != nil { + return fmt.Errorf("failed to create schedule: %w", err) + } + if out == JSONOutput { + return renderJSON(s) + } + fmt.Printf("Created schedule %q (%s), %s, next run %s.\n", s.Name, s.ID, s.State.StateName, orDash(humanizeDate(s.State.NextRunDate))) + return nil + }, + } + + cmd.Flags().StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + cmd.Flags().StringArrayVar(&f.testSuiteIDs, "test-suite-id", nil, "Suite to run. Repeatable; at least one is required.") + bindScheduleSettingsFlags(cmd, &f) + + return cmd +} + +// buildCreateScheduleOptions validates and assembles the create request, +// applying the defaults: running user = the caller, state ENABLED. +func buildCreateScheduleOptions(f scheduleFlags, callerID string) (authoring.CreateScheduleOptions, error) { + var opts authoring.CreateScheduleOptions + if f.name == "" { + return opts, errors.New("--name is required") + } + if f.cron == "" { + return opts, errors.New("--cron is required") + } + if len(f.testSuiteIDs) == 0 { + return opts, errors.New("at least one --test-suite-id is required") + } + // Observed 2026-09-06: the service validates the timezone against an IANA + // region/city list that contains neither "UTC" nor "Etc/UTC", so there is + // no safe default to fall back to. + if f.timezone == "" { + return opts, errors.New("--timezone is required: an IANA region/city zone such as Europe/Berlin (the service does not accept \"UTC\")") + } + // Observed 2026-09-06: the service's accepted list is region/city zones + // only and contained neither of these, so catch them here rather than + // spending a round trip on an INVALID_BODY. Anything else is left to the + // service, which holds the real list; a local tzdata check would not help + // because Go itself accepts "UTC". + switch strings.ToLower(f.timezone) { + case "utc", "etc/utc": + return opts, fmt.Errorf("--timezone %q is not accepted by the service; use an IANA region/city zone, e.g. Atlantic/Reykjavik for a zero offset", f.timezone) + } + if f.maxRuns < 0 { + return opts, errors.New("--max-runs must not be negative") + } + + state := authoring.ScheduleEnabled + if f.state != "" { + parsed, ok := authoring.ParseScheduleState(f.state) + if !ok { + return opts, fmt.Errorf("invalid --state %q; options: ENABLED, DISABLED", f.state) + } + state = parsed + } + + runner := f.runningUserID + if runner == "" { + runner = callerID + } + if runner == "" { + return opts, errors.New("--running-user-id is required: the current user could not be resolved") + } + + settings := authoring.ScheduleSettings{ + Cron: f.cron, + Timezone: f.timezone, + RunningUserID: runner, + StartDate: f.startDate, + EndDate: f.endDate, + TunnelName: f.tunnelName, + BuildName: f.build, + } + if f.maxRuns > 0 { + m := f.maxRuns + settings.MaxRuns = &m + } + + return authoring.CreateScheduleOptions{ + Name: f.name, + Settings: settings, + TestSuiteIDs: f.testSuiteIDs, + StateName: state, + }, nil +} diff --git a/internal/cmd/authoring/schedules_list.go b/internal/cmd/authoring/schedules_list.go new file mode 100644 index 000000000..81bb6eeb3 --- /dev/null +++ b/internal/cmd/authoring/schedules_list.go @@ -0,0 +1,148 @@ +package authoring + +import ( + "context" + "fmt" + "strings" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// SchedulesListCommand is `authoring schedules list`. +func SchedulesListCommand() *cobra.Command { + var out string + var page pageFlags + var opts authoring.ListSchedulesOptions + + cmd := &cobra.Command{ + Use: "list", + Aliases: []string{"ls"}, + Short: "List test schedules", + Example: ` saucectl authoring schedules list + saucectl authoring schedules list --test-suite-id 3f2a9c1e4b5d6e7f8a9b0c1d2e3f4a5b`, + SilenceUsage: true, + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, _ []string) error { + if err := validateOutput(out); err != nil { + return err + } + if err := page.validate(); err != nil { + return err + } + page.capture(cmd.Flags()) + items, total, err := fetchPage(cmd.Context(), page, "schedules", func(ctx context.Context, lo authoring.ListOptions) (authoring.List[authoring.TestSchedule], error) { + opts.ListOptions = lo + return scheduleService.ListSchedules(ctx, opts) + }) + if err != nil { + return fmt.Errorf("failed to list schedules: %w", err) + } + if out == JSONOutput { + return renderJSON(authoring.List[authoring.TestSchedule]{Items: items, Total: total}) + } + renderScheduleTable(items, total) + return nil + }, + } + + flags := cmd.Flags() + flags.StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + flags.StringArrayVar(&opts.IDs, "id", nil, "Only these schedule IDs. Repeatable.") + flags.StringVar(&opts.Search, "search", "", "Case-insensitive substring match on the name.") + flags.StringVar(&opts.StartDate, "start-date", "", "Only schedules created on or after this ISO 8601 date.") + flags.StringVar(&opts.EndDate, "end-date", "", "Only schedules created on or before this ISO 8601 date.") + flags.StringVar(&opts.UserID, "user-id", "", "Filter by creator user ID.") + flags.StringVar(&opts.TeamID, "team-id", "", "Filter by team ID.") + flags.StringArrayVar(&opts.TestSuiteIDs, "test-suite-id", nil, "Only schedules that trigger this suite. Repeatable.") + page.bind(flags) + + return cmd +} + +// renderScheduleTable prints the listing table, or a single line when empty. +func renderScheduleTable(items []authoring.TestSchedule, total int) { + if len(items) == 0 { + fmt.Printf("No schedules found (total: %d).\n", total) + return + } + t := newTable() + t.AppendHeader(table.Row{"ID", "Name", "State", "Cron", "Timezone", "Next Run", "Suites"}) + for _, s := range items { + t.AppendRow(table.Row{s.ID, truncate(s.Name, 40), s.State.StateName, s.Settings.Cron, s.Settings.Timezone, orDash(humanizeDate(s.State.NextRunDate)), len(s.TestSuiteIDs)}) + } + t.AppendFooter(listFooter(len(items), total, "schedules")) + fmt.Println(t.Render()) +} + +// SchedulesGetCommand is `authoring schedules get`. +func SchedulesGetCommand() *cobra.Command { + var out string + + cmd := &cobra.Command{ + Use: "get ", + Short: "Show a test schedule", + Example: ` saucectl authoring schedules get 9c1d…`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(out); err != nil { + return err + } + s, err := scheduleService.GetSchedule(cmd.Context(), args[0]) + if err != nil { + return fmt.Errorf("failed to get schedule: %w", err) + } + if out == JSONOutput { + return renderJSON(s) + } + renderScheduleDetail(s) + return nil + }, + } + + cmd.Flags().StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + + return cmd +} + +// renderScheduleDetail prints the two-column property view. +func renderScheduleDetail(s authoring.TestSchedule) { + maxRuns := "unlimited" + if s.Settings.MaxRuns != nil { + maxRuns = fmt.Sprint(*s.Settings.MaxRuns) + } + remaining := "-" + if s.State.RemainingRuns != nil { + remaining = fmt.Sprint(*s.State.RemainingRuns) + } + + t := newTable() + t.AppendHeader(table.Row{"Property", "Value"}) + t.AppendRow(table.Row{"ID", s.ID}) + t.AppendRow(table.Row{"Name", s.Name}) + t.AppendRow(table.Row{"State", s.State.StateName}) + t.AppendRow(table.Row{"Cron", s.Settings.Cron}) + t.AppendRow(table.Row{"Timezone", s.Settings.Timezone}) + t.AppendRow(table.Row{"Running User", s.Settings.RunningUserID}) + t.AppendRow(table.Row{"Start Date", orDash(s.Settings.StartDate)}) + t.AppendRow(table.Row{"End Date", orDash(s.Settings.EndDate)}) + t.AppendRow(table.Row{"Max Runs", maxRuns}) + t.AppendRow(table.Row{"Remaining Runs", remaining}) + t.AppendRow(table.Row{"Tunnel", orDash(s.Settings.TunnelName)}) + t.AppendRow(table.Row{"Build", orDash(s.Settings.BuildName)}) + t.AppendRow(table.Row{"Suites", strings.Join(s.TestSuiteIDs, "\n")}) + t.AppendRow(table.Row{"Last Run", orDash(humanizeDate(s.State.LastRunDate))}) + t.AppendRow(table.Row{"Last Run Error", orDash(s.State.LastRunError)}) + t.AppendRow(table.Row{"Next Run", orDash(humanizeDate(s.State.NextRunDate))}) + t.AppendRow(table.Row{"Created", fmt.Sprintf("%s by %s", humanizeDate(s.CreationDate), orDash(s.CreatorUserName))}) + t.AppendRow(table.Row{"Updated", fmt.Sprintf("%s by %s", humanizeDate(s.LastUpdateDate), orDash(s.LastModifierUserName))}) + fmt.Println(t.Render()) +} diff --git a/internal/cmd/authoring/schedules_state.go b/internal/cmd/authoring/schedules_state.go new file mode 100644 index 000000000..ce808d684 --- /dev/null +++ b/internal/cmd/authoring/schedules_state.go @@ -0,0 +1,117 @@ +package authoring + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// SchedulesEnableCommand is `authoring schedules enable`, a convenience over +// update that sets the state to ENABLED. +func SchedulesEnableCommand() *cobra.Command { + return scheduleStateCommand("enable", "Resume a suspended schedule", authoring.ScheduleEnabled) +} + +// SchedulesDisableCommand is `authoring schedules disable`, a convenience over +// update that sets the state to DISABLED. No runs are triggered while disabled. +func SchedulesDisableCommand() *cobra.Command { + return scheduleStateCommand("disable", "Suspend a schedule without deleting it", authoring.ScheduleDisabled) +} + +// scheduleStateCommand builds enable/disable. +func scheduleStateCommand(use, short string, state authoring.ScheduleStateName) *cobra.Command { + var out string + + cmd := &cobra.Command{ + Use: use + " ", + Short: short, + Example: fmt.Sprintf(" saucectl authoring schedules %s 9c1d…", use), + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(out); err != nil { + return err + } + // The update endpoint replaces the whole schedule, so even a + // state change is a read-modify-write of the complete object. + current, err := scheduleService.GetSchedule(cmd.Context(), args[0]) + if err != nil { + return fmt.Errorf("failed to get schedule: %w", err) + } + opts, err := buildUpdateScheduleOptions(noFlagChanges{}, scheduleUpdateFlags{scheduleFlags: scheduleFlags{state: string(state)}}, current) + if err != nil { + return err + } + s, err := scheduleService.UpdateSchedule(cmd.Context(), args[0], opts) + if err != nil { + return fmt.Errorf("failed to %s schedule: %w", use, err) + } + if out == JSONOutput { + return renderJSON(s) + } + fmt.Printf("Schedule %q (%s) is now %s.\n", s.Name, s.ID, s.State.StateName) + return nil + }, + } + + cmd.Flags().StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + + return cmd +} + +// SchedulesDeleteCommand is `authoring schedules delete`. +func SchedulesDeleteCommand() *cobra.Command { + var yes bool + + cmd := &cobra.Command{ + Use: "delete ", + Aliases: []string{"rm"}, + Short: "Delete a test schedule", + Example: ` saucectl authoring schedules delete 9c1d… --yes`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return deleteSchedule(cmd.Context(), args[0], yes) + }, + } + + cmd.Flags().BoolVarP(&yes, "yes", "y", false, "Skip the confirmation prompt. Required when not running interactively.") + + return cmd +} + +// deleteSchedule confirms, naming the schedule's state, cadence and suites, +// then deletes. +func deleteSchedule(ctx context.Context, id string, yes bool) error { + s, err := scheduleService.GetSchedule(ctx, id) + if err != nil { + return fmt.Errorf("failed to get schedule: %w", err) + } + + affects := []string{ + fmt.Sprintf("it is %s and runs %q (%s)", s.State.StateName, s.Settings.Cron, s.Settings.Timezone), + fmt.Sprintf("it triggers %d suite(s): %s", len(s.TestSuiteIDs), joinOrDash(s.TestSuiteIDs)), + } + if s.State.NextRunDate != "" && s.State.StateName == authoring.ScheduleEnabled { + affects = append(affects, fmt.Sprintf("the next run at %s will not happen", humanizeDate(s.State.NextRunDate))) + } + + if err := confirmDestructive(yes, fmt.Sprintf("schedule %q (%s)", s.Name, s.ID), affects); err != nil { + return err + } + + if err := scheduleService.DeleteSchedule(ctx, s.ID); err != nil { + return fmt.Errorf("failed to delete schedule: %w", err) + } + fmt.Printf("Deleted schedule %q (%s).\n", s.Name, s.ID) + return nil +} diff --git a/internal/cmd/authoring/schedules_update.go b/internal/cmd/authoring/schedules_update.go new file mode 100644 index 000000000..5e3ab0d8d --- /dev/null +++ b/internal/cmd/authoring/schedules_update.go @@ -0,0 +1,256 @@ +package authoring + +import ( + "errors" + "fmt" + "strings" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// unsettableFields are the settings --unset accepts. Each is cleared by +// sending an explicit null: omitting a field keeps its stored value (observed +// 2026-09-06, research Open-4). +var unsettableFields = []string{"tunnelName", "buildName", "startDate", "endDate", "maxRuns"} + +// unsetConflicts maps each unsettable field to the flag that sets it, so +// asking for both in one command can be refused rather than silently +// resolved in favour of whichever runs last. +var unsetConflicts = map[string]string{ + "tunnelName": "tunnel-name", + "buildName": "build", + "startDate": "start-date", + "endDate": "end-date", + "maxRuns": "max-runs", +} + +// scheduleUpdateFlags are the flags of `schedules update`. +type scheduleUpdateFlags struct { + scheduleFlags + addTestSuiteIDs []string + removeTestSuiteIDs []string + unset []string +} + +// SchedulesUpdateCommand is `authoring schedules update`. +func SchedulesUpdateCommand() *cobra.Command { + var out string + var f scheduleUpdateFlags + + cmd := &cobra.Command{ + Use: "update ", + Short: "Update a test schedule", + Long: `Update a test schedule. Only the given flags change; the schedule is read first and its +current settings, suites and state are re-sent in full, because the service replaces the +schedule rather than merging changes. An update therefore never erases what it does not +mention. + +--unset clears an optional setting explicitly: tunnelName, buildName, startDate, endDate or +maxRuns. --test-suite-id replaces the suites wholesale; --add-test-suite-id and +--remove-test-suite-id change them incrementally.`, + Example: ` saucectl authoring schedules update 9c1d… --cron "0 0 3 * * *" + saucectl authoring schedules update 9c1d… --add-test-suite-id 3f2a… --unset maxRuns + saucectl authoring schedules update 9c1d… --unset tunnelName`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(out); err != nil { + return err + } + current, err := scheduleService.GetSchedule(cmd.Context(), args[0]) + if err != nil { + return fmt.Errorf("failed to get schedule: %w", err) + } + opts, err := buildUpdateScheduleOptions(cmd.Flags(), f, current) + if err != nil { + return err + } + s, err := scheduleService.UpdateSchedule(cmd.Context(), args[0], opts) + if err != nil { + return fmt.Errorf("failed to update schedule: %w", err) + } + if out == JSONOutput { + return renderJSON(s) + } + fmt.Printf("Updated schedule %q (%s), %s, next run %s.\n", s.Name, s.ID, s.State.StateName, orDash(humanizeDate(s.State.NextRunDate))) + return nil + }, + } + + cmd.Flags().StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + cmd.Flags().StringArrayVar(&f.testSuiteIDs, "test-suite-id", nil, "Replace the suites with these. Repeatable.") + cmd.Flags().StringArrayVar(&f.addTestSuiteIDs, "add-test-suite-id", nil, "Add this suite. Repeatable.") + cmd.Flags().StringArrayVar(&f.removeTestSuiteIDs, "remove-test-suite-id", nil, "Remove this suite. Repeatable.") + cmd.Flags().StringArrayVar(&f.unset, "unset", nil, "Clear a setting: "+strings.Join(unsettableFields, ", ")+". Repeatable.") + bindScheduleSettingsFlags(cmd, &f.scheduleFlags) + + return cmd +} + +// noFlagChanges reports no flag as changed; used by enable/disable, which +// change only the state. +type noFlagChanges struct{} + +// Changed always returns false. +func (noFlagChanges) Changed(string) bool { return false } + +// buildUpdateScheduleOptions overlays the changed flags on the current +// schedule and produces the complete object the service requires. +// +// Observed 2026-09-06 (research Open-4, resolved): a partial body such as +// {"settings":{"cron":"…"}} is rejected with INVALID_BODY naming every +// missing required field — name, settings.timezone, settings.runningUserId, +// testSuiteIds and stateName — while omitted *optional* fields keep their +// stored values and an explicit null clears them. So the name, a settings +// object carrying the required fields, the full suite list and the state are +// always sent, --unset sends null, and --add-test-suite-id / +// --remove-test-suite-id are applied here rather than delegated. changed reports +// which flags the user set; it is the flag set in production and a stub in +// tests. +func buildUpdateScheduleOptions(changed interface{ Changed(string) bool }, f scheduleUpdateFlags, current authoring.TestSchedule) (authoring.UpdateScheduleOptions, error) { + var opts authoring.UpdateScheduleOptions + + if f.testSuiteIDs != nil && (f.addTestSuiteIDs != nil || f.removeTestSuiteIDs != nil) { + return opts, errors.New("--test-suite-id cannot be combined with --add-test-suite-id or --remove-test-suite-id") + } + + settings := authoring.PatchFromSettings(current.Settings) + touched := false + touch := func() { touched = true } + + if changed.Changed("cron") { + settings.Cron = f.cron + touch() + } + if changed.Changed("timezone") { + settings.Timezone = f.timezone + touch() + } + if changed.Changed("running-user-id") { + settings.RunningUserID = f.runningUserID + touch() + } + if changed.Changed("start-date") { + v := f.startDate + settings.StartDate = &v + touch() + } + if changed.Changed("end-date") { + v := f.endDate + settings.EndDate = &v + touch() + } + if changed.Changed("max-runs") { + if f.maxRuns < 0 { + return opts, errors.New("--max-runs must not be negative") + } + m := f.maxRuns + settings.MaxRuns = &m + settings.ClearMaxRuns = false + touch() + } + if changed.Changed("tunnel-name") { + v := f.tunnelName + settings.TunnelName = &v + touch() + } + if changed.Changed("build") { + v := f.build + settings.BuildName = &v + touch() + } + + // Setting and clearing the same field in one invocation is ambiguous. + // The unset loop runs last, so it used to win silently; say so instead. + for _, field := range f.unset { + if flag, ok := unsetConflicts[field]; ok && changed.Changed(flag) { + return opts, fmt.Errorf("--%s and --unset %s conflict: pick one", flag, field) + } + } + + for _, field := range f.unset { + empty := "" + switch field { + case "tunnelName": + settings.TunnelName = &empty + case "buildName": + settings.BuildName = &empty + case "startDate": + settings.StartDate = &empty + case "endDate": + settings.EndDate = &empty + case "maxRuns": + settings.MaxRuns = nil + settings.ClearMaxRuns = true + default: + return opts, fmt.Errorf("cannot unset %q; options: %s", field, strings.Join(unsettableFields, ", ")) + } + touch() + } + + name := current.Name + if changed.Changed("name") { + name = f.name + touch() + } + + state := current.State.StateName + if f.state != "" { + parsed, ok := authoring.ParseScheduleState(f.state) + if !ok { + return opts, fmt.Errorf("invalid --state %q; options: ENABLED, DISABLED", f.state) + } + state = parsed + touch() + } else if _, settable := authoring.ParseScheduleState(string(state)); !settable { + // RUNNING and ERRORED are observed states the service will not accept + // back. Rather than guess, make the user choose. + return opts, fmt.Errorf("schedule is currently %s; pass --state ENABLED or --state DISABLED to update it", state) + } + + suites := append([]string(nil), current.TestSuiteIDs...) + if f.testSuiteIDs != nil { + suites = append([]string(nil), f.testSuiteIDs...) + touch() + } + for _, id := range f.addTestSuiteIDs { + if !contains(suites, id) { + suites = append(suites, id) + } + touch() + } + for _, id := range f.removeTestSuiteIDs { + kept := suites[:0] + for _, s := range suites { + if s != id { + kept = append(kept, s) + } + } + suites = kept + touch() + } + if len(suites) == 0 { + return opts, errors.New("a schedule must keep at least one test suite") + } + + if !touched { + return opts, authoring.ErrEmptyUpdate + } + + return authoring.UpdateScheduleOptions{ + Name: name, + Settings: &settings, + TestSuiteIDs: suites, + StateName: state, + }, nil +} + +// Compile-time assertion that a pflag.FlagSet satisfies the Changed interface +// buildUpdateScheduleOptions takes. +var _ interface{ Changed(string) bool } = (*pflag.FlagSet)(nil) diff --git a/internal/cmd/authoring/targets.go b/internal/cmd/authoring/targets.go new file mode 100644 index 000000000..46d77b0eb --- /dev/null +++ b/internal/cmd/authoring/targets.go @@ -0,0 +1,171 @@ +package authoring + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "strings" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// ErrNoTargets is returned by commands that require at least one target. +var ErrNoTargets = errors.New("no target specified; use --target or --target-json") + +// parseTargets turns the two target flag forms into run targets. +// +// --target takes comma-separated key=value pairs, one flag per target: +// +// --target browserName=chrome,platformName="Windows 11",browserVersion=latest +// +// Dotted keys nest ("sauce:options.screenResolution=1920x1080"); the values +// "true" and "false" become booleans, everything else stays a string, since +// capability versions like "16" are strings on the wire. +// +// --target-json takes a JSON object, or @path to read one from a file. The +// object is either a capabilities object or a full target +// {"capabilities": {...}, "isRdc": true}. +func parseTargets(kvTargets, jsonTargets []string) ([]authoring.Target, error) { + var targets []authoring.Target + + for _, kv := range kvTargets { + t, err := parseKVTarget(kv) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + + for _, raw := range jsonTargets { + t, err := parseJSONTarget(raw) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + + return targets, nil +} + +// parseKVTarget parses one --target value. +func parseKVTarget(kv string) (authoring.Target, error) { + caps := map[string]any{} + for _, pair := range splitPairs(kv) { + pair = strings.TrimSpace(pair) + if pair == "" { + continue + } + key, value, ok := strings.Cut(pair, "=") + key = strings.TrimSpace(key) + if !ok || key == "" { + return authoring.Target{}, fmt.Errorf("invalid --target entry %q: expected key=value", pair) + } + setNested(caps, strings.Split(key, "."), coerce(strings.TrimSpace(value))) + } + if len(caps) == 0 { + return authoring.Target{}, fmt.Errorf("invalid --target %q: no capabilities", kv) + } + return authoring.Target{Capabilities: caps}, nil +} + +// splitPairs splits on commas that are not inside double quotes, so a value +// like "Windows 11" may be quoted to protect spaces and commas. Quotes are +// stripped from the result. +func splitPairs(s string) []string { + var parts []string + var b strings.Builder + inQuote := false + for _, r := range s { + switch { + case r == '"': + inQuote = !inQuote + case r == ',' && !inQuote: + parts = append(parts, b.String()) + b.Reset() + default: + b.WriteRune(r) + } + } + parts = append(parts, b.String()) + return parts +} + +// coerce turns the literal booleans into bool and leaves everything else a +// string. +func coerce(v string) any { + switch strings.ToLower(v) { + case "true": + return true + case "false": + return false + default: + return v + } +} + +// setNested assigns value at the dotted path inside m, creating maps as it +// goes. A conflicting scalar on the path is replaced by a map. +func setNested(m map[string]any, path []string, value any) { + for i, key := range path { + if i == len(path)-1 { + m[key] = value + return + } + next, ok := m[key].(map[string]any) + if !ok { + next = map[string]any{} + m[key] = next + } + m = next + } +} + +// parseJSONTarget parses one --target-json value, reading from a file when +// the value starts with @. +func parseJSONTarget(raw string) (authoring.Target, error) { + if strings.HasPrefix(raw, "@") { + b, err := os.ReadFile(strings.TrimPrefix(raw, "@")) + if err != nil { + return authoring.Target{}, fmt.Errorf("reading --target-json file: %w", err) + } + raw = string(b) + } + + var obj map[string]any + if err := json.Unmarshal([]byte(raw), &obj); err != nil { + return authoring.Target{}, fmt.Errorf("invalid --target-json: %w", err) + } + if len(obj) == 0 { + return authoring.Target{}, errors.New("invalid --target-json: empty object") + } + + if capsRaw, ok := obj["capabilities"]; ok { + caps, ok := capsRaw.(map[string]any) + if !ok { + return authoring.Target{}, errors.New("invalid --target-json: capabilities must be an object") + } + isRDC, _ := obj["isRdc"].(bool) + return authoring.Target{Capabilities: caps, IsRDC: isRDC}, nil + } + return authoring.Target{Capabilities: obj}, nil +} + +// describeTarget renders a target as a short label for tables and logs, +// e.g. "chrome latest / Windows 11" or "Google Pixel 9 Emulator / Android 16". +// The capability lookup itself is shared with the runner's results table via +// authoring.DescribeCapabilities, so both describe a target the same way. +func describeTarget(t authoring.Target) string { + browser, platform, device := authoring.DescribeCapabilities(t.Capabilities) + + var parts []string + for _, p := range []string{device, browser, platform} { + if p != "" { + parts = append(parts, p) + } + } + if len(parts) == 0 { + return "-" + } + return strings.Join(parts, " / ") +} diff --git a/internal/cmd/authoring/testcases.go b/internal/cmd/authoring/testcases.go new file mode 100644 index 000000000..0f161de03 --- /dev/null +++ b/internal/cmd/authoring/testcases.go @@ -0,0 +1,33 @@ +package authoring + +import "github.com/spf13/cobra" + +// TestCasesCommand is the `authoring testcases` subgroup. It defines no +// pre-run of its own: cobra runs only the closest PersistentPreRun in the +// chain, so omitting it here lets the root's setup and entitlement gate run +// for every descendant. +func TestCasesCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "testcases", + Aliases: []string{"testcase", "tc"}, + Short: "Inspect, run, author and export AI-authored test cases", + SilenceUsage: true, + } + + cmd.AddCommand( + TestCasesListCommand(), + TestCasesGetCommand(), + TestCasesDeleteCommand(), + TestCasesRenameCommand(), + TestCasesRunCommand(), + TestCasesListRunsCommand(), + TestCasesGetRunCommand(), + TestCasesListTagsCommand(), + TestCasesGenerateCommand(), + TestCasesGenerateStatusCommand(), + TestCasesCodeCommand(), + TestCasesListCodeTargetsCommand(), + ) + + return cmd +} diff --git a/internal/cmd/authoring/testcases_code.go b/internal/cmd/authoring/testcases_code.go new file mode 100644 index 000000000..322eb2bcd --- /dev/null +++ b/internal/cmd/authoring/testcases_code.go @@ -0,0 +1,261 @@ +package authoring + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "unicode" + + "github.com/AlecAivazis/survey/v2" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// codeFlags are the flags of `testcases code`. +type codeFlags struct { + out string + target string + filename string + targetDir string + force bool +} + +// TestCasesCodeCommand is `authoring testcases code`: export a test case as +// source code in a chosen language and framework. +func TestCasesCodeCommand() *cobra.Command { + var f codeFlags + + cmd := &cobra.Command{ + Use: "code ", + Short: "Export a test case as source code", + Long: `Export the latest revision of a test case as source code for a language and framework. + +Use 'list-code-targets' to see the targets available for a test case. Without --target, an +interactive session prompts for one. By default the source is printed to standard output +so it can be redirected; -f writes to a file and -d writes into a directory with a +filename derived from the target's language. Existing files are never overwritten +without --force.`, + Example: ` saucectl authoring testcases code 6a882c1dc8b4482c166e96c9 --target typescript_playwright > login.spec.ts + saucectl authoring testcases code 6a882c1dc8b4482c166e96c9 --target python_selenium -d tests/ + saucectl authoring testcases code 6a882c1dc8b4482c166e96c9 --target java_selenium -f LoginTest.java --force`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(f.out); err != nil { + return err + } + if f.filename != "" && f.targetDir != "" { + return errors.New("-f/--filename and -d/--target-dir are mutually exclusive") + } + return exportCode(cmd.Context(), args[0], f) + }, + } + + flags := cmd.Flags() + flags.StringVarP(&f.out, "out", "o", TextOutput, "Output format when printing to standard output. Options: text (raw source), json.") + flags.StringVar(&f.target, "target", "", "Code generation target, e.g. typescript_playwright. See list-code-targets.") + flags.StringVarP(&f.filename, "filename", "f", "", "Write the source to this file.") + flags.StringVarP(&f.targetDir, "target-dir", "d", "", "Write the source into this directory, deriving the filename from the target.") + flags.BoolVar(&f.force, "force", false, "Overwrite an existing file.") + + _ = cmd.RegisterFlagCompletionFunc("target", func(cmd *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) { + if len(args) == 0 || testCaseService == nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + targets, err := testCaseService.CodeTargets(cmd.Context(), args[0]) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + return targets, cobra.ShellCompDirectiveNoFileComp + }) + + return cmd +} + +// exportCode resolves the target, fetches the source and writes it where +// asked. Available targets are always resolved first: one cheap request buys +// a precise error instead of an opaque CODE_GENERATION_TARGET_NOT_FOUND. +func exportCode(ctx context.Context, id string, f codeFlags) error { + targets, err := testCaseService.CodeTargets(ctx, id) + if err != nil { + return fmt.Errorf("failed to list code targets: %w", err) + } + if len(targets) == 0 { + return fmt.Errorf("test case %s has no code export targets", id) + } + + target := f.target + if target == "" { + if !interactive() { + return fmt.Errorf("--target is required; available: %s", strings.Join(targets, ", ")) + } + if err := survey.AskOne(&survey.Select{Message: "Export target:", Options: targets}, &target); err != nil { + return err + } + } + if !contains(targets, target) { + return fmt.Errorf("target %q is not available for this test case; available: %s", target, strings.Join(targets, ", ")) + } + + code, err := testCaseService.Code(ctx, id, target) + if err != nil { + if errors.Is(err, authoring.ErrTestCaseEmpty) { + return fmt.Errorf("test case %s has no steps to export", id) + } + return fmt.Errorf("failed to export code: %w", err) + } + if strings.TrimSpace(code) == "" { + return fmt.Errorf("test case %s produced no source: it is empty", id) + } + + if f.filename == "" && f.targetDir == "" { + if f.out == JSONOutput { + return renderJSON(struct { + Target string `json:"target"` + Code string `json:"code"` + }{Target: target, Code: code}) + } + fmt.Print(code) + if !strings.HasSuffix(code, "\n") { + fmt.Println() + } + return nil + } + + dest := f.filename + if dest == "" { + tc, err := testCaseService.GetTestCase(ctx, id) + if err != nil { + return fmt.Errorf("failed to get test case for naming the file: %w", err) + } + name, known := deriveFilename(target, tc.Name, code) + if !known { + log.Warn().Msgf("Unknown language prefix in target %q; using %s. Use -f to choose a filename.", target, name) + } + dest = filepath.Join(f.targetDir, name) + } + + if !f.force { + if _, err := os.Stat(dest); err == nil { + return fmt.Errorf("%s already exists; use --force to overwrite", dest) + } + } + if dir := filepath.Dir(dest); dir != "" { + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("failed to create %s: %w", dir, err) + } + } + if err := os.WriteFile(dest, []byte(code), 0o644); err != nil { + return fmt.Errorf("failed to write %s: %w", dest, err) + } + fmt.Printf("Wrote %d bytes to %s (%s).\n", len(code), dest, target) + return nil +} + +// contains reports whether list holds s. +func contains(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} + +// javaClassPattern finds the public class a Java file must be named after. +var javaClassPattern = regexp.MustCompile(`public\s+(?:final\s+)?class\s+([A-Za-z_$][A-Za-z0-9_$]*)`) + +// deriveFilename picks a filename for the export from the target's language +// prefix (the part before the first underscore), so targets added later +// still get a sensible name. For Java the public class name is taken from the +// source, because javac requires the filename to match it. The boolean is +// false when the prefix is unknown and a generic name was used. +func deriveFilename(target, testName, code string) (string, bool) { + prefix, _, _ := strings.Cut(target, "_") + snake := snakeCase(testName) + pascal := pascalCase(testName) + + switch strings.ToLower(prefix) { + case "typescript": + return snake + ".spec.ts", true + case "javascript": + return snake + ".spec.js", true + case "python": + return "test_" + snake + ".py", true + case "csharp": + return pascal + ".cs", true + case "java": + if m := javaClassPattern.FindStringSubmatch(code); len(m) == 2 { + return m[1] + ".java", true + } + return pascal + ".java", true + case "ruby": + return snake + "_spec.rb", true + case "kotlin": + return pascal + ".kt", true + default: + return snake + ".txt", false + } +} + +// snakeCase lowercases and replaces runs of non-alphanumerics with one +// underscore; an empty result becomes "test". +func snakeCase(s string) string { + var b strings.Builder + lastUnderscore := true + for _, r := range strings.ToLower(s) { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + b.WriteRune(r) + lastUnderscore = false + continue + } + if !lastUnderscore { + b.WriteRune('_') + lastUnderscore = true + } + } + out := strings.Trim(b.String(), "_") + if out == "" { + return "test" + } + if unicode.IsDigit([]rune(out)[0]) { + out = "test_" + out + } + return out +} + +// pascalCase capitalises each alphanumeric run and drops separators; an +// empty result becomes "Test". +func pascalCase(s string) string { + var b strings.Builder + upperNext := true + for _, r := range s { + if !unicode.IsLetter(r) && !unicode.IsDigit(r) { + upperNext = true + continue + } + if upperNext { + b.WriteRune(unicode.ToUpper(r)) + upperNext = false + } else { + b.WriteRune(r) + } + } + out := b.String() + if out == "" { + return "Test" + } + if unicode.IsDigit([]rune(out)[0]) { + out = "Test" + out + } + return out +} diff --git a/internal/cmd/authoring/testcases_codetargets.go b/internal/cmd/authoring/testcases_codetargets.go new file mode 100644 index 000000000..feeed4bc9 --- /dev/null +++ b/internal/cmd/authoring/testcases_codetargets.go @@ -0,0 +1,51 @@ +package authoring + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// TestCasesListCodeTargetsCommand is `authoring testcases list-code-targets`. +func TestCasesListCodeTargetsCommand() *cobra.Command { + var out string + + cmd := &cobra.Command{ + Use: "list-code-targets ", + Aliases: []string{"code-targets"}, + Short: "List the languages and frameworks a test case can be exported to", + Example: ` saucectl authoring testcases list-code-targets 6a882c1dc8b4482c166e96c9`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(out); err != nil { + return err + } + targets, err := testCaseService.CodeTargets(cmd.Context(), args[0]) + if err != nil { + return fmt.Errorf("failed to list code targets: %w", err) + } + if out == JSONOutput { + if targets == nil { + targets = []string{} + } + return renderJSON(targets) + } + if len(targets) == 0 { + fmt.Println("No code targets available for this test case.") + return nil + } + for _, t := range targets { + fmt.Println(t) + } + return nil + }, + } + + cmd.Flags().StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + + return cmd +} diff --git a/internal/cmd/authoring/testcases_delete.go b/internal/cmd/authoring/testcases_delete.go new file mode 100644 index 000000000..de9dd7e28 --- /dev/null +++ b/internal/cmd/authoring/testcases_delete.go @@ -0,0 +1,76 @@ +package authoring + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// TestCasesDeleteCommand is `authoring testcases delete`. +func TestCasesDeleteCommand() *cobra.Command { + var yes bool + + cmd := &cobra.Command{ + Use: "delete ", + Aliases: []string{"rm"}, + Short: "Delete a test case", + Example: ` saucectl authoring testcases delete 6a882c1dc8b4482c166e96c9 --yes`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return deleteTestCase(cmd.Context(), args[0], yes) + }, + } + + cmd.Flags().BoolVarP(&yes, "yes", "y", false, "Skip the confirmation prompt. Required when not running interactively.") + + return cmd +} + +// deleteTestCase confirms, then deletes. The confirmation names the case and +// what depends on it (FR-038): its suite membership, its revisions and the +// run history that will be orphaned. +func deleteTestCase(ctx context.Context, id string, yes bool) error { + tc, err := testCaseService.GetTestCase(ctx, id) + if err != nil { + return fmt.Errorf("failed to get test case: %w", err) + } + + // These lookups exist only to fill the prompt. With --yes there is no + // prompt, so issuing them would add a wasted round trip per asset on the + // scripted path, on top of the entitlement gate's two. + var affects []string + if !yes && tc.TestSuiteID != "" { + affects = append(affects, fmt.Sprintf("it will be removed from suite %s", tc.TestSuiteID)) + } + // Observed 2026-09-06: run history outlives the test case. Runs of a + // deleted case still come back from the filtered list and from the run + // detail endpoint, so they are not lost — only orphaned. + zero := 0 + runs := authoring.List[authoring.Run]{} + if !yes { + runs, err = testCaseService.ListRuns(ctx, tc.ID, authoring.ListRunsOptions{ListOptions: authoring.ListOptions{Limit: &zero}}) + } + if !yes && err == nil && runs.Total > 0 { + affects = append(affects, fmt.Sprintf("its %d recorded run(s) stay in run history but will belong to a test case that no longer exists", runs.Total)) + } + if !yes && len(tc.Revisions) > 1 { + affects = append(affects, fmt.Sprintf("all %d revisions are deleted", len(tc.Revisions))) + } + + if err := confirmDestructive(yes, fmt.Sprintf("test case %q (%s)", tc.Name, tc.ID), affects); err != nil { + return err + } + + if err := testCaseService.DeleteTestCase(ctx, tc.ID); err != nil { + return fmt.Errorf("failed to delete test case: %w", err) + } + fmt.Printf("Deleted test case %q (%s).\n", tc.Name, tc.ID) + return nil +} diff --git a/internal/cmd/authoring/testcases_generate.go b/internal/cmd/authoring/testcases_generate.go new file mode 100644 index 000000000..b34d87ce3 --- /dev/null +++ b/internal/cmd/authoring/testcases_generate.go @@ -0,0 +1,442 @@ +package authoring + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + "unicode/utf8" + + "github.com/briandowns/spinner" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// Generation timing defaults. The service recommends polling every 2–3 s +// while a task is queued or in progress. defaultWaitTimeout is what bounds +// the wait when neither --wait-timeout nor --generation-timeout is given: the +// service's own one-hour maximum plus a margin. +const ( + defaultGenerationPoll = 3 * time.Second + waitTimeoutMargin = 2 * time.Minute + defaultWaitTimeout = time.Hour + waitTimeoutMargin + minGenerationTimeout = time.Minute + maxGenerationTimeout = time.Hour + // maxTransientPollWindow bounds how long consecutive transient poll + // failures are tolerated before the last one is reported. Propagation + // lag lasts seconds; a minute of unbroken failure is the service telling + // us something. Bounding it here rather than relying on the caller's + // deadline keeps the promise that nothing waits for ever + // (Constitution VIII) even if a caller forgets to set one. + maxTransientPollWindow = time.Minute +) + +// ErrGenerationStillRunning is returned when the wait ends (interrupt or +// timeout) before the task does. The command prints how to reattach. +var ErrGenerationStillRunning = errors.New("generation is still running on Sauce Labs") + +// generateFlags are the flags of `testcases generate`. +type generateFlags struct { + out string + name string + intent string + intentFile string + maxSteps int + testSuiteID string + tags []string + kvTargets []string + jsonTargets []string + testURL string + tunnelName string + generationTimeout time.Duration + wait bool + pollInterval time.Duration + waitTimeout time.Duration +} + +// TestCasesGenerateCommand is `authoring testcases generate`: author a test +// case from a plain-language description. +func TestCasesGenerateCommand() *cobra.Command { + var f generateFlags + + cmd := &cobra.Command{ + Use: "generate", + Short: "Author a new test case from a plain-language description", + Long: `Author a new test case: describe the journey, name the starting URL and a target, and an +AI agent drives a real browser to work out the steps, saving the result as a test case. + +Authoring is asynchronous. Without --wait the command prints the task ID and returns; with +--wait it streams each attempted action as it happens. An interrupted or timed-out wait +leaves the task running on Sauce Labs; reattach with 'testcases generate-status '. + +Three timeouts apply. --generation-timeout is the budget given to the service (1m–1h). +--wait-timeout bounds how long this command watches; when 0 it derives from +--generation-timeout plus two minutes, or defaults to 1h2m. Individual requests have +their own short timeout.`, + Example: ` saucectl authoring testcases generate --name "Login" \ + --intent "Log in as standard_user and verify the inventory page loads" \ + --test-url https://www.saucedemo.com --target browserName=chrome,platformName="Windows 11" --wait + saucectl authoring testcases generate --name "Checkout" --intent-file checkout.txt \ + --target-json @pixel9.json --tag mobile --generation-timeout 15m`, + SilenceUsage: true, + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, _ []string) error { + return runGenerate(cmd.Context(), f, os.Stdin, os.Stdout) + }, + } + + flags := cmd.Flags() + flags.StringVarP(&f.out, "out", "o", TextOutput, "Output format. Options: text, json. With json, progress is not streamed; one final object is printed.") + flags.StringVar(&f.name, "name", "", "Name of the new test case (1–255 characters). Required.") + flags.StringVar(&f.intent, "intent", "", "Plain-language description of what the test should do (1–20000 characters).") + flags.StringVar(&f.intentFile, "intent-file", "", "Read the description from a file, or - for standard input.") + flags.IntVar(&f.maxSteps, "max-steps", 0, "Maximum number of actions the agent may take (1–200). Default: service default.") + flags.StringVar(&f.testSuiteID, "test-suite-id", "", "Assign the new test case to this suite.") + flags.StringArrayVar(&f.tags, "tag", nil, "Tag for the new test case (at most 20, each at most 60 characters). Repeatable.") + flags.StringArrayVar(&f.kvTargets, "target", nil, "Target capabilities as key=value pairs, e.g. browserName=chrome,platformName=\"Windows 11\".") + flags.StringArrayVar(&f.jsonTargets, "target-json", nil, "Target capabilities as a JSON object, or @path to a file.") + flags.StringVar(&f.testURL, "test-url", "", "Starting URL (at most 2048 characters).") + flags.StringVar(&f.tunnelName, "tunnel-name", "", "Name of an active Sauce Connect tunnel to route the session through.") + flags.DurationVar(&f.generationTimeout, "generation-timeout", 0, "Service-side authoring budget, 1m to 1h. Default: service default.") + flags.BoolVar(&f.wait, "wait", false, "Wait for authoring to finish, streaming each action as it happens.") + flags.DurationVar(&f.pollInterval, "poll-interval", defaultGenerationPoll, "How often to poll while waiting.") + flags.DurationVar(&f.waitTimeout, "wait-timeout", 0, "How long to wait before giving up locally. 0 derives it from --generation-timeout.") + + return cmd +} + +// runGenerate validates the flags, starts the task and optionally waits. +func runGenerate(ctx context.Context, f generateFlags, stdin io.Reader, stdout io.Writer) error { + if err := validateOutput(f.out); err != nil { + return err + } + opts, err := buildGenerateOptions(f, stdin) + if err != nil { + return err + } + + task, err := testCaseService.Generate(ctx, opts) + if err != nil { + return fmt.Errorf("failed to start generation: %w", err) + } + + if !f.wait { + if f.out == JSONOutput { + return renderJSON(task) + } + fmt.Fprintf(stdout, "Generation task accepted.\n Task ID: %s\n Sauce job ID: %s\n\nCheck progress with: saucectl authoring testcases generate-status %s --wait\n", task.TaskID, task.SauceJobID, task.TaskID) + return nil + } + + if f.out == TextOutput { + fmt.Fprintf(stdout, "Generation task accepted.\n Task ID: %s\n Sauce job ID: %s\n\n", task.TaskID, task.SauceJobID) + } + + waitTimeout := f.waitTimeout + if waitTimeout <= 0 { + waitTimeout = defaultWaitTimeout + if f.generationTimeout > 0 { + waitTimeout = f.generationTimeout + waitTimeoutMargin + } + } + return waitForGeneration(ctx, task.TaskID, f.pollInterval, waitTimeout, f.out, stdout) +} + +// buildGenerateOptions turns flags into the request, validating the bounds +// the service enforces so the user gets a precise message instead of +// INVALID_BODY. +func buildGenerateOptions(f generateFlags, stdin io.Reader) (authoring.GenerateOptions, error) { + var opts authoring.GenerateOptions + + if strings.TrimSpace(f.name) == "" { + return opts, errors.New("--name is required") + } + if utf8.RuneCountInString(f.name) > 255 { + return opts, errors.New("--name must be at most 255 characters") + } + + intent, err := readIntent(f.intent, f.intentFile, stdin) + if err != nil { + return opts, err + } + + if f.maxSteps < 0 || f.maxSteps > 200 { + return opts, errors.New("--max-steps must be between 1 and 200") + } + if len(f.tags) > 20 { + return opts, errors.New("at most 20 tags are allowed") + } + for _, tag := range f.tags { + if utf8.RuneCountInString(tag) > 60 { + return opts, fmt.Errorf("tag %q exceeds 60 characters", tag) + } + } + if utf8.RuneCountInString(f.testURL) > 2048 { + return opts, errors.New("--test-url must be at most 2048 characters") + } + if f.generationTimeout != 0 && (f.generationTimeout < minGenerationTimeout || f.generationTimeout > maxGenerationTimeout) { + return opts, errors.New("--generation-timeout must be between 1m and 1h") + } + + targets, err := parseTargets(f.kvTargets, f.jsonTargets) + if err != nil { + return opts, err + } + if len(targets) == 0 { + return opts, ErrNoTargets + } + if len(targets) > 1 { + return opts, errors.New("authoring accepts exactly one target") + } + + opts = authoring.GenerateOptions{ + Name: f.name, + TestSuiteID: f.testSuiteID, + Tags: f.tags, + RunSettings: authoring.GenerateRunSettings{ + Target: authoring.Target{Capabilities: targets[0].Capabilities}, + TestURL: f.testURL, + TunnelName: f.tunnelName, + }, + PromptSettings: authoring.PromptSettings{ + Intent: intent, + MaxSteps: f.maxSteps, + }, + TimeoutMillis: int(f.generationTimeout / time.Millisecond), + } + return opts, nil +} + +// readIntent resolves --intent / --intent-file, which are mutually exclusive. +func readIntent(intent, intentFile string, stdin io.Reader) (string, error) { + if intent != "" && intentFile != "" { + return "", errors.New("--intent and --intent-file are mutually exclusive") + } + if intentFile != "" { + var b []byte + var err error + if intentFile == "-" { + if stdinIsTerminal() { + return "", errors.New("--intent-file - reads standard input, but standard input is a terminal") + } + b, err = io.ReadAll(stdin) + } else { + b, err = os.ReadFile(intentFile) + } + if err != nil { + return "", fmt.Errorf("reading intent: %w", err) + } + intent = strings.TrimSpace(string(b)) + } + if intent == "" { + return "", errors.New("an intent is required: use --intent or --intent-file") + } + if utf8.RuneCountInString(intent) > 20000 { + return "", errors.New("the intent must be at most 20000 characters") + } + return intent, nil +} + +// waitForGeneration watches a task until it ends or the wait is bounded, +// then reports the outcome. It never calls os.Exit: a failure is an error the +// root command turns into a non-zero status (FR-029). +func waitForGeneration(ctx context.Context, taskID string, interval, timeout time.Duration, out string, stdout io.Writer) error { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + render := io.Writer(stdout) + if out == JSONOutput { + render = io.Discard + } + + state, err := watchGeneration(ctx, testCaseService, taskID, interval, render, isTerm(os.Stdout.Fd()) && out == TextOutput) + if err != nil { + // Only an abandoned wait means the task is still running. A fatal + // error — the task does not exist, the credentials no longer work — + // must not tell the user to keep polling. Either way the cause is + // wrapped with %w so callers can match the service's sentinels. + if isAbandonedWait(err) { + // Whatever the format, the caller needs the task id to reattach: + // in text it is the printed hint, in JSON it is the emitted + // object, and it is in the error either way. + if out == TextOutput { + fmt.Fprintf(stdout, "\n%s. Check progress with: saucectl authoring testcases generate-status %s --wait\n", ErrGenerationStillRunning, taskID) + } else { + _ = renderGenerationJSON(taskID, state) + } + return fmt.Errorf("%w: task %s: %w", ErrGenerationStillRunning, taskID, err) + } + return fmt.Errorf("failed to follow generation task %s: %w", taskID, err) + } + + // The payload comes first so a script has it either way, but the task's + // own status decides the exit code in both formats: returning nil here + // for a FAILED task made `--wait -o json` exit 0 and broke CI gating. + if out == JSONOutput { + if err := renderGenerationJSON(taskID, state); err != nil { + return err + } + } + + switch state.Status { + case authoring.GenerationCompleted: + if out == TextOutput { + fmt.Fprintf(stdout, "\nGeneration completed. New test case: %s\nInspect it with: saucectl authoring testcases get %s --show-steps\n", state.TestCaseID, state.TestCaseID) + } + return nil + case authoring.GenerationFailed: + if state.Error != nil { + return *state.Error + } + return errors.New("generation failed") + default: + return fmt.Errorf("generation ended in unexpected status %q", state.Status) + } +} + +// renderGenerationJSON emits the one final object a JSON-mode wait produces. +// It always carries the task id, which is what makes an abandoned wait +// reattachable without parsing prose. +func renderGenerationJSON(taskID string, state authoring.GenerationState) error { + return renderJSON(struct { + TaskID string `json:"taskId"` + authoring.GenerationState + }{TaskID: taskID, GenerationState: state}) +} + +// isAbandonedWait reports whether the wait ended because we stopped waiting +// (interrupt or the local --wait-timeout) rather than because the task or the +// service told us something. Only in that case is the task still running. +func isAbandonedWait(err error) bool { + return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) +} + +// watchGeneration polls the task until it reaches a terminal status, the +// context ends, or a poll fails. It polls first and waits after, so an +// already-finished task returns immediately. Each step is rendered exactly +// once as it appears; the spinner is the only difference between interactive +// and non-interactive output. It returns the last state seen alongside any +// error, so callers can still report progress after an interruption. +func watchGeneration(ctx context.Context, svc authoring.TestCaseService, taskID string, interval time.Duration, out io.Writer, spin bool) (authoring.GenerationState, error) { + if interval <= 0 { + interval = defaultGenerationPoll + } + + var sp *spinner.Spinner + if spin { + sp = spinner.New(spinner.CharSets[14], 100*time.Millisecond, spinner.WithWriter(out)) + sp.Suffix = " waiting for the agent..." + sp.Start() + defer sp.Stop() + } + + var last authoring.GenerationState + // lastErr remembers the most recent transient poll failure. If the wait + // then ends on its deadline, that error is the honest explanation rather + // than "still running": a 404 tolerated as propagation lag for one tick + // is a mistyped task id once it has persisted to the deadline. + var lastErr error + var firstTransientAt time.Time + renderedSteps, renderedReasoning := 0, 0 + + for { + state, err := svc.GenerationStatus(ctx, taskID) + switch { + case err == nil: + last, lastErr, firstTransientAt = state, nil, time.Time{} + case ctx.Err() != nil: + // The wait ended at the same moment this poll failed. Route it + // through the same decision as every other exit so a deadline + // reached while polls were failing still reports the failure. + if err != nil { + lastErr = err + } + return last, pollDeadlineError(ctx, lastErr) + case authoring.IsFatalPollError(err): + return last, err + default: + // A freshly accepted task can 404 until its record propagates, + // and a truncated body fails to decode; both clear on the next + // tick. The wait is still bounded by ctx, so tolerating them + // costs nothing and matches how the runner polls a run. + lastErr = err + if firstTransientAt.IsZero() { + firstTransientAt = time.Now() + } else if time.Since(firstTransientAt) > maxTransientPollWindow { + return last, err + } + log.Debug().Err(err).Str("task", taskID).Msg("Transient error while polling generation task; retrying.") + select { + case <-ctx.Done(): + return last, pollDeadlineError(ctx, lastErr) + case <-time.After(interval): + } + continue + } + + if sp != nil { + sp.Stop() + } + for ; renderedReasoning < len(state.Reasoning); renderedReasoning++ { + r := state.Reasoning[renderedReasoning] + fmt.Fprintf(out, " * %s\n", r.Title) + } + for ; renderedSteps < len(state.Steps); renderedSteps++ { + fmt.Fprintf(out, " %s\n", formatGenerationStep(state.Steps[renderedSteps])) + } + if sp != nil && !state.Done() { + sp.Suffix = fmt.Sprintf(" %s, %d step(s) so far...", strings.ToLower(strings.ReplaceAll(string(state.Status), "_", " ")), len(state.Steps)) + sp.Start() + } + + if state.Done() { + return state, nil + } + + select { + case <-ctx.Done(): + return last, pollDeadlineError(ctx, lastErr) + case <-time.After(interval): + } + } +} + +// pollDeadlineError decides what a finished wait actually failed on. +// +// If the user interrupted us, that is the truth regardless of what the last +// poll did: they chose to stop, the task is most likely still running, and +// the reattach hint is what they need. If instead our own deadline expired +// while every poll was failing, that failure is the truth — reporting "still +// running" there would send someone back to re-poll something broken, which +// is the defect this whole path was reported for. A successful last poll +// always means the task really is still going. +func pollDeadlineError(ctx context.Context, lastErr error) error { + if lastErr != nil && errors.Is(ctx.Err(), context.DeadlineExceeded) { + return lastErr + } + return ctx.Err() +} + +// formatGenerationStep renders one attempted action with its outcome mark. +func formatGenerationStep(s authoring.GenerationStep) string { + mark := "*" + suffix := "" + if s.Result != nil { + if s.Result.Success { + mark = "✓" + } else { + mark = "✗" + if s.Result.Message != "" { + suffix = " (" + s.Result.Message + ")" + } + } + } + return fmt.Sprintf("%s %s%s", mark, s.Action.Summary(), suffix) +} diff --git a/internal/cmd/authoring/testcases_generate_status.go b/internal/cmd/authoring/testcases_generate_status.go new file mode 100644 index 000000000..386bee119 --- /dev/null +++ b/internal/cmd/authoring/testcases_generate_status.go @@ -0,0 +1,81 @@ +package authoring + +import ( + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// TestCasesGenerateStatusCommand is `authoring testcases generate-status`: it +// shows, or reattaches to, an authoring task started earlier. +func TestCasesGenerateStatusCommand() *cobra.Command { + var out string + var wait bool + var pollInterval, waitTimeout time.Duration + + cmd := &cobra.Command{ + Use: "generate-status ", + Short: "Show the status of an authoring task, or reattach and wait for it", + Example: ` saucectl authoring testcases generate-status 8b1f… + saucectl authoring testcases generate-status 8b1f… --wait`, + SilenceUsage: true, + Args: requireArgs("task-id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(out); err != nil { + return err + } + if wait { + if waitTimeout <= 0 { + waitTimeout = defaultWaitTimeout + } + return waitForGeneration(cmd.Context(), args[0], pollInterval, waitTimeout, out, os.Stdout) + } + + state, err := testCaseService.GenerationStatus(cmd.Context(), args[0]) + if err != nil { + return fmt.Errorf("failed to get generation status: %w", err) + } + if out == JSONOutput { + return renderJSON(state) + } + renderGenerationState(args[0], state) + return nil + }, + } + + flags := cmd.Flags() + flags.StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + flags.BoolVar(&wait, "wait", false, "Wait for the task to finish, streaming each action as it happens.") + flags.DurationVar(&pollInterval, "poll-interval", defaultGenerationPoll, "How often to poll while waiting.") + flags.DurationVar(&waitTimeout, "wait-timeout", 0, "How long to wait before giving up locally. 0 means 1h2m.") + + return cmd +} + +// renderGenerationState prints a one-shot snapshot of a task. +func renderGenerationState(taskID string, state authoring.GenerationState) { + fmt.Printf("Task %s: %s\n", taskID, state.Status) + for _, r := range state.Reasoning { + fmt.Printf(" * %s\n", r.Title) + } + for _, s := range state.Steps { + fmt.Printf(" %s\n", formatGenerationStep(s)) + } + switch state.Status { + case authoring.GenerationCompleted: + fmt.Printf("\nNew test case: %s\nInspect it with: saucectl authoring testcases get %s --show-steps\n", state.TestCaseID, state.TestCaseID) + case authoring.GenerationFailed: + if state.Error != nil { + fmt.Printf("\n%s\n", state.Error.Error()) + } + default: + fmt.Printf("\nStill running. Reattach with: saucectl authoring testcases generate-status %s --wait\n", taskID) + } +} diff --git a/internal/cmd/authoring/testcases_generate_test.go b/internal/cmd/authoring/testcases_generate_test.go new file mode 100644 index 000000000..7d7d3edaa --- /dev/null +++ b/internal/cmd/authoring/testcases_generate_test.go @@ -0,0 +1,184 @@ +package authoring + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/saucelabs/saucectl/internal/authoring" + "github.com/saucelabs/saucectl/internal/mocks" +) + +// step builds a generation step for the fake sequence. +func step(tool authoring.ToolType, args string, success bool) authoring.GenerationStep { + return authoring.GenerationStep{ + Action: authoring.Tool{Type: tool, Args: json.RawMessage(args)}, + Result: &authoring.StepResult{Success: success}, + } +} + +func TestWatchGeneration_RendersEachStepOnce(t *testing.T) { + steps := []authoring.GenerationStep{ + step(authoring.ToolGoToURL, `{"url":"https://saucedemo.com"}`, true), + step(authoring.ToolInputText, `{"selector":{"type":"css","value":"#user-name"},"text":"standard_user"}`, true), + step(authoring.ToolClick, `{"selector":{"type":"css","value":"#login-button"}}`, false), + step(authoring.ToolFinish, `{}`, true), + } + sequence := []authoring.GenerationState{ + {Status: authoring.GenerationQueued}, + {Status: authoring.GenerationInProgress, Steps: steps[:2], Reasoning: []authoring.Reasoning{{Title: "Navigating to the login page"}}}, + {Status: authoring.GenerationInProgress, Steps: steps[:4], Reasoning: []authoring.Reasoning{{Title: "Navigating to the login page"}}}, + {Status: authoring.GenerationCompleted, Steps: steps, TestCaseID: "new-id"}, + } + + calls := 0 + svc := &mocks.AuthoringService{GenerationStatusFn: func(_ context.Context, taskID string) (authoring.GenerationState, error) { + if taskID != "task" { + t.Errorf("taskID = %q", taskID) + } + s := sequence[calls] + if calls < len(sequence)-1 { + calls++ + } + return s, nil + }} + + var out bytes.Buffer + state, err := watchGeneration(context.Background(), svc, "task", time.Millisecond, &out, false) + if err != nil { + t.Fatal(err) + } + if state.Status != authoring.GenerationCompleted || state.TestCaseID != "new-id" { + t.Errorf("final state = %+v", state) + } + if calls != 3 { + t.Errorf("polled %d times before terminal, want 3", calls+1) + } + + text := out.String() + for _, want := range []string{ + "* Navigating to the login page", + "✓ go_to_url https://saucedemo.com", + "✓ input_text css=#user-name ← standard_user", + "✗ click css=#login-button", + "✓ finish", + } { + if strings.Count(text, want) != 1 { + t.Errorf("%q rendered %d times, want exactly once:\n%s", want, strings.Count(text, want), text) + } + } +} + +func TestWatchGeneration_PollsBeforeWaiting(t *testing.T) { + // An already-finished task must return without sleeping through an + // interval. + svc := &mocks.AuthoringService{GenerationStatusFn: func(context.Context, string) (authoring.GenerationState, error) { + return authoring.GenerationState{Status: authoring.GenerationFailed, Error: &authoring.GenerationError{Code: "X", Detail: "boom"}}, nil + }} + start := time.Now() + state, err := watchGeneration(context.Background(), svc, "task", time.Hour, &bytes.Buffer{}, false) + if err != nil || state.Status != authoring.GenerationFailed { + t.Fatalf("state = %+v, err = %v", state, err) + } + if time.Since(start) > time.Second { + t.Error("waited for an interval before the first poll") + } +} + +func TestWatchGeneration_CancelledContext(t *testing.T) { + svc := &mocks.AuthoringService{GenerationStatusFn: func(context.Context, string) (authoring.GenerationState, error) { + return authoring.GenerationState{Status: authoring.GenerationInProgress}, nil + }} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + state, err := watchGeneration(ctx, svc, "task", time.Millisecond, &bytes.Buffer{}, false) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if state.Status != authoring.GenerationInProgress { + t.Errorf("last state must be returned alongside the error, got %+v", state) + } +} + +func TestWatchGeneration_TransientPollErrorIsRetriedThenReported(t *testing.T) { + // A plain error could be propagation lag, so it is retried; when it never + // clears, the wait ends on the cause rather than on a misleading "still + // running", and the caller's deadline bounds the retrying. + // The deadline is driven by the fake rather than the wall clock: an + // earlier version of this test asserted a poll count inside 50ms and + // failed under parallel load. + boom := errors.New("boom") + calls := 0 + deadline, expire := context.WithCancel(context.Background()) + defer expire() + svc := &mocks.AuthoringService{GenerationStatusFn: func(context.Context, string) (authoring.GenerationState, error) { + calls++ + if calls == 3 { + expire() + } + return authoring.GenerationState{}, boom + }} + // A cancelled context means "the user stopped us", which reports the + // context error; the deadline case is what reports the poll failure, so + // wrap the cancellation as a deadline for this assertion. + ctx := deadlineContext{deadline} + _, err := watchGeneration(ctx, svc, "task", time.Millisecond, &bytes.Buffer{}, false) + if !errors.Is(err, boom) { + t.Fatalf("err = %v; a deadline reached while polls keep failing must report the failure", err) + } + if calls != 3 { + t.Errorf("polled %d time(s); a transient error should be retried", calls) + } +} + +func TestWatchGeneration_FatalPollErrorReturnsImmediately(t *testing.T) { + // A 4xx other than 404 cannot clear by waiting, so it must not be retried. + forbidden := &authoring.APIError{HTTPStatus: 403, Code: "UNAUTHORIZED"} + calls := 0 + svc := &mocks.AuthoringService{GenerationStatusFn: func(context.Context, string) (authoring.GenerationState, error) { + calls++ + return authoring.GenerationState{}, forbidden + }} + _, err := watchGeneration(context.Background(), svc, "task", time.Hour, &bytes.Buffer{}, false) + if !errors.Is(err, forbidden) { + t.Fatalf("err = %v", err) + } + if calls != 1 { + t.Errorf("polled %d times; a fatal error must not be retried", calls) + } +} + +func TestWatchGeneration_TransientRetryIsBounded(t *testing.T) { + // Even with an unbounded context the loop must end. + if maxTransientPollWindow <= 0 || maxTransientPollWindow > 5*time.Minute { + t.Errorf("maxTransientPollWindow = %v; want a small positive bound", maxTransientPollWindow) + } +} + +func TestFormatGenerationStep(t *testing.T) { + if got := formatGenerationStep(authoring.GenerationStep{Action: authoring.Tool{Type: authoring.ToolFinish}}); got != "* finish" { + t.Errorf("no result: %q", got) + } + s := step(authoring.ToolClick, `{"selector":{"type":"css","value":"#a"}}`, false) + s.Result.Message = "element not interactable" + if got := formatGenerationStep(s); got != "✗ click css=#a (element not interactable)" { + t.Errorf("failure: %q", got) + } +} + +// deadlineContext reports cancellation as a deadline, so a test can drive the +// "our own deadline expired" path without waiting on a real clock. It must +// stay indistinguishable from a live context until the cancellation actually +// happens, or the code under test sees an expired context on its first look. +type deadlineContext struct{ context.Context } + +func (d deadlineContext) Err() error { + if d.Context.Err() != nil { + return context.DeadlineExceeded + } + return nil +} diff --git a/internal/cmd/authoring/testcases_get.go b/internal/cmd/authoring/testcases_get.go new file mode 100644 index 000000000..c58a8c135 --- /dev/null +++ b/internal/cmd/authoring/testcases_get.go @@ -0,0 +1,177 @@ +package authoring + +import ( + "context" + "fmt" + "strings" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// TestCasesGetCommand is `authoring testcases get`. +func TestCasesGetCommand() *cobra.Command { + var out string + var revisionID string + var showSteps bool + + cmd := &cobra.Command{ + Use: "get ", + Short: "Show a test case, optionally with its recorded steps", + Example: ` saucectl authoring testcases get 6a882c1dc8b4482c166e96c9 + saucectl authoring testcases get 6a882c1dc8b4482c166e96c9 --show-steps + saucectl authoring testcases get 6a882c1dc8b4482c166e96c9 -o json | jq '.revisions[-1].steps'`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(out); err != nil { + return err + } + return getTestCase(cmd.Context(), out, args[0], revisionID, showSteps) + }, + } + + flags := cmd.Flags() + flags.StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + flags.StringVar(&revisionID, "revision", "", "Show this revision instead of the latest.") + flags.BoolVar(&showSteps, "show-steps", false, "Also print the revision's recorded steps.") + + return cmd +} + +// getTestCase fetches and renders one test case. +func getTestCase(ctx context.Context, out, id, revisionID string, showSteps bool) error { + tc, err := testCaseService.GetTestCase(ctx, id) + if err != nil { + return fmt.Errorf("failed to get test case: %w", err) + } + + rev, ok := tc.LatestRevision() + if revisionID != "" { + ok = false + for _, r := range tc.Revisions { + if r.ID == revisionID { + rev, ok = r, true + break + } + } + if !ok { + return fmt.Errorf("test case %s has no revision %s", id, revisionID) + } + } + + if out == JSONOutput { + // Narrow the document to the revision the caller pinned, otherwise + // --revision is computed and then thrown away and a script reading + // .revisions[-1] silently gets the latest instead of the one asked + // for. The shape stays the same, so `jq` expressions keep working. + if revisionID != "" && ok { + tc.Revisions = []authoring.Revision{rev} + } + return renderJSON(tc) + } + + renderTestCaseDetail(tc, rev, ok) + if showSteps && ok { + renderSteps(rev.Steps) + } + return nil +} + +// renderTestCaseDetail prints the two-column property view, following +// `devices get`. +func renderTestCaseDetail(tc authoring.TestCase, rev authoring.Revision, hasRevision bool) { + t := newTable() + t.AppendHeader(table.Row{"Property", "Value"}) + t.AppendRow(table.Row{"ID", tc.ID}) + t.AppendRow(table.Row{"Name", tc.Name}) + t.AppendRow(table.Row{"Tags", joinOrDash(tc.Tags)}) + t.AppendRow(table.Row{"Suite", orDash(tc.TestSuiteID)}) + t.AppendRow(table.Row{"Created", fmt.Sprintf("%s by %s", humanizeDate(tc.CreationDate), orDash(tc.CreatorUserName))}) + t.AppendRow(table.Row{"Updated", fmt.Sprintf("%s by %s", humanizeDate(tc.LastUpdateDate), orDash(tc.LastModifierUserName))}) + t.AppendRow(table.Row{"Test URL", orDash(tc.RunSettings.TestURL)}) + t.AppendRow(table.Row{"Tunnel", describeStoredTunnel(tc.RunSettings.TunnelName)}) + t.AppendRow(table.Row{"Primary Target", describeTarget(tc.RunSettings.PrimaryTarget)}) + targets := make([]string, 0, len(tc.RunSettings.RunTargets)) + for _, rt := range tc.RunSettings.RunTargets { + targets = append(targets, describeTarget(rt)) + } + t.AppendRow(table.Row{"Run Targets", joinOrDash(targets)}) + t.AppendRow(table.Row{"Revisions", len(tc.Revisions)}) + if hasRevision { + t.AppendRow(table.Row{"Revision", rev.ID}) + t.AppendRow(table.Row{"Intent", rev.Intent}) + t.AppendRow(table.Row{"Discovered Intent", orDash(rev.DiscoveredIntent)}) + t.AppendRow(table.Row{"Description", orDash(rev.Description)}) + t.AppendRow(table.Row{"Steps", len(rev.Steps)}) + } + fmt.Println(t.Render()) + + // The revision-level reasoning: the agent's titled paragraphs about the + // journey as a whole, distinct from each step's own reason (FR-015). + if hasRevision && len(rev.Reasoning) > 0 { + fmt.Println("Reasoning:") + for _, r := range rev.Reasoning { + fmt.Printf(" * %s\n", r.Title) + if r.Description != "" { + fmt.Printf(" %s\n", r.Description) + } + } + fmt.Println() + } +} + +// describeStoredTunnel explains the stored tunnel value, calling out the +// empty string, which is not the same as none (research R-008). +func describeStoredTunnel(name *string) string { + switch { + case name == nil: + return "-" + case *name == "": + return `"" (empty; cleared automatically on run)` + default: + return *name + } +} + +// renderSteps prints a revision's steps, one readable line each, with the +// extracted artifact identifier rather than the kilobyte-long signed URL. +func renderSteps(steps []authoring.Step) { + if len(steps) == 0 { + fmt.Println("This revision has no steps.") + return + } + + t := newTable() + t.AppendHeader(table.Row{"#", "Action", "Result", "Screenshot", "Reasoning"}) + for i, s := range steps { + t.AppendRow(table.Row{ + i + 1, + s.Tool.Summary(), + stepResult(s.Result), + orDash(s.ArtifactID()), + truncate(strings.TrimSpace(s.Tool.Reasoning()), 80), + }) + } + fmt.Println(t.Render()) +} + +// stepResult renders a step outcome as a mark plus any message. +func stepResult(r *authoring.StepResult) string { + if r == nil { + return "-" + } + mark := "✔" + if !r.Success { + mark = "✖" + } + if r.Message != "" { + return mark + " " + truncate(r.Message, 60) + } + return mark +} diff --git a/internal/cmd/authoring/testcases_list.go b/internal/cmd/authoring/testcases_list.go new file mode 100644 index 000000000..d7454ba65 --- /dev/null +++ b/internal/cmd/authoring/testcases_list.go @@ -0,0 +1,100 @@ +package authoring + +import ( + "context" + "fmt" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// TestCasesListCommand is `authoring testcases list`. +func TestCasesListCommand() *cobra.Command { + var out string + var page pageFlags + var opts authoring.ListTestCasesOptions + + cmd := &cobra.Command{ + Use: "list", + Aliases: []string{"ls"}, + Short: "List test cases", + Example: ` saucectl authoring testcases list + saucectl authoring testcases list --search checkout --tag smoke + saucectl authoring testcases list --test-suite-id 3f2a9c1e4b5d6e7f8a9b0c1d2e3f4a5b -o json + saucectl authoring testcases list --limit 0 # total count only`, + SilenceUsage: true, + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, _ []string) error { + if err := validateOutput(out); err != nil { + return err + } + if err := page.validate(); err != nil { + return err + } + page.capture(cmd.Flags()) + return listTestCases(cmd.Context(), out, page, opts) + }, + } + + flags := cmd.Flags() + flags.StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + flags.StringVar(&opts.Search, "search", "", "Case-insensitive substring match on the name.") + flags.StringVar(&opts.StartDate, "start-date", "", "Only test cases created on or after this ISO 8601 date.") + flags.StringVar(&opts.EndDate, "end-date", "", "Only test cases created on or before this ISO 8601 date.") + flags.StringVar(&opts.UserID, "user-id", "", "Filter by creator user ID.") + flags.StringVar(&opts.TeamID, "team-id", "", "Filter by team ID.") + flags.StringArrayVar(&opts.TestSuiteIDs, "test-suite-id", nil, "Filter by test suite ID. Repeatable. Use 'null' for test cases in no suite.") + flags.StringArrayVar(&opts.Tags, "tag", nil, "Filter by tag (case-sensitive). Repeatable; matches test cases with any of the tags.") + page.bind(flags) + + return cmd +} + +// listTestCases fetches and renders the listing. +func listTestCases(ctx context.Context, out string, page pageFlags, opts authoring.ListTestCasesOptions) error { + items, total, err := fetchPage(ctx, page, "test cases", func(ctx context.Context, lo authoring.ListOptions) (authoring.List[authoring.TestCase], error) { + opts.ListOptions = lo + return testCaseService.ListTestCases(ctx, opts) + }) + if err != nil { + return fmt.Errorf("failed to list test cases: %w", err) + } + + if out == JSONOutput { + return renderJSON(authoring.List[authoring.TestCase]{Items: items, Total: total}) + } + renderTestCaseTable(items, total) + return nil +} + +// renderTestCaseTable prints the listing table, or a single line when empty. +func renderTestCaseTable(items []authoring.TestCase, total int) { + if len(items) == 0 { + fmt.Printf("No test cases found (total: %d).\n", total) + return + } + + t := newTable() + t.AppendHeader(table.Row{"ID", "Name", "Tags", "Suite", "Steps", "Updated", "Modified By"}) + for _, tc := range items { + steps := "-" + if rev, ok := tc.LatestRevision(); ok { + steps = fmt.Sprint(len(rev.Steps)) + } + t.AppendRow(table.Row{ + tc.ID, + truncate(tc.Name, 50), + joinOrDash(tc.Tags), + orDash(tc.TestSuiteID), + steps, + humanizeDate(tc.LastUpdateDate), + orDash(tc.LastModifierUserName), + }) + } + t.AppendFooter(listFooter(len(items), total, "test cases")) + fmt.Println(t.Render()) +} diff --git a/internal/cmd/authoring/testcases_rename.go b/internal/cmd/authoring/testcases_rename.go new file mode 100644 index 000000000..d9a40a26f --- /dev/null +++ b/internal/cmd/authoring/testcases_rename.go @@ -0,0 +1,41 @@ +package authoring + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// TestCasesRenameCommand is `authoring testcases rename`. +func TestCasesRenameCommand() *cobra.Command { + var out string + + cmd := &cobra.Command{ + Use: "rename ", + Short: "Rename a test case", + Example: ` saucectl authoring testcases rename 6a882c1dc8b4482c166e96c9 "Checkout - add two items"`, + SilenceUsage: true, + Args: requireArgs("id", "name"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(out); err != nil { + return err + } + tc, err := testCaseService.RenameTestCase(cmd.Context(), args[0], args[1]) + if err != nil { + return fmt.Errorf("failed to rename test case: %w", err) + } + if out == JSONOutput { + return renderJSON(tc) + } + fmt.Printf("Renamed test case %s to %q.\n", tc.ID, tc.Name) + return nil + }, + } + + cmd.Flags().StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + + return cmd +} diff --git a/internal/cmd/authoring/testcases_run.go b/internal/cmd/authoring/testcases_run.go new file mode 100644 index 000000000..c3b7749fd --- /dev/null +++ b/internal/cmd/authoring/testcases_run.go @@ -0,0 +1,133 @@ +package authoring + +import ( + "context" + "fmt" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// TestCasesRunCommand is `authoring testcases run`. It starts a run and +// returns immediately with the run's identifiers; use `list-runs` and +// `get-run` to follow it, or `saucectl run` with a `kind: authoring` +// configuration to wait, report and gate a pipeline. +func TestCasesRunCommand() *cobra.Command { + var out string + var build, tunnelName, revisionID string + var kvTargets, jsonTargets []string + + cmd := &cobra.Command{ + Use: "run ", + Short: "Start a run of a test case", + Long: `Start a run of a test case against its stored run targets, or against the targets given +with --target / --target-json. The command returns as soon as the run is accepted; use +'testcases get-run' to check on it. To wait for results, report them and fail a pipeline, +use 'saucectl run' with a 'kind: authoring' configuration instead. + +Without --tunnel-name the run is started with no tunnel, which also clears any tunnel +name stored on the test case.`, + Example: ` saucectl authoring testcases run 6a882c1dc8b4482c166e96c9 --build nightly + saucectl authoring testcases run 6a882c1dc8b4482c166e96c9 --target browserName=firefox,platformName="Windows 11" + saucectl authoring testcases run 6a882c1dc8b4482c166e96c9 --target-json @pixel9.json --tunnel-name my-tunnel`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(out); err != nil { + return err + } + targets, err := parseTargets(kvTargets, jsonTargets) + if err != nil { + return err + } + return runTestCase(cmd.Context(), out, args[0], revisionID, authoring.RunOptions{ + BuildName: build, + TunnelName: tunnelName, + Targets: targets, + }) + }, + } + + flags := cmd.Flags() + flags.StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + flags.StringVar(&build, "build", "", "Build name to group the run's jobs under (max 100 characters).") + flags.StringVar(&tunnelName, "tunnel-name", "", "Name of an active Sauce Connect tunnel to route the run through.") + flags.StringArrayVar(&kvTargets, "target", nil, "Target capabilities as key=value pairs, e.g. browserName=chrome,platformName=\"Windows 11\". Repeatable.") + flags.StringArrayVar(&jsonTargets, "target-json", nil, "Target capabilities as a JSON object, or @path to a file. Repeatable.") + flags.StringVar(&revisionID, "revision", "", "Run this revision instead of the latest.") + + return cmd +} + +// runTestCase starts the run and renders its identifiers. +func runTestCase(ctx context.Context, out, id, revisionID string, opts authoring.RunOptions) error { + run, err := testCaseService.RunTestCase(ctx, id, revisionID, opts) + if err != nil { + return fmt.Errorf("failed to start run: %w", err) + } + + if out == JSONOutput { + return renderJSON(run) + } + + fmt.Printf("Run %s started for test case %s (build %q).\n", run.ID, run.TestCaseID, run.Build) + renderJobs(run.Jobs) + fmt.Printf("Check on it with: saucectl authoring testcases get-run %s %s\n", run.TestCaseID, run.ID) + return nil +} + +// renderJobs prints one row per job with its derived dashboard link. +func renderJobs(jobs []authoring.RunJob) { + if len(jobs) == 0 { + fmt.Println("No jobs reported yet.") + return + } + + t := newTable() + t.AppendHeader(table.Row{"Job", "Target", "Status", "Error", "URL"}) + for _, j := range jobs { + t.AppendRow(table.Row{ + orDash(j.SauceJobID), + describeTarget(j.Target), + jobStatus(j), + truncate(orDash(j.Error), 60), + orDash(jobURL(j.SauceJobID)), + }) + } + fmt.Println(t.Render()) +} + +// jobStatus renders a job's inferred state: there is no status field on the +// wire (research R-005). +func jobStatus(j authoring.RunJob) string { + switch { + case j.Passed(): + return "passed" + case j.Done(): + return "failed" + default: + return "in progress" + } +} + +// runStatus summarises a run's jobs as e.g. "2/3 passed" or "in progress". +func runStatus(r authoring.Run) string { + if !r.Done() { + return "in progress" + } + passed := 0 + for _, j := range r.Jobs { + if j.Passed() { + passed++ + } + } + if passed == len(r.Jobs) { + return fmt.Sprintf("passed (%d/%d)", passed, len(r.Jobs)) + } + return fmt.Sprintf("failed (%d/%d passed)", passed, len(r.Jobs)) +} diff --git a/internal/cmd/authoring/testcases_runs.go b/internal/cmd/authoring/testcases_runs.go new file mode 100644 index 000000000..1de7e7293 --- /dev/null +++ b/internal/cmd/authoring/testcases_runs.go @@ -0,0 +1,129 @@ +package authoring + +import ( + "context" + "fmt" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// TestCasesListRunsCommand is `authoring testcases list-runs`. +func TestCasesListRunsCommand() *cobra.Command { + var out string + var page pageFlags + var opts authoring.ListRunsOptions + + cmd := &cobra.Command{ + Use: "list-runs ", + Aliases: []string{"runs"}, + Short: "List the runs of a test case", + Example: ` saucectl authoring testcases list-runs 6a882c1dc8b4482c166e96c9 + saucectl authoring testcases list-runs 6a882c1dc8b4482c166e96c9 --all -o json | jq -r '[.items[].testCaseId] | unique'`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(out); err != nil { + return err + } + if err := page.validate(); err != nil { + return err + } + page.capture(cmd.Flags()) + return listRuns(cmd.Context(), out, args[0], page, opts) + }, + } + + flags := cmd.Flags() + flags.StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + flags.StringVar(&opts.StartDate, "start-date", "", "Only runs created on or after this ISO 8601 date.") + flags.StringVar(&opts.EndDate, "end-date", "", "Only runs created on or before this ISO 8601 date.") + flags.StringVar(&opts.UserID, "user-id", "", "Only runs started by this user ID.") + flags.StringVar(&opts.TeamID, "team-id", "", "Only runs belonging to this team ID.") + page.bind(flags) + + return cmd +} + +// listRuns fetches and renders the runs of one test case. The service filters +// on the testCaseId query parameter, which ListRuns always sends; without it +// the whole organisation's runs would come back (research R-004). +func listRuns(ctx context.Context, out, testCaseID string, page pageFlags, opts authoring.ListRunsOptions) error { + items, total, err := fetchPage(ctx, page, "runs", func(ctx context.Context, lo authoring.ListOptions) (authoring.List[authoring.Run], error) { + opts.ListOptions = lo + return testCaseService.ListRuns(ctx, testCaseID, opts) + }) + if err != nil { + return fmt.Errorf("failed to list runs: %w", err) + } + + if out == JSONOutput { + return renderJSON(authoring.List[authoring.Run]{Items: items, Total: total}) + } + + if len(items) == 0 { + fmt.Printf("No runs found (total: %d).\n", total) + return nil + } + t := newTable() + t.AppendHeader(table.Row{"Run ID", "Build", "Jobs", "Status", "Created"}) + for _, r := range items { + t.AppendRow(table.Row{r.ID, truncate(orDash(r.Build), 40), len(r.Jobs), runStatus(r), humanizeDate(r.CreationDate)}) + } + t.AppendFooter(listFooter(len(items), total, "runs")) + fmt.Println(t.Render()) + return nil +} + +// TestCasesGetRunCommand is `authoring testcases get-run`. +func TestCasesGetRunCommand() *cobra.Command { + var out string + + cmd := &cobra.Command{ + Use: "get-run ", + Short: "Show a single run with its jobs", + Example: ` saucectl authoring testcases get-run 6a882c1dc8b4482c166e96c9 ed320d67-5f56-4e81-b61d-72e325661c77`, + SilenceUsage: true, + Args: requireArgs("id", "run-id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(out); err != nil { + return err + } + run, err := testCaseService.GetRun(cmd.Context(), args[0], args[1]) + if err != nil { + return fmt.Errorf("failed to get run: %w", err) + } + if out == JSONOutput { + return renderJSON(run) + } + renderRunDetail(run) + return nil + }, + } + + cmd.Flags().StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + + return cmd +} + +// renderRunDetail prints a run's properties followed by its jobs. +func renderRunDetail(run authoring.Run) { + t := newTable() + t.AppendHeader(table.Row{"Property", "Value"}) + t.AppendRow(table.Row{"Run ID", run.ID}) + t.AppendRow(table.Row{"Test Case", run.TestCaseID}) + t.AppendRow(table.Row{"Build", orDash(run.Build)}) + t.AppendRow(table.Row{"Status", runStatus(run)}) + t.AppendRow(table.Row{"Created", humanizeDate(run.CreationDate)}) + t.AppendRow(table.Row{"Test URL", orDash(run.TestURL)}) + fmt.Println(t.Render()) + renderJobs(run.Jobs) +} diff --git a/internal/cmd/authoring/testcases_tags.go b/internal/cmd/authoring/testcases_tags.go new file mode 100644 index 000000000..27439499d --- /dev/null +++ b/internal/cmd/authoring/testcases_tags.go @@ -0,0 +1,52 @@ +package authoring + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// TestCasesListTagsCommand is `authoring testcases list-tags`. Tags are +// returned exactly as the service holds them: case-sensitive, not folded, not +// deduplicated — "Login" and "login" are two tags. +func TestCasesListTagsCommand() *cobra.Command { + var out string + + cmd := &cobra.Command{ + Use: "list-tags", + Aliases: []string{"tags"}, + Short: "List every tag in use across the organisation's test cases", + Example: ` saucectl authoring testcases list-tags`, + SilenceUsage: true, + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, _ []string) error { + if err := validateOutput(out); err != nil { + return err + } + tags, err := testCaseService.ListTags(cmd.Context()) + if err != nil { + return fmt.Errorf("failed to list tags: %w", err) + } + if out == JSONOutput { + if tags == nil { + tags = []string{} + } + return renderJSON(tags) + } + if len(tags) == 0 { + fmt.Println("No tags found.") + return nil + } + for _, tag := range tags { + fmt.Println(tag) + } + return nil + }, + } + + cmd.Flags().StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + + return cmd +} diff --git a/internal/cmd/authoring/testsuites.go b/internal/cmd/authoring/testsuites.go new file mode 100644 index 000000000..60cd47693 --- /dev/null +++ b/internal/cmd/authoring/testsuites.go @@ -0,0 +1,25 @@ +package authoring + +import "github.com/spf13/cobra" + +// TestSuitesCommand is the `authoring testsuites` subgroup. No pre-run of its +// own, so the root's runs (see TestCasesCommand). +func TestSuitesCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "testsuites", + Aliases: []string{"testsuite", "ts"}, + Short: "Group test cases into suites", + SilenceUsage: true, + } + + cmd.AddCommand( + TestSuitesListCommand(), + TestSuitesGetCommand(), + TestSuitesCreateCommand(), + TestSuitesUpdateCommand(), + TestSuitesDeleteCommand(), + TestSuitesRunCommand(), + ) + + return cmd +} diff --git a/internal/cmd/authoring/testsuites_create.go b/internal/cmd/authoring/testsuites_create.go new file mode 100644 index 000000000..2de6db38f --- /dev/null +++ b/internal/cmd/authoring/testsuites_create.go @@ -0,0 +1,54 @@ +package authoring + +import ( + "errors" + "fmt" + + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// TestSuitesCreateCommand is `authoring testsuites create`. +func TestSuitesCreateCommand() *cobra.Command { + var out string + var opts authoring.CreateTestSuiteOptions + + cmd := &cobra.Command{ + Use: "create", + Short: "Create a test suite", + Example: ` saucectl authoring testsuites create --name "Checkout Regression" --tag smoke \ + --test-case 6a882c1dc8b4482c166e96c9 --test-case 6a6b903c0405fb400076b2ba`, + SilenceUsage: true, + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, _ []string) error { + if err := validateOutput(out); err != nil { + return err + } + if opts.Name == "" { + return errors.New("--name is required") + } + s, err := testSuiteService.CreateTestSuite(cmd.Context(), opts) + if err != nil { + return fmt.Errorf("failed to create test suite: %w", err) + } + if out == JSONOutput { + return renderJSON(s) + } + // testCaseCount in mutation responses lags behind the change (observed + // 2026-09-06), so it is deliberately not quoted here. + fmt.Printf("Created test suite %q (%s).\n", s.Name, s.ID) + return nil + }, + } + + flags := cmd.Flags() + flags.StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + flags.StringVar(&opts.Name, "name", "", "Name of the suite (1–255 characters). Required.") + flags.StringArrayVar(&opts.Tags, "tag", nil, "Tag for the suite. Repeatable.") + flags.StringArrayVar(&opts.TestCases, "test-case", nil, "Test case ID to include. Repeatable.") + + return cmd +} diff --git a/internal/cmd/authoring/testsuites_delete.go b/internal/cmd/authoring/testsuites_delete.go new file mode 100644 index 000000000..31cdbb9f7 --- /dev/null +++ b/internal/cmd/authoring/testsuites_delete.go @@ -0,0 +1,124 @@ +package authoring + +import ( + "context" + "fmt" + + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// TestSuitesDeleteCommand is `authoring testsuites delete`. +func TestSuitesDeleteCommand() *cobra.Command { + var yes, deleteTestCases bool + + cmd := &cobra.Command{ + Use: "delete ", + Aliases: []string{"rm"}, + Short: "Delete a test suite", + Long: `Delete a test suite. Its test cases are kept and become unassigned unless +--delete-test-cases is given, in which case every test case in the suite is deleted too.`, + Example: ` saucectl authoring testsuites delete 3f2a9c1e4b5d6e7f8a9b0c1d2e3f4a5b + saucectl authoring testsuites delete 3f2a9c1e4b5d6e7f8a9b0c1d2e3f4a5b --delete-test-cases --yes`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return deleteTestSuite(cmd.Context(), args[0], yes, deleteTestCases) + }, + } + + flags := cmd.Flags() + flags.BoolVarP(&yes, "yes", "y", false, "Skip the confirmation prompt. Required when not running interactively.") + flags.BoolVar(&deleteTestCases, "delete-test-cases", false, "Also delete every test case in the suite.") + + return cmd +} + +// deleteTestSuite confirms, naming the suite's test cases and any schedules +// that trigger it, then deletes. +func deleteTestSuite(ctx context.Context, id string, yes, deleteTestCases bool) error { + s, err := testSuiteService.GetTestSuite(ctx, id) + if err != nil { + return fmt.Errorf("failed to get test suite: %w", err) + } + + // Only built when a prompt will actually be printed; see testcases_delete.go. + var affects []string + if yes { + affects = nil + } else if deleteTestCases { + affects = append(affects, fmt.Sprintf("%d test case(s) in the suite will be DELETED", s.TestCaseCount)) + } else { + affects = append(affects, fmt.Sprintf("%d test case(s) in the suite will be kept and become unassigned", s.TestCaseCount)) + } + limit := 20 + schedules := authoring.List[authoring.TestSchedule]{} + if !yes { + schedules, err = scheduleService.ListSchedules(ctx, authoring.ListSchedulesOptions{ListOptions: authoring.ListOptions{Limit: &limit}, TestSuiteIDs: []string{s.ID}}) + } + if !yes && err == nil && schedules.Total > 0 { + for _, sch := range schedules.Items { + affects = append(affects, fmt.Sprintf("schedule %q (%s) triggers this suite", sch.Name, sch.ID)) + } + if schedules.Total > len(schedules.Items) { + affects = append(affects, fmt.Sprintf("...and %d more schedule(s)", schedules.Total-len(schedules.Items))) + } + } + + if err := confirmDestructive(yes, fmt.Sprintf("test suite %q (%s)", s.Name, s.ID), affects); err != nil { + return err + } + + if err := testSuiteService.DeleteTestSuite(ctx, s.ID, deleteTestCases); err != nil { + return fmt.Errorf("failed to delete test suite: %w", err) + } + fmt.Printf("Deleted test suite %q (%s).\n", s.Name, s.ID) + return nil +} + +// TestSuitesRunCommand is `authoring testsuites run`. It is fire-and-forget by +// contract: the service returns a queued count and the build name only, with +// no per-case run identifiers to follow (research R-002). Use `saucectl run` +// with a `kind: authoring` configuration to wait for results. +func TestSuitesRunCommand() *cobra.Command { + var out, build string + + cmd := &cobra.Command{ + Use: "run ", + Short: "Queue a run of every test case in a suite (fire-and-forget)", + Long: `Queue a run of every test case in the suite. The service reports only how many runs were +queued; it returns no run identifiers, so results cannot be followed from here. To wait +for results and gate a pipeline, use 'saucectl run' with a 'kind: authoring' configuration.`, + Example: ` saucectl authoring testsuites run 3f2a9c1e4b5d6e7f8a9b0c1d2e3f4a5b --build nightly`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(out); err != nil { + return err + } + r, err := testSuiteService.RunTestSuite(cmd.Context(), args[0], build) + if err != nil { + return fmt.Errorf("failed to run test suite: %w", err) + } + if out == JSONOutput { + return renderJSON(r) + } + fmt.Printf("Queued %d run(s) for test suite %s under build %q.\n", r.RunCount, args[0], r.BuildName) + fmt.Println("Results are not followed here; see the Sauce Labs dashboard, or use 'saucectl run' with kind: authoring to wait for them.") + return nil + }, + } + + flags := cmd.Flags() + flags.StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + flags.StringVar(&build, "build", "", "Build name to group the runs under.") + + return cmd +} diff --git a/internal/cmd/authoring/testsuites_list.go b/internal/cmd/authoring/testsuites_list.go new file mode 100644 index 000000000..a46f06452 --- /dev/null +++ b/internal/cmd/authoring/testsuites_list.go @@ -0,0 +1,127 @@ +package authoring + +import ( + "context" + "fmt" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// TestSuitesListCommand is `authoring testsuites list`. +func TestSuitesListCommand() *cobra.Command { + var out string + var page pageFlags + var opts authoring.ListTestSuitesOptions + + cmd := &cobra.Command{ + Use: "list", + Aliases: []string{"ls"}, + Short: "List test suites", + Example: ` saucectl authoring testsuites list + saucectl authoring testsuites list --search regression -o json`, + SilenceUsage: true, + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, _ []string) error { + if err := validateOutput(out); err != nil { + return err + } + if err := page.validate(); err != nil { + return err + } + page.capture(cmd.Flags()) + items, total, err := fetchPage(cmd.Context(), page, "test suites", func(ctx context.Context, lo authoring.ListOptions) (authoring.List[authoring.TestSuite], error) { + opts.ListOptions = lo + return testSuiteService.ListTestSuites(ctx, opts) + }) + if err != nil { + return fmt.Errorf("failed to list test suites: %w", err) + } + if out == JSONOutput { + return renderJSON(authoring.List[authoring.TestSuite]{Items: items, Total: total}) + } + renderTestSuiteTable(items, total) + return nil + }, + } + + flags := cmd.Flags() + flags.StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + flags.StringArrayVar(&opts.IDs, "id", nil, "Only these suite IDs. Repeatable.") + flags.StringVar(&opts.Search, "search", "", "Case-insensitive substring match on the name.") + flags.StringVar(&opts.StartDate, "start-date", "", "Only suites created on or after this ISO 8601 date.") + flags.StringVar(&opts.EndDate, "end-date", "", "Only suites created on or before this ISO 8601 date.") + flags.StringVar(&opts.UserID, "user-id", "", "Filter by creator user ID.") + flags.StringVar(&opts.TeamID, "team-id", "", "Filter by team ID.") + page.bind(flags) + + return cmd +} + +// renderTestSuiteTable prints the listing table, or a single line when empty. +func renderTestSuiteTable(items []authoring.TestSuite, total int) { + if len(items) == 0 { + fmt.Printf("No test suites found (total: %d).\n", total) + return + } + t := newTable() + t.AppendHeader(table.Row{"ID", "Name", "Tags", "Test Cases", "Updated", "Modified By"}) + for _, s := range items { + t.AppendRow(table.Row{s.ID, truncate(s.Name, 50), joinOrDash(s.Tags), s.TestCaseCount, humanizeDate(s.LastUpdate), orDash(s.LastModifierUserName)}) + } + t.AppendFooter(listFooter(len(items), total, "test suites")) + fmt.Println(t.Render()) +} + +// TestSuitesGetCommand is `authoring testsuites get`. +func TestSuitesGetCommand() *cobra.Command { + var out string + + cmd := &cobra.Command{ + Use: "get ", + Short: "Show a test suite", + Example: ` saucectl authoring testsuites get 3f2a9c1e4b5d6e7f8a9b0c1d2e3f4a5b`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(out); err != nil { + return err + } + s, err := testSuiteService.GetTestSuite(cmd.Context(), args[0]) + if err != nil { + return fmt.Errorf("failed to get test suite: %w", err) + } + if out == JSONOutput { + return renderJSON(s) + } + renderTestSuiteDetail(s) + return nil + }, + } + + cmd.Flags().StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + + return cmd +} + +// renderTestSuiteDetail prints the two-column property view. +func renderTestSuiteDetail(s authoring.TestSuite) { + t := newTable() + t.AppendHeader(table.Row{"Property", "Value"}) + t.AppendRow(table.Row{"ID", s.ID}) + t.AppendRow(table.Row{"Name", s.Name}) + t.AppendRow(table.Row{"Tags", joinOrDash(s.Tags)}) + t.AppendRow(table.Row{"Test Cases", s.TestCaseCount}) + t.AppendRow(table.Row{"Team", orDash(s.TeamID)}) + t.AppendRow(table.Row{"Created", fmt.Sprintf("%s by %s", humanizeDate(s.CreationDate), orDash(s.CreatorUserName))}) + t.AppendRow(table.Row{"Updated", fmt.Sprintf("%s by %s", humanizeDate(s.LastUpdate), orDash(s.LastModifierUserName))}) + fmt.Println(t.Render()) + fmt.Printf("List its test cases with: saucectl authoring testcases list --test-suite-id %s\n", s.ID) +} diff --git a/internal/cmd/authoring/testsuites_update.go b/internal/cmd/authoring/testsuites_update.go new file mode 100644 index 000000000..b399da467 --- /dev/null +++ b/internal/cmd/authoring/testsuites_update.go @@ -0,0 +1,60 @@ +package authoring + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// TestSuitesUpdateCommand is `authoring testsuites update`. +func TestSuitesUpdateCommand() *cobra.Command { + var out string + var opts authoring.UpdateTestSuiteOptions + + cmd := &cobra.Command{ + Use: "update ", + Short: "Update a test suite's name, tags or membership", + Long: `Update a test suite. --test-case replaces the membership wholesale; --add-test-case and +--remove-test-case change it incrementally without restating the rest. The two forms +cannot be combined. At least one change is required.`, + Example: ` saucectl authoring testsuites update 3f2a… --name "Checkout Regression v2" + saucectl authoring testsuites update 3f2a… --add-test-case 6a882c1dc8b4482c166e96c9 + saucectl authoring testsuites update 3f2a… --remove-test-case 6a6b903c0405fb400076b2ba`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(out); err != nil { + return err + } + if err := opts.Validate(); err != nil { + return err + } + s, err := testSuiteService.UpdateTestSuite(cmd.Context(), args[0], opts) + if err != nil { + return fmt.Errorf("failed to update test suite: %w", err) + } + if out == JSONOutput { + return renderJSON(s) + } + // testCaseCount in mutation responses lags behind the change (observed + // 2026-09-06), so it is deliberately not quoted here. + fmt.Printf("Updated test suite %q (%s).\n", s.Name, s.ID) + return nil + }, + } + + flags := cmd.Flags() + flags.StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + flags.StringVar(&opts.Name, "name", "", "New name.") + flags.StringArrayVar(&opts.Tags, "tag", nil, "Replace the tags with these. Repeatable.") + flags.StringArrayVar(&opts.TestCases, "test-case", nil, "Replace the membership with these test case IDs. Repeatable.") + flags.StringArrayVar(&opts.AddTestCases, "add-test-case", nil, "Add this test case ID. Repeatable.") + flags.StringArrayVar(&opts.RemoveTestCases, "remove-test-case", nil, "Remove this test case ID. Repeatable.") + + return cmd +} diff --git a/internal/cmd/authoring/valuesource.go b/internal/cmd/authoring/valuesource.go new file mode 100644 index 000000000..9d47c4aa0 --- /dev/null +++ b/internal/cmd/authoring/valuesource.go @@ -0,0 +1,109 @@ +package authoring + +import ( + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/AlecAivazis/survey/v2" + "github.com/rs/zerolog/log" +) + +// valueSource is the set of flags a value can come from. Exactly one may be +// used; when none is, an interactive session prompts and a non-interactive one +// errors out naming every option (FR-023, SC-003). +type valueSource struct { + // value is --value: works, but is visible in shell history and the + // process list. + value string + // envName is --value-from-env: the name of an environment variable. + envName string + // file is --value-from-file: a path, or "-" for stdin. + file string + // secret selects a masked prompt and triggers the --value warning. + secret bool +} + +// set reports whether any source was given. +func (v valueSource) set() bool { + return v.value != "" || v.envName != "" || v.file != "" +} + +// promptValue asks for a value on the terminal, masked when secret. A +// variable so tests can replace it. +var promptValue = func(secret bool) (string, error) { + var out string + var err error + if secret { + err = survey.AskOne(&survey.Password{Message: "Value:"}, &out) + } else { + err = survey.AskOne(&survey.Input{Message: "Value:"}, &out) + } + return out, err +} + +// stdinIsTerminal reports whether stdin is a terminal, in which case reading +// a value from it would hang waiting for input nobody knows to give. A +// variable for tests. +var stdinIsTerminal = func() bool { return isTerm(os.Stdin.Fd()) } + +// resolveValue produces the value from exactly one source. stdin is what +// --value-from-file - reads. +func resolveValue(v valueSource, stdin io.Reader) (string, error) { + sources := 0 + for _, s := range []string{v.value, v.envName, v.file} { + if s != "" { + sources++ + } + } + if sources > 1 { + return "", errors.New("specify only one of --value, --value-from-env or --value-from-file") + } + + switch { + case v.envName != "": + val, ok := os.LookupEnv(v.envName) + if !ok || val == "" { + return "", fmt.Errorf("environment variable %s is not set or empty", v.envName) + } + return val, nil + + case v.file == "-": + if stdinIsTerminal() { + return "", errors.New("--value-from-file - reads standard input, but standard input is a terminal; pipe the value in or use another source") + } + b, err := io.ReadAll(stdin) + if err != nil { + return "", fmt.Errorf("reading value from standard input: %w", err) + } + return stripTrailingNewline(string(b)), nil + + case v.file != "": + b, err := os.ReadFile(v.file) + if err != nil { + return "", fmt.Errorf("reading value file: %w", err) + } + return stripTrailingNewline(string(b)), nil + + case v.value != "": + if v.secret { + log.Warn().Msg("A secret passed with --value is visible in shell history and the process list; prefer --value-from-env or --value-from-file.") + } + return v.value, nil + } + + if !isInteractive() { + return "", errors.New("no value given; supply one with --value-from-env NAME, --value-from-file PATH (or - for stdin), or --value") + } + return promptValue(v.secret) +} + +// stripTrailingNewline removes exactly one trailing line ending, so +// `echo secret | ...` behaves as expected while a deliberately multi-line +// value keeps its inner newlines. +func stripTrailingNewline(s string) string { + s = strings.TrimSuffix(s, "\n") + return strings.TrimSuffix(s, "\r") +} diff --git a/internal/cmd/authoring/variables.go b/internal/cmd/authoring/variables.go new file mode 100644 index 000000000..9ec885df5 --- /dev/null +++ b/internal/cmd/authoring/variables.go @@ -0,0 +1,24 @@ +package authoring + +import "github.com/spf13/cobra" + +// VariablesCommand is the `authoring variables` subgroup. No pre-run of its +// own, so the root's runs (see TestCasesCommand). +func VariablesCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "variables", + Aliases: []string{"variable", "var"}, + Short: "Manage shared values available to authored tests", + SilenceUsage: true, + } + + cmd.AddCommand( + VariablesListCommand(), + VariablesGetCommand(), + VariablesCreateCommand(), + VariablesUpdateCommand(), + VariablesDeleteCommand(), + ) + + return cmd +} diff --git a/internal/cmd/authoring/variables_create.go b/internal/cmd/authoring/variables_create.go new file mode 100644 index 000000000..1205717db --- /dev/null +++ b/internal/cmd/authoring/variables_create.go @@ -0,0 +1,108 @@ +package authoring + +import ( + "errors" + "fmt" + "os" + "regexp" + + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// variableNamePattern is the service's constraint on names. +var variableNamePattern = regexp.MustCompile(`^[a-z0-9_]+$`) + +// bindValueSourceFlags registers the three ways to supply a value. +func bindValueSourceFlags(cmd *cobra.Command, vs *valueSource) { + flags := cmd.Flags() + flags.StringVar(&vs.value, "value", "", "The value. Visible in shell history and the process list; prefer --value-from-env or --value-from-file for secrets.") + flags.StringVar(&vs.envName, "value-from-env", "", "Read the value from this environment variable.") + flags.StringVar(&vs.file, "value-from-file", "", "Read the value from this file, or - for standard input. One trailing newline is stripped.") +} + +// VariablesCreateCommand is `authoring variables create`. +func VariablesCreateCommand() *cobra.Command { + var out, scope string + var opts authoring.CreateVariableOptions + var vs valueSource + + cmd := &cobra.Command{ + Use: "create", + Short: "Create a variable", + Long: `Create a variable at organisation, team, suite or test case scope. + +The value comes from exactly one of --value, --value-from-env or --value-from-file. With +none of them, an interactive session prompts (masked for --secret) and a non-interactive +one fails. A secret's value is stored separately and never returned by any command.`, + Example: ` saucectl authoring variables create --scope org --name password --secret --value-from-env SAUCE_DEMO_PASSWORD + echo -n "standard_user" | saucectl authoring variables create --scope team --name username --value-from-file - + saucectl authoring variables create --scope testCase --test-case-id 6a88… --name coupon --value SAVE10`, + SilenceUsage: true, + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, _ []string) error { + if err := validateOutput(out); err != nil { + return err + } + parsed, ok := authoring.ParseVariableScope(scope) + if scope == "" || !ok { + return fmt.Errorf("--scope is required; options: org, team, testSuite, testCase") + } + opts.Scope = parsed + if err := authoring.ValidateScopePairing(opts.Scope, opts.TestSuiteID, opts.TestCaseID); err != nil { + return err + } + if opts.Name == "" { + return errors.New("--name is required") + } + if !variableNamePattern.MatchString(opts.Name) { + return fmt.Errorf("invalid --name %q: use lowercase letters, digits and underscores only", opts.Name) + } + if len(opts.Description) > 1000 { + return errors.New("--description must be at most 1000 characters") + } + + vs.secret = opts.IsSecret + value, err := resolveValue(vs, os.Stdin) + if err != nil { + return err + } + opts.Value = value + + v, err := variableService.CreateVariable(cmd.Context(), opts) + if err != nil { + return fmt.Errorf("failed to create variable: %w", err) + } + blankSecret(&v) + if out == JSONOutput { + return renderJSON(v) + } + fmt.Printf("Created %s variable %q (%s) at %s scope.\n", secretWord(v.IsSecret), v.Name, v.ID, v.Scope) + return nil + }, + } + + flags := cmd.Flags() + flags.StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + flags.StringVar(&scope, "scope", "", "Scope: org, team, testSuite or testCase. Required.") + flags.StringVar(&opts.TestSuiteID, "test-suite-id", "", "Suite ID. Required with --scope testSuite.") + flags.StringVar(&opts.TestCaseID, "test-case-id", "", "Test case ID. Required with --scope testCase.") + flags.StringVar(&opts.Name, "name", "", "Name: lowercase letters, digits and underscores (1–255 characters). Required.") + flags.StringVar(&opts.Description, "description", "", "Description (at most 1000 characters).") + flags.BoolVar(&opts.IsSecret, "secret", false, "Mark the value confidential: it is never displayed again.") + bindValueSourceFlags(cmd, &vs) + registerScopeCompletion(cmd) + + return cmd +} + +// secretWord renders "secret" or "plain" for messages. +func secretWord(secret bool) string { + if secret { + return "secret" + } + return "plain" +} diff --git a/internal/cmd/authoring/variables_delete.go b/internal/cmd/authoring/variables_delete.go new file mode 100644 index 000000000..3336bb678 --- /dev/null +++ b/internal/cmd/authoring/variables_delete.go @@ -0,0 +1,74 @@ +package authoring + +import ( + "context" + "errors" + "fmt" + + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// VariablesDeleteCommand is `authoring variables delete`. +func VariablesDeleteCommand() *cobra.Command { + var yes bool + var expectedLastUpdate string + + cmd := &cobra.Command{ + Use: "delete ", + Aliases: []string{"rm"}, + Short: "Delete a variable", + Example: ` saucectl authoring variables delete 5e7a… + saucectl authoring variables delete 5e7a… --yes --expected-last-update 2026-09-05T14:59:59.000Z`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + return deleteVariable(cmd.Context(), args[0], yes, expectedLastUpdate) + }, + } + + flags := cmd.Flags() + flags.BoolVarP(&yes, "yes", "y", false, "Skip the confirmation prompt. Required when not running interactively.") + flags.StringVar(&expectedLastUpdate, "expected-last-update", "", "The lastUpdate value from an earlier read; the delete fails if the variable changed since. Default: the current value.") + + return cmd +} + +// deleteVariable confirms, naming scope and secrecy, then deletes with +// concurrency control. The token goes in the query string on this endpoint. +func deleteVariable(ctx context.Context, id string, yes bool, expectedLastUpdate string) error { + v, err := variableService.GetVariable(ctx, id) + if err != nil { + return fmt.Errorf("failed to get variable: %w", err) + } + + affects := []string{fmt.Sprintf("it is a %s variable at %s scope", secretWord(v.IsSecret), v.Scope)} + if v.TestSuiteID != "" { + affects = append(affects, fmt.Sprintf("tests in suite %s that reference {{%s:%s}} will lose it", v.TestSuiteID, v.Scope, v.Name)) + } else if v.TestCaseID != "" { + affects = append(affects, fmt.Sprintf("test case %s references it as {{%s:%s}}", v.TestCaseID, v.Scope, v.Name)) + } else { + affects = append(affects, fmt.Sprintf("every test referencing {{%s:%s}} will lose it", v.Scope, v.Name)) + } + + if err := confirmDestructive(yes, fmt.Sprintf("variable %q (%s)", v.Name, v.ID), affects); err != nil { + return err + } + + token := expectedLastUpdate + if token == "" { + token = v.LastUpdate + } + if err := variableService.DeleteVariable(ctx, v.ID, token); err != nil { + if errors.Is(err, authoring.ErrVariableVersionConflict) { + return fmt.Errorf("variable %s was changed by someone else since it was read; re-read it with 'saucectl authoring variables get %s' and try again", id, id) + } + return fmt.Errorf("failed to delete variable: %w", err) + } + fmt.Printf("Deleted variable %q (%s).\n", v.Name, v.ID) + return nil +} diff --git a/internal/cmd/authoring/variables_list.go b/internal/cmd/authoring/variables_list.go new file mode 100644 index 000000000..4d0ad65da --- /dev/null +++ b/internal/cmd/authoring/variables_list.go @@ -0,0 +1,186 @@ +package authoring + +import ( + "context" + "fmt" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// secretPlaceholder is what a confidential value renders as, whatever the +// service returned (FR-022). +const secretPlaceholder = "" + +// VariablesListCommand is `authoring variables list`. +func VariablesListCommand() *cobra.Command { + var out, scope string + var page pageFlags + var opts authoring.ListVariablesOptions + + cmd := &cobra.Command{ + Use: "list", + Aliases: []string{"ls"}, + Short: "List variables", + Example: ` saucectl authoring variables list --scope org + saucectl authoring variables list --scope testSuite --test-suite-id 3f2a… + saucectl authoring variables list --search pass -o json`, + SilenceUsage: true, + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, _ []string) error { + if err := validateOutput(out); err != nil { + return err + } + if err := page.validate(); err != nil { + return err + } + page.capture(cmd.Flags()) + // Unlike the other listings, the variables endpoint requires + // 1 <= limit <= 200 and has no count-only mode (observed: 400 + // INVALID_QUERY "limit: Too small: expected number to be >=1"). + if !page.all && (page.limit < 1 || page.limit > 200) { + return fmt.Errorf("--limit must be between 1 and 200 for variables; the service has no count-only mode") + } + if scope != "" { + parsed, ok := authoring.ParseVariableScope(scope) + if !ok { + return fmt.Errorf("invalid --scope %q; options: org, team, testSuite, testCase", scope) + } + opts.Scope = parsed + } + // The service enforces the pairing on listing too (400 INVALID_QUERY); + // checking here gives a precise message before any request. + if err := authoring.ValidateScopePairing(opts.Scope, opts.TestSuiteID, opts.TestCaseID); err != nil { + return err + } + items, total, err := fetchPage(cmd.Context(), page, "variables", func(ctx context.Context, lo authoring.ListOptions) (authoring.List[authoring.Variable], error) { + opts.ListOptions = lo + return variableService.ListVariables(ctx, opts) + }) + if err != nil { + return fmt.Errorf("failed to list variables: %w", err) + } + for i := range items { + blankSecret(&items[i]) + } + if out == JSONOutput { + return renderJSON(authoring.List[authoring.Variable]{Items: items, Total: total}) + } + renderVariableTable(items, total) + return nil + }, + } + + flags := cmd.Flags() + flags.StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + flags.StringVar(&scope, "scope", "", "Filter by scope: org, team, testSuite or testCase.") + flags.StringVar(&opts.TestSuiteID, "test-suite-id", "", "Suite ID. Required with --scope testSuite.") + flags.StringVar(&opts.TestCaseID, "test-case-id", "", "Test case ID. Required with --scope testCase.") + flags.StringVar(&opts.Search, "search", "", "Case-insensitive substring match on the name.") + page.bind(flags) + registerScopeCompletion(cmd) + + return cmd +} + +// registerScopeCompletion completes --scope with the four scopes. +func registerScopeCompletion(cmd *cobra.Command) { + _ = cmd.RegisterFlagCompletionFunc("scope", func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + names := make([]string, len(authoring.AllVariableScopes)) + for i, s := range authoring.AllVariableScopes { + names[i] = string(s) + } + return names, cobra.ShellCompDirectiveNoFileComp + }) +} + +// blankSecret removes a confidential variable's value before rendering, +// regardless of what the service returned, so a service change can never +// leak it (FR-022). +func blankSecret(v *authoring.Variable) { + if v.IsSecret { + v.Value = "" + } +} + +// displayValue renders a variable's value, or the placeholder for secrets. +func displayValue(v authoring.Variable) string { + if v.IsSecret { + return secretPlaceholder + } + return v.Value +} + +// renderVariableTable prints the listing table, or a single line when empty. +// lastUpdate is shown raw because users paste it back as a token. +func renderVariableTable(items []authoring.Variable, total int) { + if len(items) == 0 { + fmt.Printf("No variables found (total: %d).\n", total) + return + } + t := newTable() + t.AppendHeader(table.Row{"ID", "Scope", "Name", "Secret", "Value", "Last Update"}) + for _, v := range items { + t.AppendRow(table.Row{v.ID, v.Scope, v.Name, v.IsSecret, truncate(displayValue(v), 40), v.LastUpdate}) + } + t.AppendFooter(listFooter(len(items), total, "variables")) + fmt.Println(t.Render()) +} + +// VariablesGetCommand is `authoring variables get`. +func VariablesGetCommand() *cobra.Command { + var out string + + cmd := &cobra.Command{ + Use: "get ", + Short: "Show a variable", + Example: ` saucectl authoring variables get 5e7a…`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(out); err != nil { + return err + } + v, err := variableService.GetVariable(cmd.Context(), args[0]) + if err != nil { + return fmt.Errorf("failed to get variable: %w", err) + } + blankSecret(&v) + if out == JSONOutput { + return renderJSON(v) + } + renderVariableDetail(v) + return nil + }, + } + + cmd.Flags().StringVarP(&out, "out", "o", TextOutput, "Output format. Options: text, json.") + + return cmd +} + +// renderVariableDetail prints the two-column property view. +func renderVariableDetail(v authoring.Variable) { + t := newTable() + t.AppendHeader(table.Row{"Property", "Value"}) + t.AppendRow(table.Row{"ID", v.ID}) + t.AppendRow(table.Row{"Name", v.Name}) + t.AppendRow(table.Row{"Scope", v.Scope}) + t.AppendRow(table.Row{"Suite", orDash(v.TestSuiteID)}) + t.AppendRow(table.Row{"Test Case", orDash(v.TestCaseID)}) + t.AppendRow(table.Row{"Description", orDash(v.Description)}) + t.AppendRow(table.Row{"Secret", v.IsSecret}) + t.AppendRow(table.Row{"Value", displayValue(v)}) + t.AppendRow(table.Row{"Created", fmt.Sprintf("%s by %s", humanizeDate(v.CreationDate), orDash(v.CreatorUserName))}) + t.AppendRow(table.Row{"Last Update", v.LastUpdate}) + t.AppendRow(table.Row{"Modified By", orDash(v.LastModifierUserName)}) + fmt.Println(t.Render()) + fmt.Printf("Pass --expected-last-update %q to update or delete against exactly this version.\n", v.LastUpdate) +} diff --git a/internal/cmd/authoring/variables_update.go b/internal/cmd/authoring/variables_update.go new file mode 100644 index 000000000..885334e0f --- /dev/null +++ b/internal/cmd/authoring/variables_update.go @@ -0,0 +1,143 @@ +package authoring + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" +) + +// variableUpdateFlags are the flags of `variables update`. +type variableUpdateFlags struct { + out string + name string + description string + secret bool + expectedLastUpdate string + vs valueSource +} + +// VariablesUpdateCommand is `authoring variables update`. +func VariablesUpdateCommand() *cobra.Command { + var f variableUpdateFlags + + cmd := &cobra.Command{ + Use: "update ", + Short: "Change a variable's name, description, value or secrecy", + Long: `Change a variable. Only the given flags change. + +Changes are guarded against concurrent edits: by default the variable is read first and +its current version is sent along, so a colleague's change made in between is detected +and refused rather than overwritten. Pass --expected-last-update to pin a version you +read earlier instead. There is deliberately no --force: re-reading and re-sending is +already the default.`, + Example: ` saucectl authoring variables update 5e7a… --value-from-env NEW_PASSWORD + saucectl authoring variables update 5e7a… --secret=true + saucectl authoring variables update 5e7a… --description "Rotated 2026-09" --expected-last-update 2026-09-05T14:59:59.000Z`, + SilenceUsage: true, + Args: requireArgs("id"), + PreRun: func(cmd *cobra.Command, _ []string) { + trackUsage(cmd) + }, + RunE: func(cmd *cobra.Command, args []string) error { + if err := validateOutput(f.out); err != nil { + return err + } + return updateVariable(cmd.Context(), cmd, args[0], f) + }, + } + + flags := cmd.Flags() + flags.StringVarP(&f.out, "out", "o", TextOutput, "Output format. Options: text, json.") + flags.StringVar(&f.name, "name", "", "New name: lowercase letters, digits and underscores.") + flags.StringVar(&f.description, "description", "", "New description.") + flags.BoolVar(&f.secret, "secret", false, "Set whether the value is confidential (--secret=true or --secret=false).") + flags.StringVar(&f.expectedLastUpdate, "expected-last-update", "", "The lastUpdate value from an earlier read; the update fails if the variable changed since. Default: read the current value first.") + bindValueSourceFlags(cmd, &f.vs) + + return cmd +} + +// updateVariable assembles the change and applies it with concurrency +// control, surfacing a version conflict as a message that names the command +// to re-read with (FR-024). +func updateVariable(ctx context.Context, cmd *cobra.Command, id string, f variableUpdateFlags) error { + var opts authoring.UpdateVariableOptions + changed := false + + if cmd.Flags().Changed("name") { + if !variableNamePattern.MatchString(f.name) { + return fmt.Errorf("invalid --name %q: use lowercase letters, digits and underscores only", f.name) + } + v := f.name + opts.Name = &v + changed = true + } + if cmd.Flags().Changed("description") { + if len(f.description) > 1000 { + return errors.New("--description must be at most 1000 characters") + } + v := f.description + opts.Description = &v + changed = true + } + if cmd.Flags().Changed("secret") { + v := f.secret + opts.IsSecret = &v + changed = true + } + // The variable is read when we need its concurrency token, and also + // whenever a new value is being supplied: the shell-history warning has + // to key off the *stored* secrecy, because rotating an existing secret + // with --value is precisely the case the warning exists for and + // --secret is not repeated on such a command. + needCurrent := f.expectedLastUpdate == "" || f.vs.set() + var current authoring.Variable + if needCurrent { + var err error + current, err = variableService.GetVariable(ctx, id) + if err != nil { + return fmt.Errorf("failed to read variable before updating: %w", err) + } + } + + if f.vs.set() { + f.vs.secret = current.IsSecret + if cmd.Flags().Changed("secret") { + f.vs.secret = f.vs.secret || f.secret + } + value, err := resolveValue(f.vs, os.Stdin) + if err != nil { + return err + } + opts.Value = &value + changed = true + } + if !changed { + return errors.New("nothing to update: specify --name, --description, --secret or a value source") + } + + token := f.expectedLastUpdate + if token == "" { + token = current.LastUpdate + } + opts.ExpectedLastUpdate = token + + v, err := variableService.UpdateVariable(ctx, id, opts) + if err != nil { + if errors.Is(err, authoring.ErrVariableVersionConflict) { + return fmt.Errorf("variable %s was changed by someone else since it was read; re-read it with 'saucectl authoring variables get %s' and try again", id, id) + } + return fmt.Errorf("failed to update variable: %w", err) + } + blankSecret(&v) + if f.out == JSONOutput { + return renderJSON(v) + } + fmt.Printf("Updated %s variable %q (%s); new version %s.\n", secretWord(v.IsSecret), v.Name, v.ID, v.LastUpdate) + return nil +} diff --git a/internal/cmd/run/authoring.go b/internal/cmd/run/authoring.go new file mode 100644 index 000000000..6ce4558a1 --- /dev/null +++ b/internal/cmd/run/authoring.go @@ -0,0 +1,110 @@ +package run + +import ( + "context" + "fmt" + + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + + "github.com/saucelabs/saucectl/internal/authoring" + cmds "github.com/saucelabs/saucectl/internal/cmd" + "github.com/saucelabs/saucectl/internal/config" + "github.com/saucelabs/saucectl/internal/http" + "github.com/saucelabs/saucectl/internal/region" + "github.com/saucelabs/saucectl/internal/saucecloud" + "github.com/saucelabs/saucectl/internal/usage" +) + +// runAuthoring is the `saucectl run` path for `kind: authoring`: AI-authored +// test cases run through the shared reporters, artifact download, +// concurrency control and exit codes like every other kind. +func runAuthoring(cmd *cobra.Command, isCLIDriven bool) (int, error) { + if !isCLIDriven { + config.ValidateSchema(gFlags.cfgFilePath) + } + + p, err := authoring.FromFile(gFlags.cfgFilePath) + if err != nil { + return 1, err + } + if gFlags.selectedSuite != "" { + if err := authoring.FilterSuites(&p, gFlags.selectedSuite); err != nil { + return 1, err + } + } + authoring.SetDefaults(&p) + if err := authoring.Validate(p); err != nil { + return 1, err + } + if gFlags.failFast { + log.Warn().Msg("--fail-fast is not supported for kind: authoring and will be ignored.") + } + + regio := region.FromString(p.Sauce.Region) + creds := regio.Credentials() + + svc := http.NewAuthoringService(regio, creds, authoringTimeout) + iamClient := http.NewUserService(regio.APIBaseURL(), creds, iamTimeout) + if _, err := authoring.VerifyEntitlement(cmd.Context(), &iamClient, &svc); err != nil { + return 1, err + } + + restoClient := http.NewResto(regio, creds.Username, creds.AccessKey, 0) + rdcClient := http.NewRDCService(regio, creds.Username, creds.AccessKey, rdcTimeout) + jobService := saucecloud.JobService{ + RDC: rdcClient, + Resto: restoClient, + ArtifactDownloadConfig: p.Artifacts.Download, + } + buildService := http.NewBuildService(regio, creds.Username, creds.AccessKey, buildTimeout) + + tracker := usage.DefaultClient + if regio == region.Staging { + tracker.Enabled = false + } + go func() { + tracker.Collect( + cmds.FullName(cmd), + usage.Framework("authoring", ""), + usage.Flags(cmd.Flags()), + usage.SauceConfig(p.Sauce), + usage.Artifacts(p.Artifacts), + usage.NumSuites(len(p.Suites)), + usage.Reporters(p.Reporters), + ) + _ = tracker.Close() + }() + + cleanupArtifacts(p.Artifacts) + + return runAuthoringInCloud(cmd.Context(), p, regio, &svc, jobService, &buildService, &restoClient) +} + +// runAuthoringInCloud wires the runner and executes it. +func runAuthoringInCloud(ctx context.Context, p authoring.Project, regio region.Region, svc *http.AuthoringService, jobService saucecloud.JobService, buildService *http.BuildService, restoClient *http.Resto) (int, error) { + log.Info(). + Str("region", regio.String()). + Str("tunnel", p.Sauce.Tunnel.Name). + Str("build", p.Sauce.Metadata.Build). + Msg("Running AI-authored tests in Sauce Labs.") + + r := authoring.Runner{ + Project: p, + TestCases: svc, + TestSuites: svc, + Artifacts: jobService, + Stopper: jobService, + Builds: buildService, + Tunnels: restoClient, + Region: regio, + Reporters: createReporters(p.Reporters, gFlags.async), + Async: gFlags.async, + } + + exitCode, err := r.RunProject(ctx) + if err != nil { + return exitCode, fmt.Errorf("running AI-authored tests: %w", err) + } + return exitCode, nil +} diff --git a/internal/cmd/run/run.go b/internal/cmd/run/run.go index f28e4eb93..fce32946d 100644 --- a/internal/cmd/run/run.go +++ b/internal/cmd/run/run.go @@ -20,6 +20,7 @@ import ( "github.com/spf13/pflag" "github.com/saucelabs/saucectl/internal/apitest" + "github.com/saucelabs/saucectl/internal/authoring" "github.com/saucelabs/saucectl/internal/config" "github.com/saucelabs/saucectl/internal/credentials" "github.com/saucelabs/saucectl/internal/cucumber" @@ -30,6 +31,7 @@ import ( "github.com/saucelabs/saucectl/internal/msg" "github.com/saucelabs/saucectl/internal/playwright" "github.com/saucelabs/saucectl/internal/puppeteer/replay" + "github.com/saucelabs/saucectl/internal/region" "github.com/saucelabs/saucectl/internal/report" "github.com/saucelabs/saucectl/internal/report/captor" "github.com/saucelabs/saucectl/internal/report/github" @@ -50,6 +52,7 @@ var ( buildTimeout = 10 * time.Second iamTimeout = 10 * time.Second apitestingTimeout = 30 * time.Second + authoringTimeout = 2 * time.Minute typeDef config.TypeDef @@ -102,7 +105,7 @@ func Command() *cobra.Command { cmd.PersistentFlags().BoolVar(&gFlags.failFast, "fail-fast", false, "Stops suites after the first failure") cmd.PersistentFlags().DurationVar(&gFlags.appStoreTimeout, "uploadTimeout", 5*time.Minute, "Upload timeout that limits how long saucectl will wait for an upload to finish. Supports duration values like '10s', '30m' etc.") cmd.PersistentFlags().DurationVar(&gFlags.appStoreTimeout, "upload-timeout", 5*time.Minute, "Upload timeout that limits how long saucectl will wait for an upload to finish. Supports duration values like '10s', '30m' etc.") - sc.StringP("region", "r", "sauce::region", "", "The sauce labs region. Options: us-west-1, eu-central-1.") + sc.StringP("region", "r", "sauce::region", "", fmt.Sprintf("The sauce labs region. Options: %s.", region.Options())) sc.StringToStringP("env", "e", "envFlag", map[string]string{}, "Set environment variables, e.g. -e foo=bar. Not supported for RDC or Espresso on virtual devices!") sc.Bool("show-console-log", "showConsoleLog", false, "Shows suites console.log locally. By default console.log is only shown on failures.") sc.Int("ccy", "sauce::concurrency", 2, "Concurrency specifies how many suites are run at the same time.") @@ -219,6 +222,9 @@ func Run(cmd *cobra.Command) (int, error) { if typeDef.Kind == apitest.Kind { return runApitest(cmd, false) } + if typeDef.Kind == authoring.Kind { + return runAuthoring(cmd, false) + } if typeDef.Kind == cucumber.Kind { return runCucumber(cmd, false) } From 7cbeca2dac6636480dafa78cdca0a31af7693605 Mon Sep 17 00:00:00 2001 From: Vinit Tomar Date: Thu, 17 Sep 2026 18:59:19 +0530 Subject: [PATCH 6/9] Drop spec path from package doc The spec lives in a separate PR, so the pointer would dangle here. Co-Authored-By: Claude Opus 5 (1M context) --- internal/authoring/authoring.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/authoring/authoring.go b/internal/authoring/authoring.go index 622d43d36..435ab516b 100644 --- a/internal/authoring/authoring.go +++ b/internal/authoring/authoring.go @@ -7,7 +7,7 @@ // Everything about the remote service that this package encodes was verified // against the live API rather than taken from its published specification — // several observed behaviours contradict that specification and are recorded -// where the affected code lives (see specs/001-ai-test-authoring/research.md). +// where the affected code lives. package authoring import ( From 322185753cd37aeb01e9e7898f161a665c73aeaa Mon Sep 17 00:00:00 2001 From: Vinit Tomar Date: Tue, 22 Sep 2026 08:58:16 +0530 Subject: [PATCH 7/9] Treat the entitlement lookup as advisory Sauce Labs enforces the AI authoring entitlement on every request, so the CLI's check only buys a clearer error sooner. Failing closed on it meant an entitlements outage would block every authoring command, CI pipelines included, for a verdict the service delivers anyway. A lookup that cannot be completed now warns and continues. Resolving the user stays fatal. Its identity is not only used for this check: schedules create sends it as runningUserId, so continuing with a zero-value user would write bad data rather than skip a check. A definitive "not entitled" stays fatal too, because that answer is trustworthy. Co-Authored-By: Claude Opus 5 (1M context) --- internal/authoring/entitlement.go | 25 +++++- internal/authoring/entitlement_test.go | 120 +++++++++++++++++++++++++ internal/cmd/authoring/helpers_test.go | 15 +++- 3 files changed, 152 insertions(+), 8 deletions(-) create mode 100644 internal/authoring/entitlement_test.go diff --git a/internal/authoring/entitlement.go b/internal/authoring/entitlement.go index ed41f755b..5614c2c84 100644 --- a/internal/authoring/entitlement.go +++ b/internal/authoring/entitlement.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" + "github.com/rs/zerolog/log" + "github.com/saucelabs/saucectl/internal/iam" ) @@ -15,9 +17,22 @@ import ( var ErrNotEntitled = errors.New("AI Test Authoring is not included in your Sauce Labs plan; contact your Sauce Labs account team to enable it") // VerifyEntitlement resolves the caller's organisation and checks that it is -// entitled to AI authoring. It fails closed. Any error other than -// ErrNotEntitled means "could not verify". This costs two serial requests per -// invocation, accepted for the quality of the error (research R-009). +// entitled to AI authoring. It costs two serial requests per invocation, +// accepted for the quality of the error (research R-009). +// +// The two halves fail differently, on purpose. +// +// Resolving the user is fatal when it fails: the identity it returns is not +// only used for this check. Commands such as `schedules create` send it as the +// schedule's runningUserId, so continuing with a zero-value user would write +// bad data rather than merely skip a check. +// +// The entitlement lookup is advisory. Sauce Labs enforces the entitlement on +// every authoring request, so this call only buys a clearer error sooner. When +// it cannot be completed, warn and continue: failing closed here would let an +// outage of the entitlements API block every authoring command — CI pipelines +// included — for a verdict the service delivers anyway. A definitive "not +// entitled" is still fatal, because that answer is trustworthy. func VerifyEntitlement(ctx context.Context, users iam.UserService, ents EntitlementReader) (iam.User, error) { user, err := users.User(ctx) if err != nil { @@ -29,7 +44,9 @@ func VerifyEntitlement(ctx context.Context, users iam.UserService, ents Entitlem enabled, err := ents.IsAIAuthoringEnabled(ctx, user.Organization.ID) if err != nil { - return iam.User{}, fmt.Errorf("could not verify AI authoring entitlement: %w", err) + log.Warn().Err(err).Msg( + "Could not check the AI Test Authoring entitlement; continuing. Sauce Labs will reject the request if your plan does not include it.") + return user, nil } if !enabled { return iam.User{}, ErrNotEntitled diff --git a/internal/authoring/entitlement_test.go b/internal/authoring/entitlement_test.go new file mode 100644 index 000000000..bb8246fa6 --- /dev/null +++ b/internal/authoring/entitlement_test.go @@ -0,0 +1,120 @@ +package authoring + +import ( + "context" + "errors" + "testing" + + "gotest.tools/v3/assert" + + "github.com/saucelabs/saucectl/internal/iam" +) + +// fakeUsers and fakeEntitlements are package-local: internal/mocks imports this +// package, so an in-package test cannot use the shared fakes (Constitution IV). +type fakeUsers struct { + userFn func(ctx context.Context) (iam.User, error) +} + +func (f *fakeUsers) User(ctx context.Context) (iam.User, error) { return f.userFn(ctx) } + +func (f *fakeUsers) Concurrency(context.Context) (iam.Concurrency, error) { + return iam.Concurrency{}, nil +} + +type fakeEntitlements struct { + enabledFn func(ctx context.Context, orgID string) (bool, error) +} + +func (f *fakeEntitlements) IsAIAuthoringEnabled(ctx context.Context, orgID string) (bool, error) { + return f.enabledFn(ctx, orgID) +} + +// entitledUser is a user the gate can resolve: it carries an organisation, so +// the entitlement lookup is reached. +func entitledUser() iam.User { + return iam.User{ID: "u-1", Organization: iam.Organization{ID: "org-1"}} +} + +// TestVerifyEntitlement covers the gate's two failure policies. Resolving the +// user is fatal because its identity is written to schedules; the entitlement +// lookup is advisory because the service enforces it on every request, so an +// entitlements outage must not block authoring commands. +func TestVerifyEntitlement(t *testing.T) { + boom := errors.New("service unavailable") + + tests := []struct { + name string + user func(ctx context.Context) (iam.User, error) + enabled func(ctx context.Context, orgID string) (bool, error) + wantErr string + wantUserID string + }{ + { + name: "entitled", + user: func(context.Context) (iam.User, error) { return entitledUser(), nil }, + enabled: func(context.Context, string) (bool, error) { return true, nil }, + wantUserID: "u-1", + }, + { + name: "not entitled is fatal", + user: func(context.Context) (iam.User, error) { return entitledUser(), nil }, + enabled: func(context.Context, string) (bool, error) { return false, nil }, + wantErr: ErrNotEntitled.Error(), + }, + { + name: "entitlement lookup failure is advisory", + user: func(context.Context) (iam.User, error) { return entitledUser(), nil }, + enabled: func(context.Context, string) (bool, error) { return false, boom }, + wantUserID: "u-1", + }, + { + name: "resolving the user is fatal", + user: func(context.Context) (iam.User, error) { return iam.User{}, boom }, + enabled: func(context.Context, string) (bool, error) { return true, nil }, + wantErr: "could not verify AI authoring entitlement: resolving the current user failed: service unavailable", + }, + { + name: "user without an organisation is fatal", + user: func(context.Context) (iam.User, error) { return iam.User{ID: "u-1"}, nil }, + enabled: func(context.Context, string) (bool, error) { return true, nil }, + wantErr: "could not verify AI authoring entitlement: the current user has no organisation", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + users := &fakeUsers{userFn: tt.user} + ents := &fakeEntitlements{enabledFn: tt.enabled} + + user, err := VerifyEntitlement(context.Background(), users, ents) + + if tt.wantErr != "" { + assert.Error(t, err, tt.wantErr) + return + } + assert.NilError(t, err) + assert.Equal(t, tt.wantUserID, user.ID) + }) + } +} + +// TestVerifyEntitlementAdvisoryKeepsIdentity guards the reason the lookup may +// fail open at all: the caller reuses the resolved user, so continuing must +// still hand back a usable identity rather than a zero value. +func TestVerifyEntitlementAdvisoryKeepsIdentity(t *testing.T) { + users := &fakeUsers{ + userFn: func(context.Context) (iam.User, error) { return entitledUser(), nil }, + } + ents := &fakeEntitlements{ + enabledFn: func(context.Context, string) (bool, error) { + return false, errors.New("entitlements API is down") + }, + } + + user, err := VerifyEntitlement(context.Background(), users, ents) + + assert.NilError(t, err) + assert.Equal(t, "u-1", user.ID) + assert.Equal(t, "org-1", user.Organization.ID) +} diff --git a/internal/cmd/authoring/helpers_test.go b/internal/cmd/authoring/helpers_test.go index d63d20039..e1584cb6f 100644 --- a/internal/cmd/authoring/helpers_test.go +++ b/internal/cmd/authoring/helpers_test.go @@ -367,11 +367,18 @@ func TestVerifyEntitlement(t *testing.T) { } }) - t.Run("could not verify is distinct", func(t *testing.T) { + // A lookup that cannot be completed is advisory: the service enforces the + // entitlement on every request, so an entitlements outage must not block + // authoring commands. The resolved identity still comes back, because + // callers write it to schedules as runningUserId. + t.Run("lookup failure is advisory", func(t *testing.T) { ents := &mocks.AuthoringService{IsAIAuthoringEnabledFn: func(context.Context, string) (bool, error) { return false, errors.New("503") }} - _, err := authoring.VerifyEntitlement(context.Background(), users, ents) - if err == nil || errors.Is(err, authoring.ErrNotEntitled) || !strings.Contains(err.Error(), "could not verify") { - t.Errorf("got %v", err) + u, err := authoring.VerifyEntitlement(context.Background(), users, ents) + if err != nil { + t.Errorf("got %v, want nil", err) + } + if u.ID != "u" { + t.Errorf("identity lost: got %+v", u) } }) From fb65a70def43bae0c8edb891566a928f9d42fc33 Mon Sep 17 00:00:00 2001 From: Vinit Tomar Date: Tue, 22 Sep 2026 09:11:35 +0530 Subject: [PATCH 8/9] Say plainly where the concurrency ceiling gives The comment explained why the weight is capped but left the consequence to inference. Name it: a case weighing more than the whole budget runs alone and still starts all its jobs, so more are in flight than configured while it lasts. Co-Authored-By: Claude Opus 5 (1M context) --- internal/authoring/runner.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/authoring/runner.go b/internal/authoring/runner.go index 6403966d0..f39848650 100644 --- a/internal/authoring/runner.go +++ b/internal/authoring/runner.go @@ -262,6 +262,13 @@ func (r *Runner) runCases(ctx context.Context, cases []ResolvedCase) bool { // organisation's capacity (SC-011). Weight each case by the jobs // it will start, capped at the whole budget so a case needing // more than that still runs — alone — rather than never. + // + // That cap is the one place the ceiling gives: a case weighing + // more than the budget holds every slot, runs by itself, and + // still starts all of its jobs, so more are in flight than + // configured for as long as it lasts. Uncapped it could never + // acquire enough slots and would hang for ever, which is worse. + // SC-011 records the exception. weight := expectedJobs(c) if weight > ccy { weight = ccy From c7cc4c7686f4afcf09926e2fb1bc4d47083155cb Mon Sep 17 00:00:00 2001 From: Vinit Tomar Date: Tue, 22 Sep 2026 10:25:17 +0530 Subject: [PATCH 9/9] Warn when a suite's targets exceed the concurrency limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sauce.concurrency counts Sauce jobs, and a test case starts one job per target, all from a single request. A suite declaring more targets than the limit therefore puts more jobs in flight than configured, silently. Every other kind avoids this by queueing one unit per job — espresso enumerates each device into its own job.StartOptions and feeds a pool of ccy workers, so a suite with five devices still runs ccy at a time. This runner cannot do that yet: splitting the dispatch means sending targets one at a time, and an explicit target overwrites the test case's stored run targets, so a split would rewrite shared org data. Warn for now, at the point both numbers are known and the user can act. Cases carrying their own stored run targets need a request to inspect, so only the configured case is covered. Co-Authored-By: Claude Opus 5 (1M context) --- internal/authoring/config.go | 12 +++++++++ internal/authoring/config_test.go | 45 +++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/internal/authoring/config.go b/internal/authoring/config.go index 8f3e8b372..92e48fce4 100644 --- a/internal/authoring/config.go +++ b/internal/authoring/config.go @@ -188,6 +188,18 @@ func Validate(p Project) error { return fmt.Errorf("suite name %q is used more than once", s.Name) } seen[s.Name] = true + + // One test case starts one job per target, and the service starts them + // all from a single request, so a case with more targets than the + // concurrency limit puts more jobs in flight than configured. Say so + // here, where the numbers are both in hand and the user can act on + // them. Cases carrying their own stored run targets cannot be checked + // without a request, so this catches the configured case only. + if n := len(s.Targets); n > p.Sauce.Concurrency { + log.Warn().Msgf( + "Suite %q declares %d targets but sauce.concurrency is %d: each test case starts one job per target, so up to %d jobs will run at once.", + s.Name, n, p.Sauce.Concurrency, n) + } } if p.Sauce.Retries > 0 { diff --git a/internal/authoring/config_test.go b/internal/authoring/config_test.go index e2c194975..f145146de 100644 --- a/internal/authoring/config_test.go +++ b/internal/authoring/config_test.go @@ -1,6 +1,7 @@ package authoring import ( + "bytes" "encoding/json" "os" "path/filepath" @@ -9,6 +10,9 @@ import ( "time" "unicode/utf8" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/saucelabs/saucectl/internal/config" ) @@ -198,3 +202,44 @@ func TestSetDefaults_TruncatesBuildNameOnRuneBoundary(t *testing.T) { t.Error("truncation split a rune") } } + +// TestValidateWarnsWhenTargetsExceedConcurrency guards the warning that tells a +// user their suite will put more jobs in flight than sauce.concurrency allows. +// The limit counts jobs and a case starts one per target, so the two numbers +// are only comparable once both are known — which is here. +func TestValidateWarnsWhenTargetsExceedConcurrency(t *testing.T) { + target := Target{Capabilities: map[string]interface{}{"browserName": "chrome"}} + + project := func(targets int, ccy int) Project { + p := Project{ + Sauce: config.SauceConfig{Region: "us-west-1", Concurrency: ccy}, + Suites: []Suite{{Name: "s", TestCases: []string{"tc"}}}, + } + for i := 0; i < targets; i++ { + p.Suites[0].Targets = append(p.Suites[0].Targets, target) + } + return p + } + + var buf bytes.Buffer + restore := log.Logger + log.Logger = zerolog.New(&buf) + defer func() { log.Logger = restore }() + + // Over the limit: warn, but still valid. + if err := Validate(project(3, 1)); err != nil { + t.Fatalf("Validate() = %v, want nil", err) + } + if !strings.Contains(buf.String(), "declares 3 targets but sauce.concurrency is 1") { + t.Errorf("no warning for 3 targets under concurrency 1: %s", buf.String()) + } + + // Within the limit: silent. + buf.Reset() + if err := Validate(project(2, 4)); err != nil { + t.Fatalf("Validate() = %v, want nil", err) + } + if strings.Contains(buf.String(), "sauce.concurrency") { + t.Errorf("unexpected warning for 2 targets under concurrency 4: %s", buf.String()) + } +}