SCHED-2206 Make E2E branch agnostic

.github/workflows/e2e_test.yml

@@ -13,6 +13,10 @@ on:
         required: false
         default: ""
         type: string
+      soperator_ref:
+        description: "Soperator ref to use build artifacts from"
+        required: true
+        type: string
       profile_env_var:
         description: "Name of the GitHub variable containing the profile YAML (defaults to env setting)"
         required: false
@@ -162,6 +166,7 @@ jobs:
       PATH_TO_INSTALLATION: "${{ github.workspace }}/terraform-repo/soperator/installations/example"
       TERRAFORM_REPO: "nebius/nebius-solution-library"
       TERRAFORM_REPO_REF: "${{ github.event.inputs.terraform_repo_ref || github.ref_name }}"
+      SOPERATOR_REF: "${{ inputs.soperator_ref }}"
       O11Y_ACCESS_TOKEN: ${{ secrets.E2E_O11Y_ACCESS_TOKEN }}
       PROFILE_ENV_VAR: ${{ inputs.profile_env_var || vars.PROFILE_ENV_VAR }}
       E2E_PROFILE: ${{ vars[inputs.profile_env_var || vars.PROFILE_ENV_VAR] }}
@@ -279,19 +284,19 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          echo "Looking for successful build run on branch: ${{ github.ref_name }}"
+          echo "Looking for successful build run on branch: $SOPERATOR_REF"
 
           run_info=$(gh api -X GET \
             "/repos/${{ github.repository }}/actions/workflows/one_job.yml/runs" \
-            -F branch="${{ github.ref_name }}" \
+            -F branch="$SOPERATOR_REF" \
             -F status=success \
             -F per_page=1 \
             --jq '.workflow_runs[0] | {id: .id, head_sha: .head_sha, created_at: .created_at, html_url: .html_url}')
 
           run_id=$(jq -r '.id' <<<"$run_info")
 
           if [[ "$run_id" == "null" || -z "$run_id" ]]; then
-            echo "::error::No successful build found on branch ${{ github.ref_name }}"
+            echo "::error::No successful build found on branch $SOPERATOR_REF"
             exit 1
           fi
 

.github/workflows/e2e_test_scheduler.yml

@@ -60,8 +60,9 @@ jobs:
             echo "Terraform ref: $TERRAFORM_REF"
 
             gh workflow run e2e_test.yml \
-              --ref "$REF" \
+              --ref main \
               -f profile_env_var="$PROFILE_NAME" \
+              -f soperator_ref="$REF" \
               -f terraform_repo_ref="$TERRAFORM_REF" \
               -f is_scheduled=true
 

VERSION

@@ -1 +1 @@
-4.1.7
+4.2.0

config/manager/kustomization.yaml

@@ -3,4 +3,4 @@ resources:
 images:
   - name: controller
     newName: cr.eu-north1.nebius.cloud/soperator/slurm-operator
-    newTag: 4.1.7
+    newTag: 4.2.0

config/manager/manager.yaml

@@ -84,7 +84,7 @@ spec:
             value: "false"
           - name: SLURM_OPERATOR_WATCH_NAMESPACES
             value: "*"
-        image: controller:4.1.7
+        image: controller:4.2.0
         imagePullPolicy: Always
         name: manager
         securityContext:

config/soperatorchecks/kustomization.yaml

@@ -4,7 +4,7 @@ resources:
 images:
   - name: controller
     newName: cr.eu-north1.nebius.cloud/soperator/soperatorchecks
-    newTag: 4.1.7
+    newTag: 4.2.0
 patches:
   # Serve protected metrics from the manager using controller-runtime authn/authz.
   - path: ./manager_metrics_patch.yaml

e2e/acceptance/README.md

@@ -27,14 +27,33 @@ Run the shared suite against an existing cluster:
 bin/acceptance --kubectl-context <dev-context>
 ```
 
+The target Soperator version controls which version-tagged scenarios are run.
+The CLI resolves it in this order:
+
+1. `--soperator-version`
+2. Flux HelmRelease discovery through the provided kubectl context
+
+Use `--soperator-version` for a manual override, for example
+`--soperator-version 4.2.0`. If the flag is omitted, the CLI uses kubectl to
+discover the version from the target cluster.
+
+Flux discovery reads `status.lastAttemptedRevision` from the standard Soperator
+HelmRelease: `flux-system/flux-system-soperator-fluxcd-soperator`. If
+discovery cannot do that, the CLI fails before running scenarios and asks for
+`--soperator-version`.
+
 All flags:
 
 - `--kubectl-context`: required. All local kubectl calls use this context.
 - `--slurm-cluster-name`: optional, defaults to `soperator`.
+- `--soperator-version`: optional target Soperator version. Full
+  `major.minor.patch` is required, with optional suffix allowed for deployed
+  versions, for example `4.1.5-reb85d0e5`.
 - `--run-unstable`: optional, defaults to `false`; when false, scenarios tagged
   `@unstable` are excluded.
-- `--scenario`: optional. Runs only the scenario at the provided feature path
-  and line, for example `features/internal_ssh.feature:2`. May be repeated.
+- `--scenario`: optional. Runs all compatible scenarios in the provided feature
+  file, or the single scenario at an exact `Scenario:` line, for example
+  `features/internal_ssh.feature:3`. May be repeated.
 - `--report-dir`: optional. When set, the runner writes Cucumber and JUnit
   reports into that directory.
 
@@ -47,7 +66,7 @@ scenarios tagged `@cpu` are excluded.
 For focused manual runs on a dev cluster, pass the scenario location:
 
 ```bash
-bin/acceptance --kubectl-context <dev-context> --scenario features/internal_ssh.feature:2
+bin/acceptance --kubectl-context <dev-context> --scenario features/internal_ssh.feature:3
 ```
 
 The `--scenario` flag is for local/manual investigation only. The GitHub
@@ -76,14 +95,15 @@ import (
 func run(ctx context.Context, kubectlContext string) error {
 	features := acceptance.SharedFeatureSource()
 	features.Paths = []string{
-		"features/internal_ssh.feature:2",
-		"features/topology.feature:2",
+		"features/internal_ssh.feature:3",
+		"features/topology.feature:3",
 	}
 
 	runner, err := acceptance.NewRunner(acceptance.Options{
 		SuiteName:                 "custom-soperator-acceptance",
 		KubectlContext:            kubectlContext,
 		SlurmClusterName:          "soperator",
+		TargetSoperatorVersion:    "4.2.0",
 		Features:                  features,
 		Tags:                      "~@unstable",
 		ExcludeMissingWorkerKinds: true,
@@ -104,7 +124,31 @@ func main() {
 ```
 
 The caller controls scenario selection through `FeatureSource.Paths` and Godog
-tag filtering through `Options.Tags`. Additional runner-specific steps can be
-registered by appending more `StepRegistrar` values.
+tag filtering through `Options.Tags`. The runner always pre-filters scenarios
+by `TargetSoperatorVersion` before invoking Godog.
+
+Every scenario must have exactly one `@soperator_version_...` tag. Version tags
+must be on the scenario itself and use full patch versions only, for example
+`@soperator_version_>=4.2.0`; shorthand forms such as
+`@soperator_version_>=4.2` are intentionally rejected. Comma means AND and `||`
+means OR, for example
+`@soperator_version_>=4.1.0,<4.4.0||>=5.0.0,<6.0.0`.
+
+The runner stores and uses a normalized target version for filtering. Suffixes
+and build metadata are stripped before comparison, so `4.1.5-reb85d0e5` is
+stored and compared as `4.1.5`.
+
+GitHub Actions keeps two refs separate:
+
+1. Soperator/workflow ref: the Soperator repository ref used to run the E2E
+   code and select the latest successful Soperator build artifact to deploy.
+2. Terraform repo ref: the `nebius-solution-library` ref used for cluster
+   creation. It defaults to the Soperator/workflow ref, but can be overridden
+   independently.
+
+The workflow reads the target Soperator version from the selected build artifact
+and uses it for Terraform deployment and scenario filtering. When new tests or
+behavior are added, the scenarios must be tagged with the full version lower
+bound where that behavior exists, for example `@soperator_version_>=4.2.0`.
 
 The shared steps currently assume the Soperator namespace is `soperator`.

e2e/acceptance/features/active_checks.feature

@@ -1,5 +1,5 @@
 Feature: Active checks
-  @gpu
+  @gpu @soperator_version_>=4.2.0
   Scenario: GPU ActiveCheck succeeds on all GPU workers
     Given healthy GPU workers are available for active checks
     When the GPU ActiveCheck is triggered

e2e/acceptance/features/cluster_creation.feature

@@ -1,4 +1,5 @@
 Feature: Cluster creation
+  @soperator_version_>=4.0.0
   Scenario: The provisioned cluster is ready for acceptance tests
     Then all non-job pods in soperator are Running and Ready
     And all HelmReleases are Ready

e2e/acceptance/features/docker_containers.feature

@@ -1,4 +1,5 @@
 Feature: Docker containers
+  @soperator_version_>=4.0.0
   Scenario: Docker container lifecycle uses local storage
     Given a long-running Docker container job is submitted on two workers
     When the Docker container job is running
@@ -8,7 +9,7 @@ Feature: Docker containers
     And Docker containers from the job are stopped explicitly
     Then Docker containers from the job are no longer running
 
-  @gpu
+  @gpu @soperator_version_>=4.0.0
   Scenario: Docker containers can access GPUs
     Given a Docker GPU smoke job is submitted on one GPU worker
     Then the Docker GPU smoke job succeeds and reports visible GPUs

e2e/acceptance/features/enroot_containers.feature

@@ -1,4 +1,5 @@
 Feature: Enroot containers
+  @soperator_version_>=4.0.0
   Scenario: Enroot and Pyxis cache images and clean up runtime state
     Given a long-running Enroot container job is submitted on two workers
     When the Enroot container job is running
@@ -11,7 +12,7 @@ Feature: Enroot containers
     When the repeated Enroot container job is cancelled
     Then Enroot runtime state is cleaned up
 
-  @gpu
+  @gpu @soperator_version_>=4.0.0
   Scenario: Enroot containers can access GPUs
     Given an Enroot GPU smoke job is submitted on one GPU worker
     Then the Enroot GPU smoke job succeeds and reports visible GPUs

e2e/acceptance/features/internal_ssh.feature

@@ -1,4 +1,5 @@
 Feature: Internal SSH
+  @soperator_version_>=4.0.0
   Scenario: A regular user can SSH to a worker without extra options
     Given a regular user account exists on the login node
     When the user SSHs from the login node to a worker

e2e/acceptance/features/node_replacement.feature

@@ -1,6 +1,5 @@
 Feature: Node replacement
-  @gpu
-  @unstable
+  @gpu @unstable @soperator_version_>=4.0.0
   Scenario: A maintenance event replaces the selected worker node
     Given a test job is submitted and running on a GPU worker node
     When a maintenance event is triggered for that node
@@ -10,9 +9,7 @@ Feature: Node replacement
     And a replacement node joins the cluster
     And the replacement node passes GPU validation
 
-  @cpu
-  @gpu
-  @unstable
+  @cpu @gpu @unstable @soperator_version_>=4.2.0
   Scenario: A maintenance event replaces a CPU worker node when CPU and GPU workers exist
     Given a test job is submitted and running on a CPU worker node
     When a maintenance event is triggered for that node

e2e/acceptance/features/observability.feature

@@ -1,3 +1,4 @@
 Feature: Observability stack
+  @soperator_version_>=4.2.0
   Scenario: kube-state-metrics scrape config is consumed by the vm-stack chart
     Then the kube-state-metrics VMServiceScrape carries the soperator scrape endpoints

e2e/acceptance/features/package_installation.feature

@@ -1,5 +1,5 @@
 Feature: Package installation
-  @gpu
+  @gpu @soperator_version_>=4.0.0
   Scenario: Installing jq does not break the NVIDIA driver
     Given the NVIDIA driver is working on a worker node
     When jq is installed on the worker node

e2e/acceptance/features/passive_checks.feature

@@ -1,16 +1,19 @@
 Feature: Passive checks
+  @soperator_version_>=4.2.0
   Scenario: CPU jobs run expected Prolog and Epilog passive checks
     Given a worker is selected
     When a CPU-only Slurm job runs on the selected worker
     Then the CPU job Prolog check runner output is fresh and healthy
     And the CPU job Epilog check runner output is fresh and healthy
     And GPU-only passive checks are not executed for the CPU job
 
+  @soperator_version_>=4.2.0
   Scenario: drop_page_cache runs after CPU jobs
     Given a worker is selected
     When a CPU-only Slurm job runs on the selected worker
     Then the drop_page_cache passive check completed in Epilog
 
+  @soperator_version_>=4.2.0
   Scenario: Passive Prolog drains a worker when allocated memory exceeds available memory
     Given a worker is selected
     When memory pressure is created on the selected worker
@@ -21,21 +24,22 @@ Feature: Passive checks
     Then the selected worker no longer has alloc_mem_used reason
     And the selected worker is usable after alloc_mem_used
 
-  @gpu
+  @gpu @soperator_version_>=4.2.0
   Scenario: GPU jobs run passive GPU health checks
     Given a GPU worker is selected
     When a small GPU Slurm job runs on the selected GPU worker
     Then the GPU job health-check Prolog report is fresh and passing
     And the GPU job health-check Epilog report is fresh and passing
     And raw GPU health-check command outputs are present
 
+  @soperator_version_>=4.2.0
   Scenario: Job tmpfs directory is scoped to the Slurm job lifetime
     Given a worker is selected
     When a Slurm job checks its job tmpfs directory on the selected worker
     Then the job tmpfs directory existed during the job
     And the job tmpfs directory is removed after the job exits
 
-  @gpu
+  @gpu @soperator_version_>=4.2.0
   Scenario: Passive Prolog drains a worker with unmanaged GPU processes
     Given a GPU worker is selected
     When an unmanaged GPU workload is started on the selected GPU worker

e2e/acceptance/features/system_checks.feature

@@ -1,4 +1,5 @@
 Feature: System checks
+  @soperator_version_>=4.2.0
   Scenario: Worker pod ephemeral storage pressure drains and recovers a Slurm node
     Given a healthy worker pod is selected
     When pod-local ephemeral storage is filled above the warning threshold
@@ -9,7 +10,7 @@ Feature: System checks
 
   # TODO: Make soperatorchecks --not-ready-timeout configurable through Helm and
   # set a lower value for e2e clusters, then remove @unstable from this scenario.
-  @unstable
+  @unstable @soperator_version_>=4.2.0
   Scenario: Non-responding kubelet recreates a Kubernetes node and recovers the worker
     Given a healthy worker pod is selected
     When kubelet is stopped on the selected worker Kubernetes node

e2e/acceptance/features/topology.feature

@@ -1,4 +1,5 @@
 Feature: Slurm network topology
+  @soperator_version_>=4.0.0
   Scenario: scontrol topology and SLURM_TOPOLOGY_ADDR agree across workers
     Given the Slurm topology plugin is topology/tree
     When scontrol show topology is parsed into a switch tree

e2e/acceptance/framework/model.go

@@ -12,12 +12,13 @@ type DiscoveredNodeSet struct {
 }
 
 type ClusterState struct {
-	SlurmClusterName   string
-	Workers            []WorkerRef
-	CPUWorkers         []WorkerRef
-	GPUWorkers         []WorkerRef
-	WorkersByNodeSet   map[string][]WorkerRef
-	DiscoveredNodeSets []DiscoveredNodeSet
+	SlurmClusterName       string
+	TargetSoperatorVersion string
+	Workers                []WorkerRef
+	CPUWorkers             []WorkerRef
+	GPUWorkers             []WorkerRef
+	WorkersByNodeSet       map[string][]WorkerRef
+	DiscoveredNodeSets     []DiscoveredNodeSet
 }
 
 func (s *ClusterState) PodName(podName string) string {

e2e/acceptance/framework/version.go

@@ -0,0 +1,31 @@
+package framework
+
+import (
+	"fmt"
+	"strings"
+
+	semver "github.com/Masterminds/semver/v3"
+
+	"nebius.ai/slurm-operator/e2e/versionfilter"
+)
+
+func NormalizeSoperatorVersion(raw string) (string, error) {
+	version, err := versionfilter.NormalizeVersion(raw)
+	if err != nil {
+		return "", fmt.Errorf("Soperator %w", err)
+	}
+	return version, nil
+}
+
+func ParseSoperatorBaseVersion(raw string) (*semver.Version, error) {
+	trimmed := strings.TrimSpace(raw)
+	if trimmed == "" {
+		return nil, fmt.Errorf("Soperator version is required")
+	}
+
+	version, err := versionfilter.ParseBaseVersion(raw)
+	if err != nil {
+		return nil, fmt.Errorf("Soperator %w", err)
+	}
+	return version, nil
+}

e2e/acceptance/framework/version_test.go

@@ -0,0 +1,24 @@
+package framework
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNormalizeSoperatorVersion(t *testing.T) {
+	base, err := NormalizeSoperatorVersion("4.1.5-reb85d0e5")
+	require.NoError(t, err)
+	assert.Equal(t, "4.1.5", base)
+
+	base, err = NormalizeSoperatorVersion("v4.2.0+build.1")
+	require.NoError(t, err)
+	assert.Equal(t, "4.2.0", base)
+}
+
+func TestParseSoperatorBaseVersionRejectsShortVersion(t *testing.T) {
+	_, err := ParseSoperatorBaseVersion("4.2")
+	require.Error(t, err)
+	assert.ErrorContains(t, err, "full major.minor.patch")
+}