Immutable folder support in DABs

acceptance/bundle/validate/immutable_workspace_paths/databricks.yml

@@ -0,0 +1,21 @@
+bundle:
+  name: my-bundle
+  immutable: true
+
+sync:
+  exclude:
+    # Test framework files that are not part of the bundle source.
+    - "repls.json"
+    - "user_repls.json"
+    - "script"
+    - "*.toml"
+
+resources:
+  jobs:
+    my_job:
+      name: my job
+      tasks:
+        - task_key: my_task
+          existing_cluster_id: "0101-120000-aaaaaaaa"
+          spark_python_task:
+            python_file: ./src/main.py

acceptance/bundle/validate/immutable_workspace_paths/output.txt

@@ -0,0 +1,30 @@
+
+>>> [CLI] bundle validate -o json
+Warning: Pattern user_repls.json does not match any files
+  at sync.exclude[1]
+  in databricks.yml:9:7
+
+{
+  "workspace": {
+    "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/artifacts",
+    "current_user": {
+      "domain_friendly_name": "[USERNAME]",
+      "id": "[USERID]",
+      "short_name": "[USERNAME]",
+      "userName": "[USERNAME]"
+    },
+    "file_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/files",
+    "resource_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/resources",
+    "root_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default",
+    "state_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/state"
+  },
+  "tasks": [
+    {
+      "existing_cluster_id": "0101-120000-aaaaaaaa",
+      "spark_python_task": {
+        "python_file": "${workspace.snapshot_path}/src/files/src/main.py"
+      },
+      "task_key": "my_task"
+    }
+  ]
+}

acceptance/bundle/validate/immutable_workspace_paths/script

@@ -0,0 +1 @@
+trace $CLI bundle validate -o json | jq '{workspace: .workspace, tasks: .resources.jobs.my_job.tasks}'

acceptance/bundle/validate/immutable_workspace_paths/src/main.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/validate/immutable_workspace_paths/test.toml

@@ -0,0 +1,3 @@
+Local = true
+Cloud = false
+Ignore = [".databricks"]

bundle/config/bundle.go

@@ -59,4 +59,11 @@ type Bundle struct {
 	// A stable generated UUID for the bundle. This is normally serialized by
 	// Databricks first party template when a user runs bundle init.
 	Uuid string `json:"uuid,omitempty"`
+
+	// Immutable specifies that bundle files and artifacts are uploaded as a single
+	// immutable snapshot rather than being synced individually. When true, the
+	// deployment calls /api/2.0/repos/snapshots with a zip containing all files
+	// and sets workspace.file_path and workspace.artifact_path to the returned
+	// content-addressed path. validate and plan make no mutative API calls.
+	Immutable bool `json:"immutable,omitempty"`
 }

bundle/config/mutator/resolve_variable_references.go

@@ -59,6 +59,11 @@ type resolveVariableReferences struct {
 	includeResources bool
 
 	artifactsReferenceUsed bool
+
+	// excludePaths lists variable reference paths (e.g. "workspace.file_path") whose
+	// resolution should be skipped. References to these paths remain unresolved so a
+	// later mutator can set the value and re-run resolution.
+	excludePaths []string
 }
 
 func ResolveVariableReferencesOnlyResources(prefixes ...string) bundle.Mutator {
@@ -74,6 +79,22 @@ func ResolveVariableReferencesOnlyResources(prefixes ...string) bundle.Mutator {
 	}
 }
 
+// ResolveVariableReferencesOnlyResourcesExcluding resolves variable references in
+// resources while leaving references to the specified paths unresolved.
+// Used by ProcessStaticResources for immutable bundles so that ${workspace.snapshot_path}
+// is not resolved during Initialize; it is resolved in the Deploy phase after
+// snapshot.Upload() sets workspace.snapshot_path to the API-assigned path.
+func ResolveVariableReferencesOnlyResourcesExcluding(excludePaths ...string) bundle.Mutator {
+	return &resolveVariableReferences{
+		prefixes:         defaultPrefixes,
+		lookupFn:         lookup,
+		extraRounds:      maxResolutionRounds - 1,
+		pattern:          dyn.NewPattern(dyn.Key("resources")),
+		includeResources: true,
+		excludePaths:     excludePaths,
+	}
+}
+
 func ResolveVariableReferencesWithoutResources(prefixes ...string) bundle.Mutator {
 	if len(prefixes) == 0 {
 		prefixes = defaultPrefixes
@@ -229,6 +250,9 @@ func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []dyn
 
 				// Perform resolution only if the path starts with one of the specified prefixes.
 				if slices.ContainsFunc(prefixes, path.HasPrefix) {
+					if slices.Contains(m.excludePaths, path.String()) {
+						return dyn.InvalidValue, dynvar.ErrSkipResolution
+					}
 					value, err := m.lookupFn(normalized, path, b)
 					hasUpdates = hasUpdates || (err == nil && value.IsValid())
 					return value, err

bundle/config/mutator/resolve_variable_references_test.go

@@ -6,7 +6,9 @@ import (
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/databricks-sdk-go/service/jobs"
 	"github.com/databricks/databricks-sdk-go/service/pipelines"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -63,3 +65,46 @@ func TestResolveVariableReferencesWithSourceLinkedDeployment(t *testing.T) {
 		testCase.assert(t, b)
 	}
 }
+
+// TestResolveVariableReferencesExcludePaths verifies that paths listed in excludePaths
+// are skipped during resolution and left as unresolved variable references.
+// This is used by ProcessStaticResources for immutable bundles so that
+// ${workspace.file_path} and ${workspace.artifact_path} can be resolved later
+// (in the Build phase, after artifacts are built and the correct snapshot path is known).
+func TestResolveVariableReferencesExcludePaths(t *testing.T) {
+	b := &bundle.Bundle{
+		Config: config.Root{
+			Workspace: config.Workspace{
+				FilePath:     "/snapshot/path/src/files",
+				ArtifactPath: "/snapshot/path/src/artifacts",
+			},
+			Resources: config.Resources{
+				Jobs: map[string]*resources.Job{
+					"job1": {
+						JobSettings: jobs.JobSettings{
+							Tasks: []jobs.Task{
+								{
+									SparkPythonTask: &jobs.SparkPythonTask{
+										PythonFile: "${workspace.file_path}/main.py",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	// With exclusion: ${workspace.file_path} should remain unresolved.
+	diags := bundle.Apply(t.Context(), b, ResolveVariableReferencesOnlyResourcesExcluding("workspace.file_path", "workspace.artifact_path"))
+	require.NoError(t, diags.Error())
+	assert.Equal(t, "${workspace.file_path}/main.py", b.Config.Resources.Jobs["job1"].Tasks[0].SparkPythonTask.PythonFile,
+		"reference should remain unresolved when path is excluded")
+
+	// Without exclusion: ${workspace.file_path} should resolve normally.
+	diags = bundle.Apply(t.Context(), b, ResolveVariableReferencesOnlyResources())
+	require.NoError(t, diags.Error())
+	assert.Equal(t, "/snapshot/path/src/files/main.py", b.Config.Resources.Jobs["job1"].Tasks[0].SparkPythonTask.PythonFile,
+		"reference should resolve after exclusion is lifted")
+}

bundle/config/mutator/resourcemutator/process_static_resources.go

@@ -38,13 +38,26 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia
 	// we need to resolve variables because they can change path values:
 	// - variable can be used a prefix
 	// - path can be part of a complex variable value
+
+	// For immutable bundles, defer resolving ${workspace.snapshot_path} in resources.
+	// The actual snapshot path is only known after snapshot.Upload() returns the
+	// API-assigned path in the deploy phase.
+	var resourceResolver bundle.Mutator
+	if b.Config.Bundle.Immutable {
+		resourceResolver = mutator.ResolveVariableReferencesOnlyResourcesExcluding(
+			"workspace.snapshot_path",
+		)
+	} else {
+		resourceResolver = mutator.ResolveVariableReferencesOnlyResources()
+	}
+
 	bundle.ApplySeqContext(
 		ctx,
 		b,
 		// Reads (dynamic): * (strings) (searches for variable references in string values)
 		// Updates (dynamic): resources.* (strings) (resolves variable references to their actual values)
 		// Resolves variable references in 'resources' using bundle, workspace, and variables prefixes
-		mutator.ResolveVariableReferencesOnlyResources(),
+		resourceResolver,
 		// After normal variable resolution, log all ${resources.*} references
 		mutator.LogResourceReferences(),
 		mutator.NormalizePaths(),

bundle/config/mutator/resourcemutator/resource_mutator.go

@@ -127,6 +127,19 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) {
 	)
 }
 
+// resourceVarResolver returns a mutator that resolves variable references in
+// resources. For immutable bundles, ${workspace.file_path} and
+// ${workspace.artifact_path} are excluded: the API assigns the snapshot path
+// after upload, so they must remain as-is until snapshot.Upload() has run.
+func resourceVarResolver(b *bundle.Bundle) bundle.Mutator {
+	if b.Config.Bundle.Immutable {
+		return mutator.ResolveVariableReferencesOnlyResourcesExcluding(
+			"workspace.file_path", "workspace.artifact_path",
+		)
+	}
+	return mutator.ResolveVariableReferencesOnlyResources()
+}
+
 // Normalization is applied multiple times if resource is modified during initialization
 //
 // If bundle is modified outside of 'resources' section, these changes are discarded.
@@ -139,8 +152,10 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) {
 
 		// Reads (dynamic): * (strings) (searches for variable references in string values)
 		// Updates (dynamic): resources.* (strings) (resolves variable references to their actual values)
-		// Resolves variable references in 'resources' using bundle, workspace, and variables prefixes
-		mutator.ResolveVariableReferencesOnlyResources(),
+		// Resolves variable references in 'resources' using bundle, workspace, and variables prefixes.
+		// For immutable bundles, ${workspace.file_path} and ${workspace.artifact_path} are left
+		// unresolved: the actual snapshot path is assigned by the API after upload, not pre-computed.
+		resourceVarResolver(b),
 
 		// Reads (dynamic): resources.pipelines.*.libraries (checks for notebook.path and file.path fields)
 		// Updates (dynamic): resources.pipelines.*.libraries (expands glob patterns in path fields to multiple library entries)

bundle/config/mutator/translate_paths.go

@@ -320,12 +320,21 @@ func (t *translateContext) rewriteValue(ctx context.Context, p dyn.Path, v dyn.V
 	return dyn.NewValue(out, v.Locations()), nil
 }
 
+// snapshotFilesRoot is the remote root used for file/notebook path translation
+// in immutable bundles. References to this placeholder are resolved after
+// snapshot.Upload() sets workspace.snapshot_path to the API-assigned path.
+const snapshotFilesRoot = "${workspace.snapshot_path}/src/files"
+
 func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []func(context.Context, dyn.Value) (dyn.Value, error)) diag.Diagnostics {
-	// Set the remote root to the sync root if source-linked deployment is enabled.
-	// Otherwise, set it to the workspace file path.
-	if config.IsExplicitlyEnabled(t.b.Config.Presets.SourceLinkedDeployment) {
+	switch {
+	case b.Config.Bundle.Immutable:
+		// Use a placeholder root that is resolved after snapshot.Upload() sets
+		// workspace.snapshot_path. This defers path computation until the actual
+		// content-addressed path is known.
+		t.remoteRoot = snapshotFilesRoot
+	case config.IsExplicitlyEnabled(t.b.Config.Presets.SourceLinkedDeployment):
 		t.remoteRoot = t.b.SyncRootPath
-	} else {
+	default:
 		t.remoteRoot = t.b.Config.Workspace.FilePath
 	}
 

bundle/config/workspace.go

@@ -80,6 +80,12 @@ type Workspace struct {
 	// Remote workspace path for deployment state.
 	// This defaults to "${workspace.root}/state".
 	StatePath string `json:"state_path,omitempty"`
+
+	// SnapshotPath is the workspace path of the immutable snapshot uploaded during
+	// deployment. It is set by snapshot.Upload() and used to resolve
+	// ${workspace.snapshot_path} references in resource configurations.
+	// Only populated for bundles with bundle.immutable = true.
+	SnapshotPath string `json:"snapshot_path,omitempty" bundle:"internal"`
 }
 
 type User struct {

bundle/deploy/metadata/compute.go

@@ -105,6 +105,7 @@ func (m *compute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics
 
 	// Set file upload destination of the bundle in metadata
 	b.Metadata.Config.Workspace.FilePath = b.Config.Workspace.FilePath
+	b.Metadata.Config.Workspace.SnapshotPath = b.Config.Workspace.SnapshotPath
 	// In source-linked deployment files are not copied and resources use source files, therefore we use sync path as file path in metadata
 	if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) {
 		b.Metadata.Config.Workspace.FilePath = b.SyncRootPath

bundle/deploy/metadata/load.go

@@ -0,0 +1,57 @@
+package metadata
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/metadata"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/filer"
+)
+
+type load struct{}
+
+// Load reads the metadata file written during the last deploy and populates
+// fields on the bundle that are not available locally (e.g. workspace.snapshot_path
+// for immutable bundles, which is only known after snapshot.Upload() ran).
+func Load() bundle.Mutator {
+	return &load{}
+}
+
+func (m *load) Name() string {
+	return "metadata.Load"
+}
+
+func (m *load) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	f, err := filer.NewWorkspaceFilesClient(b.WorkspaceClient(ctx), b.Config.Workspace.StatePath)
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	r, err := f.Read(ctx, metadataFileName)
+	if err != nil {
+		// Missing metadata file means the bundle was never deployed or was
+		// deployed by an older CLI version that didn't write metadata. Treat
+		// it as a no-op so destroy can still proceed.
+		return nil
+	}
+	defer r.Close()
+
+	raw, err := io.ReadAll(r)
+	if err != nil {
+		return diag.FromErr(err)
+	}
+
+	var md metadata.Metadata
+	if err := json.Unmarshal(raw, &md); err != nil {
+		return diag.FromErr(err)
+	}
+
+	if md.Config.Workspace.SnapshotPath != "" {
+		b.Config.Workspace.SnapshotPath = md.Config.Workspace.SnapshotPath
+	}
+
+	return nil
+}