direct: Fix processing of configs with dots in map keys (#4977)

denik · web-flow · commit 00de70c47c12 · 2026-04-15T13:04:41.000Z
## Changes When calculating the plan, direct engine users dyn.Walk to figure out where remaining references are. It is used dyn.Path.String() method which lost information for configuration with dots in map keys. Fixed the plan to convert dyn.Path to structpath.PathNode directly which preserves nodes. ## Why Fixes #4960 ## Tests New dedicate test that shows that this type of config did not work on direct engine (even without migration). New invariant test config with dots in maps keys to test the migration and other scenarios.
diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md
@@ -15,6 +15,7 @@
 * Fix resource references not correctly resolved in apps config section ([#4964](https://github.com/databricks/cli/pull/4964))
 * Allow run_as for dashboards with embed_credentials set to false ([#4961](https://github.com/databricks/cli/pull/4961))
 * direct: Pass changed fields into update mask for apps instead of wildcard ([#4963](https://github.com/databricks/cli/pull/4963))
+* engine/direct: Fix deploy of configurations with dots in maps keys ([#4977](https://github.com/databricks/cli/pull/4977))
 
 ### Dependency updates
 
diff --git a/acceptance/bundle/deploy/pipeline-config-dots/databricks.yml b/acceptance/bundle/deploy/pipeline-config-dots/databricks.yml
@@ -0,0 +1,24 @@
+bundle:
+  name: test-bundle
+
+variables:
+  AZURE:
+    type: complex
+    default:
+      subscription: test-sub-123
+
+resources:
+  schemas:
+    my_schema:
+      catalog_name: main
+      name: test-schema
+
+  pipelines:
+    my_pipeline:
+      name: test-pipeline
+      libraries:
+        - file:
+            path: pipeline.py
+      configuration:
+        europris.swipe.egress_streaming_schema: "${resources.schemas.my_schema.catalog_name}.${resources.schemas.my_schema.name}"
+        europris.azure.subscription: ${var.AZURE.subscription}
diff --git a/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml b/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml
diff --git a/acceptance/bundle/deploy/pipeline-config-dots/output.txt b/acceptance/bundle/deploy/pipeline-config-dots/output.txt
@@ -0,0 +1,32 @@
+
+>>> [CLI] bundle validate
+Name: test-bundle
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Validation OK!
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.pipelines.my_pipeline
+  delete resources.schemas.my_schema
+
+This action will result in the deletion of the following UC schemas. Any underlying data may be lost:
+  delete resources.schemas.my_schema
+
+This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the
+Streaming Tables (STs) and Materialized Views (MVs) managed by them:
+  delete resources.pipelines.my_pipeline
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default
+
+Deleting files...
+Destroy complete!
diff --git a/acceptance/bundle/deploy/pipeline-config-dots/pipeline.py b/acceptance/bundle/deploy/pipeline-config-dots/pipeline.py
@@ -0,0 +1 @@
+pass
diff --git a/acceptance/bundle/deploy/pipeline-config-dots/script b/acceptance/bundle/deploy/pipeline-config-dots/script
@@ -0,0 +1,8 @@
+trace $CLI bundle validate
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+trace $CLI bundle deploy
diff --git a/acceptance/bundle/invariant/configs/pipeline_config_dots.yml.tmpl b/acceptance/bundle/invariant/configs/pipeline_config_dots.yml.tmpl
@@ -0,0 +1,24 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+variables:
+  AZURE:
+    type: complex
+    default:
+      subscription: test-sub-123
+
+resources:
+  schemas:
+    my_schema:
+      catalog_name: main
+      name: test-schema-$UNIQUE_NAME
+
+  pipelines:
+    my_pipeline:
+      name: test-pipeline-$UNIQUE_NAME
+      libraries:
+        - file:
+            path: pipeline.py
+      configuration:
+        europris.swipe.egress_streaming_schema: "${resources.schemas.my_schema.catalog_name}.${resources.schemas.my_schema.name}"
+        europris.azure.subscription: ${var.AZURE.subscription}
diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml
diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml
@@ -5,3 +5,6 @@ EnvMatrixExclude.no_grant_ref = ["INPUT_CONFIG=schema_grant_ref.yml.tmpl"]
 
 # Model permissions did not work until 0.297.0 https://github.com/databricks/cli/pull/4941
 EnvMatrixExclude.no_model_with_permissions = ["INPUT_CONFIG=model_with_permissions.yml.tmpl"]
+
+# Dotted pipeline configuration keys are not supported on v0.293.0
+EnvMatrixExclude.no_pipeline_config_dots = ["INPUT_CONFIG=pipeline_config_dots.yml.tmpl"]
diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml
diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml
diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml
@@ -41,6 +41,7 @@ EnvMatrix.INPUT_CONFIG = [
     "model_with_permissions.yml.tmpl",
     "model_serving_endpoint.yml.tmpl",
     "pipeline.yml.tmpl",
+    "pipeline_config_dots.yml.tmpl",
     "postgres_branch.yml.tmpl",
     "postgres_endpoint.yml.tmpl",
     "postgres_project.yml.tmpl",
diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go
@@ -949,8 +949,11 @@ func extractReferences(root dyn.Value, node string) (map[string]string, error) {
 		if !ok {
 			return nil
 		}
-		// Store the original string that contains references, not individual references
-		refs[p.String()] = ref.Str
+		// Store the original string that contains references, not individual references.
+		// Convert dyn.Path to structpath string because refs are later parsed by structpath.ParsePath.
+		// dyn.Path.String() uses dot notation which is ambiguous for keys containing dots;
+		// structpath uses bracket notation (['key.with.dots']) which round-trips correctly.
+		refs[dynPathToStructPath(p).String()] = ref.Str
 		return nil
 	})
 	if err != nil {
@@ -959,6 +962,19 @@ func extractReferences(root dyn.Value, node string) (map[string]string, error) {
 	return refs, nil
 }
 
+// dynPathToStructPath converts a dyn.Path to a structpath.PathNode.
+func dynPathToStructPath(p dyn.Path) *structpath.PathNode {
+	var node *structpath.PathNode
+	for _, c := range p {
+		if key := c.Key(); key != "" {
+			node = structpath.NewStringKey(node, key)
+		} else {
+			node = structpath.NewIndex(node, c.Index())
+		}
+	}
+	return node
+}
+
 func (b *DeploymentBundle) getAdapterForKey(resourceKey string) (*dresources.Adapter, error) {
 	group := config.GetResourceTypeFromKey(resourceKey)
 	if group == "" {
diff --git a/bundle/direct/bundle_plan_test.go b/bundle/direct/bundle_plan_test.go
@@ -0,0 +1,37 @@
+package direct
+
+import (
+	"testing"
+
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDynPathToStructPath(t *testing.T) {
+	tests := []struct {
+		path     dyn.Path
+		expected string
+	}{
+		{
+			path:     dyn.NewPath(dyn.Key("foo"), dyn.Key("bar")),
+			expected: "foo.bar",
+		},
+		{
+			path:     dyn.NewPath(dyn.Key("foo"), dyn.Index(1), dyn.Key("bar")),
+			expected: "foo[1].bar",
+		},
+		{
+			path:     dyn.NewPath(dyn.Key("configuration"), dyn.Key("europris.swipe.egress_streaming_schema")),
+			expected: "configuration['europris.swipe.egress_streaming_schema']",
+		},
+		{
+			path:     dyn.NewPath(dyn.Key("tags"), dyn.Key("it's.here")),
+			expected: "tags['it''s.here']",
+		},
+	}
+
+	for _, tc := range tests {
+		node := dynPathToStructPath(tc.path)
+		assert.Equal(t, tc.expected, node.String())
+	}
+}
