From 4b77a5eb9553fec3e21f9dcd1d8a9fa09aa74359 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 13 Aug 2026 17:30:13 +0200 Subject: [PATCH 1/3] acc: merge repls.json into ACC_REPLS and move it out of the test directory Both files were created inside the test directory, so bundle deploy uploaded them to the workspace. There is now a single file at $ACC_REPLS, outside the test directory: the harness writes its replacements there as one JSON object per line, add_repl.py appends ":" lines as before. Co-authored-by: Isaac --- .agent/rules/testing.md | 4 +- acceptance/acceptance_test.go | 56 +++++++++------- acceptance/bin/add_repl.py | 8 ++- acceptance/bin/diff.py | 25 ++----- acceptance/bin/repls.py | 65 +++++++++++++++++++ acceptance/bin/sort_lines.py | 24 ++----- acceptance/bundle/apps/git_source/script | 2 +- .../artifacts/whl_change_version/output.txt | 1 - .../config-remote-sync/job_fields/output.txt | 4 +- .../config-remote-sync/job_fields/script | 4 +- .../multiple_resources/script | 4 +- .../deploy/wal/corrupted-wal-entry/script | 4 +- .../bundle/deployment/bind/alert/script | 2 +- .../bind/dashboard/recreation/script | 4 +- .../bind/job/already-managed-different/script | 2 +- .../deployment/bind/job/job-abort-bind/script | 2 +- .../deployment/bind/job/stale-state/script | 2 +- .../deployment/bind/pipelines/recreate/script | 2 +- .../deployment/bind/pipelines/update/script | 2 +- .../lineage-mismatch-after-redeploy/script | 4 +- acceptance/bundle/generate/alert/script | 2 +- .../generate/pipeline_and_deploy/script | 2 +- .../generate/python_job_and_deploy/script | 2 +- .../auto-migrate-empty-tfstate/output.txt | 1 - acceptance/bundle/python/script.prepare | 2 +- .../bundle/resource_deps/model_id_ref/script | 2 +- .../bundle/resources/alerts/basic/script | 2 +- .../bundle/resources/alerts/with_file/script | 2 +- .../resources/apps/inline_config/script | 2 +- .../deploy/update-and-resize-autoscale/script | 2 +- .../clusters/deploy/update-and-resize/script | 2 +- .../resize-terminated-fallback/script | 2 +- .../change-embed-credentials/script | 4 +- .../resources/dashboards/change-name/script | 4 +- .../dashboards/change-parent-path/script | 4 +- .../change-serialized-dashboard/script | 2 +- .../dashboards/dataset-catalog-schema/script | 6 +- .../delete-trashed-out-of-band/script | 6 +- .../resources/dashboards/destroy/script | 2 +- .../resources/dashboards/detect-change/script | 6 +- .../dashboards/nested-folders/script | 2 +- .../publish-failure-stale-content/script | 6 +- .../bundle/resources/dashboards/simple/script | 6 +- .../simple_outside_bundle_root/script | 2 +- .../dashboards/simple_syncroot/script | 2 +- .../dashboards/unpublish-out-of-band/script | 2 +- .../resources/genie_spaces/inline/script | 2 +- .../genie_spaces/recreate_when_gone/script | 2 +- .../resources/genie_spaces/simple/script | 2 +- .../bundle/resources/jobs/alert-task/script | 2 +- .../resources/jobs/check-metadata/script | 4 +- .../model_serving_endpoints/basic/script | 8 +-- .../drift/recreated_same_name/script | 4 +- .../recreate/catalog-name/script | 4 +- .../recreate/name-change/script | 4 +- .../recreate/route-optimized/script | 2 +- .../recreate/schema-name/script | 4 +- .../recreate/table-prefix/script | 4 +- .../update/ai-gateway/script | 2 +- .../update/both_gateway_and_tags/script | 2 +- .../update/config/script | 2 +- .../update/email-notifications/script | 2 +- .../update/tags/script | 2 +- .../models/readplan-permissions/script | 2 +- .../resources/permissions/factcheck/script | 4 +- .../genie_spaces/out_of_band_deletion/script | 2 +- .../with_permissions/script | 2 +- .../without_permissions/script | 2 +- .../models/current_can_manage/script | 2 +- .../secret_scopes/permissions/script | 2 +- .../lifecycle-started-edit/script | 2 +- .../lifecycle-started-toggle/script | 2 +- .../sql_warehouses/lifecycle-started/script | 2 +- .../drift/recreated_same_name/script | 4 +- acceptance/bundle/sync/dryrun/script | 1 - acceptance/bundle/sync/script | 1 - .../record-deployment-history/script | 2 +- .../immutable_workspace_paths/databricks.yml | 2 - .../immutable_workspace_paths/output.txt | 4 -- acceptance/bundle/variables/lookup/script | 2 +- acceptance/cmd/completion/script | 2 +- .../cmd/sync-from-file/gitignore.test-fixture | 1 - .../cmd/sync/dryrun-missing-remote/script | 1 - acceptance/cmd/sync/dryrun/script | 1 - acceptance/cmd/sync/script | 1 - .../cmd/workspace/apps/run-local-node/script | 4 +- .../cmd/workspace/apps/run-local/script | 4 +- acceptance/script.prepare | 6 ++ acceptance/selftest/acc_repls/output.txt | 4 +- acceptance/selftest/acc_repls/script | 4 +- acceptance/workspace/lakeview/publish/script | 2 +- 91 files changed, 230 insertions(+), 188 deletions(-) create mode 100644 acceptance/bin/repls.py diff --git a/.agent/rules/testing.md b/.agent/rules/testing.md index 698b69671ce..c6d051613c4 100644 --- a/.agent/rules/testing.md +++ b/.agent/rules/testing.md @@ -154,7 +154,7 @@ GOOD: ```bash trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to delete, 1 to update" trace print_requests.py //api/2.0/apps -echo "$deployment_id:DEPLOYMENT_ID" >> ACC_REPLS +add_repl "$deployment_id" DEPLOYMENT_ID ``` BAD: @@ -169,7 +169,7 @@ Available on `PATH` during test execution (from `acceptance/bin/`): - `print_requests.py //path [^//exclude] [--get] [--sort] [--unique] [--oneline] [--keep]`: print recorded HTTP requests matching path filters. Requires `RecordRequests = true` in `test.toml`. Excludes GET by default (`--get` includes them); clears `out.requests.txt` afterwards (`--keep` retains it). `^` prefix excludes a path; multiple positive filters are OR'd together. `--sort` orders output deterministically (use when the request set is order-independent), `--unique` collapses consecutive duplicates (e.g. repeated polls), `--oneline` prints one request per line. - `replace_ids.py [-t TARGET]`: read deployment state and add `[NAME_ID]` replacements for all resource IDs. - `read_id.py [-t TARGET] NAME`: read ID of a single resource from state, print it, and add a `[NAME_ID]` replacement. -- `add_repl.py VALUE REPLACEMENT`: add a custom replacement (VALUE will be replaced with `[REPLACEMENT]` in output). +- `add_repl VALUE REPLACEMENT`: add a custom replacement (VALUE will be replaced with `[REPLACEMENT]` in output). Wraps `add_repl.py`, which appends to `$ACC_REPLS` — the file holding every replacement applied to the output, read back by the harness and by `diff.py` / `sort_lines.py --repl`. - `update_file.py FILENAME OLD NEW`: replace all occurrences of OLD with NEW in FILENAME. Errors if OLD is not found. Cannot be used on `output.txt`. - `find.py REGEX [--expect N]`: find files matching regex in current directory. `--expect N` asserts an exact count. - `diff.py DIR1 DIR2` or `diff.py FILE1 FILE2`: recursive diff with test replacements applied. diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 16fd53a4785..ead8b7cf378 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -93,8 +93,6 @@ const ( CleanupScript = "script.cleanup" PrepareScript = "script.prepare" MaxFileSize = 1_000_000 - // Filename to save replacements to (used by diff.py) - ReplsFile = "repls.json" // Filename for materialized config (used as golden file) MaterializedConfigFile = "out.test.toml" @@ -103,11 +101,18 @@ const ( // The tests the don't set SERVERLESS variable or set to empty string will also be run. EnvFilterVar = "ENVFILTER" - // File where scripts can output custom replacements - // export $job_id=100200300 - // $ echo "$job_id:MY_JOB" >> ACC_REPLS # This will replace 100200300 with [MY_JOB] in the output - // TODO: this should be merged with repls.json functionality, currently these replacements are not parsed by diff.py - userReplacementsFilename = "ACC_REPLS" + // Env var with the path to the file holding all replacements applied to the output. + // It is kept outside of the test directory, otherwise "bundle deploy" uploads it. + // + // The harness writes its own replacements there, one JSON object per line. Scripts add + // literal ones with the add_repl helper, which appends ":" lines: + // + // $ job_id=100200300 + // $ add_repl "$job_id" MY_JOB # replaces 100200300 with [MY_JOB] in the output + // + // Both kinds are read back here (see loadUserReplacements) and by the python helpers + // (see bin/repls.py). + ReplsEnvVar = "ACC_REPLS" ) var ApplyCITimeoutMultipler = os.Getenv("GITHUB_WORKFLOW") != "" @@ -125,11 +130,6 @@ var Scripts = map[string]bool{ PrepareScript: true, } -var Ignored = map[string]bool{ - ReplsFile: true, - userReplacementsFilename: true, -} - func TestAccept(t *testing.T) { testAccept(t, InprocessMode, "") } @@ -888,6 +888,9 @@ func runTest(t *testing.T, cmd.Env = append(cmd.Env, "UNIQUE_NAME="+uniqueName) cmd.Env = append(cmd.Env, "TEST_TMP_DIR="+tmpDir) + replsPath := filepath.Join(t.TempDir(), ReplsEnvVar) + cmd.Env = append(cmd.Env, ReplsEnvVar+"="+replsPath) + // populate CLOUD_ENV_BASE envBase := getCloudEnvBase(cloudEnv) cmd.Env = append(cmd.Env, "CLOUD_ENV_BASE="+envBase) @@ -898,10 +901,16 @@ func runTest(t *testing.T, // User replacements: repls.Repls = append(repls.Repls, config.Repls...) - // Save replacements to temp test directory so that it can be read by diff.py - replsJson, err := json.MarshalIndent(repls.Repls, "", " ") - require.NoError(t, err) - testutil.WriteFile(t, filepath.Join(tmpDir, ReplsFile), string(replsJson)) + // Save replacements so that they can be read by the scripts (diff.py, sort_lines.py). + // One JSON object per line, because scripts append their own replacements to this file. + var replsLines strings.Builder + for _, repl := range repls.Repls { + line, err := json.Marshal(repl) + require.NoError(t, err) + replsLines.Write(line) + replsLines.WriteByte('\n') + } + testutil.WriteFile(t, replsPath, replsLines.String()) if coverDir != "" { // Creating individual coverage directory for each test, because writing to the same one @@ -1003,7 +1012,7 @@ func runTest(t *testing.T, formatOutput(out, err) require.NoError(t, out.Close()) - loadUserReplacements(t, &repls, tmpDir) + loadUserReplacements(t, &repls, replsPath) printedRepls := false @@ -1028,9 +1037,6 @@ func runTest(t *testing.T, if _, ok := outputs[relPath]; ok { continue } - if _, ok := Ignored[relPath]; ok { - continue - } if config.CompiledIgnoreObject.MatchesPath(relPath) && !strings.HasPrefix(relPath, "out") { continue } @@ -1810,8 +1816,10 @@ func setupTerraform(t *testing.T, cwd, buildDir string, repls *testdiff.Replacem repls.SetPath(terraformExecPath, "[TERRAFORM]") } -func loadUserReplacements(t *testing.T, repls *testdiff.ReplacementsContext, tmpDir string) { - b, err := os.ReadFile(filepath.Join(tmpDir, userReplacementsFilename)) +// loadUserReplacements adds replacements appended by the scripts to replsPath. +// The JSON lines written there by the harness itself are already part of repls. +func loadUserReplacements(t *testing.T, repls *testdiff.ReplacementsContext, replsPath string) { + b, err := os.ReadFile(replsPath) if errors.Is(err, fs.ErrNotExist) { return } @@ -1819,12 +1827,12 @@ func loadUserReplacements(t *testing.T, repls *testdiff.ReplacementsContext, tmp lines := strings.SplitSeq(string(b), "\n") for line := range lines { line = strings.TrimSpace(line) - if len(line) == 0 { + if len(line) == 0 || strings.HasPrefix(line, "{") { continue } items := strings.Split(line, ":") if len(items) <= 1 { - t.Errorf("Error parsing %s: %#v", userReplacementsFilename, line) + t.Errorf("Error parsing %s: %#v", ReplsEnvVar, line) continue } repl := items[len(items)-1] diff --git a/acceptance/bin/add_repl.py b/acceptance/bin/add_repl.py index b772584e532..231e99eed1b 100755 --- a/acceptance/bin/add_repl.py +++ b/acceptance/bin/add_repl.py @@ -9,14 +9,18 @@ import os from pathlib import Path -ACC_REPLS = Path(os.environ["TEST_TMP_DIR"]) / "ACC_REPLS" +ACC_REPLS = Path(os.environ["ACC_REPLS"]) def get_repls(): result = {} if ACC_REPLS.exists(): for line in ACC_REPLS.open(): - value, repl = line.strip().rsplit(":", 1) + line = line.strip() + # Skip the harness replacements; only the records added here can collide. + if not line or line.startswith("{"): + continue + value, repl = line.rsplit(":", 1) result[repl] = value return result diff --git a/acceptance/bin/diff.py b/acceptance/bin/diff.py index 84b2cdde5a6..93c75ef882b 100755 --- a/acceptance/bin/diff.py +++ b/acceptance/bin/diff.py @@ -2,33 +2,18 @@ """This script implements "diff -r -U2 dir1 dir2" but applies replacements first""" import difflib -import json -import os -import re import sys from pathlib import Path - -def replaceAll(patterns, s): - for comp, new in patterns: - s = comp.sub(new, s) - return s +sys.path.insert(0, str(Path(__file__).parent)) +from repls import compile_repls, replace_all def main(): d1, d2 = sys.argv[1:] d1, d2 = Path(d1), Path(d2) - repls_json = Path(os.environ["TEST_TMP_DIR"]) / "repls.json" - repls = json.loads(repls_json.read_text()) - - patterns = [] - for r in repls: - try: - c = re.compile(r["Old"]) - patterns.append((c, r["New"])) - except re.error as e: - print(f"Regex error for pattern {r}: {e}", file=sys.stderr) + patterns = compile_repls() if d1.is_dir() and d2.is_dir(): files1 = [str(p.relative_to(d1)) for p in d1.rglob("*") if p.is_file() and not p.name.startswith("LOG")] @@ -54,8 +39,8 @@ def main(): def diff_files(patterns, p1, p2): - a = replaceAll(patterns, p1.read_text()).splitlines(True) - b = replaceAll(patterns, p2.read_text()).splitlines(True) + a = replace_all(patterns, p1.read_text()).splitlines(True) + b = replace_all(patterns, p2.read_text()).splitlines(True) if a != b: p1_str = p1.as_posix() p2_str = p2.as_posix() diff --git a/acceptance/bin/repls.py b/acceptance/bin/repls.py new file mode 100644 index 00000000000..0599e1b1ee6 --- /dev/null +++ b/acceptance/bin/repls.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +""" +Read the replacements applied to the test output from $ACC_REPLS. + +The file holds two kinds of lines: + - JSON objects written by the test harness, where "Old" is a regular expression; + - ":" records appended by add_repl.py, where is a literal + that is replaced with []. +""" + +import json +import os +import re +import sys +from pathlib import Path + +# Order of the replacements added by the scripts. Matches loadUserReplacements in +# acceptance_test.go, which applies them before the ones written by the harness. +USER_ORDER = -100 + + +def read_repls(): + """Return (pattern, replacement) pairs in the order they must be applied.""" + entries = [] + + for line in Path(os.environ["ACC_REPLS"]).read_text().splitlines(): + line = line.strip() + if not line: + continue + + if line.startswith("{"): + item = json.loads(line) + # "Distinct" is not honoured here; unlike the harness, we do not number the matches. + entries.append((item.get("Order", 0), item["Old"], item["New"])) + continue + + value, name = line.rsplit(":", 1) + # Set() in libs/testdiff also registers the JSON-encoded form of the value, so that + # values with quotes or backslashes are replaced inside JSON output as well. + encoded = json.dumps(value, ensure_ascii=False)[1:-1] + if encoded != value: + entries.append((USER_ORDER, re.escape(encoded), f"[{name}]")) + entries.append((USER_ORDER, re.escape(value), f"[{name}]")) + + # Stable sort: replacements with the same order are applied in the order they were added. + entries.sort(key=lambda entry: entry[0]) + + return [(old, new) for _, old, new in entries] + + +def compile_repls(): + """Same as read_repls(), with patterns compiled. Invalid patterns are reported and skipped.""" + result = [] + for old, new in read_repls(): + try: + result.append((re.compile(old), new)) + except re.error as e: + print(f"Regex error for pattern {old}: {e}", file=sys.stderr) + return result + + +def replace_all(patterns, s): + for comp, new in patterns: + s = comp.sub(new, s) + return s diff --git a/acceptance/bin/sort_lines.py b/acceptance/bin/sort_lines.py index f75e53a661c..9644068b46c 100755 --- a/acceptance/bin/sort_lines.py +++ b/acceptance/bin/sort_lines.py @@ -1,34 +1,22 @@ #!/usr/bin/env python3 """ Helper to sort lines in text file. Similar to 'sort' but no dependence on locale or presence of 'sort' in PATH. -With --repl, applies TEST_TMP_DIR/repls.json replacements as sort key for stable output across different environments. +With --repl, applies the test replacements ($ACC_REPLS) as sort key for stable output across different environments. """ -import json -import os -import re import sys from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent)) +from repls import compile_repls, replace_all + use_repl = "--repl" in sys.argv[1:] lines = sys.stdin.readlines() if use_repl: - repls = json.loads((Path(os.environ["TEST_TMP_DIR"]) / "repls.json").read_text()) - patterns = [] - for r in repls: - try: - patterns.append((re.compile(r["Old"]), r["New"])) - except re.error as e: - print(f"Regex error for pattern {r}: {e}", file=sys.stderr) - - def sort_key(line): - for comp, new in patterns: - line = comp.sub(new, line) - return line - - lines.sort(key=sort_key) + patterns = compile_repls() + lines.sort(key=lambda line: replace_all(patterns, line)) else: lines.sort() diff --git a/acceptance/bundle/apps/git_source/script b/acceptance/bundle/apps/git_source/script index c33cf091fa3..f9d90c84a53 100644 --- a/acceptance/bundle/apps/git_source/script +++ b/acceptance/bundle/apps/git_source/script @@ -21,7 +21,7 @@ trace $CLI bundle run my_app title "Get app details and verify git_source configuration" app_name=$(trace $CLI bundle summary --output json | jq -r '.resources.apps.my_app.name') -echo "$app_name:APP_NAME" >> ACC_REPLS +add_repl "$app_name" APP_NAME trace $CLI apps get $app_name --output json | jq '{name, description, git_repository, git_source}' diff --git a/acceptance/bundle/artifacts/whl_change_version/output.txt b/acceptance/bundle/artifacts/whl_change_version/output.txt index 60b60eff5d3..15d5653bdc5 100644 --- a/acceptance/bundle/artifacts/whl_change_version/output.txt +++ b/acceptance/bundle/artifacts/whl_change_version/output.txt @@ -38,7 +38,6 @@ dist/my_test_code-0.1.0-py3-none-any.whl "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/__main__.py" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/setup.py" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/test.toml" diff --git a/acceptance/bundle/config-remote-sync/job_fields/output.txt b/acceptance/bundle/config-remote-sync/job_fields/output.txt index 88f32e79196..28e1758152a 100644 --- a/acceptance/bundle/config-remote-sync/job_fields/output.txt +++ b/acceptance/bundle/config-remote-sync/job_fields/output.txt @@ -64,7 +64,7 @@ Resource: resources.jobs.my_job - - job_cluster_key: test_cluster + - job_cluster_key: test_cluster_renamed new_cluster: - spark_version: [[DEFAULT_SPARK_VERSION]] + spark_version: [DEFAULT_SPARK_VERSION] @@ -34,5 +40,5 @@ notebook_task: notebook_path: /Users/{{workspace_user_name}}/notebook @@ -73,7 +73,7 @@ Resource: resources.jobs.my_job - task_key: inline_cluster_task notebook_task: @@ -41,5 +47,7 @@ - spark_version: [[DEFAULT_SPARK_VERSION]] + spark_version: [DEFAULT_SPARK_VERSION] node_type_id: [NODE_TYPE_ID] - num_workers: 1 + num_workers: 2 diff --git a/acceptance/bundle/config-remote-sync/job_fields/script b/acceptance/bundle/config-remote-sync/job_fields/script index ed2d6dd93ea..c667e273b21 100755 --- a/acceptance/bundle/config-remote-sync/job_fields/script +++ b/acceptance/bundle/config-remote-sync/job_fields/script @@ -1,8 +1,8 @@ #!/bin/bash envsubst < databricks.yml.tmpl > databricks.yml -add_repl.py "$DEFAULT_SPARK_VERSION" "[DEFAULT_SPARK_VERSION]" -add_repl.py "$NODE_TYPE_ID" "[NODE_TYPE_ID]" +add_repl "$DEFAULT_SPARK_VERSION" DEFAULT_SPARK_VERSION +add_repl "$NODE_TYPE_ID" NODE_TYPE_ID cleanup() { trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/config-remote-sync/multiple_resources/script b/acceptance/bundle/config-remote-sync/multiple_resources/script index ae3b4c486fa..7a89c168d69 100755 --- a/acceptance/bundle/config-remote-sync/multiple_resources/script +++ b/acceptance/bundle/config-remote-sync/multiple_resources/script @@ -17,8 +17,8 @@ job_one_id="$(read_id.py job_one)" job_two_id="$(read_id.py job_two)" # Add replacements for dynamic values that appear in server-side defaults -add_repl.py "$($CLI current-user me | jq -r .id)" "USER_ID" -add_repl.py "$($CLI metastores current | jq -r .metastore_id)" "METASTORE_ID" +add_repl "$($CLI current-user me | jq -r .id)" "USER_ID" +add_repl "$($CLI metastores current | jq -r .metastore_id)" "METASTORE_ID" title "Modify both jobs" edit_resource.py jobs $job_one_id <> ACC_REPLS -echo "$JOB2:JOB2_ID" >> ACC_REPLS +add_repl "$JOB1" JOB1_ID +add_repl "$JOB2" JOB2_ID mkdir -p .databricks/bundle/default cp resources.json .databricks/bundle/default/ diff --git a/acceptance/bundle/deployment/bind/alert/script b/acceptance/bundle/deployment/bind/alert/script index 88e12f15c34..9b6ae2f4085 100644 --- a/acceptance/bundle/deployment/bind/alert/script +++ b/acceptance/bundle/deployment/bind/alert/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml envsubst < alert.json.tmpl > alert.json alert_id=$(trace $CLI alerts-v2 create-alert --json @alert.json | jq -r '.id') -echo "$alert_id:ALERT_ID" >> ACC_REPLS +add_repl "$alert_id" ALERT_ID trace $CLI bundle deployment bind my_alert $alert_id --auto-approve trace $CLI bundle summary diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/script b/acceptance/bundle/deployment/bind/dashboard/recreation/script index 6ab9c0020d0..ce63de92924 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/script +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/script @@ -5,7 +5,7 @@ envsubst < databricks.yml.tmpl > databricks.yml # Create a pre-defined dashboard: DASHBOARD_ID=$($CLI lakeview create --display-name "${DASHBOARD_DISPLAY_NAME}" --warehouse-id "${TEST_DEFAULT_WAREHOUSE_ID}" --serialized-dashboard '{"pages":[{"name":"02724bf2","displayName":"Untitled page"}]}' | jq -r '.dashboard_id') -echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS +add_repl "$DASHBOARD_ID" DASHBOARD_ID cleanupRemoveDashboard() { $CLI lakeview trash "${DASHBOARD_ID}" @@ -16,7 +16,7 @@ trace $CLI bundle deployment bind dashboard1 "${DASHBOARD_ID}" --auto-approve trace print_state.py > out.state_after_bind.$DATABRICKS_BUNDLE_ENGINE.json ETAG=$(jq '.state["resources.dashboards.dashboard1"].state.etag // .resources[0].instances[0].attributes.etag // "no-etag"' < out.state_after_bind.$DATABRICKS_BUNDLE_ENGINE.json) echo $ETAG -add_repl.py $ETAG ETAG +add_repl $ETAG ETAG json_in_json_normalize.py out.state_after_bind.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/script b/acceptance/bundle/deployment/bind/job/already-managed-different/script index 0d2b5ada117..e36fdd856ad 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-different/script +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/script @@ -3,6 +3,6 @@ trace $CLI bundle deploy replace_ids.py new_job_id=$(trace $CLI jobs create --json '{"name": "My Job"}' | jq -r '.job_id') -add_repl.py $new_job_id NEW_JOB_ID +add_repl $new_job_id NEW_JOB_ID trace musterr $CLI bundle deployment bind foo $new_job_id &> out.bind.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/deployment/bind/job/job-abort-bind/script b/acceptance/bundle/deployment/bind/job/job-abort-bind/script index b0aed59093a..57a9c5e57d8 100644 --- a/acceptance/bundle/deployment/bind/job/job-abort-bind/script +++ b/acceptance/bundle/deployment/bind/job/job-abort-bind/script @@ -23,7 +23,7 @@ JOB_ID=$($CLI jobs create --json ' }' | jq -r '.job_id') echo "Created job with ID: $JOB_ID" -echo "$JOB_ID:JOB_ID" >> ACC_REPLS +add_repl "$JOB_ID" JOB_ID envsubst < $TESTDIR/../job-spark-python-task/databricks.yml.tmpl > databricks.yml diff --git a/acceptance/bundle/deployment/bind/job/stale-state/script b/acceptance/bundle/deployment/bind/job/stale-state/script index 32a2d3fb47d..3aea8be1b65 100755 --- a/acceptance/bundle/deployment/bind/job/stale-state/script +++ b/acceptance/bundle/deployment/bind/job/stale-state/script @@ -22,6 +22,6 @@ cp stale_state.json "$state_file" title "Step 6: Create external job and try to bind (should fail: remote state already has job_2)\n" job_id=$($CLI jobs create --json '{"name": "External Job"}' | jq -r '.job_id') -add_repl.py "$job_id" EXTERNAL_JOB_ID +add_repl "$job_id" EXTERNAL_JOB_ID echo "External job ID: $job_id" trace errcode $CLI bundle deployment bind job_2 $job_id --auto-approve &> out.bind.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/script b/acceptance/bundle/deployment/bind/pipelines/recreate/script index aba4cfb97a6..fd2ef19d6c0 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/script +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml envsubst < pipeline.json.tmpl > pipeline.json NEW_PIPELINE_ID=$($CLI pipelines create --json @pipeline.json | jq -r .pipeline_id) -add_repl.py $NEW_PIPELINE_ID NEW_PIPELINE_ID +add_repl $NEW_PIPELINE_ID NEW_PIPELINE_ID rm -f out.requests.txt trace musterr $CLI bundle deployment bind foo $NEW_PIPELINE_ID &> out.bind-fail.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/deployment/bind/pipelines/update/script b/acceptance/bundle/deployment/bind/pipelines/update/script index 5d2e487f10c..5277b53e30c 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/script +++ b/acceptance/bundle/deployment/bind/pipelines/update/script @@ -1,5 +1,5 @@ NEW_PIPELINE_ID=$($CLI pipelines create --json @pipeline.json | jq -r .pipeline_id) -add_repl.py $NEW_PIPELINE_ID NEW_PIPELINE_ID +add_repl $NEW_PIPELINE_ID NEW_PIPELINE_ID rm -f out.requests.txt trace musterr $CLI bundle deployment bind foo $NEW_PIPELINE_ID &> out.bind-fail.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script index 62a64358329..fb89d480852 100644 --- a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/script @@ -13,7 +13,7 @@ trace $CLI bundle destroy --auto-approve # Mask machine A's (local) lineage so the golden output is stable. LINEAGE_A=$(print_state.py | jq -r .lineage) -add_repl.py "$LINEAGE_A" LINEAGE_A +add_repl "$LINEAGE_A" LINEAGE_A # Stash A's local state and wipe it so the next deploy behaves like a fresh machine. cp -r .databricks .databricks.machineA @@ -24,7 +24,7 @@ trace $CLI bundle deploy # Mask machine B's (remote) lineage. LINEAGE_B=$(print_state.py | jq -r .lineage) -add_repl.py "$LINEAGE_B" LINEAGE_B +add_repl "$LINEAGE_B" LINEAGE_B # Back on machine A: restore the stale local state that still carries L1. rm -rf .databricks diff --git a/acceptance/bundle/generate/alert/script b/acceptance/bundle/generate/alert/script index ed533cdf933..1d5e4b165d5 100644 --- a/acceptance/bundle/generate/alert/script +++ b/acceptance/bundle/generate/alert/script @@ -3,7 +3,7 @@ trace $CLI workspace mkdirs /Workspace/Users/$CURRENT_USER_NAME/test-$UNIQUE_NAM # create an alert to import envsubst < alert.json.tmpl > alert.json alert_id=$($CLI alerts-v2 create-alert --json @alert.json | jq -r '.id') -echo "$alert_id:ALERT_ID" >> ACC_REPLS +add_repl "$alert_id" ALERT_ID rm alert.json trace $CLI bundle generate alert --existing-id $alert_id --source-dir out/alert --config-dir out/resource diff --git a/acceptance/bundle/generate/pipeline_and_deploy/script b/acceptance/bundle/generate/pipeline_and_deploy/script index 9f8a82bd265..32d4cbcbc4a 100644 --- a/acceptance/bundle/generate/pipeline_and_deploy/script +++ b/acceptance/bundle/generate/pipeline_and_deploy/script @@ -24,7 +24,7 @@ PIPELINE_ID=$($CLI pipelines create --json '{ ] }' | jq -r '.pipeline_id') echo "Created pipeline" -env -u MSYS_NO_PATHCONV add_repl.py "$PIPELINE_ID" PIPELINE_ID +add_repl "$PIPELINE_ID" PIPELINE_ID cleanup() { title "Cleanup: delete the original pipeline and files" diff --git a/acceptance/bundle/generate/python_job_and_deploy/script b/acceptance/bundle/generate/python_job_and_deploy/script index 1501374ca3a..f9629bf8e7f 100644 --- a/acceptance/bundle/generate/python_job_and_deploy/script +++ b/acceptance/bundle/generate/python_job_and_deploy/script @@ -28,7 +28,7 @@ JOB_ID=$($CLI jobs create --json '{ echo "Created job" # Disable MSYS_NO_PATHCONV when invoking python scripts: with it set, Git Bash on Windows # fails to translate the script path so the python interpreter can't find the file. -env -u MSYS_NO_PATHCONV add_repl.py "$JOB_ID" JOB_ID +add_repl "$JOB_ID" JOB_ID cleanup() { title "Cleanup: delete the original job and notebook" diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt index c4dc9f4d8af..fe2c7fdd244 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/output.txt @@ -26,7 +26,6 @@ direct_migrated_via_config true "POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/databricks.yml" "POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/out.requests.txt" "POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/output.txt" -"POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/repls.json" "POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/script" "POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" "POST /api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/deploy.lock" diff --git a/acceptance/bundle/python/script.prepare b/acceptance/bundle/python/script.prepare index c0d9f2b67d1..0f1063f6c61 100644 --- a/acceptance/bundle/python/script.prepare +++ b/acceptance/bundle/python/script.prepare @@ -4,4 +4,4 @@ if [ "$PYDAB_VERSION" = "current" ]; then else UV_ARGS="--with databricks-bundles==$PYDAB_VERSION" fi -add_repl.py "$UV_ARGS" UV_ARGS +add_repl "$UV_ARGS" UV_ARGS diff --git a/acceptance/bundle/resource_deps/model_id_ref/script b/acceptance/bundle/resource_deps/model_id_ref/script index 35dfa5beed1..58375a71476 100644 --- a/acceptance/bundle/resource_deps/model_id_ref/script +++ b/acceptance/bundle/resource_deps/model_id_ref/script @@ -8,6 +8,6 @@ trace $CLI bundle deploy # precisely rather than by the broad [NUMID] pattern. The id comes from the API (independent # of the deploy engine), so terraform and direct must both resolve to this exact value. model_id=$($CLI model-registry get-model my-model | jq -r '.registered_model_databricks.id') -add_repl.py "$model_id" MY_MODEL_ID +add_repl "$model_id" MY_MODEL_ID trace print_requests.py //mlflow/registered-models/create //jobs/create --sort diff --git a/acceptance/bundle/resources/alerts/basic/script b/acceptance/bundle/resources/alerts/basic/script index e8e4427e261..e47c734d1bd 100644 --- a/acceptance/bundle/resources/alerts/basic/script +++ b/acceptance/bundle/resources/alerts/basic/script @@ -4,7 +4,7 @@ trace $CLI bundle deploy alert_id=$($CLI bundle summary --output json | jq -r '.resources.alerts.myalert.id') -echo "$alert_id:ALERT_ID" >> ACC_REPLS +add_repl "$alert_id" ALERT_ID trace $CLI alerts-v2 get-alert $alert_id | jq '{display_name, lifecycle_state, custom_summary, evaluation, query_text, schedule, warehouse_id}' diff --git a/acceptance/bundle/resources/alerts/with_file/script b/acceptance/bundle/resources/alerts/with_file/script index d77dcadc44f..02a51e6d2cb 100755 --- a/acceptance/bundle/resources/alerts/with_file/script +++ b/acceptance/bundle/resources/alerts/with_file/script @@ -12,7 +12,7 @@ trace $CLI bundle deploy alert_id=$($CLI bundle summary --output json | jq -r '.resources.alerts.myalert.id') -echo "$alert_id:ALERT_ID" >> ACC_REPLS +add_repl "$alert_id" ALERT_ID trace $CLI alerts-v2 get-alert $alert_id | jq '{display_name, lifecycle_state, custom_summary, evaluation, query_text, schedule, warehouse_id}' diff --git a/acceptance/bundle/resources/apps/inline_config/script b/acceptance/bundle/resources/apps/inline_config/script index ec2af098e13..764ab0bc99b 100755 --- a/acceptance/bundle/resources/apps/inline_config/script +++ b/acceptance/bundle/resources/apps/inline_config/script @@ -12,6 +12,6 @@ APP_NAME=$($CLI bundle summary -o json | jq -r '.resources.apps.myapp.name') $CLI bundle run myapp > /dev/null 2>&1 DEPLOYMENT_ID=$($CLI apps get "$APP_NAME" | jq -r '.active_deployment.deployment_id') -echo $DEPLOYMENT_ID:DEPLOYMENT_ID >> ACC_REPLS +add_repl "$DEPLOYMENT_ID" DEPLOYMENT_ID trace $CLI apps get-deployment "$APP_NAME" "$DEPLOYMENT_ID" | jq '{command, env_vars}' diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script index 1370846fe4b..9125605e1d6 100755 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script @@ -12,7 +12,7 @@ trace $CLI bundle deploy title "Cluster should exist with num_workers after bundle deployment:\n" CLUSTER_ID=$($CLI bundle summary -o json | jq -r '.resources.clusters.test_cluster.id') -echo "$CLUSTER_ID:CLUSTER_ID" >> ACC_REPLS +add_repl "$CLUSTER_ID" CLUSTER_ID $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' title "Terminating the cluster\n" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script index f2d80d05de0..8bb7b3d0788 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script @@ -12,7 +12,7 @@ trace $CLI bundle deploy title "Cluster should exist after bundle deployment:\n" CLUSTER_ID=$($CLI bundle summary -o json | jq -r '.resources.clusters.test_cluster.id') -echo "$CLUSTER_ID:CLUSTER_ID" >> ACC_REPLS +add_repl "$CLUSTER_ID" CLUSTER_ID $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' title "Changing num_workers should call update API on stopped cluster\n" diff --git a/acceptance/bundle/resources/clusters/resize-terminated-fallback/script b/acceptance/bundle/resources/clusters/resize-terminated-fallback/script index 258d8e8d37f..eff7c541b94 100644 --- a/acceptance/bundle/resources/clusters/resize-terminated-fallback/script +++ b/acceptance/bundle/resources/clusters/resize-terminated-fallback/script @@ -9,7 +9,7 @@ trap cleanup EXIT trace $CLI bundle deploy CLUSTER_ID=$($CLI bundle summary -o json | jq -r '.resources.clusters.test_cluster.id') -echo "$CLUSTER_ID:CLUSTER_ID" >> ACC_REPLS +add_repl "$CLUSTER_ID" CLUSTER_ID title "Create a plan while cluster is running: should show resize\n" update_file.py databricks.yml "num_workers: 2" "num_workers: 3" diff --git a/acceptance/bundle/resources/dashboards/change-embed-credentials/script b/acceptance/bundle/resources/dashboards/change-embed-credentials/script index bfc3c9a4ad2..75046150329 100755 --- a/acceptance/bundle/resources/dashboards/change-embed-credentials/script +++ b/acceptance/bundle/resources/dashboards/change-embed-credentials/script @@ -16,8 +16,8 @@ deploy_dashboard() { dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.my_dashboard.id') DASHBOARD=$(retry $CLI lakeview get $dashboard_id) - echo "$dashboard_id:DASHBOARD_ID" >> ACC_REPLS - echo "$(echo "$DASHBOARD" | jq -r '.etag'):ETAG" >> ACC_REPLS + add_repl "$dashboard_id" DASHBOARD_ID + add_repl "$(echo "$DASHBOARD" | jq -r '.etag')" ETAG echo "$DASHBOARD" | jq "{display_name, lifecycle_state, parent_path, path, serialized_dashboard, warehouse_id}" trace $CLI lakeview get-published $dashboard_id | jq "{warehouse_id, embed_credentials}" } diff --git a/acceptance/bundle/resources/dashboards/change-name/script b/acceptance/bundle/resources/dashboards/change-name/script index 374452ed84d..f361ed4417f 100644 --- a/acceptance/bundle/resources/dashboards/change-name/script +++ b/acceptance/bundle/resources/dashboards/change-name/script @@ -23,8 +23,8 @@ deploy_dashboard # Capture the dashboard ID as a replacement. dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.my_dashboard.id') -echo "$dashboard_id:DASHBOARD_ID" >> ACC_REPLS -echo "$(echo "$DASHBOARD" | jq -r '.etag'):ETAG" >> ACC_REPLS +add_repl "$dashboard_id" DASHBOARD_ID +add_repl "$(echo "$DASHBOARD" | jq -r '.etag')" ETAG # Change the name export NAME="dashboard2" diff --git a/acceptance/bundle/resources/dashboards/change-parent-path/script b/acceptance/bundle/resources/dashboards/change-parent-path/script index 664c6867044..8f059e33b58 100755 --- a/acceptance/bundle/resources/dashboards/change-parent-path/script +++ b/acceptance/bundle/resources/dashboards/change-parent-path/script @@ -22,7 +22,7 @@ deploy_dashboard # Capture the dashboard ID as a replacement. dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.my_dashboard.id') -echo "$dashboard_id:ORIGINAL_DASHBOARD_ID" >> ACC_REPLS +add_repl "$dashboard_id" ORIGINAL_DASHBOARD_ID # Change the parent_path - this should trigger a recreate (delete + create) export PARENT_PATH=" /Users/$USERNAME/.bundle/change-parent-path-$UNIQUE_NAME/default/resources2" @@ -30,7 +30,7 @@ deploy_dashboard # Capture the new dashboard ID as a replacement. new_dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.my_dashboard.id') -echo "$new_dashboard_id:NEW_DASHBOARD_ID" >> ACC_REPLS +add_repl "$new_dashboard_id" NEW_DASHBOARD_ID # Print API requests made to create parent_path or create / update / delete the dashboard. cat out.requests.txt | \ diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/script b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/script index bcbecd86613..43327a7ff19 100755 --- a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/script +++ b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/script @@ -26,7 +26,7 @@ deploy_dashboard # Capture the dashboard ID as a replacement. dashboard_id=$($CLI bundle summary --output json | jq -r '.resources.dashboards.my_dashboard.id') -echo "$dashboard_id:DASHBOARD_ID" >> ACC_REPLS +add_repl "$dashboard_id" DASHBOARD_ID # Change the serialized_dashboard content - this should trigger an update (not recreate) export PAGE_NAME="name2" diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script index b147e47f32c..3e729767a06 100755 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script @@ -2,7 +2,7 @@ DASHBOARD_DISPLAY_NAME="test bundle-deploy-dashboard-dataset $(uuid)" if [ -z "$CLOUD_ENV" ]; then export TEST_DEFAULT_WAREHOUSE_ID="warehouse-1234" - echo "warehouse-1234:TEST_DEFAULT_WAREHOUSE_ID" >> ACC_REPLS + add_repl "warehouse-1234" TEST_DEFAULT_WAREHOUSE_ID fi export DASHBOARD_DISPLAY_NAME @@ -18,9 +18,9 @@ trace $CLI bundle deploy DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') # Capture the dashboard ID as a replacement. -echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS +add_repl "$DASHBOARD_ID" DASHBOARD_ID DASHBOARD=$(retry $CLI lakeview get $DASHBOARD_ID) -echo "$(echo "$DASHBOARD" | jq -r '.etag'):ETAG" >> ACC_REPLS +add_repl "$(echo "$DASHBOARD" | jq -r '.etag')" ETAG echo "$DASHBOARD" | jq '{lifecycle_state, parent_path, path}' diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script index f5ef99d0e60..435fd4e4891 100755 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script @@ -2,7 +2,7 @@ export DASHBOARD_DISPLAY_NAME="test bundle-deploy-dashboard $UNIQUE_NAME)" if [ -z "$CLOUD_ENV" ]; then export TEST_DEFAULT_WAREHOUSE_ID="warehouse-1234" - echo "warehouse-1234:TEST_DEFAULT_WAREHOUSE_ID" >> ACC_REPLS + add_repl "warehouse-1234" TEST_DEFAULT_WAREHOUSE_ID fi envsubst < databricks.yml.tmpl > databricks.yml @@ -12,9 +12,9 @@ trace $CLI bundle deploy DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') # Capture the dashboard ID as a replacement. -echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS +add_repl "$DASHBOARD_ID" DASHBOARD_ID DASHBOARD=$(retry $CLI lakeview get $DASHBOARD_ID) -echo "$(echo "$DASHBOARD" | jq -r '.etag'):ETAG" >> ACC_REPLS +add_repl "$(echo "$DASHBOARD" | jq -r '.etag')" ETAG # Verify dashboard was deployed echo "$DASHBOARD" | jq '{lifecycle_state, parent_path}' diff --git a/acceptance/bundle/resources/dashboards/destroy/script b/acceptance/bundle/resources/dashboards/destroy/script index 3ddb222cbdf..e0419b0507e 100755 --- a/acceptance/bundle/resources/dashboards/destroy/script +++ b/acceptance/bundle/resources/dashboards/destroy/script @@ -11,7 +11,7 @@ trace $CLI bundle deploy DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') # Capture the dashboard ID as a replacement. -echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS +add_repl "$DASHBOARD_ID" DASHBOARD_ID # Verify dashboard was deployed trace retry $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path}' diff --git a/acceptance/bundle/resources/dashboards/detect-change/script b/acceptance/bundle/resources/dashboards/detect-change/script index 98ab4cc93b7..2b4966fd87d 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/script +++ b/acceptance/bundle/resources/dashboards/detect-change/script @@ -21,9 +21,9 @@ trace cat out.summary.json DASHBOARD_ID=$(jq -r '.id' out.summary.json) # Capture the dashboard ID as a replacement. -echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS +add_repl "$DASHBOARD_ID" DASHBOARD_ID DASHBOARD=$(retry $CLI lakeview get "$DASHBOARD_ID") -echo "$(echo "$DASHBOARD" | jq -r '.etag'):ETAG_1" >> ACC_REPLS +add_repl "$(echo "$DASHBOARD" | jq -r '.etag')" ETAG_1 rm out.summary.json title "Load the dashboard by its ID and confirm its display name: " @@ -34,7 +34,7 @@ RESOURCE_ID=$($CLI workspace get-status "${DASHBOARD_PATH}" | jq -r '.resource_i DASHBOARD_JSON="{\"serialized_dashboard\": \"{}\", \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\"}" UPDATE_RESP=$($CLI lakeview update "${RESOURCE_ID}" --json "${DASHBOARD_JSON}") echo "$UPDATE_RESP" | jq '{lifecycle_state}' -echo "$(echo "$UPDATE_RESP" | jq -r '.etag'):ETAG_2" >> ACC_REPLS +add_repl "$(echo "$UPDATE_RESP" | jq -r '.etag')" ETAG_2 title "Try to redeploy the bundle and confirm that the out of band modification is detected:" trace $CLI bundle plan diff --git a/acceptance/bundle/resources/dashboards/nested-folders/script b/acceptance/bundle/resources/dashboards/nested-folders/script index 19ec3b03188..c5004f3d6a2 100644 --- a/acceptance/bundle/resources/dashboards/nested-folders/script +++ b/acceptance/bundle/resources/dashboards/nested-folders/script @@ -17,6 +17,6 @@ $CLI bundle deploy $(readplanarg tmp.plan.json) DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') # Capture the dashboard ID as a replacement. -echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS +add_repl "$DASHBOARD_ID" DASHBOARD_ID trace retry $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard: (.serialized_dashboard | fromjson | {pages: (.pages | map({name, displayName, pageType}))})}' diff --git a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script index 2fb37909fa7..41022465670 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script +++ b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script @@ -13,7 +13,7 @@ trace $CLI bundle deploy DASHBOARD_ID=$(read_id.py dashboard1) DASHBOARD=$(retry $CLI lakeview get $DASHBOARD_ID) ETAG_1=$(echo "$DASHBOARD" | jq -r '.etag') -add_repl.py "$ETAG_1" ETAG_1 +add_repl "$ETAG_1" ETAG_1 echo "$DASHBOARD" | jq '{display_name, etag}' trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}' trace $CLI bundle plan -o json | gron.py | grep -E "etag|published" @@ -31,7 +31,7 @@ errcode trace $CLI bundle deploy trace print_requests.py //lakeview/dashboards # The PATCH bumped the remote etag to ETAG_2; retry until it is visible (eventual consistency). ETAG_2=$(retry --until-not "$ETAG_1" $CLI lakeview get $DASHBOARD_ID | jq -r '.etag') -add_repl.py "$ETAG_2" ETAG_2 +add_repl "$ETAG_2" ETAG_2 trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}' trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}' trace $CLI bundle plan -o json | gron.py | grep -E "etag|published" @@ -50,7 +50,7 @@ trace $CLI bundle deploy --force trace print_requests.py //lakeview/dashboards # --force did another PATCH; retry until the new etag is visible (eventual consistency). ETAG_3=$(retry --until-not "$ETAG_2" $CLI lakeview get $DASHBOARD_ID | jq -r '.etag') -add_repl.py "$ETAG_3" ETAG_3 +add_repl "$ETAG_3" ETAG_3 trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}' trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}' trace $CLI bundle plan diff --git a/acceptance/bundle/resources/dashboards/simple/script b/acceptance/bundle/resources/dashboards/simple/script index 71e7335c569..4c8b9b55ace 100644 --- a/acceptance/bundle/resources/dashboards/simple/script +++ b/acceptance/bundle/resources/dashboards/simple/script @@ -1,7 +1,7 @@ DASHBOARD_DISPLAY_NAME="test bundle-deploy-dashboard $(uuid)" if [ -z "$CLOUD_ENV" ]; then export TEST_DEFAULT_WAREHOUSE_ID="warehouse-1234" - echo "warehouse-1234:TEST_DEFAULT_WAREHOUSE_ID" >> ACC_REPLS + add_repl "warehouse-1234" TEST_DEFAULT_WAREHOUSE_ID fi export DASHBOARD_DISPLAY_NAME @@ -16,9 +16,9 @@ trace $CLI bundle deploy DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') # Capture the dashboard ID as a replacement. -echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS +add_repl "$DASHBOARD_ID" DASHBOARD_ID DASHBOARD=$(retry $CLI lakeview get $DASHBOARD_ID) -echo "$(echo "$DASHBOARD" | jq -r '.etag'):ETAG" >> ACC_REPLS +add_repl "$(echo "$DASHBOARD" | jq -r '.etag')" ETAG echo "$DASHBOARD" | jq '{lifecycle_state, parent_path, path, serialized_dashboard}' diff --git a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/script b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/script index 09a3d9f0eaf..f1c1ae527f0 100644 --- a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/script +++ b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/script @@ -16,6 +16,6 @@ trace $CLI bundle deploy DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') # Capture the dashboard ID as a replacement. -echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS +add_repl "$DASHBOARD_ID" DASHBOARD_ID trace retry $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard}' diff --git a/acceptance/bundle/resources/dashboards/simple_syncroot/script b/acceptance/bundle/resources/dashboards/simple_syncroot/script index e6bb69c04c6..a93701b6c2d 100644 --- a/acceptance/bundle/resources/dashboards/simple_syncroot/script +++ b/acceptance/bundle/resources/dashboards/simple_syncroot/script @@ -17,6 +17,6 @@ trace $CLI bundle deploy DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') # Capture the dashboard ID as a replacement. -echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS +add_repl "$DASHBOARD_ID" DASHBOARD_ID trace retry $CLI lakeview get $DASHBOARD_ID | jq '{lifecycle_state, parent_path, path, serialized_dashboard}' diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script index 8631b921342..3be1cdd883b 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script @@ -11,7 +11,7 @@ trace $CLI bundle plan -o json > out.plan_initial.$DATABRICKS_BUNDLE_ENGINE.json # Deploy the dashboard trace $CLI bundle deploy DASHBOARD_ID=$(read_id.py dashboard1) -add_repl.py "$(retry $CLI lakeview get $DASHBOARD_ID | jq -r '.etag')" ETAG +add_repl "$(retry $CLI lakeview get $DASHBOARD_ID | jq -r '.etag')" ETAG trace print_state.py | grep publish > out.state.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/genie_spaces/inline/script b/acceptance/bundle/resources/genie_spaces/inline/script index 8f2f625b7b0..812cfb9abd4 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/script +++ b/acceptance/bundle/resources/genie_spaces/inline/script @@ -9,7 +9,7 @@ trace $CLI bundle deploy GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') # Capture the genie space ID as a replacement. -echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS +add_repl "$GENIE_SPACE_ID" GENIE_SPACE_ID # Plan after deploy must be drift-free aside from input_only fields. # Without normalization the inline serialized_space leaves a map in the diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/script b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/script index 7d8a3be80d2..0ccb456f1cf 100644 --- a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/script +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/script @@ -7,7 +7,7 @@ trap cleanup EXIT trace $CLI bundle deploy GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') -echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS +add_repl "$GENIE_SPACE_ID" GENIE_SPACE_ID # Delete the space out-of-band, simulating a UI deletion. The GET API then # returns 404, which the plan treats as "gone" so it recreates the space diff --git a/acceptance/bundle/resources/genie_spaces/simple/script b/acceptance/bundle/resources/genie_spaces/simple/script index 6a632da4232..6ad76a8ab24 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/script +++ b/acceptance/bundle/resources/genie_spaces/simple/script @@ -9,7 +9,7 @@ trace $CLI bundle deploy GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') # Capture the genie space ID as a replacement. -echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS +add_repl "$GENIE_SPACE_ID" GENIE_SPACE_ID trace $CLI genie get-space $GENIE_SPACE_ID | jq '{title, description, warehouse_id}' diff --git a/acceptance/bundle/resources/jobs/alert-task/script b/acceptance/bundle/resources/jobs/alert-task/script index f2d3d0fc9b2..1dcc1d37a0b 100644 --- a/acceptance/bundle/resources/jobs/alert-task/script +++ b/acceptance/bundle/resources/jobs/alert-task/script @@ -8,6 +8,6 @@ trap cleanup EXIT trace $CLI bundle deploy job_id=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id') -echo "$job_id:JOB_ID" >> ACC_REPLS +add_repl "$job_id" JOB_ID trace $CLI jobs get $job_id | jq '.settings.tasks[0].alert_task' diff --git a/acceptance/bundle/resources/jobs/check-metadata/script b/acceptance/bundle/resources/jobs/check-metadata/script index 0445fa0bf84..a486009c74c 100644 --- a/acceptance/bundle/resources/jobs/check-metadata/script +++ b/acceptance/bundle/resources/jobs/check-metadata/script @@ -14,14 +14,14 @@ title "Second summary, should have id and modified status should be absent" trace $CLI bundle summary -o json | jq '.resources.jobs.foo | {id, modified_status}' > out.summary.json trace cat out.summary.json JOB_1_ID=$(jq -r '.id' out.summary.json) -echo "$JOB_1_ID:JOB_1_ID" >> ACC_REPLS +add_repl "$JOB_1_ID" JOB_1_ID rm out.summary.json title "Assert job 1 is created\n" $CLI jobs get "${JOB_1_ID}" | jq '{name: .settings.name}' title "Assert job 2 is created\n" JOB_2_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.bar.id') -echo "$JOB_2_ID:JOB_2_ID" >> ACC_REPLS +add_repl "$JOB_2_ID" JOB_2_ID $CLI jobs get "${JOB_2_ID}" | jq '{name: .settings.name}' title "Read metadata object from the workspace" diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/script b/acceptance/bundle/resources/model_serving_endpoints/basic/script index 3fc875f5f58..aac11bceec4 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/script +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/script @@ -21,12 +21,12 @@ trace $CLI bundle deploy # SP and the real cloud SP. Sorting makes the recorded request stable across both. print_requests.py //serving-endpoints | gron.py --sort-arrays access_control_list > out.first-requests.$DATABRICKS_BUNDLE_ENGINE.txt -echo "$ENDPOINT_NAME:ENDPOINT_NAME_1" >> ACC_REPLS +add_repl "$ENDPOINT_NAME" ENDPOINT_NAME_1 # Record the endpoint ID for the first endpoint get_output=$($CLI serving-endpoints get $ENDPOINT_NAME) endpoint_id=$(echo "$get_output" | jq -r '.id') -echo "$endpoint_id:ENDPOINT_ID_1" >> ACC_REPLS +add_repl "$endpoint_id" ENDPOINT_ID_1 title "Print the GET endpoint details\n" echo "$get_output" | jq '{name, creator}' @@ -37,12 +37,12 @@ trace $CLI bundle plan -o json | gron.py --sort-arrays __embed__ > out.second-pl trace $CLI bundle deploy print_requests.py //serving-endpoints | gron.py --sort-arrays access_control_list > out.second-requests.$DATABRICKS_BUNDLE_ENGINE.txt -echo "$ENDPOINT_NAME:ENDPOINT_NAME_2" >> ACC_REPLS +add_repl "$ENDPOINT_NAME" ENDPOINT_NAME_2 # Record the endpoint ID for the second endpoint get_output=$($CLI serving-endpoints get $ENDPOINT_NAME) endpoint_id=$(echo "$get_output" | jq -r '.id') -echo "$endpoint_id:ENDPOINT_ID_2" >> ACC_REPLS +add_repl "$endpoint_id" ENDPOINT_ID_2 title "Print the GET endpoint details\n" echo "$get_output" | jq '{name, creator}' diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/script b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/script index 9f764d7914e..3f26d4a0738 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/script @@ -12,7 +12,7 @@ title "Initial deployment" trace $CLI bundle deploy original_endpoint_id=$($CLI serving-endpoints get "${endpoint_name}" | jq -r '.id') -add_repl.py "$original_endpoint_id" "ORIGINAL_ENDPOINT_ID" +add_repl "$original_endpoint_id" "ORIGINAL_ENDPOINT_ID" trace $CLI serving-endpoints get "${endpoint_name}" | jq '{name, creator}' title "Delete and recreate remotely with the same name" @@ -20,7 +20,7 @@ trace $CLI serving-endpoints delete "${endpoint_name}" trace $CLI serving-endpoints create "${endpoint_name}" --no-wait | jq '{name, creator}' remote_recreated_endpoint_id=$($CLI serving-endpoints get "${endpoint_name}" | jq -r '.id') -add_repl.py "$remote_recreated_endpoint_id" "REMOTE_RECREATED_ENDPOINT_ID" +add_repl "$remote_recreated_endpoint_id" "REMOTE_RECREATED_ENDPOINT_ID" trace $CLI serving-endpoints get "${endpoint_name}" | jq '{name, creator}' printf "Original endpoint id: %s\n" "$original_endpoint_id" diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script index 90e71bb338d..919470c4ca4 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script @@ -12,7 +12,7 @@ trace $CLI bundle deploy trace print_requests.py //serving-endpoints ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') -echo "$ENDPOINT_ID:ORIGINAL_ENDPOINT_ID" >> ACC_REPLS +add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config.auto_capture_config.catalog_name' trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other_catalog" @@ -22,5 +22,5 @@ trace $CLI bundle deploy trace print_requests.py //serving-endpoints NEW_ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') -echo "$NEW_ENDPOINT_ID:NEW_ENDPOINT_ID" >> ACC_REPLS +add_repl "$NEW_ENDPOINT_ID" NEW_ENDPOINT_ID trace $CLI serving-endpoints get "${NEW_ENDPOINT_ID}" | jq '.config.auto_capture_config.catalog_name' diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script index d772069d397..d48a60c0ace 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script @@ -12,7 +12,7 @@ trace $CLI bundle deploy trace print_requests.py //serving-endpoints ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') -echo "$ENDPOINT_ID:ORIGINAL_ENDPOINT_ID" >> ACC_REPLS +add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.name' trace update_file.py databricks.yml "name: test-endpoint-$UNIQUE_NAME" "name: test-endpoint-2-$UNIQUE_NAME" @@ -22,5 +22,5 @@ trace $CLI bundle deploy trace print_requests.py //serving-endpoints NEW_ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') -echo "$NEW_ENDPOINT_ID:NEW_ENDPOINT_ID" >> ACC_REPLS +add_repl "$NEW_ENDPOINT_ID" NEW_ENDPOINT_ID trace $CLI serving-endpoints get "${NEW_ENDPOINT_ID}" | jq '.name' diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script index fec000b7e18..1df74506afe 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script @@ -10,7 +10,7 @@ trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') -echo "$ENDPOINT_ID:ORIGINAL_ENDPOINT_ID" >> ACC_REPLS +add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '{route_optimized, id}' > out.first-get.$DATABRICKS_BUNDLE_ENGINE.json print_requests.py //serving-endpoints > out.first-requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script index e02abe9aa71..0e31f7a4848 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script @@ -12,7 +12,7 @@ trace $CLI bundle deploy trace print_requests.py //serving-endpoints ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') -echo "$ENDPOINT_ID:ORIGINAL_ENDPOINT_ID" >> ACC_REPLS +add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config.auto_capture_config.schema_name' trace update_file.py databricks.yml "schema_name: default" "schema_name: other_schema" @@ -22,5 +22,5 @@ trace $CLI bundle deploy trace print_requests.py //serving-endpoints NEW_ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') -echo "$NEW_ENDPOINT_ID:NEW_ENDPOINT_ID" >> ACC_REPLS +add_repl "$NEW_ENDPOINT_ID" NEW_ENDPOINT_ID trace $CLI serving-endpoints get "${NEW_ENDPOINT_ID}" | jq '.config.auto_capture_config.schema_name' diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script index 7a41b3bfcb7..7f915920ba1 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script @@ -12,7 +12,7 @@ trace $CLI bundle deploy trace print_requests.py //serving-endpoints ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') -echo "$ENDPOINT_ID:ORIGINAL_ENDPOINT_ID" >> ACC_REPLS +add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config.auto_capture_config.table_name_prefix' trace update_file.py databricks.yml "table_name_prefix: my_table" "table_name_prefix: other_table" @@ -22,5 +22,5 @@ trace $CLI bundle deploy trace print_requests.py //serving-endpoints NEW_ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') -echo "$NEW_ENDPOINT_ID:NEW_ENDPOINT_ID" >> ACC_REPLS +add_repl "$NEW_ENDPOINT_ID" NEW_ENDPOINT_ID trace $CLI serving-endpoints get "${NEW_ENDPOINT_ID}" | jq '.config.auto_capture_config.table_name_prefix' diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script index 89877faef63..6e556d2b1ab 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script @@ -10,7 +10,7 @@ trap cleanup EXIT trace $CLI bundle deploy ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') -echo "$ENDPOINT_ID:ENDPOINT_ID" >> ACC_REPLS +add_repl "$ENDPOINT_ID" ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.ai_gateway' trace update_file.py databricks.yml 'catalog_name: "first-inference-catalog"' 'catalog_name: "second-inference-catalog"' diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script index d0743fbfb73..763379ad9a6 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script @@ -10,7 +10,7 @@ trap cleanup EXIT trace $CLI bundle deploy ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') -echo "$ENDPOINT_ID:ENDPOINT_ID" >> ACC_REPLS +add_repl "$ENDPOINT_ID" ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config' # Update two fields. The patch request should be called for both. diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/config/script b/acceptance/bundle/resources/model_serving_endpoints/update/config/script index 3f28edc8438..3a28d1dcda2 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/config/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/config/script @@ -10,7 +10,7 @@ trap cleanup EXIT trace $CLI bundle deploy ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') -echo "$ENDPOINT_ID:ENDPOINT_ID" >> ACC_REPLS +add_repl "$ENDPOINT_ID" ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config' trace update_file.py databricks.yml "name: gpt-4o-mini" "name: gpt-5o-mini" diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script index 15b43b4ffda..391d369d686 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script @@ -10,7 +10,7 @@ trap cleanup EXIT trace $CLI bundle deploy ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') -echo "$ENDPOINT_ID:ENDPOINT_ID" >> ACC_REPLS +add_repl "$ENDPOINT_ID" ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.email_notifications' # Update email_notifications - change the email address diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/tags/script b/acceptance/bundle/resources/model_serving_endpoints/update/tags/script index cb1286e3aaf..6f69dc2f063 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/tags/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/tags/script @@ -10,7 +10,7 @@ trap cleanup EXIT trace $CLI bundle deploy ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') -echo "$ENDPOINT_ID:ENDPOINT_ID" >> ACC_REPLS +add_repl "$ENDPOINT_ID" ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.tags' trace update_file.py databricks.yml "value: my-team-one" "value: my-team-two" diff --git a/acceptance/bundle/resources/models/readplan-permissions/script b/acceptance/bundle/resources/models/readplan-permissions/script index 52d8975afe5..f8be26409e3 100644 --- a/acceptance/bundle/resources/models/readplan-permissions/script +++ b/acceptance/bundle/resources/models/readplan-permissions/script @@ -7,7 +7,7 @@ trap cleanup EXIT register_model_id() { # Register the model's numeric ID so it appears as [MODEL_ID] in output MODEL_ID=$($CLI model-registry get-model test-model | jq -r '.registered_model_databricks.id') - add_repl.py "$MODEL_ID" "MODEL_ID" + add_repl "$MODEL_ID" "MODEL_ID" } title "Initial deploy: model created with permissions" diff --git a/acceptance/bundle/resources/permissions/factcheck/script b/acceptance/bundle/resources/permissions/factcheck/script index 9b17c99cb4a..cbddce58087 100644 --- a/acceptance/bundle/resources/permissions/factcheck/script +++ b/acceptance/bundle/resources/permissions/factcheck/script @@ -9,10 +9,10 @@ trap cleanup EXIT trace $CLI bundle deploy job_id=$($CLI bundle summary --output json | jq -r '.resources.jobs.job.id') -echo "$job_id:JOB_ID" >> ACC_REPLS +add_repl "$job_id" JOB_ID sql_warehouse_id=$($CLI bundle summary --output json | jq -r '.resources.sql_warehouses.warehouse.id') -echo "$sql_warehouse_id:SQL_WAREHOUSE_ID" >> ACC_REPLS +add_repl "$sql_warehouse_id" SQL_WAREHOUSE_ID user=deco-test-user@databricks.com diff --git a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/script b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/script index 861e23300ab..30130e75885 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/script +++ b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/script @@ -5,7 +5,7 @@ trap cleanup EXIT trace $CLI bundle deploy SID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.foo.id') -echo "$SID:GENIE_SPACE_ID" >> ACC_REPLS +add_repl "$SID" GENIE_SPACE_ID title "Permissions are set on the live space" trace $CLI api get /api/2.0/permissions/genie/$SID | jq '{object_type, access_control_list}' diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/script b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/script index 0bfb065a862..a7ec688100b 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/script +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/script @@ -5,7 +5,7 @@ if [ -z "${TEST_SP_APPLICATION_ID:-}" ]; then fi # Mask the test SP's application_id (its userName) to a dedicated placeholder so # the assertion is env-independent instead of collapsing into the generic [UUID]. -echo "$TEST_SP_APPLICATION_ID:TEST_SP_APPLICATION_ID" >> ACC_REPLS +add_repl "$TEST_SP_APPLICATION_ID" TEST_SP_APPLICATION_ID envsubst < databricks.yml.tmpl > databricks.yml # Locally DATABRICKS_CLIENT_ID is unset; fall back to the current identity so the diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/script b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/script index 0bfb065a862..a7ec688100b 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/script +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/script @@ -5,7 +5,7 @@ if [ -z "${TEST_SP_APPLICATION_ID:-}" ]; then fi # Mask the test SP's application_id (its userName) to a dedicated placeholder so # the assertion is env-independent instead of collapsing into the generic [UUID]. -echo "$TEST_SP_APPLICATION_ID:TEST_SP_APPLICATION_ID" >> ACC_REPLS +add_repl "$TEST_SP_APPLICATION_ID" TEST_SP_APPLICATION_ID envsubst < databricks.yml.tmpl > databricks.yml # Locally DATABRICKS_CLIENT_ID is unset; fall back to the current identity so the diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/script b/acceptance/bundle/resources/permissions/models/current_can_manage/script index 9ac6f2cd41a..0ade9d8934b 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/script @@ -9,7 +9,7 @@ trace errcode $CLI bundle deploy &> out.deploy.txt # Register the model's numeric ID for output replacement. # The permissions API uses the numeric ID, not the model name. MODEL_ID=$($CLI model-registry get-model test-model | jq -r '.registered_model_databricks.id') -add_repl.py "$MODEL_ID" "FOO_MODEL_ID" +add_repl "$MODEL_ID" "FOO_MODEL_ID" print_requests.py //permissions > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/secret_scopes/permissions/script b/acceptance/bundle/resources/secret_scopes/permissions/script index 2b866baa75b..54f6a5d432c 100755 --- a/acceptance/bundle/resources/secret_scopes/permissions/script +++ b/acceptance/bundle/resources/secret_scopes/permissions/script @@ -2,7 +2,7 @@ if [ -z "$CLOUD_ENV" ]; then export TEST_SP_APPLICATION_ID="22222222-2222-2222-2222-222222222222" fi -echo "$TEST_SP_APPLICATION_ID:TEST_SP_APPLICATION_ID" >> ACC_REPLS +add_repl "$TEST_SP_APPLICATION_ID" TEST_SP_APPLICATION_ID export SECRET_SCOPE_NAME="test-scope-permissions-$UNIQUE_NAME" diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/script b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/script index 3028a1f6afa..5d444c6e4e9 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/script +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/script @@ -9,7 +9,7 @@ trap cleanup EXIT title "Deploy with started=false: warehouse created and then stopped" trace errcode $CLI bundle deploy WAREHOUSE_ID=$($CLI bundle summary -o json | jq -r '.resources.sql_warehouses.mywarehouse.id') -add_repl.py "$WAREHOUSE_ID" "WAREHOUSE_ID" +add_repl "$WAREHOUSE_ID" "WAREHOUSE_ID" trace print_requests.py //sql/warehouses rm -f out.requests.txt { trace errcode $CLI warehouses get "$WAREHOUSE_ID" | jq '.state'; } || true diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/script b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/script index 46cf94bdcc2..c9e7b940f13 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/script +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/script @@ -9,7 +9,7 @@ trap cleanup EXIT title "Deploy with started=false: warehouse created and then stopped" trace errcode $CLI bundle deploy WAREHOUSE_ID=$($CLI bundle summary -o json | jq -r '.resources.sql_warehouses.mywarehouse.id') -add_repl.py "$WAREHOUSE_ID" "WAREHOUSE_ID" +add_repl "$WAREHOUSE_ID" "WAREHOUSE_ID" trace print_requests.py //sql/warehouses rm -f out.requests.txt { trace errcode $CLI warehouses get "$WAREHOUSE_ID" | jq '.state'; } || true diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/script b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/script index 2033d4bb523..369839f4769 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/script +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/script @@ -9,7 +9,7 @@ trap cleanup EXIT title "Deploy with started=true: warehouse created and running" trace errcode $CLI bundle deploy WAREHOUSE_ID=$($CLI bundle summary -o json | jq -r '.resources.sql_warehouses.mywarehouse.id') -add_repl.py "$WAREHOUSE_ID" "WAREHOUSE_ID" +add_repl "$WAREHOUSE_ID" "WAREHOUSE_ID" trace print_requests.py //sql/warehouses rm -f out.requests.txt diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/script b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/script index dbef9250f28..9c834c66e55 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/script +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/script @@ -12,7 +12,7 @@ title "Initial deployment" trace $CLI bundle deploy original_endpoint_uuid=$($CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq -r '.id') -add_repl.py "$original_endpoint_uuid" "ORIGINAL_ENDPOINT_UUID" +add_repl "$original_endpoint_uuid" "ORIGINAL_ENDPOINT_UUID" trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{id, name, endpoint_type}' title "Delete and recreate remotely with the same name" @@ -20,7 +20,7 @@ trace $CLI vector-search-endpoints delete-endpoint "${endpoint_name}" trace $CLI vector-search-endpoints create-endpoint "${endpoint_name}" STANDARD | jq '{id, name, endpoint_type}' remote_recreated_endpoint_uuid=$($CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq -r '.id') -add_repl.py "$remote_recreated_endpoint_uuid" "REMOTE_RECREATED_ENDPOINT_UUID" +add_repl "$remote_recreated_endpoint_uuid" "REMOTE_RECREATED_ENDPOINT_UUID" trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{id, name, endpoint_type}' printf "Original endpoint UUID: %s\n" "$original_endpoint_uuid" diff --git a/acceptance/bundle/sync/dryrun/script b/acceptance/bundle/sync/dryrun/script index 10b501d9de4..7012af83248 100644 --- a/acceptance/bundle/sync/dryrun/script +++ b/acceptance/bundle/sync/dryrun/script @@ -5,7 +5,6 @@ cat > .gitignore << EOF ignored-folder/ script output.txt -repls.json EOF cleanup() { diff --git a/acceptance/bundle/sync/script b/acceptance/bundle/sync/script index b3fb2a7d62a..828c3d11eab 100644 --- a/acceptance/bundle/sync/script +++ b/acceptance/bundle/sync/script @@ -5,7 +5,6 @@ cat > .gitignore << EOF ignored-folder/ script output.txt -repls.json EOF cleanup() { diff --git a/acceptance/bundle/templates/record-deployment-history/script b/acceptance/bundle/templates/record-deployment-history/script index bad50ccad90..6a5762952ff 100644 --- a/acceptance/bundle/templates/record-deployment-history/script +++ b/acceptance/bundle/templates/record-deployment-history/script @@ -14,4 +14,4 @@ done # Keep one file in full to assert where the stanza lands in the document. mv output/default-python/my_default_python/databricks.yml out.default-python.databricks.yml -add_repl.py "$(grep -o 'uuid: .*' out.default-python.databricks.yml | cut -d ' ' -f2)" BUNDLE_UUID +add_repl "$(grep -o 'uuid: .*' out.default-python.databricks.yml | cut -d ' ' -f2)" BUNDLE_UUID diff --git a/acceptance/bundle/validate/immutable_workspace_paths/databricks.yml b/acceptance/bundle/validate/immutable_workspace_paths/databricks.yml index 001b2efbcc7..eec3c3e5158 100644 --- a/acceptance/bundle/validate/immutable_workspace_paths/databricks.yml +++ b/acceptance/bundle/validate/immutable_workspace_paths/databricks.yml @@ -7,8 +7,6 @@ experimental: sync: exclude: # Test framework files that are not part of the bundle source. - - "repls.json" - - "user_repls.json" - "script" - "*.toml" diff --git a/acceptance/bundle/validate/immutable_workspace_paths/output.txt b/acceptance/bundle/validate/immutable_workspace_paths/output.txt index fef8083634f..a41638ba774 100644 --- a/acceptance/bundle/validate/immutable_workspace_paths/output.txt +++ b/acceptance/bundle/validate/immutable_workspace_paths/output.txt @@ -1,9 +1,5 @@ >>> [CLI] bundle validate -o json -Warning: Pattern user_repls.json does not match any files - at sync.exclude[1] - in databricks.yml:11:7 - { "workspace": { "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/my-bundle/default/artifacts", diff --git a/acceptance/bundle/variables/lookup/script b/acceptance/bundle/variables/lookup/script index 462aefa4df4..58e451daaed 100644 --- a/acceptance/bundle/variables/lookup/script +++ b/acceptance/bundle/variables/lookup/script @@ -2,7 +2,7 @@ # provisioned in the cloud env, and we fall back to the name the local testserver # reports so the test passes in both. Mask it to a stable placeholder either way. export TEST_SP_DISPLAY_NAME="${TEST_SP_DISPLAY_NAME:-deco-test-spn}" -echo "$TEST_SP_DISPLAY_NAME:TEST_SP_DISPLAY_NAME" >> ACC_REPLS +add_repl "$TEST_SP_DISPLAY_NAME" TEST_SP_DISPLAY_NAME envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -o json | jq '.variables' diff --git a/acceptance/cmd/completion/script b/acceptance/cmd/completion/script index e5b0f324d65..d315d83839a 100644 --- a/acceptance/cmd/completion/script +++ b/acceptance/cmd/completion/script @@ -1,7 +1,7 @@ sethome "./home" # Track the home path for stable output across platforms. -add_repl.py "$HOME" HOME +add_repl "$HOME" HOME # Prevent Homebrew detection from affecting status output. export HOMEBREW_PREFIX=/nonexistent diff --git a/acceptance/cmd/sync-from-file/gitignore.test-fixture b/acceptance/cmd/sync-from-file/gitignore.test-fixture index eb483118609..48d52fd8cca 100644 --- a/acceptance/cmd/sync-from-file/gitignore.test-fixture +++ b/acceptance/cmd/sync-from-file/gitignore.test-fixture @@ -1,4 +1,3 @@ ignored-folder/ script output.txt -repls.json diff --git a/acceptance/cmd/sync/dryrun-missing-remote/script b/acceptance/cmd/sync/dryrun-missing-remote/script index 1cfd1e22215..65fb16ed000 100644 --- a/acceptance/cmd/sync/dryrun-missing-remote/script +++ b/acceptance/cmd/sync/dryrun-missing-remote/script @@ -5,7 +5,6 @@ script test.toml output.txt out.requests.txt -repls.json EOF # Use an ephemeral remote directory so concurrent test runs do not interfere. diff --git a/acceptance/cmd/sync/dryrun/script b/acceptance/cmd/sync/dryrun/script index e298c777b33..4018acf8872 100644 --- a/acceptance/cmd/sync/dryrun/script +++ b/acceptance/cmd/sync/dryrun/script @@ -9,7 +9,6 @@ cat > .gitignore << EOF ignored-folder/ script output.txt -repls.json EOF cleanup() { diff --git a/acceptance/cmd/sync/script b/acceptance/cmd/sync/script index 6d41a67b92f..67edd91facc 100644 --- a/acceptance/cmd/sync/script +++ b/acceptance/cmd/sync/script @@ -9,7 +9,6 @@ cat > .gitignore << EOF ignored-folder/ script output.txt -repls.json EOF cleanup() { diff --git a/acceptance/cmd/workspace/apps/run-local-node/script b/acceptance/cmd/workspace/apps/run-local-node/script index 58a139975b5..9749e979748 100755 --- a/acceptance/cmd/workspace/apps/run-local-node/script +++ b/acceptance/cmd/workspace/apps/run-local-node/script @@ -2,8 +2,8 @@ cd app # Ports are allocated per run so that tests in parallel worktrees do not collide. read -r APP_PORT PROXY_PORT DEBUG_PORT PREPARE_PORT <<< "$(free_port.py 4)" -add_repl.py "$PROXY_PORT" PROXY_PORT -add_repl.py "$DEBUG_PORT" DEBUG_PORT +add_repl "$PROXY_PORT" PROXY_PORT +add_repl "$DEBUG_PORT" DEBUG_PORT # The app and the proxy start asynchronously; allow up to 15s for them to come up. export RETRY_MAX_ATTEMPTS=30 diff --git a/acceptance/cmd/workspace/apps/run-local/script b/acceptance/cmd/workspace/apps/run-local/script index 2f293db29bb..fd8b8b3c23e 100755 --- a/acceptance/cmd/workspace/apps/run-local/script +++ b/acceptance/cmd/workspace/apps/run-local/script @@ -2,8 +2,8 @@ cd app # Ports are allocated per run so that tests in parallel worktrees do not collide. read -r APP_PORT PROXY_PORT ENTRY_POINT_PORT <<< "$(free_port.py 3)" -add_repl.py "$APP_PORT" APP_PORT -add_repl.py "$PROXY_PORT" PROXY_PORT +add_repl "$APP_PORT" APP_PORT +add_repl "$PROXY_PORT" PROXY_PORT # The app and the proxy start asynchronously; allow up to 15s for them to come up. export RETRY_MAX_ATTEMPTS=30 diff --git a/acceptance/script.prepare b/acceptance/script.prepare index c5dc9612120..d71d37aae9b 100644 --- a/acceptance/script.prepare +++ b/acceptance/script.prepare @@ -109,6 +109,12 @@ retry() { env -u MSYS_NO_PATHCONV retry.py "$@" } +add_repl() { + # Same MSYS_NO_PATHCONV workaround as envsubst above: otherwise Python cannot + # find add_repl.py on Windows. + env -u MSYS_NO_PATHCONV add_repl.py "$@" +} + # The engine_terraform_* keys reflect the deployment engine opt-in; engine_terraform_env # in particular mirrors $DATABRICKS_BUNDLE_ENGINE, which the acceptance matrix sets per # variant, so it diverges across the matrix. Drop both here so this shared helper stays diff --git a/acceptance/selftest/acc_repls/output.txt b/acceptance/selftest/acc_repls/output.txt index a4a044b6387..95f527be761 100644 --- a/acceptance/selftest/acc_repls/output.txt +++ b/acceptance/selftest/acc_repls/output.txt @@ -1,4 +1,4 @@ my job is [JOB_ID] -echo "$job_id:JOB_ID" >> ACC_REPLS -echo "[MY_TEXT]:MY_TEXT" >> ACC_REPLS +echo "$job_id:JOB_ID" >> "$ACC_REPLS" +echo "[MY_TEXT]:MY_TEXT" >> "$ACC_REPLS" grep ACC_REPLS script diff --git a/acceptance/selftest/acc_repls/script b/acceptance/selftest/acc_repls/script index 871fa13ad23..c5587d1f856 100644 --- a/acceptance/selftest/acc_repls/script +++ b/acceptance/selftest/acc_repls/script @@ -1,6 +1,6 @@ job_id=100200300 echo "my job is $job_id" -echo "$job_id:JOB_ID" >> ACC_REPLS -echo "ABCD:EFG:MY_TEXT" >> ACC_REPLS +echo "$job_id:JOB_ID" >> "$ACC_REPLS" +echo "ABCD:EFG:MY_TEXT" >> "$ACC_REPLS" grep ACC_REPLS script diff --git a/acceptance/workspace/lakeview/publish/script b/acceptance/workspace/lakeview/publish/script index 5bef1846f33..93b0044ecc1 100644 --- a/acceptance/workspace/lakeview/publish/script +++ b/acceptance/workspace/lakeview/publish/script @@ -3,5 +3,5 @@ trace $CLI workspace mkdirs /Users/tester@databricks.com trace $CLI lakeview create --json '{"display_name": "Test Dashboard", "parent_path": "/Users/tester@databricks.com", "warehouse_id": "test-warehouse"}' -o json > tmp.json dashboard_id=$(jq -r .dashboard_id < tmp.json) rm tmp.json -echo "$dashboard_id:DASHBOARD_ID" >> ACC_REPLS +add_repl "$dashboard_id" DASHBOARD_ID trace $CLI lakeview publish $dashboard_id From 5bbc12c1c36770ee5092bc25e64aab3e58df987e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 14 Aug 2026 11:18:49 +0200 Subject: [PATCH 2/3] acc: single format in ACC_REPLS Every line is now a JSON object: "Old" for a regular expression from the harness, "Literal" for a value added by add_repl.py. Previously the two kinds were told apart by a leading "{", so a value starting with "{" was silently dropped. The harness skips the lines it wrote itself by their count instead. Co-authored-by: Isaac --- .agent/rules/testing.md | 2 +- acceptance/acceptance_test.go | 45 +++++++++++++----------- acceptance/bin/add_repl.py | 22 ++++++------ acceptance/bin/repls.py | 43 ++++++++++++---------- acceptance/selftest/acc_repls/output.txt | 7 ++-- acceptance/selftest/acc_repls/script | 15 ++++++-- 6 files changed, 77 insertions(+), 57 deletions(-) diff --git a/.agent/rules/testing.md b/.agent/rules/testing.md index c6d051613c4..e3c8dfa70f8 100644 --- a/.agent/rules/testing.md +++ b/.agent/rules/testing.md @@ -169,7 +169,7 @@ Available on `PATH` during test execution (from `acceptance/bin/`): - `print_requests.py //path [^//exclude] [--get] [--sort] [--unique] [--oneline] [--keep]`: print recorded HTTP requests matching path filters. Requires `RecordRequests = true` in `test.toml`. Excludes GET by default (`--get` includes them); clears `out.requests.txt` afterwards (`--keep` retains it). `^` prefix excludes a path; multiple positive filters are OR'd together. `--sort` orders output deterministically (use when the request set is order-independent), `--unique` collapses consecutive duplicates (e.g. repeated polls), `--oneline` prints one request per line. - `replace_ids.py [-t TARGET]`: read deployment state and add `[NAME_ID]` replacements for all resource IDs. - `read_id.py [-t TARGET] NAME`: read ID of a single resource from state, print it, and add a `[NAME_ID]` replacement. -- `add_repl VALUE REPLACEMENT`: add a custom replacement (VALUE will be replaced with `[REPLACEMENT]` in output). Wraps `add_repl.py`, which appends to `$ACC_REPLS` — the file holding every replacement applied to the output, read back by the harness and by `diff.py` / `sort_lines.py --repl`. +- `add_repl VALUE REPLACEMENT`: add a custom replacement (VALUE will be replaced with `[REPLACEMENT]` in output). Wraps `add_repl.py`, which appends a JSON line to `$ACC_REPLS` — the file holding every replacement applied to the output, read back by the harness and by `diff.py` / `sort_lines.py --repl`. Always go through the helper; do not write `$ACC_REPLS` from a script. - `update_file.py FILENAME OLD NEW`: replace all occurrences of OLD with NEW in FILENAME. Errors if OLD is not found. Cannot be used on `output.txt`. - `find.py REGEX [--expect N]`: find files matching regex in current directory. `--expect N` asserts an exact count. - `diff.py DIR1 DIR2` or `diff.py FILE1 FILE2`: recursive diff with test replacements applied. diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index ead8b7cf378..6b11e812763 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -10,7 +10,6 @@ import ( "flag" "fmt" "io" - "io/fs" "maps" "math/rand/v2" "net/http" @@ -104,13 +103,14 @@ const ( // Env var with the path to the file holding all replacements applied to the output. // It is kept outside of the test directory, otherwise "bundle deploy" uploads it. // - // The harness writes its own replacements there, one JSON object per line. Scripts add - // literal ones with the add_repl helper, which appends ":" lines: + // Every line is one replacement encoded as a JSON object: "Old" is a regular expression + // (written by the harness), "Literal" is a value to replace verbatim (appended by + // add_repl.py). The harness writes its own replacements first, then the scripts add theirs: // // $ job_id=100200300 // $ add_repl "$job_id" MY_JOB # replaces 100200300 with [MY_JOB] in the output // - // Both kinds are read back here (see loadUserReplacements) and by the python helpers + // The file is read back here (see loadScriptReplacements) and by the python helpers // (see bin/repls.py). ReplsEnvVar = "ACC_REPLS" ) @@ -911,6 +911,7 @@ func runTest(t *testing.T, replsLines.WriteByte('\n') } testutil.WriteFile(t, replsPath, replsLines.String()) + replsWritten := len(repls.Repls) if coverDir != "" { // Creating individual coverage directory for each test, because writing to the same one @@ -1012,7 +1013,7 @@ func runTest(t *testing.T, formatOutput(out, err) require.NoError(t, out.Close()) - loadUserReplacements(t, &repls, replsPath) + loadScriptReplacements(t, &repls, replsPath, replsWritten) printedRepls := false @@ -1816,28 +1817,32 @@ func setupTerraform(t *testing.T, cwd, buildDir string, repls *testdiff.Replacem repls.SetPath(terraformExecPath, "[TERRAFORM]") } -// loadUserReplacements adds replacements appended by the scripts to replsPath. -// The JSON lines written there by the harness itself are already part of repls. -func loadUserReplacements(t *testing.T, repls *testdiff.ReplacementsContext, replsPath string) { +// loadScriptReplacements adds the replacements appended to replsPath by the scripts. +// The first offset lines were written by the harness itself and are already in repls. +func loadScriptReplacements(t *testing.T, repls *testdiff.ReplacementsContext, replsPath string, offset int) { b, err := os.ReadFile(replsPath) - if errors.Is(err, fs.ErrNotExist) { - return - } require.NoError(t, err) - lines := strings.SplitSeq(string(b), "\n") - for line := range lines { + lines := strings.Split(string(b), "\n") + for _, line := range lines[min(offset, len(lines)):] { line = strings.TrimSpace(line) - if len(line) == 0 || strings.HasPrefix(line, "{") { + if len(line) == 0 { + continue + } + // Scripts only add literal replacements; regular expressions come from the harness. + var entry struct { + Literal string + New string + Order int + } + if err := json.Unmarshal([]byte(line), &entry); err != nil { + t.Errorf("Error parsing %s: %#v: %s", ReplsEnvVar, line, err) continue } - items := strings.Split(line, ":") - if len(items) <= 1 { - t.Errorf("Error parsing %s: %#v", ReplsEnvVar, line) + if entry.Literal == "" || entry.New == "" { + t.Errorf("Incomplete entry in %s: %#v", ReplsEnvVar, line) continue } - repl := items[len(items)-1] - old := line[:len(line)-len(repl)-1] - repls.SetWithOrder(old, "["+repl+"]", -100) + repls.SetWithOrder(entry.Literal, entry.New, entry.Order) } } diff --git a/acceptance/bin/add_repl.py b/acceptance/bin/add_repl.py index 231e99eed1b..4b5df5c530e 100755 --- a/acceptance/bin/add_repl.py +++ b/acceptance/bin/add_repl.py @@ -6,23 +6,20 @@ """ import argparse +import json import os +import sys from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent)) +from repls import USER_ORDER, read_entries + ACC_REPLS = Path(os.environ["ACC_REPLS"]) def get_repls(): - result = {} - if ACC_REPLS.exists(): - for line in ACC_REPLS.open(): - line = line.strip() - # Skip the harness replacements; only the records added here can collide. - if not line or line.startswith("{"): - continue - value, repl = line.rsplit(":", 1) - result[repl] = value - return result + # Only the literal entries are added here, so only those can collide. + return {item["New"] for item in read_entries() if item.get("Literal") is not None} def add_repl(value, repl): @@ -32,10 +29,11 @@ def add_repl(value, repl): r = f"{repl}_{extra}" else: r = repl - if r in existing: + if f"[{r}]" in existing: continue with ACC_REPLS.open("a") as fobj: - fobj.write(f"{value}:{r}\n") + json.dump({"Literal": value, "New": f"[{r}]", "Order": USER_ORDER}, fobj) + fobj.write("\n") break diff --git a/acceptance/bin/repls.py b/acceptance/bin/repls.py index 0599e1b1ee6..1fffe2c949f 100644 --- a/acceptance/bin/repls.py +++ b/acceptance/bin/repls.py @@ -2,10 +2,9 @@ """ Read the replacements applied to the test output from $ACC_REPLS. -The file holds two kinds of lines: - - JSON objects written by the test harness, where "Old" is a regular expression; - - ":" records appended by add_repl.py, where is a literal - that is replaced with []. +Every line is one replacement encoded as a JSON object: "Old" is a regular expression +(written by the test harness), "Literal" is a value to replace verbatim (appended by +add_repl.py). "New" is the replacement, "Order" defines the order they are applied in. """ import json @@ -14,33 +13,41 @@ import sys from pathlib import Path -# Order of the replacements added by the scripts. Matches loadUserReplacements in -# acceptance_test.go, which applies them before the ones written by the harness. +# Order of the replacements added by the scripts, so that they are applied before the ones +# from the harness: a job id must become [MY_JOB] rather than [NUMID]. USER_ORDER = -100 +def read_entries(): + """Return the raw entries of $ACC_REPLS.""" + result = [] + for line in Path(os.environ["ACC_REPLS"]).read_text().splitlines(): + line = line.strip() + if line: + result.append(json.loads(line)) + return result + + def read_repls(): """Return (pattern, replacement) pairs in the order they must be applied.""" entries = [] - for line in Path(os.environ["ACC_REPLS"]).read_text().splitlines(): - line = line.strip() - if not line: - continue + for item in read_entries(): + order = item.get("Order", 0) + new = item["New"] + literal = item.get("Literal") - if line.startswith("{"): - item = json.loads(line) + if literal is None: # "Distinct" is not honoured here; unlike the harness, we do not number the matches. - entries.append((item.get("Order", 0), item["Old"], item["New"])) + entries.append((order, item["Old"], new)) continue - value, name = line.rsplit(":", 1) # Set() in libs/testdiff also registers the JSON-encoded form of the value, so that # values with quotes or backslashes are replaced inside JSON output as well. - encoded = json.dumps(value, ensure_ascii=False)[1:-1] - if encoded != value: - entries.append((USER_ORDER, re.escape(encoded), f"[{name}]")) - entries.append((USER_ORDER, re.escape(value), f"[{name}]")) + encoded = json.dumps(literal, ensure_ascii=False)[1:-1] + if encoded != literal: + entries.append((order, re.escape(encoded), new)) + entries.append((order, re.escape(literal), new)) # Stable sort: replacements with the same order are applied in the order they were added. entries.sort(key=lambda entry: entry[0]) diff --git a/acceptance/selftest/acc_repls/output.txt b/acceptance/selftest/acc_repls/output.txt index 95f527be761..1f0afec6e84 100644 --- a/acceptance/selftest/acc_repls/output.txt +++ b/acceptance/selftest/acc_repls/output.txt @@ -1,4 +1,5 @@ my job is [JOB_ID] -echo "$job_id:JOB_ID" >> "$ACC_REPLS" -echo "[MY_TEXT]:MY_TEXT" >> "$ACC_REPLS" -grep ACC_REPLS script +my text is [MY_TEXT] +my json is [CURLY] +my tricky value is [TRICKY] +{"value": "[TRICKY]"} diff --git a/acceptance/selftest/acc_repls/script b/acceptance/selftest/acc_repls/script index c5587d1f856..2ed324a85b5 100644 --- a/acceptance/selftest/acc_repls/script +++ b/acceptance/selftest/acc_repls/script @@ -1,6 +1,15 @@ job_id=100200300 +add_repl "$job_id" JOB_ID + +# Values that cannot be encoded as plain text: a colon, a leading curly brace, quotes and backslashes. +add_repl "ABCD:EFG" MY_TEXT +add_repl '{"a": 1}' CURLY +add_repl 'back\slash "quoted"' TRICKY + echo "my job is $job_id" -echo "$job_id:JOB_ID" >> "$ACC_REPLS" -echo "ABCD:EFG:MY_TEXT" >> "$ACC_REPLS" +echo "my text is ABCD:EFG" +echo 'my json is {"a": 1}' +echo 'my tricky value is back\slash "quoted"' -grep ACC_REPLS script +# The JSON-encoded form of the value is replaced as well. +echo '{"value": "back\\slash \"quoted\""}' From 2f141ab08d845a9a5d54073abb154df67694830d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 14 Aug 2026 11:19:33 +0200 Subject: [PATCH 3/3] acc: fix stale comment in read_id.py Co-authored-by: Isaac --- acceptance/bin/read_id.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bin/read_id.py b/acceptance/bin/read_id.py index 27b98c8192f..87cd2954bdf 100755 --- a/acceptance/bin/read_id.py +++ b/acceptance/bin/read_id.py @@ -3,7 +3,7 @@ Print id of the resource from the state. Update ACC_REPLS for a given ID. Example: read_id.py foo - Output job_id, e.g. "5555" and update ACC_REPLS with record "5555:FOO_ID" + Output job_id, e.g. "5555" and update ACC_REPLS to replace "5555" with [FOO_ID] Usage: [attr...] """