From 2420e841f24c093973184f703168f86d931a82d6 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Thu, 8 Oct 2026 21:51:56 -0300 Subject: [PATCH 01/18] test: align private hook fixtures with ACR pool guard Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_private_network_hooks.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_private_network_hooks.py b/tests/test_private_network_hooks.py index 665c8f90..150ed81e 100644 --- a/tests/test_private_network_hooks.py +++ b/tests/test_private_network_hooks.py @@ -15,6 +15,7 @@ PWSH = shutil.which("pwsh") BASH = shutil.which("bash") VALUES = """NETWORK_ISOLATION="true" +ACR_TASK_AGENT_POOL="test-pool" DEPLOYMENT_TOPOLOGY="classic" APP_CONFIG_ENDPOINT="https://config.azconfig.io" AZURE_RESOURCE_GROUP="test-group" @@ -273,6 +274,22 @@ def check_cases(self, shell: str) -> None: self.assertEqual([], events, output) self.assertIn("Run azd provision first", output) + for build_mode, allowed in (("acr-task", False), ("local", True)): + with self.subTest(shell=shell, build_mode=build_mode, pool_missing=True): + values = VALUES.replace('ACR_TASK_AGENT_POOL="test-pool"', 'ACR_TASK_AGENT_POOL=""') + result, events = self.run_hook( + shell, "preDeploy", 0, values=values, + process_env={"BUILD_MODE": build_mode}, + ) + output = result.stdout + result.stderr + self.assertIn("probe", events, output) + if allowed: + self.assertIn("azure-boundary", events, output) + else: + self.assertEqual(1, result.returncode, output) + self.assertNotIn("azure-boundary", events, output) + self.assertIn("ACR_TASK_AGENT_POOL is empty", output) + def integrated_cases(self, shell: str) -> None: for hook in ("postProvision", "preDeploy"): for extra, process, endpoint_fail, expected in ( From 91da94f2e1a995843c221a16b69b775f842c8971 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Thu, 8 Oct 2026 21:51:56 -0300 Subject: [PATCH 02/18] test: align private hook fixtures with ACR pool guard Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/test_private_network_hooks.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_private_network_hooks.py b/tests/test_private_network_hooks.py index 665c8f90..150ed81e 100644 --- a/tests/test_private_network_hooks.py +++ b/tests/test_private_network_hooks.py @@ -15,6 +15,7 @@ PWSH = shutil.which("pwsh") BASH = shutil.which("bash") VALUES = """NETWORK_ISOLATION="true" +ACR_TASK_AGENT_POOL="test-pool" DEPLOYMENT_TOPOLOGY="classic" APP_CONFIG_ENDPOINT="https://config.azconfig.io" AZURE_RESOURCE_GROUP="test-group" @@ -273,6 +274,22 @@ def check_cases(self, shell: str) -> None: self.assertEqual([], events, output) self.assertIn("Run azd provision first", output) + for build_mode, allowed in (("acr-task", False), ("local", True)): + with self.subTest(shell=shell, build_mode=build_mode, pool_missing=True): + values = VALUES.replace('ACR_TASK_AGENT_POOL="test-pool"', 'ACR_TASK_AGENT_POOL=""') + result, events = self.run_hook( + shell, "preDeploy", 0, values=values, + process_env={"BUILD_MODE": build_mode}, + ) + output = result.stdout + result.stderr + self.assertIn("probe", events, output) + if allowed: + self.assertIn("azure-boundary", events, output) + else: + self.assertEqual(1, result.returncode, output) + self.assertNotIn("azure-boundary", events, output) + self.assertIn("ACR_TASK_AGENT_POOL is empty", output) + def integrated_cases(self, shell: str) -> None: for hook in ("postProvision", "preDeploy"): for extra, process, endpoint_fail, expected in ( From d1bfb5f05c664e2e7770bf9262408ac0e7335637 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 08:52:37 -0300 Subject: [PATCH 03/18] fix: align custom application hooks with definition contracts Resolve directory selections in both lifecycle variants, deploy named services from the application definition project, and validate hosted greetings using the declared protocol. Narrow VM SKU preflight queries to virtual machines. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- config/deployment/hosted.py | 59 ++++- infra/scripts/Invoke-PreflightChecks.ps1 | 2 +- .../scripts/Invoke-PreflightChecks.Tests.ps1 | 27 +++ samples/custom-app/README.md | 4 +- samples/custom-app/hosted/src/main.py | 1 + scripts/preDeploy.ps1 | 24 ++- scripts/preDeploy.sh | 36 +++- scripts/preProvision.ps1 | 1 + scripts/preProvision.sh | 3 + tests/test_custom_application_hooks.py | 202 ++++++++++++++++++ tests/test_hosted_smoke.py | 25 +++ 11 files changed, 360 insertions(+), 24 deletions(-) create mode 100644 tests/test_custom_application_hooks.py diff --git a/config/deployment/hosted.py b/config/deployment/hosted.py index 5a879d9e..0add830e 100644 --- a/config/deployment/hosted.py +++ b/config/deployment/hosted.py @@ -5,13 +5,54 @@ import argparse import json import logging +from pathlib import Path import sys from urllib.parse import urlsplit, urlunsplit +import yaml MAX_SMOKE_OUTPUT = 1024 * 1024 +def service_protocol(project: Path, service_name: str) -> str: + """Choose a supported greeting protocol from the selected service.""" + document = yaml.safe_load(project.read_text(encoding="utf-8")) + if not isinstance(document, dict) or not isinstance(document.get("services"), dict): + raise ValueError("Custom hosted project must declare services.") + service = document["services"][service_name] + if not isinstance(service, dict) or not isinstance(service.get("protocols"), list): + raise ValueError("Custom hosted service must declare protocols.") + protocols = { + entry["protocol"] for entry in service["protocols"] + if isinstance(entry, dict) and isinstance(entry.get("protocol"), str) + } + for protocol in ("invocations", "responses"): + if protocol in protocols: + return protocol + raise ValueError("Custom hosted service must declare invocations or responses for its greeting smoke.") + + +def validate_responses_smoke_output(output: str) -> None: + """Validate raw Responses HTTP JSON or SSE without accepting progress text.""" + if len(output) > MAX_SMOKE_OUTPUT: + raise ValueError("Smoke response exceeds the bounded validation size.") + body = output.replace("\r\n", "\n").strip() + if body.startswith("HTTP/"): + headers, separator, body = body.partition("\n\n") + status = headers.splitlines()[0].split() + if not separator or len(status) < 2 or not status[1].isdigit() or not 200 <= int(status[1]) < 300: + raise ValueError("Hosted greeting returned an unsuccessful HTTP response; body omitted.") + if body.lstrip().startswith("{"): + try: + response = json.loads(body) + except ValueError: + raise ValueError("Malformed Responses greeting; body omitted.") from None + if not isinstance(response, dict) or response.get("object") != "response": + raise ValueError("Invalid Responses greeting envelope; body omitted.") + body = json.dumps({"type": "response.completed", "response": response}) + validate_smoke_output(body) + + def _smoke_events(output: str) -> list[dict]: """Read Responses SSE or CLI JSON event lines, allowing azd progress text. @@ -136,15 +177,29 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() mode = parser.add_mutually_exclusive_group(required=True) mode.add_argument("--invocations-endpoint") + mode.add_argument("--service-protocol", type=Path, metavar="AZURE_YAML") + parser.add_argument("--service-name") + mode.add_argument("--validate-responses-smoke", action="store_true") mode.add_argument( "--validate-smoke", action="store_true", help="Validate captured greeting Responses SSE/JSON events from stdin; no network calls.", ) args = parser.parse_args(argv) - if args.validate_smoke: + if args.service_protocol: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + try: + if not args.service_name: + raise ValueError("--service-name is required.") + print(service_protocol(args.service_protocol, args.service_name)) + except (KeyError, TypeError, OSError, ValueError, yaml.YAMLError): + logging.error("Could not resolve the custom hosted service greeting protocol.") + return 1 + return 0 + if args.validate_smoke or args.validate_responses_smoke: logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") try: - validate_smoke_output(sys.stdin.read(MAX_SMOKE_OUTPUT + 1)) + validator = validate_responses_smoke_output if args.validate_responses_smoke else validate_smoke_output + validator(sys.stdin.read(MAX_SMOKE_OUTPUT + 1)) except ValueError as exc: logging.error("%s", exc) return 1 diff --git a/infra/scripts/Invoke-PreflightChecks.ps1 b/infra/scripts/Invoke-PreflightChecks.ps1 index 19c0abce..23d381f6 100644 --- a/infra/scripts/Invoke-PreflightChecks.ps1 +++ b/infra/scripts/Invoke-PreflightChecks.ps1 @@ -1187,7 +1187,7 @@ function Get-VmSkuInfo { [Parameter(Mandatory)] [string]$VmSize ) if ([string]::IsNullOrWhiteSpace($Location) -or [string]::IsNullOrWhiteSpace($VmSize)) { return $null } - $skus = Invoke-AzCliRaw -Arguments @('vm', 'list-skus', '--location', $Location, '--size', $VmSize, '--all', '-o', 'json') + $skus = Invoke-AzCliRaw -Arguments @('vm', 'list-skus', '--location', $Location, '--size', $VmSize, '--resource-type', 'virtualMachines', '--all', '-o', 'json') if (-not $skus) { return $null } $match = @($skus | Where-Object { $_.name -eq $VmSize -and $_.resourceType -eq 'virtualMachines' } | Select-Object -First 1) if (-not $match) { return $null } diff --git a/infra/tests/scripts/Invoke-PreflightChecks.Tests.ps1 b/infra/tests/scripts/Invoke-PreflightChecks.Tests.ps1 index 10f40a78..3611eb94 100644 --- a/infra/tests/scripts/Invoke-PreflightChecks.Tests.ps1 +++ b/infra/tests/scripts/Invoke-PreflightChecks.Tests.ps1 @@ -128,6 +128,33 @@ finally { Remove-Item -Path $paramsFile -ErrorAction SilentlyContinue } +# -------------------------------------------------------------------------- +Write-Host 'VM SKU lookup is restricted to virtual machines' -ForegroundColor Cyan +$originalAzCliRaw = ${function:Invoke-AzCliRaw} +try { + function Invoke-AzCliRaw { + param([string[]]$Arguments) + $script:VmSkuArguments = $Arguments + return @([pscustomobject]@{ + name = 'Standard_D2s_v3' + resourceType = 'virtualMachines' + capabilities = @([pscustomobject]@{ name = 'vCPUs'; value = '2' }) + restrictions = @() + family = 'standardDSv3Family' + }) + } + $sku = Get-VmSkuInfo -Location 'westus3' -VmSize 'Standard_D2s_v3' + $typeIndex = [Array]::IndexOf($script:VmSkuArguments, '--resource-type') + Assert-True 'SKU request filters virtualMachines server-side' ( + $typeIndex -ge 0 -and $script:VmSkuArguments[$typeIndex + 1] -eq 'virtualMachines' + ) + Assert-True 'Filtered SKU keeps vCPU and availability facts' ( + $sku.Name -eq 'Standard_D2s_v3' -and $sku.VCpus -eq 2 + ) +} finally { + ${function:Invoke-AzCliRaw} = $originalAzCliRaw +} + # -------------------------------------------------------------------------- Write-Host 'Boolean parameter values accept only true and false' -ForegroundColor Cyan foreach ($validValue in @($true, $false, 'true', ' FALSE ')) { diff --git a/samples/custom-app/README.md b/samples/custom-app/README.md index d4b7c251..48e3837d 100644 --- a/samples/custom-app/README.md +++ b/samples/custom-app/README.md @@ -23,5 +23,5 @@ Both read the platform outputs contract (`AGENTLZ_PLATFORM_OUTPUTS`, label hosts exactly one application. Rules: no lifecycle hooks, no remote URLs, no role names; permissions come only -from capability profiles. See "Build your own application" in the Agent -Landing Zone section of the AI Landing Zones documentation for the full guide. +from capability profiles. See [Custom applications](https://azure.github.io/AI-Landing-Zones/agent-landing-zone/build-your-own-app/) +for the folder layout, source pinning, and hosted greeting protocol requirements. diff --git a/samples/custom-app/hosted/src/main.py b/samples/custom-app/hosted/src/main.py index 95b59cc0..387f7452 100644 --- a/samples/custom-app/hosted/src/main.py +++ b/samples/custom-app/hosted/src/main.py @@ -72,6 +72,7 @@ def do_POST(self) -> None: # noqa: N802 - http.server API { "type": "message", "role": "assistant", + "status": "completed", "content": [{"type": "output_text", "text": answer()}], } ], diff --git a/scripts/preDeploy.ps1 b/scripts/preDeploy.ps1 index e24d280e..835b23ff 100644 --- a/scripts/preDeploy.ps1 +++ b/scripts/preDeploy.ps1 @@ -126,6 +126,7 @@ if ($foundationMissing) { # Re-validate the application definition and compare it with the binding. $appDefinitionPath = if ($globalEnv.AGENTLZ_APP_DEFINITION) { "$($globalEnv.AGENTLZ_APP_DEFINITION)" } elseif ($env:AGENTLZ_APP_DEFINITION) { $env:AGENTLZ_APP_DEFINITION } else { Join-Path $repoRoot 'app-definition.json' } if (-not [IO.Path]::IsPathRooted($appDefinitionPath)) { $appDefinitionPath = Join-Path $repoRoot $appDefinitionPath } +if (Test-Path -LiteralPath $appDefinitionPath -PathType Container) { $appDefinitionPath = Join-Path $appDefinitionPath 'app-definition.json' } if (-not (Test-Path -LiteralPath $appDefinitionPath -PathType Leaf)) { Write-Error "AGENTLZ_APP_DEFINITION points to $appDefinitionPath, which does not exist." exit 1 @@ -478,13 +479,13 @@ foreach ($c in $manifest.components) { } # Components from the application definition that are not release-managed -# (manifest.json) components: each folder is its own azd project. +# (manifest.json) components share the definition folder's azd project. foreach ($dc in $definitionComponents) { $componentName = "$($dc.name)" if (Get-ManifestComponentForDefinition $componentName) { continue } $componentPath = Join-Path $appDefinitionDir "$($dc.path)" - if (-not (Test-Path -LiteralPath (Join-Path $componentPath 'azure.yaml') -PathType Leaf)) { - Write-Error "The app folder $componentPath must contain its own azure.yaml." + if (-not (Test-Path -LiteralPath (Join-Path $appDefinitionDir 'azure.yaml') -PathType Leaf)) { + Write-Error "The app folder $appDefinitionDir must contain its own azure.yaml." $hadErrors = $true continue } @@ -499,16 +500,18 @@ foreach ($dc in $definitionComponents) { } } if (Test-Path -LiteralPath $dotAzure) { - Copy-Item $dotAzure $componentPath -Recurse -Force -Container + Copy-Item $dotAzure $appDefinitionDir -Recurse -Force -Container } Write-Host ("Deploying {0} ({1}) from {2}" -f $componentName, $dc.kind, $componentPath) -ForegroundColor Cyan - Push-Location $componentPath + Push-Location $appDefinitionDir try { if ($sourceDigest) { & azd env set AGENTLZ_IMAGE_DIGEST $sourceDigest --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt | Out-Null if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: failed to pin image digest."; $hadErrors = $true; continue } } if ("$($dc.kind)" -eq 'azure.ai.agent') { + $smokeProtocol = (& python -c "import os, runpy, sys; sys.path.insert(0, os.environ['AGENTLZ_REPO_ROOT']); sys.argv = ['config.deployment.hosted'] + sys.argv[1:]; runpy.run_module('config.deployment.hosted', run_name='__main__')" --service-protocol (Join-Path $appDefinitionDir 'azure.yaml') --service-name $componentName | Out-String).Trim() + if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: cannot resolve the greeting protocol."; $hadErrors = $true; continue } # prepareHostedDeployment is not run here: it builds only the manifest-pinned # bundled orchestrator image (HOSTED_AGENT_*). A custom azure.ai.agent # component deploys from its own child azd project, which builds its image, @@ -521,16 +524,19 @@ foreach ($dc in $definitionComponents) { if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: hosted agent deployment failed."; $hadErrors = $true; continue } $smokePayloadPath = Join-Path ([IO.Path]::GetTempPath()) "agentlz-agent-smoke-$([guid]::NewGuid().ToString('N')).json" try { - [IO.File]::WriteAllText($smokePayloadPath, '{"messages":[{"role":"user","content":"Hello!"}]}', [Text.UTF8Encoding]::new($false)) - $smokeOutput = (& azd ai agent invoke --protocol invocations --new-session --timeout 180 --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt --input-file $smokePayloadPath 2>&1 | Out-String) + $smokePayload = if ($smokeProtocol -eq 'responses') { '{"input":[{"role":"user","content":"Hello!"}],"stream":true}' } else { '{"messages":[{"role":"user","content":"Hello!"}]}' } + $smokeOptions = if ($smokeProtocol -eq 'responses') { @('--output', 'raw') } else { @() } + $smokeValidation = if ($smokeProtocol -eq 'responses') { '--validate-responses-smoke' } else { '--validate-smoke' } + [IO.File]::WriteAllText($smokePayloadPath, $smokePayload, [Text.UTF8Encoding]::new($false)) + $smokeOutput = (& azd ai agent invoke $componentName --protocol $smokeProtocol --new-session --timeout 180 --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt --input-file $smokePayloadPath @smokeOptions 2>&1 | Out-String) if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: smoke request failed."; $hadErrors = $true; continue } - $smokeOutput | & python -c "import os, runpy, sys; sys.path.insert(0, os.environ['AGENTLZ_REPO_ROOT']); sys.argv = ['config.deployment.hosted'] + sys.argv[1:]; runpy.run_module('config.deployment.hosted', run_name='__main__')" --validate-smoke + $smokeOutput | & python -c "import os, runpy, sys; sys.path.insert(0, os.environ['AGENTLZ_REPO_ROOT']); sys.argv = ['config.deployment.hosted'] + sys.argv[1:]; runpy.run_module('config.deployment.hosted', run_name='__main__')" $smokeValidation if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: smoke test did not pass."; $hadErrors = $true; continue } } finally { Remove-Item -LiteralPath $smokePayloadPath -Force -ErrorAction SilentlyContinue } } else { - & azd deploy --all --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt + & azd deploy $componentName --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: Container App deployment failed."; $hadErrors = $true; continue } } Write-Host "${componentName}: deployed." -ForegroundColor Green diff --git a/scripts/preDeploy.sh b/scripts/preDeploy.sh index 10ff1801..cc5dd287 100755 --- a/scripts/preDeploy.sh +++ b/scripts/preDeploy.sh @@ -74,7 +74,7 @@ copy_dot_azure() { if command -v rsync >/dev/null 2>&1; then rsync -a "$1/" "$2/.azure/" >/dev/null 2>&1 || cp -R "$1" "$2"/ >/dev/null 2>&1 else - cp -R "$1" "$2"/ >/dev/null 2>&1 || true + cp -R "$1" "$2"/ >/dev/null 2>&1 fi } @@ -138,6 +138,9 @@ case "$app_definition_path" in /*) ;; *) app_definition_path="$repo_root/$app_definition_path" ;; esac +if [ -d "$app_definition_path" ]; then + app_definition_path="$app_definition_path/app-definition.json" +fi [ -f "$app_definition_path" ] || { red "AGENTLZ_APP_DEFINITION points to $app_definition_path, which does not exist."; exit 1; } command -v python3 >/dev/null 2>&1 || { red "python3 is required to validate the application definition."; exit 1; } if [ -z "${AGENTLZ_APP_ID:-}" ]; then @@ -424,8 +427,8 @@ while IFS= read -r dcomp; do c_path="$app_definition_dir/$(printf "%s" "$dcomp" | jq -r '.path')" c_commit="$(printf "%s" "$dcomp" | jq -r '.source.commit // empty')" c_digest="$(printf "%s" "$dcomp" | jq -r '.source.imageDigest // empty')" - if [ ! -f "$c_path/azure.yaml" ]; then - red "The app folder $c_path must contain its own azure.yaml." + if [ ! -f "$app_definition_dir/azure.yaml" ]; then + red "The app folder $app_definition_dir must contain its own azure.yaml." had_errors=1; continue fi if [ -n "$c_commit" ]; then @@ -435,37 +438,50 @@ while IFS= read -r dcomp; do had_errors=1; continue fi fi - copy_dot_azure "$dot_azure" "$c_path" + copy_dot_azure "$dot_azure" "$app_definition_dir" || { + red "$c_name: failed to copy the azd environment."; had_errors=1; continue + } cyan "Deploying $c_name ($c_kind) from $c_path" env_name="${AZURE_ENV_NAME:-}" if [ -n "$c_digest" ]; then - (cd "$c_path" && azd env set AGENTLZ_IMAGE_DIGEST "$c_digest" --environment "$env_name" --no-prompt >/dev/null) || { + (cd "$app_definition_dir" && azd env set AGENTLZ_IMAGE_DIGEST "$c_digest" --environment "$env_name" --no-prompt >/dev/null) || { red "$c_name: failed to pin image digest."; had_errors=1; continue } fi if [ "$c_kind" = "azure.ai.agent" ]; then + smoke_protocol="$(cd "$repo_root" && python3 -m config.deployment.hosted --service-protocol "$app_definition_dir/azure.yaml" --service-name "$c_name")" || { + red "$c_name: cannot resolve the greeting protocol."; had_errors=1; continue + } # prepareHostedDeployment is not run here: it builds only the manifest-pinned # bundled orchestrator image (HOSTED_AGENT_*). A custom azure.ai.agent # component deploys from its own child azd project, which builds its image, # or pins source.imageDigest through AGENTLZ_IMAGE_DIGEST (T079 limitation). ( - cd "$c_path" && + cd "$app_definition_dir" && azd env set FOUNDRY_PROJECT_ENDPOINT "${AZURE_AI_PROJECT_ENDPOINT:-}" --environment "$env_name" --no-prompt >/dev/null && azd env set AZURE_AI_PROJECT_ID "${AZURE_AI_PROJECT_RESOURCE_ID:-}" --environment "$env_name" --no-prompt >/dev/null && azd deploy "$c_name" --environment "$env_name" --no-prompt ) || { red "$c_name: hosted agent deployment failed."; had_errors=1; continue; } agent_smoke_payload="$(mktemp "${TMPDIR:-/tmp}/agentlz-agent-smoke-XXXXXX")" - printf '%s\n' '{"messages":[{"role":"user","content":"Hello!"}]}' >"$agent_smoke_payload" - if ! agent_smoke_output="$(cd "$c_path" && azd ai agent invoke --protocol invocations --new-session --timeout 180 --environment "$env_name" --no-prompt --input-file "$agent_smoke_payload" 2>&1)"; then + smoke_options=() + smoke_validation=--validate-smoke + if [ "$smoke_protocol" = responses ]; then + printf '%s\n' '{"input":[{"role":"user","content":"Hello!"}],"stream":true}' >"$agent_smoke_payload" + smoke_options=(--output raw) + smoke_validation=--validate-responses-smoke + else + printf '%s\n' '{"messages":[{"role":"user","content":"Hello!"}]}' >"$agent_smoke_payload" + fi + if ! agent_smoke_output="$(cd "$app_definition_dir" && azd ai agent invoke "$c_name" --protocol "$smoke_protocol" --new-session --timeout 180 --environment "$env_name" --no-prompt --input-file "$agent_smoke_payload" "${smoke_options[@]}" 2>&1)"; then rm -f "$agent_smoke_payload" red "$c_name: smoke request failed."; had_errors=1; continue fi rm -f "$agent_smoke_payload" - if ! printf '%s\n' "$agent_smoke_output" | (cd "$repo_root" && python3 -m config.deployment.hosted --validate-smoke); then + if ! printf '%s\n' "$agent_smoke_output" | (cd "$repo_root" && python3 -m config.deployment.hosted "$smoke_validation"); then red "$c_name: smoke test did not pass."; had_errors=1; continue fi else - (cd "$c_path" && azd deploy --all --environment "$env_name" --no-prompt) || { + (cd "$app_definition_dir" && azd deploy "$c_name" --environment "$env_name" --no-prompt) || { red "$c_name: Container App deployment failed."; had_errors=1; continue } fi diff --git a/scripts/preProvision.ps1 b/scripts/preProvision.ps1 index cbb3e0bd..d3d5ea4f 100755 --- a/scripts/preProvision.ps1 +++ b/scripts/preProvision.ps1 @@ -53,6 +53,7 @@ $manifestSource = Join-Path $projectRoot "manifest.json" #------------------------------------------------------------------------------- $appDefinitionPath = if ($env:AGENTLZ_APP_DEFINITION) { $env:AGENTLZ_APP_DEFINITION } else { Join-Path $projectRoot 'app-definition.json' } if (-not [IO.Path]::IsPathRooted($appDefinitionPath)) { $appDefinitionPath = Join-Path $projectRoot $appDefinitionPath } +if (Test-Path -LiteralPath $appDefinitionPath -PathType Container) { $appDefinitionPath = Join-Path $appDefinitionPath 'app-definition.json' } if (-not (Test-Path -LiteralPath $appDefinitionPath -PathType Leaf)) { Write-Host "Error: AGENTLZ_APP_DEFINITION points to $appDefinitionPath, which does not exist." -ForegroundColor Red exit 1 diff --git a/scripts/preProvision.sh b/scripts/preProvision.sh index a7917d7f..84fbd9ab 100755 --- a/scripts/preProvision.sh +++ b/scripts/preProvision.sh @@ -63,6 +63,9 @@ case "$APP_DEFINITION_PATH" in /*) ;; *) APP_DEFINITION_PATH="$PROJECT_ROOT/$APP_DEFINITION_PATH" ;; esac +if [ -d "$APP_DEFINITION_PATH" ]; then + APP_DEFINITION_PATH="$APP_DEFINITION_PATH/app-definition.json" +fi if [ ! -f "$APP_DEFINITION_PATH" ]; then echo "${YELLOW}Error: AGENTLZ_APP_DEFINITION points to $APP_DEFINITION_PATH, which does not exist.${NC}" exit 1 diff --git a/tests/test_custom_application_hooks.py b/tests/test_custom_application_hooks.py new file mode 100644 index 00000000..80c97f22 --- /dev/null +++ b/tests/test_custom_application_hooks.py @@ -0,0 +1,202 @@ +"""Run custom deployment sections with offline CLIs and shipped sample layout.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +from tests.test_hosted_smoke import completion + +ROOT = Path(__file__).resolve().parents[1] + + +class CustomApplicationHookTests(unittest.TestCase): + def check_directory_selection(self, shell: str) -> None: + suffix = "ps1" if shell == "pwsh" else "sh" + for hook in ("preProvision", "preDeploy"): + with self.subTest(hook=hook), tempfile.TemporaryDirectory(prefix="app selection ") as directory: + root = Path(directory) + folder = root / "custom app" + folder.mkdir() + definition = folder / "app-definition.json" + definition.write_text("{}", encoding="utf-8") + source = (ROOT / "scripts" / f"{hook}.{suffix}").read_text(encoding="utf-8-sig") + env = dict(os.environ, AGENTLZ_APP_DEFINITION=str(folder), FAKE_ROOT=str(root)) + if shell == "pwsh": + start = source.index("$appDefinitionPath = if") + marker = 'Write-Host "Validating and binding' if hook == "preProvision" else "$boundAppId =" + end = source.index(marker, start) + prefix = "$projectRoot = $env:FAKE_ROOT; $repoRoot = $projectRoot; $globalEnv = @{}\n" + script = root / "select.ps1" + script.write_text(prefix + source[start:end] + "\nWrite-Output $appDefinitionPath\n", + encoding="utf-8") + command = [shutil.which(shell), "-NoProfile", "-File", str(script)] + else: + variable = "APP_DEFINITION_PATH" if hook == "preProvision" else "app_definition_path" + start = source.index(f'{variable}="${{AGENTLZ_APP_DEFINITION') + marker = 'echo "${CYAN}Validating and binding' if hook == "preProvision" else "command -v python3" + end = source.index(marker, start) + prefix = r""" +if command -v cygpath >/dev/null; then + AGENTLZ_APP_DEFINITION="$(cygpath -u "$AGENTLZ_APP_DEFINITION")" + FAKE_ROOT="$(cygpath -u "$FAKE_ROOT")" +fi +PROJECT_ROOT="$FAKE_ROOT" +repo_root="$FAKE_ROOT" +red() { printf '%s\n' "$*"; } +""" + script = root / "select.sh" + script.write_text(prefix + source[start:end] + f'\nprintf "%s\\n" "${variable}"\n', + encoding="utf-8", newline="\n") + command = [shutil.which(shell), str(script)] + result = subprocess.run(command, cwd=root, env=env, capture_output=True, text=True, timeout=20) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertTrue(result.stdout.strip().endswith("app-definition.json"), result.stdout) + definition.unlink() + result = subprocess.run(command, cwd=root, env=env, capture_output=True, text=True, timeout=20) + self.assertNotEqual(0, result.returncode, result.stdout + result.stderr) + + def run_custom(self, shell: str, *, mismatch: bool = False, hosted: bool = False, + smoke_failure: bool = False) -> tuple[int, list[str], str]: + suffix = "ps1" if shell == "pwsh" else "sh" + source = (ROOT / "scripts" / f"preDeploy.{suffix}").read_text(encoding="utf-8-sig") + with tempfile.TemporaryDirectory(prefix="custom app offline ") as directory: + root = Path(directory) + app = root / "application" + shutil.copytree(ROOT / "samples" / "custom-app" / ("hosted" if hosted else "containerapp"), app) + azure = root / ".azure" + azure.mkdir() + (azure / "config.json").write_text("{}", encoding="utf-8") + definition = json.loads((app / "app-definition.json").read_text(encoding="utf-8")) + component = definition["components"][0] + component["source"] = {"commit": "1" * 40} + (app / "app-definition.json").write_text(json.dumps(definition), encoding="utf-8") + response = completion()["response"] + if smoke_failure: + response["status"] = "failed" + (root / "response").write_text(json.dumps(response), encoding="utf-8") + env = dict(os.environ, FAKE_ROOT=str(root), FAKE_APP=str(app), + FAKE_COMMIT=("2" if mismatch else "1") * 40, REAL_PYTHON=sys.executable, + AGENTLZ_REPO_ROOT=str(ROOT), REAL_REPO=str(ROOT), PYTHONPATH=str(ROOT)) + if shell == "pwsh": + start = source.index("foreach ($dc in $definitionComponents)", source.index("# Components")) + end = source.index("\nif ($hadErrors)", start) + prefix = r""" +$ErrorActionPreference = 'Continue' +$appDefinitionDir = $env:FAKE_APP +$dotAzure = Join-Path $env:FAKE_ROOT '.azure' +$definitionComponents = @((Get-Content (Join-Path $appDefinitionDir 'app-definition.json') -Raw | ConvertFrom-Json).components) +$globalEnv = [pscustomobject]@{ AZURE_ENV_NAME = 'offline' } +$hadErrors = $false +function Get-ManifestComponentForDefinition { return $null } +function git { $global:LASTEXITCODE = 0; return $env:FAKE_COMMIT } +function python { $input | & $env:REAL_PYTHON @args } +function azd { + if (-not (Test-Path 'azure.yaml') -or -not (Test-Path 'src')) { throw 'Wrong project folder' } + if (-not (Test-Path '.azure/config.json')) { throw 'Missing environment' } + Add-Content (Join-Path $env:FAKE_ROOT 'trace') ($args -join ' ') + if ($args[0] -eq 'ai') { Get-Content (Join-Path $env:FAKE_ROOT 'response') -Raw } + $global:LASTEXITCODE = 0 +} +""" + script = root / "run.ps1" + script.write_text(prefix + source[start:end] + "\nif ($hadErrors) { exit 1 }\n", encoding="utf-8") + command = [shutil.which(shell), "-NoProfile", "-NonInteractive", "-File", str(script)] + else: + fake_bin = root / "bin" + fake_bin.mkdir() + (root / "jq.py").write_text( + "import json, sys\n" + "query = sys.argv[2]\n" + "doc = json.load(open(sys.argv[3])) if len(sys.argv) > 3 else json.load(sys.stdin)\n" + "if query == '.components[]':\n" + " for item in doc['components']: print(json.dumps(item))\n" + "else:\n" + " value = doc\n" + " for key in query.split(' // ')[0].strip('.').split('.'): value = value.get(key, {})\n" + " print(value if value != {} else '')\n", encoding="utf-8") + shim = fake_bin / "jq" + shim.write_text('#!/usr/bin/env bash\nexec "$REAL_PYTHON" "$FAKE_ROOT/jq.py" "$@"\n', + encoding="utf-8", newline="\n") + shim.chmod(0o755) + python_shim = fake_bin / "python3" + python_shim.write_text('#!/usr/bin/env bash\nexec "$REAL_PYTHON" "$@"\n', + encoding="utf-8", newline="\n") + python_shim.chmod(0o755) + start = source.index('while IFS= read -r dcomp; do', source.index("# Components")) + end = source.index('\nif [ "$had_errors"', start) + prefix = r""" +set -eu +if command -v cygpath >/dev/null; then + FAKE_ROOT="$(cygpath -u "$FAKE_ROOT")" + FAKE_APP="$(cygpath -u "$FAKE_APP")" + REAL_REPO="$(cygpath -u "$REAL_REPO")" +fi +export PATH="$FAKE_ROOT/bin:$PATH" FAKE_ROOT +app_definition_dir="$FAKE_APP" +app_definition_path="$FAKE_APP/app-definition.json" +dot_azure="$FAKE_ROOT/.azure" +AZURE_ENV_NAME=offline +AZURE_AI_PROJECT_ENDPOINT=https://example.invalid/api/projects/offline +AZURE_AI_PROJECT_RESOURCE_ID=offline +repo_root="$REAL_REPO" +had_errors=0 +red() { printf '%s\n' "$*"; } +green() { :; } +cyan() { :; } +manifest_component_for() { :; } +git() { printf '%s\n' "$FAKE_COMMIT"; } +copy_dot_azure() { cp -R "$1" "$2/"; } +azd() { + [ "$PWD" = "$FAKE_APP" ] || return 91 + [ -f .azure/config.json ] || return 92 + printf '%s\n' "$*" >> "$FAKE_ROOT/trace" + if [ "$1" = ai ]; then cat "$FAKE_ROOT/response"; fi +} +""" + script = root / "run.sh" + script.write_text(prefix + source[start:end] + '\nexit "$had_errors"\n', encoding="utf-8", newline="\n") + command = [shutil.which(shell), str(script)] + result = subprocess.run(command, cwd=root, env=env, capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=30) + trace = root / "trace" + events = trace.read_text(encoding="utf-8-sig").splitlines() if trace.exists() else [] + return result.returncode, events, result.stdout + result.stderr + + @unittest.skipUnless(shutil.which("pwsh"), "PowerShell is not installed") + def test_powershell_deploys_selected_service_from_definition_folder(self) -> None: + self.check_directory_selection("pwsh") + self.check_deploy("pwsh") + + @unittest.skipUnless(shutil.which("bash"), "Bash is not installed") + def test_shell_deploys_selected_service_from_definition_folder(self) -> None: + self.check_directory_selection("bash") + self.check_deploy("bash") + + def check_deploy(self, shell: str) -> None: + code, events, output = self.run_custom(shell) + self.assertEqual(0, code, output) + self.assertEqual(["deploy web --environment offline --no-prompt"], events) + code, events, output = self.run_custom(shell, mismatch=True) + self.assertNotEqual(0, code, output) + self.assertEqual([], events) + for failed in (False, True): + code, events, output = self.run_custom(shell, hosted=True, smoke_failure=failed) + if failed: + self.assertNotEqual(0, code, output) + else: + self.assertEqual(0, code, output) + self.assertIn("deploy agent --environment offline --no-prompt", events) + smoke = next(event for event in events if event.startswith("ai agent invoke ")) + self.assertIn("ai agent invoke agent --protocol responses", smoke) + self.assertIn("--output raw", smoke) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_hosted_smoke.py b/tests/test_hosted_smoke.py index e73ac17f..4805814d 100644 --- a/tests/test_hosted_smoke.py +++ b/tests/test_hosted_smoke.py @@ -3,6 +3,8 @@ from contextlib import redirect_stdout import io import json +from pathlib import Path +import tempfile import unittest from unittest.mock import patch @@ -34,6 +36,29 @@ def sse(*events): class HostedSmokeTests(unittest.TestCase): + def test_custom_responses_json_and_raw_http(self): + response = json.dumps(completion()["response"]) + hosted.validate_responses_smoke_output(response) + hosted.validate_responses_smoke_output("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n" + response) + hosted.validate_responses_smoke_output(sse(completion())) + for body in ("HTTP/1.1 403 Forbidden\n\n" + response, + json.dumps(completion("")), '{"object":"response","status":"failed"}', + '{"object":"response","status":"completed","error":{"message":"PRIVATE-MARKER"}}'): + with self.subTest(body=body), self.assertRaises(ValueError) as caught: + hosted.validate_responses_smoke_output(body) + self.assertNotIn("PRIVATE-MARKER", str(caught.exception)) + + def test_custom_protocol_is_read_from_selected_service(self): + root = Path(__file__).resolve().parents[1] + self.assertEqual("responses", hosted.service_protocol(root / "samples/custom-app/hosted/azure.yaml", "agent")) + with tempfile.TemporaryDirectory() as directory: + project = Path(directory) / "azure.yaml" + project.write_text("services:\n agent:\n protocols:\n - protocol: invocations\n", encoding="utf-8") + self.assertEqual("invocations", hosted.service_protocol(project, "agent")) + project.write_text("services:\n agent:\n protocols:\n - protocol: a2a\n", encoding="utf-8") + with self.assertRaises(ValueError): + hosted.service_protocol(project, "agent") + def test_successful_completed_greeting_sse_not_exact_marker(self): hosted.validate_smoke_output("Invoking agent...\n" + sse( {"type": "response.output_text.delta", "delta": "Hello!"}, From 3997408b1faaa8f7754aa15433cbff55b66ca545 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 08:55:26 -0300 Subject: [PATCH 04/18] docs: record remaining Azure acceptance blockers Correct sample paths and binding negative checks; distinguish runtime failures from offline regression evidence. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../quickstart.md | 43 ++++++++++++++++--- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/specs/002-agent-landing-zone-rebrand/quickstart.md b/specs/002-agent-landing-zone-rebrand/quickstart.md index 5c6eacbf..3d1605ea 100644 --- a/specs/002-agent-landing-zone-rebrand/quickstart.md +++ b/specs/002-agent-landing-zone-rebrand/quickstart.md @@ -143,11 +143,13 @@ Evidence after the repository rename (`curl -sI`): ## S5 — Validate an application definition (US5) ```powershell -python -m config.appdefinition --validate samples/custom-app/app-definition.json +python -m config.appdefinition --validate samples/custom-app/containerapp +python -m config.appdefinition --validate samples/custom-app/hosted ``` ```sh -python -m config.appdefinition --validate samples/custom-app/app-definition.json +python -m config.appdefinition --validate samples/custom-app/containerapp +python -m config.appdefinition --validate samples/custom-app/hosted ``` Expected outcome: @@ -176,18 +178,20 @@ in `v4.0.0` they are gone, so the same command proves the full rename. ```powershell azd env new -azd env set AGENTLZ_APP_DEFINITION samples/custom-app +azd env set AGENTLZ_APP_DEFINITION samples/custom-app/containerapp azd up ``` ```sh azd env new -azd env set AGENTLZ_APP_DEFINITION samples/custom-app +azd env set AGENTLZ_APP_DEFINITION samples/custom-app/containerapp azd up ``` -Run this once with the sample's Container Apps variant and once with its -hosted-agent variant. +Replace the all-zero component source pin with the actual checkout HEAD before +deployment. Run this once with `samples/custom-app/containerapp` and once in a +separate environment with `samples/custom-app/hosted`. The definition folder +contains `azure.yaml`; `components[].path` selects its service source folder. Expected outcome: @@ -201,12 +205,37 @@ Negative checks. Both must fail **before any Azure change**: - Point `AGENTLZ_APP_DEFINITION` at an invalid definition and run `azd up`. The output includes the JSON pointer to the error. -- On the bound environment, change the definition (or point it at the trio) +- On the bound environment, select a definition with a different application + `id` (for example, point it at the trio) and run `azd up`. The output reports a binding mismatch and says to create a new azd environment (FR-015g). Re-running `azd up` with the same definition is idempotent (FR-016). +## Acceptance status (2026-10-09) + +UI `v3.2.1` and umbrella `v4.2.2` were published. Their publication does not +close the outstanding runtime acceptance scenarios: + +- Classic foundation resources were provisioned, but post-provision governance + failed with Key Vault `ForbiddenByConnection`: public access was disabled + and the validation vault had no private endpoint. Application deployment + and authenticated grounded answers remain unverified. +- The first hosted-panel provisioning attempt failed during ARM validation + with a connection reset. A retry and a separate network-isolated validation + deployment are in progress. +- The evaluation VPN Gateway was provisioned and protected with `keep=true` + and a deletion lock. Peering, private DNS, and end-to-end connectivity are + not yet acceptance evidence. +- Validation found directory-selection, custom-project working-directory, + and hosted greeting-protocol defects. The corrective change passes + 386 tests and 462 subtests (4 skipped), but those offline results do not + establish a successful S7 Azure deployment or retroactively validate + the published release. + +T035, T085, T098, and T093 remain open until their actual runtime outcomes +are captured. No network policy was disabled to obtain a passing result. + ## Fallback check (R16) If the hosted-agent automation misses the 2026-10-09 deadline, S7's From aeec7ea92e806dcc37e4273c4bdede317a9e9460 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 09:50:46 -0300 Subject: [PATCH 05/18] fix: respect Foundry local auth and resolve panel database settings Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- config/aifoundry/setup.py | 22 +++++-- config/panel/setup.py | 23 ++++++- .../quickstart.md | 28 +++++--- tests/test_aifoundry_setup.py | 64 ++++++++++++++++++ tests/test_panel_setup.py | 66 +++++++++++++++++++ 5 files changed, 189 insertions(+), 14 deletions(-) create mode 100644 tests/test_aifoundry_setup.py create mode 100644 tests/test_panel_setup.py diff --git a/config/aifoundry/setup.py b/config/aifoundry/setup.py index 4924b80a..7dcabe3f 100644 --- a/config/aifoundry/setup.py +++ b/config/aifoundry/setup.py @@ -142,18 +142,30 @@ def add_ai_foundry_account_api_key_to_key_vault( account_name: str, vault_uri: str, secret_name: str -) -> None: - """Fetches the AI Foundry Account API key and stores it in Azure Key Vault.""" +) -> bool: + """Store an evaluation API key only when the account allows local authentication.""" try: + account = mgmt_client.accounts.get(resource_group, account_name) + if account.properties.disable_local_auth is True: + logging.info( + "Local authentication is disabled for account %s; " + "evaluation API-key injection is not applicable. Use Microsoft Entra ID.", + account_name, + ) + return False logging.info("🔑 Fetching AI Foundry Account API key for account %s ...", account_name) keys = mgmt_client.accounts.list_keys(resource_group, account_name) api_key = keys.key1 + if not api_key: + raise ValueError("The Foundry account returned an empty evaluation API key.") logging.info("🔒 Storing API key in Key Vault at %s ...", vault_uri) kv_client = KeyVaultClient(vault_uri) kv_client.set_secret(secret_name, api_key) logging.info("✅ Secret %s set successfully in Key Vault.", secret_name) + return True except Exception as e: logging.error("❗️ Failed to set secret in Key Vault: %s", e) + sys.exit(1) # ── Blocklist Logic ────────────────────────────────── def configure_blocklist(client: CognitiveServicesManagementClient, resource_group: str, account_name: str, bl_def: Dict[str, Any], bl_name: str) -> None: @@ -316,14 +328,16 @@ def main() -> None: # Store API key in Key Vault logging.info("🔑 Adding AI Foundry Account API Key to Key Vault …") key_vault_uri = cfg(app_conf, "KEY_VAULT_URI") - add_ai_foundry_account_api_key_to_key_vault( + secret_stored = add_ai_foundry_account_api_key_to_key_vault( client, resource_group, account_name, key_vault_uri, SECRET_NAME ) - logging.info("✅ RAI blocklist, policy, deployment association, and secret injection complete.") + logging.info("✅ RAI blocklist, policy, and deployment association complete.") + if secret_stored: + logging.info("✅ Evaluation API-key injection complete.") if __name__ == "__main__": main() diff --git a/config/panel/setup.py b/config/panel/setup.py index 7a556597..f7f8c81a 100644 --- a/config/panel/setup.py +++ b/config/panel/setup.py @@ -343,7 +343,6 @@ def configure_panel_rbac(environment: Mapping[str, str]) -> int: resource_token = (environment.get("RESOURCE_TOKEN") or "").strip() for value, name in ( (resource_group, "AZURE_RESOURCE_GROUP"), - (account_name, "DATABASE_ACCOUNT_NAME"), (resource_token, "RESOURCE_TOKEN"), ): if not value: @@ -351,13 +350,33 @@ def configure_panel_rbac(environment: Mapping[str, str]) -> int: f"DEPLOY_ADMINISTRATIVE_PANEL is true but {name} is not set; " "cannot assign panel Cosmos RBAC." ) + database_name = (environment.get("DATABASE_NAME") or "").strip() + if not account_name and (environment.get("APP_CONFIG_ENDPOINT") or "").strip(): + endpoint = environment["APP_CONFIG_ENDPOINT"].strip() + for key in ("DATABASE_ACCOUNT_NAME", "DATABASE_NAME"): + value = _run_az([ + "appconfig", "kv", "show", "--endpoint", endpoint, + "--key", key, "--label", LABEL, "--auth-mode", "login", + "--query", "value", "--output", "tsv", + ]).strip() + if not value: + raise PanelRbacError(f"App Configuration {key} is empty; cannot assign panel Cosmos RBAC.") + if key == "DATABASE_ACCOUNT_NAME": + account_name = value + elif not database_name: + database_name = value + if not account_name: + raise PanelRbacError( + "DEPLOY_ADMINISTRATIVE_PANEL is true but DATABASE_ACCOUNT_NAME is not set; " + "cannot assign panel Cosmos RBAC." + ) subscription_id = resolve_subscription_id(environment) if not subscription_id: raise PanelRbacError( "DEPLOY_ADMINISTRATIVE_PANEL is true but AZURE_SUBSCRIPTION_ID " "could not be determined; cannot assign panel Cosmos RBAC." ) - database_name = (environment.get("DATABASE_NAME") or "").strip() or account_name + database_name = database_name or account_name app_names = { FRONTEND_APP_SERVICE_NAME: f"ca-{resource_token}-frontend", diff --git a/specs/002-agent-landing-zone-rebrand/quickstart.md b/specs/002-agent-landing-zone-rebrand/quickstart.md index 3d1605ea..ad60fb84 100644 --- a/specs/002-agent-landing-zone-rebrand/quickstart.md +++ b/specs/002-agent-landing-zone-rebrand/quickstart.md @@ -219,19 +219,31 @@ close the outstanding runtime acceptance scenarios: - Classic foundation resources were provisioned, but post-provision governance failed with Key Vault `ForbiddenByConnection`: public access was disabled - and the validation vault had no private endpoint. Application deployment - and authenticated grounded answers remain unverified. -- The first hosted-panel provisioning attempt failed during ARM validation - with a connection reset. A retry and a separate network-isolated validation - deployment are in progress. + and the validation vault had no private endpoint. Governance subsequently + passed from the jumpbox over approved recovery private endpoints, using + the exact release source and managed identity. Continuity, Container Apps + and Search setup also passed. Application deployment is in progress; + authenticated grounded answers remain unverified. +- Hosted-panel foundation, private governance recovery, continuity, Container + Apps and Search setup passed. The pinned hosted image was built and resolved + to an immutable digest. Re-provisioning its deploy handoff is blocked by a + quota preflight that treats an existing model allocation as a new request. + The separate network-isolated foundation deployment remains in progress. - The evaluation VPN Gateway was provisioned and protected with `keep=true` - and a deletion lock. Peering, private DNS, and end-to-end connectivity are - not yet acceptance evidence. + and a deletion lock. Hub/spoke peerings are Connected with gateway transit. + Private DNS and managed-identity governance calls succeeded from the actual + jumpbox. No client VPN authentication or hosted private runtime is approved. - Validation found directory-selection, custom-project working-directory, and hosted greeting-protocol defects. The corrective change passes - 386 tests and 462 subtests (4 skipped), but those offline results do not + 584 tests and 558 subtests (4 skipped), but those offline results do not establish a successful S7 Azure deployment or retroactively validate the published release. +- Validation also found false-success logging when an Entra-only Foundry + account rejects evaluation API keys, and a missing Cosmos database-name + handoff for the administrative panel. The corrective branch respects disabled + local authentication and reads published database settings before assigning + the unchanged container-scoped roles. Its live panel recovery created four + narrow grants; these fixes are not in the published release. T035, T085, T098, and T093 remain open until their actual runtime outcomes are captured. No network policy was disabled to obtain a passing result. diff --git a/tests/test_aifoundry_setup.py b/tests/test_aifoundry_setup.py new file mode 100644 index 00000000..df2454b0 --- /dev/null +++ b/tests/test_aifoundry_setup.py @@ -0,0 +1,64 @@ +"""Foundry evaluation keys respect account authentication and fail explicitly.""" + +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + +from config.aifoundry import setup + + +class FoundryEvaluationKeyTests(unittest.TestCase): + def setUp(self): + self.client = Mock() + self.client.accounts.get.return_value = SimpleNamespace( + properties=SimpleNamespace(disable_local_auth=False) + ) + self.client.accounts.list_keys.return_value = SimpleNamespace(key1="test-key") + self.vault_patch = patch.object(setup, "KeyVaultClient") + self.vault = self.vault_patch.start() + self.addCleanup(self.vault_patch.stop) + + def inject(self): + return setup.add_ai_foundry_account_api_key_to_key_vault( + self.client, "test-group", "test-foundry", + "https://test-vault.vault.azure.net/", "evaluationsModelApiKey", + ) + + def test_disabled_local_auth_never_reads_or_stores_a_key(self): + self.client.accounts.get.return_value.properties.disable_local_auth = True + with self.assertLogs(level="INFO") as logs: + self.assertFalse(self.inject()) + self.assertIn("Use Microsoft Entra ID", "\n".join(logs.output)) + self.client.accounts.list_keys.assert_not_called() + self.vault.assert_not_called() + + def test_enabled_local_auth_preserves_key_injection(self): + self.assertTrue(self.inject()) + self.client.accounts.get.assert_called_once_with("test-group", "test-foundry") + self.client.accounts.list_keys.assert_called_once_with("test-group", "test-foundry") + self.vault.return_value.set_secret.assert_called_once_with( + "evaluationsModelApiKey", "test-key" + ) + + def test_read_and_write_failures_are_not_reported_as_success(self): + for boundary in ( + self.client.accounts.get, + self.client.accounts.list_keys, + self.vault.return_value.set_secret, + ): + with self.subTest(boundary=boundary): + boundary.side_effect = RuntimeError("boundary unavailable") + with self.assertLogs(level="ERROR"), self.assertRaises(SystemExit) as error: + self.inject() + self.assertEqual(error.exception.code, 1) + boundary.side_effect = None + + def test_empty_key_is_rejected_before_vault_write(self): + self.client.accounts.list_keys.return_value.key1 = "" + with self.assertLogs(level="ERROR"), self.assertRaises(SystemExit): + self.inject() + self.vault.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_panel_setup.py b/tests/test_panel_setup.py new file mode 100644 index 00000000..d27356f6 --- /dev/null +++ b/tests/test_panel_setup.py @@ -0,0 +1,66 @@ +"""Panel RBAC resolves published database names without widening its grants.""" + +import unittest +from unittest.mock import patch + +from config.panel import setup + + +class PanelPublishedDatabaseTests(unittest.TestCase): + def setUp(self): + self.environment = { + "DEPLOY_ADMINISTRATIVE_PANEL": "true", + "AZURE_RESOURCE_GROUP": "test-group", + "RESOURCE_TOKEN": "testtoken", + "AZURE_SUBSCRIPTION_ID": "test-subscription", + "APP_CONFIG_ENDPOINT": "https://test-store.azconfig.io", + } + + def test_published_account_and_database_are_used_for_container_scopes(self): + with ( + patch.object(setup, "_run_az", side_effect=["test-account", "test-database"]) as az, + patch.object(setup, "resolve_container_app_principal_id", side_effect=["ui", "ingestion"]), + patch.object(setup, "ensure_cosmos_sql_role_assignment", return_value=True) as assign, + ): + self.assertEqual(setup.configure_panel_rbac(self.environment), 4) + self.assertEqual([call.args[0][call.args[0].index("--key") + 1] for call in az.call_args_list], + ["DATABASE_ACCOUNT_NAME", "DATABASE_NAME"]) + for call in az.call_args_list: + self.assertIn(setup.LABEL, call.args[0]) + self.assertIn("login", call.args[0]) + for call in assign.call_args_list: + self.assertEqual(call.args[0], "test-account") + self.assertIn("/dbs/test-database/colls/", call.args[-1]) + self.assertIn(call.args[3], ("ui", "ingestion")) + + def test_explicit_names_preserve_existing_behavior_without_config_read(self): + self.environment.update(DATABASE_ACCOUNT_NAME="explicit-account", DATABASE_NAME="explicit-db") + with ( + patch.object(setup, "_run_az") as az, + patch.object(setup, "resolve_container_app_principal_id", return_value="test-identity"), + patch.object(setup, "ensure_cosmos_sql_role_assignment", return_value=False), + ): + self.assertEqual(setup.configure_panel_rbac(self.environment), 0) + az.assert_not_called() + + def test_empty_published_value_fails_before_any_role_change(self): + with ( + patch.object(setup, "_run_az", return_value=""), + patch.object(setup, "ensure_cosmos_sql_role_assignment") as assign, + self.assertRaises(setup.PanelRbacError), + ): + setup.configure_panel_rbac(self.environment) + assign.assert_not_called() + + def test_config_failure_propagates_without_role_changes(self): + with ( + patch.object(setup, "_run_az", side_effect=setup.PanelRbacError("read failed")), + patch.object(setup, "ensure_cosmos_sql_role_assignment") as assign, + self.assertRaises(setup.PanelRbacError), + ): + setup.configure_panel_rbac(self.environment) + assign.assert_not_called() + + +if __name__ == "__main__": + unittest.main() From 885489552014b883b52e42103efa9a0d26c0fa5a Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 11:02:26 -0300 Subject: [PATCH 06/18] fix: validate incremental model quota for existing deployments Share conservative existing-allocation verification across both preflight gates, bind discovery to the selected environment, and retain availability and new-capacity checks. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- infra/scripts/Invoke-PreflightChecks.ps1 | 48 ++- infra/scripts/ModelQuota.ps1 | 95 ++++++ .../scripts/Invoke-PreflightChecks.Tests.ps1 | 1 + scripts/Invoke-RegionalPreflight.ps1 | 33 +- .../quickstart.md | 21 +- tests/test_regional_model_quota.py | 295 ++++++++++++++++++ 6 files changed, 463 insertions(+), 30 deletions(-) create mode 100644 infra/scripts/ModelQuota.ps1 create mode 100644 tests/test_regional_model_quota.py diff --git a/infra/scripts/Invoke-PreflightChecks.ps1 b/infra/scripts/Invoke-PreflightChecks.ps1 index 23d381f6..821ae010 100644 --- a/infra/scripts/Invoke-PreflightChecks.ps1 +++ b/infra/scripts/Invoke-PreflightChecks.ps1 @@ -79,6 +79,7 @@ param( ) $ErrorActionPreference = 'Stop' +. (Join-Path $PSScriptRoot 'ModelQuota.ps1') # -------------------------------------------------------------------------- # Emergency bypass @@ -1379,7 +1380,10 @@ function Test-ModelQuota { $deployments = @($ModelDeployments | Where-Object { $_ -ne $null }) if ($deployments.Count -eq 0) { return } - $usage = Invoke-AzCliRaw -Arguments @('cognitiveservices', 'usage', 'list', '--location', $Location, '-o', 'json') + $selectedSubscription = if ($SubscriptionId) { $SubscriptionId } else { $env:AZURE_SUBSCRIPTION_ID } + $usageArguments = @('cognitiveservices', 'usage', 'list', '--location', $Location, '-o', 'json') + if ($selectedSubscription) { $usageArguments += @('--subscription', $selectedSubscription) } + $usage = Invoke-AzCliRaw -Arguments $usageArguments if (-not $usage) { Add-Finding -Severity WARN -Code 'MODEL_QUOTA_LOOKUP' ` -Message "Could not read Cognitive Services usage/quota for '$Location'." ` @@ -1387,28 +1391,38 @@ function Test-ModelQuota { return } + $deployments = @($deployments | Where-Object { $_.model.format -eq 'OpenAI' }) + if ($deployments.Count -eq 0) { return } + $selectedEnvironment = Get-AzdEnvValues + $existing = @(Get-ExistingModelDeployments -Location $Location -SubscriptionId $selectedSubscription ` + -ProjectResourceId $selectedEnvironment['AZURE_AI_PROJECT_RESOURCE_ID'] ` + -ResourceGroup $selectedEnvironment['AZURE_RESOURCE_GROUP'] -ReadAzureJson { + param([string[]]$Arguments) + Invoke-AzCliRaw -Arguments ($Arguments + @('-o', 'json')) + } -OnWarning { + param([string]$Message) + Add-Finding -Severity WARN -Code 'MODEL_QUOTA_EXISTING_TARGET' -Message $Message + }) + try { + $requests = @(Get-ModelQuotaRequirements -Models $deployments -ExistingDeployments $existing) + } catch { + Add-Finding -Severity FAIL -Code 'MODEL_QUOTA_INVALID' -Message $_.Exception.Message + return + } $failures = @() - foreach ($d in $deployments) { - # Only OpenAI-format deployments report quota via usage list - $fmt = $null - if ($d.PSObject.Properties.Name -contains 'model' -and $d.model) { - $fmt = $d.model.format - } - if ($fmt -ne 'OpenAI') { continue } - - $modelName = [string]$d.model.name - $skuName = [string]$d.sku.name - $capacity = [double]$d.sku.capacity - $quotaName = "OpenAI.$skuName.$modelName" - + foreach ($request in $requests) { + $quotaName = $request.QuotaName $quota = @($usage | Where-Object { $_.name.value -eq $quotaName } | Select-Object -First 1) if (-not $quota) { $failures += "No quota entry '$quotaName' in $Location." continue } - $available = [double]$quota.limit - [double]$quota.currentValue - if ($available -lt $capacity) { - $failures += "$quotaName needs $capacity, $available available (used $($quota.currentValue) / limit $($quota.limit))." + $available = [double]$quota[0].limit - [double]$quota[0].currentValue + $message = "$quotaName desired $($request.DesiredCapacity), verified existing credit $($request.ExistingCredit), additional allocation $($request.AdditionalCapacity); $available available." + if ($request.AdditionalCapacity -gt 0 -and $available -lt $request.AdditionalCapacity) { + $failures += $message + } else { + Add-Finding -Severity PASS -Code 'MODEL_QUOTA_SUFFICIENT' -Message $message } } diff --git a/infra/scripts/ModelQuota.ps1 b/infra/scripts/ModelQuota.ps1 new file mode 100644 index 00000000..6b8756fa --- /dev/null +++ b/infra/scripts/ModelQuota.ps1 @@ -0,0 +1,95 @@ +function Get-ExistingModelDeployments { + param( + [Parameter(Mandatory = $true)][string]$Location, + [AllowEmptyString()][string]$SubscriptionId, + [AllowEmptyString()][string]$ProjectResourceId = $env:AZURE_AI_PROJECT_RESOURCE_ID, + [AllowEmptyString()][string]$ResourceGroup = $env:AZURE_RESOURCE_GROUP, + [Parameter(Mandatory = $true)][scriptblock]$ReadAzureJson, + [Parameter(Mandatory = $true)][scriptblock]$OnWarning + ) + + $projectId = $ProjectResourceId + if ([string]::IsNullOrWhiteSpace($projectId)) { return @() } + $target = [regex]::Match($projectId, '^/subscriptions/([^/]+)/resourceGroups/([^/]+)/providers/Microsoft\.CognitiveServices/accounts/([^/]+)/projects/[^/]+/?$', 'IgnoreCase') + if (-not $target.Success -or $target.Groups[1].Value -ne $SubscriptionId -or + [string]::IsNullOrWhiteSpace($resourceGroup) -or $target.Groups[2].Value -ne $resourceGroup) { + & $OnWarning 'Existing project output does not match the selected subscription/resource group; no allocation credit applied.' | Out-Null + return @() + } + + $accountName = $target.Groups[3].Value + $expectedId = $projectId -replace '/projects/[^/]+/?$', '' + $account = & $ReadAzureJson @('resource', 'show', '--ids', $expectedId, '--subscription', $SubscriptionId, '--query', '{id:id,location:location}') + if ($null -eq $account -or ($account -is [hashtable] -and $account.failed)) { + & $OnWarning 'Could not verify the existing account; full requested allocation will be checked.' | Out-Null + return @() + } + $actualLocation = ([string]$account.location -replace '\s+', '').ToLowerInvariant() + $desiredLocation = ($Location -replace '\s+', '').ToLowerInvariant() + if ($account.id -ne $expectedId -or $actualLocation -ne $desiredLocation) { + & $OnWarning 'Account identity/location does not match the target; no allocation credit applied.' | Out-Null + return @() + } + $deployments = & $ReadAzureJson @('cognitiveservices', 'account', 'deployment', 'list', '--name', $accountName, '--resource-group', $resourceGroup, '--subscription', $SubscriptionId) + if ($deployments -is [hashtable] -and $deployments.failed) { + & $OnWarning 'Could not verify existing deployments; full requested allocation will be checked.' | Out-Null + return @() + } + return @($deployments) +} + +function Get-ModelQuotaRequirements { + param( + [Parameter(Mandatory = $true)]$Models, + [AllowEmptyCollection()][object[]]$ExistingDeployments = @() + ) + + $requests = @{} + $seenNames = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + foreach ($deployment in @($Models)) { + $model = $deployment.model + $capacity = [double]$deployment.sku.capacity + if ($capacity -le 0 -or [double]::IsNaN($capacity) -or [double]::IsInfinity($capacity)) { + throw 'Deployment capacity must be positive and finite.' + } + $nameProperty = $deployment.PSObject.Properties['name'] + $deploymentName = if ($null -ne $nameProperty) { [string]$nameProperty.Value } else { '' } + if (-not [string]::IsNullOrWhiteSpace($deploymentName) -and -not $seenNames.Add($deploymentName)) { + throw "Duplicate deployment name '$deploymentName'; allocation credit cannot be reused." + } + $quotaName = "OpenAI.$($deployment.sku.name).$($model.name)" + $credit = 0.0 + if (-not [string]::IsNullOrWhiteSpace($deploymentName)) { + $matching = @($ExistingDeployments | Where-Object { $null -ne $_ -and $_.name -eq $deploymentName }) + if ($matching.Count -eq 1) { + $current = $matching[0] + $desiredFormat = $model.PSObject.Properties['format'] + $currentModel = $current.properties.model + $currentFormat = $currentModel.PSObject.Properties['format'] + if ($null -ne $desiredFormat -and $null -ne $currentFormat -and + $desiredFormat.Value -eq $currentFormat.Value -and + $currentModel.name -eq $model.name -and $currentModel.version -eq $model.version -and + $current.sku.name -eq $deployment.sku.name -and + $current.properties.provisioningState -eq 'Succeeded') { + $currentCapacity = [double]$current.sku.capacity + if ($currentCapacity -gt 0 -and -not [double]::IsNaN($currentCapacity) -and -not [double]::IsInfinity($currentCapacity)) { + $credit = [Math]::Min($capacity, $currentCapacity) + } + } + } + } + if (-not $requests.ContainsKey($quotaName)) { + $requests[$quotaName] = @{ desired = 0.0; credit = 0.0 } + } + $requests[$quotaName].desired += $capacity + $requests[$quotaName].credit += $credit + } + foreach ($quotaName in $requests.Keys) { + [pscustomobject]@{ + QuotaName = $quotaName + DesiredCapacity = $requests[$quotaName].desired + ExistingCredit = $requests[$quotaName].credit + AdditionalCapacity = $requests[$quotaName].desired - $requests[$quotaName].credit + } + } +} diff --git a/infra/tests/scripts/Invoke-PreflightChecks.Tests.ps1 b/infra/tests/scripts/Invoke-PreflightChecks.Tests.ps1 index 3611eb94..668c8ba5 100644 --- a/infra/tests/scripts/Invoke-PreflightChecks.Tests.ps1 +++ b/infra/tests/scripts/Invoke-PreflightChecks.Tests.ps1 @@ -25,6 +25,7 @@ $ErrorActionPreference = 'Stop' $scriptPath = Join-Path -Path (Split-Path -Parent $PSScriptRoot) -ChildPath '..\scripts\Invoke-PreflightChecks.ps1' $scriptPath = (Resolve-Path -Path $scriptPath).Path +. (Join-Path (Split-Path -Parent $scriptPath) 'ModelQuota.ps1') $raw = Get-Content -Path $scriptPath -Raw $ast = [System.Management.Automation.Language.Parser]::ParseInput($raw, [ref]$null, [ref]$null) diff --git a/scripts/Invoke-RegionalPreflight.ps1 b/scripts/Invoke-RegionalPreflight.ps1 index 8c118da3..d81c2f18 100644 --- a/scripts/Invoke-RegionalPreflight.ps1 +++ b/scripts/Invoke-RegionalPreflight.ps1 @@ -32,6 +32,7 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +. (Join-Path (Join-Path (Join-Path (Split-Path -Parent $PSScriptRoot) 'infra') 'scripts') 'ModelQuota.ps1') $script:FailureCount = 0 $script:WarningCount = 0 @@ -443,14 +444,19 @@ function Test-ModelReadiness { $usageList = @() } + $existing = @(Get-ExistingModelDeployments -Location $Location -SubscriptionId $SubscriptionId -ReadAzureJson { + param([string[]]$Arguments) + Invoke-AzJson -Arguments $Arguments -AllowFailure + } -OnWarning { + param([string]$Message) + Write-PreflightCheck -Status WARN -Name 'models:existing-target' -Message $Message + }) foreach ($deployment in @($Models)) { $model = $deployment.model $modelName = $model.name $modelVersion = $model.version $skuName = $deployment.sku.name - $capacity = [double]$deployment.sku.capacity $checkName = "model:$modelName" - $available = @($modelList | Where-Object { $_.model.name -eq $modelName -and ([string]::IsNullOrWhiteSpace($modelVersion) -or $_.model.version -eq $modelVersion) -and @@ -463,18 +469,31 @@ function Test-ModelReadiness { Write-PreflightCheck -Status FAIL -Name $checkName -Message "$modelName $modelVersion with sku $skuName is not listed in $Location." } - $usageName = "OpenAI.$skuName.$modelName" + } + + try { + $requests = @(Get-ModelQuotaRequirements -Models $Models -ExistingDeployments $existing) + } catch { + Write-PreflightCheck -Status FAIL -Name 'models:quota' -Message $_.Exception.Message + return + } + foreach ($request in $requests) { + $usageName = $request.QuotaName $quota = @($usageList | Where-Object { $_.name.value -eq $usageName } | Select-Object -First 1) if ($quota.Count -eq 0) { - Write-PreflightCheck -Status WARN -Name "${checkName}:quota" -Message "quota item $usageName was not returned." + Write-PreflightCheck -Status WARN -Name "models:quota:$usageName" -Message "quota item $usageName was not returned." continue } $remaining = [double]$quota[0].limit - [double]$quota[0].currentValue - if ($remaining -ge $capacity) { - Write-PreflightCheck -Status PASS -Name "${checkName}:quota" -Message "$remaining quota units remaining; deployment requests $capacity." + $requested = $request.DesiredCapacity + $credit = $request.ExistingCredit + $additional = $request.AdditionalCapacity + $message = "$remaining quota units remaining; desired $requested, verified existing credit $credit, additional allocation $additional." + if ($additional -eq 0 -or $remaining -ge $additional) { + Write-PreflightCheck -Status PASS -Name "models:quota:$usageName" -Message $message } else { - Write-PreflightCheck -Status FAIL -Name "${checkName}:quota" -Message "$remaining quota units remaining; deployment requests $capacity." + Write-PreflightCheck -Status FAIL -Name "models:quota:$usageName" -Message $message } } } diff --git a/specs/002-agent-landing-zone-rebrand/quickstart.md b/specs/002-agent-landing-zone-rebrand/quickstart.md index ad60fb84..dca2348f 100644 --- a/specs/002-agent-landing-zone-rebrand/quickstart.md +++ b/specs/002-agent-landing-zone-rebrand/quickstart.md @@ -222,20 +222,27 @@ close the outstanding runtime acceptance scenarios: and the validation vault had no private endpoint. Governance subsequently passed from the jumpbox over approved recovery private endpoints, using the exact release source and managed identity. Continuity, Container Apps - and Search setup also passed. Application deployment is in progress; - authenticated grounded answers remain unverified. + and Search setup also passed. Application deployment completed, but the + latest frontend revision is unhealthy with the same vault connection denial. + An HTTP 200 came from the old placeholder, not a healthy application. + Authenticated grounded answers and ingestion startup remain unverified. - Hosted-panel foundation, private governance recovery, continuity, Container Apps and Search setup passed. The pinned hosted image was built and resolved - to an immutable digest. Re-provisioning its deploy handoff is blocked by a - quota preflight that treats an existing model allocation as a new request. - The separate network-isolated foundation deployment remains in progress. + to an immutable digest. The release's quota preflight treats an existing + allocation as a new request. Its corrective branch now passes both real + preflight gates, crediting only verified matching deployments and checking + incremental capacity. That does not prove the hosted deploy handoff. + The separate network-isolated foundation and full post-provision hook + completed from the jumpbox. Owner recovery reconciled the already-declared + native Cosmos grants; the temporary management permission was then revoked. + Private hosted application deployment and authenticated runtime remain pending. - The evaluation VPN Gateway was provisioned and protected with `keep=true` and a deletion lock. Hub/spoke peerings are Connected with gateway transit. Private DNS and managed-identity governance calls succeeded from the actual jumpbox. No client VPN authentication or hosted private runtime is approved. - Validation found directory-selection, custom-project working-directory, and hosted greeting-protocol defects. The corrective change passes - 584 tests and 558 subtests (4 skipped), but those offline results do not + 598 tests and 591 subtests (4 skipped), but those offline results do not establish a successful S7 Azure deployment or retroactively validate the published release. - Validation also found false-success logging when an Entra-only Foundry @@ -244,6 +251,8 @@ close the outstanding runtime acceptance scenarios: local authentication and reads published database settings before assigning the unchanged container-scoped roles. Its live panel recovery created four narrow grants; these fixes are not in the published release. +- The pinned orchestrator still exposes the legacy `GPT-RAG Orchestrator` + OpenAPI title. No branding acceptance or new component release is claimed. T035, T085, T098, and T093 remain open until their actual runtime outcomes are captured. No network policy was disabled to obtain a passing result. diff --git a/tests/test_regional_model_quota.py b/tests/test_regional_model_quota.py new file mode 100644 index 00000000..f0888cb2 --- /dev/null +++ b/tests/test_regional_model_quota.py @@ -0,0 +1,295 @@ +"""Execute the shipped PowerShell model quota checks with a read-only CLI fixture.""" + +from __future__ import annotations + +import copy +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +PWSH = shutil.which("pwsh") +ACCOUNT_ID = ( + "/subscriptions/test-subscription/resourceGroups/test-group/" + "providers/Microsoft.CognitiveServices/accounts/test-account" +) + +DRIVER = r""" +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$SkipAzureCliChecks = $false +$SubscriptionId = 'test-subscription' +$script:FailureCount = 0 +$script:WarningCount = 0 +$script:Calls = @() +$script:Findings = [System.Collections.Generic.List[pscustomobject]]::new() +$fixture = Get-Content $env:QUOTA_FIXTURE -Raw | ConvertFrom-Json +. $env:QUOTA_HELPER +$tokens = $null +$errors = $null +$ast = [System.Management.Automation.Language.Parser]::ParseFile($env:QUOTA_SOURCE, [ref]$tokens, [ref]$errors) +if ($errors.Count -gt 0) { throw 'PowerShell source parse failed.' } +$names = @('Write-PreflightCheck', 'Get-NormalizedLocation', 'Test-ModelReadiness', 'Add-Finding', 'Test-ModelQuota') +foreach ($node in $ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -in $names }, $false)) { + Invoke-Expression $node.Extent.Text +} +function Invoke-AzJson { + param([string[]]$Arguments, [switch]$AllowFailure) + $script:Calls += ($Arguments -join ' ') + if (($Arguments[0..2] -join ' ') -eq 'cognitiveservices model list') { return $fixture.available } + if (($Arguments[0..2] -join ' ') -eq 'cognitiveservices usage list') { return $fixture.usage } + if (($Arguments[0..1] -join ' ') -eq 'resource show') { + if ($fixture.accountFailure) { return @{ failed = $true; error = 'fixture account denied' } } + return $fixture.account + } + if (($Arguments[0..3] -join ' ') -eq 'cognitiveservices account deployment list') { + if ($fixture.deploymentFailure) { return @{ failed = $true; error = 'fixture deployments denied' } } + return $fixture.deployments + } + throw "Unexpected Azure operation: $($Arguments -join ' ')" +} +function Invoke-AzCliRaw { + param([string[]]$Arguments) + $value = Invoke-AzJson -Arguments $Arguments + if ($value -is [hashtable] -and $value.failed) { return $null } + return $value +} +function Get-AzdEnvValues { + if ($fixture.PSObject.Properties['selectedEnvironment']) { + $values = @{} + foreach ($property in $fixture.selectedEnvironment.PSObject.Properties) { + $values[$property.Name] = $property.Value + } + return $values + } + return @{ + AZURE_AI_PROJECT_RESOURCE_ID = $env:AZURE_AI_PROJECT_RESOURCE_ID + AZURE_RESOURCE_GROUP = $env:AZURE_RESOURCE_GROUP + } +} +if ($env:QUOTA_GATE -eq 'foundation') { + Test-ModelQuota -Location 'westus3' -ModelDeployments $fixture.desired + $script:FailureCount = @($script:Findings | Where-Object Severity -eq 'FAIL').Count + $script:WarningCount = @($script:Findings | Where-Object Severity -eq 'WARN').Count + foreach ($finding in $script:Findings) { Write-Host "$($finding.Severity) $($finding.Message)" } +} else { + Test-ModelReadiness -Location 'westus3' -Models $fixture.desired +} +Write-Output ('RESULT:' + (@{ failures = $script:FailureCount; warnings = $script:WarningCount; calls = $script:Calls } | ConvertTo-Json -Compress)) +""" + + +def deployment(name: str = "embedding", capacity: int = 100) -> dict: + return { + "name": name, + "model": {"format": "OpenAI", "name": "text-embedding-3-large", "version": "1"}, + "sku": {"name": "Standard", "capacity": capacity}, + } + + +def fixture() -> dict: + desired = deployment() + return { + "desired": [desired], + "available": [{"model": { + "name": desired["model"]["name"], "version": "1", + "skus": [{"name": "Standard"}, {"name": "GlobalStandard"}], + }}], + "usage": [{"name": {"value": "OpenAI.Standard.text-embedding-3-large"}, + "limit": 150, "currentValue": 100}], + "account": {"id": ACCOUNT_ID, "location": "West US 3"}, + "deployments": [{ + "name": desired["name"], "sku": copy.deepcopy(desired["sku"]), + "properties": {"model": copy.deepcopy(desired["model"]), + "provisioningState": "Succeeded"}, + }], + "accountFailure": False, + "deploymentFailure": False, + } + + +@unittest.skipUnless(PWSH, "PowerShell is required for regional quota behavior tests") +class RegionalModelQuotaTests(unittest.TestCase): + def run_check(self, data: dict, *, project: str = ACCOUNT_ID + "/projects/test-project", + group: str = "test-group", gate: str = "regional") -> tuple[dict, str]: + with tempfile.TemporaryDirectory(prefix="model quota ") as directory: + folder = Path(directory) + (folder / "fixture.json").write_text(json.dumps(data), encoding="utf-8") + (folder / "run.ps1").write_text(DRIVER, encoding="utf-8") + environment = dict( + os.environ, QUOTA_FIXTURE=str(folder / "fixture.json"), + QUOTA_SOURCE=str( + ROOT / "infra" / "scripts" / "Invoke-PreflightChecks.ps1" + if gate == "foundation" else ROOT / "scripts" / "Invoke-RegionalPreflight.ps1" + ), + QUOTA_HELPER=str(ROOT / "infra" / "scripts" / "ModelQuota.ps1"), + QUOTA_GATE=gate, + AZURE_AI_PROJECT_RESOURCE_ID=project, AZURE_RESOURCE_GROUP=group, + ) + result = subprocess.run( + [PWSH, "-NoProfile", "-NonInteractive", "-File", str(folder / "run.ps1")], + env=environment, capture_output=True, text=True, timeout=30, + ) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + row = next(line for line in result.stdout.splitlines() if line.startswith("RESULT:")) + return json.loads(row.removeprefix("RESULT:")), result.stdout + + def test_existing_unchanged_allocation_needs_no_remaining_quota(self) -> None: + data = fixture() + data["usage"][0]["currentValue"] = 155 + result, output = self.run_check(data) + self.assertEqual(0, result["failures"], output) + self.assertIn("verified existing credit 100, additional allocation 0", output) + + def test_fresh_allocation_still_requires_full_capacity(self) -> None: + for available, failures in ((50, 1), (100, 0)): + with self.subTest(available=available): + data = fixture() + data["usage"][0]["limit"] = 100 + available + result, output = self.run_check(data, project="") + self.assertEqual(failures, result["failures"], output) + self.assertEqual(2, len(result["calls"])) + self.assertIn("verified existing credit 0, additional allocation 100", output) + + def test_only_increment_requires_available_quota(self) -> None: + for capacity, failures in ((150, 0), (151, 1), (75, 0)): + with self.subTest(capacity=capacity): + data = fixture() + data["desired"][0]["sku"]["capacity"] = capacity + result, output = self.run_check(data) + self.assertEqual(failures, result["failures"], output) + + def test_existing_allocation_is_not_transferable(self) -> None: + for changed in ("name", "model", "version", "format", "sku", "state", "capacity"): + with self.subTest(changed=changed): + data = fixture() + current = data["deployments"][0] + if changed == "name": + current["name"] = "another-deployment" + elif changed in {"model", "version", "format"}: + key = "name" if changed == "model" else changed + current["properties"]["model"][key] = "another-value" + elif changed == "sku": + current["sku"]["name"] = "GlobalStandard" + elif changed == "state": + current["properties"]["provisioningState"] = "Failed" + else: + current["sku"]["capacity"] = 0 + result, output = self.run_check(data) + self.assertEqual(1, result["failures"], output) + self.assertIn("verified existing credit 0", output) + + def test_existing_target_requires_scope_and_location_proof(self) -> None: + for changed in ("subscription", "group", "project", "account", "location"): + with self.subTest(changed=changed): + data = fixture() + project = ACCOUNT_ID + "/projects/test-project" + if changed == "subscription": + project = project.replace("test-subscription", "other-subscription") + elif changed == "group": + project = project.replace("test-group", "other-group") + elif changed == "project": + project = "malformed-resource-id" + else: + data["account"]["id" if changed == "account" else "location"] = "another-value" + result, output = self.run_check(data, project=project) + self.assertEqual(1, result["failures"], output) + self.assertEqual(1, result["warnings"], output) + self.assertIn("verified existing credit 0", output) + + def test_failed_existing_reads_do_not_grant_credit(self) -> None: + for boundary in ("accountFailure", "deploymentFailure"): + with self.subTest(boundary=boundary): + data = fixture() + data[boundary] = True + result, output = self.run_check(data) + self.assertEqual(1, result["failures"], output) + self.assertEqual(1, result["warnings"], output) + + def test_model_availability_is_not_skipped_for_existing_capacity(self) -> None: + data = fixture() + data["available"] = [] + result, output = self.run_check(data) + self.assertEqual(1, result["failures"], output) + self.assertIn("is not listed in westus3", output) + self.assertIn("additional allocation 0", output) + + def test_multiple_deployments_share_one_quota_budget(self) -> None: + data = fixture() + data["desired"] = [deployment("one", 30), deployment("two", 30)] + result, output = self.run_check(data) + self.assertEqual(1, result["failures"], output) + self.assertIn("additional allocation 60", output) + + def test_existing_credit_is_consumed_only_once(self) -> None: + data = fixture() + data["desired"].append(copy.deepcopy(data["desired"][0])) + result, output = self.run_check(data) + self.assertEqual(1, result["failures"], output) + self.assertIn("duplicate deployment name", output.lower()) + + def test_invalid_capacity_is_rejected(self) -> None: + for capacity in (0, -1): + with self.subTest(capacity=capacity): + data = fixture() + data["desired"][0]["sku"]["capacity"] = capacity + result, output = self.run_check(data) + self.assertEqual(1, result["failures"], output) + + def test_duplicate_existing_name_is_not_credited(self) -> None: + data = fixture() + data["deployments"].append(copy.deepcopy(data["deployments"][0])) + result, output = self.run_check(data) + self.assertEqual(1, result["failures"], output) + self.assertIn("verified existing credit 0", output) + + def test_foundation_gate_enforces_the_same_incremental_budget(self) -> None: + for case, failures in ( + ("unchanged", 0), ("fresh", 1), ("increase", 0), ("excess", 1), + ("aggregate", 1), ("read-failure", 1), ("wrong-region", 1), ("duplicate", 1), + ): + with self.subTest(case=case): + data = fixture() + project = ACCOUNT_ID + "/projects/test-project" + if case == "fresh": + project = "" + elif case in {"increase", "excess"}: + data["desired"][0]["sku"]["capacity"] = 150 if case == "increase" else 151 + elif case == "aggregate": + data["desired"] = [deployment("one", 30), deployment("two", 30)] + elif case == "read-failure": + data["accountFailure"] = True + elif case == "wrong-region": + data["account"]["location"] = "eastus" + elif case == "duplicate": + data["desired"].append(copy.deepcopy(data["desired"][0])) + else: + data["usage"][0]["currentValue"] = 155 + result, output = self.run_check(data, project=project, gate="foundation") + self.assertEqual(failures, result["failures"], output) + + def test_both_platform_hooks_invoke_the_shared_regional_gate(self) -> None: + for suffix in ("ps1", "sh"): + with self.subTest(suffix=suffix): + source = (ROOT / "scripts" / f"preProvision.{suffix}").read_text(encoding="utf-8") + self.assertIn("Invoke-RegionalPreflight.ps1", source) + self.assertIn("Invoke-PreflightChecks.ps1", source) + + def test_foundation_uses_selected_environment_not_ambient_outputs(self) -> None: + for wrong_selected_target in (False, True): + with self.subTest(wrong_selected_target=wrong_selected_target): + data = fixture() + data["selectedEnvironment"] = { + "AZURE_AI_PROJECT_RESOURCE_ID": ACCOUNT_ID + "/projects/test-project", + "AZURE_RESOURCE_GROUP": "wrong-group" if wrong_selected_target else "test-group", + } + result, output = self.run_check( + data, project=ACCOUNT_ID.replace("test-group", "ambient-group") + "/projects/stale", + group="ambient-group", gate="foundation", + ) + self.assertEqual(int(wrong_selected_target), result["failures"], output) From 03393f250868eae550ff0c5c0a98ebc5096b7700 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 11:42:14 -0300 Subject: [PATCH 07/18] docs: record private pinned image build recovery Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- specs/002-agent-landing-zone-rebrand/quickstart.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/specs/002-agent-landing-zone-rebrand/quickstart.md b/specs/002-agent-landing-zone-rebrand/quickstart.md index dca2348f..6680cdaa 100644 --- a/specs/002-agent-landing-zone-rebrand/quickstart.md +++ b/specs/002-agent-landing-zone-rebrand/quickstart.md @@ -235,6 +235,12 @@ close the outstanding runtime acceptance scenarios: The separate network-isolated foundation and full post-provision hook completed from the jumpbox. Owner recovery reconciled the already-declared native Cosmos grants; the temporary management permission was then revoked. + The private pinned image also built successfully after firewall denies + identified the frontend lockfile's Azure Artifacts feeds. The existing + application-specific build allow-list now includes those four feeds and their + verified blob CDN, restricted to build-subnet HTTPS. The immutable image + provenance was synchronized without credentials, and the actual hosted + handoff preview passed. Private hosted application deployment and authenticated runtime remain pending. - The evaluation VPN Gateway was provisioned and protected with `keep=true` and a deletion lock. Hub/spoke peerings are Connected with gateway transit. @@ -245,6 +251,11 @@ close the outstanding runtime acceptance scenarios: 598 tests and 591 subtests (4 skipped), but those offline results do not establish a successful S7 Azure deployment or retroactively validate the published release. +- Read-only checks against the actual private environment binding accepted + the exact-release default definition, rejected a valid different application + ID with exit 2, and rejected an unsupported capability with exit 1. The + environment-file hash was unchanged. These negatives do not establish + a successful custom-application deployment or repeated live provisioning. - Validation also found false-success logging when an Entra-only Foundry account rejects evaluation API keys, and a missing Cosmos database-name handoff for the administrative panel. The corrective branch respects disabled From 4b2a58c661fac513359ca1e2f02b7f8e8e4baa50 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 12:49:01 -0300 Subject: [PATCH 08/18] docs: record private hosted greeting and OAuth readiness blockers Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- specs/002-agent-landing-zone-rebrand/quickstart.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/specs/002-agent-landing-zone-rebrand/quickstart.md b/specs/002-agent-landing-zone-rebrand/quickstart.md index 6680cdaa..02990d02 100644 --- a/specs/002-agent-landing-zone-rebrand/quickstart.md +++ b/specs/002-agent-landing-zone-rebrand/quickstart.md @@ -241,7 +241,19 @@ close the outstanding runtime acceptance scenarios: verified blob CDN, restricted to build-subnet HTTPS. The immutable image provenance was synchronized without credentials, and the actual hosted handoff preview passed. - Private hosted application deployment and authenticated runtime remain pending. + The private Owner ARM handoff and full managed post-provision hook passed. + Project-scoped Foundry User resolved the runner's agent-authoring denial after + propagation. Owner reconciled the three discovered runtime grants without + granting the runner role-management permissions. Normal root deployment then + completed, and a separate managed-identity greeting passed the strict terminal + Responses validator. These are corrective-source results, not pristine release + acceptance or delegated-user retrieval evidence. + The latest UI revision is unhealthy because its default user-delegated mode + lacks OAuth client configuration; ingestion panel startup also reports missing + OAuth configuration. Old placeholder revisions remain active, so private HTTP + 200 responses do not establish readiness. Authentication has not been bypassed. + Healthy application revisions, user-delegated OBO, grounded answers, citations, + and document authorization remain unverified. - The evaluation VPN Gateway was provisioned and protected with `keep=true` and a deletion lock. Hub/spoke peerings are Connected with gateway transit. Private DNS and managed-identity governance calls succeeded from the actual From 87ce0cd9e40636a870971e56042b7d6822d3dfa3 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 13:10:20 -0300 Subject: [PATCH 09/18] docs: record healthy private application and enforced OAuth boundary Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../quickstart.md | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/specs/002-agent-landing-zone-rebrand/quickstart.md b/specs/002-agent-landing-zone-rebrand/quickstart.md index 02990d02..84404851 100644 --- a/specs/002-agent-landing-zone-rebrand/quickstart.md +++ b/specs/002-agent-landing-zone-rebrand/quickstart.md @@ -248,12 +248,20 @@ close the outstanding runtime acceptance scenarios: completed, and a separate managed-identity greeting passed the strict terminal Responses validator. These are corrective-source results, not pristine release acceptance or delegated-user retrieval evidence. - The latest UI revision is unhealthy because its default user-delegated mode - lacks OAuth client configuration; ingestion panel startup also reports missing - OAuth configuration. Old placeholder revisions remain active, so private HTTP - 200 responses do not establish readiness. Authentication has not been bypassed. - Healthy application revisions, user-delegated OBO, grounded answers, citations, - and document authorization remain unverified. + Initial UI and ingestion startup failed because OAuth configuration was + absent. A dedicated single-tenant evaluation registration now exposes the UI's + own API scope. Its short-lived credential was delivered encrypted to the + private runner, stored only in Key Vault, and referenced in App Configuration + with label `agent-lz`; tenant credential-lifetime policy was preserved. + A persistent Key Vault-backed Chainlit signing key was also configured. + Both intended pinned revisions are now Healthy/RunningAtMaxScale, match + latestReadyRevisionName, and receive 100% of traffic; placeholders no longer + serve the application. Private probes verified the real UI, enabled Entra + provider, correct login redirect, ingestion health, and HTTP 401 on + unauthenticated panel data access. Authentication has not been bypassed. + Actual user login/consent, user-delegated OBO, grounded answers, citations, + and positive document authorization remain unverified. An OAuth resource + token diagnostic did not invoke the agent and is not OBO acceptance. - The evaluation VPN Gateway was provisioned and protected with `keep=true` and a deletion lock. Hub/spoke peerings are Connected with gateway transit. Private DNS and managed-identity governance calls succeeded from the actual From aae0a3cfbda8aac029e208a46799bd9d90277664 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 13:19:29 -0300 Subject: [PATCH 10/18] docs: record private hosted authorization rejection and acceptance blockers Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../quickstart.md | 32 ++++++++++++++++++- specs/002-agent-landing-zone-rebrand/tasks.md | 4 +-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/specs/002-agent-landing-zone-rebrand/quickstart.md b/specs/002-agent-landing-zone-rebrand/quickstart.md index 84404851..d931929a 100644 --- a/specs/002-agent-landing-zone-rebrand/quickstart.md +++ b/specs/002-agent-landing-zone-rebrand/quickstart.md @@ -262,10 +262,40 @@ close the outstanding runtime acceptance scenarios: Actual user login/consent, user-delegated OBO, grounded answers, citations, and positive document authorization remain unverified. An OAuth resource token diagnostic did not invoke the agent and is not OBO acceptance. + A subsequent negative runtime test acquired a valid Foundry resource token + for the dedicated evaluation service principal, which has no project role. + The actual hosted invocation returned a structured HTTP 403 authorization + rejection. The project endpoint resolved entirely inside the evaluation + spoke. All private foundation endpoint connections were Approved/Succeeded; + Foundry, ACR, and Key Vault public access remained Disabled, ACR admin + authentication remained disabled, and Key Vault RBAC remained enabled. + Runtime grants were rechecked at their exact scopes: App Configuration + Data Reader on the store, OpenAI User on the model account, and Secrets + User only on AUDIT-HMAC-KEY. This is hosted-principal isolation evidence, + not delegated-user or document-authorization acceptance, and does not + replace deploying the custom hosted sample. - The evaluation VPN Gateway was provisioned and protected with `keep=true` and a deletion lock. Hub/spoke peerings are Connected with gateway transit. Private DNS and managed-identity governance calls succeeded from the actual - jumpbox. No client VPN authentication or hosted private runtime is approved. + jumpbox. Client VPN authentication and full delegated-user hosted runtime + acceptance remain unverified. +- Classic runtime connectivity cannot be repaired by adding an endpoint to its + current Container Apps environment: its `vnetConfiguration` is null and the + classic resource group has no virtual network. The existing recovery + endpoints make services reachable from the private jumpbox, not from apps + in the Azure-managed network. Azure does not support changing that + environment's network type after creation + ([Azure Container Apps networking](https://learn.microsoft.com/azure/container-apps/networking)). + Recovery requires a separate + VNet-integrated environment; public Key Vault access was not enabled and + existing apps/resources were not deleted or silently replaced. +- A fresh S7 foundation cannot use the current default Standard embedding + allocation unchanged: live westus3 usage is 300 of 350, leaving 50 while + the default new deployment requests 100. Existing-allocation credit is + valid only for an actual matching deployment, not a fresh account. Both + sample definitions validate, and the focused schema, roles, binding, + composition, and custom-hook suite passed 43 tests and 28 subtests. + Those checks do not constitute a successful one-command S7 deployment. - Validation found directory-selection, custom-project working-directory, and hosted greeting-protocol defects. The corrective change passes 598 tests and 591 subtests (4 skipped), but those offline results do not diff --git a/specs/002-agent-landing-zone-rebrand/tasks.md b/specs/002-agent-landing-zone-rebrand/tasks.md index bcfc0ef2..bc8c5070 100644 --- a/specs/002-agent-landing-zone-rebrand/tasks.md +++ b/specs/002-agent-landing-zone-rebrand/tasks.md @@ -212,9 +212,9 @@ modes). Fallback per R16 if FR-015a–h does not land by 2026-10-09. - [X] T084 [US5] Publish the "Build your own application" page (schema, profiles, outputs, both modes, binding rule) on the central AI Landing Zones site and register it in its nav - Status: deferred: central docs repo (external) - [ ] T085 [US5] Validate S5 (invalid and changed definitions rejected before any Azure change) and S7 (sample deploys with one `azd up` in both modes) and record evidence in specs/002-agent-landing-zone-rebrand/quickstart.md - - Status: partial: S5 covered by tests/test_app_definition*.py; S7 deploy blocked by the same MCAPS constraints as T035 + - Status: partial: S5 schema/binding negatives passed locally and on the private runner without modifying the bound environment. Both sample definitions validate; focused schema/roles/binding/composition/hooks suite passed 43 tests and 28 subtests. S7 still requires fresh one-command deploys in both modes; current default new Standard embedding allocation requests 100 with only 50 regional headroom. Existing allocations cannot be credited to a new account. - [ ] T098 [US5] Deploy the hosted mode with `NETWORK_ISOLATION=true` and verify private endpoints and RBAC for the hosted-agent component (FR-015c); record evidence in specs/002-agent-landing-zone-rebrand/quickstart.md - - Status: blocked: network-isolated deploy not possible under MCAPS policy in the maintainer subscription + - Status: partial: bundled hosted component deployed on corrective source with network isolation; managed greeting passed, private endpoints are Approved/Succeeded, public access remains disabled, exact least-privilege runtime grants verified, and a valid token from an unassigned principal was rejected by the actual private hosted endpoint with structured HTTP 403. Custom hosted sample parity (FR-015c) remains unverified; these checks do not prove user OBO or document authorization. - [X] T099 [P] [US5] Compare the role assignments generated by the profiles (`base`, `model-user`, `retrieval-reader`, `conversation-store`, `blob-delegator`, `ingestion-writer`) with the trio's current role assignments and record a parity checklist (FR-015e, SC-006) in tests/test_app_definition_schema.py and specs/002-agent-landing-zone-rebrand/quickstart.md - [X] T086 [US5] If FR-015a–h misses 2026-10-09, publish the R16 manual procedure instead and mark FR-015 as fallback in specs/002-agent-landing-zone-rebrand/spec.md and the central docs page - Status: N/A. FR-015 (app definition) shipped in v4.0.0, so the fallback was not needed. From 5141cf834e018bb97e2fe18e2cf4704779320869 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 14:43:28 -0300 Subject: [PATCH 11/18] docs: record sample private endpoint recovery evidence Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- specs/002-agent-landing-zone-rebrand/quickstart.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/specs/002-agent-landing-zone-rebrand/quickstart.md b/specs/002-agent-landing-zone-rebrand/quickstart.md index d931929a..0929d4d9 100644 --- a/specs/002-agent-landing-zone-rebrand/quickstart.md +++ b/specs/002-agent-landing-zone-rebrand/quickstart.md @@ -296,6 +296,18 @@ close the outstanding runtime acceptance scenarios: sample definitions validate, and the focused schema, roles, binding, composition, and custom-hook suite passed 43 tests and 28 subtests. Those checks do not constitute a successful one-command S7 deployment. +- Separate Container App and hosted sample evaluations now use 20 new + Standard embedding units each in evaluation-only parameter copies. Published + defaults are unchanged. Both real preProvision gates and composed previews + passed with distinct nonoverlapping spokes and independent application + bindings. Initial provisioning failed on each Container Apps private endpoint + with provider `InternalServerError`, not a proven naming conflict. Retrying + only the exact failed endpoint module, with unchanged exported parameters and + a no-deletion what-if, recovered both endpoints to Succeeded/Approved. + Reverse VPN gateway-transit peerings and direct runner-to-spoke peerings are + Connected. Normal root provisioning has been resumed; full configuration, + private DNS resolution, custom image builds, application responses, and + repeat deployment are not yet verified. - Validation found directory-selection, custom-project working-directory, and hosted greeting-protocol defects. The corrective change passes 598 tests and 591 subtests (4 skipped), but those offline results do not From 29c52eb9914cfe9292ef3b5d8fa0cd19b531f43c Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 15:29:41 -0300 Subject: [PATCH 12/18] docs: record completed sample foundations and private runner probes Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- specs/002-agent-landing-zone-rebrand/quickstart.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/specs/002-agent-landing-zone-rebrand/quickstart.md b/specs/002-agent-landing-zone-rebrand/quickstart.md index 0929d4d9..a4a9b413 100644 --- a/specs/002-agent-landing-zone-rebrand/quickstart.md +++ b/specs/002-agent-landing-zone-rebrand/quickstart.md @@ -305,9 +305,17 @@ close the outstanding runtime acceptance scenarios: only the exact failed endpoint module, with unchanged exported parameters and a no-deletion what-if, recovered both endpoints to Succeeded/Approved. Reverse VPN gateway-transit peerings and direct runner-to-spoke peerings are - Connected. Normal root provisioning has been resumed; full configuration, - private DNS resolution, custom image builds, application responses, and - repeat deployment are not yet verified. + Connected. Both normal root provisioning resumes completed successfully, + explicitly deferring private configuration rather than claiming application + readiness. Exact new-spoke private A records were added, without replacing + existing records, to the evaluation runner's already-linked DNS zones. + From the actual jumpbox, both samples' App Configuration, Key Vault, ACR, + and Foundry endpoints resolved only into their corresponding private spokes + and returned HTTPS responses with TLS verification. These are connectivity + probes, not authenticated application acceptance. Resource-scoped runner + configuration roles were reconciled without granting Owner or role-management + permissions. Full configuration, custom image builds, application responses, + and repeat deployment remain unverified. - Validation found directory-selection, custom-project working-directory, and hosted greeting-protocol defects. The corrective change passes 598 tests and 591 subtests (4 skipped), but those offline results do not From d1029d10b4e463e20e41fbcecf13ae3402e06d43 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 16:07:57 -0300 Subject: [PATCH 13/18] docs: record full private configuration and recovered sample builds Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../quickstart.md | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/specs/002-agent-landing-zone-rebrand/quickstart.md b/specs/002-agent-landing-zone-rebrand/quickstart.md index a4a9b413..5c3c7a27 100644 --- a/specs/002-agent-landing-zone-rebrand/quickstart.md +++ b/specs/002-agent-landing-zone-rebrand/quickstart.md @@ -314,8 +314,29 @@ close the outstanding runtime acceptance scenarios: and returned HTTPS responses with TLS verification. These are connectivity probes, not authenticated application acceptance. Resource-scoped runner configuration roles were reconciled without granting Owner or role-management - permissions. Full configuration, custom image builds, application responses, - and repeat deployment remain unverified. + permissions. The Container sample's full root postProvision subsequently + passed from the pinned corrective guest source: platform outputs, governance, + continuity, Foundry, Container Apps, and Search completed without deferral. + The first capability-role attempt correctly rejected the runner's lack of + role-management privileges; the parent Owner reconciled only the declared + component grants before the successful unchanged hook retry. The hosted + sample's full root postProvision also passed from the same pinned source, + with explicit guest success markers and no guest stderr for both samples. + Private image building then reached the dedicated ACR pool but failed on + the Docker Hub base-image pull: `registry-1.docker.io` was absent from the + build-subnet HTTPS allowlist. This is a build dependency, not a reason to + enable public registry access or bypass the firewall. Extending only the + build-subnet rule with the three documented Docker Hub HTTPS endpoints, + persisted through evaluation-only `additionalAcrTaskBuildFqdns`, recovered + both builds. Both images were built from the exact corrective source using + their own private ACR `build-pool`; immutable image digests and an explicit + guest completion marker were returned with no guest stderr. This separate + prebuild is not proof of the stock one-command S7 deployment. Application + responses and repeat deployment remain unverified. + The evaluation VPN client reached the existing firewall DNS proxy and + received the expected private addresses for all three evaluation spokes. + The updated client profile preserves authentication and split tunneling; + automatic OS resolution still requires profile reimport and reconnect. - Validation found directory-selection, custom-project working-directory, and hosted greeting-protocol defects. The corrective change passes 598 tests and 591 subtests (4 skipped), but those offline results do not From 5f3613b3047bb622d2f5e0951a9b03722805191d Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 16:27:02 -0300 Subject: [PATCH 14/18] docs: record actual private sample runtime responses Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../quickstart.md | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/specs/002-agent-landing-zone-rebrand/quickstart.md b/specs/002-agent-landing-zone-rebrand/quickstart.md index 5c3c7a27..3d4c519d 100644 --- a/specs/002-agent-landing-zone-rebrand/quickstart.md +++ b/specs/002-agent-landing-zone-rebrand/quickstart.md @@ -331,8 +331,28 @@ close the outstanding runtime acceptance scenarios: both builds. Both images were built from the exact corrective source using their own private ACR `build-pool`; immutable image digests and an explicit guest completion marker were returned with no guest stderr. This separate - prebuild is not proof of the stock one-command S7 deployment. Application - responses and repeat deployment remain unverified. + prebuild is not proof of the stock one-command S7 deployment. + Both samples subsequently packaged and deployed using evaluation-only + child YAML with immutable `image` and `docker.imagePassthrough: true`. + The Container App uses its existing managed identity for private ACR pulls; + its latest revision is latest-ready and matches the built digest. Its `/` + response, reached through private DNS, exactly matches live + `AGENTLZ_PLATFORM_OUTPUTS` with label `agent-lz`, schema version 1 and + `network.isolated=true`. A TLS-verified local VPN probe also received that + contract using explicit DNS, not automatic OS resolution. The published + identity map is empty despite the deployed app identity; identity-contract + acceptance is not claimed. + The hosted sample initially failed activation because the actual Foundry + project identity lacked AcrPull. An exact-registry grant recovered version + 2 to active with the built digest. Its actual runtime identity then received + App Configuration Data Reader only on its configuration store. A real + Responses invocation from the managed runner resolved the project into its + private spoke and returned a completed assistant response containing the + exact project endpoint published in live platform outputs, not `unknown` + or `unavailable`. The explicit runtime-check marker and empty guest stderr + were verified. This proves private managed-identity sample invocation, + not user-delegated OBO, grounded answers, document authorization, repeated + provisioning, or the original remote-build sample YAML. The evaluation VPN client reached the existing firewall DNS proxy and received the expected private addresses for all three evaluation spokes. The updated client profile preserves authentication and split tunneling; From 13832c80d4d3ee1b6f99d626c2f2182234e8788b Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 16:54:08 -0300 Subject: [PATCH 15/18] fix: publish provisioned component identity client IDs Discover actual Container App client IDs when no explicit identity map is supplied. Preserve the map override and fail before publication for unresolved identities. Record live private publication and repeated child runtime evidence. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- config/appdefinition/roles.py | 60 ++++++++++++++--- config/deployment/outputs.py | 16 ++++- .../quickstart.md | 24 ++++++- tests/test_app_definition_roles.py | 66 +++++++++++++++++++ tests/test_platform_outputs.py | 31 +++++++++ 5 files changed, 183 insertions(+), 14 deletions(-) diff --git a/config/appdefinition/roles.py b/config/appdefinition/roles.py index e21a9e9c..d9c60929 100644 --- a/config/appdefinition/roles.py +++ b/config/appdefinition/roles.py @@ -137,8 +137,7 @@ def resolve_scope(scope: str, environment: Mapping[str, str], resource_group: st return str(items[0]["id"]) -def resolve_principal(service: str, resource_group: str, run_az: RunAz) -> str: - """Return the single managed identity principal of the service's Container App.""" +def _resolve_identity(service: str, resource_group: str, run_az: RunAz) -> tuple[str, str, str]: apps = _json(run_az, ["containerapp", "list", "--resource-group", resource_group, "--query", f"[?tags.\"azd-service-name\"=='{service}'].{{name:name,identity:identity}}"]) apps = [app for app in apps or [] if isinstance(app, dict)] @@ -148,18 +147,60 @@ def resolve_principal(service: str, resource_group: str, run_az: RunAz) -> str: f"{resource_group}; found {len(apps)}. Run azd provision for this application first." ) identity = apps[0].get("identity") or {} - principals = [str(identity["principalId"])] if identity.get("principalId") else [] - principals += [ - str(value["principalId"]) - for value in (identity.get("userAssignedIdentities") or {}).values() + identities = [(str(identity["principalId"]), "", "")] if identity.get("principalId") else [] + identities += [ + (str(value["principalId"]), str(value.get("clientId") or ""), str(resource_id)) + for resource_id, value in (identity.get("userAssignedIdentities") or {}).items() if isinstance(value, dict) and value.get("principalId") ] - if len(principals) != 1: + if len(identities) != 1: raise RoleAssignmentError( f"Container App {apps[0].get('name')} must have exactly one managed identity to receive " - f"profile roles; found {len(principals)}." + f"profile roles; found {len(identities)}." ) - return principals[0] + return identities[0] + + +def resolve_principal(service: str, resource_group: str, run_az: RunAz) -> str: + """Return the single managed identity principal of the service's Container App.""" + return _resolve_identity(service, resource_group, run_az)[0] + + +def discover_component_identities( + definition: AppDefinition, + environment: Mapping[str, str], + *, + hosted_orchestration: bool, + run_az: RunAz | None = None, +) -> dict[str, str]: + """Read Container App client IDs for platform outputs, without role writes.""" + components = [ + component + for component in effective_components(definition, hosted_orchestration=hosted_orchestration) + if component.get("kind") == KIND_CONTAINER_APP + ] + if not components: + return {} + resource_group = (environment.get("AZURE_RESOURCE_GROUP") or "").strip() + if not resource_group: + raise RoleAssignmentError("AZURE_RESOURCE_GROUP is required to discover component identities.") + if run_az is None: + from config.deployment.appconfig import _run_az as run_az + identities: dict[str, str] = {} + for component in components: + service = component_service_name(definition, component) + principal, client, resource_id = _resolve_identity(service, resource_group, run_az) + if not client: + arguments = ( + ["identity", "show", "--ids", resource_id, "--query", "clientId"] + if resource_id else + ["ad", "sp", "show", "--id", principal, "--query", "appId"] + ) + client = _json(run_az, arguments) + if not isinstance(client, str) or not client.strip(): + raise RoleAssignmentError(f"Cannot resolve the managed identity client ID for service {service}.") + identities[str(component["name"])] = client.strip() + return identities def _ensure_rbac(principal: str, role_guid: str, scope_id: str, run_az: RunAz) -> bool: @@ -232,4 +273,3 @@ def assign_roles( if changed: created.append(item) return created - diff --git a/config/deployment/outputs.py b/config/deployment/outputs.py index 81cd224a..f83d3355 100644 --- a/config/deployment/outputs.py +++ b/config/deployment/outputs.py @@ -227,6 +227,10 @@ def publish(endpoint: str, settings: Mapping[str, str]) -> None: def main(argv: list[str] | None = None) -> int: + from config.appdefinition import AppDefinitionError, load_selected_definition, validate_definition + from config.appdefinition.roles import RoleAssignmentError, discover_component_identities + from config.deployment.composition import DeploymentMode, DeploymentTopologyError, resolve_mode + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument( "--check-foundation", @@ -243,9 +247,17 @@ def main(argv: list[str] | None = None) -> int: require_foundation(os.environ) if args.check_foundation: return 0 - document = build_platform_outputs(os.environ) + identities = None + if not (os.environ.get(IDENTITIES_ENV) or "").strip(): + definition = validate_definition(load_selected_definition(os.environ)) + identities = discover_component_identities( + definition, os.environ, + hosted_orchestration=resolve_mode(os.environ) is not DeploymentMode.CLASSIC, + ) + document = build_platform_outputs(os.environ, identities) validate_platform_outputs(document) - except (FoundationMissingError, PlatformOutputsError) as error: + except (FoundationMissingError, PlatformOutputsError, AppDefinitionError, + RoleAssignmentError, DeploymentTopologyError) as error: print(str(error), file=sys.stderr) return 1 settings = platform_settings(document) diff --git a/specs/002-agent-landing-zone-rebrand/quickstart.md b/specs/002-agent-landing-zone-rebrand/quickstart.md index 3d4c519d..b1371d8a 100644 --- a/specs/002-agent-landing-zone-rebrand/quickstart.md +++ b/specs/002-agent-landing-zone-rebrand/quickstart.md @@ -340,8 +340,16 @@ close the outstanding runtime acceptance scenarios: `AGENTLZ_PLATFORM_OUTPUTS` with label `agent-lz`, schema version 1 and `network.isolated=true`. A TLS-verified local VPN probe also received that contract using explicit DNS, not automatic OS resolution. The published - identity map is empty despite the deployed app identity; identity-contract - acceptance is not claimed. + identity map was initially empty despite the deployed app identity. + The corrective publisher now discovers the actual Container App client ID. + Two consecutive publications returned the same `web` identity in both the + JSON document and its flat key; a subsequent private `/` response exactly + matched that identity-bearing live contract. The guest temporarily staged + the corrected publisher/resolver and restored its original source afterward. + Missing endpoint aliases in this evaluation's copied azd metadata were + explicitly hydrated from the schema-validated existing App Configuration + document, not guessed. This proves the correction against the live resources, + not a pristine published-release or stock one-command S7 run. The hosted sample initially failed activation because the actual Foundry project identity lacked AcrPull. An exact-registry grant recovered version 2 to active with the built digest. Its actual runtime identity then received @@ -353,6 +361,18 @@ close the outstanding runtime acceptance scenarios: were verified. This proves private managed-identity sample invocation, not user-delegated OBO, grounded answers, document authorization, repeated provisioning, or the original remote-build sample YAML. + Repeating both child overlay deployments preserved their immutable image + digests. The Container App's new revision + `ca-ddzasx67ghev4-web--azd-1791574208` became latest-ready, and its repeated + private response again exactly matched live platform outputs. The hosted + deployment reused active `sample-hosted-agent` version 2 and the same runtime + identity; its repeated Responses check again returned the exact live project + endpoint. All 13 hosted private endpoints were Succeeded/Approved; Foundry + public access and local authentication, and ACR public/admin access, remained + disabled. These checks prove repeated child deployment, not repeated full + root provisioning or the stock one-command S7 flow. + The identity correction passed 413 tests and 500 subtests, with 4 skips; + the central documentation update `0abad14` is in Azure/AI-Landing-Zones#148. The evaluation VPN client reached the existing firewall DNS proxy and received the expected private addresses for all three evaluation spokes. The updated client profile preserves authentication and split tunneling; diff --git a/tests/test_app_definition_roles.py b/tests/test_app_definition_roles.py index ac16e2f6..52c6acd6 100644 --- a/tests/test_app_definition_roles.py +++ b/tests/test_app_definition_roles.py @@ -14,6 +14,7 @@ from config.appdefinition.roles import ( RoleAssignmentError, assign_roles, + discover_component_identities, load_role_guids, plan_assignments, ) @@ -84,6 +85,71 @@ def bundled(): class RoleAssignmentTests(unittest.TestCase): + def test_discovers_client_ids_without_role_writes(self) -> None: + calls = [] + + def az(arguments, *, required=True): + calls.append(arguments) + service = arguments[arguments.index("--query") + 1].split("=='", 1)[1].split("'", 1)[0] + return json.dumps([{ + "name": f"ca-{service}", + "identity": {"userAssignedIdentities": { + f"{SUB}/identity-{service}": {"principalId": f"p-{service}", "clientId": f"c-{service}"} + }}, + }]) + + actual = discover_component_identities( + bundled(), {"AZURE_RESOURCE_GROUP": RG}, hosted_orchestration=True, run_az=az + ) + self.assertEqual({"ui": "c-frontend", "ingestion": "c-dataingest"}, actual) + self.assertTrue(all(call[:2] == ["containerapp", "list"] for call in calls)) + + def test_client_id_lookup_is_exact_and_missing_identity_fails(self) -> None: + definition = load_definition(ROOT / "samples" / "custom-app" / "containerapp") + for system in (False, True): + with self.subTest(system=system): + calls = [] + + def az(arguments, *, required=True): + calls.append(arguments) + if arguments[:2] == ["containerapp", "list"]: + identity = {"principalId": "principal"} if system else { + "userAssignedIdentities": {f"{SUB}/uai": {"principalId": "principal"}} + } + return json.dumps([{"name": "ca-web", "identity": identity}]) + self.assertEqual( + ["ad", "sp", "show", "--id", "principal", "--query", "appId"] if system else + ["identity", "show", "--ids", f"{SUB}/uai", "--query", "clientId"], + arguments[:-3], + ) + return json.dumps("client") + + self.assertEqual({"web": "client"}, discover_component_identities( + definition, {"AZURE_RESOURCE_GROUP": RG}, hosted_orchestration=False, run_az=az + )) + self.assertEqual(2, len(calls)) + with self.assertRaisesRegex(RoleAssignmentError, "found 0"): + discover_component_identities( + definition, {"AZURE_RESOURCE_GROUP": RG}, hosted_orchestration=False, + run_az=lambda arguments, **kwargs: "[]", + ) + with self.assertRaisesRegex(RoleAssignmentError, "found 2"): + discover_component_identities( + definition, {"AZURE_RESOURCE_GROUP": RG}, hosted_orchestration=False, + run_az=lambda arguments, **kwargs: json.dumps([{ + "name": "ca-web", "identity": { + "principalId": "p1", "userAssignedIdentities": {"uai": {"principalId": "p2"}} + } + }]), + ) + + def test_hosted_only_definition_has_no_provision_time_identities(self) -> None: + definition = load_definition(ROOT / "samples" / "custom-app" / "hosted") + self.assertEqual({}, discover_component_identities( + definition, {}, hosted_orchestration=False, + run_az=lambda *args, **kwargs: self.fail("Hosted-only provision must not discover a Container App"), + )) + def test_plan_covers_only_containerapp_components_with_base_profile(self) -> None: classic = plan_assignments(bundled(), hosted_orchestration=False) self.assertEqual({"ui", "orchestrator", "ingestion"}, {item.component for item in classic}) diff --git a/tests/test_platform_outputs.py b/tests/test_platform_outputs.py index de31867e..58830ce8 100644 --- a/tests/test_platform_outputs.py +++ b/tests/test_platform_outputs.py @@ -98,6 +98,37 @@ def test_cli_dry_run_writes_nothing(monkeypatch, capsys): assert "AGENTLZ_PLATFORM_OUTPUTS" in json.loads(capsys.readouterr().out) +def test_cli_discovers_and_publishes_container_identity_when_map_is_absent(monkeypatch, capsys): + for key, value in ENVIRONMENT.items(): + monkeypatch.setenv(key, value) + monkeypatch.delenv(outputs.IDENTITIES_ENV) + monkeypatch.setenv("AGENTLZ_APP_DEFINITION", str(REPO_ROOT / "samples" / "custom-app" / "containerapp")) + with mock.patch("config.appdefinition.roles.discover_component_identities", + return_value={"web": UI_CLIENT_ID}) as discover, mock.patch.object(outputs, "publish") as publish: + assert outputs.main([]) == 0 + discover.assert_called_once() + settings = publish.call_args.args[1] + assert json.loads(settings[outputs.PLATFORM_OUTPUTS_KEY])["identities"] == [ + {"component": "web", "clientId": UI_CLIENT_ID} + ] + assert settings["AGENTLZ_IDENTITY_WEB_CLIENT_ID"] == UI_CLIENT_ID + + +def test_cli_rejects_identity_discovery_failure_before_publication(monkeypatch, capsys): + from config.appdefinition.roles import RoleAssignmentError + + for key, value in ENVIRONMENT.items(): + monkeypatch.setenv(key, value) + monkeypatch.delenv(outputs.IDENTITIES_ENV) + monkeypatch.setenv("AGENTLZ_APP_DEFINITION", str(REPO_ROOT / "samples" / "custom-app" / "containerapp")) + with mock.patch("config.appdefinition.roles.discover_component_identities", + side_effect=RoleAssignmentError("Expected exactly one identity")), \ + mock.patch.object(outputs, "publish") as publish: + assert outputs.main([]) == 1 + publish.assert_not_called() + assert "Expected exactly one identity" in capsys.readouterr().err + + def test_repo_root_detected_by_markers_not_folder_name(tmp_path): root = tmp_path / "any-folder-name" nested = root / "config" / "deployment" From a9214de71dd56ec17c9152b2eee74dcdea59cf6a Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 17:17:37 -0300 Subject: [PATCH 16/18] fix: deploy custom services with immutable private build results Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- config/deployment/custom_application.py | 168 ++++++++++++++++++++++++ scripts/preDeploy.ps1 | 10 +- scripts/preDeploy.sh | 10 +- tests/test_custom_application_deploy.py | 157 ++++++++++++++++++++++ tests/test_custom_application_hooks.py | 30 ++++- 5 files changed, 361 insertions(+), 14 deletions(-) create mode 100644 config/deployment/custom_application.py create mode 100644 tests/test_custom_application_deploy.py diff --git a/config/deployment/custom_application.py b/config/deployment/custom_application.py new file mode 100644 index 00000000..b0c9d8c4 --- /dev/null +++ b/config/deployment/custom_application.py @@ -0,0 +1,168 @@ +"""Deploy one custom service with an immutable image and private ACR builds.""" + +from __future__ import annotations + +import argparse +import copy +import logging +import os +from pathlib import Path +import re +import subprocess +from typing import Any, Mapping +from uuid import uuid4 + +import yaml + +from config.appdefinition import AppDefinitionError, load_definition, validate_definition +from config.deployment.composition import is_truthy +from config.deployment.hosted_image import ( + build_source_image, + validate_acr_agent_pool, + validate_digest, +) +from config.deployment.private_network import check_stage +from util.azure_cli import resolve_az_command + +LOGGER = logging.getLogger("config.deployment.custom_application") + + +def _expand(value: str, environment: Mapping[str, str]) -> str: + def replace(match: re.Match[str]) -> str: + name = match[1] + result = (environment.get(name) or "").strip() + if not result: + raise ValueError(f"Image requires environment variable {name}.") + return result + + return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", replace, value) + + +def prepare_image( + component: Mapping[str, Any], + service: Mapping[str, Any], + folder: Path, + environment: Mapping[str, str], +) -> str | None: + """Keep normal azd builds, except pins and isolated Docker source builds.""" + source = component["source"] + pin = source.get("imageDigest") + isolated = is_truthy(environment.get("NETWORK_ISOLATION")) + if pin: + digest = validate_digest(pin, name="source.imageDigest") + image = _expand(str(service.get("image") or ""), environment) + if not re.fullmatch( + r"[a-z0-9.-]+(?::[0-9]+)?/[a-z0-9._/-]+(?::[A-Za-z0-9_.-]+|@sha256:[0-9a-f]{64})?", + image, + ): + raise ValueError("source.imageDigest requires a qualified service image in azure.yaml.") + repository = image.split("@", 1)[0] + last_slash = repository.rfind("/") + if ":" in repository[last_slash:]: + repository = repository[:repository.rfind(":")] + return f"{repository}@{digest}" + source_dir = (folder / component["path"]).resolve() + actual = subprocess.run( + ["git", "-C", str(source_dir), "rev-parse", "HEAD"], + check=True, capture_output=True, text=True, + ).stdout.strip() + if actual.lower() != source["commit"].lower(): + raise ValueError("Custom source HEAD does not match source.commit.") + if not isolated: + return None + if service.get("language") != "docker": + raise ValueError( + "Isolated custom source builds require language: docker or source.imageDigest." + ) + docker = service.get("docker") or {} + unsupported = set(docker) - {"remoteBuild", "path", "context"} + if unsupported: + raise ValueError( + "Isolated custom builds cannot preserve these docker options: " + + ", ".join(sorted(unsupported)) + + ". Supply a prebuilt source.imageDigest instead." + ) + project_dir = (folder / str(service.get("project") or ".")).resolve() + if source_dir != project_dir: + raise ValueError("The service project must match the pinned component path.") + if docker.get("path", "Dockerfile") != "Dockerfile" or docker.get("context", ".") != ".": + raise ValueError( + "Isolated custom builds require Dockerfile and context '.' in the component path; " + "supply a prebuilt source.imageDigest for another layout." + ) + registry_endpoint = ( + environment.get("AZURE_CONTAINER_REGISTRY_ENDPOINT") + or environment.get("CONTAINER_REGISTRY_LOGIN_SERVER") or "" + ).strip() + pool = (environment.get("ACR_TASK_AGENT_POOL") or "").strip() + group = (environment.get("AZURE_RESOURCE_GROUP") or "").strip() + if not registry_endpoint or not pool or not group: + raise ValueError( + "Isolated custom builds require the registry endpoint, " + "ACR_TASK_AGENT_POOL and AZURE_RESOURCE_GROUP." + ) + registry = registry_endpoint.split(".", 1)[0] + azure_cli = resolve_az_command() + check_stage(environment, "hosted-build") + validate_acr_agent_pool( + registry=registry, resource_group=group, agent_pool=pool, azure_cli=azure_cli, + ) + image_name = f"agent-landing-zone/custom/{component['name']}" + digest = build_source_image( + registry=registry, source_dir=source_dir, image_name=image_name, + image_tag=f"{actual[:12]}-{uuid4().hex[:12]}", + agent_pool=pool, azure_cli=azure_cli, + ) + return f"{registry_endpoint}/{image_name}@{digest}" + + +def deploy_service(definition_path: str, name: str, environment: Mapping[str, str]) -> str | None: + definition = validate_definition(load_definition(definition_path)) + if definition.bundled: + raise ValueError("Custom deployment cannot deploy a bundled component.") + components = [item for item in definition.document["components"] if item["name"] == name] + if len(components) != 1: + raise ValueError(f"No unique custom component named {name}.") + env_name = (environment.get("AZURE_ENV_NAME") or "").strip() + if not env_name: + raise ValueError("AZURE_ENV_NAME is required for custom deployment.") + path = definition.folder / "azure.yaml" + original = path.read_bytes() + project = yaml.safe_load(original.decode("utf-8-sig")) + service = project["services"][name] + image = prepare_image(components[0], service, definition.folder, environment) + try: + if image: + rendered = copy.deepcopy(project) + rendered_service = rendered["services"][name] + rendered_service["image"] = image + rendered_service["docker"] = {"imagePassthrough": True} + path.write_text(yaml.safe_dump(rendered, sort_keys=False), encoding="utf-8") + LOGGER.info("Deploying custom service %s with immutable image %s", name, image) + subprocess.run( + ["azd", "deploy", name, "--environment", env_name, "--no-prompt"], + cwd=definition.folder, env={**os.environ, **environment}, check=True, + ) + finally: + if image: + path.write_bytes(original) + return image + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--definition", required=True) + parser.add_argument("--service", required=True) + parser.add_argument("--environment", required=True) + args = parser.parse_args(argv) + try: + deploy_service(args.definition, args.service, {**os.environ, "AZURE_ENV_NAME": args.environment}) + except (AppDefinitionError, ValueError, RuntimeError, OSError, subprocess.CalledProcessError, + yaml.YAMLError) as error: + LOGGER.error("Custom service deployment failed: %s", error) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/preDeploy.ps1 b/scripts/preDeploy.ps1 index 835b23ff..31503316 100644 --- a/scripts/preDeploy.ps1 +++ b/scripts/preDeploy.ps1 @@ -512,15 +512,13 @@ foreach ($dc in $definitionComponents) { if ("$($dc.kind)" -eq 'azure.ai.agent') { $smokeProtocol = (& python -c "import os, runpy, sys; sys.path.insert(0, os.environ['AGENTLZ_REPO_ROOT']); sys.argv = ['config.deployment.hosted'] + sys.argv[1:]; runpy.run_module('config.deployment.hosted', run_name='__main__')" --service-protocol (Join-Path $appDefinitionDir 'azure.yaml') --service-name $componentName | Out-String).Trim() if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: cannot resolve the greeting protocol."; $hadErrors = $true; continue } - # prepareHostedDeployment is not run here: it builds only the manifest-pinned - # bundled orchestrator image (HOSTED_AGENT_*). A custom azure.ai.agent - # component deploys from its own child azd project, which builds its image, - # or pins source.imageDigest through AGENTLZ_IMAGE_DIGEST (T079 limitation). + # Custom images use the shared immutable/private build path, not the + # manifest-pinned orchestrator's derived hosted entrypoint. & azd env set FOUNDRY_PROJECT_ENDPOINT "$($globalEnv.AZURE_AI_PROJECT_ENDPOINT)" --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt | Out-Null if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: failed to set the Foundry endpoint."; $hadErrors = $true; continue } & azd env set AZURE_AI_PROJECT_ID "$($globalEnv.AZURE_AI_PROJECT_RESOURCE_ID)" --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt | Out-Null if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: failed to set the Foundry project ID."; $hadErrors = $true; continue } - & azd deploy $componentName --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt + & python -c "import os, runpy, sys; sys.path.insert(0, os.environ['AGENTLZ_REPO_ROOT']); sys.argv = ['config.deployment.custom_application'] + sys.argv[1:]; runpy.run_module('config.deployment.custom_application', run_name='__main__')" --definition $appDefinitionPath --service $componentName --environment "$($globalEnv.AZURE_ENV_NAME)" if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: hosted agent deployment failed."; $hadErrors = $true; continue } $smokePayloadPath = Join-Path ([IO.Path]::GetTempPath()) "agentlz-agent-smoke-$([guid]::NewGuid().ToString('N')).json" try { @@ -536,7 +534,7 @@ foreach ($dc in $definitionComponents) { Remove-Item -LiteralPath $smokePayloadPath -Force -ErrorAction SilentlyContinue } } else { - & azd deploy $componentName --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt + & python -c "import os, runpy, sys; sys.path.insert(0, os.environ['AGENTLZ_REPO_ROOT']); sys.argv = ['config.deployment.custom_application'] + sys.argv[1:]; runpy.run_module('config.deployment.custom_application', run_name='__main__')" --definition $appDefinitionPath --service $componentName --environment "$($globalEnv.AZURE_ENV_NAME)" if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: Container App deployment failed."; $hadErrors = $true; continue } } Write-Host "${componentName}: deployed." -ForegroundColor Green diff --git a/scripts/preDeploy.sh b/scripts/preDeploy.sh index cc5dd287..fce5c9ac 100755 --- a/scripts/preDeploy.sh +++ b/scripts/preDeploy.sh @@ -452,15 +452,13 @@ while IFS= read -r dcomp; do smoke_protocol="$(cd "$repo_root" && python3 -m config.deployment.hosted --service-protocol "$app_definition_dir/azure.yaml" --service-name "$c_name")" || { red "$c_name: cannot resolve the greeting protocol."; had_errors=1; continue } - # prepareHostedDeployment is not run here: it builds only the manifest-pinned - # bundled orchestrator image (HOSTED_AGENT_*). A custom azure.ai.agent - # component deploys from its own child azd project, which builds its image, - # or pins source.imageDigest through AGENTLZ_IMAGE_DIGEST (T079 limitation). + # Custom images use the shared immutable/private build path, not the + # manifest-pinned orchestrator's derived hosted entrypoint. ( cd "$app_definition_dir" && azd env set FOUNDRY_PROJECT_ENDPOINT "${AZURE_AI_PROJECT_ENDPOINT:-}" --environment "$env_name" --no-prompt >/dev/null && azd env set AZURE_AI_PROJECT_ID "${AZURE_AI_PROJECT_RESOURCE_ID:-}" --environment "$env_name" --no-prompt >/dev/null && - azd deploy "$c_name" --environment "$env_name" --no-prompt + python3 -c "import os, runpy, sys; sys.path.insert(0, os.environ['AGENTLZ_REPO_ROOT']); sys.argv = ['config.deployment.custom_application'] + sys.argv[1:]; runpy.run_module('config.deployment.custom_application', run_name='__main__')" --definition "$app_definition_path" --service "$c_name" --environment "$env_name" ) || { red "$c_name: hosted agent deployment failed."; had_errors=1; continue; } agent_smoke_payload="$(mktemp "${TMPDIR:-/tmp}/agentlz-agent-smoke-XXXXXX")" smoke_options=() @@ -481,7 +479,7 @@ while IFS= read -r dcomp; do red "$c_name: smoke test did not pass."; had_errors=1; continue fi else - (cd "$app_definition_dir" && azd deploy "$c_name" --environment "$env_name" --no-prompt) || { + (cd "$app_definition_dir" && python3 -c "import os, runpy, sys; sys.path.insert(0, os.environ['AGENTLZ_REPO_ROOT']); sys.argv = ['config.deployment.custom_application'] + sys.argv[1:]; runpy.run_module('config.deployment.custom_application', run_name='__main__')" --definition "$app_definition_path" --service "$c_name" --environment "$env_name") || { red "$c_name: Container App deployment failed."; had_errors=1; continue } fi diff --git a/tests/test_custom_application_deploy.py b/tests/test_custom_application_deploy.py new file mode 100644 index 00000000..8641647d --- /dev/null +++ b/tests/test_custom_application_deploy.py @@ -0,0 +1,157 @@ +"""Immutable custom deployment and private build regressions.""" + +import json +from pathlib import Path +import shutil +import subprocess +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import yaml + +from config.deployment import custom_application as deploy + +ROOT = Path(__file__).resolve().parents[1] +DIGEST = "sha256:" + "a" * 64 +COMMIT = "1" * 40 +ENV = { + "AZURE_ENV_NAME": "offline", + "NETWORK_ISOLATION": "true", + "AZURE_CONTAINER_REGISTRY_ENDPOINT": "private.azurecr.io", + "AZURE_RESOURCE_GROUP": "offline", + "ACR_TASK_AGENT_POOL": "build-pool", +} + + +def test_pinned_digest_replaces_mutable_tag_without_build(tmp_path): + component = {"source": {"imageDigest": DIGEST}} + service = {"image": "${REGISTRY}/application:v1"} + assert deploy.prepare_image(component, service, tmp_path, {"REGISTRY": "private.azurecr.io"}) == ( + f"private.azurecr.io/application@{DIGEST}" + ) + with pytest.raises(ValueError, match="qualified service image"): + deploy.prepare_image(component, {}, tmp_path, ENV) + with pytest.raises(ValueError, match="environment variable"): + deploy.prepare_image(component, service, tmp_path, {}) + + +def test_private_build_selects_pool_and_pins_management_plane_digest(tmp_path): + component = {"name": "web", "path": "src", "source": {"commit": COMMIT}} + service = {"project": "./src", "language": "docker", "docker": {"remoteBuild": True}} + with patch.object(deploy.subprocess, "run", return_value=SimpleNamespace(stdout=COMMIT)), \ + patch.object(deploy, "check_stage") as network, \ + patch.object(deploy, "validate_acr_agent_pool") as pool, \ + patch.object(deploy, "build_source_image", return_value=DIGEST) as build: + assert deploy.prepare_image(component, service, tmp_path, ENV) == ( + f"private.azurecr.io/agent-landing-zone/custom/web@{DIGEST}" + ) + assert pool.call_args.kwargs["agent_pool"] == "build-pool" + network.assert_called_once_with(ENV, "hosted-build") + assert build.call_args.kwargs["agent_pool"] == "build-pool" + assert build.call_args.kwargs["source_dir"] == (tmp_path / "src").resolve() + + +@pytest.mark.parametrize("change,match", [ + ({"ACR_TASK_AGENT_POOL": ""}, "ACR_TASK_AGENT_POOL"), + ({"AZURE_CONTAINER_REGISTRY_ENDPOINT": ""}, "registry endpoint"), +]) +def test_private_build_rejects_missing_inputs_without_build(tmp_path, change, match): + component = {"name": "web", "path": "src", "source": {"commit": COMMIT}} + service = {"project": "./src", "language": "docker"} + with patch.object(deploy.subprocess, "run", return_value=SimpleNamespace(stdout=COMMIT)), \ + patch.object(deploy, "build_source_image") as build: + with pytest.raises(ValueError, match=match): + deploy.prepare_image(component, service, tmp_path, {**ENV, **change}) + build.assert_not_called() + + +@pytest.mark.parametrize("image", [ + "application:latest", "https://private.azurecr.io/application", + "private.azurecr.io/application --build-arg=unsafe", "${REGISTRY}/application", +]) +def test_digest_pin_rejects_unqualified_or_invalid_images(tmp_path, image): + with pytest.raises(ValueError): + deploy.prepare_image({"source": {"imageDigest": DIGEST}}, {"image": image}, tmp_path, {}) + + +def test_public_source_preserves_azd_build_and_still_checks_commit(tmp_path): + component = {"name": "web", "path": "src", "source": {"commit": COMMIT}} + with patch.object(deploy.subprocess, "run", return_value=SimpleNamespace(stdout=COMMIT)), \ + patch.object(deploy, "build_source_image") as build: + assert deploy.prepare_image(component, {"language": "python"}, tmp_path, {}) is None + build.assert_not_called() + with patch.object(deploy.subprocess, "run", return_value=SimpleNamespace(stdout="2" * 40)): + with pytest.raises(ValueError, match="HEAD does not match"): + deploy.prepare_image(component, {}, tmp_path, {}) + + +@pytest.mark.parametrize("service,match", [ + ({"project": "src", "language": "python"}, "language: docker"), + ({"project": "another", "language": "docker"}, "project must match"), + ({"project": "src", "language": "docker", "docker": {"buildArgs": {"X": "value"}}}, + "cannot preserve these docker options"), + ({"project": "src", "language": "docker", "docker": {"path": "other.Dockerfile"}}, + "require Dockerfile"), + ({"project": "src", "language": "docker", "docker": {"context": ".."}}, + "require Dockerfile"), +]) +def test_private_build_rejects_unsupported_layout_and_options(tmp_path, service, match): + component = {"name": "web", "path": "src", "source": {"commit": COMMIT}} + with patch.object(deploy.subprocess, "run", return_value=SimpleNamespace(stdout=COMMIT)), \ + patch.object(deploy, "check_stage") as network, \ + patch.object(deploy, "build_source_image") as build: + with pytest.raises(ValueError, match=match): + deploy.prepare_image(component, service, tmp_path, ENV) + network.assert_not_called() + build.assert_not_called() + + +@pytest.mark.parametrize("boundary", ["check_stage", "validate_acr_agent_pool", "build_source_image"]) +def test_private_build_propagates_network_pool_and_build_failures(tmp_path, boundary): + component = {"name": "web", "path": "src", "source": {"commit": COMMIT}} + service = {"project": "src", "language": "docker"} + with patch.object(deploy.subprocess, "run", return_value=SimpleNamespace(stdout=COMMIT)), \ + patch.object(deploy, "check_stage"), \ + patch.object(deploy, "validate_acr_agent_pool"), \ + patch.object(deploy, "build_source_image", return_value=DIGEST), \ + patch.object(deploy, boundary, side_effect=RuntimeError("explicit boundary failure")): + with pytest.raises(RuntimeError, match="explicit boundary failure"): + deploy.prepare_image(component, service, tmp_path, ENV) + + +@pytest.mark.parametrize("failure", [False, True]) +def test_deployment_consumes_pin_and_restores_original_yaml(tmp_path, failure): + app = tmp_path / "sample" + shutil.copytree(ROOT / "samples" / "custom-app" / "containerapp", app) + definition_path = app / "app-definition.json" + definition = json.loads(definition_path.read_text()) + definition["components"][0]["source"] = {"imageDigest": DIGEST} + definition_path.write_text(json.dumps(definition)) + path = app / "azure.yaml" + project = yaml.safe_load(path.read_text()) + project["services"]["web"]["image"] = "private.azurecr.io/web:mutable" + path.write_text(yaml.safe_dump(project)) + original = path.read_bytes() + + def run(args, **kwargs): + rendered = yaml.safe_load(path.read_text()) + service = rendered["services"]["web"] + assert service["image"] == f"private.azurecr.io/web@{DIGEST}" + assert service["docker"] == {"imagePassthrough": True} + assert args == ["azd", "deploy", "web", "--environment", "offline", "--no-prompt"] + assert kwargs["cwd"] == app + assert all(kwargs["env"][key] == value for key, value in ENV.items()) + if failure: + raise subprocess.CalledProcessError(1, args) + return subprocess.CompletedProcess(args, 0) + + with patch.object(deploy.subprocess, "run", side_effect=run): + if failure: + with pytest.raises(subprocess.CalledProcessError): + assert deploy.deploy_service(str(definition_path), "web", ENV) == ( + f"private.azurecr.io/web@{DIGEST}" + ) + else: + deploy.deploy_service(str(definition_path), "web", ENV) + assert path.read_bytes() == original diff --git a/tests/test_custom_application_hooks.py b/tests/test_custom_application_hooks.py index 80c97f22..7695f0fc 100644 --- a/tests/test_custom_application_hooks.py +++ b/tests/test_custom_application_hooks.py @@ -81,6 +81,22 @@ def run_custom(self, shell: str, *, mismatch: bool = False, hosted: bool = False if smoke_failure: response["status"] = "failed" (root / "response").write_text(json.dumps(response), encoding="utf-8") + (root / "custom-deploy.py").write_text( + "import os, sys, subprocess\n" + "from pathlib import Path\n" + "from unittest.mock import patch\n" + "from config.deployment import custom_application\n" + "def run(args, **kwargs):\n" + " if args[0] == 'git':\n" + " return subprocess.CompletedProcess(args, 0, stdout=os.environ['FAKE_COMMIT'])\n" + " folder = Path(kwargs['cwd'])\n" + " assert (folder / 'azure.yaml').is_file()\n" + " assert (folder / '.azure' / 'config.json').is_file()\n" + " with (Path(os.environ['FAKE_ROOT']) / 'trace').open('a', encoding='utf-8') as f:\n" + " f.write(' '.join(args[1:]) + '\\n')\n" + " return subprocess.CompletedProcess(args, 0)\n" + "with patch.object(custom_application.subprocess, 'run', side_effect=run):\n" + " sys.exit(custom_application.main(sys.argv[1:]))\n", encoding="utf-8") env = dict(os.environ, FAKE_ROOT=str(root), FAKE_APP=str(app), FAKE_COMMIT=("2" if mismatch else "1") * 40, REAL_PYTHON=sys.executable, AGENTLZ_REPO_ROOT=str(ROOT), REAL_REPO=str(ROOT), PYTHONPATH=str(ROOT)) @@ -90,13 +106,18 @@ def run_custom(self, shell: str, *, mismatch: bool = False, hosted: bool = False prefix = r""" $ErrorActionPreference = 'Continue' $appDefinitionDir = $env:FAKE_APP +$appDefinitionPath = Join-Path $appDefinitionDir 'app-definition.json' $dotAzure = Join-Path $env:FAKE_ROOT '.azure' $definitionComponents = @((Get-Content (Join-Path $appDefinitionDir 'app-definition.json') -Raw | ConvertFrom-Json).components) $globalEnv = [pscustomobject]@{ AZURE_ENV_NAME = 'offline' } $hadErrors = $false function Get-ManifestComponentForDefinition { return $null } function git { $global:LASTEXITCODE = 0; return $env:FAKE_COMMIT } -function python { $input | & $env:REAL_PYTHON @args } +function python { + if ($args.Count -ge 2 -and "$($args[1])".Contains('config.deployment.custom_application')) { + & $env:REAL_PYTHON (Join-Path $env:FAKE_ROOT 'custom-deploy.py') @($args[2..($args.Count - 1)]) + } else { $input | & $env:REAL_PYTHON @args } +} function azd { if (-not (Test-Path 'azure.yaml') -or -not (Test-Path 'src')) { throw 'Wrong project folder' } if (-not (Test-Path '.azure/config.json')) { throw 'Missing environment' } @@ -126,7 +147,12 @@ def run_custom(self, shell: str, *, mismatch: bool = False, hosted: bool = False encoding="utf-8", newline="\n") shim.chmod(0o755) python_shim = fake_bin / "python3" - python_shim.write_text('#!/usr/bin/env bash\nexec "$REAL_PYTHON" "$@"\n', + python_shim.write_text( + '#!/usr/bin/env bash\n' + 'if [[ "$*" == *config.deployment.custom_application* ]]; then\n' + ' shift 2\n' + ' exec "$REAL_PYTHON" "$FAKE_ROOT/custom-deploy.py" "$@"\n' + 'fi\nexec "$REAL_PYTHON" "$@"\n', encoding="utf-8", newline="\n") python_shim.chmod(0o755) start = source.index('while IFS= read -r dcomp; do', source.index("# Components")) From c2a6866d7286413e95af4aded425cb2d0674b162 Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 17:26:11 -0300 Subject: [PATCH 17/18] test: record real private custom build and runtime evidence Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../quickstart.md | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/specs/002-agent-landing-zone-rebrand/quickstart.md b/specs/002-agent-landing-zone-rebrand/quickstart.md index b1371d8a..8dd3df03 100644 --- a/specs/002-agent-landing-zone-rebrand/quickstart.md +++ b/specs/002-agent-landing-zone-rebrand/quickstart.md @@ -396,6 +396,30 @@ close the outstanding runtime acceptance scenarios: - The pinned orchestrator still exposes the legacy `GPT-RAG Orchestrator` OpenAPI title. No branding acceptance or new component release is claimed. +The corrective shared custom-image flow in commit `a9214de` subsequently +built and deployed both samples using the checked-in service YAML rather than +a manually prepared image overlay. Each build used its existing private ACR +pool and returned an immutable image. Container digest: +`sha256:fb33898065127a48eab6e402cab4550b0eab72828899865f3d3e25771b43185a`; +hosted digest: +`sha256:ca213b38d82ae24ebbd01ba1f2597cec2c1b94a60f316cab3a684fc67a902243`. +The helper restored each original YAML; temporary helper staging was removed. +Evaluation definitions were temporarily pinned to verified guest source +`aae0a3c` and restored afterwards; missing metadata aliases still required +explicit evaluation hydration. This is corrective-flow evidence, not pristine +release or stock root-hook acceptance. + +The resulting Container revision `ca-ddzasx67ghev4-web--azd-1791577126` +was latest-ready. Its actual private response on `10.252.2.11` exactly +matched the live platform contract, including the real `web` client ID. +Hosted version 3 was active with the exact new build digest and preserved +runtime principal `6e447da6-8aca-4f96-af56-e863aa0da041`. Its real private +Responses invocation on `10.253.2.33` returned the exact project greeting. +The combined runtime-check marker passed with empty guest stderr. +No new runtime role grant was needed. The final correction suite passed +432 tests and 500 subtests, with 4 skips; central guidance `1512486` is in +Azure/AI-Landing-Zones#148. + T035, T085, T098, and T093 remain open until their actual runtime outcomes are captured. No network policy was disabled to obtain a passing result. From 0647cf1aff2b298d800bb5290c98656b74cfde8c Mon Sep 17 00:00:00 2001 From: Paulo Lacerda Date: Fri, 9 Oct 2026 17:49:52 -0300 Subject: [PATCH 18/18] fix: reconcile custom hosted profiles and foundation upgrade metadata Publish enabled foundation outputs, recover verified quota credit for older blank outputs, and reconcile declared hosted profiles against the routed runtime identity. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- config/appdefinition/roles.py | 43 +++++++++++++++++++++++-- config/deployment/custom_application.py | 15 +++++++++ config/deployment/hosted_access.py | 30 ++++++++++++++--- infra/main.bicep | 12 ++++--- infra/scripts/ModelQuota.ps1 | 23 ++++++++++++- tests/test_app_definition_roles.py | 31 ++++++++++++++++++ tests/test_custom_application_deploy.py | 39 ++++++++++++++++++++++ tests/test_custom_application_hooks.py | 5 ++- tests/test_custom_foundation_outputs.py | 27 ++++++++++++++++ tests/test_hosted_access.py | 12 +++++++ tests/test_regional_model_quota.py | 22 ++++++++++++- 11 files changed, 244 insertions(+), 15 deletions(-) create mode 100644 tests/test_custom_foundation_outputs.py diff --git a/config/appdefinition/roles.py b/config/appdefinition/roles.py index d9c60929..cea83a07 100644 --- a/config/appdefinition/roles.py +++ b/config/appdefinition/roles.py @@ -2,12 +2,14 @@ Runs in ``postProvision`` through ``python -m config.appdefinition --assign-roles``. For every ``containerapp`` component of the selected -definition (``azure.ai.agent`` identities are owned by -:mod:`config.deployment.hosted_access`), it expands the component profiles with +definition, it expands the component profiles with :func:`~config.appdefinition.profiles.expand_profiles`, resolves the component Container App identity by its ``azd-service-name`` tag, resolves each scope to the single matching resource in ``AZURE_RESOURCE_GROUP``, and creates only the assignments that do not already exist (directly or inherited). Re-runs converge. +Custom hosted components use the same profile reconciliation after deployment, +with an instance identity verified by :mod:`config.deployment.hosted_access`. +The bundled hosted orchestrator retains its feature-owned access bootstrap. Every Azure call goes through ``az`` (:func:`config.deployment.appconfig._run_az`) so unit tests replace one boundary. @@ -240,6 +242,41 @@ def assign_roles( ) -> list[PlannedAssignment]: """Create missing profile role assignments; return the ones created.""" planned = plan_assignments(definition, hosted_orchestration=hosted_orchestration) + return _assign_planned(planned, environment, run_az=run_az, role_guids=role_guids) + + +def assign_hosted_roles( + component: Mapping[str, Any], + principal: str, + environment: Mapping[str, str], + *, + run_az: RunAz | None = None, + role_guids: Mapping[str, str] | None = None, +) -> list[PlannedAssignment]: + """Apply the selected component profiles to its verified hosted identity.""" + from uuid import UUID + + if UUID(principal).int == 0: + raise RoleAssignmentError("Hosted runtime principal must be a nonzero GUID.") + name = str(component["name"]) + planned = [ + PlannedAssignment(name, name, assignment) + for assignment in expand_profiles(component.get("profiles") or ()) + ] + return _assign_planned( + planned, environment, run_az=run_az, role_guids=role_guids, + principals={name: principal}, + ) + + +def _assign_planned( + planned: list[PlannedAssignment], + environment: Mapping[str, str], + *, + run_az: RunAz | None, + role_guids: Mapping[str, str] | None, + principals: Mapping[str, str] | None = None, +) -> list[PlannedAssignment]: if not planned: return [] resource_group = (environment.get("AZURE_RESOURCE_GROUP") or "").strip() @@ -253,7 +290,7 @@ def assign_roles( raise RoleAssignmentError(f"Role {', '.join(unknown)} is not defined in {ROLES_RELATIVE_PATH.as_posix()}.") # Resolve every identity and scope before the first write so an ambiguous # or missing resource fails with no partial assignment. - principals = { + principals = principals if principals is not None else { service: resolve_principal(service, resource_group, run_az) for service in dict.fromkeys(item.service for item in planned) } diff --git a/config/deployment/custom_application.py b/config/deployment/custom_application.py index b0c9d8c4..6a5d57cb 100644 --- a/config/deployment/custom_application.py +++ b/config/deployment/custom_application.py @@ -15,6 +15,8 @@ import yaml from config.appdefinition import AppDefinitionError, load_definition, validate_definition +from config.appdefinition.loader import KIND_HOSTED_AGENT +from config.appdefinition.roles import assign_hosted_roles from config.deployment.composition import is_truthy from config.deployment.hosted_image import ( build_source_image, @@ -22,6 +24,7 @@ validate_digest, ) from config.deployment.private_network import check_stage +from config.deployment.hosted_access import discover_instance from util.azure_cli import resolve_az_command LOGGER = logging.getLogger("config.deployment.custom_application") @@ -130,6 +133,11 @@ def deploy_service(definition_path: str, name: str, environment: Mapping[str, st original = path.read_bytes() project = yaml.safe_load(original.decode("utf-8-sig")) service = project["services"][name] + agent_name = service.get("name") + if components[0]["kind"] == KIND_HOSTED_AGENT and ( + not isinstance(agent_name, str) or not agent_name.strip() + ): + raise ValueError("Custom hosted service requires an explicit agent name.") image = prepare_image(components[0], service, definition.folder, environment) try: if image: @@ -146,6 +154,13 @@ def deploy_service(definition_path: str, name: str, environment: Mapping[str, st finally: if image: path.write_bytes(original) + if components[0]["kind"] == KIND_HOSTED_AGENT: + instance = discover_instance(environment, name=agent_name) + assign_hosted_roles(components[0], instance.principal_id, environment) + LOGGER.info( + "Verified capability-profile assignments for hosted service %s version %s.", + name, instance.agent_version, + ) return image diff --git a/config/deployment/hosted_access.py b/config/deployment/hosted_access.py index dfcc2884..39fa03b1 100644 --- a/config/deployment/hosted_access.py +++ b/config/deployment/hosted_access.py @@ -287,10 +287,19 @@ def _account_metadata(run: RunAzure, scope: str, subscription: str, group: str) return account -def discover(environment: Mapping[str, str], *, run: RunAzure = run_az) -> AccessPlan: - """Discover and validate the entire closed plan before any write.""" - if resolve_mode(environment) is DeploymentMode.CLASSIC: - return AccessPlan(classic=True) +@dataclass(frozen=True) +class HostedInstance: + subscription: str + resource_group: str + principal_id: str + agent_name: str + agent_version: str + + +def discover_instance( + environment: Mapping[str, str], *, name: str | None = None, run: RunAzure = run_az, +) -> HostedInstance: + """Resolve a live routed identity bound to the declared Foundry project.""" sub = _guid(environment.get("AZURE_SUBSCRIPTION_ID")) group = _string(environment.get("AZURE_RESOURCE_GROUP")) if not re.fullmatch(NAME, group): @@ -303,7 +312,7 @@ def discover(environment: Mapping[str, str], *, run: RunAzure = run_az) -> Acces _bound_scope(project, PROJECT_PATH, sub, group) project_name = project.rsplit("/", 1)[1] endpoint = _endpoint(endpoint, ".services.ai.azure.com", rf"/api/projects/{re.escape(project_name)}/?") - name = _consistent(environment, ("HOSTED_AGENT_NAME",), agent.get("name", environment.get("HOSTED_AGENT_NAME") or HOSTED_AGENT_NAME_DEFAULT)) + name = name if name is not None else _consistent(environment, ("HOSTED_AGENT_NAME",), agent.get("name", environment.get("HOSTED_AGENT_NAME") or HOSTED_AGENT_NAME_DEFAULT)) if not re.fullmatch(NAME, name): raise AccessError("Invalid expected hosted agent name.") account_id = project.rsplit("/", 2)[0] @@ -343,7 +352,18 @@ def retrieve(path: str) -> dict: if deployed.get("name") != name or str(deployed.get("version")) != version or definition.get("kind") != "hosted": raise AccessError("Retrieved version is not the expected hosted agent.") # Protocol extensions cannot skip identity/name/version validation. + return HostedInstance(sub, group, principal, name, version) + +def discover(environment: Mapping[str, str], *, run: RunAzure = run_az) -> AccessPlan: + """Discover and validate the entire closed plan before any write.""" + if resolve_mode(environment) is DeploymentMode.CLASSIC: + return AccessPlan(classic=True) + instance = discover_instance(environment, run=run) + sub, group, principal, name, version = ( + instance.subscription, instance.resource_group, instance.principal_id, + instance.agent_name, instance.agent_version, + ) config_endpoint = _endpoint(environment.get("APP_CONFIG_ENDPOINT"), ".azconfig.io") store_name = urlsplit(config_endpoint).hostname.split(".")[0] prefix = f"/subscriptions/{sub}/resourceGroups/{group}/providers/" diff --git a/infra/main.bicep b/infra/main.bicep index 0be111d9..caeb152c 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -4081,10 +4081,14 @@ output CONTAINER_APP_INTERNAL_FQDN string = (_deployContainerApps && length(cont output DEPLOY_HOSTED_AGENT bool = deployHostedAgent @description('True when hosted-agent prerequisites were selected through prepareHostedAgent or deployHostedAgent. This does not indicate that a hosted agent version exists.') output HOSTED_AGENT_PREPARED bool = _hostedAgentPrerequisitesEnabled -output AZURE_AI_PROJECT_RESOURCE_ID string = _hostedAgentPrerequisitesEnabled ? aiFoundryProjectResourceId : '' -output AZURE_AI_PROJECT_ENDPOINT string = _hostedAgentPrerequisitesEnabled ? aiFoundryProjectEndpoint : '' -output AZURE_CONTAINER_REGISTRY_RESOURCE_ID string = _hostedAgentPrerequisitesEnabled ? _hostedAgentContainerRegistryResourceId : '' -output AZURE_CONTAINER_REGISTRY_ENDPOINT string = _hostedAgentPrerequisitesEnabled ? _hostedAgentContainerRegistryEndpoint : '' +@description('Provisioned Foundry project ID for application deployment, including custom applications in classic topology.') +output AZURE_AI_PROJECT_RESOURCE_ID string = (deployAiFoundry || _hostedAgentPrerequisitesEnabled) ? aiFoundryProjectResourceId : '' +@description('Provisioned Foundry project endpoint, independent of bundled hosted-agent selection.') +output AZURE_AI_PROJECT_ENDPOINT string = (deployAiFoundry || _hostedAgentPrerequisitesEnabled) ? aiFoundryProjectEndpoint : '' +@description('Provisioned registry ID, or the declared external hosted-agent registry when hosted prerequisites are selected.') +output AZURE_CONTAINER_REGISTRY_RESOURCE_ID string = (deployContainerRegistry || _hostedAgentPrerequisitesEnabled) ? _hostedAgentContainerRegistryResourceId : '' +@description('Provisioned registry login server for application builds and immutable image deployment.') +output AZURE_CONTAINER_REGISTRY_ENDPOINT string = (deployContainerRegistry || _hostedAgentPrerequisitesEnabled) ? _hostedAgentContainerRegistryEndpoint : '' #disable-next-line BCP318 output HOSTED_AGENT_DEPLOYMENT object = { enabled: deployHostedAgent diff --git a/infra/scripts/ModelQuota.ps1 b/infra/scripts/ModelQuota.ps1 index 6b8756fa..cef83b5f 100644 --- a/infra/scripts/ModelQuota.ps1 +++ b/infra/scripts/ModelQuota.ps1 @@ -9,7 +9,28 @@ function Get-ExistingModelDeployments { ) $projectId = $ProjectResourceId - if ([string]::IsNullOrWhiteSpace($projectId)) { return @() } + if ([string]::IsNullOrWhiteSpace($projectId)) { + if ([string]::IsNullOrWhiteSpace($ResourceGroup) -or [string]::IsNullOrWhiteSpace($SubscriptionId)) { + return @() + } + # Older classic/custom foundations emitted no project output. + $projects = & $ReadAzureJson @( + 'resource', 'list', '--resource-group', $ResourceGroup, + '--resource-type', 'Microsoft.CognitiveServices/accounts/projects', + '--subscription', $SubscriptionId, '--query', '[].{id:id}' + ) + if ($null -eq $projects -or ($projects -is [hashtable] -and $projects.failed)) { + & $OnWarning 'Could not discover an existing project; full requested allocation will be checked.' | Out-Null + return @() + } + $candidates = @($projects) + if ($candidates.Count -eq 0) { return @() } + if ($candidates.Count -ne 1) { + & $OnWarning 'Existing project discovery is ambiguous; no allocation credit applied. Set the selected project resource ID.' | Out-Null + return @() + } + $projectId = [string]$candidates[0].id + } $target = [regex]::Match($projectId, '^/subscriptions/([^/]+)/resourceGroups/([^/]+)/providers/Microsoft\.CognitiveServices/accounts/([^/]+)/projects/[^/]+/?$', 'IgnoreCase') if (-not $target.Success -or $target.Groups[1].Value -ne $SubscriptionId -or [string]::IsNullOrWhiteSpace($resourceGroup) -or $target.Groups[2].Value -ne $resourceGroup) { diff --git a/tests/test_app_definition_roles.py b/tests/test_app_definition_roles.py index 52c6acd6..565799f1 100644 --- a/tests/test_app_definition_roles.py +++ b/tests/test_app_definition_roles.py @@ -14,6 +14,7 @@ from config.appdefinition.roles import ( RoleAssignmentError, assign_roles, + assign_hosted_roles, discover_component_identities, load_role_guids, plan_assignments, @@ -85,6 +86,36 @@ def bundled(): class RoleAssignmentTests(unittest.TestCase): + def test_hosted_runtime_expands_profiles_and_converges_without_container_lookup(self): + az = FakeAz() + principal = "22222222-2222-4222-8222-222222222222" + component = {"name": "agent", "profiles": ["retrieval-reader", "conversation-store"]} + env = {"AZURE_RESOURCE_GROUP": RG} + created = assign_hosted_roles(component, principal, env, run_az=az) + self.assertEqual( + {item.role for item in expand_profiles(component["profiles"])}, + {item.assignment.role for item in created}, + ) + self.assertFalse(any(call[:2] == ["containerapp", "list"] for call in az.calls)) + self.assertTrue(all(principal in call for call in az.creates())) + az.calls.clear() + self.assertEqual([], assign_hosted_roles(component, principal, env, run_az=az)) + self.assertEqual([], az.creates()) + + def test_hosted_runtime_fails_before_writes_for_invalid_principal_or_ambiguous_scope(self): + az = FakeAz() + with self.assertRaises(ValueError): + assign_hosted_roles({"name": "agent"}, "not-a-guid", {"AZURE_RESOURCE_GROUP": RG}, run_az=az) + with self.assertRaises(RoleAssignmentError): + assign_hosted_roles({"name": "agent"}, "00000000-0000-0000-0000-000000000000", + {"AZURE_RESOURCE_GROUP": RG}, run_az=az) + vaults = [{"name": "kv-a", "id": f"{SUB}/kva"}, {"name": "kv-b", "id": f"{SUB}/kvb"}] + with patch.dict(RESOURCES, {"Microsoft.KeyVault/vaults": vaults}): + with self.assertRaisesRegex(RoleAssignmentError, "exactly one"): + assign_hosted_roles({"name": "agent"}, "22222222-2222-4222-8222-222222222222", + {"AZURE_RESOURCE_GROUP": RG}, run_az=az) + self.assertEqual([], az.creates()) + def test_discovers_client_ids_without_role_writes(self) -> None: calls = [] diff --git a/tests/test_custom_application_deploy.py b/tests/test_custom_application_deploy.py index 8641647d..fb187721 100644 --- a/tests/test_custom_application_deploy.py +++ b/tests/test_custom_application_deploy.py @@ -155,3 +155,42 @@ def run(args, **kwargs): else: deploy.deploy_service(str(definition_path), "web", ENV) assert path.read_bytes() == original + + +@pytest.mark.parametrize("failure", ["none", "deploy", "identity", "roles"]) +def test_hosted_deployment_reconciles_only_verified_runtime_profiles(tmp_path, failure): + app = tmp_path / "hosted" + shutil.copytree(ROOT / "samples" / "custom-app" / "hosted", app) + definition_path = app / "app-definition.json" + definition = json.loads(definition_path.read_text()) + definition["components"][0]["source"] = {"commit": COMMIT} + definition_path.write_text(json.dumps(definition)) + original = (app / "azure.yaml").read_bytes() + image = f"private.azurecr.io/agent@{DIGEST}" + instance = SimpleNamespace(principal_id="verified-principal", agent_version="7") + with patch.object(deploy, "prepare_image", return_value=image), \ + patch.object(deploy.subprocess, "run") as run, \ + patch.object(deploy, "discover_instance", return_value=instance) as discover, \ + patch.object(deploy, "assign_hosted_roles") as roles: + if failure == "deploy": + run.side_effect = subprocess.CalledProcessError(1, ["azd"]) + if failure == "identity": + discover.side_effect = RuntimeError("identity mismatch") + if failure == "roles": + roles.side_effect = RuntimeError("assignment denied") + if failure == "none": + assert deploy.deploy_service(str(definition_path), "agent", ENV) == image + else: + with pytest.raises((RuntimeError, subprocess.CalledProcessError)): + deploy.deploy_service(str(definition_path), "agent", ENV) + if failure == "deploy": + discover.assert_not_called() + else: + discover.assert_called_once_with(ENV, name="sample-hosted-agent") + if failure in {"deploy", "identity"}: + roles.assert_not_called() + else: + roles.assert_called_once_with( + definition["components"][0], instance.principal_id, ENV, + ) + assert (app / "azure.yaml").read_bytes() == original diff --git a/tests/test_custom_application_hooks.py b/tests/test_custom_application_hooks.py index 7695f0fc..14dbcd92 100644 --- a/tests/test_custom_application_hooks.py +++ b/tests/test_custom_application_hooks.py @@ -85,6 +85,7 @@ def run_custom(self, shell: str, *, mismatch: bool = False, hosted: bool = False "import os, sys, subprocess\n" "from pathlib import Path\n" "from unittest.mock import patch\n" + "from types import SimpleNamespace\n" "from config.deployment import custom_application\n" "def run(args, **kwargs):\n" " if args[0] == 'git':\n" @@ -95,7 +96,9 @@ def run_custom(self, shell: str, *, mismatch: bool = False, hosted: bool = False " with (Path(os.environ['FAKE_ROOT']) / 'trace').open('a', encoding='utf-8') as f:\n" " f.write(' '.join(args[1:]) + '\\n')\n" " return subprocess.CompletedProcess(args, 0)\n" - "with patch.object(custom_application.subprocess, 'run', side_effect=run):\n" + "with patch.object(custom_application.subprocess, 'run', side_effect=run), \\\n" + " patch.object(custom_application, 'discover_instance', return_value=SimpleNamespace(principal_id='verified', agent_version='1')), \\\n" + " patch.object(custom_application, 'assign_hosted_roles'):\n" " sys.exit(custom_application.main(sys.argv[1:]))\n", encoding="utf-8") env = dict(os.environ, FAKE_ROOT=str(root), FAKE_APP=str(app), FAKE_COMMIT=("2" if mismatch else "1") * 40, REAL_PYTHON=sys.executable, diff --git a/tests/test_custom_foundation_outputs.py b/tests/test_custom_foundation_outputs.py new file mode 100644 index 00000000..6c3e2480 --- /dev/null +++ b/tests/test_custom_foundation_outputs.py @@ -0,0 +1,27 @@ +"""Foundation endpoint handoff must not depend on bundled hosted selection.""" + +from pathlib import Path +import re + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.mark.parametrize("name,flag,value", [ + ("AZURE_AI_PROJECT_RESOURCE_ID", "deployAiFoundry", "aiFoundryProjectResourceId"), + ("AZURE_AI_PROJECT_ENDPOINT", "deployAiFoundry", "aiFoundryProjectEndpoint"), + ("AZURE_CONTAINER_REGISTRY_RESOURCE_ID", "deployContainerRegistry", + "_hostedAgentContainerRegistryResourceId"), + ("AZURE_CONTAINER_REGISTRY_ENDPOINT", "deployContainerRegistry", + "_hostedAgentContainerRegistryEndpoint"), +]) +def test_foundation_handoff_preserves_disabled_and_external_hosted_cases(name, flag, value): + text = (ROOT / "infra" / "main.bicep").read_text(encoding="utf-8") + expression = re.search(rf"^output {name} string = (.+)$", text, re.MULTILINE)[1] + assert expression == f"({flag} || _hostedAgentPrerequisitesEnabled) ? {value} : ''" + for provisioned, hosted, expected in ( + (False, False, ""), (True, False, value), + (False, True, value), (True, True, value), + ): + assert (value if provisioned or hosted else "") == expected diff --git a/tests/test_hosted_access.py b/tests/test_hosted_access.py index d4945684..e4c61c3a 100644 --- a/tests/test_hosted_access.py +++ b/tests/test_hosted_access.py @@ -168,6 +168,18 @@ def test_exact_three_roles_and_actual_instance(self): (access.Role.MODEL, PREFIX + "Microsoft.CognitiveServices/accounts/test-model"), (access.Role.SECRET, SECRET), ], [(grant.role, grant.scope) for grant in plan.grants]) + + def test_custom_instance_discovery_is_read_only_and_validates_actual_name(self): + self.azure.live["name"] = self.azure.version["name"] = "sample-hosted-agent" + instance = access.discover_instance( + {**ENV, "DEPLOYMENT_TOPOLOGY": "classic"}, name="sample-hosted-agent", run=self.azure, + ) + self.assertEqual(PRINCIPAL, instance.principal_id) + self.assertEqual("7", instance.agent_version) + self.assertFalse(any(call[:2] == ["appconfig", "kv"] for call in self.azure.calls)) + self.assertEqual([], self.azure.writes) + with self.assertRaisesRegex(access.AccessError, "name does not match"): + access.discover_instance(ENV, name="wrong-agent", run=self.azure) self.assertEqual([], self.azure.writes) self.assertTrue(all("secret" not in call[:2] for call in self.azure.calls)) diff --git a/tests/test_regional_model_quota.py b/tests/test_regional_model_quota.py index f0888cb2..08f1d3a1 100644 --- a/tests/test_regional_model_quota.py +++ b/tests/test_regional_model_quota.py @@ -47,6 +47,10 @@ if ($fixture.accountFailure) { return @{ failed = $true; error = 'fixture account denied' } } return $fixture.account } + if (($Arguments[0..1] -join ' ') -eq 'resource list') { + if ($fixture.PSObject.Properties['projects']) { return $fixture.projects } + return @() + } if (($Arguments[0..3] -join ' ') -eq 'cognitiveservices account deployment list') { if ($fixture.deploymentFailure) { return @{ failed = $true; error = 'fixture deployments denied' } } return $fixture.deployments @@ -153,9 +157,25 @@ def test_fresh_allocation_still_requires_full_capacity(self) -> None: data["usage"][0]["limit"] = 100 + available result, output = self.run_check(data, project="") self.assertEqual(failures, result["failures"], output) - self.assertEqual(2, len(result["calls"])) + self.assertEqual(3, len(result["calls"])) self.assertIn("verified existing credit 0, additional allocation 100", output) + def test_old_blank_output_recovers_only_unique_verified_project(self) -> None: + data = fixture() + data["projects"] = [{"id": ACCOUNT_ID + "/projects/test-project"}] + result, output = self.run_check(data, project="") + self.assertEqual(0, result["failures"], output) + self.assertIn("verified existing credit 100, additional allocation 0", output) + for projects in ( + [*data["projects"], {"id": ACCOUNT_ID + "/projects/other"}], + [{"id": ACCOUNT_ID.replace("test-group", "other-group") + "/projects/test-project"}], + ): + with self.subTest(projects=projects): + data["projects"] = projects + result, output = self.run_check(data, project="") + self.assertEqual(1, result["failures"], output) + self.assertIn("verified existing credit 0", output) + def test_only_increment_requires_available_quota(self) -> None: for capacity, failures in ((150, 0), (151, 1), (75, 0)): with self.subTest(capacity=capacity):