diff --git a/config/aifoundry/setup.py b/config/aifoundry/setup.py index 4924b80a..7dcabe3f 100644 --- a/config/aifoundry/setup.py +++ b/config/aifoundry/setup.py @@ -142,18 +142,30 @@ def add_ai_foundry_account_api_key_to_key_vault( account_name: str, vault_uri: str, secret_name: str -) -> None: - """Fetches the AI Foundry Account API key and stores it in Azure Key Vault.""" +) -> bool: + """Store an evaluation API key only when the account allows local authentication.""" try: + account = mgmt_client.accounts.get(resource_group, account_name) + if account.properties.disable_local_auth is True: + logging.info( + "Local authentication is disabled for account %s; " + "evaluation API-key injection is not applicable. Use Microsoft Entra ID.", + account_name, + ) + return False logging.info("🔑 Fetching AI Foundry Account API key for account %s ...", account_name) keys = mgmt_client.accounts.list_keys(resource_group, account_name) api_key = keys.key1 + if not api_key: + raise ValueError("The Foundry account returned an empty evaluation API key.") logging.info("🔒 Storing API key in Key Vault at %s ...", vault_uri) kv_client = KeyVaultClient(vault_uri) kv_client.set_secret(secret_name, api_key) logging.info("✅ Secret %s set successfully in Key Vault.", secret_name) + return True except Exception as e: logging.error("❗️ Failed to set secret in Key Vault: %s", e) + sys.exit(1) # ── Blocklist Logic ────────────────────────────────── def configure_blocklist(client: CognitiveServicesManagementClient, resource_group: str, account_name: str, bl_def: Dict[str, Any], bl_name: str) -> None: @@ -316,14 +328,16 @@ def main() -> None: # Store API key in Key Vault logging.info("🔑 Adding AI Foundry Account API Key to Key Vault …") key_vault_uri = cfg(app_conf, "KEY_VAULT_URI") - add_ai_foundry_account_api_key_to_key_vault( + secret_stored = add_ai_foundry_account_api_key_to_key_vault( client, resource_group, account_name, key_vault_uri, SECRET_NAME ) - logging.info("✅ RAI blocklist, policy, deployment association, and secret injection complete.") + logging.info("✅ RAI blocklist, policy, and deployment association complete.") + if secret_stored: + logging.info("✅ Evaluation API-key injection complete.") if __name__ == "__main__": main() diff --git a/config/appdefinition/roles.py b/config/appdefinition/roles.py index e21a9e9c..cea83a07 100644 --- a/config/appdefinition/roles.py +++ b/config/appdefinition/roles.py @@ -2,12 +2,14 @@ Runs in ``postProvision`` through ``python -m config.appdefinition --assign-roles``. For every ``containerapp`` component of the selected -definition (``azure.ai.agent`` identities are owned by -:mod:`config.deployment.hosted_access`), it expands the component profiles with +definition, it expands the component profiles with :func:`~config.appdefinition.profiles.expand_profiles`, resolves the component Container App identity by its ``azd-service-name`` tag, resolves each scope to the single matching resource in ``AZURE_RESOURCE_GROUP``, and creates only the assignments that do not already exist (directly or inherited). Re-runs converge. +Custom hosted components use the same profile reconciliation after deployment, +with an instance identity verified by :mod:`config.deployment.hosted_access`. +The bundled hosted orchestrator retains its feature-owned access bootstrap. Every Azure call goes through ``az`` (:func:`config.deployment.appconfig._run_az`) so unit tests replace one boundary. @@ -137,8 +139,7 @@ def resolve_scope(scope: str, environment: Mapping[str, str], resource_group: st return str(items[0]["id"]) -def resolve_principal(service: str, resource_group: str, run_az: RunAz) -> str: - """Return the single managed identity principal of the service's Container App.""" +def _resolve_identity(service: str, resource_group: str, run_az: RunAz) -> tuple[str, str, str]: apps = _json(run_az, ["containerapp", "list", "--resource-group", resource_group, "--query", f"[?tags.\"azd-service-name\"=='{service}'].{{name:name,identity:identity}}"]) apps = [app for app in apps or [] if isinstance(app, dict)] @@ -148,18 +149,60 @@ def resolve_principal(service: str, resource_group: str, run_az: RunAz) -> str: f"{resource_group}; found {len(apps)}. Run azd provision for this application first." ) identity = apps[0].get("identity") or {} - principals = [str(identity["principalId"])] if identity.get("principalId") else [] - principals += [ - str(value["principalId"]) - for value in (identity.get("userAssignedIdentities") or {}).values() + identities = [(str(identity["principalId"]), "", "")] if identity.get("principalId") else [] + identities += [ + (str(value["principalId"]), str(value.get("clientId") or ""), str(resource_id)) + for resource_id, value in (identity.get("userAssignedIdentities") or {}).items() if isinstance(value, dict) and value.get("principalId") ] - if len(principals) != 1: + if len(identities) != 1: raise RoleAssignmentError( f"Container App {apps[0].get('name')} must have exactly one managed identity to receive " - f"profile roles; found {len(principals)}." + f"profile roles; found {len(identities)}." ) - return principals[0] + return identities[0] + + +def resolve_principal(service: str, resource_group: str, run_az: RunAz) -> str: + """Return the single managed identity principal of the service's Container App.""" + return _resolve_identity(service, resource_group, run_az)[0] + + +def discover_component_identities( + definition: AppDefinition, + environment: Mapping[str, str], + *, + hosted_orchestration: bool, + run_az: RunAz | None = None, +) -> dict[str, str]: + """Read Container App client IDs for platform outputs, without role writes.""" + components = [ + component + for component in effective_components(definition, hosted_orchestration=hosted_orchestration) + if component.get("kind") == KIND_CONTAINER_APP + ] + if not components: + return {} + resource_group = (environment.get("AZURE_RESOURCE_GROUP") or "").strip() + if not resource_group: + raise RoleAssignmentError("AZURE_RESOURCE_GROUP is required to discover component identities.") + if run_az is None: + from config.deployment.appconfig import _run_az as run_az + identities: dict[str, str] = {} + for component in components: + service = component_service_name(definition, component) + principal, client, resource_id = _resolve_identity(service, resource_group, run_az) + if not client: + arguments = ( + ["identity", "show", "--ids", resource_id, "--query", "clientId"] + if resource_id else + ["ad", "sp", "show", "--id", principal, "--query", "appId"] + ) + client = _json(run_az, arguments) + if not isinstance(client, str) or not client.strip(): + raise RoleAssignmentError(f"Cannot resolve the managed identity client ID for service {service}.") + identities[str(component["name"])] = client.strip() + return identities def _ensure_rbac(principal: str, role_guid: str, scope_id: str, run_az: RunAz) -> bool: @@ -199,6 +242,41 @@ def assign_roles( ) -> list[PlannedAssignment]: """Create missing profile role assignments; return the ones created.""" planned = plan_assignments(definition, hosted_orchestration=hosted_orchestration) + return _assign_planned(planned, environment, run_az=run_az, role_guids=role_guids) + + +def assign_hosted_roles( + component: Mapping[str, Any], + principal: str, + environment: Mapping[str, str], + *, + run_az: RunAz | None = None, + role_guids: Mapping[str, str] | None = None, +) -> list[PlannedAssignment]: + """Apply the selected component profiles to its verified hosted identity.""" + from uuid import UUID + + if UUID(principal).int == 0: + raise RoleAssignmentError("Hosted runtime principal must be a nonzero GUID.") + name = str(component["name"]) + planned = [ + PlannedAssignment(name, name, assignment) + for assignment in expand_profiles(component.get("profiles") or ()) + ] + return _assign_planned( + planned, environment, run_az=run_az, role_guids=role_guids, + principals={name: principal}, + ) + + +def _assign_planned( + planned: list[PlannedAssignment], + environment: Mapping[str, str], + *, + run_az: RunAz | None, + role_guids: Mapping[str, str] | None, + principals: Mapping[str, str] | None = None, +) -> list[PlannedAssignment]: if not planned: return [] resource_group = (environment.get("AZURE_RESOURCE_GROUP") or "").strip() @@ -212,7 +290,7 @@ def assign_roles( raise RoleAssignmentError(f"Role {', '.join(unknown)} is not defined in {ROLES_RELATIVE_PATH.as_posix()}.") # Resolve every identity and scope before the first write so an ambiguous # or missing resource fails with no partial assignment. - principals = { + principals = principals if principals is not None else { service: resolve_principal(service, resource_group, run_az) for service in dict.fromkeys(item.service for item in planned) } @@ -232,4 +310,3 @@ def assign_roles( if changed: created.append(item) return created - diff --git a/config/deployment/custom_application.py b/config/deployment/custom_application.py new file mode 100644 index 00000000..6a5d57cb --- /dev/null +++ b/config/deployment/custom_application.py @@ -0,0 +1,183 @@ +"""Deploy one custom service with an immutable image and private ACR builds.""" + +from __future__ import annotations + +import argparse +import copy +import logging +import os +from pathlib import Path +import re +import subprocess +from typing import Any, Mapping +from uuid import uuid4 + +import yaml + +from config.appdefinition import AppDefinitionError, load_definition, validate_definition +from config.appdefinition.loader import KIND_HOSTED_AGENT +from config.appdefinition.roles import assign_hosted_roles +from config.deployment.composition import is_truthy +from config.deployment.hosted_image import ( + build_source_image, + validate_acr_agent_pool, + validate_digest, +) +from config.deployment.private_network import check_stage +from config.deployment.hosted_access import discover_instance +from util.azure_cli import resolve_az_command + +LOGGER = logging.getLogger("config.deployment.custom_application") + + +def _expand(value: str, environment: Mapping[str, str]) -> str: + def replace(match: re.Match[str]) -> str: + name = match[1] + result = (environment.get(name) or "").strip() + if not result: + raise ValueError(f"Image requires environment variable {name}.") + return result + + return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", replace, value) + + +def prepare_image( + component: Mapping[str, Any], + service: Mapping[str, Any], + folder: Path, + environment: Mapping[str, str], +) -> str | None: + """Keep normal azd builds, except pins and isolated Docker source builds.""" + source = component["source"] + pin = source.get("imageDigest") + isolated = is_truthy(environment.get("NETWORK_ISOLATION")) + if pin: + digest = validate_digest(pin, name="source.imageDigest") + image = _expand(str(service.get("image") or ""), environment) + if not re.fullmatch( + r"[a-z0-9.-]+(?::[0-9]+)?/[a-z0-9._/-]+(?::[A-Za-z0-9_.-]+|@sha256:[0-9a-f]{64})?", + image, + ): + raise ValueError("source.imageDigest requires a qualified service image in azure.yaml.") + repository = image.split("@", 1)[0] + last_slash = repository.rfind("/") + if ":" in repository[last_slash:]: + repository = repository[:repository.rfind(":")] + return f"{repository}@{digest}" + source_dir = (folder / component["path"]).resolve() + actual = subprocess.run( + ["git", "-C", str(source_dir), "rev-parse", "HEAD"], + check=True, capture_output=True, text=True, + ).stdout.strip() + if actual.lower() != source["commit"].lower(): + raise ValueError("Custom source HEAD does not match source.commit.") + if not isolated: + return None + if service.get("language") != "docker": + raise ValueError( + "Isolated custom source builds require language: docker or source.imageDigest." + ) + docker = service.get("docker") or {} + unsupported = set(docker) - {"remoteBuild", "path", "context"} + if unsupported: + raise ValueError( + "Isolated custom builds cannot preserve these docker options: " + + ", ".join(sorted(unsupported)) + + ". Supply a prebuilt source.imageDigest instead." + ) + project_dir = (folder / str(service.get("project") or ".")).resolve() + if source_dir != project_dir: + raise ValueError("The service project must match the pinned component path.") + if docker.get("path", "Dockerfile") != "Dockerfile" or docker.get("context", ".") != ".": + raise ValueError( + "Isolated custom builds require Dockerfile and context '.' in the component path; " + "supply a prebuilt source.imageDigest for another layout." + ) + registry_endpoint = ( + environment.get("AZURE_CONTAINER_REGISTRY_ENDPOINT") + or environment.get("CONTAINER_REGISTRY_LOGIN_SERVER") or "" + ).strip() + pool = (environment.get("ACR_TASK_AGENT_POOL") or "").strip() + group = (environment.get("AZURE_RESOURCE_GROUP") or "").strip() + if not registry_endpoint or not pool or not group: + raise ValueError( + "Isolated custom builds require the registry endpoint, " + "ACR_TASK_AGENT_POOL and AZURE_RESOURCE_GROUP." + ) + registry = registry_endpoint.split(".", 1)[0] + azure_cli = resolve_az_command() + check_stage(environment, "hosted-build") + validate_acr_agent_pool( + registry=registry, resource_group=group, agent_pool=pool, azure_cli=azure_cli, + ) + image_name = f"agent-landing-zone/custom/{component['name']}" + digest = build_source_image( + registry=registry, source_dir=source_dir, image_name=image_name, + image_tag=f"{actual[:12]}-{uuid4().hex[:12]}", + agent_pool=pool, azure_cli=azure_cli, + ) + return f"{registry_endpoint}/{image_name}@{digest}" + + +def deploy_service(definition_path: str, name: str, environment: Mapping[str, str]) -> str | None: + definition = validate_definition(load_definition(definition_path)) + if definition.bundled: + raise ValueError("Custom deployment cannot deploy a bundled component.") + components = [item for item in definition.document["components"] if item["name"] == name] + if len(components) != 1: + raise ValueError(f"No unique custom component named {name}.") + env_name = (environment.get("AZURE_ENV_NAME") or "").strip() + if not env_name: + raise ValueError("AZURE_ENV_NAME is required for custom deployment.") + path = definition.folder / "azure.yaml" + original = path.read_bytes() + project = yaml.safe_load(original.decode("utf-8-sig")) + service = project["services"][name] + agent_name = service.get("name") + if components[0]["kind"] == KIND_HOSTED_AGENT and ( + not isinstance(agent_name, str) or not agent_name.strip() + ): + raise ValueError("Custom hosted service requires an explicit agent name.") + image = prepare_image(components[0], service, definition.folder, environment) + try: + if image: + rendered = copy.deepcopy(project) + rendered_service = rendered["services"][name] + rendered_service["image"] = image + rendered_service["docker"] = {"imagePassthrough": True} + path.write_text(yaml.safe_dump(rendered, sort_keys=False), encoding="utf-8") + LOGGER.info("Deploying custom service %s with immutable image %s", name, image) + subprocess.run( + ["azd", "deploy", name, "--environment", env_name, "--no-prompt"], + cwd=definition.folder, env={**os.environ, **environment}, check=True, + ) + finally: + if image: + path.write_bytes(original) + if components[0]["kind"] == KIND_HOSTED_AGENT: + instance = discover_instance(environment, name=agent_name) + assign_hosted_roles(components[0], instance.principal_id, environment) + LOGGER.info( + "Verified capability-profile assignments for hosted service %s version %s.", + name, instance.agent_version, + ) + return image + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--definition", required=True) + parser.add_argument("--service", required=True) + parser.add_argument("--environment", required=True) + args = parser.parse_args(argv) + try: + deploy_service(args.definition, args.service, {**os.environ, "AZURE_ENV_NAME": args.environment}) + except (AppDefinitionError, ValueError, RuntimeError, OSError, subprocess.CalledProcessError, + yaml.YAMLError) as error: + LOGGER.error("Custom service deployment failed: %s", error) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/config/deployment/hosted.py b/config/deployment/hosted.py index 5a879d9e..0add830e 100644 --- a/config/deployment/hosted.py +++ b/config/deployment/hosted.py @@ -5,13 +5,54 @@ import argparse import json import logging +from pathlib import Path import sys from urllib.parse import urlsplit, urlunsplit +import yaml MAX_SMOKE_OUTPUT = 1024 * 1024 +def service_protocol(project: Path, service_name: str) -> str: + """Choose a supported greeting protocol from the selected service.""" + document = yaml.safe_load(project.read_text(encoding="utf-8")) + if not isinstance(document, dict) or not isinstance(document.get("services"), dict): + raise ValueError("Custom hosted project must declare services.") + service = document["services"][service_name] + if not isinstance(service, dict) or not isinstance(service.get("protocols"), list): + raise ValueError("Custom hosted service must declare protocols.") + protocols = { + entry["protocol"] for entry in service["protocols"] + if isinstance(entry, dict) and isinstance(entry.get("protocol"), str) + } + for protocol in ("invocations", "responses"): + if protocol in protocols: + return protocol + raise ValueError("Custom hosted service must declare invocations or responses for its greeting smoke.") + + +def validate_responses_smoke_output(output: str) -> None: + """Validate raw Responses HTTP JSON or SSE without accepting progress text.""" + if len(output) > MAX_SMOKE_OUTPUT: + raise ValueError("Smoke response exceeds the bounded validation size.") + body = output.replace("\r\n", "\n").strip() + if body.startswith("HTTP/"): + headers, separator, body = body.partition("\n\n") + status = headers.splitlines()[0].split() + if not separator or len(status) < 2 or not status[1].isdigit() or not 200 <= int(status[1]) < 300: + raise ValueError("Hosted greeting returned an unsuccessful HTTP response; body omitted.") + if body.lstrip().startswith("{"): + try: + response = json.loads(body) + except ValueError: + raise ValueError("Malformed Responses greeting; body omitted.") from None + if not isinstance(response, dict) or response.get("object") != "response": + raise ValueError("Invalid Responses greeting envelope; body omitted.") + body = json.dumps({"type": "response.completed", "response": response}) + validate_smoke_output(body) + + def _smoke_events(output: str) -> list[dict]: """Read Responses SSE or CLI JSON event lines, allowing azd progress text. @@ -136,15 +177,29 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser() mode = parser.add_mutually_exclusive_group(required=True) mode.add_argument("--invocations-endpoint") + mode.add_argument("--service-protocol", type=Path, metavar="AZURE_YAML") + parser.add_argument("--service-name") + mode.add_argument("--validate-responses-smoke", action="store_true") mode.add_argument( "--validate-smoke", action="store_true", help="Validate captured greeting Responses SSE/JSON events from stdin; no network calls.", ) args = parser.parse_args(argv) - if args.validate_smoke: + if args.service_protocol: + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + try: + if not args.service_name: + raise ValueError("--service-name is required.") + print(service_protocol(args.service_protocol, args.service_name)) + except (KeyError, TypeError, OSError, ValueError, yaml.YAMLError): + logging.error("Could not resolve the custom hosted service greeting protocol.") + return 1 + return 0 + if args.validate_smoke or args.validate_responses_smoke: logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") try: - validate_smoke_output(sys.stdin.read(MAX_SMOKE_OUTPUT + 1)) + validator = validate_responses_smoke_output if args.validate_responses_smoke else validate_smoke_output + validator(sys.stdin.read(MAX_SMOKE_OUTPUT + 1)) except ValueError as exc: logging.error("%s", exc) return 1 diff --git a/config/deployment/hosted_access.py b/config/deployment/hosted_access.py index dfcc2884..39fa03b1 100644 --- a/config/deployment/hosted_access.py +++ b/config/deployment/hosted_access.py @@ -287,10 +287,19 @@ def _account_metadata(run: RunAzure, scope: str, subscription: str, group: str) return account -def discover(environment: Mapping[str, str], *, run: RunAzure = run_az) -> AccessPlan: - """Discover and validate the entire closed plan before any write.""" - if resolve_mode(environment) is DeploymentMode.CLASSIC: - return AccessPlan(classic=True) +@dataclass(frozen=True) +class HostedInstance: + subscription: str + resource_group: str + principal_id: str + agent_name: str + agent_version: str + + +def discover_instance( + environment: Mapping[str, str], *, name: str | None = None, run: RunAzure = run_az, +) -> HostedInstance: + """Resolve a live routed identity bound to the declared Foundry project.""" sub = _guid(environment.get("AZURE_SUBSCRIPTION_ID")) group = _string(environment.get("AZURE_RESOURCE_GROUP")) if not re.fullmatch(NAME, group): @@ -303,7 +312,7 @@ def discover(environment: Mapping[str, str], *, run: RunAzure = run_az) -> Acces _bound_scope(project, PROJECT_PATH, sub, group) project_name = project.rsplit("/", 1)[1] endpoint = _endpoint(endpoint, ".services.ai.azure.com", rf"/api/projects/{re.escape(project_name)}/?") - name = _consistent(environment, ("HOSTED_AGENT_NAME",), agent.get("name", environment.get("HOSTED_AGENT_NAME") or HOSTED_AGENT_NAME_DEFAULT)) + name = name if name is not None else _consistent(environment, ("HOSTED_AGENT_NAME",), agent.get("name", environment.get("HOSTED_AGENT_NAME") or HOSTED_AGENT_NAME_DEFAULT)) if not re.fullmatch(NAME, name): raise AccessError("Invalid expected hosted agent name.") account_id = project.rsplit("/", 2)[0] @@ -343,7 +352,18 @@ def retrieve(path: str) -> dict: if deployed.get("name") != name or str(deployed.get("version")) != version or definition.get("kind") != "hosted": raise AccessError("Retrieved version is not the expected hosted agent.") # Protocol extensions cannot skip identity/name/version validation. + return HostedInstance(sub, group, principal, name, version) + +def discover(environment: Mapping[str, str], *, run: RunAzure = run_az) -> AccessPlan: + """Discover and validate the entire closed plan before any write.""" + if resolve_mode(environment) is DeploymentMode.CLASSIC: + return AccessPlan(classic=True) + instance = discover_instance(environment, run=run) + sub, group, principal, name, version = ( + instance.subscription, instance.resource_group, instance.principal_id, + instance.agent_name, instance.agent_version, + ) config_endpoint = _endpoint(environment.get("APP_CONFIG_ENDPOINT"), ".azconfig.io") store_name = urlsplit(config_endpoint).hostname.split(".")[0] prefix = f"/subscriptions/{sub}/resourceGroups/{group}/providers/" diff --git a/config/deployment/outputs.py b/config/deployment/outputs.py index 81cd224a..f83d3355 100644 --- a/config/deployment/outputs.py +++ b/config/deployment/outputs.py @@ -227,6 +227,10 @@ def publish(endpoint: str, settings: Mapping[str, str]) -> None: def main(argv: list[str] | None = None) -> int: + from config.appdefinition import AppDefinitionError, load_selected_definition, validate_definition + from config.appdefinition.roles import RoleAssignmentError, discover_component_identities + from config.deployment.composition import DeploymentMode, DeploymentTopologyError, resolve_mode + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument( "--check-foundation", @@ -243,9 +247,17 @@ def main(argv: list[str] | None = None) -> int: require_foundation(os.environ) if args.check_foundation: return 0 - document = build_platform_outputs(os.environ) + identities = None + if not (os.environ.get(IDENTITIES_ENV) or "").strip(): + definition = validate_definition(load_selected_definition(os.environ)) + identities = discover_component_identities( + definition, os.environ, + hosted_orchestration=resolve_mode(os.environ) is not DeploymentMode.CLASSIC, + ) + document = build_platform_outputs(os.environ, identities) validate_platform_outputs(document) - except (FoundationMissingError, PlatformOutputsError) as error: + except (FoundationMissingError, PlatformOutputsError, AppDefinitionError, + RoleAssignmentError, DeploymentTopologyError) as error: print(str(error), file=sys.stderr) return 1 settings = platform_settings(document) diff --git a/config/panel/setup.py b/config/panel/setup.py index 7a556597..f7f8c81a 100644 --- a/config/panel/setup.py +++ b/config/panel/setup.py @@ -343,7 +343,6 @@ def configure_panel_rbac(environment: Mapping[str, str]) -> int: resource_token = (environment.get("RESOURCE_TOKEN") or "").strip() for value, name in ( (resource_group, "AZURE_RESOURCE_GROUP"), - (account_name, "DATABASE_ACCOUNT_NAME"), (resource_token, "RESOURCE_TOKEN"), ): if not value: @@ -351,13 +350,33 @@ def configure_panel_rbac(environment: Mapping[str, str]) -> int: f"DEPLOY_ADMINISTRATIVE_PANEL is true but {name} is not set; " "cannot assign panel Cosmos RBAC." ) + database_name = (environment.get("DATABASE_NAME") or "").strip() + if not account_name and (environment.get("APP_CONFIG_ENDPOINT") or "").strip(): + endpoint = environment["APP_CONFIG_ENDPOINT"].strip() + for key in ("DATABASE_ACCOUNT_NAME", "DATABASE_NAME"): + value = _run_az([ + "appconfig", "kv", "show", "--endpoint", endpoint, + "--key", key, "--label", LABEL, "--auth-mode", "login", + "--query", "value", "--output", "tsv", + ]).strip() + if not value: + raise PanelRbacError(f"App Configuration {key} is empty; cannot assign panel Cosmos RBAC.") + if key == "DATABASE_ACCOUNT_NAME": + account_name = value + elif not database_name: + database_name = value + if not account_name: + raise PanelRbacError( + "DEPLOY_ADMINISTRATIVE_PANEL is true but DATABASE_ACCOUNT_NAME is not set; " + "cannot assign panel Cosmos RBAC." + ) subscription_id = resolve_subscription_id(environment) if not subscription_id: raise PanelRbacError( "DEPLOY_ADMINISTRATIVE_PANEL is true but AZURE_SUBSCRIPTION_ID " "could not be determined; cannot assign panel Cosmos RBAC." ) - database_name = (environment.get("DATABASE_NAME") or "").strip() or account_name + database_name = database_name or account_name app_names = { FRONTEND_APP_SERVICE_NAME: f"ca-{resource_token}-frontend", diff --git a/infra/main.bicep b/infra/main.bicep index 0be111d9..caeb152c 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -4081,10 +4081,14 @@ output CONTAINER_APP_INTERNAL_FQDN string = (_deployContainerApps && length(cont output DEPLOY_HOSTED_AGENT bool = deployHostedAgent @description('True when hosted-agent prerequisites were selected through prepareHostedAgent or deployHostedAgent. This does not indicate that a hosted agent version exists.') output HOSTED_AGENT_PREPARED bool = _hostedAgentPrerequisitesEnabled -output AZURE_AI_PROJECT_RESOURCE_ID string = _hostedAgentPrerequisitesEnabled ? aiFoundryProjectResourceId : '' -output AZURE_AI_PROJECT_ENDPOINT string = _hostedAgentPrerequisitesEnabled ? aiFoundryProjectEndpoint : '' -output AZURE_CONTAINER_REGISTRY_RESOURCE_ID string = _hostedAgentPrerequisitesEnabled ? _hostedAgentContainerRegistryResourceId : '' -output AZURE_CONTAINER_REGISTRY_ENDPOINT string = _hostedAgentPrerequisitesEnabled ? _hostedAgentContainerRegistryEndpoint : '' +@description('Provisioned Foundry project ID for application deployment, including custom applications in classic topology.') +output AZURE_AI_PROJECT_RESOURCE_ID string = (deployAiFoundry || _hostedAgentPrerequisitesEnabled) ? aiFoundryProjectResourceId : '' +@description('Provisioned Foundry project endpoint, independent of bundled hosted-agent selection.') +output AZURE_AI_PROJECT_ENDPOINT string = (deployAiFoundry || _hostedAgentPrerequisitesEnabled) ? aiFoundryProjectEndpoint : '' +@description('Provisioned registry ID, or the declared external hosted-agent registry when hosted prerequisites are selected.') +output AZURE_CONTAINER_REGISTRY_RESOURCE_ID string = (deployContainerRegistry || _hostedAgentPrerequisitesEnabled) ? _hostedAgentContainerRegistryResourceId : '' +@description('Provisioned registry login server for application builds and immutable image deployment.') +output AZURE_CONTAINER_REGISTRY_ENDPOINT string = (deployContainerRegistry || _hostedAgentPrerequisitesEnabled) ? _hostedAgentContainerRegistryEndpoint : '' #disable-next-line BCP318 output HOSTED_AGENT_DEPLOYMENT object = { enabled: deployHostedAgent diff --git a/infra/scripts/Invoke-PreflightChecks.ps1 b/infra/scripts/Invoke-PreflightChecks.ps1 index 19c0abce..821ae010 100644 --- a/infra/scripts/Invoke-PreflightChecks.ps1 +++ b/infra/scripts/Invoke-PreflightChecks.ps1 @@ -79,6 +79,7 @@ param( ) $ErrorActionPreference = 'Stop' +. (Join-Path $PSScriptRoot 'ModelQuota.ps1') # -------------------------------------------------------------------------- # Emergency bypass @@ -1187,7 +1188,7 @@ function Get-VmSkuInfo { [Parameter(Mandatory)] [string]$VmSize ) if ([string]::IsNullOrWhiteSpace($Location) -or [string]::IsNullOrWhiteSpace($VmSize)) { return $null } - $skus = Invoke-AzCliRaw -Arguments @('vm', 'list-skus', '--location', $Location, '--size', $VmSize, '--all', '-o', 'json') + $skus = Invoke-AzCliRaw -Arguments @('vm', 'list-skus', '--location', $Location, '--size', $VmSize, '--resource-type', 'virtualMachines', '--all', '-o', 'json') if (-not $skus) { return $null } $match = @($skus | Where-Object { $_.name -eq $VmSize -and $_.resourceType -eq 'virtualMachines' } | Select-Object -First 1) if (-not $match) { return $null } @@ -1379,7 +1380,10 @@ function Test-ModelQuota { $deployments = @($ModelDeployments | Where-Object { $_ -ne $null }) if ($deployments.Count -eq 0) { return } - $usage = Invoke-AzCliRaw -Arguments @('cognitiveservices', 'usage', 'list', '--location', $Location, '-o', 'json') + $selectedSubscription = if ($SubscriptionId) { $SubscriptionId } else { $env:AZURE_SUBSCRIPTION_ID } + $usageArguments = @('cognitiveservices', 'usage', 'list', '--location', $Location, '-o', 'json') + if ($selectedSubscription) { $usageArguments += @('--subscription', $selectedSubscription) } + $usage = Invoke-AzCliRaw -Arguments $usageArguments if (-not $usage) { Add-Finding -Severity WARN -Code 'MODEL_QUOTA_LOOKUP' ` -Message "Could not read Cognitive Services usage/quota for '$Location'." ` @@ -1387,28 +1391,38 @@ function Test-ModelQuota { return } + $deployments = @($deployments | Where-Object { $_.model.format -eq 'OpenAI' }) + if ($deployments.Count -eq 0) { return } + $selectedEnvironment = Get-AzdEnvValues + $existing = @(Get-ExistingModelDeployments -Location $Location -SubscriptionId $selectedSubscription ` + -ProjectResourceId $selectedEnvironment['AZURE_AI_PROJECT_RESOURCE_ID'] ` + -ResourceGroup $selectedEnvironment['AZURE_RESOURCE_GROUP'] -ReadAzureJson { + param([string[]]$Arguments) + Invoke-AzCliRaw -Arguments ($Arguments + @('-o', 'json')) + } -OnWarning { + param([string]$Message) + Add-Finding -Severity WARN -Code 'MODEL_QUOTA_EXISTING_TARGET' -Message $Message + }) + try { + $requests = @(Get-ModelQuotaRequirements -Models $deployments -ExistingDeployments $existing) + } catch { + Add-Finding -Severity FAIL -Code 'MODEL_QUOTA_INVALID' -Message $_.Exception.Message + return + } $failures = @() - foreach ($d in $deployments) { - # Only OpenAI-format deployments report quota via usage list - $fmt = $null - if ($d.PSObject.Properties.Name -contains 'model' -and $d.model) { - $fmt = $d.model.format - } - if ($fmt -ne 'OpenAI') { continue } - - $modelName = [string]$d.model.name - $skuName = [string]$d.sku.name - $capacity = [double]$d.sku.capacity - $quotaName = "OpenAI.$skuName.$modelName" - + foreach ($request in $requests) { + $quotaName = $request.QuotaName $quota = @($usage | Where-Object { $_.name.value -eq $quotaName } | Select-Object -First 1) if (-not $quota) { $failures += "No quota entry '$quotaName' in $Location." continue } - $available = [double]$quota.limit - [double]$quota.currentValue - if ($available -lt $capacity) { - $failures += "$quotaName needs $capacity, $available available (used $($quota.currentValue) / limit $($quota.limit))." + $available = [double]$quota[0].limit - [double]$quota[0].currentValue + $message = "$quotaName desired $($request.DesiredCapacity), verified existing credit $($request.ExistingCredit), additional allocation $($request.AdditionalCapacity); $available available." + if ($request.AdditionalCapacity -gt 0 -and $available -lt $request.AdditionalCapacity) { + $failures += $message + } else { + Add-Finding -Severity PASS -Code 'MODEL_QUOTA_SUFFICIENT' -Message $message } } diff --git a/infra/scripts/ModelQuota.ps1 b/infra/scripts/ModelQuota.ps1 new file mode 100644 index 00000000..cef83b5f --- /dev/null +++ b/infra/scripts/ModelQuota.ps1 @@ -0,0 +1,116 @@ +function Get-ExistingModelDeployments { + param( + [Parameter(Mandatory = $true)][string]$Location, + [AllowEmptyString()][string]$SubscriptionId, + [AllowEmptyString()][string]$ProjectResourceId = $env:AZURE_AI_PROJECT_RESOURCE_ID, + [AllowEmptyString()][string]$ResourceGroup = $env:AZURE_RESOURCE_GROUP, + [Parameter(Mandatory = $true)][scriptblock]$ReadAzureJson, + [Parameter(Mandatory = $true)][scriptblock]$OnWarning + ) + + $projectId = $ProjectResourceId + if ([string]::IsNullOrWhiteSpace($projectId)) { + if ([string]::IsNullOrWhiteSpace($ResourceGroup) -or [string]::IsNullOrWhiteSpace($SubscriptionId)) { + return @() + } + # Older classic/custom foundations emitted no project output. + $projects = & $ReadAzureJson @( + 'resource', 'list', '--resource-group', $ResourceGroup, + '--resource-type', 'Microsoft.CognitiveServices/accounts/projects', + '--subscription', $SubscriptionId, '--query', '[].{id:id}' + ) + if ($null -eq $projects -or ($projects -is [hashtable] -and $projects.failed)) { + & $OnWarning 'Could not discover an existing project; full requested allocation will be checked.' | Out-Null + return @() + } + $candidates = @($projects) + if ($candidates.Count -eq 0) { return @() } + if ($candidates.Count -ne 1) { + & $OnWarning 'Existing project discovery is ambiguous; no allocation credit applied. Set the selected project resource ID.' | Out-Null + return @() + } + $projectId = [string]$candidates[0].id + } + $target = [regex]::Match($projectId, '^/subscriptions/([^/]+)/resourceGroups/([^/]+)/providers/Microsoft\.CognitiveServices/accounts/([^/]+)/projects/[^/]+/?$', 'IgnoreCase') + if (-not $target.Success -or $target.Groups[1].Value -ne $SubscriptionId -or + [string]::IsNullOrWhiteSpace($resourceGroup) -or $target.Groups[2].Value -ne $resourceGroup) { + & $OnWarning 'Existing project output does not match the selected subscription/resource group; no allocation credit applied.' | Out-Null + return @() + } + + $accountName = $target.Groups[3].Value + $expectedId = $projectId -replace '/projects/[^/]+/?$', '' + $account = & $ReadAzureJson @('resource', 'show', '--ids', $expectedId, '--subscription', $SubscriptionId, '--query', '{id:id,location:location}') + if ($null -eq $account -or ($account -is [hashtable] -and $account.failed)) { + & $OnWarning 'Could not verify the existing account; full requested allocation will be checked.' | Out-Null + return @() + } + $actualLocation = ([string]$account.location -replace '\s+', '').ToLowerInvariant() + $desiredLocation = ($Location -replace '\s+', '').ToLowerInvariant() + if ($account.id -ne $expectedId -or $actualLocation -ne $desiredLocation) { + & $OnWarning 'Account identity/location does not match the target; no allocation credit applied.' | Out-Null + return @() + } + $deployments = & $ReadAzureJson @('cognitiveservices', 'account', 'deployment', 'list', '--name', $accountName, '--resource-group', $resourceGroup, '--subscription', $SubscriptionId) + if ($deployments -is [hashtable] -and $deployments.failed) { + & $OnWarning 'Could not verify existing deployments; full requested allocation will be checked.' | Out-Null + return @() + } + return @($deployments) +} + +function Get-ModelQuotaRequirements { + param( + [Parameter(Mandatory = $true)]$Models, + [AllowEmptyCollection()][object[]]$ExistingDeployments = @() + ) + + $requests = @{} + $seenNames = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::OrdinalIgnoreCase) + foreach ($deployment in @($Models)) { + $model = $deployment.model + $capacity = [double]$deployment.sku.capacity + if ($capacity -le 0 -or [double]::IsNaN($capacity) -or [double]::IsInfinity($capacity)) { + throw 'Deployment capacity must be positive and finite.' + } + $nameProperty = $deployment.PSObject.Properties['name'] + $deploymentName = if ($null -ne $nameProperty) { [string]$nameProperty.Value } else { '' } + if (-not [string]::IsNullOrWhiteSpace($deploymentName) -and -not $seenNames.Add($deploymentName)) { + throw "Duplicate deployment name '$deploymentName'; allocation credit cannot be reused." + } + $quotaName = "OpenAI.$($deployment.sku.name).$($model.name)" + $credit = 0.0 + if (-not [string]::IsNullOrWhiteSpace($deploymentName)) { + $matching = @($ExistingDeployments | Where-Object { $null -ne $_ -and $_.name -eq $deploymentName }) + if ($matching.Count -eq 1) { + $current = $matching[0] + $desiredFormat = $model.PSObject.Properties['format'] + $currentModel = $current.properties.model + $currentFormat = $currentModel.PSObject.Properties['format'] + if ($null -ne $desiredFormat -and $null -ne $currentFormat -and + $desiredFormat.Value -eq $currentFormat.Value -and + $currentModel.name -eq $model.name -and $currentModel.version -eq $model.version -and + $current.sku.name -eq $deployment.sku.name -and + $current.properties.provisioningState -eq 'Succeeded') { + $currentCapacity = [double]$current.sku.capacity + if ($currentCapacity -gt 0 -and -not [double]::IsNaN($currentCapacity) -and -not [double]::IsInfinity($currentCapacity)) { + $credit = [Math]::Min($capacity, $currentCapacity) + } + } + } + } + if (-not $requests.ContainsKey($quotaName)) { + $requests[$quotaName] = @{ desired = 0.0; credit = 0.0 } + } + $requests[$quotaName].desired += $capacity + $requests[$quotaName].credit += $credit + } + foreach ($quotaName in $requests.Keys) { + [pscustomobject]@{ + QuotaName = $quotaName + DesiredCapacity = $requests[$quotaName].desired + ExistingCredit = $requests[$quotaName].credit + AdditionalCapacity = $requests[$quotaName].desired - $requests[$quotaName].credit + } + } +} diff --git a/infra/tests/scripts/Invoke-PreflightChecks.Tests.ps1 b/infra/tests/scripts/Invoke-PreflightChecks.Tests.ps1 index 10f40a78..668c8ba5 100644 --- a/infra/tests/scripts/Invoke-PreflightChecks.Tests.ps1 +++ b/infra/tests/scripts/Invoke-PreflightChecks.Tests.ps1 @@ -25,6 +25,7 @@ $ErrorActionPreference = 'Stop' $scriptPath = Join-Path -Path (Split-Path -Parent $PSScriptRoot) -ChildPath '..\scripts\Invoke-PreflightChecks.ps1' $scriptPath = (Resolve-Path -Path $scriptPath).Path +. (Join-Path (Split-Path -Parent $scriptPath) 'ModelQuota.ps1') $raw = Get-Content -Path $scriptPath -Raw $ast = [System.Management.Automation.Language.Parser]::ParseInput($raw, [ref]$null, [ref]$null) @@ -128,6 +129,33 @@ finally { Remove-Item -Path $paramsFile -ErrorAction SilentlyContinue } +# -------------------------------------------------------------------------- +Write-Host 'VM SKU lookup is restricted to virtual machines' -ForegroundColor Cyan +$originalAzCliRaw = ${function:Invoke-AzCliRaw} +try { + function Invoke-AzCliRaw { + param([string[]]$Arguments) + $script:VmSkuArguments = $Arguments + return @([pscustomobject]@{ + name = 'Standard_D2s_v3' + resourceType = 'virtualMachines' + capabilities = @([pscustomobject]@{ name = 'vCPUs'; value = '2' }) + restrictions = @() + family = 'standardDSv3Family' + }) + } + $sku = Get-VmSkuInfo -Location 'westus3' -VmSize 'Standard_D2s_v3' + $typeIndex = [Array]::IndexOf($script:VmSkuArguments, '--resource-type') + Assert-True 'SKU request filters virtualMachines server-side' ( + $typeIndex -ge 0 -and $script:VmSkuArguments[$typeIndex + 1] -eq 'virtualMachines' + ) + Assert-True 'Filtered SKU keeps vCPU and availability facts' ( + $sku.Name -eq 'Standard_D2s_v3' -and $sku.VCpus -eq 2 + ) +} finally { + ${function:Invoke-AzCliRaw} = $originalAzCliRaw +} + # -------------------------------------------------------------------------- Write-Host 'Boolean parameter values accept only true and false' -ForegroundColor Cyan foreach ($validValue in @($true, $false, 'true', ' FALSE ')) { diff --git a/samples/custom-app/README.md b/samples/custom-app/README.md index d4b7c251..48e3837d 100644 --- a/samples/custom-app/README.md +++ b/samples/custom-app/README.md @@ -23,5 +23,5 @@ Both read the platform outputs contract (`AGENTLZ_PLATFORM_OUTPUTS`, label hosts exactly one application. Rules: no lifecycle hooks, no remote URLs, no role names; permissions come only -from capability profiles. See "Build your own application" in the Agent -Landing Zone section of the AI Landing Zones documentation for the full guide. +from capability profiles. See [Custom applications](https://azure.github.io/AI-Landing-Zones/agent-landing-zone/build-your-own-app/) +for the folder layout, source pinning, and hosted greeting protocol requirements. diff --git a/samples/custom-app/hosted/src/main.py b/samples/custom-app/hosted/src/main.py index 95b59cc0..387f7452 100644 --- a/samples/custom-app/hosted/src/main.py +++ b/samples/custom-app/hosted/src/main.py @@ -72,6 +72,7 @@ def do_POST(self) -> None: # noqa: N802 - http.server API { "type": "message", "role": "assistant", + "status": "completed", "content": [{"type": "output_text", "text": answer()}], } ], diff --git a/scripts/Invoke-RegionalPreflight.ps1 b/scripts/Invoke-RegionalPreflight.ps1 index 8c118da3..d81c2f18 100644 --- a/scripts/Invoke-RegionalPreflight.ps1 +++ b/scripts/Invoke-RegionalPreflight.ps1 @@ -32,6 +32,7 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +. (Join-Path (Join-Path (Join-Path (Split-Path -Parent $PSScriptRoot) 'infra') 'scripts') 'ModelQuota.ps1') $script:FailureCount = 0 $script:WarningCount = 0 @@ -443,14 +444,19 @@ function Test-ModelReadiness { $usageList = @() } + $existing = @(Get-ExistingModelDeployments -Location $Location -SubscriptionId $SubscriptionId -ReadAzureJson { + param([string[]]$Arguments) + Invoke-AzJson -Arguments $Arguments -AllowFailure + } -OnWarning { + param([string]$Message) + Write-PreflightCheck -Status WARN -Name 'models:existing-target' -Message $Message + }) foreach ($deployment in @($Models)) { $model = $deployment.model $modelName = $model.name $modelVersion = $model.version $skuName = $deployment.sku.name - $capacity = [double]$deployment.sku.capacity $checkName = "model:$modelName" - $available = @($modelList | Where-Object { $_.model.name -eq $modelName -and ([string]::IsNullOrWhiteSpace($modelVersion) -or $_.model.version -eq $modelVersion) -and @@ -463,18 +469,31 @@ function Test-ModelReadiness { Write-PreflightCheck -Status FAIL -Name $checkName -Message "$modelName $modelVersion with sku $skuName is not listed in $Location." } - $usageName = "OpenAI.$skuName.$modelName" + } + + try { + $requests = @(Get-ModelQuotaRequirements -Models $Models -ExistingDeployments $existing) + } catch { + Write-PreflightCheck -Status FAIL -Name 'models:quota' -Message $_.Exception.Message + return + } + foreach ($request in $requests) { + $usageName = $request.QuotaName $quota = @($usageList | Where-Object { $_.name.value -eq $usageName } | Select-Object -First 1) if ($quota.Count -eq 0) { - Write-PreflightCheck -Status WARN -Name "${checkName}:quota" -Message "quota item $usageName was not returned." + Write-PreflightCheck -Status WARN -Name "models:quota:$usageName" -Message "quota item $usageName was not returned." continue } $remaining = [double]$quota[0].limit - [double]$quota[0].currentValue - if ($remaining -ge $capacity) { - Write-PreflightCheck -Status PASS -Name "${checkName}:quota" -Message "$remaining quota units remaining; deployment requests $capacity." + $requested = $request.DesiredCapacity + $credit = $request.ExistingCredit + $additional = $request.AdditionalCapacity + $message = "$remaining quota units remaining; desired $requested, verified existing credit $credit, additional allocation $additional." + if ($additional -eq 0 -or $remaining -ge $additional) { + Write-PreflightCheck -Status PASS -Name "models:quota:$usageName" -Message $message } else { - Write-PreflightCheck -Status FAIL -Name "${checkName}:quota" -Message "$remaining quota units remaining; deployment requests $capacity." + Write-PreflightCheck -Status FAIL -Name "models:quota:$usageName" -Message $message } } } diff --git a/scripts/preDeploy.ps1 b/scripts/preDeploy.ps1 index e24d280e..31503316 100644 --- a/scripts/preDeploy.ps1 +++ b/scripts/preDeploy.ps1 @@ -126,6 +126,7 @@ if ($foundationMissing) { # Re-validate the application definition and compare it with the binding. $appDefinitionPath = if ($globalEnv.AGENTLZ_APP_DEFINITION) { "$($globalEnv.AGENTLZ_APP_DEFINITION)" } elseif ($env:AGENTLZ_APP_DEFINITION) { $env:AGENTLZ_APP_DEFINITION } else { Join-Path $repoRoot 'app-definition.json' } if (-not [IO.Path]::IsPathRooted($appDefinitionPath)) { $appDefinitionPath = Join-Path $repoRoot $appDefinitionPath } +if (Test-Path -LiteralPath $appDefinitionPath -PathType Container) { $appDefinitionPath = Join-Path $appDefinitionPath 'app-definition.json' } if (-not (Test-Path -LiteralPath $appDefinitionPath -PathType Leaf)) { Write-Error "AGENTLZ_APP_DEFINITION points to $appDefinitionPath, which does not exist." exit 1 @@ -478,13 +479,13 @@ foreach ($c in $manifest.components) { } # Components from the application definition that are not release-managed -# (manifest.json) components: each folder is its own azd project. +# (manifest.json) components share the definition folder's azd project. foreach ($dc in $definitionComponents) { $componentName = "$($dc.name)" if (Get-ManifestComponentForDefinition $componentName) { continue } $componentPath = Join-Path $appDefinitionDir "$($dc.path)" - if (-not (Test-Path -LiteralPath (Join-Path $componentPath 'azure.yaml') -PathType Leaf)) { - Write-Error "The app folder $componentPath must contain its own azure.yaml." + if (-not (Test-Path -LiteralPath (Join-Path $appDefinitionDir 'azure.yaml') -PathType Leaf)) { + Write-Error "The app folder $appDefinitionDir must contain its own azure.yaml." $hadErrors = $true continue } @@ -499,38 +500,41 @@ foreach ($dc in $definitionComponents) { } } if (Test-Path -LiteralPath $dotAzure) { - Copy-Item $dotAzure $componentPath -Recurse -Force -Container + Copy-Item $dotAzure $appDefinitionDir -Recurse -Force -Container } Write-Host ("Deploying {0} ({1}) from {2}" -f $componentName, $dc.kind, $componentPath) -ForegroundColor Cyan - Push-Location $componentPath + Push-Location $appDefinitionDir try { if ($sourceDigest) { & azd env set AGENTLZ_IMAGE_DIGEST $sourceDigest --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt | Out-Null if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: failed to pin image digest."; $hadErrors = $true; continue } } if ("$($dc.kind)" -eq 'azure.ai.agent') { - # prepareHostedDeployment is not run here: it builds only the manifest-pinned - # bundled orchestrator image (HOSTED_AGENT_*). A custom azure.ai.agent - # component deploys from its own child azd project, which builds its image, - # or pins source.imageDigest through AGENTLZ_IMAGE_DIGEST (T079 limitation). + $smokeProtocol = (& python -c "import os, runpy, sys; sys.path.insert(0, os.environ['AGENTLZ_REPO_ROOT']); sys.argv = ['config.deployment.hosted'] + sys.argv[1:]; runpy.run_module('config.deployment.hosted', run_name='__main__')" --service-protocol (Join-Path $appDefinitionDir 'azure.yaml') --service-name $componentName | Out-String).Trim() + if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: cannot resolve the greeting protocol."; $hadErrors = $true; continue } + # Custom images use the shared immutable/private build path, not the + # manifest-pinned orchestrator's derived hosted entrypoint. & azd env set FOUNDRY_PROJECT_ENDPOINT "$($globalEnv.AZURE_AI_PROJECT_ENDPOINT)" --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt | Out-Null if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: failed to set the Foundry endpoint."; $hadErrors = $true; continue } & azd env set AZURE_AI_PROJECT_ID "$($globalEnv.AZURE_AI_PROJECT_RESOURCE_ID)" --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt | Out-Null if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: failed to set the Foundry project ID."; $hadErrors = $true; continue } - & azd deploy $componentName --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt + & python -c "import os, runpy, sys; sys.path.insert(0, os.environ['AGENTLZ_REPO_ROOT']); sys.argv = ['config.deployment.custom_application'] + sys.argv[1:]; runpy.run_module('config.deployment.custom_application', run_name='__main__')" --definition $appDefinitionPath --service $componentName --environment "$($globalEnv.AZURE_ENV_NAME)" if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: hosted agent deployment failed."; $hadErrors = $true; continue } $smokePayloadPath = Join-Path ([IO.Path]::GetTempPath()) "agentlz-agent-smoke-$([guid]::NewGuid().ToString('N')).json" try { - [IO.File]::WriteAllText($smokePayloadPath, '{"messages":[{"role":"user","content":"Hello!"}]}', [Text.UTF8Encoding]::new($false)) - $smokeOutput = (& azd ai agent invoke --protocol invocations --new-session --timeout 180 --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt --input-file $smokePayloadPath 2>&1 | Out-String) + $smokePayload = if ($smokeProtocol -eq 'responses') { '{"input":[{"role":"user","content":"Hello!"}],"stream":true}' } else { '{"messages":[{"role":"user","content":"Hello!"}]}' } + $smokeOptions = if ($smokeProtocol -eq 'responses') { @('--output', 'raw') } else { @() } + $smokeValidation = if ($smokeProtocol -eq 'responses') { '--validate-responses-smoke' } else { '--validate-smoke' } + [IO.File]::WriteAllText($smokePayloadPath, $smokePayload, [Text.UTF8Encoding]::new($false)) + $smokeOutput = (& azd ai agent invoke $componentName --protocol $smokeProtocol --new-session --timeout 180 --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt --input-file $smokePayloadPath @smokeOptions 2>&1 | Out-String) if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: smoke request failed."; $hadErrors = $true; continue } - $smokeOutput | & python -c "import os, runpy, sys; sys.path.insert(0, os.environ['AGENTLZ_REPO_ROOT']); sys.argv = ['config.deployment.hosted'] + sys.argv[1:]; runpy.run_module('config.deployment.hosted', run_name='__main__')" --validate-smoke + $smokeOutput | & python -c "import os, runpy, sys; sys.path.insert(0, os.environ['AGENTLZ_REPO_ROOT']); sys.argv = ['config.deployment.hosted'] + sys.argv[1:]; runpy.run_module('config.deployment.hosted', run_name='__main__')" $smokeValidation if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: smoke test did not pass."; $hadErrors = $true; continue } } finally { Remove-Item -LiteralPath $smokePayloadPath -Force -ErrorAction SilentlyContinue } } else { - & azd deploy --all --environment "$($globalEnv.AZURE_ENV_NAME)" --no-prompt + & python -c "import os, runpy, sys; sys.path.insert(0, os.environ['AGENTLZ_REPO_ROOT']); sys.argv = ['config.deployment.custom_application'] + sys.argv[1:]; runpy.run_module('config.deployment.custom_application', run_name='__main__')" --definition $appDefinitionPath --service $componentName --environment "$($globalEnv.AZURE_ENV_NAME)" if ($LASTEXITCODE -ne 0) { Write-Error "${componentName}: Container App deployment failed."; $hadErrors = $true; continue } } Write-Host "${componentName}: deployed." -ForegroundColor Green diff --git a/scripts/preDeploy.sh b/scripts/preDeploy.sh index 10ff1801..fce5c9ac 100755 --- a/scripts/preDeploy.sh +++ b/scripts/preDeploy.sh @@ -74,7 +74,7 @@ copy_dot_azure() { if command -v rsync >/dev/null 2>&1; then rsync -a "$1/" "$2/.azure/" >/dev/null 2>&1 || cp -R "$1" "$2"/ >/dev/null 2>&1 else - cp -R "$1" "$2"/ >/dev/null 2>&1 || true + cp -R "$1" "$2"/ >/dev/null 2>&1 fi } @@ -138,6 +138,9 @@ case "$app_definition_path" in /*) ;; *) app_definition_path="$repo_root/$app_definition_path" ;; esac +if [ -d "$app_definition_path" ]; then + app_definition_path="$app_definition_path/app-definition.json" +fi [ -f "$app_definition_path" ] || { red "AGENTLZ_APP_DEFINITION points to $app_definition_path, which does not exist."; exit 1; } command -v python3 >/dev/null 2>&1 || { red "python3 is required to validate the application definition."; exit 1; } if [ -z "${AGENTLZ_APP_ID:-}" ]; then @@ -424,8 +427,8 @@ while IFS= read -r dcomp; do c_path="$app_definition_dir/$(printf "%s" "$dcomp" | jq -r '.path')" c_commit="$(printf "%s" "$dcomp" | jq -r '.source.commit // empty')" c_digest="$(printf "%s" "$dcomp" | jq -r '.source.imageDigest // empty')" - if [ ! -f "$c_path/azure.yaml" ]; then - red "The app folder $c_path must contain its own azure.yaml." + if [ ! -f "$app_definition_dir/azure.yaml" ]; then + red "The app folder $app_definition_dir must contain its own azure.yaml." had_errors=1; continue fi if [ -n "$c_commit" ]; then @@ -435,37 +438,48 @@ while IFS= read -r dcomp; do had_errors=1; continue fi fi - copy_dot_azure "$dot_azure" "$c_path" + copy_dot_azure "$dot_azure" "$app_definition_dir" || { + red "$c_name: failed to copy the azd environment."; had_errors=1; continue + } cyan "Deploying $c_name ($c_kind) from $c_path" env_name="${AZURE_ENV_NAME:-}" if [ -n "$c_digest" ]; then - (cd "$c_path" && azd env set AGENTLZ_IMAGE_DIGEST "$c_digest" --environment "$env_name" --no-prompt >/dev/null) || { + (cd "$app_definition_dir" && azd env set AGENTLZ_IMAGE_DIGEST "$c_digest" --environment "$env_name" --no-prompt >/dev/null) || { red "$c_name: failed to pin image digest."; had_errors=1; continue } fi if [ "$c_kind" = "azure.ai.agent" ]; then - # prepareHostedDeployment is not run here: it builds only the manifest-pinned - # bundled orchestrator image (HOSTED_AGENT_*). A custom azure.ai.agent - # component deploys from its own child azd project, which builds its image, - # or pins source.imageDigest through AGENTLZ_IMAGE_DIGEST (T079 limitation). + smoke_protocol="$(cd "$repo_root" && python3 -m config.deployment.hosted --service-protocol "$app_definition_dir/azure.yaml" --service-name "$c_name")" || { + red "$c_name: cannot resolve the greeting protocol."; had_errors=1; continue + } + # Custom images use the shared immutable/private build path, not the + # manifest-pinned orchestrator's derived hosted entrypoint. ( - cd "$c_path" && + cd "$app_definition_dir" && azd env set FOUNDRY_PROJECT_ENDPOINT "${AZURE_AI_PROJECT_ENDPOINT:-}" --environment "$env_name" --no-prompt >/dev/null && azd env set AZURE_AI_PROJECT_ID "${AZURE_AI_PROJECT_RESOURCE_ID:-}" --environment "$env_name" --no-prompt >/dev/null && - azd deploy "$c_name" --environment "$env_name" --no-prompt + python3 -c "import os, runpy, sys; sys.path.insert(0, os.environ['AGENTLZ_REPO_ROOT']); sys.argv = ['config.deployment.custom_application'] + sys.argv[1:]; runpy.run_module('config.deployment.custom_application', run_name='__main__')" --definition "$app_definition_path" --service "$c_name" --environment "$env_name" ) || { red "$c_name: hosted agent deployment failed."; had_errors=1; continue; } agent_smoke_payload="$(mktemp "${TMPDIR:-/tmp}/agentlz-agent-smoke-XXXXXX")" - printf '%s\n' '{"messages":[{"role":"user","content":"Hello!"}]}' >"$agent_smoke_payload" - if ! agent_smoke_output="$(cd "$c_path" && azd ai agent invoke --protocol invocations --new-session --timeout 180 --environment "$env_name" --no-prompt --input-file "$agent_smoke_payload" 2>&1)"; then + smoke_options=() + smoke_validation=--validate-smoke + if [ "$smoke_protocol" = responses ]; then + printf '%s\n' '{"input":[{"role":"user","content":"Hello!"}],"stream":true}' >"$agent_smoke_payload" + smoke_options=(--output raw) + smoke_validation=--validate-responses-smoke + else + printf '%s\n' '{"messages":[{"role":"user","content":"Hello!"}]}' >"$agent_smoke_payload" + fi + if ! agent_smoke_output="$(cd "$app_definition_dir" && azd ai agent invoke "$c_name" --protocol "$smoke_protocol" --new-session --timeout 180 --environment "$env_name" --no-prompt --input-file "$agent_smoke_payload" "${smoke_options[@]}" 2>&1)"; then rm -f "$agent_smoke_payload" red "$c_name: smoke request failed."; had_errors=1; continue fi rm -f "$agent_smoke_payload" - if ! printf '%s\n' "$agent_smoke_output" | (cd "$repo_root" && python3 -m config.deployment.hosted --validate-smoke); then + if ! printf '%s\n' "$agent_smoke_output" | (cd "$repo_root" && python3 -m config.deployment.hosted "$smoke_validation"); then red "$c_name: smoke test did not pass."; had_errors=1; continue fi else - (cd "$c_path" && azd deploy --all --environment "$env_name" --no-prompt) || { + (cd "$app_definition_dir" && python3 -c "import os, runpy, sys; sys.path.insert(0, os.environ['AGENTLZ_REPO_ROOT']); sys.argv = ['config.deployment.custom_application'] + sys.argv[1:]; runpy.run_module('config.deployment.custom_application', run_name='__main__')" --definition "$app_definition_path" --service "$c_name" --environment "$env_name") || { red "$c_name: Container App deployment failed."; had_errors=1; continue } fi diff --git a/scripts/preProvision.ps1 b/scripts/preProvision.ps1 index cbb3e0bd..d3d5ea4f 100755 --- a/scripts/preProvision.ps1 +++ b/scripts/preProvision.ps1 @@ -53,6 +53,7 @@ $manifestSource = Join-Path $projectRoot "manifest.json" #------------------------------------------------------------------------------- $appDefinitionPath = if ($env:AGENTLZ_APP_DEFINITION) { $env:AGENTLZ_APP_DEFINITION } else { Join-Path $projectRoot 'app-definition.json' } if (-not [IO.Path]::IsPathRooted($appDefinitionPath)) { $appDefinitionPath = Join-Path $projectRoot $appDefinitionPath } +if (Test-Path -LiteralPath $appDefinitionPath -PathType Container) { $appDefinitionPath = Join-Path $appDefinitionPath 'app-definition.json' } if (-not (Test-Path -LiteralPath $appDefinitionPath -PathType Leaf)) { Write-Host "Error: AGENTLZ_APP_DEFINITION points to $appDefinitionPath, which does not exist." -ForegroundColor Red exit 1 diff --git a/scripts/preProvision.sh b/scripts/preProvision.sh index a7917d7f..84fbd9ab 100755 --- a/scripts/preProvision.sh +++ b/scripts/preProvision.sh @@ -63,6 +63,9 @@ case "$APP_DEFINITION_PATH" in /*) ;; *) APP_DEFINITION_PATH="$PROJECT_ROOT/$APP_DEFINITION_PATH" ;; esac +if [ -d "$APP_DEFINITION_PATH" ]; then + APP_DEFINITION_PATH="$APP_DEFINITION_PATH/app-definition.json" +fi if [ ! -f "$APP_DEFINITION_PATH" ]; then echo "${YELLOW}Error: AGENTLZ_APP_DEFINITION points to $APP_DEFINITION_PATH, which does not exist.${NC}" exit 1 diff --git a/specs/002-agent-landing-zone-rebrand/quickstart.md b/specs/002-agent-landing-zone-rebrand/quickstart.md index 5c6eacbf..8dd3df03 100644 --- a/specs/002-agent-landing-zone-rebrand/quickstart.md +++ b/specs/002-agent-landing-zone-rebrand/quickstart.md @@ -143,11 +143,13 @@ Evidence after the repository rename (`curl -sI`): ## S5 — Validate an application definition (US5) ```powershell -python -m config.appdefinition --validate samples/custom-app/app-definition.json +python -m config.appdefinition --validate samples/custom-app/containerapp +python -m config.appdefinition --validate samples/custom-app/hosted ``` ```sh -python -m config.appdefinition --validate samples/custom-app/app-definition.json +python -m config.appdefinition --validate samples/custom-app/containerapp +python -m config.appdefinition --validate samples/custom-app/hosted ``` Expected outcome: @@ -176,18 +178,20 @@ in `v4.0.0` they are gone, so the same command proves the full rename. ```powershell azd env new -azd env set AGENTLZ_APP_DEFINITION samples/custom-app +azd env set AGENTLZ_APP_DEFINITION samples/custom-app/containerapp azd up ``` ```sh azd env new -azd env set AGENTLZ_APP_DEFINITION samples/custom-app +azd env set AGENTLZ_APP_DEFINITION samples/custom-app/containerapp azd up ``` -Run this once with the sample's Container Apps variant and once with its -hosted-agent variant. +Replace the all-zero component source pin with the actual checkout HEAD before +deployment. Run this once with `samples/custom-app/containerapp` and once in a +separate environment with `samples/custom-app/hosted`. The definition folder +contains `azure.yaml`; `components[].path` selects its service source folder. Expected outcome: @@ -201,12 +205,224 @@ Negative checks. Both must fail **before any Azure change**: - Point `AGENTLZ_APP_DEFINITION` at an invalid definition and run `azd up`. The output includes the JSON pointer to the error. -- On the bound environment, change the definition (or point it at the trio) +- On the bound environment, select a definition with a different application + `id` (for example, point it at the trio) and run `azd up`. The output reports a binding mismatch and says to create a new azd environment (FR-015g). Re-running `azd up` with the same definition is idempotent (FR-016). +## Acceptance status (2026-10-09) + +UI `v3.2.1` and umbrella `v4.2.2` were published. Their publication does not +close the outstanding runtime acceptance scenarios: + +- Classic foundation resources were provisioned, but post-provision governance + failed with Key Vault `ForbiddenByConnection`: public access was disabled + and the validation vault had no private endpoint. Governance subsequently + passed from the jumpbox over approved recovery private endpoints, using + the exact release source and managed identity. Continuity, Container Apps + and Search setup also passed. Application deployment completed, but the + latest frontend revision is unhealthy with the same vault connection denial. + An HTTP 200 came from the old placeholder, not a healthy application. + Authenticated grounded answers and ingestion startup remain unverified. +- Hosted-panel foundation, private governance recovery, continuity, Container + Apps and Search setup passed. The pinned hosted image was built and resolved + to an immutable digest. The release's quota preflight treats an existing + allocation as a new request. Its corrective branch now passes both real + preflight gates, crediting only verified matching deployments and checking + incremental capacity. That does not prove the hosted deploy handoff. + The separate network-isolated foundation and full post-provision hook + completed from the jumpbox. Owner recovery reconciled the already-declared + native Cosmos grants; the temporary management permission was then revoked. + The private pinned image also built successfully after firewall denies + identified the frontend lockfile's Azure Artifacts feeds. The existing + application-specific build allow-list now includes those four feeds and their + verified blob CDN, restricted to build-subnet HTTPS. The immutable image + provenance was synchronized without credentials, and the actual hosted + handoff preview passed. + The private Owner ARM handoff and full managed post-provision hook passed. + Project-scoped Foundry User resolved the runner's agent-authoring denial after + propagation. Owner reconciled the three discovered runtime grants without + granting the runner role-management permissions. Normal root deployment then + completed, and a separate managed-identity greeting passed the strict terminal + Responses validator. These are corrective-source results, not pristine release + acceptance or delegated-user retrieval evidence. + Initial UI and ingestion startup failed because OAuth configuration was + absent. A dedicated single-tenant evaluation registration now exposes the UI's + own API scope. Its short-lived credential was delivered encrypted to the + private runner, stored only in Key Vault, and referenced in App Configuration + with label `agent-lz`; tenant credential-lifetime policy was preserved. + A persistent Key Vault-backed Chainlit signing key was also configured. + Both intended pinned revisions are now Healthy/RunningAtMaxScale, match + latestReadyRevisionName, and receive 100% of traffic; placeholders no longer + serve the application. Private probes verified the real UI, enabled Entra + provider, correct login redirect, ingestion health, and HTTP 401 on + unauthenticated panel data access. Authentication has not been bypassed. + Actual user login/consent, user-delegated OBO, grounded answers, citations, + and positive document authorization remain unverified. An OAuth resource + token diagnostic did not invoke the agent and is not OBO acceptance. + A subsequent negative runtime test acquired a valid Foundry resource token + for the dedicated evaluation service principal, which has no project role. + The actual hosted invocation returned a structured HTTP 403 authorization + rejection. The project endpoint resolved entirely inside the evaluation + spoke. All private foundation endpoint connections were Approved/Succeeded; + Foundry, ACR, and Key Vault public access remained Disabled, ACR admin + authentication remained disabled, and Key Vault RBAC remained enabled. + Runtime grants were rechecked at their exact scopes: App Configuration + Data Reader on the store, OpenAI User on the model account, and Secrets + User only on AUDIT-HMAC-KEY. This is hosted-principal isolation evidence, + not delegated-user or document-authorization acceptance, and does not + replace deploying the custom hosted sample. +- The evaluation VPN Gateway was provisioned and protected with `keep=true` + and a deletion lock. Hub/spoke peerings are Connected with gateway transit. + Private DNS and managed-identity governance calls succeeded from the actual + jumpbox. Client VPN authentication and full delegated-user hosted runtime + acceptance remain unverified. +- Classic runtime connectivity cannot be repaired by adding an endpoint to its + current Container Apps environment: its `vnetConfiguration` is null and the + classic resource group has no virtual network. The existing recovery + endpoints make services reachable from the private jumpbox, not from apps + in the Azure-managed network. Azure does not support changing that + environment's network type after creation + ([Azure Container Apps networking](https://learn.microsoft.com/azure/container-apps/networking)). + Recovery requires a separate + VNet-integrated environment; public Key Vault access was not enabled and + existing apps/resources were not deleted or silently replaced. +- A fresh S7 foundation cannot use the current default Standard embedding + allocation unchanged: live westus3 usage is 300 of 350, leaving 50 while + the default new deployment requests 100. Existing-allocation credit is + valid only for an actual matching deployment, not a fresh account. Both + sample definitions validate, and the focused schema, roles, binding, + composition, and custom-hook suite passed 43 tests and 28 subtests. + Those checks do not constitute a successful one-command S7 deployment. +- Separate Container App and hosted sample evaluations now use 20 new + Standard embedding units each in evaluation-only parameter copies. Published + defaults are unchanged. Both real preProvision gates and composed previews + passed with distinct nonoverlapping spokes and independent application + bindings. Initial provisioning failed on each Container Apps private endpoint + with provider `InternalServerError`, not a proven naming conflict. Retrying + only the exact failed endpoint module, with unchanged exported parameters and + a no-deletion what-if, recovered both endpoints to Succeeded/Approved. + Reverse VPN gateway-transit peerings and direct runner-to-spoke peerings are + Connected. Both normal root provisioning resumes completed successfully, + explicitly deferring private configuration rather than claiming application + readiness. Exact new-spoke private A records were added, without replacing + existing records, to the evaluation runner's already-linked DNS zones. + From the actual jumpbox, both samples' App Configuration, Key Vault, ACR, + and Foundry endpoints resolved only into their corresponding private spokes + and returned HTTPS responses with TLS verification. These are connectivity + probes, not authenticated application acceptance. Resource-scoped runner + configuration roles were reconciled without granting Owner or role-management + permissions. The Container sample's full root postProvision subsequently + passed from the pinned corrective guest source: platform outputs, governance, + continuity, Foundry, Container Apps, and Search completed without deferral. + The first capability-role attempt correctly rejected the runner's lack of + role-management privileges; the parent Owner reconciled only the declared + component grants before the successful unchanged hook retry. The hosted + sample's full root postProvision also passed from the same pinned source, + with explicit guest success markers and no guest stderr for both samples. + Private image building then reached the dedicated ACR pool but failed on + the Docker Hub base-image pull: `registry-1.docker.io` was absent from the + build-subnet HTTPS allowlist. This is a build dependency, not a reason to + enable public registry access or bypass the firewall. Extending only the + build-subnet rule with the three documented Docker Hub HTTPS endpoints, + persisted through evaluation-only `additionalAcrTaskBuildFqdns`, recovered + both builds. Both images were built from the exact corrective source using + their own private ACR `build-pool`; immutable image digests and an explicit + guest completion marker were returned with no guest stderr. This separate + prebuild is not proof of the stock one-command S7 deployment. + Both samples subsequently packaged and deployed using evaluation-only + child YAML with immutable `image` and `docker.imagePassthrough: true`. + The Container App uses its existing managed identity for private ACR pulls; + its latest revision is latest-ready and matches the built digest. Its `/` + response, reached through private DNS, exactly matches live + `AGENTLZ_PLATFORM_OUTPUTS` with label `agent-lz`, schema version 1 and + `network.isolated=true`. A TLS-verified local VPN probe also received that + contract using explicit DNS, not automatic OS resolution. The published + identity map was initially empty despite the deployed app identity. + The corrective publisher now discovers the actual Container App client ID. + Two consecutive publications returned the same `web` identity in both the + JSON document and its flat key; a subsequent private `/` response exactly + matched that identity-bearing live contract. The guest temporarily staged + the corrected publisher/resolver and restored its original source afterward. + Missing endpoint aliases in this evaluation's copied azd metadata were + explicitly hydrated from the schema-validated existing App Configuration + document, not guessed. This proves the correction against the live resources, + not a pristine published-release or stock one-command S7 run. + The hosted sample initially failed activation because the actual Foundry + project identity lacked AcrPull. An exact-registry grant recovered version + 2 to active with the built digest. Its actual runtime identity then received + App Configuration Data Reader only on its configuration store. A real + Responses invocation from the managed runner resolved the project into its + private spoke and returned a completed assistant response containing the + exact project endpoint published in live platform outputs, not `unknown` + or `unavailable`. The explicit runtime-check marker and empty guest stderr + were verified. This proves private managed-identity sample invocation, + not user-delegated OBO, grounded answers, document authorization, repeated + provisioning, or the original remote-build sample YAML. + Repeating both child overlay deployments preserved their immutable image + digests. The Container App's new revision + `ca-ddzasx67ghev4-web--azd-1791574208` became latest-ready, and its repeated + private response again exactly matched live platform outputs. The hosted + deployment reused active `sample-hosted-agent` version 2 and the same runtime + identity; its repeated Responses check again returned the exact live project + endpoint. All 13 hosted private endpoints were Succeeded/Approved; Foundry + public access and local authentication, and ACR public/admin access, remained + disabled. These checks prove repeated child deployment, not repeated full + root provisioning or the stock one-command S7 flow. + The identity correction passed 413 tests and 500 subtests, with 4 skips; + the central documentation update `0abad14` is in Azure/AI-Landing-Zones#148. + The evaluation VPN client reached the existing firewall DNS proxy and + received the expected private addresses for all three evaluation spokes. + The updated client profile preserves authentication and split tunneling; + automatic OS resolution still requires profile reimport and reconnect. +- Validation found directory-selection, custom-project working-directory, + and hosted greeting-protocol defects. The corrective change passes + 598 tests and 591 subtests (4 skipped), but those offline results do not + establish a successful S7 Azure deployment or retroactively validate + the published release. +- Read-only checks against the actual private environment binding accepted + the exact-release default definition, rejected a valid different application + ID with exit 2, and rejected an unsupported capability with exit 1. The + environment-file hash was unchanged. These negatives do not establish + a successful custom-application deployment or repeated live provisioning. +- Validation also found false-success logging when an Entra-only Foundry + account rejects evaluation API keys, and a missing Cosmos database-name + handoff for the administrative panel. The corrective branch respects disabled + local authentication and reads published database settings before assigning + the unchanged container-scoped roles. Its live panel recovery created four + narrow grants; these fixes are not in the published release. +- The pinned orchestrator still exposes the legacy `GPT-RAG Orchestrator` + OpenAPI title. No branding acceptance or new component release is claimed. + +The corrective shared custom-image flow in commit `a9214de` subsequently +built and deployed both samples using the checked-in service YAML rather than +a manually prepared image overlay. Each build used its existing private ACR +pool and returned an immutable image. Container digest: +`sha256:fb33898065127a48eab6e402cab4550b0eab72828899865f3d3e25771b43185a`; +hosted digest: +`sha256:ca213b38d82ae24ebbd01ba1f2597cec2c1b94a60f316cab3a684fc67a902243`. +The helper restored each original YAML; temporary helper staging was removed. +Evaluation definitions were temporarily pinned to verified guest source +`aae0a3c` and restored afterwards; missing metadata aliases still required +explicit evaluation hydration. This is corrective-flow evidence, not pristine +release or stock root-hook acceptance. + +The resulting Container revision `ca-ddzasx67ghev4-web--azd-1791577126` +was latest-ready. Its actual private response on `10.252.2.11` exactly +matched the live platform contract, including the real `web` client ID. +Hosted version 3 was active with the exact new build digest and preserved +runtime principal `6e447da6-8aca-4f96-af56-e863aa0da041`. Its real private +Responses invocation on `10.253.2.33` returned the exact project greeting. +The combined runtime-check marker passed with empty guest stderr. +No new runtime role grant was needed. The final correction suite passed +432 tests and 500 subtests, with 4 skips; central guidance `1512486` is in +Azure/AI-Landing-Zones#148. + +T035, T085, T098, and T093 remain open until their actual runtime outcomes +are captured. No network policy was disabled to obtain a passing result. + ## Fallback check (R16) If the hosted-agent automation misses the 2026-10-09 deadline, S7's diff --git a/specs/002-agent-landing-zone-rebrand/tasks.md b/specs/002-agent-landing-zone-rebrand/tasks.md index bcfc0ef2..bc8c5070 100644 --- a/specs/002-agent-landing-zone-rebrand/tasks.md +++ b/specs/002-agent-landing-zone-rebrand/tasks.md @@ -212,9 +212,9 @@ modes). Fallback per R16 if FR-015a–h does not land by 2026-10-09. - [X] T084 [US5] Publish the "Build your own application" page (schema, profiles, outputs, both modes, binding rule) on the central AI Landing Zones site and register it in its nav - Status: deferred: central docs repo (external) - [ ] T085 [US5] Validate S5 (invalid and changed definitions rejected before any Azure change) and S7 (sample deploys with one `azd up` in both modes) and record evidence in specs/002-agent-landing-zone-rebrand/quickstart.md - - Status: partial: S5 covered by tests/test_app_definition*.py; S7 deploy blocked by the same MCAPS constraints as T035 + - Status: partial: S5 schema/binding negatives passed locally and on the private runner without modifying the bound environment. Both sample definitions validate; focused schema/roles/binding/composition/hooks suite passed 43 tests and 28 subtests. S7 still requires fresh one-command deploys in both modes; current default new Standard embedding allocation requests 100 with only 50 regional headroom. Existing allocations cannot be credited to a new account. - [ ] T098 [US5] Deploy the hosted mode with `NETWORK_ISOLATION=true` and verify private endpoints and RBAC for the hosted-agent component (FR-015c); record evidence in specs/002-agent-landing-zone-rebrand/quickstart.md - - Status: blocked: network-isolated deploy not possible under MCAPS policy in the maintainer subscription + - Status: partial: bundled hosted component deployed on corrective source with network isolation; managed greeting passed, private endpoints are Approved/Succeeded, public access remains disabled, exact least-privilege runtime grants verified, and a valid token from an unassigned principal was rejected by the actual private hosted endpoint with structured HTTP 403. Custom hosted sample parity (FR-015c) remains unverified; these checks do not prove user OBO or document authorization. - [X] T099 [P] [US5] Compare the role assignments generated by the profiles (`base`, `model-user`, `retrieval-reader`, `conversation-store`, `blob-delegator`, `ingestion-writer`) with the trio's current role assignments and record a parity checklist (FR-015e, SC-006) in tests/test_app_definition_schema.py and specs/002-agent-landing-zone-rebrand/quickstart.md - [X] T086 [US5] If FR-015a–h misses 2026-10-09, publish the R16 manual procedure instead and mark FR-015 as fallback in specs/002-agent-landing-zone-rebrand/spec.md and the central docs page - Status: N/A. FR-015 (app definition) shipped in v4.0.0, so the fallback was not needed. diff --git a/tests/test_aifoundry_setup.py b/tests/test_aifoundry_setup.py new file mode 100644 index 00000000..df2454b0 --- /dev/null +++ b/tests/test_aifoundry_setup.py @@ -0,0 +1,64 @@ +"""Foundry evaluation keys respect account authentication and fail explicitly.""" + +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + +from config.aifoundry import setup + + +class FoundryEvaluationKeyTests(unittest.TestCase): + def setUp(self): + self.client = Mock() + self.client.accounts.get.return_value = SimpleNamespace( + properties=SimpleNamespace(disable_local_auth=False) + ) + self.client.accounts.list_keys.return_value = SimpleNamespace(key1="test-key") + self.vault_patch = patch.object(setup, "KeyVaultClient") + self.vault = self.vault_patch.start() + self.addCleanup(self.vault_patch.stop) + + def inject(self): + return setup.add_ai_foundry_account_api_key_to_key_vault( + self.client, "test-group", "test-foundry", + "https://test-vault.vault.azure.net/", "evaluationsModelApiKey", + ) + + def test_disabled_local_auth_never_reads_or_stores_a_key(self): + self.client.accounts.get.return_value.properties.disable_local_auth = True + with self.assertLogs(level="INFO") as logs: + self.assertFalse(self.inject()) + self.assertIn("Use Microsoft Entra ID", "\n".join(logs.output)) + self.client.accounts.list_keys.assert_not_called() + self.vault.assert_not_called() + + def test_enabled_local_auth_preserves_key_injection(self): + self.assertTrue(self.inject()) + self.client.accounts.get.assert_called_once_with("test-group", "test-foundry") + self.client.accounts.list_keys.assert_called_once_with("test-group", "test-foundry") + self.vault.return_value.set_secret.assert_called_once_with( + "evaluationsModelApiKey", "test-key" + ) + + def test_read_and_write_failures_are_not_reported_as_success(self): + for boundary in ( + self.client.accounts.get, + self.client.accounts.list_keys, + self.vault.return_value.set_secret, + ): + with self.subTest(boundary=boundary): + boundary.side_effect = RuntimeError("boundary unavailable") + with self.assertLogs(level="ERROR"), self.assertRaises(SystemExit) as error: + self.inject() + self.assertEqual(error.exception.code, 1) + boundary.side_effect = None + + def test_empty_key_is_rejected_before_vault_write(self): + self.client.accounts.list_keys.return_value.key1 = "" + with self.assertLogs(level="ERROR"), self.assertRaises(SystemExit): + self.inject() + self.vault.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_app_definition_roles.py b/tests/test_app_definition_roles.py index ac16e2f6..565799f1 100644 --- a/tests/test_app_definition_roles.py +++ b/tests/test_app_definition_roles.py @@ -14,6 +14,8 @@ from config.appdefinition.roles import ( RoleAssignmentError, assign_roles, + assign_hosted_roles, + discover_component_identities, load_role_guids, plan_assignments, ) @@ -84,6 +86,101 @@ def bundled(): class RoleAssignmentTests(unittest.TestCase): + def test_hosted_runtime_expands_profiles_and_converges_without_container_lookup(self): + az = FakeAz() + principal = "22222222-2222-4222-8222-222222222222" + component = {"name": "agent", "profiles": ["retrieval-reader", "conversation-store"]} + env = {"AZURE_RESOURCE_GROUP": RG} + created = assign_hosted_roles(component, principal, env, run_az=az) + self.assertEqual( + {item.role for item in expand_profiles(component["profiles"])}, + {item.assignment.role for item in created}, + ) + self.assertFalse(any(call[:2] == ["containerapp", "list"] for call in az.calls)) + self.assertTrue(all(principal in call for call in az.creates())) + az.calls.clear() + self.assertEqual([], assign_hosted_roles(component, principal, env, run_az=az)) + self.assertEqual([], az.creates()) + + def test_hosted_runtime_fails_before_writes_for_invalid_principal_or_ambiguous_scope(self): + az = FakeAz() + with self.assertRaises(ValueError): + assign_hosted_roles({"name": "agent"}, "not-a-guid", {"AZURE_RESOURCE_GROUP": RG}, run_az=az) + with self.assertRaises(RoleAssignmentError): + assign_hosted_roles({"name": "agent"}, "00000000-0000-0000-0000-000000000000", + {"AZURE_RESOURCE_GROUP": RG}, run_az=az) + vaults = [{"name": "kv-a", "id": f"{SUB}/kva"}, {"name": "kv-b", "id": f"{SUB}/kvb"}] + with patch.dict(RESOURCES, {"Microsoft.KeyVault/vaults": vaults}): + with self.assertRaisesRegex(RoleAssignmentError, "exactly one"): + assign_hosted_roles({"name": "agent"}, "22222222-2222-4222-8222-222222222222", + {"AZURE_RESOURCE_GROUP": RG}, run_az=az) + self.assertEqual([], az.creates()) + + def test_discovers_client_ids_without_role_writes(self) -> None: + calls = [] + + def az(arguments, *, required=True): + calls.append(arguments) + service = arguments[arguments.index("--query") + 1].split("=='", 1)[1].split("'", 1)[0] + return json.dumps([{ + "name": f"ca-{service}", + "identity": {"userAssignedIdentities": { + f"{SUB}/identity-{service}": {"principalId": f"p-{service}", "clientId": f"c-{service}"} + }}, + }]) + + actual = discover_component_identities( + bundled(), {"AZURE_RESOURCE_GROUP": RG}, hosted_orchestration=True, run_az=az + ) + self.assertEqual({"ui": "c-frontend", "ingestion": "c-dataingest"}, actual) + self.assertTrue(all(call[:2] == ["containerapp", "list"] for call in calls)) + + def test_client_id_lookup_is_exact_and_missing_identity_fails(self) -> None: + definition = load_definition(ROOT / "samples" / "custom-app" / "containerapp") + for system in (False, True): + with self.subTest(system=system): + calls = [] + + def az(arguments, *, required=True): + calls.append(arguments) + if arguments[:2] == ["containerapp", "list"]: + identity = {"principalId": "principal"} if system else { + "userAssignedIdentities": {f"{SUB}/uai": {"principalId": "principal"}} + } + return json.dumps([{"name": "ca-web", "identity": identity}]) + self.assertEqual( + ["ad", "sp", "show", "--id", "principal", "--query", "appId"] if system else + ["identity", "show", "--ids", f"{SUB}/uai", "--query", "clientId"], + arguments[:-3], + ) + return json.dumps("client") + + self.assertEqual({"web": "client"}, discover_component_identities( + definition, {"AZURE_RESOURCE_GROUP": RG}, hosted_orchestration=False, run_az=az + )) + self.assertEqual(2, len(calls)) + with self.assertRaisesRegex(RoleAssignmentError, "found 0"): + discover_component_identities( + definition, {"AZURE_RESOURCE_GROUP": RG}, hosted_orchestration=False, + run_az=lambda arguments, **kwargs: "[]", + ) + with self.assertRaisesRegex(RoleAssignmentError, "found 2"): + discover_component_identities( + definition, {"AZURE_RESOURCE_GROUP": RG}, hosted_orchestration=False, + run_az=lambda arguments, **kwargs: json.dumps([{ + "name": "ca-web", "identity": { + "principalId": "p1", "userAssignedIdentities": {"uai": {"principalId": "p2"}} + } + }]), + ) + + def test_hosted_only_definition_has_no_provision_time_identities(self) -> None: + definition = load_definition(ROOT / "samples" / "custom-app" / "hosted") + self.assertEqual({}, discover_component_identities( + definition, {}, hosted_orchestration=False, + run_az=lambda *args, **kwargs: self.fail("Hosted-only provision must not discover a Container App"), + )) + def test_plan_covers_only_containerapp_components_with_base_profile(self) -> None: classic = plan_assignments(bundled(), hosted_orchestration=False) self.assertEqual({"ui", "orchestrator", "ingestion"}, {item.component for item in classic}) diff --git a/tests/test_custom_application_deploy.py b/tests/test_custom_application_deploy.py new file mode 100644 index 00000000..fb187721 --- /dev/null +++ b/tests/test_custom_application_deploy.py @@ -0,0 +1,196 @@ +"""Immutable custom deployment and private build regressions.""" + +import json +from pathlib import Path +import shutil +import subprocess +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import yaml + +from config.deployment import custom_application as deploy + +ROOT = Path(__file__).resolve().parents[1] +DIGEST = "sha256:" + "a" * 64 +COMMIT = "1" * 40 +ENV = { + "AZURE_ENV_NAME": "offline", + "NETWORK_ISOLATION": "true", + "AZURE_CONTAINER_REGISTRY_ENDPOINT": "private.azurecr.io", + "AZURE_RESOURCE_GROUP": "offline", + "ACR_TASK_AGENT_POOL": "build-pool", +} + + +def test_pinned_digest_replaces_mutable_tag_without_build(tmp_path): + component = {"source": {"imageDigest": DIGEST}} + service = {"image": "${REGISTRY}/application:v1"} + assert deploy.prepare_image(component, service, tmp_path, {"REGISTRY": "private.azurecr.io"}) == ( + f"private.azurecr.io/application@{DIGEST}" + ) + with pytest.raises(ValueError, match="qualified service image"): + deploy.prepare_image(component, {}, tmp_path, ENV) + with pytest.raises(ValueError, match="environment variable"): + deploy.prepare_image(component, service, tmp_path, {}) + + +def test_private_build_selects_pool_and_pins_management_plane_digest(tmp_path): + component = {"name": "web", "path": "src", "source": {"commit": COMMIT}} + service = {"project": "./src", "language": "docker", "docker": {"remoteBuild": True}} + with patch.object(deploy.subprocess, "run", return_value=SimpleNamespace(stdout=COMMIT)), \ + patch.object(deploy, "check_stage") as network, \ + patch.object(deploy, "validate_acr_agent_pool") as pool, \ + patch.object(deploy, "build_source_image", return_value=DIGEST) as build: + assert deploy.prepare_image(component, service, tmp_path, ENV) == ( + f"private.azurecr.io/agent-landing-zone/custom/web@{DIGEST}" + ) + assert pool.call_args.kwargs["agent_pool"] == "build-pool" + network.assert_called_once_with(ENV, "hosted-build") + assert build.call_args.kwargs["agent_pool"] == "build-pool" + assert build.call_args.kwargs["source_dir"] == (tmp_path / "src").resolve() + + +@pytest.mark.parametrize("change,match", [ + ({"ACR_TASK_AGENT_POOL": ""}, "ACR_TASK_AGENT_POOL"), + ({"AZURE_CONTAINER_REGISTRY_ENDPOINT": ""}, "registry endpoint"), +]) +def test_private_build_rejects_missing_inputs_without_build(tmp_path, change, match): + component = {"name": "web", "path": "src", "source": {"commit": COMMIT}} + service = {"project": "./src", "language": "docker"} + with patch.object(deploy.subprocess, "run", return_value=SimpleNamespace(stdout=COMMIT)), \ + patch.object(deploy, "build_source_image") as build: + with pytest.raises(ValueError, match=match): + deploy.prepare_image(component, service, tmp_path, {**ENV, **change}) + build.assert_not_called() + + +@pytest.mark.parametrize("image", [ + "application:latest", "https://private.azurecr.io/application", + "private.azurecr.io/application --build-arg=unsafe", "${REGISTRY}/application", +]) +def test_digest_pin_rejects_unqualified_or_invalid_images(tmp_path, image): + with pytest.raises(ValueError): + deploy.prepare_image({"source": {"imageDigest": DIGEST}}, {"image": image}, tmp_path, {}) + + +def test_public_source_preserves_azd_build_and_still_checks_commit(tmp_path): + component = {"name": "web", "path": "src", "source": {"commit": COMMIT}} + with patch.object(deploy.subprocess, "run", return_value=SimpleNamespace(stdout=COMMIT)), \ + patch.object(deploy, "build_source_image") as build: + assert deploy.prepare_image(component, {"language": "python"}, tmp_path, {}) is None + build.assert_not_called() + with patch.object(deploy.subprocess, "run", return_value=SimpleNamespace(stdout="2" * 40)): + with pytest.raises(ValueError, match="HEAD does not match"): + deploy.prepare_image(component, {}, tmp_path, {}) + + +@pytest.mark.parametrize("service,match", [ + ({"project": "src", "language": "python"}, "language: docker"), + ({"project": "another", "language": "docker"}, "project must match"), + ({"project": "src", "language": "docker", "docker": {"buildArgs": {"X": "value"}}}, + "cannot preserve these docker options"), + ({"project": "src", "language": "docker", "docker": {"path": "other.Dockerfile"}}, + "require Dockerfile"), + ({"project": "src", "language": "docker", "docker": {"context": ".."}}, + "require Dockerfile"), +]) +def test_private_build_rejects_unsupported_layout_and_options(tmp_path, service, match): + component = {"name": "web", "path": "src", "source": {"commit": COMMIT}} + with patch.object(deploy.subprocess, "run", return_value=SimpleNamespace(stdout=COMMIT)), \ + patch.object(deploy, "check_stage") as network, \ + patch.object(deploy, "build_source_image") as build: + with pytest.raises(ValueError, match=match): + deploy.prepare_image(component, service, tmp_path, ENV) + network.assert_not_called() + build.assert_not_called() + + +@pytest.mark.parametrize("boundary", ["check_stage", "validate_acr_agent_pool", "build_source_image"]) +def test_private_build_propagates_network_pool_and_build_failures(tmp_path, boundary): + component = {"name": "web", "path": "src", "source": {"commit": COMMIT}} + service = {"project": "src", "language": "docker"} + with patch.object(deploy.subprocess, "run", return_value=SimpleNamespace(stdout=COMMIT)), \ + patch.object(deploy, "check_stage"), \ + patch.object(deploy, "validate_acr_agent_pool"), \ + patch.object(deploy, "build_source_image", return_value=DIGEST), \ + patch.object(deploy, boundary, side_effect=RuntimeError("explicit boundary failure")): + with pytest.raises(RuntimeError, match="explicit boundary failure"): + deploy.prepare_image(component, service, tmp_path, ENV) + + +@pytest.mark.parametrize("failure", [False, True]) +def test_deployment_consumes_pin_and_restores_original_yaml(tmp_path, failure): + app = tmp_path / "sample" + shutil.copytree(ROOT / "samples" / "custom-app" / "containerapp", app) + definition_path = app / "app-definition.json" + definition = json.loads(definition_path.read_text()) + definition["components"][0]["source"] = {"imageDigest": DIGEST} + definition_path.write_text(json.dumps(definition)) + path = app / "azure.yaml" + project = yaml.safe_load(path.read_text()) + project["services"]["web"]["image"] = "private.azurecr.io/web:mutable" + path.write_text(yaml.safe_dump(project)) + original = path.read_bytes() + + def run(args, **kwargs): + rendered = yaml.safe_load(path.read_text()) + service = rendered["services"]["web"] + assert service["image"] == f"private.azurecr.io/web@{DIGEST}" + assert service["docker"] == {"imagePassthrough": True} + assert args == ["azd", "deploy", "web", "--environment", "offline", "--no-prompt"] + assert kwargs["cwd"] == app + assert all(kwargs["env"][key] == value for key, value in ENV.items()) + if failure: + raise subprocess.CalledProcessError(1, args) + return subprocess.CompletedProcess(args, 0) + + with patch.object(deploy.subprocess, "run", side_effect=run): + if failure: + with pytest.raises(subprocess.CalledProcessError): + assert deploy.deploy_service(str(definition_path), "web", ENV) == ( + f"private.azurecr.io/web@{DIGEST}" + ) + else: + deploy.deploy_service(str(definition_path), "web", ENV) + assert path.read_bytes() == original + + +@pytest.mark.parametrize("failure", ["none", "deploy", "identity", "roles"]) +def test_hosted_deployment_reconciles_only_verified_runtime_profiles(tmp_path, failure): + app = tmp_path / "hosted" + shutil.copytree(ROOT / "samples" / "custom-app" / "hosted", app) + definition_path = app / "app-definition.json" + definition = json.loads(definition_path.read_text()) + definition["components"][0]["source"] = {"commit": COMMIT} + definition_path.write_text(json.dumps(definition)) + original = (app / "azure.yaml").read_bytes() + image = f"private.azurecr.io/agent@{DIGEST}" + instance = SimpleNamespace(principal_id="verified-principal", agent_version="7") + with patch.object(deploy, "prepare_image", return_value=image), \ + patch.object(deploy.subprocess, "run") as run, \ + patch.object(deploy, "discover_instance", return_value=instance) as discover, \ + patch.object(deploy, "assign_hosted_roles") as roles: + if failure == "deploy": + run.side_effect = subprocess.CalledProcessError(1, ["azd"]) + if failure == "identity": + discover.side_effect = RuntimeError("identity mismatch") + if failure == "roles": + roles.side_effect = RuntimeError("assignment denied") + if failure == "none": + assert deploy.deploy_service(str(definition_path), "agent", ENV) == image + else: + with pytest.raises((RuntimeError, subprocess.CalledProcessError)): + deploy.deploy_service(str(definition_path), "agent", ENV) + if failure == "deploy": + discover.assert_not_called() + else: + discover.assert_called_once_with(ENV, name="sample-hosted-agent") + if failure in {"deploy", "identity"}: + roles.assert_not_called() + else: + roles.assert_called_once_with( + definition["components"][0], instance.principal_id, ENV, + ) + assert (app / "azure.yaml").read_bytes() == original diff --git a/tests/test_custom_application_hooks.py b/tests/test_custom_application_hooks.py new file mode 100644 index 00000000..14dbcd92 --- /dev/null +++ b/tests/test_custom_application_hooks.py @@ -0,0 +1,231 @@ +"""Run custom deployment sections with offline CLIs and shipped sample layout.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +from tests.test_hosted_smoke import completion + +ROOT = Path(__file__).resolve().parents[1] + + +class CustomApplicationHookTests(unittest.TestCase): + def check_directory_selection(self, shell: str) -> None: + suffix = "ps1" if shell == "pwsh" else "sh" + for hook in ("preProvision", "preDeploy"): + with self.subTest(hook=hook), tempfile.TemporaryDirectory(prefix="app selection ") as directory: + root = Path(directory) + folder = root / "custom app" + folder.mkdir() + definition = folder / "app-definition.json" + definition.write_text("{}", encoding="utf-8") + source = (ROOT / "scripts" / f"{hook}.{suffix}").read_text(encoding="utf-8-sig") + env = dict(os.environ, AGENTLZ_APP_DEFINITION=str(folder), FAKE_ROOT=str(root)) + if shell == "pwsh": + start = source.index("$appDefinitionPath = if") + marker = 'Write-Host "Validating and binding' if hook == "preProvision" else "$boundAppId =" + end = source.index(marker, start) + prefix = "$projectRoot = $env:FAKE_ROOT; $repoRoot = $projectRoot; $globalEnv = @{}\n" + script = root / "select.ps1" + script.write_text(prefix + source[start:end] + "\nWrite-Output $appDefinitionPath\n", + encoding="utf-8") + command = [shutil.which(shell), "-NoProfile", "-File", str(script)] + else: + variable = "APP_DEFINITION_PATH" if hook == "preProvision" else "app_definition_path" + start = source.index(f'{variable}="${{AGENTLZ_APP_DEFINITION') + marker = 'echo "${CYAN}Validating and binding' if hook == "preProvision" else "command -v python3" + end = source.index(marker, start) + prefix = r""" +if command -v cygpath >/dev/null; then + AGENTLZ_APP_DEFINITION="$(cygpath -u "$AGENTLZ_APP_DEFINITION")" + FAKE_ROOT="$(cygpath -u "$FAKE_ROOT")" +fi +PROJECT_ROOT="$FAKE_ROOT" +repo_root="$FAKE_ROOT" +red() { printf '%s\n' "$*"; } +""" + script = root / "select.sh" + script.write_text(prefix + source[start:end] + f'\nprintf "%s\\n" "${variable}"\n', + encoding="utf-8", newline="\n") + command = [shutil.which(shell), str(script)] + result = subprocess.run(command, cwd=root, env=env, capture_output=True, text=True, timeout=20) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertTrue(result.stdout.strip().endswith("app-definition.json"), result.stdout) + definition.unlink() + result = subprocess.run(command, cwd=root, env=env, capture_output=True, text=True, timeout=20) + self.assertNotEqual(0, result.returncode, result.stdout + result.stderr) + + def run_custom(self, shell: str, *, mismatch: bool = False, hosted: bool = False, + smoke_failure: bool = False) -> tuple[int, list[str], str]: + suffix = "ps1" if shell == "pwsh" else "sh" + source = (ROOT / "scripts" / f"preDeploy.{suffix}").read_text(encoding="utf-8-sig") + with tempfile.TemporaryDirectory(prefix="custom app offline ") as directory: + root = Path(directory) + app = root / "application" + shutil.copytree(ROOT / "samples" / "custom-app" / ("hosted" if hosted else "containerapp"), app) + azure = root / ".azure" + azure.mkdir() + (azure / "config.json").write_text("{}", encoding="utf-8") + definition = json.loads((app / "app-definition.json").read_text(encoding="utf-8")) + component = definition["components"][0] + component["source"] = {"commit": "1" * 40} + (app / "app-definition.json").write_text(json.dumps(definition), encoding="utf-8") + response = completion()["response"] + if smoke_failure: + response["status"] = "failed" + (root / "response").write_text(json.dumps(response), encoding="utf-8") + (root / "custom-deploy.py").write_text( + "import os, sys, subprocess\n" + "from pathlib import Path\n" + "from unittest.mock import patch\n" + "from types import SimpleNamespace\n" + "from config.deployment import custom_application\n" + "def run(args, **kwargs):\n" + " if args[0] == 'git':\n" + " return subprocess.CompletedProcess(args, 0, stdout=os.environ['FAKE_COMMIT'])\n" + " folder = Path(kwargs['cwd'])\n" + " assert (folder / 'azure.yaml').is_file()\n" + " assert (folder / '.azure' / 'config.json').is_file()\n" + " with (Path(os.environ['FAKE_ROOT']) / 'trace').open('a', encoding='utf-8') as f:\n" + " f.write(' '.join(args[1:]) + '\\n')\n" + " return subprocess.CompletedProcess(args, 0)\n" + "with patch.object(custom_application.subprocess, 'run', side_effect=run), \\\n" + " patch.object(custom_application, 'discover_instance', return_value=SimpleNamespace(principal_id='verified', agent_version='1')), \\\n" + " patch.object(custom_application, 'assign_hosted_roles'):\n" + " sys.exit(custom_application.main(sys.argv[1:]))\n", encoding="utf-8") + env = dict(os.environ, FAKE_ROOT=str(root), FAKE_APP=str(app), + FAKE_COMMIT=("2" if mismatch else "1") * 40, REAL_PYTHON=sys.executable, + AGENTLZ_REPO_ROOT=str(ROOT), REAL_REPO=str(ROOT), PYTHONPATH=str(ROOT)) + if shell == "pwsh": + start = source.index("foreach ($dc in $definitionComponents)", source.index("# Components")) + end = source.index("\nif ($hadErrors)", start) + prefix = r""" +$ErrorActionPreference = 'Continue' +$appDefinitionDir = $env:FAKE_APP +$appDefinitionPath = Join-Path $appDefinitionDir 'app-definition.json' +$dotAzure = Join-Path $env:FAKE_ROOT '.azure' +$definitionComponents = @((Get-Content (Join-Path $appDefinitionDir 'app-definition.json') -Raw | ConvertFrom-Json).components) +$globalEnv = [pscustomobject]@{ AZURE_ENV_NAME = 'offline' } +$hadErrors = $false +function Get-ManifestComponentForDefinition { return $null } +function git { $global:LASTEXITCODE = 0; return $env:FAKE_COMMIT } +function python { + if ($args.Count -ge 2 -and "$($args[1])".Contains('config.deployment.custom_application')) { + & $env:REAL_PYTHON (Join-Path $env:FAKE_ROOT 'custom-deploy.py') @($args[2..($args.Count - 1)]) + } else { $input | & $env:REAL_PYTHON @args } +} +function azd { + if (-not (Test-Path 'azure.yaml') -or -not (Test-Path 'src')) { throw 'Wrong project folder' } + if (-not (Test-Path '.azure/config.json')) { throw 'Missing environment' } + Add-Content (Join-Path $env:FAKE_ROOT 'trace') ($args -join ' ') + if ($args[0] -eq 'ai') { Get-Content (Join-Path $env:FAKE_ROOT 'response') -Raw } + $global:LASTEXITCODE = 0 +} +""" + script = root / "run.ps1" + script.write_text(prefix + source[start:end] + "\nif ($hadErrors) { exit 1 }\n", encoding="utf-8") + command = [shutil.which(shell), "-NoProfile", "-NonInteractive", "-File", str(script)] + else: + fake_bin = root / "bin" + fake_bin.mkdir() + (root / "jq.py").write_text( + "import json, sys\n" + "query = sys.argv[2]\n" + "doc = json.load(open(sys.argv[3])) if len(sys.argv) > 3 else json.load(sys.stdin)\n" + "if query == '.components[]':\n" + " for item in doc['components']: print(json.dumps(item))\n" + "else:\n" + " value = doc\n" + " for key in query.split(' // ')[0].strip('.').split('.'): value = value.get(key, {})\n" + " print(value if value != {} else '')\n", encoding="utf-8") + shim = fake_bin / "jq" + shim.write_text('#!/usr/bin/env bash\nexec "$REAL_PYTHON" "$FAKE_ROOT/jq.py" "$@"\n', + encoding="utf-8", newline="\n") + shim.chmod(0o755) + python_shim = fake_bin / "python3" + python_shim.write_text( + '#!/usr/bin/env bash\n' + 'if [[ "$*" == *config.deployment.custom_application* ]]; then\n' + ' shift 2\n' + ' exec "$REAL_PYTHON" "$FAKE_ROOT/custom-deploy.py" "$@"\n' + 'fi\nexec "$REAL_PYTHON" "$@"\n', + encoding="utf-8", newline="\n") + python_shim.chmod(0o755) + start = source.index('while IFS= read -r dcomp; do', source.index("# Components")) + end = source.index('\nif [ "$had_errors"', start) + prefix = r""" +set -eu +if command -v cygpath >/dev/null; then + FAKE_ROOT="$(cygpath -u "$FAKE_ROOT")" + FAKE_APP="$(cygpath -u "$FAKE_APP")" + REAL_REPO="$(cygpath -u "$REAL_REPO")" +fi +export PATH="$FAKE_ROOT/bin:$PATH" FAKE_ROOT +app_definition_dir="$FAKE_APP" +app_definition_path="$FAKE_APP/app-definition.json" +dot_azure="$FAKE_ROOT/.azure" +AZURE_ENV_NAME=offline +AZURE_AI_PROJECT_ENDPOINT=https://example.invalid/api/projects/offline +AZURE_AI_PROJECT_RESOURCE_ID=offline +repo_root="$REAL_REPO" +had_errors=0 +red() { printf '%s\n' "$*"; } +green() { :; } +cyan() { :; } +manifest_component_for() { :; } +git() { printf '%s\n' "$FAKE_COMMIT"; } +copy_dot_azure() { cp -R "$1" "$2/"; } +azd() { + [ "$PWD" = "$FAKE_APP" ] || return 91 + [ -f .azure/config.json ] || return 92 + printf '%s\n' "$*" >> "$FAKE_ROOT/trace" + if [ "$1" = ai ]; then cat "$FAKE_ROOT/response"; fi +} +""" + script = root / "run.sh" + script.write_text(prefix + source[start:end] + '\nexit "$had_errors"\n', encoding="utf-8", newline="\n") + command = [shutil.which(shell), str(script)] + result = subprocess.run(command, cwd=root, env=env, capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=30) + trace = root / "trace" + events = trace.read_text(encoding="utf-8-sig").splitlines() if trace.exists() else [] + return result.returncode, events, result.stdout + result.stderr + + @unittest.skipUnless(shutil.which("pwsh"), "PowerShell is not installed") + def test_powershell_deploys_selected_service_from_definition_folder(self) -> None: + self.check_directory_selection("pwsh") + self.check_deploy("pwsh") + + @unittest.skipUnless(shutil.which("bash"), "Bash is not installed") + def test_shell_deploys_selected_service_from_definition_folder(self) -> None: + self.check_directory_selection("bash") + self.check_deploy("bash") + + def check_deploy(self, shell: str) -> None: + code, events, output = self.run_custom(shell) + self.assertEqual(0, code, output) + self.assertEqual(["deploy web --environment offline --no-prompt"], events) + code, events, output = self.run_custom(shell, mismatch=True) + self.assertNotEqual(0, code, output) + self.assertEqual([], events) + for failed in (False, True): + code, events, output = self.run_custom(shell, hosted=True, smoke_failure=failed) + if failed: + self.assertNotEqual(0, code, output) + else: + self.assertEqual(0, code, output) + self.assertIn("deploy agent --environment offline --no-prompt", events) + smoke = next(event for event in events if event.startswith("ai agent invoke ")) + self.assertIn("ai agent invoke agent --protocol responses", smoke) + self.assertIn("--output raw", smoke) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_custom_foundation_outputs.py b/tests/test_custom_foundation_outputs.py new file mode 100644 index 00000000..6c3e2480 --- /dev/null +++ b/tests/test_custom_foundation_outputs.py @@ -0,0 +1,27 @@ +"""Foundation endpoint handoff must not depend on bundled hosted selection.""" + +from pathlib import Path +import re + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.mark.parametrize("name,flag,value", [ + ("AZURE_AI_PROJECT_RESOURCE_ID", "deployAiFoundry", "aiFoundryProjectResourceId"), + ("AZURE_AI_PROJECT_ENDPOINT", "deployAiFoundry", "aiFoundryProjectEndpoint"), + ("AZURE_CONTAINER_REGISTRY_RESOURCE_ID", "deployContainerRegistry", + "_hostedAgentContainerRegistryResourceId"), + ("AZURE_CONTAINER_REGISTRY_ENDPOINT", "deployContainerRegistry", + "_hostedAgentContainerRegistryEndpoint"), +]) +def test_foundation_handoff_preserves_disabled_and_external_hosted_cases(name, flag, value): + text = (ROOT / "infra" / "main.bicep").read_text(encoding="utf-8") + expression = re.search(rf"^output {name} string = (.+)$", text, re.MULTILINE)[1] + assert expression == f"({flag} || _hostedAgentPrerequisitesEnabled) ? {value} : ''" + for provisioned, hosted, expected in ( + (False, False, ""), (True, False, value), + (False, True, value), (True, True, value), + ): + assert (value if provisioned or hosted else "") == expected diff --git a/tests/test_hosted_access.py b/tests/test_hosted_access.py index d4945684..e4c61c3a 100644 --- a/tests/test_hosted_access.py +++ b/tests/test_hosted_access.py @@ -168,6 +168,18 @@ def test_exact_three_roles_and_actual_instance(self): (access.Role.MODEL, PREFIX + "Microsoft.CognitiveServices/accounts/test-model"), (access.Role.SECRET, SECRET), ], [(grant.role, grant.scope) for grant in plan.grants]) + + def test_custom_instance_discovery_is_read_only_and_validates_actual_name(self): + self.azure.live["name"] = self.azure.version["name"] = "sample-hosted-agent" + instance = access.discover_instance( + {**ENV, "DEPLOYMENT_TOPOLOGY": "classic"}, name="sample-hosted-agent", run=self.azure, + ) + self.assertEqual(PRINCIPAL, instance.principal_id) + self.assertEqual("7", instance.agent_version) + self.assertFalse(any(call[:2] == ["appconfig", "kv"] for call in self.azure.calls)) + self.assertEqual([], self.azure.writes) + with self.assertRaisesRegex(access.AccessError, "name does not match"): + access.discover_instance(ENV, name="wrong-agent", run=self.azure) self.assertEqual([], self.azure.writes) self.assertTrue(all("secret" not in call[:2] for call in self.azure.calls)) diff --git a/tests/test_hosted_smoke.py b/tests/test_hosted_smoke.py index e73ac17f..4805814d 100644 --- a/tests/test_hosted_smoke.py +++ b/tests/test_hosted_smoke.py @@ -3,6 +3,8 @@ from contextlib import redirect_stdout import io import json +from pathlib import Path +import tempfile import unittest from unittest.mock import patch @@ -34,6 +36,29 @@ def sse(*events): class HostedSmokeTests(unittest.TestCase): + def test_custom_responses_json_and_raw_http(self): + response = json.dumps(completion()["response"]) + hosted.validate_responses_smoke_output(response) + hosted.validate_responses_smoke_output("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n" + response) + hosted.validate_responses_smoke_output(sse(completion())) + for body in ("HTTP/1.1 403 Forbidden\n\n" + response, + json.dumps(completion("")), '{"object":"response","status":"failed"}', + '{"object":"response","status":"completed","error":{"message":"PRIVATE-MARKER"}}'): + with self.subTest(body=body), self.assertRaises(ValueError) as caught: + hosted.validate_responses_smoke_output(body) + self.assertNotIn("PRIVATE-MARKER", str(caught.exception)) + + def test_custom_protocol_is_read_from_selected_service(self): + root = Path(__file__).resolve().parents[1] + self.assertEqual("responses", hosted.service_protocol(root / "samples/custom-app/hosted/azure.yaml", "agent")) + with tempfile.TemporaryDirectory() as directory: + project = Path(directory) / "azure.yaml" + project.write_text("services:\n agent:\n protocols:\n - protocol: invocations\n", encoding="utf-8") + self.assertEqual("invocations", hosted.service_protocol(project, "agent")) + project.write_text("services:\n agent:\n protocols:\n - protocol: a2a\n", encoding="utf-8") + with self.assertRaises(ValueError): + hosted.service_protocol(project, "agent") + def test_successful_completed_greeting_sse_not_exact_marker(self): hosted.validate_smoke_output("Invoking agent...\n" + sse( {"type": "response.output_text.delta", "delta": "Hello!"}, diff --git a/tests/test_panel_setup.py b/tests/test_panel_setup.py new file mode 100644 index 00000000..d27356f6 --- /dev/null +++ b/tests/test_panel_setup.py @@ -0,0 +1,66 @@ +"""Panel RBAC resolves published database names without widening its grants.""" + +import unittest +from unittest.mock import patch + +from config.panel import setup + + +class PanelPublishedDatabaseTests(unittest.TestCase): + def setUp(self): + self.environment = { + "DEPLOY_ADMINISTRATIVE_PANEL": "true", + "AZURE_RESOURCE_GROUP": "test-group", + "RESOURCE_TOKEN": "testtoken", + "AZURE_SUBSCRIPTION_ID": "test-subscription", + "APP_CONFIG_ENDPOINT": "https://test-store.azconfig.io", + } + + def test_published_account_and_database_are_used_for_container_scopes(self): + with ( + patch.object(setup, "_run_az", side_effect=["test-account", "test-database"]) as az, + patch.object(setup, "resolve_container_app_principal_id", side_effect=["ui", "ingestion"]), + patch.object(setup, "ensure_cosmos_sql_role_assignment", return_value=True) as assign, + ): + self.assertEqual(setup.configure_panel_rbac(self.environment), 4) + self.assertEqual([call.args[0][call.args[0].index("--key") + 1] for call in az.call_args_list], + ["DATABASE_ACCOUNT_NAME", "DATABASE_NAME"]) + for call in az.call_args_list: + self.assertIn(setup.LABEL, call.args[0]) + self.assertIn("login", call.args[0]) + for call in assign.call_args_list: + self.assertEqual(call.args[0], "test-account") + self.assertIn("/dbs/test-database/colls/", call.args[-1]) + self.assertIn(call.args[3], ("ui", "ingestion")) + + def test_explicit_names_preserve_existing_behavior_without_config_read(self): + self.environment.update(DATABASE_ACCOUNT_NAME="explicit-account", DATABASE_NAME="explicit-db") + with ( + patch.object(setup, "_run_az") as az, + patch.object(setup, "resolve_container_app_principal_id", return_value="test-identity"), + patch.object(setup, "ensure_cosmos_sql_role_assignment", return_value=False), + ): + self.assertEqual(setup.configure_panel_rbac(self.environment), 0) + az.assert_not_called() + + def test_empty_published_value_fails_before_any_role_change(self): + with ( + patch.object(setup, "_run_az", return_value=""), + patch.object(setup, "ensure_cosmos_sql_role_assignment") as assign, + self.assertRaises(setup.PanelRbacError), + ): + setup.configure_panel_rbac(self.environment) + assign.assert_not_called() + + def test_config_failure_propagates_without_role_changes(self): + with ( + patch.object(setup, "_run_az", side_effect=setup.PanelRbacError("read failed")), + patch.object(setup, "ensure_cosmos_sql_role_assignment") as assign, + self.assertRaises(setup.PanelRbacError), + ): + setup.configure_panel_rbac(self.environment) + assign.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_platform_outputs.py b/tests/test_platform_outputs.py index de31867e..58830ce8 100644 --- a/tests/test_platform_outputs.py +++ b/tests/test_platform_outputs.py @@ -98,6 +98,37 @@ def test_cli_dry_run_writes_nothing(monkeypatch, capsys): assert "AGENTLZ_PLATFORM_OUTPUTS" in json.loads(capsys.readouterr().out) +def test_cli_discovers_and_publishes_container_identity_when_map_is_absent(monkeypatch, capsys): + for key, value in ENVIRONMENT.items(): + monkeypatch.setenv(key, value) + monkeypatch.delenv(outputs.IDENTITIES_ENV) + monkeypatch.setenv("AGENTLZ_APP_DEFINITION", str(REPO_ROOT / "samples" / "custom-app" / "containerapp")) + with mock.patch("config.appdefinition.roles.discover_component_identities", + return_value={"web": UI_CLIENT_ID}) as discover, mock.patch.object(outputs, "publish") as publish: + assert outputs.main([]) == 0 + discover.assert_called_once() + settings = publish.call_args.args[1] + assert json.loads(settings[outputs.PLATFORM_OUTPUTS_KEY])["identities"] == [ + {"component": "web", "clientId": UI_CLIENT_ID} + ] + assert settings["AGENTLZ_IDENTITY_WEB_CLIENT_ID"] == UI_CLIENT_ID + + +def test_cli_rejects_identity_discovery_failure_before_publication(monkeypatch, capsys): + from config.appdefinition.roles import RoleAssignmentError + + for key, value in ENVIRONMENT.items(): + monkeypatch.setenv(key, value) + monkeypatch.delenv(outputs.IDENTITIES_ENV) + monkeypatch.setenv("AGENTLZ_APP_DEFINITION", str(REPO_ROOT / "samples" / "custom-app" / "containerapp")) + with mock.patch("config.appdefinition.roles.discover_component_identities", + side_effect=RoleAssignmentError("Expected exactly one identity")), \ + mock.patch.object(outputs, "publish") as publish: + assert outputs.main([]) == 1 + publish.assert_not_called() + assert "Expected exactly one identity" in capsys.readouterr().err + + def test_repo_root_detected_by_markers_not_folder_name(tmp_path): root = tmp_path / "any-folder-name" nested = root / "config" / "deployment" diff --git a/tests/test_private_network_hooks.py b/tests/test_private_network_hooks.py index 665c8f90..150ed81e 100644 --- a/tests/test_private_network_hooks.py +++ b/tests/test_private_network_hooks.py @@ -15,6 +15,7 @@ PWSH = shutil.which("pwsh") BASH = shutil.which("bash") VALUES = """NETWORK_ISOLATION="true" +ACR_TASK_AGENT_POOL="test-pool" DEPLOYMENT_TOPOLOGY="classic" APP_CONFIG_ENDPOINT="https://config.azconfig.io" AZURE_RESOURCE_GROUP="test-group" @@ -273,6 +274,22 @@ def check_cases(self, shell: str) -> None: self.assertEqual([], events, output) self.assertIn("Run azd provision first", output) + for build_mode, allowed in (("acr-task", False), ("local", True)): + with self.subTest(shell=shell, build_mode=build_mode, pool_missing=True): + values = VALUES.replace('ACR_TASK_AGENT_POOL="test-pool"', 'ACR_TASK_AGENT_POOL=""') + result, events = self.run_hook( + shell, "preDeploy", 0, values=values, + process_env={"BUILD_MODE": build_mode}, + ) + output = result.stdout + result.stderr + self.assertIn("probe", events, output) + if allowed: + self.assertIn("azure-boundary", events, output) + else: + self.assertEqual(1, result.returncode, output) + self.assertNotIn("azure-boundary", events, output) + self.assertIn("ACR_TASK_AGENT_POOL is empty", output) + def integrated_cases(self, shell: str) -> None: for hook in ("postProvision", "preDeploy"): for extra, process, endpoint_fail, expected in ( diff --git a/tests/test_regional_model_quota.py b/tests/test_regional_model_quota.py new file mode 100644 index 00000000..08f1d3a1 --- /dev/null +++ b/tests/test_regional_model_quota.py @@ -0,0 +1,315 @@ +"""Execute the shipped PowerShell model quota checks with a read-only CLI fixture.""" + +from __future__ import annotations + +import copy +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +PWSH = shutil.which("pwsh") +ACCOUNT_ID = ( + "/subscriptions/test-subscription/resourceGroups/test-group/" + "providers/Microsoft.CognitiveServices/accounts/test-account" +) + +DRIVER = r""" +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$SkipAzureCliChecks = $false +$SubscriptionId = 'test-subscription' +$script:FailureCount = 0 +$script:WarningCount = 0 +$script:Calls = @() +$script:Findings = [System.Collections.Generic.List[pscustomobject]]::new() +$fixture = Get-Content $env:QUOTA_FIXTURE -Raw | ConvertFrom-Json +. $env:QUOTA_HELPER +$tokens = $null +$errors = $null +$ast = [System.Management.Automation.Language.Parser]::ParseFile($env:QUOTA_SOURCE, [ref]$tokens, [ref]$errors) +if ($errors.Count -gt 0) { throw 'PowerShell source parse failed.' } +$names = @('Write-PreflightCheck', 'Get-NormalizedLocation', 'Test-ModelReadiness', 'Add-Finding', 'Test-ModelQuota') +foreach ($node in $ast.FindAll({ param($node) $node -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $node.Name -in $names }, $false)) { + Invoke-Expression $node.Extent.Text +} +function Invoke-AzJson { + param([string[]]$Arguments, [switch]$AllowFailure) + $script:Calls += ($Arguments -join ' ') + if (($Arguments[0..2] -join ' ') -eq 'cognitiveservices model list') { return $fixture.available } + if (($Arguments[0..2] -join ' ') -eq 'cognitiveservices usage list') { return $fixture.usage } + if (($Arguments[0..1] -join ' ') -eq 'resource show') { + if ($fixture.accountFailure) { return @{ failed = $true; error = 'fixture account denied' } } + return $fixture.account + } + if (($Arguments[0..1] -join ' ') -eq 'resource list') { + if ($fixture.PSObject.Properties['projects']) { return $fixture.projects } + return @() + } + if (($Arguments[0..3] -join ' ') -eq 'cognitiveservices account deployment list') { + if ($fixture.deploymentFailure) { return @{ failed = $true; error = 'fixture deployments denied' } } + return $fixture.deployments + } + throw "Unexpected Azure operation: $($Arguments -join ' ')" +} +function Invoke-AzCliRaw { + param([string[]]$Arguments) + $value = Invoke-AzJson -Arguments $Arguments + if ($value -is [hashtable] -and $value.failed) { return $null } + return $value +} +function Get-AzdEnvValues { + if ($fixture.PSObject.Properties['selectedEnvironment']) { + $values = @{} + foreach ($property in $fixture.selectedEnvironment.PSObject.Properties) { + $values[$property.Name] = $property.Value + } + return $values + } + return @{ + AZURE_AI_PROJECT_RESOURCE_ID = $env:AZURE_AI_PROJECT_RESOURCE_ID + AZURE_RESOURCE_GROUP = $env:AZURE_RESOURCE_GROUP + } +} +if ($env:QUOTA_GATE -eq 'foundation') { + Test-ModelQuota -Location 'westus3' -ModelDeployments $fixture.desired + $script:FailureCount = @($script:Findings | Where-Object Severity -eq 'FAIL').Count + $script:WarningCount = @($script:Findings | Where-Object Severity -eq 'WARN').Count + foreach ($finding in $script:Findings) { Write-Host "$($finding.Severity) $($finding.Message)" } +} else { + Test-ModelReadiness -Location 'westus3' -Models $fixture.desired +} +Write-Output ('RESULT:' + (@{ failures = $script:FailureCount; warnings = $script:WarningCount; calls = $script:Calls } | ConvertTo-Json -Compress)) +""" + + +def deployment(name: str = "embedding", capacity: int = 100) -> dict: + return { + "name": name, + "model": {"format": "OpenAI", "name": "text-embedding-3-large", "version": "1"}, + "sku": {"name": "Standard", "capacity": capacity}, + } + + +def fixture() -> dict: + desired = deployment() + return { + "desired": [desired], + "available": [{"model": { + "name": desired["model"]["name"], "version": "1", + "skus": [{"name": "Standard"}, {"name": "GlobalStandard"}], + }}], + "usage": [{"name": {"value": "OpenAI.Standard.text-embedding-3-large"}, + "limit": 150, "currentValue": 100}], + "account": {"id": ACCOUNT_ID, "location": "West US 3"}, + "deployments": [{ + "name": desired["name"], "sku": copy.deepcopy(desired["sku"]), + "properties": {"model": copy.deepcopy(desired["model"]), + "provisioningState": "Succeeded"}, + }], + "accountFailure": False, + "deploymentFailure": False, + } + + +@unittest.skipUnless(PWSH, "PowerShell is required for regional quota behavior tests") +class RegionalModelQuotaTests(unittest.TestCase): + def run_check(self, data: dict, *, project: str = ACCOUNT_ID + "/projects/test-project", + group: str = "test-group", gate: str = "regional") -> tuple[dict, str]: + with tempfile.TemporaryDirectory(prefix="model quota ") as directory: + folder = Path(directory) + (folder / "fixture.json").write_text(json.dumps(data), encoding="utf-8") + (folder / "run.ps1").write_text(DRIVER, encoding="utf-8") + environment = dict( + os.environ, QUOTA_FIXTURE=str(folder / "fixture.json"), + QUOTA_SOURCE=str( + ROOT / "infra" / "scripts" / "Invoke-PreflightChecks.ps1" + if gate == "foundation" else ROOT / "scripts" / "Invoke-RegionalPreflight.ps1" + ), + QUOTA_HELPER=str(ROOT / "infra" / "scripts" / "ModelQuota.ps1"), + QUOTA_GATE=gate, + AZURE_AI_PROJECT_RESOURCE_ID=project, AZURE_RESOURCE_GROUP=group, + ) + result = subprocess.run( + [PWSH, "-NoProfile", "-NonInteractive", "-File", str(folder / "run.ps1")], + env=environment, capture_output=True, text=True, timeout=30, + ) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + row = next(line for line in result.stdout.splitlines() if line.startswith("RESULT:")) + return json.loads(row.removeprefix("RESULT:")), result.stdout + + def test_existing_unchanged_allocation_needs_no_remaining_quota(self) -> None: + data = fixture() + data["usage"][0]["currentValue"] = 155 + result, output = self.run_check(data) + self.assertEqual(0, result["failures"], output) + self.assertIn("verified existing credit 100, additional allocation 0", output) + + def test_fresh_allocation_still_requires_full_capacity(self) -> None: + for available, failures in ((50, 1), (100, 0)): + with self.subTest(available=available): + data = fixture() + data["usage"][0]["limit"] = 100 + available + result, output = self.run_check(data, project="") + self.assertEqual(failures, result["failures"], output) + self.assertEqual(3, len(result["calls"])) + self.assertIn("verified existing credit 0, additional allocation 100", output) + + def test_old_blank_output_recovers_only_unique_verified_project(self) -> None: + data = fixture() + data["projects"] = [{"id": ACCOUNT_ID + "/projects/test-project"}] + result, output = self.run_check(data, project="") + self.assertEqual(0, result["failures"], output) + self.assertIn("verified existing credit 100, additional allocation 0", output) + for projects in ( + [*data["projects"], {"id": ACCOUNT_ID + "/projects/other"}], + [{"id": ACCOUNT_ID.replace("test-group", "other-group") + "/projects/test-project"}], + ): + with self.subTest(projects=projects): + data["projects"] = projects + result, output = self.run_check(data, project="") + self.assertEqual(1, result["failures"], output) + self.assertIn("verified existing credit 0", output) + + def test_only_increment_requires_available_quota(self) -> None: + for capacity, failures in ((150, 0), (151, 1), (75, 0)): + with self.subTest(capacity=capacity): + data = fixture() + data["desired"][0]["sku"]["capacity"] = capacity + result, output = self.run_check(data) + self.assertEqual(failures, result["failures"], output) + + def test_existing_allocation_is_not_transferable(self) -> None: + for changed in ("name", "model", "version", "format", "sku", "state", "capacity"): + with self.subTest(changed=changed): + data = fixture() + current = data["deployments"][0] + if changed == "name": + current["name"] = "another-deployment" + elif changed in {"model", "version", "format"}: + key = "name" if changed == "model" else changed + current["properties"]["model"][key] = "another-value" + elif changed == "sku": + current["sku"]["name"] = "GlobalStandard" + elif changed == "state": + current["properties"]["provisioningState"] = "Failed" + else: + current["sku"]["capacity"] = 0 + result, output = self.run_check(data) + self.assertEqual(1, result["failures"], output) + self.assertIn("verified existing credit 0", output) + + def test_existing_target_requires_scope_and_location_proof(self) -> None: + for changed in ("subscription", "group", "project", "account", "location"): + with self.subTest(changed=changed): + data = fixture() + project = ACCOUNT_ID + "/projects/test-project" + if changed == "subscription": + project = project.replace("test-subscription", "other-subscription") + elif changed == "group": + project = project.replace("test-group", "other-group") + elif changed == "project": + project = "malformed-resource-id" + else: + data["account"]["id" if changed == "account" else "location"] = "another-value" + result, output = self.run_check(data, project=project) + self.assertEqual(1, result["failures"], output) + self.assertEqual(1, result["warnings"], output) + self.assertIn("verified existing credit 0", output) + + def test_failed_existing_reads_do_not_grant_credit(self) -> None: + for boundary in ("accountFailure", "deploymentFailure"): + with self.subTest(boundary=boundary): + data = fixture() + data[boundary] = True + result, output = self.run_check(data) + self.assertEqual(1, result["failures"], output) + self.assertEqual(1, result["warnings"], output) + + def test_model_availability_is_not_skipped_for_existing_capacity(self) -> None: + data = fixture() + data["available"] = [] + result, output = self.run_check(data) + self.assertEqual(1, result["failures"], output) + self.assertIn("is not listed in westus3", output) + self.assertIn("additional allocation 0", output) + + def test_multiple_deployments_share_one_quota_budget(self) -> None: + data = fixture() + data["desired"] = [deployment("one", 30), deployment("two", 30)] + result, output = self.run_check(data) + self.assertEqual(1, result["failures"], output) + self.assertIn("additional allocation 60", output) + + def test_existing_credit_is_consumed_only_once(self) -> None: + data = fixture() + data["desired"].append(copy.deepcopy(data["desired"][0])) + result, output = self.run_check(data) + self.assertEqual(1, result["failures"], output) + self.assertIn("duplicate deployment name", output.lower()) + + def test_invalid_capacity_is_rejected(self) -> None: + for capacity in (0, -1): + with self.subTest(capacity=capacity): + data = fixture() + data["desired"][0]["sku"]["capacity"] = capacity + result, output = self.run_check(data) + self.assertEqual(1, result["failures"], output) + + def test_duplicate_existing_name_is_not_credited(self) -> None: + data = fixture() + data["deployments"].append(copy.deepcopy(data["deployments"][0])) + result, output = self.run_check(data) + self.assertEqual(1, result["failures"], output) + self.assertIn("verified existing credit 0", output) + + def test_foundation_gate_enforces_the_same_incremental_budget(self) -> None: + for case, failures in ( + ("unchanged", 0), ("fresh", 1), ("increase", 0), ("excess", 1), + ("aggregate", 1), ("read-failure", 1), ("wrong-region", 1), ("duplicate", 1), + ): + with self.subTest(case=case): + data = fixture() + project = ACCOUNT_ID + "/projects/test-project" + if case == "fresh": + project = "" + elif case in {"increase", "excess"}: + data["desired"][0]["sku"]["capacity"] = 150 if case == "increase" else 151 + elif case == "aggregate": + data["desired"] = [deployment("one", 30), deployment("two", 30)] + elif case == "read-failure": + data["accountFailure"] = True + elif case == "wrong-region": + data["account"]["location"] = "eastus" + elif case == "duplicate": + data["desired"].append(copy.deepcopy(data["desired"][0])) + else: + data["usage"][0]["currentValue"] = 155 + result, output = self.run_check(data, project=project, gate="foundation") + self.assertEqual(failures, result["failures"], output) + + def test_both_platform_hooks_invoke_the_shared_regional_gate(self) -> None: + for suffix in ("ps1", "sh"): + with self.subTest(suffix=suffix): + source = (ROOT / "scripts" / f"preProvision.{suffix}").read_text(encoding="utf-8") + self.assertIn("Invoke-RegionalPreflight.ps1", source) + self.assertIn("Invoke-PreflightChecks.ps1", source) + + def test_foundation_uses_selected_environment_not_ambient_outputs(self) -> None: + for wrong_selected_target in (False, True): + with self.subTest(wrong_selected_target=wrong_selected_target): + data = fixture() + data["selectedEnvironment"] = { + "AZURE_AI_PROJECT_RESOURCE_ID": ACCOUNT_ID + "/projects/test-project", + "AZURE_RESOURCE_GROUP": "wrong-group" if wrong_selected_target else "test-group", + } + result, output = self.run_check( + data, project=ACCOUNT_ID.replace("test-group", "ambient-group") + "/projects/stale", + group="ambient-group", gate="foundation", + ) + self.assertEqual(int(wrong_selected_target), result["failures"], output)