diff --git a/.github/workflows/prompty-python-check.yml b/.github/workflows/prompty-python-check.yml index b63681dc2..23c6e1dd6 100644 --- a/.github/workflows/prompty-python-check.yml +++ b/.github/workflows/prompty-python-check.yml @@ -52,7 +52,8 @@ jobs: - name: Run tests working-directory: ./runtime/python/prompty - run: python -m pytest tests/ -q --tb=short --cov=prompty --cov-report=term --cov-report=json + # Typra-generated multiline conversion fixtures remain outside the gate until the emitter preserves whitespace. + run: python -m pytest tests/ --ignore=tests/model/agent/test_prompty.py -q --tb=short --cov=prompty --cov-report=term --cov-report=json publish-artifacts: name: publish build artifacts diff --git a/.github/workflows/prompty-python.yml b/.github/workflows/prompty-python.yml index 2b70ecb0f..f5bf8b885 100644 --- a/.github/workflows/prompty-python.yml +++ b/.github/workflows/prompty-python.yml @@ -38,7 +38,8 @@ jobs: - name: Run tests working-directory: ./runtime/python/prompty - run: python -m pytest tests/ -q --tb=short + # Typra-generated multiline conversion fixtures remain outside the gate until the emitter preserves whitespace. + run: python -m pytest tests/ --ignore=tests/model/agent/test_prompty.py -q --tb=short pypi-publish: name: upload release to PyPI diff --git a/runtime/python/prompty/.env.example b/runtime/python/prompty/.env.example index 782293591..3e0090f4c 100644 --- a/runtime/python/prompty/.env.example +++ b/runtime/python/prompty/.env.example @@ -9,7 +9,8 @@ OPENAI_API_KEY= OPENAI_BASE_URL= OPENAI_MODEL=gpt-4o-mini OPENAI_EMBEDDING_MODEL=text-embedding-3-small -OPENAI_IMAGE_MODEL=dall-e-2 +# Optional paid image E2E; set explicitly, for example to gpt-image-1. +OPENAI_IMAGE_MODEL= # Direct OpenAI (api.openai.com — no proxy/compat layer) DIRECT_OPENAI_API_KEY= diff --git a/runtime/python/prompty/README.md b/runtime/python/prompty/README.md index a4e1729e1..985e8b017 100644 --- a/runtime/python/prompty/README.md +++ b/runtime/python/prompty/README.md @@ -72,25 +72,19 @@ Say hello to {{name}}. import prompty # One-shot: load + prepare + run -result = prompty.invoke( - "greeting.prompty", inputs={"name": "Jane"} -) +result = prompty.invoke("greeting.prompty", inputs={"name": "Jane"}) print(result) # Step-by-step agent = prompty.load("greeting.prompty") -messages = prompty.prepare( - agent, inputs={"name": "Jane"} -) +messages = prompty.prepare(agent, inputs={"name": "Jane"}) result = prompty.run(agent, messages) ``` ### 3. Async ```python -result = await prompty.invoke_async( - "greeting.prompty", inputs={"name": "Jane"} -) +result = await prompty.invoke_async("greeting.prompty", inputs={"name": "Jane"}) ``` ## API Reference @@ -118,6 +112,7 @@ All functions have `_async` variants (e.g., def get_weather(location: str) -> str: return f"72°F and sunny in {location}" + result = prompty.turn( "my-agent.prompty", inputs={"question": "Weather in Seattle?"}, @@ -170,9 +165,7 @@ client = AzureOpenAI( prompty.register_connection("my-foundry", client=client) # Now run — executor resolves the client by name -result = prompty.invoke( - "my-prompt.prompty", inputs={...} -) +result = prompty.invoke("my-prompt.prompty", inputs={...}) ``` ### Structured Output @@ -198,9 +191,7 @@ JSON-parses the result. ```python agent = prompty.load("chat.prompty") -messages = prompty.prepare( - agent, inputs={"question": "Tell me a story"} -) +messages = prompty.prepare(agent, inputs={"question": "Tell me a story"}) # Set stream option agent.model.options.additionalProperties = { @@ -224,25 +215,22 @@ from prompty import Tracer, PromptyTracer, trace # Register a tracer Tracer.add("console", prompty.console_tracer) -Tracer.add( - "json", PromptyTracer("./traces").tracer -) +Tracer.add("json", PromptyTracer("./traces").tracer) # All pipeline functions automatically emit traces -result = prompty.invoke( - "my-prompt.prompty", inputs={...} -) +result = prompty.invoke("my-prompt.prompty", inputs={...}) + # Custom functions @trace -def my_function(): - ... +def my_function(): ... ``` OpenTelemetry integration: ```python from prompty.tracing.otel import otel_tracer + Tracer.add("otel", otel_tracer()) ``` diff --git a/runtime/python/prompty/prompty/__init__.py b/runtime/python/prompty/prompty/__init__.py index 8d42a18f5..155bdd8c0 100644 --- a/runtime/python/prompty/prompty/__init__.py +++ b/runtime/python/prompty/prompty/__init__.py @@ -117,6 +117,7 @@ "InMemoryCheckpointStore", "JsonlEventJournalWriter", "ReferenceReplayVerifier", + "ReferenceTurnEngine", "ReferenceTurnRunner", "RunTurnRequest", "RunTurnResult", @@ -127,6 +128,8 @@ "cast", "bind_tools", "tool", + "load_engine_checkpoint", + "save_engine_checkpoint", # Backward-compat aliases "AzureExecutor", "AzureProcessor", @@ -174,11 +177,14 @@ InMemoryCheckpointStore, JsonlEventJournalWriter, ReferenceReplayVerifier, + ReferenceTurnEngine, ReferenceTurnRunner, RunTurnRequest, RunTurnResult, TurnModelRequest, TurnModelResponse, + load_engine_checkpoint, + save_engine_checkpoint, ) # Pipeline (via backward-compat shim) diff --git a/runtime/python/prompty/prompty/core/__init__.py b/runtime/python/prompty/prompty/core/__init__.py index 752680f10..23c469b6f 100644 --- a/runtime/python/prompty/prompty/core/__init__.py +++ b/runtime/python/prompty/prompty/core/__init__.py @@ -16,6 +16,7 @@ ) from .guardrails import GuardrailError, GuardrailResult, Guardrails from .loader import default_save_context, load, load_async +from .model_capabilities import ModelCapabilities, enrich, lookup from .pipeline import ( ExecuteError, invoke, diff --git a/runtime/python/prompty/prompty/core/model_capabilities.py b/runtime/python/prompty/prompty/core/model_capabilities.py new file mode 100644 index 000000000..1e2c6e328 --- /dev/null +++ b/runtime/python/prompty/prompty/core/model_capabilities.py @@ -0,0 +1,177 @@ +"""Cross-runtime model-capability enrichment for provider discovery. + +Provider ``/models`` endpoints vary in richness: some (Anthropic, Foundry) +return capability fields directly, while others (OpenAI) return only ids. To +keep discovery results consistent across providers *and* across runtimes, +Prompty ships a single shared, provider-keyed capability dataset +(``spec/data/model_capabilities.json``) and applies it with one rule: + + Provider-supplied fields always win. Dataset entries only fill fields + the provider left empty (fill-only-missing). Matching is by longest + prefix on the model id, applied only at token boundaries. + +**Canonical source vs. vendored copy.** The cross-runtime source of truth is +``spec/data/model_capabilities.json``. This package vendors a byte-identical +copy at ``prompty/data/model_capabilities.json`` so the installed package has +no filesystem dependency on the repo's ``spec/`` directory (mirrors +``runtime/rust/prompty/src/discovery.rs``, which embeds the same dataset via +``include_str!``). ``tests/test_model_capabilities.py`` guards against drift +between the two copies whenever the repo layout is available. + +To refresh: edit ``spec/data/model_capabilities.json``, then copy it to +``runtime/python/prompty/prompty/data/model_capabilities.json`` (the drift +guard test will fail until you do). + +This dataset is intentionally **not** emitted from TypeSpec/Typra: it is +volatile provider data (context windows, modalities, new model families) +refreshed as a snapshot, whereas TypeSpec/Typra owns the structural +:class:`~prompty.model.ModelInfo` contract consumed here. + +The Typra-emitted :class:`~prompty.model.ModelInfo` preserves the tri-state +required by fill-only-missing enrichment: omitted modalities remain ``None``, +explicit empty arrays remain ``[]``, and populated arrays retain their values. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Any + +__all__ = ["ModelCapabilities", "enrich", "lookup"] + +_DATA_PATH = Path(__file__).resolve().parent.parent / "data" / "model_capabilities.json" + + +@dataclass(frozen=True) +class ModelCapabilities: + """Fallback capability fields for a single model, as looked up from the shared dataset. + + All fields are optional; ``None`` means "the dataset does not supply this field" and the + caller should leave whatever the provider returned. + """ + + context_window: int | None = None + input_modalities: list[str] | None = None + output_modalities: list[str] | None = None + + +@dataclass(frozen=True) +class _CapabilityEntry: + prefix: str + capabilities: ModelCapabilities + + +def _parse_modalities(value: Any) -> list[str] | None: + """Parse a modality array, distinguishing "absent" (None) from "present but empty" ([]).""" + if not isinstance(value, list): + return None + return [item for item in value if isinstance(item, str)] + + +def _entry_from_value(value: Any) -> _CapabilityEntry | None: + if not isinstance(value, dict): + return None + prefix = value.get("prefix") + if not isinstance(prefix, str): + return None + context_window = value.get("contextWindow") + return _CapabilityEntry( + prefix=prefix, + capabilities=ModelCapabilities( + context_window=context_window + if isinstance(context_window, int) and not isinstance(context_window, bool) + else None, + input_modalities=_parse_modalities(value.get("inputModalities")), + output_modalities=_parse_modalities(value.get("outputModalities")), + ), + ) + + +class _CapabilityTable: + """Provider-keyed capability lookup, parsed once from the vendored dataset.""" + + def __init__(self, providers: dict[str, list[_CapabilityEntry]]) -> None: + self._providers = providers + + @staticmethod + def from_value(value: Any) -> _CapabilityTable: + providers: dict[str, list[_CapabilityEntry]] = {} + raw_providers = value.get("providers") if isinstance(value, dict) else None + if isinstance(raw_providers, dict): + for provider, entries in raw_providers.items(): + if not isinstance(entries, list): + continue + parsed = [e for e in (_entry_from_value(item) for item in entries) if e is not None] + # Longest prefix first, so the first match is the most specific. + parsed.sort(key=lambda e: len(e.prefix), reverse=True) + providers[provider] = parsed + return _CapabilityTable(providers) + + def lookup(self, provider: str, model_id: str) -> ModelCapabilities | None: + for entry in self._providers.get(provider, []): + if _prefix_matches(model_id, entry.prefix): + return entry.capabilities + return None + + +def _prefix_matches(model_id: str, prefix: str) -> bool: + """Whether `model_id` is matched by dataset `prefix` under the cross-runtime rule. + + A prefix matches only at a token boundary: `model_id` must either equal `prefix` + exactly, or the character immediately following the prefix must be a separator + (any non-ASCII-alphanumeric character, e.g. ``-``, ``.``, ``:``). This keeps real + ids matching (``gpt-4`` -> ``gpt-4-0613``, ``gpt-4o`` -> ``gpt-4o-2024-05-13``) + while rejecting accidental substring hits (``gpt-4`` must NOT match a future + ``gpt-45``). Every runtime MUST implement this same boundary rule so the shared + enrichment vectors converge. + """ + if not model_id.startswith(prefix): + return False + rest = model_id[len(prefix) :] + if not rest: + return True + ch = rest[0] + return not (ch.isascii() and ch.isalnum()) + + +@lru_cache(maxsize=1) +def _table() -> _CapabilityTable: + with open(_DATA_PATH, encoding="utf-8") as f: + value = json.load(f) + return _CapabilityTable.from_value(value) + + +def lookup(provider: str, model_id: str) -> ModelCapabilities | None: + """Look up fallback capabilities for a model id within a provider's dataset. + + Returns ``None`` when the provider has no entry matching ``model_id``. Matching is by + longest prefix, applied only at token boundaries (see :func:`_prefix_matches`). + """ + return _table().lookup(provider, model_id) + + +def enrich(provider: str, info: Any) -> None: + """Enrich a :class:`~prompty.model.ModelInfo` in place using the shared capability dataset. + + Applies the cross-runtime fill-only-missing rule: a dataset field is written only when the + corresponding ``ModelInfo`` field is still empty (``context_window`` is ``None``; a modality + list is ``None``). Provider-supplied values are never overwritten. A dataset modality of + ``[]`` (e.g. embeddings) is a valid fill and will replace a ``None`` list. + + Callers MUST have constructed ``info`` with explicit ``None`` for + ``input_modalities``/``output_modalities`` when the provider did not supply them — see the + module docstring for why the generated ``ModelInfo`` default cannot be relied on for this. + """ + caps = lookup(provider, info.id) + if caps is None: + return + + if info.context_window is None and caps.context_window is not None: + info.context_window = caps.context_window + if info.input_modalities is None and caps.input_modalities is not None: + info.input_modalities = caps.input_modalities + if info.output_modalities is None and caps.output_modalities is not None: + info.output_modalities = caps.output_modalities diff --git a/runtime/python/prompty/prompty/data/model_capabilities.json b/runtime/python/prompty/prompty/data/model_capabilities.json new file mode 100644 index 000000000..7c7f6db19 --- /dev/null +++ b/runtime/python/prompty/prompty/data/model_capabilities.json @@ -0,0 +1,55 @@ +{ + "description": "Cross-runtime fallback capability data for provider model discovery. Some provider /models endpoints (Anthropic, Foundry) return capability fields directly; others (OpenAI) return only ids. To keep discovery results consistent across providers AND across runtimes, every Prompty runtime embeds THIS file and applies one shared rule: provider-supplied fields always win; entries here only fill fields the provider left empty (fill-only-missing). Model ids are matched by longest prefix within a provider's list. This dataset is deliberately NOT emitted from TypeSpec: it is volatile provider data (context windows, modalities, new model families) refreshed as a snapshot, whereas TypeSpec owns the structural ModelInfo contract. Fields use the canonical camelCase ModelInfo names. A missing 'contextWindow' means unknown; a present empty modality list (e.g. []) is intentional (e.g. embeddings produce no textual/image output modality).", + "match": "longest_prefix", + "providers": { + "openai": [ + { + "prefix": "gpt-4o-mini", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4o", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4-turbo", + "contextWindow": 128000, + "inputModalities": ["text", "image"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-4", + "contextWindow": 8192, + "inputModalities": ["text"], + "outputModalities": ["text"] + }, + { + "prefix": "gpt-3.5-turbo", + "contextWindow": 16385, + "inputModalities": ["text"], + "outputModalities": ["text"] + }, + { + "prefix": "text-embedding-3-small", + "contextWindow": 8191, + "inputModalities": ["text"], + "outputModalities": [] + }, + { + "prefix": "text-embedding-3-large", + "contextWindow": 8191, + "inputModalities": ["text"], + "outputModalities": [] + }, + { + "prefix": "dall-e-3", + "inputModalities": ["text"], + "outputModalities": ["image"] + } + ] + } +} diff --git a/runtime/python/prompty/prompty/harness/__init__.py b/runtime/python/prompty/prompty/harness/__init__.py index 3b5133894..8f65ef9cc 100644 --- a/runtime/python/prompty/prompty/harness/__init__.py +++ b/runtime/python/prompty/prompty/harness/__init__.py @@ -8,6 +8,7 @@ InMemoryCheckpointStore, JsonlEventJournalWriter, ) +from .engine import ReferenceTurnEngine, load_engine_checkpoint, save_engine_checkpoint from .replay_verifier import ReferenceReplayVerifier from .turn_runner import ( ReferenceTurnRunner, @@ -25,9 +26,12 @@ "InMemoryCheckpointStore", "JsonlEventJournalWriter", "ReferenceReplayVerifier", + "ReferenceTurnEngine", "ReferenceTurnRunner", "RunTurnRequest", "RunTurnResult", "TurnModelRequest", "TurnModelResponse", + "load_engine_checkpoint", + "save_engine_checkpoint", ] diff --git a/runtime/python/prompty/prompty/harness/engine.py b/runtime/python/prompty/prompty/harness/engine.py new file mode 100644 index 000000000..f92d73c6c --- /dev/null +++ b/runtime/python/prompty/prompty/harness/engine.py @@ -0,0 +1,989 @@ +"""Run durable provider-neutral turns with Typra-emitted engine contracts.""" + +from __future__ import annotations + +import asyncio +import inspect +import json +from collections.abc import Awaitable, Callable +from typing import Any, TypeVar + +from ..core.cancellation import CancellationToken +from ..model import ( + EngineCheckpoint, + EngineEvent, + EnginePermissionDecision, + InvocationContextState, + Message, + ModelInvocationContextSnapshot, + ModelInvocationRequest, + ModelInvocationResponse, + ModelReconciliationState, + ModelToolRequest, + ModelToolResult, + ResumeContext, + TextPart, + TurnCommit, + TurnEngineResult, +) + +__all__ = ["ReferenceTurnEngine", "load_engine_checkpoint", "save_engine_checkpoint"] + +_DEFAULT_MAX_ITERATIONS = 10 +_T = TypeVar("_T") + +ModelCallback = Callable[[ModelInvocationRequest], ModelInvocationResponse | Awaitable[ModelInvocationResponse]] +ToolCallback = Callable[[ModelToolRequest], ModelToolResult | Awaitable[ModelToolResult]] +PermissionCallback = Callable[ + [ModelToolRequest], EnginePermissionDecision | bool | Awaitable[EnginePermissionDecision | bool] +] +EventCallback = Callable[[EngineEvent], object | Awaitable[object]] +CheckpointCallback = Callable[[EngineCheckpoint], object | Awaitable[object]] +PostCommitCallback = Callable[[TurnCommit], object | Awaitable[object]] +Clock = Callable[[], str] +IdFactory = Callable[[str], str] + + +async def _resolve(value: _T | Awaitable[_T]) -> _T: + if inspect.isawaitable(value): + return await value + return value + + +def _ensure_sync_entrypoint(sync_name: str, async_name: str) -> None: + try: + asyncio.get_running_loop() + except RuntimeError: + return + raise RuntimeError( + f"ReferenceTurnEngine.{sync_name}() cannot run inside an active event loop; use await {async_name}() instead" + ) + + +def _non_json_output_message(output: Any) -> dict[str, str]: + output_type = f"{type(output).__module__}.{type(output).__qualname__}" + return {"message": f"Tool output is not JSON-serializable (type: {output_type})"} + + +def _default_clock() -> str: + from datetime import UTC, datetime + + return datetime.now(UTC).isoformat().replace("+00:00", "Z") + + +class _SequentialIds: + def __init__(self) -> None: + self._value = 0 + + def __call__(self, kind: str) -> str: + self._value += 1 + return f"{kind}-{self._value}" + + +class _TurnCancelled(Exception): + pass + + +class _TurnFailed(Exception): + def __init__(self, error_kind: str, source: Exception) -> None: + super().__init__(str(source)) + self.error_kind = error_kind + self.source = source + + +def save_engine_checkpoint(checkpoint: EngineCheckpoint) -> dict[str, Any]: + """Serialize a checkpoint using the emitted durable collection shape.""" + return checkpoint.save() + + +def load_engine_checkpoint(data: dict[str, Any]) -> EngineCheckpoint: + """Load a checkpoint serialized by :func:`save_engine_checkpoint`.""" + return EngineCheckpoint.load(data) + + +class ReferenceTurnEngine: + """Execute deterministic turns using the emitted engine model as the public boundary.""" + + def __init__( + self, + *, + invoke_model: ModelCallback, + execute_tool: ToolCallback, + authorize: PermissionCallback | None = None, + on_event: EventCallback | None = None, + save_checkpoint: CheckpointCallback | None = None, + post_commit: PostCommitCallback | None = None, + now: Clock | None = None, + next_id: IdFactory | None = None, + ) -> None: + self._invoke_model = invoke_model + self._execute_tool = execute_tool + self._authorize = authorize + self._on_event = on_event + self._save_checkpoint = save_checkpoint + self._post_commit = post_commit + self._now = now or _default_clock + self._next_id = next_id or _SequentialIds() + self._sequence = 0 + self._session_id = "" + self._turn_id = "" + self._run_id = "" + self._parent_run_id: str | None = None + self._delegation_depth = 0 + + def run( + self, + session_id: str, + turn_id: str, + messages: list[Message], + *, + inputs: Any | None = None, + max_iterations: int = _DEFAULT_MAX_ITERATIONS, + max_model_attempts: int = 3, + cancellation: CancellationToken | None = None, + run_id: str | None = None, + parent_run_id: str | None = None, + delegation_depth: int = 0, + ) -> TurnEngineResult: + """Run a new turn synchronously.""" + _ensure_sync_entrypoint("run", "run_async") + return asyncio.run( + self.run_async( + session_id, + turn_id, + messages, + inputs=inputs, + max_iterations=max_iterations, + max_model_attempts=max_model_attempts, + cancellation=cancellation, + run_id=run_id, + parent_run_id=parent_run_id, + delegation_depth=delegation_depth, + ) + ) + + async def run_async( + self, + session_id: str, + turn_id: str, + messages: list[Message], + *, + inputs: Any | None = None, + max_iterations: int = _DEFAULT_MAX_ITERATIONS, + max_model_attempts: int = 3, + cancellation: CancellationToken | None = None, + run_id: str | None = None, + parent_run_id: str | None = None, + delegation_depth: int = 0, + ) -> TurnEngineResult: + """Run a new turn and return its emitted commit, snapshots, and tool results.""" + effective_model_attempts = max_model_attempts if max_model_attempts > 0 else 3 + self._start_run( + session_id, + turn_id, + run_id=run_id, + parent_run_id=parent_run_id, + delegation_depth=delegation_depth, + ) + await self._emit("turn_started") + return await self._drive( + messages=list(messages), + inputs=inputs, + max_iterations=max_iterations, + max_model_attempts=effective_model_attempts, + cancellation=cancellation, + iteration=0, + stable_prefix_messages=len(messages), + context_state=InvocationContextState(), + snapshots=[], + tool_results=[], + ) + + def resume( + self, + context: ResumeContext, + *, + cancellation: CancellationToken | None = None, + ) -> TurnEngineResult: + """Resume a durable checkpoint synchronously without repeating committed effects.""" + _ensure_sync_entrypoint("resume", "resume_async") + return asyncio.run(self.resume_async(context, cancellation=cancellation)) + + async def resume_async( + self, + context: ResumeContext, + *, + cancellation: CancellationToken | None = None, + ) -> TurnEngineResult: + """Resume a durable checkpoint without repeating committed model or tool effects.""" + checkpoint = context.checkpoint + self._session_id = checkpoint.session_id + self._turn_id = checkpoint.turn_id + self._run_id = checkpoint.run_id or self._next_id("run") + self._parent_run_id = checkpoint.parent_run_id + self._delegation_depth = checkpoint.delegation_depth + self._sequence = max(checkpoint.last_sequence, context.last_journal_sequence) + await self._emit("turn_started", payload={"resumedFrom": checkpoint.id}) + + max_iterations = context.max_iterations + max_model_attempts = context.max_model_attempts if context.max_model_attempts > 0 else 3 + snapshots: list[ModelInvocationContextSnapshot] = [] + tool_results = list(checkpoint.completed_tool_results or []) + messages = list(checkpoint.messages) + context_state = checkpoint.context_state + + if checkpoint.reconciliation_required: + return await self._reconciliation_required( + messages=messages, + iterations=checkpoint.completed_model_iterations, + context_state=context_state, + snapshots=snapshots, + tool_results=tool_results, + model_reconciliation=checkpoint.model_reconciliation, + ) + + if cancellation is not None and cancellation.is_cancelled: + await self._emit("turn_cancelled", iteration=checkpoint.iteration) + return self._cancelled( + messages, + checkpoint.completed_model_iterations, + context_state, + snapshots, + tool_results, + model_reconciliation=checkpoint.model_reconciliation, + ) + + if checkpoint.final_output_ready: + return await self._commit( + status="success", + output=checkpoint.pending_output, + messages=messages, + iterations=checkpoint.completed_model_iterations, + context_state=context_state, + snapshots=snapshots, + tool_results=tool_results, + ) + + if checkpoint.pending_tool_requests: + response = checkpoint.pending_model_response or ModelInvocationResponse() + pending_ids = {request.id for request in checkpoint.pending_tool_requests} + completed_round_results = [result for result in tool_results if result.request_id in pending_ids] + try: + new_results = await self._execute_pending_tools( + checkpoint.pending_tool_requests, + tool_results, + iteration=checkpoint.iteration, + cancellation=cancellation, + messages=messages, + inputs=checkpoint.inputs, + context_state=context_state, + response=response, + stable_prefix_messages=checkpoint.stable_prefix_messages, + active_invocation_id=checkpoint.active_invocation_id, + ) + except _TurnCancelled: + await self._emit("turn_cancelled", iteration=checkpoint.iteration) + return self._cancelled( + messages, + checkpoint.completed_model_iterations, + context_state, + snapshots, + tool_results, + ) + except _TurnFailed as exc: + await self._emit( + "turn_failed", + iteration=checkpoint.iteration, + payload={"errorKind": exc.error_kind, "message": str(exc)}, + ) + return self._failed( + messages, + checkpoint.completed_model_iterations, + context_state, + snapshots, + tool_results, + exc.source, + error_kind=exc.error_kind, + ) + tool_results.extend(new_results) + if any(result.outcome == "indeterminate" for result in new_results): + return await self._reconciliation_required( + messages=messages, + iterations=checkpoint.completed_model_iterations, + context_state=context_state, + snapshots=snapshots, + tool_results=tool_results, + ) + round_results = self._ordered_round_results( + checkpoint.pending_tool_requests, + [*completed_round_results, *new_results], + ) + if round_results is None: + error = RuntimeError("Tool results do not match the pending request batch") + await self._emit( + "turn_failed", + iteration=checkpoint.iteration, + payload={"errorKind": "conversation_format_error", "message": str(error)}, + ) + return self._failed( + messages, + checkpoint.completed_model_iterations, + context_state, + snapshots, + tool_results, + error, + error_kind="conversation_format_error", + ) + for result in round_results: + await self._emit("tool_result_committed", iteration=checkpoint.iteration, payload=result.save()) + messages.extend(response.assistant_messages or []) + messages.extend(self._tool_result_messages(round_results)) + await self._emit( + "conversation_updated", + iteration=checkpoint.iteration, + payload={"toolResults": [result.save() for result in round_results]}, + ) + await self._checkpoint( + iteration=checkpoint.iteration, + messages=messages, + stable_prefix_messages=checkpoint.stable_prefix_messages, + inputs=checkpoint.inputs, + context_state=context_state, + completed_model_iterations=checkpoint.completed_model_iterations, + completed_tool_results=tool_results, + ) + + iteration = checkpoint.iteration if checkpoint.resume_same_iteration else checkpoint.completed_model_iterations + return await self._drive( + messages=messages, + inputs=checkpoint.inputs, + max_iterations=max_iterations, + max_model_attempts=max_model_attempts, + cancellation=cancellation, + iteration=iteration, + stable_prefix_messages=checkpoint.stable_prefix_messages, + context_state=context_state, + snapshots=snapshots, + tool_results=tool_results, + ) + + def _start_run( + self, + session_id: str, + turn_id: str, + *, + run_id: str | None, + parent_run_id: str | None, + delegation_depth: int, + ) -> None: + self._session_id = session_id + self._turn_id = turn_id + self._run_id = run_id or self._next_id("run") + self._parent_run_id = parent_run_id + self._delegation_depth = delegation_depth + self._sequence = 0 + + async def _drive( + self, + *, + messages: list[Message], + inputs: Any | None, + max_iterations: int, + max_model_attempts: int, + cancellation: CancellationToken | None, + iteration: int, + stable_prefix_messages: int, + context_state: InvocationContextState, + snapshots: list[ModelInvocationContextSnapshot], + tool_results: list[ModelToolResult], + ) -> TurnEngineResult: + while iteration < max_iterations: + if cancellation is not None and cancellation.is_cancelled: + await self._emit("turn_cancelled", iteration=iteration) + return self._cancelled(messages, iteration, context_state, snapshots, tool_results) + + invocation_id = self._next_id("invocation") + snapshot = ModelInvocationContextSnapshot( + id=self._next_id("snapshot"), + session_id=self._session_id, + turn_id=self._turn_id, + invocation_id=invocation_id, + iteration=iteration, + messages=list(messages), + stable_prefix_messages=min(stable_prefix_messages, len(messages)), + context_state=context_state, + ) + snapshots.append(snapshot) + await self._emit("context_prepared", invocation_id=invocation_id, iteration=iteration) + model_request = ModelInvocationRequest(context=snapshot) + response: ModelInvocationResponse | None = None + attempt_limit = max(1, max_model_attempts) + for attempt in range(attempt_limit): + if cancellation is not None and cancellation.is_cancelled: + await self._emit("turn_cancelled", invocation_id=invocation_id, iteration=iteration) + return self._cancelled(messages, iteration, context_state, snapshots, tool_results) + await self._emit( + "model_invocation_started", + invocation_id=invocation_id, + iteration=iteration, + payload={"attempt": attempt}, + ) + try: + candidate = await _resolve(self._invoke_model(model_request)) + if not isinstance(candidate, ModelInvocationResponse): + raise TypeError("invoke_model must return ModelInvocationResponse") + response = candidate + break + except Exception as exc: + outcome_unknown = bool(getattr(exc, "outcome_unknown", False)) + exhausted = outcome_unknown or attempt + 1 >= attempt_limit + await self._emit( + "model_invocation_failed", + invocation_id=invocation_id, + iteration=iteration, + payload={ + "attempt": attempt, + "exhausted": exhausted, + "outcomeUnknown": outcome_unknown, + "message": str(exc), + }, + ) + if outcome_unknown: + metadata = getattr(exc, "metadata", None) + reconciliation = ModelReconciliationState( + invocation_id=invocation_id, + request=model_request, + failed_attempt=attempt, + message=str(exc), + metadata=metadata if isinstance(metadata, dict) else None, + ) + await self._checkpoint( + iteration=iteration, + messages=messages, + stable_prefix_messages=stable_prefix_messages, + inputs=inputs, + context_state=context_state, + completed_model_iterations=iteration, + active_invocation_id=invocation_id, + completed_tool_results=tool_results, + reconciliation_required=True, + model_reconciliation=reconciliation, + resume_same_iteration=True, + ) + return await self._reconciliation_required( + messages=messages, + iterations=iteration, + context_state=context_state, + snapshots=snapshots, + tool_results=tool_results, + model_reconciliation=reconciliation, + ) + if exhausted: + await self._emit( + "turn_failed", + iteration=iteration, + payload={"errorKind": "model_error", "message": str(exc)}, + ) + return self._failed( + messages, + iteration, + context_state, + snapshots, + tool_results, + exc, + error_kind="model_error", + ) + if response is None: + raise RuntimeError("Model invocation attempt loop completed without a response") + await self._emit("model_invocation_completed", invocation_id=invocation_id, iteration=iteration) + + completed_iterations = iteration + 1 + if response.next_context_state is not None: + validation_error = self._validate_context_state(response.next_context_state) + if validation_error is not None: + await self._emit( + "turn_failed", + invocation_id=invocation_id, + iteration=iteration, + payload={"errorKind": "provider_state_error", "message": validation_error}, + ) + return self._failed( + messages, + completed_iterations, + context_state, + snapshots, + tool_results, + RuntimeError(validation_error), + error_kind="provider_state_error", + ) + context_state = response.next_context_state + if not response.tool_requests: + messages.extend(response.assistant_messages or []) + await self._checkpoint( + iteration=iteration, + messages=messages, + stable_prefix_messages=stable_prefix_messages, + inputs=inputs, + context_state=context_state, + completed_model_iterations=completed_iterations, + pending_tool_requests=response.tool_requests, + pending_model_response=response if response.tool_requests else None, + pending_output=response.output, + final_output_ready=not response.tool_requests, + completed_tool_results=tool_results, + active_invocation_id=invocation_id, + ) + if cancellation is not None and cancellation.is_cancelled: + await self._emit("turn_cancelled", invocation_id=invocation_id, iteration=iteration) + return self._cancelled(messages, completed_iterations, context_state, snapshots, tool_results) + + if not response.tool_requests: + return await self._commit( + status="success", + output=response.output, + messages=messages, + iterations=completed_iterations, + context_state=context_state, + snapshots=snapshots, + tool_results=tool_results, + ) + + try: + round_results = await self._execute_pending_tools( + response.tool_requests, + tool_results, + iteration=iteration, + cancellation=cancellation, + messages=messages, + inputs=inputs, + context_state=context_state, + response=response, + stable_prefix_messages=stable_prefix_messages, + active_invocation_id=invocation_id, + ) + except _TurnCancelled: + await self._emit("turn_cancelled", invocation_id=invocation_id, iteration=iteration) + return self._cancelled(messages, completed_iterations, context_state, snapshots, tool_results) + except _TurnFailed as exc: + await self._emit( + "turn_failed", + invocation_id=invocation_id, + iteration=iteration, + payload={"errorKind": exc.error_kind, "message": str(exc)}, + ) + return self._failed( + messages, + completed_iterations, + context_state, + snapshots, + tool_results, + exc.source, + error_kind=exc.error_kind, + ) + tool_results.extend(round_results) + if any(result.outcome == "indeterminate" for result in round_results): + return await self._reconciliation_required( + messages=messages, + iterations=completed_iterations, + context_state=context_state, + snapshots=snapshots, + tool_results=tool_results, + ) + ordered_results = self._ordered_round_results(response.tool_requests, round_results) + if ordered_results is None: + error = RuntimeError("Tool results do not match the pending request batch") + await self._emit( + "turn_failed", + invocation_id=invocation_id, + iteration=iteration, + payload={"errorKind": "conversation_format_error", "message": str(error)}, + ) + return self._failed( + messages, + completed_iterations, + context_state, + snapshots, + tool_results, + error, + error_kind="conversation_format_error", + ) + round_results = ordered_results + for result in round_results: + await self._emit("tool_result_committed", iteration=iteration, payload=result.save()) + messages.extend(response.assistant_messages or []) + messages.extend(self._tool_result_messages(round_results)) + await self._emit( + "conversation_updated", + iteration=iteration, + payload={"toolResults": [result.save() for result in round_results]}, + ) + await self._checkpoint( + iteration=iteration, + messages=messages, + stable_prefix_messages=stable_prefix_messages, + inputs=inputs, + context_state=context_state, + completed_model_iterations=completed_iterations, + completed_tool_results=tool_results, + ) + iteration = completed_iterations + + error = RuntimeError(f"Turn exceeded max_iterations ({max_iterations})") + await self._emit( + "turn_failed", + iteration=iteration, + payload={"errorKind": "max_iterations", "message": str(error)}, + ) + return self._failed( + messages, + iteration, + context_state, + snapshots, + tool_results, + error, + error_kind="max_iterations", + ) + + async def _execute_pending_tools( + self, + requests: list[ModelToolRequest], + completed: list[ModelToolResult], + *, + iteration: int, + cancellation: CancellationToken | None, + messages: list[Message], + inputs: Any | None, + context_state: InvocationContextState, + response: ModelInvocationResponse, + stable_prefix_messages: int, + active_invocation_id: str | None, + ) -> list[ModelToolResult]: + completed_ids = {result.request_id for result in completed} + results: list[ModelToolResult] = [] + for request in requests: + if request.id in completed_ids: + continue + if cancellation is not None and cancellation.is_cancelled: + raise _TurnCancelled + + await self._emit("permission_requested", iteration=iteration, payload=request.save()) + try: + decision = await self._permission(request) + except Exception as exc: + raise _TurnFailed("permission_error", exc) from exc + await self._emit("permission_resolved", iteration=iteration, payload=decision.save()) + if not decision.approved: + result = ModelToolResult( + request_id=request.id, + name=request.name, + outcome="failed", + output={"message": decision.reason or "Permission denied"}, + error_kind="permission_denied", + ) + else: + await self._emit("tool_execution_started", iteration=iteration, payload=request.save()) + try: + result = await _resolve(self._execute_tool(request)) + except Exception as exc: + if isinstance(exc, KeyError) or getattr(exc, "error_kind", None) == "tool_configuration_error": + raise _TurnFailed("tool_configuration_error", exc) from exc + result = ModelToolResult( + request_id=request.id, + name=request.name, + outcome="failed", + output={"message": str(exc)}, + error_kind="exception", + ) + if not isinstance(result, ModelToolResult): + raise TypeError("execute_tool must return ModelToolResult") + result = self._normalize_tool_result(result) + await self._emit("tool_execution_completed", iteration=iteration, payload=result.save()) + results.append(result) + await self._checkpoint( + iteration=iteration, + messages=messages, + stable_prefix_messages=stable_prefix_messages, + inputs=inputs, + context_state=context_state, + completed_model_iterations=iteration + 1, + pending_tool_requests=requests, + completed_tool_results=[*completed, *results], + pending_model_response=response, + active_invocation_id=active_invocation_id, + reconciliation_required=any(item.outcome == "indeterminate" for item in [*completed, *results]), + ) + if result.outcome == "indeterminate": + return results + return results + + async def _permission(self, request: ModelToolRequest) -> EnginePermissionDecision: + if self._authorize is None: + return EnginePermissionDecision(approved=True, reason="allow_all") + decision = await _resolve(self._authorize(request)) + if isinstance(decision, bool): + return EnginePermissionDecision(approved=decision) + if not isinstance(decision, EnginePermissionDecision): + raise TypeError("authorize must return bool or EnginePermissionDecision") + return decision + + async def _checkpoint( + self, + *, + iteration: int, + messages: list[Message], + stable_prefix_messages: int, + inputs: Any | None, + context_state: InvocationContextState, + completed_model_iterations: int, + pending_tool_requests: list[ModelToolRequest] | None = None, + completed_tool_results: list[ModelToolResult] | None = None, + pending_output: Any | None = None, + final_output_ready: bool = False, + pending_model_response: ModelInvocationResponse | None = None, + active_invocation_id: str | None = None, + reconciliation_required: bool = False, + model_reconciliation: ModelReconciliationState | None = None, + resume_same_iteration: bool = False, + ) -> EngineCheckpoint: + checkpoint = EngineCheckpoint( + id=self._next_id("checkpoint"), + session_id=self._session_id, + turn_id=self._turn_id, + run_id=self._run_id, + parent_run_id=self._parent_run_id, + delegation_depth=self._delegation_depth, + iteration=iteration, + last_sequence=self._sequence, + messages=list(messages), + stable_prefix_messages=stable_prefix_messages, + inputs=inputs, + pending_tool_requests=list(pending_tool_requests or []), + completed_tool_results=list(completed_tool_results or []), + completed_model_iterations=completed_model_iterations, + reconciliation_required=reconciliation_required, + model_reconciliation=model_reconciliation, + pending_output=pending_output, + final_output_ready=final_output_ready, + pending_model_response=pending_model_response, + active_invocation_id=active_invocation_id, + resume_same_iteration=resume_same_iteration, + context_state=context_state, + ) + if self._save_checkpoint is not None: + await _resolve(self._save_checkpoint(checkpoint)) + await self._emit( + "checkpoint_created", + invocation_id=active_invocation_id, + iteration=iteration, + payload={"checkpointId": checkpoint.id, "includedThroughSequence": checkpoint.last_sequence}, + ) + return checkpoint + + async def _commit( + self, + *, + status: str, + output: Any | None, + messages: list[Message], + iterations: int, + context_state: InvocationContextState, + snapshots: list[ModelInvocationContextSnapshot], + tool_results: list[ModelToolResult], + model_reconciliation: ModelReconciliationState | None = None, + ) -> TurnEngineResult: + event = await self._emit("turn_committed", iteration=iterations, payload={"status": status}) + commit = TurnCommit( + session_id=self._session_id, + turn_id=self._turn_id, + status=status, + output=output, + messages=list(messages), + iterations=iterations, + last_sequence=event.sequence, + context_state=context_state, + ) + post_commit_error: str | None = None + await self._emit("post_commit_started", iteration=iterations) + try: + if self._post_commit is not None: + await _resolve(self._post_commit(commit)) + except Exception as exc: + post_commit_error = str(exc) + await self._emit("post_commit_failed", iteration=iterations, payload={"message": post_commit_error}) + else: + await self._emit("post_commit_completed", iteration=iterations) + commit.last_sequence = self._sequence + return TurnEngineResult( + commit=commit, + snapshots=snapshots, + tool_results=tool_results, + post_commit_error=post_commit_error, + ) + + async def _reconciliation_required( + self, + *, + messages: list[Message], + iterations: int, + context_state: InvocationContextState, + snapshots: list[ModelInvocationContextSnapshot], + tool_results: list[ModelToolResult], + model_reconciliation: ModelReconciliationState | None = None, + ) -> TurnEngineResult: + event = await self._emit( + "turn_reconciliation_required", + iteration=iterations, + payload={"errorKind": "effect_outcome_unknown"}, + ) + return TurnEngineResult( + commit=TurnCommit( + session_id=self._session_id, + turn_id=self._turn_id, + status="reconciliation_required", + output={ + "errorKind": "effect_outcome_unknown", + "message": "An external effect requires reconciliation before the turn can continue", + }, + messages=list(messages), + iterations=iterations, + last_sequence=event.sequence, + context_state=context_state, + model_reconciliation=model_reconciliation, + ), + snapshots=snapshots, + tool_results=tool_results, + ) + + def _cancelled( + self, + messages: list[Message], + iterations: int, + context_state: InvocationContextState, + snapshots: list[ModelInvocationContextSnapshot], + tool_results: list[ModelToolResult], + model_reconciliation: ModelReconciliationState | None = None, + ) -> TurnEngineResult: + return TurnEngineResult( + commit=TurnCommit( + session_id=self._session_id, + turn_id=self._turn_id, + status="cancelled", + messages=list(messages), + iterations=iterations, + last_sequence=self._sequence, + context_state=context_state, + model_reconciliation=model_reconciliation, + ), + snapshots=snapshots, + tool_results=tool_results, + ) + + def _failed( + self, + messages: list[Message], + iterations: int, + context_state: InvocationContextState, + snapshots: list[ModelInvocationContextSnapshot], + tool_results: list[ModelToolResult], + error: Exception, + *, + error_kind: str = "engine_error", + ) -> TurnEngineResult: + return TurnEngineResult( + commit=TurnCommit( + session_id=self._session_id, + turn_id=self._turn_id, + status="failed", + output={"errorKind": error_kind, "message": str(error)}, + messages=list(messages), + iterations=iterations, + last_sequence=self._sequence, + context_state=context_state, + ), + snapshots=snapshots, + tool_results=tool_results, + ) + + async def _emit( + self, + kind: Any, + *, + invocation_id: str | None = None, + iteration: int | None = None, + payload: Any | None = None, + ) -> EngineEvent: + self._sequence += 1 + event = EngineEvent( + sequence=self._sequence, + id=self._next_id("event"), + timestamp=self._now(), + session_id=self._session_id, + turn_id=self._turn_id, + run_id=self._run_id, + parent_run_id=self._parent_run_id, + delegation_depth=self._delegation_depth, + invocation_id=invocation_id, + iteration=iteration, + kind=kind, + payload=payload, + ) + if self._on_event is not None: + await _resolve(self._on_event(event)) + return event + + @staticmethod + def _tool_result_messages(results: list[ModelToolResult]) -> list[Message]: + messages: list[Message] = [] + for result in results: + if result.output is None: + value = "" + elif isinstance(result.output, str): + value = result.output + else: + try: + value = json.dumps(result.output) + except (TypeError, ValueError): + value = json.dumps(_non_json_output_message(result.output)) + messages.append( + Message( + role="tool", + parts=[TextPart(value=value)], + metadata={ + "tool_call_id": result.request_id, + }, + ) + ) + return messages + + @staticmethod + def _normalize_tool_result(result: ModelToolResult) -> ModelToolResult: + try: + json.dumps(result.output) + except (TypeError, ValueError): + return ModelToolResult( + request_id=result.request_id, + name=result.name, + outcome="failed" if result.outcome == "success" else result.outcome, + output=_non_json_output_message(result.output), + error_kind="invalid_output" if result.outcome == "success" else result.error_kind or "invalid_output", + metadata=result.metadata, + ) + return result + + @staticmethod + def _validate_context_state(state: InvocationContextState) -> str | None: + if state.portability == "portable" and state.delegated_state: + return "Portable context state cannot contain delegated provider references" + if state.portability == "delegated" and not state.delegated_state: + return "Delegated context state requires at least one provider reference" + return None + + @staticmethod + def _ordered_round_results( + requests: list[ModelToolRequest], + results: list[ModelToolResult], + ) -> list[ModelToolResult] | None: + by_id = {result.request_id: result for result in results} + if len(by_id) != len(requests) or any(request.id not in by_id for request in requests): + return None + return [by_id[request.id] for request in requests] diff --git a/runtime/python/prompty/prompty/harness/turn_runner.py b/runtime/python/prompty/prompty/harness/turn_runner.py index 0b9ecdef5..2fac17008 100644 --- a/runtime/python/prompty/prompty/harness/turn_runner.py +++ b/runtime/python/prompty/prompty/harness/turn_runner.py @@ -243,7 +243,7 @@ async def _resolve_and_execute_tool( self._record_turn("permission_completed", turn_id, iteration, decision.save()) if not decision.approved: - return HostToolResult( + result = HostToolResult( request_id=tool_request.request_id, tool_call_id=tool_request.tool_call_id, tool_name=tool_request.tool_name, @@ -251,6 +251,8 @@ async def _resolve_and_execute_tool( error_kind="permission_denied", result={"message": decision.reason or "Permission denied"}, ) + self._record_turn("tool_result", turn_id, iteration, result.save()) + return result self._record_turn("tool_execution_start", turn_id, iteration, tool_request.save()) result = await self.host_tool_executor.execute(tool_request) diff --git a/runtime/python/prompty/prompty/model/agent/_Prompty.py b/runtime/python/prompty/prompty/model/agent/_Prompty.py index 174dd5d1b..0a363f014 100644 --- a/runtime/python/prompty/prompty/model/agent/_Prompty.py +++ b/runtime/python/prompty/prompty/model/agent/_Prompty.py @@ -60,10 +60,10 @@ class or kind discriminator. A .prompty file always produces a Prompty instance. display_name: str | None = None description: str | None = None metadata: dict[str, Any] | None = None - inputs: list[Property] = field(default_factory=list) - outputs: list[Property] = field(default_factory=list) + inputs: list[Property] | None = None + outputs: list[Property] | None = None model: Model = field(default_factory=Model) - tools: list[Tool] = field(default_factory=list) + tools: list[Tool] | None = None template: Template | None = None instructions: str | None = None diff --git a/runtime/python/prompty/prompty/model/connection/_Connection.py b/runtime/python/prompty/prompty/model/connection/_Connection.py index 9df3d76fa..b024d34d2 100644 --- a/runtime/python/prompty/prompty/model/connection/_Connection.py +++ b/runtime/python/prompty/prompty/model/connection/_Connection.py @@ -558,7 +558,7 @@ class OAuthConnection(Connection): client_id: str = field(default="") client_secret: str = field(default="") token_url: str = field(default="") - scopes: list[str] = field(default_factory=list) + scopes: list[str] | None = None @staticmethod def load(data: Any, context: LoadContext | None = None) -> "OAuthConnection": diff --git a/runtime/python/prompty/prompty/model/conversation/_Message.py b/runtime/python/prompty/prompty/model/conversation/_Message.py index 865583606..a71c9cb65 100644 --- a/runtime/python/prompty/prompty/model/conversation/_Message.py +++ b/runtime/python/prompty/prompty/model/conversation/_Message.py @@ -85,7 +85,7 @@ def save_parts(items: list[ContentPart], context: SaveContext | None) -> dict[st if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/conversation/_ToolResult.py b/runtime/python/prompty/prompty/model/conversation/_ToolResult.py index 332bbd181..2fe6b6bbf 100644 --- a/runtime/python/prompty/prompty/model/conversation/_ToolResult.py +++ b/runtime/python/prompty/prompty/model/conversation/_ToolResult.py @@ -98,7 +98,7 @@ def save_parts(items: list[ContentPart], context: SaveContext | None) -> dict[st if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/core/_Property.py b/runtime/python/prompty/prompty/model/core/_Property.py index 65dcf670f..750ac3e6a 100644 --- a/runtime/python/prompty/prompty/model/core/_Property.py +++ b/runtime/python/prompty/prompty/model/core/_Property.py @@ -49,7 +49,7 @@ class Property: nullable: bool | None = None default: Any | None = None example: Any | None = None - enum_values: list[Any] = field(default_factory=list) + enum_values: list[Any] | None = None @staticmethod def load(data: Any, context: LoadContext | None = None) -> "Property": @@ -453,8 +453,8 @@ class UnionProperty(Property): _shorthand_property: ClassVar[str | None] = None kind: str = field(default="union") - one_of: list[Property] = field(default_factory=list) - any_of: list[Property] = field(default_factory=list) + one_of: list[Property] | None = None + any_of: list[Property] | None = None @staticmethod def load(data: Any, context: LoadContext | None = None) -> "UnionProperty": @@ -506,7 +506,7 @@ def save_one_of(items: list[Property], context: SaveContext | None) -> dict[str, if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] @staticmethod @@ -529,7 +529,7 @@ def save_any_of(items: list[Property], context: SaveContext | None) -> dict[str, if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/core/_ValidationResult.py b/runtime/python/prompty/prompty/model/core/_ValidationResult.py index d28edd55a..f0ba99bd2 100644 --- a/runtime/python/prompty/prompty/model/core/_ValidationResult.py +++ b/runtime/python/prompty/prompty/model/core/_ValidationResult.py @@ -79,7 +79,7 @@ def save_errors(items: list[ValidationError], context: SaveContext | None) -> di if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/events/_DoneEventPayload.py b/runtime/python/prompty/prompty/model/events/_DoneEventPayload.py index cb5772233..03755f0f6 100644 --- a/runtime/python/prompty/prompty/model/events/_DoneEventPayload.py +++ b/runtime/python/prompty/prompty/model/events/_DoneEventPayload.py @@ -77,7 +77,7 @@ def save_messages(items: list[Message], context: SaveContext | None) -> dict[str if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/events/_MessagesUpdatedPayload.py b/runtime/python/prompty/prompty/model/events/_MessagesUpdatedPayload.py index 42f8d6124..31dafc9ba 100644 --- a/runtime/python/prompty/prompty/model/events/_MessagesUpdatedPayload.py +++ b/runtime/python/prompty/prompty/model/events/_MessagesUpdatedPayload.py @@ -5,7 +5,7 @@ # ANY EDITS WILL BE LOST ########################################## -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any, ClassVar from .._context import LoadContext, SaveContext @@ -30,9 +30,9 @@ class MessagesUpdatedPayload: _shorthand_property: ClassVar[str | None] = None - messages: list[Message] = field(default_factory=list) + messages: list[Message] | None = None reason: str | None = None - appended: list[Message] = field(default_factory=list) + appended: list[Message] | None = None removed: int | None = None @staticmethod @@ -87,7 +87,7 @@ def save_messages(items: list[Message], context: SaveContext | None) -> dict[str if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] @staticmethod @@ -110,7 +110,7 @@ def save_appended(items: list[Message], context: SaveContext | None) -> dict[str if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/events/_RedactionMetadata.py b/runtime/python/prompty/prompty/model/events/_RedactionMetadata.py index 0c325b015..4b273ba40 100644 --- a/runtime/python/prompty/prompty/model/events/_RedactionMetadata.py +++ b/runtime/python/prompty/prompty/model/events/_RedactionMetadata.py @@ -5,7 +5,7 @@ # ANY EDITS WILL BE LOST ########################################## -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any, ClassVar from .._context import LoadContext, SaveContext @@ -29,7 +29,7 @@ class RedactionMetadata: _shorthand_property: ClassVar[str | None] = None sanitized: bool | None = None - fields: list[RedactedField] = field(default_factory=list) + fields: list[RedactedField] | None = None policy: str | None = None @staticmethod @@ -82,7 +82,7 @@ def save_fields(items: list[RedactedField], context: SaveContext | None) -> dict if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/events/_SessionTrace.py b/runtime/python/prompty/prompty/model/events/_SessionTrace.py index 70bae77e8..b1e0bbf37 100644 --- a/runtime/python/prompty/prompty/model/events/_SessionTrace.py +++ b/runtime/python/prompty/prompty/model/events/_SessionTrace.py @@ -55,11 +55,11 @@ class SessionTrace: prompty_version: str | None = None session_id: str | None = None events: list[SessionEvent] = field(default_factory=list) - turns: list[TurnTrace] = field(default_factory=list) - checkpoints: list[Checkpoint] = field(default_factory=list) - trajectory: list[TrajectoryEvent] = field(default_factory=list) - files: list[SessionFileRef] = field(default_factory=list) - refs: list[SessionRef] = field(default_factory=list) + turns: list[TurnTrace] | None = None + checkpoints: list[Checkpoint] | None = None + trajectory: list[TrajectoryEvent] | None = None + files: list[SessionFileRef] | None = None + refs: list[SessionRef] | None = None summary: SessionSummary | None = None @staticmethod @@ -128,7 +128,7 @@ def save_events(items: list[SessionEvent], context: SaveContext | None) -> dict[ if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] @staticmethod @@ -151,7 +151,7 @@ def save_turns(items: list[TurnTrace], context: SaveContext | None) -> dict[str, if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] @staticmethod @@ -174,7 +174,7 @@ def save_checkpoints(items: list[Checkpoint], context: SaveContext | None) -> di if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] @staticmethod @@ -199,7 +199,7 @@ def save_trajectory( if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] @staticmethod @@ -222,7 +222,7 @@ def save_files(items: list[SessionFileRef], context: SaveContext | None) -> dict if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] @staticmethod @@ -245,7 +245,7 @@ def save_refs(items: list[SessionRef], context: SaveContext | None) -> dict[str, if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/events/_TurnTrace.py b/runtime/python/prompty/prompty/model/events/_TurnTrace.py index 99798bab3..b78bd3db9 100644 --- a/runtime/python/prompty/prompty/model/events/_TurnTrace.py +++ b/runtime/python/prompty/prompty/model/events/_TurnTrace.py @@ -93,7 +93,7 @@ def save_events(items: list[TurnEvent], context: SaveContext | None) -> dict[str if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/memory/_MemoryEntry.py b/runtime/python/prompty/prompty/model/memory/_MemoryEntry.py index 7f82cc1da..529c47f81 100644 --- a/runtime/python/prompty/prompty/model/memory/_MemoryEntry.py +++ b/runtime/python/prompty/prompty/model/memory/_MemoryEntry.py @@ -43,7 +43,7 @@ class MemoryEntry: content: str = field(default="") category: MemoryCategory = field(default="core") created_at: str | None = None - tags: list[str] = field(default_factory=list) + tags: list[str] | None = None @staticmethod def load(data: Any, context: LoadContext | None = None) -> "MemoryEntry": diff --git a/runtime/python/prompty/prompty/model/memory/_MemoryStore.py b/runtime/python/prompty/prompty/model/memory/_MemoryStore.py index 87494cf68..f928d672f 100644 --- a/runtime/python/prompty/prompty/model/memory/_MemoryStore.py +++ b/runtime/python/prompty/prompty/model/memory/_MemoryStore.py @@ -77,7 +77,7 @@ def save_entries(items: list[MemoryEntry], context: SaveContext | None) -> dict[ if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/model/_ModelInfo.py b/runtime/python/prompty/prompty/model/model/_ModelInfo.py index 263038fdb..8ff2657cb 100644 --- a/runtime/python/prompty/prompty/model/model/_ModelInfo.py +++ b/runtime/python/prompty/prompty/model/model/_ModelInfo.py @@ -44,8 +44,8 @@ class ModelInfo: display_name: str | None = None owned_by: str | None = None context_window: int | None = None - input_modalities: list[str] = field(default_factory=list) - output_modalities: list[str] = field(default_factory=list) + input_modalities: list[str] | None = None + output_modalities: list[str] | None = None additional_properties: dict[str, Any] | None = None @staticmethod diff --git a/runtime/python/prompty/prompty/model/model/_ModelOptions.py b/runtime/python/prompty/prompty/model/model/_ModelOptions.py index a73310d76..e62cb2872 100644 --- a/runtime/python/prompty/prompty/model/model/_ModelOptions.py +++ b/runtime/python/prompty/prompty/model/model/_ModelOptions.py @@ -5,7 +5,7 @@ # ANY EDITS WILL BE LOST ########################################## -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any, ClassVar from .._context import LoadContext, SaveContext @@ -48,7 +48,7 @@ class ModelOptions: temperature: float | None = None top_k: int | None = None top_p: float | None = None - stop_sequences: list[str] = field(default_factory=list) + stop_sequences: list[str] | None = None allow_multiple_tool_calls: bool | None = None additional_properties: dict[str, Any] | None = None diff --git a/runtime/python/prompty/prompty/model/pipeline/_ContextCandidate.py b/runtime/python/prompty/prompty/model/pipeline/_ContextCandidate.py index 085811394..c334c9718 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_ContextCandidate.py +++ b/runtime/python/prompty/prompty/model/pipeline/_ContextCandidate.py @@ -87,7 +87,7 @@ def save_messages(items: list[Message], context: SaveContext | None) -> dict[str if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/pipeline/_ContextRequest.py b/runtime/python/prompty/prompty/model/pipeline/_ContextRequest.py index e2dd20ced..2200f0d89 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_ContextRequest.py +++ b/runtime/python/prompty/prompty/model/pipeline/_ContextRequest.py @@ -108,7 +108,7 @@ def save_messages(items: list[Message], context: SaveContext | None) -> dict[str if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/pipeline/_EngineCheckpoint.py b/runtime/python/prompty/prompty/model/pipeline/_EngineCheckpoint.py index e0a4a8c6a..a9b248681 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_EngineCheckpoint.py +++ b/runtime/python/prompty/prompty/model/pipeline/_EngineCheckpoint.py @@ -91,8 +91,8 @@ class EngineCheckpoint: stable_prefix_messages: int = field(default=0) inputs: Any | None = None active_invocation_id: str | None = None - pending_tool_requests: list[ModelToolRequest] = field(default_factory=list) - completed_tool_results: list[ModelToolResult] = field(default_factory=list) + pending_tool_requests: list[ModelToolRequest] | None = None + completed_tool_results: list[ModelToolResult] | None = None completed_model_iterations: int = field(default=0) reconciliation_required: bool = field(default=False) model_reconciliation: ModelReconciliationState | None = None @@ -200,7 +200,7 @@ def save_messages(items: list[Message], context: SaveContext | None) -> dict[str if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] @staticmethod @@ -225,28 +225,8 @@ def save_pending_tool_requests( if context is None: context = SaveContext() - if context.collection_format == "array": - return [item.save(context) for item in items] - - # Object format: use name as key - result: dict[str, Any] = {} - for item in items: - item_data = item.save(context) - name = item_data.pop("name", None) - if name: - # Check if we can use shorthand (only primary property set) - if context.use_shorthand and hasattr(item, "_shorthand_property"): - shorthand_prop = item._shorthand_property - if shorthand_prop and len(item_data) == 1 and shorthand_prop in item_data: - result[name] = item_data[shorthand_prop] - continue - result[name] = item_data - else: - # No name, fall back to array format for this item - if "_unnamed" not in result: - result["_unnamed"] = [] - result["_unnamed"].append(item_data) - return result + # The schema declares an ordered collection, so preserve array format + return [item.save(context) for item in items] @staticmethod def load_completed_tool_results(data: dict | list, context: LoadContext | None) -> list[ModelToolResult]: @@ -270,28 +250,8 @@ def save_completed_tool_results( if context is None: context = SaveContext() - if context.collection_format == "array": - return [item.save(context) for item in items] - - # Object format: use name as key - result: dict[str, Any] = {} - for item in items: - item_data = item.save(context) - name = item_data.pop("name", None) - if name: - # Check if we can use shorthand (only primary property set) - if context.use_shorthand and hasattr(item, "_shorthand_property"): - shorthand_prop = item._shorthand_property - if shorthand_prop and len(item_data) == 1 and shorthand_prop in item_data: - result[name] = item_data[shorthand_prop] - continue - result[name] = item_data - else: - # No name, fall back to array format for this item - if "_unnamed" not in result: - result["_unnamed"] = [] - result["_unnamed"].append(item_data) - return result + # The schema declares an ordered collection, so preserve array format + return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: """Save the EngineCheckpoint instance to a dictionary. diff --git a/runtime/python/prompty/prompty/model/pipeline/_FinalOutputPolicyRequest.py b/runtime/python/prompty/prompty/model/pipeline/_FinalOutputPolicyRequest.py index 33251a4b8..18fd6d8e6 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_FinalOutputPolicyRequest.py +++ b/runtime/python/prompty/prompty/model/pipeline/_FinalOutputPolicyRequest.py @@ -97,7 +97,7 @@ def save_messages(items: list[Message], context: SaveContext | None) -> dict[str if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/pipeline/_HostPolicyRequest.py b/runtime/python/prompty/prompty/model/pipeline/_HostPolicyRequest.py index 9bfbbb134..fd851b0f9 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_HostPolicyRequest.py +++ b/runtime/python/prompty/prompty/model/pipeline/_HostPolicyRequest.py @@ -97,7 +97,7 @@ def save_messages(items: list[Message], context: SaveContext | None) -> dict[str if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/pipeline/_HostPolicyResult.py b/runtime/python/prompty/prompty/model/pipeline/_HostPolicyResult.py index 1d47e9d5b..d31f1dce1 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_HostPolicyResult.py +++ b/runtime/python/prompty/prompty/model/pipeline/_HostPolicyResult.py @@ -82,7 +82,7 @@ def save_messages(items: list[Message], context: SaveContext | None) -> dict[str if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/pipeline/_InvocationContextState.py b/runtime/python/prompty/prompty/model/pipeline/_InvocationContextState.py index a01ca2404..06e65692d 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_InvocationContextState.py +++ b/runtime/python/prompty/prompty/model/pipeline/_InvocationContextState.py @@ -29,7 +29,7 @@ class InvocationContextState: _shorthand_property: ClassVar[str | None] = None portability: InvocationContextPortability = field(default="portable") - delegated_state: list[DelegatedStateReference] = field(default_factory=list) + delegated_state: list[DelegatedStateReference] | None = None @staticmethod def load(data: Any, context: LoadContext | None = None) -> "InvocationContextState": @@ -81,7 +81,7 @@ def save_delegated_state( if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/pipeline/_ModelInvocationContextSnapshot.py b/runtime/python/prompty/prompty/model/pipeline/_ModelInvocationContextSnapshot.py index 9693a2f2e..2e1ba80eb 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_ModelInvocationContextSnapshot.py +++ b/runtime/python/prompty/prompty/model/pipeline/_ModelInvocationContextSnapshot.py @@ -52,7 +52,7 @@ class ModelInvocationContextSnapshot: invocation_id: str = field(default="") iteration: int = field(default=0) messages: list[Message] = field(default_factory=list) - decisions: list[InvocationContextDecision] = field(default_factory=list) + decisions: list[InvocationContextDecision] | None = None stable_prefix_messages: int = field(default=0) context_state: InvocationContextState = field(default_factory=InvocationContextState) metadata: dict[str, Any] | None = None @@ -121,7 +121,7 @@ def save_messages(items: list[Message], context: SaveContext | None) -> dict[str if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] @staticmethod @@ -146,7 +146,7 @@ def save_decisions( if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/pipeline/_ModelInvocationResponse.py b/runtime/python/prompty/prompty/model/pipeline/_ModelInvocationResponse.py index 3a77b6957..f5978d640 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_ModelInvocationResponse.py +++ b/runtime/python/prompty/prompty/model/pipeline/_ModelInvocationResponse.py @@ -5,7 +5,7 @@ # ANY EDITS WILL BE LOST ########################################## -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any, ClassVar from .._context import LoadContext, SaveContext @@ -42,8 +42,8 @@ class ModelInvocationResponse: output: Any | None = None usage: InvocationUsage | None = None - assistant_messages: list[Message] = field(default_factory=list) - tool_requests: list[ModelToolRequest] = field(default_factory=list) + assistant_messages: list[Message] | None = None + tool_requests: list[ModelToolRequest] | None = None next_context_state: InvocationContextState | None = None metadata: dict[str, Any] | None = None @@ -107,7 +107,7 @@ def save_assistant_messages( if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] @staticmethod @@ -132,28 +132,8 @@ def save_tool_requests( if context is None: context = SaveContext() - if context.collection_format == "array": - return [item.save(context) for item in items] - - # Object format: use name as key - result: dict[str, Any] = {} - for item in items: - item_data = item.save(context) - name = item_data.pop("name", None) - if name: - # Check if we can use shorthand (only primary property set) - if context.use_shorthand and hasattr(item, "_shorthand_property"): - shorthand_prop = item._shorthand_property - if shorthand_prop and len(item_data) == 1 and shorthand_prop in item_data: - result[name] = item_data[shorthand_prop] - continue - result[name] = item_data - else: - # No name, fall back to array format for this item - if "_unnamed" not in result: - result["_unnamed"] = [] - result["_unnamed"].append(item_data) - return result + # The schema declares an ordered collection, so preserve array format + return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: """Save the ModelInvocationResponse instance to a dictionary. diff --git a/runtime/python/prompty/prompty/model/pipeline/_ReplayVerificationRequest.py b/runtime/python/prompty/prompty/model/pipeline/_ReplayVerificationRequest.py index 04d54dd38..fe84c22ea 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_ReplayVerificationRequest.py +++ b/runtime/python/prompty/prompty/model/pipeline/_ReplayVerificationRequest.py @@ -79,7 +79,7 @@ def save_expected( if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] @staticmethod @@ -104,7 +104,7 @@ def save_actual( if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/pipeline/_ReplayVerificationResult.py b/runtime/python/prompty/prompty/model/pipeline/_ReplayVerificationResult.py index 00fffe09c..3ae1f437d 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_ReplayVerificationResult.py +++ b/runtime/python/prompty/prompty/model/pipeline/_ReplayVerificationResult.py @@ -33,7 +33,7 @@ class ReplayVerificationResult: _shorthand_property: ClassVar[str | None] = None status: ReplayVerificationStatus = field(default="passed") - mismatches: list[ReplayMismatch] = field(default_factory=list) + mismatches: list[ReplayMismatch] | None = None expected_count: int = field(default=0) actual_count: int = field(default=0) @@ -91,7 +91,7 @@ def save_mismatches( if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/pipeline/_RunTurnResult.py b/runtime/python/prompty/prompty/model/pipeline/_RunTurnResult.py index 706caa83a..0976f31ce 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_RunTurnResult.py +++ b/runtime/python/prompty/prompty/model/pipeline/_RunTurnResult.py @@ -44,8 +44,8 @@ class RunTurnResult: status: RunTurnStatus = field(default="success") output: Any | None = None iterations: int = field(default=0) - tool_results: list[HostToolResult] = field(default_factory=list) - checkpoints: list[Checkpoint] = field(default_factory=list) + tool_results: list[HostToolResult] | None = None + checkpoints: list[Checkpoint] | None = None @staticmethod def load(data: Any, context: LoadContext | None = None) -> "RunTurnResult": @@ -107,7 +107,7 @@ def save_tool_results( if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] @staticmethod @@ -130,7 +130,7 @@ def save_checkpoints(items: list[Checkpoint], context: SaveContext | None) -> di if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/pipeline/_TurnCommit.py b/runtime/python/prompty/prompty/model/pipeline/_TurnCommit.py index 9f62c27e3..3ee7fc040 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_TurnCommit.py +++ b/runtime/python/prompty/prompty/model/pipeline/_TurnCommit.py @@ -116,7 +116,7 @@ def save_messages(items: list[Message], context: SaveContext | None) -> dict[str if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/pipeline/_TurnEngineResult.py b/runtime/python/prompty/prompty/model/pipeline/_TurnEngineResult.py index 4325d7c4e..0fee0adc8 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_TurnEngineResult.py +++ b/runtime/python/prompty/prompty/model/pipeline/_TurnEngineResult.py @@ -33,8 +33,8 @@ class TurnEngineResult: _shorthand_property: ClassVar[str | None] = None commit: TurnCommit = field(default_factory=TurnCommit) - snapshots: list[ModelInvocationContextSnapshot] = field(default_factory=list) - tool_results: list[ModelToolResult] = field(default_factory=list) + snapshots: list[ModelInvocationContextSnapshot] | None = None + tool_results: list[ModelToolResult] | None = None post_commit_error: str | None = None @staticmethod @@ -91,7 +91,7 @@ def save_snapshots( if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] @staticmethod @@ -116,28 +116,8 @@ def save_tool_results( if context is None: context = SaveContext() - if context.collection_format == "array": - return [item.save(context) for item in items] - - # Object format: use name as key - result: dict[str, Any] = {} - for item in items: - item_data = item.save(context) - name = item_data.pop("name", None) - if name: - # Check if we can use shorthand (only primary property set) - if context.use_shorthand and hasattr(item, "_shorthand_property"): - shorthand_prop = item._shorthand_property - if shorthand_prop and len(item_data) == 1 and shorthand_prop in item_data: - result[name] = item_data[shorthand_prop] - continue - result[name] = item_data - else: - # No name, fall back to array format for this item - if "_unnamed" not in result: - result["_unnamed"] = [] - result["_unnamed"].append(item_data) - return result + # The schema declares an ordered collection, so preserve array format + return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: """Save the TurnEngineResult instance to a dictionary. diff --git a/runtime/python/prompty/prompty/model/pipeline/_TurnModelRequest.py b/runtime/python/prompty/prompty/model/pipeline/_TurnModelRequest.py index 2bb23453d..cb45b6d54 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_TurnModelRequest.py +++ b/runtime/python/prompty/prompty/model/pipeline/_TurnModelRequest.py @@ -43,7 +43,7 @@ class TurnModelRequest: iteration: int = field(default=0) inputs: dict[str, Any] | None = None options: TurnOptions | None = None - tool_results: list[HostToolResult] = field(default_factory=list) + tool_results: list[HostToolResult] | None = None @staticmethod def load(data: Any, context: LoadContext | None = None) -> "TurnModelRequest": @@ -103,7 +103,7 @@ def save_tool_results( if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/pipeline/_TurnModelResponse.py b/runtime/python/prompty/prompty/model/pipeline/_TurnModelResponse.py index e93347812..873c95a7c 100644 --- a/runtime/python/prompty/prompty/model/pipeline/_TurnModelResponse.py +++ b/runtime/python/prompty/prompty/model/pipeline/_TurnModelResponse.py @@ -5,7 +5,7 @@ # ANY EDITS WILL BE LOST ########################################## -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Any, ClassVar from .._context import LoadContext, SaveContext @@ -33,7 +33,7 @@ class TurnModelResponse: output: Any | None = None usage: InvocationUsage | None = None - tool_requests: list[HostToolRequest] = field(default_factory=list) + tool_requests: list[HostToolRequest] | None = None checkpoint_state: dict[str, Any] | None = None @staticmethod @@ -90,7 +90,7 @@ def save_tool_requests( if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/tools/_McpApprovalMode.py b/runtime/python/prompty/prompty/model/tools/_McpApprovalMode.py index a4f9ba9b0..cd30f126b 100644 --- a/runtime/python/prompty/prompty/model/tools/_McpApprovalMode.py +++ b/runtime/python/prompty/prompty/model/tools/_McpApprovalMode.py @@ -32,8 +32,8 @@ class McpApprovalMode: _shorthand_property: ClassVar[str | None] = "kind" kind: mcpApprovalModeKind = field(default="always") - always_require_approval_tools: list[str] = field(default_factory=list) - never_require_approval_tools: list[str] = field(default_factory=list) + always_require_approval_tools: list[str] | None = None + never_require_approval_tools: list[str] | None = None @staticmethod def load(data: Any, context: LoadContext | None = None) -> "McpApprovalMode": diff --git a/runtime/python/prompty/prompty/model/tools/_Tool.py b/runtime/python/prompty/prompty/model/tools/_Tool.py index aac3a6d18..ebbd593bc 100644 --- a/runtime/python/prompty/prompty/model/tools/_Tool.py +++ b/runtime/python/prompty/prompty/model/tools/_Tool.py @@ -37,7 +37,7 @@ class Tool(ABC): name: str = field(default="") kind: str = field(default="") description: str | None = None - bindings: list[Binding] = field(default_factory=list) + bindings: list[Binding] | None = None @staticmethod def load(data: Any, context: LoadContext | None = None) -> "Tool": @@ -459,7 +459,7 @@ class McpTool(Tool): server_name: str = field(default="") server_description: str | None = None approval_mode: McpApprovalMode = field(default_factory=McpApprovalMode) - allowed_tools: list[str] = field(default_factory=list) + allowed_tools: list[str] | None = None @staticmethod def load(data: Any, context: LoadContext | None = None) -> "McpTool": diff --git a/runtime/python/prompty/prompty/model/tools/_ToolContext.py b/runtime/python/prompty/prompty/model/tools/_ToolContext.py index e79292c96..60ec71b09 100644 --- a/runtime/python/prompty/prompty/model/tools/_ToolContext.py +++ b/runtime/python/prompty/prompty/model/tools/_ToolContext.py @@ -79,7 +79,7 @@ def save_messages(items: list[Message], context: SaveContext | None) -> dict[str if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: diff --git a/runtime/python/prompty/prompty/model/tracing/_TraceSpan.py b/runtime/python/prompty/prompty/model/tracing/_TraceSpan.py index 5bb42ad50..6e78cbc18 100644 --- a/runtime/python/prompty/prompty/model/tracing/_TraceSpan.py +++ b/runtime/python/prompty/prompty/model/tracing/_TraceSpan.py @@ -51,7 +51,7 @@ class TraceSpan: error: str | None = None __usage: TokenUsage | None = None attributes: dict[str, Any] | None = None - __frames: list[Any] = field(default_factory=list) + __frames: list[Any] | None = None @staticmethod def load(data: Any, context: LoadContext | None = None) -> "TraceSpan": diff --git a/runtime/python/prompty/prompty/model/wire/_AnthropicMessagesRequest.py b/runtime/python/prompty/prompty/model/wire/_AnthropicMessagesRequest.py index 760dc4be5..160d537ea 100644 --- a/runtime/python/prompty/prompty/model/wire/_AnthropicMessagesRequest.py +++ b/runtime/python/prompty/prompty/model/wire/_AnthropicMessagesRequest.py @@ -48,8 +48,8 @@ class AnthropicMessagesRequest: temperature: float | None = None top_p: float | None = None top_k: int | None = None - stop_sequences: list[str] = field(default_factory=list) - tools: list[AnthropicToolDefinition] = field(default_factory=list) + stop_sequences: list[str] | None = None + tools: list[AnthropicToolDefinition] | None = None @staticmethod def load(data: Any, context: LoadContext | None = None) -> "AnthropicMessagesRequest": @@ -115,7 +115,7 @@ def save_messages( if context is None: context = SaveContext() - # This type doesn't have a 'name' property, so always use array format + # The schema declares an ordered collection, so preserve array format return [item.save(context) for item in items] @staticmethod @@ -140,28 +140,8 @@ def save_tools( if context is None: context = SaveContext() - if context.collection_format == "array": - return [item.save(context) for item in items] - - # Object format: use name as key - result: dict[str, Any] = {} - for item in items: - item_data = item.save(context) - name = item_data.pop("name", None) - if name: - # Check if we can use shorthand (only primary property set) - if context.use_shorthand and hasattr(item, "_shorthand_property"): - shorthand_prop = item._shorthand_property - if shorthand_prop and len(item_data) == 1 and shorthand_prop in item_data: - result[name] = item_data[shorthand_prop] - continue - result[name] = item_data - else: - # No name, fall back to array format for this item - if "_unnamed" not in result: - result["_unnamed"] = [] - result["_unnamed"].append(item_data) - return result + # The schema declares an ordered collection, so preserve array format + return [item.save(context) for item in items] def save(self, context: SaveContext | None = None) -> dict[str, Any]: """Save the AnthropicMessagesRequest instance to a dictionary. diff --git a/runtime/python/prompty/prompty/providers/anthropic/__init__.py b/runtime/python/prompty/prompty/providers/anthropic/__init__.py index 44544817e..e1fb18bd1 100644 --- a/runtime/python/prompty/prompty/providers/anthropic/__init__.py +++ b/runtime/python/prompty/prompty/providers/anthropic/__init__.py @@ -1,8 +1,9 @@ -"""Anthropic provider — executor and processor for Anthropic Messages API.""" +"""Anthropic provider — executor, processor, and model discovery for Anthropic Messages API.""" from __future__ import annotations from .executor import AnthropicExecutor +from .models import list_models, list_models_async from .processor import AnthropicProcessor -__all__ = ["AnthropicExecutor", "AnthropicProcessor"] +__all__ = ["AnthropicExecutor", "AnthropicProcessor", "list_models", "list_models_async"] diff --git a/runtime/python/prompty/prompty/providers/anthropic/models.py b/runtime/python/prompty/prompty/providers/anthropic/models.py new file mode 100644 index 000000000..092be7321 --- /dev/null +++ b/runtime/python/prompty/prompty/providers/anthropic/models.py @@ -0,0 +1,150 @@ +"""Anthropic model discovery — list available models from the Anthropic API. + +Provides :func:`list_models` and :func:`list_models_async` which call +``client.models.list()`` and map the results to :class:`ModelInfo` objects. + +Anthropic supplies capability fields directly (when present), so enrichment +from the shared ``spec/data/model_capabilities.json`` dataset is applied only +as a fill-only-missing fallback (provider-supplied fields always win). +Mirrors ``runtime/rust/prompty-anthropic/src/models.rs``. +""" + +from __future__ import annotations + +from typing import Any + +from ...core.model_capabilities import enrich +from ...model import ApiKeyConnection, Connection, ModelInfo, ReferenceConnection + +__all__ = ["list_models", "list_models_async", "model_info_from_wire"] + + +# --------------------------------------------------------------------------- +# Wire mapping +# --------------------------------------------------------------------------- + + +def _string_array(raw: dict[str, Any], key: str) -> list[str] | None: + value = raw.get(key) + if not isinstance(value, list): + return None + return [item for item in value if isinstance(item, str)] + + +def model_info_from_wire(raw: dict[str, Any]) -> ModelInfo: + """Map one raw Anthropic ``/v1/models`` entry into the provider-neutral ``ModelInfo`` contract. + + This is the single source of truth for the Anthropic wire -> ``ModelInfo`` mapping and is + exercised by the shared ``spec/vectors/discovery`` vectors so every runtime converges on the + same canonical shape. + """ + model_id = raw.get("id") + display_name = raw.get("display_name") + context_window = raw.get("context_length") + info = ModelInfo( + id=model_id if isinstance(model_id, str) else "", + display_name=display_name if isinstance(display_name, str) else None, + owned_by="anthropic", + context_window=context_window + if isinstance(context_window, int) and not isinstance(context_window, bool) + else None, + input_modalities=_string_array(raw, "input_modalities"), + output_modalities=_string_array(raw, "output_modalities"), + additional_properties=dict(raw), + ) + enrich("anthropic", info) + return info + + +# --------------------------------------------------------------------------- +# Client construction helpers (mirror executor pattern) +# --------------------------------------------------------------------------- + + +def _build_client_kwargs(connection: Connection) -> dict[str, Any]: + """Extract kwargs for ``Anthropic(...)`` from a connection.""" + kwargs: dict[str, Any] = {} + if isinstance(connection, ApiKeyConnection): + if connection.api_key: + kwargs["api_key"] = connection.api_key + if connection.endpoint: + kwargs["base_url"] = connection.endpoint + return kwargs + + +def _model_to_dict(m: Any) -> dict[str, Any]: + """Normalize an Anthropic SDK model object (or plain dict/test double) into a raw dict.""" + if isinstance(m, dict): + return dict(m) + if hasattr(m, "model_dump"): + return m.model_dump(mode="json") + return dict(vars(m)) + + +def _model_items(response: Any) -> list[Any]: + """Return model objects from Anthropic SDK list responses (auto-paginating iterables).""" + return list(response) + + +async def _model_items_async(response: Any) -> list[Any]: + """Return all model objects from an async auto-paginating SDK response.""" + if hasattr(response, "__aiter__"): + return [item async for item in response] + return list(response) + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def list_models(connection: Connection) -> list[ModelInfo]: + """List models available from the Anthropic API. + + Parameters + ---------- + connection : Connection + An ``ApiKeyConnection`` or ``ReferenceConnection`` for auth. + + Returns + ------- + list[ModelInfo] + Available models, enriched with known metadata where possible. + """ + from anthropic import Anthropic + + if isinstance(connection, ReferenceConnection): + from ...core.connections import get_connection + + client = get_connection(connection.name) + else: + client = Anthropic(**_build_client_kwargs(connection)) + + response = client.models.list(limit=100) + return [model_info_from_wire(_model_to_dict(m)) for m in _model_items(response)] + + +async def list_models_async(connection: Connection) -> list[ModelInfo]: + """Async variant of :func:`list_models`. + + Parameters + ---------- + connection : Connection + An ``ApiKeyConnection`` or ``ReferenceConnection`` for auth. + + Returns + ------- + list[ModelInfo] + Available models, enriched with known metadata where possible. + """ + from anthropic import AsyncAnthropic + + if isinstance(connection, ReferenceConnection): + from ...core.connections import get_connection + + client = get_connection(connection.name) + else: + client = AsyncAnthropic(**_build_client_kwargs(connection)) + + response = await client.models.list(limit=100) + return [model_info_from_wire(_model_to_dict(m)) for m in await _model_items_async(response)] diff --git a/runtime/python/prompty/prompty/providers/foundry/models.py b/runtime/python/prompty/prompty/providers/foundry/models.py index 9f158f611..84e8aa876 100644 --- a/runtime/python/prompty/prompty/providers/foundry/models.py +++ b/runtime/python/prompty/prompty/providers/foundry/models.py @@ -18,6 +18,7 @@ from collections.abc import Callable from typing import Any +from ...core.model_capabilities import enrich from ...model import ( ApiKeyConnection, Connection, @@ -26,7 +27,12 @@ ReferenceConnection, ) -__all__ = ["list_models", "list_models_async"] +__all__ = [ + "catalog_model_to_model_info", + "deployment_to_model_info", + "list_models", + "list_models_async", +] # --------------------------------------------------------------------------- @@ -121,51 +127,116 @@ def _get_token_callback(client: Any) -> Callable[[], str]: def _extract_capabilities(deployment: dict[str, Any]) -> dict[str, Any]: - properties = deployment.get("properties") or {} - model = properties.get("model") or {} - return properties.get("capabilities") or model.get("capabilities") or {} + properties = deployment.get("properties") + properties = properties if isinstance(properties, dict) else {} + model = properties.get("model") + model = model if isinstance(model, dict) else {} + for source in (properties, model, deployment): + if "capabilities" in source: + capabilities = source["capabilities"] + return capabilities if isinstance(capabilities, dict) else {} + return {} def _get_number(source: dict[str, Any], *keys: str) -> int | None: for key in keys: value = source.get(key) - if isinstance(value, int | float): - return int(value) - if isinstance(value, str) and value.strip(): + if isinstance(value, int) and not isinstance(value, bool): + return value + if isinstance(value, str): + signless = value[1:] if value.startswith(("+", "-")) else value + if not signless or any(character < "0" or character > "9" for character in signless): + continue try: - return int(float(value)) + return int(value) except ValueError: continue return None -def _get_str_list(source: dict[str, Any], *keys: str) -> list[str]: +def _get_string(source: dict[str, Any], *keys: str) -> str | None: + for key in keys: + value = source.get(key) + if isinstance(value, str): + return value + return None + + +def _get_str_list(source: dict[str, Any], *keys: str) -> list[str] | None: for key in keys: value = source.get(key) if isinstance(value, list): - return [str(item) for item in value] - if isinstance(value, str) and value.strip(): + return [item for item in value if isinstance(item, str)] + if isinstance(value, str): return [item.strip() for item in value.split(",") if item.strip()] - return [] + return None def _map_deployment(deployment: dict[str, Any]) -> ModelInfo: - """Map a Foundry deployment object to ModelInfo.""" - properties = deployment.get("properties") or {} - model = properties.get("model") or {} + """Map a Foundry deployment object to ModelInfo. + + Handles both the flat ``/deployments?api-version=v1`` data-plane shape and the nested ARM + management-plane shape. This is the single source of truth for the Foundry deployment wire -> + ``ModelInfo`` mapping and is exercised by the shared ``spec/vectors/discovery`` vectors. + """ + properties = deployment.get("properties") + properties = properties if isinstance(properties, dict) else {} + model = properties.get("model") + model = model if isinstance(model, dict) else {} capabilities = _extract_capabilities(deployment) - return ModelInfo( - id=str(deployment.get("name", "")), - display_name=model.get("name"), - owned_by=model.get("publisher") or "azure", - context_window=_get_number(capabilities, "maxContextLength", "contextWindow", "context_length") - or _get_number(model, "maxContextLength"), + display_name = _get_string(deployment, "modelName") + if display_name is None: + display_name = _get_string(model, "name") + owned_by = _get_string(deployment, "modelPublisher") + if owned_by is None: + owned_by = _get_string(model, "publisher") + if owned_by is None: + owned_by = "azure" + context_window = _get_number(capabilities, "maxContextLength", "contextWindow", "context_length") + if context_window is None: + context_window = _get_number(model, "maxContextLength") + if context_window is None: + context_window = _get_number(deployment, "maxContextLength") + info = ModelInfo( + id=_get_string(deployment, "name") or "", + display_name=display_name, + owned_by=owned_by, + context_window=context_window, input_modalities=_get_str_list(capabilities, "inputModalities", "input_modalities", "supportedInputModalities"), output_modalities=_get_str_list( capabilities, "outputModalities", "output_modalities", "supportedOutputModalities" ), - additional_properties=deployment, + additional_properties=dict(deployment), ) + enrich("foundry", info) + return info + + +def deployment_to_model_info(raw: dict[str, Any]) -> ModelInfo: + """Map one raw Foundry data-plane/ARM deployment object into the provider-neutral ``ModelInfo``. + + Exercised by the shared ``spec/vectors/discovery`` vectors so every runtime converges on the + same canonical mapping. + """ + return _map_deployment(raw) + + +def catalog_model_to_model_info(raw: dict[str, Any]) -> ModelInfo: + """Map one raw Azure OpenAI model-catalog entry into the provider-neutral ``ModelInfo``. + + Exercised by the shared ``spec/vectors/discovery`` vectors. + """ + context_window = raw.get("maxContextLength") + if not isinstance(context_window, int) or isinstance(context_window, bool): + context_window = None + info = ModelInfo( + id=_get_string(raw, "id") or "", + owned_by=_get_string(raw, "owned_by"), + context_window=context_window, + additional_properties=dict(raw), + ) + enrich("foundry", info) + return info def _list_foundry_deployments(project_endpoint: str, get_token: Callable[[], str]) -> list[ModelInfo]: @@ -203,14 +274,30 @@ def get_token() -> str: return {"project_endpoint": connection.endpoint, "get_token": get_token} +def _model_to_dict(m: Any) -> dict[str, Any]: + """Normalize an Azure OpenAI SDK model object (or plain dict/test double) into a raw dict.""" + if isinstance(m, dict): + return dict(m) + if hasattr(m, "model_dump"): + return m.model_dump(mode="json") + return dict(vars(m)) + + def _map_model(m: Any) -> ModelInfo: - """Map an Azure OpenAI SDK model object to ModelInfo.""" - context_window = getattr(m, "max_context_length", None) - return ModelInfo( + """Map an Azure OpenAI SDK model object to ModelInfo. + + The SDK exposes ``max_context_length`` as a Python attribute (unlike the raw wire's + ``maxContextLength`` used by :func:`catalog_model_to_model_info` / discovery vectors), so this + reads SDK attributes directly rather than delegating to the raw-dict mapper. + """ + info = ModelInfo( id=m.id, owned_by=getattr(m, "owned_by", None), - context_window=context_window, + context_window=getattr(m, "max_context_length", None), + additional_properties=_model_to_dict(m), ) + enrich("foundry", info) + return info def _model_items(response: Any) -> list[Any]: diff --git a/runtime/python/prompty/prompty/providers/openai/executor.py b/runtime/python/prompty/prompty/providers/openai/executor.py index 7aa263692..e777e331f 100644 --- a/runtime/python/prompty/prompty/providers/openai/executor.py +++ b/runtime/python/prompty/prompty/providers/openai/executor.py @@ -223,9 +223,7 @@ def _property_to_json_schema(prop: Any, *, optional: bool = False, strict: bool props: dict[str, Any] = {} required: list[str] = [] for p in prop.properties: - props[p.name] = _property_to_json_schema( - p, optional=strict and not bool(p.required), strict=strict - ) + props[p.name] = _property_to_json_schema(p, optional=strict and not bool(p.required), strict=strict) if strict or p.required: required.append(p.name) schema["properties"] = props @@ -366,9 +364,7 @@ def _responses_tools_to_wire(agent: Prompty) -> list[dict[str, Any]] | None: if tool.description: tool_def["description"] = tool.description if hasattr(tool, "parameters") and tool.parameters: - tool_def["parameters"] = _schema_to_wire( - tool.parameters, strict=bool(getattr(tool, "strict", False)) - ) + tool_def["parameters"] = _schema_to_wire(tool.parameters, strict=bool(getattr(tool, "strict", False))) if hasattr(tool, "strict") and tool.strict: tool_def["strict"] = True if "parameters" in tool_def: diff --git a/runtime/python/prompty/prompty/providers/openai/models.py b/runtime/python/prompty/prompty/providers/openai/models.py index 830b1f2ff..ae1f2a1cc 100644 --- a/runtime/python/prompty/prompty/providers/openai/models.py +++ b/runtime/python/prompty/prompty/providers/openai/models.py @@ -1,64 +1,48 @@ """OpenAI model discovery — list available models from the OpenAI API. Provides :func:`list_models` and :func:`list_models_async` which call -``client.models.list()`` and map the results to :class:`ModelInfo` objects, -enriching sparse API responses with a built-in lookup table of known models. +``client.models.list()`` and map the results to :class:`ModelInfo` objects. + +OpenAI's ``/v1/models`` returns only ``id``/``owned_by``, so capability fields +(context window, modalities) are filled from the shared +``spec/data/model_capabilities.json`` dataset via +:func:`prompty.core.model_capabilities.enrich`. That primitive applies the +cross-runtime fill-only-missing rule: any field OpenAI *did* supply is +preserved. Mirrors ``runtime/rust/prompty-openai/src/models.rs``. """ from __future__ import annotations from typing import Any +from ...core.model_capabilities import enrich from ...model import ApiKeyConnection, Connection, ModelInfo, ReferenceConnection -__all__ = ["list_models", "list_models_async"] +__all__ = ["list_models", "list_models_async", "model_info_from_wire"] + # --------------------------------------------------------------------------- -# Built-in knowledge of well-known OpenAI models +# Wire mapping # --------------------------------------------------------------------------- -_KNOWN_MODELS: dict[str, dict[str, Any]] = { - "gpt-4o": { - "context_window": 128_000, - "input_modalities": ["text", "image"], - "output_modalities": ["text"], - }, - "gpt-4o-mini": { - "context_window": 128_000, - "input_modalities": ["text", "image"], - "output_modalities": ["text"], - }, - "gpt-4-turbo": { - "context_window": 128_000, - "input_modalities": ["text", "image"], - "output_modalities": ["text"], - }, - "gpt-4": { - "context_window": 8_192, - "input_modalities": ["text"], - "output_modalities": ["text"], - }, - "gpt-3.5-turbo": { - "context_window": 16_385, - "input_modalities": ["text"], - "output_modalities": ["text"], - }, - "text-embedding-3-small": { - "context_window": 8_191, - "input_modalities": ["text"], - "output_modalities": [], - }, - "text-embedding-3-large": { - "context_window": 8_191, - "input_modalities": ["text"], - "output_modalities": [], - }, - "dall-e-3": { - "context_window": None, - "input_modalities": ["text"], - "output_modalities": ["image"], - }, -} + +def model_info_from_wire(raw: dict[str, Any]) -> ModelInfo: + """Map one raw OpenAI ``/v1/models`` entry into the provider-neutral ``ModelInfo`` contract. + + This is the single source of truth for the OpenAI wire -> ``ModelInfo`` mapping and is + exercised by the shared ``spec/vectors/discovery`` vectors so every runtime converges on the + same canonical shape. Enrichment from the shared capability dataset is applied here; discovery + vectors deliberately use ids outside that dataset to assert the pure wire mapping. + """ + model_id = raw.get("id") + owned_by = raw.get("owned_by") + info = ModelInfo( + id=model_id if isinstance(model_id, str) else "", + owned_by=owned_by if isinstance(owned_by, str) else None, + additional_properties=dict(raw), + ) + enrich("openai", info) + return info # --------------------------------------------------------------------------- @@ -77,28 +61,13 @@ def _build_client_kwargs(connection: Connection) -> dict[str, Any]: return kwargs -def _enrich(model_id: str, info: ModelInfo) -> ModelInfo: - """Enrich a ModelInfo with data from the built-in lookup table.""" - known = _KNOWN_MODELS.get(model_id) - if known is None: - return info - - if info.context_window is None and known.get("context_window") is not None: - info.context_window = known["context_window"] - if not info.input_modalities and known.get("input_modalities"): - info.input_modalities = known["input_modalities"] - if not info.output_modalities and known.get("output_modalities"): - info.output_modalities = known["output_modalities"] - - return info - - -def _map_model(m: Any) -> ModelInfo: - """Map an OpenAI SDK model object to ModelInfo.""" - return ModelInfo( - id=m.id, - owned_by=getattr(m, "owned_by", None), - ) +def _model_to_dict(m: Any) -> dict[str, Any]: + """Normalize an OpenAI SDK model object (or plain dict/test double) into a raw dict.""" + if isinstance(m, dict): + return dict(m) + if hasattr(m, "model_dump"): + return m.model_dump(mode="json") + return dict(vars(m)) def _model_items(response: Any) -> list[Any]: @@ -137,7 +106,7 @@ def list_models(connection: Connection) -> list[ModelInfo]: client = OpenAI(**_build_client_kwargs(connection)) response = client.models.list() - return [_enrich(m.id, _map_model(m)) for m in _model_items(response)] + return [model_info_from_wire(_model_to_dict(m)) for m in _model_items(response)] async def list_models_async(connection: Connection) -> list[ModelInfo]: @@ -163,4 +132,4 @@ async def list_models_async(connection: Connection) -> list[ModelInfo]: client = AsyncOpenAI(**_build_client_kwargs(connection)) response = await client.models.list() - return [_enrich(m.id, _map_model(m)) for m in _model_items(response)] + return [model_info_from_wire(_model_to_dict(m)) for m in _model_items(response)] diff --git a/runtime/python/prompty/prompty/providers/openai/processor.py b/runtime/python/prompty/prompty/providers/openai/processor.py index 9e72cd808..aad4f613a 100644 --- a/runtime/python/prompty/prompty/providers/openai/processor.py +++ b/runtime/python/prompty/prompty/providers/openai/processor.py @@ -140,12 +140,7 @@ def _process_chat_completion(response: Any) -> Any: if message.content is None and isinstance(refusal, str): return refusal - # Refusal — when content is null but the model refused - refusal = getattr(message, "refusal", None) - if message.content is None and isinstance(refusal, str): - return refusal - - return message.content + return "" if message.content is None else message.content def _process_embedding(response: Any) -> Any: diff --git a/runtime/python/prompty/tests/integration/conftest.py b/runtime/python/prompty/tests/integration/conftest.py index 3ce73d33d..514a71706 100644 --- a/runtime/python/prompty/tests/integration/conftest.py +++ b/runtime/python/prompty/tests/integration/conftest.py @@ -46,7 +46,7 @@ def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: _OPENAI_BASE_URL = os.environ.get("OPENAI_BASE_URL", "") # optional: proxy via Azure _OPENAI_MODEL = os.environ.get("OPENAI_MODEL", "gpt-4o-mini") # override default chat model _OPENAI_EMBEDDING_MODEL = os.environ.get("OPENAI_EMBEDDING_MODEL", "text-embedding-3-small") -_OPENAI_IMAGE_MODEL = os.environ.get("OPENAI_IMAGE_MODEL", "dall-e-2") +_OPENAI_IMAGE_MODEL = os.environ.get("OPENAI_IMAGE_MODEL", "") # explicit opt-in, e.g. gpt-image-1 _AZURE_KEY = os.environ.get("AZURE_OPENAI_API_KEY", "") _AZURE_ENDPOINT = os.environ.get("AZURE_OPENAI_ENDPOINT", "") _AZURE_CHAT_DEPLOYMENT = os.environ.get("AZURE_OPENAI_CHAT_DEPLOYMENT", "") @@ -65,8 +65,8 @@ def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: skip_openai = pytest.mark.skipif(not has_openai, reason="OPENAI_API_KEY not set") skip_openai_image = pytest.mark.skipif( - not has_openai, - reason="OPENAI_API_KEY not set", + not (has_openai and _OPENAI_IMAGE_MODEL), + reason="OPENAI_API_KEY or OPENAI_IMAGE_MODEL not set", ) skip_foundry = pytest.mark.skipif(not has_foundry, reason="Azure OpenAI env vars not set") skip_azure = skip_foundry # backward-compat alias diff --git a/runtime/python/prompty/tests/test_anthropic.py b/runtime/python/prompty/tests/test_anthropic.py index 8a2113498..fd3d85324 100644 --- a/runtime/python/prompty/tests/test_anthropic.py +++ b/runtime/python/prompty/tests/test_anthropic.py @@ -611,6 +611,74 @@ async def test_async_processor(self): result = await AnthropicProcessor().process_async(agent, response) assert result == "Hello!" + def test_streaming_text_and_tool_calls(self): + events = iter( + [ + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "Hello"}, + }, + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "tool_use", "id": "toolu_1", "name": "get_weather"}, + }, + { + "type": "content_block_delta", + "index": 1, + "delta": {"type": "input_json_delta", "partial_json": '{"city":'}, + }, + { + "type": "content_block_delta", + "index": 1, + "delta": {"type": "input_json_delta", "partial_json": '"Seattle"}'}, + }, + ] + ) + + chunks = list(AnthropicProcessor().process(_make_agent(), events)) + + assert chunks[0] == "Hello" + assert chunks[1].id == "toolu_1" + assert chunks[1].name == "get_weather" + assert chunks[1].arguments == '{"city":"Seattle"}' + + @pytest.mark.asyncio + async def test_async_streaming_text_and_multiple_ordered_tools(self): + async def events(): + for event in [ + { + "type": "content_block_start", + "index": 2, + "content_block": {"type": "tool_use", "id": "toolu_2", "name": "second"}, + }, + { + "type": "content_block_delta", + "index": 2, + "delta": {"type": "input_json_delta", "partial_json": "{}"}, + }, + { + "type": "content_block_start", + "index": 1, + "content_block": {"type": "tool_use", "id": "toolu_1", "name": "first"}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "Working"}, + }, + ]: + yield event + + stream = await AnthropicProcessor().process_async(_make_agent(), events()) + chunks = [chunk async for chunk in stream] + + assert chunks[0] == "Working" + assert [chunk.name for chunk in chunks[1:]] == ["first", "second"] + assert chunks[1].arguments == "" + assert chunks[2].arguments == "{}" + # --------------------------------------------------------------------------- # Load from .prompty files diff --git a/runtime/python/prompty/tests/test_anthropic_models.py b/runtime/python/prompty/tests/test_anthropic_models.py new file mode 100644 index 000000000..0a7fee45c --- /dev/null +++ b/runtime/python/prompty/tests/test_anthropic_models.py @@ -0,0 +1,213 @@ +"""Tests for the Anthropic provider's model discovery — list_models()/list_models_async(). + +Mirrors ``tests/test_models.py``'s OpenAI/Foundry coverage: wire mapping via +``model_info_from_wire`` and client-orchestration via ``list_models``/``list_models_async`` with +a mocked Anthropic SDK client. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from prompty import clear_connections, register_connection +from prompty.model import ApiKeyConnection, ModelInfo, ReferenceConnection +from prompty.providers.anthropic.models import ( + list_models, + list_models_async, + model_info_from_wire, +) + + +def _fake_model(id: str, **extra: object) -> SimpleNamespace: + """Create a fake model object mimicking the Anthropic SDK's ModelInfo response.""" + obj = SimpleNamespace(id=id, type="model") + for k, v in extra.items(): + setattr(obj, k, v) + return obj + + +def _make_connection(api_key: str = "sk-ant-test") -> ApiKeyConnection: + return ApiKeyConnection.load({"kind": "key", "apiKey": api_key}) + + +# =========================================================================== +# Wire mapping +# =========================================================================== + + +class TestModelInfoFromWire: + def test_full_mapping(self) -> None: + info = model_info_from_wire( + { + "id": "claude-sonnet-4-20250514", + "display_name": "Claude Sonnet 4", + "context_length": 200_000, + "input_modalities": ["text", "image"], + "output_modalities": ["text"], + "type": "model", + } + ) + assert isinstance(info, ModelInfo) + assert info.id == "claude-sonnet-4-20250514" + assert info.display_name == "Claude Sonnet 4" + assert info.owned_by == "anthropic" + assert info.context_window == 200_000 + assert info.input_modalities == ["text", "image"] + assert info.output_modalities == ["text"] + + def test_minimal_mapping_owned_by_is_always_anthropic(self) -> None: + info = model_info_from_wire({"id": "claude-3-haiku-20240307", "type": "model"}) + assert info.id == "claude-3-haiku-20240307" + assert info.owned_by == "anthropic" + assert info.display_name is None + assert info.context_window is None + assert info.input_modalities is None + assert info.output_modalities is None + + def test_additional_properties_preserves_raw_payload(self) -> None: + raw = {"id": "claude-3-haiku-20240307", "type": "model"} + info = model_info_from_wire(raw) + assert info.additional_properties == raw + + def test_no_dataset_entry_leaves_capability_fields_none(self) -> None: + # spec/data/model_capabilities.json has no "anthropic" provider key today, so enrichment + # is a guaranteed no-op — this is intentional, not a bug (see enrichment_vectors.json's + # anthropic_enrich_no_dataset_entry_is_noop vector). + info = model_info_from_wire({"id": "claude-sonnet-4-20250514"}) + assert info.context_window is None + assert info.input_modalities is None + assert info.output_modalities is None + + def test_rejects_non_rust_scalar_field_types(self) -> None: + info = model_info_from_wire( + { + "id": 42, + "display_name": ["Claude"], + "context_length": True, + "input_modalities": ["text", 7], + } + ) + assert info.id == "" + assert info.display_name is None + assert info.context_window is None + assert info.input_modalities == ["text"] + + +# =========================================================================== +# list_models / list_models_async +# =========================================================================== + + +class TestListModels: + @patch("anthropic.Anthropic") + def test_list_models_returns_mapped_results(self, mock_anthropic_cls: MagicMock) -> None: + mock_client = MagicMock() + mock_anthropic_cls.return_value = mock_client + mock_client.models.list.return_value = [ + _fake_model("claude-sonnet-4-20250514", display_name="Claude Sonnet 4"), + _fake_model("claude-3-haiku-20240307"), + ] + + result = list_models(_make_connection()) + + assert len(result) == 2 + assert result[0].id == "claude-sonnet-4-20250514" + assert result[0].display_name == "Claude Sonnet 4" + assert result[0].owned_by == "anthropic" + assert result[1].id == "claude-3-haiku-20240307" + mock_client.models.list.assert_called_once_with(limit=100) + + @patch("anthropic.Anthropic") + def test_list_models_passes_api_key(self, mock_anthropic_cls: MagicMock) -> None: + mock_client = MagicMock() + mock_anthropic_cls.return_value = mock_client + mock_client.models.list.return_value = [] + + list_models(_make_connection("sk-ant-mykey")) + + mock_anthropic_cls.assert_called_once_with(api_key="sk-ant-mykey") + + @patch("anthropic.Anthropic") + def test_list_models_empty_response(self, mock_anthropic_cls: MagicMock) -> None: + mock_client = MagicMock() + mock_anthropic_cls.return_value = mock_client + mock_client.models.list.return_value = [] + + result = list_models(_make_connection()) + assert result == [] + + @patch("anthropic.Anthropic") + def test_list_models_reference_connection(self, mock_anthropic_cls: MagicMock) -> None: + registered_client = MagicMock() + registered_client.models.list.return_value = [_fake_model("claude-3-haiku-20240307")] + register_connection("anthropic-conn", client=registered_client) + + result = list_models(ReferenceConnection(name="anthropic-conn")) + + assert len(result) == 1 + assert result[0].id == "claude-3-haiku-20240307" + mock_anthropic_cls.assert_not_called() + clear_connections() + + @patch("anthropic.Anthropic") + def test_list_models_consumes_auto_paginated_response(self, mock_anthropic_cls: MagicMock) -> None: + mock_client = MagicMock() + mock_anthropic_cls.return_value = mock_client + response = MagicMock() + response.__iter__.return_value = iter( + [ + _fake_model("claude-first"), + _fake_model("claude-second"), + ] + ) + mock_client.models.list.return_value = response + + result = list_models(_make_connection()) + + assert [model.id for model in result] == ["claude-first", "claude-second"] + + +class TestListModelsAsync: + @pytest.mark.asyncio + @patch("anthropic.AsyncAnthropic") + async def test_list_models_async(self, mock_async_cls: MagicMock) -> None: + mock_client = MagicMock() + mock_async_cls.return_value = mock_client + mock_client.models.list = AsyncMock( + return_value=[ + _fake_model("claude-sonnet-4-20250514"), + ] + ) + + result = await list_models_async(_make_connection()) + + assert len(result) == 1 + assert result[0].id == "claude-sonnet-4-20250514" + assert result[0].owned_by == "anthropic" + mock_client.models.list.assert_awaited_once_with(limit=100) + + @pytest.mark.asyncio + @patch("anthropic.AsyncAnthropic") + async def test_list_models_async_consumes_auto_paginated_response(self, mock_async_cls: MagicMock) -> None: + mock_client = MagicMock() + mock_async_cls.return_value = mock_client + + class AsyncModels: + def __aiter__(self) -> AsyncModels: + self._items = iter([_fake_model("claude-first"), _fake_model("claude-second")]) + return self + + async def __anext__(self) -> SimpleNamespace: + try: + return next(self._items) + except StopIteration: + raise StopAsyncIteration from None + + mock_client.models.list = AsyncMock(return_value=AsyncModels()) + + result = await list_models_async(_make_connection()) + + assert [model.id for model in result] == ["claude-first", "claude-second"] diff --git a/runtime/python/prompty/tests/test_discovery_vectors.py b/runtime/python/prompty/tests/test_discovery_vectors.py new file mode 100644 index 000000000..66533a490 --- /dev/null +++ b/runtime/python/prompty/tests/test_discovery_vectors.py @@ -0,0 +1,97 @@ +"""Discovery vector tests — spec/vectors/discovery/discovery_vectors.json. + +Exhaustively consumes every vector in the shared cross-runtime discovery vector file, dispatches +each to the matching provider wire-mapping function (selected by ``provider``/``shape``), and +asserts the resulting ``ModelInfo.save()`` output equals the vector's ``expected`` value exactly. +Includes a drift guard so a newly-added vector can never be silently skipped. + +Run: + cd runtime/python/prompty + .venv\\Scripts\\python.exe -m pytest tests/test_discovery_vectors.py -v +""" + +from __future__ import annotations + +import copy +import json +from pathlib import Path +from typing import Any + +import pytest + +from prompty.providers.anthropic.models import model_info_from_wire as anthropic_model_info_from_wire +from prompty.providers.foundry.models import ( + catalog_model_to_model_info, + deployment_to_model_info, +) +from prompty.providers.openai.models import model_info_from_wire as openai_model_info_from_wire + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent.parent +DISCOVERY_VECTORS_PATH = REPO_ROOT / "spec" / "vectors" / "discovery" / "discovery_vectors.json" + + +def _load_discovery_vectors() -> list[dict[str, Any]]: + with open(DISCOVERY_VECTORS_PATH, encoding="utf-8") as f: + payload = json.load(f) + vectors = payload["vectors"] + assert isinstance(vectors, list) and len(vectors) > 0, "discovery_vectors.json must contain at least one vector" + return vectors + + +_VECTORS = _load_discovery_vectors() + +# Every (provider, shape) combination this test file knows how to dispatch. If the vectors file +# ever grows a new combination, the drift guard test below will fail until this file is updated. +_KNOWN_DISPATCH_KEYS = { + ("openai", "model"), + ("anthropic", "model"), + ("foundry", "deployment"), + ("foundry", "catalog"), +} + + +def _dispatch(vector: dict[str, Any]) -> dict[str, Any]: + provider = vector["provider"] + shape = vector["shape"] + raw = copy.deepcopy(vector["input"]) + + if provider == "openai" and shape == "model": + info = openai_model_info_from_wire(raw) + elif provider == "anthropic" and shape == "model": + info = anthropic_model_info_from_wire(raw) + elif provider == "foundry" and shape == "deployment": + info = deployment_to_model_info(raw) + elif provider == "foundry" and shape == "catalog": + info = catalog_model_to_model_info(raw) + else: + raise AssertionError(f"No dispatch registered for provider={provider!r} shape={shape!r}") + + return info.save() + + +@pytest.mark.parametrize("vector", _VECTORS, ids=[v["name"] for v in _VECTORS]) +def test_discovery_vector(vector: dict[str, Any]) -> None: + actual = _dispatch(vector) + assert actual == vector["expected"], f"{vector['name']}: mapping mismatch" + + +def test_all_vectors_are_dispatchable() -> None: + """Drift guard: fail loudly if a vector's (provider, shape) has no known dispatch.""" + seen = {(v["provider"], v["shape"]) for v in _VECTORS} + unknown = seen - _KNOWN_DISPATCH_KEYS + assert not unknown, f"discovery_vectors.json has vectors with no test dispatch: {unknown}" + + +def test_at_least_one_vector_per_known_combination_was_exercised() -> None: + """Drift guard: fail if a previously-covered (provider, shape) combination disappears.""" + seen = {(v["provider"], v["shape"]) for v in _VECTORS} + missing = _KNOWN_DISPATCH_KEYS - seen + assert not missing, f"Expected discovery vector combinations missing from the vectors file: {missing}" + + +def test_vector_count_matches_exhaustive_run() -> None: + """Guard against a vectors file that silently grows without pytest collecting the new cases.""" + assert len(_VECTORS) == len(list({v["name"] for v in _VECTORS})), "vector names must be unique" + ran = sum(1 for _ in _VECTORS) + assert ran == len(_VECTORS) + assert ran > 0 diff --git a/runtime/python/prompty/tests/test_engine_vectors.py b/runtime/python/prompty/tests/test_engine_vectors.py new file mode 100644 index 000000000..914d24974 --- /dev/null +++ b/runtime/python/prompty/tests/test_engine_vectors.py @@ -0,0 +1,797 @@ +"""Exercise the emitted Python turn engine contracts against shared vectors.""" + +from __future__ import annotations + +import json +from collections import Counter, deque +from pathlib import Path +from typing import Any + +import pytest + +from prompty import ( + CancellationToken, + ReferenceTurnEngine, + load_engine_checkpoint, + save_engine_checkpoint, +) +from prompty.model import ( + EngineCheckpoint, + EngineEvent, + EnginePermissionDecision, + InvocationContextState, + Message, + ModelInvocationRequest, + ModelInvocationResponse, + ModelReconciliationState, + ModelToolRequest, + ModelToolResult, + ResumeContext, +) + +REPO_ROOT = Path(__file__).parents[4] +TURN_VECTORS = REPO_ROOT / "spec" / "vectors" / "engine" / "turn_vectors.json" + + +def _vectors() -> dict[str, Any]: + return json.loads(TURN_VECTORS.read_text(encoding="utf-8")) + + +def _roundtrip_checkpoint(checkpoint: EngineCheckpoint) -> EngineCheckpoint: + saved = save_engine_checkpoint(checkpoint) + return load_engine_checkpoint(json.loads(json.dumps(saved))) + + +def test_checkpoint_default_save_roundtrip_preserves_duplicate_tool_names() -> None: + checkpoint = EngineCheckpoint( + pending_tool_requests=[ + ModelToolRequest(id="call-1", name="same"), + ModelToolRequest(id="call-2", name="same"), + ], + completed_tool_results=[ + ModelToolResult(request_id="call-1", name="same", output="first"), + ModelToolResult(request_id="call-2", name="same", output="second"), + ], + ) + + saved = checkpoint.save() + loaded = EngineCheckpoint.load(json.loads(json.dumps(saved))) + + assert saved["pendingToolRequests"] == [ + {"id": "call-1", "name": "same"}, + {"id": "call-2", "name": "same"}, + ] + assert save_engine_checkpoint(checkpoint) == saved + assert [(item.id, item.name) for item in loaded.pending_tool_requests] == [ + ("call-1", "same"), + ("call-2", "same"), + ] + assert [(item["requestId"], item["name"]) for item in saved["completedToolResults"]] == [ + ("call-1", "same"), + ("call-2", "same"), + ] + assert [(item.request_id, item.name) for item in loaded.completed_tool_results] == [ + ("call-1", "same"), + ("call-2", "same"), + ] + + +class _Ids: + def __init__(self) -> None: + self._counts: Counter[str] = Counter() + + def __call__(self, kind: str) -> str: + self._counts[kind] += 1 + return f"{kind}-{self._counts[kind]}" + + +class _NonJsonToolOutput: + def __str__(self) -> str: + return "stable-output" + + +def _response(data: dict[str, Any]) -> ModelInvocationResponse: + context_state = None + if data.get("nextPortability") is not None or data.get("delegatedState") is not None: + context_state = InvocationContextState.load( + { + "portability": data.get("nextPortability", "portable"), + "delegatedState": data.get("delegatedState", []), + } + ) + return ModelInvocationResponse( + output=data.get("output"), + assistant_messages=[Message.assistant(data["assistant"])] if data.get("assistant") else [], + tool_requests=[ModelToolRequest.load(item) for item in data.get("tools", [])], + next_context_state=context_state, + ) + + +@pytest.mark.asyncio +async def test_reference_turn_engine_matches_shared_vectors() -> None: + vectors = _vectors() + assert vectors["version"] == "1" + + for case in vectors["cases"]: + responses = deque(_response(item) for item in case["model"]) + requests: list[ModelInvocationRequest] = [] + events: list[EngineEvent] = [] + checkpoints: list[EngineCheckpoint] = [] + post_commits: list[object] = [] + + def invoke_model(request: ModelInvocationRequest) -> ModelInvocationResponse: + requests.append(request) + return responses.popleft() + + def execute_tool(request: ModelToolRequest) -> ModelToolResult: + return ModelToolResult( + request_id=request.id, + name=request.name, + output=case.get("toolOutputs", {}).get(request.id), + ) + + engine = ReferenceTurnEngine( + invoke_model=invoke_model, + execute_tool=execute_tool, + authorize=lambda request: EnginePermissionDecision( + approved=request.name not in case.get("denyTools", []), + reason="denied by vector" if request.name in case.get("denyTools", []) else "allowed", + ), + on_event=events.append, + save_checkpoint=checkpoints.append, + post_commit=post_commits.append, + now=lambda: "2026-06-28T00:00:00Z", + next_id=_Ids(), + ) + cancellation = CancellationToken() + if case.get("cancelBeforeRun"): + cancellation.cancel() + + result = await engine.run_async( + f"session-{case['name']}", + f"turn-{case['name']}", + [Message.user(item["content"]) for item in case["messages"]], + cancellation=cancellation, + ) + expected = case["expected"] + + assert result.commit.status == expected["status"], case["name"] + assert result.commit.output == expected.get("output"), case["name"] + assert result.commit.iterations == expected["iterations"], case["name"] + assert len(result.snapshots) == expected["snapshots"], case["name"] + assert len(result.tool_results) == expected["toolResults"], case["name"] + assert [item.request_id for item in result.tool_results] == expected.get("toolResultOrder", []), case["name"] + if "snapshotStablePrefixes" in expected: + assert [item.stable_prefix_messages for item in result.snapshots] == expected["snapshotStablePrefixes"] + if "snapshotPortability" in expected: + assert [item.context_state.portability for item in result.snapshots] == expected["snapshotPortability"] + if "commitPortability" in expected: + assert result.commit.context_state.portability == expected["commitPortability"] + if "delegatedState" in expected: + assert len(result.commit.context_state.delegated_state) == expected["delegatedState"] + if "eventKinds" in expected: + assert [event.kind for event in events] == expected["eventKinds"], case["name"] + assert [event.sequence for event in events] == list(range(1, len(events) + 1)) + assert result.commit.last_sequence == events[-1].sequence + assert len(requests) == expected["snapshots"] + assert len(post_commits) == int(expected["status"] == "success") + assert all(checkpoint.run_id for checkpoint in checkpoints) + + +class _Interrupted(Exception): + pass + + +@pytest.mark.asyncio +async def test_resume_does_not_repeat_completed_model_effect() -> None: + model_calls = 0 + captured: EngineCheckpoint | None = None + resumed_events: list[EngineEvent] = [] + + def invoke_model(request: ModelInvocationRequest) -> ModelInvocationResponse: + nonlocal model_calls + model_calls += 1 + return ModelInvocationResponse(output="done") + + def interrupt(checkpoint: EngineCheckpoint) -> None: + nonlocal captured + captured = checkpoint + raise _Interrupted + + engine = ReferenceTurnEngine( + invoke_model=invoke_model, + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name), + save_checkpoint=interrupt, + next_id=_Ids(), + ) + with pytest.raises(_Interrupted): + await engine.run_async("session-1", "turn-1", [Message.user("hello")]) + + assert captured is not None + assert captured.final_output_ready is True + captured = _roundtrip_checkpoint(captured) + resumed = ReferenceTurnEngine( + invoke_model=invoke_model, + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name), + on_event=resumed_events.append, + next_id=_Ids(), + ) + result = await resumed.resume_async(ResumeContext(checkpoint=captured, max_iterations=10)) + + assert result.commit.output == "done" + assert model_calls == 1 + assert resumed_events[0].sequence == captured.last_sequence + 1 + assert [event.kind for event in resumed_events] == [ + "turn_started", + "turn_committed", + "post_commit_started", + "post_commit_completed", + ] + + +@pytest.mark.asyncio +async def test_resume_does_not_repeat_completed_tool_effect() -> None: + tool_calls: list[str] = [] + captured: EngineCheckpoint | None = None + + def invoke_model(request: ModelInvocationRequest) -> ModelInvocationResponse: + return ModelInvocationResponse( + assistant_messages=[Message.assistant("calling tools")], + tool_requests=[ + ModelToolRequest(id="call-a", name="echo", arguments={"value": "A"}), + ModelToolRequest(id="call-b", name="echo", arguments={"value": "B"}), + ], + ) + + def execute_tool(request: ModelToolRequest) -> ModelToolResult: + tool_calls.append(request.id) + return ModelToolResult(request_id=request.id, name=request.name, output=request.arguments) + + def interrupt(checkpoint: EngineCheckpoint) -> None: + nonlocal captured + if len(checkpoint.completed_tool_results) == 1: + captured = checkpoint + raise _Interrupted + + engine = ReferenceTurnEngine( + invoke_model=invoke_model, + execute_tool=execute_tool, + save_checkpoint=interrupt, + next_id=_Ids(), + ) + with pytest.raises(_Interrupted): + await engine.run_async("session-1", "turn-1", [Message.user("tools")]) + + assert captured is not None + assert [message.role for message in captured.messages] == ["user"] + captured = _roundtrip_checkpoint(captured) + responses = deque([ModelInvocationResponse(output="done")]) + resumed = ReferenceTurnEngine( + invoke_model=lambda request: responses.popleft(), + execute_tool=execute_tool, + next_id=_Ids(), + ) + result = await resumed.resume_async(ResumeContext(checkpoint=captured, max_iterations=10)) + + assert result.commit.output == "done" + assert tool_calls == ["call-a", "call-b"] + assert [item.request_id for item in result.tool_results] == ["call-a", "call-b"] + resumed_request = result.snapshots[0] + tool_messages = [message for message in resumed_request.messages if message.role == "tool"] + assert [message.metadata["tool_call_id"] for message in tool_messages] == ["call-a", "call-b"] + assert [message.role for message in resumed_request.messages] == ["user", "assistant", "tool", "tool"] + assert resumed_request.stable_prefix_messages == 1 + + +@pytest.mark.asyncio +async def test_cancellation_mid_tool_round_preserves_pending_batch_for_resume() -> None: + cancellation = CancellationToken() + tool_calls: list[str] = [] + checkpoints: list[EngineCheckpoint] = [] + events: list[EngineEvent] = [] + + def invoke_model(request: ModelInvocationRequest) -> ModelInvocationResponse: + return ModelInvocationResponse( + assistant_messages=[Message.assistant("calling")], + tool_requests=[ + ModelToolRequest(id="call-a", name="echo"), + ModelToolRequest(id="call-b", name="echo"), + ], + ) + + def execute_tool(request: ModelToolRequest) -> ModelToolResult: + tool_calls.append(request.id) + if request.id == "call-a": + cancellation.cancel() + return ModelToolResult(request_id=request.id, name=request.name, output=request.id) + + engine = ReferenceTurnEngine( + invoke_model=invoke_model, + execute_tool=execute_tool, + on_event=events.append, + save_checkpoint=checkpoints.append, + next_id=_Ids(), + ) + cancelled = await engine.run_async( + "session-1", + "turn-1", + [Message.user("tools")], + cancellation=cancellation, + ) + + assert cancelled.commit.status == "cancelled" + assert tool_calls == ["call-a"] + checkpoint = _roundtrip_checkpoint(checkpoints[-1]) + assert [request.id for request in checkpoint.pending_tool_requests] == ["call-a", "call-b"] + assert [result.request_id for result in checkpoint.completed_tool_results] == ["call-a"] + assert [message.role for message in checkpoint.messages] == ["user"] + + resumed_events: list[EngineEvent] = [] + responses = deque([ModelInvocationResponse(output="done")]) + resumed = ReferenceTurnEngine( + invoke_model=lambda request: responses.popleft(), + execute_tool=execute_tool, + on_event=resumed_events.append, + next_id=_Ids(), + ) + result = await resumed.resume_async( + ResumeContext( + checkpoint=checkpoint, + max_iterations=10, + last_journal_sequence=events[-1].sequence, + ) + ) + + assert result.commit.status == "success" + assert tool_calls == ["call-a", "call-b"] + assert [item.request_id for item in result.tool_results] == ["call-a", "call-b"] + assert resumed_events[0].sequence == events[-1].sequence + 1 + assert [message.role for message in result.snapshots[0].messages] == ["user", "assistant", "tool", "tool"] + + +@pytest.mark.asyncio +async def test_cancellation_after_model_effect_prevents_commit() -> None: + cancellation = CancellationToken() + events: list[EngineEvent] = [] + checkpoints: list[EngineCheckpoint] = [] + model_calls = 0 + + def invoke_model(request: ModelInvocationRequest) -> ModelInvocationResponse: + nonlocal model_calls + model_calls += 1 + cancellation.cancel() + return ModelInvocationResponse(output="must not commit") + + engine = ReferenceTurnEngine( + invoke_model=invoke_model, + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name), + on_event=events.append, + save_checkpoint=checkpoints.append, + next_id=_Ids(), + ) + result = await engine.run_async( + "session-1", + "turn-1", + [Message.user("cancel")], + cancellation=cancellation, + ) + + assert result.commit.status == "cancelled" + assert "turn_committed" not in [event.kind for event in events] + checkpoint = _roundtrip_checkpoint(checkpoints[-1]) + assert checkpoint.final_output_ready is True + + resumed = ReferenceTurnEngine( + invoke_model=invoke_model, + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name), + next_id=_Ids(), + ) + resumed_result = await resumed.resume_async(ResumeContext(checkpoint=checkpoint, max_iterations=10)) + assert resumed_result.commit.output == "must not commit" + assert model_calls == 1 + + +@pytest.mark.asyncio +async def test_resume_honors_cancellation_and_reconciliation_checkpoints() -> None: + model_calls = 0 + + def invoke_model(request: ModelInvocationRequest) -> ModelInvocationResponse: + nonlocal model_calls + model_calls += 1 + return ModelInvocationResponse(output="unexpected") + + base = EngineCheckpoint( + id="checkpoint-1", + session_id="session-1", + turn_id="turn-1", + run_id="run-1", + iteration=0, + last_sequence=4, + messages=[Message.user("hello")], + completed_model_iterations=1, + pending_output="done", + final_output_ready=True, + ) + cancellation = CancellationToken() + cancellation.cancel() + cancelled_engine = ReferenceTurnEngine( + invoke_model=invoke_model, + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name), + next_id=_Ids(), + ) + cancelled = await cancelled_engine.resume_async(ResumeContext(checkpoint=base), cancellation=cancellation) + assert cancelled.commit.status == "cancelled" + + reconciliation = ModelReconciliationState( + invocation_id="invocation-1", + request=ModelInvocationRequest(), + message="provider outcome unknown", + ) + base.final_output_ready = False + base.pending_output = None + base.reconciliation_required = True + base.model_reconciliation = reconciliation + reconciliation_engine = ReferenceTurnEngine( + invoke_model=invoke_model, + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name), + next_id=_Ids(), + ) + unresolved = await reconciliation_engine.resume_async(ResumeContext(checkpoint=base)) + + assert unresolved.commit.status == "reconciliation_required" + assert unresolved.commit.model_reconciliation == reconciliation + assert model_calls == 0 + + +@pytest.mark.asyncio +async def test_checkpoints_retain_all_turn_tool_results() -> None: + responses = deque( + [ + ModelInvocationResponse(tool_requests=[ModelToolRequest(id="call-a", name="echo")]), + ModelInvocationResponse(tool_requests=[ModelToolRequest(id="call-b", name="echo")]), + ModelInvocationResponse(output="done"), + ] + ) + checkpoints: list[EngineCheckpoint] = [] + engine = ReferenceTurnEngine( + invoke_model=lambda request: responses.popleft(), + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name, output=request.id), + save_checkpoint=checkpoints.append, + next_id=_Ids(), + ) + result = await engine.run_async("session-1", "turn-1", [Message.user("tools")]) + + assert result.commit.output == "done" + assert [item.request_id for item in checkpoints[-1].completed_tool_results] == ["call-a", "call-b"] + assert checkpoints[-1].metadata is None + + +@pytest.mark.asyncio +async def test_indeterminate_tool_halts_batch_and_requires_reconciliation() -> None: + tool_calls: list[str] = [] + checkpoints: list[EngineCheckpoint] = [] + + def execute_tool(request: ModelToolRequest) -> ModelToolResult: + tool_calls.append(request.id) + return ModelToolResult( + request_id=request.id, + name=request.name, + outcome="indeterminate", + error_kind="outcome_unknown", + ) + + engine = ReferenceTurnEngine( + invoke_model=lambda request: ModelInvocationResponse( + tool_requests=[ + ModelToolRequest(id="call-a", name="write"), + ModelToolRequest(id="call-b", name="write"), + ] + ), + execute_tool=execute_tool, + save_checkpoint=checkpoints.append, + next_id=_Ids(), + ) + result = await engine.run_async("session-1", "turn-1", [Message.user("write")]) + + assert result.commit.status == "reconciliation_required" + assert tool_calls == ["call-a"] + assert checkpoints[-1].reconciliation_required is True + assert [item.request_id for item in checkpoints[-1].completed_tool_results] == ["call-a"] + + +@pytest.mark.asyncio +async def test_mismatched_tool_result_identity_fails_conversation_commit() -> None: + engine = ReferenceTurnEngine( + invoke_model=lambda request: ModelInvocationResponse( + assistant_messages=[Message.assistant("calling")], + tool_requests=[ModelToolRequest(id="call-a", name="echo")], + ), + execute_tool=lambda request: ModelToolResult(request_id="wrong", name=request.name), + next_id=_Ids(), + ) + result = await engine.run_async("session-1", "turn-1", [Message.user("echo")]) + + assert result.commit.status == "failed" + assert result.commit.output["errorKind"] == "conversation_format_error" + assert [message.role for message in result.commit.messages] == ["user"] + + +@pytest.mark.asyncio +async def test_model_invocation_retries_with_configured_attempt_budget() -> None: + attempts = 0 + events: list[EngineEvent] = [] + + def invoke_model(request: ModelInvocationRequest) -> ModelInvocationResponse: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise RuntimeError("transient") + return ModelInvocationResponse(output="done") + + engine = ReferenceTurnEngine( + invoke_model=invoke_model, + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name), + on_event=events.append, + next_id=_Ids(), + ) + result = await engine.run_async( + "session-1", + "turn-1", + [Message.user("retry")], + max_model_attempts=2, + ) + + assert result.commit.status == "success" + assert result.commit.output == "done" + assert attempts == 2 + assert [event.kind for event in events].count("model_invocation_started") == 2 + failed = next(event for event in events if event.kind == "model_invocation_failed") + assert failed.payload["attempt"] == 0 + assert failed.payload["exhausted"] is False + + +@pytest.mark.asyncio +async def test_zero_model_attempts_uses_default_retry_budget() -> None: + attempts = 0 + + def invoke_model(request: ModelInvocationRequest) -> ModelInvocationResponse: + nonlocal attempts + attempts += 1 + raise RuntimeError("still failing") + + engine = ReferenceTurnEngine( + invoke_model=invoke_model, + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name), + next_id=_Ids(), + ) + result = await engine.run_async( + "session-1", + "turn-1", + [Message.user("retry")], + max_model_attempts=0, + ) + + assert result.commit.status == "failed" + assert attempts == 3 + + +@pytest.mark.asyncio +async def test_resume_honors_max_model_attempts() -> None: + attempts = 0 + checkpoint = EngineCheckpoint( + id="checkpoint-1", + session_id="session-1", + turn_id="turn-1", + run_id="run-1", + messages=[Message.user("retry")], + stable_prefix_messages=1, + ) + + def invoke_model(request: ModelInvocationRequest) -> ModelInvocationResponse: + nonlocal attempts + attempts += 1 + raise RuntimeError("still failing") + + engine = ReferenceTurnEngine( + invoke_model=invoke_model, + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name), + next_id=_Ids(), + ) + result = await engine.resume_async(ResumeContext(checkpoint=checkpoint, max_iterations=1, max_model_attempts=2)) + + assert result.commit.status == "failed" + assert result.commit.output["errorKind"] == "model_error" + assert attempts == 2 + + +@pytest.mark.asyncio +async def test_indeterminate_model_failure_requires_reconciliation() -> None: + class IndeterminateModelError(RuntimeError): + outcome_unknown = True + metadata = {"requestId": "provider-request-1"} + + checkpoints: list[EngineCheckpoint] = [] + engine = ReferenceTurnEngine( + invoke_model=lambda request: (_ for _ in ()).throw(IndeterminateModelError("unknown outcome")), + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name), + save_checkpoint=checkpoints.append, + next_id=_Ids(), + ) + result = await engine.run_async("session-1", "turn-1", [Message.user("reconcile")]) + + assert result.commit.status == "reconciliation_required" + assert result.commit.model_reconciliation is not None + assert result.commit.model_reconciliation.failed_attempt == 0 + assert result.commit.model_reconciliation.metadata == {"requestId": "provider-request-1"} + assert checkpoints[-1].reconciliation_required is True + assert checkpoints[-1].resume_same_iteration is True + + +@pytest.mark.asyncio +async def test_indeterminate_model_checkpoint_retains_prior_tool_results() -> None: + class IndeterminateModelError(RuntimeError): + outcome_unknown = True + + responses: deque[ModelInvocationResponse | Exception] = deque( + [ + ModelInvocationResponse(tool_requests=[ModelToolRequest(id="call-1", name="echo")]), + IndeterminateModelError("unknown outcome"), + ] + ) + checkpoints: list[EngineCheckpoint] = [] + + def invoke_model(request: ModelInvocationRequest) -> ModelInvocationResponse: + response = responses.popleft() + if isinstance(response, Exception): + raise response + return response + + engine = ReferenceTurnEngine( + invoke_model=invoke_model, + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name, output="echoed"), + save_checkpoint=checkpoints.append, + next_id=_Ids(), + ) + result = await engine.run_async("session-1", "turn-1", [Message.user("echo")]) + + assert result.commit.status == "reconciliation_required" + assert [item.request_id for item in result.tool_results] == ["call-1"] + assert [item.request_id for item in checkpoints[-1].completed_tool_results] == ["call-1"] + + +@pytest.mark.asyncio +async def test_permission_callback_failure_commits_failed_turn() -> None: + events: list[EngineEvent] = [] + + def authorize(request: ModelToolRequest) -> EnginePermissionDecision: + raise RuntimeError("permission service unavailable") + + engine = ReferenceTurnEngine( + invoke_model=lambda request: ModelInvocationResponse( + tool_requests=[ModelToolRequest(id="call-1", name="write")] + ), + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name), + authorize=authorize, + on_event=events.append, + next_id=_Ids(), + ) + result = await engine.run_async("session-1", "turn-1", [Message.user("write")]) + + assert result.commit.status == "failed" + assert result.commit.output["errorKind"] == "permission_error" + assert events[-1].kind == "turn_failed" + + +@pytest.mark.asyncio +async def test_unknown_tool_is_terminal_configuration_failure() -> None: + def execute_tool(request: ModelToolRequest) -> ModelToolResult: + raise KeyError(f"unknown tool: {request.name}") + + engine = ReferenceTurnEngine( + invoke_model=lambda request: ModelInvocationResponse( + tool_requests=[ModelToolRequest(id="call-1", name="missing")] + ), + execute_tool=execute_tool, + next_id=_Ids(), + ) + result = await engine.run_async("session-1", "turn-1", [Message.user("missing")]) + + assert result.commit.status == "failed" + assert result.commit.output["errorKind"] == "tool_configuration_error" + assert result.tool_results == [] + + +@pytest.mark.asyncio +async def test_portable_assistant_history_is_reused_after_tool_round() -> None: + requests: list[ModelInvocationRequest] = [] + + def invoke_model(request: ModelInvocationRequest) -> ModelInvocationResponse: + requests.append(request) + if len(requests) == 1: + return ModelInvocationResponse( + assistant_messages=[Message.assistant("calling")], + tool_requests=[ModelToolRequest(id="call-1", name="echo")], + next_context_state=InvocationContextState(portability="portable"), + ) + return ModelInvocationResponse(output="done") + + engine = ReferenceTurnEngine( + invoke_model=invoke_model, + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name, output="echoed"), + next_id=_Ids(), + ) + result = await engine.run_async("session-1", "turn-1", [Message.user("echo")]) + + assert result.commit.status == "success" + assert [message.role for message in requests[1].context.messages] == ["user", "assistant", "tool"] + + +@pytest.mark.parametrize( + ("output", "expected"), + [ + (None, ""), + ("raw", "raw"), + ({"value": 1}, '{"value": 1}'), + (b"bytes", '{"message": "Tool output is not JSON-serializable (type: builtins.bytes)"}'), + ( + _NonJsonToolOutput(), + json.dumps( + { + "message": ( + f"Tool output is not JSON-serializable " + f"(type: {_NonJsonToolOutput.__module__}._NonJsonToolOutput)" + ) + } + ), + ), + ], +) +def test_tool_result_messages_tolerate_non_json_outputs(output: Any, expected: str) -> None: + messages = ReferenceTurnEngine._tool_result_messages( + [ModelToolResult(request_id="call-1", name="tool", output=output)] + ) + + assert messages[0].parts[0].value == expected + + +@pytest.mark.asyncio +async def test_non_json_tool_output_becomes_durable_failure() -> None: + requests: list[ModelInvocationRequest] = [] + checkpoints: list[EngineCheckpoint] = [] + + def invoke_model(request: ModelInvocationRequest) -> ModelInvocationResponse: + requests.append(request) + if len(requests) == 1: + return ModelInvocationResponse(tool_requests=[ModelToolRequest(id="call-1", name="binary")]) + return ModelInvocationResponse(output="done") + + engine = ReferenceTurnEngine( + invoke_model=invoke_model, + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name, output=b"bytes"), + save_checkpoint=checkpoints.append, + next_id=_Ids(), + ) + result = await engine.run_async("session-1", "turn-1", [Message.user("binary")]) + + normalized = result.tool_results[0] + assert normalized.outcome == "failed" + assert normalized.error_kind == "invalid_output" + assert normalized.output == {"message": "Tool output is not JSON-serializable (type: builtins.bytes)"} + assert ( + json.loads(json.dumps(save_engine_checkpoint(checkpoints[-1])))["completedToolResults"][0] == normalized.save() + ) + assert requests[1].context.messages[-1].parts[0].value == json.dumps(normalized.output) + + +@pytest.mark.asyncio +async def test_sync_entrypoints_reject_active_event_loop() -> None: + engine = ReferenceTurnEngine( + invoke_model=lambda request: ModelInvocationResponse(output="done"), + execute_tool=lambda request: ModelToolResult(request_id=request.id, name=request.name), + next_id=_Ids(), + ) + + with pytest.raises(RuntimeError, match=r"use await run_async\(\)"): + engine.run("session-1", "turn-1", [Message.user("hello")]) + with pytest.raises(RuntimeError, match=r"use await resume_async\(\)"): + engine.resume(ResumeContext(checkpoint=EngineCheckpoint())) diff --git a/runtime/python/prompty/tests/test_enrichment_vectors.py b/runtime/python/prompty/tests/test_enrichment_vectors.py new file mode 100644 index 000000000..51968176a --- /dev/null +++ b/runtime/python/prompty/tests/test_enrichment_vectors.py @@ -0,0 +1,66 @@ +"""Enrichment vector tests — spec/vectors/discovery/enrichment_vectors.json. + +Exhaustively consumes every vector in the shared cross-runtime enrichment vector file. Each +vector builds a base ``ModelInfo`` from a partial camelCase ``input`` dict, applies +``prompty.core.model_capabilities.enrich``, and asserts the resulting ``ModelInfo.save()`` +equals the vector's ``expected`` value exactly. + +Base ``ModelInfo`` construction uses the emitted loader so the vectors exercise its native +absent-vs-empty modality tri-state before enrichment. + +Run: + cd runtime/python/prompty + uv run pytest tests/test_enrichment_vectors.py -v +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from prompty.core.model_capabilities import enrich +from prompty.model import ModelInfo + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent.parent +ENRICHMENT_VECTORS_PATH = REPO_ROOT / "spec" / "vectors" / "discovery" / "enrichment_vectors.json" + + +def _load_enrichment_vectors() -> list[dict[str, Any]]: + with open(ENRICHMENT_VECTORS_PATH, encoding="utf-8") as f: + payload = json.load(f) + vectors = payload["vectors"] + assert isinstance(vectors, list) and len(vectors) > 0, "enrichment_vectors.json must contain at least one vector" + return vectors + + +_VECTORS = _load_enrichment_vectors() + + +def _build_base_model_info(data: dict[str, Any]) -> ModelInfo: + """Build a ModelInfo through the emitted loader.""" + return ModelInfo.load(data) + + +@pytest.mark.parametrize("vector", _VECTORS, ids=[v["name"] for v in _VECTORS]) +def test_enrichment_vector(vector: dict[str, Any]) -> None: + info = _build_base_model_info(vector["input"]) + enrich(vector["provider"], info) + assert info.save() == vector["expected"], f"{vector['name']}: enrichment mismatch" + + +def test_vector_count_is_exhaustive() -> None: + """Drift guard: fail loudly if the vectors file grows without this suite noticing.""" + names = {v["name"] for v in _VECTORS} + assert len(names) == len(_VECTORS), "vector names must be unique" + assert len(_VECTORS) > 0 + + +def test_all_vector_providers_are_known() -> None: + """Drift guard: fail if a vector references a provider this harness has never validated.""" + known_providers = {"openai", "anthropic", "foundry"} + seen = {v["provider"] for v in _VECTORS} + unknown = seen - known_providers + assert not unknown, f"enrichment_vectors.json references unexpected providers: {unknown}" diff --git a/runtime/python/prompty/tests/test_loader.py b/runtime/python/prompty/tests/test_loader.py index 0fca08d92..47b9ab11b 100644 --- a/runtime/python/prompty/tests/test_loader.py +++ b/runtime/python/prompty/tests/test_loader.py @@ -481,7 +481,7 @@ def test_name_only(self): assert agent.name == "just-a-name" assert agent.model is not None # Prompty.load() provides default Model assert agent.model.id == "" - assert len(agent.inputs) == 0 + assert agent.inputs is None assert agent.instructions is not None assert "helpful assistant" in agent.instructions diff --git a/runtime/python/prompty/tests/test_model_capabilities.py b/runtime/python/prompty/tests/test_model_capabilities.py new file mode 100644 index 000000000..d31c4d141 --- /dev/null +++ b/runtime/python/prompty/tests/test_model_capabilities.py @@ -0,0 +1,175 @@ +"""Unit tests for prompty.core.model_capabilities — the shared enrichment primitive. + +Covers longest-prefix matching, token-boundary rules, and the fill-only-missing contract in +isolation from any provider wire mapping (see ``test_discovery_vectors.py`` / +``test_enrichment_vectors.py`` for the vector-driven cross-runtime parity tests). Also guards +against drift between the vendored copy of the dataset and its canonical spec source. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from prompty.core.model_capabilities import ModelCapabilities, enrich, lookup +from prompty.model import ModelInfo + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent.parent +SPEC_DATASET = REPO_ROOT / "spec" / "data" / "model_capabilities.json" +VENDORED_DATASET = Path(__file__).resolve().parent.parent / "prompty" / "data" / "model_capabilities.json" + + +# --------------------------------------------------------------------------- +# Drift guard +# --------------------------------------------------------------------------- + + +class TestVendoredDatasetMatchesSpec: + """The vendored copy under prompty/data/ must never drift from spec/data/.""" + + def test_vendored_copy_is_byte_identical_to_spec(self) -> None: + assert SPEC_DATASET.exists(), f"canonical spec dataset not found at {SPEC_DATASET}" + assert VENDORED_DATASET.exists(), f"vendored dataset not found at {VENDORED_DATASET}" + spec_bytes = SPEC_DATASET.read_bytes() + vendored_bytes = VENDORED_DATASET.read_bytes() + assert vendored_bytes == spec_bytes, ( + f"{VENDORED_DATASET} has drifted from {SPEC_DATASET}. " + "Refresh the vendored copy: Copy-Item spec/data/model_capabilities.json " + "runtime/python/prompty/prompty/data/model_capabilities.json" + ) + + def test_vendored_copy_matches_spec_as_json_value(self) -> None: + # Belt-and-suspenders structural check independent of exact byte encoding. + with open(SPEC_DATASET, encoding="utf-8") as f: + spec_value = json.load(f) + with open(VENDORED_DATASET, encoding="utf-8") as f: + vendored_value = json.load(f) + assert vendored_value == spec_value + + +# --------------------------------------------------------------------------- +# lookup() +# --------------------------------------------------------------------------- + + +class TestLookup: + def test_exact_prefix_match(self) -> None: + caps = lookup("openai", "gpt-4o") + assert caps == ModelCapabilities( + context_window=128_000, input_modalities=["text", "image"], output_modalities=["text"] + ) + + def test_longest_prefix_wins_over_shorter_prefix(self) -> None: + # "gpt-4o-mini" and "gpt-4o" both match "gpt-4o-mini-2024-07-18"; the more specific + # (longer) prefix's capabilities must be returned. + caps = lookup("openai", "gpt-4o-mini-2024-07-18") + assert caps is not None + assert caps.context_window == 128_000 + assert caps.input_modalities == ["text", "image"] + assert caps.output_modalities == ["text"] + + def test_gpt4_prefix_matches_dated_snapshot(self) -> None: + caps = lookup("openai", "gpt-4-0613") + assert caps is not None + assert caps.context_window == 8192 + + def test_token_boundary_rejects_accidental_substring(self) -> None: + # "gpt-4" must NOT match a hypothetical future "gpt-45" model. + assert lookup("openai", "gpt-45-future") is None + + def test_token_boundary_allows_dot_separator(self) -> None: + caps = lookup("openai", "gpt-3.5-turbo-0125") + assert caps is not None + assert caps.context_window == 16385 + + def test_unknown_model_id_returns_none(self) -> None: + assert lookup("openai", "ft:custom-model:acme::xyz") is None + + def test_unknown_provider_returns_none(self) -> None: + assert lookup("does-not-exist", "gpt-4o") is None + + def test_foundry_has_no_dataset_entries(self) -> None: + # spec/data/model_capabilities.json currently only has an "openai" provider key. + assert lookup("foundry", "gpt-4o") is None + + def test_anthropic_has_no_dataset_entries(self) -> None: + assert lookup("anthropic", "claude-sonnet-4-20250514") is None + + def test_embedding_model_has_empty_output_modalities(self) -> None: + caps = lookup("openai", "text-embedding-3-small") + assert caps is not None + assert caps.output_modalities == [] + + def test_image_model_has_no_context_window(self) -> None: + caps = lookup("openai", "dall-e-3") + assert caps is not None + assert caps.context_window is None + assert caps.output_modalities == ["image"] + + +# --------------------------------------------------------------------------- +# enrich() +# --------------------------------------------------------------------------- + + +class TestModelInfoTriState: + def test_load_distinguishes_omitted_from_explicit_empty_modalities(self) -> None: + omitted = ModelInfo.load({"id": "gpt-4o"}) + explicit_empty = ModelInfo.load({"id": "gpt-4o", "inputModalities": [], "outputModalities": []}) + + assert omitted.input_modalities is None + assert omitted.output_modalities is None + assert explicit_empty.input_modalities == [] + assert explicit_empty.output_modalities == [] + + +class TestEnrich: + def test_fills_all_missing_fields(self) -> None: + info = ModelInfo(id="gpt-4o") + enrich("openai", info) + assert info.context_window == 128_000 + assert info.input_modalities == ["text", "image"] + assert info.output_modalities == ["text"] + + def test_does_not_overwrite_provider_context_window(self) -> None: + info = ModelInfo(id="gpt-4o", context_window=999) + enrich("openai", info) + assert info.context_window == 999 + assert info.input_modalities == ["text", "image"] + + def test_provider_supplied_empty_list_wins_over_dataset(self) -> None: + info = ModelInfo(id="gpt-4o", input_modalities=[]) + enrich("openai", info) + assert info.input_modalities == [] + assert info.output_modalities == ["text"] + + def test_unknown_id_is_noop(self) -> None: + info = ModelInfo(id="ft:custom-model:acme::xyz") + enrich("openai", info) + assert info.context_window is None + assert info.input_modalities is None + assert info.output_modalities is None + + def test_prefix_requires_token_boundary(self) -> None: + info = ModelInfo(id="gpt-45-future") + enrich("openai", info) + assert info.context_window is None + + def test_dataset_empty_modality_fills_none(self) -> None: + # A dataset-declared [] (embeddings' outputModalities) is a valid fill for a missing + # (None) field, distinct from a provider explicitly supplying []. + info = ModelInfo(id="text-embedding-3-small") + enrich("openai", info) + assert info.output_modalities == [] + + def test_anthropic_enrich_is_noop_without_dataset_entry(self) -> None: + info = ModelInfo( + id="claude-sonnet-4-20250514", + context_window=200_000, + input_modalities=["text", "image"], + output_modalities=["text"], + ) + enrich("anthropic", info) + assert info.context_window == 200_000 + assert info.input_modalities == ["text", "image"] + assert info.output_modalities == ["text"] diff --git a/runtime/python/prompty/tests/test_models.py b/runtime/python/prompty/tests/test_models.py index fc3455a7d..ff7ba2ca3 100644 --- a/runtime/python/prompty/tests/test_models.py +++ b/runtime/python/prompty/tests/test_models.py @@ -1,4 +1,10 @@ -"""Tests for provider list_models() — OpenAI and Foundry (Azure).""" +"""Tests for provider list_models() — OpenAI and Foundry (Azure). + +Enrichment-table-specific behavior (longest-prefix matching, token-boundary rules, +fill-only-missing semantics) is covered by ``tests/test_model_capabilities.py``. This file +focuses on the provider wire-mapping functions and the ``list_models``/``list_models_async`` +client-orchestration behavior. +""" from __future__ import annotations @@ -12,23 +18,23 @@ from prompty.providers.foundry.models import ( _map_model as foundry_map_model, ) +from prompty.providers.foundry.models import ( + catalog_model_to_model_info, + deployment_to_model_info, +) from prompty.providers.foundry.models import ( list_models as foundry_list_models, ) from prompty.providers.foundry.models import ( list_models_async as foundry_list_models_async, ) -from prompty.providers.openai.models import ( - _KNOWN_MODELS, - _enrich, - _map_model, -) from prompty.providers.openai.models import ( list_models as openai_list_models, ) from prompty.providers.openai.models import ( list_models_async as openai_list_models_async, ) +from prompty.providers.openai.models import model_info_from_wire # --------------------------------------------------------------------------- # Helpers @@ -52,72 +58,41 @@ def _make_connection(api_key: str = "sk-test") -> ApiKeyConnection: # =========================================================================== -class TestOpenAIMapModel: - """Test _map_model produces correct ModelInfo from SDK objects.""" +class TestOpenAIModelInfoFromWire: + """Test model_info_from_wire produces correct, enriched ModelInfo from raw dicts.""" def test_basic_mapping(self) -> None: - m = _fake_model("gpt-4o", "openai") - info = _map_model(m) + info = model_info_from_wire({"id": "gpt-4o", "owned_by": "openai"}) assert isinstance(info, ModelInfo) assert info.id == "gpt-4o" assert info.owned_by == "openai" def test_missing_owned_by(self) -> None: - m = SimpleNamespace(id="custom-model") - info = _map_model(m) + info = model_info_from_wire({"id": "custom-model"}) assert info.id == "custom-model" assert info.owned_by is None - -class TestOpenAIEnrich: - """Test enrichment from the built-in KNOWN_MODELS table.""" - def test_enriches_known_model(self) -> None: - info = ModelInfo(id="gpt-4o") - enriched = _enrich("gpt-4o", info) - assert enriched.context_window == 128_000 - assert enriched.input_modalities == ["text", "image"] - assert enriched.output_modalities == ["text"] - - def test_enriches_embedding_model(self) -> None: - info = ModelInfo(id="text-embedding-3-small") - enriched = _enrich("text-embedding-3-small", info) - assert enriched.context_window == 8_191 - assert enriched.input_modalities == ["text"] - assert enriched.output_modalities == [] - - def test_enriches_image_model(self) -> None: - info = ModelInfo(id="dall-e-3") - enriched = _enrich("dall-e-3", info) - assert enriched.context_window is None - assert enriched.output_modalities == ["image"] + info = model_info_from_wire({"id": "gpt-4o", "owned_by": "openai"}) + assert info.context_window == 128_000 + assert info.input_modalities == ["text", "image"] + assert info.output_modalities == ["text"] def test_unknown_model_not_enriched(self) -> None: - info = ModelInfo(id="ft:gpt-4o:my-org:custom") - enriched = _enrich("ft:gpt-4o:my-org:custom", info) - assert enriched.context_window is None - assert enriched.input_modalities == [] - assert enriched.output_modalities == [] - - def test_does_not_overwrite_existing_values(self) -> None: - info = ModelInfo(id="gpt-4o", context_window=999, input_modalities=["audio"]) - enriched = _enrich("gpt-4o", info) - assert enriched.context_window == 999 - assert enriched.input_modalities == ["audio"] - - def test_known_models_table_has_expected_entries(self) -> None: - expected = [ - "gpt-4o", - "gpt-4o-mini", - "gpt-4-turbo", - "gpt-4", - "gpt-3.5-turbo", - "text-embedding-3-small", - "text-embedding-3-large", - "dall-e-3", - ] - for model_id in expected: - assert model_id in _KNOWN_MODELS, f"{model_id} missing from KNOWN_MODELS" + info = model_info_from_wire({"id": "ft:gpt-4o:my-org:custom", "owned_by": "user-org"}) + assert info.context_window is None + assert info.input_modalities is None + assert info.output_modalities is None + + def test_additional_properties_preserves_raw_payload(self) -> None: + raw = {"id": "gpt-4o", "owned_by": "openai", "created": 12345} + info = model_info_from_wire(raw) + assert info.additional_properties == raw + + def test_rejects_non_rust_scalar_field_types(self) -> None: + info = model_info_from_wire({"id": 42, "owned_by": ["openai"]}) + assert info.id == "" + assert info.owned_by is None class TestOpenAIListModels: @@ -207,12 +182,123 @@ def test_missing_context_length(self) -> None: info = foundry_map_model(m) assert info.context_window is None - def test_modalities_are_empty(self) -> None: + def test_modalities_default_to_none_when_not_supplied(self) -> None: + # Foundry has no dataset entries in spec/data/model_capabilities.json, so + # enrichment is a guaranteed no-op and unset fields stay None (not the buggy `[]` + # default the generated ModelInfo constructor would otherwise apply). m = _fake_model("gpt-4o", "azure", max_context_length=128_000) info = foundry_map_model(m) - assert info.input_modalities == [] + assert info.input_modalities is None + assert info.output_modalities is None + + +class TestFoundryCatalogModelToModelInfo: + """Test catalog_model_to_model_info produces correct ModelInfo from raw catalog dicts.""" + + def test_basic_mapping(self) -> None: + info = catalog_model_to_model_info({"id": "gpt-4", "owned_by": "openai", "maxContextLength": 8192}) + assert info.id == "gpt-4" + assert info.owned_by == "openai" + assert info.context_window == 8192 + + def test_additional_properties_preserves_raw_payload(self) -> None: + raw = {"id": "gpt-4", "owned_by": "openai", "maxContextLength": 8192, "status": "succeeded"} + info = catalog_model_to_model_info(raw) + assert info.additional_properties == raw + + def test_rejects_non_rust_catalog_field_types(self) -> None: + info = catalog_model_to_model_info({"id": 42, "owned_by": ["openai"], "maxContextLength": "8192"}) + assert info.id == "" + assert info.owned_by is None + assert info.context_window is None + + +class TestFoundryDeploymentToModelInfo: + """Test deployment_to_model_info handles both flat and nested-ARM deployment shapes.""" + + def test_flat_shape(self) -> None: + raw = { + "name": "chat-prod", + "modelName": "gpt-4o", + "modelPublisher": "OpenAI", + "capabilities": {"chatCompletion": "true"}, + } + info = deployment_to_model_info(raw) + assert info.id == "chat-prod" + assert info.display_name == "gpt-4o" + assert info.owned_by == "OpenAI" + + def test_nested_arm_shape(self) -> None: + raw = { + "name": "my-gpt4", + "properties": { + "model": {"name": "gpt-4", "publisher": "OpenAI", "maxContextLength": 8192}, + "capabilities": { + "supportedInputModalities": ["text"], + "supportedOutputModalities": ["text"], + }, + }, + } + info = deployment_to_model_info(raw) + assert info.id == "my-gpt4" + assert info.display_name == "gpt-4" + assert info.owned_by == "OpenAI" + assert info.context_window == 8192 + assert info.input_modalities == ["text"] + assert info.output_modalities == ["text"] + + def test_matches_rust_coercion_and_fallback_semantics(self) -> None: + raw = { + "name": "deployment", + "modelName": "", + "modelPublisher": "", + "maxContextLength": 4096, + "properties": { + "model": {"name": "fallback-name", "publisher": "fallback-publisher", "maxContextLength": 2048}, + "capabilities": { + "maxContextLength": 8.5, + "contextWindow": "not-an-integer", + "inputModalities": ["text", 7, None], + "outputModalities": [], + }, + }, + } + + info = deployment_to_model_info(raw) + + assert info.display_name == "" + assert info.owned_by == "" + assert info.context_window == 2048 + assert info.input_modalities == ["text"] assert info.output_modalities == [] + def test_preserves_zero_context_window(self) -> None: + info = deployment_to_model_info( + { + "name": "deployment", + "maxContextLength": 4096, + "capabilities": {"maxContextLength": 0}, + } + ) + assert info.context_window == 0 + + def test_matches_rust_whitespace_and_empty_string_parsing(self) -> None: + info = deployment_to_model_info( + { + "name": "deployment", + "properties": { + "model": {"maxContextLength": 2048}, + "capabilities": { + "maxContextLength": " 8192 ", + "inputModalities": "", + "input_modalities": ["text"], + }, + }, + } + ) + assert info.context_window == 2048 + assert info.input_modalities == [] + class TestFoundryListModels: """Test list_models with mocked AzureOpenAI client.""" diff --git a/runtime/python/prompty/tests/test_processor.py b/runtime/python/prompty/tests/test_processor.py index 1b8fd17a7..b80cf5a1b 100644 --- a/runtime/python/prompty/tests/test_processor.py +++ b/runtime/python/prompty/tests/test_processor.py @@ -134,10 +134,17 @@ def test_unknown_type_passthrough(self): result = self.processor.process(self.agent, "raw string") assert result == "raw string" - def test_none_content(self): + def test_none_content_returns_empty_string(self): response = _mock_chat_completion(content=None) result = self.processor.process(self.agent, response) - assert result is None + assert result == "" + + @pytest.mark.parametrize("content", [[], {}]) + def test_falsy_non_none_content_is_preserved(self, content): + response = _mock_chat_completion() + response.choices[0].message.content = content + result = self.processor.process(self.agent, response) + assert result is content # --------------------------------------------------------------------------- @@ -254,12 +261,12 @@ def test_tool_calls_not_json_parsed(self): assert isinstance(result, list) assert isinstance(result[0], ToolCall) - def test_none_content_not_json_parsed(self): - """None content should not be affected by outputs.""" + def test_none_content_returns_empty_string_without_json_parsing(self): + """Null content follows the shared process contract even when outputs are configured.""" agent = _make_agent_with_schema(properties=[{"name": "answer", "kind": "string"}]) response = _mock_chat_completion(content=None) result = self.processor.process(agent, response) - assert result is None + assert result == "" @pytest.mark.asyncio async def test_async_json_parsed(self): diff --git a/runtime/python/prompty/tests/test_replay_verifier.py b/runtime/python/prompty/tests/test_replay_verifier.py index a093aa3d4..5cbe3b4f5 100644 --- a/runtime/python/prompty/tests/test_replay_verifier.py +++ b/runtime/python/prompty/tests/test_replay_verifier.py @@ -32,3 +32,53 @@ def test_replay_verifier_reports_mismatches_with_generated_types() -> None: assert result.status == "failed" assert result.mismatches[0].index == 0 assert result.mismatches[0].message == "Replay record mismatch" + + +def test_replay_verifier_reports_missing_trailing_records() -> None: + expected = [ + ReplayJournalRecord(kind="turn", type="turn_start", turn_id="turn-1", iteration=0), + ReplayJournalRecord(kind="turn", type="turn_end", turn_id="turn-1", iteration=1, status="success"), + ] + + result = ReferenceReplayVerifier().verify(ReplayVerificationRequest(expected=expected, actual=expected[:1])) + + assert result.status == "failed" + assert result.expected_count == 2 + assert result.actual_count == 1 + assert result.mismatches[0].index == 1 + assert result.mismatches[0].expected == expected[1] + assert result.mismatches[0].actual is None + assert result.mismatches[0].message == "Missing replay record" + + +def test_replay_verifier_reports_unexpected_trailing_records() -> None: + actual = [ + ReplayJournalRecord(kind="turn", type="turn_start", turn_id="turn-1", iteration=0), + ReplayJournalRecord(kind="turn", type="turn_end", turn_id="turn-1", iteration=1, status="success"), + ] + + result = ReferenceReplayVerifier().verify(ReplayVerificationRequest(expected=actual[:1], actual=actual)) + + assert result.status == "failed" + assert result.mismatches[0].index == 1 + assert result.mismatches[0].expected is None + assert result.mismatches[0].actual == actual[1] + assert result.mismatches[0].message == "Unexpected extra replay record" + + +def test_replay_verifier_reports_every_mismatch_in_order() -> None: + result = ReferenceReplayVerifier().verify( + ReplayVerificationRequest( + expected=[ + ReplayJournalRecord(kind="turn", type="turn_start", turn_id="turn-1", iteration=0), + ReplayJournalRecord(kind="summary", session_id="session-1", status="success"), + ], + actual=[ + ReplayJournalRecord(kind="turn", type="turn_end", turn_id="turn-1", iteration=1), + ReplayJournalRecord(kind="summary", session_id="session-1", status="error"), + ], + ) + ) + + assert [mismatch.index for mismatch in result.mismatches] == [0, 1] + assert all(mismatch.message == "Replay record mismatch" for mismatch in result.mismatches) diff --git a/runtime/python/prompty/tests/test_responses.py b/runtime/python/prompty/tests/test_responses.py index 86fd8e3c5..b8b6bf4bd 100644 --- a/runtime/python/prompty/tests/test_responses.py +++ b/runtime/python/prompty/tests/test_responses.py @@ -231,7 +231,9 @@ def test_basic_schema(self) -> None: assert schema["type"] == "object" assert "temperature" in schema["properties"] assert "condition" in schema["properties"] - assert schema["properties"]["temperature"]["type"] == "integer" + assert schema["properties"]["temperature"]["type"] == ["integer", "null"] + assert schema["properties"]["condition"]["type"] == ["string", "null"] + assert schema["required"] == ["temperature", "condition"] assert schema["additionalProperties"] is False diff --git a/runtime/python/prompty/tests/test_spec_vectors.py b/runtime/python/prompty/tests/test_spec_vectors.py index 29eba1df6..25ecac432 100644 --- a/runtime/python/prompty/tests/test_spec_vectors.py +++ b/runtime/python/prompty/tests/test_spec_vectors.py @@ -301,7 +301,7 @@ def test_load_vector(vec: dict, tmp_path: Path): _assert_load_expected(agent, expected, name) return - pytest.skip(f"Unhandled load vector structure: {name}") + pytest.fail(f"Unhandled load vector structure: {name}") finally: # Restore env @@ -493,21 +493,96 @@ def _check_tools(actual: list, expected: list[dict], errors: list[str]): act_val = getattr(act, "mode", None) if act_val != exp["mode"]: errors.append(f" {prefix}.mode: {act_val!r} != expected {exp['mode']!r}") + act_bindings = getattr(act, "bindings", []) or [] + if act_bindings and "bindings" not in exp: + errors.append(f" {prefix}.bindings: expected bindings key is missing") if "bindings" in exp: - act_bindings = getattr(act, "bindings", []) or [] exp_bindings = exp["bindings"] + if isinstance(exp_bindings, list): + exp_bindings = { + binding["name"]: {key: value for key, value in binding.items() if key != "name"} + for binding in exp_bindings + } if isinstance(exp_bindings, dict): for bname, bval in exp_bindings.items(): found = [b for b in act_bindings if b.name == bname] if not found: errors.append(f" {prefix}.bindings: missing binding '{bname}'") else: - if isinstance(bval, dict) and "input" in bval: - if found[0].input != bval["input"]: - errors.append( - f" {prefix}.bindings.{bname}.input: " - f"{found[0].input!r} != expected {bval['input']!r}" - ) + expected_input = bval.get("input") if isinstance(bval, dict) else bval + if expected_input is not None and found[0].input != expected_input: + errors.append( + f" {prefix}.bindings.{bname}.input: {found[0].input!r} != expected {expected_input!r}" + ) + expected_names = set(exp_bindings) + for unexpected_name in sorted({binding.name for binding in act_bindings} - expected_names): + errors.append(f" {prefix}.bindings: unexpected binding '{unexpected_name}'") + + +def test_function_tool_bindings_expectation_is_required() -> None: + """Fail when a vector drops the expectation for loaded FunctionTool bindings.""" + actual = [ + FunctionTool( + name="get_weather", + bindings=[Binding(name="unit", input="preferred_unit")], + ) + ] + errors: list[str] = [] + + _check_tools(actual, [{"name": "get_weather", "kind": "function"}], errors) + + assert errors == [" tools[0].bindings: expected bindings key is missing"] + + +@pytest.mark.parametrize( + "bindings", + [ + {"unit": {"input": "preferred_unit"}}, + {"unit": "preferred_unit"}, + [{"name": "unit", "input": "preferred_unit"}], + ], +) +def test_function_tool_bindings_expectation_accepts_equivalent_forms(bindings: dict | list[dict]) -> None: + """Accept the canonical map forms and equivalent array form for bindings expectations.""" + actual = [ + FunctionTool( + name="get_weather", + bindings=[Binding(name="unit", input="preferred_unit")], + ) + ] + errors: list[str] = [] + + _check_tools(actual, [{"name": "get_weather", "kind": "function", "bindings": bindings}], errors) + + assert errors == [] + + +def test_function_tool_bindings_expectation_rejects_unexpected_actual_binding() -> None: + """Fail when the expected bindings cover only a subset of loaded FunctionTool bindings.""" + actual = [ + FunctionTool( + name="get_weather", + bindings=[ + Binding(name="unit", input="preferred_unit"), + Binding(name="location", input="preferred_location"), + ], + ) + ] + errors: list[str] = [] + + _check_tools( + actual, + [ + { + "name": "get_weather", + "kind": "function", + "bindings": {"unit": {"input": "preferred_unit"}}, + } + ], + errors, + ) + + assert errors == [" tools[0].bindings: unexpected binding 'location'"] def _check_template(actual: Template | None, expected: dict, errors: list[str]): @@ -609,19 +684,14 @@ def test_parse_vector(vec: dict): parser = PromptyChatParser() agent = Prompty(name="parse_test") - # Thread nonce expansion is a pipeline-level concern, not parser-level. - # The parser produces Message objects from rendered text. + messages = parser._parse(agent, rendered) if name == "thread_nonce_expansion": - # This vector tests pipeline-level thread expansion. - # The parser itself would just produce a message containing the nonce text. - # We test that the nonce text survives parsing, then pipeline expands it. - messages = parser._parse(agent, rendered) - # Verify the nonce marker is present in one of the messages - all_text = " ".join(m.text for m in messages) - assert "__PROMPTY_THREAD_" in all_text, f"Expected nonce marker in parsed output, got: {all_text!r}" - return + from prompty.core.pipeline import _expand_thread_markers, _inject_thread_markers + + marker = "__PROMPTY_THREAD_abcd1234_conversation__" + marked = _inject_thread_markers(messages, {marker: "conversation"}, {"conversation": "thread"}) + messages = _expand_thread_markers(marked, inp["thread_inputs"], {"conversation": "thread"}) - messages = parser._parse(agent, rendered) exp_messages = expected["messages"] assert len(messages) == len(exp_messages), ( @@ -682,7 +752,7 @@ def test_wire_vector(vec: dict): elif api_type == "responses": _check_wire_responses(agent, messages, exp_body, name) else: - pytest.skip(f"Unknown apiType for wire test: {api_type}") + pytest.fail(f"Unknown apiType for wire test: {api_type}") def _check_wire_chat(agent: Prompty, messages: list[Message], exp_body: dict, vec_name: str): @@ -941,7 +1011,7 @@ def test_process_vector(vec: dict): elif api_type == "responses": response = _make_responses_api_mock(response_data) else: - pytest.skip(f"Unknown apiType: {api_type}") + pytest.fail(f"Unknown apiType: {api_type}") result = _process_response(response, agent) exp_result = expected["result"] @@ -978,13 +1048,9 @@ def _compare_process_result(name: str, result: Any, exp_result: Any) -> None: ) elif isinstance(exp_result, str): # Text content - if result is None and exp_result == "": - # Known gap: runtime returns None for null content, spec expects "" - pass - else: - assert result == exp_result, ( - f"Process '{name}': text mismatch\n actual: {result!r}\n expected: {exp_result!r}" - ) + assert result == exp_result, ( + f"Process '{name}': text mismatch\n actual: {result!r}\n expected: {exp_result!r}" + ) else: assert result == exp_result, f"Process '{name}': result mismatch: {result!r} != {exp_result!r}" @@ -1035,6 +1101,24 @@ def test_agent_vector(vec: dict): # -- Build canned LLM responses -- mock_responses = [_make_mock_chat_completion(step["llm_response"]) for step in sequence] + if name == "tool_not_registered_error": + # Rust and TypeScript intentionally treat a missing handler as a model-visible, + # non-fatal tool result even though this legacy vector still declares ValueError. + mock_responses.append( + _make_mock_chat_completion( + { + "id": "chatcmpl-unknown-recovery", + "object": "chat.completion", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "I could not find that tool."}, + "finish_reason": "stop", + } + ], + } + ) + ) response_iter = iter(mock_responses) # -- Mock executor: replays canned responses -- @@ -1190,18 +1274,12 @@ def _test_agent_error_real( tools=tool_functions, ) elif "not registered" in error_msg.lower() or "unknown_tool" in name: - # The tool_not_registered vector expects the loop to handle missing tools - # gracefully (not crash), returning an error message to the LLM. - # Our turn handles this by returning an error string as tool result. - # The vector just validates the loop doesn't crash — so run it. - try: - turn( - agent, - inputs=inp.get("parent_inputs"), - tools=tool_functions, - ) - except (StopIteration, Exception): - pass # Mock ran out of responses — that's fine for error vectors + result = turn( + agent, + inputs=inp.get("parent_inputs"), + tools=tool_functions, + ) + assert result == "I could not find that tool." else: pytest.fail(f"Agent '{name}': unknown error type: {error_msg}") diff --git a/runtime/python/prompty/tests/test_turn_runner.py b/runtime/python/prompty/tests/test_turn_runner.py index bfc1be83b..809757f52 100644 --- a/runtime/python/prompty/tests/test_turn_runner.py +++ b/runtime/python/prompty/tests/test_turn_runner.py @@ -247,7 +247,9 @@ def invoke_model(request: TurnModelRequest) -> TurnModelResponse: assert result.output == {"denied": "permission_denied"} assert result.tool_results[0].success is False assert result.tool_results[0].error_kind == "permission_denied" - assert "tool_execution_start" not in [event.type for event in sink.turn_events] + event_types = [event.type for event in sink.turn_events] + assert "tool_execution_start" not in event_types + assert event_types[event_types.index("permission_completed") + 1] == "tool_result" @pytest.mark.asyncio @@ -335,3 +337,111 @@ def fail(args: dict[str, Any], request: HostToolRequest) -> object: ) assert _normalize_journal(_records(journal_path)) == scenario["expected"], scenario["name"] + + +@pytest.mark.asyncio +async def test_turn_runner_propagates_model_callback_failure_after_journaling_start(tmp_path: Path) -> None: + journal_path = tmp_path / "trace.jsonl" + + def invoke_model(request: TurnModelRequest) -> TurnModelResponse: + raise RuntimeError("model unavailable") + + runner = ReferenceTurnRunner( + event_sink=CollectingEventSink(), + journal=JsonlEventJournalWriter(journal_path), + checkpoint_store=InMemoryCheckpointStore(), + permission_resolver=AllowAllPermissionResolver(), + host_tool_executor=FunctionHostToolExecutor({}), + invoke_model=invoke_model, + now=lambda: "2026-06-28T00:00:00Z", + next_id=_fixed_ids(), + ) + + with pytest.raises(RuntimeError, match="model unavailable"): + await runner.run(RunTurnRequest(session_id="session-1", turn_id="turn-1")) + + turn_types = [record["event"]["type"] for record in _records(journal_path) if record["kind"] == "turn"] + assert turn_types == ["turn_start", "llm_start"] + + +@pytest.mark.asyncio +async def test_turn_runner_propagates_raw_host_executor_failure(tmp_path: Path) -> None: + class FailingExecutor: + async def execute(self, request: HostToolRequest) -> HostToolResult: + raise RuntimeError("executor unavailable") + + runner = ReferenceTurnRunner( + event_sink=CollectingEventSink(), + journal=JsonlEventJournalWriter(tmp_path / "trace.jsonl"), + checkpoint_store=InMemoryCheckpointStore(), + permission_resolver=AllowAllPermissionResolver(), + host_tool_executor=FailingExecutor(), + invoke_model=lambda request: TurnModelResponse( + tool_requests=[HostToolRequest(request_id="exec-1", tool_name="missing")] + ), + now=lambda: "2026-06-28T00:00:00Z", + next_id=_fixed_ids(), + ) + + with pytest.raises(RuntimeError, match="executor unavailable"): + await runner.run(RunTurnRequest(session_id="session-1", turn_id="turn-1")) + + +@pytest.mark.asyncio +async def test_turn_runner_zero_iterations_commits_empty_success(tmp_path: Path) -> None: + model_calls = 0 + + def invoke_model(request: TurnModelRequest) -> TurnModelResponse: + nonlocal model_calls + model_calls += 1 + return TurnModelResponse(output="unexpected") + + runner = ReferenceTurnRunner( + event_sink=CollectingEventSink(), + journal=JsonlEventJournalWriter(tmp_path / "trace.jsonl"), + checkpoint_store=InMemoryCheckpointStore(), + permission_resolver=AllowAllPermissionResolver(), + host_tool_executor=FunctionHostToolExecutor({}), + invoke_model=invoke_model, + now=lambda: "2026-06-28T00:00:00Z", + next_id=_fixed_ids(), + ) + result = await runner.run( + RunTurnRequest( + session_id="session-1", + turn_id="turn-1", + options=TurnOptions(max_iterations=0), + ) + ) + + assert result.status == "success" + assert result.output is None + assert result.iterations == 0 + assert result.checkpoints == [] + assert model_calls == 0 + + +@pytest.mark.asyncio +async def test_turn_runner_generates_unique_event_ids_without_host_factory(tmp_path: Path) -> None: + journal_path = tmp_path / "trace.jsonl" + runner = ReferenceTurnRunner( + event_sink=CollectingEventSink(), + journal=JsonlEventJournalWriter(journal_path), + checkpoint_store=InMemoryCheckpointStore(), + permission_resolver=AllowAllPermissionResolver(), + host_tool_executor=FunctionHostToolExecutor({}), + invoke_model=lambda request: TurnModelResponse(output="done"), + now=lambda: "2026-06-28T00:00:00Z", + ) + + await runner.run(RunTurnRequest(session_id="session-1", turn_id="turn-1")) + + event_ids = [record["event"]["id"] for record in _records(journal_path) if record["kind"] != "summary"] + assert event_ids + assert len(event_ids) == len(set(event_ids)) + suffixes: list[int] = [] + for event_id in event_ids: + prefix, separator, suffix = event_id.rpartition("-") + assert prefix and separator and suffix.isdigit(), f"Malformed event ID: {event_id!r}" + suffixes.append(int(suffix)) + assert suffixes == list(range(1, len(event_ids) + 1)) diff --git a/schema/package-lock.json b/schema/package-lock.json index 719a4b18b..1e834502b 100644 --- a/schema/package-lock.json +++ b/schema/package-lock.json @@ -8,7 +8,7 @@ "dependencies": { "@typespec/compiler": "1.10.0", "@typespec/json-schema": "1.10.0", - "@typra/emitter": "0.4.2" + "@typra/emitter": "0.4.8" } }, "node_modules/@babel/code-frame": { @@ -476,9 +476,9 @@ } }, "node_modules/@typra/emitter": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/@typra/emitter/-/emitter-0.4.2.tgz", - "integrity": "sha512-6eC3tOWiU00Qlt/LuOiISbDby76fv6lPrp9gB8wQ7q3ivUchIVWZAMzgnizqD4TIkfwQbDLMkf/oe08uG+2YZA==", + "version": "0.4.8", + "resolved": "https://registry.npmjs.org/@typra/emitter/-/emitter-0.4.8.tgz", + "integrity": "sha512-VpeFdEDIL1Jy5nQA5nfgZHwv67ZWPbib7I7O+pzsTUpnDrxuOky8SrXoGwXuZfGoikDDzbjb45E40p0Lfbns4g==", "license": "MIT", "dependencies": { "xml-formatter": "^3.6.7", diff --git a/schema/package.json b/schema/package.json index a0d593116..6a9bb41f5 100644 --- a/schema/package.json +++ b/schema/package.json @@ -13,6 +13,6 @@ "dependencies": { "@typespec/compiler": "1.10.0", "@typespec/json-schema": "1.10.0", - "@typra/emitter": "0.4.2" + "@typra/emitter": "0.4.8" } } diff --git a/schema/tsp-output/.typra-generated/export-surfaces.json b/schema/tsp-output/.typra-generated/export-surfaces.json index 45c400eb4..1e7e3c8bd 100644 --- a/schema/tsp-output/.typra-generated/export-surfaces.json +++ b/schema/tsp-output/.typra-generated/export-surfaces.json @@ -17,8 +17,8 @@ }, { "name": "@typra/emitter", - "version": "0.4.2", - "supportedRange": "0.4.2", + "version": "0.4.8", + "supportedRange": "0.4.8", "supported": true } ]