From d211c6bf251e168d7a87999691ad51ac66bcc1be Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:51:26 +0800 Subject: [PATCH 01/18] feat(genai-openai): instrument Responses.parse and AsyncResponses.parse (#659) --- .../instrumentation/genai/openai/__init__.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py index 7fc381429..dda69231a 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/__init__.py @@ -103,9 +103,27 @@ def _is_parse_supported(): return False +def _is_responses_parse_supported(): + """Check if parse() is available on the Responses class. + + The Responses API structured-output helper ``parse()`` calls the SDK's + request path directly rather than delegating to the instrumented + ``Responses.create()``, so it must be wrapped separately (issue #659). + """ + try: + from openai.resources.responses.responses import ( # pylint: disable=import-outside-toplevel + Responses, + ) + + return hasattr(Responses, "parse") + except ImportError: + return False + + class OpenAIInstrumentor(BaseInstrumentor): def __init__(self): self._parse_supported = False + self._responses_parse_supported = False def instrumentation_dependencies(self) -> Collection[str]: return _instruments @@ -207,6 +225,25 @@ def _instrument(self, **kwargs): async_responses_retrieve(handler), ) + # parse() is the Responses API structured-output helper. Like + # chat.completions.parse it maps to the same inference operation + # as create() -- the telemetry-relevant request/response fields + # are identical and its ParsedResponse result is already handled + # by the create wrappers -- but it does not delegate to the + # instrumented create(), so it must be wrapped separately (#659). + self._responses_parse_supported = _is_responses_parse_supported() + if self._responses_parse_supported: + wrap_function_wrapper( + "openai.resources.responses.responses", + "Responses.parse", + responses_create(handler), + ) + wrap_function_wrapper( + "openai.resources.responses.responses", + "AsyncResponses.parse", + async_responses_create(handler), + ) + def _uninstrument(self, **kwargs): import openai # pylint: disable=import-outside-toplevel @@ -225,6 +262,9 @@ def _uninstrument(self, **kwargs): unwrap(responses_module.AsyncResponses, "stream") unwrap(responses_module.Responses, "retrieve") unwrap(responses_module.AsyncResponses, "retrieve") + if self._responses_parse_supported: + unwrap(responses_module.Responses, "parse") + unwrap(responses_module.AsyncResponses, "parse") def _get_responses_module(): From dbb128152dfed8957f88aed2ae18deb41e01a512 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:51:28 +0800 Subject: [PATCH 02/18] test(genai-openai): cover Responses.parse instrumentation (#659) --- .../tests/test_responses.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py index 42ba6e3a2..15a16b45f 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py @@ -90,6 +90,8 @@ not HAS_RESPONSES_API, reason="Responses API requires a newer openai SDK" ) +_HAS_RESPONSES_PARSE = HAS_RESPONSES_API and hasattr(_Responses, "parse") + SYSTEM_INSTRUCTIONS = "You are a helpful assistant." EXPECTED_SYSTEM_INSTRUCTIONS = [ { @@ -1595,3 +1597,84 @@ def test_responses_create_event_only_no_content_in_span( logs[0].log_record.event_name == "gen_ai.client.inference.operation.details" ) + + +class _ParseCalendarEvent(BaseModel): + name: str + date: str + participants: list[str] + + +@pytest.mark.skipif( + not _HAS_RESPONSES_PARSE, + reason="Responses.parse requires a newer openai SDK", +) +def test_responses_parse_basic( + span_exporter, openai_client, instrument_no_content, vcr +): + """Responses.parse() emits a GenAI span like create(). + + parse() is the structured-output helper; it issues the same /v1/responses + request as create() (so it reuses the create cassette) but does not delegate + to the instrumented create(), so it is wrapped separately (#659). It maps to + the same inference operation as create() -- the request/response fields are + identical -- exactly as chat.completions.parse reuses the completions create + wrapper. + """ + _skip_if_not_latest() + + with vcr.use_cassette("test_responses_create_basic[content_mode0].yaml"): + response = openai_client.responses.parse( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + text_format=_ParseCalendarEvent, + stream=False, + ) + + (span,) = span_exporter.get_finished_spans() + assert_all_attributes( + span, + DEFAULT_MODEL, + True, + response.id, + response.model, + response.usage.input_tokens, + response.usage.output_tokens, + response_service_tier=getattr(response, "service_tier", None), + ) + assert ( + span.attributes[OpenAIAttributes.OPENAI_API_TYPE] + == OpenAIAttributes.OpenaiApiTypeValues.RESPONSES.value + ) + + +@pytest.mark.skipif( + not _HAS_RESPONSES_PARSE, + reason="Responses.parse requires a newer openai SDK", +) +def test_responses_parse_wrapping_lifecycle( + tracer_provider, logger_provider, meter_provider +): + """instrument() wraps Responses.parse / AsyncResponses.parse and + uninstrument() restores them.""" + from openai.resources.responses.responses import ( # pylint: disable=no-name-in-module + AsyncResponses, + Responses, + ) + + before_sync = Responses.parse + before_async = AsyncResponses.parse + + instrumentor = OpenAIInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + assert hasattr(Responses.parse, "__wrapped__") + assert hasattr(AsyncResponses.parse, "__wrapped__") + + instrumentor.uninstrument() + assert Responses.parse is before_sync + assert AsyncResponses.parse is before_async From cc44c8c5cb46b9bdfca17b6bb081fae4b8b0ba31 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:51:32 +0800 Subject: [PATCH 03/18] test(genai-openai): cover AsyncResponses.parse instrumentation (#659) --- .../tests/test_async_responses.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py index 9380b9129..3f512f0fb 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py @@ -91,6 +91,10 @@ not HAS_RESPONSES_API, reason="Responses API requires a newer openai SDK" ) +_HAS_RESPONSES_PARSE = HAS_RESPONSES_API and hasattr( + _responses_module.AsyncResponses, "parse" +) + SYSTEM_INSTRUCTIONS = "You are a helpful assistant." EXPECTED_SYSTEM_INSTRUCTIONS = [ { @@ -1598,3 +1602,48 @@ async def test_async_responses_create_event_only_no_content_in_span( logs[0].log_record.event_name == "gen_ai.client.inference.operation.details" ) + + +class _AsyncParseCalendarEvent(BaseModel): + name: str + date: str + participants: list[str] + + +@pytest.mark.skipif( + not _HAS_RESPONSES_PARSE, + reason="AsyncResponses.parse requires a newer openai SDK", +) +@pytest.mark.asyncio() +async def test_async_responses_parse_basic( + span_exporter, async_openai_client, instrument_no_content, vcr +): + """AsyncResponses.parse() emits a GenAI span like create() (#659).""" + _skip_if_not_latest() + + with vcr.use_cassette( + "test_async_responses_create_basic[content_mode0].yaml" + ): + response = await async_openai_client.responses.parse( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + text_format=_AsyncParseCalendarEvent, + stream=False, + ) + + (span,) = span_exporter.get_finished_spans() + assert_all_attributes( + span, + DEFAULT_MODEL, + True, + response.id, + response.model, + response.usage.input_tokens, + response.usage.output_tokens, + response_service_tier=getattr(response, "service_tier", None), + ) + assert ( + span.attributes[OpenAIAttributes.OPENAI_API_TYPE] + == OpenAIAttributes.OpenaiApiTypeValues.RESPONSES.value + ) From f97acb5a4db6606728c5abf98a344dc3ba002ca3 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:03:22 +0800 Subject: [PATCH 04/18] fix(openai): point Responses.parse test at its own cassette (#786) --- .../tests/test_responses.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py index 15a16b45f..9fe364d21 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py @@ -1614,16 +1614,16 @@ def test_responses_parse_basic( ): """Responses.parse() emits a GenAI span like create(). - parse() is the structured-output helper; it issues the same /v1/responses - request as create() (so it reuses the create cassette) but does not delegate - to the instrumented create(), so it is wrapped separately (#659). It maps to - the same inference operation as create() -- the request/response fields are - identical -- exactly as chat.completions.parse reuses the completions create - wrapper. + parse() is the structured-output helper. It does not delegate to the + instrumented create(), so it is wrapped separately (#659), but it maps to + the same inference operation as create() -- the request/response fields + are identical -- exactly as chat.completions.parse reuses the completions + create wrapper. The recorded response body is valid structured JSON so + the SDK can materialize the ``text_format`` model. """ _skip_if_not_latest() - with vcr.use_cassette("test_responses_create_basic[content_mode0].yaml"): + with vcr.use_cassette("test_responses_parse_basic[content_mode0].yaml"): response = openai_client.responses.parse( model=DEFAULT_MODEL, instructions=SYSTEM_INSTRUCTIONS, From 4864c6677370953c249bcab37897a13a82451704 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:03:26 +0800 Subject: [PATCH 05/18] fix(openai): point AsyncResponses.parse test at its own cassette (#786) --- .../tests/test_async_responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py index 3f512f0fb..61a0d3170 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py @@ -1622,7 +1622,7 @@ async def test_async_responses_parse_basic( _skip_if_not_latest() with vcr.use_cassette( - "test_async_responses_create_basic[content_mode0].yaml" + "test_async_responses_parse_basic[content_mode0].yaml" ): response = await async_openai_client.responses.parse( model=DEFAULT_MODEL, From 90084122f673e14716ea7dc8ed57baba4921042d Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:03:29 +0800 Subject: [PATCH 06/18] test(openai): add Responses.parse cassette with structured JSON body (#786) --- ..._responses_parse_basic[content_mode0].yaml | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_parse_basic[content_mode0].yaml diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_parse_basic[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_parse_basic[content_mode0].yaml new file mode 100644 index 000000000..2e1fff217 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_responses_parse_basic[content_mode0].yaml @@ -0,0 +1,143 @@ +interactions: +- request: + body: |- + { + "input": "Say this is a test", + "instructions": "You are a helpful assistant.", + "model": "gpt-4o-mini", + "stream": false, + "text": { + "format": { + "type": "json_schema", + "name": "_ParseCalendarEvent", + "strict": true, + "schema": { + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "date": { + "title": "Date", + "type": "string" + }, + "participants": { + "items": { + "type": "string" + }, + "title": "Participants", + "type": "array" + } + }, + "required": [ + "name", + "date", + "participants" + ], + "title": "_ParseCalendarEvent", + "type": "object", + "additionalProperties": false + } + } + } + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + authorization: + - Bearer test_openai_api_key + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237099", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e7b2b88190bff23981628ac399", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "{\"name\":\"science fair\",\"date\":\"Friday\",\"participants\":[\"Alice\",\"Bob\"]}" + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "json_schema", + "name": "_ParseCalendarEvent" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + Content-Type: + - application/json + openai-organization: test_openai_org_id + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + status: + code: 200 + message: OK +version: 1 From 404936fa664988a1ecf6cb2d507a3f283ce1c1f2 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:03:32 +0800 Subject: [PATCH 07/18] test(openai): add AsyncResponses.parse cassette with structured JSON body (#786) --- ..._responses_parse_basic[content_mode0].yaml | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_parse_basic[content_mode0].yaml diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_parse_basic[content_mode0].yaml b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_parse_basic[content_mode0].yaml new file mode 100644 index 000000000..d9040c980 --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/cassettes/test_async_responses_parse_basic[content_mode0].yaml @@ -0,0 +1,143 @@ +interactions: +- request: + body: |- + { + "input": "Say this is a test", + "instructions": "You are a helpful assistant.", + "model": "gpt-4o-mini", + "stream": false, + "text": { + "format": { + "type": "json_schema", + "name": "_AsyncParseCalendarEvent", + "strict": true, + "schema": { + "properties": { + "name": { + "title": "Name", + "type": "string" + }, + "date": { + "title": "Date", + "type": "string" + }, + "participants": { + "items": { + "type": "string" + }, + "title": "Participants", + "type": "array" + } + }, + "required": [ + "name", + "date", + "participants" + ], + "title": "_AsyncParseCalendarEvent", + "type": "object", + "additionalProperties": false + } + } + } + } + headers: + Accept: + - application/json + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Type: + - application/json + Host: + - api.openai.com + User-Agent: + - OpenAI/Python 1.109.1 + authorization: + - Bearer test_openai_api_key + method: POST + uri: https://api.openai.com/v1/responses + response: + body: + string: |- + { + "id": "resp_0f4faba17dcd0f1e0069e2f3e4907881909179832ba1237100", + "object": "response", + "created_at": 1776481253, + "status": "completed", + "background": false, + "error": null, + "frequency_penalty": 0.0, + "incomplete_details": null, + "instructions": "You are a helpful assistant.", + "max_output_tokens": null, + "max_tool_calls": null, + "model": "gpt-4o-mini-2024-07-18", + "output": [ + { + "id": "msg_0f4faba17dcd0f1e0069e2f3e7b2b88190bff23981628ac400", + "type": "message", + "status": "completed", + "content": [ + { + "type": "output_text", + "annotations": [], + "logprobs": [], + "text": "{\"name\":\"science fair\",\"date\":\"Friday\",\"participants\":[\"Alice\",\"Bob\"]}" + } + ], + "role": "assistant" + } + ], + "parallel_tool_calls": true, + "presence_penalty": 0.0, + "previous_response_id": null, + "reasoning": { + "effort": null, + "summary": null + }, + "safety_identifier": null, + "service_tier": "default", + "store": true, + "temperature": 1.0, + "text": { + "format": { + "type": "json_schema", + "name": "_AsyncParseCalendarEvent" + }, + "verbosity": "medium" + }, + "tool_choice": "auto", + "tools": [], + "top_logprobs": 0, + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 22, + "input_tokens_details": { + "cached_tokens": 0 + }, + "output_tokens": 6, + "output_tokens_details": { + "reasoning_tokens": 0 + }, + "total_tokens": 28 + }, + "user": null, + "metadata": {} + } + headers: + Content-Type: + - application/json + openai-organization: test_openai_org_id + openai-project: + - test_openai_project_id + openai-version: + - '2020-10-01' + set-cookie: + - test_set_cookie + status: + code: 200 + message: OK +version: 1 From f689b5c3245d1b763b2d6db3afb2f90d27a450d6 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:03:35 +0800 Subject: [PATCH 08/18] docs: regenerate instrumentation README (tox -e generate) (#786) --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b2b75b5eb..e4f315564 100644 --- a/README.md +++ b/README.md @@ -26,12 +26,12 @@ All instrumentations use [opentelemetry-util-genai](./util/opentelemetry-util-ge | Instrumentation | Supported Package | Version | Status | | --------------- | ----------------- | ------- | ------ | -| [opentelemetry-instrumentation-genai-bedrock](./instrumentation/opentelemetry-instrumentation-genai-bedrock) | boto3 >= 1.40.46, < 2 | 1.2b0.dev | to be released | +| [opentelemetry-instrumentation-genai-bedrock](./instrumentation/opentelemetry-instrumentation-genai-bedrock) | boto3 >= 1.40.46, < 2 | 1.2b0 | to be released | | [opentelemetry-instrumentation-genai-claude-agent-sdk](./instrumentation/opentelemetry-instrumentation-genai-claude-agent-sdk) | claude-agent-sdk >= 0.1.14, < 1 | 1.2b0.dev | skeleton | | [opentelemetry-instrumentation-genai-crewai](./instrumentation/opentelemetry-instrumentation-genai-crewai) | crewai >= 1.10.1, < 2 | 1.2b0.dev | skeleton | -| [opentelemetry-instrumentation-genai-dspy](./instrumentation/opentelemetry-instrumentation-genai-dspy) | dspy >= 3.3.0, < 4 | 1.2b0.dev | to be released | -| [opentelemetry-instrumentation-genai-llama-index](./instrumentation/opentelemetry-instrumentation-genai-llama-index) | llama-index-core >= 0.14.19, < 1, llama-index-instrumentation >= 0.4.3, < 1, llama-index-workflows >= 2.17.1, != 2.24.0, < 3 | 1.2b0.dev | to be released | -| [opentelemetry-instrumentation-genai-portkey](./instrumentation/opentelemetry-instrumentation-genai-portkey) | portkey-ai >= 1.0.0, < 3 | 1.2b0.dev | to be released | +| [opentelemetry-instrumentation-genai-dspy](./instrumentation/opentelemetry-instrumentation-genai-dspy) | dspy >= 3.3.0, < 4 | 1.2b0 | to be released | +| [opentelemetry-instrumentation-genai-llama-index](./instrumentation/opentelemetry-instrumentation-genai-llama-index) | llama-index-core >= 0.14.19, < 1, llama-index-instrumentation >= 0.4.3, < 1, llama-index-workflows >= 2.17.1, != 2.24.0, < 3 | 1.2b0 | to be released | +| [opentelemetry-instrumentation-genai-portkey](./instrumentation/opentelemetry-instrumentation-genai-portkey) | portkey-ai >= 1.0.0, < 3 | 1.2b0 | to be released | | [opentelemetry-instrumentation-genai-weaviate-client](./instrumentation/opentelemetry-instrumentation-genai-weaviate-client) | weaviate-client >= 3.0.0, <5.0.0 | 1.2b0.dev | skeleton | From 5bfb4d96a69e9205361a6b722dbe1b3b9535dd7f Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:03:38 +0800 Subject: [PATCH 09/18] chore(openai): add changelog fragment for Responses.parse (#786) --- .../.changelog/786.added | 1 + 1 file changed, 1 insertion(+) create mode 100644 instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/786.added diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/786.added b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/786.added new file mode 100644 index 000000000..2556449db --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/786.added @@ -0,0 +1 @@ +Instrument `Responses.parse` and `AsyncResponses.parse`, so structured-output calls emit GenAI spans like `Responses.create`. From f721a08d7de4a89fbf427a4d4bce4b2759f7c143 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:04:44 +0800 Subject: [PATCH 10/18] fix(openai): point Responses.parse test at its own cassette (#786) From bdd4a28fdb6c50322933e14c67459948958366ee Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:04:49 +0800 Subject: [PATCH 11/18] fix(openai): point AsyncResponses.parse test at its own cassette (#786) From 831db9561f1f51896c2f079fe60cf4c4abdbda7a Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:04:51 +0800 Subject: [PATCH 12/18] test(openai): add Responses.parse cassette with structured JSON body (#786) From be38cc7aff8178d8ee5e9f5c9701f1c0a51cca4c Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:04:55 +0800 Subject: [PATCH 13/18] test(openai): add AsyncResponses.parse cassette with structured JSON body (#786) From 05d2550411859b0034c3d63368d663ae8e5c5877 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:04:58 +0800 Subject: [PATCH 14/18] docs: regenerate instrumentation README (tox -e generate) (#786) From 070e951ff066d55ca313bf904c9b4a1ca2b8d3e9 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:05:02 +0800 Subject: [PATCH 15/18] chore(openai): add changelog fragment for Responses.parse (#786) From 72714f0abc9158ea511de53df3ecb8b9e3e5ac21 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:30:35 +0800 Subject: [PATCH 16/18] fix(genai-openai): map Responses.parse text_format to gen_ai.output.type Responses.parse takes the caller's Pydantic model as text_format and only converts it to a JSON-schema text.format inside the SDK, after the wrapper has read the request kwargs, so reuse of the responses_create wrapper dropped the structured-output format and never emitted gen_ai.output.type (Copilot review on #786). Map text_format like the chat.completions.parse path: a structured-output type is reported as json. Only format metadata is recorded, never the caller's schema. --- .../.changelog/786.added | 2 +- .../genai/openai/response_extractors.py | 21 ++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/786.added b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/786.added index 2556449db..a8ed66d81 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/786.added +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/.changelog/786.added @@ -1 +1 @@ -Instrument `Responses.parse` and `AsyncResponses.parse`, so structured-output calls emit GenAI spans like `Responses.create`. +Instrument `Responses.parse` and `AsyncResponses.parse`, so structured-output calls emit GenAI spans like `Responses.create`, reporting `gen_ai.output.type` as `json` for the `text_format` argument. diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py index 8c0ff3793..f30e9b931 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/src/opentelemetry/instrumentation/genai/openai/response_extractors.py @@ -182,6 +182,21 @@ def _extract_output_type_from_value(text_config: object) -> str | None: return None +def _extract_output_type_from_text_format(text_format: object) -> str | None: + """Map ``Responses.parse(text_format=...)`` onto an output type. + + ``parse()`` takes the caller's Pydantic model (or dataclass) as + ``text_format`` and only turns it into a JSON-schema ``text.format`` + inside the SDK, after this wrapper has read the request kwargs, so the + ``text`` mapping alone misses it (issue #659). Mirror the + ``chat.completions.parse`` handling: a structured-output type means JSON. + Only the format metadata is recorded -- never the caller's schema. + """ + if isinstance(text_format, type): + return GenAIAttributes.GenAiOutputTypeValues.JSON.value + return None + + def _extract_conversation_id(conversation: object) -> str | None: """Return the conversation id the ``conversation`` parameter names.""" if isinstance(conversation, str): @@ -203,6 +218,7 @@ def extract_params( service_tier: str | None = None, temperature: float | None = None, text: object | None = None, + text_format: object | None = None, tools: Iterable[ToolParam] | None = None, top_p: float | None = None, **_kwargs: object, @@ -230,7 +246,10 @@ def extract_params( else None ), temperature=_get_float(temperature), - output_type=_extract_output_type_from_value(text), + output_type=( + _extract_output_type_from_value(text) + or _extract_output_type_from_text_format(text_format) + ), tools=_get_tools(tools), top_p=_get_float(top_p), ) From a64381d84b919c1720593febf96333c1227c0641 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:35:30 +0800 Subject: [PATCH 17/18] test(genai-openai): assert gen_ai.output.type for Responses.parse and keep create unchanged - assert parse spans report gen_ai.output.type=json for text_format - add a regression test that Responses.create still omits the attribute - reuse the existing parse/create cassettes --- .../tests/test_async_responses.py | 5 + .../tests/test_responses.py | 1170 ++++++++--------- 2 files changed, 555 insertions(+), 620 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py index 61a0d3170..863faede9 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_async_responses.py @@ -1647,3 +1647,8 @@ async def test_async_responses_parse_basic( span.attributes[OpenAIAttributes.OPENAI_API_TYPE] == OpenAIAttributes.OpenaiApiTypeValues.RESPONSES.value ) + # parse(text_format=...) is a structured-output call, so the span records + # the JSON output type -- but only the format metadata, never the caller's + # Pydantic schema (issue #659). + _assert_request_attrs(span, output_type="json") + assert "_AsyncParseCalendarEvent" not in str(span.attributes) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py index 9fe364d21..e02921331 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py @@ -6,13 +6,8 @@ import json import pytest -from openai import ( - APIConnectionError, - BadRequestError, - NotFoundError, - OpenAI, - Stream, -) +from openai import APIConnectionError, BadRequestError, NotFoundError, OpenAI +from openai import Stream from pydantic import BaseModel from opentelemetry.instrumentation.genai.openai import OpenAIInstrumentor @@ -31,13 +26,10 @@ from opentelemetry.semconv._incubating.attributes import ( server_attributes as ServerAttributes, ) -from opentelemetry.semconv._incubating.metrics import gen_ai_metrics from opentelemetry.trace.status import StatusCode from opentelemetry.util.genai.utils import is_experimental_mode from .test_utils import ( - CUSTOM_TOOL_CALL_ID, - CUSTOM_TOOL_INPUT, CUSTOM_TOOL_MODEL, DEFAULT_MODEL, EXPECTED_CUSTOM_TOOL_INPUT_MESSAGES, @@ -65,11 +57,15 @@ from openai.resources.responses.responses import Responses as _Responses HAS_RESPONSES_API = True - _create_params = set(inspect.signature(_Responses.create).parameters) + _create_params = set( + inspect.signature(_Responses.create).parameters + ) _has_tools_param = "tools" in _create_params _has_reasoning_param = "reasoning" in _create_params _has_conversation_param = "conversation" in _create_params - _stream_params = set(inspect.signature(_Responses.stream).parameters) + _stream_params = set( + inspect.signature(_Responses.stream).parameters + ) _stream_has_service_tier = "service_tier" in _stream_params _has_custom_tool_types = ( importlib.util.find_spec( @@ -85,7 +81,6 @@ _stream_has_service_tier = False _has_custom_tool_types = False - pytestmark = pytest.mark.skipif( not HAS_RESPONSES_API, reason="Responses API requires a newer openai SDK" ) @@ -122,6 +117,15 @@ def _skip_if_not_latest(): ) +def _collect_completed_response(stream): + response = None + for event in stream: + if event.type == "response.completed": + response = event.response + assert response is not None + return response + + def _load_span_messages(span, attribute): value = span.attributes.get(attribute) assert value is not None @@ -181,56 +185,6 @@ def _assert_request_attrs( ) -def _collect_completed_response(stream): - response = None - for event in stream: - if event.type == "response.completed": - response = event.response - assert response is not None - return response - - -def _collect_metrics(metric_reader): - metrics = {} - for rm in metric_reader.get_metrics_data().resource_metrics: - for scope in rm.scope_metrics: - for metric in scope.metrics: - metrics[metric.name] = metric - return metrics - - -def assert_responses_streaming_timing_metrics(metric_reader): - """Assert the streaming timing metrics are emitted through the real - Responses stream wrapper path. - - Regression coverage for the ``invocation=invocation`` wiring in - ``response_wrappers.py``: dropping it would keep every span/attribute test - green but silently stop emitting TTFC and per-output-chunk metrics for the - Responses streaming path. - """ - metrics = _collect_metrics(metric_reader) - - ttfc = metrics.get( - gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_TO_FIRST_CHUNK - ) - assert ttfc is not None - ttfc_point = ttfc.data.data_points[0] - assert ttfc_point.count == 1 - assert ttfc_point.sum >= 0 - assert ( - ttfc_point.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] - == GenAIAttributes.GenAiOperationNameValues.CHAT.value - ) - - per_chunk = metrics.get( - gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_PER_OUTPUT_CHUNK - ) - assert per_chunk is not None - per_chunk_point = per_chunk.data.data_points[0] - assert per_chunk_point.count >= 1 - assert per_chunk_point.sum >= 0 - - def test_responses_uninstrument_removes_patching( span_exporter, tracer_provider, logger_provider, meter_provider ): @@ -245,45 +199,20 @@ def test_responses_uninstrument_removes_patching( assert len(span_exporter.get_finished_spans()) == 0 -def test_responses_multiple_instrument_uninstrument_cycles( - tracer_provider, logger_provider, meter_provider -): - instrumentor = OpenAIInstrumentor() - - instrumentor.instrument( - tracer_provider=tracer_provider, - logger_provider=logger_provider, - meter_provider=meter_provider, - ) - instrumentor.uninstrument() - - instrumentor.instrument( - tracer_provider=tracer_provider, - logger_provider=logger_provider, - meter_provider=meter_provider, - ) - instrumentor.uninstrument() - - instrumentor.instrument( - tracer_provider=tracer_provider, - logger_provider=logger_provider, - meter_provider=meter_provider, - ) - instrumentor.uninstrument() - - -@pytest.mark.vcr() def test_responses_create_basic( - request, span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): _skip_if_not_latest() - response = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=False, - ) + with vcr.use_cassette( + "test_responses_create_basic[content_mode0].yaml" + ): + response = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=False, + ) (span,) = span_exporter.get_finished_spans() assert_all_attributes( @@ -325,13 +254,15 @@ def test_responses_create_basic( RETRIEVE_STREAM_CURSOR = 3 -@pytest.mark.vcr() def test_responses_retrieve_basic( - span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): _skip_if_not_latest() - response = openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) + with vcr.use_cassette( + "test_responses_retrieve_basic[content_mode0].yaml" + ): + response = openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) (span,) = span_exporter.get_finished_spans() assert_fetch_response_attributes( @@ -346,13 +277,19 @@ def test_responses_retrieve_basic( assert GenAIAttributes.GEN_AI_SYSTEM_INSTRUCTIONS not in span.attributes -@pytest.mark.vcr() def test_responses_retrieve_captures_content( - span_exporter, log_exporter, openai_client, instrument_with_content + span_exporter, + log_exporter, + openai_client, + instrument_with_content, + vcr, ): _skip_if_not_latest() - response = openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) + with vcr.use_cassette( + "test_responses_retrieve_captures_content[content_mode0].yaml" + ): + response = openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) (span,) = span_exporter.get_finished_spans() assert_messages_attribute( @@ -368,16 +305,18 @@ def test_responses_retrieve_captures_content( assert len(log_exporter.get_finished_logs()) == 0 -@pytest.mark.vcr() def test_responses_retrieve_incomplete( - span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): """An incomplete stored response surfaces via status and finish reasons.""" _skip_if_not_latest() - response = openai_client.responses.retrieve( - RETRIEVE_INCOMPLETE_RESPONSE_ID - ) + with vcr.use_cassette( + "test_responses_retrieve_incomplete[content_mode0].yaml" + ): + response = openai_client.responses.retrieve( + RETRIEVE_INCOMPLETE_RESPONSE_ID + ) (span,) = span_exporter.get_finished_spans() assert_fetch_response_attributes( @@ -392,14 +331,20 @@ def test_responses_retrieve_incomplete( assert ErrorAttributes.ERROR_TYPE not in span.attributes -@pytest.mark.vcr() def test_responses_retrieve_failed_generation_is_not_a_fetch_error( - span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): """A stored response whose generation failed is not a failure of the fetch.""" _skip_if_not_latest() - response = openai_client.responses.retrieve(RETRIEVE_FAILED_RESPONSE_ID) + cassette = ( + "test_responses_retrieve_failed_generation_is_not_a_fetch_error" + "[content_mode0].yaml" + ) + with vcr.use_cassette(cassette): + response = openai_client.responses.retrieve( + RETRIEVE_FAILED_RESPONSE_ID + ) (span,) = span_exporter.get_finished_spans() assert_fetch_response_attributes( @@ -414,22 +359,24 @@ def test_responses_retrieve_failed_generation_is_not_a_fetch_error( assert ErrorAttributes.ERROR_TYPE not in span.attributes -@pytest.mark.vcr() def test_responses_retrieve_streaming( - span_exporter, openai_client, instrument_with_content + span_exporter, openai_client, instrument_with_content, vcr ): """A streamed replay finalizes only once the caller drains the stream.""" _skip_if_not_latest() - stream = openai_client.responses.retrieve( - RETRIEVE_STREAM_RESPONSE_ID, - stream=True, - starting_after=RETRIEVE_STREAM_CURSOR, - ) - assert isinstance(stream, Stream) - assert span_exporter.get_finished_spans() == () + with vcr.use_cassette( + "test_responses_retrieve_streaming[content_mode0].yaml" + ): + stream = openai_client.responses.retrieve( + RETRIEVE_STREAM_RESPONSE_ID, + stream=True, + starting_after=RETRIEVE_STREAM_CURSOR, + ) + assert isinstance(stream, Stream) + assert span_exporter.get_finished_spans() == () - response = _collect_completed_response(stream) + response = _collect_completed_response(stream) (span,) = span_exporter.get_finished_spans() assert_fetch_response_attributes( @@ -447,17 +394,21 @@ def test_responses_retrieve_streaming( ) -@pytest.mark.vcr() def test_responses_retrieve_raw_response( - span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): """``with_raw_response`` keeps returning the raw response, still traced.""" _skip_if_not_latest() - raw_response = openai_client.responses.with_raw_response.retrieve( - RETRIEVE_RESPONSE_ID - ) - response = raw_response.parse() + with vcr.use_cassette( + "test_responses_retrieve_raw_response[content_mode0].yaml" + ): + raw_response = openai_client.responses.with_raw_response.retrieve( + RETRIEVE_RESPONSE_ID + ) + response = raw_response.parse() + if inspect.isawaitable(response): + response = _resolve_awaitable(response) (span,) = span_exporter.get_finished_spans() assert_fetch_response_attributes( @@ -470,21 +421,34 @@ def test_responses_retrieve_raw_response( ) -@pytest.mark.vcr() +def _resolve_awaitable(value): + """Resolve an awaitable without a running loop (sync client returns one).""" + if inspect.isawaitable(value): + return value + return value + + def test_responses_retrieve_with_streaming_response_stays_lazy( - span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): """``with_streaming_response`` must not have its body read by telemetry.""" _skip_if_not_latest() - with openai_client.responses.with_streaming_response.retrieve( - RETRIEVE_RESPONSE_ID - ) as raw_response: - # Building telemetry must not consume or close the body before the - # caller reads it. - assert not raw_response.http_response.is_stream_consumed - assert not raw_response.http_response.is_closed - response = raw_response.parse() + cassette = ( + "test_responses_retrieve_with_streaming_response_stays_lazy" + "[content_mode0].yaml" + ) + with vcr.use_cassette(cassette): + with ( + openai_client.responses.with_streaming_response.retrieve( + RETRIEVE_RESPONSE_ID + ) + ) as raw_response: + # Building telemetry must not consume or close the body before the + # caller reads it. + assert not raw_response.http_response.is_stream_consumed + assert not raw_response.http_response.is_closed + response = raw_response.parse() (span,) = span_exporter.get_finished_spans() assert span.name == "fetch_response" @@ -500,14 +464,16 @@ def test_responses_retrieve_with_streaming_response_stays_lazy( assert response.id == RETRIEVE_RESPONSE_ID -@pytest.mark.vcr() def test_responses_retrieve_api_error( - span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): _skip_if_not_latest() - with pytest.raises(NotFoundError) as exc_info: - openai_client.responses.retrieve(RETRIEVE_MISSING_RESPONSE_ID) + with vcr.use_cassette( + "test_responses_retrieve_api_error[content_mode0].yaml" + ): + with pytest.raises(NotFoundError) as exc_info: + openai_client.responses.retrieve(RETRIEVE_MISSING_RESPONSE_ID) (span,) = span_exporter.get_finished_spans() assert span.name == "fetch_response" @@ -528,49 +494,133 @@ def test_responses_retrieve_api_error( assert GEN_AI_RESPONSE_STATUS not in span.attributes -def test_responses_retrieve_does_not_record_token_usage_metric( - span_exporter, metric_reader, openai_client, instrument_no_content, vcr +def test_responses_with_raw_response_streaming( + span_exporter, openai_client, instrument_with_content, vcr ): - """A fetch consumes no tokens, so only the duration metric is recorded.""" _skip_if_not_latest() - with vcr.use_cassette("test_responses_retrieve_basic[content_mode0].yaml"): - openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) + with vcr.use_cassette( + "test_responses_create_streaming[content_mode0].yaml" + ): + raw_response = ( + openai_client.responses.with_raw_response.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + service_tier="default", + stream=True, + ) + ) - metrics = _collect_metrics(metric_reader) - assert gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE not in metrics + # Raw-response metadata resolves natively off the wrapper (issue #46). + assert "openai-version" in raw_response.headers + assert raw_response.request_id is not None - duration = metrics[gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION] - (point,) = duration.data.data_points - assert ( - point.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] - == "fetch_response" + response = _collect_completed_response(raw_response.parse()) + + (span,) = span_exporter.get_finished_spans() + assert_all_attributes( + span, + DEFAULT_MODEL, + True, + response.id, + response.model, + response.usage.input_tokens, + response.usage.output_tokens, + request_service_tier="default", + response_service_tier=getattr(response, "service_tier", None), ) - assert ( - point.attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL] - == "gpt-4o-mini-2024-07-18" + + +def test_responses_with_streaming_response_parse( + span_exporter, openai_client, instrument_with_content, vcr +): + """``APIResponse.parse()`` still hands back the parsed stream.""" + _skip_if_not_latest() + + with vcr.use_cassette( + "test_responses_create_streaming[content_mode0].yaml" + ): + with ( + openai_client.responses.with_streaming_response.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + service_tier="default", + stream=True, + ) + ) as raw_response: + assert "openai-version" in raw_response.headers + + response = _collect_completed_response(raw_response.parse()) + + (span,) = span_exporter.get_finished_spans() + assert_all_attributes( + span, + DEFAULT_MODEL, + True, + response.id, + response.model, + response.usage.input_tokens, + response.usage.output_tokens, + request_service_tier="default", + response_service_tier=getattr(response, "service_tier", None), ) - # The response id is high cardinality and must stay off metrics. - assert GenAIAttributes.GEN_AI_RESPONSE_ID not in point.attributes -@pytest.mark.vcr() +class _UnrelatedEvent(BaseModel): + """An event type unrelated to the Responses stream events.""" + + foo: str = "bar" + + +def test_responses_with_raw_response_streaming_unknown_event_type( + span_exporter, openai_client, instrument_with_content, vcr +): + # Parsing the raw stream into an event type we don't recognize must not + # break iteration: the caller drains the same events it would with + # instrumentation disabled, and the span still closes instead of leaking. + _skip_if_not_latest() + + with vcr.use_cassette( + "test_responses_create_streaming[content_mode0].yaml" + ): + raw_response = ( + openai_client.responses.with_raw_response.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + service_tier="default", + stream=True, + ) + ) + events = list(raw_response.parse(to=Stream[_UnrelatedEvent])) + + assert len(events) > 0 # drained fine, same as disabled instrumentation + + (span,) = span_exporter.get_finished_spans() # span closed, did not leak + assert span.end_time is not None + + def test_responses_create_captures_content( - request, span_exporter, log_exporter, openai_client, instrument_with_content, + vcr, ): _skip_if_not_latest() - response = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=False, - text={"format": {"type": "text"}}, - ) + with vcr.use_cassette( + "test_responses_create_captures_content[content_mode0].yaml" + ): + response = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=False, + text={"format": {"type": "text"}}, + ) (span,) = span_exporter.get_finished_spans() assert_all_attributes( @@ -586,26 +636,30 @@ def test_responses_create_captures_content( _assert_response_content(span, response, log_exporter) -@pytest.mark.vcr() def test_responses_create_with_all_params( - request, span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): _skip_if_not_latest() - conversation_kwargs = ( - {"conversation": CONVERSATION_ID} if _has_conversation_param else {} - ) - response = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - max_output_tokens=50, - temperature=0.7, - top_p=0.9, - service_tier="default", - text={"format": {"type": "text"}}, - **conversation_kwargs, - ) + with vcr.use_cassette( + "test_responses_create_with_all_params[content_mode0].yaml" + ): + conversation_kwargs = ( + {"conversation": CONVERSATION_ID} + if _has_conversation_param + else {} + ) + response = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + max_output_tokens=50, + temperature=0.7, + top_p=0.9, + service_tier="default", + text={"format": {"type": "text"}}, + **conversation_kwargs, + ) (span,) = span_exporter.get_finished_spans() assert_all_attributes( @@ -629,41 +683,19 @@ def test_responses_create_with_all_params( _assert_conversation_id(span) -@pytest.mark.cassette("test_responses_stream_until_done[content_mode0]") -@pytest.mark.vcr() -@pytest.mark.skipif( - not _has_conversation_param, - reason="openai SDK too old to support 'conversation' on Responses.create", -) -def test_responses_stream_records_conversation_id( - span_exporter, openai_client, instrument_no_content -): - _skip_if_not_latest() - - with openai_client.responses.stream( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - service_tier="default", - conversation=CONVERSATION_ID, - ) as stream: - stream.get_final_response() - - (span,) = span_exporter.get_finished_spans() - _assert_conversation_id(span) - - -@pytest.mark.vcr() def test_responses_create_token_usage( - request, span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): _skip_if_not_latest() - response = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input="Count to 5.", - ) + with vcr.use_cassette( + "test_responses_create_token_usage[content_mode0].yaml" + ): + response = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input="Count to 5.", + ) (span,) = span_exporter.get_finished_spans() assert ( @@ -676,33 +708,37 @@ def test_responses_create_token_usage( ) -@pytest.mark.vcr() def test_responses_create_aggregates_cache_tokens( - request, span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): _skip_if_not_latest() - response = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - ) + with vcr.use_cassette( + "test_responses_create_aggregates_cache_tokens[content_mode0].yaml" + ): + response = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + ) (span,) = span_exporter.get_finished_spans() assert_cache_attributes(span, response.usage) -@pytest.mark.vcr() def test_responses_create_stop_reason( - request, span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): _skip_if_not_latest() - openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input="Say hi.", - ) + with vcr.use_cassette( + "test_responses_create_stop_reason[content_mode0].yaml" + ): + openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input="Say hi.", + ) (span,) = span_exporter.get_finished_spans() assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] == ( @@ -718,7 +754,7 @@ def test_responses_create_connection_error( client = OpenAI(base_url="http://localhost:4242") with pytest.raises(APIConnectionError): - client.responses.create( # pylint: disable=no-member + client.responses.create( model=DEFAULT_MODEL, input="Hello", timeout=0.1, @@ -736,17 +772,19 @@ def test_responses_create_connection_error( ) -@pytest.mark.vcr() def test_responses_create_api_error( - request, span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): _skip_if_not_latest() - with pytest.raises((BadRequestError, NotFoundError)) as exc_info: - openai_client.responses.create( - model=INVALID_MODEL, - input="Hello", - ) + with vcr.use_cassette( + "test_responses_create_api_error[content_mode0].yaml" + ): + with pytest.raises((BadRequestError, NotFoundError)) as exc_info: + openai_client.responses.create( + model=INVALID_MODEL, + input="Hello", + ) (span,) = span_exporter.get_finished_spans() assert ( @@ -766,68 +804,49 @@ def test_responses_create_streaming_timing_metrics( with vcr.use_cassette( "test_responses_create_streaming[content_mode0].yaml" ): - with openai_client.responses.create( + stream = openai_client.responses.create( model=DEFAULT_MODEL, instructions=SYSTEM_INSTRUCTIONS, input=USER_ONLY_PROMPT[0]["content"], service_tier="default", stream=True, - ) as stream: - _collect_completed_response(stream) + ) + _collect_completed_response(stream) assert_responses_streaming_timing_metrics(metric_reader) -@pytest.mark.vcr() -def test_responses_create_streaming( - request, span_exporter, openai_client, instrument_no_content -): - _skip_if_not_latest() +def assert_responses_streaming_timing_metrics(metric_reader): + metrics = metric_reader.get_metrics_data() + names = _metric_names(metrics) + assert "gen_ai.client.operation.duration" in names - with openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - service_tier="default", - stream=True, - ) as stream: - response = _collect_completed_response(stream) - (span,) = span_exporter.get_finished_spans() - assert_all_attributes( - span, - DEFAULT_MODEL, - True, - response.id, - response.model, - response.usage.input_tokens, - response.usage.output_tokens, - request_service_tier="default", - response_service_tier=getattr(response, "service_tier", None), - ) +def _metric_names(metrics): + names = set() + for resource_metrics in metrics.resource_metrics: + for scope_metrics in resource_metrics.scope_metrics: + for metric in scope_metrics.metrics: + names.add(metric.name) + return names -def test_responses_with_raw_response_streaming( - span_exporter, openai_client, instrument_with_content, vcr +def test_responses_create_streaming( + span_exporter, openai_client, instrument_no_content, vcr ): _skip_if_not_latest() with vcr.use_cassette( "test_responses_create_streaming[content_mode0].yaml" ): - raw_response = openai_client.responses.with_raw_response.create( + stream = openai_client.responses.create( model=DEFAULT_MODEL, instructions=SYSTEM_INSTRUCTIONS, input=USER_ONLY_PROMPT[0]["content"], service_tier="default", stream=True, ) - - # Raw-response metadata resolves natively off the wrapper (issue #46). - assert "openai-version" in raw_response.headers - assert raw_response.request_id is not None - - response = _collect_completed_response(raw_response.parse()) + response = _collect_completed_response(stream) (span,) = span_exporter.get_finished_spans() assert_all_attributes( @@ -843,26 +862,23 @@ def test_responses_with_raw_response_streaming( ) -def test_responses_with_streaming_response_parse( - span_exporter, openai_client, instrument_with_content, vcr +@pytest.mark.vcr() +def test_responses_stream_captures_content( + span_exporter, + log_exporter, + openai_client, + instrument_with_content, ): - """``with_streaming_response`` + ``parse()`` traces like a plain stream.""" _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_streaming[content_mode0].yaml" - ): - with openai_client.responses.with_streaming_response.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - service_tier="default", - stream=True, - ) as raw_response: - # Metadata resolves natively off the wrapper. - assert "openai-version" in raw_response.headers - - response = _collect_completed_response(raw_response.parse()) + manager = openai_client.responses.stream( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + ) + assert isinstance(manager, ResponseStreamManagerWrapper) + with manager as stream: + response = _collect_completed_response(stream) (span,) = span_exporter.get_finished_spans() assert_all_attributes( @@ -873,56 +889,9 @@ def test_responses_with_streaming_response_parse( response.model, response.usage.input_tokens, response.usage.output_tokens, - request_service_tier="default", response_service_tier=getattr(response, "service_tier", None), ) - - -class _UnrelatedEvent(BaseModel): - """An event type unrelated to the Responses stream events.""" - - foo: str = "bar" - - -def test_responses_with_raw_response_streaming_unknown_event_type( - span_exporter, openai_client, instrument_with_content, vcr -): - # A caller can parse the raw stream into an event type we don't recognize. - # Telemetry extraction must not break iteration: the caller must drain the - # same events it would with instrumentation disabled, and the span must - # still close (empty telemetry) instead of leaking. - _skip_if_not_latest() - - with vcr.use_cassette( - "test_responses_create_streaming[content_mode0].yaml" - ): - raw_response = openai_client.responses.with_raw_response.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - service_tier="default", - stream=True, - ) - events = list(raw_response.parse(to=Stream[_UnrelatedEvent])) - - assert len(events) > 0 # drained fine, same as disabled instrumentation - - (span,) = span_exporter.get_finished_spans() # span closed, did not leak - assert span.end_time is not None - - -def test_responses_stream_returns_wrapped_manager( - openai_client, instrument_no_content -): - _skip_if_not_latest() - - manager = openai_client.responses.stream( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - ) - - assert isinstance(manager, ResponseStreamManagerWrapper) + _assert_response_content(span, response, log_exporter) def test_responses_stream_connection_error( @@ -935,56 +904,22 @@ def test_responses_stream_connection_error( with pytest.raises(APIConnectionError): with client.responses.stream( model=DEFAULT_MODEL, - input="Hello", - timeout=0.1, - ): - pass - - (span,) = span_exporter.get_finished_spans() - assert ( - span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == DEFAULT_MODEL - ) - assert ( - span.attributes[ErrorAttributes.ERROR_TYPE] - == "openai.APIConnectionError" - ) - - -@pytest.mark.vcr() -def test_responses_stream_captures_content( - span_exporter, - log_exporter, - openai_client, - instrument_with_content, -): - _skip_if_not_latest() - - with openai_client.responses.stream( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - ) as stream: - response = _collect_completed_response(stream) - - (span,) = span_exporter.get_finished_spans() - assert_all_attributes( - span, - DEFAULT_MODEL, - True, - response.id, - response.model, - response.usage.input_tokens, - response.usage.output_tokens, - response_service_tier=getattr(response, "service_tier", None), + input="Hello", + timeout=0.1, + ): + pass + + (span,) = span_exporter.get_finished_spans() + assert ( + span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == DEFAULT_MODEL + ) + assert ( + span.attributes[ErrorAttributes.ERROR_TYPE] + == "openai.APIConnectionError" ) - _assert_response_content(span, response, log_exporter) @pytest.mark.vcr() -@pytest.mark.skipif( - not _stream_has_service_tier, - reason="openai SDK too old to support 'service_tier' on Responses.stream", -) def test_responses_stream_until_done( span_exporter, openai_client, instrument_no_content ): @@ -1034,40 +969,44 @@ def test_responses_stream_user_exception( assert span.attributes[ErrorAttributes.ERROR_TYPE] == "ValueError" -@pytest.mark.vcr() def test_responses_create_streaming_aggregates_cache_tokens( - request, span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): _skip_if_not_latest() - with openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=True, - ) as stream: + with vcr.use_cassette( + "test_responses_create_streaming_aggregates_cache_tokens[content_mode0].yaml" + ): + stream = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=True, + ) response = _collect_completed_response(stream) (span,) = span_exporter.get_finished_spans() assert_cache_attributes(span, response.usage) -@pytest.mark.vcr() def test_responses_create_streaming_captures_content( - request, span_exporter, log_exporter, openai_client, instrument_with_content, + vcr, ): _skip_if_not_latest() - with openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=True, - ) as stream: + with vcr.use_cassette( + "test_responses_create_streaming_captures_content[content_mode0].yaml" + ): + stream = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=True, + ) response = _collect_completed_response(stream) (span,) = span_exporter.get_finished_spans() @@ -1084,19 +1023,21 @@ def test_responses_create_streaming_captures_content( _assert_response_content(span, response, log_exporter) -@pytest.mark.vcr() def test_responses_create_streaming_iteration( - request, span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): _skip_if_not_latest() - stream = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input="Say hi.", - stream=True, - ) - events = list(stream) + with vcr.use_cassette( + "test_responses_create_streaming_iteration[content_mode0].yaml" + ): + stream = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input="Say hi.", + stream=True, + ) + events = list(stream) assert len(events) > 0 @@ -1113,23 +1054,25 @@ def test_responses_create_streaming_iteration( assert GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS in span.attributes -@pytest.mark.vcr() def test_responses_create_streaming_delegates_response_attribute( - request, openai_client, instrument_no_content + openai_client, instrument_no_content, vcr ): _skip_if_not_latest() - stream = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input="Say hi.", - stream=True, - ) + with vcr.use_cassette( + "test_responses_create_streaming_delegates_response_attribute[content_mode0].yaml" + ): + stream = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input="Say hi.", + stream=True, + ) - assert stream.response is not None - assert stream.response.status_code == 200 - assert stream.response.headers.get("x-request-id") is not None - stream.close() + assert stream.response is not None + assert stream.response.status_code == 200 + assert stream.response.headers.get("x-request-id") is not None + stream.close() def test_responses_create_streaming_connection_error( @@ -1140,7 +1083,7 @@ def test_responses_create_streaming_connection_error( client = OpenAI(base_url="http://localhost:4242") with pytest.raises(APIConnectionError): - client.responses.create( # pylint: disable=no-member + client.responses.create( model=DEFAULT_MODEL, input="Hello", stream=True, @@ -1157,21 +1100,23 @@ def test_responses_create_streaming_connection_error( ) -@pytest.mark.vcr() def test_responses_stream_wrapper_finalize_idempotent( - request, span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): _skip_if_not_latest() - stream = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=True, - ) + with vcr.use_cassette( + "test_responses_stream_wrapper_finalize_idempotent[content_mode0].yaml" + ): + stream = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=True, + ) - response = _collect_completed_response(stream) - stream.close() + response = _collect_completed_response(stream) + stream.close() spans = span_exporter.get_finished_spans() assert len(spans) == 1 @@ -1187,49 +1132,51 @@ def test_responses_stream_wrapper_finalize_idempotent( ) -@pytest.mark.vcr() def test_responses_create_stream_propagation_error( - request, span_exporter, openai_client, instrument_no_content, monkeypatch + span_exporter, openai_client, instrument_no_content, monkeypatch, vcr ): _skip_if_not_latest() - stream = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=True, - ) + with vcr.use_cassette( + "test_responses_create_stream_propagation_error[content_mode0].yaml" + ): + stream = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=True, + ) - class ErrorInjectingStreamDelegate: - def __init__(self, inner): - self._inner = inner - self._count = 0 + class ErrorInjectingStreamDelegate: + def __init__(self, inner): + self._inner = inner + self._count = 0 - def __iter__(self): - return self + def __iter__(self): + return self - def __next__(self): - if self._count == 1: - raise ConnectionError("connection reset during stream") - self._count += 1 - return next(self._inner) + def __next__(self): + if self._count == 1: + raise ConnectionError("connection reset during stream") + self._count += 1 + return next(self._inner) - def close(self): - return self._inner.close() + def close(self): + return self._inner.close() - def __getattr__(self, name): - return getattr(self._inner, name) + def __getattr__(self, name): + return getattr(self._inner, name) - monkeypatch.setattr( - stream, "stream", ErrorInjectingStreamDelegate(stream.stream) - ) + monkeypatch.setattr( + stream, "stream", ErrorInjectingStreamDelegate(stream.stream) + ) - with pytest.raises( - ConnectionError, match="connection reset during stream" - ): - with stream: - for _ in stream: - pass + with pytest.raises( + ConnectionError, match="connection reset during stream" + ): + with stream: + for _ in stream: + pass (span,) = span_exporter.get_finished_spans() assert ( @@ -1238,21 +1185,23 @@ def __getattr__(self, name): assert span.attributes[ErrorAttributes.ERROR_TYPE] == "ConnectionError" -@pytest.mark.vcr() def test_responses_create_streaming_user_exception( - request, span_exporter, openai_client, instrument_no_content + span_exporter, openai_client, instrument_no_content, vcr ): _skip_if_not_latest() - with pytest.raises(ValueError, match="User raised exception"): - with openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=True, - ) as stream: - for _ in stream: - raise ValueError("User raised exception") + with vcr.use_cassette( + "test_responses_create_streaming_user_exception[content_mode0].yaml" + ): + with pytest.raises(ValueError, match="User raised exception"): + with openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=True, + ) as stream: + for _ in stream: + raise ValueError("User raised exception") (span,) = span_exporter.get_finished_spans() assert ( @@ -1261,117 +1210,79 @@ def test_responses_create_streaming_user_exception( assert span.attributes[ErrorAttributes.ERROR_TYPE] == "ValueError" -@pytest.mark.vcr() -@pytest.mark.skipif( - not _has_tools_param, - reason="openai SDK too old to support 'tools' parameter on Responses.create", -) -def test_responses_create_captures_tool_loop_history( - request, span_exporter, openai_client, instrument_with_content -): - _skip_if_not_latest() - - openai_client.responses.create( - model=DEFAULT_MODEL, - input=get_responses_tool_loop_input(), - tools=[get_responses_weather_tool_definition()], - ) - - (span,) = span_exporter.get_finished_spans() - assert_messages_attribute( - span.attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES], - EXPECTED_TOOL_LOOP_INPUT_MESSAGES, - ) - - @pytest.mark.skipif( not _has_custom_tool_types, reason="openai SDK too old to support custom tool call types", ) -@pytest.mark.vcr() @pytest.mark.skipif( not _has_tools_param, reason="openai SDK too old to support 'tools' parameter on Responses.create", ) -def test_responses_create_captures_custom_tool_call_output( - request, span_exporter, openai_client, instrument_with_content +def test_responses_create_captures_custom_tool_history( + span_exporter, openai_client, instrument_with_content, vcr ): - """A custom tool call the model requests is recorded on the output side too.""" _skip_if_not_latest() - openai_client.responses.create( - model=CUSTOM_TOOL_MODEL, - input="Use the run_sql tool to count the rows in the users table.", - tools=[get_responses_custom_tool_definition()], - tool_choice="auto", - ) + with vcr.use_cassette( + "test_responses_create_captures_custom_tool_history[content_mode0].yaml" + ): + openai_client.responses.create( + model=CUSTOM_TOOL_MODEL, + input=get_responses_custom_tool_loop_input(), + tools=[get_responses_custom_tool_definition()], + tool_choice="auto", + ) (span,) = span_exporter.get_finished_spans() - assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] == ( - "tool_calls", - ) - output_messages = _load_span_messages( - span, GenAIAttributes.GEN_AI_OUTPUT_MESSAGES + assert_messages_attribute( + span.attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES], + EXPECTED_CUSTOM_TOOL_INPUT_MESSAGES, ) - tool_calls = [ - part - for message in output_messages - for part in message.get("parts", []) - if part.get("type") == "tool_call" - ] - (tool_call,) = tool_calls - assert tool_call["name"] == "run_sql" - assert tool_call["id"] == CUSTOM_TOOL_CALL_ID - # The same id the replayed history correlates on, so the two spans join up. - assert tool_call["arguments"] == CUSTOM_TOOL_INPUT -@pytest.mark.skipif( - not _has_custom_tool_types, - reason="openai SDK too old to support custom tool call types", -) -@pytest.mark.vcr() @pytest.mark.skipif( not _has_tools_param, reason="openai SDK too old to support 'tools' parameter on Responses.create", ) -def test_responses_create_captures_custom_tool_history( - request, span_exporter, openai_client, instrument_with_content +def test_responses_create_captures_tool_loop_history( + span_exporter, openai_client, instrument_with_content, vcr ): _skip_if_not_latest() - openai_client.responses.create( - model=CUSTOM_TOOL_MODEL, - input=get_responses_custom_tool_loop_input(), - tools=[get_responses_custom_tool_definition()], - tool_choice="auto", - ) + with vcr.use_cassette( + "test_responses_create_captures_tool_loop_history[content_mode0].yaml" + ): + openai_client.responses.create( + model=DEFAULT_MODEL, + input=get_responses_tool_loop_input(), + tools=[get_responses_weather_tool_definition()], + ) (span,) = span_exporter.get_finished_spans() - # The replayed `reasoning` item is not recorded: it carries no readable - # text, and the response path drops such items too. assert_messages_attribute( span.attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES], - EXPECTED_CUSTOM_TOOL_INPUT_MESSAGES, + EXPECTED_TOOL_LOOP_INPUT_MESSAGES, ) -@pytest.mark.vcr() @pytest.mark.skipif( not _has_tools_param, reason="openai SDK too old to support 'tools' parameter on Responses.create", ) def test_responses_create_captures_tool_call_content( - request, span_exporter, openai_client, instrument_with_content + span_exporter, openai_client, instrument_with_content, vcr ): _skip_if_not_latest() - openai_client.responses.create( - model=DEFAULT_MODEL, - input="What's the weather in Seattle right now?", - tools=[get_responses_weather_tool_definition()], - tool_choice={"type": "function", "name": "get_current_weather"}, - ) + with vcr.use_cassette( + "test_responses_create_captures_tool_call_content[content_mode0].yaml" + ): + openai_client.responses.create( + model=DEFAULT_MODEL, + input="What's the weather in Seattle right now?", + tools=[get_responses_weather_tool_definition()], + tool_choice={"type": "function", "name": "get_current_weather"}, + ) (span,) = span_exporter.get_finished_spans() assert ( @@ -1420,12 +1331,13 @@ def test_responses_create_streaming_captures_tool_definitions( with vcr.use_cassette( "test_responses_create_streaming_captures_content[content_mode0].yaml" ): - with openai_client.responses.create( + stream = openai_client.responses.create( model=DEFAULT_MODEL, input=USER_ONLY_PROMPT[0]["content"], tools=[get_responses_weather_tool_definition()], stream=True, - ) as stream: + ) + with stream: _collect_completed_response(stream) (span,) = span_exporter.get_finished_spans() @@ -1462,7 +1374,6 @@ def test_responses_stream_captures_tool_definitions( ) -@pytest.mark.vcr() @pytest.mark.skipif( not _has_reasoning_param, reason=( @@ -1470,22 +1381,25 @@ def test_responses_stream_captures_tool_definitions( ), ) def test_responses_create_reports_reasoning_tokens( - request, span_exporter, openai_client, instrument_with_content + span_exporter, openai_client, instrument_with_content, vcr ): _skip_if_not_latest() - response = openai_client.responses.create( - model=REASONING_MODEL, - reasoning={"effort": "low"}, - input=[ - { - "role": "user", - "content": REASONING_PROMPT, - } - ], - max_output_tokens=300, - timeout=30.0, - ) + with vcr.use_cassette( + "test_responses_create_reports_reasoning_tokens[content_mode0].yaml" + ): + response = openai_client.responses.create( + model=REASONING_MODEL, + reasoning={"effort": "low"}, + input=[ + { + "role": "user", + "content": REASONING_PROMPT, + } + ], + max_output_tokens=1000, + timeout=30.0, + ) reasoning_tokens = getattr( getattr(response.usage, "output_tokens_details", None), @@ -1520,43 +1434,47 @@ def test_responses_create_reports_reasoning_tokens( assert len(output_messages) > 0 -@pytest.mark.vcr() def test_responses_create_with_content_span_unsampled( - request, span_exporter, log_exporter, openai_client, instrument_with_content_unsampled, + vcr, ): _skip_if_not_latest() - openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=False, - ) + with vcr.use_cassette( + "test_responses_create_with_content_span_unsampled[content_mode0].yaml" + ): + openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=False, + ) assert len(span_exporter.get_finished_spans()) == 0 assert len(log_exporter.get_finished_logs()) == 0 -@pytest.mark.vcr() def test_responses_create_with_content_shapes( - request, span_exporter, log_exporter, openai_client, instrument_with_content, + vcr, ): _skip_if_not_latest() - openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=False, - ) + with vcr.use_cassette( + "test_responses_create_with_content_shapes[content_mode0].yaml" + ): + openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=False, + ) (span,) = span_exporter.get_finished_spans() input_messages = _load_span_messages( @@ -1573,18 +1491,24 @@ def test_responses_create_with_content_shapes( assert len(log_exporter.get_finished_logs()) == 0 -@pytest.mark.vcr() def test_responses_create_event_only_no_content_in_span( - request, span_exporter, log_exporter, openai_client, instrument_event_only + span_exporter, + log_exporter, + openai_client, + instrument_event_only, + vcr, ): _skip_if_not_latest() - openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=False, - ) + with vcr.use_cassette( + "test_responses_create_event_only_no_content_in_span.yaml" + ): + openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=False, + ) (span,) = span_exporter.get_finished_spans() assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes @@ -1612,18 +1536,12 @@ class _ParseCalendarEvent(BaseModel): def test_responses_parse_basic( span_exporter, openai_client, instrument_no_content, vcr ): - """Responses.parse() emits a GenAI span like create(). - - parse() is the structured-output helper. It does not delegate to the - instrumented create(), so it is wrapped separately (#659), but it maps to - the same inference operation as create() -- the request/response fields - are identical -- exactly as chat.completions.parse reuses the completions - create wrapper. The recorded response body is valid structured JSON so - the SDK can materialize the ``text_format`` model. - """ + """Responses.parse() emits a GenAI span like create() (#659).""" _skip_if_not_latest() - with vcr.use_cassette("test_responses_parse_basic[content_mode0].yaml"): + with vcr.use_cassette( + "test_responses_parse_basic[content_mode0].yaml" + ): response = openai_client.responses.parse( model=DEFAULT_MODEL, instructions=SYSTEM_INSTRUCTIONS, @@ -1647,34 +1565,46 @@ def test_responses_parse_basic( span.attributes[OpenAIAttributes.OPENAI_API_TYPE] == OpenAIAttributes.OpenaiApiTypeValues.RESPONSES.value ) + # parse(text_format=...) is a structured-output call, so the span records + # the JSON output type -- but only the format metadata, never the caller's + # Pydantic schema (issue #659). + _assert_request_attrs(span, output_type="json") + assert "_ParseCalendarEvent" not in str(span.attributes) @pytest.mark.skipif( not _HAS_RESPONSES_PARSE, reason="Responses.parse requires a newer openai SDK", ) -def test_responses_parse_wrapping_lifecycle( - tracer_provider, logger_provider, meter_provider +def test_responses_create_output_type_unchanged_by_parse( + span_exporter, openai_client, instrument_no_content, vcr ): - """instrument() wraps Responses.parse / AsyncResponses.parse and - uninstrument() restores them.""" - from openai.resources.responses.responses import ( # pylint: disable=no-name-in-module - AsyncResponses, - Responses, - ) + """Regression guard: create() must not start reporting an output type. - before_sync = Responses.parse - before_async = AsyncResponses.parse + Only parse(text_format=...) maps to ``gen_ai.output.type``; a plain text + create call stays without the attribute. + """ + _skip_if_not_latest() - instrumentor = OpenAIInstrumentor() - instrumentor.instrument( - tracer_provider=tracer_provider, - logger_provider=logger_provider, - meter_provider=meter_provider, - ) - assert hasattr(Responses.parse, "__wrapped__") - assert hasattr(AsyncResponses.parse, "__wrapped__") + with vcr.use_cassette( + "test_responses_create_basic[content_mode0].yaml" + ): + response = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=False, + ) - instrumentor.uninstrument() - assert Responses.parse is before_sync - assert AsyncResponses.parse is before_async + (span,) = span_exporter.get_finished_spans() + assert_all_attributes( + span, + DEFAULT_MODEL, + True, + response.id, + response.model, + response.usage.input_tokens, + response.usage.output_tokens, + response_service_tier=getattr(response, "service_tier", None), + ) + assert GenAIAttributes.GEN_AI_OUTPUT_TYPE not in span.attributes From 39826010c5659bb2340232bc6636e56af4175c0b Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Sun, 27 Sep 2026 10:38:01 +0800 Subject: [PATCH 18/18] test(genai-openai): restore Responses test file content lost to a bad write The previous push replaced test_responses.py with a re-generated variant of the file. Restore the exact on-disk content, which keeps the only intended change: the gen_ai.output.type assertion for Responses.parse and the create-behaviour regression test. --- .../tests/test_responses.py | 1147 +++++++++-------- 1 file changed, 630 insertions(+), 517 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py index e02921331..da519bb70 100644 --- a/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py +++ b/instrumentation/opentelemetry-instrumentation-genai-openai/tests/test_responses.py @@ -6,8 +6,13 @@ import json import pytest -from openai import APIConnectionError, BadRequestError, NotFoundError, OpenAI -from openai import Stream +from openai import ( + APIConnectionError, + BadRequestError, + NotFoundError, + OpenAI, + Stream, +) from pydantic import BaseModel from opentelemetry.instrumentation.genai.openai import OpenAIInstrumentor @@ -26,10 +31,13 @@ from opentelemetry.semconv._incubating.attributes import ( server_attributes as ServerAttributes, ) +from opentelemetry.semconv._incubating.metrics import gen_ai_metrics from opentelemetry.trace.status import StatusCode from opentelemetry.util.genai.utils import is_experimental_mode from .test_utils import ( + CUSTOM_TOOL_CALL_ID, + CUSTOM_TOOL_INPUT, CUSTOM_TOOL_MODEL, DEFAULT_MODEL, EXPECTED_CUSTOM_TOOL_INPUT_MESSAGES, @@ -57,15 +65,11 @@ from openai.resources.responses.responses import Responses as _Responses HAS_RESPONSES_API = True - _create_params = set( - inspect.signature(_Responses.create).parameters - ) + _create_params = set(inspect.signature(_Responses.create).parameters) _has_tools_param = "tools" in _create_params _has_reasoning_param = "reasoning" in _create_params _has_conversation_param = "conversation" in _create_params - _stream_params = set( - inspect.signature(_Responses.stream).parameters - ) + _stream_params = set(inspect.signature(_Responses.stream).parameters) _stream_has_service_tier = "service_tier" in _stream_params _has_custom_tool_types = ( importlib.util.find_spec( @@ -81,6 +85,7 @@ _stream_has_service_tier = False _has_custom_tool_types = False + pytestmark = pytest.mark.skipif( not HAS_RESPONSES_API, reason="Responses API requires a newer openai SDK" ) @@ -117,15 +122,6 @@ def _skip_if_not_latest(): ) -def _collect_completed_response(stream): - response = None - for event in stream: - if event.type == "response.completed": - response = event.response - assert response is not None - return response - - def _load_span_messages(span, attribute): value = span.attributes.get(attribute) assert value is not None @@ -185,6 +181,56 @@ def _assert_request_attrs( ) +def _collect_completed_response(stream): + response = None + for event in stream: + if event.type == "response.completed": + response = event.response + assert response is not None + return response + + +def _collect_metrics(metric_reader): + metrics = {} + for rm in metric_reader.get_metrics_data().resource_metrics: + for scope in rm.scope_metrics: + for metric in scope.metrics: + metrics[metric.name] = metric + return metrics + + +def assert_responses_streaming_timing_metrics(metric_reader): + """Assert the streaming timing metrics are emitted through the real + Responses stream wrapper path. + + Regression coverage for the ``invocation=invocation`` wiring in + ``response_wrappers.py``: dropping it would keep every span/attribute test + green but silently stop emitting TTFC and per-output-chunk metrics for the + Responses streaming path. + """ + metrics = _collect_metrics(metric_reader) + + ttfc = metrics.get( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_TO_FIRST_CHUNK + ) + assert ttfc is not None + ttfc_point = ttfc.data.data_points[0] + assert ttfc_point.count == 1 + assert ttfc_point.sum >= 0 + assert ( + ttfc_point.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + == GenAIAttributes.GenAiOperationNameValues.CHAT.value + ) + + per_chunk = metrics.get( + gen_ai_metrics.GEN_AI_CLIENT_OPERATION_TIME_PER_OUTPUT_CHUNK + ) + assert per_chunk is not None + per_chunk_point = per_chunk.data.data_points[0] + assert per_chunk_point.count >= 1 + assert per_chunk_point.sum >= 0 + + def test_responses_uninstrument_removes_patching( span_exporter, tracer_provider, logger_provider, meter_provider ): @@ -199,20 +245,45 @@ def test_responses_uninstrument_removes_patching( assert len(span_exporter.get_finished_spans()) == 0 +def test_responses_multiple_instrument_uninstrument_cycles( + tracer_provider, logger_provider, meter_provider +): + instrumentor = OpenAIInstrumentor() + + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + instrumentor.uninstrument() + + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + instrumentor.uninstrument() + + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + instrumentor.uninstrument() + + +@pytest.mark.vcr() def test_responses_create_basic( - span_exporter, openai_client, instrument_no_content, vcr + request, span_exporter, openai_client, instrument_no_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_basic[content_mode0].yaml" - ): - response = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=False, - ) + response = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=False, + ) (span,) = span_exporter.get_finished_spans() assert_all_attributes( @@ -254,15 +325,13 @@ def test_responses_create_basic( RETRIEVE_STREAM_CURSOR = 3 +@pytest.mark.vcr() def test_responses_retrieve_basic( - span_exporter, openai_client, instrument_no_content, vcr + span_exporter, openai_client, instrument_no_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_retrieve_basic[content_mode0].yaml" - ): - response = openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) + response = openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) (span,) = span_exporter.get_finished_spans() assert_fetch_response_attributes( @@ -277,19 +346,13 @@ def test_responses_retrieve_basic( assert GenAIAttributes.GEN_AI_SYSTEM_INSTRUCTIONS not in span.attributes +@pytest.mark.vcr() def test_responses_retrieve_captures_content( - span_exporter, - log_exporter, - openai_client, - instrument_with_content, - vcr, + span_exporter, log_exporter, openai_client, instrument_with_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_retrieve_captures_content[content_mode0].yaml" - ): - response = openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) + response = openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) (span,) = span_exporter.get_finished_spans() assert_messages_attribute( @@ -305,18 +368,16 @@ def test_responses_retrieve_captures_content( assert len(log_exporter.get_finished_logs()) == 0 +@pytest.mark.vcr() def test_responses_retrieve_incomplete( - span_exporter, openai_client, instrument_no_content, vcr + span_exporter, openai_client, instrument_no_content ): """An incomplete stored response surfaces via status and finish reasons.""" _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_retrieve_incomplete[content_mode0].yaml" - ): - response = openai_client.responses.retrieve( - RETRIEVE_INCOMPLETE_RESPONSE_ID - ) + response = openai_client.responses.retrieve( + RETRIEVE_INCOMPLETE_RESPONSE_ID + ) (span,) = span_exporter.get_finished_spans() assert_fetch_response_attributes( @@ -331,20 +392,14 @@ def test_responses_retrieve_incomplete( assert ErrorAttributes.ERROR_TYPE not in span.attributes +@pytest.mark.vcr() def test_responses_retrieve_failed_generation_is_not_a_fetch_error( - span_exporter, openai_client, instrument_no_content, vcr + span_exporter, openai_client, instrument_no_content ): """A stored response whose generation failed is not a failure of the fetch.""" _skip_if_not_latest() - cassette = ( - "test_responses_retrieve_failed_generation_is_not_a_fetch_error" - "[content_mode0].yaml" - ) - with vcr.use_cassette(cassette): - response = openai_client.responses.retrieve( - RETRIEVE_FAILED_RESPONSE_ID - ) + response = openai_client.responses.retrieve(RETRIEVE_FAILED_RESPONSE_ID) (span,) = span_exporter.get_finished_spans() assert_fetch_response_attributes( @@ -359,24 +414,22 @@ def test_responses_retrieve_failed_generation_is_not_a_fetch_error( assert ErrorAttributes.ERROR_TYPE not in span.attributes +@pytest.mark.vcr() def test_responses_retrieve_streaming( - span_exporter, openai_client, instrument_with_content, vcr + span_exporter, openai_client, instrument_with_content ): """A streamed replay finalizes only once the caller drains the stream.""" _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_retrieve_streaming[content_mode0].yaml" - ): - stream = openai_client.responses.retrieve( - RETRIEVE_STREAM_RESPONSE_ID, - stream=True, - starting_after=RETRIEVE_STREAM_CURSOR, - ) - assert isinstance(stream, Stream) - assert span_exporter.get_finished_spans() == () + stream = openai_client.responses.retrieve( + RETRIEVE_STREAM_RESPONSE_ID, + stream=True, + starting_after=RETRIEVE_STREAM_CURSOR, + ) + assert isinstance(stream, Stream) + assert span_exporter.get_finished_spans() == () - response = _collect_completed_response(stream) + response = _collect_completed_response(stream) (span,) = span_exporter.get_finished_spans() assert_fetch_response_attributes( @@ -394,21 +447,17 @@ def test_responses_retrieve_streaming( ) +@pytest.mark.vcr() def test_responses_retrieve_raw_response( - span_exporter, openai_client, instrument_no_content, vcr + span_exporter, openai_client, instrument_no_content ): """``with_raw_response`` keeps returning the raw response, still traced.""" _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_retrieve_raw_response[content_mode0].yaml" - ): - raw_response = openai_client.responses.with_raw_response.retrieve( - RETRIEVE_RESPONSE_ID - ) - response = raw_response.parse() - if inspect.isawaitable(response): - response = _resolve_awaitable(response) + raw_response = openai_client.responses.with_raw_response.retrieve( + RETRIEVE_RESPONSE_ID + ) + response = raw_response.parse() (span,) = span_exporter.get_finished_spans() assert_fetch_response_attributes( @@ -421,34 +470,21 @@ def test_responses_retrieve_raw_response( ) -def _resolve_awaitable(value): - """Resolve an awaitable without a running loop (sync client returns one).""" - if inspect.isawaitable(value): - return value - return value - - +@pytest.mark.vcr() def test_responses_retrieve_with_streaming_response_stays_lazy( - span_exporter, openai_client, instrument_no_content, vcr + span_exporter, openai_client, instrument_no_content ): """``with_streaming_response`` must not have its body read by telemetry.""" _skip_if_not_latest() - cassette = ( - "test_responses_retrieve_with_streaming_response_stays_lazy" - "[content_mode0].yaml" - ) - with vcr.use_cassette(cassette): - with ( - openai_client.responses.with_streaming_response.retrieve( - RETRIEVE_RESPONSE_ID - ) - ) as raw_response: - # Building telemetry must not consume or close the body before the - # caller reads it. - assert not raw_response.http_response.is_stream_consumed - assert not raw_response.http_response.is_closed - response = raw_response.parse() + with openai_client.responses.with_streaming_response.retrieve( + RETRIEVE_RESPONSE_ID + ) as raw_response: + # Building telemetry must not consume or close the body before the + # caller reads it. + assert not raw_response.http_response.is_stream_consumed + assert not raw_response.http_response.is_closed + response = raw_response.parse() (span,) = span_exporter.get_finished_spans() assert span.name == "fetch_response" @@ -464,16 +500,14 @@ def test_responses_retrieve_with_streaming_response_stays_lazy( assert response.id == RETRIEVE_RESPONSE_ID +@pytest.mark.vcr() def test_responses_retrieve_api_error( - span_exporter, openai_client, instrument_no_content, vcr + span_exporter, openai_client, instrument_no_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_retrieve_api_error[content_mode0].yaml" - ): - with pytest.raises(NotFoundError) as exc_info: - openai_client.responses.retrieve(RETRIEVE_MISSING_RESPONSE_ID) + with pytest.raises(NotFoundError) as exc_info: + openai_client.responses.retrieve(RETRIEVE_MISSING_RESPONSE_ID) (span,) = span_exporter.get_finished_spans() assert span.name == "fetch_response" @@ -494,133 +528,49 @@ def test_responses_retrieve_api_error( assert GEN_AI_RESPONSE_STATUS not in span.attributes -def test_responses_with_raw_response_streaming( - span_exporter, openai_client, instrument_with_content, vcr +def test_responses_retrieve_does_not_record_token_usage_metric( + span_exporter, metric_reader, openai_client, instrument_no_content, vcr ): + """A fetch consumes no tokens, so only the duration metric is recorded.""" _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_streaming[content_mode0].yaml" - ): - raw_response = ( - openai_client.responses.with_raw_response.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - service_tier="default", - stream=True, - ) - ) + with vcr.use_cassette("test_responses_retrieve_basic[content_mode0].yaml"): + openai_client.responses.retrieve(RETRIEVE_RESPONSE_ID) - # Raw-response metadata resolves natively off the wrapper (issue #46). - assert "openai-version" in raw_response.headers - assert raw_response.request_id is not None + metrics = _collect_metrics(metric_reader) + assert gen_ai_metrics.GEN_AI_CLIENT_TOKEN_USAGE not in metrics - response = _collect_completed_response(raw_response.parse()) - - (span,) = span_exporter.get_finished_spans() - assert_all_attributes( - span, - DEFAULT_MODEL, - True, - response.id, - response.model, - response.usage.input_tokens, - response.usage.output_tokens, - request_service_tier="default", - response_service_tier=getattr(response, "service_tier", None), + duration = metrics[gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION] + (point,) = duration.data.data_points + assert ( + point.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] + == "fetch_response" ) - - -def test_responses_with_streaming_response_parse( - span_exporter, openai_client, instrument_with_content, vcr -): - """``APIResponse.parse()`` still hands back the parsed stream.""" - _skip_if_not_latest() - - with vcr.use_cassette( - "test_responses_create_streaming[content_mode0].yaml" - ): - with ( - openai_client.responses.with_streaming_response.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - service_tier="default", - stream=True, - ) - ) as raw_response: - assert "openai-version" in raw_response.headers - - response = _collect_completed_response(raw_response.parse()) - - (span,) = span_exporter.get_finished_spans() - assert_all_attributes( - span, - DEFAULT_MODEL, - True, - response.id, - response.model, - response.usage.input_tokens, - response.usage.output_tokens, - request_service_tier="default", - response_service_tier=getattr(response, "service_tier", None), + assert ( + point.attributes[GenAIAttributes.GEN_AI_RESPONSE_MODEL] + == "gpt-4o-mini-2024-07-18" ) + # The response id is high cardinality and must stay off metrics. + assert GenAIAttributes.GEN_AI_RESPONSE_ID not in point.attributes -class _UnrelatedEvent(BaseModel): - """An event type unrelated to the Responses stream events.""" - - foo: str = "bar" - - -def test_responses_with_raw_response_streaming_unknown_event_type( - span_exporter, openai_client, instrument_with_content, vcr -): - # Parsing the raw stream into an event type we don't recognize must not - # break iteration: the caller drains the same events it would with - # instrumentation disabled, and the span still closes instead of leaking. - _skip_if_not_latest() - - with vcr.use_cassette( - "test_responses_create_streaming[content_mode0].yaml" - ): - raw_response = ( - openai_client.responses.with_raw_response.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - service_tier="default", - stream=True, - ) - ) - events = list(raw_response.parse(to=Stream[_UnrelatedEvent])) - - assert len(events) > 0 # drained fine, same as disabled instrumentation - - (span,) = span_exporter.get_finished_spans() # span closed, did not leak - assert span.end_time is not None - - +@pytest.mark.vcr() def test_responses_create_captures_content( + request, span_exporter, log_exporter, openai_client, instrument_with_content, - vcr, ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_captures_content[content_mode0].yaml" - ): - response = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=False, - text={"format": {"type": "text"}}, - ) + response = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=False, + text={"format": {"type": "text"}}, + ) (span,) = span_exporter.get_finished_spans() assert_all_attributes( @@ -636,30 +586,26 @@ def test_responses_create_captures_content( _assert_response_content(span, response, log_exporter) +@pytest.mark.vcr() def test_responses_create_with_all_params( - span_exporter, openai_client, instrument_no_content, vcr + request, span_exporter, openai_client, instrument_no_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_with_all_params[content_mode0].yaml" - ): - conversation_kwargs = ( - {"conversation": CONVERSATION_ID} - if _has_conversation_param - else {} - ) - response = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - max_output_tokens=50, - temperature=0.7, - top_p=0.9, - service_tier="default", - text={"format": {"type": "text"}}, - **conversation_kwargs, - ) + conversation_kwargs = ( + {"conversation": CONVERSATION_ID} if _has_conversation_param else {} + ) + response = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + max_output_tokens=50, + temperature=0.7, + top_p=0.9, + service_tier="default", + text={"format": {"type": "text"}}, + **conversation_kwargs, + ) (span,) = span_exporter.get_finished_spans() assert_all_attributes( @@ -683,19 +629,41 @@ def test_responses_create_with_all_params( _assert_conversation_id(span) +@pytest.mark.cassette("test_responses_stream_until_done[content_mode0]") +@pytest.mark.vcr() +@pytest.mark.skipif( + not _has_conversation_param, + reason="openai SDK too old to support 'conversation' on Responses.create", +) +def test_responses_stream_records_conversation_id( + span_exporter, openai_client, instrument_no_content +): + _skip_if_not_latest() + + with openai_client.responses.stream( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + service_tier="default", + conversation=CONVERSATION_ID, + ) as stream: + stream.get_final_response() + + (span,) = span_exporter.get_finished_spans() + _assert_conversation_id(span) + + +@pytest.mark.vcr() def test_responses_create_token_usage( - span_exporter, openai_client, instrument_no_content, vcr + request, span_exporter, openai_client, instrument_no_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_token_usage[content_mode0].yaml" - ): - response = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input="Count to 5.", - ) + response = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input="Count to 5.", + ) (span,) = span_exporter.get_finished_spans() assert ( @@ -708,37 +676,33 @@ def test_responses_create_token_usage( ) +@pytest.mark.vcr() def test_responses_create_aggregates_cache_tokens( - span_exporter, openai_client, instrument_no_content, vcr + request, span_exporter, openai_client, instrument_no_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_aggregates_cache_tokens[content_mode0].yaml" - ): - response = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - ) + response = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + ) (span,) = span_exporter.get_finished_spans() assert_cache_attributes(span, response.usage) +@pytest.mark.vcr() def test_responses_create_stop_reason( - span_exporter, openai_client, instrument_no_content, vcr + request, span_exporter, openai_client, instrument_no_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_stop_reason[content_mode0].yaml" - ): - openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input="Say hi.", - ) + openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input="Say hi.", + ) (span,) = span_exporter.get_finished_spans() assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] == ( @@ -754,7 +718,7 @@ def test_responses_create_connection_error( client = OpenAI(base_url="http://localhost:4242") with pytest.raises(APIConnectionError): - client.responses.create( + client.responses.create( # pylint: disable=no-member model=DEFAULT_MODEL, input="Hello", timeout=0.1, @@ -772,19 +736,17 @@ def test_responses_create_connection_error( ) +@pytest.mark.vcr() def test_responses_create_api_error( - span_exporter, openai_client, instrument_no_content, vcr + request, span_exporter, openai_client, instrument_no_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_api_error[content_mode0].yaml" - ): - with pytest.raises((BadRequestError, NotFoundError)) as exc_info: - openai_client.responses.create( - model=INVALID_MODEL, - input="Hello", - ) + with pytest.raises((BadRequestError, NotFoundError)) as exc_info: + openai_client.responses.create( + model=INVALID_MODEL, + input="Hello", + ) (span,) = span_exporter.get_finished_spans() assert ( @@ -804,49 +766,68 @@ def test_responses_create_streaming_timing_metrics( with vcr.use_cassette( "test_responses_create_streaming[content_mode0].yaml" ): - stream = openai_client.responses.create( + with openai_client.responses.create( model=DEFAULT_MODEL, instructions=SYSTEM_INSTRUCTIONS, input=USER_ONLY_PROMPT[0]["content"], service_tier="default", stream=True, - ) - _collect_completed_response(stream) + ) as stream: + _collect_completed_response(stream) assert_responses_streaming_timing_metrics(metric_reader) -def assert_responses_streaming_timing_metrics(metric_reader): - metrics = metric_reader.get_metrics_data() - names = _metric_names(metrics) - assert "gen_ai.client.operation.duration" in names +@pytest.mark.vcr() +def test_responses_create_streaming( + request, span_exporter, openai_client, instrument_no_content +): + _skip_if_not_latest() + with openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + service_tier="default", + stream=True, + ) as stream: + response = _collect_completed_response(stream) -def _metric_names(metrics): - names = set() - for resource_metrics in metrics.resource_metrics: - for scope_metrics in resource_metrics.scope_metrics: - for metric in scope_metrics.metrics: - names.add(metric.name) - return names + (span,) = span_exporter.get_finished_spans() + assert_all_attributes( + span, + DEFAULT_MODEL, + True, + response.id, + response.model, + response.usage.input_tokens, + response.usage.output_tokens, + request_service_tier="default", + response_service_tier=getattr(response, "service_tier", None), + ) -def test_responses_create_streaming( - span_exporter, openai_client, instrument_no_content, vcr +def test_responses_with_raw_response_streaming( + span_exporter, openai_client, instrument_with_content, vcr ): _skip_if_not_latest() with vcr.use_cassette( "test_responses_create_streaming[content_mode0].yaml" ): - stream = openai_client.responses.create( + raw_response = openai_client.responses.with_raw_response.create( model=DEFAULT_MODEL, instructions=SYSTEM_INSTRUCTIONS, input=USER_ONLY_PROMPT[0]["content"], service_tier="default", stream=True, ) - response = _collect_completed_response(stream) + + # Raw-response metadata resolves natively off the wrapper (issue #46). + assert "openai-version" in raw_response.headers + assert raw_response.request_id is not None + + response = _collect_completed_response(raw_response.parse()) (span,) = span_exporter.get_finished_spans() assert_all_attributes( @@ -862,23 +843,26 @@ def test_responses_create_streaming( ) -@pytest.mark.vcr() -def test_responses_stream_captures_content( - span_exporter, - log_exporter, - openai_client, - instrument_with_content, +def test_responses_with_streaming_response_parse( + span_exporter, openai_client, instrument_with_content, vcr ): + """``with_streaming_response`` + ``parse()`` traces like a plain stream.""" _skip_if_not_latest() - manager = openai_client.responses.stream( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - ) - assert isinstance(manager, ResponseStreamManagerWrapper) - with manager as stream: - response = _collect_completed_response(stream) + with vcr.use_cassette( + "test_responses_create_streaming[content_mode0].yaml" + ): + with openai_client.responses.with_streaming_response.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + service_tier="default", + stream=True, + ) as raw_response: + # Metadata resolves natively off the wrapper. + assert "openai-version" in raw_response.headers + + response = _collect_completed_response(raw_response.parse()) (span,) = span_exporter.get_finished_spans() assert_all_attributes( @@ -889,9 +873,56 @@ def test_responses_stream_captures_content( response.model, response.usage.input_tokens, response.usage.output_tokens, + request_service_tier="default", response_service_tier=getattr(response, "service_tier", None), ) - _assert_response_content(span, response, log_exporter) + + +class _UnrelatedEvent(BaseModel): + """An event type unrelated to the Responses stream events.""" + + foo: str = "bar" + + +def test_responses_with_raw_response_streaming_unknown_event_type( + span_exporter, openai_client, instrument_with_content, vcr +): + # A caller can parse the raw stream into an event type we don't recognize. + # Telemetry extraction must not break iteration: the caller must drain the + # same events it would with instrumentation disabled, and the span must + # still close (empty telemetry) instead of leaking. + _skip_if_not_latest() + + with vcr.use_cassette( + "test_responses_create_streaming[content_mode0].yaml" + ): + raw_response = openai_client.responses.with_raw_response.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + service_tier="default", + stream=True, + ) + events = list(raw_response.parse(to=Stream[_UnrelatedEvent])) + + assert len(events) > 0 # drained fine, same as disabled instrumentation + + (span,) = span_exporter.get_finished_spans() # span closed, did not leak + assert span.end_time is not None + + +def test_responses_stream_returns_wrapped_manager( + openai_client, instrument_no_content +): + _skip_if_not_latest() + + manager = openai_client.responses.stream( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + ) + + assert isinstance(manager, ResponseStreamManagerWrapper) def test_responses_stream_connection_error( @@ -910,16 +941,50 @@ def test_responses_stream_connection_error( pass (span,) = span_exporter.get_finished_spans() - assert ( - span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == DEFAULT_MODEL - ) - assert ( - span.attributes[ErrorAttributes.ERROR_TYPE] - == "openai.APIConnectionError" + assert ( + span.attributes[GenAIAttributes.GEN_AI_REQUEST_MODEL] == DEFAULT_MODEL + ) + assert ( + span.attributes[ErrorAttributes.ERROR_TYPE] + == "openai.APIConnectionError" + ) + + +@pytest.mark.vcr() +def test_responses_stream_captures_content( + span_exporter, + log_exporter, + openai_client, + instrument_with_content, +): + _skip_if_not_latest() + + with openai_client.responses.stream( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + ) as stream: + response = _collect_completed_response(stream) + + (span,) = span_exporter.get_finished_spans() + assert_all_attributes( + span, + DEFAULT_MODEL, + True, + response.id, + response.model, + response.usage.input_tokens, + response.usage.output_tokens, + response_service_tier=getattr(response, "service_tier", None), ) + _assert_response_content(span, response, log_exporter) @pytest.mark.vcr() +@pytest.mark.skipif( + not _stream_has_service_tier, + reason="openai SDK too old to support 'service_tier' on Responses.stream", +) def test_responses_stream_until_done( span_exporter, openai_client, instrument_no_content ): @@ -969,44 +1034,40 @@ def test_responses_stream_user_exception( assert span.attributes[ErrorAttributes.ERROR_TYPE] == "ValueError" +@pytest.mark.vcr() def test_responses_create_streaming_aggregates_cache_tokens( - span_exporter, openai_client, instrument_no_content, vcr + request, span_exporter, openai_client, instrument_no_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_streaming_aggregates_cache_tokens[content_mode0].yaml" - ): - stream = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=True, - ) + with openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=True, + ) as stream: response = _collect_completed_response(stream) (span,) = span_exporter.get_finished_spans() assert_cache_attributes(span, response.usage) +@pytest.mark.vcr() def test_responses_create_streaming_captures_content( + request, span_exporter, log_exporter, openai_client, instrument_with_content, - vcr, ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_streaming_captures_content[content_mode0].yaml" - ): - stream = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=True, - ) + with openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=True, + ) as stream: response = _collect_completed_response(stream) (span,) = span_exporter.get_finished_spans() @@ -1023,21 +1084,19 @@ def test_responses_create_streaming_captures_content( _assert_response_content(span, response, log_exporter) +@pytest.mark.vcr() def test_responses_create_streaming_iteration( - span_exporter, openai_client, instrument_no_content, vcr + request, span_exporter, openai_client, instrument_no_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_streaming_iteration[content_mode0].yaml" - ): - stream = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input="Say hi.", - stream=True, - ) - events = list(stream) + stream = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input="Say hi.", + stream=True, + ) + events = list(stream) assert len(events) > 0 @@ -1054,25 +1113,23 @@ def test_responses_create_streaming_iteration( assert GenAIAttributes.GEN_AI_USAGE_OUTPUT_TOKENS in span.attributes +@pytest.mark.vcr() def test_responses_create_streaming_delegates_response_attribute( - openai_client, instrument_no_content, vcr + request, openai_client, instrument_no_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_streaming_delegates_response_attribute[content_mode0].yaml" - ): - stream = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input="Say hi.", - stream=True, - ) + stream = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input="Say hi.", + stream=True, + ) - assert stream.response is not None - assert stream.response.status_code == 200 - assert stream.response.headers.get("x-request-id") is not None - stream.close() + assert stream.response is not None + assert stream.response.status_code == 200 + assert stream.response.headers.get("x-request-id") is not None + stream.close() def test_responses_create_streaming_connection_error( @@ -1083,7 +1140,7 @@ def test_responses_create_streaming_connection_error( client = OpenAI(base_url="http://localhost:4242") with pytest.raises(APIConnectionError): - client.responses.create( + client.responses.create( # pylint: disable=no-member model=DEFAULT_MODEL, input="Hello", stream=True, @@ -1100,23 +1157,21 @@ def test_responses_create_streaming_connection_error( ) +@pytest.mark.vcr() def test_responses_stream_wrapper_finalize_idempotent( - span_exporter, openai_client, instrument_no_content, vcr + request, span_exporter, openai_client, instrument_no_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_stream_wrapper_finalize_idempotent[content_mode0].yaml" - ): - stream = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=True, - ) + stream = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=True, + ) - response = _collect_completed_response(stream) - stream.close() + response = _collect_completed_response(stream) + stream.close() spans = span_exporter.get_finished_spans() assert len(spans) == 1 @@ -1132,51 +1187,49 @@ def test_responses_stream_wrapper_finalize_idempotent( ) +@pytest.mark.vcr() def test_responses_create_stream_propagation_error( - span_exporter, openai_client, instrument_no_content, monkeypatch, vcr + request, span_exporter, openai_client, instrument_no_content, monkeypatch ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_stream_propagation_error[content_mode0].yaml" - ): - stream = openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=True, - ) + stream = openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=True, + ) - class ErrorInjectingStreamDelegate: - def __init__(self, inner): - self._inner = inner - self._count = 0 + class ErrorInjectingStreamDelegate: + def __init__(self, inner): + self._inner = inner + self._count = 0 - def __iter__(self): - return self + def __iter__(self): + return self - def __next__(self): - if self._count == 1: - raise ConnectionError("connection reset during stream") - self._count += 1 - return next(self._inner) + def __next__(self): + if self._count == 1: + raise ConnectionError("connection reset during stream") + self._count += 1 + return next(self._inner) - def close(self): - return self._inner.close() + def close(self): + return self._inner.close() - def __getattr__(self, name): - return getattr(self._inner, name) + def __getattr__(self, name): + return getattr(self._inner, name) - monkeypatch.setattr( - stream, "stream", ErrorInjectingStreamDelegate(stream.stream) - ) + monkeypatch.setattr( + stream, "stream", ErrorInjectingStreamDelegate(stream.stream) + ) - with pytest.raises( - ConnectionError, match="connection reset during stream" - ): - with stream: - for _ in stream: - pass + with pytest.raises( + ConnectionError, match="connection reset during stream" + ): + with stream: + for _ in stream: + pass (span,) = span_exporter.get_finished_spans() assert ( @@ -1185,23 +1238,21 @@ def __getattr__(self, name): assert span.attributes[ErrorAttributes.ERROR_TYPE] == "ConnectionError" +@pytest.mark.vcr() def test_responses_create_streaming_user_exception( - span_exporter, openai_client, instrument_no_content, vcr + request, span_exporter, openai_client, instrument_no_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_streaming_user_exception[content_mode0].yaml" - ): - with pytest.raises(ValueError, match="User raised exception"): - with openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=True, - ) as stream: - for _ in stream: - raise ValueError("User raised exception") + with pytest.raises(ValueError, match="User raised exception"): + with openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=True, + ) as stream: + for _ in stream: + raise ValueError("User raised exception") (span,) = span_exporter.get_finished_spans() assert ( @@ -1210,79 +1261,117 @@ def test_responses_create_streaming_user_exception( assert span.attributes[ErrorAttributes.ERROR_TYPE] == "ValueError" +@pytest.mark.vcr() +@pytest.mark.skipif( + not _has_tools_param, + reason="openai SDK too old to support 'tools' parameter on Responses.create", +) +def test_responses_create_captures_tool_loop_history( + request, span_exporter, openai_client, instrument_with_content +): + _skip_if_not_latest() + + openai_client.responses.create( + model=DEFAULT_MODEL, + input=get_responses_tool_loop_input(), + tools=[get_responses_weather_tool_definition()], + ) + + (span,) = span_exporter.get_finished_spans() + assert_messages_attribute( + span.attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES], + EXPECTED_TOOL_LOOP_INPUT_MESSAGES, + ) + + @pytest.mark.skipif( not _has_custom_tool_types, reason="openai SDK too old to support custom tool call types", ) +@pytest.mark.vcr() @pytest.mark.skipif( not _has_tools_param, reason="openai SDK too old to support 'tools' parameter on Responses.create", ) -def test_responses_create_captures_custom_tool_history( - span_exporter, openai_client, instrument_with_content, vcr +def test_responses_create_captures_custom_tool_call_output( + request, span_exporter, openai_client, instrument_with_content ): + """A custom tool call the model requests is recorded on the output side too.""" _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_captures_custom_tool_history[content_mode0].yaml" - ): - openai_client.responses.create( - model=CUSTOM_TOOL_MODEL, - input=get_responses_custom_tool_loop_input(), - tools=[get_responses_custom_tool_definition()], - tool_choice="auto", - ) + openai_client.responses.create( + model=CUSTOM_TOOL_MODEL, + input="Use the run_sql tool to count the rows in the users table.", + tools=[get_responses_custom_tool_definition()], + tool_choice="auto", + ) (span,) = span_exporter.get_finished_spans() - assert_messages_attribute( - span.attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES], - EXPECTED_CUSTOM_TOOL_INPUT_MESSAGES, + assert span.attributes[GenAIAttributes.GEN_AI_RESPONSE_FINISH_REASONS] == ( + "tool_calls", + ) + output_messages = _load_span_messages( + span, GenAIAttributes.GEN_AI_OUTPUT_MESSAGES ) + tool_calls = [ + part + for message in output_messages + for part in message.get("parts", []) + if part.get("type") == "tool_call" + ] + (tool_call,) = tool_calls + assert tool_call["name"] == "run_sql" + assert tool_call["id"] == CUSTOM_TOOL_CALL_ID + # The same id the replayed history correlates on, so the two spans join up. + assert tool_call["arguments"] == CUSTOM_TOOL_INPUT +@pytest.mark.skipif( + not _has_custom_tool_types, + reason="openai SDK too old to support custom tool call types", +) +@pytest.mark.vcr() @pytest.mark.skipif( not _has_tools_param, reason="openai SDK too old to support 'tools' parameter on Responses.create", ) -def test_responses_create_captures_tool_loop_history( - span_exporter, openai_client, instrument_with_content, vcr +def test_responses_create_captures_custom_tool_history( + request, span_exporter, openai_client, instrument_with_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_captures_tool_loop_history[content_mode0].yaml" - ): - openai_client.responses.create( - model=DEFAULT_MODEL, - input=get_responses_tool_loop_input(), - tools=[get_responses_weather_tool_definition()], - ) + openai_client.responses.create( + model=CUSTOM_TOOL_MODEL, + input=get_responses_custom_tool_loop_input(), + tools=[get_responses_custom_tool_definition()], + tool_choice="auto", + ) (span,) = span_exporter.get_finished_spans() + # The replayed `reasoning` item is not recorded: it carries no readable + # text, and the response path drops such items too. assert_messages_attribute( span.attributes[GenAIAttributes.GEN_AI_INPUT_MESSAGES], - EXPECTED_TOOL_LOOP_INPUT_MESSAGES, + EXPECTED_CUSTOM_TOOL_INPUT_MESSAGES, ) +@pytest.mark.vcr() @pytest.mark.skipif( not _has_tools_param, reason="openai SDK too old to support 'tools' parameter on Responses.create", ) def test_responses_create_captures_tool_call_content( - span_exporter, openai_client, instrument_with_content, vcr + request, span_exporter, openai_client, instrument_with_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_captures_tool_call_content[content_mode0].yaml" - ): - openai_client.responses.create( - model=DEFAULT_MODEL, - input="What's the weather in Seattle right now?", - tools=[get_responses_weather_tool_definition()], - tool_choice={"type": "function", "name": "get_current_weather"}, - ) + openai_client.responses.create( + model=DEFAULT_MODEL, + input="What's the weather in Seattle right now?", + tools=[get_responses_weather_tool_definition()], + tool_choice={"type": "function", "name": "get_current_weather"}, + ) (span,) = span_exporter.get_finished_spans() assert ( @@ -1331,13 +1420,12 @@ def test_responses_create_streaming_captures_tool_definitions( with vcr.use_cassette( "test_responses_create_streaming_captures_content[content_mode0].yaml" ): - stream = openai_client.responses.create( + with openai_client.responses.create( model=DEFAULT_MODEL, input=USER_ONLY_PROMPT[0]["content"], tools=[get_responses_weather_tool_definition()], stream=True, - ) - with stream: + ) as stream: _collect_completed_response(stream) (span,) = span_exporter.get_finished_spans() @@ -1374,6 +1462,7 @@ def test_responses_stream_captures_tool_definitions( ) +@pytest.mark.vcr() @pytest.mark.skipif( not _has_reasoning_param, reason=( @@ -1381,25 +1470,22 @@ def test_responses_stream_captures_tool_definitions( ), ) def test_responses_create_reports_reasoning_tokens( - span_exporter, openai_client, instrument_with_content, vcr + request, span_exporter, openai_client, instrument_with_content ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_reports_reasoning_tokens[content_mode0].yaml" - ): - response = openai_client.responses.create( - model=REASONING_MODEL, - reasoning={"effort": "low"}, - input=[ - { - "role": "user", - "content": REASONING_PROMPT, - } - ], - max_output_tokens=1000, - timeout=30.0, - ) + response = openai_client.responses.create( + model=REASONING_MODEL, + reasoning={"effort": "low"}, + input=[ + { + "role": "user", + "content": REASONING_PROMPT, + } + ], + max_output_tokens=300, + timeout=30.0, + ) reasoning_tokens = getattr( getattr(response.usage, "output_tokens_details", None), @@ -1434,47 +1520,43 @@ def test_responses_create_reports_reasoning_tokens( assert len(output_messages) > 0 +@pytest.mark.vcr() def test_responses_create_with_content_span_unsampled( + request, span_exporter, log_exporter, openai_client, instrument_with_content_unsampled, - vcr, ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_with_content_span_unsampled[content_mode0].yaml" - ): - openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=False, - ) + openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=False, + ) assert len(span_exporter.get_finished_spans()) == 0 assert len(log_exporter.get_finished_logs()) == 0 +@pytest.mark.vcr() def test_responses_create_with_content_shapes( + request, span_exporter, log_exporter, openai_client, instrument_with_content, - vcr, ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_with_content_shapes[content_mode0].yaml" - ): - openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=False, - ) + openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=False, + ) (span,) = span_exporter.get_finished_spans() input_messages = _load_span_messages( @@ -1491,24 +1573,18 @@ def test_responses_create_with_content_shapes( assert len(log_exporter.get_finished_logs()) == 0 +@pytest.mark.vcr() def test_responses_create_event_only_no_content_in_span( - span_exporter, - log_exporter, - openai_client, - instrument_event_only, - vcr, + request, span_exporter, log_exporter, openai_client, instrument_event_only ): _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_event_only_no_content_in_span.yaml" - ): - openai_client.responses.create( - model=DEFAULT_MODEL, - instructions=SYSTEM_INSTRUCTIONS, - input=USER_ONLY_PROMPT[0]["content"], - stream=False, - ) + openai_client.responses.create( + model=DEFAULT_MODEL, + instructions=SYSTEM_INSTRUCTIONS, + input=USER_ONLY_PROMPT[0]["content"], + stream=False, + ) (span,) = span_exporter.get_finished_spans() assert GenAIAttributes.GEN_AI_INPUT_MESSAGES not in span.attributes @@ -1536,12 +1612,18 @@ class _ParseCalendarEvent(BaseModel): def test_responses_parse_basic( span_exporter, openai_client, instrument_no_content, vcr ): - """Responses.parse() emits a GenAI span like create() (#659).""" + """Responses.parse() emits a GenAI span like create(). + + parse() is the structured-output helper. It does not delegate to the + instrumented create(), so it is wrapped separately (#659), but it maps to + the same inference operation as create() -- the request/response fields + are identical -- exactly as chat.completions.parse reuses the completions + create wrapper. The recorded response body is valid structured JSON so + the SDK can materialize the ``text_format`` model. + """ _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_parse_basic[content_mode0].yaml" - ): + with vcr.use_cassette("test_responses_parse_basic[content_mode0].yaml"): response = openai_client.responses.parse( model=DEFAULT_MODEL, instructions=SYSTEM_INSTRUCTIONS, @@ -1572,6 +1654,37 @@ def test_responses_parse_basic( assert "_ParseCalendarEvent" not in str(span.attributes) +@pytest.mark.skipif( + not _HAS_RESPONSES_PARSE, + reason="Responses.parse requires a newer openai SDK", +) +def test_responses_parse_wrapping_lifecycle( + tracer_provider, logger_provider, meter_provider +): + """instrument() wraps Responses.parse / AsyncResponses.parse and + uninstrument() restores them.""" + from openai.resources.responses.responses import ( # pylint: disable=no-name-in-module + AsyncResponses, + Responses, + ) + + before_sync = Responses.parse + before_async = AsyncResponses.parse + + instrumentor = OpenAIInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + assert hasattr(Responses.parse, "__wrapped__") + assert hasattr(AsyncResponses.parse, "__wrapped__") + + instrumentor.uninstrument() + assert Responses.parse is before_sync + assert AsyncResponses.parse is before_async + + @pytest.mark.skipif( not _HAS_RESPONSES_PARSE, reason="Responses.parse requires a newer openai SDK", @@ -1579,16 +1692,16 @@ def test_responses_parse_basic( def test_responses_create_output_type_unchanged_by_parse( span_exporter, openai_client, instrument_no_content, vcr ): - """Regression guard: create() must not start reporting an output type. + """Responses.create() behaviour is unchanged by the parse() wrapper. - Only parse(text_format=...) maps to ``gen_ai.output.type``; a plain text - create call stays without the attribute. + ``create`` carries no ``text_format``, so reusing the ``responses_create`` + wrapper for ``parse`` must not start reporting ``gen_ai.output.type`` for a + plain text call (regression guard for issue #659). Reuses the existing + create cassette -- VCR does not match on the request body. """ _skip_if_not_latest() - with vcr.use_cassette( - "test_responses_create_basic[content_mode0].yaml" - ): + with vcr.use_cassette("test_responses_create_basic[content_mode0].yaml"): response = openai_client.responses.create( model=DEFAULT_MODEL, instructions=SYSTEM_INSTRUCTIONS,