diff --git a/javascript/sentry-conventions/src/attributes.ts b/javascript/sentry-conventions/src/attributes.ts index 22c05f019..5cf853cfc 100644 --- a/javascript/sentry-conventions/src/attributes.ts +++ b/javascript/sentry-conventions/src/attributes.ts @@ -7768,7 +7768,7 @@ export type GEN_AI_FUNCTION_ID_TYPE = string; // Path: model/attributes/gen_ai/gen_ai__input__messages.json /** - * The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. `gen_ai.input.messages` + * The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them. `gen_ai.input.messages` * * Attribute Value Type: `string` {@link GEN_AI_INPUT_MESSAGES_TYPE} * @@ -7833,7 +7833,7 @@ export type GEN_AI_OPERATION_TYPE_TYPE = string; // Path: model/attributes/gen_ai/gen_ai__output__messages.json /** - * The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. `gen_ai.output.messages` + * The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", answers: {...}}` with the answers keyed by question name, as the provider returned them. `gen_ai.output.messages` * * Attribute Value Type: `string` {@link GEN_AI_OUTPUT_MESSAGES_TYPE} * @@ -26677,7 +26677,7 @@ export const ATTRIBUTE_METADATA: Record = { }, 'gen_ai.input.messages': { brief: - 'The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`.', + 'The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them.', type: 'string', keys: [ 'gen_ai.input.messages', @@ -26696,6 +26696,7 @@ export const ATTRIBUTE_METADATA: Record = { '[{"role": "user", "parts": [{"type": "text", "content": "Weather in Paris?"}]}, {"role": "assistant", "parts": [{"type": "tool_call", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "name": "get_weather", "arguments": {"location": "Paris"}}]}, {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "result": "rainy, 57°F"}]}]', aliases: ['ai.texts', 'ai.prompt.messages', 'gen_ai.prompt', 'ai.prompt'], changelog: [ + { version: 'next', prs: [650], description: 'Describe the evaluation message shape for gen_ai.evaluate' }, { version: '0.21.0', prs: [583], description: 'Added ai.prompt as an alias' }, { version: '0.5.0', prs: [264] }, { version: '0.4.0', prs: [221] }, @@ -26735,7 +26736,7 @@ export const ATTRIBUTE_METADATA: Record = { }, 'gen_ai.output.messages': { brief: - "The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls.", + 'The model\'s response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", answers: {...}}` with the answers keyed by question name, as the provider returned them.', type: 'string', keys: ['gen_ai.output.messages', 'ai.response.text', 'ai.response.toolCalls', 'ai.responses', 'ai.tool_calls'], applyScrubbing: { @@ -26746,7 +26747,10 @@ export const ATTRIBUTE_METADATA: Record = { example: '[{"role": "assistant", "parts": [{"type": "text", "content": "The weather in Paris is currently rainy with a temperature of 57°F."}], "finish_reason": "stop"}]', aliases: ['ai.response.toolCalls', 'ai.response.text'], - changelog: [{ version: '0.4.0', prs: [221] }], + changelog: [ + { version: 'next', prs: [650], description: 'Describe the evaluation message shape for gen_ai.evaluate' }, + { version: '0.4.0', prs: [221] }, + ], }, 'gen_ai.pipeline.name': { brief: 'Name of the AI pipeline or chain being executed.', diff --git a/javascript/sentry-conventions/src/op.ts b/javascript/sentry-conventions/src/op.ts index e2976fff4..5e3caa677 100644 --- a/javascript/sentry-conventions/src/op.ts +++ b/javascript/sentry-conventions/src/op.ts @@ -344,6 +344,11 @@ export const GEN_AI_GENERATE_CONTENT = 'gen_ai.generate_content'; */ export const GEN_AI_RERANK = 'gen_ai.rerank'; +/** + * Evaluation of typed questions (e.g., boolean, choice, or score) about a shared state by a generative AI model + */ +export const GEN_AI_EVALUATE = 'gen_ai.evaluate'; + /** * Creation of an AI agent that can later be invoked to perform a task */ diff --git a/javascript/sentry-conventions/src/search.ts b/javascript/sentry-conventions/src/search.ts index def669ecf..9dae53d2d 100644 --- a/javascript/sentry-conventions/src/search.ts +++ b/javascript/sentry-conventions/src/search.ts @@ -8161,7 +8161,7 @@ export const ATTRIBUTE_SEARCH_METADATA: Record canonicalName: 'gen_ai.input.messages', type: 'string', brief: - 'The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`.', + 'The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them.', deprecationChain: [ 'gen_ai.input.messages', 'ai.input_messages', @@ -8190,7 +8190,7 @@ export const ATTRIBUTE_SEARCH_METADATA: Record canonicalName: 'gen_ai.output.messages', type: 'string', brief: - "The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls.", + 'The model\'s response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", answers: {...}}` with the answers keyed by question name, as the provider returned them.', deprecationChain: [ 'gen_ai.output.messages', 'ai.response.text', diff --git a/model/attributes/gen_ai/gen_ai__input__messages.json b/model/attributes/gen_ai/gen_ai__input__messages.json index 0e5215641..00b3c4656 100644 --- a/model/attributes/gen_ai/gen_ai__input__messages.json +++ b/model/attributes/gen_ai/gen_ai__input__messages.json @@ -1,6 +1,6 @@ { "key": "gen_ai.input.messages", - "brief": "The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `\"user\"`, `\"assistant\"`, `\"tool\"`, or `\"system\"`. For messages of the role `\"tool\"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: \"text\", text:\"...\"}`.", + "brief": "The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `\"user\"`, `\"assistant\"`, `\"tool\"`, or `\"system\"`. For messages of the role `\"tool\"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: \"text\", text:\"...\"}`. For `gen_ai.evaluate` operations, the array holds one object `{type: \"evaluation\", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them.", "type": "string", "apply_scrubbing": { "key": "manual" @@ -10,6 +10,11 @@ "alias": ["ai.texts", "ai.prompt.messages", "gen_ai.prompt", "ai.prompt"], "visibility": "public", "changelog": [ + { + "version": "next", + "prs": [650], + "description": "Describe the evaluation message shape for gen_ai.evaluate" + }, { "version": "0.21.0", "prs": [583], diff --git a/model/attributes/gen_ai/gen_ai__output__messages.json b/model/attributes/gen_ai/gen_ai__output__messages.json index 5b80a9245..754068b10 100644 --- a/model/attributes/gen_ai/gen_ai__output__messages.json +++ b/model/attributes/gen_ai/gen_ai__output__messages.json @@ -1,6 +1,6 @@ { "key": "gen_ai.output.messages", - "brief": "The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls.", + "brief": "The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: \"evaluation\", answers: {...}}` with the answers keyed by question name, as the provider returned them.", "type": "string", "apply_scrubbing": { "key": "manual" @@ -10,6 +10,11 @@ "alias": ["ai.response.toolCalls", "ai.response.text"], "visibility": "public", "changelog": [ + { + "version": "next", + "prs": [650], + "description": "Describe the evaluation message shape for gen_ai.evaluate" + }, { "version": "0.4.0", "prs": [221] diff --git a/model/description/gen_ai.json b/model/description/gen_ai.json index 21e0da778..1ee5d21a1 100644 --- a/model/description/gen_ai.json +++ b/model/description/gen_ai.json @@ -24,6 +24,7 @@ "gen_ai", "gen_ai.chat", "gen_ai.embeddings", + "gen_ai.evaluate", "gen_ai.generate_content", "gen_ai.rerank", "gen_ai.responses", @@ -36,6 +37,7 @@ "text_completion gpt-3.5-turbo-instruct", "generate_content gemini-2.0-flash-001", "embeddings text-embedding-3-small", + "evaluate jev-latest", "chat" ] }, diff --git a/model/name/gen_ai.json b/model/name/gen_ai.json index e46c807a2..f84fab001 100644 --- a/model/name/gen_ai.json +++ b/model/name/gen_ai.json @@ -31,6 +31,7 @@ "gen_ai", "gen_ai.chat", "gen_ai.embeddings", + "gen_ai.evaluate", "gen_ai.execute_tool", "gen_ai.generate_content", "gen_ai.rerank", @@ -48,6 +49,7 @@ "embeddings huggingface", "chat claude-opus-4", "execute_tool get_weather", + "evaluate jev-latest", "chat" ] } diff --git a/model/op/gen_ai.json b/model/op/gen_ai.json index e2d5bce37..c82fe5763 100644 --- a/model/op/gen_ai.json +++ b/model/op/gen_ai.json @@ -30,6 +30,10 @@ "name": "gen_ai.rerank", "description": "Reranking of documents or results by a generative AI model" }, + { + "name": "gen_ai.evaluate", + "description": "Evaluation of typed questions (e.g., boolean, choice, or score) about a shared state by a generative AI model" + }, { "name": "gen_ai.create_agent", "description": "Creation of an AI agent that can later be invoked to perform a task" diff --git a/python/src/sentry_conventions/attributes.py b/python/src/sentry_conventions/attributes.py index 7af75fa4d..3440fd6b8 100644 --- a/python/src/sentry_conventions/attributes.py +++ b/python/src/sentry_conventions/attributes.py @@ -4792,7 +4792,7 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta): # Path: model/attributes/gen_ai/gen_ai__input__messages.json GEN_AI_INPUT_MESSAGES: Literal["gen_ai.input.messages"] = "gen_ai.input.messages" - """The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. + """The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them. Type: str Apply Scrubbing: manual @@ -4826,7 +4826,7 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta): # Path: model/attributes/gen_ai/gen_ai__output__messages.json GEN_AI_OUTPUT_MESSAGES: Literal["gen_ai.output.messages"] = "gen_ai.output.messages" - """The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. + """The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", answers: {...}}` with the answers keyed by question name, as the provider returned them. Type: str Apply Scrubbing: manual @@ -17849,7 +17849,7 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta): ], ), "gen_ai.input.messages": AttributeMetadata( - brief='The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`.', + brief='The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them.', type=AttributeType.STRING, keys=( "gen_ai.input.messages", @@ -17865,6 +17865,11 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta): example='[{"role": "user", "parts": [{"type": "text", "content": "Weather in Paris?"}]}, {"role": "assistant", "parts": [{"type": "tool_call", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "name": "get_weather", "arguments": {"location": "Paris"}}]}, {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "result": "rainy, 57°F"}]}]', aliases=["ai.texts", "ai.prompt.messages", "gen_ai.prompt", "ai.prompt"], changelog=[ + ChangelogEntry( + version="next", + prs=[650], + description="Describe the evaluation message shape for gen_ai.evaluate", + ), ChangelogEntry( version="0.21.0", prs=[583], description="Added ai.prompt as an alias" ), @@ -17899,7 +17904,7 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta): ], ), "gen_ai.output.messages": AttributeMetadata( - brief="The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls.", + brief='The model\'s response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", answers: {...}}` with the answers keyed by question name, as the provider returned them.', type=AttributeType.STRING, keys=( "gen_ai.output.messages", @@ -17914,6 +17919,11 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta): example='[{"role": "assistant", "parts": [{"type": "text", "content": "The weather in Paris is currently rainy with a temperature of 57°F."}], "finish_reason": "stop"}]', aliases=["ai.response.toolCalls", "ai.response.text"], changelog=[ + ChangelogEntry( + version="next", + prs=[650], + description="Describe the evaluation message shape for gen_ai.evaluate", + ), ChangelogEntry(version="0.4.0", prs=[221]), ], ), diff --git a/rust/src/op.rs b/rust/src/op.rs index 9a46c965d..c8cb8e73b 100644 --- a/rust/src/op.rs +++ b/rust/src/op.rs @@ -224,6 +224,9 @@ pub const GEN_AI_GENERATE_CONTENT: &str = "gen_ai.generate_content"; /// Reranking of documents or results by a generative AI model pub const GEN_AI_RERANK: &str = "gen_ai.rerank"; +/// Evaluation of typed questions (e.g., boolean, choice, or score) about a shared state by a generative AI model +pub const GEN_AI_EVALUATE: &str = "gen_ai.evaluate"; + /// Creation of an AI agent that can later be invoked to perform a task pub const GEN_AI_CREATE_AGENT: &str = "gen_ai.create_agent";