From a390142068432729db0ab0b6ffaffbd8a6a6d65a Mon Sep 17 00:00:00 2001 From: kassvl Date: Sun, 25 Oct 2026 06:08:03 +0100 Subject: [PATCH 1/5] feat(unreal): Add a stack trace from the crash context when there is no minidump Unreal's Crash Report Client does not write a minidump on Linux, so these events had no stack trace. The crash context still contains the crashing call stack. Build an exception from it when the report has no minidump or Apple crash report and the event has no exception yet. --- relay-server/src/utils/unreal.rs | 668 ++++++++++++++++++++++++++++++- tests/integration/test_unreal.py | 76 ++++ 2 files changed, 737 insertions(+), 7 deletions(-) diff --git a/relay-server/src/utils/unreal.rs b/relay-server/src/utils/unreal.rs index 13369e25ffd..1ab3dcf6ba1 100644 --- a/relay-server/src/utils/unreal.rs +++ b/relay-server/src/utils/unreal.rs @@ -2,13 +2,15 @@ use bytes::Bytes; use chrono::{TimeZone, Utc}; use relay_config::ConfigSnapshot; use relay_event_schema::protocol::{ - AsPair, Breadcrumb, ClientSdkInfo, Context, Contexts, DeviceContext, Event, EventId, - GpuContext, LenientString, Level, LogEntry, Message, OsContext, TagEntry, Tags, Timestamp, - User, UserReport, Values, + Addr, AsPair, Breadcrumb, ClientSdkInfo, Context, Contexts, DeviceContext, Event, EventId, + Exception, Frame, GpuContext, JsonLenientString, LenientString, Level, LogEntry, Mechanism, + Message, OsContext, RawStacktrace, Stacktrace, TagEntry, Tags, Timestamp, User, UserReport, + Values, }; use relay_protocol::{Annotated, Array, Empty, Object, Value}; use symbolic_unreal::{ - Unreal4Context, Unreal4Crash, Unreal4Error, Unreal4FileType, Unreal4LogEntry, + Unreal4Context, Unreal4ContextRuntimeProperties, Unreal4Crash, Unreal4Error, Unreal4FileType, + Unreal4LogEntry, }; use crate::constants::{ @@ -20,6 +22,9 @@ use crate::services::processor::ProcessingError; /// Maximum number of unreal logs to parse for breadcrumbs. const MAX_NUM_UNREAL_LOGS: usize = 40; +/// Maximum number of stack frames taken from the call stacks in the Unreal crash context. +const MAX_NUM_UNREAL_FRAMES: usize = 256; + /// Client SDK name used for the event payload to identify the UE4 crash reporter. const CLIENT_SDK_NAME: &str = "unreal.crashreporter"; @@ -344,18 +349,249 @@ fn merge_unreal_context(event: &mut Event, context: Unreal4Context) { } } +/// Parses one line of the legacy `CallStack` into a frame. +/// +/// Lines look like `Module!Function(...) [File:Line]`, where everything after the module is +/// optional. Unsymbolicated frames are written as `Module!UnknownFunction(0xOffset)`, and symbols +/// without debug information carry their offset, like `Module!Function(+0x8a)`. +/// +/// Returns the frame and the module offset of an unsymbolicated frame. +fn parse_legacy_frame(line: &str) -> Option<(Frame, Option)> { + let mut line = line.trim(); + if let Some((address, rest)) = line.split_once(' ') + && address.starts_with("0x") + { + line = rest.trim_start(); + } + + if line.is_empty() { + return None; + } + + let (package, mut function) = line.split_once('!').unwrap_or((line, "")); + + let mut frame = Frame { + package: Annotated::from(package.to_owned()), + ..Frame::default() + }; + + if let Some((name, location)) = function.strip_suffix(']').and_then(|f| f.rsplit_once(" [")) { + if let Some((filename, lineno)) = location.rsplit_once(':') + && let Ok(lineno) = lineno.parse::() + { + function = name; + frame.filename = Annotated::new(filename.into()); + frame.lineno = Annotated::new(lineno); + } else if location.is_empty() || location == "UnknownFile" { + function = name; + } + } + + // Symbols with an offset are the closest exported symbol, found without debug information. + // They are often wrong and would group unrelated crashes together, so they are skipped. + let function = function.trim_end(); + let mut offset = None; + let is_symbol = match function.strip_suffix(')').and_then(|f| f.rsplit_once('(')) { + Some((_, args)) if args.starts_with("+0x") => false, + Some((_, args)) => match args.strip_prefix("0x") { + Some(hex) => { + offset = u64::from_str_radix(hex, 16).ok(); + false + } + None => true, + }, + // The last line is cut off when the call stack exceeds the engine's buffer. + None => function.matches('(').count() == function.matches(')').count(), + }; + + if is_symbol && !function.is_empty() && function != "UnknownFunction" { + frame.function = Annotated::new(function.to_owned()); + } + + Some((frame, offset)) +} + +/// A frame of the portable `PCallStack`. +#[derive(Debug, PartialEq)] +struct PortableFrame<'a> { + module: Option<&'a str>, + offset: u64, + address: u64, +} + +/// Parses the portable `PCallStack`. +/// +/// Each frame is written as `Module 0xImageBase + Offset`, older versions omit the module. +/// Returns `None` if the call stack cannot be fully parsed or contains placeholder addresses. +fn parse_portable_call_stack(call_stack: &str) -> Option>> { + let tokens: Vec<&str> = call_stack.split_whitespace().collect(); + let mut frames = Vec::new(); + let mut tokens = tokens.as_slice(); + + while !tokens.is_empty() && frames.len() < MAX_NUM_UNREAL_FRAMES { + let (module, base, offset, rest) = match tokens { + [base, "+", offset, rest @ ..] => (None, base, offset, rest), + [module, base, "+", offset, rest @ ..] => (Some(*module), base, offset, rest), + _ => return None, + }; + + let base = u64::from_str_radix(base.strip_prefix("0x")?, 16).ok()?; + let offset = u64::from_str_radix(offset, 16).ok()?; + if base == 0xffff_ffff { + return None; + } + + frames.push(PortableFrame { + module, + offset, + address: base.checked_add(offset)?, + }); + tokens = rest; + } + + Some(frames) +} + +/// Returns whether a module of the legacy call stack is the module of the portable call stack. +/// +/// The legacy call stack can differ in case, replace `-` with `_`, or add a version suffix, like +/// `libc.so.6` for `libc.so`. +fn is_same_module(legacy: &str, portable: &str) -> bool { + let legacy = legacy.to_ascii_lowercase().replace('-', "_"); + let portable = portable.to_ascii_lowercase().replace('-', "_"); + + legacy == portable + || legacy + .strip_prefix(&portable) + .is_some_and(|s| s.starts_with('.')) + || portable + .strip_prefix(&legacy) + .is_some_and(|s| s.starts_with('.')) +} + +/// Returns whether the engine writes full image bases into the portable call stack. +/// +/// Unreal Engine 4.25 and older truncate image bases to 32 bits. +fn has_full_image_bases(engine_version: Option<&str>) -> bool { + let mut parts = engine_version.unwrap_or_default().split(['.', '-']); + let major = parts.next().and_then(|part| part.parse::().ok()); + let minor = parts.next().and_then(|part| part.parse::().ok()); + + match (major, minor) { + (Some(major), Some(minor)) => (major, minor) >= (4, 26), + _ => true, + } +} + +/// Creates an exception with the stack trace from the Unreal crash context. +/// +/// Platforms that do not write a minidump, like Linux, still store the crashing call stack in the +/// crash context. Frames are taken from the legacy `CallStack`, and their instruction addresses +/// from the portable `PCallStack` if it has at least as many frames and both agree on every frame. +fn unreal_exception(runtime_props: &Unreal4ContextRuntimeProperties) -> Option { + let legacy: Vec<(Frame, Option)> = runtime_props + .legacy_call_stack + .as_deref() + .unwrap_or_default() + .lines() + .filter_map(parse_legacy_frame) + .take(MAX_NUM_UNREAL_FRAMES) + .collect(); + + let portable = runtime_props + .portable_call_stack + .as_deref() + .filter(|_| has_full_image_bases(runtime_props.engine_version.as_deref())) + .and_then(parse_portable_call_stack) + .unwrap_or_default(); + + // Unknown functions are written with the offset of their return address minus one. + let stacks_match = legacy.len() <= portable.len() + && legacy + .iter() + .zip(&portable) + .all(|((frame, offset), portable)| { + let same_module = match (frame.package.as_str(), portable.module) { + (Some(legacy), Some(portable)) => is_same_module(legacy, portable), + _ => false, + }; + same_module + && offset.is_none_or(|offset| offset.checked_add(1) == Some(portable.offset)) + }); + + let mut frames: Vec = legacy.into_iter().map(|(frame, _)| frame).collect(); + + if stacks_match { + for (frame, portable) in frames.iter_mut().zip(&portable) { + frame.instruction_addr = Annotated::new(Addr(portable.address)); + } + + // The legacy call stack can be shorter, for example it may omit system libraries. + let missing = portable.get(frames.len()..).unwrap_or_default(); + frames.extend(missing.iter().map(|portable| Frame { + package: Annotated::from(portable.module.map(str::to_owned)), + instruction_addr: Annotated::new(Addr(portable.address)), + ..Frame::default() + })); + } + + if frames.is_empty() { + return None; + } + + // Unreal lists the crashing frame first, Sentry expects it last. + frames.reverse(); + + let crash_type = runtime_props + .crash_type + .as_deref() + .filter(|crash_type| !crash_type.is_empty()); + let is_ensure = runtime_props.is_ensure.unwrap_or(false) || crash_type == Some("Ensure"); + let is_stall = runtime_props + .custom + .get("IsStall") + .is_some_and(|v| v == "true") + || crash_type == Some("Stall"); + + let ty = match crash_type { + Some(crash_type) => crash_type, + None if is_ensure => "Ensure", + None if is_stall => "Stall", + None if runtime_props.is_assert.unwrap_or(false) => "Assert", + None => "Crash", + }; + + // These reports are sent while the application keeps running. + let handled = is_ensure || is_stall || crash_type == Some("VerseRuntimeError"); + + Some(Exception { + ty: Annotated::new(ty.to_owned()), + value: Annotated::from(runtime_props.error_message.clone().map(JsonLenientString)), + stacktrace: Annotated::new(Stacktrace(RawStacktrace { + frames: Annotated::new(frames.into_iter().map(Annotated::new).collect()), + ..RawStacktrace::default() + })), + mechanism: Annotated::new(Mechanism { + ty: Annotated::new("unreal".to_owned()), + handled: Annotated::new(handled), + ..Mechanism::default() + }), + ..Exception::default() + }) +} + /// Processes an unreal crash report. /// /// The `user_header` should be extracted from the [`crate::constants::UNREAL_USER_HEADER`] envelope header. -pub fn process_unreal<'a>( +pub fn process_unreal( context: Option, event_id: EventId, event: &mut Annotated, - attachments: impl IntoIterator, + attachments: &[Item], user_header: Option<&str>, ) -> Result, Unreal4Error> { let mut logs_items = attachments - .into_iter() + .iter() .filter(|item| item.attachment_type() == Some(AttachmentType::UnrealLogs)) .map(|item| item.payload()) .peekable(); @@ -381,6 +617,24 @@ pub fn process_unreal<'a>( user_reports.push(report); } + // Minidumps and Apple crash reports are processed later and provide a better stack trace. + let has_native_crash = attachments.iter().any(|item| { + matches!( + item.attachment_type(), + Some(AttachmentType::Minidump | AttachmentType::AppleCrashReport) + ) + }); + + if !has_native_crash + && event.exceptions.is_empty() + && let Some(exception) = context + .runtime_properties + .as_ref() + .and_then(unreal_exception) + { + event.exceptions = Annotated::new(Values::new(vec![Annotated::new(exception)])); + } + merge_unreal_context(event, context); } @@ -404,6 +658,7 @@ pub struct ProcessedUnrealReport { mod tests { use relay_config::Config; + use relay_event_schema::protocol::NativeImagePath; use relay_protocol::SerializableAnnotated; use super::*; @@ -701,4 +956,403 @@ mod tests { } "###); } + + /// Crash context of an ensure on Linux, which does not write a minidump. + /// + /// Taken from the sample in https://github.com/getsentry/relay/issues/5192. + fn get_linux_context() -> Unreal4Context { + let raw_context = br#" + + + true + false + Ensure + Ensure condition failed: false [File:../../Game/Source/Game/GameInstance.cpp] [Line: 112] Stack: 0x000000000b5a7402 Game-Linux-Test!UnknownFunction(0xb3a7401) 0x000000000898ce09 Game-Linux-Test!UnknownFunction(0x878ce08) 0x00007f0b3822a28b libc.so.6!__libc_start_main(+0x8a) 0x0000000004a71029 Game-Linux-Test!UnknownFunction(0x4871028) + 5.4.4-35769+++Proto+Release + Game-Linux-Test!UnknownFunction(0xb3a7401) +Game-Linux-Test!UnknownFunction(0x878ce08) +libc.so.6!__libc_start_main(+0x8a) +Game-Linux-Test!UnknownFunction(0x4871028) + + Game-Linux-Test 0x0000000000200000 + b3a7402 Game-Linux-Test 0x0000000000200000 + 878ce09 libc.so 0x00007f0b38200000 + 2a28b Game-Linux-Test 0x0000000000200000 + 4871029 + + +"#; + + Unreal4Context::parse(raw_context).unwrap() + } + + fn get_exception(runtime_properties: &str) -> Option { + let raw_context = format!( + "{runtime_properties}" + ); + let context = Unreal4Context::parse(raw_context.as_bytes()).unwrap(); + unreal_exception(context.runtime_properties.as_ref()?) + } + + fn process_with_attachments(attachments: &[Item]) -> Annotated { + let mut event = Annotated::empty(); + process_unreal( + Some(get_linux_context()), + EventId::new(), + &mut event, + attachments, + None, + ) + .unwrap(); + event + } + + #[test] + fn test_parse_legacy_frame() { + let cases = [ + ( + "MyProject!AActor::BeginPlay()", + ( + Some("MyProject"), + Some("AActor::BeginPlay()"), + None, + None, + None, + ), + ), + ( + r"MyProject!__scrt_common_main_seh() [f:\dd\vctools\crt\vcstartup\src\startup\exe_common.inl:288]", + ( + Some("MyProject"), + Some("__scrt_common_main_seh()"), + Some(r"f:\dd\vctools\crt\vcstartup\src\startup\exe_common.inl"), + Some(288), + None, + ), + ), + ( + "Game!Foo::operator[]() [Foo.cpp:3]", + ( + Some("Game"), + Some("Foo::operator[]()"), + Some("Foo.cpp"), + Some(3), + None, + ), + ), + ( + "Game!Foo(int) [clone .cold]", + ( + Some("Game"), + Some("Foo(int) [clone .cold]"), + None, + None, + None, + ), + ), + ( + "Game!Foo() []", + (Some("Game"), Some("Foo()"), None, None, None), + ), + ( + "0x02a3b4c5 Game!Foo() [UnknownFile]", + (Some("Game"), Some("Foo()"), None, None, None), + ), + ("kernel32", (Some("kernel32"), None, None, None, None)), + ( + "Game!UnknownFunction(0xb3a7401)", + (Some("Game"), None, None, None, Some(0xb3a7401)), + ), + ( + "Game!UnknownFunction(0x10) [Foo.cpp:3]", + (Some("Game"), None, Some("Foo.cpp"), Some(3), Some(0x10)), + ), + ( + "libc.so.6!__libc_start_main(+0x8a)", + (Some("libc.so.6"), None, None, None, None), + ), + ( + "Game!UnknownFunction(0x48", + (Some("Game"), None, None, None, None), + ), + ]; + + for (line, expected) in cases { + let (frame, offset) = parse_legacy_frame(line).unwrap(); + let actual = ( + frame.package.as_str(), + frame.function.as_str(), + frame.filename.value().map(NativeImagePath::as_str), + frame.lineno.value().copied(), + offset, + ); + assert_eq!(actual, expected, "{line}"); + } + + assert!(parse_legacy_frame(" ").is_none()); + } + + #[test] + fn test_parse_portable_call_stack() { + let addresses = |call_stack| { + parse_portable_call_stack(call_stack).map(|frames| { + frames + .iter() + .map(|f| (f.module, f.address)) + .collect::>() + }) + }; + + assert_eq!( + addresses("Game 0x0000000000200000 + b3a7402 libc.so 0x00007f0b38200000 + 2a28b"), + Some(vec![ + (Some("Game"), 0xb5a7402), + (Some("libc.so"), 0x7f0b3822a28b) + ]) + ); + assert_eq!( + addresses("0x0000000000200000 + 10"), + Some(vec![(None, 0x200010)]) + ); + + // Placeholder addresses written by older engine versions. + assert_eq!(addresses("0x00000000ffffffff + ffffffff"), None); + assert_eq!(addresses("Game 0x200000 +"), None); + assert_eq!(addresses("Game 200000 + 10"), None); + assert_eq!(addresses("Game 0xffffffffffffffff + 1"), None); + } + + #[test] + fn test_is_same_module() { + assert!(is_same_module("Game", "Game")); + assert!(is_same_module("kernel32", "KERNEL32")); + assert!(is_same_module("UnrealEditor_Core", "UnrealEditor-Core")); + assert!(is_same_module("libc.so.6", "libc.so")); + assert!(is_same_module("kernel32", "KERNEL32.DLL")); + + assert!(!is_same_module("UnrealEditor", "UnrealEditor-Core")); + assert!(!is_same_module("Game", "GameServer")); + assert!(!is_same_module("libc.so.6", "libm.so")); + } + + #[test] + fn test_has_full_image_bases() { + assert!(has_full_image_bases(Some("5.4.4-35769+++Proto+Release"))); + assert!(has_full_image_bases(Some( + "4.26.2-15973114+++UE4+Release-4.26" + ))); + assert!(!has_full_image_bases(Some( + "4.24.3-11590370+++UE4+Release-4.24" + ))); + assert!(has_full_image_bases(Some("unknown"))); + assert!(has_full_image_bases(None)); + } + + #[test] + fn test_process_unreal_stacktrace_without_minidump() { + let event = process_with_attachments(&[]); + let exceptions = &event.value().unwrap().exceptions; + + insta::assert_snapshot!(exceptions.to_json_pretty().unwrap(), @r#" + { + "values": [ + { + "type": "Ensure", + "value": "Ensure condition failed: false [File:../../Game/Source/Game/GameInstance.cpp] [Line: 112] Stack: 0x000000000b5a7402 Game-Linux-Test!UnknownFunction(0xb3a7401) 0x000000000898ce09 Game-Linux-Test!UnknownFunction(0x878ce08) 0x00007f0b3822a28b libc.so.6!__libc_start_main(+0x8a) 0x0000000004a71029 Game-Linux-Test!UnknownFunction(0x4871028)", + "stacktrace": { + "frames": [ + { + "package": "Game-Linux-Test", + "instruction_addr": "0x4a71029" + }, + { + "package": "libc.so.6", + "instruction_addr": "0x7f0b3822a28b" + }, + { + "package": "Game-Linux-Test", + "instruction_addr": "0x898ce09" + }, + { + "package": "Game-Linux-Test", + "instruction_addr": "0xb5a7402" + } + ] + }, + "mechanism": { + "type": "unreal", + "handled": true + } + } + ] + } + "#); + } + + #[test] + fn test_process_unreal_stacktrace_with_native_crash() { + for attachment_type in [AttachmentType::Minidump, AttachmentType::AppleCrashReport] { + let mut item = Item::new(ItemType::Attachment); + item.set_attachment_type(attachment_type); + item.set_payload(ContentType::OctetStream, ""); + + let event = process_with_attachments(&[item]); + assert!(event.value().unwrap().exceptions.is_empty()); + } + } + + #[test] + fn test_process_unreal_stacktrace_keeps_exceptions() { + let mut event = Annotated::new(Event { + exceptions: Annotated::new(Values::new(vec![Annotated::new(Exception { + ty: Annotated::new("ClientError".to_owned()), + ..Exception::default() + })])), + ..Event::default() + }); + + process_unreal( + Some(get_linux_context()), + EventId::new(), + &mut event, + &[], + None, + ) + .unwrap(); + + let exceptions = event.value().unwrap().exceptions.value().unwrap(); + let exceptions = exceptions.values.value().unwrap(); + assert_eq!(exceptions.len(), 1); + assert_eq!( + exceptions[0].value().unwrap().ty.as_str(), + Some("ClientError") + ); + } + + #[test] + fn test_unreal_exception_addresses() { + let cases = [ + // Only the portable call stack. + ( + "", + "Game 0x200000 + 10 libc.so 0x7f0000000000 + 20", + "", + vec![ + (Some("libc.so"), Some(0x7f0000000020)), + (Some("Game"), Some(0x200010)), + ], + ), + // The legacy call stack omits the last module. + ( + "Game!Inner()", + "Game 0x200000 + 10 kernel32 0x7f0000000000 + 20", + "", + vec![ + (Some("kernel32"), Some(0x7f0000000020)), + (Some("Game"), Some(0x200010)), + ], + ), + // Different modules. + ( + "Game!Inner()\nGame!Outer()", + "Game 0x200000 + 10 Other 0x300000 + 20", + "", + vec![(Some("Game"), None), (Some("Game"), None)], + ), + // The legacy call stack has more frames. + ( + "Game!Inner()\nGame!Outer()", + "Game 0x200000 + 10", + "", + vec![(Some("Game"), None), (Some("Game"), None)], + ), + // Offsets of unknown functions do not match. + ( + "Game!UnknownFunction(0x100)\nGame!UnknownFunction(0x200)", + "Game 0x1000 + 201 Game 0x1000 + 301", + "", + vec![(Some("Game"), None), (Some("Game"), None)], + ), + // Image bases are truncated by old engine versions. + ( + "Game!Inner()", + "Game 0x200000 + 10", + "4.24.3-11590370+++UE4+Release-4.24", + vec![(Some("Game"), None)], + ), + ]; + + for (legacy, portable, properties, expected) in cases { + let exception = get_exception(&format!( + "{legacy}{portable}{properties}" + )) + .unwrap(); + + let frames = exception + .stacktrace + .value() + .unwrap() + .frames + .value() + .unwrap(); + let actual: Vec<_> = frames + .iter() + .map(|frame| { + let frame = frame.value().unwrap(); + ( + frame.package.as_str(), + frame.instruction_addr.value().map(|a| a.0), + ) + }) + .collect(); + + assert_eq!(actual, expected, "{legacy:?} {portable:?}"); + } + } + + #[test] + fn test_unreal_exception_frame_limit() { + let exception = get_exception(&format!( + "{}", + "Game!Main()\n".repeat(1000) + )) + .unwrap(); + let frames = exception + .stacktrace + .value() + .unwrap() + .frames + .value() + .unwrap(); + assert_eq!(frames.len(), MAX_NUM_UNREAL_FRAMES); + } + + #[test] + fn test_unreal_exception_crash_types() { + let cases = [ + ("Stall", "Stall", true), + ("true", "Stall", true), + ("Ensure", "Ensure", true), + ("true", "Ensure", true), + ( + "VerseRuntimeError", + "VerseRuntimeError", + true, + ), + ("true", "Assert", false), + ("", "Crash", false), + ("GPUCrash", "GPUCrash", false), + ]; + + for (properties, ty, handled) in cases { + let exception = + get_exception(&format!("{properties}Game!Main()")).unwrap(); + + assert_eq!(exception.ty.as_str(), Some(ty), "{properties}"); + let mechanism = exception.mechanism.value().unwrap(); + assert_eq!(mechanism.handled.value(), Some(&handled), "{properties}"); + } + } + + #[test] + fn test_unreal_exception_without_call_stack() { + let runtime_properties = get_context().runtime_properties.unwrap(); + assert!(unreal_exception(&runtime_properties).is_none()); + } } diff --git a/tests/integration/test_unreal.py b/tests/integration/test_unreal.py index add011efb7c..1694e5f6157 100644 --- a/tests/integration/test_unreal.py +++ b/tests/integration/test_unreal.py @@ -1,4 +1,7 @@ import os +import struct +import zlib + import pytest @@ -297,3 +300,76 @@ def test_unreal_proxy_mode(mini_sentry, relay): item = envelope.items[0] assert item.headers.get("type") == "unreal_report" assert item.headers.get("content_type") == "application/octet-stream" + + +def make_unreal_report(files): + """Builds a compressed Unreal crash report in the `CR1` format.""" + + def ansi_string(value): + data = value.encode() + b"\0" + return struct.pack(" + + + true + Ensure + Ensure condition failed: false [File:../../Game/Source/Game/GameInstance.cpp] [Line: 112] + Game-Linux-Test!UnknownFunction(0xb3a7401) +libc.so.6!__libc_start_main(+0x8a) + + Game-Linux-Test 0x0000000000200000 + b3a7402 libc.so 0x00007f0b38200000 + 2a28b + + +""" + + +def test_unreal_stacktrace_without_minidump_with_processing( + mini_sentry, relay_with_processing, attachments_consumer, objectstore +): + project_id = 42 + relay = relay_with_processing() + attachments_consumer = attachments_consumer() + objectstore("attachments", project_id) + + mini_sentry.add_full_project_config(project_id) + unreal_content = make_unreal_report( + [("CrashContext.runtime-xml", LINUX_CRASH_CONTEXT)] + ) + + relay.send_unreal_request(project_id, unreal_content) + + event_data, event = attachments_consumer.get_event() + assert [a["name"] for a in event["attachments"]] == ["CrashContext.runtime-xml"] + + assert event_data["exception"]["values"] == [ + { + "type": "Ensure", + "value": "Ensure condition failed: false [File:../../Game/Source/Game/GameInstance.cpp] [Line: 112]", + "stacktrace": { + "frames": [ + { + "package": "libc.so.6", + "instruction_addr": "0x7f0b3822a28b", + }, + { + "package": "Game-Linux-Test", + "instruction_addr": "0xb5a7402", + }, + ] + }, + "mechanism": {"type": "unreal", "handled": True}, + } + ] From e3989382cc5fdf1a61ebc4c7a74b6b1b994dcd93 Mon Sep 17 00:00:00 2001 From: kassvl Date: Wed, 28 Oct 2026 14:50:22 +0100 Subject: [PATCH 2/5] feat(unreal): Put the crash context stack trace behind a feature flag Gate the exception behind projects:relay-unreal-crash-context-stacktrace, mark it synthetic and the event native, and fall back to a fatal level after the existing assert and ensure mapping. --- relay-dynamic-config/src/feature.rs | 3 + .../processing/errors/integrations/unreal.rs | 17 ++- relay-server/src/utils/unreal.rs | 101 +++++++++++++++++- tests/integration/test_unreal.py | 23 +++- 4 files changed, 134 insertions(+), 10 deletions(-) diff --git a/relay-dynamic-config/src/feature.rs b/relay-dynamic-config/src/feature.rs index ee1923ba6d5..912bf0f1cb3 100644 --- a/relay-dynamic-config/src/feature.rs +++ b/relay-dynamic-config/src/feature.rs @@ -97,6 +97,9 @@ pub enum Feature { /// Enable Nintendo event rewrite. #[serde(rename = "projects:relay-nintendo-event-rewrite")] NintendoEventRewrite, + /// Add a stack trace from the Unreal crash context to crashes without a minidump. + #[serde(rename = "projects:relay-unreal-crash-context-stacktrace")] + UnrealCrashContextStacktrace, /// Stream minidumps to objectstore. #[serde(rename = "projects:relay-minidump-uploads")] MinidumpUploads, diff --git a/relay-server/src/processing/errors/integrations/unreal.rs b/relay-server/src/processing/errors/integrations/unreal.rs index 6cf2a1611d3..cc904d592a3 100644 --- a/relay-server/src/processing/errors/integrations/unreal.rs +++ b/relay-server/src/processing/errors/integrations/unreal.rs @@ -1,4 +1,5 @@ use relay_cogs::{AppFeature, FeatureWeights}; +use relay_dynamic_config::Feature; use relay_protocol::Empty; use relay_quotas::DataCategory; @@ -86,9 +87,19 @@ fn convert( .get_header(crate::constants::UNREAL_USER_HEADER) .and_then(|v| v.as_str()); - let report = - crate::utils::process_unreal(context, event_id, &mut event, &attachments, user_header) - .map_err(ProcessingError::InvalidUnrealReport)?; + let context_stacktrace = ctx + .project_info + .has_feature(Feature::UnrealCrashContextStacktrace); + + let report = crate::utils::process_unreal( + context, + event_id, + &mut event, + &attachments, + user_header, + context_stacktrace, + ) + .map_err(ProcessingError::InvalidUnrealReport)?; let mut envelope = Envelope::from_request(Some(event_id), unreal.headers.meta().clone()); diff --git a/relay-server/src/utils/unreal.rs b/relay-server/src/utils/unreal.rs index 1ab3dcf6ba1..11d48cb4291 100644 --- a/relay-server/src/utils/unreal.rs +++ b/relay-server/src/utils/unreal.rs @@ -574,6 +574,7 @@ fn unreal_exception(runtime_props: &Unreal4ContextRuntimeProperties) -> Option Option, event_id: EventId, event: &mut Annotated, attachments: &[Item], user_header: Option<&str>, + context_stacktrace: bool, ) -> Result, Unreal4Error> { let mut logs_items = attachments .iter() @@ -625,14 +630,19 @@ pub fn process_unreal( ) }); - if !has_native_crash - && event.exceptions.is_empty() - && let Some(exception) = context + let exception = if context_stacktrace && !has_native_crash && event.exceptions.is_empty() { + context .runtime_properties .as_ref() .and_then(unreal_exception) - { + } else { + None + }; + + if let Some(exception) = exception { event.exceptions = Annotated::new(Values::new(vec![Annotated::new(exception)])); + // The frames are native, like those of a minidump. + *event.platform.value_mut() = Some("native".to_owned()); } merge_unreal_context(event, context); @@ -998,11 +1008,23 @@ Game-Linux-Test!UnknownFunction(0x4871028) &mut event, attachments, None, + true, ) .unwrap(); event } + fn process_runtime_properties(runtime_properties: &str) -> Event { + let raw_context = format!( + "{runtime_properties}" + ); + let context = Unreal4Context::parse(raw_context.as_bytes()).unwrap(); + + let mut event = Annotated::empty(); + process_unreal(Some(context), EventId::new(), &mut event, &[], None, true).unwrap(); + event.into_value().unwrap() + } + #[test] fn test_parse_legacy_frame() { let cases = [ @@ -1177,6 +1199,7 @@ Game-Linux-Test!UnknownFunction(0x4871028) }, "mechanism": { "type": "unreal", + "synthetic": true, "handled": true } } @@ -1185,6 +1208,75 @@ Game-Linux-Test!UnknownFunction(0x4871028) "#); } + #[test] + fn test_process_unreal_stacktrace_native_event() { + let event = process_with_attachments(&[]); + let event = event.value().unwrap(); + + assert_eq!(event.platform.as_str(), Some("native")); + // The crash context of the sample is an ensure. + assert_eq!(event.level.value(), Some(&Level::Warning)); + } + + #[test] + fn test_process_unreal_stacktrace_disabled() { + let mut event = Annotated::empty(); + process_unreal( + Some(get_linux_context()), + EventId::new(), + &mut event, + &[], + None, + false, + ) + .unwrap(); + let event = event.value().unwrap(); + + assert!(event.exceptions.is_empty()); + assert!(event.platform.is_empty()); + assert_eq!(event.level.value(), Some(&Level::Warning)); + } + + #[test] + fn test_process_unreal_stacktrace_level() { + let cases = [ + ("true", Level::Error), + ("true", Level::Warning), + ("Crash", Level::Fatal), + ("Stall", Level::Fatal), + ("", Level::Fatal), + ]; + + for (properties, level) in cases { + let event = process_runtime_properties(&format!( + "{properties}Game!Main()" + )); + assert_eq!(event.level.value(), Some(&level), "{properties}"); + } + + // A level sent by the client is kept. + let mut event = Annotated::new(Event { + level: Annotated::new(Level::Info), + ..Event::default() + }); + process_unreal( + Some(get_linux_context()), + EventId::new(), + &mut event, + &[], + None, + true, + ) + .unwrap(); + let event = event.into_value().unwrap(); + assert!(!event.exceptions.is_empty()); + assert_eq!(event.level.value(), Some(&Level::Info)); + + // Without a call stack, there is no exception. + let event = process_runtime_properties("Crash"); + assert!(event.exceptions.is_empty()); + } + #[test] fn test_process_unreal_stacktrace_with_native_crash() { for attachment_type in [AttachmentType::Minidump, AttachmentType::AppleCrashReport] { @@ -1213,6 +1305,7 @@ Game-Linux-Test!UnknownFunction(0x4871028) &mut event, &[], None, + true, ) .unwrap(); diff --git a/tests/integration/test_unreal.py b/tests/integration/test_unreal.py index 1694e5f6157..30e78cbdecf 100644 --- a/tests/integration/test_unreal.py +++ b/tests/integration/test_unreal.py @@ -336,15 +336,24 @@ def ansi_string(value): """ +@pytest.mark.parametrize("feature_enabled", [True, False]) def test_unreal_stacktrace_without_minidump_with_processing( - mini_sentry, relay_with_processing, attachments_consumer, objectstore + mini_sentry, + relay_with_processing, + attachments_consumer, + objectstore, + feature_enabled, ): project_id = 42 relay = relay_with_processing() attachments_consumer = attachments_consumer() objectstore("attachments", project_id) - mini_sentry.add_full_project_config(project_id) + project_config = mini_sentry.add_full_project_config(project_id) + if feature_enabled: + project_config["config"]["features"] = [ + "projects:relay-unreal-crash-context-stacktrace" + ] unreal_content = make_unreal_report( [("CrashContext.runtime-xml", LINUX_CRASH_CONTEXT)] ) @@ -354,6 +363,14 @@ def test_unreal_stacktrace_without_minidump_with_processing( event_data, event = attachments_consumer.get_event() assert [a["name"] for a in event["attachments"]] == ["CrashContext.runtime-xml"] + if not feature_enabled: + assert "exception" not in event_data + assert event_data["platform"] == "other" + assert event_data["level"] == "warning" + return + + assert event_data["platform"] == "native" + assert event_data["level"] == "warning" assert event_data["exception"]["values"] == [ { "type": "Ensure", @@ -370,6 +387,6 @@ def test_unreal_stacktrace_without_minidump_with_processing( }, ] }, - "mechanism": {"type": "unreal", "handled": True}, + "mechanism": {"type": "unreal", "synthetic": True, "handled": True}, } ] From 3d4c1e24127adf26ce4e2a078f0d4f901083a129 Mon Sep 17 00:00:00 2001 From: kassvl Date: Wed, 28 Oct 2026 15:02:22 +0100 Subject: [PATCH 3/5] meta: Add CHANGELOG entry for #6581 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2460178516..fa27787c295 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Derive the `object` span category from `object.*` span operations, such as Cloudflare R2 and Amazon S3 object storage calls. ([#6514](https://github.com/getsentry/relay/pull/6514)) - Process `event.flamegraph` attachments associated with events. ([#6473](https://github.com/getsentry/relay/pull/6473)) - Extract the crashed session embedded in PlayStation crash reports by sentry-playstation and process it as a session update for release health. ([#6544](https://github.com/getsentry/relay/pull/6544)) +- Add a stack trace from the crash context to Unreal crash reports without a minidump, such as those from Linux, behind the `projects:relay-unreal-crash-context-stacktrace` feature. ([#6581](https://github.com/getsentry/relay/pull/6581)) **Bug Fixes**: From d5b4a25e747e911d0d2e04ab149f1f0c752150e5 Mon Sep 17 00:00:00 2001 From: kassvl Date: Thu, 29 Oct 2026 18:33:19 +0100 Subject: [PATCH 4/5] fix(unreal): Parse call stack lines without a module or with spaces in it Address review feedback on the crash context stack trace: - Legacy call stack lines may omit the module, like `UnknownFunction []`. Such frames take the module from the portable call stack, and only count as matching if their offset agrees or they carry no symbol to misplace. - Module names in the portable call stack can contain whitespace, so a frame is split at the `+` between image base and offset. - The engine does not move the crashing frame back to the call site, so the first frame matches with or without that adjustment. --- relay-server/src/utils/unreal.rs | 160 ++++++++++++++++++++++++++----- 1 file changed, 138 insertions(+), 22 deletions(-) diff --git a/relay-server/src/utils/unreal.rs b/relay-server/src/utils/unreal.rs index 20f8e09907c..67e8d6fdebe 100644 --- a/relay-server/src/utils/unreal.rs +++ b/relay-server/src/utils/unreal.rs @@ -360,7 +360,8 @@ fn merge_unreal_context(event: &mut Event, context: Unreal4Context) { /// /// Lines look like `Module!Function(...) [File:Line]`, where everything after the module is /// optional. Unsymbolicated frames are written as `Module!UnknownFunction(0xOffset)`, and symbols -/// without debug information carry their offset, like `Module!Function(+0x8a)`. +/// without debug information carry their offset, like `Module!Function(+0x8a)`. The module can +/// also be omitted, like in `UnknownFunction []`. /// /// Returns the frame and the module offset of an unsymbolicated frame. fn parse_legacy_frame(line: &str) -> Option<(Frame, Option)> { @@ -375,10 +376,17 @@ fn parse_legacy_frame(line: &str) -> Option<(Frame, Option)> { return None; } - let (package, mut function) = line.split_once('!').unwrap_or((line, "")); + let (package, mut function) = match line.split_once('!') { + Some((package, function)) => (Some(package), function), + // Without a module, the function is followed by its arguments or file. + None if line.contains('(') || line.ends_with(']') || line == "UnknownFunction" => { + (None, line) + } + None => (Some(line), ""), + }; let mut frame = Frame { - package: Annotated::from(package.to_owned()), + package: Annotated::from(package.map(str::to_owned)), ..Frame::default() }; @@ -428,23 +436,27 @@ struct PortableFrame<'a> { /// Parses the portable `PCallStack`. /// -/// Each frame is written as `Module 0xImageBase + Offset`, older versions omit the module. +/// Each frame is written as `Module 0xImageBase + Offset`, older versions omit the module. Module +/// names can contain whitespace, so a frame ends with the offset after the `+`. /// Returns `None` if the call stack cannot be fully parsed or contains placeholder addresses. fn parse_portable_call_stack(call_stack: &str) -> Option>> { - // A frame has at most four tokens, so this is enough for `MAX_NUM_UNREAL_FRAMES` frames. - let tokens: Vec<&str> = call_stack - .split_whitespace() - .take(MAX_NUM_UNREAL_FRAMES * 4) - .collect(); let mut frames = Vec::new(); - let mut tokens = tokens.as_slice(); - - while !tokens.is_empty() && frames.len() < MAX_NUM_UNREAL_FRAMES { - let (module, base, offset, rest) = match tokens { - [base, "+", offset, rest @ ..] => (None, base, offset, rest), - [module, base, "+", offset, rest @ ..] => (Some(*module), base, offset, rest), - _ => return None, + let mut rest = call_stack.trim_start(); + + while !rest.is_empty() && frames.len() < MAX_NUM_UNREAL_FRAMES { + // The `+` between image base and offset, not one in a module name like `libstdc++.so`. + let (plus, _) = rest.match_indices('+').find(|&(index, _)| { + rest[..index].ends_with(char::is_whitespace) + && rest[index + 1..].starts_with(char::is_whitespace) + })?; + let head = rest[..plus].trim_end(); + let tail = rest[plus + 1..].trim_start(); + + let (module, base) = match head.rsplit_once(char::is_whitespace) { + Some((module, base)) => (Some(module.trim_end()), base), + None => (None, head), }; + let (offset, tail) = tail.split_once(char::is_whitespace).unwrap_or((tail, "")); let base = u64::from_str_radix(base.strip_prefix("0x")?, 16).ok()?; let offset = u64::from_str_radix(offset, 16).ok()?; @@ -457,7 +469,7 @@ fn parse_portable_call_stack(call_stack: &str) -> Option>> offset, address: base.checked_add(offset)?, }); - tokens = rest; + rest = tail.trim_start(); } Some(frames) @@ -516,18 +528,29 @@ fn unreal_exception(runtime_props: &Unreal4ContextRuntimeProperties) -> Option is_same_module(legacy, portable), - _ => false, + // Without a module, only the offset can confirm the frame. Frames without + // an offset are only kept in place if they have nothing to misattribute. + (None, _) => { + offset.is_some() || (frame.function.is_empty() && frame.filename.is_empty()) + } + (Some(_), None) => false, }; - same_module - && offset.is_none_or(|offset| offset.checked_add(1) == Some(portable.offset)) + let same_offset = offset.is_none_or(|offset| { + offset.checked_add(1) == Some(portable.offset) + || (index == 0 && offset == portable.offset) + }); + same_module && same_offset }); let mut frames: Vec = legacy.into_iter().map(|(frame, _)| frame).collect(); @@ -535,6 +558,9 @@ fn unreal_exception(runtime_props: &Unreal4ContextRuntimeProperties) -> Option Date: Thu, 29 Oct 2026 18:33:39 +0100 Subject: [PATCH 5/5] meta: Move changelog entry to the new Unreleased section --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07acb661f65..e440e7d58c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ **Features**: +- Add a stack trace from the crash context to Unreal crash reports without a minidump, such as those from Linux, behind the `projects:relay-unreal-crash-context-stacktrace` feature. ([#6581](https://github.com/getsentry/relay/pull/6581)) - Implement support for splitting GPU dumps from CPU dumps for Unreal crash reports. ([#6583](https://github.com/getsentry/relay/pull/6583)) - Retain client-provided exceptions in PlayStation crash events. ([#6594](https://github.com/getsentry/relay/pull/6594)) - Update device classification for new iPhone and iPad models. ([#6595](https://github.com/getsentry/relay/pull/6595)) @@ -26,7 +27,6 @@ - Process `event.flamegraph` attachments associated with events. ([#6473](https://github.com/getsentry/relay/pull/6473)) - Extract the crashed session embedded in PlayStation crash reports by sentry-playstation and process it as a session update for release health. ([#6544](https://github.com/getsentry/relay/pull/6544)) - Allow pii scrubbing of the `script_sample` field on Csp. ([#6582](https://github.com/getsentry/relay/pull/6582)) -- Add a stack trace from the crash context to Unreal crash reports without a minidump, such as those from Linux, behind the `projects:relay-unreal-crash-context-stacktrace` feature. ([#6581](https://github.com/getsentry/relay/pull/6581)) **Bug Fixes**: