From dd23f06a8b637ddd282c8ae87711c99faef40543 Mon Sep 17 00:00:00 2001 From: komaruworld Date: Sat, 12 Sep 2026 21:45:48 +0300 Subject: [PATCH 01/58] init vr --- .gitmodules | 3 + CMakeLists.txt | 3 + README.md | 193 +------ cmake/MocktailVR.cmake | 60 ++ config/mocktail.example.yaml | 7 + include/mocktail/vr/openxr_preview.h | 34 ++ include/mocktail/vr/openxr_probe.h | 48 ++ include/runtime/command_line.h | 2 + include/runtime/runtime_config.h | 4 + scripts/vr_simulated.py | 187 ++++++ src/legacy/legacy_runtime.cc | 15 +- src/main.cc | 52 +- src/runtime/command_line.cc | 10 + src/runtime/runtime_config.cc | 2 + src/runtime/runtime_config_bootstrap.cc | 7 + src/runtime/runtime_config_file.cc | 20 +- src/vr/diagnostics.cc | 75 +++ src/vr/openxr_disabled.cc | 23 + src/vr/openxr_preview.cc | 729 ++++++++++++++++++++++++ src/vr/openxr_probe.cc | 209 +++++++ src/vr/probe_main.cc | 50 ++ tests/CMakeLists.txt | 28 + tests/command_line_test.cc | 47 ++ tests/runtime_config_file_test.cc | 53 ++ tests/runtime_config_test.cc | 12 + tests/vr/fake_openxr_preview.cc | 508 +++++++++++++++++ tests/vr/fake_openxr_runtime.cc | 170 ++++++ tests/vr/preview_contract_test.py | 141 +++++ tests/vr/probe_contract_test.py | 82 +++ tests/vr/simulator_launcher_test.py | 89 +++ third_party/OpenXR-SDK | 1 + 31 files changed, 2650 insertions(+), 214 deletions(-) create mode 100644 cmake/MocktailVR.cmake create mode 100644 include/mocktail/vr/openxr_preview.h create mode 100644 include/mocktail/vr/openxr_probe.h create mode 100644 scripts/vr_simulated.py create mode 100644 src/vr/diagnostics.cc create mode 100644 src/vr/openxr_disabled.cc create mode 100644 src/vr/openxr_preview.cc create mode 100644 src/vr/openxr_probe.cc create mode 100644 src/vr/probe_main.cc create mode 100644 tests/vr/fake_openxr_preview.cc create mode 100644 tests/vr/fake_openxr_runtime.cc create mode 100644 tests/vr/preview_contract_test.py create mode 100644 tests/vr/probe_contract_test.py create mode 100644 tests/vr/simulator_launcher_test.py create mode 160000 third_party/OpenXR-SDK diff --git a/.gitmodules b/.gitmodules index 9ac662a1..db7a1c34 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "third_party/Vulkan-Headers"] path = third_party/Vulkan-Headers url = https://github.com/KhronosGroup/Vulkan-Headers.git +[submodule "third_party/OpenXR-SDK"] + path = third_party/OpenXR-SDK + url = https://github.com/KhronosGroup/OpenXR-SDK.git diff --git a/CMakeLists.txt b/CMakeLists.txt index ae285405..a060d5bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -125,6 +125,8 @@ set(VULKAN_HEADERS_ENABLE_INSTALL OFF CACHE BOOL "Install vendored Vulkan-Headers" FORCE) add_subdirectory(third_party/Vulkan-Headers EXCLUDE_FROM_ALL) +include(${CMAKE_SOURCE_DIR}/cmake/MocktailVR.cmake) + include(CTest) if(BUILD_TESTING) # Package builds disable tests and do not fetch Google Test. @@ -627,6 +629,7 @@ target_compile_definitions(mocktail PRIVATE "MOCKTAIL_DEFAULT_COMPATIBILITY_MANIFEST=\"${MOCKTAIL_DEFAULT_COMPATIBILITY_MANIFEST}\"" ) target_link_libraries(mocktail PRIVATE + Mocktail::VR mocktail_legacy_runtime mocktail_auth_runtime_composition mocktail_failure_dialog_runtime diff --git a/README.md b/README.md index 04120ea0..99341c33 100644 --- a/README.md +++ b/README.md @@ -1,192 +1,5 @@ -# Mocktail +This is a dev branch for vr support in Mocktail. -[![CI](https://github.com/komaruworld/mocktail/actions/workflows/ci.yml/badge.svg)](https://github.com/komaruworld/mocktail/actions/workflows/ci.yml) -[![Stars](https://img.shields.io/github/stars/komaruworld/mocktail?style=flat&logo=github)](https://github.com/komaruworld/mocktail/stargazers) -[![Downloads](https://img.shields.io/github/downloads/komaruworld/mocktail/total?logo=github)](https://github.com/komaruworld/mocktail/releases/latest) -[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) -[![Discord](https://img.shields.io/discord/1543955607869063269?label=discord)](https://discord.gg/rhgpfcmFSD) +idk if I can do this. -Mocktail runs the Android `x86_64` Roblox client on Linux, including a Linux -userspace hosted by FreeBSD's Linuxulator. It provides the Android ABI and JNI -pieces the client expects, then connects them to SDL3 and Vulkan or OpenGL on -the Linux side. - -Mocktail is an independent community project. It is not affiliated with Roblox -Corporation or VinegarHQ and does not distribute the Roblox client. - -## How it works - -``` -Roblox APK -> signature and ABI checks -> Bionic + JNI -> SDL3 + Vulkan/OpenGL -``` - -The APK is checked before any native code is loaded. It is downloaded on first -launch and is not bundled with Mocktail. The last working copy is kept in case -an update fails. - -## Install with Flatpak - -Install the stable release from Flathub: - -```bash -flatpak install flathub space.bigrat.mocktail -flatpak run space.bigrat.mocktail -``` - -Nightly builds are produced automatically from the latest `main` branch: - -```bash -flatpak install --user https://mocktail.bigrat.space/mocktail.flatpakref -flatpak run space.bigrat.mocktail -``` - -## Install from the AUR - -Arch Linux users can install either the pinned source release (`mocktail`) or -the current development version (`mocktail-git`) with an AUR helper: - -```bash -paru -S mocktail-git -# or -yay -S mocktail-git -``` - -Use `mocktail` instead of `mocktail-git` to build the pinned release from -source, or install the stable prebuilt package with `paru -S mocktail-bin` or -`yay -S mocktail-bin`. - -## Install with DNF - -Fedora 44 users can install either the stable `mocktail` package or the -`mocktail-nightly` package built from `main`: - -```bash -sudo curl -fsSL https://mocktail.bigrat.space/rpm/mocktail.repo \ - -o /etc/yum.repos.d/mocktail.repo -sudo dnf install mocktail -# or -sudo dnf install mocktail-nightly -``` - -## Install with APT - -Ubuntu 26.04 users can install either the stable `mocktail` package or the -`mocktail-nightly` package built from `main`: - -```bash -sudo install -d -m 0755 /etc/apt/keyrings -sudo curl -fsSL https://mocktail.bigrat.space/mocktail-packages.gpg \ - -o /etc/apt/keyrings/mocktail.gpg -echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/mocktail.gpg] https://mocktail.bigrat.space/apt mocktail main" | \ - sudo tee /etc/apt/sources.list.d/mocktail.list >/dev/null -sudo apt update -sudo apt install mocktail -# or -sudo apt install mocktail-nightly -``` - -## Direct downloads - -Direct AppImage, DEB, and RPM downloads are available from the -[Website](https://mocktail.bigrat.space/) and -[GitHub Releases](https://github.com/komaruworld/mocktail/releases). - -
-Screenshots - -![Roblox home in Mocktail](assets/screenshots/flatpak-home.png) - -![Roblox gameplay in Mocktail](assets/screenshots/flatpak-gameplay-tower.png) - -![Roblox experience in Mocktail](assets/screenshots/flatpak-gameplay-lobby.png) - -
- -## FFlag overrides - -Put a JSON object in `$XDG_CONFIG_HOME/mocktail/fflags.json` (usually -`~/.config/mocktail/fflags.json`). Mocktail applies it on the next launch. - -```json -{ - "FFlagExample": "True", - "DFIntExample": "120" -} -``` -## FAQ - -To see the most Frequently Asked Question go check [FAQ.md](FAQ.md) - - -## Building - -Linux `x86_64` is supported. Experimental FreeBSD 15.1 Linuxulator support has -been tested with an `x86_64` Fedora 44 userspace. On FreeBSD, Mocktail runs -inside Linuxulator; it is not a native FreeBSD binary. Building requires CMake -3.20+, Git, pkg-config, LLD, binutils, a C++17 compiler, SDL 3.4+, SDL3_ttf, -Vulkan, EGL, libplacebo, fontconfig, libcurl, OpenSSL, libelf, libyaml, minizip, -Capstone 5, utf8proc, nlohmann/json, GTK4, libadwaita 1.6+, and WebKitGTK 6.0. - -See the [FreeBSD Guide](packaging/freebsd/README.md) for setup and launch instructions. - -
-Ubuntu 26.04+ - -```bash -sudo apt update -sudo apt install build-essential cmake git ninja-build pkg-config lld \ - libsdl3-dev libsdl3-ttf-dev libcurl4-openssl-dev libssl-dev \ - nlohmann-json3-dev libyaml-dev libelf-dev libminizip-dev \ - libcapstone-dev libgtk-4-dev libadwaita-1-dev libwebkitgtk-6.0-dev \ - libutf8proc-dev libfontconfig1-dev libegl-dev libvulkan-dev \ - libplacebo-dev zlib1g-dev -``` -
- -
-Arch Linux - -```bash -sudo pacman -S --needed base-devel cmake git ninja pkgconf lld sdl3 sdl3_ttf \ - curl openssl nlohmann-json libyaml libelf minizip capstone gtk4 \ - libadwaita webkitgtk-6.0 libutf8proc fontconfig libglvnd \ - libplacebo vulkan-headers vulkan-icd-loader zlib -``` -
- -
-Fedora 44+ - -```bash -sudo dnf install gcc-c++ cmake git ninja-build pkgconf-pkg-config lld \ - SDL3-devel SDL3_ttf-devel libcurl-devel openssl-devel \ - nlohmann-json-devel libyaml-devel elfutils-libelf-devel minizip-ng-compat-devel \ - capstone-devel gtk4-devel libadwaita-devel webkitgtk6.0-devel \ - utf8proc-devel fontconfig-devel libglvnd-devel vulkan-headers \ - vulkan-loader-devel libplacebo-devel zlib-ng-compat-devel -``` -
- -```bash -git clone --recurse-submodules https://github.com/komaruworld/mocktail.git -cd mocktail -make build -./build/mocktail -``` - -## Known limitations - -Mocktail does not currently work with `hardened_malloc`. Using -`hardened_malloc` may cause Mocktail to fail to start or crash during runtime. -Run Mocktail without `hardened_malloc` enabled. - -## License - -[Apache License 2.0](LICENSE). Third-party components keep their own licenses. - -## Support - -You can support the project by giving it a star or with cryptocurrency: - -- USDT (TON): `UQCi6Yzcc9cOctoij6n_r1K90-OdVxAT0D_xo2UzGKkQaJDY` -- USDT (TRC20): `TNPMG9Vig2xiuo2r1QqnXRChPH7Vu28Jmx` +But it will be fun :) diff --git a/cmake/MocktailVR.cmake b/cmake/MocktailVR.cmake new file mode 100644 index 00000000..5c7f4b62 --- /dev/null +++ b/cmake/MocktailVR.cmake @@ -0,0 +1,60 @@ +# Copyright 2026 Mocktail Project Authors +# Licensed under the Apache License, Version 2.0. + +include_guard(GLOBAL) + +option(MOCKTAIL_ENABLE_VR "Build experimental native OpenXR support" OFF) + +add_library(mocktail_vr STATIC src/vr/diagnostics.cc) +add_library(Mocktail::VR ALIAS mocktail_vr) +target_include_directories(mocktail_vr PUBLIC "${CMAKE_SOURCE_DIR}/include") +mocktail_apply_compile_options(mocktail_vr) + +if(MOCKTAIL_ENABLE_VR) + if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(FATAL_ERROR "Mocktail OpenXR support currently targets Linux") + endif() + if(NOT EXISTS "${CMAKE_SOURCE_DIR}/third_party/OpenXR-SDK/CMakeLists.txt") + message(FATAL_ERROR + "Initialize OpenXR: git submodule update --init third_party/OpenXR-SDK") + endif() + + # Scope upstream's generic option names to this dependency. Static linking + # packages the pinned loader without requiring a matching system loader. + function(mocktail_add_openxr_sdk) + set(BUILD_LOADER ON) + set(DYNAMIC_LOADER OFF) + set(BUILD_WITH_SYSTEM_JSONCPP OFF) + set(BUILD_TESTS OFF) + set(BUILD_API_LAYERS OFF) + set(BUILD_CONFORMANCE_TESTS OFF) + set(BUILD_SDK_TESTS OFF) + set(BUILD_FORCE_GENERATION OFF) + add_subdirectory("${CMAKE_SOURCE_DIR}/third_party/OpenXR-SDK" + "${CMAKE_BINARY_DIR}/third_party/OpenXR-SDK" EXCLUDE_FROM_ALL) + endfunction() + mocktail_add_openxr_sdk() + target_sources(mocktail_vr PRIVATE src/vr/openxr_probe.cc src/vr/openxr_preview.cc) + # Error unwinding is confined to this native session implementation; its + # public entry point catches errors before returning to the guest runtime. + set_source_files_properties(src/vr/openxr_preview.cc PROPERTIES + COMPILE_OPTIONS -fexceptions) + target_compile_definitions(mocktail_vr PRIVATE XR_USE_GRAPHICS_API_VULKAN) + target_link_libraries(mocktail_vr PRIVATE OpenXR::openxr_loader Vulkan::Headers ${CMAKE_DL_LIBS}) +else() + target_sources(mocktail_vr PRIVATE src/vr/openxr_disabled.cc) +endif() + +add_executable(mocktail_vr_probe src/vr/probe_main.cc) +set_target_properties(mocktail_vr_probe PROPERTIES OUTPUT_NAME mocktail-vr-probe) +target_link_libraries(mocktail_vr_probe PRIVATE Mocktail::VR) +mocktail_apply_compile_options(mocktail_vr_probe) +install(TARGETS mocktail_vr_probe RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") + +# Development launcher for an independently built Monado runtime. --prepare +# keeps its shared library and a relative manifest under this build directory. +configure_file("${CMAKE_SOURCE_DIR}/scripts/vr_simulated.py" + "${CMAKE_BINARY_DIR}/mocktail-vr-simulated" COPYONLY) +file(CHMOD "${CMAKE_BINARY_DIR}/mocktail-vr-simulated" + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE + WORLD_READ WORLD_EXECUTE) diff --git a/config/mocktail.example.yaml b/config/mocktail.example.yaml index 986bb65f..aa10b5a8 100644 --- a/config/mocktail.example.yaml +++ b/config/mocktail.example.yaml @@ -49,6 +49,13 @@ graphics: # Optional presentation synchronization override: auto, on, or off. # vsync: off +vr: + # Boolean (default: false): launch the Roblox experimental VR path. + # Only Build ID 2998 is currently accepted; eye resources and OpenXR output + # are NOT yet connected. Standalone scene: mocktail-vr-probe --scene. + # Use --no-vr to launch Roblox normally. + enabled: false + performance: # Boolean (default: false): size Roblox scheduler/render queues from every # physical CPU core. A place's Lua/main thread can still remain serial. diff --git a/include/mocktail/vr/openxr_preview.h b/include/mocktail/vr/openxr_preview.h new file mode 100644 index 00000000..c7e38107 --- /dev/null +++ b/include/mocktail/vr/openxr_preview.h @@ -0,0 +1,34 @@ +#ifndef MOCKTAIL_VR_OPENXR_PREVIEW_H_ +#define MOCKTAIL_VR_OPENXR_PREVIEW_H_ + +#include +#include +#include + +namespace mocktail::vr { + +struct PreviewOptions { + // Zero keeps the scene open until the runtime or user requests exit. + std::uint32_t frame_limit = 0; +}; + +struct PreviewResult { + bool success = false; + std::string message; + std::uint64_t frames = 0; + std::uint64_t rendered_frames = 0; + std::uint64_t tracked_frames = 0; + std::array head_position{}; + std::array head_orientation{0, 0, 0, 1}; +}; + +// Runs a native Vulkan/OpenXR scene with a panel fixed in LOCAL space. The +// compositor projects it into both eyes. Roblox images/input are not connected. +// Call on the main thread: SIGINT/SIGTERM request a graceful session exit, and +// their previous handlers are restored before returning. +PreviewResult RunOpenXrPreview(const PreviewOptions& options = {}); +std::string FormatPreviewResult(const PreviewResult& result); + +} // namespace mocktail::vr + +#endif // MOCKTAIL_VR_OPENXR_PREVIEW_H_ diff --git a/include/mocktail/vr/openxr_probe.h b/include/mocktail/vr/openxr_probe.h new file mode 100644 index 00000000..c406ad2b --- /dev/null +++ b/include/mocktail/vr/openxr_probe.h @@ -0,0 +1,48 @@ +#ifndef MOCKTAIL_VR_OPENXR_PROBE_H_ +#define MOCKTAIL_VR_OPENXR_PROBE_H_ + +#include +#include +#include + +namespace mocktail::vr { + +enum class ProbeStatus { + kReady, + kBuildDisabled, + kRuntimeUnavailable, + kHeadsetUnavailable, + kIncompatibleRuntime, + kError, +}; + +struct ViewConfiguration { + std::uint32_t width = 0; + std::uint32_t height = 0; + std::uint32_t samples = 0; +}; + +struct ProbeReport { + ProbeStatus status = ProbeStatus::kError; + std::string message; + std::string sdk_version; + std::string runtime_name; + std::string runtime_version; + std::string system_name; + std::vector extensions; + std::vector views; + bool orientation_tracking = false; + bool position_tracking = false; + std::string minimum_vulkan_version; + std::string maximum_vulkan_version; +}; + +// Native host discovery only. Does not load Roblox, change the active runtime, +// create a graphics device/session, or claim that VR rendering is implemented. +ProbeReport ProbeOpenXr(); +std::string FormatProbeReport(const ProbeReport& report); +int ProbeExitCode(ProbeStatus status); + +} // namespace mocktail::vr + +#endif // MOCKTAIL_VR_OPENXR_PROBE_H_ diff --git a/include/runtime/command_line.h b/include/runtime/command_line.h index 8a491a96..f74291f5 100644 --- a/include/runtime/command_line.h +++ b/include/runtime/command_line.h @@ -1,6 +1,7 @@ #ifndef MOCKTAIL_RUNTIME_COMMAND_LINE_H_ #define MOCKTAIL_RUNTIME_COMMAND_LINE_H_ +#include #include #include @@ -21,6 +22,7 @@ enum class WindowMode { struct CommandLineOptions { CommandMode mode = CommandMode::kRun; WindowMode window_mode = WindowMode::kUnspecified; + std::optional vr_enabled; std::string program_name = "mocktail"; std::string roblox_library_path; std::string graphics_backend; diff --git a/include/runtime/runtime_config.h b/include/runtime/runtime_config.h index 6c4ddbdc..faa6f5c9 100644 --- a/include/runtime/runtime_config.h +++ b/include/runtime/runtime_config.h @@ -77,6 +77,8 @@ class RuntimeConfig { static RuntimeConfig FromEnvironment(const Environment& environment); bool headless() const { return headless_; } + bool vr_enabled() const { return vr_enabled_; } + bool vr_valid() const { return vr_valid_; } const std::filesystem::path& roblox_library_path() const { return roblox_library_path_; } @@ -135,6 +137,8 @@ class RuntimeConfig { private: bool headless_ = false; + bool vr_enabled_ = false; + bool vr_valid_ = true; std::filesystem::path roblox_library_path_ = "rbx_bin/libroblox.so"; GraphicsBackend graphics_backend_ = GraphicsBackend::kVulkan; std::string graphics_backend_name_ = "direct-vulkan"; diff --git a/scripts/vr_simulated.py b/scripts/vr_simulated.py new file mode 100644 index 00000000..0d4c9580 --- /dev/null +++ b/scripts/vr_simulated.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Launch the VR preview with a separately built Monado simulated runtime.""" + +import argparse +import json +import os +from pathlib import Path +import select +import shutil +import signal +import subprocess +import sys +import tempfile + + +def positive_integer(value): + number = int(value) + if not 0 < number <= 0xFFFFFFFF: + raise argparse.ArgumentTypeError("must be a positive 32-bit integer") + return number + + +def prepare_runtime(source, destination): + """Keep the development runtime beside the build, independent of /tmp.""" + source = source.resolve() + document = json.loads(source.read_text()) + library = Path(document["runtime"]["library_path"]) + if not library.is_absolute(): + library = source.parent / library + if not library.is_file(): + raise ValueError(f"Runtime library does not exist: {library}") + destination.mkdir(parents=True, exist_ok=True) + output_library = destination / "libopenxr_monado.so" + if library.resolve() != output_library.resolve(): + shutil.copy2(library, output_library) + output_manifest = destination / "runtime.json" + output_manifest.write_text(json.dumps({ + "file_format_version": "1.0.0", + "runtime": {"name": "Monado (Mocktail simulator)", + "library_path": "./libopenxr_monado.so"}, + }, indent=2) + "\n") + print(f"Prepared simulator: {output_manifest}") + + +def simulator_environment(parent, manifest, display, wayland, software): + environment = parent.copy() + # These overrides belong to this child process; no active_runtime.json is + # written and the user's WiVRn/system runtime selection remains intact. + environment.update( + XR_RUNTIME_JSON=str(manifest.resolve()), + SIMULATED_ENABLE="1", + XRT_COMPOSITOR_NULL="0", + XRT_COMPOSITOR_FORCE_XCB="1" if display else "0", + XRT_COMPOSITOR_FORCE_WAYLAND="1" if not display and wayland else "0", + XRT_COMPOSITOR_FORCE_NVIDIA="0", + XRT_COMPOSITOR_FORCE_RANDR="0", + XRT_COMPOSITOR_FORCE_WAYLAND_DIRECT="0", + XRT_COMPOSITOR_FORCE_VK_DISPLAY="-1", + XRT_COMPOSITOR_XCB_FULLSCREEN="0", + ) + environment.setdefault("XRT_COMPOSITOR_DEFAULT_FRAMERATE", "30") + if display: + environment["DISPLAY"] = display + environment.pop("WAYLAND_DISPLAY", None) + if software: + candidates = [directory / "lvp_icd.json" for directory in + (Path("/usr/share/vulkan/icd.d"), Path("/usr/local/share/vulkan/icd.d"))] + manifest_path = next((path for path in candidates if path.is_file()), None) + if manifest_path is None: + raise ValueError("Mesa Lavapipe is not installed; omit --software to use your GPU") + environment["VK_DRIVER_FILES"] = str(manifest_path) + environment.pop("VK_ICD_FILENAMES", None) + environment["XRT_COMPOSITOR_FORCE_GPU_INDEX"] = "0" + environment["XRT_COMPOSITOR_FORCE_CLIENT_GPU_INDEX"] = "0" + return environment + + +def main(): + location = Path(__file__).resolve().parent + default_build = location.parent / "build" if location.name == "scripts" else location + parser = argparse.ArgumentParser( + description="Show the Mocktail VR test scene on the desktop using a simulated Monado headset.") + parser.add_argument("--build-dir", type=Path, default=default_build, + help="Mocktail build directory (defaults to the directory of this launcher)") + parser.add_argument("--runtime", type=Path, + help="Manifest of an in-process Monado build with the simulated driver enabled") + parser.add_argument("--prepare", type=Path, metavar="MONADO_MANIFEST", + help="Copy an already built Monado runtime next to this build, then exit") + parser.add_argument("--frames", type=positive_integer, + help="Request exit after this many frames; otherwise run until Ctrl+C") + parser.add_argument("--software", action="store_true", help="Use Mesa Lavapipe instead of a physical GPU") + parser.add_argument("--headless", action="store_true", + help="Use a private Xvfb display for automated checks (defaults to 180 frames)") + args = parser.parse_args() + build = args.build_dir.resolve() + if args.prepare: + prepare_runtime(args.prepare, build / "vr-simulator") + return 0 + manifest = args.runtime or build / "vr-simulator/runtime.json" + if not manifest.is_file(): + parser.error("No simulated Monado runtime is prepared. Use --runtime /path/to/openxr_monado-dev.json " + "or --prepare /path/to/openxr_monado-dev.json with a separately built Monado runtime.") + probe = build / "mocktail-vr-probe" + if not probe.is_file(): + parser.error(f"Build mocktail_vr_probe with MOCKTAIL_ENABLE_VR=ON first: {probe}") + display = os.environ.get("DISPLAY") + wayland = os.environ.get("WAYLAND_DISPLAY") + if not args.headless and not (display or wayland): + parser.error("Run from a graphical desktop terminal to see the scene, or use --headless for an automated check.") + + xvfb = None + try: + if args.headless: + if not shutil.which("Xvfb"): + parser.error("--headless requires Xvfb") + read_fd, write_fd = os.pipe() + try: + xvfb = subprocess.Popen( + ["Xvfb", "-displayfd", str(write_fd), "-screen", "0", "1280x720x24", "-nolisten", "tcp"], + pass_fds=(write_fd,), stdout=subprocess.DEVNULL, start_new_session=True) + except OSError: + os.close(read_fd) + raise + finally: + os.close(write_fd) + with os.fdopen(read_fd) as number: + if not select.select([number], [], [], 10)[0]: + raise ValueError("Xvfb did not start within 10 seconds") + display_number = number.readline().strip() + if not display_number.isdigit(): + raise ValueError("Xvfb failed to allocate a display") + display = ":" + display_number + wayland = None + env = simulator_environment(os.environ, manifest, display, wayland, args.software) + # Keep simulator settings and runtime files private and temporary. The + # desktop's XDG_RUNTIME_DIR is retained for Wayland authentication. + with tempfile.TemporaryDirectory(prefix="mocktail-vr-simulated-") as directory: + root = Path(directory) + env["XDG_CONFIG_HOME"] = str(root / "config") + env["XDG_DATA_HOME"] = str(root / "data") + if not env.get("XDG_RUNTIME_DIR"): + env["XDG_RUNTIME_DIR"] = str(root) + command = [str(probe), "--scene"] + frames = args.frames or (180 if args.headless else None) + if frames: + command += ["--frames", str(frames)] + print(f"Simulated Monado headset: {manifest.resolve()}", flush=True) + print("The test scene will appear in a desktop window. Ctrl+C exits." if not args.headless + else "Checking the test scene on a private Xvfb display.", flush=True) + child = None + previous = {} + + def forward_signal(signum, _frame): + if child is not None and child.poll() is None: + child.send_signal(signum) + + try: + child = subprocess.Popen(command, env=env, start_new_session=True) + for signum in (signal.SIGINT, signal.SIGTERM): + previous[signum] = signal.signal(signum, forward_signal) + return child.wait() + finally: + if child is not None and child.poll() is None: + child.terminate() + try: + child.wait(timeout=10) + except subprocess.TimeoutExpired: + child.kill() + child.wait() + for signum, handler in previous.items(): + signal.signal(signum, handler) + finally: + if xvfb is not None: + xvfb.terminate() + try: + xvfb.wait(timeout=5) + except subprocess.TimeoutExpired: + xvfb.kill() + xvfb.wait() + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, ValueError, KeyError) as error: + print(f"VR simulator: {error}", file=sys.stderr) + sys.exit(1) diff --git a/src/legacy/legacy_runtime.cc b/src/legacy/legacy_runtime.cc index fe45a91c..4c93dc32 100644 --- a/src/legacy/legacy_runtime.cc +++ b/src/legacy/legacy_runtime.cc @@ -2476,7 +2476,8 @@ jobject BuildAppBridgeInitParams(JNIEnv* env, jstring client_settings, SetStringField(env, params, "userAgent", user_agent.c_str()); SetBooleanField(env, params, "isTablet", JNI_FALSE); SetBooleanField(env, params, "isPotato", JNI_FALSE); - SetBooleanField(env, params, "isVrDevice", JNI_FALSE); + SetBooleanField(env, params, "isVrDevice", + IsEnabled("MOCKTAIL_VR_ENABLED") ? JNI_TRUE : JNI_FALSE); SetStringField(env, params, "buildVariant", "googleProdRelease"); SetObjectField(env, params, "vrContext", "Landroid/app/Activity;", activity); @@ -4507,6 +4508,18 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, const mocktail::compat::BuildProfile& build_profile = *profile_result.profile; + // The experimental VR ABI has only been observed on the active 2998 + // payload. Do not pass VR device parameters to unrelated builds, even when + // they have a general-purpose compatibility profile. Eye initialization and + // OpenXR projection are not yet implemented; this is a diagnostic launch. + if (IsEnabled("MOCKTAIL_VR_ENABLED") && + build_profile.elf_build_id != + "ade08266c67aee88ec9c1d00902150e1684dad3a") { + std::cerr << "[FATAL] Experimental Roblox VR requires exact Build ID " + "ade08266c67aee88ec9c1d00902150e1684dad3a; loaded " + << build_profile.elf_build_id << "\n"; + return EXIT_FAILURE; + } const mocktail::compat::HostAbiProfile* host_abi_profile = mocktail::compat::FindHostAbiProfile(build_profile.elf_build_id); const bool experiment_allowed = diff --git a/src/main.cc b/src/main.cc index c3899aff..1f9352c5 100644 --- a/src/main.cc +++ b/src/main.cc @@ -308,6 +308,36 @@ int main(int argc, char* argv[]) { } mocktail::runtime::ScrubCommandLineLaunchArguments(&command_line.options, argc, argv); + + const mocktail::runtime::RuntimeConfigBootstrapResult config_bootstrap = + mocktail::runtime::EnsureRuntimeConfigFile(paths.config_file()); + if (!config_bootstrap) { + std::cerr << "[FATAL] Cannot prepare " << paths.config_file() << ": " + << config_bootstrap.error << '\n'; + if (command_line.options.mode == mocktail::runtime::CommandMode::kRun) { + (void)mocktail::runtime::ShowFailureDialog( + environment, "Mocktail could not prepare its configuration."); + } + return EXIT_FAILURE; + } + mocktail::runtime::RuntimeConfigLoadResult runtime_config = + mocktail::runtime::LoadRuntimeConfig(environment, paths.config_file()); + if (!runtime_config) { + std::cerr << "[FATAL] Cannot load " << paths.config_file() << ": " + << runtime_config.error << '\n'; + if (command_line.options.mode == mocktail::runtime::CommandMode::kRun) { + (void)mocktail::runtime::ShowFailureDialog( + environment, "Mocktail could not load its configuration."); + } + return EXIT_FAILURE; + } + // The standalone OpenXR scene belongs to mocktail-vr-probe --scene. + // Continue through Roblox startup to inspect its native VR device. + if (runtime_config.config.vr_enabled()) { + std::cerr << " [vr] Roblox VR initialization requested (experimental); " + "stereo eye resources and OpenXR output are not yet connected. " + "Standalone scene: mocktail-vr-probe --scene\n"; + } mocktail::runtime::SessionLog session_log; mocktail::runtime::FailureSupportBundleGuard support_bundle_guard( environment, paths, @@ -355,28 +385,6 @@ int main(int argc, char* argv[]) { return EXIT_FAILURE; } } - const mocktail::runtime::RuntimeConfigBootstrapResult config_bootstrap = - mocktail::runtime::EnsureRuntimeConfigFile(paths.config_file()); - if (!config_bootstrap) { - std::cerr << "[FATAL] Cannot prepare " << paths.config_file() << ": " - << config_bootstrap.error << '\n'; - if (command_line.options.mode == mocktail::runtime::CommandMode::kRun) { - (void)mocktail::runtime::ShowFailureDialog( - environment, "Mocktail could not prepare its configuration."); - } - return EXIT_FAILURE; - } - mocktail::runtime::RuntimeConfigLoadResult runtime_config = - mocktail::runtime::LoadRuntimeConfig(environment, paths.config_file()); - if (!runtime_config) { - std::cerr << "[FATAL] Cannot load " << paths.config_file() << ": " - << runtime_config.error << '\n'; - if (command_line.options.mode == mocktail::runtime::CommandMode::kRun) { - (void)mocktail::runtime::ShowFailureDialog( - environment, "Mocktail could not load its configuration."); - } - return EXIT_FAILURE; - } if (runtime_config.config.use_system_proxy()) { const mocktail::runtime::SystemProxyResult system_proxy = mocktail::runtime::ResolveSystemProxy(); diff --git a/src/runtime/command_line.cc b/src/runtime/command_line.cc index 42dcc9d2..08b9acf7 100644 --- a/src/runtime/command_line.cc +++ b/src/runtime/command_line.cc @@ -74,6 +74,8 @@ CommandLineParseResult ParseCommandLine(int argc, const char* const argv[]) { &result.options.graphics_backend, &result.error)) { return result; } + } else if (argument == "--vr" || argument == "--no-vr") { + result.options.vr_enabled = argument == "--vr"; } else if (argument == "--allow-unverified-build") { result.options.allow_unverified_build = true; } else if (argument == "--force-run-latest") { @@ -141,6 +143,7 @@ CommandLineParseResult ParseCommandLine(int argc, const char* const argv[]) { (!result.options.roblox_library_path.empty() || result.options.window_mode != WindowMode::kUnspecified || !result.options.graphics_backend.empty() || + result.options.vr_enabled.has_value() || result.options.allow_unverified_build || !result.options.launch_request_json.empty())) { result.error = "--force-run-latest must be used on its own"; @@ -218,6 +221,11 @@ void ScrubCommandLineLaunchArguments(CommandLineOptions* options, int argc, bool ApplyCommandLineEnvironment(const CommandLineOptions& options, std::string* error) { + if (options.vr_enabled.has_value() && + !SetEnvironment("MOCKTAIL_VR_ENABLED", *options.vr_enabled ? "1" : "0", + error)) { + return false; + } if (!options.roblox_library_path.empty() && !SetEnvironment("ROBLOX_LIB_PATH", options.roblox_library_path, error)) { return false; @@ -248,6 +256,8 @@ std::string CommandLineUsage(const std::string& program_name) { "x86_64 libroblox.so\n" << " --headless Run without creating an SDL window\n" << " --windowed Force windowed startup (default)\n" + << " --vr / --no-vr Launch Roblox in experimental VR mode or " + "disable vr.enabled\n" << " --graphics direct-vulkan | opengl | system | " "angle-vulkan (default: direct-vulkan)\n" << " --allow-unverified-build Run a known but unverified Build-ID " diff --git a/src/runtime/runtime_config.cc b/src/runtime/runtime_config.cc index f3e9d9a1..18e09137 100644 --- a/src/runtime/runtime_config.cc +++ b/src/runtime/runtime_config.cc @@ -170,6 +170,8 @@ std::string BuildNetworkProxyUrl(const NetworkProxyConfig& proxy) { RuntimeConfig RuntimeConfig::FromEnvironment(const Environment& environment) { RuntimeConfig config; config.headless_ = LegacyEnabled(environment, "MOCKTAIL_HEADLESS"); + config.vr_enabled_ = + ReadBoolean(environment, "MOCKTAIL_VR_ENABLED", false, &config.vr_valid_); config.roblox_library_path_ = environment.GetOr( "ROBLOX_LIB_PATH", config.roblox_library_path_.string()); config.graphics_backend_name_ = environment.GetOr( diff --git a/src/runtime/runtime_config_bootstrap.cc b/src/runtime/runtime_config_bootstrap.cc index f0d53868..3905077d 100644 --- a/src/runtime/runtime_config_bootstrap.cc +++ b/src/runtime/runtime_config_bootstrap.cc @@ -67,6 +67,13 @@ device: pc-windows-11 # Optional presentation synchronization override: auto, on, or off. # vsync: off +vr: + # Boolean (default: false): launch the Roblox experimental VR path. + # Only Build ID 2998 is currently accepted; eye resources and OpenXR output + # are NOT yet connected. Standalone scene: mocktail-vr-probe --scene. + # Use --no-vr to launch Roblox normally. + enabled: false + performance: # Boolean (default: false): size Roblox scheduler/render queues from every # physical CPU core. A place's Lua/main thread can still remain serial. diff --git a/src/runtime/runtime_config_file.cc b/src/runtime/runtime_config_file.cc index faee55b6..9ad3422c 100644 --- a/src/runtime/runtime_config_file.cc +++ b/src/runtime/runtime_config_file.cc @@ -168,6 +168,7 @@ bool ValidateAndMap(const ValueMap& yaml, ValueMap* environment, "device.keyboard", "runtime.headless", "runtime.roblox_library", + "vr.enabled", "appearance.theme", "graphics.backend", "graphics.frame_rate_limit", @@ -308,6 +309,14 @@ bool ValidateAndMap(const ValueMap& yaml, ValueMap* environment, } (*environment)["MOCKTAIL_THEME"] = *theme; } + if (const auto enabled = value("vr.enabled"); enabled.has_value()) { + bool parsed = false; + if (!ParseBoolean(*enabled, &parsed)) { + *error = "vr.enabled must be true or false"; + return false; + } + (*environment)["MOCKTAIL_VR_ENABLED"] = parsed ? "1" : "0"; + } if (const auto backend = value("graphics.backend"); backend.has_value()) { if (RuntimeConfig::ParseGraphicsBackend(*backend) == GraphicsBackend::kUnknown) { @@ -656,7 +665,7 @@ bool LoadYaml(const std::filesystem::path& path, ValueMap* values, bool* loaded, *error = "device must be a preset scalar or detailed mapping"; valid = false; } - } else if (key == "runtime" || key == "appearance" || + } else if (key == "runtime" || key == "appearance" || key == "vr" || key == "graphics" || key == "performance" || key == "audio" || key == "window" || key == "input" || key == "compatibility" || key == "network") { @@ -725,6 +734,9 @@ RuntimeConfigLoadResult LoadRuntimeConfig( result.config = RuntimeConfig::FromEnvironment(layered); if (!result.config.frame_rate().valid()) { result.error = "frame-rate policy is invalid"; + } else if (!result.config.vr_valid()) { + result.error = "VR configuration is invalid: MOCKTAIL_VR_ENABLED must be " + "1/0, true/false, or on/off"; } else if (!result.config.device_profile_valid()) { result.error = "device profile is invalid"; } else if (!result.config.input_capabilities().touch_enabled && @@ -761,6 +773,10 @@ RuntimeConfigLoadResult LoadRuntimeConfig( bool ExportRuntimeConfigEnvironment(const RuntimeConfig& config, std::string* error) { + if (!config.vr_valid()) { + if (error != nullptr) *error = "cannot export an invalid VR configuration"; + return false; + } if (!config.device_profile_valid()) { if (error != nullptr) { *error = "cannot export an invalid device profile"; @@ -813,6 +829,8 @@ bool ExportRuntimeConfigEnvironment(const RuntimeConfig& config, const bool base_exported = SetEnvironmentValue("MOCKTAIL_HEADLESS", config.headless() ? "1" : "0", error) && + SetEnvironmentValue("MOCKTAIL_VR_ENABLED", config.vr_enabled() ? "1" : "0", + error) && SetEnvironmentValue("MOCKTAIL_DEVICE_PROFILE", std::string(device.name), error) && SetEnvironmentValue("MOCKTAIL_DEVICE_CLASS", diff --git a/src/vr/diagnostics.cc b/src/vr/diagnostics.cc new file mode 100644 index 00000000..2d374358 --- /dev/null +++ b/src/vr/diagnostics.cc @@ -0,0 +1,75 @@ +#include "mocktail/vr/openxr_probe.h" +#include "mocktail/vr/openxr_preview.h" + +#include + +namespace mocktail::vr { + +std::string FormatPreviewResult(const PreviewResult& result) { + std::ostringstream output; + output << "[VR] " << result.message << '\n' + << "Frames: " << result.frames + << ", rendered: " << result.rendered_frames + << ", tracked: " << result.tracked_frames << '\n'; + if (result.tracked_frames) { + output << "Head position (metres): " << result.head_position[0] << ", " + << result.head_position[1] << ", " << result.head_position[2] << '\n' + << "Head orientation (xyzw): " << result.head_orientation[0] << ", " + << result.head_orientation[1] << ", " << result.head_orientation[2] + << ", " << result.head_orientation[3] << '\n'; + } + return output.str(); +} + +int ProbeExitCode(ProbeStatus status) { + switch (status) { + case ProbeStatus::kReady: + return 0; + case ProbeStatus::kBuildDisabled: + return 2; + case ProbeStatus::kRuntimeUnavailable: + return 3; + case ProbeStatus::kHeadsetUnavailable: + return 4; + case ProbeStatus::kIncompatibleRuntime: + return 5; + case ProbeStatus::kError: + return 1; + } + return 1; +} + +std::string FormatProbeReport(const ProbeReport& report) { + std::ostringstream output; + output << "Mocktail OpenXR discovery\n"; + if (!report.sdk_version.empty()) + output << "SDK: " << report.sdk_version << '\n'; + if (!report.runtime_name.empty()) { + output << "Runtime: " << report.runtime_name << " (" + << report.runtime_version << ")\n"; + } + if (!report.system_name.empty()) { + output << "System: " << report.system_name << '\n' + << "Tracking: orientation=" << report.orientation_tracking + << " position=" << report.position_tracking << '\n'; + } + for (std::size_t eye = 0; eye < report.views.size(); ++eye) { + const auto& view = report.views[eye]; + output << "Eye " << eye << ": " << view.width << 'x' << view.height + << ", samples=" << view.samples << '\n'; + } + if (!report.minimum_vulkan_version.empty()) { + output << "Vulkan: minimum=" << report.minimum_vulkan_version + << ", highest runtime-tested=" << report.maximum_vulkan_version + << '\n'; + } + output << "Extensions (" << report.extensions.size() << "):\n"; + for (const auto& extension : report.extensions) + output << " " << extension << '\n'; + output << "Discovery: " << report.message << '\n' + << "Roblox VR rendering: not implemented yet.\n" + << "This probe does not create an XR session or render frames.\n"; + return output.str(); +} + +} // namespace mocktail::vr diff --git a/src/vr/openxr_disabled.cc b/src/vr/openxr_disabled.cc new file mode 100644 index 00000000..f967e102 --- /dev/null +++ b/src/vr/openxr_disabled.cc @@ -0,0 +1,23 @@ +#include "mocktail/vr/openxr_probe.h" +#include "mocktail/vr/openxr_preview.h" + +namespace mocktail::vr { + +PreviewResult RunOpenXrPreview(const PreviewOptions&) { + PreviewResult result; + result.message = + "This build does not include OpenXR support. Use a VR-enabled build of " + "Mocktail."; + return result; +} + +ProbeReport ProbeOpenXr() { + ProbeReport report; + report.status = ProbeStatus::kBuildDisabled; + report.message = + "This build does not include OpenXR support. Use a " + "VR-enabled build of Mocktail."; + return report; +} + +} // namespace mocktail::vr diff --git a/src/vr/openxr_preview.cc b/src/vr/openxr_preview.cc new file mode 100644 index 00000000..2644d5b1 --- /dev/null +++ b/src/vr/openxr_preview.cc @@ -0,0 +1,729 @@ +#include "mocktail/vr/openxr_preview.h" + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mocktail::vr { +namespace { + +using Clock = std::chrono::steady_clock; +volatile sig_atomic_t interrupted = 0; +void Interrupt(int) { interrupted = 1; } + +class SignalScope { + public: + SignalScope() { + interrupted = 0; + struct sigaction action{}; + action.sa_handler = Interrupt; + sigemptyset(&action.sa_mask); + have_int_ = sigaction(SIGINT, &action, &old_int_) == 0; + have_term_ = sigaction(SIGTERM, &action, &old_term_) == 0; + } + ~SignalScope() { + if (have_int_) sigaction(SIGINT, &old_int_, nullptr); + if (have_term_) sigaction(SIGTERM, &old_term_, nullptr); + } + + private: + struct sigaction old_int_{}, old_term_{}; + bool have_int_ = false, have_term_ = false; +}; + +template +T XrInfo(XrStructureType type) { + T value{}; + value.type = type; + return value; +} + +template +T VkInfo(VkStructureType type) { + T value{}; + value.sType = type; + return value; +} + +void Require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +void CheckVk(VkResult result, const char* operation) { + Require(result == VK_SUCCESS, + std::string(operation) + " failed: " + std::to_string(result)); +} + +// Every Vulkan call uses the host loader explicitly. Linking global vk* +// symbols here could accidentally enter Mocktail's Android Vulkan adapter. +struct Vulkan { + void* library = nullptr; + PFN_vkGetInstanceProcAddr get_instance_proc = nullptr; + PFN_vkGetDeviceProcAddr get_device_proc = nullptr; + VkInstance instance = VK_NULL_HANDLE; + VkPhysicalDevice physical_device = VK_NULL_HANDLE; + VkDevice device = VK_NULL_HANDLE; + VkQueue queue = VK_NULL_HANDLE; + std::uint32_t queue_family = 0; + PFN_vkDestroyInstance DestroyInstance = nullptr; + +#define MOCKTAIL_PREVIEW_VK_FUNCTIONS(X) \ + X(DestroyDevice) \ + X(GetDeviceQueue) \ + X(DeviceWaitIdle) \ + X(CreateImageView) \ + X(DestroyImageView) \ + X(CreateRenderPass) \ + X(DestroyRenderPass) \ + X(CreateFramebuffer) \ + X(DestroyFramebuffer) \ + X(CreateCommandPool) \ + X(DestroyCommandPool) \ + X(AllocateCommandBuffers) \ + X(ResetCommandBuffer) \ + X(BeginCommandBuffer) \ + X(EndCommandBuffer) \ + X(CmdBeginRenderPass) \ + X(CmdEndRenderPass) \ + X(CmdClearAttachments) \ + X(CreateFence) \ + X(DestroyFence) \ + X(ResetFences) \ + X(WaitForFences) \ + X(QueueSubmit) +#define DECLARE(name) PFN_vk##name name = nullptr; + MOCKTAIL_PREVIEW_VK_FUNCTIONS(DECLARE) +#undef DECLARE + + Vulkan() = default; + Vulkan(const Vulkan&) = delete; + Vulkan& operator=(const Vulkan&) = delete; + ~Vulkan() { + if (device && DestroyDevice) DestroyDevice(device, nullptr); + if (instance && DestroyInstance) DestroyInstance(instance, nullptr); + if (library) dlclose(library); + } + + void OpenLoader() { + library = dlopen("libvulkan.so.1", RTLD_NOW | RTLD_LOCAL); + Require(library != nullptr, + "Cannot load the host Vulkan loader libvulkan.so.1"); + get_instance_proc = reinterpret_cast( + dlsym(library, "vkGetInstanceProcAddr")); + Require(get_instance_proc != nullptr, + "Host vkGetInstanceProcAddr is missing"); + } + + template + T InstanceFunction(const char* name) { + const auto function = + reinterpret_cast(get_instance_proc(instance, name)); + Require(function != nullptr, + std::string("Missing Vulkan function: ") + name); + return function; + } + + void LoadDeviceFunctions() { +#define LOAD(name) \ + name = reinterpret_cast(get_device_proc(device, "vk" #name)); \ + Require(name != nullptr, "Missing Vulkan function: vk" #name); + MOCKTAIL_PREVIEW_VK_FUNCTIONS(LOAD) +#undef LOAD + GetDeviceQueue(device, queue_family, 0, &queue); + } +}; +#undef MOCKTAIL_PREVIEW_VK_FUNCTIONS + +class Preview { + public: + Preview() = default; + Preview(const Preview&) = delete; + Preview& operator=(const Preview&) = delete; + ~Preview() { + // The runtime still needs our VkDevice when destroying its + // swapchain/session. + if (vk_.device && vk_.DeviceWaitIdle) (void)vk_.DeviceWaitIdle(vk_.device); + for (auto framebuffer : framebuffers_) + vk_.DestroyFramebuffer(vk_.device, framebuffer, nullptr); + for (auto view : image_views_) + vk_.DestroyImageView(vk_.device, view, nullptr); + if (render_pass_) vk_.DestroyRenderPass(vk_.device, render_pass_, nullptr); + if (fence_) vk_.DestroyFence(vk_.device, fence_, nullptr); + if (pool_) vk_.DestroyCommandPool(vk_.device, pool_, nullptr); + if (swapchain_) (void)xrDestroySwapchain(swapchain_); + if (view_space_) (void)xrDestroySpace(view_space_); + if (local_space_) (void)xrDestroySpace(local_space_); + if (session_) (void)xrDestroySession(session_); + if (instance_) (void)xrDestroyInstance(instance_); + } + + void Initialize() { + const char* extension = XR_KHR_VULKAN_ENABLE2_EXTENSION_NAME; + auto info = XrInfo(XR_TYPE_INSTANCE_CREATE_INFO); + std::strcpy(info.applicationInfo.applicationName, "Mocktail VR preview"); + std::strcpy(info.applicationInfo.engineName, "Mocktail"); + info.applicationInfo.apiVersion = XR_MAKE_VERSION(1, 0, 0); + info.enabledExtensionCount = 1; + info.enabledExtensionNames = &extension; + Check(xrCreateInstance(&info, &instance_), "xrCreateInstance"); + auto system = XrInfo(XR_TYPE_SYSTEM_GET_INFO); + system.formFactor = XR_FORM_FACTOR_HEAD_MOUNTED_DISPLAY; + Check(xrGetSystem(instance_, &system, &system_), "xrGetSystem"); + + const auto views = Enumerate( + XR_TYPE_VIEW_CONFIGURATION_VIEW, + [&](auto capacity, auto* count, auto* data) { + return xrEnumerateViewConfigurationViews( + instance_, system_, XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, + capacity, count, data); + }); + Require(views.size() == 2, + "The VR preview requires PRIMARY_STEREO support"); + const auto blends = Enumerate( + XR_TYPE_UNKNOWN, [&](auto capacity, auto* count, auto* data) { + return xrEnumerateEnvironmentBlendModes( + instance_, system_, XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, + capacity, count, data); + }); + Require(!blends.empty(), "The runtime exposes no environment blend modes"); + blend_ = std::find(blends.begin(), blends.end(), + XR_ENVIRONMENT_BLEND_MODE_OPAQUE) != blends.end() + ? XR_ENVIRONMENT_BLEND_MODE_OPAQUE + : blends.front(); + CreateGraphics(); + auto binding = XrInfo( + XR_TYPE_GRAPHICS_BINDING_VULKAN2_KHR); + binding.instance = vk_.instance; + binding.physicalDevice = vk_.physical_device; + binding.device = vk_.device; + binding.queueFamilyIndex = vk_.queue_family; + auto session_info = + XrInfo(XR_TYPE_SESSION_CREATE_INFO); + session_info.next = &binding; + session_info.systemId = system_; + Check(xrCreateSession(instance_, &session_info, &session_), + "xrCreateSession"); + auto space_info = + XrInfo(XR_TYPE_REFERENCE_SPACE_CREATE_INFO); + space_info.poseInReferenceSpace.orientation.w = 1; + space_info.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_LOCAL; + Check(xrCreateReferenceSpace(session_, &space_info, &local_space_), + "xrCreateReferenceSpace LOCAL"); + space_info.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_VIEW; + Check(xrCreateReferenceSpace(session_, &space_info, &view_space_), + "xrCreateReferenceSpace VIEW"); + CreateImages(); + } + + void Run(const PreviewOptions& options, PreviewResult* report) { + const auto ready_deadline = Clock::now() + std::chrono::seconds(30); + auto exit_deadline = Clock::time_point::max(); + bool running = false, exit_requested = false; + while (true) { + while (true) { + auto event = XrInfo(XR_TYPE_EVENT_DATA_BUFFER); + const auto result = xrPollEvent(instance_, &event); + if (result == XR_EVENT_UNAVAILABLE) break; + Check(result, "xrPollEvent"); + if (event.type == XR_TYPE_EVENT_DATA_INSTANCE_LOSS_PENDING) + throw std::runtime_error("The OpenXR runtime was lost"); + if (event.type != XR_TYPE_EVENT_DATA_SESSION_STATE_CHANGED) continue; + const auto& changed = + reinterpret_cast(event); + if (changed.session != session_) continue; + switch (changed.state) { + case XR_SESSION_STATE_READY: { + auto begin = XrInfo(XR_TYPE_SESSION_BEGIN_INFO); + begin.primaryViewConfigurationType = + XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO; + Check(xrBeginSession(session_, &begin), "xrBeginSession"); + running = true; + std::cerr << "[VR] Session started. A test panel is fixed two " + "metres ahead.\n"; + break; + } + case XR_SESSION_STATE_STOPPING: + if (running) Check(xrEndSession(session_), "xrEndSession"); + return; + case XR_SESSION_STATE_EXITING: + return; + case XR_SESSION_STATE_LOSS_PENDING: + throw std::runtime_error( + "The OpenXR session was lost; reconnect and launch again"); + default: + break; + } + } + if (interrupted || + (options.frame_limit && report->frames >= options.frame_limit)) { + if (!running) return; + if (!exit_requested) { + Check(xrRequestExitSession(session_), "xrRequestExitSession"); + exit_requested = true; + exit_deadline = Clock::now() + std::chrono::seconds(5); + // Consume an immediate STOPPING event before starting another frame. + continue; + } + } + Require(Clock::now() < exit_deadline, + "The runtime did not stop the VR session after an exit request"); + if (!running) { + Require(Clock::now() < ready_deadline, + "Timed out waiting for the VR session to become READY"); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + continue; + } + Frame(report); + } + } + + private: + void Check(XrResult result, const char* operation) const { + if (XR_SUCCEEDED(result)) return; + char text[XR_MAX_RESULT_STRING_SIZE]{}; + if (instance_) (void)xrResultToString(instance_, result, text); + if (!text[0] && result == XR_ERROR_RUNTIME_UNAVAILABLE) + std::strcpy(text, "XR_ERROR_RUNTIME_UNAVAILABLE"); + std::string message = + std::string(operation) + + " failed: " + (text[0] ? std::string(text) : std::to_string(result)); + if (result == XR_ERROR_RUNTIME_UNAVAILABLE) + message += ". No active OpenXR runtime was found. Start WiVRn for a real " + "headset. Without a headset, run mocktail-vr-simulated from " + "this build directory."; + else if (result == XR_ERROR_FORM_FACTOR_UNAVAILABLE) + message += ". Connect a headset or select a simulated runtime."; + else if (result == XR_ERROR_EXTENSION_NOT_PRESENT) + message += ". XR_KHR_vulkan_enable2 is required."; + throw std::runtime_error(message); + } + + template + T Function(const char* name) { + PFN_xrVoidFunction function = nullptr; + Check(xrGetInstanceProcAddr(instance_, name, &function), name); + Require(function != nullptr, + std::string("Missing OpenXR function: ") + name); + return reinterpret_cast(function); + } + + template + std::vector Enumerate(XrStructureType type, F function) { + for (int retry = 0; retry < 3; ++retry) { + std::uint32_t count = 0; + Check(function(0, &count, static_cast(nullptr)), + "OpenXR enumeration count"); + Require(count <= 4096, "OpenXR enumeration exceeds the supported size"); + std::vector values(count); + if constexpr (std::is_class_v) + for (auto& value : values) value.type = type; + if (!count) return values; + const auto result = function(count, &count, values.data()); + if (result == XR_ERROR_SIZE_INSUFFICIENT) continue; + Check(result, "OpenXR enumeration data"); + Require(count <= values.size(), + "OpenXR enumeration returned an invalid size"); + values.resize(count); + return values; + } + throw std::runtime_error("OpenXR enumeration kept changing size"); + } + + void CreateGraphics() { + auto requirements = XrInfo( + XR_TYPE_GRAPHICS_REQUIREMENTS_VULKAN2_KHR); + Check(Function( + "xrGetVulkanGraphicsRequirements2KHR")(instance_, system_, + &requirements), + "xrGetVulkanGraphicsRequirements2KHR"); + vk_.OpenLoader(); + auto application = + VkInfo(VK_STRUCTURE_TYPE_APPLICATION_INFO); + application.pApplicationName = "Mocktail VR preview"; + const auto version = std::max( + requirements.minApiVersionSupported, XR_MAKE_VERSION(1, 0, 0)); + application.apiVersion = VK_MAKE_API_VERSION(0, XR_VERSION_MAJOR(version), + XR_VERSION_MINOR(version), 0); + auto instance_info = + VkInfo(VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO); + instance_info.pApplicationInfo = &application; + auto xr_instance_info = XrInfo( + XR_TYPE_VULKAN_INSTANCE_CREATE_INFO_KHR); + xr_instance_info.systemId = system_; + xr_instance_info.pfnGetInstanceProcAddr = vk_.get_instance_proc; + xr_instance_info.vulkanCreateInfo = &instance_info; + VkResult result = VK_ERROR_INITIALIZATION_FAILED; + Check(Function("xrCreateVulkanInstanceKHR")( + instance_, &xr_instance_info, &vk_.instance, &result), + "xrCreateVulkanInstanceKHR"); + CheckVk(result, "vkCreateInstance"); + vk_.DestroyInstance = + vk_.InstanceFunction("vkDestroyInstance"); + vk_.get_device_proc = + vk_.InstanceFunction("vkGetDeviceProcAddr"); + auto device_info = XrInfo( + XR_TYPE_VULKAN_GRAPHICS_DEVICE_GET_INFO_KHR); + device_info.systemId = system_; + device_info.vulkanInstance = vk_.instance; + Check(Function( + "xrGetVulkanGraphicsDevice2KHR")(instance_, &device_info, + &vk_.physical_device), + "xrGetVulkanGraphicsDevice2KHR"); + VkPhysicalDeviceProperties properties{}; + vk_.InstanceFunction( + "vkGetPhysicalDeviceProperties")(vk_.physical_device, &properties); + Require(properties.apiVersion >= application.apiVersion, + "The runtime-selected GPU does not support the required Vulkan " + "version"); + const auto get_families = + vk_.InstanceFunction( + "vkGetPhysicalDeviceQueueFamilyProperties"); + std::uint32_t count = 0; + get_families(vk_.physical_device, &count, nullptr); + Require(count > 0 && count <= 256, + "The runtime-selected GPU has no usable queue families"); + std::vector families(count); + get_families(vk_.physical_device, &count, families.data()); + const auto family = + std::find_if(families.begin(), families.end(), [](const auto& value) { + return value.queueCount && (value.queueFlags & VK_QUEUE_GRAPHICS_BIT); + }); + Require(family != families.end(), + "The runtime-selected GPU has no graphics queue"); + vk_.queue_family = static_cast(family - families.begin()); + const float priority = 1; + auto queue = VkInfo( + VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO); + queue.queueFamilyIndex = vk_.queue_family; + queue.queueCount = 1; + queue.pQueuePriorities = &priority; + auto create = + VkInfo(VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO); + create.queueCreateInfoCount = 1; + create.pQueueCreateInfos = &queue; + auto xr_create = XrInfo( + XR_TYPE_VULKAN_DEVICE_CREATE_INFO_KHR); + xr_create.systemId = system_; + xr_create.pfnGetInstanceProcAddr = vk_.get_instance_proc; + xr_create.vulkanPhysicalDevice = vk_.physical_device; + xr_create.vulkanCreateInfo = &create; + Check(Function("xrCreateVulkanDeviceKHR")( + instance_, &xr_create, &vk_.device, &result), + "xrCreateVulkanDeviceKHR"); + CheckVk(result, "vkCreateDevice"); + vk_.LoadDeviceFunctions(); + std::cerr << "[VR] Vulkan device: " << properties.deviceName << '\n'; + } + + void CreateImages() { + const auto formats = Enumerate( + XR_TYPE_UNKNOWN, [&](auto capacity, auto* count, auto* data) { + return xrEnumerateSwapchainFormats(session_, capacity, count, data); + }); + VkFormat format = VK_FORMAT_UNDEFINED; + for (auto candidate : + {VK_FORMAT_R8G8B8A8_SRGB, VK_FORMAT_B8G8R8A8_SRGB, + VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_B8G8R8A8_UNORM}) { + if (std::find(formats.begin(), formats.end(), candidate) != + formats.end()) { + format = candidate; + break; + } + } + Require(format != VK_FORMAT_UNDEFINED, + "The runtime exposes no supported RGBA swapchain format"); + auto system_properties = + XrInfo(XR_TYPE_SYSTEM_PROPERTIES); + Check(xrGetSystemProperties(instance_, system_, &system_properties), + "xrGetSystemProperties"); + width_ = std::min( + 512u, system_properties.graphicsProperties.maxSwapchainImageWidth); + height_ = std::min( + 512u, system_properties.graphicsProperties.maxSwapchainImageHeight); + Require(width_ >= 64 && height_ >= 64 && + system_properties.graphicsProperties.maxLayerCount > 0, + "The runtime cannot display the preview panel"); + auto swapchain = + XrInfo(XR_TYPE_SWAPCHAIN_CREATE_INFO); + swapchain.usageFlags = XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT; + swapchain.format = format; + swapchain.sampleCount = 1; + swapchain.width = width_; + swapchain.height = height_; + swapchain.faceCount = swapchain.arraySize = swapchain.mipCount = 1; + Check(xrCreateSwapchain(session_, &swapchain, &swapchain_), + "xrCreateSwapchain"); + const auto images = Enumerate( + XR_TYPE_SWAPCHAIN_IMAGE_VULKAN_KHR, + [&](auto capacity, auto* count, auto* data) { + return xrEnumerateSwapchainImages( + swapchain_, capacity, count, + reinterpret_cast(data)); + }); + Require(!images.empty(), "The runtime returned an empty swapchain"); + + VkAttachmentDescription attachment{}; + attachment.format = format; + attachment.samples = VK_SAMPLE_COUNT_1_BIT; + attachment.loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; + attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE; + attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + attachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + attachment.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + attachment.finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + VkAttachmentReference reference{0, + VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}; + VkSubpassDescription subpass{}; + subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; + subpass.colorAttachmentCount = 1; + subpass.pColorAttachments = &reference; + VkSubpassDependency dependency{}; + dependency.srcSubpass = VK_SUBPASS_EXTERNAL; + dependency.dstSubpass = 0; + dependency.srcStageMask = dependency.dstStageMask = + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; + dependency.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + auto pass = VkInfo( + VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO); + pass.attachmentCount = 1; + pass.pAttachments = &attachment; + pass.subpassCount = 1; + pass.pSubpasses = &subpass; + pass.dependencyCount = 1; + pass.pDependencies = &dependency; + CheckVk(vk_.CreateRenderPass(vk_.device, &pass, nullptr, &render_pass_), + "vkCreateRenderPass"); + for (const auto& image : images) { + auto view_info = VkInfo( + VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO); + view_info.image = image.image; + view_info.viewType = VK_IMAGE_VIEW_TYPE_2D; + view_info.format = format; + view_info.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + VkImageView view = VK_NULL_HANDLE; + CheckVk(vk_.CreateImageView(vk_.device, &view_info, nullptr, &view), + "vkCreateImageView"); + image_views_.push_back(view); + auto framebuffer_info = VkInfo( + VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO); + framebuffer_info.renderPass = render_pass_; + framebuffer_info.attachmentCount = 1; + framebuffer_info.pAttachments = &image_views_.back(); + framebuffer_info.width = width_; + framebuffer_info.height = height_; + framebuffer_info.layers = 1; + VkFramebuffer framebuffer = VK_NULL_HANDLE; + CheckVk(vk_.CreateFramebuffer(vk_.device, &framebuffer_info, nullptr, + &framebuffer), + "vkCreateFramebuffer"); + framebuffers_.push_back(framebuffer); + } + auto pool = VkInfo( + VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO); + pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + pool.queueFamilyIndex = vk_.queue_family; + CheckVk(vk_.CreateCommandPool(vk_.device, &pool, nullptr, &pool_), + "vkCreateCommandPool"); + auto allocate = VkInfo( + VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO); + allocate.commandPool = pool_; + allocate.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + allocate.commandBufferCount = 1; + CheckVk(vk_.AllocateCommandBuffers(vk_.device, &allocate, &command_), + "vkAllocateCommandBuffers"); + auto fence = VkInfo(VK_STRUCTURE_TYPE_FENCE_CREATE_INFO); + CheckVk(vk_.CreateFence(vk_.device, &fence, nullptr, &fence_), + "vkCreateFence"); + } + + void Render(std::uint32_t image, std::uint64_t frame) { + Require(image < framebuffers_.size(), + "The runtime acquired an invalid swapchain image index"); + CheckVk(vk_.ResetCommandBuffer(command_, 0), "vkResetCommandBuffer"); + auto begin = VkInfo( + VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO); + begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + CheckVk(vk_.BeginCommandBuffer(command_, &begin), "vkBeginCommandBuffer"); + VkClearValue background{}; + background.color = {{0.018f, 0.024f, 0.04f, 1.f}}; + auto pass = + VkInfo(VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO); + pass.renderPass = render_pass_; + pass.framebuffer = framebuffers_[image]; + pass.renderArea.extent = {width_, height_}; + pass.clearValueCount = 1; + pass.pClearValues = &background; + vk_.CmdBeginRenderPass(command_, &pass, VK_SUBPASS_CONTENTS_INLINE); + auto rectangle = [&](std::uint32_t x, std::uint32_t y, std::uint32_t w, + std::uint32_t h, VkClearColorValue color) { + VkClearAttachment attachment{}; + attachment.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + attachment.clearValue.color = color; + VkClearRect rect{ + {{static_cast(x), static_cast(y)}, + {w, h}}, + 0, + 1}; + vk_.CmdClearAttachments(command_, 1, &attachment, 1, &rect); + }; + for (std::uint32_t y = 0; y < 8; ++y) + for (std::uint32_t x = 0; x < 8; ++x) + if ((x + y) % 2 == 0) + rectangle(x * width_ / 8, y * height_ / 8, width_ / 8, height_ / 8, + x < 4 ? VkClearColorValue{{0.025f, 0.3f, 0.38f, 1.f}} + : VkClearColorValue{{0.23f, 0.08f, 0.4f, 1.f}}); + rectangle(width_ / 2 - 1, height_ / 4, 2, height_ / 2, + {{0.9f, 0.9f, 0.9f, 1.f}}); + rectangle(width_ / 4, height_ / 2 - 1, width_ / 2, 2, + {{0.9f, 0.9f, 0.9f, 1.f}}); + rectangle(static_cast(frame % (width_ - 16)), height_ - 12, + 16, 6, {{0.8f, 0.55f, 0.08f, 1.f}}); + vk_.CmdEndRenderPass(command_); + CheckVk(vk_.EndCommandBuffer(command_), "vkEndCommandBuffer"); + CheckVk(vk_.ResetFences(vk_.device, 1, &fence_), "vkResetFences"); + auto submit = VkInfo(VK_STRUCTURE_TYPE_SUBMIT_INFO); + submit.commandBufferCount = 1; + submit.pCommandBuffers = &command_; + CheckVk(vk_.QueueSubmit(vk_.queue, 1, &submit, fence_), "vkQueueSubmit"); + // One frame in flight for the first prototype: all GPU writes complete + // before release and command-buffer reuse. No runtime-owned image is freed. + CheckVk( + vk_.WaitForFences(vk_.device, 1, &fence_, VK_TRUE, 5'000'000'000ULL), + "vkWaitForFences"); + } + + void Frame(PreviewResult* report) { + auto wait = XrInfo(XR_TYPE_FRAME_WAIT_INFO); + auto state = XrInfo(XR_TYPE_FRAME_STATE); + Check(xrWaitFrame(session_, &wait, &state), "xrWaitFrame"); + auto begin = XrInfo(XR_TYPE_FRAME_BEGIN_INFO); + Check(xrBeginFrame(session_, &begin), "xrBeginFrame"); + auto end = XrInfo(XR_TYPE_FRAME_END_INFO); + end.displayTime = state.predictedDisplayTime; + end.environmentBlendMode = blend_; + bool frame_open = true, image_waited = false; + try { + auto head = XrInfo(XR_TYPE_SPACE_LOCATION); + Check(xrLocateSpace(view_space_, local_space_, state.predictedDisplayTime, + &head), + "xrLocateSpace"); + constexpr auto valid = XR_SPACE_LOCATION_ORIENTATION_VALID_BIT | + XR_SPACE_LOCATION_POSITION_VALID_BIT; + const bool pose_valid = (head.locationFlags & valid) == valid; + if (pose_valid) { + ++report->tracked_frames; + report->head_position = {head.pose.position.x, head.pose.position.y, + head.pose.position.z}; + report->head_orientation = { + head.pose.orientation.x, head.pose.orientation.y, + head.pose.orientation.z, head.pose.orientation.w}; + } + auto layer = + XrInfo(XR_TYPE_COMPOSITION_LAYER_QUAD); + const XrCompositionLayerBaseHeader* layer_pointer = + reinterpret_cast(&layer); + if (state.shouldRender && pose_valid) { + auto acquire = XrInfo( + XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO); + std::uint32_t image = 0; + Check(xrAcquireSwapchainImage(swapchain_, &acquire, &image), + "xrAcquireSwapchainImage"); + auto image_wait = + XrInfo(XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO); + image_wait.timeout = 50'000'000; + const auto deadline = Clock::now() + std::chrono::seconds(5); + while (true) { + const auto result = xrWaitSwapchainImage(swapchain_, &image_wait); + if (result != XR_TIMEOUT_EXPIRED) { + Check(result, "xrWaitSwapchainImage"); + break; + } + Require(!interrupted && Clock::now() < deadline, + "Timed out waiting for an OpenXR swapchain image"); + } + image_waited = true; + Render(image, report->frames); + auto release = XrInfo( + XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO); + image_waited = false; + Check(xrReleaseSwapchainImage(swapchain_, &release), + "xrReleaseSwapchainImage"); + layer.space = local_space_; + layer.eyeVisibility = XR_EYE_VISIBILITY_BOTH; + layer.subImage.swapchain = swapchain_; + layer.subImage.imageRect.extent = {static_cast(width_), + static_cast(height_)}; + layer.pose.orientation.w = 1; + layer.pose.position.z = -2; + layer.size = {1.4f, 1.4f}; + end.layerCount = 1; + end.layers = &layer_pointer; + } + frame_open = false; + Check(xrEndFrame(session_, &end), "xrEndFrame"); + ++report->frames; + if (end.layerCount) ++report->rendered_frames; + } catch (...) { + if (image_waited) { + if (vk_.DeviceWaitIdle) (void)vk_.DeviceWaitIdle(vk_.device); + auto release = XrInfo( + XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO); + (void)xrReleaseSwapchainImage(swapchain_, &release); + } + if (frame_open) { + end.layerCount = 0; + end.layers = nullptr; + (void)xrEndFrame(session_, &end); + } + throw; + } + } + + Vulkan vk_; + XrInstance instance_ = XR_NULL_HANDLE; + XrSystemId system_ = XR_NULL_SYSTEM_ID; + XrSession session_ = XR_NULL_HANDLE; + XrSpace local_space_ = XR_NULL_HANDLE, view_space_ = XR_NULL_HANDLE; + XrSwapchain swapchain_ = XR_NULL_HANDLE; + XrEnvironmentBlendMode blend_ = XR_ENVIRONMENT_BLEND_MODE_OPAQUE; + std::uint32_t width_ = 512, height_ = 512; + VkRenderPass render_pass_ = VK_NULL_HANDLE; + VkCommandPool pool_ = VK_NULL_HANDLE; + VkCommandBuffer command_ = VK_NULL_HANDLE; + VkFence fence_ = VK_NULL_HANDLE; + std::vector image_views_; + std::vector framebuffers_; +}; + +} // namespace + +PreviewResult RunOpenXrPreview(const PreviewOptions& options) { + PreviewResult result; + SignalScope signals; + try { + Preview preview; + preview.Initialize(); + std::cerr << "[VR] Experimental test scene. Roblox rendering is not " + "connected. Ctrl+C exits.\n"; + preview.Run(options, &result); + result.success = true; + result.message = "VR preview closed"; + } catch (const std::exception& error) { + result.message = error.what(); + } + return result; +} + +} // namespace mocktail::vr diff --git a/src/vr/openxr_probe.cc b/src/vr/openxr_probe.cc new file mode 100644 index 00000000..19639796 --- /dev/null +++ b/src/vr/openxr_probe.cc @@ -0,0 +1,209 @@ +#include "mocktail/vr/openxr_probe.h" + +#include +#include +#include + +#include +#include +#include + +namespace mocktail::vr { +namespace { + +std::string Version(XrVersion version) { + return std::to_string(XR_VERSION_MAJOR(version)) + "." + + std::to_string(XR_VERSION_MINOR(version)) + "." + + std::to_string(XR_VERSION_PATCH(version)); +} + +struct InstanceScope { + XrInstance instance = XR_NULL_HANDLE; + ~InstanceScope() { + if (instance != XR_NULL_HANDLE) (void)xrDestroyInstance(instance); + } +}; + +void Failure(ProbeReport* report, XrInstance instance, const char* operation, + XrResult result) { + report->status = ProbeStatus::kError; + if (result == XR_ERROR_RUNTIME_UNAVAILABLE) { + report->status = ProbeStatus::kRuntimeUnavailable; + } else if (result == XR_ERROR_FORM_FACTOR_UNAVAILABLE) { + report->status = ProbeStatus::kHeadsetUnavailable; + } else if (result == XR_ERROR_FORM_FACTOR_UNSUPPORTED || + result == XR_ERROR_VIEW_CONFIGURATION_TYPE_UNSUPPORTED) { + report->status = ProbeStatus::kIncompatibleRuntime; + } + char name[XR_MAX_RESULT_STRING_SIZE] = {}; + if (instance != XR_NULL_HANDLE) + (void)xrResultToString(instance, result, name); + report->message = std::string(operation) + " failed: " + + (name[0] ? std::string(name) : std::to_string(result)); + if (report->status == ProbeStatus::kRuntimeUnavailable) { + report->message += ". Start an OpenXR runtime such as WiVRn or Monado."; + } else if (report->status == ProbeStatus::kHeadsetUnavailable) { + report->message += + ". Connect a headset, or use a simulated runtime for development."; + } +} + +// Counts can change between the two enumeration calls. Retry a bounded number +// of times and never consume an output array after XR_ERROR_SIZE_INSUFFICIENT. +template +XrResult EnumerateProperties(XrStructureType type, Enumerate enumerate, + std::vector* values) { + for (int attempt = 0; attempt < 3; ++attempt) { + std::uint32_t count = 0; + XrResult result = enumerate(0, &count, nullptr); + if (XR_FAILED(result)) return result; + if (count > 4096) return XR_ERROR_LIMIT_REACHED; + values->assign(count, T{}); + for (auto& value : *values) value.type = type; + if (count == 0) return XR_SUCCESS; + result = enumerate(count, &count, values->data()); + if (result == XR_ERROR_SIZE_INSUFFICIENT) continue; + if (XR_FAILED(result)) return result; + if (count > values->size()) return XR_ERROR_RUNTIME_FAILURE; + values->resize(count); + return XR_SUCCESS; + } + return XR_ERROR_SIZE_INSUFFICIENT; +} + +} // namespace + +ProbeReport ProbeOpenXr() { + ProbeReport report; + report.sdk_version = Version(XR_CURRENT_API_VERSION); + std::vector extensions; + XrResult result = EnumerateProperties( + XR_TYPE_EXTENSION_PROPERTIES, + [](std::uint32_t capacity, std::uint32_t* count, + XrExtensionProperties* data) { + return xrEnumerateInstanceExtensionProperties(nullptr, capacity, count, + data); + }, + &extensions); + if (XR_FAILED(result)) { + Failure(&report, XR_NULL_HANDLE, "xrEnumerateInstanceExtensionProperties", + result); + return report; + } + for (const auto& extension : extensions) { + report.extensions.emplace_back(extension.extensionName); + } + const bool vulkan = + std::find(report.extensions.begin(), report.extensions.end(), + XR_KHR_VULKAN_ENABLE2_EXTENSION_NAME) != + report.extensions.end(); + const char* enabled_extension = XR_KHR_VULKAN_ENABLE2_EXTENSION_NAME; + XrInstanceCreateInfo create_info{}; + create_info.type = XR_TYPE_INSTANCE_CREATE_INFO; + std::strcpy(create_info.applicationInfo.applicationName, "Mocktail VR probe"); + std::strcpy(create_info.applicationInfo.engineName, "Mocktail"); + // Use the latest SDK while requiring only the core API this probe actually + // uses, so runtimes exposing OpenXR 1.0 can also be diagnosed. + create_info.applicationInfo.apiVersion = XR_MAKE_VERSION(1, 0, 0); + create_info.enabledExtensionCount = vulkan ? 1 : 0; + create_info.enabledExtensionNames = vulkan ? &enabled_extension : nullptr; + InstanceScope scope; + result = xrCreateInstance(&create_info, &scope.instance); + if (XR_FAILED(result)) { + Failure(&report, XR_NULL_HANDLE, "xrCreateInstance", result); + return report; + } + XrInstanceProperties properties{}; + properties.type = XR_TYPE_INSTANCE_PROPERTIES; + result = xrGetInstanceProperties(scope.instance, &properties); + if (XR_FAILED(result)) { + Failure(&report, scope.instance, "xrGetInstanceProperties", result); + return report; + } + report.runtime_name = properties.runtimeName; + report.runtime_version = Version(properties.runtimeVersion); + + XrSystemGetInfo system_info{}; + system_info.type = XR_TYPE_SYSTEM_GET_INFO; + system_info.formFactor = XR_FORM_FACTOR_HEAD_MOUNTED_DISPLAY; + XrSystemId system = XR_NULL_SYSTEM_ID; + result = xrGetSystem(scope.instance, &system_info, &system); + if (XR_FAILED(result)) { + Failure(&report, scope.instance, "xrGetSystem", result); + return report; + } + XrSystemProperties system_properties{}; + system_properties.type = XR_TYPE_SYSTEM_PROPERTIES; + result = xrGetSystemProperties(scope.instance, system, &system_properties); + if (XR_FAILED(result)) { + Failure(&report, scope.instance, "xrGetSystemProperties", result); + return report; + } + report.system_name = system_properties.systemName; + report.orientation_tracking = + system_properties.trackingProperties.orientationTracking == XR_TRUE; + report.position_tracking = + system_properties.trackingProperties.positionTracking == XR_TRUE; + if (!vulkan) { + report.status = ProbeStatus::kIncompatibleRuntime; + report.message = "The runtime does not expose XR_KHR_vulkan_enable2."; + return report; + } + + std::vector views; + result = EnumerateProperties( + XR_TYPE_VIEW_CONFIGURATION_VIEW, + [&](std::uint32_t capacity, std::uint32_t* count, + XrViewConfigurationView* data) { + return xrEnumerateViewConfigurationViews( + scope.instance, system, XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, + capacity, count, data); + }, + &views); + if (XR_FAILED(result)) { + Failure(&report, scope.instance, "xrEnumerateViewConfigurationViews", + result); + return report; + } + for (const auto& view : views) { + report.views.push_back({view.recommendedImageRectWidth, + view.recommendedImageRectHeight, + view.recommendedSwapchainSampleCount}); + } + if (views.size() != 2 || std::any_of(report.views.begin(), report.views.end(), + [](const ViewConfiguration& view) { + return !view.width || !view.height || + !view.samples; + })) { + report.status = ProbeStatus::kIncompatibleRuntime; + report.message = "The runtime did not provide two usable stereo views."; + return report; + } + + PFN_xrVoidFunction function = nullptr; + result = xrGetInstanceProcAddr( + scope.instance, "xrGetVulkanGraphicsRequirements2KHR", &function); + if (XR_FAILED(result) || function == nullptr) { + Failure(&report, scope.instance, + "xrGetVulkanGraphicsRequirements2KHR lookup", + XR_FAILED(result) ? result : XR_ERROR_FUNCTION_UNSUPPORTED); + return report; + } + XrGraphicsRequirementsVulkan2KHR requirements{}; + requirements.type = XR_TYPE_GRAPHICS_REQUIREMENTS_VULKAN2_KHR; + result = reinterpret_cast(function)( + scope.instance, system, &requirements); + if (XR_FAILED(result)) { + Failure(&report, scope.instance, "xrGetVulkanGraphicsRequirements2KHR", + result); + return report; + } + report.minimum_vulkan_version = Version(requirements.minApiVersionSupported); + report.maximum_vulkan_version = Version(requirements.maxApiVersionSupported); + report.status = ProbeStatus::kReady; + report.message = + "OpenXR HMD and Vulkan requirements discovered successfully."; + return report; +} + +} // namespace mocktail::vr diff --git a/src/vr/probe_main.cc b/src/vr/probe_main.cc new file mode 100644 index 00000000..a8b6a853 --- /dev/null +++ b/src/vr/probe_main.cc @@ -0,0 +1,50 @@ +#include "mocktail/vr/openxr_probe.h" +#include "mocktail/vr/openxr_preview.h" + +#include +#include +#include + +int main(int argc, char* argv[]) { + if (argc == 2 && (std::string_view(argv[1]) == "--help" || + std::string_view(argv[1]) == "-h")) { + std::cout + << "Usage: mocktail-vr-probe [--scene [--frames N]]\n" + "Inspect the host OpenXR runtime and HMD capabilities.\n" + "Uses the active runtime, or XR_RUNTIME_JSON if set.\n" + "Without arguments: discovery only, no XR session.\n" + "--scene: render a tracked test panel in a real Vulkan/XR session.\n" + "--frames N: request exit after N frames (positive integer).\n" + "Ctrl+C or the runtime closes the scene. Does not load Roblox.\n" + "Exit codes: 0=discovery ready, 1=API error, 2=build disabled,\n" + "3=runtime unavailable, 4=HMD unavailable, 5=incompatible " + "runtime.\n"; + return 0; + } + if (argc >= 2 && std::string_view(argv[1]) == "--scene") { + mocktail::vr::PreviewOptions options; + if (argc == 4 && std::string_view(argv[2]) == "--frames") { + const std::string_view value(argv[3]); + const auto parsed = std::from_chars( + value.data(), value.data() + value.size(), options.frame_limit); + if (parsed.ec != std::errc() || + parsed.ptr != value.data() + value.size() || !options.frame_limit) { + std::cerr << "--frames requires a positive integer.\n"; + return 1; + } + } else if (argc != 2) { + std::cerr << "Use --scene [--frames N].\n"; + return 1; + } + const auto result = mocktail::vr::RunOpenXrPreview(options); + std::cout << mocktail::vr::FormatPreviewResult(result); + return result.success ? 0 : 1; + } + if (argc != 1) { + std::cerr << "Use mocktail-vr-probe --help for supported options.\n"; + return 1; + } + const auto report = mocktail::vr::ProbeOpenXr(); + std::cout << mocktail::vr::FormatProbeReport(report); + return mocktail::vr::ProbeExitCode(report.status); +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ceb6ee23..21bb1e03 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -2,6 +2,34 @@ # # Licensed under the Apache License, Version 2.0 (the "License") +find_package(Python3 COMPONENTS Interpreter REQUIRED) +add_test(NAME OpenXrSimulatorLauncher + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/vr/simulator_launcher_test.py" + "${CMAKE_BINARY_DIR}/mocktail-vr-simulated") +if(MOCKTAIL_ENABLE_VR) + add_library(mocktail_test_openxr_runtime MODULE + vr/fake_openxr_runtime.cc vr/fake_openxr_preview.cc) + target_link_libraries(mocktail_test_openxr_runtime PRIVATE + OpenXR::headers Vulkan::Headers) + target_compile_definitions(mocktail_test_openxr_runtime PRIVATE + XR_USE_GRAPHICS_API_VULKAN) + set_target_properties(mocktail_test_openxr_runtime PROPERTIES + CXX_VISIBILITY_PRESET hidden) + mocktail_apply_compile_options(mocktail_test_openxr_runtime) + add_test(NAME OpenXrProbeContract + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/vr/probe_contract_test.py" + "$" "$") + add_test(NAME OpenXrPreviewContract + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/vr/preview_contract_test.py" + "$" "$" + "$") + set_tests_properties(OpenXrPreviewContract PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 90) +else() + add_test(NAME OpenXrProbeDisabled + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/vr/probe_contract_test.py" + "$" --disabled "$") +endif() + add_executable(jnivm_test jnivm_test.cc ) diff --git a/tests/command_line_test.cc b/tests/command_line_test.cc index 75fdfe98..ee69e957 100644 --- a/tests/command_line_test.cc +++ b/tests/command_line_test.cc @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -11,6 +12,52 @@ namespace mocktail { namespace runtime { namespace { +TEST(CommandLineTest, VrOverrideSurvivesReexecAndCanDisableEnvironmentDefault) { + const std::array arguments = {"mocktail", "--vr", + "--headless", "--no-vr"}; + const auto parsed = ParseCommandLine(arguments.size(), arguments.data()); + ASSERT_TRUE(parsed) << parsed.error; + ASSERT_TRUE(parsed.options.vr_enabled.has_value()); + EXPECT_FALSE(*parsed.options.vr_enabled); + std::vector reexec; + std::string error; + ASSERT_TRUE(BuildCommandLineReexecArguments( + parsed.options, arguments.size(), arguments.data(), &reexec, &error)) + << error; + EXPECT_EQ(reexec, + (std::vector{"--vr", "--headless", "--no-vr"})); + + const char* previous = std::getenv("MOCKTAIL_VR_ENABLED"); + const std::optional saved = + previous ? std::optional(previous) : std::nullopt; + const char* previous_headless = std::getenv("MOCKTAIL_HEADLESS"); + const std::optional saved_headless = + previous_headless ? std::optional(previous_headless) + : std::nullopt; + setenv("MOCKTAIL_VR_ENABLED", "true", 1); + EXPECT_TRUE(ApplyCommandLineEnvironment(parsed.options, &error)) << error; + EXPECT_STREQ(std::getenv("MOCKTAIL_VR_ENABLED"), "0"); + if (saved) + setenv("MOCKTAIL_VR_ENABLED", saved->c_str(), 1); + else + unsetenv("MOCKTAIL_VR_ENABLED"); + if (saved_headless) + setenv("MOCKTAIL_HEADLESS", saved_headless->c_str(), 1); + else + unsetenv("MOCKTAIL_HEADLESS"); +} + +TEST(CommandLineTest, ParsesExplicitVrAndKeepsUnspecifiedDistinctFromFalse) { + const char* enabled[] = {"mocktail", "--vr"}; + const auto parsed = ParseCommandLine(2, enabled); + ASSERT_TRUE(parsed) << parsed.error; + EXPECT_EQ(parsed.options.vr_enabled, std::optional(true)); + const char* normal[] = {"mocktail"}; + EXPECT_FALSE(ParseCommandLine(1, normal).options.vr_enabled.has_value()); + const char* incompatible[] = {"mocktail", "--force-run-latest", "--vr"}; + EXPECT_FALSE(ParseCommandLine(3, incompatible)); +} + TEST(CommandLineTest, ParsesCompleteRunContractOnce) { const std::array arguments = { "mocktail", diff --git a/tests/runtime_config_file_test.cc b/tests/runtime_config_file_test.cc index cb4b5552..e4ce0b2e 100644 --- a/tests/runtime_config_file_test.cc +++ b/tests/runtime_config_file_test.cc @@ -99,6 +99,7 @@ TEST(RuntimeConfigBootstrapTest, CreatesCompletePrivateFirstRunFile) { LoadRuntimeConfig(MapEnvironment(), file); ASSERT_TRUE(loaded) << loaded.error; EXPECT_TRUE(loaded.file_loaded); + EXPECT_FALSE(loaded.config.vr_enabled()); EXPECT_EQ(loaded.config.graphics_backend(), GraphicsBackend::kVulkan); EXPECT_EQ(loaded.config.theme_mode(), "roblox"); EXPECT_EQ(loaded.config.frame_rate().mode, FrameRateLimitMode::kUnmanaged); @@ -115,6 +116,58 @@ TEST(RuntimeConfigBootstrapTest, CreatesCompletePrivateFirstRunFile) { EXPECT_EQ(loaded.config.device_profile().name, "pc-windows-11"); } +TEST(RuntimeConfigFileTest, + VrEnvironmentOverridesYamlIncludingExplicitDisable) { + TemporaryDirectory temporary; + const auto file = temporary.Write("version: 1\nvr:\n enabled: true\n"); + const auto enabled = LoadRuntimeConfig(MapEnvironment(), file); + ASSERT_TRUE(enabled) << enabled.error; + EXPECT_TRUE(enabled.config.vr_enabled()); + const auto disabled = + LoadRuntimeConfig(MapEnvironment({{"MOCKTAIL_VR_ENABLED", "0"}}), file); + ASSERT_TRUE(disabled) << disabled.error; + EXPECT_FALSE(disabled.config.vr_enabled()); + const auto invalid = LoadRuntimeConfig( + MapEnvironment({{"MOCKTAIL_VR_ENABLED", "maybe"}}), file); + EXPECT_FALSE(invalid); + EXPECT_NE(invalid.error.find("VR configuration"), std::string::npos); +} + +TEST(RuntimeConfigFileTest, RejectsMalformedVrSectionAndUnsupportedKeys) { + TemporaryDirectory temporary; + for (const auto* yaml : + {"vr: true\n", "vr:\n enabled: perhaps\n", "vr:\n enabled: [true]\n", + "vr:\n enable: true\n", "vr:\n enabled: true\n enabled: false\n"}) { + const auto result = + LoadRuntimeConfig(MapEnvironment(), temporary.Write(yaml)); + EXPECT_FALSE(result) << yaml; + } +} + +TEST(RuntimeConfigFileTest, ExportsResolvedVrStateAndRejectsInvalidState) { + const char* previous = std::getenv("MOCKTAIL_VR_ENABLED"); + const std::optional saved = + previous ? std::optional(previous) : std::nullopt; + std::string error; + const auto enabled = RuntimeConfig::FromEnvironment( + MapEnvironment({{"MOCKTAIL_VR_ENABLED", "on"}})); + EXPECT_TRUE(ExportRuntimeConfigEnvironment(enabled, &error)) << error; + EXPECT_STREQ(std::getenv("MOCKTAIL_VR_ENABLED"), "1"); + EXPECT_TRUE(ExportRuntimeConfigEnvironment( + RuntimeConfig::FromEnvironment(MapEnvironment()), &error)) + << error; + EXPECT_STREQ(std::getenv("MOCKTAIL_VR_ENABLED"), "0"); + EXPECT_FALSE(ExportRuntimeConfigEnvironment( + RuntimeConfig::FromEnvironment( + MapEnvironment({{"MOCKTAIL_VR_ENABLED", "invalid"}})), + &error)); + EXPECT_STREQ(std::getenv("MOCKTAIL_VR_ENABLED"), "0"); + if (saved) + setenv("MOCKTAIL_VR_ENABLED", saved->c_str(), 1); + else + unsetenv("MOCKTAIL_VR_ENABLED"); +} + TEST(RuntimeConfigBootstrapTest, MatchesShippedExampleAndDocumentsEveryDefault) { const std::filesystem::path example = diff --git a/tests/runtime_config_test.cc b/tests/runtime_config_test.cc index 357f2240..74ac2971 100644 --- a/tests/runtime_config_test.cc +++ b/tests/runtime_config_test.cc @@ -37,6 +37,8 @@ TEST(RuntimeConfigTest, UsesSupportedDefaults) { const RuntimeConfig config = RuntimeConfig::FromEnvironment(environment); EXPECT_FALSE(config.headless()); + EXPECT_FALSE(config.vr_enabled()); + EXPECT_TRUE(config.vr_valid()); EXPECT_EQ(config.roblox_library_path(), "rbx_bin/libroblox.so"); EXPECT_EQ(config.graphics_backend(), GraphicsBackend::kVulkan); EXPECT_EQ(config.graphics_backend_name(), "direct-vulkan"); @@ -89,9 +91,17 @@ TEST(RuntimeConfigTest, UsesSupportedDefaults) { EXPECT_TRUE(config.unsafe_detached_thread_overrides().empty()); } +TEST(RuntimeConfigTest, RejectsMalformedVrEnvironmentInsteadOfEnablingIt) { + const auto config = RuntimeConfig::FromEnvironment( + MapEnvironment({{"MOCKTAIL_VR_ENABLED", "maybe"}})); + EXPECT_FALSE(config.vr_valid()); + EXPECT_FALSE(config.vr_enabled()); +} + TEST(RuntimeConfigTest, ReadsTypedRuntimeValues) { const MapEnvironment environment({ {"MOCKTAIL_HEADLESS", "1"}, + {"MOCKTAIL_VR_ENABLED", "true"}, {"ROBLOX_LIB_PATH", "/tmp/libroblox.so"}, {"MOCKTAIL_GRAPHICS_BACKEND", "vulkan"}, {"MOCKTAIL_WIN_WIDTH", "1920"}, @@ -122,6 +132,8 @@ TEST(RuntimeConfigTest, ReadsTypedRuntimeValues) { const RuntimeConfig config = RuntimeConfig::FromEnvironment(environment); EXPECT_TRUE(config.headless()); + EXPECT_TRUE(config.vr_enabled()); + EXPECT_TRUE(config.vr_valid()); EXPECT_EQ(config.roblox_library_path(), "/tmp/libroblox.so"); EXPECT_EQ(config.graphics_backend(), GraphicsBackend::kVulkan); EXPECT_EQ(config.window().width, 1920); diff --git a/tests/vr/fake_openxr_preview.cc b/tests/vr/fake_openxr_preview.cc new file mode 100644 index 00000000..eb9d673f --- /dev/null +++ b/tests/vr/fake_openxr_preview.cc @@ -0,0 +1,508 @@ +// Session contract fixture. Vulkan devices, images, rendering and readback are +// real; OpenXR scheduling/tracking are scripted to exercise error paths. +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +bool Scenario(const char* name) { + const char* value = std::getenv("MOCKTAIL_TEST_XR_SCENARIO"); + return value && std::strcmp(value, name) == 0; +} + +void Trace(const char* event) { + if (const char* path = std::getenv("MOCKTAIL_TEST_XR_TRACE")) { + if (auto* file = std::fopen(path, "a")) { + std::fprintf(file, "%s\n", event); + std::fclose(file); + } + } +} + +#define REQUIRE(value) \ + if (!(value)) { \ + Trace("contract-error: " #value); \ + return XR_ERROR_VALIDATION_FAILURE; \ + } + +struct Fixture { + PFN_vkGetInstanceProcAddr get_instance_proc = nullptr; + PFN_vkGetDeviceProcAddr get_device_proc = nullptr; + VkInstance instance = VK_NULL_HANDLE; + VkPhysicalDevice physical = VK_NULL_HANDLE; + VkDevice device = VK_NULL_HANDLE; + VkQueue queue = VK_NULL_HANDLE; + std::uint32_t queue_family = 0; + std::array images{}; + std::array memory{}; + std::uint32_t width = 0, height = 0, index = 0, frames = 0; + bool running = false, begun = false, acquired = false, waited = false; + bool timeout_returned = false, pixels_verified = false; + int local = 0, view = 0, swapchain = 0; + XrSessionState state = XR_SESSION_STATE_IDLE; + std::deque events; +} fixture; + +XrSession Session() { return reinterpret_cast(&fixture); } +XrSwapchain Swapchain() { + return reinterpret_cast(&fixture.swapchain); +} +XrSpace Local() { return reinterpret_cast(&fixture.local); } +XrSpace View() { return reinterpret_cast(&fixture.view); } + +template +T Device(const char* name) { + return reinterpret_cast(fixture.get_device_proc(fixture.device, name)); +} +#define VK(name, ...) Device("vk" #name)(__VA_ARGS__) + +std::uint32_t MemoryType(std::uint32_t bits, VkMemoryPropertyFlags flags) { + VkPhysicalDeviceMemoryProperties memory{}; + reinterpret_cast( + fixture.get_instance_proc(fixture.instance, + "vkGetPhysicalDeviceMemoryProperties"))( + fixture.physical, &memory); + for (std::uint32_t i = 0; i < memory.memoryTypeCount; ++i) + if ((bits & (1u << i)) && + (memory.memoryTypes[i].propertyFlags & flags) == flags) + return i; + return UINT32_MAX; +} + +// Independently inspect submitted pixels, so a successful xrEndFrame alone +// cannot make an empty or unrendered texture pass the integration test. +bool VerifyPixels() { + struct Resources { + VkBuffer buffer = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + VkCommandPool pool = VK_NULL_HANDLE; + ~Resources() { + VK(DeviceWaitIdle, fixture.device); + if (pool) VK(DestroyCommandPool, fixture.device, pool, nullptr); + if (buffer) VK(DestroyBuffer, fixture.device, buffer, nullptr); + if (memory) VK(FreeMemory, fixture.device, memory, nullptr); + } + } resources; + VkBufferCreateInfo buffer{}; + buffer.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + buffer.size = fixture.width * fixture.height * 4; + buffer.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + if (VK(CreateBuffer, fixture.device, &buffer, nullptr, &resources.buffer) != + VK_SUCCESS) + return false; + VkMemoryRequirements requirements{}; + VK(GetBufferMemoryRequirements, fixture.device, resources.buffer, + &requirements); + VkMemoryAllocateInfo allocate{}; + allocate.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + allocate.allocationSize = requirements.size; + allocate.memoryTypeIndex = MemoryType( + requirements.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + if (allocate.memoryTypeIndex == UINT32_MAX || + VK(AllocateMemory, fixture.device, &allocate, nullptr, + &resources.memory) != VK_SUCCESS || + VK(BindBufferMemory, fixture.device, resources.buffer, resources.memory, + 0) != VK_SUCCESS) + return false; + VkCommandPoolCreateInfo pool{}; + pool.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + pool.queueFamilyIndex = fixture.queue_family; + if (VK(CreateCommandPool, fixture.device, &pool, nullptr, &resources.pool) != + VK_SUCCESS) + return false; + VkCommandBufferAllocateInfo commands{}; + commands.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + commands.commandPool = resources.pool; + commands.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + commands.commandBufferCount = 1; + VkCommandBuffer command = VK_NULL_HANDLE; + if (VK(AllocateCommandBuffers, fixture.device, &commands, &command) != + VK_SUCCESS) + return false; + VkCommandBufferBeginInfo begin{}; + begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + if (VK(BeginCommandBuffer, command, &begin) != VK_SUCCESS) return false; + VkImageMemoryBarrier barrier{}; + barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + barrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = + VK_QUEUE_FAMILY_IGNORED; + barrier.image = fixture.images[fixture.index]; + barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + VK(CmdPipelineBarrier, command, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); + VkBufferImageCopy copy{}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {fixture.width, fixture.height, 1}; + VK(CmdCopyImageToBuffer, command, barrier.image, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, resources.buffer, 1, ©); + barrier.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + barrier.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + VK(CmdPipelineBarrier, command, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, 0, nullptr, 0, nullptr, + 1, &barrier); + if (VK(EndCommandBuffer, command) != VK_SUCCESS) return false; + VkSubmitInfo submit{}; + submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit.commandBufferCount = 1; + submit.pCommandBuffers = &command; + if (VK(QueueSubmit, fixture.queue, 1, &submit, VK_NULL_HANDLE) != + VK_SUCCESS || + VK(QueueWaitIdle, fixture.queue) != VK_SUCCESS) + return false; + void* mapped = nullptr; + if (VK(MapMemory, fixture.device, resources.memory, 0, VK_WHOLE_SIZE, 0, + &mapped) != VK_SUCCESS) + return false; + const auto* bytes = static_cast(mapped); + const auto* cyan = + bytes + (fixture.height / 16 * fixture.width + fixture.width / 16) * 4; + const auto* purple = cyan + fixture.width / 2 * 4; + const bool valid = cyan[1] > 60 && cyan[2] > 80 && cyan[0] < 15 && + purple[0] > 40 && purple[1] < 35 && purple[2] > 80; + VK(UnmapMemory, fixture.device, resources.memory); + if (valid) Trace("pixels-verified"); + return valid; +} + +XrResult XRAPI_CALL +CreateVulkanInstance(XrInstance, const XrVulkanInstanceCreateInfoKHR* info, + VkInstance* instance, VkResult* result) { + fixture.get_instance_proc = info->pfnGetInstanceProcAddr; + *result = reinterpret_cast( + fixture.get_instance_proc(VK_NULL_HANDLE, "vkCreateInstance"))( + info->vulkanCreateInfo, info->vulkanAllocator, instance); + if (*result == VK_SUCCESS) { + fixture.instance = *instance; + fixture.get_device_proc = reinterpret_cast( + fixture.get_instance_proc(*instance, "vkGetDeviceProcAddr")); + } + return XR_SUCCESS; +} + +XrResult XRAPI_CALL +GetVulkanDevice(XrInstance, const XrVulkanGraphicsDeviceGetInfoKHR* info, + VkPhysicalDevice* device) { + const auto enumerate = reinterpret_cast( + fixture.get_instance_proc(info->vulkanInstance, + "vkEnumeratePhysicalDevices")); + std::uint32_t count = 0; + if (enumerate(info->vulkanInstance, &count, nullptr) != VK_SUCCESS || !count) + return XR_ERROR_GRAPHICS_DEVICE_INVALID; + std::vector devices(count); + if (enumerate(info->vulkanInstance, &count, devices.data()) != VK_SUCCESS) + return XR_ERROR_RUNTIME_FAILURE; + *device = fixture.physical = devices.front(); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL CreateVulkanDevice(XrInstance, + const XrVulkanDeviceCreateInfoKHR* info, + VkDevice* device, VkResult* result) { + if (Scenario("device_failure")) { + *result = VK_ERROR_FEATURE_NOT_PRESENT; + return XR_SUCCESS; + } + *result = reinterpret_cast(fixture.get_instance_proc( + fixture.instance, "vkCreateDevice"))(info->vulkanPhysicalDevice, + info->vulkanCreateInfo, + info->vulkanAllocator, device); + if (*result == VK_SUCCESS) fixture.device = *device; + return XR_SUCCESS; +} + +XrResult XRAPI_CALL EnumerateBlends(XrInstance, XrSystemId, + XrViewConfigurationType, + std::uint32_t capacity, + std::uint32_t* count, + XrEnvironmentBlendMode* blends) { + *count = 1; + if (capacity) blends[0] = XR_ENVIRONMENT_BLEND_MODE_OPAQUE; + return XR_SUCCESS; +} + +XrResult XRAPI_CALL CreateSession(XrInstance, const XrSessionCreateInfo* info, + XrSession* session) { + if (Scenario("session_failure")) return XR_ERROR_INITIALIZATION_FAILED; + const auto* binding = + static_cast(info->next); + REQUIRE(binding && binding->type == XR_TYPE_GRAPHICS_BINDING_VULKAN2_KHR); + REQUIRE(binding->device == fixture.device && + binding->physicalDevice == fixture.physical); + fixture.queue_family = binding->queueFamilyIndex; + VK(GetDeviceQueue, fixture.device, binding->queueFamilyIndex, + binding->queueIndex, &fixture.queue); + *session = Session(); + fixture.events.push_back(XR_SESSION_STATE_READY); + Trace("session-create"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL DestroySession(XrSession) { + REQUIRE(fixture.images[0] == VK_NULL_HANDLE); + Trace("session-destroy"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL PollEvent(XrInstance, XrEventDataBuffer* buffer) { + if (fixture.events.empty()) return XR_EVENT_UNAVAILABLE; + XrEventDataSessionStateChanged event{}; + event.type = XR_TYPE_EVENT_DATA_SESSION_STATE_CHANGED; + event.session = Session(); + event.state = fixture.state = fixture.events.front(); + fixture.events.pop_front(); + std::memcpy(buffer, &event, sizeof(event)); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL BeginSession(XrSession, const XrSessionBeginInfo* begin) { + REQUIRE(fixture.state == XR_SESSION_STATE_READY && !fixture.running); + REQUIRE(begin->primaryViewConfigurationType == + XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO); + fixture.running = true; + fixture.events.push_back(XR_SESSION_STATE_FOCUSED); + Trace("session-begin"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL RequestExit(XrSession) { + REQUIRE(fixture.running); + fixture.events.push_back(XR_SESSION_STATE_STOPPING); + Trace("session-exit-request"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL EndSession(XrSession) { + REQUIRE(fixture.state == XR_SESSION_STATE_STOPPING && fixture.running); + fixture.running = false; + Trace("session-end"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL CreateSpace(XrSession, + const XrReferenceSpaceCreateInfo* info, + XrSpace* space) { + REQUIRE(info->poseInReferenceSpace.orientation.w == 1); + REQUIRE(info->referenceSpaceType == XR_REFERENCE_SPACE_TYPE_LOCAL || + info->referenceSpaceType == XR_REFERENCE_SPACE_TYPE_VIEW); + *space = info->referenceSpaceType == XR_REFERENCE_SPACE_TYPE_LOCAL ? Local() + : View(); + Trace("space-create"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL DestroySpace(XrSpace) { + Trace("space-destroy"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL LocateSpace(XrSpace space, XrSpace base, XrTime, + XrSpaceLocation* location) { + REQUIRE(space == View() && base == Local()); + location->locationFlags = Scenario("invalid_tracking") && fixture.frames == 0 + ? 0 + : XR_SPACE_LOCATION_ORIENTATION_VALID_BIT | + XR_SPACE_LOCATION_POSITION_VALID_BIT; + location->pose.orientation.w = 1; + location->pose.position.x = static_cast(fixture.frames) / 10; + return XR_SUCCESS; +} + +XrResult XRAPI_CALL EnumerateFormats(XrSession, std::uint32_t capacity, + std::uint32_t* count, + std::int64_t* formats) { + *count = 1; + if (capacity) formats[0] = VK_FORMAT_R8G8B8A8_UNORM; + return XR_SUCCESS; +} + +XrResult XRAPI_CALL CreateSwapchain(XrSession, + const XrSwapchainCreateInfo* info, + XrSwapchain* swapchain) { + if (Scenario("swapchain_failure")) return XR_ERROR_RUNTIME_FAILURE; + REQUIRE(info->sampleCount == 1 && info->arraySize == 1 && + info->faceCount == 1 && info->mipCount == 1); + fixture.width = info->width; + fixture.height = info->height; + VkImageCreateInfo create{}; + create.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + create.imageType = VK_IMAGE_TYPE_2D; + create.format = static_cast(info->format); + create.extent = {info->width, info->height, 1}; + create.mipLevels = create.arrayLayers = 1; + create.samples = VK_SAMPLE_COUNT_1_BIT; + create.usage = + VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + for (std::size_t i = 0; i < fixture.images.size(); ++i) { + REQUIRE(VK(CreateImage, fixture.device, &create, nullptr, + &fixture.images[i]) == VK_SUCCESS); + VkMemoryRequirements requirements{}; + VK(GetImageMemoryRequirements, fixture.device, fixture.images[i], + &requirements); + VkMemoryAllocateInfo memory{}; + memory.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + memory.allocationSize = requirements.size; + memory.memoryTypeIndex = MemoryType(requirements.memoryTypeBits, 0); + REQUIRE(memory.memoryTypeIndex != UINT32_MAX); + REQUIRE(VK(AllocateMemory, fixture.device, &memory, nullptr, + &fixture.memory[i]) == VK_SUCCESS); + REQUIRE(VK(BindImageMemory, fixture.device, fixture.images[i], + fixture.memory[i], 0) == VK_SUCCESS); + } + *swapchain = Swapchain(); + Trace("swapchain-create"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL DestroySwapchain(XrSwapchain) { + for (std::size_t i = 0; i < fixture.images.size(); ++i) { + if (fixture.images[i]) + VK(DestroyImage, fixture.device, fixture.images[i], nullptr); + if (fixture.memory[i]) + VK(FreeMemory, fixture.device, fixture.memory[i], nullptr); + fixture.images[i] = VK_NULL_HANDLE; + } + Trace("swapchain-destroy"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL EnumerateImages(XrSwapchain, std::uint32_t capacity, + std::uint32_t* count, + XrSwapchainImageBaseHeader* base) { + *count = fixture.images.size(); + if (!capacity) return XR_SUCCESS; + if (capacity < *count) return XR_ERROR_SIZE_INSUFFICIENT; + auto* images = reinterpret_cast(base); + for (std::uint32_t i = 0; i < *count; ++i) { + REQUIRE(images[i].type == XR_TYPE_SWAPCHAIN_IMAGE_VULKAN_KHR); + images[i].image = fixture.images[i]; + } + return XR_SUCCESS; +} + +XrResult XRAPI_CALL WaitFrame(XrSession, const XrFrameWaitInfo*, + XrFrameState* state) { + REQUIRE(fixture.running && !fixture.begun); + state->predictedDisplayTime = (fixture.frames + 1) * 16'666'666; + state->predictedDisplayPeriod = 16'666'666; + state->shouldRender = !(Scenario("skip_render") && fixture.frames == 0); + Trace("frame-wait"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL BeginFrame(XrSession, const XrFrameBeginInfo*) { + REQUIRE(fixture.running && !fixture.begun); + fixture.begun = true; + Trace("frame-begin"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL AcquireImage(XrSwapchain, + const XrSwapchainImageAcquireInfo*, + std::uint32_t* index) { + REQUIRE(fixture.begun && !fixture.acquired); + fixture.acquired = true; + fixture.index = fixture.frames % fixture.images.size(); + *index = Scenario("bad_image_index") ? 1000 : fixture.index; + Trace("image-acquire"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL WaitImage(XrSwapchain, const XrSwapchainImageWaitInfo*) { + REQUIRE(fixture.acquired && !fixture.waited); + if (Scenario("image_timeout") && !fixture.timeout_returned) { + fixture.timeout_returned = true; + Trace("image-timeout"); + return XR_TIMEOUT_EXPIRED; + } + fixture.waited = true; + Trace("image-wait"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL ReleaseImage(XrSwapchain, + const XrSwapchainImageReleaseInfo*) { + REQUIRE(fixture.acquired && fixture.waited); + fixture.acquired = fixture.waited = false; + Trace("image-release"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL EndFrame(XrSession, const XrFrameEndInfo* end) { + REQUIRE(fixture.begun && !fixture.acquired); + REQUIRE(end->displayTime == (fixture.frames + 1) * 16'666'666); + REQUIRE(end->environmentBlendMode == XR_ENVIRONMENT_BLEND_MODE_OPAQUE); + if (end->layerCount) { + REQUIRE(end->layerCount == 1 && + end->layers[0]->type == XR_TYPE_COMPOSITION_LAYER_QUAD); + const auto* layer = + reinterpret_cast(end->layers[0]); + REQUIRE(layer->space == Local() && + layer->eyeVisibility == XR_EYE_VISIBILITY_BOTH); + REQUIRE(layer->pose.position.z == -2 && layer->pose.orientation.w == 1); + REQUIRE(layer->subImage.swapchain == Swapchain()); + if (!fixture.pixels_verified) { + REQUIRE(VerifyPixels()); + fixture.pixels_verified = true; + } + Trace("layer-submit"); + } else { + Trace("empty-submit"); + } + fixture.begun = false; + ++fixture.frames; + if (Scenario("preview_auto_stop") && fixture.frames == 3) + fixture.events.push_back(XR_SESSION_STATE_STOPPING); + if (Scenario("session_loss")) + fixture.events.push_back(XR_SESSION_STATE_LOSS_PENDING); + Trace("frame-end"); + return XR_SUCCESS; +} + +} // namespace + +PFN_xrVoidFunction MocktailTestPreviewFunction(const char* name) { +#define MAP(xr_name, implementation) \ + if (std::strcmp(name, #xr_name) == 0) \ + return reinterpret_cast(implementation); + MAP(xrCreateVulkanInstanceKHR, CreateVulkanInstance) + MAP(xrGetVulkanGraphicsDevice2KHR, GetVulkanDevice) + MAP(xrCreateVulkanDeviceKHR, CreateVulkanDevice) + MAP(xrEnumerateEnvironmentBlendModes, EnumerateBlends) + MAP(xrCreateSession, CreateSession) + MAP(xrDestroySession, DestroySession) + MAP(xrPollEvent, PollEvent) + MAP(xrBeginSession, BeginSession) + MAP(xrRequestExitSession, RequestExit) + MAP(xrEndSession, EndSession) + MAP(xrCreateReferenceSpace, CreateSpace) + MAP(xrDestroySpace, DestroySpace) + MAP(xrLocateSpace, LocateSpace) + MAP(xrEnumerateSwapchainFormats, EnumerateFormats) + MAP(xrCreateSwapchain, CreateSwapchain) + MAP(xrDestroySwapchain, DestroySwapchain) + MAP(xrEnumerateSwapchainImages, EnumerateImages) + MAP(xrWaitFrame, WaitFrame) + MAP(xrBeginFrame, BeginFrame) + MAP(xrEndFrame, EndFrame) + MAP(xrAcquireSwapchainImage, AcquireImage) + MAP(xrWaitSwapchainImage, WaitImage) + MAP(xrReleaseSwapchainImage, ReleaseImage) +#undef MAP + return nullptr; +} diff --git a/tests/vr/fake_openxr_runtime.cc b/tests/vr/fake_openxr_runtime.cc new file mode 100644 index 00000000..fd6c53e3 --- /dev/null +++ b/tests/vr/fake_openxr_runtime.cc @@ -0,0 +1,170 @@ +// Test-only runtime for exercising the real Khronos loader without hardware. +// Implements discovery only; it is neither a compositor nor an XR emulator. +#include +#include +#include +#include + +#include +#include +#include +#include + +PFN_xrVoidFunction MocktailTestPreviewFunction(const char* name); + +namespace { + +bool Scenario(const char* expected) { + const char* value = std::getenv("MOCKTAIL_TEST_XR_SCENARIO"); + return value && std::strcmp(value, expected) == 0; +} + +void Trace(const char* event) { + const char* path = std::getenv("MOCKTAIL_TEST_XR_TRACE"); + if (path) { + if (FILE* file = std::fopen(path, "a")) { + std::fprintf(file, "%s\n", event); + std::fclose(file); + } + } +} + +XrResult XRAPI_CALL EnumerateExtensions(const char*, std::uint32_t capacity, + std::uint32_t* count, + XrExtensionProperties* properties) { + *count = Scenario("no_vulkan") ? 0 : 1; + if (capacity == 0 || *count == 0) return XR_SUCCESS; + if (properties[0].type != XR_TYPE_EXTENSION_PROPERTIES) + return XR_ERROR_VALIDATION_FAILURE; + std::strcpy(properties[0].extensionName, + XR_KHR_VULKAN_ENABLE2_EXTENSION_NAME); + properties[0].extensionVersion = XR_KHR_vulkan_enable2_SPEC_VERSION; + return XR_SUCCESS; +} + +XrResult XRAPI_CALL CreateInstance(const XrInstanceCreateInfo* info, + XrInstance* instance) { + if (Scenario("create_failure")) return XR_ERROR_INITIALIZATION_FAILED; + if (info->type != XR_TYPE_INSTANCE_CREATE_INFO || info->next != nullptr) + return XR_ERROR_VALIDATION_FAILURE; + static int handle; + *instance = reinterpret_cast(&handle); + Trace("create"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL DestroyInstance(XrInstance) { + Trace("destroy"); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL GetInstanceProperties(XrInstance, + XrInstanceProperties* properties) { + if (Scenario("api_error")) return XR_ERROR_RUNTIME_FAILURE; + std::strcpy(properties->runtimeName, "Mocktail discovery fixture"); + properties->runtimeVersion = XR_MAKE_VERSION(1, 0, 42); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL GetSystem(XrInstance, const XrSystemGetInfo* info, + XrSystemId* system) { + if (info->formFactor != XR_FORM_FACTOR_HEAD_MOUNTED_DISPLAY) + return XR_ERROR_FORM_FACTOR_UNSUPPORTED; + if (Scenario("no_headset")) return XR_ERROR_FORM_FACTOR_UNAVAILABLE; + *system = 1; + return XR_SUCCESS; +} + +XrResult XRAPI_CALL GetSystemProperties(XrInstance, XrSystemId, + XrSystemProperties* properties) { + properties->systemId = 1; + std::strcpy(properties->systemName, "Simulated test HMD"); + properties->graphicsProperties = {4096, 4096, 16}; + properties->trackingProperties = {XR_TRUE, XR_TRUE}; + return XR_SUCCESS; +} + +XrResult XRAPI_CALL EnumerateViews(XrInstance, XrSystemId, + XrViewConfigurationType type, + std::uint32_t capacity, std::uint32_t* count, + XrViewConfigurationView* views) { + if (type != XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO) + return XR_ERROR_VIEW_CONFIGURATION_TYPE_UNSUPPORTED; + *count = Scenario("bad_stereo") ? 1 : 2; + if (capacity == 0) return XR_SUCCESS; + static bool grew = false; + if (Scenario("enumeration_changed") && !grew) { + grew = true; + return XR_ERROR_SIZE_INSUFFICIENT; + } + if (capacity < *count) return XR_ERROR_SIZE_INSUFFICIENT; + for (std::uint32_t eye = 0; eye < *count; ++eye) { + if (views[eye].type != XR_TYPE_VIEW_CONFIGURATION_VIEW || views[eye].next) + return XR_ERROR_VALIDATION_FAILURE; + views[eye].recommendedImageRectWidth = 1440; + views[eye].recommendedImageRectHeight = 1600; + views[eye].maxImageRectWidth = 4096; + views[eye].maxImageRectHeight = 4096; + views[eye].recommendedSwapchainSampleCount = 1; + views[eye].maxSwapchainSampleCount = 4; + } + return XR_SUCCESS; +} + +XrResult XRAPI_CALL GetGraphicsRequirements( + XrInstance, XrSystemId, XrGraphicsRequirementsVulkan2KHR* requirements) { + if (Scenario("graphics_error")) return XR_ERROR_RUNTIME_FAILURE; + if (requirements->type != XR_TYPE_GRAPHICS_REQUIREMENTS_VULKAN2_KHR || + requirements->next) + return XR_ERROR_VALIDATION_FAILURE; + requirements->minApiVersionSupported = XR_MAKE_VERSION(1, 1, 0); + requirements->maxApiVersionSupported = XR_MAKE_VERSION(1, 3, 0); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL ResultToString(XrInstance, XrResult result, char* buffer) { + std::snprintf(buffer, XR_MAX_RESULT_STRING_SIZE, "fixture_result_%d", result); + return XR_SUCCESS; +} + +XrResult XRAPI_CALL GetProcAddr(XrInstance, const char* name, + PFN_xrVoidFunction* function) { + *function = nullptr; +#define MOCKTAIL_TEST_XR_PROC(xr_name, implementation) \ + if (std::strcmp(name, #xr_name) == 0) { \ + *function = reinterpret_cast(implementation); \ + return XR_SUCCESS; \ + } + MOCKTAIL_TEST_XR_PROC(xrGetInstanceProcAddr, GetProcAddr) + MOCKTAIL_TEST_XR_PROC(xrEnumerateInstanceExtensionProperties, + EnumerateExtensions) + MOCKTAIL_TEST_XR_PROC(xrCreateInstance, CreateInstance) + MOCKTAIL_TEST_XR_PROC(xrDestroyInstance, DestroyInstance) + MOCKTAIL_TEST_XR_PROC(xrGetInstanceProperties, GetInstanceProperties) + MOCKTAIL_TEST_XR_PROC(xrGetSystem, GetSystem) + MOCKTAIL_TEST_XR_PROC(xrGetSystemProperties, GetSystemProperties) + MOCKTAIL_TEST_XR_PROC(xrEnumerateViewConfigurationViews, EnumerateViews) + MOCKTAIL_TEST_XR_PROC(xrGetVulkanGraphicsRequirements2KHR, + GetGraphicsRequirements) + MOCKTAIL_TEST_XR_PROC(xrResultToString, ResultToString) +#undef MOCKTAIL_TEST_XR_PROC + *function = MocktailTestPreviewFunction(name); + if (*function) return XR_SUCCESS; + return XR_ERROR_FUNCTION_UNSUPPORTED; +} + +} // namespace + +extern "C" __attribute__((visibility("default"))) XrResult XRAPI_CALL +xrNegotiateLoaderRuntimeInterface(const XrNegotiateLoaderInfo* loader, + XrNegotiateRuntimeRequest* request) { + if (loader->structType != XR_LOADER_INTERFACE_STRUCT_LOADER_INFO || + request->structType != XR_LOADER_INTERFACE_STRUCT_RUNTIME_REQUEST || + loader->minInterfaceVersion > XR_CURRENT_LOADER_RUNTIME_VERSION || + loader->maxInterfaceVersion < XR_CURRENT_LOADER_RUNTIME_VERSION) + return XR_ERROR_INITIALIZATION_FAILED; + request->runtimeInterfaceVersion = XR_CURRENT_LOADER_RUNTIME_VERSION; + request->runtimeApiVersion = XR_MAKE_VERSION(1, 0, 42); + request->getInstanceProcAddr = GetProcAddr; + return XR_SUCCESS; +} diff --git a/tests/vr/preview_contract_test.py b/tests/vr/preview_contract_test.py new file mode 100644 index 00000000..ce67dc39 --- /dev/null +++ b/tests/vr/preview_contract_test.py @@ -0,0 +1,141 @@ +"""Real Vulkan rendering/readback with a scripted OpenXR session lifecycle.""" + +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time + + +probe, runtime, mocktail = map(Path, sys.argv[1:4]) +with tempfile.TemporaryDirectory(prefix="mocktail-xr-preview-") as directory: + root = Path(directory) + manifest = root / "runtime.json" + manifest.write_text(json.dumps({"file_format_version": "1.0.0", + "runtime": {"library_path": str(runtime.resolve())}})) + env = {k: v for k, v in os.environ.items() + if not k.startswith(("XR_", "MOCKTAIL_", "VK_"))} + env.update(XR_RUNTIME_JSON=str(manifest), XR_LOADER_DEBUG="none", + XDG_CONFIG_HOME=str(root / "config"), XDG_DATA_HOME=str(root / "data"), + VK_LOADER_LAYERS_DISABLE="~implicit~") + lavapipe = Path("/usr/share/vulkan/icd.d/lvp_icd.json") + if lavapipe.exists(): + env["VK_DRIVER_FILES"] = str(lavapipe) + if Path("/usr/share/vulkan/explicit_layer.d/VkLayer_khronos_validation.json").exists(): + env["VK_INSTANCE_LAYERS"] = "VK_LAYER_KHRONOS_validation" + trace = root / "trace" + env["MOCKTAIL_TEST_XR_TRACE"] = str(trace) + + def check(result, code): + output = result.stdout + result.stderr + assert result.returncode == code, (result.returncode, output) + assert "Validation Error" not in output and "VUID-" not in output, output + events = trace.read_text().splitlines() if trace.exists() else [] + assert not any(event.startswith("contract-error") for event in events), events + assert events.count("create") == events.count("destroy"), events + assert events.count("space-create") == events.count("space-destroy"), events + assert events.count("session-create") == events.count("session-destroy"), events + assert events.count("swapchain-create") == events.count("swapchain-destroy"), events + assert events.count("frame-begin") == events.count("frame-end"), events + assert events.count("image-acquire") == events.count("image-release"), events + if "swapchain-destroy" in events: + assert events.index("swapchain-destroy") < events.index("session-destroy") < events.index("destroy") + return output, events + + cases = [ + ("preview_ready", 0, "rendered: 3, tracked: 3"), + ("skip_render", 0, "rendered: 2, tracked: 3"), + ("invalid_tracking", 0, "rendered: 2, tracked: 2"), + ("image_timeout", 0, "rendered: 3, tracked: 3"), + ("device_failure", 1, "vkCreateDevice failed"), + ("session_failure", 1, "xrCreateSession failed"), + ("swapchain_failure", 1, "xrCreateSwapchain failed"), + ("bad_image_index", 1, "invalid swapchain image index"), + ("session_loss", 1, "OpenXR session was lost"), + ] + for scenario, code, message in cases: + trace.unlink(missing_ok=True) + env["MOCKTAIL_TEST_XR_SCENARIO"] = scenario + result = subprocess.run([str(probe), "--scene", "--frames", "3"], env=env, + capture_output=True, text=True, timeout=15) + if scenario == "preview_ready" and result.returncode != 0: + output = result.stdout + result.stderr + if "Cannot load the host Vulkan loader" in output or "vkCreateInstance failed: -9" in output: + print("SKIP: no Vulkan implementation available for GPU contract checks") + sys.exit(77) + output, events = check(result, code) + assert message in output, (scenario, output) + if code == 0: + assert "pixels-verified" in events, (scenario, events) + assert events.count("session-exit-request") == events.count("session-end") == 1 + assert "Head position (metres): 0.2, 0, 0" in output, output + if scenario in ("skip_render", "invalid_tracking", "bad_image_index"): + assert events.count("empty-submit") == 1, events + if scenario == "image_timeout": + assert events.count("image-timeout") == 1, events + print(scenario + ": passed") + + # VR must traverse the real Roblox startup, never fall back to this scene. + # Exercise configuration/CLI precedence with a deliberately missing payload. + for scenario, yaml_value, switches, env_value in [ + ("yaml", "true", [], None), + ("cli", "false", ["--vr"], "0"), + ("environment", "false", [], "1"), + ]: + case = root / scenario + config = case / "config" + config.mkdir(parents=True) + contents = f"version: 1\nvr:\n enabled: {yaml_value}\n" + (config / "config.yaml").write_text(contents) + env.update(MOCKTAIL_CONFIG_ROOT=str(config), MOCKTAIL_DATA_ROOT=str(case / "data"), + MOCKTAIL_STATE_ROOT=str(case / "state"), MOCKTAIL_TEST_XR_SCENARIO="preview_auto_stop") + if env_value is None: + env.pop("MOCKTAIL_VR_ENABLED", None) + else: + env["MOCKTAIL_VR_ENABLED"] = env_value + trace.unlink(missing_ok=True) + result = subprocess.run([str(mocktail), "--headless", "--roblox-lib", + str(case / "missing-libroblox.so"), *switches], + env=env, capture_output=True, text=True, timeout=15) + output = result.stdout + result.stderr + assert result.returncode != 0, (scenario, output) + assert "missing-libroblox.so" in output, (scenario, output) + assert not trace.exists(), (scenario, trace.read_text() if trace.exists() else "") + assert (config / "config.yaml").read_text() == contents + print("Mocktail VR rejects missing Roblox via " + scenario + ": passed") + + env["MOCKTAIL_TEST_XR_SCENARIO"] = "preview_ready" + trace.unlink(missing_ok=True) + child = subprocess.Popen([str(probe), "--scene"], env=env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + try: + deadline = time.monotonic() + 10 + while not trace.exists() or "layer-submit" not in trace.read_text(): + assert child.poll() is None and time.monotonic() < deadline, "Scene failed to start for interrupt test" + time.sleep(0.01) + child.send_signal(signal.SIGINT) + stdout, stderr = child.communicate(timeout=10) + _, events = check(subprocess.CompletedProcess(child.args, child.returncode, stdout, stderr), 0) + assert "session-exit-request" in events and "session-end" in events, events + finally: + if child.poll() is None: + child.kill() + child.wait() + print("SIGINT session shutdown: passed") + + for count in ("0", "-1", "abc", "4294967296", "1x"): + result = subprocess.run([str(probe), "--scene", "--frames", count], env=env, + capture_output=True, text=True, timeout=5) + assert result.returncode == 1 and "positive integer" in result.stderr, (count, result) + print("frame limit validation: passed") + + env["XR_RUNTIME_JSON"] = str(root / "missing-runtime.json") + result = subprocess.run([str(probe), "--scene", "--frames", "1"], env=env, + capture_output=True, text=True, timeout=5) + assert result.returncode == 1, result + assert "XR_ERROR_RUNTIME_UNAVAILABLE" in result.stdout, result.stdout + assert "mocktail-vr-simulated" in result.stdout, result.stdout + print("missing runtime diagnostic offers simulator: passed") diff --git a/tests/vr/probe_contract_test.py b/tests/vr/probe_contract_test.py new file mode 100644 index 00000000..9cff3d7b --- /dev/null +++ b/tests/vr/probe_contract_test.py @@ -0,0 +1,82 @@ +"""Run the actual OpenXR loader against a discovery-only test runtime.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile + + +probe = Path(sys.argv[1]) +if sys.argv[2] == "--disabled": + result = subprocess.run([str(probe)], text=True, capture_output=True, timeout=15) + assert result.returncode == 2, (result.returncode, result.stdout, result.stderr) + assert "does not include OpenXR support" in result.stdout, result.stdout + result = subprocess.run([str(probe), "--scene", "--frames", "1"], + text=True, capture_output=True, timeout=15) + assert result.returncode == 1 and "does not include OpenXR support" in result.stdout, result + with tempfile.TemporaryDirectory(prefix="mocktail-xr-disabled-") as directory: + root = Path(directory) + env = {k: v for k, v in os.environ.items() if not k.startswith(("XR_", "MOCKTAIL_"))} + for kind in ("CONFIG", "DATA", "STATE"): + env["MOCKTAIL_" + kind + "_ROOT"] = str(root / kind.lower()) + result = subprocess.run([sys.argv[3], "--headless", "--vr"], env=env, + text=True, capture_output=True, timeout=15) + assert result.returncode == 1 and "does not include OpenXR support" in result.stderr, result + assert not (root / "data").exists() and not (root / "state").exists() + print("OpenXR-disabled build: passed") + sys.exit(0) + +runtime = Path(sys.argv[2]) +with tempfile.TemporaryDirectory(prefix="mocktail-xr-probe-") as directory: + root = Path(directory) + manifest = root / "runtime.json" + manifest.write_text(json.dumps({ + "file_format_version": "1.0.0", + "runtime": {"library_path": str(runtime.resolve())}, + })) + environment = {k: v for k, v in os.environ.items() + if not k.startswith(("XR_", "MOCKTAIL_"))} + environment["XR_RUNTIME_JSON"] = str(manifest) + environment["XR_LOADER_DEBUG"] = "none" + environment["XDG_CONFIG_HOME"] = str(root / "config") + environment["XDG_DATA_HOME"] = str(root / "data") + environment["XR_API_LAYER_PATH"] = str(root / "layers") + (root / "layers").mkdir() + trace = root / "trace" + environment["MOCKTAIL_TEST_XR_TRACE"] = str(trace) + + cases = [ + ("ready", 0, "discovered successfully"), + ("no_headset", 4, "Connect a headset"), + ("no_vulkan", 5, "does not expose XR_KHR_vulkan_enable2"), + ("bad_stereo", 5, "two usable stereo views"), + ("api_error", 1, "xrGetInstanceProperties failed"), + ("create_failure", 1, "xrCreateInstance failed"), + ("enumeration_changed", 0, "discovered successfully"), + ("graphics_error", 1, "xrGetVulkanGraphicsRequirements2KHR failed"), + ] + for scenario, code, message in cases: + trace.unlink(missing_ok=True) + environment["MOCKTAIL_TEST_XR_SCENARIO"] = scenario + result = subprocess.run([str(probe)], env=environment, + text=True, capture_output=True, timeout=15) + assert result.returncode == code, (scenario, result.returncode, result.stdout, result.stderr) + assert message in result.stdout, (scenario, result.stdout, result.stderr) + assert "SDK: 1.1.63" in result.stdout, result.stdout + assert "Roblox VR rendering: not implemented yet" in result.stdout + events = trace.read_text().splitlines() if trace.exists() else [] + assert events == ([] if scenario == "create_failure" else ["create", "destroy"]), (scenario, events) + if code == 0: + assert "Eye 0: 1440x1600" in result.stdout + assert "Eye 1: 1440x1600" in result.stdout + assert "minimum=1.1.0" in result.stdout + print(scenario + ": passed") + + environment["XR_RUNTIME_JSON"] = str(root / "does-not-exist.json") + result = subprocess.run([str(probe)], env=environment, + text=True, capture_output=True, timeout=15) + assert result.returncode == 3, (result.returncode, result.stdout, result.stderr) + assert "runtime" in result.stdout.lower(), result.stdout + print("missing runtime: passed") diff --git a/tests/vr/simulator_launcher_test.py b/tests/vr/simulator_launcher_test.py new file mode 100644 index 00000000..3a43bc19 --- /dev/null +++ b/tests/vr/simulator_launcher_test.py @@ -0,0 +1,89 @@ +"""Check process-local runtime selection and preparation independently of VR hardware.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile + + +launcher = Path(sys.argv[1]).resolve() +with tempfile.TemporaryDirectory(prefix="mocktail-simulator-launcher-") as directory: + root = Path(directory) + build = root / "build with spaces" + build.mkdir() + source = root / "source" + source.mkdir() + library = source / "runtime.so" + library.write_bytes(b"runtime fixture for preparation only") + manifest = source / "manifest.json" + manifest.write_text(json.dumps({"runtime": {"library_path": "./runtime.so"}})) + command = [sys.executable, str(launcher), "--build-dir", str(build)] + result = subprocess.run([*command, "--prepare", str(manifest)], + capture_output=True, text=True, timeout=10) + assert result.returncode == 0, result.stderr + prepared = build / "vr-simulator/runtime.json" + document = json.loads(prepared.read_text()) + relative_library = Path(document["runtime"]["library_path"]) + assert not relative_library.is_absolute() + assert (prepared.parent / relative_library).read_bytes() == library.read_bytes() + + # Moving the prepared build must not depend on the original /tmp/source path. + relocated = root / "relocated build" + build.rename(relocated) + build = relocated + prepared = build / "vr-simulator/runtime.json" + command = [sys.executable, str(launcher), "--build-dir", str(build)] + probe = build / "mocktail-vr-probe" + probe.write_text("""#!/usr/bin/env python3 +import json, os, sys +names = ('XR_RUNTIME_JSON', 'SIMULATED_ENABLE', 'XRT_COMPOSITOR_NULL', + 'XRT_COMPOSITOR_FORCE_XCB', 'XRT_COMPOSITOR_FORCE_WAYLAND', + 'XDG_CONFIG_HOME', 'XDG_RUNTIME_DIR') +print(json.dumps({'args': sys.argv[1:], 'env': {key: os.environ.get(key) for key in names}})) +sys.exit(7) +""") + probe.chmod(0o755) + env = os.environ.copy() + env.update(DISPLAY=":123", WAYLAND_DISPLAY="wayland-test", + XR_RUNTIME_JSON=str(root / "real-headset.json"), + XRT_COMPOSITOR_NULL="1", XRT_COMPOSITOR_FORCE_WAYLAND="1", + XDG_CONFIG_HOME=str(root / "user-config"), XDG_RUNTIME_DIR=str(root / "desktop-runtime")) + original_selection = env["XR_RUNTIME_JSON"] + original_config = Path(env["XDG_CONFIG_HOME"]) + original_config.mkdir() + marker = original_config / "keep" + marker.write_text("preserve user settings") + + for graphical_backend in ("xcb", "wayland"): + if graphical_backend == "wayland": + env.pop("DISPLAY") + result = subprocess.run([*command, "--frames", "4"], env=env, + capture_output=True, text=True, timeout=10) + assert result.returncode == 7, (result.stdout, result.stderr) + child = json.loads(result.stdout.splitlines()[-1]) + assert child["args"] == ["--scene", "--frames", "4"] + selected = child["env"] + assert selected["XR_RUNTIME_JSON"] == str(prepared) + assert selected["SIMULATED_ENABLE"] == "1" and selected["XRT_COMPOSITOR_NULL"] == "0" + assert selected["XRT_COMPOSITOR_FORCE_XCB"] == ("1" if graphical_backend == "xcb" else "0") + assert selected["XRT_COMPOSITOR_FORCE_WAYLAND"] == ("0" if graphical_backend == "xcb" else "1") + assert selected["XDG_CONFIG_HOME"] != str(original_config) + assert selected["XDG_RUNTIME_DIR"] == env["XDG_RUNTIME_DIR"] + assert env["XR_RUNTIME_JSON"] == original_selection + assert marker.read_text() == "preserve user settings" + assert not (original_config / "openxr").exists() + print(graphical_backend + " process-local selection: passed") + + env.pop("WAYLAND_DISPLAY") + result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=10) + assert result.returncode == 2 and "graphical desktop" in result.stderr, result + result = subprocess.run([*command, "--runtime", str(root / "missing.json")], env=env, + capture_output=True, text=True, timeout=10) + assert result.returncode == 2 and "No simulated Monado runtime" in result.stderr, result + for count in ("0", "-1", "4294967296"): + result = subprocess.run([*command, "--frames", count], env=env, + capture_output=True, text=True, timeout=10) + assert result.returncode == 2 and "positive 32-bit integer" in result.stderr, result + print("missing runtime/display and frame limit diagnostics: passed") diff --git a/third_party/OpenXR-SDK b/third_party/OpenXR-SDK new file mode 160000 index 00000000..f2448a87 --- /dev/null +++ b/third_party/OpenXR-SDK @@ -0,0 +1 @@ +Subproject commit f2448a8797c85814aa892efc1ab8707900fbcc78 From b187efbedff0acebe2e3aa905fdadf7bf9425fc1 Mon Sep 17 00:00:00 2001 From: komaruworld Date: Sun, 13 Sep 2026 18:20:58 +0300 Subject: [PATCH 02/58] Full support camera NOT TESTED --- CMakeLists.txt | 3 + cmake/MocktailVR.cmake | 35 +- config/mocktail.example.yaml | 16 +- config/roblox_compatibility.json | 11 +- include/compat/build_profile.h | 17 + include/mocktail/vr/openxr_backend.h | 286 +++ include/mocktail/vr/openxr_probe.h | 4 + include/mocktail/vr/roblox_vr_device_bridge.h | 113 + include/mocktail/vr/vr_mirror.h | 105 + include/mocktail/vr/vr_mirror_shm.h | 42 + include/mocktail/vr/vr_perf.h | 97 + include/mocktail/vr/vr_pose_math.h | 171 ++ include/runtime/performance_policy.h | 5 + include/runtime/runtime_config.h | 7 +- scripts/vr_simulated.py | 187 -- src/compat/build_profile.cc | 56 +- src/graphics/bionic_vulkan_loader_adapter.cc | 394 ++- src/legacy/legacy_runtime.cc | 15 + src/main.cc | 69 +- src/runtime/command_line.cc | 7 +- src/runtime/performance_policy.cc | 22 + src/runtime/runtime_config.cc | 2 +- src/runtime/runtime_config_bootstrap.cc | 16 +- src/vr/openxr_backend.cc | 2194 +++++++++++++++++ src/vr/openxr_backend_mode.cc | 53 + src/vr/openxr_disabled.cc | 12 + src/vr/openxr_preview.cc | 3 +- src/vr/openxr_probe.cc | 2 + src/vr/roblox_vr_device_bridge.cc | 885 +++++++ src/vr/roblox_vr_device_bridge_internal.h | 82 + src/vr/vr_mirror.cc | 268 ++ src/vr/vr_perf.cc | 220 ++ tests/CMakeLists.txt | 60 +- tests/command_line_test.cc | 14 + tests/performance_policy_test.cc | 16 + tests/runtime_config_file_test.cc | 6 +- tests/runtime_config_test.cc | 4 +- tests/vr/openxr_backend_test.cc | 191 ++ tests/vr/preview_contract_test.py | 4 +- tests/vr/roblox_vr_device_bridge_test.cc | 503 ++++ tests/vr/simulator_launcher_test.py | 89 - tests/vr/vr_evidence_layer.cc | 1005 ++++++++ tests/vr/vr_evidence_layer_test.cc | 131 + tests/vr/vr_mirror_test.cc | 200 ++ tests/vr/vr_pose_math_test.cc | 187 ++ 45 files changed, 7476 insertions(+), 333 deletions(-) create mode 100644 include/mocktail/vr/openxr_backend.h create mode 100644 include/mocktail/vr/roblox_vr_device_bridge.h create mode 100644 include/mocktail/vr/vr_mirror.h create mode 100644 include/mocktail/vr/vr_mirror_shm.h create mode 100644 include/mocktail/vr/vr_perf.h create mode 100644 include/mocktail/vr/vr_pose_math.h delete mode 100644 scripts/vr_simulated.py create mode 100644 src/vr/openxr_backend.cc create mode 100644 src/vr/openxr_backend_mode.cc create mode 100644 src/vr/roblox_vr_device_bridge.cc create mode 100644 src/vr/roblox_vr_device_bridge_internal.h create mode 100644 src/vr/vr_mirror.cc create mode 100644 src/vr/vr_perf.cc create mode 100644 tests/vr/openxr_backend_test.cc create mode 100644 tests/vr/roblox_vr_device_bridge_test.cc delete mode 100644 tests/vr/simulator_launcher_test.py create mode 100644 tests/vr/vr_evidence_layer.cc create mode 100644 tests/vr/vr_evidence_layer_test.cc create mode 100644 tests/vr/vr_mirror_test.cc create mode 100644 tests/vr/vr_pose_math_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index a060d5bf..60c27c1d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -305,6 +305,9 @@ target_include_directories(mocktail_runtime PUBLIC include) target_include_directories(mocktail_runtime PUBLIC ${MOCKTAIL_JNI_INCLUDE_DIRS} ) +target_compile_definitions(mocktail_runtime PUBLIC + MOCKTAIL_VR_DEFAULT_ENABLED=$ +) target_compile_definitions(mocktail_runtime PRIVATE "MOCKTAIL_DISCORD_APPLICATION_ID=\"${MOCKTAIL_DISCORD_APPLICATION_ID}\"" ) diff --git a/cmake/MocktailVR.cmake b/cmake/MocktailVR.cmake index 5c7f4b62..f71f7cb5 100644 --- a/cmake/MocktailVR.cmake +++ b/cmake/MocktailVR.cmake @@ -3,11 +3,23 @@ include_guard(GLOBAL) -option(MOCKTAIL_ENABLE_VR "Build experimental native OpenXR support" OFF) +option(MOCKTAIL_ENABLE_VR "Build experimental native OpenXR support (VR branch default)" ON) -add_library(mocktail_vr STATIC src/vr/diagnostics.cc) +add_library(mocktail_vr STATIC + src/vr/diagnostics.cc + src/vr/openxr_backend_mode.cc + src/vr/vr_perf.cc + src/vr/vr_mirror.cc + src/vr/roblox_vr_device_bridge.cc) add_library(Mocktail::VR ALIAS mocktail_vr) target_include_directories(mocktail_vr PUBLIC "${CMAKE_SOURCE_DIR}/include") +target_include_directories(mocktail_vr PRIVATE "${CMAKE_SOURCE_DIR}/src") +target_link_libraries(mocktail_vr PUBLIC Vulkan::Headers) +# shm_open for the desktop eye mirror lives in librt on glibc. +find_library(MOCKTAIL_RT_LIBRARY rt) +if(MOCKTAIL_RT_LIBRARY) + target_link_libraries(mocktail_vr PUBLIC ${MOCKTAIL_RT_LIBRARY}) +endif() mocktail_apply_compile_options(mocktail_vr) if(MOCKTAIL_ENABLE_VR) @@ -34,11 +46,12 @@ if(MOCKTAIL_ENABLE_VR) "${CMAKE_BINARY_DIR}/third_party/OpenXR-SDK" EXCLUDE_FROM_ALL) endfunction() mocktail_add_openxr_sdk() - target_sources(mocktail_vr PRIVATE src/vr/openxr_probe.cc src/vr/openxr_preview.cc) - # Error unwinding is confined to this native session implementation; its - # public entry point catches errors before returning to the guest runtime. - set_source_files_properties(src/vr/openxr_preview.cc PROPERTIES - COMPILE_OPTIONS -fexceptions) + target_sources(mocktail_vr PRIVATE src/vr/openxr_probe.cc src/vr/openxr_preview.cc + src/vr/openxr_backend.cc) + # Error unwinding is confined to these native session implementations; their + # public entry points catch errors before returning to the guest runtime. + set_source_files_properties(src/vr/openxr_preview.cc src/vr/openxr_backend.cc + PROPERTIES COMPILE_OPTIONS -fexceptions) target_compile_definitions(mocktail_vr PRIVATE XR_USE_GRAPHICS_API_VULKAN) target_link_libraries(mocktail_vr PRIVATE OpenXR::openxr_loader Vulkan::Headers ${CMAKE_DL_LIBS}) else() @@ -50,11 +63,3 @@ set_target_properties(mocktail_vr_probe PROPERTIES OUTPUT_NAME mocktail-vr-probe target_link_libraries(mocktail_vr_probe PRIVATE Mocktail::VR) mocktail_apply_compile_options(mocktail_vr_probe) install(TARGETS mocktail_vr_probe RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") - -# Development launcher for an independently built Monado runtime. --prepare -# keeps its shared library and a relative manifest under this build directory. -configure_file("${CMAKE_SOURCE_DIR}/scripts/vr_simulated.py" - "${CMAKE_BINARY_DIR}/mocktail-vr-simulated" COPYONLY) -file(CHMOD "${CMAKE_BINARY_DIR}/mocktail-vr-simulated" - PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE - WORLD_READ WORLD_EXECUTE) diff --git a/config/mocktail.example.yaml b/config/mocktail.example.yaml index aa10b5a8..b1514705 100644 --- a/config/mocktail.example.yaml +++ b/config/mocktail.example.yaml @@ -49,12 +49,16 @@ graphics: # Optional presentation synchronization override: auto, on, or off. # vsync: off -vr: - # Boolean (default: false): launch the Roblox experimental VR path. - # Only Build ID 2998 is currently accepted; eye resources and OpenXR output - # are NOT yet connected. Standalone scene: mocktail-vr-probe --scene. - # Use --no-vr to launch Roblox normally. - enabled: false +# vr: + # Boolean (default: enabled in VR builds, disabled in non-VR builds). + # Experimental VR for the exact Build ID 2998 + # payload. Mocktail initializes the guest DebugDeviceVR eye resources (two + # distinct 500x500 eye framebuffers), the Roblox renderer draws both eyes + # every frame, and the frames are submitted to the active OpenXR runtime as + # a stereo projection layer with head pose (runtime-tracked, or scripted for + # desktop testing). Controllers are NOT connected yet. Standalone OpenXR + # test scene: mocktail-vr-probe --scene. Use --no-vr to launch normally. + # enabled: false performance: # Boolean (default: false): size Roblox scheduler/render queues from every diff --git a/config/roblox_compatibility.json b/config/roblox_compatibility.json index 67ff302b..40883da3 100644 --- a/config/roblox_compatibility.json +++ b/config/roblox_compatibility.json @@ -81,7 +81,16 @@ "current_method_rva": "0x32d0f30", "select_method_rva": "0x32d0cb4" }, - "reason": "Verified 2026-09-06: exact Build-ID profile passed two isolated no-recovery Tier C Vulkan canaries and real game sessions, including fullscreen state synchronization and host FMOD output-device interposition; legacy executable patches and signal recovery remain disabled." + "vr_debug_device_bridge": { + "vtable_rva": "0x6c87b38", + "constructor_rva": "0x3810614", + "state_getter_rva": "0x38109f6", + "eye_getter_rva": "0x38109e6", + "eye_initializer_rva": "0x3810a18", + "emulator_flag_storage_rva": "0x7369920", + "device_create_framebuffer_vtable_offset": "0x180" + }, + "reason": "Verified 2026-09-06: exact Build-ID profile passed two isolated no-recovery Tier C Vulkan canaries and real game sessions, including fullscreen state synchronization and host FMOD output-device interposition; legacy executable patches and signal recovery remain disabled. Experimental vr_debug_device_bridge metadata (DebugDeviceVR constructor/state/eye/initializer RVAs) is machine-validated at activation and gated behind --vr; native stereo output is not yet verified end-to-end." } ] } diff --git a/include/compat/build_profile.h b/include/compat/build_profile.h index 40e3f2ac..54836623 100644 --- a/include/compat/build_profile.h +++ b/include/compat/build_profile.h @@ -24,6 +24,21 @@ struct FmodOutputDeviceBridgeProfile { std::uintptr_t select_method_rva = 0; }; +// Exact-build metadata for the experimental native VR bridge. The bridge +// interposes only DebugDeviceVR virtual slots (state getter and eye +// framebuffer getter) and invokes the guest eye-resource initializer through +// its own vtable; executable guest code is never modified. All RVAs are +// validated against machine-code contracts before the first use. +struct VrDebugDeviceBridgeProfile { + std::uintptr_t vtable_rva = 0; + std::uintptr_t constructor_rva = 0; + std::uintptr_t state_getter_rva = 0; + std::uintptr_t eye_getter_rva = 0; + std::uintptr_t eye_initializer_rva = 0; + std::uintptr_t emulator_flag_storage_rva = 0; + std::uintptr_t device_create_framebuffer_vtable_offset = 0; +}; + struct BuildProfile { std::string version_name; int version_code = 0; @@ -41,6 +56,8 @@ struct BuildProfile { // output-device virtual slots and restores them during controlled teardown; // executable guest code is never modified. std::optional fmod_output_device_bridge; + // Optional exact-build experimental VR capability; see the struct above. + std::optional vr_debug_device_bridge; std::string reason; }; diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h new file mode 100644 index 00000000..bed4428e --- /dev/null +++ b/include/mocktail/vr/openxr_backend.h @@ -0,0 +1,286 @@ +#ifndef MOCKTAIL_VR_OPENXR_BACKEND_H_ +#define MOCKTAIL_VR_OPENXR_BACKEND_H_ + +#include + +#include +#include +#include +#include +#include +#include + +#include "mocktail/status.h" +#include "mocktail/vr/vr_mirror.h" + +namespace mocktail::vr { + +// Resolved --vr output mode. kXrOutput drives the real Roblox stereo images +// into an OpenXR projection layer; kNativeOnly keeps the diagnostic native +// stereo prototype without any runtime requirement. Selected through +// MOCKTAIL_VR_BACKEND ("xr" default, "native" diagnostic). +enum class VrBackendMode { + kDisabled, + kNativeOnly, + kXrOutput, +}; + +// Explicit XR_RUNTIME_JSON wins. Otherwise prefer a native WiVRn manifest, +// then let the OpenXR loader use the registered runtime (including Flatpak WiVRn). +std::string SelectVrRuntimeManifest(const std::string& explicit_manifest, + const std::vector& candidates); +std::string VrRuntimeUnavailableHint(bool user_selected, const std::string& manifest); +VrBackendMode ResolveVrBackendMode(bool vr_enabled); +const char* VrBackendModeName(VrBackendMode mode); + +// One deterministic head pose sample. Position is metres, orientation is a +// quaternion (x, y, z, w) in the OpenXR LOCAL reference space. +struct ScriptedPoseSample { + float position[3] = {0.f, 0.f, 0.f}; + float orientation[4] = {0.f, 0.f, 0.f, 1.f}; + float eye_offset[2][3] = {}; // metres in head space + float eye_fov[2][4] = {}; // left, right, up, down angles + std::uint64_t frame = 0; + bool valid = false; +}; + +// Production OpenXR backend for the native stereo prototype. +// +// Lifecycle: main arms the backend before Roblox startup (xrCreateInstance +// and xrGetSystem fail fast with a clear message when no runtime exists). +// The Vulkan adapter delegates the guest's vkCreateInstance/vkCreateDevice +// through xrCreateVulkanInstanceKHR/xrCreateVulkanDeviceKHR so the runtime's +// extension and GPU requirements are satisfied by the very device Roblox +// renders with. The session is created on that device; per-frame XR work runs +// on the render thread from the adapter's host-present notification: +// xrEndFrame(N) with the two eye images of frame N copied into the +// projection swapchains, then xrWaitFrame/xrBeginFrame/xrLocateViews for +// frame N+1 and publishing the scripted/runtime head pose that the device +// bridge injects into Roblox before its eye passes run. +// +// Eye image provenance is recorded in the adapter during the guest eye +// initializer (owner = DebugDeviceVR object) and bound to an eye index only +// through the real per-frame eye-getter request plus the actual render-pass +// attachment, mirroring the validated evidence-layer semantics. Creation +// order or size alone never selects an eye. +// +// All public entry points are exception-free at their boundary; the +// implementation is compiled with exceptions like openxr_preview.cc and +// translates failures into Status/log outcomes. +class OpenXrBackend final { + public: + OpenXrBackend() = default; + ~OpenXrBackend(); + OpenXrBackend(const OpenXrBackend&) = delete; + OpenXrBackend& operator=(const OpenXrBackend&) = delete; + + // Claims the process-wide backend slot and creates XrInstance/system. + Status Arm(); + // Idempotent full teardown (XR resources first, then the instance). + void Disarm(); + bool armed() const { return armed_.load(std::memory_order_acquire); } + + // Adapter delegation. Return false when the backend does not own creation + // (caller falls back to the plain host loader path). + bool CreateVulkanInstance(const VkInstanceCreateInfo* create_info, + const VkAllocationCallbacks* allocator, + VkInstance* instance, VkResult* result); + bool CreateVulkanDevice(VkPhysicalDevice physical_device, + const VkDeviceCreateInfo* create_info, + const VkAllocationCallbacks* allocator, + VkDevice* device, VkResult* result); + void NoteDeviceDestroyed(VkDevice device); + void NoteInstanceDestroyed(VkInstance instance); + void NoteQueue(VkDevice device, VkQueue queue, std::uint32_t family, + std::uint32_t index); + + // Resource provenance recording (adapter wrappers). + bool WantsResourceRecords() const { + return recording_.load(std::memory_order_acquire); + } + void RecordImage(VkDevice device, VkImage image, + const VkImageCreateInfo* info); + void RecordImageView(VkImageView view, VkImage image); + void RecordFramebuffer(VkFramebuffer framebuffer, + const VkFramebufferCreateInfo* info); + void RecordRenderPass(VkRenderPass render_pass, + const VkRenderPassCreateInfo* info); + void NoteDestroyImage(VkImage image); + void NoteDestroyImageView(VkImageView view); + void NoteDestroyFramebuffer(VkFramebuffer framebuffer); + void NoteRenderPassBegin(VkCommandBuffer command_buffer, + const VkRenderPassBeginInfo* info); + bool EyeBindingComplete() const { + return eyes_bound_.load(std::memory_order_acquire); + } + + // Render-thread frame cycle at host present begin. + void NoteHostPresent(VkQueue queue, VkDevice device); + + // Pose published for the frame that is about to render. The device bridge + // reads it once per frame on the render thread and injects it into the + // guest. valid=false until the XR session located a usable head pose. + ScriptedPoseSample PublishedHeadPose() const; + void NotePoseApplied(void* owner, std::uint64_t frame); + + // Diagnostics. + std::uint64_t submitted_frames() const { + return submitted_frames_.load(std::memory_order_relaxed); + } + std::uint64_t skipped_frames() const { + return skipped_frames_.load(std::memory_order_relaxed); + } + bool session_running() const { + return session_running_.load(std::memory_order_relaxed); + } + std::string runtime_name() const; + + private: + friend struct OpenXrBackendTestAccess; + struct ImageRecord { + VkImage image = VK_NULL_HANDLE; + VkDevice device = VK_NULL_HANDLE; + std::uint32_t width = 0; + std::uint32_t height = 0; + VkFormat format = VK_FORMAT_UNDEFINED; + VkImageUsageFlags usage = 0; + void* owner = nullptr; + std::uint64_t order = 0; + }; + struct EyeBinding { + bool valid = false; + std::uint64_t frame = 0; + VkImage image = VK_NULL_HANDLE; + VkFramebuffer framebuffer = VK_NULL_HANDLE; + VkImageLayout final_layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + void* owner = nullptr; + std::uint32_t width = 0; + std::uint32_t height = 0; + VkFormat format = VK_FORMAT_UNDEFINED; + }; + struct EyeSwapchain { + void* swapchain = nullptr; // XrSwapchain (opaque to keep the header XR-free) + std::vector images; + std::uint32_t acquired_index = 0; + bool image_acquired = false; + std::uint32_t width = 0; + std::uint32_t height = 0; + VkFormat format = VK_FORMAT_UNDEFINED; + }; + + bool InitializeSessionLocked(VkPhysicalDevice physical_device, + VkDevice device, + const VkDeviceCreateInfo* create_info, + std::string* error); + void DisarmLocked(); + void* ResolveFunction(const char* name) const; + void SetupDeviceAfterCreationLocked(VkPhysicalDevice physical_device, + VkDevice device, + const VkDeviceCreateInfo* create_info); + void TeardownSessionLocked(const char* reason, bool preserve_device = false); + void HandleSessionState(int state); + void HandleReferenceSpaceChange(std::int64_t change_time); + bool RecoverSessionLocked(std::uint64_t now_ns); + void PollSessionEvents(); + // Drops the pose/images of the frame currently in flight without touching + // guest ownership, so the next xrEndFrame submits no layer instead of an + // image paired with a pose that no longer describes the same origin. + void InvalidateInFlightPose(const char* reason); + void RunFrameCycle(VkQueue queue, VkDevice device); + bool CreateSwapchains(std::string* error); + void DestroySwapchains(); + bool EnsureCopyResources(VkDevice device, std::uint32_t queue_family, + std::string* error); + void DestroyCopyResources(); + // Records both eye copies (and, when the mirror is enabled, both staging + // readbacks) into one command buffer, submits once and waits on one fence. + // Replaces the previous per-eye submit+fence pair. + bool CopyEyesIntoSwapchains(VkQueue queue); + bool CaptureEyeEvidence(VkQueue queue, int eye); + ScriptedPoseSample ComputeScriptedPose(std::uint64_t frame) const; + void EnsureMirrorForExtent(std::uint32_t width, std::uint32_t height); + MirrorVkProcs MirrorProcs() const; + + mutable std::mutex mutex_; + std::atomic armed_{false}; + std::atomic recording_{false}; + std::atomic eyes_bound_{false}; + std::atomic session_running_{false}; + std::atomic submitted_frames_{0}; + std::atomic skipped_frames_{0}; + + // XR handles are stored as opaque pointers in the header; the implementation + // casts them back. This keeps vulkan-only consumers free of OpenXR headers. + void* xr_instance_ = nullptr; + std::uint64_t xr_system_ = 0; + void* xr_session_ = nullptr; + void* xr_local_space_ = nullptr; + void* xr_view_space_ = nullptr; + EyeSwapchain eye_swapchains_[2]; + int xr_blend_mode_ = 0; + // Runtime name captured at xrCreateInstance so the bind-time diagnostics line + // can report it without re-entering runtime_name() under the held mutex. + std::string runtime_name_; + + VkInstance vk_instance_ = VK_NULL_HANDLE; + VkDevice vk_device_ = VK_NULL_HANDLE; + VkPhysicalDevice vk_physical_device_ = VK_NULL_HANDLE; + std::uint32_t vk_queue_family_ = 0; + void* vk_loader_ = nullptr; + struct VkProcs; + VkProcs* vk_ = nullptr; + + VkCommandPool copy_pool_ = VK_NULL_HANDLE; + VkCommandBuffer copy_command_ = VK_NULL_HANDLE; + VkFence copy_fence_ = VK_NULL_HANDLE; + + // Optional desktop two-eye mirror (MOCKTAIL_VR_MIRROR_SHM). Publishes the + // latest complete pair through shared memory; inert when unset. + VrMirror mirror_; + + std::unordered_map images_; + std::unordered_map image_views_; + std::unordered_map> framebuffer_views_; + std::unordered_map framebuffer_render_pass_; + std::unordered_map> render_pass_final_layouts_; + std::unordered_map queue_families_; + std::uint64_t image_order_ = 0; + EyeBinding eyes_[2]; + + // Frame cycle state (render thread only). + bool frame_open_ = false; + bool frame_should_render_ = false; + // A running session is hidden in SYNCHRONIZED; shouldRender still governs rendering. + bool visibility_lost_ = false; + bool session_recovery_pending_ = false; + bool test_session_loss_injected_ = false; + std::uint64_t next_session_retry_ns_ = 0; + std::uint64_t frame_display_time_ = 0; + struct ViewState { + float position[3]; + float orientation[4]; + float fov[4]; // angleLeft, angleRight, angleUp, angleDown (radians) + bool valid; + }; + ViewState frame_views_[2] = {}; + struct PoseStorage { + float position[3] = {0.f, 0.f, 0.f}; + float orientation[4] = {0.f, 0.f, 0.f, 1.f}; + }; + PoseStorage frame_view_poses_[2]; + bool frame_views_valid_ = false; + // One-shot guard so a canted-view runtime reports the rejection reason once + // instead of once per frame per eye. + bool canted_rejection_logged_ = false; + mutable ScriptedPoseSample published_pose_; + std::uint64_t xr_frame_counter_ = 0; + std::unordered_map applied_poses_; +}; + +// Process-wide single backend owned by main. Adapter hooks resolve these +// exported C symbols; they are inert while no backend is armed. +OpenXrBackend* ActiveVrBackend(); + +} // namespace mocktail::vr + +#endif // MOCKTAIL_VR_OPENXR_BACKEND_H_ diff --git a/include/mocktail/vr/openxr_probe.h b/include/mocktail/vr/openxr_probe.h index c406ad2b..8eb76818 100644 --- a/include/mocktail/vr/openxr_probe.h +++ b/include/mocktail/vr/openxr_probe.h @@ -43,6 +43,10 @@ ProbeReport ProbeOpenXr(); std::string FormatProbeReport(const ProbeReport& report); int ProbeExitCode(ProbeStatus status); +// False in builds configured without MOCKTAIL_ENABLE_VR; --vr fails closed +// early in that case instead of starting Roblox without VR support. +bool IsOpenXrSupportCompiled(); + } // namespace mocktail::vr #endif // MOCKTAIL_VR_OPENXR_PROBE_H_ diff --git a/include/mocktail/vr/roblox_vr_device_bridge.h b/include/mocktail/vr/roblox_vr_device_bridge.h new file mode 100644 index 00000000..dddc0c42 --- /dev/null +++ b/include/mocktail/vr/roblox_vr_device_bridge.h @@ -0,0 +1,113 @@ +#ifndef MOCKTAIL_VR_ROBLOX_VR_DEVICE_BRIDGE_H_ +#define MOCKTAIL_VR_ROBLOX_VR_DEVICE_BRIDGE_H_ + +#include +#include +#include + +#include "compat/build_profile.h" +#include "mocktail/status.h" + +namespace mocktail::vr { +namespace internal { +struct VrBridgeTestAccess; +} + +// Experimental native stereo bring-up for the exact Build-ID-scoped +// RBX::Graphics::DebugDeviceVR object of the guest client. +// +// The bridge interposes the state and eye getters of the exact guest build. +// Resource readiness is checked on the live object during each state call; +// initialization never dereferences cached guest object/device pointers. The +// guest owns resource destruction. Recreated objects and reused addresses +// therefore need no cache invalidation. Initialization happens only on a thread +// observed presenting. OpenXR image submission and poses are not connected yet. +class RobloxVrDeviceBridge final { +public: + using StateGetterFn = void *(*)(void *result_buffer, void *device_object); + using EyeGetterFn = void *(*)(void *device_object, int eye_index); + using EyeInitializerFn = void (*)(void *device_object, void *graphics_device); + + RobloxVrDeviceBridge() = default; + ~RobloxVrDeviceBridge(); + + RobloxVrDeviceBridge(const RobloxVrDeviceBridge &) = delete; + RobloxVrDeviceBridge &operator=(const RobloxVrDeviceBridge &) = delete; + + // Arms the bridge for an exact-build profile that carries + // vr_debug_device_bridge metadata. Claims the process-wide bridge slot. + Status Install(const compat::BuildProfile &profile); + + // Validates machine contracts at the loaded image base and interposes the + // two DebugDeviceVR vtable slots. Called once, after libroblox relocations + // and before any guest startup code runs. + Status Activate(std::uintptr_t image_base); + + // Restores the interposed vtable slots and disarms. Safe to call twice. + void Shutdown(); + + bool installed() const { return installed_; } + bool active() const { return active_.load(std::memory_order_acquire); } + + // Render-thread notification from the Vulkan adapter's host present path. + void NoteHostPresent(); + + // Guest-side hooks invoked by the interposed vtable stubs. + void OnStateGetterCall(void *result_buffer, void *device_object); + void OnEyeGetterCall(void *device_object, int eye_index, void *framebuffer); + + StateGetterFn original_state_getter() const { + return original_state_getter_.load(std::memory_order_acquire); + } + EyeGetterFn original_eye_getter() const { + return original_eye_getter_.load(std::memory_order_acquire); + } + + // Cumulative initialization diagnostics (not a count of live devices). + std::uintptr_t image_base() const { + return image_base_.load(std::memory_order_acquire); + } + int captured_device_count() const; + int initialized_device_count() const; + std::uint64_t present_count() const { + return present_count_.load(std::memory_order_relaxed); + } + std::uint64_t eye_getter_call_count(int eye_index) const; + +private: + friend struct internal::VrBridgeTestAccess; + static constexpr std::uint64_t kEyeGetterSummaryInterval = 600; + bool ValidateObjectHeader(void *self) const; + void *ValidatedGraphicsDevice(void *self) const; + void TryInitializeEyes(void *self); + bool RestoreVtableLocked(); + + mutable std::mutex mutex_; + compat::VrDebugDeviceBridgeProfile profile_{}; + bool installed_ = false; + std::atomic active_{false}; + std::atomic image_base_{0}; + std::atomic image_size_{0}; + std::uintptr_t *vtable_ = nullptr; + std::uintptr_t original_state_slot_ = 0; + std::uintptr_t original_eye_slot_ = 0; + std::atomic original_state_getter_{nullptr}; + std::atomic original_eye_getter_{nullptr}; + std::atomic present_thread_{0}; + std::atomic present_count_{0}; + std::atomic flag_warning_logged_{false}; + std::atomic captured_count_{0}; + std::atomic initialized_count_{0}; + mutable std::atomic eye_call_totals_[2] = {}; +}; + +// Process-wide single active bridge, mirroring the audio output-device +// bridge ownership model. Install claims the slot; the destructor releases +// it. legacy_runtime forwards the loaded libroblox image base through +// NotifyRobloxVrImageLoaded; it is a successful no-op while no bridge is +// armed, and fail-closed once one is. +Status NotifyRobloxVrImageLoaded(std::uintptr_t image_base); + +} // namespace mocktail::vr + +#endif // MOCKTAIL_VR_ROBLOX_VR_DEVICE_BRIDGE_H_ diff --git a/include/mocktail/vr/vr_mirror.h b/include/mocktail/vr/vr_mirror.h new file mode 100644 index 00000000..4201ffaf --- /dev/null +++ b/include/mocktail/vr/vr_mirror.h @@ -0,0 +1,105 @@ +#ifndef MOCKTAIL_VR_MIRROR_H_ +#define MOCKTAIL_VR_MIRROR_H_ + +#include + +#include +#include + +#include "mocktail/vr/vr_mirror_shm.h" + +namespace mocktail::vr { + +// The subset of device/instance procs the mirror needs, supplied by the XR +// backend from its already-resolved guest-device table. Keeping this explicit +// avoids a second proc table and keeps the mirror independently testable. +struct MirrorVkProcs { + PFN_vkCreateBuffer CreateBuffer = nullptr; + PFN_vkDestroyBuffer DestroyBuffer = nullptr; + PFN_vkGetBufferMemoryRequirements GetBufferMemoryRequirements = nullptr; + PFN_vkAllocateMemory AllocateMemory = nullptr; + PFN_vkFreeMemory FreeMemory = nullptr; + PFN_vkBindBufferMemory BindBufferMemory = nullptr; + PFN_vkMapMemory MapMemory = nullptr; + PFN_vkUnmapMemory UnmapMemory = nullptr; + PFN_vkCmdCopyImageToBuffer CmdCopyImageToBuffer = nullptr; + PFN_vkCmdPipelineBarrier CmdPipelineBarrier = nullptr; + PFN_vkGetPhysicalDeviceMemoryProperties GetPhysicalDeviceMemoryProperties = + nullptr; +}; + +// Producer side of the desktop two-eye mirror. Maps a POSIX shared-memory +// segment (created with the name from MOCKTAIL_VR_MIRROR_SHM) and publishes +// the latest complete eye pair under a seqlock. It owns two persistent, +// bounded host-visible staging buffers reused across frames; the GPU->staging +// copies are recorded into the caller's single batched command buffer so the +// mirror adds no extra queue submission or fence wait. +// +// The mirror never touches the filesystem on the frame path and keeps only the +// newest pair (one latest slot), so a slow consumer drops rather than queues +// old frames. +class VrMirror { + public: + VrMirror() = default; + ~VrMirror(); + VrMirror(const VrMirror&) = delete; + VrMirror& operator=(const VrMirror&) = delete; + + bool enabled() const { return mapping_ != nullptr; } + + // Opens the shared segment. Returns false (and stays disabled) when the name + // is empty or mapping fails; the XR path then runs without a mirror. + bool Open(const std::string& shm_name, std::uint32_t width, + std::uint32_t height, std::string* error); + void Close(); + + // Ensures the two staging buffers match width*height*4 bytes on this device. + // Allocates once; a size/device change recreates them. Existing contents are + // discarded on resize. + bool EnsureStaging(const MirrorVkProcs& vk, VkDevice device, + VkPhysicalDevice physical, std::uint32_t width, + std::uint32_t height, std::string* error); + void DestroyStaging(const MirrorVkProcs& vk); + + // Records a source-eye -> staging-buffer copy for one eye into cmd, adding + // the layout barriers around it. The source is returned to final_layout so + // the next guest render pass sees it unchanged. + void RecordEyeCopy(const MirrorVkProcs& vk, VkCommandBuffer cmd, int eye, + VkImage source, VkImageLayout final_layout, + std::uint32_t width, std::uint32_t height); + + // After the batched fence wait completed, copies both staging buffers into + // the shared segment and bumps the seqlock with the frame id and a monotonic + // timestamp. No-op when staging does not cover width*height. + void Publish(std::uint64_t frame, VkFormat format, std::uint32_t width, + std::uint32_t height, std::uint64_t now_ns); + + std::uint64_t published_pairs() const { return published_pairs_; } + std::uint32_t staging_width() const { return staging_width_; } + std::uint32_t staging_height() const { return staging_height_; } + const void* mapping() const { return mapping_; } + + // Test seam: publish a pair from caller-supplied staging bytes without any + // Vulkan resources, exercising the same seqlock/segment write as the GPU + // path. Both buffers must hold width*height*4 bytes. + void PublishFromBytesForTest(std::uint64_t frame, VkFormat format, + std::uint32_t width, std::uint32_t height, + std::uint64_t now_ns, const void* eye0, + const void* eye1); + + private: + void* mapping_ = nullptr; + std::uint64_t mapping_bytes_ = 0; + std::string shm_name_; + VkBuffer staging_[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE}; + VkDeviceMemory staging_memory_[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE}; + void* staging_map_[2] = {nullptr, nullptr}; + std::uint32_t staging_width_ = 0; + std::uint32_t staging_height_ = 0; + VkDevice staging_device_ = VK_NULL_HANDLE; + std::uint64_t published_pairs_ = 0; +}; + +} // namespace mocktail::vr + +#endif // MOCKTAIL_VR_MIRROR_H_ diff --git a/include/mocktail/vr/vr_mirror_shm.h b/include/mocktail/vr/vr_mirror_shm.h new file mode 100644 index 00000000..db842a73 --- /dev/null +++ b/include/mocktail/vr/vr_mirror_shm.h @@ -0,0 +1,42 @@ +#ifndef MOCKTAIL_VR_MIRROR_SHM_H_ +#define MOCKTAIL_VR_MIRROR_SHM_H_ + +#include + +namespace mocktail::vr { + +// Shared-memory transport for the desktop two-eye mirror. The producer (XR +// backend, inside the client process) publishes the latest complete eye pair +// under a seqlock; the consumer (desktop viewer) reads the newest pair without +// any per-frame filesystem transport, image encode/decode, or allocation. +// +// Layout is fixed and little-endian. +// sizeof(VrMirrorHeader) == kVrMirrorHeaderBytes. +struct VrMirrorHeader { + std::uint32_t magic; // kVrMirrorMagic + std::uint32_t header_bytes; // kVrMirrorHeaderBytes (pixel area offset) + std::uint64_t seq; // seqlock: odd while writing, even when stable + std::uint64_t frame; // XR frame id the pair belongs to + std::uint32_t width; // eye width in pixels + std::uint32_t height; // eye height in pixels + std::uint32_t format; // VkFormat of the source eye images + std::uint32_t flags; // bit0: a complete, valid pair is present + std::uint64_t producer_ns; // CLOCK_MONOTONIC publish timestamp + std::uint64_t published_pairs; + std::uint64_t reserved; +}; + +inline constexpr std::uint32_t kVrMirrorMagic = 0x3152564D; // 'MVR1' +inline constexpr std::uint32_t kVrMirrorHeaderBytes = 64; +inline constexpr std::uint32_t kVrMirrorFlagValid = 1u << 0; + +// Total mapped bytes for a given eye extent (header + two RGBA/BGRA images). +inline std::uint64_t VrMirrorBytes(std::uint32_t width, std::uint32_t height) { + return static_cast(kVrMirrorHeaderBytes) + + 2ull * static_cast(width) * + static_cast(height) * 4ull; +} + +} // namespace mocktail::vr + +#endif // MOCKTAIL_VR_MIRROR_SHM_H_ diff --git a/include/mocktail/vr/vr_perf.h b/include/mocktail/vr/vr_perf.h new file mode 100644 index 00000000..6fe8ef66 --- /dev/null +++ b/include/mocktail/vr/vr_perf.h @@ -0,0 +1,97 @@ +#ifndef MOCKTAIL_VR_PERF_H_ +#define MOCKTAIL_VR_PERF_H_ + +#include +#include +#include + +namespace mocktail::vr::perf { + +// Opt-in, low-overhead stage timing and counter instrumentation. Collection is +// compiled out at runtime unless MOCKTAIL_VR_PERF is set, so the steady-state +// path only pays a single cached boolean test per record. +bool Enabled(); + +enum class Stage { + kFrameCycle, + kXrWaitFrame, + kXrBeginFrame, + kXrLocateViews, + kSwapchainAcquire, + kSwapchainWait, + kCopyRecord, + kCopySubmit, + kCopyFenceWait, + kMirrorPublish, + kEvidence, + kStageCount, +}; + +enum class Counter { + kPresents, + kFramesBegun, + kFramesSubmitted, + kFramesSkipped, + kSkipShouldRender, + kSkipPoseInvalid, + kSkipEyesUnbound, + kSkipEyeFrameMismatch, + kSubmits, + kFenceWaits, + kMirrorPairs, + kMirrorDrops, + kSkipVisibilityLost, + kReferenceSpaceChanges, + kVisibilityLost, + kVisibilityGained, + kFramesInvalidated, + kCounterCount, +}; + +const char* StageName(Stage stage); +const char* CounterName(Counter counter); + +// Bounded reservoir per stage: the most recent N samples are kept and ranked +// only when a report is emitted (once per interval), never on the frame path. +class Collector { + public: + static constexpr std::size_t kReservoir = 4096; + static constexpr std::uint64_t kIntervalNs = 1'000'000'000ULL; + + void Record(Stage stage, std::uint64_t nanos); + void Count(Counter counter, std::uint64_t delta = 1); + + // Emits one distribution line per active stage plus a counter line when at + // least kIntervalNs elapsed since the previous emission. Cheap no-op + // otherwise. now_ns is a CLOCK_MONOTONIC timestamp supplied by the caller so + // the collector stays clock-agnostic and testable. + void EmitIfDue(std::uint64_t now_ns, const char* context); + + // Test/inspection accessors. + std::uint64_t counter(Counter counter_id) const; + std::vector Percentiles(Stage stage) const; // p50,p95,p99,max + std::size_t Samples(Stage stage) const; + void Reset(); + + private: + struct StageStat { + std::vector samples; // ring, bounded by kReservoir + std::size_t head = 0; + std::size_t count = 0; + std::uint64_t total_ns = 0; + }; + StageStat stages_[static_cast(Stage::kStageCount)]{}; + std::uint64_t counters_[static_cast(Counter::kCounterCount)]{}; + std::uint64_t last_emit_ns_ = 0; + std::uint64_t window_start_ns_ = 0; +}; + +// Process-wide collector used by the XR backend frame cycle. +Collector& ProcessCollector(); + +// Monotonic nanoseconds (CLOCK_MONOTONIC), 0 on failure. +std::uint64_t NowNs(); + +} // namespace mocktail::vr::perf + +#endif // MOCKTAIL_VR_PERF_H_ diff --git a/include/mocktail/vr/vr_pose_math.h b/include/mocktail/vr/vr_pose_math.h new file mode 100644 index 00000000..e66b143c --- /dev/null +++ b/include/mocktail/vr/vr_pose_math.h @@ -0,0 +1,171 @@ +#ifndef MOCKTAIL_VR_POSE_MATH_H_ +#define MOCKTAIL_VR_POSE_MATH_H_ + +#include +#include + +namespace mocktail::vr::pose { + +// Coordinate spaces and units used across the whole camera path, documented so +// every stage is auditable: +// +// OpenXR head/views metres, right-handed, +Y up, -Z forward, LOCAL space +// -> published pose metres, quaternion (x, y, z, w), unchanged units +// -> guest state metres; the exact-build native helper converts metres +// to studs itself (factor 10/3). This layer must never +// pre-apply that scale. +// -> camera engine composes +// out.rotation = base.rotation * head.rotation +// out.position = base.position +// + base.rotation * head.position * HeadScale +// where HeadScale is an engine value distinct from the +// metres-to-studs conversion factor. +// -> render one pass per eye using the per-eye offset/FOV +// -> XR submission metres again, the very pose used to render the pair. + +struct Vec3 { + float x = 0.f, y = 0.f, z = 0.f; +}; + +struct Quat { + float x = 0.f, y = 0.f, z = 0.f, w = 1.f; +}; + +struct Pose { + Vec3 position{}; + Quat orientation{}; +}; + +inline constexpr float kPi = 3.14159265358979323846f; + +inline Quat QuatMultiply(const Quat& a, const Quat& b) { + return Quat{ + a.w * b.x + a.x * b.w + a.y * b.z - a.z * b.y, + a.w * b.y - a.x * b.z + a.y * b.w + a.z * b.x, + a.w * b.z + a.x * b.y - a.y * b.x + a.z * b.w, + a.w * b.w - a.x * b.x - a.y * b.y - a.z * b.z, + }; +} + +inline Quat QuatConjugate(const Quat& q) { return Quat{-q.x, -q.y, -q.z, q.w}; } + +inline float QuatNorm(const Quat& q) { + return std::sqrt(q.x * q.x + q.y * q.y + q.z * q.z + q.w * q.w); +} + +// Returns false for a non-finite or near-zero quaternion; the output is left +// untouched in that case so callers never publish a fabricated rotation. +inline bool QuatNormalize(const Quat& in, Quat* out) { + if (out == nullptr) { + return false; + } + if (!std::isfinite(in.x) || !std::isfinite(in.y) || !std::isfinite(in.z) || + !std::isfinite(in.w)) { + return false; + } + const float norm = QuatNorm(in); + if (!(norm > 1e-6f) || !std::isfinite(norm)) { + return false; + } + const float inverse = 1.f / norm; + *out = Quat{in.x * inverse, in.y * inverse, in.z * inverse, in.w * inverse}; + return true; +} + +// q and -q denote the same rotation. Used to prove the path is sign-invariant. +inline bool QuatSameRotation(const Quat& a, const Quat& b, float tolerance) { + const float dot = a.x * b.x + a.y * b.y + a.z * b.z + a.w * b.w; + return std::abs(std::abs(dot) - 1.f) <= tolerance; +} + +inline Vec3 QuatRotate(const Quat& q, const Vec3& v) { + // t = 2 * cross(q.xyz, v); v' = v + q.w * t + cross(q.xyz, t) + const Vec3 u{q.x, q.y, q.z}; + const Vec3 cross1{u.y * v.z - u.z * v.y, u.z * v.x - u.x * v.z, + u.x * v.y - u.y * v.x}; + const Vec3 t{2.f * cross1.x, 2.f * cross1.y, 2.f * cross1.z}; + const Vec3 cross2{u.y * t.z - u.z * t.y, u.z * t.x - u.x * t.z, + u.x * t.y - u.y * t.x}; + return Vec3{v.x + q.w * t.x + cross2.x, v.y + q.w * t.y + cross2.y, + v.z + q.w * t.z + cross2.z}; +} + +// result = a * b in SE(3): apply b first, then a. +inline Pose PoseMultiply(const Pose& a, const Pose& b) { + Pose result; + result.orientation = QuatMultiply(a.orientation, b.orientation); + const Vec3 rotated = QuatRotate(a.orientation, b.position); + result.position = Vec3{a.position.x + rotated.x, a.position.y + rotated.y, + a.position.z + rotated.z}; + return result; +} + +inline Pose PoseInverse(const Pose& p) { + Pose result; + result.orientation = QuatConjugate(p.orientation); + const Vec3 neg{-p.position.x, -p.position.y, -p.position.z}; + result.position = QuatRotate(result.orientation, neg); + return result; +} + +// Rotation for a head turned by yaw about +Y then pitched about +X, matching +// the scripted diagnostic motion. Quaternion order is (x, y, z, w). +inline Quat QuatFromYawPitch(float yaw, float pitch) { + const float cy = std::cos(yaw * 0.5f), sy = std::sin(yaw * 0.5f); + const float cp = std::cos(pitch * 0.5f), sp = std::sin(pitch * 0.5f); + return Quat{cy * sp, sy * cp, -sy * sp, cy * cp}; +} + +inline Quat QuatFromAxisAngle(const Vec3& axis, float angle) { + const float half = angle * 0.5f; + const float s = std::sin(half); + return Quat{axis.x * s, axis.y * s, axis.z * s, std::cos(half)}; +} + +// Per-eye FOV as the runtime reports it: angleLeft, angleRight, angleUp, +// angleDown in radians, each signed relative to the eye's forward axis. +struct EyeFov { + float angle_left = 0.f; + float angle_right = 0.f; + float angle_up = 0.f; + float angle_down = 0.f; +}; + +// The guest state stores projection tangents, not angles. The 2998 layout is +// up, down, left, right in that order; down/left are negated because the +// guest's tangent space measures them in the opposite direction. Getting this +// order or a sign wrong flips or shears the image, so it is pinned by test. +struct ProjectionTangents { + float up = 0.f; + float down = 0.f; + float left = 0.f; + float right = 0.f; +}; + +inline ProjectionTangents FovToTangents(const EyeFov& fov) { + return ProjectionTangents{std::tan(fov.angle_up), std::tan(-fov.angle_down), + std::tan(-fov.angle_left), std::tan(fov.angle_right)}; +} + +// The pinned guest state can only represent two parallel eye orientations. +// A canted (rotated) relative eye pose needs a different camera ABI, so it is +// rejected explicitly rather than silently submitted with a projection that +// does not match how the image was rendered. +inline constexpr float kCantedTolerance = 0.001f; + +inline bool EyeOrientationIsParallel(const Quat& relative_orientation) { + return std::abs(relative_orientation.x) <= kCantedTolerance && + std::abs(relative_orientation.y) <= kCantedTolerance && + std::abs(relative_orientation.z) <= kCantedTolerance; +} + +// Metres-to-studs factor the exact-build native helper applies itself. Exposed +// only so tests can prove this layer does NOT pre-apply it. +inline constexpr float kMetresToStuds = 10.f / 3.f; + +// Expected stud value for a metre offset, for numerical unit checks. +inline float MetresToStuds(float metres) { return metres * kMetresToStuds; } + +} // namespace mocktail::vr::pose + +#endif // MOCKTAIL_VR_POSE_MATH_H_ diff --git a/include/runtime/performance_policy.h b/include/runtime/performance_policy.h index 44894eb4..9d038305 100644 --- a/include/runtime/performance_policy.h +++ b/include/runtime/performance_policy.h @@ -62,6 +62,11 @@ bool MergeRuntimeClientSettingsOverrides(const FrameRatePolicy& frame_rate, std::string* merged_json, std::string* error); +// The resolved VR mode owns this flag in both client-settings channels. +bool MergeVrClientSettingsOverrides(bool enabled, std::string_view base_json, + std::string* merged_json, + std::string* error); + // Keeps the legacy test bypass disabled. The host PermissionsProtocol now // answers microphone authorization from capture configuration. No permission // query opens a recording device. diff --git a/include/runtime/runtime_config.h b/include/runtime/runtime_config.h index faa6f5c9..948c6f3b 100644 --- a/include/runtime/runtime_config.h +++ b/include/runtime/runtime_config.h @@ -15,6 +15,11 @@ namespace mocktail { namespace runtime { +#ifndef MOCKTAIL_VR_DEFAULT_ENABLED +#define MOCKTAIL_VR_DEFAULT_ENABLED 0 +#endif +inline constexpr bool kDefaultVrEnabled = MOCKTAIL_VR_DEFAULT_ENABLED != 0; + enum class GraphicsBackend { kAuto, kSystem, @@ -137,7 +142,7 @@ class RuntimeConfig { private: bool headless_ = false; - bool vr_enabled_ = false; + bool vr_enabled_ = kDefaultVrEnabled; bool vr_valid_ = true; std::filesystem::path roblox_library_path_ = "rbx_bin/libroblox.so"; GraphicsBackend graphics_backend_ = GraphicsBackend::kVulkan; diff --git a/scripts/vr_simulated.py b/scripts/vr_simulated.py deleted file mode 100644 index 0d4c9580..00000000 --- a/scripts/vr_simulated.py +++ /dev/null @@ -1,187 +0,0 @@ -#!/usr/bin/env python3 -"""Launch the VR preview with a separately built Monado simulated runtime.""" - -import argparse -import json -import os -from pathlib import Path -import select -import shutil -import signal -import subprocess -import sys -import tempfile - - -def positive_integer(value): - number = int(value) - if not 0 < number <= 0xFFFFFFFF: - raise argparse.ArgumentTypeError("must be a positive 32-bit integer") - return number - - -def prepare_runtime(source, destination): - """Keep the development runtime beside the build, independent of /tmp.""" - source = source.resolve() - document = json.loads(source.read_text()) - library = Path(document["runtime"]["library_path"]) - if not library.is_absolute(): - library = source.parent / library - if not library.is_file(): - raise ValueError(f"Runtime library does not exist: {library}") - destination.mkdir(parents=True, exist_ok=True) - output_library = destination / "libopenxr_monado.so" - if library.resolve() != output_library.resolve(): - shutil.copy2(library, output_library) - output_manifest = destination / "runtime.json" - output_manifest.write_text(json.dumps({ - "file_format_version": "1.0.0", - "runtime": {"name": "Monado (Mocktail simulator)", - "library_path": "./libopenxr_monado.so"}, - }, indent=2) + "\n") - print(f"Prepared simulator: {output_manifest}") - - -def simulator_environment(parent, manifest, display, wayland, software): - environment = parent.copy() - # These overrides belong to this child process; no active_runtime.json is - # written and the user's WiVRn/system runtime selection remains intact. - environment.update( - XR_RUNTIME_JSON=str(manifest.resolve()), - SIMULATED_ENABLE="1", - XRT_COMPOSITOR_NULL="0", - XRT_COMPOSITOR_FORCE_XCB="1" if display else "0", - XRT_COMPOSITOR_FORCE_WAYLAND="1" if not display and wayland else "0", - XRT_COMPOSITOR_FORCE_NVIDIA="0", - XRT_COMPOSITOR_FORCE_RANDR="0", - XRT_COMPOSITOR_FORCE_WAYLAND_DIRECT="0", - XRT_COMPOSITOR_FORCE_VK_DISPLAY="-1", - XRT_COMPOSITOR_XCB_FULLSCREEN="0", - ) - environment.setdefault("XRT_COMPOSITOR_DEFAULT_FRAMERATE", "30") - if display: - environment["DISPLAY"] = display - environment.pop("WAYLAND_DISPLAY", None) - if software: - candidates = [directory / "lvp_icd.json" for directory in - (Path("/usr/share/vulkan/icd.d"), Path("/usr/local/share/vulkan/icd.d"))] - manifest_path = next((path for path in candidates if path.is_file()), None) - if manifest_path is None: - raise ValueError("Mesa Lavapipe is not installed; omit --software to use your GPU") - environment["VK_DRIVER_FILES"] = str(manifest_path) - environment.pop("VK_ICD_FILENAMES", None) - environment["XRT_COMPOSITOR_FORCE_GPU_INDEX"] = "0" - environment["XRT_COMPOSITOR_FORCE_CLIENT_GPU_INDEX"] = "0" - return environment - - -def main(): - location = Path(__file__).resolve().parent - default_build = location.parent / "build" if location.name == "scripts" else location - parser = argparse.ArgumentParser( - description="Show the Mocktail VR test scene on the desktop using a simulated Monado headset.") - parser.add_argument("--build-dir", type=Path, default=default_build, - help="Mocktail build directory (defaults to the directory of this launcher)") - parser.add_argument("--runtime", type=Path, - help="Manifest of an in-process Monado build with the simulated driver enabled") - parser.add_argument("--prepare", type=Path, metavar="MONADO_MANIFEST", - help="Copy an already built Monado runtime next to this build, then exit") - parser.add_argument("--frames", type=positive_integer, - help="Request exit after this many frames; otherwise run until Ctrl+C") - parser.add_argument("--software", action="store_true", help="Use Mesa Lavapipe instead of a physical GPU") - parser.add_argument("--headless", action="store_true", - help="Use a private Xvfb display for automated checks (defaults to 180 frames)") - args = parser.parse_args() - build = args.build_dir.resolve() - if args.prepare: - prepare_runtime(args.prepare, build / "vr-simulator") - return 0 - manifest = args.runtime or build / "vr-simulator/runtime.json" - if not manifest.is_file(): - parser.error("No simulated Monado runtime is prepared. Use --runtime /path/to/openxr_monado-dev.json " - "or --prepare /path/to/openxr_monado-dev.json with a separately built Monado runtime.") - probe = build / "mocktail-vr-probe" - if not probe.is_file(): - parser.error(f"Build mocktail_vr_probe with MOCKTAIL_ENABLE_VR=ON first: {probe}") - display = os.environ.get("DISPLAY") - wayland = os.environ.get("WAYLAND_DISPLAY") - if not args.headless and not (display or wayland): - parser.error("Run from a graphical desktop terminal to see the scene, or use --headless for an automated check.") - - xvfb = None - try: - if args.headless: - if not shutil.which("Xvfb"): - parser.error("--headless requires Xvfb") - read_fd, write_fd = os.pipe() - try: - xvfb = subprocess.Popen( - ["Xvfb", "-displayfd", str(write_fd), "-screen", "0", "1280x720x24", "-nolisten", "tcp"], - pass_fds=(write_fd,), stdout=subprocess.DEVNULL, start_new_session=True) - except OSError: - os.close(read_fd) - raise - finally: - os.close(write_fd) - with os.fdopen(read_fd) as number: - if not select.select([number], [], [], 10)[0]: - raise ValueError("Xvfb did not start within 10 seconds") - display_number = number.readline().strip() - if not display_number.isdigit(): - raise ValueError("Xvfb failed to allocate a display") - display = ":" + display_number - wayland = None - env = simulator_environment(os.environ, manifest, display, wayland, args.software) - # Keep simulator settings and runtime files private and temporary. The - # desktop's XDG_RUNTIME_DIR is retained for Wayland authentication. - with tempfile.TemporaryDirectory(prefix="mocktail-vr-simulated-") as directory: - root = Path(directory) - env["XDG_CONFIG_HOME"] = str(root / "config") - env["XDG_DATA_HOME"] = str(root / "data") - if not env.get("XDG_RUNTIME_DIR"): - env["XDG_RUNTIME_DIR"] = str(root) - command = [str(probe), "--scene"] - frames = args.frames or (180 if args.headless else None) - if frames: - command += ["--frames", str(frames)] - print(f"Simulated Monado headset: {manifest.resolve()}", flush=True) - print("The test scene will appear in a desktop window. Ctrl+C exits." if not args.headless - else "Checking the test scene on a private Xvfb display.", flush=True) - child = None - previous = {} - - def forward_signal(signum, _frame): - if child is not None and child.poll() is None: - child.send_signal(signum) - - try: - child = subprocess.Popen(command, env=env, start_new_session=True) - for signum in (signal.SIGINT, signal.SIGTERM): - previous[signum] = signal.signal(signum, forward_signal) - return child.wait() - finally: - if child is not None and child.poll() is None: - child.terminate() - try: - child.wait(timeout=10) - except subprocess.TimeoutExpired: - child.kill() - child.wait() - for signum, handler in previous.items(): - signal.signal(signum, handler) - finally: - if xvfb is not None: - xvfb.terminate() - try: - xvfb.wait(timeout=5) - except subprocess.TimeoutExpired: - xvfb.kill() - xvfb.wait() - - -if __name__ == "__main__": - try: - sys.exit(main()) - except (OSError, ValueError, KeyError) as error: - print(f"VR simulator: {error}", file=sys.stderr) - sys.exit(1) diff --git a/src/compat/build_profile.cc b/src/compat/build_profile.cc index 2e2b0ebc..ee5aaa88 100644 --- a/src/compat/build_profile.cc +++ b/src/compat/build_profile.cc @@ -126,6 +126,57 @@ bool ParseOptionalFmodOutputDeviceBridge( return true; } +bool ParseOptionalVrDebugDeviceBridge( + const Json& object, + std::optional* bridge_profile) { + if (bridge_profile == nullptr) { + return false; + } + bridge_profile->reset(); + const auto field = object.find("vr_debug_device_bridge"); + if (field == object.end() || field->is_null()) { + return true; + } + if (!field->is_object()) { + return false; + } + + std::optional vtable_rva; + std::optional constructor_rva; + std::optional state_getter_rva; + std::optional eye_getter_rva; + std::optional eye_initializer_rva; + std::optional emulator_flag_storage_rva; + std::optional device_create_framebuffer_vtable_offset; + if (!ParseOptionalRva(*field, "vtable_rva", &vtable_rva) || + !ParseOptionalRva(*field, "constructor_rva", &constructor_rva) || + !ParseOptionalRva(*field, "state_getter_rva", &state_getter_rva) || + !ParseOptionalRva(*field, "eye_getter_rva", &eye_getter_rva) || + !ParseOptionalRva(*field, "eye_initializer_rva", &eye_initializer_rva) || + !ParseOptionalRva(*field, "emulator_flag_storage_rva", + &emulator_flag_storage_rva) || + !ParseOptionalRva(*field, "device_create_framebuffer_vtable_offset", + &device_create_framebuffer_vtable_offset) || + !vtable_rva.has_value() || !constructor_rva.has_value() || + !state_getter_rva.has_value() || !eye_getter_rva.has_value() || + !eye_initializer_rva.has_value() || + !emulator_flag_storage_rva.has_value() || + !device_create_framebuffer_vtable_offset.has_value()) { + return false; + } + + *bridge_profile = VrDebugDeviceBridgeProfile{ + *vtable_rva, + *constructor_rva, + *state_getter_rva, + *eye_getter_rva, + *eye_initializer_rva, + *emulator_flag_storage_rva, + *device_create_framebuffer_vtable_offset, + }; + return true; +} + } // namespace std::string_view BuildStatusName(BuildStatus status) noexcept { @@ -189,6 +240,7 @@ ProfileLookupResult FindBuildProfile(const std::string& manifest_path, const Json* reason = RequiredField(entry, "reason"); std::optional fullscreen_setter_rva; std::optional fmod_output_device_bridge; + std::optional vr_debug_device_bridge; if (version_name == nullptr || !version_name->is_string() || version_code == nullptr || !version_code->is_number_integer() || profile_build_id == nullptr || !profile_build_id->is_string() || @@ -205,7 +257,8 @@ ProfileLookupResult FindBuildProfile(const std::string& manifest_path, "user_game_settings_fullscreen_setter_rva", &fullscreen_setter_rva) || !ParseOptionalFmodOutputDeviceBridge( - entry, &fmod_output_device_bridge)) { + entry, &fmod_output_device_bridge) || + !ParseOptionalVrDebugDeviceBridge(entry, &vr_debug_device_bridge)) { return Failure("compatibility profile has missing or invalid fields"); } @@ -240,6 +293,7 @@ ProfileLookupResult FindBuildProfile(const std::string& manifest_path, allow_host_constructor_replay->get(), fullscreen_setter_rva, fmod_output_device_bridge, + vr_debug_device_bridge, reason->get_ref(), }; } diff --git a/src/graphics/bionic_vulkan_loader_adapter.cc b/src/graphics/bionic_vulkan_loader_adapter.cc index 89ac60fb..7fe7f4b1 100644 --- a/src/graphics/bionic_vulkan_loader_adapter.cc +++ b/src/graphics/bionic_vulkan_loader_adapter.cc @@ -121,6 +121,33 @@ VKAPI_ATTR VkResult VKAPI_CALL vkGetPhysicalDeviceSurfaceCapabilitiesKHR( VKAPI_ATTR VkResult VKAPI_CALL vkGetPhysicalDeviceSurfacePresentModesKHR( VkPhysicalDevice physical_device, VkSurfaceKHR surface, std::uint32_t* present_mode_count, VkPresentModeKHR* present_modes); +VKAPI_ATTR VkResult VKAPI_CALL vkCreateImage( + VkDevice device, const VkImageCreateInfo* create_info, + const VkAllocationCallbacks* allocator, VkImage* image); +VKAPI_ATTR void VKAPI_CALL vkDestroyImage(VkDevice device, VkImage image, + const VkAllocationCallbacks* allocator); +VKAPI_ATTR VkResult VKAPI_CALL vkCreateImageView( + VkDevice device, const VkImageViewCreateInfo* create_info, + const VkAllocationCallbacks* allocator, VkImageView* view); +VKAPI_ATTR void VKAPI_CALL +vkDestroyImageView(VkDevice device, VkImageView image_view, + const VkAllocationCallbacks* allocator); +VKAPI_ATTR VkResult VKAPI_CALL vkCreateFramebuffer( + VkDevice device, const VkFramebufferCreateInfo* create_info, + const VkAllocationCallbacks* allocator, VkFramebuffer* framebuffer); +VKAPI_ATTR void VKAPI_CALL +vkDestroyFramebuffer(VkDevice device, VkFramebuffer framebuffer, + const VkAllocationCallbacks* allocator); +VKAPI_ATTR VkResult VKAPI_CALL vkCreateRenderPass( + VkDevice device, const VkRenderPassCreateInfo* create_info, + const VkAllocationCallbacks* allocator, VkRenderPass* render_pass); +VKAPI_ATTR void VKAPI_CALL +vkDestroyRenderPass(VkDevice device, VkRenderPass render_pass, + const VkAllocationCallbacks* allocator); +VKAPI_ATTR void VKAPI_CALL +vkCmdBeginRenderPass(VkCommandBuffer command_buffer, + const VkRenderPassBeginInfo* begin_info, + VkSubpassContents contents); } namespace { @@ -130,6 +157,30 @@ using UsesDirectVulkanFn = bool (*)(); using NotePresentFn = void (*)(); using NoteHostPresentBeginFn = void (*)(); using NoteHostPresentEndFn = void (*)(std::int32_t); +using NoteVrPresentBeginFn = void (*)(); +using VrXrCreateInstanceFn = bool (*)(const VkInstanceCreateInfo*, + const VkAllocationCallbacks*, + VkInstance*, VkResult*); +using VrXrCreateDeviceFn = bool (*)(VkPhysicalDevice, + const VkDeviceCreateInfo*, + const VkAllocationCallbacks*, VkDevice*, + VkResult*); +using VrXrNoteDeviceDestroyedFn = void (*)(VkDevice); +using VrXrNoteQueueFn = void (*)(VkDevice, VkQueue, std::uint32_t, + std::uint32_t); +using VrXrRecordImageFn = void (*)(VkDevice, VkImage, + const VkImageCreateInfo*); +using VrXrRecordImageViewFn = void (*)(VkImageView, VkImage); +using VrXrRecordFramebufferFn = void (*)(VkFramebuffer, + const VkFramebufferCreateInfo*); +using VrXrRecordRenderPassFn = void (*)(VkRenderPass, + const VkRenderPassCreateInfo*); +using VrXrNoteDestroyImageFn = void (*)(VkImage); +using VrXrNoteDestroyImageViewFn = void (*)(VkImageView); +using VrXrNoteDestroyFramebufferFn = void (*)(VkFramebuffer); +using VrXrNoteRenderPassBeginFn = void (*)(VkCommandBuffer, + const VkRenderPassBeginInfo*); +using VrXrNotePresentFn = void (*)(VkQueue, VkDevice); using NoteVulkanCallBeginFn = std::uint64_t (*)(const char*); using NoteVulkanCallEndFn = void (*)(std::uint64_t, std::int32_t); using NoteSurfaceOutOfDateFn = void (*)(); @@ -160,6 +211,15 @@ struct AdapterState { PFN_vkBeginCommandBuffer begin_command_buffer = nullptr; PFN_vkEndCommandBuffer end_command_buffer = nullptr; PFN_vkResetCommandBuffer reset_command_buffer = nullptr; + PFN_vkCreateImage create_image = nullptr; + PFN_vkDestroyImage destroy_image = nullptr; + PFN_vkCreateImageView create_image_view = nullptr; + PFN_vkDestroyImageView destroy_image_view = nullptr; + PFN_vkCreateFramebuffer create_framebuffer = nullptr; + PFN_vkDestroyFramebuffer destroy_framebuffer = nullptr; + PFN_vkCreateRenderPass create_render_pass = nullptr; + PFN_vkDestroyRenderPass destroy_render_pass = nullptr; + PFN_vkCmdBeginRenderPass cmd_begin_render_pass = nullptr; }; struct QueueBinding { @@ -191,6 +251,21 @@ struct AdapterState { std::atomic note_present{nullptr}; std::atomic note_host_present_begin{nullptr}; std::atomic note_host_present_end{nullptr}; + std::atomic note_vr_present_begin{nullptr}; + std::atomic vr_create_instance{nullptr}; + std::atomic vr_create_device{nullptr}; + std::atomic vr_note_device_destroyed{nullptr}; + std::atomic vr_note_queue{nullptr}; + std::atomic vr_record_image{nullptr}; + std::atomic vr_record_image_view{nullptr}; + std::atomic vr_record_framebuffer{nullptr}; + std::atomic vr_record_render_pass{nullptr}; + std::atomic vr_note_destroy_image{nullptr}; + std::atomic vr_note_destroy_image_view{nullptr}; + std::atomic + vr_note_destroy_framebuffer{nullptr}; + std::atomic vr_note_render_pass_begin{nullptr}; + std::atomic vr_note_present{nullptr}; std::atomic note_vulkan_call_begin{nullptr}; std::atomic note_vulkan_call_end{nullptr}; std::atomic note_surface_out_of_date{nullptr}; @@ -234,6 +309,59 @@ bool EnsureInitialized() { ResolveProcessFunction( "mocktail_window_note_vulkan_host_present_end"), std::memory_order_release); + state.note_vr_present_begin.store( + ResolveProcessFunction( + "mocktail_vr_note_host_present_begin"), + std::memory_order_release); + state.vr_create_instance.store( + ResolveProcessFunction( + "mocktail_vr_xr_create_vulkan_instance"), + std::memory_order_release); + state.vr_create_device.store( + ResolveProcessFunction( + "mocktail_vr_xr_create_vulkan_device"), + std::memory_order_release); + state.vr_note_device_destroyed.store( + ResolveProcessFunction( + "mocktail_vr_xr_note_device_destroyed"), + std::memory_order_release); + state.vr_note_queue.store( + ResolveProcessFunction("mocktail_vr_xr_note_queue"), + std::memory_order_release); + state.vr_record_image.store( + ResolveProcessFunction("mocktail_vr_xr_record_image"), + std::memory_order_release); + state.vr_record_image_view.store( + ResolveProcessFunction( + "mocktail_vr_xr_record_image_view"), + std::memory_order_release); + state.vr_record_framebuffer.store( + ResolveProcessFunction( + "mocktail_vr_xr_record_framebuffer"), + std::memory_order_release); + state.vr_record_render_pass.store( + ResolveProcessFunction( + "mocktail_vr_xr_record_render_pass"), + std::memory_order_release); + state.vr_note_destroy_image.store( + ResolveProcessFunction( + "mocktail_vr_xr_note_destroy_image"), + std::memory_order_release); + state.vr_note_destroy_image_view.store( + ResolveProcessFunction( + "mocktail_vr_xr_note_destroy_image_view"), + std::memory_order_release); + state.vr_note_destroy_framebuffer.store( + ResolveProcessFunction( + "mocktail_vr_xr_note_destroy_framebuffer"), + std::memory_order_release); + state.vr_note_render_pass_begin.store( + ResolveProcessFunction( + "mocktail_vr_xr_note_render_pass_begin"), + std::memory_order_release); + state.vr_note_present.store( + ResolveProcessFunction("mocktail_vr_xr_note_present"), + std::memory_order_release); const auto call_begin = ResolveProcessFunction( "mocktail_window_note_vulkan_call_begin"); const auto call_end = ResolveProcessFunction( @@ -489,6 +617,24 @@ void RegisterHostDeviceDispatch(VkDevice device, get_device_proc_addr(device, "vkEndCommandBuffer")); dispatch.reset_command_buffer = reinterpret_cast( get_device_proc_addr(device, "vkResetCommandBuffer")); + dispatch.create_image = reinterpret_cast( + get_device_proc_addr(device, "vkCreateImage")); + dispatch.destroy_image = reinterpret_cast( + get_device_proc_addr(device, "vkDestroyImage")); + dispatch.create_image_view = reinterpret_cast( + get_device_proc_addr(device, "vkCreateImageView")); + dispatch.destroy_image_view = reinterpret_cast( + get_device_proc_addr(device, "vkDestroyImageView")); + dispatch.create_framebuffer = reinterpret_cast( + get_device_proc_addr(device, "vkCreateFramebuffer")); + dispatch.destroy_framebuffer = reinterpret_cast( + get_device_proc_addr(device, "vkDestroyFramebuffer")); + dispatch.create_render_pass = reinterpret_cast( + get_device_proc_addr(device, "vkCreateRenderPass")); + dispatch.destroy_render_pass = reinterpret_cast( + get_device_proc_addr(device, "vkDestroyRenderPass")); + dispatch.cmd_begin_render_pass = reinterpret_cast( + get_device_proc_addr(device, "vkCmdBeginRenderPass")); AdapterState& state = State(); std::lock_guard lock(state.mutex); @@ -727,6 +873,15 @@ PFN_vkVoidFunction AdapterProc(const char* name) { MOCKTAIL_VK_PROC(vkQueueWaitIdle) MOCKTAIL_VK_PROC(vkDeviceWaitIdle) MOCKTAIL_VK_PROC(vkQueuePresentKHR) + MOCKTAIL_VK_PROC(vkCreateImage) + MOCKTAIL_VK_PROC(vkDestroyImage) + MOCKTAIL_VK_PROC(vkCreateImageView) + MOCKTAIL_VK_PROC(vkDestroyImageView) + MOCKTAIL_VK_PROC(vkCreateFramebuffer) + MOCKTAIL_VK_PROC(vkDestroyFramebuffer) + MOCKTAIL_VK_PROC(vkCreateRenderPass) + MOCKTAIL_VK_PROC(vkDestroyRenderPass) + MOCKTAIL_VK_PROC(vkCmdBeginRenderPass) #undef MOCKTAIL_VK_PROC return nullptr; } @@ -758,7 +913,16 @@ bool IsDeviceAdapterProc(const char* name) { std::strcmp(name, "vkQueueBindSparse") == 0 || std::strcmp(name, "vkQueueWaitIdle") == 0 || std::strcmp(name, "vkDeviceWaitIdle") == 0 || - std::strcmp(name, "vkQueuePresentKHR") == 0); + std::strcmp(name, "vkQueuePresentKHR") == 0 || + std::strcmp(name, "vkCreateImage") == 0 || + std::strcmp(name, "vkDestroyImage") == 0 || + std::strcmp(name, "vkCreateImageView") == 0 || + std::strcmp(name, "vkDestroyImageView") == 0 || + std::strcmp(name, "vkCreateFramebuffer") == 0 || + std::strcmp(name, "vkDestroyFramebuffer") == 0 || + std::strcmp(name, "vkCreateRenderPass") == 0 || + std::strcmp(name, "vkDestroyRenderPass") == 0 || + std::strcmp(name, "vkCmdBeginRenderPass") == 0); } bool IsGlobalAdapterProc(const char* name) { @@ -988,10 +1152,12 @@ struct FpsWaitTrace { if (count == 0) { return; } - std::fprintf(stderr, " [fps] %s n=%llu avg=%llu us max=%llu us\n", name, + std::fprintf(stderr, " [fps] %s n=%llu avg=%llu us max=%llu us start_ns=%llu end_ns=%llu\n", name, static_cast(count), static_cast(total / count / 1000ULL), - static_cast(peak / 1000ULL)); + static_cast(peak / 1000ULL), + static_cast(window), + static_cast(start_ns)); } }; @@ -1028,6 +1194,16 @@ ObservedHostQueuePresent(VkQueue queue, const VkPresentInfoKHR* present_info) { if (note_begin != nullptr) { note_begin(); } + const NoteVrPresentBeginFn vr_note_begin = + state.note_vr_present_begin.load(std::memory_order_acquire); + if (vr_note_begin != nullptr) { + vr_note_begin(); + } + const VrXrNotePresentFn vr_note_present = + state.vr_note_present.load(std::memory_order_acquire); + if (vr_note_present != nullptr) { + vr_note_present(queue, HostDispatchForQueue(queue).device); + } const bool fps_trace = FpsTraceEnabled(); const std::uint64_t present_start_ns = fps_trace ? MonotonicNanos() : 0; const VkResult result = host_present(queue, present_info); @@ -1102,7 +1278,20 @@ vkCreateInstance(const VkInstanceCreateInfo* create_info, if (host_create == nullptr) { return VK_ERROR_INITIALIZATION_FAILED; } - const VkResult result = host_create(&host_info, allocator, instance); + VkResult result = VK_ERROR_INITIALIZATION_FAILED; + bool vr_handled = false; + const auto vr_create_instance = + State().vr_create_instance.load(std::memory_order_acquire); + if (vr_create_instance != nullptr) { + // The XR backend creates the instance through xrCreateVulkanInstanceKHR + // so the runtime's required instance extensions are enabled on the very + // instance Roblox renders with. + vr_handled = + vr_create_instance(&host_info, allocator, instance, &result); + } + if (!vr_handled) { + result = host_create(&host_info, allocator, instance); + } if (result == VK_SUCCESS) { AdapterState& state = State(); std::lock_guard lock(state.mutex); @@ -1124,6 +1313,10 @@ vkCreateInstance(const VkInstanceCreateInfo* create_info, VKAPI_ATTR void VKAPI_CALL vkDestroyInstance(VkInstance instance, const VkAllocationCallbacks* allocator) { + const auto note = ResolveProcessFunction( + "mocktail_vr_xr_note_instance_destroyed"); + if (note != nullptr) note(instance); + const auto host_destroy = reinterpret_cast( HostInstanceProc(instance, "vkDestroyInstance")); if (host_destroy != nullptr) { @@ -1149,6 +1342,21 @@ vkDestroyInstance(VkInstance instance, const VkAllocationCallbacks* allocator) { state.note_present.store(nullptr, std::memory_order_release); state.note_host_present_begin.store(nullptr, std::memory_order_release); state.note_host_present_end.store(nullptr, std::memory_order_release); + state.note_vr_present_begin.store(nullptr, std::memory_order_release); + state.vr_create_instance.store(nullptr, std::memory_order_release); + state.vr_create_device.store(nullptr, std::memory_order_release); + state.vr_note_device_destroyed.store(nullptr, std::memory_order_release); + state.vr_note_queue.store(nullptr, std::memory_order_release); + state.vr_record_image.store(nullptr, std::memory_order_release); + state.vr_record_image_view.store(nullptr, std::memory_order_release); + state.vr_record_framebuffer.store(nullptr, std::memory_order_release); + state.vr_record_render_pass.store(nullptr, std::memory_order_release); + state.vr_note_destroy_image.store(nullptr, std::memory_order_release); + state.vr_note_destroy_image_view.store(nullptr, std::memory_order_release); + state.vr_note_destroy_framebuffer.store(nullptr, + std::memory_order_release); + state.vr_note_render_pass_begin.store(nullptr, std::memory_order_release); + state.vr_note_present.store(nullptr, std::memory_order_release); state.note_vulkan_call_begin.store(nullptr, std::memory_order_release); state.note_vulkan_call_end.store(nullptr, std::memory_order_release); g_vulkan_call_observation_active.store(false, std::memory_order_release); @@ -1339,8 +1547,20 @@ VKAPI_ATTR VkResult VKAPI_CALL vkCreateDevice( } } - const VkResult result = - host_create(physical_device, &host_info, allocator, device); + VkResult result = VK_ERROR_INITIALIZATION_FAILED; + bool vr_handled = false; + const auto vr_create_device = + State().vr_create_device.load(std::memory_order_acquire); + if (vr_create_device != nullptr) { + // The XR backend creates the device through xrCreateVulkanDeviceKHR and + // binds the OpenXR session to it; falling back afterwards would create a + // second, XR-incompatible device, so the hook result is final. + vr_handled = vr_create_device(physical_device, &host_info, allocator, + device, &result); + } + if (!vr_handled) { + result = host_create(physical_device, &host_info, allocator, device); + } if (result != VK_SUCCESS) { return result; } @@ -1402,6 +1622,13 @@ VKAPI_ATTR void VKAPI_CALL vkDestroyDevice(VkDevice device, const VkAllocationCallbacks* allocator) { const auto host_destroy = reinterpret_cast( HostDeviceProc(device, "vkDestroyDevice")); + const auto vr_note_device_destroyed = + State().vr_note_device_destroyed.load(std::memory_order_acquire); + if (vr_note_device_destroyed != nullptr) { + // Runs while the device is still valid so the XR backend can destroy + // its session and swapchain resources first. + vr_note_device_destroyed(device); + } State().text_overlay.DestroyDevice(device); if (host_destroy != nullptr) { host_destroy(device, allocator); @@ -1423,6 +1650,11 @@ VKAPI_ATTR void VKAPI_CALL vkGetDeviceQueue(VkDevice device, RegisterHostQueueBinding(device, *queue); (void)State().text_overlay.RegisterQueue(device, *queue, queue_family_index, queue_index); + const auto vr_note_queue = + State().vr_note_queue.load(std::memory_order_acquire); + if (vr_note_queue != nullptr) { + vr_note_queue(device, *queue, queue_family_index, queue_index); + } } } @@ -1438,6 +1670,12 @@ VKAPI_ATTR void VKAPI_CALL vkGetDeviceQueue2( RegisterHostQueueBinding(device, *queue); (void)State().text_overlay.RegisterQueue( device, *queue, queue_info->queueFamilyIndex, queue_info->queueIndex); + const auto vr_note_queue = + State().vr_note_queue.load(std::memory_order_acquire); + if (vr_note_queue != nullptr) { + vr_note_queue(device, *queue, queue_info->queueFamilyIndex, + queue_info->queueIndex); + } } } @@ -2025,4 +2263,146 @@ vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR* present_info) { return normalized_result; } -} // extern "C" +// ---- VR backend resource provenance wrappers ------------------------------- +// Thin pass-throughs to the host dispatch with optional notifications for the +// experimental OpenXR backend. Creation/destruction calls are rare; the only +// per-frame entry (vkCmdBeginRenderPass) costs one atomic load and stops +// doing work once both eye bindings are established. + +VKAPI_ATTR VkResult VKAPI_CALL vkCreateImage( + VkDevice device, const VkImageCreateInfo* create_info, + const VkAllocationCallbacks* allocator, VkImage* image) { + const PFN_vkCreateImage host_create = + HostDispatchForDevice(device).create_image; + if (host_create == nullptr) { + return VK_ERROR_INITIALIZATION_FAILED; + } + const VkResult result = host_create(device, create_info, allocator, image); + const auto record = State().vr_record_image.load(std::memory_order_acquire); + if (result == VK_SUCCESS && record != nullptr) { + record(device, *image, create_info); + } + return result; +} + +VKAPI_ATTR void VKAPI_CALL vkDestroyImage(VkDevice device, VkImage image, + const VkAllocationCallbacks* allocator) { + const auto note = State().vr_note_destroy_image.load(std::memory_order_acquire); + if (note != nullptr && image != VK_NULL_HANDLE) { + note(image); + } + const PFN_vkDestroyImage host_destroy = + HostDispatchForDevice(device).destroy_image; + if (host_destroy != nullptr) { + host_destroy(device, image, allocator); + } +} + +VKAPI_ATTR VkResult VKAPI_CALL vkCreateImageView( + VkDevice device, const VkImageViewCreateInfo* create_info, + const VkAllocationCallbacks* allocator, VkImageView* view) { + const PFN_vkCreateImageView host_create = + HostDispatchForDevice(device).create_image_view; + if (host_create == nullptr) { + return VK_ERROR_INITIALIZATION_FAILED; + } + const VkResult result = host_create(device, create_info, allocator, view); + const auto record = + State().vr_record_image_view.load(std::memory_order_acquire); + if (result == VK_SUCCESS && record != nullptr && create_info != nullptr) { + record(*view, create_info->image); + } + return result; +} + +VKAPI_ATTR void VKAPI_CALL +vkDestroyImageView(VkDevice device, VkImageView image_view, + const VkAllocationCallbacks* allocator) { + const auto note = + State().vr_note_destroy_image_view.load(std::memory_order_acquire); + if (note != nullptr && image_view != VK_NULL_HANDLE) { + note(image_view); + } + const PFN_vkDestroyImageView host_destroy = + HostDispatchForDevice(device).destroy_image_view; + if (host_destroy != nullptr) { + host_destroy(device, image_view, allocator); + } +} + +VKAPI_ATTR VkResult VKAPI_CALL vkCreateFramebuffer( + VkDevice device, const VkFramebufferCreateInfo* create_info, + const VkAllocationCallbacks* allocator, VkFramebuffer* framebuffer) { + const PFN_vkCreateFramebuffer host_create = + HostDispatchForDevice(device).create_framebuffer; + if (host_create == nullptr) { + return VK_ERROR_INITIALIZATION_FAILED; + } + const VkResult result = host_create(device, create_info, allocator, framebuffer); + const auto record = + State().vr_record_framebuffer.load(std::memory_order_acquire); + if (result == VK_SUCCESS && record != nullptr) { + record(*framebuffer, create_info); + } + return result; +} + +VKAPI_ATTR void VKAPI_CALL +vkDestroyFramebuffer(VkDevice device, VkFramebuffer framebuffer, + const VkAllocationCallbacks* allocator) { + const auto note = + State().vr_note_destroy_framebuffer.load(std::memory_order_acquire); + if (note != nullptr && framebuffer != VK_NULL_HANDLE) { + note(framebuffer); + } + const PFN_vkDestroyFramebuffer host_destroy = + HostDispatchForDevice(device).destroy_framebuffer; + if (host_destroy != nullptr) { + host_destroy(device, framebuffer, allocator); + } +} + +VKAPI_ATTR VkResult VKAPI_CALL vkCreateRenderPass( + VkDevice device, const VkRenderPassCreateInfo* create_info, + const VkAllocationCallbacks* allocator, VkRenderPass* render_pass) { + const PFN_vkCreateRenderPass host_create = + HostDispatchForDevice(device).create_render_pass; + if (host_create == nullptr) { + return VK_ERROR_INITIALIZATION_FAILED; + } + const VkResult result = host_create(device, create_info, allocator, render_pass); + const auto record = + State().vr_record_render_pass.load(std::memory_order_acquire); + if (result == VK_SUCCESS && record != nullptr) { + record(*render_pass, create_info); + } + return result; +} + +VKAPI_ATTR void VKAPI_CALL +vkDestroyRenderPass(VkDevice device, VkRenderPass render_pass, + const VkAllocationCallbacks* allocator) { + const PFN_vkDestroyRenderPass host_destroy = + HostDispatchForDevice(device).destroy_render_pass; + if (host_destroy != nullptr) { + host_destroy(device, render_pass, allocator); + } +} + +VKAPI_ATTR void VKAPI_CALL +vkCmdBeginRenderPass(VkCommandBuffer command_buffer, + const VkRenderPassBeginInfo* begin_info, + VkSubpassContents contents) { + const auto note = + State().vr_note_render_pass_begin.load(std::memory_order_acquire); + if (note != nullptr) { + note(command_buffer, begin_info); + } + const PFN_vkCmdBeginRenderPass host_begin = + HostDispatchForCommandBuffer(command_buffer).cmd_begin_render_pass; + if (host_begin != nullptr) { + host_begin(command_buffer, begin_info, contents); + } +} + +} // extern "C" \ No newline at end of file diff --git a/src/legacy/legacy_runtime.cc b/src/legacy/legacy_runtime.cc index 4c93dc32..157ad947 100644 --- a/src/legacy/legacy_runtime.cc +++ b/src/legacy/legacy_runtime.cc @@ -56,6 +56,7 @@ #include "libc_shim/libc_shim.h" #include "linker/linker.h" #include "mocktail/graphics/bionic_egl_bridge.h" +#include "mocktail/vr/roblox_vr_device_bridge.h" #include "runtime/environment.h" #include "runtime/discord_rpc.h" #include "runtime/jnivm_platform_web_callbacks.h" @@ -5610,6 +5611,20 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, std::cout << " [linker] Bionic unwind metadata validated for libroblox\n" << std::flush; + // The experimental VR bridge is armed only for --vr on an exact supported + // Build ID. It validates its machine contracts and interposes the + // DebugDeviceVR vtable slots after relocations and before guest startup + // can create the object. A no-op success while no bridge is armed. + if (const mocktail::Status vr_activation = + mocktail::vr::NotifyRobloxVrImageLoaded( + linker::FindLoadedAndroidLibraryBase("libroblox")); + !vr_activation.ok()) { + std::cerr << "\n[FATAL] Experimental Roblox VR bridge activation " + "failed: " + << vr_activation.message() << '\n'; + return EXIT_FAILURE; + } + // The pre-constructor hook owns installation; repeating arena setup here is // not idempotent. if (HostAbiExperimentRequested() && diff --git a/src/main.cc b/src/main.cc index 1f9352c5..1e7a23c0 100644 --- a/src/main.cc +++ b/src/main.cc @@ -23,6 +23,9 @@ #include "mocktail/audio/fmod_jni_audio_bridge.h" #include "mocktail/audio/webrtc_jni_audio_bridge.h" #include "mocktail/audio/roblox_output_device_bridge.h" +#include "mocktail/vr/openxr_backend.h" +#include "mocktail/vr/openxr_probe.h" +#include "mocktail/vr/roblox_vr_device_bridge.h" #include "runtime/auth_runtime_composition.h" #include "runtime/command_line.h" #include "runtime/crash_report_policy.h" @@ -332,11 +335,28 @@ int main(int argc, char* argv[]) { return EXIT_FAILURE; } // The standalone OpenXR scene belongs to mocktail-vr-probe --scene. - // Continue through Roblox startup to inspect its native VR device. + // --vr continues through Roblox startup as a native stereo prototype. if (runtime_config.config.vr_enabled()) { - std::cerr << " [vr] Roblox VR initialization requested (experimental); " - "stereo eye resources and OpenXR output are not yet connected. " - "Standalone scene: mocktail-vr-probe --scene\n"; + if (!mocktail::vr::IsOpenXrSupportCompiled()) { + std::cerr << " [vr] This build does not include OpenXR support. " + "Use a VR-enabled build of Mocktail; refusing to start " + "Roblox with --vr in a non-VR build.\n"; + return EXIT_FAILURE; + } + const mocktail::vr::VrBackendMode vr_backend_mode = + mocktail::vr::ResolveVrBackendMode(true); + std::cerr << " [vr] experimental backend=" + << mocktail::vr::VrBackendModeName(vr_backend_mode) + << ": Roblox renders both eyes through its DebugDeviceVR " + "debug device (exact Build ID 2998 only)"; + if (vr_backend_mode == mocktail::vr::VrBackendMode::kXrOutput) { + std::cerr << "; eye images are submitted to the active OpenXR runtime " + "as a projection layer. Start WiVRn and connect the headset " + "before launching.\n"; + } else { + std::cerr << "; OpenXR output is disabled in this diagnostic mode " + "(MOCKTAIL_VR_BACKEND=native).\n"; + } } mocktail::runtime::SessionLog session_log; mocktail::runtime::FailureSupportBundleGuard support_bundle_guard( @@ -755,6 +775,12 @@ int main(int argc, char* argv[]) { << command_line_error << '\n'; return EXIT_FAILURE; } + if (!mocktail::runtime::MergeVrClientSettingsOverrides( + runtime_config.config.vr_enabled(), audio_capture_overrides, + &audio_capture_overrides, &command_line_error)) { + std::cerr << "[FATAL] Cannot apply VR policy: " << command_line_error << '\n'; + return EXIT_FAILURE; + } if (setenv("MOCKTAIL_CLIENT_SETTINGS_OVERRIDES_JSON", audio_capture_overrides.c_str(), 1) != 0) { std::cerr << "[FATAL] Cannot export runtime client-settings policy\n"; @@ -770,6 +796,12 @@ int main(int argc, char* argv[]) { << command_line_error << '\n'; return EXIT_FAILURE; } + if (!mocktail::runtime::MergeVrClientSettingsOverrides( + runtime_config.config.vr_enabled(), fast_flags_overrides, + &fast_flags_overrides, &command_line_error)) { + std::cerr << "[FATAL] Cannot apply VR policy: " << command_line_error << '\n'; + return EXIT_FAILURE; + } if (setenv("MOCKTAIL_FAST_FLAGS_JSON", fast_flags_overrides.c_str(), 1) != 0) { std::cerr << "[FATAL] Cannot export crash-report fast-flags policy\n"; @@ -828,6 +860,8 @@ int main(int argc, char* argv[]) { mocktail::runtime::RobloxFullscreenRuntimeBridge fullscreen_bridge; mocktail::audio::RobloxOutputDeviceBridge output_device_bridge; + mocktail::vr::RobloxVrDeviceBridge vr_device_bridge; + mocktail::vr::OpenXrBackend xr_backend; if (command_line.options.mode == mocktail::runtime::CommandMode::kRun) { const std::string compatibility_manifest = environment.GetOr("MOCKTAIL_COMPATIBILITY_MANIFEST", @@ -855,6 +889,31 @@ int main(int argc, char* argv[]) { << output_device_status.message() << '\n'; return EXIT_FAILURE; } + if (runtime_config.config.vr_enabled()) { + if (mocktail::vr::ResolveVrBackendMode(true) == + mocktail::vr::VrBackendMode::kXrOutput) { + // Fail closed and loudly when no OpenXR runtime is reachable; the + // native-stereo diagnostic mode must be requested explicitly. + const mocktail::Status xr_status = xr_backend.Arm(); + if (!xr_status.ok()) { + std::cerr << "[FATAL] Cannot arm the OpenXR backend for --vr: " + << xr_status.message() << '\n' + << " Start WiVRn, connect your headset, then run mocktail -vr again.\n" + << " For a custom runtime set XR_RUNTIME_JSON to its manifest.\n" + << " Set MOCKTAIL_VR_BACKEND=native for the " + "native-stereo diagnostic mode without a runtime.\n"; + return EXIT_FAILURE; + } + } + const mocktail::Status vr_device_status = + vr_device_bridge.Install(compatibility.profile); + if (!vr_device_status.ok()) { + std::cerr << "[FATAL] Cannot install experimental Roblox VR device " + "bridge: " + << vr_device_status.message() << '\n'; + return EXIT_FAILURE; + } + } } mocktail::legacy::RuntimeDependencies dependencies; @@ -1046,6 +1105,8 @@ int main(int argc, char* argv[]) { std::to_string(runtime_status) + "."); } support_bundle_guard.SetExitCode(runtime_status); + xr_backend.Disarm(); + vr_device_bridge.Shutdown(); output_device_bridge.Shutdown(); fullscreen_bridge.Shutdown(); memory_limit_watchdog.Stop(); diff --git a/src/runtime/command_line.cc b/src/runtime/command_line.cc index 08b9acf7..9534e5bc 100644 --- a/src/runtime/command_line.cc +++ b/src/runtime/command_line.cc @@ -74,8 +74,8 @@ CommandLineParseResult ParseCommandLine(int argc, const char* const argv[]) { &result.options.graphics_backend, &result.error)) { return result; } - } else if (argument == "--vr" || argument == "--no-vr") { - result.options.vr_enabled = argument == "--vr"; + } else if (argument == "--vr" || argument == "-vr" || argument == "--no-vr") { + result.options.vr_enabled = argument != "--no-vr"; } else if (argument == "--allow-unverified-build") { result.options.allow_unverified_build = true; } else if (argument == "--force-run-latest") { @@ -256,8 +256,9 @@ std::string CommandLineUsage(const std::string& program_name) { "x86_64 libroblox.so\n" << " --headless Run without creating an SDL window\n" << " --windowed Force windowed startup (default)\n" - << " --vr / --no-vr Launch Roblox in experimental VR mode or " + << " -vr / --vr / --no-vr Launch Roblox via OpenXR (WiVRn) or " "disable vr.enabled\n" + << " VR builds enable VR by default; use --no-vr to opt out.\n" << " --graphics direct-vulkan | opengl | system | " "angle-vulkan (default: direct-vulkan)\n" << " --allow-unverified-build Run a known but unverified Build-ID " diff --git a/src/runtime/performance_policy.cc b/src/runtime/performance_policy.cc index ed119404..e14c98a8 100644 --- a/src/runtime/performance_policy.cc +++ b/src/runtime/performance_policy.cc @@ -367,6 +367,28 @@ bool MergeRuntimeClientSettingsOverrides(const FrameRatePolicy& frame_rate, merged_json, error); } +bool MergeVrClientSettingsOverrides(bool enabled, std::string_view base_json, + std::string* merged_json, + std::string* error) { + if (merged_json == nullptr) { + if (error != nullptr) { + *error = "VR client-settings output is required"; + } + return false; + } + auto overrides = nlohmann::json::parse(base_json.empty() ? "{}" : base_json, + nullptr, false, true); + if (overrides.is_discarded() || !overrides.is_object()) { + if (error != nullptr) { + *error = "VR client-settings input must be a JSON object"; + } + return false; + } + overrides["FFlagDebugEnableVREmulator"] = enabled ? "True" : "False"; + *merged_json = overrides.dump(); + return true; +} + bool MergeAudioCaptureClientSettingsOverrides(bool microphone_enabled, std::string_view base_json, std::string* merged_json, diff --git a/src/runtime/runtime_config.cc b/src/runtime/runtime_config.cc index 18e09137..fbd5e693 100644 --- a/src/runtime/runtime_config.cc +++ b/src/runtime/runtime_config.cc @@ -171,7 +171,7 @@ RuntimeConfig RuntimeConfig::FromEnvironment(const Environment& environment) { RuntimeConfig config; config.headless_ = LegacyEnabled(environment, "MOCKTAIL_HEADLESS"); config.vr_enabled_ = - ReadBoolean(environment, "MOCKTAIL_VR_ENABLED", false, &config.vr_valid_); + ReadBoolean(environment, "MOCKTAIL_VR_ENABLED", kDefaultVrEnabled, &config.vr_valid_); config.roblox_library_path_ = environment.GetOr( "ROBLOX_LIB_PATH", config.roblox_library_path_.string()); config.graphics_backend_name_ = environment.GetOr( diff --git a/src/runtime/runtime_config_bootstrap.cc b/src/runtime/runtime_config_bootstrap.cc index 3905077d..892ffd20 100644 --- a/src/runtime/runtime_config_bootstrap.cc +++ b/src/runtime/runtime_config_bootstrap.cc @@ -67,12 +67,16 @@ device: pc-windows-11 # Optional presentation synchronization override: auto, on, or off. # vsync: off -vr: - # Boolean (default: false): launch the Roblox experimental VR path. - # Only Build ID 2998 is currently accepted; eye resources and OpenXR output - # are NOT yet connected. Standalone scene: mocktail-vr-probe --scene. - # Use --no-vr to launch Roblox normally. - enabled: false +# vr: + # Boolean (default: enabled in VR builds, disabled in non-VR builds). + # Experimental VR for the exact Build ID 2998 + # payload. Mocktail initializes the guest DebugDeviceVR eye resources (two + # distinct 500x500 eye framebuffers), the Roblox renderer draws both eyes + # every frame, and the frames are submitted to the active OpenXR runtime as + # a stereo projection layer with head pose (runtime-tracked, or scripted for + # desktop testing). Controllers are NOT connected yet. Standalone OpenXR + # test scene: mocktail-vr-probe --scene. Use --no-vr to launch normally. + # enabled: false performance: # Boolean (default: false): size Roblox scheduler/render queues from every diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc new file mode 100644 index 00000000..89a3b845 --- /dev/null +++ b/src/vr/openxr_backend.cc @@ -0,0 +1,2194 @@ +#include "mocktail/vr/openxr_backend.h" +#include "mocktail/vr/vr_perf.h" +#include "mocktail/vr/vr_pose_math.h" + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef M_PI +#define M_PI 3.14159265358979323846 +#endif + +namespace mocktail::vr { +namespace { + +using Clock = std::chrono::steady_clock; + +void Log(const char* format, ...) { + va_list arguments; + va_start(arguments, format); + std::vfprintf(stderr, format, arguments); + va_end(arguments); + std::fflush(stderr); +} + +void Require(bool condition, const std::string& message) { + if (!condition) { + throw std::runtime_error(message); + } +} + +template +T XrInfo(XrStructureType type) { + T value{}; + value.type = type; + return value; +} + +std::uint64_t EnvUnsigned(const char* name, std::uint64_t fallback) { + const char* value = std::getenv(name); + if (value == nullptr || value[0] == '\0') { + return fallback; + } + const unsigned long long parsed = std::strtoull(value, nullptr, 10); + return parsed > 0 ? static_cast(parsed) : fallback; +} + +double EnvDouble(const char* name, double fallback) { + const char* value = std::getenv(name); + if (value == nullptr || value[0] == '\0') { + return fallback; + } + return std::strtod(value, nullptr); +} + +std::string EnvString(const char* name, const char* fallback) { + const char* value = std::getenv(name); + return value != nullptr && value[0] != '\0' ? std::string(value) + : std::string(fallback); +} + +// Coordinate and pose math is shared with the unit tests through +// vr_pose_math.h, which documents every space and unit in the camera path, so +// the audited conventions cannot silently diverge from the backend. +using pose::Pose; +using pose::PoseInverse; +using pose::PoseMultiply; +using pose::Quat; +using pose::QuatFromYawPitch; +using pose::Vec3; + +Pose XrToPose(const XrPosef& pose) { + Pose result; + result.position = Vec3{pose.position.x, pose.position.y, pose.position.z}; + result.orientation = + Quat{pose.orientation.x, pose.orientation.y, pose.orientation.z, + pose.orientation.w}; + return result; +} + +// Bridge provenance exports (same-process direct linkage; the evidence layer +// resolves the identical symbols through dlsym). +extern "C" void* mocktail_vr_initializing_object(); +extern "C" bool mocktail_vr_current_eye(void** self, int* eye, + void** framebuffer); +extern "C" void mocktail_vr_consume_eye(); + +bool IsColorFormat(VkFormat format) { + switch (format) { + case VK_FORMAT_D16_UNORM: + case VK_FORMAT_D16_UNORM_S8_UINT: + case VK_FORMAT_D24_UNORM_S8_UINT: + case VK_FORMAT_D32_SFLOAT: + case VK_FORMAT_D32_SFLOAT_S8_UINT: + case VK_FORMAT_X8_D24_UNORM_PACK32: + return false; + default: + return true; + } +} + +bool FormatIsCopyCompatible(VkFormat a, VkFormat b) { + return a == b; +} + +OpenXrBackend* g_backend = nullptr; +std::mutex g_backend_slot_mutex; + +} // namespace + +struct OpenXrBackend::VkProcs { + PFN_vkGetInstanceProcAddr get_instance_proc_addr = nullptr; + PFN_vkGetDeviceProcAddr get_device_proc_addr = nullptr; + PFN_vkCreateBuffer CreateBuffer = nullptr; + PFN_vkDestroyBuffer DestroyBuffer = nullptr; + PFN_vkGetBufferMemoryRequirements GetBufferMemoryRequirements = nullptr; + PFN_vkAllocateMemory AllocateMemory = nullptr; + PFN_vkFreeMemory FreeMemory = nullptr; + PFN_vkBindBufferMemory BindBufferMemory = nullptr; + PFN_vkMapMemory MapMemory = nullptr; + PFN_vkUnmapMemory UnmapMemory = nullptr; + PFN_vkCmdCopyImageToBuffer CmdCopyImageToBuffer = nullptr; + PFN_vkCreateImageView CreateImageView = nullptr; + PFN_vkDestroyImageView DestroyImageView = nullptr; + PFN_vkCreateCommandPool CreateCommandPool = nullptr; + PFN_vkDestroyCommandPool DestroyCommandPool = nullptr; + PFN_vkAllocateCommandBuffers AllocateCommandBuffers = nullptr; + PFN_vkResetCommandBuffer ResetCommandBuffer = nullptr; + PFN_vkBeginCommandBuffer BeginCommandBuffer = nullptr; + PFN_vkEndCommandBuffer EndCommandBuffer = nullptr; + PFN_vkCmdPipelineBarrier CmdPipelineBarrier = nullptr; + PFN_vkCmdCopyImage CmdCopyImage = nullptr; + PFN_vkCmdBlitImage CmdBlitImage = nullptr; + PFN_vkCreateFence CreateFence = nullptr; + PFN_vkDestroyFence DestroyFence = nullptr; + PFN_vkResetFences ResetFences = nullptr; + PFN_vkWaitForFences WaitForFences = nullptr; + PFN_vkQueueSubmit QueueSubmit = nullptr; + PFN_vkDeviceWaitIdle DeviceWaitIdle = nullptr; + PFN_vkGetPhysicalDeviceMemoryProperties GetPhysicalDeviceMemoryProperties = + nullptr; + PFN_vkGetPhysicalDeviceQueueFamilyProperties + GetPhysicalDeviceQueueFamilyProperties = nullptr; + PFN_vkGetPhysicalDeviceProperties GetPhysicalDeviceProperties = nullptr; +}; + +namespace { + +// Records elapsed monotonic time into the opt-in perf collector. A no-op +// unless MOCKTAIL_VR_PERF is set, so the steady-state path is unaffected. +class ScopedStageTimer final { + public: + explicit ScopedStageTimer(perf::Stage stage) + : stage_(stage), start_(perf::Enabled() ? perf::NowNs() : 0) {} + ~ScopedStageTimer() { + if (start_ != 0) { + const std::uint64_t end = perf::NowNs(); + if (end >= start_) { + perf::ProcessCollector().Record(stage_, end - start_); + } + } + } + ScopedStageTimer(const ScopedStageTimer&) = delete; + ScopedStageTimer& operator=(const ScopedStageTimer&) = delete; + + private: + perf::Stage stage_; + std::uint64_t start_; +}; + +std::string XrResultText(XrInstance instance, XrResult result) { + char text[XR_MAX_RESULT_STRING_SIZE]{}; + if (instance != XR_NULL_HANDLE) { + (void)xrResultToString(instance, result, text); + } + if (!text[0]) { + return std::to_string(static_cast(result)); + } + return text; +} + +struct XrCallError : std::runtime_error { + XrResult result; + XrCallError(XrResult value, const std::string& message) + : std::runtime_error(message), result(value) {} +}; + +void CheckXr(XrInstance instance, XrResult result, const char* operation) { + if (XR_SUCCEEDED(result)) { + return; + } + std::string message = + std::string(operation) + " failed: " + XrResultText(instance, result); + throw XrCallError(result, message); +} + +} // namespace + +OpenXrBackend::~OpenXrBackend() { Disarm(); } + +OpenXrBackend* ActiveVrBackend() { + std::lock_guard lock(g_backend_slot_mutex); + return g_backend; +} + +Status OpenXrBackend::Arm() { + std::lock_guard lock(mutex_); + if (armed_.load(std::memory_order_acquire)) { + return Status::Ok(); + } + { + std::lock_guard slot_lock(g_backend_slot_mutex); + if (g_backend != nullptr && g_backend != this) { + return Status::Error(StatusCode::kFailedPrecondition, + "another OpenXR backend owns the process"); + } + g_backend = this; + } + try { + const std::string user_manifest = EnvString("XR_RUNTIME_JSON", ""); + const auto manifest = SelectVrRuntimeManifest(user_manifest, + {"/usr/local/share/openxr/1/openxr_wivrn.json", + "/usr/share/openxr/1/openxr_wivrn.json"}); + if (!manifest.empty()) { + Require(setenv("XR_RUNTIME_JSON", manifest.c_str(), 1) == 0, + "Cannot select the OpenXR runtime"); + Log(" [vr-backend] runtime manifest: %s\n", manifest.c_str()); + } else { + Log(" [vr-backend] using registered OpenXR runtime; connect the headset in WiVRn first\n"); + } + vk_loader_ = dlopen("libvulkan.so.1", RTLD_NOW | RTLD_LOCAL); + Require(vk_loader_ != nullptr, + "Cannot load the host Vulkan loader libvulkan.so.1"); + auto loader_gipa = reinterpret_cast( + dlsym(vk_loader_, "vkGetInstanceProcAddr")); + Require(loader_gipa != nullptr, "Host vkGetInstanceProcAddr is missing"); + + const char* extension = XR_KHR_VULKAN_ENABLE2_EXTENSION_NAME; + auto info = XrInfo(XR_TYPE_INSTANCE_CREATE_INFO); + std::strcpy(info.applicationInfo.applicationName, "Mocktail Roblox VR"); + std::strcpy(info.applicationInfo.engineName, "Mocktail"); + info.applicationInfo.apiVersion = XR_MAKE_VERSION(1, 0, 0); + info.enabledExtensionCount = 1; + info.enabledExtensionNames = &extension; + XrInstance instance = XR_NULL_HANDLE; + const XrResult instance_result = xrCreateInstance(&info, &instance); + if (instance_result == XR_ERROR_RUNTIME_UNAVAILABLE) { + throw std::runtime_error("xrCreateInstance: " + + VrRuntimeUnavailableHint(!user_manifest.empty(), manifest)); + } + CheckXr(XR_NULL_HANDLE, instance_result, "xrCreateInstance"); + xr_instance_ = instance; + + XrInstanceProperties properties = + XrInfo(XR_TYPE_INSTANCE_PROPERTIES); + if (XR_SUCCEEDED(xrGetInstanceProperties(instance, &properties))) { + runtime_name_ = properties.runtimeName; + Log(" [vr-backend] OpenXR runtime: %s %u.%u.%u\n", + properties.runtimeName, + XR_VERSION_MAJOR(properties.runtimeVersion), + XR_VERSION_MINOR(properties.runtimeVersion), + XR_VERSION_PATCH(properties.runtimeVersion)); + } + + auto system = XrInfo(XR_TYPE_SYSTEM_GET_INFO); + system.formFactor = XR_FORM_FACTOR_HEAD_MOUNTED_DISPLAY; + const XrResult system_result = + xrGetSystem(instance, &system, &xr_system_); + if (system_result == XR_ERROR_FORM_FACTOR_UNAVAILABLE) { + // The runtime answered but exposes no HMD: with WiVRn this is the exact + // state while the server runs with no headset connected. + throw std::runtime_error( + "The OpenXR runtime is reachable but reports no connected headset " + "(XR_ERROR_FORM_FACTOR_UNAVAILABLE). With WiVRn: start the server " + "and connect the headset first; VR applications can only run while " + "the headset connection is established."); + } + CheckXr(instance, system_result, "xrGetSystem"); + Require(xr_system_ != XR_NULL_SYSTEM_ID, + "The OpenXR runtime exposes no head-mounted display system"); + + auto graphics_requirements = + XrInfo( + XR_TYPE_GRAPHICS_REQUIREMENTS_VULKAN2_KHR); + auto get_requirements = reinterpret_cast( + ResolveFunction("xrGetVulkanGraphicsRequirements2KHR")); + CheckXr(instance, + get_requirements(instance, xr_system_, &graphics_requirements), + "xrGetVulkanGraphicsRequirements2KHR"); + Log(" [vr-backend] runtime Vulkan requirement: %u.%u - %u.%u\n", + XR_VERSION_MAJOR(graphics_requirements.minApiVersionSupported), + XR_VERSION_MINOR(graphics_requirements.minApiVersionSupported), + XR_VERSION_MAJOR(graphics_requirements.maxApiVersionSupported), + XR_VERSION_MINOR(graphics_requirements.maxApiVersionSupported)); + + armed_.store(true, std::memory_order_release); + Log(" [vr-backend] armed: OpenXR instance/system ready; awaiting the " + "guest Vulkan device\n"); + return Status::Ok(); + } catch (const std::exception& error) { + DisarmLocked(); + return Status::Error(StatusCode::kUnavailable, error.what()); + } +} + +void OpenXrBackend::Disarm() { + std::lock_guard lock(mutex_); + DisarmLocked(); +} + +void OpenXrBackend::DisarmLocked() { + TeardownSessionLocked("disarm"); + if (xr_instance_ != nullptr) { + const auto instance = static_cast(xr_instance_); + (void)xrDestroyInstance(instance); + xr_instance_ = nullptr; + } + xr_system_ = 0; + runtime_name_.clear(); + vk_instance_ = VK_NULL_HANDLE; + if (vk_loader_ != nullptr) { + dlclose(vk_loader_); + vk_loader_ = nullptr; + } + armed_.store(false, std::memory_order_release); + recording_.store(false, std::memory_order_release); + eyes_bound_.store(false, std::memory_order_release); + { + std::lock_guard slot_lock(g_backend_slot_mutex); + if (g_backend == this) { + g_backend = nullptr; + } + } +} + +std::string OpenXrBackend::runtime_name() const { + std::lock_guard lock(mutex_); + // Captured once at xrCreateInstance; reporting the cached value avoids a + // second xrGetInstanceProperties call and re-entrancy from the bind-time + // diagnostics that already hold the mutex. + return runtime_name_; +} + +void* OpenXrBackend::ResolveFunction(const char* name) const { + PFN_xrVoidFunction function = nullptr; + CheckXr(static_cast(xr_instance_), + xrGetInstanceProcAddr(static_cast(xr_instance_), name, + &function), + name); + Require(function != nullptr, std::string("Missing OpenXR function: ") + name); + return reinterpret_cast(function); +} + +bool OpenXrBackend::CreateVulkanInstance( + const VkInstanceCreateInfo* create_info, + const VkAllocationCallbacks* allocator, VkInstance* instance, + VkResult* result) { + std::lock_guard lock(mutex_); + if (!armed_.load(std::memory_order_acquire) || xr_instance_ == nullptr || + vk_loader_ == nullptr) { + return false; + } + try { + auto loader_gipa = reinterpret_cast( + dlsym(vk_loader_, "vkGetInstanceProcAddr")); + Require(loader_gipa != nullptr, "Host vkGetInstanceProcAddr is missing"); + auto xr_create = reinterpret_cast( + ResolveFunction("xrCreateVulkanInstanceKHR")); + auto xr_info = XrInfo( + XR_TYPE_VULKAN_INSTANCE_CREATE_INFO_KHR); + xr_info.systemId = xr_system_; + xr_info.pfnGetInstanceProcAddr = loader_gipa; + xr_info.vulkanCreateInfo = create_info; + VkInstance created = VK_NULL_HANDLE; + VkResult vk_result = VK_ERROR_INITIALIZATION_FAILED; + CheckXr(static_cast(xr_instance_), + xr_create(static_cast(xr_instance_), &xr_info, + &created, &vk_result), + "xrCreateVulkanInstanceKHR"); + if (vk_result != VK_SUCCESS) { + *result = vk_result; + *instance = VK_NULL_HANDLE; + return true; + } + vk_instance_ = created; + *instance = created; + *result = VK_SUCCESS; + Log(" [vr-backend] guest VkInstance created through the OpenXR runtime\n"); + return true; + } catch (const std::exception& error) { + Log(" [vr-backend] XR instance creation hook failed: %s; falling back " + "to the plain host loader path\n", + error.what()); + return false; + } +} + +bool OpenXrBackend::CreateVulkanDevice(VkPhysicalDevice physical_device, + const VkDeviceCreateInfo* create_info, + const VkAllocationCallbacks* allocator, + VkDevice* device, VkResult* result) { + std::lock_guard lock(mutex_); + if (!armed_.load(std::memory_order_acquire) || xr_instance_ == nullptr || + vk_instance_ == VK_NULL_HANDLE || vk_loader_ == nullptr) { + return false; + } + try { + auto loader_gipa = reinterpret_cast( + dlsym(vk_loader_, "vkGetInstanceProcAddr")); + Require(loader_gipa != nullptr, "Host vkGetInstanceProcAddr is missing"); + + auto get_graphics_device = + reinterpret_cast( + ResolveFunction("xrGetVulkanGraphicsDevice2KHR")); + auto device_get_info = XrInfo( + XR_TYPE_VULKAN_GRAPHICS_DEVICE_GET_INFO_KHR); + device_get_info.systemId = xr_system_; + device_get_info.vulkanInstance = vk_instance_; + VkPhysicalDevice runtime_device = VK_NULL_HANDLE; + CheckXr(static_cast(xr_instance_), + get_graphics_device(static_cast(xr_instance_), + &device_get_info, &runtime_device), + "xrGetVulkanGraphicsDevice2KHR"); + if (runtime_device != physical_device) { + *device = VK_NULL_HANDLE; + *result = VK_ERROR_INCOMPATIBLE_DRIVER; + Log(" [vr-backend] refusing a guest GPU different from the OpenXR GPU\n"); + return true; + } + + auto xr_create = reinterpret_cast( + ResolveFunction("xrCreateVulkanDeviceKHR")); + auto xr_info = XrInfo( + XR_TYPE_VULKAN_DEVICE_CREATE_INFO_KHR); + xr_info.systemId = xr_system_; + xr_info.pfnGetInstanceProcAddr = loader_gipa; + xr_info.vulkanPhysicalDevice = physical_device; + xr_info.vulkanCreateInfo = create_info; + VkDevice created = VK_NULL_HANDLE; + VkResult vk_result = VK_ERROR_INITIALIZATION_FAILED; + CheckXr(static_cast(xr_instance_), + xr_create(static_cast(xr_instance_), &xr_info, &created, + &vk_result), + "xrCreateVulkanDeviceKHR"); + if (vk_result != VK_SUCCESS) { + *result = vk_result; + *device = VK_NULL_HANDLE; + return true; + } + *device = created; + *result = VK_SUCCESS; + // From here the runtime-created device is the guest device; never fall + // back to a second plain vkCreateDevice. Session setup failures degrade + // to the native-stereo diagnostic path with an explicit log. + try { + SetupDeviceAfterCreationLocked(physical_device, created, create_info); + } catch (const std::exception& setup_error) { + Log(" [vr-backend] XR OUTPUT UNAVAILABLE: %s. Roblox continues in " + "native-stereo diagnostic mode.\n", + setup_error.what()); + } + return true; + } catch (const std::exception& error) { + Log(" [vr-backend] XR device creation hook failed: %s; falling back to " + "the plain host loader path\n", + error.what()); + return false; + } +} + +void OpenXrBackend::SetupDeviceAfterCreationLocked( + VkPhysicalDevice physical_device, VkDevice created, + const VkDeviceCreateInfo* create_info) { + vk_device_ = created; + vk_physical_device_ = physical_device; + auto loader_gipa = reinterpret_cast( + dlsym(vk_loader_, "vkGetInstanceProcAddr")); + Require(loader_gipa != nullptr, "Host vkGetInstanceProcAddr is missing"); + delete vk_; + vk_ = new VkProcs(); + vk_->get_instance_proc_addr = loader_gipa; + vk_->get_device_proc_addr = + reinterpret_cast(loader_gipa( + vk_instance_, "vkGetDeviceProcAddr")); + Require(vk_->get_device_proc_addr != nullptr, + "Host vkGetDeviceProcAddr is missing"); +#define MOCKTAIL_BACKEND_VK_FN(name) \ + vk_->name = reinterpret_cast( \ + vk_->get_device_proc_addr(created, "vk" #name)); \ + Require(vk_->name != nullptr, "Missing Vulkan function: vk" #name); + MOCKTAIL_BACKEND_VK_FN(CreateBuffer) + MOCKTAIL_BACKEND_VK_FN(DestroyBuffer) + MOCKTAIL_BACKEND_VK_FN(GetBufferMemoryRequirements) + MOCKTAIL_BACKEND_VK_FN(AllocateMemory) + MOCKTAIL_BACKEND_VK_FN(FreeMemory) + MOCKTAIL_BACKEND_VK_FN(BindBufferMemory) + MOCKTAIL_BACKEND_VK_FN(MapMemory) + MOCKTAIL_BACKEND_VK_FN(UnmapMemory) + MOCKTAIL_BACKEND_VK_FN(CmdCopyImageToBuffer) + MOCKTAIL_BACKEND_VK_FN(CreateImageView) + MOCKTAIL_BACKEND_VK_FN(DestroyImageView) + MOCKTAIL_BACKEND_VK_FN(CreateCommandPool) + MOCKTAIL_BACKEND_VK_FN(DestroyCommandPool) + MOCKTAIL_BACKEND_VK_FN(AllocateCommandBuffers) + MOCKTAIL_BACKEND_VK_FN(ResetCommandBuffer) + MOCKTAIL_BACKEND_VK_FN(BeginCommandBuffer) + MOCKTAIL_BACKEND_VK_FN(EndCommandBuffer) + MOCKTAIL_BACKEND_VK_FN(CmdPipelineBarrier) + MOCKTAIL_BACKEND_VK_FN(CmdCopyImage) + MOCKTAIL_BACKEND_VK_FN(CmdBlitImage) + MOCKTAIL_BACKEND_VK_FN(CreateFence) + MOCKTAIL_BACKEND_VK_FN(DestroyFence) + MOCKTAIL_BACKEND_VK_FN(ResetFences) + MOCKTAIL_BACKEND_VK_FN(WaitForFences) + MOCKTAIL_BACKEND_VK_FN(QueueSubmit) + MOCKTAIL_BACKEND_VK_FN(DeviceWaitIdle) +#undef MOCKTAIL_BACKEND_VK_FN +#define MOCKTAIL_BACKEND_VK_INSTANCE_FN(name) \ + vk_->name = reinterpret_cast(loader_gipa( \ + vk_instance_, "vk" #name)); \ + Require(vk_->name != nullptr, "Missing Vulkan function: vk" #name); + MOCKTAIL_BACKEND_VK_INSTANCE_FN(GetPhysicalDeviceMemoryProperties) + MOCKTAIL_BACKEND_VK_INSTANCE_FN(GetPhysicalDeviceQueueFamilyProperties) + MOCKTAIL_BACKEND_VK_INSTANCE_FN(GetPhysicalDeviceProperties) +#undef MOCKTAIL_BACKEND_VK_INSTANCE_FN + + VkPhysicalDeviceProperties device_properties{}; + vk_->GetPhysicalDeviceProperties(physical_device, &device_properties); + + std::string error; + if (!InitializeSessionLocked(physical_device, created, create_info, + &error)) { + Log(" [vr-backend] XR OUTPUT UNAVAILABLE on GPU '%s': %s. Roblox " + "continues in native-stereo diagnostic mode.\n", + device_properties.deviceName, error.c_str()); + return; + } + // One-time plain-text diagnostics required for headset acceptance: which + // runtime answered, which pose source drives the camera, and which GPU owns + // the session. No credentials, tokens or launch parameters are included. + Log(" [vr-backend] XR session bound to guest VkDevice: gpu=%s runtime=%s " + "pose_source=%s\n", + device_properties.deviceName, runtime_name_.c_str(), + EnvString("MOCKTAIL_VR_POSE_SOURCE", "runtime").c_str()); +} + +bool OpenXrBackend::InitializeSessionLocked( + VkPhysicalDevice physical_device, VkDevice device, + const VkDeviceCreateInfo* create_info, std::string* error) { + try { + // Pick the queue family the guest gave graphics queues on; the XR binding + // must name a family of this very device. + std::uint32_t family_count = 0; + vk_->GetPhysicalDeviceQueueFamilyProperties(physical_device, &family_count, + nullptr); + Require(family_count > 0 && family_count <= 256, + "the GPU has no usable queue families"); + std::vector families(family_count); + vk_->GetPhysicalDeviceQueueFamilyProperties(physical_device, &family_count, + families.data()); + std::uint32_t chosen_family = create_info ? UINT32_MAX : vk_queue_family_; + for (std::uint32_t index = 0; create_info && index < create_info->queueCreateInfoCount; + ++index) { + const VkDeviceQueueCreateInfo& queue_info = + create_info->pQueueCreateInfos[index]; + if (queue_info.queueFamilyIndex >= family_count || + queue_info.queueCount == 0) { + continue; + } + if (families[queue_info.queueFamilyIndex].queueFlags & + VK_QUEUE_GRAPHICS_BIT) { + chosen_family = queue_info.queueFamilyIndex; + break; + } + } + Require(chosen_family < family_count && (families[chosen_family].queueFlags & VK_QUEUE_GRAPHICS_BIT), + "the guest device requested no graphics queue family"); + vk_queue_family_ = chosen_family; + + auto binding = XrInfo( + XR_TYPE_GRAPHICS_BINDING_VULKAN2_KHR); + binding.instance = vk_instance_; + binding.physicalDevice = physical_device; + binding.device = device; + binding.queueFamilyIndex = chosen_family; + binding.queueIndex = 0; + auto session_info = + XrInfo(XR_TYPE_SESSION_CREATE_INFO); + session_info.next = &binding; + session_info.systemId = xr_system_; + XrSession session = XR_NULL_HANDLE; + CheckXr(static_cast(xr_instance_), + xrCreateSession(static_cast(xr_instance_), + &session_info, &session), + "xrCreateSession"); + xr_session_ = session; + + auto space_info = XrInfo( + XR_TYPE_REFERENCE_SPACE_CREATE_INFO); + space_info.poseInReferenceSpace.orientation.w = 1; + space_info.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_LOCAL; + XrSpace local_space = XR_NULL_HANDLE; + CheckXr(static_cast(xr_instance_), + xrCreateReferenceSpace(session, &space_info, &local_space), + "xrCreateReferenceSpace LOCAL"); + xr_local_space_ = local_space; + space_info.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_VIEW; + XrSpace view_space = XR_NULL_HANDLE; + CheckXr(static_cast(xr_instance_), + xrCreateReferenceSpace(session, &space_info, &view_space), + "xrCreateReferenceSpace VIEW"); + xr_view_space_ = view_space; + + auto system_properties = + XrInfo(XR_TYPE_SYSTEM_PROPERTIES); + CheckXr(static_cast(xr_instance_), + xrGetSystemProperties(static_cast(xr_instance_), + xr_system_, &system_properties), + "xrGetSystemProperties"); + Log(" [vr-backend] system '%s' max swapchain %ux%u views=%u\n", + system_properties.systemName, + system_properties.graphicsProperties.maxSwapchainImageWidth, + system_properties.graphicsProperties.maxSwapchainImageHeight, + system_properties.graphicsProperties.maxLayerCount); + + std::uint32_t blend_count = 0; + CheckXr(static_cast(xr_instance_), + xrEnumerateEnvironmentBlendModes( + static_cast(xr_instance_), xr_system_, + XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, 0, &blend_count, + nullptr), + "xrEnumerateEnvironmentBlendModes count"); + Require(blend_count > 0 && blend_count <= 16, + "the runtime exposes no environment blend modes"); + std::array blends{}; + CheckXr(static_cast(xr_instance_), + xrEnumerateEnvironmentBlendModes( + static_cast(xr_instance_), xr_system_, + XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, blend_count, + &blend_count, blends.data()), + "xrEnumerateEnvironmentBlendModes"); + xr_blend_mode_ = XR_ENVIRONMENT_BLEND_MODE_OPAQUE; + for (std::uint32_t index = 0; index < blend_count; ++index) { + if (blends[index] == XR_ENVIRONMENT_BLEND_MODE_OPAQUE) { + xr_blend_mode_ = XR_ENVIRONMENT_BLEND_MODE_OPAQUE; + break; + } + xr_blend_mode_ = static_cast(blends[index]); + } + + if (!CreateSwapchains(error)) { + TeardownSessionLocked("swapchain failure", session_recovery_pending_); + return false; + } + recording_.store(true, std::memory_order_release); + return true; + } catch (const std::exception& exception) { + *error = exception.what(); + TeardownSessionLocked("session init failure", session_recovery_pending_); + return false; + } +} + +bool OpenXrBackend::CreateSwapchains(std::string* error) { + try { + const auto session = static_cast(xr_session_); + std::uint32_t format_count = 0; + CheckXr(static_cast(xr_instance_), + xrEnumerateSwapchainFormats(session, 0, &format_count, nullptr), + "xrEnumerateSwapchainFormats count"); + Require(format_count > 0 && format_count <= 256, + "the runtime exposes no swapchain formats"); + std::vector formats(format_count); + CheckXr(static_cast(xr_instance_), + xrEnumerateSwapchainFormats(session, format_count, &format_count, + formats.data()), + "xrEnumerateSwapchainFormats"); + // The swapchain format must accept the guest eye content. Prefer the + // common 8-bit formats; the final choice is cross-checked against the + // recorded eye images before the first copy (blit converts if needed). + VkFormat chosen = VK_FORMAT_UNDEFINED; + for (const auto candidate : + {VK_FORMAT_B8G8R8A8_UNORM, VK_FORMAT_R8G8B8A8_UNORM, + VK_FORMAT_B8G8R8A8_SRGB, VK_FORMAT_R8G8B8A8_SRGB}) { + if (std::find(formats.begin(), formats.end(), + static_cast(candidate)) != formats.end()) { + chosen = candidate; + break; + } + } + Require(chosen != VK_FORMAT_UNDEFINED, + "the runtime exposes no 8-bit RGBA swapchain format"); + + // Default eye extent until the guest eye images are bound; the simulator + // runtime advertises 500x500-capable limits, and DebugDeviceVR renders + // 500x500. Re-created lazily if the recorded eye extent differs. + const std::uint32_t width = 500; + const std::uint32_t height = 500; + for (int eye = 0; eye < 2; ++eye) { + auto create_info = + XrInfo(XR_TYPE_SWAPCHAIN_CREATE_INFO); + create_info.usageFlags = XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT | + XR_SWAPCHAIN_USAGE_TRANSFER_DST_BIT; + if (!EnvString("MOCKTAIL_VR_XR_EVIDENCE_DIR", "").empty()) + create_info.usageFlags |= XR_SWAPCHAIN_USAGE_TRANSFER_SRC_BIT; + create_info.format = static_cast(chosen); + create_info.sampleCount = 1; + create_info.width = width; + create_info.height = height; + create_info.faceCount = 1; + create_info.arraySize = 1; + create_info.mipCount = 1; + XrSwapchain swapchain = XR_NULL_HANDLE; + CheckXr(static_cast(xr_instance_), + xrCreateSwapchain(session, &create_info, &swapchain), + "xrCreateSwapchain"); + std::uint32_t image_count = 0; + CheckXr(static_cast(xr_instance_), + xrEnumerateSwapchainImages(swapchain, 0, &image_count, nullptr), + "xrEnumerateSwapchainImages count"); + Require(image_count > 0 && image_count <= 16, + "the runtime returned an empty swapchain"); + std::vector images(image_count); + for (auto& image : images) { + image.type = XR_TYPE_SWAPCHAIN_IMAGE_VULKAN_KHR; + } + CheckXr(static_cast(xr_instance_), + xrEnumerateSwapchainImages( + swapchain, image_count, &image_count, + reinterpret_cast(images.data())), + "xrEnumerateSwapchainImages"); + EyeSwapchain& slot = eye_swapchains_[eye]; + slot.swapchain = swapchain; + slot.images.clear(); + for (const auto& image : images) { + slot.images.push_back(image.image); + } + slot.width = width; + slot.height = height; + slot.format = chosen; + Log(" [vr-backend] eye%d swapchain: %ux%u format=%d images=%u\n", eye, + width, height, static_cast(chosen), image_count); + } + return true; + } catch (const std::exception& exception) { + *error = exception.what(); + return false; + } +} + +void OpenXrBackend::DestroySwapchains() { + for (EyeSwapchain& slot : eye_swapchains_) { + slot.images.clear(); + if (slot.swapchain != nullptr) { + (void)xrDestroySwapchain(static_cast(slot.swapchain)); + slot.swapchain = nullptr; + } + slot.image_acquired = false; + slot.acquired_index = 0; + } +} + +void OpenXrBackend::TeardownSessionLocked(const char* reason, bool preserve_device) { + if (xr_session_ != nullptr) { + const auto session = static_cast(xr_session_); + if (frame_open_) { + auto end = XrInfo(XR_TYPE_FRAME_END_INFO); + end.displayTime = frame_display_time_; + end.environmentBlendMode = static_cast(xr_blend_mode_); + (void)xrEndFrame(session, &end); + frame_open_ = false; + } + // Destruction also ends a running session; xrEndSession is legal only in + // STOPPING (handled by PollSessionEvents). + if (vk_ && vk_device_) { + (void)vk_->DeviceWaitIdle(vk_device_); + // Mirror staging lives on this device; free it before the device goes. + mirror_.DestroyStaging(MirrorProcs()); + } + mirror_.Close(); + DestroyCopyResources(); + DestroySwapchains(); + if (xr_view_space_ != nullptr) { + (void)xrDestroySpace(static_cast(xr_view_space_)); + xr_view_space_ = nullptr; + } + if (xr_local_space_ != nullptr) { + (void)xrDestroySpace(static_cast(xr_local_space_)); + xr_local_space_ = nullptr; + } + (void)xrDestroySession(session); + xr_session_ = nullptr; + Log(" [vr-backend] XR session torn down (%s)\n", reason); + } + session_running_.store(false, std::memory_order_relaxed); + recording_.store(false, std::memory_order_release); + eyes_bound_.store(false, std::memory_order_release); + frame_open_ = false; + frame_views_valid_ = false; + visibility_lost_ = false; + canted_rejection_logged_ = false; + eyes_[0] = {}; + eyes_[1] = {}; + applied_poses_.clear(); + published_pose_.valid = false; + if (preserve_device) { + // Resource callbacks must continue tracking destruction/recreation while + // disconnected. Keep provenance, never resurrect a destroyed guest image. + recording_.store(true, std::memory_order_release); + session_recovery_pending_ = true; + return; + } + session_recovery_pending_ = false; + next_session_retry_ns_ = 0; + images_.clear(); + image_views_.clear(); + framebuffer_views_.clear(); + framebuffer_render_pass_.clear(); + render_pass_final_layouts_.clear(); + vk_device_ = VK_NULL_HANDLE; + vk_physical_device_ = VK_NULL_HANDLE; + queue_families_.clear(); + delete vk_; + vk_ = nullptr; +} + +void OpenXrBackend::NoteInstanceDestroyed(VkInstance instance) { + std::lock_guard lock(mutex_); + if (instance == vk_instance_) { + TeardownSessionLocked("guest VkInstance destroyed"); + vk_instance_ = VK_NULL_HANDLE; + } +} + +void OpenXrBackend::NoteDeviceDestroyed(VkDevice device) { + std::lock_guard lock(mutex_); + if (device != VK_NULL_HANDLE && device == vk_device_) { + // Tear down while the device is still valid; the adapter notifies before + // the host destroy call. + TeardownSessionLocked("guest VkDevice destroyed"); + } +} + +void OpenXrBackend::NoteQueue(VkDevice device, VkQueue queue, + std::uint32_t family, std::uint32_t index) { + static_cast(index); + std::lock_guard lock(mutex_); + if (device == vk_device_ && queue != VK_NULL_HANDLE) { + queue_families_[queue] = family; + } +} + +void OpenXrBackend::RecordImage(VkDevice device, VkImage image, + const VkImageCreateInfo* info) { + if (!recording_.load(std::memory_order_acquire) || info == nullptr) { + return; + } + void* owner = mocktail_vr_initializing_object(); + if (owner == nullptr) { + return; + } + std::lock_guard lock(mutex_); + ImageRecord record{}; + record.image = image; + record.device = device; + record.width = info->extent.width; + record.height = info->extent.height; + record.format = info->format; + record.usage = info->usage; + record.owner = owner; + record.order = ++image_order_; + images_[image] = record; +} + +void OpenXrBackend::RecordImageView(VkImageView view, VkImage image) { + if (!recording_.load(std::memory_order_acquire)) { + return; + } + std::lock_guard lock(mutex_); + image_views_[view] = image; +} + +void OpenXrBackend::RecordFramebuffer(VkFramebuffer framebuffer, + const VkFramebufferCreateInfo* info) { + if (!recording_.load(std::memory_order_acquire) || info == nullptr) { + return; + } + std::lock_guard lock(mutex_); + std::vector views; + views.reserve(info->attachmentCount); + for (std::uint32_t index = 0; index < info->attachmentCount; ++index) { + views.push_back(info->pAttachments[index]); + } + framebuffer_views_[framebuffer] = std::move(views); + framebuffer_render_pass_[framebuffer] = info->renderPass; +} + +void OpenXrBackend::RecordRenderPass(VkRenderPass render_pass, + const VkRenderPassCreateInfo* info) { + if (!recording_.load(std::memory_order_acquire) || info == nullptr) { + return; + } + std::lock_guard lock(mutex_); + std::vector layouts; + layouts.reserve(info->attachmentCount); + for (std::uint32_t index = 0; index < info->attachmentCount; ++index) { + layouts.push_back(info->pAttachments[index].finalLayout); + } + render_pass_final_layouts_[render_pass] = std::move(layouts); +} + +void OpenXrBackend::NoteDestroyImage(VkImage image) { + if (!recording_.load(std::memory_order_acquire)) { + return; + } + std::lock_guard lock(mutex_); + images_.erase(image); + for (int index = 0; index < 2; ++index) { + if (eyes_[index].valid && eyes_[index].image == image) { + Log(" [vr-backend] eye%d image destroyed; eye binding reset\n", index); + eyes_[index] = {}; + eyes_bound_.store(false, std::memory_order_release); + } + } +} + +void OpenXrBackend::NoteDestroyImageView(VkImageView view) { + if (!recording_.load(std::memory_order_acquire)) { + return; + } + std::lock_guard lock(mutex_); + image_views_.erase(view); +} + +void OpenXrBackend::NoteDestroyFramebuffer(VkFramebuffer framebuffer) { + if (!recording_.load(std::memory_order_acquire)) { + return; + } + std::lock_guard lock(mutex_); + framebuffer_views_.erase(framebuffer); + framebuffer_render_pass_.erase(framebuffer); + for (int index = 0; index < 2; ++index) { + if (eyes_[index].valid && eyes_[index].framebuffer == framebuffer) { + Log(" [vr-backend] eye%d framebuffer destroyed; eye binding reset\n", + index); + eyes_[index] = {}; + eyes_bound_.store(false, std::memory_order_release); + } + } +} + +void OpenXrBackend::NoteRenderPassBegin(VkCommandBuffer command_buffer, + const VkRenderPassBeginInfo* info) { + static_cast(command_buffer); + if (!recording_.load(std::memory_order_acquire) || info == nullptr) { + return; + } + std::lock_guard lock(mutex_); + void* owner = nullptr; + void* guest_framebuffer = nullptr; + int eye_index = -1; + if (!mocktail_vr_current_eye(&owner, &eye_index, &guest_framebuffer) || + eye_index < 0 || eye_index > 1) { + return; + } + const auto attachments = framebuffer_views_.find(info->framebuffer); + if (attachments == framebuffer_views_.end()) { + return; + } + const ImageRecord* target = nullptr; + bool ambiguous = false; + std::size_t target_attachment = 0; + for (std::size_t index = 0; index < attachments->second.size(); ++index) { + const auto source = image_views_.find(attachments->second[index]); + if (source == image_views_.end()) { + continue; + } + const auto image = images_.find(source->second); + if (image == images_.end() || image->second.owner != owner || image->second.device != vk_device_ || + !IsColorFormat(image->second.format) || + (image->second.usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) == 0 || + (image->second.usage & VK_IMAGE_USAGE_TRANSFER_SRC_BIT) == 0) { + continue; + } + if (target != nullptr) { + ambiguous = true; + } + target = &image->second; + target_attachment = index; + } + if (target == nullptr || ambiguous) { + return; + } + const auto layouts = render_pass_final_layouts_.find(info->renderPass); + if (layouts == render_pass_final_layouts_.end() || + target_attachment >= layouts->second.size() || + layouts->second[target_attachment] == VK_IMAGE_LAYOUT_UNDEFINED) return; + EyeBinding& binding = eyes_[eye_index]; + if ((eyes_[0].valid && eyes_[0].owner != owner) || + (eyes_[1].valid && eyes_[1].owner != owner)) { + // A new DebugDeviceVR generation replaced the old one; drop both eyes and + // rebind from the live provenance. + eyes_[0] = {}; + eyes_[1] = {}; + eyes_bound_.store(false, std::memory_order_release); + } + const bool changed = !binding.valid || binding.image != target->image || + binding.framebuffer != info->framebuffer; + binding.valid = true; + binding.frame = published_pose_.frame; + binding.image = target->image; + binding.framebuffer = info->framebuffer; + binding.owner = owner; + binding.width = target->width; + binding.height = target->height; + binding.format = target->format; + binding.final_layout = layouts->second[target_attachment]; + if (changed) Log(" [vr-backend][evidence] eye-bound eye=%d owner=%p guest_fb=%p " + "vk_fb=%p vk_image=%p extent=%ux%u format=%d final_layout=%d " + "provenance=guest_initializer_and_eye_getter\n", + eye_index, owner, guest_framebuffer, + static_cast(info->framebuffer), static_cast(target->image), + target->width, target->height, static_cast(target->format), + static_cast(binding.final_layout)); + // Evidence layer also observes this request; do not consume it here. + if (eyes_[0].valid && eyes_[1].valid && + eyes_[0].image != eyes_[1].image && eyes_[0].owner == eyes_[1].owner && + eyes_[0].width > 0 && eyes_[1].width > 0 && + eyes_[0].width == eyes_[1].width && + eyes_[0].height == eyes_[1].height) { + eyes_bound_.store(true, std::memory_order_release); + if (changed) Log(" [vr-backend] both eyes bound and verified distinct\n"); + } +} + +void OpenXrBackend::NotePoseApplied(void* owner, std::uint64_t frame) { + std::lock_guard lock(mutex_); + if (published_pose_.valid && published_pose_.frame == frame) + applied_poses_[owner] = frame; +} + +ScriptedPoseSample OpenXrBackend::ComputeScriptedPose( + std::uint64_t frame) const { + // Deterministic head motion: position translates along X and Z, yaw and + // pitch rotate independently. Periods/amplitudes are env-configurable so + // acceptance runs can verify each axis separately. + const double period_position = + static_cast(std::max(1, EnvUnsigned("MOCKTAIL_VR_POSE_PERIOD_POSITION", 240))); + const double period_rotation = + static_cast(std::max(1, EnvUnsigned("MOCKTAIL_VR_POSE_PERIOD_ROTATION", 360))); + const double amplitude_position = + EnvDouble("MOCKTAIL_VR_POSE_AMPLITUDE_POSITION", 0.25); + const double amplitude_yaw = + EnvDouble("MOCKTAIL_VR_POSE_AMPLITUDE_YAW", 0.5); // ~28.6 degrees + const double amplitude_pitch = + EnvDouble("MOCKTAIL_VR_POSE_AMPLITUDE_PITCH", 0.17); // ~10 degrees + const double phase_position = + 2.0 * M_PI * static_cast(frame % static_cast(period_position)) / + period_position; + const double phase_rotation = + 2.0 * M_PI * static_cast(frame % static_cast(period_rotation)) / + period_rotation; + ScriptedPoseSample sample; + sample.position[0] = + static_cast(amplitude_position * std::sin(phase_position)); + sample.position[1] = 0.f; + sample.position[2] = + static_cast(-0.3 * amplitude_position * std::cos(phase_position)); + const float yaw = static_cast(amplitude_yaw * std::sin(phase_rotation)); + const float pitch = + static_cast(amplitude_pitch * std::sin(2.0 * phase_rotation)); + const Quat orientation = QuatFromYawPitch(yaw, pitch); + sample.orientation[0] = orientation.x; + sample.orientation[1] = orientation.y; + sample.orientation[2] = orientation.z; + sample.orientation[3] = orientation.w; + sample.frame = frame; + sample.valid = true; + return sample; +} + +ScriptedPoseSample OpenXrBackend::PublishedHeadPose() const { + std::lock_guard lock(mutex_); + return published_pose_; +} + +void OpenXrBackend::PollSessionEvents() { + if (xr_instance_ == nullptr) { + return; + } + const auto instance = static_cast(xr_instance_); + while (true) { + auto event = XrInfo(XR_TYPE_EVENT_DATA_BUFFER); + const XrResult result = xrPollEvent(instance, &event); + if (result == XR_EVENT_UNAVAILABLE) { + break; + } + if (XR_FAILED(result)) { + Log(" [vr-backend] xrPollEvent failed: %s\n", + XrResultText(instance, result).c_str()); + break; + } + if (event.type == XR_TYPE_EVENT_DATA_INSTANCE_LOSS_PENDING) { + Log(" [vr-backend] OpenXR instance loss pending; XR output stopped\n"); + TeardownSessionLocked("instance loss pending"); + return; + } + if (event.type == XR_TYPE_EVENT_DATA_REFERENCE_SPACE_CHANGE_PENDING) { + const auto& change = + reinterpret_cast(event); + const bool affects_us = + change.session == static_cast(xr_session_) && + (change.referenceSpaceType == XR_REFERENCE_SPACE_TYPE_LOCAL || + change.referenceSpaceType == XR_REFERENCE_SPACE_TYPE_VIEW); + // poseInPreviousSpace is undefined when poseValid is false. No application + // origin offset is applied: xrLocate* uses the requested display time. + Log(" [vr-backend] reference space change pending: type=%d time=%lld poseValid=%d\n", + static_cast(change.referenceSpaceType), + static_cast(change.changeTime), change.poseValid); + if (affects_us) HandleReferenceSpaceChange(change.changeTime); + perf::ProcessCollector().Count(perf::Counter::kReferenceSpaceChanges); + continue; + } + if (event.type == XR_TYPE_EVENT_DATA_INTERACTION_PROFILE_CHANGED) { + // Controllers are out of scope; recorded so the log shows the runtime + // offered a profile change rather than silently ignoring it. + Log(" [vr-backend] interaction profile changed (controller input is not " + "implemented in this build)\n"); + continue; + } + if (event.type != XR_TYPE_EVENT_DATA_SESSION_STATE_CHANGED) { + continue; + } + const auto& changed = + reinterpret_cast(event); + if (changed.session != static_cast(xr_session_)) { + continue; + } + HandleSessionState(static_cast(changed.state)); + if (xr_session_ == nullptr) return; + } +} + +void OpenXrBackend::HandleSessionState(int state) { + switch (static_cast(state)) { + case XR_SESSION_STATE_READY: { + auto begin = + XrInfo(XR_TYPE_SESSION_BEGIN_INFO); + begin.primaryViewConfigurationType = + XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO; + const XrResult begin_result = + xrBeginSession(static_cast(xr_session_), &begin); + if (XR_SUCCEEDED(begin_result)) { + session_running_.store(true, std::memory_order_release); + visibility_lost_ = false; + Log(" [vr-backend] XR session READY; projection submission " + "starts on the next frame\n"); + } else { + Log(" [vr-backend] xrBeginSession failed: %s\n", + XrResultText(static_cast(xr_instance_), begin_result) + .c_str()); + } + break; + } + case XR_SESSION_STATE_STOPPING: { + if (frame_open_) { + auto end = XrInfo(XR_TYPE_FRAME_END_INFO); + end.displayTime = frame_display_time_; + end.environmentBlendMode = + static_cast(xr_blend_mode_); + (void)xrEndFrame(static_cast(xr_session_), &end); + frame_open_ = false; + } + if (session_running_.load(std::memory_order_relaxed)) { + (void)xrEndSession(static_cast(xr_session_)); + } + session_running_.store(false, std::memory_order_release); + Log(" [vr-backend] XR session STOPPING; submission paused\n"); + break; + } + case XR_SESSION_STATE_FOCUSED: + Log(" [vr-backend] XR session FOCUSED\n"); + if (visibility_lost_) { + // Returning from IDLE to a visible state: the frames rendered while + // idle are stale, so the first frame after regain is invalidated. + InvalidateInFlightPose("visibility regained (FOCUSED)"); + } + visibility_lost_ = false; + perf::ProcessCollector().Count(perf::Counter::kVisibilityGained); + break; + case XR_SESSION_STATE_VISIBLE: + Log(" [vr-backend] XR session VISIBLE (compositor shows our layers " + "without input focus)\n"); + if (visibility_lost_) { + InvalidateInFlightPose("visibility regained (VISIBLE)"); + } + visibility_lost_ = false; + perf::ProcessCollector().Count(perf::Counter::kVisibilityGained); + break; + case XR_SESSION_STATE_SYNCHRONIZED: + case XR_SESSION_STATE_IDLE: + Log(" [vr-backend] XR session not visible (IDLE/SYNCHRONIZED; runtime may " + "throttle frames)\n"); + visibility_lost_ = true; + InvalidateInFlightPose("session not visible"); + perf::ProcessCollector().Count(perf::Counter::kVisibilityLost); + break; + case XR_SESSION_STATE_EXITING: + Log(" [vr-backend] XR session EXITING\n"); + session_running_.store(false, std::memory_order_release); + TeardownSessionLocked("session exiting"); + return; + case XR_SESSION_STATE_LOSS_PENDING: + Log(" [vr-backend] XR session LOSS_PENDING\n"); + session_running_.store(false, std::memory_order_release); + TeardownSessionLocked("session loss pending", true); + return; + default: + break; + } +} + +void OpenXrBackend::InvalidateInFlightPose(const char* reason) { + // Mark the published pose invalid and clear the per-owner application record + // so the frame cycle's can_submit gate fails. The rendered images of the + // in-flight frame stay untouched (the guest owns them); xrEndFrame simply + // submits no projection layer for this frame rather than retagging a stale + // image with a pose describing a different origin. + if (frame_open_ || published_pose_.valid) { + Log(" [vr-backend] in-flight frame invalidated (%s); skipping its " + "projection submission\n", + reason); + } + published_pose_.valid = false; + frame_views_valid_ = false; + applied_poses_.clear(); + perf::ProcessCollector().Count(perf::Counter::kFramesInvalidated); +} + +void OpenXrBackend::HandleReferenceSpaceChange(std::int64_t change_time) { + // A future event does NOT invalidate an earlier predicted frame. Runtime + // locates automatically use the new origin for times >= changeTime. Drop + // only a frame already located when a late notification affects its time. + if (frame_open_ && frame_views_valid_ && + static_cast(frame_display_time_) >= change_time) { + InvalidateInFlightPose("late reference-space change affects rendered frame"); + } +} + +bool OpenXrBackend::RecoverSessionLocked(std::uint64_t now_ns) { + if (!session_recovery_pending_ || vk_device_ == VK_NULL_HANDLE || vk_ == nullptr || + xr_instance_ == nullptr || now_ns < next_session_retry_ns_) return false; + next_session_retry_ns_ = now_ns + 1'000'000'000ULL; + std::string error; + // nullptr means reuse the verified queue family of this existing VkDevice. + if (!InitializeSessionLocked(vk_physical_device_, vk_device_, nullptr, &error)) { + Log(" [vr-backend] waiting to restore XR session: %s\n", error.c_str()); + return false; + } + session_recovery_pending_ = false; + Log(" [vr-backend] XR session recreated on the existing guest device; awaiting READY\n"); + return true; +} + +void OpenXrBackend::NoteHostPresent(VkQueue queue, VkDevice device) { + std::lock_guard lock(mutex_); + if (!armed_.load(std::memory_order_acquire) || device != vk_device_) { + return; + } + try { + if (xr_session_ == nullptr && !RecoverSessionLocked(perf::NowNs())) return; + PollSessionEvents(); + if (!session_running_.load(std::memory_order_acquire)) { + return; + } + perf::ProcessCollector().Count(perf::Counter::kPresents); + RunFrameCycle(queue, device); + // Explicit, one-shot fault injection for the isolated correctness canary. + // Never active in a normal authenticated/headset launch. + const auto loss_frame = EnvUnsigned("MOCKTAIL_VR_TEST_SESSION_LOSS_AFTER", 0); + if (!test_session_loss_injected_ && loss_frame > 0 && + EnvString("MOCKTAIL_ISOLATED_CANARY", "") == "1" && + submitted_frames_.load() >= loss_frame) { + test_session_loss_injected_ = true; + Log(" [vr-backend][test] injecting session loss on existing guest device\n"); + HandleSessionState(XR_SESSION_STATE_LOSS_PENDING); + } + perf::ProcessCollector().EmitIfDue(perf::NowNs(), "xr-frame-cycle"); + } catch (const XrCallError& error) { + if (error.result == XR_ERROR_SESSION_LOST) { + Log(" [vr-backend] XR session lost; scheduling recovery on the existing device\n"); + TeardownSessionLocked("XR_ERROR_SESSION_LOST", true); + } else { + Log(" [vr-backend] XR frame failed: %s; restart the client if the runtime instance was lost\n", + error.what()); + TeardownSessionLocked("unrecoverable XR frame error"); + } + } catch (const std::exception& error) { + Log(" [vr-backend] frame cycle failed: %s; XR output stopped, native " + "stereo continues\n", + error.what()); + TeardownSessionLocked("frame cycle failure"); + } +} + +void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { + const ScopedStageTimer cycle_timer(perf::Stage::kFrameCycle); + const auto session = static_cast(xr_session_); + const auto instance = static_cast(xr_instance_); + + // ---- finish the XR frame whose predicted display time covered the frame + // Roblox just rendered (eye images are complete at present time). + if (frame_open_) { + auto end = XrInfo(XR_TYPE_FRAME_END_INFO); + end.displayTime = frame_display_time_; + end.environmentBlendMode = + static_cast(xr_blend_mode_); + XrCompositionLayerProjection projection = + XrInfo( + XR_TYPE_COMPOSITION_LAYER_PROJECTION); + std::array views{}; + const XrCompositionLayerBaseHeader* layers[1] = { + reinterpret_cast(&projection)}; + const bool eyes_usable = eyes_bound_.load(std::memory_order_acquire) && + eyes_[0].valid && eyes_[1].valid; + const bool pose_ready = frame_views_valid_ && published_pose_.valid; + const bool eye_frames_match = + eyes_usable && pose_ready && + eyes_[0].frame == published_pose_.frame && + eyes_[1].frame == published_pose_.frame && + applied_poses_[eyes_[0].owner] == published_pose_.frame; + const bool can_submit = frame_should_render_ && + pose_ready && eye_frames_match && eyes_usable; + if (!can_submit) { + // One dominant reason per skipped frame, checked in pipeline order. + if (visibility_lost_ && !frame_should_render_) { + perf::ProcessCollector().Count(perf::Counter::kSkipVisibilityLost); + } else if (!frame_should_render_) { + perf::ProcessCollector().Count(perf::Counter::kSkipShouldRender); + } else if (!pose_ready) { + perf::ProcessCollector().Count(perf::Counter::kSkipPoseInvalid); + } else if (!eyes_usable) { + perf::ProcessCollector().Count(perf::Counter::kSkipEyesUnbound); + } else { + perf::ProcessCollector().Count(perf::Counter::kSkipEyeFrameMismatch); + } + } + bool images_held[2] = {false, false}; + if (can_submit) { + projection.space = static_cast(xr_local_space_); + projection.viewCount = 2; + for (int eye = 0; eye < 2; ++eye) { + std::uint32_t index = 0; + { + const ScopedStageTimer acquire_timer(perf::Stage::kSwapchainAcquire); + auto acquire = XrInfo( + XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO); + CheckXr(instance, + xrAcquireSwapchainImage( + static_cast(eye_swapchains_[eye].swapchain), + &acquire, &index), + "xrAcquireSwapchainImage"); + } + { + const ScopedStageTimer wait_timer(perf::Stage::kSwapchainWait); + auto wait = XrInfo( + XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO); + wait.timeout = 50'000'000; + const auto deadline = Clock::now() + std::chrono::seconds(5); + while (true) { + const XrResult wait_result = xrWaitSwapchainImage( + static_cast(eye_swapchains_[eye].swapchain), &wait); + if (wait_result != XR_TIMEOUT_EXPIRED) { + CheckXr(instance, wait_result, "xrWaitSwapchainImage"); + break; + } + Require(Clock::now() < deadline, + "Timed out waiting for an OpenXR swapchain image"); + } + } + images_held[eye] = true; + eye_swapchains_[eye].acquired_index = index; + eye_swapchains_[eye].image_acquired = true; + } + const bool copied = [&] { + const ScopedStageTimer copy_record(perf::Stage::kCopyRecord); + return CopyEyesIntoSwapchains(queue); + }() && [&] { + const ScopedStageTimer evidence_timer(perf::Stage::kEvidence); + return CaptureEyeEvidence(queue, 0) && CaptureEyeEvidence(queue, 1); + }(); + for (int eye = 0; eye < 2; ++eye) { + if (images_held[eye]) { + auto release = XrInfo( + XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO); + (void)xrReleaseSwapchainImage( + static_cast(eye_swapchains_[eye].swapchain), + &release); + eye_swapchains_[eye].image_acquired = false; + } + } + if (copied) { + for (int eye = 0; eye < 2; ++eye) { + views[eye].type = XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW; + XrPosef submit_pose{}; + submit_pose.position = {frame_view_poses_[eye].position[0], + frame_view_poses_[eye].position[1], + frame_view_poses_[eye].position[2]}; + submit_pose.orientation = {frame_view_poses_[eye].orientation[0], + frame_view_poses_[eye].orientation[1], + frame_view_poses_[eye].orientation[2], + frame_view_poses_[eye].orientation[3]}; + views[eye].pose = submit_pose; + views[eye].fov = XrFovf{frame_views_[eye].fov[0], + frame_views_[eye].fov[1], + frame_views_[eye].fov[2], + frame_views_[eye].fov[3]}; + views[eye].subImage.swapchain = + static_cast(eye_swapchains_[eye].swapchain); + views[eye].subImage.imageRect.offset = {0, 0}; + views[eye].subImage.imageRect.extent = { + static_cast(eye_swapchains_[eye].width), + static_cast(eye_swapchains_[eye].height)}; + views[eye].subImage.imageArrayIndex = 0; + } + projection.views = views.data(); + end.layerCount = 1; + end.layers = layers; + } + } + CheckXr(instance, xrEndFrame(session, &end), "xrEndFrame"); + frame_open_ = false; + if (end.layerCount > 0) { + perf::ProcessCollector().Count(perf::Counter::kFramesSubmitted); + const std::uint64_t submitted = + submitted_frames_.fetch_add(1, std::memory_order_relaxed) + 1; + if (submitted == 1 || submitted % 120 == 0 || + (!EnvString("MOCKTAIL_VR_XR_EVIDENCE_DIR", "").empty() && + published_pose_.frame % std::max(1, + EnvUnsigned("MOCKTAIL_VR_XR_EVIDENCE_INTERVAL", 20)) == 0)) { + Log(" [vr-backend][evidence] projection submitted=%llu frame=%llu eye0_image=%p " + "eye1_image=%p eye0_swapchain_index=%u eye1_swapchain_index=%u\n", + static_cast(submitted), + static_cast(published_pose_.frame), + static_cast(eyes_[0].image), + static_cast(eyes_[1].image), + eye_swapchains_[0].acquired_index, + eye_swapchains_[1].acquired_index); + } + } else { + perf::ProcessCollector().Count(perf::Counter::kFramesSkipped); + skipped_frames_.fetch_add(1, std::memory_order_relaxed); + } + } + + // ---- open the XR frame for the Roblox frame that is about to render. + std::uint64_t predicted_display_time = 0; + bool should_render = false; + { + const ScopedStageTimer wait_timer(perf::Stage::kXrWaitFrame); + auto wait = XrInfo(XR_TYPE_FRAME_WAIT_INFO); + XrFrameState state = XrInfo(XR_TYPE_FRAME_STATE); + CheckXr(instance, xrWaitFrame(session, &wait, &state), "xrWaitFrame"); + { + const ScopedStageTimer begin_timer(perf::Stage::kXrBeginFrame); + auto begin_info = XrInfo(XR_TYPE_FRAME_BEGIN_INFO); + CheckXr(instance, xrBeginFrame(session, &begin_info), "xrBeginFrame"); + } + predicted_display_time = state.predictedDisplayTime; + should_render = state.shouldRender; + } + frame_open_ = true; + frame_display_time_ = predicted_display_time; + frame_should_render_ = should_render; + applied_poses_.clear(); + frame_views_valid_ = false; + perf::ProcessCollector().Count(perf::Counter::kFramesBegun); + + if (should_render) { + auto locate_info = XrInfo(XR_TYPE_VIEW_LOCATE_INFO); + locate_info.viewConfigurationType = + XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO; + locate_info.displayTime = predicted_display_time; + locate_info.space = static_cast(xr_local_space_); + XrViewState view_state = XrInfo(XR_TYPE_VIEW_STATE); + std::array xr_views{}; + xr_views[0].type = XR_TYPE_VIEW; + xr_views[1].type = XR_TYPE_VIEW; + std::uint32_t view_count = 0; + { + const ScopedStageTimer locate_timer(perf::Stage::kXrLocateViews); + CheckXr(instance, + xrLocateViews(session, &locate_info, &view_state, 2, &view_count, + xr_views.data()), + "xrLocateViews"); + } + constexpr XrViewStateFlags required_view_flags = + XR_VIEW_STATE_POSITION_VALID_BIT | XR_VIEW_STATE_ORIENTATION_VALID_BIT; + if (view_count == 2 && + (view_state.viewStateFlags & required_view_flags) == + required_view_flags) { + { + // Head pose in LOCAL space for the guest camera injection. + XrSpaceLocation head = XrInfo(XR_TYPE_SPACE_LOCATION); + const XrResult head_result = xrLocateSpace( + static_cast(xr_view_space_), + static_cast(xr_local_space_), predicted_display_time, + &head); + constexpr XrSpaceLocationFlags required_location = + XR_SPACE_LOCATION_ORIENTATION_VALID_BIT | + XR_SPACE_LOCATION_POSITION_VALID_BIT; + const bool head_valid = + XR_SUCCEEDED(head_result) && + (head.locationFlags & required_location) == required_location; + const std::string pose_source = + EnvString("MOCKTAIL_VR_POSE_SOURCE", "runtime"); + Pose head_pose = head_valid ? XrToPose(head.pose) : Pose{}; + if (pose_source == "script") { + const ScriptedPoseSample scripted = + ComputeScriptedPose(++xr_frame_counter_); + head_pose = Pose{Vec3{scripted.position[0], scripted.position[1], + scripted.position[2]}, + Quat{scripted.orientation[0], scripted.orientation[1], + scripted.orientation[2], + scripted.orientation[3]}}; + published_pose_ = scripted; + published_pose_.valid = head_valid; + published_pose_.frame = xr_frame_counter_; + } else { + ScriptedPoseSample sample; + sample.position[0] = head_pose.position.x; + sample.position[1] = head_pose.position.y; + sample.position[2] = head_pose.position.z; + sample.orientation[0] = head_pose.orientation.x; + sample.orientation[1] = head_pose.orientation.y; + sample.orientation[2] = head_pose.orientation.z; + sample.orientation[3] = head_pose.orientation.w; + sample.frame = ++xr_frame_counter_; + sample.valid = head_valid; + published_pose_ = sample; + } + for (int eye = 0; eye < 2; ++eye) { + const Pose runtime_eye = XrToPose(xr_views[eye].pose); + // Preserve the runtime's inter-lens offsets while the scripted (or + // runtime) head drives the motion. + const Pose relative = head_valid || pose_source != "script" + ? PoseMultiply(PoseInverse(head_valid + ? XrToPose(head.pose) + : Pose{}), + runtime_eye) + : runtime_eye; + // Without a valid head pose `relative` degenerates to the full + // LOCAL-space eye pose, which is not an inter-lens offset. Publish + // explicit zeroes so no consumer can read a stale tracking value. + if (head_valid) { + published_pose_.eye_offset[eye][0] = relative.position.x; + published_pose_.eye_offset[eye][1] = relative.position.y; + published_pose_.eye_offset[eye][2] = relative.position.z; + } else { + published_pose_.eye_offset[eye][0] = 0.f; + published_pose_.eye_offset[eye][1] = 0.f; + published_pose_.eye_offset[eye][2] = 0.f; + } + // The pinned guest state represents parallel eye orientations. A + // canted runtime needs a different camera ABI; do not submit an + // image with a projection pose that the guest cannot represent. + // Reject once per binding with the measured components so a tester + // sees why nothing reaches the headset instead of a silent freeze. + // Only judged while the head pose itself is valid: during tracking + // loss `relative` degenerates to the full LOCAL-space eye pose and + // its rotation is not a canted-eye signal. + if (head_valid && + !pose::EyeOrientationIsParallel(relative.orientation)) { + published_pose_.valid = false; + if (!canted_rejection_logged_) { + canted_rejection_logged_ = true; + Log(" [vr-backend] UNSUPPORTED CONFIGURATION: the runtime " + "reports canted eye orientations (eye%d relative rotation " + "xyz=(%.5f,%.5f,%.5f), tolerance %.3f). The exact-build " + "guest camera ABI represents only parallel eye orientations, " + "so XR submission is rejected rather than showing images " + "with a mismatched projection. This is a known unmet " + "compatibility criterion for this headset/runtime " + "combination.\n", + eye, relative.orientation.x, relative.orientation.y, + relative.orientation.z, pose::kCantedTolerance); + } + } + const Pose final_eye = PoseMultiply(head_pose, relative); + frame_view_poses_[eye].position[0] = final_eye.position.x; + frame_view_poses_[eye].position[1] = final_eye.position.y; + frame_view_poses_[eye].position[2] = final_eye.position.z; + frame_view_poses_[eye].orientation[0] = final_eye.orientation.x; + frame_view_poses_[eye].orientation[1] = final_eye.orientation.y; + frame_view_poses_[eye].orientation[2] = final_eye.orientation.z; + frame_view_poses_[eye].orientation[3] = final_eye.orientation.w; + frame_views_[eye].position[0] = final_eye.position.x; + frame_views_[eye].position[1] = final_eye.position.y; + frame_views_[eye].position[2] = final_eye.position.z; + frame_views_[eye].orientation[0] = final_eye.orientation.x; + frame_views_[eye].orientation[1] = final_eye.orientation.y; + frame_views_[eye].orientation[2] = final_eye.orientation.z; + frame_views_[eye].orientation[3] = final_eye.orientation.w; + frame_views_[eye].fov[0] = xr_views[eye].fov.angleLeft; + frame_views_[eye].fov[1] = xr_views[eye].fov.angleRight; + frame_views_[eye].fov[2] = xr_views[eye].fov.angleUp; + frame_views_[eye].fov[3] = xr_views[eye].fov.angleDown; + std::copy_n(frame_views_[eye].fov, 4, published_pose_.eye_fov[eye]); + frame_views_[eye].valid = true; + } + frame_views_valid_ = published_pose_.valid; + // Head-pose evidence cadence is tunable so the scripted-motion canary + // can observe at least two distinct samples within a short run; the + // production default stays coarse to avoid per-frame log spam. + static const std::uint64_t pose_log_interval = [] { + const std::uint64_t value = + EnvUnsigned("MOCKTAIL_VR_POSE_LOG_INTERVAL", 240); + return value == 0 ? 240 : value; + }(); + if (xr_frame_counter_ == 1 || xr_frame_counter_ % pose_log_interval == 0) { + Log(" [vr-backend][evidence] head-pose frame=%llu pos=(%.4f,%.4f," + "%.4f) ori=(%.4f,%.4f,%.4f,%.4f) source=%s predicted=%lld\n", + static_cast(xr_frame_counter_), + published_pose_.position[0], published_pose_.position[1], + published_pose_.position[2], published_pose_.orientation[0], + published_pose_.orientation[1], published_pose_.orientation[2], + published_pose_.orientation[3], pose_source.c_str(), + static_cast(predicted_display_time)); + } + } + } + if (!frame_views_valid_) { + // Invalid tracking is never published as a fresh valid pose. + published_pose_.valid = false; + } + } else { + published_pose_.valid = false; + } +} + +bool OpenXrBackend::EnsureCopyResources(VkDevice device, + std::uint32_t queue_family, + std::string* error) { + try { + if (copy_pool_ != VK_NULL_HANDLE && copy_command_ != VK_NULL_HANDLE && + copy_fence_ != VK_NULL_HANDLE) { + return true; + } + if (copy_pool_ == VK_NULL_HANDLE) { + VkCommandPoolCreateInfo pool_info{}; + pool_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + pool_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + pool_info.queueFamilyIndex = queue_family; + Require(vk_->CreateCommandPool(device, &pool_info, nullptr, + ©_pool_) == VK_SUCCESS, + "vkCreateCommandPool failed for XR copies"); + } + if (copy_command_ == VK_NULL_HANDLE) { + VkCommandBufferAllocateInfo allocate_info{}; + allocate_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + allocate_info.commandPool = copy_pool_; + allocate_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + allocate_info.commandBufferCount = 1; + Require(vk_->AllocateCommandBuffers(device, &allocate_info, + ©_command_) == VK_SUCCESS, + "vkAllocateCommandBuffers failed for XR copies"); + } + if (copy_fence_ == VK_NULL_HANDLE) { + VkFenceCreateInfo fence_info{}; + fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + Require(vk_->CreateFence(device, &fence_info, nullptr, ©_fence_) == + VK_SUCCESS, + "vkCreateFence failed for XR copies"); + } + return true; + } catch (const std::exception& exception) { + *error = exception.what(); + return false; + } +} + +void OpenXrBackend::DestroyCopyResources() { + if (vk_ == nullptr || vk_device_ == VK_NULL_HANDLE) { + copy_pool_ = VK_NULL_HANDLE; + copy_command_ = VK_NULL_HANDLE; + copy_fence_ = VK_NULL_HANDLE; + return; + } + if (copy_fence_ != VK_NULL_HANDLE) { + vk_->DestroyFence(vk_device_, copy_fence_, nullptr); + copy_fence_ = VK_NULL_HANDLE; + } + if (copy_pool_ != VK_NULL_HANDLE) { + // Frees copy_command_ as well. + vk_->DestroyCommandPool(vk_device_, copy_pool_, nullptr); + copy_pool_ = VK_NULL_HANDLE; + copy_command_ = VK_NULL_HANDLE; + } +} + +bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { + std::string error; + const auto family_entry = queue_families_.find(queue); + const std::uint32_t family = + family_entry != queue_families_.end() ? family_entry->second + : vk_queue_family_; + if (!EnsureCopyResources(vk_device_, family, &error)) { + Log(" [vr-backend] copy resources unavailable: %s\n", error.c_str()); + return false; + } + // Both eyes must match their swapchain extents; a mismatch is a hard skip + // of the whole batched submission, never a half-copied frame. + const VkImage source[2] = {eyes_[0].image, eyes_[1].image}; + for (int eye = 0; eye < 2; ++eye) { + const EyeSwapchain& swapchain = eye_swapchains_[eye]; + if (!swapchain.image_acquired || swapchain.images.empty() || + swapchain.acquired_index >= swapchain.images.size()) { + return false; + } + if (eyes_[eye].width != swapchain.width || + eyes_[eye].height != swapchain.height) { + Log(" [vr-backend] eye%d extent %ux%u does not match swapchain %ux%u\n", + eye, eyes_[eye].width, eyes_[eye].height, swapchain.width, + swapchain.height); + return false; + } + } + const VkImage destination[2] = { + eye_swapchains_[0].images[eye_swapchains_[0].acquired_index], + eye_swapchains_[1].images[eye_swapchains_[1].acquired_index]}; + EnsureMirrorForExtent(eyes_[0].width, eyes_[0].height); + + if (vk_->ResetCommandBuffer(copy_command_, 0) != VK_SUCCESS) { + return false; + } + VkCommandBufferBeginInfo begin{}; + begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + if (vk_->BeginCommandBuffer(copy_command_, &begin) != VK_SUCCESS) { + return false; + } + + // Phase 1: transition both sources to TRANSFER_SRC and both swapchain + // destinations to TRANSFER_DST in one barrier batch. + std::array prepare{}; + for (int eye = 0; eye < 2; ++eye) { + VkImageMemoryBarrier& source_barrier = prepare[eye]; + source_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + source_barrier.srcAccessMask = + VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + source_barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + source_barrier.oldLayout = eyes_[eye].final_layout; + source_barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + source_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + source_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + source_barrier.image = source[eye]; + source_barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + + VkImageMemoryBarrier& destination_barrier = prepare[2 + eye]; + destination_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + destination_barrier.srcAccessMask = 0; + destination_barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + destination_barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + destination_barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + destination_barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + destination_barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + destination_barrier.image = destination[eye]; + destination_barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, + 1}; + } + vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, + nullptr, static_cast(prepare.size()), + prepare.data()); + + // Phase 2: both eye copies (plus the mirror readbacks when enabled) in one + // recording. + const VkImageSubresourceLayers subresource{VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, + 1}; + for (int eye = 0; eye < 2; ++eye) { + const EyeBinding& binding = eyes_[eye]; + const EyeSwapchain& swapchain = eye_swapchains_[eye]; + if (FormatIsCopyCompatible(binding.format, swapchain.format)) { + VkImageCopy region{}; + region.srcSubresource = subresource; + region.dstSubresource = subresource; + region.extent = {binding.width, binding.height, 1}; + vk_->CmdCopyImage(copy_command_, source[eye], + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, destination[eye], + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); + } else { + VkImageBlit region{}; + region.srcSubresource = subresource; + region.dstSubresource = subresource; + region.srcOffsets[1] = {static_cast(binding.width), + static_cast(binding.height), 1}; + region.dstOffsets[1] = region.srcOffsets[1]; + vk_->CmdBlitImage(copy_command_, source[eye], + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, destination[eye], + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion, + VK_FILTER_NEAREST); + } + if (mirror_.enabled()) { + // Reads the source while it is already TRANSFER_SRC; the restore + // barrier below returns it to the guest's final layout afterwards. + mirror_.RecordEyeCopy(MirrorProcs(), copy_command_, eye, source[eye], + binding.final_layout, binding.width, + binding.height); + } + } + + // Phase 3: restore the eye images to their render-pass final layouts and + // leave the swapchain images as color attachments for the compositor. + std::array finish{}; + for (int eye = 0; eye < 2; ++eye) { + VkImageMemoryBarrier& restore_source = finish[eye]; + restore_source.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + restore_source.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + restore_source.dstAccessMask = + VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + restore_source.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + restore_source.newLayout = eyes_[eye].final_layout; + restore_source.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + restore_source.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + restore_source.image = source[eye]; + restore_source.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + + VkImageMemoryBarrier& finish_destination = finish[2 + eye]; + finish_destination.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + finish_destination.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + finish_destination.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + finish_destination.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + finish_destination.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + finish_destination.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + finish_destination.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + finish_destination.image = destination[eye]; + finish_destination.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, + 1}; + } + vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, + nullptr, static_cast(finish.size()), + finish.data()); + + if (vk_->EndCommandBuffer(copy_command_) != VK_SUCCESS) { + return false; + } + if (vk_->ResetFences(vk_device_, 1, ©_fence_) != VK_SUCCESS) { + return false; + } + VkSubmitInfo submit{}; + submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit.commandBufferCount = 1; + submit.pCommandBuffers = ©_command_; + // Same queue as the guest's frame submission: queue-order guarantees the + // batched copies run after the eye render passes of this frame. One frame in + // flight; the single fence wait bounds reuse of the copy command buffer and + // of the mirror staging buffers. + VkResult submit_result = VK_SUCCESS; + { + const ScopedStageTimer submit_timer(perf::Stage::kCopySubmit); + submit_result = vk_->QueueSubmit(queue, 1, &submit, copy_fence_); + } + perf::ProcessCollector().Count(perf::Counter::kSubmits); + if (submit_result != VK_SUCCESS) { + return false; + } + VkResult fence_result = VK_SUCCESS; + { + const ScopedStageTimer fence_timer(perf::Stage::kCopyFenceWait); + fence_result = vk_->WaitForFences(vk_device_, 1, ©_fence_, VK_TRUE, + 2'000'000'000ULL); + } + perf::ProcessCollector().Count(perf::Counter::kFenceWaits); + if (fence_result != VK_SUCCESS) { + (void)vk_->DeviceWaitIdle(vk_device_); + throw std::runtime_error("XR copy fence wait failed"); + } + if (mirror_.enabled()) { + const ScopedStageTimer publish_timer(perf::Stage::kMirrorPublish); + mirror_.Publish(published_pose_.frame, eyes_[0].format, eyes_[0].width, + eyes_[0].height, perf::NowNs()); + perf::ProcessCollector().Count(perf::Counter::kMirrorPairs); + } + return true; +} + +void OpenXrBackend::EnsureMirrorForExtent(std::uint32_t width, + std::uint32_t height) { + const std::string name = EnvString("MOCKTAIL_VR_MIRROR_SHM", ""); + if (name.empty()) { + return; + } + { + const bool was_enabled = mirror_.enabled(); + std::string error; + if (!mirror_.Open(name, width, height, &error)) { + Log(" [vr-backend] mirror unavailable: %s\n", error.c_str()); + return; + } + if (!was_enabled) Log(" [vr-backend] desktop mirror publishing to %s (%ux%u)\n", + name.c_str(), width, height); + } + std::string error; + if (!mirror_.EnsureStaging(MirrorProcs(), vk_device_, vk_physical_device_, + width, height, &error)) { + Log(" [vr-backend] mirror staging unavailable: %s\n", error.c_str()); + mirror_.Close(); + } +} + +MirrorVkProcs OpenXrBackend::MirrorProcs() const { + MirrorVkProcs procs{}; + if (vk_ == nullptr) { + return procs; + } + procs.CreateBuffer = vk_->CreateBuffer; + procs.DestroyBuffer = vk_->DestroyBuffer; + procs.GetBufferMemoryRequirements = vk_->GetBufferMemoryRequirements; + procs.AllocateMemory = vk_->AllocateMemory; + procs.FreeMemory = vk_->FreeMemory; + procs.BindBufferMemory = vk_->BindBufferMemory; + procs.MapMemory = vk_->MapMemory; + procs.UnmapMemory = vk_->UnmapMemory; + procs.CmdCopyImageToBuffer = vk_->CmdCopyImageToBuffer; + procs.CmdPipelineBarrier = vk_->CmdPipelineBarrier; + procs.GetPhysicalDeviceMemoryProperties = + vk_->GetPhysicalDeviceMemoryProperties; + return procs; +} + +// Diagnostic only: read BOTH images on the guest queue while the XR image is +// still acquired. Never read a released image owned by the compositor. +bool OpenXrBackend::CaptureEyeEvidence(VkQueue queue, int eye) { + const std::string directory = EnvString("MOCKTAIL_VR_XR_EVIDENCE_DIR", ""); + const auto interval = std::max(1, + EnvUnsigned("MOCKTAIL_VR_XR_EVIDENCE_INTERVAL", 20)); + if (directory.empty() || published_pose_.frame % interval != 0) return true; + VkBuffer buffers[2] = {}; + VkDeviceMemory memory[2] = {}; + bool pending = false; + auto cleanup = [&] { + if (pending) vk_->DeviceWaitIdle(vk_device_); + for (int i = 0; i < 2; ++i) { + if (buffers[i]) vk_->DestroyBuffer(vk_device_, buffers[i], nullptr); + if (memory[i]) vk_->FreeMemory(vk_device_, memory[i], nullptr); + } + }; + try { + const auto& binding = eyes_[eye]; + const auto& slot = eye_swapchains_[eye]; + const VkDeviceSize size = VkDeviceSize(binding.width)*binding.height*4; + const VkImage images[2] = {binding.image, slot.images[slot.acquired_index]}; + const VkFormat formats[2] = {binding.format, slot.format}; + const VkImageLayout layouts[2] = {binding.final_layout, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}; + VkPhysicalDeviceMemoryProperties properties{}; + vk_->GetPhysicalDeviceMemoryProperties(vk_physical_device_, &properties); + for (int i = 0; i < 2; ++i) { + Require(formats[i] == VK_FORMAT_R8G8B8A8_UNORM || + formats[i] == VK_FORMAT_B8G8R8A8_UNORM, + "evidence currently requires 8-bit UNORM images"); + VkBufferCreateInfo info{}; + info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + info.size = size; + info.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + Require(vk_->CreateBuffer(vk_device_, &info, nullptr, &buffers[i]) == VK_SUCCESS, + "evidence buffer creation failed"); + VkMemoryRequirements requirements{}; + vk_->GetBufferMemoryRequirements(vk_device_, buffers[i], &requirements); + std::uint32_t type = UINT32_MAX; + const auto flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + for (std::uint32_t t = 0; t < properties.memoryTypeCount; ++t) + if ((requirements.memoryTypeBits & (1u << t)) && + (properties.memoryTypes[t].propertyFlags & flags) == flags) { type = t; break; } + Require(type != UINT32_MAX, "no coherent evidence readback memory"); + VkMemoryAllocateInfo allocation{}; + allocation.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + allocation.allocationSize = requirements.size; + allocation.memoryTypeIndex = type; + Require(vk_->AllocateMemory(vk_device_, &allocation, nullptr, &memory[i]) == VK_SUCCESS, + "evidence memory allocation failed"); + Require(vk_->BindBufferMemory(vk_device_, buffers[i], memory[i], 0) == VK_SUCCESS, + "evidence buffer binding failed"); + } + Require(vk_->ResetCommandBuffer(copy_command_, 0) == VK_SUCCESS, "evidence reset failed"); + VkCommandBufferBeginInfo begin{}; + begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + Require(vk_->BeginCommandBuffer(copy_command_, &begin) == VK_SUCCESS, "evidence begin failed"); + for (int i = 0; i < 2; ++i) { + VkImageMemoryBarrier barrier{}; + barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.oldLayout = layouts[i]; + barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = images[i]; + barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); + VkBufferImageCopy region{}; + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.imageExtent = {binding.width, binding.height, 1}; + vk_->CmdCopyImageToBuffer(copy_command_, images[i], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + buffers[i], 1, ®ion); + barrier.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + std::swap(barrier.oldLayout, barrier.newLayout); + vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); + } + Require(vk_->EndCommandBuffer(copy_command_) == VK_SUCCESS, "evidence end failed"); + Require(vk_->ResetFences(vk_device_, 1, ©_fence_) == VK_SUCCESS, "evidence fence reset failed"); + VkSubmitInfo submit{}; + submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit.commandBufferCount = 1; + submit.pCommandBuffers = ©_command_; + Require(vk_->QueueSubmit(queue, 1, &submit, copy_fence_) == VK_SUCCESS, "evidence submit failed"); + pending = true; + Require(vk_->WaitForFences(vk_device_, 1, ©_fence_, VK_TRUE, 2000000000ULL) == VK_SUCCESS, + "evidence fence wait failed"); + pending = false; + std::filesystem::create_directories(directory); + const auto stem = std::filesystem::path(directory) / + ("frame-" + std::to_string(published_pose_.frame) + "-eye" + std::to_string(eye)); + for (int i = 0; i < 2; ++i) { + void* mapped = nullptr; + Require(vk_->MapMemory(vk_device_, memory[i], 0, size, 0, &mapped) == VK_SUCCESS, + "evidence map failed"); + const auto* bytes = static_cast(mapped); + std::vector rgb(size/4*3); + const bool bgra = formats[i] == VK_FORMAT_B8G8R8A8_UNORM; + for (std::size_t pixel = 0; pixel < size/4; ++pixel) { + rgb[pixel*3] = bytes[pixel*4 + (bgra ? 2 : 0)]; + rgb[pixel*3+1] = bytes[pixel*4+1]; + rgb[pixel*3+2] = bytes[pixel*4 + (bgra ? 0 : 2)]; + } + vk_->UnmapMemory(vk_device_, memory[i]); + std::ofstream file(stem.string() + (i ? "-destination.ppm" : "-source.ppm"), std::ios::binary); + file << "P6\n" << binding.width << ' ' << binding.height << "\n255\n"; + file.write(reinterpret_cast(rgb.data()), rgb.size()); + Require(file.good(), "evidence image write failed"); + } + std::ofstream meta(stem.string() + ".json"); + meta << "{\"frame\":" << published_pose_.frame << ",\"eye\":" << eye + << ",\"source\":\"" << images[0] << "\",\"destination\":\"" << images[1] + << "\",\"owner\":\"" << binding.owner << "\",\"pose_applied_frame\":" + << applied_poses_[binding.owner] << "}"; + Require(meta.good(), "evidence metadata write failed"); + cleanup(); + return true; + } catch (const std::exception& error) { + cleanup(); + Log(" [vr-backend] XR evidence failed: %s\n", error.what()); + return false; + } +} + +} // namespace mocktail::vr + +// Process symbols resolved by the Vulkan adapter through dlsym. They are +// inert while no backend is armed; the adapter falls back to its plain host +// loader path whenever a hook returns false. +extern "C" { + +bool mocktail_vr_xr_create_vulkan_instance( + const VkInstanceCreateInfo* create_info, + const VkAllocationCallbacks* allocator, VkInstance* instance, + VkResult* result) { + auto* backend = mocktail::vr::ActiveVrBackend(); + return backend != nullptr && + backend->CreateVulkanInstance(create_info, allocator, instance, + result); +} + +bool mocktail_vr_xr_create_vulkan_device( + VkPhysicalDevice physical_device, const VkDeviceCreateInfo* create_info, + const VkAllocationCallbacks* allocator, VkDevice* device, + VkResult* result) { + auto* backend = mocktail::vr::ActiveVrBackend(); + return backend != nullptr && + backend->CreateVulkanDevice(physical_device, create_info, allocator, + device, result); +} + +void mocktail_vr_xr_note_instance_destroyed(VkInstance instance) { + if (auto* backend = mocktail::vr::ActiveVrBackend()) backend->NoteInstanceDestroyed(instance); +} + +void mocktail_vr_xr_note_device_destroyed(VkDevice device) { + auto* backend = mocktail::vr::ActiveVrBackend(); + if (backend != nullptr) { + backend->NoteDeviceDestroyed(device); + } +} + +void mocktail_vr_xr_note_queue(VkDevice device, VkQueue queue, + unsigned int family, unsigned int index) { + auto* backend = mocktail::vr::ActiveVrBackend(); + if (backend != nullptr) { + backend->NoteQueue(device, queue, family, index); + } +} + +bool mocktail_vr_xr_wants_resource_records() { + auto* backend = mocktail::vr::ActiveVrBackend(); + return backend != nullptr && backend->WantsResourceRecords(); +} + +void mocktail_vr_xr_record_image(VkDevice device, VkImage image, + const VkImageCreateInfo* info) { + auto* backend = mocktail::vr::ActiveVrBackend(); + if (backend != nullptr) { + backend->RecordImage(device, image, info); + } +} + +void mocktail_vr_xr_record_image_view(VkImageView view, VkImage image) { + auto* backend = mocktail::vr::ActiveVrBackend(); + if (backend != nullptr) { + backend->RecordImageView(view, image); + } +} + +void mocktail_vr_xr_record_framebuffer(VkFramebuffer framebuffer, + const VkFramebufferCreateInfo* info) { + auto* backend = mocktail::vr::ActiveVrBackend(); + if (backend != nullptr) { + backend->RecordFramebuffer(framebuffer, info); + } +} + +void mocktail_vr_xr_record_render_pass(VkRenderPass render_pass, + const VkRenderPassCreateInfo* info) { + auto* backend = mocktail::vr::ActiveVrBackend(); + if (backend != nullptr) { + backend->RecordRenderPass(render_pass, info); + } +} + +void mocktail_vr_xr_note_destroy_image(VkImage image) { + auto* backend = mocktail::vr::ActiveVrBackend(); + if (backend != nullptr) { + backend->NoteDestroyImage(image); + } +} + +void mocktail_vr_xr_note_destroy_image_view(VkImageView view) { + auto* backend = mocktail::vr::ActiveVrBackend(); + if (backend != nullptr) { + backend->NoteDestroyImageView(view); + } +} + +void mocktail_vr_xr_note_destroy_framebuffer(VkFramebuffer framebuffer) { + auto* backend = mocktail::vr::ActiveVrBackend(); + if (backend != nullptr) { + backend->NoteDestroyFramebuffer(framebuffer); + } +} + +void mocktail_vr_xr_note_render_pass_begin(VkCommandBuffer command_buffer, + const VkRenderPassBeginInfo* info) { + auto* backend = mocktail::vr::ActiveVrBackend(); + if (backend != nullptr) { + backend->NoteRenderPassBegin(command_buffer, info); + } +} + +void mocktail_vr_xr_note_present(VkQueue queue, VkDevice device) { + auto* backend = mocktail::vr::ActiveVrBackend(); + if (backend != nullptr) { + backend->NoteHostPresent(queue, device); + } +} + +} // extern "C" diff --git a/src/vr/openxr_backend_mode.cc b/src/vr/openxr_backend_mode.cc new file mode 100644 index 00000000..c2764ce5 --- /dev/null +++ b/src/vr/openxr_backend_mode.cc @@ -0,0 +1,53 @@ +#include "mocktail/vr/openxr_backend.h" +#include +#include +namespace mocktail::vr { +std::string SelectVrRuntimeManifest(const std::string& explicit_manifest, + const std::vector& candidates) { + if (!explicit_manifest.empty()) return explicit_manifest; + for (const auto& path : candidates) { + std::error_code error; + if (std::filesystem::is_regular_file(path, error)) return path; + } + return {}; +} + +std::string VrRuntimeUnavailableHint(bool user_selected, const std::string& manifest) { + if (user_selected) { + return "User-selected XR_RUNTIME_JSON '" + manifest + + "' is unavailable. Check the manifest/library and that its runtime server " + "is running with a connected headset; this error alone does not distinguish them."; + } + if (!manifest.empty()) { + return "Auto-selected WiVRn runtime is unavailable. Start WiVRn and connect " + "the headset, then retry. If it remains unavailable, check the runtime installation."; + } + return "The registered OpenXR runtime is unavailable. Install/start WiVRn and " + "connect the headset, or explicitly select an installed runtime with XR_RUNTIME_JSON."; +} + +VrBackendMode ResolveVrBackendMode(bool vr_enabled) { + if (!vr_enabled) { + return VrBackendMode::kDisabled; + } + const char* value = std::getenv("MOCKTAIL_VR_BACKEND"); + const std::string mode = value ? value : "xr"; + if (mode == "native" || mode == "native-stereo") { + return VrBackendMode::kNativeOnly; + } + return VrBackendMode::kXrOutput; +} + +const char* VrBackendModeName(VrBackendMode mode) { + switch (mode) { + case VrBackendMode::kDisabled: + return "disabled"; + case VrBackendMode::kNativeOnly: + return "native-stereo-diagnostic"; + case VrBackendMode::kXrOutput: + return "openxr-output"; + } + return "disabled"; +} + +} diff --git a/src/vr/openxr_disabled.cc b/src/vr/openxr_disabled.cc index f967e102..f8dc382d 100644 --- a/src/vr/openxr_disabled.cc +++ b/src/vr/openxr_disabled.cc @@ -1,7 +1,17 @@ #include "mocktail/vr/openxr_probe.h" +#include "mocktail/vr/openxr_backend.h" #include "mocktail/vr/openxr_preview.h" namespace mocktail::vr { +OpenXrBackend::~OpenXrBackend() = default; +Status OpenXrBackend::Arm() { + return Status::Error(StatusCode::kUnavailable, "OpenXR support is disabled in this build"); +} +void OpenXrBackend::Disarm() {} +OpenXrBackend* ActiveVrBackend() { return nullptr; } +void OpenXrBackend::NotePoseApplied(void*, std::uint64_t) {} +ScriptedPoseSample OpenXrBackend::PublishedHeadPose() const { return {}; } + PreviewResult RunOpenXrPreview(const PreviewOptions&) { PreviewResult result; @@ -11,6 +21,8 @@ PreviewResult RunOpenXrPreview(const PreviewOptions&) { return result; } +bool IsOpenXrSupportCompiled() { return false; } + ProbeReport ProbeOpenXr() { ProbeReport report; report.status = ProbeStatus::kBuildDisabled; diff --git a/src/vr/openxr_preview.cc b/src/vr/openxr_preview.cc index 2644d5b1..3c28ab92 100644 --- a/src/vr/openxr_preview.cc +++ b/src/vr/openxr_preview.cc @@ -301,8 +301,7 @@ class Preview { " failed: " + (text[0] ? std::string(text) : std::to_string(result)); if (result == XR_ERROR_RUNTIME_UNAVAILABLE) message += ". No active OpenXR runtime was found. Start WiVRn for a real " - "headset. Without a headset, run mocktail-vr-simulated from " - "this build directory."; + "headset, or select an installed runtime using XR_RUNTIME_JSON."; else if (result == XR_ERROR_FORM_FACTOR_UNAVAILABLE) message += ". Connect a headset or select a simulated runtime."; else if (result == XR_ERROR_EXTENSION_NOT_PRESENT) diff --git a/src/vr/openxr_probe.cc b/src/vr/openxr_probe.cc index 19639796..45eba4d3 100644 --- a/src/vr/openxr_probe.cc +++ b/src/vr/openxr_probe.cc @@ -73,6 +73,8 @@ XrResult EnumerateProperties(XrStructureType type, Enumerate enumerate, } // namespace +bool IsOpenXrSupportCompiled() { return true; } + ProbeReport ProbeOpenXr() { ProbeReport report; report.sdk_version = Version(XR_CURRENT_API_VERSION); diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc new file mode 100644 index 00000000..ea30375b --- /dev/null +++ b/src/vr/roblox_vr_device_bridge.cc @@ -0,0 +1,885 @@ +#include "mocktail/vr/roblox_vr_device_bridge.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include "mocktail/vr/openxr_backend.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vr/roblox_vr_device_bridge_internal.h" + +namespace mocktail::vr { +namespace { + +using internal::kConstructorContractSize; +using internal::kExpectedObjectType; +using internal::kEyeGetterContractSize; +using internal::kEyeGetterVtableSlot; +using internal::kEyeInitializerVtableSlot; +using internal::kFramebufferSlotOffset; +using internal::kGraphicsDeviceOffset; +using internal::kHeightOffset; +using internal::kInitializerContractSize; +using internal::kObjectNameOffset; +using internal::kObjectTypeOffset; +using internal::kReadyByteOffset; +using internal::kStateGetterContractSize; +using internal::kStateGetterVtableSlot; +using internal::kTextureSlotOffset; +using internal::kWidthOffset; + +std::atomic g_active_bridge{nullptr}; +// Evidence queries expose only the current native call's provenance. These +// pointers are identities for the layer, never retained/dereferenced by it. +thread_local void *g_initializing_object = nullptr; +thread_local void *g_eye_object = nullptr; +thread_local void *g_eye_framebuffer = nullptr; +thread_local int g_eye_index = -1; +thread_local bool g_thread_presented = false; + +Status InvalidArgument(std::string message) { + return Status::Error(StatusCode::kInvalidArgument, std::move(message)); +} + +Status FailedPrecondition(std::string message) { + return Status::Error(StatusCode::kFailedPrecondition, std::move(message)); +} + +std::uint64_t CurrentThreadId() { + return static_cast(syscall(SYS_gettid)); +} + +bool IsValidElfImage(std::uintptr_t image_base) { + if (image_base == 0) { + return false; + } + const auto *header = reinterpret_cast(image_base); + return std::memcmp(header->e_ident, ELFMAG, SELFMAG) == 0 && + header->e_ident[EI_CLASS] == ELFCLASS64 && header->e_phoff != 0 && + header->e_phnum != 0; +} + +bool IsProgramHeaderRange(std::uintptr_t image_base, std::uintptr_t rva, + std::size_t size, std::uint32_t type, + std::uint32_t required_flags) { + if (!IsValidElfImage(image_base) || size == 0 || + rva > std::numeric_limits::max() - size) { + return false; + } + const auto *header = reinterpret_cast(image_base); + const auto *program_headers = reinterpret_cast( + image_base + static_cast(header->e_phoff)); + const std::uintptr_t range_end = rva + size; + for (std::size_t index = 0; index < header->e_phnum; ++index) { + const Elf64_Phdr &segment = program_headers[index]; + if (segment.p_type != type || + (segment.p_flags & required_flags) != required_flags || + segment.p_vaddr > + std::numeric_limits::max() - segment.p_memsz) { + continue; + } + const std::uintptr_t segment_end = + static_cast(segment.p_vaddr + segment.p_memsz); + if (rva >= segment.p_vaddr && range_end <= segment_end) { + return true; + } + } + return false; +} + +bool IsExecutableImageRange(std::uintptr_t image_base, std::uintptr_t rva, + std::size_t size) { + return IsProgramHeaderRange(image_base, rva, size, PT_LOAD, PF_R | PF_X); +} + +bool IsWritableDataImageRange(std::uintptr_t image_base, std::uintptr_t rva, + std::size_t size) { + return IsProgramHeaderRange(image_base, rva, size, PT_LOAD, PF_R | PF_W); +} + +bool IsRelroImageRange(std::uintptr_t image_base, std::uintptr_t rva, + std::size_t size) { + return IsProgramHeaderRange(image_base, rva, size, PT_GNU_RELRO, 0); +} + +std::uintptr_t ImageSize(std::uintptr_t image_base) { + const auto *header = reinterpret_cast(image_base); + const auto *program_headers = reinterpret_cast( + image_base + static_cast(header->e_phoff)); + std::uintptr_t highest = 0; + for (std::size_t index = 0; index < header->e_phnum; ++index) { + const Elf64_Phdr &segment = program_headers[index]; + if (segment.p_type != PT_LOAD || + segment.p_vaddr > + std::numeric_limits::max() - segment.p_memsz) { + continue; + } + highest = std::max(highest, static_cast(segment.p_vaddr + + segment.p_memsz)); + } + return highest; +} + +bool IsCodePointerInImage(std::uintptr_t image_base, std::uintptr_t image_size, + std::uintptr_t address) { + return address > image_base && image_size != 0 && + address - image_base < image_size; +} + +bool SetVtableRangeWritable(std::uintptr_t *first_slot, + std::uintptr_t *last_slot_exclusive, + bool writable) { + if (first_slot == nullptr || last_slot_exclusive <= first_slot) { + return false; + } + const long page_size_value = sysconf(_SC_PAGESIZE); + if (page_size_value <= 0) { + return false; + } + const std::uintptr_t page_size = static_cast(page_size_value); + const std::uintptr_t first = reinterpret_cast(first_slot); + const std::uintptr_t last = + reinterpret_cast(last_slot_exclusive); + const std::uintptr_t page_begin = first - first % page_size; + const std::uintptr_t page_end = + ((last + page_size - 1) / page_size) * page_size; + const int protection = writable ? PROT_READ | PROT_WRITE : PROT_READ; + return mprotect(reinterpret_cast(page_begin), page_end - page_begin, + protection) == 0; +} + +bool ContainsBytes(const std::uint8_t *code, std::size_t size, + const std::uint8_t *expected, std::size_t expected_size) { + if (code == nullptr || expected == nullptr || expected_size == 0 || + expected_size > size) { + return false; + } + for (std::size_t index = 0; index + expected_size <= size; ++index) { + if (std::memcmp(code + index, expected, expected_size) == 0) { + return true; + } + } + return false; +} + +template std::uintptr_t FunctionAddress(Function function) { + return reinterpret_cast(function); +} + +void Log(const char *format, ...) { + std::va_list arguments; + va_start(arguments, format); + std::vfprintf(stderr, format, arguments); + va_end(arguments); + std::fflush(stderr); +} + +} // namespace + +namespace internal { + +EyeResourceState InspectEyeResources(const void *object) { + if (object == nullptr) + return EyeResourceState::kInvalid; + const auto *bytes = static_cast(object); + std::uintptr_t fb[2], tex[2]; + std::memcpy(&fb[0], bytes + kFramebufferSlotOffset, sizeof(fb[0])); + std::memcpy(&fb[1], bytes + kFramebufferSlotOffset + kEyeSlotStride, + sizeof(fb[1])); + std::memcpy(&tex[0], bytes + kTextureSlotOffset, sizeof(tex[0])); + std::memcpy(&tex[1], bytes + kTextureSlotOffset + kEyeSlotStride, + sizeof(tex[1])); + std::uint32_t width, height; + std::memcpy(&width, bytes + kWidthOffset, sizeof(width)); + std::memcpy(&height, bytes + kHeightOffset, sizeof(height)); + if (width == 0 || height == 0 || width > 16384 || height > 16384) + return EyeResourceState::kInvalid; + if (bytes[kReadyByteOffset] == 0 && !fb[0] && !fb[1] && !tex[0] && !tex[1]) + return EyeResourceState::kEmpty; + if (bytes[kReadyByteOffset] == 1 && fb[0] && fb[1] && tex[0] && tex[1] && + fb[0] != fb[1] && tex[0] != tex[1]) + return EyeResourceState::kReady; + return EyeResourceState::kInvalid; +} + +bool HasExpectedVrStateGetterContract(const std::uint8_t *code, + std::size_t size) { + // push rbp; mov rbp,rsp; push rbx; push rax; mov rbx,rdi; add rsi,0x14; + // mov edx,0x138; call memcpy; mov rax,rbx; add rsp,8; pop rbx; pop rbp; ret + static constexpr std::array + kExpected = {0x55, 0x48, 0x89, 0xe5, 0x53, 0x50, 0x48, 0x89, 0xfb, + 0x48, 0x83, 0xc6, 0x14, 0xba, 0x38, 0x01, 0x00, 0x00, + 0xe8, 0xd3, 0xb5, 0x36, 0x03, 0x48, 0x89, 0xd8, 0x48, + 0x83, 0xc4, 0x08, 0x5b, 0x5d, 0xc3}; + return code != nullptr && size >= kExpected.size() && + std::memcmp(code, kExpected.data(), kExpected.size()) == 0; +} + +bool HasExpectedVrEyeGetterContract(const std::uint8_t *code, + std::size_t size) { + // movsxd rax,esi; shl rax,4; mov rax,[rdi+rax+0x150]; ret + static constexpr std::array kExpected = + {0x48, 0x63, 0xc6, 0x48, 0xc1, 0xe0, 0x04, 0x48, + 0x8b, 0x84, 0x07, 0x50, 0x01, 0x00, 0x00, 0xc3}; + return code != nullptr && size >= kExpected.size() && + std::memcmp(code, kExpected.data(), kExpected.size()) == 0; +} + +bool HasExpectedVrConstructorContract(const std::uint8_t *code, + std::size_t size) { + static constexpr std::array kPrologue = { + 0x55, 0x48, 0x89, 0xe5, 0x41, 0x57, 0x41, 0x56, 0x41, 0x55, + 0x41, 0x54, 0x53, 0x48, 0x83, 0xec, 0x28, 0x49, 0x89, 0xcf, + 0x41, 0x89, 0xd4, 0x41, 0x89, 0xf5, 0x48, 0x89, 0xfb}; + // lea rax,[rip -> DebugDeviceVR vtable]; mov [rdi],rax + static constexpr std::array kVtableStore = { + 0x48, 0x8d, 0x05, 0xf2, 0x74, 0x47, 0x03}; + static constexpr std::array kVptrWrite = {0x48, 0x89, 0x07}; + // mov dword ptr [rdi+0x10], 6 + static constexpr std::array kTypeStore = { + 0xc7, 0x47, 0x10, 0x06, 0x00, 0x00, 0x00}; + // lea r14,[rdi+0x14]; mov edx,0x134 (state block memset) + static constexpr std::array kStateMemset = { + 0x4c, 0x8d, 0x77, 0x14, 0xba, 0x34, 0x01, 0x00, 0x00}; + // mov [rbx+0x190],r13d ; mov [rbx+0x194],r12d + static constexpr std::array kWidthHeightStore = { + 0x44, 0x89, 0xab, 0x90, 0x01, 0x00, 0x00, + 0x44, 0x89, 0xa3, 0x94, 0x01, 0x00, 0x00}; + // mov [rbx+0x198],r15 (graphics device captured from rcx) + static constexpr std::array kDeviceStore = { + 0x4c, 0x89, 0xbb, 0x98, 0x01, 0x00, 0x00}; + if (code == nullptr || size < kConstructorContractSize) { + return false; + } + return std::memcmp(code, kPrologue.data(), kPrologue.size()) == 0 && + ContainsBytes(code, size, kVtableStore.data(), kVtableStore.size()) && + ContainsBytes(code, size, kVptrWrite.data(), kVptrWrite.size()) && + ContainsBytes(code, size, kTypeStore.data(), kTypeStore.size()) && + ContainsBytes(code, size, kStateMemset.data(), kStateMemset.size()) && + ContainsBytes(code, size, kWidthHeightStore.data(), + kWidthHeightStore.size()) && + ContainsBytes(code, size, kDeviceStore.data(), kDeviceStore.size()); +} + +bool HasExpectedVrEyeInitializerContract( + const std::uint8_t *code, std::size_t size, + const compat::VrDebugDeviceBridgeProfile &profile) { + static constexpr std::array kPrologue = { + 0x55, 0x48, 0x89, 0xe5, 0x41, 0x57, 0x41, 0x56, 0x41, 0x55, 0x41, 0x54, + 0x53, 0x48, 0x81, 0xec, 0xa8, 0x00, 0x00, 0x00, 0x48, 0x89, 0xb5, 0x50}; + // mov cl,1; mov [rdi+0x147],cl; mov [rdi+0x94],cl; mov [rdi+0x144],cl + static constexpr std::array kReadyBytes = { + 0xb1, 0x01, 0x88, 0x8f, 0x47, 0x01, 0x00, 0x00, 0x88, 0x8f, + 0x94, 0x00, 0x00, 0x00, 0x88, 0x8f, 0x44, 0x01, 0x00, 0x00}; + // lea rax,[rdi+0x170] (texture slots) + static constexpr std::array kTextureSlotBase = { + 0x48, 0x8d, 0x87, 0x70, 0x01, 0x00, 0x00}; + // lea rax,[rdi+0x150] (framebuffer slots) + static constexpr std::array kFramebufferSlotBase = { + 0x48, 0x8d, 0x87, 0x50, 0x01, 0x00, 0x00}; + // mov rsi,[rbp-0xb0]; mov rax,[rsi]; mov rax,[rax+createSlot]; call rax + std::array device_call = {0x48, 0x8b, 0x06, 0x48, 0x8b, + 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00}; + const std::uint32_t create_slot = static_cast( + profile.device_create_framebuffer_vtable_offset); + device_call[6] = static_cast(create_slot); + device_call[7] = static_cast(create_slot >> 8); + device_call[8] = static_cast(create_slot >> 16); + device_call[9] = static_cast(create_slot >> 24); + // Two-iteration eye loop: mov r15d,1; test [rbp-0xa4],1; mov ecx,0; jne back + static constexpr std::array kEyeLoopTail = { + 0x41, 0xbf, 0x01, 0x00, 0x00, 0x00, 0xf6, 0x85, + 0x5c, 0xff, 0xff, 0xff, 0x01, 0xb9, 0x00}; + if (code == nullptr || size < kInitializerContractSize) { + return false; + } + return std::memcmp(code, kPrologue.data(), kPrologue.size()) == 0 && + ContainsBytes(code, size, kReadyBytes.data(), kReadyBytes.size()) && + ContainsBytes(code, size, kTextureSlotBase.data(), + kTextureSlotBase.size()) && + ContainsBytes(code, size, kFramebufferSlotBase.data(), + kFramebufferSlotBase.size()) && + ContainsBytes(code, size, device_call.data(), 10) && + ContainsBytes(code, size, kEyeLoopTail.data(), kEyeLoopTail.size()); +} + +bool HasExpectedVrDebugDeviceVtable( + const std::uintptr_t *vtable, std::uintptr_t image_base, + const compat::VrDebugDeviceBridgeProfile &profile) { + return vtable != nullptr && image_base != 0 && + vtable[kStateGetterVtableSlot / sizeof(std::uintptr_t)] == + image_base + profile.state_getter_rva && + vtable[kEyeGetterVtableSlot / sizeof(std::uintptr_t)] == + image_base + profile.eye_getter_rva && + vtable[kEyeInitializerVtableSlot / sizeof(std::uintptr_t)] == + image_base + profile.eye_initializer_rva; +} + +} // namespace internal + +namespace { + +// Guest-callable interposed vtable entries. The guest Android ABI for these +// methods is SysV amd64: the state getter is sret (rdi = hidden 0x138-byte +// result buffer, rsi = device object, returns the buffer) and the eye getter +// takes (rdi = device object, esi = eye index) and returns the framebuffer +// object pointer. +extern "C" void *mocktail_vr_debug_device_state_getter(void *result_buffer, + void *device_object) { + RobloxVrDeviceBridge *bridge = + g_active_bridge.load(std::memory_order_acquire); + if (bridge != nullptr) { + bridge->OnStateGetterCall(result_buffer, device_object); + } + return result_buffer; +} + +extern "C" void *mocktail_vr_debug_device_eye_getter(void *device_object, + int eye_index) { + RobloxVrDeviceBridge *bridge = + g_active_bridge.load(std::memory_order_acquire); + const auto original = + bridge != nullptr ? bridge->original_eye_getter() : nullptr; + void *framebuffer = + original != nullptr ? original(device_object, eye_index) : nullptr; + if (bridge != nullptr) { + bridge->OnEyeGetterCall(device_object, eye_index, framebuffer); + } + return framebuffer; +} + +} // namespace + +extern "C" void *mocktail_vr_initializing_object() { + return g_initializing_object; +} + +extern "C" bool mocktail_vr_current_eye(void **self, int *eye, + void **framebuffer) { + *self = g_eye_object; + *eye = g_eye_index; + *framebuffer = g_eye_framebuffer; + return g_eye_object != nullptr && g_eye_index >= 0 && + g_eye_framebuffer != nullptr; +} + +extern "C" void mocktail_vr_consume_eye() { + g_eye_object = nullptr; + g_eye_framebuffer = nullptr; + g_eye_index = -1; +} + +extern "C" void mocktail_vr_note_host_present_begin() { + RobloxVrDeviceBridge *bridge = + g_active_bridge.load(std::memory_order_acquire); + if (bridge != nullptr) { + bridge->NoteHostPresent(); + } +} + +RobloxVrDeviceBridge::~RobloxVrDeviceBridge() { + Shutdown(); + RobloxVrDeviceBridge *expected = this; + (void)g_active_bridge.compare_exchange_strong( + expected, nullptr, std::memory_order_acq_rel, std::memory_order_acquire); +} + +Status RobloxVrDeviceBridge::Install(const compat::BuildProfile &profile) { + if (installed_) { + return FailedPrecondition("Roblox VR device bridge is already armed"); + } + if (!profile.vr_debug_device_bridge.has_value()) { + return FailedPrecondition( + "experimental Roblox VR requires vr_debug_device_bridge metadata in " + "the exact-build compatibility profile"); + } + if (!profile.allow_host_abi_bridges) { + return FailedPrecondition( + "Roblox VR device bridge profile requires host ABI bridges"); + } + + RobloxVrDeviceBridge *expected = nullptr; + if (!g_active_bridge.compare_exchange_strong(expected, this, + std::memory_order_acq_rel, + std::memory_order_acquire)) { + return FailedPrecondition("another Roblox VR device bridge owns the " + "process"); + } + profile_ = *profile.vr_debug_device_bridge; + installed_ = true; + std::fprintf(stderr, + " [vr-device] native stereo bridge armed for Build ID %s " + "(vtable +0x28/+0x48 interposition; init phase: %s)\n", + profile.elf_build_id.c_str(), "live render-thread state getter"); + return Status::Ok(); +} + +Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { + std::lock_guard lock(mutex_); + if (!installed_) { + return FailedPrecondition("Roblox VR device bridge is not armed"); + } + if (active_.load(std::memory_order_acquire)) { + return image_base == image_base_.load(std::memory_order_relaxed) + ? Status::Ok() + : FailedPrecondition("Roblox VR device bridge saw a second " + "image"); + } + if (!IsValidElfImage(image_base)) { + return InvalidArgument("libroblox image base is not a valid ELF64 image"); + } + const std::uintptr_t largest_rva = std::max( + {profile_.vtable_rva + kEyeInitializerVtableSlot + sizeof(std::uintptr_t), + profile_.constructor_rva + kConstructorContractSize, + profile_.state_getter_rva + kStateGetterContractSize, + profile_.eye_getter_rva + kEyeGetterContractSize, + profile_.eye_initializer_rva + kInitializerContractSize, + profile_.emulator_flag_storage_rva + 1}); + if (image_base > std::numeric_limits::max() - largest_rva) { + return InvalidArgument("VR device profile address overflows"); + } + + const std::uintptr_t vtable_rva_size = + kEyeInitializerVtableSlot + sizeof(std::uintptr_t); + if (!IsRelroImageRange(image_base, profile_.vtable_rva, vtable_rva_size)) { + return FailedPrecondition( + "DebugDeviceVR vtable is not inside the verified RELRO range"); + } + if (!IsExecutableImageRange(image_base, profile_.constructor_rva, + kConstructorContractSize) || + !IsExecutableImageRange(image_base, profile_.state_getter_rva, + kStateGetterContractSize) || + !IsExecutableImageRange(image_base, profile_.eye_getter_rva, + kEyeGetterContractSize) || + !IsExecutableImageRange(image_base, profile_.eye_initializer_rva, + kInitializerContractSize)) { + return FailedPrecondition( + "VR device profile functions are not in executable image ranges"); + } + if (!IsWritableDataImageRange(image_base, profile_.emulator_flag_storage_rva, + 1)) { + return FailedPrecondition( + "VR emulator flag storage is not in a writable data range"); + } + + const auto *vtable = reinterpret_cast( + image_base + profile_.vtable_rva); + if (!internal::HasExpectedVrDebugDeviceVtable(vtable, image_base, profile_)) { + return FailedPrecondition( + "DebugDeviceVR vtable does not match the Build-ID profile"); + } + const auto *state_getter_code = reinterpret_cast( + image_base + profile_.state_getter_rva); + const auto *eye_getter_code = reinterpret_cast( + image_base + profile_.eye_getter_rva); + const auto *constructor_code = reinterpret_cast( + image_base + profile_.constructor_rva); + const auto *initializer_code = reinterpret_cast( + image_base + profile_.eye_initializer_rva); + if (!internal::HasExpectedVrStateGetterContract(state_getter_code, + kStateGetterContractSize) || + !internal::HasExpectedVrEyeGetterContract(eye_getter_code, + kEyeGetterContractSize)) { + return FailedPrecondition( + "DebugDeviceVR getter machine contracts do not match the profile"); + } + if (!internal::HasExpectedVrConstructorContract(constructor_code, + kConstructorContractSize) || + !internal::HasExpectedVrEyeInitializerContract( + initializer_code, kInitializerContractSize, profile_)) { + return FailedPrecondition( + "DebugDeviceVR constructor/initializer machine contracts do not " + "match the profile"); + } + + image_base_.store(image_base, std::memory_order_relaxed); + image_size_.store(ImageSize(image_base), std::memory_order_relaxed); + vtable_ = + reinterpret_cast(image_base + profile_.vtable_rva); + original_state_slot_ = vtable_[kStateGetterVtableSlot / sizeof(void *)]; + original_eye_slot_ = vtable_[kEyeGetterVtableSlot / sizeof(void *)]; + + std::uintptr_t *first_slot = vtable_ + kStateGetterVtableSlot / 8; + std::uintptr_t *last_slot_exclusive = + vtable_ + kEyeInitializerVtableSlot / 8 + 1; + if (!SetVtableRangeWritable(first_slot, last_slot_exclusive, true)) { + vtable_ = nullptr; + return Status::Error(StatusCode::kPlatformError, + "cannot make DebugDeviceVR vtable writable"); + } + original_state_getter_.store( + reinterpret_cast(original_state_slot_), + std::memory_order_release); + original_eye_getter_.store(reinterpret_cast(original_eye_slot_), + std::memory_order_release); + __atomic_store_n(&vtable_[kStateGetterVtableSlot / 8], + FunctionAddress(&mocktail_vr_debug_device_state_getter), + __ATOMIC_RELEASE); + __atomic_store_n(&vtable_[kEyeGetterVtableSlot / 8], + FunctionAddress(&mocktail_vr_debug_device_eye_getter), + __ATOMIC_RELEASE); + if (!SetVtableRangeWritable(first_slot, last_slot_exclusive, false)) { + __atomic_store_n(&vtable_[kStateGetterVtableSlot / 8], original_state_slot_, + __ATOMIC_RELEASE); + __atomic_store_n(&vtable_[kEyeGetterVtableSlot / 8], original_eye_slot_, + __ATOMIC_RELEASE); + (void)SetVtableRangeWritable(first_slot, last_slot_exclusive, false); + original_state_getter_.store(nullptr, std::memory_order_release); + original_eye_getter_.store(nullptr, std::memory_order_release); + vtable_ = nullptr; + return Status::Error(StatusCode::kPlatformError, + "cannot restore DebugDeviceVR vtable RELRO " + "protection"); + } + + active_.store(true, std::memory_order_release); + Log(" [vr-device] activated: DebugDeviceVR vtable slots +0x28/+0x48 " + "interposed at image base 0x%" PRIxPTR "\n", + image_base); + return Status::Ok(); +} + +void RobloxVrDeviceBridge::Shutdown() { + if (!installed_) { + return; + } + bool restored = true; + { + std::lock_guard lock(mutex_); + if (active_.load(std::memory_order_acquire)) { + active_.store(false, std::memory_order_release); + restored = RestoreVtableLocked(); + Log(" [vr-device] shutdown: captured=%d initialized=%d " + "eye0_calls=%" PRIu64 " eye1_calls=%" PRIu64 " presents=%" PRIu64 + " vtable_restored=%d\n", + captured_device_count(), initialized_device_count(), + eye_getter_call_count(0), eye_getter_call_count(1), + present_count_.load(std::memory_order_relaxed), restored ? 1 : 0); + vtable_ = nullptr; + image_base_.store(0, std::memory_order_relaxed); + image_size_.store(0, std::memory_order_relaxed); + } + } + if (!restored) { + std::fprintf(stderr, " [vr-device] failed to restore DebugDeviceVR vtable " + "protection\n"); + } + installed_ = false; +} + +bool RobloxVrDeviceBridge::RestoreVtableLocked() { + if (vtable_ == nullptr) { + return false; + } + std::uintptr_t *first_slot = vtable_ + kStateGetterVtableSlot / 8; + std::uintptr_t *last_slot_exclusive = + vtable_ + kEyeInitializerVtableSlot / 8 + 1; + if (!SetVtableRangeWritable(first_slot, last_slot_exclusive, true)) { + return false; + } + __atomic_store_n(&vtable_[kStateGetterVtableSlot / 8], original_state_slot_, + __ATOMIC_RELEASE); + __atomic_store_n(&vtable_[kEyeGetterVtableSlot / 8], original_eye_slot_, + __ATOMIC_RELEASE); + return SetVtableRangeWritable(first_slot, last_slot_exclusive, false); +} + +void RobloxVrDeviceBridge::NoteHostPresent() { + if (!active_.load(std::memory_order_acquire)) { + return; + } + const std::uint64_t thread_id = CurrentThreadId(); + std::uint64_t expected_thread = 0; + if (present_thread_.compare_exchange_strong(expected_thread, thread_id, + std::memory_order_acq_rel)) { + Log(" [vr-device] render thread observed: tid=%" PRIu64 "\n", thread_id); + } + const std::uint64_t presents = + present_count_.fetch_add(1, std::memory_order_relaxed) + 1; + if (presents == 1) { + Log(" [vr-device] first host present completed on tid=%" PRIu64 "\n", + thread_id); + } + g_thread_presented = true; + mocktail_vr_consume_eye(); +} + +bool RobloxVrDeviceBridge::ValidateObjectHeader(void *self) const { + const std::uintptr_t base = image_base_.load(std::memory_order_acquire); + if (self == nullptr || base == 0) { + return false; + } + auto *bytes = static_cast(self); + const std::uintptr_t vptr = *reinterpret_cast(bytes); + if (vptr != base + profile_.vtable_rva) { + return false; + } + const std::uint32_t type = + *reinterpret_cast(bytes + kObjectTypeOffset); + if (type != kExpectedObjectType) { + return false; + } + const char *name = + *reinterpret_cast(bytes + kObjectNameOffset); + return name != nullptr && std::strcmp(name, "DebugDevice") == 0; +} + +void *RobloxVrDeviceBridge::ValidatedGraphicsDevice(void *self) const { + auto *bytes = static_cast(self); + void *device = + *reinterpret_cast(bytes + kGraphicsDeviceOffset); + if (device == nullptr) { + return nullptr; + } + const std::uintptr_t base = image_base_.load(std::memory_order_acquire); + const std::uintptr_t size = image_size_.load(std::memory_order_acquire); + const std::uintptr_t device_vtable = + *reinterpret_cast(device); + if (!IsCodePointerInImage(base, size, device_vtable)) { + return nullptr; + } + const std::uintptr_t create_framebuffer = + *reinterpret_cast( + device_vtable + profile_.device_create_framebuffer_vtable_offset); + if (!IsCodePointerInImage(base, size, create_framebuffer)) { + return nullptr; + } + return device; +} + +bool internal::ApplyXrPose(void* state, const ScriptedPoseSample& pose) { + if (!state) return false; + auto* bytes = static_cast(state); + bytes[0] = 0; + // No tracked controllers are supplied by this backend. + bytes[0x20] = bytes[0x40] = bytes[0x60] = 0; + if (!pose.valid) return false; + float norm = 0; + for (float v : pose.orientation) { if (!std::isfinite(v)) return false; norm += v*v; } + if (norm < 0.99f || norm > 1.01f) return false; + for (float v : pose.position) if (!std::isfinite(v)) return false; + for (int eye = 0; eye < 2; ++eye) { + for (float v : pose.eye_offset[eye]) if (!std::isfinite(v)) return false; + for (float v : pose.eye_fov[eye]) + if (!std::isfinite(v) || std::abs(v) >= 1.56f) return false; + } + // 2998's 0x37eacb8 converts metres to studs itself (factor 10/3). + std::memcpy(bytes + 4, pose.position, sizeof(pose.position)); + std::memcpy(bytes + 0x10, pose.orientation, sizeof(pose.orientation)); + for (int eye = 0; eye < 2; ++eye) { + std::memcpy(bytes + 0xf8 + eye*12, pose.eye_offset[eye], 12); + const auto* fov = pose.eye_fov[eye]; + const float tangents[4] = {std::tan(fov[2]), -std::tan(fov[3]), + -std::tan(fov[0]), std::tan(fov[1])}; + std::memcpy(bytes + 0x110 + eye*16, tangents, sizeof(tangents)); + } + bytes[0] = 1; + return true; +} + +void RobloxVrDeviceBridge::OnStateGetterCall(void *result_buffer, + void *device_object) { + // Serialize initialization with all copies of readiness: the initializer + // raises the guest ready byte before its resources have been allocated. + std::lock_guard lock(mutex_); + if (active_.load(std::memory_order_acquire) && g_thread_presented && + ValidateObjectHeader(device_object) && + internal::InspectEyeResources(device_object) == + internal::EyeResourceState::kEmpty) { + TryInitializeEyes(device_object); + } + const auto original = original_state_getter(); + if (original != nullptr) + original(result_buffer, device_object); + if (active_.load(std::memory_order_acquire) && result_buffer && original) { + if (auto* backend = ActiveVrBackend()) { + const auto pose = backend->PublishedHeadPose(); + if (internal::ApplyXrPose(result_buffer, pose)) { + backend->NotePoseApplied(device_object, pose.frame); + static thread_local std::uint64_t last_logged = 0; + if (pose.frame != last_logged && (pose.frame == 1 || pose.frame % 20 == 0)) { + last_logged = pose.frame; + Log(" [vr-device][evidence] pose-applied frame=%llu owner=%p " + "pos=(%.4f,%.4f,%.4f) ori=(%.4f,%.4f,%.4f,%.4f)\n", + static_cast(pose.frame), device_object, + pose.position[0], pose.position[1], pose.position[2], + pose.orientation[0], pose.orientation[1], pose.orientation[2], pose.orientation[3]); + } + } + } + } + if (active_.load(std::memory_order_acquire) && result_buffer != nullptr && + internal::InspectEyeResources(device_object) != + internal::EyeResourceState::kReady) { + // Fail closed in the returned state, without changing guest ownership or + // manufacturing readiness. Partial/failed resources must never be rendered. + static_cast( + result_buffer)[internal::kStateReadyOffsetInCopy] = 0; + } +} + +void RobloxVrDeviceBridge::TryInitializeEyes(void *self) { + if (self == nullptr || !ValidateObjectHeader(self)) { + Log(" [vr-device] eye-init aborted: object header no longer valid " + "(self=%p)\n", + self); + return; + } + const auto *flag = reinterpret_cast( + image_base_.load(std::memory_order_acquire) + + profile_.emulator_flag_storage_rva); + if (*flag == 0) { + bool expected = false; + if (flag_warning_logged_.compare_exchange_strong(expected, true)) { + Log(" [vr-device] eye-init deferred: FFlagDebugEnableVREmulator " + "storage byte is 0\n"); + } + return; + } + void *device = ValidatedGraphicsDevice(self); + if (device == nullptr) { + Log(" [vr-device] eye-init failed: graphics device at self+0x198 did " + "not pass the machine contract (self=%p)\n", + self); + return; + } + captured_count_.fetch_add(1, std::memory_order_relaxed); + auto *bytes = static_cast(self); + Log(" [vr-device] captured DebugDeviceVR object self=%p device=%p\n", self, + device); + const auto initializer = reinterpret_cast( + image_base_.load(std::memory_order_acquire) + + profile_.eye_initializer_rva); + Log(" [vr-device][evidence] eye-init-begin self=%p device=%p tid=%" PRIu64 + " presents=%" PRIu64 "\n", + self, device, CurrentThreadId(), + present_count_.load(std::memory_order_relaxed)); + g_initializing_object = self; + initializer(self, device); + g_initializing_object = nullptr; + Log(" [vr-device][evidence] eye-init-returned self=%p\n", self); + + const std::uint8_t ready = bytes[kReadyByteOffset]; + const void *framebuffer0 = + *reinterpret_cast(bytes + kFramebufferSlotOffset); + const void *framebuffer1 = *reinterpret_cast( + bytes + kFramebufferSlotOffset + internal::kEyeSlotStride); + const void *texture0 = + *reinterpret_cast(bytes + kTextureSlotOffset); + const void *texture1 = *reinterpret_cast( + bytes + kTextureSlotOffset + internal::kEyeSlotStride); + const std::uint32_t width = + *reinterpret_cast(bytes + kWidthOffset); + const std::uint32_t height = + *reinterpret_cast(bytes + kHeightOffset); + const bool slots_valid = framebuffer0 != nullptr && framebuffer1 != nullptr && + texture0 != nullptr && texture1 != nullptr && + framebuffer0 != framebuffer1 && texture0 != texture1; + if (ready == 1 && slots_valid && + internal::InspectEyeResources(self) == + internal::EyeResourceState::kReady) { + initialized_count_.fetch_add(1, std::memory_order_relaxed); + Log(" [vr-device][evidence] eye-init-complete self=%p ready=1 fb0=%p " + "fb1=%p tex0=%p tex1=%p width=%u height=%u\n", + self, framebuffer0, framebuffer1, texture0, texture1, width, height); + const auto *eye0_view = + reinterpret_cast(bytes + internal::kEye0ViewOffset); + const auto *eye1_view = + reinterpret_cast(bytes + internal::kEye1ViewOffset); + const auto *eye0_projection = reinterpret_cast( + bytes + internal::kEye0ProjectionOffset); + const auto *eye1_projection = reinterpret_cast( + bytes + internal::kEye1ProjectionOffset); + const auto *field_of_view = + reinterpret_cast(bytes + internal::kFieldOfViewOffset); + const float half_ipd = *reinterpret_cast( + bytes + internal::kHalfInterpupillaryOffset); + Log(" [vr-device][evidence] eye-params self=%p eye0_view=(%.4f,%.4f," + "%.4f) eye1_view=(%.4f,%.4f,%.4f) eye0_proj=(%.4f,%.4f,%.4f,%.4f) " + "eye1_proj=(%.4f,%.4f,%.4f,%.4f) fov_deg=(%.1f,%.1f,%.1f,%.1f) " + "half_ipd=%.4f\n", + self, eye0_view[0], eye0_view[1], eye0_view[2], eye1_view[0], + eye1_view[1], eye1_view[2], eye0_projection[0], eye0_projection[1], + eye0_projection[2], eye0_projection[3], eye1_projection[0], + eye1_projection[1], eye1_projection[2], eye1_projection[3], + field_of_view[0], field_of_view[1], field_of_view[2], field_of_view[3], + half_ipd); + } else { + Log(" [vr-device][evidence] eye-init-failed self=%p ready=%u fb0=%p " + "fb1=%p tex0=%p tex1=%p width=%u height=%u\n", + self, ready, framebuffer0, framebuffer1, texture0, texture1, width, + height); + } +} + +void RobloxVrDeviceBridge::OnEyeGetterCall(void *device_object, int eye_index, + void *framebuffer) { + if (!active_.load(std::memory_order_acquire)) { + return; + } + if (eye_index < 0 || eye_index > 1) { + Log(" [vr-device] eye getter called with unexpected index %d self=%p\n", + eye_index, device_object); + return; + } + g_eye_object = device_object; + g_eye_framebuffer = framebuffer; + g_eye_index = eye_index; + const std::uint64_t calls = + eye_call_totals_[eye_index].fetch_add(1, std::memory_order_relaxed) + 1; + if (calls <= 4) { + Log(" [vr-device][evidence] eye-getter self=%p eye=%d framebuffer=%p " + "call=%" PRIu64 " tid=%" PRIu64 "\n", + device_object, eye_index, framebuffer, calls, CurrentThreadId()); + } else if (calls % kEyeGetterSummaryInterval == 0) { + Log(" [vr-device][evidence] eye-getter-summary self=%p eye0=%" PRIu64 + " eye1=%" PRIu64 " last_framebuffer=%p\n", + device_object, eye_getter_call_count(0), eye_getter_call_count(1), + framebuffer); + } +} + +int RobloxVrDeviceBridge::captured_device_count() const { + return captured_count_.load(std::memory_order_relaxed); +} + +int RobloxVrDeviceBridge::initialized_device_count() const { + return initialized_count_.load(std::memory_order_relaxed); +} + +std::uint64_t RobloxVrDeviceBridge::eye_getter_call_count(int eye_index) const { + if (eye_index < 0 || eye_index > 1) { + return 0; + } + return eye_call_totals_[eye_index].load(std::memory_order_relaxed); +} + +Status NotifyRobloxVrImageLoaded(std::uintptr_t image_base) { + RobloxVrDeviceBridge *bridge = + g_active_bridge.load(std::memory_order_acquire); + if (bridge == nullptr) { + return Status::Ok(); + } + if (image_base == 0) { + return FailedPrecondition( + "Roblox VR device bridge is armed but libroblox base is unknown"); + } + return bridge->Activate(image_base); +} + +} // namespace mocktail::vr diff --git a/src/vr/roblox_vr_device_bridge_internal.h b/src/vr/roblox_vr_device_bridge_internal.h new file mode 100644 index 00000000..d7b39871 --- /dev/null +++ b/src/vr/roblox_vr_device_bridge_internal.h @@ -0,0 +1,82 @@ +#ifndef MOCKTAIL_VR_ROBLOX_VR_DEVICE_BRIDGE_INTERNAL_H_ +#define MOCKTAIL_VR_ROBLOX_VR_DEVICE_BRIDGE_INTERNAL_H_ + +#include +#include + +#include "compat/build_profile.h" + +namespace mocktail::vr { struct ScriptedPoseSample; } +namespace mocktail::vr::internal { + +// Writes only a caller-owned state copy; preserves all resource readiness. +bool ApplyXrPose(void* state, const ScriptedPoseSample& pose); + +// Guest ABI layout constants of RBX::Graphics::DebugDeviceVR. They are pinned +// per exact build by the machine-code contracts below; changing payload +// requires re-deriving both. +inline constexpr std::size_t kObjectNameOffset = 0x8; +inline constexpr std::size_t kObjectTypeOffset = 0x10; +inline constexpr std::uint32_t kExpectedObjectType = 6; +inline constexpr std::size_t kStateOffset = 0x14; +inline constexpr std::size_t kStateCopySize = 0x138; +inline constexpr std::size_t kStateReadyOffsetInCopy = 0x133; +inline constexpr std::size_t kReadyByteOffset = 0x147; +inline constexpr std::size_t kFramebufferSlotOffset = 0x150; +inline constexpr std::size_t kTextureSlotOffset = 0x170; +inline constexpr std::size_t kEyeSlotStride = 0x10; +inline constexpr std::size_t kWidthOffset = 0x190; +inline constexpr std::size_t kHeightOffset = 0x194; +inline constexpr std::size_t kGraphicsDeviceOffset = 0x198; +// Per-eye camera parameters filled by the constructor: symmetric +-0.25 stud +// view offsets (half IPD), tan-space projection values and FOV degrees. +inline constexpr std::size_t kEye0ViewOffset = 0x10c; +inline constexpr std::size_t kEye1ViewOffset = 0x118; +inline constexpr std::size_t kEye0ProjectionOffset = 0x124; +inline constexpr std::size_t kEye1ProjectionOffset = 0x134; +inline constexpr std::size_t kFieldOfViewOffset = 0x1a0; +inline constexpr std::size_t kHalfInterpupillaryOffset = 0x1b8; + +inline constexpr std::size_t kStateGetterVtableSlot = 0x28; +inline constexpr std::size_t kEyeGetterVtableSlot = 0x48; +inline constexpr std::size_t kEyeInitializerVtableSlot = 0xd0; + +inline constexpr std::size_t kStateGetterContractSize = 33; +inline constexpr std::size_t kEyeGetterContractSize = 16; +inline constexpr std::size_t kConstructorContractSize = 664; +inline constexpr std::size_t kInitializerContractSize = 700; + +enum class EyeResourceState { kEmpty, kReady, kInvalid }; +// Reads only the current live object's ABI bytes. A ready byte alone is +// insufficient; dimensions and both distinct framebuffer/texture slots count. +EyeResourceState InspectEyeResources(const void *object); + +// Exact 33-byte state getter: sret ABI (rdi = hidden 0x138-byte result +// buffer, rsi = device), copies state from self+0x14 and returns the buffer. +bool HasExpectedVrStateGetterContract(const std::uint8_t *code, + std::size_t size); + +// Exact 16-byte eye framebuffer getter: self + 0x150 + eyeIndex * 16. +bool HasExpectedVrEyeGetterContract(const std::uint8_t *code, std::size_t size); + +// Constructor: captures rcx (graphics device) into self+0x198, width/height +// into +0x190/+0x194, type dword 6 into +0x10, and clears the state block. +bool HasExpectedVrConstructorContract(const std::uint8_t *code, + std::size_t size); + +// Eye-resource initializer: raises ready byte self+0x147 plus +0x94/+0x144 +// and creates the per-eye resources through the graphics-device vtable slot +// taken from the profile (2998 uses +0x180; other builds differ). +bool HasExpectedVrEyeInitializerContract( + const std::uint8_t *code, std::size_t size, + const compat::VrDebugDeviceBridgeProfile &profile); + +// DebugDeviceVR vtable must point the three interposed/used slots at the +// exact profile RVAs before any interposition. +bool HasExpectedVrDebugDeviceVtable( + const std::uintptr_t *vtable, std::uintptr_t image_base, + const compat::VrDebugDeviceBridgeProfile &profile); + +} // namespace mocktail::vr::internal + +#endif // MOCKTAIL_VR_ROBLOX_VR_DEVICE_BRIDGE_INTERNAL_H_ diff --git a/src/vr/vr_mirror.cc b/src/vr/vr_mirror.cc new file mode 100644 index 00000000..1c3f7228 --- /dev/null +++ b/src/vr/vr_mirror.cc @@ -0,0 +1,268 @@ +#include "mocktail/vr/vr_mirror.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace mocktail::vr { +namespace { + +// Seqlock writers/reader fences: the header is plain shared memory, so +// ordering between the sequence counter and the payload is enforced here. +void StoreFence() { std::atomic_thread_fence(std::memory_order_release); } + +std::uint32_t FindHostVisibleMemoryType(const MirrorVkProcs& vk, + VkPhysicalDevice physical, + std::uint32_t type_bits) { + VkPhysicalDeviceMemoryProperties properties{}; + vk.GetPhysicalDeviceMemoryProperties(physical, &properties); + const auto required = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + // Prefer HOST_CACHED system memory: CPU reads of device-local BAR1 memory + // cost ~10ms per 2 MB pair on this GPU, while cached RAM readback is ~1ms. + for (std::uint32_t index = 0; index < properties.memoryTypeCount; ++index) { + if ((type_bits & (1u << index)) != 0 && + (properties.memoryTypes[index].propertyFlags & + (required | VK_MEMORY_PROPERTY_HOST_CACHED_BIT)) == + (required | VK_MEMORY_PROPERTY_HOST_CACHED_BIT)) { + return index; + } + } + for (std::uint32_t index = 0; index < properties.memoryTypeCount; ++index) { + if ((type_bits & (1u << index)) != 0 && + (properties.memoryTypes[index].propertyFlags & required) == required) { + return index; + } + } + return UINT32_MAX; +} + +} // namespace + +VrMirror::~VrMirror() { Close(); } + +bool VrMirror::Open(const std::string& shm_name, std::uint32_t width, + std::uint32_t height, std::string* error) { + if (mapping_ != nullptr) { + const auto* header = static_cast(mapping_); + if (shm_name_ == shm_name && header->width == width && header->height == height) { + return true; + } + Close(); + } + if (shm_name.empty() || shm_name[0] != '/' || width == 0 || height == 0) { + if (error != nullptr) { + *error = "invalid mirror segment name or extent"; + } + return false; + } + const std::uint64_t bytes = VrMirrorBytes(width, height); + const int descriptor = + shm_open(shm_name.c_str(), O_CREAT | O_RDWR | O_CLOEXEC, 0600); + if (descriptor < 0) { + if (error != nullptr) { + *error = "shm_open failed: " + std::string(std::strerror(errno)); + } + return false; + } + if (ftruncate(descriptor, static_cast(bytes)) != 0) { + if (error != nullptr) { + *error = "ftruncate failed: " + std::string(std::strerror(errno)); + } + close(descriptor); + (void)shm_unlink(shm_name.c_str()); + return false; + } + void* mapping = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_SHARED, + descriptor, 0); + close(descriptor); + if (mapping == MAP_FAILED) { + if (error != nullptr) { + *error = "mmap failed: " + std::string(std::strerror(errno)); + } + (void)shm_unlink(shm_name.c_str()); + return false; + } + mapping_ = mapping; + mapping_bytes_ = bytes; + shm_name_ = shm_name; + auto* header = static_cast(mapping_); + std::memset(header, 0, sizeof(*header)); + header->magic = kVrMirrorMagic; + header->header_bytes = kVrMirrorHeaderBytes; + header->width = width; + header->height = height; + StoreFence(); + return true; +} + +void VrMirror::Close() { + if (mapping_ != nullptr) { + munmap(mapping_, mapping_bytes_); + mapping_ = nullptr; + mapping_bytes_ = 0; + } + if (!shm_name_.empty()) { + (void)shm_unlink(shm_name_.c_str()); + shm_name_.clear(); + } + published_pairs_ = 0; +} + +bool VrMirror::EnsureStaging(const MirrorVkProcs& vk, VkDevice device, + VkPhysicalDevice physical, std::uint32_t width, + std::uint32_t height, std::string* error) { + if (staging_device_ == device && staging_[0] != VK_NULL_HANDLE && + staging_[1] != VK_NULL_HANDLE && staging_width_ == width && + staging_height_ == height) { + return true; + } + DestroyStaging(vk); + staging_device_ = device; // Also owns partially constructed allocations. + const VkDeviceSize size = + static_cast(width) * height * 4u; + VkPhysicalDeviceMemoryProperties properties{}; + vk.GetPhysicalDeviceMemoryProperties(physical, &properties); + for (int eye = 0; eye < 2; ++eye) { + VkBufferCreateInfo buffer_info{}; + buffer_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + buffer_info.size = size; + buffer_info.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + if (vk.CreateBuffer(device, &buffer_info, nullptr, &staging_[eye]) != + VK_SUCCESS) { + if (error != nullptr) { + *error = "mirror staging buffer creation failed"; + } + DestroyStaging(vk); + return false; + } + VkMemoryRequirements requirements{}; + vk.GetBufferMemoryRequirements(device, staging_[eye], &requirements); + const std::uint32_t type = + FindHostVisibleMemoryType(vk, physical, requirements.memoryTypeBits); + if (type == UINT32_MAX) { + if (error != nullptr) { + *error = "no host-visible coherent memory for the mirror"; + } + DestroyStaging(vk); + return false; + } + VkMemoryAllocateInfo allocation{}; + allocation.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + allocation.allocationSize = requirements.size; + allocation.memoryTypeIndex = type; + if (vk.AllocateMemory(device, &allocation, nullptr, + &staging_memory_[eye]) != VK_SUCCESS) { + if (error != nullptr) { + *error = "mirror staging allocation failed"; + } + DestroyStaging(vk); + return false; + } + if (vk.BindBufferMemory(device, staging_[eye], staging_memory_[eye], 0) != + VK_SUCCESS || + vk.MapMemory(device, staging_memory_[eye], 0, size, 0, + &staging_map_[eye]) != VK_SUCCESS) { + if (error != nullptr) { + *error = "mirror staging bind/map failed"; + } + DestroyStaging(vk); + return false; + } + } + staging_device_ = device; + staging_width_ = width; + staging_height_ = height; + return true; +} + +void VrMirror::DestroyStaging(const MirrorVkProcs& vk) { + for (int eye = 0; eye < 2; ++eye) { + if (staging_map_[eye] != nullptr && staging_memory_[eye] != VK_NULL_HANDLE) { + vk.UnmapMemory(staging_device_, staging_memory_[eye]); + staging_map_[eye] = nullptr; + } + if (staging_memory_[eye] != VK_NULL_HANDLE) { + vk.FreeMemory(staging_device_, staging_memory_[eye], nullptr); + staging_memory_[eye] = VK_NULL_HANDLE; + } + if (staging_[eye] != VK_NULL_HANDLE) { + vk.DestroyBuffer(staging_device_, staging_[eye], nullptr); + staging_[eye] = VK_NULL_HANDLE; + } + } + staging_width_ = 0; + staging_height_ = 0; + staging_device_ = VK_NULL_HANDLE; +} + +void VrMirror::RecordEyeCopy(const MirrorVkProcs& vk, VkCommandBuffer cmd, + int eye, VkImage source, + VkImageLayout /*final_layout*/, + std::uint32_t width, std::uint32_t height) { + // The caller's batched command buffer already holds the source in + // TRANSFER_SRC_OPTIMAL for the XR swapchain copy and restores its final + // layout afterwards; recording only the copy here keeps one transition pair + // per eye per frame. + if (eye < 0 || eye > 1 || staging_[eye] == VK_NULL_HANDLE || + staging_width_ != width || staging_height_ != height) { + return; + } + VkBufferImageCopy region{}; + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.imageExtent = {width, height, 1}; + vk.CmdCopyImageToBuffer(cmd, source, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + staging_[eye], 1, ®ion); +} + +void VrMirror::Publish(std::uint64_t frame, VkFormat format, + std::uint32_t width, std::uint32_t height, + std::uint64_t now_ns) { + if (mapping_ == nullptr || staging_map_[0] == nullptr || + staging_map_[1] == nullptr || staging_width_ != width || + staging_height_ != height) { + return; + } + PublishFromBytesForTest(frame, format, width, height, now_ns, staging_map_[0], + staging_map_[1]); +} + +void VrMirror::PublishFromBytesForTest(std::uint64_t frame, VkFormat format, + std::uint32_t width, + std::uint32_t height, + std::uint64_t now_ns, const void* eye0, + const void* eye1) { + if (mapping_ == nullptr || eye0 == nullptr || eye1 == nullptr) { + return; + } + auto* header = static_cast(mapping_); + auto* pixels = static_cast(mapping_) + kVrMirrorHeaderBytes; + const std::uint64_t eye_bytes = + static_cast(width) * height * 4u; + if (kVrMirrorHeaderBytes + 2 * eye_bytes > mapping_bytes_) { + return; + } + // Seqlock: odd sequence marks the payload unstable for readers. + const std::uint64_t sequence = header->seq + 1; + header->seq = sequence | 1ull; + StoreFence(); + header->width = width; + header->height = height; + header->format = static_cast(format); + header->frame = frame; + header->producer_ns = now_ns; + std::memcpy(pixels, eye0, eye_bytes); + std::memcpy(pixels + eye_bytes, eye1, eye_bytes); + header->flags = kVrMirrorFlagValid; + header->published_pairs = ++published_pairs_; + StoreFence(); + header->seq = sequence + 1; +} + +} // namespace mocktail::vr diff --git a/src/vr/vr_perf.cc b/src/vr/vr_perf.cc new file mode 100644 index 00000000..933cd5ba --- /dev/null +++ b/src/vr/vr_perf.cc @@ -0,0 +1,220 @@ +#include "mocktail/vr/vr_perf.h" + +#include + +#include +#include + +namespace mocktail::vr::perf { +namespace { + +Collector& Global() { + static Collector collector; + return collector; +} + +} // namespace + +bool Enabled() { + static const bool enabled = [] { + const char* value = std::getenv("MOCKTAIL_VR_PERF"); + return value != nullptr && value[0] != '\0' && *value != '0'; + }(); + if (!enabled) return false; + static const std::uint64_t start = [] { + const char* value = std::getenv("MOCKTAIL_VR_MEASURE_START_NS"); + return value ? std::strtoull(value, nullptr, 10) : 0ULL; + }(); + static const std::uint64_t end = [] { + const char* value = std::getenv("MOCKTAIL_VR_MEASURE_END_NS"); + return value ? std::strtoull(value, nullptr, 10) : ~0ULL; + }(); + if (start == 0 && end == ~0ULL) return true; + const auto now = NowNs(); + return now >= start && now <= end; +} + +std::uint64_t NowNs() { + timespec ts{}; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + return 0; + } + return static_cast(ts.tv_sec) * 1'000'000'000ULL + + static_cast(ts.tv_nsec); +} + +const char* StageName(Stage stage) { + switch (stage) { + case Stage::kFrameCycle: return "frame_cycle"; + case Stage::kXrWaitFrame: return "xr_wait_frame"; + case Stage::kXrBeginFrame: return "xr_begin_frame"; + case Stage::kXrLocateViews: return "xr_locate_views"; + case Stage::kSwapchainAcquire: return "swapchain_acquire"; + case Stage::kSwapchainWait: return "swapchain_wait"; + case Stage::kCopyRecord: return "copy_record"; + case Stage::kCopySubmit: return "copy_submit"; + case Stage::kCopyFenceWait: return "copy_fence_wait"; + case Stage::kMirrorPublish: return "mirror_publish"; + case Stage::kEvidence: return "evidence"; + case Stage::kStageCount: break; + } + return "unknown"; +} + +const char* CounterName(Counter counter) { + switch (counter) { + case Counter::kPresents: return "presents"; + case Counter::kFramesBegun: return "frames_begun"; + case Counter::kFramesSubmitted: return "frames_submitted"; + case Counter::kFramesSkipped: return "frames_skipped"; + case Counter::kSkipShouldRender: return "skip_should_render"; + case Counter::kSkipPoseInvalid: return "skip_pose_invalid"; + case Counter::kSkipEyesUnbound: return "skip_eyes_unbound"; + case Counter::kSkipEyeFrameMismatch: return "skip_eye_frame_mismatch"; + case Counter::kSubmits: return "submits"; + case Counter::kFenceWaits: return "fence_waits"; + case Counter::kMirrorPairs: return "mirror_pairs"; + case Counter::kMirrorDrops: return "mirror_drops"; + case Counter::kSkipVisibilityLost: return "skip_visibility_lost"; + case Counter::kReferenceSpaceChanges: return "reference_space_changes"; + case Counter::kVisibilityLost: return "visibility_lost"; + case Counter::kVisibilityGained: return "visibility_gained"; + case Counter::kFramesInvalidated: return "frames_invalidated"; + case Counter::kCounterCount: break; + } + return "unknown"; +} + +void Collector::Record(Stage stage, std::uint64_t nanos) { + if (!Enabled()) { + return; + } + const std::size_t index = static_cast(stage); + if (index >= static_cast(Stage::kStageCount)) { + return; + } + StageStat& stat = stages_[index]; + if (stat.samples.size() < kReservoir) { + stat.samples.resize(kReservoir, 0); + } + if (stat.count == kReservoir) { + stat.total_ns -= stat.samples[stat.head]; + } + stat.samples[stat.head] = nanos; + stat.head = (stat.head + 1) % kReservoir; + if (stat.count < kReservoir) { + ++stat.count; + } + stat.total_ns += nanos; +} + +void Collector::Count(Counter counter, std::uint64_t delta) { + if (!Enabled()) { + return; + } + const std::size_t index = static_cast(counter); + if (index >= static_cast(Counter::kCounterCount)) { + return; + } + counters_[index] += delta; +} + +std::vector Collector::Percentiles(Stage stage) const { + const std::size_t index = static_cast(stage); + if (index >= static_cast(Stage::kStageCount)) { + return {}; + } + const StageStat& stat = stages_[index]; + if (stat.count == 0) { + return {0, 0, 0, 0}; + } + std::vector sorted(stat.samples.begin(), + stat.samples.begin() + stat.count); + std::sort(sorted.begin(), sorted.end()); + auto pick = [&sorted](double fraction) { + const std::size_t position = static_cast( + fraction * static_cast(sorted.size() - 1)); + return sorted[position]; + }; + return {pick(0.50), pick(0.95), pick(0.99), sorted.back()}; +} + +std::size_t Collector::Samples(Stage stage) const { + const std::size_t index = static_cast(stage); + if (index >= static_cast(Stage::kStageCount)) { + return 0; + } + return stages_[index].count; +} + +std::uint64_t Collector::counter(Counter counter_id) const { + const std::size_t index = static_cast(counter_id); + if (index >= static_cast(Counter::kCounterCount)) { + return 0; + } + return counters_[index]; +} + +void Collector::Reset() { + for (StageStat& stat : stages_) { + stat.head = 0; + stat.count = 0; + stat.total_ns = 0; + } + for (std::uint64_t& value : counters_) { + value = 0; + } + last_emit_ns_ = 0; + window_start_ns_ = 0; +} + +void Collector::EmitIfDue(std::uint64_t now_ns, const char* context) { + if (!Enabled() || now_ns == 0) { + return; + } + if (window_start_ns_ == 0) { + window_start_ns_ = now_ns; + last_emit_ns_ = now_ns; + return; + } + if (now_ns - last_emit_ns_ < kIntervalNs) { + return; + } + const double window_seconds = + static_cast(now_ns - window_start_ns_) / 1e9; + std::fprintf(stderr, " [vr-perf] window=%.1fs context=%s\n", + window_seconds, context != nullptr ? context : ""); + for (std::size_t index = 0; + index < static_cast(Stage::kStageCount); ++index) { + const StageStat& stat = stages_[index]; + if (stat.count == 0) { + continue; + } + const std::vector ranked = + Percentiles(static_cast(index)); + std::fprintf(stderr, + " [vr-perf] stage=%-16s n=%zu avg=%.2fms p50=%.2fms " + "p95=%.2fms p99=%.2fms max=%.2fms\n", + StageName(static_cast(index)), stat.count, + static_cast(stat.total_ns) / + static_cast(stat.count) / 1e6, + ranked[0] / 1e6, ranked[1] / 1e6, ranked[2] / 1e6, + ranked[3] / 1e6); + } + for (std::size_t index = 0; + index < static_cast(Counter::kCounterCount); ++index) { + const std::uint64_t value = counters_[index]; + if (value == 0) { + continue; + } + std::fprintf(stderr, " [vr-perf] counter=%-22s total=%llu rate=%.2f/s\n", + CounterName(static_cast(index)), + static_cast(value), + static_cast(value) / std::max(window_seconds, 1e-9)); + } + last_emit_ns_ = now_ns; +} + +Collector& ProcessCollector() { return Global(); } + +} // namespace mocktail::vr::perf diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 21bb1e03..863b8e13 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -3,10 +3,60 @@ # Licensed under the Apache License, Version 2.0 (the "License") find_package(Python3 COMPONENTS Interpreter REQUIRED) -add_test(NAME OpenXrSimulatorLauncher - COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/vr/simulator_launcher_test.py" - "${CMAKE_BINARY_DIR}/mocktail-vr-simulated") + +# Test-only implicit Vulkan layer that captures GPU-level native stereo +# evidence (eye render-pass counts and per-eye pixel readback). +# Never linked into production code. +add_library(mocktail_vr_evidence_layer MODULE vr/vr_evidence_layer.cc) +target_link_libraries(mocktail_vr_evidence_layer PRIVATE Vulkan::Headers + ${CMAKE_DL_LIBS}) +mocktail_apply_compile_options(mocktail_vr_evidence_layer) +set_target_properties(mocktail_vr_evidence_layer PROPERTIES + PREFIX "lib" OUTPUT_NAME "mocktail_vr_evidence_layer") +file(GENERATE + OUTPUT "${CMAKE_BINARY_DIR}/tests/vr/VkLayer_mocktail_vr_evidence.json" + CONTENT [=[{ + "file_format_version": "1.2.0", + "layer": { + "name": "VK_LAYER_MOCKTAIL_vr_evidence", + "type": "GLOBAL", + "library_path": "$", + "api_version": "1.3.280", + "implementation_version": "1", + "description": "Mocktail VR native stereo evidence capture (test-only)", + "disable_environment": {"MOCKTAIL_VR_EVIDENCE_LAYER_DISABLE": "1"} + } +} +]=]) + +add_executable(vr_mirror_test vr/vr_mirror_test.cc) +target_link_libraries(vr_mirror_test PRIVATE Mocktail::VR GTest::gtest_main) +mocktail_apply_compile_options(vr_mirror_test) +add_executable(vr_pose_math_test vr/vr_pose_math_test.cc) +target_include_directories(vr_pose_math_test PRIVATE "${PROJECT_SOURCE_DIR}/include") +target_link_libraries(vr_pose_math_test PRIVATE GTest::gtest_main) +mocktail_apply_compile_options(vr_pose_math_test) +add_executable(vr_evidence_layer_test vr/vr_evidence_layer_test.cc) +target_link_libraries(vr_evidence_layer_test PRIVATE Vulkan::Headers + ${CMAKE_DL_LIBS} GTest::gtest_main) +mocktail_apply_compile_options(vr_evidence_layer_test) + +add_executable(roblox_vr_device_bridge_test vr/roblox_vr_device_bridge_test.cc) +target_include_directories(roblox_vr_device_bridge_test PRIVATE + "${PROJECT_SOURCE_DIR}/src") +target_compile_definitions(roblox_vr_device_bridge_test PRIVATE + "MOCKTAIL_TEST_BINARY_DIRECTORY=\"${CMAKE_CURRENT_BINARY_DIR}\"") +target_link_libraries(roblox_vr_device_bridge_test PRIVATE + Mocktail::VR + Mocktail::Compat + GTest::gtest_main +) +mocktail_apply_compile_options(roblox_vr_device_bridge_test) if(MOCKTAIL_ENABLE_VR) + add_executable(openxr_backend_test vr/openxr_backend_test.cc) + target_include_directories(openxr_backend_test PRIVATE "${PROJECT_SOURCE_DIR}/third_party/OpenXR-SDK/include") + target_link_libraries(openxr_backend_test PRIVATE Mocktail::VR GTest::gtest_main) + gtest_discover_tests(openxr_backend_test) add_library(mocktail_test_openxr_runtime MODULE vr/fake_openxr_runtime.cc vr/fake_openxr_preview.cc) target_link_libraries(mocktail_test_openxr_runtime PRIVATE @@ -888,6 +938,10 @@ target_link_libraries(headless_graphics_test PRIVATE ) include(GoogleTest) +gtest_discover_tests(roblox_vr_device_bridge_test) +gtest_discover_tests(vr_evidence_layer_test) +gtest_discover_tests(vr_mirror_test) +gtest_discover_tests(vr_pose_math_test) gtest_discover_tests(jnivm_test) gtest_discover_tests(runtime_config_test) gtest_discover_tests(runtime_config_file_test) diff --git a/tests/command_line_test.cc b/tests/command_line_test.cc index ee69e957..e36ac00c 100644 --- a/tests/command_line_test.cc +++ b/tests/command_line_test.cc @@ -12,6 +12,20 @@ namespace mocktail { namespace runtime { namespace { +TEST(CommandLineTest, ShortVrAliasSurvivesReexecAndLastOverrideWins) { + const char* arguments[] = {"mocktail", "-vr"}; + const auto parsed = ParseCommandLine(2, arguments); + ASSERT_TRUE(parsed); + ASSERT_TRUE(parsed.options.vr_enabled.has_value()); + EXPECT_TRUE(*parsed.options.vr_enabled); + std::vector reexec; + std::string error; + ASSERT_TRUE(BuildCommandLineReexecArguments(parsed.options, 2, arguments, &reexec, &error)); + EXPECT_EQ(reexec, (std::vector{"-vr"})); + const char* disabled[] = {"mocktail", "-vr", "--no-vr"}; + EXPECT_FALSE(*ParseCommandLine(3, disabled).options.vr_enabled); +} + TEST(CommandLineTest, VrOverrideSurvivesReexecAndCanDisableEnvironmentDefault) { const std::array arguments = {"mocktail", "--vr", "--headless", "--no-vr"}; diff --git a/tests/performance_policy_test.cc b/tests/performance_policy_test.cc index 819f54a3..16bef548 100644 --- a/tests/performance_policy_test.cc +++ b/tests/performance_policy_test.cc @@ -283,6 +283,22 @@ TEST(PerformancePolicyTest, PermissionProtocolDoesNotUseTheTestBypass) { EXPECT_EQ(parsed.at("DFFlagVoiceChatSkipPermissionCheckForTests"), "False"); } +TEST(PerformancePolicyTest, + VrModeOverridesConflictingFlagAndPreservesSettings) { + std::string merged, error; + ASSERT_TRUE(MergeVrClientSettingsOverrides( + true, R"({"FFlagDebugEnableVREmulator":"False","FIntUnrelated":"42"})", + &merged, &error)); + auto json = nlohmann::json::parse(merged); + EXPECT_EQ(json["FFlagDebugEnableVREmulator"], "True"); + EXPECT_EQ(json["FIntUnrelated"], "42"); + ASSERT_TRUE(MergeVrClientSettingsOverrides(false, merged, &merged, &error)); + EXPECT_EQ(nlohmann::json::parse(merged)["FFlagDebugEnableVREmulator"], + "False"); + EXPECT_FALSE(MergeVrClientSettingsOverrides(true, "[]", &merged, &error)); + EXPECT_FALSE(MergeVrClientSettingsOverrides(true, "bad", &merged, &error)); +} + } // namespace } // namespace runtime } // namespace mocktail diff --git a/tests/runtime_config_file_test.cc b/tests/runtime_config_file_test.cc index e4ce0b2e..e4a6cf06 100644 --- a/tests/runtime_config_file_test.cc +++ b/tests/runtime_config_file_test.cc @@ -99,7 +99,7 @@ TEST(RuntimeConfigBootstrapTest, CreatesCompletePrivateFirstRunFile) { LoadRuntimeConfig(MapEnvironment(), file); ASSERT_TRUE(loaded) << loaded.error; EXPECT_TRUE(loaded.file_loaded); - EXPECT_FALSE(loaded.config.vr_enabled()); + EXPECT_EQ(loaded.config.vr_enabled(), kDefaultVrEnabled); EXPECT_EQ(loaded.config.graphics_backend(), GraphicsBackend::kVulkan); EXPECT_EQ(loaded.config.theme_mode(), "roblox"); EXPECT_EQ(loaded.config.frame_rate().mode, FrameRateLimitMode::kUnmanaged); @@ -156,12 +156,12 @@ TEST(RuntimeConfigFileTest, ExportsResolvedVrStateAndRejectsInvalidState) { EXPECT_TRUE(ExportRuntimeConfigEnvironment( RuntimeConfig::FromEnvironment(MapEnvironment()), &error)) << error; - EXPECT_STREQ(std::getenv("MOCKTAIL_VR_ENABLED"), "0"); + EXPECT_STREQ(std::getenv("MOCKTAIL_VR_ENABLED"), kDefaultVrEnabled ? "1" : "0"); EXPECT_FALSE(ExportRuntimeConfigEnvironment( RuntimeConfig::FromEnvironment( MapEnvironment({{"MOCKTAIL_VR_ENABLED", "invalid"}})), &error)); - EXPECT_STREQ(std::getenv("MOCKTAIL_VR_ENABLED"), "0"); + EXPECT_STREQ(std::getenv("MOCKTAIL_VR_ENABLED"), kDefaultVrEnabled ? "1" : "0"); if (saved) setenv("MOCKTAIL_VR_ENABLED", saved->c_str(), 1); else diff --git a/tests/runtime_config_test.cc b/tests/runtime_config_test.cc index 74ac2971..3de61217 100644 --- a/tests/runtime_config_test.cc +++ b/tests/runtime_config_test.cc @@ -37,7 +37,7 @@ TEST(RuntimeConfigTest, UsesSupportedDefaults) { const RuntimeConfig config = RuntimeConfig::FromEnvironment(environment); EXPECT_FALSE(config.headless()); - EXPECT_FALSE(config.vr_enabled()); + EXPECT_EQ(config.vr_enabled(), kDefaultVrEnabled); EXPECT_TRUE(config.vr_valid()); EXPECT_EQ(config.roblox_library_path(), "rbx_bin/libroblox.so"); EXPECT_EQ(config.graphics_backend(), GraphicsBackend::kVulkan); @@ -95,7 +95,7 @@ TEST(RuntimeConfigTest, RejectsMalformedVrEnvironmentInsteadOfEnablingIt) { const auto config = RuntimeConfig::FromEnvironment( MapEnvironment({{"MOCKTAIL_VR_ENABLED", "maybe"}})); EXPECT_FALSE(config.vr_valid()); - EXPECT_FALSE(config.vr_enabled()); + EXPECT_EQ(config.vr_enabled(), kDefaultVrEnabled); } TEST(RuntimeConfigTest, ReadsTypedRuntimeValues) { diff --git a/tests/vr/openxr_backend_test.cc b/tests/vr/openxr_backend_test.cc new file mode 100644 index 00000000..be772a58 --- /dev/null +++ b/tests/vr/openxr_backend_test.cc @@ -0,0 +1,191 @@ +#include +#include +#include "mocktail/vr/openxr_backend.h" +#include "mocktail/vr/roblox_vr_device_bridge.h" + +namespace mocktail::vr { +namespace internal { +struct VrBridgeTestAccess { + static void Enable(RobloxVrDeviceBridge& bridge) { bridge.active_ = true; } +}; +} +struct OpenXrBackendTestAccess { + template static T Handle(std::uintptr_t n) { return reinterpret_cast(n); } + static void Configure(OpenXrBackend& backend) { + backend.vk_instance_ = Handle(1); + backend.vk_device_ = Handle(2); + backend.recording_ = true; + backend.published_pose_.frame = 7; + backend.published_pose_.valid = true; + } + static VkInstance Instance(const OpenXrBackend& b) { return b.vk_instance_; } + static void Attachment(OpenXrBackend& b, int id, void* owner, VkDevice device) { + auto image = Handle(id); + auto& r = b.images_[image]; + r.image = image; r.device = device; r.owner = owner; + r.width = r.height = 500; r.format = VK_FORMAT_R8G8B8A8_UNORM; + r.usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + b.image_views_[Handle(id)] = image; + b.framebuffer_views_[Handle(id)] = {Handle(id)}; + } + static void Pass(OpenXrBackend& b, RobloxVrDeviceBridge& bridge, int eye, int id, void* owner, + VkImageLayout layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL) { + bridge.OnEyeGetterCall(owner, eye, Handle(id)); + VkRenderPassBeginInfo info{}; + info.framebuffer = Handle(id); + info.renderPass = Handle(99); + b.render_pass_final_layouts_[info.renderPass] = {layout}; + b.NoteRenderPassBegin(VK_NULL_HANDLE, &info); + } + static VkImage Image(const OpenXrBackend& b, int eye) { return b.eyes_[eye].image; } + static VkImageLayout Layout(const OpenXrBackend& b, int eye) { return b.eyes_[eye].final_layout; } + static void Invalidate(OpenXrBackend& b) { + b.InvalidateInFlightPose("test"); + } + static void SetFrameOpen(OpenXrBackend& b, bool open) { b.frame_open_ = open; } + static void SetVisibilityLost(OpenXrBackend& b, bool lost) { b.visibility_lost_ = lost; } + static bool VisibilityLost(const OpenXrBackend& b) { return b.visibility_lost_; } + static void NotePose(OpenXrBackend& b, void* owner) { + b.NotePoseApplied(owner, b.published_pose_.frame); + } + static std::size_t AppliedPoses(const OpenXrBackend& b) { + return b.applied_poses_.size(); + } + static void State(OpenXrBackend& b, XrSessionState state) { b.HandleSessionState(state); } + static void ReferenceChange(OpenXrBackend& b, std::int64_t time) { b.HandleReferenceSpaceChange(time); } + static void FrameAt(OpenXrBackend& b, std::uint64_t time) { + b.frame_open_ = true; b.frame_views_valid_ = true; b.frame_display_time_ = time; + b.published_pose_.valid = true; + } + static bool Recovering(OpenXrBackend& b) { return b.session_recovery_pending_; } + static VkDevice Device(OpenXrBackend& b) { return b.vk_device_; } + static std::size_t Resources(OpenXrBackend& b) { return b.images_.size(); } + static void SeedResource(OpenXrBackend& b) { b.images_[Handle(99)] = {}; } + static void LatchCanted(OpenXrBackend& b) { b.canted_rejection_logged_ = true; } + static bool CantedLogged(const OpenXrBackend& b) { return b.canted_rejection_logged_; } +}; +using Access = OpenXrBackendTestAccess; +TEST(OpenXrBackendLifecycle, DeviceDestructionPreservesLiveInstance) { + OpenXrBackend b; + Access::Configure(b); + b.NoteDeviceDestroyed(Access::Handle(3)); + EXPECT_TRUE(b.WantsResourceRecords()); + b.NoteDeviceDestroyed(Access::Handle(2)); + EXPECT_EQ(Access::Instance(b), Access::Handle(1)); + EXPECT_FALSE(b.WantsResourceRecords()); + EXPECT_FALSE(b.PublishedHeadPose().valid); + b.NoteInstanceDestroyed(Access::Handle(1)); + EXPECT_EQ(Access::Instance(b), VK_NULL_HANDLE); +} +TEST(OpenXrBackendBinding, SwitchesOwnerWithoutWaitingForOldResourcesToDie) { + OpenXrBackend b; + RobloxVrDeviceBridge bridge; + internal::VrBridgeTestAccess::Enable(bridge); + Access::Configure(b); + auto* first = Access::Handle(100); + auto* second = Access::Handle(200); + for (int id = 10; id < 14; ++id) + Access::Attachment(b, id, id < 12 ? first : second, Access::Handle(2)); + Access::Pass(b, bridge, 0, 10, first); + Access::Pass(b, bridge, 1, 11, first); + ASSERT_TRUE(b.EyeBindingComplete()); + Access::Pass(b, bridge, 0, 12, second); + EXPECT_FALSE(b.EyeBindingComplete()); + Access::Pass(b, bridge, 1, 13, second); + EXPECT_TRUE(b.EyeBindingComplete()); + EXPECT_EQ(Access::Image(b, 0), Access::Handle(12)); + EXPECT_EQ(Access::Image(b, 1), Access::Handle(13)); + Access::Pass(b, bridge, 0, 12, second, VK_IMAGE_LAYOUT_GENERAL); + EXPECT_EQ(Access::Layout(b, 0), VK_IMAGE_LAYOUT_GENERAL); +} +TEST(OpenXrBackendBinding, RejectsAnotherDeviceImages) { + OpenXrBackend b; + RobloxVrDeviceBridge bridge; + internal::VrBridgeTestAccess::Enable(bridge); + Access::Configure(b); + auto* owner = Access::Handle(100); + Access::Attachment(b, 10, owner, Access::Handle(3)); + Access::Pass(b, bridge, 0, 10, owner); + EXPECT_EQ(Access::Image(b, 0), VK_NULL_HANDLE); + EXPECT_FALSE(b.EyeBindingComplete()); +} + +TEST(OpenXrBackendLifecycle, FutureRecenterPreservesEarlierFrameAndLateChangeDropsIt) { + OpenXrBackend b; + Access::Configure(b); + Access::FrameAt(b, 100); + Access::ReferenceChange(b, 200); + EXPECT_TRUE(b.PublishedHeadPose().valid); + Access::ReferenceChange(b, 100); + EXPECT_FALSE(b.PublishedHeadPose().valid); + Access::FrameAt(b, 300); + Access::ReferenceChange(b, 200); + EXPECT_FALSE(b.PublishedHeadPose().valid); +} + +TEST(OpenXrBackendLifecycle, SynchronizedVisibilityTransitionUsesActualStateHandler) { + OpenXrBackend b; + Access::Configure(b); + Access::State(b, XR_SESSION_STATE_VISIBLE); + Access::State(b, XR_SESSION_STATE_SYNCHRONIZED); + EXPECT_TRUE(Access::VisibilityLost(b)); + EXPECT_FALSE(b.PublishedHeadPose().valid); + Access::FrameAt(b, 100); + Access::State(b, XR_SESSION_STATE_VISIBLE); + EXPECT_FALSE(Access::VisibilityLost(b)); + EXPECT_FALSE(b.PublishedHeadPose().valid); +} + +TEST(OpenXrBackendLifecycle, SessionLossKeepsLiveDeviceProvenanceButDeviceDestructionCancelsRecovery) { + OpenXrBackend b; + Access::Configure(b); + Access::SeedResource(b); + Access::State(b, XR_SESSION_STATE_LOSS_PENDING); + EXPECT_TRUE(Access::Recovering(b)); + EXPECT_EQ(Access::Device(b), Access::Handle(2)); + EXPECT_EQ(Access::Resources(b), 1u); + EXPECT_TRUE(b.WantsResourceRecords()); + EXPECT_FALSE(b.PublishedHeadPose().valid); + b.NoteDestroyImage(Access::Handle(99)); + EXPECT_EQ(Access::Resources(b), 0u); + b.NoteDeviceDestroyed(Access::Handle(2)); + EXPECT_FALSE(Access::Recovering(b)); + EXPECT_EQ(Access::Device(b), VK_NULL_HANDLE); +} + +TEST(OpenXrBackendLifecycle, InvalidationDropsPoseAndAppliedRecord) { + OpenXrBackend b; + Access::Configure(b); + Access::SetFrameOpen(b, true); + auto* owner = Access::Handle(100); + Access::NotePose(b, owner); + ASSERT_EQ(Access::AppliedPoses(b), 1u); + ASSERT_TRUE(b.PublishedHeadPose().valid); + // A reference-space change / visibility transition must drop the pose and + // the per-owner application record so the frame cycle cannot retag the + // in-flight images with a pose from the previous origin. + Access::Invalidate(b); + EXPECT_FALSE(b.PublishedHeadPose().valid); + EXPECT_EQ(Access::AppliedPoses(b), 0u); +} + +TEST(OpenXrBackendLifecycle, VisibilityLostStateSurvivesUntilSessionReset) { + OpenXrBackend b; + Access::Configure(b); + Access::SetVisibilityLost(b, true); + EXPECT_TRUE(Access::VisibilityLost(b)); + // Teardown (device destroyed) resets the flag so a recreated session starts + // visible; it must not inherit IDLE state from the previous session. + b.NoteDeviceDestroyed(Access::Handle(2)); + EXPECT_FALSE(Access::VisibilityLost(b)); +} + +TEST(OpenXrBackendLifecycle, TeardownClearsCantedRejectionLatch) { + OpenXrBackend b; + Access::Configure(b); + Access::LatchCanted(b); + ASSERT_TRUE(Access::CantedLogged(b)); + b.NoteDeviceDestroyed(Access::Handle(2)); + EXPECT_FALSE(Access::CantedLogged(b)); +} +} // namespace mocktail::vr diff --git a/tests/vr/preview_contract_test.py b/tests/vr/preview_contract_test.py index ce67dc39..7afbe57a 100644 --- a/tests/vr/preview_contract_test.py +++ b/tests/vr/preview_contract_test.py @@ -137,5 +137,5 @@ def check(result, code): capture_output=True, text=True, timeout=5) assert result.returncode == 1, result assert "XR_ERROR_RUNTIME_UNAVAILABLE" in result.stdout, result.stdout - assert "mocktail-vr-simulated" in result.stdout, result.stdout - print("missing runtime diagnostic offers simulator: passed") + assert "XR_RUNTIME_JSON" in result.stdout, result.stdout + print("missing runtime diagnostic explains runtime selection: passed") diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc new file mode 100644 index 00000000..6f9cbe3e --- /dev/null +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -0,0 +1,503 @@ +// Contract tests for the experimental DebugDeviceVR bridge. They pin the +// machine-code contracts of the exact 2998 payload ABI and the fail-closed +// bridge lifecycle; they do not execute any guest code. +#include + +#include +#include +#include +#include +#include +#include "mocktail/vr/openxr_backend.h" +#include +#include +#include +#include +#include + +#include "compat/build_profile.h" +#include "mocktail/vr/roblox_vr_device_bridge.h" +#include "vr/roblox_vr_device_bridge_internal.h" + +namespace mocktail::vr { +namespace internal { +// Inject host fixture callbacks without executing or relaxing production ELF +// activation checks. Objects retain the guest byte layout, never host C++ ABI. +struct VrBridgeTestAccess { + static void Configure(RobloxVrDeviceBridge &bridge, std::uintptr_t base, + std::uintptr_t size, + const compat::VrDebugDeviceBridgeProfile &profile, + RobloxVrDeviceBridge::StateGetterFn getter) { + bridge.profile_ = profile; + bridge.image_base_ = base; + bridge.image_size_ = size; + bridge.original_state_getter_ = getter; + bridge.active_ = true; + } +}; +} // namespace internal +namespace { + +using internal::kConstructorContractSize; +using internal::kEyeGetterContractSize; +using internal::kInitializerContractSize; +using internal::kStateGetterContractSize; + +// Real 2998 machine code, transcribed from the verified payload disassembly +// (ade08266c67aee88ec9c1d00902150e1684dad3a). +const std::vector kStateGetter = { + 0x55, 0x48, 0x89, 0xe5, 0x53, 0x50, 0x48, 0x89, 0xfb, 0x48, 0x83, + 0xc6, 0x14, 0xba, 0x38, 0x01, 0x00, 0x00, 0xe8, 0xd3, 0xb5, 0x36, + 0x03, 0x48, 0x89, 0xd8, 0x48, 0x83, 0xc4, 0x08, 0x5b, 0x5d, 0xc3}; + +const std::vector kEyeGetter = {0x48, 0x63, 0xc6, 0x48, 0xc1, 0xe0, + 0x04, 0x48, 0x8b, 0x84, 0x07, 0x50, + 0x01, 0x00, 0x00, 0xc3}; + +std::vector ValidConstructor() { + std::vector code(kConstructorContractSize, 0x90); + const std::vector prologue = { + 0x55, 0x48, 0x89, 0xe5, 0x41, 0x57, 0x41, 0x56, 0x41, 0x55, + 0x41, 0x54, 0x53, 0x48, 0x83, 0xec, 0x28, 0x49, 0x89, 0xcf, + 0x41, 0x89, 0xd4, 0x41, 0x89, 0xf5, 0x48, 0x89, 0xfb}; + std::memcpy(code.data(), prologue.data(), prologue.size()); + const std::vector> markers = { + {0x48, 0x8d, 0x05, 0xf2, 0x74, 0x47, 0x03}, + {0x48, 0x89, 0x07}, + {0xc7, 0x47, 0x10, 0x06, 0x00, 0x00, 0x00}, + {0x4c, 0x8d, 0x77, 0x14, 0xba, 0x34, 0x01, 0x00, 0x00}, + {0x44, 0x89, 0xab, 0x90, 0x01, 0x00, 0x00, 0x44, 0x89, 0xa3, 0x94, 0x01, + 0x00, 0x00}, + {0x4c, 0x89, 0xbb, 0x98, 0x01, 0x00, 0x00}, + }; + size_t offset = 64; + for (const auto &marker : markers) { + std::memcpy(code.data() + offset, marker.data(), marker.size()); + offset += marker.size() + 8; + } + return code; +} + +compat::VrDebugDeviceBridgeProfile TestProfile() { + compat::VrDebugDeviceBridgeProfile profile{}; + profile.vtable_rva = 0x6c87b38; + profile.constructor_rva = 0x3810614; + profile.state_getter_rva = 0x38109f6; + profile.eye_getter_rva = 0x38109e6; + profile.eye_initializer_rva = 0x3810a18; + profile.emulator_flag_storage_rva = 0x7369920; + profile.device_create_framebuffer_vtable_offset = 0x180; + return profile; +} + +std::vector +ValidInitializer(const compat::VrDebugDeviceBridgeProfile &profile) { + std::vector code(kInitializerContractSize, 0x90); + const std::vector prologue = { + 0x55, 0x48, 0x89, 0xe5, 0x41, 0x57, 0x41, 0x56, 0x41, 0x55, 0x41, 0x54, + 0x53, 0x48, 0x81, 0xec, 0xa8, 0x00, 0x00, 0x00, 0x48, 0x89, 0xb5, 0x50}; + std::memcpy(code.data(), prologue.data(), prologue.size()); + const uint32_t slot = + static_cast(profile.device_create_framebuffer_vtable_offset); + const std::vector> markers = { + {0xb1, 0x01, 0x88, 0x8f, 0x47, 0x01, 0x00, 0x00, 0x88, 0x8f, + 0x94, 0x00, 0x00, 0x00, 0x88, 0x8f, 0x44, 0x01, 0x00, 0x00}, + {0x48, 0x8d, 0x87, 0x70, 0x01, 0x00, 0x00}, + {0x48, 0x8d, 0x87, 0x50, 0x01, 0x00, 0x00}, + {0x48, 0x8b, 0x06, 0x48, 0x8b, 0x80, static_cast(slot), + static_cast(slot >> 8), static_cast(slot >> 16), + static_cast(slot >> 24)}, + {0x41, 0xbf, 0x01, 0x00, 0x00, 0x00, 0xf6, 0x85, 0x5c, 0xff, 0xff, 0xff, + 0x01, 0xb9, 0x00}, + }; + size_t offset = 64; + for (const auto &marker : markers) { + std::memcpy(code.data() + offset, marker.data(), marker.size()); + offset += marker.size() + 8; + } + return code; +} + +compat::BuildProfile MakeBuildProfile(bool with_vr, bool allow_bridges) { + compat::BuildProfile profile{}; + profile.version_name = "2.736.1408"; + profile.version_code = 2998; + profile.elf_build_id = "ade08266c67aee88ec9c1d00902150e1684dad3a"; + profile.status = compat::BuildStatus::kSupported; + profile.default_allowed = true; + profile.allow_host_abi_bridges = allow_bridges; + if (with_vr) { + profile.vr_debug_device_bridge = TestProfile(); + } + return profile; +} + +struct alignas(8) GuestObject { + std::array bytes{}; + template void Put(std::size_t offset, T value) { + std::memcpy(bytes.data() + offset, &value, sizeof(value)); + } +}; +std::uint8_t fixture_flag = 1; +std::uintptr_t fixture_device_vtable[1]; +int fixture_initializations = 0; +bool fixture_partial = false; +void FixtureInitialize(void *object, void *) { + auto *guest = static_cast(object); + ++fixture_initializations; + guest->bytes[internal::kReadyByteOffset] = 1; + guest->Put(internal::kFramebufferSlotOffset, 0x100); + if (fixture_partial) + return; + guest->Put(internal::kFramebufferSlotOffset + 16, 0x200); + guest->Put(internal::kTextureSlotOffset, 0x300); + guest->Put(internal::kTextureSlotOffset + 16, 0x400); +} +void *FixtureState(void *output, void *object) { + std::memcpy(output, + static_cast(object)->bytes.data() + + internal::kStateOffset, + internal::kStateCopySize); + return output; +} + +TEST(VrBridgeLifecycle, ReinitializesReusedAddressesAndMoreThanFourObjects) { + RobloxVrDeviceBridge bridge; + fixture_initializations = 0; + fixture_partial = false; + const auto init = reinterpret_cast(&FixtureInitialize); + const auto flag = reinterpret_cast(&fixture_flag); + const auto vtable = reinterpret_cast(fixture_device_vtable); + const auto base = std::min({init, flag, vtable}) - 8; + auto profile = TestProfile(); + profile.eye_initializer_rva = init - base; + profile.emulator_flag_storage_rva = flag - base; + profile.device_create_framebuffer_vtable_offset = 0; + fixture_device_vtable[0] = init; + std::uintptr_t device = vtable; + internal::VrBridgeTestAccess::Configure( + bridge, base, std::max({init, flag, vtable}) - base + 64, profile, + &FixtureState); + bridge.NoteHostPresent(); + std::array output{}; + std::array guests; + for (int cycle = 0; cycle < 2; ++cycle) { + for (auto &guest : guests) { + guest = {}; // Constructor at a new or previously used address. + guest.Put(0, base + profile.vtable_rva); + guest.Put(internal::kObjectNameOffset, "DebugDevice"); + guest.Put(internal::kObjectTypeOffset, 6); + guest.Put(internal::kWidthOffset, 500); + guest.Put(internal::kHeightOffset, 500); + guest.Put(internal::kGraphicsDeviceOffset, &device); + const int before = fixture_initializations; + bridge.OnStateGetterCall(output.data(), &guest); + EXPECT_EQ(fixture_initializations, before + 1); + EXPECT_EQ(output[internal::kStateReadyOffsetInCopy], 1); + bridge.OnStateGetterCall(output.data(), &guest); + EXPECT_EQ(fixture_initializations, before + 1); + } + } + EXPECT_EQ(bridge.initialized_device_count(), 16); + // A partially constructed device must not expose ready to the renderer or + // repeatedly invoke the initializer on top of its existing allocations. + auto &guest = guests[0]; + for (std::size_t i = internal::kFramebufferSlotOffset; + i < internal::kWidthOffset; ++i) + guest.bytes[i] = 0; + guest.bytes[internal::kReadyByteOffset] = 0; + fixture_partial = true; + bridge.OnStateGetterCall(output.data(), &guest); + EXPECT_EQ(output[internal::kStateReadyOffsetInCopy], 0); + const int before = fixture_initializations; + bridge.OnStateGetterCall(output.data(), &guest); + EXPECT_EQ(fixture_initializations, before); + EXPECT_EQ(output[internal::kStateReadyOffsetInCopy], 0); + fixture_partial = false; +} + +TEST(VrResourceReadiness, + RejectsReadyByteAloneAliasedEyesAndInvalidDimensions) { + GuestObject guest; + guest.Put(internal::kWidthOffset, 500); + guest.Put(internal::kHeightOffset, 500); + EXPECT_EQ(internal::InspectEyeResources(&guest), + internal::EyeResourceState::kEmpty); + guest.bytes[internal::kReadyByteOffset] = 1; + EXPECT_EQ(internal::InspectEyeResources(&guest), + internal::EyeResourceState::kInvalid); + FixtureInitialize(&guest, nullptr); + EXPECT_EQ(internal::InspectEyeResources(&guest), + internal::EyeResourceState::kReady); + guest.Put(internal::kFramebufferSlotOffset + 16, 0x100); + EXPECT_EQ(internal::InspectEyeResources(&guest), + internal::EyeResourceState::kInvalid); + FixtureInitialize(&guest, nullptr); + guest.Put(internal::kWidthOffset, 0); + EXPECT_EQ(internal::InspectEyeResources(&guest), + internal::EyeResourceState::kInvalid); +} + +TEST(VrStateGetterContract, AcceptsExact2998Bytes) { + ASSERT_EQ(kStateGetter.size(), kStateGetterContractSize); + EXPECT_TRUE(internal::HasExpectedVrStateGetterContract(kStateGetter.data(), + kStateGetter.size())); +} + +TEST(VrStateGetterContract, RejectsCorruptionAndShortInput) { + EXPECT_FALSE(internal::HasExpectedVrStateGetterContract(nullptr, 0)); + EXPECT_FALSE(internal::HasExpectedVrStateGetterContract( + kStateGetter.data(), kStateGetter.size() - 1)); + std::vector corrupted = kStateGetter; + // The sret copy size (0x138) must be pinned; corrupt one byte of it. + corrupted[13] ^= 0xff; + EXPECT_FALSE(internal::HasExpectedVrStateGetterContract(corrupted.data(), + corrupted.size())); +} + +TEST(VrEyeGetterContract, AcceptsExact2998Bytes) { + ASSERT_EQ(kEyeGetter.size(), kEyeGetterContractSize); + EXPECT_TRUE(internal::HasExpectedVrEyeGetterContract(kEyeGetter.data(), + kEyeGetter.size())); +} + +TEST(VrEyeGetterContract, RejectsDifferentEyeSlotOffset) { + std::vector shifted = kEyeGetter; + // +0x150 -> +0x160: a different eye slot layout must not pass. + shifted[11] = 0x60; + EXPECT_FALSE( + internal::HasExpectedVrEyeGetterContract(shifted.data(), shifted.size())); +} + +TEST(VrConstructorContract, AcceptsMarkersAndRejectsEachMissingOne) { + const std::vector valid = ValidConstructor(); + EXPECT_TRUE( + internal::HasExpectedVrConstructorContract(valid.data(), valid.size())); + EXPECT_FALSE(internal::HasExpectedVrConstructorContract(valid.data(), + valid.size() - 1)); + // Corrupting the graphics-device capture (+0x198 store) must fail closed. + std::vector no_device_store = valid; + for (size_t index = 0; index + 7 <= no_device_store.size(); ++index) { + if (no_device_store[index] == 0x4c && no_device_store[index + 1] == 0x89 && + no_device_store[index + 2] == 0xbb) { + no_device_store[index + 3] ^= 0xff; + break; + } + } + EXPECT_FALSE(internal::HasExpectedVrConstructorContract( + no_device_store.data(), no_device_store.size())); +} + +TEST(VrInitializerContract, PinsDeviceVtableSlotFromProfile) { + const compat::VrDebugDeviceBridgeProfile profile = TestProfile(); + const std::vector valid = ValidInitializer(profile); + EXPECT_TRUE(internal::HasExpectedVrEyeInitializerContract( + valid.data(), valid.size(), profile)); + + // The 3092 payload uses +0x188; a profile mismatch must fail closed. + compat::VrDebugDeviceBridgeProfile other = profile; + other.device_create_framebuffer_vtable_offset = 0x188; + EXPECT_FALSE(internal::HasExpectedVrEyeInitializerContract( + valid.data(), valid.size(), other)); + + std::vector no_ready_byte = valid; + for (size_t index = 0; index + 8 <= no_ready_byte.size(); ++index) { + if (no_ready_byte[index] == 0xb1 && no_ready_byte[index + 1] == 0x01 && + no_ready_byte[index + 2] == 0x88) { + no_ready_byte[index + 4] ^= 0xff; + break; + } + } + EXPECT_FALSE(internal::HasExpectedVrEyeInitializerContract( + no_ready_byte.data(), no_ready_byte.size(), profile)); +} + +TEST(VrVtableContract, RequiresExactProfileRvas) { + const compat::VrDebugDeviceBridgeProfile profile = TestProfile(); + const std::uintptr_t base = 0x10000000; + std::vector vtable(internal::kEyeInitializerVtableSlot / 8 + + 1); + vtable[internal::kStateGetterVtableSlot / 8] = + base + profile.state_getter_rva; + vtable[internal::kEyeGetterVtableSlot / 8] = base + profile.eye_getter_rva; + vtable[internal::kEyeInitializerVtableSlot / 8] = + base + profile.eye_initializer_rva; + EXPECT_TRUE( + internal::HasExpectedVrDebugDeviceVtable(vtable.data(), base, profile)); + vtable[internal::kEyeInitializerVtableSlot / 8] += 16; + EXPECT_FALSE( + internal::HasExpectedVrDebugDeviceVtable(vtable.data(), base, profile)); +} + +TEST(VrBridgeLifecycle, InstallRejectsMissingMetadataAndWrongPermissions) { + { + RobloxVrDeviceBridge bridge; + const compat::BuildProfile profile = MakeBuildProfile(false, true); + const Status status = bridge.Install(profile); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.code(), StatusCode::kFailedPrecondition); + } + { + RobloxVrDeviceBridge bridge; + const compat::BuildProfile profile = MakeBuildProfile(true, false); + const Status status = bridge.Install(profile); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.code(), StatusCode::kFailedPrecondition); + } +} + +TEST(VrBridgeLifecycle, SingleProcessOwnerAndFailClosedActivation) { + RobloxVrDeviceBridge first; + ASSERT_TRUE(first.Install(MakeBuildProfile(true, true)).ok()); + { + RobloxVrDeviceBridge second; + const Status status = second.Install(MakeBuildProfile(true, true)); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.code(), StatusCode::kFailedPrecondition); + } + // A non-ELF image base must be rejected without touching memory. + const Status bad_image = first.Activate(0); + EXPECT_FALSE(bad_image.ok()); + EXPECT_EQ(bad_image.code(), StatusCode::kInvalidArgument); + EXPECT_FALSE(first.active()); + // While armed, a failed activation must surface to the caller. + const Status notified = NotifyRobloxVrImageLoaded(0); + EXPECT_FALSE(notified.ok()); + first.Shutdown(); +} + +TEST(VrBridgeLifecycle, NotificationIsNoOpWithoutArmedBridge) { + EXPECT_TRUE(NotifyRobloxVrImageLoaded(0).ok()); +} + +TEST(VrProfileParsing, ParsesAndRejectsBridgeSection) { + const std::string directory = std::string(MOCKTAIL_TEST_BINARY_DIRECTORY); + const auto write_manifest = [&directory](const char *name, + const std::string &contents) { + const std::string path = directory + "/" + name; + std::ofstream output(path); + output << contents; + output.close(); + return path; + }; + const std::string base_profile = R"({ + "schema_version": 1, + "profiles": [{ + "version_name": "2.736.1408", + "version_code": 2998, + "elf_build_id": "ade08266c67aee88ec9c1d00902150e1684dad3a", + "status": "supported", + "default_allowed": true, + "allow_legacy_binary_patches": false, + "allow_host_abi_bridges": true, + "allow_host_constructor_replay": true, + "reason": "test" + }] + })"; + { + const std::string path = + write_manifest("vr_profile_absent.json", base_profile); + const auto result = compat::FindBuildProfile( + path, "ade08266c67aee88ec9c1d00902150e1684dad3a"); + ASSERT_TRUE(result); + ASSERT_TRUE(result.profile.has_value()); + EXPECT_FALSE(result.profile->vr_debug_device_bridge.has_value()); + std::remove(path.c_str()); + } + { + std::string manifest = base_profile; + const std::string anchor = R"("reason": "test")"; + manifest.replace(manifest.find(anchor), anchor.size(), + R"("vr_debug_device_bridge": { + "vtable_rva": "0x6c87b38", + "constructor_rva": "0x3810614", + "state_getter_rva": "0x38109f6", + "eye_getter_rva": "0x38109e6", + "eye_initializer_rva": "0x3810a18", + "emulator_flag_storage_rva": "0x7369920", + "device_create_framebuffer_vtable_offset": "0x180" + }, + "reason": "test")"); + const std::string path = write_manifest("vr_profile_valid.json", manifest); + const auto result = compat::FindBuildProfile( + path, "ade08266c67aee88ec9c1d00902150e1684dad3a"); + ASSERT_TRUE(result); + ASSERT_TRUE(result.profile.has_value()); + ASSERT_TRUE(result.profile->vr_debug_device_bridge.has_value()); + const auto &bridge = *result.profile->vr_debug_device_bridge; + EXPECT_EQ(bridge.vtable_rva, 0x6c87b38u); + EXPECT_EQ(bridge.eye_initializer_rva, 0x3810a18u); + EXPECT_EQ(bridge.device_create_framebuffer_vtable_offset, 0x180u); + std::remove(path.c_str()); + } + { + std::string manifest = base_profile; + const std::string anchor = R"("reason": "test")"; + manifest.replace(manifest.find(anchor), anchor.size(), + R"("vr_debug_device_bridge": { + "vtable_rva": "0x6c87b38" + }, + "reason": "test")"); + const std::string path = + write_manifest("vr_profile_incomplete.json", manifest); + const auto result = compat::FindBuildProfile( + path, "ade08266c67aee88ec9c1d00902150e1684dad3a"); + EXPECT_FALSE(result); + std::remove(path.c_str()); + } +} + +} // namespace +} // namespace mocktail::vr + +namespace mocktail::vr { +TEST(VrPoseBridge, ConvertsRuntimePoseAndAsymmetricProjectionWithoutChangingReadiness) { + std::array state; + state.fill(0xa5); + ScriptedPoseSample pose; + pose.valid = true; + pose.position[0] = 0.4f; + pose.position[1] = 1.6f; + pose.orientation[1] = std::sin(0.2f); + pose.orientation[3] = std::cos(0.2f); + for (int eye = 0; eye < 2; ++eye) { + pose.eye_offset[eye][0] = eye ? 0.032f : -0.032f; + const float fov[] = {-0.7f, 0.8f, 0.9f, -0.6f}; + std::copy_n(fov, 4, pose.eye_fov[eye]); + } + ASSERT_TRUE(internal::ApplyXrPose(state.data(), pose)); + EXPECT_EQ(state[0], 1); + EXPECT_EQ(state[0x40], 0); + EXPECT_EQ(state[internal::kStateReadyOffsetInCopy], 0xa5); + float position[3], orientation[4], left[3], right[3], projection[4]; + std::memcpy(position, state.data()+4, 12); + std::memcpy(orientation, state.data()+0x10, 16); + std::memcpy(left, state.data()+0xf8, 12); + std::memcpy(right, state.data()+0x104, 12); + std::memcpy(projection, state.data()+0x110, 16); + EXPECT_FLOAT_EQ(position[0], 0.4f); // no double metre-to-stud conversion + EXPECT_FLOAT_EQ(position[1], 1.6f); + EXPECT_FLOAT_EQ(orientation[1], std::sin(0.2f)); + EXPECT_FLOAT_EQ(left[0], -0.032f); + EXPECT_FLOAT_EQ(right[0], 0.032f); + EXPECT_FLOAT_EQ(projection[0], std::tan(0.9f)); // up/down/left/right + EXPECT_FLOAT_EQ(projection[1], std::tan(0.6f)); + EXPECT_FLOAT_EQ(projection[2], std::tan(0.7f)); + EXPECT_FLOAT_EQ(projection[3], std::tan(0.8f)); +} +TEST(VrPoseBridge, InvalidTrackingAndNonfiniteValuesNeverBecomeValid) { + std::array state{}; + state[0] = state[internal::kStateReadyOffsetInCopy] = 1; + ScriptedPoseSample pose; + EXPECT_FALSE(internal::ApplyXrPose(state.data(), pose)); + EXPECT_EQ(state[0], 0); + EXPECT_EQ(state[internal::kStateReadyOffsetInCopy], 1); + pose.valid = true; + pose.position[2] = std::numeric_limits::quiet_NaN(); + EXPECT_FALSE(internal::ApplyXrPose(state.data(), pose)); + EXPECT_EQ(state[0], 0); + pose.position[2] = 0; + pose.orientation[3] = 0; + EXPECT_FALSE(internal::ApplyXrPose(state.data(), pose)); +} +} // namespace mocktail::vr diff --git a/tests/vr/simulator_launcher_test.py b/tests/vr/simulator_launcher_test.py deleted file mode 100644 index 3a43bc19..00000000 --- a/tests/vr/simulator_launcher_test.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Check process-local runtime selection and preparation independently of VR hardware.""" - -import json -import os -from pathlib import Path -import subprocess -import sys -import tempfile - - -launcher = Path(sys.argv[1]).resolve() -with tempfile.TemporaryDirectory(prefix="mocktail-simulator-launcher-") as directory: - root = Path(directory) - build = root / "build with spaces" - build.mkdir() - source = root / "source" - source.mkdir() - library = source / "runtime.so" - library.write_bytes(b"runtime fixture for preparation only") - manifest = source / "manifest.json" - manifest.write_text(json.dumps({"runtime": {"library_path": "./runtime.so"}})) - command = [sys.executable, str(launcher), "--build-dir", str(build)] - result = subprocess.run([*command, "--prepare", str(manifest)], - capture_output=True, text=True, timeout=10) - assert result.returncode == 0, result.stderr - prepared = build / "vr-simulator/runtime.json" - document = json.loads(prepared.read_text()) - relative_library = Path(document["runtime"]["library_path"]) - assert not relative_library.is_absolute() - assert (prepared.parent / relative_library).read_bytes() == library.read_bytes() - - # Moving the prepared build must not depend on the original /tmp/source path. - relocated = root / "relocated build" - build.rename(relocated) - build = relocated - prepared = build / "vr-simulator/runtime.json" - command = [sys.executable, str(launcher), "--build-dir", str(build)] - probe = build / "mocktail-vr-probe" - probe.write_text("""#!/usr/bin/env python3 -import json, os, sys -names = ('XR_RUNTIME_JSON', 'SIMULATED_ENABLE', 'XRT_COMPOSITOR_NULL', - 'XRT_COMPOSITOR_FORCE_XCB', 'XRT_COMPOSITOR_FORCE_WAYLAND', - 'XDG_CONFIG_HOME', 'XDG_RUNTIME_DIR') -print(json.dumps({'args': sys.argv[1:], 'env': {key: os.environ.get(key) for key in names}})) -sys.exit(7) -""") - probe.chmod(0o755) - env = os.environ.copy() - env.update(DISPLAY=":123", WAYLAND_DISPLAY="wayland-test", - XR_RUNTIME_JSON=str(root / "real-headset.json"), - XRT_COMPOSITOR_NULL="1", XRT_COMPOSITOR_FORCE_WAYLAND="1", - XDG_CONFIG_HOME=str(root / "user-config"), XDG_RUNTIME_DIR=str(root / "desktop-runtime")) - original_selection = env["XR_RUNTIME_JSON"] - original_config = Path(env["XDG_CONFIG_HOME"]) - original_config.mkdir() - marker = original_config / "keep" - marker.write_text("preserve user settings") - - for graphical_backend in ("xcb", "wayland"): - if graphical_backend == "wayland": - env.pop("DISPLAY") - result = subprocess.run([*command, "--frames", "4"], env=env, - capture_output=True, text=True, timeout=10) - assert result.returncode == 7, (result.stdout, result.stderr) - child = json.loads(result.stdout.splitlines()[-1]) - assert child["args"] == ["--scene", "--frames", "4"] - selected = child["env"] - assert selected["XR_RUNTIME_JSON"] == str(prepared) - assert selected["SIMULATED_ENABLE"] == "1" and selected["XRT_COMPOSITOR_NULL"] == "0" - assert selected["XRT_COMPOSITOR_FORCE_XCB"] == ("1" if graphical_backend == "xcb" else "0") - assert selected["XRT_COMPOSITOR_FORCE_WAYLAND"] == ("0" if graphical_backend == "xcb" else "1") - assert selected["XDG_CONFIG_HOME"] != str(original_config) - assert selected["XDG_RUNTIME_DIR"] == env["XDG_RUNTIME_DIR"] - assert env["XR_RUNTIME_JSON"] == original_selection - assert marker.read_text() == "preserve user settings" - assert not (original_config / "openxr").exists() - print(graphical_backend + " process-local selection: passed") - - env.pop("WAYLAND_DISPLAY") - result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=10) - assert result.returncode == 2 and "graphical desktop" in result.stderr, result - result = subprocess.run([*command, "--runtime", str(root / "missing.json")], env=env, - capture_output=True, text=True, timeout=10) - assert result.returncode == 2 and "No simulated Monado runtime" in result.stderr, result - for count in ("0", "-1", "4294967296"): - result = subprocess.run([*command, "--frames", count], env=env, - capture_output=True, text=True, timeout=10) - assert result.returncode == 2 and "positive 32-bit integer" in result.stderr, result - print("missing runtime/display and frame limit diagnostics: passed") diff --git a/tests/vr/vr_evidence_layer.cc b/tests/vr/vr_evidence_layer.cc new file mode 100644 index 00000000..b229e747 --- /dev/null +++ b/tests/vr/vr_evidence_layer.cc @@ -0,0 +1,1005 @@ +// Test-only implicit Vulkan layer that produces GPU-level evidence for the +// experimental native stereo bring-up: +// - records images only inside the bridge's guest eye initializer; +// - identifies left/right from the guest eye getter and the subsequent +// render pass's actual attachments, checking the initializer's owner; +// - counts vkCmdBeginRenderPass invocations per eye framebuffer (proof that +// the Roblox renderer issues separate GPU passes into both eyes); +// - at a configurable present index, reads both eye images back on the +// presenting queue and writes eye0.ppm / eye1.ppm plus summary.txt into +// MOCKTAIL_VR_EVIDENCE_DIR, including a pixel-difference verdict that +// distinguishes real per-eye cameras from a duplicated single image. +// +// The layer is inert unless MOCKTAIL_VR_EVIDENCE_DIR is set. It never aborts +// the application: every evidence operation is best-effort and logged with +// the [vr-evidence-layer] prefix. +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__GNUC__) +#define MOCKTAIL_VR_LAYER_EXPORT __attribute__((visibility("default"))) +#else +#define MOCKTAIL_VR_LAYER_EXPORT +#endif + +namespace { + +void Log(const char *format, ...) { + va_list arguments; + va_start(arguments, format); + std::vfprintf(stderr, format, arguments); + va_end(arguments); + std::fflush(stderr); +} + +std::string EvidenceDirectory() { + const char *value = std::getenv("MOCKTAIL_VR_EVIDENCE_DIR"); + return value != nullptr ? std::string(value) : std::string(); +} + +uint32_t EnvUnsigned(const char *name, uint32_t fallback) { + const char *value = std::getenv(name); + if (value == nullptr || value[0] == '\0') { + return fallback; + } + const long parsed = std::strtol(value, nullptr, 10); + return parsed > 0 ? static_cast(parsed) : fallback; +} + +const void *FindInChain(const void *chain, VkStructureType type) { + // The loader inserts several nodes sharing the same sType (link info, + // data callbacks, feature flags); only the VK_LAYER_LINK_INFO variant + // carries the pLayerInfo chain a layer must consume. + for (const auto *header = static_cast(chain); + header != nullptr; header = header->pNext) { + if (header->sType != type) { + continue; + } + const auto *link = + reinterpret_cast(header); + if (link->function == VK_LAYER_LINK_INFO) { + return header; + } + } + return nullptr; +} + +struct DeviceDispatch { + VkDevice device = VK_NULL_HANDLE; + VkPhysicalDevice physical_device = VK_NULL_HANDLE; + PFN_vkGetDeviceProcAddr get_device_proc_addr = nullptr; + PFN_vkDestroyDevice destroy_device = nullptr; + PFN_vkGetDeviceQueue get_device_queue = nullptr; + PFN_vkCreateImage create_image = nullptr; + PFN_vkDestroyImage destroy_image = nullptr; + PFN_vkDestroyFramebuffer destroy_framebuffer = nullptr; + PFN_vkCreateImageView create_image_view = nullptr; + PFN_vkCreateFramebuffer create_framebuffer = nullptr; + PFN_vkCreateRenderPass create_render_pass = nullptr; + PFN_vkCmdBeginRenderPass cmd_begin_render_pass = nullptr; + PFN_vkQueuePresentKHR queue_present = nullptr; + PFN_vkQueueSubmit queue_submit = nullptr; + PFN_vkCreateCommandPool create_command_pool = nullptr; + PFN_vkAllocateCommandBuffers allocate_command_buffers = nullptr; + PFN_vkBeginCommandBuffer begin_command_buffer = nullptr; + PFN_vkEndCommandBuffer end_command_buffer = nullptr; + PFN_vkCmdPipelineBarrier cmd_pipeline_barrier = nullptr; + PFN_vkCmdCopyImageToBuffer cmd_copy_image_to_buffer = nullptr; + PFN_vkCreateFence create_fence = nullptr; + PFN_vkWaitForFences wait_for_fences = nullptr; + PFN_vkDestroyFence destroy_fence = nullptr; + PFN_vkCreateBuffer create_buffer = nullptr; + PFN_vkDestroyBuffer destroy_buffer = nullptr; + PFN_vkGetBufferMemoryRequirements get_buffer_memory_requirements = nullptr; + PFN_vkAllocateMemory allocate_memory = nullptr; + PFN_vkFreeMemory free_memory = nullptr; + PFN_vkBindBufferMemory bind_buffer_memory = nullptr; + PFN_vkMapMemory map_memory = nullptr; + PFN_vkUnmapMemory unmap_memory = nullptr; + PFN_vkDestroyCommandPool destroy_command_pool = nullptr; + PFN_vkGetPhysicalDeviceMemoryProperties memory_properties = nullptr; +}; + +struct ImageRecord { + VkImage image = VK_NULL_HANDLE; + DeviceDispatch *dispatch = nullptr; + uint32_t width = 0; + uint32_t height = 0; + VkFormat format = VK_FORMAT_UNDEFINED; + VkImageUsageFlags usage = 0; + void *owner = nullptr; +}; + +struct EyeTarget { + bool valid = false; + ImageRecord image{}; + VkFramebuffer framebuffer = VK_NULL_HANDLE; + VkImageLayout last_final_layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + std::uint64_t render_pass_count = 0; +}; + +bool g_active = false; +uint32_t g_capture_present = 60; +uint32_t g_capture_render_passes = 10; +std::mutex g_mutex; +std::unordered_map g_dispatches; +std::unordered_map g_queue_families; +std::unordered_map g_render_pass_final_layouts; +std::unordered_map g_image_views; +std::unordered_map g_images; +std::unordered_map> g_framebuffer_images; +using InitializingObjectFn = void *(*)(); +using CurrentEyeFn = bool (*)(void **, int *, void **); +using ConsumeEyeFn = void (*)(); +InitializingObjectFn g_initializing_object = nullptr; +CurrentEyeFn g_current_eye = nullptr; +ConsumeEyeFn g_consume_eye = nullptr; +EyeTarget g_eyes[2]; +std::uint64_t g_present_count = 0; +bool g_capture_attempted = false; +PFN_vkGetInstanceProcAddr g_instance_gipa = nullptr; +VkInstance g_instance = VK_NULL_HANDLE; + +void *DispatchKey(void *handle) { + return handle != nullptr ? *reinterpret_cast(handle) : nullptr; +} + +DeviceDispatch *DispatchFor(void *handle) { + void *key = DispatchKey(handle); + if (key == nullptr) { + return nullptr; + } + std::lock_guard lock(g_mutex); + const auto entry = g_dispatches.find(key); + return entry != g_dispatches.end() ? &entry->second : nullptr; +} + +bool IsColorFormat(VkFormat format) { + switch (format) { + case VK_FORMAT_D16_UNORM: + case VK_FORMAT_D16_UNORM_S8_UINT: + case VK_FORMAT_D24_UNORM_S8_UINT: + case VK_FORMAT_D32_SFLOAT: + case VK_FORMAT_D32_SFLOAT_S8_UINT: + case VK_FORMAT_X8_D24_UNORM_PACK32: + return false; + default: + return true; + } +} + +uint32_t BytesPerPixel(VkFormat format) { + switch (format) { + case VK_FORMAT_R8G8B8A8_UNORM: + case VK_FORMAT_R8G8B8A8_SRGB: + case VK_FORMAT_B8G8R8A8_UNORM: + case VK_FORMAT_B8G8R8A8_SRGB: + case VK_FORMAT_R8G8B8A8_UINT: + case VK_FORMAT_R32_SFLOAT: + return 4; + default: + return 0; + } +} + +bool IsBgra(VkFormat format) { + return format == VK_FORMAT_B8G8R8A8_UNORM || + format == VK_FORMAT_B8G8R8A8_SRGB; +} + +bool WritePpm(const std::string &path, const uint8_t *pixels, uint32_t width, + uint32_t height, bool bgra) { + std::FILE *file = std::fopen(path.c_str(), "wb"); + if (file == nullptr) { + return false; + } + std::fprintf(file, "P6\n%u %u\n255\n", width, height); + for (uint32_t row = 0; row < height; ++row) { + for (uint32_t column = 0; column < width; ++column) { + const uint8_t *source = + pixels + (static_cast(row) * width + column) * 4; + const uint8_t red = bgra ? source[2] : source[0]; + const uint8_t green = source[1]; + const uint8_t blue = bgra ? source[0] : source[2]; + const uint8_t rgb[3] = {red, green, blue}; + std::fwrite(rgb, 1, sizeof(rgb), file); + } + } + std::fclose(file); + return true; +} + +int FindMemoryType(const DeviceDispatch &dispatch, uint32_t requirements, + VkMemoryPropertyFlags properties) { + if (dispatch.memory_properties == nullptr) { + return -1; + } + VkPhysicalDeviceMemoryProperties memory{}; + dispatch.memory_properties(dispatch.physical_device, &memory); + for (uint32_t index = 0; index < memory.memoryTypeCount; ++index) { + if ((requirements & (1u << index)) != 0 && + (memory.memoryTypes[index].propertyFlags & properties) == properties) { + return static_cast(index); + } + } + return -1; +} + +uint32_t QueueFamilyFor(VkQueue queue) { + std::lock_guard lock(g_mutex); + const auto entry = g_queue_families.find(DispatchKey(queue)); + return entry != g_queue_families.end() ? entry->second : 0; +} + +void CaptureEyes(DeviceDispatch &dispatch, VkQueue queue) { + // Runs on the presenting (render) thread after the frame that filled both + // eyes was submitted; g_mutex is NOT held here except through helpers. + EyeTarget eyes[2]; + { + std::lock_guard lock(g_mutex); + eyes[0] = g_eyes[0]; + eyes[1] = g_eyes[1]; + } + if (!eyes[0].valid || !eyes[1].valid || eyes[0].image.dispatch != &dispatch || + eyes[1].image.dispatch != &dispatch || + eyes[0].image.owner != eyes[1].image.owner || + eyes[0].image.width != eyes[1].image.width || + eyes[0].image.height != eyes[1].image.height) { + Log(" [vr-evidence-layer] capture skipped: eye targets unknown\n"); + return; + } + if (BytesPerPixel(eyes[0].image.format) != 4 || + BytesPerPixel(eyes[1].image.format) != 4) { + Log(" [vr-evidence-layer] capture skipped: unsupported eye formats " + "%d/%d\n", + static_cast(eyes[0].image.format), + static_cast(eyes[1].image.format)); + return; + } + const uint32_t width = eyes[0].image.width; + const uint32_t height = eyes[0].image.height; + const VkDeviceSize buffer_size = + static_cast(width) * height * 4; + + VkCommandPool pool = VK_NULL_HANDLE; + VkCommandPoolCreateInfo pool_info{}; + pool_info.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO; + pool_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + pool_info.queueFamilyIndex = QueueFamilyFor(queue); + if (dispatch.create_command_pool(dispatch.device, &pool_info, nullptr, + &pool) != VK_SUCCESS) { + Log(" [vr-evidence-layer] capture: command pool creation failed\n"); + return; + } + + VkCommandBuffer command = VK_NULL_HANDLE; + VkCommandBufferAllocateInfo allocate_info{}; + allocate_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + allocate_info.commandPool = pool; + allocate_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + allocate_info.commandBufferCount = 1; + if (dispatch.allocate_command_buffers(dispatch.device, &allocate_info, + &command) != VK_SUCCESS) { + dispatch.destroy_command_pool(dispatch.device, pool, nullptr); + Log(" [vr-evidence-layer] capture: command buffer allocation failed\n"); + return; + } + + VkBuffer buffers[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE}; + VkDeviceMemory memories[2] = {VK_NULL_HANDLE, VK_NULL_HANDLE}; + void *mapped[2] = {nullptr, nullptr}; + VkFence fence = VK_NULL_HANDLE; + VkFenceCreateInfo fence_info{}; + fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + bool prepared = dispatch.create_fence(dispatch.device, &fence_info, nullptr, + &fence) == VK_SUCCESS; + for (int index = 0; prepared && index < 2; ++index) { + VkBufferCreateInfo buffer_info{}; + buffer_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + buffer_info.size = buffer_size; + buffer_info.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; + prepared = dispatch.create_buffer(dispatch.device, &buffer_info, nullptr, + &buffers[index]) == VK_SUCCESS; + if (!prepared) { + break; + } + VkMemoryRequirements requirements{}; + dispatch.get_buffer_memory_requirements(dispatch.device, buffers[index], + &requirements); + const int memory_type = + FindMemoryType(dispatch, requirements.memoryTypeBits, + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + if (memory_type < 0) { + prepared = false; + break; + } + VkMemoryAllocateInfo memory_info{}; + memory_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + memory_info.allocationSize = requirements.size; + memory_info.memoryTypeIndex = static_cast(memory_type); + prepared = dispatch.allocate_memory(dispatch.device, &memory_info, nullptr, + &memories[index]) == VK_SUCCESS; + if (!prepared) { + break; + } + prepared = dispatch.bind_buffer_memory(dispatch.device, buffers[index], + memories[index], 0) == VK_SUCCESS; + } + + if (prepared) { + VkCommandBufferBeginInfo begin_info{}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + prepared = + dispatch.begin_command_buffer(command, &begin_info) == VK_SUCCESS; + } + if (prepared) { + for (int index = 0; index < 2; ++index) { + VkImageMemoryBarrier to_source{}; + to_source.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + to_source.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + to_source.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + to_source.oldLayout = eyes[index].last_final_layout; + to_source.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + to_source.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + to_source.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + to_source.image = eyes[index].image.image; + to_source.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + dispatch.cmd_pipeline_barrier( + command, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, + &to_source); + VkBufferImageCopy region{}; + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.imageExtent = {width, height, 1}; + dispatch.cmd_copy_image_to_buffer(command, eyes[index].image.image, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + buffers[index], 1, ®ion); + VkImageMemoryBarrier restore = to_source; + restore.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + restore.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + restore.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + restore.newLayout = eyes[index].last_final_layout; + dispatch.cmd_pipeline_barrier( + command, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 0, 0, nullptr, 0, + nullptr, 1, &restore); + } + prepared = dispatch.end_command_buffer(command) == VK_SUCCESS; + } + if (prepared) { + VkSubmitInfo submit{}; + submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit.commandBufferCount = 1; + submit.pCommandBuffers = &command; + prepared = dispatch.queue_submit(queue, 1, &submit, fence) == VK_SUCCESS && + dispatch.wait_for_fences(dispatch.device, 1, &fence, VK_TRUE, + 2000000000ULL) == VK_SUCCESS; + } + if (prepared) { + for (int index = 0; index < 2; ++index) { + prepared = + dispatch.map_memory(dispatch.device, memories[index], 0, buffer_size, + 0, &mapped[index]) == VK_SUCCESS; + if (!prepared) { + break; + } + } + } + + const std::string directory = EvidenceDirectory(); + bool wrote_images = false; + if (prepared && mapped[0] != nullptr && mapped[1] != nullptr) { + const auto *pixels0 = static_cast(mapped[0]); + const auto *pixels1 = static_cast(mapped[1]); + wrote_images = WritePpm(directory + "/eye0.ppm", pixels0, width, height, + IsBgra(eyes[0].image.format)) && + WritePpm(directory + "/eye1.ppm", pixels1, width, height, + IsBgra(eyes[1].image.format)); + std::size_t differing = 0; + std::uint64_t difference_sum = 0; + uint32_t maximum_difference = 0; + const std::size_t total = static_cast(width) * height; + for (std::size_t pixel = 0; pixel < total; ++pixel) { + const uint8_t *a = pixels0 + pixel * 4; + const uint8_t *b = pixels1 + pixel * 4; + uint32_t largest = 0; + for (int channel = 0; channel < 3; ++channel) { + const uint32_t difference = a[channel] > b[channel] + ? a[channel] - b[channel] + : b[channel] - a[channel]; + largest = difference > largest ? difference : largest; + } + if (largest > 0) { + ++differing; + difference_sum += largest; + maximum_difference = + largest > maximum_difference ? largest : maximum_difference; + } + } + std::FILE *summary = std::fopen((directory + "/summary.txt").c_str(), "w"); + if (summary != nullptr) { + std::fprintf(summary, + "provenance=guest_initializer_and_eye_getter owner=%p\n", + eyes[0].image.owner); + std::fprintf(summary, "capture_present=%llu\n", + static_cast(g_present_count)); + for (int index = 0; index < 2; ++index) { + const EyeTarget &eye = eyes[index]; + std::fprintf( + summary, + "eye%d_image=%p eye%d_format=%d eye%d_width=%u eye%d_height=%u " + "eye%d_usage=0x%x eye%d_framebuffer=%p " + "eye%d_render_pass_count=%llu eye%d_final_layout=%d\n", + index, static_cast(eye.image.image), index, + static_cast(eye.image.format), index, eye.image.width, index, + eye.image.height, index, eye.image.usage, index, + static_cast(eye.framebuffer), index, + static_cast(eye.render_pass_count), index, + static_cast(eye.last_final_layout)); + } + std::fprintf(summary, + "pixels_total=%zu pixels_differing=%zu max_abs_diff=%u " + "mean_abs_diff=%.4f\n", + total, differing, maximum_difference, + differing > 0 ? static_cast(difference_sum) / + static_cast(differing) + : 0.0); + std::fprintf(summary, "EYES_DIFFER=%s\n", differing > 0 ? "yes" : "no"); + std::fclose(summary); + } + Log(" [vr-evidence-layer] capture done: differing=%zu/%u max_diff=%u " + "eye0_rp=%llu eye1_rp=%llu wrote_ppm=%d\n", + differing, static_cast(total), maximum_difference, + static_cast(eyes[0].render_pass_count), + static_cast(eyes[1].render_pass_count), + wrote_images ? 1 : 0); + } else { + Log(" [vr-evidence-layer] capture FAILED (prepared=%d)\n", + prepared ? 1 : 0); + } + + for (int index = 0; index < 2; ++index) { + if (mapped[index] != nullptr) { + dispatch.unmap_memory(dispatch.device, memories[index]); + } + if (buffers[index] != VK_NULL_HANDLE) { + dispatch.destroy_buffer(dispatch.device, buffers[index], nullptr); + } + if (memories[index] != VK_NULL_HANDLE) { + dispatch.free_memory(dispatch.device, memories[index], nullptr); + } + } + if (fence != VK_NULL_HANDLE) { + dispatch.destroy_fence(dispatch.device, fence, nullptr); + } + dispatch.destroy_command_pool(dispatch.device, pool, nullptr); +} + +VKAPI_ATTR VkResult VKAPI_CALL +LayerQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR *info) { + DeviceDispatch *dispatch = DispatchFor(queue); + if (dispatch == nullptr || dispatch->queue_present == nullptr) { + return VK_ERROR_INITIALIZATION_FAILED; + } + if (!g_active) { + return dispatch->queue_present(queue, info); + } + std::uint64_t presents = 0; + bool should_capture = false; + { + std::lock_guard lock(g_mutex); + presents = ++g_present_count; + // Only capture once BOTH eyes have actually been rendered into at least + // g_capture_render_passes times; otherwise the readback could show + // uninitialized contents and a bogus pixel difference. + should_capture = !g_capture_attempted && presents >= g_capture_present && + g_eyes[0].valid && g_eyes[1].valid && + g_eyes[0].image.dispatch == dispatch && + g_eyes[1].image.dispatch == dispatch && + g_eyes[0].framebuffer != VK_NULL_HANDLE && + g_eyes[1].framebuffer != VK_NULL_HANDLE && + g_eyes[0].render_pass_count >= g_capture_render_passes && + g_eyes[1].render_pass_count >= g_capture_render_passes; + if (should_capture) { + g_capture_attempted = true; + } + } + if (should_capture) { + CaptureEyes(*dispatch, queue); + } + if (presents == 1 || presents % 120 == 0) { + std::lock_guard lock(g_mutex); + Log(" [vr-evidence-layer] presents=%llu eye0_rp=%llu eye1_rp=%llu\n", + static_cast(presents), + static_cast(g_eyes[0].render_pass_count), + static_cast(g_eyes[1].render_pass_count)); + } + return dispatch->queue_present(queue, info); +} + +VKAPI_ATTR void VKAPI_CALL LayerCmdBeginRenderPass( + VkCommandBuffer command, const VkRenderPassBeginInfo *info, + VkSubpassContents contents) { + DeviceDispatch *dispatch = DispatchFor(command); + if (dispatch == nullptr || dispatch->cmd_begin_render_pass == nullptr) { + return; + } + if (g_active && info != nullptr) { + std::lock_guard lock(g_mutex); + void *owner = nullptr; + void *guest_framebuffer = nullptr; + int index = -1; + const bool requested = g_current_eye != nullptr && + g_current_eye(&owner, &index, &guest_framebuffer) && + index >= 0 && index < 2; + const auto attachments = g_framebuffer_images.find(info->framebuffer); + if (requested && attachments != g_framebuffer_images.end()) { + const ImageRecord *target = nullptr; + bool ambiguous = false; + for (VkImage image : attachments->second) { + const auto found = g_images.find(image); + if (found != g_images.end() && found->second.owner == owner && + found->second.dispatch == dispatch && + IsColorFormat(found->second.format) && + (found->second.usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) != 0) { + if (target != nullptr) + ambiguous = true; + target = &found->second; + } + } + if (target != nullptr && !ambiguous) { + if ((g_eyes[0].valid && g_eyes[0].image.owner != owner) || + (g_eyes[1].valid && g_eyes[1].image.owner != owner)) { + g_eyes[0] = {}; + g_eyes[1] = {}; + g_capture_attempted = false; + } + EyeTarget &eye = g_eyes[index]; + if (!eye.valid || eye.image.image != target->image || + eye.framebuffer != info->framebuffer) { + eye = {}; + eye.valid = true; + eye.image = *target; + eye.framebuffer = info->framebuffer; + Log(" [vr-evidence-layer] eye framebuffer bound: eye=%d owner=%p " + "guest_fb=%p fb=%p image=%p " + "provenance=guest_initializer_and_eye_getter\n", + index, owner, guest_framebuffer, + static_cast(info->framebuffer), + static_cast(target->image)); + } + ++eye.render_pass_count; + const auto layout = g_render_pass_final_layouts.find(info->renderPass); + if (layout != g_render_pass_final_layouts.end()) + eye.last_final_layout = layout->second; + if (g_consume_eye != nullptr) + g_consume_eye(); + } + } + } + dispatch->cmd_begin_render_pass(command, info, contents); +} + +VKAPI_ATTR VkResult VKAPI_CALL +LayerCreateImage(VkDevice device, const VkImageCreateInfo *info, + const VkAllocationCallbacks *allocator, VkImage *image) { + DeviceDispatch *dispatch = DispatchFor(device); + if (dispatch == nullptr || dispatch->create_image == nullptr) { + return VK_ERROR_INITIALIZATION_FAILED; + } + const VkResult result = + dispatch->create_image(device, info, allocator, image); + void *owner = + g_initializing_object != nullptr ? g_initializing_object() : nullptr; + if (g_active && owner != nullptr && result == VK_SUCCESS && info != nullptr && + image != nullptr) { + std::lock_guard lock(g_mutex); + ImageRecord record{}; + record.image = *image; + record.dispatch = dispatch; + record.width = info->extent.width; + record.height = info->extent.height; + record.format = info->format; + record.usage = info->usage; + record.owner = owner; + g_images[*image] = record; + } + return result; +} + +VKAPI_ATTR VkResult VKAPI_CALL LayerCreateImageView( + VkDevice device, const VkImageViewCreateInfo *info, + const VkAllocationCallbacks *allocator, VkImageView *view) { + DeviceDispatch *dispatch = DispatchFor(device); + if (dispatch == nullptr || dispatch->create_image_view == nullptr) { + return VK_ERROR_INITIALIZATION_FAILED; + } + const VkResult result = + dispatch->create_image_view(device, info, allocator, view); + if (g_active && result == VK_SUCCESS && info != nullptr && view != nullptr) { + std::lock_guard lock(g_mutex); + g_image_views[*view] = info->image; + } + return result; +} + +VKAPI_ATTR VkResult VKAPI_CALL LayerCreateFramebuffer( + VkDevice device, const VkFramebufferCreateInfo *info, + const VkAllocationCallbacks *allocator, VkFramebuffer *framebuffer) { + DeviceDispatch *dispatch = DispatchFor(device); + if (dispatch == nullptr || dispatch->create_framebuffer == nullptr) { + return VK_ERROR_INITIALIZATION_FAILED; + } + const VkResult result = + dispatch->create_framebuffer(device, info, allocator, framebuffer); + if (g_active && result == VK_SUCCESS && info != nullptr && + framebuffer != nullptr) { + std::lock_guard lock(g_mutex); + auto &images = g_framebuffer_images[*framebuffer]; + images.clear(); + for (uint32_t index = 0; index < info->attachmentCount; ++index) { + const auto source = g_image_views.find(info->pAttachments[index]); + if (source != g_image_views.end()) + images.push_back(source->second); + } + } + return result; +} + +VKAPI_ATTR void VKAPI_CALL LayerDestroyImage( + VkDevice device, VkImage image, const VkAllocationCallbacks *allocator) { + auto *dispatch = DispatchFor(device); + if (dispatch == nullptr) + return; + { + std::lock_guard lock(g_mutex); + g_images.erase(image); + for (auto it = g_image_views.begin(); it != g_image_views.end();) { + if (it->second == image) + it = g_image_views.erase(it); + else + ++it; + } + if (g_eyes[0].image.image == image || g_eyes[1].image.image == image) { + g_eyes[0] = {}; + g_eyes[1] = {}; + g_capture_attempted = false; + } + } + dispatch->destroy_image(device, image, allocator); +} + +VKAPI_ATTR void VKAPI_CALL +LayerDestroyFramebuffer(VkDevice device, VkFramebuffer framebuffer, + const VkAllocationCallbacks *allocator) { + auto *dispatch = DispatchFor(device); + if (dispatch == nullptr) + return; + { + std::lock_guard lock(g_mutex); + g_framebuffer_images.erase(framebuffer); + if (g_eyes[0].framebuffer == framebuffer || + g_eyes[1].framebuffer == framebuffer) { + g_eyes[0] = {}; + g_eyes[1] = {}; + g_capture_attempted = false; + } + } + dispatch->destroy_framebuffer(device, framebuffer, allocator); +} + +VKAPI_ATTR VkResult VKAPI_CALL LayerCreateRenderPass( + VkDevice device, const VkRenderPassCreateInfo *info, + const VkAllocationCallbacks *allocator, VkRenderPass *render_pass) { + DeviceDispatch *dispatch = DispatchFor(device); + if (dispatch == nullptr || dispatch->create_render_pass == nullptr) { + return VK_ERROR_INITIALIZATION_FAILED; + } + const VkResult result = + dispatch->create_render_pass(device, info, allocator, render_pass); + if (g_active && result == VK_SUCCESS && info != nullptr && + render_pass != nullptr && info->attachmentCount > 0) { + std::lock_guard lock(g_mutex); + g_render_pass_final_layouts[*render_pass] = + info->pAttachments[0].finalLayout; + } + return result; +} + +VKAPI_ATTR void VKAPI_CALL LayerGetDeviceQueue(VkDevice device, uint32_t family, + uint32_t index, VkQueue *queue) { + DeviceDispatch *dispatch = DispatchFor(device); + if (dispatch == nullptr || dispatch->get_device_queue == nullptr) { + return; + } + dispatch->get_device_queue(device, family, index, queue); + if (g_active && queue != nullptr && *queue != VK_NULL_HANDLE) { + std::lock_guard lock(g_mutex); + g_queue_families[DispatchKey(*queue)] = family; + } +} + +VKAPI_ATTR void VKAPI_CALL +LayerDestroyDevice(VkDevice device, const VkAllocationCallbacks *allocator) { + DeviceDispatch *dispatch = DispatchFor(device); + if (dispatch == nullptr) { + return; + } + PFN_vkDestroyDevice destroy = dispatch->destroy_device; + { + std::lock_guard lock(g_mutex); + g_dispatches.erase(DispatchKey(device)); + } + if (destroy != nullptr) { + destroy(device, allocator); + } +} + +VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL +LayerGetDeviceProcAddr(VkDevice device, const char *name) { + if (name == nullptr) { + return nullptr; + } + if (g_active) { + if (std::strcmp(name, "vkGetDeviceProcAddr") == 0) { + return reinterpret_cast(LayerGetDeviceProcAddr); + } + if (std::strcmp(name, "vkDestroyImage") == 0) + return reinterpret_cast(LayerDestroyImage); + if (std::strcmp(name, "vkDestroyFramebuffer") == 0) + return reinterpret_cast(LayerDestroyFramebuffer); + if (std::strcmp(name, "vkDestroyDevice") == 0) { + return reinterpret_cast(LayerDestroyDevice); + } + if (std::strcmp(name, "vkGetDeviceQueue") == 0) { + return reinterpret_cast(LayerGetDeviceQueue); + } + if (std::strcmp(name, "vkCreateImage") == 0) { + return reinterpret_cast(LayerCreateImage); + } + if (std::strcmp(name, "vkCreateImageView") == 0) { + return reinterpret_cast(LayerCreateImageView); + } + if (std::strcmp(name, "vkCreateFramebuffer") == 0) { + return reinterpret_cast(LayerCreateFramebuffer); + } + if (std::strcmp(name, "vkCreateRenderPass") == 0) { + return reinterpret_cast(LayerCreateRenderPass); + } + if (std::strcmp(name, "vkCmdBeginRenderPass") == 0) { + return reinterpret_cast(LayerCmdBeginRenderPass); + } + if (std::strcmp(name, "vkQueuePresentKHR") == 0) { + return reinterpret_cast(LayerQueuePresentKHR); + } + } + std::lock_guard lock(g_mutex); + const auto entry = g_dispatches.find(DispatchKey(device)); + if (entry == g_dispatches.end() || + entry->second.get_device_proc_addr == nullptr) { + return nullptr; + } + return entry->second.get_device_proc_addr(device, name); +} + +VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL +LayerGetInstanceProcAddr(VkInstance instance, const char *name); + +VKAPI_ATTR VkResult VKAPI_CALL LayerCreateDevice( + VkPhysicalDevice physical_device, const VkDeviceCreateInfo *info, + const VkAllocationCallbacks *allocator, VkDevice *device) { + auto *link = const_cast( + static_cast( + FindInChain(info != nullptr ? info->pNext : nullptr, + VK_STRUCTURE_TYPE_LOADER_DEVICE_CREATE_INFO))); + if (link == nullptr || link->u.pLayerInfo == nullptr) { + return VK_ERROR_INITIALIZATION_FAILED; + } + const VkLayerDeviceLink *device_link = link->u.pLayerInfo; + PFN_vkGetInstanceProcAddr next_gipa = device_link->pfnNextGetInstanceProcAddr; + PFN_vkGetDeviceProcAddr next_gdpa = device_link->pfnNextGetDeviceProcAddr; + PFN_vkCreateDevice next_create = reinterpret_cast( + next_gipa(VK_NULL_HANDLE, "vkCreateDevice")); + if (next_create == nullptr) { + return VK_ERROR_INITIALIZATION_FAILED; + } + link->u.pLayerInfo = link->u.pLayerInfo->pNext; + const VkResult result = next_create(physical_device, info, allocator, device); + if (result != VK_SUCCESS || device == nullptr) { + return result; + } + DeviceDispatch dispatch{}; + dispatch.device = *device; + dispatch.physical_device = physical_device; + dispatch.get_device_proc_addr = next_gdpa; +#define MOCKTAIL_LAYER_DEVICE_FN(field, name) \ + dispatch.field = reinterpret_cast(next_gdpa(*device, #name)); + MOCKTAIL_LAYER_DEVICE_FN(destroy_device, vkDestroyDevice) + MOCKTAIL_LAYER_DEVICE_FN(get_device_queue, vkGetDeviceQueue) + MOCKTAIL_LAYER_DEVICE_FN(create_image, vkCreateImage) + MOCKTAIL_LAYER_DEVICE_FN(destroy_image, vkDestroyImage) + MOCKTAIL_LAYER_DEVICE_FN(destroy_framebuffer, vkDestroyFramebuffer) + MOCKTAIL_LAYER_DEVICE_FN(create_image_view, vkCreateImageView) + MOCKTAIL_LAYER_DEVICE_FN(create_framebuffer, vkCreateFramebuffer) + MOCKTAIL_LAYER_DEVICE_FN(create_render_pass, vkCreateRenderPass) + MOCKTAIL_LAYER_DEVICE_FN(cmd_begin_render_pass, vkCmdBeginRenderPass) + MOCKTAIL_LAYER_DEVICE_FN(queue_present, vkQueuePresentKHR) + MOCKTAIL_LAYER_DEVICE_FN(queue_submit, vkQueueSubmit) + MOCKTAIL_LAYER_DEVICE_FN(create_command_pool, vkCreateCommandPool) + MOCKTAIL_LAYER_DEVICE_FN(allocate_command_buffers, vkAllocateCommandBuffers) + MOCKTAIL_LAYER_DEVICE_FN(begin_command_buffer, vkBeginCommandBuffer) + MOCKTAIL_LAYER_DEVICE_FN(end_command_buffer, vkEndCommandBuffer) + MOCKTAIL_LAYER_DEVICE_FN(cmd_pipeline_barrier, vkCmdPipelineBarrier) + MOCKTAIL_LAYER_DEVICE_FN(cmd_copy_image_to_buffer, vkCmdCopyImageToBuffer) + MOCKTAIL_LAYER_DEVICE_FN(create_fence, vkCreateFence) + MOCKTAIL_LAYER_DEVICE_FN(wait_for_fences, vkWaitForFences) + MOCKTAIL_LAYER_DEVICE_FN(destroy_fence, vkDestroyFence) + MOCKTAIL_LAYER_DEVICE_FN(create_buffer, vkCreateBuffer) + MOCKTAIL_LAYER_DEVICE_FN(destroy_buffer, vkDestroyBuffer) + MOCKTAIL_LAYER_DEVICE_FN(get_buffer_memory_requirements, + vkGetBufferMemoryRequirements) + MOCKTAIL_LAYER_DEVICE_FN(allocate_memory, vkAllocateMemory) + MOCKTAIL_LAYER_DEVICE_FN(free_memory, vkFreeMemory) + MOCKTAIL_LAYER_DEVICE_FN(bind_buffer_memory, vkBindBufferMemory) + MOCKTAIL_LAYER_DEVICE_FN(map_memory, vkMapMemory) + MOCKTAIL_LAYER_DEVICE_FN(unmap_memory, vkUnmapMemory) + MOCKTAIL_LAYER_DEVICE_FN(destroy_command_pool, vkDestroyCommandPool) +#undef MOCKTAIL_LAYER_DEVICE_FN + dispatch.memory_properties = + reinterpret_cast( + next_gipa(g_instance, "vkGetPhysicalDeviceMemoryProperties")); + if (g_active) { + Log(" [vr-evidence-layer] CreateDevice: g_instance=%p chain-gipa " + "memory-props=%p physdev-props=%p enumerate-pd=%p pd=%p\n", + static_cast(g_instance), + reinterpret_cast(dispatch.memory_properties), + reinterpret_cast( + next_gipa(g_instance, "vkGetPhysicalDeviceProperties")), + reinterpret_cast( + next_gipa(g_instance, "vkEnumeratePhysicalDevices")), + static_cast(physical_device)); + } + if (dispatch.memory_properties == nullptr) { + // NOTE: dlsym(RTLD_DEFAULT) returns the loader's top-level trampoline, + // which rejects below-chain physical device handles; do not use it here. + } + if (dispatch.memory_properties == nullptr) { + Log(" [vr-evidence-layer] warning: memory-properties entry unresolved; " + "pixel readback will be unavailable\n"); + } + { + std::lock_guard lock(g_mutex); + g_dispatches[DispatchKey(*device)] = dispatch; + } + if (g_active) { + Log(" [vr-evidence-layer] device dispatch registered (%p)\n", + static_cast(*device)); + } + return result; +} + +VKAPI_ATTR VkResult VKAPI_CALL LayerCreateInstance( + const VkInstanceCreateInfo *info, const VkAllocationCallbacks *allocator, + VkInstance *instance) { + auto *link = const_cast( + static_cast( + FindInChain(info != nullptr ? info->pNext : nullptr, + VK_STRUCTURE_TYPE_LOADER_INSTANCE_CREATE_INFO))); + if (link == nullptr || link->u.pLayerInfo == nullptr) { + if (g_active) { + Log(" [vr-evidence-layer] CreateInstance: loader chain info missing " + "(link=%p)\n", + static_cast(link)); + } + return VK_ERROR_INITIALIZATION_FAILED; + } + const VkLayerInstanceLink *instance_link = link->u.pLayerInfo; + PFN_vkGetInstanceProcAddr next_gipa = + instance_link->pfnNextGetInstanceProcAddr; + PFN_vkCreateInstance next_create = reinterpret_cast( + next_gipa(VK_NULL_HANDLE, "vkCreateInstance")); + if (next_create == nullptr) { + Log(" [vr-evidence-layer] CreateInstance: next chain entry is null\n"); + return VK_ERROR_INITIALIZATION_FAILED; + } + link->u.pLayerInfo = link->u.pLayerInfo->pNext; + const VkResult result = next_create(info, allocator, instance); + if (g_active) { + Log(" [vr-evidence-layer] CreateInstance result=%d api=%u\n", + static_cast(result), + info != nullptr && info->pApplicationInfo != nullptr + ? info->pApplicationInfo->apiVersion + : 0u); + } + if (result == VK_SUCCESS) { + g_instance_gipa = next_gipa; + if (instance != nullptr) { + g_instance = *instance; + } + } + return result; +} + +VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL +LayerGetInstanceProcAddr(VkInstance instance, const char *name) { + if (name == nullptr) { + return nullptr; + } + if (std::strcmp(name, "vkGetInstanceProcAddr") == 0) { + return reinterpret_cast(LayerGetInstanceProcAddr); + } + if (std::strcmp(name, "vkCreateInstance") == 0) { + return reinterpret_cast(LayerCreateInstance); + } + if (std::strcmp(name, "vkCreateDevice") == 0) { + return reinterpret_cast(LayerCreateDevice); + } + if (g_instance_gipa != nullptr) { + return g_instance_gipa(instance, name); + } + return nullptr; +} + +} // namespace + +extern "C" { + +MOCKTAIL_VR_LAYER_EXPORT VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL +vkGetInstanceProcAddr(VkInstance instance, const char *name) { + return LayerGetInstanceProcAddr(instance, name); +} + +MOCKTAIL_VR_LAYER_EXPORT VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL +vkGetDeviceProcAddr(VkDevice device, const char *name) { + return LayerGetDeviceProcAddr(device, name); +} + +MOCKTAIL_VR_LAYER_EXPORT VKAPI_ATTR VkResult VKAPI_CALL +vkNegotiateLoaderLayerInterfaceVersion(VkNegotiateLayerInterface *version) { + if (version == nullptr || + version->sType != LAYER_NEGOTIATE_INTERFACE_STRUCT) { + return VK_ERROR_INITIALIZATION_FAILED; + } + if (!g_active) { + const std::string directory = EvidenceDirectory(); + if (!directory.empty()) { + g_active = true; + g_initializing_object = reinterpret_cast( + dlsym(RTLD_DEFAULT, "mocktail_vr_initializing_object")); + g_current_eye = reinterpret_cast( + dlsym(RTLD_DEFAULT, "mocktail_vr_current_eye")); + g_consume_eye = reinterpret_cast( + dlsym(RTLD_DEFAULT, "mocktail_vr_consume_eye")); + g_capture_present = + EnvUnsigned("MOCKTAIL_VR_EVIDENCE_CAPTURE_PRESENT", 60); + g_capture_render_passes = + EnvUnsigned("MOCKTAIL_VR_EVIDENCE_CAPTURE_RENDER_PASSES", 10); + Log(" [vr-evidence-layer] activated: dir=%s " + "capture_present=%u capture_render_passes=%u\n", + directory.c_str(), g_capture_present, g_capture_render_passes); + } + } + if (version->loaderLayerInterfaceVersion >= 2) { + version->pfnGetInstanceProcAddr = LayerGetInstanceProcAddr; + version->pfnGetDeviceProcAddr = LayerGetDeviceProcAddr; + version->pfnGetPhysicalDeviceProcAddr = nullptr; + version->loaderLayerInterfaceVersion = 2; + } + return VK_SUCCESS; +} + +} // extern "C" diff --git a/tests/vr/vr_evidence_layer_test.cc b/tests/vr/vr_evidence_layer_test.cc new file mode 100644 index 00000000..4dd7fb1d --- /dev/null +++ b/tests/vr/vr_evidence_layer_test.cc @@ -0,0 +1,131 @@ +// Exercise the layer's provenance joins without a GPU or guest payload. +#include "vr_evidence_layer.cc" +#include + +namespace { +void *test_owner = nullptr; +void *test_request_owner = nullptr; +int test_eye = -1; +std::uintptr_t next_handle = 0x1000; +void *InitializerOwner() { return test_owner; } +bool CurrentEye(void **owner, int *index, void **framebuffer) { + *owner = test_request_owner; + *index = test_eye; + *framebuffer = reinterpret_cast(0x8000); + return test_request_owner != nullptr; +} +void ConsumeEye() { test_request_owner = nullptr; } +VKAPI_ATTR VkResult VKAPI_CALL CreateImageStub(VkDevice, + const VkImageCreateInfo *, + const VkAllocationCallbacks *, + VkImage *image) { + *image = reinterpret_cast(++next_handle); + return VK_SUCCESS; +} +VKAPI_ATTR VkResult VKAPI_CALL CreateFramebufferStub( + VkDevice, const VkFramebufferCreateInfo *, const VkAllocationCallbacks *, + VkFramebuffer *framebuffer) { + *framebuffer = reinterpret_cast(++next_handle); + return VK_SUCCESS; +} +VKAPI_ATTR void VKAPI_CALL RenderPassStub(VkCommandBuffer, + const VkRenderPassBeginInfo *, + VkSubpassContents) {} +VKAPI_ATTR void VKAPI_CALL DestroyImageStub(VkDevice, VkImage, + const VkAllocationCallbacks *) {} +VKAPI_ATTR void VKAPI_CALL DestroyFramebufferStub( + VkDevice, VkFramebuffer, const VkAllocationCallbacks *) {} + +class VrEvidenceProvenance : public ::testing::Test { +protected: + void *key = this; + VkDevice device = reinterpret_cast(&key); + VkCommandBuffer command = reinterpret_cast(&key); + void SetUp() override { + g_active = true; + g_images.clear(); + g_image_views.clear(); + g_framebuffer_images.clear(); + g_dispatches.clear(); + g_eyes[0] = {}; + g_eyes[1] = {}; + g_initializing_object = InitializerOwner; + g_current_eye = CurrentEye; + g_consume_eye = ConsumeEye; + test_owner = nullptr; + test_request_owner = nullptr; + auto &dispatch = g_dispatches[key]; + dispatch.device = device; + dispatch.create_image = CreateImageStub; + dispatch.create_framebuffer = CreateFramebufferStub; + dispatch.cmd_begin_render_pass = RenderPassStub; + dispatch.destroy_image = DestroyImageStub; + dispatch.destroy_framebuffer = DestroyFramebufferStub; + } + VkImage Image() { + VkImageCreateInfo info{}; + info.extent = {500, 500, 1}; + info.format = VK_FORMAT_R8G8B8A8_UNORM; + info.usage = + VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; + VkImage image; + EXPECT_EQ(LayerCreateImage(device, &info, nullptr, &image), VK_SUCCESS); + return image; + } + VkFramebuffer Framebuffer(VkImage image) { + VkImageView view = reinterpret_cast(++next_handle); + g_image_views[view] = image; + VkFramebufferCreateInfo info{}; + info.attachmentCount = 1; + info.pAttachments = &view; + VkFramebuffer framebuffer; + EXPECT_EQ(LayerCreateFramebuffer(device, &info, nullptr, &framebuffer), + VK_SUCCESS); + return framebuffer; + } + void Render(VkFramebuffer framebuffer, void *owner, int eye) { + test_request_owner = owner; + test_eye = eye; + VkRenderPassBeginInfo info{}; + info.framebuffer = framebuffer; + LayerCmdBeginRenderPass(command, &info, VK_SUBPASS_CONTENTS_INLINE); + } +}; + +TEST_F(VrEvidenceProvenance, + IgnoresSameSizeTargetsOutsideInitializerAndWrongOwner) { + void *owner = reinterpret_cast(0x1234); + auto unrelated = Image(); + Render(Framebuffer(unrelated), owner, 0); + EXPECT_FALSE(g_eyes[0].valid); + EXPECT_TRUE(g_images.empty()); + test_owner = owner; + auto image = Image(); + auto framebuffer = Framebuffer(image); + Render(framebuffer, reinterpret_cast(0x5678), 0); + EXPECT_FALSE(g_eyes[0].valid); + Render(framebuffer, owner, 0); + EXPECT_TRUE(g_eyes[0].valid); + EXPECT_EQ(g_eyes[0].image.image, image); +} + +TEST_F(VrEvidenceProvenance, + EyeIndexComesFromGetterAndDestroyedHandlesAreForgotten) { + test_owner = reinterpret_cast(0x1234); + auto right = Image(); + auto left = Image(); + auto right_fb = Framebuffer(right); + auto left_fb = Framebuffer(left); + Render(right_fb, test_owner, 1); + Render(left_fb, test_owner, 0); + EXPECT_EQ(g_eyes[0].image.image, left); + EXPECT_EQ(g_eyes[1].image.image, right); + LayerDestroyImage(device, left, nullptr); + EXPECT_FALSE(g_eyes[0].valid); + EXPECT_FALSE(g_eyes[1].valid); + Render(left_fb, test_owner, 0); + EXPECT_FALSE(g_eyes[0].valid); + LayerDestroyFramebuffer(device, right_fb, nullptr); + EXPECT_EQ(g_framebuffer_images.count(right_fb), 0u); +} +} // namespace diff --git a/tests/vr/vr_mirror_test.cc b/tests/vr/vr_mirror_test.cc new file mode 100644 index 00000000..f2a8e35d --- /dev/null +++ b/tests/vr/vr_mirror_test.cc @@ -0,0 +1,200 @@ +#include + +#include +#include +#include + +#include +#include +#include "mocktail/vr/vr_perf.h" +#include "mocktail/vr/openxr_backend.h" +#include +#include +#include + +#include "mocktail/vr/vr_mirror.h" +#include "mocktail/vr/vr_mirror_shm.h" + +namespace mocktail::vr { +namespace { + +constexpr std::uint32_t kWidth = 8; +constexpr std::uint32_t kHeight = 4; + +std::vector EyePattern(std::uint8_t seed) { + std::vector bytes(kWidth * kHeight * 4); + for (std::size_t index = 0; index < bytes.size(); ++index) { + bytes[index] = static_cast(seed + index); + } + return bytes; +} + +// Maps the segment read-only exactly like the Python viewer does. +class SegmentReader { + public: + SegmentReader(const std::string& name, std::uint32_t width, + std::uint32_t height, bool* ok) { + *ok = false; + const int descriptor = shm_open(name.c_str(), O_RDONLY, 0); + if (descriptor < 0) { + return; + } + bytes_ = VrMirrorBytes(width, height); + mapping_ = mmap(nullptr, bytes_, PROT_READ, MAP_SHARED, descriptor, 0); + close(descriptor); + *ok = mapping_ != MAP_FAILED; + } + ~SegmentReader() { + if (mapping_ != nullptr && mapping_ != MAP_FAILED) { + munmap(mapping_, bytes_); + } + } + const VrMirrorHeader& header() const { + return *static_cast(mapping_); + } + const std::uint8_t* eye(int index) const { + return static_cast(mapping_) + kVrMirrorHeaderBytes + + static_cast(index) * kWidth * kHeight * 4; + } + SegmentReader(const SegmentReader&) = delete; + SegmentReader& operator=(const SegmentReader&) = delete; + + private: + void* mapping_ = nullptr; + std::uint64_t bytes_ = 0; +}; + +TEST(VrRuntimeSelection, RuntimeErrorsPreserveSelectionProvenance) { + EXPECT_NE(VrRuntimeUnavailableHint(true, "/custom.json").find("User-selected"), std::string::npos); + EXPECT_NE(VrRuntimeUnavailableHint(false, "/usr/share/openxr/1/openxr_wivrn.json").find("Auto-selected WiVRn"), std::string::npos); + EXPECT_NE(VrRuntimeUnavailableHint(false, "").find("registered"), std::string::npos); +} + +TEST(VrRuntimeSelection, ExplicitManifestWinsAndMissingCandidatesUseLoader) { + EXPECT_EQ(SelectVrRuntimeManifest("/explicit/runtime.json", {"/dev/null"}), + "/explicit/runtime.json"); + EXPECT_TRUE(SelectVrRuntimeManifest("", {"/definitely-missing-wivrn/runtime.json"}).empty()); + char path[] = "/tmp/mocktail-wivrn-manifest-XXXXXX"; + const int fd = mkstemp(path); + ASSERT_GE(fd, 0); + close(fd); + EXPECT_EQ(SelectVrRuntimeManifest("", {"/missing/runtime.json", path}), path); + unlink(path); +} + +TEST(VrMirrorProducer, AllocationFailureUsesOwningDevice) { + MirrorVkProcs vk{}; + vk.GetPhysicalDeviceMemoryProperties = [](VkPhysicalDevice, VkPhysicalDeviceMemoryProperties* out) { + *out = {}; + out->memoryTypeCount = 1; + out->memoryTypes[0].propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + }; + vk.CreateBuffer = [](VkDevice, const VkBufferCreateInfo*, const VkAllocationCallbacks*, VkBuffer* out) { + *out = reinterpret_cast(0x22); + return VK_SUCCESS; + }; + vk.GetBufferMemoryRequirements = [](VkDevice, VkBuffer, VkMemoryRequirements* out) { + *out = {}; out->size = 256; out->memoryTypeBits = 1; + }; + vk.AllocateMemory = [](VkDevice, const VkMemoryAllocateInfo*, const VkAllocationCallbacks*, VkDeviceMemory*) { + return VK_ERROR_OUT_OF_DEVICE_MEMORY; + }; + vk.DestroyBuffer = [](VkDevice device, VkBuffer, const VkAllocationCallbacks*) { + EXPECT_EQ(device, reinterpret_cast(0x11)); + }; + VrMirror mirror; + std::string error; + EXPECT_FALSE(mirror.EnsureStaging(vk, reinterpret_cast(0x11), + VK_NULL_HANDLE, 8, 8, &error)); +} + +TEST(VrMirrorPerf, RingAverageDoesNotAccumulateOverwrittenSamples) { + setenv("MOCKTAIL_VR_PERF", "1", 1); + unsetenv("MOCKTAIL_VR_MEASURE_START_NS"); + perf::Collector collector; + for (int i = 0; i < 8192; ++i) collector.Record(perf::Stage::kFrameCycle, 1000000); + testing::internal::CaptureStderr(); + collector.EmitIfDue(1, "test"); + collector.EmitIfDue(1000000001, "test"); + const auto output = testing::internal::GetCapturedStderr(); + EXPECT_NE(output.find("avg=1.00ms"), std::string::npos) << output; +} + +TEST(VrMirrorProducer, PublishesLatestPairUnderSeqlockAndUnlinksOnClose) { + const std::string name = "/mocktail-vr-mirror-unittest-a"; + { + VrMirror mirror; + std::string error; + ASSERT_TRUE(mirror.Open(name, kWidth, kHeight, &error)) << error; + EXPECT_TRUE(mirror.enabled()); + + bool opened = false; + SegmentReader reader(name, kWidth, kHeight, &opened); + ASSERT_TRUE(opened); + EXPECT_EQ(reader.header().magic, kVrMirrorMagic); + EXPECT_EQ(reader.header().header_bytes, kVrMirrorHeaderBytes); + EXPECT_EQ(reader.header().seq, 0u); + EXPECT_EQ(reader.header().flags & kVrMirrorFlagValid, 0u); + + const auto eye0 = EyePattern(10); + const auto eye1 = EyePattern(90); + mirror.PublishFromBytesForTest(41, VK_FORMAT_R8G8B8A8_UNORM, kWidth, + kHeight, 12345, eye0.data(), eye1.data()); + + // Even sequence: the pair is stable for readers. + EXPECT_EQ(reader.header().seq % 2, 0u); + EXPECT_EQ(reader.header().seq, 2u); + EXPECT_EQ(reader.header().frame, 41u); + EXPECT_EQ(reader.header().producer_ns, 12345u); + EXPECT_EQ(reader.header().width, kWidth); + EXPECT_EQ(reader.header().height, kHeight); + EXPECT_EQ(reader.header().flags & kVrMirrorFlagValid, kVrMirrorFlagValid); + EXPECT_EQ(reader.header().published_pairs, 1u); + EXPECT_EQ(std::memcmp(reader.eye(0), eye0.data(), eye0.size()), 0); + EXPECT_EQ(std::memcmp(reader.eye(1), eye1.data(), eye1.size()), 0); + // Eyes must never alias into a single buffer. + EXPECT_NE(std::memcmp(reader.eye(0), reader.eye(1), eye0.size()), 0); + + // A newer publish replaces the single latest slot; readers see only the + // newest pair and the sequence advances by two per publish. + const auto next0 = EyePattern(33); + const auto next1 = EyePattern(77); + mirror.PublishFromBytesForTest(42, VK_FORMAT_B8G8R8A8_UNORM, kWidth, + kHeight, 54321, next0.data(), next1.data()); + EXPECT_EQ(reader.header().seq, 4u); + EXPECT_EQ(reader.header().frame, 42u); + EXPECT_EQ(reader.header().format, + static_cast(VK_FORMAT_B8G8R8A8_UNORM)); + EXPECT_EQ(std::memcmp(reader.eye(0), next0.data(), next0.size()), 0); + EXPECT_EQ(std::memcmp(reader.eye(1), next1.data(), next1.size()), 0); + EXPECT_EQ(mirror.published_pairs(), 2u); + + mirror.Close(); + EXPECT_FALSE(mirror.enabled()); + } + // The producer owns the segment lifetime: after Close the name is unlinked. + EXPECT_EQ(shm_open(name.c_str(), O_RDONLY, 0), -1); +} + +TEST(VrMirrorProducer, RejectsInvalidNameAndOversizedPublish) { + VrMirror mirror; + std::string error; + EXPECT_FALSE(mirror.Open("", kWidth, kHeight, &error)); + EXPECT_FALSE(mirror.Open("relative-name", kWidth, kHeight, &error)); + EXPECT_FALSE(mirror.Open("/mocktail-vr-mirror-unittest-b", 0, kHeight, + &error)); + EXPECT_FALSE(mirror.enabled()); + + const std::string name = "/mocktail-vr-mirror-unittest-c"; + ASSERT_TRUE(mirror.Open(name, kWidth, kHeight, &error)) << error; + // Publishing an extent larger than the mapped segment is refused, not + // truncated into foreign memory. + const auto big0 = std::vector(kWidth * kHeight * 4 * 4, 7); + mirror.PublishFromBytesForTest(1, VK_FORMAT_R8G8B8A8_UNORM, kWidth * 2, + kHeight * 2, 0, big0.data(), big0.data()); + EXPECT_EQ(mirror.published_pairs(), 0u); + mirror.Close(); +} + +} // namespace +} // namespace mocktail::vr diff --git a/tests/vr/vr_pose_math_test.cc b/tests/vr/vr_pose_math_test.cc new file mode 100644 index 00000000..4c931b19 --- /dev/null +++ b/tests/vr/vr_pose_math_test.cc @@ -0,0 +1,187 @@ +#include + +#include +#include + +#include "mocktail/vr/vr_pose_math.h" + +namespace mocktail::vr { +namespace { + +using namespace pose; + +constexpr float kTol = 1e-5f; + +Vec3 Forward() { return Vec3{0.f, 0.f, -1.f}; } // OpenXR: -Z is forward +Vec3 Up() { return Vec3{0.f, 1.f, 0.f}; } +Vec3 Right() { return Vec3{1.f, 0.f, 0.f}; } + +void ExpectVec(const Vec3& actual, const Vec3& expected, float tolerance) { + EXPECT_NEAR(actual.x, expected.x, tolerance) << "x"; + EXPECT_NEAR(actual.y, expected.y, tolerance) << "y"; + EXPECT_NEAR(actual.z, expected.z, tolerance) << "z"; +} + +// ---- rotation axes verified independently, not only in combination ---- + +TEST(VrPoseMath, YawAloneRotatesForwardWithoutAxisInversion) { + const float yaw = 0.6f; + const Quat q = QuatFromYawPitch(yaw, 0.f); + // Rotation about +Y by yaw takes forward (0,0,-1) to (-sin, 0, -cos). + const Vec3 rotated = QuatRotate(q, Forward()); + ExpectVec(rotated, Vec3{-std::sin(yaw), 0.f, -std::cos(yaw)}, kTol); + // Up is untouched by yaw: an inverted axis would move it. + ExpectVec(QuatRotate(q, Up()), Up(), kTol); +} + +TEST(VrPoseMath, PitchAloneRotatesForwardWithoutAxisInversion) { + const float pitch = 0.4f; + const Quat q = QuatFromYawPitch(0.f, pitch); + // Rotation about +X by pitch takes (0,0,-1) to (0, sin, -cos): looking up + // (positive pitch) raises the forward vector's Y, matching head tilt up. + const Vec3 rotated = QuatRotate(q, Forward()); + ExpectVec(rotated, Vec3{0.f, std::sin(pitch), -std::cos(pitch)}, kTol); + ExpectVec(QuatRotate(q, Right()), Right(), kTol); +} + +TEST(VrPoseMath, RollAloneRotatesUpAroundForwardAxis) { + const float roll = 0.35f; + const Quat q = QuatFromAxisAngle(Vec3{0.f, 0.f, 1.f}, roll); + // Right-handed rotation about +Z takes up (0,1,0) to (-sin, cos, 0): the + // head tilting right rotates the world's up toward the viewer's left. + const Vec3 rotated_up = QuatRotate(q, Up()); + ExpectVec(rotated_up, Vec3{-std::sin(roll), std::cos(roll), 0.f}, kTol); + // The roll axis itself is invariant. + ExpectVec(QuatRotate(q, Vec3{0.f, 0.f, 1.f}), Vec3{0.f, 0.f, 1.f}, kTol); +} + +TEST(VrPoseMath, YawPitchCombinationMatchesIndependentComposition) { + const float yaw = 0.5f, pitch = 0.3f; + const Quat combined = QuatFromYawPitch(yaw, pitch); + // Documented order: R = Ry(yaw) * Rx(pitch). + const Quat yaw_only = QuatFromYawPitch(yaw, 0.f); + const Quat pitch_only = QuatFromYawPitch(0.f, pitch); + const Quat expected = QuatMultiply(yaw_only, pitch_only); + EXPECT_TRUE(QuatSameRotation(combined, expected, kTol)); + // A forward vector must agree between both constructions. + const Vec3 a = QuatRotate(combined, Forward()); + const Vec3 b = QuatRotate(expected, Forward()); + ExpectVec(a, b, kTol); + // And against explicit sequential rotation: pitch first, then yaw. + const Vec3 c = QuatRotate(yaw_only, QuatRotate(pitch_only, Forward())); + ExpectVec(a, c, kTol); +} + +TEST(VrPoseMath, PoseMultiplyAndInverseRoundTrip) { + Pose a; + a.position = Vec3{0.3f, -1.2f, 0.7f}; + a.orientation = QuatFromYawPitch(0.4f, -0.25f); + const Pose identity = PoseMultiply(a, PoseInverse(a)); + ExpectVec(identity.position, Vec3{0.f, 0.f, 0.f}, kTol); + EXPECT_TRUE(QuatSameRotation(identity.orientation, Quat{0.f, 0.f, 0.f, 1.f}, + kTol)); +} + +TEST(VrPoseMath, EyeOffsetIsHeadInverseTimesRuntimeEye) { + // The backend computes relative = inverse(head) * runtime_eye and reprojects + // final_eye = head * relative, which must reproduce the runtime eye exactly. + Pose head; + head.position = Vec3{0.2f, 1.5f, -0.4f}; + head.orientation = QuatFromYawPitch(0.7f, 0.15f); + Pose runtime_eye = head; + runtime_eye.position = PoseMultiply(head, Pose{Vec3{-0.032f, 0.f, 0.f}, + Quat{}}).position; + const Pose relative = PoseMultiply(PoseInverse(head), runtime_eye); + const Pose final_eye = PoseMultiply(head, relative); + ExpectVec(final_eye.position, runtime_eye.position, kTol); +} + +// ---- quaternion sign/normalization contract ---- + +TEST(VrPoseMath, QuaternionSignDoesNotChangeRotation) { + const Quat q = QuatFromYawPitch(0.9f, -0.4f); + const Quat negated{-q.x, -q.y, -q.z, -q.w}; + EXPECT_TRUE(QuatSameRotation(q, negated, kTol)); + ExpectVec(QuatRotate(q, Forward()), QuatRotate(negated, Forward()), kTol); + const Pose p{Vec3{0.1f, 0.2f, 0.3f}, q}; + const Pose pn{Vec3{0.1f, 0.2f, 0.3f}, negated}; + const Pose other{Vec3{-0.5f, 1.f, 0.25f}, QuatFromAxisAngle(Up(), 0.3f)}; + const Pose a = PoseMultiply(p, other); + const Pose b = PoseMultiply(pn, other); + ExpectVec(a.position, b.position, kTol); + EXPECT_TRUE(QuatSameRotation(a.orientation, b.orientation, kTol)); +} + +TEST(VrPoseMath, NormalizeRejectsNonfiniteAndNearZeroWithoutWritingOutput) { + Quat out{9.f, 9.f, 9.f, 9.f}; + Quat input{0.f, 0.f, 0.f, 2.f}; + ASSERT_TRUE(QuatNormalize(input, &out)); + ExpectVec(Vec3{out.x, out.y, out.z}, Vec3{0.f, 0.f, 0.f}, kTol); + EXPECT_NEAR(out.w, 1.f, kTol); + + const float nan = std::numeric_limits::quiet_NaN(); + input = Quat{nan, 0.f, 0.f, 1.f}; + out = Quat{9.f, 9.f, 9.f, 9.f}; + EXPECT_FALSE(QuatNormalize(input, &out)); + EXPECT_FLOAT_EQ(out.x, 9.f); // output untouched on rejection + + input = Quat{0.f, 0.f, 0.f, 0.f}; + EXPECT_FALSE(QuatNormalize(input, &out)); + EXPECT_FALSE(QuatNormalize(input, nullptr)); +} + +// ---- FOV order/sign contract (asymmetric, pinned to the 2998 guest layout) -- + +TEST(VrPoseMath, FovToTangentsPinsUpDownLeftRightOrderAndSigns) { + EyeFov fov; + fov.angle_left = -0.7f; + fov.angle_right = 0.8f; + fov.angle_up = 0.9f; + fov.angle_down = -0.6f; + const ProjectionTangents tangents = FovToTangents(fov); + // The guest state block stores up, down, left, right in this order; down and + // left are negated relative to the OpenXR angle signs. The existing bridge + // test pins the same expectation through ApplyXrPose; this pins the math. + EXPECT_FLOAT_EQ(tangents.up, std::tan(0.9f)); + EXPECT_FLOAT_EQ(tangents.down, std::tan(0.6f)); + EXPECT_FLOAT_EQ(tangents.left, std::tan(0.7f)); + EXPECT_FLOAT_EQ(tangents.right, std::tan(0.8f)); + // Asymmetric FOV must not collapse to a symmetric projection. + EXPECT_NE(tangents.left, tangents.right); + EXPECT_NE(tangents.up, tangents.down); +} + +// ---- canted-view rejection ---- + +TEST(VrPoseMath, ParallelEyeOrientationsAcceptedCantedRejected) { + EXPECT_TRUE(EyeOrientationIsParallel(Quat{0.f, 0.f, 0.f, 1.f})); + // Sign flip only (q == -q) is still parallel. + EXPECT_TRUE(EyeOrientationIsParallel(Quat{0.f, 0.f, 0.f, -1.f})); + // Exactly at tolerance. + EXPECT_TRUE(EyeOrientationIsParallel( + Quat{kCantedTolerance, 0.f, 0.f, 1.f})); + // A canted runtime rotates the eye about its view axis (+Z): must reject. + const Quat canted = QuatFromAxisAngle(Vec3{0.f, 0.f, 1.f}, 0.1f); + EXPECT_FALSE(EyeOrientationIsParallel(canted)); + // Any axis above tolerance rejects. + EXPECT_FALSE(EyeOrientationIsParallel( + Quat{0.f, 2.f * kCantedTolerance, 0.f, 1.f})); +} + +// ---- units: the 0.10 m translation check through the native conversion ---- + +TEST(VrPoseMath, TenthMetreConversionIsDocumentedAndNotPreApplied) { + // The exact-build native helper converts metres to studs itself with factor + // 10/3; 0.10 m must arrive in the guest state as metres (0.10) and become + // 1/3 stud natively. This layer only documents the expected value. + EXPECT_FLOAT_EQ(MetresToStuds(0.10f), 0.10f * kMetresToStuds); + EXPECT_NEAR(MetresToStuds(0.10f), 1.0f / 3.0f, 1e-6f); + // Engine HeadScale multiplies head.position again inside the camera + // composition (out.position = base + base.rotation * head.position * + // HeadScale); it is an engine property distinct from the 10/3 conversion and + // was verified numerically with the native-camera runtime observer. + EXPECT_NE(kMetresToStuds, 1.f); +} + +} // namespace +} // namespace mocktail::vr From 3d89f128cee789a2fde337b100a25f4861aa8d95 Mon Sep 17 00:00:00 2001 From: komaruworld Date: Sun, 13 Sep 2026 19:28:28 +0300 Subject: [PATCH 03/58] support alvr and opengl --- README.md | 126 ++++++++- cmake/MocktailVR.cmake | 6 +- include/mocktail/vr/openxr_backend.h | 40 ++- src/main.cc | 23 +- src/vr/gles_transport.cc | 395 ++++++++++++++++++++++++++ src/vr/gles_transport.h | 69 +++++ src/vr/openxr_backend.cc | 399 ++++++++++++++++++++++----- src/vr/openxr_backend_mode.cc | 259 +++++++++++++++-- src/vr/openxr_disabled.cc | 2 +- src/window/window.cc | 61 +++- tests/CMakeLists.txt | 7 +- tests/vr/gles_transport_test.cc | 260 +++++++++++++++++ tests/vr/runtime_detection_test.cc | 141 ++++++++++ tests/vr/vr_mirror_test.cc | 30 +- 14 files changed, 1711 insertions(+), 107 deletions(-) create mode 100644 src/vr/gles_transport.cc create mode 100644 src/vr/gles_transport.h create mode 100644 tests/vr/gles_transport_test.cc create mode 100644 tests/vr/runtime_detection_test.cc diff --git a/README.md b/README.md index 99341c33..e14e232f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,125 @@ -This is a dev branch for vr support in Mocktail. +# Mocktail VR (experimental) -idk if I can do this. +Build this checkout, start WiVRn or SteamVR/ALVR, then connect your headset before launching: -But it will be fun :) +```sh +git submodule update --init --recursive +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DMOCKTAIL_ENABLE_VR=ON +cmake --build build -j4 +./build/mocktail --vr +``` + +## Automatic runtime selection + +Normally just run `./build/mocktail --vr`. The default `auto` mode detects a +running SteamVR server (also used by ALVR) or a listening WiVRn IPC socket for +the current desktop user. A stale socket file does not count as a running +server. If exactly one server is detected, its runtime takes priority over an +old registration for the other server. If both or neither are detected, the +registered OpenXR runtime has priority. + +SteamVR is also found through `openvr/openvrpaths.vrpath`, standard Steam +locations, and the running server's executable path. WiVRn development builds +are discovered near the running server executable; a matching registered +WiVRn manifest is preferred, including a Flatpak registration. The selected +path and the reason for selecting it are printed at startup. + +Explicit `XR_RUNTIME_JSON` and `MOCKTAIL_VR_RUNTIME` overrides remain respected. +To return to automatic selection after manually choosing a runtime: + +```sh +unset XR_RUNTIME_JSON MOCKTAIL_VR_RUNTIME +./build/mocktail --vr +``` + +Detection does not start servers, connect a headset, or modify system runtime +registration. It observes the current process/user namespace; container and +host runtimes still need correctly shared libraries and IPC access. + +## WiVRn + +Start WiVRn and connect its headset client first. Run WiVRn and Mocktail as the +same desktop user, with the same runtime environment; do not start Mocktail +from a root shell while WiVRn runs in your user session. For containers/chroots, +the host runtime library and IPC socket must both be accessible inside. + +```sh +MOCKTAIL_VR_RUNTIME=wivrn ./build/mocktail --vr +``` + +If WiVRn disconnects and exits, reconnect/restart it before retrying Mocktail. +`Connection refused` on `wivrn/comp_ipc` means the selected server is not +accepting the connection. Installing `dbus-launch` alone does not fix that. + +## ALVR through SteamVR + +Install SteamVR and ALVR, register the ALVR driver using its dashboard, then +connect the headset and confirm SteamVR sees it. Follow the upstream +[ALVR Linux instructions](https://github.com/alvr-org/ALVR/wiki/Linux-Troubleshooting), +including the SteamVR launch-options workaround when required. + +```sh +env -u XR_RUNTIME_JSON MOCKTAIL_VR_RUNTIME=alvr \ + MOCKTAIL_GRAPHICS_BACKEND=direct-vulkan ./build/mocktail --vr +``` + +The command clears an inherited WiVRn manifest override. `steamvr` is an alias +for `alvr`. The selection uses SteamVR's OpenXR runtime; +ALVR handles the streaming. If automatic discovery cannot find a custom +installation, specify its actual manifest: + +```sh +XR_RUNTIME_JSON="/path/to/SteamVR/steamxr_linux64.json" ./build/mocktail --vr +``` + +`XR_RUNTIME_JSON` always wins. `MOCKTAIL_VR_RUNTIME=system` delegates selection +to the OpenXR loader. The default `auto` uses the server detection described above, then registered +and installed runtimes. Selection affects only this process. + +## OpenGL ES + +The `opengl`/`system` renderer uses the guest's real EGL/OpenGL ES context: + +```sh +MOCKTAIL_GRAPHICS_BACKEND=opengl ./build/mocktail --vr +``` + +The runtime must provide `XR_KHR_opengl_es_enable` and `XR_MNDX_egl_enable`, and +support sharing images with the guest EGL device. Both eye copies stay on the +GPU; no Python viewer or CPU readback is required. Camera tracking uses the +same OpenXR frame/pose pipeline as Vulkan. If the runtime cannot support this +EGL binding, use `MOCKTAIL_GRAPHICS_BACKEND=direct-vulkan`; the ALVR command +above deliberately selects Vulkan. + +## Current validation and limitations + +- Native VR camera/eye hooks are verified for Roblox build **2998**, Build ID + `ade08266c67aee88ec9c1d00902150e1684dad3a`. Approval of a newer payload for + ordinary rendering does **not** establish VR compatibility. Do not bypass + the Build-ID checks to run the old VR hooks on build 3092. +- GLES framebuffer copying, distinct eyes, state restoration, resource + invalidation, and an EGL OpenXR session with session-loss recovery have been + tested. A local unauthenticated Roblox run on hardware GLES submitted over + 240 projection frames through a simulated Monado headset. +- This host's software GLES/Vulkan combination could create XR texture names + but not usable framebuffer attachments. The copy failure is reported and + XR output stops; this is not counted as a successful rendering test. +- The local Monado test runtime reports `u_graphics_buffer_unref: Bad file + descriptor (fd: 0)` during GLES teardown, including successful hardware + runs. This remains an integration caveat; clean headset teardown is not + claimed from those simulator runs. During auto-selection validation, one + run also failed a later EGL session recreation with + `vkCreateFence: VK_ERROR_OUT_OF_HOST_MEMORY`; a fresh repeat passed. + Runtime selection and initial frame submission succeeded in both runs. +- Real headset acceptance through WiVRn and ALVR/SteamVR is still pending. + Controller input and canted eye orientations are not implemented. +- Use `./build/mocktail --no-vr` for ordinary rendering. No helper script is + needed for normal VR startup. + +The GLES OpenXR integration test is opt-in and needs an already connected +runtime (or a separately prepared simulator). Run from a graphical desktop: + +```sh +MOCKTAIL_TEST_XR_GLES=1 ./build/tests/gles_vr_transport_test \ + --gtest_filter='*OpenXrEglSession*' +``` diff --git a/cmake/MocktailVR.cmake b/cmake/MocktailVR.cmake index f71f7cb5..f46ee954 100644 --- a/cmake/MocktailVR.cmake +++ b/cmake/MocktailVR.cmake @@ -14,7 +14,7 @@ add_library(mocktail_vr STATIC add_library(Mocktail::VR ALIAS mocktail_vr) target_include_directories(mocktail_vr PUBLIC "${CMAKE_SOURCE_DIR}/include") target_include_directories(mocktail_vr PRIVATE "${CMAKE_SOURCE_DIR}/src") -target_link_libraries(mocktail_vr PUBLIC Vulkan::Headers) +target_link_libraries(mocktail_vr PUBLIC Vulkan::Headers PRIVATE nlohmann_json::nlohmann_json) # shm_open for the desktop eye mirror lives in librt on glibc. find_library(MOCKTAIL_RT_LIBRARY rt) if(MOCKTAIL_RT_LIBRARY) @@ -47,12 +47,14 @@ if(MOCKTAIL_ENABLE_VR) endfunction() mocktail_add_openxr_sdk() target_sources(mocktail_vr PRIVATE src/vr/openxr_probe.cc src/vr/openxr_preview.cc - src/vr/openxr_backend.cc) + src/vr/openxr_backend.cc src/vr/gles_transport.cc) # Error unwinding is confined to these native session implementations; their # public entry points catch errors before returning to the guest runtime. set_source_files_properties(src/vr/openxr_preview.cc src/vr/openxr_backend.cc PROPERTIES COMPILE_OPTIONS -fexceptions) target_compile_definitions(mocktail_vr PRIVATE XR_USE_GRAPHICS_API_VULKAN) + set_source_files_properties(src/vr/openxr_backend.cc PROPERTIES COMPILE_DEFINITIONS "XR_USE_GRAPHICS_API_OPENGL_ES;XR_USE_PLATFORM_EGL") + target_include_directories(mocktail_vr PRIVATE ${MOCKTAIL_EGL_INCLUDE_DIR} ${MOCKTAIL_GLES3_INCLUDE_DIR}) target_link_libraries(mocktail_vr PRIVATE OpenXR::openxr_loader Vulkan::Headers ${CMAKE_DL_LIBS}) else() target_sources(mocktail_vr PRIVATE src/vr/openxr_disabled.cc) diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index bed4428e..9d0fbb80 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -25,10 +25,21 @@ enum class VrBackendMode { kXrOutput, }; -// Explicit XR_RUNTIME_JSON wins. Otherwise prefer a native WiVRn manifest, -// then let the OpenXR loader use the registered runtime (including Flatpak WiVRn). +// Explicit XR_RUNTIME_JSON wins. Runtime preference can select SteamVR (ALVR), +// WiVRn, or the registered OpenXR runtime without changing system registration. std::string SelectVrRuntimeManifest(const std::string& explicit_manifest, const std::vector& candidates); +std::vector VrRuntimeManifestCandidates( + const std::string &preference, const std::string &home, + const std::string &config_home, const std::string &config_dirs); +struct VrRuntimeDiscovery { + std::vector candidates; + std::string reason; +}; +VrRuntimeDiscovery DiscoverVrRuntime( + const std::string& preference, const std::string& home, + const std::string& config_home, const std::string& config_dirs, + const std::string& runtime_dir, const std::string& proc_root = "/proc"); std::string VrRuntimeUnavailableHint(bool user_selected, const std::string& manifest); VrBackendMode ResolveVrBackendMode(bool vr_enabled); const char* VrBackendModeName(VrBackendMode mode); @@ -58,6 +69,10 @@ struct ScriptedPoseSample { // frame N+1 and publishing the scripted/runtime head pose that the device // bridge injects into Roblox before its eye passes run. // +// EGL/OpenGL ES uses XR_MNDX_egl_enable on the guest SDL context and GPU +// framebuffer blits into GLES swapchain textures. It shares the same frame +// scheduling, tracking, pose application and session recovery below. +// // Eye image provenance is recorded in the adapter during the guest eye // initializer (owner = DebugDeviceVR object) and bound to an eye index only // through the real per-frame eye-getter request plus the actual render-pass @@ -67,6 +82,9 @@ struct ScriptedPoseSample { // All public entry points are exception-free at their boundary; the // implementation is compiled with exceptions like openxr_preview.cc and // translates failures into Status/log outcomes. +class GlesTransport; +enum class VrGraphicsApi { kVulkan, kOpenGles }; + class OpenXrBackend final { public: OpenXrBackend() = default; @@ -75,7 +93,14 @@ class OpenXrBackend final { OpenXrBackend& operator=(const OpenXrBackend&) = delete; // Claims the process-wide backend slot and creates XrInstance/system. - Status Arm(); + Status Arm(VrGraphicsApi graphics_api = VrGraphicsApi::kVulkan); + // EGL context must be current on the calling thread. Resolver bypasses guest + // hooks. + Status AttachGlesContext(void *display, void *config, void *context, + void *egl_get_proc, void *(*resolve)(const char *)); + void DetachGlesContext(void *context); + void NoteGlesPresent(); + void *WrapGlesProcAddress(const char *name, void *raw); // Idempotent full teardown (XR resources first, then the instance). void Disarm(); bool armed() const { return armed_.load(std::memory_order_acquire); } @@ -161,6 +186,7 @@ class OpenXrBackend final { struct EyeSwapchain { void* swapchain = nullptr; // XrSwapchain (opaque to keep the header XR-free) std::vector images; + std::vector gl_images; std::uint32_t acquired_index = 0; bool image_acquired = false; std::uint32_t width = 0; @@ -172,6 +198,9 @@ class OpenXrBackend final { VkDevice device, const VkDeviceCreateInfo* create_info, std::string* error); + bool InitializeGlesSessionLocked(std::string *error); + bool InitializeSessionResourcesLocked(std::string *error); + bool CopyGlesEyesIntoSwapchains(); void DisarmLocked(); void* ResolveFunction(const char* name) const; void SetupDeviceAfterCreationLocked(VkPhysicalDevice physical_device, @@ -201,6 +230,11 @@ class OpenXrBackend final { void EnsureMirrorForExtent(std::uint32_t width, std::uint32_t height); MirrorVkProcs MirrorProcs() const; + VrGraphicsApi graphics_api_ = VrGraphicsApi::kVulkan; + GlesTransport *gles_ = nullptr; + std::uint64_t gl_min_version_ = 0; + std::uint64_t gl_max_version_ = 0; + mutable std::mutex mutex_; std::atomic armed_{false}; std::atomic recording_{false}; diff --git a/src/main.cc b/src/main.cc index 1e7a23c0..ea2a2a3f 100644 --- a/src/main.cc +++ b/src/main.cc @@ -351,7 +351,8 @@ int main(int argc, char* argv[]) { "debug device (exact Build ID 2998 only)"; if (vr_backend_mode == mocktail::vr::VrBackendMode::kXrOutput) { std::cerr << "; eye images are submitted to the active OpenXR runtime " - "as a projection layer. Start WiVRn and connect the headset " + "as a projection layer. Start WiVRn or SteamVR/ALVR and " + "connect the headset " "before launching.\n"; } else { std::cerr << "; OpenXR output is disabled in this diagnostic mode " @@ -894,14 +895,20 @@ int main(int argc, char* argv[]) { mocktail::vr::VrBackendMode::kXrOutput) { // Fail closed and loudly when no OpenXR runtime is reachable; the // native-stereo diagnostic mode must be requested explicitly. - const mocktail::Status xr_status = xr_backend.Arm(); + const mocktail::Status xr_status = + xr_backend.Arm(runtime_config.config.graphics_backend() == + mocktail::runtime::GraphicsBackend::kVulkan + ? mocktail::vr::VrGraphicsApi::kVulkan + : mocktail::vr::VrGraphicsApi::kOpenGles); if (!xr_status.ok()) { - std::cerr << "[FATAL] Cannot arm the OpenXR backend for --vr: " - << xr_status.message() << '\n' - << " Start WiVRn, connect your headset, then run mocktail -vr again.\n" - << " For a custom runtime set XR_RUNTIME_JSON to its manifest.\n" - << " Set MOCKTAIL_VR_BACKEND=native for the " - "native-stereo diagnostic mode without a runtime.\n"; + std::cerr + << "[FATAL] Cannot arm the OpenXR backend for --vr: " + << xr_status.message() << '\n' + << " Start WiVRn or SteamVR/ALVR, connect your headset, then " + "run mocktail -vr again.\n" + << " For a custom runtime set XR_RUNTIME_JSON to its manifest.\n" + << " Set MOCKTAIL_VR_BACKEND=native for the " + "native-stereo diagnostic mode without a runtime.\n"; return EXIT_FAILURE; } } diff --git a/src/vr/gles_transport.cc b/src/vr/gles_transport.cc new file mode 100644 index 00000000..f083c51c --- /dev/null +++ b/src/vr/gles_transport.cc @@ -0,0 +1,395 @@ +#include "vr/gles_transport.h" +#include "mocktail/vr/openxr_backend.h" + +#include +#include +#include +#include + +extern "C" void *mocktail_vr_initializing_object(); +extern "C" bool mocktail_vr_current_eye(void **, int *, void **); +extern "C" void mocktail_vr_consume_eye(); + +namespace mocktail::vr { +namespace { +// The window owns a single EGL context. Function pointers are loaded from +// that exact SDL EGL/GLES implementation, never from libGL or the guest shim. +#define GL_PROCS(X) \ + X(GetIntegerv, PFNGLGETINTEGERVPROC) \ + X(IsEnabled, PFNGLISENABLEDPROC) \ + X(Enable, PFNGLENABLEPROC) \ + X(Disable, PFNGLDISABLEPROC) \ + X(GenFramebuffers, PFNGLGENFRAMEBUFFERSPROC) X(DeleteFramebuffers, \ + PFNGLDELETEFRAMEBUFFERSPROC) \ + X(BindFramebuffer, PFNGLBINDFRAMEBUFFERPROC) \ + X(CheckFramebufferStatus, PFNGLCHECKFRAMEBUFFERSTATUSPROC) X( \ + GetFramebufferAttachmentParameteriv, \ + PFNGLGETFRAMEBUFFERATTACHMENTPARAMETERIVPROC) \ + X(FramebufferTexture2D, PFNGLFRAMEBUFFERTEXTURE2DPROC) X( \ + FramebufferTextureLayer, PFNGLFRAMEBUFFERTEXTURELAYERPROC) \ + X(FramebufferRenderbuffer, PFNGLFRAMEBUFFERRENDERBUFFERPROC) \ + X(TexImage2D, PFNGLTEXIMAGE2DPROC) X( \ + TexStorage2D, PFNGLTEXSTORAGE2DPROC) \ + X(RenderbufferStorage, PFNGLRENDERBUFFERSTORAGEPROC) \ + X(RenderbufferStorageMultisample, \ + PFNGLRENDERBUFFERSTORAGEMULTISAMPLEPROC) \ + X(DeleteTextures, PFNGLDELETETEXTURESPROC) \ + X(DeleteRenderbuffers, \ + PFNGLDELETERENDERBUFFERSPROC) \ + X(BlitFramebuffer, \ + PFNGLBLITFRAMEBUFFERPROC) \ + X(Flush, PFNGLFLUSHPROC) +struct Procs { +#define DECLARE(name, type) type name = nullptr; + GL_PROCS(DECLARE) +#undef DECLARE +} gl; +thread_local GlesTransport *current = nullptr; +GlesTransport *Current() { + return current && current->IsCurrent() ? current : nullptr; +} +void GL_APIENTRY GenFramebuffers(GLsizei n, GLuint *ids) { + gl.GenFramebuffers(n, ids); + if (auto *t = Current()) + t->Generated(n, ids); +} +void GL_APIENTRY DeleteFramebuffers(GLsizei n, const GLuint *ids) { + if (auto *t = Current()) + t->Deleted(n, ids); + gl.DeleteFramebuffers(n, ids); +} +void GL_APIENTRY BindFramebuffer(GLenum target, GLuint id) { + gl.BindFramebuffer(target, id); + if (auto *t = Current()) + t->Bind(target, id); +} +void GL_APIENTRY FramebufferTexture2D(GLenum target, GLenum attachment, + GLenum tex_target, GLuint texture, + GLint level) { + gl.FramebufferTexture2D(target, attachment, tex_target, texture, level); + if (auto *t = Current()) + t->AttachmentChanged(target); +} +void GL_APIENTRY FramebufferTextureLayer(GLenum target, GLenum attachment, + GLuint texture, GLint level, + GLint layer) { + gl.FramebufferTextureLayer(target, attachment, texture, level, layer); + if (auto *t = Current()) + t->AttachmentChanged(target); +} +void GL_APIENTRY FramebufferRenderbuffer(GLenum target, GLenum attachment, + GLenum rb_target, GLuint rb) { + gl.FramebufferRenderbuffer(target, attachment, rb_target, rb); + if (auto *t = Current()) + t->AttachmentChanged(target); +} +void GL_APIENTRY TexImage2D(GLenum target, GLint level, GLint internal_format, + GLsizei w, GLsizei h, GLint border, GLenum format, + GLenum type, const void *data) { + gl.TexImage2D(target, level, internal_format, w, h, border, format, type, + data); + if (target == GL_TEXTURE_2D && level == 0) + if (auto *t = Current()) + t->Storage(true, w, h, internal_format); +} +void GL_APIENTRY TexStorage2D(GLenum target, GLsizei levels, GLenum format, + GLsizei w, GLsizei h) { + gl.TexStorage2D(target, levels, format, w, h); + if (target == GL_TEXTURE_2D) + if (auto *t = Current()) + t->Storage(true, w, h, format); +} +void GL_APIENTRY RenderbufferStorage(GLenum target, GLenum format, GLsizei w, + GLsizei h) { + gl.RenderbufferStorage(target, format, w, h); + if (target == GL_RENDERBUFFER) + if (auto *t = Current()) + t->Storage(false, w, h, format); +} +void GL_APIENTRY RenderbufferStorageMultisample(GLenum target, GLsizei samples, + GLenum format, GLsizei w, + GLsizei h) { + gl.RenderbufferStorageMultisample(target, samples, format, w, h); + if (target == GL_RENDERBUFFER) + if (auto *t = Current()) + t->Storage(false, w, h, format, samples); +} +void GL_APIENTRY DeleteTextures(GLsizei n, const GLuint *ids) { + if (auto *t = Current()) + t->ObjectsDeleted(true, n, ids); + gl.DeleteTextures(n, ids); +} +void GL_APIENTRY DeleteRenderbuffers(GLsizei n, const GLuint *ids) { + if (auto *t = Current()) + t->ObjectsDeleted(false, n, ids); + gl.DeleteRenderbuffers(n, ids); +} +} // namespace + +bool GlesTransport::Initialize(void *display, void *config, void *context, + void *egl_get_proc, + void *(*resolve)(const char *), + std::string *error) { + if (!display || !config || !context || !egl_get_proc || !resolve) { + *error = + "OpenGL VR requires a real current EGL context, config and display"; + return false; + } +#define LOAD(name, type) \ + gl.name = reinterpret_cast(resolve("gl" #name)); \ + if (!gl.name) { \ + *error = "OpenGL VR is missing gl" #name; \ + return false; \ + } + GL_PROCS(LOAD) +#undef LOAD + using GetProc = __eglMustCastToProperFunctionPointerType (*)(const char *); + get_current_context_ = reinterpret_cast( + reinterpret_cast(egl_get_proc)("eglGetCurrentContext")); + if (!get_current_context_ || get_current_context_() != context) { + *error = "OpenGL VR must initialize on the current EGL render context"; + return false; + } + GLint major = 0, minor = 0; + gl.GetIntegerv(GL_MAJOR_VERSION, &major); + gl.GetIntegerv(GL_MINOR_VERSION, &minor); + if (major < 3) { + *error = "OpenGL VR requires OpenGL ES 3.0 or newer"; + return false; + } + version_ = (std::uint64_t(major) << 48) | (std::uint64_t(minor) << 32); + display_ = display; + config_ = config; + context_ = context; + egl_get_proc_ = egl_get_proc; + gl.GenFramebuffers(2, copy_fbos_); + if (!copy_fbos_[0] || !copy_fbos_[1]) { + *error = "Cannot allocate OpenGL VR copy framebuffers"; + return false; + } + MakeCurrent(); + return true; +} +bool GlesTransport::IsCurrent() const { + return get_current_context_ && get_current_context_() == context_; +} +void GlesTransport::MakeCurrent() { current = this; } +void GlesTransport::ReleaseCurrent() { current = nullptr; } +void GlesTransport::Destroy() { + if (IsCurrent()) + gl.DeleteFramebuffers(2, copy_fbos_); + copy_fbos_[0] = copy_fbos_[1] = 0; + if (current == this) + current = nullptr; + owners_.clear(); + textures_.clear(); + renderbuffers_.clear(); + ResetEyes(); + context_ = nullptr; +} +void *GlesTransport::Wrap(const char *name, void *raw) { + if (!raw || !name) + return raw; +#define WRAP(name_) \ + if (std::strcmp(name, "gl" #name_) == 0) \ + return reinterpret_cast(&name_); + WRAP(GenFramebuffers) + WRAP(DeleteFramebuffers) + WRAP(BindFramebuffer) WRAP(FramebufferTexture2D) WRAP(FramebufferRenderbuffer) + WRAP(FramebufferTextureLayer) WRAP(TexImage2D) WRAP(TexStorage2D) + WRAP(RenderbufferStorage) WRAP(RenderbufferStorageMultisample) + WRAP(DeleteTextures) WRAP(DeleteRenderbuffers) +#undef WRAP + return raw; +} +void GlesTransport::ResetEyes() { + eyes_[0] = {}; + eyes_[1] = {}; +} +void GlesTransport::Generated(int count, const unsigned *ids) { + void *owner = mocktail_vr_initializing_object(); + if (!owner || !ids) + return; + for (int i = 0; i < count; ++i) + if (ids[i]) + owners_[ids[i]] = owner; +} +void GlesTransport::Deleted(int count, const unsigned *ids) { + if (!ids) + return; + for (int i = 0; i < count; ++i) { + owners_.erase(ids[i]); + for (auto &eye : eyes_) + if (eye.framebuffer == ids[i]) + eye = {}; + } +} +void GlesTransport::ObjectsDeleted(bool texture, int count, + const unsigned *ids) { + if (!ids) + return; + for (int i = 0; i < count; ++i) { + (texture ? textures_ : renderbuffers_).erase(ids[i]); + for (auto &eye : eyes_) + if (eye.texture == texture && eye.object == ids[i]) + eye = {}; + } +} +void GlesTransport::Storage(bool texture, int w, int h, unsigned format, + int samples) { + GLint id = 0; + gl.GetIntegerv(texture ? GL_TEXTURE_BINDING_2D : GL_RENDERBUFFER_BINDING, + &id); + if (!id) + return; + // Storage redefinition invalidates previously rendered eye content. + const unsigned object = static_cast(id); + ObjectsDeleted(texture, 1, &object); + (texture ? textures_ : renderbuffers_)[object] = {w, h, format, samples}; +} +void GlesTransport::AttachmentChanged(unsigned target) { + GLint id = 0; + gl.GetIntegerv(target == GL_READ_FRAMEBUFFER ? GL_READ_FRAMEBUFFER_BINDING + : GL_DRAW_FRAMEBUFFER_BINDING, + &id); + for (auto &eye : eyes_) + if (eye.framebuffer == static_cast(id)) + eye = {}; +} +void GlesTransport::Bind(unsigned target, unsigned framebuffer, + std::uint64_t frame) { + if (!frame) + return; + if (target != GL_FRAMEBUFFER && target != GL_DRAW_FRAMEBUFFER) + return; + auto found = owners_.find(framebuffer); + if (found == owners_.end()) + return; + void *owner = nullptr; + void *guest_fb = nullptr; + int eye = -1; + if (!mocktail_vr_current_eye(&owner, &eye, &guest_fb) || eye < 0 || eye > 1 || + found->second != owner) + return; + if (frame == UINT64_MAX) { + auto *backend = ActiveVrBackend(); + const auto pose = + backend ? backend->PublishedHeadPose() : ScriptedPoseSample{}; + if (!pose.valid) + return; + frame = pose.frame; + } + GLint bound = 0; + gl.GetIntegerv(GL_DRAW_FRAMEBUFFER_BINDING, &bound); + if (static_cast(bound) != framebuffer) + return; + if (gl.CheckFramebufferStatus(GL_DRAW_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE) + return; + GLint type = 0, object = 0, level = 0, face = 0; + gl.GetFramebufferAttachmentParameteriv( + GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_FRAMEBUFFER_ATTACHMENT_OBJECT_TYPE, &type); + if (type != GL_TEXTURE && type != GL_RENDERBUFFER) + return; + gl.GetFramebufferAttachmentParameteriv( + GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_FRAMEBUFFER_ATTACHMENT_OBJECT_NAME, &object); + if (type == GL_TEXTURE) { + gl.GetFramebufferAttachmentParameteriv( + GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_FRAMEBUFFER_ATTACHMENT_TEXTURE_LEVEL, &level); + gl.GetFramebufferAttachmentParameteriv( + GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_FRAMEBUFFER_ATTACHMENT_TEXTURE_CUBE_MAP_FACE, &face); + if (level != 0 || face != 0) + return; + } + const auto &storage = type == GL_TEXTURE ? textures_ : renderbuffers_; + auto it = storage.find(object); + if (it == storage.end()) + return; + const auto &size = it->second; + // DebugDeviceVR currently creates single-sample 500x500 RGBA eyes. + // Reject unsupported formats/extents instead of copying unrelated pixels. + if (size.width != 500 || size.height != 500 || size.samples != 0 || + (size.format != GL_RGBA8 && size.format != GL_RGBA && + size.format != GL_SRGB8_ALPHA8)) + return; + + if ((eyes_[0].owner && eyes_[0].owner != owner) || + (eyes_[1].owner && eyes_[1].owner != owner)) + ResetEyes(); + const bool changed = eyes_[eye].framebuffer != framebuffer; + eyes_[eye] = {framebuffer, + static_cast(object), + type == GL_TEXTURE, + size.width, + size.height, + owner, + frame}; + mocktail_vr_consume_eye(); + if (changed) + std::fprintf( + stderr, + " [vr-backend] GLES eye-bound eye=%d owner=%p guest_fb=%p gl_fb=%u " + "color=%u extent=%dx%d provenance=guest_initializer_and_eye_getter\n", + eye, owner, guest_fb, framebuffer, object, size.width, size.height); +} +bool GlesTransport::Copy(const unsigned textures[2], unsigned width, + unsigned height) { + if (!IsCurrent() || !textures[0] || !textures[1] || !eyes_[0].object || + !eyes_[1].object || + (eyes_[0].texture == eyes_[1].texture && + eyes_[0].object == eyes_[1].object)) + return false; + GLint read_fb = 0, draw_fb = 0; + gl.GetIntegerv(GL_READ_FRAMEBUFFER_BINDING, &read_fb); + gl.GetIntegerv(GL_DRAW_FRAMEBUFFER_BINDING, &draw_fb); + const bool scissor = gl.IsEnabled(GL_SCISSOR_TEST); + gl.Disable(GL_SCISSOR_TEST); + bool copied = true; + for (int i = 0; i < 2; ++i) { + gl.BindFramebuffer(GL_READ_FRAMEBUFFER, copy_fbos_[0]); + if (eyes_[i].texture) + gl.FramebufferTexture2D(GL_READ_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, eyes_[i].object, 0); + else + gl.FramebufferRenderbuffer(GL_READ_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_RENDERBUFFER, eyes_[i].object); + gl.BindFramebuffer(GL_DRAW_FRAMEBUFFER, copy_fbos_[1]); + gl.FramebufferTexture2D(GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, textures[i], 0); + const auto read_status = gl.CheckFramebufferStatus(GL_READ_FRAMEBUFFER); + const auto draw_status = gl.CheckFramebufferStatus(GL_DRAW_FRAMEBUFFER); + if (read_status != GL_FRAMEBUFFER_COMPLETE || + draw_status != GL_FRAMEBUFFER_COMPLETE) { + static bool reported = false; + if (!reported) + std::fprintf(stderr, + " [vr-backend] GLES copy framebuffer incomplete: " + "read=0x%x draw=0x%x XR texture=%u\n", + read_status, draw_status, textures[i]); + reported = true; + copied = false; + break; + } + gl.BlitFramebuffer(0, 0, eyes_[i].width, eyes_[i].height, 0, 0, width, + height, GL_COLOR_BUFFER_BIT, GL_NEAREST); + } + // Release references to runtime textures before releasing XR swapchain + // images. + gl.BindFramebuffer(GL_READ_FRAMEBUFFER, copy_fbos_[0]); + gl.FramebufferRenderbuffer(GL_READ_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_RENDERBUFFER, 0); + gl.BindFramebuffer(GL_DRAW_FRAMEBUFFER, copy_fbos_[1]); + gl.FramebufferTexture2D(GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, 0, 0); + gl.BindFramebuffer(GL_READ_FRAMEBUFFER, read_fb); + gl.BindFramebuffer(GL_DRAW_FRAMEBUFFER, draw_fb); + if (scissor) + gl.Enable(GL_SCISSOR_TEST); + // OpenXR's GL binding synchronizes the flushed work at image release. + gl.Flush(); + return copied; +} +} // namespace mocktail::vr diff --git a/src/vr/gles_transport.h b/src/vr/gles_transport.h new file mode 100644 index 00000000..30fc95db --- /dev/null +++ b/src/vr/gles_transport.h @@ -0,0 +1,69 @@ +#ifndef MOCKTAIL_VR_GLES_TRANSPORT_H_ +#define MOCKTAIL_VR_GLES_TRANSPORT_H_ + +#include +#include +#include + +namespace mocktail::vr { +// One guest EGL context. GL callbacks and copies run on its current thread. +// Only framebuffers created inside the verified DebugDeviceVR initializer +// can become eyes; a live eye-getter request supplies the eye index. +class GlesTransport { +public: + struct Eye { + unsigned framebuffer = 0; + unsigned object = 0; + bool texture = false; + int width = 0, height = 0; + void *owner = nullptr; + std::uint64_t frame = 0; + }; + bool Initialize(void *display, void *config, void *context, + void *egl_get_proc, void *(*resolve)(const char *), + std::string *error); + bool IsCurrent() const; + void Destroy(); // Requires this context current; call before EGL destruction. + void MakeCurrent(); + static void ReleaseCurrent(); + static void *Wrap(const char *name, void *raw); + void ResetEyes(); + bool Copy(const unsigned textures[2], unsigned width, unsigned height); + const Eye &eye(int index) const { return eyes_[index]; } + void *display() const { return display_; } + void *config() const { return config_; } + void *context() const { return context_; } + void *egl_get_proc() const { return egl_get_proc_; } + std::uint64_t version() const { return version_; } + + // Called after the real GL operation, never while the backend mutex is held. + void Generated(int count, const unsigned *ids); + void Deleted(int count, const unsigned *ids); + void Storage(bool texture, int width, int height, unsigned format, + int samples = 0); + void ObjectsDeleted(bool texture, int count, const unsigned *ids); + void AttachmentChanged(unsigned target); + void Bind(unsigned target, unsigned framebuffer, + std::uint64_t frame = UINT64_MAX); + +private: + friend struct GlesTransportTestAccess; + struct StorageInfo { + int width = 0, height = 0; + unsigned format = 0; + int samples = 0; + }; + void *display_ = nullptr; + void *config_ = nullptr; + void *context_ = nullptr; + void *egl_get_proc_ = nullptr; + std::uint64_t version_ = 0; + void *(*get_current_context_)() = nullptr; + unsigned copy_fbos_[2] = {}; + std::unordered_map owners_; + std::unordered_map textures_; + std::unordered_map renderbuffers_; + Eye eyes_[2]; +}; +} // namespace mocktail::vr +#endif diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 89a3b845..e1c8bf78 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -2,10 +2,15 @@ #include "mocktail/vr/vr_perf.h" #include "mocktail/vr/vr_pose_math.h" +#include "vr/gles_transport.h" +#include +#include + #include #include #include +#include #include #include @@ -219,7 +224,7 @@ OpenXrBackend* ActiveVrBackend() { return g_backend; } -Status OpenXrBackend::Arm() { +Status OpenXrBackend::Arm(VrGraphicsApi graphics_api) { std::lock_guard lock(mutex_); if (armed_.load(std::memory_order_acquire)) { return Status::Ok(); @@ -234,36 +239,74 @@ Status OpenXrBackend::Arm() { } try { const std::string user_manifest = EnvString("XR_RUNTIME_JSON", ""); - const auto manifest = SelectVrRuntimeManifest(user_manifest, - {"/usr/local/share/openxr/1/openxr_wivrn.json", - "/usr/share/openxr/1/openxr_wivrn.json"}); + const auto preference = EnvString("MOCKTAIL_VR_RUNTIME", "auto"); + Require( + preference == "auto" || preference == "system" || + preference == "wivrn" || preference == "steamvr" || + preference == "alvr", + "MOCKTAIL_VR_RUNTIME must be auto, system, wivrn, steamvr, or alvr"); + const auto discovery = + user_manifest.empty() + ? DiscoverVrRuntime( + preference, EnvString("HOME", ""), + EnvString("XDG_CONFIG_HOME", ""), + EnvString("XDG_CONFIG_DIRS", ""), + EnvString("XDG_RUNTIME_DIR", + ("/run/user/" + std::to_string(getuid())).c_str())) + : VrRuntimeDiscovery{{}, "explicit XR_RUNTIME_JSON override"}; + const auto manifest = + SelectVrRuntimeManifest(user_manifest, discovery.candidates); + Log(" [vr-backend] runtime selection: %s\n", discovery.reason.c_str()); + Require(!manifest.empty() || + (preference != "steamvr" && preference != "alvr" && + preference != "wivrn"), + "Selected VR runtime was not found. For ALVR, install SteamVR and " + "set XR_RUNTIME_JSON " + "to its steamxr_linux64.json if automatic discovery cannot find " + "the installation. " + "For WiVRn, install its native runtime or select its manifest " + "explicitly."); if (!manifest.empty()) { Require(setenv("XR_RUNTIME_JSON", manifest.c_str(), 1) == 0, "Cannot select the OpenXR runtime"); Log(" [vr-backend] runtime manifest: %s\n", manifest.c_str()); } else { - Log(" [vr-backend] using registered OpenXR runtime; connect the headset in WiVRn first\n"); + Log(" [vr-backend] using registered OpenXR runtime; connect the headset " + "in WiVRn or SteamVR/ALVR first\n"); } - vk_loader_ = dlopen("libvulkan.so.1", RTLD_NOW | RTLD_LOCAL); - Require(vk_loader_ != nullptr, - "Cannot load the host Vulkan loader libvulkan.so.1"); - auto loader_gipa = reinterpret_cast( - dlsym(vk_loader_, "vkGetInstanceProcAddr")); - Require(loader_gipa != nullptr, "Host vkGetInstanceProcAddr is missing"); - - const char* extension = XR_KHR_VULKAN_ENABLE2_EXTENSION_NAME; + graphics_api_ = graphics_api; + if (graphics_api_ == VrGraphicsApi::kVulkan) { + vk_loader_ = dlopen("libvulkan.so.1", RTLD_NOW | RTLD_LOCAL); + Require(vk_loader_ != nullptr, + "Cannot load the host Vulkan loader libvulkan.so.1"); + auto loader_gipa = reinterpret_cast( + dlsym(vk_loader_, "vkGetInstanceProcAddr")); + Require(loader_gipa != nullptr, "Host vkGetInstanceProcAddr is missing"); + } + const char *vk_extensions[] = {XR_KHR_VULKAN_ENABLE2_EXTENSION_NAME}; + const char *gl_extensions[] = {XR_KHR_OPENGL_ES_ENABLE_EXTENSION_NAME, + XR_MNDX_EGL_ENABLE_EXTENSION_NAME}; auto info = XrInfo(XR_TYPE_INSTANCE_CREATE_INFO); std::strcpy(info.applicationInfo.applicationName, "Mocktail Roblox VR"); std::strcpy(info.applicationInfo.engineName, "Mocktail"); info.applicationInfo.apiVersion = XR_MAKE_VERSION(1, 0, 0); - info.enabledExtensionCount = 1; - info.enabledExtensionNames = &extension; + info.enabledExtensionCount = + graphics_api_ == VrGraphicsApi::kVulkan ? 1 : 2; + info.enabledExtensionNames = + graphics_api_ == VrGraphicsApi::kVulkan ? vk_extensions : gl_extensions; XrInstance instance = XR_NULL_HANDLE; const XrResult instance_result = xrCreateInstance(&info, &instance); if (instance_result == XR_ERROR_RUNTIME_UNAVAILABLE) { throw std::runtime_error("xrCreateInstance: " + VrRuntimeUnavailableHint(!user_manifest.empty(), manifest)); } + if (instance_result == XR_ERROR_EXTENSION_NOT_PRESENT && + graphics_api_ == VrGraphicsApi::kOpenGles) { + throw std::runtime_error( + "OpenGL VR requires runtime extensions XR_KHR_opengl_es_enable and " + "XR_MNDX_egl_enable; select direct-vulkan if the runtime does not " + "support EGL."); + } CheckXr(XR_NULL_HANDLE, instance_result, "xrCreateInstance"); xr_instance_ = instance; @@ -287,7 +330,8 @@ Status OpenXrBackend::Arm() { // state while the server runs with no headset connected. throw std::runtime_error( "The OpenXR runtime is reachable but reports no connected headset " - "(XR_ERROR_FORM_FACTOR_UNAVAILABLE). With WiVRn: start the server " + "(XR_ERROR_FORM_FACTOR_UNAVAILABLE). Start WiVRn or SteamVR with " + "ALVR " "and connect the headset first; VR applications can only run while " "the headset connection is established."); } @@ -295,23 +339,36 @@ Status OpenXrBackend::Arm() { Require(xr_system_ != XR_NULL_SYSTEM_ID, "The OpenXR runtime exposes no head-mounted display system"); - auto graphics_requirements = - XrInfo( - XR_TYPE_GRAPHICS_REQUIREMENTS_VULKAN2_KHR); - auto get_requirements = reinterpret_cast( - ResolveFunction("xrGetVulkanGraphicsRequirements2KHR")); - CheckXr(instance, - get_requirements(instance, xr_system_, &graphics_requirements), - "xrGetVulkanGraphicsRequirements2KHR"); - Log(" [vr-backend] runtime Vulkan requirement: %u.%u - %u.%u\n", - XR_VERSION_MAJOR(graphics_requirements.minApiVersionSupported), - XR_VERSION_MINOR(graphics_requirements.minApiVersionSupported), - XR_VERSION_MAJOR(graphics_requirements.maxApiVersionSupported), - XR_VERSION_MINOR(graphics_requirements.maxApiVersionSupported)); + if (graphics_api_ == VrGraphicsApi::kVulkan) { + auto graphics_requirements = XrInfo( + XR_TYPE_GRAPHICS_REQUIREMENTS_VULKAN2_KHR); + auto get_requirements = + reinterpret_cast( + ResolveFunction("xrGetVulkanGraphicsRequirements2KHR")); + CheckXr(instance, + get_requirements(instance, xr_system_, &graphics_requirements), + "xrGetVulkanGraphicsRequirements2KHR"); + Log(" [vr-backend] runtime Vulkan requirement: %u.%u - %u.%u\n", + XR_VERSION_MAJOR(graphics_requirements.minApiVersionSupported), + XR_VERSION_MINOR(graphics_requirements.minApiVersionSupported), + XR_VERSION_MAJOR(graphics_requirements.maxApiVersionSupported), + XR_VERSION_MINOR(graphics_requirements.maxApiVersionSupported)); + } else { + auto requirements = XrInfo( + XR_TYPE_GRAPHICS_REQUIREMENTS_OPENGL_ES_KHR); + auto get = reinterpret_cast( + ResolveFunction("xrGetOpenGLESGraphicsRequirementsKHR")); + CheckXr(instance, get(instance, xr_system_, &requirements), + "xrGetOpenGLESGraphicsRequirementsKHR"); + gl_min_version_ = requirements.minApiVersionSupported; + gl_max_version_ = requirements.maxApiVersionSupported; + } armed_.store(true, std::memory_order_release); - Log(" [vr-backend] armed: OpenXR instance/system ready; awaiting the " - "guest Vulkan device\n"); + Log(" [vr-backend] armed: OpenXR instance/system ready; awaiting guest " + "%s\n", + graphics_api_ == VrGraphicsApi::kVulkan ? "Vulkan device" + : "OpenGL ES / EGL context"); return Status::Ok(); } catch (const std::exception& error) { DisarmLocked(); @@ -326,6 +383,10 @@ void OpenXrBackend::Disarm() { void OpenXrBackend::DisarmLocked() { TeardownSessionLocked("disarm"); + if (gles_) + gles_->Destroy(); + delete gles_; + gles_ = nullptr; if (xr_instance_ != nullptr) { const auto instance = static_cast(xr_instance_); (void)xrDestroyInstance(instance); @@ -611,6 +672,141 @@ bool OpenXrBackend::InitializeSessionLocked( "xrCreateSession"); xr_session_ = session; + return InitializeSessionResourcesLocked(error); + } catch (const std::exception &exception) { + *error = exception.what(); + TeardownSessionLocked("session init failure", session_recovery_pending_); + return false; + } +} + +Status OpenXrBackend::AttachGlesContext(void *display, void *config, + void *context, void *egl_get_proc, + void *(*resolve)(const char *)) { + std::lock_guard lock(mutex_); + if (!armed_ || graphics_api_ != VrGraphicsApi::kOpenGles) + return Status::Ok(); + try { + if (gles_) { + Require(gles_->context() == context && gles_->display() == display && + gles_->IsCurrent(), + "OpenGL VR cannot switch to a different live EGL context"); + gles_->MakeCurrent(); + return Status::Ok(); + } + gles_ = new GlesTransport; + std::string error; + Require(gles_->Initialize(display, config, context, egl_get_proc, resolve, + &error), + error); + Require(gles_->version() >= gl_min_version_ && + gles_->version() <= gl_max_version_, + "EGL context version is outside the OpenXR runtime's OpenGL ES " + "requirements"); + Require(InitializeGlesSessionLocked(&error), error); + Log(" [vr-backend] bound OpenGL ES context to OpenXR; GPU eye copies " + "enabled, pose_source=%s\n", + EnvString("MOCKTAIL_VR_POSE_SOURCE", "runtime").c_str()); + return Status::Ok(); + } catch (const std::exception &error) { + TeardownSessionLocked("EGL binding failed"); + if (gles_) + gles_->Destroy(); + delete gles_; + gles_ = nullptr; + return Status::Error(StatusCode::kUnavailable, error.what()); + } +} + +bool OpenXrBackend::InitializeGlesSessionLocked(std::string *error) { + try { + Require(gles_ && gles_->IsCurrent(), + "OpenXR EGL session creation requires the current guest context"); + auto binding = + XrInfo(XR_TYPE_GRAPHICS_BINDING_EGL_MNDX); + binding.display = gles_->display(); + binding.config = gles_->config(); + binding.context = gles_->context(); + binding.getProcAddress = + reinterpret_cast(gles_->egl_get_proc()); + auto info = XrInfo(XR_TYPE_SESSION_CREATE_INFO); + info.next = &binding; + info.systemId = xr_system_; + XrSession session = XR_NULL_HANDLE; + CheckXr( + static_cast(xr_instance_), + xrCreateSession(static_cast(xr_instance_), &info, &session), + "xrCreateSession EGL"); + xr_session_ = session; + return InitializeSessionResourcesLocked(error); + } catch (const std::exception &exception) { + *error = exception.what(); + TeardownSessionLocked("EGL session init failure", + session_recovery_pending_); + return false; + } +} + +void OpenXrBackend::DetachGlesContext(void *context) { + std::lock_guard lock(mutex_); + if (!gles_ || gles_->context() != context) + return; + TeardownSessionLocked("EGL context destruction"); + gles_->Destroy(); + delete gles_; + gles_ = nullptr; +} + +void *OpenXrBackend::WrapGlesProcAddress(const char *name, void *raw) { + if (!armed_ || graphics_api_ != VrGraphicsApi::kOpenGles || !gles_) + return raw; + return GlesTransport::Wrap(name, raw); +} + +void OpenXrBackend::NoteGlesPresent() { + { + std::lock_guard lock(mutex_); + if (!armed_ || graphics_api_ != VrGraphicsApi::kOpenGles || !gles_ || + !gles_->IsCurrent()) + return; + for (int i = 0; i < 2; ++i) { + const auto &eye = gles_->eye(i); + eyes_[i].valid = eye.framebuffer != 0; + eyes_[i].frame = eye.frame; + eyes_[i].owner = eye.owner; + eyes_[i].width = eye.width; + eyes_[i].height = eye.height; + } + const auto &left = gles_->eye(0); + const auto &right = gles_->eye(1); + eyes_bound_ = + left.framebuffer && right.framebuffer && + left.framebuffer != right.framebuffer && left.owner == right.owner && + (left.texture != right.texture || left.object != right.object); + } + NoteHostPresent(VK_NULL_HANDLE, VK_NULL_HANDLE); +} + +bool OpenXrBackend::CopyGlesEyesIntoSwapchains() { + if (!gles_) + return false; + unsigned textures[2] = {}; + for (int i = 0; i < 2; ++i) { + const auto &slot = eye_swapchains_[i]; + if (!slot.image_acquired || slot.acquired_index >= slot.gl_images.size()) + return false; + textures[i] = slot.gl_images[slot.acquired_index]; + } + Require(gles_->Copy(textures, eye_swapchains_[0].width, + eye_swapchains_[0].height), + "Cannot copy GLES eyes into runtime textures; verify EGL and OpenXR " + "use the same GPU and support image sharing"); + return true; +} + +bool OpenXrBackend::InitializeSessionResourcesLocked(std::string *error) { + try { + const auto session = static_cast(xr_session_); auto space_info = XrInfo( XR_TYPE_REFERENCE_SPACE_CREATE_INFO); space_info.poseInReferenceSpace.orientation.w = 1; @@ -670,13 +866,12 @@ bool OpenXrBackend::InitializeSessionLocked( } recording_.store(true, std::memory_order_release); return true; - } catch (const std::exception& exception) { + } catch (const std::exception &exception) { *error = exception.what(); TeardownSessionLocked("session init failure", session_recovery_pending_); return false; } } - bool OpenXrBackend::CreateSwapchains(std::string* error) { try { const auto session = static_cast(xr_session_); @@ -694,10 +889,15 @@ bool OpenXrBackend::CreateSwapchains(std::string* error) { // The swapchain format must accept the guest eye content. Prefer the // common 8-bit formats; the final choice is cross-checked against the // recorded eye images before the first copy (blit converts if needed). - VkFormat chosen = VK_FORMAT_UNDEFINED; - for (const auto candidate : - {VK_FORMAT_B8G8R8A8_UNORM, VK_FORMAT_R8G8B8A8_UNORM, - VK_FORMAT_B8G8R8A8_SRGB, VK_FORMAT_R8G8B8A8_SRGB}) { + std::int64_t chosen = 0; + const std::vector candidates = + graphics_api_ == VrGraphicsApi::kVulkan + ? std::vector{VK_FORMAT_B8G8R8A8_UNORM, + VK_FORMAT_R8G8B8A8_UNORM, + VK_FORMAT_B8G8R8A8_SRGB, + VK_FORMAT_R8G8B8A8_SRGB} + : std::vector{GL_RGBA8, GL_SRGB8_ALPHA8}; + for (const auto candidate : candidates) { if (std::find(formats.begin(), formats.end(), static_cast(candidate)) != formats.end()) { chosen = candidate; @@ -730,30 +930,48 @@ bool OpenXrBackend::CreateSwapchains(std::string* error) { CheckXr(static_cast(xr_instance_), xrCreateSwapchain(session, &create_info, &swapchain), "xrCreateSwapchain"); + EyeSwapchain &slot = eye_swapchains_[eye]; + slot.swapchain = + swapchain; // Own immediately, including enumeration failures. std::uint32_t image_count = 0; CheckXr(static_cast(xr_instance_), xrEnumerateSwapchainImages(swapchain, 0, &image_count, nullptr), "xrEnumerateSwapchainImages count"); Require(image_count > 0 && image_count <= 16, "the runtime returned an empty swapchain"); - std::vector images(image_count); - for (auto& image : images) { - image.type = XR_TYPE_SWAPCHAIN_IMAGE_VULKAN_KHR; - } - CheckXr(static_cast(xr_instance_), - xrEnumerateSwapchainImages( - swapchain, image_count, &image_count, - reinterpret_cast(images.data())), - "xrEnumerateSwapchainImages"); - EyeSwapchain& slot = eye_swapchains_[eye]; - slot.swapchain = swapchain; - slot.images.clear(); - for (const auto& image : images) { - slot.images.push_back(image.image); + if (graphics_api_ == VrGraphicsApi::kVulkan) { + std::vector images(image_count); + for (auto &image : images) { + image.type = XR_TYPE_SWAPCHAIN_IMAGE_VULKAN_KHR; + } + CheckXr( + static_cast(xr_instance_), + xrEnumerateSwapchainImages( + swapchain, image_count, &image_count, + reinterpret_cast(images.data())), + "xrEnumerateSwapchainImages"); + slot.images.clear(); + for (const auto &image : images) { + slot.images.push_back(image.image); + } + } else { + std::vector images(image_count); + for (auto &image : images) + image.type = XR_TYPE_SWAPCHAIN_IMAGE_OPENGL_ES_KHR; + CheckXr( + static_cast(xr_instance_), + xrEnumerateSwapchainImages( + swapchain, image_count, &image_count, + reinterpret_cast(images.data())), + "xrEnumerateSwapchainImages GLES"); + slot.gl_images.clear(); + for (const auto &image : images) { + slot.gl_images.push_back(image.image); + } } slot.width = width; slot.height = height; - slot.format = chosen; + slot.format = static_cast(chosen); Log(" [vr-backend] eye%d swapchain: %ux%u format=%d images=%u\n", eye, width, height, static_cast(chosen), image_count); } @@ -767,6 +985,7 @@ bool OpenXrBackend::CreateSwapchains(std::string* error) { void OpenXrBackend::DestroySwapchains() { for (EyeSwapchain& slot : eye_swapchains_) { slot.images.clear(); + slot.gl_images.clear(); if (slot.swapchain != nullptr) { (void)xrDestroySwapchain(static_cast(slot.swapchain)); slot.swapchain = nullptr; @@ -817,6 +1036,8 @@ void OpenXrBackend::TeardownSessionLocked(const char* reason, bool preserve_devi canted_rejection_logged_ = false; eyes_[0] = {}; eyes_[1] = {}; + if (gles_) + gles_->ResetEyes(); applied_poses_.clear(); published_pose_.valid = false; if (preserve_device) { @@ -1264,12 +1485,19 @@ void OpenXrBackend::HandleReferenceSpaceChange(std::int64_t change_time) { } bool OpenXrBackend::RecoverSessionLocked(std::uint64_t now_ns) { - if (!session_recovery_pending_ || vk_device_ == VK_NULL_HANDLE || vk_ == nullptr || - xr_instance_ == nullptr || now_ns < next_session_retry_ns_) return false; + if (!session_recovery_pending_ || + (graphics_api_ == VrGraphicsApi::kVulkan + ? (vk_device_ == VK_NULL_HANDLE || vk_ == nullptr) + : gles_ == nullptr) || + xr_instance_ == nullptr || now_ns < next_session_retry_ns_) + return false; next_session_retry_ns_ = now_ns + 1'000'000'000ULL; std::string error; // nullptr means reuse the verified queue family of this existing VkDevice. - if (!InitializeSessionLocked(vk_physical_device_, vk_device_, nullptr, &error)) { + if (!(graphics_api_ == VrGraphicsApi::kVulkan + ? InitializeSessionLocked(vk_physical_device_, vk_device_, nullptr, + &error) + : InitializeGlesSessionLocked(&error))) { Log(" [vr-backend] waiting to restore XR session: %s\n", error.c_str()); return false; } @@ -1400,10 +1628,13 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { } const bool copied = [&] { const ScopedStageTimer copy_record(perf::Stage::kCopyRecord); - return CopyEyesIntoSwapchains(queue); + return graphics_api_ == VrGraphicsApi::kVulkan + ? CopyEyesIntoSwapchains(queue) + : CopyGlesEyesIntoSwapchains(); }() && [&] { const ScopedStageTimer evidence_timer(perf::Stage::kEvidence); - return CaptureEyeEvidence(queue, 0) && CaptureEyeEvidence(queue, 1); + return graphics_api_ == VrGraphicsApi::kOpenGles || + (CaptureEyeEvidence(queue, 0) && CaptureEyeEvidence(queue, 1)); }(); for (int eye = 0; eye < 2; ++eye) { if (images_held[eye]) { @@ -1454,14 +1685,23 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { (!EnvString("MOCKTAIL_VR_XR_EVIDENCE_DIR", "").empty() && published_pose_.frame % std::max(1, EnvUnsigned("MOCKTAIL_VR_XR_EVIDENCE_INTERVAL", 20)) == 0)) { - Log(" [vr-backend][evidence] projection submitted=%llu frame=%llu eye0_image=%p " - "eye1_image=%p eye0_swapchain_index=%u eye1_swapchain_index=%u\n", - static_cast(submitted), - static_cast(published_pose_.frame), - static_cast(eyes_[0].image), - static_cast(eyes_[1].image), - eye_swapchains_[0].acquired_index, - eye_swapchains_[1].acquired_index); + if (graphics_api_ == VrGraphicsApi::kOpenGles) { + Log(" [vr-backend][evidence] projection submitted=%llu frame=%llu " + "graphics=GLES eye0_texture=%u eye1_texture=%u\n", + static_cast(submitted), + static_cast(published_pose_.frame), + gles_->eye(0).object, gles_->eye(1).object); + } else { + Log(" [vr-backend][evidence] projection submitted=%llu frame=%llu " + "eye0_image=%p " + "eye1_image=%p eye0_swapchain_index=%u eye1_swapchain_index=%u\n", + static_cast(submitted), + static_cast(published_pose_.frame), + static_cast(eyes_[0].image), + static_cast(eyes_[1].image), + eye_swapchains_[0].acquired_index, + eye_swapchains_[1].acquired_index); + } } } else { perf::ProcessCollector().Count(perf::Counter::kFramesSkipped); @@ -2192,3 +2432,32 @@ void mocktail_vr_xr_note_present(VkQueue queue, VkDevice device) { } } // extern "C" + +extern "C" bool mocktail_vr_attach_egl(void *display, void *config, + void *context, void *egl_get_proc, + void *(*resolve)(const char *)) { + if (auto *backend = mocktail::vr::ActiveVrBackend()) { + const auto status = backend->AttachGlesContext(display, config, context, + egl_get_proc, resolve); + if (!status.ok()) + std::fprintf(stderr, " [vr-backend] %s\n", status.message().c_str()); + return status.ok(); + } + return true; +} +extern "C" void mocktail_vr_detach_egl(void *context) { + if (auto *backend = mocktail::vr::ActiveVrBackend()) + backend->DetachGlesContext(context); +} +extern "C" void mocktail_vr_release_egl() { + mocktail::vr::GlesTransport::ReleaseCurrent(); +} +extern "C" void *mocktail_vr_wrap_gles_proc(const char *name, void *raw) { + if (auto *backend = mocktail::vr::ActiveVrBackend()) + return backend->WrapGlesProcAddress(name, raw); + return raw; +} +extern "C" void mocktail_vr_gles_present() { + if (auto *backend = mocktail::vr::ActiveVrBackend()) + backend->NoteGlesPresent(); +} diff --git a/src/vr/openxr_backend_mode.cc b/src/vr/openxr_backend_mode.cc index c2764ce5..007a831b 100644 --- a/src/vr/openxr_backend_mode.cc +++ b/src/vr/openxr_backend_mode.cc @@ -1,36 +1,253 @@ #include "mocktail/vr/openxr_backend.h" +#include +#include #include #include +#include +#include +#include +#include +#define JSON_NOEXCEPTION 1 +#include namespace mocktail::vr { -std::string SelectVrRuntimeManifest(const std::string& explicit_manifest, - const std::vector& candidates) { - if (!explicit_manifest.empty()) return explicit_manifest; - for (const auto& path : candidates) { +std::string +SelectVrRuntimeManifest(const std::string &explicit_manifest, + const std::vector &candidates) { + if (!explicit_manifest.empty()) + return explicit_manifest; + for (const auto &path : candidates) { std::error_code error; - if (std::filesystem::is_regular_file(path, error)) return path; + if (std::filesystem::is_regular_file(path, error)) + return path; } return {}; } -std::string VrRuntimeUnavailableHint(bool user_selected, const std::string& manifest) { +std::vector VrRuntimeManifestCandidates( + const std::string &preference, const std::string &home, + const std::string &config_home, const std::string &config_dirs) { + if (preference == "steamvr" || preference == "alvr") { + if (home.empty()) + return {}; + std::vector paths; + const auto config = config_home.empty() ? home + "/.config" : config_home; + const auto registry = config + "/openvr/openvrpaths.vrpath"; + std::error_code error; + if (std::filesystem::file_size(registry, error) <= 1024 * 1024 && !error) { + std::ifstream input(registry); + const auto json = nlohmann::json::parse(input, nullptr, false); + if (json.is_object() && json.contains("runtime") && + json["runtime"].is_array()) { + for (const auto &item : json["runtime"]) { + if (item.is_string()) { + const auto root = std::filesystem::path(item.get()); + if (root.is_absolute()) + paths.push_back((root / "steamxr_linux64.json").string()); + } + } + } + } + const std::vector standard = { + home + + "/.local/share/Steam/steamapps/common/SteamVR/steamxr_linux64.json", + home + "/.steam/steam/steamapps/common/SteamVR/steamxr_linux64.json", + home + "/.steam/debian-installation/steamapps/common/SteamVR/" + "steamxr_linux64.json"}; + paths.insert(paths.end(), standard.begin(), standard.end()); + return paths; + } + const std::vector wivrn = { + "/usr/local/share/openxr/1/openxr_wivrn.json", + "/usr/share/openxr/1/openxr_wivrn.json"}; + if (preference == "wivrn") + return wivrn; + if (preference == "system") + return {}; // Use the loader's full discovery rules. + std::vector candidates; + const auto add_config = [&](const std::string &path) { + if (path.empty() || path[0] != '/') + return; + candidates.push_back(path + "/openxr/1/active_runtime.x86_64.json"); + candidates.push_back(path + "/openxr/1/active_runtime.json"); + }; + add_config(!config_home.empty() ? config_home + : (home.empty() ? "" : home + "/.config")); + std::istringstream directories(config_dirs.empty() ? "/etc/xdg" + : config_dirs); + std::string directory; + while (std::getline(directories, directory, ':')) + add_config(directory); + add_config("/etc"); + candidates.insert(candidates.end(), wivrn.begin(), wivrn.end()); + return candidates; +} + +VrRuntimeDiscovery DiscoverVrRuntime(const std::string &preference, + const std::string &home, + const std::string &config_home, + const std::string &config_dirs, + const std::string &runtime_dir, + const std::string &proc_root) { + VrRuntimeDiscovery result; + auto steam = + VrRuntimeManifestCandidates("steamvr", home, config_home, config_dirs); + auto wivrn = + VrRuntimeManifestCandidates("wivrn", home, config_home, config_dirs); + bool steam_running = false; + bool wivrn_listening = false; + // Read the kernel's socket table: a leftover socket pathname alone is not + // evidence of a listening server. Do not connect and create a dummy XR + // client. + const auto ipc = (std::filesystem::path(runtime_dir) / "wivrn/comp_ipc") + .lexically_normal(); + if (!runtime_dir.empty()) { + std::ifstream sockets(std::filesystem::path(proc_root) / "net/unix"); + std::string line; + while (std::getline(sockets, line)) { + std::istringstream row(line); + std::string num, refs, protocol, flags, type, state, inode, path; + if (!(row >> num >> refs >> protocol >> flags >> type >> state >> inode)) + continue; + std::getline(row >> std::ws, path); + if (flags == "00010000" && type == "0001" && + std::filesystem::path(path).lexically_normal() == ipc) { + struct stat socket_info{}; + if (stat(path.c_str(), &socket_info) == 0 && + S_ISSOCK(socket_info.st_mode) && socket_info.st_uid == getuid()) + wivrn_listening = true; + } + } + } + // Restrict process discovery to this user. In particular, never select a + // root server for a desktop client or another user's SteamVR installation. + std::error_code error; + std::filesystem::directory_iterator processes(proc_root, error); + for (; !error && processes != std::filesystem::directory_iterator(); + processes.increment(error)) { + const auto name = processes->path().filename().string(); + if (name.empty() || !std::all_of(name.begin(), name.end(), [](char c) { + return c >= '0' && c <= '9'; + })) + continue; + struct stat metadata{}; + if (stat(processes->path().c_str(), &metadata) != 0 || + metadata.st_uid != getuid()) + continue; + std::ifstream comm_file(processes->path() / "comm"); + std::string comm; + std::getline(comm_file, comm); + if (comm != "vrserver" && comm != "wivrn-server") + continue; + std::error_code exe_error; + const auto executable = + std::filesystem::read_symlink(processes->path() / "exe", exe_error); + if (exe_error || !executable.is_absolute() || executable.filename() != comm) + continue; + if (comm == "vrserver") { + steam_running = true; + steam.insert(steam.begin(), + (executable.parent_path().parent_path().parent_path() / + "steamxr_linux64.json") + .string()); + } else { + // Native developer builds keep this manifest at their build root. + auto directory = executable.parent_path(); + for (int i = 0; i < 3 && directory.has_relative_path(); + ++i, directory = directory.parent_path()) { + wivrn.insert(wivrn.begin(), + (directory / "openxr_wivrn-dev.json").string()); + } + } + } + if (preference == "steamvr" || preference == "alvr") { + result.candidates = steam; + result.reason = "explicit SteamVR/ALVR preference"; + } else if (preference == "wivrn") { + result.candidates = wivrn; + result.reason = "explicit WiVRn preference"; + } else if (preference == "system") { + result.reason = "system OpenXR loader selection"; + } else { + const auto registered = + VrRuntimeManifestCandidates("auto", home, config_home, config_dirs); + if (steam_running != wivrn_listening) { + const std::string runtime_tag = steam_running ? "steamvr" : "wivrn"; + for (const auto &path : registered) { + // Only registered entries, not native fallback paths. + if (std::filesystem::path(path).filename().string().find( + "active_runtime") != 0) + continue; + std::error_code manifest_error; + if (std::filesystem::file_size(path, manifest_error) > 1024 * 1024 || + manifest_error) + continue; + auto identity = + std::filesystem::weakly_canonical(path, manifest_error).string(); + std::ifstream input(path); + const auto json = nlohmann::json::parse(input, nullptr, false); + if (json.is_object() && json.contains("runtime") && + json["runtime"].is_object()) { + for (const auto *key : {"name", "library_path"}) { + const auto &runtime = json["runtime"]; + if (runtime.contains(key) && runtime[key].is_string()) + identity += " " + runtime[key].get(); + } + } + std::transform(identity.begin(), identity.end(), identity.begin(), + [](unsigned char c) { return std::tolower(c); }); + if (identity.find(runtime_tag) != std::string::npos) + result.candidates.push_back(path); + } + const auto &active = steam_running ? steam : wivrn; + result.candidates.insert(result.candidates.end(), active.begin(), + active.end()); + result.reason = steam_running + ? "running SteamVR detected (including ALVR)" + : "listening WiVRn server detected"; + } else { + result.reason = + steam_running + ? "both servers detected; registered runtime has priority" + : "no single running server detected; registered runtime has " + "priority"; + } + result.candidates.insert(result.candidates.end(), registered.begin(), + registered.end()); + result.candidates.insert(result.candidates.end(), steam.begin(), + steam.end()); + result.candidates.insert(result.candidates.end(), wivrn.begin(), + wivrn.end()); + } + return result; +} + +std::string VrRuntimeUnavailableHint(bool user_selected, + const std::string &manifest) { if (user_selected) { return "User-selected XR_RUNTIME_JSON '" + manifest + - "' is unavailable. Check the manifest/library and that its runtime server " - "is running with a connected headset; this error alone does not distinguish them."; + "' is unavailable. Check the manifest/library and that its runtime " + "server " + "is running with a connected headset; this error alone does not " + "distinguish them."; } if (!manifest.empty()) { - return "Auto-selected WiVRn runtime is unavailable. Start WiVRn and connect " - "the headset, then retry. If it remains unavailable, check the runtime installation."; + return "Selected OpenXR runtime '" + manifest + + "' is unavailable. Start its server " + "(WiVRn, or SteamVR with ALVR) and connect the headset. Run the " + "server and " + "Mocktail as the same desktop user in the same runtime environment."; } - return "The registered OpenXR runtime is unavailable. Install/start WiVRn and " - "connect the headset, or explicitly select an installed runtime with XR_RUNTIME_JSON."; + return "The registered OpenXR runtime is unavailable. Start WiVRn or SteamVR " + "with ALVR and " + "connect the headset, or explicitly select an installed runtime with " + "XR_RUNTIME_JSON."; } VrBackendMode ResolveVrBackendMode(bool vr_enabled) { if (!vr_enabled) { return VrBackendMode::kDisabled; } - const char* value = std::getenv("MOCKTAIL_VR_BACKEND"); + const char *value = std::getenv("MOCKTAIL_VR_BACKEND"); const std::string mode = value ? value : "xr"; if (mode == "native" || mode == "native-stereo") { return VrBackendMode::kNativeOnly; @@ -38,16 +255,16 @@ VrBackendMode ResolveVrBackendMode(bool vr_enabled) { return VrBackendMode::kXrOutput; } -const char* VrBackendModeName(VrBackendMode mode) { +const char *VrBackendModeName(VrBackendMode mode) { switch (mode) { - case VrBackendMode::kDisabled: - return "disabled"; - case VrBackendMode::kNativeOnly: - return "native-stereo-diagnostic"; - case VrBackendMode::kXrOutput: - return "openxr-output"; + case VrBackendMode::kDisabled: + return "disabled"; + case VrBackendMode::kNativeOnly: + return "native-stereo-diagnostic"; + case VrBackendMode::kXrOutput: + return "openxr-output"; } return "disabled"; } -} +} // namespace mocktail::vr diff --git a/src/vr/openxr_disabled.cc b/src/vr/openxr_disabled.cc index f8dc382d..3c75de6d 100644 --- a/src/vr/openxr_disabled.cc +++ b/src/vr/openxr_disabled.cc @@ -4,7 +4,7 @@ namespace mocktail::vr { OpenXrBackend::~OpenXrBackend() = default; -Status OpenXrBackend::Arm() { +Status OpenXrBackend::Arm(VrGraphicsApi) { return Status::Error(StatusCode::kUnavailable, "OpenXR support is disabled in this build"); } void OpenXrBackend::Disarm() {} diff --git a/src/window/window.cc b/src/window/window.cc index c393edd1..34abd807 100644 --- a/src/window/window.cc +++ b/src/window/window.cc @@ -40,6 +40,9 @@ namespace mocktail { namespace window { +namespace { +bool AttachVrEgl(); +} // Local EGL declarations avoid conflicts with the stub headers. @@ -1309,6 +1312,16 @@ bool Init(int width, int height, const char* title) { } return CreateSoftwareWaitingWindow(width, height, title); } + if (!AttachVrEgl()) { + SDL_GL_DestroyContext(sdl_context); + SDL_DestroyWindow(g_state.sdl_window); + g_state.sdl_window = nullptr; + g_state.egl_context = nullptr; + g_state.egl_display = nullptr; + g_state.egl_config = nullptr; + g_state.egl_surface = nullptr; + return false; + } fprintf(stderr, " [window] EGL context and surface initialized via SDL3\n"); if (WindowTraceEnabled()) { fprintf( @@ -1342,6 +1355,23 @@ bool Init(int width, int height, const char* title) { return true; } +namespace { +bool AttachVrEgl() { + using Attach = + bool (*)(void *, void *, void *, void *, void *(*)(const char *)); + auto attach = + reinterpret_cast(dlsym(RTLD_DEFAULT, "mocktail_vr_attach_egl")); + if (!attach) + return true; + return attach( + g_state.egl_display, g_state.egl_config, g_state.egl_context, + reinterpret_cast(SDL_EGL_GetProcAddress("eglGetProcAddress")), + +[](const char *name) -> void * { + return reinterpret_cast(SDL_GL_GetProcAddress(name)); + }); +} +} // namespace + bool MakeCurrentOnThread() { if (!g_state.initialised || !g_state.egl_context) { if (WindowTraceEnabled()) { @@ -1359,7 +1389,7 @@ bool MakeCurrentOnThread() { return false; } fprintf(stderr, " [window] EGL context now current on engine thread\n"); - return true; + return AttachVrEgl(); } bool ReleaseCurrentOnThread() { @@ -1374,6 +1404,11 @@ bool ReleaseCurrentOnThread() { SDL_GetError()); return false; } + using Release = void (*)(); + auto release = + reinterpret_cast(dlsym(RTLD_DEFAULT, "mocktail_vr_release_egl")); + if (release) + release(); if (WindowTraceEnabled()) { fprintf(stderr, " [window] EGL context released on current thread\n"); } @@ -1516,7 +1551,11 @@ void* GetGLProcAddress(const char* name) { if (!g_state.initialised || name == nullptr || name[0] == '\0') { return nullptr; } - return reinterpret_cast(SDL_GL_GetProcAddress(name)); + void *raw = reinterpret_cast(SDL_GL_GetProcAddress(name)); + using Wrap = void *(*)(const char *, void *); + static auto wrap = + reinterpret_cast(dlsym(RTLD_DEFAULT, "mocktail_vr_wrap_gles_proc")); + return wrap ? wrap(name, raw) : raw; } void ShowIfHidden() { @@ -1575,6 +1614,16 @@ bool SwapBuffers() { return false; } + using Notify = void (*)(); + static auto vr_begin = reinterpret_cast( + dlsym(RTLD_DEFAULT, "mocktail_vr_note_host_present_begin")); + static auto vr_present = + reinterpret_cast(dlsym(RTLD_DEFAULT, "mocktail_vr_gles_present")); + if (vr_begin) + vr_begin(); + if (vr_present) + vr_present(); + if (g_gles_text_overlay == nullptr) { graphics::GlesTextOverlaySource source; source.may_present = reinterpret_cast( @@ -2356,6 +2405,14 @@ void Shutdown() { // resources below. g_gles_text_overlay.reset(); if (!g_state.direct_vulkan && g_state.egl_context != nullptr) { + // Drain the XR session while its EGL context still exists and is current. + SDL_GL_MakeCurrent(g_state.sdl_window, + static_cast(g_state.egl_context)); + using Detach = void (*)(void *); + auto detach = + reinterpret_cast(dlsym(RTLD_DEFAULT, "mocktail_vr_detach_egl")); + if (detach) + detach(g_state.egl_context); SDL_GL_MakeCurrent(g_state.sdl_window, nullptr); } if (!g_state.direct_vulkan && g_state.egl_context) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 863b8e13..84876200 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -29,7 +29,7 @@ file(GENERATE } ]=]) -add_executable(vr_mirror_test vr/vr_mirror_test.cc) +add_executable(vr_mirror_test vr/vr_mirror_test.cc vr/runtime_detection_test.cc) target_link_libraries(vr_mirror_test PRIVATE Mocktail::VR GTest::gtest_main) mocktail_apply_compile_options(vr_mirror_test) add_executable(vr_pose_math_test vr/vr_pose_math_test.cc) @@ -57,6 +57,11 @@ if(MOCKTAIL_ENABLE_VR) target_include_directories(openxr_backend_test PRIVATE "${PROJECT_SOURCE_DIR}/third_party/OpenXR-SDK/include") target_link_libraries(openxr_backend_test PRIVATE Mocktail::VR GTest::gtest_main) gtest_discover_tests(openxr_backend_test) + add_executable(gles_vr_transport_test vr/gles_transport_test.cc) + target_include_directories(gles_vr_transport_test PRIVATE "${PROJECT_SOURCE_DIR}/src" "${PROJECT_SOURCE_DIR}/third_party/OpenXR-SDK/include" ${MOCKTAIL_GLES3_INCLUDE_DIR}) + target_link_libraries(gles_vr_transport_test PRIVATE Mocktail::VR SDL3::SDL3 GTest::gtest_main) + gtest_discover_tests(gles_vr_transport_test PROPERTIES ENVIRONMENT "SDL_VIDEODRIVER=offscreen") + add_library(mocktail_test_openxr_runtime MODULE vr/fake_openxr_runtime.cc vr/fake_openxr_preview.cc) target_link_libraries(mocktail_test_openxr_runtime PRIVATE diff --git a/tests/vr/gles_transport_test.cc b/tests/vr/gles_transport_test.cc new file mode 100644 index 00000000..60a960f7 --- /dev/null +++ b/tests/vr/gles_transport_test.cc @@ -0,0 +1,260 @@ +#include "mocktail/vr/openxr_backend.h" +#include "mocktail/vr/roblox_vr_device_bridge.h" +#include "vr/gles_transport.h" +#include +#include +#include +#include +#include +#include + +namespace mocktail::vr { +namespace internal { +struct VrBridgeTestAccess { + static void Enable(RobloxVrDeviceBridge &bridge) { bridge.active_ = true; } +}; +} // namespace internal +struct GlesTransportTestAccess { + static void Owner(GlesTransport &t, unsigned fb, void *owner) { + t.owners_[fb] = owner; + } +}; +struct OpenXrBackendTestAccess { + static GlesTransport &Transport(OpenXrBackend &b) { return *b.gles_; } + static void LoseSession(OpenXrBackend &b) { + b.HandleSessionState(XR_SESSION_STATE_LOSS_PENDING); + } +}; +namespace { +#define GL(name) \ + reinterpret_cast(SDL_GL_GetProcAddress("gl" #name)) +class GlesVrTransportTest : public ::testing::Test { +protected: + void SetUp() override { + SDL_SetHint(SDL_HINT_VIDEO_FORCE_EGL, "1"); + ASSERT_TRUE(SDL_Init(SDL_INIT_VIDEO)) << SDL_GetError(); + SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_ES); + SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 3); + SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 0); + window = SDL_CreateWindow("Mocktail GLES VR test", 64, 64, + SDL_WINDOW_OPENGL | SDL_WINDOW_HIDDEN); + ASSERT_NE(window, nullptr) << SDL_GetError(); + context = SDL_GL_CreateContext(window); + ASSERT_NE(context, nullptr) << SDL_GetError(); + ASSERT_TRUE(SDL_GL_MakeCurrent(window, context)); + internal::VrBridgeTestAccess::Enable(bridge); + } + void TearDown() override { + transport.Destroy(); + if (context) + SDL_GL_DestroyContext(context); + if (window) + SDL_DestroyWindow(window); + SDL_Quit(); + } + void InitTransport(GlesTransport &t) { + std::string error; + ASSERT_TRUE(t.Initialize( + SDL_EGL_GetCurrentDisplay(), SDL_EGL_GetCurrentConfig(), context, + reinterpret_cast(SDL_EGL_GetProcAddress("eglGetProcAddress")), + Resolve, &error)) + << error; + } + static void *Resolve(const char *name) { + return reinterpret_cast(SDL_GL_GetProcAddress(name)); + } + GLuint Texture(GlesTransport *t, float r, float g, float b) { + GLuint texture = 0, fb = 0; + GL(GenTextures)(1, &texture); + GL(BindTexture)(GL_TEXTURE_2D, texture); + GL(TexStorage2D)(GL_TEXTURE_2D, 1, GL_RGBA8, 500, 500); + if (t) + t->Storage(true, 500, 500, GL_RGBA8); + GL(GenFramebuffers)(1, &fb); + GL(BindFramebuffer)(GL_FRAMEBUFFER, fb); + GL(FramebufferTexture2D)(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, texture, 0); + GL(ClearColor)(r, g, b, 1); + GL(Clear)(GL_COLOR_BUFFER_BIT); + GL(DeleteFramebuffers)(1, &fb); + return texture; + } + GLuint Eye(GlesTransport &t, int eye, GLuint texture, std::uint64_t frame = 7, + void *owner = reinterpret_cast(123)) { + GLuint fb = 0; + GL(GenFramebuffers)(1, &fb); + GL(BindFramebuffer)(GL_FRAMEBUFFER, fb); + GL(FramebufferTexture2D)(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, texture, 0); + GlesTransportTestAccess::Owner(t, fb, owner); + BindEye(t, eye, fb, frame, owner); + return fb; + } + void BindEye(GlesTransport &t, int eye, GLuint fb, std::uint64_t frame, + void *owner) { + GL(BindFramebuffer)(GL_FRAMEBUFFER, fb); + bridge.OnEyeGetterCall(owner, eye, + reinterpret_cast(std::uintptr_t(fb))); + t.Bind(GL_FRAMEBUFFER, fb, frame); + } + std::array Pixel(GLuint texture, int x = 250, int y = 250) { + GLuint fb = 0; + GL(GenFramebuffers)(1, &fb); + GL(BindFramebuffer)(GL_FRAMEBUFFER, fb); + GL(FramebufferTexture2D)(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_TEXTURE_2D, texture, 0); + std::array rgba{}; + GL(ReadPixels)(x, y, 1, 1, GL_RGBA, GL_UNSIGNED_BYTE, rgba.data()); + GL(DeleteFramebuffers)(1, &fb); + return rgba; + } + SDL_Window *window = nullptr; + SDL_GLContext context = nullptr; + GlesTransport transport; + RobloxVrDeviceBridge bridge; +}; +TEST_F(GlesVrTransportTest, CopiesDistinctEyesAndRestoresGuestState) { + InitTransport(transport); + const GLuint red = Texture(&transport, 1, 0, 0), + green = Texture(&transport, 0, 1, 0); + Eye(transport, 0, red); + Eye(transport, 1, green); + const GLuint targets[2] = {Texture(nullptr, 0, 0, 0), + Texture(nullptr, 0, 0, 0)}; + // A tiny guest scissor must not crop the eye copy, and bindings must survive. + GLuint fbs[2]; + GL(GenFramebuffers)(2, fbs); + GL(BindFramebuffer)(GL_READ_FRAMEBUFFER, fbs[0]); + GL(BindFramebuffer)(GL_DRAW_FRAMEBUFFER, fbs[1]); + GL(Enable)(GL_SCISSOR_TEST); + GL(Scissor)(0, 0, 1, 1); + GL(Viewport)(2, 3, 17, 19); + ASSERT_TRUE(transport.Copy(targets, 500, 500)); + GLint read = 0, draw = 0, viewport[4]; + GL(GetIntegerv)(GL_READ_FRAMEBUFFER_BINDING, &read); + GL(GetIntegerv)(GL_DRAW_FRAMEBUFFER_BINDING, &draw); + GL(GetIntegerv)(GL_VIEWPORT, viewport); + EXPECT_EQ(static_cast(read), fbs[0]); + EXPECT_EQ(static_cast(draw), fbs[1]); + EXPECT_TRUE(GL(IsEnabled)(GL_SCISSOR_TEST)); + EXPECT_EQ(viewport[0], 2); + EXPECT_EQ(viewport[1], 3); + EXPECT_EQ(viewport[2], 17); + EXPECT_EQ(viewport[3], 19); + EXPECT_EQ(Pixel(targets[0]), (std::array{255, 0, 0, 255})); + EXPECT_EQ(Pixel(targets[1]), (std::array{0, 255, 0, 255})); + EXPECT_EQ(GL(GetError)(), GL_NO_ERROR); +} +TEST_F(GlesVrTransportTest, RejectsUnownedAndAliasedEyesAndTracksDeletion) { + InitTransport(transport); + GLuint texture = Texture(&transport, 1, 0, 0); + GLuint fbs[2]; + GL(GenFramebuffers)(2, fbs); + GL(BindFramebuffer)(GL_FRAMEBUFFER, fbs[0]); + GL(FramebufferTexture2D)(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, + texture, 0); + BindEye(transport, 0, fbs[0], 7, reinterpret_cast(123)); + EXPECT_EQ(transport.eye(0).framebuffer, 0u); + Eye(transport, 0, texture); + Eye(transport, 1, texture); + const GLuint targets[2] = {Texture(nullptr, 0, 0, 0), + Texture(nullptr, 0, 0, 0)}; + EXPECT_FALSE(transport.Copy(targets, 500, 500)); + transport.ObjectsDeleted(true, 1, &texture); + EXPECT_EQ(transport.eye(0).framebuffer, 0u); + EXPECT_EQ(transport.eye(1).framebuffer, 0u); +} +TEST_F(GlesVrTransportTest, NewOwnerAndFramebufferDeletionInvalidateOldPair) { + InitTransport(transport); + const GLuint red = Texture(&transport, 1, 0, 0), + green = Texture(&transport, 0, 1, 0); + Eye(transport, 0, red); + Eye(transport, 1, green); + const auto fb = Eye(transport, 0, red, 8, reinterpret_cast(456)); + EXPECT_EQ(transport.eye(1).framebuffer, 0u); + EXPECT_EQ(transport.eye(0).frame, 8u); + transport.Deleted(1, &fb); + EXPECT_EQ(transport.eye(0).framebuffer, 0u); +} +TEST_F(GlesVrTransportTest, RenderbufferEyesAndAttachmentMutation) { + InitTransport(transport); + GLuint rb = 0, fb = 0; + GL(GenRenderbuffers)(1, &rb); + GL(BindRenderbuffer)(GL_RENDERBUFFER, rb); + GL(RenderbufferStorage)(GL_RENDERBUFFER, GL_RGBA8, 500, 500); + transport.Storage(false, 500, 500, GL_RGBA8); + GL(GenFramebuffers)(1, &fb); + GL(BindFramebuffer)(GL_FRAMEBUFFER, fb); + GL(FramebufferRenderbuffer)(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, + GL_RENDERBUFFER, rb); + GL(ClearColor)(0, 0, 1, 1); + GL(Clear)(GL_COLOR_BUFFER_BIT); + GlesTransportTestAccess::Owner(transport, fb, reinterpret_cast(123)); + BindEye(transport, 0, fb, 7, reinterpret_cast(123)); + EXPECT_EQ(transport.eye(0).object, rb); + EXPECT_FALSE(transport.eye(0).texture); + const GLuint green = Texture(&transport, 0, 1, 0); + Eye(transport, 1, green); + const GLuint targets[2] = {Texture(nullptr, 0, 0, 0), + Texture(nullptr, 0, 0, 0)}; + ASSERT_TRUE(transport.Copy(targets, 500, 500)); + EXPECT_EQ(Pixel(targets[0]), (std::array{0, 0, 255, 255})); + GL(BindFramebuffer)(GL_FRAMEBUFFER, fb); + transport.AttachmentChanged(GL_FRAMEBUFFER); + EXPECT_EQ(transport.eye(0).framebuffer, 0u); + EXPECT_EQ(GL(GetError)(), GL_NO_ERROR); +} +TEST_F(GlesVrTransportTest, OpenXrEglSessionSubmitsAndRecovers) { + if (!std::getenv("MOCKTAIL_TEST_XR_GLES")) + GTEST_SKIP() << "Opt-in: requires a connected or simulated OpenXR runtime"; + OpenXrBackend backend; + auto status = backend.Arm(VrGraphicsApi::kOpenGles); + ASSERT_TRUE(status.ok()) << status.message(); + status = backend.AttachGlesContext( + SDL_EGL_GetCurrentDisplay(), SDL_EGL_GetCurrentConfig(), context, + reinterpret_cast(SDL_EGL_GetProcAddress("eglGetProcAddress")), + Resolve); + ASSERT_TRUE(status.ok()) << status.message(); + auto &t = OpenXrBackendTestAccess::Transport(backend); + const GLuint red = Texture(&t, 1, 0, 0), green = Texture(&t, 0, 1, 0); + auto *owner = reinterpret_cast(123); + const GLuint fbs[2] = {Eye(t, 0, red), Eye(t, 1, green)}; + bool lost_session = false; + for (int i = 0; i < 200 && backend.submitted_frames() < 20; ++i) { + if (!lost_session && backend.submitted_frames() >= 10) { + OpenXrBackendTestAccess::LoseSession(backend); + EXPECT_FALSE(backend.session_running()); + EXPECT_FALSE(backend.PublishedHeadPose().valid); + lost_session = true; + } + backend.NoteGlesPresent(); + auto pose = backend.PublishedHeadPose(); + if (!pose.valid) { + SDL_Delay(10); + continue; + } + backend.NotePoseApplied(owner, pose.frame); + BindEye(t, 0, fbs[0], pose.frame, owner); + BindEye(t, 1, fbs[1], pose.frame, owner); + EXPECT_EQ(t.eye(0).frame, pose.frame); + EXPECT_EQ(t.eye(1).frame, pose.frame); + EXPECT_TRUE(t.IsCurrent()); + } + EXPECT_TRUE(lost_session); + EXPECT_GE(backend.submitted_frames(), 20u); + EXPECT_EQ(GL(GetError)(), GL_NO_ERROR); + backend.DetachGlesContext(context); + EXPECT_FALSE(backend.session_running()); + status = backend.AttachGlesContext( + SDL_EGL_GetCurrentDisplay(), SDL_EGL_GetCurrentConfig(), context, + reinterpret_cast(SDL_EGL_GetProcAddress("eglGetProcAddress")), + Resolve); + EXPECT_TRUE(status.ok()) << status.message(); + if (status.ok()) { + EXPECT_EQ(OpenXrBackendTestAccess::Transport(backend).eye(0).framebuffer, + 0u); + backend.DetachGlesContext(context); + } +} +} // namespace +} // namespace mocktail::vr diff --git a/tests/vr/runtime_detection_test.cc b/tests/vr/runtime_detection_test.cc new file mode 100644 index 00000000..b5eca654 --- /dev/null +++ b/tests/vr/runtime_detection_test.cc @@ -0,0 +1,141 @@ +#include "mocktail/vr/openxr_backend.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mocktail::vr { +namespace { +class VrRuntimeDetectionTest : public ::testing::Test { +protected: + void SetUp() override { + char path[] = "/tmp/mocktail-xr-detect-XXXXXX"; + ASSERT_NE(mkdtemp(path), nullptr); + root = path; + std::filesystem::create_directories(root / "proc/net"); + std::filesystem::create_directories(root / "config/openxr/1"); + std::filesystem::create_directories(root / "run/wivrn"); + std::filesystem::create_symlink("/proc/net/unix", root / "proc/net/unix"); + } + void TearDown() override { + if (socket >= 0) + close(socket); + std::error_code error; + if (!root.empty()) + std::filesystem::remove_all(root, error); + } + void File(const std::filesystem::path &path, const std::string &data = "{}") { + std::filesystem::create_directories(path.parent_path()); + std::ofstream(path) << data; + } + std::string Steam() { + const auto install = root / "custom-library/SteamVR"; + File(install / "steamxr_linux64.json"); + File(install / "bin/linux64/vrserver", "test executable"); + File(root / "proc/100/comm", "vrserver\n"); + std::filesystem::create_symlink(install / "bin/linux64/vrserver", + root / "proc/100/exe"); + return (install / "steamxr_linux64.json").string(); + } + std::string WiVRn(bool listen) { + const auto install = root / "wivrn-build"; + File(install / "openxr_wivrn-dev.json"); + File(install / "server/wivrn-server", "test executable"); + File(root / "proc/200/comm", "wivrn-server\n"); + std::filesystem::create_symlink(install / "server/wivrn-server", + root / "proc/200/exe"); + const auto path = (root / "run/wivrn/comp_ipc").string(); + socket = ::socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); + EXPECT_GE(socket, 0); + sockaddr_un address{}; + address.sun_family = AF_UNIX; + EXPECT_LT(path.size(), sizeof(address.sun_path)); + std::strcpy(address.sun_path, path.c_str()); + EXPECT_EQ( + bind(socket, reinterpret_cast(&address), sizeof(address)), + 0); + if (listen) + EXPECT_EQ(::listen(socket, 1), 0); + else { + close(socket); + socket = -1; + } // Leave a stale socket inode. + return (install / "openxr_wivrn-dev.json").string(); + } + VrRuntimeDiscovery Discover(const std::string &preference = "auto") { + return DiscoverVrRuntime(preference, (root / "home").string(), + (root / "config").string(), + (root / "etc/xdg").string(), + (root / "run").string(), (root / "proc").string()); + } + std::string Selected(const std::string &preference = "auto") { + return SelectVrRuntimeManifest("", Discover(preference).candidates); + } + std::filesystem::path root; + int socket = -1; +}; +TEST_F(VrRuntimeDetectionTest, RunningSteamOverridesOldRegisteredWiVRn) { + File(root / "config/openxr/1/active_runtime.json"); + EXPECT_EQ(Selected(), + (root / "config/openxr/1/active_runtime.json").string()); + const auto steam = Steam(); + EXPECT_EQ(Selected(), steam); + EXPECT_NE(Discover().reason.find("running SteamVR"), std::string::npos); +} +TEST_F(VrRuntimeDetectionTest, ListeningWiVRnFindsDevelopmentManifest) { + File(root / "config/openxr/1/active_runtime.json"); + const auto wivrn = WiVRn(true); + EXPECT_EQ(Selected(), wivrn); + EXPECT_NE(Discover().reason.find("listening WiVRn"), std::string::npos); +} +TEST_F(VrRuntimeDetectionTest, RegisteredWiVRnBuildWinsOverNativeFallback) { + const auto registered = + (root / "config/openxr/1/active_runtime.json").string(); + File( + registered, + R"({"runtime":{"library_path":"/var/lib/flatpak/app/wivrn/libwivrn.so"}})"); + WiVRn(true); + EXPECT_EQ(Selected(), registered); +} +TEST_F(VrRuntimeDetectionTest, StaleWiVRnSocketDoesNotOverrideSteam) { + const auto steam = Steam(); + WiVRn(false); + EXPECT_EQ(Selected(), steam); +} +TEST_F(VrRuntimeDetectionTest, + BothServersRespectRegisteredRuntimeAndExplicitOverrides) { + const auto registered = + (root / "config/openxr/1/active_runtime.json").string(); + File(registered); + const auto steam = Steam(); + const auto wivrn = WiVRn(true); + EXPECT_EQ(Selected(), registered); + EXPECT_EQ(Selected("alvr"), steam); + EXPECT_EQ(Selected("wivrn"), wivrn); + EXPECT_TRUE(Discover("system").candidates.empty()); + EXPECT_EQ( + SelectVrRuntimeManifest("/custom/override.json", Discover().candidates), + "/custom/override.json"); +} +TEST_F(VrRuntimeDetectionTest, + OpenVrRegistryFindsCustomSteamLibraryWithoutRunningServer) { + const auto install = root / "elsewhere/SteamVR"; + File(install / "steamxr_linux64.json"); + File(root / "config/openvr/openvrpaths.vrpath", + "{\"runtime\":[42,\"relative/path\",\"" + install.string() + "\"]}"); + EXPECT_EQ(Selected("alvr"), (install / "steamxr_linux64.json").string()); + // No external host registration or installed WiVRn is assumed in this + // assertion. + const auto candidates = Discover().candidates; + EXPECT_NE(std::find(candidates.begin(), candidates.end(), + (install / "steamxr_linux64.json").string()), + candidates.end()); + File(root / "config/openvr/openvrpaths.vrpath", "invalid json"); + EXPECT_NE(Selected("alvr"), (install / "steamxr_linux64.json").string()); +} +} // namespace +} // namespace mocktail::vr diff --git a/tests/vr/vr_mirror_test.cc b/tests/vr/vr_mirror_test.cc index f2a8e35d..dfea1996 100644 --- a/tests/vr/vr_mirror_test.cc +++ b/tests/vr/vr_mirror_test.cc @@ -66,7 +66,10 @@ class SegmentReader { TEST(VrRuntimeSelection, RuntimeErrorsPreserveSelectionProvenance) { EXPECT_NE(VrRuntimeUnavailableHint(true, "/custom.json").find("User-selected"), std::string::npos); - EXPECT_NE(VrRuntimeUnavailableHint(false, "/usr/share/openxr/1/openxr_wivrn.json").find("Auto-selected WiVRn"), std::string::npos); + EXPECT_NE( + VrRuntimeUnavailableHint(false, "/usr/share/openxr/1/openxr_wivrn.json") + .find("Selected OpenXR"), + std::string::npos); EXPECT_NE(VrRuntimeUnavailableHint(false, "").find("registered"), std::string::npos); } @@ -198,3 +201,28 @@ TEST(VrMirrorProducer, RejectsInvalidNameAndOversizedPublish) { } // namespace } // namespace mocktail::vr + +namespace mocktail::vr { +TEST(VrRuntimeSelection, AlvrSelectsSteamVrWithoutFallingBackToWiVRn) { + const auto alvr = VrRuntimeManifestCandidates("alvr", "/home/test", "", ""); + EXPECT_EQ(alvr, VrRuntimeManifestCandidates("steamvr", "/home/test", "", "")); + ASSERT_FALSE(alvr.empty()); + for (const auto &path : alvr) { + EXPECT_NE(path.find("SteamVR/steamxr_linux64.json"), std::string::npos); + EXPECT_EQ(path.find("wivrn"), std::string::npos); + } + EXPECT_TRUE( + VrRuntimeManifestCandidates("system", "/home/test", "", "").empty()); + EXPECT_EQ(SelectVrRuntimeManifest("/custom/steamxr_linux64.json", alvr), + "/custom/steamxr_linux64.json"); +} +TEST(VrRuntimeSelection, RegisteredRuntimePrecedesInstalledWiVRn) { + const auto paths = VrRuntimeManifestCandidates( + "auto", "/home/test", "/private/config", "/opt/xdg:/etc/xdg"); + ASSERT_GE(paths.size(), 4u); + EXPECT_EQ(paths[0], "/private/config/openxr/1/active_runtime.x86_64.json"); + EXPECT_EQ(paths[1], "/private/config/openxr/1/active_runtime.json"); + EXPECT_EQ(paths[2], "/opt/xdg/openxr/1/active_runtime.x86_64.json"); + EXPECT_EQ(paths.back(), "/usr/share/openxr/1/openxr_wivrn.json"); +} +} // namespace mocktail::vr From f5080e8a998d6c30332fdb6117e275f62b31174d Mon Sep 17 00:00:00 2001 From: komaruworld Date: Sun, 13 Sep 2026 20:19:00 +0300 Subject: [PATCH 04/58] fixes --- src/window/window.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/window/window.cc b/src/window/window.cc index 34abd807..d4a7507e 100644 --- a/src/window/window.cc +++ b/src/window/window.cc @@ -1020,8 +1020,7 @@ bool Init(int width, int height, const char* title) { ConfigureGraphicsBackendBeforeSDL(); - if (StringEquals(GetEnvNonEmpty("MOCKTAIL_GRAPHICS_BACKEND"), - "direct-vulkan")) { + if (ShouldUseNativeVulkanBackend()) { if (!SDL_Init(SDL_INIT_VIDEO)) { fprintf(stderr, " [window] SDL_Init failed: %s\n", SDL_GetError()); return false; From 7e057232e4dad2ad4908a1750519cdbc97706659 Mon Sep 17 00:00:00 2001 From: komaruworld Date: Sun, 13 Sep 2026 20:36:53 +0300 Subject: [PATCH 05/58] fixes --- README.md | 6 + include/mocktail/vr/openxr_backend.h | 11 ++ src/graphics/bionic_vulkan_loader_adapter.cc | 29 +++++ src/vr/openxr_backend.cc | 127 +++++++++++++++++++ tests/vr/openxr_backend_test.cc | 25 ++++ 5 files changed, 198 insertions(+) diff --git a/README.md b/README.md index e14e232f..e4d09667 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,12 @@ XR_RUNTIME_JSON="/path/to/SteamVR/steamxr_linux64.json" ./build/mocktail --vr to the OpenXR loader. The default `auto` uses the server detection described above, then registered and installed runtimes. Selection affects only this process. +Vulkan VR mirrors the left eye into the desktop Roblox window using a GPU +blit, preserving its aspect ratio. DebugDeviceVR otherwise leaves that window +black even when the headset receives eye images. Set +`MOCKTAIL_VR_DESKTOP_MIRROR=0` to disable this copy. This is a view of the eye, +not a separate desktop camera or a replacement for VR menu input. + ## OpenGL ES The `opengl`/`system` renderer uses the guest's real EGL/OpenGL ES context: diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index 9d0fbb80..62765825 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -141,6 +141,11 @@ class OpenXrBackend final { // Render-thread frame cycle at host present begin. void NoteHostPresent(VkQueue queue, VkDevice device); + void RecordDesktopSwapchain(VkDevice device, VkSwapchainKHR swapchain, + const VkSwapchainCreateInfoKHR* info, const VkImage* images, unsigned count); + // On success the original present semaphores have been consumed and the + // copy has completed. The caller must present with no wait semaphores. + VkResult MirrorDesktop(VkQueue queue, const VkPresentInfoKHR* info); // Pose published for the frame that is about to render. The device bridge // reads it once per frame on the render thread and injects it into the @@ -273,6 +278,12 @@ class OpenXrBackend final { VrMirror mirror_; std::unordered_map images_; + struct DesktopSwapchain { + VkExtent2D extent; + VkFormat format; + std::vector images; + }; + std::unordered_map desktop_swapchains_; std::unordered_map image_views_; std::unordered_map> framebuffer_views_; std::unordered_map framebuffer_render_pass_; diff --git a/src/graphics/bionic_vulkan_loader_adapter.cc b/src/graphics/bionic_vulkan_loader_adapter.cc index 7fe7f4b1..a3a6dc42 100644 --- a/src/graphics/bionic_vulkan_loader_adapter.cc +++ b/src/graphics/bionic_vulkan_loader_adapter.cc @@ -1689,6 +1689,16 @@ VKAPI_ATTR VkResult VKAPI_CALL vkCreateSwapchainKHR( return VK_ERROR_INITIALIZATION_FAILED; } VkSwapchainCreateInfoKHR host_info = *create_info; + static const auto vr_desktop_enabled = ResolveProcessFunction("mocktail_vr_desktop_enabled"); + if (vr_desktop_enabled && vr_desktop_enabled()) { + PFN_vkGetPhysicalDeviceSurfaceCapabilitiesKHR caps; + { std::lock_guard lock(State().mutex); caps = State().host_surface_capabilities; } + VkSurfaceCapabilitiesKHR supported{}; + if (caps && caps(HostDispatchForDevice(device).physical_device, host_info.surface, + &supported) == VK_SUCCESS && + (supported.supportedUsageFlags & VK_IMAGE_USAGE_TRANSFER_DST_BIT)) + host_info.imageUsage |= VK_IMAGE_USAGE_TRANSFER_DST_BIT; + } const mocktail::graphics::PresentModePolicy present_policy = mocktail::graphics::CachedPresentModePolicy(); if (present_policy != mocktail::graphics::PresentModePolicy::kHostDefault) { @@ -1773,6 +1783,9 @@ VKAPI_ATTR VkResult VKAPI_CALL vkCreateSwapchainKHR( host_get_images(device, *swapchain, &image_count, images.data()); if (images_result == VK_SUCCESS || images_result == VK_INCOMPLETE) { images.resize(image_count); + static const auto record = ResolveProcessFunction("mocktail_vr_desktop_swapchain"); + if (record) record(device, *swapchain, &host_info, images.data(), image_count); (void)State().text_overlay.RegisterSwapchain( device, *swapchain, *create_info, images.data(), image_count); } @@ -1784,6 +1797,9 @@ vkDestroySwapchainKHR(VkDevice device, VkSwapchainKHR swapchain, const VkAllocationCallbacks* allocator) { const auto host_destroy = reinterpret_cast( HostDeviceProc(device, "vkDestroySwapchainKHR")); + static const auto record = ResolveProcessFunction("mocktail_vr_desktop_swapchain"); + if (record) record(device, swapchain, nullptr, nullptr, 0); State().text_overlay.DestroySwapchain(device, swapchain); if (host_destroy != nullptr) { host_destroy(device, swapchain, allocator); @@ -2239,6 +2255,19 @@ vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR* present_info) { observation.SetResult(VK_ERROR_INITIALIZATION_FAILED); return VK_ERROR_INITIALIZATION_FAILED; } + static const auto mirror = ResolveProcessFunction( + "mocktail_vr_desktop_present"); + VkPresentInfoKHR mirrored{}; + if (mirror && present_info) { + const VkResult mirror_result = mirror(queue, present_info); + if (mirror_result < 0) return mirror_result; + if (mirror_result == VK_SUCCESS) { + mirrored = *present_info; + mirrored.waitSemaphoreCount = 0; + mirrored.pWaitSemaphores = nullptr; + present_info = &mirrored; + } + } const VkResult result = state.text_overlay.QueuePresent( queue, present_info, ObservedHostQueuePresent); const VkResult normalized_result = NormalizeSwapchainResult(result); diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index e1c8bf78..b468dfb0 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -152,6 +152,8 @@ struct OpenXrBackend::VkProcs { PFN_vkCmdPipelineBarrier CmdPipelineBarrier = nullptr; PFN_vkCmdCopyImage CmdCopyImage = nullptr; PFN_vkCmdBlitImage CmdBlitImage = nullptr; + PFN_vkCmdClearColorImage CmdClearColorImage = nullptr; + PFN_vkGetPhysicalDeviceFormatProperties GetPhysicalDeviceFormatProperties = nullptr; PFN_vkCreateFence CreateFence = nullptr; PFN_vkDestroyFence DestroyFence = nullptr; PFN_vkResetFences ResetFences = nullptr; @@ -585,6 +587,7 @@ void OpenXrBackend::SetupDeviceAfterCreationLocked( MOCKTAIL_BACKEND_VK_FN(CmdPipelineBarrier) MOCKTAIL_BACKEND_VK_FN(CmdCopyImage) MOCKTAIL_BACKEND_VK_FN(CmdBlitImage) + MOCKTAIL_BACKEND_VK_FN(CmdClearColorImage) MOCKTAIL_BACKEND_VK_FN(CreateFence) MOCKTAIL_BACKEND_VK_FN(DestroyFence) MOCKTAIL_BACKEND_VK_FN(ResetFences) @@ -599,6 +602,7 @@ void OpenXrBackend::SetupDeviceAfterCreationLocked( MOCKTAIL_BACKEND_VK_INSTANCE_FN(GetPhysicalDeviceMemoryProperties) MOCKTAIL_BACKEND_VK_INSTANCE_FN(GetPhysicalDeviceQueueFamilyProperties) MOCKTAIL_BACKEND_VK_INSTANCE_FN(GetPhysicalDeviceProperties) + MOCKTAIL_BACKEND_VK_INSTANCE_FN(GetPhysicalDeviceFormatProperties) #undef MOCKTAIL_BACKEND_VK_INSTANCE_FN VkPhysicalDeviceProperties device_properties{}; @@ -1050,6 +1054,7 @@ void OpenXrBackend::TeardownSessionLocked(const char* reason, bool preserve_devi session_recovery_pending_ = false; next_session_retry_ns_ = 0; images_.clear(); + desktop_swapchains_.clear(); image_views_.clear(); framebuffer_views_.clear(); framebuffer_render_pass_.clear(); @@ -2185,6 +2190,106 @@ MirrorVkProcs OpenXrBackend::MirrorProcs() const { return procs; } +// DebugDeviceVR renders offscreen eyes but does not mirror them to the host +// window. Keep this copy on the guest GPU; no readback or image files in normal use. +void OpenXrBackend::RecordDesktopSwapchain(VkDevice device, VkSwapchainKHR swapchain, + const VkSwapchainCreateInfoKHR* info, const VkImage* images, unsigned count) { + std::lock_guard lock(mutex_); + desktop_swapchains_.erase(swapchain); + if (!device || device != vk_device_ || !info || !images || !count || + !(info->imageUsage & VK_IMAGE_USAGE_TRANSFER_DST_BIT) || + info->imageArrayLayers != 1 || info->imageSharingMode != VK_SHARING_MODE_EXCLUSIVE) + return; + desktop_swapchains_[swapchain] = {info->imageExtent, info->imageFormat, + {images, images + count}}; +} + +VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, const VkPresentInfoKHR* info) { + std::lock_guard lock(mutex_); + if (EnvString("MOCKTAIL_VR_DESKTOP_MIRROR", "1") == "0" || + !vk_ || !info || info->swapchainCount != 1 || + !info->pSwapchains || !info->pImageIndices || !eyes_[0].valid || + !eyes_[0].image || (info->waitSemaphoreCount && !info->pWaitSemaphores) || + !queue_families_.count(queue) || queue_families_[queue] != vk_queue_family_) + return VK_NOT_READY; + const auto found = desktop_swapchains_.find(info->pSwapchains[0]); + if (found == desktop_swapchains_.end() || + info->pImageIndices[0] >= found->second.images.size()) return VK_NOT_READY; + const auto& target = found->second; + VkFormatProperties source_properties{}, target_properties{}; + vk_->GetPhysicalDeviceFormatProperties(vk_physical_device_, eyes_[0].format, &source_properties); + vk_->GetPhysicalDeviceFormatProperties(vk_physical_device_, target.format, &target_properties); + if (!(source_properties.optimalTilingFeatures & VK_FORMAT_FEATURE_BLIT_SRC_BIT) || + !(target_properties.optimalTilingFeatures & VK_FORMAT_FEATURE_BLIT_DST_BIT)) return VK_NOT_READY; + std::string error; + if (!EnsureCopyResources(vk_device_, vk_queue_family_, &error)) return VK_NOT_READY; + if (vk_->ResetCommandBuffer(copy_command_, 0) != VK_SUCCESS) return VK_NOT_READY; + VkCommandBufferBeginInfo begin{}; + begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + if (vk_->BeginCommandBuffer(copy_command_, &begin) != VK_SUCCESS) return VK_NOT_READY; + VkImageMemoryBarrier barriers[2]{}; + for (auto& b : barriers) { + b.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + b.srcQueueFamilyIndex = b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + b.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_MEMORY_READ_BIT; + } + barriers[0].image = eyes_[0].image; + barriers[0].oldLayout = eyes_[0].final_layout; + barriers[0].newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + barriers[0].dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + barriers[1].image = target.images[info->pImageIndices[0]]; + barriers[1].oldLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; + barriers[1].newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barriers[1].dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 2, barriers); + const VkClearColorValue black{{0.f, 0.f, 0.f, 1.f}}; + vk_->CmdClearColorImage(copy_command_, barriers[1].image, barriers[1].newLayout, + &black, 1, &barriers[1].subresourceRange); + VkMemoryBarrier clear_barrier{}; + clear_barrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; + clear_barrier.srcAccessMask = clear_barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &clear_barrier, 0, nullptr, 0, nullptr); + const double scale = std::min(double(target.extent.width) / eyes_[0].width, + double(target.extent.height) / eyes_[0].height); + const int width = std::max(1, int(eyes_[0].width * scale)); + const int height = std::max(1, int(eyes_[0].height * scale)); + const int x = (int(target.extent.width) - width) / 2; + const int y = (int(target.extent.height) - height) / 2; + VkImageBlit region{}; + region.srcSubresource = region.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.srcOffsets[1] = {static_cast(eyes_[0].width), static_cast(eyes_[0].height), 1}; + region.dstOffsets[0] = {x, y, 0}; + region.dstOffsets[1] = {x + width, y + height, 1}; + vk_->CmdBlitImage(copy_command_, barriers[0].image, barriers[0].newLayout, + barriers[1].image, barriers[1].newLayout, 1, ®ion, VK_FILTER_NEAREST); + for (auto& b : barriers) { + std::swap(b.oldLayout, b.newLayout); + b.srcAccessMask = b.dstAccessMask; + b.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; + } + vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 2, barriers); + if (vk_->EndCommandBuffer(copy_command_) != VK_SUCCESS || + vk_->ResetFences(vk_device_, 1, ©_fence_) != VK_SUCCESS) return VK_NOT_READY; + std::vector stages(info->waitSemaphoreCount, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); + VkSubmitInfo submit{}; + submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit.waitSemaphoreCount = info->waitSemaphoreCount; + submit.pWaitSemaphores = info->pWaitSemaphores; + submit.pWaitDstStageMask = stages.data(); + submit.commandBufferCount = 1; + submit.pCommandBuffers = ©_command_; + VkResult result = vk_->QueueSubmit(queue, 1, &submit, copy_fence_); + if (result != VK_SUCCESS) return result; + // Once submitted, never fall back to waiting on the consumed semaphores. + result = vk_->WaitForFences(vk_device_, 1, ©_fence_, VK_TRUE, UINT64_MAX); + return result; +} + // Diagnostic only: read BOTH images on the guest queue while the XR image is // still acquired. Never read a released image owned by the compositor. bool OpenXrBackend::CaptureEyeEvidence(VkQueue queue, int eye) { @@ -2424,6 +2529,28 @@ void mocktail_vr_xr_note_render_pass_begin(VkCommandBuffer command_buffer, } } +bool mocktail_vr_desktop_enabled() { + return mocktail::vr::ActiveVrBackend() != nullptr; +} +void mocktail_vr_desktop_swapchain(VkDevice device, VkSwapchainKHR swapchain, + const VkSwapchainCreateInfoKHR* info, const VkImage* images, unsigned count) { + try { + if (auto* backend = mocktail::vr::ActiveVrBackend()) + backend->RecordDesktopSwapchain(device, swapchain, info, images, count); + } catch (const std::exception& error) { + std::fprintf(stderr, " [vr-backend] desktop swapchain registration failed: %s\n", error.what()); + } +} +VkResult mocktail_vr_desktop_present(VkQueue queue, const VkPresentInfoKHR* info) { + try { + if (auto* backend = mocktail::vr::ActiveVrBackend()) return backend->MirrorDesktop(queue, info); + return VK_NOT_READY; + } catch (const std::exception& error) { + std::fprintf(stderr, " [vr-backend] desktop mirror failed: %s\n", error.what()); + return VK_ERROR_OUT_OF_HOST_MEMORY; + } +} + void mocktail_vr_xr_note_present(VkQueue queue, VkDevice device) { auto* backend = mocktail::vr::ActiveVrBackend(); if (backend != nullptr) { diff --git a/tests/vr/openxr_backend_test.cc b/tests/vr/openxr_backend_test.cc index be772a58..d3843f83 100644 --- a/tests/vr/openxr_backend_test.cc +++ b/tests/vr/openxr_backend_test.cc @@ -10,6 +10,7 @@ struct VrBridgeTestAccess { }; } struct OpenXrBackendTestAccess { + static size_t DesktopCount(const OpenXrBackend& b) { return b.desktop_swapchains_.size(); } template static T Handle(std::uintptr_t n) { return reinterpret_cast(n); } static void Configure(OpenXrBackend& backend) { backend.vk_instance_ = Handle(1); @@ -189,3 +190,27 @@ TEST(OpenXrBackendLifecycle, TeardownClearsCantedRejectionLatch) { EXPECT_FALSE(Access::CantedLogged(b)); } } // namespace mocktail::vr + +namespace mocktail::vr { +TEST(OpenXrDesktopMirror, RequiresTransferUsageAndDropsDestroyedSwapchain) { + OpenXrBackend b; + OpenXrBackendTestAccess::Configure(b); + const auto device = OpenXrBackendTestAccess::Handle(2); + const auto chain = OpenXrBackendTestAccess::Handle(20); + const auto image = OpenXrBackendTestAccess::Handle(21); + VkSwapchainCreateInfoKHR info{}; + info.imageArrayLayers = 1; + info.imageExtent = {800, 600}; + info.imageFormat = VK_FORMAT_B8G8R8A8_UNORM; + b.RecordDesktopSwapchain(device, chain, &info, &image, 1); + EXPECT_EQ(OpenXrBackendTestAccess::DesktopCount(b), 0u); + info.imageUsage = VK_IMAGE_USAGE_TRANSFER_DST_BIT; + b.RecordDesktopSwapchain(device, chain, &info, &image, 1); + EXPECT_EQ(OpenXrBackendTestAccess::DesktopCount(b), 1u); + b.RecordDesktopSwapchain(device, chain, nullptr, nullptr, 0); + EXPECT_EQ(OpenXrBackendTestAccess::DesktopCount(b), 0u); + b.RecordDesktopSwapchain(OpenXrBackendTestAccess::Handle(3), chain, &info, &image, 1); + EXPECT_EQ(OpenXrBackendTestAccess::DesktopCount(b), 0u); + EXPECT_EQ(b.MirrorDesktop(VK_NULL_HANDLE, nullptr), VK_NOT_READY); +} +} From a014751cd42d0d8ac6ed6dd276f721e04172185e Mon Sep 17 00:00:00 2001 From: komaruworld Date: Mon, 14 Sep 2026 16:38:31 +0300 Subject: [PATCH 06/58] bla bla bla --- cmake/MocktailInput.cmake | 4 + cmake/MocktailVR.cmake | 7 +- include/mocktail/vr/openxr_backend.h | 34 + include/mocktail/vr/roblox_vr_device_bridge.h | 2 + include/mocktail/vr/vr_perf.h | 2 + include/mocktail/vr/xr_actions.h | 101 ++ include/mocktail/vr/xr_controller.h | 236 +++++ include/mocktail/vr/xr_controller_abi.h | 76 ++ include/runtime/roblox_window_input_runtime.h | 21 + include/runtime/roblox_xr_controller_input.h | 100 ++ src/legacy/legacy_runtime.cc | 6 + src/runtime/roblox_window_input_runtime.cc | 44 + src/runtime/roblox_xr_controller_input.cc | 272 ++++++ src/vr/openxr_backend.cc | 153 ++- src/vr/openxr_disabled.cc | 2 + src/vr/roblox_vr_device_bridge.cc | 352 ++++++- src/vr/roblox_vr_device_bridge_internal.h | 25 + src/vr/vr_perf.cc | 2 + src/vr/xr_actions.cc | 909 ++++++++++++++++++ src/vr/xr_actions_disabled.cc | 53 + src/vr/xr_controller.cc | 376 ++++++++ tests/CMakeLists.txt | 11 + tests/roblox_xr_controller_input_test.cc | 295 ++++++ tests/vr/roblox_vr_device_bridge_test.cc | 145 +++ tests/vr/xr_actions_test.cc | 372 +++++++ tests/vr/xr_controller_test.cc | 298 ++++++ 26 files changed, 3872 insertions(+), 26 deletions(-) create mode 100644 include/mocktail/vr/xr_actions.h create mode 100644 include/mocktail/vr/xr_controller.h create mode 100644 include/mocktail/vr/xr_controller_abi.h create mode 100644 include/runtime/roblox_xr_controller_input.h create mode 100644 src/runtime/roblox_xr_controller_input.cc create mode 100644 src/vr/xr_actions.cc create mode 100644 src/vr/xr_actions_disabled.cc create mode 100644 src/vr/xr_controller.cc create mode 100644 tests/roblox_xr_controller_input_test.cc create mode 100644 tests/vr/xr_actions_test.cc create mode 100644 tests/vr/xr_controller_test.cc diff --git a/cmake/MocktailInput.cmake b/cmake/MocktailInput.cmake index c00d8b58..83775d7c 100644 --- a/cmake/MocktailInput.cmake +++ b/cmake/MocktailInput.cmake @@ -22,6 +22,7 @@ add_library(mocktail_input_runtime STATIC ${MOCKTAIL_INPUT_ROOT}/src/runtime/roblox_text_font_resolver.cc ${MOCKTAIL_INPUT_ROOT}/src/runtime/roblox_text_surface_overlay.cc ${MOCKTAIL_INPUT_ROOT}/src/runtime/roblox_window_input_runtime.cc + ${MOCKTAIL_INPUT_ROOT}/src/runtime/roblox_xr_controller_input.cc ) add_library(Mocktail::InputRuntime ALIAS mocktail_input_runtime) target_include_directories(mocktail_input_runtime PUBLIC @@ -33,12 +34,14 @@ target_include_directories(mocktail_input_runtime PRIVATE target_link_libraries(mocktail_input_runtime PUBLIC Mocktail::PlatformSdl Mocktail::Runtime + Mocktail::VR mocktail_window PkgConfig::UTF8PROC PRIVATE PkgConfig::FONTCONFIG SDL3_ttf::SDL3_ttf nlohmann_json::nlohmann_json + ${CMAKE_DL_LIBS} ) target_compile_features(mocktail_input_runtime PUBLIC cxx_std_17) mocktail_apply_compile_options(mocktail_input_runtime) @@ -47,6 +50,7 @@ if(BUILD_TESTING AND TARGET GTest::gtest_main) add_executable(roblox_input_router_test ${MOCKTAIL_INPUT_ROOT}/tests/roblox_gamepad_input_test.cc ${MOCKTAIL_INPUT_ROOT}/tests/roblox_input_router_test.cc + ${MOCKTAIL_INPUT_ROOT}/tests/roblox_xr_controller_input_test.cc ) target_link_libraries(roblox_input_router_test PRIVATE Mocktail::InputRuntime diff --git a/cmake/MocktailVR.cmake b/cmake/MocktailVR.cmake index f46ee954..783f35de 100644 --- a/cmake/MocktailVR.cmake +++ b/cmake/MocktailVR.cmake @@ -10,6 +10,7 @@ add_library(mocktail_vr STATIC src/vr/openxr_backend_mode.cc src/vr/vr_perf.cc src/vr/vr_mirror.cc + src/vr/xr_controller.cc src/vr/roblox_vr_device_bridge.cc) add_library(Mocktail::VR ALIAS mocktail_vr) target_include_directories(mocktail_vr PUBLIC "${CMAKE_SOURCE_DIR}/include") @@ -47,17 +48,19 @@ if(MOCKTAIL_ENABLE_VR) endfunction() mocktail_add_openxr_sdk() target_sources(mocktail_vr PRIVATE src/vr/openxr_probe.cc src/vr/openxr_preview.cc - src/vr/openxr_backend.cc src/vr/gles_transport.cc) + src/vr/openxr_backend.cc src/vr/gles_transport.cc src/vr/xr_actions.cc) # Error unwinding is confined to these native session implementations; their # public entry points catch errors before returning to the guest runtime. set_source_files_properties(src/vr/openxr_preview.cc src/vr/openxr_backend.cc + src/vr/xr_actions.cc PROPERTIES COMPILE_OPTIONS -fexceptions) target_compile_definitions(mocktail_vr PRIVATE XR_USE_GRAPHICS_API_VULKAN) set_source_files_properties(src/vr/openxr_backend.cc PROPERTIES COMPILE_DEFINITIONS "XR_USE_GRAPHICS_API_OPENGL_ES;XR_USE_PLATFORM_EGL") target_include_directories(mocktail_vr PRIVATE ${MOCKTAIL_EGL_INCLUDE_DIR} ${MOCKTAIL_GLES3_INCLUDE_DIR}) target_link_libraries(mocktail_vr PRIVATE OpenXR::openxr_loader Vulkan::Headers ${CMAKE_DL_LIBS}) else() - target_sources(mocktail_vr PRIVATE src/vr/openxr_disabled.cc) + target_sources(mocktail_vr PRIVATE src/vr/openxr_disabled.cc + src/vr/xr_actions_disabled.cc) endif() add_executable(mocktail_vr_probe src/vr/probe_main.cc) diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index 62765825..ba4302f9 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -12,6 +12,8 @@ #include "mocktail/status.h" #include "mocktail/vr/vr_mirror.h" +#include "mocktail/vr/xr_actions.h" +#include "mocktail/vr/xr_controller.h" namespace mocktail::vr { @@ -46,11 +48,17 @@ const char* VrBackendModeName(VrBackendMode mode); // One deterministic head pose sample. Position is metres, orientation is a // quaternion (x, y, z, w) in the OpenXR LOCAL reference space. +// VrHandPose (tracked controller hands) is defined in xr_controller.h. struct ScriptedPoseSample { float position[3] = {0.f, 0.f, 0.f}; float orientation[4] = {0.f, 0.f, 0.f, 1.f}; float eye_offset[2][3] = {}; // metres in head space float eye_fov[2][4] = {}; // left, right, up, down angles + // Index 0 = left, 1 = right (Roblox UserCFrame LeftHand/RightHand). + VrHandPose hands[2]; + // Exact 2998 DebugDeviceVR input channels, published with the same pose. + float controller_channels[28] = {}; + bool controllers_connected = false; std::uint64_t frame = 0; bool valid = false; }; @@ -150,9 +158,23 @@ class OpenXrBackend final { // Pose published for the frame that is about to render. The device bridge // reads it once per frame on the render thread and injects it into the // guest. valid=false until the XR session located a usable head pose. + // hands[0]=left, hands[1]=right carry this frame's tracked controller grip + // poses (metres, LOCAL space) when a controller layer is active. ScriptedPoseSample PublishedHeadPose() const; void NotePoseApplied(void* owner, std::uint64_t frame); + // Controller input delivery, consumed by the window-thread input runtime + // through the C ABI in xr_controller_abi.h. TakeControllerDelivery pops one + // batch produced by the once-per-frame action sync; it returns false when + // no controller layer is active or nothing is queued. The snapshot exposes + // the current levels for overflow resync. Haptics requests are thread-safe. + bool TakeControllerDelivery(ControllerDelivery* out); + ControllerSnapshot ControllerSnapshotForFrame() const; + void RequestControllerHaptics(int hand, float amplitude, + std::uint64_t duration_ns, float frequency_hz); + + void StopControllerHaptics(int hand); + // Diagnostics. std::uint64_t submitted_frames() const { return submitted_frames_.load(std::memory_order_relaxed); @@ -203,6 +225,18 @@ class OpenXrBackend final { VkDevice device, const VkDeviceCreateInfo* create_info, std::string* error); + // Session-owned controller action layer (poses, buttons, axes, haptics). + // Created at instance level, attached per session, detached on teardown so + // a replacement session never reuses stale spaces. + XrActions actions_; + bool actions_created_ = false; + // Shared by action sync, snapshots, haptics and lifecycle operations. + // Acquired after mutex_ on the render thread, but never held across + // xrWaitFrame; the input thread takes only this controller lock. + mutable std::mutex controller_mutex_; + // Syncs actions once per frame at the predicted display time and folds the + // resulting hand poses into published_pose_ before the guest renders. + void SyncControllersLocked(std::uint64_t predicted_display_time); bool InitializeGlesSessionLocked(std::string *error); bool InitializeSessionResourcesLocked(std::string *error); bool CopyGlesEyesIntoSwapchains(); diff --git a/include/mocktail/vr/roblox_vr_device_bridge.h b/include/mocktail/vr/roblox_vr_device_bridge.h index dddc0c42..c6b45b57 100644 --- a/include/mocktail/vr/roblox_vr_device_bridge.h +++ b/include/mocktail/vr/roblox_vr_device_bridge.h @@ -53,6 +53,7 @@ class RobloxVrDeviceBridge final { void NoteHostPresent(); // Guest-side hooks invoked by the interposed vtable stubs. + void OnHapticsCall(void* device_object, int hand, float amplitude); void OnStateGetterCall(void *result_buffer, void *device_object); void OnEyeGetterCall(void *device_object, int eye_index, void *framebuffer); @@ -89,6 +90,7 @@ class RobloxVrDeviceBridge final { std::atomic image_base_{0}; std::atomic image_size_{0}; std::uintptr_t *vtable_ = nullptr; + std::uintptr_t original_haptic_slot_ = 0; std::uintptr_t original_state_slot_ = 0; std::uintptr_t original_eye_slot_ = 0; std::atomic original_state_getter_{nullptr}; diff --git a/include/mocktail/vr/vr_perf.h b/include/mocktail/vr/vr_perf.h index 6fe8ef66..fd6cd430 100644 --- a/include/mocktail/vr/vr_perf.h +++ b/include/mocktail/vr/vr_perf.h @@ -24,6 +24,7 @@ enum class Stage { kCopyFenceWait, kMirrorPublish, kEvidence, + kControllerSync, kStageCount, }; @@ -45,6 +46,7 @@ enum class Counter { kVisibilityLost, kVisibilityGained, kFramesInvalidated, + kControllerSyncs, kCounterCount, }; diff --git a/include/mocktail/vr/xr_actions.h b/include/mocktail/vr/xr_actions.h new file mode 100644 index 00000000..1ae9b57f --- /dev/null +++ b/include/mocktail/vr/xr_actions.h @@ -0,0 +1,101 @@ +#ifndef MOCKTAIL_VR_XR_ACTIONS_H_ +#define MOCKTAIL_VR_XR_ACTIONS_H_ + +#include +#include + +#include "mocktail/status.h" +#include "mocktail/vr/vr_pose_math.h" +#include "mocktail/vr/xr_controller.h" + +namespace mocktail::vr { + +// Session-owned OpenXR action layer: one action set, left/right subactions, +// grip+aim pose spaces, and interaction-profile bindings for Touch (primary), +// Quest 2, Index, Vive wands and the simple controller. All XR handles stay +// opaque here so non-OpenXR consumers (bridge, tests) need no XR headers. +// +// Frame contract: Sync() runs exactly once per frame from the backend frame +// cycle, immediately after xrWaitFrame returned the predicted display time and +// before the guest renders, so located hand poses share the head/eye frame. +// There is no second frame wait and no polling from guest getters. +class XrActions final { +public: + XrActions() = default; + ~XrActions(); + XrActions(const XrActions &) = delete; + XrActions &operator=(const XrActions &) = delete; + + // Creates the action set/actions and suggests bindings. Instance-level: + // valid before any session exists. Failure is non-fatal for XR output; the + // backend logs and continues without controller input. + Status Create(void *xr_instance); + + // Attaches to a session and creates the per-hand pose spaces. Must be + // called after xrCreateSession and before the first Sync; recreated on every + // session replacement (no stale handles survive). + Status Attach(void *xr_instance, void *xr_session, void *xr_local_space); + + // Destroys spaces and detaches; the action set survives for reattach. + void Detach(); + void Destroy(); + + bool attached() const { return attached_; } + + // One per-frame sync at predicted_display_time. Locates grip and aim poses + // in the session's LOCAL space, reads analog/boolean states, refreshes + // connection state from the runtime's current interaction profile, and + // builds the coherent snapshot + delivery batch. Returns false when actions + // are unavailable (runtime refused sync); input is reset and a disconnect + // remains drainable even while the session is detached. + bool Sync(void *xr_session, std::uint64_t predicted_display_time, + std::uint64_t frame, std::uint64_t session_generation); + + // Latest coherent snapshot (frame/session generation tagged). + const ControllerSnapshot &snapshot() const { return snapshot_; } + // Hand poses in metres/LOCAL space for the state-copy records. + const VrHandPose &hand(int index) const { return hands_[index & 1]; } + + // Drains the next pending delivery batch built by Sync. Returns false when + // nothing is queued. Thread-safe: Sync runs on the render thread, the drain + // runs on the window thread. Backed by a bounded queue, so a slow consumer + // drops oldest frames and converges via resync rather than growing. + bool TakeDelivery(ControllerDelivery *out); + + // Queues a vibration request for one hand. Safe from any thread; applied on + // the next Sync's XR call window. amplitude [0,1], duration in nanoseconds + // (0 = runtime default). A newer request for the same hand replaces the + // pending one. StopHaptics cancels. + void RequestHaptics(int hand, float amplitude, std::uint64_t duration_ns, + float frequency_hz); + + void StopHaptics(int hand); + + // Interaction-profile diagnostics (refreshed on profile-change events and + // every Sync that observes a change). + std::string ActiveProfile(int hand) const; + + // Test seam: publish an externally sourced profile name (event dispatch in + // the backend) without XR calls. + void NoteProfileChanged(); + + // Called by the backend on focus/session loss, under controller ownership. + void ResetInput(); + +private: + struct Impl; + Impl *impl_ = nullptr; + bool created_ = false; + bool attached_ = false; + ControllerStateBuilder builder_; + ControllerSnapshot snapshot_; + VrHandPose hands_[2]; + ControllerDeliveryQueue delivery_queue_; + std::uint64_t session_generation_ = 0; + // Last connection level handed to the consumer, to mark connection changes. + bool last_delivered_connected_[2] = {false, false}; +}; + +} // namespace mocktail::vr + +#endif // MOCKTAIL_VR_XR_ACTIONS_H_ diff --git a/include/mocktail/vr/xr_controller.h b/include/mocktail/vr/xr_controller.h new file mode 100644 index 00000000..e9beb47f --- /dev/null +++ b/include/mocktail/vr/xr_controller.h @@ -0,0 +1,236 @@ +#ifndef MOCKTAIL_VR_XR_CONTROLLER_H_ +#define MOCKTAIL_VR_XR_CONTROLLER_H_ + +#include "mocktail/vr/xr_controller_abi.h" +#include +#include +#include +#include +#include +#include + +namespace mocktail::vr { + +// One tracked controller hand, located at the same predicted display time as +// the head. Grip is the held-object pose; aim is the pointing pose. They are +// not interchangeable: the guest consumes grip for hands/tools and aim for UI +// rays. Position is metres, orientation a quaternion (x, y, z, w) in the +// OpenXR LOCAL reference space. The guest applies its own metres-to-studs +// conversion and hand pitch offset, so neither is pre-applied here. +struct VrHandPose { + float grip_position[3] = {0.f, 0.f, 0.f}; + float grip_orientation[4] = {0.f, 0.f, 0.f, 1.f}; + float aim_position[3] = {0.f, 0.f, 0.f}; + float aim_orientation[4] = {0.f, 0.f, 0.f, 1.f}; + // Pose validity, reported separately from button activity and connection so + // a controller can be connected with an untracked pose. + bool grip_valid = false; + bool aim_valid = false; +}; + +// Engine-facing controller capability, independent of which OpenXR +// interaction profile supplied it. The guest receives buttons and analog axes +// through its verified Android gamepad route, so this layer normalizes every +// supported profile onto that one stable contract. +enum class ControllerButton { + kTrigger, // analog click semantics; value reported separately + kSqueeze, // grip/squeeze + kThumbstick, // stick click + kFaceNorth, // right B / left Y + kFaceSouth, // right A / left X + kFaceEast, // right B equivalent on layouts that have it + kFaceWest, // left X equivalent + kMenu, // application-accessible menu only; never a system button + kCount, +}; + +// Capacitive contacts are distinct from presses. Some runtimes/profiles do +// not expose them; active=false means unavailable, not an inferred contact. +enum class ControllerTouch { + kTrigger, + kThumbstick, + kThumbrest, + kA, + kB, + kX, + kY, + kCount +}; + +enum class ControllerAxis { + kTrigger, // 0..1 + kSqueeze, // 0..1 + kThumbstickX, // -1..1 + kThumbstickY, // -1..1 + kCount, +}; + +// Analog trigger/squeeze thresholds used only where the runtime exposes a +// boolean click for an analog control. Hysteresis prevents edge chattering at +// the boundary; native analog values are always preserved alongside it. +struct AnalogClickThresholds { + float press = 0.75f; + float release = 0.6f; +}; + +// One coherent per-frame controller snapshot. Built once per frame by the +// backend after a single xrSyncActions, then read by native getters without +// re-polling. Button booleans are level state; edge detection happens once, at +// publish time, so repeated native getters never emit duplicate edges. +struct ControllerHandState { + bool connected = false; + bool pose_valid = false; + std::array(ControllerTouch::kCount)> touches{}; + std::array(ControllerTouch::kCount)> + touch_active{}; + std::array(ControllerButton::kCount)> + buttons{}; + std::array(ControllerAxis::kCount)> axes{}; + // Interaction profile actually reported by the runtime for this hand, for + // diagnostics only (never used to gate input). + std::string profile; +}; + +struct ControllerSnapshot { + ControllerHandState hands[2]; // 0 = left, 1 = right + std::uint64_t frame = 0; + std::uint64_t session_generation = 0; +}; + +// Pure state machine: maps raw OpenXR action values onto the normalized +// contract and derives edges exactly once per published frame. No OpenXR or +// JNI types, so it is unit-testable with injected values. +class ControllerStateBuilder { +public: + explicit ControllerStateBuilder(AnalogClickThresholds thresholds = {}); + + // Records one raw action value for the given hand. Values not set by the + // runtime are neutral when explicitly marked inactive. + void SetTouch(int hand, ControllerTouch touch, bool value, bool active); + void SetButton(int hand, ControllerButton button, bool value, bool active); + void SetAxis(int hand, ControllerAxis axis, float value, bool active); + void SetConnected(int hand, bool connected); + void SetProfile(int hand, std::string profile); + void SetPoseValid(int hand, bool valid); + + // Produces the snapshot for this frame and records which buttons changed so + // the caller can emit exactly one native edge per transition. frame must + // advance monotonically; publishing the same frame twice yields no edges. + struct Edge { + int hand = 0; + ControllerButton button = ControllerButton::kTrigger; + bool pressed = false; + }; + ControllerSnapshot Publish(std::uint64_t frame, std::uint64_t generation, + std::vector *edges); + + // Neutralizes all axes and releases held buttons once (focus loss, + // disconnect, session stop). Produces release edges for anything held so no + // button can stay stuck down. + void ReleaseAll(std::uint64_t frame, std::uint64_t generation, + std::vector *edges); + + // Derives the click state for an analog control with hysteresis: latches on + // at `press`, off at `release`. Call after SetAxis for the same hand. Used + // for profiles that expose a trigger value without a click action; native + // analog values are always preserved regardless. + bool AnalogClick(int hand, ControllerAxis axis); + + const ControllerSnapshot &snapshot() const { return snapshot_; } + +private: + AnalogClickThresholds thresholds_; + ControllerSnapshot snapshot_; + // Last published level per hand/button, used for edge detection. + std::array< + std::array(ControllerButton::kCount)>, 2> + published_buttons_{}; + // Hysteresis latch per (hand, analog control): [hand*2 + axis]. + std::array analog_latch_{}; + bool have_published_ = false; +}; + +// Maps the normalized contract onto the guest's verified Android gamepad +// identifiers, so XR controllers reuse the single authoritative input route +// and cannot double-fire alongside a physical SDL gamepad. hand selects the +// per-hand squeeze/thumbstick/trigger identifiers (0 = left, 1 = right). +// Returns -1 when the control has no keycode (analog click). +int32_t ControllerButtonToAndroidKey(int hand, ControllerButton button); +// Returns the Android MotionEvent axis id for one axis of the given hand +// (left/right trigger differ), or -1 when the control has no analog axis. +int32_t ControllerAxisToAndroidAxis(int hand, ControllerAxis axis); +// How many float components the guest expects for this axis (sticks 2, triggers +// 1). +int32_t ControllerAxisComponentCount(ControllerAxis axis); + +// SDL3 gamepad ordinals. XR controller delivery reuses the existing router +// (RobloxInputRouter::HandleEvent) as the single authoritative event route: +// the router performs the verified SDL->Android key/axis conversion, level +// deduplication, focus gating and release-on-disconnect itself. Synthesized +// platform events must therefore carry SDL ordinals, not Android codes. +// hand selects the per-hand squeeze/thumbstick/trigger ordinals. +int32_t ControllerButtonToSdlOrdinal(int hand, ControllerButton button); +// SDL axis ordinal per hand for stick/trigger delivery; -1 when not an axis. +// Left hand: LEFTX/LEFTY/TRIGGERLEFT; right: RIGHTX/RIGHTY/TRIGGERRIGHT. +int32_t ControllerAxisToSdlOrdinal(int hand, ControllerAxis axis); + +// One drained delivery batch from the XR backend to the input route. Built +// once per frame after xrSyncActions; edges appear exactly once. `resync` +// requests full level re-delivery (recovery after queue overflow or an +// unfocused drop) — the router deduplicates, so re-delivery is idempotent and +// guarantees no button sticks down. +struct ControllerDelivery { + std::uint64_t session_generation = 0; + bool button_levels[2][static_cast(ControllerButton::kCount)] = + {}; + bool connected[2] = {false, false}; + bool connection_changed[2] = {false, false}; + struct ButtonEdge { + int hand = 0; + ControllerButton button = ControllerButton::kTrigger; + bool pressed = false; + }; + std::vector edges; + // SDL ordinal order: LEFTX, LEFTY, RIGHTX, RIGHTY, TRIGGERLEFT, TRIGGERRIGHT. + float axes[6] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; + bool axes_valid[2] = {false, false}; + bool resync = false; + std::uint64_t frame = 0; + + // Bounded growth guard: the backend caps the edge queue and sets resync + // instead of growing without limit. + static constexpr std::size_t kMaxEdges = 64; +}; + +bool EncodeNativeControllerChannels(const ControllerSnapshot &snapshot, + float (&channels)[28]); + +void EncodeControllerDelivery(const ControllerDelivery &delivery, + MocktailVrControllerDelivery *out); + +// Thread-safe bounded queue between the XR render-thread producer (one push +// per frame after xrSyncActions) and the window-thread consumer that drains +// into the verified input route. Overflow drops the oldest frames and arms a +// resync on the next delivered frame, so the consumer always converges to the +// current levels and a full queue can never leave a button stuck down. +class ControllerDeliveryQueue { +public: + static constexpr std::size_t kCapacity = 8; + + void Push(ControllerDelivery delivery); + bool Pop(ControllerDelivery *out); + std::size_t size() const; + bool overflowed() const { return overflowed_; } + std::uint64_t dropped_frames() const { return dropped_frames_; } + void Clear(); + +private: + mutable std::mutex mutex_; + std::deque queue_; + bool overflowed_ = false; + std::uint64_t dropped_frames_ = 0; +}; + +} // namespace mocktail::vr + +#endif // MOCKTAIL_VR_XR_CONTROLLER_H_ diff --git a/include/mocktail/vr/xr_controller_abi.h b/include/mocktail/vr/xr_controller_abi.h new file mode 100644 index 00000000..0e0569e7 --- /dev/null +++ b/include/mocktail/vr/xr_controller_abi.h @@ -0,0 +1,76 @@ +#ifndef MOCKTAIL_VR_XR_CONTROLLER_ABI_H_ +#define MOCKTAIL_VR_XR_CONTROLLER_ABI_H_ + +#include +#include + +/* Plain-C mirror of mocktail::vr::ControllerDelivery for crossing the + * dlsym module boundary (XR backend -> input runtime) without exporting C++ + * types or requiring the consumer to link OpenXR. Field order and types are + * the ABI; both sides include this header. Keep it POD: no constructors, no + * std, fixed arrays only. */ + +#ifdef __cplusplus +extern "C" { +#endif + +#define MOCKTAIL_VR_CONTROLLER_MAX_EDGES 64 +#define MOCKTAIL_VR_CONTROLLER_BUTTON_COUNT 8 +#define MOCKTAIL_VR_CONTROLLER_AXIS_COUNT 6 + +/* Button indices: mirror mocktail::vr::ControllerButton enum order. */ +enum { + MOCKTAIL_VR_BUTTON_TRIGGER = 0, + MOCKTAIL_VR_BUTTON_SQUEEZE = 1, + MOCKTAIL_VR_BUTTON_THUMBSTICK = 2, + MOCKTAIL_VR_BUTTON_FACE_NORTH = 3, + MOCKTAIL_VR_BUTTON_FACE_SOUTH = 4, + MOCKTAIL_VR_BUTTON_FACE_EAST = 5, + MOCKTAIL_VR_BUTTON_FACE_WEST = 6, + MOCKTAIL_VR_BUTTON_MENU = 7 +}; + +typedef struct MocktailVrControllerEdge { + int32_t hand; /* 0 = left, 1 = right */ + int32_t button; /* MOCKTAIL_VR_BUTTON_* */ + int32_t pressed; /* 0/1 */ +} MocktailVrControllerEdge; + +typedef struct MocktailVrControllerDelivery { + uint64_t frame; + uint64_t session_generation; + int32_t connected[2]; + int32_t connection_changed[2]; + /* Current button levels per hand (MOCKTAIL_VR_BUTTON_* order). Used for + * resync after queue overflow: the consumer re-delivers every level and the + * router's deduplication makes it idempotent. */ + int32_t button_levels[2][MOCKTAIL_VR_CONTROLLER_BUTTON_COUNT]; + /* Analog levels in SDL gamepad axis order: + * 0=LEFTX 1=LEFTY 2=RIGHTX 3=RIGHTY 4=TRIGGERLEFT 5=TRIGGERRIGHT. */ + float axes[MOCKTAIL_VR_CONTROLLER_AXIS_COUNT]; + int32_t axes_valid[2]; + int32_t resync; + int32_t edge_count; + MocktailVrControllerEdge edges[MOCKTAIL_VR_CONTROLLER_MAX_EDGES]; +} MocktailVrControllerDelivery; + +/* Pops the next delivery batch produced by the XR backend's once-per-frame + * action sync. Returns 1 and fills *out when a batch was queued, 0 when the + * queue is empty or no XR controller layer is active. Inert (returns 0) in + * non-VR builds and while no backend is armed. */ +typedef int32_t (*MocktailVrControllerPopFn)(MocktailVrControllerDelivery *out); + +/* Queues a vibration request for one hand (0=left, 1=right). amplitude is + * clamped to [0,1]; duration_ns 0 means the runtime default. Safe from any + * thread; applied on the next XR sync. No-op when no controller layer is + * active. */ +typedef void (*MocktailVrControllerHapticsFn)(int32_t hand, float amplitude, + uint64_t duration_ns, + float frequency_hz); +typedef void (*MocktailVrControllerStopHapticsFn)(int32_t hand); + +#ifdef __cplusplus +} +#endif + +#endif /* MOCKTAIL_VR_XR_CONTROLLER_ABI_H_ */ diff --git a/include/runtime/roblox_window_input_runtime.h b/include/runtime/roblox_window_input_runtime.h index 00c6c45d..eea21627 100644 --- a/include/runtime/roblox_window_input_runtime.h +++ b/include/runtime/roblox_window_input_runtime.h @@ -8,6 +8,7 @@ #include "mocktail/status.h" #include "runtime/roblox_input_native_adapter.h" #include "runtime/roblox_text_surface_overlay.h" +#include "runtime/roblox_xr_controller_input.h" namespace mocktail { namespace runtime { @@ -35,19 +36,39 @@ class RobloxWindowInputRuntime final { Status QueryCurrentTextBoxInfo(RobloxNativeTextBoxInfoQueryResult* result); Status UpdateTextFocusProperties(uint64_t generation, const RobloxTextFocusProperties& properties); + // Feeds a synthetic platform event through the same authoritative router as + // real SDL events. Used by the XR controller delivery path so tracked + // controllers reuse the verified gamepad route (single event source, router + // deduplication and focus gating) instead of a second JNI input path. + void InjectPlatformEvent(const platform::PlatformEvent& event); RobloxInputSnapshot Snapshot() const; + // Drains one batch of XR tracked-controller input into the same router. + // Called once per main-loop iteration on the input thread; inert in non-VR + // builds or while no OpenXR controller layer is active. + void DrainXrControllers(); + // Routes an engine/experience vibration request to the active XR hand. + void RequestXrControllerHaptics(int hand, float amplitude, + std::uint64_t duration_ns, float frequency_hz); + void StopXrControllerHaptics(int hand); + private: static void PlatformEventCallback(void* context, const platform::PlatformEvent& event); static bool MouseLockQueryCallback(void* context, bool* locked_center); static void GamepadEventCallback(void* context, const platform::PlatformEvent& event); + static void XrControllerEventCallback(void* context, + const platform::PlatformEvent& event); RobloxTextSurfaceOverlay text_surface_overlay_; std::unique_ptr text_clipboard_; RobloxInputRuntime runtime_; platform::SdlGamepadManager gamepads_; + // XR tracked-controller producer. Feeds the same runtime_ router as SDL so + // there is exactly one authoritative input route. + RobloxXrControllerInput xr_controllers_; + bool xr_controllers_initialized_ = false; bool observer_registered_ = false; bool mouse_lock_query_registered_ = false; bool initialized_ = false; diff --git a/include/runtime/roblox_xr_controller_input.h b/include/runtime/roblox_xr_controller_input.h new file mode 100644 index 00000000..7ebc4fa2 --- /dev/null +++ b/include/runtime/roblox_xr_controller_input.h @@ -0,0 +1,100 @@ +#ifndef MOCKTAIL_RUNTIME_ROBLOX_XR_CONTROLLER_INPUT_H_ +#define MOCKTAIL_RUNTIME_ROBLOX_XR_CONTROLLER_INPUT_H_ + +#include + +#include "mocktail/platform/platform_runtime.h" +#include "mocktail/status.h" +#include "mocktail/vr/xr_controller_abi.h" + +namespace mocktail { +namespace runtime { + +// Delivers OpenXR tracked-controller input through the one authoritative +// Roblox input route (RobloxInputRouter::HandleEvent), the same verified path +// physical SDL gamepads use. Because both sources converge on that single +// route, the router's own level deduplication, focus gating and +// release-on-disconnect logic apply unchanged and an XR controller can never +// double-fire alongside a physical gamepad. +// +// Both hands are merged into ONE synthetic gamepad: the left hand supplies +// LEFTX/LEFTY, the left shoulder (squeeze) and left stick click plus X/Y, the +// right hand supplies RIGHTX/RIGHTY, the right shoulder, right stick click and +// A/B. That matches how Roblox reads VR controller buttons through +// GamepadService, and it means the guest sees a single normal controller +// rather than two competing devices. +// +// The XR backend is resolved through dlsym so this module links and runs in +// non-VR builds and while no OpenXR backend is armed; then every drain is a +// no-op. No OpenXR headers are required here. +class RobloxXrControllerInput final { +public: + using EventSink = void (*)(void *context, + const platform::PlatformEvent &event); + + // Synthetic device identity. Deliberately far outside the SDL joystick id + // range so it can never collide with a physical controller. + static constexpr std::int64_t kInstanceLeft = + static_cast(0x7FFFFFFFFFFFFFF0LL); + + RobloxXrControllerInput() = default; + ~RobloxXrControllerInput() = default; + RobloxXrControllerInput(const RobloxXrControllerInput &) = delete; + RobloxXrControllerInput &operator=(const RobloxXrControllerInput &) = delete; + + // Resolves the XR controller ABI. Returns false (and stays inert) when the + // symbols are absent, which is the normal non-VR or no-backend case; that is + // not an error condition. + bool Initialize(EventSink sink, void *context); + // Test/injection seam: binds explicit ABI function pointers instead of + // resolving them through dlsym. A null pop makes the module inert, matching + // the non-VR case. Production callers use Initialize(). + bool InitializeWithAbi(EventSink sink, void *context, + MocktailVrControllerPopFn pop, + MocktailVrControllerHapticsFn haptics, + MocktailVrControllerStopHapticsFn stop_haptics); + void Shutdown(); + bool active() const { return pop_ != nullptr; } + + // Drains every queued delivery and forwards it as platform events. Call once + // per main-loop iteration, after window::PumpEvents(), on the same thread + // that delivers SDL input. Bounded: drains at most kMaxDrainPerPump batches. + void Drain(); + void ResendState() { force_resync_ = true; } + + // Forwards a vibration request to the XR hand output. hand: 0 left, 1 right. + // No-op when the XR layer is inactive. amplitude is clamped by the backend. + void RequestHaptics(int hand, float amplitude, std::uint64_t duration_ns, + float frequency_hz); + void StopHaptics(int hand); + + // Last observed state, for diagnostics/tests. + bool connected() const { return connected_; } + std::uint64_t delivered_events() const { return delivered_events_; } + std::uint64_t resyncs() const { return resyncs_; } + +private: + static constexpr int kMaxDrainPerPump = 4; + void EmitConnection(bool connected); + void EmitButton(int sdl_button, bool pressed); + void EmitAxis(int sdl_axis, std::int16_t value); + + void *context_ = nullptr; + EventSink sink_ = nullptr; + void *pop_ = nullptr; + void *haptics_ = nullptr; + void *stop_haptics_ = nullptr; + bool force_resync_ = false; + std::uint64_t generation_ = 0; + bool connected_ = false; + bool initialized_ = false; + std::uint64_t delivered_events_ = 0; + std::uint64_t resyncs_ = 0; + // Last delivered button levels, for resync and to avoid redundant events. + bool last_buttons_[16] = {}; +}; + +} // namespace runtime +} // namespace mocktail + +#endif // MOCKTAIL_RUNTIME_ROBLOX_XR_CONTROLLER_INPUT_H_ diff --git a/src/legacy/legacy_runtime.cc b/src/legacy/legacy_runtime.cc index 157ad947..6bfa5de9 100644 --- a/src/legacy/legacy_runtime.cc +++ b/src/legacy/legacy_runtime.cc @@ -6826,6 +6826,12 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, while (true) { const uint64_t fps_tick_start_ns = fps_trace ? MonotonicNanos() : 0; const bool keep_window_open = mocktail::window::PumpEvents(); + // Deliver XR tracked-controller input on the same thread and through the + // same router as SDL input, once per loop iteration after the SDL pump. + // Inert in non-VR builds or while no OpenXR controller layer is active. + if (window_input_runtime != nullptr) { + window_input_runtime->DrainXrControllers(); + } game_surface_events_completed = drain_game_surface_events(); const uint64_t fps_after_sdl_ns = fps_trace ? MonotonicNanos() : 0; if (!game_surface_events_completed || !keep_window_open) { diff --git a/src/runtime/roblox_window_input_runtime.cc b/src/runtime/roblox_window_input_runtime.cc index fefb6ed3..3f2de548 100644 --- a/src/runtime/roblox_window_input_runtime.cc +++ b/src/runtime/roblox_window_input_runtime.cc @@ -80,6 +80,14 @@ Status RobloxWindowInputRuntime::Initialize() { gamepad_status.message().c_str()); } } + // XR tracked controllers join the same authoritative route. Initialize is a + // no-op (false) in non-VR builds or without an armed OpenXR backend; that is + // the normal state, not an error. + xr_controllers_initialized_ = xr_controllers_.Initialize( + &RobloxWindowInputRuntime::XrControllerEventCallback, this); + if (xr_controllers_initialized_) { + std::fprintf(stderr, " [vr-input] XR controller delivery armed\n"); + } return Status::Ok(); } @@ -108,6 +116,7 @@ void RobloxWindowInputRuntime::PlatformEventCallback( std::get_if(&event.payload)) { if (focus->focused) { self->gamepads_.ResendState(); + self->xr_controllers_.ResendState(); } } } @@ -124,6 +133,10 @@ Status RobloxWindowInputRuntime::Shutdown() { window::ClearMouseLockQueryCallback(); mouse_lock_query_registered_ = false; } + // Disconnect the synthetic XR controller while the router still processes + // events, so its release path runs through the normal route. + xr_controllers_.Shutdown(); + xr_controllers_initialized_ = false; gamepads_.Shutdown(); const RobloxInputSnapshot snapshot = runtime_.Snapshot(); Status status = runtime_.Deactivate(); @@ -199,6 +212,37 @@ RobloxInputSnapshot RobloxWindowInputRuntime::Snapshot() const { return runtime_.Snapshot(); } +void RobloxWindowInputRuntime::InjectPlatformEvent( + const platform::PlatformEvent& event) { + // Same router and same thread as SDL delivery: the XR controller path is a + // producer of platform events, not a parallel input stack. + (void)runtime_.HandleEvent(event); +} + +void RobloxWindowInputRuntime::XrControllerEventCallback( + void* context, const platform::PlatformEvent& event) { + auto* self = static_cast(context); + if (self != nullptr) { + self->InjectPlatformEvent(event); + } +} + +void RobloxWindowInputRuntime::DrainXrControllers() { + if (!xr_controllers_initialized_) { + return; + } + xr_controllers_.Drain(); +} + +void RobloxWindowInputRuntime::RequestXrControllerHaptics( + int hand, float amplitude, std::uint64_t duration_ns, float frequency_hz) { + xr_controllers_.RequestHaptics(hand, amplitude, duration_ns, frequency_hz); +} + +void RobloxWindowInputRuntime::StopXrControllerHaptics(int hand) { + xr_controllers_.StopHaptics(hand); +} + bool RobloxWindowInputRuntime::MouseLockQueryCallback(void* context, bool* locked_center) { if (context == nullptr || locked_center == nullptr) { diff --git a/src/runtime/roblox_xr_controller_input.cc b/src/runtime/roblox_xr_controller_input.cc new file mode 100644 index 00000000..e3afdd1d --- /dev/null +++ b/src/runtime/roblox_xr_controller_input.cc @@ -0,0 +1,272 @@ +#include "runtime/roblox_xr_controller_input.h" + +#include + +#include +#include +#include + +#include "mocktail/vr/xr_controller.h" +#include "mocktail/vr/xr_controller_abi.h" + +namespace mocktail { +namespace runtime { +namespace { + +// SDL3 gamepad ordinals for the merged synthetic controller. Both hands feed +// one device: left hand -> left stick/left shoulder/X/Y, right hand -> right +// stick/right shoulder/A/B; menu -> START. +constexpr std::int16_t kAxisFull = 32767; + +std::int16_t FloatToRaw(float value) { + const float clamped = + std::clamp(std::isfinite(value) ? value : 0.f, -1.f, 1.f); + return static_cast(std::lround(clamped * kAxisFull)); +} + +std::int16_t TriggerToRaw(float value) { + const float clamped = + std::clamp(std::isfinite(value) ? value : 0.f, 0.f, 1.f); + return static_cast(std::lround(clamped * kAxisFull)); +} + +std::uint64_t NowNs() { + timespec ts{}; + clock_gettime(CLOCK_MONOTONIC, &ts); + return static_cast(ts.tv_sec) * 1000000000ULL + + static_cast(ts.tv_nsec); +} + +} // namespace + +bool RobloxXrControllerInput::Initialize(EventSink sink, void *context) { + if (sink == nullptr) { + return false; + } + // Resolve the XR controller ABI through dlsym so this module links and runs + // in non-VR builds (symbols absent) and stays inert while no backend is + // armed. Never a hard dependency on OpenXR here. + auto pop = reinterpret_cast( + dlsym(RTLD_DEFAULT, "mocktail_vr_controller_pop_delivery")); + auto haptics = reinterpret_cast( + dlsym(RTLD_DEFAULT, "mocktail_vr_controller_request_haptics")); + auto stop_haptics = reinterpret_cast( + dlsym(RTLD_DEFAULT, "mocktail_vr_controller_stop_haptics")); + return InitializeWithAbi(sink, context, pop, haptics, stop_haptics); +} + +bool RobloxXrControllerInput::InitializeWithAbi( + EventSink sink, void *context, MocktailVrControllerPopFn pop, + MocktailVrControllerHapticsFn haptics, + MocktailVrControllerStopHapticsFn stop_haptics) { + if (sink == nullptr || pop == nullptr) { + // No XR controller layer (non-VR build or backend without controllers). + // This is the normal case for --no-vr and must not be treated as failure. + sink_ = nullptr; + context_ = nullptr; + pop_ = nullptr; + haptics_ = nullptr; + stop_haptics_ = nullptr; + initialized_ = false; + return false; + } + sink_ = sink; + context_ = context; + pop_ = reinterpret_cast(pop); + haptics_ = reinterpret_cast(haptics); + stop_haptics_ = reinterpret_cast(stop_haptics); + initialized_ = true; + return true; +} + +void RobloxXrControllerInput::Shutdown() { + if (connected_) { + // Release the synthetic controller once so no button or stick stays live + // after teardown. + EmitConnection(false); + } + initialized_ = false; + generation_ = 0; + force_resync_ = false; + sink_ = nullptr; + context_ = nullptr; + pop_ = nullptr; + haptics_ = nullptr; + stop_haptics_ = nullptr; +} + +void RobloxXrControllerInput::EmitConnection(bool connected) { + if (sink_ == nullptr) { + return; + } + platform::GamepadDescriptor descriptor; + descriptor.family = platform::GamepadFamily::kXbox; + descriptor.name = connected ? "Mocktail VR Controllers" : ""; + if (connected) { + // Advertise exactly the buttons/axes this route can emit so the router + // accepts them: SOUTH/EAST/WEST/NORTH, START, both stick clicks, both + // shoulders. + for (const std::uint8_t button : + {static_cast(vr::ControllerButtonToSdlOrdinal( + 0, vr::ControllerButton::kFaceWest)), + static_cast(vr::ControllerButtonToSdlOrdinal( + 0, vr::ControllerButton::kFaceNorth)), + static_cast(vr::ControllerButtonToSdlOrdinal( + 1, vr::ControllerButton::kFaceSouth)), + static_cast(vr::ControllerButtonToSdlOrdinal( + 1, vr::ControllerButton::kFaceEast)), + static_cast( + vr::ControllerButtonToSdlOrdinal(0, vr::ControllerButton::kMenu)), + static_cast(vr::ControllerButtonToSdlOrdinal( + 0, vr::ControllerButton::kThumbstick)), + static_cast(vr::ControllerButtonToSdlOrdinal( + 1, vr::ControllerButton::kThumbstick)), + static_cast(vr::ControllerButtonToSdlOrdinal( + 0, vr::ControllerButton::kSqueeze)), + static_cast(vr::ControllerButtonToSdlOrdinal( + 1, vr::ControllerButton::kSqueeze))}) { + descriptor.buttons |= (std::uint32_t{1} << button); + } + // Six axes: both sticks (x/y) and both triggers. + descriptor.axes = 0b111111; + } + platform::GamepadConnectionEvent event; + event.instance_id = kInstanceLeft; + event.connected = connected; + event.descriptor = std::move(descriptor); + event.remapped = false; + sink_(context_, platform::PlatformEvent{NowNs(), std::move(event)}); + connected_ = connected; + std::fill(std::begin(last_buttons_), std::end(last_buttons_), false); + if (connected) { + std::fprintf(stderr, " [vr-input] XR controllers connected to Roblox as a " + "single synthetic gamepad\n"); + } else { + std::fprintf(stderr, + " [vr-input] XR controllers disconnected from Roblox\n"); + } +} + +void RobloxXrControllerInput::EmitButton(int sdl_button, bool pressed) { + if (sink_ == nullptr || sdl_button < 0) { + return; + } + platform::GamepadButtonEvent event; + event.instance_id = kInstanceLeft; + event.button = static_cast(sdl_button); + event.pressed = pressed; + sink_(context_, platform::PlatformEvent{NowNs(), std::move(event)}); + ++delivered_events_; +} + +void RobloxXrControllerInput::EmitAxis(int sdl_axis, std::int16_t value) { + if (sink_ == nullptr || sdl_axis < 0) { + return; + } + platform::GamepadAxisEvent event; + event.instance_id = kInstanceLeft; + event.axis = static_cast(sdl_axis); + event.value = value; + sink_(context_, platform::PlatformEvent{NowNs(), std::move(event)}); + ++delivered_events_; +} + +void RobloxXrControllerInput::Drain() { + if (!initialized_ || pop_ == nullptr || sink_ == nullptr) { + return; + } + auto pop = reinterpret_cast(pop_); + for (int batch = 0; batch < kMaxDrainPerPump; ++batch) { + MocktailVrControllerDelivery delivery{}; + if (pop(&delivery) == 0) { + return; + } + if (delivery.session_generation < generation_) + continue; + if (delivery.session_generation != generation_) { + if (connected_) + EmitConnection(false); + generation_ = delivery.session_generation; + } + // Connection transitions first so subsequent button/axis events land on a + // registered device. Either hand being connected presents the merged + // controller as connected. + const bool connected = delivery.connected[0] || delivery.connected[1]; + if (connected != connected_) { + EmitConnection(connected); + } + if (!connected_) { + // Nothing to deliver while disconnected; the disconnect already released + // every control through the router. + continue; + } + + if (force_resync_) { + delivery.resync = 1; + force_resync_ = false; + } + if (delivery.resync) + ++resyncs_; + // Levels belong to this delivery, not a separately read newer snapshot. + // Merge first so a release/unbound button on one hand cannot cancel the + // other. + bool merged[16] = {}; + for (int hand = 0; hand < 2; ++hand) { + if (!delivery.connected[hand]) + continue; + for (int index = 0; index < MOCKTAIL_VR_CONTROLLER_BUTTON_COUNT; + ++index) { + const int button = vr::ControllerButtonToSdlOrdinal( + hand, static_cast(index)); + if (button >= 0 && button < 16) + merged[button] |= delivery.button_levels[hand][index] != 0; + } + } + for (int button = 0; button < 16; ++button) { + if (delivery.resync || merged[button] != last_buttons_[button]) + EmitButton(button, merged[button]); + last_buttons_[button] = merged[button]; + } + + // Analog axes. Delivered as raw int16 so the router applies its single + // deadzone; we never pre-filter (avoids a double deadzone). + if (delivery.axes_valid[0] || delivery.axes_valid[1]) { + EmitAxis( + vr::ControllerAxisToSdlOrdinal(0, vr::ControllerAxis::kThumbstickX), + FloatToRaw(delivery.axes[0])); + EmitAxis( + vr::ControllerAxisToSdlOrdinal(0, vr::ControllerAxis::kThumbstickY), + FloatToRaw(delivery.axes[1])); + EmitAxis( + vr::ControllerAxisToSdlOrdinal(1, vr::ControllerAxis::kThumbstickX), + FloatToRaw(delivery.axes[2])); + EmitAxis( + vr::ControllerAxisToSdlOrdinal(1, vr::ControllerAxis::kThumbstickY), + FloatToRaw(delivery.axes[3])); + EmitAxis(vr::ControllerAxisToSdlOrdinal(0, vr::ControllerAxis::kTrigger), + TriggerToRaw(delivery.axes[4])); + EmitAxis(vr::ControllerAxisToSdlOrdinal(1, vr::ControllerAxis::kTrigger), + TriggerToRaw(delivery.axes[5])); + } + } +} + +void RobloxXrControllerInput::RequestHaptics(int hand, float amplitude, + std::uint64_t duration_ns, + float frequency_hz) { + if (haptics_ == nullptr || (hand != 0 && hand != 1)) { + return; + } + reinterpret_cast(haptics_)( + hand, amplitude, duration_ns, frequency_hz); +} + +void RobloxXrControllerInput::StopHaptics(int hand) { + if (stop_haptics_ == nullptr || (hand != 0 && hand != 1)) { + return; + } + reinterpret_cast(stop_haptics_)(hand); +} + +} // namespace runtime +} // namespace mocktail diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index b468dfb0..0b21d725 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -366,6 +366,17 @@ Status OpenXrBackend::Arm(VrGraphicsApi graphics_api) { gl_min_version_ = requirements.minApiVersionSupported; gl_max_version_ = requirements.maxApiVersionSupported; } + // Controller actions are instance-level: create them once here so every + // session (including a recreated one) can attach without re-suggesting + // bindings. A failure degrades to no controller input, not to no XR. + std::lock_guard controller_lock(controller_mutex_); + const Status actions_status = actions_.Create(instance); + actions_created_ = actions_status.ok(); + if (!actions_created_) { + Log(" [vr-backend] controller actions unavailable: %s (XR output " + "continues without controller input)\n", + actions_status.message().c_str()); + } armed_.store(true, std::memory_order_release); Log(" [vr-backend] armed: OpenXR instance/system ready; awaiting guest " "%s\n", @@ -385,6 +396,9 @@ void OpenXrBackend::Disarm() { void OpenXrBackend::DisarmLocked() { TeardownSessionLocked("disarm"); + { std::lock_guard lock(controller_mutex_); + actions_.Destroy(); + actions_created_ = false; } if (gles_) gles_->Destroy(); delete gles_; @@ -868,6 +882,17 @@ bool OpenXrBackend::InitializeSessionResourcesLocked(std::string *error) { TeardownSessionLocked("swapchain failure", session_recovery_pending_); return false; } + // Attach the action set to this session and create its pose spaces. A + // failure leaves XR output working without controller input. + std::lock_guard controller_lock(controller_mutex_); + if (actions_created_) { + const Status attach_status = + actions_.Attach(xr_instance_, xr_session_, xr_local_space_); + if (!attach_status.ok()) { + Log(" [vr-backend] controller actions not attached: %s\n", + attach_status.message().c_str()); + } + } recording_.store(true, std::memory_order_release); return true; } catch (const std::exception &exception) { @@ -1019,6 +1044,9 @@ void OpenXrBackend::TeardownSessionLocked(const char* reason, bool preserve_devi mirror_.Close(); DestroyCopyResources(); DestroySwapchains(); + // Action pose spaces belong to this session; detach before the session and + // its spaces go away so a recreated session attaches cleanly. + { std::lock_guard lock(controller_mutex_); actions_.Detach(); } if (xr_view_space_ != nullptr) { (void)xrDestroySpace(static_cast(xr_view_space_)); xr_view_space_ = nullptr; @@ -1326,6 +1354,37 @@ ScriptedPoseSample OpenXrBackend::PublishedHeadPose() const { return published_pose_; } +bool OpenXrBackend::TakeControllerDelivery(ControllerDelivery* out) { + if (out == nullptr) { + return false; + } + // controller_mutex_ (never mutex_): the frame cycle holds mutex_ across + // xrWaitFrame runtime pacing, and this drain runs on the window thread. + std::lock_guard lock(controller_mutex_); + return actions_.TakeDelivery(out); +} + +ControllerSnapshot OpenXrBackend::ControllerSnapshotForFrame() const { + std::lock_guard lock(controller_mutex_); + return actions_.snapshot(); +} + +void OpenXrBackend::RequestControllerHaptics(int hand, float amplitude, + std::uint64_t duration_ns, + float frequency_hz) { + std::lock_guard lock(controller_mutex_); + if (actions_created_) { + actions_.RequestHaptics(hand, amplitude, duration_ns, frequency_hz); + } +} + +void OpenXrBackend::StopControllerHaptics(int hand) { + std::lock_guard lock(controller_mutex_); + if (actions_created_) { + actions_.StopHaptics(hand); + } +} + void OpenXrBackend::PollSessionEvents() { if (xr_instance_ == nullptr) { return; @@ -1364,10 +1423,8 @@ void OpenXrBackend::PollSessionEvents() { continue; } if (event.type == XR_TYPE_EVENT_DATA_INTERACTION_PROFILE_CHANGED) { - // Controllers are out of scope; recorded so the log shows the runtime - // offered a profile change rather than silently ignoring it. - Log(" [vr-backend] interaction profile changed (controller input is not " - "implemented in this build)\n"); + { std::lock_guard lock(controller_mutex_); actions_.NoteProfileChanged(); } + Log(" [vr-backend] controller interaction profile changed\n"); continue; } if (event.type != XR_TYPE_EVENT_DATA_SESSION_STATE_CHANGED) { @@ -1384,6 +1441,14 @@ void OpenXrBackend::PollSessionEvents() { } void OpenXrBackend::HandleSessionState(int state) { + if (state != XR_SESSION_STATE_FOCUSED) { + std::lock_guard lock(controller_mutex_); + actions_.ResetInput(); + published_pose_.hands[0] = {}; + published_pose_.hands[1] = {}; + published_pose_.controllers_connected = false; + std::fill(std::begin(published_pose_.controller_channels), std::end(published_pose_.controller_channels), 0.f); + } switch (static_cast(state)) { case XR_SESSION_STATE_READY: { auto begin = @@ -1897,6 +1962,44 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { } else { published_pose_.valid = false; } + + // Sync controllers exactly once per frame at the same predicted display time + // used for the head/eyes, then fold the resulting hand poses into the + // already-published frame. This adds no second frame wait and no per-getter + // polling; the device bridge reads published_pose_.hands when it injects the + // state for this frame. + SyncControllersLocked(predicted_display_time); +} + +void OpenXrBackend::SyncControllersLocked(std::uint64_t predicted_display_time) { + std::lock_guard lock(controller_mutex_); + published_pose_.hands[0] = {}; + published_pose_.hands[1] = {}; + published_pose_.controllers_connected = false; + std::fill(std::begin(published_pose_.controller_channels), std::end(published_pose_.controller_channels), 0.f); + if (!actions_created_ || !actions_.attached()) { + return; + } + const ScopedStageTimer sync_timer(perf::Stage::kControllerSync); + const bool synced = actions_.Sync(xr_session_, predicted_display_time, + published_pose_.frame, 0); + if (!synced) { + // Hands were invalidated above; Sync queued the input reset. + return; + } + // Fold this frame's hand poses into the published sample. Only valid when + // the head pose for this frame is itself valid, so hands and head always + // describe the same predicted instant. + if (published_pose_.valid) { + published_pose_.hands[0] = actions_.hand(0); + published_pose_.hands[1] = actions_.hand(1); + published_pose_.controllers_connected = EncodeNativeControllerChannels( + actions_.snapshot(), published_pose_.controller_channels); + } else { + published_pose_.hands[0] = VrHandPose{}; + published_pose_.hands[1] = VrHandPose{}; + } + perf::ProcessCollector().Count(perf::Counter::kControllerSyncs); } bool OpenXrBackend::EnsureCopyResources(VkDevice device, @@ -2560,6 +2663,48 @@ void mocktail_vr_xr_note_present(VkQueue queue, VkDevice device) { } // extern "C" +// Controller delivery ABI consumed by the input runtime through dlsym. These +// are defined only in the VR build; a non-VR build omits them and the consumer +// resolves nullptr, staying inert. +#include "mocktail/vr/xr_controller_abi.h" + +extern "C" int32_t +mocktail_vr_controller_pop_delivery(MocktailVrControllerDelivery* out) { + if (out == nullptr) { + return 0; + } + auto* backend = mocktail::vr::ActiveVrBackend(); + if (backend == nullptr) { + return 0; + } + // Exact native VR input owns Gamepad1 when the device bridge is active. + // The JNI fallback must not inject a second copy into that same device. + extern bool mocktail_vr_native_controller_input_active(); + if (mocktail_vr_native_controller_input_active()) return 0; + mocktail::vr::ControllerDelivery delivery; + if (!backend->TakeControllerDelivery(&delivery)) { + return 0; + } + mocktail::vr::EncodeControllerDelivery(delivery, out); + return 1; +} + +extern "C" void mocktail_vr_controller_request_haptics(int32_t hand, + float amplitude, + uint64_t duration_ns, + float frequency_hz) { + if (auto* backend = mocktail::vr::ActiveVrBackend()) { + backend->RequestControllerHaptics(hand, amplitude, duration_ns, + frequency_hz); + } +} + +extern "C" void mocktail_vr_controller_stop_haptics(int32_t hand) { + if (auto* backend = mocktail::vr::ActiveVrBackend()) { + backend->StopControllerHaptics(hand); + } +} + extern "C" bool mocktail_vr_attach_egl(void *display, void *config, void *context, void *egl_get_proc, void *(*resolve)(const char *)) { diff --git a/src/vr/openxr_disabled.cc b/src/vr/openxr_disabled.cc index 3c75de6d..28470a00 100644 --- a/src/vr/openxr_disabled.cc +++ b/src/vr/openxr_disabled.cc @@ -8,6 +8,8 @@ Status OpenXrBackend::Arm(VrGraphicsApi) { return Status::Error(StatusCode::kUnavailable, "OpenXR support is disabled in this build"); } void OpenXrBackend::Disarm() {} +void OpenXrBackend::RequestControllerHaptics(int, float, std::uint64_t, float) {} +void OpenXrBackend::StopControllerHaptics(int) {} OpenXrBackend* ActiveVrBackend() { return nullptr; } void OpenXrBackend::NotePoseApplied(void*, std::uint64_t) {} ScriptedPoseSample OpenXrBackend::PublishedHeadPose() const { return {}; } diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc index ea30375b..5a58cde6 100644 --- a/src/vr/roblox_vr_device_bridge.cc +++ b/src/vr/roblox_vr_device_bridge.cc @@ -11,6 +11,7 @@ #include #include #include "mocktail/vr/openxr_backend.h" +#include "mocktail/vr/vr_pose_math.h" #include #include #include @@ -50,6 +51,137 @@ thread_local void *g_eye_object = nullptr; thread_local void *g_eye_framebuffer = nullptr; thread_local int g_eye_index = -1; thread_local bool g_thread_presented = false; +std::mutex g_native_pose_mutex; +ScriptedPoseSample g_native_pose; +using PointerFrameFn = void *(*)(void *, void *); +std::atomic g_pointer_original{nullptr}; +constexpr std::uintptr_t kPointerFrameRva = 0x4d573ee; +// Whole instructions, no RIP-relative operands. 2998's sret prologue. +constexpr std::array kPointerPrologue = { + 0x55, 0x48, 0x89, 0xe5, 0x41, 0x57, 0x41, 0x56, + 0x41, 0x54, 0x53, 0x48, 0x83, 0xec, 0x70}; +std::uintptr_t g_pointer_target = 0; + +void AbsoluteJump(unsigned char *code, std::uintptr_t target) { + const unsigned char opcode[6] = {0xff, 0x25, 0, 0, 0, 0}; + std::memcpy(code, opcode, 6); + std::memcpy(code + 6, &target, 8); +} + +extern "C" void *mocktail_vr_pointer_frame(void *result, void *service) { + auto original = g_pointer_original.load(std::memory_order_acquire); + if (!original) + return result; + original(result, service); + auto *bridge = g_active_bridge.load(std::memory_order_acquire); + if (!bridge || !bridge->active()) return result; + ScriptedPoseSample native_pose; + { + std::lock_guard lock(g_native_pose_mutex); + native_pose = g_native_pose; + } + if (!native_pose.valid) return result; + auto *backend = ActiveVrBackend(); + if (!backend) + return result; + const auto current = backend->PublishedHeadPose(); + if (!current.valid) + return result; + int index = 0; + std::memcpy(&index, static_cast(service) + 0x118, 4); + if (index != 1 && index != 2) + return result; + if (!current.hands[index - 1].aim_valid) + return result; + const auto &hand = native_pose.hands[index - 1]; + if (!hand.grip_valid || !hand.aim_valid) + return result; + const auto base = bridge->image_base(); + // Same camera lookup as the original getter (see pointer-world-2998.asm). + auto get_workspace = reinterpret_cast(base + 0x263778c); + void *workspace = get_workspace(service); + if (!workspace) + return result; + auto *vtable = *static_cast(workspace); + void *camera = + reinterpret_cast(vtable[0x3a8 / 8])(workspace); + if (!camera) + return result; + float scale = 0; + int degrees = 0; + std::memcpy(&scale, static_cast(camera) + 0x140, 4); + std::memcpy(°rees, reinterpret_cast(base + 0x73677a8), 4); + if (internal::ApplyAimToWorldFrame(static_cast(result), hand, scale, + -static_cast(degrees))) { + static std::atomic logged[2]{}; + if (!logged[index - 1].exchange(true)) + std::fprintf( + stderr, " [vr-input] native menu pointer uses OpenXR aim: hand=%d\n", + index - 1); + } + return result; +} + +bool SetCodeWritable(std::uintptr_t target, bool writable) { + const auto page_size = static_cast(sysconf(_SC_PAGESIZE)); + if (!page_size) + return false; + const auto first = target & ~(page_size - 1); + const auto end = + (target + kPointerPrologue.size() + page_size - 1) & ~(page_size - 1); + return mprotect(reinterpret_cast(first), end - first, + PROT_READ | PROT_EXEC | (writable ? PROT_WRITE : 0)) == 0; +} + +bool InstallPointerHook(std::uintptr_t base) { + if (g_pointer_target) + return false; + auto *target = reinterpret_cast(base + kPointerFrameRva); + if (std::memcmp(target, kPointerPrologue.data(), kPointerPrologue.size()) != + 0) + return false; + const auto page_size = static_cast(sysconf(_SC_PAGESIZE)); + auto *code = static_cast( + mmap(nullptr, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + if (code == MAP_FAILED) + return false; + std::memcpy(code, target, kPointerPrologue.size()); + AbsoluteJump(code + kPointerPrologue.size(), + base + kPointerFrameRva + kPointerPrologue.size()); + if (mprotect(code, page_size, PROT_READ | PROT_EXEC) != 0 || + !SetCodeWritable(base + kPointerFrameRva, true)) { + munmap(code, page_size); + return false; + } + g_pointer_original.store(reinterpret_cast(code), + std::memory_order_release); + AbsoluteJump(target, + reinterpret_cast(&mocktail_vr_pointer_frame)); + target[14] = 0x90; + if (!SetCodeWritable(base + kPointerFrameRva, false)) { + std::memcpy(target, kPointerPrologue.data(), kPointerPrologue.size()); + (void)SetCodeWritable(base + kPointerFrameRva, false); + return false; + } + // Retain this single RX page for process lifetime: an in-flight guest call + // may still be returning through it when the entrypoint is restored. + g_pointer_target = base + kPointerFrameRva; + return true; +} + +bool RestorePointerHook() { + if (!g_pointer_target) + return true; + if (!SetCodeWritable(g_pointer_target, true)) + return false; + std::memcpy(reinterpret_cast(g_pointer_target), + kPointerPrologue.data(), kPointerPrologue.size()); + const bool restored = SetCodeWritable(g_pointer_target, false); + if (restored) + g_pointer_target = 0; + return restored; +} Status InvalidArgument(std::string message) { return Status::Error(StatusCode::kInvalidArgument, std::move(message)); @@ -339,6 +471,11 @@ namespace { // result buffer, rsi = device object, returns the buffer) and the eye getter // takes (rdi = device object, esi = eye index) and returns the framebuffer // object pointer. +extern "C" void mocktail_vr_debug_device_haptics(void* object, int hand, float amplitude) { + if (auto* bridge = g_active_bridge.load(std::memory_order_acquire)) + bridge->OnHapticsCall(object, hand, amplitude); +} + extern "C" void *mocktail_vr_debug_device_state_getter(void *result_buffer, void *device_object) { RobloxVrDeviceBridge *bridge = @@ -408,6 +545,8 @@ Status RobloxVrDeviceBridge::Install(const compat::BuildProfile &profile) { "experimental Roblox VR requires vr_debug_device_bridge metadata in " "the exact-build compatibility profile"); } + if (profile.elf_build_id != "ade08266c67aee88ec9c1d00902150e1684dad3a") + return FailedPrecondition("VR controller/pointer/haptic contracts are verified only for Build 2998"); if (!profile.allow_host_abi_bridges) { return FailedPrecondition( "Roblox VR device bridge profile requires host ABI bridges"); @@ -507,14 +646,25 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { "match the profile"); } + if (!IsExecutableImageRange(image_base, kPointerFrameRva, 0x1b2) || + std::memcmp(reinterpret_cast(image_base + kPointerFrameRva), + kPointerPrologue.data(), kPointerPrologue.size()) != 0) + return FailedPrecondition("2998 VR pointer function contract mismatch"); image_base_.store(image_base, std::memory_order_relaxed); image_size_.store(ImageSize(image_base), std::memory_order_relaxed); vtable_ = reinterpret_cast(image_base + profile_.vtable_rva); + // 2998's haptic sink is a verified no-op (ret at 0x1d48b70). + // The caller at 0x28a767e passes self, hand 0/1 and float amplitude. + if (vtable_[3] != image_base + 0x1d48b70 || + !IsExecutableImageRange(image_base, 0x1d48b70, 1) || + *reinterpret_cast(image_base + 0x1d48b70) != 0xc3) + { vtable_ = nullptr; return FailedPrecondition("2998 VR haptic vtable contract mismatch"); } + original_haptic_slot_ = vtable_[3]; original_state_slot_ = vtable_[kStateGetterVtableSlot / sizeof(void *)]; original_eye_slot_ = vtable_[kEyeGetterVtableSlot / sizeof(void *)]; - std::uintptr_t *first_slot = vtable_ + kStateGetterVtableSlot / 8; + std::uintptr_t *first_slot = vtable_ + 3; std::uintptr_t *last_slot_exclusive = vtable_ + kEyeInitializerVtableSlot / 8 + 1; if (!SetVtableRangeWritable(first_slot, last_slot_exclusive, true)) { @@ -527,6 +677,7 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { std::memory_order_release); original_eye_getter_.store(reinterpret_cast(original_eye_slot_), std::memory_order_release); + __atomic_store_n(&vtable_[3], FunctionAddress(&mocktail_vr_debug_device_haptics), __ATOMIC_RELEASE); __atomic_store_n(&vtable_[kStateGetterVtableSlot / 8], FunctionAddress(&mocktail_vr_debug_device_state_getter), __ATOMIC_RELEASE); @@ -534,6 +685,7 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { FunctionAddress(&mocktail_vr_debug_device_eye_getter), __ATOMIC_RELEASE); if (!SetVtableRangeWritable(first_slot, last_slot_exclusive, false)) { + __atomic_store_n(&vtable_[3], original_haptic_slot_, __ATOMIC_RELEASE); __atomic_store_n(&vtable_[kStateGetterVtableSlot / 8], original_state_slot_, __ATOMIC_RELEASE); __atomic_store_n(&vtable_[kEyeGetterVtableSlot / 8], original_eye_slot_, @@ -547,8 +699,16 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { "protection"); } + if (!InstallPointerHook(image_base)) { + (void)RestoreVtableLocked(); + return FailedPrecondition("could not install verified VR aim pointer hook"); + } + { + std::lock_guard pose_lock(g_native_pose_mutex); + g_native_pose = {}; + } active_.store(true, std::memory_order_release); - Log(" [vr-device] activated: DebugDeviceVR vtable slots +0x28/+0x48 " + Log(" [vr-device] activated: DebugDeviceVR input/eyes/haptics +0x18/+0x28/+0x48 and aim pointer " "interposed at image base 0x%" PRIxPTR "\n", image_base); return Status::Ok(); @@ -583,15 +743,17 @@ void RobloxVrDeviceBridge::Shutdown() { } bool RobloxVrDeviceBridge::RestoreVtableLocked() { + if (!RestorePointerHook()) return false; if (vtable_ == nullptr) { return false; } - std::uintptr_t *first_slot = vtable_ + kStateGetterVtableSlot / 8; + std::uintptr_t *first_slot = vtable_ + 3; std::uintptr_t *last_slot_exclusive = vtable_ + kEyeInitializerVtableSlot / 8 + 1; if (!SetVtableRangeWritable(first_slot, last_slot_exclusive, true)) { return false; } + __atomic_store_n(&vtable_[3], original_haptic_slot_, __ATOMIC_RELEASE); __atomic_store_n(&vtable_[kStateGetterVtableSlot / 8], original_state_slot_, __ATOMIC_RELEASE); __atomic_store_n(&vtable_[kEyeGetterVtableSlot / 8], original_eye_slot_, @@ -662,36 +824,171 @@ void *RobloxVrDeviceBridge::ValidatedGraphicsDevice(void *self) const { return device; } -bool internal::ApplyXrPose(void* state, const ScriptedPoseSample& pose) { - if (!state) return false; - auto* bytes = static_cast(state); +bool internal::ApplyAimToWorldFrame(float *frame, const VrHandPose &hand, + float head_scale, float degrees) { + if (!frame || !hand.grip_valid || !hand.aim_valid || + !std::isfinite(head_scale) || head_scale <= 0 || + !std::isfinite(degrees) || std::abs(degrees) > 360) + return false; + for (int i = 0; i < 12; ++i) + if (!std::isfinite(frame[i])) + return false; + for (int i = 0; i < 3; ++i) + if (!std::isfinite(hand.grip_position[i]) || + !std::isfinite(hand.aim_position[i])) + return false; + pose::Quat grip, aim; + if (!pose::QuatNormalize({hand.grip_orientation[0], hand.grip_orientation[1], + hand.grip_orientation[2], hand.grip_orientation[3]}, + &grip) || + !pose::QuatNormalize({hand.aim_orientation[0], hand.aim_orientation[1], + hand.aim_orientation[2], hand.aim_orientation[3]}, + &aim)) + return false; + const float angle = degrees * pose::kPi / 360.f; + grip = pose::QuatMultiply(grip, {std::sin(angle), 0, 0, std::cos(angle)}); + const auto inverse = pose::QuatConjugate(grip); + const auto delta = pose::QuatMultiply(inverse, aim); + auto translation = + pose::QuatRotate(inverse, {hand.aim_position[0] - hand.grip_position[0], + hand.aim_position[1] - hand.grip_position[1], + hand.aim_position[2] - hand.grip_position[2]}); + const float scale = (10.f / 3.f) * head_scale; + const float offset[3] = {translation.x * scale, translation.y * scale, + translation.z * scale}; + float out[12]; + for (int col = 0; col < 3; ++col) { + const auto axis = + pose::QuatRotate(delta, {col == 0 ? 1.f : 0.f, col == 1 ? 1.f : 0.f, + col == 2 ? 1.f : 0.f}); + const float component[3] = {axis.x, axis.y, axis.z}; + for (int row = 0; row < 3; ++row) { + out[row * 3 + col] = 0; + for (int k = 0; k < 3; ++k) + out[row * 3 + col] += frame[row * 3 + k] * component[k]; + } + } + for (int row = 0; row < 3; ++row) { + out[9 + row] = frame[9 + row]; + for (int k = 0; k < 3; ++k) + out[9 + row] += frame[row * 3 + k] * offset[k]; + } + for (float value : out) + if (!std::isfinite(value)) + return false; + std::memcpy(frame, out, sizeof(out)); + return true; +} + +bool internal::ApplyXrPose(void *state, const ScriptedPoseSample &pose) { + if (!state) + return false; + auto *bytes = static_cast(state); bytes[0] = 0; - // No tracked controllers are supplied by this backend. - bytes[0x20] = bytes[0x40] = bytes[0x60] = 0; - if (!pose.valid) return false; + bytes[0x80] = bytes[0x132] = 0; + const std::uint32_t changed = 0x0fffffff; + std::memcpy(bytes + 0x84, &changed, sizeof(changed)); + std::memset(bytes + 0x88, 0, 28 * sizeof(float)); + // Hand records are cleared unless a tracked grip pose is supplied below. + // Clearing the valid byte (never the whole record) makes the guest report + // the hand as untracked instead of holding a stale pose. The extra index-3 + // record is unused by this backend and stays invalid. + bytes[kExtraRecordOffset + kRecordValidOffset] = 0; + bytes[kLeftHandRecordOffset + kRecordValidOffset] = 0; + bytes[kRightHandRecordOffset + kRecordValidOffset] = 0; + if (!pose.valid) + return false; float norm = 0; - for (float v : pose.orientation) { if (!std::isfinite(v)) return false; norm += v*v; } - if (norm < 0.99f || norm > 1.01f) return false; - for (float v : pose.position) if (!std::isfinite(v)) return false; + for (float v : pose.orientation) { + if (!std::isfinite(v)) + return false; + norm += v * v; + } + if (norm < 0.99f || norm > 1.01f) + return false; + for (float v : pose.position) + if (!std::isfinite(v)) + return false; for (int eye = 0; eye < 2; ++eye) { - for (float v : pose.eye_offset[eye]) if (!std::isfinite(v)) return false; + for (float v : pose.eye_offset[eye]) + if (!std::isfinite(v)) + return false; for (float v : pose.eye_fov[eye]) - if (!std::isfinite(v) || std::abs(v) >= 1.56f) return false; + if (!std::isfinite(v) || std::abs(v) >= 1.56f) + return false; + } + bytes[0x80] = pose.valid && pose.controllers_connected; + for (int channel = 0; channel < 28; ++channel) { + const float raw = pose.controller_channels[channel]; + const float value = + pose.valid && pose.controllers_connected && std::isfinite(raw) + ? std::clamp(raw, -1.f, 1.f) + : 0.f; + std::memcpy(bytes + 0x88 + channel * 4, &value, sizeof(value)); } + bytes[0x132] = pose.valid && pose.controllers_connected; + // 2998's 0x37eacb8 converts metres to studs itself (factor 10/3). std::memcpy(bytes + 4, pose.position, sizeof(pose.position)); std::memcpy(bytes + 0x10, pose.orientation, sizeof(pose.orientation)); + // Tracked hands go into the same state copy the head uses, so the guest's + // own conversion (10/3 studs) and -20 deg hand pitch offset apply once, + // exactly as they do for the head. Grip is the held-object pose; a hand + // without a valid grip pose stays marked untracked rather than being + // fabricated at the origin. + for (int hand = 0; hand < 2; ++hand) { + const auto &source = pose.hands[hand]; + if (!source.grip_valid) + continue; + float hand_norm = 0; + bool finite = true; + for (float v : source.grip_position) + finite = finite && std::isfinite(v); + for (float v : source.grip_orientation) { + finite = finite && std::isfinite(v); + hand_norm += v * v; + } + if (!finite || hand_norm < 0.99f || hand_norm > 1.01f) + continue; + const std::size_t record = + hand == 0 ? kLeftHandRecordOffset : kRightHandRecordOffset; + std::memcpy(bytes + record + kRecordPositionOffset, source.grip_position, + sizeof(source.grip_position)); + std::memcpy(bytes + record + kRecordOrientationOffset, + source.grip_orientation, sizeof(source.grip_orientation)); + bytes[record + kRecordValidOffset] = 1; + } for (int eye = 0; eye < 2; ++eye) { - std::memcpy(bytes + 0xf8 + eye*12, pose.eye_offset[eye], 12); - const auto* fov = pose.eye_fov[eye]; + std::memcpy(bytes + 0xf8 + eye * 12, pose.eye_offset[eye], 12); + const auto *fov = pose.eye_fov[eye]; const float tangents[4] = {std::tan(fov[2]), -std::tan(fov[3]), - -std::tan(fov[0]), std::tan(fov[1])}; - std::memcpy(bytes + 0x110 + eye*16, tangents, sizeof(tangents)); + -std::tan(fov[0]), std::tan(fov[1])}; + std::memcpy(bytes + 0x110 + eye * 16, tangents, sizeof(tangents)); } bytes[0] = 1; return true; } +void RobloxVrDeviceBridge::OnHapticsCall(void *device_object, int hand, + float amplitude) { + std::lock_guard lock(mutex_); + if (!active() || !ValidateObjectHeader(device_object) || hand < 0 || hand > 1) + return; + if (auto *backend = ActiveVrBackend()) { + static std::atomic logged[2]{}; + if (!logged[hand].exchange(true)) + Log(" [vr-input] native haptic callback observed: hand=%d " + "amplitude=%.3f\n", + hand, amplitude); + if (!std::isfinite(amplitude) || amplitude <= 0) + backend->StopControllerHaptics(hand); + // Native pull refreshes once per engine frame. A short lease prevents a + // stalled or disconnected engine from leaving the controller vibrating. + else + backend->RequestControllerHaptics(hand, amplitude, 50000000, 0); + } +} + void RobloxVrDeviceBridge::OnStateGetterCall(void *result_buffer, void *device_object) { // Serialize initialization with all copies of readiness: the initializer @@ -709,16 +1006,21 @@ void RobloxVrDeviceBridge::OnStateGetterCall(void *result_buffer, if (active_.load(std::memory_order_acquire) && result_buffer && original) { if (auto* backend = ActiveVrBackend()) { const auto pose = backend->PublishedHeadPose(); + { + std::lock_guard pose_lock(g_native_pose_mutex); + g_native_pose = pose; + } if (internal::ApplyXrPose(result_buffer, pose)) { backend->NotePoseApplied(device_object, pose.frame); static thread_local std::uint64_t last_logged = 0; if (pose.frame != last_logged && (pose.frame == 1 || pose.frame % 20 == 0)) { last_logged = pose.frame; Log(" [vr-device][evidence] pose-applied frame=%llu owner=%p " - "pos=(%.4f,%.4f,%.4f) ori=(%.4f,%.4f,%.4f,%.4f)\n", + "pos=(%.4f,%.4f,%.4f) ori=(%.4f,%.4f,%.4f,%.4f) hands=%d/%d input=%d\n", static_cast(pose.frame), device_object, pose.position[0], pose.position[1], pose.position[2], - pose.orientation[0], pose.orientation[1], pose.orientation[2], pose.orientation[3]); + pose.orientation[0], pose.orientation[1], pose.orientation[2], pose.orientation[3], + pose.hands[0].grip_valid, pose.hands[1].grip_valid, pose.controllers_connected); } } } @@ -882,4 +1184,14 @@ Status NotifyRobloxVrImageLoaded(std::uintptr_t image_base) { return bridge->Activate(image_base); } +bool NativeControllerInputActive() { + auto* bridge = g_active_bridge.load(std::memory_order_acquire); + return bridge && bridge->active() && ActiveVrBackend(); +} + } // namespace mocktail::vr + +extern "C" bool mocktail_vr_native_controller_input_active() { + // Query through the bridge's public ownership API below. + return mocktail::vr::NativeControllerInputActive(); +} diff --git a/src/vr/roblox_vr_device_bridge_internal.h b/src/vr/roblox_vr_device_bridge_internal.h index d7b39871..fe42469a 100644 --- a/src/vr/roblox_vr_device_bridge_internal.h +++ b/src/vr/roblox_vr_device_bridge_internal.h @@ -5,12 +5,17 @@ #include #include "compat/build_profile.h" +#include "mocktail/vr/xr_controller.h" namespace mocktail::vr { struct ScriptedPoseSample; } namespace mocktail::vr::internal { // Writes only a caller-owned state copy; preserves all resource readiness. bool ApplyXrPose(void* state, const ScriptedPoseSample& pose); +// Adjust a world-space grip CoordinateFrame to OpenXR aim without changing +// the engine's hand pose. Native frame layout: row-major R[9], position[3]. +bool ApplyAimToWorldFrame(float* frame, const VrHandPose& hand, + float head_scale, float native_hand_pitch_degrees); // Guest ABI layout constants of RBX::Graphics::DebugDeviceVR. They are pinned // per exact build by the machine-code contracts below; changing payload @@ -21,6 +26,26 @@ inline constexpr std::uint32_t kExpectedObjectType = 6; inline constexpr std::size_t kStateOffset = 0x14; inline constexpr std::size_t kStateCopySize = 0x138; inline constexpr std::size_t kStateReadyOffsetInCopy = 0x133; +// Per-record pose layout inside the 0x138 state copy. Verified against 2998 +// consumer 0x28a6242 (docs/vr-agent-evidence/controller-2998): four 0x20-byte +// records are each converted by helper 0x37eacb8 (metres->studs x10/3, quat +// normalized) and stored via setUserCFrame 0x4d56906 at VRService+0x174+idx*0x30. +// copy+0x00 head -> UserCFrame index 0 +// copy+0x20 extra -> index 3 +// copy+0x40 left hand -> index 1 +// copy+0x60 right hand-> index 2 +// Record field layout (same for head and hands): valid byte +0, position +// (3 floats, metres) +4, quaternion xyzw (4 floats) +0x10. The guest applies +// the 10/3 conversion and a -20 deg hand pitch offset itself; neither is +// pre-applied here. +inline constexpr std::size_t kPoseRecordStride = 0x20; +inline constexpr std::size_t kHeadRecordOffset = 0x00; +inline constexpr std::size_t kExtraRecordOffset = 0x20; +inline constexpr std::size_t kLeftHandRecordOffset = 0x40; +inline constexpr std::size_t kRightHandRecordOffset = 0x60; +inline constexpr std::size_t kRecordValidOffset = 0x00; +inline constexpr std::size_t kRecordPositionOffset = 0x04; +inline constexpr std::size_t kRecordOrientationOffset = 0x10; inline constexpr std::size_t kReadyByteOffset = 0x147; inline constexpr std::size_t kFramebufferSlotOffset = 0x150; inline constexpr std::size_t kTextureSlotOffset = 0x170; diff --git a/src/vr/vr_perf.cc b/src/vr/vr_perf.cc index 933cd5ba..544ba833 100644 --- a/src/vr/vr_perf.cc +++ b/src/vr/vr_perf.cc @@ -56,6 +56,7 @@ const char* StageName(Stage stage) { case Stage::kCopyFenceWait: return "copy_fence_wait"; case Stage::kMirrorPublish: return "mirror_publish"; case Stage::kEvidence: return "evidence"; + case Stage::kControllerSync: return "controller_sync"; case Stage::kStageCount: break; } return "unknown"; @@ -80,6 +81,7 @@ const char* CounterName(Counter counter) { case Counter::kVisibilityLost: return "visibility_lost"; case Counter::kVisibilityGained: return "visibility_gained"; case Counter::kFramesInvalidated: return "frames_invalidated"; + case Counter::kControllerSyncs: return "controller_syncs"; case Counter::kCounterCount: break; } return "unknown"; diff --git a/src/vr/xr_actions.cc b/src/vr/xr_actions.cc new file mode 100644 index 00000000..10c7e00a --- /dev/null +++ b/src/vr/xr_actions.cc @@ -0,0 +1,909 @@ +#include "mocktail/vr/xr_actions.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mocktail::vr { +namespace { + +void Log(const char *format, ...) { + va_list arguments; + va_start(arguments, format); + std::vfprintf(stderr, format, arguments); + va_end(arguments); + std::fflush(stderr); +} + +template T XrInfo(XrStructureType type) { + T value{}; + value.type = type; + return value; +} + +std::string XrResultText(XrInstance instance, XrResult result) { + char text[XR_MAX_RESULT_STRING_SIZE]{}; + if (instance != XR_NULL_HANDLE) { + (void)xrResultToString(instance, result, text); + } + if (!text[0]) { + return std::to_string(static_cast(result)); + } + return text; +} + +void CheckXr(XrInstance instance, XrResult result, const char *operation) { + if (XR_SUCCEEDED(result)) { + return; + } + throw std::runtime_error(std::string(operation) + + " failed: " + XrResultText(instance, result)); +} + +// Registry-verified interaction profiles +// (docs/vr-agent-evidence/controller-2998/openxr-interaction-profiles.md). +constexpr const char *kProfileTouch = + "/interaction_profiles/oculus/touch_controller"; +constexpr const char *kProfileQuest2 = + "/interaction_profiles/meta/touch_controller_quest_2"; +constexpr const char *kProfileIndex = + "/interaction_profiles/valve/index_controller"; +constexpr const char *kProfileVive = + "/interaction_profiles/htc/vive_controller"; +constexpr const char *kProfileSimple = + "/interaction_profiles/khr/simple_controller"; + +} // namespace + +// All XR objects of one XrActions instance. +struct XrActions::Impl { + XrInstance instance = XR_NULL_HANDLE; + XrSession session = XR_NULL_HANDLE; + XrSpace local_space = XR_NULL_HANDLE; // borrowed from the backend + XrActionSet set = XR_NULL_HANDLE; + XrAction grip[2] = {XR_NULL_HANDLE, XR_NULL_HANDLE}; + XrAction aim[2] = {XR_NULL_HANDLE, XR_NULL_HANDLE}; + XrSpace grip_space[2] = {XR_NULL_HANDLE, XR_NULL_HANDLE}; + XrSpace aim_space[2] = {XR_NULL_HANDLE, XR_NULL_HANDLE}; + XrAction touches[static_cast(ControllerTouch::kCount)] = {}; + XrAction trigger = XR_NULL_HANDLE; + XrAction squeeze = XR_NULL_HANDLE; + XrAction stick = XR_NULL_HANDLE; + XrAction trigger_click = XR_NULL_HANDLE; + XrAction squeeze_click = XR_NULL_HANDLE; + XrAction stick_click = XR_NULL_HANDLE; + XrAction face_a = XR_NULL_HANDLE; + XrAction face_b = XR_NULL_HANDLE; + XrAction face_x = XR_NULL_HANDLE; + XrAction face_y = XR_NULL_HANDLE; + XrAction menu = XR_NULL_HANDLE; + XrAction haptic = XR_NULL_HANDLE; + XrPath subaction_left = XR_NULL_PATH; + XrPath subaction_right = XR_NULL_PATH; + + std::mutex haptic_mutex; + struct HapticRequest { + bool pending = false; + bool stop = false; + float amplitude = 0.f; + std::uint64_t duration_ns = 0; + float frequency_hz = 0.f; + }; + HapticRequest haptics[2]; + + XrPath active_profile[2] = {XR_NULL_PATH, XR_NULL_PATH}; + std::string active_profile_name[2]; + // Trigger-click actions that reported active this frame, per hand: the + // analog-derived click must not override a native click action. + bool native_trigger_click[2] = {false, false}; +}; + +XrActions::~XrActions() { Destroy(); } + +Status XrActions::Create(void *xr_instance) { + if (created_) { + return Status::Ok(); + } + if (xr_instance == nullptr) { + return Status::Error(StatusCode::kInvalidArgument, + "XR actions require an instance"); + } + const auto instance = static_cast(xr_instance); + delete impl_; + impl_ = new Impl(); + impl_->instance = instance; + try { + auto set_info = + XrInfo(XR_TYPE_ACTION_SET_CREATE_INFO); + std::strncpy(set_info.actionSetName, "mocktail_vr", + XR_MAX_ACTION_SET_NAME_SIZE - 1); + std::strncpy(set_info.localizedActionSetName, "Mocktail VR controls", + XR_MAX_LOCALIZED_ACTION_SET_NAME_SIZE - 1); + set_info.priority = 0; + CheckXr(instance, xrCreateActionSet(instance, &set_info, &impl_->set), + "xrCreateActionSet"); + CheckXr(instance, + xrStringToPath(instance, "/user/hand/left", &impl_->subaction_left), + "xrStringToPath left"); + CheckXr( + instance, + xrStringToPath(instance, "/user/hand/right", &impl_->subaction_right), + "xrStringToPath right"); + const std::array subactions{impl_->subaction_left, + impl_->subaction_right}; + + auto make = [&](XrActionType type, const char *name) { + auto info = XrInfo(XR_TYPE_ACTION_CREATE_INFO); + info.actionType = type; + info.countSubactionPaths = 2; + info.subactionPaths = subactions.data(); + std::strncpy(info.actionName, name, XR_MAX_ACTION_NAME_SIZE - 1); + // A non-empty localized name is mandatory; runtimes reject an empty one + // with XR_ERROR_LOCALIZED_NAME_INVALID. + std::strncpy(info.localizedActionName, name, + XR_MAX_LOCALIZED_ACTION_NAME_SIZE - 1); + XrAction action = XR_NULL_HANDLE; + CheckXr(instance, xrCreateAction(impl_->set, &info, &action), name); + return action; + }; + + impl_->grip[0] = make(XR_ACTION_TYPE_POSE_INPUT, "grip_left"); + impl_->grip[1] = make(XR_ACTION_TYPE_POSE_INPUT, "grip_right"); + impl_->aim[0] = make(XR_ACTION_TYPE_POSE_INPUT, "aim_left"); + impl_->aim[1] = make(XR_ACTION_TYPE_POSE_INPUT, "aim_right"); + impl_->trigger = make(XR_ACTION_TYPE_FLOAT_INPUT, "trigger"); + impl_->squeeze = make(XR_ACTION_TYPE_FLOAT_INPUT, "squeeze"); + impl_->stick = make(XR_ACTION_TYPE_VECTOR2F_INPUT, "thumbstick"); + impl_->trigger_click = make(XR_ACTION_TYPE_BOOLEAN_INPUT, "trigger_click"); + impl_->squeeze_click = make(XR_ACTION_TYPE_BOOLEAN_INPUT, "squeeze_click"); + impl_->stick_click = make(XR_ACTION_TYPE_BOOLEAN_INPUT, "thumbstick_click"); + impl_->face_a = make(XR_ACTION_TYPE_BOOLEAN_INPUT, "face_a"); + impl_->face_b = make(XR_ACTION_TYPE_BOOLEAN_INPUT, "face_b"); + impl_->face_x = make(XR_ACTION_TYPE_BOOLEAN_INPUT, "face_x"); + impl_->face_y = make(XR_ACTION_TYPE_BOOLEAN_INPUT, "face_y"); + impl_->menu = make(XR_ACTION_TYPE_BOOLEAN_INPUT, "menu"); + const char *touch_names[] = { + "trigger_touch", "thumbstick_touch", "thumbrest_touch", "a_touch", + "b_touch", "x_touch", "y_touch"}; + for (std::size_t i = 0; i < std::size(touch_names); ++i) + impl_->touches[i] = make(XR_ACTION_TYPE_BOOLEAN_INPUT, touch_names[i]); + impl_->haptic = make(XR_ACTION_TYPE_VIBRATION_OUTPUT, "haptic"); + + auto path = [&](const char *text) { + XrPath result = XR_NULL_PATH; + CheckXr(instance, xrStringToPath(instance, text, &result), text); + return result; + }; + // Each profile is suggested independently: a runtime that does not know an + // optional profile rejects only that suggestion, never the primary Touch + // binding. The system-reserved button is not bound on any profile. + auto suggest = [&](const char *profile, + std::vector> binds) { + auto touch = [&](ControllerTouch kind, const char *component) { + binds.emplace_back(impl_->touches[static_cast(kind)], + path(component)); + }; + const bool is_touch = std::strcmp(profile, kProfileTouch) == 0 || + std::strcmp(profile, kProfileQuest2) == 0; + const bool is_index = std::strcmp(profile, kProfileIndex) == 0; + if (is_touch || is_index) { + touch(ControllerTouch::kTrigger, "/user/hand/left/input/trigger/touch"); + touch(ControllerTouch::kTrigger, + "/user/hand/right/input/trigger/touch"); + touch(ControllerTouch::kThumbstick, + "/user/hand/left/input/thumbstick/touch"); + touch(ControllerTouch::kThumbstick, + "/user/hand/right/input/thumbstick/touch"); + touch(ControllerTouch::kA, "/user/hand/right/input/a/touch"); + touch(ControllerTouch::kB, "/user/hand/right/input/b/touch"); + touch(ControllerTouch::kX, is_touch ? "/user/hand/left/input/x/touch" + : "/user/hand/left/input/a/touch"); + touch(ControllerTouch::kY, is_touch ? "/user/hand/left/input/y/touch" + : "/user/hand/left/input/b/touch"); + } + if (is_touch) { + touch(ControllerTouch::kThumbrest, + "/user/hand/left/input/thumbrest/touch"); + touch(ControllerTouch::kThumbrest, + "/user/hand/right/input/thumbrest/touch"); + } + std::vector suggested; + suggested.reserve(binds.size()); + for (const auto &bind : binds) { + XrActionSuggestedBinding entry{}; + entry.action = bind.first; + entry.binding = bind.second; + suggested.push_back(entry); + } + auto info = XrInfo( + XR_TYPE_INTERACTION_PROFILE_SUGGESTED_BINDING); + info.interactionProfile = path(profile); + info.countSuggestedBindings = + static_cast(suggested.size()); + info.suggestedBindings = suggested.data(); + const XrResult result = + xrSuggestInteractionProfileBindings(instance, &info); + if (XR_FAILED(result)) { + Log(" [vr-input] bindings for %s not accepted (%s)\n", profile, + XrResultText(instance, result).c_str()); + } + }; + + const XrPath l_grip = path("/user/hand/left/input/grip/pose"); + const XrPath r_grip = path("/user/hand/right/input/grip/pose"); + const XrPath l_aim = path("/user/hand/left/input/aim/pose"); + const XrPath r_aim = path("/user/hand/right/input/aim/pose"); + const XrPath l_haptic = path("/user/hand/left/output/haptic"); + const XrPath r_haptic = path("/user/hand/right/output/haptic"); + + for (const char *profile : {kProfileTouch, kProfileQuest2}) { + suggest( + profile, + { + {impl_->grip[0], l_grip}, + {impl_->grip[1], r_grip}, + {impl_->aim[0], l_aim}, + {impl_->aim[1], r_aim}, + {impl_->trigger, path("/user/hand/left/input/trigger/value")}, + {impl_->trigger, path("/user/hand/right/input/trigger/value")}, + {impl_->squeeze, path("/user/hand/left/input/squeeze/value")}, + {impl_->squeeze, path("/user/hand/right/input/squeeze/value")}, + {impl_->stick, path("/user/hand/left/input/thumbstick")}, + {impl_->stick, path("/user/hand/right/input/thumbstick")}, + {impl_->stick_click, + path("/user/hand/left/input/thumbstick/click")}, + {impl_->stick_click, + path("/user/hand/right/input/thumbstick/click")}, + {impl_->face_x, path("/user/hand/left/input/x/click")}, + {impl_->face_y, path("/user/hand/left/input/y/click")}, + {impl_->face_a, path("/user/hand/right/input/a/click")}, + {impl_->face_b, path("/user/hand/right/input/b/click")}, + {impl_->menu, path("/user/hand/left/input/menu/click")}, + {impl_->haptic, l_haptic}, + {impl_->haptic, r_haptic}, + }); + } + suggest( + kProfileIndex, + { + {impl_->grip[0], l_grip}, + {impl_->grip[1], r_grip}, + {impl_->aim[0], l_aim}, + {impl_->aim[1], r_aim}, + {impl_->trigger, path("/user/hand/left/input/trigger/value")}, + {impl_->trigger, path("/user/hand/right/input/trigger/value")}, + {impl_->trigger_click, path("/user/hand/left/input/trigger/click")}, + {impl_->trigger_click, + path("/user/hand/right/input/trigger/click")}, + {impl_->squeeze, path("/user/hand/left/input/squeeze/force")}, + {impl_->squeeze, path("/user/hand/right/input/squeeze/force")}, + {impl_->stick, path("/user/hand/left/input/thumbstick")}, + {impl_->stick, path("/user/hand/right/input/thumbstick")}, + {impl_->stick_click, + path("/user/hand/left/input/thumbstick/click")}, + {impl_->stick_click, + path("/user/hand/right/input/thumbstick/click")}, + {impl_->face_x, path("/user/hand/left/input/a/click")}, + {impl_->face_a, path("/user/hand/right/input/a/click")}, + {impl_->face_y, path("/user/hand/left/input/b/click")}, + {impl_->face_b, path("/user/hand/right/input/b/click")}, + {impl_->haptic, l_haptic}, + {impl_->haptic, r_haptic}, + }); + suggest( + kProfileVive, + { + {impl_->squeeze_click, path("/user/hand/left/input/squeeze/click")}, + {impl_->squeeze_click, + path("/user/hand/right/input/squeeze/click")}, + {impl_->grip[0], l_grip}, + {impl_->grip[1], r_grip}, + {impl_->aim[0], l_aim}, + {impl_->aim[1], r_aim}, + {impl_->trigger, path("/user/hand/left/input/trigger/value")}, + {impl_->trigger, path("/user/hand/right/input/trigger/value")}, + {impl_->stick, path("/user/hand/left/input/trackpad")}, + {impl_->stick, path("/user/hand/right/input/trackpad")}, + {impl_->stick_click, path("/user/hand/left/input/trackpad/click")}, + {impl_->stick_click, path("/user/hand/right/input/trackpad/click")}, + {impl_->menu, path("/user/hand/left/input/menu/click")}, + {impl_->menu, path("/user/hand/right/input/menu/click")}, + {impl_->haptic, l_haptic}, + {impl_->haptic, r_haptic}, + }); + suggest( + kProfileSimple, + { + {impl_->grip[0], l_grip}, + {impl_->grip[1], r_grip}, + {impl_->aim[0], l_aim}, + {impl_->aim[1], r_aim}, + {impl_->trigger_click, path("/user/hand/left/input/select/click")}, + {impl_->trigger_click, path("/user/hand/right/input/select/click")}, + {impl_->menu, path("/user/hand/left/input/menu/click")}, + {impl_->menu, path("/user/hand/right/input/menu/click")}, + {impl_->haptic, l_haptic}, + {impl_->haptic, r_haptic}, + }); + + created_ = true; + Log(" [vr-input] action set created: grip+aim poses, trigger/squeeze " + "analog, thumbstick, face/menu buttons, haptic output; bindings " + "suggested for Touch/Quest2/Index/Vive/Simple (system button never " + "bound)\n"); + return Status::Ok(); + } catch (const std::exception &error) { + Destroy(); + return Status::Error(StatusCode::kUnavailable, error.what()); + } +} + +Status XrActions::Attach(void *xr_instance, void *xr_session, + void *xr_local_space) { + if (!created_ || impl_ == nullptr) { + return Status::Error(StatusCode::kFailedPrecondition, + "XR actions are not created"); + } + if (xr_session == nullptr || xr_local_space == nullptr) { + return Status::Error(StatusCode::kInvalidArgument, + "XR actions attach requires a session and space"); + } + const auto instance = static_cast(xr_instance); + const auto session = static_cast(xr_session); + try { + auto attach_info = XrInfo( + XR_TYPE_SESSION_ACTION_SETS_ATTACH_INFO); + attach_info.countActionSets = 1; + attach_info.actionSets = &impl_->set; + CheckXr(instance, xrAttachSessionActionSets(session, &attach_info), + "xrAttachSessionActionSets"); + + for (int hand = 0; hand < 2; ++hand) { + const XrPath subaction = + hand == 0 ? impl_->subaction_left : impl_->subaction_right; + auto space_info = + XrInfo(XR_TYPE_ACTION_SPACE_CREATE_INFO); + space_info.subactionPath = subaction; + space_info.poseInActionSpace.orientation.w = 1.f; + space_info.action = impl_->grip[hand]; + CheckXr( + instance, + xrCreateActionSpace(session, &space_info, &impl_->grip_space[hand]), + "xrCreateActionSpace grip"); + space_info.action = impl_->aim[hand]; + CheckXr( + instance, + xrCreateActionSpace(session, &space_info, &impl_->aim_space[hand]), + "xrCreateActionSpace aim"); + } + impl_->session = session; + impl_->local_space = static_cast(xr_local_space); + attached_ = true; + session_generation_++; + Log(" [vr-input] actions attached to session (generation %llu)\n", + static_cast(session_generation_)); + return Status::Ok(); + } catch (const std::exception &error) { + Detach(); + return Status::Error(StatusCode::kUnavailable, error.what()); + } +} + +void XrActions::ResetInput() { + const bool had_input = + snapshot_.hands[0].connected || snapshot_.hands[1].connected; + const auto frame = snapshot_.frame; + builder_ = ControllerStateBuilder{}; + snapshot_ = {}; + hands_[0] = {}; + hands_[1] = {}; + if (had_input) { + delivery_queue_.Clear(); + ControllerDelivery reset; + reset.frame = frame; + reset.session_generation = session_generation_; + reset.resync = true; + delivery_queue_.Push(reset); + } + last_delivered_connected_[0] = last_delivered_connected_[1] = false; + if (impl_) { + std::lock_guard lock(impl_->haptic_mutex); + for (int hand = 0; hand < 2; ++hand) { + impl_->haptics[hand] = {}; + if (attached_ && impl_->session) { + auto stop = XrInfo(XR_TYPE_HAPTIC_ACTION_INFO); + stop.action = impl_->haptic; + stop.subactionPath = + hand ? impl_->subaction_right : impl_->subaction_left; + (void)xrStopHapticFeedback(impl_->session, &stop); + } + } + } +} + +void XrActions::Detach() { + ResetInput(); + if (impl_ == nullptr) { + attached_ = false; + return; + } + // Haptics must not outlive the session. + if (attached_ && impl_->session != XR_NULL_HANDLE) { + for (int hand = 0; hand < 2; ++hand) { + auto stop = XrInfo(XR_TYPE_HAPTIC_ACTION_INFO); + stop.action = impl_->haptic; + stop.subactionPath = + hand == 0 ? impl_->subaction_left : impl_->subaction_right; + (void)xrStopHapticFeedback(impl_->session, &stop); + } + } + for (int hand = 0; hand < 2; ++hand) { + if (impl_->grip_space[hand] != XR_NULL_HANDLE) { + (void)xrDestroySpace(impl_->grip_space[hand]); + impl_->grip_space[hand] = XR_NULL_HANDLE; + } + if (impl_->aim_space[hand] != XR_NULL_HANDLE) { + (void)xrDestroySpace(impl_->aim_space[hand]); + impl_->aim_space[hand] = XR_NULL_HANDLE; + } + impl_->active_profile[hand] = XR_NULL_PATH; + impl_->active_profile_name[hand].clear(); + impl_->native_trigger_click[hand] = false; + } + impl_->session = XR_NULL_HANDLE; + impl_->local_space = XR_NULL_HANDLE; + attached_ = false; + hands_[0] = VrHandPose{}; + hands_[1] = VrHandPose{}; + // Queued deliveries belong to the detached session; a replacement session + // must start from a clean slate with fresh connection levels. + // Preserve queued reset until the input thread drains it. + last_delivered_connected_[0] = false; + last_delivered_connected_[1] = false; +} + +void XrActions::Destroy() { + Detach(); + if (impl_ != nullptr) { + const auto instance = impl_->instance; + if (instance != XR_NULL_HANDLE) { + const std::array actions{ + impl_->grip[0], impl_->grip[1], impl_->aim[0], + impl_->aim[1], impl_->trigger, impl_->squeeze, + impl_->stick, impl_->trigger_click, impl_->squeeze_click, + impl_->stick_click, impl_->face_a, impl_->face_b, + impl_->face_x, impl_->face_y, impl_->menu, + impl_->haptic}; + for (const XrAction action : actions) { + if (action != XR_NULL_HANDLE) { + (void)xrDestroyAction(action); + } + } + for (const auto action : impl_->touches) + if (action != XR_NULL_HANDLE) + (void)xrDestroyAction(action); + if (impl_->set != XR_NULL_HANDLE) { + (void)xrDestroyActionSet(impl_->set); + } + } + delete impl_; + impl_ = nullptr; + } + created_ = false; + attached_ = false; + snapshot_ = ControllerSnapshot{}; + hands_[0] = VrHandPose{}; + hands_[1] = VrHandPose{}; + // Preserve queued reset until the input thread drains it. + last_delivered_connected_[0] = false; + last_delivered_connected_[1] = false; + session_generation_ = 0; +} + +void XrActions::RequestHaptics(int hand, float amplitude, + std::uint64_t duration_ns, float frequency_hz) { + if (impl_ == nullptr || !attached_ || (hand != 0 && hand != 1)) { + return; + } + std::lock_guard lock(impl_->haptic_mutex); + auto &request = impl_->haptics[hand]; + request = {}; + request.pending = true; + request.stop = false; + request.amplitude = + std::clamp(std::isfinite(amplitude) ? amplitude : 0.f, 0.f, 1.f); + request.duration_ns = duration_ns; + request.frequency_hz = + std::isfinite(frequency_hz) ? std::max(0.f, frequency_hz) : 0.f; +} + +void XrActions::StopHaptics(int hand) { + if (impl_ == nullptr || !attached_ || (hand != 0 && hand != 1)) { + return; + } + std::lock_guard lock(impl_->haptic_mutex); + impl_->haptics[hand] = Impl::HapticRequest{}; + impl_->haptics[hand].pending = true; + impl_->haptics[hand].stop = true; +} + +void XrActions::NoteProfileChanged() { + if (impl_ == nullptr) { + return; + } + for (int hand = 0; hand < 2; ++hand) { + impl_->active_profile[hand] = XR_NULL_PATH; + impl_->active_profile_name[hand].clear(); + } +} + +std::string XrActions::ActiveProfile(int hand) const { + if (impl_ == nullptr || (hand != 0 && hand != 1)) { + return {}; + } + return impl_->active_profile_name[hand]; +} + +bool XrActions::TakeDelivery(ControllerDelivery *out) { + return delivery_queue_.Pop(out); +} + +bool XrActions::Sync(void *xr_session, std::uint64_t predicted_display_time, + std::uint64_t frame, std::uint64_t /*generation*/) { + if (!attached_ || impl_ == nullptr) { + return false; + } + const auto session = static_cast(xr_session); + const auto instance = impl_->instance; + const auto time = static_cast(predicted_display_time); + try { + auto sync_info = XrInfo(XR_TYPE_ACTIONS_SYNC_INFO); + const XrActiveActionSet active{impl_->set, XR_NULL_PATH}; + sync_info.countActiveActionSets = 1; + sync_info.activeActionSets = &active; + const XrResult sync_result = xrSyncActions(session, &sync_info); + if (sync_result == XR_SESSION_NOT_FOCUSED) { + // Normal while the system compositor owns input: no fresh state exists + // this frame and nothing may be delivered. + ResetInput(); + return false; + } + if (XR_FAILED(sync_result)) { + Log(" [vr-input] xrSyncActions failed: %s\n", + XrResultText(instance, sync_result).c_str()); + ResetInput(); + return false; + } + + for (int hand = 0; hand < 2; ++hand) { + const XrPath subaction = + hand == 0 ? impl_->subaction_left : impl_->subaction_right; + + // Inactive actions must neutralize previous state, even if a profile + // remains assigned after the device sleeps or the runtime changes + // bindings. + for (size_t i = 0; i < static_cast(ControllerButton::kCount); ++i) + builder_.SetButton(hand, static_cast(i), false, true); + for (size_t i = 0; i < static_cast(ControllerAxis::kCount); ++i) + builder_.SetAxis(hand, static_cast(i), 0.f, true); + for (std::size_t i = 0; + i < static_cast(ControllerTouch::kCount); ++i) + builder_.SetTouch(hand, static_cast(i), false, false); + // Connection state comes from the runtime's current profile for the + // hand; profile changes refresh diagnostics here and on the event. + auto profile_info = + XrInfo(XR_TYPE_INTERACTION_PROFILE_STATE); + bool connected = false; + if (XR_SUCCEEDED(xrGetCurrentInteractionProfile(session, subaction, + &profile_info)) && + profile_info.interactionProfile != XR_NULL_PATH) { + connected = true; + if (profile_info.interactionProfile != impl_->active_profile[hand]) { + impl_->active_profile[hand] = profile_info.interactionProfile; + char buffer[XR_MAX_PATH_LENGTH]{}; + std::uint32_t length = 0; + if (XR_SUCCEEDED(xrPathToString(instance, + profile_info.interactionProfile, + sizeof(buffer), &length, buffer))) { + impl_->active_profile_name[hand] = buffer; + } + builder_.SetProfile(hand, impl_->active_profile_name[hand]); + Log(" [vr-input] hand %d interaction profile: %s\n", hand, + impl_->active_profile_name[hand].c_str()); + } + } else { + if (impl_->active_profile[hand] != XR_NULL_PATH) { + Log(" [vr-input] hand %d controller disconnected\n", hand); + } + impl_->active_profile[hand] = XR_NULL_PATH; + impl_->active_profile_name[hand].clear(); + builder_.SetProfile(hand, {}); + } + if (connected && builder_.snapshot().hands[hand].profile != + impl_->active_profile_name[hand]) + builder_.SetProfile(hand, impl_->active_profile_name[hand]); + builder_.SetConnected(hand, connected); + if (!connected) { + // A disconnected hand must release anything held: force every button + // level to false so Publish emits at most one release edge per button + // and nothing can stay stuck down. + for (std::size_t index = 0; + index < static_cast(ControllerButton::kCount); + ++index) { + builder_.SetButton(hand, static_cast(index), false, + true); + } + builder_.SetAxis(hand, ControllerAxis::kTrigger, 0.f, true); + builder_.SetAxis(hand, ControllerAxis::kSqueeze, 0.f, true); + builder_.SetAxis(hand, ControllerAxis::kThumbstickX, 0.f, true); + builder_.SetAxis(hand, ControllerAxis::kThumbstickY, 0.f, true); + builder_.SetPoseValid(hand, false); + hands_[hand] = VrHandPose{}; + continue; + } + + // Grip and aim poses located at the very predicted display time the head + // and eyes used for this frame. + VrHandPose hand_pose; + bool hand_active = false; + for (int which = 0; which < 2; ++which) { + const XrSpace space = + which == 0 ? impl_->grip_space[hand] : impl_->aim_space[hand]; + XrSpaceLocation location = + XrInfo(XR_TYPE_SPACE_LOCATION); + const XrResult located = + xrLocateSpace(space, impl_->local_space, time, &location); + const bool orientation_valid = + XR_SUCCEEDED(located) && + (location.locationFlags & + XR_SPACE_LOCATION_ORIENTATION_VALID_BIT) != 0; + const bool position_valid = XR_SUCCEEDED(located) && + (location.locationFlags & + XR_SPACE_LOCATION_POSITION_VALID_BIT) != 0; + // The guest record has a single valid gate and the consumer converts + // position+quaternion together: orientation-only tracking is therefore + // reported untracked rather than paired with a stale position. + auto pose_info = + XrInfo(XR_TYPE_ACTION_STATE_GET_INFO); + pose_info.action = which == 0 ? impl_->grip[hand] : impl_->aim[hand]; + pose_info.subactionPath = subaction; + auto pose_state = XrInfo(XR_TYPE_ACTION_STATE_POSE); + const bool active = XR_SUCCEEDED(xrGetActionStatePose( + session, &pose_info, &pose_state)) && + pose_state.isActive; + hand_active |= active; + const bool usable = active && orientation_valid && position_valid; + float *position = + which == 0 ? hand_pose.grip_position : hand_pose.aim_position; + float *orientation = + which == 0 ? hand_pose.grip_orientation : hand_pose.aim_orientation; + if (usable) { + position[0] = location.pose.position.x; + position[1] = location.pose.position.y; + position[2] = location.pose.position.z; + orientation[0] = location.pose.orientation.x; + orientation[1] = location.pose.orientation.y; + orientation[2] = location.pose.orientation.z; + orientation[3] = location.pose.orientation.w; + } + if (which == 0) { + hand_pose.grip_valid = usable; + } else { + hand_pose.aim_valid = usable; + } + } + hands_[hand] = hand_pose; + builder_.SetPoseValid(hand, hand_pose.grip_valid); + + // Analog inputs. XrActionState{Float,Vector2f} carry the value in + // currentState and validity in isActive. + auto get_info = + XrInfo(XR_TYPE_ACTION_STATE_GET_INFO); + get_info.subactionPath = subaction; + + get_info.action = impl_->trigger; + XrActionStateFloat trigger_state = + XrInfo(XR_TYPE_ACTION_STATE_FLOAT); + if (XR_SUCCEEDED( + xrGetActionStateFloat(session, &get_info, &trigger_state)) && + trigger_state.isActive) { + hand_active = true; + builder_.SetAxis(hand, ControllerAxis::kTrigger, + trigger_state.currentState, true); + } + get_info.action = impl_->squeeze; + XrActionStateFloat squeeze_state = + XrInfo(XR_TYPE_ACTION_STATE_FLOAT); + if (XR_SUCCEEDED( + xrGetActionStateFloat(session, &get_info, &squeeze_state)) && + squeeze_state.isActive) { + hand_active = true; + builder_.SetAxis(hand, ControllerAxis::kSqueeze, + squeeze_state.currentState, true); + } + get_info.action = impl_->stick; + XrActionStateVector2f stick_state = + XrInfo(XR_TYPE_ACTION_STATE_VECTOR2F); + if (XR_SUCCEEDED( + xrGetActionStateVector2f(session, &get_info, &stick_state)) && + stick_state.isActive) { + hand_active = true; + builder_.SetAxis(hand, ControllerAxis::kThumbstickX, + stick_state.currentState.x, true); + builder_.SetAxis(hand, ControllerAxis::kThumbstickY, + stick_state.currentState.y, true); + } + + for (std::size_t i = 0; + i < static_cast(ControllerTouch::kCount); ++i) { + get_info.action = impl_->touches[i]; + auto state = XrInfo(XR_TYPE_ACTION_STATE_BOOLEAN); + const bool available = + XR_SUCCEEDED(xrGetActionStateBoolean(session, &get_info, &state)) && + state.isActive; + builder_.SetTouch(hand, static_cast(i), + state.currentState, available); + } + + // Boolean inputs. + const std::pair booleans[] = { + {impl_->trigger_click, ControllerButton::kTrigger}, + {impl_->squeeze_click, ControllerButton::kSqueeze}, + {impl_->stick_click, ControllerButton::kThumbstick}, + {impl_->face_a, ControllerButton::kFaceSouth}, + {impl_->face_b, ControllerButton::kFaceEast}, + {impl_->face_x, ControllerButton::kFaceWest}, + {impl_->face_y, ControllerButton::kFaceNorth}, + {impl_->menu, ControllerButton::kMenu}, + }; + bool native_trigger_click = false; + bool native_squeeze_click = false; + for (const auto &entry : booleans) { + get_info.action = entry.first; + XrActionStateBoolean state = + XrInfo(XR_TYPE_ACTION_STATE_BOOLEAN); + if (!XR_SUCCEEDED( + xrGetActionStateBoolean(session, &get_info, &state)) || + !state.isActive) { + continue; + } + hand_active = true; + const bool pressed = state.currentState != XR_FALSE; + builder_.SetButton(hand, entry.second, pressed, true); + if (entry.first == impl_->squeeze_click) + native_squeeze_click = true; + if (entry.first == impl_->trigger_click) { + native_trigger_click = true; + } + } + // Native VR has analog shoulder channels as well as trigger channels. + // Only click-only profiles (Vive) need a synthesized 0/1 squeeze value. + if (native_squeeze_click && !squeeze_state.isActive) + builder_.SetAxis( + hand, ControllerAxis::kSqueeze, + builder_.snapshot().hands[hand].buttons[static_cast( + ControllerButton::kSqueeze)] + ? 1.f + : 0.f, + true); + if (native_trigger_click && !trigger_state.isActive) + builder_.SetAxis( + hand, ControllerAxis::kTrigger, + builder_.snapshot() + .hands[hand] + .buttons[static_cast(ControllerButton::kTrigger)] + ? 1.f + : 0.f, + true); + if (!native_squeeze_click) + builder_.SetButton(hand, ControllerButton::kSqueeze, + builder_.AnalogClick(hand, ControllerAxis::kSqueeze), + true); + builder_.SetConnected(hand, hand_active); + impl_->native_trigger_click[hand] = native_trigger_click; + // Profiles without a trigger click action (Touch, Vive) derive the click + // from the analog value with hysteresis; the native analog value itself + // is always preserved. + if (!native_trigger_click) { + builder_.SetButton(hand, ControllerButton::kTrigger, + builder_.AnalogClick(hand, ControllerAxis::kTrigger), + true); + } + } + + // One coherent snapshot for the frame; edges fire exactly once here. + std::vector edges; + snapshot_ = builder_.Publish(frame, session_generation_, &edges); + + ControllerDelivery delivery; + delivery.frame = frame; + delivery.session_generation = session_generation_; + for (int hand = 0; hand < 2; ++hand) { + for (size_t i = 0; i < static_cast(ControllerButton::kCount); ++i) + delivery.button_levels[hand][i] = snapshot_.hands[hand].buttons[i]; + delivery.connected[hand] = snapshot_.hands[hand].connected; + delivery.connection_changed[hand] = + delivery.connected[hand] != last_delivered_connected_[hand]; + last_delivered_connected_[hand] = delivery.connected[hand]; + delivery.axes_valid[hand] = snapshot_.hands[hand].connected; + const auto &axes = snapshot_.hands[hand].axes; + const int stick_x = + ControllerAxisToSdlOrdinal(hand, ControllerAxis::kThumbstickX); + const int stick_y = + ControllerAxisToSdlOrdinal(hand, ControllerAxis::kThumbstickY); + const int trigger = + ControllerAxisToSdlOrdinal(hand, ControllerAxis::kTrigger); + delivery.axes[stick_x] = + axes[static_cast(ControllerAxis::kThumbstickX)]; + delivery.axes[stick_y] = + -axes[static_cast(ControllerAxis::kThumbstickY)]; + delivery.axes[trigger] = + axes[static_cast(ControllerAxis::kTrigger)]; + } + for (const auto &edge : edges) { + if (delivery.edges.size() >= ControllerDelivery::kMaxEdges) { + // Bounded growth: remaining edges are dropped, but a level resync is + // demanded so the consumer converges and nothing sticks down. + delivery.resync = true; + break; + } + delivery.edges.push_back( + ControllerDelivery::ButtonEdge{edge.hand, edge.button, edge.pressed}); + } + delivery_queue_.Push(std::move(delivery)); + + // Apply queued haptic requests inside this XR call window. + { + std::lock_guard lock(impl_->haptic_mutex); + for (int hand = 0; hand < 2; ++hand) { + auto &request = impl_->haptics[hand]; + if (!snapshot_.hands[hand].connected) { + request = {}; + request.pending = true; + request.stop = true; + } + if (!request.pending) { + continue; + } + request.pending = false; + auto haptic_info = + XrInfo(XR_TYPE_HAPTIC_ACTION_INFO); + haptic_info.action = impl_->haptic; + haptic_info.subactionPath = + hand == 0 ? impl_->subaction_left : impl_->subaction_right; + if (request.stop || !snapshot_.hands[hand].connected) { + // Never vibrate a disconnected controller. + (void)xrStopHapticFeedback(session, &haptic_info); + continue; + } + auto vibration = XrInfo(XR_TYPE_HAPTIC_VIBRATION); + vibration.amplitude = request.amplitude; + vibration.duration = request.duration_ns == 0 + ? XR_MIN_HAPTIC_DURATION + : static_cast(std::min( + request.duration_ns, INT64_MAX)); + // 0 lets the runtime choose its resonant frequency; the runtime + // clamps unsupported frequencies itself. + vibration.frequency = request.frequency_hz; + (void)xrApplyHapticFeedback( + session, &haptic_info, + reinterpret_cast(&vibration)); + } + } + return true; + } catch (const std::exception &error) { + Log(" [vr-input] action sync failed: %s\n", error.what()); + ResetInput(); + return false; + } +} + +} // namespace mocktail::vr diff --git a/src/vr/xr_actions_disabled.cc b/src/vr/xr_actions_disabled.cc new file mode 100644 index 00000000..724d7886 --- /dev/null +++ b/src/vr/xr_actions_disabled.cc @@ -0,0 +1,53 @@ +// Inert XrActions implementation for non-VR builds. The public header is +// OpenXR-free (opaque Impl pointer), so these definitions let OpenXrBackend +// hold an XrActions member and link without the OpenXR SDK. Every entry point +// is a no-op: a non-VR build has no session, no runtime and no controllers. +#include "mocktail/vr/xr_actions.h" + +namespace mocktail::vr { + +struct XrActions::Impl {}; + +XrActions::~XrActions() = default; + +Status XrActions::Create(void *) { + return Status::Error(StatusCode::kUnavailable, + "OpenXR support is disabled in this build"); +} + +Status XrActions::Attach(void *, void *, void *) { + return Status::Error(StatusCode::kUnavailable, + "OpenXR support is disabled in this build"); +} + +void XrActions::ResetInput() {} + +void XrActions::Detach() { attached_ = false; } + +void XrActions::Destroy() { + attached_ = false; + created_ = false; + snapshot_ = ControllerSnapshot{}; + hands_[0] = VrHandPose{}; + hands_[1] = VrHandPose{}; + delivery_queue_.Clear(); + last_delivered_connected_[0] = false; + last_delivered_connected_[1] = false; + session_generation_ = 0; +} + +bool XrActions::Sync(void *, std::uint64_t, std::uint64_t, std::uint64_t) { + return false; +} + +bool XrActions::TakeDelivery(ControllerDelivery *) { return false; } + +void XrActions::RequestHaptics(int, float, std::uint64_t, float) {} + +void XrActions::StopHaptics(int) {} + +void XrActions::NoteProfileChanged() {} + +std::string XrActions::ActiveProfile(int) const { return {}; } + +} // namespace mocktail::vr diff --git a/src/vr/xr_controller.cc b/src/vr/xr_controller.cc new file mode 100644 index 00000000..e52c3bbc --- /dev/null +++ b/src/vr/xr_controller.cc @@ -0,0 +1,376 @@ +#include "mocktail/vr/xr_controller.h" + +#include +#include + +namespace mocktail::vr { +namespace { + +constexpr std::size_t kButtons = + static_cast(ControllerButton::kCount); +constexpr std::size_t kAxes = static_cast(ControllerAxis::kCount); + +bool IsValidHand(int hand) { return hand == 0 || hand == 1; } + +} // namespace + +ControllerStateBuilder::ControllerStateBuilder(AnalogClickThresholds thresholds) + : thresholds_(thresholds) {} + +void ControllerStateBuilder::SetTouch(int hand, ControllerTouch touch, + bool value, bool active) { + const auto i = static_cast(touch); + if (!IsValidHand(hand) || + i >= static_cast(ControllerTouch::kCount)) + return; + snapshot_.hands[hand].touches[i] = active && value; + snapshot_.hands[hand].touch_active[i] = active; +} + +void ControllerStateBuilder::SetButton(int hand, ControllerButton button, + bool value, bool active) { + if (!IsValidHand(hand)) { + return; + } + const std::size_t index = static_cast(button); + if (index >= kButtons) { + return; + } + snapshot_.hands[hand].buttons[index] = active && value; +} + +void ControllerStateBuilder::SetAxis(int hand, ControllerAxis axis, float value, + bool active) { + if (!IsValidHand(hand)) { + return; + } + const std::size_t index = static_cast(axis); + if (index >= kAxes) { + return; + } + if (!active || !std::isfinite(value)) { + snapshot_.hands[hand].axes[index] = 0.f; + return; + } + // Clamp to the contract range; runtimes may overshoot slightly on analog + // controls and the guest expects normalized values. + if (axis == ControllerAxis::kTrigger || axis == ControllerAxis::kSqueeze) { + snapshot_.hands[hand].axes[index] = std::clamp(value, 0.f, 1.f); + } else { + snapshot_.hands[hand].axes[index] = std::clamp(value, -1.f, 1.f); + } +} + +void ControllerStateBuilder::SetConnected(int hand, bool connected) { + if (!IsValidHand(hand)) { + return; + } + snapshot_.hands[hand].connected = connected; + if (!connected) { + snapshot_.hands[hand].touches = {}; + snapshot_.hands[hand].touch_active = {}; + analog_latch_[hand * 2] = false; + analog_latch_[hand * 2 + 1] = false; + } +} + +void ControllerStateBuilder::SetProfile(int hand, std::string profile) { + if (!IsValidHand(hand)) { + return; + } + snapshot_.hands[hand].profile = std::move(profile); +} + +void ControllerStateBuilder::SetPoseValid(int hand, bool valid) { + if (!IsValidHand(hand)) { + return; + } + snapshot_.hands[hand].pose_valid = valid; +} + +bool ControllerStateBuilder::AnalogClick(int hand, ControllerAxis axis) { + if (!IsValidHand(hand) || + (axis != ControllerAxis::kTrigger && axis != ControllerAxis::kSqueeze)) { + return false; + } + const float value = + snapshot_.hands[hand].axes[static_cast(axis)]; + const std::size_t axis_index = static_cast(axis); + // Only two analog controls are click-derivable; index them 0/1. + const std::size_t latch_index = static_cast(hand) * 2 + + (axis == ControllerAxis::kTrigger ? 0u : 1u); + const bool latched = analog_latch_[latch_index]; + const bool next = + latched ? value > thresholds_.release : value >= thresholds_.press; + analog_latch_[latch_index] = next; + static_cast(axis_index); + return next; +} + +ControllerSnapshot ControllerStateBuilder::Publish(std::uint64_t frame, + std::uint64_t generation, + std::vector *edges) { + snapshot_.frame = frame; + snapshot_.session_generation = generation; + // Derive click levels for analog controls whose profile has no click action, + // then emit exactly one edge per transition. published_buttons_ starts + // all-false, so the first publish legitimately reports already-held buttons + // as press edges; later publishes report only real transitions, so repeated + // native getters reading the snapshot never see duplicate edges. + for (int hand = 0; hand < 2; ++hand) { + for (std::size_t index = 0; index < kButtons; ++index) { + const bool level = snapshot_.hands[hand].buttons[index]; + const bool previous = published_buttons_[hand][index]; + if (level != previous && edges != nullptr) { + edges->push_back( + Edge{hand, static_cast(index), level}); + } + published_buttons_[hand][index] = level; + } + } + have_published_ = true; + return snapshot_; +} + +void ControllerStateBuilder::ReleaseAll(std::uint64_t frame, + std::uint64_t generation, + std::vector *edges) { + snapshot_.frame = frame; + snapshot_.session_generation = generation; + for (int hand = 0; hand < 2; ++hand) { + snapshot_.hands[hand].touches = {}; + snapshot_.hands[hand].touch_active = {}; + for (std::size_t index = 0; index < kAxes; ++index) { + snapshot_.hands[hand].axes[index] = 0.f; + } + for (std::size_t index = 0; index < kButtons; ++index) { + if (snapshot_.hands[hand].buttons[index] && edges != nullptr) { + edges->push_back( + Edge{hand, static_cast(index), false}); + } + snapshot_.hands[hand].buttons[index] = false; + published_buttons_[hand][index] = false; + } + // Connection and pose validity are properties of the runtime state, not of + // the press levels; callers set them explicitly around this call. + } + analog_latch_.fill(false); + have_published_ = true; +} + +int32_t ControllerButtonToAndroidKey(int hand, ControllerButton button) { + // Android KEYCODE_BUTTON_* values already verified end-to-end by the + // existing physical-gamepad route (MapSdlGamepadButtonToAndroid). The + // per-hand squeeze/thumbstick map onto the shoulder/thumb identifiers the + // guest already accepts; the right hand uses the R-side variants. + const bool right = hand == 1; + switch (button) { + case ControllerButton::kFaceSouth: + return 96; // BUTTON_A + case ControllerButton::kFaceEast: + return 97; // BUTTON_B + case ControllerButton::kFaceWest: + return 99; // BUTTON_X + case ControllerButton::kFaceNorth: + return 100; // BUTTON_Y + case ControllerButton::kSqueeze: + return right ? 103 : 102; // R1 / L1 + case ControllerButton::kThumbstick: + return right ? 107 : 106; // THUMB_R / L + case ControllerButton::kMenu: + return 108; // BUTTON_START + case ControllerButton::kTrigger: + return -1; // analog click, no keycode + case ControllerButton::kCount: + break; + } + return -1; +} + +int32_t ControllerAxisToAndroidAxis(int hand, ControllerAxis axis) { + // Android MotionEvent AXIS_* values from kAndroidAxes in the verified + // gamepad route: {AXIS_X=0, AXIS_Y=1, AXIS_Z=11, AXIS_RZ=14, + // AXIS_LTRIGGER=17, AXIS_RTRIGGER=18}. + switch (axis) { + case ControllerAxis::kThumbstickX: + return hand == 1 ? 11 : 0; // Z / X + case ControllerAxis::kThumbstickY: + return hand == 1 ? 14 : 1; // RZ / Y + case ControllerAxis::kTrigger: + return hand == 1 ? 18 : 17; // R / L trigger + case ControllerAxis::kSqueeze: + return -1; // squeeze has no analog axis id + case ControllerAxis::kCount: + break; + } + return -1; +} + +int32_t ControllerAxisComponentCount(ControllerAxis axis) { + switch (axis) { + case ControllerAxis::kThumbstickX: + case ControllerAxis::kThumbstickY: + return 2; // sticks are delivered as an (x, y) motion pair + case ControllerAxis::kTrigger: + return 1; + default: + return 0; + } +} + +int32_t ControllerButtonToSdlOrdinal(int hand, ControllerButton button) { + // SDL_GamepadButton ordinals (SDL3): SOUTH=0, EAST=1, WEST=2, NORTH=3, + // BACK=4, START=6, LEFT_STICK=7, RIGHT_STICK=8, LEFT_SHOULDER=9, + // RIGHT_SHOULDER=10. The router converts these to the verified Android + // keycodes; XR delivery must not bypass that conversion. + switch (button) { + case ControllerButton::kFaceSouth: + return 0; + case ControllerButton::kFaceEast: + return 1; + case ControllerButton::kFaceWest: + return 2; + case ControllerButton::kFaceNorth: + return 3; + case ControllerButton::kThumbstick: + return hand == 1 ? 8 : 7; + case ControllerButton::kSqueeze: + return hand == 1 ? 10 : 9; + case ControllerButton::kMenu: + return 6; + case ControllerButton::kTrigger: + return -1; // analog click, no button + case ControllerButton::kCount: + break; + } + return -1; +} + +int32_t ControllerAxisToSdlOrdinal(int hand, ControllerAxis axis) { + // SDL_GamepadAxis ordinals (SDL3): LEFTX=0, LEFTY=1, RIGHTX=2, RIGHTY=3, + // TRIGGERLEFT=4, TRIGGERRIGHT=5. + switch (axis) { + case ControllerAxis::kThumbstickX: + return hand == 1 ? 2 : 0; + case ControllerAxis::kThumbstickY: + return hand == 1 ? 3 : 1; + case ControllerAxis::kTrigger: + return hand == 1 ? 5 : 4; + case ControllerAxis::kSqueeze: + return -1; // delivered as a button only + case ControllerAxis::kCount: + break; + } + return -1; +} + +bool EncodeNativeControllerChannels(const ControllerSnapshot &snapshot, + float (&channels)[28]) { + std::fill(std::begin(channels), std::end(channels), 0.f); + for (int hand = 0; hand < 2; ++hand) { + const auto &input = snapshot.hands[hand]; + if (!input.connected) + continue; + auto button = [&](ControllerButton b) { + return input.buttons[static_cast(b)] ? 1.f : 0.f; + }; + const int base = hand * 9; + channels[base] = button(ControllerButton::kThumbstick); + channels[base + 1] = + input.axes[static_cast(ControllerAxis::kThumbstickX)]; + channels[base + 2] = + input.axes[static_cast(ControllerAxis::kThumbstickY)]; + channels[base + 3] = + input.axes[static_cast(ControllerAxis::kTrigger)]; + channels[base + 4] = + input.axes[static_cast(ControllerAxis::kSqueeze)]; + channels[hand == 0 ? 24 : 22] = button( + hand == 0 ? ControllerButton::kFaceWest : ControllerButton::kFaceSouth); + channels[hand == 0 ? 25 : 23] = button( + hand == 0 ? ControllerButton::kFaceNorth : ControllerButton::kFaceEast); + channels[27] = std::max(channels[27], button(ControllerButton::kMenu)); + } + + return snapshot.hands[0].connected || snapshot.hands[1].connected; +} + +void EncodeControllerDelivery(const ControllerDelivery &delivery, + MocktailVrControllerDelivery *out) { + if (!out) + return; + *out = MocktailVrControllerDelivery{}; + out->frame = delivery.frame; + out->resync = delivery.resync ? 1 : 0; + out->session_generation = delivery.session_generation; + const std::size_t button_count = + static_cast(mocktail::vr::ControllerButton::kCount); + for (int hand = 0; hand < 2; ++hand) { + out->connected[hand] = delivery.connected[hand] ? 1 : 0; + out->connection_changed[hand] = delivery.connection_changed[hand] ? 1 : 0; + out->axes_valid[hand] = delivery.axes_valid[hand] ? 1 : 0; + for (std::size_t index = 0; + index < button_count && + index < static_cast(MOCKTAIL_VR_CONTROLLER_BUTTON_COUNT); + ++index) { + out->button_levels[hand][index] = + delivery.button_levels[hand][index] ? 1 : 0; + } + } + for (int axis = 0; axis < MOCKTAIL_VR_CONTROLLER_AXIS_COUNT; ++axis) { + out->axes[axis] = delivery.axes[axis]; + } + const std::size_t edges = std::min( + delivery.edges.size(), MOCKTAIL_VR_CONTROLLER_MAX_EDGES); + out->edge_count = static_cast(edges); + for (std::size_t index = 0; index < edges; ++index) { + out->edges[index].hand = delivery.edges[index].hand; + out->edges[index].button = + static_cast(delivery.edges[index].button); + out->edges[index].pressed = delivery.edges[index].pressed ? 1 : 0; + } +} + +void ControllerDeliveryQueue::Push(ControllerDelivery delivery) { + std::lock_guard lock(mutex_); + if (overflowed_) { + // The next delivered frame must re-sync full levels because intermediate + // edges were dropped with the discarded frames. + delivery.resync = true; + overflowed_ = false; + } + if (queue_.size() >= kCapacity) { + queue_.pop_front(); + ++dropped_frames_; + overflowed_ = true; + // Edges inside the dropped frame are lost; converge through resync on the + // newest frame so nothing can stick down. + delivery.resync = true; + } + queue_.push_back(std::move(delivery)); +} + +bool ControllerDeliveryQueue::Pop(ControllerDelivery *out) { + if (out == nullptr) { + return false; + } + std::lock_guard lock(mutex_); + if (queue_.empty()) { + return false; + } + *out = std::move(queue_.front()); + queue_.pop_front(); + return true; +} + +std::size_t ControllerDeliveryQueue::size() const { + std::lock_guard lock(mutex_); + return queue_.size(); +} + +void ControllerDeliveryQueue::Clear() { + std::lock_guard lock(mutex_); + queue_.clear(); + overflowed_ = false; +} + +} // namespace mocktail::vr diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 84876200..aa510729 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -36,6 +36,9 @@ add_executable(vr_pose_math_test vr/vr_pose_math_test.cc) target_include_directories(vr_pose_math_test PRIVATE "${PROJECT_SOURCE_DIR}/include") target_link_libraries(vr_pose_math_test PRIVATE GTest::gtest_main) mocktail_apply_compile_options(vr_pose_math_test) +add_executable(xr_controller_test vr/xr_controller_test.cc) +target_link_libraries(xr_controller_test PRIVATE Mocktail::VR GTest::gtest_main) +mocktail_apply_compile_options(xr_controller_test) add_executable(vr_evidence_layer_test vr/vr_evidence_layer_test.cc) target_link_libraries(vr_evidence_layer_test PRIVATE Vulkan::Headers ${CMAKE_DL_LIBS} GTest::gtest_main) @@ -947,6 +950,7 @@ gtest_discover_tests(roblox_vr_device_bridge_test) gtest_discover_tests(vr_evidence_layer_test) gtest_discover_tests(vr_mirror_test) gtest_discover_tests(vr_pose_math_test) +gtest_discover_tests(xr_controller_test) gtest_discover_tests(jnivm_test) gtest_discover_tests(runtime_config_test) gtest_discover_tests(runtime_config_file_test) @@ -1135,3 +1139,10 @@ add_test( ${CMAKE_BINARY_DIR} ) endif() + +if(MOCKTAIL_ENABLE_VR) + add_executable(xr_actions_test vr/xr_actions_test.cc) + target_link_libraries(xr_actions_test PRIVATE Mocktail::InputRuntime GTest::gtest_main) + target_include_directories(xr_actions_test PRIVATE "${CMAKE_SOURCE_DIR}/third_party/OpenXR-SDK/include") + gtest_discover_tests(xr_actions_test) +endif() diff --git a/tests/roblox_xr_controller_input_test.cc b/tests/roblox_xr_controller_input_test.cc new file mode 100644 index 00000000..fe95d02b --- /dev/null +++ b/tests/roblox_xr_controller_input_test.cc @@ -0,0 +1,295 @@ +#include + +#include +#include +#include +#include +#include + +#include "mocktail/platform/platform_runtime.h" +#include "mocktail/vr/xr_controller_abi.h" +#include "runtime/roblox_xr_controller_input.h" + +namespace mocktail { +namespace runtime { +namespace { + +using platform::GamepadAxisEvent; +using platform::GamepadButtonEvent; +using platform::GamepadConnectionEvent; +using platform::PlatformEvent; + +// The XR controller ABI is context-free (resolved through dlsym in +// production), so the test doubles live at file scope exactly like the real +// exported symbols would. +struct FakeBackend { + std::deque queue; + int haptic_calls = 0; + int stop_calls = 0; + float last_amplitude = 0.f; + std::uint64_t last_duration_ns = 0; + int last_hand = -1; +} g_backend; + +std::vector g_events; + +int32_t FakePop(MocktailVrControllerDelivery *out) { + if (g_backend.queue.empty()) { + return 0; + } + *out = g_backend.queue.front(); + g_backend.queue.pop_front(); + return 1; +} + +void FakeHaptics(int32_t hand, float amplitude, std::uint64_t duration_ns, + float) { + g_backend.haptic_calls++; + g_backend.last_hand = hand; + g_backend.last_amplitude = amplitude; + g_backend.last_duration_ns = duration_ns; +} + +void FakeStop(int32_t hand) { + g_backend.stop_calls++; + g_backend.last_hand = hand; +} + +void FakeSink(void *, const PlatformEvent &event) { g_events.push_back(event); } + +template std::vector Collect() { + std::vector out; + for (const auto &event : g_events) { + if (const auto *value = std::get_if(&event.payload)) { + out.push_back(*value); + } + } + return out; +} + +MocktailVrControllerDelivery MakeDelivery() { + MocktailVrControllerDelivery delivery{}; + delivery.connected[0] = 1; + delivery.connected[1] = 1; + delivery.axes_valid[0] = 1; + delivery.axes_valid[1] = 1; + return delivery; +} + +class XrControllerInputTest : public ::testing::Test { +protected: + void SetUp() override { + g_backend = FakeBackend{}; + g_events.clear(); + ASSERT_TRUE(input_.InitializeWithAbi(&FakeSink, nullptr, &FakePop, + &FakeHaptics, &FakeStop)); + ASSERT_TRUE(input_.active()); + } + RobloxXrControllerInput input_; +}; + +TEST_F(XrControllerInputTest, ConnectsOnceAndMergesBothHandsIntoOneGamepad) { + auto delivery = MakeDelivery(); + delivery.frame = 1; + // Left hand presses X (face west), right hand presses A (face south). + delivery.button_levels[0][MOCKTAIL_VR_BUTTON_FACE_WEST] = 1; + delivery.button_levels[1][MOCKTAIL_VR_BUTTON_FACE_SOUTH] = 1; + delivery.edge_count = 2; + delivery.edges[0] = {0, MOCKTAIL_VR_BUTTON_FACE_WEST, 1}; + delivery.edges[1] = {1, MOCKTAIL_VR_BUTTON_FACE_SOUTH, 1}; + g_backend.queue.push_back(delivery); + + input_.Drain(); + + const auto connections = Collect(); + ASSERT_EQ(connections.size(), 1u); + EXPECT_TRUE(connections[0].connected); + EXPECT_EQ(connections[0].instance_id, RobloxXrControllerInput::kInstanceLeft); + const auto buttons = Collect(); + ASSERT_EQ(buttons.size(), 2u); + EXPECT_EQ(buttons[0].instance_id, RobloxXrControllerInput::kInstanceLeft); + EXPECT_TRUE(buttons[0].pressed); + EXPECT_TRUE(buttons[1].pressed); + EXPECT_NE(buttons[0].button, buttons[1].button); +} + +TEST_F(XrControllerInputTest, + ResyncDoesNotCancelLeftHandWithUnboundRightButton) { + auto d = MakeDelivery(); + d.resync = 1; + d.button_levels[0][MOCKTAIL_VR_BUTTON_FACE_WEST] = 1; + g_backend.queue.push_back(d); + input_.Drain(); + auto buttons = Collect(); + bool west = false; + for (const auto &b : buttons) + if (b.button == 2) + west = b.pressed; + EXPECT_TRUE(west); +} +TEST_F(XrControllerInputTest, SharedMenuReleasesOnlyAfterBothHandsRelease) { + auto d = MakeDelivery(); + d.button_levels[0][MOCKTAIL_VR_BUTTON_MENU] = 1; + d.button_levels[1][MOCKTAIL_VR_BUTTON_MENU] = 1; + g_backend.queue.push_back(d); + input_.Drain(); + g_events.clear(); + d.button_levels[0][MOCKTAIL_VR_BUTTON_MENU] = 0; + g_backend.queue.push_back(d); + input_.Drain(); + EXPECT_TRUE(Collect().empty()); + d.button_levels[1][MOCKTAIL_VR_BUTTON_MENU] = 0; + g_backend.queue.push_back(d); + input_.Drain(); + auto buttons = Collect(); + ASSERT_EQ(buttons.size(), 1u); + EXPECT_FALSE(buttons[0].pressed); +} +TEST_F(XrControllerInputTest, NewGenerationDisconnectsAndRejectsLateOldFrames) { + auto d = MakeDelivery(); + d.session_generation = 1; + g_backend.queue.push_back(d); + input_.Drain(); + g_events.clear(); + d.session_generation = 2; + g_backend.queue.push_back(d); + input_.Drain(); + auto connections = Collect(); + ASSERT_EQ(connections.size(), 2u); + EXPECT_FALSE(connections[0].connected); + EXPECT_TRUE(connections[1].connected); + g_events.clear(); + d.session_generation = 1; + g_backend.queue.push_back(d); + input_.Drain(); + EXPECT_TRUE(g_events.empty()); +} + +TEST_F(XrControllerInputTest, AxesDeliveredAsRawInt16ForSingleDeadzone) { + auto delivery = MakeDelivery(); + delivery.frame = 1; + delivery.axes[0] = -1.0f; // LEFTX + delivery.axes[5] = 0.5f; // TRIGGERRIGHT + g_backend.queue.push_back(delivery); + + input_.Drain(); + + const auto axes = Collect(); + ASSERT_FALSE(axes.empty()); + bool saw_left_x = false, saw_right_trigger = false; + for (const auto &axis : axes) { + EXPECT_EQ(axis.instance_id, RobloxXrControllerInput::kInstanceLeft); + if (axis.axis == 0) { + EXPECT_EQ(axis.value, -32767); + saw_left_x = true; + } + if (axis.axis == 5) { + EXPECT_NEAR(axis.value, 16384, 2); + saw_right_trigger = true; + } + } + EXPECT_TRUE(saw_left_x); + EXPECT_TRUE(saw_right_trigger); +} + +TEST_F(XrControllerInputTest, DisconnectReleasesAndStopsFurtherDelivery) { + auto connected = MakeDelivery(); + connected.frame = 1; + connected.button_levels[1][MOCKTAIL_VR_BUTTON_FACE_SOUTH] = 1; + connected.edge_count = 1; + connected.edges[0] = {1, MOCKTAIL_VR_BUTTON_FACE_SOUTH, 1}; + g_backend.queue.push_back(connected); + input_.Drain(); + ASSERT_TRUE(input_.connected()); + + auto disconnected = MakeDelivery(); + disconnected.frame = 2; + disconnected.connected[0] = 0; + disconnected.connected[1] = 0; + g_backend.queue.push_back(disconnected); + input_.Drain(); + + const auto connections = Collect(); + ASSERT_EQ(connections.size(), 2u); + EXPECT_TRUE(connections[0].connected); + EXPECT_FALSE(connections[1].connected); + EXPECT_FALSE(input_.connected()); +} + +TEST_F(XrControllerInputTest, ResyncReDeliversEveryHeldLevel) { + auto delivery = MakeDelivery(); + delivery.frame = 5; + delivery.resync = 1; + // Right A held, left stick click held. + delivery.button_levels[1][MOCKTAIL_VR_BUTTON_FACE_SOUTH] = 1; + delivery.button_levels[0][MOCKTAIL_VR_BUTTON_THUMBSTICK] = 1; + g_backend.queue.push_back(delivery); + + input_.Drain(); + + EXPECT_EQ(input_.resyncs(), 1u); + const auto buttons = Collect(); + bool saw_face_south = false, saw_thumbstick = false; + for (const auto &button : buttons) { + if (!button.pressed) { + continue; + } + if (button.button == 0) + saw_face_south = true; // SOUTH ordinal 0 + if (button.button == 7) + saw_thumbstick = true; // LEFT_STICK ordinal 7 + } + EXPECT_TRUE(saw_face_south); + EXPECT_TRUE(saw_thumbstick); +} + +TEST_F(XrControllerInputTest, DrainIsBoundedPerPump) { + for (int index = 0; index < 10; ++index) { + auto delivery = MakeDelivery(); + delivery.frame = static_cast(index); + g_backend.queue.push_back(delivery); + } + input_.Drain(); + // kMaxDrainPerPump is 4, so at least 6 batches remain queued. + EXPECT_GE(g_backend.queue.size(), 6u); +} + +TEST_F(XrControllerInputTest, HapticsForwardedAndStopOnRequest) { + input_.RequestHaptics(1, 0.75f, 120000000ULL, 160.f); + EXPECT_EQ(g_backend.haptic_calls, 1); + EXPECT_EQ(g_backend.last_hand, 1); + EXPECT_FLOAT_EQ(g_backend.last_amplitude, 0.75f); + EXPECT_EQ(g_backend.last_duration_ns, 120000000ULL); + input_.StopHaptics(0); + EXPECT_EQ(g_backend.stop_calls, 1); + EXPECT_EQ(g_backend.last_hand, 0); + // Invalid hand indices are ignored. + input_.RequestHaptics(5, 1.f, 0, 0.f); + EXPECT_EQ(g_backend.haptic_calls, 1); +} + +TEST_F(XrControllerInputTest, InertWithoutPopFunction) { + RobloxXrControllerInput inert; + EXPECT_FALSE( + inert.InitializeWithAbi(&FakeSink, nullptr, nullptr, nullptr, nullptr)); + EXPECT_FALSE(inert.active()); + inert.Drain(); // safe no-op + EXPECT_TRUE(g_events.empty()); +} + +TEST_F(XrControllerInputTest, ShutdownDisconnectsSyntheticController) { + auto delivery = MakeDelivery(); + delivery.frame = 1; + g_backend.queue.push_back(delivery); + input_.Drain(); + ASSERT_TRUE(input_.connected()); + input_.Shutdown(); + const auto connections = Collect(); + ASSERT_FALSE(connections.empty()); + EXPECT_FALSE(connections.back().connected); + EXPECT_FALSE(input_.connected()); +} + +} // namespace +} // namespace runtime +} // namespace mocktail diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc index 6f9cbe3e..0dc9d165 100644 --- a/tests/vr/roblox_vr_device_bridge_test.cc +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -501,3 +501,148 @@ TEST(VrPoseBridge, InvalidTrackingAndNonfiniteValuesNeverBecomeValid) { EXPECT_FALSE(internal::ApplyXrPose(state.data(), pose)); } } // namespace mocktail::vr + +namespace mocktail::vr { +TEST(VrHandPoseBridge, ValidGripPosesFillBothHandRecordsInMetres) { + std::array state; + state.fill(0xa5); + ScriptedPoseSample pose; + pose.valid = true; + pose.orientation[3] = 1.f; + pose.hands[0].grip_valid = true; + pose.hands[0].grip_position[0] = -0.25f; + pose.hands[0].grip_position[1] = 1.0f; + pose.hands[0].grip_orientation[1] = std::sin(0.3f); + pose.hands[0].grip_orientation[3] = std::cos(0.3f); + pose.hands[1].grip_valid = true; + pose.hands[1].grip_position[0] = 0.25f; + pose.hands[1].grip_orientation[3] = 1.f; + ASSERT_TRUE(internal::ApplyXrPose(state.data(), pose)); + // Left hand record at copy+0x40 (UserCFrame index 1), right at +0x60 (2). + EXPECT_EQ(state[internal::kLeftHandRecordOffset], 1); + EXPECT_EQ(state[internal::kRightHandRecordOffset], 1); + // The unused extra record (index 3) must stay invalid. + EXPECT_EQ(state[internal::kExtraRecordOffset], 0); + float left_pos[3], left_ori[4], right_pos[3]; + std::memcpy(left_pos, state.data() + internal::kLeftHandRecordOffset + + internal::kRecordPositionOffset, 12); + std::memcpy(left_ori, state.data() + internal::kLeftHandRecordOffset + + internal::kRecordOrientationOffset, 16); + std::memcpy(right_pos, state.data() + internal::kRightHandRecordOffset + + internal::kRecordPositionOffset, 12); + // Metres only: the guest's 0x37eacb8 applies the 10/3 stud conversion. + EXPECT_FLOAT_EQ(left_pos[0], -0.25f); + EXPECT_FLOAT_EQ(left_pos[1], 1.0f); + EXPECT_FLOAT_EQ(left_ori[1], std::sin(0.3f)); + EXPECT_FLOAT_EQ(left_ori[3], std::cos(0.3f)); + EXPECT_FLOAT_EQ(right_pos[0], 0.25f); +} + +TEST(VrHandPoseBridge, UntrackedOrNonFiniteHandsStayInvalidWithoutTouchingHead) { + std::array state; + state.fill(0); + ScriptedPoseSample pose; + pose.valid = true; + pose.orientation[3] = 1.f; + // Left hand untracked (grip_valid false), right hand has a NaN quaternion. + pose.hands[0].grip_valid = false; + pose.hands[1].grip_valid = true; + pose.hands[1].grip_orientation[0] = std::numeric_limits::quiet_NaN(); + ASSERT_TRUE(internal::ApplyXrPose(state.data(), pose)); + EXPECT_EQ(state[0], 1); // head still applied + EXPECT_EQ(state[internal::kLeftHandRecordOffset], 0); + EXPECT_EQ(state[internal::kRightHandRecordOffset], 0); + // A non-normalized quaternion must also be rejected. + pose.hands[1].grip_orientation[0] = 0.f; + pose.hands[1].grip_orientation[3] = 0.5f; + ASSERT_TRUE(internal::ApplyXrPose(state.data(), pose)); + EXPECT_EQ(state[internal::kRightHandRecordOffset], 0); +} + +TEST(VrHandPoseBridge, LosingTrackingClearsHandValidByteNotWholeRecord) { + std::array state; + state.fill(0); + ScriptedPoseSample pose; + pose.valid = true; + pose.orientation[3] = 1.f; + pose.hands[0].grip_valid = true; + pose.hands[0].grip_orientation[3] = 1.f; + ASSERT_TRUE(internal::ApplyXrPose(state.data(), pose)); + ASSERT_EQ(state[internal::kLeftHandRecordOffset], 1); + // Tracking lost: valid byte clears so the guest reports untracked; the stale + // pose bytes remain but are unreachable while valid==0 (no teleport to origin). + pose.hands[0].grip_valid = false; + ASSERT_TRUE(internal::ApplyXrPose(state.data(), pose)); + EXPECT_EQ(state[internal::kLeftHandRecordOffset], 0); + float stale[3]; + std::memcpy(stale, state.data() + internal::kLeftHandRecordOffset + + internal::kRecordPositionOffset, 12); + EXPECT_FLOAT_EQ(stale[0], 0.f); // position was never written for this hand +} +} // namespace mocktail::vr + +namespace mocktail::vr { +TEST(VrNativeController, StateCopyPublishesChannelsAndHapticsWithoutChangingReadiness) { + std::array state{}; + state[0x133] = 77; + ScriptedPoseSample pose; + pose.valid = pose.controllers_connected = true; + pose.controller_channels[3] = .7f; + pose.controller_channels[11] = -1.f; + pose.controller_channels[24] = 1.f; + ASSERT_TRUE(internal::ApplyXrPose(state.data(), pose)); + EXPECT_EQ(state[0x80], 1); + EXPECT_EQ(state[0x132], 1); + EXPECT_EQ(state[0x133], 77); + float trigger, y, x; + std::memcpy(&trigger, state.data() + 0x88 + 3*4, 4); + std::memcpy(&y, state.data() + 0x88 + 11*4, 4); + std::memcpy(&x, state.data() + 0x88 + 24*4, 4); + EXPECT_FLOAT_EQ(trigger, .7f); + EXPECT_FLOAT_EQ(y, -1.f); + EXPECT_FLOAT_EQ(x, 1.f); + pose.valid = false; + EXPECT_FALSE(internal::ApplyXrPose(state.data(), pose)); + EXPECT_EQ(state[0x80], 0); + EXPECT_EQ(state[0x132], 0); + std::memcpy(&trigger, state.data() + 0x88 + 3*4, 4); + EXPECT_FLOAT_EQ(trigger, 0); +} +TEST(VrAimPointer, CorrectsOnlyPointerAndAppliesTranslationScaleOnce) { + float frame[12] = {1,0,0, 0,1,0, 0,0,1, 10,20,30}; + VrHandPose hand; + hand.grip_valid = hand.aim_valid = true; + hand.aim_position[0] = .3f; + hand.aim_orientation[1] = std::sin(.3f); + hand.aim_orientation[3] = std::cos(.3f); + ASSERT_TRUE(internal::ApplyAimToWorldFrame(frame, hand, 2, 0)); + EXPECT_NEAR(frame[9], 12, 1e-5); + EXPECT_FLOAT_EQ(frame[10], 20); + EXPECT_NEAR(frame[0], std::cos(.6f), 1e-5); + EXPECT_NEAR(frame[2], std::sin(.6f), 1e-5); + EXPECT_FLOAT_EQ(hand.grip_orientation[3], 1); +} +TEST(VrAimPointer, CancelsNativeHandPitchWithoutChangingWorldOrigin) { + const float a = -.3490658504f; + float frame[12] = {1,0,0, 0,std::cos(a),-std::sin(a), 0,std::sin(a),std::cos(a), 10,20,30}; + VrHandPose hand; + hand.grip_valid = hand.aim_valid = true; + ASSERT_TRUE(internal::ApplyAimToWorldFrame(frame, hand, 1, -20)); + EXPECT_NEAR(frame[0], 1, 1e-5); + EXPECT_NEAR(frame[5], 0, 1e-5); + EXPECT_NEAR(frame[8], 1, 1e-5); + EXPECT_FLOAT_EQ(frame[9], 10); +} +TEST(VrAimPointer, InvalidAimOrScaleKeepsOriginalFrame) { + const std::array original{1,0,0, 0,1,0, 0,0,1, 10,20,30}; + auto frame = original; + VrHandPose hand; + EXPECT_FALSE(internal::ApplyAimToWorldFrame(frame.data(), hand, 1, 0)); + EXPECT_EQ(frame, original); + hand.grip_valid = hand.aim_valid = true; + EXPECT_FALSE(internal::ApplyAimToWorldFrame(frame.data(), hand, -1, 0)); + hand.aim_orientation[0] = std::numeric_limits::quiet_NaN(); + EXPECT_FALSE(internal::ApplyAimToWorldFrame(frame.data(), hand, 1, 0)); + EXPECT_EQ(frame, original); +} +} diff --git a/tests/vr/xr_actions_test.cc b/tests/vr/xr_actions_test.cc new file mode 100644 index 00000000..cd2c6533 --- /dev/null +++ b/tests/vr/xr_actions_test.cc @@ -0,0 +1,372 @@ +#include "mocktail/vr/xr_actions.h" +#include "runtime/roblox_input_router.h" +#include "runtime/roblox_xr_controller_input.h" +#include +#include +#include +#include +#include + +namespace { +using namespace mocktail::vr; +template T Handle(uintptr_t n) { return reinterpret_cast(n); } +std::map names; +std::map paths; +uintptr_t next_handle; +bool focused, input_active, simple, touching; +float trigger, squeeze, stick_y; +bool click; +int syncs, stops, vibrations; +XrActionSet action_set; +XrTime located_time; +float last_amplitude; +XrDuration last_duration; +XrPath last_haptic_hand; +int simple_haptic_bindings; +} // namespace +extern "C" { +XRAPI_ATTR XrResult XRAPI_CALL +xrResultToString(XrInstance, XrResult, char buffer[XR_MAX_RESULT_STRING_SIZE]) { + std::strcpy(buffer, "fake"); + return XR_SUCCESS; +} +XRAPI_ATTR XrResult XRAPI_CALL xrStringToPath(XrInstance, const char *text, + XrPath *out) { + auto &p = paths[text]; + if (!p) + p = ++next_handle; + *out = p; + return XR_SUCCESS; +} +XRAPI_ATTR XrResult XRAPI_CALL xrPathToString(XrInstance, XrPath path, + uint32_t cap, uint32_t *count, + char *out) { + for (auto &p : paths) + if (p.second == path) { + *count = p.first.size() + 1; + if (cap >= *count) + std::strcpy(out, p.first.c_str()); + return XR_SUCCESS; + } + return XR_ERROR_PATH_INVALID; +} +XRAPI_ATTR XrResult XRAPI_CALL xrCreateActionSet(XrInstance, + const XrActionSetCreateInfo *, + XrActionSet *out) { + *out = action_set = Handle(++next_handle); + return XR_SUCCESS; +} +XRAPI_ATTR XrResult XRAPI_CALL xrDestroyActionSet(XrActionSet) { + return XR_SUCCESS; +} +XRAPI_ATTR XrResult XRAPI_CALL xrCreateAction(XrActionSet, + const XrActionCreateInfo *info, + XrAction *out) { + *out = Handle(++next_handle); + names[*out] = info->actionName; + return XR_SUCCESS; +} +XRAPI_ATTR XrResult XRAPI_CALL xrDestroyAction(XrAction) { return XR_SUCCESS; } +XRAPI_ATTR XrResult XRAPI_CALL xrSuggestInteractionProfileBindings( + XrInstance, const XrInteractionProfileSuggestedBinding *info) { + if (info->interactionProfile == + paths["/interaction_profiles/khr/simple_controller"]) + for (std::uint32_t i = 0; i < info->countSuggestedBindings; ++i) + if (names[info->suggestedBindings[i].action] == "haptic") + ++simple_haptic_bindings; + return XR_SUCCESS; +} +XRAPI_ATTR XrResult XRAPI_CALL xrAttachSessionActionSets( + XrSession, const XrSessionActionSetsAttachInfo *info) { + EXPECT_EQ(info->countActionSets, 1u); + EXPECT_EQ(info->actionSets[0], action_set); + return XR_SUCCESS; +} +XRAPI_ATTR XrResult XRAPI_CALL +xrCreateActionSpace(XrSession, const XrActionSpaceCreateInfo *, XrSpace *out) { + *out = Handle(++next_handle); + return XR_SUCCESS; +} +XRAPI_ATTR XrResult XRAPI_CALL xrDestroySpace(XrSpace) { return XR_SUCCESS; } +XRAPI_ATTR XrResult XRAPI_CALL xrSyncActions(XrSession, + const XrActionsSyncInfo *info) { + ++syncs; + EXPECT_EQ(info->countActiveActionSets, 1u); + if (info->countActiveActionSets != 1 || !info->activeActionSets) + return XR_ERROR_VALIDATION_FAILURE; + EXPECT_EQ(info->activeActionSets[0].actionSet, action_set); + EXPECT_EQ(info->activeActionSets[0].subactionPath, XR_NULL_PATH); + return focused ? XR_SUCCESS : XR_SESSION_NOT_FOCUSED; +} +XRAPI_ATTR XrResult XRAPI_CALL xrGetCurrentInteractionProfile( + XrSession, XrPath, XrInteractionProfileState *out) { + return xrStringToPath(nullptr, + simple + ? "/interaction_profiles/khr/simple_controller" + : "/interaction_profiles/oculus/touch_controller", + &out->interactionProfile); +} +XRAPI_ATTR XrResult XRAPI_CALL xrLocateSpace(XrSpace, XrSpace, XrTime time, + XrSpaceLocation *out) { + located_time = time; + out->locationFlags = XR_SPACE_LOCATION_POSITION_VALID_BIT | + XR_SPACE_LOCATION_ORIENTATION_VALID_BIT; + out->pose.orientation.w = 1; + out->pose.position.x = .1f; + return XR_SUCCESS; +} +XRAPI_ATTR XrResult XRAPI_CALL xrGetActionStatePose( + XrSession, const XrActionStateGetInfo *, XrActionStatePose *out) { + out->isActive = input_active; + return XR_SUCCESS; +} +XRAPI_ATTR XrResult XRAPI_CALL xrGetActionStateFloat( + XrSession, const XrActionStateGetInfo *info, XrActionStateFloat *out) { + out->isActive = input_active && !simple; + out->currentState = names[info->action] == "trigger" ? trigger : squeeze; + return XR_SUCCESS; +} +XRAPI_ATTR XrResult XRAPI_CALL xrGetActionStateVector2f( + XrSession, const XrActionStateGetInfo *, XrActionStateVector2f *out) { + out->isActive = input_active && !simple; + out->currentState = {0, stick_y}; + return XR_SUCCESS; +} +XRAPI_ATTR XrResult XRAPI_CALL xrGetActionStateBoolean( + XrSession, const XrActionStateGetInfo *info, XrActionStateBoolean *out) { + out->isActive = + input_active && simple && names[info->action] == "trigger_click"; + if (names[info->action] == "trigger_touch") { + out->isActive = input_active && !simple; + out->currentState = touching; + } else + out->currentState = click; + return XR_SUCCESS; +} +XRAPI_ATTR XrResult XRAPI_CALL +xrStopHapticFeedback(XrSession, const XrHapticActionInfo *) { + ++stops; + return XR_SUCCESS; +} +XRAPI_ATTR XrResult XRAPI_CALL xrApplyHapticFeedback( + XrSession, const XrHapticActionInfo *info, const XrHapticBaseHeader *base) { + const auto *vibration = reinterpret_cast(base); + last_amplitude = vibration->amplitude; + last_duration = vibration->duration; + last_haptic_hand = info->subactionPath; + ++vibrations; + return XR_SUCCESS; +} +} +namespace { +class XrActionsProduction : public testing::Test { +protected: + XrActions actions; + void SetUp() override { + names.clear(); + paths.clear(); + next_handle = 100; + focused = input_active = true; + simple = click = touching = false; + trigger = squeeze = stick_y = 0; + syncs = stops = vibrations = simple_haptic_bindings = 0; + last_amplitude = 0; + last_duration = 0; + last_haptic_hand = 0; + ASSERT_TRUE(actions.Create(Handle(1)).ok()); + ASSERT_TRUE( + actions.Attach(Handle(1), Handle(2), Handle(3)) + .ok()); + } + bool Sync(uint64_t frame = 1) { + return actions.Sync(Handle(2), 123456, frame, 0); + } + ControllerDelivery Pop() { + ControllerDelivery d; + EXPECT_TRUE(actions.TakeDelivery(&d)); + return d; + } +}; + +TEST_F(XrActionsProduction, HapticSamplesKeepHandLeaseAndClampAmplitude) { + EXPECT_EQ(simple_haptic_bindings, 2); + actions.RequestHaptics(1, 1.5f, 50000000, 0); + ASSERT_TRUE(Sync()); + EXPECT_EQ(vibrations, 1); + EXPECT_FLOAT_EQ(last_amplitude, 1); + EXPECT_EQ(last_duration, 50000000); + EXPECT_EQ(last_haptic_hand, paths["/user/hand/right"]); + actions.StopHaptics(1); + ASSERT_TRUE(Sync(2)); + EXPECT_EQ(stops, 1); +} +TEST_F(XrActionsProduction, + CapacitiveContactDoesNotPressTriggerAndResetsOnFocusLoss) { + touching = true; + ASSERT_TRUE(Sync()); + const auto index = static_cast(ControllerTouch::kTrigger); + EXPECT_TRUE(actions.snapshot().hands[0].touches[index]); + EXPECT_TRUE(actions.snapshot().hands[0].touch_active[index]); + EXPECT_FALSE(actions.snapshot().hands[0].buttons[static_cast( + ControllerButton::kTrigger)]); + focused = false; + EXPECT_FALSE(Sync(2)); + EXPECT_FALSE(actions.snapshot().hands[0].touches[index]); + EXPECT_FALSE(actions.snapshot().hands[0].touch_active[index]); +} +TEST_F(XrActionsProduction, ActivatesSetAndDeliversGripTriggerAndPose) { + trigger = .9f; + squeeze = .9f; + stick_y = 1.f; + ASSERT_TRUE(Sync()); + auto d = Pop(); + EXPECT_EQ(syncs, 1); + EXPECT_TRUE( + d.button_levels[0][static_cast(ControllerButton::kSqueeze)]); + EXPECT_TRUE( + d.button_levels[1][static_cast(ControllerButton::kSqueeze)]); + EXPECT_FLOAT_EQ(d.axes[4], .9f); + EXPECT_FLOAT_EQ(d.axes[1], -1.f); + EXPECT_TRUE(actions.hand(0).grip_valid); + EXPECT_TRUE(actions.hand(1).aim_valid); + EXPECT_EQ(located_time, 123456); + squeeze = .65f; + ASSERT_TRUE(Sync(2)); + EXPECT_TRUE(Pop().button_levels[0][1]); + squeeze = .5f; + ASSERT_TRUE(Sync(3)); + EXPECT_FALSE(Pop().button_levels[0][1]); +} +TEST_F(XrActionsProduction, + InactiveActionsReleaseEvenWithProfileStillAssigned) { + trigger = squeeze = 1; + ASSERT_TRUE(Sync()); + Pop(); + input_active = false; + ASSERT_TRUE(Sync(2)); + auto d = Pop(); + EXPECT_EQ(d.axes[4], 0); + EXPECT_FALSE(d.button_levels[0][1]); + EXPECT_FALSE(actions.hand(0).grip_valid); +} +TEST_F(XrActionsProduction, FocusLossDisconnectsAndStopsHaptics) { + squeeze = 1; + ASSERT_TRUE(Sync()); + Pop(); + actions.RequestHaptics(0, 1, 100, 0); + focused = false; + EXPECT_FALSE(Sync(2)); + auto d = Pop(); + EXPECT_FALSE(d.connected[0]); + EXPECT_TRUE(d.resync); + EXPECT_GT(stops, 0); + EXPECT_FALSE(actions.hand(0).grip_valid); + focused = true; + ASSERT_TRUE(Sync(3)); + EXPECT_EQ(vibrations, 0); +} +TEST_F(XrActionsProduction, ResetSurvivesDetachAndNewSessionHasFreshState) { + squeeze = 1; + ASSERT_TRUE(Sync()); + auto old = Pop(); + actions.RequestHaptics(0, 1, 100, 0); + actions.Detach(); + auto reset = Pop(); + EXPECT_FALSE(reset.connected[0]); + ASSERT_TRUE( + actions.Attach(Handle(1), Handle(4), Handle(5)) + .ok()); + squeeze = .65f; + ASSERT_TRUE(Sync(2)); + auto fresh = Pop(); + EXPECT_GT(fresh.session_generation, old.session_generation); + EXPECT_FALSE(fresh.button_levels[0][1]); + EXPECT_EQ(vibrations, 0); +} +TEST_F(XrActionsProduction, SimpleSelectBecomesAnalogTrigger) { + simple = click = true; + ASSERT_TRUE(Sync()); + EXPECT_FLOAT_EQ(Pop().axes[4], 1.f); + click = false; + ASSERT_TRUE(Sync(2)); + EXPECT_FLOAT_EQ(Pop().axes[4], 0.f); +} +TEST_F(XrActionsProduction, QueuedLevelsBelongToTheirOwnFrames) { + squeeze = 1; + ASSERT_TRUE(Sync(1)); + squeeze = 0; + ASSERT_TRUE(Sync(2)); + auto first = Pop(), second = Pop(); + EXPECT_EQ(first.frame, 1u); + EXPECT_TRUE(first.button_levels[0][1]); + EXPECT_EQ(second.frame, 2u); + EXPECT_FALSE(second.button_levels[0][1]); +} +XrActions *delivery_source; +int32_t DrainActions(MocktailVrControllerDelivery *out) { + ControllerDelivery d; + if (!delivery_source->TakeDelivery(&d)) + return 0; + EncodeControllerDelivery(d, out); + return 1; +} +TEST_F(XrActionsProduction, RoutesActiveInputsThroughAbiAndNativeRouter) { + using namespace mocktail; + using namespace mocktail::runtime; + struct SinkState { + std::map buttons; + std::map axes; + int connects = 0, disconnects = 0; + } state; + RobloxInputSink sink; + sink.gamepad = { + &state, + [](void *, int32_t, int32_t, bool, int32_t) { return Status::Ok(); }, + [](void *, int32_t, int32_t, int32_t, bool, int32_t) { + return Status::Ok(); + }, + [](void *p, int32_t, int32_t) { + ++static_cast(p)->connects; + return Status::Ok(); + }, + [](void *p, int32_t) { + ++static_cast(p)->disconnects; + return Status::Ok(); + }, + [](void *p, int32_t, int32_t key, bool pressed) { + static_cast(p)->buttons[key] = pressed; + return Status::Ok(); + }, + [](void *p, int32_t, int32_t axis, float x, float y, float z) { + static_cast(p)->axes[axis] = + axis == 0 ? y : (axis >= 17 ? z : x); + return Status::Ok(); + }}; + RobloxInputRouter router(sink); + ASSERT_TRUE(router.Activate({800, 600, 800, 600, 1}, true).ok()); + RobloxXrControllerInput input; + delivery_source = &actions; + ASSERT_TRUE(input.InitializeWithAbi( + [](void *p, const platform::PlatformEvent &e) { + EXPECT_TRUE( + static_cast(p)->HandleEvent(e).status.ok()); + }, + &router, DrainActions, nullptr, nullptr)); + squeeze = trigger = stick_y = 1; + ASSERT_TRUE(Sync()); + input.Drain(); + EXPECT_EQ(state.connects, 1); + EXPECT_TRUE(state.buttons[102]); + EXPECT_TRUE(state.buttons[103]); + EXPECT_GT(state.axes[17], .99f); + EXPECT_GT(state.axes[0], .99f); + focused = false; + EXPECT_FALSE(Sync(2)); + input.Drain(); + EXPECT_EQ(state.disconnects, 1); + EXPECT_FALSE(state.buttons[102]); + EXPECT_FALSE(state.buttons[103]); + input.Shutdown(); +} + +} // namespace diff --git a/tests/vr/xr_controller_test.cc b/tests/vr/xr_controller_test.cc new file mode 100644 index 00000000..09728af8 --- /dev/null +++ b/tests/vr/xr_controller_test.cc @@ -0,0 +1,298 @@ +#include + +#include + +#include "mocktail/vr/xr_controller.h" + +namespace mocktail::vr { +namespace { + +using Button = ControllerButton; +using Axis = ControllerAxis; + +TEST(XrControllerState, + PublishEmitsOneEdgePerTransitionDespiteRepeatedGetters) { + ControllerStateBuilder builder; + std::vector edges; + + builder.SetConnected(0, true); + builder.SetButton(0, Button::kFaceSouth, true, /*active=*/true); + const auto first = builder.Publish(1, 0, &edges); + ASSERT_EQ(edges.size(), 1u); + EXPECT_EQ(edges[0].hand, 0); + EXPECT_EQ(edges[0].button, Button::kFaceSouth); + EXPECT_TRUE(edges[0].pressed); + + // Repeated publishes at the same level emit no further edges: native getters + // reading the snapshot must never see duplicate transitions. + edges.clear(); + builder.SetButton(0, Button::kFaceSouth, true, true); + builder.Publish(2, 0, &edges); + builder.Publish(3, 0, &edges); + EXPECT_TRUE(edges.empty()); + + edges.clear(); + builder.SetButton(0, Button::kFaceSouth, false, true); + builder.Publish(4, 0, &edges); + ASSERT_EQ(edges.size(), 1u); + EXPECT_FALSE(edges[0].pressed); + EXPECT_EQ(first.hands[0].connected, true); +} + +TEST(XrControllerState, InactiveActionsNeverPressAndReleaseHeldControls) { + ControllerStateBuilder builder; + std::vector edges; + builder.SetButton(0, Button::kMenu, false, true); + builder.Publish(1, 0, &edges); + + edges.clear(); + // An inactive action reporting "true" must not fabricate a press. + builder.SetButton(0, Button::kMenu, true, /*active=*/false); + builder.Publish(2, 0, &edges); + EXPECT_TRUE(edges.empty()); + EXPECT_FALSE(builder.snapshot() + .hands[0] + .buttons[static_cast(Button::kMenu)]); +} + +TEST(XrControllerState, InactiveActionReleasesPreviouslyHeldControl) { + ControllerStateBuilder b; + b.SetButton(0, Button::kMenu, true, true); + b.Publish(1, 0, nullptr); + b.SetButton(0, Button::kMenu, true, false); + std::vector edges; + b.Publish(2, 0, &edges); + ASSERT_EQ(edges.size(), 1u); + EXPECT_FALSE(edges[0].pressed); +} + +TEST(XrControllerState, ReleaseAllEmitsReleaseForHeldButtonsOnlyAndZeroesAxes) { + ControllerStateBuilder builder; + std::vector edges; + builder.SetButton(0, Button::kFaceSouth, true, true); + builder.SetButton(1, Button::kSqueeze, true, true); + builder.SetAxis(0, Axis::kThumbstickX, 0.8f, true); + builder.SetAxis(0, Axis::kTrigger, 1.f, true); + builder.Publish(1, 0, &edges); + EXPECT_EQ(edges.size(), 2u); + + edges.clear(); + builder.ReleaseAll(2, 0, &edges); + ASSERT_EQ(edges.size(), 2u); + for (const auto &edge : edges) { + EXPECT_FALSE(edge.pressed); + } + const auto &snapshot = builder.snapshot(); + EXPECT_FLOAT_EQ( + snapshot.hands[0].axes[static_cast(Axis::kThumbstickX)], + 0.f); + EXPECT_FLOAT_EQ( + snapshot.hands[0].axes[static_cast(Axis::kTrigger)], 0.f); + // A second ReleaseAll must not re-emit anything (no stuck-down, no repeats). + edges.clear(); + builder.ReleaseAll(3, 0, &edges); + EXPECT_TRUE(edges.empty()); +} + +TEST(XrControllerState, AxesAreClampedAndNonFiniteRejected) { + ControllerStateBuilder builder; + builder.SetAxis(0, Axis::kThumbstickY, 4.5f, true); + builder.SetAxis(1, Axis::kTrigger, -2.f, true); + builder.SetAxis(0, Axis::kTrigger, 0.5f, true); + builder.SetAxis(0, Axis::kTrigger, std::nanf(""), true); + builder.Publish(1, 0, nullptr); + const auto &s = builder.snapshot(); + EXPECT_FLOAT_EQ(s.hands[0].axes[static_cast(Axis::kThumbstickY)], + 1.f); + EXPECT_FLOAT_EQ(s.hands[1].axes[static_cast(Axis::kTrigger)], + 0.f); + // Invalid input must neutralize any previously held trigger. + EXPECT_FLOAT_EQ(s.hands[0].axes[static_cast(Axis::kTrigger)], + 0.f); +} + +TEST(XrControllerState, AnalogClickHysteresis) { + ControllerStateBuilder builder; + builder.SetAxis(0, Axis::kTrigger, 0.7f, true); + EXPECT_FALSE(builder.AnalogClick(0, Axis::kTrigger)); // below press 0.75 + builder.SetAxis(0, Axis::kTrigger, 0.8f, true); + EXPECT_TRUE(builder.AnalogClick(0, Axis::kTrigger)); // latched on + builder.SetAxis(0, Axis::kTrigger, 0.65f, true); + EXPECT_TRUE(builder.AnalogClick(0, Axis::kTrigger)); // inside hysteresis band + builder.SetAxis(0, Axis::kTrigger, 0.55f, true); + EXPECT_FALSE(builder.AnalogClick(0, Axis::kTrigger)); // below release 0.6 + // Non-click axes and invalid hands are rejected. + EXPECT_FALSE(builder.AnalogClick(0, Axis::kThumbstickX)); + EXPECT_FALSE(builder.AnalogClick(5, Axis::kTrigger)); +} + +TEST(XrControllerState, SessionGenerationAndFrameTrackPublish) { + ControllerStateBuilder builder; + const auto snapshot = builder.Publish(42, 7, nullptr); + EXPECT_EQ(snapshot.frame, 42u); + EXPECT_EQ(snapshot.session_generation, 7u); +} + +TEST(XrControllerAndroidMapping, PerHandKeysAndAxesMatchVerifiedGamepadRoute) { + // Values pinned against MapSdlGamepadButtonToAndroid/kAndroidAxes. + EXPECT_EQ(ControllerButtonToAndroidKey(0, Button::kFaceSouth), 96); + EXPECT_EQ(ControllerButtonToAndroidKey(1, Button::kFaceSouth), 96); + EXPECT_EQ(ControllerButtonToAndroidKey(0, Button::kFaceEast), 97); + EXPECT_EQ(ControllerButtonToAndroidKey(0, Button::kFaceWest), 99); + EXPECT_EQ(ControllerButtonToAndroidKey(0, Button::kFaceNorth), 100); + EXPECT_EQ(ControllerButtonToAndroidKey(0, Button::kSqueeze), 102); // L1 + EXPECT_EQ(ControllerButtonToAndroidKey(1, Button::kSqueeze), 103); // R1 + EXPECT_EQ(ControllerButtonToAndroidKey(0, Button::kThumbstick), 106); + EXPECT_EQ(ControllerButtonToAndroidKey(1, Button::kThumbstick), 107); + EXPECT_EQ(ControllerButtonToAndroidKey(0, Button::kMenu), 108); + EXPECT_EQ(ControllerButtonToAndroidKey(0, Button::kTrigger), -1); // analog + EXPECT_EQ(ControllerAxisToAndroidAxis(0, Axis::kThumbstickX), 0); + EXPECT_EQ(ControllerAxisToAndroidAxis(0, Axis::kThumbstickY), 1); + EXPECT_EQ(ControllerAxisToAndroidAxis(1, Axis::kThumbstickX), 11); + EXPECT_EQ(ControllerAxisToAndroidAxis(1, Axis::kThumbstickY), 14); + EXPECT_EQ(ControllerAxisToAndroidAxis(0, Axis::kTrigger), 17); + EXPECT_EQ(ControllerAxisToAndroidAxis(1, Axis::kTrigger), 18); + EXPECT_EQ(ControllerAxisToAndroidAxis(0, Axis::kSqueeze), -1); + EXPECT_EQ(ControllerAxisComponentCount(Axis::kThumbstickX), 2); + EXPECT_EQ(ControllerAxisComponentCount(Axis::kTrigger), 1); + EXPECT_EQ(ControllerAxisComponentCount(Axis::kSqueeze), 0); +} + +TEST(XrControllerState, InvalidHandAndButtonIndicesAreIgnored) { + ControllerStateBuilder builder; + builder.SetButton(-1, Button::kFaceSouth, true, true); + builder.SetButton(9, Button::kFaceSouth, true, true); + builder.SetAxis(7, Axis::kTrigger, 1.f, true); + builder.SetConnected(3, true); + const auto snapshot = builder.Publish(1, 0, nullptr); + EXPECT_FALSE(snapshot.hands[0].connected); + EXPECT_FALSE(snapshot.hands[1].connected); + EXPECT_FALSE( + snapshot.hands[0].buttons[static_cast(Button::kFaceSouth)]); +} + +TEST(XrControllerQueue, BoundedCapacityDropsOldestAndArmsResync) { + ControllerDeliveryQueue queue; + for (std::uint64_t frame = 1; frame <= ControllerDeliveryQueue::kCapacity; + ++frame) { + ControllerDelivery delivery; + delivery.frame = frame; + queue.Push(std::move(delivery)); + } + EXPECT_EQ(queue.size(), ControllerDeliveryQueue::kCapacity); + EXPECT_FALSE(queue.overflowed()); + + // One past capacity: the oldest frame is dropped and the newest arrival + // carries the resync request. + ControllerDelivery extra; + extra.frame = 99; + queue.Push(std::move(extra)); + EXPECT_EQ(queue.size(), ControllerDeliveryQueue::kCapacity); + EXPECT_EQ(queue.dropped_frames(), 1u); + + ControllerDelivery out; + ASSERT_TRUE(queue.Pop(&out)); + EXPECT_EQ(out.frame, 2u); // frame 1 was dropped, not frame 2 + // Drain to the newest: it must demand a resync because edges were lost. + while (queue.Pop(&out)) { + } + EXPECT_TRUE(out.resync); + EXPECT_EQ(out.frame, 99u); +} + +TEST(XrControllerQueue, DeferredOverflowResyncSurvivesDroppedCarrier) { + ControllerDeliveryQueue queue; + for (std::uint64_t frame = 0; frame < ControllerDeliveryQueue::kCapacity; + ++frame) { + queue.Push(ControllerDelivery{}); + } + // First overflow: this frame carries resync but is itself dropped next. + ControllerDelivery carrier; + carrier.frame = 100; + queue.Push(std::move(carrier)); + ControllerDelivery newest; + newest.frame = 101; + queue.Push(std::move(newest)); + + ControllerDelivery out; + while (queue.Pop(&out)) { + } + // The resync must survive on the frame the consumer actually receives. + EXPECT_TRUE(out.resync); + EXPECT_EQ(out.frame, 101u); +} + +TEST(XrControllerQueue, PopOnEmptyIsFalseAndClearEmpties) { + ControllerDeliveryQueue queue; + ControllerDelivery out; + EXPECT_FALSE(queue.Pop(&out)); + EXPECT_FALSE(queue.Pop(nullptr)); + queue.Push(ControllerDelivery{}); + EXPECT_EQ(queue.size(), 1u); + queue.Clear(); + EXPECT_EQ(queue.size(), 0u); +} + +TEST(XrControllerSdlMapping, OrdinalsMatchRouterExpectations) { + // SDL3 ordinals the existing router converts through + // MapSdlGamepadButtonToAndroid/kAndroidAxes; verified pairs only. + EXPECT_EQ(ControllerButtonToSdlOrdinal(0, Button::kFaceSouth), 0); + EXPECT_EQ(ControllerButtonToSdlOrdinal(0, Button::kMenu), 6); + EXPECT_EQ(ControllerButtonToSdlOrdinal(0, Button::kSqueeze), 9); + EXPECT_EQ(ControllerButtonToSdlOrdinal(1, Button::kSqueeze), 10); + EXPECT_EQ(ControllerButtonToSdlOrdinal(0, Button::kThumbstick), 7); + EXPECT_EQ(ControllerButtonToSdlOrdinal(1, Button::kThumbstick), 8); + EXPECT_EQ(ControllerButtonToSdlOrdinal(0, Button::kTrigger), -1); + EXPECT_EQ(ControllerAxisToSdlOrdinal(0, Axis::kThumbstickX), 0); + EXPECT_EQ(ControllerAxisToSdlOrdinal(0, Axis::kThumbstickY), 1); + EXPECT_EQ(ControllerAxisToSdlOrdinal(1, Axis::kThumbstickX), 2); + EXPECT_EQ(ControllerAxisToSdlOrdinal(1, Axis::kThumbstickY), 3); + EXPECT_EQ(ControllerAxisToSdlOrdinal(0, Axis::kTrigger), 4); + EXPECT_EQ(ControllerAxisToSdlOrdinal(1, Axis::kTrigger), 5); + EXPECT_EQ(ControllerAxisToSdlOrdinal(0, Axis::kSqueeze), -1); +} + +} // namespace +} // namespace mocktail::vr + +TEST(XrControllerNative, + ExactChannelsMergeBothHandsAndNeverTurnTouchIntoClick) { + mocktail::vr::ControllerSnapshot snapshot; + snapshot.hands[0].connected = snapshot.hands[1].connected = true; + snapshot.hands[0] + .buttons[static_cast(mocktail::vr::ControllerButton::kFaceWest)] = + true; + snapshot.hands[1] + .buttons[static_cast(mocktail::vr::ControllerButton::kFaceEast)] = + true; + snapshot.hands[0] + .buttons[static_cast(mocktail::vr::ControllerButton::kMenu)] = + true; + snapshot.hands[1] + .axes[static_cast(mocktail::vr::ControllerAxis::kTrigger)] = .8f; + snapshot.hands[0] + .axes[static_cast(mocktail::vr::ControllerAxis::kThumbstickY)] = + 1; + snapshot.hands[0] + .touches[static_cast(mocktail::vr::ControllerTouch::kA)] = true; + snapshot.hands[0] + .axes[static_cast(mocktail::vr::ControllerAxis::kSqueeze)] = .37f; + snapshot.hands[0] + .buttons[static_cast(mocktail::vr::ControllerButton::kSqueeze)] = + true; + float channels[28]{}; + EXPECT_TRUE(mocktail::vr::EncodeNativeControllerChannels(snapshot, channels)); + EXPECT_FLOAT_EQ(channels[4], + .37f); // preserve analog grip, not its click latch + EXPECT_FLOAT_EQ(channels[24], 1); // X + EXPECT_FLOAT_EQ(channels[23], 1); // B + EXPECT_FLOAT_EQ(channels[27], 1); // Start/Menu + EXPECT_FLOAT_EQ(channels[22], 0); // capacitive A is not a press + EXPECT_FLOAT_EQ(channels[12], .8f); // right trigger + EXPECT_FLOAT_EQ(channels[2], 1); // OpenXR up remains positive in native route + snapshot.hands[0].connected = false; + EXPECT_TRUE(mocktail::vr::EncodeNativeControllerChannels(snapshot, channels)); + EXPECT_FLOAT_EQ(channels[24], 0); + EXPECT_FLOAT_EQ(channels[27], 0); + EXPECT_FLOAT_EQ(channels[23], 1); +} From 5e24bee4ab5ee4f8839998f6cb5afdfecac0ddfd Mon Sep 17 00:00:00 2001 From: Uwie <83516136+TheRealUwie@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:14:42 +0200 Subject: [PATCH 07/58] resolution thing or smth idk (#141) Co-authored-by: Uwie --- include/mocktail/vr/openxr_backend.h | 3 ++ src/vr/gles_transport.cc | 3 +- src/vr/openxr_backend.cc | 61 +++++++++++++++++++++++++--- src/vr/roblox_vr_device_bridge.cc | 12 ++++++ 4 files changed, 72 insertions(+), 7 deletions(-) diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index ba4302f9..88e6cd7f 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -161,6 +161,7 @@ class OpenXrBackend final { // hands[0]=left, hands[1]=right carry this frame's tracked controller grip // poses (metres, LOCAL space) when a controller layer is active. ScriptedPoseSample PublishedHeadPose() const; + bool RecommendedEyeExtent(std::uint32_t* width, std::uint32_t* height) const; void NotePoseApplied(void* owner, std::uint64_t frame); // Controller input delivery, consumed by the window-thread input runtime @@ -289,6 +290,8 @@ class OpenXrBackend final { void* xr_session_ = nullptr; void* xr_local_space_ = nullptr; void* xr_view_space_ = nullptr; + std::uint32_t xr_eye_width_[2] = {}; + std::uint32_t xr_eye_height_[2] = {}; EyeSwapchain eye_swapchains_[2]; int xr_blend_mode_ = 0; // Runtime name captured at xrCreateInstance so the bind-time diagnostics line diff --git a/src/vr/gles_transport.cc b/src/vr/gles_transport.cc index f083c51c..4ed92e47 100644 --- a/src/vr/gles_transport.cc +++ b/src/vr/gles_transport.cc @@ -309,9 +309,8 @@ void GlesTransport::Bind(unsigned target, unsigned framebuffer, if (it == storage.end()) return; const auto &size = it->second; - // DebugDeviceVR currently creates single-sample 500x500 RGBA eyes. // Reject unsupported formats/extents instead of copying unrelated pixels. - if (size.width != 500 || size.height != 500 || size.samples != 0 || + if (size.samples != 0 || (size.format != GL_RGBA8 && size.format != GL_RGBA && size.format != GL_SRGB8_ALPHA8)) return; diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 0b21d725..66a67b5d 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -878,6 +878,42 @@ bool OpenXrBackend::InitializeSessionResourcesLocked(std::string *error) { xr_blend_mode_ = static_cast(blends[index]); } + std::uint32_t view_count = 0; + const XrInstance instance = static_cast(xr_instance_); + CheckXr(instance, + xrEnumerateViewConfigurationViews( + instance, static_cast(xr_system_), + XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, 0, &view_count, + nullptr), + "xrEnumerateViewConfigurationViews count"); + Require(view_count >= 2, + "OpenXR primary stereo configuration exposes fewer than two views"); + + std::vector config_views(view_count); + for (auto& view : config_views) { + view = XrInfo(XR_TYPE_VIEW_CONFIGURATION_VIEW); + } + + CheckXr(instance, + xrEnumerateViewConfigurationViews( + instance, static_cast(xr_system_), + XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO, view_count, + &view_count, config_views.data()), + "xrEnumerateViewConfigurationViews"); + + xr_eye_width_[0] = config_views[0].recommendedImageRectWidth; + xr_eye_height_[0] = config_views[0].recommendedImageRectHeight; + xr_eye_width_[1] = config_views[1].recommendedImageRectWidth; + xr_eye_height_[1] = config_views[1].recommendedImageRectHeight; + + Require(xr_eye_width_[0] != 0 && xr_eye_height_[0] != 0 && + xr_eye_width_[1] != 0 && xr_eye_height_[1] != 0, + "OpenXR returned zero recommended stereo eye extent"); + + Log(" [vr-backend] OpenXR recommended eye extents L=%ux%u R=%ux%u\n", + xr_eye_width_[0], xr_eye_height_[0], xr_eye_width_[1], + xr_eye_height_[1]); + if (!CreateSwapchains(error)) { TeardownSessionLocked("swapchain failure", session_recovery_pending_); return false; @@ -936,11 +972,10 @@ bool OpenXrBackend::CreateSwapchains(std::string* error) { Require(chosen != VK_FORMAT_UNDEFINED, "the runtime exposes no 8-bit RGBA swapchain format"); - // Default eye extent until the guest eye images are bound; the simulator - // runtime advertises 500x500-capable limits, and DebugDeviceVR renders - // 500x500. Re-created lazily if the recorded eye extent differs. - const std::uint32_t width = 500; - const std::uint32_t height = 500; + const std::uint32_t width = + std::max(xr_eye_width_[0], xr_eye_width_[1]); + const std::uint32_t height = + std::max(xr_eye_height_[0], xr_eye_height_[1]); for (int eye = 0; eye < 2; ++eye) { auto create_info = XrInfo(XR_TYPE_SWAPCHAIN_CREATE_INFO); @@ -1354,6 +1389,22 @@ ScriptedPoseSample OpenXrBackend::PublishedHeadPose() const { return published_pose_; } +bool OpenXrBackend::RecommendedEyeExtent(std::uint32_t* width, + std::uint32_t* height) const { + if (width == nullptr || height == nullptr) { + return false; + } + std::lock_guard lock(mutex_); + if (xr_instance_ == nullptr || xr_system_ == 0 || + xr_eye_width_[0] == 0 || xr_eye_height_[0] == 0 || + xr_eye_width_[1] == 0 || xr_eye_height_[1] == 0) { + return false; + } + *width = std::max(xr_eye_width_[0], xr_eye_width_[1]); + *height = std::max(xr_eye_height_[0], xr_eye_height_[1]); + return true; +} + bool OpenXrBackend::TakeControllerDelivery(ControllerDelivery* out) { if (out == nullptr) { return false; diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc index 5a58cde6..06e4471c 100644 --- a/src/vr/roblox_vr_device_bridge.cc +++ b/src/vr/roblox_vr_device_bridge.cc @@ -1067,6 +1067,18 @@ void RobloxVrDeviceBridge::TryInitializeEyes(void *self) { const auto initializer = reinterpret_cast( image_base_.load(std::memory_order_acquire) + profile_.eye_initializer_rva); + + if (auto* backend = ActiveVrBackend()) { + std::uint32_t width = 0; + std::uint32_t height = 0; + if (backend->RecommendedEyeExtent(&width, &height)) { + *reinterpret_cast(bytes + kWidthOffset) = width; + *reinterpret_cast(bytes + kHeightOffset) = height; + Log(" [vr-device][evidence] using OpenXR recommended eye extent %ux%u\n", + width, height); + } + } + Log(" [vr-device][evidence] eye-init-begin self=%p device=%p tid=%" PRIu64 " presents=%" PRIu64 "\n", self, device, CurrentThreadId(), From 3c20ee3c5548c81c28cf1a95c2a19f043b0c04e1 Mon Sep 17 00:00:00 2001 From: komaruworld Date: Mon, 14 Sep 2026 18:25:33 +0300 Subject: [PATCH 08/58] Bla fix main menu bla --- src/runtime/performance_policy.cc | 7 +++++++ tests/performance_policy_test.cc | 12 +++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/runtime/performance_policy.cc b/src/runtime/performance_policy.cc index e14c98a8..fd72acb2 100644 --- a/src/runtime/performance_policy.cc +++ b/src/runtime/performance_policy.cc @@ -385,6 +385,13 @@ bool MergeVrClientSettingsOverrides(bool enabled, std::string_view base_json, return false; } overrides["FFlagDebugEnableVREmulator"] = enabled ? "True" : "False"; + if (enabled) { + // Roblox 2998's SurfaceController startup throttle renders at 1 Hz while + // waiting for AppStartRendering. Our XR frame loop is driven by guest + // presents, so that also stalls head/controller updates in the VR menu. + // Keep VR rendering continuous; leave the non-VR startup policy alone. + overrides["FFlagSlowDownRendering"] = "False"; + } *merged_json = overrides.dump(); return true; } diff --git a/tests/performance_policy_test.cc b/tests/performance_policy_test.cc index 16bef548..8adff84e 100644 --- a/tests/performance_policy_test.cc +++ b/tests/performance_policy_test.cc @@ -287,10 +287,11 @@ TEST(PerformancePolicyTest, VrModeOverridesConflictingFlagAndPreservesSettings) { std::string merged, error; ASSERT_TRUE(MergeVrClientSettingsOverrides( - true, R"({"FFlagDebugEnableVREmulator":"False","FIntUnrelated":"42"})", + true, R"({"FFlagDebugEnableVREmulator":"False","FFlagSlowDownRendering":"True","FIntUnrelated":"42"})", &merged, &error)); auto json = nlohmann::json::parse(merged); EXPECT_EQ(json["FFlagDebugEnableVREmulator"], "True"); + EXPECT_EQ(json["FFlagSlowDownRendering"], "False"); EXPECT_EQ(json["FIntUnrelated"], "42"); ASSERT_TRUE(MergeVrClientSettingsOverrides(false, merged, &merged, &error)); EXPECT_EQ(nlohmann::json::parse(merged)["FFlagDebugEnableVREmulator"], @@ -299,6 +300,15 @@ TEST(PerformancePolicyTest, EXPECT_FALSE(MergeVrClientSettingsOverrides(true, "bad", &merged, &error)); } +TEST(PerformancePolicyTest, NonVrPreservesStartupRenderingPolicy) { + std::string merged, error; + ASSERT_TRUE(MergeVrClientSettingsOverrides( + false, R"({"FFlagSlowDownRendering":"True"})", &merged, &error)); + EXPECT_EQ(nlohmann::json::parse(merged)["FFlagSlowDownRendering"], "True"); + ASSERT_TRUE(MergeVrClientSettingsOverrides(false, "{}", &merged, &error)); + EXPECT_FALSE(nlohmann::json::parse(merged).contains("FFlagSlowDownRendering")); +} + } // namespace } // namespace runtime } // namespace mocktail From 25cfd94c562a487985e2b49ba4cb408dbcf58d72 Mon Sep 17 00:00:00 2001 From: komaruworld Date: Thu, 17 Sep 2026 00:07:27 +0300 Subject: [PATCH 09/58] BLA BLA BLA 3 --- src/runtime/performance_policy.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/runtime/performance_policy.cc b/src/runtime/performance_policy.cc index fd72acb2..568e325d 100644 --- a/src/runtime/performance_policy.cc +++ b/src/runtime/performance_policy.cc @@ -391,6 +391,9 @@ bool MergeVrClientSettingsOverrides(bool enabled, std::string_view base_json, // presents, so that also stalls head/controller updates in the VR menu. // Keep VR rendering continuous; leave the non-VR startup policy alone. overrides["FFlagSlowDownRendering"] = "False"; + // The eye textures already use the OpenXR extent; render their contents + // at full resolution as well, without Roblox's dynamic downscaling. + overrides["FFlagDebugDisableDRS"] = "True"; } *merged_json = overrides.dump(); return true; From fbf97df7082bd8dc50223a221fc14a02c989929f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8F=8D=E6=83=9C?= Date: Tue, 22 Sep 2026 13:55:55 -0700 Subject: [PATCH 10/58] Fix VR crashing is the version is not supported (#158) Co-authored-by: HttpAnimations --- src/main.cc | 104 +++++++++++++---------- tests/vr/roblox_vr_device_bridge_test.cc | 69 +++++++++++++++ 2 files changed, 128 insertions(+), 45 deletions(-) diff --git a/src/main.cc b/src/main.cc index ea2a2a3f..d38422a1 100644 --- a/src/main.cc +++ b/src/main.cc @@ -268,6 +268,7 @@ int main(int argc, char* argv[]) { } const mocktail::runtime::ProcessEnvironment environment; + const bool vr_requested = environment.HasNonEmpty("MOCKTAIL_VR_ENABLED"); const std::string built_in_compatibility_manifest = environment.GetOr( "MOCKTAIL_UPDATE_COMPATIBILITY_PATH", environment.GetOr("MOCKTAIL_COMPATIBILITY_MANIFEST", @@ -336,28 +337,12 @@ int main(int argc, char* argv[]) { } // The standalone OpenXR scene belongs to mocktail-vr-probe --scene. // --vr continues through Roblox startup as a native stereo prototype. - if (runtime_config.config.vr_enabled()) { - if (!mocktail::vr::IsOpenXrSupportCompiled()) { - std::cerr << " [vr] This build does not include OpenXR support. " - "Use a VR-enabled build of Mocktail; refusing to start " - "Roblox with --vr in a non-VR build.\n"; - return EXIT_FAILURE; - } - const mocktail::vr::VrBackendMode vr_backend_mode = - mocktail::vr::ResolveVrBackendMode(true); - std::cerr << " [vr] experimental backend=" - << mocktail::vr::VrBackendModeName(vr_backend_mode) - << ": Roblox renders both eyes through its DebugDeviceVR " - "debug device (exact Build ID 2998 only)"; - if (vr_backend_mode == mocktail::vr::VrBackendMode::kXrOutput) { - std::cerr << "; eye images are submitted to the active OpenXR runtime " - "as a projection layer. Start WiVRn or SteamVR/ALVR and " - "connect the headset " - "before launching.\n"; - } else { - std::cerr << "; OpenXR output is disabled in this diagnostic mode " - "(MOCKTAIL_VR_BACKEND=native).\n"; - } + if (runtime_config.config.vr_enabled() && + !mocktail::vr::IsOpenXrSupportCompiled()) { + std::cerr << " [vr] This build does not include OpenXR support. " + "Use a VR-enabled build of Mocktail; refusing to start " + "Roblox with --vr in a non-VR build.\n"; + return EXIT_FAILURE; } mocktail::runtime::SessionLog session_log; mocktail::runtime::FailureSupportBundleGuard support_bundle_guard( @@ -743,7 +728,48 @@ int main(int argc, char* argv[]) { << " model=\"" << runtime_config.config.device_profile().display_name << "\"\n"; } + mocktail::compat::PayloadCompatibilityResult compatibility; + bool vr_active = false; if (command_line.options.mode == mocktail::runtime::CommandMode::kRun) { + const std::string compatibility_manifest = + environment.GetOr("MOCKTAIL_COMPATIBILITY_MANIFEST", + MOCKTAIL_DEFAULT_COMPATIBILITY_MANIFEST); + compatibility = mocktail::compat::CheckPayloadCompatibility( + runtime_config.config.roblox_library_path().string(), + compatibility_manifest, + command_line.options.allow_unverified_build); + if (!compatibility) { + std::cerr << "[FATAL] " << compatibility.error << '\n'; + return EXIT_FAILURE; + } + vr_active = runtime_config.config.vr_enabled() && + (vr_requested || + compatibility.profile.vr_debug_device_bridge.has_value()); + if (runtime_config.config.vr_enabled() && !vr_active) { + std::cerr << " [vr] DebugDeviceVR bridge is not profiled for Build ID " + << compatibility.build_id << "; continuing without VR\n"; + if (setenv("MOCKTAIL_VR_ENABLED", "0", 1) != 0) { + std::cerr << "[FATAL] Cannot disable VR for this Roblox build\n"; + return EXIT_FAILURE; + } + } + if (vr_active) { + const mocktail::vr::VrBackendMode vr_backend_mode = + mocktail::vr::ResolveVrBackendMode(true); + std::cerr << " [vr] experimental backend=" + << mocktail::vr::VrBackendModeName(vr_backend_mode) + << ": Roblox renders both eyes through its DebugDeviceVR " + "debug device (exact Build ID 2998 only)"; + if (vr_backend_mode == mocktail::vr::VrBackendMode::kXrOutput) { + std::cerr << "; eye images are submitted to the active OpenXR runtime " + "as a projection layer. Start WiVRn or SteamVR/ALVR and " + "connect the headset " + "before launching.\n"; + } else { + std::cerr << "; OpenXR output is disabled in this diagnostic mode " + "(MOCKTAIL_VR_BACKEND=native).\n"; + } + } const std::filesystem::path fflags_path = paths.config_root() / "fflags.json"; const auto fflags = mocktail::services::LoadAndMergeFflagsFile( @@ -777,7 +803,7 @@ int main(int argc, char* argv[]) { return EXIT_FAILURE; } if (!mocktail::runtime::MergeVrClientSettingsOverrides( - runtime_config.config.vr_enabled(), audio_capture_overrides, + vr_active, audio_capture_overrides, &audio_capture_overrides, &command_line_error)) { std::cerr << "[FATAL] Cannot apply VR policy: " << command_line_error << '\n'; return EXIT_FAILURE; @@ -798,7 +824,7 @@ int main(int argc, char* argv[]) { return EXIT_FAILURE; } if (!mocktail::runtime::MergeVrClientSettingsOverrides( - runtime_config.config.vr_enabled(), fast_flags_overrides, + vr_active, fast_flags_overrides, &fast_flags_overrides, &command_line_error)) { std::cerr << "[FATAL] Cannot apply VR policy: " << command_line_error << '\n'; return EXIT_FAILURE; @@ -864,18 +890,6 @@ int main(int argc, char* argv[]) { mocktail::vr::RobloxVrDeviceBridge vr_device_bridge; mocktail::vr::OpenXrBackend xr_backend; if (command_line.options.mode == mocktail::runtime::CommandMode::kRun) { - const std::string compatibility_manifest = - environment.GetOr("MOCKTAIL_COMPATIBILITY_MANIFEST", - MOCKTAIL_DEFAULT_COMPATIBILITY_MANIFEST); - const mocktail::compat::PayloadCompatibilityResult compatibility = - mocktail::compat::CheckPayloadCompatibility( - runtime_config.config.roblox_library_path().string(), - compatibility_manifest, - command_line.options.allow_unverified_build); - if (!compatibility) { - std::cerr << "[FATAL] " << compatibility.error << '\n'; - return EXIT_FAILURE; - } const mocktail::Status fullscreen_status = fullscreen_bridge.Install(compatibility.profile); if (!fullscreen_status.ok()) { @@ -890,7 +904,15 @@ int main(int argc, char* argv[]) { << output_device_status.message() << '\n'; return EXIT_FAILURE; } - if (runtime_config.config.vr_enabled()) { + if (vr_active) { + const mocktail::Status vr_device_status = + vr_device_bridge.Install(compatibility.profile); + if (!vr_device_status.ok()) { + std::cerr << "[FATAL] Cannot install experimental Roblox VR device " + "bridge: " + << vr_device_status.message() << '\n'; + return EXIT_FAILURE; + } if (mocktail::vr::ResolveVrBackendMode(true) == mocktail::vr::VrBackendMode::kXrOutput) { // Fail closed and loudly when no OpenXR runtime is reachable; the @@ -912,14 +934,6 @@ int main(int argc, char* argv[]) { return EXIT_FAILURE; } } - const mocktail::Status vr_device_status = - vr_device_bridge.Install(compatibility.profile); - if (!vr_device_status.ok()) { - std::cerr << "[FATAL] Cannot install experimental Roblox VR device " - "bridge: " - << vr_device_status.message() << '\n'; - return EXIT_FAILURE; - } } } diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc index 0dc9d165..3e190d65 100644 --- a/tests/vr/roblox_vr_device_bridge_test.cc +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -447,6 +447,75 @@ TEST(VrProfileParsing, ParsesAndRejectsBridgeSection) { } } +// Reproduces issue #153: a Roblox build whose compatibility profile arms the +// fullscreen and output-device bridges but carries no vr_debug_device_bridge +// metadata must fail closed when --vr is requested, surfacing the exact +// FATAL message the reporter saw. +TEST(VrProfileParsing, Issue153VrBridgeFailsWithoutMetadataForArmedProfile) { + const std::string directory = std::string(MOCKTAIL_TEST_BINARY_DIRECTORY); + const auto write_manifest = [&directory](const char *name, + const std::string &contents) { + const std::string path = directory + "/" + name; + std::ofstream output(path); + output << contents; + output.close(); + return path; + }; + + // The reporter's payload Build ID from the issue log. The profile is + // fully armed for fullscreen + FMOD output-device interposition, so the + // runtime reaches the VR device bridge install step, but it omits + // vr_debug_device_bridge entirely. + const std::string manifest = R"({ + "schema_version": 1, + "profiles": [{ + "version_name": "2.738.266", + "version_code": 3092, + "elf_build_id": "5f0704edd9064f566ee3d6df2bd2fabbcc709f03", + "status": "supported", + "default_allowed": true, + "allow_legacy_binary_patches": false, + "allow_host_abi_bridges": true, + "allow_host_constructor_replay": true, + "user_game_settings_fullscreen_setter_rva": "0x4612345", + "fmod_output_device_bridge": { + "vtable_rva": "0x6d00040", + "string_constructor_rva": "0x1d40000", + "count_method_rva": "0x32d1000", + "info_method_rva": "0x32d10a0", + "current_method_rva": "0x32d1050", + "select_method_rva": "0x32d0d04" + }, + "reason": "issue 153 reproduction profile: armed without vr metadata" + }] + })"; + + const std::string path = + write_manifest("vr_profile_issue153.json", manifest); + const auto result = compat::FindBuildProfile( + path, "5f0704edd9064f566ee3d6df2bd2fabbcc709f03"); + std::remove(path.c_str()); + + // The profile must resolve: fullscreen and output-device bridges would arm. + ASSERT_TRUE(result); + ASSERT_TRUE(result.profile.has_value()); + ASSERT_TRUE(result.profile->fmod_output_device_bridge.has_value()); + ASSERT_TRUE(result.profile->user_game_settings_fullscreen_setter_rva + .has_value()); + // But it carries no VR debug-device bridge metadata. + ASSERT_FALSE(result.profile->vr_debug_device_bridge.has_value()); + + // Installing the VR device bridge against this profile must fail closed with + // the exact message the reporter hit in the FATAL log line. + RobloxVrDeviceBridge bridge; + const Status status = bridge.Install(*result.profile); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.code(), StatusCode::kFailedPrecondition); + EXPECT_EQ(status.message(), + "experimental Roblox VR requires vr_debug_device_bridge metadata in " + "the exact-build compatibility profile"); +} + } // namespace } // namespace mocktail::vr From 6362283adeaf6385ed91fd71b0158712e5215976 Mon Sep 17 00:00:00 2001 From: komaruworld Date: Tue, 29 Sep 2026 19:34:17 +0300 Subject: [PATCH 11/58] fix idle 1fps bob --- include/runtime/roblox_desktop_app_policy.h | 10 ++ src/main.cc | 20 +++ src/runtime/roblox_desktop_app_policy.cc | 56 ++++++++ src/vr/openxr_disabled.cc | 3 + tests/roblox_desktop_app_policy_test.cc | 134 ++++++++++++++++++++ 5 files changed, 223 insertions(+) diff --git a/include/runtime/roblox_desktop_app_policy.h b/include/runtime/roblox_desktop_app_policy.h index 1a89e308..80f6135d 100644 --- a/include/runtime/roblox_desktop_app_policy.h +++ b/include/runtime/roblox_desktop_app_policy.h @@ -38,6 +38,16 @@ bool MergeDesktopAppPolicyClientSettingsOverride(std::string_view policy_json, std::string* merged_json, std::string* error); +// Apply after desktop policy composition. Roblox 2998's Android payload is not +// a VRAppBuild, so its LuaApp enables RunService's idle render throttle +// even with DebugDeviceVR active. Preserve the effective app-policy (the later +// fast-flags channel takes precedence) in both channels, changing only +// ThrottleFramerate. This does not write the setting to appStorage. +bool MergeVrAppPolicyClientSettingsOverrides( + bool enabled, std::string_view client_settings_json, + std::string_view fast_flags_json, std::string* merged_client_settings, + std::string* merged_fast_flags, std::string* error); + } // namespace runtime } // namespace mocktail diff --git a/src/main.cc b/src/main.cc index d38422a1..439117f5 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1064,6 +1064,26 @@ int main(int argc, char* argv[]) { << " updated=" << (desktop_policy.updated ? 1 : 0) << " runtime_override=1\n"; } + if (vr_active) { + std::string vr_client_settings; + std::string vr_fast_flags; + if (!mocktail::runtime::MergeVrAppPolicyClientSettingsOverrides( + vr_active, + environment.GetOr("MOCKTAIL_CLIENT_SETTINGS_OVERRIDES_JSON", "{}"), + environment.GetOr("MOCKTAIL_FAST_FLAGS_JSON", "{}"), + &vr_client_settings, &vr_fast_flags, &command_line_error)) { + std::cerr << "[FATAL] Cannot apply VR app-policy: " + << command_line_error << '\n'; + return EXIT_FAILURE; + } + if (setenv("MOCKTAIL_CLIENT_SETTINGS_OVERRIDES_JSON", + vr_client_settings.c_str(), 1) != 0 || + setenv("MOCKTAIL_FAST_FLAGS_JSON", vr_fast_flags.c_str(), 1) != 0) { + std::cerr << "[FATAL] Cannot export VR app-policy\n"; + return EXIT_FAILURE; + } + std::cout << " [vr] app-policy idle render throttling disabled\n"; + } auto android_window_context = std::make_shared(); jnivm::AndroidWindowCallbacks android_window_callbacks; diff --git a/src/runtime/roblox_desktop_app_policy.cc b/src/runtime/roblox_desktop_app_policy.cc index fa717ce0..860cc966 100644 --- a/src/runtime/roblox_desktop_app_policy.cc +++ b/src/runtime/roblox_desktop_app_policy.cc @@ -361,5 +361,61 @@ bool MergeDesktopAppPolicyClientSettingsOverride(std::string_view policy_json, return true; } +bool MergeVrAppPolicyClientSettingsOverrides( + bool enabled, std::string_view client_settings_json, + std::string_view fast_flags_json, std::string* merged_client_settings, + std::string* merged_fast_flags, std::string* error) { + if (merged_client_settings == nullptr || merged_fast_flags == nullptr) { + if (error != nullptr) { + *error = "VR app-policy outputs are required"; + } + return false; + } + if (!enabled) { + *merged_client_settings = client_settings_json; + *merged_fast_flags = fast_flags_json; + return true; + } + + auto client_settings = nlohmann::json::parse( + client_settings_json.empty() ? "{}" : client_settings_json, nullptr, + false, true); + auto fast_flags = nlohmann::json::parse( + fast_flags_json.empty() ? "{}" : fast_flags_json, nullptr, false, true); + if (!client_settings.is_object() || !fast_flags.is_object()) { + if (error != nullptr) { + *error = "VR client-settings and fast-flags overrides must be JSON objects"; + } + return false; + } + + nlohmann::json policy = nlohmann::json::object(); + const nlohmann::json* channels[] = {&client_settings, &fast_flags}; + for (const auto* channel : channels) { + const auto existing = channel->find(kDesktopAppPolicyOverride); + if (existing == channel->end()) { + continue; + } + // An empty FString means no app-policy override. The later fast-flags + // channel can clear/replace an earlier client-settings override. + if (existing->is_string() && + existing->get_ref().empty()) { + policy = nlohmann::json::object(); + } else if (!DecodePolicy(*existing, &policy)) { + if (error != nullptr) { + *error = "VR app-policy override must be an encoded JSON object"; + } + return false; + } + } + policy["ThrottleFramerate"] = false; + const std::string encoded_policy = policy.dump(); + client_settings[std::string(kDesktopAppPolicyOverride)] = encoded_policy; + fast_flags[std::string(kDesktopAppPolicyOverride)] = encoded_policy; + *merged_client_settings = client_settings.dump(); + *merged_fast_flags = fast_flags.dump(); + return true; +} + } // namespace runtime } // namespace mocktail diff --git a/src/vr/openxr_disabled.cc b/src/vr/openxr_disabled.cc index 28470a00..b9f7446a 100644 --- a/src/vr/openxr_disabled.cc +++ b/src/vr/openxr_disabled.cc @@ -13,6 +13,9 @@ void OpenXrBackend::StopControllerHaptics(int) {} OpenXrBackend* ActiveVrBackend() { return nullptr; } void OpenXrBackend::NotePoseApplied(void*, std::uint64_t) {} ScriptedPoseSample OpenXrBackend::PublishedHeadPose() const { return {}; } +bool OpenXrBackend::RecommendedEyeExtent(std::uint32_t*, std::uint32_t*) const { + return false; +} PreviewResult RunOpenXrPreview(const PreviewOptions&) { diff --git a/tests/roblox_desktop_app_policy_test.cc b/tests/roblox_desktop_app_policy_test.cc index 233fb0d0..88c5c7cd 100644 --- a/tests/roblox_desktop_app_policy_test.cc +++ b/tests/roblox_desktop_app_policy_test.cc @@ -239,6 +239,140 @@ TEST(RobloxDesktopAppPolicyTest, RejectsMalformedOrSymlinkedStorage) { EXPECT_FALSE(ApplyDesktopAppPolicy(outside, default_path, 42, "invalid")); } +TEST(RobloxDesktopAppPolicyTest, + VrDisablesIdleThrottleAfterDesktopCompositionWithoutPersistingIt) { + TemporaryDirectory temporary; + const auto storage_path = temporary.path() / "storage.json"; + const auto default_path = temporary.path() / "default.json"; + const nlohmann::json cached_policy = { + {"ThrottleFramerate", true}, {"EligibleForVideoCapture", false}, + {"AccountOwnedMarker", "preserve"}, + }; + const nlohmann::json configurations = { + {"GUAC:42:app-policy", cached_policy.dump()}, + }; + ASSERT_TRUE(WriteJson(storage_path, + {{"AppConfiguration", configurations.dump()}})); + const auto desktop = + ApplyDesktopAppPolicy(storage_path, default_path, 42, "dark"); + ASSERT_TRUE(desktop) << desktop.error; + const auto stored_before_vr = ReadJson(storage_path); + + std::string client, fast, error; + ASSERT_TRUE(MergeDesktopAppPolicyClientSettingsOverride( + desktop.policy_json, R"({"DFIntTaskSchedulerTargetFps":"90"})", &client, + &error)) << error; + ASSERT_TRUE(MergeVrAppPolicyClientSettingsOverrides( + true, client, R"({"FFlagUnrelated":"True"})", &client, &fast, &error)) + << error; + const auto client_json = nlohmann::json::parse(client); + const auto fast_json = nlohmann::json::parse(fast); + auto expected_policy = nlohmann::json::parse(desktop.policy_json); + expected_policy["ThrottleFramerate"] = false; + EXPECT_EQ(client_json["FStringAppConfigurationOverrideAppPolicy"], + expected_policy.dump()); + EXPECT_EQ(fast_json["FStringAppConfigurationOverrideAppPolicy"], + expected_policy.dump()); + EXPECT_EQ(client_json["DFIntTaskSchedulerTargetFps"], "90"); + EXPECT_EQ(fast_json["FFlagUnrelated"], "True"); + EXPECT_EQ(ReadJson(storage_path), stored_before_vr); + + const auto next_launch = + ApplyDesktopAppPolicy(storage_path, default_path, 42, "dark"); + ASSERT_TRUE(next_launch) << next_launch.error; + EXPECT_TRUE(nlohmann::json::parse(next_launch.policy_json) + .at("ThrottleFramerate").get()); + EXPECT_FALSE(next_launch.updated); + + const std::string first_client = client; + const std::string first_fast = fast; + ASSERT_TRUE(MergeVrAppPolicyClientSettingsOverrides( + true, client, fast, &client, &fast, &error)) << error; + EXPECT_EQ(client, first_client); + EXPECT_EQ(fast, first_fast); +} + +TEST(RobloxDesktopAppPolicyTest, VrPolicyWorksWithoutDesktopProfile) { + std::string client, fast, error; + ASSERT_TRUE(MergeVrAppPolicyClientSettingsOverrides( + true, "{}", "{}", &client, &fast, &error)) << error; + const nlohmann::json policy = {{"ThrottleFramerate", false}}; + const nlohmann::json expected = { + {"FStringAppConfigurationOverrideAppPolicy", policy.dump()}, + }; + EXPECT_EQ(nlohmann::json::parse(client), expected); + EXPECT_EQ(nlohmann::json::parse(fast), expected); +} + +TEST(RobloxDesktopAppPolicyTest, VrKeepsLateFastFlagsPolicyPrecedence) { + const nlohmann::json earlier = {{"EarlierPolicy", 1}}; + nlohmann::json later = { + {"ThrottleFramerate", true}, {"LaterPolicy", 2}, {"ForceTheme", "light"}, + }; + const nlohmann::json client_input = { + {"FStringAppConfigurationOverrideAppPolicy", earlier.dump()}, + {"FIntClientMarker", "42"}, + }; + const nlohmann::json fast_input = { + {"FStringAppConfigurationOverrideAppPolicy", later.dump()}, + {"FIntFastMarker", "7"}, + }; + std::string client, fast, error; + ASSERT_TRUE(MergeVrAppPolicyClientSettingsOverrides( + true, client_input.dump(), fast_input.dump(), &client, &fast, &error)) + << error; + later["ThrottleFramerate"] = false; + EXPECT_EQ(nlohmann::json::parse(client) + .at("FStringAppConfigurationOverrideAppPolicy"), later.dump()); + EXPECT_EQ(nlohmann::json::parse(fast) + .at("FStringAppConfigurationOverrideAppPolicy"), later.dump()); + EXPECT_EQ(nlohmann::json::parse(client).at("FIntClientMarker"), "42"); + EXPECT_EQ(nlohmann::json::parse(fast).at("FIntFastMarker"), "7"); +} + +TEST(RobloxDesktopAppPolicyTest, NonVrPreservesBothPolicyChannels) { + const std::string client_input = + R"({ "FStringAppConfigurationOverrideAppPolicy": "{\"ThrottleFramerate\":true}" })"; + const std::string fast_input = R"({ "FIntUnrelated": "42" })"; + std::string client, fast, error; + ASSERT_TRUE(MergeVrAppPolicyClientSettingsOverrides( + false, client_input, fast_input, &client, &fast, &error)) << error; + EXPECT_EQ(client, client_input); + EXPECT_EQ(fast, fast_input); +} + +TEST(RobloxDesktopAppPolicyTest, VrAcceptsEmptyAppPolicyString) { + std::string client, fast, error; + ASSERT_TRUE(MergeVrAppPolicyClientSettingsOverrides( + true, "{}", R"({"FStringAppConfigurationOverrideAppPolicy":""})", + &client, &fast, &error)) << error; + EXPECT_EQ(nlohmann::json::parse(client) + .at("FStringAppConfigurationOverrideAppPolicy"), + R"({"ThrottleFramerate":false})"); + EXPECT_EQ(client, fast); +} + +TEST(RobloxDesktopAppPolicyTest, VrRejectsMalformedOverridesWithoutErasingThem) { + const std::string bad_inputs[] = { + "[]", "bad", "null", + R"({"FStringAppConfigurationOverrideAppPolicy":true})", + R"({"FStringAppConfigurationOverrideAppPolicy":"bad"})", + R"({"FStringAppConfigurationOverrideAppPolicy":"[]"})", + R"({"FStringAppConfigurationOverrideAppPolicy":"null"})", + }; + for (const auto& input : bad_inputs) { + SCOPED_TRACE(input); + std::string client = "unchanged", fast = "unchanged", error; + EXPECT_FALSE(MergeVrAppPolicyClientSettingsOverrides( + true, input, "{}", &client, &fast, &error)); + EXPECT_FALSE(error.empty()); + EXPECT_FALSE(MergeVrAppPolicyClientSettingsOverrides( + true, "{}", input, &client, &fast, &error)); + EXPECT_EQ(client, "unchanged"); + EXPECT_EQ(fast, "unchanged"); + } +} + } // namespace } // namespace runtime } // namespace mocktail From ecb80b3ee1bda8822ac741f2fe97e14e00f26171 Mon Sep 17 00:00:00 2001 From: daeeseD Date: Thu, 1 Oct 2026 13:25:48 -0300 Subject: [PATCH 12/58] vr: capture the microphone for voice chat on Build 2998 (#181) * vr: capture the microphone for voice chat on Build 2998 On the vr branch Roblox 2.736.1408 joins voice chat and plays other players, but never opens a microphone: its FmodAudioDevice capture methods have no working host path, and main's native capture (537828a "Fix vc") is pinned to 2.738 RVAs. Port main's NativeInputCapture and interpose the FmodAudioDevice capture slots for Build 2998. 2.738 inserted two device-list slots (7 and 11), so the capture methods sit at {4, 11..15} here instead of {4, 13..17}. The input-device slots {8, 9, 10, 16}, checked against main's input RVAs for this image, anchor that layout before anything is patched, and each capture slot must be a distinct method next to the input-device code. The voice sink is called through its own vtable after checking that the vtable and its PCM entry point lie in libroblox, and the retained shared_ptr is released through the libc++ control-block ABI, so no further Build 2998 RVAs are needed. If any check fails, capture stays off and the output routes still work. MOCKTAIL_NATIVE_INPUT_CAPTURE=0 turns the capture interposition off, and audio.input_device: disabled reports no microphone to Roblox. * Requested fix at PR 181 --- .../audio/roblox_output_device_bridge.h | 17 +- src/audio/native_input_capture.h | 282 ++++++++++++++++++ src/audio/roblox_output_device_bridge.cc | 123 +++++++- 3 files changed, 414 insertions(+), 8 deletions(-) create mode 100644 src/audio/native_input_capture.h diff --git a/include/mocktail/audio/roblox_output_device_bridge.h b/include/mocktail/audio/roblox_output_device_bridge.h index 6bbb5e3f..45c3c127 100644 --- a/include/mocktail/audio/roblox_output_device_bridge.h +++ b/include/mocktail/audio/roblox_output_device_bridge.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -13,13 +14,15 @@ namespace mocktail::audio { -// Replaces only the output-device query/select slots of the exact -// Build-ID-scoped FmodAudioDevice vtable. Roblox keeps owning its FMOD engine, -// while the existing settings UI sees and selects the SDL host routes that -// consume Android AudioTrack PCM. +class NativeInputCapture; + +// Replaces the output-device query/select slots of the exact Build-ID-scoped +// FmodAudioDevice vtable, and on Build 2998 its microphone capture slots. +// Roblox keeps owning its FMOD engine, while the existing settings UI sees +// and selects the SDL host routes that consume Android AudioTrack PCM. class RobloxOutputDeviceBridge final { public: - RobloxOutputDeviceBridge() = default; + RobloxOutputDeviceBridge(); ~RobloxOutputDeviceBridge(); RobloxOutputDeviceBridge(const RobloxOutputDeviceBridge&) = delete; @@ -47,6 +50,7 @@ class RobloxOutputDeviceBridge final { Status Activate(std::uintptr_t image_base); Status PatchVtableLocked(); + Status PrepareCaptureLocked(); bool RestoreVtableLocked(); int DeviceCount(); void* DeviceInfo(void* result, int index); @@ -54,6 +58,9 @@ class RobloxOutputDeviceBridge final { void SelectDevice(int index); void ConstructGuestString(void* destination, std::string_view value) const; + std::unique_ptr capture_; + bool capture_profile_supported_ = false; + std::array original_capture_methods_{}; mutable std::mutex mutex_; compat::FmodOutputDeviceBridgeProfile profile_; std::vector devices_; diff --git a/src/audio/native_input_capture.h b/src/audio/native_input_capture.h new file mode 100644 index 00000000..bfb04b52 --- /dev/null +++ b/src/audio/native_input_capture.h @@ -0,0 +1,282 @@ +#ifndef MOCKTAIL_AUDIO_NATIVE_INPUT_CAPTURE_H_ +#define MOCKTAIL_AUDIO_NATIVE_INPUT_CAPTURE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mocktail/audio/sdl_audio_capture.h" + +// Port of main's native input capture (537828a "Fix vc") to Build 2998. +// Main pins the voice sink vtable, its PCM entry point and the shared_ptr +// destructor to 2.738 RVAs. Here the sink is dispatched through its own +// vtable, which is validated against the guest image first, and the +// shared_ptr is released through the libc++ control-block ABI, so no +// further 2998 RVAs are needed. +namespace mocktail::audio { +class NativeInputCapture; + +struct NativePcmFormat { + std::uint32_t rate = 48000; + std::uint8_t channels = 1; + std::uint8_t bytes_per_sample = 2; + std::uint8_t flags = 0; + std::uint8_t padding = 0; +}; +struct NativeOptionalFormat { + NativePcmFormat format{}; + std::uint64_t present = 0; +}; +// std::__ndk1::shared_ptr passed by value, i.e. by invisible reference. +struct NativeSharedSink { void* object; void* owner; }; +static_assert(sizeof(NativeOptionalFormat) == 16); +static_assert(sizeof(NativePcmFormat) == 8); +inline std::atomic g_native_capture{nullptr}; + +class NativeInputCapture { + public: + using DeliverPcm = std::size_t (*)(void*, const void*, std::size_t, + std::uint64_t, const NativePcmFormat*); + using OriginalLatency = void (*)(void*, std::uint32_t*, std::uint32_t*); + using ImageRange = bool (*)(std::uintptr_t image_base, std::uintptr_t rva, + std::size_t size); + struct SinkAbi { + std::uintptr_t image_base; + ImageRange is_image_relro; + ImageRange is_image_code; + OriginalLatency latency; + }; + NativeInputCapture(SinkAbi abi, bool enabled) + : abi_(abi), enabled_(enabled) {} + ~NativeInputCapture() { Shutdown(); } + + static NativeOptionalFormat Format(void*) { + auto* capture = g_native_capture.load(std::memory_order_acquire); + return {{}, capture && capture->enabled_ && + !capture->closed_.load(std::memory_order_acquire) + ? 1ULL + : 0ULL}; + } + static void Latency(void* self, std::uint32_t* playback, + std::uint32_t* recording) { + // Preserve the native playback latency and report the host capture queue. + auto* capture = g_native_capture.load(std::memory_order_acquire); + if (!capture) return; + capture->abi_.latency(self, playback, recording); + if (recording) { + std::lock_guard lock(capture->mutex_); + *recording = 10; + auto it = capture->sessions_.find(self); + if (it != capture->sessions_.end()) { + std::lock_guard queue_lock(it->second->queue_mutex); + *recording += static_cast(it->second->count * 10); + } + } + } + static bool Start(void* self, NativeSharedSink* sink) { + auto* capture = g_native_capture.load(std::memory_order_acquire); + return capture && capture->StartSink(self, sink); + } + static void Stop(void* self, NativeSharedSink* sink) { + auto* capture = g_native_capture.load(std::memory_order_acquire); + if (capture) capture->StopSink(self, sink); + } + static bool Recording(void* self) { + auto* capture = g_native_capture.load(std::memory_order_acquire); + if (!capture) return false; + std::lock_guard lock(capture->mutex_); + return capture->sessions_.find(self) != capture->sessions_.end(); + } + static void Poll(void* self) { + auto* capture = g_native_capture.load(std::memory_order_acquire); + if (capture) capture->Deliver(self); + } + void Shutdown() { + std::lock_guard lock(mutex_); + closed_.store(true, std::memory_order_release); + for (auto& [self, session] : sessions_) Close(*session); + sessions_.clear(); + } + // Roblox threads may already be inside an entry point with the old pointer, + // and its workers are never joined. A published capture is therefore only + // closed, never freed, so late calls find an empty, closed object. + static void Retire(std::unique_ptr capture) { + g_native_capture.store(nullptr, std::memory_order_release); + if (capture) capture.release()->Shutdown(); + } + + private: + struct Sink { + NativeSharedSink shared; + DeliverPcm deliver; + }; + // libc++ __shared_weak_count: both counters are stored minus one. Virtual + // slots 0/1 are the destructors, 2 is __on_zero_shared, 3 __get_deleter + // and 4 __on_zero_shared_weak. + struct GuestSharedCount { + const std::uintptr_t* vtable; + long shared_owners; + long weak_owners; + }; + using ControlBlockHook = void (*)(GuestSharedCount*); + struct Session { + std::unique_ptr input; + std::vector sinks; + std::mutex queue_mutex; + // At most 200 ms; a stalled consumer must not accumulate stale speech. + std::array, 20> frames{}; + std::size_t head = 0, count = 0; + std::uint64_t delivered = 0, dropped = 0; + bool audible = false; + }; + static void OnData(void* context, std::size_t bytes) { + auto& session = *static_cast(context); + if (bytes != sizeof(session.frames[0])) return; + std::lock_guard lock(session.queue_mutex); + if (session.count == session.frames.size()) { + session.head = (session.head + 1) % session.frames.size(); + --session.count; + ++session.dropped; + } + std::memcpy(session.frames[(session.head + session.count) % + session.frames.size()].data(), + session.input->buffer_data(), bytes); + ++session.count; + } + bool IsGuestVtable(const void* object, std::size_t slot_count, + std::initializer_list code_slots) const { + const auto base = abi_.image_base; + const auto table = *static_cast(object); + const auto address = reinterpret_cast(table); + if (address < base || + !abi_.is_image_relro(base, address - base, + slot_count * sizeof(std::uintptr_t))) + return false; + for (const std::size_t slot : code_slots) + if (table[slot] < base || !abi_.is_image_code(base, table[slot] - base, 1)) + return false; + return true; + } + bool StartSink(void* self, NativeSharedSink* sink) { + std::lock_guard lock(mutex_); + if (closed_.load(std::memory_order_relaxed) || !enabled_ || !self || + !sink || !sink->object || !sink->owner) + return false; + if (!IsGuestVtable(sink->object, 3, {2}) || + !IsGuestVtable(sink->owner, 5, {2, 4})) { + std::fprintf(stderr, " [audio-input] unsupported native PCM sink ABI\n"); + return false; + } + auto it = sessions_.find(self); + if (it != sessions_.end()) { + for (const auto& existing : it->second->sinks) + if (existing.shared.object == sink->object) return true; + } else { + auto session = std::make_unique(); + SdlAudioCaptureOptions options; + options.output_spec.channels = 1; + options.data_callback = &OnData; + options.data_context = session.get(); + auto status = CreateSdlAudioCapture(options, &session->input); + if (status.ok()) status = session->input->Start(); + if (!status.ok()) { + std::fprintf(stderr, " [audio-input] capture start failed: %s\n", + status.message().c_str()); + return false; + } + it = sessions_.emplace(self, std::move(session)).first; + std::fprintf(stderr, + " [audio-input] native capture started: 48000 Hz mono signed16\n"); + } + const auto* table = *static_cast(sink->object); + // The native parameter is a by-value shared_ptr. Retain its existing + // ownership by moving it, leaving the caller's temporary empty. + it->second->sinks.push_back({*sink, reinterpret_cast(table[2])}); + *sink = {}; + return true; + } + static void Release(Sink& sink) { + // Mirrors __shared_weak_count::__release_shared and __release_weak. + auto* count = static_cast(sink.shared.owner); + sink = {}; + if (!count || + __atomic_fetch_sub(&count->shared_owners, 1, __ATOMIC_ACQ_REL) != 0) + return; + reinterpret_cast(count->vtable[2])(count); + if (__atomic_load_n(&count->weak_owners, __ATOMIC_ACQUIRE) == 0 || + __atomic_fetch_sub(&count->weak_owners, 1, __ATOMIC_ACQ_REL) == 0) + reinterpret_cast(count->vtable[4])(count); + } + void Close(Session& session) { + session.input->Shutdown(); + for (auto& sink : session.sinks) Release(sink); + session.sinks.clear(); + std::fprintf(stderr, + " [audio-input] native capture stopped: delivered=%llu dropped=%llu\n", + static_cast(session.delivered), + static_cast(session.dropped)); + } + void StopSink(void* self, NativeSharedSink* sink) { + std::lock_guard lock(mutex_); + auto it = sessions_.find(self); + if (it == sessions_.end() || !sink) return; + auto& session = *it->second; + auto existing = std::find_if(session.sinks.begin(), session.sinks.end(), + [&](const auto& entry) { return entry.shared.object == sink->object; }); + if (existing == session.sinks.end()) return; + Release(*existing); + session.sinks.erase(existing); + if (session.sinks.empty()) { + Close(session); + sessions_.erase(it); + } + } + void Deliver(void* self) { + // Invoked by Roblox's recording thread. Guest code and allocator TLS must + // never run on SDL's capture callback thread. + std::lock_guard lock(mutex_); + auto it = sessions_.find(self); + if (it == sessions_.end()) return; + auto& session = *it->second; + for (std::size_t n = 0; n < session.frames.size(); ++n) { + std::array pcm; + std::uint64_t delay; + { + std::lock_guard queue_lock(session.queue_mutex); + if (!session.count) break; + pcm = session.frames[session.head]; + delay = session.count * 10000000ULL; + session.head = (session.head + 1) % session.frames.size(); + --session.count; + } + const NativePcmFormat format; + for (const auto& sink : session.sinks) + sink.deliver(sink.shared.object, pcm.data(), pcm.size(), delay, + &format); + if (++session.delivered == 1) + std::fprintf(stderr, " [audio-input] PCM delivered to native voice sink\n"); + if (!session.audible && std::any_of(pcm.begin(), pcm.end(), + [](auto value) { return value != 0; })) { + session.audible = true; + std::fprintf(stderr, " [audio-input] nonzero microphone PCM delivered\n"); + } + } + } + SinkAbi abi_; + bool enabled_; + std::atomic closed_{false}; + std::mutex mutex_; + std::map> sessions_; +}; + + +} // namespace mocktail::audio +#endif diff --git a/src/audio/roblox_output_device_bridge.cc b/src/audio/roblox_output_device_bridge.cc index eb6c4c7c..8ae9e9e4 100644 --- a/src/audio/roblox_output_device_bridge.cc +++ b/src/audio/roblox_output_device_bridge.cc @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include +#include "audio/native_input_capture.h" #include "audio/roblox_output_device_bridge_internal.h" #include "linker/linker.h" #include "mocktail/audio/sdl_audio_sink.h" @@ -35,6 +37,24 @@ constexpr std::size_t kGuestDeviceInfoSize = kGuestStringSize * 2 + 1; constexpr std::size_t kStringConstructorContractSize = 24; constexpr std::size_t kMaximumMenuDevices = 128; constexpr std::size_t kMaximumDeviceNameBytes = 512; +constexpr std::size_t kFirstPatchedVtableIndex = 4; + +// Microphone capture on Build 2998 (Roblox 2.736.1408 x86_64). Roblox 2.738 +// inserted device-list methods at slots 7 and 11; main verified its capture +// methods at {4, 13..17}, between the input current (12) and select (18) +// slots, so 2998 keeps them at {4, 11..15}, between input current (10) and +// select (16). The input slots, with main's input RVAs for this image, anchor +// that layout before anything is patched. +constexpr std::string_view kCaptureBuildId = + "ade08266c67aee88ec9c1d00902150e1684dad3a"; +constexpr std::uintptr_t kCaptureVtableRva = 0x6c58040; +// Format, latency, start, stop, poll, recording. +constexpr std::array kCaptureSlots{4, 11, 12, 13, 14, 15}; +constexpr std::array kInputAnchorSlots{8, 9, 10, 16}; +constexpr std::array kInputAnchorRvas{ + 0x32d12dc, 0x32d12ea, 0x32d13a6, 0x32d0a56}; +// On 2.738 the capture methods lie within 0x1400 bytes of the input methods. +constexpr std::uintptr_t kCaptureCodeWindow = 0x4000; std::atomic g_active_bridge{nullptr}; @@ -104,7 +124,7 @@ bool SetVtableWritable(std::uintptr_t* vtable, bool writable) { } const std::uintptr_t page_size = static_cast(page_size_value); const std::uintptr_t first = - reinterpret_cast(vtable + kOutputCountVtableIndex); + reinterpret_cast(vtable + kFirstPatchedVtableIndex); const std::uintptr_t last = reinterpret_cast(vtable + kSelectOutputVtableIndex + 1); if (last <= first) { @@ -178,6 +198,8 @@ std::string MakeOutputDeviceGuid(std::uint32_t playback_device_id, } // namespace internal +RobloxOutputDeviceBridge::RobloxOutputDeviceBridge() = default; + RobloxOutputDeviceBridge::~RobloxOutputDeviceBridge() { Shutdown(); } Status RobloxOutputDeviceBridge::Install(const compat::BuildProfile& profile) { @@ -200,6 +222,11 @@ Status RobloxOutputDeviceBridge::Install(const compat::BuildProfile& profile) { "another Roblox output-device bridge owns the process"); } profile_ = *profile.fmod_output_device_bridge; + const char* native_capture = std::getenv("MOCKTAIL_NATIVE_INPUT_CAPTURE"); + capture_profile_supported_ = + profile.elf_build_id == kCaptureBuildId && + profile_.vtable_rva == kCaptureVtableRva && + !(native_capture != nullptr && std::strcmp(native_capture, "0") == 0); if (!linker::SetAndroidLibraryLoadObserver(&ObserveAndroidLibrary, this)) { g_active_bridge.store(nullptr, std::memory_order_release); profile_ = {}; @@ -225,6 +252,7 @@ void RobloxOutputDeviceBridge::Shutdown() { " [audio-menu] failed to restore FmodAudioDevice " "vtable protection\n"); } + NativeInputCapture::Retire(std::move(capture_)); active_ = false; devices_.clear(); string_constructor_ = nullptr; @@ -348,9 +376,9 @@ Status RobloxOutputDeviceBridge::PatchVtableLocked() { return FailedPrecondition("FMOD output-device vtable range overflows"); } const std::uintptr_t first_slot_rva = - profile_.vtable_rva + kOutputCountVtableIndex * sizeof(std::uintptr_t); + profile_.vtable_rva + kFirstPatchedVtableIndex * sizeof(std::uintptr_t); const std::size_t slot_range_size = - (kSelectOutputVtableIndex - kOutputCountVtableIndex + 1) * + (kSelectOutputVtableIndex - kFirstPatchedVtableIndex + 1) * sizeof(std::uintptr_t); if (!IsRelroImageRange(library_base_, first_slot_rva, slot_range_size) || !IsExecutableImageRange(library_base_, profile_.string_constructor_rva, @@ -375,6 +403,15 @@ Status RobloxOutputDeviceBridge::PatchVtableLocked() { "FmodAudioDevice string ABI contract validation failed"); } + if (capture_profile_supported_) { + // The output routes do not depend on capture; keep them if it is refused. + const Status capture_status = PrepareCaptureLocked(); + if (!capture_status.ok()) { + std::fprintf(stderr, " [audio-input] native capture unavailable: %s\n", + capture_status.message().c_str()); + } + } + original_methods_ = { vtable_[kOutputCountVtableIndex], vtable_[kOutputInfoVtableIndex], @@ -382,6 +419,7 @@ Status RobloxOutputDeviceBridge::PatchVtableLocked() { vtable_[kSelectOutputVtableIndex], }; if (!SetVtableWritable(vtable_, true)) { + capture_.reset(); return Status::Error(StatusCode::kPlatformError, "cannot make FmodAudioDevice vtable writable"); } @@ -394,6 +432,18 @@ Status RobloxOutputDeviceBridge::PatchVtableLocked() { FunctionAddress(&GetCurrentOutputDevice), __ATOMIC_RELEASE); __atomic_store_n(&vtable_[kSelectOutputVtableIndex], FunctionAddress(&SetCurrentOutputDevice), __ATOMIC_RELEASE); + if (capture_) { + g_native_capture.store(capture_.get(), std::memory_order_release); + const std::array methods{ + FunctionAddress(&NativeInputCapture::Format), + FunctionAddress(&NativeInputCapture::Latency), + FunctionAddress(&NativeInputCapture::Start), + FunctionAddress(&NativeInputCapture::Stop), + FunctionAddress(&NativeInputCapture::Poll), + FunctionAddress(&NativeInputCapture::Recording)}; + for (std::size_t i = 0; i < kCaptureSlots.size(); ++i) + __atomic_store_n(&vtable_[kCaptureSlots[i]], methods[i], __ATOMIC_RELEASE); + } if (!SetVtableWritable(vtable_, false)) { __atomic_store_n(&vtable_[kOutputCountVtableIndex], original_methods_[0], __ATOMIC_RELEASE); @@ -403,10 +453,72 @@ Status RobloxOutputDeviceBridge::PatchVtableLocked() { __ATOMIC_RELEASE); __atomic_store_n(&vtable_[kSelectOutputVtableIndex], original_methods_[3], __ATOMIC_RELEASE); + if (capture_) { + for (std::size_t i = 0; i < kCaptureSlots.size(); ++i) + __atomic_store_n(&vtable_[kCaptureSlots[i]], + original_capture_methods_[i], __ATOMIC_RELEASE); + NativeInputCapture::Retire(std::move(capture_)); + } (void)SetVtableWritable(vtable_, false); return Status::Error(StatusCode::kPlatformError, "cannot restore FmodAudioDevice RELRO protection"); } + if (capture_) { + std::fprintf(stderr, + " [audio-input] native capture armed for Build ID %s " + "(slots 4/11-15 = %#" PRIxPTR "/%#" PRIxPTR "/%#" PRIxPTR + "/%#" PRIxPTR "/%#" PRIxPTR "/%#" PRIxPTR ")\n", + kCaptureBuildId.data(), + original_capture_methods_[0] - library_base_, + original_capture_methods_[1] - library_base_, + original_capture_methods_[2] - library_base_, + original_capture_methods_[3] - library_base_, + original_capture_methods_[4] - library_base_, + original_capture_methods_[5] - library_base_); + } + return Status::Ok(); +} + +Status RobloxOutputDeviceBridge::PrepareCaptureLocked() { + for (std::size_t i = 0; i < kInputAnchorSlots.size(); ++i) { + if (vtable_[kInputAnchorSlots[i]] != library_base_ + kInputAnchorRvas[i]) { + return FailedPrecondition( + "FmodAudioDevice input slots do not match Build 2998"); + } + } + const auto [lowest_input, highest_input] = + std::minmax_element(kInputAnchorRvas.begin(), kInputAnchorRvas.end()); + for (std::size_t i = 0; i < kCaptureSlots.size(); ++i) { + const std::uintptr_t method = vtable_[kCaptureSlots[i]]; + const std::uintptr_t rva = method - library_base_; + if (method < library_base_ || + !IsExecutableImageRange(library_base_, rva, 1) || + rva + kCaptureCodeWindow < *lowest_input || + rva > *highest_input + kCaptureCodeWindow) { + return FailedPrecondition( + "FmodAudioDevice capture slot is outside the device code"); + } + const bool repeated = + std::find(original_capture_methods_.begin(), + original_capture_methods_.begin() + i, + method) != original_capture_methods_.begin() + i || + std::find(kInputAnchorRvas.begin(), kInputAnchorRvas.end(), rva) != + kInputAnchorRvas.end(); + if (repeated) { + return FailedPrecondition( + "FmodAudioDevice capture slots are not distinct methods"); + } + original_capture_methods_[i] = method; + } + const char* input_device = std::getenv("MOCKTAIL_AUDIO_INPUT_DEVICE"); + const bool enabled = + input_device == nullptr || std::strcmp(input_device, "disabled") != 0; + capture_ = std::make_unique( + NativeInputCapture::SinkAbi{ + library_base_, &IsRelroImageRange, &IsExecutableImageRange, + reinterpret_cast( + original_capture_methods_[1])}, + enabled); return Status::Ok(); } @@ -422,6 +534,11 @@ bool RobloxOutputDeviceBridge::RestoreVtableLocked() { __ATOMIC_RELEASE); __atomic_store_n(&vtable_[kSelectOutputVtableIndex], original_methods_[3], __ATOMIC_RELEASE); + if (capture_) { + for (std::size_t i = 0; i < kCaptureSlots.size(); ++i) + __atomic_store_n(&vtable_[kCaptureSlots[i]], original_capture_methods_[i], + __ATOMIC_RELEASE); + } return SetVtableWritable(vtable_, false); } From 9a1ccbb5cebea7bb578bfb198d2a3246734e6a40 Mon Sep 17 00:00:00 2001 From: komaruworld Date: Sat, 3 Oct 2026 13:38:48 +0300 Subject: [PATCH 13/58] add roblox 3092 support Also fix openxr shit --- README.md | 132 +- config/mocktail.example.yaml | 6 +- config/roblox_compatibility.json | 30 + include/compat/build_profile.h | 7 +- .../audio/roblox_output_device_bridge.h | 7 +- include/mocktail/vr/openxr_backend.h | 62 +- include/mocktail/vr/roblox_vr_device_bridge.h | 9 +- include/mocktail/vr/vr_pose_math.h | 64 +- scripts/find_roblox_vr_offsets.py | 1510 +++++++++++++++++ src/audio/native_input_capture.h | 7 +- src/audio/roblox_output_device_bridge.cc | 201 ++- .../roblox_output_device_bridge_internal.h | 17 + src/compat/build_profile.cc | 9 + src/compat/host_abi_profile.cc | 42 +- src/graphics/bionic_vulkan_loader_adapter.cc | 438 ++++- src/legacy/legacy_runtime.cc | 14 +- src/main.cc | 9 +- src/runtime/runtime_config_bootstrap.cc | 6 +- src/vr/diagnostics.cc | 2 +- src/vr/openxr_backend.cc | 1113 ++++++++---- src/vr/roblox_vr_device_bridge.cc | 282 ++- src/vr/roblox_vr_device_bridge_internal.h | 27 +- src/vr/xr_actions.cc | 20 +- tests/CMakeLists.txt | 12 + tests/build_profile_test.cc | 17 + tests/host_abi_profile_test.cc | 22 + tests/native_input_capture_test.cc | 186 ++ tests/roblox_output_device_bridge_test.cc | 80 + tests/vr/openxr_backend_test.cc | 362 +++- tests/vr/probe_contract_test.py | 2 +- tests/vr/queue_submission_test.cc | 214 +++ tests/vr/roblox_vr_device_bridge_test.cc | 138 +- tests/vr/vr_pose_math_test.cc | 73 +- tests/vr/xr_actions_test.cc | 45 +- 34 files changed, 4403 insertions(+), 762 deletions(-) create mode 100755 scripts/find_roblox_vr_offsets.py create mode 100644 tests/native_input_capture_test.cc create mode 100644 tests/vr/queue_submission_test.cc diff --git a/README.md b/README.md index e4d09667..68e114e4 100644 --- a/README.md +++ b/README.md @@ -1,131 +1,35 @@ -# Mocktail VR (experimental) +# Mocktail VR -Build this checkout, start WiVRn or SteamVR/ALVR, then connect your headset before launching: +Experimental VR support for [Mocktail](https://github.com/komaruworld/mocktail). +Linux x86_64, WiVRn or SteamVR/ALVR. -```sh -git submodule update --init --recursive +## Build + +```bash +git clone --branch vr --recurse-submodules https://github.com/komaruworld/mocktail.git +cd mocktail cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DMOCKTAIL_ENABLE_VR=ON cmake --build build -j4 -./build/mocktail --vr ``` -## Automatic runtime selection - -Normally just run `./build/mocktail --vr`. The default `auto` mode detects a -running SteamVR server (also used by ALVR) or a listening WiVRn IPC socket for -the current desktop user. A stale socket file does not count as a running -server. If exactly one server is detected, its runtime takes priority over an -old registration for the other server. If both or neither are detected, the -registered OpenXR runtime has priority. +Build dependencies are listed in the [main README](https://github.com/komaruworld/mocktail#building). -SteamVR is also found through `openvr/openvrpaths.vrpath`, standard Steam -locations, and the running server's executable path. WiVRn development builds -are discovered near the running server executable; a matching registered -WiVRn manifest is preferred, including a Flatpak registration. The selected -path and the reason for selecting it are printed at startup. +## Run -Explicit `XR_RUNTIME_JSON` and `MOCKTAIL_VR_RUNTIME` overrides remain respected. -To return to automatic selection after manually choosing a runtime: +Start WiVRn or SteamVR/ALVR and connect your headset first. -```sh -unset XR_RUNTIME_JSON MOCKTAIL_VR_RUNTIME +```bash ./build/mocktail --vr ``` -Detection does not start servers, connect a headset, or modify system runtime -registration. It observes the current process/user namespace; container and -host runtimes still need correctly shared libraries and IPC access. +The runtime is detected automatically. To pick one manually: -## WiVRn - -Start WiVRn and connect its headset client first. Run WiVRn and Mocktail as the -same desktop user, with the same runtime environment; do not start Mocktail -from a root shell while WiVRn runs in your user session. For containers/chroots, -the host runtime library and IPC socket must both be accessible inside. - -```sh +```bash MOCKTAIL_VR_RUNTIME=wivrn ./build/mocktail --vr +# or +MOCKTAIL_VR_RUNTIME=alvr MOCKTAIL_GRAPHICS_BACKEND=direct-vulkan ./build/mocktail --vr ``` -If WiVRn disconnects and exits, reconnect/restart it before retrying Mocktail. -`Connection refused` on `wivrn/comp_ipc` means the selected server is not -accepting the connection. Installing `dbus-launch` alone does not fix that. - -## ALVR through SteamVR - -Install SteamVR and ALVR, register the ALVR driver using its dashboard, then -connect the headset and confirm SteamVR sees it. Follow the upstream -[ALVR Linux instructions](https://github.com/alvr-org/ALVR/wiki/Linux-Troubleshooting), -including the SteamVR launch-options workaround when required. - -```sh -env -u XR_RUNTIME_JSON MOCKTAIL_VR_RUNTIME=alvr \ - MOCKTAIL_GRAPHICS_BACKEND=direct-vulkan ./build/mocktail --vr -``` - -The command clears an inherited WiVRn manifest override. `steamvr` is an alias -for `alvr`. The selection uses SteamVR's OpenXR runtime; -ALVR handles the streaming. If automatic discovery cannot find a custom -installation, specify its actual manifest: - -```sh -XR_RUNTIME_JSON="/path/to/SteamVR/steamxr_linux64.json" ./build/mocktail --vr -``` +If you previously set `XR_RUNTIME_JSON`, unset it before switching runtimes. +Use `--no-vr` for desktop mode. -`XR_RUNTIME_JSON` always wins. `MOCKTAIL_VR_RUNTIME=system` delegates selection -to the OpenXR loader. The default `auto` uses the server detection described above, then registered -and installed runtimes. Selection affects only this process. - -Vulkan VR mirrors the left eye into the desktop Roblox window using a GPU -blit, preserving its aspect ratio. DebugDeviceVR otherwise leaves that window -black even when the headset receives eye images. Set -`MOCKTAIL_VR_DESKTOP_MIRROR=0` to disable this copy. This is a view of the eye, -not a separate desktop camera or a replacement for VR menu input. - -## OpenGL ES - -The `opengl`/`system` renderer uses the guest's real EGL/OpenGL ES context: - -```sh -MOCKTAIL_GRAPHICS_BACKEND=opengl ./build/mocktail --vr -``` - -The runtime must provide `XR_KHR_opengl_es_enable` and `XR_MNDX_egl_enable`, and -support sharing images with the guest EGL device. Both eye copies stay on the -GPU; no Python viewer or CPU readback is required. Camera tracking uses the -same OpenXR frame/pose pipeline as Vulkan. If the runtime cannot support this -EGL binding, use `MOCKTAIL_GRAPHICS_BACKEND=direct-vulkan`; the ALVR command -above deliberately selects Vulkan. - -## Current validation and limitations - -- Native VR camera/eye hooks are verified for Roblox build **2998**, Build ID - `ade08266c67aee88ec9c1d00902150e1684dad3a`. Approval of a newer payload for - ordinary rendering does **not** establish VR compatibility. Do not bypass - the Build-ID checks to run the old VR hooks on build 3092. -- GLES framebuffer copying, distinct eyes, state restoration, resource - invalidation, and an EGL OpenXR session with session-loss recovery have been - tested. A local unauthenticated Roblox run on hardware GLES submitted over - 240 projection frames through a simulated Monado headset. -- This host's software GLES/Vulkan combination could create XR texture names - but not usable framebuffer attachments. The copy failure is reported and - XR output stops; this is not counted as a successful rendering test. -- The local Monado test runtime reports `u_graphics_buffer_unref: Bad file - descriptor (fd: 0)` during GLES teardown, including successful hardware - runs. This remains an integration caveat; clean headset teardown is not - claimed from those simulator runs. During auto-selection validation, one - run also failed a later EGL session recreation with - `vkCreateFence: VK_ERROR_OUT_OF_HOST_MEMORY`; a fresh repeat passed. - Runtime selection and initial frame submission succeeded in both runs. -- Real headset acceptance through WiVRn and ALVR/SteamVR is still pending. - Controller input and canted eye orientations are not implemented. -- Use `./build/mocktail --no-vr` for ordinary rendering. No helper script is - needed for normal VR startup. - -The GLES OpenXR integration test is opt-in and needs an already connected -runtime (or a separately prepared simulator). Run from a graphical desktop: - -```sh -MOCKTAIL_TEST_XR_GLES=1 ./build/tests/gles_vr_transport_test \ - --gtest_filter='*OpenXrEglSession*' -``` diff --git a/config/mocktail.example.yaml b/config/mocktail.example.yaml index b1514705..208afbe3 100644 --- a/config/mocktail.example.yaml +++ b/config/mocktail.example.yaml @@ -51,12 +51,12 @@ graphics: # vr: # Boolean (default: enabled in VR builds, disabled in non-VR builds). - # Experimental VR for the exact Build ID 2998 - # payload. Mocktail initializes the guest DebugDeviceVR eye resources (two + # Experimental VR for verified build 3092. Mocktail initializes + # the guest DebugDeviceVR eye resources (two # distinct 500x500 eye framebuffers), the Roblox renderer draws both eyes # every frame, and the frames are submitted to the active OpenXR runtime as # a stereo projection layer with head pose (runtime-tracked, or scripted for - # desktop testing). Controllers are NOT connected yet. Standalone OpenXR + # desktop testing). Controllers use the native VR bridge. Standalone OpenXR # test scene: mocktail-vr-probe --scene. Use --no-vr to launch normally. # enabled: false diff --git a/config/roblox_compatibility.json b/config/roblox_compatibility.json index 40883da3..3c755c8b 100644 --- a/config/roblox_compatibility.json +++ b/config/roblox_compatibility.json @@ -91,6 +91,36 @@ "device_create_framebuffer_vtable_offset": "0x180" }, "reason": "Verified 2026-09-06: exact Build-ID profile passed two isolated no-recovery Tier C Vulkan canaries and real game sessions, including fullscreen state synchronization and host FMOD output-device interposition; legacy executable patches and signal recovery remain disabled. Experimental vr_debug_device_bridge metadata (DebugDeviceVR constructor/state/eye/initializer RVAs) is machine-validated at activation and gated behind --vr; native stereo output is not yet verified end-to-end." + }, + { + "allow_host_abi_bridges": true, + "allow_host_constructor_replay": true, + "allow_legacy_binary_patches": false, + "default_allowed": true, + "elf_build_id": "5f0704edd9064f566ee3d6df2bd2fabbcc709f03", + "fmod_output_device_bridge": { + "count_method_rva": "0x320c676", + "current_method_rva": "0x320c6c6", + "info_method_rva": "0x320c716", + "select_method_rva": "0x320bf78", + "string_constructor_rva": "0x1d7a038", + "vtable_layout_version": 2, + "vtable_rva": "0x6cd3ce0" + }, + "reason": "Exact-Build-ID host ABI promoted from the validated probation profile; native stereo, VR state/pointer/haptic contracts and FMOD output-device layout v2 were re-derived for 3092. Physical headset acceptance remains pending.", + "status": "supported", + "user_game_settings_fullscreen_setter_rva": "0x45c5f42", + "version_code": 3092, + "version_name": "2.738.1397", + "vr_debug_device_bridge": { + "vtable_rva": "0x6d08108", + "constructor_rva": "0x37f9fe4", + "state_getter_rva": "0x37fa3c6", + "eye_getter_rva": "0x37fa3b6", + "eye_initializer_rva": "0x37fa3e8", + "emulator_flag_storage_rva": "0x73f4690", + "device_create_framebuffer_vtable_offset": "0x188" + } } ] } diff --git a/include/compat/build_profile.h b/include/compat/build_profile.h index 54836623..4cd0bff3 100644 --- a/include/compat/build_profile.h +++ b/include/compat/build_profile.h @@ -22,13 +22,12 @@ struct FmodOutputDeviceBridgeProfile { std::uintptr_t info_method_rva = 0; std::uintptr_t current_method_rva = 0; std::uintptr_t select_method_rva = 0; + std::uint32_t vtable_layout_version = 1; }; // Exact-build metadata for the experimental native VR bridge. The bridge -// interposes only DebugDeviceVR virtual slots (state getter and eye -// framebuffer getter) and invokes the guest eye-resource initializer through -// its own vtable; executable guest code is never modified. All RVAs are -// validated against machine-code contracts before the first use. +// interposes DebugDeviceVR slots and a checked world-pointer trampoline. +// All RVAs are validated against machine-code contracts before first use. struct VrDebugDeviceBridgeProfile { std::uintptr_t vtable_rva = 0; std::uintptr_t constructor_rva = 0; diff --git a/include/mocktail/audio/roblox_output_device_bridge.h b/include/mocktail/audio/roblox_output_device_bridge.h index 45c3c127..633a85a2 100644 --- a/include/mocktail/audio/roblox_output_device_bridge.h +++ b/include/mocktail/audio/roblox_output_device_bridge.h @@ -15,9 +15,12 @@ namespace mocktail::audio { class NativeInputCapture; +namespace internal { +struct FmodInputCaptureProfile; +} // Replaces the output-device query/select slots of the exact Build-ID-scoped -// FmodAudioDevice vtable, and on Build 2998 its microphone capture slots. +// FmodAudioDevice vtable, including microphone capture on verified builds. // Roblox keeps owning its FMOD engine, while the existing settings UI sees // and selects the SDL host routes that consume Android AudioTrack PCM. class RobloxOutputDeviceBridge final { @@ -59,7 +62,7 @@ class RobloxOutputDeviceBridge final { void ConstructGuestString(void* destination, std::string_view value) const; std::unique_ptr capture_; - bool capture_profile_supported_ = false; + const internal::FmodInputCaptureProfile* capture_profile_ = nullptr; std::array original_capture_methods_{}; mutable std::mutex mutex_; compat::FmodOutputDeviceBridgeProfile profile_; diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index 88e6cd7f..bc053d8c 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -17,6 +17,16 @@ namespace mocktail::vr { +namespace internal { +// Consume binary present dependencies exactly once. On success a later present +// must omit them; on failure the caller must propagate the error, not retry. +VkResult WaitForPresentDependencies(VkDevice device, VkQueue queue, + const VkPresentInfoKHR& present, + VkFence fence, PFN_vkResetFences reset, + PFN_vkQueueSubmit submit, + PFN_vkWaitForFences wait); +} // namespace internal + // Resolved --vr output mode. kXrOutput drives the real Roblox stereo images // into an OpenXR projection layer; kNativeOnly keeps the diagnostic native // stereo prototype without any runtime requirement. Selected through @@ -56,7 +66,10 @@ struct ScriptedPoseSample { float eye_fov[2][4] = {}; // left, right, up, down angles // Index 0 = left, 1 = right (Roblox UserCFrame LeftHand/RightHand). VrHandPose hands[2]; - // Exact 2998 DebugDeviceVR input channels, published with the same pose. + float floor_position[3] = {}; + float floor_orientation[4] = {0.f, 0.f, 0.f, 1.f}; + bool floor_valid = false; + // Verified DebugDeviceVR input channels, published with the same pose. float controller_channels[28] = {}; bool controllers_connected = false; std::uint64_t frame = 0; @@ -126,6 +139,24 @@ class OpenXrBackend final { void NoteInstanceDestroyed(VkInstance instance); void NoteQueue(VkDevice device, VkQueue queue, std::uint32_t family, std::uint32_t index); + std::vector DesktopQueueFamilies(VkDevice device) const; + void RecordCommandPool(VkDevice device, VkCommandPool pool, + std::uint32_t family); + void RecordCommandBuffers(VkDevice device, VkCommandPool pool, + std::uint32_t count, + const VkCommandBuffer* buffers); + void ForgetCommandBuffers(std::uint32_t count, + const VkCommandBuffer* buffers); + void ForgetCommandPool(VkCommandPool pool); + void ResetCommandPool(VkCommandPool pool); + void ResetCommandBuffer(VkCommandBuffer buffer); + void NoteCommandBuffersSubmitted(VkQueue queue, std::uint32_t count, + const VkCommandBuffer* buffers); + VkImageLayout NormalizeDesktopBarrier(VkCommandBuffer command, VkImage image, + std::uint32_t* source, + std::uint32_t* destination, + VkImageLayout* old_layout, + VkImageLayout new_layout); // Resource provenance recording (adapter wrappers). bool WantsResourceRecords() const { @@ -149,8 +180,14 @@ class OpenXrBackend final { // Render-thread frame cycle at host present begin. void NoteHostPresent(VkQueue queue, VkDevice device); + // VK_NOT_READY leaves the present untouched until eye submission provenance + // is known. VK_SUCCESS means its waits were consumed on the graphics queue. + VkResult PrepareHostPresent(VkQueue queue, VkDevice device, + const VkPresentInfoKHR* info); void RecordDesktopSwapchain(VkDevice device, VkSwapchainKHR swapchain, - const VkSwapchainCreateInfoKHR* info, const VkImage* images, unsigned count); + const VkSwapchainCreateInfoKHR* info, + const VkImage* images, unsigned count, + bool converted_to_concurrent = false); // On success the original present semaphores have been consumed and the // copy has completed. The caller must present with no wait semaphores. VkResult MirrorDesktop(VkQueue queue, const VkPresentInfoKHR* info); @@ -197,6 +234,7 @@ class OpenXrBackend final { std::uint32_t height = 0; VkFormat format = VK_FORMAT_UNDEFINED; VkImageUsageFlags usage = 0; + VkSampleCountFlagBits samples = VK_SAMPLE_COUNT_1_BIT; void* owner = nullptr; std::uint64_t order = 0; }; @@ -251,6 +289,9 @@ class OpenXrBackend final { void HandleReferenceSpaceChange(std::int64_t change_time); bool RecoverSessionLocked(std::uint64_t now_ns); void PollSessionEvents(); + bool SupportsPresentQueueLocked(VkQueue queue) const; + VkQueue GraphicsQueueForPresentLocked(VkQueue present) const; + bool unsupported_present_queue_logged_ = false; // Drops the pose/images of the frame currently in flight without touching // guest ownership, so the next xrEndFrame submits no layer instead of an // image paired with a pose that no longer describes the same origin. @@ -290,6 +331,7 @@ class OpenXrBackend final { void* xr_session_ = nullptr; void* xr_local_space_ = nullptr; void* xr_view_space_ = nullptr; + void* xr_stage_space_ = nullptr; std::uint32_t xr_eye_width_[2] = {}; std::uint32_t xr_eye_height_[2] = {}; EyeSwapchain eye_swapchains_[2]; @@ -302,6 +344,8 @@ class OpenXrBackend final { VkDevice vk_device_ = VK_NULL_HANDLE; VkPhysicalDevice vk_physical_device_ = VK_NULL_HANDLE; std::uint32_t vk_queue_family_ = 0; + VkQueue vk_graphics_queue_ = VK_NULL_HANDLE; + std::vector device_queue_families_; void* vk_loader_ = nullptr; struct VkProcs; VkProcs* vk_ = nullptr; @@ -319,6 +363,9 @@ class OpenXrBackend final { VkExtent2D extent; VkFormat format; std::vector images; + VkSharingMode sharing_mode = VK_SHARING_MODE_EXCLUSIVE; + std::vector families; + bool converted_to_concurrent = false; }; std::unordered_map desktop_swapchains_; std::unordered_map image_views_; @@ -326,6 +373,15 @@ class OpenXrBackend final { std::unordered_map framebuffer_render_pass_; std::unordered_map> render_pass_final_layouts_; std::unordered_map queue_families_; + struct CommandRecord { + VkCommandPool pool = VK_NULL_HANDLE; + std::uint32_t family = UINT32_MAX; + std::vector> eye_images; + }; + std::unordered_map command_pool_families_; + std::unordered_map command_records_; + std::unordered_map eye_render_queues_; + std::unordered_map eye_render_frames_; std::uint64_t image_order_ = 0; EyeBinding eyes_[2]; @@ -353,7 +409,7 @@ class OpenXrBackend final { bool frame_views_valid_ = false; // One-shot guard so a canted-view runtime reports the rejection reason once // instead of once per frame per eye. - bool canted_rejection_logged_ = false; + bool projection_rejection_logged_ = false; mutable ScriptedPoseSample published_pose_; std::uint64_t xr_frame_counter_ = 0; std::unordered_map applied_poses_; diff --git a/include/mocktail/vr/roblox_vr_device_bridge.h b/include/mocktail/vr/roblox_vr_device_bridge.h index c6b45b57..7d214fa7 100644 --- a/include/mocktail/vr/roblox_vr_device_bridge.h +++ b/include/mocktail/vr/roblox_vr_device_bridge.h @@ -4,6 +4,7 @@ #include #include #include +#include #include "compat/build_profile.h" #include "mocktail/status.h" @@ -11,8 +12,12 @@ namespace mocktail::vr { namespace internal { struct VrBridgeTestAccess; +struct VrBuildContract; } +const compat::VrDebugDeviceBridgeProfile* FindRobloxVrDeviceProfile( + std::string_view build_id); + // Experimental native stereo bring-up for the exact Build-ID-scoped // RBX::Graphics::DebugDeviceVR object of the guest client. // @@ -21,7 +26,7 @@ struct VrBridgeTestAccess; // initialization never dereferences cached guest object/device pointers. The // guest owns resource destruction. Recreated objects and reused addresses // therefore need no cache invalidation. Initialization happens only on a thread -// observed presenting. OpenXR image submission and poses are not connected yet. +// observed presenting. class RobloxVrDeviceBridge final { public: using StateGetterFn = void *(*)(void *result_buffer, void *device_object); @@ -54,6 +59,7 @@ class RobloxVrDeviceBridge final { // Guest-side hooks invoked by the interposed vtable stubs. void OnHapticsCall(void* device_object, int hand, float amplitude); + void OnPointerFrameCall(void* result, void* service); void OnStateGetterCall(void *result_buffer, void *device_object); void OnEyeGetterCall(void *device_object, int eye_index, void *framebuffer); @@ -85,6 +91,7 @@ class RobloxVrDeviceBridge final { mutable std::mutex mutex_; compat::VrDebugDeviceBridgeProfile profile_{}; + const internal::VrBuildContract* contract_ = nullptr; bool installed_ = false; std::atomic active_{false}; std::atomic image_base_{0}; diff --git a/include/mocktail/vr/vr_pose_math.h b/include/mocktail/vr/vr_pose_math.h index e66b143c..27a17f29 100644 --- a/include/mocktail/vr/vr_pose_math.h +++ b/include/mocktail/vr/vr_pose_math.h @@ -1,6 +1,7 @@ #ifndef MOCKTAIL_VR_POSE_MATH_H_ #define MOCKTAIL_VR_POSE_MATH_H_ +#include #include #include @@ -147,16 +148,61 @@ inline ProjectionTangents FovToTangents(const EyeFov& fov) { std::tan(-fov.angle_left), std::tan(fov.angle_right)}; } -// The pinned guest state can only represent two parallel eye orientations. -// A canted (rotated) relative eye pose needs a different camera ABI, so it is -// rejected explicitly rather than silently submitted with a projection that -// does not match how the image was rendered. -inline constexpr float kCantedTolerance = 0.001f; +inline constexpr float kMaxEyeFovAngle = 1.56f; -inline bool EyeOrientationIsParallel(const Quat& relative_orientation) { - return std::abs(relative_orientation.x) <= kCantedTolerance && - std::abs(relative_orientation.y) <= kCantedTolerance && - std::abs(relative_orientation.z) <= kCantedTolerance; +// Enclose the display frustum in a head-parallel camera for the guest renderer. +// The compositor receives this rendered FOV and pose, not the canted display's. +inline bool ParallelEyeFov(const Quat& eye_orientation, const EyeFov& display, + EyeFov* rendered) { + Quat orientation; + if (!rendered || !QuatNormalize(eye_orientation, &orientation) || + !(display.angle_left < display.angle_right) || + !(display.angle_down < display.angle_up)) { + return false; + } + const float angles[] = {display.angle_left, display.angle_right, + display.angle_up, display.angle_down}; + for (float angle : angles) { + if (!std::isfinite(angle) || std::abs(angle) >= kMaxEyeFovAngle) + return false; + } + if (orientation.x == 0.f && orientation.y == 0.f && orientation.z == 0.f) { + *rendered = display; + return true; + } + const float xs[] = {std::tan(display.angle_left), + std::tan(display.angle_right)}; + const float ys[] = {std::tan(display.angle_down), std::tan(display.angle_up)}; + float left = 0.f, right = 0.f, down = 0.f, up = 0.f; + bool first = true; + for (float x : xs) { + for (float y : ys) { + const Vec3 ray = QuatRotate(orientation, {x, y, -1.f}); + if (!std::isfinite(ray.z) || ray.z >= -1e-6f) return false; + const float horizontal = ray.x / -ray.z; + const float vertical = ray.y / -ray.z; + if (!std::isfinite(horizontal) || !std::isfinite(vertical)) return false; + if (first) { + left = right = horizontal; + down = up = vertical; + first = false; + } else { + left = std::min(left, horizontal); + right = std::max(right, horizontal); + down = std::min(down, vertical); + up = std::max(up, vertical); + } + } + } + const EyeFov result{std::atan(left), std::atan(right), std::atan(up), + std::atan(down)}; + const float bounds[] = {result.angle_left, result.angle_right, + result.angle_up, result.angle_down}; + for (float angle : bounds) { + if (std::abs(angle) >= kMaxEyeFovAngle) return false; + } + *rendered = result; + return true; } // Metres-to-studs factor the exact-build native helper applies itself. Exposed diff --git a/scripts/find_roblox_vr_offsets.py b/scripts/find_roblox_vr_offsets.py new file mode 100755 index 00000000..80a9d0d9 --- /dev/null +++ b/scripts/find_roblox_vr_offsets.py @@ -0,0 +1,1510 @@ +#!/usr/bin/env python3 +# Copyright 2026 Mocktail Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Find native VR bridge offsets in an x86-64 Roblox ELF. + +Usage: + python3 find_roblox_vr_offsets.py --elf libroblox.so --output offsets.json +""" + +from __future__ import annotations + +import argparse +import bisect +from collections.abc import Iterator, Sequence +import dataclasses +import hashlib +import json +import mmap +import os +import pathlib +import re +import stat +import struct +import sys +import tempfile +from typing import Any + +try: + import capstone + from capstone import x86 as capstone_x86 +except ImportError: + capstone = None + capstone_x86 = None + +MAX_ELF_BYTES = 512 * 1024 * 1024 +MAX_RELA_DYN_BYTES = 256 * 1024 * 1024 +MAX_RELOCATIONS = 4 * 1024 * 1024 +MAX_FUNCTION_TABLE_ENTRIES = 2 * 1024 * 1024 +MAX_FUNCTION_BYTES = 0x20000 +MAX_JSON_OUTPUT_BYTES = 4 * 1024 * 1024 +SHA256_CHUNK_BYTES = 1024 * 1024 + +ELF_HEADER = struct.Struct("<16sHHIQQQIHHHHHH") +PROGRAM_HEADER = struct.Struct(" bool: + return (size >= 0 and self.virtual_address <= rva and + rva + size <= self.virtual_address + self.memory_size) + + def contains_file_data(self, rva: int, size: int = 1) -> bool: + return (size >= 0 and self.virtual_address <= rva and + rva + size <= self.virtual_address + self.file_size) + + +@dataclasses.dataclass(frozen=True) +class Section: + name: str + section_type: int + flags: int + address: int + offset: int + size: int + + +@dataclasses.dataclass(frozen=True) +class Relocation: + offset: int + info: int + addend: int + + +def sha256_file(file_object: Any) -> str: + digest = hashlib.sha256() + while True: + chunk = file_object.read(SHA256_CHUNK_BYTES) + if not chunk: + return digest.hexdigest() + digest.update(chunk) + + +class Sleb128Reader: + + def __init__(self, encoded: bytes): + self._encoded = encoded + self.position = 0 + + def pop(self) -> int: + value = 0 + shift = 0 + while True: + if self.position >= len(self._encoded) or shift >= 70: + raise AnalyzerError("invalid or truncated APS2 SLEB128 value") + byte = self._encoded[self.position] + self.position += 1 + value |= (byte & 0x7F) << shift + shift += 7 + if (byte & 0x80) == 0: + break + if shift < 64 and (byte & 0x40): + value -= 1 << shift + if not -(1 << 63) <= value < (1 << 64): + raise AnalyzerError("APS2 SLEB128 value exceeds 64-bit bounds") + return value + + def at_end(self) -> bool: + return self.position == len(self._encoded) + + +def decode_aps2_relocations(encoded: bytes) -> Iterator[Relocation]: + if not encoded.startswith(b"APS2"): + raise AnalyzerError(".rela.dyn does not use the required APS2 encoding") + reader = Sleb128Reader(encoded[4:]) + relocation_count = reader.pop() + relocation_offset = reader.pop() + if relocation_count < 0 or relocation_count > MAX_RELOCATIONS: + raise AnalyzerError("APS2 relocation count is invalid") + relocation_info = 0 + relocation_addend = 0 + emitted = 0 + while emitted < relocation_count: + group_size = reader.pop() + group_flags = reader.pop() + if group_size <= 0 or group_size > relocation_count - emitted: + raise AnalyzerError("APS2 relocation group is invalid") + offset_delta = reader.pop( + ) if group_flags & APS2_GROUPED_BY_OFFSET_DELTA else 0 + if group_flags & APS2_GROUPED_BY_INFO: + relocation_info = reader.pop() + addend_mode = group_flags & (APS2_GROUP_HAS_ADDEND | + APS2_GROUPED_BY_ADDEND) + if addend_mode == (APS2_GROUP_HAS_ADDEND | APS2_GROUPED_BY_ADDEND): + relocation_addend += reader.pop() + elif addend_mode != APS2_GROUP_HAS_ADDEND: + relocation_addend = 0 + for _ in range(group_size): + relocation_offset += (offset_delta if group_flags & + APS2_GROUPED_BY_OFFSET_DELTA else + reader.pop()) + if not group_flags & APS2_GROUPED_BY_INFO: + relocation_info = reader.pop() + if addend_mode == APS2_GROUP_HAS_ADDEND: + relocation_addend += reader.pop() + if (relocation_offset < 0 or relocation_info < 0 or + relocation_addend < 0 or relocation_offset >= 1 << 64 or + relocation_info >= 1 << 64 or relocation_addend >= 1 << 64): + raise AnalyzerError( + "APS2 relocation exceeds unsigned 64-bit bounds") + emitted += 1 + yield Relocation(relocation_offset, relocation_info, + relocation_addend) + if not reader.at_end(): + raise AnalyzerError("APS2 relocation stream contains trailing bytes") + + +class ElfImage: + + def __init__(self, path: pathlib.Path): + self.path = path + self._file = None + self.data = None + self.file_size = 0 + self.sha256 = "" + self.sections: dict[str, Section] = {} + self.segments: tuple[Segment, ...] = () + self.relro_ranges: tuple[tuple[int, int], ...] = () + self.build_id = "" + self._relocations: dict[int, int] | None = None + + def __enter__(self) -> "ElfImage": + try: + file_status = os.lstat(self.path) + except OSError as error: + raise AnalyzerError(f"cannot stat ELF file: {self.path}") from error + if stat.S_ISLNK( + file_status.st_mode) or not stat.S_ISREG(file_status.st_mode): + raise AnalyzerError( + f"ELF input must be a regular non-symlink: {self.path}") + if (file_status.st_size < ELF_HEADER.size or + file_status.st_size > MAX_ELF_BYTES): + raise AnalyzerError(f"ELF input has an invalid size: {self.path}") + self.file_size = file_status.st_size + try: + self._file = self.path.open("rb") + self.sha256 = sha256_file(self._file) + self._file.seek(0) + self.data = mmap.mmap(self._file.fileno(), + 0, + access=mmap.ACCESS_READ) + self._parse() + except AnalyzerError: + self.__exit__(None, None, None) + raise + except (OSError, ValueError, struct.error) as error: + self.__exit__(None, None, None) + raise AnalyzerError( + f"cannot parse ELF file: {self.path}") from error + return self + + def __exit__(self, _type, _value, _traceback) -> None: + if self.data is not None: + self.data.close() + self.data = None + if self._file is not None: + self._file.close() + self._file = None + + def _checked_range(self, offset: int, size: int, description: str) -> None: + if offset < 0 or size < 0 or offset > self.file_size - size: + raise AnalyzerError(f"ELF {description} exceeds file bounds") + + def bytes_at(self, offset: int, size: int, description: str) -> bytes: + self._checked_range(offset, size, description) + return self.data[offset:offset + size] + + def section_bytes(self, section: Section) -> bytes: + return self.bytes_at(section.offset, section.size, + f"section {section.name}") + + def _parse(self) -> None: + header = ELF_HEADER.unpack_from(self.data, 0) + identification = header[0] + if (identification[:4] != b"\x7fELF" or + identification[4] != ELF_CLASS_64 or + identification[5] != ELF_DATA_LITTLE_ENDIAN or + identification[6] != ELF_VERSION_CURRENT or + header[1] != ELF_TYPE_SHARED_OBJECT or + header[2] != ELF_MACHINE_X86_64 or + header[3] != ELF_VERSION_CURRENT): + raise AnalyzerError( + "candidate must be a little-endian x86-64 ET_DYN ELF") + + program_offset = header[5] + section_offset = header[6] + program_entry_size = header[9] + program_count = header[10] + section_entry_size = header[11] + section_count = header[12] + section_names_index = header[13] + if (program_entry_size != PROGRAM_HEADER.size or + section_entry_size != SECTION_HEADER.size or + program_count == 0 or section_count == 0 or + section_names_index == 0 or + section_names_index >= section_count): + raise AnalyzerError("ELF header tables use an unsupported layout") + self._checked_range(program_offset, program_count * program_entry_size, + "program headers") + self._checked_range(section_offset, section_count * section_entry_size, + "section headers") + + segments = [] + relro_ranges = [] + for index in range(program_count): + values = PROGRAM_HEADER.unpack_from( + self.data, program_offset + index * program_entry_size) + if values[0] == PT_GNU_RELRO: + if values[6] <= 0: + raise AnalyzerError("ELF PT_GNU_RELRO range is empty") + relro_ranges.append((values[3], values[6])) + if values[0] != PT_LOAD: + continue + segment = Segment( + offset=values[2], + virtual_address=values[3], + file_size=values[5], + memory_size=values[6], + flags=values[1], + ) + self._checked_range(segment.offset, segment.file_size, + "PT_LOAD data") + if segment.file_size > segment.memory_size: + raise AnalyzerError("ELF PT_LOAD file size exceeds memory size") + segments.append(segment) + if not segments: + raise AnalyzerError("ELF has no PT_LOAD segments") + self.segments = tuple(segments) + self.relro_ranges = tuple(relro_ranges) + + raw_sections = [ + SECTION_HEADER.unpack_from( + self.data, section_offset + index * section_entry_size) + for index in range(section_count) + ] + names_header = raw_sections[section_names_index] + self._checked_range(names_header[4], names_header[5], + "section-name table") + names = self.data[names_header[4]:names_header[4] + names_header[5]] + + by_name: dict[str, Section] = {} + for values in raw_sections: + name_offset = values[0] + if name_offset >= len(names): + raise AnalyzerError("ELF section name exceeds string table") + name_end = names.find(b"\0", name_offset) + if name_end < 0: + raise AnalyzerError("ELF section name is not terminated") + try: + name = names[name_offset:name_end].decode("ascii") + except UnicodeDecodeError as error: + raise AnalyzerError("ELF section name is not ASCII") from error + section = Section( + name=name, + section_type=values[1], + flags=values[2], + address=values[3], + offset=values[4], + size=values[5], + ) + if section.section_type != SHT_NOBITS: + self._checked_range(section.offset, section.size, + f"section {name}") + if name and name in by_name: + raise AnalyzerError(f"ELF contains duplicate section {name}") + if name: + by_name[name] = section + self.sections = by_name + + text = self._require_section(".text", SHT_PROGBITS) + if not text.flags & 2 or not text.flags & 4 or text.flags & 1: + raise AnalyzerError( + "ELF .text section must be read+execute, non-writable") + self._require_section(".rela.dyn", SHT_ANDROID_RELA) + self._require_section(".eh_frame_hdr", SHT_PROGBITS) + self._require_section(".note.gnu.build-id", SHT_NOTE) + self.build_id = self._read_build_id() + if len(self.build_id) != 40: + raise AnalyzerError("ELF GNU build ID is not a 20-byte SHA-1") + + def _require_section(self, name: str, section_type: int) -> Section: + section = self.sections.get(name) + if section is None or section.section_type != section_type: + raise AnalyzerError(f"ELF is missing required {name} section") + return section + + def _read_build_id(self) -> str: + section = self.sections[".note.gnu.build-id"] + encoded = self.section_bytes(section) + offset = 0 + while offset + NOTE_HEADER.size <= len(encoded): + namesz, descsz, note_type = NOTE_HEADER.unpack_from(encoded, offset) + offset += NOTE_HEADER.size + if offset + namesz > len(encoded): + raise AnalyzerError("GNU build-id note name is truncated") + note_name = encoded[offset:offset + namesz] + offset += namesz + offset += (-offset) % 4 + if note_type == NT_GNU_BUILD_ID and note_name.rstrip( + b"\0") == b"GNU": + if offset + descsz > len(encoded): + raise AnalyzerError("GNU build-id note data is truncated") + return encoded[offset:offset + descsz].hex() + offset += descsz + offset += (-offset) % 4 + raise AnalyzerError("ELF has no GNU build-id note") + + def segment_for_rva(self, rva: int, size: int = 1) -> Segment | None: + matches = [ + segment for segment in self.segments + if segment.contains_memory(rva, size) + ] + return matches[0] if len(matches) == 1 else None + + def rva_to_offset(self, rva: int, size: int = 1) -> int: + matches = [ + segment for segment in self.segments + if segment.contains_file_data(rva, size) + ] + if len(matches) != 1: + raise AnalyzerError(f"RVA 0x{rva:x} has no unique file mapping") + segment = matches[0] + offset = segment.offset + rva - segment.virtual_address + self._checked_range(offset, size, "RVA mapping") + return offset + + def offset_to_rva(self, offset: int) -> int | None: + for segment in self.segments: + if segment.offset <= offset < segment.offset + segment.file_size: + return offset - segment.offset + segment.virtual_address + return None + + def require_code_rva(self, rva: int, size: int = 1) -> None: + segment = self.segment_for_rva(rva, size) + if (segment is None or segment.flags & + (PF_READ | PF_EXECUTE) != (PF_READ | PF_EXECUTE) or + segment.flags & PF_WRITE or + not segment.contains_file_data(rva, size)): + raise AnalyzerError( + f"RVA 0x{rva:x} is not in a read/execute segment") + + def require_writable_rva(self, rva: int, size: int = 1) -> None: + segment = self.segment_for_rva(rva, size) + if (segment is None or + segment.flags & (PF_READ | PF_WRITE) != (PF_READ | PF_WRITE) or + segment.flags & PF_EXECUTE): + raise AnalyzerError( + f"RVA 0x{rva:x} is not in a non-executable RW segment") + + def require_relro_rva(self, rva: int, size: int = 1) -> None: + matches = [ + (begin, length) + for begin, length in self.relro_ranges + if size >= 0 and begin <= rva and rva + size <= begin + length + ] + if len(matches) != 1: + raise AnalyzerError( + f"RVA 0x{rva:x} is not in a unique PT_GNU_RELRO range") + + def text_section(self) -> Section: + return self.sections[".text"] + + def find_alloc_string(self, needle: bytes) -> int: + """Return the unique read-only mapped RVA whose bytes equal needle.""" + if b"\0" in needle: + raise AnalyzerError("anchor strings must not contain NUL bytes") + wanted = needle + b"\0" + hits: list[int] = [] + position = 0 + while True: + position = self.data.find(wanted, position) + if position < 0: + break + rva = self.offset_to_rva(position) + if rva is not None: + segment = self.segment_for_rva(rva, len(wanted)) + if segment is not None and not segment.flags & PF_WRITE: + hits.append(rva) + position += 1 + if not hits: + raise AnalyzerError(f"anchor string {needle.decode('ascii')!r}" + " was not found in the ELF") + if len(hits) > 1: + listed = ", ".join(f"0x{hit:x}" for hit in hits) + raise AnalyzerError(f"anchor string {needle.decode('ascii')!r}" + f" is ambiguous: {listed}") + return hits[0] + + def relative_relocations(self) -> dict[int, int]: + if self._relocations is not None: + return self._relocations + section = self.sections[".rela.dyn"] + if section.size > MAX_RELA_DYN_BYTES: + raise AnalyzerError(".rela.dyn section is suspiciously large") + encoded = self.section_bytes(section) + relocations: dict[int, int] = {} + count = 0 + for relocation in decode_aps2_relocations(encoded): + count += 1 + if count > MAX_RELOCATIONS: + raise AnalyzerError("APS2 relocation stream is too large") + if relocation.info & 0xFFFFFFFF in ( + R_X86_64_RELATIVE, + R_X86_64_IRELATIVE, + ): + relocations[relocation.offset] = relocation.addend + self._relocations = relocations + return relocations + + +class FunctionTable: + """Function boundaries from the .eh_frame_hdr binary-search table.""" + + def __init__(self, image: ElfImage): + self.image = image + section = image.sections[".eh_frame_hdr"] + encoded = image.section_bytes(section) + base = section.address + if len(encoded) < 12 or encoded[:4] != bytes((1, 27, 3, 59)): + raise AnalyzerError(".eh_frame_hdr uses an unsupported encoding") + count = struct.unpack_from(" MAX_FUNCTION_TABLE_ENTRIES: + raise AnalyzerError(".eh_frame_hdr table size is invalid") + if len(encoded) < 12 + count * 8: + raise AnalyzerError(".eh_frame_hdr table is truncated") + self.starts = [ + base + struct.unpack_from("= self.starts[i + 1] for i in range(count - 1)): + raise AnalyzerError(".eh_frame_hdr table is not sorted") + text = image.text_section() + self.ends = self.starts[1:] + [text.address + text.size] + + def bounds(self, rva: int) -> tuple[int, int]: + index = bisect.bisect_right(self.starts, rva) - 1 + if index < 0 or self.starts[index] > rva or rva >= self.ends[index]: + raise AnalyzerError( + f"RVA 0x{rva:x} is not covered by .eh_frame_hdr") + return self.starts[index], self.ends[index] + + +class Disassembler: + + def __init__(self, image: ElfImage): + require_capstone() + self.image = image + self.decoder = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) + self.decoder.detail = True + self._cache: dict[int, tuple[Any, ...]] = {} + + def function(self, start: int, end: int) -> tuple[Any, ...]: + cached = self._cache.get(start) + if cached is not None: + return cached + size = end - start + if size <= 0 or size > MAX_FUNCTION_BYTES: + raise AnalyzerError(f"function at 0x{start:x} has an invalid size") + offset = self.image.rva_to_offset(start, size) + code = self.image.bytes_at(offset, size, "function body") + instructions = tuple(self.decoder.disasm(code, start)) + if not instructions or instructions[0].address != start: + raise AnalyzerError(f"cannot decode function at 0x{start:x}") + self._cache[start] = instructions + return instructions + + +def require_capstone() -> None: + if capstone is None or capstone_x86 is None: + raise AnalyzerError( + "Python capstone 5 is required for VR offset analysis") + try: + version = capstone.cs_version() + except (AttributeError, TypeError, ValueError) as error: + raise AnalyzerError( + "Python capstone version cannot be verified") from error + if (not isinstance(version, tuple) or len(version) < 2 or + not all(isinstance(component, int) for component in version[:2]) or + version[0] != 5): + raise AnalyzerError("Python capstone major version 5 is required") + + +def rip_target(instruction: Any) -> Iterator[int]: + for operand in instruction.operands: + if (operand.type == capstone_x86.X86_OP_MEM and + operand.mem.base == capstone_x86.X86_REG_RIP): + yield instruction.address + instruction.size + operand.mem.disp + + +def find_text_refs( + image: ElfImage, + table: FunctionTable, + disasm: Disassembler, + targets: set[int], +) -> dict[int, list[tuple[int, int, int]]]: + """Find RIP-relative references at decoded instruction boundaries. + + Returns: + Target RVAs mapped to (instruction RVA, function start, function end). + """ + text = image.text_section() + data = image.section_bytes(text) + base = text.address + combined = re.compile( + b"(?=" + TEXT_REF_OPCODES.pattern + TEXT_REF_MODRMS.pattern + b".{4})", + re.DOTALL, + ) + candidates: dict[int, set[int]] = {} + for match in combined.finditer(data): + position = match.start() + opcode = data[position] + extra = 1 if opcode in (0x80, 0x83, + 0xC6) else 4 if opcode in (0x81, 0xC7) else 0 + target = (base + position + 6 + extra + + struct.unpack_from(" list[int]: + text = image.text_section() + data = image.section_bytes(text) + base = text.address + hits: list[int] = [] + position = 0 + while True: + position = data.find(pattern, position) + if position < 0: + return hits + hits.append(base + position) + position += 1 + + +def ends_with_ret(instructions: Sequence[Any]) -> bool: + return any( + instruction.mnemonic == "ret" for instruction in instructions[-3:]) + + +def is_state_getter(instructions: Sequence[Any]) -> bool: + """mov reg, rdi; add rsi, 0x14; mov edx, 0x138; call memcpy; ret.""" + if not 6 <= len(instructions) <= 24: + return False + calls = [ + instruction for instruction in instructions + if instruction.mnemonic == "call" and instruction.operands and + instruction.operands[0].type == capstone_x86.X86_OP_IMM + ] + if len(calls) != 1: + return False + if not ends_with_ret(instructions): + return False + has_saved_self = False + has_state_offset = False + has_state_size = False + for instruction in instructions: + operands = instruction.operands + if (instruction.mnemonic == "mov" and len(operands) == 2 and + operands[0].type == capstone_x86.X86_OP_REG and + operands[1].type == capstone_x86.X86_OP_REG and + operands[1].reg == capstone_x86.X86_REG_RDI): + has_saved_self = True + if (instruction.mnemonic == "add" and len(operands) == 2 and + operands[0].type == capstone_x86.X86_OP_REG and + operands[0].reg == capstone_x86.X86_REG_RSI and + operands[1].type == capstone_x86.X86_OP_IMM and + operands[1].imm == STATE_OFFSET): + has_state_offset = True + if (instruction.mnemonic == "mov" and len(operands) == 2 and + operands[0].type == capstone_x86.X86_OP_REG and + operands[1].type == capstone_x86.X86_OP_IMM and + operands[1].imm == STATE_COPY_SIZE): + has_state_size = True + return has_saved_self and has_state_offset and has_state_size + + +def is_eye_getter(instructions: Sequence[Any]) -> bool: + """movsxd rax, esi; shl rax, 4; mov rax, [rdi+rax+0x150]; ret.""" + if len(instructions) != 4: + return False + first, second, third, last = instructions + if first.mnemonic != "movsxd" or len(first.operands) != 2: + return False + if (first.operands[0].type != capstone_x86.X86_OP_REG or + first.operands[0].reg != capstone_x86.X86_REG_RAX or + first.operands[1].type != capstone_x86.X86_OP_REG or + first.operands[1].reg != capstone_x86.X86_REG_ESI): + return False + if (second.mnemonic != "shl" or + second.operands[0].type != capstone_x86.X86_OP_REG or + second.operands[0].reg != capstone_x86.X86_REG_RAX or + second.operands[1].type != capstone_x86.X86_OP_IMM or + second.operands[1].imm != 4): + return False + if third.mnemonic != "mov" or len(third.operands) != 2: + return False + memory = third.operands[1] + if memory.type != capstone_x86.X86_OP_MEM: + return False + if (memory.mem.base != capstone_x86.X86_REG_RDI or + memory.mem.index != capstone_x86.X86_REG_RAX or + memory.mem.disp != FRAMEBUFFER_SLOT_OFFSET): + return False + return last.mnemonic == "ret" + + +def is_eye_initializer(instructions: Sequence[Any], eye_label_rva: int) -> bool: + """The eye-resource initializer references 'VR Eye' and the slot arrays.""" + has_label = any(eye_label_rva in rip_target(instruction) + for instruction in instructions) + if not has_label: + return False + ready_hits = set() + has_framebuffer_lea = False + has_texture_lea = False + has_register_call = False + for instruction in instructions: + operands = instruction.operands + if instruction.mnemonic == "mov" and operands and operands[ + 0].type == capstone_x86.X86_OP_MEM: + if operands[0].size == 1 and operands[ + 0].mem.disp in READY_BYTE_OFFSETS: + ready_hits.add(operands[0].mem.disp) + if instruction.mnemonic == "lea" and len(operands) == 2: + memory = operands[1] + if (memory.type == capstone_x86.X86_OP_MEM and + memory.mem.index == capstone_x86.X86_REG_INVALID): + if memory.mem.disp == FRAMEBUFFER_SLOT_OFFSET: + has_framebuffer_lea = True + if memory.mem.disp == TEXTURE_SLOT_OFFSET: + has_texture_lea = True + if (instruction.mnemonic == "call" and operands and + operands[0].type == capstone_x86.X86_OP_REG): + has_register_call = True + return (len(ready_hits) >= 2 and has_framebuffer_lea and has_texture_lea and + has_register_call) + + +def is_constructor(instructions: Sequence[Any], vtable_rva: int, + device_name_rva: int) -> bool: + """The constructor stores the vtable, the 'DebugDevice' name and type 6.""" + has_vtable = any( + instruction.mnemonic == "lea" and vtable_rva in rip_target(instruction) + for instruction in instructions) + has_name = any(instruction.mnemonic == "lea" and + device_name_rva in rip_target(instruction) + for instruction in instructions) + has_type = False + has_state_size = False + has_direct_call = False + for instruction in instructions: + operands = instruction.operands + if (instruction.mnemonic == "mov" and len(operands) == 2 and + operands[0].type == capstone_x86.X86_OP_MEM and + operands[0].size == 4 and + operands[0].mem.disp == OBJECT_TYPE_OFFSET and + operands[0].mem.index == capstone_x86.X86_REG_INVALID and + operands[1].type == capstone_x86.X86_OP_IMM and + operands[1].imm == OBJECT_TYPE_DEBUG_DEVICE): + has_type = True + if (instruction.mnemonic == "mov" and len(operands) == 2 and + operands[0].type == capstone_x86.X86_OP_REG and + operands[1].type == capstone_x86.X86_OP_IMM and + operands[1].imm == CONSTRUCTOR_MEMSET_SIZE): + has_state_size = True + if (instruction.mnemonic == "call" and operands and + operands[0].type == capstone_x86.X86_OP_IMM): + has_direct_call = True + return (has_vtable and has_name and has_type and has_state_size and + has_direct_call) + + +def extract_framebuffer_slot(instructions: Sequence[Any]) -> int | None: + """Find ``mov rax, [rax+disp]`` ... ``call rax`` on a device vtable.""" + hits: set[int] = set() + for index, instruction in enumerate(instructions): + operands = instruction.operands + if instruction.mnemonic != "mov" or len(operands) != 2: + continue + if operands[0].type != capstone_x86.X86_OP_REG or operands[ + 1].type != capstone_x86.X86_OP_MEM: + continue + memory = operands[1].mem + register = operands[0].reg + if (memory.base != register or + memory.index != capstone_x86.X86_REG_INVALID or + memory.disp < 0x40): + continue + has_vtable_load = any( + previous.mnemonic == "mov" and len(previous.operands) == 2 and + previous.operands[0].type == capstone_x86.X86_OP_REG and + previous.operands[1].type == capstone_x86.X86_OP_MEM and + previous.operands[1].mem.index == capstone_x86.X86_REG_INVALID and + previous.operands[1].mem.disp == 0 and + previous.operands[1].mem.base != capstone_x86.X86_REG_INVALID and + previous.operands[1].mem.base != capstone_x86.X86_REG_RIP + for previous in instructions[max(0, index - 6):index]) + has_register_call = any( + later.mnemonic == "call" and later.operands and + later.operands[0].type == capstone_x86.X86_OP_REG and + later.operands[0].reg == register + for later in instructions[index + 1:index + 16]) + if has_vtable_load and has_register_call: + hits.add(memory.disp) + if len(hits) != 1: + return None + return next(iter(hits)) + + +def extract_registration_storage(instructions: Sequence[Any], + flag_name_rva: int) -> int | None: + """lea rsi, [flag]; lea rdx, [storage]; mov ecx, 1; mov r8d, 4; jmp.""" + storage: int | None = None + for index, instruction in enumerate(instructions): + operands = instruction.operands + if instruction.mnemonic != "lea" or len(operands) != 2: + continue + if operands[0].type != capstone_x86.X86_OP_REG or operands[ + 0].reg != capstone_x86.X86_REG_RSI: + continue + if flag_name_rva not in rip_target(instruction): + continue + for follower in instructions[index + 1:index + 4]: + follower_operands = follower.operands + if (follower.mnemonic == "lea" and len(follower_operands) == 2 and + follower_operands[0].type == capstone_x86.X86_OP_REG and + follower_operands[0].reg == capstone_x86.X86_REG_RDX): + for target in rip_target(follower): + storage = target + if storage is None: + return None + has_flag_scope = False + has_category = False + for instruction in instructions: + operands = instruction.operands + if (instruction.mnemonic == "mov" and len(operands) == 2 and + operands[0].type == capstone_x86.X86_OP_REG and + operands[0].reg == capstone_x86.X86_REG_ECX and + operands[1].type == capstone_x86.X86_OP_IMM and + operands[1].imm == 1): + has_flag_scope = True + if (instruction.mnemonic == "mov" and len(operands) == 2 and + operands[0].type == capstone_x86.X86_OP_REG and + operands[0].reg == capstone_x86.X86_REG_R8D and + operands[1].type == capstone_x86.X86_OP_IMM and + operands[1].imm == 4): + has_category = True + if not (has_flag_scope and has_category): + return None + return storage + + +def find_flag_zeroing(instructions: Sequence[Any], storage_rva: int) -> bool: + """mov byte ptr [rip+storage], 0 in the flag descriptor initializer.""" + for instruction in instructions: + operands = instruction.operands + if (instruction.mnemonic == "mov" and len(operands) == 2 and + operands[0].type == capstone_x86.X86_OP_MEM and + operands[0].size == 1 and + operands[1].type == capstone_x86.X86_OP_IMM and + operands[1].imm == 0): + if storage_rva in rip_target(instruction): + return True + return False + + +def extract_hand_pitch(instructions: Sequence[Any], + flag_name_rva: int) -> tuple[int, int] | None: + """Find hand-pitch storage and its default value in the flag initializer.""" + for index, instruction in enumerate(instructions): + operands = instruction.operands + if (instruction.mnemonic != "mov" or len(operands) != 2 or + operands[0].type != capstone_x86.X86_OP_MEM or + operands[1].type != capstone_x86.X86_OP_IMM): + continue + if operands[0].size != 4: + continue + default = operands[1].imm + if not 0 <= default < 360: + continue + storage = next(iter(rip_target(instruction)), None) + if storage is None: + continue + for follower_index in range(index + 1, min(index + 5, + len(instructions))): + follower = instructions[follower_index] + follower_operands = follower.operands + if (follower.mnemonic != "lea" or len(follower_operands) != 2 or + follower_operands[0].type != capstone_x86.X86_OP_REG): + continue + if flag_name_rva not in rip_target(follower): + continue + register = follower_operands[0].reg + if follower_index + 1 >= len(instructions): + continue + store = instructions[follower_index + 1] + store_operands = store.operands + if (store.mnemonic == "mov" and len(store_operands) == 2 and + store_operands[0].type == capstone_x86.X86_OP_MEM and + store_operands[1].type == capstone_x86.X86_OP_REG and + store_operands[1].reg == register): + name_slot = next(iter(rip_target(store)), None) + if name_slot == storage + 8: + return storage, default + return None + + +def match_pointer_getter(instructions: Sequence[Any]) -> bool: + """World-space pointer getter: UserCFrame copy + workspace/camera path.""" + has_selector = False + has_scale = False + has_stride = False + usercframe_loads = 0 + has_workspace_handoff = False + has_camera_slot_call = False + has_head_scale = False + for instruction in instructions: + operands = instruction.operands + if instruction.mnemonic == "movsxd" and len(operands) == 2: + if (operands[1].type == capstone_x86.X86_OP_MEM and + operands[1].mem.base == capstone_x86.X86_REG_RSI and + operands[1].mem.index == capstone_x86.X86_REG_INVALID and + operands[1].mem.disp == USERCFRAME_SELECTOR_OFFSET): + has_selector = True + if instruction.mnemonic == "lea" and len(operands) == 2: + memory = operands[1] + if (memory.type == capstone_x86.X86_OP_MEM and + operands[0].type == capstone_x86.X86_OP_REG and + operands[0].reg == capstone_x86.X86_REG_RAX and + memory.mem.base == capstone_x86.X86_REG_RAX and + memory.mem.index == capstone_x86.X86_REG_RAX and + memory.mem.scale == 2): + has_scale = True + if instruction.mnemonic == "shl" and len(operands) == 2: + if (operands[0].type == capstone_x86.X86_OP_REG and + operands[0].reg == capstone_x86.X86_REG_RAX and + operands[1].type == capstone_x86.X86_OP_IMM and + operands[1].imm == 4): + has_stride = True + if instruction.mnemonic in ("movups", "movupd", "movaps", + "movapd") and len(operands) == 2: + memory = operands[1] + if memory.type == capstone_x86.X86_OP_MEM and ( + memory.mem.base == capstone_x86.X86_REG_RSI and + memory.mem.index == capstone_x86.X86_REG_RAX and + memory.mem.disp in ( + USERCFRAME_BASE_OFFSET, + USERCFRAME_BASE_OFFSET + 0x10, + USERCFRAME_BASE_OFFSET + 0x20, + )): + usercframe_loads += 1 + if instruction.mnemonic == "movss" and len(operands) == 2: + memory = operands[1] + if (memory.type == capstone_x86.X86_OP_MEM and + memory.mem.index == capstone_x86.X86_REG_INVALID and + memory.mem.disp == CAMERA_HEAD_SCALE_OFFSET): + has_head_scale = True + for index, instruction in enumerate(instructions): + operands = instruction.operands + if (instruction.mnemonic == "mov" and len(operands) == 2 and + operands[0].type == capstone_x86.X86_OP_REG and + operands[0].reg == capstone_x86.X86_REG_RDI and + operands[1].type == capstone_x86.X86_OP_REG and + operands[1].reg == capstone_x86.X86_REG_RSI): + for follower in instructions[index + 1:index + 3]: + if (follower.mnemonic == "call" and follower.operands and + follower.operands[0].type == capstone_x86.X86_OP_IMM): + has_workspace_handoff = True + if (instruction.mnemonic == "mov" and len(operands) == 2 and + operands[0].type == capstone_x86.X86_OP_REG and + operands[0].reg == capstone_x86.X86_REG_RCX and + operands[1].type == capstone_x86.X86_OP_MEM and + operands[1].mem.base == capstone_x86.X86_REG_RAX and + operands[1].mem.disp == 0): + for follower_index in range(index + 1, + min(index + 4, len(instructions))): + follower = instructions[follower_index] + if (follower.mnemonic == "mov" and + len(follower.operands) == 2 and + follower.operands[0].type == capstone_x86.X86_OP_REG and + follower.operands[0].reg == capstone_x86.X86_REG_RDI and + follower.operands[1].type == capstone_x86.X86_OP_REG and + follower.operands[1].reg == capstone_x86.X86_REG_RAX): + for caller in instructions[follower_index + + 1:follower_index + 4]: + if (caller.mnemonic == "call" and caller.operands and + caller.operands[0].type + == capstone_x86.X86_OP_MEM and + caller.operands[0].mem.base + == capstone_x86.X86_REG_RCX and + caller.operands[0].mem.disp > 0x100): + has_camera_slot_call = True + return (has_selector and has_scale and has_stride and + usercframe_loads >= 3 and has_workspace_handoff and + has_camera_slot_call and has_head_scale) + + +def extract_pointer_getter_anchors(instructions: Sequence[Any], + image: ElfImage) -> tuple[int, int]: + """Return (workspace getter RVA, camera vtable slot offset).""" + workspace_getter = None + camera_slot = None + for index, instruction in enumerate(instructions): + operands = instruction.operands + if (instruction.mnemonic == "mov" and len(operands) == 2 and + operands[0].type == capstone_x86.X86_OP_REG and + operands[0].reg == capstone_x86.X86_REG_RDI and + operands[1].type == capstone_x86.X86_OP_REG and + operands[1].reg == capstone_x86.X86_REG_RSI): + for follower in instructions[index + 1:index + 3]: + if (follower.mnemonic == "call" and follower.operands and + follower.operands[0].type == capstone_x86.X86_OP_IMM): + target = follower.operands[0].imm + try: + image.require_code_rva(target) + except AnalyzerError: + continue + workspace_getter = target + if (instruction.mnemonic == "mov" and len(operands) == 2 and + operands[0].type == capstone_x86.X86_OP_REG and + operands[0].reg == capstone_x86.X86_REG_RCX and + operands[1].type == capstone_x86.X86_OP_MEM and + operands[1].mem.base == capstone_x86.X86_REG_RAX and + operands[1].mem.disp == 0): + for follower_index in range(index + 1, + min(index + 4, len(instructions))): + follower = instructions[follower_index] + if (follower.mnemonic == "mov" and + len(follower.operands) == 2 and + follower.operands[0].type == capstone_x86.X86_OP_REG and + follower.operands[0].reg == capstone_x86.X86_REG_RDI and + follower.operands[1].type == capstone_x86.X86_OP_REG and + follower.operands[1].reg == capstone_x86.X86_REG_RAX): + for caller in instructions[follower_index + + 1:follower_index + 4]: + if (caller.mnemonic == "call" and caller.operands and + caller.operands[0].type + == capstone_x86.X86_OP_MEM and + caller.operands[0].mem.base + == capstone_x86.X86_REG_RCX and + caller.operands[0].mem.disp > 0x100): + camera_slot = caller.operands[0].mem.disp + if workspace_getter is None or camera_slot is None: + raise AnalyzerError("world-space pointer getter contract matched" + " but its workspace/camera" + " anchors could not be extracted") + return workspace_getter, camera_slot + + +def vtable_slots( + image: ElfImage, + relocations: dict[int, int], + vtable_rva: int, + table: "FunctionTable", +) -> dict[int, int]: + """Collect vtable slots whose targets are covered by the function table. + + Segment flags alone cannot distinguish .text from .rodata in these builds. + """ + slots: dict[int, int] = {} + misses = 0 + for slot in range(0, MAX_VTABLE_SCAN_BYTES, 8): + target = relocations.get(vtable_rva + slot) + valid = False + if target is not None: + try: + image.require_code_rva(target) + table.bounds(target) + valid = True + except AnalyzerError: + valid = False + if valid: + slots[slot] = target + misses = 0 + else: + misses += 1 + if misses >= 8: + break + if len(slots) < 16: + raise AnalyzerError( + f"vtable candidate at 0x{vtable_rva:x} does not look like a vtable") + return slots + + +def find_vr_offsets(elf_path: pathlib.Path) -> dict[str, Any]: + """Analyze VR entry points without executing the ELF. + + Args: + elf_path: Path to an x86-64 libroblox.so. + + Returns: + ELF identity, bridge offsets, evidence, and analysis warnings. + + Raises: + AnalyzerError: The ELF or a required machine-code contract is invalid. + """ + evidence: list[str] = [] + warnings: list[str] = [] + + with ElfImage(elf_path) as image: + relocations = image.relative_relocations() + table = FunctionTable(image) + disasm = Disassembler(image) + + # Itanium RTTI: mangled name -> typeinfo -> primary vtable. + rtti_name_rva = image.find_alloc_string(RTTI_TYPE_NAME) + name_slots = sorted(offset for offset, addend in relocations.items() + if addend == rtti_name_rva and offset % 8 == 0) + if not name_slots: + raise AnalyzerError("no APS2 relative relocation references" + " the DebugDeviceVR RTTI name") + vtable_result: tuple[int, dict[int, int]] | None = None + typeinfo_result = 0 + for name_slot in name_slots: + typeinfo = name_slot - 8 + for reference in sorted( + offset for offset, addend in relocations.items() + if addend == typeinfo and offset % 8 == 0): + vtable = reference + 8 + try: + image.require_relro_rva(vtable, 0xE0) + slots = vtable_slots(image, relocations, vtable, table) + except AnalyzerError: + continue + vtable_result = (vtable, slots) + typeinfo_result = typeinfo + break + if vtable_result is not None: + break + if vtable_result is None: + raise AnalyzerError( + "DebugDeviceVR vtable could not be resolved from its typeinfo") + vtable, slots = vtable_result + offset_to_top = image.bytes_at(image.rva_to_offset(vtable - 16, 8), 8, + "vtable header") + if offset_to_top != b"\0" * 8: + raise AnalyzerError( + f"vtable at 0x{vtable:x} is not a primary vtable (offset-to-top" + " is nonzero)") + evidence.append( + f"RTTI: mangled name {RTTI_TYPE_NAME.decode('ascii')!r} at" + f" 0x{rtti_name_rva:x} -> typeinfo 0x{typeinfo_result:x} -> vtable" + f" 0x{vtable:x} through APS2 relative relocations") + + state_matches = [ + (slot, target) + for slot, target in slots.items() + if is_state_getter(disasm.function(*table.bounds(target))) + ] + eye_matches = [(slot, target) + for slot, target in slots.items() + if is_eye_getter(disasm.function(*table.bounds(target)))] + eye_label_rva = image.find_alloc_string(EYE_RESOURCE_LABEL) + initializer_matches = [ + (slot, target) + for slot, target in slots.items() + if is_eye_initializer(disasm.function( + *table.bounds(target)), eye_label_rva) + ] + for label, matches in ( + ("state getter", state_matches), + ("eye getter", eye_matches), + ("eye initializer", initializer_matches), + ): + if not matches: + raise AnalyzerError( + f"no vtable slot satisfies the {label} contract") + if len(matches) > 1: + listed = ", ".join(f"vtable+0x{slot:x} -> 0x{target:x}" + for slot, target in matches) + raise AnalyzerError( + f"ambiguous {label} contract matches: {listed}") + state_slot, state_getter = state_matches[0] + eye_slot, eye_getter = eye_matches[0] + initializer_slot, eye_initializer = initializer_matches[0] + for label, slot, expected in ( + ("state getter", state_slot, EXPECTED_STATE_GETTER_SLOT), + ("eye getter", eye_slot, EXPECTED_EYE_GETTER_SLOT), + ("eye initializer", initializer_slot, + EXPECTED_EYE_INITIALIZER_SLOT), + ): + if slot != expected: + warnings.append(f"{label} moved from vtable+0x{expected:x}" + f" to vtable+0x{slot:x};" + " update the bridge slot constants") + evidence.append( + f"state getter: sret memcpy of 0x{STATE_COPY_SIZE:x} bytes from" + f" +0x{STATE_OFFSET:x} at vtable+0x{state_slot:x}" + f" -> 0x{state_getter:x}") + evidence.append("eye getter: movsxd/shl/mov" + f" [rdi+rax+0x{FRAMEBUFFER_SLOT_OFFSET:x}] at" + f" vtable+0x{eye_slot:x} -> 0x{eye_getter:x}") + evidence.append( + f"eye initializer: {EYE_RESOURCE_LABEL.decode('ascii')!r}" + " label, ready" + f" bytes {[f'0x{x:x}' for x in READY_BYTE_OFFSETS]}, slot arrays" + f" +0x{FRAMEBUFFER_SLOT_OFFSET:x}/+0x{TEXTURE_SLOT_OFFSET:x} at" + f" vtable+0x{initializer_slot:x} -> 0x{eye_initializer:x}") + + haptic_matches = [] + for slot, target in slots.items(): + start, end = table.bounds(target) + if end - start == 1: + body = image.bytes_at(image.rva_to_offset(target, 1), 1, + "haptic sink") + if body == b"\xc3": + haptic_matches.append((slot, target)) + if not haptic_matches: + raise AnalyzerError( + "no vtable slot points at a 1-byte ret haptic sink") + haptic_slot, haptic_sink = min(haptic_matches) + if haptic_slot != EXPECTED_HAPTIC_SLOT: + warnings.append( + f"haptic sink moved from vtable+0x{EXPECTED_HAPTIC_SLOT:x} to" + f" vtable+0x{haptic_slot:x}") + evidence.append( + f"haptic sink: 1-byte ret stub at vtable+0x{haptic_slot:x} ->" + f" 0x{haptic_sink:x}") + + device_name_rva = image.find_alloc_string(DEVICE_OBJECT_NAME) + vtable_refs = find_text_refs(image, table, disasm, {vtable}) + constructor_candidates: dict[int, int] = {} + for _target, refs in vtable_refs.items(): + for _address, start, end in refs: + instructions = disasm.function(start, end) + if is_constructor(instructions, vtable, device_name_rva): + constructor_candidates[start] = end + if not constructor_candidates: + raise AnalyzerError( + "no function satisfies the DebugDeviceVR constructor contract") + if len(constructor_candidates) > 1: + listed = ", ".join( + f"0x{start:x}" for start in constructor_candidates) + raise AnalyzerError( + f"ambiguous constructor contract matches: {listed}") + constructor = next(iter(constructor_candidates)) + if constructor in slots.values(): + raise AnalyzerError( + "constructor candidate is itself a vtable slot target" + " (destructor?)") + evidence.append( + f"constructor: vtable store, {DEVICE_OBJECT_NAME.decode('ascii')!r}" + f" name at +0x8, type {OBJECT_TYPE_DEBUG_DEVICE} at" + f" +0x{OBJECT_TYPE_OFFSET:x}," + f" memset 0x{CONSTRUCTOR_MEMSET_SIZE:x} of" + f" +0x{STATE_OFFSET:x} -> 0x{constructor:x}") + + initializer_instructions = disasm.function( + *table.bounds(eye_initializer)) + framebuffer_slot = extract_framebuffer_slot(initializer_instructions) + if framebuffer_slot is None: + raise AnalyzerError( + "eye initializer does not contain a unique device vtable" + " framebuffer call") + evidence.append( + f"device createFramebuffer: mov rax,[rax+0x{framebuffer_slot:x}];" + " call rax inside the eye initializer") + + emulator_name_rva = image.find_alloc_string(EMULATOR_FLAG_NAME) + flag_refs = find_text_refs(image, table, disasm, {emulator_name_rva}) + flag_functions: dict[int, int] = {} + for _target, refs in flag_refs.items(): + for _address, start, end in refs: + flag_functions[start] = end + registration_storage = None + for start, end in sorted(flag_functions.items()): + storage = extract_registration_storage(disasm.function(start, end), + emulator_name_rva) + if storage is not None: + if (registration_storage is not None and + storage != registration_storage): + raise AnalyzerError("ambiguous emulator flag storage:" + f" 0x{registration_storage:x}" + f" and 0x{storage:x}") + registration_storage = storage + if registration_storage is None: + raise AnalyzerError( + "DebugEnableVREmulator registration contract was not found") + image.require_writable_rva(registration_storage, 1) + zeroing_seen = any( + find_flag_zeroing(disasm.function(start, end), registration_storage) + for start, end in flag_functions.items()) + if zeroing_seen: + evidence.append( + "emulator flag: registration lea rsi/rdx contract plus init" + f" byte-store cross-check -> 0x{registration_storage:x}") + else: + warnings.append("emulator flag storage derived from" + " the registration contract only;" + " the zero-initializing writer was not found") + + hand_pitch_name_rva = image.find_alloc_string(HAND_PITCH_FLAG_NAME) + pitch_refs = find_text_refs(image, table, disasm, {hand_pitch_name_rva}) + pitch_functions: dict[int, int] = {} + for _target, refs in pitch_refs.items(): + for _address, start, end in refs: + pitch_functions[start] = end + hand_pitch = None + for start, end in sorted(pitch_functions.items()): + match = extract_hand_pitch(disasm.function(start, end), + hand_pitch_name_rva) + if match is not None: + if hand_pitch is not None and match != hand_pitch: + raise AnalyzerError( + "ambiguous NewVRSystemHandPitch storage:" + f" 0x{hand_pitch[0]:x}" + f" and 0x{match[0]:x}") + hand_pitch = match + if hand_pitch is None: + raise AnalyzerError( + "NewVRSystemHandPitch storage contract was not found") + hand_pitch_storage, hand_pitch_default = hand_pitch + image.require_writable_rva(hand_pitch_storage, 4) + evidence.append( + f"hand pitch: dword default {hand_pitch_default} degrees with name" + f" pointer at storage+8 -> 0x{hand_pitch_storage:x}") + + selector_hits = scan_text_bytes(image, POINTER_GETTER_SELECTOR_PATTERN) + movups_hits = scan_text_bytes(image, POINTER_GETTER_MOVUPS_PATTERN) + getter_candidates: set[int] = set() + for hit in selector_hits + movups_hits: + try: + start, _end = table.bounds(hit) + except AnalyzerError: + continue + getter_candidates.add(start) + getter_matches: list[tuple[int, tuple[Any, ...]]] = [] + for start in sorted(getter_candidates): + instructions = disasm.function(*table.bounds(start)) + if match_pointer_getter(instructions): + getter_matches.append((start, instructions)) + if not getter_matches: + raise AnalyzerError( + "no function satisfies the world-space pointer getter contract") + if len(getter_matches) > 1: + listed = ", ".join(f"0x{start:x}" for start, _ in getter_matches) + raise AnalyzerError(f"ambiguous pointer getter matches: {listed}") + pointer_getter, getter_instructions = getter_matches[0] + workspace_getter, camera_slot = extract_pointer_getter_anchors( + getter_instructions, image) + evidence.append("pointer getter: UserCFrame selector" + f" +0x{USERCFRAME_SELECTOR_OFFSET:x}," + f" base +0x{USERCFRAME_BASE_OFFSET:x}, stride" + f" 0x{USERCFRAME_STRIDE:x}," + f" HeadScale +0x{CAMERA_HEAD_SCALE_OFFSET:x}" + f" -> 0x{pointer_getter:x}") + evidence.append(f"workspace getter 0x{workspace_getter:x}" + " via mov rdi,rsi + direct call;" + f" camera via call [rcx+0x{camera_slot:x}]") + + return { + "schema_version": 1, + "tool": "find_roblox_vr_offsets", + "elf": { + "path": str(elf_path), + "size": image.file_size, + "build_id": image.build_id, + "sha256": image.sha256, + }, + "vr_debug_device_bridge": { + "vtable_rva": + f"0x{vtable:x}", + "constructor_rva": + f"0x{constructor:x}", + "state_getter_rva": + f"0x{state_getter:x}", + "eye_getter_rva": + f"0x{eye_getter:x}", + "eye_initializer_rva": + f"0x{eye_initializer:x}", + "emulator_flag_storage_rva": + f"0x{registration_storage:x}", + "device_create_framebuffer_vtable_offset": + f"0x{framebuffer_slot:x}", + }, + "vr_bridge_supplement": { + "state_getter_vtable_slot_offset": + f"0x{state_slot:x}", + "eye_getter_vtable_slot_offset": + f"0x{eye_slot:x}", + "eye_initializer_vtable_slot_offset": + f"0x{initializer_slot:x}", + "haptic_sink_rva": + f"0x{haptic_sink:x}", + "haptic_sink_vtable_slot_offset": + f"0x{haptic_slot:x}", + "world_pointer_getter_rva": + f"0x{pointer_getter:x}", + "workspace_getter_rva": + f"0x{workspace_getter:x}", + "workspace_camera_vtable_offset": + f"0x{camera_slot:x}", + "new_vr_system_hand_pitch_storage_rva": + f"0x{hand_pitch_storage:x}", + "new_vr_system_hand_pitch_default_degrees": + hand_pitch_default, + }, + "evidence": evidence, + "warnings": warnings, + } + + +def write_json_atomic(path: pathlib.Path, payload: dict[str, Any]) -> None: + encoded = json.dumps(payload, indent=2) + "\n" + if len(encoded.encode("utf-8")) > MAX_JSON_OUTPUT_BYTES: + raise AnalyzerError("analysis result exceeds the output size limit") + parent = path.parent + try: + handle = tempfile.NamedTemporaryFile("w", + encoding="utf-8", + dir=parent, + prefix=path.name, + delete=False) + with handle: + handle.write(encoded) + handle.flush() + os.fsync(handle.fileno()) + os.replace(handle.name, path) + except OSError as error: + raise AnalyzerError(f"cannot write output file: {path}") from error + + +def parse_arguments(argv: Sequence[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog="find-roblox-vr-offsets", + description=( + "Extract and validate the native VR bridge offsets from an x86-64" + " Roblox libroblox.so payload."), + ) + parser.add_argument( + "--elf", + required=True, + type=pathlib.Path, + help="path to the libroblox.so ELF image to analyze", + ) + parser.add_argument( + "--output", + type=pathlib.Path, + default=None, + help="optional path for the JSON result (written atomically)", + ) + parser.add_argument( + "--quiet", + action="store_true", + help="do not echo the JSON result to stdout when --output is given", + ) + return parser.parse_args(list(argv)) + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = parse_arguments(sys.argv[1:] if argv is None else argv) + try: + result = find_vr_offsets(arguments.elf) + if arguments.output is not None: + write_json_atomic(arguments.output, result) + if not arguments.quiet: + print(json.dumps(result, indent=2)) + else: + print(json.dumps(result, indent=2)) + except AnalyzerError as error: + print(f"find-roblox-vr-offsets: error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/audio/native_input_capture.h b/src/audio/native_input_capture.h index bfb04b52..4a916fa0 100644 --- a/src/audio/native_input_capture.h +++ b/src/audio/native_input_capture.h @@ -15,12 +15,7 @@ #include "mocktail/audio/sdl_audio_capture.h" -// Port of main's native input capture (537828a "Fix vc") to Build 2998. -// Main pins the voice sink vtable, its PCM entry point and the shared_ptr -// destructor to 2.738 RVAs. Here the sink is dispatched through its own -// vtable, which is validated against the guest image first, and the -// shared_ptr is released through the libc++ control-block ABI, so no -// further 2998 RVAs are needed. +// Sink calls and shared ownership use the validated guest vtables. namespace mocktail::audio { class NativeInputCapture; diff --git a/src/audio/roblox_output_device_bridge.cc b/src/audio/roblox_output_device_bridge.cc index 8ae9e9e4..3eef5501 100644 --- a/src/audio/roblox_output_device_bridge.cc +++ b/src/audio/roblox_output_device_bridge.cc @@ -30,8 +30,6 @@ namespace { constexpr std::size_t kOutputCountVtableIndex = 5; constexpr std::size_t kOutputInfoVtableIndex = 6; -constexpr std::size_t kCurrentOutputVtableIndex = 7; -constexpr std::size_t kSelectOutputVtableIndex = 17; constexpr std::size_t kGuestStringSize = 24; constexpr std::size_t kGuestDeviceInfoSize = kGuestStringSize * 2 + 1; constexpr std::size_t kStringConstructorContractSize = 24; @@ -39,22 +37,15 @@ constexpr std::size_t kMaximumMenuDevices = 128; constexpr std::size_t kMaximumDeviceNameBytes = 512; constexpr std::size_t kFirstPatchedVtableIndex = 4; -// Microphone capture on Build 2998 (Roblox 2.736.1408 x86_64). Roblox 2.738 -// inserted device-list methods at slots 7 and 11; main verified its capture -// methods at {4, 13..17}, between the input current (12) and select (18) -// slots, so 2998 keeps them at {4, 11..15}, between input current (10) and -// select (16). The input slots, with main's input RVAs for this image, anchor -// that layout before anything is patched. -constexpr std::string_view kCaptureBuildId = - "ade08266c67aee88ec9c1d00902150e1684dad3a"; -constexpr std::uintptr_t kCaptureVtableRva = 0x6c58040; -// Format, latency, start, stop, poll, recording. -constexpr std::array kCaptureSlots{4, 11, 12, 13, 14, 15}; -constexpr std::array kInputAnchorSlots{8, 9, 10, 16}; -constexpr std::array kInputAnchorRvas{ - 0x32d12dc, 0x32d12ea, 0x32d13a6, 0x32d0a56}; -// On 2.738 the capture methods lie within 0x1400 bytes of the input methods. -constexpr std::uintptr_t kCaptureCodeWindow = 0x4000; +std::size_t CurrentOutputVtableIndex( + const compat::FmodOutputDeviceBridgeProfile& profile) { + return profile.vtable_layout_version == 2 ? 8 : 7; +} + +std::size_t SelectOutputVtableIndex( + const compat::FmodOutputDeviceBridgeProfile& profile) { + return profile.vtable_layout_version == 2 ? 19 : 17; +} std::atomic g_active_bridge{nullptr}; @@ -114,7 +105,8 @@ bool IsRelroImageRange(std::uintptr_t image_base, std::uintptr_t rva, return IsProgramHeaderRange(image_base, rva, size, PT_GNU_RELRO, 0); } -bool SetVtableWritable(std::uintptr_t* vtable, bool writable) { +bool SetVtableWritable(std::uintptr_t* vtable, std::size_t last_slot, + bool writable) { if (vtable == nullptr) { return false; } @@ -126,7 +118,7 @@ bool SetVtableWritable(std::uintptr_t* vtable, bool writable) { const std::uintptr_t first = reinterpret_cast(vtable + kFirstPatchedVtableIndex); const std::uintptr_t last = - reinterpret_cast(vtable + kSelectOutputVtableIndex + 1); + reinterpret_cast(vtable + last_slot + 1); if (last <= first) { return false; } @@ -150,6 +142,48 @@ std::uintptr_t FunctionAddress(Function function) { namespace internal { +const FmodInputCaptureProfile* FindFmodInputCaptureProfile( + const compat::BuildProfile& profile) { + static constexpr FmodInputCaptureProfile kProfiles[] = { + {"ade08266c67aee88ec9c1d00902150e1684dad3a", + 0x6c58040, + 1, + {4, 11, 12, 13, 14, 15}, + {0x32d0e9a, 0x32d0a34, 0x32d16a8, 0x32d1f84, 0x32d056c, 0x32d26ac}}, + {"5f0704edd9064f566ee3d6df2bd2fabbcc709f03", + 0x6cd3ce0, + 2, + {4, 13, 14, 15, 16, 17}, + {0x320c630, 0x320bcf6, 0x320d4b4, 0x320dd94, 0x320b844, 0x320e49a}}, + }; + if (!profile.allow_host_abi_bridges || !profile.fmod_output_device_bridge) { + return nullptr; + } + const auto& output = *profile.fmod_output_device_bridge; + for (const auto& capture : kProfiles) { + if (capture.build_id == profile.elf_build_id && + capture.vtable_rva == output.vtable_rva && + capture.vtable_layout_version == output.vtable_layout_version) { + return &capture; + } + } + return nullptr; +} + +bool HasExpectedFmodInputCaptureVtable(const std::uintptr_t* vtable, + std::uintptr_t image_base, + const FmodInputCaptureProfile& profile) { + if (vtable == nullptr || image_base == 0) return false; + for (std::size_t i = 0; i < profile.slots.size(); ++i) { + if (profile.method_rvas[i] > + std::numeric_limits::max() - image_base || + vtable[profile.slots[i]] != image_base + profile.method_rvas[i]) { + return false; + } + } + return true; +} + bool HasExpectedFmodStringConstructorContract(const std::uint8_t* code, std::size_t size) { constexpr std::array @@ -165,14 +199,16 @@ bool HasExpectedFmodStringConstructorContract(const std::uint8_t* code, bool HasExpectedFmodOutputDeviceVtable( const std::uintptr_t* vtable, std::uintptr_t image_base, const compat::FmodOutputDeviceBridgeProfile& profile) { - return vtable != nullptr && image_base != 0 && + return (profile.vtable_layout_version == 1 || + profile.vtable_layout_version == 2) && + vtable != nullptr && image_base != 0 && vtable[kOutputCountVtableIndex] == image_base + profile.count_method_rva && vtable[kOutputInfoVtableIndex] == image_base + profile.info_method_rva && - vtable[kCurrentOutputVtableIndex] == + vtable[CurrentOutputVtableIndex(profile)] == image_base + profile.current_method_rva && - vtable[kSelectOutputVtableIndex] == + vtable[SelectOutputVtableIndex(profile)] == image_base + profile.select_method_rva; } @@ -209,6 +245,10 @@ Status RobloxOutputDeviceBridge::Install(const compat::BuildProfile& profile) { if (!profile.fmod_output_device_bridge.has_value()) { return Status::Ok(); } + if (profile.fmod_output_device_bridge->vtable_layout_version != 1 && + profile.fmod_output_device_bridge->vtable_layout_version != 2) { + return FailedPrecondition("unsupported FMOD output-device vtable layout"); + } if (!profile.allow_host_abi_bridges) { return FailedPrecondition( "FMOD output-device profile requires host ABI bridges"); @@ -223,10 +263,10 @@ Status RobloxOutputDeviceBridge::Install(const compat::BuildProfile& profile) { } profile_ = *profile.fmod_output_device_bridge; const char* native_capture = std::getenv("MOCKTAIL_NATIVE_INPUT_CAPTURE"); - capture_profile_supported_ = - profile.elf_build_id == kCaptureBuildId && - profile_.vtable_rva == kCaptureVtableRva && - !(native_capture != nullptr && std::strcmp(native_capture, "0") == 0); + capture_profile_ = + native_capture != nullptr && std::strcmp(native_capture, "0") == 0 + ? nullptr + : internal::FindFmodInputCaptureProfile(profile); if (!linker::SetAndroidLibraryLoadObserver(&ObserveAndroidLibrary, this)) { g_active_bridge.store(nullptr, std::memory_order_release); profile_ = {}; @@ -264,6 +304,7 @@ void RobloxOutputDeviceBridge::Shutdown() { (void)g_active_bridge.compare_exchange_strong( expected, nullptr, std::memory_order_acq_rel, std::memory_order_acquire); profile_ = {}; + capture_profile_ = nullptr; installed_ = false; } @@ -372,13 +413,13 @@ Status RobloxOutputDeviceBridge::Activate(std::uintptr_t image_base) { Status RobloxOutputDeviceBridge::PatchVtableLocked() { if (profile_.vtable_rva > std::numeric_limits::max() - - (kSelectOutputVtableIndex + 1) * sizeof(std::uintptr_t)) { + (SelectOutputVtableIndex(profile_) + 1) * sizeof(std::uintptr_t)) { return FailedPrecondition("FMOD output-device vtable range overflows"); } const std::uintptr_t first_slot_rva = profile_.vtable_rva + kFirstPatchedVtableIndex * sizeof(std::uintptr_t); const std::size_t slot_range_size = - (kSelectOutputVtableIndex - kFirstPatchedVtableIndex + 1) * + (SelectOutputVtableIndex(profile_) - kFirstPatchedVtableIndex + 1) * sizeof(std::uintptr_t); if (!IsRelroImageRange(library_base_, first_slot_rva, slot_range_size) || !IsExecutableImageRange(library_base_, profile_.string_constructor_rva, @@ -403,7 +444,7 @@ Status RobloxOutputDeviceBridge::PatchVtableLocked() { "FmodAudioDevice string ABI contract validation failed"); } - if (capture_profile_supported_) { + if (capture_profile_ != nullptr) { // The output routes do not depend on capture; keep them if it is refused. const Status capture_status = PrepareCaptureLocked(); if (!capture_status.ok()) { @@ -415,10 +456,10 @@ Status RobloxOutputDeviceBridge::PatchVtableLocked() { original_methods_ = { vtable_[kOutputCountVtableIndex], vtable_[kOutputInfoVtableIndex], - vtable_[kCurrentOutputVtableIndex], - vtable_[kSelectOutputVtableIndex], + vtable_[CurrentOutputVtableIndex(profile_)], + vtable_[SelectOutputVtableIndex(profile_)], }; - if (!SetVtableWritable(vtable_, true)) { + if (!SetVtableWritable(vtable_, SelectOutputVtableIndex(profile_), true)) { capture_.reset(); return Status::Error(StatusCode::kPlatformError, "cannot make FmodAudioDevice vtable writable"); @@ -428,9 +469,9 @@ Status RobloxOutputDeviceBridge::PatchVtableLocked() { FunctionAddress(&GetOutputDeviceCount), __ATOMIC_RELEASE); __atomic_store_n(&vtable_[kOutputInfoVtableIndex], FunctionAddress(&GetOutputDeviceInfo), __ATOMIC_RELEASE); - __atomic_store_n(&vtable_[kCurrentOutputVtableIndex], + __atomic_store_n(&vtable_[CurrentOutputVtableIndex(profile_)], FunctionAddress(&GetCurrentOutputDevice), __ATOMIC_RELEASE); - __atomic_store_n(&vtable_[kSelectOutputVtableIndex], + __atomic_store_n(&vtable_[SelectOutputVtableIndex(profile_)], FunctionAddress(&SetCurrentOutputDevice), __ATOMIC_RELEASE); if (capture_) { g_native_capture.store(capture_.get(), std::memory_order_release); @@ -441,74 +482,55 @@ Status RobloxOutputDeviceBridge::PatchVtableLocked() { FunctionAddress(&NativeInputCapture::Stop), FunctionAddress(&NativeInputCapture::Poll), FunctionAddress(&NativeInputCapture::Recording)}; - for (std::size_t i = 0; i < kCaptureSlots.size(); ++i) - __atomic_store_n(&vtable_[kCaptureSlots[i]], methods[i], __ATOMIC_RELEASE); + for (std::size_t i = 0; i < capture_profile_->slots.size(); ++i) + __atomic_store_n(&vtable_[capture_profile_->slots[i]], methods[i], + __ATOMIC_RELEASE); } - if (!SetVtableWritable(vtable_, false)) { + if (!SetVtableWritable(vtable_, SelectOutputVtableIndex(profile_), false)) { __atomic_store_n(&vtable_[kOutputCountVtableIndex], original_methods_[0], __ATOMIC_RELEASE); __atomic_store_n(&vtable_[kOutputInfoVtableIndex], original_methods_[1], __ATOMIC_RELEASE); - __atomic_store_n(&vtable_[kCurrentOutputVtableIndex], original_methods_[2], - __ATOMIC_RELEASE); - __atomic_store_n(&vtable_[kSelectOutputVtableIndex], original_methods_[3], - __ATOMIC_RELEASE); + __atomic_store_n(&vtable_[CurrentOutputVtableIndex(profile_)], + original_methods_[2], __ATOMIC_RELEASE); + __atomic_store_n(&vtable_[SelectOutputVtableIndex(profile_)], + original_methods_[3], __ATOMIC_RELEASE); if (capture_) { - for (std::size_t i = 0; i < kCaptureSlots.size(); ++i) - __atomic_store_n(&vtable_[kCaptureSlots[i]], + for (std::size_t i = 0; i < capture_profile_->slots.size(); ++i) + __atomic_store_n(&vtable_[capture_profile_->slots[i]], original_capture_methods_[i], __ATOMIC_RELEASE); NativeInputCapture::Retire(std::move(capture_)); } - (void)SetVtableWritable(vtable_, false); + (void)SetVtableWritable(vtable_, SelectOutputVtableIndex(profile_), false); return Status::Error(StatusCode::kPlatformError, "cannot restore FmodAudioDevice RELRO protection"); } if (capture_) { std::fprintf(stderr, " [audio-input] native capture armed for Build ID %s " - "(slots 4/11-15 = %#" PRIxPTR "/%#" PRIxPTR "/%#" PRIxPTR - "/%#" PRIxPTR "/%#" PRIxPTR "/%#" PRIxPTR ")\n", - kCaptureBuildId.data(), - original_capture_methods_[0] - library_base_, - original_capture_methods_[1] - library_base_, - original_capture_methods_[2] - library_base_, - original_capture_methods_[3] - library_base_, - original_capture_methods_[4] - library_base_, - original_capture_methods_[5] - library_base_); + "(format/latency/start/stop/poll/recording slots " + "%zu/%zu/%zu/%zu/%zu/%zu)\n", + capture_profile_->build_id.data(), capture_profile_->slots[0], + capture_profile_->slots[1], capture_profile_->slots[2], + capture_profile_->slots[3], capture_profile_->slots[4], + capture_profile_->slots[5]); } return Status::Ok(); } Status RobloxOutputDeviceBridge::PrepareCaptureLocked() { - for (std::size_t i = 0; i < kInputAnchorSlots.size(); ++i) { - if (vtable_[kInputAnchorSlots[i]] != library_base_ + kInputAnchorRvas[i]) { - return FailedPrecondition( - "FmodAudioDevice input slots do not match Build 2998"); - } + if (!capture_profile_ || !internal::HasExpectedFmodInputCaptureVtable( + vtable_, library_base_, *capture_profile_)) { + return FailedPrecondition( + "FmodAudioDevice capture vtable does not match the Build ID"); } - const auto [lowest_input, highest_input] = - std::minmax_element(kInputAnchorRvas.begin(), kInputAnchorRvas.end()); - for (std::size_t i = 0; i < kCaptureSlots.size(); ++i) { - const std::uintptr_t method = vtable_[kCaptureSlots[i]]; - const std::uintptr_t rva = method - library_base_; - if (method < library_base_ || - !IsExecutableImageRange(library_base_, rva, 1) || - rva + kCaptureCodeWindow < *lowest_input || - rva > *highest_input + kCaptureCodeWindow) { - return FailedPrecondition( - "FmodAudioDevice capture slot is outside the device code"); - } - const bool repeated = - std::find(original_capture_methods_.begin(), - original_capture_methods_.begin() + i, - method) != original_capture_methods_.begin() + i || - std::find(kInputAnchorRvas.begin(), kInputAnchorRvas.end(), rva) != - kInputAnchorRvas.end(); - if (repeated) { + for (std::size_t i = 0; i < capture_profile_->slots.size(); ++i) { + if (!IsExecutableImageRange(library_base_, capture_profile_->method_rvas[i], + 1)) { return FailedPrecondition( - "FmodAudioDevice capture slots are not distinct methods"); + "FmodAudioDevice capture method is outside executable code"); } - original_capture_methods_[i] = method; + original_capture_methods_[i] = vtable_[capture_profile_->slots[i]]; } const char* input_device = std::getenv("MOCKTAIL_AUDIO_INPUT_DEVICE"); const bool enabled = @@ -523,23 +545,24 @@ Status RobloxOutputDeviceBridge::PrepareCaptureLocked() { } bool RobloxOutputDeviceBridge::RestoreVtableLocked() { - if (vtable_ == nullptr || !SetVtableWritable(vtable_, true)) { + if (vtable_ == nullptr || + !SetVtableWritable(vtable_, SelectOutputVtableIndex(profile_), true)) { return false; } __atomic_store_n(&vtable_[kOutputCountVtableIndex], original_methods_[0], __ATOMIC_RELEASE); __atomic_store_n(&vtable_[kOutputInfoVtableIndex], original_methods_[1], __ATOMIC_RELEASE); - __atomic_store_n(&vtable_[kCurrentOutputVtableIndex], original_methods_[2], - __ATOMIC_RELEASE); - __atomic_store_n(&vtable_[kSelectOutputVtableIndex], original_methods_[3], - __ATOMIC_RELEASE); + __atomic_store_n(&vtable_[CurrentOutputVtableIndex(profile_)], + original_methods_[2], __ATOMIC_RELEASE); + __atomic_store_n(&vtable_[SelectOutputVtableIndex(profile_)], + original_methods_[3], __ATOMIC_RELEASE); if (capture_) { - for (std::size_t i = 0; i < kCaptureSlots.size(); ++i) - __atomic_store_n(&vtable_[kCaptureSlots[i]], original_capture_methods_[i], - __ATOMIC_RELEASE); + for (std::size_t i = 0; i < capture_profile_->slots.size(); ++i) + __atomic_store_n(&vtable_[capture_profile_->slots[i]], + original_capture_methods_[i], __ATOMIC_RELEASE); } - return SetVtableWritable(vtable_, false); + return SetVtableWritable(vtable_, SelectOutputVtableIndex(profile_), false); } int RobloxOutputDeviceBridge::GetOutputDeviceCount(void* self) { diff --git a/src/audio/roblox_output_device_bridge_internal.h b/src/audio/roblox_output_device_bridge_internal.h index 1a28001a..c28c132c 100644 --- a/src/audio/roblox_output_device_bridge_internal.h +++ b/src/audio/roblox_output_device_bridge_internal.h @@ -1,6 +1,7 @@ #ifndef MOCKTAIL_AUDIO_ROBLOX_OUTPUT_DEVICE_BRIDGE_INTERNAL_H_ #define MOCKTAIL_AUDIO_ROBLOX_OUTPUT_DEVICE_BRIDGE_INTERNAL_H_ +#include #include #include #include @@ -10,6 +11,22 @@ namespace mocktail::audio::internal { +struct FmodInputCaptureProfile { + std::string_view build_id; + std::uintptr_t vtable_rva; + std::uint32_t vtable_layout_version; + // Format, latency, start, stop, poll, recording. + std::array slots; + std::array method_rvas; +}; + +const FmodInputCaptureProfile* FindFmodInputCaptureProfile( + const compat::BuildProfile& profile); + +bool HasExpectedFmodInputCaptureVtable(const std::uintptr_t* vtable, + std::uintptr_t image_base, + const FmodInputCaptureProfile& profile); + bool HasExpectedFmodStringConstructorContract(const std::uint8_t* code, std::size_t size); diff --git a/src/compat/build_profile.cc b/src/compat/build_profile.cc index ee5aaa88..1111747b 100644 --- a/src/compat/build_profile.cc +++ b/src/compat/build_profile.cc @@ -119,9 +119,18 @@ bool ParseOptionalFmodOutputDeviceBridge( return false; } + std::uint32_t layout_version = 1; + const auto layout = field->find("vtable_layout_version"); + if (layout != field->end()) { + if (!layout->is_number_integer() || (*layout != 1 && *layout != 2)) { + return false; + } + layout_version = layout->get(); + } *bridge_profile = FmodOutputDeviceBridgeProfile{ *vtable_rva, *string_constructor_rva, *count_method_rva, *info_method_rva, *current_method_rva, *select_method_rva, + layout_version, }; return true; } diff --git a/src/compat/host_abi_profile.cc b/src/compat/host_abi_profile.cc index a7693813..959fc394 100644 --- a/src/compat/host_abi_profile.cc +++ b/src/compat/host_abi_profile.cc @@ -6,6 +6,27 @@ namespace mocktail::compat { namespace { constexpr HostAbiProfile kProfiles[] = { + // Roblox 2.738.1397; promoted from the exact-build probation profile. + {"5f0704edd9064f566ee3d6df2bd2fabbcc709f03", + {{{0x1d73422, HostBridgeKind::kAllocate, "small-allocate"}, + {0x1d7667a, HostBridgeKind::kUsableSize, "usable-size"}, + {0x222f4c0, HostBridgeKind::kReallocate, "reallocate"}, + {0x1d731c2, HostBridgeKind::kAllocate, "allocate"}, + {0x1d8e7f9, HostBridgeKind::kAlignedAllocate, "aligned-allocate-direct"}, + {0x1d76c19, HostBridgeKind::kFree, "free"}}}, + 6, + {0x76e2da0, 0x77efb98, 0x7c4b780, 0x400, 0x1d73838, 0x1dffad7, 0x7b87830, + 0x7b878c0, 0x400000}, + {0x1d731c2, 0x1d76c19}, + 0x70e1618, + 3599, + {{{2, 3}, {5, 3599}}}, + 2, + {{{2, 3599}}}, + 1, + 2, + {0x22308d3, 0x7b8b998}, + HostAllocatorStrategy::kNativeMimalloc}, // Roblox 2.736.1408. Derived from the verified 2908 profile and validated // by two isolated no-recovery Vulkan canaries and real game sessions. {"ade08266c67aee88ec9c1d00902150e1684dad3a", @@ -13,12 +34,11 @@ constexpr HostAbiProfile kProfiles[] = { {0x1d2f43a, HostBridgeKind::kUsableSize, "usable-size"}, {0x21e68bf, HostBridgeKind::kReallocate, "reallocate"}, {0x1d2bf82, HostBridgeKind::kAllocate, "allocate"}, - {0x1d47a63, HostBridgeKind::kAlignedAllocate, - "aligned-allocate-direct"}, + {0x1d47a63, HostBridgeKind::kAlignedAllocate, "aligned-allocate-direct"}, {0x1d2f9d9, HostBridgeKind::kFree, "free"}}}, 6, - {0x7652e70, 0x775fd28, 0x7bb4340, 0x400, 0x1d2c5f8, 0x1db7b57, - 0x7af0430, 0x7af04c0, 0x400000}, + {0x7652e70, 0x775fd28, 0x7bb4340, 0x400, 0x1d2c5f8, 0x1db7b57, 0x7af0430, + 0x7af04c0, 0x400000}, {0x1d2bf82, 0x1d2f9d9}, 0x7057818, 3570, @@ -38,12 +58,11 @@ constexpr HostAbiProfile kProfiles[] = { {0x1cf9c3a, HostBridgeKind::kUsableSize, "usable-size"}, {0x21af33f, HostBridgeKind::kReallocate, "reallocate"}, {0x1cf677e, HostBridgeKind::kAllocate, "allocate"}, - {0x1d12197, HostBridgeKind::kAlignedAllocate, - "aligned-allocate-direct"}, + {0x1d12197, HostBridgeKind::kAlignedAllocate, "aligned-allocate-direct"}, {0x1cfa1d9, HostBridgeKind::kFree, "free"}}}, 6, - {0x7571570, 0x767e428, 0x7ad47e8, 0x400, 0x1cf6df8, 0x1d82be7, - 0x7a10730, 0x7a107c0, 0x400000}, + {0x7571570, 0x767e428, 0x7ad47e8, 0x400, 0x1cf6df8, 0x1d82be7, 0x7a10730, + 0x7a107c0, 0x400000}, {0x1cf677e, 0x1cfa1d9}, 0x6f78a18, 3556, @@ -64,12 +83,11 @@ constexpr HostAbiProfile kProfiles[] = { {0x1fe4c84, HostBridgeKind::kUsableSize, "usable-size"}, {0x203daae, HostBridgeKind::kReallocate, "reallocate"}, {0x1c39c3a, HostBridgeKind::kAllocate, "allocate"}, - {0x1c8dbf8, HostBridgeKind::kAlignedAllocate, - "aligned-allocate-direct"}, + {0x1c8dbf8, HostBridgeKind::kAlignedAllocate, "aligned-allocate-direct"}, {0x1c3eb72, HostBridgeKind::kFree, "free"}}}, 6, - {0x7030ae0, 0x703a7e8, 0x72a16d0, 0x400, 0x1c39f1d, 0x2040817, - 0x71e92f0, 0x71e9380, 0x400000}, + {0x7030ae0, 0x703a7e8, 0x72a16d0, 0x400, 0x1c39f1d, 0x2040817, 0x71e92f0, + 0x71e9380, 0x400000}, {0x1c39c3a, 0x1c3eb72}, 0x6c2c9b0, 0x6cc8 / 8, diff --git a/src/graphics/bionic_vulkan_loader_adapter.cc b/src/graphics/bionic_vulkan_loader_adapter.cc index a3a6dc42..ecef8562 100644 --- a/src/graphics/bionic_vulkan_loader_adapter.cc +++ b/src/graphics/bionic_vulkan_loader_adapter.cc @@ -10,8 +10,11 @@ #include #include #include +#include #include +#include #include +#include #include #include @@ -22,8 +25,14 @@ extern "C" { VKAPI_ATTR VkResult VKAPI_CALL -vkCreateInstance(const VkInstanceCreateInfo* create_info, - const VkAllocationCallbacks* allocator, VkInstance* instance); +mocktail_vulkan_idle_synchronized(PFN_vkDeviceWaitIdle raw, VkDevice device); +VKAPI_ATTR VkResult VKAPI_CALL mocktail_vulkan_submit_synchronized( + PFN_vkQueueSubmit raw, VkDevice device, VkQueue queue, uint32_t count, + const VkSubmitInfo *submits, VkFence fence); + +VKAPI_ATTR VkResult VKAPI_CALL +vkCreateInstance(const VkInstanceCreateInfo *create_info, + const VkAllocationCallbacks *allocator, VkInstance *instance); VKAPI_ATTR void VKAPI_CALL vkDestroyInstance(VkInstance instance, const VkAllocationCallbacks* allocator); VKAPI_ATTR VkResult VKAPI_CALL vkCreateAndroidSurfaceKHR( @@ -62,13 +71,29 @@ VKAPI_ATTR VkResult VKAPI_CALL vkAcquireNextImageKHR( VkDevice device, VkSwapchainKHR swapchain, std::uint64_t timeout, VkSemaphore semaphore, VkFence fence, std::uint32_t* image_index); VKAPI_ATTR VkResult VKAPI_CALL vkAcquireNextImage2KHR( - VkDevice device, const VkAcquireNextImageInfoKHR* acquire_info, - std::uint32_t* image_index); -VKAPI_ATTR VkResult VKAPI_CALL vkWaitForFences( - VkDevice device, std::uint32_t fence_count, const VkFence* fences, - VkBool32 wait_all, std::uint64_t timeout); -VKAPI_ATTR VkResult VKAPI_CALL vkResetFences( - VkDevice device, std::uint32_t fence_count, const VkFence* fences); + VkDevice device, const VkAcquireNextImageInfoKHR *acquire_info, + std::uint32_t *image_index); +VKAPI_ATTR VkResult VKAPI_CALL vkWaitForFences(VkDevice device, + std::uint32_t fence_count, + const VkFence *fences, + VkBool32 wait_all, + std::uint64_t timeout); +VKAPI_ATTR VkResult VKAPI_CALL vkResetFences(VkDevice device, + std::uint32_t fence_count, + const VkFence *fences); +VKAPI_ATTR VkResult VKAPI_CALL vkCreateCommandPool( + VkDevice device, const VkCommandPoolCreateInfo *info, + const VkAllocationCallbacks *allocator, VkCommandPool *pool); +VKAPI_ATTR void VKAPI_CALL vkCmdPipelineBarrier( + VkCommandBuffer command, VkPipelineStageFlags src, VkPipelineStageFlags dst, + VkDependencyFlags flags, uint32_t memory_count, + const VkMemoryBarrier *memory, uint32_t buffer_count, + const VkBufferMemoryBarrier *buffers, uint32_t image_count, + const VkImageMemoryBarrier *images); +VKAPI_ATTR void VKAPI_CALL vkCmdPipelineBarrier2(VkCommandBuffer command, + const VkDependencyInfo *info); +VKAPI_ATTR void VKAPI_CALL +vkCmdPipelineBarrier2KHR(VkCommandBuffer command, const VkDependencyInfo *info); VKAPI_ATTR VkResult VKAPI_CALL vkResetCommandPool( VkDevice device, VkCommandPool command_pool, VkCommandPoolResetFlags flags); @@ -286,6 +311,46 @@ Function ResolveProcessFunction(const char* name) { return reinterpret_cast(dlsym(RTLD_DEFAULT, name)); } +// Queue calls share the gate; device idle is exclusive. +// Release it before fence waits and backend callbacks. +std::shared_ptr DeviceIdleGate(VkDevice device) { + static std::mutex registry_mutex; + static std::unordered_map> + registry; + std::lock_guard lock(registry_mutex); + auto &gate = registry[device]; + if (!gate) + gate = std::make_shared(); + return gate; +} +std::shared_ptr QueueCallMutex(VkQueue queue) { + static std::mutex registry_mutex; + static std::unordered_map> registry; + std::lock_guard lock(registry_mutex); + auto &mutex = registry[queue]; + if (!mutex) + mutex = std::make_shared(); + return mutex; +} + +void NoteVrSubmitted(VkQueue queue, uint32_t count, + const VkSubmitInfo *submits) { + static const auto note = ResolveProcessFunction("mocktail_vr_xr_submitted"); + if (note && submits) + for (uint32_t i = 0; i < count; ++i) + note(queue, submits[i].commandBufferCount, submits[i].pCommandBuffers); +} +void NoteVrSubmitted2(VkQueue queue, uint32_t count, + const VkSubmitInfo2 *submits) { + static const auto note = ResolveProcessFunction("mocktail_vr_xr_submitted"); + if (note && submits) + for (uint32_t i = 0; i < count; ++i) + for (uint32_t j = 0; j < submits[i].commandBufferInfoCount; ++j) + note(queue, 1, &submits[i].pCommandBufferInfos[j].commandBuffer); +} + bool EnsureInitialized() { AdapterState& state = State(); std::lock_guard lock(state.mutex); @@ -293,6 +358,18 @@ bool EnsureInitialized() { return true; } + const auto register_submit = + ResolveProcessFunction("mocktail_vr_xr_queue_submit_adapter"); + if (register_submit) + register_submit(mocktail_vulkan_submit_synchronized); + const auto register_idle = ResolveProcessFunction( + "mocktail_vr_xr_device_idle_adapter"); + if (register_idle) + register_idle(mocktail_vulkan_idle_synchronized); + const BackendWindowFn backend_window = ResolveProcessFunction("mocktail_window_backend_window"); const UsesDirectVulkanFn uses_direct_vulkan = @@ -564,6 +641,57 @@ const AdapterState::DeviceDispatch& HostDispatchForCommandBuffer( return kEmptyDeviceDispatch; } +VkResult VKAPI_CALL LockedHostSubmit(VkQueue queue, uint32_t count, + const VkSubmitInfo *submits, + VkFence fence) { + return mocktail_vulkan_submit_synchronized( + HostDispatchForQueue(queue).queue_submit, + HostDispatchForQueue(queue).device, queue, count, submits, fence); +} +VkResult VKAPI_CALL LockedHostSubmit2(VkQueue queue, uint32_t count, + const VkSubmitInfo2 *submits, + VkFence fence) { + const auto raw = HostDispatchForQueue(queue).queue_submit2; + const auto gate = DeviceIdleGate(HostDispatchForQueue(queue).device); + std::shared_lock device_gate(*gate); + const auto mutex = QueueCallMutex(queue); + std::lock_guard lock(*mutex); + return raw(queue, count, submits, fence); +} +VkResult VKAPI_CALL LockedHostSubmit2KHR(VkQueue queue, uint32_t count, + const VkSubmitInfo2 *submits, + VkFence fence) { + const auto raw = HostDispatchForQueue(queue).queue_submit2_khr; + const auto gate = DeviceIdleGate(HostDispatchForQueue(queue).device); + std::shared_lock device_gate(*gate); + const auto mutex = QueueCallMutex(queue); + std::lock_guard lock(*mutex); + return raw(queue, count, submits, fence); +} +VkResult VKAPI_CALL LockedHostBindSparse(VkQueue queue, uint32_t count, + const VkBindSparseInfo *binds, + VkFence fence) { + const auto raw = HostDispatchForQueue(queue).queue_bind_sparse; + const auto gate = DeviceIdleGate(HostDispatchForQueue(queue).device); + std::shared_lock device_gate(*gate); + const auto mutex = QueueCallMutex(queue); + std::lock_guard lock(*mutex); + return raw(queue, count, binds, fence); +} +VkResult VKAPI_CALL LockedHostQueueWaitIdle(VkQueue queue) { + const auto raw = HostDispatchForQueue(queue).queue_wait_idle; + const auto gate = DeviceIdleGate(HostDispatchForQueue(queue).device); + std::shared_lock device_gate(*gate); + const auto mutex = QueueCallMutex(queue); + std::lock_guard lock(*mutex); + return raw(queue); +} + +VkResult VKAPI_CALL LockedHostDeviceWaitIdle(VkDevice device) { + return mocktail_vulkan_idle_synchronized( + HostDispatchForDevice(device).device_wait_idle, device); +} + void RegisterHostDeviceDispatch(VkDevice device, VkPhysicalDevice physical_device, PFN_vkGetDeviceProcAddr get_device_proc_addr) { @@ -834,9 +962,9 @@ PFN_vkVoidFunction AdapterProc(const char* name) { if (name == nullptr) { return nullptr; } -#define MOCKTAIL_VK_PROC(function) \ - if (std::strcmp(name, #function) == 0) { \ - return reinterpret_cast(function); \ +#define MOCKTAIL_VK_PROC(function) \ + if (std::strcmp(name, #function) == 0) { \ + return reinterpret_cast(function); \ } MOCKTAIL_VK_PROC(vkCreateInstance) MOCKTAIL_VK_PROC(vkDestroyInstance) @@ -856,6 +984,10 @@ PFN_vkVoidFunction AdapterProc(const char* name) { MOCKTAIL_VK_PROC(vkAcquireNextImage2KHR) MOCKTAIL_VK_PROC(vkWaitForFences) MOCKTAIL_VK_PROC(vkResetFences) + MOCKTAIL_VK_PROC(vkCreateCommandPool) + MOCKTAIL_VK_PROC(vkCmdPipelineBarrier) + MOCKTAIL_VK_PROC(vkCmdPipelineBarrier2) + MOCKTAIL_VK_PROC(vkCmdPipelineBarrier2KHR) MOCKTAIL_VK_PROC(vkResetCommandPool) MOCKTAIL_VK_PROC(vkGetQueryPoolResults) MOCKTAIL_VK_PROC(vkAllocateCommandBuffers) @@ -886,43 +1018,47 @@ PFN_vkVoidFunction AdapterProc(const char* name) { return nullptr; } -bool IsDeviceAdapterProc(const char* name) { - return name != nullptr && (std::strcmp(name, "vkDestroyDevice") == 0 || - std::strcmp(name, "vkGetDeviceQueue") == 0 || - std::strcmp(name, "vkGetDeviceQueue2") == 0 || - std::strcmp(name, "vkCreateSwapchainKHR") == 0 || - std::strcmp(name, "vkDestroySwapchainKHR") == 0 || - std::strcmp(name, "vkAcquireNextImageKHR") == 0 || - std::strcmp(name, "vkAcquireNextImage2KHR") == 0 || - std::strcmp(name, "vkWaitForFences") == 0 || - std::strcmp(name, "vkResetFences") == 0 || - std::strcmp(name, "vkResetCommandPool") == 0 || - std::strcmp(name, "vkGetQueryPoolResults") == 0 || - std::strcmp(name, "vkAllocateCommandBuffers") == - 0 || - std::strcmp(name, "vkFreeCommandBuffers") == 0 || - std::strcmp(name, "vkDestroyCommandPool") == 0 || - std::strcmp(name, "vkBeginCommandBuffer") == 0 || - std::strcmp(name, "vkEndCommandBuffer") == 0 || - std::strcmp(name, "vkResetCommandBuffer") == 0 || - std::strcmp(name, "vkWaitSemaphores") == 0 || - std::strcmp(name, "vkWaitSemaphoresKHR") == 0 || - std::strcmp(name, "vkQueueSubmit") == 0 || - std::strcmp(name, "vkQueueSubmit2") == 0 || - std::strcmp(name, "vkQueueSubmit2KHR") == 0 || - std::strcmp(name, "vkQueueBindSparse") == 0 || - std::strcmp(name, "vkQueueWaitIdle") == 0 || - std::strcmp(name, "vkDeviceWaitIdle") == 0 || - std::strcmp(name, "vkQueuePresentKHR") == 0 || - std::strcmp(name, "vkCreateImage") == 0 || - std::strcmp(name, "vkDestroyImage") == 0 || - std::strcmp(name, "vkCreateImageView") == 0 || - std::strcmp(name, "vkDestroyImageView") == 0 || - std::strcmp(name, "vkCreateFramebuffer") == 0 || - std::strcmp(name, "vkDestroyFramebuffer") == 0 || - std::strcmp(name, "vkCreateRenderPass") == 0 || - std::strcmp(name, "vkDestroyRenderPass") == 0 || - std::strcmp(name, "vkCmdBeginRenderPass") == 0); +bool IsDeviceAdapterProc(const char *name) { + return name != nullptr && + (std::strcmp(name, "vkDestroyDevice") == 0 || + std::strcmp(name, "vkGetDeviceQueue") == 0 || + std::strcmp(name, "vkGetDeviceQueue2") == 0 || + std::strcmp(name, "vkCreateSwapchainKHR") == 0 || + std::strcmp(name, "vkDestroySwapchainKHR") == 0 || + std::strcmp(name, "vkAcquireNextImageKHR") == 0 || + std::strcmp(name, "vkAcquireNextImage2KHR") == 0 || + std::strcmp(name, "vkWaitForFences") == 0 || + std::strcmp(name, "vkResetFences") == 0 || + std::strcmp(name, "vkCreateCommandPool") == 0 || + std::strcmp(name, "vkCmdPipelineBarrier") == 0 || + std::strcmp(name, "vkCmdPipelineBarrier2") == 0 || + std::strcmp(name, "vkCmdPipelineBarrier2KHR") == 0 || + std::strcmp(name, "vkResetCommandPool") == 0 || + std::strcmp(name, "vkGetQueryPoolResults") == 0 || + std::strcmp(name, "vkAllocateCommandBuffers") == 0 || + std::strcmp(name, "vkFreeCommandBuffers") == 0 || + std::strcmp(name, "vkDestroyCommandPool") == 0 || + std::strcmp(name, "vkBeginCommandBuffer") == 0 || + std::strcmp(name, "vkEndCommandBuffer") == 0 || + std::strcmp(name, "vkResetCommandBuffer") == 0 || + std::strcmp(name, "vkWaitSemaphores") == 0 || + std::strcmp(name, "vkWaitSemaphoresKHR") == 0 || + std::strcmp(name, "vkQueueSubmit") == 0 || + std::strcmp(name, "vkQueueSubmit2") == 0 || + std::strcmp(name, "vkQueueSubmit2KHR") == 0 || + std::strcmp(name, "vkQueueBindSparse") == 0 || + std::strcmp(name, "vkQueueWaitIdle") == 0 || + std::strcmp(name, "vkDeviceWaitIdle") == 0 || + std::strcmp(name, "vkQueuePresentKHR") == 0 || + std::strcmp(name, "vkCreateImage") == 0 || + std::strcmp(name, "vkDestroyImage") == 0 || + std::strcmp(name, "vkCreateImageView") == 0 || + std::strcmp(name, "vkDestroyImageView") == 0 || + std::strcmp(name, "vkCreateFramebuffer") == 0 || + std::strcmp(name, "vkDestroyFramebuffer") == 0 || + std::strcmp(name, "vkCreateRenderPass") == 0 || + std::strcmp(name, "vkDestroyRenderPass") == 0 || + std::strcmp(name, "vkCmdBeginRenderPass") == 0); } bool IsGlobalAdapterProc(const char* name) { @@ -1201,12 +1337,35 @@ ObservedHostQueuePresent(VkQueue queue, const VkPresentInfoKHR* present_info) { } const VrXrNotePresentFn vr_note_present = state.vr_note_present.load(std::memory_order_acquire); + VkPresentInfoKHR synchronized{}; if (vr_note_present != nullptr) { - vr_note_present(queue, HostDispatchForQueue(queue).device); + static const auto prepare = ResolveProcessFunction( + "mocktail_vr_xr_prepare_present"); + // Mirror/overlay may have consumed waits; use the final dependency list. + const VkResult prepared = + prepare + ? prepare(queue, HostDispatchForQueue(queue).device, present_info) + : VK_NOT_READY; + if (prepared < 0) + return prepared; + if (prepared == VK_SUCCESS) { + vr_note_present(queue, HostDispatchForQueue(queue).device); + synchronized = *present_info; + synchronized.waitSemaphoreCount = 0; + synchronized.pWaitSemaphores = nullptr; + present_info = &synchronized; + } } const bool fps_trace = FpsTraceEnabled(); const std::uint64_t present_start_ns = fps_trace ? MonotonicNanos() : 0; - const VkResult result = host_present(queue, present_info); + const VkResult result = [&] { + const auto gate = DeviceIdleGate(HostDispatchForQueue(queue).device); + std::shared_lock device_gate(*gate); + const auto mutex = QueueCallMutex(queue); + std::lock_guard lock(*mutex); + return host_present(queue, present_info); + }(); if (fps_trace) { PresentWaitTrace().Record(present_start_ns); } @@ -1689,7 +1848,23 @@ VKAPI_ATTR VkResult VKAPI_CALL vkCreateSwapchainKHR( return VK_ERROR_INITIALIZATION_FAILED; } VkSwapchainCreateInfoKHR host_info = *create_info; - static const auto vr_desktop_enabled = ResolveProcessFunction("mocktail_vr_desktop_enabled"); + unsigned vr_families[256]{}; + bool converted_to_concurrent = false; + static const auto shared_families = + ResolveProcessFunction( + "mocktail_vr_desktop_queue_families"); + if (shared_families && + host_info.imageSharingMode == VK_SHARING_MODE_EXCLUSIVE) { + const unsigned count = shared_families(device, 256, vr_families); + if (count > 1) { + host_info.imageSharingMode = VK_SHARING_MODE_CONCURRENT; + host_info.queueFamilyIndexCount = count; + host_info.pQueueFamilyIndices = vr_families; + converted_to_concurrent = true; + } + } + static const auto vr_desktop_enabled = + ResolveProcessFunction("mocktail_vr_desktop_enabled"); if (vr_desktop_enabled && vr_desktop_enabled()) { PFN_vkGetPhysicalDeviceSurfaceCapabilitiesKHR caps; { std::lock_guard lock(State().mutex); caps = State().host_surface_capabilities; } @@ -1783,11 +1958,15 @@ VKAPI_ATTR VkResult VKAPI_CALL vkCreateSwapchainKHR( host_get_images(device, *swapchain, &image_count, images.data()); if (images_result == VK_SUCCESS || images_result == VK_INCOMPLETE) { images.resize(image_count); - static const auto record = ResolveProcessFunction("mocktail_vr_desktop_swapchain"); - if (record) record(device, *swapchain, &host_info, images.data(), image_count); - (void)State().text_overlay.RegisterSwapchain( - device, *swapchain, *create_info, images.data(), image_count); + static const auto record = ResolveProcessFunction( + "mocktail_vr_desktop_swapchain_shared"); + if (record) + record(device, *swapchain, &host_info, images.data(), image_count, + converted_to_concurrent); + (void)State().text_overlay.RegisterSwapchain(device, *swapchain, host_info, + images.data(), image_count); } return result; } @@ -1905,6 +2084,10 @@ VKAPI_ATTR VkResult VKAPI_CALL vkResetCommandPool( return VK_ERROR_INITIALIZATION_FAILED; } const VkResult result = host_reset(device, command_pool, flags); + static const auto note = ResolveProcessFunction( + "mocktail_vr_xr_reset_pool"); + if (result == VK_SUCCESS && note) + note(command_pool); observation.SetResult(result); return result; } @@ -1944,6 +2127,12 @@ VKAPI_ATTR VkResult VKAPI_CALL vkAllocateCommandBuffers( RegisterHostCommandBuffers(device, allocate_info->commandPool, allocate_info->commandBufferCount, command_buffers); + static const auto note = ResolveProcessFunction( + "mocktail_vr_xr_command_buffers"); + if (note) + note(device, allocate_info->commandPool, + allocate_info->commandBufferCount, command_buffers); } observation.SetResult(result); return result; @@ -1960,6 +2149,11 @@ VKAPI_ATTR void VKAPI_CALL vkFreeCommandBuffers( observation.SetResult(VK_ERROR_INITIALIZATION_FAILED); return; } + static const auto note = ResolveProcessFunction( + "mocktail_vr_xr_command_buffers"); + if (note) + note(device, VK_NULL_HANDLE, command_buffer_count, command_buffers); host_free(device, command_pool, command_buffer_count, command_buffers); RemoveHostCommandBuffers(device, command_pool, command_buffer_count, command_buffers); @@ -1976,6 +2170,11 @@ VKAPI_ATTR void VKAPI_CALL vkDestroyCommandPool( observation.SetResult(VK_ERROR_INITIALIZATION_FAILED); return; } + static const auto note = + ResolveProcessFunction( + "mocktail_vr_xr_command_pool"); + if (note) + note(device, command_pool, UINT32_MAX); host_destroy(device, command_pool, allocator); RemoveHostCommandPoolBindings(device, command_pool); observation.SetResult(VK_SUCCESS); @@ -1992,6 +2191,10 @@ VKAPI_ATTR VkResult VKAPI_CALL vkBeginCommandBuffer( return VK_ERROR_INITIALIZATION_FAILED; } const VkResult result = host_begin(command_buffer, begin_info); + static const auto note = ResolveProcessFunction( + "mocktail_vr_xr_reset_command"); + if (result == VK_SUCCESS && note) + note(command_buffer); observation.SetResult(result); return result; } @@ -2020,6 +2223,10 @@ VKAPI_ATTR VkResult VKAPI_CALL vkResetCommandBuffer( return VK_ERROR_INITIALIZATION_FAILED; } const VkResult result = host_reset(command_buffer, flags); + static const auto note = ResolveProcessFunction( + "mocktail_vr_xr_reset_command"); + if (result == VK_SUCCESS && note) + note(command_buffer); observation.SetResult(result); return result; } @@ -2056,7 +2263,9 @@ VKAPI_ATTR VkResult VKAPI_CALL vkQueueSubmit( return VK_ERROR_INITIALIZATION_FAILED; } const VkResult result = State().text_overlay.QueueSubmit( - queue, submit_count, submits, fence, host_submit); + queue, submit_count, submits, fence, LockedHostSubmit); + if (result == VK_SUCCESS) + NoteVrSubmitted(queue, submit_count, submits); observation.SetResult(result); return result; } @@ -2072,7 +2281,9 @@ VKAPI_ATTR VkResult VKAPI_CALL vkQueueSubmit2( return VK_ERROR_INITIALIZATION_FAILED; } const VkResult result = State().text_overlay.QueueSubmit2( - queue, submit_count, submits, fence, host_submit); + queue, submit_count, submits, fence, LockedHostSubmit2); + if (result == VK_SUCCESS) + NoteVrSubmitted2(queue, submit_count, submits); observation.SetResult(result); return result; } @@ -2088,8 +2299,9 @@ VKAPI_ATTR VkResult VKAPI_CALL vkQueueSubmit2KHR( return VK_ERROR_INITIALIZATION_FAILED; } const VkResult result = State().text_overlay.QueueSubmit2( - queue, submit_count, submits, fence, - reinterpret_cast(host_submit)); + queue, submit_count, submits, fence, LockedHostSubmit2KHR); + if (result == VK_SUCCESS) + NoteVrSubmitted2(queue, submit_count, submits); observation.SetResult(result); return result; } @@ -2105,7 +2317,7 @@ VKAPI_ATTR VkResult VKAPI_CALL vkQueueBindSparse( return VK_ERROR_INITIALIZATION_FAILED; } const VkResult result = State().text_overlay.QueueBindSparse( - queue, bind_info_count, bind_info, fence, host_bind); + queue, bind_info_count, bind_info, fence, LockedHostBindSparse); observation.SetResult(result); return result; } @@ -2121,7 +2333,7 @@ VKAPI_ATTR VkResult VKAPI_CALL vkQueueWaitIdle(VkQueue queue) { const bool fps_trace = FpsTraceEnabled(); const std::uint64_t idle_start_ns = fps_trace ? MonotonicNanos() : 0; const VkResult result = - State().text_overlay.QueueWaitIdle(queue, host_wait); + State().text_overlay.QueueWaitIdle(queue, LockedHostQueueWaitIdle); if (fps_trace) { QueueIdleWaitTrace().Record(idle_start_ns); } @@ -2138,7 +2350,7 @@ VKAPI_ATTR VkResult VKAPI_CALL vkDeviceWaitIdle(VkDevice device) { return VK_ERROR_INITIALIZATION_FAILED; } const VkResult result = - State().text_overlay.DeviceWaitIdle(device, host_wait); + State().text_overlay.DeviceWaitIdle(device, LockedHostDeviceWaitIdle); observation.SetResult(result); return result; } @@ -2434,4 +2646,100 @@ vkCmdBeginRenderPass(VkCommandBuffer command_buffer, } } -} // extern "C" \ No newline at end of file +VKAPI_ATTR VkResult VKAPI_CALL vkCreateCommandPool( + VkDevice device, const VkCommandPoolCreateInfo *info, + const VkAllocationCallbacks *allocator, VkCommandPool *pool) { + const auto create = reinterpret_cast( + HostDeviceProc(device, "vkCreateCommandPool")); + if (!create) + return VK_ERROR_INITIALIZATION_FAILED; + const VkResult result = create(device, info, allocator, pool); + static const auto note = + ResolveProcessFunction( + "mocktail_vr_xr_command_pool"); + if (result == VK_SUCCESS && note && info && pool) + note(device, *pool, info->queueFamilyIndex); + return result; +} + +VKAPI_ATTR void VKAPI_CALL vkCmdPipelineBarrier( + VkCommandBuffer command, VkPipelineStageFlags src, VkPipelineStageFlags dst, + VkDependencyFlags flags, uint32_t memory_count, + const VkMemoryBarrier *memory, uint32_t buffer_count, + const VkBufferMemoryBarrier *buffers, uint32_t image_count, + const VkImageMemoryBarrier *images) { + const auto barrier = reinterpret_cast( + HostDeviceProc(HostDispatchForCommandBuffer(command).device, + "vkCmdPipelineBarrier")); + if (!barrier) + return; + static const auto normalize = ResolveProcessFunction("mocktail_vr_xr_desktop_barrier"); + std::vector adjusted; + if (normalize && images && image_count) { + adjusted.assign(images, images + image_count); + for (auto &image : adjusted) + image.newLayout = normalize( + command, image.image, &image.srcQueueFamilyIndex, + &image.dstQueueFamilyIndex, &image.oldLayout, image.newLayout); + images = adjusted.data(); + } + barrier(command, src, dst, flags, memory_count, memory, buffer_count, buffers, + image_count, images); +} + +static void ForwardVrBarrier2(VkCommandBuffer command, + const VkDependencyInfo *info, const char *name) { + const auto barrier = reinterpret_cast( + HostDeviceProc(HostDispatchForCommandBuffer(command).device, name)); + if (!barrier || !info) + return; + static const auto normalize = ResolveProcessFunction("mocktail_vr_xr_desktop_barrier"); + VkDependencyInfo adjusted_info = *info; + std::vector adjusted; + if (normalize && info->pImageMemoryBarriers && + info->imageMemoryBarrierCount) { + adjusted.assign(info->pImageMemoryBarriers, + info->pImageMemoryBarriers + info->imageMemoryBarrierCount); + for (auto &image : adjusted) + image.newLayout = normalize( + command, image.image, &image.srcQueueFamilyIndex, + &image.dstQueueFamilyIndex, &image.oldLayout, image.newLayout); + adjusted_info.pImageMemoryBarriers = adjusted.data(); + } + barrier(command, &adjusted_info); +} +VKAPI_ATTR void VKAPI_CALL vkCmdPipelineBarrier2(VkCommandBuffer command, + const VkDependencyInfo *info) { + ForwardVrBarrier2(command, info, "vkCmdPipelineBarrier2"); +} +VKAPI_ATTR void VKAPI_CALL vkCmdPipelineBarrier2KHR( + VkCommandBuffer command, const VkDependencyInfo *info) { + ForwardVrBarrier2(command, info, "vkCmdPipelineBarrier2KHR"); +} + +VKAPI_ATTR VkResult VKAPI_CALL mocktail_vulkan_submit_synchronized( + PFN_vkQueueSubmit raw, VkDevice device, VkQueue queue, uint32_t count, + const VkSubmitInfo *submits, VkFence fence) { + if (!raw) + return VK_ERROR_INITIALIZATION_FAILED; + const auto gate = DeviceIdleGate(device); + std::shared_lock device_gate(*gate); + const auto mutex = QueueCallMutex(queue); + std::lock_guard lock(*mutex); + return raw(queue, count, submits, fence); +} + +VKAPI_ATTR VkResult VKAPI_CALL +mocktail_vulkan_idle_synchronized(PFN_vkDeviceWaitIdle raw, VkDevice device) { + if (!raw) + return VK_ERROR_INITIALIZATION_FAILED; + const auto gate = DeviceIdleGate(device); + std::unique_lock device_gate(*gate); + return raw(device); +} + +} // extern "C" diff --git a/src/legacy/legacy_runtime.cc b/src/legacy/legacy_runtime.cc index 6bfa5de9..b3fe9ea3 100644 --- a/src/legacy/legacy_runtime.cc +++ b/src/legacy/legacy_runtime.cc @@ -4507,17 +4507,11 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, return EXIT_FAILURE; } - const mocktail::compat::BuildProfile& build_profile = - *profile_result.profile; - // The experimental VR ABI has only been observed on the active 2998 - // payload. Do not pass VR device parameters to unrelated builds, even when - // they have a general-purpose compatibility profile. Eye initialization and - // OpenXR projection are not yet implemented; this is a diagnostic launch. + const mocktail::compat::BuildProfile& build_profile = *profile_result.profile; if (IsEnabled("MOCKTAIL_VR_ENABLED") && - build_profile.elf_build_id != - "ade08266c67aee88ec9c1d00902150e1684dad3a") { - std::cerr << "[FATAL] Experimental Roblox VR requires exact Build ID " - "ade08266c67aee88ec9c1d00902150e1684dad3a; loaded " + mocktail::vr::FindRobloxVrDeviceProfile(build_profile.elf_build_id) == + nullptr) { + std::cerr << "[FATAL] Experimental Roblox VR has no verified profile for " << build_profile.elf_build_id << "\n"; return EXIT_FAILURE; } diff --git a/src/main.cc b/src/main.cc index 439117f5..4eac0ccc 100644 --- a/src/main.cc +++ b/src/main.cc @@ -742,6 +742,12 @@ int main(int argc, char* argv[]) { std::cerr << "[FATAL] " << compatibility.error << '\n'; return EXIT_FAILURE; } + if (!compatibility.profile.vr_debug_device_bridge.has_value()) { + if (const auto* vr_profile = + mocktail::vr::FindRobloxVrDeviceProfile(compatibility.build_id)) { + compatibility.profile.vr_debug_device_bridge = *vr_profile; + } + } vr_active = runtime_config.config.vr_enabled() && (vr_requested || compatibility.profile.vr_debug_device_bridge.has_value()); @@ -759,7 +765,8 @@ int main(int argc, char* argv[]) { std::cerr << " [vr] experimental backend=" << mocktail::vr::VrBackendModeName(vr_backend_mode) << ": Roblox renders both eyes through its DebugDeviceVR " - "debug device (exact Build ID 2998 only)"; + "debug device (Build ID " + << compatibility.build_id << ")"; if (vr_backend_mode == mocktail::vr::VrBackendMode::kXrOutput) { std::cerr << "; eye images are submitted to the active OpenXR runtime " "as a projection layer. Start WiVRn or SteamVR/ALVR and " diff --git a/src/runtime/runtime_config_bootstrap.cc b/src/runtime/runtime_config_bootstrap.cc index 892ffd20..34abae5d 100644 --- a/src/runtime/runtime_config_bootstrap.cc +++ b/src/runtime/runtime_config_bootstrap.cc @@ -69,12 +69,12 @@ device: pc-windows-11 # vr: # Boolean (default: enabled in VR builds, disabled in non-VR builds). - # Experimental VR for the exact Build ID 2998 - # payload. Mocktail initializes the guest DebugDeviceVR eye resources (two + # Experimental VR for verified builds 2998 and 3092. Mocktail initializes + # the guest DebugDeviceVR eye resources (two # distinct 500x500 eye framebuffers), the Roblox renderer draws both eyes # every frame, and the frames are submitted to the active OpenXR runtime as # a stereo projection layer with head pose (runtime-tracked, or scripted for - # desktop testing). Controllers are NOT connected yet. Standalone OpenXR + # desktop testing). Controllers use the native VR bridge. Standalone OpenXR # test scene: mocktail-vr-probe --scene. Use --no-vr to launch normally. # enabled: false diff --git a/src/vr/diagnostics.cc b/src/vr/diagnostics.cc index 2d374358..2decdb79 100644 --- a/src/vr/diagnostics.cc +++ b/src/vr/diagnostics.cc @@ -67,7 +67,7 @@ std::string FormatProbeReport(const ProbeReport& report) { for (const auto& extension : report.extensions) output << " " << extension << '\n'; output << "Discovery: " << report.message << '\n' - << "Roblox VR rendering: not implemented yet.\n" + << "Roblox VR rendering: run mocktail --vr with a supported client.\n" << "This probe does not create an XR session or render frames.\n"; return output.str(); } diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 66a67b5d..488a1749 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -35,8 +35,32 @@ namespace mocktail::vr { namespace { using Clock = std::chrono::steady_clock; - -void Log(const char* format, ...) { +using SynchronizedQueueSubmit = VkResult(VKAPI_PTR *)(PFN_vkQueueSubmit, + VkDevice, VkQueue, + std::uint32_t, + const VkSubmitInfo *, + VkFence); +std::atomic g_queue_submit_adapter{nullptr}; +using SynchronizedDeviceIdle = VkResult(VKAPI_PTR *)(PFN_vkDeviceWaitIdle, + VkDevice); +std::atomic g_device_idle_adapter{nullptr}; + +VkResult IdleHostDevice(PFN_vkDeviceWaitIdle raw, VkDevice device) { + const auto synchronized = + g_device_idle_adapter.load(std::memory_order_acquire); + return synchronized ? synchronized(raw, device) : raw(device); +} + +VkResult SubmitHostQueue(PFN_vkQueueSubmit raw, VkDevice device, VkQueue queue, + std::uint32_t count, const VkSubmitInfo *submits, + VkFence fence) { + const auto synchronized = + g_queue_submit_adapter.load(std::memory_order_acquire); + return synchronized ? synchronized(raw, device, queue, count, submits, fence) + : raw(queue, count, submits, fence); +} + +void Log(const char *format, ...) { va_list arguments; va_start(arguments, format); std::vfprintf(stderr, format, arguments); @@ -159,6 +183,7 @@ struct OpenXrBackend::VkProcs { PFN_vkResetFences ResetFences = nullptr; PFN_vkWaitForFences WaitForFences = nullptr; PFN_vkQueueSubmit QueueSubmit = nullptr; + PFN_vkGetDeviceQueue GetDeviceQueue = nullptr; PFN_vkDeviceWaitIdle DeviceWaitIdle = nullptr; PFN_vkGetPhysicalDeviceMemoryProperties GetPhysicalDeviceMemoryProperties = nullptr; @@ -288,16 +313,48 @@ Status OpenXrBackend::Arm(VrGraphicsApi graphics_api) { const char *vk_extensions[] = {XR_KHR_VULKAN_ENABLE2_EXTENSION_NAME}; const char *gl_extensions[] = {XR_KHR_OPENGL_ES_ENABLE_EXTENSION_NAME, XR_MNDX_EGL_ENABLE_EXTENSION_NAME}; + std::vector extensions = + graphics_api_ == VrGraphicsApi::kVulkan + ? std::vector{vk_extensions[0]} + : std::vector{gl_extensions[0], gl_extensions[1]}; + std::uint32_t extension_count = 0; + CheckXr(XR_NULL_HANDLE, + xrEnumerateInstanceExtensionProperties(nullptr, 0, &extension_count, + nullptr), + "xrEnumerateInstanceExtensionProperties count"); + Require(extension_count <= 4096, "OpenXR extension list is too large"); + std::vector available( + extension_count, + XrInfo(XR_TYPE_EXTENSION_PROPERTIES)); + if (extension_count != 0) { + CheckXr(XR_NULL_HANDLE, + xrEnumerateInstanceExtensionProperties( + nullptr, extension_count, &extension_count, available.data()), + "xrEnumerateInstanceExtensionProperties"); + } + for (const char *optional : + {XR_FB_TOUCH_CONTROLLER_PRO_EXTENSION_NAME, + XR_META_TOUCH_CONTROLLER_PLUS_EXTENSION_NAME}) { + if (std::any_of(available.begin(), available.end(), + [&](const auto &entry) { + return std::strcmp(entry.extensionName, optional) == 0; + })) { + extensions.push_back(optional); + } + } auto info = XrInfo(XR_TYPE_INSTANCE_CREATE_INFO); std::strcpy(info.applicationInfo.applicationName, "Mocktail Roblox VR"); std::strcpy(info.applicationInfo.engineName, "Mocktail"); - info.applicationInfo.apiVersion = XR_MAKE_VERSION(1, 0, 0); - info.enabledExtensionCount = - graphics_api_ == VrGraphicsApi::kVulkan ? 1 : 2; - info.enabledExtensionNames = - graphics_api_ == VrGraphicsApi::kVulkan ? vk_extensions : gl_extensions; + info.applicationInfo.apiVersion = XR_MAKE_VERSION(1, 1, 0); + info.enabledExtensionCount = static_cast(extensions.size()); + info.enabledExtensionNames = extensions.data(); XrInstance instance = XR_NULL_HANDLE; - const XrResult instance_result = xrCreateInstance(&info, &instance); + XrResult instance_result = xrCreateInstance(&info, &instance); + if (instance_result == XR_ERROR_API_VERSION_UNSUPPORTED) { + info.applicationInfo.apiVersion = XR_MAKE_VERSION(1, 0, 0); + instance = XR_NULL_HANDLE; + instance_result = xrCreateInstance(&info, &instance); + } if (instance_result == XR_ERROR_RUNTIME_UNAVAILABLE) { throw std::runtime_error("xrCreateInstance: " + VrRuntimeUnavailableHint(!user_manifest.empty(), manifest)); @@ -517,7 +574,8 @@ bool OpenXrBackend::CreateVulkanDevice(VkPhysicalDevice physical_device, if (runtime_device != physical_device) { *device = VK_NULL_HANDLE; *result = VK_ERROR_INCOMPATIBLE_DRIVER; - Log(" [vr-backend] refusing a guest GPU different from the OpenXR GPU\n"); + Log(" [vr-backend] refusing a guest GPU different from the OpenXR " + "GPU\n"); return true; } @@ -563,60 +621,60 @@ bool OpenXrBackend::CreateVulkanDevice(VkPhysicalDevice physical_device, void OpenXrBackend::SetupDeviceAfterCreationLocked( VkPhysicalDevice physical_device, VkDevice created, - const VkDeviceCreateInfo* create_info) { - vk_device_ = created; - vk_physical_device_ = physical_device; - auto loader_gipa = reinterpret_cast( - dlsym(vk_loader_, "vkGetInstanceProcAddr")); - Require(loader_gipa != nullptr, "Host vkGetInstanceProcAddr is missing"); - delete vk_; - vk_ = new VkProcs(); - vk_->get_instance_proc_addr = loader_gipa; - vk_->get_device_proc_addr = - reinterpret_cast(loader_gipa( - vk_instance_, "vkGetDeviceProcAddr")); - Require(vk_->get_device_proc_addr != nullptr, - "Host vkGetDeviceProcAddr is missing"); -#define MOCKTAIL_BACKEND_VK_FN(name) \ - vk_->name = reinterpret_cast( \ - vk_->get_device_proc_addr(created, "vk" #name)); \ - Require(vk_->name != nullptr, "Missing Vulkan function: vk" #name); - MOCKTAIL_BACKEND_VK_FN(CreateBuffer) - MOCKTAIL_BACKEND_VK_FN(DestroyBuffer) - MOCKTAIL_BACKEND_VK_FN(GetBufferMemoryRequirements) - MOCKTAIL_BACKEND_VK_FN(AllocateMemory) - MOCKTAIL_BACKEND_VK_FN(FreeMemory) - MOCKTAIL_BACKEND_VK_FN(BindBufferMemory) - MOCKTAIL_BACKEND_VK_FN(MapMemory) - MOCKTAIL_BACKEND_VK_FN(UnmapMemory) - MOCKTAIL_BACKEND_VK_FN(CmdCopyImageToBuffer) - MOCKTAIL_BACKEND_VK_FN(CreateImageView) - MOCKTAIL_BACKEND_VK_FN(DestroyImageView) - MOCKTAIL_BACKEND_VK_FN(CreateCommandPool) - MOCKTAIL_BACKEND_VK_FN(DestroyCommandPool) - MOCKTAIL_BACKEND_VK_FN(AllocateCommandBuffers) - MOCKTAIL_BACKEND_VK_FN(ResetCommandBuffer) - MOCKTAIL_BACKEND_VK_FN(BeginCommandBuffer) - MOCKTAIL_BACKEND_VK_FN(EndCommandBuffer) - MOCKTAIL_BACKEND_VK_FN(CmdPipelineBarrier) - MOCKTAIL_BACKEND_VK_FN(CmdCopyImage) - MOCKTAIL_BACKEND_VK_FN(CmdBlitImage) - MOCKTAIL_BACKEND_VK_FN(CmdClearColorImage) - MOCKTAIL_BACKEND_VK_FN(CreateFence) - MOCKTAIL_BACKEND_VK_FN(DestroyFence) - MOCKTAIL_BACKEND_VK_FN(ResetFences) - MOCKTAIL_BACKEND_VK_FN(WaitForFences) - MOCKTAIL_BACKEND_VK_FN(QueueSubmit) - MOCKTAIL_BACKEND_VK_FN(DeviceWaitIdle) + const VkDeviceCreateInfo *create_info) { + vk_device_ = created; + vk_physical_device_ = physical_device; + auto loader_gipa = reinterpret_cast( + dlsym(vk_loader_, "vkGetInstanceProcAddr")); + Require(loader_gipa != nullptr, "Host vkGetInstanceProcAddr is missing"); + delete vk_; + vk_ = new VkProcs(); + vk_->get_instance_proc_addr = loader_gipa; + vk_->get_device_proc_addr = reinterpret_cast( + loader_gipa(vk_instance_, "vkGetDeviceProcAddr")); + Require(vk_->get_device_proc_addr != nullptr, + "Host vkGetDeviceProcAddr is missing"); +#define MOCKTAIL_BACKEND_VK_FN(name) \ + vk_->name = reinterpret_cast( \ + vk_->get_device_proc_addr(created, "vk" #name)); \ + Require(vk_->name != nullptr, "Missing Vulkan function: vk" #name); + MOCKTAIL_BACKEND_VK_FN(CreateBuffer) + MOCKTAIL_BACKEND_VK_FN(DestroyBuffer) + MOCKTAIL_BACKEND_VK_FN(GetBufferMemoryRequirements) + MOCKTAIL_BACKEND_VK_FN(AllocateMemory) + MOCKTAIL_BACKEND_VK_FN(FreeMemory) + MOCKTAIL_BACKEND_VK_FN(BindBufferMemory) + MOCKTAIL_BACKEND_VK_FN(MapMemory) + MOCKTAIL_BACKEND_VK_FN(UnmapMemory) + MOCKTAIL_BACKEND_VK_FN(CmdCopyImageToBuffer) + MOCKTAIL_BACKEND_VK_FN(CreateImageView) + MOCKTAIL_BACKEND_VK_FN(DestroyImageView) + MOCKTAIL_BACKEND_VK_FN(CreateCommandPool) + MOCKTAIL_BACKEND_VK_FN(DestroyCommandPool) + MOCKTAIL_BACKEND_VK_FN(AllocateCommandBuffers) + MOCKTAIL_BACKEND_VK_FN(ResetCommandBuffer) + MOCKTAIL_BACKEND_VK_FN(BeginCommandBuffer) + MOCKTAIL_BACKEND_VK_FN(EndCommandBuffer) + MOCKTAIL_BACKEND_VK_FN(CmdPipelineBarrier) + MOCKTAIL_BACKEND_VK_FN(CmdCopyImage) + MOCKTAIL_BACKEND_VK_FN(CmdBlitImage) + MOCKTAIL_BACKEND_VK_FN(CmdClearColorImage) + MOCKTAIL_BACKEND_VK_FN(CreateFence) + MOCKTAIL_BACKEND_VK_FN(DestroyFence) + MOCKTAIL_BACKEND_VK_FN(ResetFences) + MOCKTAIL_BACKEND_VK_FN(WaitForFences) + MOCKTAIL_BACKEND_VK_FN(QueueSubmit) + MOCKTAIL_BACKEND_VK_FN(GetDeviceQueue) + MOCKTAIL_BACKEND_VK_FN(DeviceWaitIdle) #undef MOCKTAIL_BACKEND_VK_FN -#define MOCKTAIL_BACKEND_VK_INSTANCE_FN(name) \ - vk_->name = reinterpret_cast(loader_gipa( \ - vk_instance_, "vk" #name)); \ - Require(vk_->name != nullptr, "Missing Vulkan function: vk" #name); - MOCKTAIL_BACKEND_VK_INSTANCE_FN(GetPhysicalDeviceMemoryProperties) - MOCKTAIL_BACKEND_VK_INSTANCE_FN(GetPhysicalDeviceQueueFamilyProperties) - MOCKTAIL_BACKEND_VK_INSTANCE_FN(GetPhysicalDeviceProperties) - MOCKTAIL_BACKEND_VK_INSTANCE_FN(GetPhysicalDeviceFormatProperties) +#define MOCKTAIL_BACKEND_VK_INSTANCE_FN(name) \ + vk_->name = \ + reinterpret_cast(loader_gipa(vk_instance_, "vk" #name)); \ + Require(vk_->name != nullptr, "Missing Vulkan function: vk" #name); + MOCKTAIL_BACKEND_VK_INSTANCE_FN(GetPhysicalDeviceMemoryProperties) + MOCKTAIL_BACKEND_VK_INSTANCE_FN(GetPhysicalDeviceQueueFamilyProperties) + MOCKTAIL_BACKEND_VK_INSTANCE_FN(GetPhysicalDeviceProperties) + MOCKTAIL_BACKEND_VK_INSTANCE_FN(GetPhysicalDeviceFormatProperties) #undef MOCKTAIL_BACKEND_VK_INSTANCE_FN VkPhysicalDeviceProperties device_properties{}; @@ -630,9 +688,6 @@ void OpenXrBackend::SetupDeviceAfterCreationLocked( device_properties.deviceName, error.c_str()); return; } - // One-time plain-text diagnostics required for headset acceptance: which - // runtime answered, which pose source drives the camera, and which GPU owns - // the session. No credentials, tokens or launch parameters are included. Log(" [vr-backend] XR session bound to guest VkDevice: gpu=%s runtime=%s " "pose_source=%s\n", device_properties.deviceName, runtime_name_.c_str(), @@ -659,7 +714,7 @@ bool OpenXrBackend::InitializeSessionLocked( const VkDeviceQueueCreateInfo& queue_info = create_info->pQueueCreateInfos[index]; if (queue_info.queueFamilyIndex >= family_count || - queue_info.queueCount == 0) { + queue_info.queueCount == 0 || queue_info.flags != 0) { continue; } if (families[queue_info.queueFamilyIndex].queueFlags & @@ -671,6 +726,21 @@ bool OpenXrBackend::InitializeSessionLocked( Require(chosen_family < family_count && (families[chosen_family].queueFlags & VK_QUEUE_GRAPHICS_BIT), "the guest device requested no graphics queue family"); vk_queue_family_ = chosen_family; + if (create_info) { + device_queue_families_.clear(); + for (std::uint32_t i = 0; i < create_info->queueCreateInfoCount; ++i) { + const auto &requested = create_info->pQueueCreateInfos[i]; + if (requested.queueCount && requested.flags == 0 && + std::find( + device_queue_families_.begin(), device_queue_families_.end(), + requested.queueFamilyIndex) == device_queue_families_.end()) + device_queue_families_.push_back(requested.queueFamilyIndex); + } + } + vk_->GetDeviceQueue(device, chosen_family, 0, &vk_graphics_queue_); + Require(vk_graphics_queue_ != VK_NULL_HANDLE, + "missing guest graphics queue"); + queue_families_[vk_graphics_queue_] = chosen_family; auto binding = XrInfo( XR_TYPE_GRAPHICS_BINDING_VULKAN2_KHR); @@ -840,6 +910,12 @@ bool OpenXrBackend::InitializeSessionResourcesLocked(std::string *error) { xrCreateReferenceSpace(session, &space_info, &view_space), "xrCreateReferenceSpace VIEW"); xr_view_space_ = view_space; + space_info.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_STAGE; + XrSpace stage_space = XR_NULL_HANDLE; + if (XR_SUCCEEDED( + xrCreateReferenceSpace(session, &space_info, &stage_space))) { + xr_stage_space_ = stage_space; + } auto system_properties = XrInfo(XR_TYPE_SYSTEM_PROPERTIES); @@ -1072,7 +1148,7 @@ void OpenXrBackend::TeardownSessionLocked(const char* reason, bool preserve_devi // Destruction also ends a running session; xrEndSession is legal only in // STOPPING (handled by PollSessionEvents). if (vk_ && vk_device_) { - (void)vk_->DeviceWaitIdle(vk_device_); + (void)IdleHostDevice(vk_->DeviceWaitIdle, vk_device_); // Mirror staging lives on this device; free it before the device goes. mirror_.DestroyStaging(MirrorProcs()); } @@ -1081,7 +1157,14 @@ void OpenXrBackend::TeardownSessionLocked(const char* reason, bool preserve_devi DestroySwapchains(); // Action pose spaces belong to this session; detach before the session and // its spaces go away so a recreated session attaches cleanly. - { std::lock_guard lock(controller_mutex_); actions_.Detach(); } + { + std::lock_guard lock(controller_mutex_); + actions_.Detach(); + } + if (xr_stage_space_ != nullptr) { + (void)xrDestroySpace(static_cast(xr_stage_space_)); + xr_stage_space_ = nullptr; + } if (xr_view_space_ != nullptr) { (void)xrDestroySpace(static_cast(xr_view_space_)); xr_view_space_ = nullptr; @@ -1100,7 +1183,7 @@ void OpenXrBackend::TeardownSessionLocked(const char* reason, bool preserve_devi frame_open_ = false; frame_views_valid_ = false; visibility_lost_ = false; - canted_rejection_logged_ = false; + projection_rejection_logged_ = false; eyes_[0] = {}; eyes_[1] = {}; if (gles_) @@ -1125,6 +1208,12 @@ void OpenXrBackend::TeardownSessionLocked(const char* reason, bool preserve_devi vk_device_ = VK_NULL_HANDLE; vk_physical_device_ = VK_NULL_HANDLE; queue_families_.clear(); + vk_graphics_queue_ = VK_NULL_HANDLE; + device_queue_families_.clear(); + command_pool_families_.clear(); + command_records_.clear(); + eye_render_queues_.clear(); + eye_render_frames_.clear(); delete vk_; vk_ = nullptr; } @@ -1155,6 +1244,137 @@ void OpenXrBackend::NoteQueue(VkDevice device, VkQueue queue, } } +std::vector +OpenXrBackend::DesktopQueueFamilies(VkDevice device) const { + std::lock_guard lock(mutex_); + return device == vk_device_ ? device_queue_families_ + : std::vector{}; +} + +void OpenXrBackend::RecordCommandPool(VkDevice device, VkCommandPool pool, + std::uint32_t family) { + std::lock_guard lock(mutex_); + if (device == vk_device_) + command_pool_families_[pool] = family; +} + +void OpenXrBackend::RecordCommandBuffers(VkDevice device, VkCommandPool pool, + std::uint32_t count, + const VkCommandBuffer *buffers) { + std::lock_guard lock(mutex_); + const auto found = command_pool_families_.find(pool); + if (device != vk_device_ || found == command_pool_families_.end() || !buffers) + return; + for (std::uint32_t i = 0; i < count; ++i) + command_records_[buffers[i]] = {pool, found->second, {}}; +} + +void OpenXrBackend::ForgetCommandBuffers(std::uint32_t count, + const VkCommandBuffer *buffers) { + if (buffers) + for (std::uint32_t i = 0; i < count; ++i) + ResetCommandBuffer(buffers[i]); + std::lock_guard lock(mutex_); + if (buffers) + for (std::uint32_t i = 0; i < count; ++i) + command_records_.erase(buffers[i]); +} + +void OpenXrBackend::ForgetCommandPool(VkCommandPool pool) { + ResetCommandPool(pool); + std::lock_guard lock(mutex_); + command_pool_families_.erase(pool); + for (auto it = command_records_.begin(); it != command_records_.end();) { + if (it->second.pool == pool) + it = command_records_.erase(it); + else + ++it; + } +} + +void OpenXrBackend::ResetCommandBuffer(VkCommandBuffer buffer) { + std::lock_guard lock(mutex_); + const auto found = command_records_.find(buffer); + if (found != command_records_.end()) { + for (const auto &image : found->second.eye_images) { + const auto frame = eye_render_frames_.find(image.first); + if (frame != eye_render_frames_.end() && frame->second == image.second) { + eye_render_frames_.erase(frame); + eye_render_queues_.erase(image.first); + } + } + found->second.eye_images.clear(); + } +} + +void OpenXrBackend::ResetCommandPool(VkCommandPool pool) { + std::lock_guard lock(mutex_); + for (auto &entry : command_records_) { + if (entry.second.pool != pool) + continue; + for (const auto &image : entry.second.eye_images) { + const auto frame = eye_render_frames_.find(image.first); + if (frame != eye_render_frames_.end() && frame->second == image.second) { + eye_render_frames_.erase(frame); + eye_render_queues_.erase(image.first); + } + } + entry.second.eye_images.clear(); + } +} + +void OpenXrBackend::NoteCommandBuffersSubmitted( + VkQueue queue, std::uint32_t count, const VkCommandBuffer *buffers) { + std::lock_guard lock(mutex_); + const auto family = queue_families_.find(queue); + if (family == queue_families_.end() || !buffers) + return; + for (std::uint32_t i = 0; i < count; ++i) { + const auto command = command_records_.find(buffers[i]); + if (command == command_records_.end() || + command->second.family != family->second) + continue; + for (const auto &image : command->second.eye_images) { + eye_render_queues_[image.first] = queue; + eye_render_frames_[image.first] = image.second; + } + } +} + +VkImageLayout OpenXrBackend::NormalizeDesktopBarrier(VkCommandBuffer command, + VkImage image, + std::uint32_t *source, + std::uint32_t *destination, + VkImageLayout *old_layout, + VkImageLayout new_layout) { + std::lock_guard lock(mutex_); + const auto found = command_records_.find(command); + if (found == command_records_.end() || *source == *destination || + *source >= VK_QUEUE_FAMILY_FOREIGN_EXT || + *destination >= VK_QUEUE_FAMILY_FOREIGN_EXT) + return new_layout; + for (const auto &entry : desktop_swapchains_) { + const auto &swapchain = entry.second; + if (!swapchain.converted_to_concurrent || + std::find(swapchain.images.begin(), swapchain.images.end(), image) == + swapchain.images.end()) + continue; + // Concurrent sharing: graphics->present transitions on release; + // PRESENT/UNDEFINED->graphics transitions on acquire. + if (found->second.family == *destination && + *old_layout != VK_IMAGE_LAYOUT_PRESENT_SRC_KHR && + *old_layout != VK_IMAGE_LAYOUT_UNDEFINED) + *old_layout = new_layout; + // Keep PRESENT until graphics acquire, even with an explicit release. + if (found->second.family == *source && + *old_layout == VK_IMAGE_LAYOUT_PRESENT_SRC_KHR) + new_layout = *old_layout; + *source = *destination = VK_QUEUE_FAMILY_IGNORED; + return new_layout; + } + return new_layout; +} + void OpenXrBackend::RecordImage(VkDevice device, VkImage image, const VkImageCreateInfo* info) { if (!recording_.load(std::memory_order_acquire) || info == nullptr) { @@ -1172,6 +1392,7 @@ void OpenXrBackend::RecordImage(VkDevice device, VkImage image, record.height = info->extent.height; record.format = info->format; record.usage = info->usage; + record.samples = info->samples; record.owner = owner; record.order = ++image_order_; images_[image] = record; @@ -1220,6 +1441,16 @@ void OpenXrBackend::NoteDestroyImage(VkImage image) { } std::lock_guard lock(mutex_); images_.erase(image); + eye_render_queues_.erase(image); + eye_render_frames_.erase(image); + for (auto &command : command_records_) { + auto &recorded = command.second.eye_images; + recorded.erase(std::remove_if(recorded.begin(), recorded.end(), + [image](const auto &value) { + return value.first == image; + }), + recorded.end()); + } for (int index = 0; index < 2; ++index) { if (eyes_[index].valid && eyes_[index].image == image) { Log(" [vr-backend] eye%d image destroyed; eye binding reset\n", index); @@ -1283,6 +1514,7 @@ void OpenXrBackend::NoteRenderPassBegin(VkCommandBuffer command_buffer, const auto image = images_.find(source->second); if (image == images_.end() || image->second.owner != owner || image->second.device != vk_device_ || !IsColorFormat(image->second.format) || + image->second.samples != VK_SAMPLE_COUNT_1_BIT || (image->second.usage & VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) == 0 || (image->second.usage & VK_IMAGE_USAGE_TRANSFER_SRC_BIT) == 0) { continue; @@ -1320,13 +1552,23 @@ void OpenXrBackend::NoteRenderPassBegin(VkCommandBuffer command_buffer, binding.height = target->height; binding.format = target->format; binding.final_layout = layouts->second[target_attachment]; - if (changed) Log(" [vr-backend][evidence] eye-bound eye=%d owner=%p guest_fb=%p " - "vk_fb=%p vk_image=%p extent=%ux%u format=%d final_layout=%d " - "provenance=guest_initializer_and_eye_getter\n", - eye_index, owner, guest_framebuffer, - static_cast(info->framebuffer), static_cast(target->image), - target->width, target->height, static_cast(target->format), - static_cast(binding.final_layout)); + const auto command = command_records_.find(command_buffer); + if (command != command_records_.end()) { + const auto recorded = std::make_pair(target->image, binding.frame); + if (std::find(command->second.eye_images.begin(), + command->second.eye_images.end(), + recorded) == command->second.eye_images.end()) + command->second.eye_images.push_back(recorded); + } + if (changed) + Log(" [vr-backend][evidence] eye-bound eye=%d owner=%p guest_fb=%p " + "vk_fb=%p vk_image=%p extent=%ux%u format=%d final_layout=%d " + "provenance=guest_initializer_and_eye_getter\n", + eye_index, owner, guest_framebuffer, + static_cast(info->framebuffer), + static_cast(target->image), target->width, target->height, + static_cast(target->format), + static_cast(binding.final_layout)); // Evidence layer also observes this request; do not consume it here. if (eyes_[0].valid && eyes_[1].valid && eyes_[0].image != eyes_[1].image && eyes_[0].owner == eyes_[1].owner && @@ -1463,10 +1705,13 @@ void OpenXrBackend::PollSessionEvents() { const bool affects_us = change.session == static_cast(xr_session_) && (change.referenceSpaceType == XR_REFERENCE_SPACE_TYPE_LOCAL || - change.referenceSpaceType == XR_REFERENCE_SPACE_TYPE_VIEW); - // poseInPreviousSpace is undefined when poseValid is false. No application - // origin offset is applied: xrLocate* uses the requested display time. - Log(" [vr-backend] reference space change pending: type=%d time=%lld poseValid=%d\n", + change.referenceSpaceType == XR_REFERENCE_SPACE_TYPE_VIEW || + change.referenceSpaceType == XR_REFERENCE_SPACE_TYPE_STAGE); + // poseInPreviousSpace is undefined when poseValid is false. No + // application origin offset is applied: xrLocate* uses the requested + // display time. + Log(" [vr-backend] reference space change pending: type=%d time=%lld " + "poseValid=%d\n", static_cast(change.referenceSpaceType), static_cast(change.changeTime), change.poseValid); if (affects_us) HandleReferenceSpaceChange(change.changeTime); @@ -1501,81 +1746,80 @@ void OpenXrBackend::HandleSessionState(int state) { std::fill(std::begin(published_pose_.controller_channels), std::end(published_pose_.controller_channels), 0.f); } switch (static_cast(state)) { - case XR_SESSION_STATE_READY: { - auto begin = - XrInfo(XR_TYPE_SESSION_BEGIN_INFO); - begin.primaryViewConfigurationType = - XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO; - const XrResult begin_result = - xrBeginSession(static_cast(xr_session_), &begin); - if (XR_SUCCEEDED(begin_result)) { - session_running_.store(true, std::memory_order_release); - visibility_lost_ = false; - Log(" [vr-backend] XR session READY; projection submission " - "starts on the next frame\n"); - } else { - Log(" [vr-backend] xrBeginSession failed: %s\n", - XrResultText(static_cast(xr_instance_), begin_result) - .c_str()); - } - break; - } - case XR_SESSION_STATE_STOPPING: { - if (frame_open_) { - auto end = XrInfo(XR_TYPE_FRAME_END_INFO); - end.displayTime = frame_display_time_; - end.environmentBlendMode = - static_cast(xr_blend_mode_); - (void)xrEndFrame(static_cast(xr_session_), &end); - frame_open_ = false; - } - if (session_running_.load(std::memory_order_relaxed)) { - (void)xrEndSession(static_cast(xr_session_)); - } - session_running_.store(false, std::memory_order_release); - Log(" [vr-backend] XR session STOPPING; submission paused\n"); - break; - } - case XR_SESSION_STATE_FOCUSED: - Log(" [vr-backend] XR session FOCUSED\n"); - if (visibility_lost_) { - // Returning from IDLE to a visible state: the frames rendered while - // idle are stale, so the first frame after regain is invalidated. - InvalidateInFlightPose("visibility regained (FOCUSED)"); - } - visibility_lost_ = false; - perf::ProcessCollector().Count(perf::Counter::kVisibilityGained); - break; - case XR_SESSION_STATE_VISIBLE: - Log(" [vr-backend] XR session VISIBLE (compositor shows our layers " - "without input focus)\n"); - if (visibility_lost_) { - InvalidateInFlightPose("visibility regained (VISIBLE)"); - } - visibility_lost_ = false; - perf::ProcessCollector().Count(perf::Counter::kVisibilityGained); - break; - case XR_SESSION_STATE_SYNCHRONIZED: - case XR_SESSION_STATE_IDLE: - Log(" [vr-backend] XR session not visible (IDLE/SYNCHRONIZED; runtime may " - "throttle frames)\n"); - visibility_lost_ = true; - InvalidateInFlightPose("session not visible"); - perf::ProcessCollector().Count(perf::Counter::kVisibilityLost); - break; - case XR_SESSION_STATE_EXITING: - Log(" [vr-backend] XR session EXITING\n"); - session_running_.store(false, std::memory_order_release); - TeardownSessionLocked("session exiting"); - return; - case XR_SESSION_STATE_LOSS_PENDING: - Log(" [vr-backend] XR session LOSS_PENDING\n"); - session_running_.store(false, std::memory_order_release); - TeardownSessionLocked("session loss pending", true); - return; - default: - break; + case XR_SESSION_STATE_READY: { + auto begin = XrInfo(XR_TYPE_SESSION_BEGIN_INFO); + begin.primaryViewConfigurationType = + XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO; + const XrResult begin_result = + xrBeginSession(static_cast(xr_session_), &begin); + if (XR_SUCCEEDED(begin_result)) { + session_running_.store(true, std::memory_order_release); + visibility_lost_ = false; + Log(" [vr-backend] XR session READY; projection submission " + "starts on the next frame\n"); + } else { + Log(" [vr-backend] xrBeginSession failed: %s\n", + XrResultText(static_cast(xr_instance_), begin_result) + .c_str()); } + break; + } + case XR_SESSION_STATE_STOPPING: { + if (frame_open_) { + auto end = XrInfo(XR_TYPE_FRAME_END_INFO); + end.displayTime = frame_display_time_; + end.environmentBlendMode = + static_cast(xr_blend_mode_); + (void)xrEndFrame(static_cast(xr_session_), &end); + frame_open_ = false; + } + if (session_running_.load(std::memory_order_relaxed)) { + (void)xrEndSession(static_cast(xr_session_)); + } + session_running_.store(false, std::memory_order_release); + Log(" [vr-backend] XR session STOPPING; submission paused\n"); + break; + } + case XR_SESSION_STATE_FOCUSED: + Log(" [vr-backend] XR session FOCUSED\n"); + if (visibility_lost_) { + // Returning from IDLE to a visible state: the frames rendered while + // idle are stale, so the first frame after regain is invalidated. + InvalidateInFlightPose("visibility regained (FOCUSED)"); + } + visibility_lost_ = false; + perf::ProcessCollector().Count(perf::Counter::kVisibilityGained); + break; + case XR_SESSION_STATE_VISIBLE: + Log(" [vr-backend] XR session VISIBLE (compositor shows our layers " + "without input focus)\n"); + if (visibility_lost_) { + InvalidateInFlightPose("visibility regained (VISIBLE)"); + } + visibility_lost_ = false; + perf::ProcessCollector().Count(perf::Counter::kVisibilityGained); + break; + case XR_SESSION_STATE_SYNCHRONIZED: + case XR_SESSION_STATE_IDLE: + Log(" [vr-backend] XR session not visible (IDLE/SYNCHRONIZED; runtime may " + "throttle frames)\n"); + visibility_lost_ = true; + InvalidateInFlightPose("session not visible"); + perf::ProcessCollector().Count(perf::Counter::kVisibilityLost); + break; + case XR_SESSION_STATE_EXITING: + Log(" [vr-backend] XR session EXITING\n"); + session_running_.store(false, std::memory_order_release); + TeardownSessionLocked("session exiting"); + return; + case XR_SESSION_STATE_LOSS_PENDING: + Log(" [vr-backend] XR session LOSS_PENDING\n"); + session_running_.store(false, std::memory_order_release); + TeardownSessionLocked("session loss pending", true); + return; + default: + break; + } } void OpenXrBackend::InvalidateInFlightPose(const char* reason) { @@ -1623,17 +1867,106 @@ bool OpenXrBackend::RecoverSessionLocked(std::uint64_t now_ns) { return false; } session_recovery_pending_ = false; - Log(" [vr-backend] XR session recreated on the existing guest device; awaiting READY\n"); + Log(" [vr-backend] XR session recreated on the existing guest device; " + "awaiting READY\n"); return true; } +VkResult internal::WaitForPresentDependencies(VkDevice device, VkQueue queue, + const VkPresentInfoKHR &present, + VkFence fence, + PFN_vkResetFences reset, + PFN_vkQueueSubmit submit, + PFN_vkWaitForFences wait) { + if (present.waitSemaphoreCount == 0) + return VK_SUCCESS; + if (!present.pWaitSemaphores) + return VK_ERROR_INITIALIZATION_FAILED; + std::vector stages(present.waitSemaphoreCount, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); + VkSubmitInfo info{}; + info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + info.waitSemaphoreCount = present.waitSemaphoreCount; + info.pWaitSemaphores = present.pWaitSemaphores; + info.pWaitDstStageMask = stages.data(); + VkResult result = reset(device, 1, &fence); + if (result != VK_SUCCESS) + return result; + result = SubmitHostQueue(submit, device, queue, 1, &info, fence); + if (result != VK_SUCCESS) + return result; + // The completed fence permits presentation without repeating these waits. + result = wait(device, 1, &fence, VK_TRUE, UINT64_MAX); + // A consumed binary semaphore cannot be retried after a timeout. + return result == VK_SUCCESS || result < 0 ? result : VK_ERROR_DEVICE_LOST; +} + +bool OpenXrBackend::SupportsPresentQueueLocked(VkQueue queue) const { + return GraphicsQueueForPresentLocked(queue) != VK_NULL_HANDLE; +} + +VkQueue OpenXrBackend::GraphicsQueueForPresentLocked(VkQueue present) const { + if (queue_families_.find(present) == queue_families_.end()) + return VK_NULL_HANDLE; + VkQueue graphics = vk_graphics_queue_; + for (const auto &eye : eyes_) { + if (!eye.valid) + continue; + const auto rendered = eye_render_queues_.find(eye.image); + // An eye attachment is not evidence that its command buffer was submitted. + if (rendered == eye_render_queues_.end()) + return VK_NULL_HANDLE; + const auto frame = eye_render_frames_.find(eye.image); + if (frame == eye_render_frames_.end() || frame->second != eye.frame) + return VK_NULL_HANDLE; + const auto family = queue_families_.find(rendered->second); + if (family == queue_families_.end() || family->second != vk_queue_family_) + return VK_NULL_HANDLE; + graphics = rendered->second; + } + return graphics; +} + +VkResult OpenXrBackend::PrepareHostPresent(VkQueue queue, VkDevice device, + const VkPresentInfoKHR *info) { + std::lock_guard lock(mutex_); + if (!armed_ || graphics_api_ != VrGraphicsApi::kVulkan || !vk_ || + device != vk_device_ || !info) + return VK_NOT_READY; + if (!SupportsPresentQueueLocked(queue)) { + if (!unsupported_present_queue_logged_) { + unsupported_present_queue_logged_ = true; + Log(" [vr-backend] waiting for verified eye-render submissions on " + "the OpenXR graphics family before preparing presentation\n"); + } + return VK_NOT_READY; + } + if (info->waitSemaphoreCount == 0) + return VK_SUCCESS; + std::string error; + if (!EnsureCopyResources(device, vk_queue_family_, &error)) { + Log(" [vr-backend] cannot synchronize present dependencies: %s\n", + error.c_str()); + return VK_ERROR_INITIALIZATION_FAILED; + } + return internal::WaitForPresentDependencies( + device, GraphicsQueueForPresentLocked(queue), *info, copy_fence_, + vk_->ResetFences, vk_->QueueSubmit, vk_->WaitForFences); +} + void OpenXrBackend::NoteHostPresent(VkQueue queue, VkDevice device) { std::lock_guard lock(mutex_); if (!armed_.load(std::memory_order_acquire) || device != vk_device_) { return; } + if (graphics_api_ == VrGraphicsApi::kVulkan && + !SupportsPresentQueueLocked(queue)) + return; + if (graphics_api_ == VrGraphicsApi::kVulkan) + queue = GraphicsQueueForPresentLocked(queue); try { - if (xr_session_ == nullptr && !RecoverSessionLocked(perf::NowNs())) return; + if (xr_session_ == nullptr && !RecoverSessionLocked(perf::NowNs())) + return; PollSessionEvents(); if (!session_running_.load(std::memory_order_acquire)) { return; @@ -1642,25 +1975,29 @@ void OpenXrBackend::NoteHostPresent(VkQueue queue, VkDevice device) { RunFrameCycle(queue, device); // Explicit, one-shot fault injection for the isolated correctness canary. // Never active in a normal authenticated/headset launch. - const auto loss_frame = EnvUnsigned("MOCKTAIL_VR_TEST_SESSION_LOSS_AFTER", 0); + const auto loss_frame = + EnvUnsigned("MOCKTAIL_VR_TEST_SESSION_LOSS_AFTER", 0); if (!test_session_loss_injected_ && loss_frame > 0 && EnvString("MOCKTAIL_ISOLATED_CANARY", "") == "1" && submitted_frames_.load() >= loss_frame) { test_session_loss_injected_ = true; - Log(" [vr-backend][test] injecting session loss on existing guest device\n"); + Log(" [vr-backend][test] injecting session loss on existing guest " + "device\n"); HandleSessionState(XR_SESSION_STATE_LOSS_PENDING); } perf::ProcessCollector().EmitIfDue(perf::NowNs(), "xr-frame-cycle"); - } catch (const XrCallError& error) { + } catch (const XrCallError &error) { if (error.result == XR_ERROR_SESSION_LOST) { - Log(" [vr-backend] XR session lost; scheduling recovery on the existing device\n"); + Log(" [vr-backend] XR session lost; scheduling recovery on the existing " + "device\n"); TeardownSessionLocked("XR_ERROR_SESSION_LOST", true); } else { - Log(" [vr-backend] XR frame failed: %s; restart the client if the runtime instance was lost\n", + Log(" [vr-backend] XR frame failed: %s; restart the client if the " + "runtime instance was lost\n", error.what()); TeardownSessionLocked("unrecoverable XR frame error"); } - } catch (const std::exception& error) { + } catch (const std::exception &error) { Log(" [vr-backend] frame cycle failed: %s; XR output stopped, native " "stereo continues\n", error.what()); @@ -1684,18 +2021,17 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { XrInfo( XR_TYPE_COMPOSITION_LAYER_PROJECTION); std::array views{}; - const XrCompositionLayerBaseHeader* layers[1] = { - reinterpret_cast(&projection)}; + const XrCompositionLayerBaseHeader *layers[1] = { + reinterpret_cast(&projection)}; const bool eyes_usable = eyes_bound_.load(std::memory_order_acquire) && eyes_[0].valid && eyes_[1].valid; const bool pose_ready = frame_views_valid_ && published_pose_.valid; const bool eye_frames_match = - eyes_usable && pose_ready && - eyes_[0].frame == published_pose_.frame && + eyes_usable && pose_ready && eyes_[0].frame == published_pose_.frame && eyes_[1].frame == published_pose_.frame && applied_poses_[eyes_[0].owner] == published_pose_.frame; - const bool can_submit = frame_should_render_ && - pose_ready && eye_frames_match && eyes_usable; + const bool can_submit = + frame_should_render_ && pose_ready && eye_frames_match && eyes_usable; if (!can_submit) { // One dominant reason per skipped frame, checked in pipeline order. if (visibility_lost_ && !frame_should_render_) { @@ -1734,7 +2070,8 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { const auto deadline = Clock::now() + std::chrono::seconds(5); while (true) { const XrResult wait_result = xrWaitSwapchainImage( - static_cast(eye_swapchains_[eye].swapchain), &wait); + static_cast(eye_swapchains_[eye].swapchain), + &wait); if (wait_result != XR_TIMEOUT_EXPIRED) { CheckXr(instance, wait_result, "xrWaitSwapchainImage"); break; @@ -1779,10 +2116,9 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { frame_view_poses_[eye].orientation[2], frame_view_poses_[eye].orientation[3]}; views[eye].pose = submit_pose; - views[eye].fov = XrFovf{frame_views_[eye].fov[0], - frame_views_[eye].fov[1], - frame_views_[eye].fov[2], - frame_views_[eye].fov[3]}; + views[eye].fov = + XrFovf{frame_views_[eye].fov[0], frame_views_[eye].fov[1], + frame_views_[eye].fov[2], frame_views_[eye].fov[3]}; views[eye].subImage.swapchain = static_cast(eye_swapchains_[eye].swapchain); views[eye].subImage.imageRect.offset = {0, 0}; @@ -1804,8 +2140,11 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { submitted_frames_.fetch_add(1, std::memory_order_relaxed) + 1; if (submitted == 1 || submitted % 120 == 0 || (!EnvString("MOCKTAIL_VR_XR_EVIDENCE_DIR", "").empty() && - published_pose_.frame % std::max(1, - EnvUnsigned("MOCKTAIL_VR_XR_EVIDENCE_INTERVAL", 20)) == 0)) { + published_pose_.frame % + std::max( + 1, + EnvUnsigned("MOCKTAIL_VR_XR_EVIDENCE_INTERVAL", 20)) == + 0)) { if (graphics_api_ == VrGraphicsApi::kOpenGles) { Log(" [vr-backend][evidence] projection submitted=%llu frame=%llu " "graphics=GLES eye0_texture=%u eye1_texture=%u\n", @@ -1873,16 +2212,15 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { } constexpr XrViewStateFlags required_view_flags = XR_VIEW_STATE_POSITION_VALID_BIT | XR_VIEW_STATE_ORIENTATION_VALID_BIT; - if (view_count == 2 && - (view_state.viewStateFlags & required_view_flags) == - required_view_flags) { + if (view_count == 2 && (view_state.viewStateFlags & required_view_flags) == + required_view_flags) { { // Head pose in LOCAL space for the guest camera injection. XrSpaceLocation head = XrInfo(XR_TYPE_SPACE_LOCATION); - const XrResult head_result = xrLocateSpace( - static_cast(xr_view_space_), - static_cast(xr_local_space_), predicted_display_time, - &head); + const XrResult head_result = + xrLocateSpace(static_cast(xr_view_space_), + static_cast(xr_local_space_), + predicted_display_time, &head); constexpr XrSpaceLocationFlags required_location = XR_SPACE_LOCATION_ORIENTATION_VALID_BIT | XR_SPACE_LOCATION_POSITION_VALID_BIT; @@ -1895,11 +2233,11 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { if (pose_source == "script") { const ScriptedPoseSample scripted = ComputeScriptedPose(++xr_frame_counter_); - head_pose = Pose{Vec3{scripted.position[0], scripted.position[1], - scripted.position[2]}, - Quat{scripted.orientation[0], scripted.orientation[1], - scripted.orientation[2], - scripted.orientation[3]}}; + head_pose = + Pose{Vec3{scripted.position[0], scripted.position[1], + scripted.position[2]}, + Quat{scripted.orientation[0], scripted.orientation[1], + scripted.orientation[2], scripted.orientation[3]}}; published_pose_ = scripted; published_pose_.valid = head_valid; published_pose_.frame = xr_frame_counter_; @@ -1916,6 +2254,24 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { sample.valid = head_valid; published_pose_ = sample; } + if (head_valid && xr_stage_space_ != nullptr) { + auto floor = XrInfo(XR_TYPE_SPACE_LOCATION); + const auto result = + xrLocateSpace(static_cast(xr_stage_space_), + static_cast(xr_local_space_), + predicted_display_time, &floor); + if (XR_SUCCEEDED(result) && + (floor.locationFlags & required_location) == required_location) { + published_pose_.floor_position[0] = floor.pose.position.x; + published_pose_.floor_position[1] = floor.pose.position.y; + published_pose_.floor_position[2] = floor.pose.position.z; + published_pose_.floor_orientation[0] = floor.pose.orientation.x; + published_pose_.floor_orientation[1] = floor.pose.orientation.y; + published_pose_.floor_orientation[2] = floor.pose.orientation.z; + published_pose_.floor_orientation[3] = floor.pose.orientation.w; + published_pose_.floor_valid = true; + } + } for (int eye = 0; eye < 2; ++eye) { const Pose runtime_eye = XrToPose(xr_views[eye].pose); // Preserve the runtime's inter-lens offsets while the scripted (or @@ -1938,32 +2294,23 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { published_pose_.eye_offset[eye][1] = 0.f; published_pose_.eye_offset[eye][2] = 0.f; } - // The pinned guest state represents parallel eye orientations. A - // canted runtime needs a different camera ABI; do not submit an - // image with a projection pose that the guest cannot represent. - // Reject once per binding with the measured components so a tester - // sees why nothing reaches the headset instead of a silent freeze. - // Only judged while the head pose itself is valid: during tracking - // loss `relative` degenerates to the full LOCAL-space eye pose and - // its rotation is not a canted-eye signal. + pose::EyeFov rendered_fov{ + xr_views[eye].fov.angleLeft, xr_views[eye].fov.angleRight, + xr_views[eye].fov.angleUp, xr_views[eye].fov.angleDown}; if (head_valid && - !pose::EyeOrientationIsParallel(relative.orientation)) { + !pose::ParallelEyeFov(relative.orientation, rendered_fov, + &rendered_fov)) { published_pose_.valid = false; - if (!canted_rejection_logged_) { - canted_rejection_logged_ = true; - Log(" [vr-backend] UNSUPPORTED CONFIGURATION: the runtime " - "reports canted eye orientations (eye%d relative rotation " - "xyz=(%.5f,%.5f,%.5f), tolerance %.3f). The exact-build " - "guest camera ABI represents only parallel eye orientations, " - "so XR submission is rejected rather than showing images " - "with a mismatched projection. This is a known unmet " - "compatibility criterion for this headset/runtime " - "combination.\n", - eye, relative.orientation.x, relative.orientation.y, - relative.orientation.z, pose::kCantedTolerance); + if (!projection_rejection_logged_) { + projection_rejection_logged_ = true; + Log(" [vr-backend] eye%d projection exceeds the guest camera " + "FOV; " + "XR submission paused\n", + eye); } } - const Pose final_eye = PoseMultiply(head_pose, relative); + const Pose final_eye = + PoseMultiply(head_pose, Pose{relative.position, {}}); frame_view_poses_[eye].position[0] = final_eye.position.x; frame_view_poses_[eye].position[1] = final_eye.position.y; frame_view_poses_[eye].position[2] = final_eye.position.z; @@ -1978,10 +2325,10 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { frame_views_[eye].orientation[1] = final_eye.orientation.y; frame_views_[eye].orientation[2] = final_eye.orientation.z; frame_views_[eye].orientation[3] = final_eye.orientation.w; - frame_views_[eye].fov[0] = xr_views[eye].fov.angleLeft; - frame_views_[eye].fov[1] = xr_views[eye].fov.angleRight; - frame_views_[eye].fov[2] = xr_views[eye].fov.angleUp; - frame_views_[eye].fov[3] = xr_views[eye].fov.angleDown; + frame_views_[eye].fov[0] = rendered_fov.angle_left; + frame_views_[eye].fov[1] = rendered_fov.angle_right; + frame_views_[eye].fov[2] = rendered_fov.angle_up; + frame_views_[eye].fov[3] = rendered_fov.angle_down; std::copy_n(frame_views_[eye].fov, 4, published_pose_.eye_fov[eye]); frame_views_[eye].valid = true; } @@ -2115,11 +2462,10 @@ void OpenXrBackend::DestroyCopyResources() { bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { std::string error; - const auto family_entry = queue_families_.find(queue); - const std::uint32_t family = - family_entry != queue_families_.end() ? family_entry->second - : vk_queue_family_; - if (!EnsureCopyResources(vk_device_, family, &error)) { + const auto family = queue_families_.find(queue); + if (family == queue_families_.end() || family->second != vk_queue_family_) + return false; + if (!EnsureCopyResources(vk_device_, vk_queue_family_, &error)) { Log(" [vr-backend] copy resources unavailable: %s\n", error.c_str()); return false; } @@ -2267,14 +2613,13 @@ bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; submit.commandBufferCount = 1; submit.pCommandBuffers = ©_command_; - // Same queue as the guest's frame submission: queue-order guarantees the - // batched copies run after the eye render passes of this frame. One frame in - // flight; the single fence wait bounds reuse of the copy command buffer and - // of the mirror staging buffers. + // Present dependencies are complete; this fence protects reuse of the + // copy command buffer and mirror staging buffers. VkResult submit_result = VK_SUCCESS; { const ScopedStageTimer submit_timer(perf::Stage::kCopySubmit); - submit_result = vk_->QueueSubmit(queue, 1, &submit, copy_fence_); + submit_result = SubmitHostQueue(vk_->QueueSubmit, vk_device_, queue, 1, + &submit, copy_fence_); } perf::ProcessCollector().Count(perf::Counter::kSubmits); if (submit_result != VK_SUCCESS) { @@ -2288,7 +2633,7 @@ bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { } perf::ProcessCollector().Count(perf::Counter::kFenceWaits); if (fence_result != VK_SUCCESS) { - (void)vk_->DeviceWaitIdle(vk_device_); + (void)IdleHostDevice(vk_->DeviceWaitIdle, vk_device_); throw std::runtime_error("XR copy fence wait failed"); } if (mirror_.enabled()) { @@ -2313,8 +2658,9 @@ void OpenXrBackend::EnsureMirrorForExtent(std::uint32_t width, Log(" [vr-backend] mirror unavailable: %s\n", error.c_str()); return; } - if (!was_enabled) Log(" [vr-backend] desktop mirror publishing to %s (%ux%u)\n", - name.c_str(), width, height); + if (!was_enabled) + Log(" [vr-backend] desktop mirror publishing to %s (%ux%u)\n", + name.c_str(), width, height); } std::string error; if (!mirror_.EnsureStaging(MirrorProcs(), vk_device_, vk_physical_device_, @@ -2345,45 +2691,84 @@ MirrorVkProcs OpenXrBackend::MirrorProcs() const { } // DebugDeviceVR renders offscreen eyes but does not mirror them to the host -// window. Keep this copy on the guest GPU; no readback or image files in normal use. -void OpenXrBackend::RecordDesktopSwapchain(VkDevice device, VkSwapchainKHR swapchain, - const VkSwapchainCreateInfoKHR* info, const VkImage* images, unsigned count) { +// window. Keep this copy on the guest GPU; no readback or image files in normal +// use. +void OpenXrBackend::RecordDesktopSwapchain(VkDevice device, + VkSwapchainKHR swapchain, + const VkSwapchainCreateInfoKHR *info, + const VkImage *images, + unsigned count, + bool converted_to_concurrent) { std::lock_guard lock(mutex_); desktop_swapchains_.erase(swapchain); if (!device || device != vk_device_ || !info || !images || !count || !(info->imageUsage & VK_IMAGE_USAGE_TRANSFER_DST_BIT) || - info->imageArrayLayers != 1 || info->imageSharingMode != VK_SHARING_MODE_EXCLUSIVE) + info->imageArrayLayers != 1) return; - desktop_swapchains_[swapchain] = {info->imageExtent, info->imageFormat, - {images, images + count}}; + DesktopSwapchain record{}; + record.extent = info->imageExtent; + record.format = info->imageFormat; + record.images.assign(images, images + count); + record.sharing_mode = info->imageSharingMode; + record.converted_to_concurrent = converted_to_concurrent; + if (info->imageSharingMode == VK_SHARING_MODE_CONCURRENT) { + if (!info->pQueueFamilyIndices || info->queueFamilyIndexCount < 2) + return; + record.families.assign(info->pQueueFamilyIndices, + info->pQueueFamilyIndices + + info->queueFamilyIndexCount); + } + desktop_swapchains_[swapchain] = std::move(record); } -VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, const VkPresentInfoKHR* info) { +VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, + const VkPresentInfoKHR *info) { std::lock_guard lock(mutex_); - if (EnvString("MOCKTAIL_VR_DESKTOP_MIRROR", "1") == "0" || - !vk_ || !info || info->swapchainCount != 1 || - !info->pSwapchains || !info->pImageIndices || !eyes_[0].valid || - !eyes_[0].image || (info->waitSemaphoreCount && !info->pWaitSemaphores) || - !queue_families_.count(queue) || queue_families_[queue] != vk_queue_family_) + if (EnvString("MOCKTAIL_VR_DESKTOP_MIRROR", "1") == "0" || !vk_ || !info || + info->swapchainCount != 1 || !info->pSwapchains || !info->pImageIndices || + !eyes_[0].valid || !eyes_[0].image || + (info->waitSemaphoreCount && !info->pWaitSemaphores) || + !SupportsPresentQueueLocked(queue)) return VK_NOT_READY; const auto found = desktop_swapchains_.find(info->pSwapchains[0]); if (found == desktop_swapchains_.end() || - info->pImageIndices[0] >= found->second.images.size()) return VK_NOT_READY; - const auto& target = found->second; + info->pImageIndices[0] >= found->second.images.size()) + return VK_NOT_READY; + const auto &target = found->second; + const auto present_family = queue_families_.at(queue); + if (target.sharing_mode == VK_SHARING_MODE_CONCURRENT) { + if (std::find(target.families.begin(), target.families.end(), + vk_queue_family_) == target.families.end() || + std::find(target.families.begin(), target.families.end(), + present_family) == target.families.end()) + return VK_NOT_READY; + } else if (present_family != vk_queue_family_) { + // A swapchain created before VR activation has no sharing adaptation. + return VK_NOT_READY; + } + queue = GraphicsQueueForPresentLocked(queue); VkFormatProperties source_properties{}, target_properties{}; - vk_->GetPhysicalDeviceFormatProperties(vk_physical_device_, eyes_[0].format, &source_properties); - vk_->GetPhysicalDeviceFormatProperties(vk_physical_device_, target.format, &target_properties); - if (!(source_properties.optimalTilingFeatures & VK_FORMAT_FEATURE_BLIT_SRC_BIT) || - !(target_properties.optimalTilingFeatures & VK_FORMAT_FEATURE_BLIT_DST_BIT)) return VK_NOT_READY; + vk_->GetPhysicalDeviceFormatProperties(vk_physical_device_, eyes_[0].format, + &source_properties); + vk_->GetPhysicalDeviceFormatProperties(vk_physical_device_, target.format, + &target_properties); + if (!(source_properties.optimalTilingFeatures & + VK_FORMAT_FEATURE_BLIT_SRC_BIT) || + !(target_properties.optimalTilingFeatures & + VK_FORMAT_FEATURE_BLIT_DST_BIT)) + return VK_NOT_READY; std::string error; - if (!EnsureCopyResources(vk_device_, vk_queue_family_, &error)) return VK_NOT_READY; - if (vk_->ResetCommandBuffer(copy_command_, 0) != VK_SUCCESS) return VK_NOT_READY; + if (!EnsureCopyResources(vk_device_, vk_queue_family_, &error)) + return VK_NOT_READY; + if (vk_->ResetCommandBuffer(copy_command_, 0) != VK_SUCCESS) + return VK_NOT_READY; VkCommandBufferBeginInfo begin{}; begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; - if (vk_->BeginCommandBuffer(copy_command_, &begin) != VK_SUCCESS) return VK_NOT_READY; + if (vk_->BeginCommandBuffer(copy_command_, &begin) != VK_SUCCESS) + return VK_NOT_READY; VkImageMemoryBarrier barriers[2]{}; - for (auto& b : barriers) { + for (auto &b : barriers) { b.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; b.srcQueueFamilyIndex = b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; @@ -2398,15 +2783,19 @@ VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, const VkPresentInfoKHR* inf barriers[1].newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; barriers[1].dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 2, barriers); + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, + nullptr, 2, barriers); const VkClearColorValue black{{0.f, 0.f, 0.f, 1.f}}; - vk_->CmdClearColorImage(copy_command_, barriers[1].image, barriers[1].newLayout, - &black, 1, &barriers[1].subresourceRange); + vk_->CmdClearColorImage(copy_command_, barriers[1].image, + barriers[1].newLayout, &black, 1, + &barriers[1].subresourceRange); VkMemoryBarrier clear_barrier{}; clear_barrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; - clear_barrier.srcAccessMask = clear_barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + clear_barrier.srcAccessMask = clear_barrier.dstAccessMask = + VK_ACCESS_TRANSFER_WRITE_BIT; vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &clear_barrier, 0, nullptr, 0, nullptr); + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &clear_barrier, + 0, nullptr, 0, nullptr); const double scale = std::min(double(target.extent.width) / eyes_[0].width, double(target.extent.height) / eyes_[0].height); const int width = std::max(1, int(eyes_[0].width * scale)); @@ -2414,22 +2803,28 @@ VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, const VkPresentInfoKHR* inf const int x = (int(target.extent.width) - width) / 2; const int y = (int(target.extent.height) - height) / 2; VkImageBlit region{}; - region.srcSubresource = region.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; - region.srcOffsets[1] = {static_cast(eyes_[0].width), static_cast(eyes_[0].height), 1}; + region.srcSubresource = + region.dstSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.srcOffsets[1] = {static_cast(eyes_[0].width), + static_cast(eyes_[0].height), 1}; region.dstOffsets[0] = {x, y, 0}; region.dstOffsets[1] = {x + width, y + height, 1}; vk_->CmdBlitImage(copy_command_, barriers[0].image, barriers[0].newLayout, - barriers[1].image, barriers[1].newLayout, 1, ®ion, VK_FILTER_NEAREST); - for (auto& b : barriers) { + barriers[1].image, barriers[1].newLayout, 1, ®ion, + VK_FILTER_NEAREST); + for (auto &b : barriers) { std::swap(b.oldLayout, b.newLayout); b.srcAccessMask = b.dstAccessMask; b.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; } vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 2, barriers); + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, + nullptr, 2, barriers); if (vk_->EndCommandBuffer(copy_command_) != VK_SUCCESS || - vk_->ResetFences(vk_device_, 1, ©_fence_) != VK_SUCCESS) return VK_NOT_READY; - std::vector stages(info->waitSemaphoreCount, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); + vk_->ResetFences(vk_device_, 1, ©_fence_) != VK_SUCCESS) + return VK_NOT_READY; + std::vector stages(info->waitSemaphoreCount, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); VkSubmitInfo submit{}; submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; submit.waitSemaphoreCount = info->waitSemaphoreCount; @@ -2437,8 +2832,10 @@ VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, const VkPresentInfoKHR* inf submit.pWaitDstStageMask = stages.data(); submit.commandBufferCount = 1; submit.pCommandBuffers = ©_command_; - VkResult result = vk_->QueueSubmit(queue, 1, &submit, copy_fence_); - if (result != VK_SUCCESS) return result; + VkResult result = SubmitHostQueue(vk_->QueueSubmit, vk_device_, queue, 1, + &submit, copy_fence_); + if (result != VK_SUCCESS) + return result; // Once submitted, never fall back to waiting on the consumed semaphores. result = vk_->WaitForFences(vk_device_, 1, ©_fence_, VK_TRUE, UINT64_MAX); return result; @@ -2448,60 +2845,74 @@ VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, const VkPresentInfoKHR* inf // still acquired. Never read a released image owned by the compositor. bool OpenXrBackend::CaptureEyeEvidence(VkQueue queue, int eye) { const std::string directory = EnvString("MOCKTAIL_VR_XR_EVIDENCE_DIR", ""); - const auto interval = std::max(1, - EnvUnsigned("MOCKTAIL_VR_XR_EVIDENCE_INTERVAL", 20)); - if (directory.empty() || published_pose_.frame % interval != 0) return true; + const auto interval = std::max( + 1, EnvUnsigned("MOCKTAIL_VR_XR_EVIDENCE_INTERVAL", 20)); + if (directory.empty() || published_pose_.frame % interval != 0) + return true; VkBuffer buffers[2] = {}; VkDeviceMemory memory[2] = {}; bool pending = false; auto cleanup = [&] { - if (pending) vk_->DeviceWaitIdle(vk_device_); + if (pending) + IdleHostDevice(vk_->DeviceWaitIdle, vk_device_); for (int i = 0; i < 2; ++i) { - if (buffers[i]) vk_->DestroyBuffer(vk_device_, buffers[i], nullptr); - if (memory[i]) vk_->FreeMemory(vk_device_, memory[i], nullptr); + if (buffers[i]) + vk_->DestroyBuffer(vk_device_, buffers[i], nullptr); + if (memory[i]) + vk_->FreeMemory(vk_device_, memory[i], nullptr); } }; try { - const auto& binding = eyes_[eye]; - const auto& slot = eye_swapchains_[eye]; - const VkDeviceSize size = VkDeviceSize(binding.width)*binding.height*4; + const auto &binding = eyes_[eye]; + const auto &slot = eye_swapchains_[eye]; + const VkDeviceSize size = VkDeviceSize(binding.width) * binding.height * 4; const VkImage images[2] = {binding.image, slot.images[slot.acquired_index]}; const VkFormat formats[2] = {binding.format, slot.format}; - const VkImageLayout layouts[2] = {binding.final_layout, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}; + const VkImageLayout layouts[2] = {binding.final_layout, + VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}; VkPhysicalDeviceMemoryProperties properties{}; vk_->GetPhysicalDeviceMemoryProperties(vk_physical_device_, &properties); for (int i = 0; i < 2; ++i) { Require(formats[i] == VK_FORMAT_R8G8B8A8_UNORM || - formats[i] == VK_FORMAT_B8G8R8A8_UNORM, + formats[i] == VK_FORMAT_B8G8R8A8_UNORM, "evidence currently requires 8-bit UNORM images"); VkBufferCreateInfo info{}; info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; info.size = size; info.usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT; - Require(vk_->CreateBuffer(vk_device_, &info, nullptr, &buffers[i]) == VK_SUCCESS, + Require(vk_->CreateBuffer(vk_device_, &info, nullptr, &buffers[i]) == + VK_SUCCESS, "evidence buffer creation failed"); VkMemoryRequirements requirements{}; vk_->GetBufferMemoryRequirements(vk_device_, buffers[i], &requirements); std::uint32_t type = UINT32_MAX; - const auto flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + const auto flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; for (std::uint32_t t = 0; t < properties.memoryTypeCount; ++t) if ((requirements.memoryTypeBits & (1u << t)) && - (properties.memoryTypes[t].propertyFlags & flags) == flags) { type = t; break; } + (properties.memoryTypes[t].propertyFlags & flags) == flags) { + type = t; + break; + } Require(type != UINT32_MAX, "no coherent evidence readback memory"); VkMemoryAllocateInfo allocation{}; allocation.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = type; - Require(vk_->AllocateMemory(vk_device_, &allocation, nullptr, &memory[i]) == VK_SUCCESS, + Require(vk_->AllocateMemory(vk_device_, &allocation, nullptr, + &memory[i]) == VK_SUCCESS, "evidence memory allocation failed"); - Require(vk_->BindBufferMemory(vk_device_, buffers[i], memory[i], 0) == VK_SUCCESS, + Require(vk_->BindBufferMemory(vk_device_, buffers[i], memory[i], 0) == + VK_SUCCESS, "evidence buffer binding failed"); } - Require(vk_->ResetCommandBuffer(copy_command_, 0) == VK_SUCCESS, "evidence reset failed"); + Require(vk_->ResetCommandBuffer(copy_command_, 0) == VK_SUCCESS, + "evidence reset failed"); VkCommandBufferBeginInfo begin{}; begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; - Require(vk_->BeginCommandBuffer(copy_command_, &begin) == VK_SUCCESS, "evidence begin failed"); + Require(vk_->BeginCommandBuffer(copy_command_, &begin) == VK_SUCCESS, + "evidence begin failed"); for (int i = 0; i < 2; ++i) { VkImageMemoryBarrier barrier{}; barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; @@ -2509,50 +2920,64 @@ bool OpenXrBackend::CaptureEyeEvidence(VkQueue queue, int eye) { barrier.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; barrier.oldLayout = layouts[i]; barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; - barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.srcQueueFamilyIndex = barrier.dstQueueFamilyIndex = + VK_QUEUE_FAMILY_IGNORED; barrier.image = images[i]; barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, + nullptr, 1, &barrier); VkBufferImageCopy region{}; region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; region.imageExtent = {binding.width, binding.height, 1}; - vk_->CmdCopyImageToBuffer(copy_command_, images[i], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - buffers[i], 1, ®ion); + vk_->CmdCopyImageToBuffer(copy_command_, images[i], + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + buffers[i], 1, ®ion); barrier.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; - barrier.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; + barrier.dstAccessMask = + VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT; std::swap(barrier.oldLayout, barrier.newLayout); vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &barrier); + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, + 0, nullptr, 1, &barrier); } - Require(vk_->EndCommandBuffer(copy_command_) == VK_SUCCESS, "evidence end failed"); - Require(vk_->ResetFences(vk_device_, 1, ©_fence_) == VK_SUCCESS, "evidence fence reset failed"); + Require(vk_->EndCommandBuffer(copy_command_) == VK_SUCCESS, + "evidence end failed"); + Require(vk_->ResetFences(vk_device_, 1, ©_fence_) == VK_SUCCESS, + "evidence fence reset failed"); VkSubmitInfo submit{}; submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; submit.commandBufferCount = 1; submit.pCommandBuffers = ©_command_; - Require(vk_->QueueSubmit(queue, 1, &submit, copy_fence_) == VK_SUCCESS, "evidence submit failed"); + Require(SubmitHostQueue(vk_->QueueSubmit, vk_device_, queue, 1, &submit, + copy_fence_) == VK_SUCCESS, + "evidence submit failed"); pending = true; - Require(vk_->WaitForFences(vk_device_, 1, ©_fence_, VK_TRUE, 2000000000ULL) == VK_SUCCESS, + Require(vk_->WaitForFences(vk_device_, 1, ©_fence_, VK_TRUE, + 2000000000ULL) == VK_SUCCESS, "evidence fence wait failed"); pending = false; std::filesystem::create_directories(directory); const auto stem = std::filesystem::path(directory) / - ("frame-" + std::to_string(published_pose_.frame) + "-eye" + std::to_string(eye)); + ("frame-" + std::to_string(published_pose_.frame) + + "-eye" + std::to_string(eye)); for (int i = 0; i < 2; ++i) { - void* mapped = nullptr; - Require(vk_->MapMemory(vk_device_, memory[i], 0, size, 0, &mapped) == VK_SUCCESS, + void *mapped = nullptr; + Require(vk_->MapMemory(vk_device_, memory[i], 0, size, 0, &mapped) == + VK_SUCCESS, "evidence map failed"); - const auto* bytes = static_cast(mapped); - std::vector rgb(size/4*3); + const auto *bytes = static_cast(mapped); + std::vector rgb(size / 4 * 3); const bool bgra = formats[i] == VK_FORMAT_B8G8R8A8_UNORM; - for (std::size_t pixel = 0; pixel < size/4; ++pixel) { - rgb[pixel*3] = bytes[pixel*4 + (bgra ? 2 : 0)]; - rgb[pixel*3+1] = bytes[pixel*4+1]; - rgb[pixel*3+2] = bytes[pixel*4 + (bgra ? 0 : 2)]; + for (std::size_t pixel = 0; pixel < size / 4; ++pixel) { + rgb[pixel * 3] = bytes[pixel * 4 + (bgra ? 2 : 0)]; + rgb[pixel * 3 + 1] = bytes[pixel * 4 + 1]; + rgb[pixel * 3 + 2] = bytes[pixel * 4 + (bgra ? 0 : 2)]; } vk_->UnmapMemory(vk_device_, memory[i]); - std::ofstream file(stem.string() + (i ? "-destination.ppm" : "-source.ppm"), std::ios::binary); + std::ofstream file(stem.string() + + (i ? "-destination.ppm" : "-source.ppm"), + std::ios::binary); file << "P6\n" << binding.width << ' ' << binding.height << "\n255\n"; file.write(reinterpret_cast(rgb.data()), rgb.size()); Require(file.good(), "evidence image write failed"); @@ -2578,6 +3003,14 @@ bool OpenXrBackend::CaptureEyeEvidence(VkQueue queue, int eye) { // inert while no backend is armed; the adapter falls back to its plain host // loader path whenever a hook returns false. extern "C" { +void mocktail_vr_xr_device_idle_adapter( + mocktail::vr::SynchronizedDeviceIdle idle) { + mocktail::vr::g_device_idle_adapter.store(idle, std::memory_order_release); +} +void mocktail_vr_xr_queue_submit_adapter( + mocktail::vr::SynchronizedQueueSubmit submit) { + mocktail::vr::g_queue_submit_adapter.store(submit, std::memory_order_release); +} bool mocktail_vr_xr_create_vulkan_instance( const VkInstanceCreateInfo* create_info, @@ -2695,7 +3128,70 @@ void mocktail_vr_desktop_swapchain(VkDevice device, VkSwapchainKHR swapchain, std::fprintf(stderr, " [vr-backend] desktop swapchain registration failed: %s\n", error.what()); } } -VkResult mocktail_vr_desktop_present(VkQueue queue, const VkPresentInfoKHR* info) { +void mocktail_vr_desktop_swapchain_shared(VkDevice device, + VkSwapchainKHR swapchain, + const VkSwapchainCreateInfoKHR *info, + const VkImage *images, unsigned count, + bool converted) { + if (auto *backend = mocktail::vr::ActiveVrBackend()) + backend->RecordDesktopSwapchain(device, swapchain, info, images, count, + converted); +} +unsigned mocktail_vr_desktop_queue_families(VkDevice device, unsigned capacity, + unsigned *families) { + if (auto *backend = mocktail::vr::ActiveVrBackend()) { + const auto values = backend->DesktopQueueFamilies(device); + if (capacity < values.size() || !families) + return 0; + std::copy(values.begin(), values.end(), families); + return values.size(); + } + return 0; +} +void mocktail_vr_xr_command_pool(VkDevice device, VkCommandPool pool, + unsigned family) { + if (auto *backend = mocktail::vr::ActiveVrBackend()) { + if (family == UINT32_MAX) + backend->ForgetCommandPool(pool); + else + backend->RecordCommandPool(device, pool, family); + } +} +void mocktail_vr_xr_command_buffers(VkDevice device, VkCommandPool pool, + unsigned count, + const VkCommandBuffer *buffers) { + if (auto *backend = mocktail::vr::ActiveVrBackend()) { + if (pool == VK_NULL_HANDLE) + backend->ForgetCommandBuffers(count, buffers); + else + backend->RecordCommandBuffers(device, pool, count, buffers); + } +} +void mocktail_vr_xr_reset_command(VkCommandBuffer buffer) { + if (auto *backend = mocktail::vr::ActiveVrBackend()) + backend->ResetCommandBuffer(buffer); +} +void mocktail_vr_xr_reset_pool(VkCommandPool pool) { + if (auto *backend = mocktail::vr::ActiveVrBackend()) + backend->ResetCommandPool(pool); +} +void mocktail_vr_xr_submitted(VkQueue queue, unsigned count, + const VkCommandBuffer *buffers) { + if (auto *backend = mocktail::vr::ActiveVrBackend()) + backend->NoteCommandBuffersSubmitted(queue, count, buffers); +} +VkImageLayout mocktail_vr_xr_desktop_barrier(VkCommandBuffer command, + VkImage image, unsigned *source, + unsigned *destination, + VkImageLayout *old_layout, + VkImageLayout new_layout) { + if (auto *backend = mocktail::vr::ActiveVrBackend()) + return backend->NormalizeDesktopBarrier(command, image, source, destination, + old_layout, new_layout); + return new_layout; +} +VkResult mocktail_vr_desktop_present(VkQueue queue, + const VkPresentInfoKHR *info) { try { if (auto* backend = mocktail::vr::ActiveVrBackend()) return backend->MirrorDesktop(queue, info); return VK_NOT_READY; @@ -2712,7 +3208,20 @@ void mocktail_vr_xr_note_present(VkQueue queue, VkDevice device) { } } -} // extern "C" +VkResult mocktail_vr_xr_prepare_present(VkQueue queue, VkDevice device, + const VkPresentInfoKHR *info) { + try { + if (auto *backend = mocktail::vr::ActiveVrBackend()) + return backend->PrepareHostPresent(queue, device, info); + return VK_NOT_READY; + } catch (const std::exception &error) { + std::fprintf(stderr, " [vr-backend] present synchronization failed: %s\n", + error.what()); + return VK_ERROR_OUT_OF_HOST_MEMORY; + } +} + +} // extern "C" // Controller delivery ABI consumed by the input runtime through dlsym. These // are defined only in the VR build; a non-VR build omits them and the consumer diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc index 06e4471c..0dfb2db1 100644 --- a/src/vr/roblox_vr_device_bridge.cc +++ b/src/vr/roblox_vr_device_bridge.cc @@ -43,6 +43,14 @@ using internal::kStateGetterVtableSlot; using internal::kTextureSlotOffset; using internal::kWidthOffset; +std::uintptr_t RelativeTarget(const std::uint8_t* displacement, + std::uintptr_t next_instruction_rva) { + std::int32_t relative; + std::memcpy(&relative, displacement, sizeof(relative)); + return static_cast( + static_cast(next_instruction_rva) + relative); +} + std::atomic g_active_bridge{nullptr}; // Evidence queries expose only the current native call's provenance. These // pointers are identities for the layer, never retained/dereferenced by it. @@ -55,8 +63,7 @@ std::mutex g_native_pose_mutex; ScriptedPoseSample g_native_pose; using PointerFrameFn = void *(*)(void *, void *); std::atomic g_pointer_original{nullptr}; -constexpr std::uintptr_t kPointerFrameRva = 0x4d573ee; -// Whole instructions, no RIP-relative operands. 2998's sret prologue. +// Whole instructions, no RIP-relative operands; shared sret prologue. constexpr std::array kPointerPrologue = { 0x55, 0x48, 0x89, 0xe5, 0x41, 0x57, 0x41, 0x56, 0x41, 0x54, 0x53, 0x48, 0x83, 0xec, 0x70}; @@ -75,50 +82,7 @@ extern "C" void *mocktail_vr_pointer_frame(void *result, void *service) { original(result, service); auto *bridge = g_active_bridge.load(std::memory_order_acquire); if (!bridge || !bridge->active()) return result; - ScriptedPoseSample native_pose; - { - std::lock_guard lock(g_native_pose_mutex); - native_pose = g_native_pose; - } - if (!native_pose.valid) return result; - auto *backend = ActiveVrBackend(); - if (!backend) - return result; - const auto current = backend->PublishedHeadPose(); - if (!current.valid) - return result; - int index = 0; - std::memcpy(&index, static_cast(service) + 0x118, 4); - if (index != 1 && index != 2) - return result; - if (!current.hands[index - 1].aim_valid) - return result; - const auto &hand = native_pose.hands[index - 1]; - if (!hand.grip_valid || !hand.aim_valid) - return result; - const auto base = bridge->image_base(); - // Same camera lookup as the original getter (see pointer-world-2998.asm). - auto get_workspace = reinterpret_cast(base + 0x263778c); - void *workspace = get_workspace(service); - if (!workspace) - return result; - auto *vtable = *static_cast(workspace); - void *camera = - reinterpret_cast(vtable[0x3a8 / 8])(workspace); - if (!camera) - return result; - float scale = 0; - int degrees = 0; - std::memcpy(&scale, static_cast(camera) + 0x140, 4); - std::memcpy(°rees, reinterpret_cast(base + 0x73677a8), 4); - if (internal::ApplyAimToWorldFrame(static_cast(result), hand, scale, - -static_cast(degrees))) { - static std::atomic logged[2]{}; - if (!logged[index - 1].exchange(true)) - std::fprintf( - stderr, " [vr-input] native menu pointer uses OpenXR aim: hand=%d\n", - index - 1); - } + bridge->OnPointerFrameCall(result, service); return result; } @@ -133,10 +97,10 @@ bool SetCodeWritable(std::uintptr_t target, bool writable) { PROT_READ | PROT_EXEC | (writable ? PROT_WRITE : 0)) == 0; } -bool InstallPointerHook(std::uintptr_t base) { +bool InstallPointerHook(std::uintptr_t base, std::uintptr_t pointer_rva) { if (g_pointer_target) return false; - auto *target = reinterpret_cast(base + kPointerFrameRva); + auto* target = reinterpret_cast(base + pointer_rva); if (std::memcmp(target, kPointerPrologue.data(), kPointerPrologue.size()) != 0) return false; @@ -148,9 +112,9 @@ bool InstallPointerHook(std::uintptr_t base) { return false; std::memcpy(code, target, kPointerPrologue.size()); AbsoluteJump(code + kPointerPrologue.size(), - base + kPointerFrameRva + kPointerPrologue.size()); + base + pointer_rva + kPointerPrologue.size()); if (mprotect(code, page_size, PROT_READ | PROT_EXEC) != 0 || - !SetCodeWritable(base + kPointerFrameRva, true)) { + !SetCodeWritable(base + pointer_rva, true)) { munmap(code, page_size); return false; } @@ -159,14 +123,14 @@ bool InstallPointerHook(std::uintptr_t base) { AbsoluteJump(target, reinterpret_cast(&mocktail_vr_pointer_frame)); target[14] = 0x90; - if (!SetCodeWritable(base + kPointerFrameRva, false)) { + if (!SetCodeWritable(base + pointer_rva, false)) { std::memcpy(target, kPointerPrologue.data(), kPointerPrologue.size()); - (void)SetCodeWritable(base + kPointerFrameRva, false); + (void)SetCodeWritable(base + pointer_rva, false); return false; } // Retain this single RX page for process lifetime: an in-flight guest call // may still be returning through it when the entrypoint is restored. - g_pointer_target = base + kPointerFrameRva; + g_pointer_target = base + pointer_rva; return true; } @@ -324,6 +288,48 @@ void Log(const char *format, ...) { namespace internal { +const VrBuildContract* FindVrBuildContract(std::string_view build_id) { + static constexpr VrBuildContract kContracts[] = { + {"ade08266c67aee88ec9c1d00902150e1684dad3a", + {0x6c87b38, 0x3810614, 0x38109f6, 0x38109e6, 0x3810a18, 0x7369920, + 0x180}, + 0x6b7bfe0, + 0x4d573ee, + 0x263778c, + 0x3a8, + 0x73677a8, + 0x1d48b70}, + {"5f0704edd9064f566ee3d6df2bd2fabbcc709f03", + {0x6d08108, 0x37f9fe4, 0x37fa3c6, 0x37fa3b6, 0x37fa3e8, 0x73f4690, + 0x188}, + 0x6bfb180, + 0x4d97b98, + 0x265cebc, + 0x3a0, + 0x73f24c8, + 0x1d8f8c0}, + }; + for (const auto& contract : kContracts) { + if (contract.build_id == build_id) return &contract; + } + return nullptr; +} + +bool HasExpectedVrPointerContract(const std::uint8_t* code, std::size_t size, + const VrBuildContract& contract) { + if (code == nullptr || size < 0x1b2 || + std::memcmp(code, kPointerPrologue.data(), kPointerPrologue.size()) != + 0 || + code[0x59] != 0xe8 || code[0x6d] != 0xff || code[0x6e] != 0x91) { + return false; + } + std::uint32_t camera_slot; + std::memcpy(&camera_slot, code + 0x6f, sizeof(camera_slot)); + return RelativeTarget(code + 0x5a, contract.pointer_frame_rva + 0x5e) == + contract.workspace_getter_rva && + camera_slot == contract.camera_vtable_offset; +} + EyeResourceState InspectEyeResources(const void *object) { if (object == nullptr) return EyeResourceState::kInvalid; @@ -348,8 +354,9 @@ EyeResourceState InspectEyeResources(const void *object) { return EyeResourceState::kInvalid; } -bool HasExpectedVrStateGetterContract(const std::uint8_t *code, - std::size_t size) { +bool HasExpectedVrStateGetterContract(const std::uint8_t* code, + std::size_t size, + const VrBuildContract& contract) { // push rbp; mov rbp,rsp; push rbx; push rax; mov rbx,rdi; add rsi,0x14; // mov edx,0x138; call memcpy; mov rax,rbx; add rsp,8; pop rbx; pop rbp; ret static constexpr std::array @@ -358,7 +365,10 @@ bool HasExpectedVrStateGetterContract(const std::uint8_t *code, 0xe8, 0xd3, 0xb5, 0x36, 0x03, 0x48, 0x89, 0xd8, 0x48, 0x83, 0xc4, 0x08, 0x5b, 0x5d, 0xc3}; return code != nullptr && size >= kExpected.size() && - std::memcmp(code, kExpected.data(), kExpected.size()) == 0; + std::memcmp(code, kExpected.data(), 19) == 0 && + std::memcmp(code + 23, kExpected.data() + 23, 10) == 0 && + RelativeTarget(code + 19, contract.device.state_getter_rva + 23) == + contract.state_memcpy_rva; } bool HasExpectedVrEyeGetterContract(const std::uint8_t *code, @@ -371,15 +381,16 @@ bool HasExpectedVrEyeGetterContract(const std::uint8_t *code, std::memcmp(code, kExpected.data(), kExpected.size()) == 0; } -bool HasExpectedVrConstructorContract(const std::uint8_t *code, - std::size_t size) { +bool HasExpectedVrConstructorContract( + const std::uint8_t* code, std::size_t size, + const compat::VrDebugDeviceBridgeProfile& profile) { static constexpr std::array kPrologue = { 0x55, 0x48, 0x89, 0xe5, 0x41, 0x57, 0x41, 0x56, 0x41, 0x55, 0x41, 0x54, 0x53, 0x48, 0x83, 0xec, 0x28, 0x49, 0x89, 0xcf, 0x41, 0x89, 0xd4, 0x41, 0x89, 0xf5, 0x48, 0x89, 0xfb}; // lea rax,[rip -> DebugDeviceVR vtable]; mov [rdi],rax - static constexpr std::array kVtableStore = { - 0x48, 0x8d, 0x05, 0xf2, 0x74, 0x47, 0x03}; + static constexpr std::array kVtableStore = {0x48, 0x8d, + 0x05}; static constexpr std::array kVptrWrite = {0x48, 0x89, 0x07}; // mov dword ptr [rdi+0x10], 6 static constexpr std::array kTypeStore = { @@ -398,7 +409,10 @@ bool HasExpectedVrConstructorContract(const std::uint8_t *code, return false; } return std::memcmp(code, kPrologue.data(), kPrologue.size()) == 0 && - ContainsBytes(code, size, kVtableStore.data(), kVtableStore.size()) && + std::memcmp(code + 43, kVtableStore.data(), kVtableStore.size()) == + 0 && + RelativeTarget(code + 46, profile.constructor_rva + 50) == + profile.vtable_rva && ContainsBytes(code, size, kVptrWrite.data(), kVptrWrite.size()) && ContainsBytes(code, size, kTypeStore.data(), kTypeStore.size()) && ContainsBytes(code, size, kStateMemset.data(), kStateMemset.size()) && @@ -529,6 +543,12 @@ extern "C" void mocktail_vr_note_host_present_begin() { } } +const compat::VrDebugDeviceBridgeProfile* FindRobloxVrDeviceProfile( + std::string_view build_id) { + const auto* contract = internal::FindVrBuildContract(build_id); + return contract == nullptr ? nullptr : &contract->device; +} + RobloxVrDeviceBridge::~RobloxVrDeviceBridge() { Shutdown(); RobloxVrDeviceBridge *expected = this; @@ -536,6 +556,50 @@ RobloxVrDeviceBridge::~RobloxVrDeviceBridge() { expected, nullptr, std::memory_order_acq_rel, std::memory_order_acquire); } +void RobloxVrDeviceBridge::OnPointerFrameCall(void* result, void* service) { + if (!active() || contract_ == nullptr) return; + ScriptedPoseSample native_pose; + { + std::lock_guard lock(g_native_pose_mutex); + native_pose = g_native_pose; + } + if (!native_pose.valid) return; + auto* backend = ActiveVrBackend(); + if (!backend) return; + const auto current = backend->PublishedHeadPose(); + if (!current.valid) return; + int index = 0; + std::memcpy(&index, static_cast(service) + 0x118, 4); + if (index != 1 && index != 2) return; + if (!current.hands[index - 1].aim_valid) return; + const auto& hand = native_pose.hands[index - 1]; + if (!hand.grip_valid || !hand.aim_valid) return; + const auto base = image_base(); + // Follow the same workspace/camera path as the guest pointer getter. + auto get_workspace = reinterpret_cast( + base + contract_->workspace_getter_rva); + void* workspace = get_workspace(service); + if (!workspace) return; + auto* vtable = *static_cast(workspace); + void* camera = reinterpret_cast( + vtable[contract_->camera_vtable_offset / sizeof(void*)])(workspace); + if (!camera) return; + float scale = 0; + int degrees = 0; + std::memcpy(&scale, static_cast(camera) + 0x140, 4); + std::memcpy(°rees, + reinterpret_cast(base + contract_->hand_pitch_storage_rva), + 4); + if (internal::ApplyAimToWorldFrame(static_cast(result), hand, scale, + -static_cast(degrees))) { + static std::atomic logged[2]{}; + if (!logged[index - 1].exchange(true)) + std::fprintf( + stderr, " [vr-input] native menu pointer uses OpenXR aim: hand=%d\n", + index - 1); + } +} + Status RobloxVrDeviceBridge::Install(const compat::BuildProfile &profile) { if (installed_) { return FailedPrecondition("Roblox VR device bridge is already armed"); @@ -545,8 +609,25 @@ Status RobloxVrDeviceBridge::Install(const compat::BuildProfile &profile) { "experimental Roblox VR requires vr_debug_device_bridge metadata in " "the exact-build compatibility profile"); } - if (profile.elf_build_id != "ade08266c67aee88ec9c1d00902150e1684dad3a") - return FailedPrecondition("VR controller/pointer/haptic contracts are verified only for Build 2998"); + const auto* contract = internal::FindVrBuildContract(profile.elf_build_id); + if (contract == nullptr) { + return FailedPrecondition( + "VR contracts are not verified for this Build ID"); + } + const auto& actual = *profile.vr_debug_device_bridge; + const auto& expected_profile = contract->device; + if (actual.vtable_rva != expected_profile.vtable_rva || + actual.constructor_rva != expected_profile.constructor_rva || + actual.state_getter_rva != expected_profile.state_getter_rva || + actual.eye_getter_rva != expected_profile.eye_getter_rva || + actual.eye_initializer_rva != expected_profile.eye_initializer_rva || + actual.emulator_flag_storage_rva != + expected_profile.emulator_flag_storage_rva || + actual.device_create_framebuffer_vtable_offset != + expected_profile.device_create_framebuffer_vtable_offset) { + return FailedPrecondition( + "VR metadata does not match the verified Build ID"); + } if (!profile.allow_host_abi_bridges) { return FailedPrecondition( "Roblox VR device bridge profile requires host ABI bridges"); @@ -560,6 +641,7 @@ Status RobloxVrDeviceBridge::Install(const compat::BuildProfile &profile) { "process"); } profile_ = *profile.vr_debug_device_bridge; + contract_ = contract; installed_ = true; std::fprintf(stderr, " [vr-device] native stereo bridge armed for Build ID %s " @@ -588,7 +670,11 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { profile_.state_getter_rva + kStateGetterContractSize, profile_.eye_getter_rva + kEyeGetterContractSize, profile_.eye_initializer_rva + kInitializerContractSize, - profile_.emulator_flag_storage_rva + 1}); + profile_.emulator_flag_storage_rva + 1, contract_->state_memcpy_rva + 1, + contract_->pointer_frame_rva + 0x1b2, + contract_->workspace_getter_rva + 1, + contract_->hand_pitch_storage_rva + sizeof(int), + contract_->haptic_sink_rva + 1}); if (image_base > std::numeric_limits::max() - largest_rva) { return InvalidArgument("VR device profile address overflows"); } @@ -606,12 +692,16 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { !IsExecutableImageRange(image_base, profile_.eye_getter_rva, kEyeGetterContractSize) || !IsExecutableImageRange(image_base, profile_.eye_initializer_rva, - kInitializerContractSize)) { + kInitializerContractSize) || + !IsExecutableImageRange(image_base, contract_->state_memcpy_rva, 1) || + !IsExecutableImageRange(image_base, contract_->workspace_getter_rva, 1)) { return FailedPrecondition( "VR device profile functions are not in executable image ranges"); } if (!IsWritableDataImageRange(image_base, profile_.emulator_flag_storage_rva, - 1)) { + 1) || + !IsWritableDataImageRange(image_base, contract_->hand_pitch_storage_rva, + sizeof(int))) { return FailedPrecondition( "VR emulator flag storage is not in a writable data range"); } @@ -630,15 +720,15 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { image_base + profile_.constructor_rva); const auto *initializer_code = reinterpret_cast( image_base + profile_.eye_initializer_rva); - if (!internal::HasExpectedVrStateGetterContract(state_getter_code, - kStateGetterContractSize) || + if (!internal::HasExpectedVrStateGetterContract( + state_getter_code, kStateGetterContractSize, *contract_) || !internal::HasExpectedVrEyeGetterContract(eye_getter_code, kEyeGetterContractSize)) { return FailedPrecondition( "DebugDeviceVR getter machine contracts do not match the profile"); } - if (!internal::HasExpectedVrConstructorContract(constructor_code, - kConstructorContractSize) || + if (!internal::HasExpectedVrConstructorContract( + constructor_code, kConstructorContractSize, profile_) || !internal::HasExpectedVrEyeInitializerContract( initializer_code, kInitializerContractSize, profile_)) { return FailedPrecondition( @@ -646,20 +736,23 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { "match the profile"); } - if (!IsExecutableImageRange(image_base, kPointerFrameRva, 0x1b2) || - std::memcmp(reinterpret_cast(image_base + kPointerFrameRva), - kPointerPrologue.data(), kPointerPrologue.size()) != 0) - return FailedPrecondition("2998 VR pointer function contract mismatch"); + if (!IsExecutableImageRange(image_base, contract_->pointer_frame_rva, + 0x1b2) || + !internal::HasExpectedVrPointerContract( + reinterpret_cast(image_base + + contract_->pointer_frame_rva), + 0x1b2, *contract_)) { + return FailedPrecondition("VR pointer function contract mismatch"); + } + if (vtable[3] != image_base + contract_->haptic_sink_rva || + !IsExecutableImageRange(image_base, contract_->haptic_sink_rva, 1) || + *reinterpret_cast( + image_base + contract_->haptic_sink_rva) != 0xc3) { + return FailedPrecondition("VR haptic vtable contract mismatch"); + } image_base_.store(image_base, std::memory_order_relaxed); image_size_.store(ImageSize(image_base), std::memory_order_relaxed); - vtable_ = - reinterpret_cast(image_base + profile_.vtable_rva); - // 2998's haptic sink is a verified no-op (ret at 0x1d48b70). - // The caller at 0x28a767e passes self, hand 0/1 and float amplitude. - if (vtable_[3] != image_base + 0x1d48b70 || - !IsExecutableImageRange(image_base, 0x1d48b70, 1) || - *reinterpret_cast(image_base + 0x1d48b70) != 0xc3) - { vtable_ = nullptr; return FailedPrecondition("2998 VR haptic vtable contract mismatch"); } + vtable_ = reinterpret_cast(image_base + profile_.vtable_rva); original_haptic_slot_ = vtable_[3]; original_state_slot_ = vtable_[kStateGetterVtableSlot / sizeof(void *)]; original_eye_slot_ = vtable_[kEyeGetterVtableSlot / sizeof(void *)]; @@ -699,7 +792,7 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { "protection"); } - if (!InstallPointerHook(image_base)) { + if (!InstallPointerHook(image_base, contract_->pointer_frame_rva)) { (void)RestoreVtableLocked(); return FailedPrecondition("could not install verified VR aim pointer hook"); } @@ -891,8 +984,7 @@ bool internal::ApplyXrPose(void *state, const ScriptedPoseSample &pose) { std::memset(bytes + 0x88, 0, 28 * sizeof(float)); // Hand records are cleared unless a tracked grip pose is supplied below. // Clearing the valid byte (never the whole record) makes the guest report - // the hand as untracked instead of holding a stale pose. The extra index-3 - // record is unused by this backend and stays invalid. + // the hand as untracked instead of holding a stale pose. Index 3 is Floor. bytes[kExtraRecordOffset + kRecordValidOffset] = 0; bytes[kLeftHandRecordOffset + kRecordValidOffset] = 0; bytes[kRightHandRecordOffset + kRecordValidOffset] = 0; @@ -914,7 +1006,7 @@ bool internal::ApplyXrPose(void *state, const ScriptedPoseSample &pose) { if (!std::isfinite(v)) return false; for (float v : pose.eye_fov[eye]) - if (!std::isfinite(v) || std::abs(v) >= 1.56f) + if (!std::isfinite(v) || std::abs(v) >= pose::kMaxEyeFovAngle) return false; } bytes[0x80] = pose.valid && pose.controllers_connected; @@ -928,9 +1020,25 @@ bool internal::ApplyXrPose(void *state, const ScriptedPoseSample &pose) { } bytes[0x132] = pose.valid && pose.controllers_connected; - // 2998's 0x37eacb8 converts metres to studs itself (factor 10/3). + // The verified guest pose helper applies metres-to-studs conversion. std::memcpy(bytes + 4, pose.position, sizeof(pose.position)); std::memcpy(bytes + 0x10, pose.orientation, sizeof(pose.orientation)); + if (pose.floor_valid) { + bool finite = true; + float floor_norm = 0.f; + for (float value : pose.floor_position) finite &= std::isfinite(value); + for (float value : pose.floor_orientation) { + finite &= std::isfinite(value); + floor_norm += value * value; + } + if (finite && floor_norm >= .99f && floor_norm <= 1.01f) { + std::memcpy(bytes + kExtraRecordOffset + kRecordPositionOffset, + pose.floor_position, sizeof(pose.floor_position)); + std::memcpy(bytes + kExtraRecordOffset + kRecordOrientationOffset, + pose.floor_orientation, sizeof(pose.floor_orientation)); + bytes[kExtraRecordOffset + kRecordValidOffset] = 1; + } + } // Tracked hands go into the same state copy the head uses, so the guest's // own conversion (10/3 studs) and -20 deg hand pitch offset apply once, // exactly as they do for the head. Grip is the held-object pose; a hand diff --git a/src/vr/roblox_vr_device_bridge_internal.h b/src/vr/roblox_vr_device_bridge_internal.h index fe42469a..9e98e1cf 100644 --- a/src/vr/roblox_vr_device_bridge_internal.h +++ b/src/vr/roblox_vr_device_bridge_internal.h @@ -3,6 +3,7 @@ #include #include +#include #include "compat/build_profile.h" #include "mocktail/vr/xr_controller.h" @@ -10,6 +11,19 @@ namespace mocktail::vr { struct ScriptedPoseSample; } namespace mocktail::vr::internal { +struct VrBuildContract { + std::string_view build_id; + compat::VrDebugDeviceBridgeProfile device; + std::uintptr_t state_memcpy_rva; + std::uintptr_t pointer_frame_rva; + std::uintptr_t workspace_getter_rva; + std::uintptr_t camera_vtable_offset; + std::uintptr_t hand_pitch_storage_rva; + std::uintptr_t haptic_sink_rva; +}; + +const VrBuildContract* FindVrBuildContract(std::string_view build_id); + // Writes only a caller-owned state copy; preserves all resource readiness. bool ApplyXrPose(void* state, const ScriptedPoseSample& pose); // Adjust a world-space grip CoordinateFrame to OpenXR aim without changing @@ -78,16 +92,21 @@ EyeResourceState InspectEyeResources(const void *object); // Exact 33-byte state getter: sret ABI (rdi = hidden 0x138-byte result // buffer, rsi = device), copies state from self+0x14 and returns the buffer. -bool HasExpectedVrStateGetterContract(const std::uint8_t *code, - std::size_t size); +bool HasExpectedVrStateGetterContract(const std::uint8_t* code, + std::size_t size, + const VrBuildContract& contract); // Exact 16-byte eye framebuffer getter: self + 0x150 + eyeIndex * 16. bool HasExpectedVrEyeGetterContract(const std::uint8_t *code, std::size_t size); // Constructor: captures rcx (graphics device) into self+0x198, width/height // into +0x190/+0x194, type dword 6 into +0x10, and clears the state block. -bool HasExpectedVrConstructorContract(const std::uint8_t *code, - std::size_t size); +bool HasExpectedVrConstructorContract( + const std::uint8_t* code, std::size_t size, + const compat::VrDebugDeviceBridgeProfile& profile); + +bool HasExpectedVrPointerContract(const std::uint8_t* code, std::size_t size, + const VrBuildContract& contract); // Eye-resource initializer: raises ready byte self+0x147 plus +0x94/+0x144 // and creates the per-eye resources through the graphics-device vtable slot diff --git a/src/vr/xr_actions.cc b/src/vr/xr_actions.cc index 10c7e00a..016252d1 100644 --- a/src/vr/xr_actions.cc +++ b/src/vr/xr_actions.cc @@ -54,6 +54,14 @@ constexpr const char *kProfileTouch = "/interaction_profiles/oculus/touch_controller"; constexpr const char *kProfileQuest2 = "/interaction_profiles/meta/touch_controller_quest_2"; +constexpr const char* kTouchProfiles[] = { + kProfileTouch, + kProfileQuest2, + "/interaction_profiles/facebook/touch_controller_pro", + "/interaction_profiles/meta/touch_controller_plus", + "/interaction_profiles/meta/touch_pro_controller", + "/interaction_profiles/meta/touch_plus_controller", +}; constexpr const char *kProfileIndex = "/interaction_profiles/valve/index_controller"; constexpr const char *kProfileVive = @@ -191,8 +199,9 @@ Status XrActions::Create(void *xr_instance) { binds.emplace_back(impl_->touches[static_cast(kind)], path(component)); }; - const bool is_touch = std::strcmp(profile, kProfileTouch) == 0 || - std::strcmp(profile, kProfileQuest2) == 0; + const bool is_touch = std::any_of( + std::begin(kTouchProfiles), std::end(kTouchProfiles), + [&](const char* name) { return std::strcmp(profile, name) == 0; }); const bool is_index = std::strcmp(profile, kProfileIndex) == 0; if (is_touch || is_index) { touch(ControllerTouch::kTrigger, "/user/hand/left/input/trigger/touch"); @@ -209,7 +218,7 @@ Status XrActions::Create(void *xr_instance) { touch(ControllerTouch::kY, is_touch ? "/user/hand/left/input/y/touch" : "/user/hand/left/input/b/touch"); } - if (is_touch) { + if (is_touch && std::strcmp(profile, kProfileQuest2) != 0) { touch(ControllerTouch::kThumbrest, "/user/hand/left/input/thumbrest/touch"); touch(ControllerTouch::kThumbrest, @@ -244,7 +253,7 @@ Status XrActions::Create(void *xr_instance) { const XrPath l_haptic = path("/user/hand/left/output/haptic"); const XrPath r_haptic = path("/user/hand/right/output/haptic"); - for (const char *profile : {kProfileTouch, kProfileQuest2}) { + for (const char* profile : kTouchProfiles) { suggest( profile, { @@ -337,7 +346,8 @@ Status XrActions::Create(void *xr_instance) { created_ = true; Log(" [vr-input] action set created: grip+aim poses, trigger/squeeze " "analog, thumbstick, face/menu buttons, haptic output; bindings " - "suggested for Touch/Quest2/Index/Vive/Simple (system button never " + "suggested for Touch/Quest2/Touch Pro/Touch Plus/Index/Vive/Simple " + "(system button never " "bound)\n"); return Status::Ok(); } catch (const std::exception &error) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index aa510729..6f8329de 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -60,6 +60,10 @@ if(MOCKTAIL_ENABLE_VR) target_include_directories(openxr_backend_test PRIVATE "${PROJECT_SOURCE_DIR}/third_party/OpenXR-SDK/include") target_link_libraries(openxr_backend_test PRIVATE Mocktail::VR GTest::gtest_main) gtest_discover_tests(openxr_backend_test) + add_executable(vr_queue_submission_test vr/queue_submission_test.cc) + target_link_libraries(vr_queue_submission_test PRIVATE + Mocktail::VR stub_vulkan GTest::gtest_main) + gtest_discover_tests(vr_queue_submission_test) add_executable(gles_vr_transport_test vr/gles_transport_test.cc) target_include_directories(gles_vr_transport_test PRIVATE "${PROJECT_SOURCE_DIR}/src" "${PROJECT_SOURCE_DIR}/third_party/OpenXR-SDK/include" ${MOCKTAIL_GLES3_INCLUDE_DIR}) target_link_libraries(gles_vr_transport_test PRIVATE Mocktail::VR SDL3::SDL3 GTest::gtest_main) @@ -640,6 +644,14 @@ target_include_directories(roblox_output_device_bridge_test PRIVATE ) mocktail_apply_compile_options(roblox_output_device_bridge_test) +add_executable(native_input_capture_test native_input_capture_test.cc) +target_include_directories(native_input_capture_test PRIVATE + "${PROJECT_SOURCE_DIR}/src") +target_link_libraries(native_input_capture_test PRIVATE + mocktail_audio_sdl GTest::gtest_main) +mocktail_apply_compile_options(native_input_capture_test) +gtest_discover_tests(native_input_capture_test) + add_executable(window_text_input_owner_test window_text_input_owner_test.cc ) diff --git a/tests/build_profile_test.cc b/tests/build_profile_test.cc index e081eeb6..198a329b 100644 --- a/tests/build_profile_test.cc +++ b/tests/build_profile_test.cc @@ -14,6 +14,23 @@ namespace { const std::string kManifestPath = std::string(MOCKTAIL_TEST_SOURCE_DIR) + "/config/roblox_compatibility.json"; +TEST(BuildProfileTest, Payload3092IncludesVrAndSecondAudioLayout) { + const auto result = FindBuildProfile( + kManifestPath, "5f0704edd9064f566ee3d6df2bd2fabbcc709f03"); + ASSERT_TRUE(result) << result.error; + ASSERT_TRUE(result.profile.has_value()); + EXPECT_EQ(result.profile->version_name, "2.738.1397"); + EXPECT_EQ(result.profile->version_code, 3092); + EXPECT_TRUE(result.profile->default_allowed); + EXPECT_TRUE(result.profile->allow_host_abi_bridges); + EXPECT_FALSE(result.profile->allow_legacy_binary_patches); + ASSERT_TRUE(result.profile->vr_debug_device_bridge.has_value()); + EXPECT_EQ(result.profile->vr_debug_device_bridge->vtable_rva, 0x6d08108u); + ASSERT_TRUE(result.profile->fmod_output_device_bridge.has_value()); + EXPECT_EQ(result.profile->fmod_output_device_bridge->vtable_layout_version, + 2u); +} + TEST(BuildProfileTest, FindsCurrentPayloadAsSupported) { const ProfileLookupResult result = FindBuildProfile( kManifestPath, "d0cb1fa0deb3d9161b4cd77530cbcd2e50de3a21"); diff --git a/tests/host_abi_profile_test.cc b/tests/host_abi_profile_test.cc index e132de6f..67355ca5 100644 --- a/tests/host_abi_profile_test.cc +++ b/tests/host_abi_profile_test.cc @@ -28,6 +28,28 @@ namespace mocktail::compat { namespace { +TEST(HostAbiProfileTest, Payload3092KeepsVerifiedConstructorBoundaries) { + const auto* profile = + FindHostAbiProfile("5f0704edd9064f566ee3d6df2bd2fabbcc709f03"); + ASSERT_NE(profile, nullptr); + EXPECT_EQ(profile->bridge_entry_count, 6u); + EXPECT_EQ(profile->init_array_offset, 0x70e1618u); + EXPECT_EQ(profile->init_array_count, 3599u); + EXPECT_TRUE(profile->HasValidConstructorRanges()); + EXPECT_TRUE(profile->AllowsConstructor(2)); + EXPECT_FALSE(profile->AllowsConstructor(3)); + EXPECT_FALSE(profile->AllowsConstructor(4)); + EXPECT_TRUE(profile->AllowsConstructor(5)); + EXPECT_TRUE(profile->AllowsConstructor(3598)); + EXPECT_FALSE(profile->AllowsConstructor(3599)); + EXPECT_TRUE(profile->HasValidNativeMimallocConstructorRanges()); + EXPECT_FALSE(profile->AllowsNativeMimallocConstructor(1)); + EXPECT_TRUE(profile->AllowsNativeMimallocConstructor(2)); + EXPECT_TRUE(profile->AllowsNativeMimallocConstructor(3598)); + EXPECT_FALSE(profile->AllowsNativeMimallocConstructor(3599)); + EXPECT_TRUE(profile->HasValidNativePreJniBootstrap()); +} + TEST(HostAbiProfileTest, Supported2908HasExactDerivedBoundaries) { const HostAbiProfile* profile = FindHostAbiProfile("63c5109637b7d7b2bdb8ed8f858023ff5ef49326"); diff --git a/tests/native_input_capture_test.cc b/tests/native_input_capture_test.cc new file mode 100644 index 00000000..dd7319c9 --- /dev/null +++ b/tests/native_input_capture_test.cc @@ -0,0 +1,186 @@ +#include "audio/native_input_capture.h" + +#include +#include +#include + +#include +#include + +#include "mocktail/audio/sdl_audio_sink.h" + +namespace mocktail::audio { +namespace { + +struct TestSink { + const std::uintptr_t* vtable; + std::thread::id poll_thread = std::this_thread::get_id(); + int frames = 0; + bool nonzero = false; + bool invalid = false; +}; + +struct TestOwner { + const std::uintptr_t* vtable; + long shared_owners = 0; + long weak_owners = 0; + int destroyed = 0; + int released = 0; +}; + +std::size_t ReceivePcm(void* self, const void* pcm, std::size_t samples, + std::uint64_t delay, const NativePcmFormat* format) { + auto& sink = *static_cast(self); + sink.invalid |= samples != 480 || format->rate != 48000 || + format->channels != 1 || format->bytes_per_sample != 2 || + delay < 10000000 || delay > 200000000 || + std::this_thread::get_id() != sink.poll_thread; + const auto* data = static_cast(pcm); + sink.nonzero |= std::any_of(data, data + samples, + [](auto sample) { return sample != 0; }); + ++sink.frames; + return samples; +} + +void DestroySink(void* self) { ++static_cast(self)->destroyed; } +void ReleaseOwner(void* self) { ++static_cast(self)->released; } +void OriginalLatency(void*, std::uint32_t* playback, std::uint32_t* recording) { + if (playback) *playback = 42; + if (recording) *recording = 99; +} + +const std::array kSinkVtable{ + 0, 0, reinterpret_cast(&ReceivePcm)}; +const std::array kOwnerVtable{ + 0, 0, reinterpret_cast(&DestroySink), 0, + reinterpret_cast(&ReleaseOwner)}; + +bool TestRelro(std::uintptr_t base, std::uintptr_t rva, std::size_t size) { + return (base + rva == reinterpret_cast(kSinkVtable.data()) && + size <= sizeof(kSinkVtable)) || + (base + rva == reinterpret_cast(kOwnerVtable.data()) && + size <= sizeof(kOwnerVtable)); +} + +bool TestCode(std::uintptr_t base, std::uintptr_t rva, std::size_t) { + return base + rva == kSinkVtable[2] || base + rva == kOwnerVtable[2] || + base + rva == kOwnerVtable[4]; +} + +class NativeInputCaptureTest : public testing::Test { + protected: + void SetUp() override { + const int fd = mkstemp(pcm_path_); + ASSERT_GE(fd, 0); + FILE* file = fdopen(fd, "wb"); + if (!file) close(fd); + ASSERT_NE(file, nullptr); + const std::vector samples(1024 * 1024, 0x3f); + const auto written = std::fwrite(samples.data(), 1, samples.size(), file); + const int closed = std::fclose(file); + ASSERT_EQ(written, samples.size()); + ASSERT_EQ(closed, 0); + ASSERT_TRUE(SDL_SetHintWithPriority(SDL_HINT_AUDIO_DRIVER, "disk", + SDL_HINT_OVERRIDE)); + ASSERT_TRUE(SDL_SetHintWithPriority(SDL_HINT_AUDIO_DISK_INPUT_FILE, + pcm_path_, SDL_HINT_OVERRIDE)); + ASSERT_TRUE(SDL_SetHintWithPriority(SDL_HINT_AUDIO_DISK_OUTPUT_FILE, + "/dev/null", SDL_HINT_OVERRIDE)); + ASSERT_TRUE(InitializeSdlAudioSubsystem().ok()); + ASSERT_STREQ(SDL_GetCurrentAudioDriver(), "disk"); + } + + void TearDown() override { + g_native_capture.store(nullptr); + if (capture_) capture_->Shutdown(); + EXPECT_TRUE(ShutdownSdlAudioSubsystem().ok()); + SDL_ResetHint(SDL_HINT_AUDIO_DRIVER); + SDL_ResetHint(SDL_HINT_AUDIO_DISK_INPUT_FILE); + SDL_ResetHint(SDL_HINT_AUDIO_DISK_OUTPUT_FILE); + std::remove(pcm_path_); + } + + void Publish(bool enabled) { + capture_ = std::make_unique( + NativeInputCapture::SinkAbi{1, &TestRelro, &TestCode, &OriginalLatency}, + enabled); + g_native_capture.store(capture_.get()); + } + + char pcm_path_[64] = "/tmp/mocktail-native-capture-XXXXXX"; + std::unique_ptr capture_; + TestSink sink_{kSinkVtable.data()}; + TestOwner owner_{kOwnerVtable.data()}; + int device_ = 0; +}; + +TEST_F(NativeInputCaptureTest, DeliversPcmAndReopensAfterStop) { + Publish(true); + const auto format = NativeInputCapture::Format(&device_); + EXPECT_EQ(format.present, 1u); + EXPECT_EQ(format.format.rate, 48000u); + EXPECT_EQ(format.format.channels, 1u); + for (int pass = 0; pass < 2; ++pass) { + owner_ = {kOwnerVtable.data()}; + sink_ = {kSinkVtable.data()}; + NativeSharedSink parameter{&sink_, &owner_}; + ASSERT_TRUE(NativeInputCapture::Start(&device_, ¶meter)); + EXPECT_EQ(parameter.object, nullptr); + EXPECT_EQ(parameter.owner, nullptr); + EXPECT_TRUE(NativeInputCapture::Recording(&device_)); + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (sink_.frames < 3 && std::chrono::steady_clock::now() < deadline) { + NativeInputCapture::Poll(&device_); + SDL_Delay(10); + } + EXPECT_GE(sink_.frames, 3); + EXPECT_TRUE(sink_.nonzero); + EXPECT_FALSE(sink_.invalid); + std::uint32_t playback = 0, recording = 0; + NativeInputCapture::Latency(&device_, &playback, &recording); + EXPECT_EQ(playback, 42u); + EXPECT_GE(recording, 10u); + NativeSharedSink stop{&sink_, &owner_}; + NativeInputCapture::Stop(&device_, &stop); + EXPECT_FALSE(NativeInputCapture::Recording(&device_)); + EXPECT_EQ(owner_.destroyed, 1); + EXPECT_EQ(owner_.released, 1); + const int frames = sink_.frames; + NativeInputCapture::Poll(&device_); + NativeInputCapture::Stop(&device_, &stop); + EXPECT_EQ(sink_.frames, frames); + EXPECT_EQ(owner_.released, 1); + } +} + +TEST_F(NativeInputCaptureTest, RejectsDisabledAndClosedCapture) { + Publish(false); + NativeSharedSink parameter{&sink_, &owner_}; + EXPECT_EQ(NativeInputCapture::Format(&device_).present, 0u); + EXPECT_FALSE(NativeInputCapture::Start(&device_, ¶meter)); + EXPECT_EQ(parameter.object, &sink_); + Publish(true); + ASSERT_TRUE(NativeInputCapture::Start(&device_, ¶meter)); + capture_->Shutdown(); + EXPECT_EQ(NativeInputCapture::Format(&device_).present, 0u); + EXPECT_FALSE(NativeInputCapture::Recording(&device_)); + parameter = {&sink_, &owner_}; + EXPECT_FALSE(NativeInputCapture::Start(&device_, ¶meter)); + EXPECT_EQ(owner_.released, 1); +} + +TEST_F(NativeInputCaptureTest, RejectsUnverifiedSinkWithoutTakingOwnership) { + Publish(true); + const std::array unknown_vtable{}; + TestSink unknown{unknown_vtable.data()}; + NativeSharedSink parameter{&unknown, &owner_}; + EXPECT_FALSE(NativeInputCapture::Start(&device_, ¶meter)); + EXPECT_EQ(parameter.object, &unknown); + EXPECT_EQ(parameter.owner, &owner_); + EXPECT_FALSE(NativeInputCapture::Recording(&device_)); + EXPECT_EQ(owner_.released, 0); +} + +} // namespace +} // namespace mocktail::audio diff --git a/tests/roblox_output_device_bridge_test.cc b/tests/roblox_output_device_bridge_test.cc index ff58cf2e..ab266434 100644 --- a/tests/roblox_output_device_bridge_test.cc +++ b/tests/roblox_output_device_bridge_test.cc @@ -48,6 +48,86 @@ TEST(RobloxOutputDeviceBridgeTest, ValidatesEveryInterposedVtableSlot) { vtable.data(), kImageBase, profile)); } +TEST(RobloxOutputDeviceBridgeTest, Validates3092LayoutWithoutUsingListSlots) { + constexpr std::uintptr_t kBase = 0x10000000; + auto profile = TestProfile(); + profile.vtable_layout_version = 2; + std::array vtable{}; + vtable[5] = kBase + profile.count_method_rva; + vtable[6] = kBase + profile.info_method_rva; + vtable[7] = kBase + 0x7000; + vtable[8] = kBase + profile.current_method_rva; + vtable[17] = kBase + 0x8000; + vtable[19] = kBase + profile.select_method_rva; + EXPECT_TRUE(internal::HasExpectedFmodOutputDeviceVtable(vtable.data(), kBase, + profile)); + profile.vtable_layout_version = 1; + EXPECT_FALSE(internal::HasExpectedFmodOutputDeviceVtable(vtable.data(), kBase, + profile)); + profile.vtable_layout_version = 3; + EXPECT_FALSE(internal::HasExpectedFmodOutputDeviceVtable(vtable.data(), kBase, + profile)); +} + +TEST(RobloxOutputDeviceBridgeTest, PinsCaptureToBuildIdVtableAndLayout) { + compat::BuildProfile profile; + profile.allow_host_abi_bridges = true; + profile.elf_build_id = "5f0704edd9064f566ee3d6df2bd2fabbcc709f03"; + EXPECT_EQ(internal::FindFmodInputCaptureProfile(profile), nullptr); + profile.fmod_output_device_bridge = TestProfile(); + EXPECT_EQ(internal::FindFmodInputCaptureProfile(profile), nullptr); + profile.fmod_output_device_bridge->vtable_rva = 0x6cd3ce0; + EXPECT_EQ(internal::FindFmodInputCaptureProfile(profile), nullptr); + profile.fmod_output_device_bridge->vtable_layout_version = 2; + ASSERT_NE(internal::FindFmodInputCaptureProfile(profile), nullptr); + profile.allow_host_abi_bridges = false; + EXPECT_EQ(internal::FindFmodInputCaptureProfile(profile), nullptr); + profile.allow_host_abi_bridges = true; + profile.elf_build_id = "unknown"; + EXPECT_EQ(internal::FindFmodInputCaptureProfile(profile), nullptr); +} + +TEST(RobloxOutputDeviceBridgeTest, ValidatesEveryCaptureSlotOnBothBuilds) { + constexpr std::uintptr_t kBase = 0x10000000; + for (int version : {2998, 3092}) { + compat::BuildProfile profile; + profile.allow_host_abi_bridges = true; + profile.elf_build_id = version == 2998 + ? "ade08266c67aee88ec9c1d00902150e1684dad3a" + : "5f0704edd9064f566ee3d6df2bd2fabbcc709f03"; + profile.fmod_output_device_bridge = TestProfile(); + profile.fmod_output_device_bridge->vtable_rva = + version == 2998 ? 0x6c58040 : 0x6cd3ce0; + profile.fmod_output_device_bridge->vtable_layout_version = + version == 2998 ? 1 : 2; + const auto* capture = internal::FindFmodInputCaptureProfile(profile); + ASSERT_NE(capture, nullptr); + const std::array expected_slots = + version == 2998 ? std::array{4, 11, 12, 13, 14, 15} + : std::array{4, 13, 14, 15, 16, 17}; + EXPECT_EQ(capture->slots, expected_slots); + std::array vtable{}; + for (std::size_t i = 0; i < capture->slots.size(); ++i) { + vtable[capture->slots[i]] = kBase + capture->method_rvas[i]; + } + EXPECT_TRUE(internal::HasExpectedFmodInputCaptureVtable(vtable.data(), + kBase, *capture)); + for (const auto slot : capture->slots) { + ++vtable[slot]; + EXPECT_FALSE(internal::HasExpectedFmodInputCaptureVtable(vtable.data(), + kBase, *capture)) + << slot; + --vtable[slot]; + } + EXPECT_FALSE( + internal::HasExpectedFmodInputCaptureVtable(nullptr, kBase, *capture)); + EXPECT_FALSE(internal::HasExpectedFmodInputCaptureVtable(vtable.data(), 0, + *capture)); + EXPECT_FALSE(internal::HasExpectedFmodInputCaptureVtable( + vtable.data(), UINTPTR_MAX, *capture)); + } +} + TEST(RobloxOutputDeviceBridgeTest, BuildsStableDistinctHostGuids) { const std::string first = internal::MakeOutputDeviceGuid(17, "USB Headset"); EXPECT_EQ(first, internal::MakeOutputDeviceGuid(17, "USB Headset")); diff --git a/tests/vr/openxr_backend_test.cc b/tests/vr/openxr_backend_test.cc index d3843f83..d9da14a6 100644 --- a/tests/vr/openxr_backend_test.cc +++ b/tests/vr/openxr_backend_test.cc @@ -20,6 +20,18 @@ struct OpenXrBackendTestAccess { backend.published_pose_.valid = true; } static VkInstance Instance(const OpenXrBackend& b) { return b.vk_instance_; } + static void GraphicsFamily(OpenXrBackend& b, unsigned family) { + b.vk_queue_family_ = family; + b.vk_graphics_queue_ = Handle(30); + b.device_queue_families_ = {family, family + 1}; + } + static VkQueue CopyQueue(const OpenXrBackend& b, VkQueue present) { + return b.GraphicsQueueForPresentLocked(present); + } + static void NextFrame(OpenXrBackend& b) { ++b.published_pose_.frame; } + static bool SupportsQueue(const OpenXrBackend& b, VkQueue queue) { + return b.SupportsPresentQueueLocked(queue); + } static void Attachment(OpenXrBackend& b, int id, void* owner, VkDevice device) { auto image = Handle(id); auto& r = b.images_[image]; @@ -29,16 +41,36 @@ struct OpenXrBackendTestAccess { b.image_views_[Handle(id)] = image; b.framebuffer_views_[Handle(id)] = {Handle(id)}; } - static void Pass(OpenXrBackend& b, RobloxVrDeviceBridge& bridge, int eye, int id, void* owner, - VkImageLayout layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL) { + static void Pass( + OpenXrBackend& b, RobloxVrDeviceBridge& bridge, int eye, int id, + void* owner, + VkImageLayout layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + VkCommandBuffer command = VK_NULL_HANDLE) { bridge.OnEyeGetterCall(owner, eye, Handle(id)); VkRenderPassBeginInfo info{}; info.framebuffer = Handle(id); info.renderPass = Handle(99); b.render_pass_final_layouts_[info.renderPass] = {layout}; - b.NoteRenderPassBegin(VK_NULL_HANDLE, &info); + b.NoteRenderPassBegin(command, &info); } static VkImage Image(const OpenXrBackend& b, int eye) { return b.eyes_[eye].image; } + static void AddMsaaAttachment(OpenXrBackend& b, int framebuffer, int msaa) { + b.images_[Handle(msaa)].samples = VK_SAMPLE_COUNT_4_BIT; + b.framebuffer_views_[Handle(framebuffer)].insert( + b.framebuffer_views_[Handle(framebuffer)].begin(), + Handle(msaa)); + } + static void ResolvedPass(OpenXrBackend& b, RobloxVrDeviceBridge& bridge, + void* owner, int framebuffer) { + bridge.OnEyeGetterCall(owner, 0, Handle(framebuffer)); + VkRenderPassBeginInfo info{}; + info.framebuffer = Handle(framebuffer); + info.renderPass = Handle(99); + b.render_pass_final_layouts_[info.renderPass] = { + VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, + VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL}; + b.NoteRenderPassBegin(VK_NULL_HANDLE, &info); + } static VkImageLayout Layout(const OpenXrBackend& b, int eye) { return b.eyes_[eye].final_layout; } static void Invalidate(OpenXrBackend& b) { b.InvalidateInFlightPose("test"); @@ -62,8 +94,12 @@ struct OpenXrBackendTestAccess { static VkDevice Device(OpenXrBackend& b) { return b.vk_device_; } static std::size_t Resources(OpenXrBackend& b) { return b.images_.size(); } static void SeedResource(OpenXrBackend& b) { b.images_[Handle(99)] = {}; } - static void LatchCanted(OpenXrBackend& b) { b.canted_rejection_logged_ = true; } - static bool CantedLogged(const OpenXrBackend& b) { return b.canted_rejection_logged_; } + static void LatchProjectionRejection(OpenXrBackend& b) { + b.projection_rejection_logged_ = true; + } + static bool ProjectionRejectionLogged(const OpenXrBackend& b) { + return b.projection_rejection_logged_; + } }; using Access = OpenXrBackendTestAccess; TEST(OpenXrBackendLifecycle, DeviceDestructionPreservesLiveInstance) { @@ -181,13 +217,13 @@ TEST(OpenXrBackendLifecycle, VisibilityLostStateSurvivesUntilSessionReset) { EXPECT_FALSE(Access::VisibilityLost(b)); } -TEST(OpenXrBackendLifecycle, TeardownClearsCantedRejectionLatch) { +TEST(OpenXrBackendLifecycle, TeardownClearsProjectionRejectionLatch) { OpenXrBackend b; Access::Configure(b); - Access::LatchCanted(b); - ASSERT_TRUE(Access::CantedLogged(b)); + Access::LatchProjectionRejection(b); + ASSERT_TRUE(Access::ProjectionRejectionLogged(b)); b.NoteDeviceDestroyed(Access::Handle(2)); - EXPECT_FALSE(Access::CantedLogged(b)); + EXPECT_FALSE(Access::ProjectionRejectionLogged(b)); } } // namespace mocktail::vr @@ -214,3 +250,311 @@ TEST(OpenXrDesktopMirror, RequiresTransferUsageAndDropsDestroyedSwapchain) { EXPECT_EQ(b.MirrorDesktop(VK_NULL_HANDLE, nullptr), VK_NOT_READY); } } + +namespace mocktail::vr { +namespace { +VkResult reset_result, submit_result, wait_result; +int resets, submits, waits; +VkQueue expected_queue; +VkSemaphore expected_semaphores[2]; +VKAPI_ATTR VkResult VKAPI_CALL ResetPresentFence(VkDevice, uint32_t count, + const VkFence*) { + EXPECT_EQ(count, 1u); + ++resets; + return reset_result; +} +VKAPI_ATTR VkResult VKAPI_CALL SubmitPresentWait(VkQueue queue, uint32_t count, + const VkSubmitInfo* info, + VkFence fence) { + ++submits; + EXPECT_EQ(queue, expected_queue); + EXPECT_EQ(count, 1u); + EXPECT_NE(fence, VK_NULL_HANDLE); + EXPECT_EQ(info->commandBufferCount, 0u); + EXPECT_EQ(info->waitSemaphoreCount, 2u); + EXPECT_EQ(info->signalSemaphoreCount, 0u); + for (unsigned i = 0; i < 2; ++i) { + EXPECT_EQ(info->pWaitSemaphores[i], expected_semaphores[i]); + EXPECT_EQ(info->pWaitDstStageMask[i], VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); + } + return submit_result; +} +VKAPI_ATTR VkResult VKAPI_CALL WaitPresentFence(VkDevice, uint32_t count, + const VkFence*, VkBool32 all, + uint64_t timeout) { + ++waits; + EXPECT_EQ(count, 1u); + EXPECT_EQ(all, VK_TRUE); + EXPECT_EQ(timeout, UINT64_MAX); + return wait_result; +} +class OpenXrPresentDependencies : public testing::Test { + protected: + VkPresentInfoKHR info{}; + void SetUp() override { + reset_result = submit_result = wait_result = VK_SUCCESS; + resets = submits = waits = 0; + expected_queue = Access::Handle(30); + expected_semaphores[0] = Access::Handle(31); + expected_semaphores[1] = Access::Handle(32); + info.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR; + info.waitSemaphoreCount = 2; + info.pWaitSemaphores = expected_semaphores; + } + VkResult Run() { + return internal::WaitForPresentDependencies( + Access::Handle(2), expected_queue, info, + Access::Handle(33), ResetPresentFence, SubmitPresentWait, + WaitPresentFence); + } +}; +TEST_F(OpenXrPresentDependencies, ConsumesDependenciesBeforePermittingCopies) { + EXPECT_EQ(Run(), VK_SUCCESS); + EXPECT_EQ(resets, 1); + EXPECT_EQ(submits, 1); + EXPECT_EQ(waits, 1); + EXPECT_EQ(info.waitSemaphoreCount, 2u); +} +TEST_F(OpenXrPresentDependencies, MirrorConsumedWaitsNeedNoSecondSubmission) { + info.waitSemaphoreCount = 0; + info.pWaitSemaphores = nullptr; + EXPECT_EQ(Run(), VK_SUCCESS); + EXPECT_EQ(resets + submits + waits, 0); +} +TEST_F(OpenXrPresentDependencies, SubmitFailureDoesNotWaitForUnsignalledFence) { + submit_result = VK_ERROR_DEVICE_LOST; + EXPECT_EQ(Run(), VK_ERROR_DEVICE_LOST); + EXPECT_EQ(submits, 1); + EXPECT_EQ(waits, 0); +} +TEST_F(OpenXrPresentDependencies, + ConsumedWaitsNeverAuthorizeFallbackOnTimeout) { + wait_result = VK_TIMEOUT; + EXPECT_LT(Run(), VK_SUCCESS); + EXPECT_EQ(submits, 1); + EXPECT_EQ(waits, 1); +} +TEST_F(OpenXrPresentDependencies, InvalidDependenciesDoNotSubmit) { + info.pWaitSemaphores = nullptr; + EXPECT_LT(Run(), VK_SUCCESS); + EXPECT_EQ(resets + submits + waits, 0); +} +TEST(OpenXrPresentQueue, RoutesSameAndSplitFamilyPresentationToGraphicsQueue) { + OpenXrBackend backend; + Access::Configure(backend); + Access::GraphicsFamily(backend, 3); + const auto device = Access::Handle(2); + const auto graphics = Access::Handle(30); + const auto same_family = Access::Handle(31); + const auto split_family = Access::Handle(32); + backend.NoteQueue(device, graphics, 3, 0); + backend.NoteQueue(device, same_family, 3, 1); + backend.NoteQueue(device, split_family, 4, 0); + EXPECT_TRUE(Access::SupportsQueue(backend, graphics)); + EXPECT_TRUE(Access::SupportsQueue(backend, same_family)); + EXPECT_TRUE(Access::SupportsQueue(backend, split_family)); + EXPECT_EQ(Access::CopyQueue(backend, split_family), graphics); + EXPECT_FALSE(Access::SupportsQueue(backend, Access::Handle(99))); +} +} // namespace +} // namespace mocktail::vr + +namespace mocktail::vr { +TEST(OpenXrBackendBinding, + SelectsSingleSampleResolveAlongsideMultisampleColor) { + OpenXrBackend backend; + RobloxVrDeviceBridge bridge; + internal::VrBridgeTestAccess::Enable(bridge); + Access::Configure(backend); + auto* owner = Access::Handle(100); + const auto device = Access::Handle(2); + Access::Attachment(backend, 10, owner, device); + Access::Attachment(backend, 11, owner, device); + Access::AddMsaaAttachment(backend, 10, 11); + Access::ResolvedPass(backend, bridge, owner, 10); + EXPECT_EQ(Access::Image(backend, 0), Access::Handle(10)); + EXPECT_EQ(Access::Layout(backend, 0), + VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL); +} +TEST(OpenXrBackendBinding, RejectsMultisampleImageWithoutResolve) { + OpenXrBackend backend; + RobloxVrDeviceBridge bridge; + internal::VrBridgeTestAccess::Enable(bridge); + Access::Configure(backend); + auto* owner = Access::Handle(100); + Access::Attachment(backend, 10, owner, Access::Handle(2)); + Access::AddMsaaAttachment(backend, 10, 10); + Access::ResolvedPass(backend, bridge, owner, 10); + EXPECT_EQ(Access::Image(backend, 0), VK_NULL_HANDLE); +} +} // namespace mocktail::vr + +namespace mocktail::vr { +namespace { +class OpenXrSplitPresentation : public testing::Test { + protected: + OpenXrBackend backend; + RobloxVrDeviceBridge bridge; + const VkDevice device = Access::Handle(2); + const VkQueue graphics = Access::Handle(30); + const VkQueue present = Access::Handle(31); + const VkQueue graphics_second = Access::Handle(32); + const VkCommandPool graphics_pool = Access::Handle(40); + const VkCommandPool present_pool = Access::Handle(41); + const VkCommandBuffer draw = Access::Handle(50); + const VkCommandBuffer acquire = Access::Handle(51); + const VkSwapchainKHR chain = Access::Handle(60); + const VkImage desktop = Access::Handle(61); + void* const owner = Access::Handle(100); + void SetUp() override { + Access::Configure(backend); + Access::GraphicsFamily(backend, 3); + internal::VrBridgeTestAccess::Enable(bridge); + backend.NoteQueue(device, graphics, 3, 0); + backend.NoteQueue(device, present, 4, 0); + backend.NoteQueue(device, graphics_second, 3, 1); + backend.RecordCommandPool(device, graphics_pool, 3); + backend.RecordCommandPool(device, present_pool, 4); + backend.RecordCommandBuffers(device, graphics_pool, 1, &draw); + backend.RecordCommandBuffers(device, present_pool, 1, &acquire); + RegisterDesktop(true); + } + void RegisterDesktop(bool converted) { + const unsigned families[] = {3, 4}; + VkSwapchainCreateInfoKHR info{}; + info.imageArrayLayers = 1; + info.imageExtent = {800, 600}; + info.imageFormat = VK_FORMAT_B8G8R8A8_UNORM; + info.imageUsage = VK_IMAGE_USAGE_TRANSFER_DST_BIT; + info.imageSharingMode = VK_SHARING_MODE_CONCURRENT; + info.queueFamilyIndexCount = 2; + info.pQueueFamilyIndices = families; + backend.RecordDesktopSwapchain(device, chain, &info, &desktop, 1, + converted); + } + void Draw() { + Access::Attachment(backend, 10, owner, device); + Access::Pass(backend, bridge, 0, 10, owner, + VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, draw); + } +}; +TEST_F(OpenXrSplitPresentation, ActualSubmittedGraphicsQueueOwnsTheEyeCopy) { + Draw(); + EXPECT_EQ(Access::CopyQueue(backend, present), VK_NULL_HANDLE); + backend.NoteCommandBuffersSubmitted(present, 1, &draw); // Wrong pool family. + EXPECT_EQ(Access::CopyQueue(backend, present), VK_NULL_HANDLE); + backend.NoteCommandBuffersSubmitted(graphics_second, 1, &draw); + EXPECT_EQ(Access::CopyQueue(backend, present), graphics_second); + Access::NextFrame(backend); + backend.ResetCommandBuffer(draw); + Draw(); + EXPECT_EQ(Access::CopyQueue(backend, present), VK_NULL_HANDLE); + backend.NoteCommandBuffersSubmitted(graphics, 1, &draw); + EXPECT_EQ(Access::CopyQueue(backend, present), graphics); +} +TEST_F(OpenXrSplitPresentation, + ResetAndFreeDoNotResurrectUnsubmittedEyeCommands) { + Draw(); + backend.ResetCommandBuffer(draw); + backend.NoteCommandBuffersSubmitted(graphics, 1, &draw); + EXPECT_EQ(Access::CopyQueue(backend, present), VK_NULL_HANDLE); + Draw(); + backend.ForgetCommandPool(graphics_pool); + backend.NoteCommandBuffersSubmitted(graphics, 1, &draw); + EXPECT_EQ(Access::CopyQueue(backend, present), VK_NULL_HANDLE); +} +TEST_F(OpenXrSplitPresentation, ConcurrentSwapchainTracksBothFamilies) { + EXPECT_EQ(Access::DesktopCount(backend), 1u); + EXPECT_EQ(backend.DesktopQueueFamilies(device), + (std::vector{3, 4})); + EXPECT_TRUE( + backend.DesktopQueueFamilies(Access::Handle(9)).empty()); +} +TEST_F(OpenXrSplitPresentation, PoolResetAndReusedImageRequireNewSubmission) { + Draw(); + backend.NoteCommandBuffersSubmitted(graphics, 1, &draw); + ASSERT_EQ(Access::CopyQueue(backend, present), graphics); + backend.ResetCommandPool(graphics_pool); + EXPECT_EQ(Access::CopyQueue(backend, present), VK_NULL_HANDLE); + Draw(); + backend.NoteCommandBuffersSubmitted(graphics, 1, &draw); + ASSERT_EQ(Access::CopyQueue(backend, present), graphics); + backend.NoteDestroyImage(Access::Handle(10)); + Draw(); + EXPECT_EQ(Access::CopyQueue(backend, present), VK_NULL_HANDLE); + backend.NoteCommandBuffersSubmitted(graphics_second, 1, &draw); + EXPECT_EQ(Access::CopyQueue(backend, present), graphics_second); +} +TEST_F(OpenXrSplitPresentation, + ReleaseTransitionsOnceAndMatchingAcquireDoesNotRepeat) { + unsigned source = 3, destination = 4; + VkImageLayout old_layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + backend.NormalizeDesktopBarrier(draw, desktop, &source, &destination, + &old_layout, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR); + EXPECT_EQ(source, VK_QUEUE_FAMILY_IGNORED); + EXPECT_EQ(destination, VK_QUEUE_FAMILY_IGNORED); + EXPECT_EQ(old_layout, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); + source = 3; + destination = 4; + backend.NormalizeDesktopBarrier(acquire, desktop, &source, &destination, + &old_layout, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR); + EXPECT_EQ(source, VK_QUEUE_FAMILY_IGNORED); + EXPECT_EQ(destination, VK_QUEUE_FAMILY_IGNORED); + EXPECT_EQ(old_layout, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR); +} +TEST_F(OpenXrSplitPresentation, WsiAcquirePreservesPresentToColorTransition) { + unsigned source = 4, destination = 3; + VkImageLayout old_layout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; + backend.NormalizeDesktopBarrier(draw, desktop, &source, &destination, + &old_layout, + VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); + EXPECT_EQ(source, VK_QUEUE_FAMILY_IGNORED); + EXPECT_EQ(destination, VK_QUEUE_FAMILY_IGNORED); + EXPECT_EQ(old_layout, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR); + source = 4; + destination = 3; + old_layout = VK_IMAGE_LAYOUT_UNDEFINED; + backend.NormalizeDesktopBarrier(draw, desktop, &source, &destination, + &old_layout, + VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); + EXPECT_EQ(old_layout, VK_IMAGE_LAYOUT_UNDEFINED); +} +TEST_F(OpenXrSplitPresentation, + ExplicitPresentReleaseDefersLayoutTransitionToAcquire) { + unsigned source = 4, destination = 3; + VkImageLayout old_layout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; + const auto released_layout = backend.NormalizeDesktopBarrier( + acquire, desktop, &source, &destination, &old_layout, + VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); + EXPECT_EQ(old_layout, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR); + EXPECT_EQ(released_layout, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR); + EXPECT_EQ(source, VK_QUEUE_FAMILY_IGNORED); + source = 4; + destination = 3; + const auto acquired_layout = backend.NormalizeDesktopBarrier( + draw, desktop, &source, &destination, &old_layout, + VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); + EXPECT_EQ(old_layout, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR); + EXPECT_EQ(acquired_layout, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); +} +TEST_F(OpenXrSplitPresentation, + OriginalConcurrentAndUnrelatedImagesRemainUnchanged) { + RegisterDesktop(false); + unsigned source = 3, destination = 4; + VkImageLayout old_layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + backend.NormalizeDesktopBarrier(acquire, desktop, &source, &destination, + &old_layout, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR); + EXPECT_EQ(source, 3u); + EXPECT_EQ(destination, 4u); + RegisterDesktop(true); + backend.NormalizeDesktopBarrier(acquire, Access::Handle(99), &source, + &destination, &old_layout, + VK_IMAGE_LAYOUT_PRESENT_SRC_KHR); + EXPECT_EQ(source, 3u); + backend.RecordDesktopSwapchain(device, chain, nullptr, nullptr, 0); + backend.NormalizeDesktopBarrier(acquire, desktop, &source, &destination, + &old_layout, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR); + EXPECT_EQ(source, 3u); +} +} // namespace +} // namespace mocktail::vr diff --git a/tests/vr/probe_contract_test.py b/tests/vr/probe_contract_test.py index 9cff3d7b..c93f58a8 100644 --- a/tests/vr/probe_contract_test.py +++ b/tests/vr/probe_contract_test.py @@ -65,7 +65,7 @@ assert result.returncode == code, (scenario, result.returncode, result.stdout, result.stderr) assert message in result.stdout, (scenario, result.stdout, result.stderr) assert "SDK: 1.1.63" in result.stdout, result.stdout - assert "Roblox VR rendering: not implemented yet" in result.stdout + assert "Roblox VR rendering: run mocktail --vr" in result.stdout events = trace.read_text().splitlines() if trace.exists() else [] assert events == ([] if scenario == "create_failure" else ["create", "destroy"]), (scenario, events) if code == 0: diff --git a/tests/vr/queue_submission_test.cc b/tests/vr/queue_submission_test.cc new file mode 100644 index 00000000..fe4b5457 --- /dev/null +++ b/tests/vr/queue_submission_test.cc @@ -0,0 +1,214 @@ +#include + +#include +#include +#include +#include +#include + +#include "mocktail/vr/openxr_backend.h" + +using SubmitAdapter = VkResult(VKAPI_PTR *)(PFN_vkQueueSubmit, VkDevice, + VkQueue, uint32_t, + const VkSubmitInfo *, VkFence); +extern "C" VKAPI_ATTR VkResult VKAPI_CALL mocktail_vulkan_submit_synchronized( + PFN_vkQueueSubmit raw, VkDevice device, VkQueue queue, uint32_t count, + const VkSubmitInfo *submits, VkFence fence); +extern "C" void mocktail_vr_xr_queue_submit_adapter(SubmitAdapter); +extern "C" VKAPI_ATTR VkResult VKAPI_CALL +mocktail_vulkan_idle_synchronized(PFN_vkDeviceWaitIdle raw, VkDevice device); + +namespace { +std::mutex state_mutex; +std::condition_variable state_changed; +bool waiting, signal_submitted; +bool idle_release; +std::atomic inside_driver{0}, max_inside{0}; +VKAPI_ATTR VkResult VKAPI_CALL Reset(VkDevice, uint32_t, const VkFence *) { + return VK_SUCCESS; +} +VKAPI_ATTR VkResult VKAPI_CALL Submit(VkQueue, uint32_t, + const VkSubmitInfo *info, VkFence) { + if (info && info->signalSemaphoreCount) { + std::lock_guard lock(state_mutex); + signal_submitted = true; + state_changed.notify_all(); + } + return VK_SUCCESS; +} +VKAPI_ATTR VkResult VKAPI_CALL Wait(VkDevice, uint32_t, const VkFence *, + VkBool32, uint64_t) { + std::unique_lock lock(state_mutex); + waiting = true; + state_changed.notify_all(); + return state_changed.wait_for(lock, std::chrono::seconds(2), + [] { return signal_submitted; }) + ? VK_SUCCESS + : VK_TIMEOUT; +} +VKAPI_ATTR VkResult VKAPI_CALL SlowSubmit(VkQueue, uint32_t, + const VkSubmitInfo *, VkFence) { + const int active = ++inside_driver; + int previous = max_inside.load(); + while (active > previous && + !max_inside.compare_exchange_weak(previous, active)) { + } + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + --inside_driver; + return VK_SUCCESS; +} +VKAPI_ATTR VkResult VKAPI_CALL BlockingIdle(VkDevice) { + std::unique_lock lock(state_mutex); + waiting = true; + state_changed.notify_all(); + return state_changed.wait_for(lock, std::chrono::seconds(2), + [] { return idle_release; }) + ? VK_SUCCESS + : VK_TIMEOUT; +} +TEST(VrQueueSubmission, DeviceIdleExcludesPrivateSubmitsUntilDriverReturns) { + waiting = signal_submitted = idle_release = false; + VkResult idle_result = VK_NOT_READY; + std::thread idler([&] { + idle_result = mocktail_vulkan_idle_synchronized( + BlockingIdle, reinterpret_cast(2)); + }); + bool entered = false; + { + std::unique_lock lock(state_mutex); + entered = state_changed.wait_for(lock, std::chrono::seconds(1), + [] { return waiting; }); + } + const auto semaphore = reinterpret_cast(32); + VkSubmitInfo signal{}; + signal.signalSemaphoreCount = 1; + signal.pSignalSemaphores = &semaphore; + std::thread submitter([&] { + EXPECT_EQ(mocktail_vulkan_submit_synchronized( + Submit, reinterpret_cast(2), + reinterpret_cast(30), 1, &signal, VK_NULL_HANDLE), + VK_SUCCESS); + }); + { + std::unique_lock lock(state_mutex); + EXPECT_FALSE(state_changed.wait_for(lock, std::chrono::milliseconds(50), + [] { return signal_submitted; })); + idle_release = true; + state_changed.notify_all(); + } + idler.join(); + submitter.join(); + EXPECT_TRUE(entered); + EXPECT_EQ(idle_result, VK_SUCCESS); + EXPECT_TRUE(signal_submitted); +} +TEST(VrQueueSubmission, OtherDeviceCanSignalWhileDeviceIdleIsPending) { + waiting = signal_submitted = idle_release = false; + VkResult idle_result = VK_NOT_READY; + // Wait emulates device A waiting for an imported semaphore signal from B. + auto idle = [](VkDevice) -> VkResult { + return Wait(VK_NULL_HANDLE, 0, nullptr, VK_TRUE, UINT64_MAX); + }; + std::thread idler([&] { + idle_result = + mocktail_vulkan_idle_synchronized(idle, reinterpret_cast(2)); + }); + bool entered = false; + { + std::unique_lock lock(state_mutex); + entered = state_changed.wait_for(lock, std::chrono::seconds(1), + [] { return waiting; }); + } + const auto semaphore = reinterpret_cast(32); + VkSubmitInfo signal{}; + signal.signalSemaphoreCount = 1; + signal.pSignalSemaphores = &semaphore; + EXPECT_EQ(mocktail_vulkan_submit_synchronized( + Submit, reinterpret_cast(3), + reinterpret_cast(31), 1, &signal, VK_NULL_HANDLE), + VK_SUCCESS); + idler.join(); + EXPECT_TRUE(entered); + EXPECT_EQ(idle_result, VK_SUCCESS); +} +TEST(VrQueueSubmission, AnotherQueueCanSignalWhilePresentWaitsForFence) { + waiting = signal_submitted = false; + mocktail_vr_xr_queue_submit_adapter(mocktail_vulkan_submit_synchronized); + const auto graphics = reinterpret_cast(30); + const auto signal_queue = reinterpret_cast(31); + const auto semaphore = reinterpret_cast(32); + VkPresentInfoKHR present{}; + present.waitSemaphoreCount = 1; + present.pWaitSemaphores = &semaphore; + VkResult prepared = VK_NOT_READY; + std::thread presenter([&] { + prepared = mocktail::vr::internal::WaitForPresentDependencies( + reinterpret_cast(2), graphics, present, + reinterpret_cast(33), Reset, Submit, Wait); + }); + bool entered = false; + { + std::unique_lock lock(state_mutex); + entered = state_changed.wait_for(lock, std::chrono::seconds(1), + [] { return waiting; }); + } + VkSubmitInfo signal{}; + signal.signalSemaphoreCount = 1; + signal.pSignalSemaphores = &semaphore; + EXPECT_EQ(mocktail_vulkan_submit_synchronized( + Submit, reinterpret_cast(2), signal_queue, 1, &signal, + VK_NULL_HANDLE), + VK_SUCCESS); + presenter.join(); + mocktail_vr_xr_queue_submit_adapter(nullptr); + EXPECT_TRUE(entered); + EXPECT_EQ(prepared, VK_SUCCESS); +} +TEST(VrQueueSubmission, PrivateAndGuestCallsCannotEnterOneQueueConcurrently) { + inside_driver = max_inside = 0; + const auto queue = reinterpret_cast(30); + auto call = [&] { + EXPECT_EQ(mocktail_vulkan_submit_synchronized( + SlowSubmit, reinterpret_cast(2), queue, 0, nullptr, + VK_NULL_HANDLE), + VK_SUCCESS); + }; + std::thread first(call), second(call); + first.join(); + second.join(); + EXPECT_EQ(max_inside, 1); +} +TEST(VrQueueSubmission, FenceWaitDoesNotKeepItsOwnQueueHostLocked) { + waiting = signal_submitted = false; + mocktail_vr_xr_queue_submit_adapter(mocktail_vulkan_submit_synchronized); + const auto queue = reinterpret_cast(30); + const auto semaphore = reinterpret_cast(32); + VkPresentInfoKHR present{}; + present.waitSemaphoreCount = 1; + present.pWaitSemaphores = &semaphore; + VkResult result = VK_NOT_READY; + std::thread presenter([&] { + result = mocktail::vr::internal::WaitForPresentDependencies( + reinterpret_cast(2), queue, present, + reinterpret_cast(33), Reset, Submit, Wait); + }); + { + std::unique_lock lock(state_mutex); + state_changed.wait_for(lock, std::chrono::seconds(1), + [] { return waiting; }); + } + // An unrelated already-ready batch can still be enqueued to this queue. + EXPECT_EQ( + mocktail_vulkan_submit_synchronized(Submit, reinterpret_cast(2), + queue, 0, nullptr, VK_NULL_HANDLE), + VK_SUCCESS); + { + std::lock_guard lock(state_mutex); + signal_submitted = true; + state_changed.notify_all(); + } + presenter.join(); + mocktail_vr_xr_queue_submit_adapter(nullptr); + EXPECT_EQ(result, VK_SUCCESS); +} +} // namespace diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc index 3e190d65..3f323181 100644 --- a/tests/vr/roblox_vr_device_bridge_test.cc +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -54,6 +54,11 @@ const std::vector kEyeGetter = {0x48, 0x63, 0xc6, 0x48, 0xc1, 0xe0, 0x04, 0x48, 0x8b, 0x84, 0x07, 0x50, 0x01, 0x00, 0x00, 0xc3}; +const internal::VrBuildContract& TestContract() { + return *internal::FindVrBuildContract( + "ade08266c67aee88ec9c1d00902150e1684dad3a"); +} + std::vector ValidConstructor() { std::vector code(kConstructorContractSize, 0x90); const std::vector prologue = { @@ -70,6 +75,7 @@ std::vector ValidConstructor() { 0x00, 0x00}, {0x4c, 0x89, 0xbb, 0x98, 0x01, 0x00, 0x00}, }; + std::memcpy(code.data() + 43, markers.front().data(), markers.front().size()); size_t offset = 64; for (const auto &marker : markers) { std::memcpy(code.data() + offset, marker.data(), marker.size()); @@ -240,19 +246,100 @@ TEST(VrResourceReadiness, TEST(VrStateGetterContract, AcceptsExact2998Bytes) { ASSERT_EQ(kStateGetter.size(), kStateGetterContractSize); - EXPECT_TRUE(internal::HasExpectedVrStateGetterContract(kStateGetter.data(), - kStateGetter.size())); + EXPECT_TRUE(internal::HasExpectedVrStateGetterContract( + kStateGetter.data(), kStateGetter.size(), TestContract())); } TEST(VrStateGetterContract, RejectsCorruptionAndShortInput) { - EXPECT_FALSE(internal::HasExpectedVrStateGetterContract(nullptr, 0)); + EXPECT_FALSE( + internal::HasExpectedVrStateGetterContract(nullptr, 0, TestContract())); EXPECT_FALSE(internal::HasExpectedVrStateGetterContract( - kStateGetter.data(), kStateGetter.size() - 1)); + kStateGetter.data(), kStateGetter.size() - 1, TestContract())); std::vector corrupted = kStateGetter; // The sret copy size (0x138) must be pinned; corrupt one byte of it. corrupted[13] ^= 0xff; - EXPECT_FALSE(internal::HasExpectedVrStateGetterContract(corrupted.data(), - corrupted.size())); + EXPECT_FALSE(internal::HasExpectedVrStateGetterContract( + corrupted.data(), corrupted.size(), TestContract())); +} + +TEST(VrBuildContracts, RejectsUnknownBuildsAndMismatchedMetadata) { + EXPECT_EQ(FindRobloxVrDeviceProfile("unknown"), nullptr); + auto profile = MakeBuildProfile(true, true); + profile.elf_build_id = "5f0704edd9064f566ee3d6df2bd2fabbcc709f03"; + RobloxVrDeviceBridge bridge; + EXPECT_FALSE(bridge.Install(profile).ok()); + profile.vr_debug_device_bridge = + *FindRobloxVrDeviceProfile(profile.elf_build_id); + EXPECT_TRUE(bridge.Install(profile).ok()); + EXPECT_FALSE(bridge.active()); +} + +TEST(VrBuildContracts, RestoresVrForApprovedProfileWithoutVrMetadata) { + auto profile = MakeBuildProfile(false, true); + profile.elf_build_id = "5f0704edd9064f566ee3d6df2bd2fabbcc709f03"; + const auto* vr = FindRobloxVrDeviceProfile(profile.elf_build_id); + ASSERT_NE(vr, nullptr); + EXPECT_EQ(vr->device_create_framebuffer_vtable_offset, 0x188u); + profile.vr_debug_device_bridge = *vr; + RobloxVrDeviceBridge bridge; + EXPECT_TRUE(bridge.Install(profile).ok()); +} + +TEST(VrStateGetterContract, Verifies3092CopyTargetAndRejectsWrongBuild) { + const auto& latest = *internal::FindVrBuildContract( + "5f0704edd9064f566ee3d6df2bd2fabbcc709f03"); + auto code = kStateGetter; + const std::uint8_t displacement[] = {0xa3, 0x0d, 0x40, 0x03}; + std::memcpy(code.data() + 19, displacement, sizeof(displacement)); + EXPECT_TRUE(internal::HasExpectedVrStateGetterContract(code.data(), + code.size(), latest)); + EXPECT_FALSE(internal::HasExpectedVrStateGetterContract( + code.data(), code.size(), TestContract())); + code[19] ^= 1; + EXPECT_FALSE(internal::HasExpectedVrStateGetterContract(code.data(), + code.size(), latest)); +} + +TEST(VrConstructorContract, VerifiesRelocatedVtableFor3092) { + const auto& latest = + *FindRobloxVrDeviceProfile("5f0704edd9064f566ee3d6df2bd2fabbcc709f03"); + auto code = ValidConstructor(); + const std::uint8_t displacement[] = {0xf2, 0xe0, 0x50, 0x03}; + std::memcpy(code.data() + 46, displacement, sizeof(displacement)); + EXPECT_TRUE(internal::HasExpectedVrConstructorContract(code.data(), + code.size(), latest)); + EXPECT_FALSE(internal::HasExpectedVrConstructorContract( + code.data(), code.size(), TestProfile())); +} + +TEST(VrPointerContract, PinsWorkspaceAndCameraSlotForEachBuild) { + for (const auto* id : {"ade08266c67aee88ec9c1d00902150e1684dad3a", + "5f0704edd9064f566ee3d6df2bd2fabbcc709f03"}) { + const auto& contract = *internal::FindVrBuildContract(id); + std::array code{}; + const std::uint8_t prologue[] = {0x55, 0x48, 0x89, 0xe5, 0x41, + 0x57, 0x41, 0x56, 0x41, 0x54, + 0x53, 0x48, 0x83, 0xec, 0x70}; + std::memcpy(code.data(), prologue, sizeof(prologue)); + code[0x59] = 0xe8; + const std::int32_t displacement = + static_cast(contract.workspace_getter_rva) - + (contract.pointer_frame_rva + 0x5e); + std::memcpy(code.data() + 0x5a, &displacement, sizeof(displacement)); + code[0x6d] = 0xff; + code[0x6e] = 0x91; + const std::uint32_t slot = contract.camera_vtable_offset; + std::memcpy(code.data() + 0x6f, &slot, sizeof(slot)); + EXPECT_TRUE(internal::HasExpectedVrPointerContract(code.data(), code.size(), + contract)); + code[0x6f] ^= 8; + EXPECT_FALSE(internal::HasExpectedVrPointerContract(code.data(), + code.size(), contract)); + code[0x6f] ^= 8; + code[0x5a] ^= 1; + EXPECT_FALSE(internal::HasExpectedVrPointerContract(code.data(), + code.size(), contract)); + } } TEST(VrEyeGetterContract, AcceptsExact2998Bytes) { @@ -271,10 +358,10 @@ TEST(VrEyeGetterContract, RejectsDifferentEyeSlotOffset) { TEST(VrConstructorContract, AcceptsMarkersAndRejectsEachMissingOne) { const std::vector valid = ValidConstructor(); - EXPECT_TRUE( - internal::HasExpectedVrConstructorContract(valid.data(), valid.size())); - EXPECT_FALSE(internal::HasExpectedVrConstructorContract(valid.data(), - valid.size() - 1)); + EXPECT_TRUE(internal::HasExpectedVrConstructorContract( + valid.data(), valid.size(), TestProfile())); + EXPECT_FALSE(internal::HasExpectedVrConstructorContract( + valid.data(), valid.size() - 1, TestProfile())); // Corrupting the graphics-device capture (+0x198 store) must fail closed. std::vector no_device_store = valid; for (size_t index = 0; index + 7 <= no_device_store.size(); ++index) { @@ -285,7 +372,7 @@ TEST(VrConstructorContract, AcceptsMarkersAndRejectsEachMissingOne) { } } EXPECT_FALSE(internal::HasExpectedVrConstructorContract( - no_device_store.data(), no_device_store.size())); + no_device_store.data(), no_device_store.size(), TestProfile())); } TEST(VrInitializerContract, PinsDeviceVtableSlotFromProfile) { @@ -484,7 +571,8 @@ TEST(VrProfileParsing, Issue153VrBridgeFailsWithoutMetadataForArmedProfile) { "count_method_rva": "0x32d1000", "info_method_rva": "0x32d10a0", "current_method_rva": "0x32d1050", - "select_method_rva": "0x32d0d04" + "select_method_rva": "0x32d0d04", + "vtable_layout_version": 2 }, "reason": "issue 153 reproduction profile: armed without vr metadata" }] @@ -500,6 +588,8 @@ TEST(VrProfileParsing, Issue153VrBridgeFailsWithoutMetadataForArmedProfile) { ASSERT_TRUE(result); ASSERT_TRUE(result.profile.has_value()); ASSERT_TRUE(result.profile->fmod_output_device_bridge.has_value()); + EXPECT_EQ(result.profile->fmod_output_device_bridge->vtable_layout_version, + 2u); ASSERT_TRUE(result.profile->user_game_settings_fullscreen_setter_rva .has_value()); // But it carries no VR debug-device bridge metadata. @@ -607,6 +697,30 @@ TEST(VrHandPoseBridge, ValidGripPosesFillBothHandRecordsInMetres) { EXPECT_FLOAT_EQ(right_pos[0], 0.25f); } +TEST(VrFloorPoseBridge, PublishesStagePoseInMetresAndClearsTrackingLoss) { + std::array state{}; + state[internal::kStateReadyOffsetInCopy] = 1; + ScriptedPoseSample pose; + pose.valid = pose.floor_valid = true; + pose.floor_position[1] = -1.7f; + pose.floor_orientation[1] = std::sin(.2f); + pose.floor_orientation[3] = std::cos(.2f); + ASSERT_TRUE(internal::ApplyXrPose(state.data(), pose)); + EXPECT_EQ(state[internal::kExtraRecordOffset], 1); + float height = 0; + std::memcpy(&height, state.data() + internal::kExtraRecordOffset + 8, 4); + EXPECT_FLOAT_EQ(height, -1.7f); + EXPECT_EQ(state[internal::kStateReadyOffsetInCopy], 1); + pose.floor_valid = false; + ASSERT_TRUE(internal::ApplyXrPose(state.data(), pose)); + EXPECT_EQ(state[internal::kExtraRecordOffset], 0); + EXPECT_EQ(state[internal::kHeadRecordOffset], 1); + pose.floor_valid = true; + pose.floor_position[1] = std::numeric_limits::quiet_NaN(); + ASSERT_TRUE(internal::ApplyXrPose(state.data(), pose)); + EXPECT_EQ(state[internal::kExtraRecordOffset], 0); +} + TEST(VrHandPoseBridge, UntrackedOrNonFiniteHandsStayInvalidWithoutTouchingHead) { std::array state; state.fill(0); diff --git a/tests/vr/vr_pose_math_test.cc b/tests/vr/vr_pose_math_test.cc index 4c931b19..53098d54 100644 --- a/tests/vr/vr_pose_math_test.cc +++ b/tests/vr/vr_pose_math_test.cc @@ -151,21 +151,64 @@ TEST(VrPoseMath, FovToTangentsPinsUpDownLeftRightOrderAndSigns) { EXPECT_NE(tangents.up, tangents.down); } -// ---- canted-view rejection ---- - -TEST(VrPoseMath, ParallelEyeOrientationsAcceptedCantedRejected) { - EXPECT_TRUE(EyeOrientationIsParallel(Quat{0.f, 0.f, 0.f, 1.f})); - // Sign flip only (q == -q) is still parallel. - EXPECT_TRUE(EyeOrientationIsParallel(Quat{0.f, 0.f, 0.f, -1.f})); - // Exactly at tolerance. - EXPECT_TRUE(EyeOrientationIsParallel( - Quat{kCantedTolerance, 0.f, 0.f, 1.f})); - // A canted runtime rotates the eye about its view axis (+Z): must reject. - const Quat canted = QuatFromAxisAngle(Vec3{0.f, 0.f, 1.f}, 0.1f); - EXPECT_FALSE(EyeOrientationIsParallel(canted)); - // Any axis above tolerance rejects. - EXPECT_FALSE(EyeOrientationIsParallel( - Quat{0.f, 2.f * kCantedTolerance, 0.f, 1.f})); +TEST(VrPoseMath, ParallelProjectionKeepsUnrotatedFovExact) { + const EyeFov display{-.7f, .8f, .9f, -.6f}; + for (float sign : {1.f, -1.f}) { + EyeFov rendered; + ASSERT_TRUE(ParallelEyeFov({0, 0, 0, sign}, display, &rendered)); + EXPECT_EQ(rendered.angle_left, display.angle_left); + EXPECT_EQ(rendered.angle_right, display.angle_right); + EXPECT_EQ(rendered.angle_up, display.angle_up); + EXPECT_EQ(rendered.angle_down, display.angle_down); + } +} + +TEST(VrPoseMath, ParallelProjectionContainsEveryCantedDisplayRay) { + const EyeFov display{-.7f, .8f, .9f, -.6f}; + for (const Vec3 axis : {Vec3{1, 0, 0}, Vec3{0, 1, 0}, Vec3{0, 0, 1}}) { + for (float angle : {-.2f, .2f}) { + const Quat rotation = QuatFromAxisAngle(axis, angle); + EyeFov rendered; + ASSERT_TRUE(ParallelEyeFov(rotation, display, &rendered)); + for (int x = 0; x <= 4; ++x) { + for (int y = 0; y <= 4; ++y) { + const Vec3 ray = + QuatRotate(rotation, {std::tan(display.angle_left) + + x * .25f * + (std::tan(display.angle_right) - + std::tan(display.angle_left)), + std::tan(display.angle_down) + + y * .25f * + (std::tan(display.angle_up) - + std::tan(display.angle_down)), + -1}); + const float horizontal = std::atan2(ray.x, -ray.z); + const float vertical = std::atan2(ray.y, -ray.z); + EXPECT_GE(horizontal, rendered.angle_left - 1e-6f); + EXPECT_LE(horizontal, rendered.angle_right + 1e-6f); + EXPECT_GE(vertical, rendered.angle_down - 1e-6f); + EXPECT_LE(vertical, rendered.angle_up + 1e-6f); + } + } + const Pose head{{1, 2, 3}, QuatFromYawPitch(.4f, -.1f)}; + const Pose parallel_eye{{-.03f, .01f, .02f}, {}}; + const Pose submitted = PoseMultiply(head, parallel_eye); + EXPECT_TRUE( + QuatSameRotation(submitted.orientation, head.orientation, 1e-6f)); + } + } +} + +TEST(VrPoseMath, UnrepresentableProjectionDoesNotModifyOutput) { + const EyeFov display{-.7f, .8f, .9f, -.6f}; + EyeFov rendered{1, 2, 3, 4}; + EXPECT_FALSE(ParallelEyeFov({0, 0, 0, 0}, display, &rendered)); + EXPECT_FALSE(ParallelEyeFov({}, {.8f, -.7f, .9f, -.6f}, &rendered)); + EXPECT_FALSE( + ParallelEyeFov(QuatFromAxisAngle({0, 1, 0}, 1.5f), display, &rendered)); + EXPECT_FALSE(ParallelEyeFov({}, display, nullptr)); + EXPECT_EQ(rendered.angle_left, 1); + EXPECT_EQ(rendered.angle_down, 4); } // ---- units: the 0.10 m translation check through the native conversion ---- diff --git a/tests/vr/xr_actions_test.cc b/tests/vr/xr_actions_test.cc index cd2c6533..1978e8b4 100644 --- a/tests/vr/xr_actions_test.cc +++ b/tests/vr/xr_actions_test.cc @@ -1,12 +1,16 @@ #include "mocktail/vr/xr_actions.h" -#include "runtime/roblox_input_router.h" -#include "runtime/roblox_xr_controller_input.h" -#include + #include -#include #include + +#include +#include +#include #include +#include "runtime/roblox_input_router.h" +#include "runtime/roblox_xr_controller_input.h" + namespace { using namespace mocktail::vr; template T Handle(uintptr_t n) { return reinterpret_cast(n); } @@ -23,6 +27,7 @@ float last_amplitude; XrDuration last_duration; XrPath last_haptic_hand; int simple_haptic_bindings; +std::map> suggested_paths; } // namespace extern "C" { XRAPI_ATTR XrResult XRAPI_CALL @@ -69,6 +74,10 @@ XRAPI_ATTR XrResult XRAPI_CALL xrCreateAction(XrActionSet, XRAPI_ATTR XrResult XRAPI_CALL xrDestroyAction(XrAction) { return XR_SUCCESS; } XRAPI_ATTR XrResult XRAPI_CALL xrSuggestInteractionProfileBindings( XrInstance, const XrInteractionProfileSuggestedBinding *info) { + for (std::uint32_t i = 0; i < info->countSuggestedBindings; ++i) { + suggested_paths[info->interactionProfile].push_back( + info->suggestedBindings[i].binding); + } if (info->interactionProfile == paths["/interaction_profiles/khr/simple_controller"]) for (std::uint32_t i = 0; i < info->countSuggestedBindings; ++i) @@ -165,6 +174,7 @@ class XrActionsProduction : public testing::Test { void SetUp() override { names.clear(); paths.clear(); + suggested_paths.clear(); next_handle = 100; focused = input_active = true; simple = click = touching = false; @@ -188,6 +198,33 @@ class XrActionsProduction : public testing::Test { } }; +TEST_F(XrActionsProduction, BindsTouchProAndPlusAndOmitsQuest2Thumbrest) { + for (const char* name : + {"/interaction_profiles/facebook/touch_controller_pro", + "/interaction_profiles/meta/touch_controller_plus", + "/interaction_profiles/meta/touch_pro_controller", + "/interaction_profiles/meta/touch_plus_controller"}) { + const auto& bindings = suggested_paths.at(paths.at(name)); + for (const char* component : + {"/user/hand/left/input/grip/pose", "/user/hand/right/input/aim/pose", + "/user/hand/left/input/trigger/value", + "/user/hand/right/input/squeeze/value", + "/user/hand/left/output/haptic", "/user/hand/right/output/haptic"}) { + EXPECT_NE( + std::find(bindings.begin(), bindings.end(), paths.at(component)), + bindings.end()) + << name << " " << component; + } + } + const auto& quest = suggested_paths.at( + paths.at("/interaction_profiles/meta/touch_controller_quest_2")); + for (const char* component : {"/user/hand/left/input/thumbrest/touch", + "/user/hand/right/input/thumbrest/touch"}) { + EXPECT_EQ(std::find(quest.begin(), quest.end(), paths.at(component)), + quest.end()); + } +} + TEST_F(XrActionsProduction, HapticSamplesKeepHandLeaseAndClampAmplitude) { EXPECT_EQ(simple_haptic_bindings, 2); actions.RequestHaptics(1, 1.5f, 50000000, 0); From d54d20de6437534b692736f0cd327bf6d3f34679 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Mon, 12 Oct 2026 01:45:08 -0500 Subject: [PATCH 14/58] audio: align input capture test with shared_ptr atomic store and restore FMOD profile functions --- src/audio/roblox_output_device_bridge.cc | 44 ++++++++++++++++++++++++ tests/native_input_capture_test.cc | 8 ++--- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/audio/roblox_output_device_bridge.cc b/src/audio/roblox_output_device_bridge.cc index 0134c214..0fcde4bc 100644 --- a/src/audio/roblox_output_device_bridge.cc +++ b/src/audio/roblox_output_device_bridge.cc @@ -134,6 +134,50 @@ RobloxOutputDeviceBridge::RobloxOutputDeviceBridge() = default; namespace internal { +const FmodInputCaptureProfile* FindFmodInputCaptureProfile( + const compat::BuildProfile& profile) { + static constexpr FmodInputCaptureProfile kProfiles[] = { + {"ade08266c67aee88ec9c1d00902150e1684dad3a", + 0x6c58040, + 1, + {4, 11, 12, 13, 14, 15}, + {0x32d0e9a, 0x32d0a34, 0x32d16a8, 0x32d1f84, 0x32d056c, 0x32d26ac}}, + {"5f0704edd9064f566ee3d6df2bd2fabbcc709f03", + 0x6cd3ce0, + 2, + {4, 13, 14, 15, 16, 17}, + {0x320c630, 0x320bcf6, 0x320d4b4, 0x320dd94, 0x320b844, 0x320e49a}}, + }; + if (!profile.allow_host_abi_bridges || !profile.fmod_output_device_bridge) { + return nullptr; + } + const auto& output = *profile.fmod_output_device_bridge; + for (const auto& capture : kProfiles) { + if (capture.build_id == profile.elf_build_id && + capture.vtable_rva == output.vtable_rva && + output.vtable_layout_version >= 0 && + capture.vtable_layout_version == + static_cast(output.vtable_layout_version)) { + return &capture; + } + } + return nullptr; +} + +bool HasExpectedFmodInputCaptureVtable(const std::uintptr_t* vtable, + std::uintptr_t image_base, + const FmodInputCaptureProfile& profile) { + if (vtable == nullptr || image_base == 0) return false; + for (std::size_t i = 0; i < profile.slots.size(); ++i) { + if (profile.method_rvas[i] > + std::numeric_limits::max() - image_base || + vtable[profile.slots[i]] != image_base + profile.method_rvas[i]) { + return false; + } + } + return true; +} + bool HasExpectedFmodStringConstructorContract(const std::uint8_t* code, std::size_t size) { constexpr std::array diff --git a/tests/native_input_capture_test.cc b/tests/native_input_capture_test.cc index dd7319c9..2e470314 100644 --- a/tests/native_input_capture_test.cc +++ b/tests/native_input_capture_test.cc @@ -91,7 +91,7 @@ class NativeInputCaptureTest : public testing::Test { } void TearDown() override { - g_native_capture.store(nullptr); + std::atomic_store(&g_native_capture, std::shared_ptr{}); if (capture_) capture_->Shutdown(); EXPECT_TRUE(ShutdownSdlAudioSubsystem().ok()); SDL_ResetHint(SDL_HINT_AUDIO_DRIVER); @@ -101,14 +101,14 @@ class NativeInputCaptureTest : public testing::Test { } void Publish(bool enabled) { - capture_ = std::make_unique( + capture_ = std::make_shared( NativeInputCapture::SinkAbi{1, &TestRelro, &TestCode, &OriginalLatency}, enabled); - g_native_capture.store(capture_.get()); + std::atomic_store(&g_native_capture, capture_); } char pcm_path_[64] = "/tmp/mocktail-native-capture-XXXXXX"; - std::unique_ptr capture_; + std::shared_ptr capture_; TestSink sink_{kSinkVtable.data()}; TestOwner owner_{kOwnerVtable.data()}; int device_ = 0; From a441fa7110c443b82385cb51f1c6699fd139b08f Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Mon, 12 Oct 2026 02:16:21 -0500 Subject: [PATCH 15/58] assets: fix shader pack path resolution and root directory normalization --- src/legacy/legacy_runtime.cc | 8 ++++++-- src/main.cc | 13 +++++++++++-- stubs/libandroid_stub.cc | 34 ++++++++++++++++++++++++++++++++-- tests/asset_manager_test.cc | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 6 deletions(-) diff --git a/src/legacy/legacy_runtime.cc b/src/legacy/legacy_runtime.cc index 60c2e90f..0de58c46 100644 --- a/src/legacy/legacy_runtime.cc +++ b/src/legacy/legacy_runtime.cc @@ -2198,8 +2198,12 @@ void MocktailSetAssetPath(JNIEnv* env, jstring asset_path) { if (path.empty()) { path = DefaultAssetPath(); } - setenv("MOCKTAIL_ASSET_ROOT", path.c_str(), 1); - std::cout << " [engine] asset root set to " << path << '\n' << std::flush; + std::filesystem::path root_path(path); + if (root_path.filename() == "content") { + root_path = root_path.parent_path(); + } + setenv("MOCKTAIL_ASSET_ROOT", root_path.c_str(), 1); + std::cout << " [engine] asset root set to " << root_path.string() << '\n' << std::flush; } void MocktailAppBridgeInit(JNIEnv* env, jstring app_params) { diff --git a/src/main.cc b/src/main.cc index 88081305..8aeb294c 100644 --- a/src/main.cc +++ b/src/main.cc @@ -626,6 +626,10 @@ int main(int argc, char* argv[]) { std::cerr << "[FATAL] Cannot bind assets to explicit Roblox library\n"; return EXIT_FAILURE; } + const std::filesystem::path adjacent_root = + adjacent_assets.filename() == "content" ? adjacent_assets.parent_path() + : adjacent_assets; + setenv("MOCKTAIL_ASSET_ROOT", adjacent_root.c_str(), 1); std::cout << " [runtime] explicit Roblox library uses adjacent assets: " << adjacent_assets << '\n'; } @@ -717,8 +721,13 @@ int main(int argc, char* argv[]) { if ((needs_active_library && setenv("ROBLOX_LIB_PATH", active.roblox_library.c_str(), 1) != 0) || (needs_active_assets && - setenv("MOCKTAIL_ASSET_PATH", active.assets_content.c_str(), 1) != - 0)) { + (setenv("MOCKTAIL_ASSET_PATH", active.assets_content.c_str(), 1) != 0 || + setenv("MOCKTAIL_ASSET_ROOT", + (active.assets_content.filename() == "content" + ? active.assets_content.parent_path() + : active.assets_content) + .c_str(), + 1) != 0))) { std::cerr << "[FATAL] Cannot activate resolved Roblox payload\n"; return EXIT_FAILURE; } diff --git a/stubs/libandroid_stub.cc b/stubs/libandroid_stub.cc index c951a4eb..81f8de91 100644 --- a/stubs/libandroid_stub.cc +++ b/stubs/libandroid_stub.cc @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -159,7 +160,13 @@ ResolvedAsset ResolveAssetPath(const char* filename) { if (filename == nullptr) { return {}; } - const std::string_view requested_view = StripAssetUriPrefix(filename); + std::string_view requested_view = StripAssetUriPrefix(filename); + std::string normalized_storage; + if (requested_view.find("..") != std::string_view::npos) { + std::filesystem::path p(requested_view); + normalized_storage = p.lexically_normal().string(); + requested_view = normalized_storage; + } if (HasUnsafePathSegment(requested_view)) { return {}; } @@ -167,6 +174,13 @@ ResolvedAsset ResolveAssetPath(const char* filename) { const std::string requested(requested_view); const char* root_env = GetEnvNonEmpty("MOCKTAIL_ASSET_ROOT"); const std::string root = root_env != nullptr ? root_env : "rbx_bin/assets"; + std::string root_parent; + { + std::filesystem::path rp(root); + if (rp.filename() == "content") { + root_parent = rp.parent_path().string(); + } + } const char* overlay_env = GetEnvNonEmpty("MOCKTAIL_ASSET_OVERLAY_DIR"); const std::string overlay = overlay_env != nullptr ? overlay_env : "rbx_bin/asset_overlay"; @@ -201,7 +215,7 @@ ResolvedAsset ResolveAssetPath(const char* filename) { } } - const std::vector root_candidates = { + std::vector root_candidates = { root + "/" + requested, root + "/" + stripped, root + "/content/" + requested, @@ -209,6 +223,14 @@ ResolvedAsset ResolveAssetPath(const char* filename) { root + "/ExtraContent/" + requested, root + "/ExtraContent/" + stripped, }; + if (!root_parent.empty()) { + root_candidates.push_back(root_parent + "/" + requested); + root_candidates.push_back(root_parent + "/" + stripped); + root_candidates.push_back(root_parent + "/content/" + requested); + root_candidates.push_back(root_parent + "/content/" + stripped); + root_candidates.push_back(root_parent + "/ExtraContent/" + requested); + root_candidates.push_back(root_parent + "/ExtraContent/" + stripped); + } for (const std::string& candidate : root_candidates) { if (FileExists(candidate)) { return CacheResolvedAsset(cache_key, {candidate, false}); @@ -365,6 +387,9 @@ AAsset* AAssetManager_open(AAssetManager* mgr, const char* filename, } ResolvedAsset resolved = ResolveAssetPath(filename); if (resolved.path.empty()) { + if (AssetTraceEnabled()) { + std::fprintf(stderr, "[asset] open failed for %s\n", filename); + } return nullptr; } auto* asset = new AAsset; @@ -506,6 +531,11 @@ int AAsset_openFileDescriptor(AAsset* asset, off_t* outStart, fd = ::open(asset->path.c_str(), O_RDONLY | O_CLOEXEC); } if (fd < 0) { + if (AssetTraceEnabled()) { + std::fprintf(stderr, + "[asset] openFileDescriptor failed for %s (errno=%d)\n", + asset->path.c_str(), errno); + } return -1; } if (outStart != nullptr) { diff --git a/tests/asset_manager_test.cc b/tests/asset_manager_test.cc index 21464623..8516ca58 100644 --- a/tests/asset_manager_test.cc +++ b/tests/asset_manager_test.cc @@ -411,4 +411,40 @@ TEST_F(AssetManagerTest, CacheSeparatesOverlayDirectoryChanges) { AAsset_close(second); } +TEST_F(AssetManagerTest, ResolvesShadersWhenAssetRootPointsToContentOrAssets) { + WriteAsset("shaders/shaders_vulkan_mobile.pack", "shader-pack-data"); + WriteAsset("content/test.txt", "content-data"); + + AAssetManager* manager = AAssetManager_fromJava(nullptr, nullptr); + + // 1. Standard asset root pointing to parent assets directory: + AAsset* asset = AAssetManager_open(manager, "shaders/shaders_vulkan_mobile.pack", 0); + ASSERT_NE(asset, nullptr); + off_t start = -1; + off_t length = -1; + int fd = AAsset_openFileDescriptor(asset, &start, &length); + ASSERT_GE(fd, 0); + EXPECT_EQ(length, 16); + close(fd); + AAsset_close(asset); + + // 2. Relative traversal inside asset folder should normalize safely: + AAsset* relative_asset = AAssetManager_open( + manager, "content/../shaders/shaders_vulkan_mobile.pack", 0); + ASSERT_NE(relative_asset, nullptr); + AAsset_close(relative_asset); + + // 3. Fallback when MOCKTAIL_ASSET_ROOT accidentally points to /content directory: + const std::string content_root = root_ + "/content"; + ASSERT_EQ(setenv("MOCKTAIL_ASSET_ROOT", content_root.c_str(), 1), 0); + AAsset* fallback_asset = AAssetManager_open( + manager, "shaders/shaders_vulkan_mobile.pack", 0); + ASSERT_NE(fallback_asset, nullptr); + fd = AAsset_openFileDescriptor(fallback_asset, &start, &length); + ASSERT_GE(fd, 0); + EXPECT_EQ(length, 16); + close(fd); + AAsset_close(fallback_asset); +} + } // namespace From 509822974bb545a27b1156c5551eddb1cd364af5 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Mon, 12 Oct 2026 05:42:33 -0500 Subject: [PATCH 16/58] vr: Recreate OpenXR instance on guest Vulkan instance recreation SteamVR (vrclient.so) binds its Vulkan renderer to the lifetime of the XrInstance. When Roblox destroys the LuaApp Vulkan device and creates a second VkInstance/VkDevice for UGC experiences, reusing the original XrInstance causes SteamVR to dereference freed device handles in xrEndFrame. Tear down and recreate the XrInstance whenever a new guest Vulkan instance is initialized, resetting session state, spaces, and controller actions. --- include/mocktail/vr/openxr_backend.h | 5 + src/vr/openxr_backend.cc | 179 +++++++++++++++++---------- 2 files changed, 116 insertions(+), 68 deletions(-) diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index bc053d8c..d5edbd23 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -279,6 +279,8 @@ class OpenXrBackend final { bool InitializeGlesSessionLocked(std::string *error); bool InitializeSessionResourcesLocked(std::string *error); bool CopyGlesEyesIntoSwapchains(); + bool EnsureXrInstanceLocked(std::string* error = nullptr); + void DestroyXrInstanceLocked(); void DisarmLocked(); void* ResolveFunction(const char* name) const; void SetupDeviceAfterCreationLocked(VkPhysicalDevice physical_device, @@ -339,6 +341,9 @@ class OpenXrBackend final { // Runtime name captured at xrCreateInstance so the bind-time diagnostics line // can report it without re-entering runtime_name() under the held mutex. std::string runtime_name_; + bool xr_instance_used_ = false; + std::string user_manifest_; + std::string runtime_manifest_; VkInstance vk_instance_ = VK_NULL_HANDLE; VkDevice vk_device_ = VK_NULL_HANDLE; diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 488a1749..8ca9153b 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -251,65 +251,26 @@ OpenXrBackend* ActiveVrBackend() { return g_backend; } -Status OpenXrBackend::Arm(VrGraphicsApi graphics_api) { - std::lock_guard lock(mutex_); - if (armed_.load(std::memory_order_acquire)) { - return Status::Ok(); - } +void OpenXrBackend::DestroyXrInstanceLocked() { { - std::lock_guard slot_lock(g_backend_slot_mutex); - if (g_backend != nullptr && g_backend != this) { - return Status::Error(StatusCode::kFailedPrecondition, - "another OpenXR backend owns the process"); - } - g_backend = this; + std::lock_guard lock(controller_mutex_); + actions_.Destroy(); + actions_created_ = false; + } + if (xr_instance_ != nullptr) { + const auto instance = static_cast(xr_instance_); + (void)xrDestroyInstance(instance); + xr_instance_ = nullptr; + } + xr_system_ = 0; + xr_instance_used_ = false; +} + +bool OpenXrBackend::EnsureXrInstanceLocked(std::string* error) { + if (xr_instance_ != nullptr) { + return true; } try { - const std::string user_manifest = EnvString("XR_RUNTIME_JSON", ""); - const auto preference = EnvString("MOCKTAIL_VR_RUNTIME", "auto"); - Require( - preference == "auto" || preference == "system" || - preference == "wivrn" || preference == "steamvr" || - preference == "alvr", - "MOCKTAIL_VR_RUNTIME must be auto, system, wivrn, steamvr, or alvr"); - const auto discovery = - user_manifest.empty() - ? DiscoverVrRuntime( - preference, EnvString("HOME", ""), - EnvString("XDG_CONFIG_HOME", ""), - EnvString("XDG_CONFIG_DIRS", ""), - EnvString("XDG_RUNTIME_DIR", - ("/run/user/" + std::to_string(getuid())).c_str())) - : VrRuntimeDiscovery{{}, "explicit XR_RUNTIME_JSON override"}; - const auto manifest = - SelectVrRuntimeManifest(user_manifest, discovery.candidates); - Log(" [vr-backend] runtime selection: %s\n", discovery.reason.c_str()); - Require(!manifest.empty() || - (preference != "steamvr" && preference != "alvr" && - preference != "wivrn"), - "Selected VR runtime was not found. For ALVR, install SteamVR and " - "set XR_RUNTIME_JSON " - "to its steamxr_linux64.json if automatic discovery cannot find " - "the installation. " - "For WiVRn, install its native runtime or select its manifest " - "explicitly."); - if (!manifest.empty()) { - Require(setenv("XR_RUNTIME_JSON", manifest.c_str(), 1) == 0, - "Cannot select the OpenXR runtime"); - Log(" [vr-backend] runtime manifest: %s\n", manifest.c_str()); - } else { - Log(" [vr-backend] using registered OpenXR runtime; connect the headset " - "in WiVRn or SteamVR/ALVR first\n"); - } - graphics_api_ = graphics_api; - if (graphics_api_ == VrGraphicsApi::kVulkan) { - vk_loader_ = dlopen("libvulkan.so.1", RTLD_NOW | RTLD_LOCAL); - Require(vk_loader_ != nullptr, - "Cannot load the host Vulkan loader libvulkan.so.1"); - auto loader_gipa = reinterpret_cast( - dlsym(vk_loader_, "vkGetInstanceProcAddr")); - Require(loader_gipa != nullptr, "Host vkGetInstanceProcAddr is missing"); - } const char *vk_extensions[] = {XR_KHR_VULKAN_ENABLE2_EXTENSION_NAME}; const char *gl_extensions[] = {XR_KHR_OPENGL_ES_ENABLE_EXTENSION_NAME, XR_MNDX_EGL_ENABLE_EXTENSION_NAME}; @@ -357,7 +318,7 @@ Status OpenXrBackend::Arm(VrGraphicsApi graphics_api) { } if (instance_result == XR_ERROR_RUNTIME_UNAVAILABLE) { throw std::runtime_error("xrCreateInstance: " + - VrRuntimeUnavailableHint(!user_manifest.empty(), manifest)); + VrRuntimeUnavailableHint(!user_manifest_.empty(), runtime_manifest_)); } if (instance_result == XR_ERROR_EXTENSION_NOT_PRESENT && graphics_api_ == VrGraphicsApi::kOpenGles) { @@ -434,6 +395,79 @@ Status OpenXrBackend::Arm(VrGraphicsApi graphics_api) { "continues without controller input)\n", actions_status.message().c_str()); } + return true; + } catch (const std::exception& err) { + if (error != nullptr) { + *error = err.what(); + } + DestroyXrInstanceLocked(); + return false; + } +} + +Status OpenXrBackend::Arm(VrGraphicsApi graphics_api) { + std::lock_guard lock(mutex_); + if (armed_.load(std::memory_order_acquire)) { + return Status::Ok(); + } + { + std::lock_guard slot_lock(g_backend_slot_mutex); + if (g_backend != nullptr && g_backend != this) { + return Status::Error(StatusCode::kFailedPrecondition, + "another OpenXR backend owns the process"); + } + g_backend = this; + } + try { + user_manifest_ = EnvString("XR_RUNTIME_JSON", ""); + const auto preference = EnvString("MOCKTAIL_VR_RUNTIME", "auto"); + Require( + preference == "auto" || preference == "system" || + preference == "wivrn" || preference == "steamvr" || + preference == "alvr", + "MOCKTAIL_VR_RUNTIME must be auto, system, wivrn, steamvr, or alvr"); + const auto discovery = + user_manifest_.empty() + ? DiscoverVrRuntime( + preference, EnvString("HOME", ""), + EnvString("XDG_CONFIG_HOME", ""), + EnvString("XDG_CONFIG_DIRS", ""), + EnvString("XDG_RUNTIME_DIR", + ("/run/user/" + std::to_string(getuid())).c_str())) + : VrRuntimeDiscovery{{}, "explicit XR_RUNTIME_JSON override"}; + runtime_manifest_ = + SelectVrRuntimeManifest(user_manifest_, discovery.candidates); + Log(" [vr-backend] runtime selection: %s\n", discovery.reason.c_str()); + Require(!runtime_manifest_.empty() || + (preference != "steamvr" && preference != "alvr" && + preference != "wivrn"), + "Selected VR runtime was not found. For ALVR, install SteamVR and " + "set XR_RUNTIME_JSON " + "to its steamxr_linux64.json if automatic discovery cannot find " + "the installation. " + "For WiVRn, install its native runtime or select its manifest " + "explicitly."); + if (!runtime_manifest_.empty()) { + Require(setenv("XR_RUNTIME_JSON", runtime_manifest_.c_str(), 1) == 0, + "Cannot select the OpenXR runtime"); + Log(" [vr-backend] runtime manifest: %s\n", runtime_manifest_.c_str()); + } else { + Log(" [vr-backend] using registered OpenXR runtime; connect the headset " + "in WiVRn or SteamVR/ALVR first\n"); + } + graphics_api_ = graphics_api; + if (graphics_api_ == VrGraphicsApi::kVulkan) { + vk_loader_ = dlopen("libvulkan.so.1", RTLD_NOW | RTLD_LOCAL); + Require(vk_loader_ != nullptr, + "Cannot load the host Vulkan loader libvulkan.so.1"); + auto loader_gipa = reinterpret_cast( + dlsym(vk_loader_, "vkGetInstanceProcAddr")); + Require(loader_gipa != nullptr, "Host vkGetInstanceProcAddr is missing"); + } + std::string err; + if (!EnsureXrInstanceLocked(&err)) { + throw std::runtime_error(err); + } armed_.store(true, std::memory_order_release); Log(" [vr-backend] armed: OpenXR instance/system ready; awaiting guest " "%s\n", @@ -453,25 +487,19 @@ void OpenXrBackend::Disarm() { void OpenXrBackend::DisarmLocked() { TeardownSessionLocked("disarm"); - { std::lock_guard lock(controller_mutex_); - actions_.Destroy(); - actions_created_ = false; } + DestroyXrInstanceLocked(); if (gles_) gles_->Destroy(); delete gles_; gles_ = nullptr; - if (xr_instance_ != nullptr) { - const auto instance = static_cast(xr_instance_); - (void)xrDestroyInstance(instance); - xr_instance_ = nullptr; - } - xr_system_ = 0; runtime_name_.clear(); vk_instance_ = VK_NULL_HANDLE; if (vk_loader_ != nullptr) { dlclose(vk_loader_); vk_loader_ = nullptr; } + user_manifest_.clear(); + runtime_manifest_.clear(); armed_.store(false, std::memory_order_release); recording_.store(false, std::memory_order_release); eyes_bound_.store(false, std::memory_order_release); @@ -506,11 +534,24 @@ bool OpenXrBackend::CreateVulkanInstance( const VkAllocationCallbacks* allocator, VkInstance* instance, VkResult* result) { std::lock_guard lock(mutex_); - if (!armed_.load(std::memory_order_acquire) || xr_instance_ == nullptr || - vk_loader_ == nullptr) { + if (!armed_.load(std::memory_order_acquire) || vk_loader_ == nullptr) { return false; } try { + if (xr_instance_used_ || vk_instance_ != VK_NULL_HANDLE || + xr_session_ != nullptr) { + TeardownSessionLocked("recycling OpenXR instance for new Vulkan instance"); + DestroyXrInstanceLocked(); + vk_instance_ = VK_NULL_HANDLE; + } + if (xr_instance_ == nullptr) { + std::string err; + if (!EnsureXrInstanceLocked(&err)) { + Log(" [vr-backend] failed to recreate OpenXR instance: %s\n", + err.c_str()); + return false; + } + } auto loader_gipa = reinterpret_cast( dlsym(vk_loader_, "vkGetInstanceProcAddr")); Require(loader_gipa != nullptr, "Host vkGetInstanceProcAddr is missing"); @@ -533,6 +574,7 @@ bool OpenXrBackend::CreateVulkanInstance( return true; } vk_instance_ = created; + xr_instance_used_ = true; *instance = created; *result = VK_SUCCESS; Log(" [vr-backend] guest VkInstance created through the OpenXR runtime\n"); @@ -1222,6 +1264,7 @@ void OpenXrBackend::NoteInstanceDestroyed(VkInstance instance) { std::lock_guard lock(mutex_); if (instance == vk_instance_) { TeardownSessionLocked("guest VkInstance destroyed"); + DestroyXrInstanceLocked(); vk_instance_ = VK_NULL_HANDLE; } } From 68f8ec82d2c5e274c78ef2b3bb6fb44dd5b1dac2 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Mon, 12 Oct 2026 20:25:21 -0500 Subject: [PATCH 17/58] vr: fix sRGB double-gamma, home world avatar spawn, and controller aim alignment --- src/runtime/performance_policy.cc | 3 + src/vr/openxr_backend.cc | 19 ++- src/vr/roblox_vr_device_bridge.cc | 145 ++++++++++++++++++++++- tests/performance_policy_test.cc | 1 + tests/vr/roblox_vr_device_bridge_test.cc | 39 ++++++ 5 files changed, 198 insertions(+), 9 deletions(-) diff --git a/src/runtime/performance_policy.cc b/src/runtime/performance_policy.cc index a68ce12a..db572812 100644 --- a/src/runtime/performance_policy.cc +++ b/src/runtime/performance_policy.cc @@ -402,6 +402,9 @@ bool MergeVrClientSettingsOverrides(bool enabled, std::string_view base_json, // The eye textures already use the OpenXR extent; render their contents // at full resolution as well, without Roblox's dynamic downscaling. overrides["FFlagDebugDisableDRS"] = "True"; + // Place the Avatar Editor character 6 studs in front of the VR camera + // instead of at the origin when IsVRAppBuild() is active. + overrides["FFlagFixVRAvatarPlacement"] = "True"; } *merged_json = overrides.dump(); return true; diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 8ca9153b..58c830b1 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -145,7 +145,16 @@ bool IsColorFormat(VkFormat format) { } bool FormatIsCopyCompatible(VkFormat a, VkFormat b) { - return a == b; + if (a == b) return true; + if ((a == VK_FORMAT_R8G8B8A8_UNORM && b == VK_FORMAT_R8G8B8A8_SRGB) || + (a == VK_FORMAT_R8G8B8A8_SRGB && b == VK_FORMAT_R8G8B8A8_UNORM)) { + return true; + } + if ((a == VK_FORMAT_B8G8R8A8_UNORM && b == VK_FORMAT_B8G8R8A8_SRGB) || + (a == VK_FORMAT_B8G8R8A8_SRGB && b == VK_FORMAT_B8G8R8A8_UNORM)) { + return true; + } + return false; } OpenXrBackend* g_backend = nullptr; @@ -1075,10 +1084,10 @@ bool OpenXrBackend::CreateSwapchains(std::string* error) { std::int64_t chosen = 0; const std::vector candidates = graphics_api_ == VrGraphicsApi::kVulkan - ? std::vector{VK_FORMAT_B8G8R8A8_UNORM, - VK_FORMAT_R8G8B8A8_UNORM, - VK_FORMAT_B8G8R8A8_SRGB, - VK_FORMAT_R8G8B8A8_SRGB} + ? std::vector{VK_FORMAT_R8G8B8A8_UNORM, + VK_FORMAT_B8G8R8A8_UNORM, + VK_FORMAT_R8G8B8A8_SRGB, + VK_FORMAT_B8G8R8A8_SRGB} : std::vector{GL_RGBA8, GL_SRGB8_ALPHA8}; for (const auto candidate : candidates) { if (std::find(formats.begin(), formats.end(), diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc index 0dfb2db1..8b1db3de 100644 --- a/src/vr/roblox_vr_device_bridge.cc +++ b/src/vr/roblox_vr_device_bridge.cc @@ -284,6 +284,95 @@ void Log(const char *format, ...) { std::fflush(stderr); } +std::uintptr_t* g_is_vr_app_build_slot = nullptr; +std::uintptr_t g_is_vr_app_build_original = 0; + +extern "C" bool mocktail_vr_is_vr_app_build(const void*) { + auto* bridge = g_active_bridge.load(std::memory_order_acquire); + return bridge != nullptr && bridge->active(); +} + +void InstallIsVrAppBuildHook(std::uintptr_t image_base) { + if (g_is_vr_app_build_slot != nullptr || !IsValidElfImage(image_base)) { + return; + } + const std::uintptr_t image_size = ImageSize(image_base); + const auto* header = reinterpret_cast(image_base); + const auto* program_headers = reinterpret_cast( + image_base + static_cast(header->e_phoff)); + static constexpr std::uint8_t kLeaRdxRip[3] = {0x48, 0x8d, 0x15}; + static constexpr char kMethodName[] = "IsVRAppBuild"; + for (std::size_t index = 0; index < header->e_phnum; ++index) { + const Elf64_Phdr& segment = program_headers[index]; + if (segment.p_type != PT_LOAD || + (segment.p_flags & (PF_R | PF_X)) != (PF_R | PF_X) || + segment.p_memsz < 21) { + continue; + } + const auto* seg_begin = + reinterpret_cast(image_base + segment.p_vaddr); + const std::size_t seg_size = static_cast(segment.p_memsz); + std::size_t offset = 14; + while (offset + 7 <= seg_size) { + const void* match = + memmem(seg_begin + offset, seg_size - offset - 4, kLeaRdxRip, + sizeof(kLeaRdxRip)); + if (match == nullptr) { + break; + } + const std::size_t idx = + static_cast(match) - seg_begin; + offset = idx + 1; + if (idx < 14 || seg_begin[idx - 7] != 0x48 || + seg_begin[idx - 6] != 0x8d || seg_begin[idx - 5] != 0x35 || + seg_begin[idx - 14] != 0x48 || seg_begin[idx - 13] != 0x8d || + seg_begin[idx - 12] != 0x3d) { + continue; + } + const std::uintptr_t rdx_rva = segment.p_vaddr + idx; + const std::uintptr_t str_rva = + RelativeTarget(seg_begin + idx + 3, rdx_rva + 7); + if (str_rva >= image_size || image_size - str_rva < sizeof(kMethodName) || + std::memcmp(reinterpret_cast(image_base + str_rva), + kMethodName, sizeof(kMethodName)) != 0) { + continue; + } + const std::uintptr_t desc_rva = + RelativeTarget(seg_begin + idx - 11, rdx_rva - 7); + const std::uintptr_t fn_rva = + RelativeTarget(seg_begin + idx - 4, rdx_rva); + if (!IsWritableDataImageRange(image_base, desc_rva, 0x98) || + !IsExecutableImageRange(image_base, fn_rva, 3)) { + continue; + } + const std::uintptr_t expected_fn = image_base + fn_rva; + for (std::size_t slot_off : {std::size_t{0x80}, std::size_t{0x88}}) { + auto* slot = + reinterpret_cast(image_base + desc_rva + slot_off); + if (*slot == expected_fn) { + g_is_vr_app_build_original = expected_fn; + g_is_vr_app_build_slot = slot; + __atomic_store_n(slot, FunctionAddress(&mocktail_vr_is_vr_app_build), + __ATOMIC_RELEASE); + Log(" [vr-device] interposed VRService::IsVRAppBuild descriptor " + "at +0x%" PRIxPTR "\n", + desc_rva + slot_off); + return; + } + } + } + } +} + +void RestoreIsVrAppBuildHook() { + if (g_is_vr_app_build_slot != nullptr) { + __atomic_store_n(g_is_vr_app_build_slot, g_is_vr_app_build_original, + __ATOMIC_RELEASE); + g_is_vr_app_build_slot = nullptr; + g_is_vr_app_build_original = 0; + } +} + } // namespace namespace internal { @@ -590,8 +679,22 @@ void RobloxVrDeviceBridge::OnPointerFrameCall(void* result, void* service) { std::memcpy(°rees, reinterpret_cast(base + contract_->hand_pitch_storage_rva), 4); - if (internal::ApplyAimToWorldFrame(static_cast(result), hand, scale, - -static_cast(degrees))) { + VrHandPose pointer_hand = hand; + pose::Quat aim{}; + if (pose::QuatNormalize({hand.aim_orientation[0], hand.aim_orientation[1], + hand.aim_orientation[2], hand.aim_orientation[3]}, + &aim)) { + constexpr float kNativeHandPitchHalfAngle = 20.f * pose::kPi / 360.f; + const pose::Quat pitched = pose::QuatMultiply( + aim, {std::sin(kNativeHandPitchHalfAngle), 0.f, 0.f, + std::cos(kNativeHandPitchHalfAngle)}); + pointer_hand.grip_orientation[0] = pitched.x; + pointer_hand.grip_orientation[1] = pitched.y; + pointer_hand.grip_orientation[2] = pitched.z; + pointer_hand.grip_orientation[3] = pitched.w; + } + if (internal::ApplyAimToWorldFrame(static_cast(result), pointer_hand, + scale, -static_cast(degrees))) { static std::atomic logged[2]{}; if (!logged[index - 1].exchange(true)) std::fprintf( @@ -796,6 +899,7 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { (void)RestoreVtableLocked(); return FailedPrecondition("could not install verified VR aim pointer hook"); } + InstallIsVrAppBuildHook(image_base); { std::lock_guard pose_lock(g_native_pose_mutex); g_native_pose = {}; @@ -836,6 +940,7 @@ void RobloxVrDeviceBridge::Shutdown() { } bool RobloxVrDeviceBridge::RestoreVtableLocked() { + RestoreIsVrAppBuildHook(); if (!RestorePointerHook()) return false; if (vtable_ == nullptr) { return false; @@ -1058,12 +1163,44 @@ bool internal::ApplyXrPose(void *state, const ScriptedPoseSample &pose) { } if (!finite || hand_norm < 0.99f || hand_norm > 1.01f) continue; + float orientation[4] = { + source.grip_orientation[0], source.grip_orientation[1], + source.grip_orientation[2], source.grip_orientation[3]}; + if (source.aim_valid) { + float aim_norm = 0.f; + bool aim_finite = true; + for (float v : source.aim_orientation) { + aim_finite = aim_finite && std::isfinite(v); + aim_norm += v * v; + } + pose::Quat aim{}; + if (aim_finite && aim_norm >= 0.99f && aim_norm <= 1.01f && + pose::QuatNormalize( + {source.aim_orientation[0], source.aim_orientation[1], + source.aim_orientation[2], source.aim_orientation[3]}, + &aim)) { + // The guest's VRService consumer (0x28a6242) pitches each hand record + // by -NewVRSystemHandPitch (-20 deg) to form UserCFrame[hand], which + // Luau's LaserPointer uses directly as lookVector and TouchController + // pitches back by +20 deg (0.34906585 rad). Pre-applying +20 deg to + // the OpenXR aim orientation makes UserCFrame[hand] match the OpenXR + // aim/handmodel orientation while keeping the grip palm position. + constexpr float kNativeHandPitchHalfAngle = 20.f * pose::kPi / 360.f; + const pose::Quat pitched = pose::QuatMultiply( + aim, {std::sin(kNativeHandPitchHalfAngle), 0.f, 0.f, + std::cos(kNativeHandPitchHalfAngle)}); + orientation[0] = pitched.x; + orientation[1] = pitched.y; + orientation[2] = pitched.z; + orientation[3] = pitched.w; + } + } const std::size_t record = hand == 0 ? kLeftHandRecordOffset : kRightHandRecordOffset; std::memcpy(bytes + record + kRecordPositionOffset, source.grip_position, sizeof(source.grip_position)); - std::memcpy(bytes + record + kRecordOrientationOffset, - source.grip_orientation, sizeof(source.grip_orientation)); + std::memcpy(bytes + record + kRecordOrientationOffset, orientation, + sizeof(orientation)); bytes[record + kRecordValidOffset] = 1; } for (int eye = 0; eye < 2; ++eye) { diff --git a/tests/performance_policy_test.cc b/tests/performance_policy_test.cc index eb9d1e5e..d3dd3dc8 100644 --- a/tests/performance_policy_test.cc +++ b/tests/performance_policy_test.cc @@ -356,6 +356,7 @@ TEST(PerformancePolicyTest, auto json = nlohmann::json::parse(merged); EXPECT_EQ(json["FFlagDebugEnableVREmulator"], "True"); EXPECT_EQ(json["FFlagSlowDownRendering"], "False"); + EXPECT_EQ(json["FFlagFixVRAvatarPlacement"], "True"); EXPECT_EQ(json["FIntUnrelated"], "42"); ASSERT_TRUE(MergeVrClientSettingsOverrides(false, merged, &merged, &error)); EXPECT_EQ(nlohmann::json::parse(merged)["FFlagDebugEnableVREmulator"], diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc index 3f323181..87f9ee12 100644 --- a/tests/vr/roblox_vr_device_bridge_test.cc +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -697,6 +697,45 @@ TEST(VrHandPoseBridge, ValidGripPosesFillBothHandRecordsInMetres) { EXPECT_FLOAT_EQ(right_pos[0], 0.25f); } +TEST(VrHandPoseBridge, ValidAimPosePreAppliesHandPitchWhileKeepingGripOrigin) { + std::array state{}; + ScriptedPoseSample pose; + pose.valid = true; + pose.orientation[3] = 1.f; + pose.hands[1].grip_valid = true; + pose.hands[1].grip_position[0] = 0.25f; + pose.hands[1].grip_position[1] = 1.1f; + pose.hands[1].grip_position[2] = -0.4f; + pose.hands[1].grip_orientation[0] = std::sin(0.18f); + pose.hands[1].grip_orientation[3] = std::cos(0.18f); + pose.hands[1].aim_valid = true; + pose.hands[1].aim_position[2] = -0.5f; + pose.hands[1].aim_orientation[0] = std::sin(-0.34f); + pose.hands[1].aim_orientation[3] = std::cos(-0.34f); + ASSERT_TRUE(internal::ApplyXrPose(state.data(), pose)); + + float right_pos[3], right_ori[4]; + std::memcpy(right_pos, state.data() + internal::kRightHandRecordOffset + + internal::kRecordPositionOffset, 12); + std::memcpy(right_ori, state.data() + internal::kRightHandRecordOffset + + internal::kRecordOrientationOffset, 16); + EXPECT_FLOAT_EQ(right_pos[0], 0.25f); + EXPECT_FLOAT_EQ(right_pos[1], 1.1f); + EXPECT_FLOAT_EQ(right_pos[2], -0.4f); + + // Applying Roblox's native -20 deg pitch offset to the stored record + // orientation must recover the exact OpenXR aim orientation. + constexpr float kMinus20HalfAngle = -20.f * pose::kPi / 360.f; + const pose::Quat stored{right_ori[0], right_ori[1], right_ori[2], right_ori[3]}; + const pose::Quat after_guest_pitch = pose::QuatMultiply( + stored, {std::sin(kMinus20HalfAngle), 0.f, 0.f, + std::cos(kMinus20HalfAngle)}); + EXPECT_NEAR(after_guest_pitch.x, pose.hands[1].aim_orientation[0], 1e-5f); + EXPECT_NEAR(after_guest_pitch.y, 0.f, 1e-5f); + EXPECT_NEAR(after_guest_pitch.z, 0.f, 1e-5f); + EXPECT_NEAR(after_guest_pitch.w, pose.hands[1].aim_orientation[3], 1e-5f); +} + TEST(VrFloorPoseBridge, PublishesStagePoseInMetresAndClearsTrackingLoss) { std::array state{}; state[internal::kStateReadyOffsetInCopy] = 1; From 88bb4c940c6e711f762439fa248fffd0cfb0939c Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Mon, 12 Oct 2026 21:28:20 -0500 Subject: [PATCH 18/58] vr: align pointer ray pitch with hand orientation and reset world scale to 1.0 --- src/vr/roblox_vr_device_bridge.cc | 208 ++++++++++++++++------ src/vr/roblox_vr_device_bridge_internal.h | 5 + tests/vr/roblox_vr_device_bridge_test.cc | 20 +-- 3 files changed, 171 insertions(+), 62 deletions(-) diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc index 8b1db3de..9fd05b43 100644 --- a/src/vr/roblox_vr_device_bridge.cc +++ b/src/vr/roblox_vr_device_bridge.cc @@ -638,6 +638,95 @@ const compat::VrDebugDeviceBridgeProfile* FindRobloxVrDeviceProfile( return contract == nullptr ? nullptr : &contract->device; } +namespace internal { + +float GetHandPitchOffsetDeg() { + static const float offset = [] { + if (const char* env = std::getenv("MOCKTAIL_VR_HAND_PITCH_OFFSET")) { + char* end = nullptr; + const float val = std::strtof(env, &end); + if (end != env && std::isfinite(val)) { + Log(" [vr-device] hand pitch offset (MOCKTAIL_VR_HAND_PITCH_OFFSET): %.1f deg\n", val); + return val; + } + } + // Default -20 deg matches user preference for controller/laser pitch. + return -20.f; + }(); + return offset; +} + +float GetTotalHandPitchDeg() { + static const float total = [] { + if (const char* env = std::getenv("MOCKTAIL_VR_HAND_PITCH")) { + char* end = nullptr; + const float val = std::strtof(env, &end); + if (end != env && std::isfinite(val)) { + Log(" [vr-device] hand pitch override (MOCKTAIL_VR_HAND_PITCH): %.1f deg\n", val); + return val; + } + } + return 20.f + GetHandPitchOffsetDeg(); + }(); + return total; +} + +pose::Quat ComputeHandRecordOrientation(const VrHandPose& source) { + const float total_pitch_deg = GetTotalHandPitchDeg(); + + static const bool use_grip = [] { + if (const char* env = std::getenv("MOCKTAIL_VR_HAND_POSE_SOURCE")) { + if (std::strcmp(env, "grip") == 0) { + Log(" [vr-device] hand pose source override: grip\n"); + return true; + } + } + return false; + }(); + + if (!source.aim_valid && !use_grip) { + pose::Quat grip{}; + if (pose::QuatNormalize( + {source.grip_orientation[0], source.grip_orientation[1], + source.grip_orientation[2], source.grip_orientation[3]}, + &grip)) { + return grip; + } + return {source.grip_orientation[0], source.grip_orientation[1], + source.grip_orientation[2], source.grip_orientation[3]}; + } + + pose::Quat base_ori{}; + if (!use_grip && source.aim_valid) { + float aim_norm = 0.f; + bool aim_finite = true; + for (float v : source.aim_orientation) { + aim_finite = aim_finite && std::isfinite(v); + aim_norm += v * v; + } + if (aim_finite && aim_norm >= 0.99f && aim_norm <= 1.01f) { + (void)pose::QuatNormalize( + {source.aim_orientation[0], source.aim_orientation[1], + source.aim_orientation[2], source.aim_orientation[3]}, + &base_ori); + } + } + if (!std::isfinite(base_ori.w) || QuatNorm(base_ori) < 0.5f) { + if (!pose::QuatNormalize( + {source.grip_orientation[0], source.grip_orientation[1], + source.grip_orientation[2], source.grip_orientation[3]}, + &base_ori)) { + base_ori = {0.f, 0.f, 0.f, 1.f}; + } + } + + const float half_angle = total_pitch_deg * pose::kPi / 360.f; + return pose::QuatMultiply( + base_ori, {std::sin(half_angle), 0.f, 0.f, std::cos(half_angle)}); +} + +} // namespace internal + RobloxVrDeviceBridge::~RobloxVrDeviceBridge() { Shutdown(); RobloxVrDeviceBridge *expected = this; @@ -679,20 +768,40 @@ void RobloxVrDeviceBridge::OnPointerFrameCall(void* result, void* service) { std::memcpy(°rees, reinterpret_cast(base + contract_->hand_pitch_storage_rva), 4); - VrHandPose pointer_hand = hand; - pose::Quat aim{}; - if (pose::QuatNormalize({hand.aim_orientation[0], hand.aim_orientation[1], - hand.aim_orientation[2], hand.aim_orientation[3]}, - &aim)) { - constexpr float kNativeHandPitchHalfAngle = 20.f * pose::kPi / 360.f; - const pose::Quat pitched = pose::QuatMultiply( - aim, {std::sin(kNativeHandPitchHalfAngle), 0.f, 0.f, - std::cos(kNativeHandPitchHalfAngle)}); - pointer_hand.grip_orientation[0] = pitched.x; - pointer_hand.grip_orientation[1] = pitched.y; - pointer_hand.grip_orientation[2] = pitched.z; - pointer_hand.grip_orientation[3] = pitched.w; + { + static std::atomic last_scale{-1.f}; + const float previous = last_scale.exchange(scale); + if (std::abs(previous - scale) > 1e-4f) + std::fprintf(stderr, + " [vr-device][evidence] camera HeadScale=%.4f hand_pitch=%d\n", + scale, degrees); } + VrHandPose pointer_hand = hand; + const pose::Quat pitched = internal::ComputeHandRecordOrientation(hand); + pointer_hand.grip_orientation[0] = pitched.x; + pointer_hand.grip_orientation[1] = pitched.y; + pointer_hand.grip_orientation[2] = pitched.z; + pointer_hand.grip_orientation[3] = pitched.w; + // Apply the same hand pitch offset to aim orientation so that the native + // pointer ray rotates in lockstep with the controller and laser beam. + const float pitch_offset_rad = + internal::GetHandPitchOffsetDeg() * pose::kPi / 180.f; + const float half_pitch = pitch_offset_rad * 0.5f; + const pose::Quat aim_rot = pose::QuatMultiply( + {pointer_hand.aim_orientation[0], pointer_hand.aim_orientation[1], + pointer_hand.aim_orientation[2], pointer_hand.aim_orientation[3]}, + {std::sin(half_pitch), 0.f, 0.f, std::cos(half_pitch)}); + pointer_hand.aim_orientation[0] = aim_rot.x; + pointer_hand.aim_orientation[1] = aim_rot.y; + pointer_hand.aim_orientation[2] = aim_rot.z; + pointer_hand.aim_orientation[3] = aim_rot.w; + // Keep ray origin identical to UserCFrame[hand].p (grip origin), so the + // C++ pointer ray and Luau's LaserPointer share the exact same origin + // with zero vertical or lateral offset. + pointer_hand.aim_position[0] = pointer_hand.grip_position[0]; + pointer_hand.aim_position[1] = pointer_hand.grip_position[1]; + pointer_hand.aim_position[2] = pointer_hand.grip_position[2]; + if (internal::ApplyAimToWorldFrame(static_cast(result), pointer_hand, scale, -static_cast(degrees))) { static std::atomic logged[2]{}; @@ -1125,8 +1234,28 @@ bool internal::ApplyXrPose(void *state, const ScriptedPoseSample &pose) { } bytes[0x132] = pose.valid && pose.controllers_connected; + // World scale factor: scales spatial tracking metrics (head, hands, floor, + // eye separation). Defaults to 1.0f (raw metres, unscaled); configurable + // via MOCKTAIL_VR_WORLD_SCALE or MOCKTAIL_VR_EYE_OFFSET_SCALE. + static const float world_scale = [] { + for (const char* var : {"MOCKTAIL_VR_WORLD_SCALE", "MOCKTAIL_VR_EYE_OFFSET_SCALE"}) { + if (const char *env = std::getenv(var)) { + char *end = nullptr; + const float value = std::strtof(env, &end); + if (end != env && std::isfinite(value) && value > 0.f && value < 100.f) { + Log(" [vr-device] world scale override (%s): %.4f\n", var, value); + return value; + } + } + } + return 1.0f; + }(); + // The verified guest pose helper applies metres-to-studs conversion. - std::memcpy(bytes + 4, pose.position, sizeof(pose.position)); + const float head_pos[3] = {pose.position[0] * world_scale, + pose.position[1] * world_scale, + pose.position[2] * world_scale}; + std::memcpy(bytes + 4, head_pos, sizeof(head_pos)); std::memcpy(bytes + 0x10, pose.orientation, sizeof(pose.orientation)); if (pose.floor_valid) { bool finite = true; @@ -1137,8 +1266,11 @@ bool internal::ApplyXrPose(void *state, const ScriptedPoseSample &pose) { floor_norm += value * value; } if (finite && floor_norm >= .99f && floor_norm <= 1.01f) { + const float floor_pos[3] = {pose.floor_position[0] * world_scale, + pose.floor_position[1] * world_scale, + pose.floor_position[2] * world_scale}; std::memcpy(bytes + kExtraRecordOffset + kRecordPositionOffset, - pose.floor_position, sizeof(pose.floor_position)); + floor_pos, sizeof(floor_pos)); std::memcpy(bytes + kExtraRecordOffset + kRecordOrientationOffset, pose.floor_orientation, sizeof(pose.floor_orientation)); bytes[kExtraRecordOffset + kRecordValidOffset] = 1; @@ -1163,48 +1295,24 @@ bool internal::ApplyXrPose(void *state, const ScriptedPoseSample &pose) { } if (!finite || hand_norm < 0.99f || hand_norm > 1.01f) continue; - float orientation[4] = { - source.grip_orientation[0], source.grip_orientation[1], - source.grip_orientation[2], source.grip_orientation[3]}; - if (source.aim_valid) { - float aim_norm = 0.f; - bool aim_finite = true; - for (float v : source.aim_orientation) { - aim_finite = aim_finite && std::isfinite(v); - aim_norm += v * v; - } - pose::Quat aim{}; - if (aim_finite && aim_norm >= 0.99f && aim_norm <= 1.01f && - pose::QuatNormalize( - {source.aim_orientation[0], source.aim_orientation[1], - source.aim_orientation[2], source.aim_orientation[3]}, - &aim)) { - // The guest's VRService consumer (0x28a6242) pitches each hand record - // by -NewVRSystemHandPitch (-20 deg) to form UserCFrame[hand], which - // Luau's LaserPointer uses directly as lookVector and TouchController - // pitches back by +20 deg (0.34906585 rad). Pre-applying +20 deg to - // the OpenXR aim orientation makes UserCFrame[hand] match the OpenXR - // aim/handmodel orientation while keeping the grip palm position. - constexpr float kNativeHandPitchHalfAngle = 20.f * pose::kPi / 360.f; - const pose::Quat pitched = pose::QuatMultiply( - aim, {std::sin(kNativeHandPitchHalfAngle), 0.f, 0.f, - std::cos(kNativeHandPitchHalfAngle)}); - orientation[0] = pitched.x; - orientation[1] = pitched.y; - orientation[2] = pitched.z; - orientation[3] = pitched.w; - } - } + const pose::Quat pitched = ComputeHandRecordOrientation(source); + const float orientation[4] = {pitched.x, pitched.y, pitched.z, pitched.w}; const std::size_t record = hand == 0 ? kLeftHandRecordOffset : kRightHandRecordOffset; - std::memcpy(bytes + record + kRecordPositionOffset, source.grip_position, - sizeof(source.grip_position)); + const float grip_pos[3] = {source.grip_position[0] * world_scale, + source.grip_position[1] * world_scale, + source.grip_position[2] * world_scale}; + std::memcpy(bytes + record + kRecordPositionOffset, grip_pos, + sizeof(grip_pos)); std::memcpy(bytes + record + kRecordOrientationOffset, orientation, sizeof(orientation)); bytes[record + kRecordValidOffset] = 1; } for (int eye = 0; eye < 2; ++eye) { - std::memcpy(bytes + 0xf8 + eye * 12, pose.eye_offset[eye], 12); + const float offset[3] = {pose.eye_offset[eye][0] * world_scale, + pose.eye_offset[eye][1] * world_scale, + pose.eye_offset[eye][2] * world_scale}; + std::memcpy(bytes + 0xf8 + eye * 12, offset, sizeof(offset)); const auto *fov = pose.eye_fov[eye]; const float tangents[4] = {std::tan(fov[2]), -std::tan(fov[3]), -std::tan(fov[0]), std::tan(fov[1])}; diff --git a/src/vr/roblox_vr_device_bridge_internal.h b/src/vr/roblox_vr_device_bridge_internal.h index 9e98e1cf..f1e43868 100644 --- a/src/vr/roblox_vr_device_bridge_internal.h +++ b/src/vr/roblox_vr_device_bridge_internal.h @@ -24,6 +24,11 @@ struct VrBuildContract { const VrBuildContract* FindVrBuildContract(std::string_view build_id); +// Computes the record orientation for a hand pose, incorporating native +// hand pitch cancellation and configurable pitch offsets. +pose::Quat ComputeHandRecordOrientation(const VrHandPose& source); +float GetHandPitchOffsetDeg(); + // Writes only a caller-owned state copy; preserves all resource readiness. bool ApplyXrPose(void* state, const ScriptedPoseSample& pose); // Adjust a world-space grip CoordinateFrame to OpenXR aim without changing diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc index 87f9ee12..b354f171 100644 --- a/tests/vr/roblox_vr_device_bridge_test.cc +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -634,7 +634,7 @@ TEST(VrPoseBridge, ConvertsRuntimePoseAndAsymmetricProjectionWithoutChangingRead std::memcpy(left, state.data()+0xf8, 12); std::memcpy(right, state.data()+0x104, 12); std::memcpy(projection, state.data()+0x110, 16); - EXPECT_FLOAT_EQ(position[0], 0.4f); // no double metre-to-stud conversion + EXPECT_FLOAT_EQ(position[0], 0.4f); EXPECT_FLOAT_EQ(position[1], 1.6f); EXPECT_FLOAT_EQ(orientation[1], std::sin(0.2f)); EXPECT_FLOAT_EQ(left[0], -0.032f); @@ -689,7 +689,7 @@ TEST(VrHandPoseBridge, ValidGripPosesFillBothHandRecordsInMetres) { internal::kRecordOrientationOffset, 16); std::memcpy(right_pos, state.data() + internal::kRightHandRecordOffset + internal::kRecordPositionOffset, 12); - // Metres only: the guest's 0x37eacb8 applies the 10/3 stud conversion. + // Unscaled (1.0f): guest's 0x37eacb8 applies 10/3 stud conversion. EXPECT_FLOAT_EQ(left_pos[0], -0.25f); EXPECT_FLOAT_EQ(left_pos[1], 1.0f); EXPECT_FLOAT_EQ(left_ori[1], std::sin(0.3f)); @@ -723,17 +723,13 @@ TEST(VrHandPoseBridge, ValidAimPosePreAppliesHandPitchWhileKeepingGripOrigin) { EXPECT_FLOAT_EQ(right_pos[1], 1.1f); EXPECT_FLOAT_EQ(right_pos[2], -0.4f); - // Applying Roblox's native -20 deg pitch offset to the stored record - // orientation must recover the exact OpenXR aim orientation. - constexpr float kMinus20HalfAngle = -20.f * pose::kPi / 360.f; + // Stored orientation contains total_pitch = 20 + offset (where offset defaults to -20, so total_pitch = 0). + // Applying guest's native -20 deg pitch offset rotates by -20 deg relative to aim orientation. const pose::Quat stored{right_ori[0], right_ori[1], right_ori[2], right_ori[3]}; - const pose::Quat after_guest_pitch = pose::QuatMultiply( - stored, {std::sin(kMinus20HalfAngle), 0.f, 0.f, - std::cos(kMinus20HalfAngle)}); - EXPECT_NEAR(after_guest_pitch.x, pose.hands[1].aim_orientation[0], 1e-5f); - EXPECT_NEAR(after_guest_pitch.y, 0.f, 1e-5f); - EXPECT_NEAR(after_guest_pitch.z, 0.f, 1e-5f); - EXPECT_NEAR(after_guest_pitch.w, pose.hands[1].aim_orientation[3], 1e-5f); + EXPECT_NEAR(stored.x, pose.hands[1].aim_orientation[0], 1e-5f); + EXPECT_NEAR(stored.y, 0.f, 1e-5f); + EXPECT_NEAR(stored.z, 0.f, 1e-5f); + EXPECT_NEAR(stored.w, pose.hands[1].aim_orientation[3], 1e-5f); } TEST(VrFloorPoseBridge, PublishesStagePoseInMetresAndClearsTrackingLoss) { From ddc985218b7b0cbbb1be2b9dbc76d5fb497276cd Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Mon, 12 Oct 2026 22:18:32 -0500 Subject: [PATCH 19/58] vr: add separate VR frame-rate limit, render scale and unpaced benchmark mode vr.frame_rate_limit / MOCKTAIL_VR_FRAME_RATE_LIMIT replaces graphics.frame_rate_limit while VR is active and takes the same values. It defaults to unlimited so xrWaitFrame paces Roblox at the headset refresh rate instead of Roblox's own 60 FPS cap. MOCKTAIL_VR_RENDER_SCALE scales the runtime's recommended eye extent. SteamVR's recommendation for a Quest 2 at 90 Hz (2244x2352 per eye) missed the 11.1 ms budget and halved the rate to 45 FPS. MOCKTAIL_VR_UNPACED=1 stops guest frames from blocking in xrWaitFrame: the open XR frame spans several guest frames and is submitted once per display period, for measuring maximum render rate. --- README.md | 2 ++ config/mocktail.example.yaml | 6 ++++ docs/PERFORMANCE.md | 25 ++++++++++++++ include/mocktail/vr/openxr_backend.h | 3 ++ include/runtime/runtime_config.h | 7 ++++ src/graphics/present_mode_policy.cc | 7 +++- src/legacy/legacy_runtime.cc | 5 +++ src/main.cc | 11 ++++-- src/runtime/graphics_launch_policy.cc | 2 +- src/runtime/runtime_config.cc | 2 ++ src/runtime/runtime_config_bootstrap.cc | 6 ++++ src/runtime/runtime_config_file.cc | 13 +++++++ src/vr/openxr_backend.cc | 45 +++++++++++++++++++++++++ tests/runtime_config_file_test.cc | 37 ++++++++++++++++++++ 14 files changed, 166 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c4fcbe63..924533f2 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,8 @@ Edit `~/.config/mocktail/config.yaml`. Useful options: - `graphics.frame_rate_limit: display` to unlock FPS to your monitor's rate. - `graphics.vsync: off` for the lowest lag, at the cost of tearing. +- `vr.frame_rate_limit` sets the cap in VR. It defaults to `unlimited`, so the + headset's refresh rate paces frames; use `-1` for Roblox's 60 FPS cap. - `graphics.backend: opengl` if Vulkan does not work on your GPU. ## Building diff --git a/config/mocktail.example.yaml b/config/mocktail.example.yaml index e2c81210..332eb1d6 100644 --- a/config/mocktail.example.yaml +++ b/config/mocktail.example.yaml @@ -62,6 +62,12 @@ graphics: # desktop testing). Controllers use the native VR bridge. Standalone OpenXR # test scene: mocktail-vr-probe --scene. Use --no-vr to launch normally. # enabled: false + # Optional string or integer: frame-rate limit used instead of + # graphics.frame_rate_limit while VR is active. Same values: -1 (Roblox's + # own cap, usually 60), display, unlimited, or any positive integer. The + # default, unlimited, lets the OpenXR runtime pace frames at the headset's + # refresh rate. + # frame_rate_limit: unlimited performance: # Boolean (default: false): size Roblox scheduler/render queues from every diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index fb7656c5..b74385d4 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -157,6 +157,31 @@ MOCKTAIL_SMALL_TEXTURE_UPSCALE=1 mocktail framerate cap in charge; Nightcap sets no `DFIntTaskSchedulerTargetFps` override. A fixed value forwards that number to the scheduler. +In VR, `vr.frame_rate_limit` (`MOCKTAIL_VR_FRAME_RATE_LIMIT`) replaces +`graphics.frame_rate_limit` and takes the same values. Its default is +`unlimited`: the scheduler target goes to 240 and `xrWaitFrame` holds each +frame to the headset's refresh rate. Set `-1` to restore Roblox's own cap, or a +number such as `90` to pin it. + +`MOCKTAIL_VR_RENDER_SCALE` (default `1.0`, range `0.25`–`2.0`) scales each +axis of the OpenXR runtime's recommended eye size. SteamVR's recommendation +already includes its own supersampling, so on a Quest 2 at 90 Hz it is about +2244x2352 per eye. If a frame misses the 11.1 ms budget, the runtime halves +the rate to 45 FPS; `0.75` renders about 56% of the pixels. + +`MOCKTAIL_VR_UNPACED=1` is a benchmark mode. Roblox no longer waits for the +headset between frames; each XR frame stays open across several Roblox frames +and is submitted with the newest eyes once per display period. Pair it with a +fixed scheduler target above 240 and `MOCKTAIL_VSYNC=off` to find the +maximum rate: + +```bash +MOCKTAIL_VR_UNPACED=1 MOCKTAIL_VR_FRAME_RATE_LIMIT=9999 MOCKTAIL_VSYNC=off mocktail --vr +``` + +Frames beyond the headset refresh are never displayed, and they all reuse the +open frame's head pose, so expect judder. Do not play in this mode. + `graphics.vsync` (default `auto`) selects the presentation mode. `auto` and `on` both pick the lowest-latency synchronized mode the driver offers, in order: `FIFO_LATEST_READY`, `MAILBOX`, `FIFO_RELAXED`, `FIFO`. The first two do diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index d5edbd23..3a5f294e 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -399,6 +399,9 @@ class OpenXrBackend final { bool test_session_loss_injected_ = false; std::uint64_t next_session_retry_ns_ = 0; std::uint64_t frame_display_time_ = 0; + // MOCKTAIL_VR_UNPACED benchmark mode: guest presents before this monotonic + // time leave the open XR frame alone instead of blocking in xrWaitFrame. + std::uint64_t frame_due_ns_ = 0; struct ViewState { float position[3]; float orientation[4]; diff --git a/include/runtime/runtime_config.h b/include/runtime/runtime_config.h index bdfb815f..610c9336 100644 --- a/include/runtime/runtime_config.h +++ b/include/runtime/runtime_config.h @@ -128,6 +128,12 @@ class RuntimeConfig { return roblox_http_user_agent_; } const FrameRatePolicy& frame_rate() const { return frame_rate_; } + // Replaces frame_rate() while VR is enabled. Defaults to unlimited so the + // OpenXR runtime's xrWaitFrame paces Roblox at the headset refresh rate. + const FrameRatePolicy& vr_frame_rate() const { return vr_frame_rate_; } + const FrameRatePolicy& active_frame_rate() const { + return vr_enabled_ ? vr_frame_rate_ : frame_rate_; + } const std::string& vsync_mode() const { return vsync_mode_; } const PerformancePolicy& performance() const { return performance_; } const std::string& audio_output_device() const { @@ -180,6 +186,7 @@ class RuntimeConfig { bool device_profile_valid_ = true; std::optional roblox_http_user_agent_; FrameRatePolicy frame_rate_; + FrameRatePolicy vr_frame_rate_{FrameRateLimitMode::kUnlimited, 0}; std::string vsync_mode_ = "auto"; PerformancePolicy performance_; std::string audio_output_device_ = "default"; diff --git a/src/graphics/present_mode_policy.cc b/src/graphics/present_mode_policy.cc index d9c5f80e..03522cd8 100644 --- a/src/graphics/present_mode_policy.cc +++ b/src/graphics/present_mode_policy.cc @@ -37,9 +37,14 @@ PresentModePolicy ResolvePresentModePolicy(std::string_view vsync_value, PresentModePolicy CachedPresentModePolicy() { static const PresentModePolicy policy = [] { + // main() clears MOCKTAIL_VR_ENABLED when VR cannot start for this build, + // so the VR limit only applies while VR is actually active. + const std::string_view vr = EnvironmentOr("MOCKTAIL_VR_ENABLED", "0"); + const bool vr_enabled = vr == "1" || vr == "true" || vr == "on"; return ResolvePresentModePolicy( EnvironmentOr("MOCKTAIL_VSYNC", "auto"), - EnvironmentOr("MOCKTAIL_FRAME_RATE_LIMIT", "-1")); + vr_enabled ? EnvironmentOr("MOCKTAIL_VR_FRAME_RATE_LIMIT", "unlimited") + : EnvironmentOr("MOCKTAIL_FRAME_RATE_LIMIT", "-1")); }(); return policy; } diff --git a/src/legacy/legacy_runtime.cc b/src/legacy/legacy_runtime.cc index 02fee3f7..b70cdb0e 100644 --- a/src/legacy/legacy_runtime.cc +++ b/src/legacy/legacy_runtime.cc @@ -4669,6 +4669,11 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, "-1, display, unlimited, or a positive integer\n"; return EXIT_FAILURE; } + if (!runtime_config.vr_frame_rate().valid()) { + std::cerr << "[FATAL] Invalid MOCKTAIL_VR_FRAME_RATE_LIMIT; expected " + "-1, display, unlimited, or a positive integer\n"; + return EXIT_FAILURE; + } if (!runtime_config.theme_mode_valid()) { std::cerr << "[FATAL] Invalid MOCKTAIL_THEME; expected roblox, system, light, " diff --git a/src/main.cc b/src/main.cc index 8aeb294c..5f02d772 100644 --- a/src/main.cc +++ b/src/main.cc @@ -859,7 +859,8 @@ int main(int argc, char* argv[]) { std::string client_settings_overrides; std::vector kept_fflag_overrides; if (!mocktail::runtime::MergeRuntimeClientSettingsOverrides( - runtime_config.config.frame_rate(), + vr_active ? runtime_config.config.vr_frame_rate() + : runtime_config.config.frame_rate(), runtime_config.config.performance(), fflags.json, &client_settings_overrides, &command_line_error, &kept_fflag_overrides)) { @@ -947,10 +948,14 @@ int main(int argc, char* argv[]) { "counts under Roblox client-settings control\n"; } } + const mocktail::runtime::FrameRatePolicy& active_frame_rate = + vr_active ? runtime_config.config.vr_frame_rate() + : runtime_config.config.frame_rate(); if (command_line.options.mode == mocktail::runtime::CommandMode::kRun && - runtime_config.config.frame_rate().mode == + active_frame_rate.mode == mocktail::runtime::FrameRateLimitMode::kUnlimited) { - std::cout << " [runtime] frame-rate mode=unlimited scheduler_target=" + std::cout << " [runtime] " << (vr_active ? "vr " : "") + << "frame-rate mode=unlimited scheduler_target=" << mocktail::runtime::kMaximumSupportedRobloxSchedulerFps << " vsync=" << runtime_config.config.vsync_mode() << '\n'; } diff --git a/src/runtime/graphics_launch_policy.cc b/src/runtime/graphics_launch_policy.cc index 8474ea42..b5aee3b7 100644 --- a/src/runtime/graphics_launch_policy.cc +++ b/src/runtime/graphics_launch_policy.cc @@ -93,7 +93,7 @@ bool UnthrottledPresentation(const RuntimeConfig& config) { if (config.vsync_mode() == "off" || config.vsync_mode() == "0") { return true; } - return config.frame_rate().mode == FrameRateLimitMode::kUnlimited; + return config.active_frame_rate().mode == FrameRateLimitMode::kUnlimited; } bool ParsePciVendor(const std::string& raw, unsigned int* vendor) { diff --git a/src/runtime/runtime_config.cc b/src/runtime/runtime_config.cc index db731b19..a9c4c638 100644 --- a/src/runtime/runtime_config.cc +++ b/src/runtime/runtime_config.cc @@ -283,6 +283,8 @@ RuntimeConfig RuntimeConfig::FromEnvironment(const Environment& environment) { } config.frame_rate_ = ParseFrameRatePolicy( environment.GetOr("MOCKTAIL_FRAME_RATE_LIMIT", "-1")); + config.vr_frame_rate_ = ParseFrameRatePolicy( + environment.GetOr("MOCKTAIL_VR_FRAME_RATE_LIMIT", "unlimited")); config.vsync_mode_ = environment.GetOr("MOCKTAIL_VSYNC", "auto"); config.performance_ = ParsePerformancePolicy( environment.GetOr("MOCKTAIL_MULTITHREADED_RENDERING", "0"), diff --git a/src/runtime/runtime_config_bootstrap.cc b/src/runtime/runtime_config_bootstrap.cc index 38c74537..eef6a651 100644 --- a/src/runtime/runtime_config_bootstrap.cc +++ b/src/runtime/runtime_config_bootstrap.cc @@ -81,6 +81,12 @@ device: pc-windows-11 # desktop testing). Controllers use the native VR bridge. Standalone OpenXR # test scene: mocktail-vr-probe --scene. Use --no-vr to launch normally. # enabled: false + # Optional string or integer: frame-rate limit used instead of + # graphics.frame_rate_limit while VR is active. Same values: -1 (Roblox's + # own cap, usually 60), display, unlimited, or any positive integer. The + # default, unlimited, lets the OpenXR runtime pace frames at the headset's + # refresh rate. + # frame_rate_limit: unlimited performance: # Boolean (default: false): size Roblox scheduler/render queues from every diff --git a/src/runtime/runtime_config_file.cc b/src/runtime/runtime_config_file.cc index dcf53632..f8638236 100644 --- a/src/runtime/runtime_config_file.cc +++ b/src/runtime/runtime_config_file.cc @@ -169,6 +169,7 @@ bool ValidateAndMap(const ValueMap& yaml, ValueMap* environment, "runtime.headless", "runtime.roblox_library", "vr.enabled", + "vr.frame_rate_limit", "appearance.theme", "graphics.backend", "graphics.frame_rate_limit", @@ -327,6 +328,14 @@ bool ValidateAndMap(const ValueMap& yaml, ValueMap* environment, } (*environment)["MOCKTAIL_VR_ENABLED"] = parsed ? "1" : "0"; } + if (const auto frame_rate = value("vr.frame_rate_limit"); + frame_rate.has_value()) { + if (!ParseFrameRatePolicy(*frame_rate).valid()) { + *error = "vr.frame_rate_limit is not supported: " + *frame_rate; + return false; + } + (*environment)["MOCKTAIL_VR_FRAME_RATE_LIMIT"] = *frame_rate; + } if (const auto backend = value("graphics.backend"); backend.has_value()) { if (RuntimeConfig::ParseGraphicsBackend(*backend) == GraphicsBackend::kUnknown) { @@ -790,6 +799,8 @@ RuntimeConfigLoadResult LoadRuntimeConfig( result.config = RuntimeConfig::FromEnvironment(layered); if (!result.config.frame_rate().valid()) { result.error = "frame-rate policy is invalid"; + } else if (!result.config.vr_frame_rate().valid()) { + result.error = "VR frame-rate policy is invalid"; } else if (!result.config.vr_valid()) { result.error = "VR configuration is invalid: MOCKTAIL_VR_ENABLED must be " "1/0, true/false, or on/off"; @@ -944,6 +955,8 @@ bool ExportRuntimeConfigEnvironment(const RuntimeConfig& config, config.desktop_playability() ? "1" : "0", error) && SetEnvironmentValue("MOCKTAIL_FRAME_RATE_LIMIT", FrameRateValue(config.frame_rate()), error) && + SetEnvironmentValue("MOCKTAIL_VR_FRAME_RATE_LIMIT", + FrameRateValue(config.vr_frame_rate()), error) && SetEnvironmentValue("MOCKTAIL_VSYNC", config.vsync_mode(), error) && SetEnvironmentValue( "MOCKTAIL_MULTITHREADED_RENDERING", diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 58c830b1..e48d7710 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -1041,6 +1041,40 @@ bool OpenXrBackend::InitializeSessionResourcesLocked(std::string *error) { xr_eye_width_[0], xr_eye_height_[0], xr_eye_width_[1], xr_eye_height_[1]); + // Per-axis scale on the runtime's recommendation. The swapchains, the + // guest eye framebuffers and the submitted image rect all derive from + // these extents, so Roblox renders at the scaled size end to end. + double render_scale = EnvDouble("MOCKTAIL_VR_RENDER_SCALE", 1.0); + if (!std::isfinite(render_scale) || render_scale <= 0.0) { + render_scale = 1.0; + } + render_scale = std::clamp(render_scale, 0.25, 2.0); + if (render_scale != 1.0) { + const std::uint32_t max_width = + config_views[0].maxImageRectWidth != 0 + ? std::min(config_views[0].maxImageRectWidth, + config_views[1].maxImageRectWidth) + : UINT32_MAX; + const std::uint32_t max_height = + config_views[0].maxImageRectHeight != 0 + ? std::min(config_views[0].maxImageRectHeight, + config_views[1].maxImageRectHeight) + : UINT32_MAX; + for (int eye = 0; eye < 2; ++eye) { + xr_eye_width_[eye] = std::clamp( + static_cast( + std::lround(xr_eye_width_[eye] * render_scale)), + 1, max_width); + xr_eye_height_[eye] = std::clamp( + static_cast( + std::lround(xr_eye_height_[eye] * render_scale)), + 1, max_height); + } + Log(" [vr-backend] render scale %.2f: eye extents L=%ux%u R=%ux%u\n", + render_scale, xr_eye_width_[0], xr_eye_height_[0], + xr_eye_width_[1], xr_eye_height_[1]); + } + if (!CreateSwapchains(error)) { TeardownSessionLocked("swapchain failure", session_recovery_pending_); return false; @@ -2058,6 +2092,14 @@ void OpenXrBackend::NoteHostPresent(VkQueue queue, VkDevice device) { } void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { + // Benchmark mode: let Roblox render as fast as it can. The open XR frame + // spans several guest frames (all rendered with its pose) and is submitted + // with the newest eyes once per display period. Frames beyond the headset + // refresh are never shown, so this is for measuring, not for playing. + static const bool unpaced = EnvString("MOCKTAIL_VR_UNPACED", "0") == "1"; + if (unpaced && frame_open_ && perf::NowNs() < frame_due_ns_) { + return; + } const ScopedStageTimer cycle_timer(perf::Stage::kFrameCycle); const auto session = static_cast(xr_session_); const auto instance = static_cast(xr_instance_); @@ -2236,6 +2278,9 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { } predicted_display_time = state.predictedDisplayTime; should_render = state.shouldRender; + frame_due_ns_ = + perf::NowNs() + static_cast( + std::max(0, state.predictedDisplayPeriod)); } frame_open_ = true; frame_display_time_ = predicted_display_time; diff --git a/tests/runtime_config_file_test.cc b/tests/runtime_config_file_test.cc index 1acdf3cf..26e576f7 100644 --- a/tests/runtime_config_file_test.cc +++ b/tests/runtime_config_file_test.cc @@ -134,6 +134,43 @@ TEST(RuntimeConfigFileTest, EXPECT_NE(invalid.error.find("VR configuration"), std::string::npos); } +TEST(RuntimeConfigFileTest, VrFrameRateLimitIsSeparateFromDesktopLimit) { + TemporaryDirectory temporary; + const auto defaults = LoadRuntimeConfig( + MapEnvironment({{"MOCKTAIL_VR_ENABLED", "1"}}), + temporary.Write("version: 1\n")); + ASSERT_TRUE(defaults) << defaults.error; + EXPECT_EQ(defaults.config.frame_rate().mode, FrameRateLimitMode::kUnmanaged); + EXPECT_EQ(defaults.config.vr_frame_rate().mode, + FrameRateLimitMode::kUnlimited); + EXPECT_EQ(defaults.config.active_frame_rate().mode, + FrameRateLimitMode::kUnlimited); + + const auto file = temporary.Write( + "version: 1\ngraphics:\n frame_rate_limit: 60\n" + "vr:\n enabled: true\n frame_rate_limit: 90\n"); + const auto configured = LoadRuntimeConfig(MapEnvironment(), file); + ASSERT_TRUE(configured) << configured.error; + EXPECT_EQ(configured.config.frame_rate().fixed_fps, 60); + EXPECT_EQ(configured.config.vr_frame_rate().fixed_fps, 90); + EXPECT_EQ(configured.config.active_frame_rate().fixed_fps, 90); + + const auto desktop = + LoadRuntimeConfig(MapEnvironment({{"MOCKTAIL_VR_ENABLED", "0"}, + {"MOCKTAIL_VR_FRAME_RATE_LIMIT", "-1"}}), + file); + ASSERT_TRUE(desktop) << desktop.error; + EXPECT_EQ(desktop.config.vr_frame_rate().mode, + FrameRateLimitMode::kUnmanaged); + EXPECT_EQ(desktop.config.active_frame_rate().fixed_fps, 60); + + const auto invalid = LoadRuntimeConfig( + MapEnvironment(), + temporary.Write("version: 1\nvr:\n frame_rate_limit: fastest\n")); + EXPECT_FALSE(invalid); + EXPECT_NE(invalid.error.find("vr.frame_rate_limit"), std::string::npos); +} + TEST(RuntimeConfigFileTest, RejectsMalformedVrSectionAndUnsupportedKeys) { TemporaryDirectory temporary; for (const auto* yaml : From 4cc0e41ef3ea50d7e32d40e7cf47161deaefbb6e Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:45:51 -0500 Subject: [PATCH 20/58] vr: add in-headset virtual keyboard for TextBox input When Roblox asks for a keyboard (showKeyboard), a world-locked panel opens below eye level as an OpenXR quad layer. Controllers point at keys and the trigger types; text and key presses go through the same platform events as the desktop keyboard, so the focused TextBox edits identically. The layout is US QWERTY without the function row or Ctrl/Alt, with one-shot Shift, Caps Lock, Tab, Backspace auto-repeat, Enter and Close. A trigger aimed at the panel is withheld from Roblox so a key press is not also a click. A text field above the keys mirrors the focused TextBox, whose desktop overlay never reaches the eye images. MOCKTAIL_VR_KEYBOARD=0 turns it off. --- README.md | 10 + cmake/MocktailVR.cmake | 9 +- include/mocktail/vr/openxr_backend.h | 26 ++ include/mocktail/vr/virtual_keyboard.h | 188 +++++++++ include/runtime/roblox_text_editor.h | 6 + src/main.cc | 34 ++ src/runtime/roblox_text_surface_overlay.cc | 14 + src/vr/gles_transport.cc | 32 +- src/vr/gles_transport.h | 2 + src/vr/openxr_backend.cc | 336 ++++++++++++++- src/vr/virtual_keyboard.cc | 371 ++++++++++++++++ src/vr/virtual_keyboard_raster.cc | 325 ++++++++++++++ src/window/window.cc | 77 +++- src/window/window.h | 9 + tests/CMakeLists.txt | 4 + tests/vr/virtual_keyboard_test.cc | 469 +++++++++++++++++++++ 16 files changed, 1904 insertions(+), 8 deletions(-) create mode 100644 include/mocktail/vr/virtual_keyboard.h create mode 100644 src/vr/virtual_keyboard.cc create mode 100644 src/vr/virtual_keyboard_raster.cc create mode 100644 tests/vr/virtual_keyboard_test.cc diff --git a/README.md b/README.md index 924533f2..1c06c4a6 100644 --- a/README.md +++ b/README.md @@ -264,6 +264,16 @@ MOCKTAIL_VR_RUNTIME=alvr MOCKTAIL_GRAPHICS_BACKEND=direct-vulkan ./build/mocktai If you previously set `XR_RUNTIME_JSON`, unset it before switching runtimes. Use `--no-vr` for desktop mode. +### Typing in VR + +When a text box takes focus (chat, search), a keyboard opens in front of you, +below eye level. It is a full US layout without the function row or +Ctrl/Alt. Point at a key and pull the trigger. While you point at the keyboard, +the trigger types and does not click the game behind it. Hold Backspace to +delete repeatedly. Shift applies to the next key and selects its upper symbol; +Caps Lock stays on until you tap it again. Close dismisses the keyboard and +leaves the text box. Set `MOCKTAIL_VR_KEYBOARD=0` to turn the keyboard off. + ## Known limitations Nightcap does not currently work with `hardened_malloc`. Using diff --git a/cmake/MocktailVR.cmake b/cmake/MocktailVR.cmake index 783f35de..5ca55d33 100644 --- a/cmake/MocktailVR.cmake +++ b/cmake/MocktailVR.cmake @@ -11,11 +11,18 @@ add_library(mocktail_vr STATIC src/vr/vr_perf.cc src/vr/vr_mirror.cc src/vr/xr_controller.cc - src/vr/roblox_vr_device_bridge.cc) + src/vr/roblox_vr_device_bridge.cc + src/vr/virtual_keyboard.cc + src/vr/virtual_keyboard_raster.cc) add_library(Mocktail::VR ALIAS mocktail_vr) target_include_directories(mocktail_vr PUBLIC "${CMAKE_SOURCE_DIR}/include") target_include_directories(mocktail_vr PRIVATE "${CMAKE_SOURCE_DIR}/src") target_link_libraries(mocktail_vr PUBLIC Vulkan::Headers PRIVATE nlohmann_json::nlohmann_json) +# The in-headset keyboard draws its labels with the same SDL3_ttf/fontconfig +# stack as the TextBox overlay. MocktailInput/MocktailAudio find these +# packages later in the same directory scope; links resolve at generate time. +target_link_libraries(mocktail_vr PRIVATE SDL3_ttf::SDL3_ttf SDL3::SDL3 + PkgConfig::FONTCONFIG) # shm_open for the desktop eye mirror lives in librt on glibc. find_library(MOCKTAIL_RT_LIBRARY rt) if(MOCKTAIL_RT_LIBRARY) diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index 3a5f294e..c044892c 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -10,7 +10,10 @@ #include #include +#include + #include "mocktail/status.h" +#include "mocktail/vr/virtual_keyboard.h" #include "mocktail/vr/vr_mirror.h" #include "mocktail/vr/xr_actions.h" #include "mocktail/vr/xr_controller.h" @@ -308,6 +311,14 @@ class OpenXrBackend final { // readbacks) into one command buffer, submits once and waits on one fence. // Replaces the previous per-eye submit+fence pair. bool CopyEyesIntoSwapchains(VkQueue queue); + // In-headset keyboard: show/hide from the host's TextBox request, pointer + // input, and trigger masking. Runs inside the controller sync. + void UpdateVirtualKeyboardLocked(); + // Rasterizes and uploads the panel when it changed. Returns true when the + // keyboard swapchain holds a released image ready for a quad layer. + bool PrepareVirtualKeyboardLayerLocked(VkQueue queue); + bool UploadVirtualKeyboardVulkan(VkQueue queue, VkImage image); + void DestroyVirtualKeyboardResources(); bool CaptureEyeEvidence(VkQueue queue, int eye); ScriptedPoseSample ComputeScriptedPose(std::uint64_t frame) const; void EnsureMirrorForExtent(std::uint32_t width, std::uint32_t height); @@ -337,6 +348,21 @@ class OpenXrBackend final { std::uint32_t xr_eye_width_[2] = {}; std::uint32_t xr_eye_height_[2] = {}; EyeSwapchain eye_swapchains_[2]; + VirtualKeyboard keyboard_; + std::unique_ptr keyboard_raster_; + EyeSwapchain keyboard_swapchain_; + bool keyboard_swapchain_failed_ = false; + bool keyboard_image_ready_ = false; + std::uint64_t keyboard_uploaded_revision_ = 0; + // A new showKeyboard request may open the panel; Close disarms it until + // Roblox drops that request. + bool keyboard_armed_ = true; + pose::Pose keyboard_head_{}; + bool keyboard_head_valid_ = false; + std::vector keyboard_pixels_; + VkBuffer keyboard_staging_ = VK_NULL_HANDLE; + VkDeviceMemory keyboard_staging_memory_ = VK_NULL_HANDLE; + void* keyboard_staging_mapped_ = nullptr; int xr_blend_mode_ = 0; // Runtime name captured at xrCreateInstance so the bind-time diagnostics line // can report it without re-entering runtime_name() under the held mutex. diff --git a/include/mocktail/vr/virtual_keyboard.h b/include/mocktail/vr/virtual_keyboard.h new file mode 100644 index 00000000..a9cb2303 --- /dev/null +++ b/include/mocktail/vr/virtual_keyboard.h @@ -0,0 +1,188 @@ +#ifndef MOCKTAIL_VR_VIRTUAL_KEYBOARD_H_ +#define MOCKTAIL_VR_VIRTUAL_KEYBOARD_H_ + +#include +#include +#include + +#include "mocktail/vr/vr_pose_math.h" + +namespace mocktail::vr { + +// Host side of the in-headset keyboard. The window module owns Roblox's +// TextBox focus and the SDL event path; main wires these callbacks so the VR +// library never links the window library. Every callback may be called from +// the render thread and must be thread-safe. +struct VirtualKeyboardHost { + void* context = nullptr; + // True while Roblox has asked for an on-screen keyboard (showKeyboard). + bool (*input_requested)(void* context) = nullptr; + // Commits UTF-8 text exactly like an SDL_EVENT_TEXT_INPUT. + void (*commit_text)(void* context, const char* utf8) = nullptr; + // Presses and releases one SDL scancode exactly like a physical key. + void (*press_key)(void* context, std::uint32_t scancode) = nullptr; +}; + +// Installs the process-wide host. Passing nullptr clears it. +void SetVirtualKeyboardHost(const VirtualKeyboardHost* host); +VirtualKeyboardHost CurrentVirtualKeyboardHost(); + +// The focused TextBox's current contents, shown in the panel's text field. +// The desktop draws TextBox text as a window overlay that never reaches the +// eye images, so the headset would otherwise show an empty box. Safe from any +// thread; the text is sensitive and is never logged. +struct VirtualKeyboardText { + std::string utf8; + int cursor_utf16 = 0; + bool password = false; + std::uint64_t revision = 0; +}; +void SetVirtualKeyboardText(const char* utf8, std::size_t size, + int cursor_utf16, bool password); +void ClearVirtualKeyboardText(); +VirtualKeyboardText CurrentVirtualKeyboardText(); + +enum class VirtualKeyAction { + kText, // commits text, or shift_text under Shift/Caps Lock + kBackspace, + kEnter, + kTab, + kShift, // one-shot + kCapsLock, + kSpace, + kClose, // Escape: leaves the TextBox and hides the panel +}; + +struct VirtualKey { + std::string label; + std::string text; // committed text for kText + std::string shift_text; // committed under Shift (Caps Lock for letters) + VirtualKeyAction action = VirtualKeyAction::kText; + // Panel UV rectangle, origin top-left, both axes 0..1. + float u0 = 0.f, v0 = 0.f, u1 = 0.f, v1 = 0.f; +}; + +// One controller's pointing state for a frame. Positions are metres in the +// OpenXR LOCAL space; the ray points along the aim pose's -Z axis. +struct VirtualKeyboardPointer { + bool valid = false; + pose::Pose aim{}; + float trigger = 0.f; // 0..1 +}; + +// What the keyboard asks the host to do after an Update. +struct VirtualKeyboardOutput { + std::vector text; // committed in order + std::vector scancodes; // pressed in order, after text + bool closed = false; // user pressed Close +}; + +// Ray/quad intersection. The quad is centred on `quad` with its face along +// +Z, `width` x `height` metres. Returns false when the ray misses or points +// away. u/v are panel UVs (origin top-left). +bool IntersectVirtualKeyboardPanel(const pose::Pose& quad, float width, + float height, const pose::Pose& aim, + float* u, float* v, float* distance); + +// Pure keyboard state machine: layout, paging, shift, hover, trigger edges and +// backspace auto-repeat. No OpenXR, Vulkan or SDL types, so it is unit-tested +// directly. +class VirtualKeyboard { + public: + // A text field over 15 x 5 normal-key units, 4 cm per unit. + static constexpr float kPanelWidthMetres = 0.60f; + static constexpr float kPanelHeightMetres = 0.248f; + // Texture aspect matches the panel so keys are not stretched. + static constexpr int kTextureWidth = 1500; + static constexpr int kTextureHeight = 620; + // Top share of the panel holding the text field; keys fill the rest. + static constexpr float kFieldFraction = 120.f / 620.f; + + VirtualKeyboard(); + + // Opens the panel in front of `head` (LOCAL space), below eye level and + // tilted up toward the viewer. Resets Shift and hover; keeps Caps Lock. + void Show(const pose::Pose& head); + void Hide(); + bool visible() const { return visible_; } + const pose::Pose& panel_pose() const { return panel_; } + + // Text shown in the field. Bumps the revision only when it changed. + void SetText(const VirtualKeyboardText& text); + const std::string& text() const { return text_.utf8; } + int text_cursor_utf16() const { return text_.cursor_utf16; } + bool text_password() const { return text_.password; } + + // Processes one frame of both pointers. now_ns is monotonic. + VirtualKeyboardOutput Update(std::uint64_t now_ns, + const VirtualKeyboardPointer (&pointers)[2]); + + // True when this hand's trigger belongs to the keyboard this frame (aimed + // at the panel, or a press that began on it is still held). The backend + // withholds that trigger from Roblox so a key press is not also a click. + bool CapturesTrigger(int hand) const; + + // Current layout, with Shift/Caps Lock applied to the labels. + const std::vector& keys() const; + // -1 when the hand is not over a key. + int hovered_key(int hand) const { return hands_[hand & 1].hovered; } + int pressed_key(int hand) const { return hands_[hand & 1].pressed; } + // Pointer position on the panel, when the hand is over it. + bool cursor(int hand, float* u, float* v) const; + bool shifted() const { return shift_; } + bool caps_lock() const { return caps_lock_; } + + // Advances whenever anything visible changes; the backend re-rasterizes + // and re-uploads only when this moves. + std::uint64_t revision() const { return revision_; } + + private: + struct Hand { + int hovered = -1; + int pressed = -1; // key under an active press, -1 otherwise + bool trigger_down = false; + bool captured = false; + bool over_panel = false; + float u = 0.f, v = 0.f; + std::uint64_t repeat_at_ns = 0; + }; + + void RebuildLayout(); + int KeyAt(float u, float v) const; + std::string Output(const VirtualKey& key) const; + void Activate(const VirtualKey& key, VirtualKeyboardOutput* output); + void Touch() { ++revision_; } + + bool visible_ = false; + bool shift_ = false; + bool caps_lock_ = false; + pose::Pose panel_{}; + VirtualKeyboardText text_; + Hand hands_[2]; + std::vector keys_; + std::uint64_t revision_ = 1; +}; + +// Draws the keyboard into a tightly packed RGBA8 buffer +// (VirtualKeyboard::kTextureWidth x kTextureHeight, sRGB-encoded, opaque). +// Glyphs come from SDL3_ttf with fontconfig's sans fallback chain and are +// cached per label, so redraws after the first are plain fills and blits. +class VirtualKeyboardRasterizer { + public: + VirtualKeyboardRasterizer(); + ~VirtualKeyboardRasterizer(); + VirtualKeyboardRasterizer(const VirtualKeyboardRasterizer&) = delete; + VirtualKeyboardRasterizer& operator=(const VirtualKeyboardRasterizer&) = + delete; + + // Returns false when no font could be opened; keys still draw, unlabelled. + bool Draw(const VirtualKeyboard& keyboard, std::vector* rgba); + + private: + struct Impl; + Impl* impl_ = nullptr; +}; + +} // namespace mocktail::vr + +#endif // MOCKTAIL_VR_VIRTUAL_KEYBOARD_H_ diff --git a/include/runtime/roblox_text_editor.h b/include/runtime/roblox_text_editor.h index a620170e..e288bcfe 100644 --- a/include/runtime/roblox_text_editor.h +++ b/include/runtime/roblox_text_editor.h @@ -89,6 +89,12 @@ struct RobloxTextDisplaySink { UpdateFn update = nullptr; }; +// Process-wide second observer of the focused TextBox display, called +// synchronously after the surface overlay with the same sensitivity contract. +// The VR keyboard uses it to show the text being typed in the headset, where +// the desktop overlay is not visible. Passing an empty sink clears it. +void SetRobloxTextDisplayMirror(RobloxTextDisplaySink sink); + struct RobloxTextSink { using SyncFn = Status (*)(void* context, const char* utf8, std::size_t size, int32_t cursor_utf16); diff --git a/src/main.cc b/src/main.cc index 5f02d772..4d20ded8 100644 --- a/src/main.cc +++ b/src/main.cc @@ -27,6 +27,7 @@ #include "mocktail/vr/openxr_backend.h" #include "mocktail/vr/openxr_probe.h" #include "mocktail/vr/roblox_vr_device_bridge.h" +#include "mocktail/vr/virtual_keyboard.h" #include "runtime/auth_runtime_composition.h" #include "runtime/command_line.h" #include "runtime/crash_report_policy.h" @@ -45,6 +46,7 @@ #include "runtime/roblox_desktop_app_policy.h" #include "runtime/roblox_experience_launch_bridge.h" #include "runtime/roblox_fullscreen_runtime_bridge.h" +#include "runtime/roblox_text_editor.h" #include "runtime/runtime_config_bootstrap.h" #include "runtime/runtime_config_file.h" #include "runtime/runtime_paths.h" @@ -1031,6 +1033,38 @@ int main(int argc, char* argv[]) { "native-stereo diagnostic mode without a runtime.\n"; return EXIT_FAILURE; } + // The in-headset keyboard follows Roblox's TextBox keyboard requests + // and types through the same events as the desktop keyboard. + mocktail::vr::VirtualKeyboardHost keyboard_host; + keyboard_host.input_requested = [](void*) { + return mocktail::window::TextInputRequested(); + }; + keyboard_host.commit_text = [](void*, const char* utf8) { + mocktail::window::QueueVirtualTextInput(utf8); + }; + keyboard_host.press_key = [](void*, std::uint32_t scancode) { + mocktail::window::QueueVirtualKeyPress(scancode); + }; + mocktail::vr::SetVirtualKeyboardHost(&keyboard_host); + // TextBox text is drawn as a desktop window overlay that never reaches + // the eye images; mirror it into the keyboard's text field. + mocktail::runtime::SetRobloxTextDisplayMirror( + {nullptr, [](void*, + const mocktail::runtime::RobloxTextDisplayUpdate& + update) { + if (update.event == + mocktail::runtime::RobloxTextDisplayEvent::kHide) { + mocktail::vr::ClearVirtualKeyboardText(); + return; + } + // RbxKeyboard password input types (see the SDL mapping). + const bool password = update.text_input_type == 5 || + update.text_input_type == 9 || + update.text_input_type == 10; + mocktail::vr::SetVirtualKeyboardText( + update.utf8, update.utf8_size, update.cursor_utf16, + password); + }}); } } } diff --git a/src/runtime/roblox_text_surface_overlay.cc b/src/runtime/roblox_text_surface_overlay.cc index 5982d70e..c62530d4 100644 --- a/src/runtime/roblox_text_surface_overlay.cc +++ b/src/runtime/roblox_text_surface_overlay.cc @@ -325,6 +325,16 @@ Status RobloxTextSurfaceOverlay::UpdateViewport( return Status::Ok(); } +namespace { +std::mutex g_display_mirror_mutex; +RobloxTextDisplaySink g_display_mirror; +} // namespace + +void SetRobloxTextDisplayMirror(RobloxTextDisplaySink sink) { + std::lock_guard lock(g_display_mirror_mutex); + g_display_mirror = sink; +} + RobloxTextDisplaySink RobloxTextSurfaceOverlay::sink() { return {this, &RobloxTextSurfaceOverlay::UpdateCallback}; } @@ -334,6 +344,10 @@ void RobloxTextSurfaceOverlay::UpdateCallback( if (context != nullptr) { static_cast(context)->ApplyUpdate(update); } + std::lock_guard lock(g_display_mirror_mutex); + if (g_display_mirror.update != nullptr) { + g_display_mirror.update(g_display_mirror.context, update); + } } void RobloxTextSurfaceOverlay::ApplyUpdate( diff --git a/src/vr/gles_transport.cc b/src/vr/gles_transport.cc index 4ed92e47..3731ccce 100644 --- a/src/vr/gles_transport.cc +++ b/src/vr/gles_transport.cc @@ -38,7 +38,13 @@ namespace { PFNGLDELETERENDERBUFFERSPROC) \ X(BlitFramebuffer, \ PFNGLBLITFRAMEBUFFERPROC) \ - X(Flush, PFNGLFLUSHPROC) + X(Flush, PFNGLFLUSHPROC) \ + X(BindTexture, \ + PFNGLBINDTEXTUREPROC) \ + X(TexSubImage2D, \ + PFNGLTEXSUBIMAGE2DPROC)\ + X(PixelStorei, \ + PFNGLPIXELSTOREIPROC) struct Procs { #define DECLARE(name, type) type name = nullptr; GL_PROCS(DECLARE) @@ -391,4 +397,28 @@ bool GlesTransport::Copy(const unsigned textures[2], unsigned width, gl.Flush(); return copied; } + +bool GlesTransport::Upload(unsigned texture, int width, int height, + const void *rgba) { + if (!IsCurrent() || texture == 0 || rgba == nullptr) + return false; + // Restore the guest's bindings and unpack state around the upload. + GLint bound = 0, alignment = 4, row_length = 0, unpack_buffer = 0; + gl.GetIntegerv(GL_TEXTURE_BINDING_2D, &bound); + gl.GetIntegerv(GL_UNPACK_ALIGNMENT, &alignment); + gl.GetIntegerv(GL_UNPACK_ROW_LENGTH, &row_length); + gl.GetIntegerv(GL_PIXEL_UNPACK_BUFFER_BINDING, &unpack_buffer); + if (unpack_buffer != 0) + return false; // a bound PBO would reinterpret the pointer as an offset + gl.PixelStorei(GL_UNPACK_ALIGNMENT, 4); + gl.PixelStorei(GL_UNPACK_ROW_LENGTH, 0); + gl.BindTexture(GL_TEXTURE_2D, texture); + gl.TexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, width, height, GL_RGBA, + GL_UNSIGNED_BYTE, rgba); + gl.BindTexture(GL_TEXTURE_2D, static_cast(bound)); + gl.PixelStorei(GL_UNPACK_ALIGNMENT, alignment); + gl.PixelStorei(GL_UNPACK_ROW_LENGTH, row_length); + gl.Flush(); + return true; +} } // namespace mocktail::vr diff --git a/src/vr/gles_transport.h b/src/vr/gles_transport.h index 30fc95db..9dfb400a 100644 --- a/src/vr/gles_transport.h +++ b/src/vr/gles_transport.h @@ -29,6 +29,8 @@ class GlesTransport { static void *Wrap(const char *name, void *raw); void ResetEyes(); bool Copy(const unsigned textures[2], unsigned width, unsigned height); + // Uploads tightly packed RGBA8 pixels into a runtime swapchain texture. + bool Upload(unsigned texture, int width, int height, const void *rgba); const Eye &eye(int index) const { return eyes_[index]; } void *display() const { return display_; } void *config() const { return config_; } diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index e48d7710..6b32f702 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -174,6 +174,7 @@ struct OpenXrBackend::VkProcs { PFN_vkMapMemory MapMemory = nullptr; PFN_vkUnmapMemory UnmapMemory = nullptr; PFN_vkCmdCopyImageToBuffer CmdCopyImageToBuffer = nullptr; + PFN_vkCmdCopyBufferToImage CmdCopyBufferToImage = nullptr; PFN_vkCreateImageView CreateImageView = nullptr; PFN_vkDestroyImageView DestroyImageView = nullptr; PFN_vkCreateCommandPool CreateCommandPool = nullptr; @@ -698,6 +699,7 @@ void OpenXrBackend::SetupDeviceAfterCreationLocked( MOCKTAIL_BACKEND_VK_FN(MapMemory) MOCKTAIL_BACKEND_VK_FN(UnmapMemory) MOCKTAIL_BACKEND_VK_FN(CmdCopyImageToBuffer) + MOCKTAIL_BACKEND_VK_FN(CmdCopyBufferToImage) MOCKTAIL_BACKEND_VK_FN(CreateImageView) MOCKTAIL_BACKEND_VK_FN(DestroyImageView) MOCKTAIL_BACKEND_VK_FN(CreateCommandPool) @@ -1238,6 +1240,7 @@ void OpenXrBackend::TeardownSessionLocked(const char* reason, bool preserve_devi mirror_.DestroyStaging(MirrorProcs()); } mirror_.Close(); + DestroyVirtualKeyboardResources(); DestroyCopyResources(); DestroySwapchains(); // Action pose spaces belong to this session; detach before the session and @@ -2115,8 +2118,12 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { XrInfo( XR_TYPE_COMPOSITION_LAYER_PROJECTION); std::array views{}; - const XrCompositionLayerBaseHeader *layers[1] = { - reinterpret_cast(&projection)}; + XrCompositionLayerQuad keyboard_layer = + XrInfo(XR_TYPE_COMPOSITION_LAYER_QUAD); + const XrCompositionLayerBaseHeader *layers[2] = { + reinterpret_cast(&projection), + reinterpret_cast( + &keyboard_layer)}; const bool eyes_usable = eyes_bound_.load(std::memory_order_acquire) && eyes_[0].valid && eyes_[1].valid; const bool pose_ready = frame_views_valid_ && published_pose_.valid; @@ -2224,6 +2231,24 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { projection.views = views.data(); end.layerCount = 1; end.layers = layers; + if (PrepareVirtualKeyboardLayerLocked(queue)) { + // World-locked panel in LOCAL space, composited over the eyes. + const Pose &panel = keyboard_.panel_pose(); + keyboard_layer.space = static_cast(xr_local_space_); + keyboard_layer.eyeVisibility = XR_EYE_VISIBILITY_BOTH; + keyboard_layer.subImage.swapchain = + static_cast(keyboard_swapchain_.swapchain); + keyboard_layer.subImage.imageRect.extent = { + VirtualKeyboard::kTextureWidth, VirtualKeyboard::kTextureHeight}; + keyboard_layer.pose.position = {panel.position.x, panel.position.y, + panel.position.z}; + keyboard_layer.pose.orientation = { + panel.orientation.x, panel.orientation.y, panel.orientation.z, + panel.orientation.w}; + keyboard_layer.size = {VirtualKeyboard::kPanelWidthMetres, + VirtualKeyboard::kPanelHeightMetres}; + end.layerCount = 2; + } } } CheckXr(instance, xrEndFrame(session, &end), "xrEndFrame"); @@ -2327,6 +2352,12 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { const std::string pose_source = EnvString("MOCKTAIL_VR_POSE_SOURCE", "runtime"); Pose head_pose = head_valid ? XrToPose(head.pose) : Pose{}; + if (head_valid) { + // Tracked LOCAL head pose, independent of the scripted test source; + // the keyboard panel opens in front of it. + keyboard_head_ = head_pose; + keyboard_head_valid_ = true; + } if (pose_source == "script") { const ScriptedPoseSample scripted = ComputeScriptedPose(++xr_frame_counter_); @@ -2494,9 +2525,310 @@ void OpenXrBackend::SyncControllersLocked(std::uint64_t predicted_display_time) published_pose_.hands[0] = VrHandPose{}; published_pose_.hands[1] = VrHandPose{}; } + UpdateVirtualKeyboardLocked(); perf::ProcessCollector().Count(perf::Counter::kControllerSyncs); } +void OpenXrBackend::UpdateVirtualKeyboardLocked() { + static const bool enabled = EnvString("MOCKTAIL_VR_KEYBOARD", "1") != "0"; + if (!enabled) { + return; + } + const VirtualKeyboardHost host = CurrentVirtualKeyboardHost(); + const bool requested = + host.input_requested != nullptr && host.input_requested(host.context); + if (!requested) { + keyboard_.Hide(); + keyboard_armed_ = true; + } else if (keyboard_armed_ && keyboard_head_valid_) { + keyboard_.Show(keyboard_head_); + keyboard_armed_ = false; + Log(" [vr-keyboard] shown for focused TextBox\n"); + } + if (!keyboard_.visible()) { + return; + } + { + VirtualKeyboardText text = CurrentVirtualKeyboardText(); + keyboard_.SetText(text); + std::fill(text.utf8.begin(), text.utf8.end(), '\0'); + } + + VirtualKeyboardPointer pointers[2]; + const ControllerSnapshot& snapshot = actions_.snapshot(); + for (int hand = 0; hand < 2; ++hand) { + const VrHandPose& pose = actions_.hand(hand); + const ControllerHandState& state = snapshot.hands[hand]; + pointers[hand].valid = state.connected && pose.aim_valid; + pointers[hand].aim.position = + Vec3{pose.aim_position[0], pose.aim_position[1], pose.aim_position[2]}; + pointers[hand].aim.orientation = + Quat{pose.aim_orientation[0], pose.aim_orientation[1], + pose.aim_orientation[2], pose.aim_orientation[3]}; + pointers[hand].trigger = + state.axes[static_cast(ControllerAxis::kTrigger)]; + } + const VirtualKeyboardOutput output = + keyboard_.Update(perf::NowNs(), pointers); + for (const std::string& text : output.text) { + if (host.commit_text != nullptr) host.commit_text(host.context, text.c_str()); + } + for (const std::uint32_t scancode : output.scancodes) { + if (host.press_key != nullptr) host.press_key(host.context, scancode); + } + // A trigger aimed at the panel types; withhold it from Roblox so it is not + // also a click on whatever UI sits behind the keyboard. + for (int hand = 0; hand < 2; ++hand) { + if (keyboard_.CapturesTrigger(hand)) { + published_pose_.controller_channels[hand * 9 + 3] = 0.f; + } + } +} + +bool OpenXrBackend::PrepareVirtualKeyboardLayerLocked(VkQueue queue) { + if (!keyboard_.visible() || keyboard_swapchain_failed_) { + return false; + } + const auto instance = static_cast(xr_instance_); + try { + EyeSwapchain& slot = keyboard_swapchain_; + if (slot.swapchain == nullptr) { + auto create_info = + XrInfo(XR_TYPE_SWAPCHAIN_CREATE_INFO); + create_info.usageFlags = XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT | + XR_SWAPCHAIN_USAGE_TRANSFER_DST_BIT; + // Same format as the eyes, so the panel's sRGB-encoded bytes are + // interpreted exactly like the guest's eye pixels. + create_info.format = + static_cast(eye_swapchains_[0].format); + create_info.sampleCount = 1; + create_info.width = VirtualKeyboard::kTextureWidth; + create_info.height = VirtualKeyboard::kTextureHeight; + create_info.faceCount = 1; + create_info.arraySize = 1; + create_info.mipCount = 1; + XrSwapchain swapchain = XR_NULL_HANDLE; + CheckXr(instance, + xrCreateSwapchain(static_cast(xr_session_), + &create_info, &swapchain), + "xrCreateSwapchain keyboard"); + slot.swapchain = swapchain; + std::uint32_t count = 0; + CheckXr(instance, + xrEnumerateSwapchainImages(swapchain, 0, &count, nullptr), + "xrEnumerateSwapchainImages keyboard count"); + Require(count > 0 && count <= 16, "empty keyboard swapchain"); + if (graphics_api_ == VrGraphicsApi::kVulkan) { + std::vector images(count); + for (auto& image : images) image.type = XR_TYPE_SWAPCHAIN_IMAGE_VULKAN_KHR; + CheckXr(instance, + xrEnumerateSwapchainImages( + swapchain, count, &count, + reinterpret_cast( + images.data())), + "xrEnumerateSwapchainImages keyboard"); + for (const auto& image : images) slot.images.push_back(image.image); + } else { + std::vector images(count); + for (auto& image : images) + image.type = XR_TYPE_SWAPCHAIN_IMAGE_OPENGL_ES_KHR; + CheckXr(instance, + xrEnumerateSwapchainImages( + swapchain, count, &count, + reinterpret_cast( + images.data())), + "xrEnumerateSwapchainImages keyboard GLES"); + for (const auto& image : images) slot.gl_images.push_back(image.image); + } + slot.width = VirtualKeyboard::kTextureWidth; + slot.height = VirtualKeyboard::kTextureHeight; + slot.format = eye_swapchains_[0].format; + keyboard_image_ready_ = false; + Log(" [vr-keyboard] panel swapchain %ux%u images=%u\n", slot.width, + slot.height, count); + } + if (keyboard_image_ready_ && + keyboard_uploaded_revision_ == keyboard_.revision()) { + return true; + } + if (!keyboard_raster_) { + keyboard_raster_ = std::make_unique(); + } + (void)keyboard_raster_->Draw(keyboard_, &keyboard_pixels_); + + std::uint32_t index = 0; + auto acquire = XrInfo( + XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO); + CheckXr(instance, + xrAcquireSwapchainImage(static_cast(slot.swapchain), + &acquire, &index), + "xrAcquireSwapchainImage keyboard"); + auto wait = + XrInfo(XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO); + wait.timeout = 100'000'000; + const XrResult wait_result = + xrWaitSwapchainImage(static_cast(slot.swapchain), &wait); + bool uploaded = false; + if (XR_SUCCEEDED(wait_result) && wait_result != XR_TIMEOUT_EXPIRED) { + if (graphics_api_ == VrGraphicsApi::kVulkan) { + uploaded = index < slot.images.size() && + UploadVirtualKeyboardVulkan(queue, slot.images[index]); + } else { + uploaded = gles_ != nullptr && index < slot.gl_images.size() && + gles_->Upload(slot.gl_images[index], + VirtualKeyboard::kTextureWidth, + VirtualKeyboard::kTextureHeight, + keyboard_pixels_.data()); + } + } + auto release = XrInfo( + XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO); + (void)xrReleaseSwapchainImage(static_cast(slot.swapchain), + &release); + // A released image is what the quad shows, even if this upload failed + // after an earlier one succeeded. + if (uploaded) { + keyboard_image_ready_ = true; + keyboard_uploaded_revision_ = keyboard_.revision(); + } + return keyboard_image_ready_; + } catch (const std::exception& exception) { + Log(" [vr-keyboard] disabled: %s\n", exception.what()); + keyboard_swapchain_failed_ = true; + return false; + } +} + +bool OpenXrBackend::UploadVirtualKeyboardVulkan(VkQueue queue, VkImage image) { + std::string error; + if (vk_ == nullptr || + !EnsureCopyResources(vk_device_, vk_queue_family_, &error)) { + return false; + } + const VkDeviceSize size = static_cast(keyboard_pixels_.size()); + if (keyboard_staging_ == VK_NULL_HANDLE) { + VkBufferCreateInfo info{}; + info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + info.size = size; + info.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + if (vk_->CreateBuffer(vk_device_, &info, nullptr, &keyboard_staging_) != + VK_SUCCESS) { + keyboard_staging_ = VK_NULL_HANDLE; + return false; + } + VkMemoryRequirements requirements{}; + vk_->GetBufferMemoryRequirements(vk_device_, keyboard_staging_, + &requirements); + VkPhysicalDeviceMemoryProperties properties{}; + vk_->GetPhysicalDeviceMemoryProperties(vk_physical_device_, &properties); + const auto flags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + std::uint32_t type = UINT32_MAX; + for (std::uint32_t t = 0; t < properties.memoryTypeCount; ++t) { + if ((requirements.memoryTypeBits & (1u << t)) && + (properties.memoryTypes[t].propertyFlags & flags) == flags) { + type = t; + break; + } + } + VkMemoryAllocateInfo allocation{}; + allocation.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + allocation.allocationSize = requirements.size; + allocation.memoryTypeIndex = type; + if (type == UINT32_MAX || + vk_->AllocateMemory(vk_device_, &allocation, nullptr, + &keyboard_staging_memory_) != VK_SUCCESS || + vk_->BindBufferMemory(vk_device_, keyboard_staging_, + keyboard_staging_memory_, 0) != VK_SUCCESS || + vk_->MapMemory(vk_device_, keyboard_staging_memory_, 0, size, 0, + &keyboard_staging_mapped_) != VK_SUCCESS) { + DestroyVirtualKeyboardResources(); + return false; + } + } + std::memcpy(keyboard_staging_mapped_, keyboard_pixels_.data(), + keyboard_pixels_.size()); + + if (vk_->ResetCommandBuffer(copy_command_, 0) != VK_SUCCESS) return false; + VkCommandBufferBeginInfo begin{}; + begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + if (vk_->BeginCommandBuffer(copy_command_, &begin) != VK_SUCCESS) { + return false; + } + VkImageMemoryBarrier barrier{}; + barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + barrier.srcAccessMask = 0; + barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + barrier.image = image; + barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, + nullptr, 1, &barrier); + VkBufferImageCopy region{}; + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.imageExtent = {VirtualKeyboard::kTextureWidth, + VirtualKeyboard::kTextureHeight, 1}; + vk_->CmdCopyBufferToImage(copy_command_, keyboard_staging_, image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); + barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + barrier.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, + nullptr, 1, &barrier); + if (vk_->EndCommandBuffer(copy_command_) != VK_SUCCESS || + vk_->ResetFences(vk_device_, 1, ©_fence_) != VK_SUCCESS) { + return false; + } + VkSubmitInfo submit{}; + submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit.commandBufferCount = 1; + submit.pCommandBuffers = ©_command_; + if (SubmitHostQueue(vk_->QueueSubmit, vk_device_, queue, 1, &submit, + copy_fence_) != VK_SUCCESS) { + return false; + } + if (vk_->WaitForFences(vk_device_, 1, ©_fence_, VK_TRUE, + 2'000'000'000ULL) != VK_SUCCESS) { + (void)IdleHostDevice(vk_->DeviceWaitIdle, vk_device_); + throw std::runtime_error("keyboard upload fence wait failed"); + } + return true; +} + +void OpenXrBackend::DestroyVirtualKeyboardResources() { + if (vk_ != nullptr && vk_device_ != VK_NULL_HANDLE) { + if (keyboard_staging_mapped_ != nullptr) { + vk_->UnmapMemory(vk_device_, keyboard_staging_memory_); + } + if (keyboard_staging_ != VK_NULL_HANDLE) { + vk_->DestroyBuffer(vk_device_, keyboard_staging_, nullptr); + } + if (keyboard_staging_memory_ != VK_NULL_HANDLE) { + vk_->FreeMemory(vk_device_, keyboard_staging_memory_, nullptr); + } + } + keyboard_staging_mapped_ = nullptr; + keyboard_staging_ = VK_NULL_HANDLE; + keyboard_staging_memory_ = VK_NULL_HANDLE; + EyeSwapchain& slot = keyboard_swapchain_; + slot.images.clear(); + slot.gl_images.clear(); + if (slot.swapchain != nullptr) { + (void)xrDestroySwapchain(static_cast(slot.swapchain)); + slot.swapchain = nullptr; + } + keyboard_image_ready_ = false; + keyboard_uploaded_revision_ = 0; + keyboard_swapchain_failed_ = false; +} + bool OpenXrBackend::EnsureCopyResources(VkDevice device, std::uint32_t queue_family, std::string* error) { diff --git a/src/vr/virtual_keyboard.cc b/src/vr/virtual_keyboard.cc new file mode 100644 index 00000000..83333c8d --- /dev/null +++ b/src/vr/virtual_keyboard.cc @@ -0,0 +1,371 @@ +#include "mocktail/vr/virtual_keyboard.h" + +#include +#include +#include +#include + +namespace mocktail::vr { +namespace { + +using pose::kPi; +using pose::Pose; +using pose::Quat; +using pose::QuatConjugate; +using pose::QuatFromAxisAngle; +using pose::QuatMultiply; +using pose::QuatRotate; +using pose::Vec3; + +// SDL scancodes, kept numeric so this pure module needs no SDL headers. +constexpr std::uint32_t kScancodeReturn = 40; // SDL_SCANCODE_RETURN +constexpr std::uint32_t kScancodeEscape = 41; // SDL_SCANCODE_ESCAPE +constexpr std::uint32_t kScancodeBackspace = 42; // SDL_SCANCODE_BACKSPACE +constexpr std::uint32_t kScancodeTab = 43; // SDL_SCANCODE_TAB + +// Matches ControllerStateBuilder's analog click hysteresis. +constexpr float kTriggerPress = 0.75f; +constexpr float kTriggerRelease = 0.6f; +constexpr std::uint64_t kRepeatDelayNs = 450'000'000ULL; +constexpr std::uint64_t kRepeatIntervalNs = 60'000'000ULL; +// Cursor moves smaller than this (in UV) do not force a redraw. +constexpr float kCursorRedrawStep = 0.004f; + +constexpr float kPanelDistanceMetres = 0.45f; +constexpr float kPanelDropMetres = 0.28f; +constexpr float kPanelTiltRadians = 35.f * kPi / 180.f; + +// US QWERTY without the function row or Ctrl/Alt/Menu, in normal-key units. +// Widths follow the desktop shell OSK (Tab 1.6, Shift 2.5, wide keys fill). +constexpr float kColumns = 15.f; +constexpr int kRows = 5; + +std::mutex g_host_mutex; +VirtualKeyboardHost g_host; +std::mutex g_text_mutex; +VirtualKeyboardText g_text; + +void ScrubText(std::string* text) { + std::fill(text->begin(), text->end(), '\0'); + text->clear(); +} + +struct KeySpec { + const char* label; + const char* text; + const char* shift_text; + VirtualKeyAction action; + float width; +}; + +KeySpec Char(const char* text, const char* shift_text) { + return {text, text, shift_text, VirtualKeyAction::kText, 1.f}; +} +KeySpec Special(const char* label, VirtualKeyAction action, float width) { + return {label, "", "", action, width}; +} + +std::vector> Rows() { + using A = VirtualKeyAction; + return { + {Char("`", "~"), Char("1", "!"), Char("2", "@"), Char("3", "#"), + Char("4", "$"), Char("5", "%"), Char("6", "^"), Char("7", "&"), + Char("8", "*"), Char("9", "("), Char("0", ")"), Char("-", "_"), + Char("=", "+"), Special("Backspace", A::kBackspace, 2.f)}, + {Special("Tab", A::kTab, 1.6f), Char("q", "Q"), Char("w", "W"), + Char("e", "E"), Char("r", "R"), Char("t", "T"), Char("y", "Y"), + Char("u", "U"), Char("i", "I"), Char("o", "O"), Char("p", "P"), + Char("[", "{"), Char("]", "}"), {"\\", "\\", "|", A::kText, 1.4f}}, + {Special("Caps", A::kCapsLock, 2.f), Char("a", "A"), Char("s", "S"), + Char("d", "D"), Char("f", "F"), Char("g", "G"), Char("h", "H"), + Char("j", "J"), Char("k", "K"), Char("l", "L"), Char(";", ":"), + Char("'", "\""), Special("Enter", A::kEnter, 2.f)}, + {Special("Shift", A::kShift, 2.5f), Char("z", "Z"), Char("x", "X"), + Char("c", "C"), Char("v", "V"), Char("b", "B"), Char("n", "N"), + Char("m", "M"), Char(",", "<"), Char(".", ">"), Char("/", "?"), + Special("Shift", A::kShift, 2.5f)}, + {Special("Space", A::kSpace, 12.5f), Special("Close", A::kClose, 2.5f)}, + }; +} + +bool IsLetter(const std::string& text) { + return text.size() == 1 && + std::isalpha(static_cast(text[0])) != 0; +} + +float Dot(const Vec3& a, const Vec3& b) { + return a.x * b.x + a.y * b.y + a.z * b.z; +} +Vec3 Sub(const Vec3& a, const Vec3& b) { + return {a.x - b.x, a.y - b.y, a.z - b.z}; +} + +} // namespace + +void SetVirtualKeyboardHost(const VirtualKeyboardHost* host) { + std::lock_guard lock(g_host_mutex); + g_host = host != nullptr ? *host : VirtualKeyboardHost{}; +} + +VirtualKeyboardHost CurrentVirtualKeyboardHost() { + std::lock_guard lock(g_host_mutex); + return g_host; +} + +void SetVirtualKeyboardText(const char* utf8, std::size_t size, + int cursor_utf16, bool password) { + std::lock_guard lock(g_text_mutex); + ScrubText(&g_text.utf8); + if (utf8 != nullptr) g_text.utf8.assign(utf8, size); + g_text.cursor_utf16 = std::max(0, cursor_utf16); + g_text.password = password; + ++g_text.revision; +} + +void ClearVirtualKeyboardText() { + std::lock_guard lock(g_text_mutex); + ScrubText(&g_text.utf8); + g_text.cursor_utf16 = 0; + g_text.password = false; + ++g_text.revision; +} + +VirtualKeyboardText CurrentVirtualKeyboardText() { + std::lock_guard lock(g_text_mutex); + return g_text; +} + +bool IntersectVirtualKeyboardPanel(const Pose& quad, float width, float height, + const Pose& aim, float* u, float* v, + float* distance) { + const Vec3 normal = QuatRotate(quad.orientation, Vec3{0.f, 0.f, 1.f}); + const Vec3 direction = QuatRotate(aim.orientation, Vec3{0.f, 0.f, -1.f}); + const float denominator = Dot(direction, normal); + // Only the front face takes input; a ray from behind passes through. + if (denominator > -1e-4f) return false; + const float t = Dot(Sub(quad.position, aim.position), normal) / denominator; + if (!(t > 0.f)) return false; + const Vec3 hit{aim.position.x + direction.x * t, + aim.position.y + direction.y * t, + aim.position.z + direction.z * t}; + const Vec3 local = + QuatRotate(QuatConjugate(quad.orientation), Sub(hit, quad.position)); + const float hit_u = local.x / width + 0.5f; + const float hit_v = 0.5f - local.y / height; + if (hit_u < 0.f || hit_u > 1.f || hit_v < 0.f || hit_v > 1.f) return false; + if (u != nullptr) *u = hit_u; + if (v != nullptr) *v = hit_v; + if (distance != nullptr) *distance = t; + return true; +} + +VirtualKeyboard::VirtualKeyboard() { RebuildLayout(); } + +void VirtualKeyboard::RebuildLayout() { + const auto rows = Rows(); + keys_.clear(); + for (int row = 0; row < kRows; ++row) { + float column = 0.f; + for (const KeySpec& spec : rows[row]) { + VirtualKey key; + key.action = spec.action; + key.text = spec.text; + key.shift_text = spec.shift_text; + key.label = spec.action == VirtualKeyAction::kText ? Output(key) + : spec.label; + key.u0 = column / kColumns; + key.u1 = std::min(1.f, (column + spec.width) / kColumns); + constexpr float kField = VirtualKeyboard::kFieldFraction; + key.v0 = kField + (1.f - kField) * static_cast(row) / kRows; + key.v1 = kField + (1.f - kField) * static_cast(row + 1) / kRows; + keys_.push_back(std::move(key)); + column += spec.width; + } + } + for (Hand& hand : hands_) { + hand.hovered = -1; + hand.pressed = -1; + } + Touch(); +} + +std::string VirtualKeyboard::Output(const VirtualKey& key) const { + // Caps Lock only affects letters; Shift inverts it for letters and selects + // the upper symbol on every other key. + if (IsLetter(key.text)) { + return (shift_ != caps_lock_) ? key.shift_text : key.text; + } + return shift_ ? key.shift_text : key.text; +} + +const std::vector& VirtualKeyboard::keys() const { return keys_; } + +void VirtualKeyboard::Show(const Pose& head) { + Vec3 forward = QuatRotate(head.orientation, Vec3{0.f, 0.f, -1.f}); + forward.y = 0.f; + const float length = std::sqrt(forward.x * forward.x + forward.z * forward.z); + if (length < 1e-3f) { + forward = Vec3{0.f, 0.f, -1.f}; + } else { + forward.x /= length; + forward.z /= length; + } + const float yaw = std::atan2(-forward.x, -forward.z); + panel_.orientation = + QuatMultiply(QuatFromAxisAngle(Vec3{0.f, 1.f, 0.f}, yaw), + QuatFromAxisAngle(Vec3{1.f, 0.f, 0.f}, -kPanelTiltRadians)); + panel_.position = + Vec3{head.position.x + forward.x * kPanelDistanceMetres, + head.position.y - kPanelDropMetres, + head.position.z + forward.z * kPanelDistanceMetres}; + visible_ = true; + shift_ = false; + for (Hand& hand : hands_) hand = Hand{}; + RebuildLayout(); +} + +void VirtualKeyboard::SetText(const VirtualKeyboardText& text) { + if (text.revision == text_.revision) return; + const bool changed = text.utf8 != text_.utf8 || + text.cursor_utf16 != text_.cursor_utf16 || + text.password != text_.password; + ScrubText(&text_.utf8); + text_ = text; + if (changed) Touch(); +} + +void VirtualKeyboard::Hide() { + ScrubText(&text_.utf8); + text_.revision = UINT64_MAX; // the next SetText always reloads + if (!visible_) return; + visible_ = false; + for (Hand& hand : hands_) hand = Hand{}; + Touch(); +} + +int VirtualKeyboard::KeyAt(float u, float v) const { + for (std::size_t index = 0; index < keys_.size(); ++index) { + const VirtualKey& key = keys_[index]; + if (u >= key.u0 && u < key.u1 && v >= key.v0 && v < key.v1) { + return static_cast(index); + } + } + return -1; +} + +bool VirtualKeyboard::CapturesTrigger(int hand) const { + const Hand& state = hands_[hand & 1]; + return visible_ && (state.over_panel || state.captured); +} + +bool VirtualKeyboard::cursor(int hand, float* u, float* v) const { + const Hand& state = hands_[hand & 1]; + if (!visible_ || !state.over_panel) return false; + *u = state.u; + *v = state.v; + return true; +} + +void VirtualKeyboard::Activate(const VirtualKey& key, + VirtualKeyboardOutput* output) { + switch (key.action) { + case VirtualKeyAction::kText: + output->text.push_back(Output(key)); + if (shift_) { + shift_ = false; // one-shot, like a phone keyboard + RebuildLayout(); + } + return; + case VirtualKeyAction::kSpace: + output->text.emplace_back(" "); + return; + case VirtualKeyAction::kBackspace: + output->scancodes.push_back(kScancodeBackspace); + return; + case VirtualKeyAction::kEnter: + output->scancodes.push_back(kScancodeReturn); + return; + case VirtualKeyAction::kTab: + output->scancodes.push_back(kScancodeTab); + return; + case VirtualKeyAction::kShift: + shift_ = !shift_; + RebuildLayout(); + return; + case VirtualKeyAction::kCapsLock: + caps_lock_ = !caps_lock_; + RebuildLayout(); + return; + case VirtualKeyAction::kClose: + output->scancodes.push_back(kScancodeEscape); + output->closed = true; + Hide(); + return; + } +} + +VirtualKeyboardOutput VirtualKeyboard::Update( + std::uint64_t now_ns, const VirtualKeyboardPointer (&pointers)[2]) { + VirtualKeyboardOutput output; + if (!visible_) return output; + for (int index = 0; index < 2 && visible_; ++index) { + Hand& hand = hands_[index]; + const VirtualKeyboardPointer& pointer = pointers[index]; + float u = 0.f, v = 0.f; + const bool over = + pointer.valid && + IntersectVirtualKeyboardPanel(panel_, kPanelWidthMetres, + kPanelHeightMetres, pointer.aim, &u, &v, + nullptr); + const int hovered = over ? KeyAt(u, v) : -1; + if (over != hand.over_panel || hovered != hand.hovered || + (over && (std::abs(u - hand.u) > kCursorRedrawStep || + std::abs(v - hand.v) > kCursorRedrawStep))) { + Touch(); + hand.u = u; + hand.v = v; + } + hand.over_panel = over; + hand.hovered = hovered; + + const bool was_down = hand.trigger_down; + if (!pointer.valid) { + hand.trigger_down = false; + } else if (was_down) { + hand.trigger_down = pointer.trigger > kTriggerRelease; + } else { + hand.trigger_down = pointer.trigger >= kTriggerPress; + } + + if (hand.trigger_down && !was_down) { + hand.captured = over; + if (hovered >= 0) { + hand.pressed = hovered; + hand.repeat_at_ns = now_ns + kRepeatDelayNs; + Touch(); + // Activate may rebuild the layout and clear pressed/hovered. + const VirtualKey key = keys_[hovered]; + Activate(key, &output); + } + } else if (hand.trigger_down) { + if (hand.pressed >= 0 && hand.pressed != hovered) { + hand.pressed = -1; + Touch(); + } else if (hand.pressed >= 0 && + keys_[hand.pressed].action == VirtualKeyAction::kBackspace && + now_ns >= hand.repeat_at_ns) { + output.scancodes.push_back(kScancodeBackspace); + hand.repeat_at_ns = now_ns + kRepeatIntervalNs; + } + } else if (was_down) { + hand.captured = false; + if (hand.pressed >= 0) { + hand.pressed = -1; + Touch(); + } + } + } + return output; +} + +} // namespace mocktail::vr diff --git a/src/vr/virtual_keyboard_raster.cc b/src/vr/virtual_keyboard_raster.cc new file mode 100644 index 00000000..530623ae --- /dev/null +++ b/src/vr/virtual_keyboard_raster.cc @@ -0,0 +1,325 @@ +#include "mocktail/vr/virtual_keyboard.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace mocktail::vr { +namespace { + +struct Rgb { + std::uint8_t r, g, b; +}; + +constexpr Rgb kBackground{24, 25, 30}; +constexpr Rgb kKey{58, 60, 70}; +constexpr Rgb kSpecialKey{42, 44, 52}; +constexpr Rgb kHover{92, 96, 112}; +constexpr Rgb kAccent{82, 132, 255}; +constexpr Rgb kAccentHover{120, 160, 255}; +constexpr Rgb kLabel{240, 242, 248}; +constexpr Rgb kField{12, 13, 17}; +constexpr int kFieldPadding = 24; +constexpr int kGap = 8; +constexpr int kRadius = 12; +constexpr float kLetterPoints = 40.f; +constexpr float kSpecialPoints = 28.f; + +// fontconfig's best sans face; MOCKTAIL_TEXT_FONT wins like the TextBox +// overlay. +std::string ResolveSansFont() { + const char* configured = std::getenv("MOCKTAIL_TEXT_FONT"); + if (configured != nullptr && configured[0] != '\0') return configured; + std::string file; + if (!FcInit()) return file; + FcPattern* pattern = + FcNameParse(reinterpret_cast("sans:weight=medium")); + if (pattern == nullptr) return file; + FcConfigSubstitute(nullptr, pattern, FcMatchPattern); + FcDefaultSubstitute(pattern); + FcResult result = FcResultNoMatch; + FcPattern* match = FcFontMatch(nullptr, pattern, &result); + FcPatternDestroy(pattern); + if (match == nullptr) return file; + FcChar8* path = nullptr; + if (FcPatternGetString(match, FC_FILE, 0, &path) == FcResultMatch && + path != nullptr) { + file = reinterpret_cast(path); + } + FcPatternDestroy(match); + return file; +} + +struct AlphaMask { + int width = 0; + int height = 0; + std::vector alpha; +}; + +// Field text as drawn: password input shows one dot per code point. +std::string DisplayText(const VirtualKeyboard& keyboard) { + if (!keyboard.text_password()) return keyboard.text(); + std::string dots; + for (const char c : keyboard.text()) + if ((static_cast(c) & 0xC0) != 0x80) dots += "\u2022"; + return dots; +} + +// Byte offset in `text` of the UTF-16 cursor. Password dots are one UTF-16 +// unit each, like the BMP characters they stand for. +std::size_t CursorByte(const std::string& text, int cursor_utf16) { + std::size_t byte = 0; + int units = 0; + while (byte < text.size() && units < cursor_utf16) { + const auto lead = static_cast(text[byte]); + const std::size_t length = lead < 0x80 ? 1 : lead < 0xE0 ? 2 + : lead < 0xF0 ? 3 : 4; + units += length == 4 ? 2 : 1; + byte = std::min(text.size(), byte + length); + } + return byte; +} + +void Scrub(std::vector* bytes) { + std::fill(bytes->begin(), bytes->end(), 0); + bytes->clear(); +} + +bool InsideRoundedRect(int x, int y, int left, int top, int right, int bottom, + int radius) { + const int cx = std::clamp(x, left + radius, right - 1 - radius); + const int cy = std::clamp(y, top + radius, bottom - 1 - radius); + const int dx = x - cx; + const int dy = y - cy; + return dx * dx + dy * dy <= radius * radius; +} + +} // namespace + +struct VirtualKeyboardRasterizer::Impl { + bool font_attempted = false; + bool ttf_initialized = false; + TTF_Font* letter_font = nullptr; + TTF_Font* special_font = nullptr; + std::map, AlphaMask> masks; + // Last rendered field text; not cached by content, since it is sensitive. + AlphaMask field; + + ~Impl() { + Scrub(&field.alpha); + if (letter_font != nullptr) TTF_CloseFont(letter_font); + if (special_font != nullptr) TTF_CloseFont(special_font); + if (ttf_initialized) TTF_Quit(); + } + + bool EnsureFonts() { + if (font_attempted) return letter_font != nullptr; + font_attempted = true; + if (!TTF_Init()) { + std::fprintf(stderr, " [vr-keyboard] SDL3_ttf unavailable: %s\n", + SDL_GetError()); + return false; + } + ttf_initialized = true; + const std::string file = ResolveSansFont(); + if (!file.empty()) { + letter_font = TTF_OpenFont(file.c_str(), kLetterPoints); + special_font = TTF_OpenFont(file.c_str(), kSpecialPoints); + } + if (letter_font == nullptr || special_font == nullptr) { + std::fprintf(stderr, + " [vr-keyboard] no usable font (%s); keys are unlabelled\n", + file.empty() ? "fontconfig found none" : file.c_str()); + return false; + } + std::fprintf(stderr, " [vr-keyboard] labels use %s\n", file.c_str()); + return true; + } + + static AlphaMask Render(TTF_Font* font, const std::string& text) { + AlphaMask mask; + SDL_Surface* rendered = TTF_RenderText_Blended( + font, text.c_str(), text.size(), SDL_Color{255, 255, 255, 255}); + if (rendered != nullptr) { + SDL_Surface* rgba = SDL_ConvertSurface(rendered, SDL_PIXELFORMAT_RGBA32); + if (rgba != nullptr && SDL_LockSurface(rgba)) { + mask.width = rgba->w; + mask.height = rgba->h; + mask.alpha.resize(static_cast(mask.width) * mask.height); + for (int y = 0; y < mask.height; ++y) { + const auto* row = static_cast(rgba->pixels) + + static_cast(y) * rgba->pitch; + for (int x = 0; x < mask.width; ++x) { + mask.alpha[static_cast(y) * mask.width + x] = + row[x * 4 + 3]; + } + } + SDL_UnlockSurface(rgba); + } + if (rgba != nullptr) SDL_DestroySurface(rgba); + SDL_DestroySurface(rendered); + } + return mask; + } + + const AlphaMask* Mask(const std::string& label, bool special) { + if (label.empty() || !EnsureFonts()) return nullptr; + const auto key = std::make_pair(label, special); + auto found = masks.find(key); + if (found != masks.end()) return &found->second; + return &masks + .emplace(key, Render(special ? special_font : letter_font, + label)) + .first->second; + } +}; + +VirtualKeyboardRasterizer::VirtualKeyboardRasterizer() : impl_(new Impl) {} +VirtualKeyboardRasterizer::~VirtualKeyboardRasterizer() { delete impl_; } + +bool VirtualKeyboardRasterizer::Draw(const VirtualKeyboard& keyboard, + std::vector* rgba) { + constexpr int width = VirtualKeyboard::kTextureWidth; + constexpr int height = VirtualKeyboard::kTextureHeight; + rgba->resize(static_cast(width) * height * 4); + std::uint8_t* pixels = rgba->data(); + auto put = [&](int x, int y, Rgb color) { + std::uint8_t* p = pixels + (static_cast(y) * width + x) * 4; + p[0] = color.r; + p[1] = color.g; + p[2] = color.b; + p[3] = 255; + }; + auto blend = [&](int x, int y, Rgb color, std::uint8_t alpha) { + if (x < 0 || y < 0 || x >= width || y >= height || alpha == 0) return; + std::uint8_t* p = pixels + (static_cast(y) * width + x) * 4; + const int a = alpha; + p[0] = static_cast((color.r * a + p[0] * (255 - a)) / 255); + p[1] = static_cast((color.g * a + p[1] * (255 - a)) / 255); + p[2] = static_cast((color.b * a + p[2] * (255 - a)) / 255); + }; + for (int y = 0; y < height; ++y) + for (int x = 0; x < width; ++x) put(x, y, kBackground); + + // Text field: the focused TextBox's contents and caret, scrolled so the + // caret stays visible. + { + const int left = kGap / 2; + const int right = width - kGap / 2; + const int top = kGap / 2; + const int bottom = static_cast(std::lround( + VirtualKeyboard::kFieldFraction * height)) - + kGap / 2; + for (int y = top; y < bottom; ++y) + for (int x = left; x < right; ++x) + if (InsideRoundedRect(x, y, left, top, right, bottom, kRadius)) + put(x, y, kField); + const int clip_left = left + kFieldPadding; + const int clip_right = right - kFieldPadding; + const int inner = clip_right - clip_left; + const std::string shown = DisplayText(keyboard); + Scrub(&impl_->field.alpha); + impl_->field = AlphaMask{}; + int caret_x = 0; + int line_height = static_cast(kLetterPoints * 1.2f); + if (impl_->EnsureFonts()) { + line_height = TTF_GetFontHeight(impl_->letter_font); + if (!shown.empty()) { + impl_->field = Impl::Render(impl_->letter_font, shown); + const std::size_t cursor = + CursorByte(shown, keyboard.text_cursor_utf16()); + int unused = 0; + if (cursor > 0) + (void)TTF_GetStringSize(impl_->letter_font, shown.c_str(), cursor, + &caret_x, &unused); + } + } + const int scroll = std::max(0, caret_x - inner + 4); + const int origin_y = (top + bottom - line_height) / 2; + const AlphaMask& text = impl_->field; + for (int y = 0; y < text.height; ++y) + for (int x = 0; x < text.width; ++x) { + const int px = clip_left + x - scroll; + if (px < clip_left || px >= clip_right) continue; + blend(px, origin_y + y, kLabel, + text.alpha[static_cast(y) * text.width + x]); + } + const int caret = clip_left + caret_x - scroll; + const int caret_top = origin_y + line_height / 8; + const int caret_bottom = origin_y + line_height - line_height / 8; + for (int y = caret_top; y < caret_bottom; ++y) + for (int x = caret; x < caret + 3; ++x) + if (x >= clip_left - 2 && x < clip_right + 2) put(x, y, kAccent); + } + + bool labelled = true; + const auto& keys = keyboard.keys(); + for (std::size_t index = 0; index < keys.size(); ++index) { + const VirtualKey& key = keys[index]; + const int left = static_cast(std::lround(key.u0 * width)) + kGap / 2; + const int right = static_cast(std::lround(key.u1 * width)) - kGap / 2; + const int top = static_cast(std::lround(key.v0 * height)) + kGap / 2; + const int bottom = + static_cast(std::lround(key.v1 * height)) - kGap / 2; + const int i = static_cast(index); + const bool pressed = + keyboard.pressed_key(0) == i || keyboard.pressed_key(1) == i; + const bool hovered = + keyboard.hovered_key(0) == i || keyboard.hovered_key(1) == i; + const bool special = key.action != VirtualKeyAction::kText; + // Latched modifiers stay lit; hover still shows on top of them. + const bool latched = + (key.action == VirtualKeyAction::kShift && keyboard.shifted()) || + (key.action == VirtualKeyAction::kCapsLock && keyboard.caps_lock()); + Rgb fill = special ? kSpecialKey : kKey; + if (latched) fill = kAccent; + if (hovered) fill = latched ? kAccentHover : kHover; + if (pressed) fill = kAccent; + for (int y = top; y < bottom; ++y) + for (int x = left; x < right; ++x) + if (InsideRoundedRect(x, y, left, top, right, bottom, kRadius)) + put(x, y, fill); + + const AlphaMask* mask = impl_->Mask(key.label, special); + if (!key.label.empty() && (mask == nullptr || mask->alpha.empty())) { + labelled = false; + continue; + } + if (mask == nullptr) continue; + const int origin_x = (left + right - mask->width) / 2; + const int origin_y = (top + bottom - mask->height) / 2; + for (int y = 0; y < mask->height; ++y) + for (int x = 0; x < mask->width; ++x) + blend(origin_x + x, origin_y + y, kLabel, + mask->alpha[static_cast(y) * mask->width + x]); + } + + // Pointer dots, one per hand over the panel. + for (int hand = 0; hand < 2; ++hand) { + float u = 0.f, v = 0.f; + if (!keyboard.cursor(hand, &u, &v)) continue; + const int cx = static_cast(std::lround(u * width)); + const int cy = static_cast(std::lround(v * height)); + constexpr int kOuter = 11, kInner = 8; + for (int y = -kOuter; y <= kOuter; ++y) + for (int x = -kOuter; x <= kOuter; ++x) { + const int d2 = x * x + y * y; + if (d2 <= kInner * kInner) + blend(cx + x, cy + y, Rgb{255, 255, 255}, 255); + else if (d2 <= kOuter * kOuter) + blend(cx + x, cy + y, Rgb{0, 0, 0}, 200); + } + } + return labelled; +} + +} // namespace mocktail::vr diff --git a/src/window/window.cc b/src/window/window.cc index a3a54c33..c2b4e529 100644 --- a/src/window/window.cc +++ b/src/window/window.cc @@ -11,8 +11,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -168,6 +170,15 @@ static WindowFullscreenStateSync g_fullscreen_state_sync; static std::unique_ptr g_resize_readiness_gate; static std::unique_ptr g_text_input_backend; static std::unique_ptr g_text_input_owner; +// Generation of the showKeyboard request still awaiting its hideKeyboard; +// 0 when Roblox wants no keyboard. +static std::atomic g_text_input_requested_generation{0}; +struct VirtualInput { + std::string text; // committed text when non-empty + uint32_t scancode = 0; // otherwise one key press +}; +static std::mutex g_virtual_input_mutex; +static std::deque g_virtual_input; static std::unique_ptr g_pointer_capture_backend; // The owner is created and destroyed on the SDL thread, but the experience // composition reaches it from its own thread. The mutex guards the pointer @@ -1650,18 +1661,51 @@ bool SetPreTextInputPumpCallback(PreTextInputPumpCallback callback, } void ClearPreTextInputPumpCallback() { g_pre_text_input_pump_gate.Clear(); } void SetWindowTextInputOwnerEnabled(bool enabled) { + if (!enabled) { + g_text_input_requested_generation.store(0, std::memory_order_release); + } if (g_text_input_owner != nullptr) { g_text_input_owner->SetEnabled(enabled); } } bool RequestShowTextInput(uint64_t generation, const TextInputArea& area, const TextInputOptions& options) { - return g_text_input_owner != nullptr && - g_text_input_owner->RequestShowTextInput(generation, area, options); + const bool accepted = + g_text_input_owner != nullptr && + g_text_input_owner->RequestShowTextInput(generation, area, options); + if (accepted) { + g_text_input_requested_generation.store(generation, + std::memory_order_release); + } + return accepted; } bool RequestHideTextInput(uint64_t generation) { - return g_text_input_owner != nullptr && - g_text_input_owner->RequestHideTextInput(generation); + const bool accepted = g_text_input_owner != nullptr && + g_text_input_owner->RequestHideTextInput(generation); + if (accepted) { + uint64_t expected = generation; + g_text_input_requested_generation.compare_exchange_strong( + expected, 0, std::memory_order_acq_rel); + } + return accepted; +} +bool TextInputRequested() { + return g_text_input_requested_generation.load(std::memory_order_acquire) != + 0; +} +void QueueVirtualTextInput(const char* utf8) { + if (utf8 == nullptr || utf8[0] == '\0') { + return; + } + std::lock_guard lock(g_virtual_input_mutex); + g_virtual_input.push_back(VirtualInput{utf8, 0}); +} +void QueueVirtualKeyPress(uint32_t scancode) { + if (scancode == 0) { + return; + } + std::lock_guard lock(g_virtual_input_mutex); + g_virtual_input.push_back(VirtualInput{{}, scancode}); } void* GetGLProcAddress(const char* name) { if (!g_state.initialised || name == nullptr || name[0] == '\0') { @@ -2276,6 +2320,31 @@ bool PumpEvents() { g_text_input_owner->active())) { fprintf(stderr, " [input] SDL pointer capture state update failed\n"); } + { + std::deque virtual_input; + { + std::lock_guard lock(g_virtual_input_mutex); + virtual_input.swap(g_virtual_input); + } + for (const VirtualInput& input : virtual_input) { + platform::PlatformEvent event{}; + event.timestamp_ns = SDL_GetTicksNS(); + if (!input.text.empty()) { + event.payload = platform::TextInputEvent{input.text}; + g_platform_event_observer.Notify(event); + continue; + } + const auto scancode = static_cast(input.scancode); + const auto keycode = static_cast( + SDL_GetKeyFromScancode(scancode, SDL_KMOD_NONE, false)); + event.payload = platform::KeyEvent{true, false, input.scancode, keycode, + 0}; + g_platform_event_observer.Notify(event); + event.payload = platform::KeyEvent{false, false, input.scancode, keycode, + 0}; + g_platform_event_observer.Notify(event); + } + } platform::PlatformEvent pending_motion{}; bool has_pending_motion = false; diff --git a/src/window/window.h b/src/window/window.h index 5472f1b7..07dc85b5 100644 --- a/src/window/window.h +++ b/src/window/window.h @@ -98,6 +98,15 @@ bool RequestShowTextInput(uint64_t generation, const TextInputArea& area, const TextInputOptions& options); bool RequestHideTextInput(uint64_t generation); +// In-headset keyboard support. TextInputRequested reports whether Roblox +// currently wants an on-screen keyboard (between showKeyboard and the matching +// hideKeyboard). The Queue calls are safe from any thread; PumpEvents delivers +// them on the main thread as the same platform events a physical keyboard +// produces, so the focused TextBox edits them identically. +bool TextInputRequested(); +void QueueVirtualTextInput(const char* utf8); +void QueueVirtualKeyPress(uint32_t scancode); + void* GetEGLDisplay(); void* GetEGLSurface(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0166d2a2..7557cc08 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -40,6 +40,9 @@ mocktail_apply_compile_options(vr_pose_math_test) add_executable(xr_controller_test vr/xr_controller_test.cc) target_link_libraries(xr_controller_test PRIVATE Mocktail::VR GTest::gtest_main) mocktail_apply_compile_options(xr_controller_test) +add_executable(virtual_keyboard_test vr/virtual_keyboard_test.cc) +target_link_libraries(virtual_keyboard_test PRIVATE Mocktail::VR GTest::gtest_main) +mocktail_apply_compile_options(virtual_keyboard_test) add_executable(vr_evidence_layer_test vr/vr_evidence_layer_test.cc) target_link_libraries(vr_evidence_layer_test PRIVATE Vulkan::Headers ${CMAKE_DL_LIBS} GTest::gtest_main) @@ -1127,6 +1130,7 @@ gtest_discover_tests(vr_evidence_layer_test) gtest_discover_tests(vr_mirror_test) gtest_discover_tests(vr_pose_math_test) gtest_discover_tests(xr_controller_test) +gtest_discover_tests(virtual_keyboard_test) gtest_discover_tests(jnivm_test) gtest_discover_tests(jnivm_fields_test) gtest_discover_tests(runtime_config_test) diff --git a/tests/vr/virtual_keyboard_test.cc b/tests/vr/virtual_keyboard_test.cc new file mode 100644 index 00000000..1dfb6da8 --- /dev/null +++ b/tests/vr/virtual_keyboard_test.cc @@ -0,0 +1,469 @@ +#include + +#include +#include +#include +#include +#include + +#include "mocktail/vr/virtual_keyboard.h" + +namespace mocktail::vr { +namespace { + +using pose::Pose; +using pose::Quat; +using pose::QuatFromAxisAngle; +using pose::QuatRotate; +using pose::Vec3; + +constexpr std::uint64_t kMs = 1'000'000ULL; +constexpr std::uint32_t kBackspace = 42; +constexpr std::uint32_t kReturn = 40; +constexpr std::uint32_t kEscape = 41; +constexpr std::uint32_t kTab = 43; + +Pose HeadAt(float x, float y, float z, float yaw = 0.f) { + Pose head; + head.position = Vec3{x, y, z}; + head.orientation = QuatFromAxisAngle(Vec3{0.f, 1.f, 0.f}, yaw); + return head; +} + +// An aim pose 30 cm in front of the panel, pointing straight at (u, v). +Pose AimAt(const VirtualKeyboard& keyboard, float u, float v) { + const Pose& panel = keyboard.panel_pose(); + const Vec3 local{(u - 0.5f) * VirtualKeyboard::kPanelWidthMetres, + (0.5f - v) * VirtualKeyboard::kPanelHeightMetres, 0.3f}; + const Vec3 offset = QuatRotate(panel.orientation, local); + Pose aim; + aim.position = Vec3{panel.position.x + offset.x, panel.position.y + offset.y, + panel.position.z + offset.z}; + aim.orientation = panel.orientation; // -Z of the aim is -normal + return aim; +} + +int FindKey(const VirtualKeyboard& keyboard, const std::string& label) { + const auto& keys = keyboard.keys(); + for (std::size_t i = 0; i < keys.size(); ++i) + if (keys[i].label == label) return static_cast(i); + return -1; +} + +Pose AimAtKey(const VirtualKeyboard& keyboard, const std::string& label) { + const int index = FindKey(keyboard, label); + EXPECT_GE(index, 0) << label; + const VirtualKey& key = keyboard.keys()[index]; + return AimAt(keyboard, (key.u0 + key.u1) / 2, (key.v0 + key.v1) / 2); +} + +struct Driver { + VirtualKeyboard keyboard; + std::uint64_t now = 1'000 * kMs; + VirtualKeyboardOutput Frame(const Pose& aim, float trigger, int hand = 1, + std::uint64_t step = 11 * kMs) { + now += step; + VirtualKeyboardPointer pointers[2]; + pointers[hand].valid = true; + pointers[hand].aim = aim; + pointers[hand].trigger = trigger; + return keyboard.Update(now, pointers); + } + // Press and release a key, returning everything it produced. + VirtualKeyboardOutput Tap(const std::string& label) { + const Pose aim = AimAtKey(keyboard, label); + VirtualKeyboardOutput out = Frame(aim, 1.f); + const VirtualKeyboardOutput up = Frame(aim, 0.f); + out.text.insert(out.text.end(), up.text.begin(), up.text.end()); + out.scancodes.insert(out.scancodes.end(), up.scancodes.begin(), + up.scancodes.end()); + return out; + } +}; + +TEST(VirtualKeyboardTest, RayHitsQuadCentreAndRejectsBackAndAway) { + Pose quad; + quad.position = Vec3{0.f, 0.f, -1.f}; + Pose aim; // origin, looking down -Z + float u = 0.f, v = 0.f, distance = 0.f; + ASSERT_TRUE(IntersectVirtualKeyboardPanel(quad, 0.5f, 0.2f, aim, &u, &v, + &distance)); + EXPECT_NEAR(u, 0.5f, 1e-5f); + EXPECT_NEAR(v, 0.5f, 1e-5f); + EXPECT_NEAR(distance, 1.f, 1e-5f); + + // Up and to the right maps to the top-right of the panel (v grows down). + aim.position = Vec3{0.2f, 0.08f, 0.f}; + ASSERT_TRUE(IntersectVirtualKeyboardPanel(quad, 0.5f, 0.2f, aim, &u, &v, + nullptr)); + EXPECT_NEAR(u, 0.9f, 1e-5f); + EXPECT_NEAR(v, 0.1f, 1e-5f); + + aim.position = Vec3{0.3f, 0.f, 0.f}; // misses the 0.5 m width + EXPECT_FALSE(IntersectVirtualKeyboardPanel(quad, 0.5f, 0.2f, aim, &u, &v, + nullptr)); + aim.position = Vec3{}; + aim.orientation = QuatFromAxisAngle(Vec3{0.f, 1.f, 0.f}, pose::kPi); + EXPECT_FALSE(IntersectVirtualKeyboardPanel(quad, 0.5f, 0.2f, aim, &u, &v, + nullptr)); + // Behind the panel looking back at it: the back face takes no input. + aim.position = Vec3{0.f, 0.f, -2.f}; + EXPECT_FALSE(IntersectVirtualKeyboardPanel(quad, 0.5f, 0.2f, aim, &u, &v, + nullptr)); +} + +TEST(VirtualKeyboardTest, ShowPlacesPanelBelowAndInFrontFacingTheHead) { + VirtualKeyboard keyboard; + EXPECT_FALSE(keyboard.visible()); + // Facing +X (yaw -90 degrees) at standing height. + keyboard.Show(HeadAt(1.f, 1.6f, 2.f, -pose::kPi / 2)); + ASSERT_TRUE(keyboard.visible()); + const Pose& panel = keyboard.panel_pose(); + EXPECT_NEAR(panel.position.x, 1.45f, 1e-4f); + EXPECT_NEAR(panel.position.y, 1.32f, 1e-4f); + EXPECT_NEAR(panel.position.z, 2.f, 1e-4f); + const Vec3 normal = QuatRotate(panel.orientation, Vec3{0.f, 0.f, 1.f}); + EXPECT_LT(normal.x, -0.5f); // faces back toward the head + EXPECT_GT(normal.y, 0.3f); // tilted up toward the eyes + + // Looking from the eyes at the panel centre lands mid-panel. + Pose eyes = HeadAt(1.f, 1.6f, 2.f); + const Vec3 to_panel{panel.position.x - 1.f, panel.position.y - 1.6f, 0.f}; + const float pitch = std::atan2(to_panel.y, to_panel.x); + eyes.orientation = pose::QuatMultiply( + QuatFromAxisAngle(Vec3{0.f, 1.f, 0.f}, -pose::kPi / 2), + QuatFromAxisAngle(Vec3{1.f, 0.f, 0.f}, pitch)); + float u = 0.f, v = 0.f; + ASSERT_TRUE(IntersectVirtualKeyboardPanel( + panel, VirtualKeyboard::kPanelWidthMetres, + VirtualKeyboard::kPanelHeightMetres, eyes, &u, &v, nullptr)); + EXPECT_NEAR(u, 0.5f, 1e-3f); + EXPECT_NEAR(v, 0.5f, 1e-3f); +} + +TEST(VirtualKeyboardTest, TriggerPressTypesHoveredKeyOnce) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const Pose aim = AimAtKey(d.keyboard, "q"); + EXPECT_TRUE(d.Frame(aim, 0.f).text.empty()); + EXPECT_EQ(d.keyboard.hovered_key(1), FindKey(d.keyboard, "q")); + const auto pressed = d.Frame(aim, 0.9f); + ASSERT_EQ(pressed.text.size(), 1u); + EXPECT_EQ(pressed.text[0], "q"); + EXPECT_EQ(d.keyboard.pressed_key(1), FindKey(d.keyboard, "q")); + // Held, and hovering between the hysteresis thresholds: no repeat for text. + EXPECT_TRUE(d.Frame(aim, 0.7f, 1, 600 * kMs).text.empty()); + EXPECT_TRUE(d.Frame(aim, 0.f).text.empty()); + EXPECT_EQ(d.keyboard.pressed_key(1), -1); + EXPECT_EQ(d.Tap("Enter").scancodes, std::vector{kReturn}); + EXPECT_EQ(d.Tap("Space").text, std::vector{" "}); + EXPECT_EQ(d.Tap("Tab").scancodes, std::vector{kTab}); +} + +TEST(VirtualKeyboardTest, ShiftIsOneShotAndSelectsUpperSymbols) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + d.Tap("Shift"); + EXPECT_TRUE(d.keyboard.shifted()); + EXPECT_GE(FindKey(d.keyboard, "!"), 0); // labels follow Shift + EXPECT_EQ(d.Tap("A").text, std::vector{"A"}); + EXPECT_FALSE(d.keyboard.shifted()); + EXPECT_EQ(d.Tap("b").text, std::vector{"b"}); + d.Tap("Shift"); + EXPECT_EQ(d.Tap("!").text, std::vector{"!"}); + EXPECT_EQ(d.Tap("1").text, std::vector{"1"}); + d.Tap("Shift"); + EXPECT_EQ(d.Tap("|").text, std::vector{"|"}); + EXPECT_EQ(d.Tap("\\").text, std::vector{"\\"}); + // Tapping Shift again cancels it. + d.Tap("Shift"); + d.Tap("Shift"); + EXPECT_FALSE(d.keyboard.shifted()); +} + +// US QWERTY pairs from the desktop shell OSK (KiiteShell layouts.json, +// "English (US)"), in layout order. +const std::vector> kUsShiftPairs = { + {"`", "~"}, {"1", "!"}, {"2", "@"}, {"3", "#"}, {"4", "$"}, + {"5", "%"}, {"6", "^"}, {"7", "&"}, {"8", "*"}, {"9", "("}, + {"0", ")"}, {"-", "_"}, {"=", "+"}, {"q", "Q"}, {"w", "W"}, + {"e", "E"}, {"r", "R"}, {"t", "T"}, {"y", "Y"}, {"u", "U"}, + {"i", "I"}, {"o", "O"}, {"p", "P"}, {"[", "{"}, {"]", "}"}, + {"\\", "|"}, {"a", "A"}, {"s", "S"}, {"d", "D"}, {"f", "F"}, + {"g", "G"}, {"h", "H"}, {"j", "J"}, {"k", "K"}, {"l", "L"}, + {";", ":"}, {"'", "\""}, {"z", "Z"}, {"x", "X"}, {"c", "C"}, + {"v", "V"}, {"b", "B"}, {"n", "N"}, {"m", "M"}, {",", "<"}, + {".", ">"}, {"/", "?"}}; + +std::vector TextKeyIndices(const VirtualKeyboard& keyboard) { + std::vector indices; + for (std::size_t i = 0; i < keyboard.keys().size(); ++i) + if (keyboard.keys()[i].action == VirtualKeyAction::kText) + indices.push_back(static_cast(i)); + return indices; +} + +TEST(VirtualKeyboardTest, EveryKeyShiftsToItsUsQwertyCharacter) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const std::vector text_keys = TextKeyIndices(d.keyboard); + ASSERT_EQ(text_keys.size(), kUsShiftPairs.size()); + for (std::size_t n = 0; n < kUsShiftPairs.size(); ++n) { + const auto& [plain, shifted] = kUsShiftPairs[n]; + const int index = text_keys[n]; + SCOPED_TRACE("key " + plain); + ASSERT_EQ(d.keyboard.keys()[index].label, plain); + const VirtualKey key = d.keyboard.keys()[index]; + const Pose aim = + AimAt(d.keyboard, (key.u0 + key.u1) / 2, (key.v0 + key.v1) / 2); + + // Unshifted press types the plain character. + d.Frame(aim, 0.f); + EXPECT_EQ(d.Frame(aim, 1.f).text, std::vector{plain}); + d.Frame(aim, 0.f); + + // Shift relabels this key, the press types the shifted character, and + // the one-shot Shift then puts the plain label back. + d.Tap("Shift"); + ASSERT_TRUE(d.keyboard.shifted()); + EXPECT_EQ(d.keyboard.keys()[index].label, shifted); + d.Frame(aim, 0.f); + EXPECT_EQ(d.Frame(aim, 1.f).text, std::vector{shifted}); + d.Frame(aim, 0.f); + EXPECT_FALSE(d.keyboard.shifted()); + EXPECT_EQ(d.keyboard.keys()[index].label, plain); + } +} + +TEST(VirtualKeyboardTest, CapsLockAndShiftCombineLikeAPhysicalKeyboard) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const std::vector text_keys = TextKeyIndices(d.keyboard); + ASSERT_EQ(text_keys.size(), kUsShiftPairs.size()); + d.Tap("Caps"); + ASSERT_TRUE(d.keyboard.caps_lock()); + for (std::size_t n = 0; n < kUsShiftPairs.size(); ++n) { + const auto& [plain, shifted] = kUsShiftPairs[n]; + const bool letter = plain.size() == 1 && std::isalpha( + static_cast(plain[0])); + const int index = text_keys[n]; + SCOPED_TRACE("key " + plain); + // Caps Lock: letters upper, everything else unchanged. + const std::string caps = letter ? shifted : plain; + EXPECT_EQ(d.keyboard.keys()[index].label, caps); + const VirtualKey key = d.keyboard.keys()[index]; + const Pose aim = + AimAt(d.keyboard, (key.u0 + key.u1) / 2, (key.v0 + key.v1) / 2); + d.Frame(aim, 0.f); + EXPECT_EQ(d.Frame(aim, 1.f).text, std::vector{caps}); + d.Frame(aim, 0.f); + // Caps Lock + Shift: letters back to lower, symbols shifted. + d.Tap("Shift"); + const std::string both = letter ? plain : shifted; + EXPECT_EQ(d.keyboard.keys()[index].label, both); + d.Frame(aim, 0.f); + EXPECT_EQ(d.Frame(aim, 1.f).text, std::vector{both}); + d.Frame(aim, 0.f); + EXPECT_TRUE(d.keyboard.caps_lock()); + } +} + +TEST(VirtualKeyboardTest, CapsLockAffectsLettersOnlyAndShiftInvertsIt) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + d.Tap("Caps"); + EXPECT_TRUE(d.keyboard.caps_lock()); + EXPECT_EQ(d.Tap("C").text, std::vector{"C"}); + EXPECT_EQ(d.Tap("D").text, std::vector{"D"}); + EXPECT_EQ(d.Tap("1").text, std::vector{"1"}); + d.Tap("Shift"); + EXPECT_EQ(d.Tap("e").text, std::vector{"e"}); + EXPECT_TRUE(d.keyboard.caps_lock()); + d.Tap("Caps"); + EXPECT_EQ(d.Tap("f").text, std::vector{"f"}); +} + +TEST(VirtualKeyboardTest, BackspaceRepeatsWhileHeld) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const Pose aim = AimAtKey(d.keyboard, "Backspace"); + EXPECT_EQ(d.Frame(aim, 1.f).scancodes, + std::vector{kBackspace}); + EXPECT_TRUE(d.Frame(aim, 1.f, 1, 300 * kMs).scancodes.empty()); + EXPECT_EQ(d.Frame(aim, 1.f, 1, 200 * kMs).scancodes.size(), 1u); + EXPECT_TRUE(d.Frame(aim, 1.f, 1, 30 * kMs).scancodes.empty()); + EXPECT_EQ(d.Frame(aim, 1.f, 1, 40 * kMs).scancodes.size(), 1u); + // Sliding off the key stops the repeat. + const Pose off = AimAtKey(d.keyboard, "p"); + EXPECT_TRUE(d.Frame(off, 1.f, 1, 500 * kMs).scancodes.empty()); +} + +TEST(VirtualKeyboardTest, CapturesTriggerOnPanelAndUntilARelease) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + Pose away = AimAt(d.keyboard, 0.5f, 0.5f); + away.orientation = QuatFromAxisAngle(Vec3{0.f, 1.f, 0.f}, pose::kPi); + d.Frame(away, 0.f); + EXPECT_FALSE(d.keyboard.CapturesTrigger(1)); + const Pose on = AimAtKey(d.keyboard, "g"); + d.Frame(on, 0.f); + EXPECT_TRUE(d.keyboard.CapturesTrigger(1)); + EXPECT_FALSE(d.keyboard.CapturesTrigger(0)); + d.Frame(on, 1.f); + d.Frame(away, 1.f); // dragged off while held: still the keyboard's + EXPECT_TRUE(d.keyboard.CapturesTrigger(1)); + d.Frame(away, 0.f); + EXPECT_FALSE(d.keyboard.CapturesTrigger(1)); + // A press that starts off the panel is never captured. + d.Frame(away, 1.f); + d.Frame(on, 1.f); + EXPECT_TRUE(d.keyboard.CapturesTrigger(1)); // aimed at it now + d.Frame(away, 1.f); + EXPECT_FALSE(d.keyboard.CapturesTrigger(1)); +} + +TEST(VirtualKeyboardTest, LayoutRowsFillThePanelAndCloseKeyHides) { + VirtualKeyboard keyboard; + // Five rows; every row starts at the left edge and ends at the right edge. + int rows = 0; + for (const VirtualKey& key : keyboard.keys()) { + if (key.u0 == 0.f) ++rows; + } + EXPECT_EQ(rows, 5); + for (const VirtualKey& key : keyboard.keys()) { + EXPECT_LT(key.u0, key.u1); + EXPECT_LE(key.u1, 1.f); + } + for (const char* label : {"`", "=", "Backspace", "Tab", "[", "]", "\\", + "Caps", ";", "'", "Enter", "Shift", "/", "Space", + "Close"}) { + EXPECT_GE(FindKey(keyboard, label), 0) << label; + } + for (const char* label : {"Esc", "F1", "Ctrl", "Alt", "Menu"}) { + EXPECT_EQ(FindKey(keyboard, label), -1) << label; + } + + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const auto closed = d.Frame(AimAtKey(d.keyboard, "Close"), 1.f); + EXPECT_TRUE(closed.closed); + EXPECT_EQ(closed.scancodes, std::vector{kEscape}); + EXPECT_FALSE(d.keyboard.visible()); + EXPECT_FALSE(d.keyboard.CapturesTrigger(1)); + EXPECT_TRUE(d.Frame(AimAtKey(d.keyboard, "q"), 0.f).text.empty()); +} + +TEST(VirtualKeyboardTest, RevisionMovesOnlyWhenTheImageChanges) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const Pose aim = AimAtKey(d.keyboard, "h"); + d.Frame(aim, 0.f); + const std::uint64_t hovered = d.keyboard.revision(); + d.Frame(aim, 0.f); + EXPECT_EQ(d.keyboard.revision(), hovered); + d.Frame(AimAtKey(d.keyboard, "j"), 0.f); + EXPECT_GT(d.keyboard.revision(), hovered); +} + +TEST(VirtualKeyboardTest, TextFieldSitsAboveTheKeysAndTypesNothing) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + for (const VirtualKey& key : d.keyboard.keys()) + EXPECT_GE(key.v0, VirtualKeyboard::kFieldFraction - 1e-6f) << key.label; + const Pose field = AimAt(d.keyboard, 0.5f, VirtualKeyboard::kFieldFraction / 2); + d.Frame(field, 0.f); + EXPECT_EQ(d.keyboard.hovered_key(1), -1); + EXPECT_TRUE(d.keyboard.CapturesTrigger(1)); // still the keyboard's + const auto out = d.Frame(field, 1.f); + EXPECT_TRUE(out.text.empty()); + EXPECT_TRUE(out.scancodes.empty()); +} + +TEST(VirtualKeyboardTest, TextMirrorFollowsTheFocusedTextBox) { + ClearVirtualKeyboardText(); + SetVirtualKeyboardText("hi", 2, 2, false); + VirtualKeyboardText text = CurrentVirtualKeyboardText(); + EXPECT_EQ(text.utf8, "hi"); + EXPECT_EQ(text.cursor_utf16, 2); + + VirtualKeyboard keyboard; + keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const std::uint64_t before = keyboard.revision(); + keyboard.SetText(text); + EXPECT_EQ(keyboard.text(), "hi"); + EXPECT_GT(keyboard.revision(), before); + // Same mirror revision: nothing to redraw. + const std::uint64_t same = keyboard.revision(); + keyboard.SetText(text); + EXPECT_EQ(keyboard.revision(), same); + // Cursor moves alone still redraw. + SetVirtualKeyboardText("hi", 2, 1, false); + keyboard.SetText(CurrentVirtualKeyboardText()); + EXPECT_EQ(keyboard.text_cursor_utf16(), 1); + EXPECT_GT(keyboard.revision(), same); + + // Hiding forgets the text; reopening reloads it even without a new update. + keyboard.Hide(); + EXPECT_TRUE(keyboard.text().empty()); + keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + keyboard.SetText(CurrentVirtualKeyboardText()); + EXPECT_EQ(keyboard.text(), "hi"); + + ClearVirtualKeyboardText(); + EXPECT_TRUE(CurrentVirtualKeyboardText().utf8.empty()); +} + +TEST(VirtualKeyboardTest, RasterizerDrawsFieldTextAndCaret) { + VirtualKeyboard keyboard; + keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + VirtualKeyboardRasterizer rasterizer; + auto field_light_pixels = [&](const std::vector& rgba) { + const int bottom = static_cast(VirtualKeyboard::kFieldFraction * + VirtualKeyboard::kTextureHeight); + int count = 0; + for (int y = 8; y < bottom - 8; ++y) + for (int x = 30; x < VirtualKeyboard::kTextureWidth - 30; ++x) + if (rgba[(static_cast(y) * + VirtualKeyboard::kTextureWidth + x) * 4 + 1] > 120) + ++count; + return count; + }; + std::vector empty, typed, masked; + if (!rasterizer.Draw(keyboard, &empty)) GTEST_SKIP() << "no host font"; + VirtualKeyboardText text; + text.utf8 = "Hello world"; + text.cursor_utf16 = 11; + text.revision = 1; + keyboard.SetText(text); + ASSERT_TRUE(rasterizer.Draw(keyboard, &typed)); + // Text adds light glyph pixels; the empty field only has its caret. + EXPECT_GT(field_light_pixels(typed), field_light_pixels(empty) + 200); + text.password = true; + text.revision = 2; + keyboard.SetText(text); + ASSERT_TRUE(rasterizer.Draw(keyboard, &masked)); + EXPECT_NE(masked, typed); // dots, not the letters + EXPECT_GT(field_light_pixels(masked), field_light_pixels(empty)); +} + +TEST(VirtualKeyboardTest, RasterizerFillsAnOpaqueTexture) { + VirtualKeyboard keyboard; + keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + VirtualKeyboardRasterizer rasterizer; + std::vector rgba; + (void)rasterizer.Draw(keyboard, &rgba); // labels depend on host fonts + ASSERT_EQ(rgba.size(), static_cast( + VirtualKeyboard::kTextureWidth) * + VirtualKeyboard::kTextureHeight * 4); + for (std::size_t i = 3; i < rgba.size(); i += 4) ASSERT_EQ(rgba[i], 255); + // A key is lighter than the background behind it. + const auto& q = keyboard.keys()[FindKey(keyboard, "q")]; + const int x = static_cast(q.u0 * VirtualKeyboard::kTextureWidth) + 20; + const int y = static_cast(q.v0 * VirtualKeyboard::kTextureHeight) + 20; + const std::size_t key_px = + (static_cast(y) * VirtualKeyboard::kTextureWidth + x) * 4; + EXPECT_GT(rgba[key_px], rgba[0]); +} + +} // namespace +} // namespace mocktail::vr From 360ab6af585d9329bf769e3d3ad5c1a46612aa23 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:48:32 -0500 Subject: [PATCH 21/58] vr: fix CI failures from the 3092 profile Mark the 3092 profile experimental. It still runs when selected with --roblox-lib, but it no longer becomes the preferred update target for every x86_64 user, so the shipped payload metadata (2998) is again the preferred profile. Select the 2998 entry by Build ID in the partial-input profile test instead of relying on manifest order, and drop the shipped 3092 entry before the test re-identifies 2998 as 3092. Align the example config's VR build wording with the bootstrap template. --- config/mocktail.example.yaml | 2 +- config/roblox_compatibility.json | 4 ++-- tests/build_profile_test.cc | 20 +++++++++++++++++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/config/mocktail.example.yaml b/config/mocktail.example.yaml index 332eb1d6..6033104c 100644 --- a/config/mocktail.example.yaml +++ b/config/mocktail.example.yaml @@ -54,7 +54,7 @@ graphics: # vr: # Boolean (default: enabled in VR builds, disabled in non-VR builds). - # Experimental VR for verified build 3092. Mocktail initializes + # Experimental VR for verified builds 2998 and 3092. Mocktail initializes # the guest DebugDeviceVR eye resources (two # distinct 500x500 eye framebuffers), the Roblox renderer draws both eyes # every frame, and the frames are submitted to the active OpenXR runtime as diff --git a/config/roblox_compatibility.json b/config/roblox_compatibility.json index bab9b6f2..b7e86f60 100644 --- a/config/roblox_compatibility.json +++ b/config/roblox_compatibility.json @@ -128,8 +128,8 @@ "vtable_layout_version": 2, "vtable_rva": "0x6cd3ce0" }, - "reason": "Exact-Build-ID host ABI promoted from the validated probation profile; native stereo, VR state/pointer/haptic contracts and FMOD output-device layout v2 were re-derived for 3092. Physical headset acceptance remains pending.", - "status": "supported", + "reason": "Exact-Build-ID host ABI promoted from the validated probation profile; native stereo, VR state/pointer/haptic contracts and FMOD output-device layout v2 were re-derived for 3092. Experimental: runs when selected explicitly (--roblox-lib) but is never an update target, so non-VR users stay on the default payload.", + "status": "experimental", "user_game_settings_fullscreen_setter_rva": "0x45c5f42", "version_code": 3092, "version_name": "2.738.1397", diff --git a/tests/build_profile_test.cc b/tests/build_profile_test.cc index 8f3f5aaf..229eefc6 100644 --- a/tests/build_profile_test.cc +++ b/tests/build_profile_test.cc @@ -1,5 +1,6 @@ #include "compat/build_profile.h" +#include #include #include @@ -177,7 +178,24 @@ TEST(BuildProfileTest, RejectsPartialInputProfilesAndUpgradesCachedLayouts) { std::ifstream input(kManifestPath); input >> document; } - auto &entry = document["profiles"].back(); + // Start from build 2998, whose input RVAs the assertions below expect, and + // later re-identify it as 3092 to test the cached-layout upgrade. Drop the + // shipped 3092 entry first so the Build ID stays unique. + auto &profiles = document["profiles"]; + for (auto it = profiles.begin(); it != profiles.end();) { + if (it->value("elf_build_id", "") == + "5f0704edd9064f566ee3d6df2bd2fabbcc709f03") + it = profiles.erase(it); + else + ++it; + } + const auto found = std::find_if( + profiles.begin(), profiles.end(), [](const nlohmann::json &profile) { + return profile.value("elf_build_id", "") == + "ade08266c67aee88ec9c1d00902150e1684dad3a"; + }); + ASSERT_NE(found, profiles.end()); + auto &entry = *found; auto &bridge = entry["fmod_output_device_bridge"]; char path[] = "/tmp/mocktail-input-profile-XXXXXX"; int fd = mkstemp(path); From 3ef81c3d935db2342a65c30881f2112b5d45440d Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:53:31 -0500 Subject: [PATCH 22/58] vr: make VR opt-in and build without the OpenXR submodule VR now stays off at runtime in every build until --vr or vr.enabled: true, so a normal launch on a VR-capable build never tries to reach an OpenXR runtime and no longer exits when none is running. MOCKTAIL_ENABLE_VR defaults to AUTO: OpenXR support is built when the third_party/OpenXR-SDK submodule is present and skipped with a notice otherwise, so source packages that do not fetch submodules (Nix, AUR) keep building. ON still requires the submodule. mocktail-vr-probe is only built and installed in VR builds. CI gains an x86_64 job that builds and tests with MOCKTAIL_ENABLE_VR=OFF. The config docs and README describe the new defaults. --- .github/workflows/ci.yml | 12 ++++++-- CMakeLists.txt | 3 -- README.md | 10 ++++++- cmake/MocktailVR.cmake | 38 ++++++++++++++++++++----- config/mocktail.example.yaml | 2 +- include/runtime/runtime_config.h | 7 ++--- src/runtime/runtime_config_bootstrap.cc | 2 +- tests/CMakeLists.txt | 2 +- tests/vr/probe_contract_test.py | 12 +++----- 9 files changed, 60 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f6f2aa2..d18ab1b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,7 @@ jobs: bash tests/anylinux_dependencies_test.sh . native-build: - name: Native build (${{ matrix.arch }}) + name: Native build (${{ matrix.arch }}, VR ${{ matrix.vr }}) runs-on: ${{ matrix.runner }} timeout-minutes: 60 strategy: @@ -86,8 +86,15 @@ jobs: include: - arch: x86_64 runner: ubuntu-26.04 + vr: 'ON' - arch: aarch64 runner: ubuntu-26.04-arm + vr: 'ON' + # Packaging recipes build without the OpenXR submodule; keep the + # non-VR configuration compiling and passing on its own. + - arch: x86_64 + runner: ubuntu-26.04 + vr: 'OFF' steps: - name: Check out source uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -112,7 +119,8 @@ jobs: run: | test "$(uname -m)" = "${{ matrix.arch }}" cmake -S . -B build-ci -G Ninja \ - -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON + -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=ON \ + -DMOCKTAIL_ENABLE_VR=${{ matrix.vr }} cmake --build build-ci --parallel 4 ./build-ci/mocktail --help ./build-ci/mocktail_updater status diff --git a/CMakeLists.txt b/CMakeLists.txt index f6cd25cd..43a1dbb7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -319,9 +319,6 @@ target_include_directories(mocktail_runtime PUBLIC include) target_include_directories(mocktail_runtime PUBLIC ${MOCKTAIL_JNI_INCLUDE_DIRS} ) -target_compile_definitions(mocktail_runtime PUBLIC - MOCKTAIL_VR_DEFAULT_ENABLED=$ -) target_compile_definitions(mocktail_runtime PRIVATE "MOCKTAIL_DISCORD_APPLICATION_ID=\"${MOCKTAIL_DISCORD_APPLICATION_ID}\"" ) diff --git a/README.md b/README.md index 1c06c4a6..6ce26006 100644 --- a/README.md +++ b/README.md @@ -240,11 +240,19 @@ Experimental OpenXR VR support on Linux `x86_64` using WiVRn or SteamVR/ALVR. ### Build with VR +VR support is compiled in when the OpenXR SDK submodule is checked out +(`MOCKTAIL_ENABLE_VR` defaults to `AUTO`). Without the submodule the build +skips VR. Pass `-DMOCKTAIL_ENABLE_VR=ON` to make a missing submodule an error, +or `OFF` to leave VR out: + ```bash -cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DMOCKTAIL_ENABLE_VR=ON +git submodule update --init third_party/OpenXR-SDK +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release cmake --build build -j4 ``` +VR stays off at runtime until you pass `--vr` or set `vr.enabled: true`. + ### Run Start WiVRn or SteamVR/ALVR and connect your headset first: diff --git a/cmake/MocktailVR.cmake b/cmake/MocktailVR.cmake index 5ca55d33..9b610d2c 100644 --- a/cmake/MocktailVR.cmake +++ b/cmake/MocktailVR.cmake @@ -3,7 +3,27 @@ include_guard(GLOBAL) -option(MOCKTAIL_ENABLE_VR "Build experimental native OpenXR support (VR branch default)" ON) +# AUTO builds OpenXR support when the third_party/OpenXR-SDK submodule is +# checked out and skips it otherwise, so source packages that do not fetch +# submodules still build. ON requires the submodule; OFF never builds it. +set(MOCKTAIL_ENABLE_VR "AUTO" CACHE STRING + "Build experimental OpenXR support: AUTO, ON or OFF") +set_property(CACHE MOCKTAIL_ENABLE_VR PROPERTY STRINGS AUTO ON OFF) +set(_mocktail_openxr_sdk "${CMAKE_SOURCE_DIR}/third_party/OpenXR-SDK/CMakeLists.txt") +string(TOUPPER "${MOCKTAIL_ENABLE_VR}" _mocktail_vr_request) +if(_mocktail_vr_request STREQUAL "AUTO") + if(EXISTS "${_mocktail_openxr_sdk}" AND CMAKE_SYSTEM_NAME STREQUAL "Linux") + set(MOCKTAIL_ENABLE_VR ON) + else() + message(STATUS "OpenXR SDK submodule not found; building without VR " + "(git submodule update --init third_party/OpenXR-SDK to enable it)") + set(MOCKTAIL_ENABLE_VR OFF) + endif() +elseif(MOCKTAIL_ENABLE_VR) + set(MOCKTAIL_ENABLE_VR ON) +else() + set(MOCKTAIL_ENABLE_VR OFF) +endif() add_library(mocktail_vr STATIC src/vr/diagnostics.cc @@ -34,7 +54,7 @@ if(MOCKTAIL_ENABLE_VR) if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") message(FATAL_ERROR "Mocktail OpenXR support currently targets Linux") endif() - if(NOT EXISTS "${CMAKE_SOURCE_DIR}/third_party/OpenXR-SDK/CMakeLists.txt") + if(NOT EXISTS "${_mocktail_openxr_sdk}") message(FATAL_ERROR "Initialize OpenXR: git submodule update --init third_party/OpenXR-SDK") endif() @@ -70,8 +90,12 @@ else() src/vr/xr_actions_disabled.cc) endif() -add_executable(mocktail_vr_probe src/vr/probe_main.cc) -set_target_properties(mocktail_vr_probe PROPERTIES OUTPUT_NAME mocktail-vr-probe) -target_link_libraries(mocktail_vr_probe PRIVATE Mocktail::VR) -mocktail_apply_compile_options(mocktail_vr_probe) -install(TARGETS mocktail_vr_probe RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") +if(MOCKTAIL_ENABLE_VR) + add_executable(mocktail_vr_probe src/vr/probe_main.cc) + set_target_properties(mocktail_vr_probe PROPERTIES + OUTPUT_NAME mocktail-vr-probe) + target_link_libraries(mocktail_vr_probe PRIVATE Mocktail::VR) + mocktail_apply_compile_options(mocktail_vr_probe) + install(TARGETS mocktail_vr_probe + RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") +endif() diff --git a/config/mocktail.example.yaml b/config/mocktail.example.yaml index 6033104c..66b1b790 100644 --- a/config/mocktail.example.yaml +++ b/config/mocktail.example.yaml @@ -53,7 +53,7 @@ graphics: # vsync: off # vr: - # Boolean (default: enabled in VR builds, disabled in non-VR builds). + # Boolean (default: false). Same as launching with --vr. # Experimental VR for verified builds 2998 and 3092. Mocktail initializes # the guest DebugDeviceVR eye resources (two # distinct 500x500 eye framebuffers), the Roblox renderer draws both eyes diff --git a/include/runtime/runtime_config.h b/include/runtime/runtime_config.h index 610c9336..9be1877a 100644 --- a/include/runtime/runtime_config.h +++ b/include/runtime/runtime_config.h @@ -16,10 +16,9 @@ namespace mocktail { namespace runtime { -#ifndef MOCKTAIL_VR_DEFAULT_ENABLED -#define MOCKTAIL_VR_DEFAULT_ENABLED 0 -#endif -inline constexpr bool kDefaultVrEnabled = MOCKTAIL_VR_DEFAULT_ENABLED != 0; +// VR is opt-in in every build (--vr or vr.enabled: true). A VR-capable build +// must not try to reach an OpenXR runtime on a normal desktop launch. +inline constexpr bool kDefaultVrEnabled = false; enum class GraphicsBackend { kAuto, diff --git a/src/runtime/runtime_config_bootstrap.cc b/src/runtime/runtime_config_bootstrap.cc index eef6a651..21726ef3 100644 --- a/src/runtime/runtime_config_bootstrap.cc +++ b/src/runtime/runtime_config_bootstrap.cc @@ -72,7 +72,7 @@ device: pc-windows-11 # vsync: off # vr: - # Boolean (default: enabled in VR builds, disabled in non-VR builds). + # Boolean (default: false). Same as launching with --vr. # Experimental VR for verified builds 2998 and 3092. Mocktail initializes # the guest DebugDeviceVR eye resources (two # distinct 500x500 eye framebuffers), the Roblox renderer draws both eyes diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7557cc08..10cb6cf3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -93,7 +93,7 @@ if(MOCKTAIL_ENABLE_VR) else() add_test(NAME OpenXrProbeDisabled COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/vr/probe_contract_test.py" - "$" --disabled "$") + --disabled "$") endif() add_executable(jnivm_test diff --git a/tests/vr/probe_contract_test.py b/tests/vr/probe_contract_test.py index c93f58a8..d84e24ca 100644 --- a/tests/vr/probe_contract_test.py +++ b/tests/vr/probe_contract_test.py @@ -9,19 +9,15 @@ probe = Path(sys.argv[1]) -if sys.argv[2] == "--disabled": - result = subprocess.run([str(probe)], text=True, capture_output=True, timeout=15) - assert result.returncode == 2, (result.returncode, result.stdout, result.stderr) - assert "does not include OpenXR support" in result.stdout, result.stdout - result = subprocess.run([str(probe), "--scene", "--frames", "1"], - text=True, capture_output=True, timeout=15) - assert result.returncode == 1 and "does not include OpenXR support" in result.stdout, result +if sys.argv[1] == "--disabled": + # Non-VR builds install no probe; --vr must still fail clearly and early. + mocktail = sys.argv[2] with tempfile.TemporaryDirectory(prefix="mocktail-xr-disabled-") as directory: root = Path(directory) env = {k: v for k, v in os.environ.items() if not k.startswith(("XR_", "MOCKTAIL_"))} for kind in ("CONFIG", "DATA", "STATE"): env["MOCKTAIL_" + kind + "_ROOT"] = str(root / kind.lower()) - result = subprocess.run([sys.argv[3], "--headless", "--vr"], env=env, + result = subprocess.run([mocktail, "--headless", "--vr"], env=env, text=True, capture_output=True, timeout=15) assert result.returncode == 1 and "does not include OpenXR support" in result.stderr, result assert not (root / "data").exists() and not (root / "state").exists() From bafbe4268c43de4a536b76a72a215c3c38e64f33 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:55:37 -0500 Subject: [PATCH 23/58] vr: name the build and the supported builds when VR is unavailable An explicit --vr on a Roblox build without VR support now exits with the build's version and Build ID, the builds VR supports, and how to select one (--roblox-lib). When VR is only enabled in the config, the launch continues without VR and logs the same details. The bridge's own Install error names both as well. The verified build contracts carry their version labels and move to namespace scope so they can be listed. --- include/mocktail/vr/roblox_vr_device_bridge.h | 5 ++ src/main.cc | 27 +++++++-- src/vr/roblox_vr_device_bridge.cc | 59 ++++++++++++------- src/vr/roblox_vr_device_bridge_internal.h | 2 + tests/vr/roblox_vr_device_bridge_test.cc | 24 ++++++++ 5 files changed, 90 insertions(+), 27 deletions(-) diff --git a/include/mocktail/vr/roblox_vr_device_bridge.h b/include/mocktail/vr/roblox_vr_device_bridge.h index 7d214fa7..5c6fa36e 100644 --- a/include/mocktail/vr/roblox_vr_device_bridge.h +++ b/include/mocktail/vr/roblox_vr_device_bridge.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "compat/build_profile.h" @@ -18,6 +19,10 @@ struct VrBuildContract; const compat::VrDebugDeviceBridgeProfile* FindRobloxVrDeviceProfile( std::string_view build_id); +// Human-readable list of the Roblox builds VR supports, e.g. +// "2998 (2.736.1408) and 3092 (2.738.1397)", for error messages. +std::string DescribeSupportedVrBuilds(); + // Experimental native stereo bring-up for the exact Build-ID-scoped // RBX::Graphics::DebugDeviceVR object of the guest client. // diff --git a/src/main.cc b/src/main.cc index 4d20ded8..7f37dd4c 100644 --- a/src/main.cc +++ b/src/main.cc @@ -815,12 +815,29 @@ int main(int argc, char* argv[]) { compatibility.profile.vr_debug_device_bridge = *vr_profile; } } - vr_active = runtime_config.config.vr_enabled() && - (vr_requested || - compatibility.profile.vr_debug_device_bridge.has_value()); + const bool vr_build_supported = + compatibility.profile.vr_debug_device_bridge.has_value(); + if (runtime_config.config.vr_enabled() && vr_requested && + !vr_build_supported) { + // An explicit --vr on a build without VR support is a user error, not + // something to silently ignore; say which builds work and how to pick. + std::cerr << "[FATAL] VR is not available for Roblox " + << compatibility.profile.version_name << " (version " + << compatibility.profile.version_code << ", Build ID " + << compatibility.build_id << ").\n" + << " VR supports builds " + << mocktail::vr::DescribeSupportedVrBuilds() + << ". Point --roblox-lib at one of those payloads'\n" + << " libroblox.so, or launch without --vr.\n"; + return EXIT_FAILURE; + } + vr_active = runtime_config.config.vr_enabled() && vr_build_supported; if (runtime_config.config.vr_enabled() && !vr_active) { - std::cerr << " [vr] DebugDeviceVR bridge is not profiled for Build ID " - << compatibility.build_id << "; continuing without VR\n"; + std::cerr << " [vr] Roblox " << compatibility.profile.version_name + << " (Build ID " << compatibility.build_id + << ") has no VR support (supported: " + << mocktail::vr::DescribeSupportedVrBuilds() + << "); continuing without VR\n"; if (setenv("MOCKTAIL_VR_ENABLED", "0", 1) != 0) { std::cerr << "[FATAL] Cannot disable VR for this Roblox build\n"; return EXIT_FAILURE; diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc index 9fd05b43..0bfd62e3 100644 --- a/src/vr/roblox_vr_device_bridge.cc +++ b/src/vr/roblox_vr_device_bridge.cc @@ -19,6 +19,7 @@ #include #include #include +#include #include #include "vr/roblox_vr_device_bridge_internal.h" @@ -377,27 +378,30 @@ void RestoreIsVrAppBuildHook() { namespace internal { +constexpr VrBuildContract kContracts[] = { + {"ade08266c67aee88ec9c1d00902150e1684dad3a", + {0x6c87b38, 0x3810614, 0x38109f6, 0x38109e6, 0x3810a18, 0x7369920, + 0x180}, + 0x6b7bfe0, + 0x4d573ee, + 0x263778c, + 0x3a8, + 0x73677a8, + 0x1d48b70, + "2998 (2.736.1408)"}, + {"5f0704edd9064f566ee3d6df2bd2fabbcc709f03", + {0x6d08108, 0x37f9fe4, 0x37fa3c6, 0x37fa3b6, 0x37fa3e8, 0x73f4690, + 0x188}, + 0x6bfb180, + 0x4d97b98, + 0x265cebc, + 0x3a0, + 0x73f24c8, + 0x1d8f8c0, + "3092 (2.738.1397)"}, +}; + const VrBuildContract* FindVrBuildContract(std::string_view build_id) { - static constexpr VrBuildContract kContracts[] = { - {"ade08266c67aee88ec9c1d00902150e1684dad3a", - {0x6c87b38, 0x3810614, 0x38109f6, 0x38109e6, 0x3810a18, 0x7369920, - 0x180}, - 0x6b7bfe0, - 0x4d573ee, - 0x263778c, - 0x3a8, - 0x73677a8, - 0x1d48b70}, - {"5f0704edd9064f566ee3d6df2bd2fabbcc709f03", - {0x6d08108, 0x37f9fe4, 0x37fa3c6, 0x37fa3b6, 0x37fa3e8, 0x73f4690, - 0x188}, - 0x6bfb180, - 0x4d97b98, - 0x265cebc, - 0x3a0, - 0x73f24c8, - 0x1d8f8c0}, - }; for (const auto& contract : kContracts) { if (contract.build_id == build_id) return &contract; } @@ -638,6 +642,16 @@ const compat::VrDebugDeviceBridgeProfile* FindRobloxVrDeviceProfile( return contract == nullptr ? nullptr : &contract->device; } +std::string DescribeSupportedVrBuilds() { + std::string description; + const std::size_t count = std::size(internal::kContracts); + for (std::size_t index = 0; index < count; ++index) { + if (index > 0) description += index + 1 == count ? " and " : ", "; + description += internal::kContracts[index].version; + } + return description; +} + namespace internal { float GetHandPitchOffsetDeg() { @@ -823,8 +837,9 @@ Status RobloxVrDeviceBridge::Install(const compat::BuildProfile &profile) { } const auto* contract = internal::FindVrBuildContract(profile.elf_build_id); if (contract == nullptr) { - return FailedPrecondition( - "VR contracts are not verified for this Build ID"); + return FailedPrecondition("VR is not verified for Roblox Build ID " + + profile.elf_build_id + "; VR supports builds " + + DescribeSupportedVrBuilds()); } const auto& actual = *profile.vr_debug_device_bridge; const auto& expected_profile = contract->device; diff --git a/src/vr/roblox_vr_device_bridge_internal.h b/src/vr/roblox_vr_device_bridge_internal.h index f1e43868..b64690fc 100644 --- a/src/vr/roblox_vr_device_bridge_internal.h +++ b/src/vr/roblox_vr_device_bridge_internal.h @@ -20,6 +20,8 @@ struct VrBuildContract { std::uintptr_t camera_vtable_offset; std::uintptr_t hand_pitch_storage_rva; std::uintptr_t haptic_sink_rva; + // Roblox version code and name, for messages that list supported builds. + std::string_view version; }; const VrBuildContract* FindVrBuildContract(std::string_view build_id); diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc index b354f171..91083f1e 100644 --- a/tests/vr/roblox_vr_device_bridge_test.cc +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -864,3 +864,27 @@ TEST(VrAimPointer, InvalidAimOrScaleKeepsOriginalFrame) { EXPECT_EQ(frame, original); } } + +namespace mocktail::vr { +namespace { + +TEST(VrSupportedBuilds, NamesEveryVerifiedBuild) { + EXPECT_EQ(DescribeSupportedVrBuilds(), + "2998 (2.736.1408) and 3092 (2.738.1397)"); +} + +TEST(VrSupportedBuilds, InstallNamesTheBuildAndTheSupportedOnes) { + compat::BuildProfile profile; + profile.elf_build_id = "a67a91ef96c700c0730acd36a250c2f93cfafbc1"; // 3222 + profile.vr_debug_device_bridge = *FindRobloxVrDeviceProfile( + "ade08266c67aee88ec9c1d00902150e1684dad3a"); + RobloxVrDeviceBridge bridge; + const Status status = bridge.Install(profile); + ASSERT_FALSE(status.ok()); + EXPECT_NE(status.message().find(profile.elf_build_id), std::string::npos); + EXPECT_NE(status.message().find(DescribeSupportedVrBuilds()), + std::string::npos); +} + +} // namespace +} // namespace mocktail::vr From 0b5578d80b72c2ea0d8ec8cff947444b719123e2 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Mon, 12 Oct 2026 23:59:46 -0500 Subject: [PATCH 24/58] vr: keep non-VR launches on their original paths Queue and device-idle locks in the Vulkan adapter are now taken only after an armed OpenXR backend calls mocktail_vulkan_enable_queue_sharing(). The guest synchronizes its own queue use; only the runtime and the eye copies add a second user. Without VR, and in builds without it, submits and presents take no locks. FFlagDebugEnableVREmulator is only written when VR is active, so a user's own fflags.json value is no longer overridden on every launch. Config loading is back after the single-instance lock and the support bundle guard. A non-VR build still rejects --vr before creating any data or state directories, from the command line alone; a vr.enabled config in such a build is rejected once the config is loaded. --- src/graphics/bionic_vulkan_loader_adapter.cc | 89 ++++++++++++++------ src/main.cc | 58 +++++++------ src/runtime/performance_policy.cc | 4 +- src/vr/openxr_backend.cc | 10 +++ tests/performance_policy_test.cc | 12 ++- tests/vr/queue_submission_test.cc | 19 +++++ 6 files changed, 135 insertions(+), 57 deletions(-) diff --git a/src/graphics/bionic_vulkan_loader_adapter.cc b/src/graphics/bionic_vulkan_loader_adapter.cc index e853669c..a7afe06a 100644 --- a/src/graphics/bionic_vulkan_loader_adapter.cc +++ b/src/graphics/bionic_vulkan_loader_adapter.cc @@ -317,8 +317,19 @@ Function ResolveProcessFunction(const char* name) { return reinterpret_cast(dlsym(RTLD_DEFAULT, name)); } -// Queue calls share the gate; device idle is exclusive. -// Release it before fence waits and backend callbacks. +// The guest synchronizes its own queue use. An armed OpenXR backend adds +// another user of the same queue (eye copies, and the runtime inside +// xrEndFrame), so only then must every queue call be serialized: queue calls +// share a per-device gate and a per-queue mutex, and device idle takes the +// gate exclusively. Without VR (or in a build without it) nothing is locked. +// Set by mocktail_vulkan_enable_queue_sharing() when the VR backend arms; +// never cleared, since the runtime may use the queue until the process ends. +std::atomic g_queue_sharing{false}; + +bool QueueSharingActive() { + return g_queue_sharing.load(std::memory_order_acquire); +} + std::shared_ptr DeviceIdleGate(VkDevice device) { static std::mutex registry_mutex; static std::unordered_map> @@ -339,6 +350,41 @@ std::shared_ptr QueueCallMutex(VkQueue queue) { return mutex; } +// Held for one queue call while VR shares the queue; a no-op otherwise. +// Release it before fence waits and backend callbacks. +class QueueCallGuard { +public: + QueueCallGuard(VkDevice device, VkQueue queue) { + if (!QueueSharingActive()) + return; + gate_ = DeviceIdleGate(device); + gate_lock_ = std::shared_lock(*gate_); + mutex_ = QueueCallMutex(queue); + lock_ = std::unique_lock(*mutex_); + } + +private: + std::shared_ptr gate_; + std::shared_lock gate_lock_; + std::shared_ptr mutex_; + std::unique_lock lock_; +}; + +// Exclusive device idle while VR shares the queue; a no-op otherwise. +class DeviceIdleGuard { +public: + explicit DeviceIdleGuard(VkDevice device) { + if (!QueueSharingActive()) + return; + gate_ = DeviceIdleGate(device); + lock_ = std::unique_lock(*gate_); + } + +private: + std::shared_ptr gate_; + std::unique_lock lock_; +}; + void NoteVrSubmitted(VkQueue queue, uint32_t count, const VkSubmitInfo *submits) { static const auto note = ResolveProcessFunction device_gate(*gate); - const auto mutex = QueueCallMutex(queue); - std::lock_guard lock(*mutex); + const QueueCallGuard guard(HostDispatchForQueue(queue).device, queue); return raw(queue, count, submits, fence); } VkResult VKAPI_CALL LockedHostSubmit2KHR(VkQueue queue, uint32_t count, const VkSubmitInfo2 *submits, VkFence fence) { const auto raw = HostDispatchForQueue(queue).queue_submit2_khr; - const auto gate = DeviceIdleGate(HostDispatchForQueue(queue).device); - std::shared_lock device_gate(*gate); - const auto mutex = QueueCallMutex(queue); - std::lock_guard lock(*mutex); + const QueueCallGuard guard(HostDispatchForQueue(queue).device, queue); return raw(queue, count, submits, fence); } VkResult VKAPI_CALL LockedHostBindSparse(VkQueue queue, uint32_t count, const VkBindSparseInfo *binds, VkFence fence) { const auto raw = HostDispatchForQueue(queue).queue_bind_sparse; - const auto gate = DeviceIdleGate(HostDispatchForQueue(queue).device); - std::shared_lock device_gate(*gate); - const auto mutex = QueueCallMutex(queue); - std::lock_guard lock(*mutex); + const QueueCallGuard guard(HostDispatchForQueue(queue).device, queue); return raw(queue, count, binds, fence); } VkResult VKAPI_CALL LockedHostQueueWaitIdle(VkQueue queue) { const auto raw = HostDispatchForQueue(queue).queue_wait_idle; - const auto gate = DeviceIdleGate(HostDispatchForQueue(queue).device); - std::shared_lock device_gate(*gate); - const auto mutex = QueueCallMutex(queue); - std::lock_guard lock(*mutex); + const QueueCallGuard guard(HostDispatchForQueue(queue).device, queue); return raw(queue); } @@ -1457,10 +1491,7 @@ ObservedHostQueuePresent(VkQueue queue, const VkPresentInfoKHR* present_info) { const bool fps_trace = FpsTraceEnabled(); const std::uint64_t present_start_ns = fps_trace ? MonotonicNanos() : 0; const VkResult result = [&] { - const auto gate = DeviceIdleGate(HostDispatchForQueue(queue).device); - std::shared_lock device_gate(*gate); - const auto mutex = QueueCallMutex(queue); - std::lock_guard lock(*mutex); + const QueueCallGuard guard(HostDispatchForQueue(queue).device, queue); mocktail::graphics::TraceScope scope( mocktail::graphics::ActiveProfileTrace(), "host present", "present"); return host_present(queue, present_info); @@ -3472,15 +3503,18 @@ VKAPI_ATTR void VKAPI_CALL vkCmdPipelineBarrier2KHR( ForwardVrBarrier2(command, info, "vkCmdPipelineBarrier2KHR"); } +// Called once by the VR backend when it arms, before Roblox creates its +// device: from then on queue calls are serialized (see QueueCallGuard). +void mocktail_vulkan_enable_queue_sharing() { + g_queue_sharing.store(true, std::memory_order_release); +} + VKAPI_ATTR VkResult VKAPI_CALL mocktail_vulkan_submit_synchronized( PFN_vkQueueSubmit raw, VkDevice device, VkQueue queue, uint32_t count, const VkSubmitInfo *submits, VkFence fence) { if (!raw) return VK_ERROR_INITIALIZATION_FAILED; - const auto gate = DeviceIdleGate(device); - std::shared_lock device_gate(*gate); - const auto mutex = QueueCallMutex(queue); - std::lock_guard lock(*mutex); + const QueueCallGuard guard(device, queue); return raw(queue, count, submits, fence); } @@ -3488,8 +3522,7 @@ VKAPI_ATTR VkResult VKAPI_CALL mocktail_vulkan_idle_synchronized(PFN_vkDeviceWaitIdle raw, VkDevice device) { if (!raw) return VK_ERROR_INITIALIZATION_FAILED; - const auto gate = DeviceIdleGate(device); - std::unique_lock device_gate(*gate); + const DeviceIdleGuard guard(device); return raw(device); } diff --git a/src/main.cc b/src/main.cc index 7f37dd4c..2d89487b 100644 --- a/src/main.cc +++ b/src/main.cc @@ -329,31 +329,10 @@ int main(int argc, char* argv[]) { mocktail::runtime::ScrubCommandLineLaunchArguments(&command_line.options, argc, argv); - const mocktail::runtime::RuntimeConfigBootstrapResult config_bootstrap = - mocktail::runtime::EnsureRuntimeConfigFile(paths.config_file()); - if (!config_bootstrap) { - std::cerr << "[FATAL] Cannot prepare " << paths.config_file() << ": " - << config_bootstrap.error << '\n'; - if (command_line.options.mode == mocktail::runtime::CommandMode::kRun) { - (void)mocktail::runtime::ShowFailureDialog( - environment, "Mocktail could not prepare its configuration."); - } - return EXIT_FAILURE; - } - mocktail::runtime::RuntimeConfigLoadResult runtime_config = - mocktail::runtime::LoadRuntimeConfig(environment, paths.config_file()); - if (!runtime_config) { - std::cerr << "[FATAL] Cannot load " << paths.config_file() << ": " - << runtime_config.error << '\n'; - if (command_line.options.mode == mocktail::runtime::CommandMode::kRun) { - (void)mocktail::runtime::ShowFailureDialog( - environment, "Mocktail could not load its configuration."); - } - return EXIT_FAILURE; - } - // The standalone OpenXR scene belongs to mocktail-vr-probe --scene. - // --vr continues through Roblox startup as a native stereo prototype. - if (runtime_config.config.vr_enabled() && + // Reject --vr in a build without OpenXR before touching any data or state + // directories. A vr.enabled config is checked once the config is loaded. + const std::string vr_request = environment.GetOr("MOCKTAIL_VR_ENABLED", ""); + if ((vr_request == "1" || vr_request == "true" || vr_request == "on") && !mocktail::vr::IsOpenXrSupportCompiled()) { std::cerr << " [vr] This build does not include OpenXR support. " "Use a VR-enabled build of Mocktail; refusing to start " @@ -407,6 +386,35 @@ int main(int argc, char* argv[]) { return EXIT_FAILURE; } } + const mocktail::runtime::RuntimeConfigBootstrapResult config_bootstrap = + mocktail::runtime::EnsureRuntimeConfigFile(paths.config_file()); + if (!config_bootstrap) { + std::cerr << "[FATAL] Cannot prepare " << paths.config_file() << ": " + << config_bootstrap.error << '\n'; + if (command_line.options.mode == mocktail::runtime::CommandMode::kRun) { + (void)mocktail::runtime::ShowFailureDialog( + environment, "Mocktail could not prepare its configuration."); + } + return EXIT_FAILURE; + } + mocktail::runtime::RuntimeConfigLoadResult runtime_config = + mocktail::runtime::LoadRuntimeConfig(environment, paths.config_file()); + if (!runtime_config) { + std::cerr << "[FATAL] Cannot load " << paths.config_file() << ": " + << runtime_config.error << '\n'; + if (command_line.options.mode == mocktail::runtime::CommandMode::kRun) { + (void)mocktail::runtime::ShowFailureDialog( + environment, "Mocktail could not load its configuration."); + } + return EXIT_FAILURE; + } + if (runtime_config.config.vr_enabled() && + !mocktail::vr::IsOpenXrSupportCompiled()) { + std::cerr << " [vr] This build does not include OpenXR support, but " + "vr.enabled is set; disable it in " << paths.config_file() + << ".\n"; + return EXIT_FAILURE; + } if (runtime_config.config.use_system_proxy()) { const mocktail::runtime::SystemProxyResult system_proxy = mocktail::runtime::ResolveSystemProxy(); diff --git a/src/runtime/performance_policy.cc b/src/runtime/performance_policy.cc index db572812..6bbd948a 100644 --- a/src/runtime/performance_policy.cc +++ b/src/runtime/performance_policy.cc @@ -392,8 +392,10 @@ bool MergeVrClientSettingsOverrides(bool enabled, std::string_view base_json, } return false; } - overrides["FFlagDebugEnableVREmulator"] = enabled ? "True" : "False"; + // Without VR the flags are left exactly as the user (fflags.json) set + // them; only an active VR session forces the emulator on. if (enabled) { + overrides["FFlagDebugEnableVREmulator"] = "True"; // Roblox 2998's SurfaceController startup throttle renders at 1 Hz while // waiting for AppStartRendering. Our XR frame loop is driven by guest // presents, so that also stalls head/controller updates in the VR menu. diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 6b32f702..69d91a08 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -31,6 +31,9 @@ #define M_PI 3.14159265358979323846 #endif +// Defined by the Vulkan adapter (bionic_vulkan_loader_adapter.cc). +extern "C" __attribute__((weak)) void mocktail_vulkan_enable_queue_sharing(); + namespace mocktail::vr { namespace { @@ -479,6 +482,13 @@ Status OpenXrBackend::Arm(VrGraphicsApi graphics_api) { throw std::runtime_error(err); } armed_.store(true, std::memory_order_release); + // From here the runtime and the eye copies share the guest's queue, so + // the Vulkan adapter must serialize queue calls. Armed before Roblox + // creates its device, so no queue call ever started unlocked. Weak: the + // adapter is absent from VR-only test binaries. + if (mocktail_vulkan_enable_queue_sharing != nullptr) { + mocktail_vulkan_enable_queue_sharing(); + } Log(" [vr-backend] armed: OpenXR instance/system ready; awaiting guest " "%s\n", graphics_api_ == VrGraphicsApi::kVulkan ? "Vulkan device" diff --git a/tests/performance_policy_test.cc b/tests/performance_policy_test.cc index d3dd3dc8..8e48df68 100644 --- a/tests/performance_policy_test.cc +++ b/tests/performance_policy_test.cc @@ -358,9 +358,15 @@ TEST(PerformancePolicyTest, EXPECT_EQ(json["FFlagSlowDownRendering"], "False"); EXPECT_EQ(json["FFlagFixVRAvatarPlacement"], "True"); EXPECT_EQ(json["FIntUnrelated"], "42"); - ASSERT_TRUE(MergeVrClientSettingsOverrides(false, merged, &merged, &error)); - EXPECT_EQ(nlohmann::json::parse(merged)["FFlagDebugEnableVREmulator"], - "False"); + // Without VR the user's own value is kept, and nothing is added. + std::string kept; + ASSERT_TRUE(MergeVrClientSettingsOverrides( + false, R"({"FFlagDebugEnableVREmulator":"True","FIntUnrelated":"42"})", + &kept, &error)); + EXPECT_EQ(nlohmann::json::parse(kept)["FFlagDebugEnableVREmulator"], "True"); + ASSERT_TRUE(MergeVrClientSettingsOverrides(false, "{}", &kept, &error)); + EXPECT_FALSE(nlohmann::json::parse(kept).contains( + "FFlagDebugEnableVREmulator")); EXPECT_FALSE(MergeVrClientSettingsOverrides(true, "[]", &merged, &error)); EXPECT_FALSE(MergeVrClientSettingsOverrides(true, "bad", &merged, &error)); } diff --git a/tests/vr/queue_submission_test.cc b/tests/vr/queue_submission_test.cc index fe4b5457..5bbc6da0 100644 --- a/tests/vr/queue_submission_test.cc +++ b/tests/vr/queue_submission_test.cc @@ -17,6 +17,7 @@ extern "C" VKAPI_ATTR VkResult VKAPI_CALL mocktail_vulkan_submit_synchronized( extern "C" void mocktail_vr_xr_queue_submit_adapter(SubmitAdapter); extern "C" VKAPI_ATTR VkResult VKAPI_CALL mocktail_vulkan_idle_synchronized(PFN_vkDeviceWaitIdle raw, VkDevice device); +extern "C" void mocktail_vulkan_enable_queue_sharing(); namespace { std::mutex state_mutex; @@ -67,6 +68,7 @@ VKAPI_ATTR VkResult VKAPI_CALL BlockingIdle(VkDevice) { : VK_TIMEOUT; } TEST(VrQueueSubmission, DeviceIdleExcludesPrivateSubmitsUntilDriverReturns) { + mocktail_vulkan_enable_queue_sharing(); // an armed VR backend does this waiting = signal_submitted = idle_release = false; VkResult idle_result = VK_NOT_READY; std::thread idler([&] { @@ -164,7 +166,24 @@ TEST(VrQueueSubmission, AnotherQueueCanSignalWhilePresentWaitsForFence) { EXPECT_TRUE(entered); EXPECT_EQ(prepared, VK_SUCCESS); } +TEST(VrQueueSubmission, GuestCallsAreNotSerializedWithoutVr) { + // ctest runs each test in its own process, so sharing starts disabled here: + // without an armed VR backend the guest owns its queue synchronization. + inside_driver = max_inside = 0; + const auto queue = reinterpret_cast(30); + auto call = [&] { + EXPECT_EQ(mocktail_vulkan_submit_synchronized( + SlowSubmit, reinterpret_cast(2), queue, 0, nullptr, + VK_NULL_HANDLE), + VK_SUCCESS); + }; + std::thread first(call), second(call); + first.join(); + second.join(); + EXPECT_EQ(max_inside, 2); +} TEST(VrQueueSubmission, PrivateAndGuestCallsCannotEnterOneQueueConcurrently) { + mocktail_vulkan_enable_queue_sharing(); inside_driver = max_inside = 0; const auto queue = reinterpret_cast(30); auto call = [&] { From 0603734ff6e4e6095f1027a2d7cef5f7e6c2374f Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:04:42 -0500 Subject: [PATCH 25/58] vr: recover from XR failures instead of giving up on headset output - Every frame error, including the swapchain-image and copy-fence timeouts, now tears the session down recoverably instead of ending XR output for the rest of the run. - Session recovery backs off (1, 2, 4, 8 s) and recreates the XrInstance after an instance-loss event, an instance/runtime failure, or three failed attempts, so VR returns after SteamVR or WiVRn restarts instead of retrying forever. - While no XR session runs (headset idle, session not READY, recovering) presents are paced at 60 Hz outside the backend lock instead of running unthrottled. - A PrepareHostPresent failure that consumed nothing (copy resources unavailable, failed fence reset or submit) presents normally instead of failing the guest's vkQueuePresentKHR; only device loss is propagated. - A runtime/guest GPU mismatch falls back to the plain host device and keeps the game running without headset output, naming both GPUs. - A failed EGL attach on the GLES path no longer makes window init fail; the game runs without headset output and the bind is not retried. --- include/mocktail/vr/openxr_backend.h | 13 ++- src/vr/openxr_backend.cc | 141 ++++++++++++++++++++------- tests/vr/openxr_backend_test.cc | 6 ++ 3 files changed, 124 insertions(+), 36 deletions(-) diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index c044892c..9f31587b 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -22,7 +22,9 @@ namespace mocktail::vr { namespace internal { // Consume binary present dependencies exactly once. On success a later present -// must omit them; on failure the caller must propagate the error, not retry. +// must omit them. VK_NOT_READY means nothing was consumed and the guest's +// present proceeds unchanged; any other failure consumed them (or lost the +// device) and must be propagated, not retried. VkResult WaitForPresentDependencies(VkDevice device, VkQueue queue, const VkPresentInfoKHR& present, VkFence fence, PFN_vkResetFences reset, @@ -302,6 +304,9 @@ class OpenXrBackend final { // image paired with a pose that no longer describes the same origin. void InvalidateInFlightPose(const char* reason); void RunFrameCycle(VkQueue queue, VkDevice device); + // Returns true when the backend owns this present but ran no XR frame, so + // the caller paces the guest itself. + bool NoteHostPresentLocked(VkQueue queue, VkDevice device); bool CreateSwapchains(std::string* error); void DestroySwapchains(); bool EnsureCopyResources(VkDevice device, std::uint32_t queue_family, @@ -424,6 +429,12 @@ class OpenXrBackend final { bool session_recovery_pending_ = false; bool test_session_loss_injected_ = false; std::uint64_t next_session_retry_ns_ = 0; + // Consecutive failed session recoveries, for backoff and for deciding when + // the XrInstance itself must be recreated (e.g. after a runtime restart). + std::uint32_t session_recovery_failures_ = 0; + bool xr_instance_stale_ = false; + bool present_sync_failure_logged_ = false; + bool gles_attach_failed_ = false; std::uint64_t frame_display_time_ = 0; // MOCKTAIL_VR_UNPACED benchmark mode: guest presents before this monotonic // time leave the open XR frame alone instead of blocking in xrWaitFrame. diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 69d91a08..59f9eede 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include #ifndef M_PI @@ -634,11 +635,26 @@ bool OpenXrBackend::CreateVulkanDevice(VkPhysicalDevice physical_device, &device_get_info, &runtime_device), "xrGetVulkanGraphicsDevice2KHR"); if (runtime_device != physical_device) { - *device = VK_NULL_HANDLE; - *result = VK_ERROR_INCOMPATIBLE_DRIVER; - Log(" [vr-backend] refusing a guest GPU different from the OpenXR " - "GPU\n"); - return true; + // The runtime can only share images on its own GPU. Using it for + // Roblox is not possible here (Roblox already chose its device, queue + // families and memory types), so let the plain loader create Roblox's + // device and keep playing without headset output. + const auto properties_of = + reinterpret_cast( + loader_gipa(vk_instance_, "vkGetPhysicalDeviceProperties")); + VkPhysicalDeviceProperties guest_gpu{}, runtime_gpu{}; + if (properties_of != nullptr) { + properties_of(physical_device, &guest_gpu); + if (runtime_device != VK_NULL_HANDLE) + properties_of(runtime_device, &runtime_gpu); + } + Log(" [vr-backend] XR OUTPUT UNAVAILABLE: Roblox renders on '%s' but " + "the OpenXR runtime uses '%s'. Continuing without headset output; " + "select the runtime's GPU for Roblox (e.g. DRI_PRIME or " + "MESA_VK_DEVICE_SELECT) to use VR.\n", + guest_gpu.deviceName[0] ? guest_gpu.deviceName : "unknown GPU", + runtime_gpu.deviceName[0] ? runtime_gpu.deviceName : "unknown GPU"); + return false; } auto xr_create = reinterpret_cast( @@ -835,7 +851,8 @@ Status OpenXrBackend::AttachGlesContext(void *display, void *config, void *context, void *egl_get_proc, void *(*resolve)(const char *)) { std::lock_guard lock(mutex_); - if (!armed_ || graphics_api_ != VrGraphicsApi::kOpenGles) + if (!armed_ || graphics_api_ != VrGraphicsApi::kOpenGles || + gles_attach_failed_) return Status::Ok(); try { if (gles_) { @@ -865,6 +882,8 @@ Status OpenXrBackend::AttachGlesContext(void *display, void *config, gles_->Destroy(); delete gles_; gles_ = nullptr; + // Every later MakeCurrent would retry the same failing bind; stay inert. + gles_attach_failed_ = true; return Status::Error(StatusCode::kUnavailable, error.what()); } } @@ -1794,8 +1813,10 @@ void OpenXrBackend::PollSessionEvents() { break; } if (event.type == XR_TYPE_EVENT_DATA_INSTANCE_LOSS_PENDING) { - Log(" [vr-backend] OpenXR instance loss pending; XR output stopped\n"); - TeardownSessionLocked("instance loss pending"); + Log(" [vr-backend] OpenXR instance loss pending; recreating it once " + "the runtime is back\n"); + xr_instance_stale_ = true; + TeardownSessionLocked("instance loss pending", true); return; } if (event.type == XR_TYPE_EVENT_DATA_REFERENCE_SPACE_CHANGE_PENDING) { @@ -1953,19 +1974,38 @@ bool OpenXrBackend::RecoverSessionLocked(std::uint64_t now_ns) { (graphics_api_ == VrGraphicsApi::kVulkan ? (vk_device_ == VK_NULL_HANDLE || vk_ == nullptr) : gles_ == nullptr) || - xr_instance_ == nullptr || now_ns < next_session_retry_ns_) + now_ns < next_session_retry_ns_) return false; - next_session_retry_ns_ = now_ns + 1'000'000'000ULL; + // Back off 1, 2, 4 then 8 s so a runtime that stays down costs nothing. + const std::uint32_t shift = std::min(session_recovery_failures_, 3); + next_session_retry_ns_ = now_ns + (1'000'000'000ULL << shift); std::string error; + // A restarted runtime (SteamVR, WiVRn) invalidates the old XrInstance; a + // session can never be created on it again. Recreate the instance when the + // runtime said so, or when recovery keeps failing on the current one. + if (xr_instance_stale_ || session_recovery_failures_ >= 3 || + xr_instance_ == nullptr) { + DestroyXrInstanceLocked(); + if (!EnsureXrInstanceLocked(&error)) { + ++session_recovery_failures_; + Log(" [vr-backend] waiting for the OpenXR runtime: %s\n", + error.c_str()); + return false; + } + xr_instance_stale_ = false; + Log(" [vr-backend] OpenXR instance recreated for session recovery\n"); + } // nullptr means reuse the verified queue family of this existing VkDevice. if (!(graphics_api_ == VrGraphicsApi::kVulkan ? InitializeSessionLocked(vk_physical_device_, vk_device_, nullptr, &error) : InitializeGlesSessionLocked(&error))) { + ++session_recovery_failures_; Log(" [vr-backend] waiting to restore XR session: %s\n", error.c_str()); return false; } session_recovery_pending_ = false; + session_recovery_failures_ = 0; Log(" [vr-backend] XR session recreated on the existing guest device; " "awaiting READY\n"); return true; @@ -1988,12 +2028,15 @@ VkResult internal::WaitForPresentDependencies(VkDevice device, VkQueue queue, info.waitSemaphoreCount = present.waitSemaphoreCount; info.pWaitSemaphores = present.pWaitSemaphores; info.pWaitDstStageMask = stages.data(); + // Before the submit succeeds nothing is consumed (a failed vkQueueSubmit + // leaves its semaphores unaffected), so the guest's own present can still + // wait on them: VK_NOT_READY. Device loss stays an error either way. VkResult result = reset(device, 1, &fence); if (result != VK_SUCCESS) - return result; + return result == VK_ERROR_DEVICE_LOST ? result : VK_NOT_READY; result = SubmitHostQueue(submit, device, queue, 1, &info, fence); if (result != VK_SUCCESS) - return result; + return result == VK_ERROR_DEVICE_LOST ? result : VK_NOT_READY; // The completed fence permits presentation without repeating these waits. result = wait(device, 1, &fence, VK_TRUE, UINT64_MAX); // A consumed binary semaphore cannot be retried after a timeout. @@ -2044,9 +2087,14 @@ VkResult OpenXrBackend::PrepareHostPresent(VkQueue queue, VkDevice device, return VK_SUCCESS; std::string error; if (!EnsureCopyResources(device, vk_queue_family_, &error)) { - Log(" [vr-backend] cannot synchronize present dependencies: %s\n", - error.c_str()); - return VK_ERROR_INITIALIZATION_FAILED; + // Nothing consumed yet: present normally and skip VR for this frame. + if (!present_sync_failure_logged_) { + present_sync_failure_logged_ = true; + Log(" [vr-backend] cannot synchronize present dependencies: %s; " + "presenting without XR this frame\n", + error.c_str()); + } + return VK_NOT_READY; } return internal::WaitForPresentDependencies( device, GraphicsQueueForPresentLocked(queue), *info, copy_fence_, @@ -2054,21 +2102,38 @@ VkResult OpenXrBackend::PrepareHostPresent(VkQueue queue, VkDevice device, } void OpenXrBackend::NoteHostPresent(VkQueue queue, VkDevice device) { - std::lock_guard lock(mutex_); + bool idle = false; + { + std::lock_guard lock(mutex_); + idle = NoteHostPresentLocked(queue, device); + } + if (idle) { + // No running XR session (headset off or idle, session not READY, or + // recovering): nothing paces the guest, and with VR the desktop present + // is unthrottled. Cap it at 60 Hz, outside the backend lock. + constexpr auto kIdleFrame = std::chrono::nanoseconds(16'666'667); + static thread_local auto next = std::chrono::steady_clock::now(); + const auto now = std::chrono::steady_clock::now(); + if (next > now) std::this_thread::sleep_until(next); + next = std::max(next, now) + kIdleFrame; + } +} + +bool OpenXrBackend::NoteHostPresentLocked(VkQueue queue, VkDevice device) { if (!armed_.load(std::memory_order_acquire) || device != vk_device_) { - return; + return false; } if (graphics_api_ == VrGraphicsApi::kVulkan && !SupportsPresentQueueLocked(queue)) - return; + return false; if (graphics_api_ == VrGraphicsApi::kVulkan) queue = GraphicsQueueForPresentLocked(queue); try { if (xr_session_ == nullptr && !RecoverSessionLocked(perf::NowNs())) - return; + return true; PollSessionEvents(); if (!session_running_.load(std::memory_order_acquire)) { - return; + return true; } perf::ProcessCollector().Count(perf::Counter::kPresents); RunFrameCycle(queue, device); @@ -2085,23 +2150,26 @@ void OpenXrBackend::NoteHostPresent(VkQueue queue, VkDevice device) { HandleSessionState(XR_SESSION_STATE_LOSS_PENDING); } perf::ProcessCollector().EmitIfDue(perf::NowNs(), "xr-frame-cycle"); + return false; } catch (const XrCallError &error) { - if (error.result == XR_ERROR_SESSION_LOST) { - Log(" [vr-backend] XR session lost; scheduling recovery on the existing " - "device\n"); - TeardownSessionLocked("XR_ERROR_SESSION_LOST", true); - } else { - Log(" [vr-backend] XR frame failed: %s; restart the client if the " - "runtime instance was lost\n", - error.what()); - TeardownSessionLocked("unrecoverable XR frame error"); - } + // Every frame error is retried on the existing guest device with backoff; + // a lost or failed runtime additionally gets a fresh XrInstance. + if (error.result == XR_ERROR_INSTANCE_LOST || + error.result == XR_ERROR_RUNTIME_FAILURE || + error.result == XR_ERROR_RUNTIME_UNAVAILABLE) { + xr_instance_stale_ = true; + } + Log(" [vr-backend] XR frame failed: %s; recovering the session\n", + error.what()); + TeardownSessionLocked("XR frame error", true); } catch (const std::exception &error) { - Log(" [vr-backend] frame cycle failed: %s; XR output stopped, native " - "stereo continues\n", + // Includes the swapchain-image and copy-fence timeouts: transient stalls + // (a hitching compositor) must not end VR for the rest of the session. + Log(" [vr-backend] frame cycle failed: %s; recovering the session\n", error.what()); - TeardownSessionLocked("frame cycle failure"); + TeardownSessionLocked("frame cycle failure", true); } + return true; } void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { @@ -3710,9 +3778,12 @@ extern "C" bool mocktail_vr_attach_egl(void *display, void *config, if (auto *backend = mocktail::vr::ActiveVrBackend()) { const auto status = backend->AttachGlesContext(display, config, context, egl_get_proc, resolve); + // Headset output is optional: the game still runs on the desktop window. if (!status.ok()) - std::fprintf(stderr, " [vr-backend] %s\n", status.message().c_str()); - return status.ok(); + std::fprintf(stderr, + " [vr-backend] XR OUTPUT UNAVAILABLE: %s. Continuing " + "without headset output.\n", + status.message().c_str()); } return true; } diff --git a/tests/vr/openxr_backend_test.cc b/tests/vr/openxr_backend_test.cc index d9da14a6..65ad4e6a 100644 --- a/tests/vr/openxr_backend_test.cc +++ b/tests/vr/openxr_backend_test.cc @@ -327,6 +327,12 @@ TEST_F(OpenXrPresentDependencies, SubmitFailureDoesNotWaitForUnsignalledFence) { EXPECT_EQ(submits, 1); EXPECT_EQ(waits, 0); } +TEST_F(OpenXrPresentDependencies, FailedSubmitLeavesThePresentToTheGuest) { + submit_result = VK_ERROR_OUT_OF_HOST_MEMORY; + EXPECT_EQ(Run(), VK_NOT_READY); + EXPECT_EQ(submits, 1); + EXPECT_EQ(waits, 0); +} TEST_F(OpenXrPresentDependencies, ConsumedWaitsNeverAuthorizeFallbackOnTimeout) { wait_result = VK_TIMEOUT; From 8106203c8e490085d60a5e72a8c1a9668aa587b1 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:08:22 -0500 Subject: [PATCH 26/58] vr: fix eye colour formats, mirror segment safety and frame-wait locking - Once the guest eye format is known, the eye swapchains are recreated once in the format that keeps its bytes: the eye format, else its UNORM/SRGB sibling (raw copy), else the other channel order with the same sRGB-ness (a blit that only swizzles). Mixed order-and-gamma cases no longer go through a gamma-converting blit; if only that remains, it is logged. - The desktop mirror segment is created with O_EXCL after removing our own stale segment (a crashed run's leftover), and must be private to this user before eye images are written to it. - A new frame_mutex_ is held for the whole XR frame cycle and by every path that can destroy the session (taken before mutex_). That lets the cycle release mutex_ while it blocks in xrWaitFrame, so guest render-thread hooks no longer wait out the runtime's pacing. Swapchain-image and fence waits stay under mutex_; they return promptly unless the GPU or compositor stalls, and they run while guest eye bindings are copied. --- include/mocktail/vr/openxr_backend.h | 21 +++++ src/vr/openxr_backend.cc | 112 ++++++++++++++++++++++++++- src/vr/vr_mirror.cc | 21 ++++- tests/vr/openxr_backend_test.cc | 41 ++++++++++ tests/vr/vr_mirror_test.cc | 29 +++++++ 5 files changed, 219 insertions(+), 5 deletions(-) diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index 9f31587b..60ddbbec 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -21,6 +21,13 @@ namespace mocktail::vr { namespace internal { +// Swapchain format for guest eyes of `eye` given the runtime's formats: +// `eye` itself, else its UNORM/SRGB sibling (a raw byte copy, since the guest +// writes sRGB-encoded bytes either way), else the other channel order with +// the same sRGB-ness (a blit that only swizzles). VK_FORMAT_UNDEFINED when +// only a gamma-converting blit remains. +VkFormat PreferredEyeSwapchainFormat(VkFormat eye, + const std::vector& available); // Consume binary present dependencies exactly once. On success a later present // must omit them. VK_NOT_READY means nothing was consumed and the guest's // present proceeds unchanged; any other failure consumed them (or lost the @@ -308,6 +315,9 @@ class OpenXrBackend final { // the caller paces the guest itself. bool NoteHostPresentLocked(VkQueue queue, VkDevice device); bool CreateSwapchains(std::string* error); + // Recreates the eye swapchains once in the format that copies the guest's + // eye bytes without a gamma change (see PreferredEyeSwapchainFormat). + void MatchSwapchainFormatToEyesLocked(); void DestroySwapchains(); bool EnsureCopyResources(VkDevice device, std::uint32_t queue_family, std::string* error); @@ -334,7 +344,13 @@ class OpenXrBackend final { std::uint64_t gl_min_version_ = 0; std::uint64_t gl_max_version_ = 0; + // Held for a whole XR frame cycle and by every path that can destroy the + // session; always taken before mutex_. + std::mutex frame_mutex_; mutable std::mutex mutex_; + // The frame cycle's hold on mutex_, so it can release it around + // xrWaitFrame. Only set while NoteHostPresent runs the cycle. + std::unique_lock* state_lock_ = nullptr; std::atomic armed_{false}; std::atomic recording_{false}; std::atomic eyes_bound_{false}; @@ -435,6 +451,11 @@ class OpenXrBackend final { bool xr_instance_stale_ = false; bool present_sync_failure_logged_ = false; bool gles_attach_failed_ = false; + // Runtime swapchain formats, and the format chosen once the guest eye + // format is known (VK_FORMAT_UNDEFINED until then). + std::vector xr_swapchain_formats_; + VkFormat eye_swapchain_format_ = VK_FORMAT_UNDEFINED; + VkFormat eye_format_checked_ = VK_FORMAT_UNDEFINED; std::uint64_t frame_display_time_ = 0; // MOCKTAIL_VR_UNPACED benchmark mode: guest presents before this monotonic // time leave the open XR frame alone instead of blocking in xrWaitFrame. diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 59f9eede..9255965c 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -161,6 +161,26 @@ bool FormatIsCopyCompatible(VkFormat a, VkFormat b) { return false; } +VkFormat OtherChannelOrder(VkFormat format) { + switch (format) { + case VK_FORMAT_R8G8B8A8_UNORM: return VK_FORMAT_B8G8R8A8_UNORM; + case VK_FORMAT_R8G8B8A8_SRGB: return VK_FORMAT_B8G8R8A8_SRGB; + case VK_FORMAT_B8G8R8A8_UNORM: return VK_FORMAT_R8G8B8A8_UNORM; + case VK_FORMAT_B8G8R8A8_SRGB: return VK_FORMAT_R8G8B8A8_SRGB; + default: return VK_FORMAT_UNDEFINED; + } +} + +VkFormat SrgbSibling(VkFormat format) { + switch (format) { + case VK_FORMAT_R8G8B8A8_UNORM: return VK_FORMAT_R8G8B8A8_SRGB; + case VK_FORMAT_R8G8B8A8_SRGB: return VK_FORMAT_R8G8B8A8_UNORM; + case VK_FORMAT_B8G8R8A8_UNORM: return VK_FORMAT_B8G8R8A8_SRGB; + case VK_FORMAT_B8G8R8A8_SRGB: return VK_FORMAT_B8G8R8A8_UNORM; + default: return VK_FORMAT_UNDEFINED; + } +} + OpenXrBackend* g_backend = nullptr; std::mutex g_backend_slot_mutex; @@ -502,6 +522,8 @@ Status OpenXrBackend::Arm(VrGraphicsApi graphics_api) { } void OpenXrBackend::Disarm() { + // Lock order: frame_mutex_, then mutex_ (see NoteHostPresent). + std::lock_guard frame(frame_mutex_); std::lock_guard lock(mutex_); DisarmLocked(); } @@ -554,6 +576,8 @@ bool OpenXrBackend::CreateVulkanInstance( const VkInstanceCreateInfo* create_info, const VkAllocationCallbacks* allocator, VkInstance* instance, VkResult* result) { + // Lock order: frame_mutex_, then mutex_ (see NoteHostPresent). + std::lock_guard frame(frame_mutex_); std::lock_guard lock(mutex_); if (!armed_.load(std::memory_order_acquire) || vk_loader_ == nullptr) { return false; @@ -612,6 +636,8 @@ bool OpenXrBackend::CreateVulkanDevice(VkPhysicalDevice physical_device, const VkDeviceCreateInfo* create_info, const VkAllocationCallbacks* allocator, VkDevice* device, VkResult* result) { + // Lock order: frame_mutex_, then mutex_ (see NoteHostPresent). + std::lock_guard frame(frame_mutex_); std::lock_guard lock(mutex_); if (!armed_.load(std::memory_order_acquire) || xr_instance_ == nullptr || vk_instance_ == VK_NULL_HANDLE || vk_loader_ == nullptr) { @@ -850,6 +876,8 @@ bool OpenXrBackend::InitializeSessionLocked( Status OpenXrBackend::AttachGlesContext(void *display, void *config, void *context, void *egl_get_proc, void *(*resolve)(const char *)) { + // Lock order: frame_mutex_, then mutex_ (see NoteHostPresent). + std::lock_guard frame(frame_mutex_); std::lock_guard lock(mutex_); if (!armed_ || graphics_api_ != VrGraphicsApi::kOpenGles || gles_attach_failed_) @@ -918,6 +946,8 @@ bool OpenXrBackend::InitializeGlesSessionLocked(std::string *error) { } void OpenXrBackend::DetachGlesContext(void *context) { + // Lock order: frame_mutex_, then mutex_ (see NoteHostPresent). + std::lock_guard frame(frame_mutex_); std::lock_guard lock(mutex_); if (!gles_ || gles_->context() != context) return; @@ -1146,7 +1176,14 @@ bool OpenXrBackend::CreateSwapchains(std::string* error) { // The swapchain format must accept the guest eye content. Prefer the // common 8-bit formats; the final choice is cross-checked against the // recorded eye images before the first copy (blit converts if needed). + xr_swapchain_formats_ = formats; std::int64_t chosen = 0; + if (eye_swapchain_format_ != VK_FORMAT_UNDEFINED && + std::find(formats.begin(), formats.end(), + static_cast(eye_swapchain_format_)) != + formats.end()) { + chosen = eye_swapchain_format_; + } const std::vector candidates = graphics_api_ == VrGraphicsApi::kVulkan ? std::vector{VK_FORMAT_R8G8B8A8_UNORM, @@ -1155,6 +1192,7 @@ bool OpenXrBackend::CreateSwapchains(std::string* error) { VK_FORMAT_B8G8R8A8_SRGB} : std::vector{GL_RGBA8, GL_SRGB8_ALPHA8}; for (const auto candidate : candidates) { + if (chosen != 0) break; if (std::find(formats.begin(), formats.end(), static_cast(candidate)) != formats.end()) { chosen = candidate; @@ -1238,6 +1276,38 @@ bool OpenXrBackend::CreateSwapchains(std::string* error) { } } +void OpenXrBackend::MatchSwapchainFormatToEyesLocked() { + const VkFormat eye = eyes_[0].format; + if (graphics_api_ != VrGraphicsApi::kVulkan || eye == VK_FORMAT_UNDEFINED || + eye == eye_format_checked_) { + return; + } + eye_format_checked_ = eye; + const VkFormat current = eye_swapchains_[0].format; + const VkFormat preferred = + internal::PreferredEyeSwapchainFormat(eye, xr_swapchain_formats_); + if (preferred == VK_FORMAT_UNDEFINED) { + if (!FormatIsCopyCompatible(eye, current)) { + Log(" [vr-backend] runtime offers no swapchain format that keeps guest " + "eye format %d bytes; colours may be off\n", + static_cast(eye)); + } + return; + } + if (preferred == current || + (FormatIsCopyCompatible(eye, current) && + FormatIsCopyCompatible(eye, preferred))) { + return; // already a raw copy + } + eye_swapchain_format_ = preferred; + DestroySwapchains(); + std::string error; + Require(CreateSwapchains(&error), error); + Log(" [vr-backend] eye swapchains recreated in format %d to match guest " + "eye format %d\n", + static_cast(preferred), static_cast(eye)); +} + void OpenXrBackend::DestroySwapchains() { for (EyeSwapchain& slot : eye_swapchains_) { slot.images.clear(); @@ -1336,6 +1406,8 @@ void OpenXrBackend::TeardownSessionLocked(const char* reason, bool preserve_devi } void OpenXrBackend::NoteInstanceDestroyed(VkInstance instance) { + // Lock order: frame_mutex_, then mutex_ (see NoteHostPresent). + std::lock_guard frame(frame_mutex_); std::lock_guard lock(mutex_); if (instance == vk_instance_) { TeardownSessionLocked("guest VkInstance destroyed"); @@ -1345,6 +1417,8 @@ void OpenXrBackend::NoteInstanceDestroyed(VkInstance instance) { } void OpenXrBackend::NoteDeviceDestroyed(VkDevice device) { + // Lock order: frame_mutex_, then mutex_ (see NoteHostPresent). + std::lock_guard frame(frame_mutex_); std::lock_guard lock(mutex_); if (device != VK_NULL_HANDLE && device == vk_device_) { // Tear down while the device is still valid; the adapter notifies before @@ -2011,6 +2085,20 @@ bool OpenXrBackend::RecoverSessionLocked(std::uint64_t now_ns) { return true; } +VkFormat internal::PreferredEyeSwapchainFormat( + VkFormat eye, const std::vector& available) { + const auto offered = [&](VkFormat format) { + return format != VK_FORMAT_UNDEFINED && + std::find(available.begin(), available.end(), + static_cast(format)) != available.end(); + }; + for (const VkFormat candidate : + {eye, SrgbSibling(eye), OtherChannelOrder(eye)}) { + if (offered(candidate)) return candidate; + } + return VK_FORMAT_UNDEFINED; +} + VkResult internal::WaitForPresentDependencies(VkDevice device, VkQueue queue, const VkPresentInfoKHR &present, VkFence fence, @@ -2104,8 +2192,14 @@ VkResult OpenXrBackend::PrepareHostPresent(VkQueue queue, VkDevice device, void OpenXrBackend::NoteHostPresent(VkQueue queue, VkDevice device) { bool idle = false; { - std::lock_guard lock(mutex_); + // frame_mutex_ keeps the session alive for the whole frame cycle, so + // mutex_ (which guest render-thread hooks take) can be released while the + // cycle blocks in xrWaitFrame. Teardown paths take frame_mutex_ first. + std::lock_guard frame(frame_mutex_); + std::unique_lock lock(mutex_); + state_lock_ = &lock; idle = NoteHostPresentLocked(queue, device); + state_lock_ = nullptr; } if (idle) { // No running XR session (headset off or idle, session not READY, or @@ -2227,6 +2321,7 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { } bool images_held[2] = {false, false}; if (can_submit) { + MatchSwapchainFormatToEyesLocked(); projection.space = static_cast(xr_local_space_); projection.viewCount = 2; for (int eye = 0; eye < 2; ++eye) { @@ -2373,7 +2468,20 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { const ScopedStageTimer wait_timer(perf::Stage::kXrWaitFrame); auto wait = XrInfo(XR_TYPE_FRAME_WAIT_INFO); XrFrameState state = XrInfo(XR_TYPE_FRAME_STATE); - CheckXr(instance, xrWaitFrame(session, &wait, &state), "xrWaitFrame"); + XrResult wait_result = XR_SUCCESS; + { + // Runtime pacing can block for a whole display period. Guest hooks + // must not wait on it; frame_mutex_ still excludes teardown. + struct Relock { + std::unique_lock* lock; + ~Relock() { + if (lock != nullptr) lock->lock(); + } + } relock{state_lock_}; + if (state_lock_ != nullptr) state_lock_->unlock(); + wait_result = xrWaitFrame(session, &wait, &state); + } + CheckXr(instance, wait_result, "xrWaitFrame"); { const ScopedStageTimer begin_timer(perf::Stage::kXrBeginFrame); auto begin_info = XrInfo(XR_TYPE_FRAME_BEGIN_INFO); diff --git a/src/vr/vr_mirror.cc b/src/vr/vr_mirror.cc index 1c3f7228..4903d951 100644 --- a/src/vr/vr_mirror.cc +++ b/src/vr/vr_mirror.cc @@ -63,14 +63,29 @@ bool VrMirror::Open(const std::string& shm_name, std::uint32_t width, return false; } const std::uint64_t bytes = VrMirrorBytes(width, height); - const int descriptor = - shm_open(shm_name.c_str(), O_CREAT | O_RDWR | O_CLOEXEC, 0600); + // A crashed run leaves its segment behind; remove our own stale one. In the + // sticky /dev/shm this cannot remove another user's segment, and O_EXCL + // then refuses to adopt it, so eye images never go into a segment someone + // else created or can read. + (void)shm_unlink(shm_name.c_str()); + const int descriptor = shm_open(shm_name.c_str(), + O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, 0600); if (descriptor < 0) { if (error != nullptr) { - *error = "shm_open failed: " + std::string(std::strerror(errno)); + *error = "shm_open failed: " + std::string(std::strerror(errno)) + + (errno == EEXIST ? " (segment owned by another user)" : ""); } return false; } + struct stat status{}; + if (fstat(descriptor, &status) != 0 || status.st_uid != getuid() || + (status.st_mode & 077) != 0) { + if (error != nullptr) { + *error = "mirror segment is not private to this user"; + } + close(descriptor); + return false; + } if (ftruncate(descriptor, static_cast(bytes)) != 0) { if (error != nullptr) { *error = "ftruncate failed: " + std::string(std::strerror(errno)); diff --git a/tests/vr/openxr_backend_test.cc b/tests/vr/openxr_backend_test.cc index 65ad4e6a..36068384 100644 --- a/tests/vr/openxr_backend_test.cc +++ b/tests/vr/openxr_backend_test.cc @@ -564,3 +564,44 @@ TEST_F(OpenXrSplitPresentation, } } // namespace } // namespace mocktail::vr + +namespace mocktail::vr { +namespace { + +TEST(OpenXrEyeSwapchainFormat, KeepsGuestEyeBytesWithoutGammaConversion) { + using internal::PreferredEyeSwapchainFormat; + const auto formats = [](std::initializer_list list) { + std::vector out; + for (const VkFormat format : list) out.push_back(format); + return out; + }; + // The guest's own format wins. + EXPECT_EQ(PreferredEyeSwapchainFormat( + VK_FORMAT_R8G8B8A8_UNORM, + formats({VK_FORMAT_B8G8R8A8_SRGB, VK_FORMAT_R8G8B8A8_UNORM})), + VK_FORMAT_R8G8B8A8_UNORM); + // SteamVR-style SRGB-only runtime: the same-order sibling is a raw copy. + EXPECT_EQ(PreferredEyeSwapchainFormat( + VK_FORMAT_R8G8B8A8_UNORM, + formats({VK_FORMAT_B8G8R8A8_SRGB, VK_FORMAT_R8G8B8A8_SRGB})), + VK_FORMAT_R8G8B8A8_SRGB); + // Only the other channel order: keep the sRGB-ness so the blit only + // swizzles. + EXPECT_EQ(PreferredEyeSwapchainFormat( + VK_FORMAT_R8G8B8A8_UNORM, + formats({VK_FORMAT_B8G8R8A8_SRGB, VK_FORMAT_B8G8R8A8_UNORM})), + VK_FORMAT_B8G8R8A8_UNORM); + EXPECT_EQ(PreferredEyeSwapchainFormat(VK_FORMAT_B8G8R8A8_SRGB, + formats({VK_FORMAT_R8G8B8A8_SRGB})), + VK_FORMAT_R8G8B8A8_SRGB); + // Only a gamma-converting blit remains. + EXPECT_EQ(PreferredEyeSwapchainFormat(VK_FORMAT_R8G8B8A8_UNORM, + formats({VK_FORMAT_B8G8R8A8_SRGB})), + VK_FORMAT_UNDEFINED); + EXPECT_EQ(PreferredEyeSwapchainFormat(VK_FORMAT_R16G16B16A16_SFLOAT, + formats({VK_FORMAT_R8G8B8A8_SRGB})), + VK_FORMAT_UNDEFINED); +} + +} // namespace +} // namespace mocktail::vr diff --git a/tests/vr/vr_mirror_test.cc b/tests/vr/vr_mirror_test.cc index dfea1996..be59c3bc 100644 --- a/tests/vr/vr_mirror_test.cc +++ b/tests/vr/vr_mirror_test.cc @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -226,3 +227,31 @@ TEST(VrRuntimeSelection, RegisteredRuntimePrecedesInstalledWiVRn) { EXPECT_EQ(paths.back(), "/usr/share/openxr/1/openxr_wivrn.json"); } } // namespace mocktail::vr + +namespace mocktail::vr { +namespace { + +TEST(VrMirrorProducer, ReplacesAStaleSegmentWithAPrivateOne) { + const std::string name = "/mocktail-vr-mirror-unittest-stale"; + // A crashed run left a world-readable segment with this name behind. + const int stale = shm_open(name.c_str(), O_CREAT | O_RDWR, 0644); + ASSERT_GE(stale, 0); + ASSERT_EQ(fchmod(stale, 0644), 0); + close(stale); + + VrMirror mirror; + std::string error; + ASSERT_TRUE(mirror.Open(name, 4, 4, &error)) << error; + const int fresh = shm_open(name.c_str(), O_RDONLY, 0); + ASSERT_GE(fresh, 0); + struct stat status{}; + ASSERT_EQ(fstat(fresh, &status), 0); + close(fresh); + EXPECT_EQ(status.st_uid, getuid()); + EXPECT_EQ(status.st_mode & 077, 0u); // not the stale 0644 segment + mirror.Close(); + EXPECT_LT(shm_open(name.c_str(), O_RDONLY, 0), 0); // unlinked on close +} + +} // namespace +} // namespace mocktail::vr From a06c7de93e4ead8ce97721148a293c6ee87ee557 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:24:45 -0500 Subject: [PATCH 27/58] vr: fix bridge hand math and make its shutdown safe for live guest threads - MOCKTAIL_VR_HAND_POSE_SOURCE=grip now rotates hand records from the grip pose. base_ori started as the identity quaternion, so the norm check never fell back to it and hand rotation was dropped. - Hand records cancel the engine's stored hand pitch (read from hand_pitch_storage_rva by the pointer hook, 20 until read) instead of a hard-coded 20, so they agree with the pointer ray. - Shutdown never rewrites the aim-pointer prologue while guest threads may be executing it; the jump stays and only forwards once the bridge is gone, and a later activation reuses it. - Guest hooks count themselves in flight before loading the bridge pointer; Shutdown unpublishes the bridge and waits for in-flight calls to drain, so no hook keeps a pointer to a destroyed bridge. - Roblox's original state and eye getters are kept for the process lifetime. A hook reached with no bridge, including after a failed vtable restore, forwards to them instead of returning an unfilled state buffer. Restoring the vtable no longer depends on the pointer hook. - The activation log names the vtable slots in the right order. --- src/vr/roblox_vr_device_bridge.cc | 136 ++++++++++++++++------ src/vr/roblox_vr_device_bridge_internal.h | 12 +- tests/vr/roblox_vr_device_bridge_test.cc | 52 +++++++++ 3 files changed, 163 insertions(+), 37 deletions(-) diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc index 0bfd62e3..4cfc1cda 100644 --- a/src/vr/roblox_vr_device_bridge.cc +++ b/src/vr/roblox_vr_device_bridge.cc @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include "mocktail/vr/openxr_backend.h" @@ -18,9 +19,11 @@ #include #include #include -#include #include +#include +#include #include +#include #include "vr/roblox_vr_device_bridge_internal.h" @@ -53,6 +56,21 @@ std::uintptr_t RelativeTarget(const std::uint8_t* displacement, } std::atomic g_active_bridge{nullptr}; +// Guest calls currently inside an interposed hook. Shutdown unpublishes the +// bridge and then waits for this to drain, so no hook ever dereferences a +// bridge that is being destroyed. +std::atomic g_hook_calls{0}; +struct HookCall { + HookCall() { g_hook_calls.fetch_add(1, std::memory_order_seq_cst); } + ~HookCall() { g_hook_calls.fetch_sub(1, std::memory_order_release); } + HookCall(const HookCall&) = delete; + HookCall& operator=(const HookCall&) = delete; +}; +// Roblox's original getters, kept for the process lifetime (libroblox is +// never unloaded) so a hook still reachable after shutdown, or whose vtable +// could not be restored, always forwards instead of returning garbage. +std::atomic g_original_state_getter{nullptr}; +std::atomic g_original_eye_getter{nullptr}; // Evidence queries expose only the current native call's provenance. These // pointers are identities for the layer, never retained/dereferenced by it. thread_local void *g_initializing_object = nullptr; @@ -77,11 +95,12 @@ void AbsoluteJump(unsigned char *code, std::uintptr_t target) { } extern "C" void *mocktail_vr_pointer_frame(void *result, void *service) { + HookCall in_flight; auto original = g_pointer_original.load(std::memory_order_acquire); if (!original) return result; original(result, service); - auto *bridge = g_active_bridge.load(std::memory_order_acquire); + auto *bridge = g_active_bridge.load(std::memory_order_seq_cst); if (!bridge || !bridge->active()) return result; bridge->OnPointerFrameCall(result, service); return result; @@ -99,8 +118,10 @@ bool SetCodeWritable(std::uintptr_t target, bool writable) { } bool InstallPointerHook(std::uintptr_t base, std::uintptr_t pointer_rva) { + // The jump stays installed for the process lifetime (see + // RestorePointerHook); a later activation reuses it. if (g_pointer_target) - return false; + return g_pointer_target == base + pointer_rva; auto* target = reinterpret_cast(base + pointer_rva); if (std::memcmp(target, kPointerPrologue.data(), kPointerPrologue.size()) != 0) @@ -135,18 +156,11 @@ bool InstallPointerHook(std::uintptr_t base, std::uintptr_t pointer_rva) { return true; } -bool RestorePointerHook() { - if (!g_pointer_target) - return true; - if (!SetCodeWritable(g_pointer_target, true)) - return false; - std::memcpy(reinterpret_cast(g_pointer_target), - kPointerPrologue.data(), kPointerPrologue.size()); - const bool restored = SetCodeWritable(g_pointer_target, false); - if (restored) - g_pointer_target = 0; - return restored; -} +// Never rewrites the guest prologue: guest threads may be executing those +// bytes, and a torn 15-byte store would crash them. The installed jump stays; +// with no active bridge mocktail_vr_pointer_frame only forwards to the +// original code through the retained trampoline. +bool RestorePointerHook() { return true; } Status InvalidArgument(std::string message) { return Status::Error(StatusCode::kInvalidArgument, std::move(message)); @@ -289,7 +303,8 @@ std::uintptr_t* g_is_vr_app_build_slot = nullptr; std::uintptr_t g_is_vr_app_build_original = 0; extern "C" bool mocktail_vr_is_vr_app_build(const void*) { - auto* bridge = g_active_bridge.load(std::memory_order_acquire); + HookCall in_flight; + auto* bridge = g_active_bridge.load(std::memory_order_seq_cst); return bridge != nullptr && bridge->active(); } @@ -579,26 +594,32 @@ namespace { // takes (rdi = device object, esi = eye index) and returns the framebuffer // object pointer. extern "C" void mocktail_vr_debug_device_haptics(void* object, int hand, float amplitude) { - if (auto* bridge = g_active_bridge.load(std::memory_order_acquire)) + HookCall in_flight; + if (auto* bridge = g_active_bridge.load(std::memory_order_seq_cst)) bridge->OnHapticsCall(object, hand, amplitude); } extern "C" void *mocktail_vr_debug_device_state_getter(void *result_buffer, void *device_object) { + HookCall in_flight; RobloxVrDeviceBridge *bridge = - g_active_bridge.load(std::memory_order_acquire); + g_active_bridge.load(std::memory_order_seq_cst); if (bridge != nullptr) { bridge->OnStateGetterCall(result_buffer, device_object); + } else if (const auto original = + g_original_state_getter.load(std::memory_order_acquire)) { + // Bridge gone but the slot still points here: Roblox's own answer. + original(result_buffer, device_object); } return result_buffer; } extern "C" void *mocktail_vr_debug_device_eye_getter(void *device_object, int eye_index) { + HookCall in_flight; RobloxVrDeviceBridge *bridge = - g_active_bridge.load(std::memory_order_acquire); - const auto original = - bridge != nullptr ? bridge->original_eye_getter() : nullptr; + g_active_bridge.load(std::memory_order_seq_cst); + const auto original = g_original_eye_getter.load(std::memory_order_acquire); void *framebuffer = original != nullptr ? original(device_object, eye_index) : nullptr; if (bridge != nullptr) { @@ -629,8 +650,9 @@ extern "C" void mocktail_vr_consume_eye() { } extern "C" void mocktail_vr_note_host_present_begin() { + HookCall in_flight; RobloxVrDeviceBridge *bridge = - g_active_bridge.load(std::memory_order_acquire); + g_active_bridge.load(std::memory_order_seq_cst); if (bridge != nullptr) { bridge->NoteHostPresent(); } @@ -670,8 +692,24 @@ float GetHandPitchOffsetDeg() { return offset; } -float GetTotalHandPitchDeg() { - static const float total = [] { +namespace { +// Engine default hand pitch; replaced by the guest's stored value once read. +std::atomic g_native_hand_pitch_deg{20.f}; +} // namespace + +void NoteNativeHandPitchDeg(int degrees) { + if (std::abs(degrees) <= 360) + g_native_hand_pitch_deg.store(static_cast(degrees), + std::memory_order_relaxed); +} + +float NativeHandPitchDeg() { + return g_native_hand_pitch_deg.load(std::memory_order_relaxed); +} + +float GetTotalHandPitchDeg(float native_pitch_deg) { + static const std::optional override_deg = + []() -> std::optional { if (const char* env = std::getenv("MOCKTAIL_VR_HAND_PITCH")) { char* end = nullptr; const float val = std::strtof(env, &end); @@ -680,13 +718,17 @@ float GetTotalHandPitchDeg() { return val; } } - return 20.f + GetHandPitchOffsetDeg(); + return std::nullopt; }(); - return total; + // Cancel the pitch the engine itself applies (read from the guest, not + // assumed), then add the configurable offset. + return override_deg ? *override_deg + : native_pitch_deg + GetHandPitchOffsetDeg(); } -pose::Quat ComputeHandRecordOrientation(const VrHandPose& source) { - const float total_pitch_deg = GetTotalHandPitchDeg(); +pose::Quat ComputeHandRecordOrientation(const VrHandPose& source, + float native_pitch_deg) { + const float total_pitch_deg = GetTotalHandPitchDeg(native_pitch_deg); static const bool use_grip = [] { if (const char* env = std::getenv("MOCKTAIL_VR_HAND_POSE_SOURCE")) { @@ -711,6 +753,7 @@ pose::Quat ComputeHandRecordOrientation(const VrHandPose& source) { } pose::Quat base_ori{}; + bool have_base = false; if (!use_grip && source.aim_valid) { float aim_norm = 0.f; bool aim_finite = true; @@ -719,13 +762,15 @@ pose::Quat ComputeHandRecordOrientation(const VrHandPose& source) { aim_norm += v * v; } if (aim_finite && aim_norm >= 0.99f && aim_norm <= 1.01f) { - (void)pose::QuatNormalize( + have_base = pose::QuatNormalize( {source.aim_orientation[0], source.aim_orientation[1], source.aim_orientation[2], source.aim_orientation[3]}, &base_ori); } } - if (!std::isfinite(base_ori.w) || QuatNorm(base_ori) < 0.5f) { + // The grip source, or an unusable aim pose: rotate from the grip pose. + // (base_ori starts as identity, so a norm check alone never caught this.) + if (!have_base) { if (!pose::QuatNormalize( {source.grip_orientation[0], source.grip_orientation[1], source.grip_orientation[2], source.grip_orientation[3]}, @@ -782,6 +827,7 @@ void RobloxVrDeviceBridge::OnPointerFrameCall(void* result, void* service) { std::memcpy(°rees, reinterpret_cast(base + contract_->hand_pitch_storage_rva), 4); + internal::NoteNativeHandPitchDeg(degrees); { static std::atomic last_scale{-1.f}; const float previous = last_scale.exchange(scale); @@ -791,7 +837,8 @@ void RobloxVrDeviceBridge::OnPointerFrameCall(void* result, void* service) { scale, degrees); } VrHandPose pointer_hand = hand; - const pose::Quat pitched = internal::ComputeHandRecordOrientation(hand); + const pose::Quat pitched = + internal::ComputeHandRecordOrientation(hand, static_cast(degrees)); pointer_hand.grip_orientation[0] = pitched.x; pointer_hand.grip_orientation[1] = pitched.y; pointer_hand.grip_orientation[2] = pitched.z; @@ -997,6 +1044,11 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { std::memory_order_release); original_eye_getter_.store(reinterpret_cast(original_eye_slot_), std::memory_order_release); + g_original_state_getter.store( + reinterpret_cast(original_state_slot_), + std::memory_order_release); + g_original_eye_getter.store(reinterpret_cast(original_eye_slot_), + std::memory_order_release); __atomic_store_n(&vtable_[3], FunctionAddress(&mocktail_vr_debug_device_haptics), __ATOMIC_RELEASE); __atomic_store_n(&vtable_[kStateGetterVtableSlot / 8], FunctionAddress(&mocktail_vr_debug_device_state_getter), @@ -1029,7 +1081,7 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { g_native_pose = {}; } active_.store(true, std::memory_order_release); - Log(" [vr-device] activated: DebugDeviceVR input/eyes/haptics +0x18/+0x28/+0x48 and aim pointer " + Log(" [vr-device] activated: DebugDeviceVR haptics/input/eyes +0x18/+0x28/+0x48 and aim pointer " "interposed at image base 0x%" PRIxPTR "\n", image_base); return Status::Ok(); @@ -1060,12 +1112,27 @@ void RobloxVrDeviceBridge::Shutdown() { std::fprintf(stderr, " [vr-device] failed to restore DebugDeviceVR vtable " "protection\n"); } + // Unpublish, then wait out guest calls already inside a hook: they hold a + // raw pointer to this bridge. New calls see no bridge and only forward. + RobloxVrDeviceBridge *expected = this; + (void)g_active_bridge.compare_exchange_strong( + expected, nullptr, std::memory_order_seq_cst, std::memory_order_seq_cst); + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (g_hook_calls.load(std::memory_order_acquire) != 0 && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + if (g_hook_calls.load(std::memory_order_acquire) != 0) { + std::fprintf(stderr, " [vr-device] shutdown: a guest call is still " + "inside a VR hook after 2 s\n"); + } installed_ = false; } bool RobloxVrDeviceBridge::RestoreVtableLocked() { RestoreIsVrAppBuildHook(); - if (!RestorePointerHook()) return false; + (void)RestorePointerHook(); if (vtable_ == nullptr) { return false; } @@ -1310,7 +1377,8 @@ bool internal::ApplyXrPose(void *state, const ScriptedPoseSample &pose) { } if (!finite || hand_norm < 0.99f || hand_norm > 1.01f) continue; - const pose::Quat pitched = ComputeHandRecordOrientation(source); + const pose::Quat pitched = + ComputeHandRecordOrientation(source, NativeHandPitchDeg()); const float orientation[4] = {pitched.x, pitched.y, pitched.z, pitched.w}; const std::size_t record = hand == 0 ? kLeftHandRecordOffset : kRightHandRecordOffset; diff --git a/src/vr/roblox_vr_device_bridge_internal.h b/src/vr/roblox_vr_device_bridge_internal.h index b64690fc..65083f71 100644 --- a/src/vr/roblox_vr_device_bridge_internal.h +++ b/src/vr/roblox_vr_device_bridge_internal.h @@ -26,10 +26,16 @@ struct VrBuildContract { const VrBuildContract* FindVrBuildContract(std::string_view build_id); -// Computes the record orientation for a hand pose, incorporating native -// hand pitch cancellation and configurable pitch offsets. -pose::Quat ComputeHandRecordOrientation(const VrHandPose& source); +// Computes the record orientation for a hand pose: the aim pose (or, with +// MOCKTAIL_VR_HAND_POSE_SOURCE=grip, the grip pose) pitched by the engine's +// own hand pitch plus the configurable offset. +pose::Quat ComputeHandRecordOrientation(const VrHandPose& source, + float native_pitch_deg); float GetHandPitchOffsetDeg(); +// The engine's stored hand pitch (degrees), as last read from the guest; +// 20 until it has been read. +void NoteNativeHandPitchDeg(int degrees); +float NativeHandPitchDeg(); // Writes only a caller-owned state copy; preserves all resource readiness. bool ApplyXrPose(void* state, const ScriptedPoseSample& pose); diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc index 91083f1e..91e70871 100644 --- a/tests/vr/roblox_vr_device_bridge_test.cc +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -888,3 +888,55 @@ TEST(VrSupportedBuilds, InstallNamesTheBuildAndTheSupportedOnes) { } // namespace } // namespace mocktail::vr + +namespace mocktail::vr { +namespace { + +VrHandPose HandWith(const pose::Quat& aim, const pose::Quat& grip) { + VrHandPose hand; + hand.aim_valid = hand.grip_valid = true; + const float a[4] = {aim.x, aim.y, aim.z, aim.w}; + const float g[4] = {grip.x, grip.y, grip.z, grip.w}; + std::copy(a, a + 4, hand.aim_orientation); + std::copy(g, g + 4, hand.grip_orientation); + return hand; +} + +pose::Quat Pitch(float degrees) { + return pose::QuatFromAxisAngle({1.f, 0.f, 0.f}, degrees * pose::kPi / 180.f); +} + +TEST(VrHandOrientation, CancelsTheEnginesStoredPitchNotAnAssumedOne) { + const pose::Quat yaw = pose::QuatFromAxisAngle({0.f, 1.f, 0.f}, 0.7f); + const VrHandPose hand = HandWith(yaw, pose::Quat{}); + const float offset = internal::GetHandPitchOffsetDeg(); + for (const float native : {20.f, 35.f, 0.f}) { + const pose::Quat result = + internal::ComputeHandRecordOrientation(hand, native); + EXPECT_TRUE(pose::QuatSameRotation( + result, pose::QuatMultiply(yaw, Pitch(native + offset)), 1e-4f)) + << native; + } + internal::NoteNativeHandPitchDeg(35); + EXPECT_FLOAT_EQ(internal::NativeHandPitchDeg(), 35.f); + internal::NoteNativeHandPitchDeg(100000); // garbage is ignored + EXPECT_FLOAT_EQ(internal::NativeHandPitchDeg(), 35.f); +} + +TEST(VrHandOrientation, GripSourceKeepsTheGripRotation) { + // ctest runs each test in its own process; the source is read once. + setenv("MOCKTAIL_VR_HAND_POSE_SOURCE", "grip", 1); + const pose::Quat grip = pose::QuatFromAxisAngle({0.f, 1.f, 0.f}, 1.2f); + const pose::Quat aim = pose::QuatFromAxisAngle({0.f, 1.f, 0.f}, -0.4f); + const pose::Quat result = internal::ComputeHandRecordOrientation( + HandWith(aim, grip), 20.f); + EXPECT_TRUE(pose::QuatSameRotation( + result, + pose::QuatMultiply(grip, + Pitch(20.f + internal::GetHandPitchOffsetDeg())), + 1e-4f)); + unsetenv("MOCKTAIL_VR_HAND_POSE_SOURCE"); +} + +} // namespace +} // namespace mocktail::vr From 228121157b5bbc2e2608d9d172c22a142979a685 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Tue, 13 Oct 2026 00:28:26 -0500 Subject: [PATCH 28/58] vr: drop test hooks and dead code, document every VR variable - Remove the MOCKTAIL_VR_TEST_SESSION_LOSS_AFTER fault injection; nothing used it. - The scripted head pose and the eye evidence dumper only work in builds configured with -DMOCKTAIL_VR_DIAGNOSTICS=ON (default OFF); release builds ignore their environment variables. - Remove FindFmodInputCaptureProfile and HasExpectedFmodInputCaptureVtable (only their own tests called them) and the never-defined PrepareCaptureLocked declaration; the audio bridge files match upstream again. - docs/VR.md lists every MOCKTAIL_VR_* variable with its default, linked from the README. --- README.md | 3 + cmake/MocktailVR.cmake | 5 ++ docs/VR.md | 49 +++++++++++++++ .../audio/roblox_output_device_bridge.h | 3 +- include/mocktail/vr/openxr_backend.h | 1 - src/audio/roblox_output_device_bridge.cc | 44 -------------- .../roblox_output_device_bridge_internal.h | 17 ------ src/vr/openxr_backend.cc | 40 +++++++------ tests/roblox_output_device_bridge_test.cc | 59 ------------------- 9 files changed, 80 insertions(+), 141 deletions(-) create mode 100644 docs/VR.md diff --git a/README.md b/README.md index 6ce26006..30d623d8 100644 --- a/README.md +++ b/README.md @@ -272,6 +272,9 @@ MOCKTAIL_VR_RUNTIME=alvr MOCKTAIL_GRAPHICS_BACKEND=direct-vulkan ./build/mocktai If you previously set `XR_RUNTIME_JSON`, unset it before switching runtimes. Use `--no-vr` for desktop mode. +Every VR environment variable, from frame-rate limits to hand pitch, is listed +in [docs/VR.md](docs/VR.md). + ### Typing in VR When a text box takes focus (chat, search), a keyboard opens in front of you, diff --git a/cmake/MocktailVR.cmake b/cmake/MocktailVR.cmake index 9b610d2c..4e1e813a 100644 --- a/cmake/MocktailVR.cmake +++ b/cmake/MocktailVR.cmake @@ -82,6 +82,11 @@ if(MOCKTAIL_ENABLE_VR) src/vr/xr_actions.cc PROPERTIES COMPILE_OPTIONS -fexceptions) target_compile_definitions(mocktail_vr PRIVATE XR_USE_GRAPHICS_API_VULKAN) + option(MOCKTAIL_VR_DIAGNOSTICS + "Compile VR diagnostics: scripted head pose, eye evidence dumps" OFF) + if(MOCKTAIL_VR_DIAGNOSTICS) + target_compile_definitions(mocktail_vr PRIVATE MOCKTAIL_VR_DIAGNOSTICS=1) + endif() set_source_files_properties(src/vr/openxr_backend.cc PROPERTIES COMPILE_DEFINITIONS "XR_USE_GRAPHICS_API_OPENGL_ES;XR_USE_PLATFORM_EGL") target_include_directories(mocktail_vr PRIVATE ${MOCKTAIL_EGL_INCLUDE_DIR} ${MOCKTAIL_GLES3_INCLUDE_DIR}) target_link_libraries(mocktail_vr PRIVATE OpenXR::openxr_loader Vulkan::Headers ${CMAKE_DL_LIBS}) diff --git a/docs/VR.md b/docs/VR.md new file mode 100644 index 00000000..0f2ba044 --- /dev/null +++ b/docs/VR.md @@ -0,0 +1,49 @@ +# VR settings + +Experimental OpenXR VR is opt-in: launch with `--vr`, or set `vr.enabled: true` +in `config.yaml`. These environment variables tune it. Everything here only +applies while VR is active. + +## Runtime and display + +| Variable | Default | Effect | +| --- | --- | --- | +| `MOCKTAIL_VR_ENABLED` | `0` | Same as `--vr` (`1`) or `--no-vr` (`0`). | +| `MOCKTAIL_VR_RUNTIME` | `auto` | OpenXR runtime: `auto`, `system`, `wivrn`, `steamvr` or `alvr`. `XR_RUNTIME_JSON` overrides it. | +| `MOCKTAIL_VR_BACKEND` | `xr` | `native` keeps Roblox's stereo rendering without an OpenXR runtime, for debugging. | +| `MOCKTAIL_VR_FRAME_RATE_LIMIT` | `unlimited` | Frame-rate cap in VR, same values as `graphics.frame_rate_limit` (also `vr.frame_rate_limit`). With `unlimited` the headset's refresh rate paces frames. | +| `MOCKTAIL_VR_RENDER_SCALE` | `1.0` | Scales each axis of the runtime's recommended eye size, `0.25` to `2.0`. | +| `MOCKTAIL_VR_UNPACED` | `0` | `1` stops waiting for the headset between frames, to measure the maximum frame rate. Frames past the headset refresh are not shown. Not for playing. | +| `MOCKTAIL_VR_DESKTOP_MIRROR` | `1` | `0` stops copying the left eye to the desktop window. | +| `MOCKTAIL_VR_MIRROR_SHM` | unset | A shared-memory name such as `/nightcap-mirror` publishes both eyes for an external viewer. The segment is private to your user. | +| `MOCKTAIL_VR_KEYBOARD` | `1` | `0` turns off the in-headset keyboard. | + +## Hands and scale + +| Variable | Default | Effect | +| --- | --- | --- | +| `MOCKTAIL_VR_HAND_POSE_SOURCE` | `aim` | `grip` orients hands from the controller grip pose instead of the aim pose. | +| `MOCKTAIL_VR_HAND_PITCH_OFFSET` | `-20` | Degrees added to Roblox's own hand pitch. | +| `MOCKTAIL_VR_HAND_PITCH` | unset | Replaces the total hand pitch, in degrees. | +| `MOCKTAIL_VR_WORLD_SCALE` | `1.0` | Scales head, hand and eye offsets. `MOCKTAIL_VR_EYE_OFFSET_SCALE` is an older name for it. | + +## Performance tracing + +| Variable | Default | Effect | +| --- | --- | --- | +| `MOCKTAIL_VR_PERF` | unset | `1` logs per-stage frame timings and counters once a second. | +| `MOCKTAIL_VR_MEASURE_START_NS`, `MOCKTAIL_VR_MEASURE_END_NS` | unset | Limit `MOCKTAIL_VR_PERF` to a `CLOCK_MONOTONIC` window. | +| `MOCKTAIL_VR_POSE_LOG_INTERVAL` | `240` | Frames between head-pose log lines. | + +## Diagnostics builds + +These only work in builds configured with `-DMOCKTAIL_VR_DIAGNOSTICS=ON`. +Release builds ignore them. + +| Variable | Default | Effect | +| --- | --- | --- | +| `MOCKTAIL_VR_POSE_SOURCE` | `runtime` | `script` drives the head from a scripted motion instead of the headset, for testing without wearing it. | +| `MOCKTAIL_VR_POSE_AMPLITUDE_POSITION`, `_YAW`, `_PITCH` | `0.25`, `0.5`, `0.17` | Scripted motion size, in metres and radians. | +| `MOCKTAIL_VR_POSE_PERIOD_POSITION`, `_ROTATION` | `240`, `360` | Scripted motion period, in frames. | +| `MOCKTAIL_VR_XR_EVIDENCE_DIR` | unset | Writes submitted eye images to this directory. | +| `MOCKTAIL_VR_XR_EVIDENCE_INTERVAL` | `20` | Frames between evidence images. | diff --git a/include/mocktail/audio/roblox_output_device_bridge.h b/include/mocktail/audio/roblox_output_device_bridge.h index 04c93c0b..f9bf61c3 100644 --- a/include/mocktail/audio/roblox_output_device_bridge.h +++ b/include/mocktail/audio/roblox_output_device_bridge.h @@ -3,7 +3,6 @@ #include #include -#include #include #include #include @@ -16,6 +15,7 @@ namespace mocktail::audio { class NativeInputCapture; + // Replaces the profiled input/output-device query/select slots of the exact // Build-ID-scoped FmodAudioDevice vtable. Roblox keeps owning its FMOD engine, // while the existing settings UI sees and selects the SDL host routes that @@ -50,7 +50,6 @@ class RobloxOutputDeviceBridge final { Status Activate(std::uintptr_t image_base); Status PatchVtableLocked(); - Status PrepareCaptureLocked(); bool RestoreVtableLocked(); static int GetInputDeviceCount(void *self); static void *GetInputDeviceInfo(void *result, void *self, int index); diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index 60ddbbec..eda3ff42 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -443,7 +443,6 @@ class OpenXrBackend final { // A running session is hidden in SYNCHRONIZED; shouldRender still governs rendering. bool visibility_lost_ = false; bool session_recovery_pending_ = false; - bool test_session_loss_injected_ = false; std::uint64_t next_session_retry_ns_ = 0; // Consecutive failed session recoveries, for backoff and for deciding when // the XrInstance itself must be recreated (e.g. after a runtime restart). diff --git a/src/audio/roblox_output_device_bridge.cc b/src/audio/roblox_output_device_bridge.cc index 0fcde4bc..0134c214 100644 --- a/src/audio/roblox_output_device_bridge.cc +++ b/src/audio/roblox_output_device_bridge.cc @@ -134,50 +134,6 @@ RobloxOutputDeviceBridge::RobloxOutputDeviceBridge() = default; namespace internal { -const FmodInputCaptureProfile* FindFmodInputCaptureProfile( - const compat::BuildProfile& profile) { - static constexpr FmodInputCaptureProfile kProfiles[] = { - {"ade08266c67aee88ec9c1d00902150e1684dad3a", - 0x6c58040, - 1, - {4, 11, 12, 13, 14, 15}, - {0x32d0e9a, 0x32d0a34, 0x32d16a8, 0x32d1f84, 0x32d056c, 0x32d26ac}}, - {"5f0704edd9064f566ee3d6df2bd2fabbcc709f03", - 0x6cd3ce0, - 2, - {4, 13, 14, 15, 16, 17}, - {0x320c630, 0x320bcf6, 0x320d4b4, 0x320dd94, 0x320b844, 0x320e49a}}, - }; - if (!profile.allow_host_abi_bridges || !profile.fmod_output_device_bridge) { - return nullptr; - } - const auto& output = *profile.fmod_output_device_bridge; - for (const auto& capture : kProfiles) { - if (capture.build_id == profile.elf_build_id && - capture.vtable_rva == output.vtable_rva && - output.vtable_layout_version >= 0 && - capture.vtable_layout_version == - static_cast(output.vtable_layout_version)) { - return &capture; - } - } - return nullptr; -} - -bool HasExpectedFmodInputCaptureVtable(const std::uintptr_t* vtable, - std::uintptr_t image_base, - const FmodInputCaptureProfile& profile) { - if (vtable == nullptr || image_base == 0) return false; - for (std::size_t i = 0; i < profile.slots.size(); ++i) { - if (profile.method_rvas[i] > - std::numeric_limits::max() - image_base || - vtable[profile.slots[i]] != image_base + profile.method_rvas[i]) { - return false; - } - } - return true; -} - bool HasExpectedFmodStringConstructorContract(const std::uint8_t* code, std::size_t size) { constexpr std::array diff --git a/src/audio/roblox_output_device_bridge_internal.h b/src/audio/roblox_output_device_bridge_internal.h index c28c132c..1a28001a 100644 --- a/src/audio/roblox_output_device_bridge_internal.h +++ b/src/audio/roblox_output_device_bridge_internal.h @@ -1,7 +1,6 @@ #ifndef MOCKTAIL_AUDIO_ROBLOX_OUTPUT_DEVICE_BRIDGE_INTERNAL_H_ #define MOCKTAIL_AUDIO_ROBLOX_OUTPUT_DEVICE_BRIDGE_INTERNAL_H_ -#include #include #include #include @@ -11,22 +10,6 @@ namespace mocktail::audio::internal { -struct FmodInputCaptureProfile { - std::string_view build_id; - std::uintptr_t vtable_rva; - std::uint32_t vtable_layout_version; - // Format, latency, start, stop, poll, recording. - std::array slots; - std::array method_rvas; -}; - -const FmodInputCaptureProfile* FindFmodInputCaptureProfile( - const compat::BuildProfile& profile); - -bool HasExpectedFmodInputCaptureVtable(const std::uintptr_t* vtable, - std::uintptr_t image_base, - const FmodInputCaptureProfile& profile); - bool HasExpectedFmodStringConstructorContract(const std::uint8_t* code, std::size_t size); diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 9255965c..1f9109b4 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -108,6 +108,22 @@ std::string EnvString(const char* name, const char* fallback) { : std::string(fallback); } +#ifndef MOCKTAIL_VR_DIAGNOSTICS +#define MOCKTAIL_VR_DIAGNOSTICS 0 +#endif +// Desktop/acceptance diagnostics (scripted head pose, eye evidence dumps) +// exist only in builds configured with -DMOCKTAIL_VR_DIAGNOSTICS=ON; in +// normal builds their environment variables have no effect. +std::string EvidenceDirectory() { + return MOCKTAIL_VR_DIAGNOSTICS ? EnvString("MOCKTAIL_VR_XR_EVIDENCE_DIR", "") + : std::string(); +} +std::string PoseSource() { + return MOCKTAIL_VR_DIAGNOSTICS + ? EnvString("MOCKTAIL_VR_POSE_SOURCE", "runtime") + : std::string("runtime"); +} + // Coordinate and pose math is shared with the unit tests through // vr_pose_math.h, which documents every space and unit in the camera path, so // the audited conventions cannot silently diverge from the backend. @@ -796,7 +812,7 @@ void OpenXrBackend::SetupDeviceAfterCreationLocked( Log(" [vr-backend] XR session bound to guest VkDevice: gpu=%s runtime=%s " "pose_source=%s\n", device_properties.deviceName, runtime_name_.c_str(), - EnvString("MOCKTAIL_VR_POSE_SOURCE", "runtime").c_str()); + PoseSource().c_str()); } bool OpenXrBackend::InitializeSessionLocked( @@ -902,7 +918,7 @@ Status OpenXrBackend::AttachGlesContext(void *display, void *config, Require(InitializeGlesSessionLocked(&error), error); Log(" [vr-backend] bound OpenGL ES context to OpenXR; GPU eye copies " "enabled, pose_source=%s\n", - EnvString("MOCKTAIL_VR_POSE_SOURCE", "runtime").c_str()); + PoseSource().c_str()); return Status::Ok(); } catch (const std::exception &error) { TeardownSessionLocked("EGL binding failed"); @@ -1211,7 +1227,7 @@ bool OpenXrBackend::CreateSwapchains(std::string* error) { XrInfo(XR_TYPE_SWAPCHAIN_CREATE_INFO); create_info.usageFlags = XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT | XR_SWAPCHAIN_USAGE_TRANSFER_DST_BIT; - if (!EnvString("MOCKTAIL_VR_XR_EVIDENCE_DIR", "").empty()) + if (!EvidenceDirectory().empty()) create_info.usageFlags |= XR_SWAPCHAIN_USAGE_TRANSFER_SRC_BIT; create_info.format = static_cast(chosen); create_info.sampleCount = 1; @@ -2231,18 +2247,6 @@ bool OpenXrBackend::NoteHostPresentLocked(VkQueue queue, VkDevice device) { } perf::ProcessCollector().Count(perf::Counter::kPresents); RunFrameCycle(queue, device); - // Explicit, one-shot fault injection for the isolated correctness canary. - // Never active in a normal authenticated/headset launch. - const auto loss_frame = - EnvUnsigned("MOCKTAIL_VR_TEST_SESSION_LOSS_AFTER", 0); - if (!test_session_loss_injected_ && loss_frame > 0 && - EnvString("MOCKTAIL_ISOLATED_CANARY", "") == "1" && - submitted_frames_.load() >= loss_frame) { - test_session_loss_injected_ = true; - Log(" [vr-backend][test] injecting session loss on existing guest " - "device\n"); - HandleSessionState(XR_SESSION_STATE_LOSS_PENDING); - } perf::ProcessCollector().EmitIfDue(perf::NowNs(), "xr-frame-cycle"); return false; } catch (const XrCallError &error) { @@ -2431,7 +2435,7 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { const std::uint64_t submitted = submitted_frames_.fetch_add(1, std::memory_order_relaxed) + 1; if (submitted == 1 || submitted % 120 == 0 || - (!EnvString("MOCKTAIL_VR_XR_EVIDENCE_DIR", "").empty() && + (!EvidenceDirectory().empty() && published_pose_.frame % std::max( 1, @@ -2536,7 +2540,7 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { XR_SUCCEEDED(head_result) && (head.locationFlags & required_location) == required_location; const std::string pose_source = - EnvString("MOCKTAIL_VR_POSE_SOURCE", "runtime"); + PoseSource(); Pose head_pose = head_valid ? XrToPose(head.pose) : Pose{}; if (head_valid) { // Tracked LOCAL head pose, independent of the scripted test source; @@ -3459,7 +3463,7 @@ VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, // Diagnostic only: read BOTH images on the guest queue while the XR image is // still acquired. Never read a released image owned by the compositor. bool OpenXrBackend::CaptureEyeEvidence(VkQueue queue, int eye) { - const std::string directory = EnvString("MOCKTAIL_VR_XR_EVIDENCE_DIR", ""); + const std::string directory = EvidenceDirectory(); const auto interval = std::max( 1, EnvUnsigned("MOCKTAIL_VR_XR_EVIDENCE_INTERVAL", 20)); if (directory.empty() || published_pose_.frame % interval != 0) diff --git a/tests/roblox_output_device_bridge_test.cc b/tests/roblox_output_device_bridge_test.cc index fabc9117..57f5b3f4 100644 --- a/tests/roblox_output_device_bridge_test.cc +++ b/tests/roblox_output_device_bridge_test.cc @@ -79,65 +79,6 @@ TEST(RobloxOutputDeviceBridgeTest, Validates3092LayoutWithoutUsingListSlots) { profile)); } -TEST(RobloxOutputDeviceBridgeTest, PinsCaptureToBuildIdVtableAndLayout) { - compat::BuildProfile profile; - profile.allow_host_abi_bridges = true; - profile.elf_build_id = "5f0704edd9064f566ee3d6df2bd2fabbcc709f03"; - EXPECT_EQ(internal::FindFmodInputCaptureProfile(profile), nullptr); - profile.fmod_output_device_bridge = TestProfile(); - EXPECT_EQ(internal::FindFmodInputCaptureProfile(profile), nullptr); - profile.fmod_output_device_bridge->vtable_rva = 0x6cd3ce0; - EXPECT_EQ(internal::FindFmodInputCaptureProfile(profile), nullptr); - profile.fmod_output_device_bridge->vtable_layout_version = 2; - ASSERT_NE(internal::FindFmodInputCaptureProfile(profile), nullptr); - profile.allow_host_abi_bridges = false; - EXPECT_EQ(internal::FindFmodInputCaptureProfile(profile), nullptr); - profile.allow_host_abi_bridges = true; - profile.elf_build_id = "unknown"; - EXPECT_EQ(internal::FindFmodInputCaptureProfile(profile), nullptr); -} - -TEST(RobloxOutputDeviceBridgeTest, ValidatesEveryCaptureSlotOnBothBuilds) { - constexpr std::uintptr_t kBase = 0x10000000; - for (int version : {2998, 3092}) { - compat::BuildProfile profile; - profile.allow_host_abi_bridges = true; - profile.elf_build_id = version == 2998 - ? "ade08266c67aee88ec9c1d00902150e1684dad3a" - : "5f0704edd9064f566ee3d6df2bd2fabbcc709f03"; - profile.fmod_output_device_bridge = TestProfile(); - profile.fmod_output_device_bridge->vtable_rva = - version == 2998 ? 0x6c58040 : 0x6cd3ce0; - profile.fmod_output_device_bridge->vtable_layout_version = - version == 2998 ? 1 : 2; - const auto* capture = internal::FindFmodInputCaptureProfile(profile); - ASSERT_NE(capture, nullptr); - const std::array expected_slots = - version == 2998 ? std::array{4, 11, 12, 13, 14, 15} - : std::array{4, 13, 14, 15, 16, 17}; - EXPECT_EQ(capture->slots, expected_slots); - std::array vtable{}; - for (std::size_t i = 0; i < capture->slots.size(); ++i) { - vtable[capture->slots[i]] = kBase + capture->method_rvas[i]; - } - EXPECT_TRUE(internal::HasExpectedFmodInputCaptureVtable(vtable.data(), - kBase, *capture)); - for (const auto slot : capture->slots) { - ++vtable[slot]; - EXPECT_FALSE(internal::HasExpectedFmodInputCaptureVtable(vtable.data(), - kBase, *capture)) - << slot; - --vtable[slot]; - } - EXPECT_FALSE( - internal::HasExpectedFmodInputCaptureVtable(nullptr, kBase, *capture)); - EXPECT_FALSE(internal::HasExpectedFmodInputCaptureVtable(vtable.data(), 0, - *capture)); - EXPECT_FALSE(internal::HasExpectedFmodInputCaptureVtable( - vtable.data(), UINTPTR_MAX, *capture)); - } -} - TEST(RobloxOutputDeviceBridgeTest, BuildsStableDistinctHostGuids) { const std::string first = internal::MakeOutputDeviceGuid(17, "USB Headset"); EXPECT_EQ(first, internal::MakeOutputDeviceGuid(17, "USB Headset")); From 4b671769701e22fe2a68bfbdc003dc1588ab348a Mon Sep 17 00:00:00 2001 From: vii Date: Tue, 13 Oct 2026 17:15:39 -0500 Subject: [PATCH 29/58] fix(scripts): find VR offsets when Roblox moves the pointer fields Build 3222 moved the hand selector, hand frame and camera HeadScale fields, so the fixed byte patterns found nothing. Match the code shape instead, read the three offsets from it and write them to the output. --- scripts/find_roblox_vr_offsets.py | 99 ++++++++++++++++++------------- 1 file changed, 58 insertions(+), 41 deletions(-) diff --git a/scripts/find_roblox_vr_offsets.py b/scripts/find_roblox_vr_offsets.py index 80a9d0d9..a90eff1b 100755 --- a/scripts/find_roblox_vr_offsets.py +++ b/scripts/find_roblox_vr_offsets.py @@ -95,10 +95,8 @@ READY_BYTE_OFFSETS = (0x94, 0x144, 0x147) FRAMEBUFFER_SLOT_OFFSET = 0x150 TEXTURE_SLOT_OFFSET = 0x170 -USERCFRAME_SELECTOR_OFFSET = 0x118 -USERCFRAME_BASE_OFFSET = 0x174 USERCFRAME_STRIDE = 0x30 -CAMERA_HEAD_SCALE_OFFSET = 0x140 +MIN_CAMERA_HEAD_SCALE_OFFSET = 0x100 EXPECTED_STATE_GETTER_SLOT = 0x28 EXPECTED_EYE_GETTER_SLOT = 0x48 @@ -106,9 +104,12 @@ EXPECTED_HAPTIC_SLOT = 0x18 MAX_VTABLE_SCAN_BYTES = 0x200 -# movsxd rax, [rsi+0x118]; movups xmm0, [rsi+rax+0x174]. -POINTER_GETTER_SELECTOR_PATTERN = bytes.fromhex("48 63 86 18 01 00 00") -POINTER_GETTER_MOVUPS_PATTERN = bytes.fromhex("0f 10 84 06 74 01 00 00") +# movsxd rax, [rsi+selector]; lea rax, [rax+rax*2]; shl rax, 4; +# movups xmm0, [rsi+rax+base]. The stride is fixed; the selector and base +# displacements are captured because they differ between builds. +POINTER_GETTER_PATTERN = re.compile( + rb"\x48\x63\x86(.{4})\x48\x8d\x04\x40\x48\xc1\xe0\x04" + rb"\x0f\x10\x84\x06(.{4})", re.DOTALL) TEXT_REF_OPCODES = re.compile( rb"[\x8d\x8b\x89\x8a\x88\x3b\x39\x3a\x38\x03\x33\x2b\x85\x80\x81\x83" @@ -661,18 +662,17 @@ def find_text_refs( return results -def scan_text_bytes(image: ElfImage, pattern: bytes) -> list[int]: +def scan_pointer_getter_sites(image: ElfImage) -> list[tuple[int, int, int]]: + """Return (address, selector offset, UserCFrame base offset) per site.""" text = image.text_section() data = image.section_bytes(text) - base = text.address - hits: list[int] = [] - position = 0 - while True: - position = data.find(pattern, position) - if position < 0: - return hits - hits.append(base + position) - position += 1 + sites: list[tuple[int, int, int]] = [] + for match in POINTER_GETTER_PATTERN.finditer(data): + selector = struct.unpack(" 0 and base > 0: + sites.append((text.address + match.start(), selector, base)) + return sites def ends_with_ret(instructions: Sequence[Any]) -> bool: @@ -948,22 +948,28 @@ def extract_hand_pitch(instructions: Sequence[Any], return None -def match_pointer_getter(instructions: Sequence[Any]) -> bool: - """World-space pointer getter: UserCFrame copy + workspace/camera path.""" +def match_pointer_getter(instructions: Sequence[Any], selector_offset: int, + base_offset: int) -> int | None: + """World-space pointer getter: UserCFrame copy + workspace/camera path. + + Returns the camera HeadScale offset, or None when the contract fails. + """ has_selector = False has_scale = False has_stride = False usercframe_loads = 0 has_workspace_handoff = False has_camera_slot_call = False - has_head_scale = False + head_scale_offsets: set[int] = set() + frame_registers = (capstone_x86.X86_REG_RBP, capstone_x86.X86_REG_RSP, + capstone_x86.X86_REG_RIP) for instruction in instructions: operands = instruction.operands if instruction.mnemonic == "movsxd" and len(operands) == 2: if (operands[1].type == capstone_x86.X86_OP_MEM and operands[1].mem.base == capstone_x86.X86_REG_RSI and operands[1].mem.index == capstone_x86.X86_REG_INVALID and - operands[1].mem.disp == USERCFRAME_SELECTOR_OFFSET): + operands[1].mem.disp == selector_offset): has_selector = True if instruction.mnemonic == "lea" and len(operands) == 2: memory = operands[1] @@ -987,17 +993,19 @@ def match_pointer_getter(instructions: Sequence[Any]) -> bool: memory.mem.base == capstone_x86.X86_REG_RSI and memory.mem.index == capstone_x86.X86_REG_RAX and memory.mem.disp in ( - USERCFRAME_BASE_OFFSET, - USERCFRAME_BASE_OFFSET + 0x10, - USERCFRAME_BASE_OFFSET + 0x20, + base_offset, + base_offset + 0x10, + base_offset + 0x20, )): usercframe_loads += 1 if instruction.mnemonic == "movss" and len(operands) == 2: memory = operands[1] + # The only float the getter loads from an object is HeadScale. if (memory.type == capstone_x86.X86_OP_MEM and memory.mem.index == capstone_x86.X86_REG_INVALID and - memory.mem.disp == CAMERA_HEAD_SCALE_OFFSET): - has_head_scale = True + memory.mem.base not in frame_registers and + memory.mem.disp >= MIN_CAMERA_HEAD_SCALE_OFFSET): + head_scale_offsets.add(memory.mem.disp) for index, instruction in enumerate(instructions): operands = instruction.operands if (instruction.mnemonic == "mov" and len(operands) == 2 and @@ -1033,9 +1041,11 @@ def match_pointer_getter(instructions: Sequence[Any]) -> bool: == capstone_x86.X86_REG_RCX and caller.operands[0].mem.disp > 0x100): has_camera_slot_call = True - return (has_selector and has_scale and has_stride and + if not (has_selector and has_scale and has_stride and usercframe_loads >= 3 and has_workspace_handoff and - has_camera_slot_call and has_head_scale) + has_camera_slot_call and len(head_scale_offsets) == 1): + return None + return next(iter(head_scale_offsets)) def extract_pointer_getter_anchors(instructions: Sequence[Any], @@ -1358,34 +1368,35 @@ def find_vr_offsets(elf_path: pathlib.Path) -> dict[str, Any]: f"hand pitch: dword default {hand_pitch_default} degrees with name" f" pointer at storage+8 -> 0x{hand_pitch_storage:x}") - selector_hits = scan_text_bytes(image, POINTER_GETTER_SELECTOR_PATTERN) - movups_hits = scan_text_bytes(image, POINTER_GETTER_MOVUPS_PATTERN) - getter_candidates: set[int] = set() - for hit in selector_hits + movups_hits: + getter_candidates: dict[int, tuple[int, int]] = {} + for hit, selector, base in scan_pointer_getter_sites(image): try: start, _end = table.bounds(hit) except AnalyzerError: continue - getter_candidates.add(start) - getter_matches: list[tuple[int, tuple[Any, ...]]] = [] - for start in sorted(getter_candidates): + getter_candidates[start] = (selector, base) + getter_matches: list[tuple[int, tuple[Any, ...], int, int, int]] = [] + for start, (selector, base) in sorted(getter_candidates.items()): instructions = disasm.function(*table.bounds(start)) - if match_pointer_getter(instructions): - getter_matches.append((start, instructions)) + head_scale = match_pointer_getter(instructions, selector, base) + if head_scale is not None: + getter_matches.append( + (start, instructions, selector, base, head_scale)) if not getter_matches: raise AnalyzerError( "no function satisfies the world-space pointer getter contract") if len(getter_matches) > 1: - listed = ", ".join(f"0x{start:x}" for start, _ in getter_matches) + listed = ", ".join(f"0x{match[0]:x}" for match in getter_matches) raise AnalyzerError(f"ambiguous pointer getter matches: {listed}") - pointer_getter, getter_instructions = getter_matches[0] + (pointer_getter, getter_instructions, selector_offset, base_offset, + head_scale_offset) = getter_matches[0] workspace_getter, camera_slot = extract_pointer_getter_anchors( getter_instructions, image) evidence.append("pointer getter: UserCFrame selector" - f" +0x{USERCFRAME_SELECTOR_OFFSET:x}," - f" base +0x{USERCFRAME_BASE_OFFSET:x}, stride" + f" +0x{selector_offset:x}," + f" base +0x{base_offset:x}, stride" f" 0x{USERCFRAME_STRIDE:x}," - f" HeadScale +0x{CAMERA_HEAD_SCALE_OFFSET:x}" + f" HeadScale +0x{head_scale_offset:x}" f" -> 0x{pointer_getter:x}") evidence.append(f"workspace getter 0x{workspace_getter:x}" " via mov rdi,rsi + direct call;" @@ -1433,6 +1444,12 @@ def find_vr_offsets(elf_path: pathlib.Path) -> dict[str, Any]: f"0x{workspace_getter:x}", "workspace_camera_vtable_offset": f"0x{camera_slot:x}", + "usercframe_selector_offset": + f"0x{selector_offset:x}", + "usercframe_base_offset": + f"0x{base_offset:x}", + "camera_head_scale_offset": + f"0x{head_scale_offset:x}", "new_vr_system_hand_pitch_storage_rva": f"0x{hand_pitch_storage:x}", "new_vr_system_hand_pitch_default_degrees": From 7294a46e8a06ba7b924df0b3b376005beb7099ae Mon Sep 17 00:00:00 2001 From: vii Date: Tue, 13 Oct 2026 18:37:19 -0500 Subject: [PATCH 30/58] fix(vr): turn on Vulkan queue locking when the adapter loads after the backend --- include/mocktail/vr/openxr_backend.h | 3 ++ src/graphics/bionic_vulkan_loader_adapter.cc | 20 ++++---- src/vr/openxr_backend.cc | 25 +++++++--- tests/CMakeLists.txt | 13 +++++ tests/vr/queue_sharing_load_test.cc | 52 ++++++++++++++++++++ tests/vr/queue_submission_test.cc | 6 +-- 6 files changed, 99 insertions(+), 20 deletions(-) create mode 100644 tests/vr/queue_sharing_load_test.cc diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index eda3ff42..6a1dc0d4 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -37,6 +37,9 @@ VkResult WaitForPresentDependencies(VkDevice device, VkQueue queue, VkFence fence, PFN_vkResetFences reset, PFN_vkQueueSubmit submit, PFN_vkWaitForFences wait); +// Makes the Vulkan adapter serialize queue calls for the rest of the process. +// Called when a backend arms; the adapter library may load before or after. +void RequestVulkanQueueSharing(); } // namespace internal // Resolved --vr output mode. kXrOutput drives the real Roblox stereo images diff --git a/src/graphics/bionic_vulkan_loader_adapter.cc b/src/graphics/bionic_vulkan_loader_adapter.cc index a7afe06a..ca92d891 100644 --- a/src/graphics/bionic_vulkan_loader_adapter.cc +++ b/src/graphics/bionic_vulkan_loader_adapter.cc @@ -322,12 +322,20 @@ Function ResolveProcessFunction(const char* name) { // xrEndFrame), so only then must every queue call be serialized: queue calls // share a per-device gate and a per-queue mutex, and device idle takes the // gate exclusively. Without VR (or in a build without it) nothing is locked. -// Set by mocktail_vulkan_enable_queue_sharing() when the VR backend arms; -// never cleared, since the runtime may use the queue until the process ends. +// Latched once the VR backend reports it armed; never cleared, since the +// runtime may use the queue until the process ends. The backend lives in the +// executable and arms before this library is loaded, so it is asked here. std::atomic g_queue_sharing{false}; bool QueueSharingActive() { - return g_queue_sharing.load(std::memory_order_acquire); + if (g_queue_sharing.load(std::memory_order_acquire)) + return true; + static const auto requested = ResolveProcessFunction( + "mocktail_vr_xr_queue_sharing_requested"); + if (requested == nullptr || !requested()) + return false; + g_queue_sharing.store(true, std::memory_order_release); + return true; } std::shared_ptr DeviceIdleGate(VkDevice device) { @@ -3503,12 +3511,6 @@ VKAPI_ATTR void VKAPI_CALL vkCmdPipelineBarrier2KHR( ForwardVrBarrier2(command, info, "vkCmdPipelineBarrier2KHR"); } -// Called once by the VR backend when it arms, before Roblox creates its -// device: from then on queue calls are serialized (see QueueCallGuard). -void mocktail_vulkan_enable_queue_sharing() { - g_queue_sharing.store(true, std::memory_order_release); -} - VKAPI_ATTR VkResult VKAPI_CALL mocktail_vulkan_submit_synchronized( PFN_vkQueueSubmit raw, VkDevice device, VkQueue queue, uint32_t count, const VkSubmitInfo *submits, VkFence fence) { diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 1f9109b4..537a5088 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -32,9 +32,6 @@ #define M_PI 3.14159265358979323846 #endif -// Defined by the Vulkan adapter (bionic_vulkan_loader_adapter.cc). -extern "C" __attribute__((weak)) void mocktail_vulkan_enable_queue_sharing(); - namespace mocktail::vr { namespace { @@ -198,6 +195,8 @@ VkFormat SrgbSibling(VkFormat format) { } OpenXrBackend* g_backend = nullptr; +// Read by the Vulkan adapter through mocktail_vr_xr_queue_sharing_requested(). +std::atomic g_queue_sharing_requested{false}; std::mutex g_backend_slot_mutex; } // namespace @@ -455,6 +454,10 @@ bool OpenXrBackend::EnsureXrInstanceLocked(std::string* error) { } } +void internal::RequestVulkanQueueSharing() { + g_queue_sharing_requested.store(true, std::memory_order_release); +} + Status OpenXrBackend::Arm(VrGraphicsApi graphics_api) { std::lock_guard lock(mutex_); if (armed_.load(std::memory_order_acquire)) { @@ -521,11 +524,8 @@ Status OpenXrBackend::Arm(VrGraphicsApi graphics_api) { armed_.store(true, std::memory_order_release); // From here the runtime and the eye copies share the guest's queue, so // the Vulkan adapter must serialize queue calls. Armed before Roblox - // creates its device, so no queue call ever started unlocked. Weak: the - // adapter is absent from VR-only test binaries. - if (mocktail_vulkan_enable_queue_sharing != nullptr) { - mocktail_vulkan_enable_queue_sharing(); - } + // creates its device, so no queue call ever started unlocked. + internal::RequestVulkanQueueSharing(); Log(" [vr-backend] armed: OpenXR instance/system ready; awaiting guest " "%s\n", graphics_api_ == VrGraphicsApi::kVulkan ? "Vulkan device" @@ -3246,6 +3246,8 @@ bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { } VkResult fence_result = VK_SUCCESS; { + // mutex_ stays held: the guest's vkDestroyImage waits on it in + // NoteDestroyImage, which keeps the eye image alive until the copy ends. const ScopedStageTimer fence_timer(perf::Stage::kCopyFenceWait); fence_result = vk_->WaitForFences(vk_device_, 1, ©_fence_, VK_TRUE, 2'000'000'000ULL); @@ -3827,6 +3829,13 @@ void mocktail_vr_xr_note_present(VkQueue queue, VkDevice device) { } } +// The Vulkan adapter is loaded with dlopen() after the backend arms, so it +// asks for this instead of being called. +bool mocktail_vr_xr_queue_sharing_requested() { + return mocktail::vr::g_queue_sharing_requested.load( + std::memory_order_acquire); +} + VkResult mocktail_vr_xr_prepare_present(VkQueue queue, VkDevice device, const VkPresentInfoKHR *info) { try { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 10cb6cf3..26ba95a5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -67,7 +67,20 @@ if(MOCKTAIL_ENABLE_VR) add_executable(vr_queue_submission_test vr/queue_submission_test.cc) target_link_libraries(vr_queue_submission_test PRIVATE Mocktail::VR stub_vulkan GTest::gtest_main) + # The adapter looks up the backend's exported symbols at run time. + set_target_properties(vr_queue_submission_test PROPERTIES ENABLE_EXPORTS ON) gtest_discover_tests(vr_queue_submission_test) + # Loads the adapter with dlopen() like the runtime does, so it must not + # link it; the adapter finds the backend through the exported symbols. + add_executable(vr_queue_sharing_load_test vr/queue_sharing_load_test.cc) + target_link_libraries(vr_queue_sharing_load_test PRIVATE + Mocktail::VR ${CMAKE_DL_LIBS} GTest::gtest_main) + target_compile_definitions(vr_queue_sharing_load_test PRIVATE + "MOCKTAIL_TEST_VULKAN_ADAPTER=\"$\"") + set_target_properties(vr_queue_sharing_load_test PROPERTIES + ENABLE_EXPORTS ON) + add_dependencies(vr_queue_sharing_load_test stub_vulkan) + gtest_discover_tests(vr_queue_sharing_load_test) add_executable(gles_vr_transport_test vr/gles_transport_test.cc) target_include_directories(gles_vr_transport_test PRIVATE "${PROJECT_SOURCE_DIR}/src" "${PROJECT_SOURCE_DIR}/third_party/OpenXR-SDK/include" ${MOCKTAIL_GLES3_INCLUDE_DIR}) target_link_libraries(gles_vr_transport_test PRIVATE Mocktail::VR SDL3::SDL3 GTest::gtest_main) diff --git a/tests/vr/queue_sharing_load_test.cc b/tests/vr/queue_sharing_load_test.cc new file mode 100644 index 00000000..270cc23c --- /dev/null +++ b/tests/vr/queue_sharing_load_test.cc @@ -0,0 +1,52 @@ +#include + +#include + +#include +#include +#include + +#include "mocktail/vr/openxr_backend.h" + +namespace { + +using SubmitSynchronized = VkResult(VKAPI_PTR *)(PFN_vkQueueSubmit, VkDevice, + VkQueue, uint32_t, + const VkSubmitInfo *, VkFence); + +std::atomic inside_driver{0}, max_inside{0}; + +VKAPI_ATTR VkResult VKAPI_CALL SlowSubmit(VkQueue, uint32_t, + const VkSubmitInfo *, VkFence) { + const int active = ++inside_driver; + int previous = max_inside.load(); + while (active > previous && + !max_inside.compare_exchange_weak(previous, active)) { + } + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + --inside_driver; + return VK_SUCCESS; +} + +// The runtime arms the backend first and loads the Vulkan adapter with +// dlopen() afterwards; this binary does the same and does not link it. +TEST(VrQueueSharingLoad, AdapterLoadedAfterArmingSerializesQueueCalls) { + mocktail::vr::internal::RequestVulkanQueueSharing(); + void *adapter = dlopen(MOCKTAIL_TEST_VULKAN_ADAPTER, RTLD_LAZY | RTLD_GLOBAL); + ASSERT_NE(adapter, nullptr) << dlerror(); + const auto submit = reinterpret_cast( + dlsym(adapter, "mocktail_vulkan_submit_synchronized")); + ASSERT_NE(submit, nullptr); + const auto queue = reinterpret_cast(30); + auto call = [&] { + EXPECT_EQ(submit(SlowSubmit, reinterpret_cast(2), queue, 0, + nullptr, VK_NULL_HANDLE), + VK_SUCCESS); + }; + std::thread first(call), second(call); + first.join(); + second.join(); + EXPECT_EQ(max_inside, 1); +} + +} // namespace diff --git a/tests/vr/queue_submission_test.cc b/tests/vr/queue_submission_test.cc index 5bbc6da0..95ee1288 100644 --- a/tests/vr/queue_submission_test.cc +++ b/tests/vr/queue_submission_test.cc @@ -17,7 +17,6 @@ extern "C" VKAPI_ATTR VkResult VKAPI_CALL mocktail_vulkan_submit_synchronized( extern "C" void mocktail_vr_xr_queue_submit_adapter(SubmitAdapter); extern "C" VKAPI_ATTR VkResult VKAPI_CALL mocktail_vulkan_idle_synchronized(PFN_vkDeviceWaitIdle raw, VkDevice device); -extern "C" void mocktail_vulkan_enable_queue_sharing(); namespace { std::mutex state_mutex; @@ -68,7 +67,8 @@ VKAPI_ATTR VkResult VKAPI_CALL BlockingIdle(VkDevice) { : VK_TIMEOUT; } TEST(VrQueueSubmission, DeviceIdleExcludesPrivateSubmitsUntilDriverReturns) { - mocktail_vulkan_enable_queue_sharing(); // an armed VR backend does this + // An armed VR backend does this. + mocktail::vr::internal::RequestVulkanQueueSharing(); waiting = signal_submitted = idle_release = false; VkResult idle_result = VK_NOT_READY; std::thread idler([&] { @@ -183,7 +183,7 @@ TEST(VrQueueSubmission, GuestCallsAreNotSerializedWithoutVr) { EXPECT_EQ(max_inside, 2); } TEST(VrQueueSubmission, PrivateAndGuestCallsCannotEnterOneQueueConcurrently) { - mocktail_vulkan_enable_queue_sharing(); + mocktail::vr::internal::RequestVulkanQueueSharing(); inside_driver = max_inside = 0; const auto queue = reinterpret_cast(30); auto call = [&] { From f9289d9947c65c5d00e9ca14d862caca136fec79 Mon Sep 17 00:00:00 2001 From: vii Date: Tue, 13 Oct 2026 18:37:26 -0500 Subject: [PATCH 31/58] fix(vr): read the pointer hand and HeadScale offsets per build and check them at start-up --- src/vr/roblox_vr_device_bridge.cc | 47 ++++++++++++++++++--- src/vr/roblox_vr_device_bridge_internal.h | 8 ++++ tests/vr/roblox_vr_device_bridge_test.cc | 50 +++++++++++++++++++++++ 3 files changed, 99 insertions(+), 6 deletions(-) diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc index 4cfc1cda..b494c637 100644 --- a/src/vr/roblox_vr_device_bridge.cc +++ b/src/vr/roblox_vr_device_bridge.cc @@ -403,6 +403,8 @@ constexpr VrBuildContract kContracts[] = { 0x3a8, 0x73677a8, 0x1d48b70, + 0x118, + 0x140, "2998 (2.736.1408)"}, {"5f0704edd9064f566ee3d6df2bd2fabbcc709f03", {0x6d08108, 0x37f9fe4, 0x37fa3c6, 0x37fa3b6, 0x37fa3e8, 0x73f4690, @@ -413,6 +415,8 @@ constexpr VrBuildContract kContracts[] = { 0x3a0, 0x73f24c8, 0x1d8f8c0, + 0x118, + 0x140, "3092 (2.738.1397)"}, }; @@ -431,11 +435,26 @@ bool HasExpectedVrPointerContract(const std::uint8_t* code, std::size_t size, code[0x59] != 0xe8 || code[0x6d] != 0xff || code[0x6e] != 0x91) { return false; } - std::uint32_t camera_slot; + // movsxd rax,[rsi+selector] at +0x24; movss xmm0,[r14+HeadScale] at +0x13d. + static constexpr std::array kSelectorLoad = {0x48, 0x63, + 0x86}; + static constexpr std::array kHeadScaleLoad = { + 0xf3, 0x41, 0x0f, 0x10, 0x86}; + if (std::memcmp(code + 0x24, kSelectorLoad.data(), kSelectorLoad.size()) != + 0 || + std::memcmp(code + 0x13d, kHeadScaleLoad.data(), + kHeadScaleLoad.size()) != 0) { + return false; + } + std::uint32_t camera_slot, selector, head_scale; std::memcpy(&camera_slot, code + 0x6f, sizeof(camera_slot)); + std::memcpy(&selector, code + 0x27, sizeof(selector)); + std::memcpy(&head_scale, code + 0x142, sizeof(head_scale)); return RelativeTarget(code + 0x5a, contract.pointer_frame_rva + 0x5e) == contract.workspace_getter_rva && - camera_slot == contract.camera_vtable_offset; + camera_slot == contract.camera_vtable_offset && + selector == contract.pointer_hand_selector_offset && + head_scale == contract.camera_head_scale_offset; } EyeResourceState InspectEyeResources(const void *object) { @@ -676,6 +695,24 @@ std::string DescribeSupportedVrBuilds() { namespace internal { +int ReadPointerHandIndex(const void* service, const VrBuildContract& contract) { + int index = 0; + std::memcpy(&index, + static_cast(service) + + contract.pointer_hand_selector_offset, + sizeof(index)); + return index; +} + +float ReadCameraHeadScale(const void* camera, const VrBuildContract& contract) { + float scale = 0.f; + std::memcpy(&scale, + static_cast(camera) + + contract.camera_head_scale_offset, + sizeof(scale)); + return scale; +} + float GetHandPitchOffsetDeg() { static const float offset = [] { if (const char* env = std::getenv("MOCKTAIL_VR_HAND_PITCH_OFFSET")) { @@ -805,8 +842,7 @@ void RobloxVrDeviceBridge::OnPointerFrameCall(void* result, void* service) { if (!backend) return; const auto current = backend->PublishedHeadPose(); if (!current.valid) return; - int index = 0; - std::memcpy(&index, static_cast(service) + 0x118, 4); + const int index = internal::ReadPointerHandIndex(service, *contract_); if (index != 1 && index != 2) return; if (!current.hands[index - 1].aim_valid) return; const auto& hand = native_pose.hands[index - 1]; @@ -821,9 +857,8 @@ void RobloxVrDeviceBridge::OnPointerFrameCall(void* result, void* service) { void* camera = reinterpret_cast( vtable[contract_->camera_vtable_offset / sizeof(void*)])(workspace); if (!camera) return; - float scale = 0; + const float scale = internal::ReadCameraHeadScale(camera, *contract_); int degrees = 0; - std::memcpy(&scale, static_cast(camera) + 0x140, 4); std::memcpy(°rees, reinterpret_cast(base + contract_->hand_pitch_storage_rva), 4); diff --git a/src/vr/roblox_vr_device_bridge_internal.h b/src/vr/roblox_vr_device_bridge_internal.h index 65083f71..c9014795 100644 --- a/src/vr/roblox_vr_device_bridge_internal.h +++ b/src/vr/roblox_vr_device_bridge_internal.h @@ -20,12 +20,20 @@ struct VrBuildContract { std::uintptr_t camera_vtable_offset; std::uintptr_t hand_pitch_storage_rva; std::uintptr_t haptic_sink_rva; + // Field offsets the pointer hook reads: the hand index in the pointer + // service and HeadScale in the workspace camera. + std::uintptr_t pointer_hand_selector_offset; + std::uintptr_t camera_head_scale_offset; // Roblox version code and name, for messages that list supported builds. std::string_view version; }; const VrBuildContract* FindVrBuildContract(std::string_view build_id); +// The hand the pointer service asks about: 1 left, 2 right. +int ReadPointerHandIndex(const void* service, const VrBuildContract& contract); +float ReadCameraHeadScale(const void* camera, const VrBuildContract& contract); + // Computes the record orientation for a hand pose: the aim pose (or, with // MOCKTAIL_VR_HAND_POSE_SOURCE=grip, the grip pose) pitched by the engine's // own hand pitch plus the configurable offset. diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc index 91e70871..93b278d5 100644 --- a/tests/vr/roblox_vr_device_bridge_test.cc +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -317,6 +317,15 @@ TEST(VrPointerContract, PinsWorkspaceAndCameraSlotForEachBuild) { "5f0704edd9064f566ee3d6df2bd2fabbcc709f03"}) { const auto& contract = *internal::FindVrBuildContract(id); std::array code{}; + // movsxd rax,[rsi+selector] and movss xmm0,[r14+HeadScale]. + const std::uint8_t selector_load[] = {0x48, 0x63, 0x86}; + std::memcpy(code.data() + 0x24, selector_load, sizeof(selector_load)); + const std::uint32_t selector = contract.pointer_hand_selector_offset; + std::memcpy(code.data() + 0x27, &selector, sizeof(selector)); + const std::uint8_t scale_load[] = {0xf3, 0x41, 0x0f, 0x10, 0x86}; + std::memcpy(code.data() + 0x13d, scale_load, sizeof(scale_load)); + const std::uint32_t head_scale = contract.camera_head_scale_offset; + std::memcpy(code.data() + 0x142, &head_scale, sizeof(head_scale)); const std::uint8_t prologue[] = {0x55, 0x48, 0x89, 0xe5, 0x41, 0x57, 0x41, 0x56, 0x41, 0x54, 0x53, 0x48, 0x83, 0xec, 0x70}; @@ -336,6 +345,14 @@ TEST(VrPointerContract, PinsWorkspaceAndCameraSlotForEachBuild) { EXPECT_FALSE(internal::HasExpectedVrPointerContract(code.data(), code.size(), contract)); code[0x6f] ^= 8; + code[0x27] ^= 0x10; + EXPECT_FALSE(internal::HasExpectedVrPointerContract(code.data(), + code.size(), contract)); + code[0x27] ^= 0x10; + code[0x142] ^= 0x10; + EXPECT_FALSE(internal::HasExpectedVrPointerContract(code.data(), + code.size(), contract)); + code[0x142] ^= 0x10; code[0x5a] ^= 1; EXPECT_FALSE(internal::HasExpectedVrPointerContract(code.data(), code.size(), contract)); @@ -940,3 +957,36 @@ TEST(VrHandOrientation, GripSourceKeepsTheGripRotation) { } // namespace } // namespace mocktail::vr + +namespace mocktail::vr { +namespace { + +TEST(VrPointerLayout, ReadsHandIndexAndHeadScaleAtTheBuildsOwnOffsets) { + // Roblox moves these fields between builds; a fixed offset reads garbage. + internal::VrBuildContract contract{}; + contract.pointer_hand_selector_offset = 0x108; + contract.camera_head_scale_offset = 0x130; + std::array service{}, camera{}; + const int hand = 2, stale_hand = 7; + const float scale = 1.5f, stale_scale = 9.f; + std::memcpy(service.data() + 0x108, &hand, 4); + std::memcpy(service.data() + 0x118, &stale_hand, 4); + std::memcpy(camera.data() + 0x130, &scale, 4); + std::memcpy(camera.data() + 0x140, &stale_scale, 4); + EXPECT_EQ(internal::ReadPointerHandIndex(service.data(), contract), 2); + EXPECT_FLOAT_EQ(internal::ReadCameraHeadScale(camera.data(), contract), + 1.5f); +} + +TEST(VrPointerLayout, VerifiedBuildsKeepTheirOffsets) { + for (const char* id : {"ade08266c67aee88ec9c1d00902150e1684dad3a", + "5f0704edd9064f566ee3d6df2bd2fabbcc709f03"}) { + const auto* contract = internal::FindVrBuildContract(id); + ASSERT_NE(contract, nullptr); + EXPECT_EQ(contract->pointer_hand_selector_offset, 0x118u) << id; + EXPECT_EQ(contract->camera_head_scale_offset, 0x140u) << id; + } +} + +} // namespace +} // namespace mocktail::vr From 7ec8304d8f95ce66c8ffa2a5e4252575218d1316 Mon Sep 17 00:00:00 2001 From: vii Date: Tue, 13 Oct 2026 18:37:34 -0500 Subject: [PATCH 32/58] fix(vr): stop keyboard click-through, redraw churn and upload leaks; build its drawing only with VR --- cmake/MocktailVR.cmake | 19 ++- include/mocktail/vr/openxr_backend.h | 1 + include/mocktail/vr/virtual_keyboard.h | 9 +- src/main.cc | 8 +- src/runtime/roblox_text_surface_overlay.cc | 7 + src/vr/openxr_backend.cc | 30 +++- src/vr/openxr_disabled.cc | 3 + src/vr/virtual_keyboard.cc | 34 +++- src/window/window.cc | 6 +- tests/CMakeLists.txt | 6 + tests/vr/virtual_keyboard_raster_test.cc | 96 +++++++++++ tests/vr/virtual_keyboard_test.cc | 184 +++++++++++++++------ 12 files changed, 318 insertions(+), 85 deletions(-) create mode 100644 tests/vr/virtual_keyboard_raster_test.cc diff --git a/cmake/MocktailVR.cmake b/cmake/MocktailVR.cmake index 4e1e813a..7c76c850 100644 --- a/cmake/MocktailVR.cmake +++ b/cmake/MocktailVR.cmake @@ -32,17 +32,11 @@ add_library(mocktail_vr STATIC src/vr/vr_mirror.cc src/vr/xr_controller.cc src/vr/roblox_vr_device_bridge.cc - src/vr/virtual_keyboard.cc - src/vr/virtual_keyboard_raster.cc) + src/vr/virtual_keyboard.cc) add_library(Mocktail::VR ALIAS mocktail_vr) target_include_directories(mocktail_vr PUBLIC "${CMAKE_SOURCE_DIR}/include") target_include_directories(mocktail_vr PRIVATE "${CMAKE_SOURCE_DIR}/src") target_link_libraries(mocktail_vr PUBLIC Vulkan::Headers PRIVATE nlohmann_json::nlohmann_json) -# The in-headset keyboard draws its labels with the same SDL3_ttf/fontconfig -# stack as the TextBox overlay. MocktailInput/MocktailAudio find these -# packages later in the same directory scope; links resolve at generate time. -target_link_libraries(mocktail_vr PRIVATE SDL3_ttf::SDL3_ttf SDL3::SDL3 - PkgConfig::FONTCONFIG) # shm_open for the desktop eye mirror lives in librt on glibc. find_library(MOCKTAIL_RT_LIBRARY rt) if(MOCKTAIL_RT_LIBRARY) @@ -75,7 +69,16 @@ if(MOCKTAIL_ENABLE_VR) endfunction() mocktail_add_openxr_sdk() target_sources(mocktail_vr PRIVATE src/vr/openxr_probe.cc src/vr/openxr_preview.cc - src/vr/openxr_backend.cc src/vr/gles_transport.cc src/vr/xr_actions.cc) + src/vr/openxr_backend.cc src/vr/gles_transport.cc src/vr/xr_actions.cc + src/vr/virtual_keyboard_raster.cc) + # The in-headset keyboard draws its labels with the same SDL3_ttf and + # fontconfig stack as the TextBox overlay. + find_package(SDL3 3.4 REQUIRED CONFIG) + find_package(SDL3_ttf REQUIRED CONFIG) + find_package(PkgConfig REQUIRED) + pkg_check_modules(FONTCONFIG REQUIRED IMPORTED_TARGET fontconfig) + target_link_libraries(mocktail_vr PRIVATE SDL3_ttf::SDL3_ttf SDL3::SDL3 + PkgConfig::FONTCONFIG) # Error unwinding is confined to these native session implementations; their # public entry points catch errors before returning to the guest runtime. set_source_files_properties(src/vr/openxr_preview.cc src/vr/openxr_backend.cc diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index 6a1dc0d4..2b94a87b 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -336,6 +336,7 @@ class OpenXrBackend final { // keyboard swapchain holds a released image ready for a quad layer. bool PrepareVirtualKeyboardLayerLocked(VkQueue queue); bool UploadVirtualKeyboardVulkan(VkQueue queue, VkImage image); + void DestroyVirtualKeyboardStaging(); void DestroyVirtualKeyboardResources(); bool CaptureEyeEvidence(VkQueue queue, int eye); ScriptedPoseSample ComputeScriptedPose(std::uint64_t frame) const; diff --git a/include/mocktail/vr/virtual_keyboard.h b/include/mocktail/vr/virtual_keyboard.h index a9cb2303..ece5ffa3 100644 --- a/include/mocktail/vr/virtual_keyboard.h +++ b/include/mocktail/vr/virtual_keyboard.h @@ -74,7 +74,6 @@ struct VirtualKeyboardPointer { struct VirtualKeyboardOutput { std::vector text; // committed in order std::vector scancodes; // pressed in order, after text - bool closed = false; // user pressed Close }; // Ray/quad intersection. The quad is centred on `quad` with its face along @@ -118,8 +117,9 @@ class VirtualKeyboard { const VirtualKeyboardPointer (&pointers)[2]); // True when this hand's trigger belongs to the keyboard this frame (aimed - // at the panel, or a press that began on it is still held). The backend - // withholds that trigger from Roblox so a key press is not also a click. + // at the panel, or a press that began on it is still held, including after + // the panel hid mid-press). The backend withholds that trigger from Roblox + // so a key press is not also a click. bool CapturesTrigger(int hand) const; // Current layout, with Shift/Caps Lock applied to the labels. @@ -159,6 +159,9 @@ class VirtualKeyboard { pose::Pose panel_{}; VirtualKeyboardText text_; Hand hands_[2]; + // A press the keyboard owned when it hid; held until that trigger releases. + bool release_pending_[2] = {false, false}; + std::uint64_t cursor_drawn_ns_ = 0; std::vector keys_; std::uint64_t revision_ = 1; }; diff --git a/src/main.cc b/src/main.cc index 2d89487b..46cc1bb0 100644 --- a/src/main.cc +++ b/src/main.cc @@ -46,6 +46,7 @@ #include "runtime/roblox_desktop_app_policy.h" #include "runtime/roblox_experience_launch_bridge.h" #include "runtime/roblox_fullscreen_runtime_bridge.h" +#include "runtime/roblox_text_display_state.h" #include "runtime/roblox_text_editor.h" #include "runtime/runtime_config_bootstrap.h" #include "runtime/runtime_config_file.h" @@ -1082,13 +1083,10 @@ int main(int argc, char* argv[]) { mocktail::vr::ClearVirtualKeyboardText(); return; } - // RbxKeyboard password input types (see the SDL mapping). - const bool password = update.text_input_type == 5 || - update.text_input_type == 9 || - update.text_input_type == 10; mocktail::vr::SetVirtualKeyboardText( update.utf8, update.utf8_size, update.cursor_utf16, - password); + mocktail::runtime::IsRobloxPasswordTextInputType( + update.text_input_type)); }}); } } diff --git a/src/runtime/roblox_text_surface_overlay.cc b/src/runtime/roblox_text_surface_overlay.cc index c62530d4..2465ee46 100644 --- a/src/runtime/roblox_text_surface_overlay.cc +++ b/src/runtime/roblox_text_surface_overlay.cc @@ -328,11 +328,15 @@ Status RobloxTextSurfaceOverlay::UpdateViewport( namespace { std::mutex g_display_mirror_mutex; RobloxTextDisplaySink g_display_mirror; +// Lets a text update skip the lock while no mirror is installed. +std::atomic g_display_mirror_set{false}; } // namespace void SetRobloxTextDisplayMirror(RobloxTextDisplaySink sink) { std::lock_guard lock(g_display_mirror_mutex); g_display_mirror = sink; + g_display_mirror_set.store(sink.update != nullptr, + std::memory_order_release); } RobloxTextDisplaySink RobloxTextSurfaceOverlay::sink() { @@ -344,6 +348,9 @@ void RobloxTextSurfaceOverlay::UpdateCallback( if (context != nullptr) { static_cast(context)->ApplyUpdate(update); } + if (!g_display_mirror_set.load(std::memory_order_acquire)) { + return; + } std::lock_guard lock(g_display_mirror_mutex); if (g_display_mirror.update != nullptr) { g_display_mirror.update(g_display_mirror.context, update); diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 537a5088..5dec0cd6 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -2725,8 +2725,10 @@ void OpenXrBackend::UpdateVirtualKeyboardLocked() { return; } const VirtualKeyboardHost host = CurrentVirtualKeyboardHost(); + // A panel that cannot be drawn must not take triggers either. const bool requested = - host.input_requested != nullptr && host.input_requested(host.context); + !keyboard_swapchain_failed_ && host.input_requested != nullptr && + host.input_requested(host.context); if (!requested) { keyboard_.Hide(); keyboard_armed_ = true; @@ -2735,10 +2737,9 @@ void OpenXrBackend::UpdateVirtualKeyboardLocked() { keyboard_armed_ = false; Log(" [vr-keyboard] shown for focused TextBox\n"); } - if (!keyboard_.visible()) { - return; - } - { + // A hidden keyboard still runs below: it keeps a trigger that was pressed + // on the panel away from Roblox until that trigger is released. + if (keyboard_.visible()) { VirtualKeyboardText text = CurrentVirtualKeyboardText(); keyboard_.SetText(text); std::fill(text.utf8.begin(), text.utf8.end(), '\0'); @@ -2904,6 +2905,8 @@ bool OpenXrBackend::UploadVirtualKeyboardVulkan(VkQueue queue, VkImage image) { if (vk_->CreateBuffer(vk_device_, &info, nullptr, &keyboard_staging_) != VK_SUCCESS) { keyboard_staging_ = VK_NULL_HANDLE; + keyboard_swapchain_failed_ = true; + Log(" [vr-keyboard] disabled: no staging buffer\n"); return false; } VkMemoryRequirements requirements{}; @@ -2932,7 +2935,11 @@ bool OpenXrBackend::UploadVirtualKeyboardVulkan(VkQueue queue, VkImage image) { keyboard_staging_memory_, 0) != VK_SUCCESS || vk_->MapMemory(vk_device_, keyboard_staging_memory_, 0, size, 0, &keyboard_staging_mapped_) != VK_SUCCESS) { - DestroyVirtualKeyboardResources(); + // The caller still holds an acquired swapchain image, so only the + // staging objects go; the latch stops a retry on every frame. + DestroyVirtualKeyboardStaging(); + keyboard_swapchain_failed_ = true; + Log(" [vr-keyboard] disabled: no staging memory\n"); return false; } } @@ -2987,12 +2994,15 @@ bool OpenXrBackend::UploadVirtualKeyboardVulkan(VkQueue queue, VkImage image) { if (vk_->WaitForFences(vk_device_, 1, ©_fence_, VK_TRUE, 2'000'000'000ULL) != VK_SUCCESS) { (void)IdleHostDevice(vk_->DeviceWaitIdle, vk_device_); - throw std::runtime_error("keyboard upload fence wait failed"); + // Returning lets the caller release the acquired swapchain image. + keyboard_swapchain_failed_ = true; + Log(" [vr-keyboard] disabled: upload fence wait failed\n"); + return false; } return true; } -void OpenXrBackend::DestroyVirtualKeyboardResources() { +void OpenXrBackend::DestroyVirtualKeyboardStaging() { if (vk_ != nullptr && vk_device_ != VK_NULL_HANDLE) { if (keyboard_staging_mapped_ != nullptr) { vk_->UnmapMemory(vk_device_, keyboard_staging_memory_); @@ -3007,6 +3017,10 @@ void OpenXrBackend::DestroyVirtualKeyboardResources() { keyboard_staging_mapped_ = nullptr; keyboard_staging_ = VK_NULL_HANDLE; keyboard_staging_memory_ = VK_NULL_HANDLE; +} + +void OpenXrBackend::DestroyVirtualKeyboardResources() { + DestroyVirtualKeyboardStaging(); EyeSwapchain& slot = keyboard_swapchain_; slot.images.clear(); slot.gl_images.clear(); diff --git a/src/vr/openxr_disabled.cc b/src/vr/openxr_disabled.cc index b9f7446a..0345fec9 100644 --- a/src/vr/openxr_disabled.cc +++ b/src/vr/openxr_disabled.cc @@ -4,6 +4,9 @@ namespace mocktail::vr { OpenXrBackend::~OpenXrBackend() = default; +// The backend's keyboard_raster_ member needs this destructor. No rasterizer +// is ever created here, so SDL3_ttf and fontconfig stay out of this build. +VirtualKeyboardRasterizer::~VirtualKeyboardRasterizer() = default; Status OpenXrBackend::Arm(VrGraphicsApi) { return Status::Error(StatusCode::kUnavailable, "OpenXR support is disabled in this build"); } diff --git a/src/vr/virtual_keyboard.cc b/src/vr/virtual_keyboard.cc index 83333c8d..c737a873 100644 --- a/src/vr/virtual_keyboard.cc +++ b/src/vr/virtual_keyboard.cc @@ -30,6 +30,9 @@ constexpr std::uint64_t kRepeatDelayNs = 450'000'000ULL; constexpr std::uint64_t kRepeatIntervalNs = 60'000'000ULL; // Cursor moves smaller than this (in UV) do not force a redraw. constexpr float kCursorRedrawStep = 0.004f; +// A moving cursor redraws the panel at most this often; each redraw is a +// full texture upload on the frame thread. +constexpr std::uint64_t kCursorRedrawIntervalNs = 33'000'000ULL; constexpr float kPanelDistanceMetres = 0.45f; constexpr float kPanelDropMetres = 0.28f; @@ -239,7 +242,13 @@ void VirtualKeyboard::Hide() { text_.revision = UINT64_MAX; // the next SetText always reloads if (!visible_) return; visible_ = false; - for (Hand& hand : hands_) hand = Hand{}; + for (int index = 0; index < 2; ++index) { + // Keeps a press already pending from an earlier hide of this same press. + release_pending_[index] = + release_pending_[index] || + (hands_[index].trigger_down && hands_[index].captured); + hands_[index] = Hand{}; + } Touch(); } @@ -255,7 +264,8 @@ int VirtualKeyboard::KeyAt(float u, float v) const { bool VirtualKeyboard::CapturesTrigger(int hand) const { const Hand& state = hands_[hand & 1]; - return visible_ && (state.over_panel || state.captured); + return release_pending_[hand & 1] || + (visible_ && (state.over_panel || state.captured)); } bool VirtualKeyboard::cursor(int hand, float* u, float* v) const { @@ -298,7 +308,6 @@ void VirtualKeyboard::Activate(const VirtualKey& key, return; case VirtualKeyAction::kClose: output->scancodes.push_back(kScancodeEscape); - output->closed = true; Hide(); return; } @@ -307,8 +316,18 @@ void VirtualKeyboard::Activate(const VirtualKey& key, VirtualKeyboardOutput VirtualKeyboard::Update( std::uint64_t now_ns, const VirtualKeyboardPointer (&pointers)[2]) { VirtualKeyboardOutput output; - if (!visible_) return output; + bool held_over[2] = {false, false}; + for (int index = 0; index < 2; ++index) { + if (!release_pending_[index]) continue; + // Losing the aim pose is not a release: the trigger is read regardless. + held_over[index] = pointers[index].trigger > kTriggerRelease; + release_pending_[index] = held_over[index]; + } + // Decided once per frame, or one moving hand starves the other's cursor. + const bool cursor_redraw_due = + now_ns - cursor_drawn_ns_ >= kCursorRedrawIntervalNs; for (int index = 0; index < 2 && visible_; ++index) { + if (held_over[index]) continue; Hand& hand = hands_[index]; const VirtualKeyboardPointer& pointer = pointers[index]; float u = 0.f, v = 0.f; @@ -318,12 +337,15 @@ VirtualKeyboardOutput VirtualKeyboard::Update( kPanelHeightMetres, pointer.aim, &u, &v, nullptr); const int hovered = over ? KeyAt(u, v) : -1; + const bool cursor_moved = + over && (std::abs(u - hand.u) > kCursorRedrawStep || + std::abs(v - hand.v) > kCursorRedrawStep); if (over != hand.over_panel || hovered != hand.hovered || - (over && (std::abs(u - hand.u) > kCursorRedrawStep || - std::abs(v - hand.v) > kCursorRedrawStep))) { + (cursor_moved && cursor_redraw_due)) { Touch(); hand.u = u; hand.v = v; + cursor_drawn_ns_ = now_ns; } hand.over_panel = over; hand.hovered = hovered; diff --git a/src/window/window.cc b/src/window/window.cc index c2b4e529..877d206a 100644 --- a/src/window/window.cc +++ b/src/window/window.cc @@ -179,6 +179,8 @@ struct VirtualInput { }; static std::mutex g_virtual_input_mutex; static std::deque g_virtual_input; +// Set while the queue may hold input, so a pump with none takes no lock. +static std::atomic g_virtual_input_pending{false}; static std::unique_ptr g_pointer_capture_backend; // The owner is created and destroyed on the SDL thread, but the experience // composition reaches it from its own thread. The mutex guards the pointer @@ -1699,6 +1701,7 @@ void QueueVirtualTextInput(const char* utf8) { } std::lock_guard lock(g_virtual_input_mutex); g_virtual_input.push_back(VirtualInput{utf8, 0}); + g_virtual_input_pending.store(true, std::memory_order_release); } void QueueVirtualKeyPress(uint32_t scancode) { if (scancode == 0) { @@ -1706,6 +1709,7 @@ void QueueVirtualKeyPress(uint32_t scancode) { } std::lock_guard lock(g_virtual_input_mutex); g_virtual_input.push_back(VirtualInput{{}, scancode}); + g_virtual_input_pending.store(true, std::memory_order_release); } void* GetGLProcAddress(const char* name) { if (!g_state.initialised || name == nullptr || name[0] == '\0') { @@ -2320,7 +2324,7 @@ bool PumpEvents() { g_text_input_owner->active())) { fprintf(stderr, " [input] SDL pointer capture state update failed\n"); } - { + if (g_virtual_input_pending.exchange(false, std::memory_order_acq_rel)) { std::deque virtual_input; { std::lock_guard lock(g_virtual_input_mutex); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 26ba95a5..895d7c04 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -64,6 +64,12 @@ if(MOCKTAIL_ENABLE_VR) target_include_directories(openxr_backend_test PRIVATE "${PROJECT_SOURCE_DIR}/third_party/OpenXR-SDK/include") target_link_libraries(openxr_backend_test PRIVATE Mocktail::VR GTest::gtest_main) gtest_discover_tests(openxr_backend_test) + add_executable(virtual_keyboard_raster_test + vr/virtual_keyboard_raster_test.cc) + target_link_libraries(virtual_keyboard_raster_test PRIVATE + Mocktail::VR GTest::gtest_main) + mocktail_apply_compile_options(virtual_keyboard_raster_test) + gtest_discover_tests(virtual_keyboard_raster_test) add_executable(vr_queue_submission_test vr/queue_submission_test.cc) target_link_libraries(vr_queue_submission_test PRIVATE Mocktail::VR stub_vulkan GTest::gtest_main) diff --git a/tests/vr/virtual_keyboard_raster_test.cc b/tests/vr/virtual_keyboard_raster_test.cc new file mode 100644 index 00000000..7c5a78bd --- /dev/null +++ b/tests/vr/virtual_keyboard_raster_test.cc @@ -0,0 +1,96 @@ +#include + +#include +#include +#include + +#include "mocktail/vr/virtual_keyboard.h" + +namespace mocktail::vr { +namespace { + +using pose::Pose; +using pose::Vec3; + +Pose Head() { + Pose head; + head.position = Vec3{0.f, 1.6f, 0.f}; + return head; +} + +int FindKey(const VirtualKeyboard& keyboard, const std::string& label) { + const auto& keys = keyboard.keys(); + for (std::size_t i = 0; i < keys.size(); ++i) + if (keys[i].label == label) return static_cast(i); + return -1; +} + +void SetText(VirtualKeyboard* keyboard, const std::string& utf8, bool password, + std::uint64_t revision) { + VirtualKeyboardText text; + text.utf8 = utf8; + text.cursor_utf16 = static_cast(utf8.size()); + text.password = password; + text.revision = revision; + keyboard->SetText(text); +} + +TEST(VirtualKeyboardRasterTest, DrawsFieldTextAndCaret) { + VirtualKeyboard keyboard; + keyboard.Show(Head()); + VirtualKeyboardRasterizer rasterizer; + auto field_light_pixels = [&](const std::vector& rgba) { + const int bottom = static_cast(VirtualKeyboard::kFieldFraction * + VirtualKeyboard::kTextureHeight); + int count = 0; + for (int y = 8; y < bottom - 8; ++y) + for (int x = 30; x < VirtualKeyboard::kTextureWidth - 30; ++x) + if (rgba[(static_cast(y) * + VirtualKeyboard::kTextureWidth + x) * 4 + 1] > 120) + ++count; + return count; + }; + std::vector empty, typed, masked; + if (!rasterizer.Draw(keyboard, &empty)) GTEST_SKIP() << "no host font"; + SetText(&keyboard, "Hello world", false, 1); + ASSERT_TRUE(rasterizer.Draw(keyboard, &typed)); + EXPECT_GT(field_light_pixels(typed), field_light_pixels(empty) + 200); + SetText(&keyboard, "Hello world", true, 2); + ASSERT_TRUE(rasterizer.Draw(keyboard, &masked)); + EXPECT_NE(masked, typed); // dots, not the letters + EXPECT_GT(field_light_pixels(masked), field_light_pixels(empty)); +} + +TEST(VirtualKeyboardRasterTest, PasswordPictureDoesNotDependOnItsLetters) { + VirtualKeyboard keyboard; + keyboard.Show(Head()); + VirtualKeyboardRasterizer rasterizer; + std::vector first, second; + SetText(&keyboard, "Hello world", true, 1); + if (!rasterizer.Draw(keyboard, &first)) GTEST_SKIP() << "no host font"; + SetText(&keyboard, "Jumpy vixen", true, 2); + ASSERT_TRUE(rasterizer.Draw(keyboard, &second)); + EXPECT_EQ(first, second); +} + +TEST(VirtualKeyboardRasterTest, FillsAnOpaqueTexture) { + VirtualKeyboard keyboard; + keyboard.Show(Head()); + VirtualKeyboardRasterizer rasterizer; + std::vector rgba; + (void)rasterizer.Draw(keyboard, &rgba); // labels depend on host fonts + ASSERT_EQ(rgba.size(), static_cast( + VirtualKeyboard::kTextureWidth) * + VirtualKeyboard::kTextureHeight * 4); + for (std::size_t i = 3; i < rgba.size(); i += 4) ASSERT_EQ(rgba[i], 255); + // A key is lighter than the background behind it. + const auto& q = keyboard.keys()[FindKey(keyboard, "q")]; + const int x = static_cast(q.u0 * VirtualKeyboard::kTextureWidth) + 20; + const int y = static_cast(q.v0 * VirtualKeyboard::kTextureHeight) + 20; + const std::size_t key_px = + (static_cast(y) * VirtualKeyboard::kTextureWidth + x) * 4; + EXPECT_GT(rgba[key_px], rgba[0]); +} + +} // namespace +} // namespace mocktail::vr diff --git a/tests/vr/virtual_keyboard_test.cc b/tests/vr/virtual_keyboard_test.cc index 1dfb6da8..448e5944 100644 --- a/tests/vr/virtual_keyboard_test.cc +++ b/tests/vr/virtual_keyboard_test.cc @@ -346,13 +346,141 @@ TEST(VirtualKeyboardTest, LayoutRowsFillThePanelAndCloseKeyHides) { Driver d; d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); const auto closed = d.Frame(AimAtKey(d.keyboard, "Close"), 1.f); - EXPECT_TRUE(closed.closed); EXPECT_EQ(closed.scancodes, std::vector{kEscape}); EXPECT_FALSE(d.keyboard.visible()); - EXPECT_FALSE(d.keyboard.CapturesTrigger(1)); EXPECT_TRUE(d.Frame(AimAtKey(d.keyboard, "q"), 0.f).text.empty()); } +TEST(VirtualKeyboardTest, TriggerStaysWithheldAfterCloseUntilReleased) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const Pose close = AimAtKey(d.keyboard, "Close"); + d.Frame(close, 1.f); + ASSERT_FALSE(d.keyboard.visible()); + // The trigger that pressed Close is still down; Roblox must not see it as + // a click on whatever is behind the panel. + EXPECT_TRUE(d.keyboard.CapturesTrigger(1)); + EXPECT_FALSE(d.keyboard.CapturesTrigger(0)); + d.Frame(close, 1.f); + EXPECT_TRUE(d.keyboard.CapturesTrigger(1)); + d.Frame(close, 0.f); + EXPECT_FALSE(d.keyboard.CapturesTrigger(1)); + d.Frame(close, 1.f); // a new press belongs to Roblox + EXPECT_FALSE(d.keyboard.CapturesTrigger(1)); +} + +TEST(VirtualKeyboardTest, TriggerStaysWithheldWhenThePanelHidesMidPress) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const Pose enter = AimAtKey(d.keyboard, "Enter"); + d.Frame(enter, 1.f); + d.keyboard.Hide(); // Enter made the TextBox release focus + EXPECT_TRUE(d.keyboard.CapturesTrigger(1)); + d.Frame(enter, 0.f); + EXPECT_FALSE(d.keyboard.CapturesTrigger(1)); + // A hand that was only hovering is free at once. + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + d.Frame(enter, 0.f); + d.keyboard.Hide(); + EXPECT_FALSE(d.keyboard.CapturesTrigger(1)); +} + +TEST(VirtualKeyboardTest, HeldTriggerTypesNothingWhenThePanelReopens) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + d.Frame(AimAtKey(d.keyboard, "Close"), 1.f); + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const Pose q = AimAtKey(d.keyboard, "q"); + EXPECT_TRUE(d.Frame(q, 1.f).text.empty()); + d.Frame(q, 0.f); + EXPECT_EQ(d.Frame(q, 1.f).text, std::vector{"q"}); +} + +TEST(VirtualKeyboardTest, CursorMovesRedrawAtMostThirtyTimesASecond) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const VirtualKey space = d.keyboard.keys()[FindKey(d.keyboard, "Space")]; + const float v = (space.v0 + space.v1) / 2; + float u = 0.10f; + d.Frame(AimAt(d.keyboard, u, v), 0.f); + const std::uint64_t before = d.keyboard.revision(); + // 30 frames at 90 Hz sliding along one key: 330 ms of pointer motion. + for (int frame = 0; frame < 30; ++frame) { + u += 0.01f; + d.Frame(AimAt(d.keyboard, u, v), 0.f); + } + const std::uint64_t redraws = d.keyboard.revision() - before; + EXPECT_LE(redraws, 11u); + EXPECT_GE(redraws, 5u); + // Once the pointer rests, the drawn cursor catches up with it. + d.Frame(AimAt(d.keyboard, u, v), 0.f, 1, 40 * kMs); + float drawn_u = 0.f, drawn_v = 0.f; + ASSERT_TRUE(d.keyboard.cursor(1, &drawn_u, &drawn_v)); + EXPECT_NEAR(drawn_u, u, 1e-3f); +} + +TEST(VirtualKeyboardTest, BothCursorsKeepUpWhileBothHandsMove) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const VirtualKey space = d.keyboard.keys()[FindKey(d.keyboard, "Space")]; + const float v = (space.v0 + space.v1) / 2; + float left_u = 0.10f, right_u = 0.50f; + auto frame = [&] { + VirtualKeyboardPointer pointers[2]; + pointers[0].valid = pointers[1].valid = true; + pointers[0].aim = AimAt(d.keyboard, left_u, v); + pointers[1].aim = AimAt(d.keyboard, right_u, v); + d.now += 11 * kMs; + d.keyboard.Update(d.now, pointers); + }; + frame(); + // 30 frames at 90 Hz, both hands sliding along one key. + for (int step = 0; step < 30; ++step) { + left_u += 0.005f; + right_u += 0.005f; + frame(); + } + // Neither drawn cursor trails its hand by more than one redraw interval. + float drawn_u = 0.f, drawn_v = 0.f; + ASSERT_TRUE(d.keyboard.cursor(0, &drawn_u, &drawn_v)); + EXPECT_NEAR(drawn_u, left_u, 0.02f); + ASSERT_TRUE(d.keyboard.cursor(1, &drawn_u, &drawn_v)); + EXPECT_NEAR(drawn_u, right_u, 0.02f); +} + +TEST(VirtualKeyboardTest, HeldTriggerStaysWithheldWhenAReopenedPanelHides) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + const Pose close = AimAtKey(d.keyboard, "Close"); + d.Frame(close, 1.f); + // The panel reopens and hides again during the same trigger press. + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + d.Frame(close, 1.f); + d.keyboard.Hide(); + EXPECT_TRUE(d.keyboard.CapturesTrigger(1)); + d.Frame(close, 1.f); + EXPECT_TRUE(d.keyboard.CapturesTrigger(1)); + d.Frame(close, 0.f); + EXPECT_FALSE(d.keyboard.CapturesTrigger(1)); +} + +TEST(VirtualKeyboardTest, HeldTriggerStaysWithheldWhileAimTrackingDrops) { + Driver d; + d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); + d.Frame(AimAtKey(d.keyboard, "Close"), 1.f); + ASSERT_TRUE(d.keyboard.CapturesTrigger(1)); + // The aim pose is lost for a frame; the trigger is still down. + VirtualKeyboardPointer lost[2]; + lost[1].trigger = 1.f; + d.now += 11 * kMs; + d.keyboard.Update(d.now, lost); + EXPECT_TRUE(d.keyboard.CapturesTrigger(1)); + lost[1].trigger = 0.f; + d.now += 11 * kMs; + d.keyboard.Update(d.now, lost); + EXPECT_FALSE(d.keyboard.CapturesTrigger(1)); +} + TEST(VirtualKeyboardTest, RevisionMovesOnlyWhenTheImageChanges) { Driver d; d.keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); @@ -413,57 +541,5 @@ TEST(VirtualKeyboardTest, TextMirrorFollowsTheFocusedTextBox) { EXPECT_TRUE(CurrentVirtualKeyboardText().utf8.empty()); } -TEST(VirtualKeyboardTest, RasterizerDrawsFieldTextAndCaret) { - VirtualKeyboard keyboard; - keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); - VirtualKeyboardRasterizer rasterizer; - auto field_light_pixels = [&](const std::vector& rgba) { - const int bottom = static_cast(VirtualKeyboard::kFieldFraction * - VirtualKeyboard::kTextureHeight); - int count = 0; - for (int y = 8; y < bottom - 8; ++y) - for (int x = 30; x < VirtualKeyboard::kTextureWidth - 30; ++x) - if (rgba[(static_cast(y) * - VirtualKeyboard::kTextureWidth + x) * 4 + 1] > 120) - ++count; - return count; - }; - std::vector empty, typed, masked; - if (!rasterizer.Draw(keyboard, &empty)) GTEST_SKIP() << "no host font"; - VirtualKeyboardText text; - text.utf8 = "Hello world"; - text.cursor_utf16 = 11; - text.revision = 1; - keyboard.SetText(text); - ASSERT_TRUE(rasterizer.Draw(keyboard, &typed)); - // Text adds light glyph pixels; the empty field only has its caret. - EXPECT_GT(field_light_pixels(typed), field_light_pixels(empty) + 200); - text.password = true; - text.revision = 2; - keyboard.SetText(text); - ASSERT_TRUE(rasterizer.Draw(keyboard, &masked)); - EXPECT_NE(masked, typed); // dots, not the letters - EXPECT_GT(field_light_pixels(masked), field_light_pixels(empty)); -} - -TEST(VirtualKeyboardTest, RasterizerFillsAnOpaqueTexture) { - VirtualKeyboard keyboard; - keyboard.Show(HeadAt(0.f, 1.6f, 0.f)); - VirtualKeyboardRasterizer rasterizer; - std::vector rgba; - (void)rasterizer.Draw(keyboard, &rgba); // labels depend on host fonts - ASSERT_EQ(rgba.size(), static_cast( - VirtualKeyboard::kTextureWidth) * - VirtualKeyboard::kTextureHeight * 4); - for (std::size_t i = 3; i < rgba.size(); i += 4) ASSERT_EQ(rgba[i], 255); - // A key is lighter than the background behind it. - const auto& q = keyboard.keys()[FindKey(keyboard, "q")]; - const int x = static_cast(q.u0 * VirtualKeyboard::kTextureWidth) + 20; - const int y = static_cast(q.v0 * VirtualKeyboard::kTextureHeight) + 20; - const std::size_t key_px = - (static_cast(y) * VirtualKeyboard::kTextureWidth + x) * 4; - EXPECT_GT(rgba[key_px], rgba[0]); -} - } // namespace } // namespace mocktail::vr From 393297a43924fbf58e2d6b7a0542ceb5b587e65a Mon Sep 17 00:00:00 2001 From: vii Date: Tue, 13 Oct 2026 18:37:35 -0500 Subject: [PATCH 33/58] feat(vr): support Roblox 3222 (2.742.1059) --- scripts/find_roblox_vr_offsets.py | 8 +++ src/vr/roblox_vr_device_bridge.cc | 12 ++++ tests/vr/roblox_vr_device_bridge_test.cc | 91 +++++++++++++++++++++++- 3 files changed, 108 insertions(+), 3 deletions(-) diff --git a/scripts/find_roblox_vr_offsets.py b/scripts/find_roblox_vr_offsets.py index a90eff1b..c4259aea 100755 --- a/scripts/find_roblox_vr_offsets.py +++ b/scripts/find_roblox_vr_offsets.py @@ -1224,6 +1224,12 @@ def find_vr_offsets(elf_path: pathlib.Path) -> dict[str, Any]: raise AnalyzerError( f"ambiguous {label} contract matches: {listed}") state_slot, state_getter = state_matches[0] + # is_state_getter accepts exactly one direct call: the memcpy. + state_memcpy = next( + instruction.operands[0].imm + for instruction in disasm.function(*table.bounds(state_getter)) + if instruction.mnemonic == "call" and instruction.operands and + instruction.operands[0].type == capstone_x86.X86_OP_IMM) eye_slot, eye_getter = eye_matches[0] initializer_slot, eye_initializer = initializer_matches[0] for label, slot, expected in ( @@ -1430,6 +1436,8 @@ def find_vr_offsets(elf_path: pathlib.Path) -> dict[str, Any]: "vr_bridge_supplement": { "state_getter_vtable_slot_offset": f"0x{state_slot:x}", + "state_getter_memcpy_rva": + f"0x{state_memcpy:x}", "eye_getter_vtable_slot_offset": f"0x{eye_slot:x}", "eye_initializer_vtable_slot_offset": diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc index b494c637..923d08fb 100644 --- a/src/vr/roblox_vr_device_bridge.cc +++ b/src/vr/roblox_vr_device_bridge.cc @@ -418,6 +418,18 @@ constexpr VrBuildContract kContracts[] = { 0x118, 0x140, "3092 (2.738.1397)"}, + {"a67a91ef96c700c0730acd36a250c2f93cfafbc1", + {0x7083a48, 0x549c884, 0x549cc66, 0x549cc56, 0x549cc88, 0x74d1640, + 0x190}, + 0x6daad30, + 0x4fd8cb0, + 0x264bb76, + 0x3a0, + 0x74cf430, + 0x1e2fcf0, + 0x108, + 0x130, + "3222 (2.742.1059)"}, }; const VrBuildContract* FindVrBuildContract(std::string_view build_id) { diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc index 93b278d5..60c364c9 100644 --- a/tests/vr/roblox_vr_device_bridge_test.cc +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -10,9 +10,12 @@ #include #include "mocktail/vr/openxr_backend.h" #include +#include #include #include +#include #include +#include #include #include "compat/build_profile.h" @@ -314,7 +317,8 @@ TEST(VrConstructorContract, VerifiesRelocatedVtableFor3092) { TEST(VrPointerContract, PinsWorkspaceAndCameraSlotForEachBuild) { for (const auto* id : {"ade08266c67aee88ec9c1d00902150e1684dad3a", - "5f0704edd9064f566ee3d6df2bd2fabbcc709f03"}) { + "5f0704edd9064f566ee3d6df2bd2fabbcc709f03", + "a67a91ef96c700c0730acd36a250c2f93cfafbc1"}) { const auto& contract = *internal::FindVrBuildContract(id); std::array code{}; // movsxd rax,[rsi+selector] and movss xmm0,[r14+HeadScale]. @@ -887,12 +891,12 @@ namespace { TEST(VrSupportedBuilds, NamesEveryVerifiedBuild) { EXPECT_EQ(DescribeSupportedVrBuilds(), - "2998 (2.736.1408) and 3092 (2.738.1397)"); + "2998 (2.736.1408), 3092 (2.738.1397) and 3222 (2.742.1059)"); } TEST(VrSupportedBuilds, InstallNamesTheBuildAndTheSupportedOnes) { compat::BuildProfile profile; - profile.elf_build_id = "a67a91ef96c700c0730acd36a250c2f93cfafbc1"; // 3222 + profile.elf_build_id = "00112233445566778899aabbccddeeff00112233"; profile.vr_debug_device_bridge = *FindRobloxVrDeviceProfile( "ade08266c67aee88ec9c1d00902150e1684dad3a"); RobloxVrDeviceBridge bridge; @@ -990,3 +994,84 @@ TEST(VrPointerLayout, VerifiedBuildsKeepTheirOffsets) { } // namespace } // namespace mocktail::vr + +namespace mocktail::vr { +namespace { + +TEST(VrSupportedBuilds, Build3222CarriesTheOffsetFindersValues) { + // scripts/find_roblox_vr_offsets.py on the 2.742.1059 x86-64 libroblox.so. + const auto* contract = internal::FindVrBuildContract( + "a67a91ef96c700c0730acd36a250c2f93cfafbc1"); + ASSERT_NE(contract, nullptr); + EXPECT_EQ(contract->device.vtable_rva, 0x7083a48u); + EXPECT_EQ(contract->device.constructor_rva, 0x549c884u); + EXPECT_EQ(contract->device.state_getter_rva, 0x549cc66u); + EXPECT_EQ(contract->device.eye_getter_rva, 0x549cc56u); + EXPECT_EQ(contract->device.eye_initializer_rva, 0x549cc88u); + EXPECT_EQ(contract->device.emulator_flag_storage_rva, 0x74d1640u); + EXPECT_EQ(contract->device.device_create_framebuffer_vtable_offset, 0x190u); + EXPECT_EQ(contract->state_memcpy_rva, 0x6daad30u); + EXPECT_EQ(contract->pointer_frame_rva, 0x4fd8cb0u); + EXPECT_EQ(contract->workspace_getter_rva, 0x264bb76u); + EXPECT_EQ(contract->camera_vtable_offset, 0x3a0u); + EXPECT_EQ(contract->hand_pitch_storage_rva, 0x74cf430u); + EXPECT_EQ(contract->haptic_sink_rva, 0x1e2fcf0u); + EXPECT_EQ(contract->pointer_hand_selector_offset, 0x108u); + EXPECT_EQ(contract->camera_head_scale_offset, 0x130u); +} + +// MOCKTAIL_TEST_VR_PAYLOADS names a payload store: -/libroblox.so. +TEST(VrSupportedBuilds, ContractsMatchTheRealLibraries) { + const char* root = std::getenv("MOCKTAIL_TEST_VR_PAYLOADS"); + if (root == nullptr || root[0] == '\0') { + GTEST_SKIP() << "MOCKTAIL_TEST_VR_PAYLOADS is not set"; + } + int checked = 0; + for (const auto& [code_name, id] : + {std::pair{ + "2998", "ade08266c67aee88ec9c1d00902150e1684dad3a"}, + {"3092", "5f0704edd9064f566ee3d6df2bd2fabbcc709f03"}, + {"3222", "a67a91ef96c700c0730acd36a250c2f93cfafbc1"}}) { + const std::string path = + std::string(root) + "/" + code_name + "-" + id + "/libroblox.so"; + std::ifstream file(path, std::ios::binary); + if (!file) continue; + const std::vector bytes((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + const auto* contract = internal::FindVrBuildContract(id); + ASSERT_NE(contract, nullptr) << id; + // The first load segment maps file offset 0 at address 0, so a code RVA + // is also its file offset. + const auto code_at = [&](std::uintptr_t rva) { + return reinterpret_cast(bytes.data()) + rva; + }; + const auto size_at = [&](std::uintptr_t rva) { + return rva < bytes.size() ? bytes.size() - rva : 0; + }; + const auto& device = contract->device; + EXPECT_TRUE(internal::HasExpectedVrStateGetterContract( + code_at(device.state_getter_rva), size_at(device.state_getter_rva), + *contract)) + << code_name; + EXPECT_TRUE(internal::HasExpectedVrEyeGetterContract( + code_at(device.eye_getter_rva), size_at(device.eye_getter_rva))) + << code_name; + EXPECT_TRUE(internal::HasExpectedVrConstructorContract( + code_at(device.constructor_rva), size_at(device.constructor_rva), + device)) + << code_name; + EXPECT_TRUE(internal::HasExpectedVrEyeInitializerContract( + code_at(device.eye_initializer_rva), + size_at(device.eye_initializer_rva), device)) + << code_name; + EXPECT_TRUE(internal::HasExpectedVrPointerContract( + code_at(contract->pointer_frame_rva), + size_at(contract->pointer_frame_rva), *contract)) + << code_name; + ++checked; + } + if (checked == 0) GTEST_SKIP() << "no payload found under " << root; +} + +} // namespace +} // namespace mocktail::vr From 123968be89c4eed324d02092d62503e1d50a22aa Mon Sep 17 00:00:00 2001 From: vii Date: Tue, 13 Oct 2026 18:37:35 -0500 Subject: [PATCH 34/58] test(assets): pin overlay priority and path safety when the root is the content folder --- tests/asset_manager_test.cc | 48 +++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/asset_manager_test.cc b/tests/asset_manager_test.cc index 8516ca58..9da27691 100644 --- a/tests/asset_manager_test.cc +++ b/tests/asset_manager_test.cc @@ -447,4 +447,52 @@ TEST_F(AssetManagerTest, ResolvesShadersWhenAssetRootPointsToContentOrAssets) { AAsset_close(fallback_asset); } +TEST_F(AssetManagerTest, OverlayStillWinsWhenTheRootIsTheContentFolder) { + WriteAsset("content/textures/ArrowCursor.png", "stock-arrow"); + const std::string overlay_root = MakeTempRoot(); + ASSERT_FALSE(overlay_root.empty()); + WriteAssetAt(overlay_root, "content/textures/ArrowCursor.png", + "custom-overlay-arrow"); + ASSERT_EQ(setenv("MOCKTAIL_ASSET_OVERLAY_DIR", overlay_root.c_str(), 1), 0); + const std::string content_root = root_ + "/content"; + ASSERT_EQ(setenv("MOCKTAIL_ASSET_ROOT", content_root.c_str(), 1), 0); + + AAssetManager* manager = AAssetManager_fromJava(nullptr, nullptr); + for (const char* name : + {"content/textures/ArrowCursor.png", "textures/ArrowCursor.png", + "content/fonts/../textures/ArrowCursor.png"}) { + AAsset* asset = AAssetManager_open(manager, name, 0); + ASSERT_NE(asset, nullptr) << name; + EXPECT_EQ(std::string(static_cast(AAsset_getBuffer(asset)), + static_cast(AAsset_getLength(asset))), + "custom-overlay-arrow") + << name; + AAsset_close(asset); + } +} + +TEST_F(AssetManagerTest, DotDotPathsCannotLeaveTheAssetFolders) { + // The file sits beside the asset root, where only an escape can reach it. + const std::string outside = root_ + "/../outside-" + + std::to_string(static_cast(getpid())) + + ".txt"; + { std::ofstream(outside) << "secret"; } + const std::string outside_name = + outside.substr(outside.find_last_of('/') + 1); + const std::string content_root = root_ + "/content"; + WriteAsset("content/inside.txt", "inside"); + AAssetManager* manager = AAssetManager_fromJava(nullptr, nullptr); + for (const std::string& root : {root_, content_root}) { + ASSERT_EQ(setenv("MOCKTAIL_ASSET_ROOT", root.c_str(), 1), 0); + for (const std::string& name : + {"../" + outside_name, "content/../../" + outside_name, + "content/../../../" + outside_name, "a/../../" + outside_name, + std::string("content/.."), std::string("content/../")}) { + EXPECT_EQ(AAssetManager_open(manager, name.c_str(), 0), nullptr) + << root << " " << name; + } + } + std::remove(outside.c_str()); +} + } // namespace From 1ff05e4e6fe3d538d60009dfc04e48f1cbf56091 Mon Sep 17 00:00:00 2001 From: vii Date: Tue, 13 Oct 2026 18:37:35 -0500 Subject: [PATCH 35/58] fix(vr): do not recover an XR session when the restarted runtime uses another GPU --- include/mocktail/vr/openxr_backend.h | 2 ++ src/vr/openxr_backend.cc | 30 ++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index 2b94a87b..18b05464 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -305,6 +305,8 @@ class OpenXrBackend final { void HandleSessionState(int state); void HandleReferenceSpaceChange(std::int64_t change_time); bool RecoverSessionLocked(std::uint64_t now_ns); + // False only when the runtime names a GPU other than the guest device's. + bool RuntimeUsesGuestGpuLocked(); void PollSessionEvents(); bool SupportsPresentQueueLocked(VkQueue queue) const; VkQueue GraphicsQueueForPresentLocked(VkQueue present) const; diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 5dec0cd6..64615eeb 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -2059,6 +2059,27 @@ void OpenXrBackend::HandleReferenceSpaceChange(std::int64_t change_time) { } } +bool OpenXrBackend::RuntimeUsesGuestGpuLocked() { + // A runtime that cannot answer for this VkInstance is not proof of a + // mismatch; xrCreateSession then decides. + if (vk_instance_ == VK_NULL_HANDLE) return true; + PFN_xrVoidFunction function = nullptr; + if (XR_FAILED(xrGetInstanceProcAddr(static_cast(xr_instance_), + "xrGetVulkanGraphicsDevice2KHR", + &function)) || + function == nullptr) + return true; + auto info = XrInfo( + XR_TYPE_VULKAN_GRAPHICS_DEVICE_GET_INFO_KHR); + info.systemId = xr_system_; + info.vulkanInstance = vk_instance_; + VkPhysicalDevice runtime_device = VK_NULL_HANDLE; + if (XR_FAILED(reinterpret_cast(function)( + static_cast(xr_instance_), &info, &runtime_device))) + return true; + return runtime_device == vk_physical_device_; +} + bool OpenXrBackend::RecoverSessionLocked(std::uint64_t now_ns) { if (!session_recovery_pending_ || (graphics_api_ == VrGraphicsApi::kVulkan @@ -2084,6 +2105,15 @@ bool OpenXrBackend::RecoverSessionLocked(std::uint64_t now_ns) { } xr_instance_stale_ = false; Log(" [vr-backend] OpenXR instance recreated for session recovery\n"); + if (graphics_api_ == VrGraphicsApi::kVulkan && + !RuntimeUsesGuestGpuLocked()) { + // The guest device cannot move to another GPU, so no later retry can + // succeed: stop recovering and keep playing without headset output. + session_recovery_pending_ = false; + Log(" [vr-backend] XR OUTPUT UNAVAILABLE: the restarted OpenXR runtime " + "uses a different GPU than Roblox. Restart Nightcap to use VR.\n"); + return false; + } } // nullptr means reuse the verified queue family of this existing VkDevice. if (!(graphics_api_ == VrGraphicsApi::kVulkan From a30f05fc5b805bf4bbecaea1ca2c6b6cc15f5b44 Mon Sep 17 00:00:00 2001 From: vii Date: Tue, 13 Oct 2026 18:37:35 -0500 Subject: [PATCH 36/58] fix(vr): keep the bridge and backend alive until exit for guest threads still in a hook --- src/main.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main.cc b/src/main.cc index 46cc1bb0..5d579ab8 100644 --- a/src/main.cc +++ b/src/main.cc @@ -998,8 +998,14 @@ int main(int argc, char* argv[]) { mocktail::runtime::RobloxFullscreenRuntimeBridge fullscreen_bridge; mocktail::audio::RobloxOutputDeviceBridge output_device_bridge; - mocktail::vr::RobloxVrDeviceBridge vr_device_bridge; - mocktail::vr::OpenXrBackend xr_backend; + // Never destroyed: Shutdown() stops waiting after 2 s, and a guest thread + // still inside a hook then keeps using this object until the process ends. + static auto& vr_device_bridge = *new mocktail::vr::RobloxVrDeviceBridge; + // Never destroyed, for the same reason: the hooks call into the backend. + static auto& xr_backend = *new mocktail::vr::OpenXrBackend; + struct DisarmOnExit { + ~DisarmOnExit() { xr_backend.Disarm(); } + } disarm_xr_backend_on_exit; if (command_line.options.mode == mocktail::runtime::CommandMode::kRun) { const mocktail::Status fullscreen_status = fullscreen_bridge.Install(compatibility.profile); From 9826535102cbf241d5ac42985338a02f07a67746 Mon Sep 17 00:00:00 2001 From: vii Date: Tue, 13 Oct 2026 19:58:40 -0500 Subject: [PATCH 37/58] fix(vr): align the IsVRAppBuild hook so unoptimized builds do not crash Roblox --- src/vr/roblox_vr_device_bridge.cc | 5 ++++- tests/vr/roblox_vr_device_bridge_test.cc | 16 ++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc index 923d08fb..aa756782 100644 --- a/src/vr/roblox_vr_device_bridge.cc +++ b/src/vr/roblox_vr_device_bridge.cc @@ -302,7 +302,10 @@ void Log(const char *format, ...) { std::uintptr_t* g_is_vr_app_build_slot = nullptr; std::uintptr_t g_is_vr_app_build_original = 0; -extern "C" bool mocktail_vr_is_vr_app_build(const void*) { +// Aligned: Roblox calls this through a C++ member function pointer, which +// reads an odd address as a vtable offset. Unoptimized builds do not align. +extern "C" __attribute__((aligned(16))) bool mocktail_vr_is_vr_app_build( + const void*) { HookCall in_flight; auto* bridge = g_active_bridge.load(std::memory_order_seq_cst); return bridge != nullptr && bridge->active(); diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc index 60c364c9..75287272 100644 --- a/tests/vr/roblox_vr_device_bridge_test.cc +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -1075,3 +1075,19 @@ TEST(VrSupportedBuilds, ContractsMatchTheRealLibraries) { } // namespace } // namespace mocktail::vr + +extern "C" bool mocktail_vr_is_vr_app_build(const void*); + +namespace mocktail::vr { +namespace { + +// In a C++ member function pointer an odd address means a virtual call. +TEST(VrIsVrAppBuildHook, HasAnEvenAddress) { + // volatile: the compiler otherwise assumes the low bit is clear. + bool (*volatile hook)(const void*) = &mocktail_vr_is_vr_app_build; + const auto address = reinterpret_cast(hook); + EXPECT_EQ(address % 2, 0u) << std::hex << address; +} + +} // namespace +} // namespace mocktail::vr From fcc8b5382a53add8e360e01f284954a5f97800b1 Mon Sep 17 00:00:00 2001 From: vii Date: Tue, 13 Oct 2026 19:58:40 -0500 Subject: [PATCH 38/58] fix(vr): turn off the desktop flyout side panel in VR --- src/runtime/roblox_desktop_app_policy.cc | 10 ++++++++- tests/roblox_desktop_app_policy_test.cc | 28 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/runtime/roblox_desktop_app_policy.cc b/src/runtime/roblox_desktop_app_policy.cc index ee2b0239..84eb2e6b 100644 --- a/src/runtime/roblox_desktop_app_policy.cc +++ b/src/runtime/roblox_desktop_app_policy.cc @@ -22,6 +22,8 @@ namespace { constexpr std::uintmax_t kMaximumJsonBytes = 16U * 1024U * 1024U; constexpr std::string_view kAppPolicySuffix = ":app-policy"; +constexpr const char* kFlyoutVariantFlag = "FIntFlyoutTestVariantRollout12"; +constexpr const char* kDesktopFlyoutVariant = "2"; constexpr std::string_view kDesktopAppPolicyOverride = "FStringAppConfigurationOverrideAppPolicy"; std::atomic g_temporary_sequence{0}; @@ -364,7 +366,7 @@ bool MergeDesktopAppPolicyClientSettingsOverride(std::string_view policy_json, // Roblox enrolls desktop clients in the flyout side panel through a // server-side experiment that does not reach this client. Variant 2 is the // desktop side panel; an explicit user override is preserved. - overrides.emplace("FIntFlyoutTestVariantRollout12", "2"); + overrides.emplace(kFlyoutVariantFlag, kDesktopFlyoutVariant); *merged_json = overrides.dump(); return true; } @@ -417,6 +419,12 @@ bool MergeVrAppPolicyClientSettingsOverrides( } } policy["ThrottleFramerate"] = false; + // The desktop side panel cannot be reached with the VR pointer. fflags.json + // is the later channel, so a user's own value there still decides. + const auto flyout = client_settings.find(kFlyoutVariantFlag); + if (flyout != client_settings.end() && *flyout == kDesktopFlyoutVariant) { + *flyout = "0"; + } const std::string encoded_policy = policy.dump(); client_settings[std::string(kDesktopAppPolicyOverride)] = encoded_policy; fast_flags[std::string(kDesktopAppPolicyOverride)] = encoded_policy; diff --git a/tests/roblox_desktop_app_policy_test.cc b/tests/roblox_desktop_app_policy_test.cc index d7b1b52f..606d0558 100644 --- a/tests/roblox_desktop_app_policy_test.cc +++ b/tests/roblox_desktop_app_policy_test.cc @@ -391,6 +391,34 @@ TEST(RobloxDesktopAppPolicyTest, VrRejectsMalformedOverridesWithoutErasingThem) } } +TEST(RobloxDesktopAppPolicyTest, VrTurnsOffTheDesktopFlyoutSidePanel) { + std::string desktop, client, fast, error; + ASSERT_TRUE(MergeDesktopAppPolicyClientSettingsOverride("{}", "{}", &desktop, + &error)) + << error; + ASSERT_EQ(nlohmann::json::parse(desktop)["FIntFlyoutTestVariantRollout12"], + "2"); + + ASSERT_TRUE(MergeVrAppPolicyClientSettingsOverrides(false, desktop, "{}", + &client, &fast, &error)); + EXPECT_EQ(nlohmann::json::parse(client)["FIntFlyoutTestVariantRollout12"], + "2"); + + ASSERT_TRUE(MergeVrAppPolicyClientSettingsOverrides(true, desktop, "{}", + &client, &fast, &error)); + EXPECT_EQ(nlohmann::json::parse(client)["FIntFlyoutTestVariantRollout12"], + "0"); + EXPECT_FALSE( + nlohmann::json::parse(fast).contains("FIntFlyoutTestVariantRollout12")); + + // fflags.json is the later channel, so a user's own value still decides. + ASSERT_TRUE(MergeVrAppPolicyClientSettingsOverrides( + true, desktop, R"({"FIntFlyoutTestVariantRollout12":"2"})", &client, + &fast, &error)); + EXPECT_EQ(nlohmann::json::parse(fast)["FIntFlyoutTestVariantRollout12"], + "2"); +} + } // namespace } // namespace runtime } // namespace mocktail From 87ac6ee5721d7b4266f11eaa8658315f0ef53005 Mon Sep 17 00:00:00 2001 From: vii Date: Tue, 13 Oct 2026 19:58:40 -0500 Subject: [PATCH 39/58] test(vr): cover the keyboard upload and quad layer on a real OpenXR runtime --- tests/vr/gles_transport_test.cc | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/vr/gles_transport_test.cc b/tests/vr/gles_transport_test.cc index 60a960f7..eda6d768 100644 --- a/tests/vr/gles_transport_test.cc +++ b/tests/vr/gles_transport_test.cc @@ -24,6 +24,14 @@ struct OpenXrBackendTestAccess { static void LoseSession(OpenXrBackend &b) { b.HandleSessionState(XR_SESSION_STATE_LOSS_PENDING); } + static bool KeyboardVisible(OpenXrBackend &b) { return b.keyboard_.visible(); } + static bool KeyboardFailed(OpenXrBackend &b) { + return b.keyboard_swapchain_failed_; + } + static bool KeyboardUploaded(OpenXrBackend &b) { + return b.keyboard_uploaded_revision_ != 0 && + b.keyboard_uploaded_revision_ == b.keyboard_.revision(); + } }; namespace { #define GL(name) \ @@ -256,5 +264,49 @@ TEST_F(GlesVrTransportTest, OpenXrEglSessionSubmitsAndRecovers) { backend.DetachGlesContext(context); } } +TEST_F(GlesVrTransportTest, OpenXrKeyboardUploadsWhileRobloxWantsInput) { + if (!std::getenv("MOCKTAIL_TEST_XR_GLES")) + GTEST_SKIP() << "Opt-in: requires a connected or simulated OpenXR runtime"; + static bool wants_input = true; + VirtualKeyboardHost host; + host.input_requested = [](void *) { return wants_input; }; + SetVirtualKeyboardHost(&host); + OpenXrBackend backend; + auto status = backend.Arm(VrGraphicsApi::kOpenGles); + ASSERT_TRUE(status.ok()) << status.message(); + status = backend.AttachGlesContext( + SDL_EGL_GetCurrentDisplay(), SDL_EGL_GetCurrentConfig(), context, + reinterpret_cast(SDL_EGL_GetProcAddress("eglGetProcAddress")), + Resolve); + ASSERT_TRUE(status.ok()) << status.message(); + auto &t = OpenXrBackendTestAccess::Transport(backend); + const GLuint red = Texture(&t, 1, 0, 0), green = Texture(&t, 0, 1, 0); + auto *owner = reinterpret_cast(123); + const GLuint fbs[2] = {Eye(t, 0, red), Eye(t, 1, green)}; + const auto run_frames = [&](std::uint64_t until) { + for (int i = 0; i < 300 && backend.submitted_frames() < until; ++i) { + backend.NoteGlesPresent(); + auto pose = backend.PublishedHeadPose(); + if (!pose.valid) { + SDL_Delay(10); + continue; + } + backend.NotePoseApplied(owner, pose.frame); + BindEye(t, 0, fbs[0], pose.frame, owner); + BindEye(t, 1, fbs[1], pose.frame, owner); + } + }; + run_frames(10); + EXPECT_GE(backend.submitted_frames(), 10u); + EXPECT_TRUE(OpenXrBackendTestAccess::KeyboardVisible(backend)); + EXPECT_TRUE(OpenXrBackendTestAccess::KeyboardUploaded(backend)); + EXPECT_FALSE(OpenXrBackendTestAccess::KeyboardFailed(backend)); + wants_input = false; + run_frames(backend.submitted_frames() + 5); + EXPECT_FALSE(OpenXrBackendTestAccess::KeyboardVisible(backend)); + EXPECT_EQ(GL(GetError)(), GL_NO_ERROR); + SetVirtualKeyboardHost(nullptr); + backend.DetachGlesContext(context); +} } // namespace } // namespace mocktail::vr From 0b0d8067d8252bf914a6a4d2318dee9f26fa8d12 Mon Sep 17 00:00:00 2001 From: vii Date: Tue, 13 Oct 2026 20:19:12 -0500 Subject: [PATCH 40/58] docs(vr): list supported builds, move VR detail into docs/VR.md and update the FAQ --- FAQ.md | 14 +++++++----- README.md | 24 +++++++++---------- docs/VR.md | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 84 insertions(+), 21 deletions(-) diff --git a/FAQ.md b/FAQ.md index 0bb03bab..529189ec 100644 --- a/FAQ.md +++ b/FAQ.md @@ -56,15 +56,14 @@ after Roblox added measures against running multiple clients. ## Can I play in VR? -VR is experimental and lives on upstream's `vr` branch, which does not -include the Nightcap changes. Install the +Yes, as an experiment. VR lives on Nightcap's `vr` branch and is not in the +releases yet. It works on Roblox builds 2998, 3092 and 3222. Install the [build dependencies](README.md#building), then switch to that branch and build: ```bash -git remote add upstream https://github.com/komaruworld/mocktail.git -git fetch upstream -git switch -c vr upstream/vr +git fetch origin +git switch vr git submodule update --init --recursive cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DMOCKTAIL_ENABLE_VR=ON cmake --build build -j4 @@ -73,5 +72,8 @@ cmake --build build -j4 Start WiVRn or SteamVR/ALVR, connect your headset, then run: ```bash -./build/mocktail -vr +./build/mocktail --vr ``` + +The [README](README.md#experimental-vr) and [docs/VR.md](docs/VR.md) have the +details, including how to try it without a headset. diff --git a/README.md b/README.md index 30d623d8..756ee7d6 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,15 @@ lefthook install Experimental OpenXR VR support on Linux `x86_64` using WiVRn or SteamVR/ALVR. +VR works on these Roblox builds. On any other build, `--vr` stops at start-up +and names them: + +| Build | Roblox version | +| --- | --- | +| 2998 | 2.736.1408 | +| 3092 | 2.738.1397 | +| 3222 | 2.742.1059 | + ### Build with VR VR support is compiled in when the OpenXR SDK submodule is checked out @@ -272,18 +281,9 @@ MOCKTAIL_VR_RUNTIME=alvr MOCKTAIL_GRAPHICS_BACKEND=direct-vulkan ./build/mocktai If you previously set `XR_RUNTIME_JSON`, unset it before switching runtimes. Use `--no-vr` for desktop mode. -Every VR environment variable, from frame-rate limits to hand pitch, is listed -in [docs/VR.md](docs/VR.md). - -### Typing in VR - -When a text box takes focus (chat, search), a keyboard opens in front of you, -below eye level. It is a full US layout without the function row or -Ctrl/Alt. Point at a key and pull the trigger. While you point at the keyboard, -the trigger types and does not click the game behind it. Hold Backspace to -delete repeatedly. Shift applies to the next key and selects its upper symbol; -Caps Lock stays on until you tap it again. Close dismisses the keyboard and -leaves the text box. Set `MOCKTAIL_VR_KEYBOARD=0` to turn the keyboard off. +A keyboard opens in the headset when a text box takes focus. The rest is in +[docs/VR.md](docs/VR.md): menus and typing, every VR environment variable, +adding a Roblox build, and testing without a headset. ## Known limitations diff --git a/docs/VR.md b/docs/VR.md index 0f2ba044..93d036ca 100644 --- a/docs/VR.md +++ b/docs/VR.md @@ -1,8 +1,23 @@ -# VR settings +# VR Experimental OpenXR VR is opt-in: launch with `--vr`, or set `vr.enabled: true` -in `config.yaml`. These environment variables tune it. Everything here only -applies while VR is active. +in `config.yaml`. Everything here only applies while VR is active. The +[README](../README.md#experimental-vr) covers building and running. + +## Menus and typing + +The home menu uses Roblox's `More` button. The desktop side panel is turned +off in VR, because the VR pointer cannot click it. + +When a text box takes focus (chat, search), a keyboard opens in front of you, +below eye level. It is a full US layout without the function row or +Ctrl/Alt. Point at a key and pull the trigger. While you point at the keyboard, +the trigger types and does not click the game behind it. Hold Backspace to +delete repeatedly. Shift applies to the next key and selects its upper symbol; +Caps Lock stays on until you tap it again. Close dismisses the keyboard and +leaves the text box. Set `MOCKTAIL_VR_KEYBOARD=0` to turn the keyboard off. + +## Environment variables ## Runtime and display @@ -47,3 +62,49 @@ Release builds ignore them. | `MOCKTAIL_VR_POSE_PERIOD_POSITION`, `_ROTATION` | `240`, `360` | Scripted motion period, in frames. | | `MOCKTAIL_VR_XR_EVIDENCE_DIR` | unset | Writes submitted eye images to this directory. | | `MOCKTAIL_VR_XR_EVIDENCE_INTERVAL` | `20` | Frames between evidence images. | + +## Adding a Roblox build + +VR needs addresses that are specific to each Roblox build. To use a supported +build that is not the active one, point `--roblox-lib` at its `libroblox.so`. + +To add a build, run the offset finder on its library: + +```bash +python3 scripts/find_roblox_vr_offsets.py --elf --output offsets.json +``` + +It needs the `capstone` Python package. Add its values to `kContracts` in +`src/vr/roblox_vr_device_bridge.cc`. Nightcap checks those addresses against +Roblox's code before it hooks anything, and leaves VR off if they do not +match. To run the same checks on real libraries, set +`MOCKTAIL_TEST_VR_PAYLOADS` to a folder of `-/libroblox.so` +payloads and run `ctest -R VrSupportedBuilds`. + +## Testing without a headset + +[Monado](https://monado.freedesktop.org) can stand in for a headset. Its +`qwerty` driver gives a head and two controllers that you move with the +keyboard and mouse: + +```bash +QWERTY_ENABLE=1 XRT_DEBUG_GUI=1 monado-service +``` + +Then, in another terminal: + +```bash +XR_RUNTIME_JSON=/usr/share/openxr/1/openxr_monado.json ./build/mocktail --vr +``` + +The keys work while Monado's debug window has focus. The simulated display is +small, so text in the headset view looks coarse. + +Two tests need an OpenXR runtime and skip without one. To run them, start +Monado with no window and opt in: + +```bash +XRT_COMPOSITOR_NULL=1 monado-service & +XR_RUNTIME_JSON=/usr/share/openxr/1/openxr_monado.json MOCKTAIL_TEST_XR_GLES=1 \ + ctest --test-dir build -R GlesVrTransportTest.OpenXr +``` From f3e68fc3b719faf3eb6cdd4d178db622beb7dd66 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Wed, 14 Oct 2026 01:32:32 -0500 Subject: [PATCH 41/58] feat(vr): implement OpenXR controller haptics and bypass mobile suppression - Implement RobloxHapticProtocolBridge for MessageBus HapticProtocol (SupportsHaptics, GetMotors, SetMotorState, UpdateSingletonVibration). - Wire bridge into RobloxExperienceComposition and direct game runtime. - Call JNI NativeGLInterface.nativeUpdateMobileHapticsState during startup and game completion. - Enable FFlagKeepHapticsOnMobileWhenGamepadConnected and FFlagVRHaptics in VR client settings overrides to bypass gamepad haptic suppression in libroblox. - Directly activate KeepHaptics and VRHaptics fast flag bytes in memory within RobloxVrDeviceBridge::Activate for builds 2998 and 3222. - Check xrApplyHapticFeedback result code and add diagnostic logging. --- CMakeLists.txt | 1 + .../runtime/roblox_experience_composition.h | 2 + .../runtime/roblox_haptic_protocol_bridge.h | 41 +++ src/legacy/legacy_runtime.cc | 57 ++- src/runtime/performance_policy.cc | 4 + src/runtime/roblox_experience_composition.cc | 27 +- src/runtime/roblox_haptic_protocol_bridge.cc | 295 +++++++++++++++ src/vr/roblox_vr_device_bridge.cc | 25 +- src/vr/roblox_vr_device_bridge_internal.h | 2 + src/vr/xr_actions.cc | 15 +- tests/CMakeLists.txt | 10 + tests/performance_policy_test.cc | 2 + tests/roblox_experience_composition_test.cc | 12 +- tests/roblox_haptic_protocol_bridge_test.cc | 344 ++++++++++++++++++ tests/vr/roblox_vr_device_bridge_test.cc | 8 + 15 files changed, 832 insertions(+), 13 deletions(-) create mode 100644 include/runtime/roblox_haptic_protocol_bridge.h create mode 100644 src/runtime/roblox_haptic_protocol_bridge.cc create mode 100644 tests/roblox_haptic_protocol_bridge_test.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 43a1dbb7..cc0874c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -297,6 +297,7 @@ add_library(mocktail_runtime STATIC src/runtime/roblox_account_bridge.cc src/runtime/roblox_message_bus_request_bridge.cc src/runtime/roblox_call_protocol_bridge.cc + src/runtime/roblox_haptic_protocol_bridge.cc src/runtime/payload_update_preflight.cc src/runtime/process_launch_policy.cc src/runtime/process_diagnostics.cc diff --git a/include/runtime/roblox_experience_composition.h b/include/runtime/roblox_experience_composition.h index c3bc04f8..d95577fb 100644 --- a/include/runtime/roblox_experience_composition.h +++ b/include/runtime/roblox_experience_composition.h @@ -16,6 +16,7 @@ #include "runtime/owned_pthread.h" #include "runtime/roblox_browser_service_bridge.h" #include "runtime/roblox_call_protocol_bridge.h" +#include "runtime/roblox_haptic_protocol_bridge.h" #include "runtime/roblox_experience_launch_bridge.h" #include "runtime/roblox_experience_presence.h" #include "runtime/roblox_fresh_game_launch_controller.h" @@ -260,6 +261,7 @@ class RobloxExperienceComposition final { std::unique_ptr browser_service_bridge_; std::unique_ptr permissions_bridge_; std::unique_ptr call_protocol_bridge_; + std::unique_ptr haptic_protocol_bridge_; // Optional: a payload without the AccountProtocol entrypoints still starts. std::unique_ptr account_bridge_; std::shared_ptr web_surface_process_; diff --git a/include/runtime/roblox_haptic_protocol_bridge.h b/include/runtime/roblox_haptic_protocol_bridge.h new file mode 100644 index 00000000..f1b54a81 --- /dev/null +++ b/include/runtime/roblox_haptic_protocol_bridge.h @@ -0,0 +1,41 @@ +#ifndef MOCKTAIL_RUNTIME_ROBLOX_HAPTIC_PROTOCOL_BRIDGE_H_ +#define MOCKTAIL_RUNTIME_ROBLOX_HAPTIC_PROTOCOL_BRIDGE_H_ + +#include +#include +#include +#include + +#include "runtime/roblox_message_bus_request_bridge.h" + +namespace mocktail { +namespace runtime { + +using HapticSinkFn = std::function; +using HapticStopFn = std::function; +using HapticStateSyncFn = std::function; + +class RobloxHapticProtocolBridge final { + public: + RobloxHapticProtocolBridge( + JniEnvironmentProvider environment, + RobloxMessageBusSymbols symbols, + RobloxMessageBusObjects objects, + HapticSinkFn haptic_sink = nullptr, + HapticStopFn haptic_stop = nullptr, + HapticStateSyncFn haptic_state_sync = nullptr); + ~RobloxHapticProtocolBridge(); + + Status Initialize(); + Status Shutdown(); + + static std::string GetMotorsJson(); + + private: + std::unique_ptr bridge_; +}; + +} // namespace runtime +} // namespace mocktail +#endif diff --git a/src/legacy/legacy_runtime.cc b/src/legacy/legacy_runtime.cc index b70cdb0e..2afef4f8 100644 --- a/src/legacy/legacy_runtime.cc +++ b/src/legacy/legacy_runtime.cc @@ -75,6 +75,7 @@ #include "runtime/roblox_capability_resolver.h" #include "runtime/roblox_platform_web_symbols.h" #include "runtime/roblox_experience_composition.h" +#include "runtime/roblox_haptic_protocol_bridge.h" #include "runtime/roblox_game_session_native_adapter.h" #include "runtime/roblox_window_input_runtime.h" #include "runtime/roblox_text_input_jni_bridge.h" @@ -214,6 +215,7 @@ using NativeSendGameLoadedFn = void (*)(JNIEnv*, jclass, jstring, jstring, using NativePassSupportedRefreshRatesFn = void (*)(JNIEnv*, jclass, jfloatArray); using NativePassCurrentDisplayRefreshRateFn = void (*)(JNIEnv*, jclass, jfloat); +using NativeUpdateMobileHapticsStateFn = void (*)(JNIEnv*, jclass, jboolean, jstring); using NativeSetStringParamFn = void (*)(JNIEnv*, jclass, jstring); using NativeSetBaseUrlFn = void (*)(JNIEnv*, jclass, jstring, jstring); using NativeSetTaskSchedulerBackgroundModeFn = void (*)(JNIEnv*, jclass, @@ -381,6 +383,7 @@ struct EngineStartupContext { NativePassSupportedRefreshRatesFn native_pass_supported_refresh_rates; NativePassCurrentDisplayRefreshRateFn native_pass_current_display_refresh_rate; NativeNoArgFn native_start_lua_app_dm; + NativeUpdateMobileHapticsStateFn native_update_mobile_haptics_state; }; @@ -4314,6 +4317,23 @@ std::cerr << " [engine] nativeAppBridgeV2InitWithParams recovered\n" std::cout << " [engine] updateAppUISizes returned\n" << std::flush; } + if (context->native_update_mobile_haptics_state) { + env = ensure_env(); + std::cout << " [engine] NativeGLInterface.nativeUpdateMobileHapticsState\n" + << std::flush; + const std::string motors_json = + mocktail::runtime::RobloxHapticProtocolBridge::GetMotorsJson(); + jstring motors_str = env->NewStringUTF(motors_json.c_str()); + if (motors_str != nullptr) { + context->native_update_mobile_haptics_state(env, native_gl_class, + JNI_TRUE, motors_str); + env->DeleteLocalRef(motors_str); + } + std::cout + << " [engine] NativeGLInterface.nativeUpdateMobileHapticsState returned\n" + << std::flush; + } + int init_delay_ms = GetEnvInt("MOCKTAIL_APPBRIDGE_INIT_DELAY_MS", 0); if (init_delay_ms > 0) { std::cout << " [engine] wait after AppBridge init: " << init_delay_ms @@ -4594,6 +4614,16 @@ std::cerr << " [engine] nativeAppBridgeV2InitWithParams recovered\n" std::cout << " [engine] nativeAppBridgeV2SendAppEventOnGameLoaded returned\n" << std::flush; + if (context->native_update_mobile_haptics_state) { + const std::string motors_json = + mocktail::runtime::RobloxHapticProtocolBridge::GetMotorsJson(); + jstring motors_str = env->NewStringUTF(motors_json.c_str()); + if (motors_str != nullptr) { + context->native_update_mobile_haptics_state(env, native_gl_class, + JNI_TRUE, motors_str); + env->DeleteLocalRef(motors_str); + } + } } } @@ -6082,6 +6112,8 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, eager_permissions_bridge; std::unique_ptr eager_call_protocol_bridge; + std::unique_ptr + eager_haptic_protocol_bridge; std::shared_ptr experience_lifecycle_target; std::shared_ptr window_input_runtime; @@ -6378,6 +6410,11 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, roblox_handle, "Java_com_roblox_engine_jni_NativeGLInterface_" "nativeAppBridgeV2SendAppEventOnGameLoaded")); + auto* native_update_mobile_haptics_state = + reinterpret_cast(linker::ResolveSymbol( + roblox_handle, + "Java_com_roblox_engine_jni_NativeGLInterface_" + "nativeUpdateMobileHapticsState")); auto* native_set_asset_path = reinterpret_cast(linker::ResolveSymbol( roblox_handle, @@ -6456,6 +6493,10 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, << reinterpret_cast(native_send_app_ready) << '\n' << " native_send_game_loaded=" << reinterpret_cast(native_send_game_loaded) << '\n' + << " native_update_mobile_haptics_state=" + << reinterpret_cast( + native_update_mobile_haptics_state) + << '\n' << " native_start_lua_app_dm=" << reinterpret_cast(native_start_lua_app_dm) << '\n' << " native_call_messages_from_main_thread=" @@ -6796,17 +6837,22 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, eager_call_protocol_bridge = std::make_unique( environment, symbols.permissions, objects); + eager_haptic_protocol_bridge = + std::make_unique( + environment, symbols.permissions, objects); status = eager_permissions_bridge->Initialize(); if (status.ok()) status = eager_call_protocol_bridge->Initialize(); + if (status.ok()) + status = eager_haptic_protocol_bridge->Initialize(); env->DeleteLocalRef(bus); if (!status.ok()) { std::cerr << "[FATAL] GAME platform protocols did not initialize: " << status.message() << '\n'; return EXIT_FAILURE; } - std::cout << " [platform] direct GAME PermissionsProtocol and " - "CallProtocol ready\n" + std::cout << " [platform] direct GAME PermissionsProtocol, " + "CallProtocol and HapticProtocol ready\n" << std::flush; } @@ -6960,7 +7006,8 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, native_on_fragment_start, native_pass_supported_refresh_rates, native_pass_current_display_refresh_rate, - native_start_lua_app_dm}; + native_start_lua_app_dm, + native_update_mobile_haptics_state}; if (game_session_runtime != nullptr) { if (!game_present_observer.Register( &mocktail::runtime::RobloxGameSessionRuntime:: @@ -7307,6 +7354,10 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, input_shutdown_completed = window_input_runtime->Shutdown().ok() && input_shutdown_completed; } + if (eager_haptic_protocol_bridge != nullptr) { + (void)eager_haptic_protocol_bridge->Shutdown(); + eager_haptic_protocol_bridge.reset(); + } if (eager_call_protocol_bridge != nullptr) { (void)eager_call_protocol_bridge->Shutdown(); eager_call_protocol_bridge.reset(); diff --git a/src/runtime/performance_policy.cc b/src/runtime/performance_policy.cc index 6bbd948a..739478e6 100644 --- a/src/runtime/performance_policy.cc +++ b/src/runtime/performance_policy.cc @@ -407,6 +407,10 @@ bool MergeVrClientSettingsOverrides(bool enabled, std::string_view base_json, // Place the Avatar Editor character 6 studs in front of the VR camera // instead of at the origin when IsVRAppBuild() is active. overrides["FFlagFixVRAvatarPlacement"] = "True"; + // Mobile suppresses vibration when a gamepad connects unless KeepHaptics + // is enabled; VR controllers connect as Gamepad1. + overrides["FFlagKeepHapticsOnMobileWhenGamepadConnected"] = "True"; + overrides["FFlagVRHaptics"] = "True"; } *merged_json = overrides.dump(); return true; diff --git a/src/runtime/roblox_experience_composition.cc b/src/runtime/roblox_experience_composition.cc index cc42fb07..5832e29c 100644 --- a/src/runtime/roblox_experience_composition.cc +++ b/src/runtime/roblox_experience_composition.cc @@ -466,6 +466,17 @@ Status RobloxExperienceComposition::InitializePlatformProtocols() { (void)ReleaseGlobalObjects(); return status; } + auto haptic_protocol_bridge = std::make_unique( + environment_, permissions_symbols_, permissions_objects); + status = haptic_protocol_bridge->Initialize(); + if (!status.ok()) { + (void)call_protocol_bridge->Shutdown(); + (void)permissions_bridge->Shutdown(); + (void)browser_service_bridge->Shutdown(); + (void)web_view_bridge->Shutdown(); + (void)ReleaseGlobalObjects(); + return status; + } // AccountProtocol only shortens a device-integrity login challenge that // cannot succeed here, so its absence or failure never blocks startup. std::unique_ptr account_bridge; @@ -493,12 +504,13 @@ Status RobloxExperienceComposition::InitializePlatformProtocols() { browser_service_bridge_ = std::move(browser_service_bridge); permissions_bridge_ = std::move(permissions_bridge); call_protocol_bridge_ = std::move(call_protocol_bridge); + haptic_protocol_bridge_ = std::move(haptic_protocol_bridge); account_bridge_ = std::move(account_bridge); platform_protocols_initialized_ = true; } std::fprintf(stderr, - " [platform] Android WebViewProtocol, BrowserService and " - "PermissionsProtocol and CallProtocol " + " [platform] Android WebViewProtocol, BrowserService, " + "PermissionsProtocol, CallProtocol and HapticProtocol " "bridges ready\n"); return Status::Ok(); } @@ -1693,6 +1705,7 @@ Status RobloxExperienceComposition::Shutdown() { std::unique_ptr browser_service_bridge; std::unique_ptr permissions_bridge; std::unique_ptr call_protocol_bridge; + std::unique_ptr haptic_protocol_bridge; std::unique_ptr account_bridge; std::shared_ptr web_surface_process; jnivm::VM* late_lifecycle_vm = nullptr; @@ -1719,6 +1732,7 @@ Status RobloxExperienceComposition::Shutdown() { browser_service_bridge = std::move(browser_service_bridge_); permissions_bridge = std::move(permissions_bridge_); call_protocol_bridge = std::move(call_protocol_bridge_); + haptic_protocol_bridge = std::move(haptic_protocol_bridge_); account_bridge = std::move(account_bridge_); web_surface_process = std::move(web_surface_process_); web_surface_logical_exit_observer_ = {}; @@ -1736,9 +1750,14 @@ Status RobloxExperienceComposition::Shutdown() { if (web_surface_process != nullptr) { (void)web_surface_process->RequestClose(); } - Status status = call_protocol_bridge != nullptr - ? call_protocol_bridge->Shutdown() + Status status = haptic_protocol_bridge != nullptr + ? haptic_protocol_bridge->Shutdown() : Status::Ok(); + const Status call_status = call_protocol_bridge != nullptr + ? call_protocol_bridge->Shutdown() + : Status::Ok(); + if (status.ok()) + status = call_status; const Status account_status = account_bridge != nullptr ? account_bridge->Shutdown() : Status::Ok(); if (status.ok()) diff --git a/src/runtime/roblox_haptic_protocol_bridge.cc b/src/runtime/roblox_haptic_protocol_bridge.cc new file mode 100644 index 00000000..f146b269 --- /dev/null +++ b/src/runtime/roblox_haptic_protocol_bridge.cc @@ -0,0 +1,295 @@ +#include "runtime/roblox_haptic_protocol_bridge.h" + +#include + +#include +#include +#include +#include +#include + +extern "C" { +void mocktail_vr_controller_request_haptics(int32_t hand, float amplitude, + uint64_t duration_ns, + float frequency_hz) __attribute__((weak)); +void mocktail_vr_controller_stop_haptics(int32_t hand) __attribute__((weak)); +} + +namespace mocktail { +namespace runtime { +namespace { + +constexpr std::size_t kMaximumJsonBytes = 64 * 1024; +constexpr uint64_t kMinimumDurationNs = 50000000ULL; // 50ms +constexpr uint64_t kMaximumDurationNs = 5000000000ULL; // 5 seconds + +bool HasBoundedDepth(const std::string& json) { + int depth = 0; + bool quoted = false; + bool escaped = false; + for (const char ch : json) { + if (quoted) { + if (escaped) + escaped = false; + else if (ch == '\\') + escaped = true; + else if (ch == '"') + quoted = false; + } else if (ch == '"') { + quoted = true; + } else if (ch == '{' || ch == '[') { + if (++depth > 8) + return false; + } else if (ch == '}' || ch == ']') { + --depth; + } + } + return true; +} + +std::optional ParseJsonMessage(JNIEnv* env, jstring message) { + if (!message) + return std::nullopt; + const jsize size = env->GetStringUTFLength(message); + if (size <= 0 || static_cast(size) > kMaximumJsonBytes) + return std::nullopt; + const char* bytes = env->GetStringUTFChars(message, nullptr); + if (!bytes) + return std::nullopt; + const std::string owned(bytes, static_cast(size)); + env->ReleaseStringUTFChars(message, bytes); + if (!HasBoundedDepth(owned)) + return std::nullopt; + auto parsed = nlohmann::json::parse(owned, nullptr, false); + if (parsed.is_discarded() || !parsed.is_object()) + return std::nullopt; + return parsed; +} + +void DispatchHapticForMotor(const std::string& motor_name, float amplitude, + uint64_t duration_ns, float frequency_hz, + const HapticSinkFn& sink, + const HapticStopFn& stop) { + int hand = -1; + std::string lower_name = motor_name; + for (char& c : lower_name) { + c = static_cast(std::tolower(static_cast(c))); + } + if (lower_name.find("left") != std::string::npos || + lower_name.find("large") != std::string::npos) { + hand = 0; + } else if (lower_name.find("right") != std::string::npos || + lower_name.find("small") != std::string::npos) { + hand = 1; + } + + if (amplitude <= 0.0f || !std::isfinite(amplitude)) { + if (stop) { + if (hand == -1) { + stop(0); + stop(1); + } else { + stop(hand); + } + } + } else { + amplitude = std::clamp(amplitude, 0.0f, 1.0f); + duration_ns = std::clamp(duration_ns, kMinimumDurationNs, kMaximumDurationNs); + std::fprintf(stderr, + " [haptic] dispatch: hand=%d motor='%s' amp=%.3f " + "dur_ms=%.1f\n", + hand, motor_name.c_str(), amplitude, + duration_ns / 1000000.0); + if (sink) { + if (hand == -1) { + sink(0, amplitude, duration_ns, frequency_hz); + sink(1, amplitude, duration_ns, frequency_hz); + } else { + sink(hand, amplitude, duration_ns, frequency_hz); + } + } + } +} + +RobloxMessageBusReply Answer(JNIEnv* env, jstring message, std::size_t method, + const HapticSinkFn& sink, + const HapticStopFn& stop, + const HapticStateSyncFn& state_sync) { + // Method 0: SupportsHaptics + if (method == 0) { + if (state_sync) { + state_sync(env); + } + return RobloxMessageBusReply{R"({"supportsHaptics":true})", 0, "supported"}; + } + + // Method 1: GetMotors + if (method == 1) { + if (state_sync) { + state_sync(env); + } + return RobloxMessageBusReply{RobloxHapticProtocolBridge::GetMotorsJson(), 0, + "ok"}; + } + + // Method 2: SetMotorState + if (method == 2) { + const auto parsed = ParseJsonMessage(env, message); + if (!parsed.has_value()) { + return RobloxMessageBusReply{"{}", 13, "invalid_request"}; + } + const auto& root = *parsed; + for (auto it = root.begin(); it != root.end(); ++it) { + const std::string& motor_name = it.key(); + const auto& motor_val = it.value(); + if (!motor_val.is_object()) + continue; + + float max_intensity = 0.0f; + std::size_t sample_count = 1; + double ms_per_sample = 20.0; + + if (motor_val.contains("millisecondsPerSample") && + motor_val["millisecondsPerSample"].is_number()) { + ms_per_sample = motor_val["millisecondsPerSample"].get(); + } + + if (motor_val.contains("intensities") && + motor_val["intensities"].is_array()) { + const auto& arr = motor_val["intensities"]; + sample_count = std::max(1, arr.size()); + for (const auto& item : arr) { + if (item.is_number()) { + float val = item.get(); + if (val > max_intensity) + max_intensity = val; + } + } + } else if (motor_val.contains("intensity") && + motor_val["intensity"].is_number()) { + max_intensity = motor_val["intensity"].get(); + } + + uint64_t duration_ns = static_cast( + std::max(0.0, ms_per_sample) * sample_count * 1000000.0); + DispatchHapticForMotor(motor_name, max_intensity, duration_ns, 0.0f, sink, + stop); + } + return RobloxMessageBusReply{"{}", 0, "ok"}; + } + + // Method 3: UpdateSingletonVibration + if (method == 3) { + const auto parsed = ParseJsonMessage(env, message); + if (!parsed.has_value()) { + return RobloxMessageBusReply{"{}", 13, "invalid_request"}; + } + float intensity = 0.0f; + if (parsed->contains("intensity") && (*parsed)["intensity"].is_number()) { + intensity = (*parsed)["intensity"].get(); + } + DispatchHapticForMotor("defaultMotor", intensity, kMinimumDurationNs, 0.0f, + sink, stop); + return RobloxMessageBusReply{"{}", 0, "ok"}; + } + + return RobloxMessageBusReply{"{}", 13, "unknown_method"}; +} + +} // namespace + +std::string RobloxHapticProtocolBridge::GetMotorsJson() { + const nlohmann::json motors = { + {"leftHand", {{"x", -0.2}, {"y", 0.0}, {"z", 0.0}}}, + {"rightHand", {{"x", 0.2}, {"y", 0.0}, {"z", 0.0}}}, + {"LeftHand", {{"x", -0.2}, {"y", 0.0}, {"z", 0.0}}}, + {"RightHand", {{"x", 0.2}, {"y", 0.0}, {"z", 0.0}}}, + {"Large", {{"x", -0.2}, {"y", 0.0}, {"z", 0.0}}}, + {"Small", {{"x", 0.2}, {"y", 0.0}, {"z", 0.0}}}, + {"large", {{"x", -0.2}, {"y", 0.0}, {"z", 0.0}}}, + {"small", {{"x", 0.2}, {"y", 0.0}, {"z", 0.0}}}, + {"defaultMotor", {{"x", 0.0}, {"y", 0.0}, {"z", 0.0}}}, + }; + return motors.dump(); +} + +RobloxHapticProtocolBridge::RobloxHapticProtocolBridge( + JniEnvironmentProvider environment, RobloxMessageBusSymbols symbols, + RobloxMessageBusObjects objects, HapticSinkFn haptic_sink, + HapticStopFn haptic_stop, HapticStateSyncFn haptic_state_sync) { + if (!haptic_sink) { + haptic_sink = [](int hand, float amplitude, uint64_t duration_ns, + float frequency_hz) { + if (&mocktail_vr_controller_request_haptics != nullptr) { + mocktail_vr_controller_request_haptics(hand, amplitude, duration_ns, + frequency_hz); + } else { + using RequestFn = void (*)(int32_t, float, uint64_t, float); + auto fn = reinterpret_cast( + dlsym(RTLD_DEFAULT, "mocktail_vr_controller_request_haptics")); + if (fn != nullptr) { + fn(hand, amplitude, duration_ns, frequency_hz); + } + } + }; + } + if (!haptic_stop) { + haptic_stop = [](int hand) { + if (&mocktail_vr_controller_stop_haptics != nullptr) { + mocktail_vr_controller_stop_haptics(hand); + } else { + using StopFn = void (*)(int32_t); + auto fn = reinterpret_cast( + dlsym(RTLD_DEFAULT, "mocktail_vr_controller_stop_haptics")); + if (fn != nullptr) { + fn(hand); + } + } + }; + } + if (!haptic_state_sync) { + using NativeUpdateMobileHapticsStateFn = + void (*)(JNIEnv*, jclass, jboolean, jstring); + auto fn = reinterpret_cast( + dlsym(RTLD_DEFAULT, + "Java_com_roblox_engine_jni_NativeGLInterface_" + "nativeUpdateMobileHapticsState")); + if (fn != nullptr) { + haptic_state_sync = [fn](JNIEnv* env) { + if (!env) return; + jclass clazz = env->FindClass("com/roblox/engine/jni/NativeGLInterface"); + if (!clazz) return; + const std::string motors = RobloxHapticProtocolBridge::GetMotorsJson(); + jstring str = env->NewStringUTF(motors.c_str()); + if (str) { + fn(env, clazz, JNI_TRUE, str); + env->DeleteLocalRef(str); + } + env->DeleteLocalRef(clazz); + }; + } + } + + bridge_ = std::make_unique( + environment, symbols, objects, "HapticProtocol", + std::vector{"SupportsHaptics", "GetMotors", "SetMotorState", + "UpdateSingletonVibration"}, + [sink = std::move(haptic_sink), stop = std::move(haptic_stop), + sync = std::move(haptic_state_sync)]( + JNIEnv* env, jstring message, std::size_t method) { + return Answer(env, message, method, sink, stop, sync); + }); +} + +RobloxHapticProtocolBridge::~RobloxHapticProtocolBridge() = default; + +Status RobloxHapticProtocolBridge::Initialize() { + return bridge_->Initialize(); +} + +Status RobloxHapticProtocolBridge::Shutdown() { + return bridge_->Shutdown(); +} + +} // namespace runtime +} // namespace mocktail diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc index aa756782..f36cb983 100644 --- a/src/vr/roblox_vr_device_bridge.cc +++ b/src/vr/roblox_vr_device_bridge.cc @@ -408,6 +408,8 @@ constexpr VrBuildContract kContracts[] = { 0x1d48b70, 0x118, 0x140, + 0x72fcd18, + 0x7367588, "2998 (2.736.1408)"}, {"5f0704edd9064f566ee3d6df2bd2fabbcc709f03", {0x6d08108, 0x37f9fe4, 0x37fa3c6, 0x37fa3b6, 0x37fa3e8, 0x73f4690, @@ -420,6 +422,8 @@ constexpr VrBuildContract kContracts[] = { 0x1d8f8c0, 0x118, 0x140, + 0, + 0, "3092 (2.738.1397)"}, {"a67a91ef96c700c0730acd36a250c2f93cfafbc1", {0x7083a48, 0x549c884, 0x549cc66, 0x549cc56, 0x549cc88, 0x74d1640, @@ -432,6 +436,8 @@ constexpr VrBuildContract kContracts[] = { 0x1e2fcf0, 0x108, 0x130, + 0x763fef8, + 0x74cf1e0, "3222 (2.742.1059)"}, }; @@ -998,7 +1004,9 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { contract_->pointer_frame_rva + 0x1b2, contract_->workspace_getter_rva + 1, contract_->hand_pitch_storage_rva + sizeof(int), - contract_->haptic_sink_rva + 1}); + contract_->haptic_sink_rva + 1, + contract_->keep_haptics_flag_rva + 1, + contract_->vr_haptics_flag_rva + 1}); if (image_base > std::numeric_limits::max() - largest_rva) { return InvalidArgument("VR device profile address overflows"); } @@ -1126,6 +1134,21 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { return FailedPrecondition("could not install verified VR aim pointer hook"); } InstallIsVrAppBuildHook(image_base); + if (contract_->keep_haptics_flag_rva != 0 && + IsWritableDataImageRange(image_base, contract_->keep_haptics_flag_rva, + 1)) { + *reinterpret_cast(image_base + + contract_->keep_haptics_flag_rva) = 1; + Log(" [vr-device] enabled KeepHapticsOnMobileWhenGamepadConnected at +0x%" PRIxPTR "\n", + contract_->keep_haptics_flag_rva); + } + if (contract_->vr_haptics_flag_rva != 0 && + IsWritableDataImageRange(image_base, contract_->vr_haptics_flag_rva, 1)) { + *reinterpret_cast(image_base + contract_->vr_haptics_flag_rva) = + 1; + Log(" [vr-device] enabled VRHaptics at +0x%" PRIxPTR "\n", + contract_->vr_haptics_flag_rva); + } { std::lock_guard pose_lock(g_native_pose_mutex); g_native_pose = {}; diff --git a/src/vr/roblox_vr_device_bridge_internal.h b/src/vr/roblox_vr_device_bridge_internal.h index c9014795..119e4129 100644 --- a/src/vr/roblox_vr_device_bridge_internal.h +++ b/src/vr/roblox_vr_device_bridge_internal.h @@ -24,6 +24,8 @@ struct VrBuildContract { // service and HeadScale in the workspace camera. std::uintptr_t pointer_hand_selector_offset; std::uintptr_t camera_head_scale_offset; + std::uintptr_t keep_haptics_flag_rva; + std::uintptr_t vr_haptics_flag_rva; // Roblox version code and name, for messages that list supported builds. std::string_view version; }; diff --git a/src/vr/xr_actions.cc b/src/vr/xr_actions.cc index 016252d1..2c5f32c7 100644 --- a/src/vr/xr_actions.cc +++ b/src/vr/xr_actions.cc @@ -4,6 +4,8 @@ #include #include +#include +#include #include #include #include @@ -903,9 +905,20 @@ bool XrActions::Sync(void *xr_session, std::uint64_t predicted_display_time, // 0 lets the runtime choose its resonant frequency; the runtime // clamps unsupported frequencies itself. vibration.frequency = request.frequency_hz; - (void)xrApplyHapticFeedback( + const XrResult result = xrApplyHapticFeedback( session, &haptic_info, reinterpret_cast(&vibration)); + if (XR_FAILED(result)) { + Log(" [vr-input] xrApplyHapticFeedback failed for hand %d: %d\n", + hand, static_cast(result)); + } else { + static std::atomic logged[2]{}; + if (!logged[hand].exchange(true)) { + Log(" [vr-input] xrApplyHapticFeedback active for hand %d: amp=%.2f dur_ns=%" PRIu64 "\n", + hand, request.amplitude, + static_cast(vibration.duration)); + } + } } } return true; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 895d7c04..ef04b522 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -508,6 +508,15 @@ target_link_libraries(roblox_permissions_bridge_test PRIVATE ) mocktail_apply_compile_options(roblox_permissions_bridge_test) +add_executable(roblox_haptic_protocol_bridge_test roblox_haptic_protocol_bridge_test.cc) +target_link_libraries(roblox_haptic_protocol_bridge_test PRIVATE + Mocktail::LegacyJni + Mocktail::Runtime + GTest::gtest_main + Threads::Threads +) +mocktail_apply_compile_options(roblox_haptic_protocol_bridge_test) + add_executable(roblox_account_bridge_test roblox_account_bridge_test.cc) target_link_libraries(roblox_account_bridge_test PRIVATE Mocktail::LegacyJni @@ -1200,6 +1209,7 @@ gtest_discover_tests(roblox_game_session_native_adapter_test) gtest_discover_tests(roblox_experience_launch_bridge_test) gtest_discover_tests(roblox_web_view_bridge_test) gtest_discover_tests(roblox_permissions_bridge_test) +gtest_discover_tests(roblox_haptic_protocol_bridge_test) gtest_discover_tests(roblox_account_bridge_test) gtest_discover_tests(roblox_browser_service_bridge_test) gtest_discover_tests(roblox_fresh_game_launch_controller_test) diff --git a/tests/performance_policy_test.cc b/tests/performance_policy_test.cc index 8e48df68..010e71aa 100644 --- a/tests/performance_policy_test.cc +++ b/tests/performance_policy_test.cc @@ -357,6 +357,8 @@ TEST(PerformancePolicyTest, EXPECT_EQ(json["FFlagDebugEnableVREmulator"], "True"); EXPECT_EQ(json["FFlagSlowDownRendering"], "False"); EXPECT_EQ(json["FFlagFixVRAvatarPlacement"], "True"); + EXPECT_EQ(json["FFlagKeepHapticsOnMobileWhenGamepadConnected"], "True"); + EXPECT_EQ(json["FFlagVRHaptics"], "True"); EXPECT_EQ(json["FIntUnrelated"], "42"); // Without VR the user's own value is kept, and nothing is added. std::string kept; diff --git a/tests/roblox_experience_composition_test.cc b/tests/roblox_experience_composition_test.cc index b48f86cd..1da71a0d 100644 --- a/tests/roblox_experience_composition_test.cc +++ b/tests/roblox_experience_composition_test.cc @@ -954,9 +954,13 @@ TEST(RobloxExperienceCompositionTest, ASSERT_TRUE(composition.InitializePlatformProtocols().ok()); EXPECT_FALSE(composition.subscribed()); - EXPECT_EQ(probe.permission_subscriptions, 6); - EXPECT_EQ(probe.permission_handlers.size(), 6u); + EXPECT_EQ(probe.permission_subscriptions, 10); + EXPECT_EQ(probe.permission_handlers.size(), 10u); EXPECT_EQ(probe.permission_handlers.count("getCallState"), 1u); + EXPECT_EQ(probe.permission_handlers.count("SupportsHaptics"), 1u); + EXPECT_EQ(probe.permission_handlers.count("GetMotors"), 1u); + EXPECT_EQ(probe.permission_handlers.count("SetMotorState"), 1u); + EXPECT_EQ(probe.permission_handlers.count("UpdateSingletonVibration"), 1u); EXPECT_EQ(QueryMicrophone(&probe).at("status"), "DENIED"); EXPECT_EQ(probe.web_view_initializations, 1); EXPECT_EQ(probe.browser_bindings, 4); @@ -965,8 +969,8 @@ TEST(RobloxExperienceCompositionTest, EXPECT_EQ(probe.browser_releases, 0); EXPECT_EQ(probe.browser_callbacks_cleared, 0); EXPECT_TRUE(composition.Shutdown().ok()); - EXPECT_EQ(probe.permission_disconnects, 6); - EXPECT_EQ(probe.permission_handlers_cleared, 6); + EXPECT_EQ(probe.permission_disconnects, 10); + EXPECT_EQ(probe.permission_handlers_cleared, 10); EXPECT_EQ(probe.disconnects, 3); EXPECT_EQ(probe.browser_disconnects, 4); EXPECT_EQ(probe.browser_releases, 4); diff --git a/tests/roblox_haptic_protocol_bridge_test.cc b/tests/roblox_haptic_protocol_bridge_test.cc new file mode 100644 index 00000000..fe12fe78 --- /dev/null +++ b/tests/roblox_haptic_protocol_bridge_test.cc @@ -0,0 +1,344 @@ +#include "runtime/roblox_haptic_protocol_bridge.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "jnivm/jnivm.h" + +namespace mocktail { +namespace runtime { +namespace { + +using Json = nlohmann::json; + +std::string Copy(JNIEnv* env, jstring value) { + if (!value) + return {}; + const char* chars = env->GetStringUTFChars(value, nullptr); + const std::string copy = chars ? chars : ""; + if (chars) + env->ReleaseStringUTFChars(value, chars); + return copy; +} + +struct RecordedHaptic { + int hand = -1; + float amplitude = 0.0f; + uint64_t duration_ns = 0; + float frequency_hz = 0.0f; +}; + +struct Probe { + jnivm::VM* vm = nullptr; + std::string expected_protocol = "HapticProtocol"; + std::map legacy; + std::map handlers; + int callbacks_created = 0; + int callbacks_cleared = 0; + int handlers_created = 0; + int handlers_cleared = 0; + int subscriptions = 0; + int disconnected = 0; + int native_cleared = 0; + std::mutex mutex; + + struct Published { + std::string method; + Json body; + jint code; + }; + + std::vector published; + std::vector> resolved; + + std::vector recorded_sinks; + std::vector recorded_stops; +}; + +Probe* g_probe = nullptr; + +void Prepare(void* context) { + static_cast(context)->RestoreFunctions(); +} + +jobject CreateRaw(void* context, std::shared_ptr target, + void (*run)(void*, JNIEnv*, jstring)) { + auto* probe = static_cast(context); + ++probe->callbacks_created; + return probe->vm->CreateMessageBusRawCallback( + std::move(target), jnivm::MessageBusRawCallbacks{run}); +} + +void ClearRaw(void* context, jobject callback) { + auto* probe = static_cast(context); + ++probe->callbacks_cleared; + probe->vm->ClearMessageBusRawCallback(callback); +} + +jobject CreateAsync(void* context, std::shared_ptr target, + void (*run)(void*, JNIEnv*, jstring, jstring)) { + auto* probe = static_cast(context); + ++probe->handlers_created; + return probe->vm->CreateMessageBusAsyncRequestHandler( + std::move(target), jnivm::MessageBusAsyncRequestHandlerCallbacks{run}); +} + +void ClearAsync(void* context, jobject handler) { + auto* probe = static_cast(context); + ++probe->handlers_cleared; + probe->vm->ClearMessageBusAsyncRequestHandler(handler); +} + +void InvokeLegacy(JNIEnv* env, jobject callback, const char* payload) { + jclass cls = env->GetObjectClass(callback); + jmethodID run = env->GetMethodID(cls, "run", "(Ljava/lang/String;)V"); + jstring message = env->NewStringUTF(payload); + env->CallVoidMethod(callback, run, message); + env->DeleteLocalRef(message); + env->DeleteLocalRef(cls); +} + +void InvokeAsync(JNIEnv* env, jobject handler, const char* payload, + const char* response_id) { + jclass cls = env->GetObjectClass(handler); + jmethodID run = + env->GetMethodID(cls, "run", "(Ljava/lang/String;Ljava/lang/String;)V"); + jstring message = payload ? env->NewStringUTF(payload) : nullptr; + jstring id = env->NewStringUTF(response_id); + jvalue args[2]{}; + args[0].l = message; + args[1].l = id; + env->CallVoidMethodA(handler, run, args); + if (message) + env->DeleteLocalRef(message); + env->DeleteLocalRef(id); + env->DeleteLocalRef(cls); +} + +jobject Subscribe(JNIEnv* env, jobject, jstring protocol, jstring method, + jobject callback, jboolean once) { + EXPECT_EQ(Copy(env, protocol), g_probe->expected_protocol); + EXPECT_EQ(once, JNI_FALSE); + const int handle = ++g_probe->subscriptions; + g_probe->legacy[Copy(env, method)] = callback; + jclass cls = env->FindClass("com/roblox/universalapp/messagebus/Connection"); + jmethodID ctor = env->GetMethodID(cls, "", "(J)V"); + jobject connection = env->NewObject(cls, ctor, static_cast(handle)); + env->DeleteLocalRef(cls); + return connection; +} + +void Disconnect(JNIEnv*, jobject, jlong handle) { + EXPECT_GT(handle, 0); + ++g_probe->disconnected; +} + +void SetHandler(JNIEnv* env, jobject, jstring protocol, jstring method, + jobject handler) { + EXPECT_EQ(Copy(env, protocol), g_probe->expected_protocol); + g_probe->handlers[Copy(env, method)] = handler; +} + +void ClearHandler(JNIEnv* env, jobject, jstring protocol, jstring) { + EXPECT_EQ(Copy(env, protocol), g_probe->expected_protocol); + ++g_probe->native_cleared; +} + +void Publish(JNIEnv* env, jobject, jstring protocol, jstring method, + jstring response, jint code, jstring telemetry) { + EXPECT_EQ(Copy(env, protocol), g_probe->expected_protocol); + EXPECT_EQ(Copy(env, telemetry), "{}"); + std::unique_lock lock(g_probe->mutex); + g_probe->published.push_back( + {Copy(env, method), Json::parse(Copy(env, response)), code}); +} + +void Resolve(JNIEnv* env, jobject, jstring id, jstring body) { + std::lock_guard lock(g_probe->mutex); + g_probe->resolved.emplace_back(Copy(env, id), Json::parse(Copy(env, body))); +} + +class RobloxHapticProtocolBridgeTest : public testing::Test { + protected: + void SetUp() override { + probe.vm = &vm; + g_probe = &probe; + env = vm.GetJNIEnv(); + jclass cls = + env->FindClass("com/roblox/universalapp/messagebus/MessageBus"); + bus = env->AllocObject(cls); + env->DeleteLocalRef(cls); + } + + void TearDown() override { + bridge.reset(); + env->DeleteLocalRef(bus); + g_probe = nullptr; + } + + Status Initialize() { + auto sink = [](int hand, float amplitude, uint64_t duration_ns, + float frequency_hz) { + if (g_probe) { + std::lock_guard lock(g_probe->mutex); + g_probe->recorded_sinks.push_back( + {hand, amplitude, duration_ns, frequency_hz}); + } + }; + auto stop = [](int hand) { + if (g_probe) { + std::lock_guard lock(g_probe->mutex); + g_probe->recorded_stops.push_back(hand); + } + }; + + bridge = std::make_unique( + JniEnvironmentProvider{vm.GetJavaVM(), &vm, Prepare}, + RobloxMessageBusSymbols{Subscribe, Disconnect, Publish, SetHandler, + ClearHandler, Resolve}, + RobloxMessageBusObjects{bus, &probe, CreateRaw, ClearRaw, CreateAsync, + ClearAsync}, + sink, stop); + return bridge->Initialize(); + } + + Probe::Published Legacy(const char* method, const char* payload = "{}") { + const std::size_t mark = probe.published.size(); + InvokeLegacy(env, probe.legacy.at(method), payload); + EXPECT_EQ(probe.published.size(), mark + 1); + return probe.published.back(); + } + + std::pair Async(const char* method, + const char* payload = "{}", + const char* id = "id-123") { + const std::size_t mark = probe.resolved.size(); + InvokeAsync(env, probe.handlers.at(method), payload, id); + EXPECT_EQ(probe.resolved.size(), mark + 1); + return probe.resolved.back(); + } + + jnivm::VM vm; + JNIEnv* env = nullptr; + jobject bus = nullptr; + Probe probe; + std::unique_ptr bridge; +}; + +TEST_F(RobloxHapticProtocolBridgeTest, InitializesAndShutsDownCleanly) { + ASSERT_TRUE(Initialize().ok()); + EXPECT_EQ(probe.subscriptions, 4); + EXPECT_EQ(probe.handlers_created, 4); + EXPECT_TRUE(bridge->Shutdown().ok()); + EXPECT_EQ(probe.disconnected, 4); + EXPECT_EQ(probe.handlers_cleared, 4); +} + +TEST_F(RobloxHapticProtocolBridgeTest, SupportsHapticsRepliesTrue) { + ASSERT_TRUE(Initialize().ok()); + + const auto reply = Legacy("SupportsHaptics"); + EXPECT_EQ(reply.code, 0); + EXPECT_TRUE(reply.body.value("supportsHaptics", false)); + + const auto async_reply = Async("SupportsHaptics", "{}", "async-1"); + EXPECT_EQ(async_reply.first, "async-1"); + EXPECT_TRUE(async_reply.second.value("supportsHaptics", false)); +} + +TEST_F(RobloxHapticProtocolBridgeTest, GetMotorsReturnsValidPositions) { + ASSERT_TRUE(Initialize().ok()); + + const auto reply = Legacy("GetMotors"); + EXPECT_EQ(reply.code, 0); + EXPECT_TRUE(reply.body.contains("leftHand")); + EXPECT_TRUE(reply.body.contains("rightHand")); + EXPECT_TRUE(reply.body.contains("defaultMotor")); + + EXPECT_LT(reply.body["leftHand"]["x"].get(), 0.0); + EXPECT_GT(reply.body["rightHand"]["x"].get(), 0.0); +} + +TEST_F(RobloxHapticProtocolBridgeTest, SetMotorStateRoutesHands) { + ASSERT_TRUE(Initialize().ok()); + + const char* payload_left = R"({ + "leftHand": { + "millisecondsPerSample": 20.0, + "intensities": [0.2, 0.8, 0.5] + } + })"; + + const auto reply = Legacy("SetMotorState", payload_left); + EXPECT_EQ(reply.code, 0); + ASSERT_EQ(probe.recorded_sinks.size(), 1u); + EXPECT_EQ(probe.recorded_sinks[0].hand, 0); + EXPECT_FLOAT_EQ(probe.recorded_sinks[0].amplitude, 0.8f); + EXPECT_GE(probe.recorded_sinks[0].duration_ns, 50000000ULL); + + const char* payload_right = R"({ + "rightHand": { + "millisecondsPerSample": 10.0, + "intensities": [0.4] + } + })"; + + const auto reply_right = Legacy("SetMotorState", payload_right); + EXPECT_EQ(reply_right.code, 0); + ASSERT_EQ(probe.recorded_sinks.size(), 2u); + EXPECT_EQ(probe.recorded_sinks[1].hand, 1); + EXPECT_FLOAT_EQ(probe.recorded_sinks[1].amplitude, 0.4f); +} + +TEST_F(RobloxHapticProtocolBridgeTest, SetMotorStateZeroStopsHaptics) { + ASSERT_TRUE(Initialize().ok()); + + const char* payload = R"({ + "leftHand": { + "millisecondsPerSample": 20.0, + "intensities": [0.0] + } + })"; + + const auto reply = Legacy("SetMotorState", payload); + EXPECT_EQ(reply.code, 0); + ASSERT_EQ(probe.recorded_stops.size(), 1u); + EXPECT_EQ(probe.recorded_stops[0], 0); +} + +TEST_F(RobloxHapticProtocolBridgeTest, UpdateSingletonVibrationBroadcasts) { + ASSERT_TRUE(Initialize().ok()); + + const auto reply = Legacy("UpdateSingletonVibration", R"({"intensity": 0.6})"); + EXPECT_EQ(reply.code, 0); + ASSERT_EQ(probe.recorded_sinks.size(), 2u); + EXPECT_EQ(probe.recorded_sinks[0].hand, 0); + EXPECT_FLOAT_EQ(probe.recorded_sinks[0].amplitude, 0.6f); + EXPECT_EQ(probe.recorded_sinks[1].hand, 1); + EXPECT_FLOAT_EQ(probe.recorded_sinks[1].amplitude, 0.6f); + + const auto stop_reply = + Legacy("UpdateSingletonVibration", R"({"intensity": 0.0})"); + EXPECT_EQ(stop_reply.code, 0); + ASSERT_GE(probe.recorded_stops.size(), 2u); +} + +TEST_F(RobloxHapticProtocolBridgeTest, RejectsMalformedJsonDefensively) { + ASSERT_TRUE(Initialize().ok()); + + const auto reply = Legacy("SetMotorState", "{not valid json"); + EXPECT_EQ(reply.code, 13); + EXPECT_TRUE(probe.recorded_sinks.empty()); +} + +} // namespace +} // namespace runtime +} // namespace mocktail diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc index 75287272..4f360224 100644 --- a/tests/vr/roblox_vr_device_bridge_test.cc +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -1018,6 +1018,14 @@ TEST(VrSupportedBuilds, Build3222CarriesTheOffsetFindersValues) { EXPECT_EQ(contract->haptic_sink_rva, 0x1e2fcf0u); EXPECT_EQ(contract->pointer_hand_selector_offset, 0x108u); EXPECT_EQ(contract->camera_head_scale_offset, 0x130u); + EXPECT_EQ(contract->keep_haptics_flag_rva, 0x763fef8u); + EXPECT_EQ(contract->vr_haptics_flag_rva, 0x74cf1e0u); + + const auto* contract_2998 = internal::FindVrBuildContract( + "ade08266c67aee88ec9c1d00902150e1684dad3a"); + ASSERT_NE(contract_2998, nullptr); + EXPECT_EQ(contract_2998->keep_haptics_flag_rva, 0x72fcd18u); + EXPECT_EQ(contract_2998->vr_haptics_flag_rva, 0x7367588u); } // MOCKTAIL_TEST_VR_PAYLOADS names a payload store: -/libroblox.so. From 3788026d10efa81c6110a7efef1cb763fc41ac6f Mon Sep 17 00:00:00 2001 From: vii Date: Wed, 14 Oct 2026 03:08:58 -0500 Subject: [PATCH 42/58] fix(vr): limit haptics to VR sessions and check flag names before writing - Register HapticProtocol and report motors only when MOCKTAIL_VR_ENABLED is on. - Drop the per-message [haptic] dispatch log; log each OpenXR haptic result once per hand. - Write a fast flag only when its name pointer matches the expected name. - Remove the state-sync fallback that dlsym can never resolve. --- .../runtime/roblox_experience_composition.h | 2 +- .../runtime/roblox_haptic_protocol_bridge.h | 3 + src/legacy/legacy_runtime.cc | 26 +++++--- src/runtime/roblox_experience_composition.cc | 27 ++++---- src/runtime/roblox_haptic_protocol_bridge.cc | 38 +++-------- src/vr/roblox_vr_device_bridge.cc | 41 +++++++++--- src/vr/roblox_vr_device_bridge_internal.h | 5 ++ src/vr/xr_actions.cc | 11 +++- tests/roblox_experience_composition_test.cc | 39 ++++++++--- tests/roblox_haptic_protocol_bridge_test.cc | 24 +++++++ tests/vr/roblox_vr_device_bridge_test.cc | 64 +++++++++++++++++++ 11 files changed, 206 insertions(+), 74 deletions(-) diff --git a/include/runtime/roblox_experience_composition.h b/include/runtime/roblox_experience_composition.h index d95577fb..435dd071 100644 --- a/include/runtime/roblox_experience_composition.h +++ b/include/runtime/roblox_experience_composition.h @@ -16,10 +16,10 @@ #include "runtime/owned_pthread.h" #include "runtime/roblox_browser_service_bridge.h" #include "runtime/roblox_call_protocol_bridge.h" -#include "runtime/roblox_haptic_protocol_bridge.h" #include "runtime/roblox_experience_launch_bridge.h" #include "runtime/roblox_experience_presence.h" #include "runtime/roblox_fresh_game_launch_controller.h" +#include "runtime/roblox_haptic_protocol_bridge.h" #include "runtime/roblox_account_bridge.h" #include "runtime/roblox_permissions_bridge.h" #include "runtime/roblox_web_view_bridge.h" diff --git a/include/runtime/roblox_haptic_protocol_bridge.h b/include/runtime/roblox_haptic_protocol_bridge.h index f1b54a81..bc3b742e 100644 --- a/include/runtime/roblox_haptic_protocol_bridge.h +++ b/include/runtime/roblox_haptic_protocol_bridge.h @@ -31,6 +31,9 @@ class RobloxHapticProtocolBridge final { Status Shutdown(); static std::string GetMotorsJson(); + // True during a VR session; main() sets MOCKTAIL_VR_ENABLED to 0 when VR + // cannot start, so the bridge stays off for desktop play. + static bool EnabledForThisSession(); private: std::unique_ptr bridge_; diff --git a/src/legacy/legacy_runtime.cc b/src/legacy/legacy_runtime.cc index 2afef4f8..a398c35c 100644 --- a/src/legacy/legacy_runtime.cc +++ b/src/legacy/legacy_runtime.cc @@ -6411,10 +6411,13 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, "Java_com_roblox_engine_jni_NativeGLInterface_" "nativeAppBridgeV2SendAppEventOnGameLoaded")); auto* native_update_mobile_haptics_state = - reinterpret_cast(linker::ResolveSymbol( - roblox_handle, - "Java_com_roblox_engine_jni_NativeGLInterface_" - "nativeUpdateMobileHapticsState")); + mocktail::runtime::RobloxHapticProtocolBridge::EnabledForThisSession() + ? reinterpret_cast( + linker::ResolveSymbol( + roblox_handle, + "Java_com_roblox_engine_jni_NativeGLInterface_" + "nativeUpdateMobileHapticsState")) + : nullptr; auto* native_set_asset_path = reinterpret_cast(linker::ResolveSymbol( roblox_handle, @@ -6837,13 +6840,16 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, eager_call_protocol_bridge = std::make_unique( environment, symbols.permissions, objects); - eager_haptic_protocol_bridge = - std::make_unique( - environment, symbols.permissions, objects); + if (mocktail::runtime::RobloxHapticProtocolBridge:: + EnabledForThisSession()) { + eager_haptic_protocol_bridge = + std::make_unique( + environment, symbols.permissions, objects); + } status = eager_permissions_bridge->Initialize(); if (status.ok()) status = eager_call_protocol_bridge->Initialize(); - if (status.ok()) + if (status.ok() && eager_haptic_protocol_bridge != nullptr) status = eager_haptic_protocol_bridge->Initialize(); env->DeleteLocalRef(bus); if (!status.ok()) { @@ -6851,8 +6857,8 @@ int mocktail::legacy::Run(const runtime::CommandLineOptions& options, << status.message() << '\n'; return EXIT_FAILURE; } - std::cout << " [platform] direct GAME PermissionsProtocol, " - "CallProtocol and HapticProtocol ready\n" + std::cout << " [platform] direct GAME PermissionsProtocol and " + "CallProtocol ready\n" << std::flush; } diff --git a/src/runtime/roblox_experience_composition.cc b/src/runtime/roblox_experience_composition.cc index 5832e29c..839a5f72 100644 --- a/src/runtime/roblox_experience_composition.cc +++ b/src/runtime/roblox_experience_composition.cc @@ -466,16 +466,19 @@ Status RobloxExperienceComposition::InitializePlatformProtocols() { (void)ReleaseGlobalObjects(); return status; } - auto haptic_protocol_bridge = std::make_unique( - environment_, permissions_symbols_, permissions_objects); - status = haptic_protocol_bridge->Initialize(); - if (!status.ok()) { - (void)call_protocol_bridge->Shutdown(); - (void)permissions_bridge->Shutdown(); - (void)browser_service_bridge->Shutdown(); - (void)web_view_bridge->Shutdown(); - (void)ReleaseGlobalObjects(); - return status; + std::unique_ptr haptic_protocol_bridge; + if (RobloxHapticProtocolBridge::EnabledForThisSession()) { + haptic_protocol_bridge = std::make_unique( + environment_, permissions_symbols_, permissions_objects); + status = haptic_protocol_bridge->Initialize(); + if (!status.ok()) { + (void)call_protocol_bridge->Shutdown(); + (void)permissions_bridge->Shutdown(); + (void)browser_service_bridge->Shutdown(); + (void)web_view_bridge->Shutdown(); + (void)ReleaseGlobalObjects(); + return status; + } } // AccountProtocol only shortens a device-integrity login challenge that // cannot succeed here, so its absence or failure never blocks startup. @@ -509,8 +512,8 @@ Status RobloxExperienceComposition::InitializePlatformProtocols() { platform_protocols_initialized_ = true; } std::fprintf(stderr, - " [platform] Android WebViewProtocol, BrowserService, " - "PermissionsProtocol, CallProtocol and HapticProtocol " + " [platform] Android WebViewProtocol, BrowserService and " + "PermissionsProtocol and CallProtocol " "bridges ready\n"); return Status::Ok(); } diff --git a/src/runtime/roblox_haptic_protocol_bridge.cc b/src/runtime/roblox_haptic_protocol_bridge.cc index f146b269..8d666821 100644 --- a/src/runtime/roblox_haptic_protocol_bridge.cc +++ b/src/runtime/roblox_haptic_protocol_bridge.cc @@ -5,8 +5,10 @@ #include #include #include -#include +#include #include +#include +#include extern "C" { void mocktail_vr_controller_request_haptics(int32_t hand, float amplitude, @@ -95,11 +97,6 @@ void DispatchHapticForMotor(const std::string& motor_name, float amplitude, } else { amplitude = std::clamp(amplitude, 0.0f, 1.0f); duration_ns = std::clamp(duration_ns, kMinimumDurationNs, kMaximumDurationNs); - std::fprintf(stderr, - " [haptic] dispatch: hand=%d motor='%s' amp=%.3f " - "dur_ms=%.1f\n", - hand, motor_name.c_str(), amplitude, - duration_ns / 1000000.0); if (sink) { if (hand == -1) { sink(0, amplitude, duration_ns, frequency_hz); @@ -198,6 +195,12 @@ RobloxMessageBusReply Answer(JNIEnv* env, jstring message, std::size_t method, } // namespace +bool RobloxHapticProtocolBridge::EnabledForThisSession() { + const char* value = std::getenv("MOCKTAIL_VR_ENABLED"); + const std::string_view vr = value != nullptr ? value : ""; + return vr == "1" || vr == "true" || vr == "on"; +} + std::string RobloxHapticProtocolBridge::GetMotorsJson() { const nlohmann::json motors = { {"leftHand", {{"x", -0.2}, {"y", 0.0}, {"z", 0.0}}}, @@ -247,29 +250,6 @@ RobloxHapticProtocolBridge::RobloxHapticProtocolBridge( } }; } - if (!haptic_state_sync) { - using NativeUpdateMobileHapticsStateFn = - void (*)(JNIEnv*, jclass, jboolean, jstring); - auto fn = reinterpret_cast( - dlsym(RTLD_DEFAULT, - "Java_com_roblox_engine_jni_NativeGLInterface_" - "nativeUpdateMobileHapticsState")); - if (fn != nullptr) { - haptic_state_sync = [fn](JNIEnv* env) { - if (!env) return; - jclass clazz = env->FindClass("com/roblox/engine/jni/NativeGLInterface"); - if (!clazz) return; - const std::string motors = RobloxHapticProtocolBridge::GetMotorsJson(); - jstring str = env->NewStringUTF(motors.c_str()); - if (str) { - fn(env, clazz, JNI_TRUE, str); - env->DeleteLocalRef(str); - } - env->DeleteLocalRef(clazz); - }; - } - } - bridge_ = std::make_unique( environment, symbols, objects, "HapticProtocol", std::vector{"SupportsHaptics", "GetMotors", "SetMotorState", diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc index f36cb983..1921ed91 100644 --- a/src/vr/roblox_vr_device_bridge.cc +++ b/src/vr/roblox_vr_device_bridge.cc @@ -734,6 +734,33 @@ float ReadCameraHeadScale(const void* camera, const VrBuildContract& contract) { return scale; } +bool EnableNamedFastFlag(std::uintptr_t image_base, std::uintptr_t rva, + std::string_view name) { + constexpr std::size_t kNamePointerOffset = 8; + if (rva == 0 || !IsWritableDataImageRange( + image_base, rva, kNamePointerOffset + sizeof(void*))) { + return false; + } + std::uintptr_t name_address = 0; + std::memcpy(&name_address, + reinterpret_cast(image_base + rva + + kNamePointerOffset), + sizeof(name_address)); + // The stored name is NUL-terminated, so a longer flag name never matches. + if (name_address < image_base || + !IsProgramHeaderRange(image_base, name_address - image_base, + name.size() + 1, PT_LOAD, PF_R)) { + return false; + } + const auto* stored = reinterpret_cast(name_address); + if (std::memcmp(stored, name.data(), name.size()) != 0 || + stored[name.size()] != '\0') { + return false; + } + *reinterpret_cast(image_base + rva) = 1; + return true; +} + float GetHandPitchOffsetDeg() { static const float offset = [] { if (const char* env = std::getenv("MOCKTAIL_VR_HAND_PITCH_OFFSET")) { @@ -1134,18 +1161,14 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { return FailedPrecondition("could not install verified VR aim pointer hook"); } InstallIsVrAppBuildHook(image_base); - if (contract_->keep_haptics_flag_rva != 0 && - IsWritableDataImageRange(image_base, contract_->keep_haptics_flag_rva, - 1)) { - *reinterpret_cast(image_base + - contract_->keep_haptics_flag_rva) = 1; + if (internal::EnableNamedFastFlag(image_base, + contract_->keep_haptics_flag_rva, + "KeepHapticsOnMobileWhenGamepadConnected")) { Log(" [vr-device] enabled KeepHapticsOnMobileWhenGamepadConnected at +0x%" PRIxPTR "\n", contract_->keep_haptics_flag_rva); } - if (contract_->vr_haptics_flag_rva != 0 && - IsWritableDataImageRange(image_base, contract_->vr_haptics_flag_rva, 1)) { - *reinterpret_cast(image_base + contract_->vr_haptics_flag_rva) = - 1; + if (internal::EnableNamedFastFlag(image_base, contract_->vr_haptics_flag_rva, + "VRHaptics")) { Log(" [vr-device] enabled VRHaptics at +0x%" PRIxPTR "\n", contract_->vr_haptics_flag_rva); } diff --git a/src/vr/roblox_vr_device_bridge_internal.h b/src/vr/roblox_vr_device_bridge_internal.h index 119e4129..7074ce10 100644 --- a/src/vr/roblox_vr_device_bridge_internal.h +++ b/src/vr/roblox_vr_device_bridge_internal.h @@ -36,6 +36,11 @@ const VrBuildContract* FindVrBuildContract(std::string_view build_id); int ReadPointerHandIndex(const void* service, const VrBuildContract& contract); float ReadCameraHeadScale(const void* camera, const VrBuildContract& contract); +// Sets the fast flag at `rva` to true only when its name pointer (+8) points +// at `name` inside the image. Returns whether it wrote. +bool EnableNamedFastFlag(std::uintptr_t image_base, std::uintptr_t rva, + std::string_view name); + // Computes the record orientation for a hand pose: the aim pose (or, with // MOCKTAIL_VR_HAND_POSE_SOURCE=grip, the grip pose) pitched by the engine's // own hand pitch plus the configurable offset. diff --git a/src/vr/xr_actions.cc b/src/vr/xr_actions.cc index 2c5f32c7..f9ba5ec2 100644 --- a/src/vr/xr_actions.cc +++ b/src/vr/xr_actions.cc @@ -908,11 +908,16 @@ bool XrActions::Sync(void *xr_session, std::uint64_t predicted_display_time, const XrResult result = xrApplyHapticFeedback( session, &haptic_info, reinterpret_cast(&vibration)); + // Rumble requests repeat every few frames, so each outcome is logged + // once per hand. + static std::atomic logged_failure[2]{}; + static std::atomic logged[2]{}; if (XR_FAILED(result)) { - Log(" [vr-input] xrApplyHapticFeedback failed for hand %d: %d\n", - hand, static_cast(result)); + if (!logged_failure[hand].exchange(true)) { + Log(" [vr-input] xrApplyHapticFeedback failed for hand %d: %d\n", + hand, static_cast(result)); + } } else { - static std::atomic logged[2]{}; if (!logged[hand].exchange(true)) { Log(" [vr-input] xrApplyHapticFeedback active for hand %d: amp=%.2f dur_ns=%" PRIu64 "\n", hand, request.amplitude, diff --git a/tests/roblox_experience_composition_test.cc b/tests/roblox_experience_composition_test.cc index 1da71a0d..161dc19e 100644 --- a/tests/roblox_experience_composition_test.cc +++ b/tests/roblox_experience_composition_test.cc @@ -926,8 +926,14 @@ TEST(RobloxExperienceCompositionTest, EXPECT_FALSE(composition.subscribed()); } -TEST(RobloxExperienceCompositionTest, - InitializesDurablePlatformProtocolsBeforeNativeBootstrap) { +// HapticProtocol adds four handlers, only during a VR session. +void ExpectDurablePlatformProtocols(bool vr_session) { + if (vr_session) + ASSERT_EQ(setenv("MOCKTAIL_VR_ENABLED", "1", 1), 0); + else + unsetenv("MOCKTAIL_VR_ENABLED"); + const int handlers = vr_session ? 10 : 6; + const std::size_t haptic = vr_session ? 1u : 0u; jnivm::VM vm; Probe probe{&vm}; g_probe = &probe; @@ -954,13 +960,15 @@ TEST(RobloxExperienceCompositionTest, ASSERT_TRUE(composition.InitializePlatformProtocols().ok()); EXPECT_FALSE(composition.subscribed()); - EXPECT_EQ(probe.permission_subscriptions, 10); - EXPECT_EQ(probe.permission_handlers.size(), 10u); + EXPECT_EQ(probe.permission_subscriptions, handlers); + EXPECT_EQ(probe.permission_handlers.size(), + static_cast(handlers)); EXPECT_EQ(probe.permission_handlers.count("getCallState"), 1u); - EXPECT_EQ(probe.permission_handlers.count("SupportsHaptics"), 1u); - EXPECT_EQ(probe.permission_handlers.count("GetMotors"), 1u); - EXPECT_EQ(probe.permission_handlers.count("SetMotorState"), 1u); - EXPECT_EQ(probe.permission_handlers.count("UpdateSingletonVibration"), 1u); + EXPECT_EQ(probe.permission_handlers.count("SupportsHaptics"), haptic); + EXPECT_EQ(probe.permission_handlers.count("GetMotors"), haptic); + EXPECT_EQ(probe.permission_handlers.count("SetMotorState"), haptic); + EXPECT_EQ(probe.permission_handlers.count("UpdateSingletonVibration"), + haptic); EXPECT_EQ(QueryMicrophone(&probe).at("status"), "DENIED"); EXPECT_EQ(probe.web_view_initializations, 1); EXPECT_EQ(probe.browser_bindings, 4); @@ -969,14 +977,25 @@ TEST(RobloxExperienceCompositionTest, EXPECT_EQ(probe.browser_releases, 0); EXPECT_EQ(probe.browser_callbacks_cleared, 0); EXPECT_TRUE(composition.Shutdown().ok()); - EXPECT_EQ(probe.permission_disconnects, 10); - EXPECT_EQ(probe.permission_handlers_cleared, 10); + EXPECT_EQ(probe.permission_disconnects, handlers); + EXPECT_EQ(probe.permission_handlers_cleared, handlers); EXPECT_EQ(probe.disconnects, 3); EXPECT_EQ(probe.browser_disconnects, 4); EXPECT_EQ(probe.browser_releases, 4); EXPECT_EQ(probe.browser_callbacks_cleared, 4); EXPECT_TRUE(probe.account_handlers.empty()); g_probe = nullptr; + unsetenv("MOCKTAIL_VR_ENABLED"); +} + +TEST(RobloxExperienceCompositionTest, + InitializesDurablePlatformProtocolsBeforeNativeBootstrap) { + ExpectDurablePlatformProtocols(false); +} + +TEST(RobloxExperienceCompositionTest, + InitializesHapticProtocolDuringAVrSession) { + ExpectDurablePlatformProtocols(true); } TEST(RobloxExperienceCompositionTest, diff --git a/tests/roblox_haptic_protocol_bridge_test.cc b/tests/roblox_haptic_protocol_bridge_test.cc index fe12fe78..c0521f3f 100644 --- a/tests/roblox_haptic_protocol_bridge_test.cc +++ b/tests/roblox_haptic_protocol_bridge_test.cc @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -339,6 +340,29 @@ TEST_F(RobloxHapticProtocolBridgeTest, RejectsMalformedJsonDefensively) { EXPECT_TRUE(probe.recorded_sinks.empty()); } +// The shared MessageBus bridge logs one line per call; the haptic bridge adds +// none of its own. +TEST_F(RobloxHapticProtocolBridgeTest, RumbleMessagesAddNoHapticLogLines) { + ASSERT_TRUE(Initialize().ok()); + + testing::internal::CaptureStderr(); + Legacy("SetMotorState", R"({"leftHand": {"intensities": [0.5]}})"); + Legacy("UpdateSingletonVibration", R"({"intensity": 0.5})"); + EXPECT_EQ(testing::internal::GetCapturedStderr().find("[haptic]"), + std::string::npos); + EXPECT_EQ(probe.recorded_sinks.size(), 3u); +} + +TEST(RobloxHapticProtocolBridgeSession, RunsOnlyDuringAVrSession) { + unsetenv("MOCKTAIL_VR_ENABLED"); + EXPECT_FALSE(RobloxHapticProtocolBridge::EnabledForThisSession()); + ASSERT_EQ(setenv("MOCKTAIL_VR_ENABLED", "0", 1), 0); + EXPECT_FALSE(RobloxHapticProtocolBridge::EnabledForThisSession()); + ASSERT_EQ(setenv("MOCKTAIL_VR_ENABLED", "1", 1), 0); + EXPECT_TRUE(RobloxHapticProtocolBridge::EnabledForThisSession()); + unsetenv("MOCKTAIL_VR_ENABLED"); +} + } // namespace } // namespace runtime } // namespace mocktail diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc index 4f360224..23bc6c70 100644 --- a/tests/vr/roblox_vr_device_bridge_test.cc +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -1,6 +1,7 @@ // Contract tests for the experimental DebugDeviceVR bridge. They pin the // machine-code contracts of the exact 2998 payload ABI and the fail-closed // bridge lifecycle; they do not execute any guest code. +#include #include #include @@ -1099,3 +1100,66 @@ TEST(VrIsVrAppBuildHook, HasAnEvenAddress) { } // namespace } // namespace mocktail::vr + +namespace mocktail::vr { +namespace { + +// A fake image with a read-only segment for names and a writable one for +// flags. A Roblox fast flag keeps its value byte at +0 and its name at +8. +class VrNamedFastFlag : public testing::Test { + protected: + void SetUp() override { + image_.fill(0); + auto* header = reinterpret_cast(image_.data()); + std::memcpy(header->e_ident, ELFMAG, SELFMAG); + header->e_ident[EI_CLASS] = ELFCLASS64; + header->e_phoff = sizeof(Elf64_Ehdr); + header->e_phnum = 2; + auto* segments = + reinterpret_cast(image_.data() + sizeof(Elf64_Ehdr)); + segments[0] = Elf64_Phdr{PT_LOAD, PF_R, 0, 0, 0, 0x1000, 0x1000, 0x1000}; + segments[1] = + Elf64_Phdr{PT_LOAD, PF_R | PF_W, 0, 0x1000, 0, 0x1000, 0x1000, 0x1000}; + std::memcpy(image_.data() + 0x800, "VRHaptics", 10); + std::memcpy(image_.data() + 0x820, "OtherFlag", 10); + } + + std::uintptr_t base() const { + return reinterpret_cast(image_.data()); + } + + void SetName(std::uintptr_t rva, const void* name) { + const auto address = reinterpret_cast(name); + std::memcpy(image_.data() + rva + 8, &address, sizeof(address)); + } + + alignas(64) std::array image_{}; +}; + +TEST_F(VrNamedFastFlag, SetsTheFlagWhenItsNameMatches) { + SetName(0x1000, image_.data() + 0x800); + EXPECT_TRUE(internal::EnableNamedFastFlag(base(), 0x1000, "VRHaptics")); + EXPECT_EQ(image_[0x1000], 1); +} + +TEST_F(VrNamedFastFlag, LeavesAFlagWithAnotherNameAlone) { + SetName(0x1100, image_.data() + 0x820); + EXPECT_FALSE(internal::EnableNamedFastFlag(base(), 0x1100, "VRHaptics")); + EXPECT_EQ(image_[0x1100], 0); + SetName(0x1200, image_.data() + 0x800); + EXPECT_FALSE(internal::EnableNamedFastFlag(base(), 0x1200, "VRHaptic")); + EXPECT_EQ(image_[0x1200], 0); +} + +TEST_F(VrNamedFastFlag, RejectsNamesAndFlagsOutsideTheImage) { + static const char outside[] = "VRHaptics"; + SetName(0x1300, outside); + EXPECT_FALSE(internal::EnableNamedFastFlag(base(), 0x1300, "VRHaptics")); + EXPECT_EQ(image_[0x1300], 0); + EXPECT_FALSE(internal::EnableNamedFastFlag(base(), 0, "VRHaptics")); + EXPECT_FALSE(internal::EnableNamedFastFlag(base(), 0x900, "VRHaptics")); + EXPECT_FALSE(internal::EnableNamedFastFlag(base(), 0x1ff8, "VRHaptics")); +} + +} // namespace +} // namespace mocktail::vr From 6cf641d472c698889db7a992434a84b74fb47e06 Mon Sep 17 00:00:00 2001 From: vii Date: Wed, 14 Oct 2026 03:14:16 -0500 Subject: [PATCH 43/58] feat(vr): find the haptic flags with the offset finder and enable them on 3092 --- scripts/find_roblox_vr_offsets.py | 72 ++++++++++++++++++++++++ src/vr/roblox_vr_device_bridge.cc | 4 +- tests/vr/roblox_vr_device_bridge_test.cc | 18 ++++-- 3 files changed, 87 insertions(+), 7 deletions(-) diff --git a/scripts/find_roblox_vr_offsets.py b/scripts/find_roblox_vr_offsets.py index c4259aea..6030fec1 100755 --- a/scripts/find_roblox_vr_offsets.py +++ b/scripts/find_roblox_vr_offsets.py @@ -86,6 +86,11 @@ EYE_RESOURCE_LABEL = b"VR Eye" EMULATOR_FLAG_NAME = b"DebugEnableVREmulator" HAND_PITCH_FLAG_NAME = b"NewVRSystemHandPitch" +# Boolean fast flags Nightcap turns on in VR, keyed by contract field. +HAPTIC_FLAG_NAMES = { + "keep_haptics_flag_rva": b"KeepHapticsOnMobileWhenGamepadConnected", + "vr_haptics_flag_rva": b"VRHaptics", +} OBJECT_TYPE_OFFSET = 0x10 OBJECT_TYPE_DEBUG_DEVICE = 6 @@ -948,6 +953,69 @@ def extract_hand_pitch(instructions: Sequence[Any], return None +def extract_bool_flag_storage(instructions: Sequence[Any], + flag_name_rva: int) -> int | None: + """Value byte of a boolean flag: a byte or word store to it, then + lea reg, [name]; mov [storage+8], reg.""" + for index, instruction in enumerate(instructions): + operands = instruction.operands + if (instruction.mnemonic != "lea" or len(operands) != 2 or + operands[0].type != capstone_x86.X86_OP_REG or + flag_name_rva not in rip_target(instruction) or + index + 1 >= len(instructions)): + continue + store = instructions[index + 1] + store_operands = store.operands + if (store.mnemonic != "mov" or len(store_operands) != 2 or + store_operands[0].type != capstone_x86.X86_OP_MEM or + store_operands[1].type != capstone_x86.X86_OP_REG or + store_operands[1].reg != operands[0].reg): + continue + name_slot = next(iter(rip_target(store)), None) + if name_slot is None: + continue + storage = name_slot - 8 + for writer in instructions[max(0, index - 3):index]: + writer_operands = writer.operands + if (writer.mnemonic == "mov" and len(writer_operands) == 2 and + writer_operands[0].type == capstone_x86.X86_OP_MEM and + writer_operands[0].size in (1, 2) and + storage in rip_target(writer)): + return storage + return None + + +def find_bool_flags(image: ElfImage, table: FunctionTable, + disasm: Disassembler, names: dict[str, bytes], + evidence: list[str], warnings: list[str]) -> dict[str, int]: + """Value-byte RVA per field; 0 when the build lacks the flag.""" + found: dict[str, int] = {} + for field, name in names.items(): + label = name.decode("ascii") + try: + name_rva = image.find_alloc_string(name) + except AnalyzerError as error: + warnings.append(f"{label}: {error}; {field} left at 0") + found[field] = 0 + continue + matches: set[int] = set() + for refs in find_text_refs(image, table, disasm, {name_rva}).values(): + for _address, start, end in refs: + storage = extract_bool_flag_storage(disasm.function(start, end), + name_rva) + if storage is not None: + matches.add(storage) + if len(matches) != 1: + listed = ", ".join(f"0x{rva:x}" for rva in sorted(matches)) + raise AnalyzerError(f"{label} flag storage is not unique: [{listed}]") + storage = matches.pop() + image.require_writable_rva(storage, 16) + evidence.append(f"{label}: value byte with name pointer at storage+8" + f" -> 0x{storage:x}") + found[field] = storage + return found + + def match_pointer_getter(instructions: Sequence[Any], selector_offset: int, base_offset: int) -> int | None: """World-space pointer getter: UserCFrame copy + workspace/camera path. @@ -1374,6 +1442,9 @@ def find_vr_offsets(elf_path: pathlib.Path) -> dict[str, Any]: f"hand pitch: dword default {hand_pitch_default} degrees with name" f" pointer at storage+8 -> 0x{hand_pitch_storage:x}") + haptic_flags = find_bool_flags(image, table, disasm, HAPTIC_FLAG_NAMES, + evidence, warnings) + getter_candidates: dict[int, tuple[int, int]] = {} for hit, selector, base in scan_pointer_getter_sites(image): try: @@ -1462,6 +1533,7 @@ def find_vr_offsets(elf_path: pathlib.Path) -> dict[str, Any]: f"0x{hand_pitch_storage:x}", "new_vr_system_hand_pitch_default_degrees": hand_pitch_default, + **{field: f"0x{rva:x}" for field, rva in haptic_flags.items()}, }, "evidence": evidence, "warnings": warnings, diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc index 1921ed91..dc667210 100644 --- a/src/vr/roblox_vr_device_bridge.cc +++ b/src/vr/roblox_vr_device_bridge.cc @@ -422,8 +422,8 @@ constexpr VrBuildContract kContracts[] = { 0x1d8f8c0, 0x118, 0x140, - 0, - 0, + 0x7387118, + 0x73f2268, "3092 (2.738.1397)"}, {"a67a91ef96c700c0730acd36a250c2f93cfafbc1", {0x7083a48, 0x549c884, 0x549cc66, 0x549cc56, 0x549cc88, 0x74d1640, diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc index 23bc6c70..2acfa83f 100644 --- a/tests/vr/roblox_vr_device_bridge_test.cc +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -1021,12 +1022,19 @@ TEST(VrSupportedBuilds, Build3222CarriesTheOffsetFindersValues) { EXPECT_EQ(contract->camera_head_scale_offset, 0x130u); EXPECT_EQ(contract->keep_haptics_flag_rva, 0x763fef8u); EXPECT_EQ(contract->vr_haptics_flag_rva, 0x74cf1e0u); +} - const auto* contract_2998 = internal::FindVrBuildContract( - "ade08266c67aee88ec9c1d00902150e1684dad3a"); - ASSERT_NE(contract_2998, nullptr); - EXPECT_EQ(contract_2998->keep_haptics_flag_rva, 0x72fcd18u); - EXPECT_EQ(contract_2998->vr_haptics_flag_rva, 0x7367588u); +TEST(VrSupportedBuilds, EveryBuildCarriesBothHapticFlags) { + for (const auto& [id, keep, vr] : + {std::tuple{ + "ade08266c67aee88ec9c1d00902150e1684dad3a", 0x72fcd18, 0x7367588}, + {"5f0704edd9064f566ee3d6df2bd2fabbcc709f03", 0x7387118, 0x73f2268}, + {"a67a91ef96c700c0730acd36a250c2f93cfafbc1", 0x763fef8, 0x74cf1e0}}) { + const auto* contract = internal::FindVrBuildContract(id); + ASSERT_NE(contract, nullptr) << id; + EXPECT_EQ(contract->keep_haptics_flag_rva, keep) << id; + EXPECT_EQ(contract->vr_haptics_flag_rva, vr) << id; + } } // MOCKTAIL_TEST_VR_PAYLOADS names a payload store: -/libroblox.so. From 2be319a979a0a0acf7e2bf11f5cb54b12945f310 Mon Sep 17 00:00:00 2001 From: vii Date: Wed, 14 Oct 2026 04:19:58 -0500 Subject: [PATCH 44/58] fix(vr): keep the OpenXR runtime loaded so a game join does not deadlock --- include/mocktail/vr/openxr_backend.h | 7 +++++ src/vr/openxr_backend.cc | 41 ++++++++++++++++++++++++++++ tests/CMakeLists.txt | 3 ++ tests/vr/openxr_backend_test.cc | 23 ++++++++++++++++ 4 files changed, 74 insertions(+) diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index 18b05464..96534115 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -490,6 +490,13 @@ class OpenXrBackend final { // exported C symbols; they are inert while no backend is armed. OpenXrBackend* ActiveVrBackend(); +// Paths of the shared objects mapped into the process. +std::vector LoadedSharedObjects(); +// Keeps every shared object loaded since `before` mapped until exit and +// returns the paths it pinned. +std::vector PinSharedObjectsLoadedSince( + const std::vector& before); + } // namespace mocktail::vr #endif // MOCKTAIL_VR_OPENXR_BACKEND_H_ diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 64615eeb..a9792ae5 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -10,6 +10,7 @@ #include #include +#include #include #include @@ -300,6 +301,36 @@ OpenXrBackend* ActiveVrBackend() { return g_backend; } +std::vector LoadedSharedObjects() { + std::vector paths; + dl_iterate_phdr( + [](dl_phdr_info* info, std::size_t, void* data) { + if (info->dlpi_name != nullptr && info->dlpi_name[0] != '\0') { + static_cast*>(data)->emplace_back( + info->dlpi_name); + } + return 0; + }, + &paths); + return paths; +} + +std::vector PinSharedObjectsLoadedSince( + const std::vector& before) { + std::vector pinned; + for (const auto& path : LoadedSharedObjects()) { + if (std::find(before.begin(), before.end(), path) != before.end()) { + continue; + } + // RTLD_NOLOAD only adds a reference; RTLD_NODELETE makes it permanent. + if (dlopen(path.c_str(), RTLD_NOW | RTLD_NOLOAD | RTLD_NODELETE) != + nullptr) { + pinned.push_back(path); + } + } + return pinned; +} + void OpenXrBackend::DestroyXrInstanceLocked() { { std::lock_guard lock(controller_mutex_); @@ -320,6 +351,7 @@ bool OpenXrBackend::EnsureXrInstanceLocked(std::string* error) { return true; } try { + const auto libraries_before_runtime = LoadedSharedObjects(); const char *vk_extensions[] = {XR_KHR_VULKAN_ENABLE2_EXTENSION_NAME}; const char *gl_extensions[] = {XR_KHR_OPENGL_ES_ENABLE_EXTENSION_NAME, XR_MNDX_EGL_ENABLE_EXTENSION_NAME}; @@ -378,6 +410,15 @@ bool OpenXrBackend::EnsureXrInstanceLocked(std::string* error) { } CheckXr(XR_NULL_HANDLE, instance_result, "xrCreateInstance"); xr_instance_ = instance; + // The loader unloads the runtime with each instance. Loaded again after + // the Vulkan adapter (RTLD_GLOBAL), the runtime would call Vulkan through + // the adapter's hooks, which re-enter this backend under mutex_. + const auto pinned = PinSharedObjectsLoadedSince(libraries_before_runtime); + if (!pinned.empty()) { + Log(" [vr-backend] kept %zu runtime libraries loaded for later " + "sessions\n", + pinned.size()); + } XrInstanceProperties properties = XrInfo(XR_TYPE_INSTANCE_PROPERTIES); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 895d7c04..d4dfb09c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -109,6 +109,9 @@ if(MOCKTAIL_ENABLE_VR) "$" "$" "$") set_tests_properties(OpenXrPreviewContract PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 90) + target_compile_definitions(openxr_backend_test PRIVATE + "MOCKTAIL_TEST_OPENXR_RUNTIME=\"$\"") + add_dependencies(openxr_backend_test mocktail_test_openxr_runtime) else() add_test(NAME OpenXrProbeDisabled COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/vr/probe_contract_test.py" diff --git a/tests/vr/openxr_backend_test.cc b/tests/vr/openxr_backend_test.cc index 36068384..1d185a5d 100644 --- a/tests/vr/openxr_backend_test.cc +++ b/tests/vr/openxr_backend_test.cc @@ -1,5 +1,9 @@ +#include #include #include + +#include +#include #include "mocktail/vr/openxr_backend.h" #include "mocktail/vr/roblox_vr_device_bridge.h" @@ -603,5 +607,24 @@ TEST(OpenXrEyeSwapchainFormat, KeepsGuestEyeBytesWithoutGammaConversion) { VK_FORMAT_UNDEFINED); } +// The OpenXR loader unloads the runtime with its instance. A reload would +// bind the runtime's Vulkan calls to the adapter loaded since, so the +// libraries from the first instance stay mapped. +TEST(OpenXrRuntimeLibraries, PinnedLibrariesSurviveTheLoadersClose) { + const char* runtime = MOCKTAIL_TEST_OPENXR_RUNTIME; + ASSERT_EQ(dlopen(runtime, RTLD_NOW | RTLD_NOLOAD), nullptr); + const auto before = LoadedSharedObjects(); + void* handle = dlopen(runtime, RTLD_NOW | RTLD_LOCAL); + ASSERT_NE(handle, nullptr) << dlerror(); + const auto pinned = PinSharedObjectsLoadedSince(before); + EXPECT_NE(std::find(pinned.begin(), pinned.end(), std::string(runtime)), + pinned.end()); + EXPECT_EQ(PinSharedObjectsLoadedSince(LoadedSharedObjects()).size(), 0u); + ASSERT_EQ(dlclose(handle), 0); + void* still_loaded = dlopen(runtime, RTLD_NOW | RTLD_NOLOAD); + EXPECT_NE(still_loaded, nullptr); + if (still_loaded != nullptr) dlclose(still_loaded); +} + } // namespace } // namespace mocktail::vr From 2eaba9048b2852704f739601195686a8dcfecca7 Mon Sep 17 00:00:00 2001 From: vii Date: Wed, 14 Oct 2026 22:31:55 -0500 Subject: [PATCH 45/58] fix(vr): reload the OpenXR runtime with its own Vulkan loader instead of pinning it The Vulkan adapter exports vk* symbols globally, so a runtime loaded again after it bound its Vulkan calls to the adapter and deadlocked the join. Pinning avoided the reload but left SteamVR with its old VkInstance, which aborted the join with "vkGetInstanceProcAddr: Invalid instance". Later instances now load the runtime libraries with RTLD_DEEPBIND and let them unload with their instance. --- include/mocktail/vr/openxr_backend.h | 24 ++++++++++++++--- src/vr/openxr_backend.cc | 40 ++++++++++++++++------------ tests/vr/openxr_backend_test.cc | 26 ++++++++++-------- 3 files changed, 59 insertions(+), 31 deletions(-) diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index 96534115..343df123 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -395,6 +395,9 @@ class OpenXrBackend final { // can report it without re-entering runtime_name() under the held mutex. std::string runtime_name_; bool xr_instance_used_ = false; + // Libraries the runtime's first instance loaded; later instances load them + // with RTLD_DEEPBIND (see EnsureXrInstanceLocked). + std::vector runtime_libraries_; std::string user_manifest_; std::string runtime_manifest_; @@ -492,11 +495,26 @@ OpenXrBackend* ActiveVrBackend(); // Paths of the shared objects mapped into the process. std::vector LoadedSharedObjects(); -// Keeps every shared object loaded since `before` mapped until exit and -// returns the paths it pinned. -std::vector PinSharedObjectsLoadedSince( +// Paths mapped now that were not in `before`. +std::vector SharedObjectsLoadedSince( const std::vector& before); +// Holds the runtime's libraries open with RTLD_DEEPBIND while the OpenXR +// loader loads the runtime, so its Vulkan calls bind to its own libvulkan +// dependency instead of the Vulkan adapter's exported hooks. Released after +// the loader holds its own reference, so the runtime still unloads with its +// instance. +class DeepBoundLibraries { + public: + explicit DeepBoundLibraries(const std::vector& paths); + ~DeepBoundLibraries(); + DeepBoundLibraries(const DeepBoundLibraries&) = delete; + DeepBoundLibraries& operator=(const DeepBoundLibraries&) = delete; + + private: + std::vector handles_; +}; + } // namespace mocktail::vr #endif // MOCKTAIL_VR_OPENXR_BACKEND_H_ diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index a9792ae5..38c9c6b5 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -315,20 +315,31 @@ std::vector LoadedSharedObjects() { return paths; } -std::vector PinSharedObjectsLoadedSince( +std::vector SharedObjectsLoadedSince( const std::vector& before) { - std::vector pinned; + std::vector added; for (const auto& path : LoadedSharedObjects()) { - if (std::find(before.begin(), before.end(), path) != before.end()) { - continue; + if (std::find(before.begin(), before.end(), path) == before.end()) { + added.push_back(path); } - // RTLD_NOLOAD only adds a reference; RTLD_NODELETE makes it permanent. - if (dlopen(path.c_str(), RTLD_NOW | RTLD_NOLOAD | RTLD_NODELETE) != - nullptr) { - pinned.push_back(path); + } + return added; +} + +DeepBoundLibraries::DeepBoundLibraries( + const std::vector& paths) { + for (const auto& path : paths) { + if (void* handle = + dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND)) { + handles_.push_back(handle); } } - return pinned; +} + +DeepBoundLibraries::~DeepBoundLibraries() { + for (auto it = handles_.rbegin(); it != handles_.rend(); ++it) { + dlclose(*it); + } } void OpenXrBackend::DestroyXrInstanceLocked() { @@ -352,6 +363,7 @@ bool OpenXrBackend::EnsureXrInstanceLocked(std::string* error) { } try { const auto libraries_before_runtime = LoadedSharedObjects(); + const DeepBoundLibraries runtime_libraries(runtime_libraries_); const char *vk_extensions[] = {XR_KHR_VULKAN_ENABLE2_EXTENSION_NAME}; const char *gl_extensions[] = {XR_KHR_OPENGL_ES_ENABLE_EXTENSION_NAME, XR_MNDX_EGL_ENABLE_EXTENSION_NAME}; @@ -410,14 +422,8 @@ bool OpenXrBackend::EnsureXrInstanceLocked(std::string* error) { } CheckXr(XR_NULL_HANDLE, instance_result, "xrCreateInstance"); xr_instance_ = instance; - // The loader unloads the runtime with each instance. Loaded again after - // the Vulkan adapter (RTLD_GLOBAL), the runtime would call Vulkan through - // the adapter's hooks, which re-enter this backend under mutex_. - const auto pinned = PinSharedObjectsLoadedSince(libraries_before_runtime); - if (!pinned.empty()) { - Log(" [vr-backend] kept %zu runtime libraries loaded for later " - "sessions\n", - pinned.size()); + if (runtime_libraries_.empty()) { + runtime_libraries_ = SharedObjectsLoadedSince(libraries_before_runtime); } XrInstanceProperties properties = diff --git a/tests/vr/openxr_backend_test.cc b/tests/vr/openxr_backend_test.cc index 1d185a5d..12933c1a 100644 --- a/tests/vr/openxr_backend_test.cc +++ b/tests/vr/openxr_backend_test.cc @@ -607,23 +607,27 @@ TEST(OpenXrEyeSwapchainFormat, KeepsGuestEyeBytesWithoutGammaConversion) { VK_FORMAT_UNDEFINED); } -// The OpenXR loader unloads the runtime with its instance. A reload would -// bind the runtime's Vulkan calls to the adapter loaded since, so the -// libraries from the first instance stay mapped. -TEST(OpenXrRuntimeLibraries, PinnedLibrariesSurviveTheLoadersClose) { +// A runtime loaded through DeepBoundLibraries is reported as newly loaded +// and still unloads once every handle is closed, so each XR instance gets a +// fresh copy of the runtime. +TEST(OpenXrRuntimeLibraries, DeepBoundRuntimeUnloadsWithItsLastHandle) { const char* runtime = MOCKTAIL_TEST_OPENXR_RUNTIME; ASSERT_EQ(dlopen(runtime, RTLD_NOW | RTLD_NOLOAD), nullptr); const auto before = LoadedSharedObjects(); - void* handle = dlopen(runtime, RTLD_NOW | RTLD_LOCAL); - ASSERT_NE(handle, nullptr) << dlerror(); - const auto pinned = PinSharedObjectsLoadedSince(before); - EXPECT_NE(std::find(pinned.begin(), pinned.end(), std::string(runtime)), - pinned.end()); - EXPECT_EQ(PinSharedObjectsLoadedSince(LoadedSharedObjects()).size(), 0u); - ASSERT_EQ(dlclose(handle), 0); + void* loader_handle = nullptr; + { + const DeepBoundLibraries deep_bound({runtime}); + const auto loaded = SharedObjectsLoadedSince(before); + EXPECT_NE(std::find(loaded.begin(), loaded.end(), std::string(runtime)), + loaded.end()); + loader_handle = dlopen(runtime, RTLD_NOW | RTLD_LOCAL); + ASSERT_NE(loader_handle, nullptr) << dlerror(); + } void* still_loaded = dlopen(runtime, RTLD_NOW | RTLD_NOLOAD); EXPECT_NE(still_loaded, nullptr); if (still_loaded != nullptr) dlclose(still_loaded); + ASSERT_EQ(dlclose(loader_handle), 0); + EXPECT_EQ(dlopen(runtime, RTLD_NOW | RTLD_NOLOAD), nullptr); } } // namespace From 42b5138b114f9de4f7ec6e1467b7e61823e9c6ee Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Thu, 15 Oct 2026 01:12:01 -0500 Subject: [PATCH 46/58] fix(vr): deliver hand-motor and HapticEffect haptics the way games aim them The Android client drops HapticService:SetMotor on the Gamepad1 LeftHand and RightHand motors: its haptics backend only consumes Large and Small. VR games such as VR Hands drive those motors, so they never vibrated. - Interpose HapticService::SetMotor through its reflection descriptor, call the original, and forward Gamepad1 LeftHand/RightHand to OpenXR. The descriptor and the engine's intensity reader are found by pattern at load, for both the 2998 (tuple span) and 3222 (shared tuple) reader ABIs. The held level renews a 50 ms lease every presented frame. - Report the hand motors at x = -1/+1 in GetMotors. Positional HapticEffects aim at a hand there with a 0.5 radius (VOXEL DESTRUCT 2 does), so at +-0.2 every effect mixed to zero. - Play SetMotorState buffers sample by sample with trailing silence trimmed. Roblox sends 100 x 25 ms look-ahead buffers; taking the peak for the whole buffer turned a short bump into 2.5 s at full strength. - Stop vibrating from the DebugDeviceVR haptic slot. The engine pulls it every frame but with each mix buffer's first sample, held about a second, so it duplicated the buffers as a long buzz. - Enable SupportGenericHapticsOnVR and FixRegisterMotors with the other haptic flags. --- include/mocktail/vr/roblox_vr_device_bridge.h | 3 + .../runtime/roblox_haptic_protocol_bridge.h | 3 + scripts/find_roblox_vr_offsets.py | 2 + src/runtime/roblox_haptic_protocol_bridge.cc | 358 ++++++++++++++---- src/vr/roblox_vr_device_bridge.cc | 323 +++++++++++++++- src/vr/roblox_vr_device_bridge_internal.h | 46 +++ tests/roblox_haptic_protocol_bridge_test.cc | 156 ++++++-- tests/vr/roblox_vr_device_bridge_test.cc | 179 ++++++++- 8 files changed, 947 insertions(+), 123 deletions(-) diff --git a/include/mocktail/vr/roblox_vr_device_bridge.h b/include/mocktail/vr/roblox_vr_device_bridge.h index 5c6fa36e..ff7ee023 100644 --- a/include/mocktail/vr/roblox_vr_device_bridge.h +++ b/include/mocktail/vr/roblox_vr_device_bridge.h @@ -64,6 +64,9 @@ class RobloxVrDeviceBridge final { // Guest-side hooks invoked by the interposed vtable stubs. void OnHapticsCall(void* device_object, int hand, float amplitude); + // HapticService:SetMotor on Gamepad1 LeftHand (0) / RightHand (1); the + // level holds until the game sets it again. + void OnSetMotorCall(int hand, float amplitude); void OnPointerFrameCall(void* result, void* service); void OnStateGetterCall(void *result_buffer, void *device_object); void OnEyeGetterCall(void *device_object, int eye_index, void *framebuffer); diff --git a/include/runtime/roblox_haptic_protocol_bridge.h b/include/runtime/roblox_haptic_protocol_bridge.h index bc3b742e..4d96903d 100644 --- a/include/runtime/roblox_haptic_protocol_bridge.h +++ b/include/runtime/roblox_haptic_protocol_bridge.h @@ -16,6 +16,8 @@ using HapticSinkFn = std::function; using HapticStateSyncFn = std::function; +class HapticWaveformPlayer; + class RobloxHapticProtocolBridge final { public: RobloxHapticProtocolBridge( @@ -36,6 +38,7 @@ class RobloxHapticProtocolBridge final { static bool EnabledForThisSession(); private: + std::shared_ptr player_; std::unique_ptr bridge_; }; diff --git a/scripts/find_roblox_vr_offsets.py b/scripts/find_roblox_vr_offsets.py index 6030fec1..e1bb23a0 100755 --- a/scripts/find_roblox_vr_offsets.py +++ b/scripts/find_roblox_vr_offsets.py @@ -90,6 +90,8 @@ HAPTIC_FLAG_NAMES = { "keep_haptics_flag_rva": b"KeepHapticsOnMobileWhenGamepadConnected", "vr_haptics_flag_rva": b"VRHaptics", + "support_generic_haptics_flag_rva": b"SupportGenericHapticsOnVR", + "fix_register_motors_flag_rva": b"FixRegisterMotors", } OBJECT_TYPE_OFFSET = 0x10 diff --git a/src/runtime/roblox_haptic_protocol_bridge.cc b/src/runtime/roblox_haptic_protocol_bridge.cc index 8d666821..92f42d2a 100644 --- a/src/runtime/roblox_haptic_protocol_bridge.cc +++ b/src/runtime/roblox_haptic_protocol_bridge.cc @@ -4,11 +4,16 @@ #include #include +#include #include +#include #include +#include #include #include #include +#include +#include extern "C" { void mocktail_vr_controller_request_haptics(int32_t hand, float amplitude, @@ -68,42 +73,238 @@ std::optional ParseJsonMessage(JNIEnv* env, jstring message) { return parsed; } -void DispatchHapticForMotor(const std::string& motor_name, float amplitude, - uint64_t duration_ns, float frequency_hz, - const HapticSinkFn& sink, - const HapticStopFn& stop) { - int hand = -1; +// 0 left, 1 right, -1 both (a motor that names neither hand). +int HandForMotor(const std::string& motor_name) { std::string lower_name = motor_name; for (char& c : lower_name) { c = static_cast(std::tolower(static_cast(c))); } if (lower_name.find("left") != std::string::npos || lower_name.find("large") != std::string::npos) { - hand = 0; - } else if (lower_name.find("right") != std::string::npos || - lower_name.find("small") != std::string::npos) { - hand = 1; + return 0; } + if (lower_name.find("right") != std::string::npos || + lower_name.find("small") != std::string::npos) { + return 1; + } + return -1; +} - if (amplitude <= 0.0f || !std::isfinite(amplitude)) { - if (stop) { - if (hand == -1) { - stop(0); - stop(1); - } else { +// One message's demand on each controller: a hand vibrates at the strongest +// motor that maps to it instead of at whichever one came last. +struct HandDemand { + bool present = false; + float amplitude = 0.0f; + uint64_t duration_ns = 0; +}; + +void AddMotorDemand(HandDemand (&hands)[2], const std::string& motor_name, + float amplitude, uint64_t duration_ns) { + if (!std::isfinite(amplitude)) + amplitude = 0.0f; + const int hand = HandForMotor(motor_name); + for (int index = 0; index < 2; ++index) { + if (hand != -1 && hand != index) + continue; + auto& demand = hands[index]; + demand.present = true; + if (amplitude > demand.amplitude) { + demand.amplitude = amplitude; + demand.duration_ns = duration_ns; + } + } +} + +void DispatchHandDemands(const HandDemand (&hands)[2], const HapticSinkFn& sink, + const HapticStopFn& stop) { + for (int hand = 0; hand < 2; ++hand) { + const auto& demand = hands[hand]; + if (!demand.present) + continue; + if (demand.amplitude <= 0.0f) { + if (stop) stop(hand); + } else if (sink) { + sink(hand, std::clamp(demand.amplitude, 0.0f, 1.0f), + std::clamp(demand.duration_ns, kMinimumDurationNs, + kMaximumDurationNs), + 0.0f); + } + } +} + +} // namespace + +// Plays SetMotorState buffers sample by sample. Roblox sends HapticEffect +// output as look-ahead waveforms (e.g. 100 samples x 25 ms) and replaces them +// as it mixes, so each hand follows its latest buffer in real time. +class HapticWaveformPlayer { + public: + using Clock = std::chrono::steady_clock; + + HapticWaveformPlayer(HapticSinkFn sink, HapticStopFn stop) + : sink_(std::move(sink)), stop_(std::move(stop)) {} + ~HapticWaveformPlayer() { Halt(); } + + HapticWaveformPlayer(const HapticWaveformPlayer&) = delete; + HapticWaveformPlayer& operator=(const HapticWaveformPlayer&) = delete; + + // Starts `samples` (already trimmed, at least one) on `hand` now. + void Play(int hand, std::vector samples, uint64_t ns_per_sample) { + { + std::lock_guard lock(mutex_); + if (halted_) + return; + tracks_[hand] = Track{std::move(samples), ns_per_sample, Clock::now(), 0, + true}; + if (!thread_.joinable()) + thread_ = std::thread([this] { Run(); }); + } + wake_.notify_all(); + } + + void Stop(int hand) { + { + std::lock_guard lock(mutex_); + tracks_[hand].active = false; + } + if (stop_) + stop_(hand); + } + + // Accepts buffers again after Halt. + void Resume() { + std::lock_guard lock(mutex_); + halted_ = false; + } + + // Ends playback, stopping any hand still vibrating. + void Halt() { + bool was_active[2] = {}; + { + std::lock_guard lock(mutex_); + halted_ = true; + for (int hand = 0; hand < 2; ++hand) { + was_active[hand] = tracks_[hand].active; + tracks_[hand].active = false; } } - } else { - amplitude = std::clamp(amplitude, 0.0f, 1.0f); - duration_ns = std::clamp(duration_ns, kMinimumDurationNs, kMaximumDurationNs); - if (sink) { - if (hand == -1) { - sink(0, amplitude, duration_ns, frequency_hz); - sink(1, amplitude, duration_ns, frequency_hz); - } else { - sink(hand, amplitude, duration_ns, frequency_hz); + wake_.notify_all(); + if (thread_.joinable()) + thread_.join(); + for (int hand = 0; hand < 2; ++hand) { + if (was_active[hand] && stop_) + stop_(hand); + } + } + + private: + struct Track { + std::vector samples; + uint64_t ns_per_sample = 0; + Clock::time_point start; + std::size_t next = 0; + bool active = false; + }; + struct Step { + int hand; + float amplitude; + uint64_t duration_ns; + }; + + void Run() { + std::unique_lock lock(mutex_); + while (!halted_) { + const auto now = Clock::now(); + std::optional wake_at; + std::vector steps; + for (int hand = 0; hand < 2; ++hand) { + Track& track = tracks_[hand]; + if (!track.active) + continue; + const auto period = std::chrono::nanoseconds(track.ns_per_sample); + const auto index = static_cast((now - track.start) / period); + if (index >= track.samples.size()) { + track.active = false; + steps.push_back({hand, 0.0f, 0}); + continue; + } + // A late wake skips to the current sample rather than replaying. + if (index >= track.next) { + steps.push_back({hand, track.samples[index], track.ns_per_sample}); + track.next = index + 1; + } + const auto due = track.start + period * track.next; + if (!wake_at || due < *wake_at) + wake_at = due; + } + lock.unlock(); + for (const Step& step : steps) { + if (step.amplitude <= 0.0f) { + if (stop_) + stop_(step.hand); + } else if (sink_) { + // The lease outlives one sample a little so steps join seamlessly; + // the next sample, or the final stop, replaces it. + sink_(step.hand, step.amplitude, + std::clamp(step.duration_ns, kMinimumDurationNs, + kMaximumDurationNs), + 0.0f); + } } + lock.lock(); + if (halted_) + break; + if (wake_at) + wake_.wait_until(lock, *wake_at); + else + wake_.wait(lock); + } + } + + const HapticSinkFn sink_; + const HapticStopFn stop_; + std::mutex mutex_; + std::condition_variable wake_; + Track tracks_[2]; + bool halted_ = false; + std::thread thread_; +}; + +namespace { + +// One hand's share of a SetMotorState: the per-sample maximum of its motors, +// on the period of the first motor that reached it. +struct HandWaveform { + bool present = false; + uint64_t ns_per_sample = 0; + std::vector samples; +}; + +void AddMotorWaveform(HandWaveform (&hands)[2], const std::string& motor_name, + const std::vector& samples, + uint64_t ns_per_sample) { + const int hand = HandForMotor(motor_name); + for (int index = 0; index < 2; ++index) { + if (hand != -1 && hand != index) + continue; + HandWaveform& wave = hands[index]; + if (!wave.present) { + wave.present = true; + wave.ns_per_sample = ns_per_sample; + } + // Resample by time onto the hand's period (Roblox uses one period for + // every motor, so this is normally index for index). + const double ratio = static_cast(wave.ns_per_sample) / + static_cast(ns_per_sample); + const auto length = static_cast( + std::ceil(static_cast(samples.size()) / ratio)); + if (wave.samples.size() < length) + wave.samples.resize(length, 0.0f); + for (std::size_t at = 0; at < length; ++at) { + const auto source = static_cast(at * ratio); + if (source < samples.size()) + wave.samples[at] = std::max(wave.samples[at], samples[source]); } } } @@ -111,7 +312,8 @@ void DispatchHapticForMotor(const std::string& motor_name, float amplitude, RobloxMessageBusReply Answer(JNIEnv* env, jstring message, std::size_t method, const HapticSinkFn& sink, const HapticStopFn& stop, - const HapticStateSyncFn& state_sync) { + const HapticStateSyncFn& state_sync, + HapticWaveformPlayer& player) { // Method 0: SupportsHaptics if (method == 0) { if (state_sync) { @@ -136,41 +338,50 @@ RobloxMessageBusReply Answer(JNIEnv* env, jstring message, std::size_t method, return RobloxMessageBusReply{"{}", 13, "invalid_request"}; } const auto& root = *parsed; + HandWaveform hands[2]; for (auto it = root.begin(); it != root.end(); ++it) { - const std::string& motor_name = it.key(); - const auto& motor_val = it.value(); - if (!motor_val.is_object()) + const auto& motor = it.value(); + if (!motor.is_object()) continue; - - float max_intensity = 0.0f; - std::size_t sample_count = 1; double ms_per_sample = 20.0; - - if (motor_val.contains("millisecondsPerSample") && - motor_val["millisecondsPerSample"].is_number()) { - ms_per_sample = motor_val["millisecondsPerSample"].get(); + if (motor.contains("millisecondsPerSample") && + motor["millisecondsPerSample"].is_number()) { + ms_per_sample = motor["millisecondsPerSample"].get(); } - - if (motor_val.contains("intensities") && - motor_val["intensities"].is_array()) { - const auto& arr = motor_val["intensities"]; - sample_count = std::max(1, arr.size()); - for (const auto& item : arr) { - if (item.is_number()) { - float val = item.get(); - if (val > max_intensity) - max_intensity = val; - } + if (!std::isfinite(ms_per_sample) || ms_per_sample <= 0.0) + continue; + ms_per_sample = std::clamp(ms_per_sample, 1.0, 1000.0); + const auto ns_per_sample = + static_cast(ms_per_sample * 1000000.0); + std::vector samples; + if (motor.contains("intensities") && motor["intensities"].is_array()) { + for (const auto& item : motor["intensities"]) { + const float value = item.is_number() ? item.get() : 0.0f; + samples.push_back(std::isfinite(value) ? std::clamp(value, 0.0f, 1.0f) + : 0.0f); } - } else if (motor_val.contains("intensity") && - motor_val["intensity"].is_number()) { - max_intensity = motor_val["intensity"].get(); + } else if (motor.contains("intensity") && motor["intensity"].is_number()) { + const float value = motor["intensity"].get(); + samples.push_back( + std::isfinite(value) ? std::clamp(value, 0.0f, 1.0f) : 0.0f); } - - uint64_t duration_ns = static_cast( - std::max(0.0, ms_per_sample) * sample_count * 1000000.0); - DispatchHapticForMotor(motor_name, max_intensity, duration_ns, 0.0f, sink, - stop); + AddMotorWaveform(hands, it.key(), samples, ns_per_sample); + } + for (int hand = 0; hand < 2; ++hand) { + HandWaveform& wave = hands[hand]; + if (!wave.present) + continue; + // The buffer is padded with silence; play only up to its last bump. + while (!wave.samples.empty() && wave.samples.back() <= 0.0f) + wave.samples.pop_back(); + const std::size_t longest = + static_cast(kMaximumDurationNs / wave.ns_per_sample); + if (wave.samples.size() > longest) + wave.samples.resize(longest); + if (wave.samples.empty()) + player.Stop(hand); + else + player.Play(hand, std::move(wave.samples), wave.ns_per_sample); } return RobloxMessageBusReply{"{}", 0, "ok"}; } @@ -185,8 +396,9 @@ RobloxMessageBusReply Answer(JNIEnv* env, jstring message, std::size_t method, if (parsed->contains("intensity") && (*parsed)["intensity"].is_number()) { intensity = (*parsed)["intensity"].get(); } - DispatchHapticForMotor("defaultMotor", intensity, kMinimumDurationNs, 0.0f, - sink, stop); + HandDemand hands[2]; + AddMotorDemand(hands, "defaultMotor", intensity, kMinimumDurationNs); + DispatchHandDemands(hands, sink, stop); return RobloxMessageBusReply{"{}", 0, "ok"}; } @@ -202,15 +414,17 @@ bool RobloxHapticProtocolBridge::EnabledForThisSession() { } std::string RobloxHapticProtocolBridge::GetMotorsJson() { + // Positional HapticEffects place hands at x = -1 (left) and +1 (right): + // VOXEL DESTRUCT 2 aims its per-hand effects there with a 0.5 radius. const nlohmann::json motors = { - {"leftHand", {{"x", -0.2}, {"y", 0.0}, {"z", 0.0}}}, - {"rightHand", {{"x", 0.2}, {"y", 0.0}, {"z", 0.0}}}, - {"LeftHand", {{"x", -0.2}, {"y", 0.0}, {"z", 0.0}}}, - {"RightHand", {{"x", 0.2}, {"y", 0.0}, {"z", 0.0}}}, - {"Large", {{"x", -0.2}, {"y", 0.0}, {"z", 0.0}}}, - {"Small", {{"x", 0.2}, {"y", 0.0}, {"z", 0.0}}}, - {"large", {{"x", -0.2}, {"y", 0.0}, {"z", 0.0}}}, - {"small", {{"x", 0.2}, {"y", 0.0}, {"z", 0.0}}}, + {"leftHand", {{"x", -1.0}, {"y", 0.0}, {"z", 0.0}}}, + {"rightHand", {{"x", 1.0}, {"y", 0.0}, {"z", 0.0}}}, + {"LeftHand", {{"x", -1.0}, {"y", 0.0}, {"z", 0.0}}}, + {"RightHand", {{"x", 1.0}, {"y", 0.0}, {"z", 0.0}}}, + {"Large", {{"x", -1.0}, {"y", 0.0}, {"z", 0.0}}}, + {"Small", {{"x", 1.0}, {"y", 0.0}, {"z", 0.0}}}, + {"large", {{"x", -1.0}, {"y", 0.0}, {"z", 0.0}}}, + {"small", {{"x", 1.0}, {"y", 0.0}, {"z", 0.0}}}, {"defaultMotor", {{"x", 0.0}, {"y", 0.0}, {"z", 0.0}}}, }; return motors.dump(); @@ -250,25 +464,33 @@ RobloxHapticProtocolBridge::RobloxHapticProtocolBridge( } }; } + player_ = std::make_shared(haptic_sink, haptic_stop); bridge_ = std::make_unique( environment, symbols, objects, "HapticProtocol", std::vector{"SupportsHaptics", "GetMotors", "SetMotorState", "UpdateSingletonVibration"}, [sink = std::move(haptic_sink), stop = std::move(haptic_stop), - sync = std::move(haptic_state_sync)]( + sync = std::move(haptic_state_sync), player = player_]( JNIEnv* env, jstring message, std::size_t method) { - return Answer(env, message, method, sink, stop, sync); + return Answer(env, message, method, sink, stop, sync, *player); }); } -RobloxHapticProtocolBridge::~RobloxHapticProtocolBridge() = default; +RobloxHapticProtocolBridge::~RobloxHapticProtocolBridge() { + // A handler may outlive the bridge inside the message bus; never let its + // player keep a controller vibrating. + player_->Halt(); +} Status RobloxHapticProtocolBridge::Initialize() { + player_->Resume(); return bridge_->Initialize(); } Status RobloxHapticProtocolBridge::Shutdown() { - return bridge_->Shutdown(); + const Status status = bridge_->Shutdown(); + player_->Halt(); + return status; } } // namespace runtime diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc index dc667210..fcaec917 100644 --- a/src/vr/roblox_vr_device_bridge.cc +++ b/src/vr/roblox_vr_device_bridge.cc @@ -392,6 +392,158 @@ void RestoreIsVrAppBuildHook() { } } +// HapticService:SetMotor on the hand motors: a level the game holds until it +// sets another. (HapticEffect output plays from SetMotorState instead.) +constexpr std::uint64_t kHapticLeaseNs = 50000000; +std::atomic g_set_motor_level[2] = {}; + +void ApplyHandHaptics(int hand) { + auto* backend = ActiveVrBackend(); + if (backend == nullptr) return; + const float level = g_set_motor_level[hand].load(std::memory_order_relaxed); + // A short lease, refreshed every engine frame, so a stalled or + // disconnected engine never leaves a controller vibrating. + if (level > 0) + backend->RequestControllerHaptics(hand, level, kHapticLeaseNs, 0); + else + backend->StopControllerHaptics(hand); +} + +void ResetHandHaptics() { + for (int hand = 0; hand < 2; ++hand) + g_set_motor_level[hand].store(0, std::memory_order_relaxed); +} + +using SetMotorFn = void (*)(void* service, std::int32_t input_type, + std::int32_t motor, const void* values); +std::uintptr_t* g_set_motor_slot = nullptr; +// Kept for the process lifetime, like the getters: a call already past the +// slot load still forwards after the slot is restored. +std::atomic g_set_motor_original{nullptr}; +std::atomic g_motor_intensity_reader{0}; +std::atomic g_motor_intensity_abi{ + internal::MotorIntensityAbi::kNone}; + +// `values` is the guest's shared_ptr; a Tuple starts with its +// std::vector. +std::optional ReadMotorIntensity(const void* values) { + const auto reader = g_motor_intensity_reader.load(std::memory_order_acquire); + if (reader == 0 || values == nullptr) return std::nullopt; + const void* tuple = nullptr; + std::memcpy(&tuple, values, sizeof(tuple)); + if (tuple == nullptr) return std::nullopt; + const void* span[2] = {}; + std::memcpy(span, tuple, sizeof(span)); + // SetMotor rejects an empty tuple; the reader would read past it. + if (span[0] == nullptr || span[0] == span[1]) return std::nullopt; + float intensity = 0; + switch (g_motor_intensity_abi.load(std::memory_order_acquire)) { + case internal::MotorIntensityAbi::kSharedTuple: + intensity = reinterpret_cast(reader)(values); + break; + case internal::MotorIntensityAbi::kTupleSpan: + intensity = reinterpret_cast(reader)( + span[0], span[1]); + break; + case internal::MotorIntensityAbi::kNone: + return std::nullopt; + } + if (!std::isfinite(intensity)) return 0.f; + return std::clamp(intensity, 0.f, 1.f); +} + +// Aligned for the same member function pointer reason as IsVRAppBuild. +extern "C" __attribute__((aligned(16))) void mocktail_vr_haptic_set_motor( + void* service, std::int32_t input_type, std::int32_t motor, + const void* values) { + HookCall in_flight; + if (const auto original = + g_set_motor_original.load(std::memory_order_acquire)) { + original(service, input_type, motor, values); + } + if (input_type != internal::kUserInputTypeGamepad1 || + (motor != internal::kVibrationMotorLeftHand && + motor != internal::kVibrationMotorRightHand)) { + return; + } + auto* bridge = g_active_bridge.load(std::memory_order_seq_cst); + if (bridge == nullptr || !bridge->active()) return; + if (const auto intensity = ReadMotorIntensity(values)) { + bridge->OnSetMotorCall(motor - internal::kVibrationMotorLeftHand, + *intensity); + } +} + +bool IsExecutableRva(std::uintptr_t image_base, std::uintptr_t rva) { + return IsExecutableImageRange(image_base, rva, 1); +} + +void InstallSetMotorHook(std::uintptr_t image_base) { + if (g_set_motor_slot != nullptr || !IsValidElfImage(image_base)) return; + const auto* image = reinterpret_cast(image_base); + const std::size_t image_size = ImageSize(image_base); + const auto* header = reinterpret_cast(image_base); + const auto* program_headers = reinterpret_cast( + image_base + static_cast(header->e_phoff)); + internal::MotorIntensityReader reader; + std::uintptr_t descriptor = 0; + for (std::size_t index = 0; index < header->e_phnum; ++index) { + const Elf64_Phdr& segment = program_headers[index]; + if (segment.p_type != PT_LOAD || + (segment.p_flags & (PF_R | PF_X)) != (PF_R | PF_X)) { + continue; + } + if (reader.abi == internal::MotorIntensityAbi::kNone) { + reader = internal::FindMotorIntensityReader( + image, image_size, segment.p_vaddr, segment.p_memsz); + } + if (descriptor == 0) { + descriptor = internal::FindSetMotorDescriptor( + image, image_size, segment.p_vaddr, segment.p_memsz); + } + } + if (reader.abi == internal::MotorIntensityAbi::kNone || + !IsExecutableImageRange(image_base, reader.rva, 1) || descriptor == 0 || + !IsWritableDataImageRange(image_base, descriptor, 0x98)) { + Log(" [vr-device] SetMotor hand haptics unavailable: reader=+0x%" PRIxPTR + " descriptor=+0x%" PRIxPTR "\n", + reader.rva, descriptor); + return; + } + const std::size_t slot_offset = + internal::SetMotorDescriptorSlot(image_base, descriptor, &IsExecutableRva); + if (slot_offset == 0) { + Log(" [vr-device] SetMotor hand haptics unavailable: no method slot in " + "descriptor +0x%" PRIxPTR "\n", + descriptor); + return; + } + auto* slot = + reinterpret_cast(image_base + descriptor + slot_offset); + g_motor_intensity_reader.store(image_base + reader.rva, + std::memory_order_release); + g_motor_intensity_abi.store(reader.abi, std::memory_order_release); + g_set_motor_original.store(reinterpret_cast(*slot), + std::memory_order_release); + g_set_motor_slot = slot; + __atomic_store_n(slot, FunctionAddress(&mocktail_vr_haptic_set_motor), + __ATOMIC_RELEASE); + Log(" [vr-device] interposed HapticService::SetMotor descriptor at +0x%" PRIxPTR + " (hand motors -> OpenXR, reader +0x%" PRIxPTR ")\n", + descriptor + slot_offset, reader.rva); +} + +void RestoreSetMotorHook() { + if (g_set_motor_slot != nullptr) { + __atomic_store_n( + g_set_motor_slot, + reinterpret_cast( + g_set_motor_original.load(std::memory_order_acquire)), + __ATOMIC_RELEASE); + g_set_motor_slot = nullptr; + } +} + } // namespace namespace internal { @@ -410,6 +562,8 @@ constexpr VrBuildContract kContracts[] = { 0x140, 0x72fcd18, 0x7367588, + 0x73677d0, + 0x7367da0, "2998 (2.736.1408)"}, {"5f0704edd9064f566ee3d6df2bd2fabbcc709f03", {0x6d08108, 0x37f9fe4, 0x37fa3c6, 0x37fa3b6, 0x37fa3e8, 0x73f4690, @@ -424,6 +578,8 @@ constexpr VrBuildContract kContracts[] = { 0x140, 0x7387118, 0x73f2268, + 0x73f24f0, + 0x73f2aa8, "3092 (2.738.1397)"}, {"a67a91ef96c700c0730acd36a250c2f93cfafbc1", {0x7083a48, 0x549c884, 0x549cc66, 0x549cc56, 0x549cc88, 0x74d1640, @@ -438,6 +594,8 @@ constexpr VrBuildContract kContracts[] = { 0x130, 0x763fef8, 0x74cf1e0, + 0x74cf458, + 0x74cfa10, "3222 (2.742.1059)"}, }; @@ -761,6 +919,114 @@ bool EnableNamedFastFlag(std::uintptr_t image_base, std::uintptr_t rva, return true; } +MotorIntensityReader FindMotorIntensityReader(const std::uint8_t* image, + std::size_t image_size, + std::uintptr_t scan_rva, + std::size_t scan_size) { + // cmp esi,0xc; jne rel32; cmp edx,1; je rel8; test edx,edx; jne rel8 + static constexpr std::uint8_t kDispatch[] = {0x83, 0xfe, 0x0c, 0x0f, 0x85}; + static constexpr std::size_t kDispatchSize = 18; + // 3222: lea r15,[rbp+disp8]; mov rdi,r15 (a copy of the shared_ptr). + static constexpr std::uint8_t kSharedTuple[] = {0x4c, 0x89, 0xff, 0xe8}; + // 2998: mov rdi,[rax]; mov rsi,[rax+8] (the tuple's begin and end). + static constexpr std::uint8_t kTupleSpan[] = {0x48, 0x8b, 0x38, 0x48, + 0x8b, 0x70, 0x08, 0xe8}; + // movss [r14+0x54],xmm0: the Large motor store after the call. + static constexpr std::uint8_t kLargeStore[] = {0xf3, 0x41, 0x0f, 0x11, + 0x46, 0x54}; + static constexpr std::size_t kCallWindow = 48; + // Longest read past a window offset: the span setup, rel32 and the store. + static constexpr std::size_t kWindowTail = + sizeof(kTupleSpan) + 4 + sizeof(kLargeStore); + if (scan_rva > image_size || scan_size > image_size - scan_rva) return {}; + MotorIntensityReader found; + int matches = 0; + for (std::size_t at = scan_rva; + at + kDispatchSize + kCallWindow + kWindowTail <= scan_rva + scan_size; + ++at) { + const std::uint8_t* code = image + at; + if (std::memcmp(code, kDispatch, sizeof(kDispatch)) != 0 || + code[9] != 0x83 || code[10] != 0xfa || code[11] != 0x01 || + code[12] != 0x74 || code[14] != 0x85 || code[15] != 0xd2 || + code[16] != 0x75) { + continue; + } + for (std::size_t offset = kDispatchSize; offset < kDispatchSize + kCallWindow; + ++offset) { + MotorIntensityAbi abi = MotorIntensityAbi::kNone; + std::size_t call = 0; + if (std::memcmp(code + offset, kSharedTuple, sizeof(kSharedTuple)) == 0) { + abi = MotorIntensityAbi::kSharedTuple; + call = offset + sizeof(kSharedTuple) - 1; + } else if (std::memcmp(code + offset, kTupleSpan, sizeof(kTupleSpan)) == + 0) { + abi = MotorIntensityAbi::kTupleSpan; + call = offset + sizeof(kTupleSpan) - 1; + } else { + continue; + } + if (std::memcmp(code + call + 5, kLargeStore, sizeof(kLargeStore)) != 0) { + break; + } + found = {RelativeTarget(code + call + 1, at + call + 5), abi}; + ++matches; + break; + } + } + if (matches != 1 || found.rva >= image_size) return {}; + return found; +} + +std::uintptr_t FindSetMotorDescriptor(const std::uint8_t* image, + std::size_t image_size, + std::uintptr_t scan_rva, + std::size_t scan_size) { + static constexpr char kName[] = "SetMotor"; + if (scan_rva > image_size || scan_size > image_size - scan_rva) return 0; + std::uintptr_t found = 0; + int matches = 0; + // lea rdi,[rip+descriptor]; lea rdx,[rip+"SetMotor"] + for (std::size_t at = scan_rva; at + 14 <= scan_rva + scan_size; ++at) { + const std::uint8_t* code = image + at; + if (code[0] != 0x48 || code[1] != 0x8d || code[2] != 0x3d || + code[7] != 0x48 || code[8] != 0x8d || code[9] != 0x15) { + continue; + } + const std::uintptr_t name = RelativeTarget(code + 10, at + 14); + if (name >= image_size || image_size - name < sizeof(kName) || + std::memcmp(image + name, kName, sizeof(kName)) != 0) { + continue; + } + found = RelativeTarget(code + 3, at + 7); + ++matches; + } + return matches == 1 ? found : 0; +} + +std::size_t SetMotorDescriptorSlot(std::uintptr_t image_base, + std::uintptr_t descriptor_rva, + bool (*is_code)(std::uintptr_t image_base, + std::uintptr_t rva)) { + for (const std::size_t slot : {std::size_t{0x80}, std::size_t{0x88}}) { + std::uintptr_t function = 0; + std::uintptr_t adjust = 0; + std::memcpy(&function, + reinterpret_cast(image_base + descriptor_rva + + slot), + sizeof(function)); + std::memcpy(&adjust, + reinterpret_cast(image_base + descriptor_rva + + slot + sizeof(function)), + sizeof(adjust)); + // An odd pointer is a virtual-call offset, not code. + if (function > image_base && function % 2 == 0 && adjust == 0 && + is_code(image_base, function - image_base)) { + return slot; + } + } + return 0; +} + float GetHandPitchOffsetDeg() { static const float offset = [] { if (const char* env = std::getenv("MOCKTAIL_VR_HAND_PITCH_OFFSET")) { @@ -1033,7 +1299,9 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { contract_->hand_pitch_storage_rva + sizeof(int), contract_->haptic_sink_rva + 1, contract_->keep_haptics_flag_rva + 1, - contract_->vr_haptics_flag_rva + 1}); + contract_->vr_haptics_flag_rva + 1, + contract_->support_generic_haptics_flag_rva + 1, + contract_->fix_register_motors_flag_rva + 1}); if (image_base > std::numeric_limits::max() - largest_rva) { return InvalidArgument("VR device profile address overflows"); } @@ -1161,6 +1429,8 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { return FailedPrecondition("could not install verified VR aim pointer hook"); } InstallIsVrAppBuildHook(image_base); + ResetHandHaptics(); + InstallSetMotorHook(image_base); if (internal::EnableNamedFastFlag(image_base, contract_->keep_haptics_flag_rva, "KeepHapticsOnMobileWhenGamepadConnected")) { @@ -1172,6 +1442,18 @@ Status RobloxVrDeviceBridge::Activate(std::uintptr_t image_base) { Log(" [vr-device] enabled VRHaptics at +0x%" PRIxPTR "\n", contract_->vr_haptics_flag_rva); } + if (internal::EnableNamedFastFlag(image_base, + contract_->support_generic_haptics_flag_rva, + "SupportGenericHapticsOnVR")) { + Log(" [vr-device] enabled SupportGenericHapticsOnVR at +0x%" PRIxPTR "\n", + contract_->support_generic_haptics_flag_rva); + } + if (internal::EnableNamedFastFlag(image_base, + contract_->fix_register_motors_flag_rva, + "FixRegisterMotors")) { + Log(" [vr-device] enabled FixRegisterMotors at +0x%" PRIxPTR "\n", + contract_->fix_register_motors_flag_rva); + } { std::lock_guard pose_lock(g_native_pose_mutex); g_native_pose = {}; @@ -1228,6 +1510,8 @@ void RobloxVrDeviceBridge::Shutdown() { bool RobloxVrDeviceBridge::RestoreVtableLocked() { RestoreIsVrAppBuildHook(); + RestoreSetMotorHook(); + ResetHandHaptics(); (void)RestorePointerHook(); if (vtable_ == nullptr) { return false; @@ -1263,6 +1547,11 @@ void RobloxVrDeviceBridge::NoteHostPresent() { thread_id); } g_thread_presented = true; + // SetMotor is a held level with no engine-side refresh; renew its lease. + for (int hand = 0; hand < 2; ++hand) { + if (g_set_motor_level[hand].load(std::memory_order_relaxed) > 0) + ApplyHandHaptics(hand); + } mocktail_vr_consume_eye(); } @@ -1506,19 +1795,25 @@ void RobloxVrDeviceBridge::OnHapticsCall(void *device_object, int hand, std::lock_guard lock(mutex_); if (!active() || !ValidateObjectHeader(device_object) || hand < 0 || hand > 1) return; - if (auto *backend = ActiveVrBackend()) { - static std::atomic logged[2]{}; - if (!logged[hand].exchange(true)) - Log(" [vr-input] native haptic callback observed: hand=%d " - "amplitude=%.3f\n", - hand, amplitude); - if (!std::isfinite(amplitude) || amplitude <= 0) - backend->StopControllerHaptics(hand); - // Native pull refreshes once per engine frame. A short lease prevents a - // stalled or disconnected engine from leaving the controller vibrating. - else - backend->RequestControllerHaptics(hand, amplitude, 50000000, 0); - } + // Observed only. The engine pulls this every frame but with each mix + // buffer's first sample, which it holds until the next buffer (~1 s): a + // 50 ms click arrives as a 1 s buzz. The same HapticEffect output plays + // sample by sample from SetMotorState in the HapticProtocol bridge. + static std::atomic logged[2]{}; + if (amplitude > 0 && !logged[hand].exchange(true)) + Log(" [vr-input] native haptic callback observed: hand=%d " + "amplitude=%.3f (played from SetMotorState instead)\n", + hand, amplitude); +} + +void RobloxVrDeviceBridge::OnSetMotorCall(int hand, float amplitude) { + if (!active() || hand < 0 || hand > 1) return; + static std::atomic logged[2]{}; + if (amplitude > 0 && !logged[hand].exchange(true)) + Log(" [vr-input] SetMotor hand haptics observed: hand=%d amplitude=%.3f\n", + hand, amplitude); + g_set_motor_level[hand].store(amplitude, std::memory_order_relaxed); + ApplyHandHaptics(hand); } void RobloxVrDeviceBridge::OnStateGetterCall(void *result_buffer, diff --git a/src/vr/roblox_vr_device_bridge_internal.h b/src/vr/roblox_vr_device_bridge_internal.h index 7074ce10..43ac5d91 100644 --- a/src/vr/roblox_vr_device_bridge_internal.h +++ b/src/vr/roblox_vr_device_bridge_internal.h @@ -26,6 +26,8 @@ struct VrBuildContract { std::uintptr_t camera_head_scale_offset; std::uintptr_t keep_haptics_flag_rva; std::uintptr_t vr_haptics_flag_rva; + std::uintptr_t support_generic_haptics_flag_rva; + std::uintptr_t fix_register_motors_flag_rva; // Roblox version code and name, for messages that list supported builds. std::string_view version; }; @@ -41,6 +43,50 @@ float ReadCameraHeadScale(const void* camera, const VrBuildContract& contract); bool EnableNamedFastFlag(std::uintptr_t image_base, std::uintptr_t rva, std::string_view name); +// HapticService:SetMotor arguments the Android client drops: its haptics +// backend only consumes Gamepad1 Large/Small, so LeftHand/RightHand need a +// VR sink of their own. +inline constexpr std::int32_t kUserInputTypeGamepad1 = 12; +inline constexpr std::int32_t kVibrationMotorLeftHand = 4; +inline constexpr std::int32_t kVibrationMotorRightHand = 5; + +// How the engine's SetMotor intensity reader takes the value tuple. +enum class MotorIntensityAbi { + kNone, + // float(const shared_ptr*), 3222. + kSharedTuple, + // float(const Variant* begin, const Variant* end), 2998. + kTupleSpan, +}; +struct MotorIntensityReader { + std::uintptr_t rva = 0; + MotorIntensityAbi abi = MotorIntensityAbi::kNone; +}; + +// Both scan [scan_rva, scan_rva + scan_size) of a flat `image` view whose +// offsets are RVAs, and succeed only on exactly one match. +// +// The reader is the call in the Android backend's motor slot: +// cmp esi,Gamepad1; jne; cmp edx,1; je; test edx,edx; jne; ...; call reader; +// movss [r14+0x54],xmm0 +MotorIntensityReader FindMotorIntensityReader(const std::uint8_t* image, + std::size_t image_size, + std::uintptr_t scan_rva, + std::size_t scan_size); +// The descriptor is the one registered as lea rdi,[descriptor]; +// lea rdx,["SetMotor"]. Returns its RVA, or 0. +std::uintptr_t FindSetMotorDescriptor(const std::uint8_t* image, + std::size_t image_size, + std::uintptr_t scan_rva, + std::size_t scan_size); +// Offset of the member function pointer in that descriptor: the first of +// +0x80/+0x88 holding an even code address followed by a zero this-adjust. +// `is_code` says whether an RVA is executable. Returns 0 when neither fits. +std::size_t SetMotorDescriptorSlot(std::uintptr_t image_base, + std::uintptr_t descriptor_rva, + bool (*is_code)(std::uintptr_t image_base, + std::uintptr_t rva)); + // Computes the record orientation for a hand pose: the aim pose (or, with // MOCKTAIL_VR_HAND_POSE_SOURCE=grip, the grip pose) pitched by the engine's // own hand pitch plus the configurable offset. diff --git a/tests/roblox_haptic_protocol_bridge_test.cc b/tests/roblox_haptic_protocol_bridge_test.cc index c0521f3f..741edf45 100644 --- a/tests/roblox_haptic_protocol_bridge_test.cc +++ b/tests/roblox_haptic_protocol_bridge_test.cc @@ -10,7 +10,9 @@ #include #include #include +#include #include +#include #include "jnivm/jnivm.h" @@ -268,51 +270,135 @@ TEST_F(RobloxHapticProtocolBridgeTest, GetMotorsReturnsValidPositions) { EXPECT_GT(reply.body["rightHand"]["x"].get(), 0.0); } -TEST_F(RobloxHapticProtocolBridgeTest, SetMotorStateRoutesHands) { +// Waits for the waveform player thread; returns whether `done` became true. +template +bool WaitFor(Probe& probe, Predicate done) { + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(2); + while (std::chrono::steady_clock::now() < deadline) { + { + std::lock_guard lock(probe.mutex); + if (done()) + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return false; +} + +// SetMotorState is a look-ahead waveform. Each sample plays for its own slot +// (a short lease, never the whole buffer), in order, then the hand stops. +TEST_F(RobloxHapticProtocolBridgeTest, SetMotorStateStepsThroughTheSamples) { ASSERT_TRUE(Initialize().ok()); - const char* payload_left = R"({ - "leftHand": { - "millisecondsPerSample": 20.0, - "intensities": [0.2, 0.8, 0.5] - } + const char* payload = R"({ + "LeftHand": {"millisecondsPerSample": 20.0, "intensities": [0.0, 0.0]}, + "RightHand": {"millisecondsPerSample": 20.0, + "intensities": [0.2, 0.8, 0.5, 0.0, 0.0, 0.0]}, + "Small": {"millisecondsPerSample": 20.0, "intensities": [0.0, 0.0, 0.0]} })"; - const auto reply = Legacy("SetMotorState", payload_left); - EXPECT_EQ(reply.code, 0); - ASSERT_EQ(probe.recorded_sinks.size(), 1u); - EXPECT_EQ(probe.recorded_sinks[0].hand, 0); - EXPECT_FLOAT_EQ(probe.recorded_sinks[0].amplitude, 0.8f); - EXPECT_GE(probe.recorded_sinks[0].duration_ns, 50000000ULL); + const auto started = std::chrono::steady_clock::now(); + EXPECT_EQ(Legacy("SetMotorState", payload).code, 0); + ASSERT_TRUE(WaitFor(probe, [&] { + return std::count(probe.recorded_stops.begin(), probe.recorded_stops.end(), + 1) > 0; + })); + const auto elapsed = std::chrono::steady_clock::now() - started; + + std::lock_guard lock(probe.mutex); + // The silent left hand stops at once; the right hand plays its samples in + // order (a late wake may skip one) and nothing ever holds the peak. + EXPECT_EQ(probe.recorded_stops.front(), 0); + const std::vector expected = {0.2f, 0.8f, 0.5f}; + std::size_t next = 0; + for (const auto& sink : probe.recorded_sinks) { + EXPECT_EQ(sink.hand, 1); + EXPECT_EQ(sink.duration_ns, 50000000ULL); + while (next < expected.size() && expected[next] != sink.amplitude) + ++next; + EXPECT_LT(next, expected.size()) << sink.amplitude << " out of order"; + } + ASSERT_FALSE(probe.recorded_sinks.empty()); + EXPECT_FLOAT_EQ(probe.recorded_sinks.front().amplitude, 0.2f); + // Three 20 ms samples: the trailing silence is not played. + EXPECT_GE(elapsed, std::chrono::milliseconds(55)); + EXPECT_LT(elapsed, std::chrono::milliseconds(1000)); +} - const char* payload_right = R"({ - "rightHand": { - "millisecondsPerSample": 10.0, - "intensities": [0.4] - } - })"; +// Roblox pads a short bump to 100 x 25 ms; it must end after the bump, not +// after 2.5 s. +TEST_F(RobloxHapticProtocolBridgeTest, SetMotorStateTrimsTrailingSilence) { + ASSERT_TRUE(Initialize().ok()); - const auto reply_right = Legacy("SetMotorState", payload_right); - EXPECT_EQ(reply_right.code, 0); - ASSERT_EQ(probe.recorded_sinks.size(), 2u); - EXPECT_EQ(probe.recorded_sinks[1].hand, 1); - EXPECT_FLOAT_EQ(probe.recorded_sinks[1].amplitude, 0.4f); + Json motor = {{"millisecondsPerSample", 25.0}}; + std::vector intensities(100, 0.0); + intensities[0] = 0.6; + motor["intensities"] = intensities; + const std::string payload = Json{{"RightHand", motor}}.dump(); + + const auto started = std::chrono::steady_clock::now(); + EXPECT_EQ(Legacy("SetMotorState", payload.c_str()).code, 0); + ASSERT_TRUE(WaitFor(probe, [&] { return !probe.recorded_stops.empty(); })); + EXPECT_LT(std::chrono::steady_clock::now() - started, + std::chrono::milliseconds(500)); + + std::lock_guard lock(probe.mutex); + ASSERT_EQ(probe.recorded_sinks.size(), 1u); + EXPECT_FLOAT_EQ(probe.recorded_sinks[0].amplitude, 0.6f); + EXPECT_EQ(probe.recorded_stops, std::vector{1}); } -TEST_F(RobloxHapticProtocolBridgeTest, SetMotorStateZeroStopsHaptics) { +// A newer buffer replaces the one playing, and shutting down stops the hand. +TEST_F(RobloxHapticProtocolBridgeTest, SetMotorStateFollowsTheLatestBuffer) { ASSERT_TRUE(Initialize().ok()); - const char* payload = R"({ - "leftHand": { - "millisecondsPerSample": 20.0, - "intensities": [0.0] - } - })"; + const std::string longer = + Json{{"LeftHand", + {{"millisecondsPerSample", 25.0}, + {"intensities", std::vector(40, 0.3)}}}} + .dump(); + EXPECT_EQ(Legacy("SetMotorState", longer.c_str()).code, 0); + ASSERT_TRUE(WaitFor(probe, [&] { return !probe.recorded_sinks.empty(); })); + EXPECT_EQ(Legacy("SetMotorState", + R"({"LeftHand": {"millisecondsPerSample": 25.0, + "intensities": [0.9]}})") + .code, + 0); + ASSERT_TRUE(WaitFor(probe, [&] { + return !probe.recorded_sinks.empty() && + probe.recorded_sinks.back().amplitude == 0.9f; + })); + ASSERT_TRUE(WaitFor(probe, [&] { return !probe.recorded_stops.empty(); })); + { + std::lock_guard lock(probe.mutex); + EXPECT_EQ(probe.recorded_sinks.back().amplitude, 0.9f); + } - const auto reply = Legacy("SetMotorState", payload); - EXPECT_EQ(reply.code, 0); - ASSERT_EQ(probe.recorded_stops.size(), 1u); - EXPECT_EQ(probe.recorded_stops[0], 0); + EXPECT_EQ(Legacy("SetMotorState", longer.c_str()).code, 0); + ASSERT_TRUE(WaitFor(probe, [&] { + return probe.recorded_sinks.back().amplitude == 0.3f; + })); + std::size_t stops_before = 0; + { + std::lock_guard lock(probe.mutex); + stops_before = probe.recorded_stops.size(); + } + ASSERT_TRUE(bridge->Shutdown().ok()); + std::lock_guard lock(probe.mutex); + EXPECT_EQ(probe.recorded_stops.size(), stops_before + 1); +} + +// Positional effects aimed at a hand sit at x = +-1 with a 0.5 radius, so the +// hand motors must be reported there. +TEST_F(RobloxHapticProtocolBridgeTest, GetMotorsPlacesHandsAtUnitOffsets) { + ASSERT_TRUE(Initialize().ok()); + + const auto reply = Legacy("GetMotors"); + for (const char* left : {"leftHand", "LeftHand"}) + EXPECT_DOUBLE_EQ(reply.body[left]["x"].get(), -1.0) << left; + for (const char* right : {"rightHand", "RightHand"}) + EXPECT_DOUBLE_EQ(reply.body[right]["x"].get(), 1.0) << right; } TEST_F(RobloxHapticProtocolBridgeTest, UpdateSingletonVibrationBroadcasts) { @@ -348,9 +434,9 @@ TEST_F(RobloxHapticProtocolBridgeTest, RumbleMessagesAddNoHapticLogLines) { testing::internal::CaptureStderr(); Legacy("SetMotorState", R"({"leftHand": {"intensities": [0.5]}})"); Legacy("UpdateSingletonVibration", R"({"intensity": 0.5})"); + ASSERT_TRUE(WaitFor(probe, [&] { return probe.recorded_sinks.size() >= 3; })); EXPECT_EQ(testing::internal::GetCapturedStderr().find("[haptic]"), std::string::npos); - EXPECT_EQ(probe.recorded_sinks.size(), 3u); } TEST(RobloxHapticProtocolBridgeSession, RunsOnlyDuringAVrSession) { diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc index 2acfa83f..51a8eb17 100644 --- a/tests/vr/roblox_vr_device_bridge_test.cc +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -1022,18 +1022,26 @@ TEST(VrSupportedBuilds, Build3222CarriesTheOffsetFindersValues) { EXPECT_EQ(contract->camera_head_scale_offset, 0x130u); EXPECT_EQ(contract->keep_haptics_flag_rva, 0x763fef8u); EXPECT_EQ(contract->vr_haptics_flag_rva, 0x74cf1e0u); + EXPECT_EQ(contract->support_generic_haptics_flag_rva, 0x74cf458u); + EXPECT_EQ(contract->fix_register_motors_flag_rva, 0x74cfa10u); } -TEST(VrSupportedBuilds, EveryBuildCarriesBothHapticFlags) { - for (const auto& [id, keep, vr] : - {std::tuple{ - "ade08266c67aee88ec9c1d00902150e1684dad3a", 0x72fcd18, 0x7367588}, - {"5f0704edd9064f566ee3d6df2bd2fabbcc709f03", 0x7387118, 0x73f2268}, - {"a67a91ef96c700c0730acd36a250c2f93cfafbc1", 0x763fef8, 0x74cf1e0}}) { +TEST(VrSupportedBuilds, EveryBuildCarriesAllHapticFlags) { + for (const auto& [id, keep, vr, generic, motors] : + {std::tuple{ + "ade08266c67aee88ec9c1d00902150e1684dad3a", 0x72fcd18, 0x7367588, + 0x73677d0, 0x7367da0}, + {"5f0704edd9064f566ee3d6df2bd2fabbcc709f03", 0x7387118, 0x73f2268, + 0x73f24f0, 0x73f2aa8}, + {"a67a91ef96c700c0730acd36a250c2f93cfafbc1", 0x763fef8, 0x74cf1e0, + 0x74cf458, 0x74cfa10}}) { const auto* contract = internal::FindVrBuildContract(id); ASSERT_NE(contract, nullptr) << id; EXPECT_EQ(contract->keep_haptics_flag_rva, keep) << id; EXPECT_EQ(contract->vr_haptics_flag_rva, vr) << id; + EXPECT_EQ(contract->support_generic_haptics_flag_rva, generic) << id; + EXPECT_EQ(contract->fix_register_motors_flag_rva, motors) << id; } } @@ -1171,3 +1179,162 @@ TEST_F(VrNamedFastFlag, RejectsNamesAndFlagsOutsideTheImage) { } // namespace } // namespace mocktail::vr + +extern "C" void mocktail_vr_haptic_set_motor(void*, std::int32_t, std::int32_t, + const void*); + +namespace mocktail::vr { +namespace { + +// The Android backend's motor slot as 3222 compiles it: the Gamepad1 and +// Small/Large dispatch, a shared_ptr copy, the reader call, the Large store. +std::vector SharedTupleMotorSlot(std::int32_t reader_rel) { + std::vector code = { + 0x83, 0xfe, 0x0c, 0x0f, 0x85, 0x83, 0x00, 0x00, 0x00, // cmp; jne + 0x83, 0xfa, 0x01, 0x74, 0x2a, 0x85, 0xd2, 0x75, 0x7a, // cmp; je; test; jne + 0x48, 0x89, 0x45, 0xc8, 0x48, 0x89, 0x5d, 0xd0, 0x48, 0x85, 0xdb, + 0x74, 0x05, 0xf0, 0x48, 0xff, 0x43, 0x08, // copy the shared_ptr + 0x4c, 0x8d, 0x7d, 0xc8, 0x4c, 0x89, 0xff, 0xe8}; // lea r15; mov rdi; call + for (int shift = 0; shift < 32; shift += 8) + code.push_back(static_cast(reader_rel >> shift)); + for (const std::uint8_t byte : {0xf3, 0x41, 0x0f, 0x11, 0x46, 0x54}) + code.push_back(byte); + return code; +} + +TEST(VrSetMotorHook, FindsTheMotorIntensityReaderAfterTheGamepadDispatch) { + std::vector image(0x400, 0xcc); + const auto slot = SharedTupleMotorSlot(-0x100); + std::copy(slot.begin(), slot.end(), image.begin() + 0x200); + const auto reader = + internal::FindMotorIntensityReader(image.data(), image.size(), 0, image.size()); + EXPECT_EQ(reader.abi, internal::MotorIntensityAbi::kSharedTuple); + // The call ends at 0x200 + 44 + 4; its target is 0x100 bytes before that. + EXPECT_EQ(reader.rva, 0x200u + 48 - 0x100); +} + +TEST(VrSetMotorHook, RejectsAReaderWithoutTheLargeStoreOrSeenTwice) { + std::vector image(0x400, 0xcc); + auto slot = SharedTupleMotorSlot(-0x100); + slot.back() = 0x50; // the Small motor's field, not Large + std::copy(slot.begin(), slot.end(), image.begin() + 0x100); + EXPECT_EQ(internal::FindMotorIntensityReader(image.data(), image.size(), 0, + image.size()) + .abi, + internal::MotorIntensityAbi::kNone); + slot = SharedTupleMotorSlot(-0x100); + std::copy(slot.begin(), slot.end(), image.begin() + 0x100); + std::copy(slot.begin(), slot.end(), image.begin() + 0x200); + EXPECT_EQ(internal::FindMotorIntensityReader(image.data(), image.size(), 0, + image.size()) + .abi, + internal::MotorIntensityAbi::kNone); +} + +TEST(VrSetMotorHook, FindsOnlyTheDescriptorRegisteredWithTheExactName) { + std::vector image(0x400, 0xcc); + std::memcpy(image.data() + 0x300, "SetMotor", 9); + std::memcpy(image.data() + 0x320, "SetMotors", 10); + const auto registration = [&](std::size_t at, std::uintptr_t descriptor, + std::uintptr_t name) { + const std::uint8_t lea_rdi[] = {0x48, 0x8d, 0x3d}; + const std::uint8_t lea_rdx[] = {0x48, 0x8d, 0x15}; + const auto descriptor_rel = static_cast(descriptor - (at + 7)); + const auto name_rel = static_cast(name - (at + 14)); + std::memcpy(image.data() + at, lea_rdi, 3); + std::memcpy(image.data() + at + 3, &descriptor_rel, 4); + std::memcpy(image.data() + at + 7, lea_rdx, 3); + std::memcpy(image.data() + at + 10, &name_rel, 4); + }; + registration(0x40, 0x380, 0x320); // SetMotors: a different method + EXPECT_EQ(internal::FindSetMotorDescriptor(image.data(), image.size(), 0, 0x300), + 0u); + registration(0x80, 0x3a0, 0x300); + EXPECT_EQ(internal::FindSetMotorDescriptor(image.data(), image.size(), 0, 0x300), + 0x3a0u); + registration(0xc0, 0x3c0, 0x300); // ambiguous + EXPECT_EQ(internal::FindSetMotorDescriptor(image.data(), image.size(), 0, 0x300), + 0u); +} + +TEST(VrSetMotorHook, PicksTheMethodSlotWithACodeAddressAndNoAdjust) { + alignas(16) std::array image{}; + const auto base = reinterpret_cast(image.data()); + const auto put = [&](std::size_t offset, std::uintptr_t value) { + std::memcpy(image.data() + offset, &value, sizeof(value)); + }; + static bool (*const below_0x100)(std::uintptr_t, std::uintptr_t) = + [](std::uintptr_t, std::uintptr_t rva) { return rva < 0x100; }; + // 3222: +0x80 is empty, the method is at +0x88. + put(0x88, base + 0x40); + EXPECT_EQ(internal::SetMotorDescriptorSlot(base, 0, below_0x100), 0x88u); + // 2998: the method is at +0x80 and +0x88 holds its zero adjust. + image.fill(0); + put(0x80, base + 0x40); + EXPECT_EQ(internal::SetMotorDescriptorSlot(base, 0, below_0x100), 0x80u); + // A virtual (odd) pointer, an adjust, or data is not a method. + image.fill(0); + put(0x88, base + 0x41); + EXPECT_EQ(internal::SetMotorDescriptorSlot(base, 0, below_0x100), 0u); + put(0x88, base + 0x40); + put(0x90, 8); + EXPECT_EQ(internal::SetMotorDescriptorSlot(base, 0, below_0x100), 0u); + put(0x90, 0); + put(0x88, base + 0x180); + EXPECT_EQ(internal::SetMotorDescriptorSlot(base, 0, below_0x100), 0u); +} + +TEST(VrSetMotorHook, ForwardsNothingWithoutAnInstalledOriginal) { + // Not installed: the hook must neither crash nor read the arguments. + mocktail_vr_haptic_set_motor(nullptr, internal::kUserInputTypeGamepad1, + internal::kVibrationMotorLeftHand, nullptr); +} + +// Pins what the scans find in the real libraries (see +// ContractsMatchTheRealLibraries for MOCKTAIL_TEST_VR_PAYLOADS). +TEST(VrSetMotorHook, FindsTheReaderAndDescriptorInTheRealLibraries) { + const char* root = std::getenv("MOCKTAIL_TEST_VR_PAYLOADS"); + if (root == nullptr || root[0] == '\0') { + GTEST_SKIP() << "MOCKTAIL_TEST_VR_PAYLOADS is not set"; + } + struct Expected { + const char* code_name; + const char* id; + std::uintptr_t reader; + internal::MotorIntensityAbi abi; + std::uintptr_t descriptor; + }; + int checked = 0; + for (const Expected& expected : + {Expected{"2998", "ade08266c67aee88ec9c1d00902150e1684dad3a", 0x30b2d2e, + internal::MotorIntensityAbi::kTupleSpan, 0x7433fd0}, + Expected{"3222", "a67a91ef96c700c0730acd36a250c2f93cfafbc1", 0x61cd6f2, + internal::MotorIntensityAbi::kSharedTuple, 0x759ac70}}) { + const std::string path = std::string(root) + "/" + expected.code_name + + "-" + expected.id + "/libroblox.so"; + std::ifstream file(path, std::ios::binary); + if (!file) continue; + const std::vector bytes((std::istreambuf_iterator(file)), + std::istreambuf_iterator()); + const auto* image = reinterpret_cast(bytes.data()); + // The first load segment maps file offset 0 at address 0. + const auto* header = reinterpret_cast(image); + const auto* text = reinterpret_cast(image + header->e_phoff); + while (text->p_type != PT_LOAD) ++text; + ASSERT_EQ(text->p_vaddr, 0u); + ASSERT_NE(text->p_flags & PF_X, 0u); + const auto reader = internal::FindMotorIntensityReader( + image, bytes.size(), 0, text->p_filesz); + EXPECT_EQ(reader.rva, expected.reader) << expected.code_name; + EXPECT_EQ(reader.abi, expected.abi) << expected.code_name; + EXPECT_EQ(internal::FindSetMotorDescriptor(image, bytes.size(), 0, + text->p_filesz), + expected.descriptor) + << expected.code_name; + ++checked; + } + if (checked == 0) GTEST_SKIP() << "no payload found under " << root; +} + +} // namespace +} // namespace mocktail::vr From e553db20012d697044024c849bb11f6a62173af1 Mon Sep 17 00:00:00 2001 From: vii Date: Thu, 15 Oct 2026 02:20:13 -0500 Subject: [PATCH 47/58] fix(vr): point 3092 FixRegisterMotors at the flag Roblox reads The 3092 table had the address of another field in the same registration, so the flag never turned on. --- src/vr/roblox_vr_device_bridge.cc | 2 +- tests/vr/roblox_vr_device_bridge_test.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/vr/roblox_vr_device_bridge.cc b/src/vr/roblox_vr_device_bridge.cc index fcaec917..3daa34d3 100644 --- a/src/vr/roblox_vr_device_bridge.cc +++ b/src/vr/roblox_vr_device_bridge.cc @@ -579,7 +579,7 @@ constexpr VrBuildContract kContracts[] = { 0x7387118, 0x73f2268, 0x73f24f0, - 0x73f2aa8, + 0x73f2ac0, "3092 (2.738.1397)"}, {"a67a91ef96c700c0730acd36a250c2f93cfafbc1", {0x7083a48, 0x549c884, 0x549cc66, 0x549cc56, 0x549cc88, 0x74d1640, diff --git a/tests/vr/roblox_vr_device_bridge_test.cc b/tests/vr/roblox_vr_device_bridge_test.cc index 51a8eb17..1e40990c 100644 --- a/tests/vr/roblox_vr_device_bridge_test.cc +++ b/tests/vr/roblox_vr_device_bridge_test.cc @@ -1033,7 +1033,7 @@ TEST(VrSupportedBuilds, EveryBuildCarriesAllHapticFlags) { "ade08266c67aee88ec9c1d00902150e1684dad3a", 0x72fcd18, 0x7367588, 0x73677d0, 0x7367da0}, {"5f0704edd9064f566ee3d6df2bd2fabbcc709f03", 0x7387118, 0x73f2268, - 0x73f24f0, 0x73f2aa8}, + 0x73f24f0, 0x73f2ac0}, {"a67a91ef96c700c0730acd36a250c2f93cfafbc1", 0x763fef8, 0x74cf1e0, 0x74cf458, 0x74cfa10}}) { const auto* contract = internal::FindVrBuildContract(id); From de0ddf7864a33fc5025b5bf51695fb7c251188a0 Mon Sep 17 00:00:00 2001 From: vii Date: Thu, 15 Oct 2026 02:20:13 -0500 Subject: [PATCH 48/58] feat(update): solve the VR device bridge and compare it with the VR offset finder --- docs/VR.md | 14 + scripts/roblox_abi_check.py | 70 ++++- scripts/roblox_abi_solver.py | 7 +- scripts/roblox_abi_vr.py | 262 +++++++++++++++++++ tests/fixtures/host_abi_regression/2998.json | 23 +- tests/fixtures/host_abi_regression/3092.json | 23 +- tests/fixtures/host_abi_regression/3222.json | 23 +- tests/roblox_abi_solver_test.py | 120 ++++++++- 8 files changed, 530 insertions(+), 12 deletions(-) create mode 100644 scripts/roblox_abi_vr.py diff --git a/docs/VR.md b/docs/VR.md index 93d036ca..89ded1a8 100644 --- a/docs/VR.md +++ b/docs/VR.md @@ -81,6 +81,20 @@ match. To run the same checks on real libraries, set `MOCKTAIL_TEST_VR_PAYLOADS` to a folder of `-/libroblox.so` payloads and run `ctest -R VrSupportedBuilds`. +For a second opinion, solve the new build from the previous one and compare +the two answers. It needs the previous build's HostAbi profile and +compatibility manifest: + +```bash +python3 scripts/roblox_abi_check.py --reference-lib \ + --reference-profile \ + --reference-compatibility config/roblox_compatibility.json \ + --candidate-lib --vr +``` + +It lists every field where the finder and the solver disagree and exits 0 +when they agree on all of them. + ## Testing without a headset [Monado](https://monado.freedesktop.org) can stand in for a headset. Its diff --git a/scripts/roblox_abi_check.py b/scripts/roblox_abi_check.py index bcb43ad5..babe5042 100644 --- a/scripts/roblox_abi_check.py +++ b/scripts/roblox_abi_check.py @@ -23,6 +23,10 @@ Prints every field where the two disagree. Exit status: 0 agree, 1 differ or one side failed, 2 both failed. + +With --vr, the VR device bridge is compared too. `find_roblox_vr_offsets.py` +finds it in each build on its own: its answer for the reference build is the +solver's input, and its answer for the candidate joins the deriver's side. """ from __future__ import annotations @@ -34,25 +38,73 @@ from typing import Any, Sequence -def _load_solver(): - name = "mocktail_roblox_abi_solver" +def _load_sibling(name: str, file_name: str): if name in sys.modules: return sys.modules[name] - path = Path(__file__).resolve().with_name("roblox_abi_solver.py") + path = Path(__file__).resolve().with_name(file_name) specification = importlib.util.spec_from_file_location(name, path) if specification is None or specification.loader is None: - raise RuntimeError("cannot load roblox_abi_solver.py") + raise RuntimeError(f"cannot load {file_name}") module = importlib.util.module_from_spec(specification) sys.modules[name] = module specification.loader.exec_module(module) return module -SOLVER = _load_solver() +SOLVER = _load_sibling("mocktail_roblox_abi_solver", "roblox_abi_solver.py") +FINDER = _load_sibling("mocktail_find_roblox_vr_offsets", "find_roblox_vr_offsets.py") DERIVER = SOLVER.DERIVER AnalyzerError = SOLVER.AnalyzerError format_rva = SOLVER.format_rva +# Where find_roblox_vr_offsets.py reports each VR bridge field. +_FINDER_FIELDS = { + "vtable_rva": ("vr_debug_device_bridge", "vtable_rva"), + "constructor_rva": ("vr_debug_device_bridge", "constructor_rva"), + "state_getter_rva": ("vr_debug_device_bridge", "state_getter_rva"), + "eye_getter_rva": ("vr_debug_device_bridge", "eye_getter_rva"), + "eye_initializer_rva": ("vr_debug_device_bridge", "eye_initializer_rva"), + "emulator_flag_storage_rva": ("vr_debug_device_bridge", "emulator_flag_storage_rva"), + "device_create_framebuffer_vtable_offset": ( + "vr_debug_device_bridge", "device_create_framebuffer_vtable_offset" + ), + "state_memcpy_rva": ("vr_bridge_supplement", "state_getter_memcpy_rva"), + "pointer_frame_rva": ("vr_bridge_supplement", "world_pointer_getter_rva"), + "workspace_getter_rva": ("vr_bridge_supplement", "workspace_getter_rva"), + "camera_vtable_offset": ("vr_bridge_supplement", "workspace_camera_vtable_offset"), + "hand_pitch_storage_rva": ("vr_bridge_supplement", "new_vr_system_hand_pitch_storage_rva"), + "haptic_sink_rva": ("vr_bridge_supplement", "haptic_sink_rva"), + "pointer_hand_selector_offset": ("vr_bridge_supplement", "usercframe_selector_offset"), + "camera_head_scale_offset": ("vr_bridge_supplement", "camera_head_scale_offset"), + "keep_haptics_flag_rva": ("vr_bridge_supplement", "keep_haptics_flag_rva"), + "vr_haptics_flag_rva": ("vr_bridge_supplement", "vr_haptics_flag_rva"), + "support_generic_haptics_flag_rva": ( + "vr_bridge_supplement", "support_generic_haptics_flag_rva" + ), + "fix_register_motors_flag_rva": ("vr_bridge_supplement", "fix_register_motors_flag_rva"), +} + + +def vr_bridge_from_finder( + result: dict[str, Any], fields: Sequence[str] = SOLVER.VR.VR_FIELDS +) -> dict[str, str]: + """The VR bridge fields from a find_roblox_vr_offsets.py result.""" + bridge = {} + for field in fields: + section, key = _FINDER_FIELDS[field] + try: + bridge[field] = result[section][key] + except (KeyError, TypeError) as error: + raise AnalyzerError(f"VR offset finder result lacks {key}") from error + return bridge + + +def _find_vr_bridge(library: Path) -> dict[str, str]: + try: + return vr_bridge_from_finder(FINDER.find_vr_offsets(library)) + except FINDER.AnalyzerError as error: + raise AnalyzerError(f"VR offset finder: {error}") from error + def compare_results(derived: Any, solved: Any, path: str = "") -> list[str]: """Returns one line for every leaf value where the two results differ.""" @@ -121,6 +173,7 @@ def check(arguments: argparse.Namespace) -> int: manifests = [Path(path) for path in arguments.reference_compatibility] cache_dir = Path(arguments.cache_dir) if arguments.cache_dir else None derived = solved = deriver_error = solver_error = None + reference_vr = _find_vr_bridge(Path(arguments.reference_lib)) if arguments.vr else None with DERIVER.ElfImage(Path(arguments.reference_lib)) as reference, DERIVER.ElfImage( Path(arguments.candidate_lib) ) as candidate: @@ -128,6 +181,8 @@ def check(arguments: argparse.Namespace) -> int: profile, anchors = DERIVER.derive_profile(reference, candidate, sidecar) runtime = DERIVER.derive_runtime_compatibility(reference, candidate, manifests) derived = {"profile": profile, "derivation_anchors": anchors, "runtime": runtime} + if reference_vr is not None: + derived["vr_device_bridge"] = _find_vr_bridge(Path(arguments.candidate_lib)) except Exception as error: deriver_error = _describe(error) try: @@ -138,6 +193,8 @@ def check(arguments: argparse.Namespace) -> int: source = DERIVER.load_reference_runtime_profile(manifests, reference.build_id) runtime = solver.derive_runtime_compatibility(_solver_runtime_source(source)) solved = {"profile": profile, "derivation_anchors": anchors, "runtime": runtime} + if reference_vr is not None: + solved["vr_device_bridge"] = solver.derive_vr(reference_vr) except Exception as error: solver_error = _describe(error) status, lines = check_status(derived, solved, deriver_error, solver_error) @@ -161,6 +218,9 @@ def main(argv: Sequence[str] | None = None) -> int: default=str(Path.home() / ".cache" / "mocktail" / "function_index"), help="where function indexes are kept; empty to disable", ) + parser.add_argument( + "--vr", action="store_true", help="also compare the VR device bridge" + ) arguments = parser.parse_args(sys.argv[1:] if argv is None else argv) try: return check(arguments) diff --git a/scripts/roblox_abi_solver.py b/scripts/roblox_abi_solver.py index a29f4fd4..108a4238 100644 --- a/scripts/roblox_abi_solver.py +++ b/scripts/roblox_abi_solver.py @@ -27,8 +27,8 @@ A match is accepted only with a score of at least ACCEPT_SCORE and a lead of at least ACCEPT_LEAD over the runner-up. `roblox_abi_targets.py` turns the matches into a profile with the same shape as the one -`derive_roblox_host_abi_profile.py` writes, and `roblox_abi_check.py` -compares the two on one update. +`derive_roblox_host_abi_profile.py` writes, `roblox_abi_vr.py` into the VR +device bridge, and `roblox_abi_check.py` compares the two on one update. """ from __future__ import annotations @@ -58,6 +58,7 @@ def _load_sibling(name: str, file_name: str): INDEX = _load_sibling("mocktail_roblox_function_index", "roblox_function_index.py") TARGETS = _load_sibling("mocktail_roblox_abi_targets", "roblox_abi_targets.py") +VR = _load_sibling("mocktail_roblox_abi_vr", "roblox_abi_vr.py") DERIVER = INDEX.DERIVER FunctionIndex = INDEX.FunctionIndex Instruction = INDEX.Instruction @@ -136,7 +137,7 @@ def _weighted(parts: Sequence[tuple[float | None, float]]) -> float: ) -class Solver(TARGETS.TargetDerivation): +class Solver(TARGETS.TargetDerivation, VR.VrTargets): def __init__(self, reference: Any, candidate: Any, cache_dir: Path | None = None): self._start(FunctionIndex(reference, cache_dir), FunctionIndex(candidate, cache_dir)) diff --git a/scripts/roblox_abi_vr.py b/scripts/roblox_abi_vr.py new file mode 100644 index 00000000..517c619f --- /dev/null +++ b/scripts/roblox_abi_vr.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +# Copyright 2026 Mocktail Project Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""VR device bridge targets that `roblox_abi_solver.py` derives. + +`VrTargets.derive_vr` takes one build's VR bridge (the `VrBuildContract` +fields in `src/vr/roblox_vr_device_bridge.cc`, as hex strings) and returns +the same fields for the candidate build. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import re +import sys +from typing import Any + + +def _load_index(): + name = "mocktail_roblox_function_index" + if name in sys.modules: + return sys.modules[name] + path = Path(__file__).resolve().with_name("roblox_function_index.py") + specification = importlib.util.spec_from_file_location(name, path) + if specification is None or specification.loader is None: + raise RuntimeError("cannot load roblox_function_index.py") + module = importlib.util.module_from_spec(specification) + sys.modules[name] = module + specification.loader.exec_module(module) + return module + + +DERIVER = _load_index().DERIVER +AnalyzerError = DERIVER.AnalyzerError +format_rva = DERIVER.format_rva + +VR_FIELDS = ( + "vtable_rva", + "constructor_rva", + "state_getter_rva", + "eye_getter_rva", + "eye_initializer_rva", + "emulator_flag_storage_rva", + "device_create_framebuffer_vtable_offset", + "state_memcpy_rva", + "pointer_frame_rva", + "workspace_getter_rva", + "camera_vtable_offset", + "hand_pitch_storage_rva", + "haptic_sink_rva", + "pointer_hand_selector_offset", + "camera_head_scale_offset", + "keep_haptics_flag_rva", + "vr_haptics_flag_rva", + "support_generic_haptics_flag_rva", + "fix_register_motors_flag_rva", +) +# Fast flag and setting storage, found through the functions that use it. +_STORAGE_FIELDS = ( + "emulator_flag_storage_rva", + "hand_pitch_storage_rva", + "keep_haptics_flag_rva", + "vr_haptics_flag_rva", + "support_generic_haptics_flag_rva", + "fix_register_motors_flag_rva", +) +# Storage users tried, smallest first; registration functions are small. +_STORAGE_USER_LIMIT = 8 +_VTABLE_SLOT_LIMIT = 0x400 +# The constructor stores the vtable address plus the Itanium header. +_VTABLE_REFERENCE_SPAN = 0x40 +_NUMBER = r"(0x[0-9a-f]+|\d+)" +_SLOT_LOAD = re.compile(r"^mov (\w+), qword ptr \[\1 \+ ") +# How far back the load of the vtable pointer itself may sit. +_VTABLE_LOAD_WINDOW = 6 + + +def _vtable_call(instructions: list[Any], index: int) -> bool: + """A call through a vtable slot, or a load of the slot from a vtable + pointer loaded just before into the same register.""" + shape = instructions[index].shape + if shape.startswith("call qword ptr ["): + return True + load = _SLOT_LOAD.match(shape) + if load is None: + return False + vtable_load = re.compile(rf"^mov {load.group(1)}, qword ptr \[\w+\]$") + return any( + vtable_load.match(previous.shape) + for previous in instructions[max(0, index - _VTABLE_LOAD_WINDOW) : index] + ) + + +class VrTargets: + """Mixin for `Solver`: needs `reference`, `candidate`, `matches`, + `match_function`, `map_data` and `_aligned`.""" + + def map_operand( + self, name: str, function_name: str, start: int, value: int, vtable_call: bool = False + ) -> int: + """Maps a field offset that a matched function uses as a displacement. + + Stack-relative uses are ignored; with `vtable_call`, so is every use + that is not a virtual call. An instruction whose shape changed is + looked for between its nearest aligned neighbours. + """ + reference, candidate, mapping = self._aligned(function_name, start) + token = re.compile(rf"(? index), default=None) + low = 0 if below is None else mapping[below] + 1 + high = len(candidate) if above is None else mapping[above] + window = range(low, high) + values = [] + for position in window: + if vtable_call and not _vtable_call(candidate, position): + continue + matched = pattern.match(candidate[position].shape) + if matched is not None and len({*matched.groups()}) == 1: + values.append(int(matched.group(1), 0)) + if len(values) != 1: + raise AnalyzerError(f"{name} has {len(values)} candidate offsets") + found.add(values[0]) + if len(found) != 1: + raise AnalyzerError(f"{name} maps to {len(found)} offsets") + return found.pop() + + def map_call_target(self, name: str, function_name: str, start: int, target: int) -> int: + """Maps the target of a direct call inside a matched function.""" + reference, candidate, mapping = self._aligned(function_name, start) + found = set() + for index, item in enumerate(reference): + if item.shape != "call" or item.branch_target != target: + continue + if index not in mapping: + raise AnalyzerError(f"{name} call has no counterpart") + found.add(candidate[mapping[index]].branch_target) + if len(found) != 1 or None in found: + raise AnalyzerError(f"{name} has no unique call target") + return found.pop() + + def _map_storage(self, name: str, target: int) -> int: + users = sorted( + self.reference.data_users.get(target, ()), + key=lambda start: self.reference.functions[start].size, + )[:_STORAGE_USER_LIMIT] + found = set() + for user in users: + user_name = f"{name} user {format_rva(user)}" + try: + self.match_function(user_name, user) + found.add(self.map_data(name, user_name, user, target)) + except AnalyzerError: + continue + if len(found) != 1: + raise AnalyzerError(f"{name} maps to {len(found)} addresses") + return found.pop() + + def _map_vtable(self, constructor: int, vtable: int) -> int: + found = set() + for item in self.reference.instructions(constructor): + for target in item.rip_targets: + if 0 <= target - vtable < _VTABLE_REFERENCE_SPAN: + mapped = self.map_data("VR vtable", "vr-constructor", constructor, target) + found.add(mapped - (target - vtable)) + if len(found) != 1: + raise AnalyzerError(f"VR vtable maps to {len(found)} addresses") + return found.pop() + + @staticmethod + def _slot(relocations: dict[int, int], vtable: int, method: int) -> int: + slots = [ + offset + for offset in range(0, _VTABLE_SLOT_LIMIT, 8) + if relocations.get(vtable + offset) == method + ] + if not slots: + raise AnalyzerError(f"reference VR vtable does not hold {format_rva(method)}") + return slots[0] + + def derive_vr(self, source: dict[str, str]) -> dict[str, str]: + missing = [field for field in VR_FIELDS if field not in source] + if missing: + raise AnalyzerError("reference VR bridge lacks " + ", ".join(missing)) + value = {field: DERIVER.parse_rva(source[field], field) for field in VR_FIELDS} + result = {} + constructor = self.match_function("vr-constructor", value["constructor_rva"]) + result["constructor_rva"] = constructor + vtable = self._map_vtable(value["constructor_rva"], value["vtable_rva"]) + result["vtable_rva"] = vtable + reference_relocations = DERIVER.relative_relocation_map(self.reference.image) + relocations = DERIVER.relative_relocation_map(self.candidate.image) + slots = { + field: self._slot(reference_relocations, value["vtable_rva"], value[field]) + for field in ( + "state_getter_rva", "eye_getter_rva", "eye_initializer_rva", "haptic_sink_rva" + ) + } + self.candidate.image.require_relro_rva(vtable, max(slots.values()) + 8) + for field in ("state_getter_rva", "eye_getter_rva", "eye_initializer_rva"): + result[field] = self.match_function( + f"vr-{field}", value[field], vtable_slot=relocations.get(vtable + slots[field]) + ) + # The haptic sink is a one-byte ret stub, so only its slot identifies it. + sink = relocations.get(vtable + slots["haptic_sink_rva"]) + if sink is None or sink not in self.candidate.functions or ( + self.candidate.functions[sink].size != 1 + ): + raise AnalyzerError("VR haptic sink slot does not hold a ret stub") + result["haptic_sink_rva"] = sink + result["state_memcpy_rva"] = self.map_call_target( + "VR state memcpy", "vr-state_getter_rva", + value["state_getter_rva"], value["state_memcpy_rva"], + ) + result["device_create_framebuffer_vtable_offset"] = self.map_operand( + "VR framebuffer slot", "vr-eye_initializer_rva", + value["eye_initializer_rva"], value["device_create_framebuffer_vtable_offset"], + vtable_call=True, + ) + result["pointer_frame_rva"] = self.match_function( + "vr-pointer-frame", value["pointer_frame_rva"] + ) + result["workspace_getter_rva"] = self.map_call_target( + "VR workspace getter", "vr-pointer-frame", + value["pointer_frame_rva"], value["workspace_getter_rva"], + ) + for field in ( + "camera_vtable_offset", "pointer_hand_selector_offset", "camera_head_scale_offset" + ): + result[field] = self.map_operand( + field, "vr-pointer-frame", value["pointer_frame_rva"], value[field], + vtable_call=field == "camera_vtable_offset", + ) + for field in _STORAGE_FIELDS: + result[field] = self._map_storage(field, value[field]) + self.candidate.image.require_writable_rva(result[field], 1) + return {field: format_rva(result[field]) for field in VR_FIELDS} diff --git a/tests/fixtures/host_abi_regression/2998.json b/tests/fixtures/host_abi_regression/2998.json index ab833cff..2e954699 100644 --- a/tests/fixtures/host_abi_regression/2998.json +++ b/tests/fixtures/host_abi_regression/2998.json @@ -109,5 +109,26 @@ }, "schema_version": 1, "version_code": 2998, - "version_name": "2.736.1408" + "version_name": "2.736.1408", + "vr_device_bridge": { + "camera_head_scale_offset": "0x140", + "camera_vtable_offset": "0x3a8", + "constructor_rva": "0x3810614", + "device_create_framebuffer_vtable_offset": "0x180", + "emulator_flag_storage_rva": "0x7369920", + "eye_getter_rva": "0x38109e6", + "eye_initializer_rva": "0x3810a18", + "fix_register_motors_flag_rva": "0x7367da0", + "hand_pitch_storage_rva": "0x73677a8", + "haptic_sink_rva": "0x1d48b70", + "keep_haptics_flag_rva": "0x72fcd18", + "pointer_frame_rva": "0x4d573ee", + "pointer_hand_selector_offset": "0x118", + "state_getter_rva": "0x38109f6", + "state_memcpy_rva": "0x6b7bfe0", + "support_generic_haptics_flag_rva": "0x73677d0", + "vr_haptics_flag_rva": "0x7367588", + "vtable_rva": "0x6c87b38", + "workspace_getter_rva": "0x263778c" + } } diff --git a/tests/fixtures/host_abi_regression/3092.json b/tests/fixtures/host_abi_regression/3092.json index 441c25e3..bb238b30 100644 --- a/tests/fixtures/host_abi_regression/3092.json +++ b/tests/fixtures/host_abi_regression/3092.json @@ -110,5 +110,26 @@ }, "schema_version": 1, "version_code": 3092, - "version_name": "2.738.1397" + "version_name": "2.738.1397", + "vr_device_bridge": { + "camera_head_scale_offset": "0x140", + "camera_vtable_offset": "0x3a0", + "constructor_rva": "0x37f9fe4", + "device_create_framebuffer_vtable_offset": "0x188", + "emulator_flag_storage_rva": "0x73f4690", + "eye_getter_rva": "0x37fa3b6", + "eye_initializer_rva": "0x37fa3e8", + "fix_register_motors_flag_rva": "0x73f2ac0", + "hand_pitch_storage_rva": "0x73f24c8", + "haptic_sink_rva": "0x1d8f8c0", + "keep_haptics_flag_rva": "0x7387118", + "pointer_frame_rva": "0x4d97b98", + "pointer_hand_selector_offset": "0x118", + "state_getter_rva": "0x37fa3c6", + "state_memcpy_rva": "0x6bfb180", + "support_generic_haptics_flag_rva": "0x73f24f0", + "vr_haptics_flag_rva": "0x73f2268", + "vtable_rva": "0x6d08108", + "workspace_getter_rva": "0x265cebc" + } } diff --git a/tests/fixtures/host_abi_regression/3222.json b/tests/fixtures/host_abi_regression/3222.json index 8603d50c..2ad87607 100644 --- a/tests/fixtures/host_abi_regression/3222.json +++ b/tests/fixtures/host_abi_regression/3222.json @@ -110,5 +110,26 @@ }, "schema_version": 1, "version_code": 3222, - "version_name": "2.742.1059" + "version_name": "2.742.1059", + "vr_device_bridge": { + "camera_head_scale_offset": "0x130", + "camera_vtable_offset": "0x3a0", + "constructor_rva": "0x549c884", + "device_create_framebuffer_vtable_offset": "0x190", + "emulator_flag_storage_rva": "0x74d1640", + "eye_getter_rva": "0x549cc56", + "eye_initializer_rva": "0x549cc88", + "fix_register_motors_flag_rva": "0x74cfa10", + "hand_pitch_storage_rva": "0x74cf430", + "haptic_sink_rva": "0x1e2fcf0", + "keep_haptics_flag_rva": "0x763fef8", + "pointer_frame_rva": "0x4fd8cb0", + "pointer_hand_selector_offset": "0x108", + "state_getter_rva": "0x549cc66", + "state_memcpy_rva": "0x6daad30", + "support_generic_haptics_flag_rva": "0x74cf458", + "vr_haptics_flag_rva": "0x74cf1e0", + "vtable_rva": "0x7083a48", + "workspace_getter_rva": "0x264bb76" + } } diff --git a/tests/roblox_abi_solver_test.py b/tests/roblox_abi_solver_test.py index 85588be0..35e10709 100644 --- a/tests/roblox_abi_solver_test.py +++ b/tests/roblox_abi_solver_test.py @@ -139,6 +139,74 @@ def plain(address: int): return SOLVER.Instruction(address, "nop ", (), None) +def shaped(address: int, shape: str, target: int | None = None): + return SOLVER.Instruction(address, shape, (), target) + + +class VrOperandTest(unittest.TestCase): + def solver(self, reference_code, candidate_code): + solver = SOLVER.Solver.from_indexes( + FakeIndex({0x100: function()}, instructions={0x100: reference_code}), + FakeIndex({0x1100: function()}, instructions={0x1100: candidate_code}), + ) + solver.matches["pointer"] = SOLVER.Match(0x1100, 1.0, 1.0, "score") + return solver + + def test_reads_a_changed_field_offset_between_aligned_neighbours(self) -> None: + solver = self.solver( + [shaped(0x100, "push rbp"), shaped(0x101, "mov eax, dword ptr [rdi + 0x118]"), + shaped(0x102, "ret ")], + [shaped(0x1100, "push rbp"), shaped(0x1101, "nop "), + shaped(0x1102, "mov eax, dword ptr [rdi + 0x108]"), shaped(0x1103, "ret ")], + ) + self.assertEqual(solver.map_operand("selector", "pointer", 0x100, 0x118), 0x108) + + def test_ignores_stack_slots_with_the_same_value(self) -> None: + solver = self.solver( + [shaped(0x100, "mov qword ptr [rsp + 0x140], rax"), + shaped(0x101, "movss xmm0, dword ptr [rax + 0x140]"), shaped(0x102, "ret ")], + [shaped(0x1100, "mov qword ptr [rsp + 0x140], rax"), + shaped(0x1101, "movss xmm0, dword ptr [rax + 0x130]"), shaped(0x1102, "ret ")], + ) + self.assertEqual(solver.map_operand("head scale", "pointer", 0x100, 0x140), 0x130) + + def test_a_vtable_offset_skips_data_fields_with_the_same_value(self) -> None: + solver = self.solver( + [shaped(0x100, "mov r12d, dword ptr [rax + 0x190]"), + shaped(0x101, "mov rax, qword ptr [rsi]"), + shaped(0x102, "mov rax, qword ptr [rax + 0x190]"), shaped(0x103, "mov rdi, r14"), + shaped(0x104, "call qword ptr [rcx + 0x3a8]"), shaped(0x105, "ret ")], + [shaped(0x1100, "mov r12d, dword ptr [rax + 0x190]"), + shaped(0x1101, "mov rax, qword ptr [rsi]"), + shaped(0x1102, "mov rax, qword ptr [rax + 0x188]"), shaped(0x1103, "mov rdi, r14"), + shaped(0x1104, "call qword ptr [rcx + 0x3a0]"), shaped(0x1105, "ret ")], + ) + self.assertEqual( + solver.map_operand("framebuffer", "pointer", 0x100, 0x190, vtable_call=True), 0x188 + ) + self.assertEqual( + solver.map_operand("camera", "pointer", 0x100, 0x3a8, vtable_call=True), 0x3a0 + ) + + def test_rejects_two_candidates_for_one_offset(self) -> None: + solver = self.solver( + [shaped(0x100, "push rbp"), shaped(0x101, "mov eax, dword ptr [rdi + 0x118]"), + shaped(0x102, "ret ")], + [shaped(0x1100, "push rbp"), shaped(0x1101, "mov eax, dword ptr [rdi + 0x108]"), + shaped(0x1102, "mov eax, dword ptr [rdi + 0x110]"), shaped(0x1103, "ret ")], + ) + with self.assertRaisesRegex(SOLVER.AnalyzerError, "selector"): + solver.map_operand("selector", "pointer", 0x100, 0x118) + + def test_maps_a_call_target(self) -> None: + solver = self.solver( + [shaped(0x100, "push rbp"), shaped(0x101, "call", 0x500), shaped(0x102, "ret ")], + [shaped(0x1100, "push rbp"), shaped(0x1101, "call", 0x1500), + shaped(0x1102, "ret ")], + ) + self.assertEqual(solver.map_call_target("memcpy", "pointer", 0x100, 0x500), 0x1500) + + class FunctionLinkTest(unittest.TestCase): def test_lists_each_caller_and_data_user_once(self) -> None: callers, data_users = SOLVER.INDEX.link_functions({ @@ -432,7 +500,9 @@ def test_exit_status_follows_agreement(self) -> None: @unittest.skipUnless(DERIVER.capstone is not None, "Python capstone is unavailable") class CheckCommandTest(unittest.TestCase): - def run_check(self, reference: dict, reference_library, candidate_library): + def run_check( + self, reference: dict, reference_library, candidate_library, extra=() + ): with tempfile.TemporaryDirectory() as temporary: sidecar = Path(temporary) / "reference.json" sidecar.write_text(json.dumps(regression_sidecar(reference)), encoding="utf-8") @@ -446,6 +516,7 @@ def run_check(self, reference: dict, reference_library, candidate_library): "--reference-compatibility", str(manifest), "--candidate-lib", str(candidate_library), "--cache-dir", str(REGRESSION_CACHE / "function_index"), + *extra, ]) return status, output.getvalue() @@ -457,6 +528,16 @@ def test_reports_agreement_on_a_real_update(self) -> None: self.assertEqual(status, 0, output) self.assertIn("deriver and solver agree", output) + def test_compares_the_vr_bridge_with_the_offset_finder(self) -> None: + reference, _, reference_library, candidate_library = regression_images( + self, "2998", "3092" + ) + status, output = self.run_check( + reference, reference_library, candidate_library, ("--vr",) + ) + self.assertEqual(status, 0, output) + self.assertIn("deriver and solver agree", output) + def test_rejects_a_wrong_reference_before_indexing(self) -> None: reference, _, _, candidate_library = regression_images(self, "2998", "3092") third = regression_library(load_regression_build("3222")) @@ -494,6 +575,43 @@ def test_reports_any_error_as_that_side_failing(self) -> None: } +# Builds with VR answers; each pair solves the second build's VR bridge. +VR_PAIRS = (("2998", "3092"), ("3092", "3222"), ("3222", "3092")) + + +@unittest.skipUnless(DERIVER.capstone is not None, "Python capstone is unavailable") +class VrRegressionTest(unittest.TestCase): + def test_solves_each_vr_bridge_from_its_predecessor(self) -> None: + for reference_name, candidate_name in VR_PAIRS: + with self.subTest(reference=reference_name, candidate=candidate_name): + reference, candidate, reference_library, candidate_library = ( + regression_images(self, reference_name, candidate_name) + ) + with DERIVER.ElfImage(reference_library) as reference_image, DERIVER.ElfImage( + candidate_library + ) as candidate_image: + solver = SOLVER.Solver( + reference_image, candidate_image, REGRESSION_CACHE / "function_index" + ) + solved = solver.derive_vr(reference["vr_device_bridge"]) + self.assertEqual(solved, candidate["vr_device_bridge"]) + + def test_offset_finder_output_converts_to_the_bridge_fields(self) -> None: + finder = { + "vr_debug_device_bridge": {"vtable_rva": "0x10"}, + "vr_bridge_supplement": { + "world_pointer_getter_rva": "0x20", "usercframe_selector_offset": "0x118", + }, + } + converted = CHECK.vr_bridge_from_finder(finder, ( + "vtable_rva", "pointer_frame_rva", "pointer_hand_selector_offset" + )) + self.assertEqual(converted, { + "vtable_rva": "0x10", "pointer_frame_rva": "0x20", + "pointer_hand_selector_offset": "0x118", + }) + + @unittest.skipUnless(DERIVER.capstone is not None, "Python capstone is unavailable") class SolverRegressionTest(unittest.TestCase): def test_solves_each_build_from_its_predecessor(self) -> None: From e094a4c89debb82435d4932d58bd3f16e04ca635 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Thu, 15 Oct 2026 01:37:12 -0500 Subject: [PATCH 49/58] feat(vr): time the desktop mirror, present prepare and frame interval MOCKTAIL_VR_PERF now reports how long the CPU waits in the desktop mirror and in present preparation, and the time between presents, so per-frame stalls outside the XR frame cycle show up in the numbers. --- include/mocktail/vr/vr_perf.h | 3 +++ src/vr/openxr_backend.cc | 10 ++++++++++ src/vr/vr_perf.cc | 3 +++ 3 files changed, 16 insertions(+) diff --git a/include/mocktail/vr/vr_perf.h b/include/mocktail/vr/vr_perf.h index fd6cd430..38f72f3a 100644 --- a/include/mocktail/vr/vr_perf.h +++ b/include/mocktail/vr/vr_perf.h @@ -25,6 +25,9 @@ enum class Stage { kMirrorPublish, kEvidence, kControllerSync, + kDesktopMirror, + kPresentPrepare, + kPresentInterval, kStageCount, }; diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 38c9c6b5..dcd04b09 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -2252,6 +2252,7 @@ VkQueue OpenXrBackend::GraphicsQueueForPresentLocked(VkQueue present) const { VkResult OpenXrBackend::PrepareHostPresent(VkQueue queue, VkDevice device, const VkPresentInfoKHR *info) { + const ScopedStageTimer prepare_timer(perf::Stage::kPresentPrepare); std::lock_guard lock(mutex_); if (!armed_ || graphics_api_ != VrGraphicsApi::kVulkan || !vk_ || device != vk_device_ || !info) @@ -2283,6 +2284,14 @@ VkResult OpenXrBackend::PrepareHostPresent(VkQueue queue, VkDevice device, } void OpenXrBackend::NoteHostPresent(VkQueue queue, VkDevice device) { + if (perf::Enabled()) { + static thread_local std::uint64_t previous_ns = 0; + const std::uint64_t now_ns = perf::NowNs(); + if (previous_ns != 0 && now_ns > previous_ns) + perf::ProcessCollector().Record(perf::Stage::kPresentInterval, + now_ns - previous_ns); + previous_ns = now_ns; + } bool idle = false; { // frame_mutex_ keeps the session alive for the whole frame cycle, so @@ -3435,6 +3444,7 @@ void OpenXrBackend::RecordDesktopSwapchain(VkDevice device, VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, const VkPresentInfoKHR *info) { + const ScopedStageTimer mirror_timer(perf::Stage::kDesktopMirror); std::lock_guard lock(mutex_); if (EnvString("MOCKTAIL_VR_DESKTOP_MIRROR", "1") == "0" || !vk_ || !info || info->swapchainCount != 1 || !info->pSwapchains || !info->pImageIndices || diff --git a/src/vr/vr_perf.cc b/src/vr/vr_perf.cc index 544ba833..13e37fd6 100644 --- a/src/vr/vr_perf.cc +++ b/src/vr/vr_perf.cc @@ -57,6 +57,9 @@ const char* StageName(Stage stage) { case Stage::kMirrorPublish: return "mirror_publish"; case Stage::kEvidence: return "evidence"; case Stage::kControllerSync: return "controller_sync"; + case Stage::kDesktopMirror: return "desktop_mirror"; + case Stage::kPresentPrepare: return "present_prepare"; + case Stage::kPresentInterval: return "present_interval"; case Stage::kStageCount: break; } return "unknown"; From 7296520064338f0c20581db165deab17d0bde1c8 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Thu, 15 Oct 2026 04:52:37 -0500 Subject: [PATCH 50/58] perf(vr): order eye copies and the desktop mirror on the GPU Every frame the CPU waited for the GPU to finish Roblox's whole frame in the desktop mirror, then again for the eye copy, so CPU and GPU work never overlapped. The mirror blit now consumes the guest's present semaphores and signals a per-image semaphore that the present waits on. Present preparation hands remaining waits to the GPU the same way. Eye copies are submitted from a small ring of command buffers and fences and rely on queue order, so the CPU only waits when the GPU is three frames behind or the guest destroys an image a copy still reads. On Monado at 120 Hz with 1280x1440 eyes this removed 2 to 5 ms of waiting per frame and kept the frame rate at 118 to 119 fps instead of 103 to 118. Also free the copy resources when the guest device goes away without a session, so a later device never reuses stale handles. --- include/mocktail/vr/openxr_backend.h | 55 ++- src/graphics/bionic_vulkan_loader_adapter.cc | 27 +- src/vr/openxr_backend.cc | 350 +++++++++++++++---- tests/vr/openxr_backend_test.cc | 5 +- 4 files changed, 347 insertions(+), 90 deletions(-) diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index 343df123..e5487d9e 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -3,6 +3,7 @@ #include +#include #include #include #include @@ -196,16 +197,20 @@ class OpenXrBackend final { // Render-thread frame cycle at host present begin. void NoteHostPresent(VkQueue queue, VkDevice device); // VK_NOT_READY leaves the present untouched until eye submission provenance - // is known. VK_SUCCESS means its waits were consumed on the graphics queue. + // is known. VK_SUCCESS means its waits were consumed on the graphics queue, + // so the eye copies that follow there are ordered after them. The present + // must then wait on *ready instead (none when it is VK_NULL_HANDLE). VkResult PrepareHostPresent(VkQueue queue, VkDevice device, - const VkPresentInfoKHR* info); + const VkPresentInfoKHR* info, VkSemaphore* ready); void RecordDesktopSwapchain(VkDevice device, VkSwapchainKHR swapchain, const VkSwapchainCreateInfoKHR* info, const VkImage* images, unsigned count, bool converted_to_concurrent = false); - // On success the original present semaphores have been consumed and the - // copy has completed. The caller must present with no wait semaphores. - VkResult MirrorDesktop(VkQueue queue, const VkPresentInfoKHR* info); + // On success the original present semaphores have been consumed by the + // mirror blit on the GPU. The caller must present waiting only on *ready, + // which the blit signals. The CPU never waits for the guest's frame here. + VkResult MirrorDesktop(VkQueue queue, const VkPresentInfoKHR* info, + VkSemaphore* ready); // Pose published for the frame that is about to render. The device bridge // reads it once per frame on the render thread and injects it into the @@ -328,9 +333,32 @@ class OpenXrBackend final { std::string* error); void DestroyCopyResources(); // Records both eye copies (and, when the mirror is enabled, both staging - // readbacks) into one command buffer, submits once and waits on one fence. - // Replaces the previous per-eye submit+fence pair. + // readbacks) into one command buffer and submits it once. Only the shared + // memory mirror waits for it; the headset path leaves it to the GPU. bool CopyEyesIntoSwapchains(VkQueue queue); + // Per-frame GPU work (desktop mirror, eye copies) is submitted without a CPU + // wait. A slot is reused only after its fence signalled; its images let a + // guest destroy wait for the submissions that still read them. + struct AsyncSubmission { + VkCommandBuffer command = VK_NULL_HANDLE; + VkFence fence = VK_NULL_HANDLE; + bool pending = false; + std::vector images; + }; + // The next slot, recorded-ready, or nullptr when its previous submission + // did not finish in time (the device is hung or lost). + AsyncSubmission* NextAsyncSubmissionLocked(); + VkResult SubmitAsyncLocked(VkQueue queue, AsyncSubmission* slot, + const VkSubmitInfo& submit); + // Waits for pending submissions reading `image`, or all of them for + // VK_NULL_HANDLE. + void WaitAsyncSubmissionsLocked(VkImage image); + // Semaphore the present of `image_index` waits on, created on first use. + enum class PresentStep { kMirror, kPrepare }; + VkSemaphore PresentReadySemaphoreLocked(VkSwapchainKHR swapchain, + std::uint32_t image_index, + PresentStep step); + void DestroyRetiredSemaphoresLocked(); // In-headset keyboard: show/hide from the host's TextBox request, pointer // input, and trigger masking. Runs inside the controller sync. void UpdateVirtualKeyboardLocked(); @@ -412,8 +440,19 @@ class OpenXrBackend final { VkProcs* vk_ = nullptr; VkCommandPool copy_pool_ = VK_NULL_HANDLE; + // Synchronous one-off work: keyboard uploads, evidence, present fallback. VkCommandBuffer copy_command_ = VK_NULL_HANDLE; VkFence copy_fence_ = VK_NULL_HANDLE; + // Two submissions per frame, so three frames may be in flight. + static constexpr std::size_t kAsyncSubmissions = 6; + std::array async_{}; + std::size_t next_async_ = 0; + // The mirror's signal for this present, which PrepareHostPresent passes on. + VkSemaphore mirror_ready_ = VK_NULL_HANDLE; + // Semaphores of destroyed desktop swapchains. A present may still wait on + // one, so they are destroyed only once the device is idle. + std::vector retired_semaphores_; + std::unordered_map blit_features_; // Optional desktop two-eye mirror (MOCKTAIL_VR_MIRROR_SHM). Publishes the // latest complete pair through shared memory; inert when unset. @@ -427,6 +466,8 @@ class OpenXrBackend final { VkSharingMode sharing_mode = VK_SHARING_MODE_EXCLUSIVE; std::vector families; bool converted_to_concurrent = false; + // Two per image, see PresentReadySemaphoreLocked. + std::vector ready; }; std::unordered_map desktop_swapchains_; std::unordered_map image_views_; diff --git a/src/graphics/bionic_vulkan_loader_adapter.cc b/src/graphics/bionic_vulkan_loader_adapter.cc index ca92d891..bf678b28 100644 --- a/src/graphics/bionic_vulkan_loader_adapter.cc +++ b/src/graphics/bionic_vulkan_loader_adapter.cc @@ -1477,22 +1477,24 @@ ObservedHostQueuePresent(VkQueue queue, const VkPresentInfoKHR* present_info) { const VrXrNotePresentFn vr_note_present = state.vr_note_present.load(std::memory_order_acquire); VkPresentInfoKHR synchronized{}; + VkSemaphore ready = VK_NULL_HANDLE; if (vr_note_present != nullptr) { static const auto prepare = ResolveProcessFunction( + VkQueue, VkDevice, const VkPresentInfoKHR *, VkSemaphore *)>( "mocktail_vr_xr_prepare_present"); - // Mirror/overlay may have consumed waits; use the final dependency list. + // Mirror/overlay may have replaced waits; use the final dependency list. const VkResult prepared = - prepare - ? prepare(queue, HostDispatchForQueue(queue).device, present_info) - : VK_NOT_READY; + prepare ? prepare(queue, HostDispatchForQueue(queue).device, + present_info, &ready) + : VK_NOT_READY; if (prepared < 0) return prepared; if (prepared == VK_SUCCESS) { vr_note_present(queue, HostDispatchForQueue(queue).device); + // The waits went to the GPU; present after the semaphore they signal. synchronized = *present_info; - synchronized.waitSemaphoreCount = 0; - synchronized.pWaitSemaphores = nullptr; + synchronized.waitSemaphoreCount = ready != VK_NULL_HANDLE ? 1 : 0; + synchronized.pWaitSemaphores = ready != VK_NULL_HANDLE ? &ready : nullptr; present_info = &synchronized; } } @@ -3026,16 +3028,19 @@ vkQueuePresentKHR(VkQueue queue, const VkPresentInfoKHR* present_info) { if (trace != nullptr) { RecordProfiledFrame(trace, mocktail::graphics::TraceClockNanos()); } - static const auto mirror = ResolveProcessFunction( + static const auto mirror = ResolveProcessFunction( "mocktail_vr_desktop_present"); VkPresentInfoKHR mirrored{}; + VkSemaphore mirror_ready = VK_NULL_HANDLE; if (mirror && present_info) { - const VkResult mirror_result = mirror(queue, present_info); + const VkResult mirror_result = mirror(queue, present_info, &mirror_ready); if (mirror_result < 0) return mirror_result; if (mirror_result == VK_SUCCESS) { + // The blit consumed the guest's waits and signals mirror_ready. mirrored = *present_info; - mirrored.waitSemaphoreCount = 0; - mirrored.pWaitSemaphores = nullptr; + mirrored.waitSemaphoreCount = 1; + mirrored.pWaitSemaphores = &mirror_ready; present_info = &mirrored; } } diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index dcd04b09..ded9aefb 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -230,6 +230,8 @@ struct OpenXrBackend::VkProcs { PFN_vkGetPhysicalDeviceFormatProperties GetPhysicalDeviceFormatProperties = nullptr; PFN_vkCreateFence CreateFence = nullptr; PFN_vkDestroyFence DestroyFence = nullptr; + PFN_vkCreateSemaphore CreateSemaphore = nullptr; + PFN_vkDestroySemaphore DestroySemaphore = nullptr; PFN_vkResetFences ResetFences = nullptr; PFN_vkWaitForFences WaitForFences = nullptr; PFN_vkQueueSubmit QueueSubmit = nullptr; @@ -829,6 +831,8 @@ void OpenXrBackend::SetupDeviceAfterCreationLocked( MOCKTAIL_BACKEND_VK_FN(CmdClearColorImage) MOCKTAIL_BACKEND_VK_FN(CreateFence) MOCKTAIL_BACKEND_VK_FN(DestroyFence) + MOCKTAIL_BACKEND_VK_FN(CreateSemaphore) + MOCKTAIL_BACKEND_VK_FN(DestroySemaphore) MOCKTAIL_BACKEND_VK_FN(ResetFences) MOCKTAIL_BACKEND_VK_FN(WaitForFences) MOCKTAIL_BACKEND_VK_FN(QueueSubmit) @@ -1363,6 +1367,7 @@ void OpenXrBackend::MatchSwapchainFormatToEyesLocked() { return; // already a raw copy } eye_swapchain_format_ = preferred; + WaitAsyncSubmissionsLocked(VK_NULL_HANDLE); DestroySwapchains(); std::string error; Require(CreateSwapchains(&error), error); @@ -1405,6 +1410,7 @@ void OpenXrBackend::TeardownSessionLocked(const char* reason, bool preserve_devi DestroyVirtualKeyboardResources(); DestroyCopyResources(); DestroySwapchains(); + DestroyRetiredSemaphoresLocked(); // Action pose spaces belong to this session; detach before the session and // its spaces go away so a recreated session attaches cleanly. { @@ -1450,7 +1456,18 @@ void OpenXrBackend::TeardownSessionLocked(const char* reason, bool preserve_devi session_recovery_pending_ = false; next_session_retry_ns_ = 0; images_.clear(); + if (copy_pool_ != VK_NULL_HANDLE && vk_ && vk_device_) { + // Mirror blits can outlive the session while it recovers. + (void)IdleHostDevice(vk_->DeviceWaitIdle, vk_device_); + DestroyCopyResources(); + } + for (const auto &entry : desktop_swapchains_) + for (const VkSemaphore semaphore : entry.second.ready) + if (semaphore != VK_NULL_HANDLE) + retired_semaphores_.push_back(semaphore); + DestroyRetiredSemaphoresLocked(); desktop_swapchains_.clear(); + blit_features_.clear(); image_views_.clear(); framebuffer_views_.clear(); framebuffer_render_pass_.clear(); @@ -1695,6 +1712,8 @@ void OpenXrBackend::NoteDestroyImage(VkImage image) { return; } std::lock_guard lock(mutex_); + // A frame copy or mirror blit may still read it on the GPU. + WaitAsyncSubmissionsLocked(image); images_.erase(image); eye_render_queues_.erase(image); eye_render_frames_.erase(image); @@ -2251,9 +2270,15 @@ VkQueue OpenXrBackend::GraphicsQueueForPresentLocked(VkQueue present) const { } VkResult OpenXrBackend::PrepareHostPresent(VkQueue queue, VkDevice device, - const VkPresentInfoKHR *info) { + const VkPresentInfoKHR *info, + VkSemaphore *ready) { const ScopedStageTimer prepare_timer(perf::Stage::kPresentPrepare); std::lock_guard lock(mutex_); + // Only valid for the present it was signalled for. + const VkSemaphore mirrored = std::exchange(mirror_ready_, VK_NULL_HANDLE); + if (!ready) + return VK_NOT_READY; + *ready = VK_NULL_HANDLE; if (!armed_ || graphics_api_ != VrGraphicsApi::kVulkan || !vk_ || device != vk_device_ || !info) return VK_NOT_READY; @@ -2267,6 +2292,44 @@ VkResult OpenXrBackend::PrepareHostPresent(VkQueue queue, VkDevice device, } if (info->waitSemaphoreCount == 0) return VK_SUCCESS; + if (!info->pWaitSemaphores) + return VK_NOT_READY; + // The mirror already waited on the guest's semaphores on the graphics queue. + if (mirrored != VK_NULL_HANDLE && info->waitSemaphoreCount == 1 && + info->pWaitSemaphores[0] == mirrored) { + *ready = mirrored; + return VK_SUCCESS; + } + const VkQueue graphics = GraphicsQueueForPresentLocked(queue); + const VkSemaphore signal = + info->swapchainCount == 1 && info->pSwapchains && info->pImageIndices + ? PresentReadySemaphoreLocked(info->pSwapchains[0], + info->pImageIndices[0], + PresentStep::kPrepare) + : VK_NULL_HANDLE; + if (signal != VK_NULL_HANDLE) { + // Hand the waits to the GPU: an empty batch on the graphics queue waits + // on them, which orders the eye copies submitted after it, and signals + // the one semaphore the present waits on instead. + std::vector stages(info->waitSemaphoreCount, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); + VkSubmitInfo submit{}; + submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit.waitSemaphoreCount = info->waitSemaphoreCount; + submit.pWaitSemaphores = info->pWaitSemaphores; + submit.pWaitDstStageMask = stages.data(); + submit.signalSemaphoreCount = 1; + submit.pSignalSemaphores = &signal; + const VkResult result = SubmitHostQueue(vk_->QueueSubmit, vk_device_, + graphics, 1, &submit, + VK_NULL_HANDLE); + // A failed submit consumed nothing, so the guest's present stays valid. + if (result != VK_SUCCESS) + return result == VK_ERROR_DEVICE_LOST ? result : VK_NOT_READY; + *ready = signal; + return VK_SUCCESS; + } + // An unregistered swapchain has no semaphores of ours: wait on the CPU. std::string error; if (!EnsureCopyResources(device, vk_queue_family_, &error)) { // Nothing consumed yet: present normally and skip VR for this frame. @@ -2279,8 +2342,8 @@ VkResult OpenXrBackend::PrepareHostPresent(VkQueue queue, VkDevice device, return VK_NOT_READY; } return internal::WaitForPresentDependencies( - device, GraphicsQueueForPresentLocked(queue), *info, copy_fence_, - vk_->ResetFences, vk_->QueueSubmit, vk_->WaitForFences); + device, graphics, *info, copy_fence_, vk_->ResetFences, vk_->QueueSubmit, + vk_->WaitForFences); } void OpenXrBackend::NoteHostPresent(VkQueue queue, VkDevice device) { @@ -3124,7 +3187,8 @@ bool OpenXrBackend::EnsureCopyResources(VkDevice device, std::string* error) { try { if (copy_pool_ != VK_NULL_HANDLE && copy_command_ != VK_NULL_HANDLE && - copy_fence_ != VK_NULL_HANDLE) { + copy_fence_ != VK_NULL_HANDLE && + async_.back().fence != VK_NULL_HANDLE) { return true; } if (copy_pool_ == VK_NULL_HANDLE) { @@ -3153,6 +3217,25 @@ bool OpenXrBackend::EnsureCopyResources(VkDevice device, VK_SUCCESS, "vkCreateFence failed for XR copies"); } + for (auto &slot : async_) { + if (slot.command == VK_NULL_HANDLE) { + VkCommandBufferAllocateInfo allocate_info{}; + allocate_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + allocate_info.commandPool = copy_pool_; + allocate_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + allocate_info.commandBufferCount = 1; + Require(vk_->AllocateCommandBuffers(device, &allocate_info, + &slot.command) == VK_SUCCESS, + "vkAllocateCommandBuffers failed for XR frame copies"); + } + if (slot.fence == VK_NULL_HANDLE) { + VkFenceCreateInfo fence_info{}; + fence_info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO; + Require(vk_->CreateFence(device, &fence_info, nullptr, &slot.fence) == + VK_SUCCESS, + "vkCreateFence failed for XR frame copies"); + } + } return true; } catch (const std::exception& exception) { *error = exception.what(); @@ -3161,12 +3244,21 @@ bool OpenXrBackend::EnsureCopyResources(VkDevice device, } void OpenXrBackend::DestroyCopyResources() { + mirror_ready_ = VK_NULL_HANDLE; + next_async_ = 0; if (vk_ == nullptr || vk_device_ == VK_NULL_HANDLE) { copy_pool_ = VK_NULL_HANDLE; copy_command_ = VK_NULL_HANDLE; copy_fence_ = VK_NULL_HANDLE; + async_ = {}; return; } + // Callers idle the device first, so no slot is still pending. + for (auto &slot : async_) { + if (slot.fence != VK_NULL_HANDLE) + vk_->DestroyFence(vk_device_, slot.fence, nullptr); + slot = {}; // command buffers are freed with the pool + } if (copy_fence_ != VK_NULL_HANDLE) { vk_->DestroyFence(vk_device_, copy_fence_, nullptr); copy_fence_ = VK_NULL_HANDLE; @@ -3179,6 +3271,95 @@ void OpenXrBackend::DestroyCopyResources() { } } +OpenXrBackend::AsyncSubmission *OpenXrBackend::NextAsyncSubmissionLocked() { + AsyncSubmission &slot = async_[next_async_]; + if (slot.fence == VK_NULL_HANDLE || slot.command == VK_NULL_HANDLE) + return nullptr; + if (slot.pending) { + // Normally signalled frames ago; only a GPU running three frames behind + // waits here, which is the backpressure it needs. + const ScopedStageTimer fence_timer(perf::Stage::kCopyFenceWait); + perf::ProcessCollector().Count(perf::Counter::kFenceWaits); + if (vk_->WaitForFences(vk_device_, 1, &slot.fence, VK_TRUE, + 2'000'000'000ULL) != VK_SUCCESS) + return nullptr; + slot.pending = false; + } + slot.images.clear(); + if (vk_->ResetFences(vk_device_, 1, &slot.fence) != VK_SUCCESS || + vk_->ResetCommandBuffer(slot.command, 0) != VK_SUCCESS) + return nullptr; + next_async_ = (next_async_ + 1) % async_.size(); + return &slot; +} + +VkResult OpenXrBackend::SubmitAsyncLocked(VkQueue queue, AsyncSubmission *slot, + const VkSubmitInfo &submit) { + VkResult result = VK_SUCCESS; + { + const ScopedStageTimer submit_timer(perf::Stage::kCopySubmit); + result = SubmitHostQueue(vk_->QueueSubmit, vk_device_, queue, 1, &submit, + slot->fence); + } + perf::ProcessCollector().Count(perf::Counter::kSubmits); + slot->pending = result == VK_SUCCESS; + if (!slot->pending) + slot->images.clear(); + return result; +} + +void OpenXrBackend::WaitAsyncSubmissionsLocked(VkImage image) { + if (vk_ == nullptr || vk_device_ == VK_NULL_HANDLE) + return; + for (auto &slot : async_) { + if (!slot.pending || + (image != VK_NULL_HANDLE && + std::find(slot.images.begin(), slot.images.end(), image) == + slot.images.end())) + continue; + if (vk_->WaitForFences(vk_device_, 1, &slot.fence, VK_TRUE, + 2'000'000'000ULL) != VK_SUCCESS) + (void)IdleHostDevice(vk_->DeviceWaitIdle, vk_device_); + slot.pending = false; + slot.images.clear(); + } +} + +VkSemaphore OpenXrBackend::PresentReadySemaphoreLocked( + VkSwapchainKHR swapchain, std::uint32_t image_index, PresentStep step) { + const auto found = desktop_swapchains_.find(swapchain); + if (found == desktop_swapchains_.end() || + image_index >= found->second.images.size()) + return VK_NULL_HANDLE; + // One per image and step: when the guest acquires an image again, the + // present that waited on its semaphore has finished with it. The mirror and + // the prepare step never share one, because the overlay between them may + // wait on the mirror's from another queue. + auto &ready = found->second.ready; + ready.resize(found->second.images.size() * 2, VK_NULL_HANDLE); + VkSemaphore &semaphore = + ready[image_index * 2 + (step == PresentStep::kPrepare ? 1 : 0)]; + if (semaphore == VK_NULL_HANDLE) { + VkSemaphoreCreateInfo info{}; + info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO; + if (vk_->CreateSemaphore(vk_device_, &info, nullptr, &semaphore) != + VK_SUCCESS) + semaphore = VK_NULL_HANDLE; + } + return semaphore; +} + +void OpenXrBackend::DestroyRetiredSemaphoresLocked() { + if (retired_semaphores_.empty()) + return; + if (vk_ != nullptr && vk_device_ != VK_NULL_HANDLE) { + (void)IdleHostDevice(vk_->DeviceWaitIdle, vk_device_); + for (const VkSemaphore semaphore : retired_semaphores_) + vk_->DestroySemaphore(vk_device_, semaphore, nullptr); + } + retired_semaphores_.clear(); +} + bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { std::string error; const auto family = queue_families_.find(queue); @@ -3210,13 +3391,16 @@ bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { eye_swapchains_[1].images[eye_swapchains_[1].acquired_index]}; EnsureMirrorForExtent(eyes_[0].width, eyes_[0].height); - if (vk_->ResetCommandBuffer(copy_command_, 0) != VK_SUCCESS) { - return false; + AsyncSubmission *const slot = NextAsyncSubmissionLocked(); + if (slot == nullptr) { + (void)IdleHostDevice(vk_->DeviceWaitIdle, vk_device_); + throw std::runtime_error("XR copy fence wait failed"); } + const VkCommandBuffer command = slot->command; VkCommandBufferBeginInfo begin{}; begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; - if (vk_->BeginCommandBuffer(copy_command_, &begin) != VK_SUCCESS) { + if (vk_->BeginCommandBuffer(command, &begin) != VK_SUCCESS) { return false; } @@ -3248,7 +3432,7 @@ bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { destination_barrier.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; } - vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + vk_->CmdPipelineBarrier(command, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, static_cast(prepare.size()), prepare.data()); @@ -3265,7 +3449,7 @@ bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { region.srcSubresource = subresource; region.dstSubresource = subresource; region.extent = {binding.width, binding.height, 1}; - vk_->CmdCopyImage(copy_command_, source[eye], + vk_->CmdCopyImage(command, source[eye], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, destination[eye], VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion); } else { @@ -3275,7 +3459,7 @@ bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { region.srcOffsets[1] = {static_cast(binding.width), static_cast(binding.height), 1}; region.dstOffsets[1] = region.srcOffsets[1]; - vk_->CmdBlitImage(copy_command_, source[eye], + vk_->CmdBlitImage(command, source[eye], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, destination[eye], VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ®ion, VK_FILTER_NEAREST); @@ -3283,7 +3467,7 @@ bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { if (mirror_.enabled()) { // Reads the source while it is already TRANSFER_SRC; the restore // barrier below returns it to the guest's final layout afterwards. - mirror_.RecordEyeCopy(MirrorProcs(), copy_command_, eye, source[eye], + mirror_.RecordEyeCopy(MirrorProcs(), command, eye, source[eye], binding.final_layout, binding.width, binding.height); } @@ -3317,47 +3501,41 @@ bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { finish_destination.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; } - vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_TRANSFER_BIT, + vk_->CmdPipelineBarrier(command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, static_cast(finish.size()), finish.data()); - if (vk_->EndCommandBuffer(copy_command_) != VK_SUCCESS) { - return false; - } - if (vk_->ResetFences(vk_device_, 1, ©_fence_) != VK_SUCCESS) { + if (vk_->EndCommandBuffer(command) != VK_SUCCESS) { return false; } VkSubmitInfo submit{}; submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; submit.commandBufferCount = 1; - submit.pCommandBuffers = ©_command_; - // Present dependencies are complete; this fence protects reuse of the - // copy command buffer and mirror staging buffers. - VkResult submit_result = VK_SUCCESS; - { - const ScopedStageTimer submit_timer(perf::Stage::kCopySubmit); - submit_result = SubmitHostQueue(vk_->QueueSubmit, vk_device_, queue, 1, - &submit, copy_fence_); - } - perf::ProcessCollector().Count(perf::Counter::kSubmits); - if (submit_result != VK_SUCCESS) { + submit.pCommandBuffers = &command; + // The present dependencies were consumed earlier on this queue, so the + // copy runs after the guest's eye passes without the CPU waiting for them. + // xrReleaseSwapchainImage only needs the copy submitted to this queue. + // NoteDestroyImage waits for this slot before the guest frees a source. + slot->images = {source[0], source[1], destination[0], destination[1]}; + if (SubmitAsyncLocked(queue, slot, submit) != VK_SUCCESS) { return false; } - VkResult fence_result = VK_SUCCESS; - { - // mutex_ stays held: the guest's vkDestroyImage waits on it in - // NoteDestroyImage, which keeps the eye image alive until the copy ends. - const ScopedStageTimer fence_timer(perf::Stage::kCopyFenceWait); - fence_result = vk_->WaitForFences(vk_device_, 1, ©_fence_, VK_TRUE, - 2'000'000'000ULL); - } - perf::ProcessCollector().Count(perf::Counter::kFenceWaits); - if (fence_result != VK_SUCCESS) { - (void)IdleHostDevice(vk_->DeviceWaitIdle, vk_device_); - throw std::runtime_error("XR copy fence wait failed"); - } if (mirror_.enabled()) { + // The shared-memory readback must land before it is published. + VkResult fence_result = VK_SUCCESS; + { + const ScopedStageTimer fence_timer(perf::Stage::kCopyFenceWait); + fence_result = vk_->WaitForFences(vk_device_, 1, &slot->fence, VK_TRUE, + 2'000'000'000ULL); + } + perf::ProcessCollector().Count(perf::Counter::kFenceWaits); + if (fence_result != VK_SUCCESS) { + (void)IdleHostDevice(vk_->DeviceWaitIdle, vk_device_); + throw std::runtime_error("XR copy fence wait failed"); + } + slot->pending = false; + slot->images.clear(); const ScopedStageTimer publish_timer(perf::Stage::kMirrorPublish); mirror_.Publish(published_pose_.frame, eyes_[0].format, eyes_[0].width, eyes_[0].height, perf::NowNs()); @@ -3421,7 +3599,18 @@ void OpenXrBackend::RecordDesktopSwapchain(VkDevice device, unsigned count, bool converted_to_concurrent) { std::lock_guard lock(mutex_); - desktop_swapchains_.erase(swapchain); + const auto previous = desktop_swapchains_.find(swapchain); + if (previous != desktop_swapchains_.end()) { + for (const VkImage image : previous->second.images) + WaitAsyncSubmissionsLocked(image); + for (const VkSemaphore semaphore : previous->second.ready) + if (semaphore != VK_NULL_HANDLE) + retired_semaphores_.push_back(semaphore); + desktop_swapchains_.erase(previous); + // Bounded: a resize retires a few; idle the device only rarely. + if (retired_semaphores_.size() > 64) + DestroyRetiredSemaphoresLocked(); + } if (!device || device != vk_device_ || !info || !images || !count || !(info->imageUsage & VK_IMAGE_USAGE_TRANSFER_DST_BIT) || info->imageArrayLayers != 1) @@ -3443,10 +3632,16 @@ void OpenXrBackend::RecordDesktopSwapchain(VkDevice device, } VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, - const VkPresentInfoKHR *info) { + const VkPresentInfoKHR *info, + VkSemaphore *ready) { const ScopedStageTimer mirror_timer(perf::Stage::kDesktopMirror); + static const bool enabled = + EnvString("MOCKTAIL_VR_DESKTOP_MIRROR", "1") != "0"; + if (ready) + *ready = VK_NULL_HANDLE; std::lock_guard lock(mutex_); - if (EnvString("MOCKTAIL_VR_DESKTOP_MIRROR", "1") == "0" || !vk_ || !info || + mirror_ready_ = VK_NULL_HANDLE; + if (!enabled || !ready || !vk_ || !info || info->swapchainCount != 1 || !info->pSwapchains || !info->pImageIndices || !eyes_[0].valid || !eyes_[0].image || (info->waitSemaphoreCount && !info->pWaitSemaphores) || @@ -3469,25 +3664,33 @@ VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, return VK_NOT_READY; } queue = GraphicsQueueForPresentLocked(queue); - VkFormatProperties source_properties{}, target_properties{}; - vk_->GetPhysicalDeviceFormatProperties(vk_physical_device_, eyes_[0].format, - &source_properties); - vk_->GetPhysicalDeviceFormatProperties(vk_physical_device_, target.format, - &target_properties); - if (!(source_properties.optimalTilingFeatures & - VK_FORMAT_FEATURE_BLIT_SRC_BIT) || - !(target_properties.optimalTilingFeatures & - VK_FORMAT_FEATURE_BLIT_DST_BIT)) + const auto blit_features = [&](VkFormat format) { + const auto cached = blit_features_.find(format); + if (cached != blit_features_.end()) + return cached->second; + VkFormatProperties properties{}; + vk_->GetPhysicalDeviceFormatProperties(vk_physical_device_, format, + &properties); + return blit_features_[format] = properties.optimalTilingFeatures; + }; + if (!(blit_features(eyes_[0].format) & VK_FORMAT_FEATURE_BLIT_SRC_BIT) || + !(blit_features(target.format) & VK_FORMAT_FEATURE_BLIT_DST_BIT)) return VK_NOT_READY; std::string error; if (!EnsureCopyResources(vk_device_, vk_queue_family_, &error)) return VK_NOT_READY; - if (vk_->ResetCommandBuffer(copy_command_, 0) != VK_SUCCESS) + const VkSemaphore signal = PresentReadySemaphoreLocked( + info->pSwapchains[0], info->pImageIndices[0], PresentStep::kMirror); + if (signal == VK_NULL_HANDLE) return VK_NOT_READY; + AsyncSubmission *const slot = NextAsyncSubmissionLocked(); + if (slot == nullptr) + return VK_NOT_READY; + const VkCommandBuffer command = slot->command; VkCommandBufferBeginInfo begin{}; begin.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; - if (vk_->BeginCommandBuffer(copy_command_, &begin) != VK_SUCCESS) + if (vk_->BeginCommandBuffer(command, &begin) != VK_SUCCESS) return VK_NOT_READY; VkImageMemoryBarrier barriers[2]{}; for (auto &b : barriers) { @@ -3504,18 +3707,18 @@ VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, barriers[1].oldLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; barriers[1].newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; barriers[1].dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + vk_->CmdPipelineBarrier(command, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 2, barriers); const VkClearColorValue black{{0.f, 0.f, 0.f, 1.f}}; - vk_->CmdClearColorImage(copy_command_, barriers[1].image, + vk_->CmdClearColorImage(command, barriers[1].image, barriers[1].newLayout, &black, 1, &barriers[1].subresourceRange); VkMemoryBarrier clear_barrier{}; clear_barrier.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; clear_barrier.srcAccessMask = clear_barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_TRANSFER_BIT, + vk_->CmdPipelineBarrier(command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &clear_barrier, 0, nullptr, 0, nullptr); const double scale = std::min(double(target.extent.width) / eyes_[0].width, @@ -3531,7 +3734,7 @@ VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, static_cast(eyes_[0].height), 1}; region.dstOffsets[0] = {x, y, 0}; region.dstOffsets[1] = {x + width, y + height, 1}; - vk_->CmdBlitImage(copy_command_, barriers[0].image, barriers[0].newLayout, + vk_->CmdBlitImage(command, barriers[0].image, barriers[0].newLayout, barriers[1].image, barriers[1].newLayout, 1, ®ion, VK_FILTER_NEAREST); for (auto &b : barriers) { @@ -3539,11 +3742,10 @@ VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, b.srcAccessMask = b.dstAccessMask; b.dstAccessMask = VK_ACCESS_MEMORY_READ_BIT; } - vk_->CmdPipelineBarrier(copy_command_, VK_PIPELINE_STAGE_TRANSFER_BIT, + vk_->CmdPipelineBarrier(command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 2, barriers); - if (vk_->EndCommandBuffer(copy_command_) != VK_SUCCESS || - vk_->ResetFences(vk_device_, 1, ©_fence_) != VK_SUCCESS) + if (vk_->EndCommandBuffer(command) != VK_SUCCESS) return VK_NOT_READY; std::vector stages(info->waitSemaphoreCount, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); @@ -3553,14 +3755,18 @@ VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, submit.pWaitSemaphores = info->pWaitSemaphores; submit.pWaitDstStageMask = stages.data(); submit.commandBufferCount = 1; - submit.pCommandBuffers = ©_command_; - VkResult result = SubmitHostQueue(vk_->QueueSubmit, vk_device_, queue, 1, - &submit, copy_fence_); + submit.pCommandBuffers = &command; + // The present waits on `signal` instead of the guest's semaphores. Their + // wait also orders the eye copies submitted later on this queue. + submit.signalSemaphoreCount = 1; + submit.pSignalSemaphores = &signal; + slot->images = {eyes_[0].image, barriers[1].image}; + const VkResult result = SubmitAsyncLocked(queue, slot, submit); if (result != VK_SUCCESS) return result; - // Once submitted, never fall back to waiting on the consumed semaphores. - result = vk_->WaitForFences(vk_device_, 1, ©_fence_, VK_TRUE, UINT64_MAX); - return result; + mirror_ready_ = signal; + *ready = signal; + return VK_SUCCESS; } // Diagnostic only: read BOTH images on the guest queue while the XR image is @@ -3913,9 +4119,10 @@ VkImageLayout mocktail_vr_xr_desktop_barrier(VkCommandBuffer command, return new_layout; } VkResult mocktail_vr_desktop_present(VkQueue queue, - const VkPresentInfoKHR *info) { + const VkPresentInfoKHR *info, + VkSemaphore *ready) { try { - if (auto* backend = mocktail::vr::ActiveVrBackend()) return backend->MirrorDesktop(queue, info); + if (auto* backend = mocktail::vr::ActiveVrBackend()) return backend->MirrorDesktop(queue, info, ready); return VK_NOT_READY; } catch (const std::exception& error) { std::fprintf(stderr, " [vr-backend] desktop mirror failed: %s\n", error.what()); @@ -3938,10 +4145,11 @@ bool mocktail_vr_xr_queue_sharing_requested() { } VkResult mocktail_vr_xr_prepare_present(VkQueue queue, VkDevice device, - const VkPresentInfoKHR *info) { + const VkPresentInfoKHR *info, + VkSemaphore *ready) { try { if (auto *backend = mocktail::vr::ActiveVrBackend()) - return backend->PrepareHostPresent(queue, device, info); + return backend->PrepareHostPresent(queue, device, info, ready); return VK_NOT_READY; } catch (const std::exception &error) { std::fprintf(stderr, " [vr-backend] present synchronization failed: %s\n", diff --git a/tests/vr/openxr_backend_test.cc b/tests/vr/openxr_backend_test.cc index 12933c1a..3d0dbac6 100644 --- a/tests/vr/openxr_backend_test.cc +++ b/tests/vr/openxr_backend_test.cc @@ -251,7 +251,9 @@ TEST(OpenXrDesktopMirror, RequiresTransferUsageAndDropsDestroyedSwapchain) { EXPECT_EQ(OpenXrBackendTestAccess::DesktopCount(b), 0u); b.RecordDesktopSwapchain(OpenXrBackendTestAccess::Handle(3), chain, &info, &image, 1); EXPECT_EQ(OpenXrBackendTestAccess::DesktopCount(b), 0u); - EXPECT_EQ(b.MirrorDesktop(VK_NULL_HANDLE, nullptr), VK_NOT_READY); + VkSemaphore ready = VK_NULL_HANDLE; + EXPECT_EQ(b.MirrorDesktop(VK_NULL_HANDLE, nullptr, &ready), VK_NOT_READY); + EXPECT_EQ(ready, VK_NULL_HANDLE); } } @@ -349,6 +351,7 @@ TEST_F(OpenXrPresentDependencies, InvalidDependenciesDoNotSubmit) { EXPECT_LT(Run(), VK_SUCCESS); EXPECT_EQ(resets + submits + waits, 0); } + TEST(OpenXrPresentQueue, RoutesSameAndSplitFamilyPresentationToGraphicsQueue) { OpenXrBackend backend; Access::Configure(backend); From 9ae5667cd715dd5532779d94671de075e2f1307b Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:19:04 -0500 Subject: [PATCH 51/58] perf(vr): read per-frame settings once and skip the lock for other passes The frame cycle read the pose source, evidence and mirror settings from the environment every frame. Read them once. Check the thread-local eye request before taking the backend lock, so render passes that are not eye passes do not contend with the frame cycle. --- src/vr/openxr_backend.cc | 46 ++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index ded9aefb..c5e64bdc 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -111,15 +111,24 @@ std::string EnvString(const char* name, const char* fallback) { #endif // Desktop/acceptance diagnostics (scripted head pose, eye evidence dumps) // exist only in builds configured with -DMOCKTAIL_VR_DIAGNOSTICS=ON; in -// normal builds their environment variables have no effect. -std::string EvidenceDirectory() { - return MOCKTAIL_VR_DIAGNOSTICS ? EnvString("MOCKTAIL_VR_XR_EVIDENCE_DIR", "") - : std::string(); +// normal builds their environment variables have no effect. Read once: the +// frame cycle consults them every frame. +const std::string& EvidenceDirectory() { + static const std::string directory = + MOCKTAIL_VR_DIAGNOSTICS ? EnvString("MOCKTAIL_VR_XR_EVIDENCE_DIR", "") + : std::string(); + return directory; +} +std::uint64_t EvidenceInterval() { + static const std::uint64_t interval = std::max( + 1, EnvUnsigned("MOCKTAIL_VR_XR_EVIDENCE_INTERVAL", 20)); + return interval; } -std::string PoseSource() { - return MOCKTAIL_VR_DIAGNOSTICS - ? EnvString("MOCKTAIL_VR_POSE_SOURCE", "runtime") - : std::string("runtime"); +const std::string& PoseSource() { + static const std::string source = + MOCKTAIL_VR_DIAGNOSTICS ? EnvString("MOCKTAIL_VR_POSE_SOURCE", "runtime") + : std::string("runtime"); + return source; } // Coordinate and pose math is shared with the unit tests through @@ -1765,7 +1774,8 @@ void OpenXrBackend::NoteRenderPassBegin(VkCommandBuffer command_buffer, if (!recording_.load(std::memory_order_acquire) || info == nullptr) { return; } - std::lock_guard lock(mutex_); + // Thread-local eye request: most render passes are not eye passes and + // return here without contending for the backend lock. void* owner = nullptr; void* guest_framebuffer = nullptr; int eye_index = -1; @@ -1773,6 +1783,7 @@ void OpenXrBackend::NoteRenderPassBegin(VkCommandBuffer command_buffer, eye_index < 0 || eye_index > 1) { return; } + std::lock_guard lock(mutex_); const auto attachments = framebuffer_views_.find(info->framebuffer); if (attachments == framebuffer_views_.end()) { return; @@ -2585,11 +2596,7 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { submitted_frames_.fetch_add(1, std::memory_order_relaxed) + 1; if (submitted == 1 || submitted % 120 == 0 || (!EvidenceDirectory().empty() && - published_pose_.frame % - std::max( - 1, - EnvUnsigned("MOCKTAIL_VR_XR_EVIDENCE_INTERVAL", 20)) == - 0)) { + published_pose_.frame % EvidenceInterval() == 0)) { if (graphics_api_ == VrGraphicsApi::kOpenGles) { Log(" [vr-backend][evidence] projection submitted=%llu frame=%llu " "graphics=GLES eye0_texture=%u eye1_texture=%u\n", @@ -2688,8 +2695,7 @@ void OpenXrBackend::RunFrameCycle(VkQueue queue, VkDevice device) { const bool head_valid = XR_SUCCEEDED(head_result) && (head.locationFlags & required_location) == required_location; - const std::string pose_source = - PoseSource(); + const std::string &pose_source = PoseSource(); Pose head_pose = head_valid ? XrToPose(head.pose) : Pose{}; if (head_valid) { // Tracked LOCAL head pose, independent of the scripted test source; @@ -3546,7 +3552,7 @@ bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { void OpenXrBackend::EnsureMirrorForExtent(std::uint32_t width, std::uint32_t height) { - const std::string name = EnvString("MOCKTAIL_VR_MIRROR_SHM", ""); + static const std::string name = EnvString("MOCKTAIL_VR_MIRROR_SHM", ""); if (name.empty()) { return; } @@ -3772,10 +3778,8 @@ VkResult OpenXrBackend::MirrorDesktop(VkQueue queue, // Diagnostic only: read BOTH images on the guest queue while the XR image is // still acquired. Never read a released image owned by the compositor. bool OpenXrBackend::CaptureEyeEvidence(VkQueue queue, int eye) { - const std::string directory = EvidenceDirectory(); - const auto interval = std::max( - 1, EnvUnsigned("MOCKTAIL_VR_XR_EVIDENCE_INTERVAL", 20)); - if (directory.empty() || published_pose_.frame % interval != 0) + const std::string &directory = EvidenceDirectory(); + if (directory.empty() || published_pose_.frame % EvidenceInterval() != 0) return true; VkBuffer buffers[2] = {}; VkDeviceMemory memory[2] = {}; From dea1d5fd598950bab19b2052c04edbe6bf4d385a Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:43:51 -0500 Subject: [PATCH 52/58] test(vr): cover handing present dependencies to the GPU Move the empty wait-and-signal batch into internal::ChainPresentDependencies, like WaitForPresentDependencies, and test that it waits on every present semaphore, signals the replacement, never blocks the CPU, and leaves a failed submit to the guest. --- include/mocktail/vr/openxr_backend.h | 7 ++++ src/vr/openxr_backend.cc | 51 +++++++++++++++++----------- tests/vr/openxr_backend_test.cc | 48 ++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 20 deletions(-) diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index e5487d9e..6408ffa5 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -38,6 +38,13 @@ VkResult WaitForPresentDependencies(VkDevice device, VkQueue queue, VkFence fence, PFN_vkResetFences reset, PFN_vkQueueSubmit submit, PFN_vkWaitForFences wait); +// Hands binary present dependencies to the GPU instead: one empty batch on +// `queue` waits on them and signals `signal`, which the present waits on +// instead. Work submitted to `queue` afterwards is ordered after the waits. +// The CPU never blocks. VK_NOT_READY means nothing was consumed. +VkResult ChainPresentDependencies(VkDevice device, VkQueue queue, + const VkPresentInfoKHR& present, + VkSemaphore signal, PFN_vkQueueSubmit submit); // Makes the Vulkan adapter serialize queue calls for the rest of the process. // Called when a backend arms; the adapter library may load before or after. void RequestVulkanQueueSharing(); diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index c5e64bdc..9b5f8a05 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -2254,6 +2254,30 @@ VkResult internal::WaitForPresentDependencies(VkDevice device, VkQueue queue, return result == VK_SUCCESS || result < 0 ? result : VK_ERROR_DEVICE_LOST; } +VkResult internal::ChainPresentDependencies(VkDevice device, VkQueue queue, + const VkPresentInfoKHR &present, + VkSemaphore signal, + PFN_vkQueueSubmit submit) { + if (present.waitSemaphoreCount == 0 || !present.pWaitSemaphores || + signal == VK_NULL_HANDLE) + return VK_NOT_READY; + std::vector stages(present.waitSemaphoreCount, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); + VkSubmitInfo info{}; + info.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + info.waitSemaphoreCount = present.waitSemaphoreCount; + info.pWaitSemaphores = present.pWaitSemaphores; + info.pWaitDstStageMask = stages.data(); + info.signalSemaphoreCount = 1; + info.pSignalSemaphores = &signal; + const VkResult result = + SubmitHostQueue(submit, device, queue, 1, &info, VK_NULL_HANDLE); + // A failed submit consumed nothing, so the guest's present stays valid. + if (result != VK_SUCCESS) + return result == VK_ERROR_DEVICE_LOST ? result : VK_NOT_READY; + return VK_SUCCESS; +} + bool OpenXrBackend::SupportsPresentQueueLocked(VkQueue queue) const { return GraphicsQueueForPresentLocked(queue) != VK_NULL_HANDLE; } @@ -2319,26 +2343,13 @@ VkResult OpenXrBackend::PrepareHostPresent(VkQueue queue, VkDevice device, PresentStep::kPrepare) : VK_NULL_HANDLE; if (signal != VK_NULL_HANDLE) { - // Hand the waits to the GPU: an empty batch on the graphics queue waits - // on them, which orders the eye copies submitted after it, and signals - // the one semaphore the present waits on instead. - std::vector stages(info->waitSemaphoreCount, - VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); - VkSubmitInfo submit{}; - submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; - submit.waitSemaphoreCount = info->waitSemaphoreCount; - submit.pWaitSemaphores = info->pWaitSemaphores; - submit.pWaitDstStageMask = stages.data(); - submit.signalSemaphoreCount = 1; - submit.pSignalSemaphores = &signal; - const VkResult result = SubmitHostQueue(vk_->QueueSubmit, vk_device_, - graphics, 1, &submit, - VK_NULL_HANDLE); - // A failed submit consumed nothing, so the guest's present stays valid. - if (result != VK_SUCCESS) - return result == VK_ERROR_DEVICE_LOST ? result : VK_NOT_READY; - *ready = signal; - return VK_SUCCESS; + // The eye copies submitted after this on the graphics queue are ordered + // after the guest's semaphores without the CPU waiting for them. + const VkResult result = internal::ChainPresentDependencies( + device, graphics, *info, signal, vk_->QueueSubmit); + if (result == VK_SUCCESS) + *ready = signal; + return result; } // An unregistered swapchain has no semaphores of ours: wait on the CPU. std::string error; diff --git a/tests/vr/openxr_backend_test.cc b/tests/vr/openxr_backend_test.cc index 3d0dbac6..6966ef30 100644 --- a/tests/vr/openxr_backend_test.cc +++ b/tests/vr/openxr_backend_test.cc @@ -352,6 +352,54 @@ TEST_F(OpenXrPresentDependencies, InvalidDependenciesDoNotSubmit) { EXPECT_EQ(resets + submits + waits, 0); } +VkSemaphore chained_signal; +VKAPI_ATTR VkResult VKAPI_CALL SubmitChainedPresent(VkQueue queue, + uint32_t count, + const VkSubmitInfo* info, + VkFence fence) { + ++submits; + EXPECT_EQ(queue, expected_queue); + EXPECT_EQ(count, 1u); + EXPECT_EQ(fence, VK_NULL_HANDLE); + EXPECT_EQ(info->commandBufferCount, 0u); + EXPECT_EQ(info->waitSemaphoreCount, 2u); + for (unsigned i = 0; i < 2; ++i) { + EXPECT_EQ(info->pWaitSemaphores[i], expected_semaphores[i]); + EXPECT_EQ(info->pWaitDstStageMask[i], VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); + } + EXPECT_EQ(info->signalSemaphoreCount, 1u); + EXPECT_EQ(info->pSignalSemaphores[0], chained_signal); + return submit_result; +} +class OpenXrChainedPresent : public OpenXrPresentDependencies { + protected: + VkResult Chain() { + chained_signal = Access::Handle(34); + return internal::ChainPresentDependencies(Access::Handle(2), + expected_queue, info, + chained_signal, + SubmitChainedPresent); + } +}; +TEST_F(OpenXrChainedPresent, HandsDependenciesToTheGpuWithoutWaiting) { + EXPECT_EQ(Chain(), VK_SUCCESS); + EXPECT_EQ(submits, 1); + EXPECT_EQ(resets + waits, 0); +} +TEST_F(OpenXrChainedPresent, FailedSubmitLeavesThePresentToTheGuest) { + submit_result = VK_ERROR_OUT_OF_HOST_MEMORY; + EXPECT_EQ(Chain(), VK_NOT_READY); + submit_result = VK_ERROR_DEVICE_LOST; + EXPECT_EQ(Chain(), VK_ERROR_DEVICE_LOST); +} +TEST_F(OpenXrChainedPresent, NothingToChainDoesNotSubmit) { + info.waitSemaphoreCount = 0; + EXPECT_EQ(Chain(), VK_NOT_READY); + info.waitSemaphoreCount = 2; + info.pWaitSemaphores = nullptr; + EXPECT_EQ(Chain(), VK_NOT_READY); + EXPECT_EQ(submits, 0); +} TEST(OpenXrPresentQueue, RoutesSameAndSplitFamilyPresentationToGraphicsQueue) { OpenXrBackend backend; Access::Configure(backend); From df025d2eb7cf16065d0066eec815bfac45ba23f1 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:58:26 -0500 Subject: [PATCH 53/58] fix(vr): publish the shared-memory mirror a frame later instead of waiting With the eye copies no longer waited for, MOCKTAIL_VR_MIRROR_SHM made the CPU wait for the whole frame before publishing, which cut the frame rate from 117 to 82 fps at 120 Hz. Publish each readback at the next copy, when the GPU has finished it. The external viewer sees each frame one frame later. --- include/mocktail/vr/openxr_backend.h | 14 +++++++++ src/vr/openxr_backend.cc | 43 ++++++++++++++++------------ 2 files changed, 39 insertions(+), 18 deletions(-) diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index 6408ffa5..c3df8170 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -350,8 +350,21 @@ class OpenXrBackend final { VkCommandBuffer command = VK_NULL_HANDLE; VkFence fence = VK_NULL_HANDLE; bool pending = false; + std::uint64_t serial = 0; // bumped per submission std::vector images; }; + // The shared-memory mirror's readback from the previous frame copy. It is + // published at the next copy, when the GPU has long finished it, so neither + // the headset nor the mirror makes the CPU wait for the frame. + struct MirrorReadback { + AsyncSubmission* slot = nullptr; + std::uint64_t serial = 0; + std::uint64_t frame = 0; + VkFormat format = VK_FORMAT_UNDEFINED; + std::uint32_t width = 0; + std::uint32_t height = 0; + }; + void PublishMirrorReadbackLocked(); // The next slot, recorded-ready, or nullptr when its previous submission // did not finish in time (the device is hung or lost). AsyncSubmission* NextAsyncSubmissionLocked(); @@ -454,6 +467,7 @@ class OpenXrBackend final { static constexpr std::size_t kAsyncSubmissions = 6; std::array async_{}; std::size_t next_async_ = 0; + MirrorReadback mirror_readback_{}; // The mirror's signal for this present, which PrepareHostPresent passes on. VkSemaphore mirror_ready_ = VK_NULL_HANDLE; // Semaphores of destroyed desktop swapchains. A present may still wait on diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index 9b5f8a05..f2f118c2 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -3262,6 +3262,7 @@ bool OpenXrBackend::EnsureCopyResources(VkDevice device, void OpenXrBackend::DestroyCopyResources() { mirror_ready_ = VK_NULL_HANDLE; + mirror_readback_ = {}; next_async_ = 0; if (vk_ == nullptr || vk_device_ == VK_NULL_HANDLE) { copy_pool_ = VK_NULL_HANDLE; @@ -3319,6 +3320,7 @@ VkResult OpenXrBackend::SubmitAsyncLocked(VkQueue queue, AsyncSubmission *slot, slot->fence); } perf::ProcessCollector().Count(perf::Counter::kSubmits); + ++slot->serial; slot->pending = result == VK_SUCCESS; if (!slot->pending) slot->images.clear(); @@ -3406,6 +3408,8 @@ bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { const VkImage destination[2] = { eye_swapchains_[0].images[eye_swapchains_[0].acquired_index], eye_swapchains_[1].images[eye_swapchains_[1].acquired_index]}; + // Publish the last readback before its staging is resized or overwritten. + PublishMirrorReadbackLocked(); EnsureMirrorForExtent(eyes_[0].width, eyes_[0].height); AsyncSubmission *const slot = NextAsyncSubmissionLocked(); @@ -3538,27 +3542,30 @@ bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { if (SubmitAsyncLocked(queue, slot, submit) != VK_SUCCESS) { return false; } - if (mirror_.enabled()) { - // The shared-memory readback must land before it is published. - VkResult fence_result = VK_SUCCESS; - { - const ScopedStageTimer fence_timer(perf::Stage::kCopyFenceWait); - fence_result = vk_->WaitForFences(vk_device_, 1, &slot->fence, VK_TRUE, - 2'000'000'000ULL); - } + if (mirror_.enabled()) + mirror_readback_ = {slot, slot->serial, published_pose_.frame, + eyes_[0].format, eyes_[0].width, eyes_[0].height}; + return true; +} + +void OpenXrBackend::PublishMirrorReadbackLocked() { + const MirrorReadback readback = std::exchange(mirror_readback_, {}); + if (readback.slot == nullptr || !mirror_.enabled()) + return; + // A reused slot was waited for before reuse, so its readback landed. + if (readback.slot->serial == readback.serial && readback.slot->pending) { + const ScopedStageTimer fence_timer(perf::Stage::kCopyFenceWait); perf::ProcessCollector().Count(perf::Counter::kFenceWaits); - if (fence_result != VK_SUCCESS) { - (void)IdleHostDevice(vk_->DeviceWaitIdle, vk_device_); - throw std::runtime_error("XR copy fence wait failed"); + if (vk_->WaitForFences(vk_device_, 1, &readback.slot->fence, VK_TRUE, + 2'000'000'000ULL) != VK_SUCCESS) { + perf::ProcessCollector().Count(perf::Counter::kMirrorDrops); + return; } - slot->pending = false; - slot->images.clear(); - const ScopedStageTimer publish_timer(perf::Stage::kMirrorPublish); - mirror_.Publish(published_pose_.frame, eyes_[0].format, eyes_[0].width, - eyes_[0].height, perf::NowNs()); - perf::ProcessCollector().Count(perf::Counter::kMirrorPairs); } - return true; + const ScopedStageTimer publish_timer(perf::Stage::kMirrorPublish); + mirror_.Publish(readback.frame, readback.format, readback.width, + readback.height, perf::NowNs()); + perf::ProcessCollector().Count(perf::Counter::kMirrorPairs); } void OpenXrBackend::EnsureMirrorForExtent(std::uint32_t width, From 35033454b0c149ff88bae9bdb3c4fa6646ce4999 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Thu, 15 Oct 2026 17:21:09 -0500 Subject: [PATCH 54/58] docs(vr): note the frame-stall fix and the new perf stages --- CHANGELOG.md | 8 ++++++++ docs/VR.md | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcd2edfa..2c4642a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,14 @@ Release notes on GitHub are taken from the matching section here. - Nightcap answers Roblox's Java calls with less work per call, and the audio thread no longer allocates memory for each block of sound it plays. +### Changed + +- VR on Vulkan no longer makes the CPU wait for the GPU each frame. The eye copies, the desktop mirror and + the `MOCKTAIL_VR_MIRROR_SHM` readback are ordered on the GPU, so Roblox prepares the next frame while the + last one renders. On a simulated 120 Hz headset this removed 2 to 5 ms of waiting per frame and steadied + the frame rate. The shared-memory mirror now shows each frame one frame later. +- `MOCKTAIL_VR_PERF` also reports `desktop_mirror`, `present_prepare` and `present_interval`. + ## [0.3.4] - 2026-10-11 ### Added diff --git a/docs/VR.md b/docs/VR.md index 89ded1a8..9aeecae2 100644 --- a/docs/VR.md +++ b/docs/VR.md @@ -46,7 +46,7 @@ leaves the text box. Set `MOCKTAIL_VR_KEYBOARD=0` to turn the keyboard off. | Variable | Default | Effect | | --- | --- | --- | -| `MOCKTAIL_VR_PERF` | unset | `1` logs per-stage frame timings and counters once a second. | +| `MOCKTAIL_VR_PERF` | unset | `1` logs per-stage frame timings and counters once a second. `present_interval` is the frame time; `desktop_mirror`, `copy_record` and `copy_fence_wait` show time the CPU spent on VR copies. | | `MOCKTAIL_VR_MEASURE_START_NS`, `MOCKTAIL_VR_MEASURE_END_NS` | unset | Limit `MOCKTAIL_VR_PERF` to a `CLOCK_MONOTONIC` window. | | `MOCKTAIL_VR_POSE_LOG_INTERVAL` | `240` | Frames between head-pose log lines. | From ea248b1ecfc13ab640e1edcfce2e0aeae514d823 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:37:32 -0500 Subject: [PATCH 55/58] feat(vr): add vr.mode with on, off and auto vr.mode in config.yaml picks how Nightcap starts: off on the desktop, on in VR, auto in VR only when this user's SteamVR is running and its OpenXR runtime reports a headset. auto never starts SteamVR, and asks its runtime from a short-lived copy of the process so the runtime's libraries do not load before the VR backend arms. vr.enabled still works as on/off, and --vr or --no-vr still override the file for one launch. --- cmake/MocktailVR.cmake | 1 + config/mocktail.example.yaml | 17 ++-- include/mocktail/vr/headset_detection.h | 35 ++++++++ include/mocktail/vr/openxr_backend.h | 2 + include/runtime/runtime_config.h | 8 ++ src/main.cc | 36 ++++++++- src/runtime/runtime_config.cc | 28 ++++++- src/runtime/runtime_config_bootstrap.cc | 17 ++-- src/runtime/runtime_config_file.cc | 23 +++++- src/vr/headset_detection.cc | 101 ++++++++++++++++++++++++ src/vr/openxr_backend_mode.cc | 80 ++++++++++++------- tests/runtime_config_file_test.cc | 42 ++++++++++ tests/runtime_config_test.cc | 20 +++++ tests/vr/runtime_detection_test.cc | 66 ++++++++++++++++ 14 files changed, 423 insertions(+), 53 deletions(-) create mode 100644 include/mocktail/vr/headset_detection.h create mode 100644 src/vr/headset_detection.cc diff --git a/cmake/MocktailVR.cmake b/cmake/MocktailVR.cmake index 7c76c850..67fb47b9 100644 --- a/cmake/MocktailVR.cmake +++ b/cmake/MocktailVR.cmake @@ -27,6 +27,7 @@ endif() add_library(mocktail_vr STATIC src/vr/diagnostics.cc + src/vr/headset_detection.cc src/vr/openxr_backend_mode.cc src/vr/vr_perf.cc src/vr/vr_mirror.cc diff --git a/config/mocktail.example.yaml b/config/mocktail.example.yaml index 66b1b790..37919832 100644 --- a/config/mocktail.example.yaml +++ b/config/mocktail.example.yaml @@ -53,15 +53,14 @@ graphics: # vsync: off # vr: - # Boolean (default: false). Same as launching with --vr. - # Experimental VR for verified builds 2998 and 3092. Mocktail initializes - # the guest DebugDeviceVR eye resources (two - # distinct 500x500 eye framebuffers), the Roblox renderer draws both eyes - # every frame, and the frames are submitted to the active OpenXR runtime as - # a stereo projection layer with head pose (runtime-tracked, or scripted for - # desktop testing). Controllers use the native VR bridge. Standalone OpenXR - # test scene: mocktail-vr-probe --scene. Use --no-vr to launch normally. - # enabled: false + # String (default: off): experimental VR through OpenXR on supported Roblox + # builds. off always starts on the desktop. on always starts in VR, like + # --vr. auto starts in VR when SteamVR is already running with a connected + # headset (ALVR included), and on the desktop otherwise. auto never starts + # SteamVR. --vr and --no-vr override this for one launch. The older + # enabled: true/false still works as on/off; set one or the other. + # Standalone OpenXR test scene: mocktail-vr-probe --scene. + # mode: off # Optional string or integer: frame-rate limit used instead of # graphics.frame_rate_limit while VR is active. Same values: -1 (Roblox's # own cap, usually 60), display, unlimited, or any positive integer. The diff --git a/include/mocktail/vr/headset_detection.h b/include/mocktail/vr/headset_detection.h new file mode 100644 index 00000000..d295b14b --- /dev/null +++ b/include/mocktail/vr/headset_detection.h @@ -0,0 +1,35 @@ +#ifndef MOCKTAIL_VR_HEADSET_DETECTION_H_ +#define MOCKTAIL_VR_HEADSET_DETECTION_H_ + +#include +#include + +namespace mocktail::vr { + +// Set on the child process that answers DetectSteamVrHeadset. main() checks +// it first and returns RunHeadsetProbeChild() without starting Roblox. +inline constexpr const char* kHeadsetProbeEnvironment = + "MOCKTAIL_VR_HEADSET_PROBE"; + +struct HeadsetDetection { + bool headset = false; + std::string reason; +}; + +// vr.mode: auto. True only when this user's SteamVR is running and its OpenXR +// runtime reports a head-mounted display. SteamVR is never started. The +// runtime is asked from a child process (`self_executable` with +// kHeadsetProbeEnvironment set) so its libraries never load into this one +// before the VR backend arms. +HeadsetDetection DetectSteamVrHeadset( + const std::string& self_executable, + std::chrono::milliseconds timeout = std::chrono::seconds(5), + const std::string& proc_root = "/proc"); + +// The child side: probes the OpenXR runtime in XR_RUNTIME_JSON and returns +// the probe's exit code (0 when a headset is available). +int RunHeadsetProbeChild(); + +} // namespace mocktail::vr + +#endif // MOCKTAIL_VR_HEADSET_DETECTION_H_ diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index c3df8170..2c34e5fe 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -76,6 +76,8 @@ VrRuntimeDiscovery DiscoverVrRuntime( const std::string& config_home, const std::string& config_dirs, const std::string& runtime_dir, const std::string& proc_root = "/proc"); std::string VrRuntimeUnavailableHint(bool user_selected, const std::string& manifest); +// SteamVR's OpenXR manifest when this user's vrserver is running, else empty. +std::string RunningSteamVrManifest(const std::string& proc_root = "/proc"); VrBackendMode ResolveVrBackendMode(bool vr_enabled); const char* VrBackendModeName(VrBackendMode mode); diff --git a/include/runtime/runtime_config.h b/include/runtime/runtime_config.h index 9be1877a..a38515ca 100644 --- a/include/runtime/runtime_config.h +++ b/include/runtime/runtime_config.h @@ -20,6 +20,11 @@ namespace runtime { // must not try to reach an OpenXR runtime on a normal desktop launch. inline constexpr bool kDefaultVrEnabled = false; +// vr.mode: on and off force VR or the desktop. auto starts in VR only when +// SteamVR is running with a headset, which main() checks before launch. +enum class VrMode { kOff, kOn, kAuto }; +const char* VrModeName(VrMode mode); + enum class GraphicsBackend { kAuto, kSystem, @@ -101,6 +106,8 @@ class RuntimeConfig { bool headless() const { return headless_; } bool vr_enabled() const { return vr_enabled_; } + // MOCKTAIL_VR_ENABLED (--vr, --no-vr) wins over MOCKTAIL_VR_MODE. + VrMode vr_mode() const { return vr_mode_; } bool vr_valid() const { return vr_valid_; } const std::filesystem::path& roblox_library_path() const { return roblox_library_path_; @@ -174,6 +181,7 @@ class RuntimeConfig { private: bool headless_ = false; bool vr_enabled_ = kDefaultVrEnabled; + VrMode vr_mode_ = kDefaultVrEnabled ? VrMode::kOn : VrMode::kOff; bool vr_valid_ = true; std::filesystem::path roblox_library_path_ = "rbx_bin/libroblox.so"; GraphicsBackend graphics_backend_ = GraphicsBackend::kVulkan; diff --git a/src/main.cc b/src/main.cc index 5d579ab8..50436fa1 100644 --- a/src/main.cc +++ b/src/main.cc @@ -24,6 +24,7 @@ #include "mocktail/audio/fmod_jni_audio_bridge.h" #include "mocktail/audio/webrtc_jni_audio_bridge.h" #include "mocktail/audio/roblox_output_device_bridge.h" +#include "mocktail/vr/headset_detection.h" #include "mocktail/vr/openxr_backend.h" #include "mocktail/vr/openxr_probe.h" #include "mocktail/vr/roblox_vr_device_bridge.h" @@ -251,6 +252,11 @@ void ConfigureHostDriverEnvironment() { int main(int argc, char* argv[]) { ConfigureHostDriverEnvironment(); + // vr.mode: auto asks SteamVR for a headset from a copy of this process. + if (const char* probe = std::getenv(mocktail::vr::kHeadsetProbeEnvironment); + probe != nullptr && std::string_view(probe) == "1") { + return mocktail::vr::RunHeadsetProbeChild(); + } const auto process_started_at = std::chrono::system_clock::now(); mocktail::runtime::CommandLineParseResult command_line = mocktail::runtime::ParseCommandLine(argc, argv); @@ -409,10 +415,38 @@ int main(int argc, char* argv[]) { } return EXIT_FAILURE; } + if (command_line.options.mode == mocktail::runtime::CommandMode::kRun && + runtime_config.config.vr_mode() == mocktail::runtime::VrMode::kAuto) { + std::error_code self_error; + const std::filesystem::path self = + std::filesystem::read_symlink("/proc/self/exe", self_error); + const mocktail::vr::HeadsetDetection detection = + mocktail::vr::DetectSteamVrHeadset(self_error ? std::string() + : self.string()); + std::cerr << " [vr] vr.mode is auto: " << detection.reason + << (detection.headset ? "; starting in VR\n" + : "; starting on the desktop\n"); + if (detection.headset) { + // A detected headset is not an explicit --vr: a Roblox build without + // VR support still starts on the desktop. + if (setenv("MOCKTAIL_VR_ENABLED", "1", 1) != 0) { + std::cerr << "[FATAL] Cannot enable VR for the detected headset\n"; + return EXIT_FAILURE; + } + runtime_config = mocktail::runtime::LoadRuntimeConfig( + environment, paths.config_file()); + if (!runtime_config) { + std::cerr << "[FATAL] Cannot load " << paths.config_file() << ": " + << runtime_config.error << '\n'; + return EXIT_FAILURE; + } + } + } if (runtime_config.config.vr_enabled() && !mocktail::vr::IsOpenXrSupportCompiled()) { std::cerr << " [vr] This build does not include OpenXR support, but " - "vr.enabled is set; disable it in " << paths.config_file() + "VR is on; set vr.mode to off or auto in " + << paths.config_file() << ".\n"; return EXIT_FAILURE; } diff --git a/src/runtime/runtime_config.cc b/src/runtime/runtime_config.cc index a9c4c638..1412bbf7 100644 --- a/src/runtime/runtime_config.cc +++ b/src/runtime/runtime_config.cc @@ -173,11 +173,35 @@ std::string BuildNetworkProxyUrl(const NetworkProxyConfig& proxy) { (ipv6 ? "]" : "") + ":" + std::to_string(proxy.port); } +const char* VrModeName(VrMode mode) { + switch (mode) { + case VrMode::kOn: + return "on"; + case VrMode::kAuto: + return "auto"; + case VrMode::kOff: + break; + } + return "off"; +} + RuntimeConfig RuntimeConfig::FromEnvironment(const Environment& environment) { RuntimeConfig config; config.headless_ = LegacyEnabled(environment, "MOCKTAIL_HEADLESS"); - config.vr_enabled_ = - ReadBoolean(environment, "MOCKTAIL_VR_ENABLED", kDefaultVrEnabled, &config.vr_valid_); + if (environment.Get("MOCKTAIL_VR_ENABLED").has_value()) { + config.vr_enabled_ = ReadBoolean(environment, "MOCKTAIL_VR_ENABLED", + kDefaultVrEnabled, &config.vr_valid_); + config.vr_mode_ = config.vr_enabled_ ? VrMode::kOn : VrMode::kOff; + } else if (const auto mode = environment.Get("MOCKTAIL_VR_MODE"); + mode.has_value()) { + if (*mode == "auto") { + config.vr_mode_ = VrMode::kAuto; + } else { + config.vr_enabled_ = ReadBoolean(environment, "MOCKTAIL_VR_MODE", + kDefaultVrEnabled, &config.vr_valid_); + config.vr_mode_ = config.vr_enabled_ ? VrMode::kOn : VrMode::kOff; + } + } config.roblox_library_path_ = environment.GetOr( "ROBLOX_LIB_PATH", config.roblox_library_path_.string()); config.graphics_backend_name_ = environment.GetOr( diff --git a/src/runtime/runtime_config_bootstrap.cc b/src/runtime/runtime_config_bootstrap.cc index 21726ef3..5c670406 100644 --- a/src/runtime/runtime_config_bootstrap.cc +++ b/src/runtime/runtime_config_bootstrap.cc @@ -72,15 +72,14 @@ device: pc-windows-11 # vsync: off # vr: - # Boolean (default: false). Same as launching with --vr. - # Experimental VR for verified builds 2998 and 3092. Mocktail initializes - # the guest DebugDeviceVR eye resources (two - # distinct 500x500 eye framebuffers), the Roblox renderer draws both eyes - # every frame, and the frames are submitted to the active OpenXR runtime as - # a stereo projection layer with head pose (runtime-tracked, or scripted for - # desktop testing). Controllers use the native VR bridge. Standalone OpenXR - # test scene: mocktail-vr-probe --scene. Use --no-vr to launch normally. - # enabled: false + # String (default: off): experimental VR through OpenXR on supported Roblox + # builds. off always starts on the desktop. on always starts in VR, like + # --vr. auto starts in VR when SteamVR is already running with a connected + # headset (ALVR included), and on the desktop otherwise. auto never starts + # SteamVR. --vr and --no-vr override this for one launch. The older + # enabled: true/false still works as on/off; set one or the other. + # Standalone OpenXR test scene: mocktail-vr-probe --scene. + # mode: off # Optional string or integer: frame-rate limit used instead of # graphics.frame_rate_limit while VR is active. Same values: -1 (Roblox's # own cap, usually 60), display, unlimited, or any positive integer. The diff --git a/src/runtime/runtime_config_file.cc b/src/runtime/runtime_config_file.cc index f8638236..4b332694 100644 --- a/src/runtime/runtime_config_file.cc +++ b/src/runtime/runtime_config_file.cc @@ -169,6 +169,7 @@ bool ValidateAndMap(const ValueMap& yaml, ValueMap* environment, "runtime.headless", "runtime.roblox_library", "vr.enabled", + "vr.mode", "vr.frame_rate_limit", "appearance.theme", "graphics.backend", @@ -320,13 +321,28 @@ bool ValidateAndMap(const ValueMap& yaml, ValueMap* environment, } (*environment)["MOCKTAIL_THEME"] = *theme; } - if (const auto enabled = value("vr.enabled"); enabled.has_value()) { + // The file only sets MOCKTAIL_VR_MODE, so --vr and --no-vr + // (MOCKTAIL_VR_ENABLED) always win over it. + const auto enabled = value("vr.enabled"); + const auto mode = value("vr.mode"); + if (enabled.has_value() && mode.has_value()) { + *error = "set either vr.mode or vr.enabled, not both"; + return false; + } + if (enabled.has_value()) { bool parsed = false; if (!ParseBoolean(*enabled, &parsed)) { *error = "vr.enabled must be true or false"; return false; } - (*environment)["MOCKTAIL_VR_ENABLED"] = parsed ? "1" : "0"; + (*environment)["MOCKTAIL_VR_MODE"] = parsed ? "on" : "off"; + } + if (mode.has_value()) { + if (*mode != "on" && *mode != "off" && *mode != "auto") { + *error = "vr.mode must be on, off, or auto"; + return false; + } + (*environment)["MOCKTAIL_VR_MODE"] = *mode; } if (const auto frame_rate = value("vr.frame_rate_limit"); frame_rate.has_value()) { @@ -803,7 +819,8 @@ RuntimeConfigLoadResult LoadRuntimeConfig( result.error = "VR frame-rate policy is invalid"; } else if (!result.config.vr_valid()) { result.error = "VR configuration is invalid: MOCKTAIL_VR_ENABLED must be " - "1/0, true/false, or on/off"; + "1/0, true/false, or on/off, and MOCKTAIL_VR_MODE on, off, " + "or auto"; } else if (!result.config.device_profile_valid()) { result.error = "device profile is invalid"; } else if (!result.config.input_capabilities().touch_enabled && diff --git a/src/vr/headset_detection.cc b/src/vr/headset_detection.cc new file mode 100644 index 00000000..0a702830 --- /dev/null +++ b/src/vr/headset_detection.cc @@ -0,0 +1,101 @@ +#include "mocktail/vr/headset_detection.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "mocktail/vr/openxr_backend.h" +#include "mocktail/vr/openxr_probe.h" + +extern char** environ; + +namespace mocktail::vr { + +HeadsetDetection DetectSteamVrHeadset(const std::string& self_executable, + std::chrono::milliseconds timeout, + const std::string& proc_root) { + if (!IsOpenXrSupportCompiled()) { + return {false, "this build has no OpenXR support"}; + } + const std::string manifest = RunningSteamVrManifest(proc_root); + if (manifest.empty()) { + return {false, "SteamVR is not running"}; + } + std::error_code error; + if (!std::filesystem::is_regular_file(manifest, error)) { + return {false, "SteamVR's OpenXR runtime is missing at " + manifest}; + } + + // The child inherits this environment with the runtime and probe mode set. + const std::string runtime = "XR_RUNTIME_JSON=" + manifest; + const std::string probe = std::string(kHeadsetProbeEnvironment) + "=1"; + std::vector environment; + for (char** entry = environ; entry != nullptr && *entry != nullptr; ++entry) { + if (std::strncmp(*entry, "XR_RUNTIME_JSON=", 16) != 0 && + std::strncmp(*entry, probe.c_str(), probe.size() - 1) != 0) { + environment.push_back(*entry); + } + } + environment.push_back(const_cast(runtime.c_str())); + environment.push_back(const_cast(probe.c_str())); + environment.push_back(nullptr); + std::vector arguments = {const_cast(self_executable.c_str()), + nullptr}; + + posix_spawn_file_actions_t actions; + posix_spawn_file_actions_init(&actions); + posix_spawn_file_actions_addopen(&actions, STDOUT_FILENO, "/dev/null", + O_WRONLY, 0); + posix_spawn_file_actions_addopen(&actions, STDERR_FILENO, "/dev/null", + O_WRONLY, 0); + pid_t child = -1; + const int spawned = + posix_spawn(&child, self_executable.c_str(), &actions, nullptr, + arguments.data(), environment.data()); + posix_spawn_file_actions_destroy(&actions); + if (spawned != 0) { + return {false, std::string("cannot start the headset check: ") + + std::strerror(spawned)}; + } + + const auto deadline = std::chrono::steady_clock::now() + timeout; + int status = 0; + while (true) { + const pid_t done = waitpid(child, &status, WNOHANG); + if (done == child) break; + if (done < 0) { + return {false, "the headset check was lost"}; + } + if (std::chrono::steady_clock::now() >= deadline) { + kill(child, SIGKILL); + (void)waitpid(child, &status, 0); + return {false, "SteamVR did not answer the headset check in time"}; + } + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + if (!WIFEXITED(status)) { + return {false, "the headset check crashed"}; + } + const int code = WEXITSTATUS(status); + if (code == ProbeExitCode(ProbeStatus::kReady)) { + return {true, "SteamVR reports a connected headset"}; + } + if (code == ProbeExitCode(ProbeStatus::kHeadsetUnavailable)) { + return {false, "SteamVR is running without a connected headset"}; + } + if (code == ProbeExitCode(ProbeStatus::kRuntimeUnavailable)) { + return {false, "SteamVR's OpenXR runtime is unavailable"}; + } + return {false, "SteamVR's OpenXR runtime failed the headset check"}; +} + +int RunHeadsetProbeChild() { return ProbeExitCode(ProbeOpenXr().status); } + +} // namespace mocktail::vr diff --git a/src/vr/openxr_backend_mode.cc b/src/vr/openxr_backend_mode.cc index 007a831b..1175b25f 100644 --- a/src/vr/openxr_backend_mode.cc +++ b/src/vr/openxr_backend_mode.cc @@ -82,6 +82,55 @@ std::vector VrRuntimeManifestCandidates( return candidates; } +namespace { +// Restrict process discovery to this user. In particular, never select a +// root server for a desktop client or another user's SteamVR installation. +std::vector> +UserVrServers(const std::string &proc_root) { + std::vector> servers; + std::error_code error; + std::filesystem::directory_iterator processes(proc_root, error); + for (; !error && processes != std::filesystem::directory_iterator(); + processes.increment(error)) { + const auto name = processes->path().filename().string(); + if (name.empty() || !std::all_of(name.begin(), name.end(), [](char c) { + return c >= '0' && c <= '9'; + })) + continue; + struct stat metadata{}; + if (stat(processes->path().c_str(), &metadata) != 0 || + metadata.st_uid != getuid()) + continue; + std::ifstream comm_file(processes->path() / "comm"); + std::string comm; + std::getline(comm_file, comm); + if (comm != "vrserver" && comm != "wivrn-server") + continue; + std::error_code exe_error; + const auto executable = + std::filesystem::read_symlink(processes->path() / "exe", exe_error); + if (exe_error || !executable.is_absolute() || executable.filename() != comm) + continue; + servers.emplace_back(comm, executable); + } + return servers; +} + +// vrserver lives in /bin/linux64/. +std::string SteamVrManifestForServer(const std::filesystem::path &vrserver) { + return (vrserver.parent_path().parent_path().parent_path() / + "steamxr_linux64.json") + .string(); +} +} // namespace + +std::string RunningSteamVrManifest(const std::string &proc_root) { + for (const auto &[comm, executable] : UserVrServers(proc_root)) + if (comm == "vrserver") + return SteamVrManifestForServer(executable); + return {}; +} + VrRuntimeDiscovery DiscoverVrRuntime(const std::string &preference, const std::string &home, const std::string &config_home, @@ -118,37 +167,10 @@ VrRuntimeDiscovery DiscoverVrRuntime(const std::string &preference, } } } - // Restrict process discovery to this user. In particular, never select a - // root server for a desktop client or another user's SteamVR installation. - std::error_code error; - std::filesystem::directory_iterator processes(proc_root, error); - for (; !error && processes != std::filesystem::directory_iterator(); - processes.increment(error)) { - const auto name = processes->path().filename().string(); - if (name.empty() || !std::all_of(name.begin(), name.end(), [](char c) { - return c >= '0' && c <= '9'; - })) - continue; - struct stat metadata{}; - if (stat(processes->path().c_str(), &metadata) != 0 || - metadata.st_uid != getuid()) - continue; - std::ifstream comm_file(processes->path() / "comm"); - std::string comm; - std::getline(comm_file, comm); - if (comm != "vrserver" && comm != "wivrn-server") - continue; - std::error_code exe_error; - const auto executable = - std::filesystem::read_symlink(processes->path() / "exe", exe_error); - if (exe_error || !executable.is_absolute() || executable.filename() != comm) - continue; + for (const auto &[comm, executable] : UserVrServers(proc_root)) { if (comm == "vrserver") { steam_running = true; - steam.insert(steam.begin(), - (executable.parent_path().parent_path().parent_path() / - "steamxr_linux64.json") - .string()); + steam.insert(steam.begin(), SteamVrManifestForServer(executable)); } else { // Native developer builds keep this manifest at their build root. auto directory = executable.parent_path(); diff --git a/tests/runtime_config_file_test.cc b/tests/runtime_config_file_test.cc index 26e576f7..e00e3fd8 100644 --- a/tests/runtime_config_file_test.cc +++ b/tests/runtime_config_file_test.cc @@ -134,6 +134,48 @@ TEST(RuntimeConfigFileTest, EXPECT_NE(invalid.error.find("VR configuration"), std::string::npos); } +TEST(RuntimeConfigFileTest, VrModeSelectsOnOffOrAuto) { + TemporaryDirectory temporary; + const auto load = [&](const std::string& vr, MapEnvironment environment = + MapEnvironment()) { + return LoadRuntimeConfig(environment, + temporary.Write("version: 1\nvr:\n" + vr)); + }; + const auto on = load(" mode: on\n"); + ASSERT_TRUE(on) << on.error; + EXPECT_EQ(on.config.vr_mode(), VrMode::kOn); + EXPECT_TRUE(on.config.vr_enabled()); + const auto off = load(" mode: off\n"); + ASSERT_TRUE(off) << off.error; + EXPECT_EQ(off.config.vr_mode(), VrMode::kOff); + EXPECT_FALSE(off.config.vr_enabled()); + // auto stays on the desktop until main() finds a headset. + const auto automatic = load(" mode: auto\n"); + ASSERT_TRUE(automatic) << automatic.error; + EXPECT_EQ(automatic.config.vr_mode(), VrMode::kAuto); + EXPECT_FALSE(automatic.config.vr_enabled()); + // The older key still means on/off. + EXPECT_EQ(load(" enabled: true\n").config.vr_mode(), VrMode::kOn); + EXPECT_EQ(load(" enabled: false\n").config.vr_mode(), VrMode::kOff); + // --vr / --no-vr (and a detected headset) win over the file. + const auto forced = load(" mode: auto\n", + MapEnvironment({{"MOCKTAIL_VR_ENABLED", "1"}})); + ASSERT_TRUE(forced) << forced.error; + EXPECT_EQ(forced.config.vr_mode(), VrMode::kOn); + EXPECT_TRUE(forced.config.vr_enabled()); + const auto desktop = load(" mode: on\n", + MapEnvironment({{"MOCKTAIL_VR_ENABLED", "0"}})); + ASSERT_TRUE(desktop) << desktop.error; + EXPECT_FALSE(desktop.config.vr_enabled()); + + const auto both = load(" mode: auto\n enabled: true\n"); + EXPECT_FALSE(both); + EXPECT_NE(both.error.find("vr.mode"), std::string::npos); + const auto invalid = load(" mode: sometimes\n"); + EXPECT_FALSE(invalid); + EXPECT_NE(invalid.error.find("vr.mode"), std::string::npos); +} + TEST(RuntimeConfigFileTest, VrFrameRateLimitIsSeparateFromDesktopLimit) { TemporaryDirectory temporary; const auto defaults = LoadRuntimeConfig( diff --git a/tests/runtime_config_test.cc b/tests/runtime_config_test.cc index 74ab0abe..880077ee 100644 --- a/tests/runtime_config_test.cc +++ b/tests/runtime_config_test.cc @@ -106,6 +106,26 @@ TEST(RuntimeConfigTest, RejectsMalformedVrEnvironmentInsteadOfEnablingIt) { EXPECT_EQ(config.vr_enabled(), kDefaultVrEnabled); } +TEST(RuntimeConfigTest, ReadsVrModeAndLetsVrEnabledWin) { + EXPECT_EQ(RuntimeConfig::FromEnvironment(MapEnvironment()).vr_mode(), + kDefaultVrEnabled ? VrMode::kOn : VrMode::kOff); + const auto automatic = RuntimeConfig::FromEnvironment( + MapEnvironment({{"MOCKTAIL_VR_MODE", "auto"}})); + EXPECT_TRUE(automatic.vr_valid()); + EXPECT_EQ(automatic.vr_mode(), VrMode::kAuto); + EXPECT_FALSE(automatic.vr_enabled()); + EXPECT_TRUE(RuntimeConfig::FromEnvironment( + MapEnvironment({{"MOCKTAIL_VR_MODE", "on"}})) + .vr_enabled()); + const auto overridden = RuntimeConfig::FromEnvironment(MapEnvironment( + {{"MOCKTAIL_VR_MODE", "auto"}, {"MOCKTAIL_VR_ENABLED", "0"}})); + EXPECT_EQ(overridden.vr_mode(), VrMode::kOff); + EXPECT_FALSE(RuntimeConfig::FromEnvironment( + MapEnvironment({{"MOCKTAIL_VR_MODE", "sometimes"}})) + .vr_valid()); + EXPECT_STREQ(VrModeName(VrMode::kAuto), "auto"); +} + TEST(RuntimeConfigTest, ReadsTypedRuntimeValues) { const MapEnvironment environment({ {"MOCKTAIL_HEADLESS", "1"}, diff --git a/tests/vr/runtime_detection_test.cc b/tests/vr/runtime_detection_test.cc index b5eca654..ec37e032 100644 --- a/tests/vr/runtime_detection_test.cc +++ b/tests/vr/runtime_detection_test.cc @@ -1,4 +1,6 @@ +#include "mocktail/vr/headset_detection.h" #include "mocktail/vr/openxr_backend.h" +#include "mocktail/vr/openxr_probe.h" #include #include #include @@ -137,5 +139,69 @@ TEST_F(VrRuntimeDetectionTest, File(root / "config/openvr/openvrpaths.vrpath", "invalid json"); EXPECT_NE(Selected("alvr"), (install / "steamxr_linux64.json").string()); } +TEST_F(VrRuntimeDetectionTest, RunningSteamVrManifestNeedsThisUsersServer) { + EXPECT_EQ(RunningSteamVrManifest((root / "proc").string()), ""); + const auto manifest = Steam(); + EXPECT_EQ(RunningSteamVrManifest((root / "proc").string()), manifest); +} + +class VrHeadsetDetectionTest : public VrRuntimeDetectionTest { +protected: + // Stands in for the probe child: a script that sees the same environment. + std::string Probe(const std::string &body) { + const auto path = root / "probe.sh"; + File(path, "#!/bin/sh\n" + body + "\n"); + std::filesystem::permissions(path, std::filesystem::perms::owner_all); + return path.string(); + } + HeadsetDetection Detect(const std::string &probe, + std::chrono::milliseconds timeout = + std::chrono::seconds(5)) { + return DetectSteamVrHeadset(probe, timeout, (root / "proc").string()); + } +}; +TEST_F(VrHeadsetDetectionTest, NoSteamVrMeansDesktopWithoutStartingIt) { + if (!IsOpenXrSupportCompiled()) + GTEST_SKIP() << "built without OpenXR"; + const auto marker = root / "started"; + const auto result = Detect(Probe("touch " + marker.string() + "; exit 0")); + EXPECT_FALSE(result.headset); + EXPECT_NE(result.reason.find("not running"), std::string::npos); + EXPECT_FALSE(std::filesystem::exists(marker)); +} +TEST_F(VrHeadsetDetectionTest, AsksSteamVrsRuntimeInAProbeChild) { + if (!IsOpenXrSupportCompiled()) + GTEST_SKIP() << "built without OpenXR"; + const auto manifest = Steam(); + const auto result = Detect(Probe( + "[ \"$XR_RUNTIME_JSON\" = \"" + manifest + "\" ] && [ \"$" + + kHeadsetProbeEnvironment + "\" = 1 ] && exit 0; exit 9")); + EXPECT_TRUE(result.headset) << result.reason; +} +TEST_F(VrHeadsetDetectionTest, MapsProbeOutcomesToDesktop) { + if (!IsOpenXrSupportCompiled()) + GTEST_SKIP() << "built without OpenXR"; + Steam(); + const auto missing = + Detect(Probe("exit " + std::to_string(ProbeExitCode( + ProbeStatus::kHeadsetUnavailable)))); + EXPECT_FALSE(missing.headset); + EXPECT_NE(missing.reason.find("without a connected headset"), + std::string::npos); + EXPECT_FALSE(Detect(Probe("exit 1")).headset); + EXPECT_FALSE(Detect(Probe("kill -9 $$")).headset); + EXPECT_FALSE(Detect(root / "no-such-probe").headset); +} +TEST_F(VrHeadsetDetectionTest, GivesUpOnASteamVrThatDoesNotAnswer) { + if (!IsOpenXrSupportCompiled()) + GTEST_SKIP() << "built without OpenXR"; + Steam(); + const auto started = std::chrono::steady_clock::now(); + const auto result = + Detect(Probe("exec sleep 30"), std::chrono::milliseconds(200)); + EXPECT_FALSE(result.headset); + EXPECT_NE(result.reason.find("in time"), std::string::npos); + EXPECT_LT(std::chrono::steady_clock::now() - started, std::chrono::seconds(5)); +} } // namespace } // namespace mocktail::vr From 043964a5dcf5ba5fd2b8e244a3b5602618685a0f Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:42:40 -0500 Subject: [PATCH 56/58] fix(vr): say why auto found no headset, and document vr.mode Without a headset SteamVR refuses the OpenXR client at xrCreateInstance (VRInitError_Driver_WirelessHmdNotConnected) instead of reporting no system, so say that it usually means no headset is connected. Describe vr.mode in the README, docs/VR.md and the changelog. --- CHANGELOG.md | 3 +++ README.md | 4 +++- docs/VR.md | 16 +++++++++++++--- src/vr/headset_detection.cc | 8 ++++---- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c4642a7..f718f9b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ Release notes on GitHub are taken from the matching section here. - `MOCKTAIL_ASSET_TRACE=1` now also prints the files Roblox asks for and does not get, as `[asset] miss`. - Arch Linux: `nightcap-git` is on the AUR and builds the current `main` branch. Thanks to @dim-ghub, who maintains it. +- `vr.mode` in `config.yaml`: `off` starts on the desktop, `on` starts in VR, and `auto` starts in VR only + when SteamVR is already running with a connected headset. `auto` never starts SteamVR. `vr.enabled` still + works. - The log now names each Roblox Java call that Nightcap has no handler for, once per call, as `[JNI] no handler for .`. This shows what a new Roblox version needs. diff --git a/README.md b/README.md index 307c0d13..ea6c4d58 100644 --- a/README.md +++ b/README.md @@ -258,7 +258,9 @@ cmake -S . -B build -DCMAKE_BUILD_TYPE=Release cmake --build build -j4 ``` -VR stays off at runtime until you pass `--vr` or set `vr.enabled: true`. +VR stays off at runtime until you pass `--vr` or set `vr.mode: on` in +`config.yaml`. `vr.mode: auto` starts in VR only when SteamVR is running with a +headset connected. ### Run diff --git a/docs/VR.md b/docs/VR.md index 9aeecae2..9af2e9d6 100644 --- a/docs/VR.md +++ b/docs/VR.md @@ -1,8 +1,18 @@ # VR -Experimental OpenXR VR is opt-in: launch with `--vr`, or set `vr.enabled: true` -in `config.yaml`. Everything here only applies while VR is active. The -[README](../README.md#experimental-vr) covers building and running. +Experimental OpenXR VR is opt-in: launch with `--vr`, or set `vr.mode` in +`config.yaml`: + +- `off` (default) always starts on the desktop. +- `on` always starts in VR. +- `auto` starts in VR when SteamVR is already running with a connected + headset, ALVR included, and on the desktop otherwise. It never starts + SteamVR, so start SteamVR and connect the headset first. + +`--vr` and `--no-vr` override `vr.mode` for one launch. The older +`vr.enabled: true`/`false` still works as `on`/`off`. Everything here only +applies while VR is active. The [README](../README.md#experimental-vr) covers +building and running. ## Menus and typing diff --git a/src/vr/headset_detection.cc b/src/vr/headset_detection.cc index 0a702830..50ca46c2 100644 --- a/src/vr/headset_detection.cc +++ b/src/vr/headset_detection.cc @@ -90,10 +90,10 @@ HeadsetDetection DetectSteamVrHeadset(const std::string& self_executable, if (code == ProbeExitCode(ProbeStatus::kHeadsetUnavailable)) { return {false, "SteamVR is running without a connected headset"}; } - if (code == ProbeExitCode(ProbeStatus::kRuntimeUnavailable)) { - return {false, "SteamVR's OpenXR runtime is unavailable"}; - } - return {false, "SteamVR's OpenXR runtime failed the headset check"}; + // Without a headset SteamVR refuses the OpenXR client outright + // (VRInitError_Driver_*HmdNotConnected) rather than reporting no system. + return {false, "SteamVR refused the headset check, which usually means no " + "headset is connected"}; } int RunHeadsetProbeChild() { return ProbeExitCode(ProbeOpenXr().status); } From 569df68729d3d2f39f2fd02dc89921aeb9993190 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:55:35 -0500 Subject: [PATCH 57/58] fix(vr): stop trapping the desktop cursor in the window during VR Roblox's mouse lock and the RMB camera drag put the window into relative mouse mode with a grab, which locked the cursor inside it while the headset was in use. In VR the controllers aim and the window only mirrors the left eye, so the pointer is never captured and the system cursor stays visible. --- src/window/window.cc | 3 +++ src/window/window_pointer_capture_owner.cc | 6 ++++- src/window/window_pointer_capture_owner.h | 4 ++++ tests/window_pointer_capture_owner_test.cc | 26 ++++++++++++++++++++++ 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/window/window.cc b/src/window/window.cc index 877d206a..2d578055 100644 --- a/src/window/window.cc +++ b/src/window/window.cc @@ -982,6 +982,9 @@ bool ActivateWindowEventLifecycles() { std::lock_guard lock(g_pointer_capture_owner_mutex); g_pointer_capture_owner = std::make_shared( g_pointer_capture_backend.get()); + // main() exports the resolved VR state before the window exists. + const char* vr = GetEnvNonEmpty("MOCKTAIL_VR_ENABLED"); + g_pointer_capture_owner->SetVrActive(StringEquals(vr, "1")); } fprintf(stderr, " [input] SDL event pump target=%llu Hz raw-motion, no smoothing\n", diff --git a/src/window/window_pointer_capture_owner.cc b/src/window/window_pointer_capture_owner.cc index 511e7f2b..d9836d31 100644 --- a/src/window/window_pointer_capture_owner.cc +++ b/src/window/window_pointer_capture_owner.cc @@ -58,7 +58,7 @@ void WindowPointerCaptureOwner::ClearQuery() { } bool WindowPointerCaptureOwner::Pump(bool text_input_active) { - if (!focused_) { + if (!focused_ || vr_active_) { return Apply(false, true); } MouseLockQueryCallback callback = nullptr; @@ -126,6 +126,10 @@ void WindowPointerCaptureOwner::SetGameSessionActive(bool active) { game_session_active_ = active; } +void WindowPointerCaptureOwner::SetVrActive(bool active) { + vr_active_ = active; +} + bool WindowPointerCaptureOwner::OnShiftKeyPressed(bool text_input_active) { focused_ = true; return Pump(text_input_active); diff --git a/src/window/window_pointer_capture_owner.h b/src/window/window_pointer_capture_owner.h index d087c2d7..e93a8c0b 100644 --- a/src/window/window_pointer_capture_owner.h +++ b/src/window/window_pointer_capture_owner.h @@ -36,6 +36,9 @@ class WindowPointerCaptureOwner final { // is none, and capturing there freezes the cursor over Roblox's own UI. // Starts false: the app surface is what the client shows first. void SetGameSessionActive(bool active); + // In VR the controllers aim and the desktop window only mirrors the left + // eye, so neither Roblox's mouse lock nor RMB drag may trap the pointer. + void SetVrActive(bool active); // Shift can enable Roblox's persistent mouse lock while a transient RMB // camera drag is active. Refresh the native state after dispatching that // key so the subsequent RMB release preserves the intentional lock. @@ -71,6 +74,7 @@ class WindowPointerCaptureOwner final { bool right_button_held_ = false; // Written by the experience composition, read on the SDL thread. std::atomic game_session_active_{false}; + std::atomic vr_active_{false}; bool native_lock_observed_ = false; bool native_lock_was_active_before_right_drag_ = false; bool shift_key_pressed_during_right_drag_ = false; diff --git a/tests/window_pointer_capture_owner_test.cc b/tests/window_pointer_capture_owner_test.cc index 1bb02a34..7d4c5c01 100644 --- a/tests/window_pointer_capture_owner_test.cc +++ b/tests/window_pointer_capture_owner_test.cc @@ -341,6 +341,32 @@ TEST(WindowPointerCaptureOwnerTest, RightDragCapturesOnceTheGameSurfaceIsLive) { EXPECT_FALSE(owner.captured()); } +TEST(WindowPointerCaptureOwnerTest, VrNeverTrapsThePointer) { + FakeBackend backend; + QueryState query{true, true}; + WindowPointerCaptureOwner owner(&backend); + ASSERT_TRUE(owner.RegisterQuery(Query, &query)); + owner.SetGameSessionActive(true); + ASSERT_TRUE(owner.Pump(false)); + ASSERT_TRUE(owner.captured()); + + // Turning VR on releases a held lock and shows the desktop cursor. + owner.SetVrActive(true); + EXPECT_TRUE(owner.Pump(false)); + EXPECT_FALSE(owner.captured()); + EXPECT_TRUE(owner.cursor_visible()); + // Neither Roblox's mouse lock nor an RMB drag captures it again. + EXPECT_TRUE(owner.OnRightButton(true, false)); + EXPECT_FALSE(owner.captured()); + EXPECT_TRUE(owner.OnShiftKeyPressed(false)); + EXPECT_FALSE(owner.captured()); + EXPECT_TRUE(owner.OnFocusGained(false)); + EXPECT_FALSE(owner.captured()); + EXPECT_TRUE(owner.cursor_visible()); + ASSERT_FALSE(backend.calls.empty()); + EXPECT_FALSE(backend.calls.back().relative_mode); +} + TEST(WindowPointerCaptureOwnerTest, RightDragCapturesUntilButtonRelease) { FakeBackend backend; QueryState query{true, false}; From 753a3f8106e4d7fdafea457601a5ff238e77c2d0 Mon Sep 17 00:00:00 2001 From: dim-ghub <59436063+dim-ghub@users.noreply.github.com> Date: Thu, 15 Oct 2026 20:19:17 -0500 Subject: [PATCH 58/58] feat(vr): anti-alias the headset eyes with FXAA Roblox's VR debug device renders both eyes into single-sample textures, so edges were jagged in the headset. When the eyes go to the runtime's swapchain, an FXAA draw now replaces the plain copy. It samples the eye directly, and re-encodes for sRGB swapchains so the guest's bytes round-trip exactly. Anything it cannot set up falls back to the copy. vr.anti_aliasing (fxaa, the default, or off) controls it. The SPIR-V is generated from shaders/vr by scripts/generate_vr_shaders.py and checked in, so builds need no shader compiler; a test checks it against the sources when glslc is installed. On a Quest 2 through SteamVR at 90 Hz (2244x2352 per eye) frame times did not change. Captured output keeps 97 to 98 percent of pixels byte-identical and the average brightness unchanged, and smooths the edges. --- CHANGELOG.md | 3 + config/mocktail.example.yaml | 5 + docs/VR.md | 1 + include/mocktail/vr/openxr_backend.h | 33 ++ scripts/generate_vr_shaders.py | 79 ++++ shaders/vr/fullscreen.vert | 8 + shaders/vr/fxaa.frag | 59 +++ src/runtime/runtime_config_bootstrap.cc | 5 + src/runtime/runtime_config_file.cc | 9 + src/vr/openxr_backend.cc | 517 +++++++++++++++++++++++- src/vr/vr_shaders_spirv.h | 220 ++++++++++ tests/CMakeLists.txt | 8 + tests/runtime_config_file_test.cc | 16 + 13 files changed, 946 insertions(+), 17 deletions(-) create mode 100755 scripts/generate_vr_shaders.py create mode 100644 shaders/vr/fullscreen.vert create mode 100644 shaders/vr/fxaa.frag create mode 100644 src/vr/vr_shaders_spirv.h diff --git a/CHANGELOG.md b/CHANGELOG.md index f718f9b0..10db5f9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ Release notes on GitHub are taken from the matching section here. - `vr.mode` in `config.yaml`: `off` starts on the desktop, `on` starts in VR, and `auto` starts in VR only when SteamVR is already running with a connected headset. `auto` never starts SteamVR. `vr.enabled` still works. +- Anti-aliasing in VR. Roblox draws the VR eyes without it, so edges were jagged in the headset. FXAA now + smooths each eye on its way to the headset, with no measurable frame-time cost on a Quest 2 at 90 Hz. + `vr.anti_aliasing: off` turns it off. - The log now names each Roblox Java call that Nightcap has no handler for, once per call, as `[JNI] no handler for .`. This shows what a new Roblox version needs. diff --git a/config/mocktail.example.yaml b/config/mocktail.example.yaml index 37919832..8a476c20 100644 --- a/config/mocktail.example.yaml +++ b/config/mocktail.example.yaml @@ -61,6 +61,11 @@ graphics: # enabled: true/false still works as on/off; set one or the other. # Standalone OpenXR test scene: mocktail-vr-probe --scene. # mode: off + # String (default: fxaa): smooth jagged edges in the headset. Roblox draws + # the VR eyes without anti-aliasing; fxaa filters each eye on its way to the + # headset, which costs well under a millisecond. off sends the eyes as + # rendered. + # anti_aliasing: fxaa # Optional string or integer: frame-rate limit used instead of # graphics.frame_rate_limit while VR is active. Same values: -1 (Roblox's # own cap, usually 60), display, unlimited, or any positive integer. The diff --git a/docs/VR.md b/docs/VR.md index 9af2e9d6..3ccc256c 100644 --- a/docs/VR.md +++ b/docs/VR.md @@ -37,6 +37,7 @@ leaves the text box. Set `MOCKTAIL_VR_KEYBOARD=0` to turn the keyboard off. | `MOCKTAIL_VR_RUNTIME` | `auto` | OpenXR runtime: `auto`, `system`, `wivrn`, `steamvr` or `alvr`. `XR_RUNTIME_JSON` overrides it. | | `MOCKTAIL_VR_BACKEND` | `xr` | `native` keeps Roblox's stereo rendering without an OpenXR runtime, for debugging. | | `MOCKTAIL_VR_FRAME_RATE_LIMIT` | `unlimited` | Frame-rate cap in VR, same values as `graphics.frame_rate_limit` (also `vr.frame_rate_limit`). With `unlimited` the headset's refresh rate paces frames. | +| `MOCKTAIL_VR_ANTI_ALIASING` | `fxaa` | Same as `vr.anti_aliasing`. Roblox draws the VR eyes without anti-aliasing; `fxaa` smooths each eye on its way to the headset (Vulkan only), `off` sends them as rendered. | | `MOCKTAIL_VR_RENDER_SCALE` | `1.0` | Scales each axis of the runtime's recommended eye size, `0.25` to `2.0`. | | `MOCKTAIL_VR_UNPACED` | `0` | `1` stops waiting for the headset between frames, to measure the maximum frame rate. Frames past the headset refresh are not shown. Not for playing. | | `MOCKTAIL_VR_DESKTOP_MIRROR` | `1` | `0` stops copying the left eye to the desktop window. | diff --git a/include/mocktail/vr/openxr_backend.h b/include/mocktail/vr/openxr_backend.h index 2c34e5fe..fc3a2445 100644 --- a/include/mocktail/vr/openxr_backend.h +++ b/include/mocktail/vr/openxr_backend.h @@ -345,6 +345,9 @@ class OpenXrBackend final { // readbacks) into one command buffer and submits it once. Only the shared // memory mirror waits for it; the headset path leaves it to the GPU. bool CopyEyesIntoSwapchains(VkQueue queue); + void RecordEyeCopiesLocked(VkCommandBuffer command, const VkImage source[2], + const VkImage destination[2]); + void RecordFxaaEyesLocked(VkCommandBuffer command); // Per-frame GPU work (desktop mirror, eye copies) is submitted without a CPU // wait. A slot is reused only after its fence signalled; its images let a // guest destroy wait for the submissions that still read them. @@ -367,6 +370,35 @@ class OpenXrBackend final { std::uint32_t height = 0; }; void PublishMirrorReadbackLocked(); + // vr.anti_aliasing: fxaa. The eye copy becomes one FXAA draw per eye into + // the runtime's swapchain image. Anything it cannot set up falls back to + // the plain copy, which never loses headset output. + struct FxaaState { + VkFormat target_format = VK_FORMAT_UNDEFINED; + VkShaderModule vertex = VK_NULL_HANDLE; + VkShaderModule fragment = VK_NULL_HANDLE; + VkSampler sampler = VK_NULL_HANDLE; + VkDescriptorSetLayout set_layout = VK_NULL_HANDLE; + VkDescriptorPool pool = VK_NULL_HANDLE; + VkDescriptorSet sets[2] = {}; + VkPipelineLayout layout = VK_NULL_HANDLE; + VkRenderPass render_pass = VK_NULL_HANDLE; + VkPipeline pipeline = VK_NULL_HANDLE; + // The eye image each set samples, and its view. + VkImage sources[2] = {}; + VkImageView source_views[2] = {}; + // Per eye, one view and framebuffer per runtime swapchain image. + std::vector target_views[2]; + std::vector framebuffers[2]; + bool failed = false; + }; + bool FxaaEnabled() const; + // Pipeline, targets and eye bindings for this frame; false means copy. + bool PrepareFxaaLocked(); + bool CreateFxaaPipelineLocked(VkFormat target_format); + void DestroyFxaaTargetsLocked(); + void ForgetFxaaSourceLocked(VkImage image); + void DestroyFxaaLocked(); // The next slot, recorded-ready, or nullptr when its previous submission // did not finish in time (the device is hung or lost). AsyncSubmission* NextAsyncSubmissionLocked(); @@ -470,6 +502,7 @@ class OpenXrBackend final { std::array async_{}; std::size_t next_async_ = 0; MirrorReadback mirror_readback_{}; + FxaaState fxaa_{}; // The mirror's signal for this present, which PrepareHostPresent passes on. VkSemaphore mirror_ready_ = VK_NULL_HANDLE; // Semaphores of destroyed desktop swapchains. A present may still wait on diff --git a/scripts/generate_vr_shaders.py b/scripts/generate_vr_shaders.py new file mode 100755 index 00000000..28d5cf02 --- /dev/null +++ b/scripts/generate_vr_shaders.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Regenerate src/vr/vr_shaders_spirv.h from shaders/vr with glslc. + +The header is checked in so a build never needs a shader compiler. Run this +after editing a shader; VrShaders.SpirvMatchesSources checks it when glslc is +installed. +""" + +import argparse +import pathlib +import struct +import subprocess +import sys +import tempfile + +ROOT = pathlib.Path(__file__).resolve().parent.parent +SHADERS = [ + ("kFullscreenVertexSpirv", "shaders/vr/fullscreen.vert"), + ("kFxaaFragmentSpirv", "shaders/vr/fxaa.frag"), +] +OUTPUT = ROOT / "src/vr/vr_shaders_spirv.h" + + +def compile_words(glslc, source): + with tempfile.TemporaryDirectory() as directory: + output = pathlib.Path(directory) / "shader.spv" + subprocess.run( + [glslc, "-O", "--target-env=vulkan1.0", str(ROOT / source), "-o", + str(output)], + check=True) + data = output.read_bytes() + return struct.unpack("<%dI" % (len(data) // 4), data) + + +def render(glslc): + lines = [ + "// Generated by scripts/generate_vr_shaders.py from shaders/vr. Do not edit.", + "#ifndef MOCKTAIL_VR_VR_SHADERS_SPIRV_H_", + "#define MOCKTAIL_VR_VR_SHADERS_SPIRV_H_", + "", + "#include ", + "", + "namespace mocktail::vr {", + "", + ] + for name, source in SHADERS: + words = compile_words(glslc, source) + lines.append("// %s" % source) + lines.append("inline constexpr std::uint32_t %s[] = {" % name) + for start in range(0, len(words), 6): + chunk = ", ".join("0x%08x" % word for word in words[start:start + 6]) + lines.append(" %s," % chunk) + lines.append("};") + lines.append("") + lines += ["} // namespace mocktail::vr", "", + "#endif // MOCKTAIL_VR_VR_SHADERS_SPIRV_H_", ""] + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--glslc", default="glslc") + parser.add_argument("--check", action="store_true", + help="exit 1 when the header is out of date") + arguments = parser.parse_args() + text = render(arguments.glslc) + if arguments.check: + current = OUTPUT.read_text() if OUTPUT.exists() else "" + if current != text: + print("%s is out of date; run %s" % (OUTPUT, __file__), + file=sys.stderr) + return 1 + return 0 + OUTPUT.write_text(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/shaders/vr/fullscreen.vert b/shaders/vr/fullscreen.vert new file mode 100644 index 00000000..e4a17bce --- /dev/null +++ b/shaders/vr/fullscreen.vert @@ -0,0 +1,8 @@ +#version 450 +// One triangle that covers the viewport; uv spans 0..1 over the target. +layout(location = 0) out vec2 uv; + +void main() { + uv = vec2((gl_VertexIndex << 1) & 2, gl_VertexIndex & 2); + gl_Position = vec4(uv * 2.0 - 1.0, 0.0, 1.0); +} diff --git a/shaders/vr/fxaa.frag b/shaders/vr/fxaa.frag new file mode 100644 index 00000000..60aa3eba --- /dev/null +++ b/shaders/vr/fxaa.frag @@ -0,0 +1,59 @@ +#version 450 +// FXAA for one VR eye, after Timothy Lottes' FXAA (console variant). The eye +// holds sRGB-encoded bytes, so luma is computed on perceptual values, which +// is what FXAA expects. +layout(set = 0, binding = 0) uniform sampler2D source; +layout(push_constant) uniform Push { + vec2 texel; // 1 / source extent + uint linearize; // 1 when the target view re-encodes to sRGB +} push; +layout(location = 0) in vec2 uv; +layout(location = 0) out vec4 color; + +const float kEdgeThreshold = 1.0 / 8.0; +const float kEdgeThresholdMin = 1.0 / 24.0; +const float kReduceMul = 1.0 / 8.0; +const float kReduceMin = 1.0 / 128.0; +const float kSpanMax = 8.0; + +float Luma(vec3 rgb) { return dot(rgb, vec3(0.299, 0.587, 0.114)); } + +vec3 Sample(vec2 at) { return textureLod(source, at, 0.0).rgb; } + +// The target encodes linear values back to sRGB, which restores these bytes. +vec3 Linearize(vec3 c) { + return mix(c / 12.92, pow((c + 0.055) / 1.055, vec3(2.4)), + step(vec3(0.04045), c)); +} + +void main() { + const vec4 center = textureLod(source, uv, 0.0); + const float lumaM = Luma(center.rgb); + const float lumaNW = Luma(Sample(uv + vec2(-1.0, -1.0) * push.texel)); + const float lumaNE = Luma(Sample(uv + vec2(1.0, -1.0) * push.texel)); + const float lumaSW = Luma(Sample(uv + vec2(-1.0, 1.0) * push.texel)); + const float lumaSE = Luma(Sample(uv + vec2(1.0, 1.0) * push.texel)); + const float lumaMin = + min(lumaM, min(min(lumaNW, lumaNE), min(lumaSW, lumaSE))); + const float lumaMax = + max(lumaM, max(max(lumaNW, lumaNE), max(lumaSW, lumaSE))); + + vec3 rgb = center.rgb; + // Flat areas, where nothing aliases, keep their exact pixels. + if (lumaMax - lumaMin >= max(kEdgeThresholdMin, lumaMax * kEdgeThreshold)) { + vec2 dir = vec2(-((lumaNW + lumaNE) - (lumaSW + lumaSE)), + (lumaNW + lumaSW) - (lumaNE + lumaSE)); + const float reduce = max((lumaNW + lumaNE + lumaSW + lumaSE) * + (0.25 * kReduceMul), + kReduceMin); + const float rcpDirMin = 1.0 / (min(abs(dir.x), abs(dir.y)) + reduce); + dir = clamp(dir * rcpDirMin, vec2(-kSpanMax), vec2(kSpanMax)) * push.texel; + const vec3 rgbA = 0.5 * (Sample(uv + dir * (1.0 / 3.0 - 0.5)) + + Sample(uv + dir * (2.0 / 3.0 - 0.5))); + const vec3 rgbB = + rgbA * 0.5 + 0.25 * (Sample(uv + dir * -0.5) + Sample(uv + dir * 0.5)); + const float lumaB = Luma(rgbB); + rgb = (lumaB < lumaMin || lumaB > lumaMax) ? rgbA : rgbB; + } + color = vec4(push.linearize != 0u ? Linearize(rgb) : rgb, center.a); +} diff --git a/src/runtime/runtime_config_bootstrap.cc b/src/runtime/runtime_config_bootstrap.cc index 5c670406..2c6d0ddc 100644 --- a/src/runtime/runtime_config_bootstrap.cc +++ b/src/runtime/runtime_config_bootstrap.cc @@ -80,6 +80,11 @@ device: pc-windows-11 # enabled: true/false still works as on/off; set one or the other. # Standalone OpenXR test scene: mocktail-vr-probe --scene. # mode: off + # String (default: fxaa): smooth jagged edges in the headset. Roblox draws + # the VR eyes without anti-aliasing; fxaa filters each eye on its way to the + # headset, which costs well under a millisecond. off sends the eyes as + # rendered. + # anti_aliasing: fxaa # Optional string or integer: frame-rate limit used instead of # graphics.frame_rate_limit while VR is active. Same values: -1 (Roblox's # own cap, usually 60), display, unlimited, or any positive integer. The diff --git a/src/runtime/runtime_config_file.cc b/src/runtime/runtime_config_file.cc index 4b332694..704a7534 100644 --- a/src/runtime/runtime_config_file.cc +++ b/src/runtime/runtime_config_file.cc @@ -170,6 +170,7 @@ bool ValidateAndMap(const ValueMap& yaml, ValueMap* environment, "runtime.roblox_library", "vr.enabled", "vr.mode", + "vr.anti_aliasing", "vr.frame_rate_limit", "appearance.theme", "graphics.backend", @@ -344,6 +345,14 @@ bool ValidateAndMap(const ValueMap& yaml, ValueMap* environment, } (*environment)["MOCKTAIL_VR_MODE"] = *mode; } + if (const auto anti_aliasing = value("vr.anti_aliasing"); + anti_aliasing.has_value()) { + if (*anti_aliasing != "fxaa" && *anti_aliasing != "off") { + *error = "vr.anti_aliasing must be fxaa or off"; + return false; + } + (*environment)["MOCKTAIL_VR_ANTI_ALIASING"] = *anti_aliasing; + } if (const auto frame_rate = value("vr.frame_rate_limit"); frame_rate.has_value()) { if (!ParseFrameRatePolicy(*frame_rate).valid()) { diff --git a/src/vr/openxr_backend.cc b/src/vr/openxr_backend.cc index f2f118c2..78274c15 100644 --- a/src/vr/openxr_backend.cc +++ b/src/vr/openxr_backend.cc @@ -3,6 +3,7 @@ #include "mocktail/vr/vr_pose_math.h" #include "vr/gles_transport.h" +#include "vr/vr_shaders_spirv.h" #include #include @@ -251,6 +252,33 @@ struct OpenXrBackend::VkProcs { PFN_vkGetPhysicalDeviceQueueFamilyProperties GetPhysicalDeviceQueueFamilyProperties = nullptr; PFN_vkGetPhysicalDeviceProperties GetPhysicalDeviceProperties = nullptr; + // FXAA pass. + PFN_vkCreateShaderModule CreateShaderModule = nullptr; + PFN_vkDestroyShaderModule DestroyShaderModule = nullptr; + PFN_vkCreateRenderPass CreateRenderPass = nullptr; + PFN_vkDestroyRenderPass DestroyRenderPass = nullptr; + PFN_vkCreateFramebuffer CreateFramebuffer = nullptr; + PFN_vkDestroyFramebuffer DestroyFramebuffer = nullptr; + PFN_vkCreatePipelineLayout CreatePipelineLayout = nullptr; + PFN_vkDestroyPipelineLayout DestroyPipelineLayout = nullptr; + PFN_vkCreateGraphicsPipelines CreateGraphicsPipelines = nullptr; + PFN_vkDestroyPipeline DestroyPipeline = nullptr; + PFN_vkCreateDescriptorSetLayout CreateDescriptorSetLayout = nullptr; + PFN_vkDestroyDescriptorSetLayout DestroyDescriptorSetLayout = nullptr; + PFN_vkCreateDescriptorPool CreateDescriptorPool = nullptr; + PFN_vkDestroyDescriptorPool DestroyDescriptorPool = nullptr; + PFN_vkAllocateDescriptorSets AllocateDescriptorSets = nullptr; + PFN_vkUpdateDescriptorSets UpdateDescriptorSets = nullptr; + PFN_vkCreateSampler CreateSampler = nullptr; + PFN_vkDestroySampler DestroySampler = nullptr; + PFN_vkCmdBeginRenderPass CmdBeginRenderPass = nullptr; + PFN_vkCmdEndRenderPass CmdEndRenderPass = nullptr; + PFN_vkCmdBindPipeline CmdBindPipeline = nullptr; + PFN_vkCmdBindDescriptorSets CmdBindDescriptorSets = nullptr; + PFN_vkCmdPushConstants CmdPushConstants = nullptr; + PFN_vkCmdSetViewport CmdSetViewport = nullptr; + PFN_vkCmdSetScissor CmdSetScissor = nullptr; + PFN_vkCmdDraw CmdDraw = nullptr; }; namespace { @@ -847,6 +875,32 @@ void OpenXrBackend::SetupDeviceAfterCreationLocked( MOCKTAIL_BACKEND_VK_FN(QueueSubmit) MOCKTAIL_BACKEND_VK_FN(GetDeviceQueue) MOCKTAIL_BACKEND_VK_FN(DeviceWaitIdle) + MOCKTAIL_BACKEND_VK_FN(CreateShaderModule) + MOCKTAIL_BACKEND_VK_FN(DestroyShaderModule) + MOCKTAIL_BACKEND_VK_FN(CreateRenderPass) + MOCKTAIL_BACKEND_VK_FN(DestroyRenderPass) + MOCKTAIL_BACKEND_VK_FN(CreateFramebuffer) + MOCKTAIL_BACKEND_VK_FN(DestroyFramebuffer) + MOCKTAIL_BACKEND_VK_FN(CreatePipelineLayout) + MOCKTAIL_BACKEND_VK_FN(DestroyPipelineLayout) + MOCKTAIL_BACKEND_VK_FN(CreateGraphicsPipelines) + MOCKTAIL_BACKEND_VK_FN(DestroyPipeline) + MOCKTAIL_BACKEND_VK_FN(CreateDescriptorSetLayout) + MOCKTAIL_BACKEND_VK_FN(DestroyDescriptorSetLayout) + MOCKTAIL_BACKEND_VK_FN(CreateDescriptorPool) + MOCKTAIL_BACKEND_VK_FN(DestroyDescriptorPool) + MOCKTAIL_BACKEND_VK_FN(AllocateDescriptorSets) + MOCKTAIL_BACKEND_VK_FN(UpdateDescriptorSets) + MOCKTAIL_BACKEND_VK_FN(CreateSampler) + MOCKTAIL_BACKEND_VK_FN(DestroySampler) + MOCKTAIL_BACKEND_VK_FN(CmdBeginRenderPass) + MOCKTAIL_BACKEND_VK_FN(CmdEndRenderPass) + MOCKTAIL_BACKEND_VK_FN(CmdBindPipeline) + MOCKTAIL_BACKEND_VK_FN(CmdBindDescriptorSets) + MOCKTAIL_BACKEND_VK_FN(CmdPushConstants) + MOCKTAIL_BACKEND_VK_FN(CmdSetViewport) + MOCKTAIL_BACKEND_VK_FN(CmdSetScissor) + MOCKTAIL_BACKEND_VK_FN(CmdDraw) #undef MOCKTAIL_BACKEND_VK_FN #define MOCKTAIL_BACKEND_VK_INSTANCE_FN(name) \ vk_->name = \ @@ -1386,6 +1440,7 @@ void OpenXrBackend::MatchSwapchainFormatToEyesLocked() { } void OpenXrBackend::DestroySwapchains() { + DestroyFxaaTargetsLocked(); // views of these images for (EyeSwapchain& slot : eye_swapchains_) { slot.images.clear(); slot.gl_images.clear(); @@ -1723,6 +1778,7 @@ void OpenXrBackend::NoteDestroyImage(VkImage image) { std::lock_guard lock(mutex_); // A frame copy or mirror blit may still read it on the GPU. WaitAsyncSubmissionsLocked(image); + ForgetFxaaSourceLocked(image); images_.erase(image); eye_render_queues_.erase(image); eye_render_frames_.erase(image); @@ -3261,6 +3317,7 @@ bool OpenXrBackend::EnsureCopyResources(VkDevice device, } void OpenXrBackend::DestroyCopyResources() { + DestroyFxaaLocked(); // callers idle the device first mirror_ready_ = VK_NULL_HANDLE; mirror_readback_ = {}; next_async_ = 0; @@ -3412,6 +3469,8 @@ bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { PublishMirrorReadbackLocked(); EnsureMirrorForExtent(eyes_[0].width, eyes_[0].height); + // Before taking a slot: rebinding an eye waits for every pending one. + const bool fxaa = PrepareFxaaLocked(); AsyncSubmission *const slot = NextAsyncSubmissionLocked(); if (slot == nullptr) { (void)IdleHostDevice(vk_->DeviceWaitIdle, vk_device_); @@ -3425,6 +3484,35 @@ bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { return false; } + if (fxaa) + RecordFxaaEyesLocked(command); + else + RecordEyeCopiesLocked(command, source, destination); + + if (vk_->EndCommandBuffer(command) != VK_SUCCESS) { + return false; + } + VkSubmitInfo submit{}; + submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit.commandBufferCount = 1; + submit.pCommandBuffers = &command; + // The present dependencies were consumed earlier on this queue, so the + // copy runs after the guest's eye passes without the CPU waiting for them. + // xrReleaseSwapchainImage only needs the copy submitted to this queue. + // NoteDestroyImage waits for this slot before the guest frees a source. + slot->images = {source[0], source[1], destination[0], destination[1]}; + if (SubmitAsyncLocked(queue, slot, submit) != VK_SUCCESS) { + return false; + } + if (mirror_.enabled()) + mirror_readback_ = {slot, slot->serial, published_pose_.frame, + eyes_[0].format, eyes_[0].width, eyes_[0].height}; + return true; +} + +void OpenXrBackend::RecordEyeCopiesLocked(VkCommandBuffer command, + const VkImage source[2], + const VkImage destination[2]) { // Phase 1: transition both sources to TRANSFER_SRC and both swapchain // destinations to TRANSFER_DST in one barrier batch. std::array prepare{}; @@ -3526,28 +3614,419 @@ bool OpenXrBackend::CopyEyesIntoSwapchains(VkQueue queue) { VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, static_cast(finish.size()), finish.data()); +} - if (vk_->EndCommandBuffer(command) != VK_SUCCESS) { +bool OpenXrBackend::FxaaEnabled() const { + static const bool enabled = [] { + const std::string mode = EnvString("MOCKTAIL_VR_ANTI_ALIASING", "fxaa"); + if (mode != "fxaa" && mode != "off") + Log(" [vr-backend] unknown MOCKTAIL_VR_ANTI_ALIASING '%s'; using off\n", + mode.c_str()); + return mode == "fxaa"; + }(); + return enabled && graphics_api_ == VrGraphicsApi::kVulkan; +} + +bool OpenXrBackend::CreateFxaaPipelineLocked(VkFormat target_format) { + const auto module = [&](const std::uint32_t *words, std::size_t bytes, + VkShaderModule *out) { + VkShaderModuleCreateInfo info{}; + info.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO; + info.codeSize = bytes; + info.pCode = words; + return vk_->CreateShaderModule(vk_device_, &info, nullptr, out) == + VK_SUCCESS; + }; + if (fxaa_.vertex == VK_NULL_HANDLE && + !module(kFullscreenVertexSpirv, sizeof(kFullscreenVertexSpirv), + &fxaa_.vertex)) + return false; + if (fxaa_.fragment == VK_NULL_HANDLE && + !module(kFxaaFragmentSpirv, sizeof(kFxaaFragmentSpirv), &fxaa_.fragment)) return false; + if (fxaa_.sampler == VK_NULL_HANDLE) { + VkSamplerCreateInfo info{}; + info.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO; + info.magFilter = info.minFilter = VK_FILTER_LINEAR; + info.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST; + info.addressModeU = info.addressModeV = info.addressModeW = + VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + if (vk_->CreateSampler(vk_device_, &info, nullptr, &fxaa_.sampler) != + VK_SUCCESS) + return false; } - VkSubmitInfo submit{}; - submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; - submit.commandBufferCount = 1; - submit.pCommandBuffers = &command; - // The present dependencies were consumed earlier on this queue, so the - // copy runs after the guest's eye passes without the CPU waiting for them. - // xrReleaseSwapchainImage only needs the copy submitted to this queue. - // NoteDestroyImage waits for this slot before the guest frees a source. - slot->images = {source[0], source[1], destination[0], destination[1]}; - if (SubmitAsyncLocked(queue, slot, submit) != VK_SUCCESS) { + if (fxaa_.set_layout == VK_NULL_HANDLE) { + VkDescriptorSetLayoutBinding binding{}; + binding.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + binding.descriptorCount = 1; + binding.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT; + binding.pImmutableSamplers = &fxaa_.sampler; + VkDescriptorSetLayoutCreateInfo info{}; + info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO; + info.bindingCount = 1; + info.pBindings = &binding; + if (vk_->CreateDescriptorSetLayout(vk_device_, &info, nullptr, + &fxaa_.set_layout) != VK_SUCCESS) + return false; + } + if (fxaa_.pool == VK_NULL_HANDLE) { + VkDescriptorPoolSize size{VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 2}; + VkDescriptorPoolCreateInfo info{}; + info.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO; + info.maxSets = 2; + info.poolSizeCount = 1; + info.pPoolSizes = &size; + if (vk_->CreateDescriptorPool(vk_device_, &info, nullptr, &fxaa_.pool) != + VK_SUCCESS) + return false; + const VkDescriptorSetLayout layouts[2] = {fxaa_.set_layout, + fxaa_.set_layout}; + VkDescriptorSetAllocateInfo allocate{}; + allocate.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO; + allocate.descriptorPool = fxaa_.pool; + allocate.descriptorSetCount = 2; + allocate.pSetLayouts = layouts; + if (vk_->AllocateDescriptorSets(vk_device_, &allocate, fxaa_.sets) != + VK_SUCCESS) + return false; + } + if (fxaa_.layout == VK_NULL_HANDLE) { + VkPushConstantRange range{VK_SHADER_STAGE_FRAGMENT_BIT, 0, + 3 * sizeof(std::uint32_t)}; + VkPipelineLayoutCreateInfo info{}; + info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + info.setLayoutCount = 1; + info.pSetLayouts = &fxaa_.set_layout; + info.pushConstantRangeCount = 1; + info.pPushConstantRanges = ⦥ + if (vk_->CreatePipelineLayout(vk_device_, &info, nullptr, &fxaa_.layout) != + VK_SUCCESS) + return false; + } + if (fxaa_.target_format == target_format && fxaa_.pipeline != VK_NULL_HANDLE) + return true; + if (fxaa_.pipeline != VK_NULL_HANDLE) + vk_->DestroyPipeline(vk_device_, fxaa_.pipeline, nullptr); + if (fxaa_.render_pass != VK_NULL_HANDLE) + vk_->DestroyRenderPass(vk_device_, fxaa_.render_pass, nullptr); + fxaa_.pipeline = VK_NULL_HANDLE; + fxaa_.render_pass = VK_NULL_HANDLE; + fxaa_.target_format = VK_FORMAT_UNDEFINED; + + // The swapchain image arrives (and stays) a color attachment; every pixel + // is overwritten, so its old contents are never loaded. + VkAttachmentDescription attachment{}; + attachment.format = target_format; + attachment.samples = VK_SAMPLE_COUNT_1_BIT; + attachment.loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE; + attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + attachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + attachment.initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + attachment.finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + VkAttachmentReference reference{0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}; + VkSubpassDescription subpass{}; + subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; + subpass.colorAttachmentCount = 1; + subpass.pColorAttachments = &reference; + VkRenderPassCreateInfo pass{}; + pass.sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO; + pass.attachmentCount = 1; + pass.pAttachments = &attachment; + pass.subpassCount = 1; + pass.pSubpasses = &subpass; + if (vk_->CreateRenderPass(vk_device_, &pass, nullptr, &fxaa_.render_pass) != + VK_SUCCESS) + return false; + + VkPipelineShaderStageCreateInfo stages[2]{}; + stages[0].sType = stages[1].sType = + VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + stages[0].stage = VK_SHADER_STAGE_VERTEX_BIT; + stages[0].module = fxaa_.vertex; + stages[0].pName = "main"; + stages[1].stage = VK_SHADER_STAGE_FRAGMENT_BIT; + stages[1].module = fxaa_.fragment; + stages[1].pName = "main"; + VkPipelineVertexInputStateCreateInfo vertex_input{}; + vertex_input.sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO; + VkPipelineInputAssemblyStateCreateInfo assembly{}; + assembly.sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO; + assembly.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + VkPipelineViewportStateCreateInfo viewport{}; + viewport.sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO; + viewport.viewportCount = viewport.scissorCount = 1; + VkPipelineRasterizationStateCreateInfo raster{}; + raster.sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO; + raster.polygonMode = VK_POLYGON_MODE_FILL; + raster.cullMode = VK_CULL_MODE_NONE; + raster.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + raster.lineWidth = 1.0f; + VkPipelineMultisampleStateCreateInfo multisample{}; + multisample.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO; + multisample.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; + VkPipelineColorBlendAttachmentState blend{}; + blend.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | + VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT; + VkPipelineColorBlendStateCreateInfo color{}; + color.sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO; + color.attachmentCount = 1; + color.pAttachments = &blend; + const VkDynamicState dynamic_states[2] = {VK_DYNAMIC_STATE_VIEWPORT, + VK_DYNAMIC_STATE_SCISSOR}; + VkPipelineDynamicStateCreateInfo dynamic{}; + dynamic.sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO; + dynamic.dynamicStateCount = 2; + dynamic.pDynamicStates = dynamic_states; + VkGraphicsPipelineCreateInfo info{}; + info.sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO; + info.stageCount = 2; + info.pStages = stages; + info.pVertexInputState = &vertex_input; + info.pInputAssemblyState = &assembly; + info.pViewportState = &viewport; + info.pRasterizationState = &raster; + info.pMultisampleState = &multisample; + info.pColorBlendState = &color; + info.pDynamicState = &dynamic; + info.layout = fxaa_.layout; + info.renderPass = fxaa_.render_pass; + if (vk_->CreateGraphicsPipelines(vk_device_, VK_NULL_HANDLE, 1, &info, + nullptr, &fxaa_.pipeline) != VK_SUCCESS) { + fxaa_.pipeline = VK_NULL_HANDLE; return false; } - if (mirror_.enabled()) - mirror_readback_ = {slot, slot->serial, published_pose_.frame, - eyes_[0].format, eyes_[0].width, eyes_[0].height}; + fxaa_.target_format = target_format; + return true; +} + +bool OpenXrBackend::PrepareFxaaLocked() { + if (!FxaaEnabled() || fxaa_.failed || vk_ == nullptr) + return false; + // Sampling needs SAMPLED usage on the guest's eye images. + for (int eye = 0; eye < 2; ++eye) { + const auto record = images_.find(eyes_[eye].image); + if (record == images_.end() || + !(record->second.usage & VK_IMAGE_USAGE_SAMPLED_BIT)) + return false; + } + const auto fail = [this](const char *what) { + fxaa_.failed = true; + Log(" [vr-backend] FXAA unavailable (%s); copying the eyes without " + "anti-aliasing\n", + what); + return false; + }; + const VkFormat target_format = eye_swapchains_[0].format; + if (fxaa_.target_format != target_format) { + // Framebuffers name the old render pass; pending draws use both. + WaitAsyncSubmissionsLocked(VK_NULL_HANDLE); + DestroyFxaaTargetsLocked(); + } + if (!CreateFxaaPipelineLocked(target_format)) + return fail("pipeline"); + for (int eye = 0; eye < 2; ++eye) { + const EyeSwapchain &swapchain = eye_swapchains_[eye]; + auto &views = fxaa_.target_views[eye]; + auto &framebuffers = fxaa_.framebuffers[eye]; + if (framebuffers.size() == swapchain.images.size()) + continue; + for (std::size_t i = framebuffers.size(); i < swapchain.images.size(); ++i) { + VkImageViewCreateInfo view{}; + view.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + view.image = swapchain.images[i]; + view.viewType = VK_IMAGE_VIEW_TYPE_2D; + view.format = swapchain.format; + view.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + VkImageView target = VK_NULL_HANDLE; + if (vk_->CreateImageView(vk_device_, &view, nullptr, &target) != + VK_SUCCESS) + return fail("swapchain view"); + views.push_back(target); + VkFramebufferCreateInfo info{}; + info.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO; + info.renderPass = fxaa_.render_pass; + info.attachmentCount = 1; + info.pAttachments = ⌖ + info.width = swapchain.width; + info.height = swapchain.height; + info.layers = 1; + VkFramebuffer framebuffer = VK_NULL_HANDLE; + if (vk_->CreateFramebuffer(vk_device_, &info, nullptr, &framebuffer) != + VK_SUCCESS) + return fail("framebuffer"); + framebuffers.push_back(framebuffer); + } + } + for (int eye = 0; eye < 2; ++eye) { + if (fxaa_.sources[eye] == eyes_[eye].image && + fxaa_.source_views[eye] != VK_NULL_HANDLE) + continue; + // A set must not change while a pending draw reads it. Eyes rebind + // rarely (Roblox recreated its eye targets), so waiting here is cheap. + WaitAsyncSubmissionsLocked(VK_NULL_HANDLE); + if (fxaa_.source_views[eye] != VK_NULL_HANDLE) + vk_->DestroyImageView(vk_device_, fxaa_.source_views[eye], nullptr); + fxaa_.source_views[eye] = VK_NULL_HANDLE; + fxaa_.sources[eye] = VK_NULL_HANDLE; + VkImageViewCreateInfo view{}; + view.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; + view.image = eyes_[eye].image; + view.viewType = VK_IMAGE_VIEW_TYPE_2D; + view.format = eyes_[eye].format; + view.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + if (vk_->CreateImageView(vk_device_, &view, nullptr, + &fxaa_.source_views[eye]) != VK_SUCCESS) { + fxaa_.source_views[eye] = VK_NULL_HANDLE; + return fail("eye view"); + } + fxaa_.sources[eye] = eyes_[eye].image; + VkDescriptorImageInfo image{}; + image.imageView = fxaa_.source_views[eye]; + image.imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + VkWriteDescriptorSet write{}; + write.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + write.dstSet = fxaa_.sets[eye]; + write.descriptorCount = 1; + write.descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER; + write.pImageInfo = ℑ + vk_->UpdateDescriptorSets(vk_device_, 1, &write, 0, nullptr); + } + static bool logged = false; + if (!logged) { + logged = true; + Log(" [vr-backend] FXAA anti-aliasing on (%ux%u per eye)\n", + eye_swapchains_[0].width, eye_swapchains_[0].height); + } return true; } +void OpenXrBackend::RecordFxaaEyesLocked(VkCommandBuffer command) { + const auto barrier = [&](VkImage image, VkImageLayout from, VkImageLayout to, + VkAccessFlags src_access, VkAccessFlags dst_access, + VkPipelineStageFlags src_stage, + VkPipelineStageFlags dst_stage) { + VkImageMemoryBarrier b{}; + b.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + b.srcAccessMask = src_access; + b.dstAccessMask = dst_access; + b.oldLayout = from; + b.newLayout = to; + b.srcQueueFamilyIndex = b.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + b.image = image; + b.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + vk_->CmdPipelineBarrier(command, src_stage, dst_stage, 0, 0, nullptr, 0, + nullptr, 1, &b); + }; + const bool linearize = eye_swapchains_[0].format == VK_FORMAT_R8G8B8A8_SRGB || + eye_swapchains_[0].format == VK_FORMAT_B8G8R8A8_SRGB; + for (int eye = 0; eye < 2; ++eye) { + const EyeBinding &binding = eyes_[eye]; + const EyeSwapchain &swapchain = eye_swapchains_[eye]; + barrier(binding.image, binding.final_layout, + VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, + VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, + VK_ACCESS_SHADER_READ_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT); + barrier(swapchain.images[swapchain.acquired_index], + VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, + 0, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); + VkRenderPassBeginInfo begin{}; + begin.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO; + begin.renderPass = fxaa_.render_pass; + begin.framebuffer = fxaa_.framebuffers[eye][swapchain.acquired_index]; + begin.renderArea.extent = {swapchain.width, swapchain.height}; + vk_->CmdBeginRenderPass(command, &begin, VK_SUBPASS_CONTENTS_INLINE); + vk_->CmdBindPipeline(command, VK_PIPELINE_BIND_POINT_GRAPHICS, + fxaa_.pipeline); + vk_->CmdBindDescriptorSets(command, VK_PIPELINE_BIND_POINT_GRAPHICS, + fxaa_.layout, 0, 1, &fxaa_.sets[eye], 0, + nullptr); + const VkViewport viewport{0.f, 0.f, float(swapchain.width), + float(swapchain.height), 0.f, 1.f}; + const VkRect2D scissor{{0, 0}, {swapchain.width, swapchain.height}}; + vk_->CmdSetViewport(command, 0, 1, &viewport); + vk_->CmdSetScissor(command, 0, 1, &scissor); + struct { + float texel[2]; + std::uint32_t linearize; + } push{{1.f / float(binding.width), 1.f / float(binding.height)}, + linearize ? 1u : 0u}; + vk_->CmdPushConstants(command, fxaa_.layout, VK_SHADER_STAGE_FRAGMENT_BIT, + 0, sizeof(push), &push); + vk_->CmdDraw(command, 3, 1, 0, 0); + vk_->CmdEndRenderPass(command); + VkImageLayout layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; + VkAccessFlags access = VK_ACCESS_SHADER_READ_BIT; + VkPipelineStageFlags stage = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; + if (mirror_.enabled()) { + // The shared-memory mirror still reads the guest's own pixels. + barrier(binding.image, layout, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + access, VK_ACCESS_TRANSFER_READ_BIT, stage, + VK_PIPELINE_STAGE_TRANSFER_BIT); + mirror_.RecordEyeCopy(MirrorProcs(), command, eye, binding.image, + binding.final_layout, binding.width, + binding.height); + layout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + access = VK_ACCESS_TRANSFER_READ_BIT; + stage = VK_PIPELINE_STAGE_TRANSFER_BIT; + } + // Back to the guest's layout before its next frame renders into it. + barrier(binding.image, layout, binding.final_layout, access, + VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, stage, + VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); + } +} + +void OpenXrBackend::DestroyFxaaTargetsLocked() { + for (int eye = 0; eye < 2; ++eye) { + if (vk_ != nullptr && vk_device_ != VK_NULL_HANDLE) { + for (const VkFramebuffer framebuffer : fxaa_.framebuffers[eye]) + vk_->DestroyFramebuffer(vk_device_, framebuffer, nullptr); + for (const VkImageView view : fxaa_.target_views[eye]) + vk_->DestroyImageView(vk_device_, view, nullptr); + } + fxaa_.framebuffers[eye].clear(); + fxaa_.target_views[eye].clear(); + } +} + +void OpenXrBackend::ForgetFxaaSourceLocked(VkImage image) { + for (int eye = 0; eye < 2; ++eye) { + if (image == VK_NULL_HANDLE || fxaa_.sources[eye] != image) + continue; + if (vk_ != nullptr && vk_device_ != VK_NULL_HANDLE && + fxaa_.source_views[eye] != VK_NULL_HANDLE) + vk_->DestroyImageView(vk_device_, fxaa_.source_views[eye], nullptr); + fxaa_.source_views[eye] = VK_NULL_HANDLE; + fxaa_.sources[eye] = VK_NULL_HANDLE; + } +} + +void OpenXrBackend::DestroyFxaaLocked() { + DestroyFxaaTargetsLocked(); + ForgetFxaaSourceLocked(fxaa_.sources[0]); + ForgetFxaaSourceLocked(fxaa_.sources[1]); + if (vk_ != nullptr && vk_device_ != VK_NULL_HANDLE) { + // The pool frees its sets. + if (fxaa_.pipeline) vk_->DestroyPipeline(vk_device_, fxaa_.pipeline, nullptr); + if (fxaa_.render_pass) + vk_->DestroyRenderPass(vk_device_, fxaa_.render_pass, nullptr); + if (fxaa_.layout) + vk_->DestroyPipelineLayout(vk_device_, fxaa_.layout, nullptr); + if (fxaa_.pool) vk_->DestroyDescriptorPool(vk_device_, fxaa_.pool, nullptr); + if (fxaa_.set_layout) + vk_->DestroyDescriptorSetLayout(vk_device_, fxaa_.set_layout, nullptr); + if (fxaa_.sampler) vk_->DestroySampler(vk_device_, fxaa_.sampler, nullptr); + if (fxaa_.vertex) vk_->DestroyShaderModule(vk_device_, fxaa_.vertex, nullptr); + if (fxaa_.fragment) + vk_->DestroyShaderModule(vk_device_, fxaa_.fragment, nullptr); + } + fxaa_ = {}; +} + void OpenXrBackend::PublishMirrorReadbackLocked() { const MirrorReadback readback = std::exchange(mirror_readback_, {}); if (readback.slot == nullptr || !mirror_.enabled()) @@ -3823,9 +4302,12 @@ bool OpenXrBackend::CaptureEyeEvidence(VkQueue queue, int eye) { VkPhysicalDeviceMemoryProperties properties{}; vk_->GetPhysicalDeviceMemoryProperties(vk_physical_device_, &properties); for (int i = 0; i < 2; ++i) { + // sRGB formats hold the same bytes; only the channel order matters. Require(formats[i] == VK_FORMAT_R8G8B8A8_UNORM || - formats[i] == VK_FORMAT_B8G8R8A8_UNORM, - "evidence currently requires 8-bit UNORM images"); + formats[i] == VK_FORMAT_B8G8R8A8_UNORM || + formats[i] == VK_FORMAT_R8G8B8A8_SRGB || + formats[i] == VK_FORMAT_B8G8R8A8_SRGB, + "evidence currently requires 8-bit RGBA images"); VkBufferCreateInfo info{}; info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; info.size = size; @@ -3918,7 +4400,8 @@ bool OpenXrBackend::CaptureEyeEvidence(VkQueue queue, int eye) { "evidence map failed"); const auto *bytes = static_cast(mapped); std::vector rgb(size / 4 * 3); - const bool bgra = formats[i] == VK_FORMAT_B8G8R8A8_UNORM; + const bool bgra = formats[i] == VK_FORMAT_B8G8R8A8_UNORM || + formats[i] == VK_FORMAT_B8G8R8A8_SRGB; for (std::size_t pixel = 0; pixel < size / 4; ++pixel) { rgb[pixel * 3] = bytes[pixel * 4 + (bgra ? 2 : 0)]; rgb[pixel * 3 + 1] = bytes[pixel * 4 + 1]; diff --git a/src/vr/vr_shaders_spirv.h b/src/vr/vr_shaders_spirv.h new file mode 100644 index 00000000..d8c47992 --- /dev/null +++ b/src/vr/vr_shaders_spirv.h @@ -0,0 +1,220 @@ +// Generated by scripts/generate_vr_shaders.py from shaders/vr. Do not edit. +#ifndef MOCKTAIL_VR_VR_SHADERS_SPIRV_H_ +#define MOCKTAIL_VR_VR_SHADERS_SPIRV_H_ + +#include + +namespace mocktail::vr { + +// shaders/vr/fullscreen.vert +inline constexpr std::uint32_t kFullscreenVertexSpirv[] = { + 0x07230203, 0x00010000, 0x000d000b, 0x0000002c, 0x00000000, 0x00020011, + 0x00000001, 0x0006000b, 0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, + 0x00000000, 0x0003000e, 0x00000000, 0x00000001, 0x0008000f, 0x00000000, + 0x00000004, 0x6e69616d, 0x00000000, 0x00000009, 0x0000000c, 0x0000001d, + 0x00040047, 0x00000009, 0x0000001e, 0x00000000, 0x00040047, 0x0000000c, + 0x0000000b, 0x0000002a, 0x00030047, 0x0000001b, 0x00000002, 0x00050048, + 0x0000001b, 0x00000000, 0x0000000b, 0x00000000, 0x00050048, 0x0000001b, + 0x00000001, 0x0000000b, 0x00000001, 0x00050048, 0x0000001b, 0x00000002, + 0x0000000b, 0x00000003, 0x00050048, 0x0000001b, 0x00000003, 0x0000000b, + 0x00000004, 0x00020013, 0x00000002, 0x00030021, 0x00000003, 0x00000002, + 0x00030016, 0x00000006, 0x00000020, 0x00040017, 0x00000007, 0x00000006, + 0x00000002, 0x00040020, 0x00000008, 0x00000003, 0x00000007, 0x0004003b, + 0x00000008, 0x00000009, 0x00000003, 0x00040015, 0x0000000a, 0x00000020, + 0x00000001, 0x00040020, 0x0000000b, 0x00000001, 0x0000000a, 0x0004003b, + 0x0000000b, 0x0000000c, 0x00000001, 0x0004002b, 0x0000000a, 0x0000000e, + 0x00000001, 0x0004002b, 0x0000000a, 0x00000010, 0x00000002, 0x00040017, + 0x00000017, 0x00000006, 0x00000004, 0x00040015, 0x00000018, 0x00000020, + 0x00000000, 0x0004002b, 0x00000018, 0x00000019, 0x00000001, 0x0004001c, + 0x0000001a, 0x00000006, 0x00000019, 0x0006001e, 0x0000001b, 0x00000017, + 0x00000006, 0x0000001a, 0x0000001a, 0x00040020, 0x0000001c, 0x00000003, + 0x0000001b, 0x0004003b, 0x0000001c, 0x0000001d, 0x00000003, 0x0004002b, + 0x0000000a, 0x0000001e, 0x00000000, 0x0004002b, 0x00000006, 0x00000020, + 0x40000000, 0x0004002b, 0x00000006, 0x00000022, 0x3f800000, 0x0004002b, + 0x00000006, 0x00000025, 0x00000000, 0x00040020, 0x00000029, 0x00000003, + 0x00000017, 0x0005002c, 0x00000007, 0x0000002b, 0x00000022, 0x00000022, + 0x00050036, 0x00000002, 0x00000004, 0x00000000, 0x00000003, 0x000200f8, + 0x00000005, 0x0004003d, 0x0000000a, 0x0000000d, 0x0000000c, 0x000500c4, + 0x0000000a, 0x0000000f, 0x0000000d, 0x0000000e, 0x000500c7, 0x0000000a, + 0x00000011, 0x0000000f, 0x00000010, 0x0004006f, 0x00000006, 0x00000012, + 0x00000011, 0x000500c7, 0x0000000a, 0x00000014, 0x0000000d, 0x00000010, + 0x0004006f, 0x00000006, 0x00000015, 0x00000014, 0x00050050, 0x00000007, + 0x00000016, 0x00000012, 0x00000015, 0x0003003e, 0x00000009, 0x00000016, + 0x0004003d, 0x00000007, 0x0000001f, 0x00000009, 0x0005008e, 0x00000007, + 0x00000021, 0x0000001f, 0x00000020, 0x00050083, 0x00000007, 0x00000024, + 0x00000021, 0x0000002b, 0x00050051, 0x00000006, 0x00000026, 0x00000024, + 0x00000000, 0x00050051, 0x00000006, 0x00000027, 0x00000024, 0x00000001, + 0x00070050, 0x00000017, 0x00000028, 0x00000026, 0x00000027, 0x00000025, + 0x00000022, 0x00050041, 0x00000029, 0x0000002a, 0x0000001d, 0x0000001e, + 0x0003003e, 0x0000002a, 0x00000028, 0x000100fd, 0x00010038, +}; + +// shaders/vr/fxaa.frag +inline constexpr std::uint32_t kFxaaFragmentSpirv[] = { + 0x07230203, 0x00010000, 0x000d000b, 0x00000189, 0x00000000, 0x00020011, + 0x00000001, 0x0006000b, 0x00000001, 0x4c534c47, 0x6474732e, 0x3035342e, + 0x00000000, 0x0003000e, 0x00000000, 0x00000001, 0x0007000f, 0x00000004, + 0x00000004, 0x6e69616d, 0x00000000, 0x00000044, 0x00000111, 0x00030010, + 0x00000004, 0x00000007, 0x00040047, 0x00000022, 0x00000021, 0x00000000, + 0x00040047, 0x00000022, 0x00000022, 0x00000000, 0x00040047, 0x00000044, + 0x0000001e, 0x00000000, 0x00030047, 0x00000052, 0x00000002, 0x00050048, + 0x00000052, 0x00000000, 0x00000023, 0x00000000, 0x00050048, 0x00000052, + 0x00000001, 0x00000023, 0x00000008, 0x00040047, 0x00000111, 0x0000001e, + 0x00000000, 0x00020013, 0x00000002, 0x00030021, 0x00000003, 0x00000002, + 0x00030016, 0x00000006, 0x00000020, 0x00040017, 0x00000007, 0x00000006, + 0x00000003, 0x00040017, 0x0000000d, 0x00000006, 0x00000002, 0x0004002b, + 0x00000006, 0x00000018, 0x3e991687, 0x0004002b, 0x00000006, 0x00000019, + 0x3f1645a2, 0x0004002b, 0x00000006, 0x0000001a, 0x3de978d5, 0x0006002c, + 0x00000007, 0x0000001b, 0x00000018, 0x00000019, 0x0000001a, 0x00090019, + 0x0000001f, 0x00000006, 0x00000001, 0x00000000, 0x00000000, 0x00000000, + 0x00000001, 0x00000000, 0x0003001b, 0x00000020, 0x0000001f, 0x00040020, + 0x00000021, 0x00000000, 0x00000020, 0x0004003b, 0x00000021, 0x00000022, + 0x00000000, 0x0004002b, 0x00000006, 0x00000025, 0x00000000, 0x00040017, + 0x00000026, 0x00000006, 0x00000004, 0x0004002b, 0x00000006, 0x00000030, + 0x3d6147ae, 0x0004002b, 0x00000006, 0x00000036, 0x4019999a, 0x0006002c, + 0x00000007, 0x00000037, 0x00000036, 0x00000036, 0x00000036, 0x0004002b, + 0x00000006, 0x00000039, 0x3d25aee6, 0x0006002c, 0x00000007, 0x0000003a, + 0x00000039, 0x00000039, 0x00000039, 0x00040020, 0x00000043, 0x00000001, + 0x0000000d, 0x0004003b, 0x00000043, 0x00000044, 0x00000001, 0x0004002b, + 0x00000006, 0x0000004f, 0xbf800000, 0x0005002c, 0x0000000d, 0x00000050, + 0x0000004f, 0x0000004f, 0x00040015, 0x00000051, 0x00000020, 0x00000000, + 0x0004001e, 0x00000052, 0x0000000d, 0x00000051, 0x00040020, 0x00000053, + 0x00000009, 0x00000052, 0x0004003b, 0x00000053, 0x00000054, 0x00000009, + 0x00040015, 0x00000055, 0x00000020, 0x00000001, 0x0004002b, 0x00000055, + 0x00000056, 0x00000000, 0x00040020, 0x00000057, 0x00000009, 0x0000000d, + 0x0004002b, 0x00000006, 0x00000062, 0x3f800000, 0x0005002c, 0x0000000d, + 0x00000063, 0x00000062, 0x0000004f, 0x0005002c, 0x0000000d, 0x0000006e, + 0x0000004f, 0x00000062, 0x0004002b, 0x00000006, 0x0000009c, 0x3d2aaaab, + 0x0004002b, 0x00000006, 0x0000009e, 0x3e000000, 0x00020014, 0x000000a1, + 0x0004002b, 0x00000006, 0x000000be, 0x3d000000, 0x0004002b, 0x00000006, + 0x000000c0, 0x3c000000, 0x0004002b, 0x00000051, 0x000000c3, 0x00000000, + 0x0004002b, 0x00000006, 0x000000d2, 0xc1000000, 0x0005002c, 0x0000000d, + 0x000000d3, 0x000000d2, 0x000000d2, 0x0004002b, 0x00000006, 0x000000d4, + 0x41000000, 0x0005002c, 0x0000000d, 0x000000d5, 0x000000d4, 0x000000d4, + 0x0004002b, 0x00000006, 0x000000db, 0x3f000000, 0x0004002b, 0x00000006, + 0x000000de, 0xbe2aaaab, 0x0004002b, 0x00000006, 0x000000e5, 0x3e2aaaab, + 0x0004002b, 0x00000006, 0x000000ef, 0x3e800000, 0x0004002b, 0x00000006, + 0x000000f2, 0xbf000000, 0x00040017, 0x0000010d, 0x000000a1, 0x00000003, + 0x00040020, 0x00000110, 0x00000003, 0x00000026, 0x0004003b, 0x00000110, + 0x00000111, 0x00000003, 0x0004002b, 0x00000055, 0x00000112, 0x00000001, + 0x00040020, 0x00000113, 0x00000009, 0x00000051, 0x0006002c, 0x00000007, + 0x00000183, 0x00000030, 0x00000030, 0x00000030, 0x0004002b, 0x00000006, + 0x00000185, 0x3d9e8391, 0x0006002c, 0x00000007, 0x00000186, 0x00000185, + 0x00000185, 0x00000185, 0x0004002b, 0x00000006, 0x00000187, 0x3f72a76f, + 0x0006002c, 0x00000007, 0x00000188, 0x00000187, 0x00000187, 0x00000187, + 0x00050036, 0x00000002, 0x00000004, 0x00000000, 0x00000003, 0x000200f8, + 0x00000005, 0x0004003d, 0x00000020, 0x00000042, 0x00000022, 0x0004003d, + 0x0000000d, 0x00000045, 0x00000044, 0x00070058, 0x00000026, 0x00000046, + 0x00000042, 0x00000045, 0x00000002, 0x00000025, 0x0008004f, 0x00000007, + 0x0000004b, 0x00000046, 0x00000046, 0x00000000, 0x00000001, 0x00000002, + 0x00050094, 0x00000006, 0x0000012a, 0x0000004b, 0x0000001b, 0x00050041, + 0x00000057, 0x00000058, 0x00000054, 0x00000056, 0x0004003d, 0x0000000d, + 0x00000059, 0x00000058, 0x00050085, 0x0000000d, 0x0000005a, 0x00000050, + 0x00000059, 0x00050081, 0x0000000d, 0x0000005b, 0x00000045, 0x0000005a, + 0x0004003d, 0x00000020, 0x0000012d, 0x00000022, 0x00070058, 0x00000026, + 0x0000012f, 0x0000012d, 0x0000005b, 0x00000002, 0x00000025, 0x0008004f, + 0x00000007, 0x00000130, 0x0000012f, 0x0000012f, 0x00000000, 0x00000001, + 0x00000002, 0x00050094, 0x00000006, 0x00000134, 0x00000130, 0x0000001b, + 0x00050085, 0x0000000d, 0x00000066, 0x00000063, 0x00000059, 0x00050081, + 0x0000000d, 0x00000067, 0x00000045, 0x00000066, 0x0004003d, 0x00000020, + 0x00000137, 0x00000022, 0x00070058, 0x00000026, 0x00000139, 0x00000137, + 0x00000067, 0x00000002, 0x00000025, 0x0008004f, 0x00000007, 0x0000013a, + 0x00000139, 0x00000139, 0x00000000, 0x00000001, 0x00000002, 0x00050094, + 0x00000006, 0x0000013e, 0x0000013a, 0x0000001b, 0x00050085, 0x0000000d, + 0x00000071, 0x0000006e, 0x00000059, 0x00050081, 0x0000000d, 0x00000072, + 0x00000045, 0x00000071, 0x0004003d, 0x00000020, 0x00000141, 0x00000022, + 0x00070058, 0x00000026, 0x00000143, 0x00000141, 0x00000072, 0x00000002, + 0x00000025, 0x0008004f, 0x00000007, 0x00000144, 0x00000143, 0x00000143, + 0x00000000, 0x00000001, 0x00000002, 0x00050094, 0x00000006, 0x00000148, + 0x00000144, 0x0000001b, 0x00050081, 0x0000000d, 0x0000007d, 0x00000045, + 0x00000059, 0x0004003d, 0x00000020, 0x0000014b, 0x00000022, 0x00070058, + 0x00000026, 0x0000014d, 0x0000014b, 0x0000007d, 0x00000002, 0x00000025, + 0x0008004f, 0x00000007, 0x0000014e, 0x0000014d, 0x0000014d, 0x00000000, + 0x00000001, 0x00000002, 0x00050094, 0x00000006, 0x00000152, 0x0000014e, + 0x0000001b, 0x0007000c, 0x00000006, 0x00000086, 0x00000001, 0x00000025, + 0x00000134, 0x0000013e, 0x0007000c, 0x00000006, 0x00000089, 0x00000001, + 0x00000025, 0x00000148, 0x00000152, 0x0007000c, 0x00000006, 0x0000008a, + 0x00000001, 0x00000025, 0x00000086, 0x00000089, 0x0007000c, 0x00000006, + 0x0000008b, 0x00000001, 0x00000025, 0x0000012a, 0x0000008a, 0x0007000c, + 0x00000006, 0x00000090, 0x00000001, 0x00000028, 0x00000134, 0x0000013e, + 0x0007000c, 0x00000006, 0x00000093, 0x00000001, 0x00000028, 0x00000148, + 0x00000152, 0x0007000c, 0x00000006, 0x00000094, 0x00000001, 0x00000028, + 0x00000090, 0x00000093, 0x0007000c, 0x00000006, 0x00000095, 0x00000001, + 0x00000028, 0x0000012a, 0x00000094, 0x00050083, 0x00000006, 0x0000009b, + 0x00000095, 0x0000008b, 0x00050085, 0x00000006, 0x0000009f, 0x00000095, + 0x0000009e, 0x0007000c, 0x00000006, 0x000000a0, 0x00000001, 0x00000028, + 0x0000009c, 0x0000009f, 0x000500be, 0x000000a1, 0x000000a2, 0x0000009b, + 0x000000a0, 0x000300f7, 0x000000a4, 0x00000000, 0x000400fa, 0x000000a2, + 0x000000a3, 0x000000a4, 0x000200f8, 0x000000a3, 0x00050081, 0x00000006, + 0x000000a8, 0x00000134, 0x0000013e, 0x00050081, 0x00000006, 0x000000ab, + 0x00000148, 0x00000152, 0x00050083, 0x00000006, 0x000000ac, 0x000000a8, + 0x000000ab, 0x0004007f, 0x00000006, 0x000000ad, 0x000000ac, 0x00050081, + 0x00000006, 0x000000b0, 0x00000134, 0x00000148, 0x00050081, 0x00000006, + 0x000000b3, 0x0000013e, 0x00000152, 0x00050083, 0x00000006, 0x000000b4, + 0x000000b0, 0x000000b3, 0x00050050, 0x0000000d, 0x000000b5, 0x000000ad, + 0x000000b4, 0x00050081, 0x00000006, 0x000000bb, 0x000000a8, 0x00000148, + 0x00050081, 0x00000006, 0x000000bd, 0x000000bb, 0x00000152, 0x00050085, + 0x00000006, 0x000000bf, 0x000000bd, 0x000000be, 0x0007000c, 0x00000006, + 0x000000c1, 0x00000001, 0x00000028, 0x000000bf, 0x000000c0, 0x0006000c, + 0x00000006, 0x000000c6, 0x00000001, 0x00000004, 0x000000ad, 0x0006000c, + 0x00000006, 0x000000ca, 0x00000001, 0x00000004, 0x000000b4, 0x0007000c, + 0x00000006, 0x000000cb, 0x00000001, 0x00000025, 0x000000c6, 0x000000ca, + 0x00050081, 0x00000006, 0x000000cd, 0x000000cb, 0x000000c1, 0x00050088, + 0x00000006, 0x000000ce, 0x00000062, 0x000000cd, 0x0005008e, 0x0000000d, + 0x000000d1, 0x000000b5, 0x000000ce, 0x0008000c, 0x0000000d, 0x000000d6, + 0x00000001, 0x0000002b, 0x000000d1, 0x000000d3, 0x000000d5, 0x00050085, + 0x0000000d, 0x000000d9, 0x000000d6, 0x00000059, 0x0005008e, 0x0000000d, + 0x000000df, 0x000000d9, 0x000000de, 0x00050081, 0x0000000d, 0x000000e0, + 0x00000045, 0x000000df, 0x0004003d, 0x00000020, 0x00000155, 0x00000022, + 0x00070058, 0x00000026, 0x00000157, 0x00000155, 0x000000e0, 0x00000002, + 0x00000025, 0x0008004f, 0x00000007, 0x00000158, 0x00000157, 0x00000157, + 0x00000000, 0x00000001, 0x00000002, 0x0005008e, 0x0000000d, 0x000000e6, + 0x000000d9, 0x000000e5, 0x00050081, 0x0000000d, 0x000000e7, 0x00000045, + 0x000000e6, 0x0004003d, 0x00000020, 0x0000015b, 0x00000022, 0x00070058, + 0x00000026, 0x0000015d, 0x0000015b, 0x000000e7, 0x00000002, 0x00000025, + 0x0008004f, 0x00000007, 0x0000015e, 0x0000015d, 0x0000015d, 0x00000000, + 0x00000001, 0x00000002, 0x00050081, 0x00000007, 0x000000ea, 0x00000158, + 0x0000015e, 0x0005008e, 0x00000007, 0x000000eb, 0x000000ea, 0x000000db, + 0x0005008e, 0x00000007, 0x000000ee, 0x000000eb, 0x000000db, 0x0005008e, + 0x0000000d, 0x000000f3, 0x000000d9, 0x000000f2, 0x00050081, 0x0000000d, + 0x000000f4, 0x00000045, 0x000000f3, 0x0004003d, 0x00000020, 0x00000161, + 0x00000022, 0x00070058, 0x00000026, 0x00000163, 0x00000161, 0x000000f4, + 0x00000002, 0x00000025, 0x0008004f, 0x00000007, 0x00000164, 0x00000163, + 0x00000163, 0x00000000, 0x00000001, 0x00000002, 0x0005008e, 0x0000000d, + 0x000000f9, 0x000000d9, 0x000000db, 0x00050081, 0x0000000d, 0x000000fa, + 0x00000045, 0x000000f9, 0x0004003d, 0x00000020, 0x00000167, 0x00000022, + 0x00070058, 0x00000026, 0x00000169, 0x00000167, 0x000000fa, 0x00000002, + 0x00000025, 0x0008004f, 0x00000007, 0x0000016a, 0x00000169, 0x00000169, + 0x00000000, 0x00000001, 0x00000002, 0x00050081, 0x00000007, 0x000000fd, + 0x00000164, 0x0000016a, 0x0005008e, 0x00000007, 0x000000fe, 0x000000fd, + 0x000000ef, 0x00050081, 0x00000007, 0x000000ff, 0x000000ee, 0x000000fe, + 0x00050094, 0x00000006, 0x0000016e, 0x000000ff, 0x0000001b, 0x000500b8, + 0x000000a1, 0x00000106, 0x0000016e, 0x0000008b, 0x000500ba, 0x000000a1, + 0x00000109, 0x0000016e, 0x00000095, 0x000500a6, 0x000000a1, 0x0000010a, + 0x00000106, 0x00000109, 0x00060050, 0x0000010d, 0x0000010e, 0x0000010a, + 0x0000010a, 0x0000010a, 0x000600a9, 0x00000007, 0x0000010f, 0x0000010e, + 0x000000eb, 0x000000ff, 0x000200f9, 0x000000a4, 0x000200f8, 0x000000a4, + 0x000700f5, 0x00000007, 0x00000180, 0x0000004b, 0x00000005, 0x0000010f, + 0x000000a3, 0x00050041, 0x00000113, 0x00000114, 0x00000054, 0x00000112, + 0x0004003d, 0x00000051, 0x00000115, 0x00000114, 0x000500ab, 0x000000a1, + 0x00000116, 0x00000115, 0x000000c3, 0x000300f7, 0x00000119, 0x00000000, + 0x000400fa, 0x00000116, 0x00000118, 0x0000011d, 0x000200f8, 0x00000118, + 0x00050085, 0x00000007, 0x00000173, 0x00000180, 0x00000186, 0x00050081, + 0x00000007, 0x00000176, 0x00000180, 0x00000183, 0x00050085, 0x00000007, + 0x00000178, 0x00000176, 0x00000188, 0x0007000c, 0x00000007, 0x00000179, + 0x00000001, 0x0000001a, 0x00000178, 0x00000037, 0x0007000c, 0x00000007, + 0x0000017b, 0x00000001, 0x00000030, 0x0000003a, 0x00000180, 0x0008000c, + 0x00000007, 0x0000017c, 0x00000001, 0x0000002e, 0x00000173, 0x00000179, + 0x0000017b, 0x000200f9, 0x00000119, 0x000200f8, 0x0000011d, 0x000200f9, + 0x00000119, 0x000200f8, 0x00000119, 0x000700f5, 0x00000007, 0x00000181, + 0x0000017c, 0x00000118, 0x00000180, 0x0000011d, 0x00050051, 0x00000006, + 0x00000122, 0x00000046, 0x00000003, 0x00050051, 0x00000006, 0x00000123, + 0x00000181, 0x00000000, 0x00050051, 0x00000006, 0x00000124, 0x00000181, + 0x00000001, 0x00050051, 0x00000006, 0x00000125, 0x00000181, 0x00000002, + 0x00070050, 0x00000026, 0x00000126, 0x00000123, 0x00000124, 0x00000125, + 0x00000122, 0x0003003e, 0x00000111, 0x00000126, 0x000100fd, 0x00010038, +}; + +} // namespace mocktail::vr + +#endif // MOCKTAIL_VR_VR_SHADERS_SPIRV_H_ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 44681d07..ea253fd0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -109,6 +109,14 @@ if(MOCKTAIL_ENABLE_VR) "$" "$" "$") set_tests_properties(OpenXrPreviewContract PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 90) + # The checked-in SPIR-V must match shaders/vr. Needs glslc to check. + find_program(MOCKTAIL_GLSLC glslc) + if(MOCKTAIL_GLSLC) + add_test(NAME VrShaders.SpirvMatchesSources + COMMAND "${Python3_EXECUTABLE}" + "${PROJECT_SOURCE_DIR}/scripts/generate_vr_shaders.py" --check + --glslc "${MOCKTAIL_GLSLC}") + endif() target_compile_definitions(openxr_backend_test PRIVATE "MOCKTAIL_TEST_OPENXR_RUNTIME=\"$\"") add_dependencies(openxr_backend_test mocktail_test_openxr_runtime) diff --git a/tests/runtime_config_file_test.cc b/tests/runtime_config_file_test.cc index e00e3fd8..dfec93a9 100644 --- a/tests/runtime_config_file_test.cc +++ b/tests/runtime_config_file_test.cc @@ -176,6 +176,22 @@ TEST(RuntimeConfigFileTest, VrModeSelectsOnOffOrAuto) { EXPECT_NE(invalid.error.find("vr.mode"), std::string::npos); } +TEST(RuntimeConfigFileTest, VrAntiAliasingAcceptsFxaaOrOff) { + TemporaryDirectory temporary; + for (const char* value : {"fxaa", "off"}) { + const auto loaded = LoadRuntimeConfig( + MapEnvironment(), + temporary.Write(std::string("version: 1\nvr:\n anti_aliasing: ") + + value + "\n")); + EXPECT_TRUE(loaded) << value << ": " << loaded.error; + } + const auto invalid = LoadRuntimeConfig( + MapEnvironment(), + temporary.Write("version: 1\nvr:\n anti_aliasing: msaa\n")); + EXPECT_FALSE(invalid); + EXPECT_NE(invalid.error.find("vr.anti_aliasing"), std::string::npos); +} + TEST(RuntimeConfigFileTest, VrFrameRateLimitIsSeparateFromDesktopLimit) { TemporaryDirectory temporary; const auto defaults = LoadRuntimeConfig(