From e9530439c8372d12de2525a8465e804db146224c Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Tue, 6 Oct 2026 19:18:04 +0200 Subject: [PATCH 1/3] [Patch] Run the editor as an app: its identity, its settings and a development bundle A bare executable, as `swift run` and Xcode start the editor, has no bundle identifier and no LaunchServices record. The Apple Vision Pro preview's device picker traps without the identifier and refuses to connect without the record. - `Sources/UntoldEditor/Info.plist` is embedded in the executable (`-sectcreate __TEXT __info_plist`, from the manifest's linker settings): the identifier of the packaged app, `com.untoldengine.studio`. The launch line logs it. - The settings move with the identity: `EditorSettingsDomain` copies what the old domain, `UntoldEditor`, holds into the new one, once. - `scripts/dev-app.sh [--run] [debug|release]` and `make run-app` wrap the build in `Untold Engine Studio.app` inside the build folder: the executable copied, the resource bundles and the exporter scripts linked, the bundle registered. `ComponentSDK` and `EditorEnginePackage` follow its `BuildProducts` link to the modules and the `Package.resolved`. --- Makefile | 6 ++ Package.swift | 15 ++++ README.md | 10 +++ .../Components/ComponentSDK.swift | 15 ++++ .../Components/EditorEnginePackage.swift | 10 ++- .../Config/EditorSettingsDomain.swift | 50 ++++++++++++ Sources/UntoldEditor/Info.plist | 20 +++++ Sources/UntoldEditor/UntoldEditorApp.swift | 3 +- .../UntoldEditorTests/ComponentSDKTests.swift | 16 ++++ .../EditorEnginePackageTests.swift | 13 ++++ .../EditorSettingsDomainTests.swift | 66 ++++++++++++++++ scripts/dev-app.sh | 77 +++++++++++++++++++ 12 files changed, 297 insertions(+), 4 deletions(-) create mode 100644 Sources/UntoldEditor/Config/EditorSettingsDomain.swift create mode 100644 Sources/UntoldEditor/Info.plist create mode 100644 Tests/UntoldEditorTests/EditorSettingsDomainTests.swift create mode 100755 scripts/dev-app.sh diff --git a/Makefile b/Makefile index d69732c..de04692 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,12 @@ build: swift build +# Build, wrap the result in a development app bundle and start it: what the Apple Vision Pro +# preview needs (see scripts/dev-app.sh). CONFIG=release for the release build. +CONFIG ?= debug +run-app: + scripts/dev-app.sh --run $(CONFIG) + # Clean build artifact clean: diff --git a/Package.swift b/Package.swift index 75da016..f7f2acb 100644 --- a/Package.swift +++ b/Package.swift @@ -23,6 +23,9 @@ let package = Package( .product(name: "UntoldComponentKit", package: "UntoldEngine"), ], path: "Sources/UntoldEditor", + exclude: [ + "Info.plist", + ], resources: [ .process("Resources/Thumbnails"), ], @@ -34,6 +37,18 @@ let package = Package( .linkedFramework("Metal"), .linkedFramework("MetalKit"), .linkedFramework("QuartzCore"), + // The identity of the executable run from the build folder (`swift run`, Xcode), + // which has no app bundle around it: its bundle identifier and name, read from + // the Info.plist embedded in the binary. The system's device picker of the Apple + // Vision Pro preview traps without one, and the settings live under it, shared + // with the packaged app (create_app_bundle.sh writes the bundle's own Info.plist). + // An incremental build does not notice a change to the file: clean, or touch a source. + .unsafeFlags([ + "-Xlinker", "-sectcreate", + "-Xlinker", "__TEXT", + "-Xlinker", "__info_plist", + "-Xlinker", Context.packageDirectory + "/Sources/UntoldEditor/Info.plist", + ]), ] ), diff --git a/README.md b/README.md index 8d5f600..4e57b1a 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,16 @@ swift build swift run UntoldEditor ``` +### Run as an app, for the Apple Vision Pro preview + +`swift run` and Xcode start the editor as a bare executable. View ▸ Preview on Apple Vision Pro (macOS 26, a Vision Pro on visionOS 26 nearby, both on the same Apple Account) needs the editor to run as an app that LaunchServices knows, because the system's device picker asks LaunchServices for the app behind the request. This makes a development app bundle around the build in seconds and starts it: + +```bash +scripts/dev-app.sh --run # the debug build; add `release` for the release build +``` + +It builds, wraps the result in `Untold Engine Studio.app` inside the build folder (the executable copied, the resources and the build products linked), registers the bundle, and starts it with its output in the terminal; `make run-app` does the same. Run it again after a build. The packaged app from `create_app_bundle.sh` works as well. + ### Open in Xcode (recommended) 1. Open Xcode → File ▸ Open → select the Package.swift in this repo diff --git a/Sources/UntoldEditor/Components/ComponentSDK.swift b/Sources/UntoldEditor/Components/ComponentSDK.swift index d36dff7..fef61a3 100644 --- a/Sources/UntoldEditor/Components/ComponentSDK.swift +++ b/Sources/UntoldEditor/Components/ComponentSDK.swift @@ -34,6 +34,9 @@ struct ComponentSDK: Equatable { static let bundleDirectoryName = "ComponentSDK" static let manifestFileName = "sdk.json" + /// A development app bundle (`scripts/dev-app.sh`) links the build products it wraps here, + /// beside its resources: its executable is a copy that sits apart from them. + static let buildProductsLinkName = "BuildProducts" /// The engine's own C module: every SDK has it. static let cShaderTypesModule = "CShaderTypes" @@ -73,12 +76,24 @@ struct ComponentSDK: Equatable { { return bundled } + if let products = developmentBuildProducts(resourceURL: resourceURL, fileManager: fileManager) { + return resolveFromBuildProducts(productsDirectory: products, fileManager: fileManager) + } guard let executableURL else { return nil } // `.build/debug` is a symlink; the module maps are found relative to the real directory. let products = executableURL.resolvingSymlinksInPath().deletingLastPathComponent() return resolveFromBuildProducts(productsDirectory: products, fileManager: fileManager) } + /// The build products a development app bundle wraps, from the link beside its resources; + /// nil for a packaged app and for an executable run from the build folder. + static func developmentBuildProducts(resourceURL: URL?, fileManager: FileManager = .default) -> URL? { + guard let link = resourceURL?.appendingPathComponent(buildProductsLinkName, isDirectory: true) else { return nil } + var isDirectory: ObjCBool = false + guard fileManager.fileExists(atPath: link.path, isDirectory: &isDirectory), isDirectory.boolValue else { return nil } + return link.resolvingSymlinksInPath() + } + static func resolveBundled(at sdkRoot: URL, fileManager: FileManager = .default) -> ComponentSDK? { let modules = sdkRoot.appendingPathComponent("Modules", isDirectory: true) let manifestURL = sdkRoot.appendingPathComponent(manifestFileName) diff --git a/Sources/UntoldEditor/Components/EditorEnginePackage.swift b/Sources/UntoldEditor/Components/EditorEnginePackage.swift index 1a2ebd8..ee54f61 100644 --- a/Sources/UntoldEditor/Components/EditorEnginePackage.swift +++ b/Sources/UntoldEditor/Components/EditorEnginePackage.swift @@ -19,16 +19,20 @@ import UntoldEngine /// engine is what makes "it compiles in Xcode" and "it compiles in the editor" mean the same. enum EditorEnginePackage { /// A packaged editor reads it from its Component SDK; an editor run from source finds the - /// `Package.resolved` of its own checkout above the executable. `nil` when neither exists, - /// in which case new projects keep the engine's default reference and get no plugins folder. + /// `Package.resolved` of its own checkout above the executable, or above the build products + /// a development app bundle links. `nil` when none exists, in which case new projects keep + /// the engine's default reference and get no plugins folder. static func resolve( sdk: ComponentSDK? = ComponentSDK.resolve(), + resourceURL: URL? = Bundle.main.resourceURL, executableURL: URL? = Bundle.main.executableURL ) -> EnginePackageReference? { if let sdk, let url = sdk.engineURL, let revision = sdk.engineRevision { return EnginePackageReference(url: url, requirement: .revision(revision)) } - guard var directory = executableURL?.resolvingSymlinksInPath().deletingLastPathComponent() else { return nil } + guard var directory = ComponentSDK.developmentBuildProducts(resourceURL: resourceURL) + ?? executableURL?.resolvingSymlinksInPath().deletingLastPathComponent() + else { return nil } for _ in 0 ..< 8 { let candidate = directory.appendingPathComponent("Package.resolved") if let reference = reference(fromResolvedFile: candidate) { diff --git a/Sources/UntoldEditor/Config/EditorSettingsDomain.swift b/Sources/UntoldEditor/Config/EditorSettingsDomain.swift new file mode 100644 index 0000000..813b4b7 --- /dev/null +++ b/Sources/UntoldEditor/Config/EditorSettingsDomain.swift @@ -0,0 +1,50 @@ +// +// EditorSettingsDomain.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// + +import Foundation + +/// Where the editor's settings live. An executable run from the build folder +/// (`swift run`, Xcode) had no bundle, so `UserDefaults` kept them under its +/// name, `UntoldEditor`. With the identity the executable carries now they +/// live under the app's identifier, as the packaged app's do: the first launch +/// with the identifier takes over what the old domain holds. +enum EditorSettingsDomain { + /// The domain an executable without an identity gets: its name. + static let legacyName = "UntoldEditor" + + /// Set in the current domain once the old one has been looked at. + static let adoptedKey = "EditorSettingsDomain.adoptedLegacySettings" + + /// Copies every setting of the old domain that the current one lacks, once; + /// a setting the current domain already has stays. Returns the keys copied. + @discardableResult + static func adoptLegacySettings( + defaults: UserDefaults = .standard, + current: String? = Bundle.main.bundleIdentifier, + legacy: String = legacyName + ) -> [String] { + guard let current, current != legacy else { + return [] + } + var settings = defaults.persistentDomain(forName: current) ?? [:] + guard settings[adoptedKey] == nil else { + return [] + } + settings[adoptedKey] = true + var copied: [String] = [] + for (key, value) in defaults.persistentDomain(forName: legacy) ?? [:] where settings[key] == nil { + settings[key] = value + copied.append(key) + } + defaults.setPersistentDomain(settings, forName: current) + return copied.sorted() + } +} diff --git a/Sources/UntoldEditor/Info.plist b/Sources/UntoldEditor/Info.plist new file mode 100644 index 0000000..34718e9 --- /dev/null +++ b/Sources/UntoldEditor/Info.plist @@ -0,0 +1,20 @@ + + + + + CFBundleDisplayName + Untold Engine Studio + CFBundleExecutable + UntoldEditor + CFBundleIdentifier + com.untoldengine.studio + CFBundleName + Untold Engine Studio + CFBundlePackageType + APPL + LSMinimumSystemVersion + 14.0 + NSHighResolutionCapable + + + diff --git a/Sources/UntoldEditor/UntoldEditorApp.swift b/Sources/UntoldEditor/UntoldEditorApp.swift index dda50d8..38c75c6 100644 --- a/Sources/UntoldEditor/UntoldEditorApp.swift +++ b/Sources/UntoldEditor/UntoldEditorApp.swift @@ -55,7 +55,8 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate { private var splatLevelModeItems: [SplatLevelModeOption: NSMenuItem] = [:] func applicationDidFinishLaunching(_: Notification) { - Logger.log(message: "Launching \(appName) v\(Self.editorVersion)") + Logger.log(message: "Launching \(appName) v\(Self.editorVersion) as \(Bundle.main.bundleIdentifier ?? "an executable without an identity")") + EditorSettingsDomain.adoptLegacySettings() setupMainMenu() diff --git a/Tests/UntoldEditorTests/ComponentSDKTests.swift b/Tests/UntoldEditorTests/ComponentSDKTests.swift index fe61093..4a90907 100644 --- a/Tests/UntoldEditorTests/ComponentSDKTests.swift +++ b/Tests/UntoldEditorTests/ComponentSDKTests.swift @@ -62,6 +62,22 @@ final class ComponentSDKTests: XCTestCase { XCTAssertNotNil(sdk, "../.. must be taken from the real products directory, not from the symlink") } + func test_developmentAppBundle_findsTheBuildProductsItLinks() throws { + let scratch = try ScratchDirectory() + let products = try scratch.directory("build/out/Products/Debug") + try addModules(["UntoldEngine", "UntoldComponentKit"], to: products) + try scratch.write("module CShaderTypes {}", to: "build/out/Intermediates.noindex/GeneratedModuleMaps/CShaderTypes.modulemap") + let executable = try scratch.write("", to: "build/out/Products/Debug/Untold Engine Studio.app/Contents/MacOS/UntoldEditor") + let resources = try scratch.directory("build/out/Products/Debug/Untold Engine Studio.app/Contents/Resources") + try FileManager.default.createSymbolicLink(at: resources.appendingPathComponent(ComponentSDK.buildProductsLinkName), withDestinationURL: products) + + let sdk = try XCTUnwrap(ComponentSDK.resolve(resourceURL: resources, executableURL: executable)) + + XCTAssertEqual(sdk.modulesDirectory.resolvingSymlinksInPath().path, products.resolvingSymlinksInPath().path) + XCTAssertFalse(sdk.isBundled) + XCTAssertNil(ComponentSDK.resolve(resourceURL: resources.appendingPathComponent("elsewhere"), executableURL: executable), "a copied executable with no link beside the resources finds nothing") + } + func test_bundledSDK_winsAndCarriesTheRecordedCompiler() throws { let scratch = try ScratchDirectory() let resources = try scratch.directory("App.app/Contents/Resources") diff --git a/Tests/UntoldEditorTests/EditorEnginePackageTests.swift b/Tests/UntoldEditorTests/EditorEnginePackageTests.swift index 807fc56..74f017c 100644 --- a/Tests/UntoldEditorTests/EditorEnginePackageTests.swift +++ b/Tests/UntoldEditorTests/EditorEnginePackageTests.swift @@ -50,6 +50,19 @@ final class EditorEnginePackageTests: XCTestCase { XCTAssertEqual(reference, EnginePackageReference(url: "https://github.com/miolabs/UntoldEngine.git", requirement: .revision("engine123"))) } + func test_developmentAppBundle_findsThePackageResolvedAboveTheBuildProductsItLinks() throws { + let scratch = try ScratchDirectory("EditorEnginePackage") + try scratch.write(resolvedFile, to: "UntoldEditor/Package.resolved") + let products = try scratch.directory("UntoldEditor/.build/out/Products/Debug") + let executable = try scratch.write("", to: "Elsewhere/Untold Engine Studio.app/Contents/MacOS/UntoldEditor") + let resources = try scratch.directory("Elsewhere/Untold Engine Studio.app/Contents/Resources") + try FileManager.default.createSymbolicLink(at: resources.appendingPathComponent(ComponentSDK.buildProductsLinkName), withDestinationURL: products) + + let reference = EditorEnginePackage.resolve(sdk: makeTestSDK(), resourceURL: resources, executableURL: executable) + + XCTAssertEqual(reference, EnginePackageReference(url: "https://github.com/miolabs/UntoldEngine.git", requirement: .revision("engine123"))) + } + func test_withoutAnyRecord_thereIsNoReference() throws { let scratch = try ScratchDirectory("EditorEnginePackage") let executable = try scratch.write("", to: "Somewhere/UntoldEditor") diff --git a/Tests/UntoldEditorTests/EditorSettingsDomainTests.swift b/Tests/UntoldEditorTests/EditorSettingsDomainTests.swift new file mode 100644 index 0000000..47b6e0b --- /dev/null +++ b/Tests/UntoldEditorTests/EditorSettingsDomainTests.swift @@ -0,0 +1,66 @@ +// +// EditorSettingsDomainTests.swift +// UntoldEditorTests +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// + +@testable import UntoldEditor +import XCTest + +final class EditorSettingsDomainTests: XCTestCase { + private let legacy = "untold.test.settings.legacy.\(UUID().uuidString)" + private let current = "untold.test.settings.current.\(UUID().uuidString)" + private let defaults = UserDefaults.standard + + override func tearDown() { + defaults.removePersistentDomain(forName: legacy) + defaults.removePersistentDomain(forName: current) + super.tearDown() + } + + func test_takesOverWhatTheOldDomainHas_andKeepsWhatTheCurrentOneHas() { + defaults.setPersistentDomain(["layout": "old", "snap": true], forName: legacy) + defaults.setPersistentDomain(["layout": "new"], forName: current) + + let copied = EditorSettingsDomain.adoptLegacySettings(defaults: defaults, current: current, legacy: legacy) + + XCTAssertEqual(copied, ["snap"]) + let settings = defaults.persistentDomain(forName: current) + XCTAssertEqual(settings?["layout"] as? String, "new") + XCTAssertEqual(settings?["snap"] as? Bool, true) + XCTAssertEqual(defaults.persistentDomain(forName: legacy)?["layout"] as? String, "old", "the old domain is left as it is") + } + + func test_onlyOnce_aLaterChangeOfTheOldDomainIsNotCopied() { + defaults.setPersistentDomain(["snap": true], forName: legacy) + EditorSettingsDomain.adoptLegacySettings(defaults: defaults, current: current, legacy: legacy) + defaults.setPersistentDomain(["snap": true, "later": 1], forName: legacy) + + let copied = EditorSettingsDomain.adoptLegacySettings(defaults: defaults, current: current, legacy: legacy) + + XCTAssertEqual(copied, []) + XCTAssertNil(defaults.persistentDomain(forName: current)?["later"]) + } + + func test_anEmptyOldDomain_isLookedAtOnceToo() { + let copied = EditorSettingsDomain.adoptLegacySettings(defaults: defaults, current: current, legacy: legacy) + + XCTAssertEqual(copied, []) + XCTAssertEqual(defaults.persistentDomain(forName: current)?[EditorSettingsDomain.adoptedKey] as? Bool, true) + } + + func test_withoutAnIdentity_orWithTheOldDomainAsTheCurrentOne_nothingIsWritten() { + defaults.setPersistentDomain(["snap": true], forName: legacy) + + XCTAssertEqual(EditorSettingsDomain.adoptLegacySettings(defaults: defaults, current: nil, legacy: legacy), []) + XCTAssertEqual(EditorSettingsDomain.adoptLegacySettings(defaults: defaults, current: legacy, legacy: legacy), []) + + XCTAssertNil(defaults.persistentDomain(forName: current)) + XCTAssertNil(defaults.persistentDomain(forName: legacy)?[EditorSettingsDomain.adoptedKey]) + } +} diff --git a/scripts/dev-app.sh b/scripts/dev-app.sh new file mode 100755 index 0000000..705bcbf --- /dev/null +++ b/scripts/dev-app.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Wraps the editor that `swift build` made in a development app bundle and registers it with +# LaunchServices, so that it runs as an app the system knows: the device picker of the Apple +# Vision Pro preview asks LaunchServices for the app behind the request and refuses a bare +# executable. The settings and the window state are the packaged app's (same identifier). +# +# The executable is copied (LaunchServices wants the running file inside the bundle); the resource +# bundles, the exporter scripts and the build products are linked, and the editor finds its +# Component SDK through the BuildProducts link. Run it again after a build, or use --run. +# +# scripts/dev-app.sh [debug|release] makes (or refreshes) the bundle and prints its path +# scripts/dev-app.sh --run [debug|release] builds, makes it and starts it, output in this terminal +set -euo pipefail +cd "$(dirname "$0")/.." + +RUN=0 +CONFIG=debug +for arg in "$@"; do + case "$arg" in + --run) RUN=1 ;; + debug | release) CONFIG="$arg" ;; + *) echo "usage: scripts/dev-app.sh [--run] [debug|release]" >&2; exit 2 ;; + esac +done + +if [ "$RUN" = 1 ]; then + swift build -c "$CONFIG" +fi +BIN_DIR="$(swift build -c "$CONFIG" --show-bin-path)" +if [ ! -x "$BIN_DIR/UntoldEditor" ]; then + echo "No $CONFIG build in $BIN_DIR: run 'swift build -c $CONFIG' first." >&2 + exit 1 +fi + +APP="$BIN_DIR/Untold Engine Studio.app" +mkdir -p "$APP/Contents/MacOS" "$APP/Contents/Resources" +rm -f "$APP/Contents/MacOS/UntoldEditor" +cp "$BIN_DIR/UntoldEditor" "$APP/Contents/MacOS/UntoldEditor" +cp Sources/UntoldEditor/Info.plist "$APP/Contents/Info.plist" +if [ -f Resources/AppIcon.icns ]; then + cp Resources/AppIcon.icns "$APP/Contents/Resources/AppIcon.icns" + /usr/libexec/PlistBuddy -c "Add :CFBundleIconFile string AppIcon" "$APP/Contents/Info.plist" > /dev/null +fi +# The resource bundles of the engine and the editor, as the build folder has them (for the +# `Bundle.module` accessors), and their files flattened into Resources as the packaged app has +# them (for the lookups through `Bundle.main`); the build products for the Component SDK; the +# exporter scripts. +for bundle in "$BIN_DIR"/*.bundle; do + ln -sfn "$bundle" "$APP/Contents/Resources/$(basename "$bundle")" + resource_root="$bundle" + if [ -d "$bundle/Contents/Resources" ]; then + resource_root="$bundle/Contents/Resources" + fi + for entry in "$resource_root"/*; do + [ -e "$entry" ] || continue + target="$APP/Contents/Resources/$(basename "$entry")" + if [ -e "$target" ] && [ ! -L "$target" ]; then + echo "warning: $(basename "$entry") of $(basename "$bundle") is left out: a file of that name is in the bundle already" >&2 + continue + fi + ln -sfn "$entry" "$target" + done +done +ln -sfn "$BIN_DIR" "$APP/Contents/Resources/BuildProducts" +SCRIPTS=.build/checkouts/UntoldEngine/scripts +if [ ! -d "$SCRIPTS" ]; then + SCRIPTS=../UntoldEngine/scripts +fi +if [ -d "$SCRIPTS" ]; then + ln -sfn "$(cd "$SCRIPTS" && pwd)" "$APP/Contents/Resources/scripts" +fi +/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister -f "$APP" + +echo "$APP" +if [ "$RUN" = 1 ]; then + exec "$APP/Contents/MacOS/UntoldEditor" +fi From c0a0ee5ffa49a1669ccc70f1071570c0f9b30709 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Tue, 6 Oct 2026 19:18:04 +0200 Subject: [PATCH 2/3] [Feature] Preview the scene on an Apple Vision Pro, drawn by this Mac macOS 26 lets a Mac app open a `RemoteImmersiveSpace` on a Vision Pro nearby and draw it with `CompositorLayer`, as a visionOS app would. The editor is a SwiftUI `App` around its AppKit delegate, window and menus, and declares that space. View > Preview on Apple Vision Pro, or the Preview pill beside the build target, asks for it; the headset's wearer accepts; the scene shows in full immersion, drawn by the editor's own renderer through the engine's public XR API, on a thread of its own, frame by frame as the headset asks. - The headset rides the editor's camera. W A S D fly it level, Q E up and down, the pointer turns it as a second head, hidden and held while the preview runs, on top of the wearer's own head. Esc ends the preview. A card over the viewport says so. - The Mac's viewport mirrors the left eye; the panels edit the live scene; clicks select nothing and the overlays are off. Play, a scene load and the headset closing the space end it. - Steering and drawing take turns under a lock, the camera's pose is read from its position and rotation, and the engine's own per-frame fly and its head write stand down meanwhile. - The two frameworks are weak-linked: the editor still starts on macOS 14. - Tests on a fake frame source: the placement, the flight, the loop rendering into textures, the mirror under Metal's validation, the session's states. Plan and decisions: EditorVisionProPreview.md in the engine's proposal drafts. --- .../Controls/EditorPillTextButton.swift | 55 +++ .../Editor/Toolbar/EditorToolbarView.swift | 5 +- .../Editor/Toolbar/PreviewControl.swift | 80 ++++ .../Editor/Toolbar/PreviewDestination.swift | 42 ++ .../Editor/Viewport/ViewportCamera.swift | 24 +- .../VisionPro/CompositorFrameSource.swift | 170 ++++++++ .../Editor/VisionPro/EngineStereoMode.swift | 49 +++ .../Editor/VisionPro/VisionProFlight.swift | 60 +++ .../VisionPro/VisionProFrameSource.swift | 58 +++ .../Editor/VisionPro/VisionProMirror.swift | 225 +++++++++++ .../Editor/VisionPro/VisionProPlacement.swift | 60 +++ .../VisionPro/VisionProPointerCapture.swift | 111 ++++++ .../VisionPro/VisionProPreviewActions.swift | 58 +++ .../VisionPro/VisionProPreviewBridge.swift | 24 ++ .../VisionPro/VisionProPreviewContent.swift | 50 +++ .../VisionPro/VisionProPreviewHints.swift | 57 +++ .../VisionPro/VisionProPreviewLabel.swift | 54 +++ .../VisionPro/VisionProPreviewLoop.swift | 261 ++++++++++++ .../VisionPro/VisionProPreviewScene.swift | 29 ++ .../VisionPro/VisionProPreviewSession.swift | 246 ++++++++++++ .../VisionPro/VisionProPreviewState.swift | 52 +++ .../Editor/Window/EditorView+PlayMode.swift | 2 + .../Editor/Window/EditorView+Viewport.swift | 13 + .../Window/EditorView+ViewportControls.swift | 8 +- .../Editor/Window/EditorView.swift | 2 + .../Renderer/EditorUntoldRenderer.swift | 8 + .../Systems/EditorInputSystemAppKit.swift | 40 +- Sources/UntoldEditor/UntoldEditorApp.swift | 44 ++- .../EditorRenderingSystemTests.swift | 14 + .../ViewportCameraTests.swift | 11 + .../VisionProFakeFrames.swift | 159 ++++++++ .../VisionProFlightTests.swift | 53 +++ .../VisionProMirrorTests.swift | 131 +++++++ .../VisionProPlacementTests.swift | 100 +++++ .../VisionProPreviewLoopTests.swift | 370 ++++++++++++++++++ .../VisionProPreviewSessionTests.swift | 228 +++++++++++ 36 files changed, 2926 insertions(+), 27 deletions(-) create mode 100644 Sources/UntoldEditor/Editor/Shared/Controls/EditorPillTextButton.swift create mode 100644 Sources/UntoldEditor/Editor/Toolbar/PreviewControl.swift create mode 100644 Sources/UntoldEditor/Editor/Toolbar/PreviewDestination.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/CompositorFrameSource.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/EngineStereoMode.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/VisionProFlight.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/VisionProFrameSource.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/VisionProMirror.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/VisionProPlacement.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/VisionProPointerCapture.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewActions.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewBridge.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewContent.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewHints.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewLabel.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewLoop.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewScene.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewSession.swift create mode 100644 Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewState.swift create mode 100644 Tests/UntoldEditorTests/VisionProFakeFrames.swift create mode 100644 Tests/UntoldEditorTests/VisionProFlightTests.swift create mode 100644 Tests/UntoldEditorTests/VisionProMirrorTests.swift create mode 100644 Tests/UntoldEditorTests/VisionProPlacementTests.swift create mode 100644 Tests/UntoldEditorTests/VisionProPreviewLoopTests.swift create mode 100644 Tests/UntoldEditorTests/VisionProPreviewSessionTests.swift diff --git a/Sources/UntoldEditor/Editor/Shared/Controls/EditorPillTextButton.swift b/Sources/UntoldEditor/Editor/Shared/Controls/EditorPillTextButton.swift new file mode 100644 index 0000000..7f1e916 --- /dev/null +++ b/Sources/UntoldEditor/Editor/Shared/Controls/EditorPillTextButton.swift @@ -0,0 +1,55 @@ +// +// EditorPillTextButton.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// + +import SwiftUI + +/// A text button inside an `EditorPillGroup` that does something at once: an +/// optional glyph and the title, 26 pt, lit in the accent colour while what it +/// started goes on, with no fill of its own so the group's pill is the surface. +/// The preview control uses it. +struct EditorPillTextButton: View { + let title: String + var systemImage: String? + var isActive = false + var isEnabled = true + var help = "" + let action: () -> Void + + var body: some View { + Button(action: action) { + HStack(spacing: 6) { + if let systemImage { + Image(systemName: systemImage) + .font(.system(size: 12, weight: .semibold)) + } + Text(title) + .font(EditorType.body) + .lineLimit(1) + .fixedSize() + } + .foregroundColor(color) + .padding(.horizontal, 10) + .frame(height: 26) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .focusable(false) + .disabled(isEnabled == false) + .help(help) + } + + private var color: Color { + if isEnabled == false { + return Color.editorTextDisabled + } + return isActive ? Color.editorAccent : Color.editorTextPrimary + } +} diff --git a/Sources/UntoldEditor/Editor/Toolbar/EditorToolbarView.swift b/Sources/UntoldEditor/Editor/Toolbar/EditorToolbarView.swift index 92a50aa..aac35f9 100644 --- a/Sources/UntoldEditor/Editor/Toolbar/EditorToolbarView.swift +++ b/Sources/UntoldEditor/Editor/Toolbar/EditorToolbarView.swift @@ -12,7 +12,7 @@ import SwiftUI /// The toolbar row that shares the window's title bar: the traffic lights on /// the left (drawn by macOS), the project chip, Undo / Redo / History, the play -/// controls in the middle, and the build target on the right. The search field +/// controls in the middle, and the preview and the build target on the right. The search field /// of the mockup joins it with the command palette (stage 1.9). Global chrome: /// it shows in every experience mode. Dragging its empty space moves the /// window, as the title bar it replaces did. @@ -44,8 +44,11 @@ struct EditorToolbarView: View { ) UndoRedoControls() Spacer(minLength: 0) + PreviewControl() BuildTargetMenu(target: $buildTarget) + .padding(.leading, 12) } + .background(VisionProPreviewBridge()) .padding(.trailing, 12) .frame(maxWidth: .infinity) .frame(height: Self.height) diff --git a/Sources/UntoldEditor/Editor/Toolbar/PreviewControl.swift b/Sources/UntoldEditor/Editor/Toolbar/PreviewControl.swift new file mode 100644 index 0000000..abe4df7 --- /dev/null +++ b/Sources/UntoldEditor/Editor/Toolbar/PreviewControl.swift @@ -0,0 +1,80 @@ +// +// PreviewControl.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// + +import SwiftUI + +/// The preview pill at the right of the toolbar, beside the build target: one +/// click shows the scene on the chosen destination, the next ends it. Wide, +/// so that it can be hit with a headset on; lit while the preview goes on. +/// The destination is picked from a chevron once there is more than one to +/// pick from. Hidden where this Mac has no destination, as the View menu's +/// item is. +struct PreviewControl: View { + @ObservedObject private var session = VisionProPreviewSession.shared + @ObservedObject private var playback = EditorPlaybackSettings.shared + @State private var destination: PreviewDestination = .visionPro + @State private var showsDestinations = false + + var body: some View { + let destinations = PreviewDestination.available(visionPro: session.state) + if destinations.isEmpty == false { + let item = VisionProPreviewSession.menuItem(for: session.state, isPlaying: ViewportCameras.isPlaying) + EditorPillGroup { + EditorPillTextButton( + title: Self.title(for: session.state), + systemImage: destination.systemImage, + isActive: session.state == .connecting || session.state == .previewing, + isEnabled: item.isEnabled, + help: item.title, + action: session.toggle + ) + if destinations.count > 1 { + EditorPillMenuButton( + title: destination.title, + isEnabled: session.state == .idle, + help: "Where the preview shows the scene" + ) { + showsDestinations.toggle() + } + .popover(isPresented: $showsDestinations, arrowEdge: .bottom) { + VStack(alignment: .leading, spacing: 2) { + ForEach(destinations) { choice in + Button { + destination = choice + showsDestinations = false + } label: { + Label(choice.title, systemImage: choice.systemImage) + .frame(maxWidth: .infinity, alignment: .leading) + } + .buttonStyle(.plain) + } + } + .padding(8) + } + } + } + } + } + + /// What the button reads in each state of the preview. + static func title(for state: VisionProPreviewSession.State) -> String { + switch state { + case .unavailable, .idle: + return "Preview" + case .connecting: + return "Connecting…" + case .previewing: + return "Stop Preview" + case .ending: + return "Ending…" + } + } +} diff --git a/Sources/UntoldEditor/Editor/Toolbar/PreviewDestination.swift b/Sources/UntoldEditor/Editor/Toolbar/PreviewDestination.swift new file mode 100644 index 0000000..f9744de --- /dev/null +++ b/Sources/UntoldEditor/Editor/Toolbar/PreviewDestination.swift @@ -0,0 +1,42 @@ +// +// PreviewDestination.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// + +import Foundation + +/// Where the toolbar's preview shows the scene, besides the viewport: an +/// Apple Vision Pro nearby. A window of its own and the full screen are the +/// destinations to come; the control offers the choice once there is one. +enum PreviewDestination: String, CaseIterable, Identifiable { + case visionPro + + var id: String { + rawValue + } + + var title: String { + switch self { + case .visionPro: + return "Apple Vision Pro" + } + } + + var systemImage: String { + switch self { + case .visionPro: + return "visionpro" + } + } + + /// The destinations this Mac can preview on now. + static func available(visionPro: VisionProPreviewSession.State) -> [PreviewDestination] { + visionPro == .unavailable ? [] : [.visionPro] + } +} diff --git a/Sources/UntoldEditor/Editor/Viewport/ViewportCamera.swift b/Sources/UntoldEditor/Editor/Viewport/ViewportCamera.swift index ddbe22e..f8c832c 100644 --- a/Sources/UntoldEditor/Editor/Viewport/ViewportCamera.swift +++ b/Sources/UntoldEditor/Editor/Viewport/ViewportCamera.swift @@ -84,10 +84,12 @@ enum ViewportCameras { /// The camera the keys and the mouse steer now, or nil when they steer /// none. While editing it is the editor's, and a game camera shown as a - /// locked preview stays where it is. While playing it is the camera the - /// viewport shows: the game's, unless View > Steer the Camera While - /// Playing is off and the game steers it alone, or the editor's when the - /// View menu keeps the viewport on it. + /// locked preview stays where it is. While a headset previews the scene it + /// is the editor's too, which carries the headset: the keys and the mouse + /// fly it, and the headset sees the scene from there. While playing it is + /// the camera the viewport shows: the game's, unless View > Steer the + /// Camera While Playing is off and the game steers it alone, or the + /// editor's when the View menu keeps the viewport on it. static var steered: EntityID? { if let active = CameraSystem.shared.activeCamera, isGameCamera(active) { return isPlaying && playback.steersCameraWhilePlaying ? active : nil @@ -95,13 +97,19 @@ enum ViewportCameras { return findSceneCamera() } - /// True while the viewport is a locked preview of a game camera: the mouse - /// and the keys move nothing, clicks select nothing and the editor draws - /// none of its overlays, until the editor's camera is chosen again. + /// True while the viewport is a locked preview of a game camera, or while + /// a headset previews the scene: clicks select nothing and the editor + /// draws none of its overlays, until the editor's camera is chosen again + /// or the preview ends. The keys and the mouse move nothing in a game + /// camera's preview; with a headset they fly the editor's camera. static var isLockedPreview: Bool { - current != .editor + current != .editor || isPreviewingOnHeadset() } + /// Whether the scene is previewed on a headset, which the editor's camera + /// carries then. Tests put their own answer here. + static var isPreviewingOnHeadset: () -> Bool = { VisionProPreviewState.shared.isPreviewing } + /// An entity is about to leave the scene. A game camera the viewport is /// locked on, or an ancestor of it, which goes with it, gives the viewport /// back to the editor's camera first: the engine never clears its active diff --git a/Sources/UntoldEditor/Editor/VisionPro/CompositorFrameSource.swift b/Sources/UntoldEditor/Editor/VisionPro/CompositorFrameSource.swift new file mode 100644 index 0000000..909d7ad --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/CompositorFrameSource.swift @@ -0,0 +1,170 @@ +// +// CompositorFrameSource.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +#if canImport(CompositorServices) && canImport(ARKit) + @_weakLinked import ARKit + @_weakLinked import CompositorServices + import Foundation + import Metal + import QuartzCore + import simd + import SwiftUI + import UntoldEngine + + /// The headset's frames through CompositorServices and its pose through + /// ARKit, for a Mac on macOS 26 drawing for an Apple Vision Pro nearby. + @available(macOS 26.0, *) + final class CompositorFrameSource: VisionProFrameSource, @unchecked Sendable { + private let layerRenderer: LayerRenderer + private let session: ARKitSession + private let worldTracking = WorldTrackingProvider() + private let lock = NSLock() + private var stopped = false + private var lastAnchor: DeviceAnchor? + + init(layerRenderer: LayerRenderer, device: RemoteDeviceIdentifier) { + self.layerRenderer = layerRenderer + session = ARKitSession(device: device) + let session = session + let worldTracking = worldTracking + Task { + do { + try await session.run([worldTracking]) + } catch { + Logger.log(message: "Vision Pro preview: the headset's tracking could not start (\(error.localizedDescription)).") + } + } + } + + private var isStopped: Bool { + lock.lock() + defer { lock.unlock() } + return stopped + } + + func nextFrame() -> VisionProFrame? { + while isStopped == false { + switch layerRenderer.state { + case .paused: + // Not `waitUntilRunning()`: a stop has to be seen meanwhile. + Thread.sleep(forTimeInterval: 0.01) + case .running: + if let frame = layerRenderer.queryNextFrame() { + return CompositorFrame(frame: frame, layerRenderer: layerRenderer, source: self) + } + case .invalidated: + return nil + @unknown default: + return nil + } + } + return nil + } + + func stop() { + lock.lock() + stopped = true + lock.unlock() + session.stop() + } + + /// The headset's pose for a moment, else the last one known. + func deviceAnchor(for time: TimeInterval) -> DeviceAnchor? { + if worldTracking.state == .running, + let anchor = worldTracking.queryDeviceAnchor(atTimestamp: time) ?? worldTracking.queryDeviceAnchor(atTimestamp: CACurrentMediaTime()) + { + lock.lock() + lastAnchor = anchor + lock.unlock() + return anchor + } + lock.lock() + defer { lock.unlock() } + return lastAnchor + } + } + + /// One of the compositor's frames. + @available(macOS 26.0, *) + final class CompositorFrame: VisionProFrame { + private let frame: LayerRenderer.Frame + private let layerRenderer: LayerRenderer + private let source: CompositorFrameSource + private var timing: LayerRenderer.Frame.Timing? + private var drawables: [LayerRenderer.Drawable] = [] + + init(frame: LayerRenderer.Frame, layerRenderer: LayerRenderer, source: CompositorFrameSource) { + self.frame = frame + self.layerRenderer = layerRenderer + self.source = source + } + + func beginUpdate() { + timing = frame.predictTiming() + frame.startUpdate() + } + + func endUpdate() { + frame.endUpdate() + } + + func waitForDrawingTime() { + guard let timing else { return } + LayerRenderer.Clock().wait(until: timing.optimalInputTime, tolerance: .zero) + } + + func beginDrawing() -> Bool { + guard layerRenderer.state == .running else { return false } + frame.startSubmission() + return true + } + + func acquireEyes() -> VisionProFrameEyes? { + // The headset is the first drawable; another target the compositor + // hands over is presented and left blank. + drawables = frame.queryDrawables() + guard let drawable = drawables.first else { + return nil + } + let presentationTime = timing.map { Self.time(of: $0.presentationTime) } ?? CACurrentMediaTime() + guard let anchor = source.deviceAnchor(for: presentationTime) else { + return VisionProFrameEyes(originFromDevice: nil, eyes: []) + } + drawable.deviceAnchor = anchor + let originFromDevice = anchor.originFromAnchorTransform + let eyes = drawable.views.indices.map { index in + VisionProEye( + colorTexture: drawable.colorTextures[index], + depthTexture: drawable.depthTextures[index], + viewFromOrigin: simd_inverse(simd_mul(originFromDevice, drawable.views[index].transform)), + projection: drawable.computeProjection(convention: .rightUpBack, viewIndex: index) + ) + } + return VisionProFrameEyes(originFromDevice: originFromDevice, eyes: eyes) + } + + func present(commandBuffer: MTLCommandBuffer) { + for drawable in drawables { + drawable.encodePresent(commandBuffer: commandBuffer) + } + } + + func endDrawing() { + guard layerRenderer.state == .running else { return } + frame.endSubmission() + } + + /// A moment of the compositor's clock as `CACurrentMediaTime` measures it. + private static func time(of instant: LayerRenderer.Clock.Instant) -> TimeInterval { + let ahead = LayerRenderer.Clock().now.duration(to: instant) + return CACurrentMediaTime() + Double(ahead.components.seconds) + Double(ahead.components.attoseconds) / 1e18 + } + } +#endif diff --git a/Sources/UntoldEditor/Editor/VisionPro/EngineStereoMode.swift b/Sources/UntoldEditor/Editor/VisionPro/EngineStereoMode.swift new file mode 100644 index 0000000..8b12ca1 --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/EngineStereoMode.swift @@ -0,0 +1,49 @@ +// +// EngineStereoMode.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +import MetalKit +import simd +import UntoldEngine + +/// The engine's render targets for the headset or for the viewport. The +/// engine makes a renderer of its own for a headset (`createXR`), which the +/// editor cannot use: it would be a second renderer over the open scene, +/// with a camera and a light of its own. The editor keeps its one renderer +/// and switches its targets: two eyes of the headset's size, or the viewport. +extension UntoldRenderer { + /// Switches to stereo. The eyes' size is not known until the headset + /// hands over its first frame; `fitStereoTargets(to:)` sizes them then. + func enterStereo() { + renderInfo.isXRStereoMode = true + renderInfo.currentEye = 0 + } + + /// Sizes the targets to one eye, when the eye's size is not what they + /// have. True when they were made again. + @discardableResult + func fitStereoTargets(to eyeSize: simd_float2) -> Bool { + guard renderInfo.viewPort != eyeSize else { + return false + } + renderInfo.viewPort = eyeSize + initSizeableResources() + return true + } + + /// Switches back to the viewport's targets, at its size, and to its + /// projection. + func leaveStereo(viewport: MTKView) { + renderInfo.isXRStereoMode = false + renderInfo.currentEye = 0 + renderInfo.viewPort = simd_float2(Float(viewport.drawableSize.width), Float(viewport.drawableSize.height)) + initSizeableResources() + mtkView(viewport, drawableSizeWillChange: viewport.drawableSize) + } +} diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProFlight.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProFlight.swift new file mode 100644 index 0000000..662287b --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProFlight.swift @@ -0,0 +1,60 @@ +// +// VisionProFlight.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// + +import simd +import UntoldEngine + +/// How the keys fly the camera the headset rides: level. W and S go along +/// the level part of where the camera faces, A and D to its sides, Q and E +/// straight up and down, at the editor's flying speed. The editor's own +/// flying goes where the camera looks, which with a headset on tilts the +/// floor under the wearer. +enum VisionProFlight { + struct Keys: Equatable { + var w = false + var a = false + var s = false + var d = false + var q = false + var e = false + } + + /// The editor flies `speed × 0.1` a frame at 60 frames a second. + static let unitsPerSecondPerSpeed: Float = 6 + + /// The level directions ahead and to the right of a camera facing `forward`. + static func levelAxes(forward: simd_float3) -> (ahead: simd_float3, right: simd_float3) { + let yaw = VisionProPlacement.levelYaw(of: forward) + return (simd_float3(-sin(yaw), 0, -cos(yaw)), simd_float3(cos(yaw), 0, -sin(yaw))) + } + + /// Where the keys take a camera facing `forward` in `deltaTime` seconds. + static func displacement(keys: Keys, forward: simd_float3, speed: Float, deltaTime: Float) -> simd_float3 { + let axes = levelAxes(forward: forward) + let ahead: Float = (keys.w ? 1 : 0) - (keys.s ? 1 : 0) + let aside: Float = (keys.d ? 1 : 0) - (keys.a ? 1 : 0) + let up: Float = (keys.q ? 1 : 0) - (keys.e ? 1 : 0) + let direction = axes.ahead * ahead + axes.right * aside + simd_float3(0, up, 0) + return direction * (speed * unitsPerSecondPerSpeed * deltaTime) + } + + /// Flies `camera` by the keys held, for `deltaTime` seconds. + static func fly(camera: EntityID, keys: Keys, speed: Float, deltaTime: Float) { + guard let pose = VisionProPlacement.pose(of: camera) else { + return + } + let delta = displacement(keys: keys, forward: pose.forward, speed: speed, deltaTime: deltaTime) + guard simd_length_squared(delta) > 0 else { + return + } + cameraMoveBy(entityId: camera, delta: delta, space: .world) + } +} diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProFrameSource.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProFrameSource.swift new file mode 100644 index 0000000..5b89d8a --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProFrameSource.swift @@ -0,0 +1,58 @@ +// +// VisionProFrameSource.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +import Metal +import simd + +/// One eye of a frame for the headset: the textures to draw it into, how +/// the eye sees the headset's own space, and its projection. +struct VisionProEye { + let colorTexture: MTLTexture + let depthTexture: MTLTexture + /// From the headset's origin, where its tracking began, into this eye's view. + let viewFromOrigin: simd_float4x4 + let projection: simd_float4x4 +} + +/// What a frame hands over to draw: where the headset is, from its origin, +/// and its eyes. No eyes while the headset is not tracked: the frame is then +/// presented with nothing drawn, as the compositor asks. +struct VisionProFrameEyes { + let originFromDevice: simd_float4x4? + let eyes: [VisionProEye] +} + +/// One frame the headset asked for, through the phases the compositor +/// wants them in: an update, a wait for the right moment, then the drawing. +protocol VisionProFrame: AnyObject { + func beginUpdate() + func endUpdate() + /// Waits until the moment the compositor wants the drawing to begin. + func waitForDrawingTime() + /// Begins the drawing; false when the frame is no longer wanted. + func beginDrawing() -> Bool + /// The eyes to draw, or nil when the compositor gives none for this + /// frame: nothing is drawn or presented then, and the drawing is not + /// ended. + func acquireEyes() -> VisionProFrameEyes? + /// Hands the eyes to the headset, encoded after the drawing. + func present(commandBuffer: MTLCommandBuffer) + func endDrawing() +} + +/// Where the frames come from: the headset through CompositorServices, or +/// a test with textures of its own. +protocol VisionProFrameSource: AnyObject { + /// The next frame, waiting for the headset to ask for one; nil once the + /// headset is gone or the source was stopped. + func nextFrame() -> VisionProFrame? + /// Ends the source: tracking stops and `nextFrame` returns nil. + func stop() +} diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProMirror.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProMirror.swift new file mode 100644 index 0000000..7ec17ab --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProMirror.swift @@ -0,0 +1,225 @@ +// +// VisionProMirror.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +import Metal +import MetalKit +import simd +import UntoldEngine + +/// What the Mac's viewport shows while the headset shows the scene: the +/// left eye, copied after each of the headset's frames and drawn into the +/// viewport with its own proportions kept. The viewport's Metal view draws +/// through it meanwhile, instead of through the engine, which the headset's +/// loop drives then. +final class VisionProMirror: NSObject, MTKViewDelegate { + static let shaderSource = """ + #include + using namespace metal; + + struct MirrorVertexOut { + float4 position [[position]]; + float2 uv; + }; + + // One triangle over the whole view; the eye lies in the middle of it at + // its own proportions, `fit` being the part of the view it takes each way. + vertex MirrorVertexOut visionProMirrorVertex(uint id [[vertex_id]], constant float2 &fit [[buffer(0)]]) { + float2 corner = float2((id << 1) & 2, id & 2); + MirrorVertexOut out; + out.position = float4(corner * 2.0 - 1.0, 0.0, 1.0); + float2 uv = float2(corner.x, 1.0 - corner.y); + out.uv = (uv - 0.5) / fit + 0.5; + return out; + } + + fragment float4 visionProMirrorFragment( + MirrorVertexOut in [[stage_in]], + texture2d eye [[texture(0)]], + constant float4 &background [[buffer(0)]]) + { + if (any(in.uv < 0.0) || any(in.uv > 1.0)) { + return background; + } + constexpr sampler linearSampler(filter::linear, address::clamp_to_edge); + return float4(eye.sample(linearSampler, in.uv).rgb, 1.0); + } + """ + + private let device: MTLDevice + private let lock = NSLock() + /// Run on the main thread before each of the viewport's frames, as the + /// engine's own draw would run the editor's per-frame input: the session + /// puts the key-up check there. + var onFrame: (() -> Void)? + /// Two copies, written in turn, so the one on screen is never the one + /// being written. + private var copies: [MTLTexture] = [] + private var nextCopy = 0 + private var latest: MTLTexture? + private var pipeline: MTLRenderPipelineState? + private var pipelineFormats: (color: MTLPixelFormat, depthStencil: MTLPixelFormat)? + + init(device: MTLDevice) { + self.device = device + } + + /// The eye the viewport shows, once the GPU has copied it. + var latestEye: MTLTexture? { + lock.lock() + defer { lock.unlock() } + return latest + } + + /// Copies the eye in the headset's own command buffer; the viewport shows + /// the copy once that buffer is done. + func copy(eye: MTLTexture, commandBuffer: MTLCommandBuffer) { + guard let target = copyTexture(matching: eye), let blit = commandBuffer.makeBlitCommandEncoder() else { + return + } + blit.label = "Vision Pro Mirror Copy" + blit.copy(from: eye, to: target) + blit.endEncoding() + commandBuffer.addCompletedHandler { [weak self] _ in + guard let self else { return } + lock.lock() + latest = target + lock.unlock() + } + } + + private func copyTexture(matching eye: MTLTexture) -> MTLTexture? { + lock.lock() + defer { lock.unlock() } + if copies.count != 2 || copies[0].width != eye.width || copies[0].height != eye.height || copies[0].pixelFormat != eye.pixelFormat { + let descriptor = MTLTextureDescriptor.texture2DDescriptor( + pixelFormat: eye.pixelFormat, width: eye.width, height: eye.height, mipmapped: false + ) + descriptor.usage = [.shaderRead] + descriptor.storageMode = .private + copies = (0 ..< 2).compactMap { index in + let texture = device.makeTexture(descriptor: descriptor) + texture?.label = "Vision Pro Mirror \(index)" + return texture + } + latest = nil + nextCopy = 0 + } + guard copies.count == 2 else { + return nil + } + let target = copies[nextCopy] + nextCopy = (nextCopy + 1) % 2 + return target + } + + /// How much of the view the eye takes each way, keeping its proportions. + static func fit(eye: CGSize, in view: CGSize) -> simd_float2 { + guard eye.width > 0, eye.height > 0, view.width > 0, view.height > 0 else { + return simd_float2(1, 1) + } + let eyeAspect = eye.width / eye.height + let viewAspect = view.width / view.height + if eyeAspect > viewAspect { + return simd_float2(1, Float(viewAspect / eyeAspect)) + } + return simd_float2(Float(eyeAspect / viewAspect), 1) + } + + // MARK: - MTKViewDelegate + + func mtkView(_: MTKView, drawableSizeWillChange _: CGSize) {} + + func draw(in view: MTKView) { + onFrame?() + guard let drawable = view.currentDrawable, + let descriptor = view.currentRenderPassDescriptor, + let commandBuffer = renderInfo.commandQueue.makeCommandBuffer() + else { + return + } + commandBuffer.label = "Vision Pro Mirror" + encode( + into: descriptor, size: view.drawableSize, + colorFormat: view.colorPixelFormat, depthStencilFormat: view.depthStencilPixelFormat, + commandBuffer: commandBuffer + ) + commandBuffer.present(drawable) + commandBuffer.commit() + } + + /// Draws the latest eye over the background, or the background alone, + /// into the pass described: the view's drawable, or a test's textures. + func encode( + into descriptor: MTLRenderPassDescriptor, size: CGSize, + colorFormat: MTLPixelFormat, depthStencilFormat: MTLPixelFormat, + commandBuffer: MTLCommandBuffer + ) { + let background = EditorViewportResizePolicy.exposedBackgroundColor + descriptor.colorAttachments[0].loadAction = .clear + descriptor.colorAttachments[0].clearColor = MTLClearColor( + red: Double(background.x), green: Double(background.y), blue: Double(background.z), alpha: 1 + ) + // The mirror writes no depth and no stencil. + descriptor.depthAttachment.loadAction = .dontCare + descriptor.depthAttachment.storeAction = .dontCare + descriptor.stencilAttachment.loadAction = .dontCare + descriptor.stencilAttachment.storeAction = .dontCare + guard let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: descriptor) else { + return + } + if let eye = latestEye, + let pipeline = makePipelineIfNeeded(colorFormat: colorFormat, depthStencilFormat: depthStencilFormat) + { + var fit = Self.fit(eye: CGSize(width: eye.width, height: eye.height), in: size) + var backgroundColor = background + encoder.setRenderPipelineState(pipeline) + encoder.setVertexBytes(&fit, length: MemoryLayout.stride, index: 0) + encoder.setFragmentBytes(&backgroundColor, length: MemoryLayout.stride, index: 0) + encoder.setFragmentTexture(eye, index: 0) + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) + } + encoder.endEncoding() + } + + private func makePipelineIfNeeded(colorFormat: MTLPixelFormat, depthStencilFormat: MTLPixelFormat) -> MTLRenderPipelineState? { + if let pipeline, let formats = pipelineFormats, formats.color == colorFormat, formats.depthStencil == depthStencilFormat { + return pipeline + } + do { + let library = try device.makeLibrary(source: Self.shaderSource, options: nil) + let descriptor = Self.pipelineDescriptor(colorFormat: colorFormat, depthStencilFormat: depthStencilFormat, library: library) + let made = try device.makeRenderPipelineState(descriptor: descriptor) + pipeline = made + pipelineFormats = (colorFormat, depthStencilFormat) + return made + } catch { + Logger.log(message: "Vision Pro preview: the viewport cannot show the headset's eye (\(error.localizedDescription)).") + return nil + } + } + + /// The pipeline for a view's formats: its colour, and the depth and the + /// stencil its drawable comes with, which a pipeline has to name even + /// though the mirror writes neither. + static func pipelineDescriptor(colorFormat: MTLPixelFormat, depthStencilFormat: MTLPixelFormat, library: MTLLibrary) -> MTLRenderPipelineDescriptor { + let descriptor = MTLRenderPipelineDescriptor() + descriptor.label = "Vision Pro Mirror Pipeline" + descriptor.vertexFunction = library.makeFunction(name: "visionProMirrorVertex") + descriptor.fragmentFunction = library.makeFunction(name: "visionProMirrorFragment") + descriptor.colorAttachments[0].pixelFormat = colorFormat + if [.depth16Unorm, .depth32Float, .depth24Unorm_stencil8, .depth32Float_stencil8].contains(depthStencilFormat) { + descriptor.depthAttachmentPixelFormat = depthStencilFormat + } + if [.stencil8, .depth24Unorm_stencil8, .depth32Float_stencil8, .x32_stencil8, .x24_stencil8].contains(depthStencilFormat) { + descriptor.stencilAttachmentPixelFormat = depthStencilFormat + } + return descriptor + } +} diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProPlacement.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProPlacement.swift new file mode 100644 index 0000000..b8e3b20 --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProPlacement.swift @@ -0,0 +1,60 @@ +// +// VisionProPlacement.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +import simd +import UntoldEngine + +/// Where the headset stands in the scene. ARKit measures the headset from +/// its own origin, where its tracking began; the preview puts that origin +/// where it makes the headset, as it stood when the preview began, stand +/// where the editor's camera stood, facing where it faced. Only turns about +/// the vertical axis are used, so the floor stays level. +enum VisionProPlacement { + /// The turn about the vertical axis that takes the forward direction, + /// (0, 0, -1), to the level part of `direction`; zero for a direction + /// straight up or down. + static func levelYaw(of direction: simd_float3) -> Float { + guard direction.x * direction.x + direction.z * direction.z > 1e-8 else { + return 0 + } + return atan2(-direction.x, -direction.z) + } + + /// Where a camera stands and faces, from the two fields every way of + /// steering it writes: its position and its rotation. The camera's `eye` + /// and `target` are written by `cameraLookAt` alone, so the fly keys and + /// the mouse leave them behind. Nil for an entity that is no camera. + static func pose(of camera: EntityID) -> (eye: simd_float3, forward: simd_float3)? { + guard let component = scene.get(component: CameraComponent.self, for: camera) else { + return nil + } + // The camera looks down its own -Z; its rotation is the view's, from + // the world into the camera, so the inverse carries -Z out. + return (component.localPosition, simd_act(simd_conjugate(component.rotation), simd_float3(0, 0, -1))) + } + + /// The transform that carries the headset's own coordinates into the + /// scene's: the headset where it was, `originFromDevice`, lands on + /// `cameraEye` facing the level part of `cameraForward`. + static func sceneFromOrigin( + cameraEye: simd_float3, + cameraForward: simd_float3, + originFromDevice: simd_float4x4 + ) -> simd_float4x4 { + let devicePosition = simd_float3(originFromDevice.columns.3.x, originFromDevice.columns.3.y, originFromDevice.columns.3.z) + // The headset looks down its own -Z. + let deviceForward = -simd_float3(originFromDevice.columns.2.x, originFromDevice.columns.2.y, originFromDevice.columns.2.z) + let turn = simd_quatf(angle: levelYaw(of: cameraForward) - levelYaw(of: deviceForward), axis: simd_float3(0, 1, 0)) + return simd_mul( + simd_mul(matrix4x4Translation(cameraEye.x, cameraEye.y, cameraEye.z), simd_float4x4(turn)), + matrix4x4Translation(-devicePosition.x, -devicePosition.y, -devicePosition.z) + ) + } +} diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProPointerCapture.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProPointerCapture.swift new file mode 100644 index 0000000..8604cfa --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProPointerCapture.swift @@ -0,0 +1,111 @@ +// +// VisionProPointerCapture.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// + +import AppKit +import simd + +/// The pointer as a second head while the headset shows the scene: its +/// movement turns the view, with no button held, on top of what the real +/// head does, since looking back is hard sitting down. Taken as a game takes +/// it: the cursor hidden and held where it is, so that the movement never +/// leaves the viewport, and every mouse, Magic Mouse or finger on a trackpad +/// does the same. Let go while the editor is not the active app, and when +/// the preview ends. +final class VisionProPointerCapture { + /// The turn a movement of the pointer asks for, in the units of a look + /// drag: to the right for a move to the right, up for a move up (the + /// event's Y grows down the screen). + static func turn(forPointerDelta delta: simd_float2) -> simd_float2 { + guard delta.x.isFinite, delta.y.isFinite else { + return .zero + } + return simd_float2(delta.x, -delta.y) + } + + private(set) var isCapturing = false + private var isHolding = false + private var monitors: [Any] = [] + private var observers: [NSObjectProtocol] = [] + private let onTurn: (simd_float2) -> Void + + init(onTurn: @escaping (simd_float2) -> Void) { + self.onTurn = onTurn + } + + deinit { + stop() + } + + /// Takes the pointer: from now until `stop()`, its movement turns. + func start() { + guard isCapturing == false else { + return + } + isCapturing = true + let moves: NSEvent.EventTypeMask = [.mouseMoved, .leftMouseDragged, .rightMouseDragged, .otherMouseDragged] + if let monitor = NSEvent.addLocalMonitorForEvents(matching: moves, handler: { [weak self] event in + guard let self, isHolding else { + return event + } + onTurn(Self.turn(forPointerDelta: simd_float2(Float(event.deltaX), Float(event.deltaY)))) + return event + }) { + monitors.append(monitor) + } + let center = NotificationCenter.default + observers.append(center.addObserver(forName: NSApplication.didBecomeActiveNotification, object: nil, queue: .main) { [weak self] _ in + self?.hold() + }) + observers.append(center.addObserver(forName: NSApplication.didResignActiveNotification, object: nil, queue: .main) { [weak self] _ in + self?.release() + }) + if NSApp.isActive { + hold() + } + } + + /// Gives the pointer back. + func stop() { + guard isCapturing else { + return + } + release() + for monitor in monitors { + NSEvent.removeMonitor(monitor) + } + monitors = [] + for observer in observers { + NotificationCenter.default.removeObserver(observer) + } + observers = [] + isCapturing = false + } + + /// Hides the cursor and holds it where it is; the movement still arrives. + private func hold() { + guard isCapturing, isHolding == false else { + return + } + isHolding = true + NSApp.windows.forEach { $0.acceptsMouseMovedEvents = true } + CGAssociateMouseAndMouseCursorPosition(0) + NSCursor.hide() + } + + private func release() { + guard isHolding else { + return + } + isHolding = false + CGAssociateMouseAndMouseCursorPosition(1) + NSCursor.unhide() + } +} diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewActions.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewActions.swift new file mode 100644 index 0000000..015fd1d --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewActions.swift @@ -0,0 +1,58 @@ +// +// VisionProPreviewActions.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +import SwiftUI +import UntoldEngine + +/// The part of the bridge that has the actions: they are environment values +/// of a SwiftUI view, so a view keeps them for the session. +@available(macOS 26.0, *) +struct VisionProPreviewActions: View { + @Environment(\.supportsRemoteScenes) private var supportsRemoteScenes + @Environment(\.openImmersiveSpace) private var openImmersiveSpace + @Environment(\.dismissImmersiveSpace) private var dismissImmersiveSpace + + var body: some View { + Color.clear + .frame(width: 0, height: 0) + .onAppear(perform: handOver) + .onChange(of: supportsRemoteScenes) { _, _ in + handOver() + } + } + + private func handOver() { + let session = VisionProPreviewSession.shared + #if canImport(CompositorServices) + let openImmersiveSpace = openImmersiveSpace + session.openSpace = { + let result = await openImmersiveSpace(id: VisionProPreviewScene.id) + switch result { + case .opened: + return true + case .userCancelled: + Logger.log(message: "Vision Pro preview: the request was cancelled.") + return false + case .error: + Logger.log(message: "Vision Pro preview: the system could not open the space on a headset.") + return false + @unknown default: + Logger.log(message: "Vision Pro preview: the system answered \(result).") + return false + } + } + #endif + let dismissImmersiveSpace = dismissImmersiveSpace + session.dismissSpace = { + await dismissImmersiveSpace() + } + session.setAvailable(supportsRemoteScenes) + } +} diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewBridge.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewBridge.swift new file mode 100644 index 0000000..c4865e2 --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewBridge.swift @@ -0,0 +1,24 @@ +// +// VisionProPreviewBridge.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +import SwiftUI + +/// Hands SwiftUI's actions that open and close the remote space to the +/// session, which the View menu drives. Draws nothing. Before macOS 26 there +/// is nothing to hand over and the session stays unavailable. +struct VisionProPreviewBridge: View { + var body: some View { + if #available(macOS 26.0, *) { + VisionProPreviewActions() + } else { + Color.clear.frame(width: 0, height: 0) + } + } +} diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewContent.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewContent.swift new file mode 100644 index 0000000..aebbcd4 --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewContent.swift @@ -0,0 +1,50 @@ +// +// VisionProPreviewContent.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +#if canImport(CompositorServices) + @_weakLinked import CompositorServices + import SwiftUI + import UntoldEngine + + /// What the space draws: the editor's renderer through a compositor + /// layer, for the headset the system chose. + @available(macOS 26.0, *) + struct VisionProPreviewContent: CompositorContent { + @Environment(\.remoteDeviceIdentifier) private var remoteDevice + + var body: some CompositorContent { + CompositorLayer(configuration: VisionProLayerConfiguration()) { @MainActor layerRenderer in + guard let remoteDevice else { + Logger.log(message: "Vision Pro preview: the space opened without a headset.") + return + } + VisionProPreviewSession.shared.spaceDidOpen( + frames: CompositorFrameSource(layerRenderer: layerRenderer, device: remoteDevice) + ) + } + } + } + + /// The layer as the engine draws it: one texture per eye, the viewport's + /// formats, no foveation. + @available(macOS 26.0, *) + struct VisionProLayerConfiguration: CompositorLayerConfiguration { + func makeConfiguration(capabilities: LayerRenderer.Capabilities, configuration: inout LayerRenderer.Configuration) { + configuration.layout = .dedicated + configuration.isFoveationEnabled = false + if capabilities.supportedColorFormats(options: []).contains(.bgra8Unorm_srgb) { + configuration.colorFormat = .bgra8Unorm_srgb + } + if capabilities.supportedDepthFormats.contains(.depth32Float) { + configuration.depthFormat = .depth32Float + } + } + } +#endif diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewHints.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewHints.swift new file mode 100644 index 0000000..12666da --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewHints.swift @@ -0,0 +1,57 @@ +// +// VisionProPreviewHints.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// + +import SwiftUI + +/// How to move while the headset shows the scene, under the label over the +/// viewport: the keys, the trackpad, and the way out. Large enough to read +/// through the headset. +struct VisionProPreviewHints: View { + struct Line: Identifiable { + let keys: String + let does: String + + var id: String { + keys + } + } + + static let lines = [ + Line(keys: "Move the mouse, or a finger", does: "Turn the view, as your head does"), + Line(keys: "W A S D", does: "Move along the floor, where you face"), + Line(keys: "Q E", does: "Up and down"), + Line(keys: "Esc", does: "End the preview"), + ] + + var body: some View { + VStack(alignment: .leading, spacing: 5) { + ForEach(Self.lines) { line in + HStack(alignment: .firstTextBaseline, spacing: 10) { + Text(line.keys) + .font(EditorType.badge) + .foregroundColor(.editorTextPrimary) + .frame(width: 200, alignment: .trailing) + Text(line.does) + .font(EditorType.hint) + .foregroundColor(.editorTextTertiary) + } + } + } + .padding(.horizontal, 14) + .padding(.vertical, 10) + .background(Color.editorScrim) + .overlay( + RoundedRectangle(cornerRadius: EditorType.Radius.card) + .stroke(Color.editorDivider, lineWidth: 1) + ) + .cornerRadius(EditorType.Radius.card) + } +} diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewLabel.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewLabel.swift new file mode 100644 index 0000000..8ce071c --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewLabel.swift @@ -0,0 +1,54 @@ +// +// VisionProPreviewLabel.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +import SwiftUI + +/// The label over the viewport while the scene is previewed on a headset: +/// what is going on, how the keys fly, and the way to stop. +struct VisionProPreviewLabel: View { + let state: VisionProPreviewSession.State + let onStop: () -> Void + + var body: some View { + HStack(spacing: 8) { + Image(systemName: "visionpro") + .font(.system(size: 11, weight: .semibold)) + .foregroundColor(.editorAccent) + Text(state == .connecting ? "Connecting to Apple Vision Pro" : "Previewing on Apple Vision Pro") + .font(EditorType.badge) + .foregroundColor(.editorTextPrimary) + .lineLimit(1) + Text(state == .connecting ? "Accept on the headset" : "Drawn by this Mac") + .font(EditorType.hint) + .foregroundColor(.editorTextTertiary) + .fixedSize() + Color.editorDivider.frame(width: 1, height: 14) + Button(action: onStop) { + Text("Stop") + .font(EditorType.badge) + .foregroundColor(.editorAccent) + .fixedSize() + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .focusable(false) + .help("Ends the preview; the viewport goes back to the editor's camera") + } + .padding(.horizontal, 12) + .frame(height: 28) + .background(Color.editorScrim) + .overlay( + RoundedRectangle(cornerRadius: EditorType.Radius.card) + .stroke(Color.editorDivider, lineWidth: 1) + ) + .cornerRadius(EditorType.Radius.card) + .help("The scene shows in the Apple Vision Pro, drawn by this Mac; the viewport shows the headset's left eye. The panels still edit the scene; the viewport takes no camera or scene input until the preview ends.") + } +} diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewLoop.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewLoop.swift new file mode 100644 index 0000000..2599319 --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewLoop.swift @@ -0,0 +1,261 @@ +// +// VisionProPreviewLoop.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +import Foundation +import Metal +import QuartzCore +import simd +import UntoldEngine + +/// Draws the scene for the headset, frame after frame, on a thread of its +/// own: the headset asks for frames at its own rate, and the compositor +/// wants each frame's update, wait and drawing in order. It drives the +/// engine as the engine's own visionOS loop does, through the engine's +/// public API: one update, the culling once, then each eye. +final class VisionProPreviewLoop { + /// The camera the headset rides, and where it stood when the preview + /// began. The headset, as it stood when its first tracked pose arrived, + /// stands where the camera stands and faces its way; the camera's pose is + /// read again every frame, so the keys and the mouse carry the headset. + struct Start { + /// The engine's active camera at the start, whose pose the headset + /// follows; the loop ends once another takes its place, as a scene + /// load or a change of camera does. Nil keeps the pose below, for tests. + var camera: EntityID? + let cameraEye: simd_float3 + let cameraForward: simd_float3 + } + + private let renderer: UntoldRenderer + private let frames: VisionProFrameSource + private let start: Start + private let mirror: VisionProMirror? + + private let lock = NSLock() + private var stopped = false + private var sceneFromOrigin: simd_float4x4? + private var originFromScene = matrix_identity_float4x4 + /// The headset's first tracked pose: what stands on the camera. + private var deviceStart: simd_float4x4? + private let passDescriptors = [MTLRenderPassDescriptor(), MTLRenderPassDescriptor()] + + /// Frames in which the eyes were drawn, for the tests. + private(set) var framesDrawn = 0 + + /// The fly keys held now and the editor's flying speed, read once a + /// frame; tests put their own here. + var keysHeld: () -> VisionProFlight.Keys = { + let state = InputSystem.shared.keyState + return VisionProFlight.Keys(w: state.wPressed, a: state.aPressed, s: state.sPressed, d: state.dPressed, q: state.qPressed, e: state.ePressed) + } + + var flyingSpeed: () -> Float = { EditorViewportSettings.shared.speedMultiplier } + private var lastFlight: CFTimeInterval? + + init(renderer: UntoldRenderer, frames: VisionProFrameSource, start: Start, mirror: VisionProMirror?) { + self.renderer = renderer + self.frames = frames + self.start = start + self.mirror = mirror + } + + /// Where the headset's coordinates land in the scene, once its first + /// tracked pose placed it on the camera; nil before. It follows the + /// camera frame by frame. + var placement: simd_float4x4? { + lock.lock() + defer { lock.unlock() } + return sceneFromOrigin + } + + private var isStopped: Bool { + lock.lock() + defer { lock.unlock() } + return stopped + } + + /// Draws frames until the source ends, `stop()` is called, or the camera + /// the headset drives is no longer the engine's active one. Blocking: + /// call it on a thread of its own. + func run() { + while isStopped == false, drivesTheActiveCamera, let frame = frames.nextFrame() { + autoreleasepool { + draw(frame) + } + } + } + + private var drivesTheActiveCamera: Bool { + guard let camera = start.camera else { return true } + return CameraSystem.shared.activeCamera == camera + } + + /// Ends the loop after the frame it is on. + func stop() { + lock.lock() + stopped = true + lock.unlock() + frames.stop() + } + + private func draw(_ frame: VisionProFrame) { + frame.beginUpdate() + fly() + // The progressive loader wants the main thread, as the engine's own + // loop gives it. + DispatchQueue.main.async { + ProgressiveAssetLoader.shared.tick() + } + let loading = AssetLoadingGate.shared.isLoadingAny + + // The engine's own loop would write the headset's position into the + // camera here, for streaming and LOD to follow it. Not this one: the + // camera is what the keys and the mouse fly and what the headset rides, + // read back every frame, so a write would feed the head's motion back + // into the placement. Streaming, LOD and shading go by the camera, + // which the headset stays within a room of. + if !loading { + renderer.updateXR(useExternalStatsLifecycle: true) + } + frame.endUpdate() + + frame.waitForDrawingTime() + guard isStopped == false, frame.beginDrawing() else { + return + } + // No eyes this frame: the compositor's frame is left as it is, as + // the engine's own loop leaves it. + guard let acquired = frame.acquireEyes() else { + return + } + submit(acquired, frame: frame, loading: loading) + frame.endDrawing() + renderer.finalizeXRStatsAndMonitors(frameStartTime: 0) + } + + private func submit(_ acquired: VisionProFrameEyes, frame: VisionProFrame, loading: Bool) { + commandBufferSemaphore.wait() + guard let commandBuffer = renderInfo.commandQueue.makeCommandBuffer() else { + commandBufferSemaphore.signal() + return + } + commandBuffer.label = "Vision Pro Preview" + renderInfo.currentInFlightFrameSlot = acquireUniformFrameSlot() + if let first = acquired.eyes.first { + renderer.fitStereoTargets(to: simd_float2(Float(first.colorTexture.width), Float(first.colorTexture.height))) + } + if !loading { + visibleEntityIds = tripleVisibleEntities.snapshotForRead(frame: cullFrameIndex) + } + + // No move of the camera from the keys or the mouse until both eyes + // are drawn and the frame is handed over: `VisionProPreviewState.cameraLock`. + let drawsEyes = acquired.originFromDevice != nil && acquired.eyes.isEmpty == false + if drawsEyes { + VisionProPreviewState.shared.cameraLock.lock() + } + defer { + if drawsEyes { + VisionProPreviewState.shared.cameraLock.unlock() + } + } + if let originFromDevice = acquired.originFromDevice, acquired.eyes.isEmpty == false { + place(originFromDevice) + + // The culling and the splat sort once for both eyes, as the + // engine's own loop does. + if !loading { + SceneRootTransform.shared.updateIfNeeded() + performFrustumCulling(commandBuffer: commandBuffer) + executeGaussianFrustumCulling(commandBuffer) + executeGaussianPreprocess(commandBuffer) + executeRadixSort(commandBuffer) + } + + for (index, eye) in acquired.eyes.enumerated() { + let descriptor = passDescriptors[min(index, passDescriptors.count - 1)] + descriptor.colorAttachments[0].texture = eye.colorTexture + descriptor.colorAttachments[0].loadAction = .clear + descriptor.colorAttachments[0].storeAction = .store + descriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 1) + descriptor.depthAttachment.texture = eye.depthTexture + descriptor.depthAttachment.loadAction = .clear + descriptor.depthAttachment.storeAction = .store + // Reverse Z, as the engine keeps depth. + descriptor.depthAttachment.clearDepth = 0 + + renderInfo.currentEye = index + renderer.renderXR( + commandBuffer: commandBuffer, + passDescriptor: descriptor, + viewMatrix: simd_mul(eye.viewFromOrigin, originFromScene), + projectionMatrix: eye.projection, + eyeIndex: index + ) + } + buildHZBDepthPyramid(commandBuffer) + if let left = acquired.eyes.first { + mirror?.copy(eye: left.colorTexture, commandBuffer: commandBuffer) + } + framesDrawn += 1 + } + + frame.present(commandBuffer: commandBuffer) + commandBuffer.addCompletedHandler { _ in + commandBufferSemaphore.signal() + } + commandBuffer.commit() + } + + /// Flies the camera the headset rides by the keys held, once a frame, by + /// the time since the last frame: level, as `VisionProFlight` has it. On + /// this thread, so that every frame of the headset sees a fresh step, and + /// between the eyes of two frames, as every move of the camera is. + private func fly() { + guard let camera = start.camera else { + return + } + let now = CACurrentMediaTime() + defer { lastFlight = now } + guard let last = lastFlight else { + return + } + let deltaTime = Float(min(max(now - last, 0), 0.1)) + VisionProPreviewState.shared.steerCamera { + VisionProFlight.fly(camera: camera, keys: keysHeld(), speed: flyingSpeed(), deltaTime: deltaTime) + } + } + + /// Puts the headset, as it first stood, on the camera as it stands now. + private func place(_ originFromDevice: simd_float4x4) { + let (eye, forward) = cameraPose() + lock.lock() + defer { lock.unlock() } + if deviceStart == nil { + deviceStart = originFromDevice + } + let placed = VisionProPlacement.sceneFromOrigin( + cameraEye: eye, + cameraForward: forward, + originFromDevice: deviceStart ?? originFromDevice + ) + sceneFromOrigin = placed + originFromScene = simd_inverse(placed) + } + + /// Where the camera stands and faces now: the camera the headset rides, + /// or the start's pose when there is none. + private func cameraPose() -> (eye: simd_float3, forward: simd_float3) { + guard let camera = start.camera, let pose = VisionProPlacement.pose(of: camera) else { + return (start.cameraEye, start.cameraForward) + } + return pose + } +} diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewScene.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewScene.swift new file mode 100644 index 0000000..ea0d34b --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewScene.swift @@ -0,0 +1,29 @@ +// +// VisionProPreviewScene.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +#if canImport(CompositorServices) + @_weakLinked import CompositorServices + import SwiftUI + import UntoldEngine + + /// The space the headset shows: the scene, drawn by this Mac. The app + /// declares it; the session opens and closes it. + @available(macOS 26.0, *) + struct VisionProPreviewScene: SwiftUI.Scene { + static let id = "untold.editor.visionProPreview" + + var body: some SwiftUI.Scene { + RemoteImmersiveSpace(id: Self.id) { + VisionProPreviewContent() + } + .immersionStyle(selection: .constant(.full), in: .full) + } + } +#endif diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewSession.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewSession.swift new file mode 100644 index 0000000..2b490e8 --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewSession.swift @@ -0,0 +1,246 @@ +// +// VisionProPreviewSession.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +import AppKit +import Combine +import Foundation +import MetalKit +import simd +import UntoldEngine + +/// The preview of the scene on an Apple Vision Pro: asked for from the View +/// menu, shown once the headset's wearer accepts, ended from the menu or by +/// the headset. It owns the loop that draws for the headset and puts the +/// editor back as it was. Main thread only, as the editor's other settings. +final class VisionProPreviewSession: ObservableObject { + static let shared = VisionProPreviewSession() + + enum State: Equatable { + /// This Mac cannot show a scene on a headset: before macOS 26, or + /// the system says so. + case unavailable + case idle + /// Asked for; the headset's wearer is being asked to accept. + case connecting + case previewing + case ending + } + + @Published private(set) var state: State = .unavailable { + didSet { + // Escape ends a preview from the moment it is asked for. + if state == .connecting { + watchEscape() + } else if state == .idle || state == .unavailable { + unwatchEscape() + } + } + } + + private var escapeMonitor: Any? + + /// Opens the remote space; true when it opened. Handed in by the bridge + /// view, which has SwiftUI's action for it. + var openSpace: (() async -> Bool)? + /// Closes the remote space. Handed in by the bridge view too. + var dismissSpace: (() async -> Void)? + /// The renderer that draws the viewport, handed in by the editor view. + weak var renderer: UntoldRenderer? + + private var loop: VisionProPreviewLoop? + private var loopDone: DispatchSemaphore? + private var mirror: VisionProMirror? + private var pointer: VisionProPointerCapture? + + var isPreviewing: Bool { + state == .previewing + } + + /// Whether this Mac can show a scene on a headset, from the bridge view. + func setAvailable(_ available: Bool) { + if available, state == .unavailable { + state = .idle + } else if available == false, state == .idle { + state = .unavailable + } + } + + /// The View menu's item: Preview, or Stop while one is asked for or shown. + func toggle() { + switch state { + case .idle: + begin() + case .connecting, .previewing: + end() + case .unavailable, .ending: + break + } + } + + /// Asks for the preview: the system asks the headset's wearer, and the + /// space opens on acceptance. Not while the game plays. + func begin() { + guard state == .idle, let openSpace, renderer != nil, ViewportCameras.isPlaying == false else { + return + } + state = .connecting + Task { @MainActor [weak self] in + let opened = await openSpace() + guard let self, opened == false, state == .connecting else { return } + state = .idle + } + } + + /// The compositor handed the layer over: the headset shows the space. + func spaceDidOpen(frames: VisionProFrameSource) { + guard let renderer, state == .connecting || state == .idle else { + frames.stop() + return + } + startLoop(renderer: renderer, frames: frames) + state = .previewing + } + + /// Ends the preview: the loop stops, the editor is put back, the space + /// closes. + func end() { + guard state == .connecting || state == .previewing else { + return + } + let wasPreviewing = state == .previewing + state = .ending + if wasPreviewing { + stopLoopAndRestore() + } + Task { @MainActor [weak self] in + await self?.dismissSpace?() + guard let self, state == .ending else { return } + state = .idle + } + } + + /// The loop ended on its own: the headset closed the space, or the + /// stream broke. + private func loopDidEnd() { + guard state == .previewing else { + return + } + stopLoopAndRestore() + state = .idle + } + + private func startLoop(renderer: UntoldRenderer, frames: VisionProFrameSource) { + // The headset rides the editor's camera, which the keys and the mouse + // fly meanwhile; a game camera shown as a locked preview is left for it. + ViewportCameras.show(.editor) + let camera = findSceneCamera() + let pose = VisionProPlacement.pose(of: camera) ?? (eye: .zero, forward: simd_float3(0, 0, -1)) + + VisionProPreviewState.shared.isPreviewing = true + renderer.enterStereo() + let mirror = VisionProMirror(device: renderInfo.device) + self.mirror = mirror + // The viewport shows the headset's eye; the loop drives the engine. + // The keys keep flying the camera, which the headset rides. + // The loop flies the camera by the keys; the viewport's frames only + // let go of a fly key whose key-up was lost, as the editor's own + // frames do. + mirror.onFrame = { + InputSystem.shared.releaseFlyKeysTheKeyboardLetGo() + } + renderer.metalView.delegate = mirror + // The pointer is a second head: its movement turns the view. + let pointer = VisionProPointerCapture { delta in + VisionProPreviewState.shared.steerCamera { + InputSystem.shared.lookAroundSceneCamera(by: delta) + } + } + self.pointer = pointer + pointer.start() + + let loop = VisionProPreviewLoop( + renderer: renderer, + frames: frames, + start: VisionProPreviewLoop.Start(camera: camera, cameraEye: pose.eye, cameraForward: pose.forward), + mirror: mirror + ) + self.loop = loop + let done = DispatchSemaphore(value: 0) + loopDone = done + let thread = Thread { [weak self] in + loop.run() + done.signal() + DispatchQueue.main.async { + self?.loopDidEnd() + } + } + thread.name = "Vision Pro Preview" + thread.qualityOfService = .userInteractive + thread.start() + } + + private func stopLoopAndRestore() { + loop?.stop() + // The loop finishes the frame it is on. + _ = loopDone?.wait(timeout: .now() + .seconds(2)) + loop = nil + loopDone = nil + + if let renderer { + renderer.metalView.delegate = renderer + renderer.leaveStereo(viewport: renderer.metalView) + } + pointer?.stop() + pointer = nil + // The camera stays where the keys and the mouse flew it. + mirror = nil + VisionProPreviewState.shared.isPreviewing = false + } + + /// Whether a key ends the preview: Escape, while one is asked for or shown. + static func endsOnKey(_ keyCode: UInt16, state: State) -> Bool { + keyCode == 53 && (state == .connecting || state == .previewing) + } + + private func watchEscape() { + guard escapeMonitor == nil else { + return + } + escapeMonitor = NSEvent.addLocalMonitorForEvents(matching: .keyDown) { [weak self] event in + guard let self, Self.endsOnKey(event.keyCode, state: state) else { + return event + } + end() + return nil + } + } + + private func unwatchEscape() { + if let escapeMonitor { + NSEvent.removeMonitor(escapeMonitor) + } + escapeMonitor = nil + } + + /// The View menu's item for a state: its title, whether it can be chosen, + /// and whether it is shown at all. + static func menuItem(for state: State, isPlaying: Bool) -> (title: String, isEnabled: Bool, isHidden: Bool) { + switch state { + case .unavailable: + return ("Preview on Apple Vision Pro", false, true) + case .idle: + return ("Preview on Apple Vision Pro", isPlaying == false, false) + case .connecting, .previewing: + return ("Stop the Preview on Apple Vision Pro", true, false) + case .ending: + return ("Stop the Preview on Apple Vision Pro", false, false) + } + } +} diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewState.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewState.swift new file mode 100644 index 0000000..5057571 --- /dev/null +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewState.swift @@ -0,0 +1,52 @@ +// +// VisionProPreviewState.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +import Foundation + +/// Whether the scene is previewed on a headset now. The session writes it +/// on the main thread; the input system and the render extension read it, +/// the latter from the headset's render thread, so it is kept under a lock. +final class VisionProPreviewState: @unchecked Sendable { + static let shared = VisionProPreviewState() + + private let lock = NSLock() + private var previewing = false + + var isPreviewing: Bool { + get { + lock.lock() + defer { lock.unlock() } + return previewing + } + set { + lock.lock() + defer { lock.unlock() } + previewing = newValue + } + } + + /// Held by the headset's loop while it draws the eyes, and by the main + /// thread while it steers the camera: the engine keeps the camera's view + /// matrix in the camera itself, and the per-eye render sets and reads it, + /// so a move from the keys or the mouse in the middle of an eye would + /// draw part of that eye from the camera's own view, stuck to the head. + let cameraLock = NSLock() + + /// Runs a move of the camera from the keys or the mouse: while a headset + /// previews, between two of its frames. + func steerCamera(_ body: () throws -> T) rethrows -> T { + guard isPreviewing else { + return try body() + } + cameraLock.lock() + defer { cameraLock.unlock() } + return try body() + } +} diff --git a/Sources/UntoldEditor/Editor/Window/EditorView+PlayMode.swift b/Sources/UntoldEditor/Editor/Window/EditorView+PlayMode.swift index 98ea8c8..d981d4d 100644 --- a/Sources/UntoldEditor/Editor/Window/EditorView+PlayMode.swift +++ b/Sources/UntoldEditor/Editor/Window/EditorView+PlayMode.swift @@ -44,6 +44,8 @@ extension EditorView { } if shouldPlay { + // A preview on a headset ends before the game takes the camera. + VisionProPreviewSession.shared.end() if capturesSnapshot { let snapshot = serializeScene() playModeSnapshot = snapshot diff --git a/Sources/UntoldEditor/Editor/Window/EditorView+Viewport.swift b/Sources/UntoldEditor/Editor/Window/EditorView+Viewport.swift index a449699..731c0b1 100644 --- a/Sources/UntoldEditor/Editor/Window/EditorView+Viewport.swift +++ b/Sources/UntoldEditor/Editor/Window/EditorView+Viewport.swift @@ -71,6 +71,19 @@ extension EditorView { .padding(.top, 10) } } + .overlay(alignment: .top) { + if experienceMode == .edit, editor_showsHeadsetPreviewLabel { + VStack(spacing: 8) { + VisionProPreviewLabel(state: visionProPreview.state) { + visionProPreview.end() + } + if visionProPreview.isPreviewing { + VisionProPreviewHints() + } + } + .padding(.top, 10) + } + } .overlay { if shouldShowDemoGallery { DemoGalleryView( diff --git a/Sources/UntoldEditor/Editor/Window/EditorView+ViewportControls.swift b/Sources/UntoldEditor/Editor/Window/EditorView+ViewportControls.swift index b757058..82cc892 100644 --- a/Sources/UntoldEditor/Editor/Window/EditorView+ViewportControls.swift +++ b/Sources/UntoldEditor/Editor/Window/EditorView+ViewportControls.swift @@ -60,7 +60,7 @@ extension EditorView { /// as a locked preview. Not during a play session, when the play flow owns /// the camera. func editor_showViewportCamera(_ camera: ViewportCamera) { - guard experienceMode == .edit, isPlaying == false, ViewportCameras.show(camera) else { return } + guard experienceMode == .edit, isPlaying == false, visionProPreview.isPreviewing == false, ViewportCameras.show(camera) else { return } viewportSettings.camera = camera } @@ -78,6 +78,12 @@ extension EditorView { return ViewportCameras.gameCameras().first { $0.entityId == entityId } } + /// True while a preview on a headset is asked for or shown, when the + /// viewport says so. + var editor_showsHeadsetPreviewLabel: Bool { + visionProPreview.state == .connecting || visionProPreview.state == .previewing + } + /// F: frames the selection, keeping the camera's direction. func editor_frameSelection() { guard experienceMode == .edit, isPlaying == false, ViewportCameras.isLockedPreview == false, diff --git a/Sources/UntoldEditor/Editor/Window/EditorView.swift b/Sources/UntoldEditor/Editor/Window/EditorView.swift index 462f9e4..b48b2ea 100644 --- a/Sources/UntoldEditor/Editor/Window/EditorView.swift +++ b/Sources/UntoldEditor/Editor/Window/EditorView.swift @@ -85,6 +85,7 @@ public struct EditorView: View { @State var dropStatusIsError = false @ObservedObject var buildTargetSettings = EditorBuildTargetSettings.shared @ObservedObject var viewportSettings = EditorViewportSettings.shared + @ObservedObject var visionProPreview = VisionProPreviewSession.shared var renderer: UntoldRenderer? @@ -93,6 +94,7 @@ public struct EditorView: View { _selectionManager = StateObject(wrappedValue: sharedSelectionManager) editorController = EditorController(selectionManager: sharedSelectionManager) renderer = UntoldRenderer.create(configuration: .editor) + VisionProPreviewSession.shared.renderer = renderer // Extensions that create pipelines must be registered after the renderer // has initialized Metal and loaded the engine shader library. registerEditorRenderExtension() diff --git a/Sources/UntoldEditor/Renderer/EditorUntoldRenderer.swift b/Sources/UntoldEditor/Renderer/EditorUntoldRenderer.swift index 56f30dd..fd1665e 100644 --- a/Sources/UntoldEditor/Renderer/EditorUntoldRenderer.swift +++ b/Sources/UntoldEditor/Renderer/EditorUntoldRenderer.swift @@ -14,6 +14,14 @@ import UntoldEngine extension UntoldRenderer { func handleSceneInput() { + // While a headset previews the scene the engine's XR update runs this + // from the headset's loop too. The keys fly the camera level from that + // loop then (`VisionProFlight`), the pointer turns it, and the gizmo + // takes no input: the editor's own flying, along the camera's tilted + // forward, would climb and dive on top. + guard VisionProPreviewState.shared.isPreviewing == false else { + return + } // A key whose release was never reported must not fly the camera forever. #if os(macOS) InputSystem.shared.releaseFlyKeysTheKeyboardLetGo() diff --git a/Sources/UntoldEditor/Systems/EditorInputSystemAppKit.swift b/Sources/UntoldEditor/Systems/EditorInputSystemAppKit.swift index bd061f6..585aa97 100644 --- a/Sources/UntoldEditor/Systems/EditorInputSystemAppKit.swift +++ b/Sources/UntoldEditor/Systems/EditorInputSystemAppKit.swift @@ -247,29 +247,42 @@ } } + // While a headset previews the scene, a move of the camera waits for + // the eyes being drawn: `VisionProPreviewState.steerCamera`. + func canvasRightMouseDown(_ event: NSEvent) { syncModifiers(from: event) keyState.rightMousePressed = true - beginCameraDrag() + VisionProPreviewState.shared.steerCamera { + beginCameraDrag() + } } func canvasRightMouseDragged(_ event: NSEvent) { // The event's Y grows down the screen; the view's Y points up. - moveCameraDrag(by: simd_float2(Float(event.deltaX), Float(-event.deltaY))) + VisionProPreviewState.shared.steerCamera { + moveCameraDrag(by: simd_float2(Float(event.deltaX), Float(-event.deltaY))) + } } func canvasRightMouseUp(_: NSEvent) { keyState.rightMousePressed = false - endCameraDrag() + VisionProPreviewState.shared.steerCamera { + endCameraDrag() + } } func canvasScrolled(_ event: NSEvent) { syncModifiers(from: event) - handleMouseScroll(event) + VisionProPreviewState.shared.steerCamera { + handleMouseScroll(event) + } } func canvasMagnified(_ event: NSEvent) { - handleMagnify(by: event.magnification, phase: event.phase) + VisionProPreviewState.shared.steerCamera { + handleMagnify(by: event.magnification, phase: event.phase) + } } /// A key pressed while the canvas has the keyboard. False when the key is @@ -349,6 +362,13 @@ } let precise = event.hasPreciseScrollingDeltas + // With a headset on, the scroll does nothing: the orbit would swing + // the wearer round a point ahead, the pointer's own movement turns + // the view, and the keys move. + if VisionProPreviewState.shared.isPreviewing { + return + } + let now = ProcessInfo.processInfo.systemUptime if now - editorInputTargetViewRef.lastScrollNavigationTime > InputSystem.scrollSessionGap { reanchorSceneCameraTarget() @@ -705,7 +725,10 @@ if magnification != 0 { pinchDelta = 3.0 * simd_float3(0.0, 0.0, Float(magnification)) - zoomSceneCamera(by: Float(magnification) * 8.0) + // With a headset on, the pinch does nothing: the keys move. + if VisionProPreviewState.shared.isPreviewing == false { + zoomSceneCamera(by: Float(magnification) * 8.0) + } previousScale += magnification currentPinchGestureState = .changed } @@ -1270,6 +1293,11 @@ guard editorInputTargetViewRef.isCameraDragActive else { return } + // With a headset on, the pointer's movement turns the view, button + // or not: `VisionProPointerCapture` has it. + if VisionProPreviewState.shared.isPreviewing { + return + } switch editorInputTargetViewRef.activeDragAction { case .look: lookAroundSceneCamera(by: delta) diff --git a/Sources/UntoldEditor/UntoldEditorApp.swift b/Sources/UntoldEditor/UntoldEditorApp.swift index 38c75c6..b820dda 100644 --- a/Sources/UntoldEditor/UntoldEditorApp.swift +++ b/Sources/UntoldEditor/UntoldEditorApp.swift @@ -41,6 +41,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate { private var overlayItems: [ViewportOverlay: NSMenuItem] = [:] private var sceneCamItem: NSMenuItem? private var steerWhilePlayingItem: NSMenuItem? + private var visionProItem: NSMenuItem? private var cameraMenu: NSMenu? private var panelMenuItems: [PanelID: [NSMenuItem]] = [:] private var dockMenuItem: NSMenuItem? @@ -185,6 +186,9 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate { sceneCamItem?.toolTip = "While playing, keep the viewport on the editor's camera" steerWhilePlayingItem = addItem(to: viewMenu, title: "Steer the Camera While Playing", action: #selector(menuToggleSteerWhilePlaying), key: "") steerWhilePlayingItem?.toolTip = "While playing, the keys and the mouse steer the game's camera as they steer the editor's. Switch it off for a game that steers its camera itself." + // Shown only where the Mac can draw for a headset (macOS 26 and later). + visionProItem = addItem(to: viewMenu, title: "Preview on Apple Vision Pro", action: #selector(menuToggleVisionProPreview), key: "") + visionProItem?.toolTip = "Shows the scene in an Apple Vision Pro nearby, drawn by this Mac; the headset asks to accept. The headset moves the camera, and the viewport shows its left eye." viewMenu.addItem(.separator()) // Camera navigation style (radio-style checkmarks, synced in menuNeedsUpdate). @@ -408,6 +412,10 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate { } sceneCamItem?.state = EditorPlaybackSettings.shared.useSceneCameraDuringPlay ? .on : .off steerWhilePlayingItem?.state = EditorPlaybackSettings.shared.steersCameraWhilePlaying ? .on : .off + let preview = VisionProPreviewSession.menuItem(for: VisionProPreviewSession.shared.state, isPlaying: ViewportCameras.isPlaying) + visionProItem?.title = preview.title + visionProItem?.isEnabled = preview.isEnabled + visionProItem?.isHidden = preview.isHidden let layout = EditorDockLayout.shared for (panel, items) in panelMenuItems { @@ -485,6 +493,10 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate { EditorPlaybackSettings.shared.useSceneCameraDuringPlay.toggle() } + @objc private func menuToggleVisionProPreview() { + VisionProPreviewSession.shared.toggle() + } + @objc private func menuToggleSteerWhilePlaying() { EditorPlaybackSettings.shared.steersCameraWhilePlaying.toggle() } @@ -495,7 +507,8 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate { private func fillCameraMenu(_ menu: NSMenu) { menu.removeAllItems() let shown = CameraSystem.shared.activeCamera - let canChoose = EditorPlaybackSettings.shared.isSessionActive == false + // Not while the game plays, nor while a headset moves the camera. + let canChoose = EditorPlaybackSettings.shared.isSessionActive == false && VisionProPreviewSession.shared.isPreviewing == false let cameras = ViewportCameras.gameCameras() let editorItem = addItem(to: menu, title: "Editor Camera", action: #selector(menuShowViewportCamera(_:)), key: "") @@ -617,17 +630,26 @@ class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate { } } -/// Entry point. An `@main` type rather than top-level code in a `main.swift`: Xcode 26 -/// compiles a package executable that test targets import with `-parse-as-library` -/// (so `@testable import UntoldEditor` works), which forbids top-level statements. -/// `swift build` / `swift test` handle `@main` executables the same way. +/// Entry point: a SwiftUI app, so that it can declare the space the Apple Vision Pro +/// preview shows (`RemoteImmersiveSpace` is a SwiftUI scene), with the editor's own +/// AppKit delegate, window and menus as before; the app opens no window of its own. +/// An `@main` type rather than top-level code in a `main.swift`: Xcode 26 compiles a +/// package executable that test targets import with `-parse-as-library` (so +/// `@testable import UntoldEditor` works), which forbids top-level statements. @main -enum UntoldEditorApp { - static func main() { - let app = NSApplication.shared - let delegate = AppDelegate() - app.delegate = delegate - app.run() +struct UntoldEditorApp: App { + @NSApplicationDelegateAdaptor(AppDelegate.self) private var delegate + + var body: some SwiftUI.Scene { + #if canImport(CompositorServices) + if #available(macOS 26.0, *) { + VisionProPreviewScene() + } + #endif + // A scene nothing opens: the editor's window is the delegate's. + Settings { + EmptyView() + } } } diff --git a/Tests/UntoldEditorTests/EditorRenderingSystemTests.swift b/Tests/UntoldEditorTests/EditorRenderingSystemTests.swift index 4099ac0..925bb6e 100644 --- a/Tests/UntoldEditorTests/EditorRenderingSystemTests.swift +++ b/Tests/UntoldEditorTests/EditorRenderingSystemTests.swift @@ -118,6 +118,20 @@ final class EditorRenderingSystemTests: XCTestCase { XCTAssertNotNil(graph["untold.editor.clearGizmoLayer"]) } + func testHeadsetPreviewUsesRuntimeGraphWithOnlyTheClearingPass() throws { + gameMode = false + let original = ViewportCameras.isPreviewingOnHeadset + ViewportCameras.isPreviewingOnHeadset = { true } + defer { ViewportCameras.isPreviewingOnHeadset = original } + + let (graph, _) = try buildGameModeGraph() + + XCTAssertNil(graph["untold.editor.highlight"]) + XCTAssertNil(graph["untold.editor.lightVisuals"]) + XCTAssertNil(graph["untold.editor.gizmo"]) + XCTAssertNotNil(graph["untold.editor.clearGizmoLayer"], "nothing of the editor's shows in the headset") + } + func testEditingAddsNoClearingPassOfItsOwn() throws { gameMode = false diff --git a/Tests/UntoldEditorTests/ViewportCameraTests.swift b/Tests/UntoldEditorTests/ViewportCameraTests.swift index e97e1d1..3808434 100644 --- a/Tests/UntoldEditorTests/ViewportCameraTests.swift +++ b/Tests/UntoldEditorTests/ViewportCameraTests.swift @@ -90,6 +90,17 @@ final class ViewportCameraTests: XCTestCase { XCTAssertFalse(ViewportCameras.isLockedPreview) } + func test_whileAHeadsetPreviews_theViewportIsLocked_andTheKeysSteerTheEditorsCamera() { + let original = ViewportCameras.isPreviewingOnHeadset + ViewportCameras.isPreviewingOnHeadset = { true } + defer { ViewportCameras.isPreviewingOnHeadset = original } + let camera = findSceneCamera() + + XCTAssertEqual(ViewportCameras.current, .editor, "the editor's camera carries the headset") + XCTAssertTrue(ViewportCameras.isLockedPreview, "clicks select nothing and no overlay is drawn") + XCTAssertEqual(ViewportCameras.steered, camera, "the keys and the mouse fly the camera the headset rides") + } + func test_show_refusesAnEntityThatIsNotAGameCamera() { let sceneCamera = findSceneCamera() let prop = createEntity() diff --git a/Tests/UntoldEditorTests/VisionProFakeFrames.swift b/Tests/UntoldEditorTests/VisionProFakeFrames.swift new file mode 100644 index 0000000..c3b11a3 --- /dev/null +++ b/Tests/UntoldEditorTests/VisionProFakeFrames.swift @@ -0,0 +1,159 @@ +// +// VisionProFakeFrames.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +import Foundation +import Metal +import simd +@testable import UntoldEditor +@testable import UntoldEngine + +/// Frames for the headset's loop without a headset: two eyes drawn into +/// textures of the test's own, from a pose the test gives, a set number of +/// times; or, kept open, frames with nothing to draw until the source is +/// stopped, as a headset that shows nothing yet. +final class VisionProFakeFrames: VisionProFrameSource { + final class Frame: VisionProFrame { + let eyes: VisionProFrameEyes? + private(set) var phases: [String] = [] + private(set) var presentedIn: MTLCommandBuffer? + /// Whether the camera was held against the keys and the mouse when + /// the frame was presented, right after its eyes were drawn. + private(set) var cameraWasHeldAtPresent = false + + init(eyes: VisionProFrameEyes?) { + self.eyes = eyes + } + + func beginUpdate() { + phases.append("beginUpdate") + } + + func endUpdate() { + phases.append("endUpdate") + } + + func waitForDrawingTime() { + phases.append("wait") + } + + func beginDrawing() -> Bool { + phases.append("beginDrawing") + return true + } + + func acquireEyes() -> VisionProFrameEyes? { + phases.append("acquire") + return eyes + } + + func present(commandBuffer: MTLCommandBuffer) { + phases.append("present") + presentedIn = commandBuffer + if VisionProPreviewState.shared.cameraLock.try() { + VisionProPreviewState.shared.cameraLock.unlock() + } else { + cameraWasHeldAtPresent = true + } + } + + func endDrawing() { + phases.append("endDrawing") + } + } + + /// Half the distance between the eyes, as on a headset. + static let halfEyeDistance: Float = 0.032 + + let colorTextures: [MTLTexture] + let depthTextures: [MTLTexture] + private(set) var frames: [Frame] = [] + private var left: Int + private let keepsOpen: Bool + private let lock = NSLock() + private var stopped = false + private let opened = DispatchSemaphore(value: 0) + /// Where the headset is, from its origin, for the frames to come. + var originFromDevice = matrix_identity_float4x4 + /// Open-ended frames carry eyes, as a headset that shows the scene; off, + /// they carry none, as one that shows nothing yet. + var showsEyesWhileOpen = false + + /// Textures of `eyeSize` for `count` frames; `count` nil keeps the source + /// open with empty frames until it is stopped. + init(device: MTLDevice, eyeSize: (width: Int, height: Int), count: Int?) { + let color = MTLTextureDescriptor.texture2DDescriptor( + pixelFormat: .bgra8Unorm_srgb, width: eyeSize.width, height: eyeSize.height, mipmapped: false + ) + color.usage = [.renderTarget, .shaderRead] + color.storageMode = .shared + let depth = MTLTextureDescriptor.texture2DDescriptor( + pixelFormat: .depth32Float, width: eyeSize.width, height: eyeSize.height, mipmapped: false + ) + depth.usage = [.renderTarget, .shaderRead] + depth.storageMode = .private + colorTextures = (0 ..< 2).compactMap { _ in device.makeTexture(descriptor: color) } + depthTextures = (0 ..< 2).compactMap { _ in device.makeTexture(descriptor: depth) } + left = count ?? 0 + keepsOpen = count == nil + } + + private var isStopped: Bool { + lock.lock() + defer { lock.unlock() } + return stopped + } + + func nextFrame() -> VisionProFrame? { + if keepsOpen { + // A headset that is on but shows nothing yet: a frame now and then. + _ = opened.wait(timeout: .now() + .milliseconds(20)) + guard isStopped == false else { return nil } + let frame = Frame(eyes: showsEyesWhileOpen ? eyes() : nil) + frames.append(frame) + return frame + } + guard isStopped == false, left > 0 else { return nil } + left -= 1 + let frame = Frame(eyes: eyes()) + frames.append(frame) + return frame + } + + func stop() { + lock.lock() + stopped = true + lock.unlock() + opened.signal() + } + + private func eyes() -> VisionProFrameEyes { + let aspect = Float(colorTextures[0].width) / Float(colorTextures[0].height) + let projection = matrixPerspectiveRightHandReverseZ(fovyRadians: .pi / 2, aspectRatio: aspect, nearZ: 0.1, farZ: 100) + let eyes = (0 ..< 2).map { index -> VisionProEye in + let side = index == 0 ? -Self.halfEyeDistance : Self.halfEyeDistance + let deviceFromView = matrix4x4Translation(side, 0, 0) + return VisionProEye( + colorTexture: colorTextures[index], + depthTexture: depthTextures[index], + viewFromOrigin: simd_inverse(simd_mul(originFromDevice, deviceFromView)), + projection: projection + ) + } + return VisionProFrameEyes(originFromDevice: originFromDevice, eyes: eyes) + } + + /// A pixel of a color texture as red, green, blue, alpha from 0 to 255. + static func pixel(of texture: MTLTexture, x: Int, y: Int) -> (r: UInt8, g: UInt8, b: UInt8, a: UInt8) { + var bytes = [UInt8](repeating: 0, count: 4) + texture.getBytes(&bytes, bytesPerRow: 4, from: MTLRegionMake2D(x, y, 1, 1), mipmapLevel: 0) + // The texture is BGRA. + return (bytes[2], bytes[1], bytes[0], bytes[3]) + } +} diff --git a/Tests/UntoldEditorTests/VisionProFlightTests.swift b/Tests/UntoldEditorTests/VisionProFlightTests.swift new file mode 100644 index 0000000..6a7c855 --- /dev/null +++ b/Tests/UntoldEditorTests/VisionProFlightTests.swift @@ -0,0 +1,53 @@ +// +// VisionProFlightTests.swift +// UntoldEditorTests +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// + +import simd +@testable import UntoldEditor +import XCTest + +/// The keys fly the camera the headset rides level: ahead and aside along the +/// floor whatever the camera's tilt, Q and E straight up and down. +final class VisionProFlightTests: XCTestCase { + private func assertEqual(_ a: simd_float3, _ b: simd_float3, _ message: String = "", line: UInt = #line) { + XCTAssertEqual(simd_distance(a, b), 0, accuracy: 0.001, "\(message) \(a) vs \(b)", line: line) + } + + func test_theLevelAxes_followTheCamerasYaw_notItsTilt() { + let ahead = VisionProFlight.levelAxes(forward: simd_float3(0, 0, -1)) + assertEqual(ahead.ahead, simd_float3(0, 0, -1)) + assertEqual(ahead.right, simd_float3(1, 0, 0)) + + let leftAndDown = VisionProFlight.levelAxes(forward: simd_normalize(simd_float3(-1, -1, 0))) + assertEqual(leftAndDown.ahead, simd_float3(-1, 0, 0), "the tilt down is left out") + assertEqual(leftAndDown.right, simd_float3(0, 0, -1)) + } + + func test_theKeys_moveAlongTheFloor_andQAndEStraightUpAndDown() { + let tiltedDown = simd_normalize(simd_float3(0, -1, -1)) + let speed: Float = 2 + let deltaTime: Float = 0.5 + let aStep = speed * VisionProFlight.unitsPerSecondPerSpeed * deltaTime + + assertEqual(VisionProFlight.displacement(keys: .init(w: true), forward: tiltedDown, speed: speed, deltaTime: deltaTime), simd_float3(0, 0, -aStep), "W keeps the height") + assertEqual(VisionProFlight.displacement(keys: .init(s: true), forward: tiltedDown, speed: speed, deltaTime: deltaTime), simd_float3(0, 0, aStep)) + assertEqual(VisionProFlight.displacement(keys: .init(d: true), forward: tiltedDown, speed: speed, deltaTime: deltaTime), simd_float3(aStep, 0, 0)) + assertEqual(VisionProFlight.displacement(keys: .init(a: true), forward: tiltedDown, speed: speed, deltaTime: deltaTime), simd_float3(-aStep, 0, 0)) + assertEqual(VisionProFlight.displacement(keys: .init(q: true), forward: tiltedDown, speed: speed, deltaTime: deltaTime), simd_float3(0, aStep, 0), "Q rises") + assertEqual(VisionProFlight.displacement(keys: .init(e: true), forward: tiltedDown, speed: speed, deltaTime: deltaTime), simd_float3(0, -aStep, 0), "E sinks") + assertEqual(VisionProFlight.displacement(keys: .init(w: true, s: true), forward: tiltedDown, speed: speed, deltaTime: deltaTime), .zero, "opposite keys cancel") + assertEqual(VisionProFlight.displacement(keys: .init(), forward: tiltedDown, speed: speed, deltaTime: deltaTime), .zero) + } + + func test_thePointersMovement_turnsAsALookDragDoes() { + XCTAssertEqual(VisionProPointerCapture.turn(forPointerDelta: simd_float2(3, -2)), simd_float2(3, 2), "a move up looks up") + XCTAssertEqual(VisionProPointerCapture.turn(forPointerDelta: simd_float2(.infinity, 0)), .zero) + } +} diff --git a/Tests/UntoldEditorTests/VisionProMirrorTests.swift b/Tests/UntoldEditorTests/VisionProMirrorTests.swift new file mode 100644 index 0000000..b610a5a --- /dev/null +++ b/Tests/UntoldEditorTests/VisionProMirrorTests.swift @@ -0,0 +1,131 @@ +// +// VisionProMirrorTests.swift +// UntoldEditorTests +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// + +import MetalKit +import simd +@testable import UntoldEditor +@testable import UntoldEngine +import XCTest + +/// The viewport's mirror of the headset's eye: its pipeline names what the +/// viewport's drawable comes with, and the eye lands in the middle of the +/// view at its own proportions. Run with `MTL_DEBUG_LAYER=1` in the +/// environment for Metal's own check of the pipeline against the drawable. +@MainActor +final class VisionProMirrorTests: XCTestCase { + private var window: NSWindow! + private var renderer: UntoldRenderer! + + override func setUp() { + super.setUp() + window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 64, height: 32), + styleMask: [.titled], + backing: .buffered, + defer: false + ) + guard let created = UntoldRenderer.create() else { + XCTFail("Failed to initialize the renderer") + return + } + renderer = created + window.contentView = created.metalView + } + + override func tearDown() { + renderer = nil + window = nil + super.tearDown() + } + + func test_thePipelineNamesTheDepthAndTheStencil_theViewsDrawableComesWith() throws { + let device = try XCTUnwrap(renderInfo.device) + let library = try device.makeLibrary(source: VisionProMirror.shaderSource, options: nil) + + let depthOnly = VisionProMirror.pipelineDescriptor(colorFormat: .bgra8Unorm_srgb, depthStencilFormat: .depth32Float, library: library) + XCTAssertEqual(depthOnly.colorAttachments[0].pixelFormat, .bgra8Unorm_srgb) + XCTAssertEqual(depthOnly.depthAttachmentPixelFormat, .depth32Float) + XCTAssertEqual(depthOnly.stencilAttachmentPixelFormat, MTLPixelFormat.invalid) + XCTAssertNoThrow(try device.makeRenderPipelineState(descriptor: depthOnly)) + + let both = VisionProMirror.pipelineDescriptor(colorFormat: .bgra8Unorm, depthStencilFormat: .depth32Float_stencil8, library: library) + XCTAssertEqual(both.depthAttachmentPixelFormat, .depth32Float_stencil8) + XCTAssertEqual(both.stencilAttachmentPixelFormat, .depth32Float_stencil8) + + let none = VisionProMirror.pipelineDescriptor(colorFormat: .bgra8Unorm, depthStencilFormat: .invalid, library: library) + XCTAssertEqual(none.depthAttachmentPixelFormat, MTLPixelFormat.invalid) + XCTAssertEqual(none.stencilAttachmentPixelFormat, MTLPixelFormat.invalid) + } + + func test_theMirrorDrawsTheEye_intoAPassWithADepthAttachment_asTheViewportsDrawableIs() throws { + let view = renderer.metalView + XCTAssertEqual(view.depthStencilPixelFormat, .depth32Float, "the viewport's drawable comes with a depth attachment") + let device = try XCTUnwrap(renderInfo.device) + let mirror = VisionProMirror(device: device) + + // A square red eye in a wide pass: the eye takes the middle, the + // sides show the background. + let eye = try XCTUnwrap(Self.makeRedEye(device: device, size: 16)) + let copy = try XCTUnwrap(renderInfo.commandQueue.makeCommandBuffer()) + mirror.copy(eye: eye, commandBuffer: copy) + copy.commit() + copy.waitUntilCompleted() + let deadline = Date().addingTimeInterval(2) + while mirror.latestEye == nil, Date() < deadline { + RunLoop.main.run(until: Date().addingTimeInterval(0.01)) + } + XCTAssertNotNil(mirror.latestEye) + + let (color, depth) = try Self.makePass(device: device, width: 64, height: 32, colorFormat: view.colorPixelFormat, depthFormat: view.depthStencilPixelFormat) + let descriptor = MTLRenderPassDescriptor() + descriptor.colorAttachments[0].texture = color + descriptor.depthAttachment.texture = depth + let commandBuffer = try XCTUnwrap(renderInfo.commandQueue.makeCommandBuffer()) + mirror.encode( + into: descriptor, size: CGSize(width: 64, height: 32), + colorFormat: view.colorPixelFormat, depthStencilFormat: view.depthStencilPixelFormat, + commandBuffer: commandBuffer + ) + commandBuffer.commit() + commandBuffer.waitUntilCompleted() + XCTAssertNil(commandBuffer.error) + + let middle = VisionProFakeFrames.pixel(of: color, x: 32, y: 16) + let side = VisionProFakeFrames.pixel(of: color, x: 2, y: 16) + XCTAssertGreaterThan(Int(middle.r), 200, "the eye, red, in the middle") + XCTAssertLessThan(Int(middle.g), 60) + XCTAssertLessThan(Int(side.r), 120, "the background at the side") + XCTAssertLessThan(abs(Int(side.r) - Int(side.g)), 20, "the background is a grey") + } + + private static func makePass(device: MTLDevice, width: Int, height: Int, colorFormat: MTLPixelFormat, depthFormat: MTLPixelFormat) throws -> (MTLTexture, MTLTexture) { + let color = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: colorFormat, width: width, height: height, mipmapped: false) + color.usage = [.renderTarget, .shaderRead] + color.storageMode = .shared + let depth = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: depthFormat, width: width, height: height, mipmapped: false) + depth.usage = [.renderTarget] + depth.storageMode = .private + return try (XCTUnwrap(device.makeTexture(descriptor: color)), XCTUnwrap(device.makeTexture(descriptor: depth))) + } + + private static func makeRedEye(device: MTLDevice, size: Int) -> MTLTexture? { + let descriptor = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: .bgra8Unorm_srgb, width: size, height: size, mipmapped: false) + descriptor.usage = [.shaderRead] + descriptor.storageMode = .shared + guard let texture = device.makeTexture(descriptor: descriptor) else { return nil } + // BGRA. + let bytes = [UInt8](repeating: 0, count: size * size * 4).enumerated().map { index, _ -> UInt8 in + index % 4 == 2 || index % 4 == 3 ? 255 : 0 + } + texture.replace(region: MTLRegionMake2D(0, 0, size, size), mipmapLevel: 0, withBytes: bytes, bytesPerRow: size * 4) + return texture + } +} diff --git a/Tests/UntoldEditorTests/VisionProPlacementTests.swift b/Tests/UntoldEditorTests/VisionProPlacementTests.swift new file mode 100644 index 0000000..717cda2 --- /dev/null +++ b/Tests/UntoldEditorTests/VisionProPlacementTests.swift @@ -0,0 +1,100 @@ +// +// VisionProPlacementTests.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +import simd +@testable import UntoldEditor +@testable import UntoldEngine +import XCTest + +/// Where the headset stands in the scene: at the editor's camera when the +/// preview begins, and from there wherever it walks and turns. +final class VisionProPlacementTests: XCTestCase { + private func point(_ matrix: simd_float4x4, _ p: simd_float3) -> simd_float3 { + let moved = simd_mul(matrix, simd_float4(p, 1)) + return simd_float3(moved.x, moved.y, moved.z) + } + + private func direction(_ matrix: simd_float4x4, _ d: simd_float3) -> simd_float3 { + let turned = simd_mul(matrix, simd_float4(d, 0)) + return simd_float3(turned.x, turned.y, turned.z) + } + + private func assertNearlyEqual(_ lhs: simd_float3, _ rhs: simd_float3, accuracy: Float = 0.001, _ message: String = "", file: StaticString = #filePath, line: UInt = #line) { + XCTAssertEqual(lhs.x, rhs.x, accuracy: accuracy, message, file: file, line: line) + XCTAssertEqual(lhs.y, rhs.y, accuracy: accuracy, message, file: file, line: line) + XCTAssertEqual(lhs.z, rhs.z, accuracy: accuracy, message, file: file, line: line) + } + + /// A pose of the headset: where it is and the turn about the vertical + /// axis, with a tilt down when asked. + private func pose(at position: simd_float3, yaw: Float, pitch: Float = 0) -> simd_float4x4 { + let turn = simd_mul(simd_quatf(angle: yaw, axis: simd_float3(0, 1, 0)), simd_quatf(angle: pitch, axis: simd_float3(1, 0, 0))) + return simd_mul(matrix4x4Translation(position.x, position.y, position.z), simd_float4x4(turn)) + } + + func test_theLevelYaw_isZeroLookingForward_andAQuarterTurnLookingLeft() { + XCTAssertEqual(VisionProPlacement.levelYaw(of: simd_float3(0, 0, -1)), 0, accuracy: 0.0001) + XCTAssertEqual(VisionProPlacement.levelYaw(of: simd_float3(-1, 0, 0)), .pi / 2, accuracy: 0.0001) + XCTAssertEqual(VisionProPlacement.levelYaw(of: simd_float3(1, 0, 0)), -.pi / 2, accuracy: 0.0001) + XCTAssertEqual(VisionProPlacement.levelYaw(of: simd_float3(-1, -3, -1)), .pi / 4, accuracy: 0.0001, "the tilt does not count") + XCTAssertEqual(VisionProPlacement.levelYaw(of: simd_float3(0, -1, 0)), 0, "straight down has no yaw") + } + + func test_theHeadsetBegins_whereTheCameraStood_facingItsWay() { + let eye = simd_float3(3, 1.6, -2) + let placed = VisionProPlacement.sceneFromOrigin( + cameraEye: eye, + cameraForward: simd_float3(1, 0, 0), + originFromDevice: pose(at: simd_float3(0, 1.5, 0), yaw: 0) + ) + + assertNearlyEqual(point(placed, simd_float3(0, 1.5, 0)), eye, "the headset stands at the camera's eye") + assertNearlyEqual(direction(placed, simd_float3(0, 0, -1)), simd_float3(1, 0, 0), "and looks where it looked") + assertNearlyEqual(direction(placed, simd_float3(0, 1, 0)), simd_float3(0, 1, 0), "the floor stays level") + } + + func test_howTheHeadsetWasTurnedAtTheStart_doesNotCount_butItsTiltDoes() { + let placed = VisionProPlacement.sceneFromOrigin( + cameraEye: .zero, + cameraForward: simd_float3(0, 0, -1), + originFromDevice: pose(at: simd_float3(2, 1.5, 1), yaw: .pi / 2, pitch: -0.3) + ) + let start = pose(at: simd_float3(2, 1.5, 1), yaw: .pi / 2, pitch: -0.3) + let lookingAt = direction(simd_mul(placed, start), simd_float3(0, 0, -1)) + + assertNearlyEqual(point(placed, simd_float3(2, 1.5, 1)), .zero) + XCTAssertEqual(lookingAt.x, 0, accuracy: 0.001, "the turn to the left at the start is taken out") + XCTAssertEqual(lookingAt.y, sin(-0.3), accuracy: 0.001, "the tilt down stays") + XCTAssertLessThan(lookingAt.z, 0, "it looks the camera's way") + } + + func test_walkingForward_goesTheCamerasWay_andTurningTurnsTheView() { + let placed = VisionProPlacement.sceneFromOrigin( + cameraEye: simd_float3(10, 1.6, 10), + cameraForward: simd_float3(-1, 0, 0), + originFromDevice: pose(at: simd_float3(0, 1.5, 0), yaw: 0) + ) + + // One step along the headset's own forward, which was -Z. + let stepped = pose(at: simd_float3(0, 1.5, -1), yaw: 0) + assertNearlyEqual(point(simd_mul(placed, stepped), .zero), simd_float3(9, 1.6, 10), "a metre along the camera's way") + + // A quarter turn to the left: the view turns from -X to +Z. + let turned = pose(at: simd_float3(0, 1.5, 0), yaw: .pi / 2) + assertNearlyEqual(direction(simd_mul(placed, turned), simd_float3(0, 0, -1)), simd_float3(0, 0, 1)) + } + + func test_theMirrorFitsTheEye_keepingItsProportions() { + XCTAssertEqual(VisionProMirror.fit(eye: CGSize(width: 200, height: 100), in: CGSize(width: 100, height: 100)), simd_float2(1, 0.5)) + XCTAssertEqual(VisionProMirror.fit(eye: CGSize(width: 100, height: 200), in: CGSize(width: 100, height: 100)), simd_float2(0.5, 1)) + XCTAssertEqual(VisionProMirror.fit(eye: CGSize(width: 400, height: 300), in: CGSize(width: 800, height: 600)), simd_float2(1, 1)) + XCTAssertEqual(VisionProMirror.fit(eye: .zero, in: CGSize(width: 100, height: 100)), simd_float2(1, 1)) + } +} diff --git a/Tests/UntoldEditorTests/VisionProPreviewLoopTests.swift b/Tests/UntoldEditorTests/VisionProPreviewLoopTests.swift new file mode 100644 index 0000000..fbb46b3 --- /dev/null +++ b/Tests/UntoldEditorTests/VisionProPreviewLoopTests.swift @@ -0,0 +1,370 @@ +// +// VisionProPreviewLoopTests.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +import Metal +import MetalKit +import simd +@testable import UntoldEditor +@testable import UntoldEngine +import XCTest + +/// The loop that draws for the headset, run with frames of the test's own: +/// the scene is drawn into both eyes from where the editor's camera stood, +/// the engine is switched to stereo and back, and the viewport's mirror gets +/// the left eye. +@MainActor +final class VisionProPreviewLoopTests: XCTestCase { + private var originalScene: Scene! + private var originalActiveCamera: EntityID? + private var originalRenderEnvironment = false + private var window: NSWindow! + private var renderer: UntoldRenderer! + + override func setUp() { + super.setUp() + window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 640, height: 480), + styleMask: [.titled], + backing: .buffered, + defer: false + ) + guard let created = UntoldRenderer.create() else { + XCTFail("Failed to initialize the renderer") + return + } + renderer = created + window.contentView = created.metalView + created.initResources() + originalScene = scene + originalActiveCamera = CameraSystem.shared.activeCamera + // The background is the engine's clear colour, whatever another test + // left on. + originalRenderEnvironment = renderEnvironment + renderEnvironment = false + } + + override func tearDown() { + renderEnvironment = originalRenderEnvironment + renderer?.leaveStereo(viewport: renderer.metalView) + CameraSystem.shared.activeCamera = originalActiveCamera + scene = originalScene + originalScene = nil + renderer = nil + window = nil + super.tearDown() + } + + /// A lit cube of a unit at the origin, and the active camera five units + /// in front of it, looking at it. + private func makeCubeInFrontOfTheCamera() -> EntityID { + let cube = createEntity() + setEntityName(entityId: cube, name: "Cube") + setEntityMeshDirect(entityId: cube, meshes: BasicPrimitives.createCube(extent: 1), assetName: "Cube") + if let camera = CameraSystem.shared.activeCamera { + cameraLookAt(entityId: camera, eye: simd_float3(0, 0, 5), target: .zero, up: simd_float3(0, 1, 0)) + } + return cube + } + + private func start() -> VisionProPreviewLoop.Start { + VisionProPreviewLoop.Start(cameraEye: simd_float3(0, 0, 5), cameraForward: simd_float3(0, 0, -1)) + } + + private func brightness(_ pixel: (r: UInt8, g: UInt8, b: UInt8, a: UInt8)) -> Int { + Int(pixel.r) + Int(pixel.g) + Int(pixel.b) + } + + func test_theSceneIsDrawnIntoBothEyes_fromWhereTheCameraStood() throws { + _ = makeCubeInFrontOfTheCamera() + // The culling of one frame is drawn a frame or two later, as in the + // engine's own loop: a few frames, and the last one is looked at. + let frames = VisionProFakeFrames(device: renderInfo.device, eyeSize: (256, 192), count: 6) + renderer.enterStereo() + let loop = VisionProPreviewLoop(renderer: renderer, frames: frames, start: start(), mirror: nil) + + loop.run() + try XCTUnwrap(frames.frames.last?.presentedIn).waitUntilCompleted() + + XCTAssertEqual(loop.framesDrawn, 6) + XCTAssertEqual(renderInfo.viewPort, simd_float2(256, 192), "the engine's targets took the eye's size") + XCTAssertTrue(renderInfo.isXRStereoMode) + for (index, eye) in frames.colorTextures.enumerated() { + let middle = VisionProFakeFrames.pixel(of: eye, x: 128, y: 96) + let corner = VisionProFakeFrames.pixel(of: eye, x: 4, y: 4) + // The corner shows the background; the middle shows the lit cube. + let farCorner = VisionProFakeFrames.pixel(of: eye, x: 251, y: 187) + XCTAssertEqual(brightness(corner), brightness(farCorner), accuracy: 6, "eye \(index) shows the same background in both corners: \(corner), \(farCorner)") + XCTAssertGreaterThan(abs(brightness(middle) - brightness(corner)), 40, "eye \(index) shows the cube in the middle: \(middle) over \(corner)") + } + XCTAssertEqual(frames.frames[0].phases, ["beginUpdate", "endUpdate", "wait", "beginDrawing", "acquire", "present", "endDrawing"]) + XCTAssertTrue(frames.frames.allSatisfy(\.cameraWasHeldAtPresent), "the keys and the mouse wait while the eyes are drawn") + } + + func test_theHeadsetIsPlacedAtTheCamera_byItsFirstPose() { + _ = makeCubeInFrontOfTheCamera() + let frames = VisionProFakeFrames(device: renderInfo.device, eyeSize: (128, 96), count: 1) + // The headset began a metre and a half up, looking to its left. + frames.originFromDevice = simd_mul(matrix4x4Translation(0, 1.5, 0), simd_float4x4(simd_quatf(angle: .pi / 2, axis: simd_float3(0, 1, 0)))) + renderer.enterStereo() + let loop = VisionProPreviewLoop(renderer: renderer, frames: frames, start: start(), mirror: nil) + + XCTAssertNil(loop.placement, "placed by the first pose, not before") + loop.run() + + let placement = loop.placement + XCTAssertNotNil(placement) + if let placement { + let headset = simd_mul(placement, simd_float4(0, 1.5, 0, 1)) + XCTAssertEqual(headset.x, 0, accuracy: 0.001) + XCTAssertEqual(headset.y, 0, accuracy: 0.001) + XCTAssertEqual(headset.z, 5, accuracy: 0.001) + } + } + + func test_theHeadsetRidesTheCamera_whereTheKeysFlyIt() throws { + _ = makeCubeInFrontOfTheCamera() + let camera = try XCTUnwrap(CameraSystem.shared.activeCamera) + let frames = VisionProFakeFrames(device: renderInfo.device, eyeSize: (64, 48), count: nil) + frames.showsEyesWhileOpen = true + renderer.enterStereo() + let loop = VisionProPreviewLoop( + renderer: renderer, frames: frames, + start: VisionProPreviewLoop.Start(camera: camera, cameraEye: .zero, cameraForward: simd_float3(0, 0, -1)), + mirror: nil + ) + let thread = Thread { loop.run() } + thread.start() + defer { loop.stop() } + + try waitUntil { loop.framesDrawn >= 2 } + let before = try XCTUnwrap(loop.placement) + XCTAssertEqual(before.columns.3.z, 5, accuracy: 0.001, "the headset began where the camera stood") + + // The keys flew the camera three units to the right. + cameraLookAt(entityId: camera, eye: simd_float3(3, 0, 5), target: simd_float3(3, 0, 0), up: simd_float3(0, 1, 0)) + let drawn = loop.framesDrawn + try waitUntil { loop.framesDrawn >= drawn + 2 } + + let after = try XCTUnwrap(loop.placement) + XCTAssertEqual(after.columns.3.x, 3, accuracy: 0.001, "the headset went with the camera") + XCTAssertEqual(after.columns.3.z, 5, accuracy: 0.001) + } + + private func waitUntil(_ condition: () -> Bool) throws { + let deadline = Date().addingTimeInterval(5) + while condition() == false { + guard Date() < deadline else { + throw XCTSkip("the loop did not get there in time") + } + RunLoop.main.run(until: Date().addingTimeInterval(0.02)) + } + } + + func test_theCamerasPose_comesFromWhatTheKeysAndTheMouseMove_notFromLookAtAlone() throws { + let camera = try XCTUnwrap(CameraSystem.shared.activeCamera) + cameraLookAt(entityId: camera, eye: simd_float3(0, 0, 5), target: .zero, up: simd_float3(0, 1, 0)) + var pose = try XCTUnwrap(VisionProPlacement.pose(of: camera)) + XCTAssertEqual(simd_distance(pose.eye, simd_float3(0, 0, 5)), 0, accuracy: 0.001) + XCTAssertEqual(simd_distance(pose.forward, simd_float3(0, 0, -1)), 0, accuracy: 0.001) + + // The D key: two to the right. `eye` is left behind, the pose is not. + moveCameraBy(entityId: camera, delU: 2, delV: 0, delN: 0) + pose = try XCTUnwrap(VisionProPlacement.pose(of: camera)) + XCTAssertEqual(simd_distance(pose.eye, simd_float3(2, 0, 5)), 0, accuracy: 0.001) + XCTAssertEqual(getCameraEye(entityId: camera).x, 0, accuracy: 0.001, "what cameraLookAt wrote stays") + + // The mouse: a quarter turn. The W key then flies the way the pose faces. + rotateCamera(entityId: camera, pitch: 0, yaw: .pi / 2) + pose = try XCTUnwrap(VisionProPlacement.pose(of: camera)) + XCTAssertEqual(pose.forward.y, 0, accuracy: 0.001, "a turn about the vertical axis keeps the view level") + XCTAssertEqual(abs(pose.forward.x), 1, accuracy: 0.001) + let before = pose.eye + moveCameraBy(entityId: camera, delU: 0, delV: 0, delN: -1) + let after = try XCTUnwrap(VisionProPlacement.pose(of: camera)).eye + XCTAssertEqual(simd_distance(after - before, pose.forward), 0, accuracy: 0.01, "W goes where the pose faces") + } + + func test_theHeadMovingAbout_doesNotMoveThePlacement() throws { + _ = makeCubeInFrontOfTheCamera() + let camera = try XCTUnwrap(CameraSystem.shared.activeCamera) + let frames = VisionProFakeFrames(device: renderInfo.device, eyeSize: (64, 48), count: nil) + frames.showsEyesWhileOpen = true + frames.originFromDevice = matrix4x4Translation(0, 1.6, 0) + renderer.enterStereo() + let loop = VisionProPreviewLoop( + renderer: renderer, frames: frames, + start: VisionProPreviewLoop.Start(camera: camera, cameraEye: .zero, cameraForward: simd_float3(0, 0, -1)), + mirror: nil + ) + let thread = Thread { loop.run() } + thread.start() + defer { loop.stop() } + try waitUntil { loop.framesDrawn >= 2 } + let before = try XCTUnwrap(loop.placement) + + // The wearer walks half a metre to the right and a little forward. + frames.originFromDevice = matrix4x4Translation(0.5, 1.6, -0.3) + let drawn = loop.framesDrawn + try waitUntil { loop.framesDrawn >= drawn + 3 } + + let after = try XCTUnwrap(loop.placement) + for column in 0 ..< 4 { + XCTAssertEqual(simd_distance(before[column], after[column]), 0, accuracy: 0.001, "the placement stands still while the head moves") + } + let cameraPose = try XCTUnwrap(VisionProPlacement.pose(of: camera)) + XCTAssertEqual(simd_distance(cameraPose.eye, simd_float3(0, 0, 5)), 0, accuracy: 0.001, "the head's motion is not written into the camera") + } + + func test_theLoopFliesTheCamera_byTheKeysHeld_levelAndAtTheEditorsSpeed() throws { + _ = makeCubeInFrontOfTheCamera() + let camera = try XCTUnwrap(CameraSystem.shared.activeCamera) + // The camera looks down at the cube; W must not take it down. + cameraLookAt(entityId: camera, eye: simd_float3(0, 2, 5), target: .zero, up: simd_float3(0, 1, 0)) + let frames = VisionProFakeFrames(device: renderInfo.device, eyeSize: (64, 48), count: nil) + frames.showsEyesWhileOpen = true + renderer.enterStereo() + let loop = VisionProPreviewLoop( + renderer: renderer, frames: frames, + start: VisionProPreviewLoop.Start(camera: camera, cameraEye: .zero, cameraForward: simd_float3(0, 0, -1)), + mirror: nil + ) + let held = HeldKeys(VisionProFlight.Keys(w: true)) + loop.keysHeld = { held.value } + loop.flyingSpeed = { 1 } + let thread = Thread { loop.run() } + thread.start() + defer { loop.stop() } + let started = Date() + + try waitUntil { (VisionProPlacement.pose(of: camera)?.eye.z ?? 5) < 4 } + let flown = Date().timeIntervalSince(started) + // W let go: the camera stands, and the headset's placement catches up. + held.value = VisionProFlight.Keys() + let drawn = loop.framesDrawn + try waitUntil { loop.framesDrawn >= drawn + 2 } + let pose = try XCTUnwrap(VisionProPlacement.pose(of: camera)) + + XCTAssertEqual(pose.eye.y, 2, accuracy: 0.001, "W keeps the height") + XCTAssertEqual(pose.eye.x, 0, accuracy: 0.001) + XCTAssertLessThan(flown, 2, "a unit at the editor's speed takes a sixth of a second") + let placement = try XCTUnwrap(loop.placement) + XCTAssertEqual(placement.columns.3.z, pose.eye.z, accuracy: 0.001, "the headset went along") + } + + /// The keys a test holds, read from the loop's thread. + private final class HeldKeys { + private let lock = NSLock() + private var keys: VisionProFlight.Keys + + init(_ keys: VisionProFlight.Keys) { + self.keys = keys + } + + var value: VisionProFlight.Keys { + get { + lock.lock() + defer { lock.unlock() } + return keys + } + set { + lock.lock() + defer { lock.unlock() } + keys = newValue + } + } + } + + func test_theEditorsOwnFlying_standsDown_whileAHeadsetPreviews() throws { + // The editor's camera, steered: whatever camera another test left active. + let camera = findSceneCamera() + CameraSystem.shared.activeCamera = camera + InputSystem.shared.cameraControlMode = .idle + // The camera looks up the hall; the editor's own W would climb. + cameraLookAt(entityId: camera, eye: simd_float3(0, 1, 5), target: simd_float3(0, 3, 0), up: simd_float3(0, 1, 0)) + let keysBefore = InputSystem.shared.keyState + InputSystem.shared.keyState.wPressed = true + defer { InputSystem.shared.keyState = keysBefore } + let state = VisionProPreviewState.shared + state.isPreviewing = true + defer { state.isPreviewing = false } + + renderer.handleSceneInput() + + let held = try XCTUnwrap(VisionProPlacement.pose(of: camera)) + XCTAssertEqual(simd_distance(held.eye, simd_float3(0, 1, 5)), 0, accuracy: 0.001, "the headset's loop flies, the editor's own frame does not") + + state.isPreviewing = false + renderer.handleSceneInput() + let flown = try XCTUnwrap(VisionProPlacement.pose(of: camera)) + XCTAssertGreaterThan(flown.eye.y, 1, "without a headset the editor's own W climbs where the camera looks") + } + + func test_aFrameWithoutEyes_isPresentedWithNothingDrawn() { + _ = makeCubeInFrontOfTheCamera() + let frames = VisionProFakeFrames(device: renderInfo.device, eyeSize: (64, 48), count: 1) + let loop = VisionProPreviewLoop(renderer: renderer, frames: frames, start: start(), mirror: nil) + let untracked = VisionProFakeFrames.Frame(eyes: VisionProFrameEyes(originFromDevice: nil, eyes: [])) + let source = SingleFrame(frame: untracked) + + VisionProPreviewLoop(renderer: renderer, frames: source, start: start(), mirror: nil).run() + + XCTAssertEqual(untracked.phases, ["beginUpdate", "endUpdate", "wait", "beginDrawing", "acquire", "present", "endDrawing"]) + XCTAssertNotNil(untracked.presentedIn, "the compositor wants every frame presented") + XCTAssertEqual(loop.framesDrawn, 0) + _ = frames + } + + func test_theMirrorGetsTheLeftEye() throws { + _ = makeCubeInFrontOfTheCamera() + let frames = VisionProFakeFrames(device: renderInfo.device, eyeSize: (160, 120), count: 1) + let mirror = VisionProMirror(device: renderInfo.device) + renderer.enterStereo() + + VisionProPreviewLoop(renderer: renderer, frames: frames, start: start(), mirror: mirror).run() + try XCTUnwrap(frames.frames.last?.presentedIn).waitUntilCompleted() + + let eye = try XCTUnwrap(mirror.latestEye) + XCTAssertEqual(eye.width, 160) + XCTAssertEqual(eye.height, 120) + XCTAssertEqual(eye.pixelFormat, frames.colorTextures[0].pixelFormat) + } + + func test_leavingStereo_givesTheViewportItsTargetsBack() { + let frames = VisionProFakeFrames(device: renderInfo.device, eyeSize: (256, 192), count: 1) + let before = renderInfo.viewPort + renderer.enterStereo() + VisionProPreviewLoop(renderer: renderer, frames: frames, start: start(), mirror: nil).run() + XCTAssertEqual(renderInfo.viewPort, simd_float2(256, 192)) + + renderer.leaveStereo(viewport: renderer.metalView) + + XCTAssertFalse(renderInfo.isXRStereoMode) + XCTAssertEqual(renderInfo.currentEye, 0) + XCTAssertEqual(renderInfo.viewPort, before) + } + + /// One frame, then none. + private final class SingleFrame: VisionProFrameSource { + private var frame: VisionProFakeFrames.Frame? + + init(frame: VisionProFakeFrames.Frame) { + self.frame = frame + } + + func nextFrame() -> VisionProFrame? { + defer { frame = nil } + return frame + } + + func stop() { + frame = nil + } + } +} diff --git a/Tests/UntoldEditorTests/VisionProPreviewSessionTests.swift b/Tests/UntoldEditorTests/VisionProPreviewSessionTests.swift new file mode 100644 index 0000000..110fb02 --- /dev/null +++ b/Tests/UntoldEditorTests/VisionProPreviewSessionTests.swift @@ -0,0 +1,228 @@ +// +// VisionProPreviewSessionTests.swift +// UntoldEditor +// +// Copyright (C) Untold Engine Studios +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +// +import Combine +import MetalKit +import simd +@testable import UntoldEditor +@testable import UntoldEngine +import XCTest + +/// The preview as the View menu drives it: asked for, shown once the space +/// opens, ended from the menu or by the headset, with the editor put back. +@MainActor +final class VisionProPreviewSessionTests: XCTestCase { + private var originalScene: Scene! + private var originalActiveCamera: EntityID? + private var originalGameMode = false + private var window: NSWindow! + private var renderer: UntoldRenderer! + private var session: VisionProPreviewSession! + + override func setUp() { + super.setUp() + window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 640, height: 480), + styleMask: [.titled], + backing: .buffered, + defer: false + ) + guard let created = UntoldRenderer.create() else { + XCTFail("Failed to initialize the renderer") + return + } + renderer = created + window.contentView = created.metalView + created.initResources() + originalScene = scene + originalActiveCamera = CameraSystem.shared.activeCamera + originalGameMode = gameMode + gameMode = false + session = VisionProPreviewSession() + session.renderer = renderer + session.setAvailable(true) + } + + override func tearDown() { + session.end() + session = nil + VisionProPreviewState.shared.isPreviewing = false + gameMode = originalGameMode + CameraSystem.shared.activeCamera = originalActiveCamera + scene = originalScene + originalScene = nil + renderer = nil + window = nil + super.tearDown() + } + + /// Waits until the session is in `state`, or fails. + private func waitForState(_ state: VisionProPreviewSession.State, file: StaticString = #filePath, line: UInt = #line) { + let deadline = Date().addingTimeInterval(3) + while session.state != state, Date() < deadline { + RunLoop.main.run(until: Date().addingTimeInterval(0.01)) + } + XCTAssertEqual(session.state, state, file: file, line: line) + } + + func test_beforeTheBridgeSaysSo_thereIsNoPreview() { + let fresh = VisionProPreviewSession() + + XCTAssertEqual(fresh.state, .unavailable) + fresh.begin() + XCTAssertEqual(fresh.state, .unavailable) + } + + func test_askingForThePreview_opensTheSpace_andAHeadsetThatDeclines_leavesItIdle() { + var asked = 0 + session.openSpace = { + asked += 1 + return false + } + + session.begin() + + XCTAssertEqual(session.state, .connecting) + waitForState(.idle) + XCTAssertEqual(asked, 1) + } + + func test_whileTheSpaceOpens_theSceneIsPreviewed_andTheEditorIsLockedOut() { + session.openSpace = { true } + session.begin() + let frames = VisionProFakeFrames(device: renderInfo.device, eyeSize: (64, 48), count: nil) + + session.spaceDidOpen(frames: frames) + + XCTAssertEqual(session.state, .previewing) + XCTAssertTrue(VisionProPreviewState.shared.isPreviewing) + XCTAssertTrue(ViewportCameras.isLockedPreview) + XCTAssertEqual(ViewportCameras.steered, findSceneCamera(), "the keys and the mouse fly the camera the headset rides") + XCTAssertTrue(renderInfo.isXRStereoMode) + XCTAssertFalse(renderer.metalView.delegate === renderer, "the viewport shows the mirror") + } + + func test_steeringTheCamera_waitsForTheEyes_onlyWhileAHeadsetPreviews() { + let state = VisionProPreviewState.shared + state.isPreviewing = false + state.steerCamera { + XCTAssertTrue(state.cameraLock.try(), "nothing is held while no headset previews") + state.cameraLock.unlock() + } + + state.isPreviewing = true + defer { state.isPreviewing = false } + state.steerCamera { + XCTAssertFalse(state.cameraLock.try(), "a move of the camera holds what the loop holds while it draws the eyes") + } + } + + func test_escapeEndsThePreview_fromTheMomentItIsAskedFor() { + XCTAssertTrue(VisionProPreviewSession.endsOnKey(53, state: .connecting)) + XCTAssertTrue(VisionProPreviewSession.endsOnKey(53, state: .previewing)) + XCTAssertFalse(VisionProPreviewSession.endsOnKey(53, state: .idle)) + XCTAssertFalse(VisionProPreviewSession.endsOnKey(53, state: .ending)) + XCTAssertFalse(VisionProPreviewSession.endsOnKey(13, state: .previewing), "W flies") + } + + func test_thePointerCapture_startsAndStopsOnce_andTurnsOnlyWhileHeld() { + var turns: [simd_float2] = [] + let capture = VisionProPointerCapture { turns.append($0) } + XCTAssertFalse(capture.isCapturing) + capture.start() + capture.start() + XCTAssertTrue(capture.isCapturing) + capture.stop() + capture.stop() + XCTAssertFalse(capture.isCapturing) + XCTAssertTrue(turns.isEmpty) + } + + func test_endingThePreview_putsTheEditorBack() { + let camera = findSceneCamera() + cameraLookAt(entityId: camera, eye: simd_float3(1, 2, 3), target: .zero, up: simd_float3(0, 1, 0)) + var dismissed = 0 + session.openSpace = { true } + session.dismissSpace = { dismissed += 1 } + session.begin() + session.spaceDidOpen(frames: VisionProFakeFrames(device: renderInfo.device, eyeSize: (64, 48), count: nil)) + // The keys flew the camera meanwhile. + cameraLookAt(entityId: camera, eye: simd_float3(4, 5, 6), target: .zero, up: simd_float3(0, 1, 0)) + + session.end() + + XCTAssertFalse(VisionProPreviewState.shared.isPreviewing) + XCTAssertFalse(ViewportCameras.isLockedPreview) + XCTAssertFalse(renderInfo.isXRStereoMode) + XCTAssertTrue(renderer.metalView.delegate === renderer) + let eye = getCameraEye(entityId: camera) + XCTAssertEqual(eye.x, 4, accuracy: 0.001, "the camera stays where it was flown") + XCTAssertEqual(eye.y, 5, accuracy: 0.001) + XCTAssertEqual(eye.z, 6, accuracy: 0.001) + waitForState(.idle) + XCTAssertEqual(dismissed, 1) + } + + func test_aHeadsetThatCloses_endsThePreviewByItself() { + session.openSpace = { true } + session.begin() + let frames = VisionProFakeFrames(device: renderInfo.device, eyeSize: (64, 48), count: nil) + session.spaceDidOpen(frames: frames) + + frames.stop() + + waitForState(.idle) + XCTAssertFalse(VisionProPreviewState.shared.isPreviewing) + XCTAssertFalse(renderInfo.isXRStereoMode) + } + + func test_anotherCameraTakingOver_endsThePreview_andLeavesThatCameraAlone() { + session.openSpace = { true } + session.begin() + session.spaceDidOpen(frames: VisionProFakeFrames(device: renderInfo.device, eyeSize: (64, 48), count: nil)) + + // The scene was loaded again: a new camera is the active one. + let newCamera = createEntity() + registerComponent(entityId: newCamera, componentType: CameraComponent.self) + cameraLookAt(entityId: newCamera, eye: simd_float3(7, 8, 9), target: .zero, up: simd_float3(0, 1, 0)) + CameraSystem.shared.activeCamera = newCamera + + waitForState(.idle) + XCTAssertFalse(VisionProPreviewState.shared.isPreviewing) + XCTAssertFalse(renderInfo.isXRStereoMode) + let eye = getCameraEye(entityId: newCamera) + XCTAssertEqual(eye.x, 7, accuracy: 0.001, "the new camera is left where the scene put it") + XCTAssertEqual(eye.y, 8, accuracy: 0.001) + XCTAssertEqual(eye.z, 9, accuracy: 0.001) + } + + func test_thePreviewIsNotAskedFor_whileTheGamePlays() { + gameMode = true + session.openSpace = { true } + + session.begin() + + XCTAssertEqual(session.state, .idle) + } + + func test_theMenuItem_followsTheState() { + XCTAssertTrue(VisionProPreviewSession.menuItem(for: .unavailable, isPlaying: false).isHidden) + let idle = VisionProPreviewSession.menuItem(for: .idle, isPlaying: false) + XCTAssertEqual(idle.title, "Preview on Apple Vision Pro") + XCTAssertTrue(idle.isEnabled) + XCTAssertFalse(idle.isHidden) + XCTAssertFalse(VisionProPreviewSession.menuItem(for: .idle, isPlaying: true).isEnabled, "not while the game plays") + let previewing = VisionProPreviewSession.menuItem(for: .previewing, isPlaying: false) + XCTAssertEqual(previewing.title, "Stop the Preview on Apple Vision Pro") + XCTAssertTrue(previewing.isEnabled) + XCTAssertEqual(VisionProPreviewSession.menuItem(for: .connecting, isPlaying: false).title, "Stop the Preview on Apple Vision Pro") + XCTAssertFalse(VisionProPreviewSession.menuItem(for: .ending, isPlaying: false).isEnabled) + } +} From d238e1ec2c61349c2cb46692f889f335419c2aa4 Mon Sep 17 00:00:00 2001 From: Javier Segura Date: Wed, 7 Oct 2026 09:04:04 +0200 Subject: [PATCH 3/3] [Patch] Vision Pro preview: the editor waits for a loop that outlives the stop, undrawn drawables are cleared, two eyes at most From Harold's review of #153. - `stopLoopAndRestore` went on after its wait and re-made the renderer's stereo targets under a loop that might still draw into them. A timeout is logged now, the session stays in `ending`, and the editor is put back when the loop's thread ends. The wait is `loopStopTimeout`, which tests shorten. - Every drawable the compositor hands over gets the head's anchor, and a drawable beyond the first is cleared before it is presented, so that it shows nothing rather than what its textures held. - The loop draws the first two eyes, the engine's stereo, and says once when a frame brings more, instead of drawing a third eye over the second. Tests: a loop held inside a frame outlives the stop and the editor is put back when it is released; a three-eye frame draws two and leaves the third untouched. --- .../VisionPro/CompositorFrameSource.swift | 18 ++++++++- .../VisionPro/VisionProPreviewLoop.swift | 12 +++++- .../VisionPro/VisionProPreviewSession.swift | 38 ++++++++++++++++--- .../VisionProFakeFrames.swift | 13 +++++-- .../VisionProPreviewLoopTests.swift | 20 ++++++++++ .../VisionProPreviewSessionTests.swift | 38 +++++++++++++++++++ 6 files changed, 126 insertions(+), 13 deletions(-) diff --git a/Sources/UntoldEditor/Editor/VisionPro/CompositorFrameSource.swift b/Sources/UntoldEditor/Editor/VisionPro/CompositorFrameSource.swift index 909d7ad..b1910e4 100644 --- a/Sources/UntoldEditor/Editor/VisionPro/CompositorFrameSource.swift +++ b/Sources/UntoldEditor/Editor/VisionPro/CompositorFrameSource.swift @@ -128,7 +128,7 @@ func acquireEyes() -> VisionProFrameEyes? { // The headset is the first drawable; another target the compositor - // hands over is presented and left blank. + // hands over is cleared and presented, see `present`. drawables = frame.queryDrawables() guard let drawable = drawables.first else { return nil @@ -137,7 +137,9 @@ guard let anchor = source.deviceAnchor(for: presentationTime) else { return VisionProFrameEyes(originFromDevice: nil, eyes: []) } - drawable.deviceAnchor = anchor + for drawable in drawables { + drawable.deviceAnchor = anchor + } let originFromDevice = anchor.originFromAnchorTransform let eyes = drawable.views.indices.map { index in VisionProEye( @@ -151,6 +153,18 @@ } func present(commandBuffer: MTLCommandBuffer) { + // A drawable beyond the first was not drawn: it is cleared, so + // that it shows nothing rather than what its textures held. + for drawable in drawables.dropFirst() { + for texture in drawable.colorTextures { + let pass = MTLRenderPassDescriptor() + pass.colorAttachments[0].texture = texture + pass.colorAttachments[0].loadAction = .clear + pass.colorAttachments[0].storeAction = .store + pass.colorAttachments[0].clearColor = MTLClearColor(red: 0, green: 0, blue: 0, alpha: 1) + commandBuffer.makeRenderCommandEncoder(descriptor: pass)?.endEncoding() + } + } for drawable in drawables { drawable.encodePresent(commandBuffer: commandBuffer) } diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewLoop.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewLoop.swift index 2599319..d9525a1 100644 --- a/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewLoop.swift +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewLoop.swift @@ -44,7 +44,11 @@ final class VisionProPreviewLoop { private var originFromScene = matrix_identity_float4x4 /// The headset's first tracked pose: what stands on the camera. private var deviceStart: simd_float4x4? + /// One per eye the engine draws: its stereo path has two, left and right + /// (`renderInfo.currentEye`). A frame with more eyes gets these two drawn + /// and the rest left, said once. private let passDescriptors = [MTLRenderPassDescriptor(), MTLRenderPassDescriptor()] + private var saidTooManyEyes = false /// Frames in which the eyes were drawn, for the tests. private(set) var framesDrawn = 0 @@ -179,8 +183,12 @@ final class VisionProPreviewLoop { executeRadixSort(commandBuffer) } - for (index, eye) in acquired.eyes.enumerated() { - let descriptor = passDescriptors[min(index, passDescriptors.count - 1)] + if acquired.eyes.count > passDescriptors.count, saidTooManyEyes == false { + saidTooManyEyes = true + Logger.log(message: "Vision Pro preview: the headset's frame has \(acquired.eyes.count) eyes; the engine draws two, the rest are left blank.") + } + for (index, eye) in acquired.eyes.prefix(passDescriptors.count).enumerated() { + let descriptor = passDescriptors[index] descriptor.colorAttachments[0].texture = eye.colorTexture descriptor.colorAttachments[0].loadAction = .clear descriptor.colorAttachments[0].storeAction = .store diff --git a/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewSession.swift b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewSession.swift index 2b490e8..5644e4c 100644 --- a/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewSession.swift +++ b/Sources/UntoldEditor/Editor/VisionPro/VisionProPreviewSession.swift @@ -58,6 +58,11 @@ final class VisionProPreviewSession: ObservableObject { private var loopDone: DispatchSemaphore? private var mirror: VisionProMirror? private var pointer: VisionProPointerCapture? + /// How long `end()` waits for the loop to finish the frame it is on + /// before leaving the rest to the loop's own end; tests shorten it. + var loopStopTimeout: TimeInterval = 2 + /// The loop outlived the wait: the editor is put back when it ends. + private(set) var restoreWaitsForTheLoop = false var isPreviewing: Bool { state == .previewing @@ -121,14 +126,23 @@ final class VisionProPreviewSession: ObservableObject { } Task { @MainActor [weak self] in await self?.dismissSpace?() - guard let self, state == .ending else { return } + // A loop that outlived the wait puts the editor back, and ends + // the ending, when it ends. + guard let self, state == .ending, restoreWaitsForTheLoop == false else { return } state = .idle } } - /// The loop ended on its own: the headset closed the space, or the - /// stream broke. + /// The loop's thread ended: the headset closed the space, the stream + /// broke, or a loop that outlived `end()`'s wait is done at last. private func loopDidEnd() { + if restoreWaitsForTheLoop { + restoreEditor() + if state == .ending { + state = .idle + } + return + } guard state == .previewing else { return } @@ -186,12 +200,26 @@ final class VisionProPreviewSession: ObservableObject { thread.start() } + /// Stops the loop and puts the editor back once the loop has ended: the + /// loop finishes the frame it is on, through the renderer's stereo targets + /// and the mirror, so neither is taken from under it. Should the frame + /// take longer than the wait, as a command buffer that never completes + /// would, the editor is put back when the loop's thread ends; the session + /// stays in `ending` meanwhile. private func stopLoopAndRestore() { loop?.stop() - // The loop finishes the frame it is on. - _ = loopDone?.wait(timeout: .now() + .seconds(2)) + guard loopDone?.wait(timeout: .now() + loopStopTimeout) == .success else { + Logger.log(message: "Vision Pro preview: the headset's loop did not end within \(loopStopTimeout) s; the editor is put back when it does.") + restoreWaitsForTheLoop = true + return + } + restoreEditor() + } + + private func restoreEditor() { loop = nil loopDone = nil + restoreWaitsForTheLoop = false if let renderer { renderer.metalView.delegate = renderer diff --git a/Tests/UntoldEditorTests/VisionProFakeFrames.swift b/Tests/UntoldEditorTests/VisionProFakeFrames.swift index c3b11a3..1941ea6 100644 --- a/Tests/UntoldEditorTests/VisionProFakeFrames.swift +++ b/Tests/UntoldEditorTests/VisionProFakeFrames.swift @@ -84,10 +84,14 @@ final class VisionProFakeFrames: VisionProFrameSource { /// Open-ended frames carry eyes, as a headset that shows the scene; off, /// they carry none, as one that shows nothing yet. var showsEyesWhileOpen = false + /// Run on the loop's thread before each open-ended frame is handed over: + /// a test holds the loop there. + var beforeEachFrame: (() -> Void)? /// Textures of `eyeSize` for `count` frames; `count` nil keeps the source /// open with empty frames until it is stopped. - init(device: MTLDevice, eyeSize: (width: Int, height: Int), count: Int?) { + /// `eyes` beyond two stand for a headset with more views than the engine draws. + init(device: MTLDevice, eyeSize: (width: Int, height: Int), count: Int?, eyes: Int = 2) { let color = MTLTextureDescriptor.texture2DDescriptor( pixelFormat: .bgra8Unorm_srgb, width: eyeSize.width, height: eyeSize.height, mipmapped: false ) @@ -98,8 +102,8 @@ final class VisionProFakeFrames: VisionProFrameSource { ) depth.usage = [.renderTarget, .shaderRead] depth.storageMode = .private - colorTextures = (0 ..< 2).compactMap { _ in device.makeTexture(descriptor: color) } - depthTextures = (0 ..< 2).compactMap { _ in device.makeTexture(descriptor: depth) } + colorTextures = (0 ..< eyes).compactMap { _ in device.makeTexture(descriptor: color) } + depthTextures = (0 ..< eyes).compactMap { _ in device.makeTexture(descriptor: depth) } left = count ?? 0 keepsOpen = count == nil } @@ -114,6 +118,7 @@ final class VisionProFakeFrames: VisionProFrameSource { if keepsOpen { // A headset that is on but shows nothing yet: a frame now and then. _ = opened.wait(timeout: .now() + .milliseconds(20)) + beforeEachFrame?() guard isStopped == false else { return nil } let frame = Frame(eyes: showsEyesWhileOpen ? eyes() : nil) frames.append(frame) @@ -136,7 +141,7 @@ final class VisionProFakeFrames: VisionProFrameSource { private func eyes() -> VisionProFrameEyes { let aspect = Float(colorTextures[0].width) / Float(colorTextures[0].height) let projection = matrixPerspectiveRightHandReverseZ(fovyRadians: .pi / 2, aspectRatio: aspect, nearZ: 0.1, farZ: 100) - let eyes = (0 ..< 2).map { index -> VisionProEye in + let eyes = colorTextures.indices.map { index -> VisionProEye in let side = index == 0 ? -Self.halfEyeDistance : Self.halfEyeDistance let deviceFromView = matrix4x4Translation(side, 0, 0) return VisionProEye( diff --git a/Tests/UntoldEditorTests/VisionProPreviewLoopTests.swift b/Tests/UntoldEditorTests/VisionProPreviewLoopTests.swift index fbb46b3..fc207d7 100644 --- a/Tests/UntoldEditorTests/VisionProPreviewLoopTests.swift +++ b/Tests/UntoldEditorTests/VisionProPreviewLoopTests.swift @@ -306,6 +306,26 @@ final class VisionProPreviewLoopTests: XCTestCase { XCTAssertGreaterThan(flown.eye.y, 1, "without a headset the editor's own W climbs where the camera looks") } + func test_aFrameWithMoreEyesThanTheEngineDraws_getsTwo_andClobbersNone() throws { + _ = makeCubeInFrontOfTheCamera() + let frames = VisionProFakeFrames(device: renderInfo.device, eyeSize: (128, 96), count: 6, eyes: 3) + renderer.enterStereo() + let loop = VisionProPreviewLoop(renderer: renderer, frames: frames, start: start(), mirror: nil) + + loop.run() + try XCTUnwrap(frames.frames.last?.presentedIn).waitUntilCompleted() + + XCTAssertEqual(loop.framesDrawn, 6) + for index in 0 ..< 2 { + let eye = frames.colorTextures[index] + let middle = VisionProFakeFrames.pixel(of: eye, x: 64, y: 48) + let corner = VisionProFakeFrames.pixel(of: eye, x: 4, y: 4) + XCTAssertGreaterThan(abs(brightness(middle) - brightness(corner)), 40, "eye \(index) shows the cube") + } + let third = VisionProFakeFrames.pixel(of: frames.colorTextures[2], x: 64, y: 48) + XCTAssertEqual(brightness(third), 0, "the third eye is left as it was: nothing drawn, nothing clobbered") + } + func test_aFrameWithoutEyes_isPresentedWithNothingDrawn() { _ = makeCubeInFrontOfTheCamera() let frames = VisionProFakeFrames(device: renderInfo.device, eyeSize: (64, 48), count: 1) diff --git a/Tests/UntoldEditorTests/VisionProPreviewSessionTests.swift b/Tests/UntoldEditorTests/VisionProPreviewSessionTests.swift index 110fb02..39032f4 100644 --- a/Tests/UntoldEditorTests/VisionProPreviewSessionTests.swift +++ b/Tests/UntoldEditorTests/VisionProPreviewSessionTests.swift @@ -145,6 +145,44 @@ final class VisionProPreviewSessionTests: XCTestCase { XCTAssertTrue(turns.isEmpty) } + func test_aLoopThatOutlivesTheStop_putsTheEditorBack_whenItEnds() { + session.loopStopTimeout = 0.05 + var dismissed = 0 + session.openSpace = { true } + session.dismissSpace = { dismissed += 1 } + let frames = VisionProFakeFrames(device: renderInfo.device, eyeSize: (64, 48), count: nil) + // The loop is held in the middle of a frame, as a command buffer that + // never completes would hold it. + let gate = DispatchSemaphore(value: 0) + let arrived = DispatchSemaphore(value: 0) + frames.beforeEachFrame = { + arrived.signal() + gate.wait() + } + session.begin() + session.spaceDidOpen(frames: frames) + XCTAssertEqual(arrived.wait(timeout: .now() + .seconds(5)), .success, "the loop reached the frame it is held in") + + session.end() + + XCTAssertEqual(session.state, .ending, "the ending waits for the loop") + XCTAssertTrue(session.restoreWaitsForTheLoop) + XCTAssertTrue(VisionProPreviewState.shared.isPreviewing, "the loop still draws: nothing is taken from under it") + XCTAssertTrue(renderInfo.isXRStereoMode) + XCTAssertFalse(renderer.metalView.delegate === renderer) + RunLoop.main.run(until: Date().addingTimeInterval(0.1)) + XCTAssertEqual(session.state, .ending, "the dismiss alone does not end the ending") + + gate.signal() + waitForState(.idle) + + XCTAssertFalse(session.restoreWaitsForTheLoop) + XCTAssertFalse(VisionProPreviewState.shared.isPreviewing) + XCTAssertFalse(renderInfo.isXRStereoMode) + XCTAssertTrue(renderer.metalView.delegate === renderer) + XCTAssertEqual(dismissed, 1) + } + func test_endingThePreview_putsTheEditorBack() { let camera = findSceneCamera() cameraLookAt(entityId: camera, eye: simd_float3(1, 2, 3), target: .zero, up: simd_float3(0, 1, 0))