From 0374a2562a17a618c474a5e1ace12cfb3b70e47f Mon Sep 17 00:00:00 2001 From: michal Date: Thu, 22 Oct 2026 17:28:17 +0200 Subject: [PATCH 1/3] feat: get output timestamp in the audiocontext --- .claude/skills/thread-safety-itc/SKILL.md | 2 + .claude/skills/web-audio-api/SKILL.md | 2 +- .../FabricExampleTests/AudioPlayerTests.mm | 5 +- .../audiodocs/docs/core/audio-context.mdx | 25 +++++ .../docs/other/web-audio-api-coverage.mdx | 2 +- .../android/core/AndroidAudioPlayer.cpp | 41 +++++++ .../android/core/AndroidAudioPlayer.h | 5 + .../HostObjects/AudioContextHostObject.cpp | 13 ++- .../HostObjects/AudioContextHostObject.h | 1 + .../common/cpp/audioapi/core/AudioContext.cpp | 15 +++ .../common/cpp/audioapi/core/AudioContext.h | 7 ++ .../common/cpp/audioapi/core/AudioPlayer.h | 42 +++++++ .../cpp/audioapi/core/types/AudioTimestamp.h | 14 +++ .../cpp/audioapi/core/utils/OutputTimestamp.h | 60 ++++++++++ .../src/core/utils/OutputTimestampTest.cpp | 104 ++++++++++++++++++ .../ios/audioapi/ios/core/IOSAudioPlayer.h | 8 +- .../ios/audioapi/ios/core/IOSAudioPlayer.mm | 51 ++++++++- .../ios/audioapi/ios/core/NativeAudioPlayer.h | 6 +- .../ios/audioapi/ios/core/NativeAudioPlayer.m | 2 +- .../react-native-audio-api/src/api.web.ts | 1 + .../src/core/AudioContext.ts | 11 +- .../src/jsi-interfaces.ts | 2 + .../react-native-audio-api/src/mock/index.ts | 6 +- packages/react-native-audio-api/src/types.ts | 11 ++ .../src/web-core/AudioContext.web.ts | 13 ++- .../tests/audio-context-options.test.ts | 12 ++ .../react-native-audio-api/tests/mock.test.ts | 4 + 27 files changed, 450 insertions(+), 15 deletions(-) create mode 100644 packages/react-native-audio-api/common/cpp/audioapi/core/types/AudioTimestamp.h create mode 100644 packages/react-native-audio-api/common/cpp/audioapi/core/utils/OutputTimestamp.h create mode 100644 packages/react-native-audio-api/common/cpp/test/src/core/utils/OutputTimestampTest.cpp diff --git a/.claude/skills/thread-safety-itc/SKILL.md b/.claude/skills/thread-safety-itc/SKILL.md index 621a346bc..c2b804b16 100644 --- a/.claude/skills/thread-safety-itc/SKILL.md +++ b/.claude/skills/thread-safety-itc/SKILL.md @@ -193,6 +193,8 @@ On Android, `AndroidAudioPlayer::onErrorAfterClose` also takes `driverMutex_` be **Player base class:** all three drivers (`AndroidAudioPlayer` on Android, `IOSAudioPlayer`, `NodeAudioPlayer` in `wpt_tests/src/`) derive from `AudioPlayer` (`common/cpp/audioapi/core/AudioPlayer.h`). It owns the shared driver state as protected members: `renderAudio_`, `renderBuffer_` (one render quantum, allocated once in the constructor), `sampleRate_`, `channelCount_`, `currentRenders_`, `context_`, `driverMutex_`, and `isRunning_`. It also takes one constructor signature for every platform. `AudioContext::initialize()` calls `createPlatformAudioPlayer()` (declared in `AudioPlayer.h`, defined next to each subclass; the recorder has the matching `createPlatformAudioRecorder()` in `AudioRecorder.h`) instead of including a platform header. Keep it that way: `IOSAudioPlayer.h` and `IOSAudioRecorder.h` are Objective-C++ only, and a plain-C++ `.cpp` that includes them sees a different class definition (raw pointers instead of ARC `__strong` members), so any constructor or inline member emitted from that TU skips the retain and the native object is freed early. Render through `renderNormalizedQuantum()`, which keeps the output peak-limiting in the player so offline renders are never limited. `AudioPlayerTests.mm` (fabric-example XCTest) hand-copies `IOSAudioPlayer`'s member layout to reach its private members, so any change to `IOSAudioPlayer`'s data members must also be made there. +**Output timestamps (`getOutputTimestamp`):** the audio thread never asks the platform when a buffer will be heard — `AVAudioSession.outputLatency` is an Objective-C session call and Oboe's `getTimestamp` enters AAudio. Each platform callback only publishes an `OutputAnchor` (context frame of its first output sample, the callback's host time, and on Android `getFramesWritten()`) through the base class's `TripleBuffer`; the JS thread, its single reader, adds the latency or queries the device position in `getOutputPresentation()`, and `outputTimestampAt()` (`core/utils/OutputTimestamp.h`, pure and unit-tested) extrapolates that to now. The context frame comes from the player's own `renderedFrames_`, which equals the context's `currentSampleFrame_` only because the realtime context renders exclusively through `renderNormalizedQuantum()` — keep it that way. On iOS subtract `pendingSavedCount_`: a hardware buffer can start with the unplayed tail of the previous quantum. Players that cannot tell (the WPT `NodeAudioPlayer`) inherit the `std::nullopt` default, which the context reports as the spec's `{0, 0}`. + **Graph producer self-drain:** `Graph::enableProducerSelfDrain()` makes producer threads drain the event channels themselves after each enqueue; `disableProducerSelfDrain()` hands consumption back to the audio/render thread. Both flush the channels internally (no separate `processEvents()` call needed) and serialize with in-flight drains via `selfDrainMutex_`, because two producers can drain concurrently: the JS thread (mutations) and the GC finalizer thread (`removeNode`, which self-drains after its Channel B orphan send — otherwise a finalizer burst with no consumer fills the bounded channel and blocks forever, e.g. at process exit). Enable only when there is no audio/render consumer (realtime: construction + after `suspend`/`close` quiescence; offline: before `startRendering` and after a scheduled suspend); disable *before* starting the audio/render consumer; re-enable if start/resume fails. **Context lifecycle promises:** HostObjects wrap JSI `Promise`s via `ContextPromiseResolver` diff --git a/.claude/skills/web-audio-api/SKILL.md b/.claude/skills/web-audio-api/SKILL.md index 63d252702..484f5a775 100644 --- a/.claude/skills/web-audio-api/SKILL.md +++ b/.claude/skills/web-audio-api/SKILL.md @@ -113,7 +113,7 @@ Current status (from `packages/audiodocs/docs/other/web-audio-api-coverage.mdx`) ### Partially implemented 🚧 | Interface | What's available | |---|---| -| `AudioContext` | `close`, `suspend`, `resume`, `currentTime`, `destination`, `sampleRate`, `state` | +| `AudioContext` | `baseLatency`, `outputLatency`, `getOutputTimestamp`, `close`, `suspend`, `resume`, `currentTime`, `destination`, `sampleRate`, `state` | | `BaseAudioContext` | `currentTime`, `destination`, `listener`, `sampleRate`, `state`, `decodeAudioData`, all `create*` for available nodes | | `AudioListener` | All nine AudioParams; used by `PannerNode` for 3D spatialization; deprecated `setPosition` / `setOrientation` provided in JS (set AudioParam values) | | `PannerNode` | Equal-power spatialization, distance models, cone gain; `panningModel` is `'equalpower'` only | diff --git a/apps/fabric-example/ios/FabricExampleTests/AudioPlayerTests.mm b/apps/fabric-example/ios/FabricExampleTests/AudioPlayerTests.mm index f70ceafec..7548fc3ad 100644 --- a/apps/fabric-example/ios/FabricExampleTests/AudioPlayerTests.mm +++ b/apps/fabric-example/ios/FabricExampleTests/AudioPlayerTests.mm @@ -36,6 +36,7 @@ [[nodiscard]] double getBaseLatency() const override; [[nodiscard]] double getOutputLatency() const override; + [[nodiscard]] std::optional getOutputPresentation() override; protected: NativeAudioPlayer *createNativePlayer(); @@ -334,7 +335,7 @@ - (void)tearDown - (NativeAudioPlayer *)createPlayerWithRenderCallCount:(NSInteger *)renderCallCount { - return [[NativeAudioPlayer alloc] initWithRenderAudio:^(AudioBufferList *outputBuffer, int numFrames) { + return [[NativeAudioPlayer alloc] initWithRenderAudio:^(AudioBufferList *outputBuffer, int numFrames, const AudioTimeStamp *) { if (renderCallCount != nullptr) { *renderCallCount += 1; } @@ -399,7 +400,7 @@ - (void)testRenderBlockInvokesRenderAudioWhenBufferCountMatches __block AVAudioFrameCount renderedFrameCount = 0; __block AudioBufferList *renderedBufferList = nullptr; NativeAudioPlayer *player = [[NativeAudioPlayer alloc] - initWithRenderAudio:^(AudioBufferList *outputBuffer, int numFrames) { + initWithRenderAudio:^(AudioBufferList *outputBuffer, int numFrames, const AudioTimeStamp *) { renderCallCount += 1; renderedFrameCount = static_cast(numFrames); renderedBufferList = outputBuffer; diff --git a/packages/audiodocs/docs/core/audio-context.mdx b/packages/audiodocs/docs/core/audio-context.mdx index ab5d158d0..1779fd64c 100644 --- a/packages/audiodocs/docs/core/audio-context.mdx +++ b/packages/audiodocs/docs/core/audio-context.mdx @@ -54,6 +54,31 @@ Creates [`MediaElementAudioSourceNode`](../sources/media-element-audio-source-no #### Returns [`MediaElementAudioSourceNode`](../sources/media-element-audio-source-node.mdx). +### `getOutputTimestamp` + +Returns the context time of the sample leaving the output device now, together with now on the `performance.now()` clock. Use it to line audio up with anything else timed by `performance.now()`: animations, network messages, other devices. + +The pair is measured on the audio thread from the platform's own presentation timestamps, so output latency is already included and it is not limited to the render-quantum steps `currentTime` moves in. Both values are `0` until the context has rendered its first block. While the context is suspended the values stop advancing. + +```tsx +const { contextTime, performanceTime } = audioContext.getOutputTimestamp(); + +// The context time at which a sound is heard at `perfMs` (a performance.now() value): +const heardAt = (perfMs: number) => + contextTime + (perfMs - performanceTime) / 1000; + +oscillator.start(heardAt(performance.now() + 500)); // heard ~500 ms from now +``` + +On iOS the presentation time is CoreAudio's host time for each output buffer plus `AVAudioSession.outputLatency`. On Android it comes from Oboe's `getTimestamp(CLOCK_MONOTONIC)`, falling back to callback time plus `outputLatency` when the stream does not report timestamps (OpenSL ES). + +#### Returns `AudioTimestamp`. + +| Name | Type | Description | +| :---: | :---: | :---- | +| `contextTime` | `number` | Seconds, on the [`currentTime`](./base-audio-context.mdx#properties) axis. Never exceeds `currentTime`. | +| `performanceTime` | `number` | Milliseconds, on the `performance.now()` axis. | + ### `close` Closes the audio context, releasing any system audio resources that it uses. diff --git a/packages/audiodocs/docs/other/web-audio-api-coverage.mdx b/packages/audiodocs/docs/other/web-audio-api-coverage.mdx index 402ff0a79..766cd6e45 100644 --- a/packages/audiodocs/docs/other/web-audio-api-coverage.mdx +++ b/packages/audiodocs/docs/other/web-audio-api-coverage.mdx @@ -36,7 +36,7 @@ This page tracks how closely React Native Audio API follows the [Web Audio API]( | PeriodicWave | ✅ | | StereoPannerNode | ✅ | | WaveShaperNode | ✅ | -| AudioContext | 🚧 | Available props and methods: `baseLatency`, `outputLatency`, `close`, `suspend`, `resume` | +| AudioContext | 🚧 | Available props and methods: `baseLatency`, `outputLatency`, `getOutputTimestamp`, `close`, `suspend`, `resume` | | BaseAudioContext | 🚧 | Available props and methods: `currentTime`, `destination`, `listener`, `sampleRate`, `state`, `decodeAudioData`, all create methods for available or partially implemented nodes | | PannerNode | 🚧 | Equal-power spatialization, distance models, and cone gain. `panningModel` is `'equalpower'` only (`'HRTF'` not exposed yet). | | AudioSinkInfo | ❌ | diff --git a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.cpp b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.cpp index 7680f1f89..aa6b4bc44 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.cpp +++ b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.cpp @@ -8,8 +8,13 @@ #include +#include + #include +#include +#include #include +#include #include #include #include @@ -176,6 +181,17 @@ AndroidAudioPlayer::onAudioReady(AudioStream *oboeStream, void *audioData, int32 lastCallbackFrameCount_.store(numFrames, std::memory_order_release); } + if (isRunning_.load(std::memory_order_acquire)) { + // Oboe counts this buffer into getFramesWritten() only after the callback returns. + publishOutputAnchor( + {.contextFrame = renderedFrames_, + .hostTimeNs = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(), + .streamFrame = oboeStream->getFramesWritten(), + .isValid = true}); + } + const CurrentRenderScope renderScope(currentRenders_); auto *buffer = static_cast(audioData); @@ -298,6 +314,31 @@ double AndroidAudioPlayer::getBaseLatency() const { return static_cast(RENDER_QUANTUM_SIZE) / static_cast(sampleRate_); } +std::optional AndroidAudioPlayer::getOutputPresentation() { + const auto anchor = latestOutputAnchor(); + if (!anchor) { + return std::nullopt; + } + + { + std::scoped_lock lock(streamMutex_); + if (isRunningLocked()) { + const auto presented = mStream_->getTimestamp(CLOCK_MONOTONIC); + if (presented) { + return OutputPresentation{ + .contextFrame = + anchor->contextFrame + (presented.value().position - anchor->streamFrame), + .presentationTimeNs = presented.value().timestamp}; + } + } + } + + const auto outputLatencyNs = static_cast(std::llround(getOutputLatency() * 1e9)); + return OutputPresentation{ + .contextFrame = anchor->contextFrame, + .presentationTimeNs = anchor->hostTimeNs + outputLatencyNs}; +} + double AndroidAudioPlayer::getOutputLatency() const { std::scoped_lock lock(streamMutex_); if (mStream_ == nullptr || !isInitialized_.load(std::memory_order_acquire) || diff --git a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.h b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.h index 214273515..0a1e683b1 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.h +++ b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.h @@ -47,6 +47,11 @@ class AndroidAudioPlayer : public AudioPlayer, [[nodiscard]] double getBaseLatency() const override; [[nodiscard]] double getOutputLatency() const override; + /// Maps the stream position Oboe reports as presented (`getTimestamp`) onto context frames. + /// Without a device timestamp (OpenSL ES, or right after start) it assumes the last callback's + /// first frame plays one output latency after that callback ran. + [[nodiscard]] std::optional getOutputPresentation() override; + DataCallbackResult onAudioReady(AudioStream *oboeStream, void *audioData, int32_t numFrames) override; diff --git a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.cpp b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.cpp index 4e9bbaee9..6bbe68431 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.cpp +++ b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.cpp @@ -31,7 +31,8 @@ AudioContextHostObject::AudioContextHostObject( JSI_EXPORT_FUNCTION(AudioContextHostObject, close), JSI_EXPORT_FUNCTION(AudioContextHostObject, resume), JSI_EXPORT_FUNCTION(AudioContextHostObject, suspend), - JSI_EXPORT_FUNCTION(AudioContextHostObject, createMediaElementSource)); + JSI_EXPORT_FUNCTION(AudioContextHostObject, createMediaElementSource), + JSI_EXPORT_FUNCTION(AudioContextHostObject, getOutputTimestamp)); } AudioContextHostObject::~AudioContextHostObject() { @@ -64,6 +65,16 @@ JSI_PROPERTY_GETTER_IMPL(AudioContextHostObject, outputLatency) { return {audioContext->getOutputLatency()}; } +JSI_HOST_FUNCTION_IMPL(AudioContextHostObject, getOutputTimestamp) { + auto audioContext = std::static_pointer_cast(context_); + const auto timestamp = audioContext->getOutputTimestamp(); + + auto jsTimestamp = jsi::Object(runtime); + jsTimestamp.setProperty(runtime, "contextTime", timestamp.contextTime); + jsTimestamp.setProperty(runtime, "performanceTime", timestamp.performanceTime); + return jsTimestamp; +} + JSI_PROPERTY_GETTER_IMPL(AudioContextHostObject, baseLatency) { auto audioContext = std::static_pointer_cast(context_); return {audioContext->getBaseLatency()}; diff --git a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.h b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.h index b3cf114ca..506007dd5 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/HostObjects/AudioContextHostObject.h @@ -31,6 +31,7 @@ class AudioContextHostObject : public BaseAudioContextHostObject { JSI_HOST_FUNCTION_DECL(createMediaElementSource); JSI_PROPERTY_GETTER_DECL(outputLatency); + JSI_HOST_FUNCTION_DECL(getOutputTimestamp); JSI_PROPERTY_GETTER_DECL(baseLatency); JSI_PROPERTY_SETTER_DECL(onerror); }; diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.cpp b/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.cpp index db7270bec..05ab56012 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.cpp +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -173,6 +174,20 @@ double AudioContext::getBaseLatency() const { return audioPlayer_->getBaseLatency(); } +AudioTimestamp AudioContext::getOutputTimestamp() const { + if (audioPlayer_ == nullptr) { + return {}; + } + + const auto presentation = audioPlayer_->getOutputPresentation(); + if (!presentation) { + return {}; + } + + return outputTimestampAt( + *presentation, getSampleRate(), getCurrentSampleFrame(), audioPlayer_->isRunning()); +} + double AudioContext::getOutputLatency() const { if (audioPlayer_ == nullptr) { return 0.0; diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.h b/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.h index 065482dde..20730c5af 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/AudioContext.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,12 @@ class AudioContext : public BaseAudioContext { /// @returns The output latency in seconds. [[nodiscard]] double getOutputLatency() const; + /// @brief Spec `getOutputTimestamp()`: the context time of the sample leaving the output device + /// now, and now on the `performance.now()` clock. Both are zero before the first rendered block + /// or where the platform cannot tell. + /// @note JS Thread only. + [[nodiscard]] AudioTimestamp getOutputTimestamp() const; + /// @brief Called when the audio stream failed to rebuild. /// @note This method is called when the audio stream fails. void onStreamFail(); diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/AudioPlayer.h b/packages/react-native-audio-api/common/cpp/audioapi/core/AudioPlayer.h index c42cd5328..0ebf923ed 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/AudioPlayer.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/AudioPlayer.h @@ -3,14 +3,18 @@ #include #include #include +#include #include #include +#include #include +#include #include #include #include #include +#include #include namespace audioapi { @@ -50,7 +54,37 @@ class AudioPlayer { [[nodiscard]] virtual bool isRunning() const = 0; + /// @brief Which context frame leaves the output device when, or `std::nullopt` before the + /// first rendered callback or on a platform that cannot tell (the default). + /// @note JS Thread only: it is the single reader of the anchors published from the audio thread. + [[nodiscard]] virtual std::optional getOutputPresentation() { + return std::nullopt; + } + protected: + /// One render callback's first output sample, tied to the platform's own timeline. + struct OutputAnchor { + /// Context frame of the callback's first output sample. + std::int64_t contextFrame = 0; + /// `steady_clock` time the platform attached to that sample, before output latency. + std::int64_t hostTimeNs = 0; + /// Index of that sample in the device stream, counted by platform. + std::int64_t streamFrame = 0; + bool isValid = false; + }; + + /// @note Audio Thread only. + void publishOutputAnchor(const OutputAnchor &anchor) { + *outputAnchors_.getForWriter() = anchor; + outputAnchors_.publish(); + } + + /// @note JS Thread only. + [[nodiscard]] std::optional latestOutputAnchor() { + const OutputAnchor anchor = *outputAnchors_.getForReader(); + return anchor.isValid ? std::optional(anchor) : std::nullopt; + } + /// @brief Pulls `framesToProcess` frames from the graph into `renderBuffer_` and peak-normalizes /// them. /// @note Audio Thread only. The limiting lives in the player (not the destination node) so @@ -58,6 +92,7 @@ class AudioPlayer { void renderNormalizedQuantum(int framesToProcess) { renderAudio_(renderBuffer_.get(), framesToProcess); renderBuffer_->normalize(); + renderedFrames_ += framesToProcess; } std::function renderAudio_; @@ -70,6 +105,13 @@ class AudioPlayer { std::mutex *driverMutex_; std::atomic isRunning_{false}; AudioContextLatencyHint latencyHint_; + /// Frames pulled from the graph so far. The realtime context renders only through this player, + /// and `processGraph` advances its frame counter by exactly these frames, so this equals the + /// context frame of the next frame to render. Audio Thread only. + std::int64_t renderedFrames_ = 0; + + private: + TripleBuffer outputAnchors_; }; /// Builds the platform's AudioPlayer. diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/types/AudioTimestamp.h b/packages/react-native-audio-api/common/cpp/audioapi/core/types/AudioTimestamp.h new file mode 100644 index 000000000..a00effe8a --- /dev/null +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/types/AudioTimestamp.h @@ -0,0 +1,14 @@ +#pragma once + +namespace audioapi { + +/// Web Audio `AudioTimestamp`: one moment expressed on the context clock and on the +/// `performance.now()` clock. +struct AudioTimestamp { + /// Seconds, on the context's `currentTime` axis. + double contextTime = 0.0; + /// Milliseconds, on the `performance.now()` axis (`std::chrono::steady_clock`). + double performanceTime = 0.0; +}; + +} // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/utils/OutputTimestamp.h b/packages/react-native-audio-api/common/cpp/audioapi/core/utils/OutputTimestamp.h new file mode 100644 index 000000000..1cd25641b --- /dev/null +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/utils/OutputTimestamp.h @@ -0,0 +1,60 @@ +#pragma once + +#include + +#include +#include +#include + +namespace audioapi { + +/// A context sample frame and the `steady_clock` moment it leaves the output device. +struct OutputPresentation { + /// Signed: a platform may report a device position from before the context's first frame. + std::int64_t contextFrame = 0; + std::int64_t presentationTimeNs = 0; +}; + +/// @brief Spec `getOutputTimestamp()`: the frame leaving the device now, and the `steady_clock` +/// time of now. +/// @param presentation the latest frame/moment pair the platform reported. +/// @param renderedFrames frames the graph has rendered +[[nodiscard]] inline AudioTimestamp outputTimestampAt( + const OutputPresentation &presentation, + float sampleRate, + std::size_t renderedFrames, + bool isPlaying) { + constexpr double NANOSECONDS_PER_SECOND = 1e9; + constexpr double NANOSECONDS_PER_MILLISECOND = 1e6; + + const auto nowNs = std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + + auto frame = static_cast(presentation.contextFrame); + auto timeNs = static_cast(presentation.presentationTimeNs); + + // interpolation + if (isPlaying) { + frame += (static_cast(nowNs) - timeNs) * sampleRate / NANOSECONDS_PER_SECOND; + timeNs = static_cast(nowNs); + } + + const auto rendered = static_cast(renderedFrames); + // can happen if the audio stalls or is interrupted + if (frame > rendered) { + timeNs -= (frame - rendered) / sampleRate * NANOSECONDS_PER_SECOND; + frame = rendered; + } + + // frame 0 hasnt reach the speaker yet + if (frame < 0.0) { + timeNs -= frame / sampleRate * NANOSECONDS_PER_SECOND; + frame = 0.0; + } + + return { + .contextTime = frame / sampleRate, .performanceTime = timeNs / NANOSECONDS_PER_MILLISECOND}; +} + +} // namespace audioapi diff --git a/packages/react-native-audio-api/common/cpp/test/src/core/utils/OutputTimestampTest.cpp b/packages/react-native-audio-api/common/cpp/test/src/core/utils/OutputTimestampTest.cpp new file mode 100644 index 000000000..9da28d3c1 --- /dev/null +++ b/packages/react-native-audio-api/common/cpp/test/src/core/utils/OutputTimestampTest.cpp @@ -0,0 +1,104 @@ +#include +#include + +#include + +using namespace audioapi; + +namespace { + +constexpr float kSampleRate = 48000.0f; +constexpr std::int64_t kMs = 1'000'000; +/// Real time passing between the test reading the clock and the function reading it. +constexpr double kClockSkewMs = 5.0; + +std::int64_t steadyNowNs() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} + +double toMs(std::int64_t ns) { + return static_cast(ns) / kMs; +} + +} // namespace + +TEST(OutputTimestampTest, ExtrapolatesToNowWhilePlaying) { + // Frame 48000 (1 s) left the speaker 20 ms ago; the device is now ~960 frames on. + const auto nowNs = steadyNowNs(); + const OutputPresentation presentation{ + .contextFrame = 48000, .presentationTimeNs = nowNs - 20 * kMs}; + + const auto timestamp = + outputTimestampAt(presentation, kSampleRate, /*renderedFrames=*/96000, true); + + EXPECT_NEAR(timestamp.contextTime, 1.020, kClockSkewMs / 1000.0); + EXPECT_NEAR(timestamp.performanceTime, toMs(nowNs), kClockSkewMs); +} + +TEST(OutputTimestampTest, AcceptsAPresentationStillInTheFuture) { + // iOS reports the moment a buffer *will* play; looking back from before it is still linear. + const auto nowNs = steadyNowNs(); + const OutputPresentation presentation{ + .contextFrame = 48000, .presentationTimeNs = nowNs + 10 * kMs}; + + const auto timestamp = outputTimestampAt(presentation, kSampleRate, 96000, true); + + EXPECT_NEAR(timestamp.contextTime, 0.990, kClockSkewMs / 1000.0); + EXPECT_NEAR(timestamp.performanceTime, toMs(nowNs), kClockSkewMs); +} + +TEST(OutputTimestampTest, FreezesWhenTheDeviceIsNotPlaying) { + const auto presentationTimeNs = steadyNowNs() - 5000 * kMs; + const OutputPresentation presentation{ + .contextFrame = 48000, .presentationTimeNs = presentationTimeNs}; + + const auto timestamp = outputTimestampAt(presentation, kSampleRate, 96000, false); + + EXPECT_NEAR(timestamp.contextTime, 1.0, 1e-9); + EXPECT_NEAR(timestamp.performanceTime, toMs(presentationTimeNs), 1e-6); +} + +TEST(OutputTimestampTest, NeverRunsAheadOfRenderedFrames) { + // A stale report extrapolated over a long stall would claim frames the graph never rendered. + const auto presentationTimeNs = steadyNowNs() - 2000 * kMs; + const OutputPresentation presentation{ + .contextFrame = 48000, .presentationTimeNs = presentationTimeNs}; + + const auto timestamp = + outputTimestampAt(presentation, kSampleRate, /*renderedFrames=*/72000, true); + + EXPECT_NEAR(timestamp.contextTime, 1.5, 1e-9); + // Slid back along the same line: frame 72000 played 0.5 s after frame 48000. + EXPECT_NEAR(timestamp.performanceTime, toMs(presentationTimeNs) + 500.0, 1e-3); +} + +TEST(OutputTimestampTest, NeverReportsANegativeContextTime) { + // Right after start the device is still playing frames from before the context's first one. + const auto presentationTimeNs = steadyNowNs(); + const OutputPresentation presentation{ + .contextFrame = -480, .presentationTimeNs = presentationTimeNs}; + + const auto timestamp = outputTimestampAt(presentation, kSampleRate, 4096, true); + + EXPECT_DOUBLE_EQ(timestamp.contextTime, 0.0); + // Frame 0 reaches the speaker 10 ms after frame -480. + EXPECT_NEAR(timestamp.performanceTime, toMs(presentationTimeNs) + 10.0, 1e-3); +} + +TEST(OutputTimestampTest, KeepsBothCoordinatesOnOneMoment) { + const auto nowNs = steadyNowNs(); + + for (const std::int64_t offsetMs : {-77, 0, 23, 123}) { + const auto presentationTimeNs = nowNs + offsetMs * kMs; + const OutputPresentation presentation{ + .contextFrame = 12345, .presentationTimeNs = presentationTimeNs}; + + const auto timestamp = outputTimestampAt(presentation, kSampleRate, 1'000'000, true); + const double framesFromPresentation = + (timestamp.performanceTime - toMs(presentationTimeNs)) / 1000.0 * kSampleRate; + EXPECT_NEAR(timestamp.contextTime * kSampleRate, 12345.0 + framesFromPresentation, 1e-3) + << "presentation offset = " << offsetMs << " ms"; + } +} diff --git a/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.h b/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.h index 61507bfe9..5d0d94ecb 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.h +++ b/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.h @@ -33,13 +33,19 @@ class IOSAudioPlayer : public AudioPlayer { [[nodiscard]] double getBaseLatency() const override; [[nodiscard]] double getOutputLatency() const override; + /// The host time CoreAudio attaches to a buffer is when its first frame leaves the I/O + /// unit; the session's `outputLatency` covers the rest of the way to the speaker. + [[nodiscard]] std::optional getOutputPresentation() override; + private: void clearPendingSaved(); /// @note Audio Thread only /// Always pulls the graph in steps of RENDER_QUANTUM_SIZE; if the system /// buffer size is not a multiple of 128, the unused tail of the last quantum /// is kept (max 128 frames) and played at the start of the next callback. - void deliverOutputBuffers(AudioBufferList *outputData, int numFrames); + /// Publishes the context frame of the buffer's first sample against `timestamp`'s host time. + void + deliverOutputBuffers(AudioBufferList *outputData, int numFrames, const AudioTimeStamp *timestamp); /// Builds the native player and points its render and stream-failure blocks at this object. /// Runs as a member initializer, before the members below exist: the blocks only capture diff --git a/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.mm b/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.mm index 1698ab462..17c62156d 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.mm +++ b/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.mm @@ -2,7 +2,10 @@ #include #include +#include + #include +#include #include #include @@ -21,6 +24,19 @@ namespace { +const mach_timebase_info_data_t kHostTimebase = [] { + mach_timebase_info_data_t timebase{}; + mach_timebase_info(&timebase); + return timebase; +}(); + +/// Mach host time is the clock `steady_clock` reads on Apple platforms; only the unit differs. +std::int64_t hostTimeToNs(UInt64 hostTime) +{ + return static_cast( + static_cast(hostTime) * kHostTimebase.numer / kHostTimebase.denom); +} + /// In frames of the session's own rate, which may differ from the context's. int preferredIOBufferFramesFor(AudioContextLatencyHint latencyHint) { @@ -61,9 +77,10 @@ void reportStreamFailToContext( NativeAudioPlayer *IOSAudioPlayer::createNativePlayer() { - RenderAudioBlock renderAudioBlock = ^(AudioBufferList *outputData, int numFrames) { - deliverOutputBuffers(outputData, numFrames); - }; + RenderAudioBlock renderAudioBlock = + ^(AudioBufferList *outputData, int numFrames, const AudioTimeStamp *timestamp) { + deliverOutputBuffers(outputData, numFrames, timestamp); + }; NativeAudioPlayer *nativePlayer = [[NativeAudioPlayer alloc] initWithRenderAudio:renderAudioBlock @@ -89,7 +106,10 @@ void reportStreamFailToContext( pendingSaved_.zero(); } -void IOSAudioPlayer::deliverOutputBuffers(AudioBufferList *outputData, int numFrames) +void IOSAudioPlayer::deliverOutputBuffers( + AudioBufferList *outputData, + int numFrames, + const AudioTimeStamp *timestamp) { const CurrentRenderScope renderScope(currentRenders_); @@ -107,6 +127,14 @@ void reportStreamFailToContext( return; } + if (timestamp != nullptr && (timestamp->mFlags & kAudioTimeStampHostTimeValid) != 0) { + // Frames left over from the previous quantum play first; they were rendered but not delivered. + publishOutputAnchor( + {.contextFrame = renderedFrames_ - pendingSavedCount_, + .hostTimeNs = hostTimeToNs(timestamp->mHostTime), + .isValid = true}); + } + int outPos = 0; while (outPos < numFrames) { const int need = numFrames - outPos; @@ -233,6 +261,21 @@ void reportStreamFailToContext( return [sessionManager outputLatencySeconds] + [sessionManager ioBufferDurationSeconds]; } +std::optional IOSAudioPlayer::getOutputPresentation() +{ + const auto anchor = latestOutputAnchor(); + if (!anchor) { + return std::nullopt; + } + + AudioSessionManager *sessionManager = [AudioSessionManager sharedInstance]; + const auto outputLatencyNs = + static_cast(std::llround([sessionManager outputLatencySeconds] * 1e9)); + return OutputPresentation{ + .contextFrame = anchor->contextFrame, + .presentationTimeNs = anchor->hostTimeNs + outputLatencyNs}; +} + std::shared_ptr createPlatformAudioPlayer( const std::function &renderAudio, float sampleRate, diff --git a/packages/react-native-audio-api/ios/audioapi/ios/core/NativeAudioPlayer.h b/packages/react-native-audio-api/ios/audioapi/ios/core/NativeAudioPlayer.h index 8b19ddda9..0842d2f34 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/core/NativeAudioPlayer.h +++ b/packages/react-native-audio-api/ios/audioapi/ios/core/NativeAudioPlayer.h @@ -3,7 +3,11 @@ #import #import -typedef void (^RenderAudioBlock)(AudioBufferList *outputBuffer, int numFrames); +/// `timestamp` is CoreAudio's for the buffer's first frame; may be NULL. +typedef void (^RenderAudioBlock)( + AudioBufferList *outputBuffer, + int numFrames, + const AudioTimeStamp *timestamp); @interface NativeAudioPlayer : NSObject diff --git a/packages/react-native-audio-api/ios/audioapi/ios/core/NativeAudioPlayer.m b/packages/react-native-audio-api/ios/audioapi/ios/core/NativeAudioPlayer.m index 16130a195..f3774c222 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/core/NativeAudioPlayer.m +++ b/packages/react-native-audio-api/ios/audioapi/ios/core/NativeAudioPlayer.m @@ -67,7 +67,7 @@ - (instancetype)initWithRenderAudio:(RenderAudioBlock)renderAudio return kAudioServicesBadPropertySizeError; } - weakSelf.renderAudio(outputData, frameCount); + weakSelf.renderAudio(outputData, frameCount, timestamp); return kAudioServicesNoError; }; diff --git a/packages/react-native-audio-api/src/api.web.ts b/packages/react-native-audio-api/src/api.web.ts index 0192a03cb..ba59c967d 100644 --- a/packages/react-native-audio-api/src/api.web.ts +++ b/packages/react-native-audio-api/src/api.web.ts @@ -54,6 +54,7 @@ export type { ChannelCountMode, ChannelInterpretation, ContextState, + AudioTimestamp, } from './types'; export type { diff --git a/packages/react-native-audio-api/src/core/AudioContext.ts b/packages/react-native-audio-api/src/core/AudioContext.ts index 694bfa1ee..9a1e3055d 100644 --- a/packages/react-native-audio-api/src/core/AudioContext.ts +++ b/packages/react-native-audio-api/src/core/AudioContext.ts @@ -4,7 +4,7 @@ import { AudioTagHandle } from '../Audio/types'; import { AudioEventEmitter } from '../events'; import { IAudioContext } from '../jsi-interfaces'; import AudioManager from '../system'; -import { AudioContextOptions, ContextState } from '../types'; +import { AudioContextOptions, AudioTimestamp, ContextState } from '../types'; import BaseAudioContext from './BaseAudioContext'; import MediaElementAudioSourceNode from './MediaElementAudioSourceNode'; @@ -44,6 +44,15 @@ export default class AudioContext extends BaseAudioContext { return (this.context as IAudioContext).outputLatency; } + /** + * The context time of the sample leaving the output device now, and now on + * the `performance.now()` clock. Both are `0` until the context has rendered + * its first block. + */ + getOutputTimestamp(): AudioTimestamp { + return (this.context as IAudioContext).getOutputTimestamp(); + } + async close(): Promise { if (this._state === 'closed') { throw new InvalidStateError('Cannot close a closed audio context.'); diff --git a/packages/react-native-audio-api/src/jsi-interfaces.ts b/packages/react-native-audio-api/src/jsi-interfaces.ts index c79e8c0f7..314869221 100644 --- a/packages/react-native-audio-api/src/jsi-interfaces.ts +++ b/packages/react-native-audio-api/src/jsi-interfaces.ts @@ -1,6 +1,7 @@ import { AudioEventCallback, AudioEventName } from './events/types'; import type { AnalyserOptions, + AudioTimestamp, AudioBufferSourceOptions, AudioRecorderCallbackOptions, AudioRecorderFileOptions, @@ -93,6 +94,7 @@ export interface IBaseAudioContext { export interface IAudioContext extends IBaseAudioContext { readonly baseLatency: number; readonly outputLatency: number; + getOutputTimestamp(): AudioTimestamp; createMediaElementSource: ( mediaElement: IAudioFileSourceNode ) => IMediaElementAudioSourceNode; diff --git a/packages/react-native-audio-api/src/mock/index.ts b/packages/react-native-audio-api/src/mock/index.ts index 2a51408b9..1604ccce4 100644 --- a/packages/react-native-audio-api/src/mock/index.ts +++ b/packages/react-native-audio-api/src/mock/index.ts @@ -32,7 +32,7 @@ import type { DistanceModelType, WaveShaperOptions, } from '../types'; -import { FileDirectory, FileFormat } from '../types'; +import { AudioTimestamp, FileDirectory, FileFormat } from '../types'; import { toFloat32Array } from '../utils'; /* eslint-disable no-useless-constructor */ @@ -950,6 +950,10 @@ class AudioContextMock extends BaseAudioContextMock { return 0.01; } + getOutputTimestamp(): AudioTimestamp { + return { contextTime: 0, performanceTime: 0 }; + } + close(): Promise { this._state = 'closed'; return Promise.resolve(); diff --git a/packages/react-native-audio-api/src/types.ts b/packages/react-native-audio-api/src/types.ts index 32c98d763..af2eb4183 100644 --- a/packages/react-native-audio-api/src/types.ts +++ b/packages/react-native-audio-api/src/types.ts @@ -63,6 +63,17 @@ export interface AudioContextOptions { export type AndroidOutputProfile = 'media' | 'voiceCommunication'; +/** + * One moment on two clocks: the context's `currentTime` axis and the + * `performance.now()` axis. Returned by `AudioContext.getOutputTimestamp()`. + */ +export interface AudioTimestamp { + /** Seconds, on the context's `currentTime` axis. */ + contextTime: number; + /** Milliseconds, on the `performance.now()` axis. */ + performanceTime: number; +} + export interface OfflineAudioContextOptions { numberOfChannels: number; length: number; diff --git a/packages/react-native-audio-api/src/web-core/AudioContext.web.ts b/packages/react-native-audio-api/src/web-core/AudioContext.web.ts index 370f939a5..6bb33c804 100644 --- a/packages/react-native-audio-api/src/web-core/AudioContext.web.ts +++ b/packages/react-native-audio-api/src/web-core/AudioContext.web.ts @@ -1,6 +1,11 @@ import { InvalidAccessError, NotSupportedError } from '../errors'; import { assertSupportedSampleRate } from '../utils/validation'; -import { AudioContextOptions, ContextState, DecodeDataInput } from '../types'; +import { + AudioContextOptions, + AudioTimestamp, + ContextState, + DecodeDataInput, +} from '../types'; import AnalyserNode from './AnalyserNode.web'; import AudioBuffer from './AudioBuffer.web'; import AudioBufferSourceNode from './AudioBufferSourceNode.web'; @@ -64,6 +69,12 @@ export default class AudioContext implements BaseAudioContext { return this.context.outputLatency; } + getOutputTimestamp(): AudioTimestamp { + const { contextTime = 0, performanceTime = 0 } = + this.context.getOutputTimestamp(); + return { contextTime, performanceTime }; + } + createOscillator(): OscillatorNode { return new OscillatorNode(this); } diff --git a/packages/react-native-audio-api/tests/audio-context-options.test.ts b/packages/react-native-audio-api/tests/audio-context-options.test.ts index 8a314c109..78caf57fd 100644 --- a/packages/react-native-audio-api/tests/audio-context-options.test.ts +++ b/packages/react-native-audio-api/tests/audio-context-options.test.ts @@ -45,4 +45,16 @@ describe('AudioContext options', () => { 'voiceCommunication' ); }); + + it('returns the native output timestamp unchanged', () => { + const timestamp = { contextTime: 1.25, performanceTime: 98765.4 }; + const nativeContext = { + getOutputTimestamp: jest.fn(() => timestamp), + } as unknown as IAudioContext; + globalThis.createAudioContext = jest.fn(() => nativeContext); + const AudioContext = require('../src/core/AudioContext') + .default as typeof import('../src/core/AudioContext').default; + + expect(new AudioContext().getOutputTimestamp()).toEqual(timestamp); + }); }); diff --git a/packages/react-native-audio-api/tests/mock.test.ts b/packages/react-native-audio-api/tests/mock.test.ts index 25610122e..7d7e6c126 100644 --- a/packages/react-native-audio-api/tests/mock.test.ts +++ b/packages/react-native-audio-api/tests/mock.test.ts @@ -11,6 +11,10 @@ describe('React Native Audio API Mocks', () => { expect(context.state).toBe('running'); expect(context.baseLatency).toBe(0.005); expect(context.outputLatency).toBe(0.01); + expect(context.getOutputTimestamp()).toEqual({ + contextTime: 0, + performanceTime: 0, + }); expect(context.destination).toBeInstanceOf(MockAPI.AudioDestinationNode); }); From 9b0f86ac1c705c5debe540e7a55b50844414b4bd Mon Sep 17 00:00:00 2001 From: michal Date: Thu, 22 Oct 2026 18:41:35 +0200 Subject: [PATCH 2/3] feat: all players return output timestamp --- .../common/cpp/audioapi/core/AudioPlayer.h | 6 +-- .../wpt_tests/src/NodeAudioPlayer.cpp | 37 ++++++++++++++++++- .../wpt_tests/src/NodeAudioPlayer.h | 5 +++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/AudioPlayer.h b/packages/react-native-audio-api/common/cpp/audioapi/core/AudioPlayer.h index 0ebf923ed..74767e354 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/AudioPlayer.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/AudioPlayer.h @@ -55,11 +55,9 @@ class AudioPlayer { [[nodiscard]] virtual bool isRunning() const = 0; /// @brief Which context frame leaves the output device when, or `std::nullopt` before the - /// first rendered callback or on a platform that cannot tell (the default). + /// first rendered callback or on a platform that cannot tell. /// @note JS Thread only: it is the single reader of the anchors published from the audio thread. - [[nodiscard]] virtual std::optional getOutputPresentation() { - return std::nullopt; - } + [[nodiscard]] virtual std::optional getOutputPresentation() = 0; protected: /// One render callback's first output sample, tied to the platform's own timeline. diff --git a/packages/react-native-audio-api/wpt_tests/src/NodeAudioPlayer.cpp b/packages/react-native-audio-api/wpt_tests/src/NodeAudioPlayer.cpp index 051c588c0..10f43e32c 100644 --- a/packages/react-native-audio-api/wpt_tests/src/NodeAudioPlayer.cpp +++ b/packages/react-native-audio-api/wpt_tests/src/NodeAudioPlayer.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace audioapi { @@ -111,20 +112,54 @@ double NodeAudioPlayer::getBaseLatency() const { return 0.0; } +std::optional NodeAudioPlayer::getOutputPresentation() { + const auto anchor = latestOutputAnchor(); + if (!anchor) { + return std::nullopt; + } + + return OutputPresentation{ + .contextFrame = anchor->contextFrame, .presentationTimeNs = anchor->hostTimeNs}; +} + void NodeAudioPlayer::run() { const auto sampleRate = std::max(sampleRate_, 1.0f); const auto quantumDuration = std::chrono::microseconds(static_cast( (static_cast(RENDER_QUANTUM_SIZE) * 1'000'000.0) / sampleRate)); + // The virtual device's own clock: when the quantum about to be rendered leaves it. Paced by + // absolute deadlines so a render's cost does not accumulate into the reported timestamps. + auto nextQuantumAt = std::chrono::steady_clock::now(); + while (!shouldStop_.load(std::memory_order_acquire)) { if (isPaused_.load(std::memory_order_acquire)) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); + // Resume plays on from now, instead of racing to catch up on the silent time. + nextQuantumAt = std::chrono::steady_clock::now(); continue; } + // Before the render, so `renderedFrames_` is still the quantum's first frame. + publishOutputAnchor( + {.contextFrame = renderedFrames_, + .hostTimeNs = std::chrono::duration_cast( + nextQuantumAt.time_since_epoch()) + .count(), + .isValid = true}); + renderBuffer_->zero(); renderNormalizedQuantum(RENDER_QUANTUM_SIZE); - std::this_thread::sleep_for(quantumDuration); + + nextQuantumAt += quantumDuration; + const auto now = std::chrono::steady_clock::now(); + if (nextQuantumAt <= now) { + // The render did not keep up; a real device would have underrun here. Drop the backlog + // rather than busy-rendering to catch up. + nextQuantumAt = now; + continue; + } + + std::this_thread::sleep_until(nextQuantumAt); } } diff --git a/packages/react-native-audio-api/wpt_tests/src/NodeAudioPlayer.h b/packages/react-native-audio-api/wpt_tests/src/NodeAudioPlayer.h index b6ed6dfd3..e4865f54f 100644 --- a/packages/react-native-audio-api/wpt_tests/src/NodeAudioPlayer.h +++ b/packages/react-native-audio-api/wpt_tests/src/NodeAudioPlayer.h @@ -3,6 +3,7 @@ #include #include +#include #include namespace audioapi { @@ -24,6 +25,10 @@ class NodeAudioPlayer final : public AudioPlayer { [[nodiscard]] double getOutputLatency() const override; [[nodiscard]] double getBaseLatency() const override; + /// The virtual device plays a quantum at the deadline the render loop paced it to, with no + /// output latency between the two. + [[nodiscard]] std::optional getOutputPresentation() override; + private: void run(); /// Signal the worker to exit and join it. Safe to call repeatedly. From c5cd693f09c8fa6ee41343f5e21e56f06e0c490d Mon Sep 17 00:00:00 2001 From: michal Date: Fri, 23 Oct 2026 10:58:21 +0200 Subject: [PATCH 3/3] feat: slight improvements --- packages/audiodocs/docs/core/audio-context.mdx | 2 -- .../android/core/AndroidAudioPlayer.cpp | 12 +++++++----- .../audioapi/android/core/AndroidAudioPlayer.h | 2 ++ .../cpp/audioapi/core/utils/OutputTimestamp.h | 11 ++++++++--- .../ios/audioapi/ios/core/IOSAudioPlayer.mm | 18 ++++++++++++++---- 5 files changed, 31 insertions(+), 14 deletions(-) diff --git a/packages/audiodocs/docs/core/audio-context.mdx b/packages/audiodocs/docs/core/audio-context.mdx index 1779fd64c..45f6eee86 100644 --- a/packages/audiodocs/docs/core/audio-context.mdx +++ b/packages/audiodocs/docs/core/audio-context.mdx @@ -70,8 +70,6 @@ const heardAt = (perfMs: number) => oscillator.start(heardAt(performance.now() + 500)); // heard ~500 ms from now ``` -On iOS the presentation time is CoreAudio's host time for each output buffer plus `AVAudioSession.outputLatency`. On Android it comes from Oboe's `getTimestamp(CLOCK_MONOTONIC)`, falling back to callback time plus `outputLatency` when the stream does not report timestamps (OpenSL ES). - #### Returns `AudioTimestamp`. | Name | Type | Description | diff --git a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.cpp b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.cpp index aa6b4bc44..1067c2021 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.cpp +++ b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.cpp @@ -94,6 +94,7 @@ bool AndroidAudioPlayer::rebuildStreamLocked() { } bool AndroidAudioPlayer::startStreamLocked() { + streamStartedAtNs_ = steadyClockNowNs(); const bool started = mStream_ != nullptr && mStream_->requestStart() == oboe::Result::OK; isRunning_.store(started, std::memory_order_release); return started; @@ -185,9 +186,7 @@ AndroidAudioPlayer::onAudioReady(AudioStream *oboeStream, void *audioData, int32 // Oboe counts this buffer into getFramesWritten() only after the callback returns. publishOutputAnchor( {.contextFrame = renderedFrames_, - .hostTimeNs = std::chrono::duration_cast( - std::chrono::steady_clock::now().time_since_epoch()) - .count(), + .hostTimeNs = steadyClockNowNs(), .streamFrame = oboeStream->getFramesWritten(), .isValid = true}); } @@ -322,9 +321,12 @@ std::optional AndroidAudioPlayer::getOutputPresentation() { { std::scoped_lock lock(streamMutex_); - if (isRunningLocked()) { + // An anchor from before the last start may belong to a stream a rebuild replaced, whose + // positions the current stream does not continue. A device timestamp from before it is + // AAudio's stale pre-pause report, which would interpolate across the pause. + if (isRunningLocked() && anchor->hostTimeNs >= streamStartedAtNs_) { const auto presented = mStream_->getTimestamp(CLOCK_MONOTONIC); - if (presented) { + if (presented && presented.value().timestamp >= streamStartedAtNs_) { return OutputPresentation{ .contextFrame = anchor->contextFrame + (presented.value().position - anchor->streamFrame), diff --git a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.h b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.h index 0a1e683b1..2c4ac69ad 100644 --- a/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.h +++ b/packages/react-native-audio-api/android/src/main/cpp/audioapi/android/core/AndroidAudioPlayer.h @@ -66,6 +66,8 @@ class AndroidAudioPlayer : public AudioPlayer, /// Updated on the audio thread from each Oboe callback `numFrames`. std::atomic lastCallbackFrameCount_{0}; AndroidOutputProfile outputProfile_; + /// `steady_clock` time of the last `startStreamLocked`; guarded by `streamMutex_`. + std::int64_t streamStartedAtNs_ = 0; bool openAudioStreamLocked(); bool rebuildStreamLocked(); diff --git a/packages/react-native-audio-api/common/cpp/audioapi/core/utils/OutputTimestamp.h b/packages/react-native-audio-api/common/cpp/audioapi/core/utils/OutputTimestamp.h index 1cd25641b..ba36c355f 100644 --- a/packages/react-native-audio-api/common/cpp/audioapi/core/utils/OutputTimestamp.h +++ b/packages/react-native-audio-api/common/cpp/audioapi/core/utils/OutputTimestamp.h @@ -8,6 +8,13 @@ namespace audioapi { +/// Now on the `performance.now()` axis, which every presentation time is expressed in. +[[nodiscard]] inline std::int64_t steadyClockNowNs() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} + /// A context sample frame and the `steady_clock` moment it leaves the output device. struct OutputPresentation { /// Signed: a platform may report a device position from before the context's first frame. @@ -27,9 +34,7 @@ struct OutputPresentation { constexpr double NANOSECONDS_PER_SECOND = 1e9; constexpr double NANOSECONDS_PER_MILLISECOND = 1e6; - const auto nowNs = std::chrono::duration_cast( - std::chrono::steady_clock::now().time_since_epoch()) - .count(); + const auto nowNs = steadyClockNowNs(); auto frame = static_cast(presentation.contextFrame); auto timeNs = static_cast(presentation.presentationTimeNs); diff --git a/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.mm b/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.mm index 17c62156d..7c0c8fe13 100644 --- a/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.mm +++ b/packages/react-native-audio-api/ios/audioapi/ios/core/IOSAudioPlayer.mm @@ -30,11 +30,21 @@ return timebase; }(); -/// Mach host time is the clock `steady_clock` reads on Apple platforms; only the unit differs. -std::int64_t hostTimeToNs(UInt64 hostTime) +std::int64_t machTicksToNs(std::int64_t ticks) { return static_cast( - static_cast(hostTime) * kHostTimebase.numer / kHostTimebase.denom); + static_cast<__int128>(ticks) * kHostTimebase.numer / kHostTimebase.denom); +} + +/// Mach host time stops while the device sleeps, but libc++ `steady_clock` (`CLOCK_MONOTONIC_RAW`) +/// keeps counting, so the two drift apart by the total sleep. Translating only the distance from +/// now keeps the drift out. +std::int64_t hostTimeToSteadyClockNs(UInt64 hostTime) +{ + const auto hostNow = mach_absolute_time(); + const auto steadyNowNs = steadyClockNowNs(); + return steadyNowNs - + machTicksToNs(static_cast(hostNow) - static_cast(hostTime)); } /// In frames of the session's own rate, which may differ from the context's. @@ -131,7 +141,7 @@ void reportStreamFailToContext( // Frames left over from the previous quantum play first; they were rendered but not delivered. publishOutputAnchor( {.contextFrame = renderedFrames_ - pendingSavedCount_, - .hostTimeNs = hostTimeToNs(timestamp->mHostTime), + .hostTimeNs = hostTimeToSteadyClockNs(timestamp->mHostTime), .isValid = true}); }