Add idle frame support to Performance timeline (#57952)

Summary:
Pull Request resolved: #57952

Implement Idle frame spans in React Native DevTools, by emitting synthetic `NeedsBeginFrameChanged` + `BeginFrame` events.

**Definition**

An idle frame is emitted whenever the gap between two consecutive frames exceeds one vsync interval, derived from the display's refresh rate (`CADisplayLink.duration` on iOS, `Display.refreshRate` on Android, falling back to 60 Hz).

```
   frame N                  gap > 1 vsync                 frame N+1
+------------+  +--------------------------------+  +------------+
| BeginFrame |  | NeedsBeginFrameChanged         |  | BeginFrame |
| DrawFrame  |  | BeginFrame     (no DrawFrame)  |  | DrawFrame  |
+------------+  +--------------------------------+  +------------+
                    rendered as an "Idle frame"
```

**Implementation notes**

- Drop frames that begin before the recording start — Android `FrameMetrics` can deliver frames from app startup, and the first iOS `CADisplayLink` callback reports the previous vsync.
- Sort frames by begin timestamp before serializing, since async screenshot encoding can deliver them out of order and break gap detection.
- iOS: skip the frame event when the screenshot is unchanged, letting the gap render as an idle frame.

Changelog: [Internal]

Reviewed By: hoxyq

Differential Revision: D97502569

fbshipit-source-id: 7cb19e0a462ad878f0b44b66f6b5de3f76426ffe

Author: huntie

packages/react-native/React/DevSupport/RCTFrameTimingsObserver.mm

@@ -35,6 +35,7 @@
   jsinspector_modern::tracing::ThreadId threadId;
   HighResTimeStamp beginTimestamp;
   HighResTimeStamp endTimestamp;
+  HighResDuration vsyncInterval;
 };
 
 } // namespace
@@ -84,9 +85,11 @@ - (void)start
     _lastFrameData.reset();
   }
 
-  // Emit initial frame event
+  // Emit an initial render frame. Keep it zero-duration: the frontend derives
+  // each frame's rendered length from the gap to the next frame, so a synthetic
+  // end time here can overlap the first real frame and corrupt the frames track.
   auto now = HighResTimeStamp::now();
-  [self _emitFrameTimingWithBeginTimestamp:now endTimestamp:now];
+  [self _emitFrameTimingWithBeginTimestamp:now endTimestamp:now vsyncInterval:HighResDuration::zero()];
 
   _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(_displayLinkTick:)];
   [_displayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSRunLoopCommonModes];
@@ -115,11 +118,14 @@ - (void)_displayLinkTick:(CADisplayLink *)sender
       std::chrono::steady_clock::time_point(std::chrono::nanoseconds(beginNanos)));
   auto endTimestamp = HighResTimeStamp::fromChronoSteadyClockTimePoint(
       std::chrono::steady_clock::time_point(std::chrono::nanoseconds(endNanos)));
+  auto vsyncInterval = HighResDuration::fromNanoseconds(static_cast<int64_t>(sender.duration * 1e9));
 
-  [self _emitFrameTimingWithBeginTimestamp:beginTimestamp endTimestamp:endTimestamp];
+  [self _emitFrameTimingWithBeginTimestamp:beginTimestamp endTimestamp:endTimestamp vsyncInterval:vsyncInterval];
 }
 
-- (void)_emitFrameTimingWithBeginTimestamp:(HighResTimeStamp)beginTimestamp endTimestamp:(HighResTimeStamp)endTimestamp
+- (void)_emitFrameTimingWithBeginTimestamp:(HighResTimeStamp)beginTimestamp
+                              endTimestamp:(HighResTimeStamp)endTimestamp
+                             vsyncInterval:(HighResDuration)vsyncInterval
 {
   uint64_t frameId = _frameCounter++;
   auto threadId = static_cast<jsinspector_modern::tracing::ThreadId>(pthread_mach_thread_np(pthread_self()));
@@ -130,22 +136,21 @@ - (void)_emitFrameTimingWithBeginTimestamp:(HighResTimeStamp)beginTimestamp endT
                             threadId:threadId
                       beginTimestamp:beginTimestamp
                         endTimestamp:endTimestamp
-                          screenshot:std::nullopt];
+                          screenshot:std::nullopt
+                       vsyncInterval:vsyncInterval];
     return;
   }
 
   UIImage *image = [self _captureScreenshot];
   if (image == nil) {
-    // Failed to capture (e.g. no window, duplicate hash) - emit without screenshot
-    [self _emitFrameEventWithFrameId:frameId
-                            threadId:threadId
-                      beginTimestamp:beginTimestamp
-                        endTimestamp:endTimestamp
-                          screenshot:std::nullopt];
+    // Screenshot unchanged (duplicate hash) or capture failed — don't emit
+    // a frame event. The serializer will fill the resulting gap with an idle
+    // frame, matching Chrome's native behavior where idle = vsync with no
+    // new rendering.
     return;
   }
 
-  FrameData frameData{image, frameId, threadId, beginTimestamp, endTimestamp};
+  FrameData frameData{image, frameId, threadId, beginTimestamp, endTimestamp, vsyncInterval};
 
   bool expected = false;
   if (_encodingInProgress.compare_exchange_strong(expected, true)) {
@@ -165,7 +170,8 @@ - (void)_emitFrameTimingWithBeginTimestamp:(HighResTimeStamp)beginTimestamp endT
                               threadId:oldFrame->threadId
                         beginTimestamp:oldFrame->beginTimestamp
                           endTimestamp:oldFrame->endTimestamp
-                            screenshot:std::nullopt];
+                            screenshot:std::nullopt
+                         vsyncInterval:oldFrame->vsyncInterval];
     }
   }
 }
@@ -175,13 +181,14 @@ - (void)_emitFrameEventWithFrameId:(uint64_t)frameId
                     beginTimestamp:(HighResTimeStamp)beginTimestamp
                       endTimestamp:(HighResTimeStamp)endTimestamp
                         screenshot:(std::optional<std::vector<uint8_t>>)screenshot
+                     vsyncInterval:(HighResDuration)vsyncInterval
 {
   dispatch_async(dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0), ^{
     if (!self->_running.load(std::memory_order_relaxed)) {
       return;
     }
     jsinspector_modern::tracing::FrameTimingSequence sequence{
-        frameId, threadId, beginTimestamp, endTimestamp, std::move(screenshot)};
+        frameId, threadId, beginTimestamp, endTimestamp, std::move(screenshot), vsyncInterval};
     self->_callback(std::move(sequence));
   });
 }
@@ -198,7 +205,8 @@ - (void)_encodeFrame:(FrameData)frameData
                             threadId:frameData.threadId
                       beginTimestamp:frameData.beginTimestamp
                         endTimestamp:frameData.endTimestamp
-                          screenshot:std::move(screenshot)];
+                          screenshot:std::move(screenshot)
+                       vsyncInterval:frameData.vsyncInterval];
 
     // Clear encoding flag early, allowing new frames to start fresh encoding
     // sessions
@@ -221,7 +229,8 @@ - (void)_encodeFrame:(FrameData)frameData
                               threadId:tailFrame->threadId
                         beginTimestamp:tailFrame->beginTimestamp
                           endTimestamp:tailFrame->endTimestamp
-                            screenshot:std::move(tailScreenshot)];
+                            screenshot:std::move(tailScreenshot)
+                         vsyncInterval:tailFrame->vsyncInterval];
     }
   });
 }

packages/react-native/ReactAndroid/src/main/java/com/facebook/react/devsupport/inspector/FrameTimingSequence.kt

@@ -13,4 +13,5 @@ internal data class FrameTimingSequence(
     val beginTimestamp: Long,
     val endTimestamp: Long,
     val screenshot: ByteArray? = null,
+    val vsyncIntervalNanos: Long = 0,
 )

packages/react-native/ReactAndroid/src/main/java/com/facebook/react/devsupport/inspector/FrameTimingsObserver.kt

@@ -54,6 +54,7 @@ internal class FrameTimingsObserver(
       val threadId: Int,
       val beginTimestamp: Long,
       val endTimestamp: Long,
+      val vsyncIntervalNanos: Long,
   )
 
   fun start() {
@@ -66,9 +67,11 @@ internal class FrameTimingsObserver(
     lastFrameBuffer.set(null)
     isTracing = true
 
-    // Emit initial frame event
+    // Emit an initial render frame. Keep it zero-duration: the frontend derives
+    // each frame's rendered length from the gap to the next frame, so a synthetic
+    // end time here can overlap the first real frame and corrupt the frames track.
     val timestamp = System.nanoTime()
-    emitFrameTiming(timestamp, timestamp)
+    emitFrameTiming(timestamp, timestamp, vsyncIntervalNanos = 0)
 
     currentWindow?.addOnFrameMetricsAvailableListener(frameMetricsListener, mainHandler)
   }
@@ -97,27 +100,35 @@ internal class FrameTimingsObserver(
     }
   }
 
-  private val frameMetricsListener = Window.OnFrameMetricsAvailableListener { _, frameMetrics, _ ->
-    // Guard against calls after stop()
-    if (!isTracing) {
-      return@OnFrameMetricsAvailableListener
-    }
-    val beginTimestamp = frameMetrics.getMetric(FrameMetrics.VSYNC_TIMESTAMP)
-    val endTimestamp = beginTimestamp + frameMetrics.getMetric(FrameMetrics.TOTAL_DURATION)
-    emitFrameTiming(beginTimestamp, endTimestamp)
-  }
+  private val frameMetricsListener =
+      Window.OnFrameMetricsAvailableListener { window, frameMetrics, _ ->
+        // Guard against calls after stop()
+        if (!isTracing) {
+          return@OnFrameMetricsAvailableListener
+        }
+        val beginTimestamp = frameMetrics.getMetric(FrameMetrics.VSYNC_TIMESTAMP)
+        val endTimestamp = beginTimestamp + frameMetrics.getMetric(FrameMetrics.TOTAL_DURATION)
+        val refreshRate = window.decorView.display?.refreshRate ?: 60f
+        val vsyncIntervalNanos = (1_000_000_000L / refreshRate).toLong()
+        emitFrameTiming(beginTimestamp, endTimestamp, vsyncIntervalNanos)
+      }
 
-  private fun emitFrameTiming(beginTimestamp: Long, endTimestamp: Long) {
+  private fun emitFrameTiming(
+      beginTimestamp: Long,
+      endTimestamp: Long,
+      vsyncIntervalNanos: Long = 0,
+  ) {
     val frameId = frameCounter++
     val threadId = Process.myTid()
 
     if (!screenshotsEnabled) {
       // Screenshots disabled - emit without screenshot
-      emitFrameEvent(frameId, threadId, beginTimestamp, endTimestamp, null)
+      emitFrameEvent(frameId, threadId, beginTimestamp, endTimestamp, null, vsyncIntervalNanos)
       return
     }
 
-    captureScreenshot(frameId, threadId, beginTimestamp, endTimestamp) { frameData ->
+    captureScreenshot(frameId, threadId, beginTimestamp, endTimestamp, vsyncIntervalNanos) {
+        frameData ->
       if (frameData != null) {
         if (encodingInProgress.compareAndSet(false, true)) {
           // Not encoding - encode this frame immediately
@@ -133,13 +144,14 @@ internal class FrameTimingsObserver(
                 oldFrameData.beginTimestamp,
                 oldFrameData.endTimestamp,
                 null,
+                oldFrameData.vsyncIntervalNanos,
             )
             oldFrameData.bitmap.recycle()
           }
         }
       } else {
         // Failed to capture (e.g. timeout) - emit without screenshot
-        emitFrameEvent(frameId, threadId, beginTimestamp, endTimestamp, null)
+        emitFrameEvent(frameId, threadId, beginTimestamp, endTimestamp, null, vsyncIntervalNanos)
       }
     }
   }
@@ -150,10 +162,18 @@ internal class FrameTimingsObserver(
       beginTimestamp: Long,
       endTimestamp: Long,
       screenshot: ByteArray?,
+      vsyncIntervalNanos: Long = 0,
   ) {
     CoroutineScope(Dispatchers.Default).launch {
       onFrameTimingSequence(
-          FrameTimingSequence(frameId, threadId, beginTimestamp, endTimestamp, screenshot),
+          FrameTimingSequence(
+              frameId,
+              threadId,
+              beginTimestamp,
+              endTimestamp,
+              screenshot,
+              vsyncIntervalNanos,
+          ),
       )
     }
   }
@@ -168,6 +188,7 @@ internal class FrameTimingsObserver(
             frameData.beginTimestamp,
             frameData.endTimestamp,
             screenshot,
+            frameData.vsyncIntervalNanos,
         )
       } finally {
         frameData.bitmap.recycle()
@@ -187,6 +208,7 @@ internal class FrameTimingsObserver(
               tailFrame.beginTimestamp,
               tailFrame.endTimestamp,
               screenshot,
+              tailFrame.vsyncIntervalNanos,
           )
         } finally {
           tailFrame.bitmap.recycle()
@@ -201,6 +223,7 @@ internal class FrameTimingsObserver(
       threadId: Int,
       beginTimestamp: Long,
       endTimestamp: Long,
+      vsyncIntervalNanos: Long,
       callback: (FrameData?) -> Unit,
   ) {
     if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
@@ -226,7 +249,16 @@ internal class FrameTimingsObserver(
         bitmap,
         { copyResult ->
           if (copyResult == PixelCopy.SUCCESS) {
-            callback(FrameData(bitmap, frameId, threadId, beginTimestamp, endTimestamp))
+            callback(
+                FrameData(
+                    bitmap,
+                    frameId,
+                    threadId,
+                    beginTimestamp,
+                    endTimestamp,
+                    vsyncIntervalNanos,
+                ),
+            )
           } else {
             bitmap.recycle()
             callback(null)

packages/react-native/ReactAndroid/src/main/jni/react/runtime/jni/JReactHostInspectorTarget.cpp

@@ -251,6 +251,7 @@ void JReactHostInspectorTarget::recordFrameTimings(
       frameTimingSequence->getBeginTimestamp(),
       frameTimingSequence->getEndTimestamp(),
       frameTimingSequence->getScreenshot(),
+      frameTimingSequence->getVsyncInterval(),
   });
 }
 

packages/react-native/ReactAndroid/src/main/jni/react/runtime/jni/JReactHostInspectorTarget.h

@@ -113,6 +113,12 @@ struct JFrameTimingSequence : public jni::JavaClass<JFrameTimingSequence> {
     }
     return std::nullopt;
   }
+
+  HighResDuration getVsyncInterval() const
+  {
+    auto field = javaClassStatic()->getField<jlong>("vsyncIntervalNanos");
+    return HighResDuration::fromNanoseconds(static_cast<int64_t>(getFieldValue(field)));
+  }
 };
 
 struct JReactHostImpl : public jni::JavaClass<JReactHostImpl> {

packages/react-native/ReactCommon/jsinspector-modern/tracing/FrameTimingSequence.h

@@ -30,12 +30,14 @@ struct FrameTimingSequence {
       ThreadId threadId,
       HighResTimeStamp beginTimestamp,
       HighResTimeStamp endTimestamp,
-      std::optional<std::vector<uint8_t>> screenshot = std::nullopt)
+      std::optional<std::vector<uint8_t>> screenshot = std::nullopt,
+      HighResDuration vsyncInterval = HighResDuration::zero())
       : id(id),
         threadId(threadId),
         beginTimestamp(beginTimestamp),
         endTimestamp(endTimestamp),
-        screenshot(std::move(screenshot))
+        screenshot(std::move(screenshot)),
+        vsyncInterval(vsyncInterval)
   {
   }
 
@@ -56,6 +58,12 @@ struct FrameTimingSequence {
    * Optional screenshot data captured during the frame.
    */
   std::optional<std::vector<uint8_t>> screenshot;
+
+  /**
+   * Duration of one vsync interval from the device's display refresh rate.
+   * Zero when unknown (e.g. the initial synthetic frame).
+   */
+  HighResDuration vsyncInterval = HighResDuration::zero();
 };
 
 } // namespace facebook::react::jsinspector_modern::tracing