feat(nextjs): Mark pageloads that were served from a cached shell

CHANGELOG.md

@@ -8,6 +8,10 @@
 
   Pageloads of apps with `cacheComponents` enabled are connected to their server request again. Pages that Next.js prerenders no longer carry trace information, so a cached page can no longer connect visitors to a request that was not theirs. This also applies to ISR pages. The pageload waits for the trace information when it arrives with the part of the page that streams in after a cached shell.
 
+- **feat(nextjs): Mark pageloads that were served from a cached shell**
+
+  A pageload whose document was served from a cache, for example an ISR page or a `cacheComponents` page generated on first visit, now carries the `document.cached` attribute and a `cache_origin` span link to the request that prerendered the page. That tells apart "nothing ran on the server for this visit" from missing server instrumentation. The link is never continued as a parent, so pageloads keep their own traces and sampling decisions.
+
 ## 11.7.0
 
 ### Important Changes

dev-packages/e2e-tests/test-applications/nextjs-16-cacheComponents/tests/cacheOrigin-pageloadConnection.spec.ts

@@ -7,7 +7,8 @@ const APP = 'nextjs-16-cacheComponents';
 // The server only writes `sentry-trace` meta tags for a render that happened for the current request,
 // never while it prerenders a shell. So a tag in the document can always be continued; the only
 // question is when it arrives. The resumed part of a prerendered shell arrives after the SDK started,
-// which the pageload instrumentation waits for.
+// which the pageload instrumentation waits for. A shell prerendered at runtime carries a `sentry-prerender`
+// marker instead, which marks the pageload as served from a cached shell and links the prerender.
 
 test('connects the pageload trace to the runtime server request', async ({ page }) => {
   // The shell is prerendered, but the page's dynamic part (`DynamicContent` awaits `headers()`)
@@ -30,7 +31,6 @@ test('connects the pageload trace to the runtime server request', async ({ page
 
   expect(serverTx.contexts?.trace?.trace_id).toBeTruthy();
   expect(pageloadTx.contexts?.trace?.trace_id).toBe(serverTx.contexts?.trace?.trace_id);
-  expect(pageloadTx.contexts?.trace?.links).toBeUndefined();
 });
 
 test('connects the pageload trace to a blocking render', async ({ page }) => {
@@ -58,10 +58,11 @@ test('does not share a trace between visitors of a shell generated at request ti
   test.skip(process.env.TEST_ENV !== 'production', 'Shells are only cached in production builds');
 
   // `[id]` only lists `test` in `generateStaticParams`, so the first visit prerenders and caches the
-  // shell for this id. The prerender writes no trace tag, so every visitor starts a fresh trace.
+  // shell for this id. The prerender writes no trace tag, only a `sentry-prerender` marker with its own
+  // trace, so every visitor starts a fresh trace that links the prerender and is marked as a cached shell.
   const id = `shell-${Date.now()}`;
 
-  const visit = async (): Promise<{ traceId: string | undefined; links: unknown }> => {
+  const visit = async (): Promise<{ traceId: string | undefined; links: unknown; documentCached: unknown }> => {
     const context = await browser.newContext();
     const page: Page = await context.newPage();
 
@@ -75,11 +76,16 @@ test('does not share a trace between visitors of a shell generated at request ti
     await page.goto(`/${id}`);
     await expect(page.locator('#result')).toHaveText(`Hello, ${id}!`);
     expect(await page.locator('meta[name="sentry-trace"]').count()).toBe(0);
+    expect(await page.locator('meta[name="sentry-prerender"]').count()).toBe(1);
 
     const pageloadTx = await pageloadTxPromise;
     await context.close();
 
-    return { traceId: pageloadTx.contexts?.trace?.trace_id, links: pageloadTx.contexts?.trace?.links };
+    return {
+      traceId: pageloadTx.contexts?.trace?.trace_id,
+      links: pageloadTx.contexts?.trace?.links,
+      documentCached: pageloadTx.contexts?.trace?.data?.['document.cached'],
+    };
   };
 
   const first = await visit();
@@ -88,8 +94,17 @@ test('does not share a trace between visitors of a shell generated at request ti
   expect(first.traceId).toBeTruthy();
   expect(second.traceId).toBeTruthy();
   expect(second.traceId).not.toBe(first.traceId);
-  expect(first.links).toBeUndefined();
-  expect(second.links).toBeUndefined();
+  expect(first.documentCached).toBe(true);
+  expect(second.documentCached).toBe(true);
+
+  // Both link the one request that prerendered the shell, which is neither visitor's own trace.
+  const linkOf = (links: unknown): { trace_id: string; attributes?: Record<string, unknown> } => {
+    const link = (links as Array<{ trace_id: string; attributes?: Record<string, unknown> }>)[0];
+    expect(link?.attributes?.['sentry.link.type']).toBe('cache_origin');
+    return link!;
+  };
+  expect(linkOf(first.links).trace_id).toBe(linkOf(second.links).trace_id);
+  expect(linkOf(second.links).trace_id).not.toBe(second.traceId);
 });
 
 test('starts a fresh trace on a fully static page', async ({ page }) => {
@@ -105,8 +120,10 @@ test('starts a fresh trace on a fully static page', async ({ page }) => {
 
   const pageloadTx = await pageloadTxPromise;
 
-  // A build-time shell carries no trace context at all.
+  // A build-time shell carries no trace context and no marker, the SDK does not run during the build.
   expect(await page.locator('meta[name="sentry-trace"]').count()).toBe(0);
+  expect(await page.locator('meta[name="sentry-prerender"]').count()).toBe(0);
   expect(pageloadTx.contexts?.trace?.trace_id).toBeTruthy();
   expect(pageloadTx.contexts?.trace?.links).toBeUndefined();
+  expect(pageloadTx.contexts?.trace?.data?.['document.cached']).toBeUndefined();
 });

dev-packages/e2e-tests/test-applications/nextjs-16-streaming-cacheComponents/tests/cacheOrigin-pageloadConnection.spec.ts

@@ -7,7 +7,8 @@ const APP = 'nextjs-16-streaming-cacheComponents';
 // The server only writes `sentry-trace` meta tags for a render that happened for the current request,
 // never while it prerenders a shell. So a tag in the document can always be continued; the only
 // question is when it arrives. The resumed part of a prerendered shell arrives after the SDK started,
-// which the pageload instrumentation waits for.
+// which the pageload instrumentation waits for. A shell prerendered at runtime carries a `sentry-prerender`
+// marker instead, which marks the pageload as served from a cached shell and links the prerender.
 
 test('connects the pageload trace to the runtime server request', async ({ page }) => {
   // The shell is prerendered, but the page's dynamic part (`DynamicContent` awaits `headers()`)
@@ -28,7 +29,6 @@ test('connects the pageload trace to the runtime server request', async ({ page
 
   expect(serverSpan.trace_id).toBeTruthy();
   expect(pageloadSpan.trace_id).toBe(serverSpan.trace_id);
-  expect(pageloadSpan.links).toBeUndefined();
 });
 
 test('connects the pageload trace to a blocking render', async ({ page }) => {
@@ -56,10 +56,11 @@ test('does not share a trace between visitors of a shell generated at request ti
   test.skip(process.env.TEST_ENV !== 'production', 'Shells are only cached in production builds');
 
   // `[id]` only lists `test` in `generateStaticParams`, so the first visit prerenders and caches the
-  // shell for this id. The prerender writes no trace tag, so every visitor starts a fresh trace.
+  // shell for this id. The prerender writes no trace tag, only a `sentry-prerender` marker with its own
+  // trace, so every visitor starts a fresh trace that links the prerender and is marked as a cached shell.
   const id = `shell-${Date.now()}`;
 
-  const visit = async (): Promise<{ traceId: string | undefined; links: unknown }> => {
+  const visit = async (): Promise<{ traceId: string | undefined; links: unknown; documentCached: unknown }> => {
     const context = await browser.newContext();
     const page: Page = await context.newPage();
 
@@ -70,11 +71,16 @@ test('does not share a trace between visitors of a shell generated at request ti
     await page.goto(`/${id}`);
     await expect(page.locator('#result')).toHaveText(`Hello, ${id}!`);
     expect(await page.locator('meta[name="sentry-trace"]').count()).toBe(0);
+    expect(await page.locator('meta[name="sentry-prerender"]').count()).toBe(1);
 
     const pageloadSpan = await pageloadSpanPromise;
     await context.close();
 
-    return { traceId: pageloadSpan.trace_id, links: pageloadSpan.links };
+    return {
+      traceId: pageloadSpan.trace_id,
+      links: pageloadSpan.links,
+      documentCached: pageloadSpan.attributes['document.cached']?.value,
+    };
   };
 
   const first = await visit();
@@ -83,8 +89,17 @@ test('does not share a trace between visitors of a shell generated at request ti
   expect(first.traceId).toBeTruthy();
   expect(second.traceId).toBeTruthy();
   expect(second.traceId).not.toBe(first.traceId);
-  expect(first.links).toBeUndefined();
-  expect(second.links).toBeUndefined();
+  expect(first.documentCached).toBe(true);
+  expect(second.documentCached).toBe(true);
+
+  // Both link the one request that prerendered the shell, which is neither visitor's own trace.
+  const linkOf = (links: unknown): { trace_id: string; attributes?: Record<string, { value: unknown }> } => {
+    const link = (links as Array<{ trace_id: string; attributes?: Record<string, { value: unknown }> }>)[0];
+    expect(link?.attributes?.['sentry.link.type']?.value).toBe('cache_origin');
+    return link!;
+  };
+  expect(linkOf(first.links).trace_id).toBe(linkOf(second.links).trace_id);
+  expect(linkOf(second.links).trace_id).not.toBe(second.traceId);
 });
 
 test('starts a fresh trace on a fully static page', async ({ page }) => {
@@ -100,8 +115,10 @@ test('starts a fresh trace on a fully static page', async ({ page }) => {
 
   const pageloadSpan = await pageloadSpanPromise;
 
-  // A build-time shell carries no trace context at all.
+  // A build-time shell carries no trace context and no marker, the SDK does not run during the build.
   expect(await page.locator('meta[name="sentry-trace"]').count()).toBe(0);
+  expect(await page.locator('meta[name="sentry-prerender"]').count()).toBe(0);
   expect(pageloadSpan.trace_id).toBeTruthy();
   expect(pageloadSpan.links).toBeUndefined();
+  expect(pageloadSpan.attributes['document.cached']).toBeUndefined();
 });

packages/nextjs/src/client/browserTracingIntegration.ts

@@ -1,6 +1,6 @@
 import type { Integration } from '@sentry/core';
 import { browserTracingIntegration as originalBrowserTracingIntegration, isBotUserAgent } from '@sentry/react';
-import { settlePendingPageloadWait } from './routing/appRouterRoutingInstrumentation';
+import { settlePendingPageloadWait } from './routing/appRouterPageloadInstrumentation';
 import { nextRouterInstrumentNavigation, nextRouterInstrumentPageLoad } from './routing/nextRoutingInstrumentation';
 
 /**

packages/nextjs/src/client/routing/appRouterPageloadInstrumentation.ts

@@ -0,0 +1,125 @@
+import {
+  SENTRY_OP,
+  SENTRY_ORIGIN,
+  SENTRY_SEGMENT_NAME_SOURCE,
+  URL_FULL,
+  URL_PATH,
+  URL_TEMPLATE,
+} from '@sentry/conventions/attributes';
+import { PAGELOAD } from '@sentry/conventions/op';
+import type { Client, Span, StartSpanOptions } from '@sentry/core';
+import {
+  filterCollectedUrl,
+  getCurrentScope,
+  hasSpanStreamingEnabled,
+  PAGELOAD_SPAN_NAME_FALLBACK,
+} from '@sentry/core';
+import { resolveCurrentRoute, startBrowserTracingPageLoadSpan, WINDOW } from '@sentry/react';
+import { stripTrailingSlash } from './parameterization';
+import type { TraceMetaTagValues, TraceMetaTagWait } from './traceMetaTags';
+import {
+  addTraceMetaTagLink,
+  cachedDocumentSpanOptions,
+  canWaitForTraceMetaTag,
+  readPrerenderMarker,
+  readTraceMetaTags,
+  waitForTraceMetaTag,
+} from './traceMetaTags';
+
+/**
+ * How long the pageload start may wait for the resumed part of a Cache Components document. The wait
+ * normally ends with `DOMContentLoaded`; the cap only matters for a response the server keeps open.
+ */
+const PAGELOAD_TRACE_META_TAG_TIMEOUT_MS = 10_000;
+
+let pendingPageloadWait: TraceMetaTagWait | undefined;
+
+/**
+ * Starts a pageload that is still waiting for its trace meta tag with a fresh trace. Called when a
+ * navigation starts, before the navigation span is created, so the navigation ends the pageload like
+ * any pageload it interrupts, and the pageload is still recorded.
+ */
+export function settlePendingPageloadWait(): void {
+  pendingPageloadWait?.giveUp();
+}
+
+/**
+ * Starts the pageload span with the request data of the document. The span may start after a
+ * navigation already changed the location, so the browser SDK would otherwise read the new one.
+ */
+function startPageloadSpan(
+  client: Client,
+  spanOptions: StartSpanOptions,
+  documentUrl: string,
+  traceMetaTags?: TraceMetaTagValues,
+): Span | undefined {
+  const prerenderMarker = readPrerenderMarker();
+  const prerenderOptions = prerenderMarker ? cachedDocumentSpanOptions(prerenderMarker) : undefined;
+  const span = startBrowserTracingPageLoadSpan(
+    client,
+    prerenderOptions
+      ? {
+          ...spanOptions,
+          attributes: { ...spanOptions.attributes, ...prerenderOptions.attributes },
+          links: prerenderOptions.links,
+        }
+      : spanOptions,
+    traceMetaTags,
+  );
+  const scope = getCurrentScope();
+  const { normalizedRequest } = scope.getScopeData().sdkProcessingMetadata;
+  if (normalizedRequest) {
+    scope.setSDKProcessingMetadata({ normalizedRequest: { ...normalizedRequest, url: documentUrl } });
+  }
+  return span;
+}
+
+/** Instruments the Next.js app router for pageloads. */
+export function appRouterInstrumentPageLoad(client: Client): void {
+  const documentUrl = WINDOW.location.href;
+  const pathname = stripTrailingSlash(WINDOW.location.pathname);
+  const parameterizedPathname = resolveCurrentRoute(client);
+  const spanOptions: StartSpanOptions = {
+    // With span streaming, span names have to be low cardinality, so we can't fall back to the URL.
+    name: parameterizedPathname ?? (hasSpanStreamingEnabled(client) ? PAGELOAD_SPAN_NAME_FALLBACK : pathname),
+    // pageload should always start at timeOrigin (and needs to be in s, not ms)
+    attributes: {
+      [SENTRY_OP]: PAGELOAD,
+      [SENTRY_ORIGIN]: 'auto.pageload.nextjs.app_router_instrumentation',
+      [SENTRY_SEGMENT_NAME_SOURCE]: parameterizedPathname ? 'route' : 'url',
+      [URL_PATH]: WINDOW.location.pathname,
+      [URL_FULL]: filterCollectedUrl(documentUrl, client),
+      ...(parameterizedPathname && { [URL_TEMPLATE]: parameterizedPathname }),
+    },
+  };
+
+  // A tag that is already in the document was rendered for this request and can be continued. A
+  // Cache Components document that resumes a prerendered shell gets its tag with the resumed part,
+  // which usually arrives after the SDK started. Waiting for it lets the pageload continue the server
+  // trace instead of linking it; the span's start is backdated to the time origin either way.
+  const traceMetaTags = readTraceMetaTags();
+  if (traceMetaTags || !canWaitForTraceMetaTag()) {
+    startPageloadSpan(client, spanOptions, documentUrl, traceMetaTags);
+    return;
+  }
+
+  pendingPageloadWait = waitForTraceMetaTag(
+    values => {
+      pendingPageloadWait = undefined;
+      startPageloadSpan(client, spanOptions, documentUrl, values);
+    },
+    () => {
+      pendingPageloadWait = undefined;
+      const span = startPageloadSpan(client, spanOptions, documentUrl);
+      // Only reachable through the cap or a navigation while the document still streams: a tag that
+      // shows up after this can no longer be continued, but the server request can still be linked.
+      if (span) {
+        waitForTraceMetaTag(
+          values => span.isRecording() && addTraceMetaTagLink(span, values),
+          () => undefined,
+        );
+      }
+    },
+    PAGELOAD_TRACE_META_TAG_TIMEOUT_MS,
+  );
+}