feat(cloudflare): Propagate traceparent to RPC calls - via fetch (#19991)

relates to #19327

related to #16898 (it is not really closing it as we just add context
propagation without adding spans for individual calls. It needs to be
defined if we need it)

It is important to know that these RPC calls do only work with the
`.fetch` call:

```js
const id = env.MY_DURABLE_OBJECT.idFromName('workflow-test');
const stub = env.MY_DURABLE_OBJECT.get(id);

await stub.fetch(new Request('http://my-worker/my-do-call'));
```

This adds RPC fetch calls between:
- Workers -> Workers ([Service
bindings](https://developers.cloudflare.com/workers/runtime-apis/bindings/service-bindings/rpc/))
- Workers -> DurableObjects (via [standard
RPC](https://developers.cloudflare.com/workers/runtime-apis/rpc/))
- Workflows -> DurableObjects (also via standard RPC)

This works by instrumenting `env` (via `instrumentEnv`), which then goes
over the bindings and see if there is a DurableObject or a normal
Fetcher (full list of current bindings:
https://developers.cloudflare.com/workers/runtime-apis/bindings/). This
got inspired by how `otel-cf-workers` instruments their env:
https://github.com/evanderkoogh/otel-cf-workers/blob/effeb549f0a4ed1c55ea0c4f0d8e8e37e5494fb3/src/instrumentation/env.ts

With this PR I added a lot of tests to check if trace propagation works
(so this PR might look like it added a lot of LoC, but it is mostly
tests). So I added it for `schedule` and `queue`, but it is not possible
for `email` and `tail` with `wrangler dev`.

## Potential things to change

### Trace propagagtion

I added the `addTraceHeaders.ts` helper, as there is currently no way to
reuse the existing logic (it is baked-in into the fetch
instrumentations). It would be nice once #19960 lands that we can reuse
it in Cloudflare to reuse existing code. I tried to write couple of
tests so we don't have duplicated headers.

### Adding extra spans

So there is actually a guide by OTel to [add RPC
spans](https://opentelemetry.io/docs/specs/semconv/rpc/rpc-spans/), but
was talking with someone from the OTel maintainers and they meant that
this wouldn't be necessary as we already have an `http.server` span from
out instrumented DurableObjects (and other resources) - so it wouldn't
add much of information.

Without RPC span:

<img width="451" height="130" alt="Screenshot 2026-03-25 at 10 59 01"
src="https://github.com/user-attachments/assets/dc280b38-0879-4306-8d61-7fcc5e9cacc0"
/>

With RPC span:

<img width="433" height="170" alt="Screenshot 2026-03-25 at 10 55 48"
src="https://github.com/user-attachments/assets/e30fa84b-068a-4eca-aa06-5668f3f0081c"
/>

Author: JPeer264

dev-packages/cloudflare-integration-tests/runner.ts

@@ -68,12 +68,17 @@ export function createRunner(...paths: string[]) {
   // By default, we ignore session & sessions
   const ignored: Set<EnvelopeItemType> = new Set(['session', 'sessions', 'client_report']);
   let serverUrl: string | undefined;
+  const extraWranglerArgs: string[] = [];
 
   return {
     withServerUrl: function (url: string) {
       serverUrl = url;
       return this;
     },
+    withWranglerArgs: function (...args: string[]) {
+      extraWranglerArgs.push(...args);
+      return this;
+    },
     expect: function (expected: Expected) {
       expectedEnvelopes.push(expected);
       return this;
@@ -237,6 +242,7 @@ export function createRunner(...paths: string[]) {
               `SENTRY_DSN:http://public@localhost:${mockServerPort}/1337`,
               '--var',
               `SERVER_URL:${serverUrl}`,
+              ...extraWranglerArgs,
             ],
             { stdio, signal },
           );

dev-packages/cloudflare-integration-tests/suites/tracing/instrument-fetcher/index.ts

@@ -0,0 +1,94 @@
+import { instrumentDurableObjectWithSentry, withSentry } from '@sentry/cloudflare';
+import { DurableObject } from 'cloudflare:workers';
+
+interface Env {
+  SENTRY_DSN: string;
+  ECHO_HEADERS_DO: DurableObjectNamespace;
+}
+
+class EchoHeadersDurableObjectBase extends DurableObject<Env> {
+  async fetch(incoming: Request): Promise<Response> {
+    return Response.json({
+      sentryTrace: incoming.headers.get('sentry-trace'),
+      baggage: incoming.headers.get('baggage'),
+      authorization: incoming.headers.get('authorization'),
+      xFromInit: incoming.headers.get('x-from-init'),
+      xExtra: incoming.headers.get('x-extra'),
+      xMergeProbe: incoming.headers.get('x-merge-probe'),
+    });
+  }
+}
+
+export const EchoHeadersDurableObject = instrumentDurableObjectWithSentry(
+  (env: Env) => ({
+    dsn: env.SENTRY_DSN,
+    tracesSampleRate: 1.0,
+  }),
+  EchoHeadersDurableObjectBase,
+);
+
+export default withSentry(
+  (env: Env) => ({
+    dsn: env.SENTRY_DSN,
+    tracesSampleRate: 1.0,
+  }),
+  {
+    async fetch(request, env) {
+      const url = new URL(request.url);
+      const id = env.ECHO_HEADERS_DO.idFromName('instrument-fetcher-echo');
+      const stub = env.ECHO_HEADERS_DO.get(id);
+      const doUrl = new URL(request.url);
+
+      let subResponse: Response;
+
+      if (url.pathname === '/via-init') {
+        subResponse = await stub.fetch(doUrl, {
+          headers: {
+            Authorization: 'Bearer from-init',
+            'X-Extra': 'init-extra',
+            'X-Merge-Probe': 'via-init-probe',
+          },
+        });
+      } else if (url.pathname === '/via-request') {
+        subResponse = await stub.fetch(
+          new Request(doUrl, {
+            headers: {
+              Authorization: 'Bearer from-request',
+              'X-Extra': 'request-extra',
+              'X-Merge-Probe': 'via-request-probe',
+            },
+          }),
+        );
+      } else if (url.pathname === '/via-request-and-init') {
+        subResponse = await stub.fetch(
+          new Request(doUrl, {
+            headers: {
+              Authorization: 'Bearer from-request',
+              'X-Extra': 'request-extra',
+              'X-Merge-Probe': 'dropped-from-request',
+            },
+          }),
+          {
+            headers: {
+              'X-From-Init': '1',
+              'X-Merge-Probe': 'via-init-wins',
+            },
+          },
+        );
+      } else if (url.pathname === '/with-preset-sentry-baggage') {
+        subResponse = await stub.fetch(
+          new Request(doUrl, {
+            headers: {
+              baggage: 'sentry-environment=preset,acme=vendor',
+            },
+          }),
+        );
+      } else {
+        return new Response('not found', { status: 404 });
+      }
+
+      const payload: unknown = await subResponse.json();
+      return Response.json(payload);
+    },
+  } satisfies ExportedHandler<Env>,
+);

dev-packages/cloudflare-integration-tests/suites/tracing/instrument-fetcher/test.ts

@@ -0,0 +1,131 @@
+import { expect, it } from 'vitest';
+import type { Event } from '@sentry/core';
+import { createRunner } from '../../../runner';
+
+type EchoedHeaders = {
+  sentryTrace: string | null;
+  baggage: string | null;
+  authorization: string | null;
+  xFromInit: string | null;
+  xExtra: string | null;
+  xMergeProbe: string | null;
+};
+
+const SENTRY_TRACE_HEADER_RE = /^[0-9a-f]{32}-[0-9a-f]{16}-[01]$/;
+
+type ScenarioPath = '/via-init' | '/via-request' | '/via-request-and-init' | '/with-preset-sentry-baggage';
+
+function startStubFetchScenario(path: ScenarioPath, signal: AbortSignal) {
+  let mainTraceId: string | undefined;
+  let mainSpanId: string | undefined;
+  let doTraceId: string | undefined;
+  let doParentSpanId: string | undefined;
+
+  const traceBase = {
+    op: 'http.server',
+    data: expect.objectContaining({
+      'sentry.origin': 'auto.http.cloudflare',
+    }),
+    origin: 'auto.http.cloudflare',
+  };
+
+  const { makeRequest, completed } = createRunner(__dirname)
+    .expect(envelope => {
+      const transactionEvent = envelope[1]?.[0]?.[1] as Event;
+      const parentSpanId = transactionEvent.contexts?.trace?.parent_span_id;
+
+      expect(transactionEvent).toEqual(
+        expect.objectContaining({
+          contexts: expect.objectContaining({
+            trace: expect.objectContaining(traceBase),
+          }),
+          transaction: `GET ${path}`,
+        }),
+      );
+      expect(parentSpanId).toBeUndefined();
+
+      mainTraceId = transactionEvent.contexts?.trace?.trace_id as string;
+      mainSpanId = transactionEvent.contexts?.trace?.span_id as string;
+    })
+    .expect(envelope => {
+      const transactionEvent = envelope[1]?.[0]?.[1] as Event;
+      const parentSpanId = transactionEvent.contexts?.trace?.parent_span_id;
+
+      expect(transactionEvent).toEqual(
+        expect.objectContaining({
+          contexts: expect.objectContaining({
+            trace: expect.objectContaining(traceBase),
+          }),
+          transaction: `GET ${path}`,
+        }),
+      );
+      expect(parentSpanId).toBeDefined();
+
+      doTraceId = transactionEvent.contexts?.trace?.trace_id as string;
+      doParentSpanId = parentSpanId as string;
+    })
+    .unordered()
+    .start(signal);
+
+  return {
+    makeRequest,
+    async completedWithTraceCheck(): Promise<void> {
+      await completed();
+      expect(mainTraceId).toBeDefined();
+      expect(doTraceId).toBeDefined();
+      expect(mainTraceId).toBe(doTraceId);
+      expect(mainSpanId).toBeDefined();
+      expect(doParentSpanId).toBeDefined();
+      expect(doParentSpanId).toBe(mainSpanId);
+    },
+  };
+}
+
+it('stub.fetch: headers in init (URL string + init)', async ({ signal }) => {
+  const { makeRequest, completedWithTraceCheck } = startStubFetchScenario('/via-init', signal);
+  const body = await makeRequest<EchoedHeaders>('get', '/via-init');
+  await completedWithTraceCheck();
+
+  expect(body?.sentryTrace).toEqual(expect.stringMatching(SENTRY_TRACE_HEADER_RE));
+  expect(body?.baggage).toContain('sentry-environment=production,sentry-public_key=public,sentry-trace_id=');
+  expect(body?.authorization).toBe('Bearer from-init');
+  expect(body?.xExtra).toBe('init-extra');
+  expect(body?.xMergeProbe).toBe('via-init-probe');
+  expect(body?.xFromInit).toBeNull();
+});
+
+it('stub.fetch: headers on Request (URL from incoming request)', async ({ signal }) => {
+  const { makeRequest, completedWithTraceCheck } = startStubFetchScenario('/via-request', signal);
+  const body = await makeRequest<EchoedHeaders>('get', '/via-request');
+  await completedWithTraceCheck();
+
+  expect(body?.sentryTrace).toEqual(expect.stringMatching(SENTRY_TRACE_HEADER_RE));
+  expect(body?.baggage).toContain('sentry-environment=production,sentry-public_key=public,sentry-trace_id=');
+  expect(body?.authorization).toBe('Bearer from-request');
+  expect(body?.xExtra).toBe('request-extra');
+  expect(body?.xMergeProbe).toBe('via-request-probe');
+  expect(body?.xFromInit).toBeNull();
+});
+
+it('stub.fetch: Request + init — only init headers are sent', async ({ signal }) => {
+  const { makeRequest, completedWithTraceCheck } = startStubFetchScenario('/via-request-and-init', signal);
+  const body = await makeRequest<EchoedHeaders>('get', '/via-request-and-init');
+  await completedWithTraceCheck();
+
+  expect(body?.sentryTrace).toEqual(expect.stringMatching(SENTRY_TRACE_HEADER_RE));
+  expect(body?.baggage).toContain('sentry-environment=production,sentry-public_key=public,sentry-trace_id=');
+  expect(body?.authorization).toBeNull();
+  expect(body?.xExtra).toBeNull();
+  expect(body?.xMergeProbe).toBe('via-init-wins');
+  expect(body?.xFromInit).toBe('1');
+});
+
+it('stub.fetch: does not append SDK baggage when the Request already includes Sentry baggage', async ({ signal }) => {
+  const { makeRequest, completedWithTraceCheck } = startStubFetchScenario('/with-preset-sentry-baggage', signal);
+  const body = await makeRequest<EchoedHeaders>('get', '/with-preset-sentry-baggage');
+  await completedWithTraceCheck();
+
+  expect(body?.sentryTrace).toEqual(expect.stringMatching(SENTRY_TRACE_HEADER_RE));
+  // Dynamic SDK baggage includes `sentry-trace_id=…`; appending it again would change this string.
+  expect(body?.baggage).toBe('sentry-environment=preset,acme=vendor');
+});

dev-packages/cloudflare-integration-tests/suites/tracing/instrument-fetcher/wrangler.jsonc

@@ -0,0 +1,20 @@
+{
+  "name": "cloudflare-instrument-fetcher",
+  "main": "index.ts",
+  "compatibility_date": "2025-06-17",
+  "compatibility_flags": ["nodejs_als"],
+  "migrations": [
+    {
+      "new_sqlite_classes": ["EchoHeadersDurableObject"],
+      "tag": "v1",
+    },
+  ],
+  "durable_objects": {
+    "bindings": [
+      {
+        "class_name": "EchoHeadersDurableObject",
+        "name": "ECHO_HEADERS_DO",
+      },
+    ],
+  },
+}

dev-packages/cloudflare-integration-tests/suites/tracing/propagation/worker-do/index.ts

@@ -0,0 +1,60 @@
+import * as Sentry from '@sentry/cloudflare';
+import { DurableObject } from 'cloudflare:workers';
+
+interface Env {
+  SENTRY_DSN: string;
+  MY_DURABLE_OBJECT: DurableObjectNamespace;
+  MY_QUEUE: Queue;
+}
+
+class MyDurableObjectBase extends DurableObject<Env> {
+  async fetch(request: Request) {
+    return new Response('DO is fine');
+  }
+}
+
+export const MyDurableObject = Sentry.instrumentDurableObjectWithSentry(
+  (env: Env) => ({
+    dsn: env.SENTRY_DSN,
+    tracesSampleRate: 1.0,
+  }),
+  MyDurableObjectBase,
+);
+
+export default Sentry.withSentry(
+  (env: Env) => ({
+    dsn: env.SENTRY_DSN,
+    tracesSampleRate: 1.0,
+  }),
+  {
+    async fetch(request, env) {
+      const url = new URL(request.url);
+
+      if (url.pathname === '/queue/send') {
+        await env.MY_QUEUE.send({ action: 'test' });
+        return new Response('Queued');
+      }
+
+      const id = env.MY_DURABLE_OBJECT.idFromName('test');
+      const stub = env.MY_DURABLE_OBJECT.get(id);
+      const response = await stub.fetch(new Request('http://fake-host/hello'));
+      const text = await response.text();
+      return new Response(text);
+    },
+
+    async queue(batch, env, _ctx) {
+      const id = env.MY_DURABLE_OBJECT.idFromName('test');
+      const stub = env.MY_DURABLE_OBJECT.get(id);
+      for (const message of batch.messages) {
+        await stub.fetch(new Request('http://fake-host/hello'));
+        message.ack();
+      }
+    },
+
+    async scheduled(controller, env, _ctx) {
+      const id = env.MY_DURABLE_OBJECT.idFromName('test');
+      const stub = env.MY_DURABLE_OBJECT.get(id);
+      await stub.fetch(new Request('http://fake-host/hello'));
+    },
+  } satisfies ExportedHandler<Env>,
+);