fix(core): Deduplicate native HTTP breadcrumbs for URLs with a query string

CHANGELOG.md

@@ -16,6 +16,7 @@
 ### Fixes
 
 - Do not mark `sendDefaultPii` as deprecated in the React Native options ([#6864](https://github.com/getsentry/sentry-react-native/pull/6864))
+- Fix duplicate HTTP breadcrumbs for requests with a query string or differently formatted URLs ([#6881](https://github.com/getsentry/sentry-react-native/pull/6881))
 
 ### Dependencies
 

packages/core/src/js/integrations/devicecontext.ts

@@ -119,19 +119,25 @@ function deduplicateNativeHttpBreadcrumbs(nativeBreadcrumbs: Breadcrumb[], jsBre
   }
 
   const consumedJsIndices = new Set<number>();
+  const jsUrls = jsHttpBreadcrumbs.map(b => normalizeJsHttpUrl(b.data?.url));
 
   return nativeBreadcrumbs.filter(nativeBreadcrumb => {
     if (nativeBreadcrumb.type !== 'http' || nativeBreadcrumb.category !== 'http') {
       return true;
     }
 
+    const nativeUrl = normalizeNativeHttpUrl(nativeBreadcrumb.data?.url);
+
     const matchIndex = jsHttpBreadcrumbs.findIndex((jsBreadcrumb, index) => {
       if (consumedJsIndices.has(index)) {
         return false;
       }
 
+      const jsUrl = jsUrls[index];
       const sameMethod = nativeBreadcrumb.data?.method === jsBreadcrumb.data?.method;
-      const sameUrl = nativeBreadcrumb.data?.url === jsBreadcrumb.data?.url;
+      const sameUrl =
+        nativeBreadcrumb.data?.url === jsBreadcrumb.data?.url ||
+        (nativeUrl !== undefined && jsUrl !== undefined && isSameHttpUrl(nativeUrl, jsUrl));
       const nativeStatus = nativeBreadcrumb.data?.status_code;
       const jsStatus = jsBreadcrumb.data?.status_code;
       const sameStatus =
@@ -155,3 +161,56 @@ function deduplicateNativeHttpBreadcrumbs(nativeBreadcrumbs: Breadcrumb[], jsBre
     return true;
   });
 }
+
+/*
+ * The JS breadcrumb keeps the raw request URL. The native one is sanitized: credentials are replaced,
+ * the fragment is removed, the query is moved to `data["http.query"]` or kept filtered, and the URL
+ * is percent-decoded. The query is ignored because native may filter or drop it.
+ */
+
+/** Returns the decoded JS URL without query, fragment or trailing slash. */
+function normalizeJsHttpUrl(url: unknown): string | undefined {
+  if (typeof url !== 'string') {
+    return undefined;
+  }
+
+  // Cut on the raw URL, where `?` and `#` are always delimiters.
+  const queryOrFragmentStart = url.search(/[?#]/);
+  const normalized = decodeUrl(
+    normalizeUrlAuthority(queryOrFragmentStart === -1 ? url : url.slice(0, queryOrFragmentStart)),
+  );
+
+  return normalized.length > 1 && normalized.endsWith('/') ? normalized.slice(0, -1) : normalized;
+}
+
+/** Returns the decoded native URL. It may still contain a (filtered) query. */
+function normalizeNativeHttpUrl(url: unknown): string | undefined {
+  return typeof url === 'string' ? decodeUrl(normalizeUrlAuthority(url)) : undefined;
+}
+
+/** Matches the native URL to the JS one, with or without a trailing slash or query. */
+function isSameHttpUrl(nativeUrl: string, jsUrl: string): boolean {
+  return (
+    nativeUrl === jsUrl ||
+    nativeUrl === `${jsUrl}/` ||
+    nativeUrl.startsWith(`${jsUrl}?`) ||
+    nativeUrl.startsWith(`${jsUrl}/?`)
+  );
+}
+
+/** Removes credentials and lowercases the scheme and host. */
+function normalizeUrlAuthority(url: string): string {
+  return url.replace(/^([a-zA-Z][a-zA-Z0-9+.-]*:\/\/)([^/?#]*)/, (_match, scheme: string, authority: string) => {
+    const host = authority.slice(authority.lastIndexOf('@') + 1);
+    return `${scheme}${host}`.toLowerCase();
+  });
+}
+
+function decodeUrl(url: string): string {
+  try {
+    return decodeURIComponent(url);
+  } catch {
+    // Malformed percent-encoding, compare as is.
+    return url;
+  }
+}

packages/core/test/integrations/devicecontext.test.ts

@@ -614,6 +614,119 @@ describe('Device Context Integration', () => {
         },
       ]);
     });
+
+    it.each([
+      ['the js url has a query string', 'https://api.example.com/posts', 'https://api.example.com/posts?page=1'],
+      ['the js url has a fragment', 'https://api.example.com/posts', 'https://api.example.com/posts#top'],
+      [
+        'the native url keeps a filtered query',
+        'https://api.example.com/posts?page=1&token=[Filtered]',
+        'https://api.example.com/posts?page=1&token=abc',
+      ],
+      ['host casing and trailing slashes differ', 'https://API.Example.com/data/', 'https://api.example.com/data'],
+      ['the url is the root with a port', 'https://api.example.com:8080/', 'https://api.example.com:8080'],
+      [
+        'the native url is percent-decoded',
+        'https://api.example.com/users/john doe',
+        'https://api.example.com/users/john%20doe',
+      ],
+      ['the path has an encoded question mark', 'https://api.example.com/a?b', 'https://api.example.com/a%3Fb?x=1'],
+      [
+        'the native url has filtered credentials',
+        'https://[Filtered]:[Filtered]@api.example.com/data',
+        'https://user:Pass%2Fword@api.example.com/data',
+      ],
+      ['the url has malformed percent-encoding', 'https://api.example.com/100%', 'https://api.example.com/100%?x=1'],
+      [
+        'the urls are identical with a fragment',
+        'https://api.example.com/docs#intro',
+        'https://api.example.com/docs#intro',
+      ],
+    ])('deduplicates when %s', async (_name, nativeUrl, jsUrl) => {
+      const jsBreadcrumb = {
+        category: 'xhr',
+        type: 'http',
+        data: { method: 'GET', url: jsUrl, status_code: 200 },
+        timestamp: 1730985899,
+      };
+
+      const { processedEvent } = await processEventWith(
+        {
+          nativeContexts: {
+            breadcrumbs: [
+              {
+                category: 'http',
+                type: 'http',
+                data: { method: 'GET', url: nativeUrl, status_code: 200 },
+                timestamp: 'Thursday, November 7, 2024 3:24:59 PM GMT+02:00',
+              },
+            ],
+          },
+          mockEvent: { breadcrumbs: [jsBreadcrumb] },
+        },
+        mockClient,
+      );
+
+      expect(processedEvent?.breadcrumbs).toStrictEqual([jsBreadcrumb]);
+    });
+
+    it.each([
+      ['the paths differ only in case', 'https://api.example.com/Data', 'https://api.example.com/data'],
+      ['the native path is longer', 'https://api.example.com/data-archive', 'https://api.example.com/data'],
+    ])('keeps both breadcrumbs when %s', async (_name, nativeUrl, jsUrl) => {
+      const nativeBreadcrumb = {
+        category: 'http',
+        type: 'http',
+        data: { method: 'GET', url: nativeUrl, status_code: 200 },
+        timestamp: 1730985899,
+      };
+      const jsBreadcrumb = {
+        category: 'xhr',
+        type: 'http',
+        data: { method: 'GET', url: jsUrl, status_code: 200 },
+        timestamp: 1730985899,
+      };
+
+      const { processedEvent } = await processEventWith(
+        {
+          nativeContexts: {
+            breadcrumbs: [{ ...nativeBreadcrumb, timestamp: 'Thursday, November 7, 2024 3:24:59 PM GMT+02:00' }],
+          },
+          mockEvent: { breadcrumbs: [jsBreadcrumb] },
+        },
+        mockClient,
+      );
+
+      expect(processedEvent?.breadcrumbs).toStrictEqual([nativeBreadcrumb, jsBreadcrumb]);
+    });
+
+    it('deduplicates when the native query differs, since the query is ignored', async () => {
+      const jsBreadcrumb = {
+        category: 'xhr',
+        type: 'http',
+        data: { method: 'GET', url: 'https://api.example.com/img?w=1', status_code: 200 },
+        timestamp: 1730985899,
+      };
+
+      const { processedEvent } = await processEventWith(
+        {
+          nativeContexts: {
+            breadcrumbs: [
+              {
+                category: 'http',
+                type: 'http',
+                data: { method: 'GET', url: 'https://api.example.com/img', 'http.query': 'w=2', status_code: 200 },
+                timestamp: 'Thursday, November 7, 2024 3:24:59 PM GMT+02:00',
+              },
+            ],
+          },
+          mockEvent: { breadcrumbs: [jsBreadcrumb] },
+        },
+        mockClient,
+      );
+
+      expect(processedEvent?.breadcrumbs).toStrictEqual([jsBreadcrumb]);
+    });
   });
 
   it('do not add in_foreground if unknown', async () => {