From bb427f7f26533c09e45aaa5090563b532548ab69 Mon Sep 17 00:00:00 2001 From: William Conti Date: Wed, 26 Aug 2026 15:30:36 -0400 Subject: [PATCH 1/9] feat(turbopack): support bundled instrumentation --- .github/CODEOWNERS | 1 + .github/workflows/instrumentation.yml | 15 ++ package.json | 3 + packages/datadog-turbopack/index.js | 59 +++++ packages/datadog-turbopack/src/loader.js | 125 +++++++++ packages/datadog-turbopack/src/targets.js | 237 ++++++++++++++++++ .../datadog-turbopack/test/plugin.spec.js | 134 ++++++++++ turbopack.js | 3 + 8 files changed, 577 insertions(+) create mode 100644 packages/datadog-turbopack/index.js create mode 100644 packages/datadog-turbopack/src/loader.js create mode 100644 packages/datadog-turbopack/src/targets.js create mode 100644 packages/datadog-turbopack/test/plugin.spec.js create mode 100644 turbopack.js diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 28f851466a6..4033c1b49cc 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -102,6 +102,7 @@ /integration-tests/pino.spec.js @DataDog/dd-trace-js @DataDog/apm-idm-js /packages/datadog-esbuild/ @DataDog/dd-trace-js @DataDog/apm-idm-js +/packages/datadog-turbopack/ @DataDog/dd-trace-js @DataDog/apm-idm-js /packages/datadog-webpack/ @DataDog/dd-trace-js @DataDog/apm-idm-js /packages/datadog-plugin-*/ @DataDog/dd-trace-js @DataDog/apm-idm-js /packages/datadog-instrumentations/ @DataDog/dd-trace-js @DataDog/apm-idm-js diff --git a/.github/workflows/instrumentation.yml b/.github/workflows/instrumentation.yml index 63179f945a5..b953fb8c315 100644 --- a/.github/workflows/instrumentation.yml +++ b/.github/workflows/instrumentation.yml @@ -36,6 +36,21 @@ jobs: with: flags: platform-esbuild + turbopack: + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/node/oldest-maintenance-lts + - uses: ./.github/actions/install + - run: npm run test:turbopack:ci + - uses: ./.github/actions/node/latest + - run: npm run test:turbopack:ci + - uses: ./.github/actions/coverage + with: + flags: platform-turbopack + webpack: runs-on: ubuntu-latest permissions: diff --git a/package.json b/package.json index 5ab8b3c3018..ae2254bf90f 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,8 @@ "test:trace:guardrails:ci": "node scripts/c8-ci.js test:trace:guardrails", "test:esbuild": "mocha \"packages/datadog-esbuild/test/**/*.spec.js\"", "test:esbuild:ci": "node scripts/c8-ci.js test:esbuild", + "test:turbopack": "mocha \"packages/datadog-turbopack/test/**/*.spec.js\"", + "test:turbopack:ci": "node scripts/c8-ci.js test:turbopack", "test:webpack": "mocha \"packages/datadog-webpack/test/**/*.spec.js\"", "test:webpack:ci": "node scripts/c8-ci.js test:webpack", "test:instrumentations": "mocha --fail-zero \"packages/datadog-instrumentations/test/@(${PLUGINS}).spec.js\"", @@ -150,6 +152,7 @@ "ci/**/*", "cypress/**/*", "esbuild.js", + "turbopack.js", "webpack.js", "ext/**/*", "index.d.ts", diff --git a/packages/datadog-turbopack/index.js b/packages/datadog-turbopack/index.js new file mode 100644 index 00000000000..da242883d1a --- /dev/null +++ b/packages/datadog-turbopack/index.js @@ -0,0 +1,59 @@ +'use strict' + +const { createManifest } = require('./src/targets') + +const loader = require.resolve('./src/loader') + +/** + * Adds Datadog instrumentation rules to a Turbopack configuration. Generated + * aliases and loader metadata are local to the application and apply only to + * Node.js bundles. Browser and Edge bundles retain their original modules. + * + * @param {object} [turbopack] + * @param {string} [projectDir] + * @returns {Promise} + */ +function withDatadogTurbopack (turbopack, projectDir) { + return addRules(turbopack, projectDir) +} + +/** + * @param {object} turbopack + * @param {string} [projectDir] + * @returns {Promise} + */ +async function addRules (turbopack = {}, projectDir = process.cwd()) { + const manifest = await createManifest(projectDir) + if (!manifest.packagePathPattern || !manifest.path) return turbopack + + const aliases = { ...turbopack.resolveAlias } + + for (const [specifier, alias] of Object.entries(manifest.aliases)) { + // Do not replace application aliases, which would change customer behavior. + aliases[specifier] ??= alias + } + + const rules = { ...turbopack.rules } + for (const extension of ['*.js', '*.cjs', '*.mjs']) { + const existing = rules[extension] + const rule = { + condition: { + all: ['foreign', 'node', { path: manifest.packagePathPattern }], + }, + loaders: [{ loader, options: { manifestPath: manifest.path } }], + } + rules[extension] = existing + ? [...(Array.isArray(existing) ? existing : [existing]), rule] + : rule + } + + return { + ...turbopack, + resolveAlias: aliases, + rules, + } +} + +module.exports = { + withDatadogTurbopack, +} diff --git a/packages/datadog-turbopack/src/loader.js b/packages/datadog-turbopack/src/loader.js new file mode 100644 index 00000000000..c025082a5c8 --- /dev/null +++ b/packages/datadog-turbopack/src/loader.js @@ -0,0 +1,125 @@ +'use strict' + +const fs = require('node:fs') +const path = require('node:path') + +const { create } = require('../../../vendor/dist/@apm-js-collab/code-transformer') +const { isESMFile } = require('../../datadog-esbuild/src/utils') + +const CHANNEL = 'dd-trace:bundler:load' + +/** + * Instruments bundled modules known to dd-trace. CommonJS modules publish + * through the existing bundler channel. ESM modules instead rewrite only + * imports that resolve to generated live-binding proxies. + * + * @param {string} source + * @returns {string} + */ +module.exports = function loader (source) { + const { manifestPath } = this.getOptions() + const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) + const target = manifest.targets[normalizePath(this.resourcePath)] + + if (isESMFile(this.resourcePath)) { + return rewriteImports(source, this.resourcePath, manifest.targets) + } + + if (!target) return source + + return `${source} +;{ + const __dd_dc = require('dc-polyfill') + const __dd_ch = __dd_dc.channel('${CHANNEL}') + const __dd_payload = { + module: module.exports, + version: ${JSON.stringify(target.version)}, + package: ${JSON.stringify(target.name)}, + path: ${JSON.stringify(target.path)}, + } + __dd_ch.publish(__dd_payload) + module.exports = __dd_payload.module +} +` +} + +/** + * @param {string} source + * @param {string} resourcePath + * @param {Record} targets + * @returns {string} + */ +function rewriteImports (source, resourcePath, targets) { + resourcePath = normalizePath(resourcePath) + let rewritten = false + const matcher = create([{ + module: { name: 'dd-trace-turbopack', versionRange: '*', filePath: /.*/ }, + astQuery: 'Program', + transform: 'rewriteImports', + }]) + + matcher.addTransform('rewriteImports', (_state, program) => { + visit(program, node => { + if (!isModuleSource(node)) return + + const resolved = resolveFrom(resourcePath, node.source.value) + const target = resolved && targets[resolved] + if (!target?.esm || !target.proxyPath) return + + const proxySpecifier = relativeImport(path.dirname(resourcePath), target.proxyPath) + // The code transformer emits `raw` when present, so update both fields. + node.source.value = proxySpecifier + node.source.raw = JSON.stringify(proxySpecifier) + rewritten = true + }) + }) + + const transformer = matcher.getTransformer('dd-trace-turbopack', '1.0.0', resourcePath) + if (!transformer) return source + + try { + const output = transformer.transform(source, 'esm').code + return rewritten ? output : source + } catch { + // A parser failure must never prevent an application from building. + return source + } +} + +function isModuleSource (node) { + return (node.type === 'ImportDeclaration' || + node.type === 'ExportNamedDeclaration' || + node.type === 'ExportAllDeclaration' || + node.type === 'ImportExpression') && + node.source?.type === 'Literal' && typeof node.source.value === 'string' +} + +function visit (node, callback) { + if (!node || typeof node !== 'object') return + callback(node) + for (const value of Object.values(node)) { + if (Array.isArray(value)) value.forEach(child => visit(child, callback)) + else visit(value, callback) + } +} + +function resolveFrom (resourcePath, specifier) { + try { + return normalizePath(require.resolve(specifier, { + paths: [path.dirname(resourcePath)], + conditions: new Set(['import', 'node']), + })) + } catch {} +} + +function relativeImport (from, to) { + let value = path.relative(from, to).replaceAll('\\', '/') + if (!value.startsWith('.')) value = `./${value}` + return value +} + +function normalizePath (value) { + return fs.realpathSync(value).replaceAll('\\', '/') +} + +module.exports.rewriteImports = rewriteImports diff --git a/packages/datadog-turbopack/src/targets.js b/packages/datadog-turbopack/src/targets.js new file mode 100644 index 00000000000..d0eea18b22f --- /dev/null +++ b/packages/datadog-turbopack/src/targets.js @@ -0,0 +1,237 @@ +'use strict' + +const fs = require('node:fs/promises') +const fsSync = require('node:fs') +const Module = require('node:module') +const path = require('node:path') +const { pathToFileURL } = require('node:url') + +const instrumentations = require('../../datadog-instrumentations/src/helpers/instrumentations') +const hooks = require('../../datadog-instrumentations/src/helpers/hooks') +const { filename, matchVersion } = require('../../datadog-instrumentations/src/helpers/register') +const { isESMFile, processModule } = require('../../datadog-esbuild/src/utils') + +const CACHE_DIRECTORY = path.join('node_modules', '.cache', 'dd-trace', 'turbopack') + +/** + * Builds the Turbopack manifest and generated ESM proxies for supported + * instrumentation targets installed in an application. + * + * @param {string} projectDir + * @returns {Promise<{ aliases: Record, packagePathPattern?: RegExp, path?: string }>} + */ +async function createManifest (projectDir) { + loadInstrumentations() + + const appRequire = Module.createRequire(path.join(projectDir, 'package.json')) + const cacheDirectory = path.join(projectDir, CACHE_DIRECTORY) + const targets = getTargets(appRequire) + const manifestTargets = {} + const aliases = {} + + if (targets.length === 0) return { aliases } + + await fs.mkdir(cacheDirectory, { recursive: true }) + + for (const [index, target] of targets.entries()) { + const entry = { + esm: target.esm, + name: target.name, + path: target.instrumentationPath, + version: target.version, + } + + if (target.esm) { + const proxyPath = path.join(cacheDirectory, `${index}.mjs`) + try { + // Proxies are build-time artifacts; preserve source order for stable paths. + // eslint-disable-next-line no-await-in-loop + await fs.writeFile(proxyPath, await createEsmProxy(target.path, proxyPath, target.specifier)) + } catch { + // An unsupported dependency must not prevent the customer's build. Its + // original module remains bundled without instrumentation instead. + continue + } + entry.proxyPath = normalizePath(proxyPath) + + if (target.entrypoint || target.specifier !== target.name) { + aliases[target.specifier] = { + browser: relativeImport(projectDir, target.path), + default: relativeImport(projectDir, proxyPath), + } + } + } + + manifestTargets[normalizePath(target.path)] = entry + } + + const manifestPath = path.join(cacheDirectory, 'manifest.json') + await fs.writeFile(manifestPath, JSON.stringify({ targets: manifestTargets })) + + const packageNames = [...new Set(targets.map(target => target.name))] + const packagePathPattern = new RegExp( + `(?:^|/)node_modules/(?:${packageNames.map(escapeRegExp).join('|')})(?:/|$)` + ) + + return { aliases, packagePathPattern, path: manifestPath } +} + +/** + * Ensures the existing instrumentation declarations have populated their + * shared registry before we inspect it at build time. + */ +function loadInstrumentations () { + for (const hook of Object.values(hooks)) { + const load = hook?.fn ?? hook + if (typeof load === 'function') load() + } +} + +/** + * @param {Function & { resolve: Function }} appRequire + * @returns {Array<{ + * esm: boolean, entrypoint: boolean, instrumentationPath: string, name: string, + * path: string, specifier: string, version: string + * }>} + */ +function getTargets (appRequire) { + const targets = new Map() + + for (const [name, entries] of Object.entries(instrumentations)) { + if (name.startsWith('node:') || name.startsWith('.')) continue + + let entrypoint + try { + entrypoint = resolveImport(appRequire, name) + } catch { + continue + } + + const packageRoot = findPackageRoot(entrypoint) + if (!packageRoot) continue + + let packageJson + try { + packageJson = JSON.parse(fsSync.readFileSync(path.join(packageRoot, 'package.json'), 'utf8')) + } catch { + continue + } + + for (const entry of entries) { + if (!matchVersion(packageJson.version, entry.versions)) continue + + const files = entry.file + ? [path.join(packageRoot, entry.file)] + : entry.filePattern + ? findMatchingFiles(packageRoot, new RegExp(entry.filePattern)) + : [entrypoint] + + for (const file of files) { + if (!fsSync.existsSync(file)) continue + const modulePath = entry.file || (entry.filePattern && + path.relative(packageRoot, file).replaceAll('\\', '/')) + + targets.set(normalizePath(file), { + esm: isESMFile(file, path.join(packageRoot, 'package.json'), packageJson), + entrypoint: !entry.file && !entry.filePattern && samePath(file, entrypoint), + instrumentationPath: filename(name, modulePath), + name, + path: file, + specifier: modulePath ? `${name}/${modulePath}` : name, + version: packageJson.version, + }) + } + } + } + + return [...targets.values()] +} + +/** + * Resolves with import conditions so an ESM package uses the same entrypoint + * as a Turbopack Node bundle instead of the CommonJS require entrypoint. + * + * @param {Function & { resolve: Function }} appRequire + * @param {string} specifier + * @returns {string} + */ +function resolveImport (appRequire, specifier) { + return appRequire.resolve(specifier, { conditions: new Set(['import', 'node']) }) +} + +/** + * @param {string} file + * @returns {string|undefined} + */ +function findPackageRoot (file) { + let directory = path.dirname(file) + while (directory !== path.dirname(directory)) { + if (fsSync.existsSync(path.join(directory, 'package.json'))) return directory + directory = path.dirname(directory) + } +} + +/** + * @param {string} directory + * @param {RegExp} pattern + * @returns {string[]} + */ +function findMatchingFiles (directory, pattern) { + const files = [] + const pending = [directory] + while (pending.length > 0) { + const current = pending.pop() + for (const entry of fsSync.readdirSync(current, { withFileTypes: true })) { + const target = path.join(current, entry.name) + if (entry.isDirectory() && entry.name !== 'node_modules') { + pending.push(target) + } else if (entry.isFile()) { + const relativePath = path.relative(directory, target).replaceAll('\\', '/') + if (pattern.test(relativePath)) files.push(target) + } + } + } + return files +} + +/** + * @param {string} sourcePath + * @param {string} proxyPath + * @param {string} specifier + * @returns {Promise} + */ +async function createEsmProxy (sourcePath, proxyPath, specifier) { + const setters = await processModule({ path: sourcePath, context: { format: 'module' } }) + return `import { register } from 'import-in-the-middle/lib/register.js'; +import * as namespace from ${JSON.stringify(relativeImport(path.dirname(proxyPath), sourcePath))}; +const _ = Object.create(null, { [Symbol.toStringTag]: { value: 'Module' } }); +const set = {}; +const get = {}; +${[...setters.values()].join(';\n')}; +register(${JSON.stringify(pathToFileURL(sourcePath).href)}, _, set, get, ${JSON.stringify(specifier)}); +` +} + +function relativeImport (from, to) { + let value = path.relative(from, to).replaceAll('\\', '/') + if (!value.startsWith('.')) value = `./${value}` + return value +} + +function normalizePath (value) { + return fsSync.realpathSync(value).replaceAll('\\', '/') +} + +function samePath (left, right) { + return normalizePath(left) === normalizePath(right) +} + +function escapeRegExp (value) { + return value.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`) +} + +module.exports = { + createEsmProxy, + createManifest, + getTargets, +} diff --git a/packages/datadog-turbopack/test/plugin.spec.js b/packages/datadog-turbopack/test/plugin.spec.js new file mode 100644 index 00000000000..dc563f95f00 --- /dev/null +++ b/packages/datadog-turbopack/test/plugin.spec.js @@ -0,0 +1,134 @@ +'use strict' + +const assert = require('node:assert/strict') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { afterEach, describe, it } = require('mocha') + +const { withDatadogTurbopack } = require('..') +const loader = require('../src/loader') + +const directories = [] + +afterEach(() => { + while (directories.length > 0) { + fs.rmSync(directories.pop(), { force: true, recursive: true }) + } +}) + +describe('datadog-turbopack loader', () => { + it('routes an internal ESM import through its generated proxy', () => { + const directory = createPackage('openai', { type: 'module' }) + const client = write(directory, 'client.mjs', "import { Models } from './resources/models.mjs'\nexport { Models }") + const models = write(directory, 'resources/models.mjs', 'export class Models {}') + const proxy = write(directory, '../.cache/dd-trace/turbopack/models.mjs', 'export {}') + + const result = loader.rewriteImports(fs.readFileSync(client, 'utf8'), client, { + [realpath(models)]: { esm: true, proxyPath: realpath(proxy) }, + }) + + assert.match(result, /from "\.\.\/\.cache\/dd-trace\/turbopack\/models\.mjs"/) + }) + + it('routes a dynamic ESM import through its generated proxy', () => { + const directory = createPackage('openai', { type: 'module' }) + const client = write(directory, 'client.mjs', "export const loadModels = () => import('./resources/models.mjs')") + const models = write(directory, 'resources/models.mjs', 'export class Models {}') + const proxy = write(directory, '../.cache/dd-trace/turbopack/models.mjs', 'export {}') + + const result = loader.rewriteImports(fs.readFileSync(client, 'utf8'), client, { + [realpath(models)]: { esm: true, proxyPath: realpath(proxy) }, + }) + + assert.match(result, /import\("\.\.\/\.cache\/dd-trace\/turbopack\/models\.mjs"\)/) + }) + + it('preserves ESM modules without an instrumented import', () => { + const directory = createPackage('openai', { type: 'module' }) + const client = write(directory, 'client.mjs', "import { Models } from './resources/models.mjs'\nexport { Models }") + const source = fs.readFileSync(client, 'utf8') + + const result = loader.rewriteImports(source, client, {}) + + assert.equal(result, source) + }) + + it('publishes CommonJS exports through the existing bundler channel', () => { + const directory = createPackage('ioredis') + const resourcePath = write(directory, 'built/index.js', 'module.exports = {}') + const manifestPath = write(directory, 'manifest.json', JSON.stringify({ + targets: { + [realpath(resourcePath)]: { + esm: false, + name: 'ioredis', + path: 'ioredis', + version: '5.0.0', + }, + }, + })) + + const result = loader.call({ + getOptions: () => ({ manifestPath }), + resourcePath, + }, 'module.exports = {}') + + assert.match(result, /dd-trace:bundler:load/) + assert.match(result, /package: "ioredis"/) + assert.match(result, /path: "ioredis"/) + }) +}) + +describe('datadog-turbopack configuration', () => { + it('does not add rules when no supported package is installed', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) + directories.push(directory) + fs.writeFileSync(path.join(directory, 'package.json'), '{}') + const config = { rules: { '*.js': { loaders: ['existing-loader'] } } } + + assert.strictEqual(await withDatadogTurbopack(config, directory), config) + }) + + it('does not require Next.js and preserves existing Turbopack settings', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) + directories.push(directory) + fs.writeFileSync(path.join(directory, 'package.json'), '{}') + const packagePath = createPackageIn(directory, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packagePath, 'index.js', 'module.exports = {}') + + const config = await withDatadogTurbopack({ + resolveAlias: { existing: './existing.js' }, + rules: { '*.js': { loaders: ['existing-loader'] } }, + }, directory) + + assert.equal(config.resolveAlias.existing, './existing.js') + assert.equal(config.rules['*.js'].length, 2) + assert.deepEqual(config.rules['*.js'][0], { loaders: ['existing-loader'] }) + assert.match(config.rules['*.js'][1].condition.all[2].path.source, /node_modules/) + }) +}) + +function createPackage (name, manifest = {}) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) + directories.push(directory) + + return createPackageIn(directory, name, manifest) +} + +function createPackageIn (directory, name, manifest = {}) { + const packagePath = path.join(directory, 'node_modules', name) + fs.mkdirSync(packagePath, { recursive: true }) + fs.writeFileSync(path.join(packagePath, 'package.json'), JSON.stringify({ name, ...manifest })) + return packagePath +} + +function write (directory, relativePath, content) { + const target = path.resolve(directory, relativePath) + fs.mkdirSync(path.dirname(target), { recursive: true }) + fs.writeFileSync(target, content) + return target +} + +function realpath (file) { + return fs.realpathSync(file).replaceAll('\\', '/') +} diff --git a/turbopack.js b/turbopack.js new file mode 100644 index 00000000000..03f10834988 --- /dev/null +++ b/turbopack.js @@ -0,0 +1,3 @@ +'use strict' + +module.exports = require('./packages/datadog-turbopack') From 7ea5d9a0950ef912e005bdf13c36546353a32178 Mon Sep 17 00:00:00 2001 From: William Conti Date: Wed, 26 Aug 2026 16:12:34 -0400 Subject: [PATCH 2/9] fix(turbopack): resolve generated runtime imports --- packages/datadog-turbopack/src/loader.js | 7 +++++- packages/datadog-turbopack/src/targets.js | 6 ++++- .../datadog-turbopack/test/plugin.spec.js | 22 +++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/packages/datadog-turbopack/src/loader.js b/packages/datadog-turbopack/src/loader.js index c025082a5c8..4fb98e40e61 100644 --- a/packages/datadog-turbopack/src/loader.js +++ b/packages/datadog-turbopack/src/loader.js @@ -27,9 +27,14 @@ module.exports = function loader (source) { if (!target) return source + const dcPolyfillPath = relativeImport( + path.dirname(this.resourcePath), + require.resolve('dc-polyfill') + ) + return `${source} ;{ - const __dd_dc = require('dc-polyfill') + const __dd_dc = require(${JSON.stringify(dcPolyfillPath)}) const __dd_ch = __dd_dc.channel('${CHANNEL}') const __dd_payload = { module: module.exports, diff --git a/packages/datadog-turbopack/src/targets.js b/packages/datadog-turbopack/src/targets.js index d0eea18b22f..e360cf6a786 100644 --- a/packages/datadog-turbopack/src/targets.js +++ b/packages/datadog-turbopack/src/targets.js @@ -202,7 +202,11 @@ function findMatchingFiles (directory, pattern) { */ async function createEsmProxy (sourcePath, proxyPath, specifier) { const setters = await processModule({ path: sourcePath, context: { format: 'module' } }) - return `import { register } from 'import-in-the-middle/lib/register.js'; + const registerPath = relativeImport( + path.dirname(proxyPath), + require.resolve('import-in-the-middle/lib/register.js') + ) + return `import { register } from ${JSON.stringify(registerPath)}; import * as namespace from ${JSON.stringify(relativeImport(path.dirname(proxyPath), sourcePath))}; const _ = Object.create(null, { [Symbol.toStringTag]: { value: 'Module' } }); const set = {}; diff --git a/packages/datadog-turbopack/test/plugin.spec.js b/packages/datadog-turbopack/test/plugin.spec.js index dc563f95f00..b34b64a84a0 100644 --- a/packages/datadog-turbopack/test/plugin.spec.js +++ b/packages/datadog-turbopack/test/plugin.spec.js @@ -8,6 +8,7 @@ const { afterEach, describe, it } = require('mocha') const { withDatadogTurbopack } = require('..') const loader = require('../src/loader') +const { createEsmProxy } = require('../src/targets') const directories = [] @@ -76,6 +77,21 @@ describe('datadog-turbopack loader', () => { assert.match(result, /dd-trace:bundler:load/) assert.match(result, /package: "ioredis"/) assert.match(result, /path: "ioredis"/) + assert.ok(result.includes(`require(${JSON.stringify(relativeImport( + path.dirname(resourcePath), require.resolve('dc-polyfill') + ))})`)) + }) + + it('resolves generated ESM proxy dependencies from dd-trace', async () => { + const directory = createPackage('openai', { type: 'module' }) + const resourcePath = write(directory, 'index.mjs', 'export const client = true') + const proxyPath = write(directory, '../.cache/dd-trace/turbopack/openai.mjs', '') + + const result = await createEsmProxy(resourcePath, proxyPath, 'openai') + + assert.ok(result.includes(`from ${JSON.stringify(relativeImport( + path.dirname(proxyPath), require.resolve('import-in-the-middle/lib/register.js') + ))}`)) }) }) @@ -132,3 +148,9 @@ function write (directory, relativePath, content) { function realpath (file) { return fs.realpathSync(file).replaceAll('\\', '/') } + +function relativeImport (from, to) { + let value = path.relative(from, to).replaceAll('\\', '/') + if (!value.startsWith('.')) value = `./${value}` + return value +} From cf04bc039e9d46536b15c2fdd5e154fbf1505b79 Mon Sep 17 00:00:00 2001 From: William Conti Date: Wed, 26 Aug 2026 16:56:07 -0400 Subject: [PATCH 3/9] fix(turbopack): activate bundled ESM integrations --- .../src/helpers/bundler-register.js | 10 +++++--- packages/datadog-turbopack/src/targets.js | 25 +++++++++++++------ .../datadog-turbopack/test/plugin.spec.js | 9 ++++--- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/packages/datadog-instrumentations/src/helpers/bundler-register.js b/packages/datadog-instrumentations/src/helpers/bundler-register.js index 57da69cda5e..afa5d551429 100644 --- a/packages/datadog-instrumentations/src/helpers/bundler-register.js +++ b/packages/datadog-instrumentations/src/helpers/bundler-register.js @@ -104,14 +104,18 @@ dc.subscribe(CHANNEL, (message) => { return } - for (const { file, versions, hook } of instrumentation) { - if (payload.path !== filename(name, file) || !matchVersion(payload.version, versions)) { + for (const { file, filePattern, versions, hook } of instrumentation) { + const matchesFile = payload.path === filename(name, file) || + (filePattern && new RegExp(filename(name, filePattern)).test(payload.path)) + if (!matchesFile || !matchVersion(payload.version, versions)) { continue } try { loadChannel.publish({ name, version: payload.version, file }) - payload.module = hook(payload.module, payload.version) ?? payload.module + const exports = hook(payload.module, payload.version) ?? payload.module + payload.module = exports + payload.apply?.(exports) } catch (e) { log.error('Error executing bundler hook', e) } diff --git a/packages/datadog-turbopack/src/targets.js b/packages/datadog-turbopack/src/targets.js index e360cf6a786..b179744ae95 100644 --- a/packages/datadog-turbopack/src/targets.js +++ b/packages/datadog-turbopack/src/targets.js @@ -4,7 +4,6 @@ const fs = require('node:fs/promises') const fsSync = require('node:fs') const Module = require('node:module') const path = require('node:path') -const { pathToFileURL } = require('node:url') const instrumentations = require('../../datadog-instrumentations/src/helpers/instrumentations') const hooks = require('../../datadog-instrumentations/src/helpers/hooks') @@ -46,7 +45,9 @@ async function createManifest (projectDir) { try { // Proxies are build-time artifacts; preserve source order for stable paths. // eslint-disable-next-line no-await-in-loop - await fs.writeFile(proxyPath, await createEsmProxy(target.path, proxyPath, target.specifier)) + await fs.writeFile(proxyPath, await createEsmProxy( + target.path, proxyPath, target.name, target.specifier, target.version + )) } catch { // An unsupported dependency must not prevent the customer's build. Its // original module remains bundled without instrumentation instead. @@ -197,22 +198,32 @@ function findMatchingFiles (directory, pattern) { /** * @param {string} sourcePath * @param {string} proxyPath + * @param {string} name * @param {string} specifier + * @param {string} version * @returns {Promise} */ -async function createEsmProxy (sourcePath, proxyPath, specifier) { +async function createEsmProxy (sourcePath, proxyPath, name, specifier, version) { const setters = await processModule({ path: sourcePath, context: { format: 'module' } }) - const registerPath = relativeImport( + const dcPolyfillPath = relativeImport( path.dirname(proxyPath), - require.resolve('import-in-the-middle/lib/register.js') + require.resolve('dc-polyfill') ) - return `import { register } from ${JSON.stringify(registerPath)}; + return `import { channel } from ${JSON.stringify(dcPolyfillPath)}; import * as namespace from ${JSON.stringify(relativeImport(path.dirname(proxyPath), sourcePath))}; const _ = Object.create(null, { [Symbol.toStringTag]: { value: 'Module' } }); const set = {}; const get = {}; ${[...setters.values()].join(';\n')}; -register(${JSON.stringify(pathToFileURL(sourcePath).href)}, _, set, get, ${JSON.stringify(specifier)}); +channel('dd-trace:bundler:load').publish({ + package: ${JSON.stringify(name)}, + module: _, + path: ${JSON.stringify(specifier)}, + version: ${JSON.stringify(version)}, + apply (exports) { + for (const name of Object.keys(exports)) set[name]?.(exports[name]); + }, +}); ` } diff --git a/packages/datadog-turbopack/test/plugin.spec.js b/packages/datadog-turbopack/test/plugin.spec.js index b34b64a84a0..4ad7908c60f 100644 --- a/packages/datadog-turbopack/test/plugin.spec.js +++ b/packages/datadog-turbopack/test/plugin.spec.js @@ -82,16 +82,19 @@ describe('datadog-turbopack loader', () => { ))})`)) }) - it('resolves generated ESM proxy dependencies from dd-trace', async () => { + it('publishes generated ESM proxies through the existing bundler channel', async () => { const directory = createPackage('openai', { type: 'module' }) const resourcePath = write(directory, 'index.mjs', 'export const client = true') const proxyPath = write(directory, '../.cache/dd-trace/turbopack/openai.mjs', '') - const result = await createEsmProxy(resourcePath, proxyPath, 'openai') + const result = await createEsmProxy(resourcePath, proxyPath, 'openai', 'openai', '5.0.0') assert.ok(result.includes(`from ${JSON.stringify(relativeImport( - path.dirname(proxyPath), require.resolve('import-in-the-middle/lib/register.js') + path.dirname(proxyPath), require.resolve('dc-polyfill') ))}`)) + assert.match(result, /dd-trace:bundler:load/) + assert.match(result, /apply \(exports\)/) + assert.doesNotMatch(result, /import-in-the-middle/) }) }) From 4433171f56dd9e4c2747d2f54d05d9d6f227b1c4 Mon Sep 17 00:00:00 2001 From: William Conti Date: Wed, 26 Aug 2026 19:15:59 -0400 Subject: [PATCH 4/9] fix(turbopack): isolate bundled ESM instrumentation --- packages/datadog-turbopack/index.js | 32 ++++--- packages/datadog-turbopack/src/loader.js | 36 +++++--- packages/datadog-turbopack/src/targets.js | 35 +++----- .../datadog-turbopack/test/plugin.spec.js | 86 ++++++++++++++++++- 4 files changed, 143 insertions(+), 46 deletions(-) diff --git a/packages/datadog-turbopack/index.js b/packages/datadog-turbopack/index.js index da242883d1a..4d56ff1df5c 100644 --- a/packages/datadog-turbopack/index.js +++ b/packages/datadog-turbopack/index.js @@ -6,8 +6,8 @@ const loader = require.resolve('./src/loader') /** * Adds Datadog instrumentation rules to a Turbopack configuration. Generated - * aliases and loader metadata are local to the application and apply only to - * Node.js bundles. Browser and Edge bundles retain their original modules. + * loader metadata is local to the application and applies only to Node.js + * bundles. Browser and Edge bundles retain their original modules. * * @param {object} [turbopack] * @param {string} [projectDir] @@ -26,34 +26,40 @@ async function addRules (turbopack = {}, projectDir = process.cwd()) { const manifest = await createManifest(projectDir) if (!manifest.packagePathPattern || !manifest.path) return turbopack - const aliases = { ...turbopack.resolveAlias } - - for (const [specifier, alias] of Object.entries(manifest.aliases)) { - // Do not replace application aliases, which would change customer behavior. - aliases[specifier] ??= alias - } - const rules = { ...turbopack.rules } for (const extension of ['*.js', '*.cjs', '*.mjs']) { const existing = rules[extension] - const rule = { + if (hasDatadogLoader(existing)) continue + + const datadogRules = [{ condition: { all: ['foreign', 'node', { path: manifest.packagePathPattern }], }, loaders: [{ loader, options: { manifestPath: manifest.path } }], + }] + if (manifest.esmImportPattern) { + datadogRules.push({ + condition: { + all: ['node', { not: 'foreign' }, { content: manifest.esmImportPattern }], + }, + loaders: [{ loader, options: { manifestPath: manifest.path, rewriteApplicationImports: true } }], + }) } rules[extension] = existing - ? [...(Array.isArray(existing) ? existing : [existing]), rule] - : rule + ? [...(Array.isArray(existing) ? existing : [existing]), ...datadogRules] + : datadogRules.length === 1 ? datadogRules[0] : datadogRules } return { ...turbopack, - resolveAlias: aliases, rules, } } +function hasDatadogLoader (rules) { + return [rules].flat().some(rule => rule?.loaders?.some(item => item?.loader === loader)) +} + module.exports = { withDatadogTurbopack, } diff --git a/packages/datadog-turbopack/src/loader.js b/packages/datadog-turbopack/src/loader.js index 4fb98e40e61..39de994645c 100644 --- a/packages/datadog-turbopack/src/loader.js +++ b/packages/datadog-turbopack/src/loader.js @@ -17,12 +17,14 @@ const CHANNEL = 'dd-trace:bundler:load' * @returns {string} */ module.exports = function loader (source) { - const { manifestPath } = this.getOptions() + const { manifestPath, rewriteApplicationImports } = this.getOptions() const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) const target = manifest.targets[normalizePath(this.resourcePath)] + const esm = isESMFile(this.resourcePath) - if (isESMFile(this.resourcePath)) { - return rewriteImports(source, this.resourcePath, manifest.targets) + if (rewriteApplicationImports || esm) { + const rewritten = rewriteImports(source, this.resourcePath, manifest.targets) + if (rewritten !== source || esm) return rewritten } if (!target) return source @@ -65,16 +67,17 @@ function rewriteImports (source, resourcePath, targets) { matcher.addTransform('rewriteImports', (_state, program) => { visit(program, node => { - if (!isModuleSource(node)) return + const source = getModuleSource(node) + if (!source) return - const resolved = resolveFrom(resourcePath, node.source.value) + const resolved = resolveFrom(resourcePath, source.value) const target = resolved && targets[resolved] if (!target?.esm || !target.proxyPath) return const proxySpecifier = relativeImport(path.dirname(resourcePath), target.proxyPath) // The code transformer emits `raw` when present, so update both fields. - node.source.value = proxySpecifier - node.source.raw = JSON.stringify(proxySpecifier) + source.value = proxySpecifier + source.raw = JSON.stringify(proxySpecifier) rewritten = true }) }) @@ -91,12 +94,23 @@ function rewriteImports (source, resourcePath, targets) { } } -function isModuleSource (node) { - return (node.type === 'ImportDeclaration' || +function getModuleSource (node) { + if (node.type === 'ImportDeclaration' || node.type === 'ExportNamedDeclaration' || node.type === 'ExportAllDeclaration' || - node.type === 'ImportExpression') && - node.source?.type === 'Literal' && typeof node.source.value === 'string' + node.type === 'ImportExpression') { + return isStringLiteral(node.source) && node.source + } + + if (node.type === 'CallExpression' && + node.callee?.type === 'Identifier' && node.callee.name === 'require' && + node.arguments?.length === 1) { + return isStringLiteral(node.arguments[0]) && node.arguments[0] + } +} + +function isStringLiteral (node) { + return node?.type === 'Literal' && typeof node.value === 'string' } function visit (node, callback) { diff --git a/packages/datadog-turbopack/src/targets.js b/packages/datadog-turbopack/src/targets.js index b179744ae95..eb2533b8d74 100644 --- a/packages/datadog-turbopack/src/targets.js +++ b/packages/datadog-turbopack/src/targets.js @@ -17,7 +17,7 @@ const CACHE_DIRECTORY = path.join('node_modules', '.cache', 'dd-trace', 'turbopa * instrumentation targets installed in an application. * * @param {string} projectDir - * @returns {Promise<{ aliases: Record, packagePathPattern?: RegExp, path?: string }>} + * @returns {Promise<{ esmImportPattern?: RegExp, packagePathPattern?: RegExp, path?: string }>} */ async function createManifest (projectDir) { loadInstrumentations() @@ -26,11 +26,11 @@ async function createManifest (projectDir) { const cacheDirectory = path.join(projectDir, CACHE_DIRECTORY) const targets = getTargets(appRequire) const manifestTargets = {} - const aliases = {} - if (targets.length === 0) return { aliases } + if (targets.length === 0) return {} await fs.mkdir(cacheDirectory, { recursive: true }) + const realCacheDirectory = normalizePath(cacheDirectory) for (const [index, target] of targets.entries()) { const entry = { @@ -41,7 +41,7 @@ async function createManifest (projectDir) { } if (target.esm) { - const proxyPath = path.join(cacheDirectory, `${index}.mjs`) + const proxyPath = path.join(realCacheDirectory, `${index}.mjs`) try { // Proxies are build-time artifacts; preserve source order for stable paths. // eslint-disable-next-line no-await-in-loop @@ -54,27 +54,25 @@ async function createManifest (projectDir) { continue } entry.proxyPath = normalizePath(proxyPath) - - if (target.entrypoint || target.specifier !== target.name) { - aliases[target.specifier] = { - browser: relativeImport(projectDir, target.path), - default: relativeImport(projectDir, proxyPath), - } - } } manifestTargets[normalizePath(target.path)] = entry } - const manifestPath = path.join(cacheDirectory, 'manifest.json') + const manifestPath = path.join(realCacheDirectory, 'manifest.json') await fs.writeFile(manifestPath, JSON.stringify({ targets: manifestTargets })) const packageNames = [...new Set(targets.map(target => target.name))] const packagePathPattern = new RegExp( `(?:^|/)node_modules/(?:${packageNames.map(escapeRegExp).join('|')})(?:/|$)` ) + const esmPackageNames = [...new Set(targets.filter(target => target.esm).map(target => target.name))] + const esmPackagePattern = esmPackageNames.map(escapeRegExp).join('|') + const esmImportPattern = esmPackageNames.length > 0 && new RegExp( + String.raw`\b(?:from\s*|import\s*(?:\(\s*)?|require\s*\(\s*)["'](?:${esmPackagePattern})(?:/[^"']*)?["']` + ) - return { aliases, packagePathPattern, path: manifestPath } + return { esmImportPattern, packagePathPattern, path: manifestPath } } /** @@ -91,7 +89,7 @@ function loadInstrumentations () { /** * @param {Function & { resolve: Function }} appRequire * @returns {Array<{ - * esm: boolean, entrypoint: boolean, instrumentationPath: string, name: string, + * esm: boolean, instrumentationPath: string, name: string, * path: string, specifier: string, version: string * }>} */ @@ -134,7 +132,6 @@ function getTargets (appRequire) { targets.set(normalizePath(file), { esm: isESMFile(file, path.join(packageRoot, 'package.json'), packageJson), - entrypoint: !entry.file && !entry.filePattern && samePath(file, entrypoint), instrumentationPath: filename(name, modulePath), name, path: file, @@ -209,13 +206,13 @@ async function createEsmProxy (sourcePath, proxyPath, name, specifier, version) path.dirname(proxyPath), require.resolve('dc-polyfill') ) - return `import { channel } from ${JSON.stringify(dcPolyfillPath)}; + return `import dc from ${JSON.stringify(dcPolyfillPath)}; import * as namespace from ${JSON.stringify(relativeImport(path.dirname(proxyPath), sourcePath))}; const _ = Object.create(null, { [Symbol.toStringTag]: { value: 'Module' } }); const set = {}; const get = {}; ${[...setters.values()].join(';\n')}; -channel('dd-trace:bundler:load').publish({ +dc.channel('dd-trace:bundler:load').publish({ package: ${JSON.stringify(name)}, module: _, path: ${JSON.stringify(specifier)}, @@ -237,10 +234,6 @@ function normalizePath (value) { return fsSync.realpathSync(value).replaceAll('\\', '/') } -function samePath (left, right) { - return normalizePath(left) === normalizePath(right) -} - function escapeRegExp (value) { return value.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`) } diff --git a/packages/datadog-turbopack/test/plugin.spec.js b/packages/datadog-turbopack/test/plugin.spec.js index 4ad7908c60f..568c3dd013c 100644 --- a/packages/datadog-turbopack/test/plugin.spec.js +++ b/packages/datadog-turbopack/test/plugin.spec.js @@ -4,11 +4,12 @@ const assert = require('node:assert/strict') const fs = require('node:fs') const os = require('node:os') const path = require('node:path') +const { pathToFileURL } = require('node:url') const { afterEach, describe, it } = require('mocha') const { withDatadogTurbopack } = require('..') const loader = require('../src/loader') -const { createEsmProxy } = require('../src/targets') +const { createEsmProxy, createManifest } = require('../src/targets') const directories = [] @@ -19,6 +20,21 @@ afterEach(() => { }) describe('datadog-turbopack loader', () => { + it('uses CommonJS-compatible diagnostics channel imports in ESM proxies', async () => { + const directory = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-'))) + directories.push(directory) + const source = write(directory, 'module.mjs', 'export const value = 1') + const proxyPath = path.join(directory, 'proxy.mjs') + const proxy = await createEsmProxy(source, proxyPath, 'ai', 'ai', '7.0.0') + + assert.match(proxy, /import dc from /) + assert.match(proxy, /dc\.channel\('dd-trace:bundler:load'\)/) + assert.doesNotMatch(proxy, /import \{ channel \} from /) + + fs.writeFileSync(proxyPath, proxy) + await import(pathToFileURL(proxyPath).href) + }) + it('routes an internal ESM import through its generated proxy', () => { const directory = createPackage('openai', { type: 'module' }) const client = write(directory, 'client.mjs', "import { Models } from './resources/models.mjs'\nexport { Models }") @@ -45,6 +61,40 @@ describe('datadog-turbopack loader', () => { assert.match(result, /import\("\.\.\/\.cache\/dd-trace\/turbopack\/models\.mjs"\)/) }) + it('routes an application ESM import through its generated proxy', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) + directories.push(directory) + fs.writeFileSync(path.join(directory, 'package.json'), '{}') + const packagePath = createPackageIn(directory, 'ai', { main: 'index.mjs', type: 'module', version: '7.0.0' }) + write(packagePath, 'index.mjs', 'export const generateText = () => {}') + const appPath = write(directory, 'app/route.js', "import { generateText } from 'ai'") + const manifest = await createManifest(directory) + + const result = loader.call({ + getOptions: () => ({ manifestPath: manifest.path, rewriteApplicationImports: true }), + resourcePath: appPath, + }, fs.readFileSync(appPath, 'utf8')) + + assert.match(result, /from "\.\.\/node_modules\/\.cache\/dd-trace\/turbopack\/0\.mjs"/) + }) + + it('routes an application CommonJS require through its generated proxy', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) + directories.push(directory) + fs.writeFileSync(path.join(directory, 'package.json'), '{}') + const packagePath = createPackageIn(directory, 'ai', { main: 'index.mjs', type: 'module', version: '7.0.0' }) + write(packagePath, 'index.mjs', 'export const generateText = () => {}') + const appPath = write(directory, 'pages/api/route.js', "const { generateText } = require('ai')") + const manifest = await createManifest(directory) + + const result = loader.call({ + getOptions: () => ({ manifestPath: manifest.path, rewriteApplicationImports: true }), + resourcePath: appPath, + }, fs.readFileSync(appPath, 'utf8')) + + assert.match(result, /require\("\.\.\/\.\.\/node_modules\/\.cache\/dd-trace\/turbopack\/0\.mjs"\)/) + }) + it('preserves ESM modules without an instrumented import', () => { const directory = createPackage('openai', { type: 'module' }) const client = write(directory, 'client.mjs', "import { Models } from './resources/models.mjs'\nexport { Models }") @@ -125,6 +175,40 @@ describe('datadog-turbopack configuration', () => { assert.deepEqual(config.rules['*.js'][0], { loaders: ['existing-loader'] }) assert.match(config.rules['*.js'][1].condition.all[2].path.source, /node_modules/) }) + + it('does not add the Datadog loader twice when composed repeatedly', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) + directories.push(directory) + fs.writeFileSync(path.join(directory, 'package.json'), '{}') + const packagePath = createPackageIn(directory, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packagePath, 'index.js', 'module.exports = {}') + + const once = await withDatadogTurbopack({}, directory) + const twice = await withDatadogTurbopack(once, directory) + + for (const extension of ['*.js', '*.cjs', '*.mjs']) { + const rules = [twice.rules[extension]].flat() + assert.equal(rules.filter(rule => rule.loaders.some(item => item.loader.includes('datadog-turbopack'))).length, 1) + } + }) + + it('adds a Node-only rule for application ESM imports and requires', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) + directories.push(directory) + fs.writeFileSync(path.join(directory, 'package.json'), '{}') + const packagePath = createPackageIn(directory, 'ai', { main: 'index.mjs', type: 'module', version: '7.0.0' }) + write(packagePath, 'index.mjs', 'export const generateText = () => {}') + + const config = await withDatadogTurbopack({}, directory) + const rules = [config.rules['*.js']].flat() + const applicationRule = rules.find(rule => rule.condition.all.some(condition => condition?.not === 'foreign')) + + assert.deepEqual(applicationRule.condition.all.slice(0, 2), ['node', { not: 'foreign' }]) + assert.match('import { generateText } from "ai"', applicationRule.condition.all[2].content) + assert.match('const { generateText } = require("ai")', applicationRule.condition.all[2].content) + assert.doesNotMatch('import { something } from "unrelated"', applicationRule.condition.all[2].content) + assert.equal(applicationRule.loaders[0].options.rewriteApplicationImports, true) + }) }) function createPackage (name, manifest = {}) { From 296736837078158618ab49939f8fae2c91820a20 Mon Sep 17 00:00:00 2001 From: William Conti Date: Wed, 26 Aug 2026 20:50:50 -0400 Subject: [PATCH 5/9] feat(turbopack): instrument bundled dependencies --- package.json | 1 + .../src/helpers/bundler-register.js | 17 +- .../src/helpers/instrumentation-utils.js | 51 ++++ .../src/helpers/register.js | 53 +--- .../test/helpers/bundler-register.spec.js | 150 +++++++++++ packages/datadog-turbopack/index.js | 23 +- packages/datadog-turbopack/src/loader.js | 58 +++- packages/datadog-turbopack/src/targets.js | 255 +++++++++++++++--- .../datadog-turbopack/test/plugin.spec.js | 122 ++++++++- turbopack.d.ts | 12 + 10 files changed, 638 insertions(+), 104 deletions(-) create mode 100644 packages/datadog-instrumentations/src/helpers/instrumentation-utils.js create mode 100644 packages/datadog-instrumentations/test/helpers/bundler-register.spec.js create mode 100644 turbopack.d.ts diff --git a/package.json b/package.json index ae2254bf90f..a4171e3013a 100644 --- a/package.json +++ b/package.json @@ -153,6 +153,7 @@ "cypress/**/*", "esbuild.js", "turbopack.js", + "turbopack.d.ts", "webpack.js", "ext/**/*", "index.d.ts", diff --git a/packages/datadog-instrumentations/src/helpers/bundler-register.js b/packages/datadog-instrumentations/src/helpers/bundler-register.js index afa5d551429..12fa55264ca 100644 --- a/packages/datadog-instrumentations/src/helpers/bundler-register.js +++ b/packages/datadog-instrumentations/src/helpers/bundler-register.js @@ -9,8 +9,11 @@ const { loadChannel, matchVersion, } = require('./register.js') +const { getDisabledInstrumentations } = require('./instrumentation-utils') const hooks = require('./hooks') const instrumentations = require('./instrumentations') +const { isRelativeRequire } = require('./shared-utils') +const disabledInstrumentations = getDisabledInstrumentations() // register.js has now set up ritm (require-in-the-middle). In bundled // environments (webpack, esbuild), Node.js built-in modules required by @@ -81,6 +84,7 @@ const instrumentedNodeModules = new Set() dc.subscribe(CHANNEL, (message) => { const payload = /** @type {Payload} */ (message) const name = payload.package + if (disabledInstrumentations.has(name)) return const isPrefixedWithNode = name.startsWith('node:') @@ -104,8 +108,8 @@ dc.subscribe(CHANNEL, (message) => { return } - for (const { file, filePattern, versions, hook } of instrumentation) { - const matchesFile = payload.path === filename(name, file) || + for (const { file, filePattern, patchDefault, versions, hook } of instrumentation) { + const matchesFile = isRelativeRequire(name) || payload.path === filename(name, file) || (filePattern && new RegExp(filename(name, filePattern)).test(payload.path)) if (!matchesFile || !matchVersion(payload.version, versions)) { continue @@ -113,9 +117,14 @@ dc.subscribe(CHANNEL, (message) => { try { loadChannel.publish({ name, version: payload.version, file }) - const exports = hook(payload.module, payload.version) ?? payload.module + let exports = payload.module + if (patchDefault === !!exports.default) { + if (patchDefault) exports = exports.default + else continue + } + exports = hook(exports, payload.version) ?? exports payload.module = exports - payload.apply?.(exports) + payload.apply?.(exports, patchDefault) } catch (e) { log.error('Error executing bundler hook', e) } diff --git a/packages/datadog-instrumentations/src/helpers/instrumentation-utils.js b/packages/datadog-instrumentations/src/helpers/instrumentation-utils.js new file mode 100644 index 00000000000..c0504a1a4bc --- /dev/null +++ b/packages/datadog-instrumentations/src/helpers/instrumentation-utils.js @@ -0,0 +1,51 @@ +'use strict' + +const { builtinModules } = require('node:module') + +const satisfies = require('../../../../vendor/dist/semifies') +const { getValueFromEnvSources } = require('../../../dd-trace/src/config/helper') + +/** + * @param {string|undefined} version + * @param {string[]|undefined} ranges + * @returns {boolean} + */ +function matchVersion (version, ranges) { + return !version || !ranges || ranges.some(range => satisfies(version, range)) +} + +/** + * @param {string} name + * @param {string} [file] + * @returns {string} + */ +function filename (name, file) { + return file ? `${name}/${file}` : name +} + +/** + * @returns {Set} + */ +function getDisabledInstrumentations () { + const disabled = new Set( + getValueFromEnvSources('DD_TRACE_DISABLED_INSTRUMENTATIONS')?.split(',').filter(Boolean) + ) + const expanded = new Set(disabled) + const builtins = new Set(builtinModules) + + for (const name of disabled) { + const prefixed = name.startsWith('node:') + if (!prefixed && !builtins.has(name)) continue + + const counterpart = prefixed ? name.slice(5) : `node:${name}` + expanded.add(counterpart) + } + + return expanded +} + +module.exports = { + filename, + getDisabledInstrumentations, + matchVersion, +} diff --git a/packages/datadog-instrumentations/src/helpers/register.js b/packages/datadog-instrumentations/src/helpers/register.js index a1f9277371f..adc6789421e 100644 --- a/packages/datadog-instrumentations/src/helpers/register.js +++ b/packages/datadog-instrumentations/src/helpers/register.js @@ -1,20 +1,21 @@ 'use strict' -const { builtinModules } = require('module') const path = require('path') const { channel } = require('dc-polyfill') -const satisfies = require('../../../../vendor/dist/semifies') const log = require('../../../dd-trace/src/log') const telemetry = require('../../../dd-trace/src/guardrails/telemetry') const { IS_SERVERLESS } = require('../../../dd-trace/src/serverless') const { getValueFromEnvSources } = require('../../../dd-trace/src/config/helper') const checkRequireCache = require('./check-require-cache') const Hook = require('./hook') +const { + filename, + getDisabledInstrumentations, + matchVersion, +} = require('./instrumentation-utils') const { isRelativeRequire } = require('./shared-utils') const rewriter = require('./rewriter') -const DD_TRACE_DISABLED_INSTRUMENTATIONS = - getValueFromEnvSources('DD_TRACE_DISABLED_INSTRUMENTATIONS') const DD_TRACE_DEBUG = getValueFromEnvSources('DD_TRACE_DEBUG') const hooks = require('./hooks') @@ -22,9 +23,7 @@ const instrumentations = require('./instrumentations') const names = Object.keys(hooks) const pathSepExpr = new RegExp(`\\${path.sep}`, 'g') -const disabledInstrumentations = new Set( - DD_TRACE_DISABLED_INSTRUMENTATIONS?.split(',') -) +const disabledInstrumentations = getDisabledInstrumentations() const loadChannel = channel('dd-trace:instrumentation:load') @@ -54,29 +53,6 @@ const instrumentedIntegrationsSuccess = new Map() /** @type {Set} */ const alreadyLoggedIncompatibleIntegrations = new Set() -// Always disable prefixed and unprefixed node modules if one is disabled. -if (disabledInstrumentations.size) { - const builtinsSet = new Set(builtinModules) - const disabledBuiltinCounterparts = [] - for (const name of disabledInstrumentations) { - const hasPrefix = name.startsWith('node:') - if (hasPrefix || builtinsSet.has(name)) { - if (hasPrefix) { - const unprefixedName = name.slice(5) - if (!disabledInstrumentations.has(unprefixedName)) { - disabledBuiltinCounterparts.push(unprefixedName) - } - } else if (!disabledInstrumentations.has(`node:${name}`)) { - disabledBuiltinCounterparts.push(`node:${name}`) - } - } - } - for (const name of disabledBuiltinCounterparts) { - disabledInstrumentations.add(name) - } - builtinsSet.clear() -} - for (const name of names) { if (disabledInstrumentations.has(name)) continue @@ -188,23 +164,6 @@ function logAbortedIntegrations () { instrumentedIntegrationsSuccess.clear() } -/** - * @param {string|undefined} version - * @param {string[]|undefined} ranges - */ -function matchVersion (version, ranges) { - return !version || !ranges || ranges.some(range => satisfies(version, range)) -} - -/** - * @param {string} name - * @param {string} [file] - * @returns {string} - */ -function filename (name, file) { - return file ? `${name}/${file}` : name -} - module.exports = { filename, pathSepExpr, diff --git a/packages/datadog-instrumentations/test/helpers/bundler-register.spec.js b/packages/datadog-instrumentations/test/helpers/bundler-register.spec.js new file mode 100644 index 00000000000..68f2472fa04 --- /dev/null +++ b/packages/datadog-instrumentations/test/helpers/bundler-register.spec.js @@ -0,0 +1,150 @@ +'use strict' + +const Module = require('node:module') + +const sinon = require('sinon') + +const CHANNEL = 'dd-trace:bundler:load' + +describe('bundler register', () => { + let originalRequire + + beforeEach(() => { + originalRequire = Module.prototype.require + }) + + afterEach(() => { + Module.prototype.require = originalRequire + sinon.restore() + }) + + it('honors patchDefault when applying an ESM proxy update', () => { + const Original = class Original {} + const Patched = class Patched {} + const hook = sinon.stub().returns(Patched) + const apply = sinon.stub() + const { loadChannel, publish } = loadBundlerRegister({ + hooks: { 'test-default-export': sinon.stub() }, + instrumentations: { + 'test-default-export': [{ file: 'index.mjs', hook, patchDefault: true }], + }, + }) + + publish({ + apply, + module: { default: Original }, + package: 'test-default-export', + path: 'test-default-export/index.mjs', + version: '1.0.0', + }) + + sinon.assert.calledOnceWithExactly(hook, Original, '1.0.0') + sinon.assert.calledOnceWithExactly(apply, Patched, true) + sinon.assert.calledOnceWithExactly(loadChannel.publish, { + file: 'index.mjs', + name: 'test-default-export', + version: '1.0.0', + }) + }) + + it('does not activate explicitly disabled bundled integrations', () => { + const hook = sinon.stub() + const integrationHook = sinon.stub() + const { publish } = loadBundlerRegister({ + disabled: new Set(['test-disabled-integration']), + hooks: { 'test-disabled-integration': hook }, + instrumentations: { + 'test-disabled-integration': [{ hook: integrationHook }], + }, + }) + + publish({ + module: {}, + package: 'test-disabled-integration', + path: 'test-disabled-integration', + version: '1.0.0', + }) + + sinon.assert.notCalled(hook) + sinon.assert.notCalled(integrationHook) + }) + + it('matches bundled file-pattern hooks', () => { + const hook = sinon.stub() + const integrationHook = sinon.stub() + const { publish } = loadBundlerRegister({ + hooks: { 'test-pattern-hook': hook }, + instrumentations: { + 'test-pattern-hook': [{ filePattern: 'dist/cli.*', hook: integrationHook }], + }, + }) + + publish({ + module: {}, + package: 'test-pattern-hook', + path: 'test-pattern-hook/dist/cli-123.js', + version: '1.0.0', + }) + + sinon.assert.calledOnceWithExactly(integrationHook, {}, '1.0.0') + }) + + it('matches bundled relative-module hooks', () => { + const hook = sinon.stub() + const integrationHook = sinon.stub() + const { publish } = loadBundlerRegister({ + hooks: { './runtime/library.js': hook }, + instrumentations: { + './runtime/library.js': [{ file: 'runtime/library.js', hook: integrationHook }], + }, + }) + + publish({ + module: {}, + package: './runtime/library.js', + path: './runtime/library.js', + version: '6.1.0', + }) + + sinon.assert.calledOnceWithExactly(integrationHook, {}, '6.1.0') + }) +}) + +function loadBundlerRegister ({ disabled = new Set(), hooks, instrumentations }) { + const bundlerRegisterPath = require.resolve('../../src/helpers/bundler-register') + const originalRequire = Module.prototype.require + const loadChannel = { publish: sinon.stub() } + let bundledModuleSubscriber + const register = { + filename: (name, file) => file ? `${name}/${file}` : name, + loadChannel, + matchVersion: () => true, + } + + Module.prototype.require = function (request) { + if (this.filename === bundlerRegisterPath) { + const stubs = { + './hooks': hooks, + './instrumentation-utils': { getDisabledInstrumentations: () => disabled }, + './instrumentations': instrumentations, + './register.js': register, + '../../../dd-trace/src/log': { error: sinon.stub() }, + 'dc-polyfill': { + subscribe: (channel, callback) => { + if (channel === CHANNEL) bundledModuleSubscriber = callback + }, + }, + } + return stubs[request] || originalRequire.call(this, request) + } + return originalRequire.call(this, request) + } + delete require.cache[bundlerRegisterPath] + require('../../src/helpers/bundler-register') + Module.prototype.require = originalRequire + + return { + loadChannel, + publish: message => bundledModuleSubscriber(message), + } +} diff --git a/packages/datadog-turbopack/index.js b/packages/datadog-turbopack/index.js index 4d56ff1df5c..f8023dd2cbc 100644 --- a/packages/datadog-turbopack/index.js +++ b/packages/datadog-turbopack/index.js @@ -27,7 +27,8 @@ async function addRules (turbopack = {}, projectDir = process.cwd()) { if (!manifest.packagePathPattern || !manifest.path) return turbopack const rules = { ...turbopack.rules } - for (const extension of ['*.js', '*.cjs', '*.mjs']) { + const aliases = Object.keys(turbopack.resolveAlias ?? {}) + for (const extension of ['*.js', '*.cjs', '*.mjs', '*.jsx', '*.ts', '*.tsx']) { const existing = rules[extension] if (hasDatadogLoader(existing)) continue @@ -35,14 +36,30 @@ async function addRules (turbopack = {}, projectDir = process.cwd()) { condition: { all: ['foreign', 'node', { path: manifest.packagePathPattern }], }, - loaders: [{ loader, options: { manifestPath: manifest.path } }], + loaders: [{ loader, options: { manifestHash: manifest.hash, manifestPath: manifest.path } }], }] if (manifest.esmImportPattern) { datadogRules.push({ condition: { all: ['node', { not: 'foreign' }, { content: manifest.esmImportPattern }], }, - loaders: [{ loader, options: { manifestPath: manifest.path, rewriteApplicationImports: true } }], + loaders: [{ + loader, + options: { + aliases, + manifestHash: manifest.hash, + manifestPath: manifest.path, + rewriteApplicationImports: true, + }, + }], + }) + } + if (manifest.relativePathPattern) { + datadogRules.push({ + condition: { + all: ['node', { not: 'foreign' }, { path: manifest.relativePathPattern }], + }, + loaders: [{ loader, options: { manifestHash: manifest.hash, manifestPath: manifest.path } }], }) } rules[extension] = existing diff --git a/packages/datadog-turbopack/src/loader.js b/packages/datadog-turbopack/src/loader.js index 39de994645c..34596b67681 100644 --- a/packages/datadog-turbopack/src/loader.js +++ b/packages/datadog-turbopack/src/loader.js @@ -17,13 +17,14 @@ const CHANNEL = 'dd-trace:bundler:load' * @returns {string} */ module.exports = function loader (source) { - const { manifestPath, rewriteApplicationImports } = this.getOptions() + const { aliases, manifestPath, rewriteApplicationImports } = this.getOptions() const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) - const target = manifest.targets[normalizePath(this.resourcePath)] + const target = manifest.targets[normalizePath(this.resourcePath)] || + getRelativeTarget(this.resourcePath, manifest.relativeTargets) const esm = isESMFile(this.resourcePath) if (rewriteApplicationImports || esm) { - const rewritten = rewriteImports(source, this.resourcePath, manifest.targets) + const rewritten = rewriteImports(source, this.resourcePath, manifest.targets, aliases) if (rewritten !== source || esm) return rewritten } @@ -54,9 +55,10 @@ module.exports = function loader (source) { * @param {string} source * @param {string} resourcePath * @param {Record} targets + * @param {string[]} [aliases] * @returns {string} */ -function rewriteImports (source, resourcePath, targets) { +function rewriteImports (source, resourcePath, targets, aliases = []) { resourcePath = normalizePath(resourcePath) let rewritten = false const matcher = create([{ @@ -66,9 +68,11 @@ function rewriteImports (source, resourcePath, targets) { }]) matcher.addTransform('rewriteImports', (_state, program) => { + const hasLocalRequire = declaresRequire(program) visit(program, node => { - const source = getModuleSource(node) - if (!source) return + const source = getModuleSource(node, hasLocalRequire) + + if (!source || matchesAlias(source.value, aliases)) return const resolved = resolveFrom(resourcePath, source.value) const target = resolved && targets[resolved] @@ -94,7 +98,7 @@ function rewriteImports (source, resourcePath, targets) { } } -function getModuleSource (node) { +function getModuleSource (node, hasLocalRequire) { if (node.type === 'ImportDeclaration' || node.type === 'ExportNamedDeclaration' || node.type === 'ExportAllDeclaration' || @@ -105,10 +109,48 @@ function getModuleSource (node) { if (node.type === 'CallExpression' && node.callee?.type === 'Identifier' && node.callee.name === 'require' && node.arguments?.length === 1) { - return isStringLiteral(node.arguments[0]) && node.arguments[0] + return !hasLocalRequire && isStringLiteral(node.arguments[0]) && node.arguments[0] } } +function matchesAlias (specifier, aliases) { + return aliases.some(alias => specifier === alias || specifier.startsWith(`${alias}/`)) +} + +function getRelativeTarget (resourcePath, targets = []) { + const normalizedPath = normalizePath(resourcePath) + return targets.find(target => normalizedPath.endsWith(`/${target.file}`)) +} + +// We deliberately decline all CommonJS rewrites in a file with a lexical +// `require` binding. Rewriting an application-defined function is worse than +// leaving an uncommon module load uninstrumented. +function declaresRequire (node) { + let declared = false + visit(node, child => { + if (child.type === 'VariableDeclarator' || child.type === 'CatchClause') { + declared ||= bindingIncludesRequire(child.id ?? child.param) + } else if (child.type === 'FunctionDeclaration' || + child.type === 'FunctionExpression' || child.type === 'ArrowFunctionExpression') { + declared ||= child.params.some(bindingIncludesRequire) || child.id?.name === 'require' + } else if (child.type === 'ImportSpecifier' || child.type === 'ImportDefaultSpecifier' || + child.type === 'ImportNamespaceSpecifier') { + declared ||= child.local?.name === 'require' + } else if (child.type === 'ClassDeclaration') { + declared ||= child.id?.name === 'require' + } + }) + return declared +} + +function bindingIncludesRequire (node) { + if (!node || typeof node !== 'object') return false + if (node.type === 'Identifier') return node.name === 'require' + return Object.values(node).some(value => Array.isArray(value) + ? value.some(bindingIncludesRequire) + : bindingIncludesRequire(value)) +} + function isStringLiteral (node) { return node?.type === 'Literal' && typeof node.value === 'string' } diff --git a/packages/datadog-turbopack/src/targets.js b/packages/datadog-turbopack/src/targets.js index eb2533b8d74..0ac7424e204 100644 --- a/packages/datadog-turbopack/src/targets.js +++ b/packages/datadog-turbopack/src/targets.js @@ -4,10 +4,15 @@ const fs = require('node:fs/promises') const fsSync = require('node:fs') const Module = require('node:module') const path = require('node:path') +const { createHash } = require('node:crypto') const instrumentations = require('../../datadog-instrumentations/src/helpers/instrumentations') const hooks = require('../../datadog-instrumentations/src/helpers/hooks') -const { filename, matchVersion } = require('../../datadog-instrumentations/src/helpers/register') +const { + filename, + getDisabledInstrumentations, + matchVersion, +} = require('../../datadog-instrumentations/src/helpers/instrumentation-utils') const { isESMFile, processModule } = require('../../datadog-esbuild/src/utils') const CACHE_DIRECTORY = path.join('node_modules', '.cache', 'dd-trace', 'turbopack') @@ -17,19 +22,26 @@ const CACHE_DIRECTORY = path.join('node_modules', '.cache', 'dd-trace', 'turbopa * instrumentation targets installed in an application. * * @param {string} projectDir - * @returns {Promise<{ esmImportPattern?: RegExp, packagePathPattern?: RegExp, path?: string }>} + * @returns {Promise<{ esmImportPattern?: RegExp, hash?: string, packagePathPattern?: RegExp, path?: string }>} */ async function createManifest (projectDir) { + projectDir = path.resolve(projectDir) + const disabledInstrumentations = getDisabledInstrumentations() loadInstrumentations() const appRequire = Module.createRequire(path.join(projectDir, 'package.json')) const cacheDirectory = path.join(projectDir, CACHE_DIRECTORY) - const targets = getTargets(appRequire) + const targets = getTargets(appRequire, disabledInstrumentations, projectDir) + const relativeTargets = getRelativeTargets(targets, disabledInstrumentations) const manifestTargets = {} - if (targets.length === 0) return {} + if (targets.length === 0 && relativeTargets.length === 0) return {} - await fs.mkdir(cacheDirectory, { recursive: true }) + try { + await fs.mkdir(cacheDirectory, { recursive: true }) + } catch { + return {} + } const realCacheDirectory = normalizePath(cacheDirectory) for (const [index, target] of targets.entries()) { @@ -60,7 +72,12 @@ async function createManifest (projectDir) { } const manifestPath = path.join(realCacheDirectory, 'manifest.json') - await fs.writeFile(manifestPath, JSON.stringify({ targets: manifestTargets })) + const manifest = JSON.stringify({ relativeTargets, targets: manifestTargets }) + try { + await fs.writeFile(manifestPath, manifest) + } catch { + return {} + } const packageNames = [...new Set(targets.map(target => target.name))] const packagePathPattern = new RegExp( @@ -72,7 +89,17 @@ async function createManifest (projectDir) { String.raw`\b(?:from\s*|import\s*(?:\(\s*)?|require\s*\(\s*)["'](?:${esmPackagePattern})(?:/[^"']*)?["']` ) - return { esmImportPattern, packagePathPattern, path: manifestPath } + const relativePathPattern = relativeTargets.length > 0 && new RegExp( + `(?:^|/)(?:${relativeTargets.map(target => escapeRegExp(target.file)).join('|')})$` + ) + + return { + esmImportPattern, + hash: createHash('sha256').update(manifest).digest('hex'), + packagePathPattern, + path: manifestPath, + relativePathPattern, + } } /** @@ -80,7 +107,9 @@ async function createManifest (projectDir) { * shared registry before we inspect it at build time. */ function loadInstrumentations () { - for (const hook of Object.values(hooks)) { + const disabledInstrumentations = getDisabledInstrumentations() + for (const [name, hook] of Object.entries(hooks)) { + if (disabledInstrumentations.has(name)) continue const load = hook?.fn ?? hook if (typeof load === 'function') load() } @@ -88,61 +117,207 @@ function loadInstrumentations () { /** * @param {Function & { resolve: Function }} appRequire + * @param {Set} [disabledInstrumentations] + * @param {string} [projectDir] * @returns {Array<{ * esm: boolean, instrumentationPath: string, name: string, * path: string, specifier: string, version: string * }>} */ -function getTargets (appRequire) { +function getTargets (appRequire, disabledInstrumentations = new Set(), projectDir) { const targets = new Map() + const packageNames = new Set( + Object.keys(instrumentations).filter(name => !name.startsWith('node:') && !name.startsWith('.')) + ) + const packageRootsByName = projectDir ? findPackageRoots(projectDir, packageNames) : new Map() for (const [name, entries] of Object.entries(instrumentations)) { - if (name.startsWith('node:') || name.startsWith('.')) continue + if (name.startsWith('node:') || name.startsWith('.') || disabledInstrumentations.has(name)) continue + + const packageRoots = [...(packageRootsByName.get(name) ?? [])] + if (packageRoots.length === 0) { + try { + const entrypoint = resolveImport(appRequire, name) + const packageRoot = findPackageRoot(entrypoint) + if (packageRoot) packageRoots.push(packageRoot) + } catch { + continue + } + } + + for (const packageRoot of packageRoots) { + addTargets(targets, packageRoot, name, entries) + } + } + + return [...targets.values()] +} - let entrypoint +function getRelativeTargets (targets, disabledInstrumentations) { + const relativeTargets = [] + + for (const [name, entries] of Object.entries(instrumentations)) { + if (!name.startsWith('.') || disabledInstrumentations.has(name)) continue + + for (const entry of entries) { + if (!entry.file) continue + const compatibleTarget = targets.find(target => matchVersion(target.version, entry.versions) && + instrumentations[target.name]?.some(candidate => + candidate.hook === entry.hook && matchVersion(target.version, candidate.versions) + ) + ) + if (!compatibleTarget) continue + + relativeTargets.push({ + file: entry.file, + name, + path: name, + version: compatibleTarget.version, + }) + } + } + + return relativeTargets +} + +function addTargets (targets, packageRoot, name, entries) { + let packageJson + let entrypoint + try { + packageJson = JSON.parse(fsSync.readFileSync(path.join(packageRoot, 'package.json'), 'utf8')) + entrypoint = resolvePackageEntrypoint(packageRoot, name) + } catch { + return + } + + for (const entry of entries) { + if (!matchVersion(packageJson.version, entry.versions)) continue + + let files try { - entrypoint = resolveImport(appRequire, name) + files = entry.file + ? [path.join(packageRoot, entry.file)] + : entry.filePattern + ? findMatchingFiles(packageRoot, new RegExp(entry.filePattern)) + : [entrypoint] } catch { continue } - const packageRoot = findPackageRoot(entrypoint) - if (!packageRoot) continue + for (const file of files) { + if (!fsSync.existsSync(file)) continue + const modulePath = entry.file || (entry.filePattern && + path.relative(packageRoot, file).replaceAll('\\', '/')) + + targets.set(normalizePath(file), { + esm: isESMFile(file, path.join(packageRoot, 'package.json'), packageJson), + instrumentationPath: filename(name, modulePath), + name, + path: file, + specifier: modulePath ? `${name}/${modulePath}` : name, + version: packageJson.version, + }) + } + } +} + +// Visit package boundaries only: this reaches nested dependency copies without +// walking each package's source tree during Next configuration. +function findPackageRoots (projectDir, names) { + const packageRoots = new Map() + const pending = [path.join(projectDir, 'node_modules')] + const seen = new Set() - let packageJson + while (pending.length > 0) { + const nodeModules = pending.pop() + let directory try { - packageJson = JSON.parse(fsSync.readFileSync(path.join(packageRoot, 'package.json'), 'utf8')) + directory = normalizePath(nodeModules) } catch { continue } + if (seen.has(directory)) continue + seen.add(directory) + let entries + try { + entries = fsSync.readdirSync(directory, { withFileTypes: true }) + } catch { + continue + } for (const entry of entries) { - if (!matchVersion(packageJson.version, entry.versions)) continue - - const files = entry.file - ? [path.join(packageRoot, entry.file)] - : entry.filePattern - ? findMatchingFiles(packageRoot, new RegExp(entry.filePattern)) - : [entrypoint] - - for (const file of files) { - if (!fsSync.existsSync(file)) continue - const modulePath = entry.file || (entry.filePattern && - path.relative(packageRoot, file).replaceAll('\\', '/')) - - targets.set(normalizePath(file), { - esm: isESMFile(file, path.join(packageRoot, 'package.json'), packageJson), - instrumentationPath: filename(name, modulePath), - name, - path: file, - specifier: modulePath ? `${name}/${modulePath}` : name, - version: packageJson.version, - }) + if (entry.name === '.bin') continue + if (!entry.isDirectory() && !entry.isSymbolicLink()) continue + const entryPath = path.join(directory, entry.name) + if (entry.name.startsWith('@')) { + addScopedPackageRoots(entryPath, names, packageRoots, pending) + } else { + addPackageRoot(entryPath, entry.name, names, packageRoots, pending) } } } - return [...targets.values()] + return packageRoots +} + +function addScopedPackageRoots (scopePath, names, packageRoots, pending) { + let entries + try { + entries = fsSync.readdirSync(scopePath, { withFileTypes: true }) + } catch { + return + } + for (const entry of entries) { + if (!entry.isDirectory() && !entry.isSymbolicLink()) continue + addPackageRoot( + path.join(scopePath, entry.name), + `${path.basename(scopePath)}/${entry.name}`, + names, + packageRoots, + pending + ) + } +} + +function addPackageRoot (packageRoot, packageName, names, packageRoots, pending) { + let realPackageRoot + try { + realPackageRoot = normalizePath(packageRoot) + } catch { + return + } + + if (names.has(packageName)) { + const roots = packageRoots.get(packageName) ?? new Set() + roots.add(realPackageRoot) + packageRoots.set(packageName, roots) + } + pending.push(path.join(realPackageRoot, 'node_modules')) +} + +function createPackageRequire (packageRoot) { + const nodeModules = findNodeModulesRoot(packageRoot) + return Module.createRequire(path.join( + nodeModules ? path.dirname(nodeModules) : packageRoot, + 'package.json' + )) +} + +function findNodeModulesRoot (packageRoot) { + let directory = packageRoot + while (directory !== path.dirname(directory)) { + if (path.basename(directory) === 'node_modules') return directory + directory = path.dirname(directory) + } +} + +function resolvePackageEntrypoint (packageRoot, name) { + const packageRequire = createPackageRequire(packageRoot) + try { + return resolveImport(packageRequire, name) + } catch { + return resolveImport(Module.createRequire(path.join(packageRoot, 'package.json')), '.') + } } /** @@ -217,7 +392,8 @@ dc.channel('dd-trace:bundler:load').publish({ module: _, path: ${JSON.stringify(specifier)}, version: ${JSON.stringify(version)}, - apply (exports) { + apply (exports, patchDefault) { + if (patchDefault) return set.default?.(exports); for (const name of Object.keys(exports)) set[name]?.(exports[name]); }, }); @@ -241,5 +417,6 @@ function escapeRegExp (value) { module.exports = { createEsmProxy, createManifest, + getRelativeTargets, getTargets, } diff --git a/packages/datadog-turbopack/test/plugin.spec.js b/packages/datadog-turbopack/test/plugin.spec.js index 568c3dd013c..054f0bcfb63 100644 --- a/packages/datadog-turbopack/test/plugin.spec.js +++ b/packages/datadog-turbopack/test/plugin.spec.js @@ -9,7 +9,7 @@ const { afterEach, describe, it } = require('mocha') const { withDatadogTurbopack } = require('..') const loader = require('../src/loader') -const { createEsmProxy, createManifest } = require('../src/targets') +const { createEsmProxy, createManifest, getRelativeTargets } = require('../src/targets') const directories = [] @@ -95,6 +95,34 @@ describe('datadog-turbopack loader', () => { assert.match(result, /require\("\.\.\/\.\.\/node_modules\/\.cache\/dd-trace\/turbopack\/0\.mjs"\)/) }) + it('does not rewrite an application-defined require function', () => { + const directory = createPackage('ai', { main: 'index.mjs', type: 'module' }) + const target = write(directory, 'index.mjs', 'export const generateText = () => {}') + const proxy = write(directory, '../.cache/dd-trace/turbopack/ai.mjs', 'export {}') + const appPath = write(path.dirname(path.dirname(directory)), 'route.js', '') + const source = "function load (require) { return require('ai') }" + + const result = loader.rewriteImports(source, appPath, { + [realpath(target)]: { esm: true, proxyPath: realpath(proxy) }, + }) + + assert.equal(result, source) + }) + + it('preserves configured aliases for instrumented packages', () => { + const directory = createPackage('ai', { main: 'index.mjs', type: 'module' }) + const target = write(directory, 'index.mjs', 'export const generateText = () => {}') + const proxy = write(directory, '../.cache/dd-trace/turbopack/ai.mjs', 'export {}') + const appPath = write(path.dirname(path.dirname(directory)), 'route.js', '') + const source = "import { generateText } from 'ai'" + + const result = loader.rewriteImports(source, appPath, { + [realpath(target)]: { esm: true, proxyPath: realpath(proxy) }, + }, ['ai']) + + assert.equal(result, source) + }) + it('preserves ESM modules without an instrumented import', () => { const directory = createPackage('openai', { type: 'module' }) const client = write(directory, 'client.mjs', "import { Models } from './resources/models.mjs'\nexport { Models }") @@ -132,6 +160,29 @@ describe('datadog-turbopack loader', () => { ))})`)) }) + it('publishes supported relative runtime modules through the bundler channel', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) + directories.push(directory) + const resourcePath = write(directory, 'generated/prisma/runtime/library.js', 'module.exports = {}') + const manifestPath = write(directory, 'manifest.json', JSON.stringify({ + relativeTargets: [{ + file: 'runtime/library.js', + name: './runtime/library.js', + path: './runtime/library.js', + version: '6.1.0', + }], + targets: {}, + })) + + const result = loader.call({ + getOptions: () => ({ manifestPath }), + resourcePath, + }, 'module.exports = {}') + + assert.match(result, /package: "\.\/runtime\/library\.js"/) + assert.match(result, /path: "\.\/runtime\/library\.js"/) + }) + it('publishes generated ESM proxies through the existing bundler channel', async () => { const directory = createPackage('openai', { type: 'module' }) const resourcePath = write(directory, 'index.mjs', 'export const client = true') @@ -143,12 +194,61 @@ describe('datadog-turbopack loader', () => { path.dirname(proxyPath), require.resolve('dc-polyfill') ))}`)) assert.match(result, /dd-trace:bundler:load/) - assert.match(result, /apply \(exports\)/) + assert.match(result, /apply \(exports, patchDefault\)/) + assert.match(result, /set\.default\?\.\(exports\)/) assert.doesNotMatch(result, /import-in-the-middle/) }) }) describe('datadog-turbopack configuration', () => { + it('limits relative runtime rules to compatible package versions', () => { + require('../../datadog-instrumentations/src/prisma') + + const supported = getRelativeTargets([{ name: '@prisma/client', version: '6.1.0' }], new Set()) + const unsupported = getRelativeTargets([{ name: '@prisma/client', version: '7.0.0' }], new Set()) + + assert.deepEqual(supported, [{ + file: 'runtime/library.js', + name: './runtime/library.js', + path: './runtime/library.js', + version: '6.1.0', + }]) + assert.deepEqual(unsupported, []) + }) + + it('discovers nested copies of supported packages', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) + directories.push(directory) + fs.writeFileSync(path.join(directory, 'package.json'), '{}') + const nested = createPackageIn(directory, 'parent/node_modules/ioredis', { + main: 'index.js', + version: '5.0.0', + }) + const target = write(nested, 'index.js', 'module.exports = {}') + + const manifest = await createManifest(directory) + const targets = JSON.parse(fs.readFileSync(manifest.path, 'utf8')).targets + + assert.equal(targets[realpath(target)].name, 'ioredis') + }) + + it('does not generate targets for disabled integrations', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) + directories.push(directory) + fs.writeFileSync(path.join(directory, 'package.json'), '{}') + const packagePath = createPackageIn(directory, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packagePath, 'index.js', 'module.exports = {}') + const previous = process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS + process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS = 'ioredis' + + try { + assert.deepEqual(await createManifest(directory), {}) + } finally { + if (previous === undefined) delete process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS + else process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS = previous + } + }) + it('does not add rules when no supported package is installed', async () => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) directories.push(directory) @@ -158,6 +258,18 @@ describe('datadog-turbopack configuration', () => { assert.strictEqual(await withDatadogTurbopack(config, directory), config) }) + it('leaves configuration unchanged when its cache directory cannot be created', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) + directories.push(directory) + fs.writeFileSync(path.join(directory, 'package.json'), '{}') + const packagePath = createPackageIn(directory, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packagePath, 'index.js', 'module.exports = {}') + fs.writeFileSync(path.join(directory, 'node_modules', '.cache'), '') + const config = { rules: { '*.js': { loaders: ['existing-loader'] } } } + + assert.strictEqual(await withDatadogTurbopack(config, directory), config) + }) + it('does not require Next.js and preserves existing Turbopack settings', async () => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) directories.push(directory) @@ -186,7 +298,7 @@ describe('datadog-turbopack configuration', () => { const once = await withDatadogTurbopack({}, directory) const twice = await withDatadogTurbopack(once, directory) - for (const extension of ['*.js', '*.cjs', '*.mjs']) { + for (const extension of ['*.js', '*.cjs', '*.mjs', '*.jsx', '*.ts', '*.tsx']) { const rules = [twice.rules[extension]].flat() assert.equal(rules.filter(rule => rule.loaders.some(item => item.loader.includes('datadog-turbopack'))).length, 1) } @@ -208,6 +320,10 @@ describe('datadog-turbopack configuration', () => { assert.match('const { generateText } = require("ai")', applicationRule.condition.all[2].content) assert.doesNotMatch('import { something } from "unrelated"', applicationRule.condition.all[2].content) assert.equal(applicationRule.loaders[0].options.rewriteApplicationImports, true) + assert.match(applicationRule.loaders[0].options.manifestHash, /^[a-f0-9]{64}$/) + assert.deepEqual(applicationRule.loaders[0].options.aliases, []) + assert.ok(config.rules['*.ts']) + assert.ok(config.rules['*.tsx']) }) }) diff --git a/turbopack.d.ts b/turbopack.d.ts new file mode 100644 index 00000000000..a8d4a240114 --- /dev/null +++ b/turbopack.d.ts @@ -0,0 +1,12 @@ +export interface TurbopackConfiguration { + rules?: Record + resolveAlias?: Record +} + +/** + * Adds Node.js Turbopack rules for supported dd-trace integrations. + */ +export function withDatadogTurbopack ( + turbopack?: TurbopackConfiguration, + projectDir?: string +): Promise From 96cc212eda03e7da5c4e0242e3b374f12fb510a8 Mon Sep 17 00:00:00 2001 From: William Conti Date: Wed, 26 Aug 2026 21:42:39 -0400 Subject: [PATCH 6/9] fix(turbopack): preserve bundled instrumentation --- .../src/helpers/bundler-register.js | 3 ++- .../test/helpers/bundler-register.spec.js | 24 +++++++++++++++++++ packages/datadog-turbopack/src/loader.js | 21 +++++++++++----- .../datadog-turbopack/test/plugin.spec.js | 23 ++++++++++++++++++ turbopack.d.ts | 6 ++--- 5 files changed, 67 insertions(+), 10 deletions(-) diff --git a/packages/datadog-instrumentations/src/helpers/bundler-register.js b/packages/datadog-instrumentations/src/helpers/bundler-register.js index 12fa55264ca..6edd0ca1f77 100644 --- a/packages/datadog-instrumentations/src/helpers/bundler-register.js +++ b/packages/datadog-instrumentations/src/helpers/bundler-register.js @@ -118,7 +118,8 @@ dc.subscribe(CHANNEL, (message) => { try { loadChannel.publish({ name, version: payload.version, file }) let exports = payload.module - if (patchDefault === !!exports.default) { + // Only generated ESM proxy payloads need default-export unwrapping. + if (typeof payload.apply === 'function' && patchDefault === !!exports.default) { if (patchDefault) exports = exports.default else continue } diff --git a/packages/datadog-instrumentations/test/helpers/bundler-register.spec.js b/packages/datadog-instrumentations/test/helpers/bundler-register.spec.js index 68f2472fa04..ba92a0540e4 100644 --- a/packages/datadog-instrumentations/test/helpers/bundler-register.spec.js +++ b/packages/datadog-instrumentations/test/helpers/bundler-register.spec.js @@ -1,5 +1,6 @@ 'use strict' +const assert = require('node:assert/strict') const Module = require('node:module') const sinon = require('sinon') @@ -47,6 +48,29 @@ describe('bundler register', () => { }) }) + it('patches a CommonJS object when its hook does not use a default export', () => { + const Original = class Original {} + const Patched = class Patched {} + const hook = sinon.stub().returns(Patched) + const { publish } = loadBundlerRegister({ + hooks: { 'test-commonjs-export': sinon.stub() }, + instrumentations: { + 'test-commonjs-export': [{ file: 'index.js', hook, patchDefault: false }], + }, + }) + const payload = { + module: { Original }, + package: 'test-commonjs-export', + path: 'test-commonjs-export/index.js', + version: '1.0.0', + } + + publish(payload) + + sinon.assert.calledOnceWithExactly(hook, { Original }, '1.0.0') + assert.equal(payload.module, Patched) + }) + it('does not activate explicitly disabled bundled integrations', () => { const hook = sinon.stub() const integrationHook = sinon.stub() diff --git a/packages/datadog-turbopack/src/loader.js b/packages/datadog-turbopack/src/loader.js index 34596b67681..c5b7f55acfb 100644 --- a/packages/datadog-turbopack/src/loader.js +++ b/packages/datadog-turbopack/src/loader.js @@ -5,8 +5,12 @@ const path = require('node:path') const { create } = require('../../../vendor/dist/@apm-js-collab/code-transformer') const { isESMFile } = require('../../datadog-esbuild/src/utils') +const { rewrite } = require('../../datadog-instrumentations/src/helpers/rewriter') const CHANNEL = 'dd-trace:bundler:load' +// Keep the marker split so source-map scanners do not treat this file as mapped. +// eslint-disable-next-line unicorn/no-useless-concat -- Keep the marker non-contiguous. +const SOURCE_MAP_PREFIX = '//# sourceMapping' + 'URL=data:application/json;base64,' /** * Instruments bundled modules known to dd-trace. CommonJS modules publish @@ -23,13 +27,13 @@ module.exports = function loader (source) { getRelativeTarget(this.resourcePath, manifest.relativeTargets) const esm = isESMFile(this.resourcePath) - if (rewriteApplicationImports || esm) { - const rewritten = rewriteImports(source, this.resourcePath, manifest.targets, aliases) - if (rewritten !== source || esm) return rewritten - } + if (rewriteApplicationImports || esm) source = rewriteImports(source, this.resourcePath, manifest.targets, aliases) if (!target) return source + source = rewrite(source, this.resourcePath, esm ? 'module' : 'commonjs') + if (esm) return source + const dcPolyfillPath = relativeImport( path.dirname(this.resourcePath), require.resolve('dc-polyfill') @@ -90,14 +94,19 @@ function rewriteImports (source, resourcePath, targets, aliases = []) { if (!transformer) return source try { - const output = transformer.transform(source, 'esm').code - return rewritten ? output : source + const { code, map } = transformer.transform(source, 'esm') + return rewritten ? withInlineSourceMap(code, map) : source } catch { // A parser failure must never prevent an application from building. return source } } +function withInlineSourceMap (code, map) { + if (!map) return code + return `${code}\n${SOURCE_MAP_PREFIX}${Buffer.from(map).toString('base64')}` +} + function getModuleSource (node, hasLocalRequire) { if (node.type === 'ImportDeclaration' || node.type === 'ExportNamedDeclaration' || diff --git a/packages/datadog-turbopack/test/plugin.spec.js b/packages/datadog-turbopack/test/plugin.spec.js index 054f0bcfb63..73b611ecee5 100644 --- a/packages/datadog-turbopack/test/plugin.spec.js +++ b/packages/datadog-turbopack/test/plugin.spec.js @@ -76,6 +76,7 @@ describe('datadog-turbopack loader', () => { }, fs.readFileSync(appPath, 'utf8')) assert.match(result, /from "\.\.\/node_modules\/\.cache\/dd-trace\/turbopack\/0\.mjs"/) + assert.match(result, /sourceMappingURL=data:application\/json;base64,/) }) it('routes an application CommonJS require through its generated proxy', async () => { @@ -198,6 +199,28 @@ describe('datadog-turbopack loader', () => { assert.match(result, /set\.default\?\.\(exports\)/) assert.doesNotMatch(result, /import-in-the-middle/) }) + + it('applies existing rewriter instrumentation to an ESM target', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) + directories.push(directory) + fs.writeFileSync(path.join(directory, 'package.json'), '{}') + const packagePath = createPackageIn(directory, 'ai', { main: 'dist/index.js', type: 'module', version: '6.0.0' }) + const target = write(packagePath, 'dist/index.js', [ + 'export function resolveLanguageModel (model) {', + ' return model', + '}', + '', + ].join('\n')) + const manifest = await createManifest(directory) + + const result = loader.call({ + getOptions: () => ({ manifestPath: manifest.path }), + resourcePath: target, + }, fs.readFileSync(target, 'utf8')) + + assert.notEqual(result, fs.readFileSync(target, 'utf8')) + assert.match(result, /sourceMappingURL=data:application\/json;base64,/) + }) }) describe('datadog-turbopack configuration', () => { diff --git a/turbopack.d.ts b/turbopack.d.ts index a8d4a240114..7415dd7827e 100644 --- a/turbopack.d.ts +++ b/turbopack.d.ts @@ -6,7 +6,7 @@ export interface TurbopackConfiguration { /** * Adds Node.js Turbopack rules for supported dd-trace integrations. */ -export function withDatadogTurbopack ( - turbopack?: TurbopackConfiguration, +export function withDatadogTurbopack ( + turbopack?: T, projectDir?: string -): Promise +): Promise From 593cb4a895b165ffa4c99aebc6c66f1e68cdc672 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Fri, 28 Aug 2026 15:13:16 +0200 Subject: [PATCH 7/9] feat(turbopack): complete bundled instrumentation Turbopack compiles dependency modules before runtime hooks can observe their exports. Build-time plans and live-binding proxies preserve instrumentation without adding discovery or source analysis to the request path. --- .github/CODEOWNERS | 1 + .github/workflows/instrumentation.yml | 1 + LICENSE-3rdparty.csv | 3 + README.md | 11 + .../turbopack/app/api/esm/route.js | 15 + integration-tests/turbopack/app/model.js | 19 + integration-tests/turbopack/index.spec.js | 67 ++ integration-tests/turbopack/next.config.js | 9 + integration-tests/turbopack/pages/api/cjs.js | 18 + integration-tests/turbopack/server.js | 20 + next.d.ts | 15 + turbopack.js => next.js | 0 package.json | 6 +- packages/datadog-esbuild/src/utils.js | 4 +- packages/datadog-esbuild/test/utils.spec.js | 8 + .../src/helpers/bundler-constants.js | 5 + .../src/helpers/bundler-register.js | 53 +- .../src/helpers/instrumentation-utils.js | 17 + .../src/helpers/register.js | 22 +- .../src/helpers/rewriter/index.js | 140 ++- .../test/helpers/bundler-register.spec.js | 177 +++- .../test/helpers/rewriter/index.spec.js | 88 ++ packages/datadog-turbopack/README.md | 111 +++ packages/datadog-turbopack/index.js | 326 +++++-- packages/datadog-turbopack/src/loader.js | 692 ++++++++++++--- packages/datadog-turbopack/src/targets.js | 684 ++++++++++---- .../datadog-turbopack/test/config.spec.js | 563 ++++++++++++ packages/datadog-turbopack/test/helpers.js | 88 ++ .../datadog-turbopack/test/loader.spec.js | 831 ++++++++++++++++++ .../datadog-turbopack/test/plugin.spec.js | 382 -------- turbopack.d.ts | 12 - 31 files changed, 3537 insertions(+), 851 deletions(-) create mode 100644 integration-tests/turbopack/app/api/esm/route.js create mode 100644 integration-tests/turbopack/app/model.js create mode 100644 integration-tests/turbopack/index.spec.js create mode 100644 integration-tests/turbopack/next.config.js create mode 100644 integration-tests/turbopack/pages/api/cjs.js create mode 100644 integration-tests/turbopack/server.js create mode 100644 next.d.ts rename turbopack.js => next.js (100%) create mode 100644 packages/datadog-instrumentations/src/helpers/bundler-constants.js create mode 100644 packages/datadog-turbopack/README.md create mode 100644 packages/datadog-turbopack/test/config.spec.js create mode 100644 packages/datadog-turbopack/test/helpers.js create mode 100644 packages/datadog-turbopack/test/loader.spec.js delete mode 100644 packages/datadog-turbopack/test/plugin.spec.js delete mode 100644 turbopack.d.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4033c1b49cc..03fb77ba57c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -97,6 +97,7 @@ /index.electron.js @DataDog/dd-trace-js @DataDog/apm-idm-js /integration-tests/electron/ @DataDog/dd-trace-js @DataDog/apm-idm-js /integration-tests/esbuild/ @DataDog/dd-trace-js @DataDog/apm-idm-js +/integration-tests/turbopack/ @DataDog/dd-trace-js @DataDog/apm-idm-js /integration-tests/webpack/ @DataDog/dd-trace-js @DataDog/apm-idm-js /integration-tests/pino/ @DataDog/dd-trace-js @DataDog/apm-idm-js /integration-tests/pino.spec.js @DataDog/dd-trace-js @DataDog/apm-idm-js diff --git a/.github/workflows/instrumentation.yml b/.github/workflows/instrumentation.yml index b953fb8c315..66ecafecd7d 100644 --- a/.github/workflows/instrumentation.yml +++ b/.github/workflows/instrumentation.yml @@ -47,6 +47,7 @@ jobs: - run: npm run test:turbopack:ci - uses: ./.github/actions/node/latest - run: npm run test:turbopack:ci + - run: npm run test:integration:turbopack - uses: ./.github/actions/coverage with: flags: platform-turbopack diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 9fae5b5de66..97f14794226 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -53,11 +53,13 @@ "dd-trace","https://github.com/DataDog/dd-trace-js","['(Apache-2.0 OR BSD-3-Clause)']","['Datadog Inc. ']" "debug","https://github.com/debug-js/debug","['MIT']","['Josh Junon']" "detect-newline","https://github.com/sindresorhus/detect-newline","['MIT']","['Sindre Sorhus']" +"enhanced-resolve","https://github.com/webpack/enhanced-resolve","['MIT']","['JS Foundation and other contributors']" "es-module-lexer","https://github.com/guybedford/es-module-lexer","['MIT']","['Guy Bedford']" "escape-string-regexp","https://github.com/sindresorhus/escape-string-regexp","['MIT']","['Sindre Sorhus']" "esquery","https://github.com/estools/esquery","['BSD-3-Clause']","['Joel Feenstra']" "estraverse","https://github.com/estools/estraverse","['BSD-2-Clause']","['estools']" "fast-fifo","https://github.com/mafintosh/fast-fifo","['MIT']","['Mathias Buus']" +"graceful-fs","https://github.com/isaacs/node-graceful-fs","['ISC']","['Isaac Z. Schlueter, Ben Noordhuis, and Contributors']" "https-proxy-agent","https://github.com/TooTallNate/proxy-agents","['MIT']","['Nathan Rajlich']" "import-in-the-middle","https://github.com/nodejs/import-in-the-middle","['Apache-2.0']","['Bryan English']" "istanbul-lib-coverage","https://github.com/istanbuljs/istanbuljs","['BSD-3-Clause']","['Krishnan Anantheswaran']" @@ -87,6 +89,7 @@ "shell-quote","https://github.com/ljharb/shell-quote","['MIT']","['James Halliday']" "source-map","https://github.com/mozilla/source-map","['BSD-3-Clause']","['Nick Fitzgerald']" "spark-md5","https://github.com/satazor/js-spark-md5","['(WTFPL OR MIT)']","['André Cruz']" +"tapable","https://github.com/webpack/tapable","['MIT']","['JS Foundation and other contributors']" "tlhunter-sorted-set","https://github.com/tlhunter/node-sorted-set","['MIT']","['Thomas Hunter II']" "tslib","https://github.com/microsoft/tslib","['0BSD']","['Microsoft Corp.']" "ttl-set","https://github.com/watson/ttl-set","['MIT']","['Thomas Watson']" diff --git a/README.md b/README.md index 04a8ee2b321..e2836fdfe25 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,17 @@ Regardless of where you open the issue, someone at Datadog will try to help. If you would like to trace your bundled application then please read this page on [bundling and dd-trace](https://docs.datadoghq.com/tracing/trace_collection/automatic_instrumentation/dd_libraries/nodejs/#bundling). It includes information on how to use our ESBuild plugin and includes caveats for other bundlers. +Next.js applications that use Turbopack can wrap their existing configuration: + +```javascript +const { withDatadogTurbopack } = require('dd-trace/next') + +module.exports = withDatadogTurbopack({}) +``` + +Preload `dd-trace/init` before application modules load. The generated modules can then publish their exports to the tracer. + +See the [Turbopack implementation](packages/datadog-turbopack/README.md) for the internal build and runtime flow. ## Security Vulnerabilities diff --git a/integration-tests/turbopack/app/api/esm/route.js b/integration-tests/turbopack/app/api/esm/route.js new file mode 100644 index 00000000000..9d27b67782c --- /dev/null +++ b/integration-tests/turbopack/app/api/esm/route.js @@ -0,0 +1,15 @@ +'use strict' + +async function GET () { + // eslint-disable-next-line n/no-missing-import -- dependency is installed in the integration sandbox + const { generateText } = await import('ai') + const model = require('../../model') + const result = await generateText({ + model, + prompt: 'Say ok', + experimental_telemetry: { isEnabled: true }, + }) + return Response.json({ dependency: typeof generateText === 'function' ? 'ai' : 'missing', text: result.text }) +} + +module.exports = { GET } diff --git a/integration-tests/turbopack/app/model.js b/integration-tests/turbopack/app/model.js new file mode 100644 index 00000000000..bd040fc4041 --- /dev/null +++ b/integration-tests/turbopack/app/model.js @@ -0,0 +1,19 @@ +'use strict' + +module.exports = { + specificationVersion: 'v3', + provider: 'turbopack-test', + modelId: 'turbopack-test', + supportedUrls: {}, + doGenerate () { + return Promise.resolve({ + content: [{ type: 'text', text: 'ok' }], + finishReason: { unified: 'stop', raw: undefined }, + usage: { + inputTokens: { total: 1 }, + outputTokens: { total: 1 }, + }, + warnings: [], + }) + }, +} diff --git a/integration-tests/turbopack/index.spec.js b/integration-tests/turbopack/index.spec.js new file mode 100644 index 00000000000..02bd1e9f1cb --- /dev/null +++ b/integration-tests/turbopack/index.spec.js @@ -0,0 +1,67 @@ +'use strict' + +const assert = require('node:assert/strict') +const { execSync } = require('node:child_process') +const path = require('node:path') + +const axios = require('axios') + +const { + FakeAgent, + checkSpansForServiceName, + sandboxCwd, + spawnPluginIntegrationTestProc, + stopProc, + useSandbox, +} = require('../helpers') + +for (const nextVersion of ['15.5.0', 'latest']) { + describe(`Turbopack integration with Next.js ${nextVersion}`, () => { + useSandbox([`next@${nextVersion}`, 'react', 'react-dom', 'ai', 'express'], false, [__dirname]) + + let agent + let proc + + before(function () { + this.timeout(300_000) + execSync('npm exec -- next build --turbopack', { cwd: appCwd(), stdio: 'inherit' }) + }) + + beforeEach(async () => { + agent = await new FakeAgent().start() + proc = await spawnPluginIntegrationTestProc(appCwd(), 'server.js', agent.port, { + NODE_OPTIONS: '-r dd-trace/init', + }) + }) + + afterEach(async () => { + await stopProc(proc) + await agent.stop() + }) + + it('runs bundled CommonJS and ESM dependencies', async () => { + const assertCjsTrace = agent.assertMessageReceived(({ payload }) => { + assert.strictEqual(checkSpansForServiceName(payload, 'next.request'), true) + assert.strictEqual(checkSpansForServiceName(payload, 'express.request'), true) + assert.strictEqual(checkSpansForServiceName(payload, 'generateText'), true) + }, 10_000, 1, true) + + const response = await axios.get(`${proc.url}/api/cjs`) + assert.deepStrictEqual(response.data, { dependency: 'express', text: 'ok' }) + await assertCjsTrace + + const assertEsmTrace = agent.assertMessageReceived(({ payload }) => { + assert.strictEqual(checkSpansForServiceName(payload, 'next.request'), true) + assert.strictEqual(checkSpansForServiceName(payload, 'generateText'), true) + }, 10_000, 1, true) + + const esmResponse = await axios.get(`${proc.url}/api/esm`) + assert.deepStrictEqual(esmResponse.data, { dependency: 'ai', text: 'ok' }) + await assertEsmTrace + }) + }) +} + +function appCwd () { + return path.join(sandboxCwd(), 'turbopack') +} diff --git a/integration-tests/turbopack/next.config.js b/integration-tests/turbopack/next.config.js new file mode 100644 index 00000000000..06c44c9d3ab --- /dev/null +++ b/integration-tests/turbopack/next.config.js @@ -0,0 +1,9 @@ +'use strict' + +const path = require('node:path') + +const { withDatadogTurbopack } = require('dd-trace/next') + +const root = path.dirname(__dirname) + +module.exports = withDatadogTurbopack({ turbopack: { root } }, { projectDir: root }) diff --git a/integration-tests/turbopack/pages/api/cjs.js b/integration-tests/turbopack/pages/api/cjs.js new file mode 100644 index 00000000000..dd1fb0166a4 --- /dev/null +++ b/integration-tests/turbopack/pages/api/cjs.js @@ -0,0 +1,18 @@ +'use strict' + +const express = require('express') +const { generateText } = require('ai') + +const model = require('../../app/model') + +const app = express() +app.use(async (_request, response) => { + const result = await generateText({ + model, + prompt: 'Say ok', + experimental_telemetry: { isEnabled: true }, + }) + response.json({ dependency: 'express', text: result.text }) +}) + +module.exports = (request, response) => app(request, response) diff --git a/integration-tests/turbopack/server.js b/integration-tests/turbopack/server.js new file mode 100644 index 00000000000..26a9fd52b4a --- /dev/null +++ b/integration-tests/turbopack/server.js @@ -0,0 +1,20 @@ +'use strict' + +const { createServer } = require('node:http') + +const next = require('next') + +const nextApp = next({ dev: false }) +const handle = nextApp.getRequestHandler() + +async function start () { + await nextApp.prepare() + + const server = createServer((request, response) => handle(request, response)) + server.listen(0, () => { + const port = server.address().port + process.send({ port }) + }) +} + +start() diff --git a/next.d.ts b/next.d.ts new file mode 100644 index 00000000000..cfb290e7b57 --- /dev/null +++ b/next.d.ts @@ -0,0 +1,15 @@ +import type { NextConfig } from 'next' + +export interface DatadogTurbopackOptions { + projectDir?: string +} + +export function withDatadogTurbopack ( + nextConfig: (...args: TArguments) => NextConfig | Promise, + options?: DatadogTurbopackOptions +): (...args: TArguments) => Promise + +export function withDatadogTurbopack ( + nextConfig?: NextConfig | Promise, + options?: DatadogTurbopackOptions +): Promise diff --git a/turbopack.js b/next.js similarity index 100% rename from turbopack.js rename to next.js diff --git a/package.json b/package.json index a4171e3013a..49c4bba023b 100644 --- a/package.json +++ b/package.json @@ -93,6 +93,7 @@ "test:integration:electron": "mocha \"integration-tests/electron/*.spec.js\"", "test:integration:esbuild": "mocha --timeout 60000 \"integration-tests/esbuild/*.spec.js\"", "test:integration:esbuild:coverage": "node ./integration-tests/coverage/run-suite.js --timeout 60000 \"integration-tests/esbuild/*.spec.js\"", + "test:integration:turbopack": "mocha --timeout 60000 \"integration-tests/turbopack/*.spec.js\"", "test:integration:webpack": "mocha --timeout 60000 \"integration-tests/webpack/*.spec.js\"", "test:integration:openfeature": "mocha --timeout 60000 \"integration-tests/openfeature/*.spec.js\"", "test:integration:openfeature:coverage": "node ./integration-tests/coverage/run-suite.js --timeout 60000 \"integration-tests/openfeature/*.spec.js\"", @@ -152,8 +153,8 @@ "ci/**/*", "cypress/**/*", "esbuild.js", - "turbopack.js", - "turbopack.d.ts", + "next.js", + "next.d.ts", "webpack.js", "ext/**/*", "index.d.ts", @@ -183,6 +184,7 @@ ], "dependencies": { "dc-polyfill": "^0.1.11", + "enhanced-resolve": "^5.17.1", "import-in-the-middle": "^3.3.2", "opentracing": ">=0.14.7" }, diff --git a/packages/datadog-esbuild/src/utils.js b/packages/datadog-esbuild/src/utils.js index df3b8de0286..6c20ccaea74 100644 --- a/packages/datadog-esbuild/src/utils.js +++ b/packages/datadog-esbuild/src/utils.js @@ -246,8 +246,8 @@ async function processModule ({ path, internal = false, context, excludeDefault * @returns {boolean} */ function isESMFile (fullPathToModule, modulePackageJsonPath, packageJson = {}) { - if (fullPathToModule.endsWith('.mjs')) return true - if (fullPathToModule.endsWith('.cjs')) return false + if (fullPathToModule.endsWith('.mjs') || fullPathToModule.endsWith('.mts')) return true + if (fullPathToModule.endsWith('.cjs') || fullPathToModule.endsWith('.cts')) return false const pathParts = fullPathToModule.split(path.sep) do { diff --git a/packages/datadog-esbuild/test/utils.spec.js b/packages/datadog-esbuild/test/utils.spec.js index eacc3f4aabe..274fc45b5d4 100644 --- a/packages/datadog-esbuild/test/utils.spec.js +++ b/packages/datadog-esbuild/test/utils.spec.js @@ -75,6 +75,14 @@ describe('esbuild utils', () => { assert.strictEqual(isESMFile('/path/to/test.cjs'), false) }) + it('should return true if the file has a .mts extension in a CommonJS package', () => { + assert.strictEqual(isESMFile('/path/to/test.mts', '/path/to/package.json', { type: 'commonjs' }), true) + }) + + it('should return false if the file has a .cts extension in an ESM package', () => { + assert.strictEqual(isESMFile('/path/to/test.cts', '/path/to/package.json', { type: 'module' }), false) + }) + it('should return true if the file is in a directory with a package.json that has a type of module', () => { assert.strictEqual(isESMFile('/path/to/test.js', '/path/to/package.json', { type: 'module' }), true) }) diff --git a/packages/datadog-instrumentations/src/helpers/bundler-constants.js b/packages/datadog-instrumentations/src/helpers/bundler-constants.js new file mode 100644 index 00000000000..3bba3c72836 --- /dev/null +++ b/packages/datadog-instrumentations/src/helpers/bundler-constants.js @@ -0,0 +1,5 @@ +'use strict' + +const BUNDLER_DC_GLOBAL = 'dd-trace:bundler:dc' + +module.exports = { BUNDLER_DC_GLOBAL } diff --git a/packages/datadog-instrumentations/src/helpers/bundler-register.js b/packages/datadog-instrumentations/src/helpers/bundler-register.js index 6edd0ca1f77..0a1271b98a1 100644 --- a/packages/datadog-instrumentations/src/helpers/bundler-register.js +++ b/packages/datadog-instrumentations/src/helpers/bundler-register.js @@ -4,17 +4,19 @@ const Module = require('module') const dc = require('dc-polyfill') const log = require('../../../dd-trace/src/log') +const { BUNDLER_DC_GLOBAL } = require('./bundler-constants') +const { loadChannel } = require('./register.js') const { - filename, - loadChannel, + getDisabledInstrumentations, + matchesInstrumentation, matchVersion, -} = require('./register.js') -const { getDisabledInstrumentations } = require('./instrumentation-utils') +} = require('./instrumentation-utils') const hooks = require('./hooks') const instrumentations = require('./instrumentations') -const { isRelativeRequire } = require('./shared-utils') const disabledInstrumentations = getDisabledInstrumentations() +globalThis[Symbol.for(BUNDLER_DC_GLOBAL)] = dc + // register.js has now set up ritm (require-in-the-middle). In bundled // environments (webpack, esbuild), Node.js built-in modules required by // dd-trace internal modules (e.g. http from request.js) may have been loaded @@ -72,15 +74,25 @@ function doHook (name) { try { hookFn() - } catch { - log.error('esbuild-wrapped %s hook failed', name) + } catch (error) { + log.error('esbuild-wrapped %s hook failed: %s', name, error.message, error) } } /** @type {Set} */ const instrumentedNodeModules = new Set() -/** @typedef {{ package: string, module: unknown, version: string, path: string }} Payload */ +/** + * @typedef {object} Payload + * @property {string} package + * @property {unknown} module + * @property {string} version + * @property {string} path + * @property {number[]} [instrumentationIndexes] + * @property {string} [moduleBaseDir] + * @property {string} [moduleName] + * @property {(exports: unknown, patchDefault: boolean) => void} [apply] + */ dc.subscribe(CHANNEL, (message) => { const payload = /** @type {Payload} */ (message) const name = payload.package @@ -108,26 +120,35 @@ dc.subscribe(CHANNEL, (message) => { return } - for (const { file, filePattern, patchDefault, versions, hook } of instrumentation) { - const matchesFile = isRelativeRequire(name) || payload.path === filename(name, file) || - (filePattern && new RegExp(filename(name, filePattern)).test(payload.path)) - if (!matchesFile || !matchVersion(payload.version, versions)) { + const indexes = payload.instrumentationIndexes ?? instrumentation.keys() + for (const index of indexes) { + const entry = instrumentation[index] + if (!entry) { + log.error('Bundled %s instrumentation index %s does not exist', name, index) continue } + if (payload.instrumentationIndexes === undefined && + !matchesInstrumentation(name, payload.version, payload.path, entry)) continue + + const { patchDefault, versions, hook } = entry + if (!matchVersion(payload.version, versions)) continue try { - loadChannel.publish({ name, version: payload.version, file }) + loadChannel.publish({ name }) let exports = payload.module // Only generated ESM proxy payloads need default-export unwrapping. if (typeof payload.apply === 'function' && patchDefault === !!exports.default) { if (patchDefault) exports = exports.default else continue } - exports = hook(exports, payload.version) ?? exports + exports = hook(exports, payload.version, false, { + moduleBaseDir: payload.moduleBaseDir, + moduleName: payload.moduleName ?? payload.path, + }) ?? exports payload.module = exports payload.apply?.(exports, patchDefault) - } catch (e) { - log.error('Error executing bundler hook', e) + } catch (error) { + log.error('Error executing bundler hook: %s', error.message, error) } } }) diff --git a/packages/datadog-instrumentations/src/helpers/instrumentation-utils.js b/packages/datadog-instrumentations/src/helpers/instrumentation-utils.js index c0504a1a4bc..272fd192727 100644 --- a/packages/datadog-instrumentations/src/helpers/instrumentation-utils.js +++ b/packages/datadog-instrumentations/src/helpers/instrumentation-utils.js @@ -4,6 +4,7 @@ const { builtinModules } = require('node:module') const satisfies = require('../../../../vendor/dist/semifies') const { getValueFromEnvSources } = require('../../../dd-trace/src/config/helper') +const { isRelativeRequire } = require('./shared-utils') /** * @param {string|undefined} version @@ -23,6 +24,21 @@ function filename (name, file) { return file ? `${name}/${file}` : name } +/** + * @param {string} name + * @param {string|undefined} version + * @param {string} moduleName + * @param {{ file?: string, filePattern?: string, versions?: string[] }} instrumentation + * @returns {boolean} + */ +function matchesInstrumentation (name, version, moduleName, instrumentation) { + const { file, filePattern, versions } = instrumentation + if (!matchVersion(version, versions)) return false + if (isRelativeRequire(name)) return true + if (moduleName === filename(name, file)) return true + return filePattern !== undefined && new RegExp(filename(name, filePattern)).test(moduleName) +} + /** * @returns {Set} */ @@ -47,5 +63,6 @@ function getDisabledInstrumentations () { module.exports = { filename, getDisabledInstrumentations, + matchesInstrumentation, matchVersion, } diff --git a/packages/datadog-instrumentations/src/helpers/register.js b/packages/datadog-instrumentations/src/helpers/register.js index adc6789421e..99612bb81fd 100644 --- a/packages/datadog-instrumentations/src/helpers/register.js +++ b/packages/datadog-instrumentations/src/helpers/register.js @@ -11,9 +11,8 @@ const Hook = require('./hook') const { filename, getDisabledInstrumentations, - matchVersion, + matchesInstrumentation, } = require('./instrumentation-utils') -const { isRelativeRequire } = require('./shared-utils') const rewriter = require('./rewriter') const DD_TRACE_DEBUG = getValueFromEnvSources('DD_TRACE_DEBUG') @@ -87,21 +86,9 @@ for (const name of names) { instrumentedNodeModules.set(name, moduleExports) } - for (const { file, versions, hook, filePattern, patchDefault } of instrumentations[name]) { - const fullFilename = filename(name, file) - - let matchesFile = moduleName === fullFilename - - if (!matchesFile && isRelativeRequire(name)) matchesFile = true - - const fullFilePattern = filePattern && filename(name, filePattern) - if (fullFilePattern) { - // Some libraries include a hash in their filenames when installed, - // so our instrumentation has to include a '.*' to match them for more than a single version. - matchesFile ||= new RegExp(fullFilePattern).test(moduleName) - } - - if (matchesFile && matchVersion(moduleVersion, versions)) { + for (const instrumentation of instrumentations[name]) { + if (matchesInstrumentation(name, moduleVersion, moduleName, instrumentation)) { + const { hook, patchDefault } = instrumentation // IITM invokes this callback for every module in the package. Only unwrap the namespace after its file and // version match, otherwise a default export from an unrelated internal module can replace that module. if (isIitm && patchDefault === !!moduleExports.default) { @@ -168,5 +155,4 @@ module.exports = { filename, pathSepExpr, loadChannel, - matchVersion, } diff --git a/packages/datadog-instrumentations/src/helpers/rewriter/index.js b/packages/datadog-instrumentations/src/helpers/rewriter/index.js index 33a39fa200c..be1287c8f95 100644 --- a/packages/datadog-instrumentations/src/helpers/rewriter/index.js +++ b/packages/datadog-instrumentations/src/helpers/rewriter/index.js @@ -1,10 +1,12 @@ 'use strict' -const { readFileSync } = require('fs') -const { join } = require('path') -const { pathToFileURL } = require('url') +const { readFileSync } = require('node:fs') +const { join } = require('node:path') +const { pathToFileURL } = require('node:url') + const log = require('../../../../dd-trace/src/log') const { create } = require('../../../../../vendor/dist/@apm-js-collab/code-transformer') +const { BUNDLER_DC_GLOBAL } = require('../bundler-constants') const instrumentations = require('./instrumentations') const { getRewriteTarget } = require('./targets') const { awaitContextCallback, waitForAsyncEnd } = require('./transforms') @@ -33,11 +35,81 @@ const moduleVersions = {} const disabled = new Set() const matcherCjs = create(instrumentations, dcPolyfillCjs) const matcherEsm = create(instrumentations, dcPolyfillEsm) +const matcherBundler = create(instrumentations, 'node:diagnostics_channel') -for (const matcher of [matcherCjs, matcherEsm]) { +for (const matcher of [matcherCjs, matcherEsm, matcherBundler]) { matcher.addTransform('awaitContextCallback', awaitContextCallback) matcher.addTransform('waitForAsyncEnd', waitForAsyncEnd) } +matcherBundler.addTransform('tracingChannelImport', addBundlerTracingChannelImport) + +/** + * Reuses the process-wide polyfill installed by bundler-register while keeping + * the native diagnostics channel as the inactive-tracer fallback. + * + * @param {{ transforms: { defaults: { tracingChannelImport: Function } } }} state + * @param {{ body: object[] }} program + */ +function addBundlerTracingChannelImport (state, program) { + const previousLength = program.body.length + state.transforms.defaults.tracingChannelImport(state, program) + if (program.body.length === previousLength) return + + const index = program.body.findIndex(isNativeDcDeclaration) + const statement = program.body[index] + const identifier = statement.type === 'ImportDeclaration' + ? statement.specifiers[0].local + : statement.declarations[0].id + + identifier.name = 'tr_ch_apm_native_dc' + program.body.splice(index + 1, 0, createBundlerDcDeclaration()) +} + +/** + * @param {object} statement + * @returns {boolean} + */ +function isNativeDcDeclaration (statement) { + if (statement.type === 'ImportDeclaration') { + return statement.source?.value === 'node:diagnostics_channel' + } + const declaration = statement.declarations?.[0] + return declaration?.init?.arguments?.[0]?.value === 'node:diagnostics_channel' +} + +/** + * @returns {object} + */ +function createBundlerDcDeclaration () { + return { + type: 'VariableDeclaration', + declarations: [{ + type: 'VariableDeclarator', + id: { type: 'Identifier', name: 'tr_ch_apm_dc' }, + init: { + type: 'LogicalExpression', + operator: '??', + left: { + type: 'MemberExpression', + computed: true, + object: { type: 'Identifier', name: 'globalThis' }, + property: { + type: 'CallExpression', + arguments: [{ type: 'Literal', value: BUNDLER_DC_GLOBAL }], + callee: { + type: 'MemberExpression', + computed: false, + object: { type: 'Identifier', name: 'Symbol' }, + property: { type: 'Identifier', name: 'for' }, + }, + }, + }, + right: { type: 'Identifier', name: 'tr_ch_apm_native_dc' }, + }, + }], + kind: 'const', + } +} // Keep the marker split: source-map scanners can read a contiguous token in // string literals as this file's own inline map. @@ -52,10 +124,41 @@ const SOURCE_MAP_PREFIX = '//# sourceMapping' + 'URL=data:application/json;base6 * @returns {string|Buffer|ArrayBuffer|Uint8Array} */ function rewrite (content, filename, format, target) { - if (!content) return content + const { code, map } = rewriteWithMatcher(content, filename, format, target) + if (!map) return code + + return code + '\n' + SOURCE_MAP_PREFIX + Buffer.from(map).toString('base64') +} + +/** + * Rewrites source with a package specifier that bundlers can include in their + * output. + * + * @param {string|Buffer|ArrayBuffer|Uint8Array} content + * @param {string} filename + * @param {string} [format] + * @param {{ moduleName: string, filePath: string }} [target] + * @param {string|object} [sourceMap] + * @returns {{ code: string|Buffer|ArrayBuffer|Uint8Array, map?: string|object }} + */ +function rewriteBundledWithSourceMap (content, filename, format, target, sourceMap) { + return rewriteWithMatcher(content, filename, format, target, sourceMap, matcherBundler) +} + +/** + * @param {string|Buffer|ArrayBuffer|Uint8Array} content + * @param {string} filename + * @param {string} [format] + * @param {{ moduleName: string, filePath: string }} [target] + * @param {string|object} [sourceMap] + * @param {object} [bundlerMatcher] + * @returns {{ code: string|Buffer|ArrayBuffer|Uint8Array, map?: string|object }} + */ +function rewriteWithMatcher (content, filename, format, target, sourceMap, bundlerMatcher) { + if (!content) return { code: content, map: sourceMap } target ||= getRewriteTarget(filename) - if (!target) return content + if (!target) return { code: content, map: sourceMap } filename = filename.replace('file://', '') @@ -63,29 +166,22 @@ function rewrite (content, filename, format, target) { const { moduleName, filePath } = target const version = getVersion(filename, filePath) - if (disabled.has(moduleName)) return content + if (disabled.has(moduleName)) return { code: content, map: sourceMap } - const matcher = moduleType === 'esm' ? matcherEsm : matcherCjs + const matcher = bundlerMatcher ?? (moduleType === 'esm' ? matcherEsm : matcherCjs) const transformer = matcher.getTransformer(moduleName, version, filePath) - if (!transformer) return content + if (!transformer) return { code: content, map: sourceMap } try { const source = getSourceText(content) - - // TODO: pass existing sourcemap as input for remapping - const { code, map } = transformer.transform(source, moduleType) - - if (!map) return code - - const inlineMap = Buffer.from(map).toString('base64') - - return code + '\n' + SOURCE_MAP_PREFIX + inlineMap - } catch (e) { - log.error(e) + const { code, map } = transformer.transform(source, moduleType, sourceMap) + return { code, map } + } catch (error) { + log.error(error) } - return content + return { code: content, map: sourceMap } } /** @typedef {{ buffer: ArrayBuffer | SharedArrayBuffer, byteLength: number, byteOffset: number }} BufferView */ @@ -124,4 +220,4 @@ function getVersion (filename, filePath) { return moduleVersions[basename] } -module.exports = { rewrite, disable } +module.exports = { rewrite, rewriteBundledWithSourceMap, disable } diff --git a/packages/datadog-instrumentations/test/helpers/bundler-register.spec.js b/packages/datadog-instrumentations/test/helpers/bundler-register.spec.js index ba92a0540e4..42d8d9e3c9a 100644 --- a/packages/datadog-instrumentations/test/helpers/bundler-register.spec.js +++ b/packages/datadog-instrumentations/test/helpers/bundler-register.spec.js @@ -5,20 +5,37 @@ const Module = require('node:module') const sinon = require('sinon') +const { BUNDLER_DC_GLOBAL } = require('../../src/helpers/bundler-constants') +const instrumentationUtils = require('../../src/helpers/instrumentation-utils') + const CHANNEL = 'dd-trace:bundler:load' describe('bundler register', () => { + const dcGlobal = Symbol.for(BUNDLER_DC_GLOBAL) + let originalDc let originalRequire beforeEach(() => { + originalDc = globalThis[dcGlobal] originalRequire = Module.prototype.require }) afterEach(() => { + if (originalDc === undefined) { + delete globalThis[dcGlobal] + } else { + globalThis[dcGlobal] = originalDc + } Module.prototype.require = originalRequire sinon.restore() }) + it('shares the polyfilled diagnostics channel with generated bundles', () => { + const { dc } = loadBundlerRegister({ hooks: {}, instrumentations: {} }) + + assert.strictEqual(globalThis[dcGlobal], dc) + }) + it('honors patchDefault when applying an ESM proxy update', () => { const Original = class Original {} const Patched = class Patched {} @@ -33,19 +50,21 @@ describe('bundler register', () => { publish({ apply, + instrumentationIndexes: [0], module: { default: Original }, + moduleBaseDir: '/app/node_modules/test-default-export', + moduleName: 'test-default-export/index.mjs', package: 'test-default-export', path: 'test-default-export/index.mjs', version: '1.0.0', }) - sinon.assert.calledOnceWithExactly(hook, Original, '1.0.0') - sinon.assert.calledOnceWithExactly(apply, Patched, true) - sinon.assert.calledOnceWithExactly(loadChannel.publish, { - file: 'index.mjs', - name: 'test-default-export', - version: '1.0.0', + sinon.assert.calledOnceWithExactly(hook, Original, '1.0.0', false, { + moduleBaseDir: '/app/node_modules/test-default-export', + moduleName: 'test-default-export/index.mjs', }) + sinon.assert.calledOnceWithExactly(apply, Patched, true) + sinon.assert.calledOnceWithExactly(loadChannel.publish, { name: 'test-default-export' }) }) it('patches a CommonJS object when its hook does not use a default export', () => { @@ -67,7 +86,10 @@ describe('bundler register', () => { publish(payload) - sinon.assert.calledOnceWithExactly(hook, { Original }, '1.0.0') + sinon.assert.calledOnceWithExactly(hook, { Original }, '1.0.0', false, { + moduleBaseDir: undefined, + moduleName: 'test-commonjs-export/index.js', + }) assert.equal(payload.module, Patched) }) @@ -110,7 +132,10 @@ describe('bundler register', () => { version: '1.0.0', }) - sinon.assert.calledOnceWithExactly(integrationHook, {}, '1.0.0') + sinon.assert.calledOnceWithExactly(integrationHook, {}, '1.0.0', false, { + moduleBaseDir: undefined, + moduleName: 'test-pattern-hook/dist/cli-123.js', + }) }) it('matches bundled relative-module hooks', () => { @@ -130,7 +155,114 @@ describe('bundler register', () => { version: '6.1.0', }) - sinon.assert.calledOnceWithExactly(integrationHook, {}, '6.1.0') + sinon.assert.calledOnceWithExactly(integrationHook, {}, '6.1.0', false, { + moduleBaseDir: undefined, + moduleName: './runtime/library.js', + }) + }) + + it('uses build-plan indexes without running sibling hooks', () => { + const skippedHook = sinon.stub() + const selectedHook = sinon.stub() + const { publish } = loadBundlerRegister({ + hooks: { 'test-indexed-hook': sinon.stub() }, + instrumentations: { + 'test-indexed-hook': [ + { file: 'first.js', hook: skippedHook }, + { file: 'second.js', hook: selectedHook }, + ], + }, + }) + + publish({ + instrumentationIndexes: [1], + module: {}, + package: 'test-indexed-hook', + path: 'test-indexed-hook/second.js', + version: '1.0.0', + }) + + sinon.assert.notCalled(skippedHook) + sinon.assert.calledOnce(selectedHook) + }) + + it('rejects stale build-plan entries and incompatible versions', () => { + const integrationHook = sinon.stub() + const { log, publish } = loadBundlerRegister({ + hooks: { 'test-stale-plan': sinon.stub() }, + instrumentations: { + 'test-stale-plan': [{ file: 'index.js', hook: integrationHook, versions: ['>=2'] }], + }, + }) + + publish({ + instrumentationIndexes: [1], + module: {}, + package: 'test-stale-plan', + path: 'test-stale-plan/index.js', + version: '2.0.0', + }) + publish({ + instrumentationIndexes: [0], + module: {}, + package: 'test-stale-plan', + path: 'test-stale-plan/index.js', + version: '1.0.0', + }) + publish({ + module: {}, + package: 'test-stale-plan', + path: 'test-stale-plan/other.js', + version: '2.0.0', + }) + + sinon.assert.notCalled(integrationHook) + sinon.assert.calledWithMatch(log.error, 'Bundled %s instrumentation index %s does not exist', 'test-stale-plan', 1) + }) + + it('contains loader and instrumentation failures', () => { + const loadHook = sinon.stub().throws(new Error('load failed')) + const integrationHook = sinon.stub().throws(new Error('patch failed')) + const { log, publish } = loadBundlerRegister({ + hooks: { 'test-hook-errors': loadHook }, + instrumentations: { + 'test-hook-errors': [{ hook: integrationHook }], + }, + }) + + publish({ + instrumentationIndexes: [0], + module: {}, + package: 'test-hook-errors', + path: 'test-hook-errors', + version: '1.0.0', + }) + + sinon.assert.calledWithMatch(log.error, 'esbuild-wrapped %s hook failed: %s', 'test-hook-errors', 'load failed') + sinon.assert.calledWithMatch(log.error, 'Error executing bundler hook: %s', 'patch failed') + }) + + it('does not apply an ESM hook without the export shape it expects', () => { + const integrationHook = sinon.stub() + const apply = sinon.stub() + const { publish } = loadBundlerRegister({ + hooks: { 'test-missing-export': sinon.stub() }, + instrumentations: { + 'test-missing-export': [{ hook: integrationHook, patchDefault: false }], + }, + }) + + publish({ + apply, + instrumentationIndexes: [0], + module: {}, + package: 'test-missing-export', + path: 'test-missing-export', + version: '1.0.0', + }) + + sinon.assert.notCalled(integrationHook) + sinon.assert.notCalled(apply) }) }) @@ -138,26 +270,27 @@ function loadBundlerRegister ({ disabled = new Set(), hooks, instrumentations }) const bundlerRegisterPath = require.resolve('../../src/helpers/bundler-register') const originalRequire = Module.prototype.require const loadChannel = { publish: sinon.stub() } - let bundledModuleSubscriber - const register = { - filename: (name, file) => file ? `${name}/${file}` : name, - loadChannel, - matchVersion: () => true, + const log = { error: sinon.stub() } + const dc = { + subscribe: (channel, callback) => { + if (channel === CHANNEL) bundledModuleSubscriber = callback + }, } + let bundledModuleSubscriber + const register = { loadChannel } Module.prototype.require = function (request) { if (this.filename === bundlerRegisterPath) { const stubs = { './hooks': hooks, - './instrumentation-utils': { getDisabledInstrumentations: () => disabled }, + './instrumentation-utils': { + ...instrumentationUtils, + getDisabledInstrumentations: () => disabled, + }, './instrumentations': instrumentations, './register.js': register, - '../../../dd-trace/src/log': { error: sinon.stub() }, - 'dc-polyfill': { - subscribe: (channel, callback) => { - if (channel === CHANNEL) bundledModuleSubscriber = callback - }, - }, + '../../../dd-trace/src/log': log, + 'dc-polyfill': dc, } return stubs[request] || originalRequire.call(this, request) } @@ -168,7 +301,9 @@ function loadBundlerRegister ({ disabled = new Set(), hooks, instrumentations }) Module.prototype.require = originalRequire return { + dc, loadChannel, + log, publish: message => bundledModuleSubscriber(message), } } diff --git a/packages/datadog-instrumentations/test/helpers/rewriter/index.spec.js b/packages/datadog-instrumentations/test/helpers/rewriter/index.spec.js index a95506216cb..bce9e80b259 100644 --- a/packages/datadog-instrumentations/test/helpers/rewriter/index.spec.js +++ b/packages/datadog-instrumentations/test/helpers/rewriter/index.spec.js @@ -419,6 +419,20 @@ describe('check-require-cache', () => { }, channelName: 'pregel_stream', }, + { + module: { + name: 'test-esm', + versionRange: '>=0.1', + filePath: 'pregel-class.js', + }, + functionQuery: { + methodName: 'stream', + className: 'Pregel', + kind: 'Sync', + returnKind: 'AsyncIterator', + }, + channelName: 'pregel_stream_secondary', + }, ], }) }) @@ -931,6 +945,80 @@ describe('check-require-cache', () => { assert.strictEqual(rewriter.rewrite(source, filename, 'module'), source) }) + it('should compose an existing source map for bundler consumers', () => { + const filename = resolve(__dirname, 'node_modules', 'test-trace-sync', 'index.js') + const source = readFileSync(filename, 'utf8') + const sourceMap = { + file: filename, + mappings: 'AAAA', + names: [], + sources: ['original.js'], + sourcesContent: [source], + version: 3, + } + + const result = rewriter.rewriteBundledWithSourceMap(source, filename, 'commonjs', { + moduleName: 'test-trace-sync', + filePath: 'index.js', + }, sourceMap) + const map = JSON.parse(result.map) + + assert.match(result.code, /tr_ch_apm_tracingChannel/) + assert.strictEqual(map.sources[0], 'original.js') + assert.strictEqual(map.sourcesContent[0], source) + assert.strictEqual(map.sources.includes('test-trace-sync/index.js'), true) + }) + + it('should use the shared bundler diagnostics channel with a native fallback', () => { + const filename = resolve(__dirname, 'node_modules', 'test-esm', 'pregel-class.js') + const source = readFileSync(filename, 'utf8') + const result = rewriter.rewriteBundledWithSourceMap(source, filename, 'module', { + moduleName: 'test-esm', + filePath: 'pregel-class.js', + }) + + assert.match(result.code, /\bimport\s+.+\s+from\s+"node:diagnostics_channel"/) + assert.match(result.code, /Symbol\.for\("dd-trace:bundler:dc"\)/) + assert.doesNotMatch(result.code, /dc-polyfill/) + assert.strictEqual(result.code.match(/node:diagnostics_channel/g)?.length, 1) + + const commonJsFilename = resolve(__dirname, 'node_modules', 'test-trace-sync', 'index.js') + const commonJsSource = readFileSync(commonJsFilename, 'utf8') + const commonJsResult = rewriter.rewriteBundledWithSourceMap( + commonJsSource, + commonJsFilename, + 'commonjs', + { moduleName: 'test-trace-sync', filePath: 'index.js' } + ) + + assert.match(commonJsResult.code, /require\("node:diagnostics_channel"\)/) + assert.match(commonJsResult.code, /Symbol\.for\("dd-trace:bundler:dc"\)/) + }) + + it('should preserve bundled sources that cannot be rewritten', () => { + const sourceMap = { mappings: '', version: 3 } + assert.deepStrictEqual( + rewriter.rewriteBundledWithSourceMap('', '/project/empty.js', 'module', undefined, sourceMap), + { code: '', map: sourceMap } + ) + assert.deepStrictEqual(rewriter.rewriteBundledWithSourceMap( + 'module.exports = true', + '/project/node_modules/missing/index.js', + 'commonjs', + { moduleName: 'missing', filePath: 'index.js' }, + sourceMap + ), { code: 'module.exports = true', map: sourceMap }) + + rewriter.disable('test-disabled') + assert.deepStrictEqual(rewriter.rewriteBundledWithSourceMap( + 'module.exports = true', + '/project/node_modules/test-disabled/index.js', + 'commonjs', + { moduleName: 'test-disabled', filePath: 'index.js' }, + sourceMap + ), { code: 'module.exports = true', map: sourceMap }) + }) + it('should use import when rewriting esm modules', () => { const filename = resolve(__dirname, 'node_modules', 'test-esm', 'pregel-class.js') diff --git a/packages/datadog-turbopack/README.md b/packages/datadog-turbopack/README.md new file mode 100644 index 00000000000..b7a9983bb89 --- /dev/null +++ b/packages/datadog-turbopack/README.md @@ -0,0 +1,111 @@ +# Turbopack instrumentation + +The Turbopack integration instruments server-side dependency modules during a Next.js build. +It keeps target discovery and source analysis outside the application request path. + +This document describes the internal implementation. See the [root README](../../README.md#bundling) for setup instructions. + +## Architecture + +```mermaid +flowchart LR + A["Next.js configuration"] --> B["withDatadogTurbopack"] + B --> C["createBuildPlan"] + C --> D["Build plan
and ESM proxies"] + B --> E["Turbopack rules"] + D --> F["Datadog loader"] + E --> F + F --> G["Instrumented server modules"] + G --> H["dd-trace:bundler:load"] + H --> I["Bundler instrumentation registry"] +``` + +`withDatadogTurbopack` connects the configuration phase and the build phase. +The configuration phase creates immutable artifacts and Turbopack rules. +The build phase transforms only the modules that match these rules. +The generated modules publish their exports when the application loads them. + +## Configuration phase + +`withDatadogTurbopack` normalizes the supplied Next.js configuration. +It resolves Next.js from the application project directory. +It preserves existing Turbopack settings and appends the Datadog rules. +Repeated wrapping does not add the Datadog loader again. + +`createBuildPlan` loads the existing Datadog instrumentation declarations. +It finds installed packages that match these declarations. +It resolves package entry points with the Node.js `import` and `require` conditions. +It also records relative target files for integrations that instrument files below a package root. + +The planner hashes each target source file. +It creates export setters and proxies for successful ESM targets. +It writes the plan and proxies as content-addressed artifacts. +Concurrent configuration calls can safely use the same artifacts. + +The plan contains the target metadata and the expected source hashes. +The wrapper passes the plan path and hash to the loader. +The loader rejects plan content that does not match this hash. + +## Rule registration + +The wrapper selects the rule shape that the detected Next.js release accepts. +It adds direct rules for installed instrumentation targets. +It adds an import-inspection rule when the plan contains an ESM target. +It adds a separate rule for relative targets when the plan contains them. + +The direct rules transform known dependency files. +The import-inspection rule finds active edges to planned ESM targets. +The relative rule matches a file by its relative name and source hash. + +## Loader phase + +Turbopack calls the Datadog loader for each matching server module. +The loader verifies and caches the build plan before it transforms source. +The plan cache and source-hash cache have fixed bounds. + +For a direct target, the loader compares the current source hash with the plan. +It skips a changed target because the stored instrumentation data can be stale. +It sends matching source to the shared bundler rewriter and preserves the source map. + +The loader adds a guarded publication block to each CommonJS target. +The subscriber can replace the published exports before the module returns them. + +An ESM importer cannot replace the imported module namespace. +The loader therefore redirects active edges to a generated proxy. +It uses the Turbopack resolver for each edge. +An `import` edge uses import conditions, and a `require` edge uses require conditions. +This preserves the application resolver behavior and its aliases. + +The proxy imports the original module and maintains live export bindings. +The instrumentation subscriber applies changed exports through generated setters. +Type-only TypeScript imports do not create runtime edges. +Node.js built-in modules do not enter edge resolution. + +## Runtime publication + +`dd-trace/init` installs the bundler subscriber before application modules load. +Generated code gets the shared diagnostic channel through a global symbol. +It uses the native diagnostic channel when the shared channel is not present. + +Generated code checks `channel.hasSubscribers` before it creates publication payloads. +A payload identifies the package, version, path, exports, and selected instrumentation entries. +The subscriber checks disabled integrations and version rules before it runs a hook. + +For CommonJS, the subscriber updates the payload module. +The generated block then copies the result to `module.exports`. +For ESM, the proxy applies the result to its live bindings. +The subscriber catches and logs hook errors so that they do not stop the application. + +## Failure behavior + +| Condition | Behavior | +| --- | --- | +| No installed target matches | The wrapper returns the original Next.js configuration. | +| A target cannot produce valid instrumentation data | The planner warns once and omits that target. | +| The plan hash or plan version is invalid | The loader stops the build instead of using invalid plan data. | +| A direct target source hash changed | The loader warns once and skips direct instrumentation for that target. | +| Import parsing, resolver setup, or source generation fails | The loader keeps the original module edges and continues direct target handling. | +| The runtime channel has no subscriber | Generated code does not create payloads or run instrumentation hooks. | + +The bounded warning set prevents repeated build warnings for the same failure. +The shared module-format classifier keeps ESM and CommonJS decisions consistent with the other bundler integration. diff --git a/packages/datadog-turbopack/index.js b/packages/datadog-turbopack/index.js index f8023dd2cbc..abbc40faeea 100644 --- a/packages/datadog-turbopack/index.js +++ b/packages/datadog-turbopack/index.js @@ -1,80 +1,302 @@ 'use strict' -const { createManifest } = require('./src/targets') +const fs = require('node:fs') +const Module = require('node:module') +const path = require('node:path') + +const { createBuildPlan } = require('./src/targets') const loader = require.resolve('./src/loader') +const SOURCE_EXTENSIONS = ['*.js', '*.cjs', '*.mjs', '*.jsx', '*.ts', '*.cts', '*.mts', '*.tsx'] +const SOURCE_PATH_PATTERN = /\.(?:cjs|cts|js|jsx|mjs|mts|ts|tsx)$/ /** - * Adds Datadog instrumentation rules to a Turbopack configuration. Generated - * loader metadata is local to the application and applies only to Node.js - * bundles. Browser and Edge bundles retain their original modules. + * Adds Datadog instrumentation to a Next.js configuration. * - * @param {object} [turbopack] - * @param {string} [projectDir] - * @returns {Promise} + * @param {object|Promise|Function} [nextConfig] + * @param {{ projectDir?: string }} [options] + * @returns {Promise|Function} */ -function withDatadogTurbopack (turbopack, projectDir) { - return addRules(turbopack, projectDir) +function withDatadogTurbopack (nextConfig = {}, options = {}) { + if (!options || typeof options !== 'object' || Array.isArray(options)) { + throw new TypeError('withDatadogTurbopack options must be an object') + } + if (options.projectDir !== undefined && typeof options.projectDir !== 'string') { + throw new TypeError('withDatadogTurbopack options.projectDir must be a string') + } + + const projectDir = path.resolve(options.projectDir ?? process.cwd()) + const nextInfo = getNextInfo(projectDir) + if (typeof nextConfig === 'function') { + return async function datadogNextConfig (...args) { + const config = await nextConfig.apply(this, args) + return addDatadogConfig(normalizeConfig(config), projectDir, nextInfo) + } + } + + return Promise.resolve(nextConfig).then(config => + addDatadogConfig(normalizeConfig(config), projectDir, nextInfo) + ) } /** - * @param {object} turbopack - * @param {string} [projectDir] + * @param {object} nextConfig + * @param {string} projectDir + * @param {{ compiler: { generator: string, parser: string, traverse: string }, major: number }} nextInfo * @returns {Promise} */ -async function addRules (turbopack = {}, projectDir = process.cwd()) { - const manifest = await createManifest(projectDir) - if (!manifest.packagePathPattern || !manifest.path) return turbopack +async function addDatadogConfig (nextConfig, projectDir, nextInfo) { + const plan = await createBuildPlan(projectDir) + if (!plan.packagePathPattern || !plan.targetPathPattern || !plan.path || !plan.hash) return nextConfig + + const turbopack = nextConfig.turbopack ?? {} + const configured = nextInfo.major === 15 + ? addLegacyRules(turbopack, plan, nextInfo.compiler) + : addModernRules(turbopack, plan, nextInfo.compiler) + + return { + ...nextConfig, + turbopack: configured, + } +} + +/** + * @param {object|undefined} config + * @returns {object} + */ +function normalizeConfig (config) { + if (config === undefined) return {} + if (!config || typeof config !== 'object' || Array.isArray(config)) { + throw new TypeError('withDatadogTurbopack expects a Next.js configuration object, promise, or function') + } + + const { turbopack } = config + if (turbopack !== undefined && (!turbopack || typeof turbopack !== 'object' || Array.isArray(turbopack))) { + throw new TypeError('nextConfig.turbopack must be an object') + } + if (turbopack?.rules !== undefined && + (!turbopack.rules || typeof turbopack.rules !== 'object' || Array.isArray(turbopack.rules))) { + throw new TypeError('nextConfig.turbopack.rules must be an object') + } + if (turbopack?.conditions !== undefined && + (!turbopack.conditions || typeof turbopack.conditions !== 'object' || Array.isArray(turbopack.conditions))) { + throw new TypeError('nextConfig.turbopack.conditions must be an object') + } + if (turbopack?.resolveAlias !== undefined && + (!turbopack.resolveAlias || typeof turbopack.resolveAlias !== 'object' || Array.isArray(turbopack.resolveAlias))) { + throw new TypeError('nextConfig.turbopack.resolveAlias must be an object') + } + return config +} + +/** + * @param {object} turbopack + * @param {object} plan + * @param {{ generator: string, parser: string, traverse: string }} compiler + * @returns {object} + */ +function addModernRules (turbopack, plan, compiler) { const rules = { ...turbopack.rules } - const aliases = Object.keys(turbopack.resolveAlias ?? {}) - for (const extension of ['*.js', '*.cjs', '*.mjs', '*.jsx', '*.ts', '*.tsx']) { + + for (const extension of SOURCE_EXTENSIONS) { const existing = rules[extension] if (hasDatadogLoader(existing)) continue - const datadogRules = [{ - condition: { - all: ['foreign', 'node', { path: manifest.packagePathPattern }], - }, - loaders: [{ loader, options: { manifestHash: manifest.hash, manifestPath: manifest.path } }], - }] - if (manifest.esmImportPattern) { - datadogRules.push({ - condition: { - all: ['node', { not: 'foreign' }, { content: manifest.esmImportPattern }], - }, - loaders: [{ - loader, - options: { - aliases, - manifestHash: manifest.hash, - manifestPath: manifest.path, - rewriteApplicationImports: true, - }, - }], - }) - } - if (manifest.relativePathPattern) { - datadogRules.push({ - condition: { - all: ['node', { not: 'foreign' }, { path: manifest.relativePathPattern }], - }, - loaders: [{ loader, options: { manifestHash: manifest.hash, manifestPath: manifest.path } }], - }) + const additions = [createModernTargetRule(plan, compiler)] + if (plan.moduleSyntaxPattern) additions.push(createModernImportRule(plan, compiler)) + if (plan.relativePathPattern) additions.push(createModernRelativeRule(plan)) + + if (existing === undefined) { + rules[extension] = additions.length === 1 ? additions[0] : additions + } else { + rules[extension] = Array.isArray(existing) ? [...existing, ...additions] : [existing, ...additions] } - rules[extension] = existing - ? [...(Array.isArray(existing) ? existing : [existing]), ...datadogRules] - : datadogRules.length === 1 ? datadogRules[0] : datadogRules } - return { - ...turbopack, + return { ...turbopack, rules } +} + +/** + * @param {object} turbopack + * @param {object} plan + * @param {{ generator: string, parser: string, traverse: string }} compiler + * @returns {object} + */ +function addLegacyRules (turbopack, plan, compiler) { + const conditions = { ...turbopack.conditions } + const rules = { ...turbopack.rules } + + addLegacyRule( + conditions, rules, + '#dd-trace/target', + { + path: new RegExp( + `(?:${plan.packagePathPattern.source}.*${SOURCE_PATH_PATTERN.source}|${plan.targetPathPattern.source})` + ), + }, + { node: { loaders: [createLoader(plan, { compiler, rewriteEdges: true, targetScope: 'direct' })] } } + ) + + if (plan.moduleSyntaxPattern) { + addLegacyRule( + conditions, + rules, + '#dd-trace/import', + { content: plan.moduleSyntaxPattern, path: SOURCE_PATH_PATTERN }, + { node: { foreign: false, loaders: [createLoader(plan, { compiler, rewriteEdges: true })] } } + ) + } + + if (plan.relativePathPattern) { + addLegacyRule( + conditions, + rules, + '#dd-trace/relative', + { path: plan.relativePathPattern }, + { node: { foreign: false, loaders: [createLoader(plan, { targetScope: 'relative' })] } } + ) } + + return { ...turbopack, conditions, rules } } -function hasDatadogLoader (rules) { - return [rules].flat().some(rule => rule?.loaders?.some(item => item?.loader === loader)) +/** + * @param {Record} conditions + * @param {Record} rules + * @param {string} name + * @param {object} condition + * @param {object} rule + */ +function addLegacyRule (conditions, rules, name, condition, rule) { + if (hasDatadogLoader(rules[name])) return + if (Object.hasOwn(conditions, name) || Object.hasOwn(rules, name)) { + throw new Error(`Next.js Turbopack configuration already uses the reserved condition ${name}`) + } + conditions[name] = condition + rules[name] = rule +} + +/** + * @param {object} plan + * @param {{ generator: string, parser: string, traverse: string }} compiler + * @returns {object} + */ +function createModernTargetRule (plan, compiler) { + return { + condition: { + all: ['node', { + any: [ + { path: plan.packagePathPattern }, + { path: plan.targetPathPattern }, + ], + }], + }, + loaders: [createLoader(plan, { compiler, rewriteEdges: true, targetScope: 'direct' })], + } +} + +/** + * @param {object} plan + * @param {{ generator: string, parser: string, traverse: string }} compiler + * @returns {object} + */ +function createModernImportRule (plan, compiler) { + return { + condition: { all: ['node', { not: 'foreign' }, { content: plan.moduleSyntaxPattern }] }, + loaders: [createLoader(plan, { compiler, rewriteEdges: true })], + } +} + +/** + * @param {object} plan + * @returns {object} + */ +function createModernRelativeRule (plan) { + return { + condition: { all: ['node', { not: 'foreign' }, { path: plan.relativePathPattern }] }, + loaders: [createLoader(plan, { targetScope: 'relative' })], + } +} + +/** + * @param {object} plan + * @param {{ + * compiler?: { generator: string, parser: string, traverse: string }, + * rewriteEdges?: boolean, + * targetScope?: 'direct'|'relative' + * }} [settings] + * @returns {object} + */ +function createLoader (plan, settings = {}) { + const options = { + manifestHash: plan.hash, + manifestPath: plan.path, + } + if (settings.compiler) options.compiler = settings.compiler + if (settings.rewriteEdges) options.rewriteEdges = true + if (settings.targetScope) options.targetScope = settings.targetScope + return { loader, options } +} + +/** + * @param {unknown} value + * @returns {boolean} + */ +function hasDatadogLoader (value) { + if (Array.isArray(value)) { + for (const item of value) { + if (hasDatadogLoader(item)) return true + } + return false + } + if (!value || typeof value !== 'object') return false + if (value.loader === loader) return true + + for (const key of Object.keys(value)) { + if (hasDatadogLoader(value[key])) return true + } + return false +} + +/** + * @param {string} projectDir + * @returns {{ compiler: { generator: string, parser: string, traverse: string }, major: number }} + */ +function getNextInfo (projectDir) { + let version + let appRequire + try { + appRequire = Module.createRequire(path.join(projectDir, 'package.json')) + const packagePath = appRequire.resolve('next/package.json') + version = JSON.parse(fs.readFileSync(packagePath, 'utf8')).version + } catch (error) { + throw new Error(`withDatadogTurbopack could not resolve Next.js from ${projectDir}`, { cause: error }) + } + + const match = /^(\d+)\.(\d+)\./.exec(version) + if (!match) throw new Error(`withDatadogTurbopack could not parse Next.js version ${version}`) + const major = Number(match[1]) + const minor = Number(match[2]) + if (major < 15 || (major === 15 && minor < 5)) { + throw new RangeError(`withDatadogTurbopack requires Next.js 15.5 or newer; found ${version}`) + } + + try { + return { + compiler: { + generator: appRequire.resolve('next/dist/compiled/babel/generator'), + parser: appRequire.resolve('next/dist/compiled/babel/parser'), + traverse: appRequire.resolve('next/dist/compiled/babel/traverse'), + }, + major, + } + } catch (error) { + throw new Error(`Next.js ${version} does not provide the compiler required by withDatadogTurbopack`, { + cause: error, + }) + } } module.exports = { diff --git a/packages/datadog-turbopack/src/loader.js b/packages/datadog-turbopack/src/loader.js index c5b7f55acfb..dd2249f309c 100644 --- a/packages/datadog-turbopack/src/loader.js +++ b/packages/datadog-turbopack/src/loader.js @@ -1,185 +1,600 @@ 'use strict' +const { createHash } = require('node:crypto') const fs = require('node:fs') +const { builtinModules } = require('node:module') const path = require('node:path') -const { create } = require('../../../vendor/dist/@apm-js-collab/code-transformer') +const { BUNDLER_DC_GLOBAL } = require('../../datadog-instrumentations/src/helpers/bundler-constants') const { isESMFile } = require('../../datadog-esbuild/src/utils') -const { rewrite } = require('../../datadog-instrumentations/src/helpers/rewriter') +const { rewriteBundledWithSourceMap } = require('../../datadog-instrumentations/src/helpers/rewriter') +const BUILTIN_MODULES = new Set(builtinModules) const CHANNEL = 'dd-trace:bundler:load' -// Keep the marker split so source-map scanners do not treat this file as mapped. -// eslint-disable-next-line unicorn/no-useless-concat -- Keep the marker non-contiguous. -const SOURCE_MAP_PREFIX = '//# sourceMapping' + 'URL=data:application/json;base64,' +const IMPORT_RESOLVE_OPTIONS = { conditionNames: ['node', 'import'] } +const BASE_PARSER_PLUGINS = [ + 'decorators-legacy', + 'explicitResourceManagement', + 'importAttributes', + 'jsx', +] +const JAVASCRIPT_PARSER_PLUGINS = [...BASE_PARSER_PLUGINS, 'flow'] +const MAX_CACHED_FILES = 2048 +const MAX_CACHED_PLANS = 16 +const MAX_WARNINGS = 128 +const MODULE_SYNTAX_PATTERN = /\b(?:export|import|require)\b/ +const PLAN_VERSION = 3 +const REQUIRE_RESOLVE_OPTIONS = { conditionNames: ['node', 'require'] } +const TYPESCRIPT_PARSER_PLUGINS = [...BASE_PARSER_PLUGINS, 'typescript'] + +/** @type {Map} */ +const fileHashes = new Map() +/** @type {Map} */ +const plans = new Map() +/** @type {Set} */ +const warnedErrors = new Set() + +/** + * @typedef {object} PlanTarget + * @property {boolean} esm + * @property {object[]} payloads + * @property {string} [proxyPath] + * @property {string} sourceHash + */ + +/** + * @typedef {object} BuildPlan + * @property {Record} proxies + * @property {Array} relativeTargets + * @property {Record} targets + * @property {number} version + */ + +/** + * @typedef {object} ModuleEdge + * @property {'import'|'require'} kind + * @property {object[]} nodes + * @property {string} specifier + */ /** - * Instruments bundled modules known to dd-trace. CommonJS modules publish - * through the existing bundler channel. ESM modules instead rewrite only - * imports that resolve to generated live-binding proxies. + * @typedef {object} CollectState + * @property {Map} edges + */ + +/** + * @typedef {object} ResolutionState + * @property {object} ast + * @property {(code: string, sourceMap?: object) => void} callback + * @property {Function} generate + * @property {object} [inputSourceMap] + * @property {{ emitWarning?: (warning: Error) => void }} loaderContext + * @property {number} pending + * @property {string} resourcePath + * @property {boolean} rewritten + * @property {string} source + * @property {Record} targets + */ + +/** + * Instruments modules selected by a build plan generated from the installed + * dd-trace integrations. * * @param {string} source - * @returns {string} + * @param {object} [inputSourceMap] + * @returns {void} */ -module.exports = function loader (source) { - const { aliases, manifestPath, rewriteApplicationImports } = this.getOptions() - const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) - const target = manifest.targets[normalizePath(this.resourcePath)] || - getRelativeTarget(this.resourcePath, manifest.relativeTargets) - const esm = isESMFile(this.resourcePath) +module.exports = function loader (source, inputSourceMap) { + const callback = this.async() + try { + load.call(this, source, inputSourceMap, callback) + } catch (error) { + callback(error) + } +} - if (rewriteApplicationImports || esm) source = rewriteImports(source, this.resourcePath, manifest.targets, aliases) +/** + * @this {{ + * async: Function, + * emitWarning?: (warning: Error) => void, + * getOptions: Function, + * getResolve: Function, + * resourcePath: string + * }} + * @param {string} source + * @param {object} [inputSourceMap] + * @param {(error?: Error, code?: string, sourceMap?: object) => void} callback + */ +function load (source, inputSourceMap, callback) { + const options = this.getOptions() + const plan = getPlan(options.manifestPath, options.manifestHash) + const resourcePath = normalizePath(this.resourcePath) + const esm = isESMFile(resourcePath) + const match = findTarget(resourcePath, plan, options.targetScope, this) - if (!target) return source + /** + * @param {string} code + * @param {object} [sourceMap] + */ + function onRewritten (code, sourceMap) { + finishLoad(code, sourceMap, resourcePath, match, esm, callback) + } - source = rewrite(source, this.resourcePath, esm ? 'module' : 'commonjs') - if (esm) return source + if (plan.proxies[resourcePath] || !options.rewriteEdges || !MODULE_SYNTAX_PATTERN.test(source)) { + onRewritten(source, inputSourceMap) + return + } - const dcPolyfillPath = relativeImport( - path.dirname(this.resourcePath), - require.resolve('dc-polyfill') - ) + rewriteModuleEdges(source, inputSourceMap, resourcePath, plan.targets, options.compiler, this, onRewritten) +} - return `${source} -;{ - const __dd_dc = require(${JSON.stringify(dcPolyfillPath)}) - const __dd_ch = __dd_dc.channel('${CHANNEL}') - const __dd_payload = { - module: module.exports, - version: ${JSON.stringify(target.version)}, - package: ${JSON.stringify(target.name)}, - path: ${JSON.stringify(target.path)}, - } - __dd_ch.publish(__dd_payload) - module.exports = __dd_payload.module +/** + * @param {string} manifestPath + * @param {string} manifestHash + * @returns {BuildPlan} + */ +function getPlan (manifestPath, manifestHash) { + if (typeof manifestPath !== 'string' || typeof manifestHash !== 'string') { + throw new TypeError('The Datadog Turbopack loader requires a build plan path and hash') + } + + const key = `${manifestPath}\0${manifestHash}` + const cached = plans.get(key) + if (cached) return cached + + const serialized = fs.readFileSync(manifestPath, 'utf8') + if (hash(serialized) !== manifestHash) { + throw new Error(`The Datadog Turbopack build plan at ${manifestPath} failed its integrity check`) + } + + const plan = JSON.parse(serialized) + if (plan?.version !== PLAN_VERSION || !plan.proxies || typeof plan.proxies !== 'object' || + Array.isArray(plan.proxies) || !Array.isArray(plan.relativeTargets) || + !plan.targets || typeof plan.targets !== 'object') { + throw new Error(`The Datadog Turbopack build plan at ${manifestPath} is not supported`) + } + + if (plans.size >= MAX_CACHED_PLANS) plans.delete(plans.keys().next().value) + plans.set(key, plan) + return plan } -` + +/** + * @param {string} resourcePath + * @param {BuildPlan} plan + * @param {'direct'|'relative'} [targetScope] + * @param {{ emitWarning?: (warning: Error) => void }} loaderContext + * @returns {PlanTarget|undefined} + */ +function findTarget (resourcePath, plan, targetScope, loaderContext) { + const direct = plan.targets[resourcePath] + if (targetScope === 'direct') { + if (!direct) return + if (matchesSource(resourcePath, direct.sourceHash)) return direct + warnOnce(loaderContext, `changed:${resourcePath}`, `Skipped changed dependency ${resourcePath}`) + return + } + if (targetScope !== 'relative' || direct) return + + let sourceHash + for (const target of plan.relativeTargets) { + if (!resourcePath.endsWith(`/${target.file}`)) continue + sourceHash ??= getFileHash(resourcePath) + if (sourceHash !== target.sourceHash) continue + return target + } } /** * @param {string} source + * @param {object} [inputSourceMap] * @param {string} resourcePath - * @param {Record} targets - * @param {string[]} [aliases] - * @returns {string} + * @param {Record} targets + * @param {{ generator: string, parser: string, traverse: string }} compiler + * @param {{ emitWarning?: (warning: Error) => void, getResolve: Function }} loaderContext + * @param {(code: string, sourceMap?: object) => void} callback */ -function rewriteImports (source, resourcePath, targets, aliases = []) { - resourcePath = normalizePath(resourcePath) - let rewritten = false - const matcher = create([{ - module: { name: 'dd-trace-turbopack', versionRange: '*', filePath: /.*/ }, - astQuery: 'Program', - transform: 'rewriteImports', - }]) - - matcher.addTransform('rewriteImports', (_state, program) => { - const hasLocalRequire = declaresRequire(program) - visit(program, node => { - const source = getModuleSource(node, hasLocalRequire) - - if (!source || matchesAlias(source.value, aliases)) return - - const resolved = resolveFrom(resourcePath, source.value) - const target = resolved && targets[resolved] - if (!target?.esm || !target.proxyPath) return - - const proxySpecifier = relativeImport(path.dirname(resourcePath), target.proxyPath) - // The code transformer emits `raw` when present, so update both fields. - source.value = proxySpecifier - source.raw = JSON.stringify(proxySpecifier) - rewritten = true - }) - }) - - const transformer = matcher.getTransformer('dd-trace-turbopack', '1.0.0', resourcePath) - if (!transformer) return source +function rewriteModuleEdges (source, inputSourceMap, resourcePath, targets, compiler, loaderContext, callback) { + let ast + let generate + const state = { edges: new Map() } try { - const { code, map } = transformer.transform(source, 'esm') - return rewritten ? withInlineSourceMap(code, map) : source - } catch { - // A parser failure must never prevent an application from building. - return source + const { parse } = require(compiler.parser) + const traverse = require(compiler.traverse).default + generate = require(compiler.generator).default + const plugins = /\.(?:cts|mts|ts|tsx)$/.test(resourcePath) + ? TYPESCRIPT_PARSER_PLUGINS + : JAVASCRIPT_PARSER_PLUGINS + ast = parse(source, { plugins, sourceType: 'unambiguous' }) + traverse(ast, IMPORT_VISITORS, undefined, state) + } catch (error) { + warnOnce( + loaderContext, + `imports:${resourcePath}`, + `Could not inspect imports in ${resourcePath}: ${error.message}`, + error + ) + callback(source, inputSourceMap) + return + } + + if (state.edges.size === 0) { + callback(source, inputSourceMap) + return } + + resolveModuleEdges( + state.edges, + source, + inputSourceMap, + resourcePath, + targets, + ast, + generate, + loaderContext, + callback + ) } -function withInlineSourceMap (code, map) { - if (!map) return code - return `${code}\n${SOURCE_MAP_PREFIX}${Buffer.from(map).toString('base64')}` +/** + * @param {Map} edges + * @param {string} source + * @param {object} inputSourceMap + * @param {string} resourcePath + * @param {Record} targets + * @param {object} ast + * @param {Function} generate + * @param {{ emitWarning?: (warning: Error) => void, getResolve: Function }} loaderContext + * @param {(code: string, sourceMap?: object) => void} callback + */ +function resolveModuleEdges ( + edges, + source, + inputSourceMap, + resourcePath, + targets, + ast, + generate, + loaderContext, + callback +) { + let importResolve + let requireResolve + try { + importResolve = loaderContext.getResolve(IMPORT_RESOLVE_OPTIONS) + requireResolve = loaderContext.getResolve(REQUIRE_RESOLVE_OPTIONS) + } catch (error) { + warnOnce( + loaderContext, + `resolver:${resourcePath}`, + `Could not initialize import resolution in ${resourcePath}: ${error.message}`, + error + ) + callback(source, inputSourceMap) + return + } + + const state = { + ast, + callback, + generate, + inputSourceMap, + loaderContext, + pending: edges.size, + resourcePath, + rewritten: false, + source, + targets, + } + const directory = path.dirname(resourcePath) + for (const edge of edges.values()) { + resolveModuleEdge(edge.kind === 'require' ? requireResolve : importResolve, directory, edge, state) + } } -function getModuleSource (node, hasLocalRequire) { - if (node.type === 'ImportDeclaration' || - node.type === 'ExportNamedDeclaration' || - node.type === 'ExportAllDeclaration' || - node.type === 'ImportExpression') { - return isStringLiteral(node.source) && node.source +/** + * @param {Function} resolve + * @param {string} directory + * @param {ModuleEdge} edge + * @param {ResolutionState} state + */ +function resolveModuleEdge (resolve, directory, edge, state) { + let settled = false + + /** + * @param {Error|null|undefined} error + * @param {string} [resolved] + */ + function onResolved (error, resolved) { + if (settled) return + settled = true + if (!error && resolved) { + state.rewritten = rewriteResolvedEdge( + edge, + resolved, + state.resourcePath, + state.targets, + state.loaderContext + ) || state.rewritten + } + completeModuleEdge(state) } - if (node.type === 'CallExpression' && - node.callee?.type === 'Identifier' && node.callee.name === 'require' && - node.arguments?.length === 1) { - return !hasLocalRequire && isStringLiteral(node.arguments[0]) && node.arguments[0] + try { + resolve(directory, edge.specifier, onResolved) + } catch (error) { + onResolved(error) } } -function matchesAlias (specifier, aliases) { - return aliases.some(alias => specifier === alias || specifier.startsWith(`${alias}/`)) +/** + * @param {ResolutionState} state + */ +function completeModuleEdge (state) { + state.pending-- + if (state.pending > 0) return + if (!state.rewritten) { + state.callback(state.source, state.inputSourceMap) + return + } + + try { + const { code, map } = state.generate(state.ast, { + inputSourceMap: state.inputSourceMap, + retainLines: true, + sourceFileName: state.resourcePath, + sourceMaps: true, + }, state.source) + state.callback(code, map) + } catch (error) { + warnOnce( + state.loaderContext, + `generate:${state.resourcePath}`, + `Could not generate rewritten imports in ${state.resourcePath}: ${error.message}`, + error + ) + state.callback(state.source, state.inputSourceMap) + } } -function getRelativeTarget (resourcePath, targets = []) { - const normalizedPath = normalizePath(resourcePath) - return targets.find(target => normalizedPath.endsWith(`/${target.file}`)) +/** + * @param {{ node: object }} modulePath + * @param {CollectState} state + */ +function collectModuleDeclaration (modulePath, state) { + if (isTypeOnlyDeclaration(modulePath.node)) return + collectModuleSource(modulePath.node.source, 'import', state) } -// We deliberately decline all CommonJS rewrites in a file with a lexical -// `require` binding. Rewriting an application-defined function is worse than -// leaving an uncommon module load uninstrumented. -function declaresRequire (node) { - let declared = false - visit(node, child => { - if (child.type === 'VariableDeclarator' || child.type === 'CatchClause') { - declared ||= bindingIncludesRequire(child.id ?? child.param) - } else if (child.type === 'FunctionDeclaration' || - child.type === 'FunctionExpression' || child.type === 'ArrowFunctionExpression') { - declared ||= child.params.some(bindingIncludesRequire) || child.id?.name === 'require' - } else if (child.type === 'ImportSpecifier' || child.type === 'ImportDefaultSpecifier' || - child.type === 'ImportNamespaceSpecifier') { - declared ||= child.local?.name === 'require' - } else if (child.type === 'ClassDeclaration') { - declared ||= child.id?.name === 'require' - } - }) - return declared +/** + * @param {{ node: { arguments?: object[], callee?: object }, scope: { hasBinding: Function } }} modulePath + * @param {CollectState} state + */ +function collectCallExpression (modulePath, state) { + const { arguments: args, callee } = modulePath.node + if (callee?.type === 'Import' && args?.length > 0) { + collectModuleSource(args[0], 'import', state) + return + } + if (callee?.type === 'Identifier' && callee.name === 'require' && args?.length === 1 && + !modulePath.scope.hasBinding('require', true)) { + collectModuleSource(args[0], 'require', state) + } +} + +/** + * @param {{ node: { source?: object } }} modulePath + * @param {CollectState} state + */ +function collectImportExpression (modulePath, state) { + collectModuleSource(modulePath.node.source, 'import', state) } -function bindingIncludesRequire (node) { - if (!node || typeof node !== 'object') return false - if (node.type === 'Identifier') return node.name === 'require' - return Object.values(node).some(value => Array.isArray(value) - ? value.some(bindingIncludesRequire) - : bindingIncludesRequire(value)) +/** + * @param {{ node: { importKind?: string, moduleReference?: { expression?: object, type?: string } } }} modulePath + * @param {CollectState} state + */ +function collectTypescriptImport (modulePath, state) { + const { importKind, moduleReference } = modulePath.node + if (importKind === 'type' || moduleReference?.type !== 'TSExternalModuleReference') return + collectModuleSource(moduleReference.expression, 'require', state) +} + +/** + * @param {object|undefined} moduleSource + * @param {'import'|'require'} kind + * @param {CollectState} state + */ +function collectModuleSource (moduleSource, kind, state) { + const specifier = getModuleSpecifier(moduleSource) + if (specifier === undefined || specifier.startsWith('node:') || BUILTIN_MODULES.has(specifier)) return + + const key = `${kind}\0${specifier}` + const existing = state.edges.get(key) + if (existing) { + existing.nodes.push(moduleSource) + return + } + state.edges.set(key, { kind, nodes: [moduleSource], specifier }) } -function isStringLiteral (node) { - return node?.type === 'Literal' && typeof node.value === 'string' +/** + * @param {object|undefined} moduleSource + * @returns {string|undefined} + */ +function getModuleSpecifier (moduleSource) { + if (moduleSource?.type === 'StringLiteral') return moduleSource.value + if (moduleSource?.type === 'TemplateLiteral' && moduleSource.expressions.length === 0) { + return moduleSource.quasis[0].value.cooked + } } -function visit (node, callback) { - if (!node || typeof node !== 'object') return - callback(node) - for (const value of Object.values(node)) { - if (Array.isArray(value)) value.forEach(child => visit(child, callback)) - else visit(value, callback) +/** + * @param {object} declaration + * @returns {boolean} + */ +function isTypeOnlyDeclaration (declaration) { + if (declaration.importKind === 'type' || declaration.importKind === 'typeof' || + declaration.exportKind === 'type') return true + const typeKind = declaration.type === 'ImportDeclaration' ? 'importKind' : 'exportKind' + if ((declaration.type !== 'ImportDeclaration' && declaration.type !== 'ExportNamedDeclaration') || + declaration.specifiers.length === 0) return false + for (const specifier of declaration.specifiers) { + if (specifier[typeKind] !== 'type' && specifier[typeKind] !== 'typeof') return false } + return true +} + +const IMPORT_VISITORS = { + CallExpression: collectCallExpression, + ExportAllDeclaration: collectModuleDeclaration, + ExportNamedDeclaration: collectModuleDeclaration, + ImportDeclaration: collectModuleDeclaration, + ImportExpression: collectImportExpression, + TSImportEqualsDeclaration: collectTypescriptImport, } -function resolveFrom (resourcePath, specifier) { +/** + * @param {ModuleEdge} edge + * @param {string} resolved + * @param {string} resourcePath + * @param {Record} targets + * @param {{ emitWarning?: (warning: Error) => void }} loaderContext + * @returns {boolean} + */ +function rewriteResolvedEdge (edge, resolved, resourcePath, targets, loaderContext) { + let resolvedPath try { - return normalizePath(require.resolve(specifier, { - paths: [path.dirname(resourcePath)], - conditions: new Set(['import', 'node']), - })) - } catch {} + resolvedPath = normalizePath(resolved) + } catch { + return false + } + + const target = targets[resolvedPath] + if (!target?.esm || !target.proxyPath) return false + if (!matchesSource(resolvedPath, target.sourceHash)) { + warnOnce(loaderContext, `changed:${resolvedPath}`, `Skipped changed dependency ${resolvedPath}`) + return false + } + + const replacement = relativeImport(path.dirname(resourcePath), target.proxyPath) + for (const node of edge.nodes) setModuleSpecifier(node, replacement) + return true +} + +/** + * @param {object} moduleSource + * @param {string} value + */ +function setModuleSpecifier (moduleSource, value) { + if (moduleSource.type === 'StringLiteral') { + moduleSource.value = value + moduleSource.extra = undefined + return + } + + const templateValue = moduleSource.quasis[0].value + templateValue.cooked = value + templateValue.raw = value + .replaceAll('\\', '\\\\') + .replaceAll('`', '\\`') + .replaceAll('${', '\\${') +} + +/** + * @param {string} source + * @param {object} [sourceMap] + * @param {string} resourcePath + * @param {PlanTarget} [match] + * @param {boolean} esm + * @param {(error?: Error, code?: string, sourceMap?: object) => void} callback + */ +function finishLoad (source, sourceMap, resourcePath, match, esm, callback) { + if (!match) { + callback(undefined, source, sourceMap) + return + } + + const rewritten = rewriteBundledWithSourceMap( + source, + resourcePath, + esm ? 'module' : 'commonjs', + undefined, + sourceMap + ) + const code = esm ? rewritten.code : appendCommonJsPublications(rewritten.code, resourcePath, match) + callback(undefined, code, rewritten.map) +} + +/** + * @param {string} source + * @param {string} resourcePath + * @param {{ payloads: object[] }} match + * @returns {string} + */ +function appendCommonJsPublications (source, resourcePath, match) { + let publications = '' + let publicationIndex = 0 + + for (const payload of match.payloads) { + const payloadName = `payload${publicationIndex++}` + publications += ` const ${payloadName} = { + instrumentationIndexes: ${JSON.stringify(payload.instrumentationIndexes)}, + module: module.exports, + moduleName: ${JSON.stringify(payload.moduleName)}, + package: ${JSON.stringify(payload.package)}, + path: ${JSON.stringify(payload.path)}, + version: ${JSON.stringify(payload.version)}, + } + channel.publish(${payloadName}) + module.exports = ${payloadName}.module +` + } + + return `${source} +{ + /* eslint-disable @stylistic/quotes */ + // eslint-disable-next-line n/no-unsupported-features/node-builtins + const nativeDc = require('node:diagnostics_channel') + const dc = globalThis[Symbol.for(${JSON.stringify(BUNDLER_DC_GLOBAL)})] ?? nativeDc + const channel = dc.channel('${CHANNEL}') + if (channel.hasSubscribers) { +${publications} } +} +` +} + +/** + * @param {string} file + * @param {string} expectedHash + * @returns {boolean} + */ +function matchesSource (file, expectedHash) { + return getFileHash(file) === expectedHash +} + +/** + * @param {string} file + * @returns {string} + */ +function getFileHash (file) { + const { ctimeMs, mtimeMs, size } = fs.statSync(file) + const cached = fileHashes.get(file) + if (cached?.ctimeMs === ctimeMs && cached.mtimeMs === mtimeMs && cached.size === size) return cached.hash + + const value = hash(fs.readFileSync(file)) + if (fileHashes.size >= MAX_CACHED_FILES) fileHashes.delete(fileHashes.keys().next().value) + fileHashes.set(file, { ctimeMs, hash: value, mtimeMs, size }) + return value +} + +/** + * @param {string|Buffer} value + * @returns {string} + */ +function hash (value) { + return createHash('sha256').update(value).digest('hex') } function relativeImport (from, to) { @@ -192,4 +607,17 @@ function normalizePath (value) { return fs.realpathSync(value).replaceAll('\\', '/') } -module.exports.rewriteImports = rewriteImports +/** + * @param {{ emitWarning?: (warning: Error) => void }} loaderContext + * @param {string} key + * @param {string} message + * @param {Error} [cause] + */ +function warnOnce (loaderContext, key, message, cause) { + if (warnedErrors.has(key) || warnedErrors.size >= MAX_WARNINGS) return + warnedErrors.add(key) + const warning = new Error(message, { cause }) + warning.name = 'DatadogTurbopackWarning' + if (typeof loaderContext.emitWarning === 'function') loaderContext.emitWarning(warning) + else process.emitWarning(warning) +} diff --git a/packages/datadog-turbopack/src/targets.js b/packages/datadog-turbopack/src/targets.js index 0ac7424e204..201b4bc1459 100644 --- a/packages/datadog-turbopack/src/targets.js +++ b/packages/datadog-turbopack/src/targets.js @@ -1,228 +1,356 @@ 'use strict' -const fs = require('node:fs/promises') +const { createHash, randomUUID } = require('node:crypto') const fsSync = require('node:fs') -const Module = require('node:module') +const fs = require('node:fs/promises') const path = require('node:path') -const { createHash } = require('node:crypto') +const enhancedResolve = require('enhanced-resolve') + +const { BUNDLER_DC_GLOBAL } = require('../../datadog-instrumentations/src/helpers/bundler-constants') const instrumentations = require('../../datadog-instrumentations/src/helpers/instrumentations') const hooks = require('../../datadog-instrumentations/src/helpers/hooks') const { filename, - getDisabledInstrumentations, matchVersion, } = require('../../datadog-instrumentations/src/helpers/instrumentation-utils') const { isESMFile, processModule } = require('../../datadog-esbuild/src/utils') const CACHE_DIRECTORY = path.join('node_modules', '.cache', 'dd-trace', 'turbopack') +const CHANNEL = 'dd-trace:bundler:load' +const MAX_WARNINGS = 128 +const PLAN_VERSION = 3 +const TRAILING_WHITESPACE = /[ \t]+$/gm +const resolveImport = enhancedResolve.create.sync({ conditionNames: ['node', 'import'] }) +const resolveRequire = enhancedResolve.create.sync({ conditionNames: ['node', 'require'] }) + +/** @type {Set} */ +const emittedWarnings = new Set() /** - * Builds the Turbopack manifest and generated ESM proxies for supported - * instrumentation targets installed in an application. + * @typedef {object} InstrumentationPayload + * @property {number[]} instrumentationIndexes + * @property {string} moduleName + * @property {string} package + * @property {string} path + * @property {string} version + */ + +/** + * @typedef {object} Target + * @property {boolean} esm + * @property {Array<{ hook: Function, payload: InstrumentationPayload, version: string }>} matches + * @property {InstrumentationPayload[]} payloads + * @property {string} path + * @property {Set} rulePaths + * @property {string} sourceHash + * @property {string[]} [setters] + */ + +/** + * Compiles installed integration targets into immutable build artifacts. * * @param {string} projectDir - * @returns {Promise<{ esmImportPattern?: RegExp, hash?: string, packagePathPattern?: RegExp, path?: string }>} + * @returns {Promise<{ + * hash?: string, + * moduleSyntaxPattern?: RegExp, + * packagePathPattern?: RegExp, + * path?: string, + * relativePathPattern?: RegExp, + * targetPathPattern?: RegExp + * }>} */ -async function createManifest (projectDir) { +async function createBuildPlan (projectDir) { projectDir = path.resolve(projectDir) - const disabledInstrumentations = getDisabledInstrumentations() loadInstrumentations() - const appRequire = Module.createRequire(path.join(projectDir, 'package.json')) - const cacheDirectory = path.join(projectDir, CACHE_DIRECTORY) - const targets = getTargets(appRequire, disabledInstrumentations, projectDir) - const relativeTargets = getRelativeTargets(targets, disabledInstrumentations) - const manifestTargets = {} + const targets = getTargets(projectDir) + const compiledTargets = [] + let includesEsmTarget = false - if (targets.length === 0 && relativeTargets.length === 0) return {} + for (const target of targets) { + try { + target.sourceHash = hash(fsSync.readFileSync(target.path)) + if (target.esm) { + // Export discovery is asynchronous in import-in-the-middle and belongs at build time. + // eslint-disable-next-line no-await-in-loop + const setters = await processModule({ path: target.path, context: { format: 'module' } }) + target.setters = [...setters.values()].map(setter => setter.replaceAll(TRAILING_WHITESPACE, '')) + includesEsmTarget = true + } + compiledTargets.push(target) + } catch (error) { + warnOnce(`target:${target.path}`, `Could not instrument ${target.path}: ${error.message}`) + } + } + + const relativeTargets = getRelativeTargets(compiledTargets) + if (compiledTargets.length === 0 && relativeTargets.length === 0) return {} + compiledTargets.sort(compareTargets) + relativeTargets.sort(compareRelativeTargets) + const identity = createPlanIdentity(compiledTargets, relativeTargets) + const planId = hash(identity) + const artifactDirectory = path.join(projectDir, CACHE_DIRECTORY, planId) try { - await fs.mkdir(cacheDirectory, { recursive: true }) - } catch { - return {} + await fs.mkdir(artifactDirectory, { recursive: true }) + } catch (error) { + throw new Error(`Could not create the Datadog Turbopack cache at ${artifactDirectory}: ${error.message}`, { + cause: error, + }) } - const realCacheDirectory = normalizePath(cacheDirectory) + const realArtifactDirectory = normalizePath(artifactDirectory) + const planProxies = {} + const planTargets = {} - for (const [index, target] of targets.entries()) { + for (const target of compiledTargets) { const entry = { esm: target.esm, - name: target.name, - path: target.instrumentationPath, - version: target.version, + payloads: target.payloads, + sourceHash: target.sourceHash, } if (target.esm) { - const proxyPath = path.join(realCacheDirectory, `${index}.mjs`) - try { - // Proxies are build-time artifacts; preserve source order for stable paths. - // eslint-disable-next-line no-await-in-loop - await fs.writeFile(proxyPath, await createEsmProxy( - target.path, proxyPath, target.name, target.specifier, target.version - )) - } catch { - // An unsupported dependency must not prevent the customer's build. Its - // original module remains bundled without instrumentation instead. - continue - } + const proxy = createEsmProxy(target, path.join(realArtifactDirectory, 'proxy.mjs')) + const proxyId = hash(proxy) + const proxyPath = path.join(realArtifactDirectory, `${proxyId}.mjs`) + // Files are content-addressed and safe for concurrent config evaluation. + // eslint-disable-next-line no-await-in-loop + await writeArtifact(proxyPath, proxy) entry.proxyPath = normalizePath(proxyPath) + planProxies[entry.proxyPath] = true } - manifestTargets[normalizePath(target.path)] = entry - } - - const manifestPath = path.join(realCacheDirectory, 'manifest.json') - const manifest = JSON.stringify({ relativeTargets, targets: manifestTargets }) - try { - await fs.writeFile(manifestPath, manifest) - } catch { - return {} + planTargets[target.path] = entry } - const packageNames = [...new Set(targets.map(target => target.name))] - const packagePathPattern = new RegExp( - `(?:^|/)node_modules/(?:${packageNames.map(escapeRegExp).join('|')})(?:/|$)` - ) - const esmPackageNames = [...new Set(targets.filter(target => target.esm).map(target => target.name))] - const esmPackagePattern = esmPackageNames.map(escapeRegExp).join('|') - const esmImportPattern = esmPackageNames.length > 0 && new RegExp( - String.raw`\b(?:from\s*|import\s*(?:\(\s*)?|require\s*\(\s*)["'](?:${esmPackagePattern})(?:/[^"']*)?["']` - ) - - const relativePathPattern = relativeTargets.length > 0 && new RegExp( - `(?:^|/)(?:${relativeTargets.map(target => escapeRegExp(target.file)).join('|')})$` - ) + const plan = JSON.stringify({ + proxies: planProxies, + relativeTargets, + targets: planTargets, + version: PLAN_VERSION, + }) + const planHash = hash(plan) + const planPath = path.join(realArtifactDirectory, `${planHash}.json`) + await writeArtifact(planPath, plan) return { - esmImportPattern, - hash: createHash('sha256').update(manifest).digest('hex'), - packagePathPattern, - path: manifestPath, - relativePathPattern, + hash: planHash, + moduleSyntaxPattern: includesEsmTarget ? /\b(?:export|import|require)\b/ : undefined, + packagePathPattern: createPackagePathPattern(compiledTargets), + path: planPath, + relativePathPattern: createRelativePathPattern(relativeTargets), + targetPathPattern: createTargetPathPattern(compiledTargets), } } -/** - * Ensures the existing instrumentation declarations have populated their - * shared registry before we inspect it at build time. - */ +/** Loads each instrumentation declaration before target discovery. */ function loadInstrumentations () { - const disabledInstrumentations = getDisabledInstrumentations() for (const [name, hook] of Object.entries(hooks)) { - if (disabledInstrumentations.has(name)) continue const load = hook?.fn ?? hook - if (typeof load === 'function') load() + if (typeof load !== 'function') continue + + try { + load() + } catch (error) { + warnOnce(`hook:${name}`, `Could not load the ${name} instrumentation: ${error.message}`) + } } } /** - * @param {Function & { resolve: Function }} appRequire - * @param {Set} [disabledInstrumentations] - * @param {string} [projectDir] - * @returns {Array<{ - * esm: boolean, instrumentationPath: string, name: string, - * path: string, specifier: string, version: string - * }>} + * @param {string} projectDir + * @returns {Target[]} */ -function getTargets (appRequire, disabledInstrumentations = new Set(), projectDir) { +function getTargets (projectDir) { const targets = new Map() - const packageNames = new Set( - Object.keys(instrumentations).filter(name => !name.startsWith('node:') && !name.startsWith('.')) - ) - const packageRootsByName = projectDir ? findPackageRoots(projectDir, packageNames) : new Map() + const packageNames = new Set() + + for (const name of Object.keys(instrumentations)) { + if (!name.startsWith('node:') && !name.startsWith('.')) packageNames.add(name) + } + const packageRootsByName = findPackageRoots(projectDir, packageNames) for (const [name, entries] of Object.entries(instrumentations)) { - if (name.startsWith('node:') || name.startsWith('.') || disabledInstrumentations.has(name)) continue + if (name.startsWith('node:') || name.startsWith('.')) continue const packageRoots = [...(packageRootsByName.get(name) ?? [])] if (packageRoots.length === 0) { - try { - const entrypoint = resolveImport(appRequire, name) - const packageRoot = findPackageRoot(entrypoint) - if (packageRoot) packageRoots.push(packageRoot) - } catch { - continue - } + addResolvedPackageRoot(packageRoots, projectDir, name, resolveImport) + addResolvedPackageRoot(packageRoots, projectDir, name, resolveRequire) } - for (const packageRoot of packageRoots) { - addTargets(targets, packageRoot, name, entries) - } + for (const packageRoot of packageRoots) addTargets(targets, packageRoot, name, entries) } return [...targets.values()] } -function getRelativeTargets (targets, disabledInstrumentations) { - const relativeTargets = [] +/** + * @param {Target[]} targets + * @returns {Array<{ file: string, payloads: InstrumentationPayload[], sourceHash: string }>} + */ +function getRelativeTargets (targets) { + const relativeTargets = new Map() + const ambiguousTargets = new Set() for (const [name, entries] of Object.entries(instrumentations)) { - if (!name.startsWith('.') || disabledInstrumentations.has(name)) continue + if (!name.startsWith('.')) continue - for (const entry of entries) { + for (let index = 0; index < entries.length; index++) { + const entry = entries[index] if (!entry.file) continue - const compatibleTarget = targets.find(target => matchVersion(target.version, entry.versions) && - instrumentations[target.name]?.some(candidate => - candidate.hook === entry.hook && matchVersion(target.version, candidate.versions) - ) - ) - if (!compatibleTarget) continue - - relativeTargets.push({ - file: entry.file, - name, - path: name, - version: compatibleTarget.version, - }) + + for (const target of targets) { + for (const match of target.matches) { + if (match.hook !== entry.hook || !matchVersion(match.version, entry.versions)) continue + + const key = `${entry.file}\0${target.sourceHash}` + if (ambiguousTargets.has(key)) continue + + const existing = relativeTargets.get(key) + if (existing && existing.payloads[0].version !== match.version) { + relativeTargets.delete(key) + ambiguousTargets.add(key) + continue + } + + if (existing) { + if (!existing.payloads[0].instrumentationIndexes.includes(index)) { + existing.payloads[0].instrumentationIndexes.push(index) + } + continue + } + + relativeTargets.set(key, { + file: entry.file.replaceAll('\\', '/'), + payloads: [{ + instrumentationIndexes: [index], + moduleName: name, + package: name, + path: name, + version: match.payload.version, + }], + sourceHash: target.sourceHash, + }) + } + } } } - return relativeTargets + return [...relativeTargets.values()] } +/** + * @param {Map} targets + * @param {string} packageRoot + * @param {string} name + * @param {Array} entries + */ function addTargets (targets, packageRoot, name, entries) { let packageJson - let entrypoint try { packageJson = JSON.parse(fsSync.readFileSync(path.join(packageRoot, 'package.json'), 'utf8')) - entrypoint = resolvePackageEntrypoint(packageRoot, name) } catch { return } - for (const entry of entries) { + let entrypoints + for (let index = 0; index < entries.length; index++) { + const entry = entries[index] if (!matchVersion(packageJson.version, entry.versions)) continue let files try { - files = entry.file - ? [path.join(packageRoot, entry.file)] - : entry.filePattern - ? findMatchingFiles(packageRoot, new RegExp(entry.filePattern)) - : [entrypoint] + if (entry.file) { + files = [path.join(packageRoot, entry.file)] + } else if (entry.filePattern) { + files = findMatchingFiles(packageRoot, new RegExp(entry.filePattern)) + } else { + entrypoints ??= resolvePackageEntrypoints(packageRoot, name) + files = entrypoints + } } catch { continue } for (const file of files) { if (!fsSync.existsSync(file)) continue - const modulePath = entry.file || (entry.filePattern && - path.relative(packageRoot, file).replaceAll('\\', '/')) - - targets.set(normalizePath(file), { - esm: isESMFile(file, path.join(packageRoot, 'package.json'), packageJson), - instrumentationPath: filename(name, modulePath), - name, - path: file, - specifier: modulePath ? `${name}/${modulePath}` : name, - version: packageJson.version, - }) + + const targetPath = normalizePath(file) + const relativePath = path.relative(packageRoot, file).replaceAll('\\', '/') + const modulePath = entry.file || (entry.filePattern && relativePath) + const moduleName = filename(name, modulePath) + let target = targets.get(targetPath) + if (!target) { + target = { + esm: isESMFile(file, path.join(packageRoot, 'package.json'), packageJson), + matches: [], + path: targetPath, + payloads: [], + rulePaths: new Set(), + sourceHash: '', + } + targets.set(targetPath, target) + } + target.rulePaths.add(`${name}/${relativePath}`) + target.rulePaths.add(`${path.basename(packageRoot)}/${relativePath}`) + + let payload = findPayload(target.payloads, name, moduleName, packageJson.version) + if (!payload) { + payload = { + instrumentationIndexes: [], + moduleName, + package: name, + path: moduleName, + version: packageJson.version, + } + target.payloads.push(payload) + } + payload.instrumentationIndexes.push(index) + target.matches.push({ hook: entry.hook, payload, version: packageJson.version }) } } } +/** + * @param {string[]} packageRoots + * @param {string} directory + * @param {string} name + * @param {(directory: string, specifier: string) => string} resolve + */ +function addResolvedPackageRoot (packageRoots, directory, name, resolve) { + let entrypoint + try { + entrypoint = resolve(directory, name) + } catch { + return + } + + const packageRoot = findPackageRoot(entrypoint) + if (packageRoot && !packageRoots.includes(packageRoot)) packageRoots.push(packageRoot) +} + +/** + * @param {InstrumentationPayload[]} payloads + * @param {string} name + * @param {string} moduleName + * @param {string} version + * @returns {InstrumentationPayload|undefined} + */ +function findPayload (payloads, name, moduleName, version) { + for (const payload of payloads) { + if (payload.package === name && payload.moduleName === moduleName && payload.version === version) return payload + } +} + // Visit package boundaries only: this reaches nested dependency copies without // walking each package's source tree during Next configuration. +/** + * @param {string} projectDir + * @param {Set} names + * @returns {Map>} + */ function findPackageRoots (projectDir, names) { const packageRoots = new Map() const pending = [path.join(projectDir, 'node_modules')] @@ -249,7 +377,9 @@ function findPackageRoots (projectDir, names) { if (entry.name === '.bin') continue if (!entry.isDirectory() && !entry.isSymbolicLink()) continue const entryPath = path.join(directory, entry.name) - if (entry.name.startsWith('@')) { + if (entry.name === '.pnpm') { + addPnpmPackageRoots(entryPath, pending) + } else if (entry.name.startsWith('@')) { addScopedPackageRoots(entryPath, names, packageRoots, pending) } else { addPackageRoot(entryPath, entry.name, names, packageRoots, pending) @@ -260,6 +390,30 @@ function findPackageRoots (projectDir, names) { return packageRoots } +/** + * @param {string} storePath + * @param {string[]} pending + */ +function addPnpmPackageRoots (storePath, pending) { + let entries + try { + entries = fsSync.readdirSync(storePath, { withFileTypes: true }) + } catch { + return + } + for (const entry of entries) { + if (!entry.isDirectory() && !entry.isSymbolicLink()) continue + const entryPath = path.join(storePath, entry.name) + pending.push(entry.name === 'node_modules' ? entryPath : path.join(entryPath, 'node_modules')) + } +} + +/** + * @param {string} scopePath + * @param {Set} names + * @param {Map>} packageRoots + * @param {string[]} pending + */ function addScopedPackageRoots (scopePath, names, packageRoots, pending) { let entries try { @@ -279,6 +433,13 @@ function addScopedPackageRoots (scopePath, names, packageRoots, pending) { } } +/** + * @param {string} packageRoot + * @param {string} packageName + * @param {Set} names + * @param {Map>} packageRoots + * @param {string[]} pending + */ function addPackageRoot (packageRoot, packageName, names, packageRoots, pending) { let realPackageRoot try { @@ -295,41 +456,34 @@ function addPackageRoot (packageRoot, packageName, names, packageRoots, pending) pending.push(path.join(realPackageRoot, 'node_modules')) } -function createPackageRequire (packageRoot) { - const nodeModules = findNodeModulesRoot(packageRoot) - return Module.createRequire(path.join( - nodeModules ? path.dirname(nodeModules) : packageRoot, - 'package.json' - )) -} - -function findNodeModulesRoot (packageRoot) { - let directory = packageRoot - while (directory !== path.dirname(directory)) { - if (path.basename(directory) === 'node_modules') return directory - directory = path.dirname(directory) +/** + * @param {string} packageRoot + * @param {string} name + * @returns {string[]} + */ +function resolvePackageEntrypoints (packageRoot, name) { + const entrypoints = new Set() + + addResolvedEntrypoint(entrypoints, packageRoot, name, resolveImport) + addResolvedEntrypoint(entrypoints, packageRoot, name, resolveRequire) + if (entrypoints.size === 0) { + addResolvedEntrypoint(entrypoints, packageRoot, '.', resolveImport) + addResolvedEntrypoint(entrypoints, packageRoot, '.', resolveRequire) } -} -function resolvePackageEntrypoint (packageRoot, name) { - const packageRequire = createPackageRequire(packageRoot) - try { - return resolveImport(packageRequire, name) - } catch { - return resolveImport(Module.createRequire(path.join(packageRoot, 'package.json')), '.') - } + return [...entrypoints] } /** - * Resolves with import conditions so an ESM package uses the same entrypoint - * as a Turbopack Node bundle instead of the CommonJS require entrypoint. - * - * @param {Function & { resolve: Function }} appRequire + * @param {Set} entrypoints + * @param {string} directory * @param {string} specifier - * @returns {string} + * @param {(directory: string, specifier: string) => string} resolve */ -function resolveImport (appRequire, specifier) { - return appRequire.resolve(specifier, { conditions: new Set(['import', 'node']) }) +function addResolvedEntrypoint (entrypoints, directory, specifier, resolve) { + try { + entrypoints.add(resolve(directory, specifier)) + } catch {} } /** @@ -368,55 +522,201 @@ function findMatchingFiles (directory, pattern) { } /** - * @param {string} sourcePath + * @param {Target[]} targets + * @param {Array<{ file: string, payloads: InstrumentationPayload[], sourceHash: string }>} relativeTargets + * @returns {string} + */ +function createPlanIdentity (targets, relativeTargets) { + const identityTargets = [] + for (const target of targets) { + identityTargets.push({ + esm: target.esm, + path: target.path, + payloads: target.payloads, + setters: target.setters, + sourceHash: target.sourceHash, + }) + } + return JSON.stringify({ relativeTargets, targets: identityTargets, version: PLAN_VERSION }) +} + +/** + * @param {Target} target * @param {string} proxyPath - * @param {string} name - * @param {string} specifier - * @param {string} version - * @returns {Promise} + * @returns {string} */ -async function createEsmProxy (sourcePath, proxyPath, name, specifier, version) { - const setters = await processModule({ path: sourcePath, context: { format: 'module' } }) - const dcPolyfillPath = relativeImport( - path.dirname(proxyPath), - require.resolve('dc-polyfill') - ) - return `import dc from ${JSON.stringify(dcPolyfillPath)}; -import * as namespace from ${JSON.stringify(relativeImport(path.dirname(proxyPath), sourcePath))}; -const _ = Object.create(null, { [Symbol.toStringTag]: { value: 'Module' } }); -const set = {}; -const get = {}; -${[...setters.values()].join(';\n')}; -dc.channel('dd-trace:bundler:load').publish({ - package: ${JSON.stringify(name)}, - module: _, - path: ${JSON.stringify(specifier)}, - version: ${JSON.stringify(version)}, - apply (exports, patchDefault) { - if (patchDefault) return set.default?.(exports); - for (const name of Object.keys(exports)) set[name]?.(exports[name]); - }, -}); +function createEsmProxy (target, proxyPath) { + let publications = '' + let publicationIndex = 0 + + for (const payload of target.payloads) { + const payloadName = `payload${publicationIndex++}` + publications += ` const ${payloadName} = { + instrumentationIndexes: ${JSON.stringify(payload.instrumentationIndexes)}, + module: _, + moduleName: ${JSON.stringify(payload.moduleName)}, + package: ${JSON.stringify(payload.package)}, + path: ${JSON.stringify(payload.path)}, + version: ${JSON.stringify(payload.version)}, + apply (exports, patchDefault) { + if (patchDefault) return set.default?.(exports) + for (const name of Object.keys(exports)) set[name]?.(exports[name]) + }, + } + channel.publish(${payloadName}) +` + } + + return `/* eslint-disable @stylistic/quotes, @stylistic/semi */ +/* eslint-disable @stylistic/comma-spacing, dot-notation, import/no-mutable-exports */ +/* eslint-disable indent */ +import nativeDc from 'node:diagnostics_channel' +import * as namespace from + ${JSON.stringify(relativeImport(path.dirname(proxyPath), target.path))} +const dc = globalThis[Symbol.for(${JSON.stringify(BUNDLER_DC_GLOBAL)})] ?? nativeDc +const _ = Object.create(null, { [Symbol.toStringTag]: { value: 'Module' } }) +const set = {} +const get = {} +${target.setters.join(';\n')} +const channel = dc.channel('${CHANNEL}') +if (channel.hasSubscribers) { +${publications}} ` } +/** + * @param {Target[]} targets + * @returns {RegExp|undefined} + */ +function createPackagePathPattern (targets) { + const packageNames = new Set() + for (const target of targets) { + for (const payload of target.payloads) packageNames.add(payload.package) + } + return new RegExp(`(?:^|/)node_modules/(?:${[...packageNames].sort().map(escapeRegExp).join('|')})(?:/|$)`) +} + +/** + * @param {Array<{ file: string }>} relativeTargets + * @returns {RegExp|undefined} + */ +function createRelativePathPattern (relativeTargets) { + if (relativeTargets.length === 0) return + const files = new Set() + for (const target of relativeTargets) files.add(target.file) + return new RegExp(`(?:^|/)(?:${[...files].sort().map(escapeRegExp).join('|')})$`) +} + +/** + * @param {Target[]} targets + * @returns {RegExp} + */ +function createTargetPathPattern (targets) { + const paths = new Set() + for (const target of targets) { + paths.add(target.path) + for (const rulePath of target.rulePaths) paths.add(rulePath) + } + return new RegExp(`(?:^|/)(?:${[...paths].sort().map(escapeRegExp).join('|')})$`) +} + +/** + * @param {string} file + * @param {string} content + * @returns {Promise} + */ +async function writeArtifact (file, content) { + try { + const existing = await fs.readFile(file, 'utf8') + if (existing === content) return + throw new Error(`The Datadog Turbopack artifact at ${file} does not match its content address`) + } catch (error) { + if (error.code !== 'ENOENT') throw error + } + + const temporaryFile = `${file}.${process.pid}.${randomUUID()}.tmp` + await fs.writeFile(temporaryFile, content, { flag: 'wx' }) + try { + await fs.link(temporaryFile, file) + } catch (error) { + if (error.code !== 'EEXIST') throw error + const existing = await fs.readFile(file, 'utf8') + if (existing !== content) { + throw new Error(`The Datadog Turbopack artifact at ${file} does not match its content address`) + } + } finally { + try { + await fs.unlink(temporaryFile) + } catch (error) { + warnOnce( + `cleanup:${temporaryFile}`, + `Could not remove temporary Turbopack artifact ${temporaryFile}: ${error.message}` + ) + } + } +} + +/** + * @param {Target} left + * @param {Target} right + * @returns {number} + */ +function compareTargets (left, right) { + return left.path.localeCompare(right.path) +} + +/** + * @param {{ file: string, sourceHash: string }} left + * @param {{ file: string, sourceHash: string }} right + * @returns {number} + */ +function compareRelativeTargets (left, right) { + return left.file.localeCompare(right.file) || left.sourceHash.localeCompare(right.sourceHash) +} + +/** + * @param {string} from + * @param {string} to + * @returns {string} + */ function relativeImport (from, to) { - let value = path.relative(from, to).replaceAll('\\', '/') - if (!value.startsWith('.')) value = `./${value}` - return value + return path.relative(from, to).replaceAll('\\', '/') } +/** + * @param {string} value + * @returns {string} + */ function normalizePath (value) { return fsSync.realpathSync(value).replaceAll('\\', '/') } +/** + * @param {string|Buffer} value + * @returns {string} + */ +function hash (value) { + return createHash('sha256').update(value).digest('hex') +} + +/** + * @param {string} value + * @returns {string} + */ function escapeRegExp (value) { return value.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`) } +/** + * @param {string} key + * @param {string} message + */ +function warnOnce (key, message) { + if (emittedWarnings.has(key) || emittedWarnings.size >= MAX_WARNINGS) return + emittedWarnings.add(key) + process.emitWarning(message, { code: 'DD_TRACE_TURBOPACK' }) +} + module.exports = { - createEsmProxy, - createManifest, - getRelativeTargets, - getTargets, + createBuildPlan, } diff --git a/packages/datadog-turbopack/test/config.spec.js b/packages/datadog-turbopack/test/config.spec.js new file mode 100644 index 00000000000..a5483066ad1 --- /dev/null +++ b/packages/datadog-turbopack/test/config.spec.js @@ -0,0 +1,563 @@ +'use strict' + +const assert = require('node:assert/strict') +const { createHash } = require('node:crypto') +const fs = require('node:fs') +const fsPromises = require('node:fs/promises') +const path = require('node:path') +const { pathToFileURL } = require('node:url') +const { afterEach, describe, it } = require('mocha') +const sinon = require('sinon') + +const { withDatadogTurbopack } = require('../../../next') +const hooks = require('../../datadog-instrumentations/src/helpers/hooks') +const instrumentations = require('../../datadog-instrumentations/src/helpers/instrumentations') +const { + cleanup, + createPackage, + createProject, + findDatadogLoaders, + write, +} = require('./helpers') + +afterEach(() => { + cleanup() + sinon.restore() +}) + +describe('withDatadogTurbopack', () => { + it('exports the wrapper to CommonJS and ESM configurations', async () => { + const namespace = await import(pathToFileURL(require.resolve('../../../next')).href) + + assert.strictEqual(namespace.withDatadogTurbopack, withDatadogTurbopack) + }) + + it('discovers integrations independently of build-process disablement', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const previous = process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS + process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS = 'ioredis' + + try { + const config = await withDatadogTurbopack({}, { projectDir }) + assert.ok(config.turbopack.rules['*.js']) + } finally { + if (previous === undefined) delete process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS + else process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS = previous + } + }) + + it('uses Next 16 rule conditions and preserves existing configuration', async () => { + const projectDir = createProject('16.2.0') + const packageDir = createPackage(projectDir, 'ai', { main: 'index.mjs', type: 'module', version: '7.0.0' }) + write(packageDir, 'index.mjs', [ + 'export function generateText () {}', + 'export function streamText () {}', + '', + ].join('\n')) + const input = { + marker: true, + turbopack: { + resolveAlias: { existing: './existing.js' }, + rules: { + '*.cjs': [{ loaders: ['existing-array-loader'] }], + '*.js': { loaders: ['existing-loader'] }, + }, + }, + } + + const config = await withDatadogTurbopack(input, { projectDir }) + const rules = config.turbopack.rules['*.js'] + const targetRule = rules.find(rule => rule.condition?.all?.some(condition => condition?.any)) + const importRule = rules.find(rule => rule.condition?.all?.some(condition => condition?.content)) + + assert.equal(config.marker, true) + assert.equal(config.turbopack.resolveAlias.existing, './existing.js') + assert.deepEqual(rules[0], { loaders: ['existing-loader'] }) + assert.deepEqual(config.turbopack.rules['*.cjs'][0], { loaders: ['existing-array-loader'] }) + assert.equal(targetRule.condition.all[0], 'node') + assert.equal(targetRule.condition.all[1].any.length, 2) + assert.equal(importRule.condition.all[0], 'node') + assert.equal(importRule.condition.all.some(condition => condition?.not === 'foreign'), true) + assert.equal(importRule.loaders[0].options.rewriteEdges, true) + assert.equal(importRule.loaders[0].options.targetScope, undefined) + assert.equal(targetRule.loaders[0].options.rewriteEdges, true) + assert.equal(targetRule.loaders[0].options.targetScope, 'direct') + const contentPattern = importRule.condition.all.find(condition => condition?.content).content + assert.equal(contentPattern.test("import /* webpackChunkName: 'ai' */ ('ai')"), true) + assert.equal(contentPattern.test('const answer = 42'), false) + }) + + it('uses named conditions and nested built-ins for Next 15', async () => { + const projectDir = createProject('15.5.0') + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const aiDirectory = createPackage(projectDir, 'ai', { main: 'index.mjs', type: 'module', version: '7.0.0' }) + write(aiDirectory, 'index.mjs', 'export function generateText () {}\n') + const prismaDirectory = createPackage(projectDir, '@prisma/client', { main: 'index.js', version: '6.1.0' }) + write(prismaDirectory, 'index.js', 'module.exports = {}') + write(prismaDirectory, 'runtime/library.js', 'module.exports = {}\n') + + const config = await withDatadogTurbopack({}, { projectDir }) + const name = '#dd-trace/target' + const rule = config.turbopack.rules[name] + const loader = rule.node.loaders[0] + + assert.ok(config.turbopack.conditions[name].path instanceof RegExp) + assert.equal(config.turbopack.conditions[name].path.test('/app/node_modules/ioredis/index.js'), true) + assert.equal(config.turbopack.conditions[name].path.test('/app/node_modules/ioredis/package.json'), false) + assert.equal(Object.keys(config.turbopack.conditions).length, 3) + assert.ok(config.turbopack.conditions['#dd-trace/import'].content instanceof RegExp) + assert.ok(config.turbopack.conditions['#dd-trace/relative'].path instanceof RegExp) + assert.equal(config.turbopack.rules['#dd-trace/import'].node.foreign, false) + assert.equal(config.turbopack.rules['#dd-trace/import'].node.loaders.length, 1) + assert.equal(config.turbopack.rules['#dd-trace/import'].node.loaders[0].options.rewriteEdges, true) + assert.equal(config.turbopack.rules['#dd-trace/relative'].node.foreign, false) + assert.equal(config.turbopack.rules['#dd-trace/relative'].node.loaders[0].options.targetScope, 'relative') + assert.equal(rule.condition, undefined) + assert.equal(typeof loader.loader, 'string') + assert.match(loader.options.manifestHash, /^[a-f\d]{64}$/) + JSON.stringify(loader.options) + + const repeated = await withDatadogTurbopack(config, { projectDir }) + assert.equal(findDatadogLoaders(repeated).length, findDatadogLoaders(config).length) + }) + + it('leaves newer Next majors on the modern schema path', async () => { + const projectDir = createProject('17.0.0-canary.1') + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + + const config = await withDatadogTurbopack({}, { projectDir }) + + assert.equal([config.turbopack.rules['*.js']].flat().every(rule => rule.condition), true) + assert.equal(config.turbopack.conditions, undefined) + }) + + it('supports object promises and config functions without changing their contract', async () => { + const projectDir = createProject() + const promisedConfig = { promised: true } + const promiseResult = await withDatadogTurbopack(Promise.resolve(promisedConfig), { projectDir }) + const receiver = { calls: 0 } + const defaultConfig = { defaultConfig: true } + const wrapped = withDatadogTurbopack(function (phase, context) { + this.calls++ + assert.equal(phase, 'phase-production-build') + assert.strictEqual(context.defaultConfig, defaultConfig) + return { functional: true } + }, { projectDir }) + const wrappedUndefined = withDatadogTurbopack(() => undefined, { projectDir }) + + const [functionResult, undefinedResult] = await Promise.all([ + wrapped.call(receiver, 'phase-production-build', { defaultConfig }), + wrappedUndefined(), + ]) + + assert.strictEqual(promiseResult, promisedConfig) + assert.equal(receiver.calls, 1) + assert.deepEqual(functionResult, { functional: true }) + assert.deepEqual(undefinedResult, {}) + }) + + it('uses the current directory when no project option is provided', async () => { + const projectDir = createProject() + const previousDirectory = process.cwd() + + try { + process.chdir(projectDir) + assert.deepEqual(await withDatadogTurbopack(), {}) + } finally { + process.chdir(previousDirectory) + } + }) + + it('returns the original object when no supported package is installed', async () => { + const projectDir = createProject() + const config = { turbopack: { resolveAlias: { value: './value.js' } } } + + assert.strictEqual(await withDatadogTurbopack(config, { projectDir }), config) + assert.equal(fs.existsSync(path.join(projectDir, 'node_modules/.cache/dd-trace/turbopack')), false) + }) + + it('creates one immutable plan under concurrent and repeated composition', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + + const configs = await Promise.all([ + withDatadogTurbopack({}, { projectDir }), + withDatadogTurbopack({}, { projectDir }), + withDatadogTurbopack({}, { projectDir }), + withDatadogTurbopack({}, { projectDir }), + ]) + const planPaths = configs.map(config => findDatadogLoaders(config)[0].options.manifestPath) + const twice = await withDatadogTurbopack(configs[0], { projectDir }) + + assert.equal(new Set(planPaths).size, 1) + assert.match(planPaths[0], /\/[a-f\d]{64}\/[a-f\d]{64}\.json$/) + assert.equal( + path.basename(planPaths[0], '.json'), + createHash('sha256').update(fs.readFileSync(planPaths[0])).digest('hex') + ) + assert.equal(findDatadogLoaders(twice).length, findDatadogLoaders(configs[0]).length) + assert.equal(fs.readdirSync(path.dirname(path.dirname(planPaths[0]))).length, 1) + }) + + it('reports one warning for instrumentation discovery and target compilation failures', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const aiDirectory = createPackage(projectDir, 'ai', { main: 'index.mjs', type: 'module', version: '7.0.0' }) + write(aiDirectory, 'index.mjs', 'export {') + const emitWarning = sinon.stub(process, 'emitWarning') + const hookName = 'test-turbopack-load-failure' + const skippedHookName = 'test-turbopack-nonfunction-hook' + hooks[hookName] = () => { + throw new Error('load failed') + } + hooks[skippedHookName] = {} + + try { + await withDatadogTurbopack({}, { projectDir }) + await withDatadogTurbopack({}, { projectDir }) + } finally { + delete hooks[hookName] + delete hooks[skippedHookName] + } + + assert.equal(emitWarning.callCount, 2) + sinon.assert.calledWithMatch(emitWarning, /Could not load the test-turbopack-load-failure instrumentation/) + sinon.assert.calledWithMatch(emitWarning, new RegExp(`Could not instrument ${aiDirectory}`)) + }) + + it('wraps cache-directory creation failures with their build path', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const error = Object.assign(new Error('permission denied'), { code: 'EACCES' }) + sinon.stub(fsPromises, 'mkdir').rejects(error) + + await assert.rejects( + withDatadogTurbopack({}, { projectDir }), + { message: /Could not create the Datadog Turbopack cache .*permission denied/ } + ) + }) + + it('accepts an artifact completed by a concurrent build-plan writer', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const link = fsPromises.link.bind(fsPromises) + sinon.stub(fsPromises, 'link').callsFake(async (source, target) => { + await link(source, target) + throw Object.assign(new Error('already exists'), { code: 'EEXIST' }) + }) + + const config = await withDatadogTurbopack({}, { projectDir }) + + assert.ok(config.turbopack.rules['*.js']) + }) + + it('rejects conflicting and failed artifact writes', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const link = fsPromises.link.bind(fsPromises) + sinon.stub(fsPromises, 'link').callsFake(async (source, target) => { + await link(source, target) + await fsPromises.writeFile(target, 'conflict') + throw Object.assign(new Error('already exists'), { code: 'EEXIST' }) + }) + + await assert.rejects( + withDatadogTurbopack({}, { projectDir }), + { message: /artifact .* does not match its content address/ } + ) + + sinon.restore() + const failedProject = createProject() + const failedPackage = createPackage(failedProject, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(failedPackage, 'index.js', 'module.exports = {}') + sinon.stub(fsPromises, 'link').rejects(Object.assign(new Error('link denied'), { code: 'EACCES' })) + + await assert.rejects(withDatadogTurbopack({}, { projectDir: failedProject }), { message: /link denied/ }) + }) + + it('warns when a temporary artifact cannot be removed', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const emitWarning = sinon.stub(process, 'emitWarning') + sinon.stub(fsPromises, 'unlink').rejects(new Error('unlink denied')) + + const config = await withDatadogTurbopack({}, { projectDir }) + + assert.ok(config.turbopack.rules['*.js']) + sinon.assert.calledWithMatch(emitWarning, /Could not remove temporary Turbopack artifact .*unlink denied/) + }) + + it('rejects an existing artifact whose content no longer matches its address', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const config = await withDatadogTurbopack({}, { projectDir }) + const planPath = findDatadogLoaders(config)[0].options.manifestPath + fs.writeFileSync(planPath, 'conflict') + + await assert.rejects( + withDatadogTurbopack({}, { projectDir }), + { message: /artifact .* does not match its content address/ } + ) + }) + + it('discovers nested copies and records exact instrumentation indexes', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'parent/node_modules/ioredis', { + main: 'index.js', + version: '5.0.0', + }) + const target = write(packageDir, 'index.js', 'module.exports = {}') + + const config = await withDatadogTurbopack({}, { projectDir }) + const planPath = findDatadogLoaders(config)[0].options.manifestPath + const plan = JSON.parse(fs.readFileSync(planPath, 'utf8')) + const entry = plan.targets[fs.realpathSync(target).replaceAll('\\', '/')] + + assert.equal(entry.payloads[0].package, 'ioredis') + assert.deepEqual(entry.payloads[0].instrumentationIndexes, [3]) + }) + + it('discovers dependencies resolved from an ancestor project', async () => { + const projectRoot = createProject() + const packageDir = createPackage(projectRoot, 'ioredis', { + main: 'dist/index.js', + version: '5.0.0', + }) + write(packageDir, 'dist/index.js', 'module.exports = {}') + const projectDir = path.join(projectRoot, 'app') + write(projectDir, 'package.json', '{}') + + const config = await withDatadogTurbopack({}, { projectDir }) + + assert.ok(config.turbopack.rules['*.js']) + }) + + it('discovers linked workspace dependencies outside node_modules', async () => { + const workspaceDir = createProject() + const projectDir = path.join(workspaceDir, 'apps/web') + write(projectDir, 'package.json', '{}') + const packageDir = path.join(workspaceDir, 'packages/cache-client') + write(packageDir, 'package.json', JSON.stringify({ main: 'index.js', name: 'ioredis', version: '5.0.0' })) + const target = write(packageDir, 'index.js', 'module.exports = {}') + fs.symlinkSync(packageDir, path.join(workspaceDir, 'node_modules/ioredis'), 'dir') + + const config = await withDatadogTurbopack({}, { projectDir }) + const targetRule = [config.turbopack.rules['*.js']].flat().find( + rule => rule.condition?.all?.some(condition => condition?.any) + ) + const planPath = findDatadogLoaders(config)[0].options.manifestPath + const plan = JSON.parse(fs.readFileSync(planPath, 'utf8')) + + assert.ok(plan.targets[fs.realpathSync(target).replaceAll('\\', '/')]) + assert.equal( + targetRule.condition.all[1].any.some(({ path: pattern }) => + pattern.test('../../packages/cache-client/index.js')), + true + ) + }) + + it('discovers dependencies in a pnpm virtual store', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, '.pnpm/ioredis@5.0.0/node_modules/ioredis', { + main: 'index.js', + version: '5.0.0', + }) + write(packageDir, 'index.js', 'module.exports = {}') + const hoistedPackage = createPackage(projectDir, '.pnpm/node_modules/ioredis', { + main: 'index.js', + version: '5.0.0', + }) + write(hoistedPackage, 'index.js', 'module.exports = {}') + write(projectDir, 'node_modules/.pnpm/not-a-package', 'file') + + const config = await withDatadogTurbopack({}, { projectDir }) + const planPath = findDatadogLoaders(config)[0].options.manifestPath + const plan = JSON.parse(fs.readFileSync(planPath, 'utf8')) + + assert.ok(plan.targets[fs.realpathSync(path.join(packageDir, 'index.js')).replaceAll('\\', '/')]) + }) + + it('handles package-boundary traversal failures without losing valid targets', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + write(packageDir, 'node_modules', 'not a directory') + fs.mkdirSync(path.join(projectDir, 'node_modules', '.bin')) + write(projectDir, 'node_modules/not-a-package', 'file') + const scopeDirectory = path.join(projectDir, 'node_modules', '@scope') + fs.mkdirSync(scopeDirectory) + write(scopeDirectory, 'not-a-package', 'file') + fs.symlinkSync(packageDir, path.join(projectDir, 'node_modules', 'ioredis-alias'), 'dir') + fs.symlinkSync( + path.join(projectDir, 'node_modules', 'missing'), + path.join(projectDir, 'node_modules', 'broken-link'), + 'dir' + ) + fs.symlinkSync( + path.join(projectDir, 'node_modules', 'not-a-package'), + path.join(projectDir, 'node_modules', '@broken'), + 'dir' + ) + fs.symlinkSync( + path.join(projectDir, 'node_modules', 'not-a-package'), + path.join(projectDir, 'node_modules', '.pnpm'), + 'dir' + ) + + const config = await withDatadogTurbopack({}, { projectDir }) + + assert.ok(config.turbopack.rules['*.js']) + }) + + it('skips invalid package metadata and file patterns', async () => { + const projectDir = createProject() + const invalidPackage = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(invalidPackage, 'package.json', '{') + + assert.deepEqual(await withDatadogTurbopack({}, { projectDir }), {}) + + const validProject = createProject() + const validPackage = createPackage(validProject, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(validPackage, 'index.js', 'module.exports = {}') + const load = hooks.ioredis?.fn ?? hooks.ioredis + load() + const entries = instrumentations.ioredis + entries.push({ filePattern: '[', hook () {}, versions: ['>=5'] }) + + try { + const config = await withDatadogTurbopack({}, { projectDir: validProject }) + assert.ok(config.turbopack.rules['*.js']) + } finally { + entries.pop() + } + + const fallbackProject = createProject() + const fallbackPackage = createPackage(fallbackProject, 'ioredis', { + exports: { './commands': './index.js' }, + main: 'index.js', + version: '5.0.0', + }) + write(fallbackPackage, 'index.js', 'module.exports = {}') + + const fallbackConfig = await withDatadogTurbopack({}, { projectDir: fallbackProject }) + assert.ok(fallbackConfig.turbopack.rules['*.js']) + }) + + it('deduplicates relative hooks and rejects ambiguous package versions', async () => { + const projectDir = createProject() + const first = createPackage(projectDir, '@prisma/client', { main: 'index.js', version: '6.1.0' }) + const second = createPackage(projectDir, 'parent/node_modules/@prisma/client', { + main: 'index.js', + version: '6.1.0', + }) + write(first, 'index.js', 'module.exports = {}') + write(first, 'runtime/library.js', 'module.exports = { copy: 1 }\n') + write(second, 'index.js', 'module.exports = {}') + write(second, 'runtime/library.js', 'module.exports = { copy: 2 }\n') + const load = hooks['@prisma/client']?.fn ?? hooks['@prisma/client'] + load() + const relativeEntries = instrumentations['./runtime/library.js'] + relativeEntries.push({ ...relativeEntries[0] }) + relativeEntries.push({ hook () {}, versions: ['>=6.1.0 <7.0.0'] }) + + try { + const config = await withDatadogTurbopack({}, { projectDir }) + const planPath = findDatadogLoaders(config)[0].options.manifestPath + const plan = JSON.parse(fs.readFileSync(planPath, 'utf8')) + + assert.equal(plan.relativeTargets.length, 2) + assert.deepEqual(plan.relativeTargets[0].payloads[0].instrumentationIndexes, [0, 1]) + assert.deepEqual(plan.relativeTargets[1].payloads[0].instrumentationIndexes, [0, 1]) + } finally { + relativeEntries.splice(-2) + } + + const ambiguousProject = createProject() + for (const [parent, version] of [['', '6.1.0'], ['one', '6.2.0'], ['two', '6.3.0']]) { + const name = parent ? `${parent}/node_modules/@prisma/client` : '@prisma/client' + const packageDir = createPackage(ambiguousProject, name, { main: 'index.js', version }) + write(packageDir, 'index.js', 'module.exports = {}') + write(packageDir, 'runtime/library.js', 'module.exports = { same: true }\n') + } + + const ambiguousConfig = await withDatadogTurbopack({}, { projectDir: ambiguousProject }) + const ambiguousPlanPath = findDatadogLoaders(ambiguousConfig)[0].options.manifestPath + const ambiguousPlan = JSON.parse(fs.readFileSync(ambiguousPlanPath, 'utf8')) + + assert.equal(ambiguousPlan.relativeTargets.length, 0) + }) + + it('rejects unsupported versions and invalid configuration shapes', async () => { + const oldProject = createProject('15.4.9') + const invalidVersionProject = createProject('latest') + const projectDir = createProject() + const missingNext = createProject() + const missingCompiler = createProject() + fs.rmSync(path.join(missingNext, 'node_modules/next'), { force: true, recursive: true }) + fs.rmSync(path.join(missingCompiler, 'node_modules/next/dist/compiled/babel/parser.js')) + + assert.throws( + () => withDatadogTurbopack({}, { projectDir: oldProject }), + { name: 'RangeError', message: /requires Next\.js 15\.5 or newer/ } + ) + assert.throws( + () => withDatadogTurbopack({}, { projectDir: missingNext }), + { message: /could not resolve Next\.js/ } + ) + assert.throws( + () => withDatadogTurbopack({}, { projectDir: missingCompiler }), + { message: /does not provide the compiler/ } + ) + assert.throws( + () => withDatadogTurbopack({}, { projectDir: invalidVersionProject }), + { message: /could not parse Next\.js version/ } + ) + assert.throws( + () => withDatadogTurbopack({}, /** @type {object} */ (null)), + { name: 'TypeError', message: /options must be an object/ } + ) + assert.throws( + () => withDatadogTurbopack({}, { projectDir: /** @type {string} */ (42) }), + { name: 'TypeError', message: /options\.projectDir must be a string/ } + ) + await assert.rejects( + withDatadogTurbopack(42, { projectDir }), + { name: 'TypeError', message: /configuration object, promise, or function/ } + ) + for (const [config, message] of [ + [{ turbopack: null }, /turbopack must be an object/], + [{ turbopack: [] }, /turbopack must be an object/], + [{ turbopack: { rules: false } }, /turbopack\.rules must be an object/], + [{ turbopack: { rules: [] } }, /turbopack\.rules must be an object/], + [{ turbopack: { conditions: '' } }, /turbopack\.conditions must be an object/], + [{ turbopack: { conditions: [] } }, /turbopack\.conditions must be an object/], + [{ turbopack: { resolveAlias: '' } }, /turbopack\.resolveAlias must be an object/], + [{ turbopack: { resolveAlias: [] } }, /turbopack\.resolveAlias must be an object/], + ]) { + await assert.rejects(withDatadogTurbopack(config, { projectDir }), { message }) + } + }) + + it('does not overwrite a user condition in the reserved Next 15 namespace', async () => { + const projectDir = createProject('15.5.0') + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + + await assert.rejects(withDatadogTurbopack({ + turbopack: { + conditions: { '#dd-trace/target': { path: '*.custom.js' } }, + }, + }, { projectDir }), { message: /already uses the reserved condition/ }) + }) +}) diff --git a/packages/datadog-turbopack/test/helpers.js b/packages/datadog-turbopack/test/helpers.js new file mode 100644 index 00000000000..b0678152e59 --- /dev/null +++ b/packages/datadog-turbopack/test/helpers.js @@ -0,0 +1,88 @@ +'use strict' + +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +/** @type {string[]} */ +const directories = [] + +/** + * @param {string} [nextVersion] + * @returns {string} + */ +function createProject (nextVersion = '16.0.0') { + const directory = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-'))) + directories.push(directory) + write(directory, 'package.json', '{}') + const next = createPackage(directory, 'next', { main: 'index.js', version: nextVersion }) + write(next, 'index.js', 'module.exports = {}') + for (const name of ['generator', 'parser', 'traverse']) { + const modulePath = require.resolve(`@babel/${name}`) + write(next, `dist/compiled/babel/${name}.js`, `module.exports = require(${JSON.stringify(modulePath)})\n`) + } + return directory +} + +/** + * @param {string} projectDir + * @param {string} name + * @param {object} [manifest] + * @returns {string} + */ +function createPackage (projectDir, name, manifest = {}) { + const packageDirectory = path.join(projectDir, 'node_modules', name) + fs.mkdirSync(packageDirectory, { recursive: true }) + fs.writeFileSync(path.join(packageDirectory, 'package.json'), JSON.stringify({ name, ...manifest })) + return packageDirectory +} + +/** + * @param {string} directory + * @param {string} relativePath + * @param {string} content + * @returns {string} + */ +function write (directory, relativePath, content) { + const target = path.resolve(directory, relativePath) + fs.mkdirSync(path.dirname(target), { recursive: true }) + fs.writeFileSync(target, content) + return target +} + +/** + * @param {unknown} value + * @returns {object[]} + */ +function findDatadogLoaders (value) { + const loaders = [] + collectDatadogLoaders(value, loaders) + return loaders +} + +/** + * @param {unknown} value + * @param {object[]} loaders + */ +function collectDatadogLoaders (value, loaders) { + if (Array.isArray(value)) { + for (const item of value) collectDatadogLoaders(item, loaders) + return + } + if (!value || typeof value !== 'object') return + if (typeof value.loader === 'string' && value.loader.includes('datadog-turbopack')) loaders.push(value) + + for (const key of Object.keys(value)) collectDatadogLoaders(value[key], loaders) +} + +function cleanup () { + while (directories.length > 0) fs.rmSync(directories.pop(), { force: true, recursive: true }) +} + +module.exports = { + cleanup, + createPackage, + createProject, + findDatadogLoaders, + write, +} diff --git a/packages/datadog-turbopack/test/loader.spec.js b/packages/datadog-turbopack/test/loader.spec.js new file mode 100644 index 00000000000..06922db2f47 --- /dev/null +++ b/packages/datadog-turbopack/test/loader.spec.js @@ -0,0 +1,831 @@ +'use strict' + +const assert = require('node:assert/strict') +const { createHash } = require('node:crypto') +const fs = require('node:fs') +const path = require('node:path') +const vm = require('node:vm') +const { pathToFileURL } = require('node:url') +const { afterEach, describe, it } = require('mocha') + +const dc = require('dc-polyfill') +const enhancedResolve = require('enhanced-resolve') +const { ESLint } = require('eslint') +const semver = require('semver') +const sinon = require('sinon') + +const { engines, nodeMaxMajor } = require('../../../package.json') +const { withDatadogTurbopack } = require('../../../next') +const loader = require('../src/loader') +const { + cleanup, + createPackage, + createProject, + findDatadogLoaders, + write, +} = require('./helpers') + +const CHANNEL = 'dd-trace:bundler:load' +const lintRuntimeSupported = semver.satisfies(process.version, `${engines.node} <${nodeMaxMajor}`) + +afterEach(() => { + cleanup() + sinon.restore() +}) + +describe('datadog-turbopack loader', () => { + it('rewrites ESM imports and only unshadowed CommonJS requires', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/route.js', '') + const source = [ + "import { generateText } from 'ai'", + "export { streamText } from 'ai'", + "const dynamic = import('ai')", + "const attributed = import('ai', { with: { type: 'json' } })", + "const top = require('ai')", + "const commonjs = require('ioredis')", + "const missing = require('not-installed')", + "const missingTarget = require('ai/not-installed')", + "function local (require) { return require('ai') }", + 'export { attributed, commonjs, dynamic, local, missing, missingTarget, top }', + '', + ].join('\n') + + const { code: result, map } = await runLoaderResult(appPath, source, fixture.importOptions) + const proxyFile = path.basename(fixture.proxyPath) + + assert.equal(result.split(proxyFile).length - 1, 4) + assert.match(result, /const top = require\('ai'\)/) + assert.match(result, /function local\(require\) \{\s*return require\('ai'\)/) + assert.equal(map.sources[0], appPath) + assert.equal( + path.basename(fixture.proxyPath, '.mjs'), + createHash('sha256').update(fs.readFileSync(fixture.proxyPath)).digest('hex') + ) + assert.equal(await runLoader(appPath, source, fixture.importOptions), result) + }) + + it('rewrites imports in TypeScript JSX application modules', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/route.tsx', '') + const source = [ + "import { generateText } from 'ai'", + 'const prompt: string = \'hello\'', + 'export default
{prompt}
', + '', + ].join('\n') + + const result = await runLoader(appPath, source, fixture.importOptions) + + assert.match(result, new RegExp(path.basename(fixture.proxyPath))) + }) + + it('chains an input source map through module-edge rewriting', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/mapped.js', '') + const source = "import { generateText } from 'ai'\n" + const sourceMap = { + file: appPath, + mappings: 'AAAA', + names: [], + sources: ['route.ts'], + sourcesContent: [source], + version: 3, + } + + const result = await runLoaderResult(appPath, source, fixture.importOptions, { sourceMap }) + + assert.deepEqual(result.map.sources, ['route.ts']) + assert.deepEqual(result.map.sourcesContent, [source]) + }) + + it('rewrites static template imports and keeps dynamic templates unchanged', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/templates.js', '') + const source = [ + 'const imported = import(`ai`)', + 'const required = require(`ai`)', + // eslint-disable-next-line no-template-curly-in-string -- This is source for the loader under test. + 'const dynamic = import(`ai/${name}`)', + 'export { dynamic, imported, required }', + '', + ].join('\n') + + const result = await runLoader(appPath, source, fixture.importOptions) + + assert.equal(result.split(path.basename(fixture.proxyPath)).length - 1, 1) + assert.match(result, /require\(`ai`\)/) + assert.match(result, /import\(`ai\/\$\{name\}`\)/) + }) + + it('rewrites import-expression AST nodes from newer parsers', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/import-expression.js', '') + const parserPath = fixture.importOptions.compiler.parser + const parser = write(fixture.projectDir, 'parser.js', [ + `const parser = require(${JSON.stringify(parserPath)})`, + 'exports.parse = function parse (source, options) {', + ' return parser.parse(source, { ...options, createImportExpressions: true })', + '}', + '', + ].join('\n')) + const options = { + ...fixture.importOptions, + compiler: { ...fixture.importOptions.compiler, parser }, + } + + const result = await runLoader(appPath, "const value = import('ai')\n", options) + + assert.match(result, new RegExp(path.basename(fixture.proxyPath))) + }) + + it('leaves type-only module edges untouched', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/types.ts', '') + const source = [ + "import type { CoreTool } from 'ai'", + "import { type LanguageModel } from 'ai'", + "export type { ToolChoice } from 'ai'", + "export { type ToolChoice } from 'ai'", + 'import Alias = Namespace.Value', + '', + ].join('\n') + + assert.equal(await runLoader(appPath, source, fixture.importOptions), source) + }) + + it('leaves Flow type imports untouched', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/types.js', '') + const source = [ + "import type { CoreTool } from 'ai'", + "import typeof GenerateText from 'ai'", + 'const count: number = 1', + 'export { count }', + '', + ].join('\n') + + assert.equal(await runLoader(appPath, source, fixture.importOptions), source) + }) + + it('does not pass Node.js built-ins to the Turbopack resolver', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/builtins.js', '') + const requests = [] + const getResolve = () => (_directory, request, callback) => { + requests.push(request) + callback(undefined, fixture.targetPath) + } + const source = [ + "import fs from 'node:fs'", + "import path from 'path'", + "import { generateText } from 'ai'", + '', + ].join('\n') + + const result = await runLoader(appPath, source, fixture.importOptions, { getResolve }) + + assert.deepEqual(requests, ['ai']) + assert.match(result, new RegExp(path.basename(fixture.proxyPath))) + }) + + it('does not redirect a generated proxy back to itself', async () => { + const fixture = await createAiFixture() + const source = fs.readFileSync(fixture.proxyPath, 'utf8') + let resolverCalls = 0 + const getResolve = () => (_directory, _request, callback) => { + resolverCalls++ + callback(new Error('generated proxies must not be resolved')) + } + + assert.equal(await runLoader(fixture.proxyPath, source, fixture.importOptions, { getResolve }), source) + assert.equal(resolverCalls, 0) + }) + + it('parses explicit resource management syntax while rewriting imports', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/resource.js', '') + const source = [ + "import { generateText } from 'ai'", + 'using resource = { [Symbol.dispose] () {} }', + 'export { resource }', + '', + ].join('\n') + + const result = await runLoader(appPath, source, fixture.importOptions) + + assert.match(result, new RegExp(path.basename(fixture.proxyPath))) + }) + + it('does not rewrite a require that is shadowed in its own scope', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/route.js', '') + const source = "const require = load\nconst value = require('ai')\n" + + assert.equal(await runLoader(appPath, source, fixture.importOptions), source) + }) + + it('rewrites require edges only when the require resolver selects an ESM target', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/require.ts', '') + const source = "import AI = require('ai')\nconst required = require('ai')\nexport { required }\n" + const resolvedConditions = [] + const getResolve = options => (_directory, _request, callback) => { + resolvedConditions.push(options.conditionNames) + queueMicrotask(() => callback(undefined, fixture.targetPath)) + } + + const result = await runLoader(appPath, source, fixture.importOptions, { getResolve }) + + assert.equal(result.split(path.basename(fixture.proxyPath)).length - 1, 2) + assert.deepEqual(resolvedConditions, [['node', 'require']]) + }) + + it('tracks bindings from every supported declaration pattern', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/scopes.js', '') + const source = [ + "import { generateText } from 'ai'", + 'const named = function named () {}', + "const assigned = (require = load) => require('ai')", + "const array = ([require]) => require('ai')", + "const object = ({ require }) => require('ai')", + 'const { require: objectRequire, ...objectRest } = globalThis', + "const rest = (...require) => require[0]('ai')", + "function scoped () { var require = load; return require('ai') }", + "try { scoped() } catch (require) { require('ai') }", + 'try { scoped() } catch {}', + 'const Named = class Named {}', + 'const Anonymous = class {}', + 'export { Anonymous, Named, array, assigned, named, object, objectRequire, objectRest, rest, scoped }', + '', + ].join('\n') + + const result = await runLoader(appPath, source, fixture.importOptions) + + assert.equal(result.split(path.basename(fixture.proxyPath)).length - 1, 1) + assert.match(result, /return require\('ai'\)/) + }) + + it('preserves configured aliases', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ai', { main: 'index.mjs', type: 'module', version: '7.0.0' }) + write(packageDir, 'index.mjs', 'export function generateText () {}\n') + const config = await withDatadogTurbopack({ + turbopack: { resolveAlias: { ai: './replacement.js' } }, + }, { projectDir }) + const options = findDatadogLoaders(config) + .find(item => item.options.rewriteEdges && !item.options.targetScope).options + const appPath = write(projectDir, 'app/route.js', '') + const source = "import { generateText } from 'ai'\nimport value from 'ai/subpath'\n" + const replacement = write(projectDir, 'replacement.js', 'export default true\n') + const getResolve = () => (_directory, _request, callback) => callback(undefined, replacement) + + assert.equal(await runLoader(appPath, source, options, { getResolve }), source) + }) + + it('returns transformed ESM dependencies without a CommonJS publication tail', async () => { + const fixture = await createAiFixture() + const source = fs.readFileSync(fixture.targetPath, 'utf8') + + const result = await runLoader(fixture.targetPath, source, fixture.packageOptions) + + assert.doesNotMatch(result, /dd-trace:bundler:load/) + }) + + it('plans and instruments both sides of a dual package export', async () => { + const fixture = await createAiFixture() + const plan = JSON.parse(fs.readFileSync(fixture.packageOptions.manifestPath, 'utf8')) + + assert.ok(plan.targets[fs.realpathSync(fixture.targetPath)]) + assert.ok(plan.targets[fs.realpathSync(fixture.commonJsPath)]) + assert.match( + await runLoader(fixture.commonJsPath, fs.readFileSync(fixture.commonJsPath, 'utf8'), fixture.packageOptions), + /dd-trace:bundler:load/ + ) + }) + + it('classifies TypeScript module extensions independently of package type', async () => { + const moduleProject = createProject() + const moduleDirectory = createPackage(moduleProject, 'ioredis', { + main: 'index.mts', + type: 'commonjs', + version: '5.0.0', + }) + const moduleSource = 'export const original = true\n' + const modulePath = write(moduleDirectory, 'index.mts', moduleSource) + const commonJsProject = createProject() + const commonJsDirectory = createPackage(commonJsProject, 'ioredis', { + main: 'index.cts', + type: 'module', + version: '5.0.0', + }) + const commonJsSource = 'module.exports = { original: true }\n' + const commonJsPath = write(commonJsDirectory, 'index.cts', commonJsSource) + const [moduleConfig, commonJsConfig] = await Promise.all([ + withDatadogTurbopack({}, { projectDir: moduleProject }), + withDatadogTurbopack({}, { projectDir: commonJsProject }), + ]) + const moduleOptions = findDatadogLoaders(moduleConfig).find(item => item.options.targetScope === 'direct').options + const commonJsOptions = findDatadogLoaders(commonJsConfig) + .find(item => item.options.targetScope === 'direct').options + const modulePlan = JSON.parse(fs.readFileSync(moduleOptions.manifestPath, 'utf8')) + const commonJsPlan = JSON.parse(fs.readFileSync(commonJsOptions.manifestPath, 'utf8')) + const moduleTarget = modulePlan.targets[fs.realpathSync(modulePath)] + const commonJsTarget = commonJsPlan.targets[fs.realpathSync(commonJsPath)] + + assert.equal(moduleTarget.esm, true) + assert.equal(typeof moduleTarget.proxyPath, 'string') + assert.equal(commonJsTarget.esm, false) + assert.equal(commonJsTarget.proxyPath, undefined) + + const [moduleResult, commonJsResult] = await Promise.all([ + runLoader(modulePath, moduleSource, moduleOptions), + runLoader(commonJsPath, commonJsSource, commonJsOptions), + ]) + + assert.equal(moduleResult, moduleSource) + assert.match(commonJsResult, /dd-trace:bundler:load/) + }) + + it('rewrites an internal package edge to its instrumented ESM target', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'hono', { + exports: './dist/index.js', + type: 'module', + version: '4.12.19', + }) + const source = "export { Hono } from './hono.js'\n" + const entryPath = write(packageDir, 'dist/index.js', source) + const targetPath = write(packageDir, 'dist/hono.js', 'export class Hono {}\n') + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config).find(item => item.options.targetScope === 'direct').options + const plan = JSON.parse(fs.readFileSync(options.manifestPath, 'utf8')) + const proxyPath = plan.targets[fs.realpathSync(targetPath)].proxyPath + + const rewritten = await runLoader(entryPath, source, options) + + assert.match(rewritten, new RegExp(path.basename(proxyPath))) + assert.doesNotMatch(rewritten, /dd-trace:bundler:load/) + }) + + it('does not instrument a direct target through the relative-copy rule', async () => { + const fixture = await createAiFixture() + const source = fs.readFileSync(fixture.ioredisPath, 'utf8') + + const result = await runLoader(fixture.ioredisPath, source, { + ...fixture.packageOptions, + targetScope: 'relative', + }) + + assert.equal(result, source) + }) + + it('keeps source and emits one warning when import parsing fails', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/broken.js', '') + const warnings = [] + const source = 'import {' + assert.equal(await runLoader(appPath, source, fixture.importOptions, { + emitWarning: warning => warnings.push(warning), + }), source) + assert.equal(await runLoader(appPath, source, fixture.importOptions, { + emitWarning: warning => warnings.push(warning), + }), source) + assert.equal(warnings.length, 1) + assert.equal(warnings[0].name, 'DatadogTurbopackWarning') + }) + + it('falls back to process warnings when the loader context cannot emit one', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/process-warning.js', '') + const emitWarning = sinon.stub(process, 'emitWarning') + + assert.equal(await runLoader(appPath, 'import {', fixture.importOptions), 'import {') + sinon.assert.calledOnce(emitWarning) + }) + + it('keeps source when the Turbopack resolver cannot be initialized', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/resolver.js', '') + const source = "import { generateText } from 'ai'\n" + const warnings = [] + const getResolve = () => { + throw new Error('resolver unavailable') + } + + const result = await runLoader(appPath, source, fixture.importOptions, { + emitWarning: warning => warnings.push(warning), + getResolve, + }) + + assert.equal(result, source) + assert.equal(warnings.length, 1) + assert.match(warnings[0].message, /Could not initialize import resolution/) + }) + + it('settles each resolver once and contains edge-resolution failures', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/resolver-edge.js', '') + const source = "import { generateText } from 'ai'\n" + const duplicateCallback = () => (_directory, _request, callback) => { + callback(undefined, fixture.targetPath) + callback(new Error('late failure')) + } + + const rewritten = await runLoader(appPath, source, fixture.importOptions, { + getResolve: duplicateCallback, + }) + assert.match(rewritten, new RegExp(path.basename(fixture.proxyPath))) + + const throwingResolver = () => () => { + throw new Error('resolution failed') + } + assert.equal(await runLoader(appPath, source, fixture.importOptions, { + getResolve: throwingResolver, + }), source) + + const missingTarget = () => (_directory, _request, callback) => { + callback(undefined, path.join(fixture.projectDir, 'missing.js')) + } + assert.equal(await runLoader(appPath, source, fixture.importOptions, { + getResolve: missingTarget, + }), source) + }) + + it('keeps source when import generation fails', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/generator.js', '') + const generator = write( + fixture.projectDir, + 'generator.js', + "module.exports.default = () => { throw new Error('generation failed') }\n" + ) + const options = { + ...fixture.importOptions, + compiler: { ...fixture.importOptions.compiler, generator }, + } + const source = "import { generateText } from 'ai'\n" + const warnings = [] + + const result = await runLoader(appPath, source, options, { + emitWarning: warning => warnings.push(warning), + }) + + assert.equal(result, source) + assert.equal(warnings.length, 1) + assert.match(warnings[0].message, /Could not generate rewritten imports/) + }) + + it('does not redirect an edge after its planned target changes', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/changed-target.js', '') + const source = "import { generateText } from 'ai'\n" + const warnings = [] + fs.writeFileSync(fixture.targetPath, 'export function generateText () { return 1 }\n') + + const result = await runLoader(appPath, source, fixture.importOptions, { + emitWarning: warning => warnings.push(warning), + }) + + assert.equal(result, source) + assert.equal(warnings.length, 1) + assert.match(warnings[0].message, /Skipped changed dependency/) + }) + + it('publishes CommonJS targets only when the channel has subscribers', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + const resourcePath = write(packageDir, 'index.js', "'use strict'\n\nmodule.exports = { original: true }\n") + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config).find(item => item.options.targetScope === 'direct').options + const transformed = await runLoader(resourcePath, fs.readFileSync(resourcePath, 'utf8'), options) + await assertGeneratedSourceIsLintClean(transformed, 'generated.cjs') + const publications = [] + const channel = { + hasSubscribers: false, + publish: payload => publications.push(payload), + } + + const inactive = executeCommonJs(transformed, channel) + channel.hasSubscribers = true + channel.publish = payload => { + publications.push(payload) + payload.module = { patched: true } + } + const active = executeCommonJs(transformed, channel) + + assert.equal(inactive.original, true) + assert.equal(inactive.patched, undefined) + assert.equal(active.patched, true) + assert.equal(publications.length, 1) + assert.equal(publications[0].package, 'ioredis') + assert.equal(publications[0].moduleName, 'ioredis') + assert.equal(publications[0].instrumentationIndexes.length, 1) + assert.equal(publications[0].instrumentationIndexes[0], 3) + }) + + it('matches relative runtimes by suffix and source hash', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, '@prisma/client', { main: 'index.js', version: '6.1.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const source = 'module.exports = { prisma: true }\n' + write(packageDir, 'runtime/library.js', source) + const matching = write(projectDir, 'generated/runtime/library.js', source) + const unrelated = write(projectDir, 'unrelated/runtime/library.js', 'module.exports = { unrelated: true }\n') + const otherFile = write(projectDir, 'generated/runtime/other.js', source) + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config).find(item => item.options.targetScope === 'relative').options + const matchingResult = await runLoader(matching, source, options) + const unrelatedSource = fs.readFileSync(unrelated, 'utf8') + + assert.match(matchingResult, /dd-trace:bundler:load/) + assert.match(matchingResult, /package: "\.\/runtime\/library\.js"/) + assert.equal(await runLoader(unrelated, unrelatedSource, options), unrelatedSource) + assert.equal(await runLoader(otherFile, source, options), source) + }) + + it('evicts old file hashes at the cache boundary', async function () { + this.timeout(30000) + const projectDir = createProject() + const packageDir = createPackage(projectDir, '@prisma/client', { main: 'index.js', version: '6.1.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const originalSource = 'module.exports = { first: true }\n' + const changedSource = 'module.exports = { other: true }\n' + write(packageDir, 'runtime/library.js', originalSource) + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config).find(item => item.options.targetScope === 'relative').options + const stableTime = new Date('2020-01-01T00:00:00.000Z') + const files = [] + + for (let index = 0; index <= 2048; index++) { + const file = write(projectDir, `generated/${index}/runtime/library.js`, originalSource) + fs.utimesSync(file, stableTime, stableTime) + files.push(file) + await runLoader(file, originalSource, options) + } + + const firstStat = fs.statSync(files[0]) + fs.writeFileSync(files[0], changedSource) + fs.utimesSync(files[0], stableTime, stableTime) + const statSync = fs.statSync.bind(fs) + sinon.stub(fs, 'statSync').callsFake(file => file === files[0] ? firstStat : statSync(file)) + + assert.equal(Buffer.byteLength(originalSource), Buffer.byteLength(changedSource)) + assert.equal(await runLoader(files[0], changedSource, options), changedSource) + }) + + it('does not use a plan after a dependency changes', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + const before = 'module.exports = { first: true }\n' + const resourcePath = write(packageDir, 'index.js', before) + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config)[0].options + const source = 'module.exports = { other: true }\n' + const warnings = [] + const { atime, mtime } = fs.statSync(resourcePath) + fs.writeFileSync(resourcePath, source) + fs.utimesSync(resourcePath, atime, mtime) + + const result = await runLoader(resourcePath, source, options, { + emitWarning: warning => warnings.push(warning), + }) + + assert.equal(result, source) + assert.equal(Buffer.byteLength(before), Buffer.byteLength(source)) + assert.equal(warnings.length, 1) + assert.match(warnings[0].message, /Skipped changed dependency/) + }) + + it('rejects a build plan that fails its integrity check', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + const resourcePath = write(packageDir, 'index.js', 'module.exports = {}') + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config)[0].options + fs.appendFileSync(options.manifestPath, ' ') + + await assert.rejects( + runLoader(resourcePath, fs.readFileSync(resourcePath, 'utf8'), options), + { message: /failed its integrity check/ } + ) + }) + + it('rejects missing loader options and unsupported build plans', async () => { + await assert.rejects( + runLoader(__filename, '', {}), + { name: 'TypeError', message: /requires a build plan path and hash/ } + ) + + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + const resourcePath = write(packageDir, 'index.js', 'module.exports = {}') + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config)[0].options + const plan = JSON.parse(fs.readFileSync(options.manifestPath, 'utf8')) + plan.version++ + const serialized = JSON.stringify(plan) + fs.writeFileSync(options.manifestPath, serialized) + + await assert.rejects( + runLoader(resourcePath, fs.readFileSync(resourcePath, 'utf8'), { + ...options, + manifestHash: createHash('sha256').update(serialized).digest('hex'), + }), + { message: /build plan .* is not supported/ } + ) + + plan.version-- + delete plan.relativeTargets + const missingFieldPlan = JSON.stringify(plan) + fs.writeFileSync(options.manifestPath, missingFieldPlan) + + await assert.rejects( + runLoader(resourcePath, fs.readFileSync(resourcePath, 'utf8'), { + ...options, + manifestHash: createHash('sha256').update(missingFieldPlan).digest('hex'), + }), + { message: /build plan .* is not supported/ } + ) + }) + + it('evicts old build plans at the cache boundary', async function () { + this.timeout(30000) + const fixtures = [] + + for (let index = 0; index <= 16; index++) { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + const resourcePath = write(packageDir, 'index.js', `module.exports = ${index}`) + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config)[0].options + await runLoader(resourcePath, fs.readFileSync(resourcePath, 'utf8'), options) + fixtures.push({ options, resourcePath }) + } + + fs.appendFileSync(fixtures[0].options.manifestPath, ' ') + + await assert.rejects( + runLoader( + fixtures[0].resourcePath, + fs.readFileSync(fixtures[0].resourcePath, 'utf8'), + fixtures[0].options + ), + { message: /failed its integrity check/ } + ) + }) + + it('uses a relative proxy specifier for a source beside its generated proxy', async () => { + const fixture = await createAiFixture() + const appPath = write(path.dirname(fixture.proxyPath), 'route.js', '') + const result = await runLoader(appPath, "import { generateText } from 'ai'\n", fixture.importOptions) + + assert.match(result, /from ['"]\.\/[a-f\d]{64}\.mjs['"]/) + }) + + it('keeps ESM exports live and applies patches once per proxy evaluation', async () => { + const fixture = await createAiFixture() + const channel = dc.channel(CHANNEL) + await assertGeneratedSourceIsLintClean(fs.readFileSync(fixture.proxyPath, 'utf8'), 'generated.mjs') + const inactive = await import(`${pathToFileURL(fixture.proxyPath).href}?inactive`) + let publications = 0 + const subscriber = payload => { + if (payload.package !== 'ai') return + publications++ + payload.apply({ + generateText: () => 'patched', + streamText: () => 'patched-stream', + }, false) + } + channel.subscribe(subscriber) + + try { + const active = await import(`${pathToFileURL(fixture.proxyPath).href}?active`) + assert.equal(inactive.generateText(), 'original') + assert.equal(active.generateText(), 'patched') + assert.equal(publications, 1) + } finally { + channel.unsubscribe(subscriber) + } + }) +}) + +/** + * @param {string} [ioredisSource] + * @returns {Promise<{ + * commonJsPath: string, + * importOptions: object, + * ioredisPath: string, + * packageOptions: object, + * projectDir: string, + * proxyPath: string, + * targetPath: string + * }>} + */ +async function createAiFixture (ioredisSource = 'module.exports = {}') { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ai', { + exports: { + import: './index.mjs', + require: './index.cjs', + }, + main: 'index.cjs', + type: 'module', + version: '7.0.0', + }) + write(packageDir, 'index.mjs', [ + "export function generateText () { return 'original' }", + "export function streamText () { return 'original-stream' }", + '', + ].join('\n')) + const commonJsPath = write(packageDir, 'index.cjs', 'module.exports = {}\n') + const ioredisDirectory = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + const ioredisPath = write(ioredisDirectory, 'index.js', ioredisSource) + const config = await withDatadogTurbopack({}, { projectDir }) + const loaders = findDatadogLoaders(config) + const importOptions = loaders.find(item => item.options.rewriteEdges && !item.options.targetScope).options + const packageOptions = loaders.find(item => item.options.targetScope === 'direct').options + const plan = JSON.parse(fs.readFileSync(importOptions.manifestPath, 'utf8')) + const [targetPath, target] = Object.entries(plan.targets).find(([, entry]) => entry.esm) + + return { + commonJsPath, + importOptions, + ioredisPath, + packageOptions, + projectDir, + proxyPath: target.proxyPath, + targetPath, + } +} + +/** + * @param {string} resourcePath + * @param {string} source + * @param {object} options + * @param {{ emitWarning?: (warning: Error) => void, getResolve?: Function }} [settings] + * @returns {Promise} + */ +async function runLoader (resourcePath, source, options, settings = {}) { + const { code } = await runLoaderResult(resourcePath, source, options, settings) + return code +} + +/** + * @param {string} resourcePath + * @param {string} source + * @param {object} options + * @param {{ emitWarning?: (warning: Error) => void, getResolve?: Function, sourceMap?: object }} [settings] + * @returns {Promise<{ code: string, map?: object }>} + */ +function runLoaderResult (resourcePath, source, options, settings = {}) { + return new Promise((resolve, reject) => { + /** + * @param {Error} [error] + * @param {string} [code] + * @param {object} [map] + */ + function callback (error, code, map) { + if (error) reject(error) + else resolve({ code, map }) + } + + loader.call({ + async: () => callback, + emitWarning: settings.emitWarning, + getOptions: () => options, + getResolve: settings.getResolve ?? (resolveOptions => enhancedResolve.create(resolveOptions)), + resourcePath, + }, source, settings.sourceMap) + }) +} + +/** + * @param {string} source + * @param {object} channel + * @returns {object} + */ +function executeCommonJs (source, channel) { + const context = { + module: { exports: {} }, + require: () => ({ channel: () => channel }), + } + vm.runInNewContext(source, context) + return context.module.exports +} + +/** + * @param {string} source + * @param {string} filename + * @returns {Promise} + */ +async function assertGeneratedSourceIsLintClean (source, filename) { + if (!lintRuntimeSupported) return + + const eslint = new ESLint() + const [result] = await eslint.lintText(source, { + filePath: path.join(process.cwd(), 'packages/datadog-turbopack/test', filename), + }) + assert.deepEqual(result.messages, []) +} diff --git a/packages/datadog-turbopack/test/plugin.spec.js b/packages/datadog-turbopack/test/plugin.spec.js deleted file mode 100644 index 73b611ecee5..00000000000 --- a/packages/datadog-turbopack/test/plugin.spec.js +++ /dev/null @@ -1,382 +0,0 @@ -'use strict' - -const assert = require('node:assert/strict') -const fs = require('node:fs') -const os = require('node:os') -const path = require('node:path') -const { pathToFileURL } = require('node:url') -const { afterEach, describe, it } = require('mocha') - -const { withDatadogTurbopack } = require('..') -const loader = require('../src/loader') -const { createEsmProxy, createManifest, getRelativeTargets } = require('../src/targets') - -const directories = [] - -afterEach(() => { - while (directories.length > 0) { - fs.rmSync(directories.pop(), { force: true, recursive: true }) - } -}) - -describe('datadog-turbopack loader', () => { - it('uses CommonJS-compatible diagnostics channel imports in ESM proxies', async () => { - const directory = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-'))) - directories.push(directory) - const source = write(directory, 'module.mjs', 'export const value = 1') - const proxyPath = path.join(directory, 'proxy.mjs') - const proxy = await createEsmProxy(source, proxyPath, 'ai', 'ai', '7.0.0') - - assert.match(proxy, /import dc from /) - assert.match(proxy, /dc\.channel\('dd-trace:bundler:load'\)/) - assert.doesNotMatch(proxy, /import \{ channel \} from /) - - fs.writeFileSync(proxyPath, proxy) - await import(pathToFileURL(proxyPath).href) - }) - - it('routes an internal ESM import through its generated proxy', () => { - const directory = createPackage('openai', { type: 'module' }) - const client = write(directory, 'client.mjs', "import { Models } from './resources/models.mjs'\nexport { Models }") - const models = write(directory, 'resources/models.mjs', 'export class Models {}') - const proxy = write(directory, '../.cache/dd-trace/turbopack/models.mjs', 'export {}') - - const result = loader.rewriteImports(fs.readFileSync(client, 'utf8'), client, { - [realpath(models)]: { esm: true, proxyPath: realpath(proxy) }, - }) - - assert.match(result, /from "\.\.\/\.cache\/dd-trace\/turbopack\/models\.mjs"/) - }) - - it('routes a dynamic ESM import through its generated proxy', () => { - const directory = createPackage('openai', { type: 'module' }) - const client = write(directory, 'client.mjs', "export const loadModels = () => import('./resources/models.mjs')") - const models = write(directory, 'resources/models.mjs', 'export class Models {}') - const proxy = write(directory, '../.cache/dd-trace/turbopack/models.mjs', 'export {}') - - const result = loader.rewriteImports(fs.readFileSync(client, 'utf8'), client, { - [realpath(models)]: { esm: true, proxyPath: realpath(proxy) }, - }) - - assert.match(result, /import\("\.\.\/\.cache\/dd-trace\/turbopack\/models\.mjs"\)/) - }) - - it('routes an application ESM import through its generated proxy', async () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) - directories.push(directory) - fs.writeFileSync(path.join(directory, 'package.json'), '{}') - const packagePath = createPackageIn(directory, 'ai', { main: 'index.mjs', type: 'module', version: '7.0.0' }) - write(packagePath, 'index.mjs', 'export const generateText = () => {}') - const appPath = write(directory, 'app/route.js', "import { generateText } from 'ai'") - const manifest = await createManifest(directory) - - const result = loader.call({ - getOptions: () => ({ manifestPath: manifest.path, rewriteApplicationImports: true }), - resourcePath: appPath, - }, fs.readFileSync(appPath, 'utf8')) - - assert.match(result, /from "\.\.\/node_modules\/\.cache\/dd-trace\/turbopack\/0\.mjs"/) - assert.match(result, /sourceMappingURL=data:application\/json;base64,/) - }) - - it('routes an application CommonJS require through its generated proxy', async () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) - directories.push(directory) - fs.writeFileSync(path.join(directory, 'package.json'), '{}') - const packagePath = createPackageIn(directory, 'ai', { main: 'index.mjs', type: 'module', version: '7.0.0' }) - write(packagePath, 'index.mjs', 'export const generateText = () => {}') - const appPath = write(directory, 'pages/api/route.js', "const { generateText } = require('ai')") - const manifest = await createManifest(directory) - - const result = loader.call({ - getOptions: () => ({ manifestPath: manifest.path, rewriteApplicationImports: true }), - resourcePath: appPath, - }, fs.readFileSync(appPath, 'utf8')) - - assert.match(result, /require\("\.\.\/\.\.\/node_modules\/\.cache\/dd-trace\/turbopack\/0\.mjs"\)/) - }) - - it('does not rewrite an application-defined require function', () => { - const directory = createPackage('ai', { main: 'index.mjs', type: 'module' }) - const target = write(directory, 'index.mjs', 'export const generateText = () => {}') - const proxy = write(directory, '../.cache/dd-trace/turbopack/ai.mjs', 'export {}') - const appPath = write(path.dirname(path.dirname(directory)), 'route.js', '') - const source = "function load (require) { return require('ai') }" - - const result = loader.rewriteImports(source, appPath, { - [realpath(target)]: { esm: true, proxyPath: realpath(proxy) }, - }) - - assert.equal(result, source) - }) - - it('preserves configured aliases for instrumented packages', () => { - const directory = createPackage('ai', { main: 'index.mjs', type: 'module' }) - const target = write(directory, 'index.mjs', 'export const generateText = () => {}') - const proxy = write(directory, '../.cache/dd-trace/turbopack/ai.mjs', 'export {}') - const appPath = write(path.dirname(path.dirname(directory)), 'route.js', '') - const source = "import { generateText } from 'ai'" - - const result = loader.rewriteImports(source, appPath, { - [realpath(target)]: { esm: true, proxyPath: realpath(proxy) }, - }, ['ai']) - - assert.equal(result, source) - }) - - it('preserves ESM modules without an instrumented import', () => { - const directory = createPackage('openai', { type: 'module' }) - const client = write(directory, 'client.mjs', "import { Models } from './resources/models.mjs'\nexport { Models }") - const source = fs.readFileSync(client, 'utf8') - - const result = loader.rewriteImports(source, client, {}) - - assert.equal(result, source) - }) - - it('publishes CommonJS exports through the existing bundler channel', () => { - const directory = createPackage('ioredis') - const resourcePath = write(directory, 'built/index.js', 'module.exports = {}') - const manifestPath = write(directory, 'manifest.json', JSON.stringify({ - targets: { - [realpath(resourcePath)]: { - esm: false, - name: 'ioredis', - path: 'ioredis', - version: '5.0.0', - }, - }, - })) - - const result = loader.call({ - getOptions: () => ({ manifestPath }), - resourcePath, - }, 'module.exports = {}') - - assert.match(result, /dd-trace:bundler:load/) - assert.match(result, /package: "ioredis"/) - assert.match(result, /path: "ioredis"/) - assert.ok(result.includes(`require(${JSON.stringify(relativeImport( - path.dirname(resourcePath), require.resolve('dc-polyfill') - ))})`)) - }) - - it('publishes supported relative runtime modules through the bundler channel', () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) - directories.push(directory) - const resourcePath = write(directory, 'generated/prisma/runtime/library.js', 'module.exports = {}') - const manifestPath = write(directory, 'manifest.json', JSON.stringify({ - relativeTargets: [{ - file: 'runtime/library.js', - name: './runtime/library.js', - path: './runtime/library.js', - version: '6.1.0', - }], - targets: {}, - })) - - const result = loader.call({ - getOptions: () => ({ manifestPath }), - resourcePath, - }, 'module.exports = {}') - - assert.match(result, /package: "\.\/runtime\/library\.js"/) - assert.match(result, /path: "\.\/runtime\/library\.js"/) - }) - - it('publishes generated ESM proxies through the existing bundler channel', async () => { - const directory = createPackage('openai', { type: 'module' }) - const resourcePath = write(directory, 'index.mjs', 'export const client = true') - const proxyPath = write(directory, '../.cache/dd-trace/turbopack/openai.mjs', '') - - const result = await createEsmProxy(resourcePath, proxyPath, 'openai', 'openai', '5.0.0') - - assert.ok(result.includes(`from ${JSON.stringify(relativeImport( - path.dirname(proxyPath), require.resolve('dc-polyfill') - ))}`)) - assert.match(result, /dd-trace:bundler:load/) - assert.match(result, /apply \(exports, patchDefault\)/) - assert.match(result, /set\.default\?\.\(exports\)/) - assert.doesNotMatch(result, /import-in-the-middle/) - }) - - it('applies existing rewriter instrumentation to an ESM target', async () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) - directories.push(directory) - fs.writeFileSync(path.join(directory, 'package.json'), '{}') - const packagePath = createPackageIn(directory, 'ai', { main: 'dist/index.js', type: 'module', version: '6.0.0' }) - const target = write(packagePath, 'dist/index.js', [ - 'export function resolveLanguageModel (model) {', - ' return model', - '}', - '', - ].join('\n')) - const manifest = await createManifest(directory) - - const result = loader.call({ - getOptions: () => ({ manifestPath: manifest.path }), - resourcePath: target, - }, fs.readFileSync(target, 'utf8')) - - assert.notEqual(result, fs.readFileSync(target, 'utf8')) - assert.match(result, /sourceMappingURL=data:application\/json;base64,/) - }) -}) - -describe('datadog-turbopack configuration', () => { - it('limits relative runtime rules to compatible package versions', () => { - require('../../datadog-instrumentations/src/prisma') - - const supported = getRelativeTargets([{ name: '@prisma/client', version: '6.1.0' }], new Set()) - const unsupported = getRelativeTargets([{ name: '@prisma/client', version: '7.0.0' }], new Set()) - - assert.deepEqual(supported, [{ - file: 'runtime/library.js', - name: './runtime/library.js', - path: './runtime/library.js', - version: '6.1.0', - }]) - assert.deepEqual(unsupported, []) - }) - - it('discovers nested copies of supported packages', async () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) - directories.push(directory) - fs.writeFileSync(path.join(directory, 'package.json'), '{}') - const nested = createPackageIn(directory, 'parent/node_modules/ioredis', { - main: 'index.js', - version: '5.0.0', - }) - const target = write(nested, 'index.js', 'module.exports = {}') - - const manifest = await createManifest(directory) - const targets = JSON.parse(fs.readFileSync(manifest.path, 'utf8')).targets - - assert.equal(targets[realpath(target)].name, 'ioredis') - }) - - it('does not generate targets for disabled integrations', async () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) - directories.push(directory) - fs.writeFileSync(path.join(directory, 'package.json'), '{}') - const packagePath = createPackageIn(directory, 'ioredis', { main: 'index.js', version: '5.0.0' }) - write(packagePath, 'index.js', 'module.exports = {}') - const previous = process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS - process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS = 'ioredis' - - try { - assert.deepEqual(await createManifest(directory), {}) - } finally { - if (previous === undefined) delete process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS - else process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS = previous - } - }) - - it('does not add rules when no supported package is installed', async () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) - directories.push(directory) - fs.writeFileSync(path.join(directory, 'package.json'), '{}') - const config = { rules: { '*.js': { loaders: ['existing-loader'] } } } - - assert.strictEqual(await withDatadogTurbopack(config, directory), config) - }) - - it('leaves configuration unchanged when its cache directory cannot be created', async () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) - directories.push(directory) - fs.writeFileSync(path.join(directory, 'package.json'), '{}') - const packagePath = createPackageIn(directory, 'ioredis', { main: 'index.js', version: '5.0.0' }) - write(packagePath, 'index.js', 'module.exports = {}') - fs.writeFileSync(path.join(directory, 'node_modules', '.cache'), '') - const config = { rules: { '*.js': { loaders: ['existing-loader'] } } } - - assert.strictEqual(await withDatadogTurbopack(config, directory), config) - }) - - it('does not require Next.js and preserves existing Turbopack settings', async () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) - directories.push(directory) - fs.writeFileSync(path.join(directory, 'package.json'), '{}') - const packagePath = createPackageIn(directory, 'ioredis', { main: 'index.js', version: '5.0.0' }) - write(packagePath, 'index.js', 'module.exports = {}') - - const config = await withDatadogTurbopack({ - resolveAlias: { existing: './existing.js' }, - rules: { '*.js': { loaders: ['existing-loader'] } }, - }, directory) - - assert.equal(config.resolveAlias.existing, './existing.js') - assert.equal(config.rules['*.js'].length, 2) - assert.deepEqual(config.rules['*.js'][0], { loaders: ['existing-loader'] }) - assert.match(config.rules['*.js'][1].condition.all[2].path.source, /node_modules/) - }) - - it('does not add the Datadog loader twice when composed repeatedly', async () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) - directories.push(directory) - fs.writeFileSync(path.join(directory, 'package.json'), '{}') - const packagePath = createPackageIn(directory, 'ioredis', { main: 'index.js', version: '5.0.0' }) - write(packagePath, 'index.js', 'module.exports = {}') - - const once = await withDatadogTurbopack({}, directory) - const twice = await withDatadogTurbopack(once, directory) - - for (const extension of ['*.js', '*.cjs', '*.mjs', '*.jsx', '*.ts', '*.tsx']) { - const rules = [twice.rules[extension]].flat() - assert.equal(rules.filter(rule => rule.loaders.some(item => item.loader.includes('datadog-turbopack'))).length, 1) - } - }) - - it('adds a Node-only rule for application ESM imports and requires', async () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) - directories.push(directory) - fs.writeFileSync(path.join(directory, 'package.json'), '{}') - const packagePath = createPackageIn(directory, 'ai', { main: 'index.mjs', type: 'module', version: '7.0.0' }) - write(packagePath, 'index.mjs', 'export const generateText = () => {}') - - const config = await withDatadogTurbopack({}, directory) - const rules = [config.rules['*.js']].flat() - const applicationRule = rules.find(rule => rule.condition.all.some(condition => condition?.not === 'foreign')) - - assert.deepEqual(applicationRule.condition.all.slice(0, 2), ['node', { not: 'foreign' }]) - assert.match('import { generateText } from "ai"', applicationRule.condition.all[2].content) - assert.match('const { generateText } = require("ai")', applicationRule.condition.all[2].content) - assert.doesNotMatch('import { something } from "unrelated"', applicationRule.condition.all[2].content) - assert.equal(applicationRule.loaders[0].options.rewriteApplicationImports, true) - assert.match(applicationRule.loaders[0].options.manifestHash, /^[a-f0-9]{64}$/) - assert.deepEqual(applicationRule.loaders[0].options.aliases, []) - assert.ok(config.rules['*.ts']) - assert.ok(config.rules['*.tsx']) - }) -}) - -function createPackage (name, manifest = {}) { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-')) - directories.push(directory) - - return createPackageIn(directory, name, manifest) -} - -function createPackageIn (directory, name, manifest = {}) { - const packagePath = path.join(directory, 'node_modules', name) - fs.mkdirSync(packagePath, { recursive: true }) - fs.writeFileSync(path.join(packagePath, 'package.json'), JSON.stringify({ name, ...manifest })) - return packagePath -} - -function write (directory, relativePath, content) { - const target = path.resolve(directory, relativePath) - fs.mkdirSync(path.dirname(target), { recursive: true }) - fs.writeFileSync(target, content) - return target -} - -function realpath (file) { - return fs.realpathSync(file).replaceAll('\\', '/') -} - -function relativeImport (from, to) { - let value = path.relative(from, to).replaceAll('\\', '/') - if (!value.startsWith('.')) value = `./${value}` - return value -} diff --git a/turbopack.d.ts b/turbopack.d.ts deleted file mode 100644 index 7415dd7827e..00000000000 --- a/turbopack.d.ts +++ /dev/null @@ -1,12 +0,0 @@ -export interface TurbopackConfiguration { - rules?: Record - resolveAlias?: Record -} - -/** - * Adds Node.js Turbopack rules for supported dd-trace integrations. - */ -export function withDatadogTurbopack ( - turbopack?: T, - projectDir?: string -): Promise From ba4dca82129216aa52b172d91ee5e980dadbbfa6 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Mon, 31 Aug 2026 16:38:41 +0200 Subject: [PATCH 8/9] ci(turbopack): fix clean package checks 1. Clean installs hoisted Babel 8's ESM parser into the CommonJS fake Next compiler because its Babel 7 inputs were only transitive dependencies. 2. Bun applied the root files allowlist to nested READMEs while npm included them automatically, so the package archives differed. --- package.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/package.json b/package.json index 7b13fd5a7a1..2378af25a9b 100644 --- a/package.json +++ b/package.json @@ -174,6 +174,7 @@ "packages/*/lib/**/*", "packages/*/src/**/*", "packages/datadog-instrumentations/orchestrion.yml", + "packages/datadog-turbopack/README.md", "README.md", "register.js", "vendor/dist/**/*.d.ts", @@ -202,7 +203,10 @@ "devDependencies": { "@actions/core": "^3.0.1", "@actions/github": "^9.1.1", + "@babel/generator": "^7.29.7", "@babel/helpers": "^8.0.0", + "@babel/parser": "^7.29.7", + "@babel/traverse": "^7.29.7", "@eslint/eslintrc": "^3.3.6", "@eslint/js": "^10.0.1", "@eslint/plugin-kit": "^0.7.2", From 276c51ff07b5cb9749d505dad8d1dd2ae1e05eec Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Mon, 31 Aug 2026 17:39:16 +0200 Subject: [PATCH 9/9] test(turbopack): gate generated lint on ESLint support Backport CI widens the repository engine range to include Node 18 and 20. The test then loads ESLint 10 on unsupported runtimes, which fail while parsing its /v regular expressions. --- packages/datadog-turbopack/test/loader.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/datadog-turbopack/test/loader.spec.js b/packages/datadog-turbopack/test/loader.spec.js index 06922db2f47..a07d084c29e 100644 --- a/packages/datadog-turbopack/test/loader.spec.js +++ b/packages/datadog-turbopack/test/loader.spec.js @@ -11,10 +11,10 @@ const { afterEach, describe, it } = require('mocha') const dc = require('dc-polyfill') const enhancedResolve = require('enhanced-resolve') const { ESLint } = require('eslint') +const { engines: eslintEngines } = require('eslint/package.json') const semver = require('semver') const sinon = require('sinon') -const { engines, nodeMaxMajor } = require('../../../package.json') const { withDatadogTurbopack } = require('../../../next') const loader = require('../src/loader') const { @@ -26,7 +26,7 @@ const { } = require('./helpers') const CHANNEL = 'dd-trace:bundler:load' -const lintRuntimeSupported = semver.satisfies(process.version, `${engines.node} <${nodeMaxMajor}`) +const lintRuntimeSupported = semver.satisfies(process.version, eslintEngines.node) afterEach(() => { cleanup()