diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ac5b6d66a8a..ce19a6477e4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -100,11 +100,13 @@ /index.electron.js @DataDog/dd-trace-js @DataDog/apm-idm-js /integration-tests/electron/ @DataDog/dd-trace-js @DataDog/apm-idm-js /integration-tests/esbuild/ @DataDog/dd-trace-js @DataDog/apm-idm-js +/integration-tests/turbopack/ @DataDog/dd-trace-js @DataDog/apm-idm-js /integration-tests/webpack/ @DataDog/dd-trace-js @DataDog/apm-idm-js /integration-tests/pino/ @DataDog/dd-trace-js @DataDog/apm-idm-js /integration-tests/pino.spec.js @DataDog/dd-trace-js @DataDog/apm-idm-js /packages/datadog-esbuild/ @DataDog/dd-trace-js @DataDog/apm-idm-js +/packages/datadog-turbopack/ @DataDog/dd-trace-js @DataDog/apm-idm-js /packages/datadog-webpack/ @DataDog/dd-trace-js @DataDog/apm-idm-js /packages/datadog-plugin-*/ @DataDog/dd-trace-js @DataDog/apm-idm-js /packages/datadog-instrumentations/ @DataDog/dd-trace-js @DataDog/apm-idm-js diff --git a/.github/workflows/instrumentation.yml b/.github/workflows/instrumentation.yml index 09b8f1f7e65..650cb1fd709 100644 --- a/.github/workflows/instrumentation.yml +++ b/.github/workflows/instrumentation.yml @@ -36,6 +36,22 @@ jobs: with: flags: platform-esbuild + turbopack: + runs-on: ubuntu-latest + permissions: + id-token: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/node/oldest-maintenance-lts + - uses: ./.github/actions/install + - run: npm run test:turbopack:ci + - uses: ./.github/actions/node/latest + - run: npm run test:turbopack:ci + - run: npm run test:integration:turbopack + - uses: ./.github/actions/coverage + with: + flags: platform-turbopack + webpack: runs-on: ubuntu-latest permissions: diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 9fae5b5de66..97f14794226 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -53,11 +53,13 @@ "dd-trace","https://github.com/DataDog/dd-trace-js","['(Apache-2.0 OR BSD-3-Clause)']","['Datadog Inc. ']" "debug","https://github.com/debug-js/debug","['MIT']","['Josh Junon']" "detect-newline","https://github.com/sindresorhus/detect-newline","['MIT']","['Sindre Sorhus']" +"enhanced-resolve","https://github.com/webpack/enhanced-resolve","['MIT']","['JS Foundation and other contributors']" "es-module-lexer","https://github.com/guybedford/es-module-lexer","['MIT']","['Guy Bedford']" "escape-string-regexp","https://github.com/sindresorhus/escape-string-regexp","['MIT']","['Sindre Sorhus']" "esquery","https://github.com/estools/esquery","['BSD-3-Clause']","['Joel Feenstra']" "estraverse","https://github.com/estools/estraverse","['BSD-2-Clause']","['estools']" "fast-fifo","https://github.com/mafintosh/fast-fifo","['MIT']","['Mathias Buus']" +"graceful-fs","https://github.com/isaacs/node-graceful-fs","['ISC']","['Isaac Z. Schlueter, Ben Noordhuis, and Contributors']" "https-proxy-agent","https://github.com/TooTallNate/proxy-agents","['MIT']","['Nathan Rajlich']" "import-in-the-middle","https://github.com/nodejs/import-in-the-middle","['Apache-2.0']","['Bryan English']" "istanbul-lib-coverage","https://github.com/istanbuljs/istanbuljs","['BSD-3-Clause']","['Krishnan Anantheswaran']" @@ -87,6 +89,7 @@ "shell-quote","https://github.com/ljharb/shell-quote","['MIT']","['James Halliday']" "source-map","https://github.com/mozilla/source-map","['BSD-3-Clause']","['Nick Fitzgerald']" "spark-md5","https://github.com/satazor/js-spark-md5","['(WTFPL OR MIT)']","['André Cruz']" +"tapable","https://github.com/webpack/tapable","['MIT']","['JS Foundation and other contributors']" "tlhunter-sorted-set","https://github.com/tlhunter/node-sorted-set","['MIT']","['Thomas Hunter II']" "tslib","https://github.com/microsoft/tslib","['0BSD']","['Microsoft Corp.']" "ttl-set","https://github.com/watson/ttl-set","['MIT']","['Thomas Watson']" diff --git a/README.md b/README.md index 04a8ee2b321..e2836fdfe25 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,17 @@ Regardless of where you open the issue, someone at Datadog will try to help. If you would like to trace your bundled application then please read this page on [bundling and dd-trace](https://docs.datadoghq.com/tracing/trace_collection/automatic_instrumentation/dd_libraries/nodejs/#bundling). It includes information on how to use our ESBuild plugin and includes caveats for other bundlers. +Next.js applications that use Turbopack can wrap their existing configuration: + +```javascript +const { withDatadogTurbopack } = require('dd-trace/next') + +module.exports = withDatadogTurbopack({}) +``` + +Preload `dd-trace/init` before application modules load. The generated modules can then publish their exports to the tracer. + +See the [Turbopack implementation](packages/datadog-turbopack/README.md) for the internal build and runtime flow. ## Security Vulnerabilities diff --git a/integration-tests/turbopack/app/api/esm/route.js b/integration-tests/turbopack/app/api/esm/route.js new file mode 100644 index 00000000000..9d27b67782c --- /dev/null +++ b/integration-tests/turbopack/app/api/esm/route.js @@ -0,0 +1,15 @@ +'use strict' + +async function GET () { + // eslint-disable-next-line n/no-missing-import -- dependency is installed in the integration sandbox + const { generateText } = await import('ai') + const model = require('../../model') + const result = await generateText({ + model, + prompt: 'Say ok', + experimental_telemetry: { isEnabled: true }, + }) + return Response.json({ dependency: typeof generateText === 'function' ? 'ai' : 'missing', text: result.text }) +} + +module.exports = { GET } diff --git a/integration-tests/turbopack/app/model.js b/integration-tests/turbopack/app/model.js new file mode 100644 index 00000000000..bd040fc4041 --- /dev/null +++ b/integration-tests/turbopack/app/model.js @@ -0,0 +1,19 @@ +'use strict' + +module.exports = { + specificationVersion: 'v3', + provider: 'turbopack-test', + modelId: 'turbopack-test', + supportedUrls: {}, + doGenerate () { + return Promise.resolve({ + content: [{ type: 'text', text: 'ok' }], + finishReason: { unified: 'stop', raw: undefined }, + usage: { + inputTokens: { total: 1 }, + outputTokens: { total: 1 }, + }, + warnings: [], + }) + }, +} diff --git a/integration-tests/turbopack/index.spec.js b/integration-tests/turbopack/index.spec.js new file mode 100644 index 00000000000..02bd1e9f1cb --- /dev/null +++ b/integration-tests/turbopack/index.spec.js @@ -0,0 +1,67 @@ +'use strict' + +const assert = require('node:assert/strict') +const { execSync } = require('node:child_process') +const path = require('node:path') + +const axios = require('axios') + +const { + FakeAgent, + checkSpansForServiceName, + sandboxCwd, + spawnPluginIntegrationTestProc, + stopProc, + useSandbox, +} = require('../helpers') + +for (const nextVersion of ['15.5.0', 'latest']) { + describe(`Turbopack integration with Next.js ${nextVersion}`, () => { + useSandbox([`next@${nextVersion}`, 'react', 'react-dom', 'ai', 'express'], false, [__dirname]) + + let agent + let proc + + before(function () { + this.timeout(300_000) + execSync('npm exec -- next build --turbopack', { cwd: appCwd(), stdio: 'inherit' }) + }) + + beforeEach(async () => { + agent = await new FakeAgent().start() + proc = await spawnPluginIntegrationTestProc(appCwd(), 'server.js', agent.port, { + NODE_OPTIONS: '-r dd-trace/init', + }) + }) + + afterEach(async () => { + await stopProc(proc) + await agent.stop() + }) + + it('runs bundled CommonJS and ESM dependencies', async () => { + const assertCjsTrace = agent.assertMessageReceived(({ payload }) => { + assert.strictEqual(checkSpansForServiceName(payload, 'next.request'), true) + assert.strictEqual(checkSpansForServiceName(payload, 'express.request'), true) + assert.strictEqual(checkSpansForServiceName(payload, 'generateText'), true) + }, 10_000, 1, true) + + const response = await axios.get(`${proc.url}/api/cjs`) + assert.deepStrictEqual(response.data, { dependency: 'express', text: 'ok' }) + await assertCjsTrace + + const assertEsmTrace = agent.assertMessageReceived(({ payload }) => { + assert.strictEqual(checkSpansForServiceName(payload, 'next.request'), true) + assert.strictEqual(checkSpansForServiceName(payload, 'generateText'), true) + }, 10_000, 1, true) + + const esmResponse = await axios.get(`${proc.url}/api/esm`) + assert.deepStrictEqual(esmResponse.data, { dependency: 'ai', text: 'ok' }) + await assertEsmTrace + }) + }) +} + +function appCwd () { + return path.join(sandboxCwd(), 'turbopack') +} diff --git a/integration-tests/turbopack/next.config.js b/integration-tests/turbopack/next.config.js new file mode 100644 index 00000000000..06c44c9d3ab --- /dev/null +++ b/integration-tests/turbopack/next.config.js @@ -0,0 +1,9 @@ +'use strict' + +const path = require('node:path') + +const { withDatadogTurbopack } = require('dd-trace/next') + +const root = path.dirname(__dirname) + +module.exports = withDatadogTurbopack({ turbopack: { root } }, { projectDir: root }) diff --git a/integration-tests/turbopack/pages/api/cjs.js b/integration-tests/turbopack/pages/api/cjs.js new file mode 100644 index 00000000000..dd1fb0166a4 --- /dev/null +++ b/integration-tests/turbopack/pages/api/cjs.js @@ -0,0 +1,18 @@ +'use strict' + +const express = require('express') +const { generateText } = require('ai') + +const model = require('../../app/model') + +const app = express() +app.use(async (_request, response) => { + const result = await generateText({ + model, + prompt: 'Say ok', + experimental_telemetry: { isEnabled: true }, + }) + response.json({ dependency: 'express', text: result.text }) +}) + +module.exports = (request, response) => app(request, response) diff --git a/integration-tests/turbopack/server.js b/integration-tests/turbopack/server.js new file mode 100644 index 00000000000..26a9fd52b4a --- /dev/null +++ b/integration-tests/turbopack/server.js @@ -0,0 +1,20 @@ +'use strict' + +const { createServer } = require('node:http') + +const next = require('next') + +const nextApp = next({ dev: false }) +const handle = nextApp.getRequestHandler() + +async function start () { + await nextApp.prepare() + + const server = createServer((request, response) => handle(request, response)) + server.listen(0, () => { + const port = server.address().port + process.send({ port }) + }) +} + +start() diff --git a/next.d.ts b/next.d.ts new file mode 100644 index 00000000000..cfb290e7b57 --- /dev/null +++ b/next.d.ts @@ -0,0 +1,15 @@ +import type { NextConfig } from 'next' + +export interface DatadogTurbopackOptions { + projectDir?: string +} + +export function withDatadogTurbopack ( + nextConfig: (...args: TArguments) => NextConfig | Promise, + options?: DatadogTurbopackOptions +): (...args: TArguments) => Promise + +export function withDatadogTurbopack ( + nextConfig?: NextConfig | Promise, + options?: DatadogTurbopackOptions +): Promise diff --git a/next.js b/next.js new file mode 100644 index 00000000000..03f10834988 --- /dev/null +++ b/next.js @@ -0,0 +1,3 @@ +'use strict' + +module.exports = require('./packages/datadog-turbopack') diff --git a/package.json b/package.json index 1d1cce64a7a..2378af25a9b 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,8 @@ "test:trace:guardrails:ci": "node scripts/c8-ci.js test:trace:guardrails", "test:esbuild": "mocha \"packages/datadog-esbuild/test/**/*.spec.js\"", "test:esbuild:ci": "node scripts/c8-ci.js test:esbuild", + "test:turbopack": "mocha \"packages/datadog-turbopack/test/**/*.spec.js\"", + "test:turbopack:ci": "node scripts/c8-ci.js test:turbopack", "test:webpack": "mocha \"packages/datadog-webpack/test/**/*.spec.js\"", "test:webpack:ci": "node scripts/c8-ci.js test:webpack", "test:instrumentations": "mocha --fail-zero \"packages/datadog-instrumentations/test/@(${PLUGINS}).spec.js\"", @@ -91,6 +93,7 @@ "test:integration:electron": "mocha \"integration-tests/electron/*.spec.js\"", "test:integration:esbuild": "mocha --timeout 60000 \"integration-tests/esbuild/*.spec.js\"", "test:integration:esbuild:coverage": "node ./integration-tests/coverage/run-suite.js --timeout 60000 \"integration-tests/esbuild/*.spec.js\"", + "test:integration:turbopack": "mocha --timeout 60000 \"integration-tests/turbopack/*.spec.js\"", "test:integration:webpack": "mocha --timeout 60000 \"integration-tests/webpack/*.spec.js\"", "test:integration:openfeature": "mocha --timeout 60000 \"integration-tests/openfeature/*.spec.js\"", "test:integration:openfeature:coverage": "node ./integration-tests/coverage/run-suite.js --timeout 60000 \"integration-tests/openfeature/*.spec.js\"", @@ -150,6 +153,8 @@ "ci/**/*", "cypress/**/*", "esbuild.js", + "next.js", + "next.d.ts", "webpack.js", "ext/**/*", "index.d.ts", @@ -169,6 +174,7 @@ "packages/*/lib/**/*", "packages/*/src/**/*", "packages/datadog-instrumentations/orchestrion.yml", + "packages/datadog-turbopack/README.md", "README.md", "register.js", "vendor/dist/**/*.d.ts", @@ -179,6 +185,7 @@ ], "dependencies": { "dc-polyfill": "^0.1.11", + "enhanced-resolve": "^5.17.1", "import-in-the-middle": "^3.3.2", "opentracing": ">=0.14.7" }, @@ -196,7 +203,10 @@ "devDependencies": { "@actions/core": "^3.0.1", "@actions/github": "^9.1.1", + "@babel/generator": "^7.29.7", "@babel/helpers": "^8.0.0", + "@babel/parser": "^7.29.7", + "@babel/traverse": "^7.29.7", "@eslint/eslintrc": "^3.3.6", "@eslint/js": "^10.0.1", "@eslint/plugin-kit": "^0.7.2", diff --git a/packages/datadog-esbuild/src/utils.js b/packages/datadog-esbuild/src/utils.js index df3b8de0286..6c20ccaea74 100644 --- a/packages/datadog-esbuild/src/utils.js +++ b/packages/datadog-esbuild/src/utils.js @@ -246,8 +246,8 @@ async function processModule ({ path, internal = false, context, excludeDefault * @returns {boolean} */ function isESMFile (fullPathToModule, modulePackageJsonPath, packageJson = {}) { - if (fullPathToModule.endsWith('.mjs')) return true - if (fullPathToModule.endsWith('.cjs')) return false + if (fullPathToModule.endsWith('.mjs') || fullPathToModule.endsWith('.mts')) return true + if (fullPathToModule.endsWith('.cjs') || fullPathToModule.endsWith('.cts')) return false const pathParts = fullPathToModule.split(path.sep) do { diff --git a/packages/datadog-esbuild/test/utils.spec.js b/packages/datadog-esbuild/test/utils.spec.js index eacc3f4aabe..274fc45b5d4 100644 --- a/packages/datadog-esbuild/test/utils.spec.js +++ b/packages/datadog-esbuild/test/utils.spec.js @@ -75,6 +75,14 @@ describe('esbuild utils', () => { assert.strictEqual(isESMFile('/path/to/test.cjs'), false) }) + it('should return true if the file has a .mts extension in a CommonJS package', () => { + assert.strictEqual(isESMFile('/path/to/test.mts', '/path/to/package.json', { type: 'commonjs' }), true) + }) + + it('should return false if the file has a .cts extension in an ESM package', () => { + assert.strictEqual(isESMFile('/path/to/test.cts', '/path/to/package.json', { type: 'module' }), false) + }) + it('should return true if the file is in a directory with a package.json that has a type of module', () => { assert.strictEqual(isESMFile('/path/to/test.js', '/path/to/package.json', { type: 'module' }), true) }) diff --git a/packages/datadog-instrumentations/src/helpers/bundler-constants.js b/packages/datadog-instrumentations/src/helpers/bundler-constants.js new file mode 100644 index 00000000000..3bba3c72836 --- /dev/null +++ b/packages/datadog-instrumentations/src/helpers/bundler-constants.js @@ -0,0 +1,5 @@ +'use strict' + +const BUNDLER_DC_GLOBAL = 'dd-trace:bundler:dc' + +module.exports = { BUNDLER_DC_GLOBAL } diff --git a/packages/datadog-instrumentations/src/helpers/bundler-register.js b/packages/datadog-instrumentations/src/helpers/bundler-register.js index 57da69cda5e..0a1271b98a1 100644 --- a/packages/datadog-instrumentations/src/helpers/bundler-register.js +++ b/packages/datadog-instrumentations/src/helpers/bundler-register.js @@ -4,13 +4,18 @@ const Module = require('module') const dc = require('dc-polyfill') const log = require('../../../dd-trace/src/log') +const { BUNDLER_DC_GLOBAL } = require('./bundler-constants') +const { loadChannel } = require('./register.js') const { - filename, - loadChannel, + getDisabledInstrumentations, + matchesInstrumentation, matchVersion, -} = require('./register.js') +} = require('./instrumentation-utils') const hooks = require('./hooks') const instrumentations = require('./instrumentations') +const disabledInstrumentations = getDisabledInstrumentations() + +globalThis[Symbol.for(BUNDLER_DC_GLOBAL)] = dc // register.js has now set up ritm (require-in-the-middle). In bundled // environments (webpack, esbuild), Node.js built-in modules required by @@ -69,18 +74,29 @@ function doHook (name) { try { hookFn() - } catch { - log.error('esbuild-wrapped %s hook failed', name) + } catch (error) { + log.error('esbuild-wrapped %s hook failed: %s', name, error.message, error) } } /** @type {Set} */ const instrumentedNodeModules = new Set() -/** @typedef {{ package: string, module: unknown, version: string, path: string }} Payload */ +/** + * @typedef {object} Payload + * @property {string} package + * @property {unknown} module + * @property {string} version + * @property {string} path + * @property {number[]} [instrumentationIndexes] + * @property {string} [moduleBaseDir] + * @property {string} [moduleName] + * @property {(exports: unknown, patchDefault: boolean) => void} [apply] + */ dc.subscribe(CHANNEL, (message) => { const payload = /** @type {Payload} */ (message) const name = payload.package + if (disabledInstrumentations.has(name)) return const isPrefixedWithNode = name.startsWith('node:') @@ -104,16 +120,35 @@ dc.subscribe(CHANNEL, (message) => { return } - for (const { file, versions, hook } of instrumentation) { - if (payload.path !== filename(name, file) || !matchVersion(payload.version, versions)) { + const indexes = payload.instrumentationIndexes ?? instrumentation.keys() + for (const index of indexes) { + const entry = instrumentation[index] + if (!entry) { + log.error('Bundled %s instrumentation index %s does not exist', name, index) continue } + if (payload.instrumentationIndexes === undefined && + !matchesInstrumentation(name, payload.version, payload.path, entry)) continue + + const { patchDefault, versions, hook } = entry + if (!matchVersion(payload.version, versions)) continue try { - loadChannel.publish({ name, version: payload.version, file }) - payload.module = hook(payload.module, payload.version) ?? payload.module - } catch (e) { - log.error('Error executing bundler hook', e) + loadChannel.publish({ name }) + let exports = payload.module + // Only generated ESM proxy payloads need default-export unwrapping. + if (typeof payload.apply === 'function' && patchDefault === !!exports.default) { + if (patchDefault) exports = exports.default + else continue + } + exports = hook(exports, payload.version, false, { + moduleBaseDir: payload.moduleBaseDir, + moduleName: payload.moduleName ?? payload.path, + }) ?? exports + payload.module = exports + payload.apply?.(exports, patchDefault) + } catch (error) { + log.error('Error executing bundler hook: %s', error.message, error) } } }) diff --git a/packages/datadog-instrumentations/src/helpers/instrumentation-utils.js b/packages/datadog-instrumentations/src/helpers/instrumentation-utils.js new file mode 100644 index 00000000000..272fd192727 --- /dev/null +++ b/packages/datadog-instrumentations/src/helpers/instrumentation-utils.js @@ -0,0 +1,68 @@ +'use strict' + +const { builtinModules } = require('node:module') + +const satisfies = require('../../../../vendor/dist/semifies') +const { getValueFromEnvSources } = require('../../../dd-trace/src/config/helper') +const { isRelativeRequire } = require('./shared-utils') + +/** + * @param {string|undefined} version + * @param {string[]|undefined} ranges + * @returns {boolean} + */ +function matchVersion (version, ranges) { + return !version || !ranges || ranges.some(range => satisfies(version, range)) +} + +/** + * @param {string} name + * @param {string} [file] + * @returns {string} + */ +function filename (name, file) { + return file ? `${name}/${file}` : name +} + +/** + * @param {string} name + * @param {string|undefined} version + * @param {string} moduleName + * @param {{ file?: string, filePattern?: string, versions?: string[] }} instrumentation + * @returns {boolean} + */ +function matchesInstrumentation (name, version, moduleName, instrumentation) { + const { file, filePattern, versions } = instrumentation + if (!matchVersion(version, versions)) return false + if (isRelativeRequire(name)) return true + if (moduleName === filename(name, file)) return true + return filePattern !== undefined && new RegExp(filename(name, filePattern)).test(moduleName) +} + +/** + * @returns {Set} + */ +function getDisabledInstrumentations () { + const disabled = new Set( + getValueFromEnvSources('DD_TRACE_DISABLED_INSTRUMENTATIONS')?.split(',').filter(Boolean) + ) + const expanded = new Set(disabled) + const builtins = new Set(builtinModules) + + for (const name of disabled) { + const prefixed = name.startsWith('node:') + if (!prefixed && !builtins.has(name)) continue + + const counterpart = prefixed ? name.slice(5) : `node:${name}` + expanded.add(counterpart) + } + + return expanded +} + +module.exports = { + filename, + getDisabledInstrumentations, + matchesInstrumentation, + matchVersion, +} diff --git a/packages/datadog-instrumentations/src/helpers/register.js b/packages/datadog-instrumentations/src/helpers/register.js index a1f9277371f..99612bb81fd 100644 --- a/packages/datadog-instrumentations/src/helpers/register.js +++ b/packages/datadog-instrumentations/src/helpers/register.js @@ -1,20 +1,20 @@ 'use strict' -const { builtinModules } = require('module') const path = require('path') const { channel } = require('dc-polyfill') -const satisfies = require('../../../../vendor/dist/semifies') const log = require('../../../dd-trace/src/log') const telemetry = require('../../../dd-trace/src/guardrails/telemetry') const { IS_SERVERLESS } = require('../../../dd-trace/src/serverless') const { getValueFromEnvSources } = require('../../../dd-trace/src/config/helper') const checkRequireCache = require('./check-require-cache') const Hook = require('./hook') -const { isRelativeRequire } = require('./shared-utils') +const { + filename, + getDisabledInstrumentations, + matchesInstrumentation, +} = require('./instrumentation-utils') const rewriter = require('./rewriter') -const DD_TRACE_DISABLED_INSTRUMENTATIONS = - getValueFromEnvSources('DD_TRACE_DISABLED_INSTRUMENTATIONS') const DD_TRACE_DEBUG = getValueFromEnvSources('DD_TRACE_DEBUG') const hooks = require('./hooks') @@ -22,9 +22,7 @@ const instrumentations = require('./instrumentations') const names = Object.keys(hooks) const pathSepExpr = new RegExp(`\\${path.sep}`, 'g') -const disabledInstrumentations = new Set( - DD_TRACE_DISABLED_INSTRUMENTATIONS?.split(',') -) +const disabledInstrumentations = getDisabledInstrumentations() const loadChannel = channel('dd-trace:instrumentation:load') @@ -54,29 +52,6 @@ const instrumentedIntegrationsSuccess = new Map() /** @type {Set} */ const alreadyLoggedIncompatibleIntegrations = new Set() -// Always disable prefixed and unprefixed node modules if one is disabled. -if (disabledInstrumentations.size) { - const builtinsSet = new Set(builtinModules) - const disabledBuiltinCounterparts = [] - for (const name of disabledInstrumentations) { - const hasPrefix = name.startsWith('node:') - if (hasPrefix || builtinsSet.has(name)) { - if (hasPrefix) { - const unprefixedName = name.slice(5) - if (!disabledInstrumentations.has(unprefixedName)) { - disabledBuiltinCounterparts.push(unprefixedName) - } - } else if (!disabledInstrumentations.has(`node:${name}`)) { - disabledBuiltinCounterparts.push(`node:${name}`) - } - } - } - for (const name of disabledBuiltinCounterparts) { - disabledInstrumentations.add(name) - } - builtinsSet.clear() -} - for (const name of names) { if (disabledInstrumentations.has(name)) continue @@ -111,21 +86,9 @@ for (const name of names) { instrumentedNodeModules.set(name, moduleExports) } - for (const { file, versions, hook, filePattern, patchDefault } of instrumentations[name]) { - const fullFilename = filename(name, file) - - let matchesFile = moduleName === fullFilename - - if (!matchesFile && isRelativeRequire(name)) matchesFile = true - - const fullFilePattern = filePattern && filename(name, filePattern) - if (fullFilePattern) { - // Some libraries include a hash in their filenames when installed, - // so our instrumentation has to include a '.*' to match them for more than a single version. - matchesFile ||= new RegExp(fullFilePattern).test(moduleName) - } - - if (matchesFile && matchVersion(moduleVersion, versions)) { + for (const instrumentation of instrumentations[name]) { + if (matchesInstrumentation(name, moduleVersion, moduleName, instrumentation)) { + const { hook, patchDefault } = instrumentation // IITM invokes this callback for every module in the package. Only unwrap the namespace after its file and // version match, otherwise a default export from an unrelated internal module can replace that module. if (isIitm && patchDefault === !!moduleExports.default) { @@ -188,26 +151,8 @@ function logAbortedIntegrations () { instrumentedIntegrationsSuccess.clear() } -/** - * @param {string|undefined} version - * @param {string[]|undefined} ranges - */ -function matchVersion (version, ranges) { - return !version || !ranges || ranges.some(range => satisfies(version, range)) -} - -/** - * @param {string} name - * @param {string} [file] - * @returns {string} - */ -function filename (name, file) { - return file ? `${name}/${file}` : name -} - module.exports = { filename, pathSepExpr, loadChannel, - matchVersion, } diff --git a/packages/datadog-instrumentations/src/helpers/rewriter/index.js b/packages/datadog-instrumentations/src/helpers/rewriter/index.js index 33a39fa200c..be1287c8f95 100644 --- a/packages/datadog-instrumentations/src/helpers/rewriter/index.js +++ b/packages/datadog-instrumentations/src/helpers/rewriter/index.js @@ -1,10 +1,12 @@ 'use strict' -const { readFileSync } = require('fs') -const { join } = require('path') -const { pathToFileURL } = require('url') +const { readFileSync } = require('node:fs') +const { join } = require('node:path') +const { pathToFileURL } = require('node:url') + const log = require('../../../../dd-trace/src/log') const { create } = require('../../../../../vendor/dist/@apm-js-collab/code-transformer') +const { BUNDLER_DC_GLOBAL } = require('../bundler-constants') const instrumentations = require('./instrumentations') const { getRewriteTarget } = require('./targets') const { awaitContextCallback, waitForAsyncEnd } = require('./transforms') @@ -33,11 +35,81 @@ const moduleVersions = {} const disabled = new Set() const matcherCjs = create(instrumentations, dcPolyfillCjs) const matcherEsm = create(instrumentations, dcPolyfillEsm) +const matcherBundler = create(instrumentations, 'node:diagnostics_channel') -for (const matcher of [matcherCjs, matcherEsm]) { +for (const matcher of [matcherCjs, matcherEsm, matcherBundler]) { matcher.addTransform('awaitContextCallback', awaitContextCallback) matcher.addTransform('waitForAsyncEnd', waitForAsyncEnd) } +matcherBundler.addTransform('tracingChannelImport', addBundlerTracingChannelImport) + +/** + * Reuses the process-wide polyfill installed by bundler-register while keeping + * the native diagnostics channel as the inactive-tracer fallback. + * + * @param {{ transforms: { defaults: { tracingChannelImport: Function } } }} state + * @param {{ body: object[] }} program + */ +function addBundlerTracingChannelImport (state, program) { + const previousLength = program.body.length + state.transforms.defaults.tracingChannelImport(state, program) + if (program.body.length === previousLength) return + + const index = program.body.findIndex(isNativeDcDeclaration) + const statement = program.body[index] + const identifier = statement.type === 'ImportDeclaration' + ? statement.specifiers[0].local + : statement.declarations[0].id + + identifier.name = 'tr_ch_apm_native_dc' + program.body.splice(index + 1, 0, createBundlerDcDeclaration()) +} + +/** + * @param {object} statement + * @returns {boolean} + */ +function isNativeDcDeclaration (statement) { + if (statement.type === 'ImportDeclaration') { + return statement.source?.value === 'node:diagnostics_channel' + } + const declaration = statement.declarations?.[0] + return declaration?.init?.arguments?.[0]?.value === 'node:diagnostics_channel' +} + +/** + * @returns {object} + */ +function createBundlerDcDeclaration () { + return { + type: 'VariableDeclaration', + declarations: [{ + type: 'VariableDeclarator', + id: { type: 'Identifier', name: 'tr_ch_apm_dc' }, + init: { + type: 'LogicalExpression', + operator: '??', + left: { + type: 'MemberExpression', + computed: true, + object: { type: 'Identifier', name: 'globalThis' }, + property: { + type: 'CallExpression', + arguments: [{ type: 'Literal', value: BUNDLER_DC_GLOBAL }], + callee: { + type: 'MemberExpression', + computed: false, + object: { type: 'Identifier', name: 'Symbol' }, + property: { type: 'Identifier', name: 'for' }, + }, + }, + }, + right: { type: 'Identifier', name: 'tr_ch_apm_native_dc' }, + }, + }], + kind: 'const', + } +} // Keep the marker split: source-map scanners can read a contiguous token in // string literals as this file's own inline map. @@ -52,10 +124,41 @@ const SOURCE_MAP_PREFIX = '//# sourceMapping' + 'URL=data:application/json;base6 * @returns {string|Buffer|ArrayBuffer|Uint8Array} */ function rewrite (content, filename, format, target) { - if (!content) return content + const { code, map } = rewriteWithMatcher(content, filename, format, target) + if (!map) return code + + return code + '\n' + SOURCE_MAP_PREFIX + Buffer.from(map).toString('base64') +} + +/** + * Rewrites source with a package specifier that bundlers can include in their + * output. + * + * @param {string|Buffer|ArrayBuffer|Uint8Array} content + * @param {string} filename + * @param {string} [format] + * @param {{ moduleName: string, filePath: string }} [target] + * @param {string|object} [sourceMap] + * @returns {{ code: string|Buffer|ArrayBuffer|Uint8Array, map?: string|object }} + */ +function rewriteBundledWithSourceMap (content, filename, format, target, sourceMap) { + return rewriteWithMatcher(content, filename, format, target, sourceMap, matcherBundler) +} + +/** + * @param {string|Buffer|ArrayBuffer|Uint8Array} content + * @param {string} filename + * @param {string} [format] + * @param {{ moduleName: string, filePath: string }} [target] + * @param {string|object} [sourceMap] + * @param {object} [bundlerMatcher] + * @returns {{ code: string|Buffer|ArrayBuffer|Uint8Array, map?: string|object }} + */ +function rewriteWithMatcher (content, filename, format, target, sourceMap, bundlerMatcher) { + if (!content) return { code: content, map: sourceMap } target ||= getRewriteTarget(filename) - if (!target) return content + if (!target) return { code: content, map: sourceMap } filename = filename.replace('file://', '') @@ -63,29 +166,22 @@ function rewrite (content, filename, format, target) { const { moduleName, filePath } = target const version = getVersion(filename, filePath) - if (disabled.has(moduleName)) return content + if (disabled.has(moduleName)) return { code: content, map: sourceMap } - const matcher = moduleType === 'esm' ? matcherEsm : matcherCjs + const matcher = bundlerMatcher ?? (moduleType === 'esm' ? matcherEsm : matcherCjs) const transformer = matcher.getTransformer(moduleName, version, filePath) - if (!transformer) return content + if (!transformer) return { code: content, map: sourceMap } try { const source = getSourceText(content) - - // TODO: pass existing sourcemap as input for remapping - const { code, map } = transformer.transform(source, moduleType) - - if (!map) return code - - const inlineMap = Buffer.from(map).toString('base64') - - return code + '\n' + SOURCE_MAP_PREFIX + inlineMap - } catch (e) { - log.error(e) + const { code, map } = transformer.transform(source, moduleType, sourceMap) + return { code, map } + } catch (error) { + log.error(error) } - return content + return { code: content, map: sourceMap } } /** @typedef {{ buffer: ArrayBuffer | SharedArrayBuffer, byteLength: number, byteOffset: number }} BufferView */ @@ -124,4 +220,4 @@ function getVersion (filename, filePath) { return moduleVersions[basename] } -module.exports = { rewrite, disable } +module.exports = { rewrite, rewriteBundledWithSourceMap, disable } diff --git a/packages/datadog-instrumentations/test/helpers/bundler-register.spec.js b/packages/datadog-instrumentations/test/helpers/bundler-register.spec.js new file mode 100644 index 00000000000..42d8d9e3c9a --- /dev/null +++ b/packages/datadog-instrumentations/test/helpers/bundler-register.spec.js @@ -0,0 +1,309 @@ +'use strict' + +const assert = require('node:assert/strict') +const Module = require('node:module') + +const sinon = require('sinon') + +const { BUNDLER_DC_GLOBAL } = require('../../src/helpers/bundler-constants') +const instrumentationUtils = require('../../src/helpers/instrumentation-utils') + +const CHANNEL = 'dd-trace:bundler:load' + +describe('bundler register', () => { + const dcGlobal = Symbol.for(BUNDLER_DC_GLOBAL) + let originalDc + let originalRequire + + beforeEach(() => { + originalDc = globalThis[dcGlobal] + originalRequire = Module.prototype.require + }) + + afterEach(() => { + if (originalDc === undefined) { + delete globalThis[dcGlobal] + } else { + globalThis[dcGlobal] = originalDc + } + Module.prototype.require = originalRequire + sinon.restore() + }) + + it('shares the polyfilled diagnostics channel with generated bundles', () => { + const { dc } = loadBundlerRegister({ hooks: {}, instrumentations: {} }) + + assert.strictEqual(globalThis[dcGlobal], dc) + }) + + it('honors patchDefault when applying an ESM proxy update', () => { + const Original = class Original {} + const Patched = class Patched {} + const hook = sinon.stub().returns(Patched) + const apply = sinon.stub() + const { loadChannel, publish } = loadBundlerRegister({ + hooks: { 'test-default-export': sinon.stub() }, + instrumentations: { + 'test-default-export': [{ file: 'index.mjs', hook, patchDefault: true }], + }, + }) + + publish({ + apply, + instrumentationIndexes: [0], + module: { default: Original }, + moduleBaseDir: '/app/node_modules/test-default-export', + moduleName: 'test-default-export/index.mjs', + package: 'test-default-export', + path: 'test-default-export/index.mjs', + version: '1.0.0', + }) + + sinon.assert.calledOnceWithExactly(hook, Original, '1.0.0', false, { + moduleBaseDir: '/app/node_modules/test-default-export', + moduleName: 'test-default-export/index.mjs', + }) + sinon.assert.calledOnceWithExactly(apply, Patched, true) + sinon.assert.calledOnceWithExactly(loadChannel.publish, { name: 'test-default-export' }) + }) + + it('patches a CommonJS object when its hook does not use a default export', () => { + const Original = class Original {} + const Patched = class Patched {} + const hook = sinon.stub().returns(Patched) + const { publish } = loadBundlerRegister({ + hooks: { 'test-commonjs-export': sinon.stub() }, + instrumentations: { + 'test-commonjs-export': [{ file: 'index.js', hook, patchDefault: false }], + }, + }) + const payload = { + module: { Original }, + package: 'test-commonjs-export', + path: 'test-commonjs-export/index.js', + version: '1.0.0', + } + + publish(payload) + + sinon.assert.calledOnceWithExactly(hook, { Original }, '1.0.0', false, { + moduleBaseDir: undefined, + moduleName: 'test-commonjs-export/index.js', + }) + assert.equal(payload.module, Patched) + }) + + it('does not activate explicitly disabled bundled integrations', () => { + const hook = sinon.stub() + const integrationHook = sinon.stub() + const { publish } = loadBundlerRegister({ + disabled: new Set(['test-disabled-integration']), + hooks: { 'test-disabled-integration': hook }, + instrumentations: { + 'test-disabled-integration': [{ hook: integrationHook }], + }, + }) + + publish({ + module: {}, + package: 'test-disabled-integration', + path: 'test-disabled-integration', + version: '1.0.0', + }) + + sinon.assert.notCalled(hook) + sinon.assert.notCalled(integrationHook) + }) + + it('matches bundled file-pattern hooks', () => { + const hook = sinon.stub() + const integrationHook = sinon.stub() + const { publish } = loadBundlerRegister({ + hooks: { 'test-pattern-hook': hook }, + instrumentations: { + 'test-pattern-hook': [{ filePattern: 'dist/cli.*', hook: integrationHook }], + }, + }) + + publish({ + module: {}, + package: 'test-pattern-hook', + path: 'test-pattern-hook/dist/cli-123.js', + version: '1.0.0', + }) + + sinon.assert.calledOnceWithExactly(integrationHook, {}, '1.0.0', false, { + moduleBaseDir: undefined, + moduleName: 'test-pattern-hook/dist/cli-123.js', + }) + }) + + it('matches bundled relative-module hooks', () => { + const hook = sinon.stub() + const integrationHook = sinon.stub() + const { publish } = loadBundlerRegister({ + hooks: { './runtime/library.js': hook }, + instrumentations: { + './runtime/library.js': [{ file: 'runtime/library.js', hook: integrationHook }], + }, + }) + + publish({ + module: {}, + package: './runtime/library.js', + path: './runtime/library.js', + version: '6.1.0', + }) + + sinon.assert.calledOnceWithExactly(integrationHook, {}, '6.1.0', false, { + moduleBaseDir: undefined, + moduleName: './runtime/library.js', + }) + }) + + it('uses build-plan indexes without running sibling hooks', () => { + const skippedHook = sinon.stub() + const selectedHook = sinon.stub() + const { publish } = loadBundlerRegister({ + hooks: { 'test-indexed-hook': sinon.stub() }, + instrumentations: { + 'test-indexed-hook': [ + { file: 'first.js', hook: skippedHook }, + { file: 'second.js', hook: selectedHook }, + ], + }, + }) + + publish({ + instrumentationIndexes: [1], + module: {}, + package: 'test-indexed-hook', + path: 'test-indexed-hook/second.js', + version: '1.0.0', + }) + + sinon.assert.notCalled(skippedHook) + sinon.assert.calledOnce(selectedHook) + }) + + it('rejects stale build-plan entries and incompatible versions', () => { + const integrationHook = sinon.stub() + const { log, publish } = loadBundlerRegister({ + hooks: { 'test-stale-plan': sinon.stub() }, + instrumentations: { + 'test-stale-plan': [{ file: 'index.js', hook: integrationHook, versions: ['>=2'] }], + }, + }) + + publish({ + instrumentationIndexes: [1], + module: {}, + package: 'test-stale-plan', + path: 'test-stale-plan/index.js', + version: '2.0.0', + }) + publish({ + instrumentationIndexes: [0], + module: {}, + package: 'test-stale-plan', + path: 'test-stale-plan/index.js', + version: '1.0.0', + }) + publish({ + module: {}, + package: 'test-stale-plan', + path: 'test-stale-plan/other.js', + version: '2.0.0', + }) + + sinon.assert.notCalled(integrationHook) + sinon.assert.calledWithMatch(log.error, 'Bundled %s instrumentation index %s does not exist', 'test-stale-plan', 1) + }) + + it('contains loader and instrumentation failures', () => { + const loadHook = sinon.stub().throws(new Error('load failed')) + const integrationHook = sinon.stub().throws(new Error('patch failed')) + const { log, publish } = loadBundlerRegister({ + hooks: { 'test-hook-errors': loadHook }, + instrumentations: { + 'test-hook-errors': [{ hook: integrationHook }], + }, + }) + + publish({ + instrumentationIndexes: [0], + module: {}, + package: 'test-hook-errors', + path: 'test-hook-errors', + version: '1.0.0', + }) + + sinon.assert.calledWithMatch(log.error, 'esbuild-wrapped %s hook failed: %s', 'test-hook-errors', 'load failed') + sinon.assert.calledWithMatch(log.error, 'Error executing bundler hook: %s', 'patch failed') + }) + + it('does not apply an ESM hook without the export shape it expects', () => { + const integrationHook = sinon.stub() + const apply = sinon.stub() + const { publish } = loadBundlerRegister({ + hooks: { 'test-missing-export': sinon.stub() }, + instrumentations: { + 'test-missing-export': [{ hook: integrationHook, patchDefault: false }], + }, + }) + + publish({ + apply, + instrumentationIndexes: [0], + module: {}, + package: 'test-missing-export', + path: 'test-missing-export', + version: '1.0.0', + }) + + sinon.assert.notCalled(integrationHook) + sinon.assert.notCalled(apply) + }) +}) + +function loadBundlerRegister ({ disabled = new Set(), hooks, instrumentations }) { + const bundlerRegisterPath = require.resolve('../../src/helpers/bundler-register') + const originalRequire = Module.prototype.require + const loadChannel = { publish: sinon.stub() } + const log = { error: sinon.stub() } + const dc = { + subscribe: (channel, callback) => { + if (channel === CHANNEL) bundledModuleSubscriber = callback + }, + } + let bundledModuleSubscriber + const register = { loadChannel } + + Module.prototype.require = function (request) { + if (this.filename === bundlerRegisterPath) { + const stubs = { + './hooks': hooks, + './instrumentation-utils': { + ...instrumentationUtils, + getDisabledInstrumentations: () => disabled, + }, + './instrumentations': instrumentations, + './register.js': register, + '../../../dd-trace/src/log': log, + 'dc-polyfill': dc, + } + return stubs[request] || originalRequire.call(this, request) + } + return originalRequire.call(this, request) + } + delete require.cache[bundlerRegisterPath] + require('../../src/helpers/bundler-register') + Module.prototype.require = originalRequire + + return { + dc, + loadChannel, + log, + publish: message => bundledModuleSubscriber(message), + } +} diff --git a/packages/datadog-instrumentations/test/helpers/rewriter/index.spec.js b/packages/datadog-instrumentations/test/helpers/rewriter/index.spec.js index a95506216cb..bce9e80b259 100644 --- a/packages/datadog-instrumentations/test/helpers/rewriter/index.spec.js +++ b/packages/datadog-instrumentations/test/helpers/rewriter/index.spec.js @@ -419,6 +419,20 @@ describe('check-require-cache', () => { }, channelName: 'pregel_stream', }, + { + module: { + name: 'test-esm', + versionRange: '>=0.1', + filePath: 'pregel-class.js', + }, + functionQuery: { + methodName: 'stream', + className: 'Pregel', + kind: 'Sync', + returnKind: 'AsyncIterator', + }, + channelName: 'pregel_stream_secondary', + }, ], }) }) @@ -931,6 +945,80 @@ describe('check-require-cache', () => { assert.strictEqual(rewriter.rewrite(source, filename, 'module'), source) }) + it('should compose an existing source map for bundler consumers', () => { + const filename = resolve(__dirname, 'node_modules', 'test-trace-sync', 'index.js') + const source = readFileSync(filename, 'utf8') + const sourceMap = { + file: filename, + mappings: 'AAAA', + names: [], + sources: ['original.js'], + sourcesContent: [source], + version: 3, + } + + const result = rewriter.rewriteBundledWithSourceMap(source, filename, 'commonjs', { + moduleName: 'test-trace-sync', + filePath: 'index.js', + }, sourceMap) + const map = JSON.parse(result.map) + + assert.match(result.code, /tr_ch_apm_tracingChannel/) + assert.strictEqual(map.sources[0], 'original.js') + assert.strictEqual(map.sourcesContent[0], source) + assert.strictEqual(map.sources.includes('test-trace-sync/index.js'), true) + }) + + it('should use the shared bundler diagnostics channel with a native fallback', () => { + const filename = resolve(__dirname, 'node_modules', 'test-esm', 'pregel-class.js') + const source = readFileSync(filename, 'utf8') + const result = rewriter.rewriteBundledWithSourceMap(source, filename, 'module', { + moduleName: 'test-esm', + filePath: 'pregel-class.js', + }) + + assert.match(result.code, /\bimport\s+.+\s+from\s+"node:diagnostics_channel"/) + assert.match(result.code, /Symbol\.for\("dd-trace:bundler:dc"\)/) + assert.doesNotMatch(result.code, /dc-polyfill/) + assert.strictEqual(result.code.match(/node:diagnostics_channel/g)?.length, 1) + + const commonJsFilename = resolve(__dirname, 'node_modules', 'test-trace-sync', 'index.js') + const commonJsSource = readFileSync(commonJsFilename, 'utf8') + const commonJsResult = rewriter.rewriteBundledWithSourceMap( + commonJsSource, + commonJsFilename, + 'commonjs', + { moduleName: 'test-trace-sync', filePath: 'index.js' } + ) + + assert.match(commonJsResult.code, /require\("node:diagnostics_channel"\)/) + assert.match(commonJsResult.code, /Symbol\.for\("dd-trace:bundler:dc"\)/) + }) + + it('should preserve bundled sources that cannot be rewritten', () => { + const sourceMap = { mappings: '', version: 3 } + assert.deepStrictEqual( + rewriter.rewriteBundledWithSourceMap('', '/project/empty.js', 'module', undefined, sourceMap), + { code: '', map: sourceMap } + ) + assert.deepStrictEqual(rewriter.rewriteBundledWithSourceMap( + 'module.exports = true', + '/project/node_modules/missing/index.js', + 'commonjs', + { moduleName: 'missing', filePath: 'index.js' }, + sourceMap + ), { code: 'module.exports = true', map: sourceMap }) + + rewriter.disable('test-disabled') + assert.deepStrictEqual(rewriter.rewriteBundledWithSourceMap( + 'module.exports = true', + '/project/node_modules/test-disabled/index.js', + 'commonjs', + { moduleName: 'test-disabled', filePath: 'index.js' }, + sourceMap + ), { code: 'module.exports = true', map: sourceMap }) + }) + it('should use import when rewriting esm modules', () => { const filename = resolve(__dirname, 'node_modules', 'test-esm', 'pregel-class.js') diff --git a/packages/datadog-turbopack/README.md b/packages/datadog-turbopack/README.md new file mode 100644 index 00000000000..b7a9983bb89 --- /dev/null +++ b/packages/datadog-turbopack/README.md @@ -0,0 +1,111 @@ +# Turbopack instrumentation + +The Turbopack integration instruments server-side dependency modules during a Next.js build. +It keeps target discovery and source analysis outside the application request path. + +This document describes the internal implementation. See the [root README](../../README.md#bundling) for setup instructions. + +## Architecture + +```mermaid +flowchart LR + A["Next.js configuration"] --> B["withDatadogTurbopack"] + B --> C["createBuildPlan"] + C --> D["Build plan
and ESM proxies"] + B --> E["Turbopack rules"] + D --> F["Datadog loader"] + E --> F + F --> G["Instrumented server modules"] + G --> H["dd-trace:bundler:load"] + H --> I["Bundler instrumentation registry"] +``` + +`withDatadogTurbopack` connects the configuration phase and the build phase. +The configuration phase creates immutable artifacts and Turbopack rules. +The build phase transforms only the modules that match these rules. +The generated modules publish their exports when the application loads them. + +## Configuration phase + +`withDatadogTurbopack` normalizes the supplied Next.js configuration. +It resolves Next.js from the application project directory. +It preserves existing Turbopack settings and appends the Datadog rules. +Repeated wrapping does not add the Datadog loader again. + +`createBuildPlan` loads the existing Datadog instrumentation declarations. +It finds installed packages that match these declarations. +It resolves package entry points with the Node.js `import` and `require` conditions. +It also records relative target files for integrations that instrument files below a package root. + +The planner hashes each target source file. +It creates export setters and proxies for successful ESM targets. +It writes the plan and proxies as content-addressed artifacts. +Concurrent configuration calls can safely use the same artifacts. + +The plan contains the target metadata and the expected source hashes. +The wrapper passes the plan path and hash to the loader. +The loader rejects plan content that does not match this hash. + +## Rule registration + +The wrapper selects the rule shape that the detected Next.js release accepts. +It adds direct rules for installed instrumentation targets. +It adds an import-inspection rule when the plan contains an ESM target. +It adds a separate rule for relative targets when the plan contains them. + +The direct rules transform known dependency files. +The import-inspection rule finds active edges to planned ESM targets. +The relative rule matches a file by its relative name and source hash. + +## Loader phase + +Turbopack calls the Datadog loader for each matching server module. +The loader verifies and caches the build plan before it transforms source. +The plan cache and source-hash cache have fixed bounds. + +For a direct target, the loader compares the current source hash with the plan. +It skips a changed target because the stored instrumentation data can be stale. +It sends matching source to the shared bundler rewriter and preserves the source map. + +The loader adds a guarded publication block to each CommonJS target. +The subscriber can replace the published exports before the module returns them. + +An ESM importer cannot replace the imported module namespace. +The loader therefore redirects active edges to a generated proxy. +It uses the Turbopack resolver for each edge. +An `import` edge uses import conditions, and a `require` edge uses require conditions. +This preserves the application resolver behavior and its aliases. + +The proxy imports the original module and maintains live export bindings. +The instrumentation subscriber applies changed exports through generated setters. +Type-only TypeScript imports do not create runtime edges. +Node.js built-in modules do not enter edge resolution. + +## Runtime publication + +`dd-trace/init` installs the bundler subscriber before application modules load. +Generated code gets the shared diagnostic channel through a global symbol. +It uses the native diagnostic channel when the shared channel is not present. + +Generated code checks `channel.hasSubscribers` before it creates publication payloads. +A payload identifies the package, version, path, exports, and selected instrumentation entries. +The subscriber checks disabled integrations and version rules before it runs a hook. + +For CommonJS, the subscriber updates the payload module. +The generated block then copies the result to `module.exports`. +For ESM, the proxy applies the result to its live bindings. +The subscriber catches and logs hook errors so that they do not stop the application. + +## Failure behavior + +| Condition | Behavior | +| --- | --- | +| No installed target matches | The wrapper returns the original Next.js configuration. | +| A target cannot produce valid instrumentation data | The planner warns once and omits that target. | +| The plan hash or plan version is invalid | The loader stops the build instead of using invalid plan data. | +| A direct target source hash changed | The loader warns once and skips direct instrumentation for that target. | +| Import parsing, resolver setup, or source generation fails | The loader keeps the original module edges and continues direct target handling. | +| The runtime channel has no subscriber | Generated code does not create payloads or run instrumentation hooks. | + +The bounded warning set prevents repeated build warnings for the same failure. +The shared module-format classifier keeps ESM and CommonJS decisions consistent with the other bundler integration. diff --git a/packages/datadog-turbopack/index.js b/packages/datadog-turbopack/index.js new file mode 100644 index 00000000000..abbc40faeea --- /dev/null +++ b/packages/datadog-turbopack/index.js @@ -0,0 +1,304 @@ +'use strict' + +const fs = require('node:fs') +const Module = require('node:module') +const path = require('node:path') + +const { createBuildPlan } = require('./src/targets') + +const loader = require.resolve('./src/loader') +const SOURCE_EXTENSIONS = ['*.js', '*.cjs', '*.mjs', '*.jsx', '*.ts', '*.cts', '*.mts', '*.tsx'] +const SOURCE_PATH_PATTERN = /\.(?:cjs|cts|js|jsx|mjs|mts|ts|tsx)$/ + +/** + * Adds Datadog instrumentation to a Next.js configuration. + * + * @param {object|Promise|Function} [nextConfig] + * @param {{ projectDir?: string }} [options] + * @returns {Promise|Function} + */ +function withDatadogTurbopack (nextConfig = {}, options = {}) { + if (!options || typeof options !== 'object' || Array.isArray(options)) { + throw new TypeError('withDatadogTurbopack options must be an object') + } + if (options.projectDir !== undefined && typeof options.projectDir !== 'string') { + throw new TypeError('withDatadogTurbopack options.projectDir must be a string') + } + + const projectDir = path.resolve(options.projectDir ?? process.cwd()) + const nextInfo = getNextInfo(projectDir) + if (typeof nextConfig === 'function') { + return async function datadogNextConfig (...args) { + const config = await nextConfig.apply(this, args) + return addDatadogConfig(normalizeConfig(config), projectDir, nextInfo) + } + } + + return Promise.resolve(nextConfig).then(config => + addDatadogConfig(normalizeConfig(config), projectDir, nextInfo) + ) +} + +/** + * @param {object} nextConfig + * @param {string} projectDir + * @param {{ compiler: { generator: string, parser: string, traverse: string }, major: number }} nextInfo + * @returns {Promise} + */ +async function addDatadogConfig (nextConfig, projectDir, nextInfo) { + const plan = await createBuildPlan(projectDir) + if (!plan.packagePathPattern || !plan.targetPathPattern || !plan.path || !plan.hash) return nextConfig + + const turbopack = nextConfig.turbopack ?? {} + const configured = nextInfo.major === 15 + ? addLegacyRules(turbopack, plan, nextInfo.compiler) + : addModernRules(turbopack, plan, nextInfo.compiler) + + return { + ...nextConfig, + turbopack: configured, + } +} + +/** + * @param {object|undefined} config + * @returns {object} + */ +function normalizeConfig (config) { + if (config === undefined) return {} + if (!config || typeof config !== 'object' || Array.isArray(config)) { + throw new TypeError('withDatadogTurbopack expects a Next.js configuration object, promise, or function') + } + + const { turbopack } = config + if (turbopack !== undefined && (!turbopack || typeof turbopack !== 'object' || Array.isArray(turbopack))) { + throw new TypeError('nextConfig.turbopack must be an object') + } + if (turbopack?.rules !== undefined && + (!turbopack.rules || typeof turbopack.rules !== 'object' || Array.isArray(turbopack.rules))) { + throw new TypeError('nextConfig.turbopack.rules must be an object') + } + if (turbopack?.conditions !== undefined && + (!turbopack.conditions || typeof turbopack.conditions !== 'object' || Array.isArray(turbopack.conditions))) { + throw new TypeError('nextConfig.turbopack.conditions must be an object') + } + if (turbopack?.resolveAlias !== undefined && + (!turbopack.resolveAlias || typeof turbopack.resolveAlias !== 'object' || Array.isArray(turbopack.resolveAlias))) { + throw new TypeError('nextConfig.turbopack.resolveAlias must be an object') + } + + return config +} + +/** + * @param {object} turbopack + * @param {object} plan + * @param {{ generator: string, parser: string, traverse: string }} compiler + * @returns {object} + */ +function addModernRules (turbopack, plan, compiler) { + const rules = { ...turbopack.rules } + + for (const extension of SOURCE_EXTENSIONS) { + const existing = rules[extension] + if (hasDatadogLoader(existing)) continue + + const additions = [createModernTargetRule(plan, compiler)] + if (plan.moduleSyntaxPattern) additions.push(createModernImportRule(plan, compiler)) + if (plan.relativePathPattern) additions.push(createModernRelativeRule(plan)) + + if (existing === undefined) { + rules[extension] = additions.length === 1 ? additions[0] : additions + } else { + rules[extension] = Array.isArray(existing) ? [...existing, ...additions] : [existing, ...additions] + } + } + + return { ...turbopack, rules } +} + +/** + * @param {object} turbopack + * @param {object} plan + * @param {{ generator: string, parser: string, traverse: string }} compiler + * @returns {object} + */ +function addLegacyRules (turbopack, plan, compiler) { + const conditions = { ...turbopack.conditions } + const rules = { ...turbopack.rules } + + addLegacyRule( + conditions, + rules, + '#dd-trace/target', + { + path: new RegExp( + `(?:${plan.packagePathPattern.source}.*${SOURCE_PATH_PATTERN.source}|${plan.targetPathPattern.source})` + ), + }, + { node: { loaders: [createLoader(plan, { compiler, rewriteEdges: true, targetScope: 'direct' })] } } + ) + + if (plan.moduleSyntaxPattern) { + addLegacyRule( + conditions, + rules, + '#dd-trace/import', + { content: plan.moduleSyntaxPattern, path: SOURCE_PATH_PATTERN }, + { node: { foreign: false, loaders: [createLoader(plan, { compiler, rewriteEdges: true })] } } + ) + } + + if (plan.relativePathPattern) { + addLegacyRule( + conditions, + rules, + '#dd-trace/relative', + { path: plan.relativePathPattern }, + { node: { foreign: false, loaders: [createLoader(plan, { targetScope: 'relative' })] } } + ) + } + + return { ...turbopack, conditions, rules } +} + +/** + * @param {Record} conditions + * @param {Record} rules + * @param {string} name + * @param {object} condition + * @param {object} rule + */ +function addLegacyRule (conditions, rules, name, condition, rule) { + if (hasDatadogLoader(rules[name])) return + if (Object.hasOwn(conditions, name) || Object.hasOwn(rules, name)) { + throw new Error(`Next.js Turbopack configuration already uses the reserved condition ${name}`) + } + conditions[name] = condition + rules[name] = rule +} + +/** + * @param {object} plan + * @param {{ generator: string, parser: string, traverse: string }} compiler + * @returns {object} + */ +function createModernTargetRule (plan, compiler) { + return { + condition: { + all: ['node', { + any: [ + { path: plan.packagePathPattern }, + { path: plan.targetPathPattern }, + ], + }], + }, + loaders: [createLoader(plan, { compiler, rewriteEdges: true, targetScope: 'direct' })], + } +} + +/** + * @param {object} plan + * @param {{ generator: string, parser: string, traverse: string }} compiler + * @returns {object} + */ +function createModernImportRule (plan, compiler) { + return { + condition: { all: ['node', { not: 'foreign' }, { content: plan.moduleSyntaxPattern }] }, + loaders: [createLoader(plan, { compiler, rewriteEdges: true })], + } +} + +/** + * @param {object} plan + * @returns {object} + */ +function createModernRelativeRule (plan) { + return { + condition: { all: ['node', { not: 'foreign' }, { path: plan.relativePathPattern }] }, + loaders: [createLoader(plan, { targetScope: 'relative' })], + } +} + +/** + * @param {object} plan + * @param {{ + * compiler?: { generator: string, parser: string, traverse: string }, + * rewriteEdges?: boolean, + * targetScope?: 'direct'|'relative' + * }} [settings] + * @returns {object} + */ +function createLoader (plan, settings = {}) { + const options = { + manifestHash: plan.hash, + manifestPath: plan.path, + } + if (settings.compiler) options.compiler = settings.compiler + if (settings.rewriteEdges) options.rewriteEdges = true + if (settings.targetScope) options.targetScope = settings.targetScope + return { loader, options } +} + +/** + * @param {unknown} value + * @returns {boolean} + */ +function hasDatadogLoader (value) { + if (Array.isArray(value)) { + for (const item of value) { + if (hasDatadogLoader(item)) return true + } + return false + } + if (!value || typeof value !== 'object') return false + if (value.loader === loader) return true + + for (const key of Object.keys(value)) { + if (hasDatadogLoader(value[key])) return true + } + return false +} + +/** + * @param {string} projectDir + * @returns {{ compiler: { generator: string, parser: string, traverse: string }, major: number }} + */ +function getNextInfo (projectDir) { + let version + let appRequire + try { + appRequire = Module.createRequire(path.join(projectDir, 'package.json')) + const packagePath = appRequire.resolve('next/package.json') + version = JSON.parse(fs.readFileSync(packagePath, 'utf8')).version + } catch (error) { + throw new Error(`withDatadogTurbopack could not resolve Next.js from ${projectDir}`, { cause: error }) + } + + const match = /^(\d+)\.(\d+)\./.exec(version) + if (!match) throw new Error(`withDatadogTurbopack could not parse Next.js version ${version}`) + const major = Number(match[1]) + const minor = Number(match[2]) + if (major < 15 || (major === 15 && minor < 5)) { + throw new RangeError(`withDatadogTurbopack requires Next.js 15.5 or newer; found ${version}`) + } + + try { + return { + compiler: { + generator: appRequire.resolve('next/dist/compiled/babel/generator'), + parser: appRequire.resolve('next/dist/compiled/babel/parser'), + traverse: appRequire.resolve('next/dist/compiled/babel/traverse'), + }, + major, + } + } catch (error) { + throw new Error(`Next.js ${version} does not provide the compiler required by withDatadogTurbopack`, { + cause: error, + }) + } +} + +module.exports = { + withDatadogTurbopack, +} diff --git a/packages/datadog-turbopack/src/loader.js b/packages/datadog-turbopack/src/loader.js new file mode 100644 index 00000000000..dd2249f309c --- /dev/null +++ b/packages/datadog-turbopack/src/loader.js @@ -0,0 +1,623 @@ +'use strict' + +const { createHash } = require('node:crypto') +const fs = require('node:fs') +const { builtinModules } = require('node:module') +const path = require('node:path') + +const { BUNDLER_DC_GLOBAL } = require('../../datadog-instrumentations/src/helpers/bundler-constants') +const { isESMFile } = require('../../datadog-esbuild/src/utils') +const { rewriteBundledWithSourceMap } = require('../../datadog-instrumentations/src/helpers/rewriter') + +const BUILTIN_MODULES = new Set(builtinModules) +const CHANNEL = 'dd-trace:bundler:load' +const IMPORT_RESOLVE_OPTIONS = { conditionNames: ['node', 'import'] } +const BASE_PARSER_PLUGINS = [ + 'decorators-legacy', + 'explicitResourceManagement', + 'importAttributes', + 'jsx', +] +const JAVASCRIPT_PARSER_PLUGINS = [...BASE_PARSER_PLUGINS, 'flow'] +const MAX_CACHED_FILES = 2048 +const MAX_CACHED_PLANS = 16 +const MAX_WARNINGS = 128 +const MODULE_SYNTAX_PATTERN = /\b(?:export|import|require)\b/ +const PLAN_VERSION = 3 +const REQUIRE_RESOLVE_OPTIONS = { conditionNames: ['node', 'require'] } +const TYPESCRIPT_PARSER_PLUGINS = [...BASE_PARSER_PLUGINS, 'typescript'] + +/** @type {Map} */ +const fileHashes = new Map() +/** @type {Map} */ +const plans = new Map() +/** @type {Set} */ +const warnedErrors = new Set() + +/** + * @typedef {object} PlanTarget + * @property {boolean} esm + * @property {object[]} payloads + * @property {string} [proxyPath] + * @property {string} sourceHash + */ + +/** + * @typedef {object} BuildPlan + * @property {Record} proxies + * @property {Array} relativeTargets + * @property {Record} targets + * @property {number} version + */ + +/** + * @typedef {object} ModuleEdge + * @property {'import'|'require'} kind + * @property {object[]} nodes + * @property {string} specifier + */ + +/** + * @typedef {object} CollectState + * @property {Map} edges + */ + +/** + * @typedef {object} ResolutionState + * @property {object} ast + * @property {(code: string, sourceMap?: object) => void} callback + * @property {Function} generate + * @property {object} [inputSourceMap] + * @property {{ emitWarning?: (warning: Error) => void }} loaderContext + * @property {number} pending + * @property {string} resourcePath + * @property {boolean} rewritten + * @property {string} source + * @property {Record} targets + */ + +/** + * Instruments modules selected by a build plan generated from the installed + * dd-trace integrations. + * + * @param {string} source + * @param {object} [inputSourceMap] + * @returns {void} + */ +module.exports = function loader (source, inputSourceMap) { + const callback = this.async() + try { + load.call(this, source, inputSourceMap, callback) + } catch (error) { + callback(error) + } +} + +/** + * @this {{ + * async: Function, + * emitWarning?: (warning: Error) => void, + * getOptions: Function, + * getResolve: Function, + * resourcePath: string + * }} + * @param {string} source + * @param {object} [inputSourceMap] + * @param {(error?: Error, code?: string, sourceMap?: object) => void} callback + */ +function load (source, inputSourceMap, callback) { + const options = this.getOptions() + const plan = getPlan(options.manifestPath, options.manifestHash) + const resourcePath = normalizePath(this.resourcePath) + const esm = isESMFile(resourcePath) + const match = findTarget(resourcePath, plan, options.targetScope, this) + + /** + * @param {string} code + * @param {object} [sourceMap] + */ + function onRewritten (code, sourceMap) { + finishLoad(code, sourceMap, resourcePath, match, esm, callback) + } + + if (plan.proxies[resourcePath] || !options.rewriteEdges || !MODULE_SYNTAX_PATTERN.test(source)) { + onRewritten(source, inputSourceMap) + return + } + + rewriteModuleEdges(source, inputSourceMap, resourcePath, plan.targets, options.compiler, this, onRewritten) +} + +/** + * @param {string} manifestPath + * @param {string} manifestHash + * @returns {BuildPlan} + */ +function getPlan (manifestPath, manifestHash) { + if (typeof manifestPath !== 'string' || typeof manifestHash !== 'string') { + throw new TypeError('The Datadog Turbopack loader requires a build plan path and hash') + } + + const key = `${manifestPath}\0${manifestHash}` + const cached = plans.get(key) + if (cached) return cached + + const serialized = fs.readFileSync(manifestPath, 'utf8') + if (hash(serialized) !== manifestHash) { + throw new Error(`The Datadog Turbopack build plan at ${manifestPath} failed its integrity check`) + } + + const plan = JSON.parse(serialized) + if (plan?.version !== PLAN_VERSION || !plan.proxies || typeof plan.proxies !== 'object' || + Array.isArray(plan.proxies) || !Array.isArray(plan.relativeTargets) || + !plan.targets || typeof plan.targets !== 'object') { + throw new Error(`The Datadog Turbopack build plan at ${manifestPath} is not supported`) + } + + if (plans.size >= MAX_CACHED_PLANS) plans.delete(plans.keys().next().value) + plans.set(key, plan) + return plan +} + +/** + * @param {string} resourcePath + * @param {BuildPlan} plan + * @param {'direct'|'relative'} [targetScope] + * @param {{ emitWarning?: (warning: Error) => void }} loaderContext + * @returns {PlanTarget|undefined} + */ +function findTarget (resourcePath, plan, targetScope, loaderContext) { + const direct = plan.targets[resourcePath] + if (targetScope === 'direct') { + if (!direct) return + if (matchesSource(resourcePath, direct.sourceHash)) return direct + warnOnce(loaderContext, `changed:${resourcePath}`, `Skipped changed dependency ${resourcePath}`) + return + } + if (targetScope !== 'relative' || direct) return + + let sourceHash + for (const target of plan.relativeTargets) { + if (!resourcePath.endsWith(`/${target.file}`)) continue + sourceHash ??= getFileHash(resourcePath) + if (sourceHash !== target.sourceHash) continue + return target + } +} + +/** + * @param {string} source + * @param {object} [inputSourceMap] + * @param {string} resourcePath + * @param {Record} targets + * @param {{ generator: string, parser: string, traverse: string }} compiler + * @param {{ emitWarning?: (warning: Error) => void, getResolve: Function }} loaderContext + * @param {(code: string, sourceMap?: object) => void} callback + */ +function rewriteModuleEdges (source, inputSourceMap, resourcePath, targets, compiler, loaderContext, callback) { + let ast + let generate + const state = { edges: new Map() } + + try { + const { parse } = require(compiler.parser) + const traverse = require(compiler.traverse).default + generate = require(compiler.generator).default + const plugins = /\.(?:cts|mts|ts|tsx)$/.test(resourcePath) + ? TYPESCRIPT_PARSER_PLUGINS + : JAVASCRIPT_PARSER_PLUGINS + ast = parse(source, { plugins, sourceType: 'unambiguous' }) + traverse(ast, IMPORT_VISITORS, undefined, state) + } catch (error) { + warnOnce( + loaderContext, + `imports:${resourcePath}`, + `Could not inspect imports in ${resourcePath}: ${error.message}`, + error + ) + callback(source, inputSourceMap) + return + } + + if (state.edges.size === 0) { + callback(source, inputSourceMap) + return + } + + resolveModuleEdges( + state.edges, + source, + inputSourceMap, + resourcePath, + targets, + ast, + generate, + loaderContext, + callback + ) +} + +/** + * @param {Map} edges + * @param {string} source + * @param {object} inputSourceMap + * @param {string} resourcePath + * @param {Record} targets + * @param {object} ast + * @param {Function} generate + * @param {{ emitWarning?: (warning: Error) => void, getResolve: Function }} loaderContext + * @param {(code: string, sourceMap?: object) => void} callback + */ +function resolveModuleEdges ( + edges, + source, + inputSourceMap, + resourcePath, + targets, + ast, + generate, + loaderContext, + callback +) { + let importResolve + let requireResolve + try { + importResolve = loaderContext.getResolve(IMPORT_RESOLVE_OPTIONS) + requireResolve = loaderContext.getResolve(REQUIRE_RESOLVE_OPTIONS) + } catch (error) { + warnOnce( + loaderContext, + `resolver:${resourcePath}`, + `Could not initialize import resolution in ${resourcePath}: ${error.message}`, + error + ) + callback(source, inputSourceMap) + return + } + + const state = { + ast, + callback, + generate, + inputSourceMap, + loaderContext, + pending: edges.size, + resourcePath, + rewritten: false, + source, + targets, + } + const directory = path.dirname(resourcePath) + for (const edge of edges.values()) { + resolveModuleEdge(edge.kind === 'require' ? requireResolve : importResolve, directory, edge, state) + } +} + +/** + * @param {Function} resolve + * @param {string} directory + * @param {ModuleEdge} edge + * @param {ResolutionState} state + */ +function resolveModuleEdge (resolve, directory, edge, state) { + let settled = false + + /** + * @param {Error|null|undefined} error + * @param {string} [resolved] + */ + function onResolved (error, resolved) { + if (settled) return + settled = true + if (!error && resolved) { + state.rewritten = rewriteResolvedEdge( + edge, + resolved, + state.resourcePath, + state.targets, + state.loaderContext + ) || state.rewritten + } + completeModuleEdge(state) + } + + try { + resolve(directory, edge.specifier, onResolved) + } catch (error) { + onResolved(error) + } +} + +/** + * @param {ResolutionState} state + */ +function completeModuleEdge (state) { + state.pending-- + if (state.pending > 0) return + if (!state.rewritten) { + state.callback(state.source, state.inputSourceMap) + return + } + + try { + const { code, map } = state.generate(state.ast, { + inputSourceMap: state.inputSourceMap, + retainLines: true, + sourceFileName: state.resourcePath, + sourceMaps: true, + }, state.source) + state.callback(code, map) + } catch (error) { + warnOnce( + state.loaderContext, + `generate:${state.resourcePath}`, + `Could not generate rewritten imports in ${state.resourcePath}: ${error.message}`, + error + ) + state.callback(state.source, state.inputSourceMap) + } +} + +/** + * @param {{ node: object }} modulePath + * @param {CollectState} state + */ +function collectModuleDeclaration (modulePath, state) { + if (isTypeOnlyDeclaration(modulePath.node)) return + collectModuleSource(modulePath.node.source, 'import', state) +} + +/** + * @param {{ node: { arguments?: object[], callee?: object }, scope: { hasBinding: Function } }} modulePath + * @param {CollectState} state + */ +function collectCallExpression (modulePath, state) { + const { arguments: args, callee } = modulePath.node + if (callee?.type === 'Import' && args?.length > 0) { + collectModuleSource(args[0], 'import', state) + return + } + if (callee?.type === 'Identifier' && callee.name === 'require' && args?.length === 1 && + !modulePath.scope.hasBinding('require', true)) { + collectModuleSource(args[0], 'require', state) + } +} + +/** + * @param {{ node: { source?: object } }} modulePath + * @param {CollectState} state + */ +function collectImportExpression (modulePath, state) { + collectModuleSource(modulePath.node.source, 'import', state) +} + +/** + * @param {{ node: { importKind?: string, moduleReference?: { expression?: object, type?: string } } }} modulePath + * @param {CollectState} state + */ +function collectTypescriptImport (modulePath, state) { + const { importKind, moduleReference } = modulePath.node + if (importKind === 'type' || moduleReference?.type !== 'TSExternalModuleReference') return + collectModuleSource(moduleReference.expression, 'require', state) +} + +/** + * @param {object|undefined} moduleSource + * @param {'import'|'require'} kind + * @param {CollectState} state + */ +function collectModuleSource (moduleSource, kind, state) { + const specifier = getModuleSpecifier(moduleSource) + if (specifier === undefined || specifier.startsWith('node:') || BUILTIN_MODULES.has(specifier)) return + + const key = `${kind}\0${specifier}` + const existing = state.edges.get(key) + if (existing) { + existing.nodes.push(moduleSource) + return + } + state.edges.set(key, { kind, nodes: [moduleSource], specifier }) +} + +/** + * @param {object|undefined} moduleSource + * @returns {string|undefined} + */ +function getModuleSpecifier (moduleSource) { + if (moduleSource?.type === 'StringLiteral') return moduleSource.value + if (moduleSource?.type === 'TemplateLiteral' && moduleSource.expressions.length === 0) { + return moduleSource.quasis[0].value.cooked + } +} + +/** + * @param {object} declaration + * @returns {boolean} + */ +function isTypeOnlyDeclaration (declaration) { + if (declaration.importKind === 'type' || declaration.importKind === 'typeof' || + declaration.exportKind === 'type') return true + const typeKind = declaration.type === 'ImportDeclaration' ? 'importKind' : 'exportKind' + if ((declaration.type !== 'ImportDeclaration' && declaration.type !== 'ExportNamedDeclaration') || + declaration.specifiers.length === 0) return false + for (const specifier of declaration.specifiers) { + if (specifier[typeKind] !== 'type' && specifier[typeKind] !== 'typeof') return false + } + return true +} + +const IMPORT_VISITORS = { + CallExpression: collectCallExpression, + ExportAllDeclaration: collectModuleDeclaration, + ExportNamedDeclaration: collectModuleDeclaration, + ImportDeclaration: collectModuleDeclaration, + ImportExpression: collectImportExpression, + TSImportEqualsDeclaration: collectTypescriptImport, +} + +/** + * @param {ModuleEdge} edge + * @param {string} resolved + * @param {string} resourcePath + * @param {Record} targets + * @param {{ emitWarning?: (warning: Error) => void }} loaderContext + * @returns {boolean} + */ +function rewriteResolvedEdge (edge, resolved, resourcePath, targets, loaderContext) { + let resolvedPath + try { + resolvedPath = normalizePath(resolved) + } catch { + return false + } + + const target = targets[resolvedPath] + if (!target?.esm || !target.proxyPath) return false + if (!matchesSource(resolvedPath, target.sourceHash)) { + warnOnce(loaderContext, `changed:${resolvedPath}`, `Skipped changed dependency ${resolvedPath}`) + return false + } + + const replacement = relativeImport(path.dirname(resourcePath), target.proxyPath) + for (const node of edge.nodes) setModuleSpecifier(node, replacement) + return true +} + +/** + * @param {object} moduleSource + * @param {string} value + */ +function setModuleSpecifier (moduleSource, value) { + if (moduleSource.type === 'StringLiteral') { + moduleSource.value = value + moduleSource.extra = undefined + return + } + + const templateValue = moduleSource.quasis[0].value + templateValue.cooked = value + templateValue.raw = value + .replaceAll('\\', '\\\\') + .replaceAll('`', '\\`') + .replaceAll('${', '\\${') +} + +/** + * @param {string} source + * @param {object} [sourceMap] + * @param {string} resourcePath + * @param {PlanTarget} [match] + * @param {boolean} esm + * @param {(error?: Error, code?: string, sourceMap?: object) => void} callback + */ +function finishLoad (source, sourceMap, resourcePath, match, esm, callback) { + if (!match) { + callback(undefined, source, sourceMap) + return + } + + const rewritten = rewriteBundledWithSourceMap( + source, + resourcePath, + esm ? 'module' : 'commonjs', + undefined, + sourceMap + ) + const code = esm ? rewritten.code : appendCommonJsPublications(rewritten.code, resourcePath, match) + callback(undefined, code, rewritten.map) +} + +/** + * @param {string} source + * @param {string} resourcePath + * @param {{ payloads: object[] }} match + * @returns {string} + */ +function appendCommonJsPublications (source, resourcePath, match) { + let publications = '' + let publicationIndex = 0 + + for (const payload of match.payloads) { + const payloadName = `payload${publicationIndex++}` + publications += ` const ${payloadName} = { + instrumentationIndexes: ${JSON.stringify(payload.instrumentationIndexes)}, + module: module.exports, + moduleName: ${JSON.stringify(payload.moduleName)}, + package: ${JSON.stringify(payload.package)}, + path: ${JSON.stringify(payload.path)}, + version: ${JSON.stringify(payload.version)}, + } + channel.publish(${payloadName}) + module.exports = ${payloadName}.module +` + } + + return `${source} +{ + /* eslint-disable @stylistic/quotes */ + // eslint-disable-next-line n/no-unsupported-features/node-builtins + const nativeDc = require('node:diagnostics_channel') + const dc = globalThis[Symbol.for(${JSON.stringify(BUNDLER_DC_GLOBAL)})] ?? nativeDc + const channel = dc.channel('${CHANNEL}') + if (channel.hasSubscribers) { +${publications} } +} +` +} + +/** + * @param {string} file + * @param {string} expectedHash + * @returns {boolean} + */ +function matchesSource (file, expectedHash) { + return getFileHash(file) === expectedHash +} + +/** + * @param {string} file + * @returns {string} + */ +function getFileHash (file) { + const { ctimeMs, mtimeMs, size } = fs.statSync(file) + const cached = fileHashes.get(file) + if (cached?.ctimeMs === ctimeMs && cached.mtimeMs === mtimeMs && cached.size === size) return cached.hash + + const value = hash(fs.readFileSync(file)) + if (fileHashes.size >= MAX_CACHED_FILES) fileHashes.delete(fileHashes.keys().next().value) + fileHashes.set(file, { ctimeMs, hash: value, mtimeMs, size }) + return value +} + +/** + * @param {string|Buffer} value + * @returns {string} + */ +function hash (value) { + return createHash('sha256').update(value).digest('hex') +} + +function relativeImport (from, to) { + let value = path.relative(from, to).replaceAll('\\', '/') + if (!value.startsWith('.')) value = `./${value}` + return value +} + +function normalizePath (value) { + return fs.realpathSync(value).replaceAll('\\', '/') +} + +/** + * @param {{ emitWarning?: (warning: Error) => void }} loaderContext + * @param {string} key + * @param {string} message + * @param {Error} [cause] + */ +function warnOnce (loaderContext, key, message, cause) { + if (warnedErrors.has(key) || warnedErrors.size >= MAX_WARNINGS) return + warnedErrors.add(key) + const warning = new Error(message, { cause }) + warning.name = 'DatadogTurbopackWarning' + if (typeof loaderContext.emitWarning === 'function') loaderContext.emitWarning(warning) + else process.emitWarning(warning) +} diff --git a/packages/datadog-turbopack/src/targets.js b/packages/datadog-turbopack/src/targets.js new file mode 100644 index 00000000000..201b4bc1459 --- /dev/null +++ b/packages/datadog-turbopack/src/targets.js @@ -0,0 +1,722 @@ +'use strict' + +const { createHash, randomUUID } = require('node:crypto') +const fsSync = require('node:fs') +const fs = require('node:fs/promises') +const path = require('node:path') + +const enhancedResolve = require('enhanced-resolve') + +const { BUNDLER_DC_GLOBAL } = require('../../datadog-instrumentations/src/helpers/bundler-constants') +const instrumentations = require('../../datadog-instrumentations/src/helpers/instrumentations') +const hooks = require('../../datadog-instrumentations/src/helpers/hooks') +const { + filename, + matchVersion, +} = require('../../datadog-instrumentations/src/helpers/instrumentation-utils') +const { isESMFile, processModule } = require('../../datadog-esbuild/src/utils') + +const CACHE_DIRECTORY = path.join('node_modules', '.cache', 'dd-trace', 'turbopack') +const CHANNEL = 'dd-trace:bundler:load' +const MAX_WARNINGS = 128 +const PLAN_VERSION = 3 +const TRAILING_WHITESPACE = /[ \t]+$/gm +const resolveImport = enhancedResolve.create.sync({ conditionNames: ['node', 'import'] }) +const resolveRequire = enhancedResolve.create.sync({ conditionNames: ['node', 'require'] }) + +/** @type {Set} */ +const emittedWarnings = new Set() + +/** + * @typedef {object} InstrumentationPayload + * @property {number[]} instrumentationIndexes + * @property {string} moduleName + * @property {string} package + * @property {string} path + * @property {string} version + */ + +/** + * @typedef {object} Target + * @property {boolean} esm + * @property {Array<{ hook: Function, payload: InstrumentationPayload, version: string }>} matches + * @property {InstrumentationPayload[]} payloads + * @property {string} path + * @property {Set} rulePaths + * @property {string} sourceHash + * @property {string[]} [setters] + */ + +/** + * Compiles installed integration targets into immutable build artifacts. + * + * @param {string} projectDir + * @returns {Promise<{ + * hash?: string, + * moduleSyntaxPattern?: RegExp, + * packagePathPattern?: RegExp, + * path?: string, + * relativePathPattern?: RegExp, + * targetPathPattern?: RegExp + * }>} + */ +async function createBuildPlan (projectDir) { + projectDir = path.resolve(projectDir) + loadInstrumentations() + + const targets = getTargets(projectDir) + const compiledTargets = [] + let includesEsmTarget = false + + for (const target of targets) { + try { + target.sourceHash = hash(fsSync.readFileSync(target.path)) + if (target.esm) { + // Export discovery is asynchronous in import-in-the-middle and belongs at build time. + // eslint-disable-next-line no-await-in-loop + const setters = await processModule({ path: target.path, context: { format: 'module' } }) + target.setters = [...setters.values()].map(setter => setter.replaceAll(TRAILING_WHITESPACE, '')) + includesEsmTarget = true + } + compiledTargets.push(target) + } catch (error) { + warnOnce(`target:${target.path}`, `Could not instrument ${target.path}: ${error.message}`) + } + } + + const relativeTargets = getRelativeTargets(compiledTargets) + if (compiledTargets.length === 0 && relativeTargets.length === 0) return {} + + compiledTargets.sort(compareTargets) + relativeTargets.sort(compareRelativeTargets) + const identity = createPlanIdentity(compiledTargets, relativeTargets) + const planId = hash(identity) + const artifactDirectory = path.join(projectDir, CACHE_DIRECTORY, planId) + try { + await fs.mkdir(artifactDirectory, { recursive: true }) + } catch (error) { + throw new Error(`Could not create the Datadog Turbopack cache at ${artifactDirectory}: ${error.message}`, { + cause: error, + }) + } + const realArtifactDirectory = normalizePath(artifactDirectory) + const planProxies = {} + const planTargets = {} + + for (const target of compiledTargets) { + const entry = { + esm: target.esm, + payloads: target.payloads, + sourceHash: target.sourceHash, + } + + if (target.esm) { + const proxy = createEsmProxy(target, path.join(realArtifactDirectory, 'proxy.mjs')) + const proxyId = hash(proxy) + const proxyPath = path.join(realArtifactDirectory, `${proxyId}.mjs`) + // Files are content-addressed and safe for concurrent config evaluation. + // eslint-disable-next-line no-await-in-loop + await writeArtifact(proxyPath, proxy) + entry.proxyPath = normalizePath(proxyPath) + planProxies[entry.proxyPath] = true + } + + planTargets[target.path] = entry + } + + const plan = JSON.stringify({ + proxies: planProxies, + relativeTargets, + targets: planTargets, + version: PLAN_VERSION, + }) + const planHash = hash(plan) + const planPath = path.join(realArtifactDirectory, `${planHash}.json`) + await writeArtifact(planPath, plan) + + return { + hash: planHash, + moduleSyntaxPattern: includesEsmTarget ? /\b(?:export|import|require)\b/ : undefined, + packagePathPattern: createPackagePathPattern(compiledTargets), + path: planPath, + relativePathPattern: createRelativePathPattern(relativeTargets), + targetPathPattern: createTargetPathPattern(compiledTargets), + } +} + +/** Loads each instrumentation declaration before target discovery. */ +function loadInstrumentations () { + for (const [name, hook] of Object.entries(hooks)) { + const load = hook?.fn ?? hook + if (typeof load !== 'function') continue + + try { + load() + } catch (error) { + warnOnce(`hook:${name}`, `Could not load the ${name} instrumentation: ${error.message}`) + } + } +} + +/** + * @param {string} projectDir + * @returns {Target[]} + */ +function getTargets (projectDir) { + const targets = new Map() + const packageNames = new Set() + + for (const name of Object.keys(instrumentations)) { + if (!name.startsWith('node:') && !name.startsWith('.')) packageNames.add(name) + } + + const packageRootsByName = findPackageRoots(projectDir, packageNames) + for (const [name, entries] of Object.entries(instrumentations)) { + if (name.startsWith('node:') || name.startsWith('.')) continue + + const packageRoots = [...(packageRootsByName.get(name) ?? [])] + if (packageRoots.length === 0) { + addResolvedPackageRoot(packageRoots, projectDir, name, resolveImport) + addResolvedPackageRoot(packageRoots, projectDir, name, resolveRequire) + } + + for (const packageRoot of packageRoots) addTargets(targets, packageRoot, name, entries) + } + + return [...targets.values()] +} + +/** + * @param {Target[]} targets + * @returns {Array<{ file: string, payloads: InstrumentationPayload[], sourceHash: string }>} + */ +function getRelativeTargets (targets) { + const relativeTargets = new Map() + const ambiguousTargets = new Set() + + for (const [name, entries] of Object.entries(instrumentations)) { + if (!name.startsWith('.')) continue + + for (let index = 0; index < entries.length; index++) { + const entry = entries[index] + if (!entry.file) continue + + for (const target of targets) { + for (const match of target.matches) { + if (match.hook !== entry.hook || !matchVersion(match.version, entry.versions)) continue + + const key = `${entry.file}\0${target.sourceHash}` + if (ambiguousTargets.has(key)) continue + + const existing = relativeTargets.get(key) + if (existing && existing.payloads[0].version !== match.version) { + relativeTargets.delete(key) + ambiguousTargets.add(key) + continue + } + + if (existing) { + if (!existing.payloads[0].instrumentationIndexes.includes(index)) { + existing.payloads[0].instrumentationIndexes.push(index) + } + continue + } + + relativeTargets.set(key, { + file: entry.file.replaceAll('\\', '/'), + payloads: [{ + instrumentationIndexes: [index], + moduleName: name, + package: name, + path: name, + version: match.payload.version, + }], + sourceHash: target.sourceHash, + }) + } + } + } + } + + return [...relativeTargets.values()] +} + +/** + * @param {Map} targets + * @param {string} packageRoot + * @param {string} name + * @param {Array} entries + */ +function addTargets (targets, packageRoot, name, entries) { + let packageJson + try { + packageJson = JSON.parse(fsSync.readFileSync(path.join(packageRoot, 'package.json'), 'utf8')) + } catch { + return + } + + let entrypoints + for (let index = 0; index < entries.length; index++) { + const entry = entries[index] + if (!matchVersion(packageJson.version, entry.versions)) continue + + let files + try { + if (entry.file) { + files = [path.join(packageRoot, entry.file)] + } else if (entry.filePattern) { + files = findMatchingFiles(packageRoot, new RegExp(entry.filePattern)) + } else { + entrypoints ??= resolvePackageEntrypoints(packageRoot, name) + files = entrypoints + } + } catch { + continue + } + + for (const file of files) { + if (!fsSync.existsSync(file)) continue + + const targetPath = normalizePath(file) + const relativePath = path.relative(packageRoot, file).replaceAll('\\', '/') + const modulePath = entry.file || (entry.filePattern && relativePath) + const moduleName = filename(name, modulePath) + let target = targets.get(targetPath) + if (!target) { + target = { + esm: isESMFile(file, path.join(packageRoot, 'package.json'), packageJson), + matches: [], + path: targetPath, + payloads: [], + rulePaths: new Set(), + sourceHash: '', + } + targets.set(targetPath, target) + } + target.rulePaths.add(`${name}/${relativePath}`) + target.rulePaths.add(`${path.basename(packageRoot)}/${relativePath}`) + + let payload = findPayload(target.payloads, name, moduleName, packageJson.version) + if (!payload) { + payload = { + instrumentationIndexes: [], + moduleName, + package: name, + path: moduleName, + version: packageJson.version, + } + target.payloads.push(payload) + } + payload.instrumentationIndexes.push(index) + target.matches.push({ hook: entry.hook, payload, version: packageJson.version }) + } + } +} + +/** + * @param {string[]} packageRoots + * @param {string} directory + * @param {string} name + * @param {(directory: string, specifier: string) => string} resolve + */ +function addResolvedPackageRoot (packageRoots, directory, name, resolve) { + let entrypoint + try { + entrypoint = resolve(directory, name) + } catch { + return + } + + const packageRoot = findPackageRoot(entrypoint) + if (packageRoot && !packageRoots.includes(packageRoot)) packageRoots.push(packageRoot) +} + +/** + * @param {InstrumentationPayload[]} payloads + * @param {string} name + * @param {string} moduleName + * @param {string} version + * @returns {InstrumentationPayload|undefined} + */ +function findPayload (payloads, name, moduleName, version) { + for (const payload of payloads) { + if (payload.package === name && payload.moduleName === moduleName && payload.version === version) return payload + } +} + +// Visit package boundaries only: this reaches nested dependency copies without +// walking each package's source tree during Next configuration. +/** + * @param {string} projectDir + * @param {Set} names + * @returns {Map>} + */ +function findPackageRoots (projectDir, names) { + const packageRoots = new Map() + const pending = [path.join(projectDir, 'node_modules')] + const seen = new Set() + + while (pending.length > 0) { + const nodeModules = pending.pop() + let directory + try { + directory = normalizePath(nodeModules) + } catch { + continue + } + if (seen.has(directory)) continue + seen.add(directory) + + let entries + try { + entries = fsSync.readdirSync(directory, { withFileTypes: true }) + } catch { + continue + } + for (const entry of entries) { + if (entry.name === '.bin') continue + if (!entry.isDirectory() && !entry.isSymbolicLink()) continue + const entryPath = path.join(directory, entry.name) + if (entry.name === '.pnpm') { + addPnpmPackageRoots(entryPath, pending) + } else if (entry.name.startsWith('@')) { + addScopedPackageRoots(entryPath, names, packageRoots, pending) + } else { + addPackageRoot(entryPath, entry.name, names, packageRoots, pending) + } + } + } + + return packageRoots +} + +/** + * @param {string} storePath + * @param {string[]} pending + */ +function addPnpmPackageRoots (storePath, pending) { + let entries + try { + entries = fsSync.readdirSync(storePath, { withFileTypes: true }) + } catch { + return + } + for (const entry of entries) { + if (!entry.isDirectory() && !entry.isSymbolicLink()) continue + const entryPath = path.join(storePath, entry.name) + pending.push(entry.name === 'node_modules' ? entryPath : path.join(entryPath, 'node_modules')) + } +} + +/** + * @param {string} scopePath + * @param {Set} names + * @param {Map>} packageRoots + * @param {string[]} pending + */ +function addScopedPackageRoots (scopePath, names, packageRoots, pending) { + let entries + try { + entries = fsSync.readdirSync(scopePath, { withFileTypes: true }) + } catch { + return + } + for (const entry of entries) { + if (!entry.isDirectory() && !entry.isSymbolicLink()) continue + addPackageRoot( + path.join(scopePath, entry.name), + `${path.basename(scopePath)}/${entry.name}`, + names, + packageRoots, + pending + ) + } +} + +/** + * @param {string} packageRoot + * @param {string} packageName + * @param {Set} names + * @param {Map>} packageRoots + * @param {string[]} pending + */ +function addPackageRoot (packageRoot, packageName, names, packageRoots, pending) { + let realPackageRoot + try { + realPackageRoot = normalizePath(packageRoot) + } catch { + return + } + + if (names.has(packageName)) { + const roots = packageRoots.get(packageName) ?? new Set() + roots.add(realPackageRoot) + packageRoots.set(packageName, roots) + } + pending.push(path.join(realPackageRoot, 'node_modules')) +} + +/** + * @param {string} packageRoot + * @param {string} name + * @returns {string[]} + */ +function resolvePackageEntrypoints (packageRoot, name) { + const entrypoints = new Set() + + addResolvedEntrypoint(entrypoints, packageRoot, name, resolveImport) + addResolvedEntrypoint(entrypoints, packageRoot, name, resolveRequire) + if (entrypoints.size === 0) { + addResolvedEntrypoint(entrypoints, packageRoot, '.', resolveImport) + addResolvedEntrypoint(entrypoints, packageRoot, '.', resolveRequire) + } + + return [...entrypoints] +} + +/** + * @param {Set} entrypoints + * @param {string} directory + * @param {string} specifier + * @param {(directory: string, specifier: string) => string} resolve + */ +function addResolvedEntrypoint (entrypoints, directory, specifier, resolve) { + try { + entrypoints.add(resolve(directory, specifier)) + } catch {} +} + +/** + * @param {string} file + * @returns {string|undefined} + */ +function findPackageRoot (file) { + let directory = path.dirname(file) + while (directory !== path.dirname(directory)) { + if (fsSync.existsSync(path.join(directory, 'package.json'))) return directory + directory = path.dirname(directory) + } +} + +/** + * @param {string} directory + * @param {RegExp} pattern + * @returns {string[]} + */ +function findMatchingFiles (directory, pattern) { + const files = [] + const pending = [directory] + while (pending.length > 0) { + const current = pending.pop() + for (const entry of fsSync.readdirSync(current, { withFileTypes: true })) { + const target = path.join(current, entry.name) + if (entry.isDirectory() && entry.name !== 'node_modules') { + pending.push(target) + } else if (entry.isFile()) { + const relativePath = path.relative(directory, target).replaceAll('\\', '/') + if (pattern.test(relativePath)) files.push(target) + } + } + } + return files +} + +/** + * @param {Target[]} targets + * @param {Array<{ file: string, payloads: InstrumentationPayload[], sourceHash: string }>} relativeTargets + * @returns {string} + */ +function createPlanIdentity (targets, relativeTargets) { + const identityTargets = [] + for (const target of targets) { + identityTargets.push({ + esm: target.esm, + path: target.path, + payloads: target.payloads, + setters: target.setters, + sourceHash: target.sourceHash, + }) + } + return JSON.stringify({ relativeTargets, targets: identityTargets, version: PLAN_VERSION }) +} + +/** + * @param {Target} target + * @param {string} proxyPath + * @returns {string} + */ +function createEsmProxy (target, proxyPath) { + let publications = '' + let publicationIndex = 0 + + for (const payload of target.payloads) { + const payloadName = `payload${publicationIndex++}` + publications += ` const ${payloadName} = { + instrumentationIndexes: ${JSON.stringify(payload.instrumentationIndexes)}, + module: _, + moduleName: ${JSON.stringify(payload.moduleName)}, + package: ${JSON.stringify(payload.package)}, + path: ${JSON.stringify(payload.path)}, + version: ${JSON.stringify(payload.version)}, + apply (exports, patchDefault) { + if (patchDefault) return set.default?.(exports) + for (const name of Object.keys(exports)) set[name]?.(exports[name]) + }, + } + channel.publish(${payloadName}) +` + } + + return `/* eslint-disable @stylistic/quotes, @stylistic/semi */ +/* eslint-disable @stylistic/comma-spacing, dot-notation, import/no-mutable-exports */ +/* eslint-disable indent */ +import nativeDc from 'node:diagnostics_channel' +import * as namespace from + ${JSON.stringify(relativeImport(path.dirname(proxyPath), target.path))} +const dc = globalThis[Symbol.for(${JSON.stringify(BUNDLER_DC_GLOBAL)})] ?? nativeDc +const _ = Object.create(null, { [Symbol.toStringTag]: { value: 'Module' } }) +const set = {} +const get = {} +${target.setters.join(';\n')} +const channel = dc.channel('${CHANNEL}') +if (channel.hasSubscribers) { +${publications}} +` +} + +/** + * @param {Target[]} targets + * @returns {RegExp|undefined} + */ +function createPackagePathPattern (targets) { + const packageNames = new Set() + for (const target of targets) { + for (const payload of target.payloads) packageNames.add(payload.package) + } + return new RegExp(`(?:^|/)node_modules/(?:${[...packageNames].sort().map(escapeRegExp).join('|')})(?:/|$)`) +} + +/** + * @param {Array<{ file: string }>} relativeTargets + * @returns {RegExp|undefined} + */ +function createRelativePathPattern (relativeTargets) { + if (relativeTargets.length === 0) return + const files = new Set() + for (const target of relativeTargets) files.add(target.file) + return new RegExp(`(?:^|/)(?:${[...files].sort().map(escapeRegExp).join('|')})$`) +} + +/** + * @param {Target[]} targets + * @returns {RegExp} + */ +function createTargetPathPattern (targets) { + const paths = new Set() + for (const target of targets) { + paths.add(target.path) + for (const rulePath of target.rulePaths) paths.add(rulePath) + } + return new RegExp(`(?:^|/)(?:${[...paths].sort().map(escapeRegExp).join('|')})$`) +} + +/** + * @param {string} file + * @param {string} content + * @returns {Promise} + */ +async function writeArtifact (file, content) { + try { + const existing = await fs.readFile(file, 'utf8') + if (existing === content) return + throw new Error(`The Datadog Turbopack artifact at ${file} does not match its content address`) + } catch (error) { + if (error.code !== 'ENOENT') throw error + } + + const temporaryFile = `${file}.${process.pid}.${randomUUID()}.tmp` + await fs.writeFile(temporaryFile, content, { flag: 'wx' }) + try { + await fs.link(temporaryFile, file) + } catch (error) { + if (error.code !== 'EEXIST') throw error + const existing = await fs.readFile(file, 'utf8') + if (existing !== content) { + throw new Error(`The Datadog Turbopack artifact at ${file} does not match its content address`) + } + } finally { + try { + await fs.unlink(temporaryFile) + } catch (error) { + warnOnce( + `cleanup:${temporaryFile}`, + `Could not remove temporary Turbopack artifact ${temporaryFile}: ${error.message}` + ) + } + } +} + +/** + * @param {Target} left + * @param {Target} right + * @returns {number} + */ +function compareTargets (left, right) { + return left.path.localeCompare(right.path) +} + +/** + * @param {{ file: string, sourceHash: string }} left + * @param {{ file: string, sourceHash: string }} right + * @returns {number} + */ +function compareRelativeTargets (left, right) { + return left.file.localeCompare(right.file) || left.sourceHash.localeCompare(right.sourceHash) +} + +/** + * @param {string} from + * @param {string} to + * @returns {string} + */ +function relativeImport (from, to) { + return path.relative(from, to).replaceAll('\\', '/') +} + +/** + * @param {string} value + * @returns {string} + */ +function normalizePath (value) { + return fsSync.realpathSync(value).replaceAll('\\', '/') +} + +/** + * @param {string|Buffer} value + * @returns {string} + */ +function hash (value) { + return createHash('sha256').update(value).digest('hex') +} + +/** + * @param {string} value + * @returns {string} + */ +function escapeRegExp (value) { + return value.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`) +} + +/** + * @param {string} key + * @param {string} message + */ +function warnOnce (key, message) { + if (emittedWarnings.has(key) || emittedWarnings.size >= MAX_WARNINGS) return + emittedWarnings.add(key) + process.emitWarning(message, { code: 'DD_TRACE_TURBOPACK' }) +} + +module.exports = { + createBuildPlan, +} diff --git a/packages/datadog-turbopack/test/config.spec.js b/packages/datadog-turbopack/test/config.spec.js new file mode 100644 index 00000000000..a5483066ad1 --- /dev/null +++ b/packages/datadog-turbopack/test/config.spec.js @@ -0,0 +1,563 @@ +'use strict' + +const assert = require('node:assert/strict') +const { createHash } = require('node:crypto') +const fs = require('node:fs') +const fsPromises = require('node:fs/promises') +const path = require('node:path') +const { pathToFileURL } = require('node:url') +const { afterEach, describe, it } = require('mocha') +const sinon = require('sinon') + +const { withDatadogTurbopack } = require('../../../next') +const hooks = require('../../datadog-instrumentations/src/helpers/hooks') +const instrumentations = require('../../datadog-instrumentations/src/helpers/instrumentations') +const { + cleanup, + createPackage, + createProject, + findDatadogLoaders, + write, +} = require('./helpers') + +afterEach(() => { + cleanup() + sinon.restore() +}) + +describe('withDatadogTurbopack', () => { + it('exports the wrapper to CommonJS and ESM configurations', async () => { + const namespace = await import(pathToFileURL(require.resolve('../../../next')).href) + + assert.strictEqual(namespace.withDatadogTurbopack, withDatadogTurbopack) + }) + + it('discovers integrations independently of build-process disablement', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const previous = process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS + process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS = 'ioredis' + + try { + const config = await withDatadogTurbopack({}, { projectDir }) + assert.ok(config.turbopack.rules['*.js']) + } finally { + if (previous === undefined) delete process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS + else process.env.DD_TRACE_DISABLED_INSTRUMENTATIONS = previous + } + }) + + it('uses Next 16 rule conditions and preserves existing configuration', async () => { + const projectDir = createProject('16.2.0') + const packageDir = createPackage(projectDir, 'ai', { main: 'index.mjs', type: 'module', version: '7.0.0' }) + write(packageDir, 'index.mjs', [ + 'export function generateText () {}', + 'export function streamText () {}', + '', + ].join('\n')) + const input = { + marker: true, + turbopack: { + resolveAlias: { existing: './existing.js' }, + rules: { + '*.cjs': [{ loaders: ['existing-array-loader'] }], + '*.js': { loaders: ['existing-loader'] }, + }, + }, + } + + const config = await withDatadogTurbopack(input, { projectDir }) + const rules = config.turbopack.rules['*.js'] + const targetRule = rules.find(rule => rule.condition?.all?.some(condition => condition?.any)) + const importRule = rules.find(rule => rule.condition?.all?.some(condition => condition?.content)) + + assert.equal(config.marker, true) + assert.equal(config.turbopack.resolveAlias.existing, './existing.js') + assert.deepEqual(rules[0], { loaders: ['existing-loader'] }) + assert.deepEqual(config.turbopack.rules['*.cjs'][0], { loaders: ['existing-array-loader'] }) + assert.equal(targetRule.condition.all[0], 'node') + assert.equal(targetRule.condition.all[1].any.length, 2) + assert.equal(importRule.condition.all[0], 'node') + assert.equal(importRule.condition.all.some(condition => condition?.not === 'foreign'), true) + assert.equal(importRule.loaders[0].options.rewriteEdges, true) + assert.equal(importRule.loaders[0].options.targetScope, undefined) + assert.equal(targetRule.loaders[0].options.rewriteEdges, true) + assert.equal(targetRule.loaders[0].options.targetScope, 'direct') + const contentPattern = importRule.condition.all.find(condition => condition?.content).content + assert.equal(contentPattern.test("import /* webpackChunkName: 'ai' */ ('ai')"), true) + assert.equal(contentPattern.test('const answer = 42'), false) + }) + + it('uses named conditions and nested built-ins for Next 15', async () => { + const projectDir = createProject('15.5.0') + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const aiDirectory = createPackage(projectDir, 'ai', { main: 'index.mjs', type: 'module', version: '7.0.0' }) + write(aiDirectory, 'index.mjs', 'export function generateText () {}\n') + const prismaDirectory = createPackage(projectDir, '@prisma/client', { main: 'index.js', version: '6.1.0' }) + write(prismaDirectory, 'index.js', 'module.exports = {}') + write(prismaDirectory, 'runtime/library.js', 'module.exports = {}\n') + + const config = await withDatadogTurbopack({}, { projectDir }) + const name = '#dd-trace/target' + const rule = config.turbopack.rules[name] + const loader = rule.node.loaders[0] + + assert.ok(config.turbopack.conditions[name].path instanceof RegExp) + assert.equal(config.turbopack.conditions[name].path.test('/app/node_modules/ioredis/index.js'), true) + assert.equal(config.turbopack.conditions[name].path.test('/app/node_modules/ioredis/package.json'), false) + assert.equal(Object.keys(config.turbopack.conditions).length, 3) + assert.ok(config.turbopack.conditions['#dd-trace/import'].content instanceof RegExp) + assert.ok(config.turbopack.conditions['#dd-trace/relative'].path instanceof RegExp) + assert.equal(config.turbopack.rules['#dd-trace/import'].node.foreign, false) + assert.equal(config.turbopack.rules['#dd-trace/import'].node.loaders.length, 1) + assert.equal(config.turbopack.rules['#dd-trace/import'].node.loaders[0].options.rewriteEdges, true) + assert.equal(config.turbopack.rules['#dd-trace/relative'].node.foreign, false) + assert.equal(config.turbopack.rules['#dd-trace/relative'].node.loaders[0].options.targetScope, 'relative') + assert.equal(rule.condition, undefined) + assert.equal(typeof loader.loader, 'string') + assert.match(loader.options.manifestHash, /^[a-f\d]{64}$/) + JSON.stringify(loader.options) + + const repeated = await withDatadogTurbopack(config, { projectDir }) + assert.equal(findDatadogLoaders(repeated).length, findDatadogLoaders(config).length) + }) + + it('leaves newer Next majors on the modern schema path', async () => { + const projectDir = createProject('17.0.0-canary.1') + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + + const config = await withDatadogTurbopack({}, { projectDir }) + + assert.equal([config.turbopack.rules['*.js']].flat().every(rule => rule.condition), true) + assert.equal(config.turbopack.conditions, undefined) + }) + + it('supports object promises and config functions without changing their contract', async () => { + const projectDir = createProject() + const promisedConfig = { promised: true } + const promiseResult = await withDatadogTurbopack(Promise.resolve(promisedConfig), { projectDir }) + const receiver = { calls: 0 } + const defaultConfig = { defaultConfig: true } + const wrapped = withDatadogTurbopack(function (phase, context) { + this.calls++ + assert.equal(phase, 'phase-production-build') + assert.strictEqual(context.defaultConfig, defaultConfig) + return { functional: true } + }, { projectDir }) + const wrappedUndefined = withDatadogTurbopack(() => undefined, { projectDir }) + + const [functionResult, undefinedResult] = await Promise.all([ + wrapped.call(receiver, 'phase-production-build', { defaultConfig }), + wrappedUndefined(), + ]) + + assert.strictEqual(promiseResult, promisedConfig) + assert.equal(receiver.calls, 1) + assert.deepEqual(functionResult, { functional: true }) + assert.deepEqual(undefinedResult, {}) + }) + + it('uses the current directory when no project option is provided', async () => { + const projectDir = createProject() + const previousDirectory = process.cwd() + + try { + process.chdir(projectDir) + assert.deepEqual(await withDatadogTurbopack(), {}) + } finally { + process.chdir(previousDirectory) + } + }) + + it('returns the original object when no supported package is installed', async () => { + const projectDir = createProject() + const config = { turbopack: { resolveAlias: { value: './value.js' } } } + + assert.strictEqual(await withDatadogTurbopack(config, { projectDir }), config) + assert.equal(fs.existsSync(path.join(projectDir, 'node_modules/.cache/dd-trace/turbopack')), false) + }) + + it('creates one immutable plan under concurrent and repeated composition', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + + const configs = await Promise.all([ + withDatadogTurbopack({}, { projectDir }), + withDatadogTurbopack({}, { projectDir }), + withDatadogTurbopack({}, { projectDir }), + withDatadogTurbopack({}, { projectDir }), + ]) + const planPaths = configs.map(config => findDatadogLoaders(config)[0].options.manifestPath) + const twice = await withDatadogTurbopack(configs[0], { projectDir }) + + assert.equal(new Set(planPaths).size, 1) + assert.match(planPaths[0], /\/[a-f\d]{64}\/[a-f\d]{64}\.json$/) + assert.equal( + path.basename(planPaths[0], '.json'), + createHash('sha256').update(fs.readFileSync(planPaths[0])).digest('hex') + ) + assert.equal(findDatadogLoaders(twice).length, findDatadogLoaders(configs[0]).length) + assert.equal(fs.readdirSync(path.dirname(path.dirname(planPaths[0]))).length, 1) + }) + + it('reports one warning for instrumentation discovery and target compilation failures', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const aiDirectory = createPackage(projectDir, 'ai', { main: 'index.mjs', type: 'module', version: '7.0.0' }) + write(aiDirectory, 'index.mjs', 'export {') + const emitWarning = sinon.stub(process, 'emitWarning') + const hookName = 'test-turbopack-load-failure' + const skippedHookName = 'test-turbopack-nonfunction-hook' + hooks[hookName] = () => { + throw new Error('load failed') + } + hooks[skippedHookName] = {} + + try { + await withDatadogTurbopack({}, { projectDir }) + await withDatadogTurbopack({}, { projectDir }) + } finally { + delete hooks[hookName] + delete hooks[skippedHookName] + } + + assert.equal(emitWarning.callCount, 2) + sinon.assert.calledWithMatch(emitWarning, /Could not load the test-turbopack-load-failure instrumentation/) + sinon.assert.calledWithMatch(emitWarning, new RegExp(`Could not instrument ${aiDirectory}`)) + }) + + it('wraps cache-directory creation failures with their build path', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const error = Object.assign(new Error('permission denied'), { code: 'EACCES' }) + sinon.stub(fsPromises, 'mkdir').rejects(error) + + await assert.rejects( + withDatadogTurbopack({}, { projectDir }), + { message: /Could not create the Datadog Turbopack cache .*permission denied/ } + ) + }) + + it('accepts an artifact completed by a concurrent build-plan writer', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const link = fsPromises.link.bind(fsPromises) + sinon.stub(fsPromises, 'link').callsFake(async (source, target) => { + await link(source, target) + throw Object.assign(new Error('already exists'), { code: 'EEXIST' }) + }) + + const config = await withDatadogTurbopack({}, { projectDir }) + + assert.ok(config.turbopack.rules['*.js']) + }) + + it('rejects conflicting and failed artifact writes', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const link = fsPromises.link.bind(fsPromises) + sinon.stub(fsPromises, 'link').callsFake(async (source, target) => { + await link(source, target) + await fsPromises.writeFile(target, 'conflict') + throw Object.assign(new Error('already exists'), { code: 'EEXIST' }) + }) + + await assert.rejects( + withDatadogTurbopack({}, { projectDir }), + { message: /artifact .* does not match its content address/ } + ) + + sinon.restore() + const failedProject = createProject() + const failedPackage = createPackage(failedProject, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(failedPackage, 'index.js', 'module.exports = {}') + sinon.stub(fsPromises, 'link').rejects(Object.assign(new Error('link denied'), { code: 'EACCES' })) + + await assert.rejects(withDatadogTurbopack({}, { projectDir: failedProject }), { message: /link denied/ }) + }) + + it('warns when a temporary artifact cannot be removed', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const emitWarning = sinon.stub(process, 'emitWarning') + sinon.stub(fsPromises, 'unlink').rejects(new Error('unlink denied')) + + const config = await withDatadogTurbopack({}, { projectDir }) + + assert.ok(config.turbopack.rules['*.js']) + sinon.assert.calledWithMatch(emitWarning, /Could not remove temporary Turbopack artifact .*unlink denied/) + }) + + it('rejects an existing artifact whose content no longer matches its address', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const config = await withDatadogTurbopack({}, { projectDir }) + const planPath = findDatadogLoaders(config)[0].options.manifestPath + fs.writeFileSync(planPath, 'conflict') + + await assert.rejects( + withDatadogTurbopack({}, { projectDir }), + { message: /artifact .* does not match its content address/ } + ) + }) + + it('discovers nested copies and records exact instrumentation indexes', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'parent/node_modules/ioredis', { + main: 'index.js', + version: '5.0.0', + }) + const target = write(packageDir, 'index.js', 'module.exports = {}') + + const config = await withDatadogTurbopack({}, { projectDir }) + const planPath = findDatadogLoaders(config)[0].options.manifestPath + const plan = JSON.parse(fs.readFileSync(planPath, 'utf8')) + const entry = plan.targets[fs.realpathSync(target).replaceAll('\\', '/')] + + assert.equal(entry.payloads[0].package, 'ioredis') + assert.deepEqual(entry.payloads[0].instrumentationIndexes, [3]) + }) + + it('discovers dependencies resolved from an ancestor project', async () => { + const projectRoot = createProject() + const packageDir = createPackage(projectRoot, 'ioredis', { + main: 'dist/index.js', + version: '5.0.0', + }) + write(packageDir, 'dist/index.js', 'module.exports = {}') + const projectDir = path.join(projectRoot, 'app') + write(projectDir, 'package.json', '{}') + + const config = await withDatadogTurbopack({}, { projectDir }) + + assert.ok(config.turbopack.rules['*.js']) + }) + + it('discovers linked workspace dependencies outside node_modules', async () => { + const workspaceDir = createProject() + const projectDir = path.join(workspaceDir, 'apps/web') + write(projectDir, 'package.json', '{}') + const packageDir = path.join(workspaceDir, 'packages/cache-client') + write(packageDir, 'package.json', JSON.stringify({ main: 'index.js', name: 'ioredis', version: '5.0.0' })) + const target = write(packageDir, 'index.js', 'module.exports = {}') + fs.symlinkSync(packageDir, path.join(workspaceDir, 'node_modules/ioredis'), 'dir') + + const config = await withDatadogTurbopack({}, { projectDir }) + const targetRule = [config.turbopack.rules['*.js']].flat().find( + rule => rule.condition?.all?.some(condition => condition?.any) + ) + const planPath = findDatadogLoaders(config)[0].options.manifestPath + const plan = JSON.parse(fs.readFileSync(planPath, 'utf8')) + + assert.ok(plan.targets[fs.realpathSync(target).replaceAll('\\', '/')]) + assert.equal( + targetRule.condition.all[1].any.some(({ path: pattern }) => + pattern.test('../../packages/cache-client/index.js')), + true + ) + }) + + it('discovers dependencies in a pnpm virtual store', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, '.pnpm/ioredis@5.0.0/node_modules/ioredis', { + main: 'index.js', + version: '5.0.0', + }) + write(packageDir, 'index.js', 'module.exports = {}') + const hoistedPackage = createPackage(projectDir, '.pnpm/node_modules/ioredis', { + main: 'index.js', + version: '5.0.0', + }) + write(hoistedPackage, 'index.js', 'module.exports = {}') + write(projectDir, 'node_modules/.pnpm/not-a-package', 'file') + + const config = await withDatadogTurbopack({}, { projectDir }) + const planPath = findDatadogLoaders(config)[0].options.manifestPath + const plan = JSON.parse(fs.readFileSync(planPath, 'utf8')) + + assert.ok(plan.targets[fs.realpathSync(path.join(packageDir, 'index.js')).replaceAll('\\', '/')]) + }) + + it('handles package-boundary traversal failures without losing valid targets', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + write(packageDir, 'node_modules', 'not a directory') + fs.mkdirSync(path.join(projectDir, 'node_modules', '.bin')) + write(projectDir, 'node_modules/not-a-package', 'file') + const scopeDirectory = path.join(projectDir, 'node_modules', '@scope') + fs.mkdirSync(scopeDirectory) + write(scopeDirectory, 'not-a-package', 'file') + fs.symlinkSync(packageDir, path.join(projectDir, 'node_modules', 'ioredis-alias'), 'dir') + fs.symlinkSync( + path.join(projectDir, 'node_modules', 'missing'), + path.join(projectDir, 'node_modules', 'broken-link'), + 'dir' + ) + fs.symlinkSync( + path.join(projectDir, 'node_modules', 'not-a-package'), + path.join(projectDir, 'node_modules', '@broken'), + 'dir' + ) + fs.symlinkSync( + path.join(projectDir, 'node_modules', 'not-a-package'), + path.join(projectDir, 'node_modules', '.pnpm'), + 'dir' + ) + + const config = await withDatadogTurbopack({}, { projectDir }) + + assert.ok(config.turbopack.rules['*.js']) + }) + + it('skips invalid package metadata and file patterns', async () => { + const projectDir = createProject() + const invalidPackage = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(invalidPackage, 'package.json', '{') + + assert.deepEqual(await withDatadogTurbopack({}, { projectDir }), {}) + + const validProject = createProject() + const validPackage = createPackage(validProject, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(validPackage, 'index.js', 'module.exports = {}') + const load = hooks.ioredis?.fn ?? hooks.ioredis + load() + const entries = instrumentations.ioredis + entries.push({ filePattern: '[', hook () {}, versions: ['>=5'] }) + + try { + const config = await withDatadogTurbopack({}, { projectDir: validProject }) + assert.ok(config.turbopack.rules['*.js']) + } finally { + entries.pop() + } + + const fallbackProject = createProject() + const fallbackPackage = createPackage(fallbackProject, 'ioredis', { + exports: { './commands': './index.js' }, + main: 'index.js', + version: '5.0.0', + }) + write(fallbackPackage, 'index.js', 'module.exports = {}') + + const fallbackConfig = await withDatadogTurbopack({}, { projectDir: fallbackProject }) + assert.ok(fallbackConfig.turbopack.rules['*.js']) + }) + + it('deduplicates relative hooks and rejects ambiguous package versions', async () => { + const projectDir = createProject() + const first = createPackage(projectDir, '@prisma/client', { main: 'index.js', version: '6.1.0' }) + const second = createPackage(projectDir, 'parent/node_modules/@prisma/client', { + main: 'index.js', + version: '6.1.0', + }) + write(first, 'index.js', 'module.exports = {}') + write(first, 'runtime/library.js', 'module.exports = { copy: 1 }\n') + write(second, 'index.js', 'module.exports = {}') + write(second, 'runtime/library.js', 'module.exports = { copy: 2 }\n') + const load = hooks['@prisma/client']?.fn ?? hooks['@prisma/client'] + load() + const relativeEntries = instrumentations['./runtime/library.js'] + relativeEntries.push({ ...relativeEntries[0] }) + relativeEntries.push({ hook () {}, versions: ['>=6.1.0 <7.0.0'] }) + + try { + const config = await withDatadogTurbopack({}, { projectDir }) + const planPath = findDatadogLoaders(config)[0].options.manifestPath + const plan = JSON.parse(fs.readFileSync(planPath, 'utf8')) + + assert.equal(plan.relativeTargets.length, 2) + assert.deepEqual(plan.relativeTargets[0].payloads[0].instrumentationIndexes, [0, 1]) + assert.deepEqual(plan.relativeTargets[1].payloads[0].instrumentationIndexes, [0, 1]) + } finally { + relativeEntries.splice(-2) + } + + const ambiguousProject = createProject() + for (const [parent, version] of [['', '6.1.0'], ['one', '6.2.0'], ['two', '6.3.0']]) { + const name = parent ? `${parent}/node_modules/@prisma/client` : '@prisma/client' + const packageDir = createPackage(ambiguousProject, name, { main: 'index.js', version }) + write(packageDir, 'index.js', 'module.exports = {}') + write(packageDir, 'runtime/library.js', 'module.exports = { same: true }\n') + } + + const ambiguousConfig = await withDatadogTurbopack({}, { projectDir: ambiguousProject }) + const ambiguousPlanPath = findDatadogLoaders(ambiguousConfig)[0].options.manifestPath + const ambiguousPlan = JSON.parse(fs.readFileSync(ambiguousPlanPath, 'utf8')) + + assert.equal(ambiguousPlan.relativeTargets.length, 0) + }) + + it('rejects unsupported versions and invalid configuration shapes', async () => { + const oldProject = createProject('15.4.9') + const invalidVersionProject = createProject('latest') + const projectDir = createProject() + const missingNext = createProject() + const missingCompiler = createProject() + fs.rmSync(path.join(missingNext, 'node_modules/next'), { force: true, recursive: true }) + fs.rmSync(path.join(missingCompiler, 'node_modules/next/dist/compiled/babel/parser.js')) + + assert.throws( + () => withDatadogTurbopack({}, { projectDir: oldProject }), + { name: 'RangeError', message: /requires Next\.js 15\.5 or newer/ } + ) + assert.throws( + () => withDatadogTurbopack({}, { projectDir: missingNext }), + { message: /could not resolve Next\.js/ } + ) + assert.throws( + () => withDatadogTurbopack({}, { projectDir: missingCompiler }), + { message: /does not provide the compiler/ } + ) + assert.throws( + () => withDatadogTurbopack({}, { projectDir: invalidVersionProject }), + { message: /could not parse Next\.js version/ } + ) + assert.throws( + () => withDatadogTurbopack({}, /** @type {object} */ (null)), + { name: 'TypeError', message: /options must be an object/ } + ) + assert.throws( + () => withDatadogTurbopack({}, { projectDir: /** @type {string} */ (42) }), + { name: 'TypeError', message: /options\.projectDir must be a string/ } + ) + await assert.rejects( + withDatadogTurbopack(42, { projectDir }), + { name: 'TypeError', message: /configuration object, promise, or function/ } + ) + for (const [config, message] of [ + [{ turbopack: null }, /turbopack must be an object/], + [{ turbopack: [] }, /turbopack must be an object/], + [{ turbopack: { rules: false } }, /turbopack\.rules must be an object/], + [{ turbopack: { rules: [] } }, /turbopack\.rules must be an object/], + [{ turbopack: { conditions: '' } }, /turbopack\.conditions must be an object/], + [{ turbopack: { conditions: [] } }, /turbopack\.conditions must be an object/], + [{ turbopack: { resolveAlias: '' } }, /turbopack\.resolveAlias must be an object/], + [{ turbopack: { resolveAlias: [] } }, /turbopack\.resolveAlias must be an object/], + ]) { + await assert.rejects(withDatadogTurbopack(config, { projectDir }), { message }) + } + }) + + it('does not overwrite a user condition in the reserved Next 15 namespace', async () => { + const projectDir = createProject('15.5.0') + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + + await assert.rejects(withDatadogTurbopack({ + turbopack: { + conditions: { '#dd-trace/target': { path: '*.custom.js' } }, + }, + }, { projectDir }), { message: /already uses the reserved condition/ }) + }) +}) diff --git a/packages/datadog-turbopack/test/helpers.js b/packages/datadog-turbopack/test/helpers.js new file mode 100644 index 00000000000..b0678152e59 --- /dev/null +++ b/packages/datadog-turbopack/test/helpers.js @@ -0,0 +1,88 @@ +'use strict' + +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +/** @type {string[]} */ +const directories = [] + +/** + * @param {string} [nextVersion] + * @returns {string} + */ +function createProject (nextVersion = '16.0.0') { + const directory = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'dd-turbopack-'))) + directories.push(directory) + write(directory, 'package.json', '{}') + const next = createPackage(directory, 'next', { main: 'index.js', version: nextVersion }) + write(next, 'index.js', 'module.exports = {}') + for (const name of ['generator', 'parser', 'traverse']) { + const modulePath = require.resolve(`@babel/${name}`) + write(next, `dist/compiled/babel/${name}.js`, `module.exports = require(${JSON.stringify(modulePath)})\n`) + } + return directory +} + +/** + * @param {string} projectDir + * @param {string} name + * @param {object} [manifest] + * @returns {string} + */ +function createPackage (projectDir, name, manifest = {}) { + const packageDirectory = path.join(projectDir, 'node_modules', name) + fs.mkdirSync(packageDirectory, { recursive: true }) + fs.writeFileSync(path.join(packageDirectory, 'package.json'), JSON.stringify({ name, ...manifest })) + return packageDirectory +} + +/** + * @param {string} directory + * @param {string} relativePath + * @param {string} content + * @returns {string} + */ +function write (directory, relativePath, content) { + const target = path.resolve(directory, relativePath) + fs.mkdirSync(path.dirname(target), { recursive: true }) + fs.writeFileSync(target, content) + return target +} + +/** + * @param {unknown} value + * @returns {object[]} + */ +function findDatadogLoaders (value) { + const loaders = [] + collectDatadogLoaders(value, loaders) + return loaders +} + +/** + * @param {unknown} value + * @param {object[]} loaders + */ +function collectDatadogLoaders (value, loaders) { + if (Array.isArray(value)) { + for (const item of value) collectDatadogLoaders(item, loaders) + return + } + if (!value || typeof value !== 'object') return + if (typeof value.loader === 'string' && value.loader.includes('datadog-turbopack')) loaders.push(value) + + for (const key of Object.keys(value)) collectDatadogLoaders(value[key], loaders) +} + +function cleanup () { + while (directories.length > 0) fs.rmSync(directories.pop(), { force: true, recursive: true }) +} + +module.exports = { + cleanup, + createPackage, + createProject, + findDatadogLoaders, + write, +} diff --git a/packages/datadog-turbopack/test/loader.spec.js b/packages/datadog-turbopack/test/loader.spec.js new file mode 100644 index 00000000000..a07d084c29e --- /dev/null +++ b/packages/datadog-turbopack/test/loader.spec.js @@ -0,0 +1,831 @@ +'use strict' + +const assert = require('node:assert/strict') +const { createHash } = require('node:crypto') +const fs = require('node:fs') +const path = require('node:path') +const vm = require('node:vm') +const { pathToFileURL } = require('node:url') +const { afterEach, describe, it } = require('mocha') + +const dc = require('dc-polyfill') +const enhancedResolve = require('enhanced-resolve') +const { ESLint } = require('eslint') +const { engines: eslintEngines } = require('eslint/package.json') +const semver = require('semver') +const sinon = require('sinon') + +const { withDatadogTurbopack } = require('../../../next') +const loader = require('../src/loader') +const { + cleanup, + createPackage, + createProject, + findDatadogLoaders, + write, +} = require('./helpers') + +const CHANNEL = 'dd-trace:bundler:load' +const lintRuntimeSupported = semver.satisfies(process.version, eslintEngines.node) + +afterEach(() => { + cleanup() + sinon.restore() +}) + +describe('datadog-turbopack loader', () => { + it('rewrites ESM imports and only unshadowed CommonJS requires', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/route.js', '') + const source = [ + "import { generateText } from 'ai'", + "export { streamText } from 'ai'", + "const dynamic = import('ai')", + "const attributed = import('ai', { with: { type: 'json' } })", + "const top = require('ai')", + "const commonjs = require('ioredis')", + "const missing = require('not-installed')", + "const missingTarget = require('ai/not-installed')", + "function local (require) { return require('ai') }", + 'export { attributed, commonjs, dynamic, local, missing, missingTarget, top }', + '', + ].join('\n') + + const { code: result, map } = await runLoaderResult(appPath, source, fixture.importOptions) + const proxyFile = path.basename(fixture.proxyPath) + + assert.equal(result.split(proxyFile).length - 1, 4) + assert.match(result, /const top = require\('ai'\)/) + assert.match(result, /function local\(require\) \{\s*return require\('ai'\)/) + assert.equal(map.sources[0], appPath) + assert.equal( + path.basename(fixture.proxyPath, '.mjs'), + createHash('sha256').update(fs.readFileSync(fixture.proxyPath)).digest('hex') + ) + assert.equal(await runLoader(appPath, source, fixture.importOptions), result) + }) + + it('rewrites imports in TypeScript JSX application modules', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/route.tsx', '') + const source = [ + "import { generateText } from 'ai'", + 'const prompt: string = \'hello\'', + 'export default
{prompt}
', + '', + ].join('\n') + + const result = await runLoader(appPath, source, fixture.importOptions) + + assert.match(result, new RegExp(path.basename(fixture.proxyPath))) + }) + + it('chains an input source map through module-edge rewriting', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/mapped.js', '') + const source = "import { generateText } from 'ai'\n" + const sourceMap = { + file: appPath, + mappings: 'AAAA', + names: [], + sources: ['route.ts'], + sourcesContent: [source], + version: 3, + } + + const result = await runLoaderResult(appPath, source, fixture.importOptions, { sourceMap }) + + assert.deepEqual(result.map.sources, ['route.ts']) + assert.deepEqual(result.map.sourcesContent, [source]) + }) + + it('rewrites static template imports and keeps dynamic templates unchanged', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/templates.js', '') + const source = [ + 'const imported = import(`ai`)', + 'const required = require(`ai`)', + // eslint-disable-next-line no-template-curly-in-string -- This is source for the loader under test. + 'const dynamic = import(`ai/${name}`)', + 'export { dynamic, imported, required }', + '', + ].join('\n') + + const result = await runLoader(appPath, source, fixture.importOptions) + + assert.equal(result.split(path.basename(fixture.proxyPath)).length - 1, 1) + assert.match(result, /require\(`ai`\)/) + assert.match(result, /import\(`ai\/\$\{name\}`\)/) + }) + + it('rewrites import-expression AST nodes from newer parsers', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/import-expression.js', '') + const parserPath = fixture.importOptions.compiler.parser + const parser = write(fixture.projectDir, 'parser.js', [ + `const parser = require(${JSON.stringify(parserPath)})`, + 'exports.parse = function parse (source, options) {', + ' return parser.parse(source, { ...options, createImportExpressions: true })', + '}', + '', + ].join('\n')) + const options = { + ...fixture.importOptions, + compiler: { ...fixture.importOptions.compiler, parser }, + } + + const result = await runLoader(appPath, "const value = import('ai')\n", options) + + assert.match(result, new RegExp(path.basename(fixture.proxyPath))) + }) + + it('leaves type-only module edges untouched', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/types.ts', '') + const source = [ + "import type { CoreTool } from 'ai'", + "import { type LanguageModel } from 'ai'", + "export type { ToolChoice } from 'ai'", + "export { type ToolChoice } from 'ai'", + 'import Alias = Namespace.Value', + '', + ].join('\n') + + assert.equal(await runLoader(appPath, source, fixture.importOptions), source) + }) + + it('leaves Flow type imports untouched', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/types.js', '') + const source = [ + "import type { CoreTool } from 'ai'", + "import typeof GenerateText from 'ai'", + 'const count: number = 1', + 'export { count }', + '', + ].join('\n') + + assert.equal(await runLoader(appPath, source, fixture.importOptions), source) + }) + + it('does not pass Node.js built-ins to the Turbopack resolver', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/builtins.js', '') + const requests = [] + const getResolve = () => (_directory, request, callback) => { + requests.push(request) + callback(undefined, fixture.targetPath) + } + const source = [ + "import fs from 'node:fs'", + "import path from 'path'", + "import { generateText } from 'ai'", + '', + ].join('\n') + + const result = await runLoader(appPath, source, fixture.importOptions, { getResolve }) + + assert.deepEqual(requests, ['ai']) + assert.match(result, new RegExp(path.basename(fixture.proxyPath))) + }) + + it('does not redirect a generated proxy back to itself', async () => { + const fixture = await createAiFixture() + const source = fs.readFileSync(fixture.proxyPath, 'utf8') + let resolverCalls = 0 + const getResolve = () => (_directory, _request, callback) => { + resolverCalls++ + callback(new Error('generated proxies must not be resolved')) + } + + assert.equal(await runLoader(fixture.proxyPath, source, fixture.importOptions, { getResolve }), source) + assert.equal(resolverCalls, 0) + }) + + it('parses explicit resource management syntax while rewriting imports', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/resource.js', '') + const source = [ + "import { generateText } from 'ai'", + 'using resource = { [Symbol.dispose] () {} }', + 'export { resource }', + '', + ].join('\n') + + const result = await runLoader(appPath, source, fixture.importOptions) + + assert.match(result, new RegExp(path.basename(fixture.proxyPath))) + }) + + it('does not rewrite a require that is shadowed in its own scope', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/route.js', '') + const source = "const require = load\nconst value = require('ai')\n" + + assert.equal(await runLoader(appPath, source, fixture.importOptions), source) + }) + + it('rewrites require edges only when the require resolver selects an ESM target', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/require.ts', '') + const source = "import AI = require('ai')\nconst required = require('ai')\nexport { required }\n" + const resolvedConditions = [] + const getResolve = options => (_directory, _request, callback) => { + resolvedConditions.push(options.conditionNames) + queueMicrotask(() => callback(undefined, fixture.targetPath)) + } + + const result = await runLoader(appPath, source, fixture.importOptions, { getResolve }) + + assert.equal(result.split(path.basename(fixture.proxyPath)).length - 1, 2) + assert.deepEqual(resolvedConditions, [['node', 'require']]) + }) + + it('tracks bindings from every supported declaration pattern', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/scopes.js', '') + const source = [ + "import { generateText } from 'ai'", + 'const named = function named () {}', + "const assigned = (require = load) => require('ai')", + "const array = ([require]) => require('ai')", + "const object = ({ require }) => require('ai')", + 'const { require: objectRequire, ...objectRest } = globalThis', + "const rest = (...require) => require[0]('ai')", + "function scoped () { var require = load; return require('ai') }", + "try { scoped() } catch (require) { require('ai') }", + 'try { scoped() } catch {}', + 'const Named = class Named {}', + 'const Anonymous = class {}', + 'export { Anonymous, Named, array, assigned, named, object, objectRequire, objectRest, rest, scoped }', + '', + ].join('\n') + + const result = await runLoader(appPath, source, fixture.importOptions) + + assert.equal(result.split(path.basename(fixture.proxyPath)).length - 1, 1) + assert.match(result, /return require\('ai'\)/) + }) + + it('preserves configured aliases', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ai', { main: 'index.mjs', type: 'module', version: '7.0.0' }) + write(packageDir, 'index.mjs', 'export function generateText () {}\n') + const config = await withDatadogTurbopack({ + turbopack: { resolveAlias: { ai: './replacement.js' } }, + }, { projectDir }) + const options = findDatadogLoaders(config) + .find(item => item.options.rewriteEdges && !item.options.targetScope).options + const appPath = write(projectDir, 'app/route.js', '') + const source = "import { generateText } from 'ai'\nimport value from 'ai/subpath'\n" + const replacement = write(projectDir, 'replacement.js', 'export default true\n') + const getResolve = () => (_directory, _request, callback) => callback(undefined, replacement) + + assert.equal(await runLoader(appPath, source, options, { getResolve }), source) + }) + + it('returns transformed ESM dependencies without a CommonJS publication tail', async () => { + const fixture = await createAiFixture() + const source = fs.readFileSync(fixture.targetPath, 'utf8') + + const result = await runLoader(fixture.targetPath, source, fixture.packageOptions) + + assert.doesNotMatch(result, /dd-trace:bundler:load/) + }) + + it('plans and instruments both sides of a dual package export', async () => { + const fixture = await createAiFixture() + const plan = JSON.parse(fs.readFileSync(fixture.packageOptions.manifestPath, 'utf8')) + + assert.ok(plan.targets[fs.realpathSync(fixture.targetPath)]) + assert.ok(plan.targets[fs.realpathSync(fixture.commonJsPath)]) + assert.match( + await runLoader(fixture.commonJsPath, fs.readFileSync(fixture.commonJsPath, 'utf8'), fixture.packageOptions), + /dd-trace:bundler:load/ + ) + }) + + it('classifies TypeScript module extensions independently of package type', async () => { + const moduleProject = createProject() + const moduleDirectory = createPackage(moduleProject, 'ioredis', { + main: 'index.mts', + type: 'commonjs', + version: '5.0.0', + }) + const moduleSource = 'export const original = true\n' + const modulePath = write(moduleDirectory, 'index.mts', moduleSource) + const commonJsProject = createProject() + const commonJsDirectory = createPackage(commonJsProject, 'ioredis', { + main: 'index.cts', + type: 'module', + version: '5.0.0', + }) + const commonJsSource = 'module.exports = { original: true }\n' + const commonJsPath = write(commonJsDirectory, 'index.cts', commonJsSource) + const [moduleConfig, commonJsConfig] = await Promise.all([ + withDatadogTurbopack({}, { projectDir: moduleProject }), + withDatadogTurbopack({}, { projectDir: commonJsProject }), + ]) + const moduleOptions = findDatadogLoaders(moduleConfig).find(item => item.options.targetScope === 'direct').options + const commonJsOptions = findDatadogLoaders(commonJsConfig) + .find(item => item.options.targetScope === 'direct').options + const modulePlan = JSON.parse(fs.readFileSync(moduleOptions.manifestPath, 'utf8')) + const commonJsPlan = JSON.parse(fs.readFileSync(commonJsOptions.manifestPath, 'utf8')) + const moduleTarget = modulePlan.targets[fs.realpathSync(modulePath)] + const commonJsTarget = commonJsPlan.targets[fs.realpathSync(commonJsPath)] + + assert.equal(moduleTarget.esm, true) + assert.equal(typeof moduleTarget.proxyPath, 'string') + assert.equal(commonJsTarget.esm, false) + assert.equal(commonJsTarget.proxyPath, undefined) + + const [moduleResult, commonJsResult] = await Promise.all([ + runLoader(modulePath, moduleSource, moduleOptions), + runLoader(commonJsPath, commonJsSource, commonJsOptions), + ]) + + assert.equal(moduleResult, moduleSource) + assert.match(commonJsResult, /dd-trace:bundler:load/) + }) + + it('rewrites an internal package edge to its instrumented ESM target', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'hono', { + exports: './dist/index.js', + type: 'module', + version: '4.12.19', + }) + const source = "export { Hono } from './hono.js'\n" + const entryPath = write(packageDir, 'dist/index.js', source) + const targetPath = write(packageDir, 'dist/hono.js', 'export class Hono {}\n') + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config).find(item => item.options.targetScope === 'direct').options + const plan = JSON.parse(fs.readFileSync(options.manifestPath, 'utf8')) + const proxyPath = plan.targets[fs.realpathSync(targetPath)].proxyPath + + const rewritten = await runLoader(entryPath, source, options) + + assert.match(rewritten, new RegExp(path.basename(proxyPath))) + assert.doesNotMatch(rewritten, /dd-trace:bundler:load/) + }) + + it('does not instrument a direct target through the relative-copy rule', async () => { + const fixture = await createAiFixture() + const source = fs.readFileSync(fixture.ioredisPath, 'utf8') + + const result = await runLoader(fixture.ioredisPath, source, { + ...fixture.packageOptions, + targetScope: 'relative', + }) + + assert.equal(result, source) + }) + + it('keeps source and emits one warning when import parsing fails', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/broken.js', '') + const warnings = [] + const source = 'import {' + assert.equal(await runLoader(appPath, source, fixture.importOptions, { + emitWarning: warning => warnings.push(warning), + }), source) + assert.equal(await runLoader(appPath, source, fixture.importOptions, { + emitWarning: warning => warnings.push(warning), + }), source) + assert.equal(warnings.length, 1) + assert.equal(warnings[0].name, 'DatadogTurbopackWarning') + }) + + it('falls back to process warnings when the loader context cannot emit one', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/process-warning.js', '') + const emitWarning = sinon.stub(process, 'emitWarning') + + assert.equal(await runLoader(appPath, 'import {', fixture.importOptions), 'import {') + sinon.assert.calledOnce(emitWarning) + }) + + it('keeps source when the Turbopack resolver cannot be initialized', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/resolver.js', '') + const source = "import { generateText } from 'ai'\n" + const warnings = [] + const getResolve = () => { + throw new Error('resolver unavailable') + } + + const result = await runLoader(appPath, source, fixture.importOptions, { + emitWarning: warning => warnings.push(warning), + getResolve, + }) + + assert.equal(result, source) + assert.equal(warnings.length, 1) + assert.match(warnings[0].message, /Could not initialize import resolution/) + }) + + it('settles each resolver once and contains edge-resolution failures', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/resolver-edge.js', '') + const source = "import { generateText } from 'ai'\n" + const duplicateCallback = () => (_directory, _request, callback) => { + callback(undefined, fixture.targetPath) + callback(new Error('late failure')) + } + + const rewritten = await runLoader(appPath, source, fixture.importOptions, { + getResolve: duplicateCallback, + }) + assert.match(rewritten, new RegExp(path.basename(fixture.proxyPath))) + + const throwingResolver = () => () => { + throw new Error('resolution failed') + } + assert.equal(await runLoader(appPath, source, fixture.importOptions, { + getResolve: throwingResolver, + }), source) + + const missingTarget = () => (_directory, _request, callback) => { + callback(undefined, path.join(fixture.projectDir, 'missing.js')) + } + assert.equal(await runLoader(appPath, source, fixture.importOptions, { + getResolve: missingTarget, + }), source) + }) + + it('keeps source when import generation fails', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/generator.js', '') + const generator = write( + fixture.projectDir, + 'generator.js', + "module.exports.default = () => { throw new Error('generation failed') }\n" + ) + const options = { + ...fixture.importOptions, + compiler: { ...fixture.importOptions.compiler, generator }, + } + const source = "import { generateText } from 'ai'\n" + const warnings = [] + + const result = await runLoader(appPath, source, options, { + emitWarning: warning => warnings.push(warning), + }) + + assert.equal(result, source) + assert.equal(warnings.length, 1) + assert.match(warnings[0].message, /Could not generate rewritten imports/) + }) + + it('does not redirect an edge after its planned target changes', async () => { + const fixture = await createAiFixture() + const appPath = write(fixture.projectDir, 'app/changed-target.js', '') + const source = "import { generateText } from 'ai'\n" + const warnings = [] + fs.writeFileSync(fixture.targetPath, 'export function generateText () { return 1 }\n') + + const result = await runLoader(appPath, source, fixture.importOptions, { + emitWarning: warning => warnings.push(warning), + }) + + assert.equal(result, source) + assert.equal(warnings.length, 1) + assert.match(warnings[0].message, /Skipped changed dependency/) + }) + + it('publishes CommonJS targets only when the channel has subscribers', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + const resourcePath = write(packageDir, 'index.js', "'use strict'\n\nmodule.exports = { original: true }\n") + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config).find(item => item.options.targetScope === 'direct').options + const transformed = await runLoader(resourcePath, fs.readFileSync(resourcePath, 'utf8'), options) + await assertGeneratedSourceIsLintClean(transformed, 'generated.cjs') + const publications = [] + const channel = { + hasSubscribers: false, + publish: payload => publications.push(payload), + } + + const inactive = executeCommonJs(transformed, channel) + channel.hasSubscribers = true + channel.publish = payload => { + publications.push(payload) + payload.module = { patched: true } + } + const active = executeCommonJs(transformed, channel) + + assert.equal(inactive.original, true) + assert.equal(inactive.patched, undefined) + assert.equal(active.patched, true) + assert.equal(publications.length, 1) + assert.equal(publications[0].package, 'ioredis') + assert.equal(publications[0].moduleName, 'ioredis') + assert.equal(publications[0].instrumentationIndexes.length, 1) + assert.equal(publications[0].instrumentationIndexes[0], 3) + }) + + it('matches relative runtimes by suffix and source hash', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, '@prisma/client', { main: 'index.js', version: '6.1.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const source = 'module.exports = { prisma: true }\n' + write(packageDir, 'runtime/library.js', source) + const matching = write(projectDir, 'generated/runtime/library.js', source) + const unrelated = write(projectDir, 'unrelated/runtime/library.js', 'module.exports = { unrelated: true }\n') + const otherFile = write(projectDir, 'generated/runtime/other.js', source) + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config).find(item => item.options.targetScope === 'relative').options + const matchingResult = await runLoader(matching, source, options) + const unrelatedSource = fs.readFileSync(unrelated, 'utf8') + + assert.match(matchingResult, /dd-trace:bundler:load/) + assert.match(matchingResult, /package: "\.\/runtime\/library\.js"/) + assert.equal(await runLoader(unrelated, unrelatedSource, options), unrelatedSource) + assert.equal(await runLoader(otherFile, source, options), source) + }) + + it('evicts old file hashes at the cache boundary', async function () { + this.timeout(30000) + const projectDir = createProject() + const packageDir = createPackage(projectDir, '@prisma/client', { main: 'index.js', version: '6.1.0' }) + write(packageDir, 'index.js', 'module.exports = {}') + const originalSource = 'module.exports = { first: true }\n' + const changedSource = 'module.exports = { other: true }\n' + write(packageDir, 'runtime/library.js', originalSource) + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config).find(item => item.options.targetScope === 'relative').options + const stableTime = new Date('2020-01-01T00:00:00.000Z') + const files = [] + + for (let index = 0; index <= 2048; index++) { + const file = write(projectDir, `generated/${index}/runtime/library.js`, originalSource) + fs.utimesSync(file, stableTime, stableTime) + files.push(file) + await runLoader(file, originalSource, options) + } + + const firstStat = fs.statSync(files[0]) + fs.writeFileSync(files[0], changedSource) + fs.utimesSync(files[0], stableTime, stableTime) + const statSync = fs.statSync.bind(fs) + sinon.stub(fs, 'statSync').callsFake(file => file === files[0] ? firstStat : statSync(file)) + + assert.equal(Buffer.byteLength(originalSource), Buffer.byteLength(changedSource)) + assert.equal(await runLoader(files[0], changedSource, options), changedSource) + }) + + it('does not use a plan after a dependency changes', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + const before = 'module.exports = { first: true }\n' + const resourcePath = write(packageDir, 'index.js', before) + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config)[0].options + const source = 'module.exports = { other: true }\n' + const warnings = [] + const { atime, mtime } = fs.statSync(resourcePath) + fs.writeFileSync(resourcePath, source) + fs.utimesSync(resourcePath, atime, mtime) + + const result = await runLoader(resourcePath, source, options, { + emitWarning: warning => warnings.push(warning), + }) + + assert.equal(result, source) + assert.equal(Buffer.byteLength(before), Buffer.byteLength(source)) + assert.equal(warnings.length, 1) + assert.match(warnings[0].message, /Skipped changed dependency/) + }) + + it('rejects a build plan that fails its integrity check', async () => { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + const resourcePath = write(packageDir, 'index.js', 'module.exports = {}') + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config)[0].options + fs.appendFileSync(options.manifestPath, ' ') + + await assert.rejects( + runLoader(resourcePath, fs.readFileSync(resourcePath, 'utf8'), options), + { message: /failed its integrity check/ } + ) + }) + + it('rejects missing loader options and unsupported build plans', async () => { + await assert.rejects( + runLoader(__filename, '', {}), + { name: 'TypeError', message: /requires a build plan path and hash/ } + ) + + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + const resourcePath = write(packageDir, 'index.js', 'module.exports = {}') + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config)[0].options + const plan = JSON.parse(fs.readFileSync(options.manifestPath, 'utf8')) + plan.version++ + const serialized = JSON.stringify(plan) + fs.writeFileSync(options.manifestPath, serialized) + + await assert.rejects( + runLoader(resourcePath, fs.readFileSync(resourcePath, 'utf8'), { + ...options, + manifestHash: createHash('sha256').update(serialized).digest('hex'), + }), + { message: /build plan .* is not supported/ } + ) + + plan.version-- + delete plan.relativeTargets + const missingFieldPlan = JSON.stringify(plan) + fs.writeFileSync(options.manifestPath, missingFieldPlan) + + await assert.rejects( + runLoader(resourcePath, fs.readFileSync(resourcePath, 'utf8'), { + ...options, + manifestHash: createHash('sha256').update(missingFieldPlan).digest('hex'), + }), + { message: /build plan .* is not supported/ } + ) + }) + + it('evicts old build plans at the cache boundary', async function () { + this.timeout(30000) + const fixtures = [] + + for (let index = 0; index <= 16; index++) { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + const resourcePath = write(packageDir, 'index.js', `module.exports = ${index}`) + const config = await withDatadogTurbopack({}, { projectDir }) + const options = findDatadogLoaders(config)[0].options + await runLoader(resourcePath, fs.readFileSync(resourcePath, 'utf8'), options) + fixtures.push({ options, resourcePath }) + } + + fs.appendFileSync(fixtures[0].options.manifestPath, ' ') + + await assert.rejects( + runLoader( + fixtures[0].resourcePath, + fs.readFileSync(fixtures[0].resourcePath, 'utf8'), + fixtures[0].options + ), + { message: /failed its integrity check/ } + ) + }) + + it('uses a relative proxy specifier for a source beside its generated proxy', async () => { + const fixture = await createAiFixture() + const appPath = write(path.dirname(fixture.proxyPath), 'route.js', '') + const result = await runLoader(appPath, "import { generateText } from 'ai'\n", fixture.importOptions) + + assert.match(result, /from ['"]\.\/[a-f\d]{64}\.mjs['"]/) + }) + + it('keeps ESM exports live and applies patches once per proxy evaluation', async () => { + const fixture = await createAiFixture() + const channel = dc.channel(CHANNEL) + await assertGeneratedSourceIsLintClean(fs.readFileSync(fixture.proxyPath, 'utf8'), 'generated.mjs') + const inactive = await import(`${pathToFileURL(fixture.proxyPath).href}?inactive`) + let publications = 0 + const subscriber = payload => { + if (payload.package !== 'ai') return + publications++ + payload.apply({ + generateText: () => 'patched', + streamText: () => 'patched-stream', + }, false) + } + channel.subscribe(subscriber) + + try { + const active = await import(`${pathToFileURL(fixture.proxyPath).href}?active`) + assert.equal(inactive.generateText(), 'original') + assert.equal(active.generateText(), 'patched') + assert.equal(publications, 1) + } finally { + channel.unsubscribe(subscriber) + } + }) +}) + +/** + * @param {string} [ioredisSource] + * @returns {Promise<{ + * commonJsPath: string, + * importOptions: object, + * ioredisPath: string, + * packageOptions: object, + * projectDir: string, + * proxyPath: string, + * targetPath: string + * }>} + */ +async function createAiFixture (ioredisSource = 'module.exports = {}') { + const projectDir = createProject() + const packageDir = createPackage(projectDir, 'ai', { + exports: { + import: './index.mjs', + require: './index.cjs', + }, + main: 'index.cjs', + type: 'module', + version: '7.0.0', + }) + write(packageDir, 'index.mjs', [ + "export function generateText () { return 'original' }", + "export function streamText () { return 'original-stream' }", + '', + ].join('\n')) + const commonJsPath = write(packageDir, 'index.cjs', 'module.exports = {}\n') + const ioredisDirectory = createPackage(projectDir, 'ioredis', { main: 'index.js', version: '5.0.0' }) + const ioredisPath = write(ioredisDirectory, 'index.js', ioredisSource) + const config = await withDatadogTurbopack({}, { projectDir }) + const loaders = findDatadogLoaders(config) + const importOptions = loaders.find(item => item.options.rewriteEdges && !item.options.targetScope).options + const packageOptions = loaders.find(item => item.options.targetScope === 'direct').options + const plan = JSON.parse(fs.readFileSync(importOptions.manifestPath, 'utf8')) + const [targetPath, target] = Object.entries(plan.targets).find(([, entry]) => entry.esm) + + return { + commonJsPath, + importOptions, + ioredisPath, + packageOptions, + projectDir, + proxyPath: target.proxyPath, + targetPath, + } +} + +/** + * @param {string} resourcePath + * @param {string} source + * @param {object} options + * @param {{ emitWarning?: (warning: Error) => void, getResolve?: Function }} [settings] + * @returns {Promise} + */ +async function runLoader (resourcePath, source, options, settings = {}) { + const { code } = await runLoaderResult(resourcePath, source, options, settings) + return code +} + +/** + * @param {string} resourcePath + * @param {string} source + * @param {object} options + * @param {{ emitWarning?: (warning: Error) => void, getResolve?: Function, sourceMap?: object }} [settings] + * @returns {Promise<{ code: string, map?: object }>} + */ +function runLoaderResult (resourcePath, source, options, settings = {}) { + return new Promise((resolve, reject) => { + /** + * @param {Error} [error] + * @param {string} [code] + * @param {object} [map] + */ + function callback (error, code, map) { + if (error) reject(error) + else resolve({ code, map }) + } + + loader.call({ + async: () => callback, + emitWarning: settings.emitWarning, + getOptions: () => options, + getResolve: settings.getResolve ?? (resolveOptions => enhancedResolve.create(resolveOptions)), + resourcePath, + }, source, settings.sourceMap) + }) +} + +/** + * @param {string} source + * @param {object} channel + * @returns {object} + */ +function executeCommonJs (source, channel) { + const context = { + module: { exports: {} }, + require: () => ({ channel: () => channel }), + } + vm.runInNewContext(source, context) + return context.module.exports +} + +/** + * @param {string} source + * @param {string} filename + * @returns {Promise} + */ +async function assertGeneratedSourceIsLintClean (source, filename) { + if (!lintRuntimeSupported) return + + const eslint = new ESLint() + const [result] = await eslint.lintText(source, { + filePath: path.join(process.cwd(), 'packages/datadog-turbopack/test', filename), + }) + assert.deepEqual(result.messages, []) +}