perf: speed up build() and large arrays of numbers, integers, and strings

README.md

@@ -623,11 +623,15 @@ In order to improve that the user can set the `largeArrayMechanism` and
 
 `largeArrayMechanism`'s default value is `default`. Valid values for it are:
 
-- `default` - This option is a compromise between performance and feature set by
-still providing the expected functionality out of this lib but giving up some
-possible performance gain. With this option set, **large arrays** would be
-stringified by joining their stringified elements using `Array.join` instead of
-string concatenation for better performance
+- `default` - This option keeps the full feature set of this lib: every element
+is still coerced to the type declared in `items`, and type errors are still
+thrown. Arrays whose `items` schema is a plain `{ type: 'string' }` (no
+`format`, `nullable`, `const`, or combinator keyword) are serialized at or
+above `largeArraySize` by copying the elements and calling the native
+`JSON.stringify` on the copy, which is several times faster than appending one
+element at a time. If an element is not a string, that element and the ones
+after it go through the regular per-element coercion, so the output is the
+same as before.
 - `json-stringify` - This option will remove support for schema validation
 within **large arrays** completely. By doing so the overhead previously
 mentioned is nulled, greatly improving execution time. Mind there's no change
@@ -642,6 +646,32 @@ integer-like values, such as:
 - `'2e4'` - _note this will be converted to `2`, not `20000`_
 - `1.5` - _note this will be converted to `1`_
 
+##### Number and integer arrays
+
+With `largeArrayMechanism: 'default'`, arrays whose `items` schema is a plain
+`{ type: 'number' }` or `{ type: 'integer' }` (no `nullable`, `const`, `$ref`,
+or combinator keyword) use the same copy-then-`JSON.stringify` strategy, but
+with their own length cutoffs that do not depend on `largeArraySize`. The
+cutoffs are the array lengths above which the copy beats the per-element loop,
+and they depend on the V8 version:
+
+| Node.js | `number` | `integer` |
+|---|---:|---:|
+| 22 and later | 128 elements | 512 elements |
+| 20 | 300000 elements | 300000 elements |
+
+Below the cutoff the array is serialized element by element as before. Above
+it, the elements are copied until one is found that is not already a `number`
+(or not already an integer, for `integer`); that element and the ones after it
+go through the regular per-element coercion and validation (`null` still
+becomes `0`, `rounding` still applies, `NaN` still throws). The output is
+byte-identical either way. Arrays of `boolean`, `null`, objects, or any other
+`items` schema are not affected.
+
+In [standalone mode](#standalone) the cutoffs are baked into the generated
+code, so the generated module inherits the cutoffs of the Node.js version that
+ran `build()`, not of the one that runs the module.
+
 <a name="unsafe"></a>
 #### Unsafe string
 By default, the library escapes all strings. With the 'unsafe' format, the string isn't escaped. This has a potentially dangerous security issue. You can use it only if you are sure that your data doesn't need escaping. The advantage is a significant performance improvement.
@@ -748,6 +778,12 @@ may require Ajv runtime helpers used by the generated validation code. Custom
 Ajv formats used by these schemas must support Ajv's standalone code
 generation.
 
+The generated code also bakes in the length cutoffs used for
+[number and integer arrays](#largearrays). Those cutoffs are chosen for the
+Node.js version that ran `build()`; a module built on Node.js 20 and run on
+Node.js 22 or later keeps the Node.js 20 cutoffs, and vice versa. The output is
+the same either way, only the speed differs.
+
 <a name="acknowledgments"></a>
 ## Acknowledgments
 

benchmark/bench-fixtures.js

@@ -0,0 +1,54 @@
+'use strict'
+
+const arrayKinds = ['integer', 'number', 'short-string', 'long-string', 'flat-object', 'nested-object', 'nullable-number', 'Float64Array', 'Int32Array']
+const arraySizes = [1e3, 2e4, 1e5, 1e6]
+const mechanisms = ['default', 'json-stringify', 'native']
+const flatSchema = { type: 'object', properties: { s: { type: 'string' }, n: { type: 'number' }, b: { type: 'boolean' } } }
+const longString = 'a"quoted" string\\with\ncontrols '.repeat(4)
+
+function arrayFixture (kind, size) {
+  let items
+  let value
+  switch (kind) {
+    case 'integer':
+    case 'Int32Array':
+      items = { type: 'integer' }
+      value = i => (i % 2001) - 1000
+      break
+    case 'number':
+    case 'Float64Array':
+      items = { type: 'number' }
+      value = i => ((i % 2001) - 1000) / 10
+      break
+    case 'short-string':
+      items = { type: 'string' }
+      value = i => `hello ${i % 100}`
+      break
+    case 'long-string':
+      items = { type: 'string' }
+      value = () => longString
+      break
+    case 'flat-object':
+      items = flatSchema
+      value = i => ({ s: `hello ${i % 100}`, n: i % 1000, b: i % 2 === 0 })
+      break
+    case 'nested-object':
+      items = { type: 'object', properties: { id: { type: 'integer' }, inner: { type: 'array', items: flatSchema } } }
+      value = i => ({ id: i, inner: [{ s: 'hello', n: i % 1000, b: true }, { s: 'world', n: i / 10, b: false }] })
+      break
+    case 'nullable-number':
+      items = { type: 'number', nullable: true }
+      value = i => i % 4 === 0 ? null : i / 10
+      break
+    default:
+      throw new Error(`Unknown array kind: ${kind}`)
+  }
+  const input = kind === 'Float64Array'
+    ? Float64Array.from({ length: size }, (_, i) => value(i))
+    : kind === 'Int32Array'
+      ? Int32Array.from({ length: size }, (_, i) => value(i))
+      : Array.from({ length: size }, (_, i) => value(i))
+  return { schema: { type: 'array', items }, input, typed: ArrayBuffer.isView(input) }
+}
+
+module.exports = { arrayKinds, arraySizes, mechanisms, arrayFixture }

benchmark/bench-large-array.js

@@ -0,0 +1,64 @@
+'use strict'
+
+const assert = require('node:assert/strict')
+const { parseArgs } = require('node:util')
+const { isMainThread, parentPort, workerData } = require('node:worker_threads')
+const { Bench } = require('tinybench')
+const build = require('..')
+const { arrayKinds, arraySizes, mechanisms, arrayFixture } = require('./bench-fixtures')
+const { positiveNumber, runWorker, saveResults } = require('./bench-utils')
+
+async function measure ({ kind, size, mechanism, time, iterations }) {
+  const { schema, input, typed } = arrayFixture(kind, size)
+  const stringify = mechanism === 'native' ? JSON.stringify : build(schema, { largeArrayMechanism: mechanism })
+  if (typed && mechanism !== 'native') {
+    // Rejection is existing behavior; do not time exceptions as serialization.
+    assert.throws(() => stringify(input), TypeError)
+    return { kind, size, mechanism, status: 'unsupported (TypeError)' }
+  }
+  assert.equal(stringify(input), JSON.stringify(input))
+  const bytes = Buffer.byteLength(stringify(input))
+  const bench = new Bench({ time, iterations, warmupTime: 100, warmupIterations: 2, throws: true })
+  bench.add(kind, () => {
+    Buffer.byteLength(stringify(input))
+  })
+  await bench.run()
+  const { latency, throughput } = bench.tasks[0].result
+  return {
+    kind,
+    size,
+    mechanism,
+    status: 'ok',
+    bytes,
+    ops: throughput.p50,
+    meanOps: throughput.mean,
+    medianMs: latency.p50,
+    rme: latency.rme,
+    samples: latency.samplesCount
+  }
+}
+
+async function main () {
+  const { values } = parseArgs({ options: { kind: { type: 'string' }, size: { type: 'string' }, time: { type: 'string', default: '300' }, iterations: { type: 'string', default: '8' }, json: { type: 'string' } } })
+  const kinds = values.kind ? [values.kind] : arrayKinds
+  if (kinds.some(kind => !arrayKinds.includes(kind))) throw new Error(`kind must be one of ${arrayKinds.join(', ')}`)
+  const sizes = values.size ? [positiveNumber(values.size, 'size', true)] : arraySizes
+  const config = { time: positiveNumber(values.time, 'time'), iterations: positiveNumber(values.iterations, 'iterations', true) }
+  const rows = []
+  console.log('| Kind | Size | Mechanism | Median ops/sec | Latency RME % | Samples |')
+  console.log('| --- | ---: | --- | ---: | ---: | ---: |')
+  for (const size of sizes) {
+    for (const kind of kinds) {
+      for (const mechanism of mechanisms) {
+        const row = await runWorker(__filename, { kind, size, mechanism, ...config })
+        rows.push(row)
+        console.log(`| ${kind} | ${size} | ${mechanism} | ${row.ops?.toFixed(2) ?? row.status} | ${row.rme?.toFixed(2) ?? '—'} | ${row.samples ?? '—'} |`)
+      }
+    }
+  }
+  console.log('\nTyped arrays are rejected by FJS; native serializes indexed objects. Each measured call includes Buffer.byteLength().')
+  saveResults(values.json, { ...config, kinds, sizes }, rows)
+}
+
+if (isMainThread) main().catch(err => { console.error(err); process.exitCode = 1 })
+else measure(workerData).then(result => parentPort.postMessage(result)).catch(err => { throw err })

benchmark/bench-utils.js

@@ -0,0 +1,41 @@
+'use strict'
+
+const { Worker } = require('node:worker_threads')
+const { writeFileSync } = require('node:fs')
+const os = require('node:os')
+
+function positiveNumber (value, name, integer = false) {
+  const result = Number(value)
+  if (!Number.isFinite(result) || result <= 0 || (integer && !Number.isInteger(result))) {
+    throw new Error(`${name} must be a positive ${integer ? 'integer' : 'number'}`)
+  }
+  return result
+}
+
+function runWorker (file, workerData) {
+  return new Promise((resolve, reject) => {
+    const worker = new Worker(file, { workerData })
+    let result
+    worker.on('message', value => { result = value })
+    worker.on('error', reject)
+    worker.on('exit', code => {
+      if (code !== 0 || result === undefined) reject(new Error(`Benchmark worker exited ${code} without a valid result`))
+      else resolve(result)
+    })
+  })
+}
+
+function saveResults (file, config, rows) {
+  if (!file) return
+  writeFileSync(file, JSON.stringify({
+    timestamp: new Date().toISOString(),
+    node: process.version,
+    platform: `${process.platform}/${process.arch}`,
+    cpu: os.cpus()[0].model,
+    cpus: os.cpus().length,
+    config,
+    rows
+  }, null, 2) + '\n')
+}
+
+module.exports = { positiveNumber, runWorker, saveResults }