diff --git a/benchmark/_http-benchmarkers.js b/benchmark/_http-benchmarkers.js deleted file mode 100644 index ec0e80a9e7ed..000000000000 --- a/benchmark/_http-benchmarkers.js +++ /dev/null @@ -1,257 +0,0 @@ -'use strict'; - -const child_process = require('child_process'); -const path = require('path'); -const fs = require('fs'); - -const requirementsURL = - 'https://github.com/nodejs/node/blob/HEAD/benchmark/writing-and-running-benchmarks.md#http-benchmark-requirements'; - -// The port used by servers and wrk -exports.PORT = Number(process.env.PORT) || 12346; - -class AutocannonBenchmarker { - constructor() { - this.name = 'autocannon'; - this.executable = - process.platform === 'win32' ? 'autocannon.cmd' : 'autocannon'; - const result = child_process.spawnSync(this.executable, ['-h']); - this.present = !(result.error && result.error.code === 'ENOENT'); - } - - create(options) { - const args = [ - '-d', options.duration, - '-c', options.connections, - '-j', - '-n', - ]; - for (const field in options.headers) { - args.push('-H', `${field}=${options.headers[field]}`); - } - const scheme = options.scheme || 'http'; - args.push(`${scheme}://127.0.0.1:${options.port}${options.path}`); - const child = child_process.spawn(this.executable, args); - return child; - } - - processResults(output) { - let result; - try { - result = JSON.parse(output); - } catch { - return undefined; - } - if (!result || !result.requests || !result.requests.average) { - return undefined; - } - return result.requests.average; - } -} - -class WrkBenchmarker { - constructor() { - this.name = 'wrk'; - this.executable = 'wrk'; - const result = child_process.spawnSync(this.executable, ['-h']); - this.present = !(result.error && result.error.code === 'ENOENT'); - } - - create(options) { - const duration = typeof options.duration === 'number' ? - Math.max(options.duration, 1) : - options.duration; - const scheme = options.scheme || 'http'; - const args = [ - '-d', duration, - '-c', options.connections, - '-t', Math.min(options.connections, require('os').cpus().length || 8), - `${scheme}://127.0.0.1:${options.port}${options.path}`, - ]; - for (const field in options.headers) { - args.push('-H', `${field}: ${options.headers[field]}`); - } - const child = child_process.spawn(this.executable, args); - return child; - } - - processResults(output) { - const throughputRe = /Requests\/sec:[ \t]+([0-9.]+)/; - const match = output.match(throughputRe); - const throughput = match && +match[1]; - if (!isFinite(throughput)) { - return undefined; - } - return throughput; - } -} - -/** - * Simple, single-threaded benchmarker for testing if the benchmark - * works - */ -class TestDoubleBenchmarker { - constructor(type) { - // `type` is the type of benchmarker. Possible values are 'http', 'https', - // and 'http2'. - this.name = `test-double-${type}`; - this.executable = path.resolve(__dirname, '_test-double-benchmarker.js'); - this.present = fs.existsSync(this.executable); - this.type = type; - } - - create(options) { - process.env.duration = process.env.duration || options.duration || 5; - - const scheme = options.scheme || 'http'; - const env = { - test_url: `${scheme}://127.0.0.1:${options.port}${options.path}`, - ...process.env - }; - - const child = child_process.fork(this.executable, - [this.type], - { silent: true, env }); - return child; - } - - processResults(output) { - let result; - try { - result = JSON.parse(output); - } catch { - return undefined; - } - return result.throughput; - } -} - -/** - * HTTP/2 Benchmarker - */ -class H2LoadBenchmarker { - constructor() { - this.name = 'h2load'; - this.executable = 'h2load'; - const result = child_process.spawnSync(this.executable, ['-h']); - this.present = !(result.error && result.error.code === 'ENOENT'); - } - - create(options) { - const args = []; - if (typeof options.requests === 'number') - args.push('-n', options.requests); - if (typeof options.clients === 'number') - args.push('-c', options.clients); - if (typeof options.threads === 'number') - args.push('-t', options.threads); - if (typeof options.maxConcurrentStreams === 'number') - args.push('-m', options.maxConcurrentStreams); - if (typeof options.initialWindowSize === 'number') - args.push('-w', options.initialWindowSize); - if (typeof options.sessionInitialWindowSize === 'number') - args.push('-W', options.sessionInitialWindowSize); - if (typeof options.rate === 'number') - args.push('-r', options.rate); - if (typeof options.ratePeriod === 'number') - args.push(`--rate-period=${options.ratePeriod}`); - if (typeof options.duration === 'number') - args.push('-T', options.duration); - if (typeof options.timeout === 'number') - args.push('-N', options.timeout); - if (typeof options.headerTableSize === 'number') - args.push(`--header-table-size=${options.headerTableSize}`); - if (typeof options.encoderHeaderTableSize === 'number') { - args.push( - `--encoder-header-table-size=${options.encoderHeaderTableSize}`); - } - const scheme = options.scheme || 'http'; - const host = options.host || '127.0.0.1'; - args.push(`${scheme}://${host}:${options.port}${options.path}`); - const child = child_process.spawn(this.executable, args); - return child; - } - - processResults(output) { - const rex = /(\d+\.\d+) req\/s/; - return rex.exec(output)[1]; - } -} - -const http_benchmarkers = [ - new WrkBenchmarker(), - new AutocannonBenchmarker(), - new TestDoubleBenchmarker('http'), - new TestDoubleBenchmarker('https'), - new TestDoubleBenchmarker('http2'), - new H2LoadBenchmarker(), -]; - -const benchmarkers = {}; - -http_benchmarkers.forEach((benchmarker) => { - benchmarkers[benchmarker.name] = benchmarker; - if (!exports.default_http_benchmarker && benchmarker.present) { - exports.default_http_benchmarker = benchmarker.name; - } -}); - -exports.run = function(options, callback) { - options = { - port: exports.PORT, - path: '/', - connections: 100, - duration: 5, - benchmarker: exports.default_http_benchmarker, - ...options - }; - if (!options.benchmarker) { - callback(new Error('Could not locate required http benchmarker. See ' + - `${requirementsURL} for further instructions.`)); - return; - } - const benchmarker = benchmarkers[options.benchmarker]; - if (!benchmarker) { - callback(new Error(`Requested benchmarker '${options.benchmarker}' ` + - 'is not supported')); - return; - } - if (!benchmarker.present) { - callback(new Error(`Requested benchmarker '${options.benchmarker}' ` + - 'is not installed')); - return; - } - - const benchmarker_start = process.hrtime.bigint(); - - const child = benchmarker.create(options); - - child.stderr.pipe(process.stderr); - - let stdout = ''; - child.stdout.setEncoding('utf8'); - child.stdout.on('data', (chunk) => stdout += chunk); - - child.once('close', (code) => { - const benchmark_end = process.hrtime.bigint(); - if (code) { - let error_message = `${options.benchmarker} failed with ${code}.`; - if (stdout !== '') { - error_message += ` Output: ${stdout}`; - } - callback(new Error(error_message), code); - return; - } - - const result = benchmarker.processResults(stdout); - if (result === undefined) { - callback(new Error( - `${options.benchmarker} produced strange output: ${stdout}`), code); - return; - } - - const elapsed = benchmark_end - benchmarker_start; - callback(null, code, options.benchmarker, result, elapsed); - }); - -}; diff --git a/benchmark/common.js b/benchmark/common.js index 6ed230ffde42..f916b9579842 100644 --- a/benchmark/common.js +++ b/benchmark/common.js @@ -1,308 +1,5 @@ 'use strict'; - -const child_process = require('child_process'); -const http_benchmarkers = require('./_http-benchmarkers.js'); - -class Benchmark { - constructor(fn, configs, options = {}) { - // Used to make sure a benchmark only start a timer once - this._started = false; - - // Indicate that the benchmark ended - this._ended = false; - - // Holds process.hrtime value - this._time = 0n; - - // Use the file name as the name of the benchmark - this.name = require.main.filename.slice(__dirname.length + 1); - - // Execution arguments i.e. flags used to run the jobs - this.flags = process.env.NODE_BENCHMARK_FLAGS ? - process.env.NODE_BENCHMARK_FLAGS.split(/\s+/) : - []; - - // Parse job-specific configuration from the command line arguments - const argv = process.argv.slice(2); - const parsed_args = this._parseArgs(argv, configs, options); - this.options = parsed_args.cli; - this.extra_options = parsed_args.extra; - if (options.flags) { - this.flags = this.flags.concat(options.flags); - } - - // The configuration list as a queue of jobs - this.queue = this._queue(this.options); - - // The configuration of the current job, head of the queue - this.config = this.queue[0]; - - process.nextTick(() => { - if (Object.hasOwn(process.env, 'NODE_RUN_BENCHMARK_FN')) { - fn(this.config); - } else { - // _run will use fork() to create a new process for each configuration - // combination. - this._run(); - } - }); - } - - _parseArgs(argv, configs, options) { - const cliOptions = {}; - - // Check for the test mode first. - const testIndex = argv.indexOf('--test'); - if (testIndex !== -1) { - for (const [key, rawValue] of Object.entries(configs)) { - let value = Array.isArray(rawValue) ? rawValue[0] : rawValue; - // Set numbers to one by default to reduce the runtime. - if (typeof value === 'number') { - if (key === 'dur' || key === 'duration') { - value = 0.05; - } else if (value > 1) { - value = 1; - } - } - cliOptions[key] = [value]; - } - // Override specific test options. - if (options.test) { - for (const [key, value] of Object.entries(options.test)) { - cliOptions[key] = Array.isArray(value) ? value : [value]; - } - } - argv.splice(testIndex, 1); - } else { - // Accept single values instead of arrays. - for (const [key, value] of Object.entries(configs)) { - if (!Array.isArray(value)) - configs[key] = [value]; - } - } - - const extraOptions = {}; - const validArgRE = /^(.+?)=([\s\S]*)$/; - // Parse configuration arguments - for (const arg of argv) { - const match = arg.match(validArgRE); - if (!match) { - console.error(`bad argument: ${arg}`); - process.exit(1); - } - const [, key, value] = match; - if (Object.hasOwn(configs, key)) { - if (!cliOptions[key]) - cliOptions[key] = []; - cliOptions[key].push( - // Infer the type from the config object and parse accordingly - typeof configs[key][0] === 'number' ? +value : value - ); - } else { - extraOptions[key] = value; - } - } - return { cli: { ...configs, ...cliOptions }, extra: extraOptions }; - } - - _queue(options) { - const queue = []; - const keys = Object.keys(options); - - // Perform a depth-first walk through all options to generate a - // configuration list that contains all combinations. - function recursive(keyIndex, prevConfig) { - const key = keys[keyIndex]; - const values = options[key]; - - for (const value of values) { - if (typeof value !== 'number' && typeof value !== 'string') { - throw new TypeError( - `configuration "${key}" had type ${typeof value}`); - } - if (typeof value !== typeof values[0]) { - // This is a requirement for being able to consistently and - // predictably parse CLI provided configuration values. - throw new TypeError(`configuration "${key}" has mixed types`); - } - - const currConfig = { [key]: value, ...prevConfig }; - - if (keyIndex + 1 < keys.length) { - recursive(keyIndex + 1, currConfig); - } else { - queue.push(currConfig); - } - } - } - - if (keys.length > 0) { - recursive(0, {}); - } else { - queue.push({}); - } - - return queue; - } - - http(options, cb) { - const http_options = { ...options }; - http_options.benchmarker = http_options.benchmarker || - this.config.benchmarker || - this.extra_options.benchmarker || - http_benchmarkers.default_http_benchmarker; - http_benchmarkers.run( - http_options, (error, code, used_benchmarker, result, elapsed) => { - if (cb) { - cb(code); - } - if (error) { - console.error(error); - process.exit(code || 1); - } - this.config.benchmarker = used_benchmarker; - this.report(result, elapsed); - } - ); - } - - _run() { - // If forked, report to the parent. - if (process.send) { - process.send({ - type: 'config', - name: this.name, - queueLength: this.queue.length, - }); - } - - const recursive = (queueIndex) => { - const config = this.queue[queueIndex]; - - // Set NODE_RUN_BENCHMARK_FN to indicate that the child shouldn't - // construct a configuration queue, but just execute the benchmark - // function. - const childEnv = { ...process.env }; - childEnv.NODE_RUN_BENCHMARK_FN = ''; - - // Create configuration arguments - const childArgs = []; - for (const [key, value] of Object.entries(config)) { - childArgs.push(`${key}=${value}`); - } - for (const [key, value] of Object.entries(this.extra_options)) { - childArgs.push(`${key}=${value}`); - } - - const child = child_process.fork(require.main.filename, childArgs, { - env: childEnv, - execArgv: this.flags.concat(process.execArgv), - }); - child.on('message', sendResult); - child.on('close', (code) => { - if (code) { - process.exit(code); - } - - if (queueIndex + 1 < this.queue.length) { - recursive(queueIndex + 1); - } - }); - }; - - recursive(0); - } - - start() { - if (this._started) { - throw new Error('Called start more than once in a single benchmark'); - } - this._started = true; - this._time = process.hrtime.bigint(); - } - - end(operations) { - // Get elapsed time now and do error checking later for accuracy. - const time = process.hrtime.bigint(); - - if (!this._started) { - throw new Error('called end without start'); - } - if (this._ended) { - throw new Error('called end multiple times'); - } - if (typeof operations !== 'number') { - throw new Error('called end() without specifying operation count'); - } - if (!process.env.NODEJS_BENCHMARK_ZERO_ALLOWED && operations <= 0) { - throw new Error('called end() with operation count <= 0'); - } - - this._ended = true; - - if (time === this._time) { - if (!process.env.NODEJS_BENCHMARK_ZERO_ALLOWED) - throw new Error('insufficient clock precision for short benchmark'); - // Avoid dividing by zero - this.report(operations && Number.MAX_VALUE, 0n); - return; - } - - const elapsed = time - this._time; - const rate = operations / (Number(elapsed) / 1e9); - this.report(rate, elapsed); - } - - report(rate, elapsed) { - sendResult({ - name: this.name, - conf: this.config, - rate, - time: nanoSecondsToString(elapsed), - type: 'report', - }); - } -} - -function nanoSecondsToString(bigint) { - const str = bigint.toString(); - const decimalPointIndex = str.length - 9; - if (decimalPointIndex <= 0) { - return `0.${'0'.repeat(-decimalPointIndex)}${str}`; - } - return `${str.slice(0, decimalPointIndex)}.${str.slice(decimalPointIndex)}`; -} - -function formatResult(data) { - // Construct configuration string, " A=a, B=b, ..." - let conf = ''; - for (const key of Object.keys(data.conf)) { - conf += ` ${key}=${JSON.stringify(data.conf[key])}`; - } - - let rate = data.rate.toString().split('.'); - rate[0] = rate[0].replace(/(\d)(?=(?:\d\d\d)+(?!\d))/g, '$1,'); - rate = (rate[1] ? rate.join('.') : rate[0]); - return `${data.name}${conf}: ${rate}\n`; -} - -function sendResult(data) { - if (process.send) { - // If forked, report by process send - process.send(data, () => { - if (Object.hasOwn(process.env, 'NODE_RUN_BENCHMARK_FN')) { - // If, for any reason, the process is unable to self close within - // a second after completing, forcefully close it. - setTimeout(() => { - process.exit(0); - }, 5000).unref(); - } - }); - } else { - // Otherwise report by stdout - process.stdout.write(formatResult(data)); - } -} +const { Benchmark, createBenchmark } = require('node:benchmark'); const urls = { long: 'http://nodejs.org:89/docs/latest/api/foo/bar/qua/13949281/0f28b/' + @@ -393,7 +90,7 @@ function bakeUrlData(type, e = 0, withBase = false, asUrl = false) { module.exports = { Benchmark, - PORT: http_benchmarkers.PORT, + PORT: Benchmark.HTTP_BENCHMARK_PORT, bakeUrlData, binding(bindingName) { try { @@ -405,10 +102,8 @@ module.exports = { } }, buildType: process.features.debug ? 'Debug' : 'Release', - createBenchmark(fn, configs, options) { - return new Benchmark(fn, configs, options); - }, - sendResult, + createBenchmark, + sendResult: Benchmark.SendResult, searchParams, urlDataTypes: Object.keys(urls).concat(['wpt']), urls, diff --git a/benchmark/compare.js b/benchmark/compare.js index 169948e006d6..59e438fefd21 100644 --- a/benchmark/compare.js +++ b/benchmark/compare.js @@ -1,11 +1,7 @@ 'use strict'; -const { fork } = require('child_process'); -const { inspect } = require('util'); -const path = require('path'); const CLI = require('./_cli.js'); -const BenchmarkProgress = require('./_benchmark_progress.js'); - +const { compareBenchmarks } = require('node:benchmark'); // // Parse arguments // @@ -30,84 +26,24 @@ if (!cli.optional.new || !cli.optional.old) { cli.abort(cli.usage); } -const binaries = ['old', 'new']; const runs = cli.optional.runs ? parseInt(cli.optional.runs, 10) : 30; -const benchmarks = cli.benchmarks(); +const benchmarkFiles = cli.benchmarks(); -if (benchmarks.length === 0) { +if (benchmarkFiles.length === 0) { console.error('No benchmarks found'); process.exitCode = 1; return; } -// Create queue from the benchmarks list such both node versions are tested -// `runs` amount of times each. -// Note: BenchmarkProgress relies on this order to estimate -// how much runs remaining for a file. All benchmarks generated from -// the same file must be run consecutively. -const queue = []; -for (const filename of benchmarks) { - for (let iter = 0; iter < runs; iter++) { - for (const binary of binaries) { - queue.push({ binary, filename, iter }); - } +compareBenchmarks( + { + binary: { + old: cli.optional.old, + new: cli.optional.new + }, + benchmarkFiles, + showProgress: !cli.optional['no-progress'], + set: cli.optional.set, + runs, } -} -// queue.length = binary.length * runs * benchmarks.length - -// Print csv header -console.log('"binary","filename","configuration","rate","time"'); - -const kStartOfQueue = 0; - -const showProgress = !cli.optional['no-progress']; -let progress; -if (showProgress) { - progress = new BenchmarkProgress(queue, benchmarks); - progress.startQueue(kStartOfQueue); -} - -(function recursive(i) { - const job = queue[i]; - - const child = fork(path.resolve(__dirname, job.filename), cli.optional.set, { - execPath: cli.optional[job.binary] - }); - - child.on('message', (data) => { - if (data.type === 'report') { - // Construct configuration string, " A=a, B=b, ..." - let conf = ''; - for (const key of Object.keys(data.conf)) { - conf += ` ${key}=${inspect(data.conf[key])}`; - } - conf = conf.slice(1); - // Escape quotes (") for correct csv formatting - conf = conf.replace(/"/g, '""'); - - console.log(`"${job.binary}","${job.filename}","${conf}",` + - `${data.rate},${data.time}`); - if (showProgress) { - // One item in the subqueue has been completed. - progress.completeConfig(data); - } - } else if (showProgress && data.type === 'config') { - // The child has computed the configurations, ready to run subqueue. - progress.startSubqueue(data, i); - } - }); - - child.once('close', (code) => { - if (code) { - process.exit(code); - } - if (showProgress) { - progress.completeRun(job); - } - - // If there are more benchmarks execute the next - if (i + 1 < queue.length) { - recursive(i + 1); - } - }); -})(kStartOfQueue); +); diff --git a/doc/api/errors.md b/doc/api/errors.md index 9bb467af7fc7..40cbcb80794f 100644 --- a/doc/api/errors.md +++ b/doc/api/errors.md @@ -726,6 +726,77 @@ An attempt was made to register something that is not a function as an The type of an asynchronous resource was invalid. Users are also able to define their own types if using the public embedder API. + + +### `ERR_BENCHMARK_HTTP_COULD_NOT_LOCATE_BENCHMARKER` + +Could not locate any of the http benchmarkers(autocannon, wrk and h2load). + + + +### `ERR_BENCHMARK_HTTP_BENCHMARKER_NOT_SUPPROTED` + +Specified http benchmarker implementation does not exists. +Valid http benchmarkers are. + +1. autocannon +2. wrk +3. h2load + + + +### `ERR_BENCHMARK_CLOCK_PRECISION` + +Insufficient clock precision for short benchmark. + + + +### `ERR_BENCHMARK_END_CALLED_MORE_THAN_ONCE` + +This error is thrown when the end method is called multiple times. + + + +### `ERR_BENCHMARK_HTTP_BENCHMARKER_NOT_INSTALLED` + +This error is thrown when the specified http benchmarker cannot be found. + + + +### `ERR_BENCHMARK_HTTP_STRANGE_OUTPUT` + +Thrown when the specified http benchmarker produces a strange output. + + + +### `ERR_BENCHMARK_HTTP_UNKNOWN_OUTPUT` + +Thrown when the specified http benchmarker failed. + + + +### `ERR_BENCHMARK_INVALID_OPERATION_COUNT` + +The end method was called when operations count is <= 0 + + + +### `ERR_BENCHMARK_NO_OPERATION_COUNT` + +The end method was called without specifying operation count + + + +### `ERR_BENCHMARK_START_CALLED_MORE_THAN_ONCE` + +This error is thrown when the end method is called multiple times. + + + +### `ERR_BENCHMARK_START_NOT_CALLED` + +The start method was not called before calling the end method. + ### `ERR_BROTLI_COMPRESSION_FAILED` diff --git a/lib/benchmark.js b/lib/benchmark.js new file mode 100644 index 000000000000..929a780c8c6b --- /dev/null +++ b/lib/benchmark.js @@ -0,0 +1,536 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS + +'use strict'; + +const child_process = require('child_process'); +const util = require('util'); +const path = require('path'); +const { setTimeout } = require('timers'); +const { + ObjectKeys, + StringPrototypeSlice, + StringPrototypeRepeat, + ArrayPrototypePush, + ArrayPrototypeIndexOf, + ArrayPrototypeSplice, + ArrayPrototypeSlice, + ArrayIsArray, + ArrayPrototypeConcat, + BigIntPrototypeToString, + ObjectPrototypeHasOwnProperty: ObjectHasOwn, + Number, + NumberMAX_VALUE, + NumberPrototypeToString, + JSONStringify, +} = primordials; + +const http_benchmarkers = require('internal/benchmark/_http-benchmarkers'); +const BenchmarkProgress = require('internal/benchmark/_benchmark_progress'); + +const { + codes: { + ERR_BENCHMARK_START_CALLED_MORE_THAN_ONCE, + ERR_BENCHMARK_END_CALLED_MORE_THAN_ONCE, + ERR_BENCHMARK_NO_OPERATION_COUNT, + ERR_BENCHMARK_INVALID_OPERATION_COUNT, + ERR_BENCHMARK_START_NOT_CALLED, + ERR_BENCHMARK_CLOCK_PRECISION, + ERR_INVALID_ARG_TYPE + } +} = require('internal/errors'); + +const { + validateFunction, + validateString, + validateNumber, + validateArray, + validateBoolean, + validateStringArray, +} = require('internal/validators'); +const { validatePath } = require('internal/fs/utils'); + +class Benchmark { + + constructor(fn, benchmarkConfigs, options = {}) { + + validateFunction(fn); + + if (options.flags) + validateArray(options.flags); + + // Used to make sure a benchmark only start a timer once + this._started = false; + + // Indicate that the benchmark ended + this._ended = false; + + // Holds process.hrtime value + this._time = 0n; + + this.options = options; + + this.name = StringPrototypeSlice(process.mainModule.filename, process.mainModule.path.length + 1); + + // Execution arguments i.e. flags used to run the jobs + this.flags = process.env.NODE_BENCHMARK_FLAGS ? + process.env.NODE_BENCHMARK_FLAGS.split(/\s+/) : + []; + + // Parse job-specific configuration from the command line arguments + const argv = ArrayPrototypeSlice(process.argv, 2); + + const parsed_args = this._parseArgs(argv, benchmarkConfigs, options); + + this.options = parsed_args.cli; + this.extra_options = parsed_args.extra; + + if (options.flags) { + this.flags = ArrayPrototypeConcat(this.flags, options.flags); + } + + // The configuration list as a queue of jobs + this.queue = this._queue(this.options); + + // The configuration of the current job, head of the queue + this.config = this.queue[0]; + process.nextTick(() => { + if (ObjectHasOwn(process.env, 'NODE_RUN_BENCHMARK_FN')) { + fn(this.config); + } else { + // _run will use fork() to create a new process for each configuration + // combination. + this._run(); + } + }); + } + + /** + * Runs the http benchmaker specified in `options.benchmarker` + * @param {object} options + * @param {string} options.path + * @param {number} options.connections + * @param {number} options.duration + * @param {string} options.benchmarker + * @param {string} options.scheme + * @param {Function} cb + */ + http(options, cb) { + const http_options = { ...options }; + http_options.benchmarker = http_options.benchmarker || + this.config.benchmarker || + this.extra_options.benchmarker || + http_benchmarkers.default_http_benchmarker; + http_benchmarkers.run( + http_options, (error, code, used_benchmarker, result, elapsed) => { + if (cb) { + cb(code); + } + if (error) { + throw error; + } + this.config.benchmarker = used_benchmarker; + this.report(result, elapsed); + } + ); + } + + _run() { + + if (process.send) { + process.send({ + type: 'config', + name: this.name, + queueLength: this.queue.length, + }); + } + + const recursive = (queueIndex) => { + const config = this.queue[queueIndex]; + + // Set NODE_RUN_BENCHMARK_FN to indicate that the child shouldn't + // construct a configuration queue, but just execute the benchmark + // function. + const childEnv = { + ...process.env, + NODE_RUN_BENCHMARK_FN: '' + }; + + const childArgs = []; + + for (const key of ObjectKeys(config)) { + ArrayPrototypePush(childArgs, `${key}=${config[key]}`); + } + + for (const key of ObjectKeys(this.extra_options)) { + ArrayPrototypePush(childArgs, `${key}=${this.extra_options[key]}`); + } + + const child = child_process.fork(process.mainModule.filename, childArgs, { + env: childEnv, + execArgv: ArrayPrototypeConcat(this.flags, process.execArgv) + }); + + child.on('message', Benchmark.SendResult); + child.on('close', (code) => { + if (code) { + process.exit(code); + } + + if (queueIndex + 1 < this.queue.length) { + recursive(queueIndex + 1); + } + + }); + }; + + recursive(0); + } + + _parseArgs(argv, config, options) { + const cliOptions = {}; + // Check for the test mode first. + const testIndex = ArrayPrototypeIndexOf(argv, '--test'); + if (testIndex !== -1) { + for (const key of ObjectKeys(config)) { + const rawValue = config[key]; + let value = ArrayIsArray(rawValue) ? rawValue[0] : rawValue; + // Set numbers to one by default to reduce the runtime. + if (typeof value === 'number') { + if (key === 'dur' || key === 'duration') { + value = 0.05; + } else if (value > 1) { + value = 1; + } + } + cliOptions[key] = [value]; + } + // Override specific test options. + if (options.test) { + for (const key of ObjectKeys(options.test)) { + const value = options.test[key]; + cliOptions[key] = ArrayIsArray(value) ? value : [value]; + } + } + ArrayPrototypeSplice(argv, testIndex, 1); + } else { + // Accept single values instead of arrays. + for (const key of ObjectKeys(config)) { + const value = config[key]; + if (!ArrayIsArray(value)) + config[key] = [value]; + } + } + + const extraOptions = {}; + const validArgRE = /^(.+?)=([\s\S]*)$/; + // Parse configuration arguments + for (const arg of argv) { + + const match = arg.match(validArgRE); + + if (!match) { + throw new ERR_INVALID_ARG_TYPE(`bad argument: ${arg}`); + } + + const key = match[1]; + const value = match[2]; + + if (ObjectHasOwn(config, key)) { + if (!cliOptions[key]) + cliOptions[key] = []; + ArrayPrototypePush( + cliOptions[key], + // Infer the type from the config object and parse accordingly + typeof config[key][0] === 'number' ? +value : value + ); + } else { + extraOptions[key] = value; + } + } + return { cli: { ...config, ...cliOptions }, extra: extraOptions }; + } + + _queue(options) { + + const queue = []; + const keys = ObjectKeys(options); + + function recursive(keyIndex, prevConfig) { + const key = keys[keyIndex]; + const values = options[key]; + for (const value of values) { + try { + validateString(value, 'value'); + } catch (error) { + if (error.code !== 'ERR_INVALID_ARG_TYPE') { + throw error; + } + validateNumber(value, 'value'); + } + + if (typeof value !== typeof values[0]) { + // This is a requirement for being able to consistently and + // predictably parse CLI provided configuration values. + throw new ERR_INVALID_ARG_TYPE(`configuration "${key}" has mixed types`); + } + + const currConfig = { [key]: value, ...prevConfig }; + + if (keyIndex + 1 < keys.length) { + recursive(keyIndex + 1, currConfig); + } else { + ArrayPrototypePush(queue, currConfig); + } + } + } + + if (keys.length > 0) { + recursive(0, {}); + } else { + ArrayPrototypePush(queue, {}); + } + return queue; + } + + /** + * Start the benchmarker + */ + start() { + if (this._started) { + throw new ERR_BENCHMARK_START_CALLED_MORE_THAN_ONCE(); + } + this._started = true; + this._time = process.hrtime.bigint(); + } + + /** + * Stops the benchmarker + * @param {number} operations + */ + end(operations) { + // Get elapsed time now and do error checking later for accuracy. + const time = process.hrtime.bigint(); + + if (!this._started) { + throw new ERR_BENCHMARK_START_NOT_CALLED(); + } + if (this._ended) { + throw new ERR_BENCHMARK_END_CALLED_MORE_THAN_ONCE(); + } + if (typeof operations !== 'number') { + throw new ERR_BENCHMARK_NO_OPERATION_COUNT(); + } + if (!process.env.NODEJS_BENCHMARK_ZERO_ALLOWED && operations <= 0) { + throw new ERR_BENCHMARK_INVALID_OPERATION_COUNT(); + } + + this._ended = true; + + if (time === this._time) { + if (!process.env.NODEJS_BENCHMARK_ZERO_ALLOWED) + throw new ERR_BENCHMARK_CLOCK_PRECISION(); + // Avoid dividing by zero + this.report(operations && NumberMAX_VALUE, 0n); + return; + } + + const elapsed = time - this._time; + const rate = operations / (Number(elapsed) / 1e9); + this.report(rate, elapsed); + } + + report(rate, elapsed) { + Benchmark.SendResult({ + name: this.name, + conf: this.config, + rate, + time: this._nanoSecondsToString(elapsed), + type: 'report', + }); + } + + _nanoSecondsToString(bigint) { + const str = BigIntPrototypeToString(bigint); + const decimalPointIndex = str.length - 9; + if (decimalPointIndex <= 0) { + return `0.${StringPrototypeRepeat('0', -decimalPointIndex)}${str}`; + } + return `${StringPrototypeSlice(str, 0, decimalPointIndex)}.${StringPrototypeSlice(str, decimalPointIndex)}`; + } + + static SendResult(data) { + if (process.send) { + // If forked, report by process send + process.send(data, () => { + if (ObjectHasOwn(process.env, 'NODE_RUN_BENCHMARK_FN')) { + // If, for any reason, the process is unable to self close within + // a second after completing, forcefully close it. + setTimeout(() => { + process.exit(0); + }, 5000).unref(); + } + }); + } else { + // Otherwise report by stdout + process.stdout.write(Benchmark.FormatResult(data)); + } + } + + static FormatResult(data) { + // Construct configuration string, " A=a, B=b, ..." + let conf = ''; + for (const key of ObjectKeys(data.conf)) { + conf += ` ${key}=${JSONStringify(data.conf[key])}`; + } + let rate = NumberPrototypeToString(data.rate).split('.'); + rate[0] = rate[0].replace(/(\d)(?=(?:\d\d\d)+(?!\d))/g, '$1,'); + rate = (rate[1] ? rate.join('.') : rate[0]); + return `${data.name}${conf}: ${rate}\n`; + } +} + +module.exports = { + HTTP_BENCHMARK_PORT: http_benchmarkers.HTTP_BENCHMARK_PORT, + Benchmark, + /** + * Returns an instance of Benchmark + * @param {Function} fn + * @param {Object} config + * @param {{ + * flags: Array + * }} options + * @returns {Benchmark} + */ + createBenchmark(fn, config, options) { + return new Benchmark(fn, config, options); + }, + + /** + * Benchmarks an operation using + * `options.binary.new` and `options.binary.old` and + * compares the result. + * @param {object} options + * @param {number} options.runs + * @param {boolean} options.showProgress + * @param {Array} options.benchmarkFiles + * @param {object} options.binary + * @param {string} options.binary.old + * @param {string} options.binary.new + * @param {Array} options.set + */ + compareBenchmarks(options) { + const kStartOfQueue = 0; + + validateStringArray(options.benchmarkFiles); + validateBoolean(options.showProgress); + validatePath(options.binary.old); + validatePath(options.binary.new); + + options.runs = typeof options.runs !== 'number' ? 30 : 0; + + validateNumber(options.runs); + + for (const file of options.benchmarkFiles) { + validatePath(file); + } + + if (options.set) { + validateStringArray(options.set); + for (const variable of options.set) { + const tokens = variable.split('='); + if ( + tokens.length > 2 || + tokens < 2 || + /^\s+$/.test(tokens[1]) + ) { + throw new ERR_INVALID_ARG_TYPE( + `Expected ${variable} to only follow this format key=value` + ); + } + } + } + + const queue = BenchmarkProgress.QueueBenchmarkFiles( + { + binaries: ObjectKeys(options.binary), + benchmarkFiles: options.benchmarkFiles, + runs: options.runs, + } + ); + + let progress; + + if (options.showProgress) { + progress = new BenchmarkProgress(queue, options.benchmarkFiles); + progress.startQueue(kStartOfQueue); + } + + process.stdout.write('"binary","filename","configuration","rate","time"\n'); + + (function recursive(i) { + const job = queue[i]; + + const child = child_process.fork(path.resolve(process.mainModule.path, job.filename), options.set, { + execPath: options.binary[job.binary] + }); + + child.on('message', (data) => { + if (data.type === 'report') { + // Construct configuration string, " A=a, B=b, ..." + let conf = ''; + for (const key of ObjectKeys(data.conf)) { + conf += ` ${key}=${util.inspect(data.conf[key])}`; + } + conf = StringPrototypeSlice(conf, 1); + // Escape quotes (") for correct csv formatting + conf = conf.replace(/"/g, '""'); + + process.stdout.write( + `"${job.binary}","${job.filename}","${conf}",` + + `${data.rate},${data.time}\n` + ); + + if (options.showProgress) { + // One item in the subqueue has been completed. + progress.completeConfig(data); + } + } else if (options.showProgress && data.type === 'config') { + // The child has computed the configurations, ready to run subqueue. + progress.startSubqueue(data, i); + } + }); + + child.once('close', (code) => { + if (code) { + process.exit(code); + } + if (options.showProgress) { + progress.completeRun(job); + } + // If there are more benchmarks execute the next + if (i + 1 < queue.length) { + recursive(i + 1); + } + }); + })(kStartOfQueue); + } +}; diff --git a/benchmark/_benchmark_progress.js b/lib/internal/benchmark/_benchmark_progress.js similarity index 61% rename from benchmark/_benchmark_progress.js rename to lib/internal/benchmark/_benchmark_progress.js index 6c925f34e682..b9f95b4cc894 100644 --- a/benchmark/_benchmark_progress.js +++ b/lib/internal/benchmark/_benchmark_progress.js @@ -1,10 +1,44 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS + 'use strict'; +const { + String, + StringPrototypeRepeat, + MathMax, + MathCeil, + MathFloor +} = primordials; + +const { + setInterval, + clearInterval +} = require('timers'); + const readline = require('readline'); function pad(input, minLength, fill) { const result = String(input); - const padding = fill.repeat(Math.max(0, minLength - result.length)); + const padding = StringPrototypeRepeat(fill, MathMax(0, minLength - result.length)); return `${padding}${result}`; } @@ -15,9 +49,9 @@ function fraction(numerator, denominator) { } function getTime(diff) { - const time = Math.ceil(diff[0] + diff[1] / 1e9); - const hours = pad(Math.floor(time / 3600), 2, '0'); - const minutes = pad(Math.floor((time % 3600) / 60), 2, '0'); + const time = MathCeil(diff[0] + diff[1] / 1e9); + const hours = pad(MathFloor(time / 3600), 2, '0'); + const minutes = pad(MathFloor((time % 3600) / 60), 2, '0'); const seconds = pad((time % 3600) % 60, 2, '0'); return `${hours}:${minutes}:${seconds}`; } @@ -83,7 +117,7 @@ class BenchmarkProgress { // Calculate numbers for fractions. const runsPerFile = this.runsPerFile; - const completedFiles = Math.floor(completedRuns / runsPerFile); + const completedFiles = MathFloor(completedRuns / runsPerFile); const scheduledFiles = this.benchmarks.length; const completedRunsForFile = finished ? runsPerFile : completedRuns % runsPerFile; @@ -96,7 +130,7 @@ class BenchmarkProgress { runRate = completedConfig / scheduledConfig; } const completedRate = ((completedRuns + runRate) / scheduledRuns); - const percent = pad(Math.floor(completedRate * 100), 3, ' '); + const percent = pad(MathFloor(completedRate * 100), 3, ' '); const caption = finished ? 'Done\n' : this.currentFile; return `[${getTime(diff)}|% ${percent}| ` + @@ -114,6 +148,25 @@ class BenchmarkProgress { readline.cursorTo(process.stderr, 0); process.stderr.write(this.getProgress()); } + + /** + * Create queue from the benchmarks list such both node versions are tested + * `runs` amount of times each. + * Note: BenchmarkProgress relies on this order to estimate + * how much runs remaining for a file. All benchmarks generated from + * the same file must be run consecutively. + */ + static QueueBenchmarkFiles(options) { + const queue = []; + for (const filename of options.benchmarkFiles) { + for (let iter = 0; iter < options.runs; iter++) { + for (const binary of options.binaries) { + queue.push({ binary, filename, iter }); + } + } + } + return queue; + } } module.exports = BenchmarkProgress; diff --git a/lib/internal/benchmark/_http-benchmarkers.js b/lib/internal/benchmark/_http-benchmarkers.js new file mode 100644 index 000000000000..3bc64576f355 --- /dev/null +++ b/lib/internal/benchmark/_http-benchmarkers.js @@ -0,0 +1,279 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS + +'use strict'; + +const child_process = require('child_process'); + +const { + codes: { + ERR_BENCHMARK_HTTP_COULD_NOT_LOCATE_BENCHMARKER, + ERR_BENCHMARK_HTTP_BENCHMARKER_NOT_INSTALLED, + ERR_BENCHMARK_HTTP_BENCHMARKER_NOT_SUPPROTED, + ERR_BENCHMARK_HTTP_STRANGE_OUTPUT, + ERR_BENCHMARK_HTTP_UNKNOWN_OUTPUT, + } +} = require('internal/errors'); + +const { + Number, + JSONParse, + MathMax, + MathMin, + ArrayPrototypePush, + NumberIsFinite, + RegExpPrototypeExec +} = primordials; + +const { + validateFunction, + validateString, + validateNumber +} = require('internal/validators'); + +const requirementsURL = + 'https://github.com/nodejs/node/blob/HEAD/benchmark/writing-and-running-benchmarks.md#http-benchmark-requirements'; + +// The port used by servers and wrk +exports.HTTP_BENCHMARK_PORT = Number(process.env.PORT) || 12346; + +class LoadBalancerBenchmarker { + constructor(name, executable, type) { + this.name = name; + this.executable = executable; + this.type = type; + const result = child_process.spawnSync(this.executable, ['-h']); + this.present = !(result.error && result.error.code === 'ENOENT'); + } +} + +class AutocannonBenchmarker extends LoadBalancerBenchmarker { + constructor() { + super('autocannon', process.platform === 'win32' ? 'autocannon.cmd' : 'autocannon'); + } + + create(options) { + const args = [ + '-d', options.duration, + '-c', options.connections, + '-j', + '-n', + ]; + for (const field in options.headers) { + ArrayPrototypePush(args, '-H', `${field}=${options.headers[field]}`); + } + const scheme = options.scheme || 'http'; + ArrayPrototypePush(args, `${scheme}://${options.host}:${options.port}${options.path}`); + const child = child_process.spawn(this.executable, args); + return child; + } + + processResults(output) { + let result; + try { + result = JSONParse(output); + } catch { + return undefined; + } + if (!result || !result.requests || !result.requests.average) { + return undefined; + } + return result.requests.average; + } +} + +class WrkBenchmarker extends LoadBalancerBenchmarker { + constructor() { + super('wrk', 'wrk'); + } + + create(options) { + const duration = typeof options.duration === 'number' ? + MathMax(options.duration, 1) : + options.duration; + const scheme = options.scheme || 'http'; + const args = [ + '-d', duration, + '-c', options.connections, + '-t', MathMin(options.connections, require('os').cpus().length || 8), + `${scheme}://${options.host}:${options.port}${options.path}`, + ]; + for (const field in options.headers) { + ArrayPrototypePush(args, '-H', `${field}: ${options.headers[field]}`); + } + const child = child_process.spawn(this.executable, args); + return child; + } + + processResults(output) { + const throughputRe = /Requests\/sec:[ \t]+([0-9.]+)/; + const match = output.match(throughputRe); + const throughput = match && +match[1]; + if (!NumberIsFinite(throughput)) { + return undefined; + } + return throughput; + } +} + +/** + * HTTP/2 Benchmarker + */ +class H2LoadBenchmarker extends LoadBalancerBenchmarker { + constructor() { + super('h2load', 'h2load'); + } + + create(options) { + const args = []; + + if (typeof options.requests === 'number') + ArrayPrototypePush(args, '-n', options.requests); + if (typeof options.clients === 'number') + ArrayPrototypePush(args, '-c', options.clients); + if (typeof options.threads === 'number') + ArrayPrototypePush(args, '-t', options.threads); + if (typeof options.maxConcurrentStreams === 'number') + ArrayPrototypePush(args, '-m', options.maxConcurrentStreams); + if (typeof options.initialWindowSize === 'number') + ArrayPrototypePush(args, '-w', options.initialWindowSize); + if (typeof options.sessionInitialWindowSize === 'number') + ArrayPrototypePush(args, '-W', options.sessionInitialWindowSize); + if (typeof options.rate === 'number') + ArrayPrototypePush(args, '-r', options.rate); + if (typeof options.ratePeriod === 'number') + ArrayPrototypePush(args, `--rate-period=${options.ratePeriod}`); + if (typeof options.duration === 'number') + ArrayPrototypePush(args, '-T', options.duration); + if (typeof options.timeout === 'number') + ArrayPrototypePush(args, '-N', options.timeout); + if (typeof options.headerTableSize === 'number') + ArrayPrototypePush(args, `--header-table-size=${options.headerTableSize}`); + if (typeof options.encoderHeaderTableSize === 'number') { + ArrayPrototypePush(args, `--encoder-header-table-size=${options.encoderHeaderTableSize}`); + } + const scheme = options.scheme || 'http'; + ArrayPrototypePush(args, `${scheme}://${options.host}:${options.port}${options.path}`); + const child = child_process.spawn(this.executable, args); + return child; + } + + processResults(output) { + const rex = /(\d+\.\d+) req\/s/; + return RegExpPrototypeExec(rex, output)[1]; + } +} + +const http_benchmarkers = [ + new WrkBenchmarker(), + new AutocannonBenchmarker(), + new H2LoadBenchmarker(), +]; + +const benchmarkers = {}; + +http_benchmarkers.forEach((benchmarker) => { + benchmarkers[benchmarker.name] = benchmarker; + if (!exports.default_http_benchmarker && benchmarker.present) { + exports.default_http_benchmarker = benchmarker.name; + } +}); + +exports.run = function(options, callback) { + + validateFunction(callback); + + if (options.path) + validateString(options.path); + if (options.connections) + validateNumber(options.connections); + if (options.benchmarker) + validateString(options.benchmarker); + + options = { + port: exports.PORT, + path: '/', + connections: 100, + duration: 5, + host: '127.0.0.1', + benchmarker: exports.default_http_benchmarker, + ...options + }; + let err; + if (!options.benchmarker) { + err = new ERR_BENCHMARK_HTTP_COULD_NOT_LOCATE_BENCHMARKER( + 'Could not locate required http benchmarker. See ' + + `${requirementsURL} for further instructions.` + ); + callback(err); + return; + } + const benchmarker = benchmarkers[options.benchmarker]; + if (!benchmarker) { + err = new ERR_BENCHMARK_HTTP_BENCHMARKER_NOT_SUPPROTED( + `Requested benchmarker '${options.benchmarker}' ` + + 'is not supported' + ); + callback(err); + return; + } + if (!benchmarker.present) { + err = new ERR_BENCHMARK_HTTP_BENCHMARKER_NOT_INSTALLED( + `Requested benchmarker '${options.benchmarker}' ` + + 'is not installed' + ); + callback(err); + return; + } + + const benchmarker_start = process.hrtime.bigint(); + + const child = benchmarker.create(options); + + child.stderr.pipe(process.stderr); + + let stdout = ''; + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk) => stdout += chunk); + + child.once('close', (code) => { + const benchmark_end = process.hrtime.bigint(); + if (code) { + let error_message = `${options.benchmarker} failed with ${code}.`; + if (stdout !== '') { + error_message += ` Output: ${stdout}`; + } + callback(new ERR_BENCHMARK_HTTP_UNKNOWN_OUTPUT(error_message), code); + return; + } + + const result = benchmarker.processResults(stdout); + if (result === undefined) { + callback(new ERR_BENCHMARK_HTTP_STRANGE_OUTPUT( + `${options.benchmarker} produced strange output: ${stdout}`), code); + return; + } + + const elapsed = benchmark_end - benchmarker_start; + callback(null, code, options.benchmarker, result, elapsed); + }); + +}; diff --git a/lib/internal/bootstrap/loaders.js b/lib/internal/bootstrap/loaders.js index a3c3d375f709..3f5fae2bf968 100644 --- a/lib/internal/bootstrap/loaders.js +++ b/lib/internal/bootstrap/loaders.js @@ -124,6 +124,7 @@ const legacyWrapperList = new SafeSet([ // Modules that can only be imported via the node: scheme. const schemelessBlockList = new SafeSet([ 'test', + 'benchmark', ]); // Set up process.binding() and process._linkedBinding(). diff --git a/lib/internal/errors.js b/lib/internal/errors.js index f6a8ce6549c5..2fb78dd857c5 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -947,6 +947,17 @@ E('ERR_ASSERT_SNAPSHOT_NOT_SUPPORTED', 'Snapshot is not supported in this context ', TypeError); E('ERR_ASYNC_CALLBACK', '%s must be a function', TypeError); E('ERR_ASYNC_TYPE', 'Invalid name for async "type": %s', TypeError); +E('ERR_BENCHMARK_CLOCK_PRECISION', 'insufficient clock precision for short benchmark', Error); +E('ERR_BENCHMARK_END_CALLED_MORE_THAN_ONCE', 'Called end multiple times', Error); +E('ERR_BENCHMARK_HTTP_BENCHMARKER_NOT_INSTALLED', '%s', Error); +E('ERR_BENCHMARK_HTTP_BENCHMARKER_NOT_SUPPROTED', '%s', Error); +E('ERR_BENCHMARK_HTTP_COULD_NOT_LOCATE_BENCHMARKER', '%s', Error); +E('ERR_BENCHMARK_HTTP_STRANGE_OUTPUT', '%s', Error); +E('ERR_BENCHMARK_HTTP_UNKNOWN_OUTPUT', '%s', Error); +E('ERR_BENCHMARK_INVALID_OPERATION_COUNT', 'called end() with operation count <= 0', RangeError); +E('ERR_BENCHMARK_NO_OPERATION_COUNT', 'called end() without specifying operation count', Error); +E('ERR_BENCHMARK_START_CALLED_MORE_THAN_ONCE', 'Called start more than once in a single benchmark', Error); +E('ERR_BENCHMARK_START_NOT_CALLED', 'called end without start', Error); E('ERR_BROTLI_INVALID_PARAM', '%s is not a valid Brotli parameter', RangeError); E('ERR_BUFFER_OUT_OF_BOUNDS', // Using a default argument here is important so the argument is not counted diff --git a/test/common/benchmark.js b/test/benchmark/benchmark.js similarity index 98% rename from test/common/benchmark.js rename to test/benchmark/benchmark.js index 88c918766fd7..995919a47d36 100644 --- a/test/common/benchmark.js +++ b/test/benchmark/benchmark.js @@ -1,5 +1,7 @@ 'use strict'; +require('../common'); + const assert = require('assert'); const fork = require('child_process').fork; const path = require('path'); diff --git a/test/benchmark/test-benchmark-assert.js b/test/benchmark/test-benchmark-assert.js index 5ec2319c28a1..10a8292c5b1a 100644 --- a/test/benchmark/test-benchmark-assert.js +++ b/test/benchmark/test-benchmark-assert.js @@ -5,6 +5,6 @@ require('../common'); // Minimal test for assert benchmarks. This makes sure the benchmarks aren't // completely broken but nothing more than that. -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('assert'); diff --git a/test/benchmark/test-benchmark-async-hooks.js b/test/benchmark/test-benchmark-async-hooks.js index c9ea2c1e86db..72e328408db4 100644 --- a/test/benchmark/test-benchmark-async-hooks.js +++ b/test/benchmark/test-benchmark-async-hooks.js @@ -8,6 +8,6 @@ if (!common.hasCrypto) if (!common.enoughTestMem) common.skip('Insufficient memory for async_hooks benchmark test'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('async_hooks'); diff --git a/test/benchmark/test-benchmark-buffer.js b/test/benchmark/test-benchmark-buffer.js index af93842b0b92..1c486cd492f8 100644 --- a/test/benchmark/test-benchmark-buffer.js +++ b/test/benchmark/test-benchmark-buffer.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('buffers', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-child-process.js b/test/benchmark/test-benchmark-child-process.js index 043620de12f1..0d8d87b6dcc9 100644 --- a/test/benchmark/test-benchmark-child-process.js +++ b/test/benchmark/test-benchmark-child-process.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('child_process', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-cluster.js b/test/benchmark/test-benchmark-cluster.js index b24aced5d58d..7463a0a3ef12 100644 --- a/test/benchmark/test-benchmark-cluster.js +++ b/test/benchmark/test-benchmark-cluster.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('cluster'); diff --git a/test/benchmark/test-benchmark-crypto.js b/test/benchmark/test-benchmark-crypto.js index 7f6988acf234..bd87cc060113 100644 --- a/test/benchmark/test-benchmark-crypto.js +++ b/test/benchmark/test-benchmark-crypto.js @@ -8,6 +8,6 @@ if (!common.hasCrypto) if (common.hasFipsCrypto) common.skip('some benchmarks are FIPS-incompatible'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('crypto', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-dgram.js b/test/benchmark/test-benchmark-dgram.js index ceafdd77a2aa..de76dde5ae62 100644 --- a/test/benchmark/test-benchmark-dgram.js +++ b/test/benchmark/test-benchmark-dgram.js @@ -2,7 +2,7 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); // Dgram benchmarks use hardcoded ports. Thus, this test can not be run in // parallel with tests that choose random ports. diff --git a/test/benchmark/test-benchmark-dns.js b/test/benchmark/test-benchmark-dns.js index 331a4c8ff0d8..ac56719be6b4 100644 --- a/test/benchmark/test-benchmark-dns.js +++ b/test/benchmark/test-benchmark-dns.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('dns', { ...process.env, NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-domain.js b/test/benchmark/test-benchmark-domain.js index 5ebbfc5ea3b4..d423e192622c 100644 --- a/test/benchmark/test-benchmark-domain.js +++ b/test/benchmark/test-benchmark-domain.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('domain'); diff --git a/test/benchmark/test-benchmark-es.js b/test/benchmark/test-benchmark-es.js index 6886b3ce9257..faf62eaacc02 100644 --- a/test/benchmark/test-benchmark-es.js +++ b/test/benchmark/test-benchmark-es.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('es', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-esm.js b/test/benchmark/test-benchmark-esm.js index cd10ff9cb160..c45f0e93882d 100644 --- a/test/benchmark/test-benchmark-esm.js +++ b/test/benchmark/test-benchmark-esm.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('esm', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-events.js b/test/benchmark/test-benchmark-events.js index 53de4897dd75..27ca9bbe8822 100644 --- a/test/benchmark/test-benchmark-events.js +++ b/test/benchmark/test-benchmark-events.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('events', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-fs.js b/test/benchmark/test-benchmark-fs.js index 3ef6be2b7eba..14a3e64079be 100644 --- a/test/benchmark/test-benchmark-fs.js +++ b/test/benchmark/test-benchmark-fs.js @@ -1,7 +1,7 @@ 'use strict'; require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); const tmpdir = require('../common/tmpdir'); tmpdir.refresh(); diff --git a/test/benchmark/test-benchmark-http.js b/test/benchmark/test-benchmark-http.js index a3d92c7e987f..e13254759c12 100644 --- a/test/benchmark/test-benchmark-http.js +++ b/test/benchmark/test-benchmark-http.js @@ -9,6 +9,6 @@ if (!common.enoughTestMem) // rather than parallel to make sure it does not conflict with tests that choose // random available ports. -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('http', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-http2.js b/test/benchmark/test-benchmark-http2.js index 25dd771076e7..fc271a9129b9 100644 --- a/test/benchmark/test-benchmark-http2.js +++ b/test/benchmark/test-benchmark-http2.js @@ -11,6 +11,6 @@ if (!common.enoughTestMem) // rather than parallel to make sure it does not conflict with tests that choose // random available ports. -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('http2', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-misc.js b/test/benchmark/test-benchmark-misc.js index 30707bfaf736..ad221c5bda0d 100644 --- a/test/benchmark/test-benchmark-misc.js +++ b/test/benchmark/test-benchmark-misc.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('misc', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-module.js b/test/benchmark/test-benchmark-module.js index da8e108d53e1..b5a1458e8e47 100644 --- a/test/benchmark/test-benchmark-module.js +++ b/test/benchmark/test-benchmark-module.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('module'); diff --git a/test/benchmark/test-benchmark-napi.js b/test/benchmark/test-benchmark-napi.js index 5c6a8aa01187..6d6a924abaa3 100644 --- a/test/benchmark/test-benchmark-napi.js +++ b/test/benchmark/test-benchmark-napi.js @@ -13,6 +13,6 @@ if (!common.isMainThread) { if (process.features.debug) { common.skip('benchmark does not work with debug build yet'); } -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('napi', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-net.js b/test/benchmark/test-benchmark-net.js index df8ea8011693..a03ff65aa6e0 100644 --- a/test/benchmark/test-benchmark-net.js +++ b/test/benchmark/test-benchmark-net.js @@ -6,6 +6,6 @@ require('../common'); // rather than parallel to make sure it does not conflict with tests that choose // random available ports. -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('net', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-os.js b/test/benchmark/test-benchmark-os.js index dbedd7f582d9..c1cf4f60281a 100644 --- a/test/benchmark/test-benchmark-os.js +++ b/test/benchmark/test-benchmark-os.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('os'); diff --git a/test/benchmark/test-benchmark-path.js b/test/benchmark/test-benchmark-path.js index 3bca4f2a11bd..accea13d823f 100644 --- a/test/benchmark/test-benchmark-path.js +++ b/test/benchmark/test-benchmark-path.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('path', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-policy.js b/test/benchmark/test-benchmark-policy.js index 7eb0992b1f1e..af34d019230a 100644 --- a/test/benchmark/test-benchmark-policy.js +++ b/test/benchmark/test-benchmark-policy.js @@ -2,7 +2,7 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('policy', [ 'n=1', diff --git a/test/benchmark/test-benchmark-process.js b/test/benchmark/test-benchmark-process.js index c6687f302341..2083c3705f3e 100644 --- a/test/benchmark/test-benchmark-process.js +++ b/test/benchmark/test-benchmark-process.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('process', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-querystring.js b/test/benchmark/test-benchmark-querystring.js index 6fee9bb39143..c7c5bcc1485a 100644 --- a/test/benchmark/test-benchmark-querystring.js +++ b/test/benchmark/test-benchmark-querystring.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('querystring', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-streams.js b/test/benchmark/test-benchmark-streams.js index 68c8478a7398..41497cc8450d 100644 --- a/test/benchmark/test-benchmark-streams.js +++ b/test/benchmark/test-benchmark-streams.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('streams', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-string_decoder.js b/test/benchmark/test-benchmark-string_decoder.js index 721529e5ae64..537c97c23d9a 100644 --- a/test/benchmark/test-benchmark-string_decoder.js +++ b/test/benchmark/test-benchmark-string_decoder.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('string_decoder'); diff --git a/test/benchmark/test-benchmark-timers.js b/test/benchmark/test-benchmark-timers.js index db4927ab32ea..0ab0319b4666 100644 --- a/test/benchmark/test-benchmark-timers.js +++ b/test/benchmark/test-benchmark-timers.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('timers', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-tls.js b/test/benchmark/test-benchmark-tls.js index c9a87c15770d..22269caecf46 100644 --- a/test/benchmark/test-benchmark-tls.js +++ b/test/benchmark/test-benchmark-tls.js @@ -12,6 +12,6 @@ if (!common.enoughTestMem) // rather than parallel to make sure it does not conflict with tests that choose // random available ports. -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('tls', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-url.js b/test/benchmark/test-benchmark-url.js index 664e7c4d8dc8..a767733b92f9 100644 --- a/test/benchmark/test-benchmark-url.js +++ b/test/benchmark/test-benchmark-url.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('url', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-util.js b/test/benchmark/test-benchmark-util.js index d0c16c623268..488ff795ed4c 100644 --- a/test/benchmark/test-benchmark-util.js +++ b/test/benchmark/test-benchmark-util.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('util', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-v8.js b/test/benchmark/test-benchmark-v8.js index efeaac8328c7..2193e9e51e3f 100644 --- a/test/benchmark/test-benchmark-v8.js +++ b/test/benchmark/test-benchmark-v8.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('v8', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-vm.js b/test/benchmark/test-benchmark-vm.js index e9c4e3f1389f..853ebcaa1ef2 100644 --- a/test/benchmark/test-benchmark-vm.js +++ b/test/benchmark/test-benchmark-vm.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('vm', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-worker.js b/test/benchmark/test-benchmark-worker.js index a4319d4face6..1804a8248fc0 100644 --- a/test/benchmark/test-benchmark-worker.js +++ b/test/benchmark/test-benchmark-worker.js @@ -9,6 +9,6 @@ if (!common.enoughTestMem) // this should be in sequential rather than parallel to make sure // it does not conflict with tests that choose random available ports. -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('worker', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/test/benchmark/test-benchmark-zlib.js b/test/benchmark/test-benchmark-zlib.js index e3c4723aa972..87e7b5535503 100644 --- a/test/benchmark/test-benchmark-zlib.js +++ b/test/benchmark/test-benchmark-zlib.js @@ -2,6 +2,6 @@ require('../common'); -const runBenchmark = require('../common/benchmark'); +const runBenchmark = require('./benchmark'); runBenchmark('zlib', { NODEJS_BENCHMARK_ZERO_ALLOWED: 1 }); diff --git a/tsconfig.json b/tsconfig.json index 1594a3363393..4a47e6a6e8d3 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -46,6 +46,7 @@ "assert": ["./lib/assert.js"], "assert/strict": ["./lib/assert/strict.js"], "async_hooks": ["./lib/async_hooks.js"], + "benchmark": ["./lib/benchmark.js"], "buffer": ["./lib/buffer.js"], "child_process": ["./lib/child_process.js"], "cluster": ["./lib/cluster.js"],