benchmark: add progress indicator to compare.js
* Print the progress bar and the current benchmark to stderr when stderr is TTY and stdout is not. * Allow cli arguments without values via setting.boolArgs * Add --no-progress option PR-URL: https://github.com/nodejs/node/pull/10823 Fixes: https://github.com/nodejs/node/issues/8659 Reviewed-By: Andreas Madsen <amwebdk@gmail.com>
This commit is contained in:
parent
ca3d131bd4
commit
60d77bd514
120
benchmark/_benchmark_progress.js
Normal file
120
benchmark/_benchmark_progress.js
Normal file
@ -0,0 +1,120 @@
|
||||
'use strict';
|
||||
|
||||
const readline = require('readline');
|
||||
|
||||
function pad(input, minLength, fill) {
|
||||
var result = input + '';
|
||||
return fill.repeat(Math.max(0, minLength - result.length)) + result;
|
||||
}
|
||||
|
||||
function fraction(numerator, denominator) {
|
||||
const fdenominator = denominator + '';
|
||||
const fnumerator = pad(numerator, fdenominator.length, ' ');
|
||||
return `${fnumerator}/${fdenominator}`;
|
||||
}
|
||||
|
||||
function getTime(diff) {
|
||||
const time = Math.ceil(diff[0] + diff[1] / 1e9);
|
||||
const seconds = pad(time % 60, 2, '0');
|
||||
const minutes = pad(Math.floor(time / 60) % (60 * 60), 2, '0');
|
||||
const hours = pad(Math.floor(time / (60 * 60)), 2, '0');
|
||||
return `${hours}:${minutes}:${seconds}`;
|
||||
}
|
||||
|
||||
// A run is an item in the job queue: { binary, filename, iter }
|
||||
// A config is an item in the subqueue: { binary, filename, iter, configs }
|
||||
class BenchmarkProgress {
|
||||
constructor(queue, benchmarks) {
|
||||
this.queue = queue; // Scheduled runs.
|
||||
this.benchmarks = benchmarks; // Filenames of scheduled benchmarks.
|
||||
this.completedRuns = 0; // Number of completed runs.
|
||||
this.scheduledRuns = queue.length; // Number of scheduled runs.
|
||||
// Time when starting to run benchmarks.
|
||||
this.startTime = process.hrtime();
|
||||
// Number of times each file will be run (roughly).
|
||||
this.runsPerFile = queue.length / benchmarks.length;
|
||||
this.currentFile = ''; // Filename of current benchmark.
|
||||
this.currentFileConfig; // Configurations for current file
|
||||
// Number of configurations already run for the current file.
|
||||
this.completedConfig = 0;
|
||||
// Total number of configurations for the current file
|
||||
this.scheduledConfig = 0;
|
||||
this.interval = 0; // result of setInterval for updating the elapsed time
|
||||
}
|
||||
|
||||
startQueue(index) {
|
||||
this.kStartOfQueue = index;
|
||||
this.currentFile = this.queue[index].filename;
|
||||
this.interval = setInterval(() => {
|
||||
if (this.completedRuns === this.scheduledRuns) {
|
||||
clearInterval(this.interval);
|
||||
} else {
|
||||
this.updateProgress();
|
||||
}
|
||||
}, 1000);
|
||||
}
|
||||
|
||||
startSubqueue(data, index) {
|
||||
// This subqueue is generated by a new benchmark
|
||||
if (data.name !== this.currentFile || index === this.kStartOfQueue) {
|
||||
this.currentFile = data.name;
|
||||
this.scheduledConfig = data.queueLength;
|
||||
}
|
||||
this.completedConfig = 0;
|
||||
this.updateProgress();
|
||||
}
|
||||
|
||||
completeConfig(data) {
|
||||
this.completedConfig++;
|
||||
this.updateProgress();
|
||||
}
|
||||
|
||||
completeRun(job) {
|
||||
this.completedRuns++;
|
||||
this.updateProgress();
|
||||
}
|
||||
|
||||
getProgress() {
|
||||
// Get time as soon as possible.
|
||||
const diff = process.hrtime(this.startTime);
|
||||
|
||||
const completedRuns = this.completedRuns;
|
||||
const scheduledRuns = this.scheduledRuns;
|
||||
const finished = completedRuns === scheduledRuns;
|
||||
|
||||
// Calculate numbers for fractions.
|
||||
const runsPerFile = this.runsPerFile;
|
||||
const completedFiles = Math.floor(completedRuns / runsPerFile);
|
||||
const scheduledFiles = this.benchmarks.length;
|
||||
const completedRunsForFile = finished ? runsPerFile :
|
||||
completedRuns % runsPerFile;
|
||||
const completedConfig = this.completedConfig;
|
||||
const scheduledConfig = this.scheduledConfig;
|
||||
|
||||
// Calculate the percentage.
|
||||
let runRate = 0; // Rate of current incomplete run.
|
||||
if (completedConfig !== scheduledConfig) {
|
||||
runRate = completedConfig / scheduledConfig;
|
||||
}
|
||||
const completedRate = ((completedRuns + runRate) / scheduledRuns);
|
||||
const percent = pad(Math.floor(completedRate * 100), 3, ' ');
|
||||
|
||||
const caption = finished ? 'Done\n' : this.currentFile;
|
||||
return `[${getTime(diff)}|% ${percent}` +
|
||||
`| ${fraction(completedFiles, scheduledFiles)} files ` +
|
||||
`| ${fraction(completedRunsForFile, runsPerFile)} runs ` +
|
||||
`| ${fraction(completedConfig, scheduledConfig)} configs]` +
|
||||
`: ${caption}`;
|
||||
}
|
||||
|
||||
updateProgress(finished) {
|
||||
if (!process.stderr.isTTY || process.stdout.isTTY) {
|
||||
return;
|
||||
}
|
||||
readline.clearLine(process.stderr);
|
||||
readline.cursorTo(process.stderr, 0);
|
||||
process.stderr.write(this.getProgress());
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = BenchmarkProgress;
|
@ -45,13 +45,13 @@ function CLI(usage, settings) {
|
||||
currentOptional = arg.slice(1);
|
||||
}
|
||||
|
||||
// Default the value to true
|
||||
if (!settings.arrayArgs.includes(currentOptional)) {
|
||||
if (settings.boolArgs && settings.boolArgs.includes(currentOptional)) {
|
||||
this.optional[currentOptional] = true;
|
||||
mode = 'both';
|
||||
} else {
|
||||
// expect the next value to be option related (either -- or the value)
|
||||
mode = 'option';
|
||||
}
|
||||
|
||||
// expect the next value to be option related (either -- or the value)
|
||||
mode = 'option';
|
||||
} else if (mode === 'option') {
|
||||
// Optional arguments value
|
||||
|
||||
|
@ -128,6 +128,14 @@ Benchmark.prototype.http = function(options, cb) {
|
||||
|
||||
Benchmark.prototype._run = function() {
|
||||
const self = this;
|
||||
// If forked, report to the parent.
|
||||
if (process.send) {
|
||||
process.send({
|
||||
type: 'config',
|
||||
name: this.name,
|
||||
queueLength: this.queue.length
|
||||
});
|
||||
}
|
||||
|
||||
(function recursive(queueIndex) {
|
||||
const config = self.queue[queueIndex];
|
||||
@ -217,7 +225,8 @@ Benchmark.prototype.report = function(rate, elapsed) {
|
||||
name: this.name,
|
||||
conf: this.config,
|
||||
rate: rate,
|
||||
time: elapsed[0] + elapsed[1] / 1e9
|
||||
time: elapsed[0] + elapsed[1] / 1e9,
|
||||
type: 'report'
|
||||
});
|
||||
};
|
||||
|
||||
|
@ -3,6 +3,7 @@
|
||||
const fork = require('child_process').fork;
|
||||
const path = require('path');
|
||||
const CLI = require('./_cli.js');
|
||||
const BenchmarkProgress = require('./_benchmark_progress.js');
|
||||
|
||||
//
|
||||
// Parse arguments
|
||||
@ -13,13 +14,15 @@ const cli = CLI(`usage: ./node compare.js [options] [--] <category> ...
|
||||
The output is formatted as csv, which can be processed using for
|
||||
example 'compare.R'.
|
||||
|
||||
--new ./new-node-binary new node binary (required)
|
||||
--old ./old-node-binary old node binary (required)
|
||||
--runs 30 number of samples
|
||||
--filter pattern string to filter benchmark scripts
|
||||
--set variable=value set benchmark variable (can be repeated)
|
||||
--new ./new-node-binary new node binary (required)
|
||||
--old ./old-node-binary old node binary (required)
|
||||
--runs 30 number of samples
|
||||
--filter pattern string to filter benchmark scripts
|
||||
--set variable=value set benchmark variable (can be repeated)
|
||||
--no-progress don't show benchmark progress indicator
|
||||
`, {
|
||||
arrayArgs: ['set']
|
||||
arrayArgs: ['set'],
|
||||
boolArgs: ['no-progress']
|
||||
});
|
||||
|
||||
if (!cli.optional.new || !cli.optional.old) {
|
||||
@ -39,6 +42,9 @@ if (benchmarks.length === 0) {
|
||||
|
||||
// Create queue from the benchmarks list such both node versions are tested
|
||||
// `runs` amount of times each.
|
||||
// Note: BenchmarkProgress relies on this order to estimate
|
||||
// how much runs remaining for a file. All benchmarks generated from
|
||||
// the same file must be run consecutively.
|
||||
const queue = [];
|
||||
for (const filename of benchmarks) {
|
||||
for (let iter = 0; iter < runs; iter++) {
|
||||
@ -47,10 +53,20 @@ for (const filename of benchmarks) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// queue.length = binary.length * runs * benchmarks.length
|
||||
|
||||
// Print csv header
|
||||
console.log('"binary", "filename", "configuration", "rate", "time"');
|
||||
|
||||
const kStartOfQueue = 0;
|
||||
|
||||
const showProgress = !cli.optional['no-progress'];
|
||||
let progress;
|
||||
if (showProgress) {
|
||||
progress = new BenchmarkProgress(queue, benchmarks);
|
||||
progress.startQueue(kStartOfQueue);
|
||||
}
|
||||
|
||||
(function recursive(i) {
|
||||
const job = queue[i];
|
||||
|
||||
@ -59,18 +75,26 @@ console.log('"binary", "filename", "configuration", "rate", "time"');
|
||||
});
|
||||
|
||||
child.on('message', function(data) {
|
||||
// Construct configuration string, " A=a, B=b, ..."
|
||||
let conf = '';
|
||||
for (const key of Object.keys(data.conf)) {
|
||||
conf += ' ' + key + '=' + JSON.stringify(data.conf[key]);
|
||||
if (data.type === 'report') {
|
||||
// Construct configuration string, " A=a, B=b, ..."
|
||||
let conf = '';
|
||||
for (const key of Object.keys(data.conf)) {
|
||||
conf += ' ' + key + '=' + JSON.stringify(data.conf[key]);
|
||||
}
|
||||
conf = conf.slice(1);
|
||||
// Escape quotes (") for correct csv formatting
|
||||
conf = conf.replace(/"/g, '""');
|
||||
|
||||
console.log(`"${job.binary}", "${job.filename}", "${conf}", ` +
|
||||
`${data.rate}, ${data.time}`);
|
||||
if (showProgress) {
|
||||
// One item in the subqueue has been completed.
|
||||
progress.completeConfig(data);
|
||||
}
|
||||
} else if (showProgress && data.type === 'config') {
|
||||
// The child has computed the configurations, ready to run subqueue.
|
||||
progress.startSubqueue(data, i);
|
||||
}
|
||||
conf = conf.slice(1);
|
||||
|
||||
// Escape quotes (") for correct csv formatting
|
||||
conf = conf.replace(/"/g, '""');
|
||||
|
||||
console.log(`"${job.binary}", "${job.filename}", "${conf}", ` +
|
||||
`${data.rate}, ${data.time}`);
|
||||
});
|
||||
|
||||
child.once('close', function(code) {
|
||||
@ -78,10 +102,13 @@ console.log('"binary", "filename", "configuration", "rate", "time"');
|
||||
process.exit(code);
|
||||
return;
|
||||
}
|
||||
if (showProgress) {
|
||||
progress.completeRun(job);
|
||||
}
|
||||
|
||||
// If there are more benchmarks execute the next
|
||||
if (i + 1 < queue.length) {
|
||||
recursive(i + 1);
|
||||
}
|
||||
});
|
||||
})(0);
|
||||
})(kStartOfQueue);
|
||||
|
@ -44,6 +44,9 @@ if (format === 'csv') {
|
||||
}
|
||||
|
||||
child.on('message', function(data) {
|
||||
if (data.type !== 'report') {
|
||||
return;
|
||||
}
|
||||
// Construct configuration string, " A=a, B=b, ..."
|
||||
let conf = '';
|
||||
for (const key of Object.keys(data.conf)) {
|
||||
|
@ -42,6 +42,10 @@ function csvEncodeValue(value) {
|
||||
const child = fork(path.resolve(__dirname, filepath), cli.optional.set);
|
||||
|
||||
child.on('message', function(data) {
|
||||
if (data.type !== 'report') {
|
||||
return;
|
||||
}
|
||||
|
||||
// print csv header
|
||||
if (printHeader) {
|
||||
const confHeader = Object.keys(data.conf)
|
||||
|
Loading…
x
Reference in New Issue
Block a user