1berry/nodejs
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bench: Use environ to run compares more than once

Browse Source 
This will run the benchmarks the number of times specified by NODE_BENCH_RUNS,
to attempt to reduce variability.

If the number of runs is high enough, it'll also throw out the top and bottom
quartiles, since that's where the outliers will be.

It's not very fancy statistics-fu, but it's better than nothing.

Also, linted this file.  It had tabs in it.  TABS!
This commit is contained in:
isaacs 2013-03-06 12:32:59 -08:00
parent 9208c89058
commit db5d58e842
1 changed files with 63 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
benchmark/compare.js
View File

@ -1,4 +1,6 @@
var usage = 'node benchmark/compare.js <node-binary1> <node-binary2> [--html] [--red|-r] [--green|-g]';
var usage = 'node benchmark/compare.js ' +
'<node-binary1> <node-binary2> ' +
'[--html] [--red|-r] [--green|-g]';
var show = 'both';
var nodes = [];
@ -46,15 +48,26 @@ if (nodes.length !== 2)
var spawn = require('child_process').spawn;
var results = {};
var n = 2;
var n = 1;
run();
var RUNS = +process.env.NODE_BENCH_RUNS || 1;
var r = RUNS;
function run() {
if (n === 0)
return compare();
// Flip back and forth between the two binaries.
if (n === 1) {
n--;
} else {
r--;
if (r === 0)
return compare();
else
n++;
}
if (n === -1)
return compare();
var node = nodes[n];
console.error('running %s', node);
@ -85,11 +98,12 @@ function run() {
var s = line.split(':');
var num = +s.pop();
if (!num && num !== 0)
return
return;
line = s.join(':');
var res = results[line] = results[line] || {};
res[node] = num;
res[node] = res[node] || [];
res[node].push(num);
});
run();
@ -107,8 +121,8 @@ function compare() {
Object.keys(results).map(function(bench) {
var res = results[bench];
var n0 = res[nodes[0]];
var n1 = res[nodes[1]];
var n0 = avg(res[nodes[0]]);
var n1 = avg(res[nodes[1]]);
var pct = ((n0 - n1) / n1 * 100).toFixed(2);
@ -119,8 +133,8 @@ function compare() {
if (show === 'green' && !g || show === 'red' && !r)
return;
var r0 = util.format('%s%s: %d%s', g, nodes[0], n0, reset);
var r1 = util.format('%s%s: %d%s', r, nodes[1], n1, reset);
var r0 = util.format('%s%s: %d%s', g, nodes[0], n0, g ? reset : '');
var r1 = util.format('%s%s: %d%s', r, nodes[1], n1, r ? reset : '');
var pct = c + pct + '%' + reset;
var l = util.format('%s: %s %s', bench, r0, r1);
maxLen = Math.max(l.length + pct.length, maxLen);
@ -136,3 +150,14 @@ function compare() {
});
console.log(end);
}
function avg(list) {
if (list.length >= 3) {
list = list.sort();
var q = Math.floor(list.length / 4) || 1;
list = list.slice(q, -q);
}
return (list.reduce(function(a, b) {
return a + b;
}, 0) / list.length).toPrecision(5);
}