fs: Change default WriteStream config, increase perf
This increases fs.WriteStream throughput dramatically by removing the "higher default water marks" for fs.WriteStream. Also includes a benchmark. Current performance is significantly higher than v0.8 for strings at all tested levels except size=1. Buffer performance is still lackluster. Further improvement in the stream.Writable base class is required, but this is a start.
This commit is contained in:
parent
9299168f2a
commit
8476aefc8e
96
benchmark/fs-write-stream-throughput.js
Normal file
96
benchmark/fs-write-stream-throughput.js
Normal file
@ -0,0 +1,96 @@
|
|||||||
|
|
||||||
|
// If there are no args, then this is the root. Run all the benchmarks!
|
||||||
|
if (!process.argv[2])
|
||||||
|
parent();
|
||||||
|
else
|
||||||
|
runTest(+process.argv[2], +process.argv[3], process.argv[4]);
|
||||||
|
|
||||||
|
function parent() {
|
||||||
|
var types = [ 'string', 'buffer' ];
|
||||||
|
var durs = [ 1, 5 ];
|
||||||
|
var sizes = [ 1, 10, 100, 2048, 10240 ];
|
||||||
|
var queue = [];
|
||||||
|
types.forEach(function(t) {
|
||||||
|
durs.forEach(function(d) {
|
||||||
|
sizes.forEach(function(s) {
|
||||||
|
queue.push([__filename, d, s, t]);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
var spawn = require('child_process').spawn;
|
||||||
|
var node = process.execPath;
|
||||||
|
|
||||||
|
run();
|
||||||
|
|
||||||
|
function run() {
|
||||||
|
var args = queue.shift();
|
||||||
|
if (!args)
|
||||||
|
return;
|
||||||
|
var child = spawn(node, args, { stdio: 'inherit' });
|
||||||
|
child.on('close', function(code, signal) {
|
||||||
|
if (code)
|
||||||
|
throw new Error('Benchmark failed: ' + args.slice(1));
|
||||||
|
run();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function runTest(dur, size, type) {
|
||||||
|
if (type !== 'string')
|
||||||
|
type = 'buffer';
|
||||||
|
switch (type) {
|
||||||
|
case 'string':
|
||||||
|
var chunk = new Array(size + 1).join('a');
|
||||||
|
break;
|
||||||
|
case 'buffer':
|
||||||
|
var chunk = new Buffer(size);
|
||||||
|
chunk.fill('a');
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
var writes = 0;
|
||||||
|
var fs = require('fs');
|
||||||
|
try { fs.unlinkSync('write_stream_throughput'); } catch (e) {}
|
||||||
|
|
||||||
|
var start
|
||||||
|
var end;
|
||||||
|
function done() {
|
||||||
|
var time = end[0] + end[1]/1E9;
|
||||||
|
var written = fs.statSync('write_stream_throughput').size / 1024;
|
||||||
|
var rate = (written / time).toFixed(2);
|
||||||
|
console.log('fs_write_stream_dur_%d_size_%d_type_%s: %d',
|
||||||
|
dur, size, type, rate);
|
||||||
|
|
||||||
|
try { fs.unlinkSync('write_stream_throughput'); } catch (e) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
var f = require('fs').createWriteStream('write_stream_throughput');
|
||||||
|
f.on('drain', write);
|
||||||
|
f.on('open', write);
|
||||||
|
f.on('close', done);
|
||||||
|
|
||||||
|
// streams2 fs.WriteStreams will let you send a lot of writes into the
|
||||||
|
// buffer before returning false, so capture the *actual* end time when
|
||||||
|
// all the bytes have been written to the disk, indicated by 'finish'
|
||||||
|
f.on('finish', function() {
|
||||||
|
end = process.hrtime(start);
|
||||||
|
});
|
||||||
|
|
||||||
|
var ending = false;
|
||||||
|
function write() {
|
||||||
|
// don't try to write after we end, even if a 'drain' event comes.
|
||||||
|
// v0.8 streams are so sloppy!
|
||||||
|
if (ending)
|
||||||
|
return;
|
||||||
|
|
||||||
|
start = start || process.hrtime();
|
||||||
|
while (false !== f.write(chunk));
|
||||||
|
end = process.hrtime(start);
|
||||||
|
|
||||||
|
if (end[0] >= dur) {
|
||||||
|
ending = true;
|
||||||
|
f.end();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -1559,11 +1559,7 @@ function WriteStream(path, options) {
|
|||||||
if (!(this instanceof WriteStream))
|
if (!(this instanceof WriteStream))
|
||||||
return new WriteStream(path, options);
|
return new WriteStream(path, options);
|
||||||
|
|
||||||
// a little bit bigger buffer and water marks by default
|
options = options || {};
|
||||||
options = util._extend({
|
|
||||||
lowWaterMark: 16 * 1024,
|
|
||||||
highWaterMark: 64 * 1024
|
|
||||||
}, options || {});
|
|
||||||
|
|
||||||
Writable.call(this, options);
|
Writable.call(this, options);
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user