From 314c6b306038526209d284cf9fdbe978d2a634ec Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 4 Oct 2012 17:43:15 -0700 Subject: [PATCH 01/72] Don't allow invalid encodings in StringDecoder class --- lib/string_decoder.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/string_decoder.js b/lib/string_decoder.js index 31d4b247021..6b1e30895a4 100644 --- a/lib/string_decoder.js +++ b/lib/string_decoder.js @@ -19,8 +19,15 @@ // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. +function assertEncoding(encoding) { + if (encoding && !Buffer.isEncoding(encoding)) { + throw new Error('Unknown encoding: ' + encoding); + } +} + var StringDecoder = exports.StringDecoder = function(encoding) { this.encoding = (encoding || 'utf8').toLowerCase().replace(/[-_]/, ''); + assertEncoding(encoding); switch (this.encoding) { case 'utf8': // CESU-8 represents each of Surrogate Pair by 3-bytes From 372cb32dc4975983184e7c83d69dbbb5302e6d8b Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 2 Oct 2012 16:10:58 -0700 Subject: [PATCH 02/72] module: Support cycles in native module requires --- src/node.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node.js b/src/node.js index 7f762cd4130..d861fe5b257 100644 --- a/src/node.js +++ b/src/node.js @@ -701,8 +701,8 @@ var nativeModule = new NativeModule(id); - nativeModule.compile(); nativeModule.cache(); + nativeModule.compile(); return nativeModule.exports; }; From 17834ed28cb9f376f9505a15273cff4317045feb Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 2 Oct 2012 16:11:36 -0700 Subject: [PATCH 03/72] Add 'stream' as a native module in repl --- lib/repl.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/repl.js b/lib/repl.js index 55df83f9705..3438a45cc13 100644 --- a/lib/repl.js +++ b/lib/repl.js @@ -69,8 +69,8 @@ module.paths = require('module')._nodeModulePaths(module.filename); exports.writer = util.inspect; exports._builtinLibs = ['assert', 'buffer', 'child_process', 'cluster', - 'crypto', 'dgram', 'dns', 'events', 'fs', 'http', 'https', 'net', - 'os', 'path', 'punycode', 'querystring', 'readline', 'repl', + 'crypto', 'dgram', 'dns', 'events', 'fs', 'http', 'https', 'net', 'os', + 'path', 'punycode', 'querystring', 'readline', 'repl', 'stream', 'string_decoder', 'tls', 'tty', 'url', 'util', 'vm', 'zlib']; From 420e07c5777bdb2e493147d296abfc102f725015 Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 2 Oct 2012 15:44:50 -0700 Subject: [PATCH 04/72] streams2: The new stream base classes --- lib/_stream_duplex.js | 81 +++++++ lib/_stream_passthrough.js | 39 ++++ lib/_stream_readable.js | 429 +++++++++++++++++++++++++++++++++++++ lib/_stream_transform.js | 123 +++++++++++ lib/_stream_writable.js | 135 ++++++++++++ lib/stream.js | 21 +- node.gyp | 5 + 7 files changed, 829 insertions(+), 4 deletions(-) create mode 100644 lib/_stream_duplex.js create mode 100644 lib/_stream_passthrough.js create mode 100644 lib/_stream_readable.js create mode 100644 lib/_stream_transform.js create mode 100644 lib/_stream_writable.js diff --git a/lib/_stream_duplex.js b/lib/_stream_duplex.js new file mode 100644 index 00000000000..0256b0f2f22 --- /dev/null +++ b/lib/_stream_duplex.js @@ -0,0 +1,81 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +// a duplex stream is just a stream that is both readable and writable. +// Since JS doesn't have multiple prototypal inheritance, this class +// prototypally inherits from Readable, and then parasitically from +// Writable. + +module.exports = Duplex; +var util = require('util'); +var Readable = require('_stream_readable'); +var Writable = require('_stream_writable'); + +util.inherits(Duplex, Readable); + +Object.keys(Writable.prototype).forEach(function(method) { + if (!Duplex.prototype[method]) + Duplex.prototype[method] = Writable.prototype[method]; +}); + +function Duplex(options) { + Readable.call(this, options); + Writable.call(this, options); + + this.allowHalfOpen = true; + if (options && options.allowHalfOpen === false) + this.allowHalfOpen = false; + + this.once('finish', onfinish); + this.once('end', onend); +} + +// the no-half-open enforcers. +function onfinish() { + // if we allow half-open state, or if the readable side ended, + // then we're ok. + if (this.allowHalfOpen || this._readableState.ended) + return; + + // mark that we're done. + this._readableState.ended = true; + + // tell the user + if (this._readableState.length === 0) + this.emit('end'); + else + this.emit('readable'); +} + +function onend() { + // if we allow half-open state, or if the writable side ended, + // then we're ok. + if (this.allowHalfOpen || this._writableState.ended) + return; + + // just in case the user is about to call write() again. + this.write = function() { + return false; + }; + + // no more data can be written. + this.end(); +} diff --git a/lib/_stream_passthrough.js b/lib/_stream_passthrough.js new file mode 100644 index 00000000000..dd6390fc6ee --- /dev/null +++ b/lib/_stream_passthrough.js @@ -0,0 +1,39 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +// a passthrough stream. +// basically just the most minimal sort of Transform stream. +// Every written chunk gets output as-is. + +module.exports = PassThrough; + +var Transform = require('_stream_transform'); +var util = require('util'); +util.inherits(PassThrough, Transform); + +function PassThrough(options) { + Transform.call(this, options); +} + +PassThrough.prototype._transform = function(chunk, output, cb) { + output(chunk); + cb(); +}; diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js new file mode 100644 index 00000000000..b71c22aecf4 --- /dev/null +++ b/lib/_stream_readable.js @@ -0,0 +1,429 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +module.exports = Readable; + +var Stream = require('stream'); +var util = require('util'); +var assert = require('assert'); + +util.inherits(Readable, Stream); + +function ReadableState(options, stream) { + options = options || {}; + + this.bufferSize = options.bufferSize || 16 * 1024; + assert(typeof this.bufferSize === 'number'); + // cast to an int + this.bufferSize = ~~this.bufferSize; + + this.lowWaterMark = options.lowWaterMark || 1024; + this.buffer = []; + this.length = 0; + this.pipes = []; + this.flowing = false; + this.ended = false; + this.stream = stream; + this.reading = false; + + // whenever we return null, then we set a flag to say + // that we're awaiting a 'readable' event emission. + this.needReadable = false; +} + +function Readable(options) { + this._readableState = new ReadableState(options, this); + Stream.apply(this); +} + +// you can override either this method, or _read(n, cb) below. +Readable.prototype.read = function(n) { + var state = this._readableState; + + if (state.length === 0 && state.ended) { + process.nextTick(this.emit.bind(this, 'end')); + return null; + } + + if (isNaN(n) || n <= 0) + n = state.length + + // XXX: controversial. + // don't have that much. return null, unless we've ended. + // However, if the low water mark is lower than the number of bytes, + // then we still need to return what we have, or else it won't kick + // off another _read() call. For example, + // lwm=5 + // len=9 + // read(10) + // We don't have that many bytes, so it'd be tempting to return null, + // but then it won't ever cause _read to be called, so in that case, + // we just return what we have, and let the programmer deal with it. + if (n > state.length) { + if (!state.ended && state.length < state.lowWaterMark) { + state.needReadable = true; + n = 0; + } else + n = state.length; + } + + var ret = n > 0 ? fromList(n, state.buffer, state.length) : null; + + if (ret === null || ret.length === 0) + state.needReadable = true; + + state.length -= n; + + if (!state.ended && + state.length < state.lowWaterMark && + !state.reading) { + state.reading = true; + // call internal read method + this._read(state.bufferSize, function onread(er, chunk) { + state.reading = false; + if (er) + return this.emit('error', er); + + if (!chunk || !chunk.length) { + state.ended = true; + // if we've ended and we have some data left, then emit + // 'readable' now to make sure it gets picked up. + if (state.length > 0) + this.emit('readable'); + return; + } + + state.length += chunk.length; + state.buffer.push(chunk); + if (state.length < state.lowWaterMark) + this._read(state.bufferSize, onread.bind(this)); + + // now we have something to call this.read() to get. + if (state.needReadable) { + state.needReadable = false; + this.emit('readable'); + } + }.bind(this)); + } + + return ret; +}; + +// abstract method. to be overridden in specific implementation classes. +// call cb(er, data) where data is <= n in length. +// for virtual (non-string, non-buffer) streams, "length" is somewhat +// arbitrary, and perhaps not very meaningful. +Readable.prototype._read = function(n, cb) { + process.nextTick(cb.bind(this, new Error('not implemented'))); +}; + +Readable.prototype.pipe = function(dest, pipeOpts) { + var src = this; + var state = this._readableState; + if (!pipeOpts) + pipeOpts = {}; + state.pipes.push(dest); + + if ((!pipeOpts || pipeOpts.end !== false) && + dest !== process.stdout && + dest !== process.stderr) { + src.once('end', onend); + dest.on('unpipe', function(readable) { + if (readable === src) + src.removeListener('end', onend); + }); + } + + function onend() { + dest.end(); + } + + dest.emit('pipe', src); + + // start the flow. + if (!state.flowing) + process.nextTick(flow.bind(null, src, pipeOpts)); + + return dest; +}; + +function flow(src, pipeOpts) { + var state = src._readableState; + var chunk; + var dest; + var needDrain = 0; + + function ondrain() { + needDrain--; + if (needDrain === 0) + flow(src, pipeOpts); + } + + while (state.pipes.length && + null !== (chunk = src.read(pipeOpts.chunkSize))) { + state.pipes.forEach(function(dest, i, list) { + var written = dest.write(chunk); + if (false === written) { + needDrain++; + dest.once('drain', ondrain); + } + }); + src.emit('data', chunk); + + // if anyone needs a drain, then we have to wait for that. + if (needDrain > 0) + return; + } + + // if every destination was unpiped, either before entering this + // function, or in the while loop, then stop flowing. + // + // NB: This is a pretty rare edge case. + if (state.pipes.length === 0) { + state.flowing = false; + + // if there were data event listeners added, then switch to old mode. + if (this.listeners('data').length) + emitDataEvents(this); + return; + } + + // at this point, no one needed a drain, so we just ran out of data + // on the next readable event, start it over again. + src.once('readable', flow.bind(null, src, pipeOpts)); +} + +Readable.prototype.unpipe = function(dest) { + var state = this._readableState; + if (!dest) { + // remove all of them. + state.pipes.forEach(function(dest, i, list) { + dest.emit('unpipe', this); + }, this); + state.pipes.length = 0; + } else { + var i = state.pipes.indexOf(dest); + if (i !== -1) { + dest.emit('unpipe', this); + state.pipes.splice(i, 1); + } + } + return this; +}; + +// kludge for on('data', fn) consumers. Sad. +// This is *not* part of the new readable stream interface. +// It is an ugly unfortunate mess of history. +Readable.prototype.on = function(ev, fn) { + // https://github.com/isaacs/readable-stream/issues/16 + // if we're already flowing, then no need to set up data events. + if (ev === 'data' && !this._readableState.flowing) + emitDataEvents(this); + + return Stream.prototype.on.call(this, ev, fn); +}; +Readable.prototype.addListener = Readable.prototype.on; + +// pause() and resume() are remnants of the legacy readable stream API +// If the user uses them, then switch into old mode. +Readable.prototype.resume = function() { + emitDataEvents(this); + return this.resume(); +}; + +Readable.prototype.pause = function() { + emitDataEvents(this); + return this.pause(); +}; + +function emitDataEvents(stream) { + var state = stream._readableState; + + if (state.flowing) { + // https://github.com/isaacs/readable-stream/issues/16 + throw new Error('Cannot switch to old mode now.'); + } + + var paused = false; + var readable = false; + + // convert to an old-style stream. + stream.readable = true; + stream.pipe = Stream.prototype.pipe; + stream.on = stream.addEventListener = Stream.prototype.on; + + stream.on('readable', function() { + readable = true; + var c; + while (!paused && (null !== (c = stream.read()))) + stream.emit('data', c); + + if (c === null) { + readable = false; + stream._readableState.needReadable = true; + } + }); + + stream.pause = function() { + paused = true; + }; + + stream.resume = function() { + paused = false; + if (readable) + stream.emit('readable'); + }; + + // now make it start, just in case it hadn't already. + process.nextTick(function() { + stream.emit('readable'); + }); +} + +// wrap an old-style stream as the async data source. +// This is *not* part of the readable stream interface. +// It is an ugly unfortunate mess of history. +Readable.prototype.wrap = function(stream) { + var state = this._readableState; + var paused = false; + + stream.on('end', function() { + state.ended = true; + if (state.length === 0) + this.emit('end'); + }.bind(this)); + + stream.on('data', function(chunk) { + state.buffer.push(chunk); + state.length += chunk.length; + this.emit('readable'); + + // if not consumed, then pause the stream. + if (state.length > state.lowWaterMark && !paused) { + paused = true; + stream.pause(); + } + }.bind(this)); + + // proxy all the other methods. + // important when wrapping filters and duplexes. + for (var i in stream) { + if (typeof stream[i] === 'function' && + typeof this[i] === 'undefined') { + this[i] = function(method) { return function() { + return stream[method].apply(stream, arguments); + }}(i); + } + } + + // proxy certain important events. + var events = ['error', 'close', 'destroy', 'pause', 'resume']; + events.forEach(function(ev) { + stream.on(ev, this.emit.bind(this, ev)); + }.bind(this)); + + // consume some bytes. if not all is consumed, then + // pause the underlying stream. + this.read = function(n) { + if (state.length === 0) { + state.needReadable = true; + return null; + } + + if (isNaN(n) || n <= 0) + n = state.length; + + if (n > state.length) { + if (!state.ended) { + state.needReadable = true; + return null; + } else + n = state.length; + } + + var ret = fromList(n, state.buffer, state.length); + state.length -= n; + + if (state.length < state.lowWaterMark && paused) { + stream.resume(); + paused = false; + } + + if (state.length === 0 && state.ended) + process.nextTick(this.emit.bind(this, 'end')); + + return ret; + }; +}; + + + +// exposed for testing purposes only. +Readable._fromList = fromList; + +// Pluck off n bytes from an array of buffers. +// Length is the combined lengths of all the buffers in the list. +// If there's no data, then +function fromList(n, list, length) { + var ret; + + // nothing in the list, definitely empty. + if (list.length === 0) { + return null; + } + + if (length === 0) { + ret = null; + } else if (!n || n >= length) { + // read it all, truncate the array. + ret = Buffer.concat(list, length); + list.length = 0; + } else { + // read just some of it. + if (n < list[0].length) { + // just take a part of the first list item. + var buf = list[0]; + ret = buf.slice(0, n); + list[0] = buf.slice(n); + } else if (n === list[0].length) { + // first list is a perfect match + ret = list.shift(); + } else { + // complex case. + // we have enough to cover it, but it spans past the first buffer. + ret = new Buffer(n); + var c = 0; + for (var i = 0, l = list.length; i < l && c < n; i++) { + var buf = list[0]; + var cpy = Math.min(n - c, buf.length); + buf.copy(ret, c, 0, cpy); + if (cpy < buf.length) { + list[0] = buf.slice(cpy); + } else { + list.shift(); + } + c += cpy; + } + } + } + + return ret; +} diff --git a/lib/_stream_transform.js b/lib/_stream_transform.js new file mode 100644 index 00000000000..79d40cffabd --- /dev/null +++ b/lib/_stream_transform.js @@ -0,0 +1,123 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +// a transform stream is a readable/writable stream where you do +// something with the data. Sometimes it's called a "filter", +// but that's not a great name for it, since that implies a thing where +// some bits pass through, and others are simply ignored. (That would +// be a valid example of a transform, of course.) +// +// While the output is causally related to the input, it's not a +// necessarily symmetric or synchronous transformation. For example, +// a zlib stream might take multiple plain-text writes(), and then +// emit a single compressed chunk some time in the future. + +module.exports = Transform; + +var Duplex = require('_stream_duplex'); +var util = require('util'); +util.inherits(Transform, Duplex); + +function Transform(options) { + Duplex.call(this, options); + + // bind output so that it can be passed around as a regular function. + this._output = this._output.bind(this); + + // when the writable side finishes, then flush out anything remaining. + this.once('finish', function() { + if ('function' === typeof this._flush) + this._flush(this._output, done.bind(this)); + else + done.call(this); + }); +} + +// This is the part where you do stuff! +// override this function in implementation classes. +// 'chunk' is an input chunk. +// +// Call `output(newChunk)` to pass along transformed output +// to the readable side. You may call 'output' zero or more times. +// +// Call `cb(err)` when you are done with this chunk. If you pass +// an error, then that'll put the hurt on the whole operation. If you +// never call cb(), then you'll never get another chunk. +Transform.prototype._transform = function(chunk, output, cb) { + throw new Error('not implemented'); +}; + + +Transform.prototype._write = function(chunk, cb) { + this._transform(chunk, this._output, cb); +}; + +Transform.prototype._read = function(n, cb) { + var ws = this._writableState; + var rs = this._readableState; + + // basically a no-op, since the _transform will fill the + // _readableState.buffer and emit 'readable' for us, and set ended + // Usually, we want to just not call the cb, and set the reading + // flag to false, so that another _read will happen next time, + // but no state changes. + rs.reading = false; + + // however, if the writable side has ended, and its buffer is clear, + // then that means that the input has all been consumed, and no more + // will ever be provide. treat this as an EOF, and pass back 0 bytes. + if ((ws.ended || ws.ending) && ws.length === 0) + cb(); +}; + +Transform.prototype._output = function(chunk) { + if (!chunk || !chunk.length) + return; + + var state = this._readableState; + var len = state.length; + state.buffer.push(chunk); + state.length += chunk.length; + if (state.needReadable) { + state.needReadable = false; + this.emit('readable'); + } +}; + +function done(er) { + if (er) + return this.emit('error', er); + + // if there's nothing in the write buffer, then that means + // that nothing more will ever be provided + var ws = this._writableState; + var rs = this._readableState; + + rs.ended = true; + // we may have gotten a 'null' read before, and since there is + // no more data coming from the writable side, we need to emit + // now so that the consumer knows to pick up the tail bits. + if (rs.length && rs.needReadable) + this.emit('readable'); + else if (rs.length === 0) { + this.emit('end'); + } +} diff --git a/lib/_stream_writable.js b/lib/_stream_writable.js new file mode 100644 index 00000000000..e2343e63f9c --- /dev/null +++ b/lib/_stream_writable.js @@ -0,0 +1,135 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +// A bit simpler than readable streams. +// Implement an async ._write(chunk, cb), and it'll handle all +// the drain event emission and buffering. + +module.exports = Writable + +var util = require('util'); +var Stream = require('stream'); + +util.inherits(Writable, Stream); + +function WritableState(options) { + options = options || {}; + this.highWaterMark = options.highWaterMark || 16 * 1024; + this.lowWaterMark = options.lowWaterMark || 1024; + this.needDrain = false; + this.ended = false; + this.ending = false; + + // not an actual buffer we keep track of, but a measurement + // of how much we're waiting to get pushed to some underlying + // socket or file. + this.length = 0; + + this.writing = false; + this.buffer = []; +} + +function Writable(options) { + this._writableState = new WritableState(options); + + // legacy. + this.writable = true; + + Stream.call(this); +} + +// Override this method for sync streams +// override the _write(chunk, cb) method for async streams +Writable.prototype.write = function(chunk, encoding) { + var state = this._writableState; + if (state.ended) { + this.emit('error', new Error('write after end')); + return; + } + + if (typeof chunk === 'string' && encoding) + chunk = new Buffer(chunk, encoding); + + var ret = state.length >= state.highWaterMark; + if (ret === false) + state.needDrain = true; + + var l = chunk.length; + state.length += l; + + if (state.writing) { + state.buffer.push(chunk); + return ret; + } + + state.writing = true; + this._write(chunk, function writecb(er) { + state.writing = false; + if (er) { + this.emit('error', er); + return; + } + state.length -= l; + + if (state.length === 0 && (state.ended || state.ending)) { + // emit 'finish' at the very end. + this.emit('finish'); + return; + } + + // if there's something in the buffer waiting, then do that, too. + if (state.buffer.length) { + chunk = state.buffer.shift(); + l = chunk.length; + state.writing = true; + this._write(chunk, writecb.bind(this)); + } + + if (state.length < state.lowWaterMark && state.needDrain) { + // Must force callback to be called on nextTick, so that we don't + // emit 'drain' before the write() consumer gets the 'false' return + // value, and has a chance to attach a 'drain' listener. + process.nextTick(function() { + if (!state.needDrain) + return; + state.needDrain = false; + this.emit('drain'); + }.bind(this)); + } + + }.bind(this)); + + return ret; +}; + +Writable.prototype._write = function(chunk, cb) { + process.nextTick(cb.bind(this, new Error('not implemented'))); +}; + +Writable.prototype.end = function(chunk, encoding) { + var state = this._writableState; + state.ending = true; + if (chunk) + this.write(chunk, encoding); + else if (state.length === 0) + this.emit('finish'); + state.ended = true; +}; diff --git a/lib/stream.js b/lib/stream.js index 16e2e0e723f..481d7644e5e 100644 --- a/lib/stream.js +++ b/lib/stream.js @@ -19,16 +19,29 @@ // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. +module.exports = Stream; + var events = require('events'); var util = require('util'); +util.inherits(Stream, events.EventEmitter); +Stream.Readable = require('_stream_readable'); +Stream.Writable = require('_stream_writable'); +Stream.Duplex = require('_stream_duplex'); +Stream.Transform = require('_stream_transform'); +Stream.PassThrough = require('_stream_passthrough'); + +// Backwards-compat with node 0.4.x +Stream.Stream = Stream; + + + +// old-style streams. Note that the pipe method (the only relevant +// part of this class) is overridden in the Readable class. + function Stream() { events.EventEmitter.call(this); } -util.inherits(Stream, events.EventEmitter); -module.exports = Stream; -// Backwards-compat with node 0.4.x -Stream.Stream = Stream; Stream.prototype.pipe = function(dest, options) { var source = this; diff --git a/node.gyp b/node.gyp index f6651db83d7..14058eb960f 100644 --- a/node.gyp +++ b/node.gyp @@ -44,6 +44,11 @@ 'lib/readline.js', 'lib/repl.js', 'lib/stream.js', + 'lib/_stream_readable.js', + 'lib/_stream_writable.js', + 'lib/_stream_duplex.js', + 'lib/_stream_transform.js', + 'lib/_stream_passthrough.js', 'lib/string_decoder.js', 'lib/sys.js', 'lib/timers.js', From 639fbe28d11fa9d75aa1531930b1e73d7b3a1e77 Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 2 Oct 2012 16:28:02 -0700 Subject: [PATCH 05/72] streams2: Convert strings to buffers before passing to _write() --- lib/_stream_writable.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/_stream_writable.js b/lib/_stream_writable.js index e2343e63f9c..fd0cd115b4c 100644 --- a/lib/_stream_writable.js +++ b/lib/_stream_writable.js @@ -65,7 +65,7 @@ Writable.prototype.write = function(chunk, encoding) { return; } - if (typeof chunk === 'string' && encoding) + if (typeof chunk === 'string') chunk = new Buffer(chunk, encoding); var ret = state.length >= state.highWaterMark; From 51a52c43a2887b354af55a891e478a6b4b194a79 Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 3 Oct 2012 17:43:27 -0700 Subject: [PATCH 06/72] streams2: Set flowing=true when flowing --- lib/_stream_readable.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index b71c22aecf4..7b2d76c5348 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -159,8 +159,10 @@ Readable.prototype.pipe = function(dest, pipeOpts) { dest.emit('pipe', src); // start the flow. - if (!state.flowing) + if (!state.flowing) { + state.flowing = true; process.nextTick(flow.bind(null, src, pipeOpts)); + } return dest; }; @@ -168,7 +170,6 @@ Readable.prototype.pipe = function(dest, pipeOpts) { function flow(src, pipeOpts) { var state = src._readableState; var chunk; - var dest; var needDrain = 0; function ondrain() { From 9b5abe5bfe31988da1180e5a47f38b8fed03f99e Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 3 Oct 2012 16:52:14 -0700 Subject: [PATCH 07/72] streams2: setEncoding and abstract out endReadable --- lib/_stream_readable.js | 87 +++++++++++++++++++++++++++++++++-------- 1 file changed, 71 insertions(+), 16 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 7b2d76c5348..db76ab73b81 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -24,6 +24,7 @@ module.exports = Readable; var Stream = require('stream'); var util = require('util'); var assert = require('assert'); +var StringDecoder; util.inherits(Readable, Stream); @@ -41,12 +42,20 @@ function ReadableState(options, stream) { this.pipes = []; this.flowing = false; this.ended = false; + this.endEmitted = false; this.stream = stream; this.reading = false; // whenever we return null, then we set a flag to say // that we're awaiting a 'readable' event emission. this.needReadable = false; + + this.decoder = null; + if (options.encoding) { + if (!StringDecoder) + StringDecoder = require('string_decoder').StringDecoder; + this.decoder = new StringDecoder(options.encoding); + } } function Readable(options) { @@ -54,12 +63,19 @@ function Readable(options) { Stream.apply(this); } +// backwards compatibility. +Readable.prototype.setEncoding = function(enc) { + if (!StringDecoder) + StringDecoder = require('string_decoder').StringDecoder; + this._readableState.decoder = new StringDecoder(enc); +}; + // you can override either this method, or _read(n, cb) below. Readable.prototype.read = function(n) { var state = this._readableState; if (state.length === 0 && state.ended) { - process.nextTick(this.emit.bind(this, 'end')); + endReadable(this); return null; } @@ -85,7 +101,12 @@ Readable.prototype.read = function(n) { n = state.length; } - var ret = n > 0 ? fromList(n, state.buffer, state.length) : null; + + var ret; + if (n > 0) + ret = fromList(n, state.buffer, state.length, !!state.decoder); + else + ret = null; if (ret === null || ret.length === 0) state.needReadable = true; @@ -108,13 +129,26 @@ Readable.prototype.read = function(n) { // 'readable' now to make sure it gets picked up. if (state.length > 0) this.emit('readable'); + else + endReadable(this); return; } + if (state.decoder) + chunk = state.decoder.write(chunk); + state.length += chunk.length; state.buffer.push(chunk); - if (state.length < state.lowWaterMark) + + // if we haven't gotten enough to pass the lowWaterMark, + // and we haven't ended, then don't bother telling the user + // that it's time to read more data. Otherwise, that'll + // probably kick off another stream.read(), which can trigger + // another _read(n,cb) before this one returns! + if (state.length < state.lowWaterMark) { this._read(state.bufferSize, onread.bind(this)); + return; + } // now we have something to call this.read() to get. if (state.needReadable) { @@ -309,7 +343,7 @@ Readable.prototype.wrap = function(stream) { stream.on('end', function() { state.ended = true; if (state.length === 0) - this.emit('end'); + endReadable(this); }.bind(this)); stream.on('data', function(chunk) { @@ -360,7 +394,7 @@ Readable.prototype.wrap = function(stream) { n = state.length; } - var ret = fromList(n, state.buffer, state.length); + var ret = fromList(n, state.buffer, state.length, !!state.decoder); state.length -= n; if (state.length < state.lowWaterMark && paused) { @@ -369,7 +403,7 @@ Readable.prototype.wrap = function(stream) { } if (state.length === 0 && state.ended) - process.nextTick(this.emit.bind(this, 'end')); + endReadable(this); return ret; }; @@ -382,8 +416,7 @@ Readable._fromList = fromList; // Pluck off n bytes from an array of buffers. // Length is the combined lengths of all the buffers in the list. -// If there's no data, then -function fromList(n, list, length) { +function fromList(n, list, length, stringMode) { var ret; // nothing in the list, definitely empty. @@ -391,16 +424,20 @@ function fromList(n, list, length) { return null; } - if (length === 0) { + if (length === 0) ret = null; - } else if (!n || n >= length) { + else if (!n || n >= length) { // read it all, truncate the array. - ret = Buffer.concat(list, length); + if (stringMode) + ret = list.join(''); + else + ret = Buffer.concat(list, length); list.length = 0; } else { // read just some of it. if (n < list[0].length) { // just take a part of the first list item. + // slice is the same for buffers and strings. var buf = list[0]; ret = buf.slice(0, n); list[0] = buf.slice(n); @@ -410,17 +447,26 @@ function fromList(n, list, length) { } else { // complex case. // we have enough to cover it, but it spans past the first buffer. - ret = new Buffer(n); + if (stringMode) + ret = ''; + else + ret = new Buffer(n); + var c = 0; for (var i = 0, l = list.length; i < l && c < n; i++) { var buf = list[0]; var cpy = Math.min(n - c, buf.length); - buf.copy(ret, c, 0, cpy); - if (cpy < buf.length) { + + if (stringMode) + ret += buf.slice(0, cpy); + else + buf.copy(ret, c, 0, cpy); + + if (cpy < buf.length) list[0] = buf.slice(cpy); - } else { + else list.shift(); - } + c += cpy; } } @@ -428,3 +474,12 @@ function fromList(n, list, length) { return ret; } + +function endReadable(stream) { + var state = stream._readableState; + if (state.endEmitted) + return; + state.ended = true; + state.endEmitted = true; + process.nextTick(stream.emit.bind(stream, 'end')); +} From 3b59fd70f4ef26742cc66a28105d2be75590e4d2 Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 4 Oct 2012 13:26:16 -0700 Subject: [PATCH 08/72] streams2: Make Transform streams pull-style That is, the transform is triggered by a _read, not by a _write. This way, backpressure works properly. --- lib/_stream_readable.js | 1 + lib/_stream_transform.js | 118 +++++++++++++++++++++++++++++++++------ 2 files changed, 103 insertions(+), 16 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index db76ab73b81..e78dd4f0cb0 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -146,6 +146,7 @@ Readable.prototype.read = function(n) { // probably kick off another stream.read(), which can trigger // another _read(n,cb) before this one returns! if (state.length < state.lowWaterMark) { + state.reading = true; this._read(state.bufferSize, onread.bind(this)); return; } diff --git a/lib/_stream_transform.js b/lib/_stream_transform.js index 79d40cffabd..a3603f42a63 100644 --- a/lib/_stream_transform.js +++ b/lib/_stream_transform.js @@ -19,6 +19,7 @@ // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. + // a transform stream is a readable/writable stream where you do // something with the data. Sometimes it's called a "filter", // but that's not a great name for it, since that implies a thing where @@ -29,6 +30,39 @@ // necessarily symmetric or synchronous transformation. For example, // a zlib stream might take multiple plain-text writes(), and then // emit a single compressed chunk some time in the future. +// +// Here's how this works: +// +// The Transform stream has all the aspects of the readable and writable +// stream classes. When you write(chunk), that calls _write(chunk,cb) +// internally, and returns false if there's a lot of pending writes +// buffered up. When you call read(), that calls _read(n,cb) until +// there's enough pending readable data buffered up. +// +// In a transform stream, the written data is placed in a buffer. When +// _read(n,cb) is called, it transforms the queued up data, calling the +// buffered _write cb's as it consumes chunks. If consuming a single +// written chunk would result in multiple output chunks, then the first +// outputted bit calls the readcb, and subsequent chunks just go into +// the read buffer, and will cause it to emit 'readable' if necessary. +// +// This way, back-pressure is actually determined by the reading side, +// since _read has to be called to start processing a new chunk. However, +// a pathological inflate type of transform can cause excessive buffering +// here. For example, imagine a stream where every byte of input is +// interpreted as an integer from 0-255, and then results in that many +// bytes of output. Writing the 4 bytes {ff,ff,ff,ff} would result in +// 1kb of data being output. In this case, you could write a very small +// amount of input, and end up with a very large amount of output. In +// such a pathological inflating mechanism, there'd be no way to tell +// the system to stop doing the transform. A single 4MB write could +// cause the system to run out of memory. +// +// However, even in such a pathological case, only a single written chunk +// would be consumed, and then the rest would wait (un-transformed) until +// the results of the previous transformed chunk were consumed. Because +// the transform happens on-demand, it will only transform as much as is +// necessary to fill the readable buffer to the specified lowWaterMark. module.exports = Transform; @@ -36,12 +70,21 @@ var Duplex = require('_stream_duplex'); var util = require('util'); util.inherits(Transform, Duplex); +function TransformState() { + this.buffer = []; + this.transforming = false; + this.pendingReadCb = null; +} + function Transform(options) { Duplex.call(this, options); // bind output so that it can be passed around as a regular function. this._output = this._output.bind(this); + // the queue of _write chunks that are pending being transformed + this._transformState = new TransformState(); + // when the writable side finishes, then flush out anything remaining. this.once('finish', function() { if ('function' === typeof this._flush) @@ -65,33 +108,65 @@ Transform.prototype._transform = function(chunk, output, cb) { throw new Error('not implemented'); }; - Transform.prototype._write = function(chunk, cb) { - this._transform(chunk, this._output, cb); + var ts = this._transformState; + ts.buffer.push([chunk, cb]); + + // now we have something to transform, if we were waiting for it. + if (ts.pendingReadCb && !ts.transforming) { + var readcb = ts.pendingReadCb; + ts.pendingReadCb = null; + this._read(-1, readcb); + } }; -Transform.prototype._read = function(n, cb) { +Transform.prototype._read = function(n, readcb) { var ws = this._writableState; var rs = this._readableState; + var ts = this._transformState; - // basically a no-op, since the _transform will fill the - // _readableState.buffer and emit 'readable' for us, and set ended - // Usually, we want to just not call the cb, and set the reading - // flag to false, so that another _read will happen next time, - // but no state changes. - rs.reading = false; + if (ts.pendingReadCb) + throw new Error('_read while _read already in progress'); - // however, if the writable side has ended, and its buffer is clear, - // then that means that the input has all been consumed, and no more - // will ever be provide. treat this as an EOF, and pass back 0 bytes. - if ((ws.ended || ws.ending) && ws.length === 0) - cb(); + ts.pendingReadCb = readcb; + + // if there's nothing pending, then we just wait. + // if we're already transforming, then also just hold on a sec. + // we've already stashed the readcb, so we can come back later + // when we have something to transform + if (ts.buffer.length === 0 || ts.transforming) + return; + + // go ahead and transform that thing, now that someone wants it + var req = ts.buffer.shift(); + var chunk = req[0]; + var writecb = req[1]; + var output = this._output; + ts.transforming = true; + this._transform(chunk, output, function(er, data) { + ts.transforming = false; + if (data) + output(data); + writecb(er); + }.bind(this)); }; Transform.prototype._output = function(chunk) { if (!chunk || !chunk.length) return; + // if we've got a pending readcb, then just call that, + // and let Readable take care of it. If not, then we fill + // the readable buffer ourselves, and emit whatever's needed. + var ts = this._transformState; + var readcb = ts.pendingReadCb; + if (readcb) { + ts.pendingReadCb = null; + readcb(null, chunk); + return; + } + + // otherwise, it's up to us to fill the rs buffer. var state = this._readableState; var len = state.length; state.buffer.push(chunk); @@ -110,6 +185,18 @@ function done(er) { // that nothing more will ever be provided var ws = this._writableState; var rs = this._readableState; + var ts = this._transformState; + + if (ws.length) + throw new Error('calling transform done when ws.length != 0'); + + if (ts.transforming) + throw new Error('calling transform done when still transforming'); + + // if we were waiting on a read, let them know that it isn't coming. + var readcb = ts.pendingReadCb; + if (readcb) + return readcb(); rs.ended = true; // we may have gotten a 'null' read before, and since there is @@ -117,7 +204,6 @@ function done(er) { // now so that the consumer knows to pick up the tail bits. if (rs.length && rs.needReadable) this.emit('readable'); - else if (rs.length === 0) { + else if (rs.length === 0) this.emit('end'); - } } From caa853bb06c9ef1ba4203a42ad9e1072c3988a38 Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 4 Oct 2012 16:58:43 -0700 Subject: [PATCH 09/72] transform: Automatically read() on _write when read buffer is empty --- lib/_stream_transform.js | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/_stream_transform.js b/lib/_stream_transform.js index a3603f42a63..40917de7abf 100644 --- a/lib/_stream_transform.js +++ b/lib/_stream_transform.js @@ -110,14 +110,28 @@ Transform.prototype._transform = function(chunk, output, cb) { Transform.prototype._write = function(chunk, cb) { var ts = this._transformState; + var rs = this._readableState; ts.buffer.push([chunk, cb]); + // no need for auto-pull if already in the midst of one. + if (ts.transforming) + return; + // now we have something to transform, if we were waiting for it. - if (ts.pendingReadCb && !ts.transforming) { + // kick off a _read to pull it in. + if (ts.pendingReadCb) { var readcb = ts.pendingReadCb; ts.pendingReadCb = null; this._read(-1, readcb); } + + // if we weren't waiting for it, but nothing is queued up, then + // still kick off a transform, just so it's there when the user asks. + if (rs.length === 0) { + var ret = this.read(); + if (ret !== null) + return cb(new Error('invalid stream transform state')); + } }; Transform.prototype._read = function(n, readcb) { From 02f017d24f8cd93939bb4bd178b878d15cc5a08c Mon Sep 17 00:00:00 2001 From: isaacs Date: Fri, 5 Oct 2012 07:43:34 -0700 Subject: [PATCH 10/72] streams2: Allow 0 as a lowWaterMark value --- lib/_stream_readable.js | 11 ++++++----- lib/_stream_writable.js | 7 +++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index e78dd4f0cb0..ea944fcb44a 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -36,7 +36,8 @@ function ReadableState(options, stream) { // cast to an int this.bufferSize = ~~this.bufferSize; - this.lowWaterMark = options.lowWaterMark || 1024; + this.lowWaterMark = options.hasOwnProperty('lowWaterMark') ? + options.lowWaterMark : 1024; this.buffer = []; this.length = 0; this.pipes = []; @@ -94,7 +95,7 @@ Readable.prototype.read = function(n) { // but then it won't ever cause _read to be called, so in that case, // we just return what we have, and let the programmer deal with it. if (n > state.length) { - if (!state.ended && state.length < state.lowWaterMark) { + if (!state.ended && state.length <= state.lowWaterMark) { state.needReadable = true; n = 0; } else @@ -114,7 +115,7 @@ Readable.prototype.read = function(n) { state.length -= n; if (!state.ended && - state.length < state.lowWaterMark && + state.length <= state.lowWaterMark && !state.reading) { state.reading = true; // call internal read method @@ -145,7 +146,7 @@ Readable.prototype.read = function(n) { // that it's time to read more data. Otherwise, that'll // probably kick off another stream.read(), which can trigger // another _read(n,cb) before this one returns! - if (state.length < state.lowWaterMark) { + if (state.length <= state.lowWaterMark) { state.reading = true; this._read(state.bufferSize, onread.bind(this)); return; @@ -398,7 +399,7 @@ Readable.prototype.wrap = function(stream) { var ret = fromList(n, state.buffer, state.length, !!state.decoder); state.length -= n; - if (state.length < state.lowWaterMark && paused) { + if (state.length <= state.lowWaterMark && paused) { stream.resume(); paused = false; } diff --git a/lib/_stream_writable.js b/lib/_stream_writable.js index fd0cd115b4c..020d7abcc0e 100644 --- a/lib/_stream_writable.js +++ b/lib/_stream_writable.js @@ -33,7 +33,10 @@ util.inherits(Writable, Stream); function WritableState(options) { options = options || {}; this.highWaterMark = options.highWaterMark || 16 * 1024; - this.lowWaterMark = options.lowWaterMark || 1024; + this.highWaterMark = options.hasOwnProperty('highWaterMark') ? + options.highWaterMark : 16 * 1024; + this.lowWaterMark = options.hasOwnProperty('lowWaterMark') ? + options.lowWaterMark : 1024; this.needDrain = false; this.ended = false; this.ending = false; @@ -103,7 +106,7 @@ Writable.prototype.write = function(chunk, encoding) { this._write(chunk, writecb.bind(this)); } - if (state.length < state.lowWaterMark && state.needDrain) { + if (state.length <= state.lowWaterMark && state.needDrain) { // Must force callback to be called on nextTick, so that we don't // emit 'drain' before the write() consumer gets the 'false' return // value, and has a chance to attach a 'drain' listener. From 06e321d0f9692af1b6c6909cdfc49fd1b197129c Mon Sep 17 00:00:00 2001 From: isaacs Date: Fri, 5 Oct 2012 07:45:03 -0700 Subject: [PATCH 11/72] streams2: Correct drain/return logic It was testing the length *before* adding the current chunk, which is the opposite of correct. Also, the return value was flipped. --- lib/_stream_writable.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/_stream_writable.js b/lib/_stream_writable.js index 020d7abcc0e..90aa2858658 100644 --- a/lib/_stream_writable.js +++ b/lib/_stream_writable.js @@ -71,13 +71,13 @@ Writable.prototype.write = function(chunk, encoding) { if (typeof chunk === 'string') chunk = new Buffer(chunk, encoding); - var ret = state.length >= state.highWaterMark; - if (ret === false) - state.needDrain = true; - var l = chunk.length; state.length += l; + var ret = state.length < state.highWaterMark; + if (ret === false) + state.needDrain = true; + if (state.writing) { state.buffer.push(chunk); return ret; From 8acb416ad0532d249a0342f39103dd09fbbeeb2e Mon Sep 17 00:00:00 2001 From: isaacs Date: Sun, 7 Oct 2012 13:12:21 -0700 Subject: [PATCH 12/72] streams2: Handle immediate synthetic transforms properly --- lib/_stream_passthrough.js | 3 +- lib/_stream_readable.js | 140 +++++++++++++++++++++++++------------ lib/_stream_transform.js | 7 +- 3 files changed, 99 insertions(+), 51 deletions(-) diff --git a/lib/_stream_passthrough.js b/lib/_stream_passthrough.js index dd6390fc6ee..5acd27b2608 100644 --- a/lib/_stream_passthrough.js +++ b/lib/_stream_passthrough.js @@ -34,6 +34,5 @@ function PassThrough(options) { } PassThrough.prototype._transform = function(chunk, output, cb) { - output(chunk); - cb(); + cb(null, chunk); }; diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index ea944fcb44a..916ebc20561 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -28,7 +28,7 @@ var StringDecoder; util.inherits(Readable, Stream); -function ReadableState(options, stream) { +function ReadableState(options) { options = options || {}; this.bufferSize = options.bufferSize || 16 * 1024; @@ -44,7 +44,6 @@ function ReadableState(options, stream) { this.flowing = false; this.ended = false; this.endEmitted = false; - this.stream = stream; this.reading = false; // whenever we return null, then we set a flag to say @@ -71,52 +70,76 @@ Readable.prototype.setEncoding = function(enc) { this._readableState.decoder = new StringDecoder(enc); }; + +function howMuchToRead(n, state) { + if (state.length === 0 && state.ended) + return 0; + + if (isNaN(n)) + return state.length; + + if (n <= 0) + return 0; + + // don't have that much. return null, unless we've ended. + if (n > state.length) { + if (!state.ended) { + state.needReadable = true; + return 0; + } else + return state.length; + } + + return n; +} + // you can override either this method, or _read(n, cb) below. Readable.prototype.read = function(n) { var state = this._readableState; + var nOrig = n; - if (state.length === 0 && state.ended) { + n = howMuchToRead(n, state); + + // if we've ended, and we're now clear, then finish it up. + if (n === 0 && state.ended) { endReadable(this); return null; } - if (isNaN(n) || n <= 0) - n = state.length + // All the actual chunk generation logic needs to be + // *below* the call to _read. The reason is that in certain + // synthetic stream cases, such as passthrough streams, _read + // may be a completely synchronous operation which may change + // the state of the read buffer, providing enough data when + // before there was *not* enough. + // + // So, the steps are: + // 1. Figure out what the state of things will be after we do + // a read from the buffer. + // + // 2. If that resulting state will trigger a _read, then call _read. + // Note that this may be asynchronous, or synchronous. Yes, it is + // deeply ugly to write APIs this way, but that still doesn't mean + // that the Readable class should behave improperly, as streams are + // designed to be sync/async agnostic. + // Take note if the _read call is sync or async (ie, if the read call + // has returned yet), so that we know whether or not it's safe to emit + // 'readable' etc. + // + // 3. Actually pull the requested chunks out of the buffer and return. - // XXX: controversial. - // don't have that much. return null, unless we've ended. - // However, if the low water mark is lower than the number of bytes, - // then we still need to return what we have, or else it won't kick - // off another _read() call. For example, - // lwm=5 - // len=9 - // read(10) - // We don't have that many bytes, so it'd be tempting to return null, - // but then it won't ever cause _read to be called, so in that case, - // we just return what we have, and let the programmer deal with it. - if (n > state.length) { - if (!state.ended && state.length <= state.lowWaterMark) { - state.needReadable = true; - n = 0; - } else - n = state.length; - } + // if we need a readable event, then we need to do some reading. + var doRead = state.needReadable; + // if we currently have less than the lowWaterMark, then also read some + if (state.length - n <= state.lowWaterMark) + doRead = true; + // however, if we've ended, then there's no point, and if we're already + // reading, then it's unnecessary. + if (state.ended || state.reading) + doRead = false; - - var ret; - if (n > 0) - ret = fromList(n, state.buffer, state.length, !!state.decoder); - else - ret = null; - - if (ret === null || ret.length === 0) - state.needReadable = true; - - state.length -= n; - - if (!state.ended && - state.length <= state.lowWaterMark && - !state.reading) { + if (doRead) { + var sync = true; state.reading = true; // call internal read method this._read(state.bufferSize, function onread(er, chunk) { @@ -125,21 +148,27 @@ Readable.prototype.read = function(n) { return this.emit('error', er); if (!chunk || !chunk.length) { + // eof state.ended = true; // if we've ended and we have some data left, then emit // 'readable' now to make sure it gets picked up. - if (state.length > 0) - this.emit('readable'); - else - endReadable(this); + if (!sync) { + if (state.length > 0) + this.emit('readable'); + else + endReadable(this); + } return; } if (state.decoder) chunk = state.decoder.write(chunk); - state.length += chunk.length; - state.buffer.push(chunk); + // update the buffer info. + if (chunk) { + state.length += chunk.length; + state.buffer.push(chunk); + } // if we haven't gotten enough to pass the lowWaterMark, // and we haven't ended, then don't bother telling the user @@ -152,14 +181,33 @@ Readable.prototype.read = function(n) { return; } - // now we have something to call this.read() to get. - if (state.needReadable) { + if (state.needReadable && !sync) { state.needReadable = false; this.emit('readable'); } }.bind(this)); + sync = false; } + // If _read called its callback synchronously, then `reading` + // will be false, and we need to re-evaluate how much data we + // can return to the user. + if (doRead && !state.reading) + n = howMuchToRead(nOrig, state); + + var ret; + if (n > 0) + ret = fromList(n, state.buffer, state.length, !!state.decoder); + else + ret = null; + + if (ret === null || ret.length === 0) { + state.needReadable = true; + n = 0; + } + + state.length -= n; + return ret; }; diff --git a/lib/_stream_transform.js b/lib/_stream_transform.js index 40917de7abf..16f2cacb936 100644 --- a/lib/_stream_transform.js +++ b/lib/_stream_transform.js @@ -122,13 +122,14 @@ Transform.prototype._write = function(chunk, cb) { if (ts.pendingReadCb) { var readcb = ts.pendingReadCb; ts.pendingReadCb = null; - this._read(-1, readcb); + this._read(0, readcb); } // if we weren't waiting for it, but nothing is queued up, then // still kick off a transform, just so it's there when the user asks. - if (rs.length === 0) { - var ret = this.read(); + var doRead = rs.needReadable || rs.length <= rs.lowWaterMark; + if (doRead && !rs.reading) { + var ret = this.read(0); if (ret !== null) return cb(new Error('invalid stream transform state')); } From 9b1b85490b6b242aabf84054a0a59ea1cf1c8dbc Mon Sep 17 00:00:00 2001 From: isaacs Date: Sun, 7 Oct 2012 13:26:03 -0700 Subject: [PATCH 13/72] streams2: Tests of new interfaces --- test/simple/test-stream2-basic.js | 312 ++++++++++++++++++ test/simple/test-stream2-fs.js | 76 +++++ .../simple/test-stream2-readable-from-list.js | 109 ++++++ test/simple/test-stream2-set-encoding.js | 299 +++++++++++++++++ test/simple/test-stream2-transform.js | 309 +++++++++++++++++ test/simple/test-stream2-writable.js | 142 ++++++++ 6 files changed, 1247 insertions(+) create mode 100644 test/simple/test-stream2-basic.js create mode 100644 test/simple/test-stream2-fs.js create mode 100644 test/simple/test-stream2-readable-from-list.js create mode 100644 test/simple/test-stream2-set-encoding.js create mode 100644 test/simple/test-stream2-transform.js create mode 100644 test/simple/test-stream2-writable.js diff --git a/test/simple/test-stream2-basic.js b/test/simple/test-stream2-basic.js new file mode 100644 index 00000000000..c28cdc917a6 --- /dev/null +++ b/test/simple/test-stream2-basic.js @@ -0,0 +1,312 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + + +var common = require('../common.js'); +var R = require('_stream_readable'); +var assert = require('assert'); + +var util = require('util'); +var EE = require('events').EventEmitter; + +function TestReader(n) { + R.apply(this); + this._buffer = new Buffer(n || 100); + this._buffer.fill('x'); + this._pos = 0; + this._bufs = 10; +} + +util.inherits(TestReader, R); + +TestReader.prototype.read = function(n) { + var max = this._buffer.length - this._pos; + n = n || max; + n = Math.max(n, 0); + var toRead = Math.min(n, max); + if (toRead === 0) { + // simulate the read buffer filling up with some more bytes some time + // in the future. + setTimeout(function() { + this._pos = 0; + this._bufs -= 1; + if (this._bufs <= 0) { + // read them all! + if (!this.ended) { + this.emit('end'); + this.ended = true; + } + } else { + this.emit('readable'); + } + }.bind(this), 10); + return null; + } + + var ret = this._buffer.slice(this._pos, this._pos + toRead); + this._pos += toRead; + return ret; +}; + +///// + +function TestWriter() { + EE.apply(this); + this.received = []; + this.flush = false; +} + +util.inherits(TestWriter, EE); + +TestWriter.prototype.write = function(c) { + this.received.push(c.toString()); + this.emit('write', c); + return true; + + // flip back and forth between immediate acceptance and not. + this.flush = !this.flush; + if (!this.flush) setTimeout(this.emit.bind(this, 'drain'), 10); + return this.flush; +}; + +TestWriter.prototype.end = function(c) { + if (c) this.write(c); + this.emit('end', this.received); +}; + +//////// + +// tiny node-tap lookalike. +var tests = []; +function test(name, fn) { + tests.push([name, fn]); +} + +function run() { + var next = tests.shift(); + if (!next) + return console.error('ok'); + + var name = next[0]; + var fn = next[1]; + console.log('# %s', name); + fn({ + same: assert.deepEqual, + equal: assert.equal, + end: run + }); +} + +process.nextTick(run); + + +test('a most basic test', function(t) { + var r = new TestReader(20); + + var reads = []; + var expect = [ 'x', + 'xx', + 'xxx', + 'xxxx', + 'xxxxx', + 'xxxxx', + 'xxxxxxxx', + 'xxxxxxxxx', + 'xxx', + 'xxxxxxxxxxxx', + 'xxxxxxxx', + 'xxxxxxxxxxxxxxx', + 'xxxxx', + 'xxxxxxxxxxxxxxxxxx', + 'xx', + 'xxxxxxxxxxxxxxxxxxxx', + 'xxxxxxxxxxxxxxxxxxxx', + 'xxxxxxxxxxxxxxxxxxxx', + 'xxxxxxxxxxxxxxxxxxxx', + 'xxxxxxxxxxxxxxxxxxxx' ]; + + r.on('end', function() { + t.same(reads, expect); + t.end(); + }); + + var readSize = 1; + function flow() { + var res; + while (null !== (res = r.read(readSize++))) { + reads.push(res.toString()); + } + r.once('readable', flow); + } + + flow(); +}); + +test('pipe', function(t) { + var r = new TestReader(5); + + var expect = [ 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx' ] + + var w = new TestWriter; + var flush = true; + w.on('end', function(received) { + t.same(received, expect); + t.end(); + }); + + r.pipe(w); +}); + + + +[1,2,3,4,5,6,7,8,9].forEach(function(SPLIT) { + test('unpipe', function(t) { + var r = new TestReader(5); + + // unpipe after 3 writes, then write to another stream instead. + var expect = [ 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx' ]; + expect = [ expect.slice(0, SPLIT), expect.slice(SPLIT) ]; + + var w = [ new TestWriter(), new TestWriter() ]; + + var writes = SPLIT; + w[0].on('write', function() { + if (--writes === 0) { + r.unpipe(); + w[0].end(); + r.pipe(w[1]); + } + }); + + var ended = 0; + + w[0].on('end', function(results) { + ended++; + t.same(results, expect[0]); + }); + + w[1].on('end', function(results) { + ended++; + t.equal(ended, 2); + t.same(results, expect[1]); + t.end(); + }); + + r.pipe(w[0]); + }); +}); + + +// both writers should get the same exact data. +test('multipipe', function(t) { + var r = new TestReader(5); + var w = [ new TestWriter, new TestWriter ]; + + var expect = [ 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx' ]; + + var c = 2; + w[0].on('end', function(received) { + t.same(received, expect, 'first'); + if (--c === 0) t.end(); + }); + w[1].on('end', function(received) { + t.same(received, expect, 'second'); + if (--c === 0) t.end(); + }); + + r.pipe(w[0]); + r.pipe(w[1]); +}); + + +[1,2,3,4,5,6,7,8,9].forEach(function(SPLIT) { + test('multi-unpipe', function(t) { + var r = new TestReader(5); + + // unpipe after 3 writes, then write to another stream instead. + var expect = [ 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx', + 'xxxxx' ]; + expect = [ expect.slice(0, SPLIT), expect.slice(SPLIT) ]; + + var w = [ new TestWriter(), new TestWriter(), new TestWriter() ]; + + var writes = SPLIT; + w[0].on('write', function() { + if (--writes === 0) { + r.unpipe(); + w[0].end(); + r.pipe(w[1]); + } + }); + + var ended = 0; + + w[0].on('end', function(results) { + ended++; + t.same(results, expect[0]); + }); + + w[1].on('end', function(results) { + ended++; + t.equal(ended, 2); + t.same(results, expect[1]); + t.end(); + }); + + r.pipe(w[0]); + r.pipe(w[2]); + }); +}); diff --git a/test/simple/test-stream2-fs.js b/test/simple/test-stream2-fs.js new file mode 100644 index 00000000000..70cd8ba4de3 --- /dev/null +++ b/test/simple/test-stream2-fs.js @@ -0,0 +1,76 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + + +var common = require('../common.js'); +var R = require('_stream_readable'); +var assert = require('assert'); + +var fs = require('fs'); +var FSReadable = fs.ReadStream; + +var path = require('path'); +var file = path.resolve(common.fixturesDir, 'x1024.txt'); + +var size = fs.statSync(file).size; + +// expect to see chunks no more than 10 bytes each. +var expectLengths = []; +for (var i = size; i > 0; i -= 10) { + expectLengths.push(Math.min(i, 10)); +} + +var util = require('util'); +var Stream = require('stream'); + +util.inherits(TestWriter, Stream); + +function TestWriter() { + Stream.apply(this); + this.buffer = []; + this.length = 0; +} + +TestWriter.prototype.write = function(c) { + this.buffer.push(c.toString()); + this.length += c.length; + return true; +}; + +TestWriter.prototype.end = function(c) { + if (c) this.buffer.push(c.toString()); + this.emit('results', this.buffer); +} + +var r = new FSReadable(file, { bufferSize: 10 }); +var w = new TestWriter(); + +w.on('results', function(res) { + console.error(res, w.length); + assert.equal(w.length, size); + var l = 0; + assert.deepEqual(res.map(function (c) { + return c.length; + }), expectLengths); + console.log('ok'); +}); + +r.pipe(w, { chunkSize: 10 }); diff --git a/test/simple/test-stream2-readable-from-list.js b/test/simple/test-stream2-readable-from-list.js new file mode 100644 index 00000000000..a28fe343eec --- /dev/null +++ b/test/simple/test-stream2-readable-from-list.js @@ -0,0 +1,109 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +var assert = require('assert'); +var common = require('../common.js'); +var fromList = require('_stream_readable')._fromList; + +// tiny node-tap lookalike. +var tests = []; +function test(name, fn) { + tests.push([name, fn]); +} + +function run() { + var next = tests.shift(); + if (!next) + return console.error('ok'); + + var name = next[0]; + var fn = next[1]; + console.log('# %s', name); + fn({ + same: assert.deepEqual, + equal: assert.equal, + end: run + }); +} + +process.nextTick(run); + + + +test('buffers', function(t) { + // have a length + var len = 16; + var list = [ new Buffer('foog'), + new Buffer('bark'), + new Buffer('bazy'), + new Buffer('kuel') ]; + + // read more than the first element. + var ret = fromList(6, list, 16); + t.equal(ret.toString(), 'foogba'); + + // read exactly the first element. + ret = fromList(2, list, 10); + t.equal(ret.toString(), 'rk'); + + // read less than the first element. + ret = fromList(2, list, 8); + t.equal(ret.toString(), 'ba'); + + // read more than we have. + ret = fromList(100, list, 6); + t.equal(ret.toString(), 'zykuel'); + + // all consumed. + t.same(list, []); + + t.end(); +}); + +test('strings', function(t) { + // have a length + var len = 16; + var list = [ 'foog', + 'bark', + 'bazy', + 'kuel' ]; + + // read more than the first element. + var ret = fromList(6, list, 16, true); + t.equal(ret, 'foogba'); + + // read exactly the first element. + ret = fromList(2, list, 10, true); + t.equal(ret, 'rk'); + + // read less than the first element. + ret = fromList(2, list, 8, true); + t.equal(ret, 'ba'); + + // read more than we have. + ret = fromList(100, list, 6, true); + t.equal(ret, 'zykuel'); + + // all consumed. + t.same(list, []); + + t.end(); +}); diff --git a/test/simple/test-stream2-set-encoding.js b/test/simple/test-stream2-set-encoding.js new file mode 100644 index 00000000000..a1883fbbb84 --- /dev/null +++ b/test/simple/test-stream2-set-encoding.js @@ -0,0 +1,299 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + + +var common = require('../common.js'); +var assert = require('assert'); +var R = require('_stream_readable'); +var util = require('util'); + +// tiny node-tap lookalike. +var tests = []; +function test(name, fn) { + tests.push([name, fn]); +} + +function run() { + var next = tests.shift(); + if (!next) + return console.error('ok'); + + var name = next[0]; + var fn = next[1]; + console.log('# %s', name); + fn({ + same: assert.deepEqual, + equal: assert.equal, + end: run + }); +} + +process.nextTick(run); + +///// + +util.inherits(TestReader, R); + +function TestReader(n, opts) { + R.call(this, util._extend({ + bufferSize: 5 + }, opts)); + + this.pos = 0; + this.len = n || 100; +} + +TestReader.prototype._read = function(n, cb) { + setTimeout(function() { + + if (this.pos >= this.len) { + return cb(); + } + + n = Math.min(n, this.len - this.pos); + if (n <= 0) { + return cb(); + } + + this.pos += n; + var ret = new Buffer(n); + ret.fill('a'); + + return cb(null, ret); + }.bind(this), 1); +}; + +test('setEncoding utf8', function(t) { + var tr = new TestReader(100); + tr.setEncoding('utf8'); + var out = []; + var expect = + [ 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa' ]; + + tr.on('readable', function flow() { + var chunk; + while (null !== (chunk = tr.read(10))) + out.push(chunk); + }); + + tr.on('end', function() { + t.same(out, expect); + t.end(); + }); + + // just kick it off. + tr.emit('readable'); +}); + + +test('setEncoding hex', function(t) { + var tr = new TestReader(100); + tr.setEncoding('hex'); + var out = []; + var expect = + [ '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161' ]; + + tr.on('readable', function flow() { + var chunk; + while (null !== (chunk = tr.read(10))) + out.push(chunk); + }); + + tr.on('end', function() { + t.same(out, expect); + t.end(); + }); + + // just kick it off. + tr.emit('readable'); +}); + +test('setEncoding hex with read(13)', function(t) { + var tr = new TestReader(100); + tr.setEncoding('hex'); + var out = []; + var expect = + [ "6161616161616", + "1616161616161", + "6161616161616", + "1616161616161", + "6161616161616", + "1616161616161", + "6161616161616", + "1616161616161", + "6161616161616", + "1616161616161", + "6161616161616", + "1616161616161", + "6161616161616", + "1616161616161", + "6161616161616", + "16161" ]; + + tr.on('readable', function flow() { + var chunk; + while (null !== (chunk = tr.read(13))) + out.push(chunk); + }); + + tr.on('end', function() { + t.same(out, expect); + t.end(); + }); + + // just kick it off. + tr.emit('readable'); +}); + +test('encoding: utf8', function(t) { + var tr = new TestReader(100, { encoding: 'utf8' }); + var out = []; + var expect = + [ 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa', + 'aaaaaaaaaa' ]; + + tr.on('readable', function flow() { + var chunk; + while (null !== (chunk = tr.read(10))) + out.push(chunk); + }); + + tr.on('end', function() { + t.same(out, expect); + t.end(); + }); + + // just kick it off. + tr.emit('readable'); +}); + + +test('encoding: hex', function(t) { + var tr = new TestReader(100, { encoding: 'hex' }); + var out = []; + var expect = + [ '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161', + '6161616161' ]; + + tr.on('readable', function flow() { + var chunk; + while (null !== (chunk = tr.read(10))) + out.push(chunk); + }); + + tr.on('end', function() { + t.same(out, expect); + t.end(); + }); + + // just kick it off. + tr.emit('readable'); +}); + +test('encoding: hex with read(13)', function(t) { + var tr = new TestReader(100, { encoding: 'hex' }); + var out = []; + var expect = + [ "6161616161616", + "1616161616161", + "6161616161616", + "1616161616161", + "6161616161616", + "1616161616161", + "6161616161616", + "1616161616161", + "6161616161616", + "1616161616161", + "6161616161616", + "1616161616161", + "6161616161616", + "1616161616161", + "6161616161616", + "16161" ]; + + tr.on('readable', function flow() { + var chunk; + while (null !== (chunk = tr.read(13))) + out.push(chunk); + }); + + tr.on('end', function() { + t.same(out, expect); + t.end(); + }); + + // just kick it off. + tr.emit('readable'); +}); diff --git a/test/simple/test-stream2-transform.js b/test/simple/test-stream2-transform.js new file mode 100644 index 00000000000..9b6365b5ca9 --- /dev/null +++ b/test/simple/test-stream2-transform.js @@ -0,0 +1,309 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +var assert = require('assert'); +var common = require('../common.js'); +var PassThrough = require('_stream_passthrough'); +var Transform = require('_stream_transform'); + +// tiny node-tap lookalike. +var tests = []; +function test(name, fn) { + tests.push([name, fn]); +} + +function run() { + var next = tests.shift(); + if (!next) + return console.error('ok'); + + var name = next[0]; + var fn = next[1]; + console.log('# %s', name); + fn({ + same: assert.deepEqual, + equal: assert.equal, + end: run + }); +} + +process.nextTick(run); + +///// + +test('passthrough', function(t) { + var pt = new PassThrough(); + + pt.write(new Buffer('foog')); + pt.write(new Buffer('bark')); + pt.write(new Buffer('bazy')); + pt.write(new Buffer('kuel')); + pt.end(); + + t.equal(pt.read(5).toString(), 'foogb'); + t.equal(pt.read(5).toString(), 'arkba'); + t.equal(pt.read(5).toString(), 'zykue'); + t.equal(pt.read(5).toString(), 'l'); + t.end(); +}); + +test('simple transform', function(t) { + var pt = new Transform; + pt._transform = function(c, output, cb) { + var ret = new Buffer(c.length); + ret.fill('x'); + output(ret); + cb(); + }; + + pt.write(new Buffer('foog')); + pt.write(new Buffer('bark')); + pt.write(new Buffer('bazy')); + pt.write(new Buffer('kuel')); + pt.end(); + + t.equal(pt.read(5).toString(), 'xxxxx'); + t.equal(pt.read(5).toString(), 'xxxxx'); + t.equal(pt.read(5).toString(), 'xxxxx'); + t.equal(pt.read(5).toString(), 'x'); + t.end(); +}); + +test('async passthrough', function(t) { + var pt = new Transform; + pt._transform = function(chunk, output, cb) { + setTimeout(function() { + output(chunk); + cb(); + }, 10); + }; + + pt.write(new Buffer('foog')); + pt.write(new Buffer('bark')); + pt.write(new Buffer('bazy')); + pt.write(new Buffer('kuel')); + pt.end(); + + setTimeout(function() { + t.equal(pt.read(5).toString(), 'foogb'); + t.equal(pt.read(5).toString(), 'arkba'); + t.equal(pt.read(5).toString(), 'zykue'); + t.equal(pt.read(5).toString(), 'l'); + t.end(); + }, 100); +}); + +test('assymetric transform (expand)', function(t) { + var pt = new Transform; + + // emit each chunk 2 times. + pt._transform = function(chunk, output, cb) { + setTimeout(function() { + output(chunk); + setTimeout(function() { + output(chunk); + cb(); + }, 10) + }, 10); + }; + + pt.write(new Buffer('foog')); + pt.write(new Buffer('bark')); + pt.write(new Buffer('bazy')); + pt.write(new Buffer('kuel')); + pt.end(); + + setTimeout(function() { + t.equal(pt.read(5).toString(), 'foogf'); + t.equal(pt.read(5).toString(), 'oogba'); + t.equal(pt.read(5).toString(), 'rkbar'); + t.equal(pt.read(5).toString(), 'kbazy'); + t.equal(pt.read(5).toString(), 'bazyk'); + t.equal(pt.read(5).toString(), 'uelku'); + t.equal(pt.read(5).toString(), 'el'); + t.end(); + }, 100); +}); + +test('assymetric transform (compress)', function(t) { + var pt = new Transform; + + // each output is the first char of 3 consecutive chunks, + // or whatever's left. + pt.state = ''; + + pt._transform = function(chunk, output, cb) { + if (!chunk) + chunk = ''; + var s = chunk.toString(); + setTimeout(function() { + this.state += s.charAt(0); + if (this.state.length === 3) { + output(new Buffer(this.state)); + this.state = ''; + } + cb(); + }.bind(this), 10); + }; + + pt._flush = function(output, cb) { + // just output whatever we have. + setTimeout(function() { + output(new Buffer(this.state)); + this.state = ''; + cb(); + }.bind(this), 10); + }; + + pt._writableState.lowWaterMark = 3; + + pt.write(new Buffer('aaaa')); + pt.write(new Buffer('bbbb')); + pt.write(new Buffer('cccc')); + pt.write(new Buffer('dddd')); + pt.write(new Buffer('eeee')); + pt.write(new Buffer('aaaa')); + pt.write(new Buffer('bbbb')); + pt.write(new Buffer('cccc')); + pt.write(new Buffer('dddd')); + pt.write(new Buffer('eeee')); + pt.write(new Buffer('aaaa')); + pt.write(new Buffer('bbbb')); + pt.write(new Buffer('cccc')); + pt.write(new Buffer('dddd')); + pt.end(); + + // 'abcdeabcdeabcd' + setTimeout(function() { + t.equal(pt.read(5).toString(), 'abcde'); + t.equal(pt.read(5).toString(), 'abcde'); + t.equal(pt.read(5).toString(), 'abcd'); + t.end(); + }, 200); +}); + + +test('passthrough event emission', function(t) { + var pt = new PassThrough({ + lowWaterMark: 0 + }); + var emits = 0; + pt.on('readable', function() { + var state = pt._readableState; + console.error('>>> emit readable %d', emits); + emits++; + }); + + var i = 0; + + pt.write(new Buffer('foog')); + pt.write(new Buffer('bark')); + + t.equal(pt.read(5).toString(), 'foogb'); + t.equal(pt.read(5) + '', 'null'); + + console.error('need emit 0'); + + pt.write(new Buffer('bazy')); + pt.write(new Buffer('kuel')); + + console.error('should have emitted readable now'); + t.equal(emits, 1); + + t.equal(pt.read(5).toString(), 'arkba'); + t.equal(pt.read(5).toString(), 'zykue'); + t.equal(pt.read(5), null); + + console.error('need emit 1'); + + pt.end(); + + t.equal(emits, 2); + + t.equal(pt.read(5).toString(), 'l'); + t.equal(pt.read(5), null); + + console.error('should not have emitted again'); + t.equal(emits, 2); + t.end(); +}); + +test('passthrough event emission reordered', function(t) { + var pt = new PassThrough; + var emits = 0; + pt.on('readable', function() { + console.error('emit readable', emits) + emits++; + }); + + pt.write(new Buffer('foog')); + pt.write(new Buffer('bark')); + + t.equal(pt.read(5).toString(), 'foogb'); + t.equal(pt.read(5), null); + + console.error('need emit 0'); + pt.once('readable', function() { + t.equal(pt.read(5).toString(), 'arkba'); + t.equal(pt.read(5).toString(), 'zykue'); + t.equal(pt.read(5), null); + + console.error('need emit 1'); + pt.once('readable', function() { + t.equal(pt.read(5).toString(), 'l'); + t.equal(pt.read(5), null); + + t.equal(emits, 2); + t.end(); + }); + pt.end(); + }); + pt.write(new Buffer('bazy')); + pt.write(new Buffer('kuel')); +}); + +test('passthrough facaded', function(t) { + console.error('passthrough facaded'); + var pt = new PassThrough; + var datas = []; + pt.on('data', function(chunk) { + datas.push(chunk.toString()); + }); + + pt.on('end', function() { + t.same(datas, ['foog', 'bark', 'bazy', 'kuel']); + t.end(); + }); + + pt.write(new Buffer('foog')); + setTimeout(function() { + pt.write(new Buffer('bark')); + setTimeout(function() { + pt.write(new Buffer('bazy')); + setTimeout(function() { + pt.write(new Buffer('kuel')); + setTimeout(function() { + pt.end(); + }, 10); + }, 10); + }, 10); + }, 10); +}); diff --git a/test/simple/test-stream2-writable.js b/test/simple/test-stream2-writable.js new file mode 100644 index 00000000000..bc1859ce222 --- /dev/null +++ b/test/simple/test-stream2-writable.js @@ -0,0 +1,142 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +var common = require('../common.js'); +var W = require('_stream_writable'); +var assert = require('assert'); + +var util = require('util'); +util.inherits(TestWriter, W); + +function TestWriter() { + W.apply(this, arguments); + this.buffer = []; + this.written = 0; +} + +TestWriter.prototype._write = function(chunk, cb) { + // simulate a small unpredictable latency + setTimeout(function() { + this.buffer.push(chunk.toString()); + this.written += chunk.length; + cb(); + }.bind(this), Math.floor(Math.random() * 10)); +}; + +var chunks = new Array(50); +for (var i = 0; i < chunks.length; i++) { + chunks[i] = new Array(i + 1).join('x'); +} + +// tiny node-tap lookalike. +var tests = []; +function test(name, fn) { + tests.push([name, fn]); +} + +function run() { + var next = tests.shift(); + if (!next) + return console.log('ok'); + + var name = next[0]; + var fn = next[1]; + console.log('# %s', name); + fn({ + same: assert.deepEqual, + equal: assert.equal, + end: run + }); +} + +process.nextTick(run); + +test('write fast', function(t) { + var tw = new TestWriter({ + lowWaterMark: 5, + highWaterMark: 100 + }); + + tw.on('finish', function() { + t.same(tw.buffer, chunks, 'got chunks in the right order'); + t.end(); + }); + + chunks.forEach(function(chunk) { + // screw backpressure. Just buffer it all up. + tw.write(chunk); + }); + tw.end(); +}); + +test('write slow', function(t) { + var tw = new TestWriter({ + lowWaterMark: 5, + highWaterMark: 100 + }); + + tw.on('finish', function() { + t.same(tw.buffer, chunks, 'got chunks in the right order'); + t.end(); + }); + + var i = 0; + (function W() { + tw.write(chunks[i++]); + if (i < chunks.length) + setTimeout(W, 10); + else + tw.end(); + })(); +}); + +test('write backpressure', function(t) { + var tw = new TestWriter({ + lowWaterMark: 5, + highWaterMark: 50 + }); + + var drains = 0; + + tw.on('finish', function() { + t.same(tw.buffer, chunks, 'got chunks in the right order'); + t.equal(drains, 17); + t.end(); + }); + + tw.on('drain', function() { + drains++; + }); + + var i = 0; + (function W() { + do { + var ret = tw.write(chunks[i++]); + } while (ret !== false && i < chunks.length); + + if (i < chunks.length) { + assert(tw._writableState.length >= 50); + tw.once('drain', W); + } else { + tw.end(); + } + })(); +}); From 545f5126190acacd14ce54799d1e3c5d6dcd9edb Mon Sep 17 00:00:00 2001 From: isaacs Date: Mon, 8 Oct 2012 14:43:17 -0700 Subject: [PATCH 14/72] streams2: ctor guards on Stream classes --- lib/_stream_duplex.js | 3 +++ lib/_stream_passthrough.js | 3 +++ lib/_stream_readable.js | 3 +++ lib/_stream_transform.js | 3 +++ lib/_stream_writable.js | 6 ++++++ 5 files changed, 18 insertions(+) diff --git a/lib/_stream_duplex.js b/lib/_stream_duplex.js index 0256b0f2f22..30b46922332 100644 --- a/lib/_stream_duplex.js +++ b/lib/_stream_duplex.js @@ -37,6 +37,9 @@ Object.keys(Writable.prototype).forEach(function(method) { }); function Duplex(options) { + if (!(this instanceof Duplex)) + return new Duplex(options); + Readable.call(this, options); Writable.call(this, options); diff --git a/lib/_stream_passthrough.js b/lib/_stream_passthrough.js index 5acd27b2608..0f2fe14c787 100644 --- a/lib/_stream_passthrough.js +++ b/lib/_stream_passthrough.js @@ -30,6 +30,9 @@ var util = require('util'); util.inherits(PassThrough, Transform); function PassThrough(options) { + if (!(this instanceof PassThrough)) + return new PassThrough(options); + Transform.call(this, options); } diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 916ebc20561..180ab1d55b6 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -59,6 +59,9 @@ function ReadableState(options) { } function Readable(options) { + if (!(this instanceof Readable)) + return new Readable(options); + this._readableState = new ReadableState(options, this); Stream.apply(this); } diff --git a/lib/_stream_transform.js b/lib/_stream_transform.js index 16f2cacb936..abf7479fdce 100644 --- a/lib/_stream_transform.js +++ b/lib/_stream_transform.js @@ -77,6 +77,9 @@ function TransformState() { } function Transform(options) { + if (!(this instanceof Transform)) + return new Transform(options); + Duplex.call(this, options); // bind output so that it can be passed around as a regular function. diff --git a/lib/_stream_writable.js b/lib/_stream_writable.js index 90aa2858658..ce56afdff9e 100644 --- a/lib/_stream_writable.js +++ b/lib/_stream_writable.js @@ -27,6 +27,7 @@ module.exports = Writable var util = require('util'); var Stream = require('stream'); +var Duplex = Stream.Duplex; util.inherits(Writable, Stream); @@ -51,6 +52,11 @@ function WritableState(options) { } function Writable(options) { + // Writable ctor is applied to Duplexes, though they're not + // instanceof Writable, they're instanceof Readable. + if (!(this instanceof Writable) && !(this instanceof Duplex)) + return new Writable(options); + this._writableState = new WritableState(options); // legacy. From 0678480b572b3d4d2a7a25060d5acd66fdd65199 Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 9 Oct 2012 11:01:53 -0700 Subject: [PATCH 15/72] streams2: Allow Writables to opt out of pre-buffer-izing --- lib/_stream_writable.js | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/lib/_stream_writable.js b/lib/_stream_writable.js index ce56afdff9e..4eb2a5f981e 100644 --- a/lib/_stream_writable.js +++ b/lib/_stream_writable.js @@ -27,7 +27,7 @@ module.exports = Writable var util = require('util'); var Stream = require('stream'); -var Duplex = Stream.Duplex; +var Duplex = require('_stream_duplex'); util.inherits(Writable, Stream); @@ -42,6 +42,12 @@ function WritableState(options) { this.ended = false; this.ending = false; + // should we decode strings into buffers before passing to _write? + // this is here so that some node-core streams can optimize string + // handling at a lower level. + this.decodeStrings = options.hasOwnProperty('decodeStrings') ? + options.decodeStrings : true; + // not an actual buffer we keep track of, but a measurement // of how much we're waiting to get pushed to some underlying // socket or file. @@ -74,10 +80,14 @@ Writable.prototype.write = function(chunk, encoding) { return; } - if (typeof chunk === 'string') - chunk = new Buffer(chunk, encoding); - var l = chunk.length; + if (false === state.decodeStrings) + chunk = [chunk, encoding]; + else if (typeof chunk === 'string' || encoding) { + chunk = new Buffer(chunk + '', encoding); + l = chunk.length; + } + state.length += l; var ret = state.length < state.highWaterMark; @@ -90,7 +100,11 @@ Writable.prototype.write = function(chunk, encoding) { } state.writing = true; - this._write(chunk, function writecb(er) { + this._write(chunk, writecb.bind(this)); + + return ret; + + function writecb(er) { state.writing = false; if (er) { this.emit('error', er); @@ -123,10 +137,8 @@ Writable.prototype.write = function(chunk, encoding) { this.emit('drain'); }.bind(this)); } + } - }.bind(this)); - - return ret; }; Writable.prototype._write = function(chunk, cb) { From 71e2b61388e09dad116cea4f6b3a3790d0d95654 Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 9 Oct 2012 17:31:29 -0700 Subject: [PATCH 16/72] streams2: Support write(chunk,[encoding],[callback]) --- lib/_stream_writable.js | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/lib/_stream_writable.js b/lib/_stream_writable.js index 4eb2a5f981e..a84cbc3785e 100644 --- a/lib/_stream_writable.js +++ b/lib/_stream_writable.js @@ -73,13 +73,18 @@ function Writable(options) { // Override this method for sync streams // override the _write(chunk, cb) method for async streams -Writable.prototype.write = function(chunk, encoding) { +Writable.prototype.write = function(chunk, encoding, cb) { var state = this._writableState; if (state.ended) { this.emit('error', new Error('write after end')); return; } + if (typeof encoding === 'function') { + cb = encoding; + encoding = null; + } + var l = chunk.length; if (false === state.decodeStrings) chunk = [chunk, encoding]; @@ -94,24 +99,43 @@ Writable.prototype.write = function(chunk, encoding) { if (ret === false) state.needDrain = true; + // if we're already writing something, then just put this + // in the queue, and wait our turn. if (state.writing) { - state.buffer.push(chunk); + state.buffer.push([chunk, cb]); return ret; } state.writing = true; + var sync = true; this._write(chunk, writecb.bind(this)); + sync = false; return ret; function writecb(er) { state.writing = false; if (er) { - this.emit('error', er); + if (cb) { + if (sync) + process.nextTick(cb.bind(null, er)); + else + cb(er); + } else + this.emit('error', er); return; } state.length -= l; + if (cb) { + // don't call the cb until the next tick if we're in sync mode. + // also, defer if we're about to write some more right now. + if (sync || state.buffer.length) + process.nextTick(cb); + else + cb(); + } + if (state.length === 0 && (state.ended || state.ending)) { // emit 'finish' at the very end. this.emit('finish'); @@ -120,8 +144,15 @@ Writable.prototype.write = function(chunk, encoding) { // if there's something in the buffer waiting, then do that, too. if (state.buffer.length) { - chunk = state.buffer.shift(); - l = chunk.length; + var chunkCb = state.buffer.shift(); + chunk = chunkCb[0]; + cb = chunkCb[1]; + + if (false === state.decodeStrings) + l = chunk[0].length; + else + l = chunk.length; + state.writing = true; this._write(chunk, writecb.bind(this)); } From f3e71eb41707a375aa10970d5c1eda39d78dcb71 Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 9 Oct 2012 17:37:40 -0700 Subject: [PATCH 17/72] test: Writable bufferizing, non-bufferizing, and callbacks --- test/simple/test-stream2-writable.js | 98 ++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/test/simple/test-stream2-writable.js b/test/simple/test-stream2-writable.js index bc1859ce222..bfd6bb75d61 100644 --- a/test/simple/test-stream2-writable.js +++ b/test/simple/test-stream2-writable.js @@ -140,3 +140,101 @@ test('write backpressure', function(t) { } })(); }); + +test('write bufferize', function(t) { + var tw = new TestWriter({ + lowWaterMark: 5, + highWaterMark: 100 + }); + + var encodings = + [ 'hex', + 'utf8', + 'utf-8', + 'ascii', + 'binary', + 'base64', + 'ucs2', + 'ucs-2', + 'utf16le', + 'utf-16le', + undefined ]; + + tw.on('finish', function() { + t.same(tw.buffer, chunks, 'got the expected chunks'); + }); + + chunks.forEach(function(chunk, i) { + var enc = encodings[ i % encodings.length ]; + chunk = new Buffer(chunk); + tw.write(chunk.toString(enc), enc); + }); + t.end(); +}); + +test('write no bufferize', function(t) { + var tw = new TestWriter({ + lowWaterMark: 5, + highWaterMark: 100, + decodeStrings: false + }); + + tw._write = function(chunk, cb) { + assert(Array.isArray(chunk)); + assert(typeof chunk[0] === 'string'); + chunk = new Buffer(chunk[0], chunk[1]); + return TestWriter.prototype._write.call(this, chunk, cb); + }; + + var encodings = + [ 'hex', + 'utf8', + 'utf-8', + 'ascii', + 'binary', + 'base64', + 'ucs2', + 'ucs-2', + 'utf16le', + 'utf-16le', + undefined ]; + + tw.on('finish', function() { + t.same(tw.buffer, chunks, 'got the expected chunks'); + }); + + chunks.forEach(function(chunk, i) { + var enc = encodings[ i % encodings.length ]; + chunk = new Buffer(chunk); + tw.write(chunk.toString(enc), enc); + }); + t.end(); +}); + +test('write callbacks', function (t) { + var callbacks = chunks.map(function(chunk, i) { + return [i, function(er) { + callbacks._called[i] = chunk; + }]; + }).reduce(function(set, x) { + set['callback-' + x[0]] = x[1]; + return set; + }, {}); + callbacks._called = []; + + var tw = new TestWriter({ + lowWaterMark: 5, + highWaterMark: 100 + }); + + tw.on('finish', function() { + t.same(tw.buffer, chunks, 'got chunks in the right order'); + t.same(callbacks._called, chunks, 'called all callbacks'); + t.end(); + }); + + chunks.forEach(function(chunk, i) { + tw.write(chunk, callbacks['callback-' + i]); + }); + tw.end(); +}); From e82d06bef9c2de480f8c21ea195c876811b275e1 Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 9 Oct 2012 17:42:47 -0700 Subject: [PATCH 18/72] streams2: Fix regression from Duplex ctor assignment --- lib/_stream_writable.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/_stream_writable.js b/lib/_stream_writable.js index a84cbc3785e..1f6f5112a03 100644 --- a/lib/_stream_writable.js +++ b/lib/_stream_writable.js @@ -27,7 +27,6 @@ module.exports = Writable var util = require('util'); var Stream = require('stream'); -var Duplex = require('_stream_duplex'); util.inherits(Writable, Stream); @@ -60,7 +59,7 @@ function WritableState(options) { function Writable(options) { // Writable ctor is applied to Duplexes, though they're not // instanceof Writable, they're instanceof Readable. - if (!(this instanceof Writable) && !(this instanceof Duplex)) + if (!(this instanceof Writable) && !(this instanceof Stream.Duplex)) return new Writable(options); this._writableState = new WritableState(options); From acfb0ef908a595d133a47b6c0f73236a0596a487 Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 9 Oct 2012 21:56:02 -0700 Subject: [PATCH 19/72] test: fixture for streams2 testing --- test/fixtures/x1024.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 test/fixtures/x1024.txt diff --git a/test/fixtures/x1024.txt b/test/fixtures/x1024.txt new file mode 100644 index 00000000000..c6a9d2f1a50 --- /dev/null +++ b/test/fixtures/x1024.txt @@ -0,0 +1 @@ +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx \ No newline at end of file From cf0b4ba4105bb3f0377f5f5e5d1d60d929a143fb Mon Sep 17 00:00:00 2001 From: isaacs Date: Fri, 12 Oct 2012 10:03:03 -0700 Subject: [PATCH 20/72] streams2: flow() is not always bound to src --- lib/_stream_readable.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 180ab1d55b6..06797c4ea15 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -289,8 +289,8 @@ function flow(src, pipeOpts) { state.flowing = false; // if there were data event listeners added, then switch to old mode. - if (this.listeners('data').length) - emitDataEvents(this); + if (src.listeners('data').length) + emitDataEvents(src); return; } From f624ccb475aa0d0f39f7fdf9d40a08fd3b98242b Mon Sep 17 00:00:00 2001 From: isaacs Date: Fri, 12 Oct 2012 11:45:17 -0700 Subject: [PATCH 21/72] streams2: Use StringDecoder.end --- lib/_stream_readable.js | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 06797c4ea15..51201e82fec 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -153,6 +153,13 @@ Readable.prototype.read = function(n) { if (!chunk || !chunk.length) { // eof state.ended = true; + if (state.decoder) { + chunk = state.decoder.end(); + if (chunk && chunk.length) { + state.buffer.push(chunk); + state.length += chunk.length; + } + } // if we've ended and we have some data left, then emit // 'readable' now to make sure it gets picked up. if (!sync) { @@ -395,11 +402,26 @@ Readable.prototype.wrap = function(stream) { stream.on('end', function() { state.ended = true; - if (state.length === 0) + if (state.decoder) { + var chunk = state.decoder.end(); + if (chunk && chunk.length) { + state.buffer.push(chunk); + state.length += chunk.length; + } + } + + if (state.length > 0) + this.emit('readable'); + else endReadable(this); }.bind(this)); stream.on('data', function(chunk) { + if (state.decoder) + chunk = state.decoder.write(chunk); + if (!chunk || !chunk.length) + return; + state.buffer.push(chunk); state.length += chunk.length; this.emit('readable'); From 286aa04910f355699be2a99f89570fd6344e8d39 Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 31 Oct 2012 14:30:30 -0700 Subject: [PATCH 22/72] streams2: Abstract out onread function --- lib/_stream_readable.js | 114 +++++++++++++++++++++------------------- 1 file changed, 61 insertions(+), 53 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 51201e82fec..4e84eb57bf8 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -28,7 +28,7 @@ var StringDecoder; util.inherits(Readable, Stream); -function ReadableState(options) { +function ReadableState(options, stream) { options = options || {}; this.bufferSize = options.bufferSize || 16 * 1024; @@ -45,6 +45,8 @@ function ReadableState(options) { this.ended = false; this.endEmitted = false; this.reading = false; + this.sync = false; + this.onread = onread.bind(stream); // whenever we return null, then we set a flag to say // that we're awaiting a 'readable' event emission. @@ -144,59 +146,10 @@ Readable.prototype.read = function(n) { if (doRead) { var sync = true; state.reading = true; + state.sync = true; // call internal read method - this._read(state.bufferSize, function onread(er, chunk) { - state.reading = false; - if (er) - return this.emit('error', er); - - if (!chunk || !chunk.length) { - // eof - state.ended = true; - if (state.decoder) { - chunk = state.decoder.end(); - if (chunk && chunk.length) { - state.buffer.push(chunk); - state.length += chunk.length; - } - } - // if we've ended and we have some data left, then emit - // 'readable' now to make sure it gets picked up. - if (!sync) { - if (state.length > 0) - this.emit('readable'); - else - endReadable(this); - } - return; - } - - if (state.decoder) - chunk = state.decoder.write(chunk); - - // update the buffer info. - if (chunk) { - state.length += chunk.length; - state.buffer.push(chunk); - } - - // if we haven't gotten enough to pass the lowWaterMark, - // and we haven't ended, then don't bother telling the user - // that it's time to read more data. Otherwise, that'll - // probably kick off another stream.read(), which can trigger - // another _read(n,cb) before this one returns! - if (state.length <= state.lowWaterMark) { - state.reading = true; - this._read(state.bufferSize, onread.bind(this)); - return; - } - - if (state.needReadable && !sync) { - state.needReadable = false; - this.emit('readable'); - } - }.bind(this)); - sync = false; + this._read(state.bufferSize, state.onread); + state.sync = false; } // If _read called its callback synchronously, then `reading` @@ -221,6 +174,61 @@ Readable.prototype.read = function(n) { return ret; }; +function onread(er, chunk) { + var state = this._readableState; + var sync = state.sync; + + state.reading = false; + if (er) + return this.emit('error', er); + + if (!chunk || !chunk.length) { + // eof + state.ended = true; + if (state.decoder) { + chunk = state.decoder.end(); + if (chunk && chunk.length) { + state.buffer.push(chunk); + state.length += chunk.length; + } + } + // if we've ended and we have some data left, then emit + // 'readable' now to make sure it gets picked up. + if (!sync) { + if (state.length > 0) + this.emit('readable'); + else + endReadable(this); + } + return; + } + + if (state.decoder) + chunk = state.decoder.write(chunk); + + // update the buffer info. + if (chunk) { + state.length += chunk.length; + state.buffer.push(chunk); + } + + // if we haven't gotten enough to pass the lowWaterMark, + // and we haven't ended, then don't bother telling the user + // that it's time to read more data. Otherwise, that'll + // probably kick off another stream.read(), which can trigger + // another _read(n,cb) before this one returns! + if (state.length <= state.lowWaterMark) { + state.reading = true; + this._read(state.bufferSize, state.onread); + return; + } + + if (state.needReadable && !sync) { + state.needReadable = false; + this.emit('readable'); + } +} + // abstract method. to be overridden in specific implementation classes. // call cb(er, data) where data is <= n in length. // for virtual (non-string, non-buffer) streams, "length" is somewhat From 58568232234da60931817ccc98da2bf14d121177 Mon Sep 17 00:00:00 2001 From: isaacs Date: Mon, 12 Nov 2012 23:28:56 -0800 Subject: [PATCH 23/72] streams2: Fix duplex no-half-open logic --- lib/_stream_duplex.js | 27 +++------------------------ 1 file changed, 3 insertions(+), 24 deletions(-) diff --git a/lib/_stream_duplex.js b/lib/_stream_duplex.js index 30b46922332..ef74c34c7e2 100644 --- a/lib/_stream_duplex.js +++ b/lib/_stream_duplex.js @@ -47,38 +47,17 @@ function Duplex(options) { if (options && options.allowHalfOpen === false) this.allowHalfOpen = false; - this.once('finish', onfinish); this.once('end', onend); } -// the no-half-open enforcers. -function onfinish() { - // if we allow half-open state, or if the readable side ended, - // then we're ok. - if (this.allowHalfOpen || this._readableState.ended) - return; - - // mark that we're done. - this._readableState.ended = true; - - // tell the user - if (this._readableState.length === 0) - this.emit('end'); - else - this.emit('readable'); -} - +// the no-half-open enforcer function onend() { // if we allow half-open state, or if the writable side ended, // then we're ok. if (this.allowHalfOpen || this._writableState.ended) return; - // just in case the user is about to call write() again. - this.write = function() { - return false; - }; - // no more data can be written. - this.end(); + // But allow more writes to happen in this tick. + process.nextTick(this.end.bind(this)); } From 63ac07b32b0a6de8ea6e51d6ae52ea3b35f18935 Mon Sep 17 00:00:00 2001 From: isaacs Date: Mon, 12 Nov 2012 23:30:10 -0800 Subject: [PATCH 24/72] streams2: Export Readable/Writable State classes --- lib/_stream_readable.js | 1 + lib/_stream_writable.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 4e84eb57bf8..c228e52afbd 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -20,6 +20,7 @@ // USE OR OTHER DEALINGS IN THE SOFTWARE. module.exports = Readable; +Readable.ReadableState = ReadableState; var Stream = require('stream'); var util = require('util'); diff --git a/lib/_stream_writable.js b/lib/_stream_writable.js index 1f6f5112a03..744ec419452 100644 --- a/lib/_stream_writable.js +++ b/lib/_stream_writable.js @@ -23,7 +23,8 @@ // Implement an async ._write(chunk, cb), and it'll handle all // the drain event emission and buffering. -module.exports = Writable +module.exports = Writable; +Writable.WritableState = WritableState; var util = require('util'); var Stream = require('stream'); From f20fd22abd5e1147d7bb89ff9783169532644671 Mon Sep 17 00:00:00 2001 From: isaacs Date: Mon, 12 Nov 2012 23:31:25 -0800 Subject: [PATCH 25/72] streams2: Add high water mark for Readable Also, organize the numeric settings a bit on the ReadableState class --- lib/_stream_readable.js | 36 ++++++++++++++++++++++++++++++++---- lib/_stream_transform.js | 2 +- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index c228e52afbd..b1dd0cb67b4 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -32,13 +32,34 @@ util.inherits(Readable, Stream); function ReadableState(options, stream) { options = options || {}; - this.bufferSize = options.bufferSize || 16 * 1024; - assert(typeof this.bufferSize === 'number'); // cast to an int this.bufferSize = ~~this.bufferSize; + // the argument passed to this._read(n,cb) + this.bufferSize = options.hasOwnProperty('bufferSize') ? + options.bufferSize : 16 * 1024; + + // the point at which it stops calling _read() to fill the buffer + this.highWaterMark = options.hasOwnProperty('highWaterMark') ? + options.highWaterMark : 16 * 1024; + + // the minimum number of bytes to buffer before emitting 'readable' + // default to pushing everything out as fast as possible. this.lowWaterMark = options.hasOwnProperty('lowWaterMark') ? options.lowWaterMark : 1024; + + // cast to ints. + assert(typeof this.bufferSize === 'number'); + assert(typeof this.lowWaterMark === 'number'); + assert(typeof this.highWaterMark === 'number'); + this.bufferSize = ~~this.bufferSize; + this.lowWaterMark = ~~this.lowWaterMark; + this.highWaterMark = ~~this.highWaterMark; + assert(this.bufferSize >= 0); + assert(this.lowWaterMark >= 0); + assert(this.highWaterMark >= this.lowWaterMark, + this.highWaterMark + '>=' + this.lowWaterMark); + this.buffer = []; this.length = 0; this.pipes = []; @@ -136,9 +157,16 @@ Readable.prototype.read = function(n) { // if we need a readable event, then we need to do some reading. var doRead = state.needReadable; - // if we currently have less than the lowWaterMark, then also read some - if (state.length - n <= state.lowWaterMark) + + // if we currently have less than the highWaterMark, then also read some + if (state.length - n <= state.highWaterMark) doRead = true; + + // if we currently have *nothing*, then always try to get *something* + // no matter what the high water mark says. + if (state.length === 0) + doRead = true; + // however, if we've ended, then there's no point, and if we're already // reading, then it's unnecessary. if (state.ended || state.reading) diff --git a/lib/_stream_transform.js b/lib/_stream_transform.js index abf7479fdce..cf1c2e3b0eb 100644 --- a/lib/_stream_transform.js +++ b/lib/_stream_transform.js @@ -130,7 +130,7 @@ Transform.prototype._write = function(chunk, cb) { // if we weren't waiting for it, but nothing is queued up, then // still kick off a transform, just so it's there when the user asks. - var doRead = rs.needReadable || rs.length <= rs.lowWaterMark; + var doRead = rs.needReadable || rs.length <= rs.highWaterMark; if (doRead && !rs.reading) { var ret = this.read(0); if (ret !== null) From 62dd04027b67f02856713e080651bb24b38206f8 Mon Sep 17 00:00:00 2001 From: isaacs Date: Mon, 12 Nov 2012 23:31:40 -0800 Subject: [PATCH 26/72] streams2: Set Readable lwm to 0 by default --- lib/_stream_readable.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index b1dd0cb67b4..56a3e9dac38 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -46,7 +46,7 @@ function ReadableState(options, stream) { // the minimum number of bytes to buffer before emitting 'readable' // default to pushing everything out as fast as possible. this.lowWaterMark = options.hasOwnProperty('lowWaterMark') ? - options.lowWaterMark : 1024; + options.lowWaterMark : 0; // cast to ints. assert(typeof this.bufferSize === 'number'); From 286c54439a6cf200271e6bd7fe3c21d6b20501ed Mon Sep 17 00:00:00 2001 From: isaacs Date: Mon, 12 Nov 2012 23:32:05 -0800 Subject: [PATCH 27/72] streams2: Only emit 'readable' when needed --- lib/_stream_readable.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 56a3e9dac38..c598c91306f 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -73,6 +73,7 @@ function ReadableState(options, stream) { // whenever we return null, then we set a flag to say // that we're awaiting a 'readable' event emission. this.needReadable = false; + this.emittedReadable = false; this.decoder = null; if (options.encoding) { @@ -125,6 +126,9 @@ Readable.prototype.read = function(n) { var state = this._readableState; var nOrig = n; + if (typeof n !== 'number' || n > 0) + state.emittedReadable = false; + n = howMuchToRead(n, state); // if we've ended, and we're now clear, then finish it up. @@ -224,9 +228,13 @@ function onread(er, chunk) { // if we've ended and we have some data left, then emit // 'readable' now to make sure it gets picked up. if (!sync) { - if (state.length > 0) - this.emit('readable'); - else + if (state.length > 0) { + state.needReadable = false; + if (!state.emittedReadable) { + state.emittedReadable = true; + this.emit('readable'); + } + } else endReadable(this); } return; @@ -254,7 +262,10 @@ function onread(er, chunk) { if (state.needReadable && !sync) { state.needReadable = false; - this.emit('readable'); + if (!state.emittedReadable) { + state.emittedReadable = true; + this.emit('readable'); + } } } From 0118584433cb13717a509bbd67b22729dd4e8c98 Mon Sep 17 00:00:00 2001 From: isaacs Date: Mon, 12 Nov 2012 23:33:06 -0800 Subject: [PATCH 28/72] streams2: Writable organization, add 'finishing' flag --- lib/_stream_writable.js | 197 ++++++++++++++++++++++++++-------------- 1 file changed, 129 insertions(+), 68 deletions(-) diff --git a/lib/_stream_writable.js b/lib/_stream_writable.js index 744ec419452..cfcd2e25d7b 100644 --- a/lib/_stream_writable.js +++ b/lib/_stream_writable.js @@ -27,20 +27,41 @@ module.exports = Writable; Writable.WritableState = WritableState; var util = require('util'); +var assert = require('assert'); var Stream = require('stream'); util.inherits(Writable, Stream); -function WritableState(options) { +function WritableState(options, stream) { options = options || {}; - this.highWaterMark = options.highWaterMark || 16 * 1024; + + // the point at which write() starts returning false this.highWaterMark = options.hasOwnProperty('highWaterMark') ? options.highWaterMark : 16 * 1024; + + // the point that it has to get to before we call _write(chunk,cb) + // default to pushing everything out as fast as possible. this.lowWaterMark = options.hasOwnProperty('lowWaterMark') ? - options.lowWaterMark : 1024; + options.lowWaterMark : 0; + + // cast to ints. + assert(typeof this.lowWaterMark === 'number'); + assert(typeof this.highWaterMark === 'number'); + this.lowWaterMark = ~~this.lowWaterMark; + this.highWaterMark = ~~this.highWaterMark; + assert(this.lowWaterMark >= 0); + assert(this.highWaterMark >= this.lowWaterMark, + this.highWaterMark + '>=' + this.lowWaterMark); + this.needDrain = false; - this.ended = false; + // at the start of calling end() this.ending = false; + // when end() has been called, and returned + this.ended = false; + // when 'finish' has emitted + this.finished = false; + // when 'finish' is being emitted + this.finishing = false; // should we decode strings into buffers before passing to _write? // this is here so that some node-core streams can optimize string @@ -53,7 +74,22 @@ function WritableState(options) { // socket or file. this.length = 0; + // a flag to see when we're in the middle of a write. this.writing = false; + + // a flag to be able to tell if the onwrite cb is called immediately, + // or on a later tick. + this.sync = false; + + // the callback that's passed to _write(chunk,cb) + this.onwrite = onwrite.bind(stream); + + // the callback that the user supplies to write(chunk,encoding,cb) + this.writecb = null; + + // the amount that is being written when _write is called. + this.writelen = 0; + this.buffer = []; } @@ -63,7 +99,7 @@ function Writable(options) { if (!(this instanceof Writable) && !(this instanceof Stream.Duplex)) return new Writable(options); - this._writableState = new WritableState(options); + this._writableState = new WritableState(options, this); // legacy. this.writable = true; @@ -71,23 +107,26 @@ function Writable(options) { Stream.call(this); } -// Override this method for sync streams -// override the _write(chunk, cb) method for async streams +// Override this method or _write(chunk, cb) Writable.prototype.write = function(chunk, encoding, cb) { var state = this._writableState; - if (state.ended) { - this.emit('error', new Error('write after end')); - return; - } if (typeof encoding === 'function') { cb = encoding; encoding = null; } + if (state.ended) { + var er = new Error('write after end'); + if (typeof cb === 'function') + cb(er); + this.emit('error', er); + return; + } + var l = chunk.length; if (false === state.decodeStrings) - chunk = [chunk, encoding]; + chunk = [chunk, encoding || 'utf8']; else if (typeof chunk === 'string' || encoding) { chunk = new Buffer(chunk + '', encoding); l = chunk.length; @@ -107,70 +146,84 @@ Writable.prototype.write = function(chunk, encoding, cb) { } state.writing = true; - var sync = true; - this._write(chunk, writecb.bind(this)); - sync = false; + state.sync = true; + state.writelen = l; + state.writecb = cb; + this._write(chunk, state.onwrite); + state.sync = false; return ret; +}; - function writecb(er) { - state.writing = false; - if (er) { - if (cb) { - if (sync) - process.nextTick(cb.bind(null, er)); - else - cb(er); - } else - this.emit('error', er); - return; - } - state.length -= l; +function onwrite(er) { + var state = this._writableState; + var sync = state.sync; + var cb = state.writecb; + var l = state.writelen; + state.writing = false; + state.writelen = null; + state.writecb = null; + + if (er) { if (cb) { - // don't call the cb until the next tick if we're in sync mode. - // also, defer if we're about to write some more right now. - if (sync || state.buffer.length) - process.nextTick(cb); + if (sync) + process.nextTick(cb.bind(null, er)); else - cb(); - } + cb(er); + } else + this.emit('error', er); + return; + } + state.length -= l; - if (state.length === 0 && (state.ended || state.ending)) { - // emit 'finish' at the very end. - this.emit('finish'); - return; - } - - // if there's something in the buffer waiting, then do that, too. - if (state.buffer.length) { - var chunkCb = state.buffer.shift(); - chunk = chunkCb[0]; - cb = chunkCb[1]; - - if (false === state.decodeStrings) - l = chunk[0].length; - else - l = chunk.length; - - state.writing = true; - this._write(chunk, writecb.bind(this)); - } - - if (state.length <= state.lowWaterMark && state.needDrain) { - // Must force callback to be called on nextTick, so that we don't - // emit 'drain' before the write() consumer gets the 'false' return - // value, and has a chance to attach a 'drain' listener. - process.nextTick(function() { - if (!state.needDrain) - return; - state.needDrain = false; - this.emit('drain'); - }.bind(this)); - } + if (cb) { + // don't call the cb until the next tick if we're in sync mode. + // also, defer if we're about to write some more right now. + if (sync || state.buffer.length) + process.nextTick(cb); + else + cb(); } -}; + if (state.length === 0 && (state.ended || state.ending)) { + // emit 'finish' at the very end. + state.finishing = true; + this.emit('finish'); + state.finished = true; + return; + } + + // if there's something in the buffer waiting, then do that, too. + if (state.buffer.length) { + var chunkCb = state.buffer.shift(); + var chunk = chunkCb[0]; + cb = chunkCb[1]; + + if (false === state.decodeStrings) + l = chunk[0].length; + else + l = chunk.length; + + state.writelen = l; + state.writecb = cb; + state.writechunk = chunk; + state.writing = true; + this._write(chunk, state.onwrite); + } + + if (state.length <= state.lowWaterMark && state.needDrain) { + // Must force callback to be called on nextTick, so that we don't + // emit 'drain' before the write() consumer gets the 'false' return + // value, and has a chance to attach a 'drain' listener. + process.nextTick(function() { + if (!state.needDrain) + return; + state.needDrain = false; + this.emit('drain'); + }.bind(this)); + } +} Writable.prototype._write = function(chunk, cb) { process.nextTick(cb.bind(this, new Error('not implemented'))); @@ -178,10 +231,18 @@ Writable.prototype._write = function(chunk, cb) { Writable.prototype.end = function(chunk, encoding) { var state = this._writableState; + + // ignore unnecessary end() calls. + if (state.ending || state.ended || state.finished) + return; + state.ending = true; if (chunk) this.write(chunk, encoding); - else if (state.length === 0) + else if (state.length === 0) { + state.finishing = true; this.emit('finish'); + state.finished = true; + } state.ended = true; }; From c2f62d496a08e9d44bea4a459c2eab7457d724ee Mon Sep 17 00:00:00 2001 From: isaacs Date: Mon, 12 Nov 2012 23:41:17 -0800 Subject: [PATCH 29/72] test: Update stream2 transform for corrected behavior --- test/simple/test-stream2-transform.js | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/test/simple/test-stream2-transform.js b/test/simple/test-stream2-transform.js index 9b6365b5ca9..2bc008517cc 100644 --- a/test/simple/test-stream2-transform.js +++ b/test/simple/test-stream2-transform.js @@ -140,7 +140,7 @@ test('assymetric transform (expand)', function(t) { t.equal(pt.read(5).toString(), 'uelku'); t.equal(pt.read(5).toString(), 'el'); t.end(); - }, 100); + }, 200); }); test('assymetric transform (compress)', function(t) { @@ -223,9 +223,10 @@ test('passthrough event emission', function(t) { console.error('need emit 0'); pt.write(new Buffer('bazy')); + console.error('should have emitted, but not again'); pt.write(new Buffer('kuel')); - console.error('should have emitted readable now'); + console.error('should have emitted readable now 1 === %d', emits); t.equal(emits, 1); t.equal(pt.read(5).toString(), 'arkba'); @@ -263,21 +264,25 @@ test('passthrough event emission reordered', function(t) { console.error('need emit 0'); pt.once('readable', function() { t.equal(pt.read(5).toString(), 'arkba'); - t.equal(pt.read(5).toString(), 'zykue'); + t.equal(pt.read(5), null); console.error('need emit 1'); pt.once('readable', function() { - t.equal(pt.read(5).toString(), 'l'); + t.equal(pt.read(5).toString(), 'zykue'); t.equal(pt.read(5), null); - - t.equal(emits, 2); - t.end(); + pt.once('readable', function() { + t.equal(pt.read(5).toString(), 'l'); + t.equal(pt.read(5), null); + t.equal(emits, 3); + t.end(); + }); + pt.end(); }); - pt.end(); + pt.write(new Buffer('kuel')); }); + pt.write(new Buffer('bazy')); - pt.write(new Buffer('kuel')); }); test('passthrough facaded', function(t) { From 2ff499c022942e22fffc74709a2a99990bbec8ba Mon Sep 17 00:00:00 2001 From: isaacs Date: Sat, 17 Nov 2012 14:27:41 +1100 Subject: [PATCH 30/72] streams2: Do multipipe without always using forEach The Array.forEach call is too expensive. --- lib/_stream_readable.js | 100 +++++++++++++++++++++++------- test/simple/test-stream2-basic.js | 8 +++ 2 files changed, 86 insertions(+), 22 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index c598c91306f..e045ead396d 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -62,7 +62,8 @@ function ReadableState(options, stream) { this.buffer = []; this.length = 0; - this.pipes = []; + this.pipes = null; + this.pipesCount = 0; this.flowing = false; this.ended = false; this.endEmitted = false; @@ -282,7 +283,19 @@ Readable.prototype.pipe = function(dest, pipeOpts) { var state = this._readableState; if (!pipeOpts) pipeOpts = {}; - state.pipes.push(dest); + + switch (state.pipesCount) { + case 0: + state.pipes = dest; + break; + case 1: + state.pipes = [ state.pipes, dest ]; + break; + default: + state.pipes.push(dest); + break; + } + state.pipesCount += 1; if ((!pipeOpts || pipeOpts.end !== false) && dest !== process.stdout && @@ -320,15 +333,22 @@ function flow(src, pipeOpts) { flow(src, pipeOpts); } - while (state.pipes.length && + function write(dest, i, list) { + var written = dest.write(chunk); + if (false === written) { + needDrain++; + dest.once('drain', ondrain); + } + } + + while (state.pipesCount && null !== (chunk = src.read(pipeOpts.chunkSize))) { - state.pipes.forEach(function(dest, i, list) { - var written = dest.write(chunk); - if (false === written) { - needDrain++; - dest.once('drain', ondrain); - } - }); + + if (state.pipesCount === 1) + write(state.pipes, 0, null); + else + state.pipes.forEach(write); + src.emit('data', chunk); // if anyone needs a drain, then we have to wait for that. @@ -340,7 +360,7 @@ function flow(src, pipeOpts) { // function, or in the while loop, then stop flowing. // // NB: This is a pretty rare edge case. - if (state.pipes.length === 0) { + if (state.pipesCount === 0) { state.flowing = false; // if there were data event listeners added, then switch to old mode. @@ -356,19 +376,55 @@ function flow(src, pipeOpts) { Readable.prototype.unpipe = function(dest) { var state = this._readableState; - if (!dest) { - // remove all of them. - state.pipes.forEach(function(dest, i, list) { + + // if we're not piping anywhere, then do nothing. + if (state.pipesCount === 0) + return this; + + // just one destination. most common case. + if (state.pipesCount === 1) { + // passed in one, but it's not the right one. + if (dest && dest !== state.pipes) + return this; + + if (!dest) + dest = state.pipes; + + // got a match. + state.pipes = null; + state.pipesCount = 0; + if (dest) dest.emit('unpipe', this); - }, this); - state.pipes.length = 0; - } else { - var i = state.pipes.indexOf(dest); - if (i !== -1) { - dest.emit('unpipe', this); - state.pipes.splice(i, 1); - } + return this; } + + // slow case. multiple pipe destinations. + + if (!dest) { + // remove all. + var dests = state.pipes; + var len = state.pipesCount; + state.pipes = null; + state.pipesCount = 0; + + for (var i = 0; i < len; i++) + dests[i].emit('unpipe', this); + + return this; + } + + // try to find the right one. + var i = state.pipes.indexOf(dest); + if (i === -1) + return this; + + state.pipes.splice(i, 1); + state.pipesCount -= 1; + if (state.pipesCount === 1) + state.pipes = state.pipes[0]; + + dest.emit('unpipe', this); + return this; }; diff --git a/test/simple/test-stream2-basic.js b/test/simple/test-stream2-basic.js index c28cdc917a6..0b4f4cf2b5e 100644 --- a/test/simple/test-stream2-basic.js +++ b/test/simple/test-stream2-basic.js @@ -209,19 +209,27 @@ test('pipe', function(t) { w[0].on('write', function() { if (--writes === 0) { r.unpipe(); + t.equal(r._readableState.pipes, null); w[0].end(); r.pipe(w[1]); + t.equal(r._readableState.pipes, w[1]); } }); var ended = 0; + var ended0 = false; + var ended1 = false; w[0].on('end', function(results) { + t.equal(ended0, false); + ended0 = true; ended++; t.same(results, expect[0]); }); w[1].on('end', function(results) { + t.equal(ended1, false); + ended1 = true; ended++; t.equal(ended, 2); t.same(results, expect[1]); From b15e19a2324c3dc11192ad3d4b66e88d37b136f2 Mon Sep 17 00:00:00 2001 From: isaacs Date: Sat, 17 Nov 2012 15:24:14 +1100 Subject: [PATCH 31/72] streams2: Remove function.bind() usage It's too slow, unfortunately. --- lib/_stream_readable.js | 49 ++++++++++++++++++++++++---------------- lib/_stream_transform.js | 48 +++++++++++++++++++++------------------ lib/_stream_writable.js | 26 +++++++++++++-------- 3 files changed, 72 insertions(+), 51 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index e045ead396d..fff50d36fe0 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -69,7 +69,9 @@ function ReadableState(options, stream) { this.endEmitted = false; this.reading = false; this.sync = false; - this.onread = onread.bind(stream); + this.onread = function(er, data) { + onread(stream, er, data); + }; // whenever we return null, then we set a flag to say // that we're awaiting a 'readable' event emission. @@ -208,13 +210,13 @@ Readable.prototype.read = function(n) { return ret; }; -function onread(er, chunk) { - var state = this._readableState; +function onread(stream, er, chunk) { + var state = stream._readableState; var sync = state.sync; state.reading = false; if (er) - return this.emit('error', er); + return stream.emit('error', er); if (!chunk || !chunk.length) { // eof @@ -233,10 +235,10 @@ function onread(er, chunk) { state.needReadable = false; if (!state.emittedReadable) { state.emittedReadable = true; - this.emit('readable'); + stream.emit('readable'); } } else - endReadable(this); + endReadable(stream); } return; } @@ -257,7 +259,7 @@ function onread(er, chunk) { // another _read(n,cb) before this one returns! if (state.length <= state.lowWaterMark) { state.reading = true; - this._read(state.bufferSize, state.onread); + stream._read(state.bufferSize, state.onread); return; } @@ -265,7 +267,7 @@ function onread(er, chunk) { state.needReadable = false; if (!state.emittedReadable) { state.emittedReadable = true; - this.emit('readable'); + stream.emit('readable'); } } } @@ -275,7 +277,9 @@ function onread(er, chunk) { // for virtual (non-string, non-buffer) streams, "length" is somewhat // arbitrary, and perhaps not very meaningful. Readable.prototype._read = function(n, cb) { - process.nextTick(cb.bind(this, new Error('not implemented'))); + process.nextTick(function() { + cb(new Error('not implemented')); + }); }; Readable.prototype.pipe = function(dest, pipeOpts) { @@ -316,7 +320,9 @@ Readable.prototype.pipe = function(dest, pipeOpts) { // start the flow. if (!state.flowing) { state.flowing = true; - process.nextTick(flow.bind(null, src, pipeOpts)); + process.nextTick(function() { + flow(src, pipeOpts); + }); } return dest; @@ -371,7 +377,9 @@ function flow(src, pipeOpts) { // at this point, no one needed a drain, so we just ran out of data // on the next readable event, start it over again. - src.once('readable', flow.bind(null, src, pipeOpts)); + src.once('readable', function() { + flow(src, pipeOpts); + }); } Readable.prototype.unpipe = function(dest) { @@ -504,6 +512,7 @@ Readable.prototype.wrap = function(stream) { var state = this._readableState; var paused = false; + var self = this; stream.on('end', function() { state.ended = true; if (state.decoder) { @@ -515,10 +524,10 @@ Readable.prototype.wrap = function(stream) { } if (state.length > 0) - this.emit('readable'); + self.emit('readable'); else - endReadable(this); - }.bind(this)); + endReadable(self); + }); stream.on('data', function(chunk) { if (state.decoder) @@ -528,14 +537,14 @@ Readable.prototype.wrap = function(stream) { state.buffer.push(chunk); state.length += chunk.length; - this.emit('readable'); + self.emit('readable'); // if not consumed, then pause the stream. if (state.length > state.lowWaterMark && !paused) { paused = true; stream.pause(); } - }.bind(this)); + }); // proxy all the other methods. // important when wrapping filters and duplexes. @@ -551,8 +560,8 @@ Readable.prototype.wrap = function(stream) { // proxy certain important events. var events = ['error', 'close', 'destroy', 'pause', 'resume']; events.forEach(function(ev) { - stream.on(ev, this.emit.bind(this, ev)); - }.bind(this)); + stream.on(ev, self.emit.bind(self, ev)); + }); // consume some bytes. if not all is consumed, then // pause the underlying stream. @@ -660,5 +669,7 @@ function endReadable(stream) { return; state.ended = true; state.endEmitted = true; - process.nextTick(stream.emit.bind(stream, 'end')); + process.nextTick(function() { + stream.emit('end'); + }); } diff --git a/lib/_stream_transform.js b/lib/_stream_transform.js index cf1c2e3b0eb..b0819de29a5 100644 --- a/lib/_stream_transform.js +++ b/lib/_stream_transform.js @@ -70,10 +70,13 @@ var Duplex = require('_stream_duplex'); var util = require('util'); util.inherits(Transform, Duplex); -function TransformState() { +function TransformState(stream) { this.buffer = []; this.transforming = false; this.pendingReadCb = null; + this.output = function(chunk) { + stream._output(chunk); + }; } function Transform(options) { @@ -83,17 +86,19 @@ function Transform(options) { Duplex.call(this, options); // bind output so that it can be passed around as a regular function. - this._output = this._output.bind(this); + var stream = this; // the queue of _write chunks that are pending being transformed - this._transformState = new TransformState(); + var ts = this._transformState = new TransformState(stream); // when the writable side finishes, then flush out anything remaining. this.once('finish', function() { if ('function' === typeof this._flush) - this._flush(this._output, done.bind(this)); + this._flush(ts.output, function(er) { + done(stream, er); + }); else - done.call(this); + done(stream); }); } @@ -159,14 +164,13 @@ Transform.prototype._read = function(n, readcb) { var req = ts.buffer.shift(); var chunk = req[0]; var writecb = req[1]; - var output = this._output; ts.transforming = true; - this._transform(chunk, output, function(er, data) { + this._transform(chunk, ts.output, function(er, data) { ts.transforming = false; if (data) - output(data); + ts.output(data); writecb(er); - }.bind(this)); + }); }; Transform.prototype._output = function(chunk) { @@ -185,25 +189,25 @@ Transform.prototype._output = function(chunk) { } // otherwise, it's up to us to fill the rs buffer. - var state = this._readableState; - var len = state.length; - state.buffer.push(chunk); - state.length += chunk.length; - if (state.needReadable) { - state.needReadable = false; + var rs = this._readableState; + var len = rs.length; + rs.buffer.push(chunk); + rs.length += chunk.length; + if (rs.needReadable) { + rs.needReadable = false; this.emit('readable'); } }; -function done(er) { +function done(stream, er) { if (er) - return this.emit('error', er); + return stream.emit('error', er); // if there's nothing in the write buffer, then that means // that nothing more will ever be provided - var ws = this._writableState; - var rs = this._readableState; - var ts = this._transformState; + var ws = stream._writableState; + var rs = stream._readableState; + var ts = stream._transformState; if (ws.length) throw new Error('calling transform done when ws.length != 0'); @@ -221,7 +225,7 @@ function done(er) { // no more data coming from the writable side, we need to emit // now so that the consumer knows to pick up the tail bits. if (rs.length && rs.needReadable) - this.emit('readable'); + stream.emit('readable'); else if (rs.length === 0) - this.emit('end'); + stream.emit('end'); } diff --git a/lib/_stream_writable.js b/lib/_stream_writable.js index cfcd2e25d7b..00702cab9a3 100644 --- a/lib/_stream_writable.js +++ b/lib/_stream_writable.js @@ -82,7 +82,9 @@ function WritableState(options, stream) { this.sync = false; // the callback that's passed to _write(chunk,cb) - this.onwrite = onwrite.bind(stream); + this.onwrite = function(er) { + onwrite(stream, er); + }; // the callback that the user supplies to write(chunk,encoding,cb) this.writecb = null; @@ -155,8 +157,8 @@ Writable.prototype.write = function(chunk, encoding, cb) { return ret; }; -function onwrite(er) { - var state = this._writableState; +function onwrite(stream, er) { + var state = stream._writableState; var sync = state.sync; var cb = state.writecb; var l = state.writelen; @@ -168,11 +170,13 @@ function onwrite(er) { if (er) { if (cb) { if (sync) - process.nextTick(cb.bind(null, er)); + process.nextTick(function() { + cb(er); + }); else cb(er); } else - this.emit('error', er); + stream.emit('error', er); return; } state.length -= l; @@ -189,7 +193,7 @@ function onwrite(er) { if (state.length === 0 && (state.ended || state.ending)) { // emit 'finish' at the very end. state.finishing = true; - this.emit('finish'); + stream.emit('finish'); state.finished = true; return; } @@ -209,7 +213,7 @@ function onwrite(er) { state.writecb = cb; state.writechunk = chunk; state.writing = true; - this._write(chunk, state.onwrite); + stream._write(chunk, state.onwrite); } if (state.length <= state.lowWaterMark && state.needDrain) { @@ -220,13 +224,15 @@ function onwrite(er) { if (!state.needDrain) return; state.needDrain = false; - this.emit('drain'); - }.bind(this)); + stream.emit('drain'); + }); } } Writable.prototype._write = function(chunk, cb) { - process.nextTick(cb.bind(this, new Error('not implemented'))); + process.nextTick(function() { + cb(new Error('not implemented')); + }); }; Writable.prototype.end = function(chunk, encoding) { From 38e2b0053a0894b28c588a7e8b99f3842952574a Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 27 Nov 2012 18:20:16 -0800 Subject: [PATCH 32/72] streams2: Get rid of .once() usage in Readable.pipe Significant performance impact --- lib/_stream_readable.js | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index fff50d36fe0..abb23e41c70 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -78,6 +78,11 @@ function ReadableState(options, stream) { this.needReadable = false; this.emittedReadable = false; + // when piping, we only care about 'readable' events that happen + // after read()ing all the bytes and not getting any pushback. + this.ranOut = false; + this.flowChunkSize = null; + this.decoder = null; if (options.encoding) { if (!StringDecoder) @@ -285,8 +290,6 @@ Readable.prototype._read = function(n, cb) { Readable.prototype.pipe = function(dest, pipeOpts) { var src = this; var state = this._readableState; - if (!pipeOpts) - pipeOpts = {}; switch (state.pipesCount) { case 0: @@ -311,6 +314,9 @@ Readable.prototype.pipe = function(dest, pipeOpts) { }); } + if (pipeOpts && pipeOpts.chunkSize) + state.flowChunkSize = pipeOpts.chunkSize; + function onend() { dest.end(); } @@ -319,16 +325,22 @@ Readable.prototype.pipe = function(dest, pipeOpts) { // start the flow. if (!state.flowing) { + // the handler that waits for readable events after all + // the data gets sucked out in flow. + // This would be easier to follow with a .once() handler + // in flow(), but that is too slow. + this.on('readable', pipeOnReadable); + state.flowing = true; process.nextTick(function() { - flow(src, pipeOpts); + flow(src); }); } return dest; }; -function flow(src, pipeOpts) { +function flow(src) { var state = src._readableState; var chunk; var needDrain = 0; @@ -336,7 +348,7 @@ function flow(src, pipeOpts) { function ondrain() { needDrain--; if (needDrain === 0) - flow(src, pipeOpts); + flow(src); } function write(dest, i, list) { @@ -348,7 +360,7 @@ function flow(src, pipeOpts) { } while (state.pipesCount && - null !== (chunk = src.read(pipeOpts.chunkSize))) { + null !== (chunk = src.read(state.pipeChunkSize))) { if (state.pipesCount === 1) write(state.pipes, 0, null); @@ -377,11 +389,17 @@ function flow(src, pipeOpts) { // at this point, no one needed a drain, so we just ran out of data // on the next readable event, start it over again. - src.once('readable', function() { - flow(src, pipeOpts); - }); + state.ranOut = true; } +function pipeOnReadable() { + if (this._readableState.ranOut) { + this._readableState.ranOut = false; + flow(this); + } +} + + Readable.prototype.unpipe = function(dest) { var state = this._readableState; @@ -401,6 +419,7 @@ Readable.prototype.unpipe = function(dest) { // got a match. state.pipes = null; state.pipesCount = 0; + this.removeListener('readable', pipeOnReadable); if (dest) dest.emit('unpipe', this); return this; @@ -414,10 +433,10 @@ Readable.prototype.unpipe = function(dest) { var len = state.pipesCount; state.pipes = null; state.pipesCount = 0; + this.removeListener('readable', pipeOnReadable); for (var i = 0; i < len; i++) dests[i].emit('unpipe', this); - return this; } From 4b4ff2dff16ad4f5e7aa6e82673962d219057a70 Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 28 Nov 2012 01:25:39 -0800 Subject: [PATCH 33/72] streams2: Refactor out .once() usage from Readable.pipe() --- lib/_stream_readable.js | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index abb23e41c70..9c242990d4f 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -81,6 +81,9 @@ function ReadableState(options, stream) { // when piping, we only care about 'readable' events that happen // after read()ing all the bytes and not getting any pushback. this.ranOut = false; + + // the number of writers that are awaiting a drain event in .pipe()s + this.awaitDrain = 0; this.flowChunkSize = null; this.decoder = null; @@ -330,6 +333,19 @@ Readable.prototype.pipe = function(dest, pipeOpts) { // This would be easier to follow with a .once() handler // in flow(), but that is too slow. this.on('readable', pipeOnReadable); + var ondrain = pipeOnDrain(src); + dest.on('drain', ondrain); + dest.on('unpipe', function(readable) { + if (readable === src) + dest.removeListener('drain', ondrain); + + // if the reader is waiting for a drain event from this + // specific writer, then it would cause it to never start + // flowing again. + // So, if this is awaiting a drain, then we just call it now. + if (dest._writableState.needDrain) + ondrain(); + }); state.flowing = true; process.nextTick(function() { @@ -340,22 +356,25 @@ Readable.prototype.pipe = function(dest, pipeOpts) { return dest; }; +function pipeOnDrain(src) { + return function() { + var dest = this; + var state = src._readableState; + state.awaitDrain --; + if (state.awaitDrain === 0) + flow(src); + }; +} + function flow(src) { var state = src._readableState; var chunk; - var needDrain = 0; - - function ondrain() { - needDrain--; - if (needDrain === 0) - flow(src); - } + state.awaitDrain = 0; function write(dest, i, list) { var written = dest.write(chunk); if (false === written) { - needDrain++; - dest.once('drain', ondrain); + state.awaitDrain++; } } @@ -370,7 +389,7 @@ function flow(src) { src.emit('data', chunk); // if anyone needs a drain, then we have to wait for that. - if (needDrain > 0) + if (state.awaitDrain > 0) return; } From 53fa66d9f79b4ba593f071334d58fda766fa5be3 Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 27 Nov 2012 18:21:05 -0800 Subject: [PATCH 34/72] streams2: Set 'readable' flag on Readable streams --- lib/_stream_readable.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 9c242990d4f..20ce5ed2db3 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -99,6 +99,10 @@ function Readable(options) { return new Readable(options); this._readableState = new ReadableState(options, this); + + // legacy + this.readable = true; + Stream.apply(this); } From ac5a185edf16efb7738c00f4d35f00bc2f93bf04 Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 28 Nov 2012 10:46:24 -0800 Subject: [PATCH 35/72] streams2: Handle pipeChunkSize properly --- lib/_stream_readable.js | 45 ++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 20ce5ed2db3..53b920a7506 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -84,7 +84,7 @@ function ReadableState(options, stream) { // the number of writers that are awaiting a drain event in .pipe()s this.awaitDrain = 0; - this.flowChunkSize = null; + this.pipeChunkSize = null; this.decoder = null; if (options.encoding) { @@ -118,7 +118,7 @@ function howMuchToRead(n, state) { if (state.length === 0 && state.ended) return 0; - if (isNaN(n)) + if (isNaN(n) || n === null) return state.length; if (n <= 0) @@ -266,8 +266,8 @@ function onread(stream, er, chunk) { // if we haven't gotten enough to pass the lowWaterMark, // and we haven't ended, then don't bother telling the user - // that it's time to read more data. Otherwise, that'll - // probably kick off another stream.read(), which can trigger + // that it's time to read more data. Otherwise, emitting 'readable' + // probably will trigger another stream.read(), which can trigger // another _read(n,cb) before this one returns! if (state.length <= state.lowWaterMark) { state.reading = true; @@ -322,34 +322,41 @@ Readable.prototype.pipe = function(dest, pipeOpts) { } if (pipeOpts && pipeOpts.chunkSize) - state.flowChunkSize = pipeOpts.chunkSize; + state.pipeChunkSize = pipeOpts.chunkSize; function onend() { dest.end(); } + // when the dest drains, it reduces the awaitDrain counter + // on the source. This would be more elegant with a .once() + // handler in flow(), but adding and removing repeatedly is + // too slow. + var ondrain = pipeOnDrain(src); + dest.on('drain', ondrain); + dest.on('unpipe', function(readable) { + if (readable === src) + dest.removeListener('drain', ondrain); + + // if the reader is waiting for a drain event from this + // specific writer, then it would cause it to never start + // flowing again. + // So, if this is awaiting a drain, then we just call it now. + // If we don't know, then assume that we are waiting for one. + if (!dest._writableState || dest._writableState.needDrain) + ondrain(); + }); + + // tell the dest that it's being piped to dest.emit('pipe', src); - // start the flow. + // start the flow if it hasn't been started already. if (!state.flowing) { // the handler that waits for readable events after all // the data gets sucked out in flow. // This would be easier to follow with a .once() handler // in flow(), but that is too slow. this.on('readable', pipeOnReadable); - var ondrain = pipeOnDrain(src); - dest.on('drain', ondrain); - dest.on('unpipe', function(readable) { - if (readable === src) - dest.removeListener('drain', ondrain); - - // if the reader is waiting for a drain event from this - // specific writer, then it would cause it to never start - // flowing again. - // So, if this is awaiting a drain, then we just call it now. - if (dest._writableState.needDrain) - ondrain(); - }); state.flowing = true; process.nextTick(function() { From 49ea653363da50c76a099839b4af555cec1d06c8 Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 28 Nov 2012 20:45:16 -0800 Subject: [PATCH 36/72] streams2: Remove pipe if the dest emits error --- lib/_stream_readable.js | 8 ++ .../test-stream2-pipe-error-handling.js | 105 ++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 test/simple/test-stream2-pipe-error-handling.js diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 53b920a7506..814995364e7 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -347,6 +347,14 @@ Readable.prototype.pipe = function(dest, pipeOpts) { ondrain(); }); + // if the dest has an error, then stop piping into it. + // however, don't suppress the throwing behavior for this. + dest.once('error', function(er) { + src.unpipe(dest); + if (dest.listeners('error').length === 0) + dest.emit('error', er); + }); + // tell the dest that it's being piped to dest.emit('pipe', src); diff --git a/test/simple/test-stream2-pipe-error-handling.js b/test/simple/test-stream2-pipe-error-handling.js new file mode 100644 index 00000000000..c17139f5d37 --- /dev/null +++ b/test/simple/test-stream2-pipe-error-handling.js @@ -0,0 +1,105 @@ +// Copyright Joyent, Inc. and other Node contributors. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the +// "Software"), to deal in the Software without restriction, including +// without limitation the rights to use, copy, modify, merge, publish, +// distribute, sublicense, and/or sell copies of the Software, and to permit +// persons to whom the Software is furnished to do so, subject to the +// following conditions: +// +// The above copyright notice and this permission notice shall be included +// in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +// USE OR OTHER DEALINGS IN THE SOFTWARE. + +var common = require('../common'); +var assert = require('assert'); +var stream = require('stream'); + +(function testErrorListenerCatches() { + var count = 1000; + + var source = new stream.Readable(); + source._read = function(n, cb) { + n = Math.min(count, n); + count -= n; + cb(null, new Buffer(n)); + }; + + var unpipedDest; + source.unpipe = function(dest) { + unpipedDest = dest; + stream.Readable.prototype.unpipe.call(this, dest); + }; + + var dest = new stream.Writable(); + dest._write = function(chunk, cb) { + cb(); + }; + + source.pipe(dest); + + var gotErr = null; + dest.on('error', function(err) { + gotErr = err; + }); + + var unpipedSource; + dest.on('unpipe', function(src) { + unpipedSource = src; + }); + + var err = new Error('This stream turned into bacon.'); + dest.emit('error', err); + assert.strictEqual(gotErr, err); + assert.strictEqual(unpipedSource, source); + assert.strictEqual(unpipedDest, dest); +})(); + +(function testErrorWithoutListenerThrows() { + var count = 1000; + + var source = new stream.Readable(); + source._read = function(n, cb) { + n = Math.min(count, n); + count -= n; + cb(null, new Buffer(n)); + }; + + var unpipedDest; + source.unpipe = function(dest) { + unpipedDest = dest; + stream.Readable.prototype.unpipe.call(this, dest); + }; + + var dest = new stream.Writable(); + dest._write = function(chunk, cb) { + cb(); + }; + + source.pipe(dest); + + var unpipedSource; + dest.on('unpipe', function(src) { + unpipedSource = src; + }); + + var err = new Error('This stream turned into bacon.'); + + var gotErr = null; + try { + dest.emit('error', err); + } catch (e) { + gotErr = e; + } + assert.strictEqual(gotErr, err); + assert.strictEqual(unpipedSource, source); + assert.strictEqual(unpipedDest, dest); +})(); From d58f2654bce637dea9a27ee1dd9dbec1ad7614cf Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 28 Nov 2012 22:09:28 -0800 Subject: [PATCH 37/72] streams2: Unpipe on dest.emit('close') --- lib/_stream_readable.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 814995364e7..3e9253aa7df 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -350,11 +350,22 @@ Readable.prototype.pipe = function(dest, pipeOpts) { // if the dest has an error, then stop piping into it. // however, don't suppress the throwing behavior for this. dest.once('error', function(er) { - src.unpipe(dest); + unpipe(); if (dest.listeners('error').length === 0) dest.emit('error', er); }); + // if the dest emits close, then presumably there's no point writing + // to it any more. + dest.on('close', unpipe); + dest.on('finish', function() { + dest.removeListener('close', unpipe); + }); + + function unpipe() { + src.unpipe(dest); + } + // tell the dest that it's being piped to dest.emit('pipe', src); From 44b308b1f7beaa3398d869f1626f01ae6526bc0b Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 4 Oct 2012 17:44:48 -0700 Subject: [PATCH 38/72] fs: streams2 --- lib/fs.js | 439 +++++++++---------------- test/simple/test-file-write-stream.js | 35 +- test/simple/test-file-write-stream2.js | 40 ++- test/simple/test-fs-read-stream.js | 4 +- 4 files changed, 197 insertions(+), 321 deletions(-) diff --git a/lib/fs.js b/lib/fs.js index 83bacc932f2..96af63186c9 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -34,6 +34,9 @@ var fs = exports; var Stream = require('stream').Stream; var EventEmitter = require('events').EventEmitter; +var Readable = Stream.Readable; +var Writable = Stream.Writable; + var kMinPoolSpace = 128; var kPoolSize = 40 * 1024; @@ -1386,34 +1389,30 @@ fs.createReadStream = function(path, options) { return new ReadStream(path, options); }; -var ReadStream = fs.ReadStream = function(path, options) { - if (!(this instanceof ReadStream)) return new ReadStream(path, options); +util.inherits(ReadStream, Readable); +fs.ReadStream = ReadStream; - Stream.call(this); +function ReadStream(path, options) { + if (!(this instanceof ReadStream)) + return new ReadStream(path, options); - var self = this; + // a little bit bigger buffer and water marks by default + options = util._extend({ + bufferSize: 64 * 1024, + lowWaterMark: 16 * 1024, + highWaterMark: 64 * 1024 + }, options || {}); + + Readable.call(this, options); this.path = path; - this.fd = null; - this.readable = true; - this.paused = false; + this.fd = options.hasOwnProperty('fd') ? options.fd : null; + this.flags = options.hasOwnProperty('flags') ? options.flags : 'r'; + this.mode = options.hasOwnProperty('mode') ? options.mode : 438; /*=0666*/ - this.flags = 'r'; - this.mode = 438; /*=0666*/ - this.bufferSize = 64 * 1024; - - options = options || {}; - - // Mixin options into this - var keys = Object.keys(options); - for (var index = 0, length = keys.length; index < length; index++) { - var key = keys[index]; - this[key] = options[key]; - } - - assertEncoding(this.encoding); - - if (this.encoding) this.setEncoding(this.encoding); + this.start = options.hasOwnProperty('start') ? options.start : undefined; + this.end = options.hasOwnProperty('start') ? options.end : undefined; + this.pos = undefined; if (this.start !== undefined) { if ('number' !== typeof this.start) { @@ -1432,41 +1431,40 @@ var ReadStream = fs.ReadStream = function(path, options) { this.pos = this.start; } - if (this.fd !== null) { - process.nextTick(function() { - self._read(); - }); - return; - } + if (typeof this.fd !== 'number') + this.open(); - fs.open(this.path, this.flags, this.mode, function(err, fd) { - if (err) { - self.emit('error', err); - self.readable = false; + this.on('end', function() { + this.destroy(); + }); +} + +fs.FileReadStream = fs.ReadStream; // support the legacy name + +ReadStream.prototype.open = function() { + var self = this; + fs.open(this.path, this.flags, this.mode, function(er, fd) { + if (er) { + self.destroy(); + self.emit('error', er); return; } self.fd = fd; self.emit('open', fd); - self._read(); + // start the flow of data. + self.read(); }); }; -util.inherits(ReadStream, Stream); -fs.FileReadStream = fs.ReadStream; // support the legacy name +ReadStream.prototype._read = function(n, cb) { + if (typeof this.fd !== 'number') + return this.once('open', function() { + this._read(n, cb); + }); -ReadStream.prototype.setEncoding = function(encoding) { - assertEncoding(encoding); - var StringDecoder = require('string_decoder').StringDecoder; // lazy load - this._decoder = new StringDecoder(encoding); -}; - - -ReadStream.prototype._read = function() { - var self = this; - if (!this.readable || this.paused || this.reading) return; - - this.reading = true; + if (this.destroyed) + return; if (!pool || pool.length - pool.used < kMinPoolSpace) { // discard the old pool. Can't add to the free list because @@ -1475,150 +1473,111 @@ ReadStream.prototype._read = function() { allocNewPool(); } - // Grab another reference to the pool in the case that while we're in the - // thread pool another read() finishes up the pool, and allocates a new - // one. + // Grab another reference to the pool in the case that while we're + // in the thread pool another read() finishes up the pool, and + // allocates a new one. var thisPool = pool; - var toRead = Math.min(pool.length - pool.used, ~~this.bufferSize); + var toRead = Math.min(pool.length - pool.used, n); var start = pool.used; - if (this.pos !== undefined) { + if (this.pos !== undefined) toRead = Math.min(this.end - this.pos + 1, toRead); - } - function afterRead(err, bytesRead) { - self.reading = false; - if (err) { - fs.close(self.fd, function() { - self.fd = null; - self.emit('error', err); - self.readable = false; - }); - return; - } + // already read everything we were supposed to read! + // treat as EOF. + if (toRead <= 0) + return cb(); - if (bytesRead === 0) { - if (this._decoder) { - var ret = this._decoder.end(); - if (ret) - this.emit('data', ret); - } - self.emit('end'); - self.destroy(); - return; - } + // the actual read. + var self = this; + fs.read(this.fd, pool, pool.used, toRead, this.pos, onread); - var b = thisPool.slice(start, start + bytesRead); - - // Possible optimizition here? - // Reclaim some bytes if bytesRead < toRead? - // Would need to ensure that pool === thisPool. - - // do not emit events if the stream is paused - if (self.paused) { - self.buffer = b; - return; - } - - // do not emit events anymore after we declared the stream unreadable - if (!self.readable) return; - - self._emitData(b); - self._read(); - } - - fs.read(this.fd, pool, pool.used, toRead, this.pos, afterRead); - - if (this.pos !== undefined) { + // move the pool positions, and internal position for reading. + if (this.pos !== undefined) this.pos += toRead; - } pool.used += toRead; -}; + function onread(er, bytesRead) { + if (er) { + self.destroy(); + return cb(er); + } -ReadStream.prototype._emitData = function(d) { - if (this._decoder) { - var string = this._decoder.write(d); - if (string.length) this.emit('data', string); - } else { - this.emit('data', d); + var b = null; + if (bytesRead > 0) + b = thisPool.slice(start, start + bytesRead); + + cb(null, b); } }; ReadStream.prototype.destroy = function() { - var self = this; + if (this.destroyed) + return; + this.destroyed = true; - if (!this.readable) return; - this.readable = false; + if ('number' === typeof this.fd) + this.close(); +}; + + +ReadStream.prototype.close = function(cb) { + if (cb) + this.once('close', cb); + if (this.closed || 'number' !== typeof this.fd) { + if ('number' !== typeof this.fd) + this.once('open', close); + return process.nextTick(this.emit.bind(this, 'close')); + } + this.closed = true; + var self = this; + close(); function close() { - fs.close(self.fd, function(err) { - if (err) { - self.emit('error', err); - } else { + fs.close(self.fd, function(er) { + if (er) + self.emit('error', er); + else self.emit('close'); - } }); - } - - if (this.fd === null) { - this.addListener('open', close); - } else { - close(); + self.fd = null; } }; -ReadStream.prototype.pause = function() { - this.paused = true; -}; - - -ReadStream.prototype.resume = function() { - this.paused = false; - - if (this.buffer) { - var buffer = this.buffer; - this.buffer = null; - this._emitData(buffer); - } - - // hasn't opened yet. - if (null == this.fd) return; - - this._read(); -}; - fs.createWriteStream = function(path, options) { return new WriteStream(path, options); }; -var WriteStream = fs.WriteStream = function(path, options) { - if (!(this instanceof WriteStream)) return new WriteStream(path, options); +util.inherits(WriteStream, Writable); +fs.WriteStream = WriteStream; +function WriteStream(path, options) { + if (!(this instanceof WriteStream)) + return new WriteStream(path, options); - Stream.call(this); + // a little bit bigger buffer and water marks by default + options = util._extend({ + bufferSize: 64 * 1024, + lowWaterMark: 16 * 1024, + highWaterMark: 64 * 1024 + }, options || {}); + + Writable.call(this, options); this.path = path; this.fd = null; - this.writable = true; - this.flags = 'w'; - this.encoding = 'binary'; - this.mode = 438; /*=0666*/ + this.fd = options.hasOwnProperty('fd') ? options.fd : null; + this.flags = options.hasOwnProperty('flags') ? options.flags : 'w'; + this.mode = options.hasOwnProperty('mode') ? options.mode : 438; /*=0666*/ + + this.start = options.hasOwnProperty('start') ? options.start : undefined; + this.pos = undefined; this.bytesWritten = 0; - options = options || {}; - - // Mixin options into this - var keys = Object.keys(options); - for (var index = 0, length = keys.length; index < length; index++) { - var key = keys[index]; - this[key] = options[key]; - } - if (this.start !== undefined) { if ('number' !== typeof this.start) { throw TypeError('start must be a Number'); @@ -1630,154 +1589,54 @@ var WriteStream = fs.WriteStream = function(path, options) { this.pos = this.start; } - this.busy = false; - this._queue = []; + if ('number' !== typeof this.fd) + this.open(); - if (this.fd === null) { - this._open = fs.open; - this._queue.push([this._open, this.path, this.flags, this.mode, undefined]); - this.flush(); - } -}; -util.inherits(WriteStream, Stream); + // dispose on finish. + this.once('finish', this.close); +} fs.FileWriteStream = fs.WriteStream; // support the legacy name -WriteStream.prototype.flush = function() { - if (this.busy) return; + +WriteStream.prototype.open = function() { + fs.open(this.path, this.flags, this.mode, function(er, fd) { + if (er) { + this.destroy(); + this.emit('error', er); + return; + } + + this.fd = fd; + this.emit('open', fd); + }.bind(this)); +}; + + +WriteStream.prototype._write = function(data, cb) { + if (!Buffer.isBuffer(data)) + return this.emit('error', new Error('Invalid data')); + + if (typeof this.fd !== 'number') + return this.once('open', this._write.bind(this, data, cb)); + var self = this; - - var args = this._queue.shift(); - if (!args) { - if (this.drainable) { this.emit('drain'); } - return; - } - - this.busy = true; - - var method = args.shift(), - cb = args.pop(); - - args.push(function(err) { - self.busy = false; - - if (err) { - self.writable = false; - - function emit() { - self.fd = null; - if (cb) cb(err); - self.emit('error', err); - } - - if (self.fd === null) { - emit(); - } else { - fs.close(self.fd, emit); - } - - return; + fs.write(this.fd, data, 0, data.length, this.pos, function(er, bytes) { + if (er) { + self.destroy(); + return cb(er); } - - if (method == fs.write) { - self.bytesWritten += arguments[1]; - if (cb) { - // write callback - cb(null, arguments[1]); - } - - } else if (method === self._open) { - // save reference for file pointer - self.fd = arguments[1]; - self.emit('open', self.fd); - - } else if (method === fs.close) { - // stop flushing after close - if (cb) { - cb(null); - } - self.emit('close'); - return; - } - - self.flush(); + self.bytesWritten += bytes; + cb(); }); - // Inject the file pointer - if (method !== self._open) { - args.unshift(this.fd); - } - - method.apply(this, args); -}; - -WriteStream.prototype.write = function(data) { - if (!this.writable) { - this.emit('error', new Error('stream not writable')); - return false; - } - - this.drainable = true; - - var cb; - if (typeof(arguments[arguments.length - 1]) == 'function') { - cb = arguments[arguments.length - 1]; - } - - if (!Buffer.isBuffer(data)) { - var encoding = 'utf8'; - if (typeof(arguments[1]) == 'string') encoding = arguments[1]; - assertEncoding(encoding); - data = new Buffer('' + data, encoding); - } - - this._queue.push([fs.write, data, 0, data.length, this.pos, cb]); - - if (this.pos !== undefined) { + if (this.pos !== undefined) this.pos += data.length; - } - - this.flush(); - - return false; }; -WriteStream.prototype.end = function(data, encoding, cb) { - if (typeof(data) === 'function') { - cb = data; - } else if (typeof(encoding) === 'function') { - cb = encoding; - this.write(data); - } else if (arguments.length > 0) { - this.write(data, encoding); - } - this.writable = false; - this._queue.push([fs.close, cb]); - this.flush(); -}; -WriteStream.prototype.destroy = function() { - var self = this; - - if (!this.writable) return; - this.writable = false; - - function close() { - fs.close(self.fd, function(err) { - if (err) { - self.emit('error', err); - } else { - self.emit('close'); - } - }); - } - - if (this.fd === null) { - this.addListener('open', close); - } else { - close(); - } -}; +WriteStream.prototype.destroy = ReadStream.prototype.destroy; +WriteStream.prototype.close = ReadStream.prototype.close; // There is no shutdown() for files. WriteStream.prototype.destroySoon = WriteStream.prototype.end; diff --git a/test/simple/test-file-write-stream.js b/test/simple/test-file-write-stream.js index 5d2286cbd79..88295447663 100644 --- a/test/simple/test-file-write-stream.js +++ b/test/simple/test-file-write-stream.js @@ -22,46 +22,50 @@ var common = require('../common'); var assert = require('assert'); -var path = require('path'), - fs = require('fs'), - fn = path.join(common.tmpDir, 'write.txt'), - file = fs.createWriteStream(fn), +var path = require('path'); +var fs = require('fs'); +var fn = path.join(common.tmpDir, 'write.txt'); +var file = fs.createWriteStream(fn, { + lowWaterMark: 3, + highWaterMark: 10 + }); - EXPECTED = '012345678910', +var EXPECTED = '012345678910'; - callbacks = { +var callbacks = { open: -1, drain: -2, - close: -1, - endCb: -1 + close: -1 }; file .on('open', function(fd) { + console.error('open!'); callbacks.open++; assert.equal('number', typeof fd); }) .on('error', function(err) { throw err; + console.error('error!', err.stack); }) .on('drain', function() { + console.error('drain!', callbacks.drain); callbacks.drain++; if (callbacks.drain == -1) { - assert.equal(EXPECTED, fs.readFileSync(fn)); + assert.equal(EXPECTED, fs.readFileSync(fn, 'utf8')); file.write(EXPECTED); } else if (callbacks.drain == 0) { - assert.equal(EXPECTED + EXPECTED, fs.readFileSync(fn)); - file.end(function(err) { - assert.ok(!err); - callbacks.endCb++; - }); + assert.equal(EXPECTED + EXPECTED, fs.readFileSync(fn, 'utf8')); + file.end(); } }) .on('close', function() { + console.error('close!'); assert.strictEqual(file.bytesWritten, EXPECTED.length * 2); callbacks.close++; assert.throws(function() { + console.error('write after end should not be allowed'); file.write('should not work anymore'); }); @@ -70,7 +74,7 @@ file for (var i = 0; i < 11; i++) { (function(i) { - assert.strictEqual(false, file.write(i)); + file.write('' + i); })(i); } @@ -78,4 +82,5 @@ process.on('exit', function() { for (var k in callbacks) { assert.equal(0, callbacks[k], k + ' count off by ' + callbacks[k]); } + console.log('ok'); }); diff --git a/test/simple/test-file-write-stream2.js b/test/simple/test-file-write-stream2.js index 9b5d7ffef02..4f2e73ce824 100644 --- a/test/simple/test-file-write-stream2.js +++ b/test/simple/test-file-write-stream2.js @@ -22,18 +22,18 @@ var common = require('../common'); var assert = require('assert'); -var path = require('path'), - fs = require('fs'), - util = require('util'); +var path = require('path'); +var fs = require('fs'); +var util = require('util'); -var filepath = path.join(common.tmpDir, 'write.txt'), - file; +var filepath = path.join(common.tmpDir, 'write.txt'); +var file; var EXPECTED = '012345678910'; -var cb_expected = 'write open drain write drain close error ', - cb_occurred = ''; +var cb_expected = 'write open drain write drain close error '; +var cb_occurred = ''; var countDrains = 0; @@ -47,6 +47,8 @@ process.on('exit', function() { assert.strictEqual(cb_occurred, cb_expected, 'events missing or out of order: "' + cb_occurred + '" !== "' + cb_expected + '"'); + } else { + console.log('ok'); } }); @@ -59,22 +61,30 @@ function removeTestFile() { removeTestFile(); -file = fs.createWriteStream(filepath); +// drain at 0, return false at 10. +file = fs.createWriteStream(filepath, { + lowWaterMark: 0, + highWaterMark: 11 +}); file.on('open', function(fd) { + console.error('open'); cb_occurred += 'open '; assert.equal(typeof fd, 'number'); }); file.on('drain', function() { + console.error('drain'); cb_occurred += 'drain '; ++countDrains; if (countDrains === 1) { - assert.equal(fs.readFileSync(filepath), EXPECTED); - file.write(EXPECTED); + console.error('drain=1, write again'); + assert.equal(fs.readFileSync(filepath, 'utf8'), EXPECTED); + console.error('ondrain write ret=%j', file.write(EXPECTED)); cb_occurred += 'write '; } else if (countDrains == 2) { - assert.equal(fs.readFileSync(filepath), EXPECTED + EXPECTED); + console.error('second drain, end'); + assert.equal(fs.readFileSync(filepath, 'utf8'), EXPECTED + EXPECTED); file.end(); } }); @@ -88,11 +98,15 @@ file.on('close', function() { file.on('error', function(err) { cb_occurred += 'error '; - assert.ok(err.message.indexOf('not writable') >= 0); + assert.ok(err.message.indexOf('write after end') >= 0); }); for (var i = 0; i < 11; i++) { - assert.strictEqual(file.write(i), false); + var ret = file.write(i + ''); + console.error('%d %j', i, ret); + + // return false when i hits 10 + assert(ret === (i != 10)); } cb_occurred += 'write '; diff --git a/test/simple/test-fs-read-stream.js b/test/simple/test-fs-read-stream.js index 71cff2c059f..a88802b6542 100644 --- a/test/simple/test-fs-read-stream.js +++ b/test/simple/test-fs-read-stream.js @@ -60,12 +60,10 @@ file.on('data', function(data) { paused = true; file.pause(); - assert.ok(file.paused); setTimeout(function() { paused = false; file.resume(); - assert.ok(!file.paused); }, 10); }); @@ -77,7 +75,6 @@ file.on('end', function(chunk) { file.on('close', function() { callbacks.close++; - assert.ok(!file.readable); //assert.equal(fs.readFileSync(fn), fileContent); }); @@ -104,6 +101,7 @@ process.on('exit', function() { assert.equal(2, callbacks.close); assert.equal(30000, file.length); assert.equal(10000, file3.length); + console.error('ok'); }); var file4 = fs.createReadStream(rangeFile, {bufferSize: 1, start: 1, end: 2}); From 0e01d6398f14074675393a65cf430ea0a7a81ab8 Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 2 Oct 2012 16:15:39 -0700 Subject: [PATCH 39/72] zlib: streams2 --- lib/zlib.js | 213 +++++++++++-------------- src/node_zlib.cc | 13 ++ test/simple/test-zlib-destroy.js | 36 ----- test/simple/test-zlib-invalid-input.js | 7 - 4 files changed, 103 insertions(+), 166 deletions(-) delete mode 100644 test/simple/test-zlib-destroy.js diff --git a/lib/zlib.js b/lib/zlib.js index 9b562411467..bc3e9330f23 100644 --- a/lib/zlib.js +++ b/lib/zlib.js @@ -19,9 +19,10 @@ // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE // USE OR OTHER DEALINGS IN THE SOFTWARE. +var Transform = require('_stream_transform'); + var binding = process.binding('zlib'); var util = require('util'); -var Stream = require('stream'); var assert = require('assert').ok; // zlib doesn't provide these, so kludge them in following the same @@ -138,15 +139,25 @@ function zlibBuffer(engine, buffer, callback) { var buffers = []; var nread = 0; - function onError(err) { - engine.removeListener('end', onEnd); - engine.removeListener('error', onError); - callback(err); + engine.on('error', onError); + engine.on('end', onEnd); + + engine.end(buffer); + flow(); + + function flow() { + var chunk; + while (null !== (chunk = engine.read())) { + buffers.push(chunk); + nread += chunk.length; + } + engine.once('readable', flow); } - function onData(chunk) { - buffers.push(chunk); - nread += chunk.length; + function onError(err) { + engine.removeListener('end', onEnd); + engine.removeListener('readable', flow); + callback(err); } function onEnd() { @@ -154,17 +165,9 @@ function zlibBuffer(engine, buffer, callback) { buffers = []; callback(null, buf); } - - engine.on('error', onError); - engine.on('data', onData); - engine.on('end', onEnd); - - engine.write(buffer); - engine.end(); } - // generic zlib // minimal 2-byte header function Deflate(opts) { @@ -217,15 +220,13 @@ function Unzip(opts) { // you call the .write() method. function Zlib(opts, mode) { - Stream.call(this); - this._opts = opts = opts || {}; - this._queue = []; - this._processing = false; - this._ended = false; - this.readable = true; - this.writable = true; - this._flush = binding.Z_NO_FLUSH; + this._chunkSize = opts.chunkSize || exports.Z_DEFAULT_CHUNK; + + Transform.call(this, opts); + + // means a different thing there. + this._readableState.chunkSize = null; if (opts.chunkSize) { if (opts.chunkSize < exports.Z_MIN_CHUNK || @@ -274,13 +275,12 @@ function Zlib(opts, mode) { this._binding = new binding.Zlib(mode); var self = this; + this._hadError = false; this._binding.onerror = function(message, errno) { // there is no way to cleanly recover. // continuing only obscures problems. self._binding = null; self._hadError = true; - self._queue.length = 0; - self._processing = false; var error = new Error(message); error.errno = errno; @@ -294,7 +294,6 @@ function Zlib(opts, mode) { opts.strategy || exports.Z_DEFAULT_STRATEGY, opts.dictionary); - this._chunkSize = opts.chunkSize || exports.Z_DEFAULT_CHUNK; this._buffer = new Buffer(this._chunkSize); this._offset = 0; this._closed = false; @@ -302,59 +301,47 @@ function Zlib(opts, mode) { this.once('end', this.close); } -util.inherits(Zlib, Stream); - -Zlib.prototype.write = function write(chunk, cb) { - if (this._hadError) return true; - - if (this._ended) { - return this.emit('error', new Error('Cannot write after end')); - } - - if (arguments.length === 1 && typeof chunk === 'function') { - cb = chunk; - chunk = null; - } - - if (!chunk) { - chunk = null; - } else if (typeof chunk === 'string') { - chunk = new Buffer(chunk); - } else if (!Buffer.isBuffer(chunk)) { - return this.emit('error', new Error('Invalid argument')); - } - - - var empty = this._queue.length === 0; - - this._queue.push([chunk, cb]); - this._process(); - if (!empty) { - this._needDrain = true; - } - return empty; -}; +util.inherits(Zlib, Transform); Zlib.prototype.reset = function reset() { return this._binding.reset(); }; -Zlib.prototype.flush = function flush(cb) { - this._flush = binding.Z_SYNC_FLUSH; - return this.write(cb); +Zlib.prototype._flush = function(output, callback) { + var rs = this._readableState; + var self = this; + this._transform(null, output, function(er) { + if (er) + return callback(er); + + // now a weird thing happens... it could be that you called flush + // but everything had already actually been consumed, but it wasn't + // enough to get over the Readable class's lowWaterMark. + // In that case, we emit 'readable' now to make sure it's consumed. + if (rs.length && + rs.length < rs.lowWaterMark && + !rs.ended && + rs.needReadable) + self.emit('readable'); + + callback(); + }); }; -Zlib.prototype.end = function end(chunk, cb) { - if (this._hadError) return true; +Zlib.prototype.flush = function(callback) { + var ws = this._writableState; + var ts = this._transformState; - var self = this; - this._ending = true; - var ret = this.write(chunk, function() { - self.emit('end'); - if (cb) cb(); - }); - this._ended = true; - return ret; + if (ws.writing) { + ws.needDrain = true; + var self = this; + this.once('drain', function() { + self._flush(ts.output, callback); + }); + return; + } + + this._flush(ts.output, callback || function() {}); }; Zlib.prototype.close = function(callback) { @@ -368,37 +355,37 @@ Zlib.prototype.close = function(callback) { this._binding.close(); - process.nextTick(this.emit.bind(this, 'close')); + var self = this; + process.nextTick(function() { + self.emit('close'); + }); }; -Zlib.prototype._process = function() { - if (this._hadError) return; +Zlib.prototype._transform = function(chunk, output, cb) { + var flushFlag; + var ws = this._writableState; + var ending = ws.ending || ws.ended; + var last = ending && (!chunk || ws.length === chunk.length); - if (this._processing || this._paused) return; + if (chunk !== null && !Buffer.isBuffer(chunk)) + return cb(new Error('invalid input')); - if (this._queue.length === 0) { - if (this._needDrain) { - this._needDrain = false; - this.emit('drain'); - } - // nothing to do, waiting for more data at this point. - return; - } + // If it's the last chunk, or a final flush, we use the Z_FINISH flush flag. + // If it's explicitly flushing at some other time, then we use + // Z_FULL_FLUSH. Otherwise, use Z_NO_FLUSH for maximum compression + // goodness. + if (last) + flushFlag = binding.Z_FINISH; + else if (chunk === null) + flushFlag = binding.Z_FULL_FLUSH; + else + flushFlag = binding.Z_NO_FLUSH; - var req = this._queue.shift(); - var cb = req.pop(); - var chunk = req.pop(); - - if (this._ending && this._queue.length === 0) { - this._flush = binding.Z_FINISH; - } - - var self = this; var availInBefore = chunk && chunk.length; var availOutBefore = this._chunkSize - this._offset; - var inOff = 0; - var req = this._binding.write(this._flush, + + var req = this._binding.write(flushFlag, chunk, // in inOff, // in_off availInBefore, // in_len @@ -408,23 +395,23 @@ Zlib.prototype._process = function() { req.buffer = chunk; req.callback = callback; - this._processing = req; + var self = this; function callback(availInAfter, availOutAfter, buffer) { - if (self._hadError) return; + if (self._hadError) + return; var have = availOutBefore - availOutAfter; - assert(have >= 0, 'have should not go down'); if (have > 0) { var out = self._buffer.slice(self._offset, self._offset + have); self._offset += have; - self.emit('data', out); + // serve some output to the consumer. + output(out); } - // XXX Maybe have a 'min buffer' size so we don't dip into the - // thread pool with only 1 byte available or something? + // exhausted the output buffer, or used all the input create a new one. if (availOutAfter === 0 || self._offset >= self._chunkSize) { availOutBefore = self._chunkSize; self._offset = 0; @@ -439,7 +426,7 @@ Zlib.prototype._process = function() { inOff += (availInBefore - availInAfter); availInBefore = availInAfter; - var newReq = self._binding.write(self._flush, + var newReq = self._binding.write(flushFlag, chunk, inOff, availInBefore, @@ -448,34 +435,14 @@ Zlib.prototype._process = function() { self._chunkSize); newReq.callback = callback; // this same function newReq.buffer = chunk; - self._processing = newReq; return; } // finished with the chunk. - self._processing = false; - if (cb) cb(); - self._process(); + cb(); } }; -Zlib.prototype.pause = function() { - this._paused = true; - this.emit('pause'); -}; - -Zlib.prototype.resume = function() { - this._paused = false; - this._process(); -}; - -Zlib.prototype.destroy = function() { - this.readable = false; - this.writable = false; - this._ended = true; - this.emit('close'); -}; - util.inherits(Deflate, Zlib); util.inherits(Inflate, Zlib); util.inherits(Gzip, Zlib); diff --git a/src/node_zlib.cc b/src/node_zlib.cc index 13f94e9020f..881b20ce626 100644 --- a/src/node_zlib.cc +++ b/src/node_zlib.cc @@ -109,7 +109,19 @@ class ZCtx : public ObjectWrap { assert(!ctx->write_in_progress_ && "write already in progress"); ctx->write_in_progress_ = true; + assert(!args[0]->IsUndefined() && "must provide flush value"); + unsigned int flush = args[0]->Uint32Value(); + + if (flush != Z_NO_FLUSH && + flush != Z_PARTIAL_FLUSH && + flush != Z_SYNC_FLUSH && + flush != Z_FULL_FLUSH && + flush != Z_FINISH && + flush != Z_BLOCK) { + assert(0 && "Invalid flush value"); + } + Bytef *in; Bytef *out; size_t in_off, in_len, out_off, out_len; @@ -483,6 +495,7 @@ void InitZlib(Handle target) { callback_sym = NODE_PSYMBOL("callback"); onerror_sym = NODE_PSYMBOL("onerror"); + // valid flush values. NODE_DEFINE_CONSTANT(target, Z_NO_FLUSH); NODE_DEFINE_CONSTANT(target, Z_PARTIAL_FLUSH); NODE_DEFINE_CONSTANT(target, Z_SYNC_FLUSH); diff --git a/test/simple/test-zlib-destroy.js b/test/simple/test-zlib-destroy.js deleted file mode 100644 index 7a1120e2844..00000000000 --- a/test/simple/test-zlib-destroy.js +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Joyent, Inc. and other Node contributors. -// -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the -// "Software"), to deal in the Software without restriction, including -// without limitation the rights to use, copy, modify, merge, publish, -// distribute, sublicense, and/or sell copies of the Software, and to permit -// persons to whom the Software is furnished to do so, subject to the -// following conditions: -// -// The above copyright notice and this permission notice shall be included -// in all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN -// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE -// USE OR OTHER DEALINGS IN THE SOFTWARE. - -var common = require('../common'); -var assert = require('assert'); -var zlib = require('zlib'); - -['Deflate', 'Inflate', 'Gzip', 'Gunzip', 'DeflateRaw', 'InflateRaw', 'Unzip'] - .forEach(function (name) { - var a = false; - var zStream = new zlib[name](); - zStream.on('close', function () { - a = true; - }); - zStream.destroy(); - - assert.equal(a, true, name+'#destroy() must emit \'close\''); - }); diff --git a/test/simple/test-zlib-invalid-input.js b/test/simple/test-zlib-invalid-input.js index f97c5831ad1..c3d8b5b47a5 100644 --- a/test/simple/test-zlib-invalid-input.js +++ b/test/simple/test-zlib-invalid-input.js @@ -50,13 +50,6 @@ unzips.forEach(function (uz, i) { uz.on('error', function(er) { console.error('Error event', er); hadError[i] = true; - - // to be friendly to the Stream API, zlib objects just return true and - // ignore data on the floor after an error. It's up to the user to - // catch the 'error' event and do something intelligent. They do not - // emit any more data, however. - assert.equal(uz.write('also invalid'), true); - assert.equal(uz.end(), true); }); uz.on('end', function(er) { From 79fd9620f5eeb4c8165a6c354d93c62b56715696 Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 2 Oct 2012 16:54:49 -0700 Subject: [PATCH 40/72] test: Fix test-repl-autolibs inspect call --- test/simple/test-repl-autolibs.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/simple/test-repl-autolibs.js b/test/simple/test-repl-autolibs.js index 0f4ae8b387b..a8ee68ccc3c 100644 --- a/test/simple/test-repl-autolibs.js +++ b/test/simple/test-repl-autolibs.js @@ -48,8 +48,9 @@ function test1(){ putIn.write = function (data) { gotWrite = true; if (data.length) { + // inspect output matches repl output - assert.equal(data, util.inspect(require('fs'), null, null, false) + '\n'); + assert.equal(data, util.inspect(require('fs'), null, 2, false) + '\n'); // globally added lib matches required lib assert.equal(global.fs, require('fs')); test2(); From 70461c39be754b3597e0c14da15d251caf4907f6 Mon Sep 17 00:00:00 2001 From: isaacs Date: Fri, 5 Oct 2012 08:07:12 -0700 Subject: [PATCH 41/72] test: simple/test-file-write-stream needs to use 0 lowWaterMark --- test/simple/test-file-write-stream.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/simple/test-file-write-stream.js b/test/simple/test-file-write-stream.js index 88295447663..afedc5b5217 100644 --- a/test/simple/test-file-write-stream.js +++ b/test/simple/test-file-write-stream.js @@ -26,7 +26,7 @@ var path = require('path'); var fs = require('fs'); var fn = path.join(common.tmpDir, 'write.txt'); var file = fs.createWriteStream(fn, { - lowWaterMark: 3, + lowWaterMark: 0, highWaterMark: 10 }); From 3d3a0b30466b19a7879ff6f92512861404ca6c74 Mon Sep 17 00:00:00 2001 From: isaacs Date: Fri, 5 Oct 2012 08:26:49 -0700 Subject: [PATCH 42/72] test: Writable stream end() method doesn't take a callback --- test/simple/test-fs-write-stream-end.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/simple/test-fs-write-stream-end.js b/test/simple/test-fs-write-stream-end.js index 6c0f29c3441..2a85ac3eea6 100644 --- a/test/simple/test-fs-write-stream-end.js +++ b/test/simple/test-fs-write-stream-end.js @@ -31,7 +31,8 @@ var writeEndOk = false; var file = path.join(common.tmpDir, 'write-end-test.txt'); var stream = fs.createWriteStream(file); - stream.end('a\n', 'utf8', function() { + stream.end('a\n', 'utf8'); + stream.on('close', function() { var content = fs.readFileSync(file, 'utf8'); assert.equal(content, 'a\n'); writeEndOk = true; From 90de2ddb7739a0f1842066dffffcfbc83d2b3a88 Mon Sep 17 00:00:00 2001 From: isaacs Date: Mon, 29 Oct 2012 11:31:59 -0700 Subject: [PATCH 43/72] crypto: Streaming interface for Hash --- lib/crypto.js | 17 ++++++++++++++++- test/simple/test-crypto.js | 16 ++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/crypto.js b/lib/crypto.js index a787e09c348..8ea48e2a748 100644 --- a/lib/crypto.js +++ b/lib/crypto.js @@ -37,6 +37,9 @@ try { var crypto = false; } +var stream = require('stream'); +var util = require('util'); + // This is here because many functions accepted binary strings without // any explicit encoding in older versions of node, and we don't want // to break them unnecessarily. @@ -148,12 +151,24 @@ exports.createCredentials = function(options, context) { exports.createHash = exports.Hash = Hash; -function Hash(algorithm) { +function Hash(algorithm, options) { if (!(this instanceof Hash)) return new Hash(algorithm); this._binding = new binding.Hash(algorithm); + stream.Transform.call(this, options); } +util.inherits(Hash, stream.Transform); + +Hash.prototype._transform = function(chunk, output, callback) { + this._binding.update(chunk); + callback(); +}; + +Hash.prototype._flush = function(output, callback) { + output(this._binding.digest()); + callback(); +}; Hash.prototype.update = function(data, encoding) { encoding = encoding || exports.DEFAULT_ENCODING; diff --git a/test/simple/test-crypto.js b/test/simple/test-crypto.js index 6012671231c..1db94a99249 100644 --- a/test/simple/test-crypto.js +++ b/test/simple/test-crypto.js @@ -373,6 +373,18 @@ var a2 = crypto.createHash('sha256').update('Test123').digest('base64'); var a3 = crypto.createHash('sha512').update('Test123').digest(); // binary var a4 = crypto.createHash('sha1').update('Test123').digest('buffer'); +// stream interface +var a5 = crypto.createHash('sha512'); +a5.end('Test123'); +a5 = a5.read(); + +var a6 = crypto.createHash('sha512'); +a6.write('Te'); +a6.write('st'); +a6.write('123'); +a6.end(); +a6 = a6.read(); + assert.equal(a0, '8308651804facb7b9af8ffc53a33a22d6a1c8ac2', 'Test SHA1'); assert.equal(a1, 'h\u00ea\u00cb\u0097\u00d8o\fF!\u00fa+\u000e\u0017\u00ca' + '\u00bd\u008c', 'Test MD5 as binary'); @@ -392,6 +404,10 @@ assert.deepEqual(a4, new Buffer('8308651804facb7b9af8ffc53a33a22d6a1c8ac2', 'hex'), 'Test SHA1'); +// stream interface should produce the same result. +assert.deepEqual(a5, a3, 'stream interface is consistent'); +assert.deepEqual(a6, a3, 'stream interface is consistent'); + // Test multiple updates to same hash var h1 = crypto.createHash('sha1').update('Test123').digest('hex'); var h2 = crypto.createHash('sha1').update('Test').update('123').digest('hex'); From 175f78c6ba716c7fb7326da4d3532b4c0c0a9605 Mon Sep 17 00:00:00 2001 From: isaacs Date: Mon, 29 Oct 2012 15:21:25 -0700 Subject: [PATCH 44/72] crypto: Streaming api for Hmac --- lib/crypto.js | 6 +++++- test/simple/test-crypto.js | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/crypto.js b/lib/crypto.js index 8ea48e2a748..3807bf48114 100644 --- a/lib/crypto.js +++ b/lib/crypto.js @@ -189,16 +189,20 @@ Hash.prototype.digest = function(outputEncoding) { exports.createHmac = exports.Hmac = Hmac; -function Hmac(hmac, key) { +function Hmac(hmac, key, options) { if (!(this instanceof Hmac)) return new Hmac(hmac, key); this._binding = new binding.Hmac(); this._binding.init(hmac, toBuf(key)); + stream.Transform.call(this, options); } +util.inherits(Hmac, stream.Transform); Hmac.prototype.update = Hash.prototype.update; Hmac.prototype.digest = Hash.prototype.digest; +Hmac.prototype._flush = Hash.prototype._flush; +Hmac.prototype._transform = Hash.prototype._transform; function getDecoder(decoder, encoding) { diff --git a/test/simple/test-crypto.js b/test/simple/test-crypto.js index 1db94a99249..87a8d0ce7a5 100644 --- a/test/simple/test-crypto.js +++ b/test/simple/test-crypto.js @@ -230,15 +230,20 @@ var rfc4231 = [ for (var i = 0, l = rfc4231.length; i < l; i++) { for (var hash in rfc4231[i]['hmac']) { + var str = crypto.createHmac(hash, rfc4231[i].key); + str.end(rfc4231[i].data); + var strRes = str.read().toString('hex'); var result = crypto.createHmac(hash, rfc4231[i]['key']) .update(rfc4231[i]['data']) .digest('hex'); if (rfc4231[i]['truncate']) { result = result.substr(0, 32); // first 128 bits == 32 hex chars + strRes = strRes.substr(0, 32); } assert.equal(rfc4231[i]['hmac'][hash], result, 'Test HMAC-' + hash + ': Test case ' + (i + 1) + ' rfc 4231'); + assert.equal(strRes, result, 'Should get same result from stream'); } } From e336134658fe67265e4e0497e7972c088c5bb43a Mon Sep 17 00:00:00 2001 From: isaacs Date: Mon, 29 Oct 2012 16:36:20 -0700 Subject: [PATCH 45/72] crypto: Streaming interface for cipher/decipher/iv --- lib/crypto.js | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/lib/crypto.js b/lib/crypto.js index 3807bf48114..e6b03968734 100644 --- a/lib/crypto.js +++ b/lib/crypto.js @@ -213,15 +213,28 @@ function getDecoder(decoder, encoding) { exports.createCipher = exports.Cipher = Cipher; -function Cipher(cipher, password) { +function Cipher(cipher, password, options) { if (!(this instanceof Cipher)) return new Cipher(cipher, password); this._binding = new binding.Cipher; this._binding.init(cipher, toBuf(password)); this._decoder = null; + + stream.Transform.call(this, options); } +util.inherits(Cipher, stream.Transform); + +Cipher.prototype._transform = function(chunk, output, callback) { + output(this._binding.update(chunk)); + callback(); +}; + +Cipher.prototype._flush = function(output, callback) { + output(this._binding.final()); + callback(); +}; Cipher.prototype.update = function(data, inputEncoding, outputEncoding) { inputEncoding = inputEncoding || exports.DEFAULT_ENCODING; @@ -260,15 +273,20 @@ Cipher.prototype.setAutoPadding = function(ap) { exports.createCipheriv = exports.Cipheriv = Cipheriv; -function Cipheriv(cipher, key, iv) { +function Cipheriv(cipher, key, iv, options) { if (!(this instanceof Cipheriv)) return new Cipheriv(cipher, key, iv); this._binding = new binding.Cipher(); this._binding.initiv(cipher, toBuf(key), toBuf(iv)); this._decoder = null; + + stream.Transform.call(this, options); } +util.inherits(Cipheriv, stream.Transform); +Cipheriv.prototype._transform = Cipher.prototype._transform; +Cipheriv.prototype._flush = Cipher.prototype._flush; Cipheriv.prototype.update = Cipher.prototype.update; Cipheriv.prototype.final = Cipher.prototype.final; Cipheriv.prototype.setAutoPadding = Cipher.prototype.setAutoPadding; @@ -276,16 +294,21 @@ Cipheriv.prototype.setAutoPadding = Cipher.prototype.setAutoPadding; exports.createDecipher = exports.Decipher = Decipher; -function Decipher(cipher, password) { +function Decipher(cipher, password, options) { if (!(this instanceof Decipher)) return new Decipher(cipher, password); this._binding = new binding.Decipher; this._binding.init(cipher, toBuf(password)); this._decoder = null; + + stream.Transform.call(this, options); } +util.inherits(Decipher, stream.Transform); +Decipher.prototype._transform = Cipher.prototype._transform; +Decipher.prototype._flush = Cipher.prototype._flush; Decipher.prototype.update = Cipher.prototype.update; Decipher.prototype.final = Cipher.prototype.final; Decipher.prototype.finaltol = Cipher.prototype.final; @@ -294,16 +317,21 @@ Decipher.prototype.setAutoPadding = Cipher.prototype.setAutoPadding; exports.createDecipheriv = exports.Decipheriv = Decipheriv; -function Decipheriv(cipher, key, iv) { +function Decipheriv(cipher, key, iv, options) { if (!(this instanceof Decipheriv)) return new Decipheriv(cipher, key, iv); this._binding = new binding.Decipher; this._binding.initiv(cipher, toBuf(key), toBuf(iv)); this._decoder = null; + + stream.Transform.call(this, options); } +util.inherits(Decipheriv, stream.Transform); +Decipheriv.prototype._transform = Cipher.prototype._transform; +Decipheriv.prototype._flush = Cipher.prototype._flush; Decipheriv.prototype.update = Cipher.prototype.update; Decipheriv.prototype.final = Cipher.prototype.final; Decipheriv.prototype.finaltol = Cipher.prototype.final; From dd3ebb8cf643e88208adc5ec5bd2525f5288ddbb Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 30 Oct 2012 10:18:55 -0700 Subject: [PATCH 46/72] crypto: Streaming interface for Sign and Verify --- lib/crypto.js | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/crypto.js b/lib/crypto.js index e6b03968734..0033267ceee 100644 --- a/lib/crypto.js +++ b/lib/crypto.js @@ -340,17 +340,24 @@ Decipheriv.prototype.setAutoPadding = Cipher.prototype.setAutoPadding; exports.createSign = exports.Sign = Sign; -function Sign(algorithm) { +function Sign(algorithm, options) { if (!(this instanceof Sign)) return new Sign(algorithm); this._binding = new binding.Sign(); this._binding.init(algorithm); + + stream.Writable.call(this, options); } +util.inherits(Sign, stream.Writable); + +Sign.prototype._write = function(chunk, callback) { + this._binding.update(chunk); + callback(); +}; Sign.prototype.update = Hash.prototype.update; - Sign.prototype.sign = function(key, encoding) { encoding = encoding || exports.DEFAULT_ENCODING; var ret = this._binding.sign(toBuf(key)); @@ -364,17 +371,20 @@ Sign.prototype.sign = function(key, encoding) { exports.createVerify = exports.Verify = Verify; -function Verify(algorithm) { +function Verify(algorithm, options) { if (!(this instanceof Verify)) return new Verify(algorithm); this._binding = new binding.Verify; this._binding.init(algorithm); + + stream.Writable.call(this, options); } +util.inherits(Verify, stream.Writable); -Verify.prototype.update = Hash.prototype.update; - +Verify.prototype._write = Sign.prototype._write; +Verify.prototype.update = Sign.prototype.update; Verify.prototype.verify = function(object, signature, sigEncoding) { sigEncoding = sigEncoding || exports.DEFAULT_ENCODING; From e0c600e00e846d252aeb75f5c0d2a5e2ccad120d Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 30 Oct 2012 11:19:53 -0700 Subject: [PATCH 47/72] test: Tests for streaming crypto interfaces --- test/simple/test-crypto.js | 56 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/test/simple/test-crypto.js b/test/simple/test-crypto.js index 87a8d0ce7a5..8f5043ef73c 100644 --- a/test/simple/test-crypto.js +++ b/test/simple/test-crypto.js @@ -440,6 +440,11 @@ assert.throws(function() { var s1 = crypto.createSign('RSA-SHA1') .update('Test123') .sign(keyPem, 'base64'); +var s1stream = crypto.createSign('RSA-SHA1'); +s1stream.end('Test123'); +s1stream = s1stream.sign(keyPem, 'base64'); +assert.equal(s1, s1stream, 'Stream produces same output'); + var verified = crypto.createVerify('RSA-SHA1') .update('Test') .update('123') @@ -448,13 +453,25 @@ assert.strictEqual(verified, true, 'sign and verify (base 64)'); var s2 = crypto.createSign('RSA-SHA256') .update('Test123') - .sign(keyPem); // binary + .sign(keyPem, 'binary'); +var s2stream = crypto.createSign('RSA-SHA256'); +s2stream.end('Test123'); +s2stream = s2stream.sign(keyPem, 'binary'); +assert.equal(s2, s2stream, 'Stream produces same output'); + var verified = crypto.createVerify('RSA-SHA256') .update('Test') .update('123') - .verify(certPem, s2); // binary + .verify(certPem, s2, 'binary'); assert.strictEqual(verified, true, 'sign and verify (binary)'); +var verStream = crypto.createVerify('RSA-SHA256'); +verStream.write('Tes'); +verStream.write('t12'); +verStream.end('3'); +verified = verStream.verify(certPem, s2, 'binary'); +assert.strictEqual(verified, true, 'sign and verify (stream)'); + var s3 = crypto.createSign('RSA-SHA1') .update('Test123') .sign(keyPem, 'buffer'); @@ -464,6 +481,13 @@ var verified = crypto.createVerify('RSA-SHA1') .verify(certPem, s3); assert.strictEqual(verified, true, 'sign and verify (buffer)'); +var verStream = crypto.createVerify('RSA-SHA1'); +verStream.write('Tes'); +verStream.write('t12'); +verStream.end('3'); +verified = verStream.verify(certPem, s3); +assert.strictEqual(verified, true, 'sign and verify (stream)'); + function testCipher1(key) { // Test encryption and decryption @@ -481,6 +505,20 @@ function testCipher1(key) { txt += decipher.final('utf8'); assert.equal(txt, plaintext, 'encryption and decryption'); + + // streaming cipher interface + // NB: In real life, it's not guaranteed that you can get all of it + // in a single read() like this. But in this case, we know it's + // quite small, so there's no harm. + var cStream = crypto.createCipher('aes192', key); + cStream.end(plaintext); + ciph = cStream.read(); + + var dStream = crypto.createDecipher('aes192', key); + dStream.end(ciph); + txt = dStream.read().toString('utf8'); + + assert.equal(txt, plaintext, 'encryption and decryption with streams'); } @@ -521,6 +559,20 @@ function testCipher3(key, iv) { txt += decipher.final('utf8'); assert.equal(txt, plaintext, 'encryption and decryption with key and iv'); + + // streaming cipher interface + // NB: In real life, it's not guaranteed that you can get all of it + // in a single read() like this. But in this case, we know it's + // quite small, so there's no harm. + var cStream = crypto.createCipheriv('des-ede3-cbc', key, iv); + cStream.end(plaintext); + ciph = cStream.read(); + + var dStream = crypto.createDecipheriv('des-ede3-cbc', key, iv); + dStream.end(ciph); + txt = dStream.read().toString('utf8'); + + assert.equal(txt, plaintext, 'streaming cipher iv'); } From 4a32d53155558ffb3d93f66f5cd8a11fc44b92d8 Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 30 Oct 2012 15:46:43 -0700 Subject: [PATCH 48/72] doc: Crypto streaming interface --- doc/api/crypto.markdown | 51 +++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/doc/api/crypto.markdown b/doc/api/crypto.markdown index 4a5735d36df..de574bc37e4 100644 --- a/doc/api/crypto.markdown +++ b/doc/api/crypto.markdown @@ -89,6 +89,11 @@ Example: this program that takes the sha1 sum of a file The class for creating hash digests of data. +It is a [stream](stream.html) that is both readable and writable. The +written data is used to compute the hash. Once the writable side of +the stream is ended, use the `read()` method to get the computed hash +digest. The legacy `update` and `digest` methods are also supported. + Returned by `crypto.createHash`. ### hash.update(data, [input_encoding]) @@ -114,6 +119,11 @@ called. Creates and returns a hmac object, a cryptographic hmac with the given algorithm and key. +It is a [stream](stream.html) that is both readable and writable. The +written data is used to compute the hmac. Once the writable side of +the stream is ended, use the `read()` method to get the computed +digest. The legacy `update` and `digest` methods are also supported. + `algorithm` is dependent on the available algorithms supported by OpenSSL - see createHash above. `key` is the hmac key to be used. @@ -148,6 +158,11 @@ recent releases, `openssl list-cipher-algorithms` will display the available cipher algorithms. `password` is used to derive key and IV, which must be a `'binary'` encoded string or a [buffer](buffer.html). +It is a [stream](stream.html) that is both readable and writable. The +written data is used to compute the hash. Once the writable side of +the stream is ended, use the `read()` method to get the computed hash +digest. The legacy `update` and `digest` methods are also supported. + ## crypto.createCipheriv(algorithm, key, iv) Creates and returns a cipher object, with the given algorithm, key and @@ -166,6 +181,11 @@ Class for encrypting data. Returned by `crypto.createCipher` and `crypto.createCipheriv`. +Cipher objects are [streams](stream.html) that are both readable and +writable. The written plain text data is used to produce the +encrypted data on the the readable side. The legacy `update` and +`final` methods are also supported. + ### cipher.update(data, [input_encoding], [output_encoding]) Updates the cipher with `data`, the encoding of which is given in @@ -213,6 +233,11 @@ Class for decrypting data. Returned by `crypto.createDecipher` and `crypto.createDecipheriv`. +Decipher objects are [streams](stream.html) that are both readable and +writable. The written enciphered data is used to produce the +plain-text data on the the readable side. The legacy `update` and +`final` methods are also supported. + ### decipher.update(data, [input_encoding], [output_encoding]) Updates the decipher with `data`, which is encoded in `'binary'`, @@ -246,28 +271,33 @@ Creates and returns a signing object, with the given algorithm. On recent OpenSSL releases, `openssl list-public-key-algorithms` will display the available signing algorithms. Examples are `'RSA-SHA256'`. -## Class: Signer +## Class: Sign Class for generating signatures. Returned by `crypto.createSign`. -### signer.update(data) +Sign objects are writable [streams](stream.html). The written data is +used to generate the signature. Once all of the data has been +written, the `sign` method will return the signature. The legacy +`update` method is also supported. -Updates the signer object with data. This can be called many times +### sign.update(data) + +Updates the sign object with data. This can be called many times with new data as it is streamed. -### signer.sign(private_key, [output_format]) +### sign.sign(private_key, [output_format]) Calculates the signature on all the updated data passed through the -signer. `private_key` is a string containing the PEM encoded private +sign. `private_key` is a string containing the PEM encoded private key for signing. Returns the signature in `output_format` which can be `'binary'`, `'hex'` or `'base64'`. If no encoding is provided, then a buffer is returned. -Note: `signer` object can not be used after `sign()` method been +Note: `sign` object can not be used after `sign()` method been called. ## crypto.createVerify(algorithm) @@ -281,6 +311,12 @@ Class for verifying signatures. Returned by `crypto.createVerify`. +Verify objects are writable [streams](stream.html). The written data +is used to validate against the supplied signature. Once all of the +data has been written, the `verify` method will return true if the +supplied signature is valid. The legacy `update` method is also +supported. + ### verifier.update(data) Updates the verifier object with data. This can be called many times @@ -469,9 +505,6 @@ default, set the `crypto.DEFAULT_ENCODING` field to 'binary'. Note that new programs will probably expect buffers, so only use this as a temporary measure. -Also, a Streaming API will be provided, but this will be done in such -a way as to preserve the legacy API surface. - [createCipher()]: #crypto_crypto_createcipher_algorithm_password [createCipheriv()]: #crypto_crypto_createcipheriv_algorithm_key_iv From 83704f127982cc958d0a9f702c103eb8c960f2aa Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 5 Dec 2012 11:27:46 -0800 Subject: [PATCH 49/72] streams2: Set readable=false on end --- lib/_stream_readable.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 3e9253aa7df..ac21330dfe5 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -738,6 +738,7 @@ function endReadable(stream) { state.ended = true; state.endEmitted = true; process.nextTick(function() { + stream.readable = false; stream.emit('end'); }); } From 42981e2aadfdf1a16218dced31e55d9f62020c4d Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 4 Dec 2012 17:20:12 -0800 Subject: [PATCH 50/72] streams2: Switch to old-mode immediately, not nextTick This fixes the CONNECT/Upgrade HTTP functionality, which was not getting sliced properly, because readable wasn't emitted on this tick. Conflicts: test/simple/test-http-connect.js --- lib/_stream_readable.js | 4 +--- test/simple/test-http-connect.js | 6 +++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index ac21330dfe5..a0cb13c3f98 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -568,9 +568,7 @@ function emitDataEvents(stream) { }; // now make it start, just in case it hadn't already. - process.nextTick(function() { - stream.emit('readable'); - }); + stream.emit('readable'); } // wrap an old-style stream as the async data source. diff --git a/test/simple/test-http-connect.js b/test/simple/test-http-connect.js index 668dda79633..3643cec18e9 100644 --- a/test/simple/test-http-connect.js +++ b/test/simple/test-http-connect.js @@ -73,7 +73,11 @@ server.listen(common.PORT, function() { assert(!socket.onend); assert.equal(socket.listeners('connect').length, 0); assert.equal(socket.listeners('data').length, 0); - assert.equal(socket.listeners('end').length, 0); + + // the stream.Duplex onend listener + // allow 0 here, so that i can run the same test on streams1 impl + assert(socket.listeners('end').length <= 1); + assert.equal(socket.listeners('free').length, 0); assert.equal(socket.listeners('close').length, 0); assert.equal(socket.listeners('error').length, 0); From 99021b7a4f7de955074095abfb03e5cbc5416e7b Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 4 Dec 2012 17:34:17 -0800 Subject: [PATCH 51/72] streams2: pause() should be immediate --- lib/_stream_readable.js | 8 +++----- test/simple/test-fs-empty-readStream.js | 6 ++++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index a0cb13c3f98..6c4d95e5bb6 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -521,15 +521,13 @@ Readable.prototype.addListener = Readable.prototype.on; // If the user uses them, then switch into old mode. Readable.prototype.resume = function() { emitDataEvents(this); - return this.resume(); }; Readable.prototype.pause = function() { - emitDataEvents(this); - return this.pause(); + emitDataEvents(this, true); }; -function emitDataEvents(stream) { +function emitDataEvents(stream, startPaused) { var state = stream._readableState; if (state.flowing) { @@ -537,7 +535,7 @@ function emitDataEvents(stream) { throw new Error('Cannot switch to old mode now.'); } - var paused = false; + var paused = startPaused || false; var readable = false; // convert to an old-style stream. diff --git a/test/simple/test-fs-empty-readStream.js b/test/simple/test-fs-empty-readStream.js index a9b378fea1b..d181c219820 100644 --- a/test/simple/test-fs-empty-readStream.js +++ b/test/simple/test-fs-empty-readStream.js @@ -32,12 +32,13 @@ fs.open(emptyFile, 'r', function (error, fd) { var read = fs.createReadStream(emptyFile, { 'fd': fd }); read.once('data', function () { - throw new Error("data event should not emit"); + throw new Error('data event should not emit'); }); var readEmit = false; read.once('end', function () { readEmit = true; + console.error('end event 1'); }); setTimeout(function () { @@ -52,12 +53,13 @@ fs.open(emptyFile, 'r', function (error, fd) { read.pause(); read.once('data', function () { - throw new Error("data event should not emit"); + throw new Error('data event should not emit'); }); var readEmit = false; read.once('end', function () { readEmit = true; + console.error('end event 2'); }); setTimeout(function () { From dbcacc5afe4c1fea4a0796d95090ea6aff7e67ed Mon Sep 17 00:00:00 2001 From: isaacs Date: Tue, 4 Dec 2012 18:19:07 -0800 Subject: [PATCH 52/72] streams2: NextTick the emit('readable') in resume() Otherwise resume() will cause data to be emitted before it can be handled. --- lib/_stream_readable.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 6c4d95e5bb6..54ceaa102ab 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -545,6 +545,7 @@ function emitDataEvents(stream, startPaused) { stream.on('readable', function() { readable = true; + var c; while (!paused && (null !== (c = stream.read()))) stream.emit('data', c); @@ -562,7 +563,9 @@ function emitDataEvents(stream, startPaused) { stream.resume = function() { paused = false; if (readable) - stream.emit('readable'); + process.nextTick(function() { + stream.emit('readable'); + }); }; // now make it start, just in case it hadn't already. From f8bb031bdc7067f3b69de771edd2c46df6386ad9 Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 6 Dec 2012 07:29:42 -0800 Subject: [PATCH 53/72] test: Sync writables may emit finish before callbacks --- test/simple/test-stream2-writable.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/simple/test-stream2-writable.js b/test/simple/test-stream2-writable.js index bfd6bb75d61..be40664edf2 100644 --- a/test/simple/test-stream2-writable.js +++ b/test/simple/test-stream2-writable.js @@ -59,6 +59,10 @@ function run() { var name = next[0]; var fn = next[1]; + + if (!fn) + return run(); + console.log('# %s', name); fn({ same: assert.deepEqual, @@ -228,9 +232,11 @@ test('write callbacks', function (t) { }); tw.on('finish', function() { - t.same(tw.buffer, chunks, 'got chunks in the right order'); - t.same(callbacks._called, chunks, 'called all callbacks'); - t.end(); + process.nextTick(function() { + t.same(tw.buffer, chunks, 'got chunks in the right order'); + t.same(callbacks._called, chunks, 'called all callbacks'); + t.end(); + }); }); chunks.forEach(function(chunk, i) { From fc7d8d59f75b57545cd4951ef178d25ef6af9d64 Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 6 Dec 2012 10:21:22 -0800 Subject: [PATCH 54/72] lint --- lib/_stream_readable.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 54ceaa102ab..f7790c66256 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -303,7 +303,7 @@ Readable.prototype.pipe = function(dest, pipeOpts) { state.pipes = dest; break; case 1: - state.pipes = [ state.pipes, dest ]; + state.pipes = [state.pipes, dest]; break; default: state.pipes.push(dest); @@ -390,7 +390,7 @@ function pipeOnDrain(src) { return function() { var dest = this; var state = src._readableState; - state.awaitDrain --; + state.awaitDrain--; if (state.awaitDrain === 0) flow(src); }; From 8f428f3b0deaaf74987076aafa47f36768a05d99 Mon Sep 17 00:00:00 2001 From: isaacs Date: Mon, 10 Dec 2012 15:58:23 -0800 Subject: [PATCH 55/72] streams2: Call read(0) on resume() Otherwise (especially with stdin) you sometimes end up in cases where the high water mark is zero, and the current buffer is at 0, and it doesn't need a readable event, so it never calls _read(). --- lib/_debugger.js | 2 +- lib/_stream_readable.js | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/_debugger.js b/lib/_debugger.js index 19c26aa943b..239220962d8 100644 --- a/lib/_debugger.js +++ b/lib/_debugger.js @@ -36,7 +36,7 @@ exports.start = function(argv, stdin, stdout) { } // Setup input/output streams - stdin = stdin || process.openStdin(); + stdin = stdin || process.stdin; stdout = stdout || process.stdout; var args = ['--debug-brk'].concat(argv), diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index f7790c66256..3a65b53fe00 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -521,6 +521,7 @@ Readable.prototype.addListener = Readable.prototype.on; // If the user uses them, then switch into old mode. Readable.prototype.resume = function() { emitDataEvents(this); + this.read(0); }; Readable.prototype.pause = function() { @@ -566,6 +567,8 @@ function emitDataEvents(stream, startPaused) { process.nextTick(function() { stream.emit('readable'); }); + else + this.read(0); }; // now make it start, just in case it hadn't already. From 5760244cc622b986de0287a7f08bd88d9c0c21e4 Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 12 Dec 2012 00:59:54 -0800 Subject: [PATCH 56/72] streams2: Writable only emit 'finish' once --- lib/_stream_writable.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/_stream_writable.js b/lib/_stream_writable.js index 00702cab9a3..7364d3ab10c 100644 --- a/lib/_stream_writable.js +++ b/lib/_stream_writable.js @@ -190,7 +190,8 @@ function onwrite(stream, er) { cb(); } - if (state.length === 0 && (state.ended || state.ending)) { + if (state.length === 0 && (state.ended || state.ending) && + !state.finished && !state.finishing) { // emit 'finish' at the very end. state.finishing = true; stream.emit('finish'); @@ -245,7 +246,7 @@ Writable.prototype.end = function(chunk, encoding) { state.ending = true; if (chunk) this.write(chunk, encoding); - else if (state.length === 0) { + else if (state.length === 0 && !state.finishing && !state.finished) { state.finishing = true; this.emit('finish'); state.finished = true; From 8fe7b0c910ed0e82b06316eb2eb78934b6459e59 Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 12 Dec 2012 22:03:19 -0800 Subject: [PATCH 57/72] streams2: Support a Readable hwm of 0 Necessary for proper stdin functioning --- lib/_stream_readable.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 3a65b53fe00..ed65af9c2de 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -181,20 +181,17 @@ Readable.prototype.read = function(n) { if (state.length - n <= state.highWaterMark) doRead = true; - // if we currently have *nothing*, then always try to get *something* - // no matter what the high water mark says. - if (state.length === 0) - doRead = true; - // however, if we've ended, then there's no point, and if we're already // reading, then it's unnecessary. if (state.ended || state.reading) doRead = false; if (doRead) { - var sync = true; state.reading = true; state.sync = true; + // if the length is currently zero, then we *need* a readable event. + if (state.length === 0) + state.needReadable = true; // call internal read method this._read(state.bufferSize, state.onread); state.sync = false; @@ -219,6 +216,11 @@ Readable.prototype.read = function(n) { state.length -= n; + // If we have nothing in the buffer, then we want to know + // as soon as we *do* get something into the buffer. + if (state.length === 0 && !state.ended) + state.needReadable = true; + return ret; }; @@ -655,6 +657,9 @@ Readable.prototype.wrap = function(stream) { var ret = fromList(n, state.buffer, state.length, !!state.decoder); state.length -= n; + if (state.length === 0 && !state.ended) + state.needReadable = true; + if (state.length <= state.lowWaterMark && paused) { stream.resume(); paused = false; From 04541cf7bccb2be9fa4e587db8cfb21392d97c2d Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 12 Dec 2012 22:17:57 -0800 Subject: [PATCH 58/72] streams2: Emit pause/resume events --- lib/_stream_readable.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index ed65af9c2de..8c6e1b5b4fd 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -524,10 +524,12 @@ Readable.prototype.addListener = Readable.prototype.on; Readable.prototype.resume = function() { emitDataEvents(this); this.read(0); + this.emit('resume'); }; Readable.prototype.pause = function() { emitDataEvents(this, true); + this.emit('pause'); }; function emitDataEvents(stream, startPaused) { @@ -561,6 +563,7 @@ function emitDataEvents(stream, startPaused) { stream.pause = function() { paused = true; + this.emit('pause'); }; stream.resume = function() { @@ -571,6 +574,7 @@ function emitDataEvents(stream, startPaused) { }); else this.read(0); + this.emit('resume'); }; // now make it start, just in case it hadn't already. From 20a88feb8fe9a52382866166e897ddb5bfae199b Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 13 Dec 2012 11:15:49 -0800 Subject: [PATCH 59/72] docs: streams2 --- doc/api/stream.markdown | 497 ++++++++++++++++++++++++++++++++-------- 1 file changed, 398 insertions(+), 99 deletions(-) diff --git a/doc/api/stream.markdown b/doc/api/stream.markdown index 5e0c6f642db..974c5dd22c9 100644 --- a/doc/api/stream.markdown +++ b/doc/api/stream.markdown @@ -7,186 +7,485 @@ Node. For example a request to an HTTP server is a stream, as is stdout. Streams are readable, writable, or both. All streams are instances of [EventEmitter][] -You can load up the Stream base class by doing `require('stream')`. +You can load the Stream base classes by doing `require('stream')`. +There are base classes provided for Readable streams, Writable +streams, Duplex streams, and Transform streams. -## Readable Stream +## Compatibility + +In earlier versions of Node, the Readable stream interface was +simpler, but also less powerful and less useful. + +* Rather than waiting for you to call the `read()` method, `'data'` + events would start emitting immediately. If you needed to do some + I/O to decide how to handle data, then you had to store the chunks + in some kind of buffer so that they would not be lost. +* The `pause()` method was advisory, rather than guaranteed. This + meant that you still had to be prepared to receive `'data'` events + even when the stream was in a paused state. + +In Node v0.10, the Readable class described below was added. For +backwards compatibility with older Node programs, Readable streams +switch into "old mode" when a `'data'` event handler is added, or when +the `pause()` or `resume()` methods are called. The effect is that, +even if you are not using the new `read()` method and `'readable'` +event, you no longer have to worry about losing `'data'` chunks. + +Most programs will continue to function normally. However, this +introduces an edge case in the following conditions: + +* No `'data'` event handler is added. +* The `pause()` and `resume()` methods are never called. + +For example, consider the following code: + +```javascript +// WARNING! BROKEN! +net.createServer(function(socket) { + + // we add an 'end' method, but never consume the data + socket.on('end', function() { + // It will never get here. + socket.end('I got your message (but didnt read it)\n'); + }); + +}).listen(1337); +``` + +In versions of node prior to v0.10, the incoming message data would be +simply discarded. However, in Node v0.10 and beyond, the socket will +remain paused forever. + +The workaround in this situation is to call the `resume()` method to +trigger "old mode" behavior: + +```javascript +// Workaround +net.createServer(function(socket) { + + socket.on('end', function() { + socket.end('I got your message (but didnt read it)\n'); + }); + + // start the flow of data, discarding it. + socket.resume(); + +}).listen(1337); +``` + +In addition to new Readable streams switching into old-mode, pre-v0.10 +style streams can be wrapped in a Readable class using the `wrap()` +method. + +## Class: stream.Readable A `Readable Stream` has the following methods, members, and events. -### Event: 'data' +Note that `stream.Readable` is an abstract class designed to be +extended with an underlying implementation of the `_read(size, cb)` +method. (See below.) -`function (data) { }` +### new stream.Readable([options]) -The `'data'` event emits either a `Buffer` (by default) or a string if -`setEncoding()` was used. +* `options` {Object} + * `bufferSize` {Number} The size of the chunks to consume from the + underlying resource. Default=16kb + * `lowWaterMark` {Number} The minimum number of bytes to store in + the internal buffer before emitting `readable`. Default=0 + * `highWaterMark` {Number} The maximum number of bytes to store in + the internal buffer before ceasing to read from the underlying + resource. Default=16kb + * `encoding` {String} If specified, then buffers will be decoded to + strings using the specified encoding. Default=null -Note that the __data will be lost__ if there is no listener when a -`Readable Stream` emits a `'data'` event. +In classes that extend the Readable class, make sure to call the +constructor so that the buffering settings can be properly +initialized. + +### readable.\_read(size, callback) + +* `size` {Number} Number of bytes to read asynchronously +* `callback` {Function} Called with an error or with data + +All Readable stream implementations must provide a `_read` method +to fetch data from the underlying resource. + +**This function MUST NOT be called directly.** It should be +implemented by child classes, and called by the internal Readable +class methods only. + +Call the callback using the standard `callback(error, data)` pattern. +When no more data can be fetched, call `callback(null, null)` to +signal the EOF. + +This method is prefixed with an underscore because it is internal to +the class that defines it, and should not be called directly by user +programs. However, you **are** expected to override this method in +your own extension classes. + + +### readable.wrap(stream) + +* `stream` {Stream} An "old style" readable stream + +If you are using an older Node library that emits `'data'` events and +has a `pause()` method that is advisory only, then you can use the +`wrap()` method to create a Readable stream that uses the old stream +as its data source. + +For example: + +```javascript +var OldReader = require('./old-api-module.js').OldReader; +var oreader = new OldReader; +var Readable = require('stream').Readable; +var myReader = new Readable().wrap(oreader); + +myReader.on('readable', function() { + myReader.read(); // etc. +}); +``` + +### Event: 'readable' + +When there is data ready to be consumed, this event will fire. The +number of bytes that are required to be considered "readable" depends +on the `lowWaterMark` option set in the constructor. + +When this event emits, call the `read()` method to consume the data. ### Event: 'end' -`function () { }` - Emitted when the stream has received an EOF (FIN in TCP terminology). Indicates that no more `'data'` events will happen. If the stream is also writable, it may be possible to continue writing. -### Event: 'error' +### Event: 'data' -`function (exception) { }` +The `'data'` event emits either a `Buffer` (by default) or a string if +`setEncoding()` was used. + +Note that adding a `'data'` event listener will switch the Readable +stream into "old mode", where data is emitted as soon as it is +available, rather than waiting for you to call `read()` to consume it. + +### Event: 'error' Emitted if there was an error receiving data. ### Event: 'close' -`function () { }` - Emitted when the underlying resource (for example, the backing file descriptor) has been closed. Not all streams will emit this. -### stream.readable - -A boolean that is `true` by default, but turns `false` after an -`'error'` occurred, the stream came to an `'end'`, or `destroy()` was -called. - -### stream.setEncoding([encoding]) +### readable.setEncoding(encoding) Makes the `'data'` event emit a string instead of a `Buffer`. `encoding` -can be `'utf8'`, `'utf16le'` (`'ucs2'`), `'ascii'`, or `'hex'`. Defaults -to `'utf8'`. +can be `'utf8'`, `'utf16le'` (`'ucs2'`), `'ascii'`, or `'hex'`. -### stream.pause() +The encoding can also be set by specifying an `encoding` field to the +constructor. -Issues an advisory signal to the underlying communication layer, -requesting that no further data be sent until `resume()` is called. +### readable.read([size]) -Note that, due to the advisory nature, certain streams will not be -paused immediately, and so `'data'` events may be emitted for some -indeterminate period of time even after `pause()` is called. You may -wish to buffer such `'data'` events. +* `size` {Number | null} Optional number of bytes to read. +* Return: {Buffer | String | null} -### stream.resume() +Call this method to consume data once the `'readable'` event is +emitted. -Resumes the incoming `'data'` events after a `pause()`. +The `size` argument will set a minimum number of bytes that you are +interested in. If not set, then the entire content of the internal +buffer is returned. -### stream.destroy() +If there is no data to consume, or if there are fewer bytes in the +internal buffer than the `size` argument, then `null` is returned, and +a future `'readable'` event will be emitted when more is available. -Closes the underlying file descriptor. Stream is no longer `writable` -nor `readable`. The stream will not emit any more 'data', or 'end' -events. Any queued write data will not be sent. The stream should emit -'close' event once its resources have been disposed of. +Note that calling `stream.read(0)` will always return `null`, and will +trigger a refresh of the internal buffer, but otherwise be a no-op. +### readable.pipe(destination, [options]) -### stream.pipe(destination, [options]) +* `destination` {Writable Stream} +* `options` {Object} Optional + * `end` {Boolean} Default=true -This is a `Stream.prototype` method available on all `Stream`s. - -Connects this read stream to `destination` WriteStream. Incoming data on -this stream gets written to `destination`. The destination and source -streams are kept in sync by pausing and resuming as necessary. +Connects this readable stream to `destination` WriteStream. Incoming +data on this stream gets written to `destination`. Properly manages +back-pressure so that a slow destination will not be overwhelmed by a +fast readable stream. This function returns the `destination` stream. -Emulating the Unix `cat` command: - - process.stdin.resume(); process.stdin.pipe(process.stdout); +For example, emulating the Unix `cat` command: + process.stdin.pipe(process.stdout); By default `end()` is called on the destination when the source stream emits `end`, so that `destination` is no longer writable. Pass `{ end: false }` as `options` to keep the destination stream open. -This keeps `process.stdout` open so that "Goodbye" can be written at the +This keeps `writer` open so that "Goodbye" can be written at the end. - process.stdin.resume(); + reader.pipe(writer, { end: false }); + reader.on("end", function() { + writer.end("Goodbye\n"); + }); - process.stdin.pipe(process.stdout, { end: false }); +Note that `process.stderr` and `process.stdout` are never closed until +the process exits, regardless of the specified options. - process.stdin.on("end", function() { - process.stdout.write("Goodbye\n"); }); +### readable.unpipe([destination]) + +* `destination` {Writable Stream} Optional + +Undo a previously established `pipe()`. If no destination is +provided, then all previously established pipes are removed. + +### readable.pause() + +Switches the readable stream into "old mode", where data is emitted +using a `'data'` event rather than being buffered for consumption via +the `read()` method. + +Ceases the flow of data. No `'data'` events are emitted while the +stream is in a paused state. + +### readable.resume() + +Switches the readable stream into "old mode", where data is emitted +using a `'data'` event rather than being buffered for consumption via +the `read()` method. + +Resumes the incoming `'data'` events after a `pause()`. -## Writable Stream +## Class: stream.Writable -A `Writable Stream` has the following methods, members, and events. +A `Writable` Stream has the following methods, members, and events. + +Note that `stream.Writable` is an abstract class designed to be +extended with an underlying implementation of the `_write(chunk, cb)` +method. (See below.) + +### new stream.Writable([options]) + +* `options` {Object} + * `highWaterMark` {Number} Buffer level when `write()` starts + returning false. Default=16kb + * `lowWaterMark` {Number} The buffer level when `'drain'` is + emitted. Default=0 + * `decodeStrings` {Boolean} Whether or not to decode strings into + Buffers before passing them to `_write()`. Default=true + +In classes that extend the Writable class, make sure to call the +constructor so that the buffering settings can be properly +initialized. + +### writable.\_write(chunk, callback) + +* `chunk` {Buffer | Array} The data to be written +* `callback` {Function} Called with an error, or null when finished + +All Writable stream implementations must provide a `_write` method to +send data to the underlying resource. + +**This function MUST NOT be called directly.** It should be +implemented by child classes, and called by the internal Writable +class methods only. + +Call the callback using the standard `callback(error)` pattern to +signal that the write completed successfully or with an error. + +If the `decodeStrings` flag is set in the constructor options, then +`chunk` will be an array rather than a Buffer. This is to support +implementations that have an optimized handling for certain string +data encodings. + +This method is prefixed with an underscore because it is internal to +the class that defines it, and should not be called directly by user +programs. However, you **are** expected to override this method in +your own extension classes. + + +### writable.write(chunk, [encoding], [callback]) + +* `chunk` {Buffer | String} Data to be written +* `encoding` {String} Optional. If `chunk` is a string, then encoding + defaults to `'utf8'` +* `callback` {Function} Optional. Called when this chunk is + successfully written. +* Returns {Boolean} + +Writes `chunk` to the stream. Returns `true` if the data has been +flushed to the underlying resource. Returns `false` to indicate that +the buffer is full, and the data will be sent out in the future. The +`'drain'` event will indicate when the buffer is empty again. + +The specifics of when `write()` will return false, and when a +subsequent `'drain'` event will be emitted, are determined by the +`highWaterMark` and `lowWaterMark` options provided to the +constructor. + +### writable.end([chunk], [encoding]) + +* `chunk` {Buffer | String} Optional final data to be written +* `encoding` {String} Optional. If `chunk` is a string, then encoding + defaults to `'utf8'` + +Call this method to signal the end of the data being written to the +stream. ### Event: 'drain' -`function () { }` - -Emitted when the stream's write queue empties and it's safe to write without -buffering again. Listen for it when `stream.write()` returns `false`. - -The `'drain'` event can happen at *any* time, regardless of whether or not -`stream.write()` has previously returned `false`. To avoid receiving unwanted -`'drain'` events, listen using `stream.once()`. - -### Event: 'error' - -`function (exception) { }` - -Emitted on error with the exception `exception`. +Emitted when the stream's write queue empties and it's safe to write +without buffering again. Listen for it when `stream.write()` returns +`false`. ### Event: 'close' -`function () { }` - -Emitted when the underlying file descriptor has been closed. +Emitted when the underlying resource (for example, the backing file +descriptor) has been closed. Not all streams will emit this. ### Event: 'pipe' -`function (src) { }` +* `source` {Readable Stream} Emitted when the stream is passed to a readable stream's pipe method. -### stream.writable +### Event 'unpipe' -A boolean that is `true` by default, but turns `false` after an -`'error'` occurred or `end()` / `destroy()` was called. +* `source` {Readable Stream} -### stream.write(string, [encoding]) +Emitted when a previously established `pipe()` is removed using the +source Readable stream's `unpipe()` method. -Writes `string` with the given `encoding` to the stream. Returns `true` -if the string has been flushed to the kernel buffer. Returns `false` to -indicate that the kernel buffer is full, and the data will be sent out -in the future. The `'drain'` event will indicate when the kernel buffer -is empty again. The `encoding` defaults to `'utf8'`. +## Class: stream.Duplex -### stream.write(buffer) + -Same as the above except with a raw buffer. +A "duplex" stream is one that is both Readable and Writable, such as a +TCP socket connection. -### stream.end() +Note that `stream.Duplex` is an abstract class designed to be +extended with an underlying implementation of the `_read(size, cb)` +and `_write(chunk, callback)` methods as you would with a Readable or +Writable stream class. -Terminates the stream with EOF or FIN. This call will allow queued -write data to be sent before closing the stream. +Since JavaScript doesn't have multiple prototypal inheritance, this +class prototypally inherits from Readable, and then parasitically from +Writable. It is thus up to the user to implement both the lowlevel +`_read(n,cb)` method as well as the lowlevel `_write(chunk,cb)` method +on extension duplex classes. -### stream.end(string, encoding) +### new stream.Duplex(options) -Sends `string` with the given `encoding` and terminates the stream with -EOF or FIN. This is useful to reduce the number of packets sent. +* `options` {Object} Passed to both Writable and Readable + constructors. Also has the following fields: + * `allowHalfOpen` {Boolean} Default=true. If set to `false`, then + the stream will automatically end the readable side when the + writable side ends and vice versa. -### stream.end(buffer) +In classes that extend the Duplex class, make sure to call the +constructor so that the buffering settings can be properly +initialized. -Same as above but with a `buffer`. +## Class: stream.Transform -### stream.destroy() +A "transform" stream is a duplex stream where the output is causally +connected in some way to the input, such as a zlib stream or a crypto +stream. -Closes the underlying file descriptor. Stream is no longer `writable` -nor `readable`. The stream will not emit any more 'data', or 'end' -events. Any queued write data will not be sent. The stream should emit -'close' event once its resources have been disposed of. +There is no requirement that the output be the same size as the input, +the same number of chunks, or arrive at the same time. For example, a +Hash stream will only ever have a single chunk of output which is +provided when the input is ended. A zlib stream will either produce +much smaller or much larger than its input. -### stream.destroySoon() +Rather than implement the `_read()` and `_write()` methods, Transform +classes must implement the `_transform()` method, and may optionally +also implement the `_flush()` method. (See below.) + +### new stream.Transform([options]) + +* `options` {Object} Passed to both Writable and Readable + constructors. + +In classes that extend the Transform class, make sure to call the +constructor so that the buffering settings can be properly +initialized. + +### transform.\_transform(chunk, outputFn, callback) + +* `chunk` {Buffer} The chunk to be transformed. +* `outputFn` {Function} Call this function with any output data to be + passed to the readable interface. +* `callback` {Function} Call this function (optionally with an error + argument) when you are done processing the supplied chunk. + +All Transform stream implementations must provide a `_transform` +method to accept input and produce output. + +**This function MUST NOT be called directly.** It should be +implemented by child classes, and called by the internal Transform +class methods only. + +`_transform` should do whatever has to be done in this specific +Transform class, to handle the bytes being written, and pass them off +to the readable portion of the interface. Do asynchronous I/O, +process things, and so on. + +Call the callback function only when the current chunk is completely +consumed. Note that this may mean that you call the `outputFn` zero +or more times, depending on how much data you want to output as a +result of this chunk. + +This method is prefixed with an underscore because it is internal to +the class that defines it, and should not be called directly by user +programs. However, you **are** expected to override this method in +your own extension classes. + +### transform.\_flush(outputFn, callback) + +* `outputFn` {Function} Call this function with any output data to be + passed to the readable interface. +* `callback` {Function} Call this function (optionally with an error + argument) when you are done flushing any remaining data. + +**This function MUST NOT be called directly.** It MAY be implemented +by child classes, and if so, will be called by the internal Transform +class methods only. + +In some cases, your transform operation may need to emit a bit more +data at the end of the stream. For example, a `Zlib` compression +stream will store up some internal state so that it can optimally +compress the output. At the end, however, it needs to do the best it +can with what is left, so that the data will be complete. + +In those cases, you can implement a `_flush` method, which will be +called at the very end, after all the written data is consumed, but +before emitting `end` to signal the end of the readable side. Just +like with `_transform`, call `outputFn` zero or more times, as +appropriate, and call `callback` when the flush operation is complete. + +This method is prefixed with an underscore because it is internal to +the class that defines it, and should not be called directly by user +programs. However, you **are** expected to override this method in +your own extension classes. + + +## Class: stream.PassThrough + +This is a trivial implementation of a `Transform` stream that simply +passes the input bytes across to the output. Its purpose is mainly +for examples and testing, but there are occasionally use cases where +it can come in handy. -After the write queue is drained, close the file descriptor. -`destroySoon()` can still destroy straight away, as long as there is no -data left in the queue for writes. [EventEmitter]: events.html#events_class_events_eventemitter From 854171dc6f238be528e21e905c4764d9522b7033 Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 13 Dec 2012 11:15:29 -0800 Subject: [PATCH 60/72] streams2: Remove extraneous bufferSize setting --- lib/_stream_readable.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js index 8c6e1b5b4fd..9baa2e6bfa9 100644 --- a/lib/_stream_readable.js +++ b/lib/_stream_readable.js @@ -32,9 +32,6 @@ util.inherits(Readable, Stream); function ReadableState(options, stream) { options = options || {}; - // cast to an int - this.bufferSize = ~~this.bufferSize; - // the argument passed to this._read(n,cb) this.bufferSize = options.hasOwnProperty('bufferSize') ? options.bufferSize : 16 * 1024; From 7742257febb511b97d8859cc03df9e09d8b7e0c3 Mon Sep 17 00:00:00 2001 From: isaacs Date: Sun, 9 Dec 2012 15:12:19 -0800 Subject: [PATCH 61/72] benchmark: Add once() function to net-pipe benchmark fixture --- benchmark/net-pipe.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/net-pipe.js b/benchmark/net-pipe.js index e678c2a9f22..0890e292eb7 100644 --- a/benchmark/net-pipe.js +++ b/benchmark/net-pipe.js @@ -27,7 +27,7 @@ Writer.prototype.write = function(chunk, encoding, cb) { // doesn't matter, never emits anything. Writer.prototype.on = function() {}; - +Writer.prototype.once = function() {}; Writer.prototype.emit = function() {}; var statCounter = 0; From 8a3befa0c65ba2ee653e3d3b39360e974536f3ef Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 12 Dec 2012 21:18:57 -0800 Subject: [PATCH 62/72] net: Refactor to use streams2 This is a combination of 6 commits. * XXX net fixup lcase stream * net: Refactor to use streams2 Use 'socket.resume()' in many tests to trigger old-mode behavior. * net: Call destroy() if shutdown() is not provided This is important for TTY wrap streams * net: Call .end() in socket.destroySoon if necessary This makes the http 1.0 keepAlive test pass, also. * net wtf-ish stuff kinda busted * net fixup --- lib/net.js | 484 ++++++++++++++++++++++++++++------------------------- 1 file changed, 259 insertions(+), 225 deletions(-) diff --git a/lib/net.js b/lib/net.js index 81d02a5a646..d0a2c5a6279 100644 --- a/lib/net.js +++ b/lib/net.js @@ -20,7 +20,7 @@ // USE OR OTHER DEALINGS IN THE SOFTWARE. var events = require('events'); -var Stream = require('stream'); +var stream = require('stream'); var timers = require('timers'); var util = require('util'); var assert = require('assert'); @@ -42,16 +42,16 @@ function createTCP() { } -/* Bit flags for socket._flags */ -var FLAG_GOT_EOF = 1 << 0; -var FLAG_SHUTDOWN = 1 << 1; -var FLAG_DESTROY_SOON = 1 << 2; -var FLAG_SHUTDOWN_QUEUED = 1 << 3; - - var debug; if (process.env.NODE_DEBUG && /net/.test(process.env.NODE_DEBUG)) { - debug = function(x) { console.error('NET:', x); }; + var pid = process.pid; + debug = function(x) { + // if console is not set up yet, then skip this. + if (!console.error) + return; + console.error('NET: %d', pid, + util.format.apply(util, arguments).slice(0, 500)); + }; } else { debug = function() { }; } @@ -110,12 +110,8 @@ function normalizeConnectArgs(args) { exports._normalizeConnectArgs = normalizeConnectArgs; -/* called when creating new Socket, or when re-using a closed Socket */ +// called when creating new Socket, or when re-using a closed Socket function initSocketHandle(self) { - self._pendingWriteReqs = 0; - - self._flags = 0; - self._connectQueueSize = 0; self.destroyed = false; self.errorEmitted = false; self.bytesRead = 0; @@ -131,8 +127,6 @@ function initSocketHandle(self) { function Socket(options) { if (!(this instanceof Socket)) return new Socket(options); - Stream.call(this); - switch (typeof options) { case 'number': options = { fd: options }; // Legacy interface. @@ -142,7 +136,10 @@ function Socket(options) { break; } - if (typeof options.fd === 'undefined') { + this.readable = this.writable = false; + if (options.handle) { + this._handle = options.handle; // private + } else if (typeof options.fd === 'undefined') { this._handle = options && options.handle; // private } else { this._handle = createPipe(); @@ -150,17 +147,105 @@ function Socket(options) { this.readable = this.writable = true; } - initSocketHandle(this); - this.allowHalfOpen = options && options.allowHalfOpen; -} -util.inherits(Socket, Stream); + this.onend = null; + // shut down the socket when we're finished with it. + this.on('finish', onSocketFinish); + this.on('_socketEnd', onSocketEnd); + + initSocketHandle(this); + + this._pendingWrite = null; + + stream.Duplex.call(this, options); + + // handle strings directly + this._writableState.decodeStrings = false; + + // default to *not* allowing half open sockets + this.allowHalfOpen = options && options.allowHalfOpen || false; + + // if we have a handle, then start the flow of data into the + // buffer. if not, then this will happen when we connect + if (this._handle && (!options || options.readable !== false)) + this.read(0); +} +util.inherits(Socket, stream.Duplex); + +// the user has called .end(), and all the bytes have been +// sent out to the other side. +// If allowHalfOpen is false, or if the readable side has +// ended already, then destroy. +// If allowHalfOpen is true, then we need to do a shutdown, +// so that only the writable side will be cleaned up. +function onSocketFinish() { + debug('onSocketFinish'); + if (this._readableState.ended) { + debug('oSF: ended, destroy', this._readableState); + return this.destroy(); + } + + debug('oSF: not ended, call shutdown()'); + + // otherwise, just shutdown, or destroy() if not possible + if (!this._handle.shutdown) + return this.destroy(); + + var shutdownReq = this._handle.shutdown(); + + if (!shutdownReq) + return this._destroy(errnoException(errno, 'shutdown')); + + shutdownReq.oncomplete = afterShutdown; +} + + +function afterShutdown(status, handle, req) { + var self = handle.owner; + + debug('afterShutdown destroyed=%j', self.destroyed, + self._readableState); + + // callback may come after call to destroy. + if (self.destroyed) + return; + + if (self._readableState.ended) { + debug('readableState ended, destroying'); + self.destroy(); + } else { + self.once('_socketEnd', self.destroy); + } +} + +// the EOF has been received, and no more bytes are coming. +// if the writable side has ended already, then clean everything +// up. +function onSocketEnd() { + // XXX Should not have to do as much crap in this function. + // ended should already be true, since this is called *after* + // the EOF errno and onread has returned null to the _read cb. + debug('onSocketEnd', this._readableState); + this._readableState.ended = true; + if (this._readableState.endEmitted) { + this.readable = false; + } else { + this.once('end', function() { + this.readable = false; + }); + this.read(0); + } + + if (!this.allowHalfOpen) + this.destroySoon(); +} exports.Socket = Socket; exports.Stream = Socket; // Legacy naming. Socket.prototype.listen = function() { + debug('socket.listen'); var self = this; self.on('connection', arguments[0]); listen(self, null, null, null); @@ -230,96 +315,62 @@ Object.defineProperty(Socket.prototype, 'readyState', { Object.defineProperty(Socket.prototype, 'bufferSize', { get: function() { if (this._handle) { - return this._handle.writeQueueSize + this._connectQueueSize; + return this._handle.writeQueueSize; } } }); -Socket.prototype.pause = function() { - this._paused = true; - if (this._handle && !this._connecting) { - this._handle.readStop(); +// Just call handle.readStart until we have enough in the buffer +Socket.prototype._read = function(n, callback) { + debug('_read'); + if (this._connecting || !this._handle) { + debug('_read wait for connection'); + this.once('connect', this._read.bind(this, n, callback)); + return; } -}; + assert(callback === this._readableState.onread); + assert(this._readableState.reading = true); -Socket.prototype.resume = function() { - this._paused = false; - if (this._handle && !this._connecting) { - this._handle.readStart(); + if (!this._handle.reading) { + debug('Socket._read readStart'); + this._handle.reading = true; + var r = this._handle.readStart(); + if (r) + this._destroy(errnoException(errno, 'read')); + } else { + debug('readStart already has been called.'); } }; Socket.prototype.end = function(data, encoding) { - if (this._connecting && ((this._flags & FLAG_SHUTDOWN_QUEUED) == 0)) { - // still connecting, add data to buffer - if (data) this.write(data, encoding); - this.writable = false; - this._flags |= FLAG_SHUTDOWN_QUEUED; - } - - if (!this.writable) return; + stream.Duplex.prototype.end.call(this, data, encoding); this.writable = false; - - if (data) this.write(data, encoding); DTRACE_NET_STREAM_END(this); - if (!this.readable) { - this.destroySoon(); - } else { - this._flags |= FLAG_SHUTDOWN; - var shutdownReq = this._handle.shutdown(); - - if (!shutdownReq) { - this._destroy(errnoException(errno, 'shutdown')); - return false; - } - - shutdownReq.oncomplete = afterShutdown; - } - - return true; + // just in case we're waiting for an EOF. + if (!this._readableState.endEmitted) + this.read(0); + return; }; -function afterShutdown(status, handle, req) { - var self = handle.owner; - - assert.ok(self._flags & FLAG_SHUTDOWN); - assert.ok(!self.writable); - - // callback may come after call to destroy. - if (self.destroyed) { - return; - } - - if (self._flags & FLAG_GOT_EOF || !self.readable) { - self._destroy(); - } else { - } -} - - Socket.prototype.destroySoon = function() { - this.writable = false; - this._flags |= FLAG_DESTROY_SOON; + if (this.writable) + this.end(); - if (this._pendingWriteReqs == 0) { - this._destroy(); - } -}; - - -Socket.prototype._connectQueueCleanUp = function(exception) { - this._connecting = false; - this._connectQueueSize = 0; - this._connectQueue = null; + if (this._writableState.finishing || this._writableState.finished) + this.destroy(); + else + this.once('finish', this.destroy); }; Socket.prototype._destroy = function(exception, cb) { + debug('destroy'); + var self = this; function fireErrorCallbacks() { @@ -333,13 +384,12 @@ Socket.prototype._destroy = function(exception, cb) { }; if (this.destroyed) { + debug('already destroyed, fire error callbacks'); fireErrorCallbacks(); return; } - self._connectQueueCleanUp(); - - debug('destroy'); + self._connecting = false; this.readable = this.writable = false; @@ -347,6 +397,8 @@ Socket.prototype._destroy = function(exception, cb) { debug('close'); if (this._handle) { + if (this !== process.stderr) + debug('close handle'); this._handle.close(); this._handle.onread = noop; this._handle = null; @@ -355,6 +407,7 @@ Socket.prototype._destroy = function(exception, cb) { fireErrorCallbacks(); process.nextTick(function() { + debug('emit close'); self.emit('close', exception ? true : false); }); @@ -362,6 +415,7 @@ Socket.prototype._destroy = function(exception, cb) { if (this.server) { COUNTER_NET_SERVER_CONNECTION_CLOSE(this); + debug('has server'); this.server._connections--; if (this.server._emitCloseIfDrained) { this.server._emitCloseIfDrained(); @@ -371,10 +425,13 @@ Socket.prototype._destroy = function(exception, cb) { Socket.prototype.destroy = function(exception) { + debug('destroy', exception); this._destroy(exception); }; +// This function is called whenever the handle gets a +// buffer, or when there's an error reading. function onread(buffer, offset, length) { var handle = this; var self = handle.owner; @@ -383,47 +440,56 @@ function onread(buffer, offset, length) { timers.active(self); var end = offset + length; + debug('onread', global.errno, offset, length, end); if (buffer) { - // Emit 'data' event. + debug('got data'); - if (self._decoder) { - // Emit a string. - var string = self._decoder.write(buffer.slice(offset, end)); - if (string.length) self.emit('data', string); - } else { - // Emit a slice. Attempt to avoid slicing the buffer if no one is - // listening for 'data'. - if (self._events && self._events['data']) { - self.emit('data', buffer.slice(offset, end)); - } + // read success. + // In theory (and in practice) calling readStop right now + // will prevent this from being called again until _read() gets + // called again. + + // if we didn't get any bytes, that doesn't necessarily mean EOF. + // wait for the next one. + if (offset === end) { + debug('not any data, keep waiting'); + return; } + // if it's not enough data, we'll just call handle.readStart() + // again right away. self.bytesRead += length; + self._readableState.onread(null, buffer.slice(offset, end)); + + if (handle.reading && !self._readableState.reading) { + handle.reading = false; + debug('readStop'); + var r = handle.readStop(); + if (r) + self._destroy(errnoException(errno, 'read')); + } // Optimization: emit the original buffer with end points if (self.ondata) self.ondata(buffer, offset, end); } else if (errno == 'EOF') { - // EOF - self.readable = false; + debug('EOF'); - assert.ok(!(self._flags & FLAG_GOT_EOF)); - self._flags |= FLAG_GOT_EOF; + if (self._readableState.length === 0) + self.readable = false; - // We call destroy() before end(). 'close' not emitted until nextTick so - // the 'end' event will come first as required. - if (!self.writable) self._destroy(); + if (self.onend) self.once('end', self.onend); - if (!self.allowHalfOpen) self.end(); - if (self._decoder) { - var ret = self._decoder.end(); - if (ret) - self.emit('data', ret); - } - if (self._events && self._events['end']) self.emit('end'); - if (self.onend) self.onend(); + // send a null to the _read cb to signal the end of data. + self._readableState.onread(null, null); + + // internal end event so that we know that the actual socket + // is no longer readable, and we can start the shutdown + // procedure. No need to wait for all the data to be consumed. + self.emit('_socketEnd'); } else { + debug('error', errno); // Error if (errno == 'ECONNRESET') { self._destroy(); @@ -434,12 +500,6 @@ function onread(buffer, offset, length) { } -Socket.prototype.setEncoding = function(encoding) { - var StringDecoder = require('string_decoder').StringDecoder; // lazy load - this._decoder = new StringDecoder(encoding); -}; - - Socket.prototype._getpeername = function() { if (!this._handle || !this._handle.getpeername) { return {}; @@ -465,63 +525,39 @@ Socket.prototype.__defineGetter__('remotePort', function() { }); -/* - * Arguments data, [encoding], [cb] - */ -Socket.prototype.write = function(data, arg1, arg2) { - var encoding, cb; - - // parse arguments - if (arg1) { - if (typeof arg1 === 'string') { - encoding = arg1; - cb = arg2; - } else if (typeof arg1 === 'function') { - cb = arg1; - } else { - throw new Error('bad arg'); - } - } - - if (typeof data === 'string') { - encoding = (encoding || 'utf8').toLowerCase(); - switch (encoding) { - case 'utf8': - case 'utf-8': - case 'ascii': - case 'ucs2': - case 'ucs-2': - case 'utf16le': - case 'utf-16le': - // This encoding can be handled in the binding layer. - break; - - default: - data = new Buffer(data, encoding); - } - } else if (!Buffer.isBuffer(data)) { - throw new TypeError('First argument must be a buffer or a string.'); - } - - // If we are still connecting, then buffer this for later. - if (this._connecting) { - this._connectQueueSize += data.length; - if (this._connectQueue) { - this._connectQueue.push([data, encoding, cb]); - } else { - this._connectQueue = [[data, encoding, cb]]; - } - return false; - } - - return this._write(data, encoding, cb); +Socket.prototype.write = function(chunk, encoding, cb) { + if (typeof chunk !== 'string' && !Buffer.isBuffer(chunk)) + throw new TypeError('invalid data'); + return stream.Duplex.prototype.write.apply(this, arguments); }; -Socket.prototype._write = function(data, encoding, cb) { +Socket.prototype._write = function(dataEncoding, cb) { + assert(Array.isArray(dataEncoding)); + var data = dataEncoding[0]; + var encoding = dataEncoding[1] || 'utf8'; + + if (this !== process.stderr && this !== process.stdout) + debug('Socket._write'); + + // If we are still connecting, then buffer this for later. + // The Writable logic will buffer up any more writes while + // waiting for this one to be done. + if (this._connecting) { + debug('_write: waiting for connection'); + this._pendingWrite = dataEncoding; + this.once('connect', function() { + debug('_write: connected now, try again'); + this._write(dataEncoding, cb); + }); + return; + } + this._pendingWrite = null; + timers.active(this); if (!this._handle) { + debug('already destroyed'); this._destroy(new Error('This socket is closed.'), cb); return false; } @@ -550,39 +586,32 @@ Socket.prototype._write = function(data, encoding, cb) { break; default: - assert(0); + writeReq = this._handle.writeBuffer(new Buffer(data, encoding)); + break; } } - if (!writeReq || typeof writeReq !== 'object') { - this._destroy(errnoException(errno, 'write'), cb); - return false; - } + if (!writeReq || typeof writeReq !== 'object') + return this._destroy(errnoException(errno, 'write'), cb); writeReq.oncomplete = afterWrite; writeReq.cb = cb; - this._pendingWriteReqs++; this._bytesDispatched += writeReq.bytes; - - return this._handle.writeQueueSize == 0; }; Socket.prototype.__defineGetter__('bytesWritten', function() { var bytes = this._bytesDispatched, - connectQueue = this._connectQueue; + state = this._writableState, + pending = this._pendingWrite; - if (connectQueue) { - connectQueue.forEach(function(el) { - var data = el[0]; - if (Buffer.isBuffer(data)) { - bytes += data.length; - } else { - bytes += Buffer.byteLength(data, el[1]); - } - }, this); - } + state.buffer.forEach(function(el) { + bytes += Buffer.byteLength(el[0], el[1]); + }); + + if (pending) + bytes += Buffer.byteLength(pending[0], pending[1]); return bytes; }); @@ -590,30 +619,28 @@ Socket.prototype.__defineGetter__('bytesWritten', function() { function afterWrite(status, handle, req) { var self = handle.owner; + var state = self._writableState; + if (self !== process.stderr && self !== process.stdout) + debug('afterWrite', status, req); // callback may come after call to destroy. if (self.destroyed) { + debug('afterWrite destroyed'); return; } if (status) { + debug('write failure', errnoException(errno, 'write')); self._destroy(errnoException(errno, 'write'), req.cb); return; } timers.active(self); - self._pendingWriteReqs--; + if (self !== process.stderr && self !== process.stdout) + debug('afterWrite call cb'); - if (self._pendingWriteReqs == 0) { - self.emit('drain'); - } - - if (req.cb) req.cb(); - - if (self._pendingWriteReqs == 0 && self._flags & FLAG_DESTROY_SOON) { - self._destroy(); - } + req.cb.call(self); } @@ -663,10 +690,21 @@ Socket.prototype.connect = function(options, cb) { return Socket.prototype.connect.apply(this, args); } + if (this.destroyed) { + this._readableState.reading = false; + this._readableState.ended = false; + this._writableState.ended = false; + this._writableState.ending = false; + this._writableState.finished = false; + this._writableState.finishing = false; + this.destroyed = false; + this._handle = null; + } + var self = this; var pipe = !!options.path; - if (this.destroyed || !this._handle) { + if (!this._handle) { this._handle = pipe ? createPipe() : createTCP(); initSocketHandle(this); } @@ -755,28 +793,15 @@ function afterConnect(status, handle, req, readable, writable) { self.writable = writable; timers.active(self); - if (self.readable && !self._paused) { - handle.readStart(); - } - - if (self._connectQueue) { - debug('Drain the connect queue'); - var connectQueue = self._connectQueue; - for (var i = 0; i < connectQueue.length; i++) { - self._write.apply(self, connectQueue[i]); - } - self._connectQueueCleanUp(); - } - self.emit('connect'); - if (self._flags & FLAG_SHUTDOWN_QUEUED) { - // end called before connected - call end now with no data - self._flags &= ~FLAG_SHUTDOWN_QUEUED; - self.end(); - } + // start the first read, or get an immediate EOF. + // this doesn't actually consume any bytes, because len=0. + if (readable) + self.read(0); + } else { - self._connectQueueCleanUp(); + self._connecting = false; self._destroy(errnoException(errno, 'connect')); } } @@ -831,9 +856,9 @@ function Server(/* [ options, ] listener */) { configurable: true, enumerable: true }); - this.allowHalfOpen = options.allowHalfOpen || false; - this._handle = null; + + this.allowHalfOpen = options.allowHalfOpen || false; } util.inherits(Server, events.EventEmitter); exports.Server = Server; @@ -901,12 +926,14 @@ var createServerHandle = exports._createServerHandle = Server.prototype._listen2 = function(address, port, addressType, backlog, fd) { + debug('listen2', address, port, addressType, backlog); var self = this; var r = 0; // If there is not yet a handle, we need to create one and bind. // In the case of a server sent via IPC, we don't need to do this. if (!self._handle) { + debug('_listen2: create a handle'); self._handle = createServerHandle(address, port, addressType, fd); if (!self._handle) { var error = errnoException(errno, 'listen'); @@ -915,6 +942,8 @@ Server.prototype._listen2 = function(address, port, addressType, backlog, fd) { }); return; } + } else { + debug('_listen2: have a handle already'); } self._handle.onconnection = onconnection; @@ -1049,7 +1078,6 @@ function onconnection(clientHandle) { }); socket.readable = socket.writable = true; - clientHandle.readStart(); self._connections++; socket.server = self; @@ -1086,11 +1114,17 @@ Server.prototype.close = function(cb) { }; Server.prototype._emitCloseIfDrained = function() { + debug('SERVER _emitCloseIfDrained'); var self = this; - if (self._handle || self._connections) return; + if (self._handle || self._connections) { + debug('SERVER handle? %j connections? %d', + !!self._handle, self._connections); + return; + } process.nextTick(function() { + debug('SERVER: emit close'); self.emit('close'); }); }; From 695abba5ac3365792c17ae3b83ba1272fb34ba5a Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 13 Dec 2012 09:51:31 -0800 Subject: [PATCH 63/72] test: Fix many tests for streams2 net refactor --- test/simple/test-child-process-disconnect.js | 2 + test/simple/test-child-process-fork-net2.js | 58 ++++++++++++++++---- test/simple/test-cluster-message.js | 2 + test/simple/test-http-1.0-keep-alive.js | 4 ++ test/simple/test-net-after-close.js | 2 + test/simple/test-net-binary.js | 28 ++++++++-- test/simple/test-net-bytes-stats.js | 15 +++-- test/simple/test-net-can-reset-timeout.js | 2 + test/simple/test-net-connect-buffer.js | 13 ++++- test/simple/test-net-connect-options.js | 4 ++ test/simple/test-net-pingpong.js | 9 ++- test/simple/test-net-reconnect.js | 18 ++++-- test/simple/test-net-remote-address-port.js | 1 + test/simple/test-net-write-after-close.js | 4 ++ test/simple/test-pipe-file-to-http.js | 13 +++++ test/simple/test-pipe.js | 1 + test/simple/test-tcp-wrap-connect.js | 1 + test/simple/test-zlib-random-byte-pipes.js | 17 ++++++ 18 files changed, 165 insertions(+), 29 deletions(-) diff --git a/test/simple/test-child-process-disconnect.js b/test/simple/test-child-process-disconnect.js index 2136aaf3e6f..162e7dde8be 100644 --- a/test/simple/test-child-process-disconnect.js +++ b/test/simple/test-child-process-disconnect.js @@ -31,6 +31,8 @@ if (process.argv[2] === 'child') { server.on('connection', function(socket) { + socket.resume(); + process.on('disconnect', function() { socket.end((process.connected).toString()); }); diff --git a/test/simple/test-child-process-fork-net2.js b/test/simple/test-child-process-fork-net2.js index 48713566a62..be749fe3197 100644 --- a/test/simple/test-child-process-fork-net2.js +++ b/test/simple/test-child-process-fork-net2.js @@ -23,31 +23,59 @@ var assert = require('assert'); var common = require('../common'); var fork = require('child_process').fork; var net = require('net'); +var count = 12; if (process.argv[2] === 'child') { - var endMe = null; + var needEnd = []; process.on('message', function(m, socket) { if (!socket) return; + console.error('got socket', m); + // will call .end('end') or .write('write'); socket[m](m); + socket.resume(); + + socket.on('data', function() { + console.error('%d socket.data', process.pid, m); + }); + + socket.on('end', function() { + console.error('%d socket.end', process.pid, m); + }); + // store the unfinished socket if (m === 'write') { - endMe = socket; + needEnd.push(socket); } + + socket.on('close', function() { + console.error('%d socket.close', process.pid, m); + }); + + socket.on('finish', function() { + console.error('%d socket finished', process.pid, m); + }); }); process.on('message', function(m) { if (m !== 'close') return; - endMe.end('end'); - endMe = null; + console.error('got close message'); + needEnd.forEach(function(endMe, i) { + console.error('%d ending %d', process.pid, i); + endMe.end('end'); + }); }); process.on('disconnect', function() { - endMe.end('end'); + console.error('%d process disconnect, ending', process.pid); + needEnd.forEach(function(endMe, i) { + console.error('%d ending %d', process.pid, i); + endMe.end('end'); + }); endMe = null; }); @@ -61,7 +89,7 @@ if (process.argv[2] === 'child') { var connected = 0; server.on('connection', function(socket) { - switch (connected) { + switch (connected % 6) { case 0: child1.send('end', socket); break; case 1: @@ -77,7 +105,7 @@ if (process.argv[2] === 'child') { } connected += 1; - if (connected === 6) { + if (connected === count) { closeServer(); } }); @@ -85,17 +113,23 @@ if (process.argv[2] === 'child') { var disconnected = 0; server.on('listening', function() { - var j = 6, client; + var j = count, client; while (j--) { client = net.connect(common.PORT, '127.0.0.1'); client.on('close', function() { + console.error('CLIENT: close event in master'); disconnected += 1; }); + // XXX This resume() should be unnecessary. + // a stream high water mark should be enough to keep + // consuming the input. + client.resume(); } }); var closeEmitted = false; server.on('close', function() { + console.error('server close'); closeEmitted = true; child1.kill(); @@ -107,14 +141,18 @@ if (process.argv[2] === 'child') { var timeElasped = 0; var closeServer = function() { + console.error('closeServer'); var startTime = Date.now(); server.on('close', function() { + console.error('emit(close)'); timeElasped = Date.now() - startTime; }); + console.error('calling server.close'); server.close(); setTimeout(function() { + console.error('sending close to children'); child1.send('close'); child2.send('close'); child3.disconnect(); @@ -122,8 +160,8 @@ if (process.argv[2] === 'child') { }; process.on('exit', function() { - assert.equal(disconnected, 6); - assert.equal(connected, 6); + assert.equal(disconnected, count); + assert.equal(connected, count); assert.ok(closeEmitted); assert.ok(timeElasped >= 190 && timeElasped <= 1000, 'timeElasped was not between 190 and 1000 ms'); diff --git a/test/simple/test-cluster-message.js b/test/simple/test-cluster-message.js index 313355f7f6d..3a76dbc7407 100644 --- a/test/simple/test-cluster-message.js +++ b/test/simple/test-cluster-message.js @@ -81,6 +81,7 @@ else if (cluster.isMaster) { var check = function(type, result) { checks[type].receive = true; checks[type].correct = result; + console.error('check', checks); var missing = false; forEach(checks, function(type) { @@ -88,6 +89,7 @@ else if (cluster.isMaster) { }); if (missing === false) { + console.error('end client'); client.end(); } }; diff --git a/test/simple/test-http-1.0-keep-alive.js b/test/simple/test-http-1.0-keep-alive.js index 623facb1738..851409d28bb 100644 --- a/test/simple/test-http-1.0-keep-alive.js +++ b/test/simple/test-http-1.0-keep-alive.js @@ -115,6 +115,7 @@ function check(tests) { function server(req, res) { if (current + 1 === test.responses.length) this.close(); var ctx = test.responses[current]; + console.error('< SERVER SENDING RESPONSE', ctx); res.writeHead(200, ctx.headers); ctx.chunks.slice(0, -1).forEach(function(chunk) { res.write(chunk) }); res.end(ctx.chunks[ctx.chunks.length - 1]); @@ -126,16 +127,19 @@ function check(tests) { function connected() { var ctx = test.requests[current]; + console.error(' > CLIENT SENDING REQUEST', ctx); conn.setEncoding('utf8'); conn.write(ctx.data); function onclose() { + console.error(' > CLIENT CLOSE'); if (!ctx.expectClose) throw new Error('unexpected close'); client(); } conn.on('close', onclose); function ondata(s) { + console.error(' > CLIENT ONDATA %j %j', s.length, s.toString()); current++; if (ctx.expectClose) return; conn.removeListener('close', onclose); diff --git a/test/simple/test-net-after-close.js b/test/simple/test-net-after-close.js index 65fda21900d..2f3d4c37976 100644 --- a/test/simple/test-net-after-close.js +++ b/test/simple/test-net-after-close.js @@ -25,12 +25,14 @@ var net = require('net'); var closed = false; var server = net.createServer(function(s) { + console.error('SERVER: got connection'); s.end(); }); server.listen(common.PORT, function() { var c = net.createConnection(common.PORT); c.on('close', function() { + console.error('connection closed'); assert.strictEqual(c._handle, null); closed = true; assert.doesNotThrow(function() { diff --git a/test/simple/test-net-binary.js b/test/simple/test-net-binary.js index 349e2da0009..6b41d72d7b0 100644 --- a/test/simple/test-net-binary.js +++ b/test/simple/test-net-binary.js @@ -41,12 +41,15 @@ for (var i = 255; i >= 0; i--) { // safe constructor var echoServer = net.Server(function(connection) { + // connection._readableState.lowWaterMark = 0; + console.error('SERVER got connection'); connection.setEncoding('binary'); connection.on('data', function(chunk) { - common.error('recved: ' + JSON.stringify(chunk)); + common.error('SERVER recved: ' + JSON.stringify(chunk)); connection.write(chunk, 'binary'); }); connection.on('end', function() { + console.error('SERVER ending'); connection.end(); }); }); @@ -55,29 +58,44 @@ echoServer.listen(common.PORT); var recv = ''; echoServer.on('listening', function() { + console.error('SERVER listening'); var j = 0; - var c = net.createConnection(common.PORT); + var c = net.createConnection({ + port: common.PORT + }); + + // c._readableState.lowWaterMark = 0; c.setEncoding('binary'); c.on('data', function(chunk) { - if (j < 256) { - common.error('write ' + j); + console.error('CLIENT data %j', chunk); + var n = j + chunk.length; + while (j < n && j < 256) { + common.error('CLIENT write ' + j); c.write(String.fromCharCode(j), 'binary'); j++; - } else { + } + if (j === 256) { + console.error('CLIENT ending'); c.end(); } recv += chunk; }); c.on('connect', function() { + console.error('CLIENT connected, writing'); c.write(binaryString, 'binary'); }); c.on('close', function() { + console.error('CLIENT closed'); console.dir(recv); echoServer.close(); }); + + c.on('finish', function() { + console.error('CLIENT finished'); + }); }); process.on('exit', function() { diff --git a/test/simple/test-net-bytes-stats.js b/test/simple/test-net-bytes-stats.js index a406e991c8e..0cb08009ee4 100644 --- a/test/simple/test-net-bytes-stats.js +++ b/test/simple/test-net-bytes-stats.js @@ -34,33 +34,40 @@ var count = 0; var tcp = net.Server(function(s) { console.log('tcp server connection'); + // trigger old mode. + s.resume(); + s.on('end', function() { bytesRead += s.bytesRead; console.log('tcp socket disconnect #' + count); }); }); -tcp.listen(common.PORT, function() { +tcp.listen(common.PORT, function doTest() { + console.error('listening'); var socket = net.createConnection(tcpPort); socket.on('connect', function() { count++; - console.log('tcp client connection #' + count); + console.error('CLIENT connect #%d', count); socket.write('foo', function() { + console.error('CLIENT: write cb'); socket.end('bar'); }); }); - socket.on('end', function() { + socket.on('finish', function() { bytesWritten += socket.bytesWritten; - console.log('tcp client disconnect #' + count); + console.error('CLIENT end event #%d', count); }); socket.on('close', function() { + console.error('CLIENT close event #%d', count); console.log('Bytes read: ' + bytesRead); console.log('Bytes written: ' + bytesWritten); if (count < 2) { + console.error('RECONNECTING'); socket.connect(tcpPort); } else { tcp.close(); diff --git a/test/simple/test-net-can-reset-timeout.js b/test/simple/test-net-can-reset-timeout.js index bb9b071427c..b9ea97efe80 100644 --- a/test/simple/test-net-can-reset-timeout.js +++ b/test/simple/test-net-can-reset-timeout.js @@ -28,6 +28,8 @@ var timeoutCount = 0; var server = net.createServer(function(stream) { stream.setTimeout(100); + stream.resume(); + stream.on('timeout', function() { console.log('timeout'); // try to reset the timeout. diff --git a/test/simple/test-net-connect-buffer.js b/test/simple/test-net-connect-buffer.js index 09505936268..679e18e9072 100644 --- a/test/simple/test-net-connect-buffer.js +++ b/test/simple/test-net-connect-buffer.js @@ -38,6 +38,7 @@ var tcp = net.Server(function(s) { }); s.on('end', function() { + console.error('SERVER: end', buf.toString()); assert.equal(buf, "L'État, c'est moi"); console.log('tcp socket disconnect'); s.end(); @@ -50,7 +51,7 @@ var tcp = net.Server(function(s) { }); tcp.listen(common.PORT, function() { - var socket = net.Stream(); + var socket = net.Stream({ highWaterMark: 0 }); console.log('Connecting to socket '); @@ -77,6 +78,7 @@ tcp.listen(common.PORT, function() { {} ].forEach(function(v) { function f() { + console.error('write', v); socket.write(v); } assert.throws(f, TypeError); @@ -90,12 +92,17 @@ tcp.listen(common.PORT, function() { // We're still connecting at this point so the datagram is first pushed onto // the connect queue. Make sure that it's not added to `bytesWritten` again // when the actual write happens. - var r = socket.write(a, function() { + var r = socket.write(a, function(er) { + console.error('write cb'); dataWritten = true; assert.ok(connectHappened); - assert.equal(socket.bytesWritten, Buffer(a + b).length); + console.error('socket.bytesWritten', socket.bytesWritten); + //assert.equal(socket.bytesWritten, Buffer(a + b).length); console.error('data written'); }); + console.error('socket.bytesWritten', socket.bytesWritten); + console.error('write returned', r); + assert.equal(socket.bytesWritten, Buffer(a).length); assert.equal(false, r); diff --git a/test/simple/test-net-connect-options.js b/test/simple/test-net-connect-options.js index 8df692ef7e4..6be3696dae7 100644 --- a/test/simple/test-net-connect-options.js +++ b/test/simple/test-net-connect-options.js @@ -27,6 +27,7 @@ var serverGotEnd = false; var clientGotEnd = false; var server = net.createServer({allowHalfOpen: true}, function(socket) { + socket.resume(); socket.on('end', function() { serverGotEnd = true; }); @@ -39,6 +40,8 @@ server.listen(common.PORT, function() { port: common.PORT, allowHalfOpen: true }, function() { + console.error('client connect cb'); + client.resume(); client.on('end', function() { clientGotEnd = true; setTimeout(function() { @@ -53,6 +56,7 @@ server.listen(common.PORT, function() { }); process.on('exit', function() { + console.error('exit', serverGotEnd, clientGotEnd); assert(serverGotEnd); assert(clientGotEnd); }); diff --git a/test/simple/test-net-pingpong.js b/test/simple/test-net-pingpong.js index a81f0dc80d1..1e7b6681ab2 100644 --- a/test/simple/test-net-pingpong.js +++ b/test/simple/test-net-pingpong.js @@ -60,6 +60,8 @@ function pingPongTest(port, host) { }); socket.on('end', function() { + console.error(socket); + assert.equal(true, socket.allowHalfOpen); assert.equal(true, socket.writable); // because allowHalfOpen assert.equal(false, socket.readable); socket.end(); @@ -129,10 +131,11 @@ function pingPongTest(port, host) { } /* All are run at once, so run on different ports */ +console.log(common.PIPE); pingPongTest(common.PIPE); -pingPongTest(20988); -pingPongTest(20989, 'localhost'); -pingPongTest(20997, '::1'); +pingPongTest(common.PORT); +pingPongTest(common.PORT + 1, 'localhost'); +pingPongTest(common.PORT + 2, '::1'); process.on('exit', function() { assert.equal(4, tests_run); diff --git a/test/simple/test-net-reconnect.js b/test/simple/test-net-reconnect.js index f7fcb8b29be..58e0fef6277 100644 --- a/test/simple/test-net-reconnect.js +++ b/test/simple/test-net-reconnect.js @@ -30,39 +30,49 @@ var client_recv_count = 0; var disconnect_count = 0; var server = net.createServer(function(socket) { + console.error('SERVER: got socket connection'); + socket.resume(); + socket.on('connect', function() { + console.error('SERVER connect, writing'); socket.write('hello\r\n'); }); socket.on('end', function() { + console.error('SERVER socket end, calling end()'); socket.end(); }); socket.on('close', function(had_error) { - //console.log('server had_error: ' + JSON.stringify(had_error)); + console.log('SERVER had_error: ' + JSON.stringify(had_error)); assert.equal(false, had_error); }); }); server.listen(common.PORT, function() { - console.log('listening'); + console.log('SERVER listening'); var client = net.createConnection(common.PORT); client.setEncoding('UTF8'); client.on('connect', function() { - console.log('client connected.'); + console.error('CLIENT connected', client._writableState); }); client.on('data', function(chunk) { client_recv_count += 1; console.log('client_recv_count ' + client_recv_count); assert.equal('hello\r\n', chunk); + console.error('CLIENT: calling end', client._writableState); client.end(); }); + client.on('end', function() { + console.error('CLIENT end'); + }); + client.on('close', function(had_error) { - console.log('disconnect'); + console.log('CLIENT disconnect'); assert.equal(false, had_error); if (disconnect_count++ < N) client.connect(common.PORT); // reconnect diff --git a/test/simple/test-net-remote-address-port.js b/test/simple/test-net-remote-address-port.js index 9b585fce9ef..5d1ae3c8eb6 100644 --- a/test/simple/test-net-remote-address-port.js +++ b/test/simple/test-net-remote-address-port.js @@ -34,6 +34,7 @@ var server = net.createServer(function(socket) { socket.on('end', function() { if (++conns_closed == 2) server.close(); }); + socket.resume(); }); server.listen(common.PORT, 'localhost', function() { diff --git a/test/simple/test-net-write-after-close.js b/test/simple/test-net-write-after-close.js index b77e9af7240..3b98bbc42ea 100644 --- a/test/simple/test-net-write-after-close.js +++ b/test/simple/test-net-write-after-close.js @@ -32,12 +32,16 @@ process.on('exit', function() { }); var server = net.createServer(function(socket) { + socket.resume(); + socket.on('error', function(error) { + console.error('got error, closing server', error); server.close(); gotError = true; }); setTimeout(function() { + console.error('about to try to write'); socket.write('test', function(e) { gotWriteCB = true; }); diff --git a/test/simple/test-pipe-file-to-http.js b/test/simple/test-pipe-file-to-http.js index 99fad6ebb68..1b3ba7089b4 100644 --- a/test/simple/test-pipe-file-to-http.js +++ b/test/simple/test-pipe-file-to-http.js @@ -31,6 +31,7 @@ var clientReqComplete = false; var count = 0; var server = http.createServer(function(req, res) { + console.error('SERVER request'); var timeoutId; assert.equal('POST', req.method); req.pause(); @@ -63,6 +64,8 @@ server.on('listening', function() { cp.exec(cmd, function(err, stdout, stderr) { if (err) throw err; + console.error('EXEC returned successfully stdout=%d stderr=%d', + stdout.length, stderr.length); makeRequest(); }); }); @@ -75,8 +78,15 @@ function makeRequest() { }); common.error('pipe!'); + var s = fs.ReadStream(filename); s.pipe(req); + s.on('data', function(chunk) { + console.error('FS data chunk=%d', chunk.length); + }); + s.on('end', function() { + console.error('FS end'); + }); s.on('close', function(err) { if (err) throw err; clientReqComplete = true; @@ -84,7 +94,10 @@ function makeRequest() { }); req.on('response', function(res) { + console.error('RESPONSE', res.statusCode, res.headers); + res.resume(); res.on('end', function() { + console.error('RESPONSE end'); server.close(); }); }); diff --git a/test/simple/test-pipe.js b/test/simple/test-pipe.js index 3dd243725d3..9f1dae885b0 100644 --- a/test/simple/test-pipe.js +++ b/test/simple/test-pipe.js @@ -125,6 +125,7 @@ function startClient() { }); req.write(buffer); req.end(); + console.error('ended request', req); } process.on('exit', function() { diff --git a/test/simple/test-tcp-wrap-connect.js b/test/simple/test-tcp-wrap-connect.js index b8ac6aae342..3da8d932e17 100644 --- a/test/simple/test-tcp-wrap-connect.js +++ b/test/simple/test-tcp-wrap-connect.js @@ -54,6 +54,7 @@ var shutdownCount = 0; var server = require('net').Server(function(s) { console.log('got connection'); connectCount++; + s.resume(); s.on('end', function() { console.log('got eof'); endCount++; diff --git a/test/simple/test-zlib-random-byte-pipes.js b/test/simple/test-zlib-random-byte-pipes.js index f9723cc40d4..fc1db1cbba5 100644 --- a/test/simple/test-zlib-random-byte-pipes.js +++ b/test/simple/test-zlib-random-byte-pipes.js @@ -150,8 +150,25 @@ var inp = new RandomReadStream({ total: 1024, block: 256, jitter: 16 }); var out = new HashStream(); var gzip = zlib.createGzip(); var gunz = zlib.createGunzip(); + inp.pipe(gzip).pipe(gunz).pipe(out); +inp.on('data', function(c) { + console.error('inp data', c.length); +}); + +gzip.on('data', function(c) { + console.error('gzip data', c.length); +}); + +gunz.on('data', function(c) { + console.error('gunz data', c.length); +}); + +out.on('data', function(c) { + console.error('out data', c.length); +}); + var didSomething = false; out.on('data', function(c) { didSomething = true; From bb56dcc4505f1b8fe7fe5d9f975a96510787f2a7 Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 12 Dec 2012 22:06:35 -0800 Subject: [PATCH 64/72] tty/stdin: Refactor for streams2 --- lib/tty.js | 35 ++++++++++++++++++++--------------- src/node.js | 35 ++++++++++++++++++++++------------- 2 files changed, 42 insertions(+), 28 deletions(-) diff --git a/lib/tty.js b/lib/tty.js index 9fa18fd38c1..5b4036d0db8 100644 --- a/lib/tty.js +++ b/lib/tty.js @@ -40,42 +40,47 @@ exports.setRawMode = util.deprecate(function(flag) { }, 'tty.setRawMode: Use `process.stdin.setRawMode()` instead.'); -function ReadStream(fd) { - if (!(this instanceof ReadStream)) return new ReadStream(fd); - net.Socket.call(this, { +function ReadStream(fd, options) { + if (!(this instanceof ReadStream)) + return new ReadStream(fd, options); + + options = util._extend({ + highWaterMark: 0, + lowWaterMark: 0, handle: new TTY(fd, true) - }); + }, options); + + net.Socket.call(this, options); this.readable = true; this.writable = false; this.isRaw = false; + this.isTTY = true; + + // this.read = function(orig) { return function(n) { + // var ret = orig.apply(this, arguments); + // console.trace('TTY read(' + n + ') -> ' + ret); + // return ret; + // } }(this.read); } inherits(ReadStream, net.Socket); exports.ReadStream = ReadStream; -ReadStream.prototype.pause = function() { - return net.Socket.prototype.pause.call(this); -}; - -ReadStream.prototype.resume = function() { - return net.Socket.prototype.resume.call(this); -}; - ReadStream.prototype.setRawMode = function(flag) { flag = !!flag; this._handle.setRawMode(flag); this.isRaw = flag; }; -ReadStream.prototype.isTTY = true; - function WriteStream(fd) { if (!(this instanceof WriteStream)) return new WriteStream(fd); net.Socket.call(this, { - handle: new TTY(fd, false) + handle: new TTY(fd, false), + readable: false, + writable: true }); this.readable = false; diff --git a/src/node.js b/src/node.js index d861fe5b257..76c94b75923 100644 --- a/src/node.js +++ b/src/node.js @@ -140,7 +140,6 @@ } else { // Read all of stdin - execute it. - process.stdin.resume(); process.stdin.setEncoding('utf8'); var code = ''; @@ -497,17 +496,20 @@ switch (tty_wrap.guessHandleType(fd)) { case 'TTY': var tty = NativeModule.require('tty'); - stdin = new tty.ReadStream(fd); + stdin = new tty.ReadStream(fd, { + highWaterMark: 0, + lowWaterMark: 0 + }); break; case 'FILE': var fs = NativeModule.require('fs'); - stdin = new fs.ReadStream(null, {fd: fd}); + stdin = new fs.ReadStream(null, { fd: fd }); break; case 'PIPE': var net = NativeModule.require('net'); - stdin = new net.Stream(fd); + stdin = new net.Stream({ fd: fd }); stdin.readable = true; break; @@ -520,16 +522,23 @@ stdin.fd = fd; // stdin starts out life in a paused state, but node doesn't - // know yet. Call pause() explicitly to unref() it. - stdin.pause(); + // know yet. Explicitly to readStop() it to put it in the + // not-reading state. + if (stdin._handle && stdin._handle.readStop) { + stdin._handle.reading = false; + stdin._readableState.reading = false; + stdin._handle.readStop(); + } - // when piping stdin to a destination stream, - // let the data begin to flow. - var pipe = stdin.pipe; - stdin.pipe = function(dest, opts) { - stdin.resume(); - return pipe.call(stdin, dest, opts); - }; + // if the user calls stdin.pause(), then we need to stop reading + // immediately, so that the process can close down. + stdin.on('pause', function() { + if (!stdin._handle) + return; + stdin._readableState.reading = false; + stdin._handle.reading = false; + stdin._handle.readStop(); + }); return stdin; }); From b4df1e62de102de5243ed6a5da2f2686fbde1b3c Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 12 Dec 2012 22:07:05 -0800 Subject: [PATCH 65/72] test updates --- test/simple/test-child-process-ipc.js | 3 +++ .../test-http-header-response-splitting.js | 21 +++++++++++-------- test/simple/test-tls-pause.js | 15 +++++++++++-- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/test/simple/test-child-process-ipc.js b/test/simple/test-child-process-ipc.js index 8ccc505a82c..e8144e439dd 100644 --- a/test/simple/test-child-process-ipc.js +++ b/test/simple/test-child-process-ipc.js @@ -42,10 +42,13 @@ child.stdout.setEncoding('utf8'); child.stdout.on('data', function(data) { console.log('child said: ' + JSON.stringify(data)); if (!gotHelloWorld) { + console.error('testing for hello world'); assert.equal('hello world\r\n', data); gotHelloWorld = true; + console.error('writing echo me'); child.stdin.write('echo me\r\n'); } else { + console.error('testing for echo me'); assert.equal('echo me\r\n', data); gotEcho = true; child.stdin.end(); diff --git a/test/simple/test-http-header-response-splitting.js b/test/simple/test-http-header-response-splitting.js index 044618436cf..a54af12ccd5 100644 --- a/test/simple/test-http-header-response-splitting.js +++ b/test/simple/test-http-header-response-splitting.js @@ -27,6 +27,7 @@ var testIndex = 0, responses = 0; var server = http.createServer(function(req, res) { + console.error('request', testIndex); switch (testIndex++) { case 0: res.writeHead(200, { test: 'foo \r\ninvalid: bar' }); @@ -41,6 +42,7 @@ var server = http.createServer(function(req, res) { res.writeHead(200, { test: 'foo \n\n\ninvalid: bar' }); break; case 4: + console.error('send request, then close'); res.writeHead(200, { test: 'foo \r\n \r\n \r\ninvalid: bar' }); server.close(); break; @@ -49,15 +51,16 @@ var server = http.createServer(function(req, res) { } res.end('Hi mars!'); }); -server.listen(common.PORT); - -for (var i = 0; i < 5; i++) { - var req = http.get({ port: common.PORT, path: '/' }, function(res) { - assert.strictEqual(res.headers.test, 'foo invalid: bar'); - assert.strictEqual(res.headers.invalid, undefined); - responses++; - }); -} +server.listen(common.PORT, function() { + for (var i = 0; i < 5; i++) { + var req = http.get({ port: common.PORT, path: '/' }, function(res) { + assert.strictEqual(res.headers.test, 'foo invalid: bar'); + assert.strictEqual(res.headers.invalid, undefined); + responses++; + res.resume(); + }); + } +}); process.on('exit', function() { assert.strictEqual(responses, 5); diff --git a/test/simple/test-tls-pause.js b/test/simple/test-tls-pause.js index 0b29ae20d37..5eaac8da199 100644 --- a/test/simple/test-tls-pause.js +++ b/test/simple/test-tls-pause.js @@ -41,6 +41,9 @@ var received = 0; var server = tls.Server(options, function(socket) { socket.pipe(socket); + socket.on('data', function(c) { + console.error('data', c.length); + }); }); server.listen(common.PORT, function() { @@ -49,11 +52,16 @@ server.listen(common.PORT, function() { port: common.PORT, rejectUnauthorized: false }, function() { + console.error('connected'); client.pause(); common.debug('paused'); send(); function send() { - if (client.write(new Buffer(bufSize))) { + console.error('sending'); + var ret = client.write(new Buffer(bufSize)); + console.error('write => %j', ret); + if (false !== ret) { + console.error('write again'); sent += bufSize; assert.ok(sent < 100 * 1024 * 1024); // max 100MB return process.nextTick(send); @@ -62,12 +70,15 @@ server.listen(common.PORT, function() { common.debug('sent: ' + sent); resumed = true; client.resume(); - common.debug('resumed'); + console.error('resumed', client); } }); client.on('data', function(data) { + console.error('data'); assert.ok(resumed); received += data.length; + console.error('received', received); + console.error('sent', sent); if (received >= sent) { common.debug('received: ' + received); client.end(); From 81e356279dcd37fda20d827fbab3550026e74c63 Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 7 Nov 2012 17:19:14 -0800 Subject: [PATCH 66/72] child_process: Remove stream.pause/resume calls Unnecessary in streams2 --- lib/child_process.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/child_process.js b/lib/child_process.js index 1f9ddd543ae..8509d53024f 100644 --- a/lib/child_process.js +++ b/lib/child_process.js @@ -110,7 +110,6 @@ var handleConversion = { 'net.Socket': { send: function(message, socket) { // pause socket so no data is lost, will be resumed later - socket.pause(); // if the socket wsa created by net.Server if (socket.server) { @@ -142,7 +141,6 @@ var handleConversion = { got: function(message, handle, emit) { var socket = new net.Socket({handle: handle}); socket.readable = socket.writable = true; - socket.pause(); // if the socket was created by net.Server we will track the socket if (message.key) { @@ -153,7 +151,6 @@ var handleConversion = { } emit(socket); - socket.resume(); } } }; From 1d369317ea449424b98aec645f12b0d9e43876d6 Mon Sep 17 00:00:00 2001 From: isaacs Date: Wed, 12 Dec 2012 22:24:17 -0800 Subject: [PATCH 67/72] http: Refactor for streams2 Because of some of the peculiarities of http, this has a bit of special magic to handle cases where the IncomingMessage would wait forever in a paused state. In the server, if you do not begin consuming the request body by the time the response emits 'finish', then it will be flushed out. In the client, if you do not add a 'response' handler onto the request, then the response stream will be flushed out. --- lib/http.js | 142 +++++++++++++++++++++++++--------------------------- 1 file changed, 68 insertions(+), 74 deletions(-) diff --git a/lib/http.js b/lib/http.js index 62b2ffe0524..0555e699e26 100644 --- a/lib/http.js +++ b/lib/http.js @@ -114,19 +114,30 @@ function parserOnHeadersComplete(info) { return skipBody; } +// XXX This is a mess. +// TODO: http.Parser should be a Writable emits request/response events. function parserOnBody(b, start, len) { var parser = this; - var slice = b.slice(start, start + len); - if (parser.incoming._paused || parser.incoming._pendings.length) { - parser.incoming._pendings.push(slice); - } else { - parser.incoming._emitData(slice); + var stream = parser.incoming; + var rs = stream._readableState; + var socket = stream.socket; + + // pretend this was the result of a stream._read call. + if (len > 0) { + var slice = b.slice(start, start + len); + rs.onread(null, slice); } + + if (rs.length >= rs.highWaterMark) + socket.pause(); } function parserOnMessageComplete() { var parser = this; - parser.incoming.complete = true; + var stream = parser.incoming; + var socket = stream.socket; + + stream.complete = true; // Emit any trailing headers. var headers = parser._headers; @@ -140,19 +151,13 @@ function parserOnMessageComplete() { parser._url = ''; } - if (!parser.incoming.upgrade) { + if (!stream.upgrade) // For upgraded connections, also emit this after parser.execute - if (parser.incoming._paused || parser.incoming._pendings.length) { - parser.incoming._pendings.push(END_OF_FILE); - } else { - parser.incoming.readable = false; - parser.incoming._emitEnd(); - } - } + stream._readableState.onread(null, null); if (parser.socket.readable) { // force to read the next incoming message - parser.socket.resume(); + socket.resume(); } } @@ -263,9 +268,13 @@ function utcDate() { /* Abstract base class for ServerRequest and ClientResponse. */ function IncomingMessage(socket) { - Stream.call(this); + Stream.Readable.call(this); + + // XXX This implementation is kind of all over the place + // When the parser emits body chunks, they go in this list. + // _read() pulls them out, and when it finds EOF, it ends. + this._pendings = []; - // TODO Remove one of these eventually. this.socket = socket; this.connection = socket; @@ -276,78 +285,50 @@ function IncomingMessage(socket) { this.readable = true; - this._paused = false; this._pendings = []; - - this._endEmitted = false; + this._pendingIndex = 0; // request (server) only this.url = ''; - this.method = null; // response (client) only this.statusCode = null; this.client = this.socket; + + // flag for backwards compatibility grossness. + this._consuming = false; } -util.inherits(IncomingMessage, Stream); +util.inherits(IncomingMessage, Stream.Readable); exports.IncomingMessage = IncomingMessage; +IncomingMessage.prototype.read = function(n) { + this._consuming = true; + return Stream.Readable.prototype.read.call(this, n); +}; + + +IncomingMessage.prototype._read = function(n, callback) { + // We actually do almost nothing here, because the parserOnBody + // function fills up our internal buffer directly. However, we + // do need to unpause the underlying socket so that it flows. + if (!this.socket.readable) + return callback(null, null); + else + this.socket.resume(); +}; + + IncomingMessage.prototype.destroy = function(error) { this.socket.destroy(error); }; -IncomingMessage.prototype.setEncoding = function(encoding) { - var StringDecoder = require('string_decoder').StringDecoder; // lazy load - this._decoder = new StringDecoder(encoding); -}; -IncomingMessage.prototype.pause = function() { - this._paused = true; - this.socket.pause(); -}; - - -IncomingMessage.prototype.resume = function() { - this._paused = false; - if (this.socket) { - this.socket.resume(); - } - - this._emitPending(); -}; - - -IncomingMessage.prototype._emitPending = function(callback) { - if (this._pendings.length) { - var self = this; - process.nextTick(function() { - while (!self._paused && self._pendings.length) { - var chunk = self._pendings.shift(); - if (chunk !== END_OF_FILE) { - assert(Buffer.isBuffer(chunk)); - self._emitData(chunk); - } else { - assert(self._pendings.length === 0); - self.readable = false; - self._emitEnd(); - } - } - - if (callback) { - callback(); - } - }); - } else if (callback) { - callback(); - } -}; - IncomingMessage.prototype._emitData = function(d) { if (this._decoder) { @@ -1016,7 +997,7 @@ ServerResponse.prototype.writeHead = function(statusCode) { // don't keep alive connections where the client expects 100 Continue // but we sent a final status; they may put extra bytes on the wire. - if (this._expect_continue && ! this._sent100) { + if (this._expect_continue && !this._sent100) { this.shouldKeepAlive = false; } @@ -1321,11 +1302,10 @@ function socketCloseListener() { // Socket closed before we emitted 'end' below. req.res.emit('aborted'); var res = req.res; - req.res._emitPending(function() { - res._emitEnd(); + res.on('end', function() { res.emit('close'); - res = null; }); + res._readableState.onread(null, null); } else if (!req.res && !req._hadError) { // This socket error fired before we started to // receive a response. The error needs to @@ -1428,11 +1408,13 @@ function socketOnData(d, start, end) { } +// client function parserOnIncomingClient(res, shouldKeepAlive) { var parser = this; var socket = this.socket; var req = socket._httpMessage; + // propogate "domain" setting... if (req.domain && !res.domain) { debug('setting "res.domain"'); @@ -1480,15 +1462,21 @@ function parserOnIncomingClient(res, shouldKeepAlive) { DTRACE_HTTP_CLIENT_RESPONSE(socket, req); COUNTER_HTTP_CLIENT_RESPONSE(); - req.emit('response', res); req.res = res; res.req = req; - + var handled = req.emit('response', res); res.on('end', responseOnEnd); + // If the user did not listen for the 'response' event, then they + // can't possibly read the data, so we .resume() it into the void + // so that the socket doesn't hang there in a paused state. + if (!handled) + res.resume(); + return isHeadResponse; } +// client function responseOnEnd() { var res = this; var req = res.req; @@ -1784,7 +1772,7 @@ function connectionListener(socket) { incoming.push(req); var res = new ServerResponse(req); - debug('server response shouldKeepAlive: ' + shouldKeepAlive); + res.shouldKeepAlive = shouldKeepAlive; DTRACE_HTTP_SERVER_REQUEST(req, socket); COUNTER_HTTP_SERVER_REQUEST(); @@ -1806,6 +1794,12 @@ function connectionListener(socket) { incoming.shift(); + // if the user never called req.read(), and didn't pipe() or + // .resume() or .on('data'), then we call req.resume() so that the + // bytes will be pulled off the wire. + if (!req._consuming) + req.resume(); + res.detachSocket(socket); if (res._last) { From 0977638ffb016c9e1ace5a9f6299d4bd4c27688c Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 13 Dec 2012 09:52:08 -0800 Subject: [PATCH 68/72] test: Fix many tests for http streams2 refactor --- test/simple/test-cluster-http-pipe.js | 1 + test/simple/test-domain-http-server.js | 1 + test/simple/test-http-abort-client.js | 10 +++++++++- test/simple/test-http-client-agent.js | 1 + test/simple/test-http-client-pipe-end.js | 5 +++-- 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/test/simple/test-cluster-http-pipe.js b/test/simple/test-cluster-http-pipe.js index 46d429ad6d3..7123bf62756 100644 --- a/test/simple/test-cluster-http-pipe.js +++ b/test/simple/test-cluster-http-pipe.js @@ -53,6 +53,7 @@ http.createServer(function(req, res) { }).listen(common.PIPE, function() { var self = this; http.get({ socketPath: common.PIPE, path: '/' }, function(res) { + res.resume(); res.on('end', function(err) { if (err) throw err; process.send('DONE'); diff --git a/test/simple/test-domain-http-server.js b/test/simple/test-domain-http-server.js index f9962d3b8d4..b337f002dce 100644 --- a/test/simple/test-domain-http-server.js +++ b/test/simple/test-domain-http-server.js @@ -33,6 +33,7 @@ var disposeEmit = 0; var server = http.createServer(function(req, res) { var dom = domain.create(); + req.resume(); dom.add(req); dom.add(res); diff --git a/test/simple/test-http-abort-client.js b/test/simple/test-http-abort-client.js index 6acdd6f4047..f15238af160 100644 --- a/test/simple/test-http-abort-client.js +++ b/test/simple/test-http-abort-client.js @@ -46,13 +46,21 @@ server.listen(common.PORT, function() { res.on('data', function(chunk) { console.log('Read ' + chunk.length + ' bytes'); - console.log(chunk.toString()); + console.log(' chunk=%j', chunk.toString()); }); res.on('end', function() { console.log('Response ended.'); }); + res.on('aborted', function() { + console.log('Response aborted.'); + }); + + res.socket.on('close', function() { + console.log('socket closed, but not res'); + }) + // it would be nice if this worked: res.on('close', function() { console.log('Response aborted'); diff --git a/test/simple/test-http-client-agent.js b/test/simple/test-http-client-agent.js index f7112e3bfe4..aedf64ba706 100644 --- a/test/simple/test-http-client-agent.js +++ b/test/simple/test-http-client-agent.js @@ -61,6 +61,7 @@ function request(i) { server.close(); } }); + res.resume(); }); } diff --git a/test/simple/test-http-client-pipe-end.js b/test/simple/test-http-client-pipe-end.js index 7cb592e4c59..51edebbe1a7 100644 --- a/test/simple/test-http-client-pipe-end.js +++ b/test/simple/test-http-client-pipe-end.js @@ -26,6 +26,7 @@ var assert = require('assert'); var http = require('http'); var server = http.createServer(function(req, res) { + req.resume(); req.once('end', function() { res.writeHead(200); res.end(); @@ -50,9 +51,9 @@ server.listen(common.PIPE, function() { function sched(cb, ticks) { function fn() { if (--ticks) - process.nextTick(fn); + setImmediate(fn); else cb(); } - process.nextTick(fn); + setImmediate(fn); } From 19ecc3a4a6ed8b7256da4425307e18dba547492d Mon Sep 17 00:00:00 2001 From: isaacs Date: Thu, 13 Dec 2012 07:47:33 -0800 Subject: [PATCH 69/72] test updates for streams2 --- test/simple/test-domain-http-server.js | 11 +++-------- test/simple/test-http-agent.js | 10 +++++++--- test/simple/test-http-date-header.js | 1 + test/simple/test-http-default-encoding.js | 1 + test/simple/test-http-header-read.js | 1 + test/simple/test-http-host-headers.js | 6 ++++-- test/simple/test-http-keep-alive-close-on-header.js | 7 +++++-- test/simple/test-http-keep-alive.js | 3 +++ test/simple/test-http-many-keep-alive-connections.js | 1 + test/simple/test-http-parser-free.js | 1 + test/simple/test-http-request-end-twice.js | 1 + test/simple/test-http-request-end.js | 1 + .../simple/test-http-res-write-end-dont-take-array.js | 6 ++++-- test/simple/test-http-response-readable.js | 1 + test/simple/test-http-set-trailers.js | 1 + test/simple/test-http-status-code.js | 1 + test/simple/test-http-timeout.js | 2 ++ test/simple/test-https-agent.js | 1 + test/simple/test-https-socket-options.js | 2 ++ test/simple/test-https-strict.js | 1 + test/simple/test-regress-GH-877.js | 1 + 21 files changed, 43 insertions(+), 17 deletions(-) diff --git a/test/simple/test-domain-http-server.js b/test/simple/test-domain-http-server.js index b337f002dce..666f5d190ac 100644 --- a/test/simple/test-domain-http-server.js +++ b/test/simple/test-domain-http-server.js @@ -39,7 +39,7 @@ var server = http.createServer(function(req, res) { dom.on('error', function(er) { serverCaught++; - console.log('server error', er); + console.log('horray! got a server error', er); // try to send a 500. If that fails, oh well. res.writeHead(500, {'content-type':'text/plain'}); res.end(er.stack || er.message || 'Unknown error'); @@ -82,12 +82,7 @@ function next() { dom.on('error', function(er) { clientCaught++; console.log('client error', er); - // kill everything. - dom.dispose(); - }); - - dom.on('dispose', function() { - disposeEmit += 1; + req.socket.destroy(); }); var req = http.get({ host: 'localhost', port: common.PORT, path: p }); @@ -107,6 +102,7 @@ function next() { d += c; }); res.on('end', function() { + console.error('trying to parse json', d); d = JSON.parse(d); console.log('json!', d); }); @@ -117,6 +113,5 @@ function next() { process.on('exit', function() { assert.equal(serverCaught, 2); assert.equal(clientCaught, 2); - assert.equal(disposeEmit, 2); console.log('ok'); }); diff --git a/test/simple/test-http-agent.js b/test/simple/test-http-agent.js index a1ce456e5fe..fc66dc49f85 100644 --- a/test/simple/test-http-agent.js +++ b/test/simple/test-http-agent.js @@ -40,10 +40,14 @@ server.listen(common.PORT, function() { setTimeout(function() { for (var j = 0; j < M; j++) { http.get({ port: common.PORT, path: '/' }, function(res) { - console.log(res.statusCode); - if (++responses == N * M) server.close(); + console.log('%d %d', responses, res.statusCode); + if (++responses == N * M) { + console.error('Received all responses, closing server'); + server.close(); + } + res.resume(); }).on('error', function(e) { - console.log(e.message); + console.log('Error!', e); process.exit(1); }); } diff --git a/test/simple/test-http-date-header.js b/test/simple/test-http-date-header.js index 8ed52817124..b11507c0174 100644 --- a/test/simple/test-http-date-header.js +++ b/test/simple/test-http-date-header.js @@ -49,6 +49,7 @@ server.addListener('listening', function() { server.close(); process.exit(); }); + res.resume(); }); req.end(); }); diff --git a/test/simple/test-http-default-encoding.js b/test/simple/test-http-default-encoding.js index 4e4741a0611..b06f7c83a20 100644 --- a/test/simple/test-http-default-encoding.js +++ b/test/simple/test-http-default-encoding.js @@ -50,6 +50,7 @@ server.listen(common.PORT, function() { method: 'POST' }, function(res) { console.log(res.statusCode); + res.resume(); }).on('error', function(e) { console.log(e.message); process.exit(1); diff --git a/test/simple/test-http-header-read.js b/test/simple/test-http-header-read.js index e844deafcf2..33837759224 100644 --- a/test/simple/test-http-header-read.js +++ b/test/simple/test-http-header-read.js @@ -50,5 +50,6 @@ function runTest() { response.on('end', function() { s.close(); }); + response.resume(); }); } diff --git a/test/simple/test-http-host-headers.js b/test/simple/test-http-host-headers.js index 85f07a56852..ca7f70947f8 100644 --- a/test/simple/test-http-host-headers.js +++ b/test/simple/test-http-host-headers.js @@ -57,13 +57,14 @@ function testHttp() { var counter = 0; - function cb() { + function cb(res) { counter--; console.log('back from http request. counter = ' + counter); if (counter === 0) { httpServer.close(); testHttps(); } + res.resume(); } httpServer.listen(common.PORT, function(er) { @@ -124,13 +125,14 @@ function testHttps() { var counter = 0; - function cb() { + function cb(res) { counter--; console.log('back from https request. counter = ' + counter); if (counter === 0) { httpsServer.close(); console.log('ok'); } + res.resume(); } httpsServer.listen(common.PORT, function(er) { diff --git a/test/simple/test-http-keep-alive-close-on-header.js b/test/simple/test-http-keep-alive-close-on-header.js index 8fd7348dd5a..53c73ae461d 100644 --- a/test/simple/test-http-keep-alive-close-on-header.js +++ b/test/simple/test-http-keep-alive-close-on-header.js @@ -44,8 +44,9 @@ server.listen(common.PORT, function() { headers: headers, port: common.PORT, agent: agent - }, function() { + }, function(res) { assert.equal(1, agent.sockets['localhost:' + common.PORT].length); + res.resume(); }); request.on('socket', function(s) { s.on('connect', function() { @@ -60,8 +61,9 @@ server.listen(common.PORT, function() { headers: headers, port: common.PORT, agent: agent - }, function() { + }, function(res) { assert.equal(1, agent.sockets['localhost:' + common.PORT].length); + res.resume(); }); request.on('socket', function(s) { s.on('connect', function() { @@ -80,6 +82,7 @@ server.listen(common.PORT, function() { assert.equal(1, agent.sockets['localhost:' + common.PORT].length); server.close(); }); + response.resume(); }); request.on('socket', function(s) { s.on('connect', function() { diff --git a/test/simple/test-http-keep-alive.js b/test/simple/test-http-keep-alive.js index aa03639debc..4e8a6e816fe 100644 --- a/test/simple/test-http-keep-alive.js +++ b/test/simple/test-http-keep-alive.js @@ -42,6 +42,7 @@ server.listen(common.PORT, function() { }, function(response) { assert.equal(agent.sockets[name].length, 1); assert.equal(agent.requests[name].length, 2); + response.resume(); }); http.get({ @@ -49,6 +50,7 @@ server.listen(common.PORT, function() { }, function(response) { assert.equal(agent.sockets[name].length, 1); assert.equal(agent.requests[name].length, 1); + response.resume(); }); http.get({ @@ -59,6 +61,7 @@ server.listen(common.PORT, function() { assert(!agent.requests.hasOwnProperty(name)); server.close(); }); + response.resume(); }); }); diff --git a/test/simple/test-http-many-keep-alive-connections.js b/test/simple/test-http-many-keep-alive-connections.js index 4714cd523ca..adbebbdcc43 100644 --- a/test/simple/test-http-many-keep-alive-connections.js +++ b/test/simple/test-http-many-keep-alive-connections.js @@ -55,6 +55,7 @@ server.listen(common.PORT, function() { server.close(); } }); + res.resume(); }).on('error', function(e) { console.log(e.message); process.exit(1); diff --git a/test/simple/test-http-parser-free.js b/test/simple/test-http-parser-free.js index bbf4a502749..7b35781f4d3 100644 --- a/test/simple/test-http-parser-free.js +++ b/test/simple/test-http-parser-free.js @@ -44,6 +44,7 @@ server.listen(common.PORT, function() { if (++responses === N) { server.close(); } + res.resume(); }); })(i); } diff --git a/test/simple/test-http-request-end-twice.js b/test/simple/test-http-request-end-twice.js index f33cc6df3db..aa587722264 100644 --- a/test/simple/test-http-request-end-twice.js +++ b/test/simple/test-http-request-end-twice.js @@ -36,6 +36,7 @@ server.listen(common.PORT, function() { assert.ok(!req.end()); server.close(); }); + res.resume(); }); }); diff --git a/test/simple/test-http-request-end.js b/test/simple/test-http-request-end.js index f5da0faf2e5..f64dcc305ad 100644 --- a/test/simple/test-http-request-end.js +++ b/test/simple/test-http-request-end.js @@ -47,6 +47,7 @@ server.listen(common.PORT, function() { method: 'POST' }, function(res) { console.log(res.statusCode); + res.resume(); }).on('error', function(e) { console.log(e.message); process.exit(1); diff --git a/test/simple/test-http-res-write-end-dont-take-array.js b/test/simple/test-http-res-write-end-dont-take-array.js index f4b3f8ccee4..0d68afc3f80 100644 --- a/test/simple/test-http-res-write-end-dont-take-array.js +++ b/test/simple/test-http-res-write-end-dont-take-array.js @@ -53,11 +53,13 @@ var server = http.createServer(function(req, res) { server.listen(common.PORT, function() { // just make a request, other tests handle responses - http.get({port: common.PORT}, function() { + http.get({port: common.PORT}, function(res) { + res.resume(); // lazy serial test, becuase we can only call end once per request test += 1; // do it again to test .end(Buffer); - http.get({port: common.PORT}, function() { + http.get({port: common.PORT}, function(res) { + res.resume(); server.close(); }); }); diff --git a/test/simple/test-http-response-readable.js b/test/simple/test-http-response-readable.js index b31fcc3d12b..b48c06fb409 100644 --- a/test/simple/test-http-response-readable.js +++ b/test/simple/test-http-response-readable.js @@ -35,6 +35,7 @@ testServer.listen(common.PORT, function() { assert.equal(res.readable, false, 'res.readable set to false after end'); testServer.close(); }); + res.resume(); }); }); diff --git a/test/simple/test-http-set-trailers.js b/test/simple/test-http-set-trailers.js index a0896df5ac9..445a3eeaacc 100644 --- a/test/simple/test-http-set-trailers.js +++ b/test/simple/test-http-set-trailers.js @@ -106,6 +106,7 @@ server.on('listening', function() { process.exit(); } }); + res.resume(); }); outstanding_reqs++; }); diff --git a/test/simple/test-http-status-code.js b/test/simple/test-http-status-code.js index 395623869f8..ca56230d264 100644 --- a/test/simple/test-http-status-code.js +++ b/test/simple/test-http-status-code.js @@ -59,6 +59,7 @@ function nextTest() { testIdx += 1; nextTest(); }); + response.resume(); }); } diff --git a/test/simple/test-http-timeout.js b/test/simple/test-http-timeout.js index 5170d795aaf..c68a465f33d 100644 --- a/test/simple/test-http-timeout.js +++ b/test/simple/test-http-timeout.js @@ -55,6 +55,8 @@ server.listen(port, function() { server.close(); } }) + + res.resume(); }); req.setTimeout(1000, callback); diff --git a/test/simple/test-https-agent.js b/test/simple/test-https-agent.js index b54d5c38aec..34fa15c737a 100644 --- a/test/simple/test-https-agent.js +++ b/test/simple/test-https-agent.js @@ -54,6 +54,7 @@ server.listen(common.PORT, function() { port: common.PORT, rejectUnauthorized: false }, function(res) { + res.resume(); console.log(res.statusCode); if (++responses == N * M) server.close(); }).on('error', function(e) { diff --git a/test/simple/test-https-socket-options.js b/test/simple/test-https-socket-options.js index 4487cf8fa49..21b1118f7d4 100644 --- a/test/simple/test-https-socket-options.js +++ b/test/simple/test-https-socket-options.js @@ -55,6 +55,7 @@ server_http.listen(common.PORT, function() { rejectUnauthorized: false }, function(res) { server_http.close(); + res.resume(); }); // These methods should exist on the request and get passed down to the socket req.setNoDelay(true); @@ -77,6 +78,7 @@ server_https.listen(common.PORT+1, function() { rejectUnauthorized: false }, function(res) { server_https.close(); + res.resume(); }); // These methods should exist on the request and get passed down to the socket req.setNoDelay(true); diff --git a/test/simple/test-https-strict.js b/test/simple/test-https-strict.js index 43febc8e13f..720b0a6db74 100644 --- a/test/simple/test-https-strict.js +++ b/test/simple/test-https-strict.js @@ -170,6 +170,7 @@ function makeReq(path, port, error, host, ca) { server2.close(); server3.close(); } + res.resume(); }) } diff --git a/test/simple/test-regress-GH-877.js b/test/simple/test-regress-GH-877.js index d431118fdf8..1cdca9f7db9 100644 --- a/test/simple/test-regress-GH-877.js +++ b/test/simple/test-regress-GH-877.js @@ -48,6 +48,7 @@ server.listen(common.PORT, '127.0.0.1', function() { if (++responses == N) { server.close(); } + res.resume(); }); assert.equal(req.agent, agent); From 3751c0fe40ed22aaa281abda914ef9bf0c02edac Mon Sep 17 00:00:00 2001 From: isaacs Date: Fri, 14 Dec 2012 10:49:16 -0800 Subject: [PATCH 70/72] streams2: Still emit error if there was a write() cb --- lib/_stream_writable.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/_stream_writable.js b/lib/_stream_writable.js index 7364d3ab10c..619603fcad3 100644 --- a/lib/_stream_writable.js +++ b/lib/_stream_writable.js @@ -175,8 +175,10 @@ function onwrite(stream, er) { }); else cb(er); - } else - stream.emit('error', er); + } + + // backwards compatibility. still emit if there was a cb. + stream.emit('error', er); return; } state.length -= l; From abbd47e4a3774a0448dcc5ab36d2e9275c9ac805 Mon Sep 17 00:00:00 2001 From: isaacs Date: Fri, 14 Dec 2012 10:51:04 -0800 Subject: [PATCH 71/72] test: Update simple/test-fs-{write,read}-stream-err for streams2 Streams2 style streams might have already kicked off a read() or write() before emitting 'data' events. Make the test less dependent on ordering of when data events occur. --- test/simple/test-fs-read-stream-err.js | 40 +++++++++++------ test/simple/test-fs-write-stream-err.js | 60 ++++++++++++++++--------- 2 files changed, 67 insertions(+), 33 deletions(-) diff --git a/test/simple/test-fs-read-stream-err.js b/test/simple/test-fs-read-stream-err.js index 2c285f183cf..77960f06e00 100644 --- a/test/simple/test-fs-read-stream-err.js +++ b/test/simple/test-fs-read-stream-err.js @@ -23,28 +23,42 @@ var common = require('../common'); var assert = require('assert'); var fs = require('fs'); -var stream = fs.createReadStream(__filename, { bufferSize: 64 }); +var stream = fs.createReadStream(__filename, { + bufferSize: 64, + lowWaterMark: 0 +}); var err = new Error('BAM'); -stream.on('data', function(buf) { - var fd = stream.fd; +stream.on('error', common.mustCall(function errorHandler(err_) { + console.error('error event'); + process.nextTick(function() { + assert.equal(stream.fd, null); + assert.equal(err_, err); + }); +})); +fs.close = common.mustCall(function(fd_, cb) { + assert.equal(fd_, stream.fd); + process.nextTick(cb); +}); + +var read = fs.read; +fs.read = function() { + // first time is ok. + read.apply(fs, arguments); + // then it breaks fs.read = function() { var cb = arguments[arguments.length - 1]; process.nextTick(function() { cb(err); }); + // and should not be called again! + fs.read = function() { + throw new Error('BOOM!'); + }; }; +}; - fs.close = common.mustCall(function(fd_, cb) { - assert.equal(fd_, fd); - process.nextTick(cb); - }); - - stream.on('error', common.mustCall(function(err_) { - assert.equal(stream.fd, null); - assert.equal(err_, err); - })); - +stream.on('data', function(buf) { stream.on('data', assert.fail); // no more 'data' events should follow }); diff --git a/test/simple/test-fs-write-stream-err.js b/test/simple/test-fs-write-stream-err.js index 94d249f8967..a4d20200e5e 100644 --- a/test/simple/test-fs-write-stream-err.js +++ b/test/simple/test-fs-write-stream-err.js @@ -23,30 +23,50 @@ var common = require('../common'); var assert = require('assert'); var fs = require('fs'); -var stream = fs.createWriteStream(common.tmpDir + '/out'); +var stream = fs.createWriteStream(common.tmpDir + '/out', { + lowWaterMark: 0, + highWaterMark: 10 +}); var err = new Error('BAM'); +var write = fs.write; +var writeCalls = 0; +fs.write = function() { + switch (writeCalls++) { + case 0: + console.error('first write'); + // first time is ok. + return write.apply(fs, arguments); + case 1: + // then it breaks + console.error('second write'); + var cb = arguments[arguments.length - 1]; + return process.nextTick(function() { + cb(err); + }); + default: + // and should not be called again! + throw new Error('BOOM!'); + } +}; + +fs.close = common.mustCall(function(fd_, cb) { + console.error('fs.close', fd_, stream.fd); + assert.equal(fd_, stream.fd); + process.nextTick(cb); +}); + +stream.on('error', common.mustCall(function(err_) { + console.error('error handler'); + assert.equal(stream.fd, null); + assert.equal(err_, err); +})); + + stream.write(new Buffer(256), function() { - var fd = stream.fd; - - fs.write = function() { - var cb = arguments[arguments.length - 1]; - process.nextTick(function() { - cb(err); - }); - }; - - fs.close = function(fd_, cb) { - assert.equal(fd_, fd); - process.nextTick(cb); - }; - + console.error('first cb'); stream.write(new Buffer(256), common.mustCall(function(err_) { - assert.equal(err_, err); - })); - - stream.on('error', common.mustCall(function(err_) { - assert.equal(stream.fd, null); + console.error('second cb'); assert.equal(err_, err); })); }); From cd51fa8f5a97fcdf251aaccb954d324c13a3561b Mon Sep 17 00:00:00 2001 From: isaacs Date: Fri, 14 Dec 2012 17:43:02 -0800 Subject: [PATCH 72/72] test: Update message tests for streams2 --- test/message/max_tick_depth_trace.out | 16 ++++++++-------- test/message/stdin_messages.out | 16 ++++++++++------ 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/test/message/max_tick_depth_trace.out b/test/message/max_tick_depth_trace.out index 17cb7a72965..0fab6cd2c7f 100644 --- a/test/message/max_tick_depth_trace.out +++ b/test/message/max_tick_depth_trace.out @@ -8,10 +8,10 @@ tick 14 tick 13 tick 12 Trace: (node) warning: Recursive process.nextTick detected. This will break in the next version of node. Please use setImmediate for recursive deferral. - at maxTickWarn (node.js:289:17) - at process.nextTick (node.js:362:9) - at f (*test*message*max_tick_depth_trace.js:30:13) - at process._tickCallback (node.js:335:13) + at maxTickWarn (node.js:*:*) + at process.nextTick (node.js:*:* + at f (*test*message*max_tick_depth_trace.js:*:*) + at process._tickCallback (node.js:*:*) tick 11 tick 10 tick 9 @@ -23,9 +23,9 @@ tick 4 tick 3 tick 2 Trace: (node) warning: Recursive process.nextTick detected. This will break in the next version of node. Please use setImmediate for recursive deferral. - at maxTickWarn (node.js:289:17) - at process.nextTick (node.js:362:9) - at f (*test*message*max_tick_depth_trace.js:30:13) - at process._tickCallback (node.js:335:13) + at maxTickWarn (node.js:*:*) + at process.nextTick (node.js:*:* + at f (*test*message*max_tick_depth_trace.js:*:*) + at process._tickCallback (node.js:*:*) tick 1 tick 0 diff --git a/test/message/stdin_messages.out b/test/message/stdin_messages.out index e1a3790ca63..b0ad45bda48 100644 --- a/test/message/stdin_messages.out +++ b/test/message/stdin_messages.out @@ -9,7 +9,8 @@ SyntaxError: Strict mode code may not include a with statement at evalScript (node.js:*:*) at Socket. (node.js:*:*) at Socket.EventEmitter.emit (events.js:*:*) - at Pipe.onread (net.js:*:*) + at _stream_readable.js:*:* + at process._tickCallback (node.js:*:*) at process._makeCallback (node.js:*:*) 42 42 @@ -18,26 +19,28 @@ SyntaxError: Strict mode code may not include a with statement throw new Error("hello") ^ Error: hello - at [stdin]:1:7 + at [stdin]:1:* at Object. ([stdin]-wrapper:*:*) at Module._compile (module.js:*:*) at evalScript (node.js:*:*) at Socket. (node.js:*:*) at Socket.EventEmitter.emit (events.js:*:*) - at Pipe.onread (net.js:*:*) + at _stream_readable.js:*:* + at process._tickCallback (node.js:*:*) at process._makeCallback (node.js:*:*) [stdin]:1 throw new Error("hello") ^ Error: hello - at [stdin]:1:7 + at [stdin]:1:* at Object. ([stdin]-wrapper:*:*) at Module._compile (module.js:*:*) at evalScript (node.js:*:*) at Socket. (node.js:*:*) at Socket.EventEmitter.emit (events.js:*:*) - at Pipe.onread (net.js:*:*) + at _stream_readable.js:*:* + at process._tickCallback (node.js:*:*) at process._makeCallback (node.js:*:*) 100 @@ -51,7 +54,8 @@ ReferenceError: y is not defined at evalScript (node.js:*:*) at Socket. (node.js:*:*) at Socket.EventEmitter.emit (events.js:*:*) - at Pipe.onread (net.js:*:*) + at _stream_readable.js:*:* + at process._tickCallback (node.js:*:*) at process._makeCallback (node.js:*:*) [stdin]:1